diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000000000..491a83575febb --- /dev/null +++ b/.clang-format @@ -0,0 +1,22 @@ +--- +# Defaults for all languages. +BasedOnStyle: Google + +# Setting ColumnLimit to 0 so developer choices about where to break lines are maintained. +# Developers are responsible for adhering to the 120 character maximum. +ColumnLimit: 0 +SortIncludes: false +DerivePointerAlignment: false + +# if you want to customize when working locally see https://clang.llvm.org/docs/ClangFormatStyleOptions.html for options. +# See ReformatSource.ps1 for a script to update all source according to the current options in this file. +# e.g. customizations to use Allman bracing and more indenting. +# AccessModifierOffset: -2 +# BreakBeforeBraces: Allman +# CompactNamespaces: false +# IndentCaseLabels: true +# IndentWidth: 4 +# NamespaceIndentation: All + +... + diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 0000000000000..811af89e68a95 --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,30 @@ +--- +# turn off readability-braces-around-statements to allow single line statement like 'if (x == y) doSomething();' +Checks: '-*,cppcoreguidelines-*,google-*,readability-*,modernize-*,-readability-braces-around-statements,-google-runtime-references,-cppcoreguidelines-pro-type-reinterpret-cast' +WarningsAsErrors: '' +HeaderFilterRegex: '.*lotus\/core\/.*' +AnalyzeTemporaryDtors: false +FormatStyle: none +CheckOptions: + - key: google-readability-braces-around-statements.ShortStatementLines + value: '1' + - key: google-readability-function-size.StatementThreshold + value: '800' + - key: google-readability-namespace-comments.ShortNamespaceLines + value: '10' + - key: google-readability-namespace-comments.SpacesBeforeComments + value: '2' + - key: modernize-loop-convert.MaxCopySize + value: '16' + - key: modernize-loop-convert.MinConfidence + value: reasonable + - key: modernize-loop-convert.NamingStyle + value: CamelCase + - key: modernize-pass-by-value.IncludeStyle + value: google + - key: modernize-replace-auto-ptr.IncludeStyle + value: google + - key: modernize-use-nullptr.NullMacros + value: 'NULL' +... + diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000..41eae6dac52f5 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,13 @@ +# This sets the default behaviour, overriding core.autocrlf +* text=auto + +# All source files should have unix line-endings in the repository, +# but convert to native line-endings on checkout +*.cc text +*.h text + +# Windows specific files should retain windows line-endings +*.sln text eol=crlf + +# make sure build.sh retains Unix line endings, even when checked out on windows. +*.sh text eol=lf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000..7d69fe5c943db --- /dev/null +++ b/.gitignore @@ -0,0 +1,31 @@ +# build, distribute, and bins (+ python proto bindings) +build +build_host_protoc +build_android +build_ios +build_* +.build_debug/* +.build_release/* +distribute/* +*.testbin +*.bin +cmake_build +.cmake_build +gen +*~ +.vs +TestResults/ +.idea/ +lotus.egg-info +nuget_root/ +.packages/ +.vscode/ +*.code-workspace +__pycache__ +onnxruntime_profile*.json +/docs/python/*.md +/docs/python/auto_examples/* +/docs/python/media/* +/docs/python/examples/*.onnx +/docs/python/examples/graph.* +/docs/python/*_LICENSE diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000000..004d8dfacb09b --- /dev/null +++ b/.gitmodules @@ -0,0 +1,18 @@ +[submodule "external/protobuf"] + path = cmake/external/protobuf + url = https://github.com/google/protobuf.git +[submodule "cmake/external/googletest"] + path = cmake/external/googletest + url = https://github.com/google/googletest.git +[submodule "cmake/external/onnx"] + path = cmake/external/onnx + url = https://github.com/onnx/onnx +[submodule "cmake/external/tvm"] + path = cmake/external/tvm + url = https://github.com/dmlc/tvm.git +[submodule "cmake/external/date"] + path = cmake/external/date + url = https://github.com/HowardHinnant/date.git +[submodule "cmake/external/gsl"] + path = cmake/external/gsl + url = https://github.com/Microsoft/GSL.git diff --git a/BUILD.md b/BUILD.md new file mode 100644 index 0000000000000..5bbdd1d52675f --- /dev/null +++ b/BUILD.md @@ -0,0 +1,136 @@ +# Build ONNX Runtime + +## Supported dev environments + +| OS | Supports CPU | Supports GPU| Notes | +|-------------|:------------:|:------------:|------------------------------------| +|Windows 10 | YES | YES |Must use VS 2017 or the latest VS2015| +|Windows 10
Subsystem for Linux | YES | NO | | +|Ubuntu 16.x | YES | YES | | +|Ubuntu 17.x | YES | YES | | +|Ubuntu 18.x | YES | YES | | +|Fedora 24 | YES | YES | | +|Fedora 25 | YES | YES | | +|Fedora 26 | YES | YES | | +|Fedora 27 | YES | YES | | +|Fedora 28 | YES | NO |Cannot build GPU kernels but can run them | + +* Red Hat Enterprise Linux and CentOS are not supported. +* GCC 4.x and below are not supported. If you are using GCC 7.0+, you'll need to upgrade eigen to a newer version before compiling ONNX Runtime. + +OS/Compiler Matrix: + +| OS/Compiler | Supports VC | Supports GCC | Supports Clang | +|-------------|:------------:|:----------------:|:---------------:| +|Windows 10 | YES | Not tested | Not tested | +|Linux | NO | YES(gcc>=5.0) | YES | + +ONNX Runtime python binding only supports Python 3.x. Please use python 3.5+. + +## Build +Install cmake-3.11 or better from https://cmake.org/download/. + +Checkout the source tree: +``` +git clone --recursive https://github.com/Microsoft/onnxruntime +cd onnxruntime +./build.sh for Linux (or ./build.bat for Windows) +``` +The build script runs all unit tests by default. + +The complete list of build options can be found by running `./build.sh (or ./build.bat) --help` + +## Build/Test Flavors for CI + +### CI Build Environments + +| Build Job Name | Environment | Dependency | Test Coverage | Scripts | +|--------------------|---------------------|---------------------------------|--------------------------|------------------------------------------| +| Linux_CI_Dev | Ubuntu 16.04 | python=3.5 | Unit tests; ONNXModelZoo | [script](tools/ci_build/github/linux/run_build.sh) | +| Linux_CI_GPU_Dev | Ubuntu 16.04 | python=3.5; nvidia-docker | Unit tests; ONNXModelZoo | [script](tools/ci_build/github/linux/run_build.sh) | +| Windows_CI_Dev | Windows Server 2016 | python=3.5 | Unit tests; ONNXModelZoo | [script](build.bat) | +| Windows_CI_GPU_Dev | Windows Server 2016 | cuda=9.0; cudnn=7.0; python=3.5 | Unit tests; ONNXModelZoo | [script](build.bat) | + +## Additional Build Flavors +The complete list of build flavors can be seen by running `./build.sh --help` or `./build.bat --help`. Here are some common flavors. + +### Windows CUDA Build +ONNX Runtime supports CUDA builds. You will need to download and install [CUDA](https://developer.nvidia.com/cuda-toolkit) and [CUDNN](https://developer.nvidia.com/cudnn). + +ONNX Runtime is built and tested with CUDA 9.0 and CUDNN 7.0 using the Visual Studio 2017 14.11 toolset (i.e. Visual Studio 2017 v15.3). +CUDA versions up to 9.2 and CUDNN version 7.1 should also work with versions of Visual Studio 2017 up to and including v15.7, however you may need to explicitly install and use the 14.11 toolset due to CUDA and CUDNN only being compatible with earlier versions of Visual Studio 2017. + +To install the Visual Studio 2017 14.11 toolset, see + +If using this toolset with a later version of Visual Studio 2017 you have two options: + +1. Setup the Visual Studio environment variables to point to the 14.11 toolset by running vcvarsall.bat prior to running cmake + - e.g. if you have VS2017 Enterprise, an x64 build would use the following command +`"C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" amd64 -vcvars_ver=14.11` + +2. Alternatively if you have CMake 3.12 or later you can specify the toolset version in the "-T" parameter by adding "version=14.11" + - e.g. use the following with the below cmake command +`-T version=14.11,host=x64` + +CMake should automatically find the CUDA installation. If it does not, or finds a different version to the one you wish to use, specify your root CUDA installation directory via the -DCUDA_TOOLKIT_ROOT_DIR CMake parameter. + +_Side note: If you have multiple versions of CUDA installed on a Windows machine and are building with Visual Studio, CMake will use the build files for the highest version of CUDA it finds in the BuildCustomization folder. e.g. C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\Common7\IDE\VC\VCTargets\BuildCustomizations\. If you want to build with an earlier version, you must temporarily remove the 'CUDA x.y.*' files for later versions from this directory._ + +The path to the 'cuda' folder in the CUDNN installation must be provided. The 'cuda' folder should contain 'bin', 'include' and 'lib' directories. + +You can build with: + +``` +./build.sh --use_cuda --cudnn_home /usr --cuda_home /usr/local/cuda (Linux) +./build.bat --use_cuda --cudnn_home --cuda_home (Windows) +``` + +### MKL-DNN +To build ONNX Runtime with MKL-DNN support, build it with `./build.sh --use_mkldnn --use_mklml` + +### OpenBLAS +#### Windows +Instructions how to build OpenBLAS for windows can be found here https://github.com/xianyi/OpenBLAS/wiki/How-to-use-OpenBLAS-in-Microsoft-Visual-Studio#build-openblas-for-universal-windows-platform. + +Once you have the OpenBLAS binaries, build ONNX Runtime with `./build.bat --use_openblas` + +#### Linux +For Linux (e.g. Ubuntu 16.04), install libopenblas-dev package +`sudo apt-get install libopenblas-dev` and build with `./build.sh --use_openblas` + +### OpenMP +``` +./build.sh --use_openmp (for Linux) +./build.bat --use_openmp (for Windows) +``` + +### Build with Docker on Linux +Install Docker: `https://docs.docker.com/install/` + +#### CPU +``` +cd tools/ci_build/github/linux/docker +docker build -t onnxruntime_dev --build-arg OS_VERSION=16.04 -f Dockerfile.ubuntu . +docker run --rm -it onnxruntime_dev /bin/bash +``` + +#### GPU +If you need GPU support, please also install: +1. nvidia driver. Before doing this please add 'nomodeset rd.driver.blacklist=nouveau' to your linux [kernel boot parameters](https://www.kernel.org/doc/html/v4.17/admin-guide/kernel-parameters.html). +2. nvidia-docker2: [Install doc](`https://github.com/NVIDIA/nvidia-docker/wiki/Installation-(version-2.0)`) + +To test if your nvidia-docker works: +``` +docker run --runtime=nvidia --rm nvidia/cuda nvidia-smi +``` + +Then build a docker image. We provided a sample for use: +``` +cd tools/ci_build/github/linux/docker +docker build -t cuda_dev -f Dockerfile.ubuntu_gpu . +``` + +Then run it +``` +./tools/ci_build/github/linux/run_dockerbuild.sh +``` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000000..601f0cea3e649 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,50 @@ +# Contributing + +We're always looking for your help to fix bugs and improve the product. Create a pull request and we'll be happy to take a look. +Start by reading the [Engineering Design](docs/HighLevelDesign.md). + +# Checkin procedure +``` +git clone --recursive https://github.com/Microsoft/onnxruntime +git checkout -b feature_branch +# make your changes +# write unit tests +# make sure it builds and all tests pass +git commit -m "my changes" +git push origin HEAD +``` +To request merge into master send a pull request from the web ui +https://github.com/Microsoft/onnxruntime +New code *must* be accompanied by unit tests. + +# Build +[Build](BUILD.md) + +# Additional Documentation + * [Adding a custom operator](docs/AddingCustomOp.md) + +# Coding guidelines +Please see [Coding Conventions and Standards](./docs/Coding_Conventions_and_Standards.md) + +# Licensing guidelines +This project welcomes contributions and suggestions. Most contributions require you to +agree to a Contributor License Agreement (CLA) declaring that you have the right to, +and actually do, grant us the rights to use your contribution. For details, visit +https://cla.microsoft.com. + +When you submit a pull request, a CLA-bot will automatically determine whether you need +to provide a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the +instructions provided by the bot. You will only need to do this once across all repositories using our CLA. + +# Code of conduct +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). +For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) +or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. + +# Reporting Security Issues +Security issues and bugs should be reported privately, via email, to the Microsoft Security +Response Center (MSRC) at [secure@microsoft.com](mailto:secure@microsoft.com). You should +receive a response within 24 hours. If for some reason you do not, please follow up via +email to ensure we received your original message. Further information, including the +[MSRC PGP](https://technet.microsoft.com/en-us/security/dn606155) key, can be found in +the [Security TechCenter](https://technet.microsoft.com/en-us/security/default). diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000..506ab97e56e2b --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Microsoft Corporation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000000000..35ffd0f547a8f --- /dev/null +++ b/README.md @@ -0,0 +1,35 @@ +# ONNX Runtime + +[![Build Status](https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/status/Microsoft.onnxruntime)](https://dev.azure.com/onnxruntime/onnxruntime/_build/latest?definitionId=1) + +ONNX Runtime is the runtime for [ONNX](https://github.com/onnx/onnx). + +# Engineering Design +[Engineering Design](docs/HighLevelDesign.md) + +# API +| API | CPU package | GPU package | +|-----|-------------|-------------| +| [Python](https://docs.microsoft.com/en-us/python/api/overview/azure/onnx/intro?view=azure-onnx-py) | [Windows](TODO)
[Linux](https://pypi.org/project/onnxruntime/)
[Mac](TODO)| [Windows](TODO)
[Linux](https://pypi.org/project/onnxruntime-gpu/) | +| [C#](docs/CSharp_API.md) | [Windows](TODO)| Not available | +| [C](docs/C_API.md) | [Windows](TODO)
[Linux](TODO) | Not available | + +# Build +[Build](BUILD.md) + +# Contribute +[Contribute](CONTRIBUTING.md) + +# Versioning +[Versioning](docs/Versioning.md) + +# Feedback + * File a bug in [GitHub Issues](https://github.com/Microsoft/onnxruntime/issues) + +# Code of Conduct +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). +For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) +or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. + +# License +[LICENSE](LICENSE) diff --git a/ThirdPartyNotices.txt b/ThirdPartyNotices.txt new file mode 100644 index 0000000000000..aee0067b789b9 --- /dev/null +++ b/ThirdPartyNotices.txt @@ -0,0 +1,2293 @@ +THIRD PARTY SOFTWARE NOTICES AND INFORMATION + +Do Not Translate or Localize + +This software incorporates material from third parties. Microsoft makes certain +open source code available at http://3rdpartysource.microsoft.com, or you may +send a check or money order for US $5.00, including the product name, the open +source component name, and version number, to: + +Source Code Compliance Team +Microsoft Corporation +One Microsoft Way +Redmond, WA 98052 +USA + +Notwithstanding any other terms, you may reverse engineer this software to the +extent required to debug changes to any libraries licensed under the GNU Lesser +General Public License. + +_____ + +Intel Math Kernel Library (Intel MKL) + +Intel Simplified Software License (Version April 2018) + +Copyright (c) 2018 Intel Corporation. + +Use and Redistribution. You may use and redistribute the software (the “Software”), without modification, +provided the following conditions are met: + +* Redistributions must reproduce the above copyright notice and the following terms of use in the Software +and in the documentation and/or other materials provided with the distribution. + +* Neither the name of Intel nor the names of its suppliers may be used to endorse or promote products +derived from this Software without specific prior written permission. + +* No reverse engineering, decompilation, or disassembly of this Software is permitted. + +Limited patent license. Intel grants you a world-wide, royalty-free, non-exclusive license under patents it now +or hereafter owns or controls to make, have made, use, import, offer to sell and sell (“Utilize”) this Software, +but solely to the extent that any such patent is necessary to Utilize the Software alone. The patent license +shall not apply to any combinations which include this software. No hardware per se is licensed hereunder. + +Third party and other Intel programs. “Third Party Programs” are the files listed in the “third-party-programs.txt” +text file that is included with the Software and may include Intel programs under separate license terms. +Third Party Programs, even if included with the distribution of the Materials, are governed by +separate license terms and those license terms solely govern your use of those programs. + +DISCLAIMER. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT ARE DISCLAIMED. THIS SOFTWARE IS +NOT INTENDED FOR USE IN SYSTEMS OR APPLICATIONS WHERE FAILURE OF THE SOFTWARE +MAY CAUSE PERSONAL INJURY OR DEATH AND YOU AGREE THAT YOU ARE FULLY RESPONSIBLE FOR ANY +CLAIMS, COSTS, DAMAGES, EXPENSES, AND ATTORNEYS’ FEES ARISING OUT OF ANY SUCH USE, +EVEN IF ANY CLAIM ALLEGES THAT INTEL WAS NEGLIGENT REGARDING THE DESIGN OR MANUFACTURE OF +THE MATERIALS. + +LIMITATION OF LIABILITY. IN NO EVENT WILL INTEL BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY +OF SUCH DAMAGE. YOU AGREE TO INDEMNIFY AND HOLD INTEL HARMLESS AGAINST ANY CLAIMS +AND EXPENSES RESULTING FROM YOUR USE OR UNAUTHORIZED USE OF THE SOFTWARE. + +No support. Intel may make changes to the Software, at any time without notice, and is not obligated to +support, update or provide training for the Software. + +Termination. Intel may terminate your right to use the Software in the event of your breach of this Agreement +and you fail to cure the breach within a reasonable period of time. + +Feedback. Should you provide Intel with comments, modifications, corrections, enhancements or other input +(“Feedback”) related to the Software Intel will be free to use, disclose, reproduce, license or otherwise +distribute or exploit the Feedback in its sole discretion without any obligations or restrictions of any kind, +including without limitation, intellectual property rights or licensing obligations. + +Compliance with laws. You agree to comply with all relevant laws and regulations governing your use, +transfer, import or export (or prohibition thereof) of the Software. + +Governing law. All disputes will be governed by the laws of the United States of America and the State of +Delaware without reference to conflict of law principles and subject to the exclusive jurisdiction of the state or +federal courts sitting in the State of Delaware, and each party agrees that it submits to the personal +jurisdiction and venue of those courts and waives any objections. The United Nations Convention on +Contracts for the International Sale of Goods (1980) is specifically excluded and will not apply to the +Software. + +*Other names and brands may be claimed as the property of others. + +_____ + +protocolbuffers/protobuf + +Copyright 2008 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. + +_____ + +madler/zlib + +The deflate format used by zlib was defined by Phil Katz. The deflate and +zlib specifications were written by L. Peter Deutsch. Thanks to all the +people who reported problems and suggested various improvements in zlib; they +are too numerous to cite here. + +Copyright notice: + + (C) 1995-2017 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + +If you use the zlib library in a product, we would appreciate *not* receiving +lengthy legal documents to sign. The sources are provided for free but without +warranty of any kind. The library has been entirely written by Jean-loup +Gailly and Mark Adler; it does not include third-party code. + +If you redistribute modified sources, we would appreciate that you include in +the file ChangeLog history information documenting your changes. Please read +the FAQ for more information on the distribution of modified source versions. + +_____ + +pybind/pybind11 + +Copyright (c) 2016 Wenzel Jakob , All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Please also refer to the file CONTRIBUTING.md, which clarifies licensing of +external contributions to this project including patches, pull requests, etc. + +_____ + +NVlabs/cub + +Copyright (c) 2010-2011, Duane Merrill. All rights reserved. +Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the NVIDIA CORPORATION nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +_____ + +onnx +Open Neural Network Exchange + +Copyright (c) Facebook, Inc. and Microsoft Corporation. All rights reserved. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the ""Software""), to deal in the +Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR +ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH +THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +_____ + +eigen-git-mirror + +Mozilla Public License Version 2.0 + +================================== + +1. Definitions + +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions + +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities + +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination + +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. + +_____ + +intel/mkl-dnn + +Copyright 2016-2018 Intel Corporation + +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +sub-components: + +xbyak + +Copyright (c) 2007 MITSUNARI Shigeo. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. +Neither the name of the copyright owner nor the names of its contributors may +be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. + +_____ + +Microsoft/GSL + +Copyright (c) 2015 Microsoft Corporation. All rights reserved. + +This code is licensed under the MIT License (MIT). + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +_____ + +Tensorflow + +Copyright 2018 The TensorFlow Authors. All rights reserved. + +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, +and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by +the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all +other entities that control, are controlled by, or are under common +control with that entity. For the purposes of this definition, +"control" means (i) the power, direct or indirect, to cause the +direction or management of such entity, whether by contract or +otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity +exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, +including but not limited to software source code, documentation +source, and configuration files. + +"Object" form shall mean any form resulting from mechanical +transformation or translation of a Source form, including but +not limited to compiled object code, generated documentation, +and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or +Object form, made available under the License, as indicated by a +copyright notice that is included in or attached to the work +(an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object +form, that is based on (or derived from) the Work and for which the +editorial revisions, annotations, elaborations, or other modifications +represent, as a whole, an original work of authorship. For the purposes +of this License, Derivative Works shall not include works that remain +separable from, or merely link (or bind by name) to the interfaces of, +the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including +the original version of the Work and any modifications or additions +to that Work or Derivative Works thereof, that is intentionally +submitted to Licensor for inclusion in the Work by the copyright owner +or by an individual or Legal Entity authorized to submit on behalf of +the copyright owner. For the purposes of this definition, "submitted" +means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, +and issue tracking systems that are managed by, or on behalf of, the +Licensor for the purpose of discussing and improving the Work, but +excluding communication that is conspicuously marked or otherwise +designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity +on behalf of whom a Contribution has been received by Licensor and +subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of +this License, each Contributor hereby grants to You a perpetual, +worldwide, non-exclusive, no-charge, royalty-free, irrevocable +copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the +Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of +this License, each Contributor hereby grants to You a perpetual, +worldwide, non-exclusive, no-charge, royalty-free, irrevocable +(except as stated in this section) patent license to make, have made, +use, offer to sell, sell, import, and otherwise transfer the Work, +where such license applies only to those patent claims licensable +by such Contributor that are necessarily infringed by their +Contribution(s) alone or by combination of their Contribution(s) +with the Work to which such Contribution(s) was submitted. If You +institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work +or a Contribution incorporated within the Work constitutes direct +or contributory patent infringement, then any patent licenses +granted to You under this License for that Work shall terminate +as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the +Work or Derivative Works thereof in any medium, with or without +modifications, and in Source or Object form, provided that You +meet the following conditions: + +(a) You must give any other recipients of the Work or +Derivative Works a copy of this License; and + +(b) You must cause any modified files to carry prominent notices +stating that You changed the files; and + +(c) You must retain, in the Source form of any Derivative Works +that You distribute, all copyright, patent, trademark, and +attribution notices from the Source form of the Work, +excluding those notices that do not pertain to any part of +the Derivative Works; and + +(d) If the Work includes a "NOTICE" text file as part of its +distribution, then any Derivative Works that You distribute must +include a readable copy of the attribution notices contained +within such NOTICE file, excluding those notices that do not +pertain to any part of the Derivative Works, in at least one +of the following places: within a NOTICE text file distributed +as part of the Derivative Works; within the Source form or +documentation, if provided along with the Derivative Works; or, +within a display generated by the Derivative Works, if and +wherever such third-party notices normally appear. The contents +of the NOTICE file are for informational purposes only and +do not modify the License. You may add Your own attribution +notices within Derivative Works that You distribute, alongside +or as an addendum to the NOTICE text from the Work, provided +that such additional attribution notices cannot be construed +as modifying the License. + +You may add Your own copyright statement to Your modifications and +may provide additional or different license terms and conditions +for use, reproduction, or distribution of Your modifications, or +for any such Derivative Works as a whole, provided Your use, +reproduction, and distribution of the Work otherwise complies with +the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, +any Contribution intentionally submitted for inclusion in the Work +by You to the Licensor shall be under the terms and conditions of +this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify +the terms of any separate license agreement you may have executed +with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade +names, trademarks, service marks, or product names of the Licensor, +except as required for reasonable and customary use in describing the +origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or +agreed to in writing, Licensor provides the Work (and each +Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied, including, without limitation, any warranties or conditions +of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A +PARTICULAR PURPOSE. You are solely responsible for determining the +appropriateness of using or redistributing the Work and assume any +risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, +whether in tort (including negligence), contract, or otherwise, +unless required by applicable law (such as deliberate and grossly +negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, +incidental, or consequential damages of any character arising as a +result of this License or out of the use or inability to use the +Work (including but not limited to damages for loss of goodwill, +work stoppage, computer failure or malfunction, or any and all +other commercial damages or losses), even if such Contributor +has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing +the Work or Derivative Works thereof, You may choose to offer, +and charge a fee for, acceptance of support, warranty, indemnity, +or other liability obligations and/or rights consistent with this +License. However, in accepting such obligations, You may act only +on Your own behalf and on Your sole responsibility, not on behalf +of any other Contributor, and only if You agree to indemnify, +defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason +of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2017, The TensorFlow Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +_____ + +Microsoft Cognitive Toolkit (CNTK) + +Copyright (c) Microsoft Corporation. All rights reserved. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT +LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +_____ + +Pytorch / Caffe2 + +From PyTorch: + +Copyright (c) 2016- Facebook, Inc (Adam Paszke) +Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +Copyright (c) 2011-2013 NYU (Clement Farabet) +Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) +Copyright (c) 2006 Idiap Research Institute (Samy Bengio) +Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) + +From Caffe2: + +Copyright (c) 2016-present, Facebook Inc. All rights reserved. + +All contributions by Facebook: +Copyright (c) 2016 Facebook Inc. + +All contributions by Google: +Copyright (c) 2015 Google Inc. +All rights reserved. + +All contributions by Yangqing Jia: +Copyright (c) 2015 Yangqing Jia +All rights reserved. + +All contributions from Caffe: +Copyright(c) 2013, 2014, 2015, the respective contributors +All rights reserved. + +All other contributions: +Copyright(c) 2015, 2016 the respective contributors +All rights reserved. + +Caffe2 uses a copyright model similar to Caffe: each contributor holds +copyright over their contributions to Caffe2. The project versioning records +all such contribution and copyright details. If a contributor wants to further +mark their specific copyright on a particular contribution, they should +indicate their copyright solely in the commit message of the change when it is +committed. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America + and IDIAP Research Institute nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +_____ + +The LLVM Compiler Infrastructure + +============================================================================== +LLVM Release License +============================================================================== +University of Illinois/NCSA +Open Source License + +Copyright (c) 2003-2017 University of Illinois at Urbana-Champaign. +All rights reserved. + +Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at + Urbana-Champaign, nor the names of its contributors may be used to + endorse or promote products derived from this Software without specific + prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + +============================================================================== +Copyrights and Licenses for Third Party Software Distributed with LLVM: +============================================================================== +The LLVM software contains code written by third parties. Such software will +have its own individual LICENSE.TXT file in the directory in which it appears. +This file will describe the copyrights, license, and restrictions which apply +to that code. + +The disclaimer of warranty in the University of Illinois Open Source License +applies to all code in the LLVM Distribution, and nothing in any of the +other licenses gives permission to use the names of the LLVM Team or the +University of Illinois to endorse or promote products derived from this +Software. + +The following pieces of software have additional or alternate copyrights, +licenses, and/or restrictions: + +Program Directory +------- --------- +Google Test llvm/utils/unittest/googletest +OpenBSD regex llvm/lib/Support/{reg*, COPYRIGHT.regex} +pyyaml tests llvm/test/YAMLParser/{*.data, LICENSE.TXT} +ARM contributions llvm/lib/Target/ARM/LICENSE.TXT +md5 contributions llvm/lib/Support/MD5.cpp llvm/include/llvm/Support/MD5.h + +_____ + +google/benchmark + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +CONTRIBUTORS + +# People who have agreed to one of the CLAs and can contribute patches. +# The AUTHORS file lists the copyright holders; this file +# lists people. For example, Google employees are listed here +# but not in AUTHORS, because Google holds the copyright. +# +# Names should be added to this file only after verifying that +# the individual or the individual's organization has agreed to +# the appropriate Contributor License Agreement, found here: +# +# https://developers.google.com/open-source/cla/individual +# https://developers.google.com/open-source/cla/corporate +# +# The agreement for individuals can be filled out on the web. +# +# When adding J Random Contributor's name to this file, +# either J's name or J's organization's name should be +# added to the AUTHORS file, depending on whether the +# individual or corporate CLA was used. +# +# Names should be added to this file as: +# Name +# +# Please keep the list sorted. + +Albert Pretorius +Arne Beer +Billy Robert O'Neal III +Chris Kennelly +Christopher Seymour +David Coeurjolly +Deniz Evrenci +Dominic Hamon +Dominik Czarnota +Eric Fiselier +Eugene Zhuk +Evgeny Safronov +Federico Ficarelli +Felix Homann +Ismael Jimenez Martinez +Jern-Kuan Leong +JianXiong Zhou +Joao Paulo Magalhaes +John Millikin +Jussi Knuuttila +Kai Wolf +Kishan Kumar +Kaito Udagawa +Lei Xu +Matt Clarkson +Maxim Vafin +Nick Hutchinson +Oleksandr Sochka +Pascal Leroy +Paul Redmond +Pierre Phaneuf +Radoslav Yovchev +Raul Marin +Ray Glover +Robert Guo +Roman Lebedev +Shuo Chen +Tobias Ulvgĺrd +Tom Madams +Yixuan Qiu +Yusuke Suzuki +Zbigniew Skowron + +AUTHORS + +# This is the official list of benchmark authors for copyright purposes. +# This file is distinct from the CONTRIBUTORS files. +# See the latter for an explanation. +# +# Names should be added to this file as: +# Name or Organization +# The email address is not required for organizations. +# +# Please keep the list sorted. + +Albert Pretorius +Arne Beer +Carto +Christopher Seymour +David Coeurjolly +Deniz Evrenci +Dirac Research +Dominik Czarnota +Eric Fiselier +Eugene Zhuk +Evgeny Safronov +Federico Ficarelli +Felix Homann +Google Inc. +International Business Machines Corporation +Ismael Jimenez Martinez +Jern-Kuan Leong +JianXiong Zhou +Joao Paulo Magalhaes +Jussi Knuuttila +Kaito Udagawa +Kishan Kumar +Lei Xu +Matt Clarkson +Maxim Vafin +MongoDB Inc. +Nick Hutchinson +Oleksandr Sochka +Paul Redmond +Radoslav Yovchev +Roman Lebedev +Shuo Chen +Steinar H. Gunderson +Stripe, Inc. +Yixuan Qiu +Yusuke Suzuki +Zbigniew Skowron + +_____ + +HalidelR + +Copyright (c) 2016 HalideIR contributors +Copyright (c) 2012-2014 MIT CSAIL, Google Inc., and other contributors +HalideIR is derived from the Halide project. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +____ + +Distributed Machine Learning Common Codebase + +Copyright (c) 2015 by Contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +_____ + +DLPack: Open In Memory Tensor Structure + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2017 by Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +_____ + +JSON for Modern C++ + +MIT License + +Copyright (c) 2013-2018 Niels Lohmann + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +____ + +Date + +The source code in this project is released using the MIT License. There is no +global license for the project because each file is licensed individually with +different author names and/or dates. + +If you contribute to this project, please add your name to the license of each +file you modify. If you have already contributed to this project and forgot to +add your name to the license, please feel free to submit a new P/R to add your +name to the license in each file you modified. + +For convenience, here is a copy of the MIT license found in each file except +without author names or dates: + +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +____ + +TVM Open Deep Learning Compiler Stack + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +CONTRIBUTORS + +TVM Contributors +================ +TVM adopts the Apache style model and governs by merit. We believe that it is important to create an inclusive community where everyone can use, +contribute to, and influence the direction of the project. We actively invite contributors who have earned the merit to be part of the development community. + +See the [community structure document](http://docs.tvm.ai/contribute/community.html) for the explanation of community structure and contribution guidelines. + +## Committers +- [Tianqi Chen](https://github.com/tqchen) (PMC) +- [Thierry Moreau](http://homes.cs.washington.edu/~moreau/) +- [Ziheng Jiang](https://github.com/ZihengJiang) +- [Haichen Shen](http://homes.cs.washington.edu/~haichen/) +- [Yizhi Liu](https://github.com/yzhliu) + +## Code Owners +- [Aditya Atluri](https://github.com/adityaatluri) ROCM +- [Leyuan Wang](https://github.com/Laurawly) TOPI +- [Yuwei Hu](https://github.com/Huyuwei) TOPI +- [Zhixun Tan](https://github.com/phisiart) OpenGL/WebGL backend +- [Nick Hynes](https://github.com/nhynes) SGX and secured computing +- [Lianmin Zheng](https://github.com/merrymercy) AutoTVM + +## Reviewers +- [Zhi Chen](https://github.com/zhiics) +- [Xiaoqiang Dan](https://github.com/xqdan) +- [Liangfu Chen](https://github.com/liangfu) +- [Masahiro Masuda](https://github.com/masahi) +- [Kazutaka Morita](https://github.com/kazum) +- [Tatsuya Nishiyama](https://github.com/nishi-t) +- [Pariksheet Pinjari](https://github.com/PariksheetPinjari909) +- [Jared Roesch](https://github.com/jroesch) +- [Siva](https://github.com/srkreddy1238) +- [Siju Samuel](https://github.com/siju-samuel) +- [Alex Weaver](https://github.com/alex-weaver) +- [Yao Wang](https://github.com/kevinthesun) +- [Jian Weng](https://github.com/were) +- [Eddie Yan](https://github.com/eqy) +- [Joshua Z. Zhang](https://github.com/zhreshold) + +## List of Contributors +- [Full List of Contributors](https://github.com/dmlc/tvm/graphs/contributors) + - To contributors: please add your name to the list. +- [Qiao Zhang](https://github.com/zhangqiaorjc) +- [Haolong Zhang](https://github.com/haolongzhangm) +- [Cody Hao Yu](https://github.com/comaniac) +- [Chris Nuernberger](https://github.com/cnuernber) + +_____ + +jemalloc + +Unless otherwise specified, files in the jemalloc source distribution are +subject to the following license: +-------------------------------------------------------------------------------- +Copyright (C) 2002-2018 Jason Evans . +All rights reserved. +Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. +Copyright (C) 2009-2018 Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice(s), + this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice(s), + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- + +____ + +FreeBSD: getopt.c file + +Copyright (c) 1987, 1993, 1994 +The Regents of the University of California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + 1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +3. Neither the name of the University nor the names of its contributors +may be used to endorse or promote products derived from this software +without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. +_____ + + +google/googletest + +Copyright 2008, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +_____ + +G3log : Asynchronous logger with Dynamic Sinks + +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to +_____ diff --git a/VERSION_NUMBER b/VERSION_NUMBER new file mode 100644 index 0000000000000..446ba66e7eb44 --- /dev/null +++ b/VERSION_NUMBER @@ -0,0 +1 @@ +0.1.4 \ No newline at end of file diff --git a/build.amd64.1411.bat b/build.amd64.1411.bat new file mode 100644 index 0000000000000..5a289c13e0488 --- /dev/null +++ b/build.amd64.1411.bat @@ -0,0 +1,12 @@ +:: Copyright (c) Microsoft Corporation. All rights reserved. +:: Licensed under the MIT License. + +rem This will setup the VC env vars to use the 14.11 (VS2017 ver15.3) toolchain which is supported by CUDA 9.2 prior to running build.py. +rem It currently defaults to amd64 but that could be made configurable if that would be useful to developers running this locally. +@echo off + +rem Use 14.11 toolset +call "%VCINSTALLDIR%\Auxiliary\Build\vcvarsall.bat" amd64 -vcvars_ver=14.11 + +rem Requires a python 3.6 or higher install to be available in your PATH +python %~dp0\tools\ci_build\build.py --build_dir %~dp0\build\Windows %* \ No newline at end of file diff --git a/build.bat b/build.bat new file mode 100644 index 0000000000000..d7f6a8513ebd1 --- /dev/null +++ b/build.bat @@ -0,0 +1,6 @@ +:: Copyright (c) Microsoft Corporation. All rights reserved. +:: Licensed under the MIT License. + +@echo off +rem Requires a python 3.6 or higher install to be available in your PATH +python %~dp0\tools\ci_build\build.py --build_dir %~dp0\build\Windows %* \ No newline at end of file diff --git a/build.sh b/build.sh new file mode 100755 index 0000000000000..1be1e32b71023 --- /dev/null +++ b/build.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# Get directory this script is in +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +#requires python3.6 or higher +python3 $DIR/tools/ci_build/build.py --build_dir $DIR/build/Linux "$@" diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt new file mode 100644 index 0000000000000..158356b67fe60 --- /dev/null +++ b/cmake/CMakeLists.txt @@ -0,0 +1,493 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# Minimum CMake required +cmake_minimum_required(VERSION 3.11) + +# Project +project(onnxruntime C CXX) +include(CheckCXXCompilerFlag) +include(CheckLanguage) + +# Set C++14 as standard for the whole project +set(CMAKE_CXX_STANDARD 14) + +# General C# prperties +if (onnxruntime_BUILD_CSHARP) + check_language(CSharp) + if (CMAKE_CSharp_COMPILER) + enable_language(CSharp) + set(CMAKE_DOTNET_TARGET_FRAMEWORK_VERSION v4.6.1) + set(CMAKE_CSharp_FLAGS ${CMAKE_CSharp_FLAGS} "/langversion:6") + message(STATUS "CMAKE_Csharp_Compiler = ${CMAKE_CSharp_COMPILER}") + else() + message(WARNING "Language Csharp is not found in the system") + endif() +endif() + +set_property(GLOBAL PROPERTY USE_FOLDERS ON) +# NOTE: POSITION INDEPENDENT CODE hurts performance, and it only make sense on POSIX systems +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +# Enable CTest +enable_testing() + +if(NOT CMAKE_BUILD_TYPE) + message(STATUS "Build type not set - using RelWithDebInfo") + set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "Choose build type: Debug Release RelWithDebInfo." FORCE) +endif() + +# Options +option(onnxruntime_RUN_ONNX_TESTS "Enable ONNX Compatibility Testing" OFF) +option(onnxruntime_GENERATE_TEST_REPORTS "Enable test report generation" OFF) +option(onnxruntime_ENABLE_STATIC_ANALYSIS "Enable static analysis" OFF) +option(onnxruntime_ENABLE_PYTHON "Enable python buildings" OFF) +option(onnxruntime_USE_CUDA "Build with CUDA support" OFF) +option(onnxruntime_USE_EIGEN_FOR_BLAS "Use eign for blas" ON) +option(onnxruntime_USE_MLAS "Use optimized blas library for GEMM and 2D Convolution" OFF) +option(onnxruntime_USE_MKLDNN "Build with MKL-DNN support" OFF) +option(onnxruntime_USE_MKLML "Build MKL-DNN with MKL-ML binary dependency" OFF) +option(onnxruntime_USE_OPENBLAS "Use openblas" OFF) +option(onnxruntime_DEV_MODE "Enable developer warnings and treat most of them as error." OFF) +option(onnxruntime_USE_PREBUILT_PB "Use prebuilt protobuf library" OFF) +option(onnxruntime_USE_JEMALLOC "Use jecmalloc" OFF) +option(onnxruntime_MSVC_STATIC_RUNTIME "Compile for the static CRT" OFF) +option(onnxruntime_BUILD_UNIT_TESTS "Build ONNXRuntime unit tests" ON) +option(onnxruntime_USE_PREINSTALLED_EIGEN "Use pre-installed EIGEN. Need to provide eigen_SOURCE_PATH if turn this on." OFF) +option(onnxruntime_BUILD_BENCHMARKS "Build ONNXRuntime micro-benchmarks" OFF) +option(onnxruntime_USE_TVM "Build tvm for code-gen" OFF) +option(onnxruntime_USE_LLVM "Build tvm with LLVM" OFF) +option(onnxruntime_USE_OPENMP "Build with OpenMP support" OFF) +option(onnxruntime_BUILD_SHARED_LIB "Build a shared library" OFF) +option(onnxruntime_ENABLE_MICROSOFT_INTERNAL "Use this option to enable/disable microsoft internal only code" OFF) +option(onnxruntime_USE_NUPHAR "Build with Nupha" OFF) +option(onnxruntime_USE_BRAINSLICE "Build with BrainSlice" OFF) + +set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE) +set(ONNX_ML 1) + +set(REPO_ROOT ${PROJECT_SOURCE_DIR}/..) +set(ONNXRUNTIME_ROOT ${PROJECT_SOURCE_DIR}/../onnxruntime) +file (STRINGS "${REPO_ROOT}/VERSION_NUMBER" VERSION_NUMBER) + +if (MSVC) + if (onnxruntime_MSVC_STATIC_RUNTIME) + # set all of our submodules to static runtime + set(ONNX_USE_MSVC_STATIC_RUNTIME ON) + set(protobuf_MSVC_STATIC_RUNTIME ON) + set(gtest_force_shared_crt OFF) + + # In case we are building static libraries, link also the runtime library statically + # so that MSVCR*.DLL is not required at runtime. + # https://msdn.microsoft.com/en-us/library/2kzt1wy3.aspx + # This is achieved by replacing msvc option /MD with /MT and /MDd with /MTd + # http://www.cmake.org/Wiki/CMake_FAQ#How_can_I_build_my_MSVC_application_with_a_static_runtime.3F + foreach(flag_var + CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE + CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO + CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE + CMAKE_C_FLAGS_MINSIZEREL CMAKE__FLAGS_RELWITHDEBINFO) + if(${flag_var} MATCHES "/MD") + string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}") + endif(${flag_var} MATCHES "/MD") + endforeach(flag_var) + else() + set(ONNX_USE_MSVC_STATIC_RUNTIME OFF) + set(protobuf_WITH_ZLIB OFF CACHE BOOL "" FORCE) + set(protobuf_MSVC_STATIC_RUNTIME OFF CACHE BOOL "Link protobuf to static runtime libraries" FORCE) + set(gtest_force_shared_crt ON CACHE BOOL "Use shared (DLL) run-time lib for gtest" FORCE) + endif() + #Always enable exception handling, even for Windows ARM + SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc") + #Disable 4100 globally. Too many this kind errors in protobuf + SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4100") +else() + # Enable OpenMP for Non-Windows only. WinML team disallows use of OpenMP. + find_package(OpenMP) + if (OPENMP_FOUND) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") + add_definitions(-DUSE_OPENMP) + endif() +endif() +find_package(PNG) +set(ENABLE_DATE_TESTING OFF CACHE BOOL "" FORCE) +set(USE_SYSTEM_TZ_DB ON CACHE BOOL "" FORCE) + +if(CMAKE_CROSSCOMPILING) + message("Doing crosscompiling") +endif() + +#Need python to generate def file +if(onnxruntime_BUILD_SHARED_LIB OR onnxruntime_ENABLE_PYTHON) + if(onnxruntime_ENABLE_PYTHON) + find_package(PythonInterp 3.5 REQUIRED) + find_package(PythonLibs 3.5 REQUIRED) + else() + find_package(PythonInterp 3.4 REQUIRED) + find_package(PythonLibs 3.4 REQUIRED) + endif() +endif() + +if(onnxruntime_BUILD_BENCHMARKS) + if(NOT TARGET benchmark) + # We will not need to test benchmark lib itself. + set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Disable benchmark testing as we don't need it.") + # We will not need to install benchmark since we link it statically. + set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "Disable benchmark install to avoid overwriting vendor install.") + add_subdirectory(${PROJECT_SOURCE_DIR}/external/onnx/third_party/benchmark EXCLUDE_FROM_ALL) + endif() +endif() + +# External dependencies +list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/external) + +#Here we support three build mode: +#1. (recommended)onnxruntime_USE_PREBUILT_PB is set (ONNX_CUSTOM_PROTOC_EXECUTABLE should also be set) +# We will not build protobuf, will use a prebuilt binary instead. This mode can also support cross-compiling +#2. onnxruntime_USE_PREBUILT_PB is not set but ONNX_CUSTOM_PROTOC_EXECUTABLE is set +# Build Protobuf from source, except protoc.exe. This mode is mainly for cross-compiling +#3. both onnxruntime_USE_PREBUILT_PB and ONNX_CUSTOM_PROTOC_EXECUTABLE are not set +# Compile everything from source code. Slowest option. + +if(onnxruntime_USE_PREBUILT_PB) + get_filename_component( + _PROTOBUF_INSTALL_PREFIX + ${ONNX_CUSTOM_PROTOC_EXECUTABLE} + DIRECTORY) + get_filename_component( + _PROTOBUF_INSTALL_PREFIX + ${_PROTOBUF_INSTALL_PREFIX}/.. + REALPATH) + if(WIN32) + include(${_PROTOBUF_INSTALL_PREFIX}/cmake/protobuf-config.cmake) + else() + include(${_PROTOBUF_INSTALL_PREFIX}/lib64/cmake/protobuf/protobuf-config.cmake) + endif() + include(protobuf_function.cmake) +else() + # use protobuf as a submodule + add_subdirectory(${PROJECT_SOURCE_DIR}/external/protobuf/cmake EXCLUDE_FROM_ALL) + set_target_properties(libprotobuf PROPERTIES FOLDER "External/Protobuf") + set_target_properties(libprotobuf-lite PROPERTIES FOLDER "External/Protobuf") + set_target_properties(libprotoc PROPERTIES FOLDER "External/Protobuf") + set_target_properties(protoc PROPERTIES FOLDER "External/Protobuf") + add_library(protobuf::libprotobuf ALIAS libprotobuf) + add_executable(protobuf::protoc ALIAS protoc) + include(protobuf_function.cmake) +endif() + +if (onnxruntime_USE_CUDA AND "${onnxruntime_CUDNN_HOME}" STREQUAL "") + message(FATAL_ERROR "onnxruntime_CUDNN_HOME required for onnxruntime_USE_CUDA") +endif() + +if (onnxruntime_USE_EIGEN_FOR_BLAS) + add_definitions(-DUSE_EIGEN_FOR_BLAS) +endif() + +if (onnxruntime_USE_OPENBLAS AND "${onnxruntime_OPENBLAS_HOME}" STREQUAL "" AND WIN32) + # On linux we assume blas is installed via 'apt-get install libopenblas-dev' + message(FATAL_ERROR "onnxruntime_OPENBLAS_HOME required for onnxruntime_USE_OPENBLAS") +endif() + +if (onnxruntime_USE_OPENBLAS AND onnxruntime_USE_EIGEN_FOR_BLAS) + message(FATAL_ERROR "use one of onnxruntime_USE_OPENBLAS, onnxruntime_USE_EIGEN_FOR_BLAS") +endif() + +# if ON put all the unit tests in a single project so that code coverage is more comprehensive. +# defaulting to that and most likely removing option to have separate unit test projects in the near future. +set(SingleUnitTestProject ON) +if (onnxruntime_SPLIT_UNIT_TEST_PROJECTS) + set(SingleUnitTestProject OFF) +endif() + +get_filename_component(ONNXRUNTIME_ROOT "${ONNXRUNTIME_ROOT}" ABSOLUTE) +get_filename_component(REPO_ROOT "${REPO_ROOT}" ABSOLUTE) +set(ONNXRUNTIME_INCLUDE_DIR ${REPO_ROOT}/include/onnxruntime) + +add_subdirectory(external/date EXCLUDE_FROM_ALL) +add_subdirectory(external/gsl EXCLUDE_FROM_ALL) +add_library(date ALIAS tz) +add_library(gsl ALIAS GSL) + +set(date_INCLUDE_DIR $) + +# bounds checking behavior. +# throw instead of calling terminate if there's a bounds checking violation. +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DGSL_THROW_ON_CONTRACT_VIOLATION") +# no bounds checking in release build so no perf cost +set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DGSL_UNENFORCED_ON_CONTRACT_VIOLATION") +set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -DGSL_UNENFORCED_ON_CONTRACT_VIOLATION") + +include(eigen) +set(onnxruntime_EXTERNAL_LIBRARIES protobuf::libprotobuf) + +# gtest and gmock +add_subdirectory(${PROJECT_SOURCE_DIR}/external/googletest EXCLUDE_FROM_ALL) +set_target_properties(gmock PROPERTIES FOLDER "External/GTest") +set_target_properties(gmock_main PROPERTIES FOLDER "External/GTest") +set_target_properties(gtest PROPERTIES FOLDER "External/GTest") +set_target_properties(gtest_main PROPERTIES FOLDER "External/GTest") + +function(onnxruntime_add_include_to_target dst_target) + foreach(src_target ${ARGN}) + target_include_directories(${dst_target} PRIVATE $) + target_compile_definitions(${dst_target} PRIVATE $) + endforeach() +endfunction() + +# TVM +if (onnxruntime_USE_TVM) + if (onnxruntime_USE_CUDA) + set(USE_CUDA ON) + endif() + if (onnxruntime_USE_LLVM) + set(USE_LLVM ON) + add_definitions(-DUSE_TVM_WITH_LLVM) + endif() + add_subdirectory(${PROJECT_SOURCE_DIR}/external/tvm EXCLUDE_FROM_ALL) + set_target_properties(tvm PROPERTIES FOLDER "External/tvm") + set_target_properties(tvm_topi PROPERTIES FOLDER "External/tvm") + set_target_properties(tvm_runtime PROPERTIES FOLDER "External/tvm") + set_target_properties(nnvm_compiler PROPERTIES FOLDER "External/tvm") + + set(TVM_INCLUDES ${PROJECT_SOURCE_DIR}/external/tvm/include + ${PROJECT_SOURCE_DIR}/external/tvm/3rdparty/dmlc-core/include + ${PROJECT_SOURCE_DIR}/external/tvm/3rdparty/dlpack/include + $ + $ + $) + add_definitions(-DUSE_TVM) + + set(onnxruntime_tvm_libs + onnxruntime_codegen_tvm + tvm + nnvm_compiler) + + set(onnxruntime_tvm_dependencies + tvm + nnvm_compiler + onnxruntime_codegen_tvm) +endif() + +# ONNX +add_subdirectory(onnx) +set_target_properties(onnx PROPERTIES FOLDER "External/ONNX") +set_target_properties(onnx_proto PROPERTIES FOLDER "External/ONNX") +#set_target_properties(gen_onnx_proto PROPERTIES FOLDER "External/ONNX") +# fix a warning in onnx code we can't do anything about +if (MSVC) + target_compile_options(onnx_proto PRIVATE /wd4146) # unary minus operator applied to unsigned type +endif() +set(onnxruntime_EXTERNAL_DEPENDENCIES gsl onnx_proto) + +if (onnxruntime_RUN_ONNX_TESTS) + add_definitions(-DONNXRUNTIME_RUN_EXTERNAL_ONNX_TESTS) +endif() + +add_definitions(-DUSE_MLAS) +#Adjust warning flags +if (WIN32) + add_definitions(-DPLATFORM_WINDOWS -DNOGDI -DNOMINMAX -D_USE_MATH_DEFINES) + # parallel build + # These compiler opitions cannot be forwarded to NVCC, so cannot use add_compiler_options + string(APPEND CMAKE_CXX_FLAGS " /MP") + string(APPEND CMAKE_CXX_FLAGS + " /wd4503" # Decorated name length exceeded. + " /wd4127" # conditional expression is constant. + " /wd4146" # unary minus operator applied to unsigned type. Needed for Protobuf + ) + if (onnxruntime_ENABLE_STATIC_ANALYSIS) + string(APPEND CMAKE_CXX_FLAGS + " /analyze:WX- " + # disable warning because there are many occurrences from test macros + " /wd6326 " # potential comparison of a constant with another constant + ) + endif() + + # set compile warning level to 3 on CUDA build but 4 on CPU-only build + if(onnxruntime_USE_CUDA) + #CMake hardcoded /W3 in its 'Windows-NVIDIA-CUDA.cmake'. We'd better keep consistent with it. + #Change it to /W4 will result build failure + string(APPEND CMAKE_CXX_FLAGS " /W3") + else() + string(APPEND CMAKE_CXX_FLAGS " /W4") + endif() + + #only treat warning as error on x64 platform. For x86, right no there are too many warnings to fix + if (CMAKE_SIZEOF_VOID_P EQUAL 8 AND onnxruntime_DEV_MODE) + # treat warnings as errors + string(APPEND CMAKE_CXX_FLAGS " /WX") + foreach(type EXE STATIC SHARED) + set(CMAKE_${type}_LINKER_FLAGS "${CMAKE_${type}_LINKER_FLAGS} /WX") + endforeach() + endif() +else() + add_definitions(-DPLATFORM_POSIX) + # Enable warning in Linux + string(APPEND CMAKE_CXX_FLAGS " -Wall -Wextra") + string(APPEND CMAKE_C_FLAGS " -Wall -Wextra") + if(onnxruntime_DEV_MODE) + string(APPEND CMAKE_CXX_FLAGS " -Werror") + string(APPEND CMAKE_C_FLAGS " -Werror") + endif() + check_cxx_compiler_flag(-Wunused-but-set-variable HAS_UNUSED_BUT_SET_VARIABLE) + check_cxx_compiler_flag(-Wunused-parameter HAS_UNUSED_PARAMETER) + check_cxx_compiler_flag(-Wcast-function-type HAS_CAST_FUNCTION_TYPE) + check_cxx_compiler_flag(-Wparentheses HAS_PARENTHESES) + check_cxx_compiler_flag(-Wnull-dereference HAS_NULL_DEREFERENCE) + check_cxx_compiler_flag(-Wuseless-cast HAS_USELESS_CAST) + check_cxx_compiler_flag(-Wnonnull-compare HAS_NONNULL_COMPARE) + check_cxx_compiler_flag(-Wtautological-pointer-compare HAS_TAUTOLOGICAL_POINTER_COMPARE) + check_cxx_compiler_flag(-Wcatch-value HAS_CATCH_VALUE) + if(HAS_NULL_DEREFERENCE) + string(APPEND CMAKE_CXX_FLAGS " -Wnull-dereference") + string(APPEND CMAKE_C_FLAGS " -Wnull-dereference") + endif() + if(HAS_TAUTOLOGICAL_POINTER_COMPARE) + #we may have extra null pointer checkings in debug build, it's not an issue + string(APPEND CMAKE_CXX_FLAGS_DEBUG " -Wno-tautological-pointer-compare") + string(APPEND CMAKE_C_FLAGS_DEBUG " -Wno-tautological-pointer-compare") + endif() + if(HAS_NONNULL_COMPARE) + #we may have extra null pointer checkings in debug build, it's not an issue + string(APPEND CMAKE_CXX_FLAGS_DEBUG " -Wno-nonnull-compare") + string(APPEND CMAKE_C_FLAGS_DEBUG " -Wno-nonnull-compare") + endif() + string(APPEND CMAKE_CXX_FLAGS " -Wno-error=sign-compare -Wno-error=comment") + if(onnxruntime_USE_CUDA) + string(APPEND CMAKE_CXX_FLAGS " -Wno-error=reorder") + endif() + if(HAS_PARENTHESES) + string(APPEND CMAKE_CXX_FLAGS " -Wno-parentheses") + endif() + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") + string(APPEND CMAKE_CXX_FLAGS " -Wno-error=invalid-partial-specialization -Wno-error=missing-braces -Wno-error=inconsistent-missing-override") + endif() +endif() + +if (onnxruntime_USE_TVM) + if (WIN32 AND MSVC) + # wd4100: identifier' : unreferenced formal parameter + # wd4244: conversion from 'int' to 'char', possible loss of data + # wd4251: class X needs to have dll-interface to be used by clients of class Y + # wd4267: 'initializing': conversion from 'size_t' to 'int', possible loss of data + # wd4275: non dll-interface class X used as base for dll-interface class Y + # wd4389: signed/unsigned mismatch + # wd4456: declaration of X hides previous local declaration + set(DISABLED_WARNINGS_FOR_TVM "/wd4100" "/wd4244" "/wd4251" "/wd4267" "/wd4275" "/wd4389" "/wd4456") + else() + set(DISABLED_WARNINGS_FOR_TVM "-Wno-error=extra" "-Wno-error=ignored-qualifiers") + if(HAS_UNUSED_PARAMETER) + list(APPEND DISABLED_WARNINGS_FOR_TVM "-Wno-error=unused-parameter") + endif() + if(HAS_CATCH_VALUE) + #TODO: send a PR to TVM and fix it + list(APPEND DISABLED_WARNINGS_FOR_TVM "-Wno-error=catch-value") + endif() + endif() + include(onnxruntime_codegen.cmake) +endif() + +if (onnxruntime_USE_JEMALLOC) + if (Win32) + message( FATAL_ERROR "Jemalloc is not supported on Windows." ) + endif() + include(jemalloc) + add_definitions(-DUSE_JEMALLOC=1) + list(APPEND onnxruntime_EXTERNAL_LIBRARIES ${JEMALLOC_STATIC_LIBRARIES}) + list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES jemalloc) +endif() + +include_directories( + ${ONNXRUNTIME_INCLUDE_DIR} + $ +) + +if (onnxruntime_USE_MKLDNN) + add_definitions(-DUSE_MKLDNN=1) + include(mkldnn) + list(APPEND onnxruntime_EXTERNAL_LIBRARIES mkldnn) + list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES mkldnn) + link_directories(${MKLDNN_LIB_DIR}) +endif() + +if (onnxruntime_USE_OPENBLAS) + add_definitions(-DUSE_OPENBLAS=1) + if (WIN32) + include_directories(${onnxruntime_OPENBLAS_HOME}) + list(APPEND onnxruntime_EXTERNAL_LIBRARIES ${onnxruntime_OPENBLAS_HOME}/lib/libopenblas.lib) + else() + # on linux we assume blas is installed via 'apt-get install libopenblas-dev' + list(APPEND onnxruntime_EXTERNAL_LIBRARIES openblas) + endif() +endif() + +configure_file(onnxruntime_config.h.in ${CMAKE_CURRENT_BINARY_DIR}/onnxruntime_config.h) + +if (onnxruntime_USE_CUDA) + add_definitions(-DUSE_CUDA=1) + enable_language(CUDA) + set(CMAKE_CUDA_STANDARD 14) + find_package(CUDA 9.0 REQUIRED) + file(TO_CMAKE_PATH ${onnxruntime_CUDNN_HOME} onnxruntime_CUDNN_HOME) + include(cub) + set(CUDA_LINK_LIBRARIES_KEYWORD PRIVATE) + file(TO_CMAKE_PATH ${onnxruntime_CUDNN_HOME} onnxruntime_CUDNN_HOME) + if (WIN32) + link_directories(${onnxruntime_CUDNN_HOME}/lib/x64) + set(ONNXRUNTIME_CUDA_LIBRARIES cudnn cublas) + else() + link_directories(${onnxruntime_CUDNN_HOME}/lib64) + set(ONNXRUNTIME_CUDA_LIBRARIES cudnn_static cublas_static culibos) + endif() + list(APPEND onnxruntime_EXTERNAL_LIBRARIES ${ONNXRUNTIME_CUDA_LIBRARIES}) + list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES cub) + + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -gencode=arch=compute_30,code=sm_30") # K series + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -gencode=arch=compute_50,code=sm_50") # M series + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -gencode=arch=compute_60,code=sm_60") # P series + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -gencode=arch=compute_70,code=sm_70") # V series + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --default-stream per-thread") + if (NOT WIN32) + set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} --compiler-options -fPIC") + endif() +endif() + +#names in this var must match the directory names under onnxruntime/core/providers +set(ONNXRUNTIME_PROVIDER_NAMES cpu) + +include(onnxruntime_common.cmake) +include(onnxruntime_graph.cmake) +include(onnxruntime_framework.cmake) +include(onnxruntime_util.cmake) +include(onnxruntime_providers.cmake) +include(onnxruntime_session.cmake) +include(onnxruntime_mlas.cmake) + + +if (onnxruntime_BUILD_SHARED_LIB) + include(onnxruntime.cmake) +endif() + +if (onnxruntime_ENABLE_PYTHON) +if(UNIX) + set(CMAKE_SKIP_BUILD_RPATH ON) +endif() +include(onnxruntime_python.cmake) +endif() + +if (onnxruntime_BUILD_CSHARP) + message(STATUS "CSharp Build is enabled") +# set_property(GLOBAL PROPERTY VS_DOTNET_TARGET_FRAMEWORK_VERSION "netstandard2.0") + include(onnxruntime_csharp.cmake) +endif() + +# some of the tests rely on the shared libs to be +# built; hence the ordering +if (onnxruntime_BUILD_UNIT_TESTS) + # we need to make sure this is turned off since it + # turned ON by the previous step when building a shared lib + set(CMAKE_SKIP_BUILD_RPATH OFF) + include(onnxruntime_unittests.cmake) +endif() \ No newline at end of file diff --git a/cmake/ConfigureVisualStudioCodeAnalysis.props b/cmake/ConfigureVisualStudioCodeAnalysis.props new file mode 100644 index 0000000000000..bad2a87b930e7 --- /dev/null +++ b/cmake/ConfigureVisualStudioCodeAnalysis.props @@ -0,0 +1,14 @@ + + + + true + NativeRecommendedRules.ruleset + false + + $(SolutionDir);$(SolutionDir)..\..\..\cmake; + + diff --git a/cmake/EnableVisualStudioCodeAnalysis.props b/cmake/EnableVisualStudioCodeAnalysis.props new file mode 100644 index 0000000000000..9da9f0be46560 --- /dev/null +++ b/cmake/EnableVisualStudioCodeAnalysis.props @@ -0,0 +1,24 @@ + + + + true + NativeRecommendedRules.ruleset + + $(SolutionDir);$(MSBuildThisFileDirectory) + + + true + + + diff --git a/cmake/external/FindNumPy.cmake b/cmake/external/FindNumPy.cmake new file mode 100644 index 0000000000000..4155744ba3033 --- /dev/null +++ b/cmake/external/FindNumPy.cmake @@ -0,0 +1,56 @@ +# - Find the NumPy libraries +# This module finds if NumPy is installed, and sets the following variables +# indicating where it is. +# +# TODO: Update to provide the libraries and paths for linking npymath lib. +# +# NUMPY_FOUND - was NumPy found +# NUMPY_VERSION - the version of NumPy found as a string +# NUMPY_VERSION_MAJOR - the major version number of NumPy +# NUMPY_VERSION_MINOR - the minor version number of NumPy +# NUMPY_VERSION_PATCH - the patch version number of NumPy +# NUMPY_VERSION_DECIMAL - e.g. version 1.6.1 is 10601 +# NUMPY_INCLUDE_DIR - path to the NumPy include files + +unset(NUMPY_VERSION) +unset(NUMPY_INCLUDE_DIR) + +if(PYTHONINTERP_FOUND) + execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c" + "import numpy as n; print(n.__version__); print(n.get_include());" + RESULT_VARIABLE __result + OUTPUT_VARIABLE __output + OUTPUT_STRIP_TRAILING_WHITESPACE) + + if(__result MATCHES 0) + string(REGEX REPLACE ";" "\\\\;" __values ${__output}) + string(REGEX REPLACE "\r?\n" ";" __values ${__values}) + list(GET __values 0 NUMPY_VERSION) + list(GET __values 1 NUMPY_INCLUDE_DIR) + + string(REGEX MATCH "^([0-9])+\\.([0-9])+\\.([0-9])+" __ver_check "${NUMPY_VERSION}") + if(NOT "${__ver_check}" STREQUAL "") + set(NUMPY_VERSION_MAJOR ${CMAKE_MATCH_1}) + set(NUMPY_VERSION_MINOR ${CMAKE_MATCH_2}) + set(NUMPY_VERSION_PATCH ${CMAKE_MATCH_3}) + math(EXPR NUMPY_VERSION_DECIMAL + "(${NUMPY_VERSION_MAJOR} * 10000) + (${NUMPY_VERSION_MINOR} * 100) + ${NUMPY_VERSION_PATCH}") + string(REGEX REPLACE "\\\\" "/" NUMPY_INCLUDE_DIR ${NUMPY_INCLUDE_DIR}) + else() + unset(NUMPY_VERSION) + unset(NUMPY_INCLUDE_DIR) + message(STATUS "Requested NumPy version and include path, but got instead:\n${__output}\n") + endif() + endif() +else() + message(STATUS "To find NumPy Python interpretator is required to be found.") +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(NumPy REQUIRED_VARS NUMPY_INCLUDE_DIR NUMPY_VERSION + VERSION_VAR NUMPY_VERSION) + +if(NUMPY_FOUND) + message(STATUS "NumPy ver. ${NUMPY_VERSION} found (include: ${NUMPY_INCLUDE_DIR})") +endif() + diff --git a/cmake/external/cub.cmake b/cmake/external/cub.cmake new file mode 100644 index 0000000000000..080e11c7fd22e --- /dev/null +++ b/cmake/external/cub.cmake @@ -0,0 +1,30 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +include (ExternalProject) + +set(cub_URL https://github.com/NVlabs/cub/archive/v1.8.0.zip) +set(cub_HASH SHA256=6bfa06ab52a650ae7ee6963143a0bbc667d6504822cbd9670369b598f18c58c3) +set(cub_BUILD ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub) +set(cub_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub) +set(cub_ARCHIVE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/cub_archive) + +ExternalProject_Add(cub + PREFIX cub + URL ${cub_URL} + URL_HASH ${cub_HASH} + DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" + BUILD_IN_SOURCE 1 + PATCH_COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/patches/cub/CMakeLists.txt ${cub_BUILD} + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory ${cub_INCLUDE_DIR}/cub ${cub_ARCHIVE_DIR}/cub) diff --git a/cmake/external/date b/cmake/external/date new file mode 160000 index 0000000000000..e7e1482087f58 --- /dev/null +++ b/cmake/external/date @@ -0,0 +1 @@ +Subproject commit e7e1482087f58913b80a20b04d5c58d9d6d90155 diff --git a/cmake/external/eigen.cmake b/cmake/external/eigen.cmake new file mode 100644 index 0000000000000..156dee830814f --- /dev/null +++ b/cmake/external/eigen.cmake @@ -0,0 +1,28 @@ +include (ExternalProject) + +if (onnxruntime_USE_PREINSTALLED_EIGEN) + set(eigen_INCLUDE_DIRS ${eigen_SOURCE_PATH}) + ExternalProject_Add(eigen + PREFIX eigen + SOURCE_DIR ${eigen_SOURCE_PATH} + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + DOWNLOAD_COMMAND "" + UPDATE_COMMAND "" + ) +else () + set(eigen_URL "https://github.com/eigenteam/eigen-git-mirror.git") + set(eigen_TAG "3.3.4") + set(eigen_ROOT_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/eigen) + set(eigen_INCLUDE_DIRS ${eigen_ROOT_DIR}) + ExternalProject_Add(eigen + PREFIX eigen + GIT_REPOSITORY ${eigen_URL} + GIT_TAG ${eigen_TAG} + SOURCE_DIR ${eigen_ROOT_DIR} + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + ) +endif() diff --git a/cmake/external/googletest b/cmake/external/googletest new file mode 160000 index 0000000000000..9bda90b7e5e08 --- /dev/null +++ b/cmake/external/googletest @@ -0,0 +1 @@ +Subproject commit 9bda90b7e5e08c4c37a832d0cea218aed6af6470 diff --git a/cmake/external/gsl b/cmake/external/gsl new file mode 160000 index 0000000000000..cee3125af7208 --- /dev/null +++ b/cmake/external/gsl @@ -0,0 +1 @@ +Subproject commit cee3125af7208258d024a75e24f73977eddaec5b diff --git a/cmake/external/jemalloc.cmake b/cmake/external/jemalloc.cmake new file mode 100644 index 0000000000000..db0b922aee486 --- /dev/null +++ b/cmake/external/jemalloc.cmake @@ -0,0 +1,24 @@ +include (ExternalProject) + +set(JEMALLOC_URL https://github.com/jemalloc/jemalloc/releases/download/4.1.1/jemalloc-4.1.1.tar.bz2) +set(JEMALLOC_BUILD ${CMAKE_CURRENT_BINARY_DIR}/jemalloc/src/jemalloc) +set(JEMALLOC_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/jemalloc/install) + +if(NOT WIN32) + set(JEMALLOC_STATIC_LIBRARIES ${CMAKE_CURRENT_BINARY_DIR}/jemalloc/install/lib/libjemalloc_pic.a) +else() + message( FATAL_ERROR "Jemalloc is not supported on Windows." ) +endif() + +ExternalProject_Add(jemalloc + PREFIX jemalloc + URL ${JEMALLOC_URL} + INSTALL_DIR ${JEMALLOC_INSTALL} + DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" + BUILD_COMMAND $(MAKE) + BUILD_IN_SOURCE 1 + INSTALL_COMMAND $(MAKE) install + CONFIGURE_COMMAND + ${CMAKE_CURRENT_BINARY_DIR}/jemalloc/src/jemalloc/configure + --prefix=${JEMALLOC_INSTALL} + ) diff --git a/cmake/external/mkldnn.cmake b/cmake/external/mkldnn.cmake new file mode 100644 index 0000000000000..69d20b8f77c29 --- /dev/null +++ b/cmake/external/mkldnn.cmake @@ -0,0 +1,55 @@ +include (ExternalProject) + +set(MKLDNN_URL https://github.com/intel/mkl-dnn.git) +# If MKLDNN_TAG is updated, check if platform.cmake.patch or mkldnn_sgemm.patch needs to be updated. +set(MKLDNN_TAG v0.15) +set(MKLDNN_SOURCE ${CMAKE_CURRENT_BINARY_DIR}/mkl-dnn/src/mkl-dnn/src) +set(MKLDNN_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/mkl-dnn/install) +set(MKLDNN_LIB_DIR ${MKLDNN_INSTALL}/lib) +set(MKLDNN_INCLUDE_DIR ${MKLDNN_INSTALL}/include) + +# patch for mkldnn_sgemm thread safety bug. +# it can be removed once a fix is available in a validated mkldnn release version. +set(MKLDNN_PATCH_COMMAND1 git apply ${CMAKE_SOURCE_DIR}/patches/mkldnn/mkldnn_sgemm.patch) + +if(WIN32) + set(MKLDNN_SHARED_LIB mkldnn.dll) + set(MKLDNN_IMPORT_LIB mkldnn.lib) + if(onnxruntime_USE_MKLML) + set(DOWNLOAD_MKLML ${MKLDNN_SOURCE}/scripts/prepare_mkl.bat) + set(MKLML_SHARED_LIB mklml.dll) + set(IOMP5MD_SHARED_LIB libiomp5md.dll) + endif() + set(MKLDNN_PATCH_COMMAND2 "") +else() + set(MKLDNN_SHARED_LIB libmkldnn.so.0) + if(onnxruntime_USE_MKLML) + set(DOWNLOAD_MKLML ${MKLDNN_SOURCE}/scripts/prepare_mkl.sh) + set(MKLML_SHARED_LIB libmklml_intel.so) + set(IOMP5MD_SHARED_LIB libiomp5.so) + endif() + set(MKLDNN_PATCH_COMMAND2 git apply ${CMAKE_SOURCE_DIR}/patches/mkldnn/platform.cmake.patch) +endif() + +if(NOT onnxruntime_USE_MKLDNN OR EXISTS ${MKLDNN_SOURCE}/external) + set(DOWNLOAD_MKLML "") +endif() + +ExternalProject_Add(project_mkldnn + PREFIX mkl-dnn + GIT_REPOSITORY ${MKLDNN_URL} + GIT_TAG ${MKLDNN_TAG} + PATCH_COMMAND ${DOWNLOAD_MKLML} COMMAND ${MKLDNN_PATCH_COMMAND1} COMMAND ${MKLDNN_PATCH_COMMAND2} + SOURCE_DIR ${MKLDNN_SOURCE} + CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${MKLDNN_INSTALL} +) + +if(WIN32) + add_library(mkldnn STATIC IMPORTED) + set_property(TARGET mkldnn PROPERTY IMPORTED_LOCATION ${MKLDNN_LIB_DIR}/${MKLDNN_IMPORT_LIB}) +else() + add_library(mkldnn SHARED IMPORTED) + set_property(TARGET mkldnn PROPERTY IMPORTED_LOCATION ${MKLDNN_LIB_DIR}/${MKLDNN_SHARED_LIB}) +endif() +add_dependencies(mkldnn project_mkldnn) +include_directories(${MKLDNN_INCLUDE_DIR}) diff --git a/cmake/external/onnx b/cmake/external/onnx new file mode 160000 index 0000000000000..eb4b7c2cc2a0d --- /dev/null +++ b/cmake/external/onnx @@ -0,0 +1 @@ +Subproject commit eb4b7c2cc2a0d34c0127e26c2c1cb5e712467e1e diff --git a/cmake/external/protobuf b/cmake/external/protobuf new file mode 160000 index 0000000000000..48cb18e5c419d --- /dev/null +++ b/cmake/external/protobuf @@ -0,0 +1 @@ +Subproject commit 48cb18e5c419ddd23d9badcfe4e9df7bde1979b2 diff --git a/cmake/external/pybind11.cmake b/cmake/external/pybind11.cmake new file mode 100644 index 0000000000000..329cd60334df5 --- /dev/null +++ b/cmake/external/pybind11.cmake @@ -0,0 +1,14 @@ +include(ExternalProject) + +set(pybind11_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/pybind11/src/pybind11/include) +set(pybind11_URL https://github.com/pybind/pybind11.git) +set(pybind11_TAG v2.2.4) + +ExternalProject_Add(pybind11 + PREFIX pybind11 + GIT_REPOSITORY ${pybind11_URL} + GIT_TAG ${pybind11_TAG} + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" +) diff --git a/cmake/external/tvm b/cmake/external/tvm new file mode 160000 index 0000000000000..c2b3615477850 --- /dev/null +++ b/cmake/external/tvm @@ -0,0 +1 @@ +Subproject commit c2b36154778503a509a70a3b5309b201969eccab diff --git a/cmake/external/zlib.cmake b/cmake/external/zlib.cmake new file mode 100644 index 0000000000000..02014170157f0 --- /dev/null +++ b/cmake/external/zlib.cmake @@ -0,0 +1,47 @@ +include (ExternalProject) + +set(zlib_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/zlib_archive) +set(ZLIB_URL https://github.com/madler/zlib) +set(ZLIB_BUILD ${CMAKE_CURRENT_BINARY_DIR}/zlib/src/zlib) +set(ZLIB_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/zlib/install) +set(ZLIB_TAG 50893291621658f355bc5b4d450a8d06a563053d) + +if(WIN32) + set(zlib_STATIC_LIBRARIES + debug ${CMAKE_CURRENT_BINARY_DIR}/zlib/install/lib/zlibstaticd.lib + optimized ${CMAKE_CURRENT_BINARY_DIR}/zlib/install/lib/zlibstatic.lib) +else() + set(zlib_STATIC_LIBRARIES + ${CMAKE_CURRENT_BINARY_DIR}/zlib/install/lib/libz.a) +endif() + +set(ZLIB_HEADERS + "${ZLIB_INSTALL}/include/zconf.h" + "${ZLIB_INSTALL}/include/zlib.h" +) + +ExternalProject_Add(zlib + PREFIX zlib + GIT_REPOSITORY ${ZLIB_URL} + GIT_TAG ${ZLIB_TAG} + INSTALL_DIR ${ZLIB_INSTALL} + BUILD_IN_SOURCE 1 + DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" + CMAKE_CACHE_ARGS + -DCMAKE_BUILD_TYPE:STRING=Release + -DCMAKE_INSTALL_PREFIX:STRING=${ZLIB_INSTALL} + -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON +) + +# put zlib includes in the directory where they are expected +add_custom_target(zlib_create_destination_dir + COMMAND ${CMAKE_COMMAND} -E make_directory ${zlib_INCLUDE_DIR} + DEPENDS zlib) + +add_custom_target(zlib_copy_headers_to_destination + DEPENDS zlib_create_destination_dir) + +foreach(header_file ${ZLIB_HEADERS}) + add_custom_command(TARGET zlib_copy_headers_to_destination PRE_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${header_file} ${zlib_INCLUDE_DIR}) +endforeach() diff --git a/cmake/onnx/CMakeLists.txt b/cmake/onnx/CMakeLists.txt new file mode 100644 index 0000000000000..b34aa9b2fc4ef --- /dev/null +++ b/cmake/onnx/CMakeLists.txt @@ -0,0 +1,51 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +add_library(onnx_proto ${ONNXRUNTIME_ROOT}/core/protobuf/onnx-ml.proto ${ONNXRUNTIME_ROOT}/core/protobuf/onnx-operators-ml.proto) +target_include_directories(onnx_proto PUBLIC $ "${CMAKE_CURRENT_BINARY_DIR}/..") +onnxruntime_protobuf_generate(APPEND_PATH IMPORT_DIRS TARGET onnx_proto) + +# Cpp Tests were added and they require googletest +# since we have our own copy, try using that +set(ONNX_SOURCE_ROOT ${PROJECT_SOURCE_DIR}/external/onnx) +file(GLOB_RECURSE onnx_src + "${ONNX_SOURCE_ROOT}/onnx/*.h" + "${ONNX_SOURCE_ROOT}/onnx/*.cc" +) + +file(GLOB_RECURSE onnx_exclude_src + "${ONNX_SOURCE_ROOT}/onnx/py_utils.h" + "${ONNX_SOURCE_ROOT}/onnx/proto_utils.h" + "${ONNX_SOURCE_ROOT}/onnx/backend/test/cpp/*" + "${ONNX_SOURCE_ROOT}/onnx/test/*" + "${ONNX_SOURCE_ROOT}/onnx/cpp2py_export.cc" +) + +list(REMOVE_ITEM onnx_src ${onnx_exclude_src}) +add_library(onnx ${onnx_src}) +add_dependencies(onnx onnx_proto) +target_include_directories(onnx PUBLIC $ "${ONNX_SOURCE_ROOT}") +target_compile_definitions(onnx PUBLIC "ONNX_ML" "ONNX_NAMESPACE=onnx") +if (WIN32) + target_compile_options(onnx PRIVATE + /wd4800 # 'type' : forcing value to bool 'true' or 'false' (performance warning) + /wd4125 # decimal digit terminates octal escape sequence + /wd4100 # 'param' : unreferenced formal parameter + /wd4244 # 'argument' conversion from 'google::protobuf::int64' to 'int', possible loss of data + /EHsc # exception handling - C++ may throw, extern "C" will not + ) + set(onnx_static_library_flags + -IGNORE:4221 # LNK4221: This object file does not define any previously undefined public symbols, so it will not be used by any link operation that consumes this library + ) + set_target_properties(onnx PROPERTIES + STATIC_LIBRARY_FLAGS "${onnx_static_library_flags}") +else() + if(HAS_UNUSED_PARAMETER) + target_compile_options(onnx PRIVATE "-Wno-unused-parameter") + target_compile_options(onnx_proto PRIVATE "-Wno-unused-parameter") + endif() + if(HAS_UNUSED_BUT_SET_VARIABLE) + target_compile_options(onnx PRIVATE "-Wno-unused-but-set-variable") + target_compile_options(onnx_proto PRIVATE "-Wno-unused-but-set-variable") + endif() +endif() diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake new file mode 100644 index 0000000000000..901052ca7c451 --- /dev/null +++ b/cmake/onnxruntime.cmake @@ -0,0 +1,64 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +if(UNIX) + set(SYMBOL_FILE ${CMAKE_CURRENT_BINARY_DIR}/onnxruntime.lds) + set(OUTPUT_STYLE gcc) +else() + set(SYMBOL_FILE ${CMAKE_CURRENT_BINARY_DIR}/onnxruntime_dll.def) + set(OUTPUT_STYLE vc) +endif() + + +list(APPEND SYMBOL_FILES "${REPO_ROOT}/tools/ci_build/gen_def.py") +foreach(f ${ONNXRUNTIME_PROVIDER_NAMES}) + list(APPEND SYMBOL_FILES "${ONNXRUNTIME_ROOT}/core/providers/${f}/symbols.txt") +endforeach() + +add_custom_command(OUTPUT ${SYMBOL_FILE} + COMMAND ${PYTHON_EXECUTABLE} "${REPO_ROOT}/tools/ci_build/gen_def.py" --version_file "${ONNXRUNTIME_ROOT}/../VERSION_NUMBER" --src_root "${ONNXRUNTIME_ROOT}" --config ${ONNXRUNTIME_PROVIDER_NAMES} --style=${OUTPUT_STYLE} --output ${SYMBOL_FILE} + DEPENDS ${SYMBOL_FILES} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + +add_custom_target(onnxruntime_generate_def ALL DEPENDS ${SYMBOL_FILE}) +add_library(onnxruntime SHARED ${onnxruntime_session_srcs}) +set_target_properties(onnxruntime PROPERTIES VERSION ${VERSION_NUMBER}) +add_dependencies(onnxruntime onnxruntime_generate_def ${onnxruntime_EXTERNAL_DEPENDENCIES}) +target_include_directories(onnxruntime PRIVATE ${ONNXRUNTIME_ROOT} ${date_INCLUDE_DIR}) + +if(UNIX) + set(BEGIN_WHOLE_ARCHIVE -Xlinker --whole-archive) + set(END_WHOLE_ARCHIVE -Xlinker --no-whole-archive) + set(ONNXRUNTIME_SO_LINK_FLAG "-Xlinker --version-script=${SYMBOL_FILE} -Xlinker --no-undefined") +else() + set(ONNXRUNTIME_SO_LINK_FLAG "-DEF:${SYMBOL_FILE}") +endif() + +target_link_libraries(onnxruntime PRIVATE + ${BEGIN_WHOLE_ARCHIVE} + ${onnxruntime_libs} + ${PROVIDERS_CUDA} + ${PROVIDERS_MKLDNN} + onnxruntime_providers + onnxruntime_util + onnxruntime_framework + ${END_WHOLE_ARCHIVE} + onnxruntime_graph + onnxruntime_common + onnx + onnx_proto + onnxruntime_mlas + ${onnxruntime_tvm_libs} + ${onnxruntime_EXTERNAL_LIBRARIES} + ${CMAKE_THREAD_LIBS_INIT} + ${ONNXRUNTIME_CUDA_LIBRARIES}) + +set_property(TARGET onnxruntime APPEND_STRING PROPERTY LINK_FLAGS ${ONNXRUNTIME_SO_LINK_FLAG}) +set_target_properties(onnxruntime PROPERTIES LINK_DEPENDS ${SYMBOL_FILE}) + +install(TARGETS onnxruntime + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + +set_target_properties(onnxruntime PROPERTIES FOLDER "ONNXRuntime") diff --git a/cmake/onnxruntime_codegen.cmake b/cmake/onnxruntime_codegen.cmake new file mode 100644 index 0000000000000..f9185787388aa --- /dev/null +++ b/cmake/onnxruntime_codegen.cmake @@ -0,0 +1,16 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +file(GLOB_RECURSE onnxruntime_codegen_tvm_srcs + "${ONNXRUNTIME_ROOT}/core/codegen/tvm/*.h" + "${ONNXRUNTIME_ROOT}/core/codegen/tvm/*.cc" +) + +add_library(onnxruntime_codegen_tvm ${onnxruntime_codegen_tvm_srcs}) +set_target_properties(onnxruntime_codegen_tvm PROPERTIES FOLDER "ONNXRuntime") +target_include_directories(onnxruntime_codegen_tvm PRIVATE ${ONNXRUNTIME_ROOT} ${TVM_INCLUDES}) +onnxruntime_add_include_to_target(onnxruntime_codegen_tvm onnx protobuf::libprotobuf) +target_compile_options(onnxruntime_codegen_tvm PRIVATE ${DISABLED_WARNINGS_FOR_TVM}) +# need onnx to build to create headers that this project includes +add_dependencies(onnxruntime_codegen_tvm onnxruntime_framework tvm ${onnxruntime_EXTERNAL_DEPENDENCIES}) + diff --git a/cmake/onnxruntime_common.cmake b/cmake/onnxruntime_common.cmake new file mode 100644 index 0000000000000..eed99f87ef78c --- /dev/null +++ b/cmake/onnxruntime_common.cmake @@ -0,0 +1,54 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +set(onnxruntime_common_src_patterns + "${ONNXRUNTIME_INCLUDE_DIR}/core/common/*.h" + "${ONNXRUNTIME_INCLUDE_DIR}/core/common/logging/*.h" + "${ONNXRUNTIME_ROOT}/core/common/*.h" + "${ONNXRUNTIME_ROOT}/core/common/*.cc" + "${ONNXRUNTIME_ROOT}/core/common/logging/*.h" + "${ONNXRUNTIME_ROOT}/core/common/logging/*.cc" + "${ONNXRUNTIME_ROOT}/core/common/logging/sinks/*.h" + "${ONNXRUNTIME_ROOT}/core/common/logging/sinks/*.cc" + "${ONNXRUNTIME_ROOT}/core/inc/*.h" + "${ONNXRUNTIME_ROOT}/core/platform/env.h" + "${ONNXRUNTIME_ROOT}/core/platform/env.cc" + "${ONNXRUNTIME_ROOT}/core/platform/env_time.h" + "${ONNXRUNTIME_ROOT}/core/platform/env_time.cc" +) + +if(WIN32) + list(APPEND onnxruntime_common_src_patterns + "${ONNXRUNTIME_ROOT}/core/platform/windows/*.h" + "${ONNXRUNTIME_ROOT}/core/platform/windows/*.cc" + "${ONNXRUNTIME_ROOT}/core/platform/windows/logging/*.h" + "${ONNXRUNTIME_ROOT}/core/platform/windows/logging/*.cc" + ) +else() + list(APPEND onnxruntime_common_src_patterns + "${ONNXRUNTIME_ROOT}/core/platform/posix/*.h" + "${ONNXRUNTIME_ROOT}/core/platform/posix/*.cc" + ) +endif() + +file(GLOB onnxruntime_common_src ${onnxruntime_common_src_patterns}) + +source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_common_src}) + +add_library(onnxruntime_common ${onnxruntime_common_src}) + +if(NOT WIN32) + target_link_libraries(onnxruntime_common dl) +endif() +target_include_directories(onnxruntime_common PRIVATE ${ONNXRUNTIME_ROOT} ${date_INCLUDE_DIR}) +# logging uses date. threadpool uses eigen +add_dependencies(onnxruntime_common date eigen gsl) + +install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/common DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/common) +set_target_properties(onnxruntime_common PROPERTIES LINKER_LANGUAGE CXX) +set_target_properties(onnxruntime_common PROPERTIES FOLDER "ONNXRuntime") + +if(WIN32) + # Add Code Analysis properties to enable C++ Core checks. Have to do it via a props file include. + set_target_properties(onnxruntime_common PROPERTIES VS_USER_PROPS ${PROJECT_SOURCE_DIR}/EnableVisualStudioCodeAnalysis.props) +endif() diff --git a/cmake/onnxruntime_config.h.in b/cmake/onnxruntime_config.h.in new file mode 100644 index 0000000000000..7dd3c034fb9cd --- /dev/null +++ b/cmake/onnxruntime_config.h.in @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#cmakedefine HAS_UNUSED_BUT_SET_VARIABLE +#cmakedefine HAS_UNUSED_PARAMETER +#cmakedefine HAS_CAST_FUNCTION_TYPE +#cmakedefine HAS_PARENTHESES +#cmakedefine HAS_NULL_DEREFERENCE +#cmakedefine HAS_USELESS_CAST + + diff --git a/cmake/onnxruntime_csharp.cmake b/cmake/onnxruntime_csharp.cmake new file mode 100644 index 0000000000000..ddd9628ed2d5f --- /dev/null +++ b/cmake/onnxruntime_csharp.cmake @@ -0,0 +1,12 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +set (CSHARP_ROOT ${PROJECT_SOURCE_DIR}/../csharp) +set (CSHARP_MASTER_TARGET OnnxRuntime.CSharp) +set (CSHARP_MASTER_PROJECT ${CSHARP_ROOT}/OnnxRuntime.CSharp.proj ) +include(CSharpUtilities) + +include_external_msproject(${CSHARP_MASTER_TARGET} + ${CSHARP_MASTER_PROJECT} + onnxruntime # make it depend on the native onnxruntime project + ) diff --git a/cmake/onnxruntime_dependencies.dot b/cmake/onnxruntime_dependencies.dot new file mode 100644 index 0000000000000..7f1f5d9af5c41 --- /dev/null +++ b/cmake/onnxruntime_dependencies.dot @@ -0,0 +1,21 @@ +digraph "GG" { +node [ + fontsize = "12" +]; + "node12" [ label="onnxruntime_graph" shape="diamond"]; + "node10" [ label="onnxruntime_common" shape="diamond"]; + "node12" -> "node10" // onnxruntime_graph -> onnxruntime_common + "node4" [ label="onnx" shape="diamond"]; + "node12" -> "node4" // onnxruntime_graph -> onnx + "node15" [ label="onnxruntime_framework" shape="diamond"]; + "node15" -> "node12" // onnxruntime_framework -> onnxruntime_graph + "node15" -> "node10" // onnxruntime_framework -> onnxruntime_common + "node15" -> "node4" // onnxruntime_framework -> onnx + "node17" [ label="onnxruntime_providers" shape="diamond"]; + "node17" -> "node10" // onnxruntime_providers -> onnxruntime_common + "node17" -> "node15" // onnxruntime_providers -> onnxruntime_framework + "node18" [ label="onnxruntime_test_common" shape="house"]; + "node6" [ label="onnxruntime_test_framework" shape="house"]; + "node19" [ label="onnxruntime_test_ir" shape="house"]; + "node20" [ label="onnxruntime_test_providers" shape="house"]; +} diff --git a/cmake/onnxruntime_framework.cmake b/cmake/onnxruntime_framework.cmake new file mode 100644 index 0000000000000..b74cdc6a88d4f --- /dev/null +++ b/cmake/onnxruntime_framework.cmake @@ -0,0 +1,25 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +file(GLOB_RECURSE onnxruntime_framework_srcs + "${ONNXRUNTIME_INCLUDE_DIR}/core/framework/*.h" + "${ONNXRUNTIME_ROOT}/core/framework/*.h" + "${ONNXRUNTIME_ROOT}/core/framework/*.cc" +) + +source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_framework_srcs}) + +add_library(onnxruntime_framework ${onnxruntime_framework_srcs}) + +#TODO: remove ${eigen_INCLUDE_DIRS} from here +target_include_directories(onnxruntime_framework PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS}) +onnxruntime_add_include_to_target(onnxruntime_framework onnx protobuf::libprotobuf) +set_target_properties(onnxruntime_framework PROPERTIES FOLDER "ONNXRuntime") +# need onnx to build to create headers that this project includes +add_dependencies(onnxruntime_framework ${onnxruntime_EXTERNAL_DEPENDENCIES}) + +install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/framework DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/framework) +if (WIN32) + # Add Code Analysis properties to enable C++ Core checks. Have to do it via a props file include. + set_target_properties(onnxruntime_framework PROPERTIES VS_USER_PROPS ${PROJECT_SOURCE_DIR}/ConfigureVisualStudioCodeAnalysis.props) +endif() diff --git a/cmake/onnxruntime_graph.cmake b/cmake/onnxruntime_graph.cmake new file mode 100644 index 0000000000000..d83c07ca044f0 --- /dev/null +++ b/cmake/onnxruntime_graph.cmake @@ -0,0 +1,37 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +file(GLOB_RECURSE onnxruntime_graph_src + "${ONNXRUNTIME_INCLUDE_DIR}/core/graph/*.h" + "${ONNXRUNTIME_ROOT}/core/graph/*.h" + "${ONNXRUNTIME_ROOT}/core/graph/*.cc" +) + +file(GLOB_RECURSE onnxruntime_ir_defs_src + "${ONNXRUNTIME_ROOT}/core/defs/*.cc" +) + +add_library(onnxruntime_graph ${onnxruntime_graph_src} ${onnxruntime_ir_defs_src}) +add_dependencies(onnxruntime_graph onnx_proto gsl) +onnxruntime_add_include_to_target(onnxruntime_graph onnx protobuf::libprotobuf) +target_include_directories(onnxruntime_graph PRIVATE ${ONNXRUNTIME_ROOT}) +set_target_properties(onnxruntime_graph PROPERTIES FOLDER "ONNXRuntime") +set_target_properties(onnxruntime_graph PROPERTIES LINKER_LANGUAGE CXX) +install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/graph DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/graph) +source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_graph_src} ${onnxruntime_ir_defs_src}) + +if (WIN32) + set(onnxruntime_graph_static_library_flags + -IGNORE:4221 # LNK4221: This object file does not define any previously undefined public symbols, so it will not be used by any link operation that consumes this library + ) + + set_target_properties(onnxruntime_graph PROPERTIES + STATIC_LIBRARY_FLAGS "${onnxruntime_graph_static_library_flags}") + + target_compile_options(onnxruntime_graph PRIVATE + /EHsc # exception handling - C++ may throw, extern "C" will not + ) + + # Add Code Analysis properties to enable C++ Core checks. Have to do it via a props file include. + set_target_properties(onnxruntime_graph PROPERTIES VS_USER_PROPS ${PROJECT_SOURCE_DIR}/EnableVisualStudioCodeAnalysis.props) +endif() diff --git a/cmake/onnxruntime_mlas.cmake b/cmake/onnxruntime_mlas.cmake new file mode 100644 index 0000000000000..8285ef7952dad --- /dev/null +++ b/cmake/onnxruntime_mlas.cmake @@ -0,0 +1,137 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +set(mlas_common_srcs + ${ONNXRUNTIME_ROOT}/core/mlas/lib/platform.cpp + ${ONNXRUNTIME_ROOT}/core/mlas/lib/sgemm.cpp + ${ONNXRUNTIME_ROOT}/core/mlas/lib/convolve.cpp + ${ONNXRUNTIME_ROOT}/core/mlas/lib/pooling.cpp + ${ONNXRUNTIME_ROOT}/core/mlas/lib/bias.cpp +) + +if (MSVC) + + if (CMAKE_GENERATOR_PLATFORM STREQUAL "ARM") + + set(mlas_platform_srcs + ${ONNXRUNTIME_ROOT}/core/mlas/lib/arm/sgemmc.cpp + ) + + elseif (CMAKE_GENERATOR_PLATFORM STREQUAL "ARM64") + + set(asm_filename ${ONNXRUNTIME_ROOT}/core/mlas/lib/arm64/sgemma.asm) + set(pre_filename ${CMAKE_CURRENT_BINARY_DIR}/sgemma.i) + set(obj_filename ${CMAKE_CURRENT_BINARY_DIR}/sgemma.obj) + + if (CMAKE_BUILD_TYPE STREQUAL "Debug") + set(ARMASM_FLAGS "-g") + else() + set(ARMASM_FLAGS "") + endif() + + add_custom_command( + OUTPUT ${obj_filename} + COMMAND + cl.exe /P ${asm_filename} + COMMAND + armasm64.exe ${ARMASM_FLAGS} ${pre_filename} ${obj_filename} + ) + + set(mlas_platform_srcs ${obj_filename}) + + elseif (CMAKE_GENERATOR_PLATFORM STREQUAL "Win32") + + enable_language(ASM_MASM) + + set(CMAKE_ASM_MASM_FLAGS "${CMAKE_ASM_MASM_FLAGS} /safeseh") + + set(mlas_platform_srcs + ${ONNXRUNTIME_ROOT}/core/mlas/lib/i386/sgemma.asm + ) + + elseif (CMAKE_GENERATOR_PLATFORM STREQUAL "x64") + + enable_language(ASM_MASM) + + set(mlas_platform_srcs + ${ONNXRUNTIME_ROOT}/core/mlas/lib/amd64/SgemmKernelSse2.asm + ${ONNXRUNTIME_ROOT}/core/mlas/lib/amd64/SgemmKernelAvx.asm + ${ONNXRUNTIME_ROOT}/core/mlas/lib/amd64/SgemmKernelFma3.asm + ${ONNXRUNTIME_ROOT}/core/mlas/lib/amd64/SgemmKernelAvx512F.asm + ${ONNXRUNTIME_ROOT}/core/mlas/lib/amd64/sgemma.asm + ${ONNXRUNTIME_ROOT}/core/mlas/lib/amd64/cvtfp16a.asm + ) + + endif() + +else() + + execute_process( + COMMAND ${CMAKE_C_COMPILER} -dumpmachine + OUTPUT_VARIABLE dumpmachine_output + ERROR_QUIET + ) + + if (dumpmachine_output MATCHES "^arm.*") + + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mfpu=neon") + + set(mlas_platform_srcs + ${ONNXRUNTIME_ROOT}/core/mlas/lib/arm/sgemmc.cpp + ) + + elseif (dumpmachine_output MATCHES "^aarch64.*") + + enable_language(ASM) + + set(mlas_platform_srcs + ${ONNXRUNTIME_ROOT}/core/mlas/lib/aarch64/sgemma.s + ) + + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + + enable_language(ASM) + + # The LLVM assmebler does not support the .arch directive to enable instruction + # set extensions and also doesn't support AVX-512F instructions without + # turning on support via command-line option. Group the sources by the + # instruction set extension and explicitly set the compiler flag as appropriate. + + set(mlas_platform_srcs_sse2 + ${ONNXRUNTIME_ROOT}/core/mlas/lib/x86_64/SgemmKernelSse2.S + ${ONNXRUNTIME_ROOT}/core/mlas/lib/x86_64/SgemmTransposePackB16x4Sse2.S + ) + set_source_files_properties(${mlas_platform_srcs_sse} PROPERTIES COMPILE_FLAGS "-msse2") + + set(mlas_platform_srcs_avx + ${ONNXRUNTIME_ROOT}/core/mlas/lib/x86_64/SgemmKernelAvx.S + ${ONNXRUNTIME_ROOT}/core/mlas/lib/x86_64/SgemmKernelM1Avx.S + ${ONNXRUNTIME_ROOT}/core/mlas/lib/x86_64/SgemmKernelM1TransposeBAvx.S + ${ONNXRUNTIME_ROOT}/core/mlas/lib/x86_64/SgemmTransposePackB16x4Avx.S + ) + set_source_files_properties(${mlas_platform_srcs_avx} PROPERTIES COMPILE_FLAGS "-mavx") + + set(mlas_platform_srcs_avx2 + ${ONNXRUNTIME_ROOT}/core/mlas/lib/x86_64/SgemmKernelFma3.S + ) + set_source_files_properties(${mlas_platform_srcs_avx2} PROPERTIES COMPILE_FLAGS "-mavx2 -mfma") + + set(mlas_platform_srcs_avx512f + ${ONNXRUNTIME_ROOT}/core/mlas/lib/x86_64/SgemmKernelAvx512F.S + ) + set_source_files_properties(${mlas_platform_srcs_avx512f} PROPERTIES COMPILE_FLAGS "-mavx512f") + + set(mlas_platform_srcs + ${mlas_platform_srcs_sse2} + ${mlas_platform_srcs_avx} + ${mlas_platform_srcs_avx2} + ${mlas_platform_srcs_avx512f} + ) + + endif() + +endif() + +add_library(onnxruntime_mlas STATIC ${mlas_common_srcs} ${mlas_platform_srcs}) +target_include_directories(onnxruntime_mlas PRIVATE ${ONNXRUNTIME_ROOT}/core/mlas/inc ${ONNXRUNTIME_ROOT}/core/mlas/lib) +set_target_properties(onnxruntime_mlas PROPERTIES FOLDER "ONNXRuntime") diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake new file mode 100644 index 0000000000000..379738325a7f8 --- /dev/null +++ b/cmake/onnxruntime_providers.cmake @@ -0,0 +1,104 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +file(GLOB_RECURSE onnxruntime_providers_srcs + "${ONNXRUNTIME_ROOT}/core/providers/cpu/*.h" + "${ONNXRUNTIME_ROOT}/core/providers/cpu/*.cc" +) + +file(GLOB_RECURSE onnxruntime_contrib_ops_srcs + "${ONNXRUNTIME_ROOT}/contrib_ops/*.h" + "${ONNXRUNTIME_ROOT}/contrib_ops/*.cc" + "${ONNXRUNTIME_ROOT}/contrib_ops/cpu/*.h" + "${ONNXRUNTIME_ROOT}/contrib_ops/cpu/*.cc" +) + +file(GLOB onnxruntime_providers_common_srcs + "${ONNXRUNTIME_ROOT}/core/providers/*.h" + "${ONNXRUNTIME_ROOT}/core/providers/*.cc" +) + +if(onnxruntime_USE_MKLDNN) + set(PROVIDERS_MKLDNN onnxruntime_providers_mkldnn) + list(APPEND ONNXRUNTIME_PROVIDER_NAMES mkldnn) +endif() +if(onnxruntime_USE_CUDA) + set(PROVIDERS_CUDA onnxruntime_providers_cuda) + list(APPEND ONNXRUNTIME_PROVIDER_NAMES cuda) +endif() + +source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_common_srcs} ${onnxruntime_providers_srcs}) +# add using ONNXRUNTIME_ROOT so they show up under the 'contrib_ops' folder in Visual Studio +source_group(TREE ${ONNXRUNTIME_ROOT} FILES ${onnxruntime_contrib_ops_srcs}) +add_library(onnxruntime_providers ${onnxruntime_providers_common_srcs} ${onnxruntime_providers_srcs} ${onnxruntime_contrib_ops_srcs}) +onnxruntime_add_include_to_target(onnxruntime_providers onnx protobuf::libprotobuf) +target_include_directories(onnxruntime_providers PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS}) +add_dependencies(onnxruntime_providers eigen gsl onnx) +install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/cpu DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers/cpu) +set_target_properties(onnxruntime_providers PROPERTIES LINKER_LANGUAGE CXX) +set_target_properties(onnxruntime_providers PROPERTIES FOLDER "ONNXRuntime") + +if (WIN32 AND onnxruntime_USE_OPENMP) + if (${CMAKE_CXX_COMPILER_ID} STREQUAL MSVC) + add_definitions(/openmp) + endif() +endif() + +if (onnxruntime_USE_CUDA) + file(GLOB_RECURSE onnxruntime_providers_cuda_cc_srcs + "${ONNXRUNTIME_ROOT}/core/providers/cuda/*.h" + "${ONNXRUNTIME_ROOT}/core/providers/cuda/*.cc" + ) + file(GLOB_RECURSE onnxruntime_providers_cuda_cu_srcs + "${ONNXRUNTIME_ROOT}/core/providers/cuda/*.cu" + "${ONNXRUNTIME_ROOT}/core/providers/cuda/*.cuh" + ) + source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_cuda_cc_srcs} ${onnxruntime_providers_cuda_cu_srcs}) + if(NOT WIN32) + set(CUDA_CFLAGS " -std=c++14") + endif() + cuda_add_library(onnxruntime_providers_cuda ${onnxruntime_providers_cuda_cc_srcs} ${onnxruntime_providers_cuda_cu_srcs} OPTIONS ${CUDA_CFLAGS}) + onnxruntime_add_include_to_target(onnxruntime_providers_cuda onnx protobuf::libprotobuf) + add_dependencies(onnxruntime_providers_cuda eigen ${onnxruntime_EXTERNAL_DEPENDENCIES} ${onnxruntime_tvm_dependencies}) + target_include_directories(onnxruntime_providers_cuda PRIVATE ${ONNXRUNTIME_ROOT} ${onnxruntime_CUDNN_HOME}/include ${eigen_INCLUDE_DIRS} ${TVM_INCLUDES}) + install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/cuda DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers/cuda) + set_target_properties(onnxruntime_providers_cuda PROPERTIES LINKER_LANGUAGE CUDA) + set_target_properties(onnxruntime_providers_cuda PROPERTIES FOLDER "ONNXRuntime") + if (WIN32) + # *.cu cannot use PCH + foreach(src_file ${onnxruntime_providers_cuda_cc_srcs}) + set_source_files_properties(${src_file} + PROPERTIES + COMPILE_FLAGS "/Yucuda_pch.h /FIcuda_pch.h") + endforeach() + set_source_files_properties("${ONNXRUNTIME_ROOT}/core/providers/cuda/cuda_pch.cc" + PROPERTIES + COMPILE_FLAGS "/Yccuda_pch.h" + ) + # disable a warning from the CUDA headers about unreferenced local functions + target_compile_options(onnxruntime_providers_cuda PRIVATE /wd4505) + if (onnxruntime_USE_TVM) + target_compile_options(onnxruntime_providers_cuda PRIVATE ${DISABLED_WARNINGS_FOR_TVM}) + endif() + endif() +endif() + +if (onnxruntime_USE_MKLDNN) + file(GLOB_RECURSE onnxruntime_providers_mkldnn_cc_srcs + "${ONNXRUNTIME_ROOT}/core/providers/mkldnn/*.h" + "${ONNXRUNTIME_ROOT}/core/providers/mkldnn/*.cc" + ) + + source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_mkldnn_cc_srcs}) + add_library(onnxruntime_providers_mkldnn ${onnxruntime_providers_mkldnn_cc_srcs}) + onnxruntime_add_include_to_target(onnxruntime_providers_mkldnn onnx protobuf::libprotobuf) + add_dependencies(onnxruntime_providers_mkldnn eigen ${onnxruntime_EXTERNAL_DEPENDENCIES}) + set_target_properties(onnxruntime_providers_mkldnn PROPERTIES FOLDER "ONNXRuntime") + target_include_directories(onnxruntime_providers_mkldnn PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS}) + install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/mkldnn DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers/mkldnn) + set_target_properties(onnxruntime_providers_mkldnn PROPERTIES LINKER_LANGUAGE CXX) +endif() + +if (onnxruntime_ENABLE_MICROSOFT_INTERNAL) + include(onnxruntime_providers_internal.cmake) +endif() diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake new file mode 100644 index 0000000000000..8f2f55c69c222 --- /dev/null +++ b/cmake/onnxruntime_python.cmake @@ -0,0 +1,185 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +include(pybind11) +FIND_PACKAGE(NumPy) + +if(NOT PYTHON_INCLUDE_DIR) + set(PYTHON_NOT_FOUND false) + exec_program("${PYTHON_EXECUTABLE}" + ARGS "-c \"import distutils.sysconfig; print(distutils.sysconfig.get_python_inc())\"" + OUTPUT_VARIABLE PYTHON_INCLUDE_DIR + RETURN_VALUE PYTHON_NOT_FOUND) + if(${PYTHON_NOT_FOUND}) + message(FATAL_ERROR + "Cannot get Python include directory. Is distutils installed?") + endif(${PYTHON_NOT_FOUND}) +endif(NOT PYTHON_INCLUDE_DIR) + +# 2. Resolve the installed version of NumPy (for numpy/arrayobject.h). +if(NOT NUMPY_INCLUDE_DIR) + set(NUMPY_NOT_FOUND false) + exec_program("${PYTHON_EXECUTABLE}" + ARGS "-c \"import numpy; print(numpy.get_include())\"" + OUTPUT_VARIABLE NUMPY_INCLUDE_DIR + RETURN_VALUE NUMPY_NOT_FOUND) + if(${NUMPY_NOT_FOUND}) + message(FATAL_ERROR + "Cannot get NumPy include directory: Is NumPy installed?") + endif(${NUMPY_NOT_FOUND}) +endif(NOT NUMPY_INCLUDE_DIR) + +# ---[ Python + Numpy +set(onnxruntime_pybind_srcs_pattern + "${ONNXRUNTIME_ROOT}/python/*.cc" + "${ONNXRUNTIME_ROOT}/python/*.h" +) + +file(GLOB onnxruntime_pybind_srcs ${onnxruntime_pybind_srcs_pattern}) + +#TODO(): enable cuda and test it +add_library(onnxruntime_pybind11_state MODULE ${onnxruntime_pybind_srcs}) +if(HAS_CAST_FUNCTION_TYPE) +target_compile_options(onnxruntime_pybind11_state PRIVATE "-Wno-cast-function-type") +endif() +target_include_directories(onnxruntime_pybind11_state PRIVATE ${ONNXRUNTIME_ROOT} ${PYTHON_INCLUDE_DIR} ${NUMPY_INCLUDE_DIR}) +target_include_directories(onnxruntime_pybind11_state PRIVATE ${pybind11_INCLUDE_DIRS}) +if(APPLE) + set(ONNXRUNTIME_SO_LINK_FLAG "-Xlinker -exported_symbols_list ${ONNXRUNTIME_ROOT}/python/exported_symbols.lst") +elseif(UNIX) + set(ONNXRUNTIME_SO_LINK_FLAG "-Xlinker --version-script=${ONNXRUNTIME_ROOT}/python/version_script.lds -Xlinker --no-undefined") +endif() + +set(onnxruntime_pybind11_state_libs + onnxruntime_session + ${onnxruntime_libs} + ${PROVIDERS_CUDA} + ${PROVIDERS_MKLDNN} + onnxruntime_providers + onnxruntime_framework + onnxruntime_util + onnxruntime_graph + onnx + onnx_proto + onnxruntime_common + onnxruntime_mlas + ${onnxruntime_tvm_libs} +) + +set(onnxruntime_pybind11_state_dependencies + ${onnxruntime_EXTERNAL_DEPENDENCIES} + pybind11 +) + +add_dependencies(onnxruntime_pybind11_state ${onnxruntime_pybind11_state_dependencies}) +if (MSVC) + # if MSVC, pybind11 looks for release version of python lib (pybind11/detail/common.h undefs _DEBUG) + target_link_libraries(onnxruntime_pybind11_state ${onnxruntime_pybind11_state_libs} ${onnxruntime_EXTERNAL_LIBRARIES} ${PYTHON_LIBRARY_RELEASE} ${ONNXRUNTIME_SO_LINK_FLAG}) +else() + target_link_libraries(onnxruntime_pybind11_state ${onnxruntime_pybind11_state_libs} ${onnxruntime_EXTERNAL_LIBRARIES} ${PYTHON_LIBRARY} ${ONNXRUNTIME_SO_LINK_FLAG}) + if (APPLE) + set_target_properties(onnxruntime_pybind11_state PROPERTIES INSTALL_RPATH "@loader_path") + else() + set_target_properties(onnxruntime_pybind11_state PROPERTIES LINK_FLAGS "-Xlinker -rpath=\$ORIGIN") + endif() +endif() + +set_target_properties(onnxruntime_pybind11_state PROPERTIES PREFIX "") +set_target_properties(onnxruntime_pybind11_state PROPERTIES FOLDER "ONNXRuntime") + +if (MSVC) + set_target_properties(onnxruntime_pybind11_state PROPERTIES SUFFIX ".pyd") +else() + set_target_properties(onnxruntime_pybind11_state PROPERTIES SUFFIX ".so") +endif() + +file(GLOB onnxruntime_backend_srcs + "${ONNXRUNTIME_ROOT}/python/backend/*.py" +) +file(GLOB onnxruntime_python_srcs + "${ONNXRUNTIME_ROOT}/python/*.py" +) +file(GLOB onnxruntime_python_test_srcs + "${ONNXRUNTIME_ROOT}/test/python/*.py" +) +file(GLOB onnxruntime_python_tools_srcs + "${ONNXRUNTIME_ROOT}/python/tools/*.py" +) +file(GLOB onnxruntime_python_datasets_srcs + "${ONNXRUNTIME_ROOT}/python/datasets/*.py" +) +file(GLOB onnxruntime_python_datasets_data + "${ONNXRUNTIME_ROOT}/python/datasets/*.pb" + "${ONNXRUNTIME_ROOT}/python/datasets/*.onnx" +) + +# adjust based on what target/s onnxruntime_unittests.cmake created +if (SingleUnitTestProject) + set(test_data_target onnxruntime_test_all) +else() + set(test_data_target onnxruntime_test_ir) +endif() + +add_custom_command( + TARGET onnxruntime_pybind11_state POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory $/onnxruntime/backend + COMMAND ${CMAKE_COMMAND} -E make_directory $/onnxruntime/capi + COMMAND ${CMAKE_COMMAND} -E make_directory $/onnxruntime/datasets + COMMAND ${CMAKE_COMMAND} -E make_directory $/onnxruntime/tools + COMMAND ${CMAKE_COMMAND} -E copy + ${ONNXRUNTIME_ROOT}/__init__.py + $/onnxruntime/ + COMMAND ${CMAKE_COMMAND} -E copy + ${REPO_ROOT}/ThirdPartyNotices.txt + $/onnxruntime/ + COMMAND ${CMAKE_COMMAND} -E copy + ${REPO_ROOT}/LICENSE + $/onnxruntime/ + COMMAND ${CMAKE_COMMAND} -E copy + ${onnxruntime_python_test_srcs} + $ + COMMAND ${CMAKE_COMMAND} -E copy + ${onnxruntime_backend_srcs} + $/onnxruntime/backend/ + COMMAND ${CMAKE_COMMAND} -E copy + ${onnxruntime_python_srcs} + $/onnxruntime/capi/ + COMMAND ${CMAKE_COMMAND} -E copy + $ + $/onnxruntime/capi/ + COMMAND ${CMAKE_COMMAND} -E copy + ${onnxruntime_python_datasets_srcs} + $/onnxruntime/datasets/ + COMMAND ${CMAKE_COMMAND} -E copy + ${onnxruntime_python_datasets_data} + $/onnxruntime/datasets/ + COMMAND ${CMAKE_COMMAND} -E copy + ${onnxruntime_python_tools_srcs} + $/onnxruntime/tools/ +) + +if (onnxruntime_USE_MKLDNN) + add_custom_command( + TARGET onnxruntime_pybind11_state POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${MKLDNN_LIB_DIR}/${MKLDNN_SHARED_LIB} + $/onnxruntime/capi/ + ) +endif() +if (onnxruntime_USE_TVM) + add_custom_command( + TARGET onnxruntime_pybind11_state POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + $ $ + $/onnxruntime/capi/ + ) +endif() + +if (onnxruntime_USE_MKLML) + add_custom_command( + TARGET onnxruntime_pybind11_state POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${MKLDNN_LIB_DIR}/${MKLML_SHARED_LIB} ${MKLDNN_LIB_DIR}/${IOMP5MD_SHARED_LIB} + $/onnxruntime/capi/ + ) +endif() diff --git a/cmake/onnxruntime_session.cmake b/cmake/onnxruntime_session.cmake new file mode 100644 index 0000000000000..9cc92ec1f6f3f --- /dev/null +++ b/cmake/onnxruntime_session.cmake @@ -0,0 +1,19 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +file(GLOB onnxruntime_session_srcs + "${ONNXRUNTIME_INCLUDE_DIR}/core/session/*.h" + "${ONNXRUNTIME_ROOT}/core/session/*.h" + "${ONNXRUNTIME_ROOT}/core/session/*.cc" + ) + +source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_session_srcs}) + +add_library(onnxruntime_session ${onnxruntime_session_srcs}) +install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/session DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/session) +onnxruntime_add_include_to_target(onnxruntime_session onnx protobuf::libprotobuf) +target_include_directories(onnxruntime_session PRIVATE ${ONNXRUNTIME_ROOT}) +add_dependencies(onnxruntime_session ${onnxruntime_EXTERNAL_DEPENDENCIES}) +set_target_properties(onnxruntime_session PROPERTIES FOLDER "ONNXRuntime") + + diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake new file mode 100644 index 0000000000000..f4c09cdd1d5fa --- /dev/null +++ b/cmake/onnxruntime_unittests.cmake @@ -0,0 +1,563 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +find_package(Threads) + + +set(TEST_SRC_DIR ${ONNXRUNTIME_ROOT}/test) +set(TEST_INC_DIR ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${date_INCLUDE_DIR} ${CUDA_INCLUDE_DIRS} ${onnxruntime_CUDNN_HOME}/include) +if (onnxruntime_USE_TVM) + list(APPEND TEST_INC_DIR ${TVM_INCLUDES}) +endif() + +set(disabled_warnings) +set(extra_includes) + +function(AddTest) + cmake_parse_arguments(_UT "" "TARGET" "LIBS;SOURCES;DEPENDS" ${ARGN}) + + list(REMOVE_DUPLICATES _UT_LIBS) + list(REMOVE_DUPLICATES _UT_SOURCES) + + if (_UT_DEPENDS) + list(REMOVE_DUPLICATES _UT_DEPENDS) + endif(_UT_DEPENDS) + + add_executable(${_UT_TARGET} ${_UT_SOURCES}) + + source_group(TREE ${TEST_SRC_DIR} FILES ${_UT_SOURCES}) + set_target_properties(${_UT_TARGET} PROPERTIES FOLDER "ONNXRuntimeTest") + + if (_UT_DEPENDS) + add_dependencies(${_UT_TARGET} ${_UT_DEPENDS} eigen) + endif(_UT_DEPENDS) + + target_link_libraries(${_UT_TARGET} PRIVATE ${_UT_LIBS} ${onnxruntime_EXTERNAL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) + target_include_directories(${_UT_TARGET} PRIVATE ${TEST_INC_DIR}) + + if (WIN32) + if (onnxruntime_USE_CUDA) + # disable a warning from the CUDA headers about unreferenced local functions + if (MSVC) + target_compile_options(${_UT_TARGET} PRIVATE /wd4505) + endif() + endif() + target_compile_options(${_UT_TARGET} PRIVATE ${disabled_warnings}) + else() + target_compile_options(${_UT_TARGET} PRIVATE ${DISABLED_WARNINGS_FOR_TVM}) + endif() + + set(TEST_ARGS) + if (onnxruntime_GENERATE_TEST_REPORTS) + # generate a report file next to the test program + list(APPEND TEST_ARGS + "--gtest_output=xml:$.$.results.xml>") + endif(onnxruntime_GENERATE_TEST_REPORTS) + + add_test(NAME ${_UT_TARGET} + COMMAND ${_UT_TARGET} ${TEST_ARGS} + WORKING_DIRECTORY $ + ) +endfunction(AddTest) + +#Check whether C++17 header file is present +include(CheckIncludeFiles) +check_include_files("filesystem" HAS_FILESYSTEM_H LANGUAGE CXX) +check_include_files("experimental/filesystem" HAS_EXPERIMENTAL_FILESYSTEM_H LANGUAGE CXX) + +#Do not add '${TEST_SRC_DIR}/util/include' to your include directories directly +#Use onnxruntime_add_include_to_target or target_link_libraries, so that compile definitions +#can propagate correctly. + +file(GLOB onnxruntime_test_utils_src + "${TEST_SRC_DIR}/util/include/*.h" + "${TEST_SRC_DIR}/util/*.cc" +) + +file(GLOB onnxruntime_test_common_src + "${TEST_SRC_DIR}/common/*.cc" + "${TEST_SRC_DIR}/common/*.h" + "${TEST_SRC_DIR}/common/logging/*.cc" + "${TEST_SRC_DIR}/common/logging/*.h" + ) + +file(GLOB onnxruntime_test_ir_src + "${TEST_SRC_DIR}/ir/*.cc" + "${TEST_SRC_DIR}/ir/*.h" + ) + +set(onnxruntime_test_framework_src_patterns + "${TEST_SRC_DIR}/framework/*.cc" + "${TEST_SRC_DIR}/platform/*.cc" + ) + +if(WIN32) + list(APPEND onnxruntime_test_framework_src_patterns + "${TEST_SRC_DIR}/platform/windows/*.cc" + "${TEST_SRC_DIR}/platform/windows/logging/*.cc" ) +endif() + +if(onnxruntime_USE_CUDA) + list(APPEND onnxruntime_test_framework_src_patterns ${TEST_SRC_DIR}/framework/cuda/*) +endif() + +set(onnxruntime_test_providers_src_patterns + "${TEST_SRC_DIR}/contrib_ops/*.h" + "${TEST_SRC_DIR}/contrib_ops/*.cc" + "${TEST_SRC_DIR}/providers/*.h" + "${TEST_SRC_DIR}/providers/*.cc" + "${TEST_SRC_DIR}/framework/TestAllocatorManager.cc" + "${TEST_SRC_DIR}/framework/TestAllocatorManager.h" + ) + +file(GLOB onnxruntime_test_providers_src ${onnxruntime_test_providers_src_patterns}) +file(GLOB_RECURSE onnxruntime_test_providers_cpu_src + "${TEST_SRC_DIR}/providers/cpu/*" + ) +list(APPEND onnxruntime_test_providers_src ${onnxruntime_test_providers_cpu_src}) + +# tests from lowest level library up. +# the order of libraries should be maintained, with higher libraries being added first in the list + +set(onnxruntime_test_common_libs + onnxruntime_test_utils + onnxruntime_common + gtest + gmock + ) + +set(onnxruntime_test_ir_libs + onnxruntime_test_utils + onnxruntime_graph + onnx + onnx_proto + onnxruntime_common + protobuf::libprotobuf + gtest gmock + ) + +set(onnxruntime_test_framework_libs + onnxruntime_test_utils_for_framework + onnxruntime_session + onnxruntime_providers + onnxruntime_framework + onnxruntime_util + onnxruntime_graph + onnx + onnx_proto + onnxruntime_common + onnxruntime_mlas + protobuf::libprotobuf + gtest gmock + ) + +if(onnxruntime_USE_CUDA) + list(APPEND onnxruntime_test_framework_libs onnxruntime_providers_cuda) +endif() + +if(onnxruntime_USE_MKLDNN) + list(APPEND onnxruntime_test_framework_libs onnxruntime_providers_mkldnn) +endif() + +if(WIN32) + list(APPEND onnxruntime_test_framework_libs Advapi32) +elseif(HAS_FILESYSTEM_H OR HAS_EXPERIMENTAL_FILESYSTEM_H) + list(APPEND onnxruntime_test_framework_libs stdc++fs) +endif() + +set(onnxruntime_test_providers_libs + onnxruntime_test_utils_for_framework + onnxruntime_session) + +set (onnxruntime_test_providers_dependencies ${onnxruntime_EXTERNAL_DEPENDENCIES}) + +if(onnxruntime_USE_CUDA) + list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_cuda) +endif() + +if(onnxruntime_USE_MKLDNN) + list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_mkldnn) +endif() + +if( NOT WIN32 AND (HAS_FILESYSTEM_H OR HAS_EXPERIMENTAL_FILESYSTEM_H)) + list(APPEND onnxruntime_test_providers_libs stdc++fs) +endif() + +file(GLOB_RECURSE onnxruntime_test_tvm_src + "${ONNXRUNTIME_ROOT}/test/tvm/*.h" + "${ONNXRUNTIME_ROOT}/test/tvm/*.cc" + ) + +set(onnx_test_libs + onnxruntime_test_utils + onnxruntime_session) + +if (onnxruntime_ENABLE_MICROSOFT_INTERNAL) + include(onnxruntime_unittests_internal.cmake) +endif() + +list(APPEND onnxruntime_test_providers_libs + ${PROVIDERS_CUDA} + ${PROVIDERS_MKLDNN} + onnxruntime_providers + onnxruntime_framework + onnxruntime_util + onnxruntime_graph + onnx + onnx_proto + onnxruntime_common + onnxruntime_mlas + protobuf::libprotobuf + gtest gmock + ) + +if(WIN32) + if (onnxruntime_USE_TVM) + list(APPEND disabled_warnings ${DISABLED_WARNINGS_FOR_TVM}) + endif() +endif() + +file(GLOB onnxruntime_test_framework_src ${onnxruntime_test_framework_src_patterns}) + +add_library(onnxruntime_test_utils_for_framework ${onnxruntime_test_utils_src}) +onnxruntime_add_include_to_target(onnxruntime_test_utils_for_framework gtest onnx protobuf::libprotobuf) +add_dependencies(onnxruntime_test_utils_for_framework ${onnxruntime_EXTERNAL_DEPENDENCIES} eigen) +target_include_directories(onnxruntime_test_utils_for_framework PUBLIC "${TEST_SRC_DIR}/util/include" PRIVATE ${eigen_INCLUDE_DIRS} ${ONNXRUNTIME_ROOT}) +# Add the define for conditionally using the framework Environment class in TestEnvironment +target_compile_definitions(onnxruntime_test_utils_for_framework PUBLIC -DHAVE_FRAMEWORK_LIB) + +if (SingleUnitTestProject) + add_library(onnxruntime_test_utils ALIAS onnxruntime_test_utils_for_framework) +else() + add_library(onnxruntime_test_utils ${onnxruntime_test_utils_src}) + onnxruntime_add_include_to_target(onnxruntime_test_utils gtest onnx protobuf::libprotobuf) + add_dependencies(onnxruntime_test_utils ${onnxruntime_EXTERNAL_DEPENDENCIES} eigen) + target_include_directories(onnxruntime_test_utils PUBLIC "${TEST_SRC_DIR}/util/include" PRIVATE ${eigen_INCLUDE_DIRS}) +endif() + + +if (SingleUnitTestProject) + set(all_tests ${onnxruntime_test_common_src} ${onnxruntime_test_ir_src} ${onnxruntime_test_framework_src} ${onnxruntime_test_providers_src}) + set(all_libs onnxruntime_test_utils ${onnxruntime_test_providers_libs}) + set(all_dependencies ${onnxruntime_test_providers_dependencies} ) + + if (onnxruntime_USE_TVM) + list(APPEND all_tests ${onnxruntime_test_tvm_src}) + list(APPEND all_libs ${onnxruntime_tvm_libs}) + list(APPEND all_dependencies ${onnxruntime_tvm_dependencies}) + endif() + # we can only have one 'main', so remove them all and add back the providers test_main as it sets + # up everything we need for all tests + file(GLOB_RECURSE test_mains "${TEST_SRC_DIR}/*/test_main.cc") + list(REMOVE_ITEM all_tests ${test_mains}) + list(APPEND all_tests "${TEST_SRC_DIR}/providers/test_main.cc") + + # this is only added to onnxruntime_test_framework_libs above, but we use onnxruntime_test_providers_libs for the onnxruntime_test_all target. + # for now, add it here. better is probably to have onnxruntime_test_providers_libs use the full onnxruntime_test_framework_libs + # list given it's built on top of that library and needs all the same dependencies. + if(WIN32) + list(APPEND onnxruntime_test_providers_libs Advapi32) + endif() + + AddTest( + TARGET onnxruntime_test_all + SOURCES ${all_tests} + LIBS ${all_libs} ${onnxruntime_test_common_libs} + DEPENDS ${all_dependencies} + ) + + # the default logger tests conflict with the need to have an overall default logger + # so skip in this type of + target_compile_definitions(onnxruntime_test_all PUBLIC -DSKIP_DEFAULT_LOGGER_TESTS) + + set(test_data_target onnxruntime_test_all) +else() + AddTest( + TARGET onnxruntime_test_common + SOURCES ${onnxruntime_test_common_src} + LIBS ${onnxruntime_test_common_libs} + DEPENDS ${onnxruntime_EXTERNAL_DEPENDENCIES} + ) + + AddTest( + TARGET onnxruntime_test_ir + SOURCES ${onnxruntime_test_ir_src} + LIBS ${onnxruntime_test_ir_libs} + DEPENDS ${onnxruntime_EXTERNAL_DEPENDENCIES} + ) + + AddTest( + TARGET onnxruntime_test_framework + SOURCES ${onnxruntime_test_framework_src} + LIBS ${onnxruntime_test_framework_libs} + # code smell! see if CPUExecutionProvider should move to framework so onnxruntime_providers isn't needed. + DEPENDS ${onnxruntime_test_providers_dependencies} + ) + + AddTest( + TARGET onnxruntime_test_providers + SOURCES ${onnxruntime_test_providers_src} + LIBS ${onnxruntime_test_providers_libs} + DEPENDS ${onnxruntime_test_providers_dependencies} + ) + + set(test_data_target onnxruntime_test_ir) +endif() # SingleUnitTestProject + +# +# onnxruntime_ir_graph test data +# +set(TEST_DATA_SRC ${TEST_SRC_DIR}/testdata) +set(TEST_DATA_DES $/testdata) + +# Copy test data from source to destination. +add_custom_command( + TARGET ${test_data_target} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_directory + ${TEST_DATA_SRC} + ${TEST_DATA_DES}) + +add_library(onnx_test_data_proto ${TEST_SRC_DIR}/proto/tml.proto) +if(HAS_NULL_DEREFERENCE) + target_compile_options(onnx_test_data_proto PRIVATE "-Wno-null-dereference") + endif() +if(WIN32) + target_compile_options(onnx_test_data_proto PRIVATE "/wd4125" "/wd4456") +endif() +add_dependencies(onnx_test_data_proto onnx_proto ${onnxruntime_EXTERNAL_DEPENDENCIES}) + +if(NOT WIN32) + if(HAS_UNUSED_PARAMETER) + set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/tml.pb.cc PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) + endif() +endif() +onnxruntime_add_include_to_target(onnx_test_data_proto onnx_proto protobuf::libprotobuf) +target_include_directories(onnx_test_data_proto PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/onnx) +set_target_properties(onnx_test_data_proto PROPERTIES FOLDER "ONNXRuntimeTest") +onnxruntime_protobuf_generate(APPEND_PATH IMPORT_DIRS ${ONNXRUNTIME_ROOT}/core/protobuf TARGET onnx_test_data_proto) + +set(onnx_test_runner_src_dir ${TEST_SRC_DIR}/onnx) +set(onnx_test_runner_common_srcs + ${onnx_test_runner_src_dir}/TestResultStat.cc + ${onnx_test_runner_src_dir}/TestResultStat.h + ${onnx_test_runner_src_dir}/testenv.h + ${onnx_test_runner_src_dir}/FixedCountFinishCallback.h + ${onnx_test_runner_src_dir}/TestCaseResult.cc + ${onnx_test_runner_src_dir}/TestCaseResult.h + ${onnx_test_runner_src_dir}/testenv.cc + ${onnx_test_runner_src_dir}/runner.h + ${onnx_test_runner_src_dir}/runner.cc + ${onnx_test_runner_src_dir}/TestCase.cc + ${onnx_test_runner_src_dir}/TestCase.h + ${onnx_test_runner_src_dir}/path_lib.h + ${onnx_test_runner_src_dir}/sync_api.h) + +if(WIN32) + set(wide_get_opt_src_dir ${TEST_SRC_DIR}/win_getopt/wide) + add_library(win_getopt_wide ${wide_get_opt_src_dir}/getopt.cc ${wide_get_opt_src_dir}/include/getopt.h) + target_include_directories(win_getopt_wide INTERFACE ${wide_get_opt_src_dir}/include) + set_target_properties(win_getopt_wide PROPERTIES FOLDER "ONNXRuntimeTest") + set(mb_get_opt_src_dir ${TEST_SRC_DIR}/win_getopt/mb) + add_library(win_getopt_mb ${mb_get_opt_src_dir}/getopt.cc ${mb_get_opt_src_dir}/include/getopt.h) + target_include_directories(win_getopt_mb INTERFACE ${mb_get_opt_src_dir}/include) + set_target_properties(win_getopt_mb PROPERTIES FOLDER "ONNXRuntimeTest") + + set(onnx_test_runner_common_srcs ${onnx_test_runner_common_srcs} ${onnx_test_runner_src_dir}/sync_api_win.cc) + set(GETOPT_LIB_WIDE win_getopt_wide) + set(GETOPT_LIB win_getopt_mb) +else() + set(onnx_test_runner_common_srcs ${onnx_test_runner_common_srcs} ${onnx_test_runner_src_dir}/onnxruntime_event.h ${onnx_test_runner_src_dir}/simple_thread_pool.h ${onnx_test_runner_src_dir}/sync_api_linux.cc) + if(HAS_FILESYSTEM_H OR HAS_EXPERIMENTAL_FILESYSTEM_H) + set(FS_STDLIB stdc++fs) + endif() +endif() + +add_library(onnx_test_runner_common ${onnx_test_runner_common_srcs}) +onnxruntime_add_include_to_target(onnx_test_runner_common onnxruntime_test_utils onnx protobuf::libprotobuf) +add_dependencies(onnx_test_runner_common eigen onnx_test_data_proto ${onnxruntime_EXTERNAL_DEPENDENCIES}) +target_include_directories(onnx_test_runner_common PRIVATE ${eigen_INCLUDE_DIRS} ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/onnx ${ONNXRUNTIME_ROOT}) +set_target_properties(onnx_test_runner_common PROPERTIES FOLDER "ONNXRuntimeTest") + + +if(onnxruntime_USE_CUDA) + set(onnx_cuda_test_libs onnxruntime_providers_cuda) +endif() + +if(onnxruntime_USE_MKLDNN) + set(onnx_mkldnn_test_libs onnxruntime_providers_mkldnn) +endif() + +list(APPEND onnx_test_libs + ${onnx_cuda_test_libs} + ${onnxruntime_tvm_libs} + ${onnx_mkldnn_test_libs} + onnxruntime_providers + onnxruntime_framework + onnxruntime_util + onnxruntime_graph + onnx + onnx_proto + onnxruntime_common + onnxruntime_mlas + onnx_test_data_proto + ${FS_STDLIB} + ${onnxruntime_EXTERNAL_LIBRARIES} + ${ONNXRUNTIME_CUDA_LIBRARIES} + ${CMAKE_THREAD_LIBS_INIT} +) +if(WIN32) + list(APPEND onnx_test_libs Pathcch) +endif() +if (onnxruntime_USE_OPENBLAS) + if (WIN32) + list(APPEND onnx_test_libs ${onnxruntime_OPENBLAS_HOME}/lib/libopenblas.lib) + else() + list(APPEND onnx_test_libs openblas) + endif() +endif() + +if (onnxruntime_USE_MKLDNN) + list(APPEND onnx_test_libs mkldnn) + add_custom_command( + TARGET ${test_data_target} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_LIB_DIR}/${MKLDNN_SHARED_LIB} $ + ) +endif() + +if (onnxruntime_USE_MKLML) + add_custom_command( + TARGET ${test_data_target} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${MKLDNN_LIB_DIR}/${MKLML_SHARED_LIB} ${MKLDNN_LIB_DIR}/${IOMP5MD_SHARED_LIB} + $ + ) +endif() + +add_executable(onnx_test_runner ${onnx_test_runner_src_dir}/main.cc) +target_link_libraries(onnx_test_runner PRIVATE onnx_test_runner_common ${onnx_test_libs} ${GETOPT_LIB_WIDE}) +target_include_directories(onnx_test_runner PRIVATE ${ONNXRUNTIME_ROOT}) +set_target_properties(onnx_test_runner PROPERTIES FOLDER "ONNXRuntimeTest") + +install(TARGETS onnx_test_runner + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + +if(onnxruntime_BUILD_BENCHMARKS AND (HAS_FILESYSTEM_H OR HAS_EXPERIMENTAL_FILESYSTEM_H)) + add_executable(onnxruntime_benchmark ${TEST_SRC_DIR}/onnx/microbenchmark/main.cc ${TEST_SRC_DIR}/onnx/microbenchmark/modeltest.cc) + target_include_directories(onnxruntime_benchmark PRIVATE ${ONNXRUNTIME_ROOT} ${onnxruntime_graph_header} benchmark) + target_compile_options(onnxruntime_benchmark PRIVATE "/wd4141") + target_link_libraries(onnxruntime_benchmark PRIVATE ${onnx_test_libs} onnx_test_runner_common benchmark) + add_dependencies(onnxruntime_benchmark ${onnxruntime_EXTERNAL_DEPENDENCIES}) + set_target_properties(onnxruntime_benchmark PROPERTIES FOLDER "ONNXRuntimeTest") +endif() + +if(WIN32) + set(DISABLED_WARNINGS_FOR_PROTOBUF "/wd4125" "/wd4456" "/wd4505") + target_compile_options(onnx_test_runner_common PRIVATE ${DISABLED_WARNINGS_FOR_PROTOBUF} -D_CRT_SECURE_NO_WARNINGS) + target_compile_options(onnx_test_runner PRIVATE ${DISABLED_WARNINGS_FOR_PROTOBUF}) +endif() + +set(onnxruntime_exec_src_dir ${TEST_SRC_DIR}/onnxruntime_exec) +file(GLOB onnxruntime_exec_src + "${onnxruntime_exec_src_dir}/*.cc" + "${onnxruntime_exec_src_dir}/*.h" + ) + +add_executable(onnxruntime_exec ${onnxruntime_exec_src}) + +target_include_directories(onnxruntime_exec PRIVATE ${ONNXRUNTIME_ROOT}) +# we need to force these dependencies to build first. just using target_link_libraries isn't sufficient +add_dependencies(onnxruntime_exec ${onnxruntime_EXTERNAL_DEPENDENCIES}) +target_link_libraries(onnxruntime_exec PRIVATE ${onnx_test_libs}) +set_target_properties(onnxruntime_exec PROPERTIES FOLDER "ONNXRuntimeTest") + +add_test(NAME onnx_test_pytorch_converted + COMMAND onnx_test_runner ${PROJECT_SOURCE_DIR}/external/onnx/onnx/backend/test/data/pytorch-converted) +add_test(NAME onnx_test_pytorch_operator + COMMAND onnx_test_runner ${PROJECT_SOURCE_DIR}/external/onnx/onnx/backend/test/data/pytorch-operator) + +if(HAS_FILESYSTEM_H OR HAS_EXPERIMENTAL_FILESYSTEM_H) + set(onnxruntime_perf_test_src_dir ${TEST_SRC_DIR}/perftest) + set(onnxruntime_perf_test_src_patterns + "${onnxruntime_perf_test_src_dir}/*.cc" + "${onnxruntime_perf_test_src_dir}/*.h") + + if(WIN32) + list(APPEND onnxruntime_perf_test_src_patterns + "${onnxruntime_perf_test_src_dir}/windows/*.cc" + "${onnxruntime_perf_test_src_dir}/windows/*.h" ) + else () + list(APPEND onnxruntime_perf_test_src_patterns + "${onnxruntime_perf_test_src_dir}/posix/*.cc" + "${onnxruntime_perf_test_src_dir}/posix/*.h" ) + endif() + + file(GLOB onnxruntime_perf_test_src ${onnxruntime_perf_test_src_patterns}) + add_executable(onnxruntime_perf_test ${onnxruntime_perf_test_src}) + + target_include_directories(onnxruntime_perf_test PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${extra_includes} ${onnxruntime_graph_header} ${onnxruntime_exec_src_dir} ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/onnx) + if (WIN32) + target_compile_options(onnxruntime_perf_test PRIVATE ${disabled_warnings}) + endif() + + target_link_libraries(onnxruntime_perf_test PRIVATE ${onnx_test_libs} ${GETOPT_LIB}) + set_target_properties(onnxruntime_perf_test PROPERTIES FOLDER "ONNXRuntimeTest") +endif() + +# shared lib +if (onnxruntime_BUILD_SHARED_LIB) + if (UNIX) + # test custom op shared lib + file(GLOB onnxruntime_custom_op_shared_lib_test_srcs "${ONNXRUNTIME_ROOT}/test/custom_op_shared_lib/test_custom_op.cc") + add_library(onnxruntime_custom_op_shared_lib_test SHARED ${onnxruntime_custom_op_shared_lib_test_srcs}) + add_dependencies(onnxruntime_custom_op_shared_lib_test onnx_proto ${onnxruntime_EXTERNAL_DEPENDENCIES}) + target_include_directories(onnxruntime_custom_op_shared_lib_test PUBLIC "${PROJECT_SOURCE_DIR}/include") + target_link_libraries(onnxruntime_custom_op_shared_lib_test PRIVATE onnxruntime onnx onnx_proto protobuf::libprotobuf) + set_target_properties(onnxruntime_custom_op_shared_lib_test PROPERTIES FOLDER "ONNXRuntimeSharedLibTest") + set(ONNX_DLL onnxruntime) + else() + set(ONNX_DLL onnxruntime) + endif() + + ################################################################# + # test inference using shared lib + custom op + # this program shouldn't have direct depedency on CUDA + # CUDA is part of ${ONNX_DLL} + set (ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR "${ONNXRUNTIME_ROOT}/test/shared_lib") + add_executable(onnxruntime_shared_lib_test + ${ONNXRUNTIME_ROOT}/test/util/test_allocator.cc + ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_fixture.h + ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_inference.cc + ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_session_options.cc + ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_run_options.cc + ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_allocator.cc + ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_inference.cc) + onnxruntime_add_include_to_target(onnxruntime_shared_lib_test onnxruntime_test_utils) + target_include_directories(onnxruntime_shared_lib_test PRIVATE "${TEST_SRC_DIR}/util/include" "${PROJECT_SOURCE_DIR}/include") + if(WIN32) + target_compile_definitions(onnxruntime_shared_lib_test PRIVATE ONNX_RUNTIME_DLL_IMPORT) + endif() + target_link_libraries(onnxruntime_shared_lib_test PRIVATE ${ONNX_DLL} onnx onnx_proto gtest) + + set_target_properties(onnxruntime_shared_lib_test PROPERTIES FOLDER "ONNXRuntimeSharedLibTest") + + add_test(NAME onnxruntime_shared_lib_test COMMAND onnxruntime_shared_lib_test WORKING_DIRECTORY $) + #demo + if(PNG_FOUND) + add_executable(fns_candy_style_transfer "${ONNXRUNTIME_ROOT}/test/shared_lib/fns_candy_style_transfer.c") + target_include_directories(fns_candy_style_transfer PRIVATE "${TEST_SRC_DIR}/util/include" ${PNG_INCLUDE_DIRS}) + target_link_libraries(fns_candy_style_transfer PRIVATE ${ONNX_DLL} ${PNG_LIBRARIES}) + set_target_properties(fns_candy_style_transfer PROPERTIES FOLDER "ONNXRuntimeTest") + endif() +endif() + + +add_executable(onnxruntime_mlas_test ${TEST_SRC_DIR}/mlas/unittest.cpp) +target_include_directories(onnxruntime_mlas_test PRIVATE ${ONNXRUNTIME_ROOT}/core/mlas/inc) +target_link_libraries(onnxruntime_mlas_test PRIVATE onnxruntime_mlas) +set_target_properties(onnxruntime_mlas_test PROPERTIES FOLDER "ONNXRuntimeTest") + +if (onnxruntime_ENABLE_MICROSOFT_INTERNAL) + include(onnxruntime_standalone_tests_internal.cmake) +endif() diff --git a/cmake/onnxruntime_util.cmake b/cmake/onnxruntime_util.cmake new file mode 100644 index 0000000000000..525d851a92d70 --- /dev/null +++ b/cmake/onnxruntime_util.cmake @@ -0,0 +1,20 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +file(GLOB_RECURSE onnxruntime_util_srcs + "${ONNXRUNTIME_ROOT}/core/util/*.h" + "${ONNXRUNTIME_ROOT}/core/util/*.cc" +) + +source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_util_srcs}) + +add_library(onnxruntime_util ${onnxruntime_util_srcs}) +target_include_directories(onnxruntime_util PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS}) +onnxruntime_add_include_to_target(onnxruntime_util onnx protobuf::libprotobuf) +set_target_properties(onnxruntime_util PROPERTIES LINKER_LANGUAGE CXX) +set_target_properties(onnxruntime_util PROPERTIES FOLDER "ONNXRuntime") +add_dependencies(onnxruntime_util ${onnxruntime_EXTERNAL_DEPENDENCIES} eigen) +if (WIN32) + target_compile_definitions(onnxruntime_util PRIVATE _SCL_SECURE_NO_WARNINGS) + target_compile_definitions(onnxruntime_framework PRIVATE _SCL_SECURE_NO_WARNINGS) +endif() diff --git a/cmake/patches/.gitattributes b/cmake/patches/.gitattributes new file mode 100644 index 0000000000000..9812ceb1ffd9b --- /dev/null +++ b/cmake/patches/.gitattributes @@ -0,0 +1 @@ +*.patch text eol=lf diff --git a/cmake/patches/cub/CMakeLists.txt b/cmake/patches/cub/CMakeLists.txt new file mode 100644 index 0000000000000..8fbb114c345bd --- /dev/null +++ b/cmake/patches/cub/CMakeLists.txt @@ -0,0 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +cmake_minimum_required(VERSION 2.8.3) + +project(cub) diff --git a/cmake/patches/mkldnn/mkldnn_sgemm.patch b/cmake/patches/mkldnn/mkldnn_sgemm.patch new file mode 100644 index 0000000000000..e5da49c326383 --- /dev/null +++ b/cmake/patches/mkldnn/mkldnn_sgemm.patch @@ -0,0 +1,136 @@ +diff --git a/src/cpu/gemm/jit_avx2_gemm_f32.cpp b/src/cpu/gemm/jit_avx2_gemm_f32.cpp +index bf03c57..46793e7 100644 +--- a/src/cpu/gemm/jit_avx2_gemm_f32.cpp ++++ b/src/cpu/gemm/jit_avx2_gemm_f32.cpp +@@ -2349,13 +2349,18 @@ void jit_avx2_gemm_f32::sgemm(const char *transa, const char *transb, + + nthr_mn = nthr_m * nthr_n; + +- unsigned int volatile *ompstatus = (unsigned int volatile *)ompstatus_; +- if (!ompstatus) return; ++ unsigned int *ompstatus_ = nullptr; ++ unsigned int volatile *ompstatus = nullptr; + + float *c_buffers = NULL; + float *ws_buffers = NULL; + + if (nthr_k > 1) { ++ ompstatus_ = (unsigned int *)malloc( ++ sizeof(unsigned int *) * nthrs_ * CACHE_LINE_SIZE, 64); ++ ompstatus = (unsigned int volatile *)ompstatus_; ++ assert(ompstatus); ++ + for (int i = 0; i < nthr; i++) + ompstatus[i * CACHE_LINE_SIZE] = 0; + +@@ -2486,8 +2491,10 @@ void jit_avx2_gemm_f32::sgemm(const char *transa, const char *transb, + } + } + +- if (nthr_k > 1) ++ if (nthr_k > 1) { + free(c_buffers); ++ free(ompstatus_); ++ } + free(ws_buffers); + } + +@@ -2513,9 +2520,6 @@ jit_avx2_gemm_f32::jit_avx2_gemm_f32( + ker_b0_ = ker_bn_; + } + nthrs_ = omp_get_max_threads(); +- ompstatus_ = (unsigned int *)malloc( +- sizeof(unsigned int *) * nthrs_ * CACHE_LINE_SIZE, 64); +- assert(ompstatus_); + } + + jit_avx2_gemm_f32::~jit_avx2_gemm_f32() +@@ -2525,7 +2529,6 @@ jit_avx2_gemm_f32::~jit_avx2_gemm_f32() + delete ker_b1_; + if (beta_ != 0.0 || (beta_ == 0.0 && hasBias_)) + delete ker_b0_; +- free(ompstatus_); + } + + } +diff --git a/src/cpu/gemm/jit_avx2_gemm_f32.hpp b/src/cpu/gemm/jit_avx2_gemm_f32.hpp +index 7adb2a2..ebbbde0 100644 +--- a/src/cpu/gemm/jit_avx2_gemm_f32.hpp ++++ b/src/cpu/gemm/jit_avx2_gemm_f32.hpp +@@ -49,7 +49,6 @@ private: + bool hasBias_; + struct xbyak_gemm; + xbyak_gemm *ker_bn_, *ker_b1_, *ker_b0_; +- unsigned int *ompstatus_; + int nthrs_; + }; + } +diff --git a/src/cpu/gemm/jit_avx512_common_gemm_f32.cpp b/src/cpu/gemm/jit_avx512_common_gemm_f32.cpp +index 7959195..fca14f4 100644 +--- a/src/cpu/gemm/jit_avx512_common_gemm_f32.cpp ++++ b/src/cpu/gemm/jit_avx512_common_gemm_f32.cpp +@@ -1866,14 +1866,18 @@ void jit_avx512_common_gemm_f32::sgemm(const char *transa, const char *transb, + nthr = nthr_m * nthr_n * nthr_k; + + nthr_mn = nthr_m * nthr_n; +- +- unsigned int volatile *ompstatus = (unsigned int volatile *)ompstatus_; +- if (!ompstatus) return; ++ ++ unsigned int *ompstatus_ = nullptr; ++ unsigned int volatile *ompstatus = nullptr; + + float *c_buffers = NULL; + float *ws_buffers = NULL; + + if (nthr_k > 1) { ++ ompstatus_ = (unsigned int *)malloc( ++ sizeof(unsigned int *) * nthrs_ * CACHE_LINE_SIZE, 64); ++ ompstatus = (unsigned int volatile *)ompstatus_; ++ assert(ompstatus); + for (int i = 0; i < nthr; i++) + ompstatus[i * CACHE_LINE_SIZE] = 0; + +@@ -2004,8 +2008,10 @@ void jit_avx512_common_gemm_f32::sgemm(const char *transa, const char *transb, + } + } + +- if (nthr_k > 1) ++ if (nthr_k > 1) { + free(c_buffers); ++ free(ompstatus_); ++ } + free(ws_buffers); + } + +@@ -2032,10 +2038,6 @@ jit_avx512_common_gemm_f32::jit_avx512_common_gemm_f32( + } + + nthrs_ = omp_get_max_threads(); +- ompstatus_ = (unsigned int *)malloc( +- sizeof(unsigned int *) * nthrs_ * CACHE_LINE_SIZE, 64); +- assert(ompstatus_); +- + } + + jit_avx512_common_gemm_f32::~jit_avx512_common_gemm_f32() +@@ -2045,7 +2047,6 @@ jit_avx512_common_gemm_f32::~jit_avx512_common_gemm_f32() + delete ker_b1_; + if (beta_ != 0.0 || (beta_ == 0.0 && hasBias_)) + delete ker_b0_; +- free(ompstatus_); + } + } + } +diff --git a/src/cpu/gemm/jit_avx512_common_gemm_f32.hpp b/src/cpu/gemm/jit_avx512_common_gemm_f32.hpp +index ede1cf9..c057335 100644 +--- a/src/cpu/gemm/jit_avx512_common_gemm_f32.hpp ++++ b/src/cpu/gemm/jit_avx512_common_gemm_f32.hpp +@@ -49,7 +49,6 @@ private: + bool hasBias_; + struct xbyak_gemm; + xbyak_gemm *ker_bn_, *ker_b1_, *ker_b0_; +- unsigned int *ompstatus_; + int nthrs_; + }; + } diff --git a/cmake/patches/mkldnn/platform.cmake.patch b/cmake/patches/mkldnn/platform.cmake.patch new file mode 100644 index 0000000000000..03626964fc8f0 --- /dev/null +++ b/cmake/patches/mkldnn/platform.cmake.patch @@ -0,0 +1,14 @@ +diff --git a/cmake/platform.cmake b/cmake/platform.cmake +index fa51aa7..3d24fdc 100644 +--- a/cmake/platform.cmake ++++ b/cmake/platform.cmake +@@ -64,9 +64,6 @@ elseif(UNIX OR APPLE OR MINGW) + # unconditionnaly. + set(CMAKE_CCXX_FLAGS "${CMAKE_CCXX_FLAGS} -Wno-pass-failed") + elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") +- if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0) +- set(DEF_ARCH_OPT_FLAGS "-march=native -mtune=native") +- endif() + if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6.0) + # suppress warning on assumptions made regarding overflow (#146) + set(CMAKE_CCXX_FLAGS "${CMAKE_CCXX_FLAGS} -Wno-strict-overflow") diff --git a/cmake/patches/protobuf.patch b/cmake/patches/protobuf.patch new file mode 100644 index 0000000000000..111d5655ca0d2 --- /dev/null +++ b/cmake/patches/protobuf.patch @@ -0,0 +1,27 @@ +diff --git a/src/google/protobuf/compiler/cpp/cpp_file.cc b/src/google/protobuf/compiler/cpp/cpp_file.cc +index a066a6a7..636a864f 100644 +--- a/src/google/protobuf/compiler/cpp/cpp_file.cc ++++ b/src/google/protobuf/compiler/cpp/cpp_file.cc +@@ -972,6 +972,11 @@ void FileGenerator::GenerateTopHeaderGuard(io::Printer* printer, + "#ifndef PROTOBUF_$filename_identifier$__INCLUDED\n" + "#define PROTOBUF_$filename_identifier$__INCLUDED\n" + "\n" ++ "#ifdef _MSC_VER\n" ++ "#pragma warning(push)\n" ++ "#pragma warning(disable: 4800)\n" ++ "#endif // _MSC_VER\n" ++ "\n" + "#include \n", + "filename", file_->name(), "filename_identifier", filename_identifier); + printer->Print("\n"); +@@ -980,6 +985,10 @@ void FileGenerator::GenerateTopHeaderGuard(io::Printer* printer, + void FileGenerator::GenerateBottomHeaderGuard( + io::Printer* printer, const string& filename_identifier) { + printer->Print( ++ "#ifdef _MSC_VER\n" ++ "#pragma warning(pop)\n" ++ "#endif // _MSC_VER\n" ++ "\n" + "#endif // PROTOBUF_$filename_identifier$__INCLUDED\n", + "filename_identifier", filename_identifier); + } diff --git a/cmake/protobuf_function.cmake b/cmake/protobuf_function.cmake new file mode 100644 index 0000000000000..8b869b73010cf --- /dev/null +++ b/cmake/protobuf_function.cmake @@ -0,0 +1,148 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#Changelog: +#copied from https://github.com/protocolbuffers/protobuf/blob/master/cmake/protobuf-config.cmake.in +#sed -i 's/protobuf_generate/onnxruntime_protobuf_generate/g' protobuf-config.cmake.orig +#replace 'protobuf::protoc' with ${PROTOC_EXECUTABLE} and ${PROTOC_DEPS} +#remove OUTDIR + +function(onnxruntime_protobuf_generate) + include(CMakeParseArguments) + if(EXISTS "${ONNX_CUSTOM_PROTOC_EXECUTABLE}") + set(PROTOC_EXECUTABLE ${ONNX_CUSTOM_PROTOC_EXECUTABLE}) + else() + set(PROTOC_EXECUTABLE $) + set(PROTOC_DEPS protobuf::protoc) + endif() + set(_options APPEND_PATH) + set(_singleargs LANGUAGE OUT_VAR EXPORT_MACRO) + if(COMMAND target_sources) + list(APPEND _singleargs TARGET) + endif() + set(_multiargs PROTOS IMPORT_DIRS GENERATE_EXTENSIONS) + + cmake_parse_arguments(onnxruntime_protobuf_generate "${_options}" "${_singleargs}" "${_multiargs}" "${ARGN}") + + if(NOT onnxruntime_protobuf_generate_PROTOS AND NOT onnxruntime_protobuf_generate_TARGET) + message(SEND_ERROR "Error: onnxruntime_protobuf_generate called without any targets or source files") + return() + endif() + + if(NOT onnxruntime_protobuf_generate_OUT_VAR AND NOT onnxruntime_protobuf_generate_TARGET) + message(SEND_ERROR "Error: onnxruntime_protobuf_generate called without a target or output variable") + return() + endif() + + if(NOT onnxruntime_protobuf_generate_LANGUAGE) + set(onnxruntime_protobuf_generate_LANGUAGE cpp) + endif() + string(TOLOWER ${onnxruntime_protobuf_generate_LANGUAGE} onnxruntime_protobuf_generate_LANGUAGE) + + if(onnxruntime_protobuf_generate_EXPORT_MACRO AND onnxruntime_protobuf_generate_LANGUAGE STREQUAL cpp) + set(_dll_export_decl "dllexport_decl=${onnxruntime_protobuf_generate_EXPORT_MACRO}:") + endif() + + if(NOT onnxruntime_protobuf_generate_EXTENSIONS) + if(onnxruntime_protobuf_generate_LANGUAGE STREQUAL cpp) + set(onnxruntime_protobuf_generate_EXTENSIONS .pb.h .pb.cc) + elseif(onnxruntime_protobuf_generate_LANGUAGE STREQUAL python) + set(onnxruntime_protobuf_generate_EXTENSIONS _pb2.py) + else() + message(SEND_ERROR "Error: onnxruntime_protobuf_generate given unknown Language ${LANGUAGE}, please provide a value for GENERATE_EXTENSIONS") + return() + endif() + endif() + + if(onnxruntime_protobuf_generate_TARGET) + get_target_property(_source_list ${onnxruntime_protobuf_generate_TARGET} SOURCES) + foreach(_file ${_source_list}) + if(_file MATCHES "proto$") + list(APPEND onnxruntime_protobuf_generate_PROTOS ${_file}) + endif() + endforeach() + endif() + + if(NOT onnxruntime_protobuf_generate_PROTOS) + message(SEND_ERROR "Error: onnxruntime_protobuf_generate could not find any .proto files") + return() + endif() + + if(onnxruntime_protobuf_generate_APPEND_PATH) + # Create an include path for each file specified + foreach(_file ${onnxruntime_protobuf_generate_PROTOS}) + get_filename_component(_abs_file ${_file} ABSOLUTE) + get_filename_component(_abs_path ${_abs_file} PATH) + list(FIND _protobuf_include_path ${_abs_path} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _protobuf_include_path -I ${_abs_path}) + endif() + endforeach() + else() + set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR}) + endif() + + foreach(DIR ${onnxruntime_protobuf_generate_IMPORT_DIRS}) + get_filename_component(ABS_PATH ${DIR} ABSOLUTE) + list(FIND _protobuf_include_path ${ABS_PATH} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _protobuf_include_path -I ${ABS_PATH}) + endif() + endforeach() + + set(_generated_srcs_all) + foreach(_proto ${onnxruntime_protobuf_generate_PROTOS}) + get_filename_component(_abs_file ${_proto} ABSOLUTE) + get_filename_component(_basename ${_proto} NAME_WE) + + set(_generated_srcs) + foreach(_ext ${onnxruntime_protobuf_generate_EXTENSIONS}) + list(APPEND _generated_srcs "${CMAKE_CURRENT_BINARY_DIR}/${_basename}${_ext}") + endforeach() + list(APPEND _generated_srcs_all ${_generated_srcs}) + + add_custom_command( + OUTPUT ${_generated_srcs} + COMMAND ${PROTOC_EXECUTABLE} + ARGS --${onnxruntime_protobuf_generate_LANGUAGE}_out ${_dll_export_decl}${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${_abs_file} + DEPENDS ${_abs_file} ${PROTOC_DEPS} + COMMENT "Running ${onnxruntime_protobuf_generate_LANGUAGE} protocol buffer compiler on ${_proto}" + VERBATIM ) + endforeach() + + set_source_files_properties(${_generated_srcs_all} PROPERTIES GENERATED TRUE) + if(onnxruntime_protobuf_generate_OUT_VAR) + set(${onnxruntime_protobuf_generate_OUT_VAR} ${_generated_srcs_all} PARENT_SCOPE) + endif() + if(onnxruntime_protobuf_generate_TARGET) + target_sources(${onnxruntime_protobuf_generate_TARGET} PRIVATE ${_generated_srcs_all}) + endif() + +endfunction() diff --git a/csharp/CSharpUsage/CSharpUsage.csproj b/csharp/CSharpUsage/CSharpUsage.csproj new file mode 100644 index 0000000000000..ae0c068e9a772 --- /dev/null +++ b/csharp/CSharpUsage/CSharpUsage.csproj @@ -0,0 +1,30 @@ + + + + Exe + netcoreapp2.1 + ..\..\build\Windows\$(Configuration)\$(Configuration) + + + + + Always + false + + + Always + false + + + Always + false + + + + + + + + + + diff --git a/csharp/CSharpUsage/Program.cs b/csharp/CSharpUsage/Program.cs new file mode 100644 index 0000000000000..2f195c40ae329 --- /dev/null +++ b/csharp/CSharpUsage/Program.cs @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Text; +using System.IO; +using Microsoft.ML.OnnxRuntime; +using System.Numerics.Tensors; + +namespace CSharpUsage +{ + class Program + { + public static void Main(string[] args) + { + Console.WriteLine("Using API"); + UseApi(); + Console.WriteLine("Done"); + } + + + static void UseApi() + { + string modelPath = Directory.GetCurrentDirectory() + @"\testdata\squeezenet.onnx"; + + + using (var session = new InferenceSession(modelPath)) + { + var inputMeta = session.InputMetadata; + + // User should be able to detect input name/type/shape from the metadata. + // Currently InputMetadata implementation is inclomplete, so assuming Tensor of predefined dimension. + + var shape0 = new int[] { 1, 3, 224, 224 }; + float[] inputData0 = LoadInputsFloat(); + var tensor = new DenseTensor(inputData0, shape0); + + var container = new List(); + container.Add(new NamedOnnxValue("data_0", tensor)); + + // Run the inference + var results = session.Run(container); // results is an IReadOnlyList container + + // dump the results + foreach (var r in results) + { + Console.WriteLine("Output for {0}", r.Name); + Console.WriteLine(r.AsTensor().GetArrayString()); + } + + // Just try some GC collect + results = null; + container = null; + + GC.Collect(); + GC.WaitForPendingFinalizers(); + } + } + + static int[] LoadInputsInt32() + { + return null; + } + + static float[] LoadInputsFloat() + { + // input: data_0 = float32[1,3,224,224] for squeezenet model + // output: softmaxout_1 = float32[1,1000,1,1] + uint size = 1 * 3 * 224 * 224; + float[] tensor = new float[size]; + + // read data from file + using (var inputFile = new System.IO.StreamReader(@"testdata\bench.in")) + { + inputFile.ReadLine(); //skip the input name + string[] dataStr = inputFile.ReadLine().Split(new char[] { ',', '[', ']' }, StringSplitOptions.RemoveEmptyEntries); + for (int i = 0; i < dataStr.Length; i++) + { + tensor[i] = Single.Parse(dataStr[i]); + } + } + + return tensor; + } + + } +} diff --git a/csharp/CSharpUsage/testdata/bench.expected_out b/csharp/CSharpUsage/testdata/bench.expected_out new file mode 100644 index 0000000000000..403588c4e2167 --- /dev/null +++ b/csharp/CSharpUsage/testdata/bench.expected_out @@ -0,0 +1,2 @@ +softmaxout_1 +[[[[3.30038e-05]],[[0.00240706]],[[6.37481e-05]],[[0.000747601]],[[0.00150019]],[[0.00483676]],[[0.0225981]],[[1.00831e-06]],[[1.09255e-05]],[[4.99784e-07]],[[3.56863e-06]],[[2.39458e-07]],[[4.05555e-06]],[[4.125e-05]],[[2.7419e-05]],[[7.0522e-06]],[[3.13915e-06]],[[1.43166e-06]],[[3.89503e-05]],[[9.59126e-07]],[[4.04121e-05]],[[1.44285e-05]],[[3.57193e-06]],[[1.79007e-06]],[[9.95244e-07]],[[8.06105e-06]],[[0.000350968]],[[1.53663e-05]],[[6.4513e-06]],[[0.0012472]],[[5.49246e-06]],[[1.79511e-05]],[[3.62395e-05]],[[0.0108226]],[[0.00336278]],[[0.000252014]],[[0.011735]],[[0.00017365]],[[0.00218219]],[[6.12092e-07]],[[2.211e-05]],[[3.50974e-05]],[[8.72451e-06]],[[1.4989e-05]],[[0.000234527]],[[0.0068162]],[[2.21972e-05]],[[0.00151148]],[[0.000432405]],[[0.00197073]],[[0.00484609]],[[2.82015e-07]],[[0.000228804]],[[3.39269e-05]],[[0.000382118]],[[4.70206e-06]],[[1.1913e-05]],[[2.23097e-06]],[[0.00685201]],[[3.31187e-05]],[[0.000336639]],[[2.79269e-05]],[[0.000199309]],[[0.00317999]],[[0.000113043]],[[0.00128387]],[[0.00155218]],[[0.000923709]],[[0.000240535]],[[0.00253241]],[[2.83529e-05]],[[0.00133016]],[[2.0673e-06]],[[0.000300064]],[[2.02181e-05]],[[0.00039521]],[[0.00180333]],[[0.00526178]],[[0.00672168]],[[0.00165962]],[[1.2716e-06]],[[2.40267e-05]],[[7.0667e-05]],[[8.33176e-06]],[[5.46878e-06]],[[0.000184338]],[[0.000245671]],[[9.80053e-07]],[[2.81819e-07]],[[4.73917e-06]],[[2.19647e-07]],[[2.98639e-06]],[[7.11458e-07]],[[8.84598e-07]],[[1.8719e-05]],[[2.70032e-07]],[[1.23598e-05]],[[3.72205e-07]],[[1.26912e-05]],[[2.47628e-05]],[[1.79942e-06]],[[0.000126682]],[[4.94588e-05]],[[0.0102678]],[[1.58766e-06]],[[2.19647e-07]],[[7.29778e-05]],[[0.00677497]],[[3.8126e-05]],[[0.000435307]],[[2.3404e-06]],[[0.132055]],[[6.02585e-05]],[[5.89881e-05]],[[3.03346e-05]],[[1.3844e-06]],[[5.24638e-05]],[[0.000825554]],[[0.00500985]],[[0.000136619]],[[1.46635e-05]],[[7.55287e-05]],[[3.5211e-06]],[[1.12535e-06]],[[8.07075e-05]],[[0.000272862]],[[0.00321429]],[[1.22513e-05]],[[2.93262e-05]],[[1.48032e-05]],[[5.36753e-06]],[[7.58634e-05]],[[0.000112646]],[[5.02625e-05]],[[3.24783e-05]],[[1.71106e-05]],[[4.81191e-06]],[[0.000141718]],[[7.72931e-05]],[[2.35613e-06]],[[0.00012713]],[[8.392e-05]],[[2.60958e-05]],[[3.85835e-06]],[[1.14518e-05]],[[6.8281e-06]],[[1.35586e-06]],[[6.49379e-05]],[[1.18678e-05]],[[0.00341779]],[[2.61765e-05]],[[2.91871e-05]],[[2.34715e-06]],[[0.000145097]],[[4.2643e-05]],[[0.000197889]],[[7.91037e-05]],[[9.83346e-06]],[[3.39639e-07]],[[3.03579e-06]],[[1.17122e-05]],[[2.06673e-05]],[[0.000150339]],[[0.00020328]],[[1.04667e-05]],[[6.91003e-07]],[[4.05582e-06]],[[5.44345e-07]],[[4.66852e-05]],[[1.18521e-05]],[[3.95729e-07]],[[5.05007e-05]],[[7.4885e-05]],[[6.45486e-06]],[[1.28401e-06]],[[2.01996e-06]],[[5.05181e-06]],[[4.2517e-06]],[[3.03471e-05]],[[2.77055e-05]],[[2.64213e-05]],[[1.88586e-05]],[[1.32463e-05]],[[6.96098e-06]],[[2.90003e-06]],[[3.06301e-06]],[[1.12795e-06]],[[9.59877e-07]],[[2.86446e-06]],[[1.51577e-06]],[[2.72648e-06]],[[6.72556e-06]],[[6.54428e-05]],[[4.4694e-07]],[[1.67012e-06]],[[1.66503e-06]],[[1.55137e-06]],[[1.84327e-06]],[[2.53061e-07]],[[1.05396e-05]],[[1.86334e-05]],[[7.70901e-07]],[[1.40602e-05]],[[0.000601871]],[[0.000196814]],[[8.07645e-07]],[[6.08658e-06]],[[3.00776e-06]],[[2.60375e-06]],[[2.38763e-05]],[[5.27953e-06]],[[1.07436e-05]],[[2.97076e-05]],[[2.28515e-05]],[[7.25094e-06]],[[0.000152125]],[[1.37768e-06]],[[1.51765e-05]],[[2.42037e-06]],[[6.10985e-07]],[[1.8458e-06]],[[7.37962e-06]],[[1.46182e-05]],[[2.51198e-05]],[[3.27596e-07]],[[1.4108e-06]],[[6.25375e-06]],[[2.68018e-06]],[[2.69143e-05]],[[6.06063e-06]],[[0.000106133]],[[1.46003e-05]],[[4.83793e-07]],[[1.34876e-05]],[[2.31457e-06]],[[6.68377e-06]],[[4.43177e-06]],[[7.89233e-05]],[[6.51787e-06]],[[3.42215e-06]],[[2.56579e-06]],[[2.39804e-05]],[[2.38996e-05]],[[7.296e-06]],[[2.30304e-07]],[[6.76266e-05]],[[1.74024e-06]],[[1.99373e-05]],[[6.47423e-07]],[[3.43456e-06]],[[1.10051e-05]],[[1.23886e-06]],[[2.60883e-05]],[[3.96144e-06]],[[4.10274e-05]],[[9.1267e-07]],[[5.01223e-06]],[[0.000132038]],[[6.39193e-05]],[[0.000112808]],[[1.90401e-05]],[[8.00349e-06]],[[7.82643e-06]],[[2.27094e-05]],[[1.38722e-05]],[[4.99394e-06]],[[2.65734e-06]],[[7.1897e-07]],[[3.4084e-07]],[[2.5075e-06]],[[9.08597e-06]],[[1.05238e-06]],[[2.37228e-05]],[[8.48681e-05]],[[0.00015608]],[[4.96326e-06]],[[9.76552e-05]],[[0.00012748]],[[1.16663e-05]],[[0.000474339]],[[0.000125924]],[[9.32651e-07]],[[6.14761e-06]],[[1.84157e-05]],[[2.0661e-05]],[[2.15305e-06]],[[1.90258e-06]],[[6.07655e-06]],[[2.72655e-06]],[[5.12465e-07]],[[3.07574e-06]],[[5.70555e-06]],[[3.70938e-05]],[[3.89949e-06]],[[5.05176e-06]],[[1.89544e-06]],[[3.55921e-05]],[[0.000115676]],[[8.98462e-05]],[[5.97214e-05]],[[2.12628e-06]],[[6.78752e-05]],[[4.29522e-05]],[[1.56884e-06]],[[1.50015e-05]],[[0.000283533]],[[6.39846e-07]],[[9.70085e-05]],[[1.88628e-05]],[[6.64936e-07]],[[0.000356803]],[[8.39299e-06]],[[5.74065e-05]],[[7.27516e-05]],[[0.00261427]],[[6.00777e-05]],[[7.12263e-05]],[[1.26491e-06]],[[0.000868939]],[[0.000397627]],[[2.35373e-05]],[[2.28251e-07]],[[3.06452e-07]],[[2.19647e-07]],[[8.73208e-07]],[[9.02118e-07]],[[1.382e-06]],[[0.000156077]],[[0.000121389]],[[8.77277e-05]],[[0.00316587]],[[0.000807538]],[[0.00204002]],[[0.00032277]],[[0.000302543]],[[4.31282e-06]],[[2.19416e-05]],[[0.00177341]],[[1.06765e-05]],[[1.11829e-06]],[[2.17968e-06]],[[1.53142e-05]],[[6.17556e-06]],[[1.55956e-05]],[[0.00168962]],[[5.31761e-07]],[[7.22402e-05]],[[6.69545e-06]],[[3.47428e-07]],[[9.00044e-07]],[[0.000113894]],[[2.07959e-05]],[[7.21104e-06]],[[6.67658e-05]],[[1.03238e-05]],[[3.54474e-06]],[[0.000450102]],[[0.000364276]],[[0.000164161]],[[0.000321661]],[[7.09733e-05]],[[0.000874399]],[[0.000532035]],[[0.000946064]],[[1.63541e-06]],[[6.66925e-07]],[[1.00801e-05]],[[9.33275e-07]],[[2.26388e-07]],[[2.65757e-07]],[[3.81374e-07]],[[3.49317e-06]],[[5.70276e-07]],[[1.79782e-06]],[[3.96168e-06]],[[3.79932e-07]],[[9.98164e-07]],[[2.68421e-05]],[[7.04427e-06]],[[2.14382e-06]],[[5.87577e-05]],[[8.82346e-07]],[[1.65695e-06]],[[4.0358e-06]],[[3.4606e-07]],[[4.24055e-05]],[[6.4875e-06]],[[7.15376e-07]],[[2.83752e-07]],[[0.000187299]],[[5.81318e-05]],[[0.000161363]],[[1.87876e-05]],[[1.81743e-06]],[[0.00169564]],[[0.00922038]],[[2.4518e-06]],[[2.37628e-05]],[[1.56076e-06]],[[4.16376e-05]],[[2.22112e-07]],[[2.74136e-07]],[[8.87607e-07]],[[2.37552e-05]],[[5.59479e-05]],[[0.000238186]],[[4.14247e-06]],[[2.62818e-07]],[[5.15268e-06]],[[2.44908e-05]],[[5.29226e-06]],[[0.000488674]],[[0.000196068]],[[4.08526e-07]],[[1.51241e-06]],[[4.3665e-06]],[[1.00193e-05]],[[0.00267409]],[[0.000107653]],[[0.00136346]],[[2.21079e-06]],[[1.27526e-05]],[[2.54319e-06]],[[1.4435e-06]],[[2.92003e-07]],[[1.88393e-05]],[[2.84691e-05]],[[2.419e-06]],[[2.81245e-06]],[[4.00504e-06]],[[1.22796e-05]],[[0.000329972]],[[7.76645e-07]],[[8.17329e-05]],[[0.0075691]],[[0.000272345]],[[3.47348e-07]],[[1.72498e-05]],[[0.00059095]],[[2.19856e-07]],[[2.97259e-05]],[[5.70946e-05]],[[3.50248e-06]],[[0.000505348]],[[2.6645e-07]],[[1.84202e-05]],[[0.0419757]],[[1.19775e-06]],[[0.000125246]],[[3.40366e-06]],[[1.28719e-06]],[[7.57776e-07]],[[0.000119444]],[[4.55271e-06]],[[5.57566e-06]],[[1.85926e-05]],[[4.23977e-07]],[[4.13716e-07]],[[0.000238636]],[[0.00234118]],[[2.00847e-05]],[[8.80801e-07]],[[6.20842e-05]],[[0.000201885]],[[4.35091e-07]],[[0.000212658]],[[7.93442e-07]],[[2.56245e-07]],[[2.86597e-07]],[[4.9724e-05]],[[0.000595791]],[[3.17517e-07]],[[8.28939e-07]],[[2.65091e-05]],[[0.000116553]],[[1.81658e-06]],[[2.7075e-07]],[[1.80121e-06]],[[0.00671099]],[[2.88147e-07]],[[3.31486e-05]],[[0.000103751]],[[7.2855e-07]],[[2.59299e-06]],[[8.29641e-06]],[[2.8954e-05]],[[3.82065e-07]],[[1.58906e-05]],[[7.5345e-06]],[[1.88725e-06]],[[2.20567e-07]],[[2.64769e-07]],[[0.000704843]],[[7.48047e-06]],[[7.07716e-06]],[[3.24475e-06]],[[0.000229562]],[[1.04762e-05]],[[2.75673e-07]],[[5.60927e-05]],[[0.000670869]],[[5.25422e-06]],[[7.67442e-06]],[[3.99764e-05]],[[2.02051e-05]],[[4.53167e-06]],[[4.54816e-06]],[[8.61861e-06]],[[1.01519e-06]],[[3.966e-07]],[[4.79176e-05]],[[2.37593e-07]],[[4.46226e-07]],[[2.95312e-07]],[[3.49675e-07]],[[5.54914e-06]],[[0.000167629]],[[2.35371e-05]],[[9.46391e-07]],[[0.000218146]],[[0.000695242]],[[0.000177932]],[[2.13105e-05]],[[1.37545e-05]],[[4.23788e-07]],[[0.00279564]],[[1.56444e-06]],[[2.2925e-07]],[[1.14547e-05]],[[0.000838637]],[[1.94783e-05]],[[2.22737e-06]],[[0.00269667]],[[0.000587703]],[[0.000242132]],[[2.39822e-07]],[[7.88235e-07]],[[2.19647e-07]],[[3.96023e-05]],[[0.000942217]],[[0.000199963]],[[2.98115e-07]],[[0.000114011]],[[2.78705e-06]],[[1.49864e-06]],[[5.44075e-05]],[[6.16242e-06]],[[2.25314e-07]],[[2.90298e-07]],[[0.0563553]],[[2.19647e-07]],[[0.00041908]],[[2.45128e-06]],[[0.000216731]],[[0.000220765]],[[7.10003e-06]],[[0.00314294]],[[4.27621e-06]],[[1.21116e-06]],[[3.57691e-06]],[[2.19866e-07]],[[2.77421e-05]],[[1.33204e-06]],[[3.23081e-06]],[[4.16899e-07]],[[3.00326e-05]],[[2.23953e-07]],[[5.92595e-06]],[[2.59079e-07]],[[3.95766e-07]],[[6.23917e-05]],[[2.3895e-07]],[[2.59321e-05]],[[8.72774e-07]],[[1.46551e-05]],[[2.37166e-07]],[[4.85887e-07]],[[5.91673e-06]],[[1.5925e-05]],[[5.48621e-07]],[[1.29689e-06]],[[2.42972e-07]],[[2.58339e-07]],[[7.07611e-06]],[[9.51068e-06]],[[9.89523e-05]],[[4.30986e-05]],[[6.61681e-05]],[[0.000377943]],[[3.49729e-06]],[[5.08567e-06]],[[0.100853]],[[1.0899e-05]],[[1.52226e-06]],[[4.74296e-07]],[[2.98692e-07]],[[1.48701e-05]],[[2.72196e-07]],[[8.15167e-07]],[[0.000854144]],[[4.81532e-05]],[[7.4703e-05]],[[2.33504e-07]],[[4.01518e-07]],[[4.65201e-05]],[[0.000614875]],[[2.85956e-05]],[[7.15136e-07]],[[7.42455e-06]],[[6.68006e-05]],[[0.00338469]],[[0.000373694]],[[3.92498e-07]],[[8.70474e-07]],[[1.46042e-06]],[[2.39024e-05]],[[4.56457e-06]],[[7.64493e-05]],[[2.58002e-05]],[[0.000290927]],[[5.9998e-05]],[[1.0992e-06]],[[7.49084e-06]],[[0.000270476]],[[1.07873e-05]],[[8.33809e-07]],[[0.000378689]],[[2.31458e-07]],[[6.9372e-06]],[[3.28015e-05]],[[1.66907e-05]],[[0.000668885]],[[3.65539e-05]],[[4.492e-06]],[[6.63593e-07]],[[1.4175e-05]],[[5.3893e-06]],[[1.25146e-06]],[[8.96112e-06]],[[1.56599e-05]],[[0.000639026]],[[5.25124e-07]],[[4.33042e-07]],[[9.55997e-07]],[[7.20381e-05]],[[1.8769e-06]],[[0.00591436]],[[0.0142333]],[[5.78227e-05]],[[5.58047e-06]],[[1.15346e-05]],[[0.000191445]],[[2.50651e-06]],[[5.37143e-05]],[[1.61384e-06]],[[2.10579e-05]],[[2.82325e-07]],[[0.000562215]],[[7.31107e-05]],[[0.000185458]],[[9.19092e-06]],[[2.24806e-07]],[[0.00122146]],[[3.34874e-06]],[[3.1904e-05]],[[2.49545e-07]],[[0.000527636]],[[5.26276e-07]],[[4.21786e-06]],[[0.0981208]],[[2.38566e-07]],[[8.18549e-06]],[[7.68801e-06]],[[1.79197e-05]],[[0.000124411]],[[2.81175e-07]],[[2.37748e-07]],[[0.000151226]],[[1.34168e-05]],[[1.33137e-06]],[[0.00140345]],[[4.45635e-05]],[[0.000123665]],[[4.43648e-07]],[[3.71324e-07]],[[2.56964e-06]],[[6.44212e-05]],[[3.52327e-06]],[[0.000145309]],[[2.28818e-05]],[[8.69461e-07]],[[7.64946e-06]],[[0.00124899]],[[1.95925e-06]],[[1.82611e-05]],[[0.000128566]],[[6.53273e-05]],[[6.7991e-05]],[[6.96028e-07]],[[6.85518e-07]],[[0.0999919]],[[0.000171717]],[[2.45555e-07]],[[2.20557e-07]],[[6.95378e-05]],[[2.36021e-07]],[[6.83477e-05]],[[2.71391e-06]],[[0.000297223]],[[6.14753e-05]],[[3.98032e-05]],[[0.000202539]],[[0.0211731]],[[0.000109838]],[[0.000204545]],[[9.54438e-07]],[[0.000132366]],[[3.13755e-07]],[[1.7683e-05]],[[3.05367e-06]],[[8.13365e-05]],[[8.62639e-05]],[[5.79236e-05]],[[1.72338e-05]],[[5.51936e-06]],[[2.37853e-05]],[[2.38583e-06]],[[7.3413e-05]],[[0.0365768]],[[0.000350949]],[[3.97942e-07]],[[0.000166126]],[[2.62828e-07]],[[0.000119214]],[[2.59006e-07]],[[0.000244027]],[[2.20833e-07]],[[2.69368e-05]],[[2.08583e-06]],[[6.67744e-05]],[[1.15667e-06]],[[0.00231503]],[[2.33152e-05]],[[2.7366e-06]],[[3.53167e-05]],[[2.03656e-05]],[[6.22545e-05]],[[1.2622e-06]],[[1.95008e-05]],[[0.000707449]],[[0.0022835]],[[2.14179e-05]],[[6.99508e-06]],[[7.92588e-05]],[[6.41414e-07]],[[0.00251094]],[[2.4398e-05]],[[6.75114e-07]],[[8.26408e-07]],[[4.115e-07]],[[3.32577e-05]],[[3.77059e-05]],[[2.97349e-07]],[[1.01808e-06]],[[1.14929e-06]],[[2.92594e-07]],[[3.70965e-07]],[[0.000210903]],[[7.57253e-06]],[[0.000671497]],[[2.57833e-05]],[[4.24406e-05]],[[0.000115908]],[[1.37939e-05]],[[1.4135e-05]],[[8.69344e-05]],[[1.36076e-06]],[[5.66509e-06]],[[9.69823e-06]],[[2.20374e-07]],[[3.79278e-06]],[[3.20583e-07]],[[3.48438e-05]],[[2.23965e-05]],[[3.01125e-05]],[[3.28908e-07]],[[1.8647e-06]],[[3.01314e-06]],[[3.23302e-07]],[[0.000202211]],[[1.46115e-06]],[[0.000135461]],[[1.81886e-06]],[[0.00220625]],[[0.0100632]],[[4.68942e-06]],[[5.13535e-06]],[[0.000709504]],[[6.95215e-05]],[[1.05843e-05]],[[2.19647e-07]],[[5.92654e-05]],[[7.72615e-06]],[[1.30853e-05]],[[0.00035334]],[[2.60654e-05]],[[1.70955e-06]],[[2.0768e-05]],[[1.40579e-06]],[[5.68283e-05]],[[2.28711e-07]],[[3.1642e-05]],[[0.000870124]],[[0.000250426]],[[8.21169e-06]],[[0.00041412]],[[5.17313e-05]],[[2.19648e-07]],[[2.58391e-05]],[[4.21913e-07]],[[2.75305e-07]],[[2.02621e-06]],[[5.78907e-06]],[[2.06269e-06]],[[1.4204e-05]],[[2.70143e-06]],[[1.57078e-06]],[[6.74661e-07]],[[3.99983e-05]],[[5.39646e-06]],[[2.84113e-05]],[[1.13468e-05]],[[2.61936e-05]],[[0.000348203]],[[1.3984e-05]],[[5.74484e-06]],[[4.46801e-06]],[[3.07966e-07]],[[0.000679563]],[[4.93455e-07]],[[0.000668054]],[[0.000694075]],[[0.000109974]],[[2.86653e-07]],[[0.000163258]],[[0.00102639]],[[2.56474e-05]],[[5.7945e-06]],[[3.08543e-06]],[[2.04542e-05]],[[6.7593e-06]],[[1.35856e-05]],[[1.80622e-05]],[[1.42013e-06]],[[2.31614e-05]],[[9.83816e-06]],[[2.23705e-07]],[[8.88647e-06]],[[2.99608e-07]],[[2.06588e-06]],[[4.90268e-07]],[[0.00990751]],[[5.02635e-05]],[[2.991e-05]],[[6.99187e-06]],[[2.19762e-07]],[[4.38414e-06]],[[1.94361e-05]],[[0.0118964]],[[4.71439e-07]],[[3.48044e-06]],[[3.06884e-06]],[[8.46049e-07]],[[2.84214e-06]],[[2.61806e-07]],[[8.08323e-07]],[[0.000378208]],[[1.44979e-06]],[[2.48622e-07]],[[2.63819e-05]],[[2.31254e-06]],[[3.73906e-07]],[[7.06266e-07]],[[8.55171e-05]],[[0.000463059]],[[0.00519056]],[[2.3448e-07]],[[7.43225e-05]],[[1.92467e-05]],[[3.02221e-07]],[[3.01564e-06]],[[1.00334e-05]],[[0.00138754]],[[5.73591e-05]],[[4.46816e-06]],[[0.0001871]],[[0.000772637]],[[5.25714e-05]],[[0.000222698]],[[0.00011766]],[[6.29192e-05]],[[0.00104611]],[[0.000150696]],[[2.85862e-06]],[[0.0213807]],[[7.83114e-05]],[[1.22906e-05]],[[2.89221e-05]],[[0.00558292]],[[1.80053e-05]],[[6.28303e-05]],[[0.000187814]],[[2.22076e-05]],[[0.019121]],[[1.44428e-05]],[[3.48743e-05]],[[4.95461e-06]],[[2.01822e-06]],[[0.000154636]],[[3.96885e-06]],[[1.15581e-05]],[[0.0217395]],[[0.00106286]],[[1.47636e-05]],[[7.78406e-07]],[[0.000182114]],[[3.64317e-05]],[[2.45914e-05]],[[0.000218702]],[[1.77705e-05]],[[3.88213e-05]],[[6.38537e-06]],[[6.49358e-06]],[[6.16064e-07]],[[9.00705e-06]],[[6.60956e-06]],[[3.62144e-05]],[[3.51255e-06]],[[3.13264e-05]],[[6.02226e-07]],[[9.67834e-06]],[[1.9771e-06]],[[8.10685e-06]],[[4.62282e-06]],[[1.3246e-06]],[[7.31311e-07]],[[7.34991e-06]],[[2.95626e-05]],[[3.87789e-05]],[[1.1141e-05]],[[2.79194e-06]],[[3.32728e-05]],[[8.18433e-06]],[[2.76908e-07]],[[2.60412e-07]],[[4.92324e-06]],[[2.33574e-07]],[[3.27539e-06]],[[1.97259e-05]],[[2.95824e-06]],[[2.76007e-06]],[[0.00158116]],[[5.84331e-07]],[[1.81017e-06]],[[1.5555e-05]],[[7.36356e-07]],[[0.000136843]],[[1.42929e-05]],[[0.000111963]],[[0.000735147]],[[0.000271685]],[[0.00143252]],[[6.72216e-05]],[[0.000145032]],[[0.000286367]],[[0.000266569]],[[0.000258938]],[[0.00629227]],[[0.000196766]],[[0.00144487]],[[0.000658506]],[[6.19422e-06]],[[7.89064e-06]],[[0.000167934]],[[3.46261e-05]],[[2.56693e-05]],[[1.69383e-06]],[[1.93635e-05]],[[1.4758e-05]],[[1.72688e-06]],[[2.75854e-07]],[[7.37269e-07]],[[6.86722e-07]],[[2.20309e-07]],[[3.00458e-07]],[[8.82893e-06]],[[5.07031e-06]],[[3.30098e-05]],[[5.39999e-06]],[[0.0041139]]]] diff --git a/csharp/CSharpUsage/testdata/bench.in b/csharp/CSharpUsage/testdata/bench.in new file mode 100644 index 0000000000000..b28238dad41ee --- /dev/null +++ b/csharp/CSharpUsage/testdata/bench.in @@ -0,0 +1,2 @@ +data_0 +[[[[1.7640523911,0.4001572132,0.9787380099,2.2408931255,1.8675580025,-0.9772778749,0.9500884414,-0.1513572037,-0.1032188535,0.4105985165,0.1440435648,1.4542734623,0.7610377073,0.1216750145,0.4438632429,0.3336743414,1.4940791130,-0.2051582634,0.3130677044,-0.8540957570,-2.5529897213,0.6536185741,0.8644362092,-0.7421650290,2.2697546482,-1.4543657303,0.0457585156,-0.1871838570,1.5327792168,1.4693588018,0.1549474299,0.3781625330,-0.8877857327,-1.9807964563,-0.3479121625,0.1563489735,1.2302906513,1.2023798227,-0.3873268068,-0.3023027480,-1.0485529900,-1.4200179577,-1.7062702179,1.9507753849,-0.5096521974,-0.4380742908,-1.2527953386,0.7774903774,-1.6138978004,-0.2127402872,-0.8954665661,0.3869025111,-0.5108051300,-1.1806322336,-0.0281822290,0.4283318818,0.0665172189,0.3024719059,-0.6343221068,-0.3627411723,-0.6724604368,-0.3595531583,-0.8131462932,-1.7262825966,0.1774261445,-0.4017809331,-1.6301983595,0.4627822638,-0.9072983861,0.0519453958,0.7290905714,0.1289829165,1.1394007206,-1.2348258495,0.4023416340,-0.6848101020,-0.8707971573,-0.5788496733,-0.3115525246,0.0561653413,-1.1651498079,0.9008265138,0.4656624496,-1.5362436771,1.4882521629,1.8958891630,1.1787796021,-0.1799248308,-1.0707526207,1.0544517040,-0.4031769335,1.2224450111,0.2082749754,0.9766390324,0.3563663960,0.7065731883,0.0105000203,1.7858705521,0.1269120872,0.4019893706,1.8831506968,-1.3477590084,-1.2704850435,0.9693967104,-1.1731233597,1.9436211586,-0.4136189818,-0.7474548221,1.9229420424,1.4805147648,1.8675589561,0.9060446620,-0.8612256646,1.9100649357,-0.2680033743,0.8024563789,0.9472519755,-0.1550100893,0.6140793562,0.9222066998,0.3764255345,-1.0994007587,0.2982381880,1.3263858557,-0.6945678592,-0.1496345401,-0.4351535439,1.8492637873,0.6722947359,0.4074618220,-0.7699160576,0.5392491817,-0.6743326783,0.0318305567,-0.6358460784,0.6764332652,0.5765908360,-0.2082987577,0.3960067034,-1.0930615664,-1.4912575483,0.4393917024,0.1666734964,0.6350314617,2.3831448555,0.9444794655,-0.9128222466,1.1170163155,-1.3159073591,-0.4615845978,-0.0682416037,1.7133426666,-0.7447548509,-0.8264385462,-0.0984525234,-0.6634783149,1.1266359091,-1.0799314976,-1.1474686861,-0.4378200471,-0.4980324507,1.9295320511,0.9494208097,0.0875512436,-1.2254354954,0.8443629742,-1.0002152920,-1.5447710752,1.1880297661,0.3169426024,0.9208588004,0.3187276423,0.8568305969,-0.6510255933,-1.0342428684,0.6815944910,-0.8034096360,-0.6895498037,-0.4555324912,0.0174791589,-0.3539939225,-1.3749512434,-0.6436184049,-2.2234032154,0.6252314448,-1.6020576954,-1.1043833494,0.0521650799,-0.7395629883,1.5430146456,-1.2928569317,0.2670508623,-0.0392828174,-1.1680934429,0.5232766867,-0.1715463251,0.7717905641,0.8235041499,2.1632359028,1.3365279436,-0.3691818416,-0.2393791825,1.0996595621,0.6552637219,0.6401315331,-1.6169559956,-0.0243261252,-0.7380309105,0.2799246013,-0.0981503874,0.9101788998,0.3172182143,0.7863279581,-0.4664191008,-0.9444462657,-0.4100497067,-0.0170204137,0.3791517317,2.2593090534,-0.0422571525,-0.9559450150,-0.3459817767,-0.4635959864,0.4814814627],[-1.5407969952,0.0632619932,0.1565065384,0.2321810424,-0.5973160863,-0.2379217297,-1.4240609407,-0.4933198690,-0.5428614616,0.4160500467,-1.1561824083,0.7811980844,1.4944845438,-2.0699849129,0.4262587428,0.6769080162,-0.6374370456,-0.3972718120,-0.1328805834,-0.2977908850,-0.3090129793,-1.6760038137,1.1523315907,1.0796185732,-0.8133642673,-1.4664243460,0.5210648775,-0.5757879615,0.1419531703,-0.3193284273,0.6915387511,0.6947491169,-0.7255973816,-1.3833639622,-1.5829384327,0.6103793979,-1.1888592243,-0.5068163276,-0.5963140130,-0.0525672957,-1.9362797737,0.1887785941,0.5238910317,0.0884220898,-0.3108861744,0.0974001661,0.3990463316,-2.7725927830,1.9559123516,0.3900933266,-0.6524085999,-0.3909533620,0.4937417805,-0.1161039397,-2.0306844711,2.0644929409,-0.1105406582,1.0201727152,-0.6920498610,1.5363770723,0.2863436937,0.6088438630,-1.0452533960,1.2111452818,0.6898181438,1.3018462658,-0.6280875802,-0.4810271263,2.3039166927,-1.0600157976,-0.1359497011,1.1368913651,0.0977249667,0.5829536915,-0.3994490206,0.3700558841,-1.3065268993,1.6581306458,-0.1181640476,-0.6801782250,0.6663830876,-0.4607197940,-1.3342584372,-1.3467174768,0.6937731504,-0.1595734358,-0.1337015629,1.0777437687,-1.1268258095,-0.7306777239,-0.3848797977,0.0943515897,-0.0421714522,-0.2868871987,-0.0616264008,-0.1073052734,-0.7196043730,-0.8129929900,0.2745163441,-0.8909150958,-1.1573553085,-0.3122922480,-0.1576670110,2.2567234039,-0.7047002912,0.9432607293,0.7471883297,-1.1889449358,0.7732529640,-1.1838806868,-2.6591722965,0.6063195467,-1.7558906078,0.4509344697,-0.6840109229,1.6595507860,1.0685093403,-0.4533858001,-0.6878376007,-1.2140773535,-0.4409226179,-0.2803554833,-0.3646935523,0.1567038596,0.5785214901,0.3496544659,-0.7641439438,-1.4377914667,1.3645318747,-0.6894491911,-0.6522936225,-0.5211893320,-1.8430695534,-0.4779739976,-0.4796558022,0.6203582883,0.6984571218,0.0037708890,0.9318483472,0.3399649858,-0.0156821124,0.1609281749,-0.1906534880,-0.3948495090,-0.2677335441,-1.1280113459,0.2804417014,-0.9931235909,0.8416312933,-0.2494585812,0.0494949818,0.4938367903,0.6433144808,-1.5706233978,-0.2069036812,0.8801789284,-1.6981058121,0.3872804642,-2.2555642128,-1.0225068331,0.0386305526,-1.6567151546,-0.9855107665,-1.4718350172,1.6481349468,0.1642277539,0.5672903061,-0.2226751000,-0.3534317613,-1.6164741516,-0.2918373644,-0.7614921927,0.8579239249,1.1411018372,1.4665787220,0.8525519371,-0.5986539125,-1.1158969402,0.7666631937,0.3562928140,-1.7685384750,0.3554818034,0.8145198226,0.0589255877,-0.1850536764,-0.8076484799,-1.4465347528,0.8002979755,-0.3091144562,-0.2334666550,1.7327212095,0.6845011115,0.3708249927,0.1420617998,1.5199948549,1.7195893526,0.9295051098,0.5822246075,-2.0946030617,0.1237219125,-0.1301069558,0.0939532295,0.9430460930,-2.7396771908,-0.5693120360,0.2699043453,-0.4668455422,-1.4169061184,0.8689634800,0.2768719196,-0.9711045623,0.3148171902,0.8215857148,0.0052926461,0.8005648255,0.0782601759,-0.3952289820,-1.1594204903,-0.0859307647,0.1942929327,0.8758327365,-0.1151074693,0.4574156106,-0.9646120071],[-0.7826291323,-0.1103892997,-1.0546284914,0.8202478290,0.4631303251,0.2790957689,0.3389041126,2.0210435390,-0.4688642025,-2.2014412880,0.1993001997,-0.0506035425,-0.5175190568,-0.9788298607,-0.4391895235,0.1813384295,-0.5028166771,2.4124536514,-0.9605043530,-0.7931173444,-2.2886199951,0.2514844239,-2.0164065361,-0.5394546390,-0.2756705284,-0.7097279429,1.7388726473,0.9943943620,1.3191368580,-0.8824188113,1.1285940409,0.4960009456,0.7714059353,1.0294388533,-0.9087632298,-0.4243176281,0.8625960350,-2.6556191444,1.5133280754,0.5531320572,-0.0457039624,0.2205076516,-1.0299352407,-0.3499433696,1.1002843380,1.2980220318,2.6962239742,-0.0739246681,-0.6585529447,-0.5142339468,-1.0180418491,-0.0778547525,0.3827324212,-0.0342422798,1.0963468552,-0.2342157960,-0.3474506438,-0.5812684894,-1.6326345205,-1.5677677393,-1.1791579723,1.3014280796,0.8952602744,1.3749641180,-1.3322116137,-1.9686247110,-0.6600562930,0.1758189499,0.4986902773,1.0479722023,0.2842796743,1.7426687479,-0.2226056755,-0.9130792022,-1.6812182665,-0.8889713287,0.2421179563,-0.8887202740,0.9367424846,1.4123276472,-2.3695869446,0.8640522957,-2.2396039963,0.4014990628,1.2248705626,0.0648561046,-1.2796891928,-0.5854312181,-0.2616454363,-0.1822447777,-0.2028968334,-0.1098827794,0.2134800553,-1.2085736990,-0.2420198321,1.5182611942,-0.3846454322,-0.4438360929,1.0781973600,-2.5591845512,1.1813786030,-0.6319037676,0.1639285684,0.0963213593,0.9424681067,-0.2675947547,-0.6780257821,1.2978458405,-2.3641738892,0.0203341823,-1.3479254246,-0.7615733743,2.0112566948,-0.0445954278,0.1950697005,-1.7815628052,-0.7290446758,0.1965574026,0.3547576964,0.6168865561,0.0086278990,0.5270041823,0.4537819028,-1.8297404051,0.0370057225,0.7679024339,0.5898798108,-0.3638588190,-0.8056265116,-1.1183118820,-0.1310540140,1.1330798864,-1.9518041611,-0.6598917246,-1.1398024559,0.7849575281,-0.5543096066,-0.4706376493,-0.2169495672,0.4453932643,-0.3923889995,-3.0461430550,0.5433118939,0.4390429556,-0.2195410281,-1.0840365887,0.3517801166,0.3792355359,-0.4700328708,-0.2167314738,-0.9301565289,-0.1785890907,-1.5504293442,0.4173188210,-0.9443684816,0.2381031513,-1.4059629440,-0.5900576711,-0.1104894057,-1.6606998444,0.1151478738,-0.3791475594,-1.7423561811,-1.3032428026,0.6051200628,0.8955559731,-0.1319086403,0.4047618210,0.2238435596,0.3296229839,1.2859840393,-1.5069984198,0.6764607430,-0.3820089698,-0.2242589295,-0.3022497296,-0.3751471043,-1.2261961699,0.1833391935,1.6709430218,-0.0561330207,-0.0013850428,-0.6872990131,-0.1174745485,0.4661664367,-0.3702424467,-0.4538040459,0.4032645524,-0.9180047512,0.2524966300,0.8203217983,1.3599485159,-0.0903820097,1.3675972223,1.0344098806,-0.9962126613,-1.2179385424,-0.3049636483,1.0289355516,-0.0722870082,-0.6006575823,1.5522432327,0.2869044840,-2.3205943108,0.3171606362,0.5200406313,0.2256086618,0.4497120976,-0.0672756061,-1.3183958530,-0.3707039952,-0.9456157684,-0.9327409267,-1.2630683184,0.4524890780,0.0978961438,-0.4481653571,-0.6493379474,-0.0234231055,1.0791947842,-2.0042157173,0.3768765330,-0.5457119942,-1.8845858574],[-1.9457030296,-0.9127835035,0.2195095569,0.3930629194,-0.9389815927,1.0170209408,1.4229835272,0.3960865736,-0.5914026499,1.1244192123,0.7553957105,0.8674074411,-0.6564636827,-2.8345544338,2.1167910099,-1.6108783484,-0.0357680731,2.3807454109,0.3305767477,0.9492464662,-1.5023965836,-1.7776669264,-0.5327028036,1.0907497406,-0.3462494612,-0.7946363091,0.1979672909,1.0819351673,-1.4449402094,-1.2105430365,-0.7886692286,1.0946383476,0.2348215282,2.1321535110,0.9364457130,-0.0350951776,1.2650778294,0.2114970088,-0.7049213648,0.6799748540,-0.6963266730,-0.2903971076,1.3277827501,-0.1012814865,-0.8031414151,-0.4643376768,1.0217906237,-0.5525406599,-0.3868708611,-0.5102927685,0.1839254946,-0.3854897618,-1.6018360853,-0.8871809244,-0.9327890277,1.2433193922,0.8126740456,0.5872593522,-0.5053583384,-0.8157915473,-0.5075175762,-1.0518801212,2.4972004890,-2.2453217506,0.5640085340,-1.2845523357,-0.1043434888,-0.9880019426,-1.1776289940,-1.1401963234,1.7549861670,-0.1329884231,-0.7657021880,0.5557869673,0.0103493147,0.7200337648,-1.8242566586,0.3036039174,0.7726948261,-1.6615983248,0.4481952786,1.6961815357,-0.0148577038,0.8214059472,0.6705704331,-0.7075057030,0.0397667363,-1.5669946671,-0.4513030350,0.2656879723,0.7231004834,0.0246121250,0.7199837565,-1.1029062271,-0.1016972736,0.0192793850,1.8495912552,-0.2141666561,-0.4990166426,0.0213512238,-0.9191134572,0.1927538514,-0.3650552034,-1.7913275957,-0.0585865527,-0.3175430894,-1.6324232817,-0.0671341568,1.4893559217,0.5213037729,0.6119272113,-1.3414967060,0.4768983722,0.1484495848,0.5290452242,0.4226286113,-1.3597806692,-0.0414008126,-0.7578708529,-0.0500840954,-0.8974009156,1.3124703169,-0.8589723706,-0.8989421725,0.0745864064,-1.0770990849,-0.4246633053,-0.8299645782,1.4111720324,0.7858038545,-0.0574695170,-0.3912170529,0.9409176111,0.4052040875,0.4980524182,-0.0261922367,-1.6882300377,-0.1124659851,-0.5324898958,0.6450552940,1.0118424892,-0.6579510570,0.4683852196,1.7358789444,-0.6677127481,1.6819217205,-0.8525858521,0.0229597557,-0.0111456122,0.0114989001,-0.8376780152,-0.5911831260,-0.6677202582,0.3269625902,0.3300351202,2.2259442806,1.3709889650,-0.5098432302,0.3248696029,0.9971179962,0.0306018237,-0.0696415752,0.0515749417,0.8672766089,-0.8483205438,-0.3256694674,0.4704331458,0.3114470840,0.2395827621,-0.3698011637,0.9725357890,2.1338682175,0.4064154923,-0.1931767017,0.7557402849,-0.5391326547,-0.7496903539,0.0328087471,-2.5827965736,-1.1539503336,-0.3479618430,-1.3533889055,-1.0326430798,-0.4367483258,-1.6429653168,-0.4060717821,-0.5352701545,0.0254052076,1.1541839838,0.1725044101,0.0210620221,0.0994544551,0.2273927778,-1.0167386532,-0.1147753224,0.3087512553,-1.3707599640,0.8656529188,1.0813760757,-0.6313759685,-0.2413377911,-0.8781903386,0.6993804574,-1.0612223148,-0.2224770039,-0.8589199185,0.0509542786,-1.7942292690,1.3264616728,-0.9646064043,0.0598946847,-0.2125230432,-0.7621145248,-0.8877801299,0.9363985658,-0.5256406069,0.2711701989,-0.8014968634,-0.6471814513,0.4722471535,0.9304084778,-0.1753164083,-1.4219198227,1.9979560375],[-0.8565493226,-1.5415873528,2.5944244862,-0.4040322900,-1.4617327452,-0.6834397912,0.3675448895,0.1903115511,-0.8517292142,1.8227236271,-0.5215796828,-1.1846865416,0.9606934190,1.3290628195,-0.8174930811,-1.4013472795,1.0304383039,-2.0473237038,-1.2266216278,0.9674461484,-0.0553525463,-0.2639373541,0.3528166115,-0.1527744234,-1.2986867428,1.2760753632,1.3250139952,0.2053325623,0.0451340154,2.3396248817,-0.2764328420,-0.2595769763,0.3644812405,1.4713219404,1.5927706957,-0.2585726380,0.3083312511,-1.3780834675,-0.3119761050,-0.8402903676,-1.0068317652,1.6815767288,-0.7922866344,-0.5316058993,0.3658487797,1.2978252172,0.4811151326,2.7593550682,-0.0746679753,0.2587164342,0.2756006718,1.4350494146,0.5072389245,-0.1162296981,-0.9474886060,0.2444434613,1.4013447762,-0.4103817940,0.5289435983,0.2461477816,0.8635196686,-0.8047537208,2.3466470242,-1.2791610956,-0.3655510843,0.9380925298,0.2967331707,0.8299861550,-0.4961023331,-0.0748049840,0.0122319832,1.5692596436,0.6904290318,0.7966721058,-0.6579260826,0.9688826203,0.2255816609,1.3891453743,2.0140602589,-0.3067657650,-0.4063031375,-0.8640449643,-0.1435795128,-0.3820254505,0.3595044017,-0.1445668191,-0.3615992665,1.0645850897,-0.9378802180,0.4331079423,-0.4059417248,0.7243685126,1.3852615356,-0.3030982614,0.4410329163,0.1787928641,-0.7994223833,0.2407875061,0.2891204953,0.4128708243,-0.1983989030,0.0941923037,-1.1476109028,-0.3581140637,0.5559626818,0.8924738765,-0.4223148227,0.1047140285,0.2280533314,0.2014799416,0.5407735705,-1.8180776834,-0.0493240692,0.2390335947,-1.0003303289,1.6739857197,0.1615592688,1.5634047985,-0.7905229926,-0.9073001146,0.2242522240,-1.6786884069,0.2149655968,0.0972192287,1.0156652927,0.7010413408,-0.4174773395,-1.0974966288,1.7123051882,-0.7921150327,-1.0455245972,-1.0848560333,1.1173052788,-0.5189002156,-0.7537044883,0.1376898289,-0.2069447041,-0.6780954599,0.7539914846,1.0653154850,0.9853175282,0.7669196725,0.4026255310,-1.7758879662,1.6692508459,0.3019891977,0.6081564426,1.1149623394,1.4333524704,0.4183980227,0.4355461597,-0.5992242694,0.0330897495,-0.8541612625,-0.7199405432,-0.8935744166,-0.1560238898,1.0490932465,3.1709747314,0.1894996315,-1.3484131098,1.2649832964,-0.3007838726,-0.6606085896,0.2098494768,-1.2406245470,0.2224631608,-0.0883755237,0.0983779058,0.3814162612,0.0674922541,0.0163380839,0.2843145132,0.4154006243,-1.0314824581,-1.4299912453,-0.0616380535,-1.4327354431,0.0875314698,0.9387468696,0.6071116924,-1.0481704473,-0.8602624536,0.3283012807,-0.4012978077,-0.3166553080,0.5969064832,-0.9872866869,-0.4012347162,-0.8000825047,-1.0431294441,-0.8570781946,0.6774621606,0.0518203899,-0.8791606426,-0.2311016023,-1.6388072968,-0.7333127856,2.1495745182,-0.0902438462,0.7316589355,-0.0654883757,0.3481692374,0.6632580757,-1.1046166420,-0.0309362579,1.5788651705,-0.7955005765,-0.5664398670,-0.3076912761,0.2690240741,0.5249178410,1.2674117088,0.4994982481,-0.0620531254,1.2591670752,0.7041110396,-1.4956794977,2.5263681412,1.7699214220,-0.1682142168,0.3779101074,1.3243587017,-0.1722007990],[0.7303518057,1.1045784950,-1.0148259401,-0.6023318768,0.9214084148,0.4608144760,0.9237965345,-0.1325680166,-0.2890052199,-1.9986394644,-1.1460003853,0.0470660962,0.8245572448,0.5311783552,-0.1282419711,-0.2717715800,0.2171796262,0.0782111809,1.4045455456,0.1464407742,-1.4812459946,-1.2725580931,1.5187593699,-1.1711604595,0.7644974589,-0.2683727443,-0.1697582901,-0.1341327876,1.2213850021,-0.1928418279,-0.0333192833,-1.5308034420,0.2066905051,0.5310425162,0.2391455770,1.3978962898,0.0551713556,0.2989774644,1.6485040188,-1.5500141382,-0.4558253586,1.4261587858,0.9361291528,0.6783800721,0.8326507211,0.3270662129,1.6315973997,0.3777591586,0.2398671061,0.1589586735,0.1928639561,-1.1570172310,0.7706730366,-0.1304397285,1.8219151497,-0.0756504685,0.4209182858,0.2466021925,-0.6255570054,0.9921368361,1.9050636292,-0.0147772199,-0.3004787862,-0.3550287187,-1.8923618793,-0.1778131425,0.2509981096,1.0547579527,0.9600477219,-0.4164990783,-0.2768229842,1.1239053011,-0.1734638959,-0.5100295544,1.3925184011,1.0375856161,0.0187917911,-0.5937774181,-2.0118803978,0.5897036195,-0.8963696957,-1.9627319574,1.5848205090,0.6479678154,-1.1390081644,-1.2144013643,0.8709617853,-0.8779706359,1.2961498499,0.6164593101,0.5365965366,0.4046954513,0.1914508790,0.8805112243,-0.4540803730,0.0859519765,0.7519465685,0.5629897118,-1.1949868202,-0.5004096627,0.2528035045,-0.4080147147,1.7746585608,-0.3931531906,-0.1622184515,0.7694301605,0.3305327296,-0.1452744603,-0.7564935088,0.3015140593,1.0390964746,0.4790952206,-0.7781835198,1.7367749214,-1.4465779066,-1.5826855898,0.9605572224,0.2258404791,-0.5494985580,-1.0985707045,2.3207998276,0.1170908734,0.5342011452,0.3178851008,0.4348079562,0.5400944352,0.7324240208,-0.3752224147,-0.2916419804,-1.7410228252,-0.7803044319,0.2711127996,1.0450233221,0.5990395546,-0.3406923413,-1.2631728649,-2.7773592472,1.1517339945,-0.5892289877,-0.4484650195,0.1315739751,-1.4055600166,-0.3497821689,2.0234718323,0.5053869486,0.3592491448,-1.5824944973,2.2436017990,-1.4227949381,1.9223247766,-2.1150560379,1.4053654671,1.6180542707,-0.8244091272,0.4225803614,0.5474805832,-0.8137944937,-1.4491176605,-1.3177173138,0.5410082340,-0.0851156041,-0.5643010139,0.9667680264,0.5080679059,-0.7554627061,-1.2012015581,0.5232617259,-0.5375833511,0.0992048606,1.5762989521,0.5023282170,-0.8622670174,0.1606611907,-0.9526449442,1.6085221767,-0.5615787506,0.2072707415,0.3077325821,0.1592504680,-1.9585489035,-1.4464210272,-0.4523502886,0.3194318414,-0.1377792209,-0.9571474791,-1.3484243155,-0.4015575349,-0.4684760571,0.5128364563,-0.3263184726,0.6027076840,-0.5946497917,-0.2559576631,-0.3480463922,-0.7823669910,0.6251186728,-0.8135960102,-0.5216414928,-0.0731196478,-1.2973796129,-0.3249349594,-0.7113063335,-0.3881541789,-0.0599280037,-0.7999136448,-0.2200757861,1.3086687326,-0.0257985573,1.1452621222,0.3464944363,0.7741606236,-0.7744589448,0.1049071625,0.1339129210,-0.6126257181,-0.8228283525,-1.4902653694,1.4961396456,-0.9724028707,1.3462210894,-0.4674931765,-0.8624932766,0.6225191355,-0.6311919689],[0.5684589148,-0.3328117728,0.4804244936,-0.9681860805,0.8313510418,0.4879726768,-0.9196506739,2.6429357529,0.5401230454,2.2904670238,1.6002677679,-0.1888347864,-0.4122717679,-0.4034591913,-1.8300285339,-0.6958351135,0.2467660308,1.5259575844,-0.7727718949,0.8820565939,-1.2525933981,-0.5863200426,-0.4576405883,0.3718110919,0.4573096335,0.9623417258,0.7708369493,0.2431682199,0.3903649449,1.5885306597,-0.5109261870,0.7747282982,-1.8081439734,0.4113342464,-0.4832495451,0.0025711823,1.0400862694,0.1646438092,0.8851875663,1.4737647772,0.3890939653,1.1710410118,-0.3265609741,-0.0082098823,-0.5226194263,1.0429775715,0.4140913486,-0.5072344542,0.1546688378,1.0415683985,-0.0392679907,-0.9489328265,0.1319117546,-1.9805655479,0.7687706351,-0.4213275909,-0.4693107307,0.8756957054,-1.3651628494,1.9470986128,-0.4802420437,-0.5232509375,1.0212247372,0.7086952925,2.4512298107,-0.2112059891,-0.1204066351,-1.4793159962,-0.3321022689,-0.7214313149,-0.4487670064,-1.7441877127,1.6606075764,-1.4166034460,-2.8022027016,-1.1884244680,-0.6038395762,-1.1495540142,1.0983035564,-0.1378391832,0.0253856052,0.6103917360,0.2860125303,0.9785673022,-1.1094775200,-0.5475180745,0.6659671664,-2.5345544815,-1.3751845360,0.5009922385,-0.4802490473,0.9361075759,0.8091803193,-1.1980929375,0.4066570997,1.2016978264,0.1474343985,-0.9774648547,0.8793899417,0.6354245543,0.5426107645,0.7159388661,-2.9946129322,0.8809375763,1.8081318140,0.4366384745,0.1927289963,0.6964386702,0.3382254839,0.6517812610,0.0014710003,-0.7667048573,-1.0043227673,-0.9981917143,-1.3730425835,-1.0677419901,1.7612661123,0.7540956736,-0.6250274181,-0.3903926909,0.1125575304,-0.6555450559,0.0675168559,0.7776041627,-0.0357427336,0.3360157311,0.8864915371,-0.2721317708,0.2847906053,-0.3093775809,-0.0285288692,-0.3247302771,-0.5288698673,0.1737118512,0.5665453076,0.1463044435,0.4987269640,-0.7379317880,-1.2037352324,0.4170435071,0.6878814101,0.0498572662,1.3480358124,0.9076988101,2.6805708408,-0.2008085102,-0.9988487959,-0.7401368022,-0.5654978156,0.4760313928,-2.1580686569,1.3185510635,-0.2392965853,-0.2467935532,-1.0793431997,-0.1142255515,0.0132397674,-0.1219449267,0.3390592635,-0.5896320343,-0.8958157897,0.5483281016,0.0986674502,0.1971810609,1.0590271950,-1.0225644112,-0.8552404642,1.2572196722,-1.4828833342,-1.3094121218,0.8178618550,0.2382001877,0.1052321345,-0.0916594118,0.0312675461,-0.0921121165,1.3554426432,-0.3981481194,-0.1613735408,1.7944488525,0.0275097024,2.2320163250,-0.1049797013,1.3674149513,-1.6553440094,0.1536444575,-1.5844736099,0.8444542885,-1.2128678560,0.2837695479,-0.2821958661,-1.1582032442,-1.6193599701,-0.5110404491,1.7406294346,-0.2934850454,0.9172215462,-0.0570428669,0.8767267466,-1.8269113302,-0.4031883180,0.9494055510,-0.1632549465,-0.0864552855,-0.4304619133,1.1493793726,0.2975143492,0.0440222770,0.6430545449,0.5882249475,0.2125870436,1.5470315218,-0.0602875352,0.2780810595,-0.6429525614,0.1501152217,1.5877615213,-0.6432576180,-1.1335928440,0.9967596531,-0.1487661451,0.0960042030,-0.0451133028,0.0791217238],[0.8505306840,-0.8391242027,-1.0117740631,0.0849681348,-1.6064397097,-1.3730535507,1.8666831255,0.7574683428,-0.0100564715,1.2380069494,-1.0405992270,-0.3156031370,0.6234536171,0.8906716704,0.5129168630,-2.5412387848,-0.9680821300,0.4770680964,-0.3559514880,2.5402317047,0.9265583158,0.5580818653,-1.1169495583,-0.0352967381,0.2412039638,1.1277836561,0.8811311126,1.0329891443,-0.9239119887,1.4121516943,-1.3804306984,-0.5359145403,0.4307711422,-0.1498915851,-1.0060368776,-0.8215498328,-1.5482543707,0.5319746137,1.2605688572,-0.1003935039,-0.4003488123,-1.4723229408,0.9132019281,2.2113044262,-1.7974557877,-1.0634329319,-0.6795930266,-0.5643178821,0.2273459435,1.6142495871,1.0085972548,0.5275973678,-0.7239286900,-1.1196281910,-0.7967752814,1.5480668545,-0.0617433004,-0.4468362629,-0.1837557256,0.8246182203,-1.3128496408,1.4148740768,0.1564762592,-0.2163439840,0.4428461194,0.2183970660,-0.3441964686,-0.2527106702,-0.8688625693,0.6563907266,-0.5319938064,-0.9562584162,0.1658635288,1.3291412592,-0.0483446233,-0.6081012487,0.4038960338,1.9367125034,-1.4519054890,0.3822027743,0.2050866187,1.1615338326,0.9909091592,-0.1867091060,-1.6845172644,0.8065637946,-0.8351926804,-0.9467403889,1.1483505964,-0.9108504057,1.4028447866,0.3358447254,0.3191184103,0.3072647750,-1.6384236813,-1.7763886452,0.2155530602,0.5680073500,0.0826110318,-0.8215345144,0.0189221036,-0.0820341557,-0.9571580887,1.0139721632,-1.7302761078,0.5887424350,0.3843234181,1.0097118616,-1.0053118467,0.1014071479,2.1711649895,0.6620742679,0.1005812064,0.5391612649,0.0861768425,2.1908979416,0.9836362004,-0.0856149569,0.2523314357,-0.3907980025,1.2098500729,-1.4061048031,-1.6047384739,1.4587147236,2.1531198025,0.4683049023,0.1127379388,0.6572676897,-0.6470535398,0.1712435484,0.0389087051,0.6265642643,-1.5579985380,-0.5070347786,0.8449956179,-0.6755938530,-0.9933613539,2.0420720577,0.0381180011,-0.5789181590,-1.6923704147,0.7293463349,0.6991361380,-0.2987596095,-1.1022301912,-0.0245494228,-0.8358560801,-0.9420936108,-0.1032127514,-1.0513904095,0.2466489524,0.6079925299,-0.8396324515,-1.3682451248,1.5612796545,-0.9402702451,-0.6599426866,0.2130171657,0.5993693471,-0.2563169003,0.4607943296,-0.4009861648,-0.9711706638,1.4263168573,2.4884417057,1.6959695816,0.1418066323,1.8334354162,0.3557035029,-0.4772862792,0.4663795829,-0.0943925083,-0.9831181765,-0.8983219862,0.8020517230,-1.8465319872,0.6041367650,-1.6295835972,-2.1211764812,-1.8388465643,1.9667639732,-0.1962339580,0.0865831822,1.4192550182,0.9341797233,-1.3915052414,0.8690063357,0.1841812581,-0.3416780829,0.0242909137,1.2798119783,-0.8859664798,0.4008856714,-0.0096572367,-1.7971645594,-0.8022531867,0.1932135522,1.2973420620,1.0013309717,0.5972124934,-0.8152756691,1.8012139797,0.2152404636,-1.0063655376,-0.1829049736,0.8962484002,0.0076174983,0.8868646622,1.1036939621,0.4005306959,-0.8577026129,0.1354546696,0.0451658554,1.8593463898,-1.6263219118,-0.1348224580,-0.5840935707,0.3351056278,-2.4375643730,1.1149245501,0.0137484875,-1.8447011709,-0.3611131310,0.6089623570],[-1.5914478302,0.0032222164,-1.0574736595,-0.5559850335,0.0267383829,0.1834502518,-0.4707424939,0.2727963924,0.8179776073,-0.2789142728,1.4315677881,1.4622141123,-0.4287020564,-0.6378405690,-1.6641730070,-0.1265693307,-0.3634377718,0.7790512443,-1.5096615553,-0.2773914039,0.9687443972,-0.7303571105,-0.7623615265,-1.4469403028,2.6205737591,-0.7474731803,-1.3003468513,-0.8038504124,-0.7742950916,-0.2693897784,0.8253722191,-0.2983231544,-0.9228233099,-1.4513385296,0.0218573585,0.0425390750,1.5309323072,0.0924477354,-0.0990083143,-1.0506538153,-0.3059525788,-0.4384744465,-0.3701641560,-0.9592553973,0.5383296013,-0.1424454153,-0.2003534734,-1.7140461206,0.4936440885,0.4870153368,-0.8391293883,0.9901213646,-1.3647582531,-0.0218699090,-0.2712073326,-1.3171747923,0.1897026151,1.7025702000,0.0676342323,-0.4630217552,0.4470241666,0.1057199985,0.0277621318,-0.4255422056,1.4219756126,0.4563633502,-0.5286706686,-0.1080038399,-0.7408667207,-0.6082911491,-0.6407257318,-1.1343115568,0.7772769928,-0.2910414636,0.5541275740,-0.6701259017,-0.0603624955,-0.7110406160,0.7196681499,-0.2484192997,-0.7308735847,-1.6417032480,0.2756665349,-0.7083850503,-0.0157792177,-0.4917300940,0.9541895986,0.5441447496,0.4472121000,-0.6161211133,0.4662900567,1.7148315907,-0.8321860433,0.1723391414,-1.6492170095,1.3985620737,-0.3979120851,0.7825788856,-1.7232282162,1.7975393534,-0.3568715155,0.5456573367,0.1508181989,-0.2554707825,1.6857923269,-1.6480462551,0.2987136543,0.9106456637,-0.0298561212,-0.1181707829,-0.1426877081,-1.2276364565,0.0381273851,0.5127175450,0.0685992241,-0.2722761035,-0.4897250235,-0.2792966664,1.2577441931,-2.0866348743,0.0400714576,-0.3277549148,1.4558079243,0.0554922260,1.4849256277,-2.1238899231,0.4595848918,0.2800578475,1.3905339241,-1.6413486004,-0.1550358087,0.0660602599,-0.4957954884,1.2165777683,-0.3386821747,2.0347626209,1.0541778803,0.9508336782,0.5592989922,-1.0636955500,-0.4310963452,0.5727513433,0.6775570512,1.3071838617,-0.4674410224,-0.8601533771,0.8591042161,-0.8096265793,0.8733118176,1.1997361183,0.4561530352,-0.3575790226,0.0410822257,0.5934659243,0.0101855183,2.1982963085,-0.9906709194,-1.0026686192,-0.9768953919,-0.5895799398,-2.1789314747,-0.6296504140,-0.6532847285,0.0785140246,0.4178005755,-1.2402163744,0.9000542164,1.8022422791,-0.2082851082,1.5743712187,0.1989894956,1.9887318611,1.1172834635,-1.5639046431,0.0186273698,1.0543249846,0.0305465814,-0.0368835293,1.2697647810,-0.7098541856,0.0175156128,0.3236257732,-0.3337909579,-0.0201291032,0.7750232816,0.4328376353,-0.8087175488,-1.1041239500,-0.7891021967,0.0012484558,-0.1599397808,-0.8319575191,-0.5981504321,-1.5200393200,0.4178537130,-0.0400187261,-1.2597873211,0.0286205038,1.3426220417,-0.7399358749,1.3151376247,-0.3234574795,0.1978281736,0.0977508053,1.4015234709,0.1584338397,-1.1419013739,-1.3109704256,-1.5329210758,-1.7119702101,0.0461350605,-0.9583745003,-0.0808116123,-0.7038590312,-0.7707843184,-0.4808453321,0.7035855651,0.9291451573,0.3711725473,-0.9898225665,0.6436312795,0.6888966560,0.2746472061,-0.6036204100],[0.7088595629,0.4228185713,-3.1168565750,0.6444520354,-1.9137426615,0.6635615826,-0.1540724039,1.1936116219,-0.0981612131,-0.8866142631,-0.1473536640,1.0598063469,0.0262466185,-0.1143351570,0.7435535192,0.2103593647,-0.0059274058,1.3660600185,1.5551140308,0.6133262515,-0.2859591544,1.4969110489,1.1831195354,0.7188971639,-1.2160766125,0.1406719089,-0.7436721921,-0.1590122581,0.2400569320,0.1001594067,-0.4751751125,1.2729537487,-1.6961312294,0.7301835418,-1.8574832678,0.3825981319,-0.8869042993,0.8783037663,0.0864525214,0.2477063835,-1.0182793140,-0.6545701623,0.2072173953,0.5835699439,2.9290962219,0.2228583246,0.9760375023,-1.5569338799,-1.3298919201,-0.3554947674,-1.1974277496,1.4863992929,-0.4102186859,1.3821818829,1.4867824316,0.0427797213,0.5017997622,-0.0560994744,0.5384370089,0.4833418429,-0.1236496270,0.5049699545,1.7236962318,0.7130162120,0.3257996142,0.1247695237,-1.0126731396,-1.0272969007,0.3233565390,-1.3693910837,-0.7663276196,1.2815113068,1.9142297506,-1.6659560204,1.6266496181,-0.2114382982,-0.0150050875,-0.1134116277,1.0805441141,-1.6076766253,0.4561636150,-0.9448701739,0.5707885027,1.5427963734,-0.0004173264,0.3741550744,0.4095517695,-0.7995935082,1.5116393566,1.7064682245,0.7017833591,0.0732854307,-0.4618938267,-0.6264902353,1.7108365297,1.4144150019,-0.0636614859,-1.5799305439,-2.8320119381,-1.0834267139,-0.1306203902,1.4006890059,-0.6516562104,0.5048154593,1.3031809330,0.1285363138,-0.1424478740,-1.3087635040,-1.2024753094,0.4160996377,-0.2009075284,0.1225313172,-0.0472777151,0.6641440392,-0.7846873999,-0.3355806470,1.8961821795,-0.7997861505,-0.2815754414,-0.5893867016,0.4447813630,1.0223922729,-0.4982116222,-0.4314143360,-0.2789815962,0.5298337936,-0.7393953204,-0.3759599626,-2.3721938133,-1.3817449808,-0.1124437526,0.8978641629,0.2950757742,-1.0987684727,-1.4002561569,0.1746800989,-1.6528036594,1.0659267902,0.0638961941,-1.6073201895,-0.9659538865,-0.7243112922,-0.7731925249,-1.4899330139,-0.8746625185,-0.6844015718,-0.7112857699,1.1279566288,0.1048278064,-0.9932572246,-0.3346216083,-0.8795570731,-0.3000066578,0.8755091429,0.2522707880,2.2856011391,0.3759274185,-0.9135944843,0.8097407222,1.0799312592,1.0941669941,-1.0942409039,-0.1476374120,1.1318119764,-1.6847289801,-0.4994167686,-1.4269376993,-0.9325702190,-1.0124571323,1.2505698204,-0.2345380336,-0.8633555770,-1.0356057882,0.1416671723,-0.0111356275,1.3440743685,0.5000166893,-1.4317977428,-0.6289806962,1.0700725317,-0.6210827231,1.7345721722,-1.0982894897,0.5726133585,-0.8612155318,-0.5095951557,1.0985816717,-0.1270671636,0.8134522438,0.4732905924,0.7538656592,-0.8881881833,-0.2215743959,0.4242526293,-0.8490728736,1.6295000315,-0.7772280574,-0.3000035882,-1.0065590143,-2.1433081627,1.7969185114,-0.2043389380,-0.4479148388,-0.1987150609,1.4198639393,-0.9651066065,0.6795678735,-0.4237882495,-0.5966708660,0.5670582056,0.9882405996,-0.5139029622,-0.7688491344,-1.1690957546,1.1035038233,-0.5752559900,-1.8491307497,1.4099521637,-1.3698594570,0.7794605494,0.1834286451,0.2879154384,-0.5843752623,0.3655914664],[-1.6677799225,0.5880377293,1.5570100546,0.8840271831,-2.0195400715,-0.9842090011,-0.1877949238,0.4869373143,-0.1066526771,-0.4932143986,0.5953003168,1.1641517878,-0.2322940081,0.7289298773,-2.5790507793,-0.9375093579,-0.3212589324,-0.4885662198,0.3327982128,1.0137505531,0.5066690445,-0.6222254634,-1.5227681398,0.5569640994,-1.8381767273,0.6530373096,-0.1884490848,-1.1758350134,0.2872573137,-0.0028761027,-0.0365972929,-0.0842233002,0.4195241034,0.9244340062,0.4966152012,1.0121332407,-0.0441397205,1.6184593439,0.5711098313,-0.5436940193,-1.0938950777,0.2057968080,-1.3065215349,-0.9733759761,0.2390870750,-0.6078874469,-0.9333162308,-0.0344750471,0.0726778954,-0.2058340311,-0.3775469065,0.8546427488,0.3424273431,-0.2234261185,2.4643218517,0.1938317418,1.1320050955,-0.5609809756,-1.3629409075,-0.7917565107,-0.2680097818,-0.4966081977,1.3363862038,-0.1200411245,0.4614688754,-0.0464811549,-0.4335543215,0.0379960127,1.7140514851,-0.7679485679,0.7669904232,-1.0260072947,-0.4596264362,0.0035832059,0.3263750970,1.4831286669,-0.0500826426,-0.8436155915,0.6500419974,-0.3641698062,0.2386815697,-0.1162224412,-1.9434568882,0.5082991719,0.5833680034,0.9266047478,1.8004627228,-1.1951037645,0.5165074468,0.4092949927,-0.4190819860,0.3971062303,0.4996469617,-1.2186838388,0.2462227643,-0.9179843068,-0.6518564820,-1.7747448683,-0.4733609259,-0.2035706788,0.5498568416,0.0008999266,-1.5422881842,0.8621480465,-0.1185866222,0.4883705974,0.9659361243,1.4226047993,1.9612269402,-0.0722387582,0.3111244440,-1.0783610344,1.0616002083,-1.1848874092,-1.8052517176,0.8303859830,-0.5216965079,0.7776072621,0.4080746472,-1.6300026178,-2.7196793556,-1.0966017246,0.0164914876,-1.2217763662,-0.6527614594,-1.4589407444,0.1698779613,0.0908259302,-0.4813926220,1.3970652819,1.4977149963,0.5652672052,-1.7997711897,-1.1046901941,0.4071303308,-0.6285575628,-0.4870914221,0.8989673853,0.5108748078,1.3141543865,-0.4292092919,1.3752254248,-0.5541312695,1.4994914532,0.1058346406,-0.8605097532,-1.6312195063,-0.3014723063,-0.2562327087,0.8576619029,-0.1105905026,-0.4324319661,1.0770374537,-0.2248265594,-0.5762417912,0.5746089220,-0.4898282290,0.6588021517,-0.5969170928,-0.2229591757,0.1521769762,-0.3741263151,-0.0134514691,0.8154719472,0.4106017947,0.4809698462,-0.6354304552,0.8528297544,0.6695623398,1.0044192076,-0.7263658047,-0.1724586040,0.6335338950,-0.6088151336,-0.2261224687,1.9258056879,1.9517610073,1.2399405241,0.9385851622,-1.0192511082,0.5125622153,-0.3591165841,-1.0585719347,-0.5090058446,0.1156650707,-0.5473555923,-0.5507994294,0.7920414805,0.1441064924,0.2334580868,0.1118723974,-0.6757031679,-1.3705719709,0.3105646968,-0.5070366263,-2.0107822418,-0.3925672472,-1.0922179222,0.6986502409,0.5216252208,0.4968931377,-0.6650416255,0.7315515876,0.3196497858,-0.4098545313,-0.4533374310,0.8927081823,-0.4736040533,0.3036564589,1.0339570045,1.9093426466,1.6638730764,0.9008227587,-1.5059113503,-0.6890484095,-0.5480871797,1.6531498432,-0.6993179321,0.3861663640,0.1008670628,-0.9351271987,0.3818240166,0.3982960880,-1.2557748556],[1.2228775024,-2.0865099430,-0.5907571316,0.9719703197,-1.1932578087,0.3502659202,-1.2963603735,-0.0930241421,-2.3137731552,-0.8425716758,-1.5429214239,-0.4017637372,-0.4152314067,-0.6736641526,0.7979131937,-0.8868796229,0.6343866587,1.6292757988,0.1390641481,-0.8576701880,-1.2493385077,-0.7097851038,0.7046427131,0.1555907279,0.9367952347,0.7703309059,0.1408106536,0.4734882712,1.8552461863,1.4156562090,-0.3027460277,0.9896794558,0.5858508348,1.1363880634,0.6716165543,-0.9741674066,-1.6196845770,0.5726270080,1.9026181698,-0.7756640911,-0.1880897433,-1.0357477665,1.1778295040,-2.3051669598,-2.2636601925,0.3750199080,-0.0823436454,-0.4796230197,-0.3010948002,0.5369879007,-0.4138039947,-1.0969250202,-0.9273629189,0.8883388638,-0.5247419477,-1.3852776289,0.1021783277,0.5049947500,1.3289607763,0.2179033905,-0.6597112417,0.4740078747,0.7271748781,-0.0389053077,-0.0445993915,0.2601329088,-0.0698564947,0.2501139045,-1.0219132900,-1.1504377127,-0.8361113667,0.6422109604,0.2587975562,1.0402389765,-0.1866909266,-1.1436413527,1.1445535421,-0.0187670551,1.2834550142,0.5979464650,2.1886186600,-0.2197729796,0.9007239342,0.8913640976,-0.5551263690,-0.1724823117,-1.4617383480,-1.5487961769,0.1265687943,0.7930070758,0.6380240321,0.3400245905,0.8630171418,-0.5896977782,-0.2725327611,0.7375215292,0.4331187308,-0.2101888210,1.3207943439,-1.2920012474,-0.5186786652,-0.2833977640,0.8165348768,0.0023851979,-1.2614917755,0.5140041709,1.0875463486,0.7393045425,0.6191549301,-1.8743134737,-0.8998864889,0.4820806086,-0.0548881851,0.5225576162,-1.2663426399,-0.0614947639,-1.3897809982,-1.9536786079,0.2957790792,0.8425887823,0.2456164211,-0.0329964794,-1.5620143414,1.0061070919,-0.0440448970,1.9595620632,0.9423143268,-2.0051255226,0.7550497055,-1.3965352774,-0.7594954967,-0.2507566810,-0.0940624475,0.3975652158,-1.0228550434,-1.1506919861,0.6006051898,-0.0132502681,0.1743730456,-2.1936833858,-0.1771373898,-0.8907291889,-0.9206264019,0.9219347835,-1.0956711769,-1.0928965807,-0.3310106099,0.4502888322,-0.8840147257,1.2341440916,1.4498475790,-0.8814470768,-0.2450817525,-0.7786754966,-1.6853821278,0.3030110598,0.7335948944,2.0118641853,-0.8974094987,1.3362350464,1.3423537016,0.1978533119,0.6021634936,0.8732730746,1.9740999937,0.4778085649,-0.0601378866,-0.8661687970,0.3053207695,1.0241649151,0.2446103543,-0.7799232602,0.0890762061,-0.1291534752,0.2647387683,-1.6618484259,0.5507886410,0.5954231620,0.4448534250,-0.0037628172,-1.8059362173,-0.0193227921,1.0607149601,-0.8601288795,-1.9892694950,-1.5405579805,0.3140257001,0.3728760183,0.8862931728,-0.0552589968,-1.5003284216,-0.8185041547,0.8188393712,0.1404959112,0.6498296261,0.4347887933,-0.2049605548,-0.1740068346,1.8571022749,0.4146742523,-0.1285875440,0.4554199874,0.2229058146,-2.1573562622,0.6500844955,1.8209393024,-0.7802798748,1.4540357590,-0.2568697035,0.2934713960,1.0703600645,-0.7200014591,1.2424938679,-1.2142173052,-0.8751547337,-0.5935203433,0.6620053649,-0.3408744037,-1.5199744701,-0.2165328711,-0.7842214108,0.7312936187,-0.3432350457,0.0707740784],[-0.4054724574,0.4339389801,-0.1835907698,0.3251987100,-2.5933885574,0.0972508788,0.4139136672,-0.1992800534,0.6693924665,0.7386070490,1.3042138815,0.1048116088,-1.9138007164,-2.2854993343,-1.6018409729,-0.0379070602,-0.1573052853,0.2762398422,-0.6252459288,-0.7364911437,0.5550479293,0.6559244394,-0.2566501498,-0.0384766571,0.4043143392,0.5043435693,-1.1439807415,-0.7195738554,-1.2305459976,-0.5069066286,0.8123335838,0.5462718606,-1.0980979204,0.5122667551,0.0858431086,-0.4939267039,-1.4064596891,-0.1748233736,0.6799439788,-2.1630976200,-0.3961232007,2.2542836666,0.6726367474,0.2598325014,-0.7371851802,-0.6783298254,-0.0832883939,1.6028636694,0.4655891955,-0.8721584082,1.1767870188,-0.2925941944,1.6973464489,-0.5666030049,-1.0032657385,0.1746295840,0.9823269844,1.0374448299,0.1591917723,-0.9880967140,-0.5053406954,-2.0182819366,-0.9131215215,-0.1784568131,0.3890021443,-0.3394543231,-0.0569790564,-0.3961854577,0.7510253191,-0.8991129398,0.8375478983,1.9608807564,0.4727896452,-0.5270916224,-0.5362701416,1.2098371983,-1.1265894175,-0.9538044333,-1.1644484997,-1.2785137892,-1.0448163748,0.7899049520,1.1022825241,-0.6970731020,0.2073340416,0.7591567039,0.1005642042,-0.9549427629,-1.4704017639,1.0104275942,0.4961794019,0.5769559145,-1.1076469421,0.2349771857,0.6289995909,0.3140338361,-0.7450231910,1.0122605562,-1.5276319981,0.9287419319,1.0810559988,1.5723303556,-0.3424921930,-0.9994300008,0.7938803434,-0.6992152929,0.0439955108,-0.3174622059,-0.9020719528,0.3209994733,-1.3920159340,0.5922057033,-0.9669311047,-1.7317312956,-0.0501074605,0.4316338599,0.5769345760,0.8183537126,-2.3536403179,-1.0051444769,0.1066522971,1.5190032721,0.7837445140,1.9013400078,-0.5249394178,0.2744169831,-1.0999708176,-0.4043522179,-0.7352957129,-0.6339886785,-0.3934491277,0.0027175399,0.0222126637,0.5434534550,0.1399884671,-0.3440456390,-0.5225785375,-0.3071317077,-0.4490371346,0.4909710586,0.8655251861,1.2740445137,-0.7977027893,0.4693722129,-1.3946796656,0.3731747270,1.0826722383,-0.1495895088,1.0726360083,-1.1385679245,-0.8886452913,-0.1358098388,1.0222103596,-0.4174294472,-0.4535531104,-0.9916283488,0.2028810382,1.2466951609,0.7006801367,0.6966506839,-0.2069744766,-0.5633093715,0.6772459149,-0.0319110751,-0.1736082286,0.8982406259,-0.1977874488,-0.8377762437,0.9091885090,0.0807198882,-1.0370293856,-1.1129058599,0.0954118744,2.3374097347,-0.3928205967,-0.3362738490,1.5237711668,-0.0572811998,-1.4484668970,-1.5727964640,1.2266639471,0.6663545370,0.8261256814,-0.0577565581,-0.7267120481,-0.2171631157,0.1360312104,-0.8383111358,0.5614498854,-1.2595961094,-0.3327587545,-0.2040078789,-0.6910198331,-2.2055053711,0.4478696585,-0.7557507753,1.3257079124,-0.3419822752,-0.5413596034,0.0915219486,1.0534397364,-0.5634076595,1.0147377253,1.4403036833,0.9903228283,1.6264314651,1.2926460505,1.5148823261,1.6043263674,0.2080695331,-0.4292238951,-2.2622437477,-1.3227331638,-0.4482828081,-0.3817350864,-0.1527944654,-1.0007604361,-1.5957776308,-0.1302231699,-0.1894179285,-0.8075540662,-0.7421521544,-0.9401565790,-0.3965237439],[-0.8563027978,1.2598752975,0.2409967333,-0.9723179340,-0.2804477811,-1.1802855730,1.0121682882,1.3841867447,1.2520020008,-1.1446926594,-0.0912670195,-0.4015706778,0.5620130897,-1.0079097748,-0.6758916974,-0.4132170379,0.1532884687,0.6941286922,-0.3287276924,0.6639650464,0.8220763803,-0.2132152319,-1.2456581593,-1.1711903811,0.5917269588,-0.4762244225,-1.7126293182,0.6129523516,0.1295545250,-1.4059671164,1.1794199944,0.8366360068,0.1387452483,-1.2743194103,-1.4023305178,-0.3070684969,-1.7139153481,0.4050802588,-1.4108233452,0.1649127305,-0.2881314456,0.7117852569,-0.9379475713,0.2737294436,-1.3948402405,0.7955495715,-0.1149617657,0.4958506823,-1.3205252886,0.4990842640,0.3062033951,0.3636978865,0.3126339614,-0.1934638768,1.2412992716,-0.1558979899,-0.7391691804,-0.0587261915,-0.9505179524,-0.4639964104,-0.1772466153,-0.3795541227,0.1993970722,1.9457614422,0.5709498525,1.0723006725,-0.5037094355,-0.5870162845,-0.3781780601,0.8528891206,-2.1481184959,-1.0331647396,0.1023358479,-0.2240923643,1.9677296877,0.4476832151,-0.6621914506,-1.5776070356,-0.3405600488,-1.3032200336,0.4667506516,0.1611063182,0.3200319409,2.0791766644,-0.9074659944,-0.1924042106,-1.2125157118,-0.0805985183,1.5932736397,0.5687224269,-0.1144870445,0.2516302466,-1.2108556032,-0.3937337101,0.0852525756,0.0994219854,-1.5306162834,0.3276231885,0.2791965008,-0.3770512044,0.0041749990,-1.4834915400,-1.4797955751,0.1346872598,-0.6677231789,-0.0115555199,0.8394906521,-0.1739299297,-2.8106679916,-0.1506536454,-0.4810440242,-0.2346943617,0.8997308016,-1.5785301924,0.2439566255,1.5703039169,-0.6259431243,0.4723278880,0.9663057923,0.2102314383,-0.6850969791,-0.7095209956,0.7438001633,0.5921490788,-0.7864683867,-1.1764731407,-1.2808066607,1.6616518497,-0.0679451227,2.3602285385,0.5555456281,0.4395223260,0.3062724769,0.9991498590,-0.9660632014,2.1600131989,-0.1003017053,-0.7034000754,0.3025610149,1.0923389196,-1.0075548887,0.5668693781,-0.7164441347,-0.5062735081,-0.4894824326,0.7635414600,-1.1090726852,0.1926161051,-0.3434178531,-0.8472101688,-1.2135236263,-1.2028883696,-1.6337959766,0.8961672187,-0.2416531593,0.1586519331,1.1781893969,-1.2201172113,-0.9415456653,0.2547155321,-1.8240795135,-0.5787085295,-0.9248930812,0.3295224309,-0.4258158803,2.0081493855,0.9378913641,-0.8532384634,-0.3873134255,-0.3475845158,3.3065743446,-1.5101996660,0.2035396993,-2.0844321251,-0.0069374414,1.9098905325,-0.4084554315,1.1045544147,-0.0661152229,-0.4224987030,-0.2516563535,-0.5869026184,-0.6260582805,-1.3301943541,1.5068007708,-0.3930763900,0.2937743068,-0.8765318394,1.1169905663,-0.2735557854,-0.0910326689,-1.8289766312,0.3959762156,1.8115056753,-0.8690775633,-0.4582291543,-1.1383239031,0.1291621774,0.0640241951,0.7050811052,0.5514735579,-0.8125160336,0.2249480486,-0.3283011019,-1.0910329819,-0.1268558800,3.8016602993,2.3151705265,0.1398265958,1.7388571501,-0.0453833640,-0.0531383380,-1.9495717287,-0.9601055384,-0.7834992409,0.1075190306,0.0139845349,-0.5789423585,-0.5888131857,-0.1661531329,-1.3814116716,-0.6126385331,-0.3812898695,-1.2489489317],[-0.3302378953,-0.8348071575,1.2353824377,-0.2438037992,-0.1895456612,0.4280281067,0.5569683313,-1.7362418175,-0.3767841160,-0.9090323448,-0.1451702416,-0.5363325477,0.1570694596,-0.9804592729,-0.5677672625,-0.5911596417,1.0825914145,0.3680036664,0.3688887060,-0.2863182724,-0.3847178519,0.5610029101,0.7774339318,0.0151467854,1.1416479349,1.2741550207,-1.6646980047,0.4303788841,-0.0426019281,0.3882888258,1.1159765720,-0.9205381870,-1.6202740669,1.1061915159,-0.9984846711,-0.6862195134,0.2046208978,-0.6861017942,-1.5922106504,0.0341897681,-0.7814846635,0.5978598595,-0.5060765743,-0.6884461641,-0.2100005448,1.0521534681,0.9079040885,-1.0932261944,2.7997076511,-0.3257763386,-1.1524157524,0.8882319927,-0.3616724610,2.1537194252,0.8474083543,-0.1987198442,1.5753068924,0.8491151929,-1.2288951874,0.8883941174,-0.5164874196,-0.0833262876,0.1310544461,-0.8790960312,-1.3333423138,0.3677840233,-1.3882335424,-2.5752027035,-0.8361055851,0.3310924172,-0.2698811293,1.2671309710,0.1837534904,-0.7663096786,-0.4395835400,-1.4365413189,1.0857971907,-1.3811000586,-0.9204076529,-0.1602862179,0.0023532645,-1.5026503801,-0.9055358171,0.2650406063,1.1297233105,0.3490035534,-0.0258097611,-1.5624086857,-0.6173423529,0.5214942694,1.0809466839,0.8893759251,0.1380716413,1.2046004534,2.8814606667,-0.5938619375,-0.7631158233,1.5184829235,0.2354645282,0.1123076901,0.3923743367,-0.6544864774,-1.0347952843,-0.7771475315,1.2459462881,-1.4366406202,0.4986546338,-0.5576874614,-0.3533668816,0.7429509759,0.8439888954,0.3429765403,-1.8731197119,1.5709646940,1.3101965189,0.0914368331,0.0102578169,1.8014491796,0.9472242594,-0.0292944033,-0.2923386693,-0.1935371161,1.1772320271,1.0399917364,-1.6134229898,0.4646424055,0.8641213179,-1.5064631701,-0.0029647513,-1.7770435810,0.1294928342,-2.0832345486,-0.6817455292,-0.6110659242,-0.7088498473,1.4515280724,0.5355105400,-0.3995688558,-0.9330778122,-0.2387763113,-1.0291129351,0.9730799794,1.9967659712,1.0531998873,0.3316903412,-0.1656287760,-0.4051062763,1.7452845573,-0.5759356022,1.5610985756,-1.1315392256,-0.2962316573,-1.7140566111,0.1592341959,-1.2637276649,1.6650494337,0.4122722745,0.5373967290,0.2826784551,-1.0925408602,0.1241182908,1.8370807171,0.0085549261,-1.0170161724,-1.8523426056,-0.7133269906,-1.7622288465,0.8305173516,0.7811672688,-0.8756818175,0.6139813066,-0.5764546394,-0.0456142835,0.3719555438,-0.4439600110,0.4182033539,-1.6857280731,0.1174749881,-0.0349520221,-2.0463931561,-1.8096902370,-1.8595236540,0.4143068194,0.1239596233,0.2739575803,-1.3263784647,1.1389738321,0.9828411937,-0.7669630647,1.1760603189,-0.2509224117,-1.7762050629,-1.6326947212,0.7337234616,-0.1040488109,0.8812249303,-0.0883731246,0.2676708996,2.1235263348,1.3968490362,-0.4328272641,0.3749687374,0.4944454432,0.7613911629,0.0710088089,-0.4935318530,-0.0036228173,-0.4802871048,1.6833672523,1.2407262325,-0.2036150247,0.4282922745,-0.1654592603,1.1932411194,1.0488804579,0.5686108470,0.8712642789,0.6605708003,1.1740618944,0.5311313868,0.1519005299,-0.5772256255,-1.5717507601,-0.0278483797],[-0.7410555482,0.0600090660,1.1404883862,0.1728246808,-0.4150016606,-0.8531286120,-1.4301352501,1.3328052759,-1.7766909599,-0.9347830415,-2.3132019043,-0.3161415756,-0.3422845602,-0.4042944312,-0.0631299019,-0.8212651014,-0.9136556387,1.8178263903,-0.3340629339,0.9076586366,-0.8367710710,1.6127285957,1.5141820908,0.2310186774,-1.0995316505,0.0870013833,0.0473044999,0.2396239191,-0.9782206416,-1.5230001211,0.1623630375,-0.0102913165,0.0020750219,1.0268006325,-1.4751604795,1.0106936693,-0.7432275414,-0.3952220678,-0.8257793784,0.0896198601,-1.9058178663,-0.5680857301,-0.5157565475,1.2639302015,0.1506981403,0.6955183148,0.0059388145,-1.0489003658,0.9072048664,-0.8454413414,-0.5262433290,0.1820997745,0.9455388188,-0.2013845444,1.5105247498,-0.5714784265,0.6655892730,0.0036163009,1.5466718674,0.2144060135,-1.8773127794,1.0883351564,-0.0815485120,-0.5530619025,1.2229647636,-0.3313086331,0.5999867320,-0.7683833241,-0.8361301422,1.8105818033,-0.7870327234,-0.5847709179,-1.7083207369,1.6299822330,0.3979983330,0.2377796322,0.9751384258,-1.3293365240,-0.5410467982,-0.0914377347,-1.5484710932,1.3114271164,-0.0184290502,-0.3232886493,0.2362254858,-0.7525823116,0.0451130047,3.4275386333,0.6046820283,1.6683111191,-0.3550831079,-0.7515689731,0.3097035885,-1.3417049646,-2.3069577217,0.7315924764,0.6413381696,0.8338512182,0.0281698741,1.9783726931,-0.0873281881,-0.5539647341,-3.0064988136,-0.0471658669,0.8318777680,0.0068611987,1.1242216825,2.2948811054,-0.1733502150,1.2312535048,-1.5858526230,1.0313191414,0.0634904802,-0.2213905007,-0.1633989215,-0.1563034654,-0.3088029027,0.1986729652,-0.1742921323,-1.1557924747,0.4170538783,-0.6078679562,1.0479866266,-0.0338269658,0.1270239502,-2.0492320061,-1.2566801310,0.9396144152,-0.7338167429,-0.5324376822,-0.2779399753,1.3637425900,0.3741379976,1.3102645874,-0.2677477896,0.2131762654,-1.2032434940,1.1780312061,0.1086482033,0.0441290997,0.3383155465,1.4467921257,-0.2144951075,1.6630389690,-0.8515225649,0.4221846163,2.0092184544,-0.4898147285,0.2452558577,0.8775050640,-0.1378996968,-1.5003532171,-1.0559593439,0.5809326172,0.8915153146,0.7845553160,1.1464320421,0.0719851926,0.2082331777,-1.5188686848,0.3173289597,0.6126807928,-0.5832113028,0.6440017223,-1.8158888817,0.7510995865,0.3002843261,2.1106085777,1.4130855799,1.5069802999,0.8173971176,0.6466156244,-1.1816313267,-0.3350912929,1.8267284632,-1.4561644793,-0.4502818286,-1.4192340374,1.4509518147,-0.5657813549,1.5445343256,-0.4137624800,-0.5041320920,1.2785291672,0.9388317466,-2.7162802219,0.4511407912,0.6001668572,0.2098069340,-0.6576579809,0.0284084454,-0.3980614841,0.2113230228,-0.2023942620,-0.6219281554,0.1637704521,0.8024389148,0.2890059054,-0.5536423922,0.3362540305,1.0697923899,1.5954041481,1.2075525522,0.5373802185,-1.0091240406,-1.3655527830,-0.2023812085,-1.4091848135,-0.7847847342,-0.1701223105,-0.4842104316,-0.3279180527,-1.3280045986,0.2314667553,0.9965080619,-0.5481374860,0.7257553339,2.6627266407,-0.0918110311,0.6512100101,0.1967700869,0.9696237445,-1.7186498642,-1.0569567680],[0.1434639245,0.8869625926,0.1305240244,-1.6645731926,-0.8236132860,-0.7947061062,0.3889919519,-0.7620389462,-0.6808071136,1.0847475529,1.3353163004,-0.4132747948,0.4249026775,-1.8814837933,0.1983270645,1.1899780035,0.5267816782,0.1010608450,-0.3886429667,-0.6467919946,-0.1790824085,-1.5514411926,1.6104589701,0.5642106533,-0.1024370342,-0.6198047996,-0.0703396201,0.7977949381,1.0114479065,-0.9036909342,-0.9735287428,2.0782299042,1.1365928650,0.7085199356,-0.1386462450,0.9240203500,-1.2732331753,1.5317684412,-0.0357722230,0.7908614874,0.6462177634,-0.1315709054,-0.1753663570,1.2215831280,1.0264973640,-1.7722311020,-1.6924057007,-0.9462206364,-0.8935453892,-1.1185258627,0.2721106410,-0.4637044370,1.2061246634,1.4528777599,-0.0286832377,1.6834579706,0.0242143106,-0.4347906411,0.0647857413,0.9448599815,-1.6144609451,-0.2085993290,0.2974056602,0.3630846739,-0.3684363365,0.4887857437,0.2921216190,-0.5919080973,2.1815986633,0.4395501912,-0.3311833441,-0.5717190504,1.0294089317,0.1020597890,2.5481126308,-0.4359241128,-1.2426069975,-0.0276984591,0.1750674546,-2.1184506416,-0.3091685176,-0.3684154451,-0.3687635362,-0.6302257180,-1.3431925774,0.7580380440,-0.5838408470,-1.0237014294,-0.7599342465,-0.4723232388,0.1086471230,0.6683390141,-0.9531794786,-0.4792973995,-1.3455077410,-3.3922998905,0.1557939351,1.5200035572,0.5220832825,-0.5070599318,0.0964791402,-1.1748200655,-0.1222923547,-0.4277092516,-0.8527142406,0.4056522250,2.5998671055,1.6654495001,-0.0720730200,0.8841146827,0.8627074361,-0.6475380063,0.6439040899,-1.4409921169,-0.8052983880,0.2387528718,-0.4147876501,1.7564786673,0.6480404139,-0.3820381165,-0.4705797136,0.1869706064,-1.0555312634,0.5956119299,-1.3753019571,0.6230102181,-0.1645947248,0.4146135151,-1.0125858784,0.2449852079,2.4123477936,-0.4572167397,0.3173998594,1.5055669546,0.7617041469,0.4318854809,-1.0136892796,-1.2775883675,0.0534324422,-0.4632358551,-0.0190582145,0.2056566775,-0.6764278412,0.4941030145,1.8585561514,-1.0093410015,-0.4695463479,-0.0496106595,1.1404596567,-1.1863820553,-1.0651481152,-2.1636610031,-0.4403622150,0.6801457405,1.0652247667,0.3571536541,-0.6009569764,0.7064716220,0.2043185979,-1.9207055569,-1.2280948162,1.5118652582,0.3222050965,-1.3747943640,0.8199530840,1.0614349842,-0.4350340366,0.6576821208,-3.7401006222,0.9735767841,1.1751554012,-1.1247026920,0.2820853889,-0.3381205499,-0.1025294811,-0.4248804450,-1.3322954178,1.8904037476,-0.3103108406,0.1047550440,-1.0094006062,-1.0368671417,0.4125984013,0.5263921022,0.8779241443,1.1037739515,-0.2102075368,-0.4442030787,0.7468138337,-0.6374391913,0.8717585206,0.3745002747,1.1550264359,0.6703916788,-1.0544458628,-0.8656336665,0.7324852943,1.9070558548,-1.3228117228,0.0232116040,0.2816745639,-1.5257774591,0.4781250060,-0.0931227952,-2.0965573788,1.6217279434,-0.8632082343,-1.2825033665,0.4201416075,0.5574867725,0.7364113927,-0.3860033751,-0.0109143378,-0.7308067679,-1.3101973534,1.0791306496,-0.1027626842,-0.1823142618,-1.9992675781,-0.1783711016,-0.8424944878,-0.1746136546,-0.2192441523,-0.4464647770],[0.9388386607,0.4470541477,1.1271544695,-1.3248273134,-0.6489559412,-0.0402808189,-0.4066389799,-0.0792577267,-1.1821033955,-0.7161780596,-1.6415539980,-0.8900255561,0.6941767335,-0.2142068893,1.5057532787,-0.5955338478,0.1190710813,-1.2132523060,2.6006717682,-0.1786205918,0.8296298385,0.4133850336,-0.5838788152,-1.3309012651,0.1561431289,-0.5567897558,-0.1555043012,0.6513020396,0.0782411546,0.3771162927,0.1500465721,-1.4672492743,1.3960622549,1.1758522987,-1.1361649036,0.5053006411,-0.6620242596,-0.7469163537,-0.0048416615,1.7476682663,1.0579575300,0.6052213311,-1.1506056786,2.5544493198,0.8737310171,-2.3488373756,0.3994743228,-0.4886947274,0.4099823534,0.4006403685,-0.9185190797,1.8258857727,0.1997846216,0.9413478971,1.3514236212,-0.7381575704,-0.9117684960,1.1219073534,1.3928374052,-1.3770185709,2.0112431049,-0.2355033159,0.6917845011,0.5643882155,-0.9713423252,-0.8640481234,-2.0835924149,-1.1511501074,-1.4824759960,0.0401905142,1.3694021702,-0.0271449313,0.3388541639,0.7780035138,0.6797094345,-0.3858315051,-1.4633450508,-0.4298055172,0.0629593581,-0.8716452122,0.3619607091,-0.2927120924,0.6218215227,-0.8032394648,-0.9219676852,1.7740563154,0.0287562385,0.5529638529,-1.0984222889,-0.3772644699,0.6821694970,1.5656158924,-0.7244850993,-0.8029174209,-0.0226689409,-1.5243951082,-0.0301333070,-0.0647283867,0.7247491479,1.4146097898,0.5698443055,0.7415510416,0.0522789508,-0.3597445786,-1.9959769249,-0.8862208128,0.2172666788,-1.6455937624,0.2428898215,-0.4008347094,-1.0215598345,-0.4700243175,0.7287815213,0.8855010867,-1.9370213747,-0.1494840086,0.9138846397,-0.2578948736,0.1088152677,-1.4954109192,-0.4800336659,1.8287754059,-0.7880680561,-1.4406323433,0.1494717598,0.7886293530,1.1938104630,-0.5177226663,0.2224755734,0.5443548560,0.6492105126,-0.5472027659,1.7127249241,-0.6872970462,0.7078721523,-0.0219112355,-0.5872185826,-0.6428512931,-0.5863469243,-0.4468710721,-1.0188856125,0.6974096894,-0.7035152912,-0.6150208712,0.4886905849,-0.1079619452,-1.4219036102,-0.9360095263,-0.1965572387,-0.5749878287,0.7504825592,-0.7644020319,-0.9671270847,-1.0105462074,0.4066572487,0.4834717214,-1.6724445820,0.6220752001,0.8609732985,-1.6909977198,-0.6904314160,1.4288923740,1.0061017275,0.0247926600,0.5012493730,2.1120195389,0.5027969480,-1.2208088636,1.3649389744,-0.8709388971,0.9939022064,0.6562706828,0.8895135522,1.5409330130,-1.4659143686,-0.0695885569,1.9460494518,0.9763817191,0.1771583408,-1.0231730938,0.1067204922,-0.9118813276,-1.4683669806,0.5764787197,0.0653056055,-0.7735127807,0.3949481845,-0.5038898587,1.7795591354,-0.0305724442,1.5770882368,-0.8128020763,0.6133491397,1.8436999321,0.2710909843,1.1364476681,-1.7383319139,0.7071347237,0.0303861313,0.7650019526,0.8676652312,-2.2562501431,-0.4436027408,-0.6700232625,0.1521641910,-1.9405333996,-1.0905086994,1.0019207001,0.1768924445,-1.0880144835,-0.2532173693,1.0982730389,-1.8395668268,-0.2114286125,-0.2296632379,0.1869794130,0.5037794709,1.9103424549,0.5537812114,-0.5874814391,1.2579499483,-0.8586683869,0.4361870885,1.5714631081],[1.0773148537,0.8110896945,-2.2315375805,-0.1010025144,-0.5873750448,1.3248683214,0.8406484723,0.2611061931,0.7944416404,-0.6496164799,0.6342844963,0.0950026661,-1.6832067966,0.3440461457,0.7071580291,1.1934145689,0.5273885131,1.0067039728,-1.7323273420,-0.3734121621,-0.1425103992,-0.3297423720,-0.0890421197,-0.5773962736,0.7361654639,-0.9912055731,0.1251746118,0.0731527135,0.1439372301,-0.9477243423,1.3992298841,-0.2261237204,-1.4388542175,0.8013010025,-0.0033314459,-0.0969415605,-0.0958714485,0.3954369724,-0.0532430410,-0.7734996080,-1.4191855192,0.3034519255,-1.5182067156,1.1197077036,-0.9538609385,-0.8496140242,-0.9818996787,-1.3630776405,-0.7725985646,-0.2836254537,-2.3276040554,-2.4452273846,-0.7158649564,0.8833968639,-1.3004398346,-0.0763390809,1.4305567741,-1.3234086037,-0.4383561611,-0.7431524992,0.8919675946,0.4638727605,0.6176608205,2.4964170456,1.6294752359,-0.0990447029,-0.2019919604,-1.4488258362,-1.7141647339,-0.0496415719,-1.2993954420,0.6253554225,-0.7917193174,-0.5829433799,-1.5526804924,2.1101534367,0.7588295341,-0.7099302411,0.1511470377,1.3230912685,-0.9278250933,1.9065986872,-1.0321749449,-0.1773614883,-1.6503783464,-2.5385110378,1.0100908279,0.0857020915,-1.7338609695,-1.6406011581,1.1453614235,-0.1505951136,1.4314432144,0.6365867853,-0.0665628463,0.0323299803,-0.5550736189,0.0977861062,-0.0609849803,0.8375166655,-0.1634184420,0.8235554695,0.9206323028,0.1807626039,-1.3129683733,-0.1604766995,-1.9060746431,-1.2066216469,0.7304183245,0.4948927760,-0.0032088785,-0.3024331629,-0.7394009233,-0.5128120780,0.9652515054,0.4766792655,-1.3712165356,0.1988528371,0.1399628371,1.6486734152,-1.7575517893,-0.7831295133,0.9736258388,-1.1109322309,2.3856215477,-1.1789442301,0.0291221458,0.5595475435,0.8810371161,0.7152084708,-0.4620775282,0.9132069945,-0.7546525002,-0.5349771380,0.4566424489,1.5095769167,-0.2281712443,-0.8903415203,1.2097718716,-1.2648000717,1.8381814957,-0.9840826988,0.6409484744,0.9266912341,0.7850340605,0.2270026356,0.0495295525,-1.6531804800,-0.7808196545,0.7246446609,0.6633691788,-1.0378812551,0.3469790220,0.2520309985,1.7509188652,-0.4184011519,-0.5198572874,-0.9234429598,-0.9992784858,0.3749483526,-0.7043411136,1.0747038126,-0.6272990704,1.5339956284,0.4177254438,0.2583836317,-1.1504285336,0.3291141987,0.0455219783,0.6320825815,-0.5108472705,-1.4536298513,0.2752215862,0.1397872418,0.2438923568,1.0056501627,-0.9396406412,-2.3817501068,0.4750269949,0.4059125185,-0.4770562947,0.1705982089,-1.0477807522,-2.1061971188,-1.6929115057,0.0423611403,1.3827104568,-0.3895183802,0.8139379025,-0.5943319798,-0.0554390214,0.7965607047,0.1331798881,-0.5416783690,-0.8653023839,-0.0925281271,1.1821020842,-1.5706546307,0.8593307734,0.2836470008,-0.9691122770,-0.0183737297,-0.2040354460,-0.9477403760,-0.5394350886,-1.2562873363,-2.0715236664,0.1512364447,1.0444895029,1.6333490610,-1.1113785505,2.1473650932,1.5263067484,1.4234750271,-0.7856664062,-0.5622516274,-1.9383574724,0.1911592335,-0.3936029375,0.1617906839,-0.8345180154,0.6728704572,0.3431881070,-1.1441230774],[-0.0458879471,0.2846845090,2.0084414482,0.0957808495,-0.9404090047,-0.3162630498,-0.0312234666,-0.1335952878,-1.8414719105,-0.3315768838,-0.6933088899,-0.2605129480,2.1209623814,-0.8322905898,1.4379409552,1.1606181860,0.6834978461,0.0031104183,0.6543934345,-0.4499197900,-0.5464591384,-0.7478616834,0.2739015818,-0.2097705752,-0.2395850718,1.4202288389,-0.7047485113,0.7353649139,-0.5219275951,-1.5921950340,-1.4259504080,-0.4915523827,0.6296114922,0.6417863965,-0.2406871468,0.6184217334,0.0762631074,-0.2614903152,0.8547312617,1.1878938675,1.0160274506,-0.3068729639,0.5677075982,-2.1292457581,0.1950669289,0.3610023558,0.1519751996,-0.2222727239,1.3047732115,-0.0932376981,-0.1395379305,-0.2409367412,1.0341054201,-0.2924596369,-0.8343493938,-0.1087585390,1.7077137232,-0.3006861210,0.6157716513,-0.2781107426,-0.2767468095,-0.5608413815,-1.2163040638,-0.1097816303,0.7184557319,1.5761929750,0.4418694973,-0.8168610930,0.7455046177,0.4540291727,1.3983632326,2.0896103382,1.2146077156,-0.3927581906,-0.1592295468,1.1579405069,-0.5076931119,-0.1404834688,0.6343402267,1.0706061125,0.2229107618,-2.8925197124,0.3393659592,-0.3120492101,-0.9759209752,0.0241301656,1.1204642057,-1.1298772097,1.5899230242,0.9787300825,0.9341636300,-0.8147085905,-0.2512424588,-0.3828348815,0.0003409579,-0.0622048005,-0.3173488677,-0.1779273450,-0.2116203159,0.2674838603,-1.6518170834,0.4865783453,1.6972717047,1.0725008249,-0.9458972216,1.0446079969,2.2896277905,-0.2425562590,0.7210319042,2.2993803024,1.0302956104,1.5507205725,-1.0952255726,0.2356208563,0.3919691145,0.5642871261,0.7895075679,0.1561750621,-1.3123941422,1.0662723780,0.3788603246,-1.1786166430,0.7589958310,0.5529800057,0.5370345712,-0.2868351936,1.6344056129,0.5712266564,-0.7899540663,-1.5427579880,-0.7673321962,-2.3696231842,-0.2605598569,0.1289570481,-1.2076663971,-0.5454435945,1.5532826185,0.1072913334,1.5353435278,-0.1849514097,-0.1767827123,-2.1992540359,0.7292887568,1.1005582809,-0.3423680365,-0.3103042245,0.8130363822,-0.7362970710,-0.7823243141,0.3170981705,-0.6062513590,0.6110578775,-0.9203095436,1.5095089674,0.6564636827,0.0658511817,-0.8736306429,-1.0577539206,-0.8972370028,-1.4322032928,1.3404302597,0.3420748413,1.0706282854,1.7390650511,1.3066060543,0.4496167898,-0.4110504389,0.7492995262,0.2111377716,-0.2606511414,-0.5151983500,1.0792167187,1.4774627686,-0.2751087248,-0.4370077252,0.6981697679,-0.6351764202,0.6036766768,1.7369296551,-0.4178344309,0.0245132316,1.9896930456,-0.3731683195,0.1410367340,-0.9418154955,-0.7028346062,-1.3788369894,2.9252495766,0.5717520118,-0.0458755493,-1.0570642948,-1.8246930838,0.2417382896,-0.1451518238,0.5019926429,0.6923288703,1.0122750998,-0.0421771109,0.6220104694,-0.4075361192,-0.8514463902,-0.3381832540,0.7689677477,0.6064976454,-0.0126131419,0.3224277198,1.6075083017,-1.8024101257,-1.2550679445,-0.0130337598,0.7003573179,-0.2152732313,0.9030884504,-3.0074372292,-2.3304672241,-0.5678032637,2.6673221588,-0.0000683526,-2.2414488792,-0.3600297272,0.6174938679,0.9622223973,0.4705488980,-0.2358042002],[0.6787893176,1.1536179781,-0.5547327995,-0.1236007586,-0.1360976994,-1.7055424452,-1.2570964098,-2.3186249733,0.0843379274,-0.2976732552,-0.3390328884,-0.0344037488,0.5192691684,-0.6725831032,1.1736112833,-1.5922294855,-0.0136492010,-0.0983454883,0.8498770595,-0.4949856997,1.0687308311,-0.2338664085,-0.1018051058,-0.1490771621,0.6980962157,0.8531481624,-0.6047407985,-1.4088909626,-0.9811016321,0.1651830673,-0.0166458562,0.1409326643,-0.7251721025,0.7957408428,-0.1764385402,-1.6334345341,0.1553497761,0.5474259853,0.1437603086,0.1302209795,0.2271719575,-1.0537291765,1.5317530632,-0.3264935911,1.4668451548,-0.5541288853,0.9666754007,0.2401796430,0.1003144309,-0.9387914538,1.2058185339,0.4189720154,-0.0902500674,1.2073749304,0.0885801986,2.2098054886,1.0166771412,-0.3307090700,1.3583818674,0.0608563833,2.0058913231,0.0816583633,-0.0792093650,0.0939683020,0.3995066881,1.5493112803,0.2928569317,-0.1067149267,-0.4934600294,0.7856813073,-0.5543451309,-1.1688302755,-0.9422464371,-0.4649276137,-0.1244343743,1.0531721115,-0.5772435665,1.1971284151,-1.6377062798,2.6346027851,-1.5864238739,0.1980870366,-0.2175800204,-0.7853147388,0.6764087081,1.0572141409,-0.1721861064,1.2093849182,0.0640576929,-1.0637799501,-0.0791875049,-0.5274960399,0.4602147639,-0.3991668820,-0.0741831958,-1.0853574276,1.0693564415,-0.3556284308,-1.0694770813,2.6314742565,-1.4129743576,0.5074303150,-2.1636655331,0.3085227609,0.1881135255,-0.2523779571,-1.3635439873,-0.8585309386,-0.8065607548,-1.1741236448,-0.9480458498,1.4351259470,0.3912090063,1.3308275938,-0.9888421297,-0.4313032329,-1.7370495796,-1.1904556751,-0.2911131382,-0.4206063449,1.7773537636,0.1080465764,-0.2650998831,0.4594625831,0.0268785525,-1.0857034922,-0.6268877387,1.3989568949,1.3037877083,0.2572353184,1.0194852352,1.6428092718,-0.4417257011,0.1843148470,0.2210908979,-0.8268135190,0.4464316368,-1.1436780691,0.1022599861,-0.2689138949,0.4295471311,-1.3548612595,-0.5210994482,1.8831641674,-0.1348460466,0.0681441575,-0.6695713997,0.0453709252,0.3797713816,-0.4755639732,-0.2494962960,-0.1664597243,0.2957374454,1.1305173635,-0.6693257093,0.8059806824,0.8046057224,0.2548323572,1.0816518068,0.0990683660,0.1078162864,-0.1735524535,-0.2703637779,-0.2104981542,-1.8608211279,0.6002736688,-0.3222564161,-1.4688658714,0.9703220129,-0.9755422473,0.8594921231,-0.7415559888,0.9588875771,1.8989210129,-1.5950626135,-0.2549622059,0.9568085670,0.2796890736,0.2816965282,0.9156034589,-0.7888602614,-1.2830469608,-0.7138097882,1.3335521221,-0.7639064789,-1.5349581242,1.1567809582,-0.5121814013,-0.4490087628,-1.3938723803,0.5303887129,0.5679660439,-0.5504558682,-1.1937325001,-1.8920999765,0.5680583119,2.5964953899,0.4043229222,0.6083219051,0.1465612203,0.3495765924,-0.2330164611,0.2941886485,-0.8010068536,-1.2406933308,0.1112386584,0.0981622413,-1.1744223833,-0.5514314175,-0.3500730395,0.7157834768,0.1586675495,-0.1676100791,-1.5922158957,0.2704105675,0.5291992426,2.1981124878,0.4463334382,0.9241107106,0.9336605668,-1.8127021790,-0.1246180162,-0.1696453393,-0.4599695206],[0.8565337658,1.6267207861,-0.9244696498,-0.1559941918,0.4249975085,-0.7795513868,-2.1165361404,-0.2150630206,-1.1040906906,-1.9384589195,0.9018970132,1.6354812384,0.5289262533,0.4159639180,0.4340830147,-0.0776079670,0.4491689801,-0.1313019097,-0.7509349585,1.2776517868,0.7487615347,2.6572730541,-0.8186072707,-1.4451512098,1.3417631388,-0.1051895544,1.3567872047,2.0140140057,-1.3726296425,-0.8574283719,-1.6812884808,-0.1505986601,-0.7210639715,-1.9408824444,0.0999050587,1.2818363905,-0.9700794220,1.2453765869,0.6968713999,-0.0645520166,-1.0578403473,-0.4692222774,2.1283955574,-0.3892408013,0.5696129203,-0.2742994726,-1.0984168053,0.6897208691,1.6492482424,1.0651326180,1.6090450287,0.1430166811,-0.0025570951,0.5276804566,-0.2226212621,-0.4914850295,-1.5938395262,1.0183731318,0.0496852174,-1.6018487215,-1.8184107542,1.7716566324,1.0756032467,0.3834027946,-1.4669975042,-0.2647817731,-1.0320789814,-0.3644350767,-0.2168742120,-0.7252820730,-1.3937695026,0.7387053370,0.2493469417,0.5485517979,1.1348981857,1.1029473543,1.6632301807,0.9258661866,0.4591159523,-0.4123454392,-1.2897313833,0.5902590156,-0.2920606136,1.9976263046,-0.5045529604,0.2800785005,0.2846641541,-1.4732339382,-0.7650971413,-1.9866602421,0.2745036185,0.1866807640,0.7437629700,-0.7251886725,-1.0595256090,-0.3219626844,0.4132559597,-1.0829941034,0.3818486631,0.4902530909,0.8203164935,-0.3574656546,0.0871377513,1.2285019159,-0.6999685168,-0.0382436216,2.0060265064,0.3040466607,0.9432737231,0.3004783690,1.0069338083,0.4846802950,-0.2194198221,-0.2402843833,-0.2284196168,0.6577883959,0.0243720971,-0.0167382881,0.0804451779,0.2744990885,1.5632476807,-0.9265797138,1.0785713196,-0.4262202978,-0.3350518346,-0.1482925713,-1.3545845747,-0.6348443031,-0.1358720660,-0.5967708230,1.2970815897,-0.8241472840,-0.1672525704,-0.6040331125,1.2512923479,-1.3428666592,0.4152507782,1.1968697309,1.4077038765,0.1768872589,1.7283319235,-0.9516835809,-0.5515766740,-0.5391194224,0.2572713494,-2.1139318943,0.9485329390,1.2312760353,-0.2881865203,1.2581969500,0.5737562180,-0.9768746495,-0.6910295486,-0.5072914362,0.3287530839,0.9405872822,0.6720070243,-0.6202755570,0.1128496304,-0.6132038236,1.5822707415,-0.3844380975,-0.0673749968,-0.2834100127,-1.1995179653,-0.0790346786,0.8218730092,-0.1707502753,-1.6510307789,-0.2086256593,-0.7109470963,0.0523468554,0.9653596878,-0.7002084851,-0.1555075496,0.0463614166,-1.4276753664,0.3529338241,-0.2941586375,-0.9563120008,-1.3457732201,-0.0424981080,0.8205020428,0.4909951687,-1.1780422926,-0.3410792053,0.4586801231,1.1928086281,1.5076155663,0.2159299105,1.0762003660,-1.3993560076,0.5086227655,1.5924257040,-0.0414426811,1.3759416342,1.3556041718,0.6253252029,1.6755017042,-0.2901961207,-0.2289954126,1.5246464014,0.3891948462,-1.2429443598,0.0270989761,2.9799761772,-2.3688035011,1.3221940994,-0.5086752772,0.5284387469,0.8252167106,-1.3484315872,-1.5581241846,-0.8698654771,-0.0978383347,-1.6825011969,0.6562595367,0.5922316909,0.2983350754,0.2247846574,-1.3188779354,-0.3159044385,0.0416775160,0.7735174298],[0.1309816986,1.6671591997,0.6244473457,0.0545613840,0.4523994327,1.6147964001,-1.1268408298,-0.3600841165,-0.3496910334,-0.6707585454,-0.9565799832,0.5979391932,-0.4567743838,0.6110959649,-0.4604537189,-1.2620191574,1.1980472803,1.7984120846,0.2246788591,0.5230852962,1.7367670536,0.5086534619,1.1650388241,0.7131203413,1.3199769258,0.3111094236,0.2989323735,0.6086726785,-1.3495463133,-0.8733163476,1.3610419035,1.4206547737,0.4303741753,-0.9519916773,1.2027678490,0.0799262673,-1.1702611446,0.6195428371,0.8924890161,0.8761147857,-0.3095809221,-0.5196042657,-1.3068716526,1.5195988417,0.2128613889,1.5664979219,-0.2589054108,1.7114889622,-1.8208161592,0.1634945124,-0.8131170273,-0.6053546071,-1.3275238276,-0.6441715956,1.9088834524,-0.5635452867,1.0824725628,-1.9519108534,2.4412162304,-0.0172850862,0.9122820497,1.2396584749,-0.5733674169,0.4248894751,-0.2712600231,-0.6835675240,-1.5374375582,-0.1013744250,0.7466657162,0.9291818142,0.2294180095,0.4144058824,0.3097238243,-0.7374562025,-1.5369198322,-0.5622548461,-1.5995111465,0.8243899345,2.4914863110,1.0118551254,-0.2812379301,0.0167065077,1.1539196968,-0.6009150147,0.1329372674,-2.1699743271,-1.1165589094,-0.5693680048,0.6362667084,-0.7676492333,0.6184794903,0.1831612140,1.8593512774,-1.0755146742,-2.0192196369,-2.4374644756,-0.1485635042,1.1511514187,0.1853233874,0.6584609151,-0.0495361015,0.2373988479,-1.5115803480,1.9323703051,0.4824220240,-1.3157830238,-1.9636150599,0.5330681801,-0.1093232185,-0.2831834257,1.4337438345,-0.5686172247,-0.8498464823,0.7244464755,0.6058983207,-1.2902585268,0.7894331217,1.9609137774,-0.3355398178,-0.6076121926,-0.7272418737,0.8514086604,0.8884165883,-1.1113233566,1.1378443241,1.2755492926,0.2945994139,-2.1856794357,0.7923774719,-1.6250413656,0.7398307323,1.0142691135,0.9582080841,-2.6153709888,-0.7281308174,0.7424984574,-0.0821439624,-1.5249018669,-0.4381790459,-0.0053916587,-1.7120848894,1.7055863142,0.8111898899,0.1493609101,-0.7354556918,-0.3129665256,-0.6534162164,-0.9912257195,0.1489663869,1.1033235788,-0.1900487989,-0.4122638702,0.1786202043,0.7816811800,-0.5246403813,-1.3810967207,0.9454761744,-1.3616911173,0.5349876285,-0.0959592983,-0.8964068890,1.3871814013,0.9973930717,0.5764676332,0.4400008321,0.4754700959,-0.9835773110,-0.5619937181,-1.6410032511,0.2922827303,1.4331048727,-0.3837113082,0.2374925911,-0.5327551961,-0.4711798429,-0.3400845230,-1.4632056952,1.1385982037,-0.0050248299,0.9715877175,0.3023412228,-0.8062357306,-0.3169119358,-1.5868651867,0.7886871099,0.1372247785,-0.7851805091,-1.5065702200,-0.5764098167,-1.1072884798,0.1352825314,-1.0527776480,2.0522935390,0.9853089452,-0.3728401065,-1.4862358570,-0.1116613746,-1.9596153498,1.0607012510,0.7912979722,1.4486447573,-0.5566545129,0.8055643439,1.3840409517,-0.4356646836,-0.7338418365,1.3481131792,0.1454764307,0.4252978861,-0.0880398229,-1.2484735250,1.1045972109,-2.0293858051,0.9979759455,-0.2792384624,-0.1876324564,-0.5325407982,0.5268504620,0.4087450206,1.6114984751,-0.1840722412,-0.1820611060,-0.3537951708,0.1039311066],[-0.4489805400,1.1552664042,1.3704001904,-1.9223861694,-1.2888606787,-1.0187835693,0.6616163254,-0.8920385242,0.5518894196,1.3713300228,0.7708854675,-1.0448182821,-1.1999044418,-2.6680612564,0.4900120795,-1.2550389767,0.0423943661,0.2307881862,-1.3097743988,0.9433927536,-1.0359666348,-0.5941559672,-1.1822756529,0.0490396470,-0.6646436453,2.2700588703,-0.6435940862,0.1863738149,-0.2803147733,0.8866586089,0.0039090207,-1.4617713690,0.1988759339,0.4399749041,-1.2791353464,-1.0427937508,-0.6556071043,0.2571541965,-0.3568092585,-2.2522468567,-0.7602754831,1.1351559162,0.6326811314,0.6538631916,-0.0560684986,2.1729958057,-0.0687307790,-0.2313192934,-0.4723461866,-0.3986878395,0.7822806239,-2.0840001106,-1.1852180958,1.4007776976,-0.7757635117,-1.1275815964,-0.1672484130,-0.7482985854,0.2928707302,-1.1963318586,1.4991011620,-1.3109302521,-0.0839465857,0.0558208153,0.2463919818,-0.0606065616,-0.0057793492,-0.0700023621,1.3056834936,-0.1253987551,0.9635531902,-0.7322648168,-0.0024187912,-0.1301703900,1.0004709959,-0.5866019130,-0.6917225122,-0.1560804099,-0.0212033875,-2.1023504734,-0.8584808111,1.1101720333,-0.2545737624,-0.2416257262,0.5702453852,0.5405352712,1.7392754555,0.6312438250,-0.4402846098,0.0550365932,0.7741043568,0.9616689086,-0.9026238322,0.6697832346,1.7671315670,-0.1734208614,0.0113102626,-0.9254624844,-1.0618740320,0.0115344757,0.7880923748,-0.2987428606,1.1056511402,0.9078025818,-0.3661470115,0.7300938368,-0.1293408722,0.0929499343,1.0738584995,2.4386839867,-1.3962153196,-0.3303312063,1.1602600813,-0.2442411035,-0.4293623567,1.6676892042,-0.0608364716,0.7872980833,-0.3093358874,-1.5586376190,-0.5111013651,0.6837660074,0.1929822713,-0.7978610992,2.0638716221,0.1178953871,-0.9953600168,1.2995123863,0.9166028500,-0.5599343181,0.6541744471,0.7905471325,0.4623413086,0.1298543960,-1.7680779696,-0.3233806193,-0.0529145338,-0.5833628178,-0.3562459648,0.7015071511,-0.4987028837,-1.6961719990,-0.6258285046,-0.5355837345,-0.1618913412,-1.2267602682,-0.3106380999,-0.8768603802,0.7795471549,-0.1935388297,0.5475685596,0.2768119276,-1.2845706940,-0.0423595309,1.6698843241,-1.6108504534,0.9478467703,0.0986463651,-1.5106642246,0.5734161735,1.0034544468,-0.0609908327,0.3071651459,1.8223924637,-0.6054177284,1.8280153275,1.6417883635,-1.3252933025,0.1235409901,0.0669323280,1.2934104204,1.1690789461,1.5101593733,-1.6193233728,1.3771556616,0.5679165125,-1.8386728764,1.3130825758,-1.1070921421,0.7729588747,-0.5860818624,-0.2058396339,0.4436039925,-0.2096983939,-1.4521727562,0.3058028519,-1.1525199413,0.5443555713,-1.2522698641,0.1393335462,0.3642078936,2.0137174129,-1.9375973940,0.3161602020,-1.2603361607,0.4786128998,-0.1122477651,0.7691808939,1.4161857367,-0.3374640048,-0.0133701405,0.9041422606,-0.5291411281,0.4031642973,0.8388589621,-1.0761878490,-1.5249725580,0.1757129133,0.8767262697,-1.5049668550,0.4910938144,-0.3653927147,-0.1716035455,1.0316590071,0.1405045390,-0.8550847769,-0.0122549757,-0.2336222380,-0.2811496854,1.1793352365,-0.6546878815,-0.7931123972,0.2504535615,0.6389125586],[-0.1338374466,-1.2315516472,0.1087464318,2.4242548943,0.5663132668,-1.6622200012,-0.9516454935,0.0935006887,0.5418270230,-0.6956735849,2.1641728878,0.5378525257,0.3326438665,1.2569468021,0.2456189245,-1.9654610157,-0.3942227066,-0.6392866969,-0.3373064399,0.1140286326,0.6925070286,-0.7737633586,-0.7652257681,-0.0776397437,0.5756703019,0.5510339737,0.6002933979,1.1095582247,-0.3609852493,1.1201286316,0.3345442414,0.1578962952,1.8014444113,-0.3187448680,-0.7662656307,1.4651508331,-0.0448592342,1.1052986383,0.3984775245,-1.7524188757,0.2810229659,0.6246092319,-0.0725462735,0.6233510375,1.1921753883,1.4715322256,0.7919710279,-0.8447570801,0.1241444349,-0.7139037251,-0.4604227841,-0.4756816626,-0.0533989593,0.2218795717,2.0838932991,0.2475150228,1.7827460766,-0.0134890378,0.7679715157,-0.3554916978,0.0362120308,-0.7166433930,0.6927425265,-0.7791339755,-0.2974212468,0.2203360796,0.4349841774,0.5249035358,-0.4737904668,-1.3593201637,-2.2599756718,-0.8934810758,-0.6567152739,-0.6929903626,-0.8416537046,0.4606203735,-0.2749021649,-0.1371530890,-0.7862843871,1.6317793131,-0.2760257423,-2.3511528969,-0.7772335410,-1.3283784389,-1.4011971951,0.4626928866,2.3816864491,-1.2089858055,-0.4188674390,-0.7853214145,-2.1645514965,1.1445811987,-0.9505017996,0.7641232014,1.3194897175,-0.1203246713,-1.0543094873,-2.7589275837,0.6597554088,2.1434054375,-0.4508769512,0.4651074708,-0.3877440095,-0.6067355871,1.5365588665,0.1711568981,0.0135632185,0.1285218298,0.1237851009,-1.6126866341,-0.1699062586,1.7187911272,0.7952760458,-0.5621861815,0.1939502656,1.2603075504,0.3326231539,2.4959945679,0.3348725140,0.2440851331,-1.0814745426,-1.6712106466,1.0294514894,-0.3955523074,-0.7647819519,0.5640015006,-0.6139293313,0.7056366205,0.2410723269,-0.6252645254,0.2575959563,-1.4764999151,0.4396305084,-0.5134104490,-0.0369065218,1.2340645790,0.0310939327,-0.6546210647,0.9864290953,-0.0455574021,1.2032821178,-0.7458623052,-0.2978456020,-0.9822353125,-1.3218252659,0.1633568257,-0.6249672174,-0.7417123914,0.1362227798,-1.0594494343,-0.0809739381,-0.5245481730,1.8382658958,-0.8360715508,-1.3591579199,-0.5534932017,0.1547365636,-0.8778612614,-0.5545231104,-1.2758891582,-0.8472756147,0.3394901454,0.8867839575,0.3970008194,0.9629596472,-0.0797268972,0.6448428035,0.4046718478,-0.0212829933,-2.4381325245,-0.1657105386,0.9223551154,-0.8054891229,1.8176586628,0.8265794516,1.5185177326,1.7634532452,-2.1006796360,0.9573124051,0.2478752732,1.1825718880,0.9785193205,-0.5731644034,-0.9844460487,-0.9941863418,0.1693080962,-0.9317142963,-0.9543631673,-0.6126065254,0.3264756799,-0.8015695214,1.0485463142,-0.7815785408,1.3674834967,-1.0214259624,-0.2009140849,0.1025521085,-0.9636093974,-0.7346326113,0.2468834966,-0.4661616087,-0.5672651529,-0.4167720973,0.1828022152,-0.8394173384,1.6634211540,-1.8836456537,0.0390705056,-0.1487848461,-0.3561755717,-1.4013719559,0.6198907495,-1.6715638638,0.5220121145,-0.1027402878,-0.5389942527,-1.2783858776,-0.7213203311,-2.3513112068,0.6171727777,0.1380944103,0.5125414729,-1.2011198997,-0.5964149833],[0.5951352715,0.2339069694,1.1006313562,0.4247358143,0.3372429609,0.1389364749,1.4059437513,-0.4783534110,-0.6449186206,0.0131500335,-0.0374759957,-0.2436141372,0.8794044256,0.3872634768,-0.5042549968,2.1577014923,-0.5692856908,0.5889863968,-0.8519099355,-0.5406209826,-0.3512584567,1.8592058420,0.1071278453,0.1881214976,-1.4387773275,0.2278479040,-1.0301718712,-0.6761962175,0.4439955652,-0.4711591303,1.5134398937,1.2920711040,0.1850506514,0.0724349618,-0.1809644252,0.3238140941,0.4700443745,-0.2543262243,-1.1642212868,0.2567294538,0.1768714786,-0.6741221547,-0.0544025823,-0.8167592883,0.6026630402,0.1571673602,0.1848359108,0.7246237993,2.1916341782,0.0395292416,0.5885300040,1.0862364769,-0.0933451578,0.8974225521,1.2546049356,0.0124170966,2.8417673111,-1.7472499609,-0.7727107406,-1.1648753881,1.1488866806,0.2784065306,-0.7488279343,0.7987616062,-0.2916001379,2.6300568581,-0.5040947199,1.0890872478,-1.0070320368,0.5389986038,-0.4499021173,-1.3006299734,-2.0500237942,-0.2852370441,0.9818706512,0.0392058752,-1.4161587954,1.4106215239,0.4247548580,0.7822275162,0.4471131861,2.5369141102,0.1805239916,-0.8006153703,1.0798196793,-2.1828672886,0.2883101404,-0.3787364364,1.2829811573,0.0463522710,-0.7552426457,0.1391619742,-1.3081711531,1.8474140167,-0.2298999429,0.6889461875,-0.6292168498,0.4396181107,-0.0836468861,1.9265546799,-1.4717415571,-0.4133013189,-0.5089668632,-1.2310500145,-0.1285655946,0.8536369801,-0.7277516723,-0.0126291113,0.4544115961,0.5498104095,-0.5179764032,-2.4123616219,-0.4469912946,2.0694754124,-0.9394243360,-1.0521241426,0.2036963105,0.0071311919,-0.6870504022,-0.0155820046,-0.1647448540,0.3724843860,0.4417797029,-1.0829602480,0.6957066059,-0.2200885266,-0.6384267211,1.0450456142,-0.0275342409,-0.3599655032,-0.2629359663,0.5287644267,0.1829177290,0.6768316031,-1.8263455629,1.1303802729,-0.0842294767,0.5546888709,1.3973606825,1.3698474169,-1.3730745316,1.0151197910,0.2647194564,1.0315226316,-1.5784586668,-0.9907632470,0.2567261159,-0.8996278048,-0.3867754042,1.4097727537,2.2299740314,-0.1896184236,0.9524666071,0.1702094078,0.9398848414,0.5321618915,-1.2387745380,1.2894320488,-0.8582060337,-1.3007978201,1.1981871128,-0.3366500735,0.8419474363,0.5433722734,-0.9629876018,-1.6009552479,1.7573373318,0.6935952902,1.4654803276,-0.1797384024,1.2437579632,1.7125695944,-0.2490044981,-0.2686403692,0.0930381417,-0.0375618339,-2.6600220203,-0.9509128332,0.2806393504,0.2936806977,0.2567313313,-0.3014256954,-2.0332870483,0.1230432019,0.2137270868,-0.9075357318,0.3526922762,0.2563580275,-0.5407764316,-0.1236735359,0.8370244503,0.2930872738,-0.9313671589,1.5671607256,1.1615507603,-0.8167613149,-0.7182096839,-0.6333760023,0.5856393576,0.6780885458,0.5890313983,0.7123593092,-1.2848955393,-0.5609303117,-1.9114166498,0.8378915191,1.0157173872,1.0953134298,-0.4000841677,0.3343574703,0.3863209784,1.5166685581,-1.9124687910,0.7424203753,-0.3382374644,-0.3509818017,0.6303770542,0.4310825765,-0.8497053981,0.5221378803,0.4046062231,1.6502557993,0.2492208332,0.7841585279],[-1.4824200869,-0.4545456767,-0.9722738266,-0.0509347804,1.7074534893,-1.7254183292,-0.3090017140,-0.0453792103,-0.5058189034,1.3871597052,1.1207715273,1.0124270916,-0.3037402928,-0.1054088175,0.5237683654,0.1429743320,-0.8187479973,0.7458306551,-2.1103670597,1.0728235245,1.7312847376,0.3593048155,0.8765829206,-0.0282392837,-1.0374023914,1.3495305777,-0.8798107505,2.2148194313,-0.3535363674,0.0634836853,0.0064468719,0.4102476537,-0.5776757598,-0.3282352984,1.2123632431,-1.4314142466,-0.6823901534,-0.8864627481,-0.1065689325,0.0673301294,-0.2074949145,0.2912158370,0.9534795880,0.7978639603,-0.1701406687,0.2286783755,-0.8701025844,1.3222193718,0.1815055162,-2.4394092560,-0.3474123180,-0.9069854021,0.9881507158,-0.5943987966,-0.6776933074,-0.5054681301,-1.7560210228,-1.4633255005,0.5323156714,-0.8000277877,1.1683307886,-1.0447676182,0.0635707676,-0.3356823921,-1.1476477385,0.1429802775,-0.6482725739,0.0072258324,0.4359221756,-0.8878523707,2.4653964043,-0.5865577459,0.7874836326,-0.0772186443,0.0906665623,0.1741325259,-0.7149444222,-0.0434522107,0.0239767432,0.1674236953,-0.3820293546,-0.0326141678,-1.2816491127,-0.2897256017,-0.5946949720,0.0430568755,0.3749506772,0.0121182492,1.7634828091,-1.3510473967,1.1881501675,0.4915170074,0.0009339975,-1.8913590908,0.4536544681,-0.3723362088,0.2312607020,-1.4136934280,-0.5840505362,-0.1064425334,1.3762027025,-0.4251186252,-1.9075818062,0.2827370465,1.4952538013,-0.2975981832,-0.7461140156,0.3224439025,-1.4082685709,0.1460434049,-0.1548788100,-0.2577507496,-0.2077642530,-0.6747339964,-0.5161724687,-0.7427644134,0.2661053836,0.0695588514,-0.4860402048,0.5244890451,0.4730707407,1.7623431683,0.8991122842,-0.0184322987,-1.0504052639,0.2551665604,-1.1870924234,0.6593571305,-0.3136428595,1.2176970243,-1.0874655247,-0.0515389442,0.2061298043,-0.6265270710,1.3921668530,2.8257706165,1.0335572958,-0.8086478114,-1.0901274681,-0.3106675744,-0.1632628441,-0.4603413045,-1.3341411352,0.5154522061,-0.4899453223,0.3032837510,2.4529964924,-1.8736343384,-0.0681086928,0.0572270155,0.0044212798,-0.0653590336,0.2338360548,-1.1391998529,-0.0103998687,-0.5209394097,-1.0892212391,-0.3462033272,-1.8454072475,-2.5989222527,0.1982216686,-0.4841802418,0.2070287764,0.7549870610,1.3594202995,0.6960720420,0.6822013855,-0.0112141585,1.3447604179,-0.8314924240,-0.4079718292,-1.3308038712,0.3525986075,-0.5378847718,0.3934444189,0.2865182757,2.0425362587,-0.9194611907,0.1146700308,-0.1374237090,1.3655269146,-0.2616383433,-1.2862409353,-1.6546574831,0.0372327417,1.2703013420,-0.1432766169,0.7952552438,-0.0720724761,-0.7424240112,1.2256538868,-2.2053880692,-0.5306376219,0.1682143807,-0.9688461423,0.3096540272,0.9401078820,-0.1633903384,-0.3208823204,-0.4034307897,-0.6640667319,-0.1162874550,-0.4164044857,-0.2106245607,2.6117999554,-0.8170961142,0.2296934575,0.1352863908,2.0153095722,-0.9835947752,-0.0398398787,-0.7099856138,0.2519035935,1.0100510120,-0.7469947338,-0.2271399647,0.8424672484,-0.0028453339,0.0573192686,-0.7911548018,1.5115075111,-0.4292039275,-0.2814901173,-0.2986505032],[0.9713628888,1.1659340858,-0.1845057160,0.0065851845,0.5065482855,0.1957827955,0.7852159739,1.0992507935,-0.6381587982,-0.2258110791,-0.0518908724,-0.4910358787,-2.2459900379,-0.7985108495,-0.8093937039,0.2404313385,0.4376957715,0.3021634221,0.2768120170,-0.9123110771,-1.4636436701,0.8011099696,-1.4775968790,1.9602862597,0.2919983566,0.1149028838,0.3892951906,-0.4618502259,0.7220209837,-0.1333760023,-0.8379451632,0.4440839589,2.7235734463,-1.1158251762,-0.2277544439,1.3317129612,-0.4921458066,1.3590461016,1.0847727060,0.0156160211,-0.1734282523,0.9402840137,-0.7774642706,0.7608965039,1.4017525911,1.2316533327,1.0011581182,-0.9780515432,0.6779560447,-0.0694640726,-1.0573610067,-0.1401209533,-1.4334952831,-0.1790812165,-0.1230426356,0.5381953716,1.0654780865,-1.4013202190,-1.3276036978,-2.0463159084,0.8385336399,-0.6869310737,-0.5620132089,-0.0442883112,-0.0047367625,0.6841859221,-2.1602709293,-0.1973593086,-1.2325326204,1.6918396950,0.5312877893,0.3519688249,2.1720921993,-0.1462069154,0.8961949944,-1.4612011909,0.2014119923,2.2746560574,0.4980108440,-0.7518550158,-0.9755960107,-1.5738663673,0.4436374903,0.8610516787,0.1271061748,-0.8113466501,0.2418205291,-0.6867427826,1.2207347155,-0.4108690321,-0.8827690482,-0.5456284881,0.3868001401,-1.2304912806,0.6670293212,-0.2712611258,0.0818201751,-0.1299658567,2.2640588284,-0.1695053726,0.8452254534,2.4532289505,0.4257523119,1.8475958109,0.0974547565,0.7967066169,-0.5887081027,-1.1121789217,-0.9687710404,-0.0241551120,0.2857384086,-1.1169031858,-1.3783563375,-1.8767281771,2.1267092228,0.1075932160,-2.1452682018,-0.0090910634,-0.1592708379,-1.3771734238,1.1593115330,-0.4573624432,0.0412932597,-0.2843123376,0.0463854149,1.0175329447,0.6056706309,-0.4518868327,1.5754882097,1.5401930809,-1.4632025957,-1.1145495176,-0.6111642122,1.5250842571,-0.1827131957,-0.5719213486,-0.0339936987,-0.2241427749,-0.1559786499,-1.0696374178,-0.7931587696,-0.2671611905,1.3815757036,1.2891613245,0.9156867266,0.3633261919,-2.6118755341,1.7803673744,1.1180121899,-0.3693193495,-0.6483833790,0.1860720217,0.0447100475,-0.6926733851,1.4939805269,-0.6550534368,0.2498712093,-1.2042515278,-0.2552892864,1.0822662115,0.0425696447,-0.5869133472,-1.9481761456,-0.3409913480,-0.3546536267,0.7333459258,1.3222188950,-1.2185709476,0.8016905785,-0.9687362313,1.1493378878,-0.4367955327,0.5572791100,1.5488294363,-1.7036701441,0.1468585730,1.5773946047,0.7791216373,2.1659414768,-0.4962480664,1.0852104425,0.7184401751,0.0505882911,-1.0123195648,0.2237323672,1.9337600470,1.2498807907,0.4884160757,-0.4939172566,-1.4075125456,-0.1990298480,-0.9089871049,-0.3559097946,1.1026279926,1.2252410650,1.0871775150,0.5441542268,-0.5297963619,0.4389980137,-1.7267080545,1.1040461063,-1.0993429422,1.3341366053,-0.6335446239,2.2331142426,-0.2638579607,-0.1156971976,0.7624002099,-1.3418661356,0.2185997516,-1.0052831173,0.6768507361,0.4997563660,0.2243302017,1.2361823320,0.1434970349,-2.1947665215,-0.7532163262,-0.9595815539,0.6633858681,-0.2023599595,-0.6430290341,-0.8134525418,-0.3416174352],[0.4218400717,0.8895066977,0.0375063010,-0.4428788722,0.0392513834,-1.2110996246,-0.4579227865,1.4286129475,1.0722044706,2.2364633083,-0.7332018614,2.6002426147,-0.5068512559,-0.6531524062,0.2969976068,0.4913036525,0.1942404211,-0.0775747150,2.0618264675,-1.1979434490,-1.0965492725,0.2128379345,-0.0973059759,0.0509134494,2.3178815842,1.6716791391,0.6432677507,-0.7658655047,0.2551034093,-1.7418290377,-0.8488428593,0.7497497201,-0.9394742846,-0.6140592098,0.2242153883,-2.9374637604,0.2126290649,0.3135212660,0.5678259730,1.6826602221,-0.5162865520,-0.2058548182,-0.7859453559,-0.6208847761,-0.7223044038,-0.7193372250,-1.9608685970,-1.1417084932,0.7958766818,-0.3401833475,-0.4921501875,-0.4349618852,0.3137913048,-1.3271640539,0.9106199145,-2.0794861317,1.0177873373,0.4266745448,-2.0590798855,0.0865664855,0.3999687433,-1.1941734552,-0.0655964836,-1.5167351961,0.8678767085,0.7040355206,-0.0141630126,-2.5513372421,-0.0591046214,-0.1782632917,-0.6721311212,-1.4577360153,-0.8415274024,-0.4924280941,-0.1054591313,0.6587065458,0.6277593970,0.2027057856,-1.0286164284,0.1998594999,0.6571788192,0.6904782057,0.9589516521,-2.3220672607,-1.4431536198,-0.3931055367,0.7838268876,0.8431079388,-0.6274953485,2.0802054405,-1.8623497486,-0.3251871765,0.6918599010,-0.4644671977,-0.9389165044,1.2123388052,-0.1305605322,0.2123090029,1.2012878656,-0.7994313240,0.2028683871,-0.4153151810,-1.1983784437,1.0681349039,0.5389846563,0.2846174240,0.0825485736,1.4062007666,-2.5803592205,-0.2879975736,-1.4541510344,0.2424149364,1.2237983942,-0.8512282968,1.5265872478,0.3694509864,0.7817362547,0.8075862527,0.3952428102,-0.9362884760,-0.7247197032,-0.0328013301,-0.4716132581,0.5637686849,-2.2679274082,0.3426539898,-0.0979672372,-0.2402446121,-0.0963036716,-2.1943864822,0.6942761540,0.0291213989,2.4530353546,0.7187931538,0.2133912891,-1.0874562263,-0.8873418570,-0.9403048158,1.9221187830,-0.8266971707,-1.5223639011,0.9220241308,0.8729629517,-0.1005499288,0.9404553771,0.2749381661,-1.2735220194,2.1069967747,-2.2149572372,-0.3422880471,1.4668304920,1.9702908993,-0.9250925183,-0.0060714148,-0.9882918596,1.4887702465,0.2804108858,-0.3489108682,0.5542525649,-1.5422412157,-1.2575305700,0.2715023756,1.2902284861,-0.2465406358,0.3424791992,-1.0304328203,1.7649421692,0.8475595117,1.3346787691,1.6747332811,-0.2368809879,-0.3557014465,-2.0324966908,-0.4428349435,0.6714438796,1.0843613148,0.2764685154,-0.1155481040,0.9883731008,0.4112939835,-0.6270067096,-0.1460562497,0.2729308009,-1.0989434719,-2.7572638988,0.7561358213,-0.9827682972,0.9108489156,0.6604135036,-0.1289324462,1.8280123472,0.6759609580,-0.3519555330,1.6215261221,-1.0882657766,-2.5271363258,-0.3571330607,0.9479765892,0.4368820190,-0.2717285156,0.6993663907,-0.7780221701,-1.1863687038,-0.5583704710,-0.7359765768,-1.0251901150,-0.1638904214,1.2510455847,-0.3172542751,-0.9209412932,-0.5378237963,-0.2243570387,0.5573434830,0.7788372636,1.7665655613,-0.5648117661,0.5073568821,-0.1877584457,-1.0030902624,0.0020168158,-2.2090637684,-0.3919711113,-0.0892934874,-0.2768075466],[-0.7382922173,0.4781686962,-0.5271230936,-0.8565163016,-0.0114886239,-0.3409681022,0.9220369458,0.4548239112,1.1105103493,-0.8540458679,-0.5342381001,-0.1587235928,0.3786576390,1.6225638390,-2.4055118561,-0.9667271376,0.7057707906,0.2168771327,-1.8539066315,-1.3120892048,-1.1653203964,-0.5374454856,-0.4951444268,0.6106092930,1.4650698900,0.2833453417,0.0034100253,-0.4601604939,-0.7077453732,-0.1619891524,-0.3710734546,1.2191070318,1.4806418419,-1.0868707895,-2.0648705959,-0.9124275446,-1.9892864227,0.8951463103,0.8109661341,0.7963710427,0.7626373172,0.6699028611,1.3099942207,0.1779273599,0.3980103135,-0.2223629951,0.7420293093,-2.4057607651,0.7425398827,0.7739811540,1.8070101738,-1.3846660852,0.1880014688,-0.1909513474,-1.5174038410,1.2443370819,0.4424822032,0.7484818697,-1.8359574080,-1.0961536169,-0.5201159716,-0.4773720503,-2.1213893890,-0.7915535569,-0.1687713414,-0.3899119198,0.2895978987,-0.3511215150,-1.3487813473,-0.0930239484,0.3696572185,-0.3927856386,0.1512555778,2.9449841976,1.6288179159,-0.3464185596,-1.0187661648,0.1346620172,-0.1407055557,-0.2609769702,0.8170487881,0.0737538487,0.5982130170,0.1496603042,2.7200851440,-1.6137839556,1.5230054855,-2.3850162029,0.9167109132,-1.0846986771,0.1835428476,1.8487994671,0.3989471197,0.8919633031,1.2315006256,-0.0234141331,-0.7482023239,0.1487952620,0.4381481707,0.3836542070,1.5491355658,-0.7265835404,0.9751958847,-1.2847875357,2.1041853428,0.2272570580,-1.1291662455,-0.2813319564,-0.7394167185,-2.5843210220,-1.1766476631,0.9712014198,-1.2400223017,0.6785711646,0.7806016803,0.9310101867,0.2308028489,-0.9521868229,-1.0569379330,0.0101044821,1.2908110619,0.8434351683,-1.0314128399,-1.3817199469,-0.0570717938,0.4098027349,0.7228524089,-0.1619416177,-0.1127954870,-1.3605376482,1.6959846020,-0.1019152254,1.0857456923,-0.1930623353,0.5130358338,0.2026667893,0.7818601727,-0.5955975056,0.1939072609,-0.2461180836,-0.2952910960,1.1024920940,0.4419107437,-1.1785167456,-0.1340370774,-0.3758517504,0.1165554896,-1.1369833946,0.9722950459,0.1699508727,-1.1497621536,-0.5577647090,2.4325559139,2.0300633907,0.3172180355,1.0420550108,1.3970026970,-1.3860317469,0.6756493449,-0.2088022530,0.0416123271,-0.3647925854,-1.3826622963,0.0327142030,-1.7300449610,-0.6120365262,1.1658670902,0.7848781943,-0.5333843231,1.3701926470,-0.6063991785,-0.9086737037,2.2848854065,0.6883058548,0.6623967290,0.8888964653,0.5443738699,-0.7265207171,-0.6869468093,0.3359093666,0.4884953499,0.9215206504,-0.1096334979,1.8479670286,-1.3251534700,1.9301359653,1.4032901525,0.3269095421,0.3069120944,-0.2451367527,-0.8169812560,-0.4496933222,0.4065041840,2.2406930923,0.8854416013,-1.4401488304,-0.9977304935,0.9404314756,0.0276801270,-1.0297243595,-0.9591946602,-1.1910470724,1.7555153370,-0.0457942411,-0.0153770149,-0.3424043953,-0.6024621725,-1.5076104403,0.3168259561,1.2989704609,-0.3437852859,0.3600607812,1.1474263668,0.8276595473,-0.7948626280,-0.3214859068,0.1499467790,1.6915059090,1.2442883253,-0.1117230430,-1.8866000175,-0.0652765781,-0.8484117389,0.9157932997],[-0.8810158968,-0.6766892672,0.0717540085,-0.0943662152,-0.8810151219,1.5139248371,-1.0652790070,-3.0692067146,-1.7233544588,0.2145391554,-0.0761685669,-1.6524933577,-1.8591082096,0.0064446605,1.4435199499,0.4387893379,-0.8295137882,0.4456272721,-0.8297645450,-2.2846164703,0.3067142367,0.2384975702,0.9313440919,0.7382943034,-0.0273835473,0.7648501396,-0.8317270875,-1.4108572006,-1.7982788086,-0.6981588006,0.6069019437,0.8501870036,-0.8509014845,0.4383669794,0.6248463988,0.8739450574,0.3806868494,-0.9634509683,-0.1053313464,-1.0504844189,-1.2173187733,1.4710108042,0.6670583487,0.0257385876,-0.9211400151,-0.5374246240,-0.0370455347,0.9016004801,0.6242956519,-0.0050148740,0.8383492231,-0.6565697193,1.8286083937,0.5789019465,-2.3177831173,-0.5858536363,-0.3403774202,1.7211940289,-0.0403645262,-0.0005720349,0.9837250710,1.1704604626,-2.0550563335,0.5990749598,-1.3058104515,-0.1033330113,-0.3316901922,0.9040453434,2.9563987255,-1.5847589970,0.4970963597,1.2656636238,-0.1820234358,-0.6379039288,1.7642017603,-0.3535215557,-0.3904646635,0.0819236487,-0.4960451722,0.6401004791,-1.1532803774,-0.7206152678,-0.4024859965,0.7426589727,0.6694399714,1.4347842932,-0.3065627217,0.3679106236,1.2681542635,0.0654527098,0.8345687985,-1.1156513691,0.8476576805,0.2385712713,-0.4635876119,-1.1457544565,-0.0187508408,0.5387160778,0.2548680007,-0.0915771574,1.0684789419,1.0852134228,-0.8394083977,-0.1809697300,-0.3924306929,0.1450937837,-0.0587084554,0.6864700913,-1.0417679548,0.2963407636,2.9507956505,-1.9250681400,-0.1433570832,-1.1863735914,0.1866655797,0.9594634175,0.3174482286,1.4533431530,0.5670638084,1.3095489740,0.1332997233,-0.9662900567,0.3597651422,0.1602782160,0.2071735710,0.8496370912,-0.7209706306,0.1189359799,-0.7544926405,0.8743216991,-0.6681683660,0.3383395374,-1.1891311407,-0.0207240880,0.3600969017,1.0823884010,0.4390854836,0.6673156023,-0.1929547191,-0.0451419055,-0.4575970769,0.2687020898,0.5414130688,0.5603662729,0.4634231329,1.2978063822,-0.2309095711,1.7121770382,0.2484276593,-0.7386830449,0.3304316700,-0.0578057617,-0.0305827893,0.9412934780,-0.5984822512,-0.9803550243,0.2070207894,-0.6550219059,0.1320174783,0.7900218964,-1.3689039946,0.9380484223,0.4847313464,0.2433396876,-1.4307051897,-1.0672821999,-0.7570881248,-0.4750105143,0.0807691738,1.2956811190,0.5300829411,-0.4246348143,-0.6007465720,1.3769870996,0.4140109718,1.0277614594,-0.8248614073,0.2571107447,-0.7096899748,-0.1520026624,-0.6228904128,-0.1400981396,-0.1943059713,-1.3681163788,-1.1639925241,0.4308236837,0.1339069456,-0.8116710186,-0.5282785296,0.4628006220,1.3132369518,0.8331753016,-0.2018923014,0.0933108181,-1.0099724531,0.4170527160,0.4332081079,-0.2000631541,-0.4819248021,0.0526755936,0.5904321671,0.4194876850,1.4380538464,-0.2091670036,0.1405492425,0.8455461264,-0.1576862335,0.7914052606,0.5603117347,0.6225966215,-0.6495121121,1.6057735682,-1.4565155506,-0.1592467129,0.9452735782,-1.1509324312,1.1862341166,-0.9232009053,-1.4868230820,1.2887068987,-0.5039800406,-0.9293571115,1.4114164114,-1.3048022985],[-0.5259607434,0.1474925727,-0.6388627291,-0.4544603229,1.6006817818,1.8981077671,1.2952227592,-0.6770508885,-0.0343132466,0.0424407683,0.0593665354,0.1125282869,0.1721530408,0.2323742956,1.0071390867,1.1618844271,2.0738105774,-1.5517368317,0.5792925358,-1.5301368237,-0.9412553906,1.5477662086,-0.1588866711,-0.5141246915,-0.1607577950,0.5885755420,0.7499521375,-1.0819777250,0.2867581248,-0.8426262140,-0.4466435015,-0.7166017890,1.5151894093,-0.3630546331,-1.2952378988,0.2233130634,0.7022958398,1.2997992039,-0.2185886055,-0.3559314013,0.9324097633,-1.1729716063,0.8698990941,0.3515312672,0.5347394347,0.5843375325,-0.9044497013,0.2044183016,-1.1293611526,1.0726300478,-0.5822884440,1.2378178835,0.2893677652,-1.0045837164,-1.0325323343,-0.9010717869,-0.5148781538,0.4178542495,-2.0488326550,-0.9897443652,-0.3382941782,1.5038264990,-0.2582089603,-0.1545956433,-1.6558270454,-0.0935545862,-1.0900809765,0.7780081630,2.1689538956,0.5874821544,-1.1014442444,0.6476220489,-0.8646059036,-0.3191539943,0.1379110068,1.2473257780,0.1051613837,0.5986577272,0.0665234625,0.6185915470,-0.1027515382,1.2281802893,-1.4339200258,-0.7613863945,-1.1129082441,-0.6932907701,-0.9792848229,0.4943833947,-0.6510017514,-0.0364130028,-0.5350659490,-0.7569305301,0.1001948863,-1.3082510233,-1.0891143084,-1.6640188694,-0.7002798915,-0.0759273991,1.1599098444,-0.8365039229,1.5821044445,-0.1271798313,0.4336561263,0.1694084853,0.5724648237,1.8298932314,1.4186511040,-0.9776277542,-0.4195668101,0.4297143519,-1.0633279085,0.3911823630,-0.0747859403,-0.9993607402,0.3444029987,0.9736811519,-0.9430867434,1.6025952101,0.2376157790,-0.1281956732,0.3923225701,-2.5928940773,0.7752914429,0.2649167776,-0.4567572176,0.3371549845,0.4049337804,-1.0609476566,0.0936940014,-1.5922925472,1.8138062954,-0.1034576967,-0.3052410781,-1.4047080278,-0.3657502234,-0.0972745717,0.2751365006,-1.5588892698,-0.3687875867,0.7465646267,0.3279536366,0.1500969678,0.6712778807,-0.3980213404,-0.8001051545,-0.5537888408,0.7133117318,0.8906948566,0.0045022634,-0.9853961468,-0.1782789230,0.9238131642,0.7145440578,-1.0212540627,0.2322989702,-0.1549165547,-0.3999933898,-2.6583871841,-1.0034289360,1.3892837763,-0.0713517517,0.1388881058,-0.0967622548,0.4031153023,0.6281492114,0.5679972768,1.1819807291,0.4587058425,1.7037755251,1.5106276274,0.0188147873,-1.2132136822,-1.1595399380,-0.9960440993,0.1820143163,-1.2581967115,-0.0109062744,1.2203755379,0.6200798154,-0.2033996284,-0.0597796105,-1.1121650934,0.3698798716,0.2429883182,-1.4556866884,0.2045957595,0.2362975627,-0.2086562663,-0.8186010122,-0.2720654607,-0.5458545089,-1.2913902998,1.2412921190,1.4953358173,-0.2214281559,0.5088971257,-1.5477359295,-0.7341091037,-0.0617847852,-0.9167001843,1.7482943535,0.6113579273,0.1244715825,-1.7654695511,-0.8131557107,-0.2266721576,-0.2433164567,-0.9586618543,-0.3598257303,-0.1786702126,-1.8549596071,0.4828202128,-0.2374148220,-0.2234849483,-0.7928118110,-0.0088003566,-0.1138385981,-1.1613034010,0.5579746366,-1.1964149475,1.2761023045,-0.5428506732,-0.7676082253,-2.0380215645],[-0.8374180794,1.8076927662,0.6725515127,-0.4143821895,0.0190587919,0.6254171133,-0.2820325494,0.8748767376,-0.5163024068,-1.0898655653,0.1275738180,-0.1395269185,-1.5575122833,0.4249118268,-0.1607739925,1.3152391911,-0.7662281394,-0.1419369578,-1.1442737579,0.5419403315,-0.0603652373,0.0115801683,-0.4260250032,-0.8693335056,0.3321054876,-0.2232301235,0.1859180033,0.0755599067,0.4812561274,0.0805540904,-0.1881784499,-1.3111925125,-0.0887244120,1.5127700567,0.5737081766,-0.5410040021,0.1011774912,0.9945519567,0.0187130310,0.2556715012,-0.7111772299,1.1038419008,0.7498760223,-0.6901777387,-0.6161143184,-0.8754807711,1.3833988905,-1.4642595053,-1.2905743122,-1.1369878054,-1.8163366318,-2.0593073368,-1.5773495436,-0.1809103638,2.6330025196,-0.8738535047,0.4626492262,0.5717197657,-0.5768548846,-0.3781278729,-0.1874432266,0.1967199296,0.3621532023,-1.7878540754,1.1232843399,2.2314839363,-0.2592034340,0.4197042882,0.3647200167,-0.4982883036,0.8533704281,0.6698650122,1.4574915171,0.1867191195,3.0060515404,0.0711427107,-0.2154966444,0.0293234326,-0.0889864117,0.1243936792,0.0645327345,-0.2294944078,-1.0487772226,0.1695384383,-0.4823871255,-0.2830505073,-1.2472976446,0.1283144057,0.4478857815,-0.6975439787,0.1654880643,-0.3362832665,-0.1657265425,-0.6748378277,-0.6607167721,-1.2857440710,0.2204591036,-1.4501774311,-0.0336493291,0.8193687797,-0.7773550153,1.2970961332,-0.5298593044,-0.2872518003,-0.0863314644,0.3339187801,0.3005096018,0.9379919171,-0.4672140777,1.9484413862,-0.3087449372,-2.0272562504,0.2302280813,0.1488207579,3.2926943302,-0.5023747087,0.9368308187,0.4223787189,-1.2146153450,-0.3633534908,-1.0163754225,0.8161545992,-2.6420648098,-0.9995900393,-0.6842966676,-1.3786201477,-0.1166622937,-0.5009272099,1.3049273491,-1.1700606346,0.4273369908,-0.4868766069,-0.9399681091,0.1936713308,-0.1304665208,1.6364108324,-1.2473168373,0.4555480182,0.7874873281,1.0548788309,1.4028953314,-0.3403153718,-0.7662428617,0.7910608053,-0.5680841804,-1.6864377260,1.7951066494,-0.5170757771,1.3853578568,-0.4875607193,0.9538254142,-0.4127383828,1.2545660734,0.9811357856,2.5652039051,0.4855013788,-1.6319627762,-1.9827576876,0.7196250558,0.3306773603,-0.3879871368,1.6143678427,-1.7628905773,-0.2992368042,-0.2816053331,-0.4913335741,0.1062109917,0.6110439897,0.4482290745,0.8110507131,-0.0504925326,-1.0317854881,0.5823349357,0.8444985747,-0.3849413693,-1.3550829887,0.6306875348,0.3171112239,0.0095400224,0.8568302989,1.8019857407,-0.9952394366,3.2118475437,-0.1661012471,1.7766364813,-0.3340893090,-0.2285335511,-0.0810918510,-1.7271569967,0.4434600174,-0.7047592402,0.5324918628,0.4588414729,0.6069088578,-1.1168786287,-0.4205290377,-0.4082236588,0.9159150720,0.0197826736,-0.9659548998,-0.0031386658,-1.1092762947,1.1993969679,-0.9169333577,-0.3250979185,-0.6554597020,1.0403012037,-0.8468415141,-0.0201100614,1.0483227968,1.3847842216,0.2876734436,0.3542454243,0.4986132979,-0.2432114482,-0.1372609735,0.5234652758,-1.2656041384,0.4806742072,3.0031232834,-0.1512721330,-0.7243952751,0.0387897491,-0.1198186129],[0.8208485842,-1.0074971914,-0.6677929163,0.0483032688,0.1750379354,0.2083159089,-1.9190596342,0.9766510725,1.8913180828,0.4273438454,-1.3285946846,-0.1469343305,-1.3562725782,-0.3669628799,0.1919582635,1.3829925060,-1.0648114681,-0.2708503604,2.7610263824,0.6004490256,1.3111655712,0.9766030908,-0.1186602041,-0.4448900223,-1.0978698730,-0.4263503253,-0.2740243077,-0.1068669632,1.0883152485,-2.9813721180,-1.1545959711,-0.0348202735,0.2651986182,-0.4033118188,-0.8410776854,1.0309700966,0.4036042690,0.1645698398,-0.4990791678,0.8270534277,-3.1262013912,0.2744820118,0.8101330996,0.7581610084,-0.7181271911,-1.3808708191,-0.5076599717,0.0838778764,-0.8853391409,0.6123282313,-0.1626137942,0.2677460611,-1.4126522541,-0.4442411065,-1.2625015974,1.5032956600,0.6110239029,-0.4633914828,-0.2613438368,1.8707574606,0.6762051582,-0.3888108432,-1.0406956673,-2.4245371819,0.4552400112,-0.4199117124,2.0087494850,0.6389307380,-0.3146655858,-0.0531506911,-0.1081308648,-0.8942368627,-0.2720023394,1.1744161844,-0.5152111053,-1.5052090883,-1.3972624540,-0.0511602834,-0.0412393324,-0.2354413718,-1.2756769657,0.1774501950,-0.1988486648,0.8698195815,0.4150623381,-1.1867893934,-1.2666206360,1.5865522623,0.0610986762,-0.1770948023,-0.5854319930,-0.4385347962,0.0175962076,1.3314625025,1.5840754509,-0.3236637712,2.3411216736,-0.6135569811,0.9249237180,-0.2237814665,0.8911208510,0.1451560408,0.2448153049,0.3776126504,2.0296990871,0.6324076653,0.3589432240,0.4496875703,-0.3058456481,1.1094084978,0.4872805178,-0.3038507998,-1.4573172331,-0.9205236435,0.6826651692,-0.3554729223,0.3927389085,-0.3810762763,-1.7063821554,-0.1602554619,-1.6169706583,0.6162632704,2.4065866470,0.3152024150,1.2607896328,0.3310032785,-1.3002210855,0.5527735353,-0.7645601630,-1.6030761003,0.2826044559,0.1884256005,-1.1052546501,-0.0256252103,-1.4617702961,2.3069312572,0.1014108062,-0.5887804627,-0.0482289307,0.7604926825,0.7126061320,-1.1288833618,0.3887622952,-0.2175828516,1.4924032688,-0.1203889251,-0.4008507431,-0.4998865426,-0.9092680812,-1.4348453283,-1.2464061975,0.4905832410,-0.4584877491,-1.7624422312,-1.2848756313,0.1869234592,1.6680623293,1.4857335091,-1.3092820644,-0.4001376033,-0.6902616024,-0.2851842344,0.5611698031,-0.2224484831,0.1025903299,-1.3632470369,-0.5505031347,0.9418570399,0.4167126417,1.5588051081,-0.4624279737,-0.3520548344,0.6923726201,-1.3459591866,-1.3967781067,-1.2523069382,1.1929949522,-0.8578463793,0.2459804118,1.1848936081,-1.6676170826,0.3219023943,0.1425884366,0.8878026605,1.3843916655,-2.0635306835,0.4181311429,-1.6780023575,2.8656015396,-0.6755149364,-1.2139745951,-1.7235435247,-0.0115590412,-1.2834458351,0.6609151959,-0.1157040447,0.3007107079,-0.9618673921,0.6788420677,0.4106647968,0.7691630721,-1.8603491783,-0.2602683008,-0.6554810405,0.8151291013,-1.1044774055,-0.0258444790,-0.6846947670,0.7430288196,0.0927735269,-1.8108327389,1.4642354250,2.1097369194,0.5113024712,0.9214703441,-0.7961116433,0.1557842940,1.6264904737,1.6839746237,-0.1017777249,-0.0270002671,-0.5560973883,-0.3349419832,0.4305644631],[-0.3148517013,0.5323333144,0.4808635712,0.6366700530,1.1908731461,1.2183773518,-0.0871770829,0.1331706196,0.0532641746,-1.2944794893,-0.6270956993,1.0332086086,0.8411585093,-1.1188211441,0.9106353521,2.4634940624,1.3054764271,-0.5078490973,0.6401654482,-2.0949027538,-1.5359004736,-2.1329395771,-1.2214189768,-0.0510937348,-0.6269537807,0.9085376859,-1.7701388597,0.1271123588,0.2025357783,0.2193867266,-0.1419527531,0.4712587297,-1.1216319799,0.2348901033,0.2660713494,0.1221213117,-0.0339588486,-0.4119123518,0.2396080047,0.4450393617,0.1456137896,0.1783765703,-0.8795627356,0.0829178467,0.2220709324,-0.2598603666,1.2970924377,-1.6507456303,0.6115096807,-0.3253150582,0.1117031053,0.7479269505,0.4571745992,0.1007617787,1.4383536577,-1.4001889229,-1.9547203779,-0.7588565350,0.1194262281,0.7364101410,-0.6658715010,-0.0521111190,0.1420148462,-1.2008237839,-0.0141287912,0.1722817719,1.0185024738,0.3625552952,-0.2192810923,0.6853411794,1.0109658241,0.0720389709,-0.5317717195,-0.2604891360,-0.2174306065,0.9860923290,0.3199743629,-0.0393218994,1.5142209530,-1.8734128475,-0.8563058972,0.7394201159,0.9219983816,-0.8890936971,-0.5466315150,0.5607649684,0.8172806501,-0.2346559316,1.4355891943,-0.5468201041,0.1247638464,-0.6282752156,-0.2512498796,0.0405594669,0.4966886342,0.7192645073,-0.5121807456,0.2786832452,2.3277153969,1.7324640751,-0.1355056465,0.9407029152,1.7785062790,2.0651509762,1.1749883890,0.0644653961,0.3345230818,-0.2759347260,-0.9038693905,0.0737935454,1.7145204544,-0.0049464516,-0.4580802917,0.8939317465,-1.5609588623,1.1997606754,-1.3571116924,0.2147751749,0.6041502953,-0.4827028811,0.4263130724,0.3580539525,2.0956006050,-0.2793421745,-1.6219590902,-0.2702989280,1.0025880337,0.7011756897,0.1722511351,-0.1773850620,0.7790995240,0.8636000752,-1.0049010515,0.6323234439,-0.7638198137,0.3079225421,-0.3288014233,0.3946300745,0.8784893751,-0.1306656599,3.0949795246,-1.4146577120,0.3331373334,1.1519114971,-0.1283607334,-0.6941428185,0.1147115529,-0.0274392385,-1.7293875217,2.3312771320,1.6542543173,-1.7632293701,-1.9415524006,-1.1907383204,-0.0048493538,0.4222418964,0.0341976583,1.5213152170,-0.1766054034,0.2243996412,0.7282629609,0.1159321964,-1.4154875278,0.3165683150,0.8783222437,-1.1561028957,0.5572648644,1.5004160404,-0.8511629105,-0.5939493179,-1.1936300993,0.6819807291,-0.0313347243,-0.0286470875,0.8888990283,1.0018740892,-0.1425251812,0.2556823492,0.7652249932,0.1631320864,-0.2498634607,-1.5008529425,-2.9273338318,0.6607398987,0.1327477843,0.5937239528,0.1125463098,-1.6437536478,1.5341486931,2.1285750866,-1.0796158314,-0.2879979610,-2.2914717197,-0.8244517446,0.4066144824,0.6147168875,-0.5193148851,-0.7325679660,0.4611653090,-0.4268031120,-1.1843461990,-0.1097135618,-1.1662861109,1.3328620195,-1.4928250313,-0.2382298410,-1.2711412907,-1.0696352720,0.6976164579,0.3925410211,1.2439258099,-0.3567233384,1.2630944252,0.0982297659,-0.5787029862,-1.2956535816,0.8188961744,0.4646765888,1.2062302828,-0.0755145028,0.2233940512,0.0219676215,-0.2262375951,0.8005526662],[0.2583341599,0.0134167904,-0.9566358328,0.6107801199,-0.3282876909,-0.5326413512,0.3580334187,1.6714169979,1.4230730534,-0.5663484335,-0.7258203030,0.5634755492,-0.4356320798,-0.1045525447,-2.3212728500,-0.2140943706,0.5434054136,-0.5374541283,-0.3802753389,0.2654729187,-0.1454317421,0.3900747597,-0.1429263800,-0.0141180474,-0.5412923098,0.9153400064,0.7681113482,0.1016348004,0.8094421029,0.0004556435,-0.2267173529,1.2817173004,-0.0734994039,1.0696345568,0.7920151949,0.3397075832,0.6335131526,-0.3126899004,0.3656824529,0.3227659762,-0.1339740306,0.3025370538,-1.6606614590,0.1812739521,-0.3153503239,-1.0073664188,-0.7484238744,-1.7166824341,-0.1572054476,0.8618949652,-0.9956087470,-1.1582169533,1.3176295757,-1.6233000755,0.1798952520,0.6858589053,0.0602968559,-0.6038802266,1.0489137173,-2.2571678162,-0.0584337749,-0.1138860956,0.0855550766,-0.7688744068,-1.2541327477,-1.4902909994,0.1294953823,0.0926831588,-0.5188183784,-0.7847260237,-1.0042928457,-0.0707495660,-1.5369973183,-0.8811527491,-0.9041975737,0.7040268183,1.1990603209,1.8376708031,1.3268336058,-0.1728139222,-0.1250518411,-1.5261913538,-1.8103522062,0.3975400925,1.7494088411,1.7238782644,-1.3281983137,0.1908317953,-0.8550479412,0.1436198354,-0.3308296800,0.1347103268,-0.5981942415,-0.9036156535,0.8448509574,-0.5984261632,-0.3971627057,-0.1978619695,-1.2565414906,-0.4385175109,-0.6725533009,-0.5639172792,0.0460369922,-1.9070044756,0.4588394463,-0.4961534441,0.5328682065,0.2758805752,1.0665974617,-0.1897082031,-1.0725839138,0.9363661408,1.1084721088,-0.3609683514,-0.6044198275,0.7308124304,-0.2895660102,0.9214672446,1.1936705112,0.1420208365,0.9903038144,1.0248546600,0.7819829583,-1.5155851841,-0.1204732060,-0.2646034360,-0.4781559408,-1.2578017712,-0.8879620433,0.8834035993,0.3680997789,-0.3455525041,-0.1083529815,0.8362635970,-0.8118629456,-0.8376584053,1.3152875900,-0.3645234704,1.9935706854,1.5848784447,-2.1046626568,-2.5531182289,-1.2426662445,0.2019866556,-0.3053317666,-1.1955868006,-1.5779031515,0.8499122262,0.3275901377,-0.0016695884,-0.0355634280,-0.4892515540,1.9304982424,-0.2626447082,0.8259323239,-0.6432670355,-0.8289811015,-0.2027345747,-0.2578664720,0.0708145201,0.9978447556,0.2600807250,0.9250656366,1.4760758877,-1.8792523146,-0.7825285196,-1.4022051096,-1.4552857876,-1.1523354053,0.0965485498,0.2922802567,0.8867214322,0.1132206246,-0.0294816475,0.2782558203,0.9704330564,-1.4312628508,-1.4022347927,-0.1946246922,-0.5354971290,0.6042767167,-1.4157117605,-0.9723172784,1.0312830210,1.4538744688,0.2337075174,1.3080239296,0.7617688179,0.5921661854,-1.2361630201,-0.9084081650,-0.7632186413,-0.8020383716,-1.3764209747,-1.5502327681,-0.1663934290,0.7598895431,-0.2875463068,-0.3408353031,-0.4380674958,0.1424137503,-0.2995167673,-2.1005253792,2.4057154655,0.3382180035,0.3632414639,0.3133047223,0.4243564308,0.1828026623,0.4043556750,2.5423345566,-0.3791434169,-0.1495744288,-0.8346999884,-0.7742741108,1.2149647474,0.7606167197,0.3171077371,-0.1672189832,0.1230465323,-0.5340057015,-0.0623281896,0.4993795156,-0.6112615466],[0.2279778421,-1.2632958889,-1.0773020983,0.3607445359,-0.2576623857,0.7818472981,-0.4763545990,1.3054454327,0.6843120456,0.4527694881,-2.4472017288,0.6361515522,-0.5922811031,-0.1407441050,-1.1186494827,-0.2712420821,0.3353216648,0.9042813778,-2.1890122890,-0.3902392983,-1.2158573866,-1.4680877924,-0.2748048902,0.8750464320,-1.0083405972,-0.4647849798,-0.1865984648,-2.0889463425,-0.5600870848,-1.3486815691,-0.3641701937,1.1793979406,0.1997187734,1.6865342855,-1.0752303600,0.5811572075,0.3023730814,-0.5949453712,1.3834298849,0.3260892630,-1.5045257807,-0.6118640304,0.1986764967,1.0946094990,0.9827951193,-0.0680601895,-1.0457265377,-1.7469846010,-0.5958772302,-0.3935099542,0.7819852829,0.1315787882,1.3365695477,1.2137441635,0.4329839945,0.7754896283,1.2031998634,-0.4921071529,3.0196695328,0.6024559140,0.2277309895,0.5269126296,-0.1516455263,-0.2588847578,-0.5245341063,0.2717128098,-0.8874869943,-0.8964639902,-0.1968666464,0.0911976174,-0.8873035908,-1.8681821823,-0.0760981515,-0.7805968523,0.4802442491,0.0320365019,-0.7067071795,-0.4631549716,-1.1735436916,0.5445239544,-1.3017100096,0.3699762821,-0.0850021094,-0.7237022519,-0.7464592457,0.2509557009,-2.1246905327,1.6723703146,-0.7169879079,-0.5348271132,0.3575956225,-0.2395388484,1.9923259020,0.1982047707,-1.4630272388,-0.5053290725,0.8650518656,-1.3065795898,-0.6528154612,1.7014002800,0.2201984078,-1.6176035404,-0.5858163238,0.3461747766,1.7925175428,-0.8658657074,0.0194235109,-0.2472789586,-0.9830157757,-1.2460169792,-1.4011884928,-2.3403863907,-0.7892500758,-0.1878778636,-0.0241079796,-1.2374361753,0.4444930553,1.4464956522,-0.6280738115,0.0667637363,0.6859157681,1.8527070284,1.5816912651,0.0943878442,-0.6667955518,0.6660448313,0.5320387483,-1.0508078337,-0.5289185643,-1.9789937735,-0.0936506018,-0.5537081361,0.1577619612,0.3714364469,0.4536406696,1.1362136602,0.4127557874,0.0746068135,1.4729266167,-0.0521894582,-0.3334014714,-0.2249729484,-0.5179290175,-1.0466347933,0.9848658442,-0.0994437635,1.7418688536,1.1313471794,1.5630669594,-1.0602076054,1.6669305563,-1.4789799452,-1.9057705402,0.3552731574,-0.7447243929,0.1306759268,0.9287368655,-0.1092540473,0.3579307199,0.0620544627,0.9341294169,-1.3217412233,-0.1887846887,0.5450025797,1.1946133375,0.1155479774,-0.0156681519,0.0818186924,0.3128174245,-0.5175796747,-0.9493309259,-0.5009193420,0.3581947982,-0.6916647553,0.7086075544,1.2116622925,-0.5839709044,-0.1023268700,-0.4031613171,-0.0775697455,-0.1116259247,0.5571227670,0.6536220312,0.1175153852,-1.2265955210,0.2845886052,-0.3341941833,-0.9034729004,0.3816067576,-0.0520795807,1.3590296507,0.2401977628,-0.5103403926,1.4039888382,1.0099010468,-0.4682047069,-0.1903148890,0.2275711149,0.3296707869,1.3139348030,0.3545247018,1.6756818295,-0.4598955214,0.0968695879,0.9012082815,-0.6129329205,0.5607587099,1.7570564747,0.6849796176,-0.1868782192,-0.4102973640,-1.4443739653,-0.0522584990,2.2450199127,0.1798616350,0.7548625469,1.1856045723,0.2843368649,-0.4107406437,0.4364345074,1.2590898275,-0.3064253330,0.7047862411,0.4805850387],[-0.1725681275,-1.0563880205,1.0208398104,0.9267752767,0.1891755462,-0.3822020590,-1.4817459583,1.2379530668,-1.5762476921,-0.8821061850,1.7232716084,-0.5654813051,-0.0660265014,-1.1681851149,-0.6628458500,0.7722485662,0.2607228458,2.2022979259,-0.7688165307,0.0033735237,-0.1173261702,-0.9136849642,0.3267530501,-0.1441042274,-1.2224164009,0.8834843636,-0.6588438749,-0.2248045206,-0.8135863543,-1.1237211227,-1.0655337572,0.4619101584,-0.7503932118,0.3477369547,0.2655709982,0.8278338909,-1.2869812250,0.3188256621,-0.8562581539,0.7813947797,0.6154556870,0.6575708389,0.1797037572,-2.2783794403,-1.3994863033,0.7637216449,-0.6022380590,1.0573459864,-1.2417126894,0.8444338441,0.4190975428,0.5408490300,0.1426850855,-0.9931777120,-0.3794707060,0.5953972340,-0.8911392093,1.2549008131,1.0709406137,1.3681180477,0.7540420294,-1.3617898226,0.2148898542,0.7666696310,-0.5721467137,-0.9326041937,-1.3463188410,-1.0583540201,0.3082196116,0.1263058931,1.4746549129,0.0198656376,-0.1533403397,0.5427250266,-0.8982430696,1.0728341341,0.9212007523,-1.0459827185,0.2398848087,-0.2399872392,0.2015265524,1.0953737497,-0.7641457319,-1.0590814352,0.9709772468,-2.7245488167,0.0263428781,0.0311356522,-0.1787093282,0.4194993079,-0.8683885336,0.4747060835,-0.4976926148,-0.6131805778,-0.8993081450,-0.6951372623,0.2548579872,2.4627816677,1.2880226374,1.3663511276,-2.0841400623,1.0545402765,-0.6584460735,0.7016707063,-1.4842098951,0.7018342018,-0.7367026210,-1.1353683472,0.8034266233,1.9471383095,0.2818152010,0.6163467169,1.6901090145,-1.7596832514,-0.0940287039,-1.5356237888,-0.4820639789,-0.2813972235,1.9129413366,0.1789698899,0.2698389292,-0.4756282866,-0.0582442805,0.6104027033,-0.1537651569,-0.3071576655,0.0031420253,-1.1825238466,-1.8446222544,-1.3169915676,0.1479340047,-0.6539691091,-0.3660846055,-0.5772585273,-0.5850701928,-0.2924168110,-1.8485302925,-0.1489338130,-0.7512722015,-0.7153009772,1.0778415203,0.6231737137,-1.6968379021,-0.8516989946,-0.9225254059,0.9233241081,-0.3950044215,0.7099353671,-0.0429341830,-0.6496396065,-0.3185375035,-1.8720908165,0.0721271336,0.2912057340,-0.2783482969,0.6049549580,0.6706092954,0.7283976674,1.3359290361,-0.8727504015,-0.1820479333,-0.2766488492,0.5385487080,-1.2421909571,-0.6489337087,-0.8941696286,0.1014679298,-0.0520993546,-0.6106041670,-2.9797456264,-0.6892374754,-2.2069187164,-0.4327895939,-0.6774798036,0.1662548184,-0.1374444216,-0.7766288519,0.4023796916,-1.8384613991,-0.7337769270,-2.0104799271,-0.8689224720,-0.0495265275,-1.3442329168,0.0602338761,-1.7688345909,-0.8403044939,-1.0115302801,1.9035303593,-1.2843977213,-0.4898892343,0.7581721544,1.1472157240,-1.3688549995,0.6080808640,-0.5300948024,-0.6094548702,1.0624125004,-0.0401498191,-0.2764210701,1.5775593519,0.2636820376,1.2670412064,1.1847480536,0.6004807949,0.6627605557,-0.9631214142,-0.0651835725,0.3965368867,1.2247297764,-1.5966616869,-0.9161520600,2.0568938255,0.1804197133,-0.0667092502,-0.0283539798,0.4848047495,0.8173012137,0.8953057528,-0.7625162601,1.4375944138,1.1974359751,-0.6648783088,0.4911331534],[0.4361787140,2.4836254120,1.2508749962,-0.8652247190,-1.1523828506,-2.5317647457,0.6768011451,-0.8235494494,-0.0016548608,-0.2926580310,-1.7133356333,0.2775128782,-0.4684120715,-1.0212373734,0.3111077845,-0.5543714166,-0.5277050734,0.0934340209,0.0553637892,0.1839600801,1.0744355917,-0.8088902831,0.0177089591,-1.0553754568,0.0537255295,0.8920047283,-0.1436830014,0.5428864956,2.6151807308,0.9089220762,-0.6701074243,0.1462989897,-0.4177500904,-0.3071745634,0.2703181803,0.0061454624,-0.0414125137,1.2520478964,-0.5853372216,0.0183674395,-0.3882516623,1.1016137600,1.8874223232,-0.6500365734,1.4510560036,0.2006217688,0.1075251549,0.4110045135,0.2201270163,-0.9558433890,-0.0729107559,-0.2200553864,-0.1955879778,-0.3888328075,-0.1492266357,0.2064908445,-0.4907214046,1.3380523920,1.0333486795,-0.3924434781,0.2761882544,-0.0500014015,0.2783436775,-0.2208845317,0.4076091349,-1.6868382692,-1.1322995424,-0.4491987824,-1.6285598278,-0.0747513771,-0.9900171757,-0.5021103621,-0.0169003569,0.6617447734,0.5219373107,0.0710919797,-0.1033318341,-0.1241831928,0.5050722957,1.7465324402,-0.9384755492,-0.2050182074,-1.4408466816,2.0659956932,-1.2176458836,1.0436911583,-1.6367999315,-0.8311728239,0.0383930989,-1.4327731133,1.6502213478,-0.0957067087,2.2228686810,0.3076825440,-1.7243221998,-1.6951988935,0.2783026695,0.9383012056,-0.8741029501,-1.1462081671,-0.6457691789,-0.8208730817,1.1436293125,1.4125794172,0.2804195881,-1.6822501421,-0.4242359996,0.6605323553,-0.0331386216,-0.0384374969,-0.8116567135,0.1741831154,1.4778373241,-0.2192885429,-0.5614511967,-0.8825250864,-1.2342511415,1.1353919506,0.8536889553,0.6619521379,0.1940427870,1.2519853115,1.4540354013,-1.4876911640,-0.5286277533,-0.5613620281,0.4594193399,0.4614850581,-0.8415557742,-2.4047565460,-1.4944550991,-2.2047286034,-0.7104699612,-0.6865041256,0.4871855676,0.9703326821,0.0191650800,0.0305574350,2.1260392666,0.3825403750,0.1017763391,1.2437696457,0.0841875896,0.0260246377,-0.3963061869,0.2071161419,-0.4882599413,-0.1168544292,-0.9286617041,1.1061260700,-0.2454181761,-0.0635446012,-1.3290656805,-0.2628045976,1.4840424061,0.5852842927,-0.6792443991,-0.4082862735,-0.3425306380,-0.5310598016,-0.2525795102,1.2393785715,-0.6161656976,-0.8743874431,-1.1418819427,-0.5788629055,2.3537743092,1.3391598463,0.7086358070,1.0133312941,-0.6402965188,-0.4752223790,1.2483383417,-0.0412226953,-0.3494254649,1.2869471312,-1.3261907101,1.0926253796,0.3833552301,-0.6908642054,-0.0721437111,0.4729478657,-0.4645186067,0.8151687384,-0.7450098395,0.3585434854,-1.3225007057,-0.5362467766,0.0556739420,-0.5106480718,-1.4202176332,1.1659399271,0.7442878485,-0.6453229189,0.4733515680,-1.0344504118,1.7793974876,-0.7470325828,1.7667423487,1.0275626183,0.0522043034,-1.9739670753,-1.1114044189,-0.3246097863,-0.3506036103,-0.3710768819,-1.9017586708,2.0592153072,0.1685639471,0.0427768752,-1.3544977903,0.5274385810,-0.8166235089,-0.6331622601,-0.0177748725,1.5376009941,1.5840349197,-1.5073336363,0.0665762797,0.3779613674,0.9021579623,0.0178753231,-0.1060614064,0.9497752786],[1.1455736160,1.1762216091,-0.8378641009,-0.5064931512,-0.5005347729,0.0742226392,0.5423977971,0.6719839573,0.3955279887,0.1929187477,-0.0416823812,0.5823065639,0.4075008333,0.9344403744,1.6156861782,-0.4030669630,-0.5414798856,0.3706814051,-0.4143368900,1.2245029211,-1.7612340450,0.0220842361,0.0963661671,-0.9343460202,-2.0305268764,-0.6361606717,-0.4195115566,0.7116051316,0.6106399894,-0.2644272745,0.4211940169,-1.1107473373,-0.6500771642,-0.1556589901,1.8477299213,-1.4575940371,0.4544501007,0.5361469388,-1.8324573040,0.0428999849,0.5673006773,-0.3812290132,0.1586058289,-0.7432076335,-2.3003587723,-3.0178782940,-0.4192528725,0.3673498631,-0.3779532313,0.3696709871,0.7667483687,1.1323468685,0.1713690609,1.9920309782,1.0371128321,0.7972601652,-1.5348200798,-0.2240379602,-0.7211647630,1.2391200066,-0.5251198411,0.3672321141,-0.2873863578,0.1083054766,-0.3605248034,0.0712789819,0.9799603820,0.6624241471,-1.7817859650,-1.0848448277,0.5504264832,-0.4166675508,1.6126602888,-0.2049243599,-1.0211094618,0.3455152214,-0.5317375660,-0.7377064824,-0.4188673496,0.8521258235,0.2819546163,0.8450640440,-1.0280054808,0.9755204916,-0.0629693344,-0.6170272827,-1.3914250135,1.1874107122,0.1367413551,0.4460196495,-1.2611466646,0.0386255011,-0.9642368555,0.4145136178,-0.1752219051,-2.5754177570,-1.6540249586,-0.0349275842,0.0448959023,-0.4018982351,-0.7180095911,-0.0794271529,-1.0193690062,0.0466088690,1.0369007587,-0.4155933261,-0.6224948168,0.2363770604,2.0348365307,-1.3155070543,1.4027558565,0.7323297858,1.2334948778,1.3659120798,-0.0808410868,0.5328992605,-0.0881688148,-1.8309184313,-0.5434551239,0.6239242554,0.8248231411,-0.5365520716,-1.2415540218,0.7438274026,1.1258167028,0.1619975269,0.2171468437,-0.0945205837,-0.7705362439,-0.2652542293,0.3964420557,-1.0244829655,1.4870411158,-0.1595176905,-2.5220484734,0.8388492465,-1.2317235470,-0.9562405944,-0.6581323147,-0.2054009885,0.0864707008,-1.5467650890,-0.0167100113,0.0975208133,-1.6225047112,-0.6531865597,1.1103789806,0.1780981272,1.4125519991,0.6659147143,0.7821815014,-2.0617392063,0.8627902269,-0.3878584504,-0.1652761549,0.0142101552,0.0475623906,0.8808799386,0.0930731744,1.9853578806,0.2075347602,0.0492352843,1.8051396608,-0.5307197571,-1.1836994886,2.3241052628,-0.3693919778,-0.0356154442,1.7207887173,-0.0219147708,-1.0829179287,-1.5767006874,-0.1409223974,1.3477866650,-0.1534931064,-1.1728582382,-0.7144800425,-0.2241252959,0.3540037870,-0.1153135300,-0.9265352488,0.1685252041,-0.6477338076,-0.5638371706,0.1049879938,-0.0841720998,-0.5061149597,-1.2376722097,-1.2302114964,-0.2418429554,-0.0475620590,0.3470554650,-0.7907449603,0.0829786956,-0.9652466774,-0.8953530192,0.7561290264,0.4937201738,-0.3596648276,0.9256682992,1.1828033924,-1.1108970642,0.6139956713,1.3882287741,-0.2518001795,-0.7563911080,-0.9352231622,0.9392328858,0.5460475087,0.9603745937,-0.9488556385,-0.4981226325,-0.4026987553,0.5920459032,0.0090565272,-0.8863218427,0.0490998067,-0.1781470776,0.2800677419,-0.9707691669,1.3887768984,-0.6077256203,-0.2242699414,1.9329805374],[-0.8913836479,0.0187778343,1.3015447855,1.3718682528,0.5478003025,-1.0495789051,-2.1708903313,1.3192451000,-0.9026982784,-1.4011514187,2.0441510677,-2.0233173370,-0.0080679497,-1.6217567921,0.4429032803,0.0473358221,0.9135802388,-0.0456274971,0.0288143251,-0.5566142201,-0.1602125764,0.7894438505,-2.3435871601,-0.8936960697,-0.6324608922,1.4873902798,-0.7355272770,0.0185255464,0.2910588682,1.8088326454,-1.2172501087,-1.4750502110,-0.5276563764,2.0352544785,0.4579901695,0.0531067401,0.1881678998,-0.8126240969,-1.0810470581,0.6841163635,1.4170793295,0.4437287152,-0.0769727081,-1.0915343761,-0.4374568164,1.6740102768,-2.9241530895,1.6825302839,-0.1471130699,-0.3678592145,0.1373748779,-0.2201071382,-2.5014238358,-0.6038212180,0.7263562083,1.2062644958,-1.0378527641,-0.2458584905,-1.3896532059,-0.0786310136,0.9795127511,-0.2071323991,-0.4947267771,-1.6725231409,0.2150016129,-0.6723101735,-0.2168543935,-0.9003168344,-1.2477769852,-0.1771844625,-0.6590757966,1.1977955103,-1.5937381983,-0.7925133109,-0.6221466660,-1.6597610712,0.4350447357,-2.5738360882,-0.6248568296,0.3362247050,1.0735864639,-1.4852718115,-0.2946455479,-0.3111856580,1.2795861959,1.4509670734,-0.9087712765,-0.8750560880,-0.6453881860,-2.5647492409,0.2437494695,0.5734773278,1.1027342081,-0.7475714684,2.0953369141,-1.3439731598,-1.2973996401,1.1596117020,-0.9503767490,0.0014978965,0.1826970875,0.1884649992,0.3975955248,-0.6645811200,-0.1333314031,-2.8415510654,0.4180964828,-2.7764372826,0.2658073008,0.2275587916,1.4011553526,1.5555111170,0.8428824544,0.9028973579,0.3212300241,-0.5996940136,-1.0242420435,-0.6158038378,1.6927067041,0.2687541246,-0.3995000422,-0.3005733490,0.2216373682,-0.2204890847,0.6902763247,-1.4169993401,-0.8151863813,-1.9904801846,0.4423972666,0.6706801057,0.7185264826,-0.2972773910,1.1108764410,-0.8177000284,-0.5754866600,1.1977572441,-2.6281495094,1.5867403746,1.4161043167,0.7671688199,0.1658380479,0.8150725961,-1.6277327538,0.3003188074,1.3336789608,0.8292962313,-0.4848930836,0.1618258804,-0.5849781632,-0.6967284083,-1.1470516920,0.7196304798,-0.9395693541,-1.2485228777,0.9772579670,-0.9749694467,-0.9194549322,0.1552649140,-1.3407961130,1.0570300817,-0.0778691396,1.7038300037,-0.2739207745,-0.3102475405,-1.7548904419,0.4191471636,1.2825931311,-0.5296666622,1.0294588804,0.1527706832,-0.8580433726,0.4660653174,0.2710532248,-0.7573392987,1.7085819244,0.2593532801,0.2727315128,0.0305096321,0.6543181539,0.7176415920,-0.2725006044,1.1646317244,-0.1886187494,0.6548362374,0.7484856248,1.5596144199,-1.2608844042,-0.6304160357,0.6321176887,0.2912154496,-0.8312640786,-0.2090908885,-0.7051216364,-1.5571593046,1.7400833368,-1.1747038364,-0.3725011945,-0.3637978733,0.4852464497,-1.6679120064,-0.4092452228,0.4874619842,-0.0685126632,-0.2348806113,0.1828349084,1.0253461599,-0.3952368796,-1.0767440796,-0.4530940652,-0.5046754479,-0.4909371436,-0.3833635449,-1.8254991770,-1.1022430658,0.3569259942,-0.2243722230,-1.7885184288,0.9822217226,-1.4025393724,0.8213406801,0.3869772255,-1.1860983372,-1.2052828074,-0.1533375829],[1.8314754963,0.8154265881,-1.0724186897,0.8384286761,-2.0999834538,0.2436810583,-0.0349125080,0.7932613492,1.1110419035,2.4974799156,-2.8381979465,-1.0605124235,1.0589070320,0.9585606456,0.8247618079,-2.5230290890,1.5672498941,-0.2026850730,2.0308806896,-0.0637612268,0.5324293375,0.9650161266,-0.3561647236,-2.2441608906,0.5721330643,-0.0420632027,0.8333802819,-0.1294799000,0.2182023227,0.3625458479,-0.3400902748,-1.3537571430,-1.2952094078,0.7723783255,-0.0648406297,-1.2400768995,1.6827946901,1.1401575804,-1.5258969069,0.8159262538,-1.1959249973,-1.5505919456,-0.6034004092,-0.6083353758,-1.0715056658,-0.1194637045,0.0140260626,-0.9964612126,-0.1172712520,-0.5439452529,-1.6995549202,-0.0008625090,-0.9978842139,0.3596127033,-0.7026296854,-0.5324466825,-0.7600529194,-1.3232053518,-0.5340525508,-2.1644289494,0.4067259431,-1.6405332088,-0.3487933874,1.1609245539,0.2904104888,-0.1469300389,-1.5347036123,0.2856205404,0.0134263178,-2.1771950722,1.0045795441,0.8746539354,1.1157619953,-0.0907416940,1.4753232002,0.5067855716,-0.7191897631,-0.3575884700,-0.1088900194,-0.4578658938,1.6910647154,-0.3512211740,-0.3477337062,-0.0016170363,1.6011307240,0.8124353886,0.1603957415,-0.3600437343,-0.2487288564,-0.5773665905,0.8298076391,-0.4896481931,-0.7197877765,-0.0610746816,-0.3374556005,-1.5635737181,-0.6105449796,0.5172725320,-1.2310928106,0.6385520101,0.2386399955,1.5264911652,0.5826241374,-0.6799334288,-0.5346289873,-0.2036975175,-0.7439308763,0.4147465229,0.0078077312,-1.0435984135,-1.2097308636,-0.3265855610,-0.8725820184,1.7610948086,-0.9714349508,1.5349853039,0.2814086676,1.2379722595,0.4718837142,1.1676279306,-1.2804001570,-1.5776510239,0.4372701347,-1.1033475399,0.6475730538,0.4473706782,-0.1467706263,0.7794570923,-0.6803262234,0.9622517824,-0.8587225080,1.2174996138,0.2897004187,-2.5981965065,-1.5418102741,0.0819696561,1.5873248577,-0.7349403501,0.7290760875,0.9269818664,0.7080751061,-1.2577210665,1.1770211458,-1.9203807116,0.7047927976,-1.0936950445,-0.2829663157,0.6887729168,1.2502661943,0.1848202497,0.4263668954,0.0182026662,-1.5792089701,-0.3927980363,0.8779292107,-0.4019509554,0.4690700173,-1.5016077757,0.3696859777,-0.9097312689,0.7302269340,-0.7808380723,0.5710436702,-0.6982168555,1.8683104515,0.0536289178,2.7342181206,0.9965337515,-1.3732835054,0.5279285312,-1.5946539640,0.5114824772,-0.0093094138,-1.9116069078,0.5795866251,-1.3237437010,-0.9147019386,-0.5839483142,-0.1468760073,0.9125059247,-0.1213563755,0.6303017139,-0.4348902702,1.0431065559,-0.3461530209,1.0725315809,0.9376847744,-0.6338280439,-1.7862892151,-0.8356691003,0.4631983340,-1.3741754293,-0.1512538940,0.4590940177,-0.7534619570,-0.8041709065,-0.1798563600,-0.2793759406,1.3496509790,0.9117848277,0.6284425855,-0.1181348413,0.5227888823,-0.8071405888,-1.4427751303,-0.5251013637,-0.4712083340,-0.4328814149,-0.0625080392,0.2701307237,-1.2395265102,1.9819504023,0.5889578462,0.7082515955,0.1354319900,-0.4342207611,0.8350552917,-1.5380432606,2.2015075684,-1.2920140028,1.6503757238,0.4370865226,-0.6922412515,-1.8327370882],[-0.1175616831,0.0189046878,-0.1070931181,-1.7652977705,0.5713495612,-0.8332467079,0.3077534735,-0.5451762676,0.5432214737,0.9431089759,0.2331315875,0.4595543742,0.6053746343,-0.5241636634,0.6633955836,-0.5136868358,0.3695436716,2.5949618816,-1.8505828381,1.3736778498,-0.2972381115,-1.7225773335,0.3269241452,-0.9890488386,1.9549521208,0.5292824507,0.6432071328,0.2097698003,-0.3138158023,0.0110373385,-0.8368542790,0.6338981390,2.1725356579,-1.3096066713,0.5576665401,-0.8026767373,-0.0024915286,0.7201132178,1.1928230524,0.2271231711,-0.0062309671,-2.3676698208,-0.2402453274,-0.5720393062,0.8346250057,0.1938629746,0.6820136905,-0.1138827205,0.0992246643,0.6323102713,-1.8433687687,-0.2471316308,0.8616644144,0.6367298365,0.6539236307,0.5544044971,-0.3177534342,-0.7643328905,0.5900797844,0.7913125157,0.7577337623,0.7785899639,-1.0440396070,-0.6633660197,1.5012997389,-0.0091627706,0.7560212016,0.8975538015,-0.9736179709,-0.2276934981,0.9973939657,-0.3334585130,-0.8088312745,-0.3517300487,2.3488607407,0.6854371428,0.3064183891,-0.4391704798,2.6296787262,0.2384973019,-0.9323630333,-0.1401946098,-0.8720086813,0.7077708244,-0.3136989176,-0.0932615995,0.6460273266,-0.4569582045,1.7222473621,0.2486722022,-0.5947818756,-0.5724456310,-1.2928011417,0.3518567085,-0.2768443227,0.1744876951,0.7715813518,-0.3155222535,-0.6012001038,-0.0778921768,0.6547526121,-1.5268802643,-0.8698882461,-0.2165810913,1.1484020948,0.6043649912,-0.7420395613,-0.0446573719,-1.3837007284,0.5973103046,-1.2017232180,-0.4692558944,1.0996962786,0.4843816459,0.3924929500,0.5177378058,2.1317279339,1.7797539234,1.8023210764,0.5402734876,1.6492481232,-1.1998282671,-1.1383824348,-1.3544220924,1.0511277914,-0.3663887382,-0.0666134506,-1.7126017809,-0.1563147753,-1.6851294041,-1.6602188349,-1.3355588913,-0.2164773196,1.3588712215,-0.7749710679,-0.5971747041,-1.5134319067,0.6159718633,-1.0240488052,0.4399787486,-0.2938184738,1.3824003935,0.9063398838,1.7439777851,-0.8023254871,0.7868092656,0.0004580946,0.4446972311,1.3736970425,-0.3319808543,1.6903849840,-0.8114499450,1.7070233822,0.3595496714,-0.1166067645,0.6463969350,2.5515398979,-0.9076561928,-0.1777824014,-2.1957619190,-1.9981634617,0.4334587753,-0.7519865036,-1.0104613304,-0.7528412342,-0.2588641346,-1.2371978760,-0.0698891729,0.2212488353,0.4128143191,0.5448367596,-0.3559421003,0.1989956051,-0.7013852000,-0.6775157452,-0.4674378335,1.0175814629,-0.9423191547,-1.4644925594,1.0119210482,-0.5379062891,2.3790464401,-0.7766230702,-0.2486738861,-1.7352688313,-0.0490255542,0.3822934031,0.7362163663,0.7763320208,-0.1527945995,1.2485646009,0.2239975333,0.6150395274,0.1295638382,0.0960831046,-0.1421104372,0.6133358479,0.8356537819,0.7742197514,0.1762249768,0.1703704149,-0.3257652521,0.5471190810,-1.5459822416,-1.5104633570,-0.3404698074,1.1019839048,0.8252863288,-0.2277154028,-0.4301207066,-0.0688877627,0.8002319932,1.4292235374,0.4345422983,-1.4420714378,0.1606875360,0.3235762119,0.1411382407,0.1357653439,-0.4222933948,-1.0847389698,1.8023097515,-1.3667072058,1.5778913498],[-0.0131876403,0.3190685511,0.8450273871,1.1641961336,-0.7162372470,0.3938809037,-0.3219540715,-1.7239831686,-1.1850391626,-0.2674075663,1.1804103851,-0.9732410312,0.1530347764,-0.5131613612,0.5818021894,-0.1796519756,-0.3842805624,0.4687068462,-0.3080357015,0.6460461020,-1.9881218672,-1.7341786623,0.0405684933,-1.2557553053,1.1959751844,-0.2807148993,-1.1054583788,-0.9994213581,1.3731906414,0.5248828530,1.1866965294,-0.8613355756,-0.2546068728,-0.4799605608,-1.8969695568,0.2036823183,0.0273817368,-1.9169287682,-0.9908143878,0.1783454567,0.4995328784,-0.3635697961,-0.0450246893,1.6701726913,-0.1013248339,1.0435426235,-0.8696965575,-1.1632086039,0.1831671149,0.6457328796,0.2006639540,1.5764163733,1.4348359108,0.4867582619,0.0998035371,0.2122234851,-0.0686026067,-0.2263608724,0.3516403437,0.7742586732,-0.3687585294,-1.5238984823,-1.5085883141,0.2777234614,-0.3668000698,0.1228307188,-0.8946448565,-0.5748972893,-0.0552437901,0.1306430250,0.4406910539,2.4174592495,-0.9714004993,-0.5191168785,-0.5188544393,1.3178100586,0.1041980833,1.2463740110,-0.0790428743,0.6186703444,1.2864513397,-0.8152043819,0.4917009175,-1.1067799330,0.0927638784,-0.1811284125,-0.4697842598,0.3500574529,0.3294185996,0.1947860867,0.8966708183,1.9381089211,-0.9195051193,0.7260957956,0.8662853837,-0.5145770311,0.1757200509,-0.8238931298,0.1056567132,-0.1374659091,-1.2861822844,0.5849096775,1.8646203279,-1.6820768118,-0.9834036827,1.3807744980,-1.2435959578,-1.6130899191,-0.7388973832,2.2242629528,-0.4154294133,-1.7480731010,-0.2249363214,-0.7993203402,0.6460397840,0.1766046137,-1.6437238455,-0.0110550970,-1.4476402998,-0.2804417014,1.2617326975,-0.2355776876,-1.6873000860,0.8121863008,-1.6272295713,0.7626906633,0.1414132714,-0.9910776615,0.8963002563,1.4895867109,-0.7519250512,-0.1140206978,-0.2397381216,1.8317588568,0.9710619450,-2.1389555931,-1.1381371021,-0.7534425259,-1.2349971533,-0.9838739634,0.7869290113,0.7393561602,-0.9010815024,-0.1183883175,-1.5193796158,1.0234062672,0.4929726720,-0.5983226299,-0.8546621799,-0.9928486943,0.8580879569,0.8566496372,0.3198273480,-0.2430842817,0.2529213130,-0.4792161882,-1.4864950180,1.4555321932,1.3306303024,1.2000240088,-1.2556482553,0.4722168446,0.7385764122,-0.3788935244,-0.4737539589,0.9722071290,-0.9189504981,0.6632404923,-0.1334913969,-1.5663703680,-1.7486513853,-1.1225919724,0.7985507846,-0.0880931318,-0.2178262472,0.3675931692,-2.1636779308,1.5768371820,0.0332911499,-0.6151152849,0.7404027581,0.7260823846,-0.8314467072,0.6558555365,-0.3027761579,-1.3855255842,0.0839681327,-0.8914278150,-1.5680286884,-0.6328111887,-0.1193251312,0.2832503319,0.2091138959,0.0765040293,1.0707458258,1.1313108206,0.6235074401,1.1784594059,-1.4363331795,-0.4905205369,0.1248268560,0.3911684453,0.2099477947,-1.4990891218,0.5412989259,-1.1710778475,1.2187296152,0.6991159916,1.2686791420,0.0984257534,0.1025270298,1.6023763418,0.3297880888,-0.9317253828,-1.2777259350,1.5729357004,0.7261555195,-2.7456653118,2.2690129280,-0.4050147533,1.7808687687,1.9016879797,-0.3218570054,-0.2043680102],[0.1489765644,-1.4593832493,-1.5017405748,-0.8670356870,0.2126237601,1.3351058960,1.6312758923,1.6304445267,-0.4380942583,-1.3650913239,-1.4511771202,1.0316030979,0.8142898679,-1.6095172167,0.7330580950,-0.3812020123,0.5772187114,-1.0423914194,-1.7273218632,-0.0568555333,1.2256610394,1.6008268595,0.4206520319,-0.5724737048,-0.2653441727,0.5704436898,-0.2628526688,-0.7718463540,-0.1277451515,1.1820430756,0.4724800885,1.4606692791,0.0491502918,0.4095941186,-0.5304942131,1.3007168770,-1.6952812672,-0.3756690323,0.3484024704,-1.6185001135,-0.1921009719,0.0708475932,2.4357285500,0.9716811776,-0.9329621792,2.8652036190,-1.7920479774,0.5449343920,2.4494490623,0.0170834791,-1.3960881233,0.5912993550,0.7908234000,-0.2101195306,1.0908360481,0.9109295607,0.1053518280,0.9273505211,0.8930205107,0.5378038287,-0.6903609633,-1.1878529787,-0.4941872358,1.1770657301,1.0582617521,0.5110420585,0.5188018084,1.3615533113,-0.2138238251,0.4488541186,1.4647057056,-0.3835515380,0.7276459932,1.1939560175,0.2122646570,-1.0129065514,-0.5653158426,0.1305658966,0.2739067674,0.7412998080,0.0615668334,-0.4668753147,2.5491123199,1.0991567373,0.6903136969,0.6747993231,-0.4939589202,-1.8001532555,-0.2043026686,0.0035368698,-0.0057333200,1.0612637997,-1.4626442194,0.4315978885,-0.6228010058,1.2404770851,0.1704462916,1.9868983030,0.1113011688,1.2096550465,-0.4573297799,0.9231835008,1.3255188465,-0.9531663656,-0.6781297922,0.3340995908,-0.7549282312,-0.3883825243,-0.6265756488,-1.3064211607,-0.0384958498,0.1435673833,-0.7890736461,-0.0985037014,-1.4528664351,-0.9478659630,-0.8695787191,-0.3055172861,-0.6051473618,0.2248205394,-1.0384724140,0.7718191743,-0.4800092876,1.8347450495,-0.1087654233,0.4544749856,0.0987093523,-0.4583054483,-0.8570440412,-0.7744029760,0.7087873220,0.0184736028,-0.1040805802,-0.2348577380,-0.2509063482,-0.9627502561,0.4934051633,2.1330499649,0.9838882685,-1.8092817068,0.0423588157,0.5168721676,-0.0329206921,1.2981114388,-0.2021170259,-0.8332309723,1.7336002588,0.1906490028,-0.1778104007,-1.0850250721,0.9722623229,1.4314717054,-0.2450730503,-0.6232538223,-0.3959988654,-1.3585224152,1.3226025105,0.0034439384,-0.0001967701,1.5337039232,-0.7730822563,-0.4012549520,-0.1153567657,1.0115554333,1.5132355690,0.9170686007,0.7288931012,0.5520321727,-2.8032341003,-0.3960531652,-0.7131436467,1.3008337021,-0.5528658032,2.0708973408,-0.3659083247,-0.8976300359,1.4449365139,0.1729418486,0.2826884985,1.0908695459,-0.2013481557,-0.6806857586,0.8032750487,-0.3421638310,-0.4724888504,-0.6922569275,-0.0466712639,-0.6195800900,0.8695260286,-0.1468828470,0.3280917704,-1.3630435467,0.8078343272,0.6606640220,0.7788500190,1.5073202848,0.3795762062,0.6519295573,0.3520001471,-0.4837388992,-0.1357428730,-0.1349714249,1.0053361654,-0.8543254733,-1.7947654724,0.9829106331,0.8027749658,-0.4673963487,-1.4142329693,-1.1199934483,0.8871966600,0.0762820542,1.2231287956,-0.6656363010,-0.3412737548,-0.5046559572,0.3275733590,1.1423867941,0.7465314269,1.7910561562,-0.0725371093,1.2476819754,0.0711429790,-1.8477734327],[1.9659148455,-0.2410738766,-0.1578042954,0.5254576206,-1.9116709232,0.9348053932,0.8459461927,0.1184646636,0.5029144883,-0.1982770860,0.7655358315,0.9073389769,0.1973911673,1.3738481998,-0.9893300533,-0.9891023636,-2.4769854546,0.7702542543,-0.0861265808,-0.8576679826,0.6391736269,-0.2472003400,0.2333795726,0.8708141446,0.6738551855,1.6272670031,0.7516965270,-0.7883411646,-0.8082208037,-1.3736037016,-0.4045992196,1.0280630589,-1.5187823772,-0.4838001728,0.6150732040,0.8216204643,0.1266588718,0.5752500296,2.1062376499,0.8944895864,-0.7871508598,-1.3972195387,0.3273196220,0.7946544290,-0.4160199165,0.7112743855,-0.0405380018,-1.7044556141,0.5837534666,-1.0075877905,0.7251850963,0.9467189312,0.5067328811,1.6449491978,0.5131101012,0.3260095119,0.5165187716,0.1619718522,1.9142762423,0.6852089763,2.6714634895,-1.2615832090,0.0633755252,-0.6122508645,0.7064356208,-0.1490675062,0.3214509189,-0.3426703811,0.3457258642,-0.4682802856,0.2918916047,0.4650507569,0.1249749437,0.5599335432,0.6786236167,-0.4045546353,0.6692442298,0.4938224852,1.2209396362,0.1036879346,0.6975646615,-0.6695911288,-1.2909108400,0.1948612928,0.1942252070,-0.6748269200,-1.9970334768,2.4057705402,0.6200784445,0.3772981763,1.5525032282,-1.4655271769,0.3644940555,-0.0155226057,1.1985623837,-0.4178135395,-1.6951625347,0.8210743666,-1.7736728191,-1.0594503880,0.7284531593,1.1591846943,0.2815538645,-0.3945144117,-1.9853501320,-0.4423422217,-0.9395880699,-0.3203176856,-0.9223166108,-0.5297619700,1.6634757519,-2.5264670849,-2.2690434456,0.5216883421,-0.3797107339,-1.0727113485,-1.2990864515,0.1797483265,0.2679230273,0.7701867223,1.3195123672,-0.4258531332,0.0932302922,-0.1008663177,-0.0527959503,-0.2420887053,-1.2358770370,0.2188131213,0.2474349886,0.0911240950,0.9356385469,0.6642363667,-1.7988439798,-0.9775348902,1.0068738461,-0.4412638247,0.2630724311,-0.2938269973,1.2398545742,0.1530414075,0.0919402540,-1.4946030378,-1.6424170732,-0.3924821019,0.9380593896,-0.7755195498,1.4306209087,-0.5973093510,0.4153765142,-0.4966987967,0.4551673830,1.3969449997,0.9603327513,1.6694878340,-0.7250531912,-1.1356847286,-1.2331088781,-0.2271185815,-1.2854925394,-0.9318059087,-0.7606769800,3.2205018997,0.2209428847,-0.0629122630,-0.6298869848,-1.0732094049,0.0010558150,-1.0177214146,-1.5743277073,-1.6354320049,1.7043139935,-0.6617698073,-1.6599169970,-0.3333908916,0.6777613759,-0.0104675246,-1.1323215961,-0.0904404595,-0.5879105926,1.0276099443,1.4621688128,0.2452373505,-1.8568937778,-0.2075275332,0.0924542099,-0.5434443355,-0.5015999079,0.3858227134,0.5888552666,1.1586396694,-0.7475170493,-0.2434626371,-0.1310633272,0.2667567730,0.5259755254,0.1121224090,-0.5332795978,0.8407949805,1.0488711596,1.6149709225,0.9318397045,-2.2712433338,-0.7382249832,0.0473290794,0.1856919378,-1.3058172464,-0.7064233422,-0.1791879386,0.5038546920,-0.6263046265,-0.0282821339,-0.7154991627,0.4878412187,-0.0843407884,1.7737895250,-0.4433949292,-1.0820724964,-0.5127086639,-0.4460237324,1.8900141716,1.3373774290,0.3039647639,-0.1128789261,-2.2114961147],[-0.7659804821,0.0121604325,-0.5553606749,1.8555424213,0.9179546237,0.2352409810,0.8584582210,-1.1751790047,1.6242084503,2.2770774364,-0.1804874986,0.5590753555,-0.1992742270,-0.5453976393,0.5759513974,1.1652381420,1.6872155666,1.8794616461,-1.1923128366,-0.3455305696,-0.0926098600,0.0298121963,-2.2340764999,-0.0128461355,1.7809996605,-1.7698057890,-0.1657604724,-0.2962538302,0.3984484971,-0.2784538865,-0.7657407522,0.5881029963,-0.0268511921,1.2491877079,0.5969085693,0.8053591251,1.2250876427,-1.6654818058,0.7458550334,0.1767183840,-1.0294940472,-0.3047012985,-0.8348066211,-0.5481224656,-0.2275418341,-0.3919983208,1.7537639141,-0.4548853040,-0.2024148703,-0.7957806587,0.7550222874,0.5483199358,-0.6040002108,-1.0742310286,-0.2339738905,-0.7312679887,0.6031473279,-1.9090847969,-0.4644692838,1.8127008677,0.5727068782,-1.1307305098,-0.2126340717,0.9141866565,1.0289263725,-0.9336540699,0.9685837030,-0.4468255639,-1.3155292273,0.9664657712,-0.1305926740,-0.1409974098,-0.2777955234,0.1934855133,0.2790493965,-1.0827175379,-0.6172372103,-0.9062939882,1.6249476671,-0.8238903284,0.6574448943,1.3339018822,-0.7396215200,0.0353739858,-0.9580649734,-0.1081325933,-0.5253806114,0.8399999738,-0.6203387976,-0.1352863312,1.8578811884,-0.9981376529,1.3206781149,-0.0126401605,0.5569839478,-0.1270247847,0.4880434573,0.5574302673,-0.4547669888,-0.6072672606,-1.0674308538,-0.5023055673,-0.4857284427,-0.0919217393,-0.9623599648,-0.0035295901,-0.0758933797,0.0933261737,1.2281299829,-0.0178553499,-0.4720449448,0.2214139700,1.4468604326,0.4306001961,1.0677331686,-1.6240931749,-1.1872859001,-1.1063474417,1.1893283129,-1.3409972191,0.7507525086,0.2144597620,0.3168707788,1.2522690296,0.9644386172,2.1303524971,-0.7632300258,1.7955607176,-0.3486187458,-0.3859466910,0.2825184762,-1.0904860497,0.2479778975,-0.3009818196,0.4486035109,0.9551592469,1.4568217993,-0.6479675770,-0.3713506460,1.0168209076,1.2009193897,1.0800970793,-0.3941484392,-0.1019895002,-2.3731226921,-0.3524095416,-1.0711951256,0.5553830862,-1.7090671062,-0.6871978641,0.0623640977,0.0408379324,0.4368767738,-0.9474046826,1.3001302481,1.1587718725,0.4988572299,0.2281088084,-0.6101936698,-1.8242050409,-0.0441021472,2.4481766224,0.9330115318,-0.1531505287,0.0430765450,1.4130680561,1.1317970753,-0.8838879466,-0.4641161561,0.8401603103,-0.1360720992,0.7551409602,0.9949775338,-1.5624421835,-0.3360073268,0.2582748532,0.6022772789,0.1784233600,0.7269070745,1.0488575697,-0.3621068895,0.6744479537,-0.4786558449,-0.2334593087,0.4071067572,0.2839189470,0.7738007903,0.4134279788,-0.1320126951,1.0825728178,-0.8175003529,1.4486014843,1.0853774548,-0.2358137816,-1.6589474678,1.1178635359,1.0709910393,-0.3829950988,-0.1981175393,-1.3988100290,-0.0638643652,-0.1147952601,-0.3831200600,-0.8857003450,-0.8296794295,1.4530539513,-1.6511634588,-2.5654265881,-1.1252266169,0.3472560346,0.9780961275,-0.7914997339,1.9477938414,0.3580872715,0.1118140593,-0.2804694772,0.4610228837,0.6023619175,-0.5335633159,0.0542477593,-1.7664861679,-0.0508076660,1.9950895309,1.5672857761],[1.4353256226,-0.5908765197,-1.2080715895,-0.6752406359,-0.7683525085,0.2406216562,-0.8646903634,-1.6352556944,1.7879844904,0.3776165843,1.5704457760,-1.4466234446,0.8425059319,-1.2013957500,-0.3469410837,-0.7660391331,-0.6346268058,1.6559984684,-0.6233503819,0.4907566309,0.8447885513,-0.9813486338,-1.4458115101,0.6337310076,-0.5561428070,-0.5065686107,0.1102068424,-0.5101391673,-1.1948239803,0.7441099882,0.0857115984,0.0515982620,1.1483319998,0.7804873586,0.5419780016,1.2188420296,2.0769410133,-2.4862742424,-0.4223719239,1.0099729300,-0.7526311874,0.0810160041,0.5616210103,-0.4127331972,0.1325272769,-0.5238465071,-0.3826581836,0.0631827638,0.2585388422,-1.0420076847,-0.4863687158,0.0387250669,-0.4188231528,0.0140505927,-0.9987033606,-0.8189831972,0.2218448222,-0.5424128175,-1.5457696915,-0.4346087873,0.4255478680,-0.2396979183,-1.6252154112,0.3310910463,-2.3621444702,0.2520245612,-0.2275166214,0.3297084570,0.9491755962,-0.0028857999,0.3958771825,-0.3808728158,-0.7215411067,0.9620351195,-0.9630186558,-0.4433543384,1.2981798649,0.0506372303,1.3202215433,-0.0137817934,-2.1586008072,-0.2851150930,1.0693598986,-0.4877946377,1.0574339628,-0.5105757713,-0.5830003619,1.8261824846,-1.2577497959,2.4384016991,-1.2908158302,-1.2552080154,-1.0708709955,0.3558755517,2.2702765465,1.6279611588,0.9271426201,0.4194103479,0.6996855140,0.2764927447,0.0600915737,-0.2719833255,-0.0216664709,-1.0817744732,0.2898022234,-0.7958546281,-0.5543126464,0.0587645993,0.2073607743,1.6387189627,0.9563472271,-0.1144778430,0.4083743691,0.3125553429,-0.0443716496,-1.4478203058,-1.4528867006,-1.0498052835,-1.0768823624,-0.9817907810,1.0057474375,2.1739764214,-0.1706229895,-0.3973425627,-0.8106067777,-0.3117626309,1.7385970354,0.4636293352,-0.3924411833,-0.5573529005,-1.3127901554,-0.0500530154,-0.3318820298,0.6395338178,0.1124192551,1.6188535690,1.5975137949,0.7731382847,0.1306872517,-1.3235602379,-0.4722246826,-0.2985983193,-0.4397483170,0.9028589129,0.7685422301,0.9230428934,1.6842801571,1.4418077469,-2.5131144524,0.4086238444,0.0655701682,-0.3320867121,1.7466781139,0.3734170496,-0.2728291750,-1.6017853022,0.7018960714,0.4604117274,2.3683567047,0.4982431233,-0.3625016212,-1.7402305603,-1.1604846716,-0.0124342721,-0.0771125332,1.2629164457,0.2355280519,-1.0599642992,-1.4503680468,0.1968804300,-0.3745819032,-0.6852248907,1.8013814688,0.4930929840,0.7483561039,2.1840589046,0.0830427632,-0.6732596159,1.0215715170,-0.5686770678,0.8424426913,-1.9740310907,0.6989012957,0.0312862769,1.7320979834,1.0707975626,-1.0670249462,-0.2167563587,1.7714269161,-0.2345068604,2.4865136147,0.2414270341,1.7394777536,0.4267886877,-0.6875585318,1.1192632914,2.2618951797,-0.1167902946,0.4676852822,1.3006483316,2.9003543854,1.5146406889,-1.1470533609,-1.2451536655,-0.5879987478,0.5812782645,2.1253869534,-0.0506866947,0.4011137187,0.7882096171,-0.6160338521,-1.2647396326,-0.2363180667,-1.5727478266,-0.0531756170,-0.5144679546,0.2395006865,-0.8833888769,0.2919378281,0.2528843284,1.3525061607,0.1802900583,-0.9587054849,-0.1820952743],[3.0571014881,1.5167911053,-1.0126098394,-0.1719368100,0.4079130590,0.1754129082,0.3528579772,-0.2692727149,-0.8049977422,0.0460437089,-0.0798596963,1.0721807480,-1.1028991938,-0.6041131616,3.4571788311,-0.4846538603,-0.6901526451,1.0774567127,0.1411880702,-0.4632414579,0.1769790202,0.8287273049,-0.7865691185,-0.8834164739,0.8981291056,-0.3418011665,0.4376186132,-0.8193335533,0.7731699347,0.3516902030,2.3642060757,-1.3887287378,-0.3825967014,0.6642029285,-0.8499180079,0.8090372682,-0.0887294859,-0.7657515407,0.4094007313,-0.2589460313,-0.0710417926,-0.2994653583,0.1474562585,-0.0265487880,1.6107178926,-0.8812941909,0.0249902233,-0.3142261505,-1.5077643394,-0.5018021464,1.0165740252,-0.0183646958,0.3615891039,0.7975444198,1.2121458054,-0.6506839395,0.3548581898,0.2484272867,-0.2789663076,0.5858780742,2.3015351295,1.2746146917,1.0363812447,0.7774479985,1.1028864384,-0.1068114564,0.1970036775,0.1932538748,-0.3952465057,1.1583861113,1.4059976339,2.1219327450,-1.8358495235,-1.2085362673,-1.3249143362,0.4205504954,1.1037262678,0.9093187451,1.4752906561,-0.8287338018,1.0169249773,-0.6972437501,-1.4810217619,-0.7408619523,-0.6186295152,0.5369817615,-0.2822094262,-0.1443003863,-0.7923364043,-0.6148132086,-0.0632172227,0.1186423898,0.3373407423,-1.3994134665,0.5843821168,0.1835361719,-1.3693171740,0.6581259370,-2.2536697388,-0.7097249627,-0.2993446589,-0.7744182944,-0.7150697708,2.7478237152,-0.4464194179,-0.5393986702,0.5048452020,0.0309016295,0.7893177271,0.8253213167,-0.1216339916,0.8358047605,-0.5530250072,-0.6232783794,0.1180848405,-0.5877562761,0.3572616875,0.6083747149,-1.6882966757,0.3013758063,-0.9361636639,0.6878786087,-0.5113450289,1.3853739500,0.2846948504,0.4329951406,0.1250179708,-0.8994542956,0.2352239043,-1.2032506466,-1.7193822861,0.5931987762,-0.9778930545,0.2653068602,0.5067004561,0.5289206505,-0.8424483538,0.3913740814,-1.1664303541,1.4980700016,-2.3028693199,1.2116063833,-0.2439048588,-1.2052580118,-0.2917331755,0.0616047978,-0.6883229017,0.6314316392,0.2147449553,-0.4651604891,0.0913353115,-0.6877609491,2.5887622833,2.2688219547,1.6409702301,0.7080410123,-1.6533527374,-1.6032719612,1.0296443701,-0.1468841136,0.7079297900,0.7750738263,-0.5503200889,-0.1952609122,2.2069025040,-0.3737757802,1.3544288874,-0.1811147928,0.1393500119,-0.1114728898,1.0114481449,-0.6935762763,0.2787633538,-2.0546996593,-0.5709084868,-0.3062367737,-0.3156863749,-0.8897069693,-2.3231439590,0.4797623754,0.0417126454,0.0786912814,0.8310974240,-1.6643717289,-2.1256644726,1.2952638865,-0.9766326547,0.1436393559,-0.7110436559,-1.1480141878,-1.1452629566,-0.8019243479,1.5351486206,2.1707282066,0.7657800317,1.5620862246,0.1756275594,-1.3503173590,1.7917487621,0.7160843611,-0.1938566267,-0.4526286423,0.2835141718,-1.2592515945,0.3630310297,0.3490190506,0.4931244850,-0.8836598992,-1.2861914635,0.6278840899,0.5298740864,-1.4413779974,-0.2835259736,-0.5337915421,-0.5376721621,0.2081073076,0.9799979329,2.2777760029,1.2127616405,0.5642753243,0.0354485251,-1.0789268017,0.1125210300,-0.2152985483],[1.7759253979,0.9840850234,-0.0767186433,0.6679525375,-1.0115307570,-1.5240343809,0.1609255522,0.8578897119,-1.6950411797,1.2214843035,-0.1197355837,0.9035636783,-0.2301980704,1.8057299852,0.0134482170,-0.8037886024,0.4452606440,0.1248501688,-0.5301916599,-0.8451347947,1.1502391100,-0.1114130989,1.0182909966,0.1798265725,0.0725538880,-0.0725682154,0.6086733341,0.6504692435,-0.0388825051,0.6453584433,1.5192260742,-1.3471193314,0.3118595779,-0.6958822012,-0.8889675736,-1.8208668232,0.3896815777,-0.2915764749,-0.0712015107,0.4959938824,0.5245856643,0.1248565987,2.2854576111,-0.3773034513,-0.4099958241,0.2611141205,-1.1319282055,-0.8118616939,-0.0755043477,0.8247570395,-0.2658351064,-0.5308529735,-0.5981125236,0.5713994503,0.1442613006,-0.0215171184,-1.5180338621,-1.9154884815,0.9877078533,-0.2374628782,1.3302844763,0.1467256844,-0.2150432318,0.6785774827,-0.7292082906,-0.2542716265,0.4539424479,-0.0867115185,-1.2298870087,-1.1490353346,0.9752449989,-0.1623965055,-2.7027933598,1.0214067698,-1.6213696003,0.4340440631,1.3481116295,-1.2926946878,0.3220206499,-0.6569662094,1.6419132948,0.1643116474,-0.0132283932,-1.8333109617,-0.9346026182,-0.9406163096,-0.9235872626,0.6504324079,0.1044687778,1.2268142700,-0.2209423184,-0.2384083420,-1.2015172243,-0.5316178203,0.1752220094,0.6893568635,0.2345945686,0.4606382847,-0.6025770903,-0.8063659668,-0.5693377852,-1.8322895765,-0.4588719010,-0.4438908696,1.3813488483,-0.7166424394,-1.3205441236,1.6986227036,-0.9610489607,0.7402990460,0.5968779325,-2.0013959408,1.5075762272,-0.0389403030,0.4936818480,-0.3687821925,-0.3875725567,1.2237678766,0.3262037039,0.8998846412,2.1900782585,1.2703543901,2.1951560974,0.2156772017,-1.5663990974,-2.2078025341,0.0685291588,-1.5581027269,-1.0036827326,-1.3702378273,0.1694180816,-0.9137530327,1.0961093903,-0.1972991228,-0.8759288788,-0.5241637230,0.4103074372,0.1449525803,-0.7451390624,0.2333553135,1.1782257557,0.1412797421,-1.4361449480,-0.3597810268,-0.8911457658,-2.2995569706,0.0269727893,-0.5035428405,-1.4560674429,1.7810405493,-1.3310171366,-1.8869807720,-0.2242967784,0.7271280885,0.7559499741,1.4708317518,-0.0475333333,-0.5505061150,-1.2091144323,1.0972720385,0.1798228621,0.0975974947,0.6015082598,-1.5747113228,-2.3804230690,0.2242878824,0.4144230783,-2.0694725513,0.0689833388,-0.8785431981,-0.2263758332,0.2870897055,-0.1257645935,-0.5280503631,-0.6401610374,-0.9018902183,0.6883198023,-0.2870998979,2.0648100376,2.5321288109,-0.1234981641,0.3702547550,1.3969179392,-0.2872636020,-0.5175419450,0.1475308388,0.0770351812,1.0055582523,1.4378954172,-0.5518070459,-0.0454276241,-0.9547314644,-1.7417740822,0.5516366959,1.4494178295,1.2299993038,0.0344757847,0.8050377369,0.1586563140,-1.4417908192,0.8038492203,-0.4616938531,-1.5563237667,-0.2434473038,0.8458123803,-0.3156723976,-0.9523938298,-0.3382590115,0.1520356685,0.3631040454,1.6726390123,-0.6849400997,1.6337143183,0.3757461905,-1.1149408817,-0.1292464733,0.5845023394,2.7107896805,0.1954807341,2.0079009533,-0.3136621714,-1.4966130257,-0.4827790260,-0.5490916967],[0.8668015599,-0.4425382912,-0.3470218480,-0.0106614679,0.1458479911,0.9882314205,1.5065393448,1.0266677141,-1.6985048056,-1.2251684666,-0.0537470914,0.6280582547,-2.1100039482,0.0921582580,2.0108947754,0.1368566453,0.2666280866,0.5259639025,-0.5141676664,-0.1254386157,-1.9846127033,-0.6376038194,0.1830519587,-1.3448629379,1.7952604294,-0.6953235269,-0.4445840120,-0.2850975990,-0.6821460128,-0.6116260290,-0.0420884788,-1.1017171144,0.6697148681,-0.3835142255,-0.1298492849,-0.4696798623,1.4395703077,0.0283602029,0.4346496761,0.9028067589,-1.2745651007,-0.8085855246,0.2151665688,1.1069250107,1.0683196783,-1.1012074947,-0.2108737677,1.0316549540,0.0220382344,0.1956191957,-0.0308300909,0.9818850160,-0.9788653851,-0.4604631364,0.7173712850,-2.1900360584,-0.0521114245,0.0403597429,1.2453624010,0.5835828185,1.4235873222,0.9640865326,0.2892068624,0.2281533778,0.1888966411,-0.6644412875,-0.3662932515,-0.4654232264,-0.1265579760,-0.5279335380,-0.2821264863,0.5741770267,-0.9894772768,-1.6030364037,0.5362505317,-0.0642012283,-0.5990480185,-0.3981341720,-0.4884864092,0.8829444051,-0.9191870689,-3.5971634388,-1.1088981628,-0.2825691402,0.0801857784,2.0061070919,0.5381712317,-0.6989403963,-0.6342703104,0.6783562303,-1.7275750637,0.1030801684,-1.1218008995,-0.8425818682,-0.2069326192,-1.2354469299,-0.6567828059,0.1083241701,0.8044376969,0.2656084299,-1.1283918619,-0.7726039886,-0.2678033710,-0.8238725662,-0.7397587299,0.3420245945,0.6743623018,1.0771563053,-1.1560052633,-0.3988094032,-0.2347034961,-2.5536952019,-0.4352763295,-0.7365463376,0.6908099055,1.7416143417,-0.8961228728,1.1003487110,1.5549330711,0.7204515338,0.2944574058,-0.5237665772,-1.2406834364,0.1910576522,-0.5132317543,-0.1078897566,0.2912856638,-0.1775024086,-0.4434978366,-1.6129906178,-0.8831277490,2.1223089695,0.5924318433,-0.1580628008,-1.5093832016,-0.1052221730,0.4946533144,1.3044494390,-0.5286353827,0.5293801427,-0.0658370033,-2.1142232418,-1.6638370752,0.1901883334,0.0722268820,1.6248866320,-0.1596909165,0.9091033936,0.0457334258,-0.1473118365,-1.2650078535,1.2293517590,-2.2262835503,0.8036817908,-1.5079095364,-0.3022215068,-0.5339203477,-0.0028147821,0.9604151249,1.0207607746,-1.7765924931,0.3125726283,-0.5199762583,-0.7035210133,-1.5594059229,-0.9711892009,-0.8094975352,-0.2992601693,0.1038063467,0.4899407625,-0.4866145849,-0.4270956218,-0.1204021871,0.0000380513,-0.7106295824,-1.1571166515,1.2584540844,-0.9098292589,0.4606101811,-0.8967497349,-0.7487707734,1.1330285072,-0.2487686425,0.0485689640,0.9908419251,1.7194275856,1.1388652325,0.4050374627,0.7197656035,-0.0095453719,-1.7850903273,0.2807362080,0.3113456666,-0.4148780704,-0.5144474506,2.3370497227,-0.9018598795,-0.0432650372,0.6294213533,-1.2214990854,-0.3480480611,0.4941302836,-0.8427708745,-1.0920269489,1.1331650019,0.5445849895,-1.9988650084,-0.5136639476,1.5669808388,-1.4852890968,1.1697503328,0.1714032292,-0.1498004496,-1.7167567015,0.2077475339,-0.2835475206,0.6810171008,0.1799054295,-0.3234174848,1.2285729647,-0.2494829297,0.4951613247,1.2835344076,-0.2153073847],[-1.5655417442,1.0642517805,-0.4465774298,0.6258028150,-0.4518454075,-0.6958141923,-0.4009466767,-0.7438731194,-1.0151065588,0.2284471691,-0.5200065970,1.7738699913,-0.9808174372,1.6659055948,-1.2655656338,0.1619434059,0.5714436173,0.8214023709,-0.7151944041,-0.7368196845,-0.5527368784,-0.4281289577,-1.3470569849,0.2309796363,-1.1804646254,-0.1970812082,-0.6844750047,0.1222466752,-0.5097717047,1.0388318300,-0.0032982610,-0.9455435276,0.0521331877,0.9406508803,1.4224059582,0.7051896453,-0.0485493839,0.0030716613,-0.0828417838,0.3621097505,1.1506960392,-1.2411286831,0.6016132236,-1.8681024313,-0.2948068678,-0.5459722877,-1.3271028996,0.0146063436,0.0820118934,1.1872184277,0.0226242170,0.5577337146,-0.6835659146,-0.3828615844,-0.0172030423,-0.3059270680,-1.1533372402,0.2937498689,-1.1304335594,-1.5362212658,1.1067888737,-1.7159109116,0.1627099216,3.4915504456,0.8252794743,-0.6337646246,-0.6024153829,0.2472287565,0.0888550505,0.3549385667,0.6885216832,0.0299285725,-0.0664129481,-0.6327346563,-0.1739193201,-0.8258158565,0.1949875653,-0.2163696438,1.0570889711,-0.7573488951,-1.5297939777,0.3537744582,0.3407221437,1.3309781551,0.7501005530,-1.1971751451,0.3842400312,0.1106487289,0.9332572222,1.2433928251,1.0489282608,-0.7144296765,0.3993691802,-1.1924867630,-1.6388714314,0.6282942295,-0.3221462667,2.2317204475,-1.0032519102,2.5989425182,0.7262270451,1.8382234573,1.0665451288,0.6297884583,-1.7966475487,0.2026394159,-0.6314821839,-2.0691819191,-0.3227912486,-0.6972043514,-0.0853688121,-0.7535147667,0.3925627172,-1.5193345547,-0.2938063443,0.3562227488,-0.9631296992,-0.3119919896,-3.0219933987,-1.2542766333,0.3058491051,-0.0131095555,-0.0122315139,-0.0387849845,-1.0967736244,0.5564718246,-0.6666582227,0.3884274662,1.9603561163,-0.5804669857,0.8530720472,-0.3293070197,-0.7811429501,0.5486994982,-0.2156431228,0.4974171817,1.8149178028,-0.0831253380,-0.2076419592,0.4824213386,0.2035827041,0.2575459778,-1.2080625296,-0.9822015166,-0.8012655377,1.1425778866,0.7922763228,1.2311615944,-1.5688765049,0.7047674060,0.2851555347,2.2098674774,1.0570685863,-0.0321040228,-0.1283721030,0.3590631783,-1.3702954054,-1.4281027317,-0.2610388696,0.6530545950,0.4184088409,0.2936683297,-0.1556836814,0.9377830029,-0.1359964013,-0.8170792460,-1.2400767803,-0.4732416570,-1.1670972109,-0.8413016200,-1.4027423859,1.0371077061,-0.8688614964,-0.5326256752,0.5555218458,-1.4648450613,-1.0836731195,1.8800867796,-0.8621342778,0.5107352138,1.9502209425,0.6216194034,0.4190502167,2.2297959328,-0.8208199739,-1.7991086245,-0.6422991753,-0.7526358366,-0.0102676786,-0.6219030619,0.9146863818,0.6332815886,-1.3110525608,0.8213313818,-0.1096853018,-2.4599967003,0.6964056492,-1.6533551216,1.4545164108,-0.0317682959,-0.2366403639,0.8078718185,0.1043894738,1.0441939831,0.5735760927,-0.3592137694,1.6438568830,-2.3127036095,-0.6342550516,-1.0091156960,-0.3238717020,0.1599443555,0.0075996816,1.2957876921,0.1138323396,0.8152285218,-0.6953651309,-0.7397205830,0.2354884744,-0.0152770123,1.1379036903,-0.4955999851,0.6735898852,0.1157289371],[1.3253082037,-0.0473974869,0.0749195814,1.1126670837,-1.3454822302,0.5657912493,-0.3173732460,-0.6623814106,0.9390728474,-0.3503127098,0.5224634409,1.7018532753,1.1917303801,2.5286018848,-0.9827714562,-1.7515311241,0.3258644044,1.1894116402,0.2793818414,-2.5151178837,0.2461755723,-0.0248270202,0.8610759377,-1.4125248194,0.9993839264,-1.4585593939,-1.2222012281,-2.5958955288,1.7636417150,-2.3849341869,0.8637584448,-0.1760935038,-0.3307088912,1.3145556450,0.5149357915,0.5830872655,2.3947978020,1.3304606676,2.7539777756,-1.1339077950,-1.7897496223,0.0805313960,0.2522512376,0.2491337359,0.6630885005,0.2543898225,-0.1769705266,0.5460646749,-0.1675920635,0.6434882879,1.2286797762,-2.2995398045,-0.2927011251,0.6443910599,-1.0067539215,1.6835182905,0.9931317568,0.4729190767,-1.1432638168,-0.7436600327,-0.2560135126,0.9444711208,0.7911326289,1.6346222162,-0.1302263737,-1.0289372206,0.0085418718,-1.3331294060,1.1780107021,-0.9500072598,0.6754372716,1.0067199469,-0.8243820071,-1.0483652353,-1.9091670513,0.0497869141,0.2428957224,0.0930123553,0.1747187376,-0.2940579355,-1.8441371918,-0.6564555764,-1.3747949600,-0.2950574458,-0.5911638141,-0.9996911287,-1.0452963114,-0.8234270215,-1.3071545362,0.4519526660,1.1589044333,-0.5272555947,0.4144105017,0.9806813598,1.7847551107,-1.0145297050,0.5712207556,-0.5980527401,0.5029620528,-0.0612854436,0.7624505162,-1.3226306438,-0.4346112013,0.6056466103,-0.0498499461,-0.3783754706,-1.5018903017,0.0628009811,0.6265262365,0.8333978653,0.3140480220,0.0424722657,-1.9600057602,1.5053578615,-1.9780138731,-0.5857676268,0.6841420531,0.6114683747,-1.3019052744,0.2540067136,0.9127832651,0.4417865276,1.2836414576,0.1039965823,1.2140318155,-0.5549297333,-0.4985508621,0.6292501092,0.3685869873,1.1157144308,1.5523375273,-2.3205018044,0.5883346796,-1.6743589640,-1.3808029890,-1.1486417055,-0.4540960789,-0.2932817936,-1.0938034058,1.2470657825,0.1483083218,-1.3925426006,-0.8331165314,-0.2666631341,1.5057089329,-0.8068805933,-1.0885897875,2.3916010857,-0.4042384028,-0.4770764112,-0.1564542800,-0.6575467587,0.2871340513,-0.0035539989,-0.1058869883,0.1488221437,0.3605379760,-2.4916563034,-1.5001876354,0.4150368869,0.8202406168,-1.4984824657,1.2506990433,-0.9239646792,-0.9689198732,-0.6747028232,-0.9625166059,0.0919773877,1.7888215780,-0.3183748126,-1.5616273880,0.7808475494,-0.0725234970,0.7236689329,-0.8480882049,0.4332070053,-0.7184300423,-0.5073258877,-0.7155088782,0.1922626346,-1.8225835562,-1.5416775942,-0.4609018862,-0.4596909881,0.2107827514,-2.0516586304,1.5195232630,0.8270009756,-0.3949391544,1.2474613190,1.3624265194,-2.0977537632,0.3367308080,1.3497856855,-0.1149254441,-0.1106928810,-0.6452971697,0.4695451558,0.0963989571,-0.2132138759,1.3956539631,-0.4724814594,-0.8594051600,1.8760187626,-0.1349168271,-0.3604379296,0.4938758314,0.2165178359,-1.3356113434,1.1783431768,0.1670271158,-0.0458714627,0.2959707677,-0.8548012972,-0.2733450532,0.4309072495,0.5981340408,-0.6337695718,0.1154701039,-0.1036763787,0.8955789804,0.3470619619,0.3223627806,0.6703640819],[0.3695249259,-1.2931357622,2.0881593227,-0.3163700700,-0.0996335223,-0.0413534530,-1.4175074100,-0.5930390954,0.4457131326,-0.1496128440,-0.1687347889,0.3874080181,-0.8589089513,-0.5768976808,1.7151972055,-0.6996167898,-0.2785257995,-1.0260729790,1.3125190735,0.2768625617,0.7143705487,-0.2726789117,-0.0571496300,-0.0628784820,-0.7734673023,-1.7554893494,1.3222324848,0.8918345571,1.3041300774,-1.3633157015,0.0353974849,0.3286035955,0.6159794331,1.3449649811,-0.0227691717,1.6639068127,-1.9228399992,0.5120229125,-1.0251792669,0.6372367144,1.5930398703,-0.6409438252,1.4217171669,0.0658135042,1.8625446558,0.6787316203,-0.4409461915,0.1394979209,-0.7030583620,0.7338191271,-0.7291454673,-0.3782072663,-2.8154554367,-0.8627194762,-0.8333143592,1.2222462893,0.8411992788,0.3509971201,0.2359188348,0.9166232944,0.0066510001,1.5729032755,0.4003543556,-0.4913519323,0.8445338607,0.0200802702,1.0654255152,1.3981710672,-1.6055946350,0.0228539519,0.8410942554,-1.5818178654,-0.3087835014,-0.7839787602,0.8243343234,-0.9782735705,-0.1291076690,-1.1661448479,1.2579156160,0.3912162185,-0.1826521456,-1.5388045311,0.0681304187,0.7006773353,0.0295840409,0.1992514282,0.2526450753,-1.8144961596,-1.1816760302,-0.9918892384,0.6096387506,0.2016691118,-0.6762703657,-1.3589177132,0.7350046039,-0.1993409395,-2.6887750626,-1.2836247683,1.9510684013,0.1622150093,-1.4023469687,1.4098094702,0.0630933121,1.0961402655,-0.8833422065,1.8702621460,-0.1089891195,-0.3058429062,-0.7081920505,-0.3911435902,-0.7089096308,1.9077292681,-0.5598803759,0.9991017580,0.7150085568,-0.3869202733,-0.9710565805,-0.0675847158,1.1329613924,0.6529337764,1.2304496765,1.1190041304,-0.9004789591,-0.0169585124,1.5388683081,-1.9974298477,-0.0319209136,0.2239657640,0.9455083609,0.4229235351,-1.1756800413,-0.2048513144,0.9564954638,1.5861339569,0.7912840247,-0.5508337021,-0.6716780066,-0.6968273520,-1.3961554766,-0.5600498915,0.3246770799,1.4076087475,-0.0798354521,0.2515286505,0.4909705818,0.3263372183,1.4774889946,1.0507353544,-0.5025584698,-2.5827465057,-1.3963918686,0.3938339055,0.1324110627,0.6092009544,-0.6405442357,1.0710233450,0.9779106975,0.9465399981,-0.9084601998,-1.0478686094,-0.1336563677,-0.9601236582,-0.2246695161,-1.3933295012,1.6808741093,-0.8732835054,-0.4315283000,-1.2740129232,0.4324110150,1.1807173491,2.5027995110,-1.1592706442,2.4850172997,1.2996827364,-0.1811547577,-0.4567832351,0.0583423376,0.4886063039,1.9107813835,-0.4104271531,0.7122089863,0.1199572161,0.7052668929,0.1382817328,1.1929559708,-0.1740968972,0.9283349514,-0.7982419729,-0.6423220634,1.1604852676,-1.0541092157,0.7567608356,-0.1418243349,-1.3702803850,2.9446048737,-1.4440181255,-0.0868617892,-0.4741066694,-0.8283151984,0.9016938806,-0.1963388175,-0.6778028607,1.1174887419,0.2317363173,-1.0282225609,-0.2005127370,-2.2192344666,0.2017867267,0.0493783355,1.3375517130,0.4729900658,-0.7144607306,-0.9983495474,0.6037179828,-1.8254740238,0.2400026470,0.2330179662,2.5256221294,-1.4175472260,-0.3735436201,1.6245576143,0.2052359134,0.6497414112,0.4077384174],[-0.2230141908,-0.7564809322,-0.0916129127,1.0873105526,-0.2665374875,-0.5684608221,-0.7968423367,-1.6703445911,-0.8607365489,0.3183925450,-0.9658346176,0.0435120389,-0.5150961280,-1.1097168922,0.1698560864,2.9419577122,-2.3477473259,-1.3247072697,-0.4962736666,-0.1773999929,-0.0164038111,2.5538542271,1.7659779787,-0.0989361107,0.3340937793,2.1344652176,-0.4907274842,0.1163907647,-0.2446371317,-1.2983027697,-1.7936493158,-0.6849700809,0.8144246340,0.3181484938,1.9092911482,0.8085999489,0.7802957892,-0.0886350796,-0.2588721514,0.0695181787,0.3667381108,-0.3762246370,0.2074953020,-0.3414714932,0.1384399831,-0.7856284976,0.5178771019,1.3630260229,-0.6483896375,0.8072376847,-0.3309074640,0.2732403576,0.5953650475,0.4950374961,0.4725801349,-0.8133680820,0.4031637311,-0.8463309407,-1.2494440079,-0.7790490985,-0.3001246750,0.7606703043,-2.6036124229,0.2312413156,0.0621792115,1.3174229860,-0.5900319219,1.1107158661,1.0482529402,-0.2892859876,0.2211965621,-0.8559523225,0.0115584433,-0.9336676002,1.9462053776,-2.1260342598,0.2948924899,1.2464601994,-1.7223078012,0.8161273599,-1.0551950932,-1.0623240471,0.6164644957,0.6140427589,0.7598878741,1.3047753572,1.5473181009,-1.1017062664,0.3287614584,0.5281797051,2.5536885262,0.2077630758,-2.2922198772,-1.4933623075,-0.2370071262,-0.7327952385,2.1606707573,-1.1783447266,-1.3211505413,-0.7479308844,-0.5706911087,-2.5351402760,1.1879647970,-0.6937165260,0.0836460069,-0.5761220455,0.9647812843,0.3790502846,-0.5369278193,-0.9889282584,-0.0713329986,-1.9731627703,0.0177028272,-0.0183720831,0.3783574998,0.0163121391,-1.3108023405,1.7357823849,0.8464573026,-0.3903010786,-0.3368534148,0.4967666566,-2.7908391953,-0.3965183496,-0.3624084890,0.4881130457,-0.1702686101,-1.1690906286,-1.8261866570,-0.4230371714,0.4788505435,0.4658548534,0.2310337573,0.0985355750,0.4738362134,-0.7344447970,1.2384089231,-2.6306266785,-1.5320723057,-1.2753394842,1.5467416048,1.6379555464,1.0030064583,0.9898139834,0.0424625278,-0.4771284163,1.1298083067,0.6710648537,-0.3274264336,0.7314128876,-0.8347561955,-0.2209829986,0.8305916786,-0.1107638553,-1.0064861774,-0.7868315578,0.2856790125,-1.1872032881,-2.6854472160,1.1837202311,-0.5017939806,-0.7630453706,-0.7599319220,-1.7406020164,-0.3569377363,-0.3295323551,0.1544053108,-1.6152344942,-0.7816680670,0.9302698374,-0.4331679344,-1.4746901989,-1.2758826017,-0.8955430388,1.1119370461,-0.4404303133,-0.2563897967,-1.3007929325,-1.0545949936,1.7923588753,0.5344469547,-0.1055984795,1.2540366650,-0.1275109500,0.1339087486,-0.2487187982,-0.4006001949,-0.2114236802,1.1732668877,0.5091586113,-0.1582694799,0.6916762590,-0.1102267802,0.2098225951,-0.2684221864,-0.2210887969,1.4088248014,-0.4681519568,-1.2454816103,-0.7584915161,-0.5442646742,1.2987605333,-0.0903712809,-0.7038058043,-1.8618180752,1.3913886547,-2.0506701469,1.0668116808,1.0024168491,-0.9046353698,-0.3053961098,-0.2314878851,-0.1263826787,-0.8304456472,-0.8061316013,-0.2422748208,-1.9520103931,-0.7236518860,0.4473903477,0.1257790476,0.6045052409,-1.0181647539,-0.2301247269,1.0518794060],[0.2211459726,-2.1039614677,0.4767338634,-0.5758100748,-0.7040908337,-1.3656538725,-0.9187939167,0.1141875982,-0.0630888268,0.9285331964,-2.3116347790,-2.1702721119,0.6343227625,-1.6013319492,-0.2154222429,1.5275949240,-0.6772807837,1.9274326563,0.7861020565,-0.2652913034,-0.8883908987,0.2253436595,-0.5352017879,-0.5620777011,0.5740081668,0.3111010194,1.5274264812,-0.3963585198,-0.2029903829,1.3392060995,0.4060575068,-1.0550361872,-1.0509568453,-1.0039232969,-1.6214109659,-1.1161359549,-0.5431999564,-1.6879911423,2.1472158432,-0.5342582464,-1.1732395887,0.2828506827,0.6968113184,-0.3553364873,-1.3209646940,1.4841766357,1.4951782227,-0.0843770579,-1.1510633230,0.2676264048,-1.9190896749,1.3373588324,1.0322799683,0.4293705821,-1.0973135233,1.1228461266,-0.1259989738,-0.3956427276,0.5731127858,-0.7413690686,0.0254712421,-0.1747244149,-0.6090978980,-2.3484940529,-0.8988916278,1.0013848543,-0.6178749800,3.2145085335,1.7016453743,-0.1834490895,0.3850836754,2.8780665398,-0.3341577053,-0.3734532297,-0.1130664498,0.1813558340,-0.5761479735,-0.5341535807,1.4379769564,0.0405320339,-0.6183389425,0.4018271565,0.3525454700,-0.6058232188,-0.5648207068,0.0922305062,0.1551928073,1.5452309847,-0.9681372643,-0.5784638524,0.3552555740,0.4516860843,-1.9098737240,0.2499908358,-0.8666939139,0.8051800728,-0.8144145608,-0.2480166554,0.1575364619,0.2941561341,-0.1956454664,-0.3580013514,-0.9503346086,-0.6282243133,-1.3779340982,1.5391281843,0.0974658281,-1.5649412870,0.6169933677,-1.5744905472,-0.8908445835,-0.0382696539,-1.0136801004,-0.2685077190,0.4745731056,-1.8531562090,1.5317523479,0.6800500154,-0.6525881886,2.0080859661,-1.0958909988,1.0522122383,-0.3234107196,1.9245126247,0.3289987743,-0.1973149925,-0.1600109935,-0.1019958928,-0.1369468868,-0.3338777125,-0.4217321277,-0.6276172996,-0.0088310521,0.2495359033,0.6251870394,0.2042882591,-0.0481986813,-0.4357247949,0.1891266406,0.1527542472,-0.3866901100,1.0209299326,0.6823812127,-0.6945874095,-0.7691562772,-0.5777331591,0.0130191892,-0.8398479819,-0.2155789435,0.4451689422,-0.2670453787,0.1741062254,-0.5216994882,-0.8992028236,0.3179417849,-1.7559511662,2.4414460659,-1.0115315914,-0.7715908289,-0.3075437844,-0.6755710244,-0.6566454768,0.3051428497,-0.1559470147,-0.4731896818,-1.5133440495,-0.9692866802,0.1147779524,0.1330191642,-1.4187124968,-0.2106103599,-2.4852890968,0.3849193454,-0.4459509850,1.0283155441,0.4236266315,1.1467685699,-0.4037198126,-0.0717719048,-0.8492287993,0.8758452535,0.1124814972,-0.1776066273,1.2060687542,-1.8546257019,1.0201659203,-0.8215023279,0.0630850345,-1.8618688583,0.1551971585,-0.7597349882,-1.1710904837,1.4031120539,-1.3759285212,-0.4247080982,0.4434662163,0.4800909758,-1.3437353373,-0.3794100285,1.7216287851,-0.4607748687,1.8423278332,0.3057335317,0.5179230571,-2.2636806965,-0.4333676398,1.2896549702,-0.4285718203,-0.8006920815,-0.4375852048,0.5202440023,-0.9587842822,-0.5118048191,-0.9421377182,-1.1184242964,1.6118689775,-1.2746709585,0.7071536779,0.1218064204,1.1981039047,0.6363534331,-0.0728155077,0.9757786393,-0.4173601270],[-0.0015435452,-0.7620249391,-0.5343719125,0.4405129850,-1.3240416050,1.4426378012,0.4065026343,1.5632677078,0.0156175345,-0.4524750710,1.0752927065,1.2035952806,-0.8711138964,0.2041112036,0.2468991578,-0.1774842441,-1.1503459215,-1.2382020950,2.4137120247,1.1750917435,1.5967836380,-1.6567083597,-2.3314807415,1.3394186497,-1.0232579708,0.9828591347,0.6385476589,0.6664152741,-0.1034429669,1.0297343731,0.2680951357,-0.5350618362,-0.4283976257,-1.2094867229,-0.1973341703,0.5106055737,1.0990395546,-0.3337288201,-0.4588372409,-0.8280749917,-0.2634945810,-0.7192752361,-0.0046136426,-0.2803740203,1.8313490152,-0.2697892189,1.5935767889,-0.6038568020,-0.1862212420,-0.2486424148,-0.1683999300,-1.2148817778,-0.4478370547,0.3814966083,0.3098362982,0.3672737181,-0.5108301640,-1.1260827780,0.4522577822,-2.3199803829,1.2764129639,0.0071071084,0.8953126073,-0.1732992828,-0.8343470693,1.2000719309,0.6027942300,0.0065583684,-0.5235270262,1.4073104858,1.5511393547,-0.6028784513,-0.6788135171,-0.4255207181,-1.2978883982,-0.2149679661,0.1982887536,-1.8132313490,-0.7148556709,0.9376966953,0.0934126228,-2.2183358669,0.2135851234,0.9191421866,1.0439063311,0.9127506614,-0.5701822042,-0.5769571066,-0.3889191747,0.8824122548,-1.0307428837,-0.6692697406,0.4454301298,-0.3818296790,-1.5339218378,0.3329823315,1.0650533438,-0.7414758205,-0.1798739582,-0.3133811057,0.0330814943,0.8718146086,0.3320226669,-0.5496357679,-0.6352506876,0.4852578044,-1.3856655359,-1.0167318583,0.3205580413,0.4294673502,-0.7857626677,-0.1906618029,1.4213653803,-1.3946044445,0.0078004524,-1.0760178566,1.2182247639,-1.1788051128,-0.3671139181,-0.1223488450,-0.6729784608,-0.3291174173,-0.0389075093,0.5921736956,-1.6977338791,-0.2491133809,-0.4006183147,-0.3961207867,1.5913404226,-0.5850166082,0.9393358827,0.4974382520,-0.4705432653,0.4906757176,0.0837168992,0.3925708830,-0.5851889253,1.0661809444,-0.4824724197,-0.2261067629,1.2158709764,-0.1601240784,-0.0983043760,0.5352964401,1.6688851118,-0.5516088605,0.1419655532,0.3875458241,-0.3246272802,-0.5487880707,1.0352808237,-0.3842982948,1.4187676907,-1.3117232323,0.2935632169,0.6130684614,0.4549064040,-0.8751249909,-1.3332929611,-0.8147768974,1.0529824495,1.2756927013,-0.5867362618,2.0169563293,-0.2070715725,0.8094360828,-0.0061388025,-0.2174635977,0.9481768608,1.2745907307,1.1427247524,0.0107475081,0.3241678476,0.3622403741,2.0765511990,-0.8151247501,-0.0743439496,-0.9019891024,-0.0548304245,-0.4506755173,0.7100703716,-1.0151324272,1.8718290329,0.6454366446,-1.4716508389,-0.7614647150,-0.6045665145,-1.5807316303,1.3683742285,-0.7087847590,-0.5360591412,-1.1137635708,0.0546837524,-0.2155482918,0.0912592858,0.5711795092,-0.5264383554,-0.2848584652,-0.9425686598,-2.4303109646,-0.1181594357,-1.4667463303,0.9932658076,0.1965706199,0.3686066270,-0.3189594448,-1.3668688536,1.0229911804,0.3237343132,-1.1159375906,1.0948339701,-0.7159814239,0.9033344984,0.6435657144,-0.1155977696,1.0557695627,0.6256161332,-1.8282810450,0.8863859773,2.2530357838,0.9894797206,0.0942059904,0.0559607707,-0.6437538266],[-0.5194265246,-0.1113528311,-0.5954051018,0.0305511039,2.0396575928,-0.9323299527,1.1999788284,-0.1505632401,-0.0716161281,-0.1202129871,1.6836475134,-0.3819015920,1.0422122478,1.2395704985,-1.1008722782,0.3359590769,0.3583621979,0.3088277578,-0.7779789567,0.2468801737,-0.0547442622,-1.1857970953,-0.7717460394,-0.6387051344,0.5602406263,1.7625153065,-0.0663885921,-1.3087264299,-0.6310637593,-0.0381813124,-0.6173859835,0.3789446056,-0.2139604241,-1.1707614660,1.3018363714,0.7095224261,-0.0035795420,0.1458305418,0.0410517007,0.2223228961,-1.1324160099,1.3681198359,1.0462281704,-1.5376925468,-1.6998412609,1.1517218351,-0.2828924060,-0.1268702596,0.0175452475,0.6152723432,2.9246551991,0.0550507084,-0.4084461927,-0.4082174003,-0.0883197784,-0.6834801435,0.1403120756,-0.0701343417,-0.5488973260,-0.7441422343,0.0780278295,-0.5865127444,-1.9822689295,-0.5372510552,-0.1041933000,-1.2280911207,0.8918114901,0.8050571680,0.7902606726,0.1689276099,0.8103792071,-0.3139070272,-1.0741906166,0.8703773618,1.0449312925,-1.3065792322,-0.3290812373,0.0967386067,1.6248611212,1.0085247755,0.7625691891,-0.6881162524,-0.0375259668,-0.1837271601,-0.5998256207,-0.6495984197,1.8604087830,-0.3011618555,0.7878006101,0.3329676390,0.0409335904,-0.9390720725,1.2049945593,-0.1606714725,0.6641917229,1.1545108557,-0.4691017270,0.9510985017,0.3453378379,0.6326152682,0.8411407471,3.3795402050,-0.5579387546,-2.2784161568,1.2684965134,-0.1679872721,0.5747269988,-1.2267564535,2.6995685101,1.2447136641,2.0237715244,-1.2390452623,-1.2545794249,1.7316627502,-2.1253314018,0.3927900195,0.7085506320,-0.8432416916,-0.0852408782,1.6294378042,-0.6161226034,0.9238883257,2.0524621010,0.8227902055,0.4947157800,0.8637558222,-0.2730913460,0.8136610985,-0.3739970922,0.4879109561,1.6569792032,0.3815992177,-1.2899910212,1.9640835524,-0.0580297746,-0.1557988226,-0.6913321614,-1.4017077684,0.3727024496,-0.6369473934,-0.6303749084,-2.6259300709,-0.0493691675,-0.9832947254,1.0024470091,-0.3762393594,1.3069745302,-1.2268050909,-0.7823822498,-0.0879970342,-0.1371357441,-2.0683543682,1.3137185574,-0.5420494080,-1.8690586090,-0.4229525030,1.2428137064,-2.0199379921,1.1582741737,0.3339900672,-0.1540900767,-0.3291948736,0.5431203246,0.5548845530,0.8049129844,-0.9914619923,-0.2426926792,0.4396611750,1.1291645765,1.5539902449,0.0206491835,3.1093180180,-0.6115761995,0.6421800256,0.3549148440,-0.5587056279,1.3602957726,0.8574593067,2.1869783401,-2.0839545727,-0.1410952508,-0.2207788825,0.3456647098,1.7722004652,0.2104642242,-0.0937629715,-0.6939895749,-1.4438390732,-1.4316949844,1.3473109007,1.1719737053,0.1000234857,-2.5027976036,0.5600686669,-1.7411983013,0.6763187647,1.1642938852,0.1111669838,1.0283162594,-0.9822398424,-1.5453110933,-0.8225803375,-0.6440979242,-0.0711455941,1.9552521706,-0.3010765314,-2.1804435253,-0.4159669280,-0.2086515576,0.0323335044,0.7367733121,-0.1938336194,0.0218441971,-0.6198917031,-1.2360179424,0.0325217918,-0.1815406382,-0.8679505587,-1.2398601770,2.7161452770,0.8557375073,0.0299651753,-0.4124097824,1.7006686926],[-1.1124067307,0.0908856541,-0.1516654342,-0.2011841089,-0.8780253530,0.1487515718,-0.1081603616,-2.6986460686,-0.6502318382,-1.3820240498,1.3549199104,0.2082520574,1.0158964396,-0.3602983356,1.0314747095,0.7706522346,-1.4677193165,0.4941012859,0.4933527708,-0.7369694710,0.5938832760,-0.9388635159,0.3698066771,0.1977473944,2.0577509403,1.8244630098,-0.6621942520,0.9831089973,1.8373091221,-1.7026886940,-0.0270229913,-0.7254310846,-0.5342456102,-0.2495466620,0.9207409024,1.1019066572,1.4010708332,-0.4718470275,0.7765082717,-0.2912778556,0.3498292863,-2.7129755020,0.3319345415,-0.4187539518,1.3095924854,1.1868669987,0.8806769252,0.3419238627,0.3971645832,-0.5310337543,1.2703276873,1.2536746264,-0.2384586930,0.7138925791,-0.6958540082,1.0730539560,1.0104210377,-0.2720271051,2.4342498779,-0.1827040464,-1.1780284643,1.5289424658,0.2870815396,1.0993279219,1.0624066591,0.9699524045,-1.5428931713,-0.3731743395,-0.9524514675,1.6522585154,-0.6506668925,1.8939784765,0.7598444223,-1.4891389608,0.2364653498,-0.4037335515,-0.3316952884,0.4112060368,-0.4684414566,-1.5352321863,0.7067044377,-0.2318485826,-1.1240425110,1.5368109941,-0.1835157275,-0.4744376838,0.3413875699,-0.6717487574,0.7119252682,-0.2505222857,-1.0557770729,-1.3723297119,-0.3597370386,-0.4294839501,0.7981411219,0.1716908067,0.7857249975,-0.2756601572,1.3481935263,0.1696635783,0.4440221786,-0.8391209245,0.4513442814,-1.5586638451,-0.3483391404,-0.8522367477,1.5907986164,-0.3074297011,0.2480690479,0.7581233382,-0.5760253668,-1.8630077839,1.0422192812,-1.7398986816,1.2222512960,0.5329426527,-0.5442413092,-0.4942600429,-0.8551141620,0.0436583534,-0.5721877813,0.6107054353,0.3239576817,-0.7967873216,-0.6653596759,-0.9299156070,0.5174583197,2.7006502151,0.0948338807,0.5206369758,-0.3405960500,-1.6355158091,-1.1801077127,-1.0089060068,0.8826825023,0.8409978151,0.5379786491,-0.8869519234,-0.6959516406,0.8970693350,1.1439201832,0.2837764323,0.6520831585,1.5538183451,0.9409436584,1.6984336376,-0.2489234060,-0.4934352338,1.1103788614,-0.0425555967,-0.2637329102,2.3317098618,0.1342730075,-1.1002831459,-0.3531531394,-0.9949698448,-1.2437766790,-0.3254702687,-0.2078438252,0.6164678335,1.4852756262,-0.6047641039,-1.6532808542,0.4184491634,-1.6963517666,-0.2338368893,-0.7228124738,-0.1625349373,-0.4308318496,0.0099216374,0.4700077772,-1.2281514406,-2.7309982777,0.3110759258,0.4845410585,-0.2558689415,-0.7635210752,-0.3534634411,0.9853637815,2.0035877228,0.1427997351,0.3458914757,1.1226363182,-0.4557880163,-0.7759872675,1.3420548439,0.0752204135,-1.5103042126,-0.0915874243,-1.2788894176,-0.2196186185,0.2062564492,0.2331690490,0.2155169100,-0.5119232535,0.2016098946,-0.2278685421,-0.6937284470,-1.1732790470,-0.2774077356,-1.6016391516,1.0906006098,-1.1602990627,1.1262484789,-1.0025730133,0.0271053445,-0.2579020560,0.4614878893,-0.1589870751,0.1840162426,-0.6898531318,0.0125418641,-0.4625565112,-0.8035131097,-0.2153994441,-1.3669756651,-0.3434249461,0.2921564877,-1.1370795965,1.9588086605,0.8582231998,1.3342782259,-0.1519452184,-0.1291483045],[-0.6357197165,-1.1396441460,0.8062340021,-1.2459180355,-0.9046903849,-1.2425501347,-2.1115393639,-0.4970543683,-0.8733558655,1.1578514576,1.0069839954,0.4968858361,0.3666149080,0.6556609273,-0.3875073791,-0.5488761067,-0.0836496875,2.0249290466,-1.0677149296,0.3628509939,0.3246399462,-1.1875450611,0.2687779069,-1.0852653980,0.7550567389,-0.4066241086,1.2095012665,-0.1736595035,-0.3596734703,-0.6551199555,0.3217178881,0.3877495825,-0.5331507921,-0.5281913877,1.2084648609,-1.3032892942,0.7628052235,-0.1079162583,0.4455998242,1.1350136995,-0.1589067131,-0.6414845586,-0.6046472192,0.9702689648,0.7563495636,1.1022398472,0.9599217772,0.3824656308,-0.2513045669,1.9816982746,-0.1709746271,2.5292274952,-0.0726553649,0.8528701663,-0.2167818695,-1.1923350096,0.7861623168,-0.1764873415,0.5312441587,-0.6337814927,0.5093117952,-0.6839939952,0.5782969594,1.6045950651,0.8131929040,-0.7173815370,-0.3780921400,1.3238222599,-0.5112685561,0.0101344641,-0.2775341272,-0.5660531521,1.0679335594,0.1818200201,0.0210486110,0.2730336189,0.9373193383,-0.5658476949,-2.9819197655,-0.7312950492,-0.6392739415,0.3130269647,1.3852910995,-1.7225570679,-0.1778837144,-0.2917789519,2.5960144997,-2.0717620850,-0.5658675432,-0.1349422485,-0.6983388066,0.0349466354,1.1256330013,-1.4770747423,-0.5363630056,-0.2403480560,-1.1767562628,0.1195749193,-0.2448243648,0.5246302485,1.8328498602,-1.3843655586,-0.0110265408,-0.0536559224,-0.0643325523,0.1795994788,-0.9816644192,0.3176301122,-0.1802364588,0.9853456616,-1.1633157730,0.0661375895,0.1401054561,1.1376334429,-0.4585750997,1.6170740128,-0.8622001410,0.0746508464,-2.0770630836,0.9686267972,0.2395104170,0.7997642159,-0.7137439251,-0.4591530561,1.0764036179,0.7686495185,1.0426416397,-1.1665842533,-0.0190958213,0.2914540172,-0.7461980581,-1.0264773369,1.1318042278,-0.6412966847,0.4828600883,0.4435411096,-0.2497335821,0.7211852670,-0.9790833592,-0.1474432647,0.5849481821,0.3871085644,-1.6994421482,-0.9294052720,-0.4046594501,0.0394828208,-2.2656846046,0.8610637188,-0.5293418169,-0.9234024286,-0.7983475924,0.3869788647,-0.0125233578,-0.6018054485,-1.5917366743,-0.6404822469,-0.8488659859,0.3502156138,-1.9618258476,0.4609496295,0.7993075252,-1.2686504126,-2.0727105141,0.4813488722,2.2085037231,-0.1328210086,-0.1219409853,0.7389209867,-1.2908295393,-0.6629701853,0.0689686611,-1.8911678791,-0.0095606102,2.2362875938,-0.2920475602,-0.4304375947,0.6180891395,0.3166204989,-0.2677041590,0.7496351600,-1.5953862667,1.6120036840,-1.2561944723,-0.2658331990,-1.6532055140,0.3250307441,0.2280231416,-1.7944575548,0.9863104820,0.9655084610,-1.2732337713,0.8295410872,1.7446888685,0.4547727108,0.9031069279,-0.0009527904,0.1082891300,0.0627697557,-0.7638112903,0.1184428260,-0.6439966559,-0.6934063435,-0.6369379759,0.0757776573,-0.3329806030,-0.8138238192,0.6588058472,0.0405100137,-0.7962877154,0.1916862875,1.6392009258,1.5568352938,-0.2725393474,0.4475654662,-1.8609107733,0.2686832845,0.5630605221,-0.8181628585,0.7510074973,0.5397127867,0.1188118681,-1.5726583004,1.1463209391,-0.1599459797],[-1.5908477306,1.1402555704,0.0534986258,0.6712822318,-0.2257112414,0.2104907334,1.2600893974,-0.8423901200,0.4301144779,0.0580476858,-0.1085274071,0.2783560753,-0.1843588799,1.1259652376,-0.9875113368,-1.3340691328,-0.2070790976,1.1330128908,-0.5045336485,-0.0493099913,-0.9911766052,-1.0161695480,0.6145914793,0.2172343433,0.8915202618,0.7863522172,-1.5050923824,-0.4608986676,-0.9159309268,-1.0463711023,-0.4841423333,1.4623806477,0.9499843121,0.1457133293,-0.0732988268,0.4994755089,0.0105162933,-0.9202109575,-1.7985740900,-1.8778783083,0.1385161132,-0.1998910755,-0.6571277976,1.3705222607,-0.0393945649,-0.6115819812,0.1122607738,1.7633579969,1.1450352669,0.0145209311,0.4134386480,1.0941917896,-0.7587047219,0.1511386931,0.8067439795,0.3308694363,0.8130151629,-0.4325229824,-0.3996813297,0.7124310136,0.5596027970,1.2407661676,-1.5103613138,-0.7653915286,0.4409960806,-1.3893051147,0.2526411414,-1.5413987637,2.1944396496,0.6292828321,-0.8632075787,0.2486867309,-0.4590431452,-0.9867677093,0.3626977801,-1.0730080605,-1.6345175505,0.2953622937,0.2162745744,-0.2574618161,-1.3549469709,-0.6115554571,0.7602116466,1.9480760098,0.9903740883,-0.2730256915,-0.9925105572,1.0541404486,0.2880633175,-1.1912264824,0.8099787831,-0.1467864662,-1.1243214607,0.2368906140,-1.0483739376,1.5535750389,1.7349170446,1.7200393677,0.4215571284,-0.5391463637,0.7642809749,-1.4886449575,0.7880499363,-0.1383967400,0.2078452259,1.2732071877,-1.3002576828,0.4559438527,-1.1586076021,-2.5392279625,-0.1181192771,-0.1082064211,0.5446758866,-0.0360974520,0.7261715531,2.2089526653,0.2038448453,1.1848708391,0.0747086108,-0.3506720662,-1.0089811087,0.4365611374,0.2272382379,-1.2056691647,0.1030338556,0.8712006807,-1.4377597570,1.3927700520,-1.1165030003,0.1298575550,-1.0141154528,0.0596608967,-1.4940531254,-0.2160215676,-0.0512839481,0.9271569848,-0.4844455123,-0.2318405509,0.6599657536,-1.7151789665,0.7706661224,0.0119470004,0.9035109282,-0.1867048591,0.0665158778,0.8442751765,-0.0485997386,0.5673190951,-0.2960724831,-0.1498709619,-0.4024744630,-1.2394433022,1.5663323402,-1.0304254293,-1.6466841698,-0.8810069561,-1.1049762964,-0.4639779627,-1.5220855474,0.2335274220,2.7803463936,0.0539885126,0.9135933518,-0.9690374732,-0.1095990017,0.1958942860,1.7529040575,1.5590376854,1.5509411097,0.8768229485,-1.7840468884,-0.1732115000,-0.6301497221,0.3956625164,2.3877656460,-0.0268936399,0.5478615761,-0.5379067659,2.1843235493,-0.5380644202,-0.5254244804,0.5818068981,-0.7079021335,1.3369965553,0.8060417771,-1.6658140421,-0.5134732127,0.7067216635,0.8297515512,-0.4131700397,-0.2021587193,-0.5879874229,-0.7992437482,-1.0584437847,-0.8227606416,0.3689611852,-1.1542803049,-0.0566550530,1.3873462677,0.6274378896,1.1806343794,-0.0127347568,0.7829507589,-1.7611320019,-1.8452315331,-0.4864417017,-1.2627916336,-0.9706924558,0.1326923072,-0.0447083339,-1.2672939301,0.5071989894,0.1937571317,-0.4156983495,-0.7045055032,1.5209288597,1.0463624001,0.2856897712,-0.2598487735,0.3661739826,0.5295231938,-1.4221961498,-0.8767566085,0.6296004653],[-0.0801178962,0.6200874448,-0.5689187050,0.1718118489,-0.3422384560,-0.9554112554,-0.2749791145,1.5549792051,0.0348670818,-0.6264494061,0.4486315846,0.1790567935,-2.0919644833,1.0272736549,-0.3224200904,1.3721137047,0.0656904429,0.1180860549,0.1220767125,-0.1829499155,0.5675747991,0.8053595424,1.9218912125,-0.7440114021,0.0348815545,-1.8749556541,-1.4344085455,-1.3828809261,-0.0116490088,-0.9548451304,0.2953458726,-1.4136312008,-0.9416632056,-0.5594043136,0.6049207449,1.0211660862,0.2590726018,0.9106184840,-0.2311110795,-1.2518192530,-0.7415366769,0.4421019256,1.1234837770,1.3203134537,0.6332851052,0.5463161469,0.2642716169,-0.5631567836,0.2989800274,0.3328503370,-0.4342570901,-0.8483982682,-1.7796927691,0.0795050189,0.7186124921,-0.6157069802,0.1147888228,0.0838786140,-0.2524175644,2.8837597370,0.8444882035,-1.9462414980,0.7658838034,-0.4896616936,-1.7648249865,-0.3483499885,0.8363187313,1.1675939560,0.7815184593,-0.5237451196,0.0220253151,0.6432757378,0.2756863832,-1.3450036049,0.0794675127,-0.4446147978,0.4111537635,-2.1214573383,-0.7913644910,0.6275324225,-0.1632787734,0.3176547289,1.3256460428,1.9135844707,-0.3551263511,-0.1786167324,0.6787138581,0.2391502708,1.0172373056,-1.8531953096,0.3468162119,-1.4673253298,-0.2697222829,-0.7077001333,-0.6697863340,-1.4380449057,-2.5307970047,0.8736237884,1.1434020996,0.6140899062,-0.1727177799,0.5524320006,-1.1338095665,0.3060199618,-0.3793222308,1.4749680758,0.8436607718,1.7927627563,-0.3670867682,0.2940903902,0.6996473074,0.1865092516,0.2714602351,-1.1878365278,0.0535032712,-0.3654344380,-2.1078500748,-0.8450355530,0.7202682495,0.6847658157,-1.5380316973,0.1071202233,-1.4377793074,0.5285930634,1.1309717894,-0.2409960181,1.1711474657,0.0241652615,0.1474795640,1.0787435770,-0.2041695267,1.1821421385,0.7134022713,-0.1815098971,0.6097062826,-0.7310802937,1.3213235140,-0.4690505266,-0.1206440702,-1.9621025324,0.0792470798,-0.9037244320,-0.6727975011,-0.5813845992,0.7638904452,-0.7982986569,-1.1170951128,-0.0278148465,0.7436719537,-0.0349298790,-1.5607745647,-3.6942853928,0.2392628640,1.6242785454,0.1078562513,0.8389151692,-1.0812578201,-1.3334944248,-0.3950431943,0.1764431298,-1.5741150379,-0.2641482949,-0.5769725442,0.0579043850,1.6284952164,1.3821072578,-0.1604852676,0.5396218896,0.3823760748,-0.5413639545,0.3832670748,0.3760714531,0.6530348063,0.7082789540,1.6361300945,-0.8255808353,0.4770345092,0.7754092813,0.5997623801,-1.0650105476,-0.1005972773,-0.6563813090,-0.4330505729,0.2436604351,-0.1934807748,0.4373670518,-0.1382869184,-0.0957607403,-0.9223588109,0.2641588449,0.4486901760,0.7179651260,-0.2507312000,-0.3429058492,2.7186050415,-0.0031434130,-1.7863855362,-0.1491491944,-2.4144194126,1.8584879637,1.0816385746,-0.0691355243,0.0412137210,-0.1214653924,0.8756392598,0.6263403296,-0.2714163363,2.5999414921,0.0031115902,-0.6093174219,0.8194519281,0.2598125339,-1.5010932684,-0.3467047811,-0.5145673156,0.8833550811,1.5528534651,1.1378787756,0.4535325766,-0.2141902596,-1.0113316774,-0.0475542098,-1.9863569736,-0.0898115486],[0.8721480370,0.6683921218,1.5094119310,-0.5674828291,-0.2722056508,1.1545054913,0.9233204722,1.7845782042,-1.1065726280,0.8899226189,1.1991003752,0.3537445962,2.0520317554,0.4479960799,-2.4655239582,1.1212525368,-1.6055749655,-0.5201081634,0.5299128890,2.2476568222,-0.7108030319,-0.0680756792,1.3571964502,-1.8943730593,0.6292427778,-1.0711674690,-1.9200879335,-0.0095262108,-0.0612300560,0.3710441887,0.9317353368,0.2323822975,1.5968242884,1.2942872047,-0.7865222692,-0.0996802449,-0.2432516068,-1.8798357248,1.6190136671,-0.4560818374,1.0802834034,-0.3977110982,0.9913378954,-0.4810326695,1.1207609177,0.3215666711,0.6868842244,0.7012643814,0.3027723134,-0.5669187307,-0.8218886256,2.1448388100,1.5178095102,0.3204951584,1.2383538485,-0.6073427796,0.1993925571,-1.6000400782,-0.7605104446,-0.1507879049,-0.1515130103,0.1344979405,-1.3684259653,0.0471347682,0.0774593130,0.2506715953,0.7475439310,-0.5684504509,-1.4915621281,0.4500528872,0.3627835214,0.2907096446,1.4360669851,0.2089084089,0.7776738405,0.0575551018,0.1220124662,-0.5838123560,-0.0324958488,0.2039363533,0.3325089812,1.3271270990,1.4291518927,1.6142344475,-0.2566379309,0.0891720057,1.0746363401,0.0611726567,1.3872996569,-2.3101968765,1.5936372280,-0.6907869577,-0.0694889277,-0.2972775698,0.8311193585,-0.1777330637,2.0184845924,0.5656145811,-0.2051413357,0.9329389334,0.6280627251,-0.1737667471,1.2492796183,0.8439475298,1.2072235346,-1.0259077549,-0.2974030375,-0.4043617547,0.8485488892,1.0105415583,1.0167632103,-0.3439735472,-0.7064131498,-0.3029172421,1.4635292292,1.8002523184,0.4651216567,-0.3715837598,-0.2971130908,0.3571931124,-2.6784129143,0.3649374545,-0.3777738512,-0.7590556741,-0.0151699381,-0.8821194172,0.5951886773,-0.0852708519,0.9750209451,-2.0264630318,0.3239995539,-1.5468266010,-0.6036015749,1.2733149529,-0.5170837641,1.3886284828,2.0128860474,0.6841310859,0.5642735362,-1.7078286409,-0.1220685244,0.8022543192,2.9402222633,-0.5440009236,0.5440354347,-0.3028250933,-0.1236471310,1.0711059570,0.1711559743,-1.1677023172,-1.5948584080,1.1498628855,1.7074403763,-1.7957524061,-0.3149690032,0.1568339318,-0.3114894927,0.0211651828,0.1890159994,0.9555293322,-0.2745038271,2.7662432194,0.4147194326,-0.4034470320,0.3713617623,0.3691652417,-0.8164408803,-0.0419991501,0.8501044512,0.4174660742,-0.4661606550,0.7105901837,-0.1279045492,0.3523364067,-0.2073855847,-1.1988571882,-0.0351843536,0.1673676521,2.1629037857,2.3720927238,0.5707538724,-1.2859030962,-1.7754752636,-2.2168619633,2.3724796772,0.0648849458,0.4216980040,-0.4210457206,-1.1706908941,1.2849756479,0.0288783815,0.0029941904,-0.2512137294,1.0792390108,0.0430316254,-0.1150320619,1.3237221241,1.2466740608,-0.7737157941,0.3568372130,-0.7242242694,2.0552656651,0.5782729983,-2.1194407940,-0.4371688962,0.5964887738,-0.1009557024,-0.2599818408,-0.5893468261,1.4360679388,-1.2596794367,1.6888058186,0.1015370265,1.0656791925,0.5520688295,-0.1842226237,0.3630548120,0.3245004416,-1.7664282322,-0.6189075708,0.8789561391,-0.0498849228,0.1388183832,-0.6309426427],[-1.2514204979,-0.9320270419,-1.3813583851,1.6924980879,0.4783075154,-0.2157701105,-0.4409082532,1.7098357677,-0.0066842828,-0.6266974807,0.3062791824,0.8604946733,0.3906947374,0.6621009707,-0.3415372968,-0.1327738911,0.1090027690,0.5395166278,1.3441191912,0.3890197575,-1.5003491640,-1.6765406132,1.0947563648,-0.4194751680,2.2247760296,1.4415625334,-0.7291014791,-1.1591322422,0.0448737144,0.1642017961,-0.4120277464,-1.0800826550,0.0302338228,-0.1979196072,0.2417976409,-0.4217292070,2.8807368279,-0.2916334867,-0.1119498610,1.4483487606,1.3079737425,3.1414964199,1.9990832806,1.3666808605,-0.4089882970,-1.9430245161,0.1383289248,-0.2756088376,-0.3302395046,0.3767584264,-0.1456294358,-0.0736739188,-0.2632665634,-1.0246253014,-0.0997517109,1.4146862030,0.3474251032,-0.3105725646,-0.6983625293,0.1704372913,0.5889595151,0.3045978248,-0.9002690315,-1.0737453699,0.9776275754,-0.8216294050,-1.4059617519,-0.8584246039,-0.9430308342,0.0863835663,0.4300649464,1.0046936274,-0.1960627437,-0.9061456323,-1.2908878326,-0.8379617333,-1.2833580971,-0.4425460398,0.8820519447,1.7542774677,-0.9453480244,0.4118882418,-1.1179608107,0.1755656004,-0.6019179821,-0.2438964695,1.4608323574,0.6031800508,0.4058443010,1.3704522848,-0.8749694824,-0.2395764142,0.3866791129,1.1813465357,1.0303496122,0.4329538345,0.6799361706,0.1898843944,0.5167849064,1.3488928080,-0.9842775464,-0.9970665574,0.4313078225,1.4899312258,0.2684113979,-0.8607005477,1.5387905836,-0.2373507470,0.1182171926,-1.9649727345,0.1285471916,0.7161492705,0.0362120271,0.9534409046,-0.9433271289,-0.9471279979,0.4125443995,1.5807524920,-0.6251024604,0.0839564353,-1.3108118773,0.8444495201,-0.9436219931,0.2464914024,0.1501429230,-0.9166177511,0.8560885191,-1.7652616501,1.0727841854,-1.1108458042,2.2795565128,0.9886731505,0.7996646166,-0.7582955360,0.5232353806,-0.5892561674,-0.4959479570,0.5991611481,-0.9851461053,0.5707060695,-1.1462754011,0.0037861473,0.3931219280,0.6216490269,0.6596783400,0.8396199942,0.1782273948,-0.4145593047,-0.1143532768,1.4198349714,0.4542442262,1.0489712954,0.1573176235,1.6916421652,0.2117508501,0.6056104302,-0.8419842720,0.8234154582,0.9980833530,1.0220810175,0.6612729430,-1.0162453651,-0.6721482277,-1.0239974260,0.7389633656,0.6064419150,0.4327298105,0.4807211161,0.1548499912,-0.1756894737,-0.4154034555,-0.5601374507,-0.6286499500,1.0796920061,0.5231323838,1.2025828362,2.7605485916,0.9448372126,0.4897510409,0.3005301058,-0.3569986224,1.4894896746,1.0769951344,0.9457442164,1.3836812973,-1.2388128042,-0.9194841981,0.6592172980,1.1811634302,0.1504609138,1.1520256996,0.7040884495,-0.7302854657,-0.7238515019,0.6783299446,0.5535770059,-1.1894803047,0.1862907112,-0.7157256007,-1.6877734661,0.2565253377,0.9417484403,-1.1264679432,0.3717091382,-0.6848784089,0.8496810794,0.5500252247,1.2373111248,0.5267019868,-0.3370111287,1.7402325869,0.6208246946,0.1496523172,-0.0145279448,-1.0990328789,1.0914890766,0.3146514297,-0.2327844501,-0.9927524328,0.2801242769,-0.8725679517,-0.3057530224,-1.4110640287,0.0511594787],[0.0009446123,0.4381587207,1.0378230810,-0.8866230845,0.1709326804,-0.2147371024,-0.5751985908,-0.4123259783,0.3231201470,-1.2464450598,-0.1151144356,1.0316728354,-1.7826942205,-0.2277909219,0.2456097901,-0.1247552261,1.1718108654,-0.9280160069,-0.8763661385,-0.9531322718,0.3050051033,0.9160202742,1.1899218559,-0.3139192462,-0.5719921589,0.2692719996,0.6695338488,-1.4176100492,-0.1810361296,0.8919016123,0.3253822625,0.2102392316,1.1510844231,-0.2731625140,0.9161008596,-0.3529846966,0.7246912122,0.3719465435,-0.6608036757,-0.7025949955,-0.2617565691,-1.5062785149,-0.2983382642,0.7757444978,0.5407574773,-0.8948160410,-0.3278140426,-0.2843683064,0.3198806047,0.7847097516,1.0833867788,-1.1843409538,0.9079430699,0.7242349982,-0.1339356005,0.4209992290,0.3426757455,1.3023025990,-0.1115856692,1.1961381435,-0.6273810863,0.5569947362,-2.0751523972,0.4874547124,0.0112005575,-1.9131480455,-1.6525598764,-0.9860039949,-0.4189216495,-0.2969622314,-0.4724214971,-0.2920680940,-0.6366106868,1.4640799761,0.0001000688,0.6696881056,0.3056806624,0.2199319601,0.5476730466,0.2989024222,-0.3060296178,1.5866267681,-0.1923424900,-0.7535434961,2.4642388821,1.4206904173,-1.3298442364,-1.6729069948,-0.2089457661,0.1286960840,0.2165742069,-1.2509105206,-1.2517259121,0.0144478939,-0.5250476003,-1.4943634272,1.7899401188,0.7980135083,-0.2467044145,0.1137525737,-0.8360635042,-1.0790261030,-1.4593545198,0.6329082251,-0.1951130778,0.2109565586,-0.6452851892,1.3813667297,0.3914685845,2.4400918484,0.8632235527,0.4989744127,0.4575488269,0.6004958153,1.4023209810,-0.9147142768,0.3939240277,-0.2197887301,-0.9745028019,-0.8095402122,-0.1595830917,0.3253418505,-0.9514853954,-1.2631294727,-1.9905785322,-0.1589462757,-0.2686427534,-1.0362379551,-0.5929874182,2.1278758049,0.6741823554,-0.3402375579,-0.8575452566,0.1285680085,-0.1974782944,1.3091429472,-0.0491636656,-0.8006293774,-1.9171167612,-1.6133759022,0.6724346280,0.2764676511,-1.0947827101,0.7140973806,-0.4589699209,-0.5040537119,-2.6571507454,0.4938603342,-0.1112184599,-1.1413344145,0.3065828979,-0.0351181589,-0.2077741474,-1.2007638216,1.3442721367,-0.2882195115,0.7794566751,-0.0357597359,-0.4914976358,-0.6510698795,0.4476221204,-0.2434361875,0.3303336501,-0.1247829944,0.9456844330,0.3387642205,-0.0858657435,0.0836824849,0.3197313249,-0.5224598050,0.0667929351,0.9599102736,-1.2944271564,-1.6121833324,-0.4610219300,-0.0686687902,-0.6209889054,-0.5094255805,-1.3296127319,-0.6519948840,0.3874265552,0.2548698485,0.6057342887,-1.1156878471,1.7558230162,1.3769574165,-0.5067620277,-0.2540018559,0.1169501916,1.4090871811,-0.2061688751,-0.3826217651,0.0744053125,-1.0701073408,0.2666794658,0.0159707237,-1.0801004171,0.3020006120,-1.1221276522,-0.0603687689,0.4302231073,0.8030367494,0.3813634813,-0.5177665949,-0.5745231509,-1.2133439779,1.0359526873,-1.0773680210,-1.0581761599,-0.7194543481,0.7911873460,0.3575147688,0.4734067023,1.3755928278,-0.3413253725,-0.7216416001,1.4161067009,-1.0657318830,-0.5758734345,-0.0554632656,-0.5676805973,1.0491763353,-0.5460031033,0.0000377406],[-0.1134769619,-0.1902130544,1.1122241020,0.0361957811,-0.5202575326,-0.9287207723,0.4673231840,0.2076914012,-0.5268574357,-1.4158695936,0.9209938049,-1.2451177835,0.3177010119,1.3631247282,-0.6694192886,-0.5273765922,0.1168278307,-0.1416368484,1.0902903080,-1.5505670309,0.8457047343,1.2713711262,0.8615369797,-0.6803045273,-0.9436883330,-0.8042429686,1.2764884233,-1.4393085241,0.6460635662,0.1760231256,0.6022271514,-0.6289021969,0.2052420974,-0.9054312110,-2.2867314816,-0.5080083013,-0.3702924848,-0.3562673926,1.6888591051,1.0248645544,1.2868249416,-1.4498550892,0.4269231856,0.7690059543,1.2068730593,2.2509765625,-1.3853080273,-0.0437916964,0.5083934665,-0.1405894905,-1.9240996838,-0.0695686266,1.0290793180,-0.6863524318,1.5921109915,-0.5149906874,0.4313142598,1.2325224876,0.7381823063,0.0506481752,0.2718752027,-0.0458972342,-0.2813476324,0.1470736563,1.1240963936,-0.4729631841,0.3032023907,-0.0378687717,0.7551904321,0.2726889551,0.8062175512,-0.9914822578,-1.3461425304,1.3953132629,-0.3963041306,-0.1348937005,0.1861299574,-0.1282923222,1.3191727400,-1.3927209377,1.6524684429,-1.4570862055,0.1970746368,0.7873681784,-1.5872268677,0.7789785862,0.7257158160,0.9104884863,-1.0458092690,0.1311151236,0.0523036309,-0.5617845058,0.6315040588,-0.8047713637,-0.7324396968,0.5472825766,-1.1248081923,0.7605649233,1.9306453466,-0.8998404741,-0.9145870209,0.7737373114,-0.7555955052,-0.0085799778,0.6102102995,1.0703949928,0.0122935213,1.7418897152,-0.8565835357,-0.1485811919,2.5863671303,0.2296967655,0.4921732545,-0.2157451808,1.7708336115,1.4435811043,0.3088599741,0.3821006119,0.7828531265,0.8288940787,1.2654770613,0.5320999622,1.6035956144,0.0555833168,-0.7808041573,-0.4768952131,0.0609117076,-0.4685634375,0.0401823297,-0.7980151772,-0.1980319172,0.3622185290,-0.0124242250,0.0019212270,-1.8690266609,-0.6661810875,1.2603762150,0.3925990760,-0.9471579790,0.9259031415,-0.6989381909,-0.5957267284,0.8059161305,-0.0204536505,-0.7605403066,-0.8714606762,-0.0785574615,0.2176924050,-0.0952641219,-0.6237257719,0.8928356767,0.0207476560,-0.8050456643,-1.1606603861,-2.1986162663,0.5743509531,-0.3747348487,-0.9865067601,0.6025411487,-1.6260415316,0.4241379201,-1.5761916637,-0.5231423378,-1.2368144989,1.3236354589,-0.2628894150,0.9006096721,-0.5883111954,0.9315660596,0.3901229501,0.3613577783,-0.9036842585,0.6531979442,-0.5237332582,-0.4377007186,1.2412061691,-1.8077656031,1.0896279812,-0.5850965977,0.9993878603,0.9793956280,1.7526859045,-0.1627811193,0.0789898112,-0.6008516550,0.6423956156,-1.3718004227,0.8177475333,0.1333660483,-0.7116602659,-0.7372015715,0.0193281416,0.5816512108,0.6322121620,-0.9512215257,-2.3198320866,-0.1225626543,-0.7548299432,1.7849982977,-0.4172134101,1.1810698509,-0.6830892563,0.9388549328,-1.2363265753,-0.6732035279,-1.1739046574,-0.3762327731,-2.2701492310,-1.8741184473,-0.9467853308,0.1845830977,0.0290970095,-1.1332253218,1.5481897593,0.6677289009,1.6565377712,-0.1813921779,-1.4472157955,-0.9493680596,-2.3203413486,-0.1047000363,-0.5362039208,0.1503630877,-0.2119492143],[0.1951975077,0.3594886661,1.1071165800,0.5541626215,-2.2391679287,-0.0982199907,0.1499948204,-0.2038379759,-0.3641252220,1.2327725887,-1.0428520441,-1.1857684851,-0.3758674860,-0.7437927723,-0.5258880258,1.6090214252,-0.5477022529,-0.5831058025,-0.7693243027,0.9592191577,0.7612723112,0.6288821697,0.4854650795,-1.0746076107,1.0054817200,0.0261112470,-1.3465268612,0.8569097519,-0.8402596116,-1.0034821033,0.0340053290,0.6326579452,0.0624723062,-0.8272801638,0.9872714877,-0.6493257880,-0.0124687888,-0.0510785058,-1.0020663738,1.7653633356,0.6531934738,1.7806649208,-1.2149158716,0.1973568499,1.2248632908,-0.6891977787,-0.5806148648,-1.4091354609,-1.4452899694,-0.1621054560,0.2553484142,0.5607942939,-0.3715134859,-1.2819964886,0.1613192409,0.7283805609,0.9018242955,-0.4699330330,0.7522509694,0.0473475978,-1.4565970898,0.2873678505,0.6105089188,2.3087093830,-0.4378469586,1.5861054659,-0.0902395621,0.2450557947,1.6665794849,-0.2875549495,-0.6267389655,-1.1200077534,-0.1176927686,-0.9884122014,0.0998817012,-0.3104116321,1.6640614271,1.8047343493,1.7079073191,-0.2945924997,1.4536072016,-0.9042026997,-1.5250431299,-1.2416226864,0.9511591196,1.6640285254,0.9149685502,0.5348483920,0.0271247104,-0.4100965559,-0.4923259616,-0.5284524560,0.7326970100,-0.6320443749,-0.1361745298,1.7820904255,-0.4102703035,0.1149579212,0.4176089168,0.5615858436,-0.1241502464,-0.2711972296,0.8339188695,-0.3911842406,0.8035120964,0.4347385466,0.3256336153,-1.0912731886,-1.1064262390,-1.0023913383,-0.0014343157,0.6572046876,1.1768506765,-0.1612158120,0.2430538386,-0.4206996262,-0.6818819046,-1.5458599329,-0.4415317476,0.8601318598,0.5682681799,0.5093677044,0.6739342213,0.2802112103,1.0092612505,0.5739693046,0.4783741236,-0.7188705802,-0.7555210590,-1.2773789167,0.9673935771,0.7169851065,0.6318838596,-0.3510349393,-0.2784935236,0.9245770574,0.1456117630,0.8596948385,0.5870320201,0.3222205639,-1.1058843136,0.1310071945,0.1351862252,1.4976658821,0.0642576590,0.9805966020,-0.2625622451,-2.0229084492,-0.2577911317,0.8459518552,-0.9816684127,0.0494478457,0.1633282155,0.8872380853,0.2346172631,-1.4553668499,-1.2075823545,0.9899796247,-0.7265192866,-0.1070689112,0.4315021932,1.5511058569,1.0648895502,1.2513504028,-1.6179898977,-0.1960540712,-1.1174563169,-0.0627279058,0.3648060262,0.4943988621,0.4699738920,1.5897046328,0.4787073135,-0.2594979405,0.9839239120,-1.1296446323,-0.2151689082,1.8668168783,1.2359091043,-0.7876203656,2.2410216331,-0.6001842618,0.6790406108,1.9386615753,-0.2716152966,-2.4987998009,0.7988651395,-1.7352396250,-0.8488154411,-0.2405159175,0.0022603455,-0.8565176725,-1.7655363083,0.0227796603,1.1678682566,1.5977461338,0.6137763858,-0.1560899615,-1.2249263525,1.0691320896,0.8002671003,0.9079485536,-0.5812396407,-0.6096565127,0.8414008021,0.8574656844,0.8194988966,1.9166227579,-1.2481331825,0.1276202202,1.2027217150,0.6284421086,1.8388808966,0.7530941367,-0.5810328126,-0.1983797401,2.4699819088,0.5870167613,-0.3973526657,0.4335496426,0.5002970695,1.2368345261,-1.0192248821,-0.4722243547],[0.1429932714,-0.6643745303,0.9991702437,-0.7230226994,-0.9242684245,-0.6563588977,1.8147925138,1.0481163263,0.5963919759,0.3161043823,-1.5705043077,-3.0482473373,-1.4911568165,-0.5620017052,-1.5762267113,-0.2792216539,-0.0183045566,-0.3427428901,0.4325659573,0.0841940790,-1.5596717596,-2.3954472542,0.6650484204,0.9164831638,-0.6995766759,0.7592606544,1.9653995037,2.1778728962,1.3679281473,0.3184723258,-1.3753741980,-0.5408719778,-1.3242262602,-0.0486250222,-1.4743169546,-1.9059720039,0.4585015178,-1.3563024998,-2.1027057171,-1.6160247326,-0.8111845851,0.6833175421,0.3431516886,1.0679210424,0.3967325985,1.3747622967,-1.9252409935,-1.5621434450,0.3567034006,0.1609219313,-0.4597774446,-1.5141679049,0.2331478596,-1.2618752718,-0.7512057424,-1.0166230202,-0.5066205263,1.9142640829,1.3055176735,-0.3582405448,0.3410398662,0.7159633636,-0.9026330709,0.9061701894,-1.3593796492,0.2564473748,0.6371505260,0.4507238567,2.0558533669,0.4674500823,0.4008921087,1.1421283484,-0.7805560827,0.8754662871,1.1091470718,-0.4740432799,-1.0471091270,0.5696887374,1.4610911608,0.2937921584,-1.3595267534,0.5948380232,-0.5259676576,0.6979746819,2.6188032627,1.7860373259,1.3816629648,0.3645930588,0.6413857937,-0.3179526329,-0.6694340110,0.8531759977,-0.5612838268,0.6540765762,-0.7173810005,0.7171044946,-0.3337262869,-1.2696685791,-0.3191621602,-0.0951602459,1.6437281370,0.4104022980,-0.6791217327,-0.5343052149,0.1236637682,-0.6867526770,0.4327032268,-0.9899078012,0.3520361781,-0.0372290350,0.0954944342,0.8924535513,-1.3897480965,0.2908154130,0.1433742046,-1.2002483606,-1.1578544378,0.2018009126,-1.7330788374,-0.3900980949,0.3442434072,0.5612210035,-2.3735797405,0.4677275121,-0.8755277395,0.3840200901,0.7140026093,-0.0161184259,-0.2327160239,-1.3661100864,1.8017098904,0.1095980778,-0.9140314460,-2.1166992188,1.5349193811,1.5171842575,-0.7831214070,0.8759396672,-0.3350074291,-0.6258761287,1.1088163853,0.1659310460,0.3859133720,2.1467437744,-1.4819864035,-0.1044765934,0.0227515679,-1.3247855902,-0.0460397489,-0.2199831605,0.3223781884,-1.8705461025,-0.2860034704,1.3835725784,0.0756484866,0.7798700333,1.1855169535,1.0793471336,1.5322893858,0.4833815396,0.7111821175,1.2975721359,0.5630246401,1.6598074436,-0.3058865666,1.0004166365,0.2399135530,2.0036790371,-1.0272872448,-0.2898159623,0.3397452831,-0.5536059141,0.1857142448,-0.8085384965,-1.5434528589,-0.0007162808,1.0881972313,0.6845315695,-1.8424415588,0.6105918884,1.3732216358,-1.0601470470,1.1996201277,-1.2230579853,1.1186140776,-0.6385557652,1.1005522013,-1.8498262167,1.9758620262,0.3748421669,0.9024031758,0.6665591598,-0.5407711267,-1.1386766434,0.8666719794,-0.7272105813,-0.5572866797,0.3335200846,0.2419228256,0.1356631666,0.0634459481,0.5071093440,-0.1976352483,-0.1633360833,3.0307579041,-0.6317493916,-0.2643266618,-1.2277452946,-1.0420066118,-0.2836993039,0.8368556499,-1.4368258715,-0.4121444225,-0.1726893187,-2.3277754784,-0.5735732317,0.4645721018,-0.1024918482,0.9124009013,-0.9102084637,1.3875967264,0.7080764174,-1.5910292864,0.1935648620],[0.8600574136,0.3366048932,0.4506984651,0.6514768600,-0.0764857233,-0.5673771501,-0.2962603569,-1.1983201504,1.0930721760,-0.4548245668,0.9074430466,0.0770448744,0.8746569753,-0.2766015828,1.2379782200,-0.0664854795,-1.2437325716,-0.4574719965,0.7507360578,0.6556258202,-0.2704447210,-1.5917396545,0.9173870087,-1.8183697462,-0.6614665389,1.0413696766,1.2564733028,-0.5287645459,-0.7868911624,0.4760374725,1.4004799128,-0.3912179172,-0.3319635987,-0.1274384409,-0.3486401141,-0.5874551535,-0.4041832685,-1.2626019716,-0.2735873163,-1.8302440643,1.1272053719,-0.9103027582,0.9363489151,1.7044332027,-1.4405498505,0.1213527992,-0.7432028651,-0.3949774206,-0.6219618917,-1.0270428658,0.3988615274,-1.2037241459,0.0879646614,-1.0123245716,2.2703430653,-0.1986519694,1.0142803192,-0.5765676498,-0.1850306988,0.9218153358,-1.4215303659,-0.3045988679,0.8781455755,0.4839467108,0.3406828046,0.1385729313,0.5530541539,-0.2178639174,0.6572006345,-0.7660921216,-0.1504787654,-0.6955124736,-0.8039906025,0.1935697198,-0.6107688546,1.4775750637,2.1482009888,1.2706098557,0.8919548392,0.5517945290,0.4745905995,-1.3749998808,-1.9628106356,-0.2441599369,-0.2756695151,2.3750700951,-1.0125036240,2.3627867699,-1.0484228134,0.0202945489,-1.6291437149,1.5460455418,-0.2626701891,-1.3716934919,1.7448372841,-1.9003964663,0.7340160608,-0.9719927907,0.4878770411,0.8426557183,-0.0036995597,1.1608890295,-0.1446119249,-0.3867310584,-1.2060709000,-1.8427013159,-2.1762900352,0.5401457548,-0.5232191682,0.6863697767,1.7252759933,0.9453377724,-1.2271872759,-0.5287625194,-0.5573931336,1.1400207281,0.8531771302,-0.7874594331,1.4860290289,-0.3966833353,-0.8653481603,0.6682871580,-1.3460118771,1.1101393700,2.5374398232,1.5781009197,-0.2074164003,1.1810634136,1.3876578808,0.4343869984,1.0187358856,-0.3469872475,-0.0697899982,0.6849715710,-0.7938639522,1.0810682774,-0.7873340249,0.4875499606,-0.5890196562,0.6428114176,-0.5371803641,-0.0320537537,-0.6355226636,0.2840048969,-0.7231734991,1.6757985353,-1.3749076128,-0.1871342212,-0.5509265661,0.2247584015,0.0963228941,0.5343322754,0.4177618623,-1.0538265705,1.7033421993,-0.1041369513,-0.5689711571,0.1449021250,1.5101170540,-0.8077566624,0.1635077000,0.3681793809,-1.3697288036,-0.9026447535,0.1419233531,-0.4355032146,1.4709450006,-1.2994558811,-0.1398774087,0.4056711495,-0.9279340506,-0.4008336663,-0.1832318157,-0.4932772815,1.1810588837,-0.4294885397,0.2586955130,-0.6026366353,-1.6949310303,-2.4281458855,0.3360414505,0.0007651557,0.6452424526,0.0013740991,1.2619608641,-0.6294914484,1.0755714178,-1.6946045160,0.4138219059,-0.9013427496,-0.9153943658,0.2750778496,-1.7745633125,1.2423049212,1.4853067398,0.6582362056,-0.4944946170,0.8580268025,-2.5256597996,-0.4416845739,0.0358854383,0.2637412548,0.1396094710,-0.2113924474,-1.0850139856,0.8667337894,0.6863631606,-0.7880696058,0.2563993931,0.6276249886,0.5765470862,1.0376647711,-0.8950424194,-0.7374470830,-0.5733580589,1.5468299389,1.0180302858,-1.3552303314,1.5083237886,-1.5015984774,-0.4331097603,-0.7236114740,-1.2652302980,1.8355219364],[-0.9491904974,0.7845528722,-0.1004951149,-0.9171906114,-0.6248505712,-0.5434099436,0.2936888933,0.7476300597,-0.1759679466,0.7415206432,0.1636239588,0.3913640082,-1.0406374931,1.5966686010,1.7157511711,-0.9193367362,-0.5155612230,0.3866383433,0.3502348959,0.6012752652,-0.2728933096,1.0108480453,-0.3607590199,-0.9352738857,0.0113944160,-1.0877426863,-1.2035589218,-0.5796896815,0.0864668638,-0.7101565599,0.0531113632,-0.7170058489,-0.0952148288,0.3925267458,-1.0388872623,1.1641366482,0.4681651592,1.2697256804,0.5855969787,0.8054180145,0.5459367037,-0.0193008948,-0.0799392536,-0.7031532526,-0.3332782388,0.8763495684,-0.3275536895,-0.0899465531,-0.7804068923,1.5976791382,-0.0845693126,-0.8380930424,1.2919481993,0.3801421523,1.0650464296,-0.4915513992,0.3796888292,0.3082422614,-1.2515425682,0.8574884534,0.0453626625,-0.8994646668,0.1817809045,-0.3982611299,-0.4245830178,0.4759723842,-0.5196942091,0.9084286094,-0.2574448586,0.6669890881,1.4542843103,0.6965520978,-0.4026068151,0.4368975759,-0.1450076252,-0.7213081121,-1.9050275087,0.1893024594,2.1357374191,1.5123866796,1.4541821480,-1.6385452747,-1.5910087824,-0.6939245462,-0.8785220981,-0.8075929880,-0.3812121153,0.8093725443,0.2798313498,0.5270179510,-2.3664381504,0.5130461454,0.0916117877,0.6262785792,-0.7436412573,0.3277203441,-0.7760006785,-2.5068039894,0.4759555757,-0.9010194540,0.0145629812,0.8987622857,0.3353192210,0.9606677890,-0.2874678075,-2.9939599037,1.2504578829,-1.1720881462,-1.0356649160,0.6958131194,0.4948087037,-0.4572609365,-1.6435623169,1.5744616985,-0.0300552286,-1.0009658337,-0.9311393499,1.0732966661,-0.2361930311,0.2371501774,-0.5839691758,0.8537157774,-0.2181455344,1.7659276724,0.2091579586,1.5832922459,-0.1612836719,-0.6309607625,-1.7614916563,-0.5587900877,-1.0242021084,0.0377286524,0.4732848704,-0.1918084174,-0.0163135994,0.2189601362,-0.9728784561,-2.5277113914,-2.9586434364,0.6248936653,0.9954282641,0.9903995395,2.0193979740,0.4239855707,0.8024560809,-0.2874479592,0.2707243562,-0.7016677856,0.3606193960,-0.3674240112,1.8943160772,2.4282639027,1.5565392971,0.8108466268,-0.4795611799,-1.8067957163,1.3225959539,-1.4695440531,-0.4730064571,-0.1111553162,-0.1633212417,0.7373108864,0.0697204918,0.1390757263,-0.5723037124,0.1945779175,0.4934951663,0.8671396375,0.0400937945,-0.8754099011,0.3754470050,-0.1142536923,-0.4761653841,1.7966411114,-0.2497129291,-1.3456119299,-0.2531846464,-0.0173894931,-0.8581447005,-0.8182291985,-1.2103476524,-1.8957642317,-0.5462412238,-0.3278253973,0.4181403518,-0.8027678132,-0.1248840615,0.3669672012,-0.4320286512,-0.6431820393,-0.3669171929,-0.4180790484,-1.1114394665,1.5662413836,0.4966558814,-1.5271258354,1.6809345484,0.9452425838,0.7926274538,-0.1260183305,-0.3414977193,1.1150146723,0.7628927231,0.7186394930,2.0608656406,-1.1427087784,-0.6720955968,0.4238626957,-0.2674430609,-0.1200173497,1.1119060516,0.2046580315,-0.1742334217,-0.6899335980,0.4895804822,0.5928009152,-0.6894159913,-0.7927412987,-1.3092082739,0.6599462032,1.1592344046,-0.6471576691,-0.8530796766,0.2530079484],[-1.2349803448,-0.4698171914,0.6378022432,0.0646075532,1.2665129900,-0.0035302599,-0.4264021516,1.9557579756,0.3368331194,0.1942038685,0.2976615727,-0.2447934896,-1.1720229387,0.0022187224,-0.5681318045,-0.0083157700,-0.2423404306,0.4079694450,-0.4025951028,-1.6668678522,1.1076716185,1.1742789745,0.8371176124,-1.2049195766,0.8989720941,0.5069872141,1.2650052309,0.5479833484,-0.6105635166,-0.7324912548,-0.7606137395,2.3526127338,0.3232249618,-0.0974526554,0.4910657406,-0.3990236223,1.5860863924,0.8555051684,-0.1618283838,1.5360722542,-0.0187115595,0.4534055293,0.5044534802,-0.2699193060,0.7831186056,-0.0310493931,-1.5355266333,0.1740212291,-0.5708485842,-1.2032914162,1.0133572817,-0.9925648570,0.6193821430,-0.5009004474,-0.0948746130,0.5210202932,1.4905010462,-0.0342098996,-0.7431033850,-0.1009846702,1.3803369999,2.7175147533,-1.6663825512,-0.6393872499,0.2210241407,-1.6114358902,-2.2916140556,-0.1636691839,-0.5086622238,1.3264782429,-0.7173157334,-0.6772971153,-0.3950392008,1.3992792368,0.4134455919,0.9333790541,0.8993307948,0.2048672885,-0.2898763120,-1.4870306253,-0.0224908516,-0.6100093722,-0.0982251316,1.4243254662,-1.5467219353,-0.3993415534,1.2259448767,0.1247667298,-1.8018198013,0.4228524268,-0.2830335796,0.2086140662,-1.8642035723,0.2794917822,0.8912532330,0.4764015675,1.1169859171,-0.8097556233,0.6917906404,-0.5510755181,-0.1711165160,0.7242781520,-0.1862911284,-0.8004393578,0.1734851003,-2.3327710629,1.0729398727,-0.0054144766,-0.6906583905,0.1949700415,1.1526728868,0.6277722120,-0.4168083966,-0.4985950589,-0.4415447414,0.4759062827,-0.9995189905,-0.0556426719,-0.8905441165,1.5150352716,-0.4012076855,-0.5104394555,-0.3579500318,-0.8876761198,-0.1260623485,1.2100745440,0.3163279593,0.1174208298,-1.0488599539,0.8006420732,0.8193587065,-1.4236061573,0.0530269407,0.8419333100,-2.3811285496,0.3123259842,2.0316591263,-0.1334241331,0.9566749930,0.3105371296,-0.0280005373,0.1698162705,-0.6434794068,1.4687440395,0.8223428726,0.5253647566,-0.9414321780,0.2501220703,-1.4306063652,0.4417558610,-1.3391911983,-2.1790165901,-0.6563906074,1.6905251741,-1.0324158669,0.0105966255,-0.4265025556,0.1774287373,1.1015805006,-1.2504085302,1.9962170124,-0.7548727989,-0.4495831132,-0.1260795742,-0.9168621898,-0.5386515260,0.4052873552,-1.6897302866,-1.1016709805,0.3024735451,1.5943657160,-1.4092345238,2.6607553959,1.1071485281,0.2390423715,0.4573163390,-0.1807434261,-1.9098607302,-0.3015142679,-0.5953611135,0.1685141623,0.3814010024,-0.0196545552,0.1171996817,0.5379750729,0.0300317835,-0.7584958076,2.2050728798,0.7647278309,1.4125244617,-0.6500667930,0.7842895389,0.6472234130,0.1368263811,2.1448404789,-0.4679664671,-0.3690626025,-0.8305638433,0.4829550087,-0.4260619879,2.0036153793,0.8908524513,-0.4578307271,-0.4667037129,0.5653272867,-0.2103756368,0.0654499233,-1.0563793182,0.8389202952,-0.9145900011,-3.0074131489,0.8634305596,-0.3165632188,1.3181333542,0.7771042585,-0.3728661537,-0.9975465536,1.3015630245,0.5809165835,-0.1991985440,0.4615896344,1.6888397932,-0.9476919770,-1.0568165779],[0.6325836182,-0.2203704268,1.1994246244,0.2282462716,0.7648639679,-0.6254203916,0.0122858062,-0.1240646094,-0.3559269309,0.5451278687,1.2897644043,-0.1418277174,-0.0237832293,2.4164583683,0.3128513694,1.2583760023,-0.0183320828,-0.2904647887,1.1263755560,-0.1271803826,-0.6991423368,0.4350317717,-0.9930130839,-0.0732888058,-0.0096755922,0.6082351208,-1.4622994661,0.3734008372,-0.2516731918,-0.5549135208,0.2704532146,0.8839444518,0.9480336905,-1.7732236385,-0.3400033414,1.0801545382,0.4076742530,1.1092234850,-0.5090354085,0.0154662654,0.8081864119,0.1444585472,0.4838244617,0.4002939761,-1.8577429056,0.0833002552,0.1059778705,0.7952242494,0.0375305712,-0.0192010757,0.5708048940,-0.6749990582,0.6472918987,-0.5778470635,0.5768222213,2.3813226223,-0.8882810473,-0.0370802470,-1.5818849802,-0.5403636694,0.9223807454,0.8768610954,1.6823604107,0.4641687870,0.5547270775,-0.7476307750,-0.1079692096,-0.5736877918,-1.1933667660,0.2192575037,-0.1325137913,0.0294413287,-0.7202627063,0.0728821531,-1.4440402985,1.1075712442,-1.7380994558,-0.3122649789,0.5126811862,0.0520603210,-1.6752490997,-0.4572330117,-1.4403076172,0.3279295862,-2.2079944611,-0.2930724025,1.5250643492,-0.7436814904,0.1235063225,-0.9559509754,-0.7831512690,-1.1331845522,2.1102159023,1.4193875790,0.7471066713,-0.1063024849,-1.9626525640,-0.8340319991,1.9938367605,-0.6666597724,0.9762901664,-0.8030210733,1.0035098791,0.4384184480,1.1932302713,-0.2671444714,1.6537593603,-0.0300932750,1.7405285835,1.1974927187,-0.2730903327,-0.4148294330,-0.9199048877,0.9715379477,0.2495393604,1.9860477448,0.3632284403,1.0493472815,-1.7863451242,0.4960638881,-0.5253154039,1.2082391977,0.7166950703,-0.1228279471,0.5477110147,-1.1976144314,-0.4063251615,0.1708601564,-0.5023768544,1.9816210270,1.4944957495,1.3486838341,0.1301545054,2.2598488331,1.0608019829,0.6828438640,-0.1969961673,-0.0942310393,-0.1965968609,-0.0145343514,-0.4977135956,1.4621585608,-0.5365141630,-0.1809267700,0.0045570266,0.5690778494,0.7586371899,0.2046857178,-0.2727931440,1.0885236263,-1.1514253616,0.0665233359,-1.5880743265,0.8120266795,-0.0526654460,0.0728608370,-0.0096358024,-0.1773585379,0.8013668060,1.6797046661,-0.2765279710,-0.5227367282,0.4652282894,1.0062574148,-1.0238008499,-0.3171005249,2.1590240002,0.1793651879,0.2974379659,-1.7805523872,0.2365886420,0.5838900805,0.8689589500,-0.1619296968,-1.8251862526,1.0525997877,-0.7366934419,0.2232283652,-0.2124611437,1.1224877834,-0.5253707170,0.8429335952,-0.4547739327,0.2139290422,-0.3529743552,-0.4005251527,-0.3588286340,0.5342850089,1.0697213411,0.3627375960,-0.6180936098,0.7206230760,2.3563475609,0.7881179452,-0.1104493737,-0.1020342410,0.9928730726,0.8416631222,-0.2109418511,0.0594552308,0.1714573205,0.9736061096,-0.2536408901,-0.5090725422,0.2091951966,-0.3952316642,0.6734473705,1.8492509127,0.5406158566,2.1666073799,-0.4017258286,-1.0504158735,-0.5410004258,-0.1275609881,-0.8653998375,-0.7439379096,1.7201974392,-0.3531041741,1.3364866972,-0.6355758905,1.0752155781,0.1554413140,0.8499841690,0.2878955901],[1.3831596375,1.0869204998,-1.9145219326,-0.6446622014,-0.6476334333,0.5867015719,-0.1330263019,0.3355934322,-1.0635930300,1.1624379158,-0.2087804228,0.9965528846,0.4709334671,-0.7888488173,-0.9004927874,-1.5506350994,-1.1885224581,0.0462546945,-0.2447022200,0.3237609267,0.5815305710,-0.0116930585,1.9005151987,1.5096068382,1.0625275373,-2.6801900864,-1.4928581715,2.7504143715,1.5735440254,0.1196702197,0.2730636895,-0.6364715099,-1.6727474928,-0.6743252873,1.0582365990,-1.6658549309,1.0274500847,0.8210723996,-0.3279028535,-0.5612089634,-0.8480519056,-1.2279438972,-0.6800284386,0.9341332912,-0.0853653327,-0.6074674726,-0.5700142980,1.1554375887,-1.6881010532,0.8331784606,1.5076262951,1.9043989182,1.4181431532,0.6705791354,-0.2154797763,-0.1985649318,0.8634696603,-0.2186039984,-0.7260090709,0.8744863868,1.0869740248,-1.0211362839,-0.4984046817,0.0550192371,0.1830883920,0.8126010299,0.8161264658,-0.8569269180,-0.1854367256,0.9796576500,-1.7851219177,-0.4285949469,1.2304253578,1.0196049213,0.1956699491,-0.9608377814,0.0585650690,0.5979899764,-1.1106897593,-1.0379048586,-1.7021956444,0.2371710539,1.3618772030,0.1308800131,1.5423973799,0.0521697253,0.3295921087,-0.3451442122,0.0585741326,0.2526168227,0.1740647852,0.0335522965,0.6994304657,1.5105129480,-0.4503269196,-0.4363505542,1.3006982803,0.8859831691,0.6783366203,0.7363558412,-1.7234025002,1.4570312500,0.8774670362,-0.6389059424,-1.0063931942,-0.4606885016,-0.7491079569,-0.4581098258,0.0056097927,0.3033953905,-0.5941976905,1.0164443254,-0.1951033771,-2.0242991447,0.9950196743,-1.6479471922,-0.2919598818,-0.4742799997,0.8809318542,2.0386807919,-0.1976701617,-0.3874321282,0.1774139851,-1.5380210876,0.7691081166,0.2360260338,0.1340331733,-2.0685832500,-0.4424262941,-0.2943103015,0.7442401648,0.5301663876,0.0796818957,0.8675574064,0.1207659021,0.4363836944,0.3957124650,0.4611773193,0.6354420781,-0.6371799707,-0.3056533039,0.4651468992,-0.1246628910,1.1826097965,-0.0713501945,0.4811518490,0.0363793969,-0.4016169012,-0.2674632967,0.3393284678,0.5643074512,-0.9851466417,0.5367643237,0.8051172495,-0.7623518109,0.3541601598,1.1396274567,0.7583637238,0.4819799662,0.5493212342,-0.2779793739,-0.5416448116,-1.0729044676,2.4494299889,-0.4952209592,-0.9134637713,1.0599906445,0.6519761086,-1.1102234125,0.3736817837,1.1824181080,1.0432199240,-0.1811021417,0.3057804704,1.5537210703,-0.3224816918,0.3417883515,-0.7627717257,0.3877569437,-0.5905166268,3.1262331009,-0.0636707246,0.2948612869,0.8857660294,-1.3919001818,-2.0759520531,2.0139601231,0.5181505084,0.2192502469,-0.4590292275,-0.7326875925,-0.0743509084,0.2777031064,-0.0589995608,0.7715876698,-0.1587277502,-0.5620646477,0.3577314913,0.3553095162,1.7608978748,0.3887453675,0.2869249582,-1.5057801008,0.1353569478,-1.1386996508,-1.7352130413,-1.0972082615,-0.8925606608,1.0090011358,0.4416737258,-0.1895652562,0.5086488128,1.2149100304,-0.5647054315,1.0612516403,-0.0578660406,0.3708175421,-0.0541841723,1.8225225210,0.3027867377,1.0238395929,-2.0085389614,0.0481917150,0.3897301257],[0.2223383039,1.1534072161,0.8207922578,0.0908544362,-1.3841704130,-1.4576156139,-1.2129918337,-0.9277918935,0.1623429805,-1.0262908936,3.5983099937,0.6017745733,0.5672770739,0.6544624567,-0.9301851392,0.1682607383,1.4109365940,-2.0551912785,0.9204959273,0.0109659592,-0.0355977714,0.1439855695,-0.2296297550,0.9706581235,1.0231167078,-1.0316967964,1.8201168776,-0.2633411586,0.4973664284,-0.8761262298,-0.7698000073,0.9660003185,0.2249306291,0.2809285223,-0.9330500364,0.4547836483,0.5424904823,0.0041835858,0.8023304343,-0.3718265891,1.3409324884,-1.7307722569,-0.0929673240,-0.1464788914,0.6054271460,-0.3584529757,1.1562112570,0.3190774918,0.9031167030,1.1098406315,0.3319199085,-0.7225655913,0.0292326510,-0.6798599958,-1.0965301991,0.8027890325,-0.2196585387,-0.6784861684,-0.6861836314,-0.9749602675,0.6559276581,-0.2036371380,-0.5718295574,0.2653274238,0.3244380057,-0.0364570580,1.8394060135,-0.8206712008,0.5605004430,0.3706909716,-0.8948367238,-0.1099870428,0.9826195836,-1.8920612335,0.8355630040,1.1913090944,-1.7892980576,0.8185464144,1.1664609909,-0.6384063363,1.4365177155,1.8991087675,1.4480994940,0.5353578925,-0.1014933810,0.2317328602,-0.0489891469,0.2788106799,-0.3530018032,0.7007697225,0.8251960874,0.5446463823,-0.2403146476,-0.2228565663,0.6680720448,-0.9187930822,-0.0626189634,0.3346115947,-0.0144286985,-0.9312236905,-0.1410451829,-0.1779760569,-1.4948040247,-0.8491395712,-0.6012837887,-1.1472615004,-1.0493052006,0.9051018953,-0.8528327942,-0.8717535138,-0.8167239428,2.2011229992,0.5912480354,0.7781169415,-0.8884767890,-1.3819385767,-1.1633492708,1.2937792540,0.3712132275,0.1289652586,1.7088830471,-0.1212667525,2.0375425816,0.2414857298,0.5944365263,0.6591821313,1.2509437799,0.8774040341,0.0764976963,1.6224048138,0.2065740824,-0.8375450373,0.3608026505,-0.7311544418,1.8801552057,-1.2250365019,0.7259657383,1.4483156204,-1.5710409880,0.8928659558,-0.1776217222,0.5828669071,-1.1092414856,1.0693781376,1.7471616268,-0.0638344735,-0.2843283713,0.1209072024,1.7735556364,-1.4085712433,1.3586884737,-1.7795503139,-0.5770805478,1.8389601707,1.5333522558,2.2969801426,0.6170276403,0.5831583142,0.0016300381,-1.4345458746,-0.3312624991,0.7281045914,-0.0612430722,0.1704209596,-0.7399250865,-0.9588273764,1.4042122364,2.0758225918,-0.7547942400,0.0572688095,-0.9924984574,1.1447558403,2.0738089085,0.4144458771,-0.4910028577,0.8720424771,2.1747357845,-0.3411389589,1.1494208574,0.3893336654,-0.2615806162,-0.1336776763,0.3566448390,-0.1948368698,-1.1085442305,0.0452541150,-1.6467444897,-1.0986956358,-0.4581389427,0.9293973446,-0.9444404840,2.2482736111,-1.8937467337,-1.0940096378,-1.5336623192,0.9742526412,-3.2672441006,0.5969296098,-0.4229683280,1.9410246611,0.3461483121,-0.1929189414,0.8508393764,-0.8942077756,-0.0658058375,-0.9040490985,-1.6763790846,1.9827840328,0.4141021073,0.7216985822,0.3340985775,0.4529460371,0.3948349059,-0.3916791081,-0.6501089931,-0.1761995107,-0.4910579920,-0.9518061280,0.5322595835,-1.1581377983,-0.7828429937,-0.8784741163,-0.7512039542,0.2515104711],[-0.7926872373,0.9031017423,-0.6620030403,0.2099865377,0.1103555858,-0.3409232199,0.3346741498,-0.5610801578,-0.6902984977,0.6454124451,2.1148784161,-0.1916747689,-0.5036607981,1.3881807327,-1.2380584478,1.7191636562,0.3221949041,1.5209794044,0.3738695681,-0.7806591988,-0.4377743006,-0.3996760845,0.9527902007,-0.1983201355,0.2340738177,-0.3172970414,1.4159215689,0.7529594898,0.7795801163,-0.2162986398,-0.1358234584,-0.9613436460,-1.3732218742,1.2092061043,0.6537166238,0.0339915529,1.4929498434,1.3207374811,-0.7652394176,-1.0177652836,-0.6362911463,1.2755696774,0.0088382503,1.0899994373,1.2225722075,0.7833430767,0.4043075442,1.7909801006,-0.3479894698,0.7831335664,-1.4179023504,-0.1100483984,1.2900805473,0.2464116067,0.9515271783,-0.3098686337,-0.3669249713,0.0978010669,0.8864583969,-0.1877962053,0.5702041388,-0.1159990877,-0.3501803577,1.9029144049,0.3099996150,-0.5302504897,-0.6566003561,-0.4687570930,0.9224168062,2.4704847336,0.0035063766,1.6607170105,-1.1333860159,0.1573023349,0.1733097583,0.3055340052,-0.1416791677,0.3056150079,-0.5185316205,-0.2780053914,1.1015346050,-0.9035406113,-0.4951810241,0.8598588109,1.0429813862,0.2664982677,-0.0298852306,0.8533706069,0.0039714552,0.9565080404,-1.5921889544,-0.8913908601,2.3364353180,-1.1469471455,0.0677223578,-0.9267605543,-0.2041192949,0.3063176572,0.2214103341,1.0146539211,-0.2558603883,-1.3943849802,0.1468503773,1.3297446966,-0.8142588139,-0.2132356614,2.4995143414,0.0899943933,0.0429980233,-0.1537914723,-0.4266037941,0.8056005836,1.1891802549,0.2974522412,-0.1454759538,0.1457174271,-0.0381532684,-1.4634613991,-1.3462141752,0.0664003119,-0.7865975499,-0.9306924939,0.3232910931,1.3516124487,-1.3557263613,-0.5764961243,0.2937407792,0.7960445285,0.2003103644,0.3335437179,0.1145793125,-1.4473557472,-0.3551094532,-1.2691313028,-0.1487769932,0.1636830419,0.9360257983,-0.7665287256,0.5842079520,1.2309166193,-0.8302861452,1.6384637356,-1.6886717081,0.7259944081,-0.9984105229,1.0805730820,-0.4680268466,-0.3358856142,-1.3471429348,-2.3989200592,-0.8478084207,0.1898977607,0.7270219326,1.8451247215,-1.4518831968,-0.1755070686,-1.7972141504,-0.3209051192,-0.2535520792,-1.1991846561,-1.0366666317,-0.3719597459,-1.2335456610,-1.3254798651,1.1153312922,-0.5936127305,-1.2354867458,0.4393461049,1.6991671324,1.6203832626,-0.0244100485,-0.3054488301,0.9963783026,0.5320534706,1.1051325798,-0.3606890440,0.4809477031,-0.6204119921,0.6224191785,-0.9839615822,-1.0425922871,0.8285560012,0.6690504551,-0.5170633793,0.1227343306,-0.5366504192,-0.4230706096,0.4494740963,0.0801548958,0.3285671473,-1.4552829266,-0.0545614995,0.3951006234,0.8749583364,0.7818165421,-0.7438398004,-0.3983262479,0.0770372972,0.1963945776,0.9916386604,-1.0397281647,-2.1135330200,0.2499190867,0.8048771620,-0.0107534034,0.6105771065,-1.0989814997,-1.4957785606,0.5590358973,-0.4976819456,0.0038733780,-3.1699786186,-0.5023010969,-1.2277863026,0.9048526287,-0.3307199180,0.3236357868,-0.3506893814,-1.8465671539,0.2423798144,-0.8711999059,0.5552427173,-1.0252454281,-0.9851835370],[-1.0336141586,-0.3072377741,-0.3015651107,0.9300719500,-1.1634535789,1.5237438679,0.4603967071,0.9846302271,0.7602651715,1.2697321177,-0.3905704618,-1.4983217716,-1.4723336697,-0.7304906845,-0.2598758936,0.8250431418,0.7544211149,0.6444281936,-0.8383622766,0.2967797518,0.7523125410,-0.7764243484,0.7958608270,1.2056927681,-0.9662641287,-0.9335420132,-0.1679661572,0.2740947604,-1.5847927332,-0.7598161101,0.0663753822,0.4885937572,1.4334609509,0.1223380715,1.2751848698,-0.5594834089,0.4535132945,1.4081346989,-0.4110103846,-0.4689775407,0.1422861069,-0.0778675601,0.6395888329,-0.5474281907,-0.1878197789,-0.5766133666,-1.8732328415,-0.1212131009,-0.0612858348,-0.1527992487,1.8027352095,1.4275883436,0.4526786506,-0.0216174275,0.7622801661,-0.1540553421,0.4228228629,-1.2467342615,-0.5674841404,1.2695670128,-0.8308476210,-1.9346109629,-0.5754521489,0.2582443655,1.0578256845,-0.4269259870,-0.4187313020,-0.5974100828,0.7509816289,1.0372412205,1.2170906067,-0.0723076239,-0.9454078078,-0.2535687983,-0.0566087402,-0.1577985138,0.7568653226,1.4414137602,0.4087726176,-1.5048319101,0.4968555868,1.2993911505,-0.9232976437,-0.4792942405,0.6598887444,1.9238653183,0.0458117649,1.7594987154,-0.2377487421,1.2578010559,0.0903284103,-1.4757454395,-0.7747513652,0.1500543505,0.5281760097,-1.5053366423,1.3702644110,-1.5342292786,-1.8576005697,-1.5551402569,-0.8589060307,2.2405879498,1.4200407267,-0.2279152572,1.5842866898,-0.5206685662,0.5317901969,0.8036642671,-1.5581449270,0.5541169643,0.3074723184,1.3566111326,-0.6996374130,-0.2088235170,-0.2496134192,-1.2829433680,1.9111602306,-0.0551519729,2.0919880867,-0.6781550050,0.1923809499,-1.1128199100,0.3574016094,-0.1978608668,0.6969423294,0.9032924175,0.1589456499,-0.4350588024,-0.3512411118,-0.0519459359,0.9904433489,1.2984777689,0.6284416914,-1.1503527164,-1.1220369339,0.3831738234,1.0778528452,0.3843923211,0.4374722242,0.7809922099,-0.0230867900,0.9145608544,-0.5255103707,-0.8014039993,-0.5153968334,-0.8046190143,0.9278858304,0.6122110486,-1.8956546783,-0.9318542480,-0.4475701451,-1.4089059830,0.5044367313,1.0466452837,-0.4882206023,1.6315196753,-0.6935045719,3.4683833122,0.6756458282,0.4363811016,-0.0252611358,0.7786549926,-1.3182072639,0.5751082301,2.3058328629,1.3533002138,-1.3967939615,-0.3641127944,-1.0209269524,0.2126713246,0.4525144398,0.2537213862,0.1484848112,-0.8377807140,-0.3130927086,-2.0773918629,0.8348540664,0.4066103995,-1.7121460438,0.4686309993,0.9460710883,1.7069529295,-0.6233937740,-0.8569788337,0.0303167701,-0.5756971240,0.1500440240,-0.2254502177,0.5234786868,0.7500634193,-1.2132166624,0.2510019243,0.2570083737,1.0357878208,-0.6433942318,1.4982335567,-0.9077165723,1.4716104269,-1.7842148542,1.1447430849,-0.6266376376,-2.2693505287,0.3182332516,-1.0613924265,0.6442693472,-0.5000963807,-2.0452585220,1.4310381413,1.8100054264,-0.6701334715,1.5482307673,-0.5137092471,0.5081887245,-1.3406304121,1.3682693243,1.2737818956,-0.3865059316,-0.4780690670,0.7779576182,-1.7476885319,-1.1008824110,0.0263148081,1.8815983534,0.3323176205],[0.2674674392,-1.3087997437,2.1118044853,0.2675665617,-0.4922035635,1.7720580101,-0.7110042572,0.1589149386,-0.4881309569,-1.5783791542,-0.0925537944,0.5986972451,-2.1295366287,1.7461127043,-0.9785813689,0.3955598176,0.6764844060,-0.9934566617,0.7253823876,2.6917986870,-0.5455944538,0.4043146968,0.3411338329,-0.3855164051,1.6743670702,0.5616671443,1.5067958832,0.2787322402,0.8779776096,-1.8535884619,-1.3119099140,-1.4514750242,-0.3689820766,-0.6068240404,-2.3265423775,-0.2324558496,1.0002081394,-0.6160846949,0.0170312542,-0.2363373041,1.9653319120,-0.2599737346,-1.2773052454,0.4339766502,-0.6928628683,-1.0053579807,-0.1896031797,0.6886450052,-1.9172238111,0.4927026331,0.5296412110,0.2830281556,2.3585255146,0.1015405208,0.1898712814,-0.3793027103,0.2264746577,-0.1147489175,-1.2718250751,0.5322838426,-0.1509367973,-1.0796653032,0.2211570591,-1.6349580288,0.7289609909,-0.0924888924,1.2654665709,-0.7704129219,0.3740957081,-0.6158271432,-0.6735936403,-0.7281390429,-0.1193343252,0.7776176333,0.4423677027,1.3741822243,-0.4074907601,-0.1957744956,0.4908902943,-0.7967771888,-1.5564359426,0.6080636978,1.1600704193,-0.3984555602,-0.1730147898,2.0433011055,0.9244021177,1.0907392502,-0.1569561809,-0.9651011229,-1.5164184570,-0.0046005156,1.8511489630,2.0630350113,-1.8583544493,1.2922915220,-1.0781641006,0.3176600039,-1.4096624851,-0.6563232541,-0.1117633730,0.6297587752,1.2154203653,0.2888377905,0.7956738472,0.3510543406,-1.1075544357,2.0214886665,0.9916204810,0.5647026300,0.5993618965,0.5231040120,-0.8116585612,0.6569921374,-1.8315339088,-0.6885490417,0.3351188302,-0.0013942909,0.8267314434,-1.9799656868,-0.6828018427,0.7968899012,-0.2808510363,0.0512608960,0.1743437499,-0.7116479874,-1.9787137508,0.5628860593,-0.6114267111,-0.4011185169,-0.2396005392,1.0169463158,-0.8091300130,-1.1353332996,0.0133572351,1.0202590227,0.2226665467,1.1409810781,-1.5810728073,-0.3684642911,-0.1554839909,-2.3100559711,1.0542657375,1.6269677877,-1.7038469315,-1.2879344225,1.0403683186,2.2401287556,0.1703069061,-0.5179620385,1.0845952034,0.4455762208,0.2167074829,-0.4839757085,0.9151929021,-0.5111671090,-1.0387377739,0.8055750728,-0.5076743960,0.0328205563,-1.5801910162,-0.4398359656,-1.1315128803,0.2544636726,-0.6623916626,-1.1471446753,0.2642677724,-0.1311631501,0.1489771158,0.8975483775,-0.2278074473,0.1106519401,-0.4545907676,0.0902820975,-0.3923905790,-0.7976117730,-0.0126832854,-0.3523421884,-0.1040960327,-0.2254287302,-0.4944473207,0.5934621096,1.1226073503,-2.1177215576,0.6347402334,0.1774308234,0.1942605078,-0.0195572022,-0.1038614362,-1.2880922556,-1.8354381323,-1.0152963400,-1.4097815752,1.7686097622,0.6368556619,-0.1358798593,0.7228752971,-0.9184829593,0.2026240230,-1.6902302504,-0.7515771389,-1.4603588581,0.1438691318,-2.3376917839,0.8028458357,-1.7040096521,0.8609021902,1.7478538752,-1.3853265047,-1.2477948666,-0.5389990807,0.7780463696,1.2358132601,-0.9333853126,-0.4348754883,-1.2053111792,0.2426356375,1.0397968292,-1.2316269875,0.6959838867,-1.7544223070,-0.6464222670,-0.4856535196,-0.3731232882],[-2.0051338673,0.2533751428,-1.3090395927,0.8500843644,0.5513440371,1.5281857252,0.9997274876,0.7820481658,-0.4565566182,-1.1126084328,0.5382915139,-1.1646991968,0.3164305091,-2.3938493729,-1.0044484138,0.4640351832,-1.2155511379,0.1764370948,-0.8172885180,-1.2739762068,-1.3600958586,0.8626067638,-0.9916674495,-1.2624299526,0.9874475598,-0.2145728320,-0.5082772970,-1.6458400488,-0.3702574968,-0.9768534303,0.2904407978,1.3255532980,1.2100210190,0.1227659956,0.2595930994,-0.4679057598,0.4274121821,-1.3154900074,0.1236572117,-0.9854791760,0.5440257788,-0.5402190685,-1.1039479971,-0.9907112122,-1.3060643673,-0.5652656555,0.9776216745,0.2473392487,0.8499594331,0.7057187557,-1.3080376387,1.0187844038,-0.0227408782,0.6088210940,-0.6433678865,0.6544898748,0.9204548597,-2.7137091160,0.7325530052,0.0752336457,0.8366841078,0.9279086590,-1.1596821547,0.3568362594,-0.2196850330,-0.2727072835,0.5928372145,-0.1771852970,-0.2883036137,1.1079210043,-1.0769922733,0.4753883481,0.3450099528,-0.3523810506,1.4889559746,-0.2710278332,0.2396248132,0.6586426497,1.0482380390,0.3379995525,-0.4739660025,1.6513723135,-0.8296175599,-1.3067449331,-0.6117827892,-1.7295495272,-0.9451754689,-0.6887831688,0.4632112086,0.0821003988,-1.1309273243,-0.1603619158,-0.5950310230,0.3923758268,-1.5252598524,0.4210940599,-1.5164136887,-1.2393224239,-1.7814563513,-0.2897252738,-2.1949856281,2.8120102882,-0.8257092834,0.1937599629,1.0681219101,-0.8102371693,1.1713981628,0.3316555321,-1.2564464808,-1.2102553844,-1.8110905886,-0.3118630052,-0.6994720697,-1.5547536612,2.2070877552,-0.5162882209,1.3778606653,-0.7898684740,0.2299599946,0.1487733722,-0.8022645116,0.5625656247,-0.8308770061,-1.2218902111,0.7719486356,1.6793210506,0.6410716176,0.5552347302,-0.6866862774,-2.3685967922,-0.5928680897,-1.2760992050,-2.5390007496,1.2587535381,0.4895009398,0.1038385183,0.5110086799,0.3381683230,0.7548806071,0.1276102215,0.6869242191,1.2325774431,-0.6868656874,-0.8387863636,-0.8901911378,-0.8299873471,0.6570521593,-0.8497098088,-1.9196294546,0.1074195951,0.9909681082,1.0813125372,1.2957479954,0.7512951493,0.6021413803,-1.1488620043,0.6352616549,0.0198759958,0.9322074652,-1.4032728672,-0.5972887278,-1.1572170258,-0.1391252577,-0.0026114394,-1.2078139782,-0.6075099111,-0.1559657305,1.2921199799,-0.6285111904,-0.5723410845,-0.0440929458,2.2640471458,1.1017858982,0.1785074621,-0.4585222602,-1.4878432751,-0.4616599381,2.7842957973,-0.8562133312,-0.8348052502,0.5125629306,-1.6848424673,1.6984417439,0.4297846258,0.6659538746,-1.0364086628,-0.8324087262,0.2216985375,0.5052349567,-0.4887041748,-0.1160030439,0.7263070941,1.2204111814,0.1790825576,1.0658636093,0.2977754474,-0.5876668096,-0.7762431502,-0.5158827305,0.2565769553,0.8809493780,0.8431380987,-0.4027089179,-0.1782636046,-0.1587935835,1.3338432312,-1.4606239796,-2.0378668308,-0.1473344862,1.1285791397,-0.4220150411,1.2866398096,-0.2262062728,-0.6299557686,-0.7837253809,0.7857016325,-0.2874922752,-0.7144582868,-1.5693187714,0.4702760875,1.3465423584,1.0922623873,0.3292343020,-0.3397429287],[-1.1707133055,-0.8345708847,0.1053590998,-0.9551098943,-0.0158422422,-0.6733519435,0.5780305862,0.7031232715,-0.0393282436,-0.9383910894,0.3805969954,0.7196177244,0.2210744172,0.5715443492,0.2615196705,1.4784181118,-0.0361505449,-0.9886692762,-2.5481803417,1.3661826849,-0.3472016156,-1.6215758324,-2.0042810440,0.2201344520,-0.1717495918,0.6369396448,1.3920063972,1.8210952282,-0.6172340512,0.1448362023,-0.4834059179,1.1414402723,1.8423522711,1.4584639072,2.5065975189,-0.4695457816,-0.2602095008,-0.5626283288,1.9504405260,0.4526340365,0.4930631518,0.6368220448,-0.7115779519,0.9143373370,1.0674004555,-1.4705325365,-0.1610876769,0.0818736106,0.1498768330,1.6491193771,0.0072635375,0.7203763723,1.1762765646,0.2638449371,1.9991233349,1.0177111626,0.3286144733,-1.2706656456,-0.1446818113,1.7641743422,0.8927143812,0.4896416962,-0.9942132235,0.9834810495,0.1882795244,0.7997056246,-0.6269438267,-0.7839465141,-0.7389545441,0.7751408219,0.7793362141,0.5096878409,0.2314013690,-1.4001361132,-1.2783879042,0.3915791214,-1.6315677166,1.2537792921,-0.7252321839,0.9186237454,-0.5066030622,0.5364947319,0.3846390545,-0.7608844638,-0.9879293442,1.0536998510,0.1239340007,-0.6024615765,0.7722412348,0.6044166684,-0.1003129780,0.7670496702,-0.5458506942,-1.4519466162,0.2771151066,0.3518697321,0.0249520000,2.6057703495,-0.9137113690,0.5218591094,0.0153903477,2.6444854736,1.3167178631,0.3327669799,0.6593927741,0.2390110046,-1.4895582199,0.6340134144,-1.6390534639,-1.2675498724,1.0433050394,1.2519178391,0.3263645172,-0.1726632714,0.5519032478,-0.1143158376,0.2162200809,0.9341704249,0.3033621013,-0.5463930368,0.8753668070,1.3833848238,0.8291720152,-1.2218199968,0.5906565189,-0.7905145288,-1.7524110079,0.3663987815,2.1609265804,0.8671112657,-0.5826405287,0.1804178357,0.6935610771,-0.6312169433,-0.2545692325,0.1374749988,-1.4491913319,-0.0883082896,-0.0105298012,-1.2318902016,-0.8289853930,-0.3602082729,-0.8703395128,-0.7262350917,-0.6620940566,0.6471897364,0.0936652049,-1.3356847763,-1.9162960052,-0.1310082227,0.8927040696,-1.8789820671,-0.6941149831,-0.8801056743,-2.4179081917,-0.3510309160,1.0250413418,-2.0253155231,-0.2171654552,-0.6943956614,0.4614755809,-0.2940581143,1.0993890762,-0.4676396549,-1.0508223772,1.7287107706,-0.6683145165,-0.1298807859,2.1324386597,0.9666262865,0.8443111181,-0.6013779640,0.7276886106,1.0302677155,-1.2724297047,0.9751001000,-0.1170990020,-0.2573932111,0.0356474034,-0.5298755169,-0.9047068954,-0.1097842306,0.8076342344,-0.3035034239,0.7364196777,-0.1532740593,-1.5517117977,1.3448683023,0.9069454074,-1.1535201073,-1.1836688519,0.0531910323,-0.9787717462,-0.8477154970,1.5395288467,0.1366031617,1.1396311522,2.3474342823,-0.0828050002,0.5587758422,-0.1701462269,0.2563884556,0.4576572180,-0.5936612487,-0.0666432232,-1.2495532036,-1.7759221792,-1.0503594875,1.4247668982,0.3914461732,0.0761216953,0.2842491865,0.6038943529,1.2931584120,-0.2688145936,0.6839090586,-0.0863594636,0.1521898508,-2.1591539383,0.3447727859,-0.4143142998,0.3156183362,0.1544964314,1.3025699854],[-0.0636318251,0.6497617364,0.5397651196,-0.2517716885,-0.9968451858,0.2427535653,-1.6707130671,0.4256668091,0.4201525450,1.6644111872,-0.1637274027,-1.0096265078,-0.7501395345,0.3889355659,-0.3280121088,0.0683555230,-0.2293541282,0.0059699183,-1.2834771872,-1.4719812870,1.0352134705,-0.1490982920,0.2533560693,-1.1749132872,0.5028715730,-0.2666368186,0.7678294182,0.5390011072,0.1143793166,0.0369447507,-2.1100959778,-0.1035841182,0.1734765768,-0.5206263065,-1.5388579369,0.2944219708,-0.5542598367,0.3960934281,-0.5660918951,-0.5781425238,-0.4664564431,0.0804147124,0.4390437007,-0.0616029091,0.7003066540,0.6478360295,-0.2125269622,0.9341881275,-0.9784104824,0.4180209339,-0.6914352179,0.7919614911,-1.1545484066,-0.0029162788,0.8164563775,-2.1450593472,-0.7141282558,-0.1273373812,0.9063172936,0.7292864323,-1.4445300102,-0.5211780071,0.7087510228,1.8131198883,-1.2041488886,1.4940724373,-1.3090559244,1.7167582512,-1.7583614588,0.3108649254,-1.5711164474,-0.6901423931,-1.0158981085,0.4401728809,1.6992042065,1.7186495066,0.5683168769,0.7231294513,0.0740922242,-0.3422453701,-0.4967070222,0.7054743171,1.3211479187,0.2724389136,0.6408790946,-0.9621573091,-0.3153292537,-1.3749207258,0.6791490316,-0.7940768600,-1.0095255375,-1.3924701214,-0.2026451379,0.0677531809,-0.1508983523,0.5414183140,0.7144338489,-0.6705932021,-1.3298588991,-1.3412828445,-0.5634455681,-0.2916662991,0.7729437947,1.5142472982,-0.0278050378,-0.5018460751,0.6177078485,0.0424504094,0.2765903771,0.6403551698,-0.5025407672,0.5923534632,0.6478619576,-1.5444386005,-1.2579747438,1.2809257507,0.6353840828,0.4131743014,0.3158742487,1.1801162958,-0.4564170837,0.4091522694,-1.4798978567,1.2729399204,-0.8004751205,-0.4432139397,0.3686844409,0.7507426739,0.9426909089,1.6492742300,-1.5455055237,-1.2340234518,-0.9009966254,-1.2735397816,-0.0615484640,1.8640985489,-0.3493136168,0.2173888832,-2.4374730587,-0.0383728780,-0.2510394156,-0.7139562964,0.8909270167,1.3366739750,-0.0830413997,2.5130395889,0.0980813131,-0.8486409783,0.4998627305,-0.4094617665,0.5458596945,-0.4862162173,-2.0455002785,1.0393195152,0.1168493107,-1.7165883780,-1.1070877314,0.2518376708,-1.2448687553,0.5589826107,1.7770667076,-1.5959331989,1.5407427549,-0.5897306204,-1.8142468929,0.1698196530,-0.1295103729,0.5277545452,0.2446128577,-0.3235076964,1.2561858892,0.7550396919,0.1513881832,-0.4214838743,0.8148363829,1.4301729202,0.1300896555,0.4603887796,-0.8699390888,-0.0237311311,-1.1238359213,0.1377335340,0.2441793680,-1.7290854454,-0.4972402155,1.3003982306,-0.5680015087,0.5945675373,0.7442512512,-0.2569022775,-0.9850519896,0.2363358736,0.3354244232,-2.0168704987,0.6541719437,0.1009543091,0.0797480121,1.6220499277,1.1278858185,1.5643399954,1.7326484919,0.0972388163,-0.0682537928,0.5457283258,-1.9053533077,1.7635300159,-1.5515018702,-1.2203807831,-0.1801347584,-2.6934776306,0.1715345532,0.6029487848,0.0573038310,2.3573427200,-0.2083336562,-0.4516532719,-0.2417075187,0.4663243592,-0.6326389909,0.1208775267,0.2149387449,1.3439264297,1.2751500607,-0.4505316317],[-0.5756392479,1.4063438177,-1.4878020287,-0.5038126111,0.9840235710,-1.0844315290,-0.2784172595,0.4415511191,-0.1162850186,0.1757821590,0.7411425710,-0.3668819070,-0.7205744386,-0.5024018884,0.2062256783,0.7241781950,1.5902775526,0.0932773799,-0.6890472174,-1.9990137815,-0.4262224734,-1.1738171577,1.1300529242,0.4181618392,-2.3131039143,0.5922805667,-0.3001381159,1.7097831964,-1.1595673561,-2.1293334961,-0.3307192922,1.2602394819,-0.7109816670,1.7607816458,-0.6528120637,1.6467469931,-0.9482585788,-0.8347230554,-0.6540571451,-0.3115817606,-1.6141192913,-1.0731090307,0.0426733084,-1.0998947620,1.0489722490,1.4080376625,-0.3871131837,0.2852624655,1.3540599346,0.5000309348,-0.0193734169,1.1629503965,1.7707066536,-0.7773700953,0.8161155581,-0.6074299812,1.9348500967,-0.7318091393,0.9646677375,0.4912351668,0.5005277395,-0.1886811703,0.6468701959,-0.6115835905,-1.4020545483,-0.2524648607,0.0008449311,-0.3852461576,0.2591911256,-0.2682701349,-1.5723091364,0.3320451081,3.2006881237,0.6155536175,0.8023455739,0.1190806627,0.1179326773,-0.0133221503,-0.1462981254,0.0558719449,0.0060142102,0.7527711987,0.2324405760,0.5915004611,-1.0211442709,0.1468193084,0.9947727323,-0.1153474972,-2.2038846016,-0.3664562106,0.8300547600,0.7540668845,-1.2107092142,0.2661699057,1.3505961895,1.9395216703,-0.1923337281,-0.5798934102,1.1088138819,-0.5727385283,0.6018813848,1.7457513809,-0.7539621592,0.3392607570,-1.8162858486,0.4703977704,-1.6360388994,0.5180320740,0.3072411418,-0.3721626103,1.7841976881,1.7781287432,0.7256065607,-0.5469256043,-1.1149164438,-1.2010357380,-0.0934013724,-0.4959301949,-0.2157954574,-2.6348752975,-0.7417191863,-0.5041836500,-1.9211819172,-0.4793742895,-1.3138118982,-0.6303048134,-0.8648637533,-1.5271095037,1.2077393532,1.3537611961,-1.0298615694,0.2516765296,-1.1857849360,-1.2759015560,-0.1323886216,0.0077520325,-1.1594430208,0.1734642833,-0.8420557976,-0.1126872674,-0.4606709182,-1.1694540977,2.2664411068,0.4393515587,0.8670080304,-0.6363406777,1.0128105879,0.5352993011,1.2980554104,-0.3366172612,0.7446729541,0.0706164017,0.5329166055,1.5777875185,-0.7912862301,-2.4273786545,-0.0999521613,0.4401312470,-0.2123771459,-0.8119220734,-1.1485599279,-0.4177002013,-0.1898192316,-1.5691518784,0.7123882771,-0.4033349156,0.1992595643,-1.0107963085,-0.7487183809,-0.3474726081,0.5490239859,-1.3117270470,0.4529985487,1.2649879456,-1.3282953501,2.2694823742,-1.9726030827,-0.1417467296,2.0489106178,0.6848044991,-1.1864144802,0.0095336009,0.5513510108,0.4967232049,-0.5022209287,0.7636085153,0.7493492961,0.9887427092,1.0534604788,-0.5208474398,1.5930726528,1.1946299076,-1.1940947771,-0.6941130757,-0.2753409147,0.3418467939,-0.3992665112,0.0144357430,-0.1594496369,-0.4563882053,1.4027935266,0.2411439270,0.0072475886,-0.8765354753,0.4561424851,-0.0980236828,-1.0085912943,-1.3533272743,-0.8581139445,0.8061968088,-1.0639472008,-0.1058177575,0.4276168644,-0.8239603639,1.8089847565,-1.5798636675,-0.0302213319,-0.3193961382,0.5277996659,-1.4658607244,0.5939242244,-0.8696262836,0.3683566749,0.4510522187],[0.5968563557,-1.1456565857,-0.0630941987,-0.0725090280,-0.6521866322,1.9407103062,-1.9997001886,-0.4733425677,-0.6821983457,-2.0067911148,-0.1969090551,0.0117811309,0.8634021282,0.5490431786,0.5036259890,0.8777194619,-0.5009964108,-0.2476846427,0.7691996694,-0.4389582276,0.7626179457,-0.9946994185,-0.9002199769,-0.2358198315,0.5558941364,-0.5835582614,0.0424835086,0.5476747155,0.4497909844,2.0062038898,0.5630492568,-0.4302353561,-0.3927549124,0.6127817035,-0.3549419045,-0.0465914086,0.7972400188,-0.0291995928,-0.1873065531,-0.2666181028,-0.5908914208,0.4490607083,0.4655361474,1.3770449162,0.3744198084,2.0639455318,-0.5035088062,2.2640516758,0.8200385571,0.3762246370,0.1349862367,-0.7485954762,-0.6919678450,-1.5081450939,-0.8550764918,-0.3346778750,-0.2704392374,0.7448489666,0.6368254423,1.1909401417,-0.3652415872,0.0266696922,0.1857023090,-2.1289024353,-0.0612011552,-0.1647111028,1.5982892513,0.3783888221,-0.8879693747,0.1596832424,0.9702088833,-0.4322770238,0.7622994184,-0.9912603498,-0.0206081625,-1.8675857782,1.3297318220,-0.2855650485,0.4054629803,-0.6809203625,-0.5133872032,-1.8930710554,-0.0781741962,-0.4591271281,1.4383069277,0.9537320733,-0.9098660350,-0.5606174469,-1.1290600300,1.0253957510,-1.3523454666,0.3395365477,0.7096471786,-0.4674813449,-1.6451238394,0.8107517362,1.7173854113,-0.0553687960,0.3889346719,-1.2086811066,-0.0176033434,1.2727808952,-2.3618505001,-0.0911023468,-0.6099271774,1.4351190329,-0.2916943431,0.2546273768,1.4699568748,-0.0347926132,1.3543151617,0.0582686812,0.7119902968,1.3041653633,-0.2682042718,-0.3871650100,1.2333360910,-1.1758619547,-1.5743108988,0.2008384019,1.6558495760,1.3954097033,0.2431987077,0.6873512268,-1.5398448706,-2.1012260914,-0.8874242306,1.0868637562,-0.8933115005,-2.3817610741,-0.2322047204,0.5744912624,1.1864047050,2.0981147289,-1.0142970085,-0.2725763023,1.1405632496,-0.3188109994,-1.6895825863,-0.9098422527,-0.9992399812,0.2970742285,0.4739212394,-0.3946261406,0.9657480121,0.1714322120,-0.4725795090,0.0793206245,-1.2109538317,-0.9714989066,-1.4903097153,0.5481170416,-1.3274215460,0.1033664346,0.3240928352,-0.0927641466,-0.7859684229,-0.4100660086,0.1349842697,-0.6123028398,-0.4821889997,-0.0113635045,-0.6305410862,-0.6447528601,0.9588113427,2.0857424736,1.6361919641,-0.0809310004,0.4175556898,0.5576475263,1.1547952890,-0.9088178873,-1.3321971893,-0.6993666887,0.8384341002,-0.0056715696,0.0963145494,-2.0406420231,-0.5699337721,0.2109227180,0.1758637279,-0.1132779494,-0.1388639659,-0.5674633980,-1.0927742720,0.4664836824,1.3453660011,0.9642984271,0.1735391319,-0.6066111326,-1.8372893333,0.0989775285,-1.0595743656,-1.7484507561,1.3843749762,0.0083545428,0.3311144710,-0.2330202758,-0.4341944754,-0.3969227970,-0.7791808248,1.1586873531,-0.7870743871,-1.3215303421,0.5297895074,-0.5042585731,-1.6013978720,0.6434103251,-1.1106889248,1.0323313475,0.0253233518,-0.5326119661,-0.7091501355,1.1146080494,0.3246107399,0.4328122735,0.1061507687,0.9968009591,-1.5700018406,0.9111320972,-0.1880164742,0.0720187649,-0.1722674072,-1.2956403494],[0.1623233408,0.9533376098,0.8453432918,1.0247024298,0.8314163089,0.2169153392,-1.1605566740,0.6202650666,0.7922874093,-1.1443600655,-0.1555470228,0.2054934651,-0.0696882978,1.5672497749,-0.6392518878,0.6004613042,0.2808816433,0.9697634578,1.3994048834,-0.9369864464,-1.0598493814,0.6915630698,0.0257297009,0.5673299432,-1.9622377157,-0.7675710917,0.3625481427,0.7994940281,-0.0520695336,0.0688188747,0.7390265465,0.2610666752,-1.5965384245,0.3458299041,-0.7521305680,-1.3208951950,0.5133634806,0.6769243479,0.8414777517,1.7870870829,0.5016800761,0.6279888749,-0.0259170141,0.0858539417,-0.1149455160,-0.9918748140,1.3966687918,0.5319203734,0.0872511491,1.2238022089,-0.6457444429,0.6148810387,-0.2943848968,-0.7676346898,-1.9957956076,0.0006206976,-0.4282977879,-0.3669142425,-0.2334992588,-0.1813053340,2.3843736649,0.0274813995,-0.2801706195,0.0149513474,-0.2649163604,0.2022246122,-0.3847317100,-1.1576153040,1.6478676796,0.5820747018,-0.2079120427,-0.1404580623,0.9750667810,-0.6656797528,-1.0515507460,-0.5181403160,-0.0723514110,0.9246936440,1.0558516979,-0.2617908716,0.0100078853,1.4384540319,-1.5206338167,0.7696095109,-0.7830299139,-0.6658165455,0.8587329984,0.0302361734,0.7325266004,0.6763762832,-0.9670042396,0.5436664820,-0.0928966329,0.2728548050,0.7476511598,0.8182846308,0.0401502326,0.0703607574,0.7100747228,-1.1809411049,0.1525679231,-1.4360591173,-0.9362903833,0.6512863636,2.5559821129,-0.4898336828,-0.5570479035,0.3082244098,-0.1268011928,-0.1690266579,1.1079031229,0.5577710867,1.2079291344,0.3057589829,1.4704600573,-1.5299013853,-0.4372470081,-0.1838792264,0.4103325903,-0.0079925116,-1.4350907803,3.0050628185,1.9440249205,-0.5308479667,0.7670062780,-0.1695310771,-0.4937400520,-1.0065364838,-0.3335910738,0.2164574564,2.0567500591,1.0385917425,-0.9172860384,0.1360678822,-1.4758296013,0.2109170258,-0.6283934116,0.4619432986,1.2396088839,-1.2901101112,-0.4787948728,-0.0482119285,-1.3737345934,0.3587618470,-0.1970571727,-1.1884554625,-0.3847505152,-0.4059247375,-0.8413720131,-0.2346312255,-0.5250424743,-0.8797713518,-2.2436685562,-0.1204453632,-0.9047447443,0.1623364985,-0.1249333546,-0.4604585767,-0.1806154251,-1.5818452835,0.7564077377,-0.3526729941,-0.5011270642,-0.4888699055,0.2328237146,-0.8190051317,-0.8744320869,-1.6677303314,1.3656442165,1.3829454184,-0.1329779178,0.6396308541,-1.6524884701,0.8636744618,-0.6694845557,0.8651663661,-0.9834919572,0.9883199930,-1.2616648674,0.2136829346,-1.4948030710,-0.5576620698,-2.1199064255,-2.6051251888,0.8021505475,0.2092306316,0.8647927046,-1.0634721518,-1.7716604471,0.1749333441,-1.3047918081,0.8585357666,-0.0653507933,0.8114531040,-0.8026093245,-0.2348457277,-0.1972251832,-0.1731291860,-0.9763115644,-0.4425777495,0.1876048744,0.2390181720,0.5495560765,-0.4430482984,0.5191142559,0.3313212991,-0.2732692659,-0.1305333972,-0.3552298546,-1.2423542738,-0.4154689610,-0.5192009211,-1.0146884918,0.0872632042,1.0132828951,0.7355144620,-0.8626582026,-0.9035837650,-0.1659100503,-0.4304459989,0.8710014820,0.2093449235,1.7577981949,-1.4495869875],[0.3389512897,0.3277698159,-1.3833289146,-1.0309791565,0.5938034654,0.3381133080,-0.7506350279,0.5864959955,0.6818603873,0.9190769792,-0.9943068027,0.7560220361,-0.2823075950,0.4619690776,-1.0569438934,-1.6896722317,2.2968416214,0.1863039136,-0.4671124518,-1.7164677382,-0.0640422776,-0.9713795185,0.9030221105,-1.0318793058,0.7022774220,-2.5937387943,0.9347560406,-0.0407111496,-0.0842949972,2.1857056618,0.8122920990,-0.1597754359,-1.2797119617,-2.5321006775,-0.9836844802,-0.3681376874,1.4671288729,-2.6063854694,0.1413185447,1.1272912025,-0.3143338859,0.2970275581,-0.9995012879,0.3840419352,-0.8541126847,0.1262938529,0.8874447942,3.3216011524,-0.0434766375,-0.2402396351,-1.7334960699,-1.1922539473,0.0395632051,0.3123318255,1.5156829357,0.1709397286,0.4083366990,-0.2023052424,1.5522542000,-0.4802343547,1.6968504190,1.4456083775,0.2222046554,-1.3635166883,0.4523965716,-0.1486860216,-1.3813056946,0.2926163673,-1.4815069437,0.0460004359,0.0685515702,0.8657517433,-0.4926210642,0.3616339266,0.2384285331,-0.5062540174,0.9793159366,-0.3106910288,1.4108459949,1.4620524645,0.2490852475,1.3678778410,-0.6981250644,1.0042148829,0.9979116917,0.4013634026,-0.5294713378,-0.9604782462,0.7616765499,1.0867910385,2.4451165199,0.5009768009,0.5941057801,2.1568233967,-0.2864334583,-0.7203481793,0.2201959938,0.0211936776,0.2499561757,-0.5913699269,0.4207766056,0.5149331689,-2.2674925327,-1.3170949221,0.0806484967,1.6441719532,-1.6811925173,-0.7950924039,-0.5914666057,-0.4738128781,1.3944123983,-1.1955771446,-1.4249742031,0.6435713768,-1.5451512337,-0.0101473620,2.2087171078,-0.9800808430,-1.4776380062,0.7782774568,-0.7789295912,-0.0102034928,-0.0880911723,-0.4840727150,0.7514572740,0.0549944900,-2.4430112839,2.1914079189,-1.0618168116,1.4040321112,-0.2890518904,-1.8754194975,-0.4907815754,-0.0630276427,-0.7582322359,-1.1303482056,-0.7832784057,-0.5160414577,0.0700457245,0.6622155905,1.1924040318,1.4414311647,0.6065171361,-0.4135439694,-0.1386700422,-1.0272814035,0.4866032004,0.1530758739,-0.3026266694,0.2846779227,0.0424417257,0.5401003361,-0.7115367651,1.5831142664,-0.2675635815,-0.0213743374,-0.0043467307,-1.5642577410,0.9680214524,-1.3408775330,0.1488857120,0.2985931039,-0.5775466561,-0.7951164246,-0.4007186294,-0.5162972808,0.4733025432,0.2995747328,0.8256702423,1.2272002697,-0.0659088045,0.4785670638,-1.1270323992,-0.5993215442,-1.5867478848,0.1878347248,-1.2973229885,2.1392738819,-0.5237012506,0.4061898589,0.1321331561,0.1819093823,0.3166199625,0.1830487847,1.1566855907,-0.9360304475,-0.1669008881,-0.5197641850,0.7517486811,0.0689731315,-0.3164053559,0.2283985317,-0.4720411599,0.3385706842,0.4731729925,0.4821418226,-0.7884815335,0.3731731176,-0.7758998275,0.7502189875,1.0179754496,-0.2132595032,-0.6256816387,0.6563635468,-2.0351250172,-1.6866027117,0.7681083083,-1.0538374186,0.0967316404,-1.3432598114,-0.2516896725,1.4454326630,0.1537240297,0.7852683067,0.1986494809,-0.2295962125,0.3377442062,0.9981122017,0.7033475637,0.8179015517,2.8854703903,0.1449834108,1.5653721094,-0.8830209970],[0.2447814792,-1.6657313108,-1.0418515205,-0.0914706290,-2.2588729858,0.2607210279,-0.6904564500,-0.2766918540,0.4198768735,0.6650484800,1.2362228632,0.2318393737,-1.7664381266,0.2203081250,-0.6439874768,1.3943929672,-0.2212222815,0.5396524668,-0.9326766729,-1.1483886242,-0.5559464097,-2.1605238914,-1.5237659216,-0.8512112498,1.4548479319,1.4710714817,-0.0255904552,-0.0392868333,0.8066657782,0.2605836093,-0.0717089623,-0.4921136498,-0.7088971138,1.3106839657,-0.7427183390,0.5950650573,0.4102731943,-0.7450366020,1.2448486090,0.8819051385,1.2929413319,0.7543784380,1.5012474060,0.5012357831,1.6025744677,-0.1244885102,-0.7437287569,0.7625656128,1.5684475899,-0.8715553284,1.5655452013,-0.4011086524,0.3054696023,1.0253591537,-0.6708995700,0.3336648345,-0.7808531523,0.2032589763,0.4629889429,0.8305130005,1.3792150021,1.0666061640,-0.5386751294,1.8384336233,1.6489225626,-0.5408121943,1.1491128206,0.3872258961,0.7378184795,0.5292190313,0.1566731632,-0.8639983535,-0.4543217719,-1.1172450781,-0.1373617202,0.0493386127,-0.6242133379,-0.0747201517,-0.0418696329,0.1286173165,-0.2732076347,-1.2780323029,0.0635503307,-0.4641813934,-0.2046342939,-1.7704610825,-0.6292266846,-1.5954760313,-0.6125565171,1.3553122282,-0.6214655042,0.3244607449,-0.1392270476,-1.7893319130,-1.6337937117,1.2505484819,-0.3817860782,1.0203552246,-1.7707964182,1.5794938803,0.9360316396,0.6239099503,-0.5742901564,0.3519190848,1.6146284342,0.4477892816,-0.3142979741,0.4781792164,0.6606804729,-0.0335218385,-0.5427355766,-2.5328576565,2.1415202618,-1.8951349258,-0.2973324358,1.1050219536,1.2794615030,-1.4703121185,-0.4997144043,-1.1340399981,0.5188848376,1.5461864471,0.6832302809,-1.0181527138,-1.8280473948,0.0570660420,0.2686289847,-1.2347159386,0.6066080928,0.8530008197,0.3568423390,-0.3545398414,0.3357949257,-1.1221921444,0.1614009887,-1.2751108408,0.1513240039,0.6052228808,-0.0645731688,0.3507217765,-0.3744005859,-2.7758972645,-1.1287082434,-0.0116145117,1.3980032206,-0.0176374651,-0.1791180372,-0.4948559999,-0.4348399043,0.9603962302,-0.9801307917,-0.6622002721,-0.0265429523,1.6936491728,-0.0890061557,-0.1063767895,-0.6456767321,-0.6210477948,-2.4629504681,0.5300250649,-1.1274645329,0.1202629134,0.7539504766,-1.6694158316,0.8432223797,-0.6346231103,0.5511178374,-1.6847029924,-0.3897745609,-0.1799215525,-0.3502997756,0.7974809408,0.7187066674,0.5191764832,-0.1159625053,0.9157214761,-2.1128296852,0.5010185242,-0.4072211087,-1.2765816450,1.9876989126,1.3443856239,0.2349775285,0.6620453000,1.3393974304,-1.2028733492,-1.1018798351,-0.2779258788,-0.5651109219,0.4557473958,0.4294156134,0.3529987633,0.7264522910,1.4002771378,-1.0009080172,0.0965322256,0.9219058752,0.3410715461,-0.7426944971,-1.0091520548,1.4181830883,1.5620485544,0.4852831960,0.4569746554,-1.5205644369,0.8606964350,-0.0159703139,-0.1952470392,-1.7966439724,1.2196379900,-0.2568109930,0.1493610740,0.1471642554,-0.1334927678,-0.0951507837,0.8027544022,1.8461946249,0.5527061224,1.3992179632,-0.4277961552,1.2142347097,0.0890079811,1.1724892855,0.8421200514],[-2.0531296730,-0.3815230131,-1.2456768751,-0.2631421089,0.3498506546,3.3013672829,-0.3395608068,0.9844453931,0.7795133591,-0.3801021874,0.4742499292,-0.5515266657,0.3347168565,0.4604980946,0.8324033022,-0.0461973809,-0.1935420334,-1.0834821463,-0.5094074011,1.4313070774,1.1016554832,-0.2138853818,0.4686948061,1.3630285263,-0.5100106001,-0.1556339264,0.3429749906,-0.3238503933,-1.0426410437,0.1445600837,0.1049434692,-0.2222164124,0.4424103796,-0.1254242659,0.5358566642,1.5508053303,0.6763385534,0.6407976747,0.1865347624,-1.2624466419,-0.4919446707,1.0839577913,0.8675064445,0.2160370052,1.4639556408,1.1394736767,-0.7107564807,0.0310300067,0.4451051354,1.2646448612,0.6961842179,0.3952502310,0.8698752522,-0.8570802212,0.2444748431,-0.3380377889,-0.3063204885,-0.6506505609,1.0126060247,-1.2486658096,-0.4531163275,1.3865495920,0.8400343060,0.8887103796,0.1331875622,-0.4949502647,0.2909305990,1.3711076975,-4.4466323853,-0.3461068869,-0.4214496017,-0.3191107810,1.4321105480,0.5082851648,-0.4887852967,-0.2842614055,-0.4167383611,-0.5153347254,0.1215999797,-0.9909867048,0.2912752628,-0.0309490580,0.0686672628,0.3116374612,-0.9446466565,-1.4318141937,-2.3070585728,0.1934504807,0.0642832592,1.6173255444,-1.4001069069,0.9930692315,0.4986160100,-2.1691753864,0.3272755146,-0.5290307999,0.3029612601,0.7106252313,-0.5628217459,0.3532442451,-1.2171857357,1.0224491358,0.0653357655,1.4942281246,-0.9784055352,0.9602708220,-0.2665057480,0.4682205617,-0.6420611143,-0.7896304131,-0.2709776163,0.8798081875,-0.2442070842,0.1087937206,-0.3002286255,-0.5277259946,0.4140106440,-0.2964256108,1.4545035362,0.4128693938,-0.8971816301,1.0674988031,0.1346273124,-1.0002290010,-1.5374671221,-3.4322154522,-2.1081695557,1.6781947613,0.0430080853,-0.2318437546,0.4126449823,1.4264055490,-0.5483173728,1.1825624704,0.0128414826,0.3367869854,-1.4143692255,0.1019542515,-1.7843068838,0.2959744334,0.6726202965,-0.4266151190,0.1067341939,0.2738974392,-1.5534147024,-0.8846866488,0.4381192923,0.8158077002,-0.0652576610,-1.3206679821,0.6635075212,2.6380484104,-0.6187989116,-1.0136746168,1.8584141731,0.6533104777,-0.4259442091,-0.1774014086,1.4402084351,-0.1898958385,-1.1205271482,-0.5485401154,0.1698873043,0.0687155649,0.0311105829,0.3088907599,0.3787349463,0.3809392750,0.0914585218,-1.4571095705,0.9450003505,-0.7444524765,-0.7551084161,0.9248359203,-0.2947985828,0.9806703329,0.6765164137,-0.4960588217,0.8475109935,0.7173162103,0.7837095261,0.4317607582,1.0102860928,2.1032080650,0.1576048732,0.0173003376,0.7079991102,0.6061809063,-0.6047534943,-0.1859838217,-0.9648866057,0.3602762520,1.5325679779,0.0651429519,1.0650039911,2.5460152626,-0.6730990410,0.2153566033,0.9968594313,-0.4184798002,0.3913473785,0.2801388502,-0.1887837499,1.6170681715,0.3825345039,-1.9914906025,0.4941851795,-0.1115392447,0.0824190080,0.2398149073,0.7439398170,0.2233340293,-0.7043433785,2.8407576084,0.3171818852,-0.0780908912,1.0117206573,-1.3970212936,0.2704565227,1.1053961515,-0.8552007079,0.1994792670,-0.7967630029,-1.7464600801],[-1.4587714672,-0.9694950581,0.4347965121,0.7123994231,2.5555636883,0.2164577097,0.4949950576,0.8548553586,0.5338478088,1.8773450851,1.8840982914,-0.6057417393,-0.2921206653,0.3764616251,-0.8626120687,0.6312596798,0.1972151399,0.8950186372,2.1432530880,-0.5009615421,-0.0870208442,0.2992241681,0.1273721606,-0.5634948611,0.8877972960,-2.2128298283,0.4280473888,-0.2503516078,-0.0595523603,-0.0990573093,-1.3535153866,-1.0699493885,0.9600315690,0.3506530225,-0.0972179323,0.6288717389,-1.3509629965,0.1111599132,0.2762899995,0.8108143210,-0.1749507338,0.6180928946,0.0537807047,0.4083303213,0.7938087583,1.3735338449,0.1129539311,0.7087584138,0.2543578446,-0.7406477928,1.1616770029,-0.4258893132,0.5054414868,-0.4079273939,-0.8973106742,1.5773397684,0.0967438072,2.5411305428,1.1561323404,-0.3894842565,-1.9563227892,-0.3720968068,-0.4485727549,1.4407653809,2.2971429825,-0.1074938774,-0.1706637144,0.2393594086,-1.2897298336,-0.2838257849,1.4366940260,0.5021362901,-1.3120448589,1.3362712860,-0.9349203110,-0.1795876771,0.2796979547,0.3275337815,-1.5682973862,-0.1581210196,-0.4314252436,0.5257216692,0.5361158252,0.9633139372,0.0385950543,-0.2250328809,0.0394976810,-0.6151430607,-0.4402093291,-1.7778775692,-1.0987372398,-1.1117931604,-1.4587020874,-0.5599415898,-0.8956044912,0.5017939210,-0.3279520869,0.1389401406,0.3314218521,0.9030095935,-1.1500355005,-1.9140057564,-0.4541152418,1.1774168015,0.3544850051,1.1513677835,-1.0737146139,-0.2920063734,-1.4028655291,1.5057044029,0.3650828898,-0.8197543621,-0.6998680234,0.3804240525,0.4712976813,0.5005731583,0.2035487741,0.9716621041,0.5432396531,1.3573894501,-1.0096626282,1.6208111048,-0.5277240276,0.8372216225,1.8990112543,0.0555527322,-0.7414680123,-1.0119155645,-0.1866538525,0.8219956756,0.2626342177,-0.0517387129,1.0967673063,0.4631964564,-2.2763576508,-0.9005764127,0.4502929151,1.2675552368,1.4029026031,0.0410192646,0.4613499939,0.8826645017,-1.8186287880,-0.7178622484,-0.1207559109,1.4767705202,-0.0755815879,0.7688866854,0.4810449481,0.6946743131,-1.5904605389,-0.7692117095,-1.4830653667,-2.5812559128,-0.9567979574,-1.4820371866,-0.6474533081,-0.4920964837,0.6667062640,-0.0434637107,-0.4990424216,1.0992600918,0.7525292039,-1.6032221317,0.5581198335,1.3076291084,-1.6271916628,-0.9884853959,-0.3797832131,-1.3973912001,-0.3615023196,-1.9054090977,-1.2950693369,-0.2564415634,-0.4667300582,-1.2958893776,-0.5529566407,0.0508099534,-0.5288753510,-0.7055816650,1.2113600969,0.3943111897,0.2387282550,-1.2374818325,0.2701815963,-0.3978709877,-1.1572402716,3.0878937244,0.8014529943,1.8457676172,-0.0689089596,0.9194198847,2.2617175579,0.6498615742,-0.7515003085,-1.0202875137,-0.3537638783,0.3832994401,3.3604583740,1.0071740150,-1.1935818195,0.2676483095,0.3094398081,1.2086896896,-1.2866652012,0.7168079019,0.2362437248,-0.5765281320,0.7539612055,1.2162506580,0.2970718741,-1.2590700388,-1.1305162907,0.6088635325,0.4285758138,0.8790427446,1.0297600031,-0.0150263263,0.5672517419,1.9943733215,-0.2904198468,2.2879085541,-0.6683076024,-0.9014266133],[1.1198561192,-0.3908662796,-0.7119098902,-0.1672258079,-1.8771710396,1.4835146666,1.5566693544,-0.5456945300,-0.3992888927,-1.8576182127,-0.0591329224,1.1905106306,-0.7533163428,1.3647427559,1.4910933971,-0.8707145452,-1.2028700113,0.1947313249,1.4388641119,-0.7994881272,-0.1219984218,1.3156124353,0.4465557039,-2.8038535118,-1.6329493523,0.4930388033,1.1889687777,0.9552270770,-0.8060735464,-1.2271238565,1.5816148520,1.0718077421,0.0845048428,1.0317893028,0.3432446420,0.4551599324,0.2916109264,-1.2060781717,-1.1404657364,0.1325116307,-1.1530866623,1.4436823130,1.3028328419,0.4774630666,-2.2254891396,-0.4252972901,-0.3897869885,0.3375076354,0.8366788030,-1.5183181763,-0.3942394257,0.3418150246,-0.9899711609,1.7337936163,-0.6356321573,-1.4572966099,1.1070363522,-0.3804219961,1.2175976038,0.1318281144,0.3227569163,-0.3193053603,1.1206564903,-0.9782357216,2.5470190048,-0.0272714552,1.3424437046,1.3656324148,0.1930369586,0.4632759094,-1.6178817749,0.5364732146,-0.5179991126,0.8289573789,-2.2828421593,-0.7208089232,0.5070265532,0.0186515842,0.0308151282,-0.5973554254,-0.3513200283,-0.2779895365,3.2370834351,0.0105593828,-1.0757038593,0.9850459695,-0.0400293693,-0.2541744411,0.2447330952,-0.1110943928,1.3795638084,-0.2299806774,1.0006225109,0.5272231102,0.8905154467,2.0872952938,0.6532689929,-0.3763174713,0.6781877279,0.3656844199,0.9306739569,0.2416982949,-0.3654942513,1.8575373888,0.1327817738,1.4138320684,-0.7374867797,-0.6212695837,-1.1035248041,-1.7859196663,-0.3236762583,0.2797504067,-1.7126622200,0.3749800622,-0.3638271987,-0.2331794649,0.4900467694,-0.1618707776,-1.4507075548,0.4063182771,-0.1040311977,0.7966028452,-0.3308908939,2.0540111065,-0.8186672330,0.0951378867,0.3161187470,0.3237067461,-0.2970733643,0.0833072290,1.5167295933,0.5328765512,0.3028484285,-1.4206554890,-0.4329741299,1.3647885323,-0.0198237337,1.1204465628,-2.4348003864,0.6366173625,-0.3564464450,0.6139984727,0.0905427411,-0.0487639718,-0.7035763860,0.3491133153,-1.6848409176,0.2357283682,1.1717361212,-0.8481423855,-0.2195025980,0.3186892569,0.2741073966,0.3729113638,0.2715400457,-0.4458433986,-0.3774275780,0.6609749198,-1.0435019732,-1.1683075428,0.2231950015,-0.1027064696,0.5333796144,-0.4214748740,-0.8696775436,-0.5515634418,-0.0729684234,-0.3588608801,-0.7902694345,0.2904454768,1.5837885141,-1.0763326883,0.0376655497,-1.5118037462,0.0069348137,-0.3564294577,0.6037105322,0.0964799896,1.5362436771,0.2995141745,1.2214409113,0.7883645296,-0.0879966095,0.6117028594,-0.3020986915,-1.8245121241,-0.2724688649,-0.1164358258,-0.5263771415,0.2055741847,-1.6351301670,-0.1721231043,0.4942176640,-0.6478079557,-0.7820081115,1.2534043789,-0.9469304085,2.0706932545,0.6533148289,0.5774577856,-1.9339703321,0.7606521249,1.7944775820,-1.0832855701,0.7078484893,-1.5409542322,-1.5427272320,-0.6692368388,1.0229578018,1.4193599224,-0.9133544564,0.4840072691,-0.9753501415,0.2186499238,0.1535844058,-1.3288750648,0.3165779114,-0.8043466806,-0.7737863064,-0.0225233603,-2.1306989193,-0.0007973393,-0.1462996751,0.3758448362],[-0.5482758284,-0.2904385328,-0.2203367203,-0.1360739022,0.9308471084,0.5719411969,-0.8307803869,1.6780029535,-1.1897013187,-1.1363524199,0.5443426371,0.6116832495,0.9746663570,-0.3817377985,1.8900529146,-0.7435458899,2.4997756481,0.2572008967,-0.1710775048,1.5151948929,0.4258111119,0.7934648991,0.6231244206,-0.4840977490,-0.9201597571,-0.4832959473,1.1362259388,-0.7039580345,0.8110657930,0.6199864149,-1.2173436880,0.7433096766,-1.2307972908,0.8163778782,1.2898740768,-0.9440120459,0.7128977180,0.3834163845,-0.2448080033,-0.4208971560,1.0776454210,-0.9177082777,-1.3809242249,-0.0549273714,0.4324325323,0.8900623322,-0.1035629958,0.6625125408,-1.1518876553,0.2836413383,1.2821938992,-0.2739796638,0.4010577202,-0.3207268715,1.7261518240,0.5400056243,0.1708711684,0.6841276288,0.5838326216,-1.1475074291,0.9617252946,-0.5272442102,-0.3061755896,-0.2802444696,-1.2695152760,-1.6422095299,0.3058715761,-0.0128703928,-2.6293754578,-0.3578370512,-0.4003086686,0.9074838758,0.9345986843,0.9679060578,0.3018855453,0.0361836366,0.0973763913,-1.6076494455,-1.4042066336,-0.7118547559,0.5561019182,-0.0364919156,-1.5172846317,2.4007532597,-0.1681730747,-0.9137525558,-1.4355233908,0.7524472475,2.1874260902,0.5050536990,-0.4081895053,-0.2869866192,-0.7814391255,-0.9182626009,2.3791458607,-1.1824381351,0.0357667170,-1.0054905415,0.2820036411,-0.2671211064,-0.1642470062,1.4117362499,-0.2460403889,-0.7910364866,0.9709326029,1.2122645378,0.3736355901,0.9546735287,-0.5189890862,0.8345152140,0.1865234822,0.9267693162,-0.7936452031,-0.1931727082,-1.2181763649,-1.1549400091,-0.3107994795,0.3828459978,-2.1461999416,0.6490299702,2.1667606831,0.5197604299,-1.0844227076,-0.5551661849,0.5941517949,0.8355113268,0.0238041356,-0.1456642449,-1.7534958124,1.0213720798,-2.0443327427,-0.8794829249,-0.8769536018,-0.2262274176,2.2516703606,1.2893469334,-1.1498097181,-1.9233324528,0.4795956910,-1.0476869345,-0.1603702307,-1.3869994879,-0.4466053545,-0.1670096517,-1.2587105036,-0.6671476960,-0.1938714534,1.3857107162,0.3265101612,0.3107939661,2.0093312263,0.8311404586,-0.3633439541,0.9202536345,1.6982054710,1.0785430670,-1.1998401880,1.0437088013,0.9129307866,-0.1036761850,0.9082722664,-0.1097550467,-1.1004810333,-1.0039086342,-0.5625741482,-0.7018459439,1.0801191330,0.2250289023,0.6520404220,-0.2414115965,-1.3095985651,-0.2076982260,0.1262160689,-0.5799643993,0.1192129403,-1.1732382774,0.9104181528,-1.1093168259,1.3726590872,0.5925967097,-0.5359659195,0.7879594564,-0.5177975297,-0.1359538883,0.8724327087,1.2267098427,-0.2726623714,-1.2737469673,0.3153249323,-0.3683904409,-0.3502188623,-0.2695860863,-0.3834236562,-0.7484129071,-1.8965317011,0.4718063474,0.8372075558,-1.8106716871,-2.5100286007,1.0228426456,-0.2571284175,0.4561365843,0.1014745757,-1.9939613342,1.5270358324,2.7336032391,0.5241632462,0.0706301928,1.5364835262,0.4563824534,-0.2476233393,0.9482578039,0.4184074104,0.4652556181,-0.4185591042,-1.7924579382,-0.0816290453,0.1231039241,-2.1013731956,0.3075747490,0.9703420401,-1.1419631243,-0.2391041815,0.1807586849],[1.3525646925,-2.1389710903,-1.8200509548,1.4221153259,-0.8525925279,-0.1797548085,-0.8431107998,-0.9901415706,1.7518792152,-0.6028918028,1.2872824669,-0.2191602588,0.9963245392,-1.0853056908,0.5814145207,-0.2415531278,-0.2949104905,-0.1386898309,0.9096865058,-0.2297956049,-0.5122372508,0.6257262230,-0.5922887325,-0.7369153500,-0.0611394197,-0.4971470535,0.4321023226,0.9778766632,0.1208294109,0.2494894415,0.0673965141,-0.1767411530,0.9871430397,0.2811503112,1.9028134346,-0.2427451164,-0.9117431641,-1.3278282881,0.0232584253,-1.6634082794,0.5697232485,-0.7888104916,0.2057036310,0.3429163396,-0.7559013367,0.9190707803,1.5341836214,1.0631592274,-0.7089481354,1.8540906906,0.6824290156,-0.8776482344,-0.9802616835,2.3940370083,1.6527678967,2.0181610584,0.0669684559,2.7364752293,-1.0129966736,0.2716617286,-0.1089972779,-0.0572592542,-1.0589312315,-0.3265284300,0.3300458789,-0.0004799794,0.8181158900,0.4282137156,-2.5039472580,0.1204811335,0.8078928590,0.6021209955,-0.8651900291,-0.1533203572,-0.2404905558,-0.0607633479,0.5328661203,0.7644009590,-0.8560833335,-1.6823405027,0.6530694366,0.9302550554,0.4051826000,0.2040024847,-0.8227655888,-1.2768743038,-0.2264368683,-1.0160527229,-0.6572948098,0.2939909101,0.6200525165,0.7796336412,0.4549840093,-1.3034356833,-0.5644497275,-0.6971324086,1.0378397703,1.5601544380,-1.4602061510,-1.9396758080,0.5265857577,1.6603387594,-0.3638090491,-0.0310451910,-0.3636223972,1.2418016195,-1.6686720848,0.6132101417,0.0434691086,0.1165482178,-0.5786048174,-1.2646787167,-0.1174873859,-0.4851864874,1.0230669975,0.1483554542,-0.0294206236,0.8758327961,0.1348077953,0.0840644166,0.8931159973,-0.1384363919,-0.2515294552,0.5490271449,1.6555695534,-1.3272736073,-0.1325270534,-2.0754849911,0.6027877927,0.4861270487,2.0449700356,-0.6894755363,0.3789910376,0.3894831240,1.2121812105,0.1136562079,-1.0378799438,1.5067687035,-0.4767748713,1.2916351557,1.2961734533,1.1144324541,0.1773619503,-0.2875421047,1.4568520784,0.8501911163,-0.3830829263,-0.6213039756,2.2090291977,1.0792033672,1.1268278360,0.1464464664,-1.0442385674,0.3112875223,0.9527289867,-0.1369006485,0.1579798460,0.9468470812,1.1619044542,0.1117127314,-1.2547981739,0.0041935598,-1.8870812654,1.8109247684,1.2467620373,-1.1706187725,-0.8198552132,-1.0388509035,-0.3795161247,0.2471856773,0.0978375450,-0.3687669337,1.4281040430,-1.2668262720,-0.7314748168,-0.2361954153,-0.0640455484,1.5850062370,0.5684098601,-1.0112850666,-1.1607600451,1.6386686563,0.0010163832,0.6599608064,-0.5040879846,1.7607916594,-0.2987151444,-0.0699679181,-0.2390695810,0.0480869710,-0.0561540872,-1.4012265205,-0.4074691832,1.0456372499,-2.2115867138,0.1003632694,-1.1718195677,1.9943586588,-0.7404477596,-1.8472335339,-0.3663715124,1.6781787872,-0.3644346595,0.2316042483,-0.6349893808,-1.0591756105,-0.3665927351,-1.3370912075,1.3054594994,0.0549971089,-1.1565880775,-0.1878262907,-1.7153264284,-0.2808305323,0.3602373302,-0.7212537527,0.5203909278,0.6193087101,0.0597162433,1.3419901133,1.7610182762,-0.4293690622,0.7105035186,-0.2299219370],[-1.1954588890,1.5433597565,0.2379141748,0.7672137022,-2.1098141670,0.6415581703,-1.4304503202,-0.5032683015,-0.1808628291,-0.9698618650,0.9286208153,1.2245706320,-0.8532410860,0.3880198300,-1.2975735664,-0.7356634736,0.1969772279,1.2833096981,-0.2106740475,-0.1530257016,-0.1989236623,0.6560999155,0.1675494611,0.4150153399,-0.2940008044,-0.4093782008,-0.8216152191,-0.4769310653,0.9040605426,1.2138260603,0.8652194738,-1.1292105913,1.0701618195,-0.1425126493,-0.8724672794,-1.6130620241,0.0751923323,0.4281897247,0.1331243962,-2.1410734653,-0.9063301682,-0.3643827438,-1.4411940575,-0.8983129859,0.4188375175,-0.6246270537,1.0137139559,-0.0459714942,-1.2153115273,-0.8810849786,0.9074236155,-2.5559902191,2.1576261520,-0.4590293169,-0.9930305481,1.8669599295,-0.4173523188,0.9735121727,-0.5194160342,1.8311252594,-0.2143068463,-0.3267007768,-0.7139260173,-2.0381011963,-0.5063050389,0.6719281077,1.2255204916,-1.0252617598,0.2913454175,-0.5917704105,0.7488190532,1.6724610329,0.8395947814,1.7179481983,-1.6458688974,-1.7235046625,1.6781872511,0.6279851794,-1.2300579548,0.2063685060,1.7433674335,0.2346530110,0.7607139349,-0.3385611773,-0.2102975249,0.0059588552,1.4260768890,0.0142158372,-0.4406615496,0.2704971731,-1.0747419596,-1.3426505327,-0.1716880351,-1.2444826365,-0.8974450827,-0.0273043569,1.3485187292,-0.2789664567,1.0251704454,0.1180881709,-0.3778805137,-0.2141260207,0.4078404903,0.3173700273,0.3696385920,-3.5518553257,-0.0249717198,1.0215418339,0.6116573811,-0.8735608459,1.9158036709,1.0758155584,1.0407550335,-0.1262805015,-0.1615180820,-1.1025336981,0.7169651389,1.9114420414,-0.7175180912,-0.2838610411,0.1815304458,0.0509980470,0.1601001769,1.3206021786,-2.0074746609,1.1405119896,1.0774667263,-0.3010169268,-0.0158685166,0.2949124277,-0.9155491590,0.0511295386,-0.9183694124,0.4059500396,-0.9847790599,-0.3063639998,0.2768558264,0.6799152493,0.1374379843,-1.7768172026,1.0458937883,-1.5003950596,0.2859581411,0.4456777871,2.7115471363,0.0316340886,0.0237582047,1.3859419823,-0.2100570202,0.5256478190,-0.0925751105,-1.4566472769,1.6074087620,-0.4122121632,-0.8000978231,-0.4829004407,0.7124879360,2.1299157143,-0.0259689819,-0.2674769461,-0.5260988474,-0.1544209272,0.8619582653,0.1432756037,1.1954751015,-0.4071525037,-1.1334422827,-0.1827943474,-0.1172728464,0.1429066360,-0.6256256700,-1.1532642841,-0.7034975290,0.5680677891,0.2888707519,0.1756383628,0.5803961158,-0.5129320025,0.0149797229,0.9116480350,-0.1075092405,2.3236284256,-1.1821206808,1.3360717297,0.5781747103,0.2877194285,0.4588770568,-0.1316705644,0.0050465642,0.5311812162,-0.7386845350,1.6503698826,0.8862819672,1.2689412832,1.4950553179,0.0704524592,0.9109133482,0.0946484506,-1.0497871637,0.7867195606,0.2191394717,0.8150247335,0.0630580559,0.1004397869,1.4962004423,0.3818966448,-0.3216921687,1.6232447624,0.4632739425,0.3479743302,0.2854761779,0.0905378759,0.0078849401,1.1418205500,-0.2064098269,0.1464061737,-0.1620241702,-1.1185684204,0.0551032908,0.0955945477,0.1914982051,-0.2445030063,1.0773905516,0.7771847248],[-2.0026953220,-1.3879367113,1.8168145418,-1.3441421986,-0.1909300387,-0.2537579536,-0.4333573282,-0.5043718219,-1.1934229136,0.8883563876,1.3796744347,-0.8058362007,0.6302874088,-1.4562950134,-0.2413089424,-1.1239010096,-0.8434281945,-0.6111188531,-0.3214215040,1.7906646729,-0.2953863144,-1.0996087790,-0.0280012544,1.5195906162,0.4385769963,-0.0715610012,-1.2215670347,-1.1551032066,-0.5126031041,0.5079829693,-1.5987869501,0.6407077312,1.2264078856,-0.5521286726,0.5383154750,0.9565272927,-1.1502499580,0.4597047567,0.3260488808,-2.5554530621,0.0539155044,0.9961486459,-2.3894460201,-0.3041664660,-0.2683731318,-0.8905698657,-0.0944725424,0.5605733991,0.7430984974,-0.6170213819,1.4618774652,-0.4896700978,-1.6775000095,0.9276668429,-0.3350292444,-0.2878435254,0.2499760538,-0.8336799145,1.0695707798,-0.8759759665,0.8804914355,-0.4413791895,-1.4477114677,0.3207118511,-0.6660116315,-0.0039797751,0.2596048117,-1.3988531828,0.0217267051,-0.3165823221,1.4104597569,-0.8589660525,0.3823044598,0.5117325783,0.5799040198,-0.7965207100,-1.0794136524,-0.5460695624,-1.4399230480,0.9373409748,-0.6249065399,-0.7326609492,1.7352846861,-0.7039022446,-0.8619210720,0.2775456607,1.8297880888,-1.3504639864,0.1678970009,1.6588230133,-1.4726707935,0.2412999719,-0.0883939713,0.8553674817,-1.1602106094,-0.0671377331,0.2164906412,-1.9931809902,0.0053256475,-0.2441381961,1.7421977520,0.2616285682,0.6127197742,-0.6840193272,-0.3926095366,0.4958389103,-0.4157512188,-1.8020431995,0.5348650813,-0.4459702671,-1.0132100582,1.6183987856,1.8912308216,1.4364794493,0.4951191545,0.3030346632,0.6038584709,0.6475570798,-0.2035506368,0.4782030582,0.8421400785,1.6720960140,-0.8565639257,-0.4995645881,-0.0788188726,2.6190998554,0.0172765702,0.8030301332,1.9209679365,0.6042689681,-1.6229704618,-1.5398095846,1.2159051895,-0.0491375811,0.6701576114,-2.0786049366,0.2396868467,-0.1663447022,0.4873751402,1.6635489464,-0.7188113332,-2.3959128857,1.7327207327,-0.5349614620,-0.0728966668,-0.2948603034,-1.4774221182,0.4834442735,0.3681408465,0.9823635817,0.5640633106,-0.0505462214,0.3036163151,0.9760785103,-0.2052964270,0.0603897423,-0.0227735676,0.4560826123,-0.7479364872,0.4386453032,1.5767741203,1.2626835108,0.5652157664,1.5275896788,0.0146089382,-0.8305668831,-0.0492977872,-0.4252792895,1.2362649441,0.0874866694,0.4226440191,0.0723808184,-0.0925615653,-0.5521541238,-0.5364710689,-0.5070555806,0.1121965498,-1.1298216581,-0.7371627092,0.8932670355,-0.6637581587,0.4557986557,-0.6397003531,-0.3043342829,-1.1985019445,-0.5069193840,0.5818718076,0.9713091850,-0.5977748036,0.0146248564,-2.4917397499,-2.0943746567,1.2473956347,-1.9712914228,-1.3773270845,0.1807457060,1.2754870653,-0.6485044360,2.5466191769,-1.3412190676,0.1961534172,1.7364504337,0.1384211183,-0.2164301127,1.4340209961,-1.1674877405,2.2805016041,0.7271856070,-1.6405873299,-1.7613757849,1.2478111982,-0.7066447139,0.4562221169,1.0106225014,0.6289210916,-0.2545123994,0.8075569868,0.9933471084,-0.7091018558,2.4793047905,-0.1058447361,1.4626127481,-1.1512520313,2.2819995880],[0.1980908364,-2.0391972065,0.4623443484,-0.4528153837,-0.3034771681,0.6253050566,-0.9960688353,0.1848763525,0.1992302835,1.3969924450,-2.1879806519,-3.5258898735,-0.3500127494,0.1608139724,0.5911024213,0.6979445219,-0.4094097912,0.1062802821,1.5368102789,-1.2191261053,-0.1408000141,-0.3725816011,-1.0421752930,-0.1061916351,-1.8223229647,0.6922587156,0.0466201380,2.0245492458,1.8404192924,-0.5316298604,1.8038771152,-0.9891858101,-0.7872034311,-0.9601790905,1.5571957827,-0.0051668053,1.6011846066,-0.1465292424,1.2160656452,0.4363673925,0.0321315080,0.7290571928,-0.5699969530,-0.1844813824,-0.0893535689,0.4522967935,1.4669539928,-0.2522238791,0.4661012590,1.4432981014,-2.7434496880,0.3867551684,-2.0132300854,1.6866211891,0.7507246137,1.0172561407,-0.2487859428,0.7585685849,0.5755932927,1.2352458239,0.8978250623,1.2966716290,-0.6571815610,1.4671807289,0.7853207588,0.3756480515,0.6194713712,0.3272837400,-0.3676315248,-0.0267670508,-0.2408681363,-0.7468521595,0.6240339279,2.1852478981,1.1214398146,2.0001266003,-1.5732886791,-0.0183822736,-0.8087029457,0.9497622252,-1.0744439363,0.9904448986,1.4737164974,0.1133142486,0.3727804124,-0.5018841624,0.6833029389,0.3201475739,0.2428314686,-0.8037180305,0.2282627523,-0.6574258208,1.8786723614,-0.7279502749,0.5236622691,0.3721244335,0.7680162191,0.7290078998,-1.6448494196,-1.3555030823,-0.9520250559,0.7258281708,-0.2495432496,-0.3343724608,-0.1035859659,-1.5604089499,-0.0533355586,1.2266353369,-1.9780186415,-1.2835073471,1.0977239609,-0.0809845626,-0.1254453063,1.2240229845,0.8162678480,1.1728838682,-0.4857140183,2.0652470589,-1.0620183945,0.4987939894,0.1875279099,-0.1605241150,0.2976987064,-1.7253868580,-1.2821964025,-0.5454786420,-0.6711188555,0.4872035384,1.3012408018,-0.0574941002,1.0852347612,0.2164134383,-0.8638213873,0.0029188166,-0.3997426331,0.8708361983,1.0494933128,1.2122185230,1.0808790922,0.0889089853,0.3659733832,0.6266746521,-0.7211550474,-0.0947615877,2.5941832066,-0.6472509503,-0.0892662406,-1.1688027382,-0.7755636573,-0.1888562292,0.4613487422,-0.1133504286,-1.0579370260,-0.3661361039,-0.9344208837,1.4105546474,0.3662054837,-0.1800113618,-0.8287143111,-0.4588594139,-0.5348659754,0.1927637607,-0.1688094139,-1.0985659361,1.7208666801,-0.9949175119,0.0071640117,0.2181097269,-0.5293247700,-1.2477115393,-0.4222748876,-0.8102868199,1.5723097324,0.7872313261,-1.7233170271,0.5807046890,0.1063605845,-0.5338957906,-2.1653342247,0.4575393498,-0.5575693846,-0.8172625303,-0.3406208754,0.0603416599,-1.5120033026,1.1769851446,-0.2431340963,-0.8860734105,0.2002744973,0.0624899231,-2.2787888050,-1.7242875099,-0.3660179377,-1.5788301229,-0.9279945493,-1.8265781403,-0.8975494504,0.2838334143,1.9209350348,-0.1576115489,-0.5317687988,-1.6259893179,0.0153308073,-0.9110952616,-0.1653829366,-0.4306715727,-0.6055718064,-0.8026126623,1.3090204000,-1.2156563997,-0.0519316867,-0.6713653803,-0.7353394628,1.4131215811,1.1170443296,-1.1881471872,0.2349975854,-0.2238769233,0.2461022288,0.1120637581,0.2336885929,-0.5243279338,-2.1437044144,-0.2431624234],[-1.2464995384,0.4675020576,-0.7527608275,0.5624199510,1.3365873098,0.7289454341,1.7958503962,0.2928060591,0.7216563225,0.9394800067,-0.6092904806,0.5123190880,1.6707273722,1.7536890507,1.2125877142,0.8943855762,-0.8880759478,-1.3625150919,-0.2121257335,-0.6096383333,-0.1692983806,0.1635778695,-0.1416323781,0.3522885740,-1.0253274441,0.0980968177,-0.1966700852,0.7525386214,0.0875271410,-0.4656350613,-0.4846600294,1.0480329990,-2.1517102718,0.4131481051,0.2350990921,0.8666293025,-0.0862590522,-1.3615747690,0.3523576856,0.6853317618,-0.3136028051,0.1880191416,0.4099854231,0.5680887699,0.5689167380,1.6204892397,-0.1595118791,-0.1066793799,-1.1540112495,0.6110669971,-0.7147044539,-1.7258614302,0.7982810736,-0.7267000079,0.1675357521,-0.8346014619,1.2481591702,-0.9113322496,-1.5161044598,-0.1560480595,-0.3392308652,-0.1593530327,-0.1827998012,-0.5442073941,1.3976205587,1.0422940254,-0.2430956662,2.9464750290,-1.0710835457,-1.2913874388,0.4633493125,1.0320210457,-0.1943645924,0.4055425823,0.8175639510,-0.6497596502,1.2451591492,0.3004733324,1.1675512791,-0.6333433390,-0.9965176582,1.0963352919,1.3911732435,-0.2135262489,0.0029021860,0.0531421266,0.3033952117,0.5173051953,-0.0102700051,-0.0946862027,0.2938453853,1.3135150671,0.4863734245,-0.8989269137,-0.7194559574,-0.1590253711,0.4843160808,-1.1027262211,0.0849133283,0.5688287616,0.2679769695,0.4811837673,-0.7134509087,0.2008953393,-0.8335529566,-0.7201511860,0.5242791772,0.0110169752,0.4427189529,0.4736892581,-0.5382049680,1.6526933908,-0.2942489386,-0.4903966188,1.2739274502,1.1805559397,-0.1975703090,0.4837063253,-2.3744602203,0.2897094488,0.6084982157,-1.6204299927,1.3142671585,0.6344429851,-0.4190022051,-0.4888667166,0.6072349548,0.9508922696,-0.6400114894,1.4980759621,-2.1872255802,-0.9253449440,-1.6960446835,1.5540304184,1.1076310873,0.4102774858,0.3571720719,0.0244482420,1.1432708502,1.6378223896,0.1038910747,1.1907352209,-0.7008403540,-0.4064160585,1.1399818659,0.8482441902,-0.8278619051,1.2165918350,-0.1842192709,0.5383604765,0.0136463754,-1.6233984232,-0.3791120946,-0.8511069417,-1.2866246700,-1.8059158325,1.1002595425,-0.6931541562,-0.2090755552,0.0067911642,-1.0231842995,-1.4700142145,1.5429402590,0.5305857658,0.7299745679,0.4535187185,0.1681299061,1.6024149656,-0.9856672287,-1.6728117466,1.1804482937,1.7814029455,0.0313383043,0.1245605722,1.5340962410,0.1487563401,-1.3175044060,0.5722743273,-0.8558244705,1.0446487665,-0.9774014950,1.4014579058,-1.2876857519,1.9653085470,-0.2574622631,0.8741406202,0.1923532635,0.4771126211,0.5219726563,1.0568250418,0.1275015920,0.3534825146,-0.4791073799,0.0526588932,0.8323126435,0.7390776277,1.3197847605,0.3206358850,-0.8508493304,0.1079496369,0.0510789007,0.8233492374,-0.2772503793,-1.0964258909,-1.0618132353,0.0157843046,-0.9365561008,-0.7326688170,0.1077768654,-0.2073359340,-2.3172490597,1.3560752869,1.5103831291,-0.1958469301,1.0512095690,-0.0394407101,0.3858150244,-0.0032381886,0.9143221378,0.9421868324,1.6064001322,0.4654370844,-0.2540049553,1.4979096651],[0.6909629703,0.4411643445,-1.0131149292,-0.1733857840,-1.7086960077,-0.3259657919,-2.0024752617,-0.6706923842,0.6668214202,-1.9155961275,-0.3144879341,-1.2456922531,-1.2796248198,-2.2090945244,0.7396605611,0.0296590049,0.9433910847,2.1231267452,-1.0864417553,-0.6030449271,0.3438302875,-0.6608654857,-1.0862615108,1.3887225389,-0.7988653183,1.3075400591,0.9552127123,0.3432019651,-0.2984606922,-0.1288973540,0.7399637103,0.0231899377,0.6450478435,-1.3013180494,-1.3024494648,-1.0995938778,-0.2466881722,-0.3775995970,0.4923250675,0.1694402844,0.9104561806,0.6665503383,-0.5168366432,-0.3762043715,-0.9129214287,-0.2788790166,-0.4600901902,-0.8824477792,0.5361903310,0.6113188267,1.3454030752,-0.8275414705,0.4882041514,0.8205421567,0.8798395395,-0.6477968693,2.4074959755,0.5788549781,-0.0437356532,-0.7579018474,0.9680426121,0.5040457845,1.1636078358,1.8771742582,0.6200423837,1.5920174122,-1.8497363329,-0.8436110020,0.3149974346,-0.4593172967,0.8473721743,0.0478686653,0.4404908121,0.3380852044,-1.1299993992,-0.1233110130,-0.9066924453,0.4301193058,-0.3466370702,-1.0565040112,0.2993363738,-2.2224280834,-1.1136168242,-0.5317560434,-1.9586926699,1.2876089811,0.5533798337,0.8821557760,1.1840175390,-0.1996572614,-1.7286516428,-1.8491935730,0.5178694725,1.7988730669,-0.2523056269,-0.4970327318,-1.4278050661,0.3399576545,0.7023395896,-0.7998167872,0.7301787734,0.5362419486,0.8577725291,-0.9817480445,-0.9755565524,0.3299261034,1.4980843067,0.7955431342,0.8185396194,0.0570088066,-0.1452435553,-0.4598515928,-0.4803588092,-0.6821164489,0.4632911086,-0.0423791558,1.8142098188,1.3901891708,-1.0697890520,-0.0289973561,-0.6405655146,0.0753127635,-0.5270310640,0.1859117001,-0.1663290560,0.5048159957,-1.7322213650,1.6413090229,-0.7120727897,-0.4916498661,0.8107194304,0.9926199913,-0.1052645668,-0.0880601779,0.6687352657,-1.4801020622,-0.7137554288,-0.3793790638,-0.1603921801,0.7068908215,1.2170107365,-0.4102213979,0.4785642028,0.3500928879,0.4325503111,-0.6816869974,0.2588504553,-2.3803224564,0.0774483383,-0.5226783156,-0.3486301601,0.1307638288,-0.4308033586,1.0323166847,0.7308865786,-1.0509271622,-0.3705799282,-1.4526411295,0.7582592368,-0.1112433150,0.2123344541,-0.1703404486,0.5684955716,1.2879304886,0.4955765605,0.9762223959,0.5969173312,1.0294094086,-0.4016108215,1.4354219437,-0.4525334239,0.9244177341,-1.8113485575,-1.8544753790,0.6716580987,-0.1637547463,-1.0912935734,1.0384893417,-2.0325262547,0.6750365496,0.1859313697,1.4355543852,0.9272583723,0.3481788635,-1.7764997482,0.6461026073,0.3343574703,-1.1932055950,-0.7114098072,-1.1107187271,0.2314597964,-0.3236814439,-0.2080765665,-1.4484943151,0.4726499617,1.2705886364,0.0462085642,-0.8196042776,-0.4398490787,-0.5813989043,0.0109049398,-0.3579022586,-0.3998372853,1.3548592329,-1.9293205738,-0.6319025159,-0.7778295875,-2.0569281578,-0.7499248981,0.9562718868,0.7526506782,0.3659623861,-2.2221159935,-0.5005191565,0.0293950923,0.0451079011,0.2878231704,1.1077967882,-2.5320174694,-2.6052191257,0.5428759456,-0.3326431513,-0.2977385223,-0.3886250556],[0.3837181628,-3.0844507217,-0.0326572843,1.9840731621,-0.0990928710,-2.0813033581,0.9364513755,-1.5018221140,-0.7127351761,-0.4184252620,-1.6085178852,-0.5712712407,0.3170658648,1.3702354431,0.2474628687,1.6732935905,0.3019446135,-0.5106189251,-1.6296691895,0.0827088729,-0.2414676398,0.0117205493,-0.3977093697,-0.8551368713,0.5991954803,0.4162049294,1.5830416679,-0.1344192475,-0.8080347776,0.8270174265,-0.2177505791,0.7213084102,2.7542774677,-1.9037790298,-1.1765846014,0.9574951530,-2.2370584011,-0.1641188562,-1.5068861246,-0.5019809604,0.2476276606,-1.3522347212,1.5981073380,-0.0016236352,0.0727113858,-1.4785469770,-0.8532192111,1.3951283693,-1.0664182901,0.4442366064,-0.4066230357,0.2653382123,1.0576475859,-0.2093126327,0.0964964181,0.1187895015,-0.1235642135,0.1212674677,-0.6063463688,-1.7976351976,-0.2944546044,0.5414992571,0.5756024122,-0.4685805142,-0.7462616563,-0.3464683294,0.2230277807,0.9854597449,0.0767948404,-0.4874532223,-0.1219522431,0.8327487111,-0.1716076732,-0.3945173025,-1.1791951656,-0.4452678263,1.3629028797,-1.1066451073,0.3573346436,0.6323838830,-0.3071047366,1.6411852837,0.8643095493,-0.2245212346,1.4297059774,1.3671625853,0.2288126200,-2.0948269367,0.9121999145,0.1160216853,-0.0257257298,0.2509836555,-0.1600100994,-0.6034733057,-0.1384988427,-0.0839320645,-0.6945816278,1.9426094294,0.0452554338,0.3266722560,1.2575550079,1.1093206406,0.8573603034,-1.2389947176,-2.0883388519,2.1227445602,1.1516443491,0.2675343156,-0.3369225860,-0.4755612016,-1.1894302368,-0.2873914242,-0.1477799714,-0.1865244955,0.9614298344,0.1122136638,-0.1507401168,-0.7496002913,-0.8663163185,1.8104102612,0.1771599203,-0.7490789890,0.0366497710,2.0983855724,-1.0290652514,-0.5826900601,1.1658000946,1.2506613731,-0.1822900325,0.0264755636,-1.3294763565,0.1857622564,0.1860099584,0.6812687516,-0.0765349269,-0.5551304817,0.2183326185,1.0382562876,-0.7524948716,-1.2267206907,-0.5154917836,-0.9957588315,-0.8302957416,1.1273059845,-0.7931482196,0.0290784836,-1.8754261732,0.6611202955,0.7537636161,1.4504109621,0.4328519702,-0.1413241476,-0.3728679121,0.6295785308,-0.2826263309,0.1017487198,0.1004890427,-0.7599399686,1.1586389542,-0.4124349654,-0.9641899467,0.9970339537,0.6460083127,-0.4832358658,-0.6841650009,-0.0303084180,0.5566793084,1.2560545206,1.2719508410,-0.0263155978,0.0070461296,-0.8024127483,0.1088609323,-1.8889316320,1.7075667381,0.4763677418,0.8923625946,-0.1581884325,0.3295147121,-0.6005237103,-0.7768769860,-0.4446044564,2.0271074772,1.3566521406,0.0894624814,-2.5341632366,2.3792328835,-0.2505889535,-1.3664253950,-0.5644386411,-2.1931164265,1.1769444942,1.0361630917,1.8065854311,1.3853068352,0.8310387731,-0.6718800068,0.0839435682,0.5932793617,-0.3277965784,0.4227188528,-0.9974304438,0.1251470149,-0.4001867771,0.2866265476,-1.7253122330,0.4782764316,0.7806475163,-0.5707991719,0.4938725829,2.1363813877,-0.6386837363,-2.1948287487,1.2262840271,-0.0510553345,-1.0744620562,0.5451381803,-1.9778690338,0.9243052006,-0.2363083810,-0.8462770581,-0.4339647591,1.5218746662,-0.2496304214],[-1.2015702724,0.4348722398,-0.7702450752,-0.3616005480,-0.9668240547,-0.7775117755,1.6584910154,1.2157908678,-0.0582093261,0.4874089956,-0.4351541996,-1.2981131077,0.4196226597,-1.1504046917,-1.3714263439,-1.1537238359,0.2113851160,0.9948299527,-0.0589868426,-0.2105388194,-0.1264684796,-1.0223534107,-0.1095140055,0.3037421107,-0.2303951383,-0.3564314842,-0.2918756604,0.3242609799,1.4333026409,1.8529117107,0.2137945294,1.9478644133,0.6609766483,0.2107582986,-0.0890568718,-1.0639896393,1.5600599051,-1.1038756371,-1.2550096512,0.1083938032,0.5406846404,-0.2521128654,0.4976765513,-1.5165120363,0.7626850009,0.2167731673,0.6663960218,0.4201067984,-0.5433947444,0.3073146641,0.7519953847,-0.5402916670,0.5032405257,-0.7342537045,-0.9444994330,-0.4982891083,1.3228744268,-0.1457812637,0.5063264370,0.5172562003,1.5089118481,1.8802934885,-1.1349375248,0.1486520320,-0.3159368038,-0.9561010003,-0.1483890116,-0.6002311110,0.8183563948,0.9246582985,-0.2708984911,-0.1344445795,-0.3975899816,0.2588158548,-0.8930866718,-1.6686511040,-0.5920096636,-0.0261793677,-1.0039272308,0.5324293375,-0.7660702467,-1.5267822742,-0.0449126177,-0.5578223467,0.4411966503,2.8361527920,-1.0847097635,0.5951077342,0.6513732076,0.5490559936,0.2389404327,-0.6742807031,0.2803015709,-0.5785861611,-0.4543779492,-0.6440399289,-0.9178314805,0.3433896899,-0.0781365111,-0.0110655092,1.0301516056,0.2924049199,0.9450477362,-0.7342799902,-0.4853533804,-0.0152373305,0.3297406137,-0.9591660500,0.6535556316,0.6672778726,0.5473683476,-0.0690325350,1.7266031504,-0.6833693385,-0.0771031976,1.3326834440,0.6728719473,0.8581472635,-0.7994928956,1.7931659222,-0.2488678694,-0.9435475469,-1.3550130129,-0.1657632738,1.8103445768,1.6421945095,-0.5307902694,0.3177648783,-0.8274542093,-1.5274112225,1.0199569464,-0.8939806819,-1.1168608665,1.3539602757,0.2608950734,-1.5025458336,-0.0350464173,1.6370078325,-0.2023550272,-0.9884869456,-0.7876907587,0.3030428588,-0.8543554544,-2.0759766102,0.8074741960,0.0092417505,-0.7486258149,0.9306184053,1.5228734016,1.0625956059,-1.0638818741,0.9000504017,0.9773253202,1.4914863110,-0.9541801214,-0.1188048348,0.0520063490,-0.7230827808,0.4520131350,-0.6191820502,-0.9545143247,0.1884564161,0.3061656952,0.4476604462,0.0418105461,-0.1309568137,0.4180471897,1.2613639832,-0.7609384656,0.3685899377,-0.2012770772,-0.4936135709,1.1598070860,0.3737017810,-0.1421750039,0.2495179474,-0.8559209704,-0.6964498758,-0.2374830842,-0.9256697297,-1.7266376019,0.2695619464,-1.4074134827,0.8068033457,0.6471933722,-0.5924095511,0.1117355973,0.2366000563,-1.4429482222,0.8711776733,1.5175033808,-0.2762643397,-1.2968262434,-1.2299476862,0.9790905714,-0.6197896004,0.2056121975,1.1886742115,0.6697474122,0.1087135598,1.2664176226,0.1910816878,-0.3005948663,0.6051598191,0.1337996721,2.2987043858,-0.2989132106,0.0533511490,1.4336971045,0.3602710962,-1.4897760153,1.9669363499,1.7474601269,0.7100765705,0.2949651182,-0.0514402352,-0.3578273058,0.3582149446,-1.3475846052,0.0255273320,-0.7661463022,0.6330846548,1.4332324266,0.9169499874],[0.8839311600,0.7519271374,-0.1242360175,-0.3691127002,-0.8278617859,0.3434691429,1.4831691980,0.8366777301,1.0739248991,1.0072058439,-0.6175259948,-1.3308913708,1.1121007204,-0.8164995909,0.8835690618,-0.9938970208,0.9998544455,0.4616194069,-1.1370314360,0.9062680602,1.1257185936,-1.9194012880,0.2764996290,0.5737554431,1.4669361115,1.4267357588,1.0081113577,0.7518727183,0.9374237061,0.6022189856,0.9636828899,0.6104877591,1.2044204473,0.9677898288,-0.5647366643,1.0105677843,-0.3122237325,0.3334992528,-1.9033569098,-0.9630427361,0.2652240694,0.5199043751,-1.1540907621,0.7361152768,1.2738237381,0.1880766749,-2.2973453999,0.5291508436,0.0009351905,-0.6024293900,-0.2233156264,1.5331068039,-0.9884901643,1.5078092813,0.7398353219,2.3887550831,-0.3120129704,0.5433498025,0.0288962387,-0.2012146115,1.0563759804,0.0455531701,-0.1074009314,-0.9435821772,0.8474586010,2.1150949001,-0.5216084123,0.6223477125,-0.4835988879,0.0323945619,0.1332448572,-0.6117167473,1.2821873426,0.6829779744,1.7952840328,1.4735364914,-0.0281432271,0.3344296217,0.0498930812,-0.7253366709,-0.1882753372,0.7500153780,0.3084684014,-1.2596098185,0.4998308122,-0.2986431718,0.7846732736,-0.5215204358,0.4127131701,-1.4233584404,0.3591822088,0.8100859523,0.6673883200,-1.4757587910,-2.2996366024,-0.5354201198,-0.6260387301,0.0607966073,-0.9204249382,-0.3795864880,-0.0245653503,0.4642808735,-0.2157164365,0.8455864191,2.2098424435,-0.4032137394,1.2565199137,0.3011314571,-0.4515376091,-2.2521011829,0.2975798845,-0.6609811783,1.2324833870,-0.2599743605,0.6994997263,-0.9482492208,-0.0442115664,-0.1683008969,1.7778540850,1.5128661394,1.1165469885,-0.9578713179,0.5861684084,0.4876947403,-0.0258086044,0.8223834634,1.0231527090,1.5395146608,0.5583790541,-0.5228635669,0.8827819824,0.4835412502,0.4577231705,0.5126315951,-0.7414305806,-1.8140503168,-1.3544577360,0.8708856702,1.0170553923,1.0010755062,-0.6828664541,-0.0983215198,0.2329484820,-0.5799430013,-0.9481555223,1.3907268047,0.6413714886,0.3823638558,0.9870128632,-0.4725233614,-1.5783783197,-0.6470431089,-0.9037789106,0.0999176055,-0.6165252924,-0.5567381978,0.6479242444,0.5634599924,0.4559148848,-0.7018021941,0.5007602572,-0.7826791406,0.3516834974,-0.5544981360,0.9681010842,-0.6531418562,-0.0493310317,-0.3987984359,-0.0768239871,1.2028565407,0.9053220749,-1.5396063328,-1.5210251808,0.0095656775,0.4015446603,0.4952757657,-1.9695576429,-0.9904660583,0.0043475674,-0.3832959235,1.5924043655,0.1618719697,1.1238229275,-0.3456900418,0.7589526772,-0.2558067739,-0.1103068292,0.3348076940,-0.2165825069,-1.7158200741,-2.2370626926,0.7226798534,0.1446095556,-0.3551731110,0.7914757133,-1.0397866964,0.3701458871,1.5328626633,0.1711807400,0.0900413617,-0.1533249617,0.7270753384,0.1712650359,1.2431062460,0.2759212554,-1.4751939774,0.1763630956,3.6361017227,-1.5242985487,-0.0279763695,-0.9603900313,1.1799722910,0.2726994753,-0.2330887020,-0.6826753616,1.0471214056,1.1989165545,-0.4046612084,-0.7465475798,0.8356605172,-1.0643728971,-0.3423130512,1.3552594185,0.0537857786],[-1.5534422398,-0.5955978036,1.1433280706,0.2265868187,-0.2054847330,1.4488072395,-0.9428060055,0.2496462762,-0.5333206058,0.9197281003,0.0167320166,-0.5308976173,-0.3981518447,0.1968156695,1.2565236092,-0.1500016153,1.7518627644,-0.5710981488,-0.7607555389,-0.5640133619,-2.1015295982,0.2339061052,0.3560619652,-0.2133811563,1.6464656591,0.8782684207,0.6951352954,-0.8930280805,-0.5173587203,-1.0700474977,-0.6147041917,-2.5534784794,0.4594208002,-2.9251899719,-0.6844087839,-0.1010654792,-0.9624158740,-1.0140864849,-0.0757387206,1.1583821774,-0.5384712815,-0.9480490088,0.9694117904,-1.3669112921,-1.2707827091,0.7595877051,0.9194093347,-0.9817193747,-0.7923594713,-0.9515684247,0.1903454214,-0.6589611173,0.2844074667,-0.0866462439,1.4720683098,-0.1156150550,-3.1793930531,1.5431214571,-0.6267422438,-0.5408167839,-1.4073745012,0.0020372916,-0.9605144858,0.0653555542,-0.0072504482,-0.2048006058,-0.8758507967,0.6794900894,-0.5280171633,-0.2964494228,-0.6805481315,0.7795145512,1.9110471010,-1.3809238672,-0.0489975736,1.4007698298,0.4083929360,0.2260774076,-0.6497412920,-1.0933283567,-0.2311290503,1.1378812790,-0.7990297079,-0.7805021405,2.1594064236,0.0872589722,-0.3067907393,-0.5868695974,0.4651990831,0.0277806502,0.3595694602,-1.7833522558,-0.9235187769,-0.1305120736,1.2421206236,0.7602189779,1.3953794241,0.3049729168,-0.5557566881,-1.0600878000,-1.5434368849,-0.4848669171,0.6762301326,0.3975794613,1.4395723343,0.6811670661,-1.5078858137,-0.2627333403,0.4361654222,-0.0008911950,1.0187122822,-1.0375159979,-1.6047658920,-0.0847408697,-1.3486804962,-1.8990767002,-0.7349280119,-0.1065333709,0.6665663719,0.5214542150,-0.0061568609,0.9659379721,0.7524757981,-0.0891807601,0.1273224354,0.8268703222,-0.1176565886,-0.7092362046,0.2952494621,-0.2299559861,-1.0425001383,1.2988185883,0.5426243544,0.4739787579,0.8060304523,1.5848567486,-0.0154908923,-1.9596747160,0.0608664788,-0.8224123120,0.2057334930,-0.3345183134,-0.7199540734,-1.0148650408,1.2570089102,1.7332231998,2.7068395615,0.5335254669,-2.0145659447,1.3850419521,-0.8592329025,0.6600543261,-0.4218591750,-0.0107094571,-0.6457693577,0.9560267329,0.5790085793,0.0617983118,0.3226754069,1.0685539246,-1.2620589733,-0.2347259670,-0.2421554327,-1.3379868269,0.9540162086,0.5375425816,-0.1855080128,-0.8155971169,1.2428810596,-1.7945463657,-0.0921459422,-1.4434609413,0.1425899863,0.0288435649,-0.7485082150,-1.0032165051,0.7188972831,-0.9643496871,1.1487607956,0.7042000294,1.0104765892,-1.3018821478,1.7104345560,-0.2635222673,-1.3582605124,-1.5040088892,-1.0269665718,-0.6213265061,0.1832721084,-0.3864678442,0.4071185589,0.4705953002,-0.4236857295,-0.1801551133,0.7529713511,0.8858818412,0.2697959542,1.4118680954,2.3722064495,-0.9191628695,-0.9307866096,0.3029454648,0.9904204607,1.2289518118,0.4089305103,-0.8350077271,0.5419706106,-0.5536469817,0.1028060094,0.3641077876,-2.9135649204,-1.1976279020,1.7268692255,1.0548107624,0.9957117438,-1.4481039047,-2.2000696659,0.4746692479,-0.3013611734,2.5931897163,0.1785014421,-0.9042234421,0.7220194340,0.5479753017],[0.8089565635,0.2825019360,1.6301807165,-0.9289301038,0.7758326530,1.7873525620,-1.7252171040,-2.6020746231,0.5115439296,3.2497107983,-0.5192397833,-1.0423642397,0.0324750915,0.4287697673,0.8982563019,1.3349349499,0.3121462464,-0.3937326074,2.0630812645,-1.4304995537,-1.3159050941,-0.0241598841,-0.7293787599,-0.3655036390,1.4027618170,0.1256354004,-0.0866579339,1.6698479652,0.6147372127,2.1514334679,-0.0860073492,0.5820814371,-1.2603700161,0.0365599617,0.1094137803,-1.0534101725,0.3607339859,-0.9958238006,-1.4315663576,0.7793776989,0.1461087465,-0.1628145576,0.3107595444,1.3687645197,0.5772598982,-0.4336755574,1.1175009012,-0.2714384198,-0.1034518629,-0.2754325569,0.9069437385,-0.0283357389,-0.1800245643,-0.1651595682,-0.5861871243,-0.4601516128,-0.0606546029,1.5763812065,-1.0784891844,0.9564955831,0.9455194473,0.7763608098,0.4902954996,-0.2564931214,-1.0015263557,0.3603806794,-0.7381950021,0.2104441971,-0.2726085186,-1.4666761160,-2.0567760468,1.1105402708,0.3748869598,-0.8810894489,-0.0609646738,-1.1932916641,0.0129567189,0.5431383252,0.6572669744,-0.6381964087,0.6827172041,0.2055616975,-0.4754789472,2.6849081516,0.9908787012,0.4223089516,-0.9532756209,-1.4316188097,0.4283581078,-1.3681161404,-0.0958754867,0.5441727042,1.0810705423,-0.0415074416,2.5692317486,-0.3227604330,0.4241096675,1.4338536263,-0.4114264250,0.0011840090,0.3396936655,-0.7032704353,0.0369640626,1.9964704514,-1.0196714401,0.2410306334,-0.6645501256,-1.9832742214,2.1286649704,-0.2816694379,0.6200950742,-2.5297253132,0.2783054113,-2.2589633465,1.5027683973,-0.6791571379,0.7946748734,0.5594283938,-0.4154709876,0.0868373662,-1.1731896400,-1.0965510607,0.4667754173,0.3479546905,0.5917227268,-0.1208579317,-1.9895873070,0.0767334402,0.0827785879,-1.5488442183,0.4962759912,0.6963671446,-0.6112600565,-1.9577354193,0.9674578309,0.7537819743,2.4563322067,-0.0252350532,-1.5132838488,1.2217682600,-0.7690975666,-0.2130514085,-1.2223173380,-0.5371199846,-1.0274674892,1.1752959490,0.0587048046,-1.2375228405,0.4799143374,-0.8488432169,-0.6043387055,1.1303751469,-0.3233990967,-0.6648087502,0.9539896250,-0.3432818949,-0.5456913710,1.2928091288,0.6582537293,-0.3367332220,0.1220468283,-1.5532795191,-1.7195379734,-1.0341162682,0.2998998463,0.8046797514,0.4961921871,-0.6684690118,-1.5009387732,1.2318353653,0.3643293083,0.4558056891,-0.7469726801,-0.1064849347,-1.1267255545,0.2651161849,-0.3827407956,-0.8086588979,0.0461993329,1.4123934507,0.1515855044,1.5572073460,1.3240995407,-1.5372346640,-0.2648573220,-1.5860991478,0.0319794863,-0.9175467491,-0.8246659040,-0.2483655810,0.1255396158,-0.5857534409,-1.9942765236,0.0182692204,-1.7602024078,0.3323732615,-1.4235423803,-0.0769748762,-0.6448416114,1.1782761812,-0.8038555384,0.5822150707,-1.4353785515,0.1170133278,-0.8189358115,-1.2466543913,0.6469664574,0.9749919772,1.2244031429,-0.2720718384,-0.0915584937,-0.6139442921,2.7357959747,0.6321662068,-0.4468442202,0.0670683905,-0.0152648930,1.4001054764,0.8164184093,0.2930732667,-0.6081119776,-0.9192171693,-0.1245781928,2.3893449306],[-2.4778037071,0.5976509452,-1.4283977747,0.4154342115,-0.4454664290,0.3893599212,0.2045798749,0.1197161973,-0.0726990327,-0.0892046988,0.2428955585,0.6287866831,-0.8733956814,0.1548168063,-0.8002090454,-1.3946036100,0.5990428925,-1.1735966206,2.5494780540,-0.7590735555,0.1940673888,-0.4431787431,-1.4156470299,-1.9468938112,0.3451087773,0.5958864689,1.1357485056,0.8258775473,-0.5660755038,1.9902119637,1.3801629543,-1.7834206820,0.8641321659,-2.3902966976,0.6164762378,-1.0545002222,0.6365401149,-0.6782473922,0.5563191175,0.7363837957,-1.8891561031,0.0214900393,-0.7824299335,-0.2060978413,0.3763463199,1.4707410336,0.4039954841,0.6742861867,-0.6028607488,1.2719975710,-0.7390785813,-2.4224550724,-1.2123428583,2.3222174644,-0.4605011046,3.1352612972,0.2794305980,-0.5493283272,1.0512810946,0.1013304591,0.3027843535,0.7218084931,0.2959215641,0.6120178699,-2.3468084335,-1.1663336754,-0.8147548437,-0.4591985345,1.3389530182,-0.0083134025,0.0621182621,1.2398926020,-1.1599477530,0.0201061536,1.5943176746,-1.4953566790,0.0830490291,0.2470071912,-1.9120575190,-0.1153577715,-0.4227010310,0.8099350333,0.0726526678,0.8789698482,0.6540535688,-0.6828325987,-0.4248169661,0.3941554427,-0.6652766466,0.0736672878,0.4986302853,1.2773303986,-0.2276820540,-1.7044115067,-0.9628150463,2.3503878117,-0.9085405469,-0.1369735599,0.4999858141,-2.4511468410,-0.1574627161,-1.2132976055,0.7054603696,-0.8242201209,0.9384362698,0.0222000740,0.5847182274,1.7617032528,-0.7561365366,-0.0969338343,2.0506048203,-0.6521903872,-0.5247980952,-0.8815501928,-0.2154593319,-1.5231957436,0.1173115671,0.8983911276,-1.5951839685,-1.7393609285,-0.4271378517,0.2468763441,-0.6876968741,-0.8700183034,-0.3693754673,-1.0298929214,0.4024917483,-0.4774999917,-0.8142039776,1.4894051552,0.6848152876,-0.6682641506,0.5156447887,-0.0836239308,-0.1595827490,-0.9284272194,1.6347107887,-0.5399640799,1.0173338652,0.2623037398,-0.4373755455,1.2972157001,0.9749194384,0.9706473351,0.6446200609,0.8166325092,1.1084922552,-0.1151099503,-0.0970648900,-1.6904464960,1.3225225210,-0.7746454477,-2.2212016582,1.1361017227,-1.3346679211,0.4757602215,-0.9677015543,-0.4630626142,0.0695191398,1.3389387131,-0.1383368075,-1.3319382668,0.6405091286,1.2340289354,-0.6869156957,-0.4758132398,-1.9471237659,-1.2886620760,1.2084196806,-0.5310587883,-1.3227537870,-0.7771701813,-0.0368499942,0.4158807993,0.3814026713,1.0844006538,3.4120693207,1.7345339060,-0.0976254642,-0.2757738233,-0.8358352780,0.1355589926,0.7318641543,-0.3644649684,0.3209478557,0.2786023319,0.7931556106,-0.5511992574,-0.4364145994,-1.2345582247,0.3531896770,-0.1258494258,1.9726265669,1.5069609880,1.0744990110,0.7512360215,0.5726920962,0.0459811017,0.6249359250,1.7143579721,0.9048161507,-0.1954852939,0.0158866066,-0.4643318653,-1.1683137417,-2.2276227474,0.8133713007,-0.5477312207,1.1287671328,0.4268198907,-0.4344830215,0.5964285731,-0.5707772374,0.1042475775,-0.2392384261,-0.3934238553,2.3036134243,-0.9504826069,-0.6469936371,-1.0541242361,0.0337269567,-0.7810605168,0.9689751267,1.9031298161],[0.2525197864,0.1838586181,-0.2395608574,-0.5237963200,-1.1532931328,0.4638881385,0.4851041436,2.1061365604,-1.5883505344,0.6773467660,0.9099257588,0.4675730467,-0.5751948953,0.8763380051,0.7442619205,0.0406341441,-2.2689523697,0.7088384032,0.2085940242,-1.6460503340,-1.2184493542,-0.8154140115,-0.2806397378,0.2383890152,0.7780022025,0.7270069718,-1.3557515144,1.2708170414,-0.1392763555,-0.3877072334,-1.4174968004,0.3400408626,-0.8829170465,-1.9422985315,1.4105211496,0.4198240936,0.3245728314,0.0032399215,-0.4103808403,-0.1612507552,-0.3893881440,0.5924805999,0.0768460557,0.2591560483,-0.7614800334,-0.4818507731,0.3876158595,-1.8072429895,0.6448305845,-0.4991023242,0.2990922332,0.7611392140,0.2680123746,1.9994397163,0.3640748560,-0.2594336271,-1.1131106615,-0.8219708204,1.1891075373,-0.1939040869,-1.7173801661,0.2022092044,-0.5829202533,0.5355664492,-1.4946354628,0.9865855575,-0.8716754317,-0.9377834797,0.3892740905,0.6675186753,2.0363080502,0.9215611815,-2.0991868973,-1.0165593624,2.0159192085,0.4246093333,-1.3649692535,0.8177539110,0.1407611221,-0.7992921472,-0.2672897279,0.7641906142,-0.3375808597,0.2430761158,-0.2130232900,-0.4740942121,1.6659946442,1.5688807964,0.1841378659,0.4639534056,-0.5424900055,1.4749709368,-1.3607293367,-0.6117712259,-1.2864717245,0.7173013687,-0.8206724524,-0.8166086674,-1.1200137138,-0.1569412053,0.4703750908,-0.7065168619,0.4275971949,-0.3475115299,-0.7177425623,1.2331171036,0.1298312247,-0.9110089540,-0.1160564497,-0.1007026881,0.1473488063,-0.0265515819,-0.1134301201,-0.6335337758,1.0367230177,1.2019270658,0.2903762460,0.8871808648,-1.3211030960,-1.2637345791,-0.3478819430,2.6312584877,-0.0779268518,-0.0033409558,-0.6762089133,-0.2439787388,0.8177816868,1.4604249001,-1.5679810047,-0.1339834630,-0.1631641537,1.5747507811,0.1084988192,1.4038569927,-1.6337524652,0.0177575890,1.0167950392,-1.6949776411,0.0757225081,1.4419057369,0.2556048930,-0.6593601108,-0.2549015582,-0.9439916611,-1.8636031151,0.3080304861,0.4328792691,-0.7933393717,-0.2392593324,0.4455813468,-1.1091521978,0.6833079457,-0.1153917611,-1.4299768209,-2.4306547642,-0.4589000940,-0.9767031670,-1.1525596380,-1.1114211082,-0.6227270365,-0.3940295875,-1.1870974302,0.3540764153,0.8213087320,0.5123792291,-0.0097675789,0.7527971864,0.4728479683,1.2856576443,-1.5182529688,0.7346980572,-0.1847112030,-0.7216877937,1.3291749954,1.1263566017,-0.7484042645,-0.2636847794,0.0217362791,-0.9375768900,-0.7247433066,-0.0415024720,0.0492623039,-0.7459357977,1.1220898628,-0.5154392719,-0.4776915610,1.5804840326,0.8040207028,2.2452106476,1.7335001230,-2.4033691883,-0.2907517850,1.0620511770,0.9767885208,0.2632334530,0.5237601995,-0.5357385278,0.4868034720,-2.4270212650,1.2570142746,0.2010706663,0.6235077977,0.7071534991,-1.2176045179,-0.5152527690,0.3981330395,0.2556486428,-0.5797071457,-0.6267167330,-0.9684706330,0.2104416192,-0.8124510050,0.7867467999,0.5026560426,-0.0581949949,1.6213607788,-0.0828385055,1.9328351021,1.7954280376,0.2441668212,0.4763677716,-2.6320970058,-0.0609419681,0.4138411880],[-2.8224081993,0.8544120193,0.3406496048,-1.3216235638,-0.1166282445,0.0615011118,-0.9227725863,-0.7776309252,0.6261429191,0.6446788907,-0.2852533460,0.4866887033,-1.7207942009,0.3032245636,-0.7486038804,0.2438063323,-0.2671179175,-0.3899340332,0.0800387189,1.1771892309,-2.8977830410,0.5417762399,0.5842677355,0.5015832186,-0.4940728843,-1.4268690348,1.7689621449,1.7392749786,-0.3849787414,-0.9646254182,-0.1681044698,0.1822709441,0.6294948459,-0.4607347548,-0.4217625558,0.4491379857,-1.9762825966,-0.8236123919,-0.6104609966,-0.3675303161,-0.7454993129,2.4073290825,1.1063375473,2.4108457565,0.2904265821,-0.0319271833,-0.1898043603,-1.1604915857,1.1043064594,-0.4222970903,0.1301062554,0.3959206343,-0.1617540270,1.0322108269,-1.7450871468,0.2481280863,0.5010092854,-1.0896248817,1.2918280363,0.1254145205,-0.9438206553,0.5197620988,0.8137003779,-0.5376303196,0.2467346787,-0.6972445846,0.9036880732,-1.7012495995,0.1490175277,1.6514542103,-0.3002925217,0.6282120943,-0.0789799169,-0.5649306178,1.1362684965,-0.1409714818,0.7433593869,-0.4661725760,1.0111490488,-0.1314715445,-0.2794724107,-0.5579557419,-3.0807359219,0.6628274322,-0.4908138514,0.2899478078,-0.0956695154,0.4372552633,-1.0438014269,-0.1414678246,0.0443393439,-0.0560218208,-0.4559345841,-1.9249411821,-2.0573368073,-0.0334194154,0.0930603072,1.3541816473,-0.0132205859,0.1044306085,0.2731702626,-1.0667283535,-1.3606183529,-0.3784544766,1.4905642271,1.3429602385,-0.7809277177,-0.3124582767,0.1603041440,0.7163369656,-0.5278331637,-0.8617659211,0.0707306340,0.5078257918,0.5343469381,2.7920517921,1.1080427170,0.6272335052,-0.2741323411,0.5719816685,0.7550273538,0.8943145871,0.1229175404,-0.6765108109,1.5254110098,2.3936822414,0.8236930370,-0.1658976078,-0.5570540428,0.1447302103,-0.0118487366,0.2371482849,0.2607527077,-0.3373178542,0.0334553272,0.4224862456,2.3579299450,-0.2408792973,-1.9752658606,-1.0364269018,3.8317902088,-0.9200903773,-0.6679349542,-1.6793940067,1.1850756407,0.0062380657,0.9284568429,-0.1546033323,-0.8065575957,-0.3873867691,0.6424305439,0.2068197876,0.1537742168,-2.0683746338,-1.7794313431,-0.9078343511,0.3276721835,0.0317510068,-0.9162718654,-0.9382501841,0.8591877818,0.0205972660,0.3160986602,0.4791103005,2.3585975170,1.2266488075,0.0675020814,-0.0706233829,0.1332499087,-1.2125068903,0.9365451932,0.1137873232,0.2121182531,0.9187198281,-0.9210603833,-1.3618185520,-0.8186307549,0.2531672716,1.0069762468,0.8433437347,1.1979171038,-0.5553349257,1.0772843361,0.3243967593,1.4009695053,-0.7212144732,0.2164977342,0.1635417789,-0.4878661633,0.2016631365,0.6704857945,0.1343654990,0.9387943149,0.0494832285,-0.1038764939,1.4206699133,0.1582160592,-1.6772825718,2.0133848190,0.1406247765,-0.4359038472,1.9959181547,0.0242856946,-0.9826890230,-0.8497070670,0.9738262892,-0.9124195576,1.5310759544,1.1907900572,0.8818747997,-0.2512267828,-0.4001435935,-1.6142051220,0.7161724567,0.1099669784,-0.7687640786,0.1460857093,-1.2593784332,-1.1577454805,-0.8739498258,-0.1982318014,-0.4073867500,-1.3021171093,-0.1136029884],[-1.5907045603,-1.1428781748,0.1213455573,0.3092034757,0.0334026329,-2.4200618267,3.3238034248,1.1879281998,1.9430119991,0.2992740571,1.6709127426,-0.3273898959,1.4901436567,-0.6158698797,0.9194712043,-1.7540516853,0.4838142991,1.2830245495,0.5275840163,0.8925089836,-0.4407205284,-0.0098689431,1.5410522223,0.2787226439,0.6298201084,-1.2696967125,-0.1690566242,-1.3529920578,0.4250549376,1.9450334311,0.9914284945,-1.6382533312,-0.4501300156,-0.0474638417,-1.6697709560,1.3145624399,-0.8170112371,-0.8111792803,-1.0784217119,-0.3456541896,0.4999735057,-0.4648705721,-0.7333679199,0.0848675817,-1.6081210375,0.0173648894,-0.0103555061,0.8468238115,-1.1412696838,0.3661522269,-0.4450252354,0.8290936947,1.1968002319,0.5475606918,0.0861864313,0.5601829886,-1.5211524963,-0.5370119214,-0.5964001417,-0.5213727951,1.0056358576,0.3153733909,-0.8991996646,-0.5396741033,0.9581548572,-0.5757839084,-1.0513538122,1.1611233950,1.8634634018,0.4477458298,-0.2449746579,1.5770003796,-0.2315210402,-0.3703002334,0.0056428425,-0.4994624555,0.0295562055,-0.0041788672,0.8292986751,-0.0069422261,0.4954382181,-0.7247942090,1.1682143211,0.0745441839,0.0941772833,-0.7176083922,0.8280134201,-1.6582189798,0.6430417299,0.1367926598,0.7049418092,0.0571377203,1.1631231308,0.6296694875,-0.0647290796,-1.2796778679,-0.5895133018,0.8610818982,0.3743764460,-0.8071950078,1.3061199188,-0.8803336620,-0.0079116700,-0.5633738637,0.3912031353,-0.1816237271,-0.1595174670,-0.8321972489,0.2964517474,0.9229819775,1.4701541662,0.6736425757,0.5532526374,-0.1107472777,1.7293556929,-0.5153669119,-0.6763880253,0.2341462821,-0.5053901076,-0.1485505253,1.2039222717,0.4330215752,1.6672991514,0.3798584044,0.5994235277,-0.3695307672,0.6090422869,-1.8857115507,1.0357469320,-0.6409605742,0.5081863403,0.8336287737,0.2464233786,0.1892288029,-1.5773829222,0.7635652423,1.6710473299,-0.3723566532,-0.0935106426,0.4004655480,1.6516311169,-0.0239151251,1.3460192680,-0.0556113571,0.7014779449,-1.1409986019,-1.9849413633,0.7863185406,0.1040583923,-0.5113148689,-0.6147990823,-0.1334390640,-0.8577879071,-1.6896690130,0.0303357765,1.0349303484,-1.3887677193,1.4465106726,1.4462199211,0.9695534110,-0.9796079397,1.0599244833,-0.3817900717,0.5725802779,-1.0808739662,0.6993675828,0.0584711172,-0.5152519345,-0.3773689270,-0.9231536388,-0.0607250929,2.4555163383,-2.2077910900,1.5051277876,-1.2260414362,-0.2329563051,-1.2366898060,-0.3229627013,-1.4195483923,-0.1831594706,1.7906260490,0.2295542806,0.2179629952,0.3471575677,0.2313944250,-0.0609209612,-0.1078507155,0.4665036798,1.1243994236,0.9174999595,0.2093791664,1.6390047073,-1.3195793629,-0.1680346727,-0.5173400640,0.6296047568,1.0929300785,-1.5066785812,1.2682543993,-0.3669987917,-0.0628513321,-0.3588504791,-0.8302126527,0.2542398572,-0.3541894555,0.8507209420,-0.6196182966,-1.1007553339,0.5014449358,-0.8606661558,0.4498639703,0.2191863060,-1.4525839090,-0.1573316753,-0.3670399189,1.1949836016,1.2781003714,-0.3769445121,-0.8998956680,-0.1330458224,-2.1117086411,-0.3949528337,1.4143472910,0.0138987098],[2.0456752777,-1.7744398117,-0.0091569005,-2.6830213070,-0.2830215991,0.5834649801,2.3459873199,-0.9336617589,-1.0544435978,0.9978377223,0.8136739731,0.8705741763,-0.4490517080,-0.4460271597,-0.8419141173,0.2584621310,0.2546221018,0.0829493180,0.1972437054,-2.2637302876,-0.3299877644,-1.9285436869,1.0270245075,-1.5507107973,-0.4767905176,0.3876853883,0.9809426665,-0.6491833925,0.2031453401,0.8613374829,0.9279358983,0.3166069984,0.3596234322,0.0556599312,2.6604681015,-0.3559695184,-0.8140570521,1.2609890699,0.7009537816,1.6023476124,-1.6866204739,-0.3539562225,0.2053398639,-0.1173354909,0.0861274227,-3.4895308018,-0.4967344999,-0.0145240240,0.3744366169,-0.9175367355,-1.4568159580,0.7877227068,0.6968756914,-1.1660602093,1.4751977921,-0.1938352287,-0.0264743380,-1.3262991905,0.0237702597,-2.4104363918,-0.6477180719,0.5059946775,0.3510814011,0.3875126243,1.2187185287,-0.2408093661,0.6088964343,-0.3309341669,1.8326859474,0.2369977683,-0.3282163143,-0.2274724394,-0.2073322684,-1.0838879347,-2.1840865612,-0.4663523138,2.0218403339,0.9449817538,-0.9978976846,1.3662493229,1.7149190903,-0.2701505125,-0.8451486230,-1.6304798126,-0.7960706353,1.2277041674,-1.7852749825,1.2480875254,-0.3981883824,-0.7367050648,-0.4135238230,0.5079085231,0.0900633857,-1.2990952730,-0.6961169243,0.3780362606,0.9619675279,2.1818091869,0.4237046242,0.6840161681,0.6984956861,-0.2514249980,-1.7293785810,0.3667742610,1.1325265169,0.7683765888,-1.1529784203,0.4323818088,2.3151834011,0.3081058562,-0.6248998046,0.5047844052,0.6498021483,1.7978770733,0.3899771869,-0.4997722208,-0.1578063518,-1.9672470093,-0.3175988495,-0.0856336206,-0.9187674522,0.9462944865,-0.1890290082,0.0526048094,0.7671905756,0.0052128769,0.6099808812,0.5507966280,-1.3356704712,1.1185127497,-1.0913438797,1.2091479301,0.7399811745,0.1125142574,-2.0963685513,-0.2536096871,-1.4916297197,0.6988286972,1.0503125191,-0.7035870552,-0.4649752080,0.3794374168,0.7661656141,-0.5972264409,-1.0623397827,-0.0284309927,1.3018770218,-0.9428182840,-0.2252209038,-0.3813332319,-0.9502642751,0.0767621696,0.7689592242,-0.0794237703,0.2420342267,1.4336521626,1.5250978470,-1.3550601006,0.5420228243,0.7595084310,0.1850525141,0.9257733226,0.2702014446,1.1696821451,0.8106309175,-1.0838518143,0.4734386802,-0.4565702081,1.3057682514,1.4480742216,-0.5279594064,0.6467399597,1.6548110247,-0.0588355176,-1.4721015692,-0.9004341364,0.0964303613,-0.6494836211,-1.2516726255,-1.5430279970,1.6699434519,-0.2119133621,-0.3817618191,0.3938360512,2.3915538788,-1.0785287619,1.3833017349,-0.5712346435,0.0911998898,1.0019682646,0.4382897615,1.1474125385,1.1693954468,0.1031880304,0.7955002785,1.3405262232,0.0209042039,1.6260020733,0.3945917785,-0.3832558692,0.7924461961,0.0312300827,0.7102786899,-1.7634069920,-1.3943989277,-0.7769746184,-2.2571027279,-1.1994607449,1.0450494289,0.3268254697,-0.6890351772,-2.0280230045,-0.4204472303,0.4531336129,0.1239523813,-1.0549724102,-0.4190306664,1.0953723192,0.2018714398,0.7987915874,-0.2929423451,-0.1676543951,0.5970343351,-1.2773684263],[1.2236326933,0.9840433598,0.1225740537,1.1968276501,-0.3415886462,-0.6405214667,0.7638618350,1.5436466932,-1.0006358624,0.1138499379,-1.0741703510,-1.5411993265,-0.7750322819,-0.9824083447,1.0209515095,0.7557222247,1.0524106026,0.2535609007,-0.3112972081,1.7932996750,-1.2139261961,2.0946898460,-0.0090470267,0.6447563171,-0.7594442964,-0.1354953796,0.0660691410,-1.0660305023,0.8187485933,-0.3587822914,0.4471886158,0.8411183357,0.5693242550,-1.4816662073,-0.7689180970,-0.7788417339,-0.9212349057,0.7576090693,-1.2389307022,0.6051481366,-1.1756383181,-1.2799351215,-1.8131713867,-0.0544976145,-1.5947587490,0.1935119927,-0.3740875423,-1.1885846853,-0.0792386606,1.1125651598,0.3793795705,0.5841192603,0.0571746901,-0.1136863083,-0.5652914047,0.7779823542,0.4327304661,-1.0822675228,1.0263322592,-0.4865728915,0.0487550497,-0.2343792617,-0.2356239855,0.0258828122,-1.5499720573,0.2839837670,0.0145347295,-0.1443930119,1.1493870020,-0.6262537241,-1.7196328640,-1.7135055065,0.4295227230,-1.3662779331,-0.4895913601,0.2493050396,-3.3102571964,0.4204231799,-1.0910575390,-0.9758739471,-0.1436772048,-2.1989107132,0.3001672029,-1.7423200607,-0.5652357936,1.4203227758,0.3454657495,-0.8482355475,-1.0011060238,0.8571538925,-0.4992506206,0.7553210258,-0.8703880310,0.7454122305,1.4743127823,-0.3983528912,0.8992680907,-0.8724070191,-0.4594956636,-0.2891397178,1.6464035511,-0.5800548196,0.1192378998,0.3251797855,1.5719088316,0.2433685362,1.6238752604,0.9168971181,-1.7127541304,0.4310311675,-0.2766275108,1.5945236683,0.7060499787,-0.2566100061,1.0110602379,1.4094588757,0.9080002904,2.1751139164,1.6054965258,0.9616665244,2.2810950279,0.4121274054,-0.0153397154,-0.3080755472,1.4366900921,-0.4038511515,0.7201999426,-0.7327430844,2.0949237347,0.0393173620,2.2166905403,1.2483799458,0.4711368680,-0.4010816813,0.7753511071,-0.5172150135,0.3040603995,-0.2500374317,-1.0669492483,-0.0891368464,0.0055941949,-0.9773228765,1.1023194790,0.4938498437,-0.5543298721,0.0906332210,0.0505249016,0.5267651677,-0.5476112962,-1.3049540520,0.8578358293,-2.0298936367,-0.1674510688,-1.0671063662,-1.4121545553,-0.9709029794,-0.3150966167,1.9244461060,0.4939165413,-0.0465229526,0.4197863042,-0.4554887116,-0.8888947368,-0.4829266965,-1.9815404415,-0.1060535312,-1.4770417213,0.6611261964,-0.8000780344,-0.0008512373,0.0904324129,-2.3193302155,1.2738482952,0.8960350156,0.3338158429,-0.8541143537,0.3179859519,-0.2948468626,2.4262568951,-0.6027595401,-0.1137509346,0.3913904130,-1.4957355261,0.4632006288,-1.5061677694,-0.0406272337,1.3672878742,0.9088773727,0.0280201621,-0.3759260476,-0.5266292095,-1.6783390045,0.7885782123,0.3884814978,0.5505877137,-0.2127685249,1.5057326555,-0.3375179768,-0.4418465793,0.7669808865,0.4645462036,-0.4733898342,0.0666616634,0.6061386466,-0.4868551195,0.4049857855,-1.6582262516,1.4912316799,0.1251794696,-1.2685055733,0.3584597707,-0.0061725895,0.1182529852,-0.4459925294,0.0649669096,-0.5827793479,-0.9905433059,-0.9163452387,1.7737463713,-0.4419859648,1.9480772018,1.4152547121,0.9885281920,-0.7954316735],[-0.6662418246,-1.1790033579,-0.2741505206,-2.0678997040,0.3266300261,1.3303706646,-0.2867909372,-0.9021807313,0.2714713812,-0.5843097568,1.2097675800,0.2966309488,-1.0622887611,0.6333710551,0.3833867610,0.2917357981,-0.5103815198,-0.3244171441,1.2576582432,0.9913121462,-0.1317096502,-0.5059637427,1.1085257530,0.1522968709,1.1645395756,-1.2047892809,2.2415204048,0.1009635627,-0.1189174354,-0.3255894184,-1.5088373423,-0.1761455089,-1.2958545685,0.4631999731,-1.4431842566,-0.0815225467,0.0264114328,-0.0134794405,-1.7359671593,-0.8405629992,-0.0171891171,-1.5212892294,-1.3272206783,0.3295044005,-0.3159745634,-0.4405059218,-1.2775102854,-2.0660405159,-1.3951530457,-0.3871063888,-1.9635423422,0.3878398240,0.2240091264,-0.6311438084,1.1141709089,0.2829698622,0.1319483519,-0.9153652787,1.4351322651,-0.6812025309,0.1347745210,0.8289664388,-0.1638492942,-0.8752292991,0.6741952300,-0.8481402397,-0.4189682901,-0.8234933615,0.2876498699,0.0553242527,-1.3581985235,0.1725343168,1.4257600307,0.3076897562,-0.6096960306,-0.6603984237,0.2657020390,-0.3375091851,1.0193190575,0.3068545163,0.3049698770,-0.8007757664,-1.4549874067,-0.7384971380,-1.3470081091,-1.8314106464,0.0719065815,-0.4571023285,1.0792492628,2.2859106064,0.2279076427,-0.9891816378,-0.6099649668,-0.6531402469,0.4977306128,0.8263016343,1.8157972097,-2.3561737537,0.7034680843,0.3216913640,-0.2272416651,-0.8027136326,-2.7809767723,-0.7257278562,-0.2273125350,0.2905314267,-0.3215314150,-1.8055938482,1.4343173504,-0.9048855305,0.1905162185,0.0468965657,0.8480873108,1.7085329294,0.4332759082,-0.3628672361,-0.6735289097,-1.1163346767,1.3994438648,-0.4484830201,-0.9012812376,-0.4600181282,-0.4331746101,0.1538828015,2.2690322399,-0.1937780529,0.1015669405,-0.5000990629,0.1091853082,0.6234668493,-0.4246721566,-0.0613945201,-0.8901450634,-1.1397343874,-0.8808919787,-1.0332076550,-1.1599528790,0.0582559519,1.4459686279,0.1238117516,-1.9827600718,-1.5158689022,-0.6710330844,-1.8118439913,-1.6237982512,0.0679273456,-1.3040943146,0.3018227518,1.1907341480,-0.0415180326,-3.0691003799,-1.3719874620,1.0663380623,0.4812533259,0.4310096502,1.4828522205,-0.5797268152,0.4582100511,-0.6201152205,0.1096500233,-0.6994256377,0.3923276663,1.5864090919,-0.9220345020,-2.6383047104,0.1972300112,-1.7775818110,1.6898244619,0.5709993839,-0.5445369482,-0.2049824893,-0.4456205368,0.4843392372,-1.4021123648,-1.1148446798,1.3989207745,-0.1107301638,1.9977177382,0.0048229685,0.6804937720,-0.4943865836,0.0055954242,-0.8179009557,0.6151177883,1.7471002340,0.5628360510,-0.6135786772,0.1982985437,0.3813651800,0.1579486132,-0.7328537107,-0.0750542954,-0.1824423671,1.0054148436,-0.0228289776,-0.3487198949,-1.0940510035,-0.7222530842,-0.6091982126,-0.6036486626,-0.2050597668,1.1312741041,2.0681664944,1.1864436865,-0.2236890197,-0.8625605106,-0.7263569832,0.3035775125,0.0642245412,0.4337040782,-0.2739275396,-0.3962904513,0.8403483033,1.0489102602,-0.4937454462,-0.5222024918,2.6116652489,0.1352985650,0.0618634447,-0.3024633825,-0.8286788464,0.1451219916,0.5880844593,-0.6094891429],[0.3219236434,-1.3695703745,-0.0585177056,0.1700653136,0.4964697361,-0.9321046472,-0.1507416666,-0.3361282051,-1.9466036558,1.1729111671,-0.2739106417,1.6313178539,-0.1329207718,0.2038348913,-0.7290941477,0.8423547745,-0.1522175819,0.0712911412,0.9698452353,-0.7045807838,-0.3561033309,0.0883658156,-0.4302078485,-1.2478718758,0.4989421368,-1.7669965029,-0.4198913872,0.6975441575,0.4738608301,-0.0651089847,1.0222551823,-0.5519806147,-0.3389316499,1.5764091015,-0.7053538561,-1.2888039351,0.3234673142,-0.1417339593,0.9278524518,-0.2274197787,0.7508451343,1.4466332197,0.6488268971,-0.9954628348,-0.6652079225,0.2864868939,-0.3673034012,-1.4958170652,0.2327007651,-1.5480067730,-1.3630514145,-0.4443548620,1.6024609804,-1.0053298473,0.7348178625,0.8800488114,0.8419062495,0.0995379835,-0.2248645723,-0.5294179916,0.3160320520,0.0924004465,-0.4043036699,0.8951402903,-1.2198828459,-0.0668722764,0.4917221963,0.5889586806,-1.8725498915,-1.7988634109,0.3151299059,-1.3311706781,-0.4863022566,-0.3315199912,-0.1942166537,1.7517579794,0.2368437648,0.5656731129,-0.0282656793,1.7823673487,-0.1720479727,-1.3562507629,1.0161684752,-0.9181568027,-0.6771322489,-0.2678036094,-0.6415093541,-0.9906380773,1.3346472979,-0.7320179343,0.6933484674,-0.3105112314,0.0937701985,0.5368575454,0.0130084530,-0.9673388600,0.5386470556,0.1005189866,2.0324866772,1.2055453062,0.8220839500,-1.5560021400,0.1882465482,1.6134934425,-0.0579883754,-0.4294087291,2.3164374828,1.3468019962,0.7022433877,0.2779477239,-0.5003404617,0.8945329785,0.2074485123,0.0166519284,0.9358554482,-0.3249890804,1.4698566198,-0.4166862071,-0.5823774338,0.6684930921,0.0280649178,-0.3356093764,0.5318160653,-1.4786239862,-1.0148028135,1.1849619150,0.2391745150,-0.6188251972,1.0286505222,-0.7010456324,-0.5719571114,0.1553662270,1.0689032078,0.0141669363,-0.0817926750,1.2871496677,0.8973370790,1.0335407257,0.5311743021,0.2168038338,0.3711017966,-0.7261734009,-0.6836606860,-0.4303300679,-0.9623488784,-0.2232476622,-2.8081274033,-0.1310659349,-0.1354033500,1.3199428320,-0.4372418821,0.3999006748,-2.7013614178,0.2619364262,0.1507003158,0.0715506971,0.8289863467,1.2401783466,-1.0547454357,-0.0138927512,-1.1025797129,-1.3504421711,1.4697799683,-0.4074902534,-1.1207348108,0.0986788496,0.3246071041,-0.8392274976,1.0582296848,0.2235145271,-0.5397490263,1.1688777208,0.2009891868,0.1039046645,-1.0009261370,-0.8749092817,-0.2685181201,-1.0310175419,-0.4049829543,-3.1205077171,-0.3098204732,0.5935285687,0.1903157085,1.1804158688,-0.3473457992,0.8149939179,-0.7612107396,-0.8262937069,0.6924495101,-1.3365644217,-0.7892350554,0.1343418360,0.7926063538,-0.1322494149,-0.4287471771,-0.3642910719,0.1303860843,-0.1418085694,-1.5912516117,-0.0276062898,0.4361262321,1.0095407963,-0.0749416277,0.2285602689,-0.1831205785,-0.0460781157,0.2784950733,-1.8076405525,-0.3249053955,-0.7071861625,-1.1327867508,-0.0700493380,1.8727740049,-0.9032890201,-0.4349606037,0.4104333520,0.6316285133,1.0076644421,-0.8771318197,-0.9682021737,0.4976361096,0.8091021180,-0.0814546943,0.1587979943],[0.5097995996,0.5882976651,0.5308931470,-0.0849480703,1.5427222252,-0.3993819952,-0.6722313762,0.6797321439,0.3277135193,-0.0085896691,-0.3661646247,-0.9072237015,-0.4623303115,-0.2023209929,0.6599558592,-0.7539873719,0.8019641042,-0.1096544489,0.0187816937,-0.2943032682,1.1049185991,0.8500798345,-0.8090435266,-0.9830206633,-0.2533683479,1.5624762774,-0.8042505980,0.0995938256,-1.0303261280,1.0731709003,0.5126503110,-0.3870714307,-1.0472843647,0.3077230453,-1.7716023922,-0.1703357995,-0.3809374273,0.2320100963,-1.0177683830,1.2246047258,-0.6436703205,-0.2181359231,1.5420596600,-1.2188315392,0.5913078785,-1.0695223808,-1.0774040222,0.4448950589,1.1289564371,-1.2322168350,-0.9961451292,0.1253382862,-1.2857888937,0.8030779958,0.5583163500,-0.6780901551,-0.0292556807,0.6156740189,0.6637461782,-0.2593209445,0.1577128321,-0.6255224347,-0.5165698528,1.3434090614,-0.1373769790,0.8557762504,0.7699329853,0.0583121888,-1.5436519384,1.2966266870,0.5823848844,-0.3058349490,-0.1300800890,-0.3919242322,-1.1922485828,-1.0443159342,-1.5529685020,-2.7964434624,1.2522928715,1.2292847633,-0.2496839911,-1.6875931025,-0.2166756839,-1.1366854906,1.1348092556,1.0292310715,0.2931405902,-0.0401573479,-0.9168517590,3.1208748817,-3.0247054100,0.1153336465,-0.2590649724,-0.5816124082,1.8359223604,0.8361560702,1.3724067211,1.9320363998,-0.8699535131,0.3323044777,0.9254535437,0.9190768003,2.0381422043,1.0914663076,0.7399589419,0.6411070824,0.1178541556,-0.8540045023,1.2046840191,-1.7498428822,0.1689570099,-0.7534065247,-1.4080204964,1.3100963831,2.1610531807,-0.6405741572,-1.1049610376,-0.5449877977,-0.2860692143,0.3250898123,-0.9200837612,0.9884458780,-0.5333630443,-0.6851798296,-1.0003677607,-1.5293353796,0.0621513054,-0.0578085035,0.6272996068,2.0618607998,-0.2794499397,-0.4480098784,0.1708033234,0.8488475680,-0.0460077003,1.1200356483,0.3858909607,-0.1463131756,-0.8500095010,1.1876649857,0.8287144899,-0.3788069487,0.2699967027,0.7841120958,0.9621093869,1.0886086226,-1.4207915068,0.8679859638,1.1370193958,1.1478097439,-0.0892617479,0.4420532882,0.4368574619,-0.1616665870,1.2207653522,0.3205839992,-1.3781871796,-0.5439456105,0.6632543206,-0.2686441541,-1.0038115978,-0.1014877558,0.5503932238,-1.3899176121,0.0010357754,1.5733733177,0.4335486889,0.0456469469,1.9128178358,2.6157686710,0.3550651968,2.7038369179,1.0162559748,0.6567167640,1.4713921547,-1.3117628098,1.0215899944,-0.9033085108,-0.1266045272,-1.4393845797,1.2639056444,0.7589184046,0.8724140525,0.3952097595,2.1992070675,-0.5563782454,1.0330309868,2.2867393494,-0.6047636867,1.7792675495,-0.8073445559,0.4877997935,-0.2104053199,-0.7016320229,1.4434002638,0.0233583841,0.4562973678,-0.4751683176,0.4907547235,-0.1294404715,0.5382491946,-1.3764859438,-1.3180493116,0.6598469019,-0.9768490791,-0.5608139038,0.7176390886,1.1750555038,-0.1087838933,0.8607848883,0.0236848835,-1.5650117397,-0.8263857365,0.2376575172,-0.3216416240,-0.0618195906,0.5945374966,0.3223091960,1.4338846207,-1.7596757412,0.0521411188,0.6781387925,-1.2394995689,-0.9775453806],[1.7855298519,0.4754302204,1.0716226101,-0.0919560567,1.0172238350,-0.3709467351,0.1175377220,0.3524052501,-0.0831708163,-1.4916843176,-0.0503998026,-0.6599224210,-0.2826690972,1.7006320953,-0.6317148805,-0.3335561156,-1.8706378937,0.4418658316,-0.9441236258,-0.3783243299,-0.9927918315,-0.3547174335,-0.6230579019,0.2012146562,-0.4615501761,0.6221498847,-0.2511247694,-1.4008331299,0.3153209388,-0.1327913404,-1.9306480885,0.2888429463,-0.1532785892,-0.7431642413,0.3893904388,0.5003846884,0.5587242246,-0.7005661726,-0.5150581598,0.6435684562,0.3248820007,-0.8303997517,-0.6451705098,-1.5633621216,0.4436821043,-0.5834668279,1.4078401327,0.9660906792,-0.2825075686,-2.9486505985,0.4625095129,0.3063019514,-0.4668817818,-0.8732379675,-0.3178085983,1.3156703711,1.1733013391,0.0139046516,-0.7732798457,1.6154451370,-0.7379982471,0.2065338343,-0.9543405175,0.8040487766,-0.4187107980,1.3101384640,-0.0562577173,1.0397031307,0.8087363839,1.0376975536,0.6207315326,-0.0367724709,0.1032700464,-0.7664158344,0.7779768705,-1.7194195986,-0.9962170124,-0.4599902630,-1.0737614632,0.4924404621,0.5062992573,-2.3763055801,0.3256783783,1.0295050144,-1.0963584185,1.0508966446,2.2131047249,-0.3020640910,0.3280752301,-0.0142530557,0.0741175264,0.3232976496,1.8511095047,-0.0932176560,-0.1747188568,-1.9773966074,0.8212793469,1.3709951639,1.3017138243,0.5213615298,-1.8491699696,-2.2682118416,0.7733955383,0.2187506855,1.3515372276,-0.5978830457,-0.1860491633,0.7750968337,-1.1614238024,0.5150699019,-2.3115155697,-1.1966111660,0.1801932007,0.2726337612,-0.1692290753,-0.5952160954,0.0005237133,0.6088415980,-0.3638955355,-2.2119069099,-0.6087537408,0.8907001019,-0.7753039002,0.1969347894,0.4207064807,-0.8330823183,0.0483885109,1.9549907446,-1.6341890097,0.1474251449,0.3637137711,0.9511996508,-1.1052323580,-0.9953976274,-1.4866085052,-0.2924993634,-0.0756657571,0.8346943855,-0.0225741137,-2.0840604305,0.4949536622,-0.3814873397,1.1613310575,-0.3354426920,-1.0039441586,-1.1599848270,2.5265002251,-0.3191393018,0.3884355426,0.1033144817,1.4330396652,-0.0158385038,-0.5452295542,-0.1095836759,-0.7352720499,-0.9227036238,0.0774430260,0.0953032821,-0.3727390468,-1.0648605824,-0.2965101898,0.8064007163,0.8640318513,1.0614836216,-0.4732660949,-0.9734390974,-0.4409328401,-0.4907247722,-0.4260222316,-0.3214211166,-1.2692657709,-0.6843496561,-1.0450330973,0.9267991781,1.0043290854,-1.9433656931,-0.1234595403,-0.0358733200,-1.0778970718,0.8430383801,-1.6297807693,0.4672100842,0.1302661300,-1.3048177958,0.7166115046,-0.4405407906,-1.6556293964,-0.7538868785,-0.8053990006,1.7398165464,0.8470565677,-1.0320606232,0.6498590112,0.6128501892,-0.2097587138,1.7268390656,-0.4334447682,-0.9774934649,0.0173344910,-0.8519948125,-0.2736777067,1.1822780371,-0.7529136539,1.2937862873,-0.4342118800,-0.4783442914,-0.1367436051,1.1648869514,-1.3607287407,-1.2247972488,0.5062522888,-0.3501808345,-2.3892455101,-0.3553076386,0.0785791352,-0.4684511423,-0.9138480425,-0.7515913844,0.3910466731,1.8633239269,0.0983788744,-0.5529601574,-0.8192595840,0.0123285549],[0.9051536322,-0.5296494961,0.6193763018,-0.7575354576,-0.6588627696,1.4482965469,0.1804517806,-0.0969546363,-0.8561577797,-0.1636443138,0.7998893261,0.4571678042,-0.0476388596,-1.4426069260,-0.1742287725,-0.5233820081,0.7883758545,0.3852189183,0.3091126084,0.5430634022,0.4737339020,-1.0917273760,-1.0901318789,0.4070112109,-0.9084571004,-0.7717982531,0.0192359034,0.9061601162,0.5126354098,-1.3967828751,0.7774495482,0.5376854539,0.1111765280,-0.8687179089,-0.6647869945,-0.0788185447,-0.2937527895,-1.9742597342,0.9528505206,-0.6390838027,1.2707260847,-0.6682310104,0.4715318680,-2.0942647457,1.7908589840,0.2074149996,-0.4311299622,0.4696592391,-0.6548035145,0.4858731925,-1.1523069143,0.5596313477,-1.4902211428,-1.7995990515,1.6622927189,-0.2245040238,-0.5269060731,-0.0428438373,-1.2792727947,-0.7476548553,0.0938511789,0.8331119418,1.3060201406,0.3938763738,0.9923769236,-1.6811056137,-1.1295176744,0.2734489143,-1.6697459221,-0.6384560466,0.9949476719,-0.5062407255,1.0034281015,0.0994017720,-0.0723804981,-0.0227089208,2.1757483482,0.1033674479,-1.2962023020,1.6298233271,0.0310833752,0.0043407893,-0.7852146626,-0.7798393369,-0.7882912159,-1.0835288763,0.3038004935,-0.0921095908,-1.4625827074,-1.5864539146,-1.0401488543,0.3578006923,-0.6439752579,0.6123004556,-0.8602662683,-0.0679839328,-1.9755679369,1.0582910776,0.0997800231,-1.1577682495,-1.5413347483,-0.3838081360,0.4210002422,0.4642764032,-0.0339855291,0.1298860610,-0.3884651065,-0.8865935206,-1.6143238544,3.6056587696,0.1462464035,1.5213328600,-1.0632134676,-0.5663945079,-0.1952020824,-1.3035510778,-0.0436232910,-0.4883011281,-1.0581581593,1.6403815746,-0.2546814978,0.3874523938,0.2777298093,1.1789630651,-0.2963931262,-2.3299434185,-1.0283929110,0.4161494374,0.7006410360,1.7180471420,1.0294718742,0.8499308228,-0.1957364976,-2.0873782635,-0.9732530713,0.3517284989,0.6282029748,0.7819412351,-0.6832048893,-1.0266739130,1.5057219267,-0.9331673384,-2.0270714760,-0.7710177898,-0.6412399411,-0.2176401466,1.9960709810,0.1312145889,0.7394256592,1.3804028034,-3.1236875057,0.8488565683,0.7830206752,-1.2917408943,0.4925467372,0.1587381214,-1.1164747477,1.5651832819,-1.0977840424,-0.6612092853,1.8947798014,-0.7653524280,-0.3012660742,0.4719370008,-0.8919570446,0.1150531024,0.2762588859,-0.4841531515,1.3143765926,0.2696977258,-0.2131957114,-0.6583449841,-1.3425199986,0.2763188183,1.1737834215,-0.8113678694,0.4808428288,-0.8791851997,0.2634968162,-0.6410830617,-0.0812414438,-0.6814115644,1.5104262829,0.5177908540,-0.6783463359,-0.1256770492,-0.2949545383,-0.7587720752,-0.8653795123,0.3860862255,1.2165676355,1.2671173811,0.2852470577,-0.5262190104,1.2160490751,-0.2968252599,0.6153306961,-0.6760441661,-0.0796759948,1.0532268286,1.3543426991,1.5206708908,-0.2734349668,1.6068272591,-1.2159032822,0.4170197248,-0.4496528506,0.5826635361,-0.1970722526,-0.2496076673,-1.2450256348,-0.1965055168,-0.5271847844,0.4371919930,-2.1151013374,-0.2608734667,0.0152701726,0.5723733902,0.0681537613,0.0613985993,0.3411239684,-1.2768346071,0.3332662880,0.5430427194],[-0.0432091579,1.6785727739,-0.7715849876,-0.1714796424,-0.3674465716,0.6206015348,-1.6828806400,1.1599649191,0.4741328657,0.8494206071,1.0655183792,0.8664425611,0.8542308211,0.2530686855,1.4608001709,-1.4641044140,0.9866617918,1.7032502890,0.9210450649,0.8014882803,0.3322421610,-1.5023698807,-0.3505462706,0.0389856398,0.0831000507,-0.3857187629,-1.5359857082,1.2615644932,1.2758626938,0.1480398178,1.2954460382,-0.7860206962,-1.1860078573,-2.2061941624,-1.0470296144,0.3352247775,0.5302492976,-0.0405291393,1.4120001793,0.4016290307,-0.1539154351,0.9125192761,-0.5446004272,1.2182688713,-0.8864365220,1.1918206215,1.0045500994,-1.6376503706,0.9360725284,-1.2072974443,-0.7660700679,0.5314910412,-0.8932678103,1.1890400648,0.3187058568,1.6210167408,0.4157439768,-0.4446620047,-0.6844602227,-0.8145654798,0.6531211734,0.8158108592,1.4798234701,0.4006395042,-0.7666428089,0.9254111648,-0.3484019637,-1.1469308138,-0.7036637664,-0.0442226976,-0.2382203043,0.4113468826,-0.5789852142,0.7120308876,-0.6015210152,-1.1392376423,1.1177431345,0.4594758749,1.5914516449,-1.0618811846,-0.8215261102,-0.7494154572,0.7032563090,-0.1234351471,0.0701818541,-0.5553544760,0.0761191547,0.3339363635,-2.1919014454,-0.3116528094,0.8165085912,0.2899995446,0.2868409753,2.1124336720,-1.1487253904,-1.4357818365,-0.3476918936,-1.3994435072,0.8405013680,-0.9114860296,0.4391860366,-1.6877720356,-0.2196552604,1.3988096714,-0.6408332586,-0.1922979355,-0.2654271424,0.0860480741,0.2465860546,0.2520399392,2.9076354504,-1.4887257814,0.6890859604,0.4454374611,0.6194879413,-0.0967055187,1.1441223621,0.1449663192,-0.5741001368,0.0515831709,2.8111915588,0.2500806749,-1.0168375969,0.0459030941,0.5006407499,1.2243378162,-0.5594944358,1.5234234333,-0.5856988430,0.8465883732,-0.1062590182,0.7699720860,0.7508335114,-0.5606443286,-1.7602663040,0.4370567799,0.3021217883,0.1467457712,0.6101540327,1.5521664619,0.0121918637,0.0001930556,-0.7419973612,-1.3801542521,-0.0754940435,1.1674489975,-0.0283292662,-1.2254366875,1.6937772036,0.7008804679,0.1472029239,-0.2088548094,-0.9180703163,-0.9908519983,-0.8559581637,0.5744030476,0.2971726060,0.7592765689,0.5797044039,-0.3228662908,-0.3427666128,-0.1186992005,0.2182934582,0.9786879420,-1.4661244154,0.7656714916,-0.3914667666,-0.4707569182,-0.0234617610,-2.1558485031,-0.8340327144,-0.4761915803,-0.1583889872,-1.3076349497,0.6191383600,-0.3673848808,0.4211728573,0.0395293981,-0.2179098874,0.1381019950,-0.0317963026,-0.4639281929,-0.0641642138,-1.1540219784,-0.9627858996,-1.6770368814,-0.5858719945,-0.0825671777,-1.5955755711,-0.0500167795,-0.0760848224,-0.0237301663,-0.0268115401,0.2086068839,-1.6112877131,0.0350952968,-1.4237899780,-1.2756197453,-1.6744543314,-1.3942058086,-1.4011231661,0.5572247505,-1.6251987219,0.0624367483,-0.9734458923,1.9991265535,0.3975417614,1.9408439398,-0.4244121313,-0.7449286580,1.5648454428,-0.1315009445,-1.1770961285,-0.1874196529,-1.0828311443,-0.4274128377,-0.3763251603,-1.8707411289,-0.3534707427,-0.0243204199,1.9254937172,0.5000748634,-0.0090672439,-0.1229096130],[-0.1555079669,1.1247876883,0.6508572698,1.3430335522,0.3828526735,-0.7241315842,-0.0204289500,-1.2060266733,1.8055220842,-0.0309408773,1.6533294916,0.9167777300,0.6583053470,2.4935271740,0.1820680201,-0.4153838158,-1.7359157801,-0.9003521204,1.3195201159,1.1472709179,-0.3788455427,0.3824750483,0.5139021873,1.7121081352,1.1626836061,0.1476384997,-0.5816458464,0.2407868803,-1.5572460890,-0.9878185987,-1.6791173220,-0.1426104307,-0.6590248942,-0.6210706234,0.9594504833,0.2867678702,-0.0755748749,-0.0945067704,1.8625407219,1.0528297424,-0.6334456205,-0.5078506470,-1.3274567127,-0.7107422948,-0.2926018834,-2.0036118031,0.1115319952,-0.0476051159,-1.3324609995,-0.7805432081,0.3220021725,0.9446145296,1.6483644247,-0.2507813573,-1.4725340605,-2.6068005562,-1.0014157295,-1.0183038712,1.2119328976,-0.8830393553,0.5760194063,0.4326705337,-0.6804663539,1.8935378790,1.0359658003,-2.4166157246,1.1839493513,-0.2268783003,-0.5061920881,0.9377138019,-1.2114729881,-0.7506918907,0.4980424941,0.8761253357,-0.4148249626,1.1402990818,0.1848191768,-1.8540214300,-0.8729506731,0.8739518523,1.9581677914,2.4186632633,-1.1326782703,0.5931650996,-1.1454510689,0.1306309998,-0.7470979095,-0.5057881474,-0.9316557646,0.0993926078,-0.7553661466,0.1219220161,-0.2455041856,0.3656617999,-0.8616326451,0.1999523640,0.1532318890,-0.7006615996,0.4613142610,1.3785855770,0.6721458435,0.3734226227,-1.3855634928,0.4933677316,-2.0586693287,-1.8312758207,-0.5378765464,0.1734742075,1.1764724255,-1.6062233448,0.9842114449,-0.1152400449,-0.0080436319,2.1545655727,0.9554353952,0.1760829836,-0.4261439741,-1.0313150883,-0.4449267685,-1.6515427828,0.8663641810,-0.3967720568,-0.3075744510,-0.8735423088,0.6620267034,-1.0369021893,0.5346903801,-1.5483399630,-0.3606949151,-1.5275582075,-0.1260908693,1.1827498674,-1.3001586199,-0.8946741223,-0.4685489535,-1.0407261848,0.2360029519,-0.1681945175,0.7322675586,-0.1572569311,1.2127138376,0.0375163071,-1.0115327835,-0.0792735964,-1.0087019205,-0.2862460911,0.9906303883,1.2327841520,1.3395975828,-0.1617410332,-3.1198022366,1.2752115726,-0.9587536454,0.0452648103,-1.0152710676,-0.5220885873,1.2208900452,-0.0604511015,0.6024882793,0.4948946238,0.0399792418,-0.2535513341,1.5232809782,-1.2082173824,0.1494268030,0.7835847139,-0.2103703767,-0.0482308120,0.1537257582,1.2463809252,0.6994246840,0.5315209627,2.1850254536,0.4130679369,-1.9432587624,1.1612490416,0.5338376760,-0.0464589708,1.1988745928,-0.9518498182,0.1815148890,-0.7856477499,-1.3307493925,1.2344479561,1.0216497183,0.6948999166,1.3199999332,-0.3467248082,-0.6526562572,-0.2523265183,-0.4008188844,0.8922727704,-0.4961834848,-0.2049113363,1.6276949644,-0.1190614551,0.1953179687,-0.0473487377,-0.3821465373,1.0792948008,-0.5341405869,-0.7568749189,-0.2242150456,-1.9106316566,0.4949327111,0.8403900266,0.4573747814,-0.8882073760,-0.5454280376,1.3229922056,-0.0953648686,0.1615354121,0.3111441433,-1.2016353607,-1.1228525639,1.5913999081,0.4555059075,-0.0846310630,0.0696771741,1.3984402418,0.8977686167,-0.0743149444,-0.8610787392,-0.0034345402],[0.4643716812,0.6500335932,0.3972373605,0.6056055427,-1.1137615442,1.2621937990,-1.0029988289,0.7454153299,-0.9797261357,-1.0222493410,0.2281863987,-0.2279123515,1.2973057032,0.7009657621,-0.1237428635,0.6601970792,1.6168614626,-0.1405349374,0.4543885589,-0.9073397517,-1.3353450298,-1.4128642082,-0.8668837547,0.2140561193,0.6704117656,0.2983856201,0.1711228490,0.0040384457,0.2274931222,-0.5629349947,-0.2145272195,-0.6500881314,1.0181735754,-1.0987073183,-1.1695632935,1.4457358122,0.9086647630,0.5250781178,1.1210864782,-0.5814261436,-1.9068636894,1.2717565298,0.3779475689,1.1875561476,2.1112318039,1.3692845106,0.4907457232,-0.1053989232,1.7720201015,0.3179871738,0.6021529436,2.0522155762,-0.8119975924,-1.2906919718,1.9221531153,0.2619466484,1.4544030428,-0.0270028636,-1.0798466206,2.4531943798,0.4816752672,2.3576691151,0.8212106228,0.6885333061,-1.1239271164,-0.3848465085,-0.5353651047,1.2736561298,-0.4609678388,0.1968630999,0.5500658154,2.0417609215,0.5277529955,-0.4230228662,-1.5167474747,-0.4852030277,1.0694479942,0.3578872681,0.3670755923,0.3847409785,0.5441589952,-1.1518552303,-0.3627217412,-0.2236316949,-0.1309448630,0.3189993501,0.7243591547,0.6128135324,0.6833281517,0.1058774963,-1.0476717949,1.2348972559,0.1849824041,-1.0603389740,2.7461400032,1.0114446878,1.3566025496,-0.6985847950,0.9919859767,-0.1195081174,-0.5770586729,-1.6611901522,-1.4490962029,-0.1540308297,-0.0394946598,-1.5549813509,0.4014787078,-1.4460828304,0.3835073709,-0.2769566774,-0.1825580746,-1.3795892000,-1.0845396519,0.2838324010,-1.9333698750,-1.5066390038,-0.8090303540,-2.3525457382,0.3431841731,-1.8326971531,0.4958780706,-0.6096104383,-0.1282366365,0.0555926375,0.8264662027,-0.5319551826,-1.3494676352,0.0178227182,0.2657372057,1.5358214378,1.4378138781,-1.5428961515,-1.0020878315,-0.1664336324,0.3720614016,-0.9722996950,0.4478465617,-0.5604529977,-2.8267419338,-0.1732289642,-1.3194926977,0.7245579362,-0.4114677310,-1.0185942650,0.0171113219,-0.3274767399,0.8993369341,0.4165078700,-0.4409069121,2.3318879604,1.0301353931,0.0106273629,0.5165274739,0.3694303930,-0.7169600725,-2.0160875320,-1.5899252892,-0.7279121280,1.0887588263,0.2646250725,-0.2574935853,0.2369214147,-0.1405727863,1.1408565044,1.3749165535,-0.3872929215,0.3741849363,-1.1731528044,-0.8771096468,1.7028417587,0.7422641516,0.7952908874,-0.6359503269,0.0625767857,-1.4387636185,0.7526410222,0.5943829417,-0.6834931970,0.4352502227,0.0361339785,-0.3459370732,-1.3282161951,-0.3925054371,-0.3875874877,-0.4427610338,0.6347910166,0.7603368759,-0.4462294281,0.2494061291,-1.7198584080,-2.5241043568,1.0394392014,-0.8128814697,-1.3440935612,-0.9509823918,0.5185998082,0.6612313390,-0.3765857816,1.2585207224,-0.2729130089,-0.1594479978,-0.8824411631,0.3794788122,1.0085316896,0.3316577375,0.5210019946,-0.2534552217,-1.6504944563,0.1905167997,0.3426152468,-0.4149730504,0.4229606390,0.5006102324,-0.6884571910,-0.0085293930,0.2035993785,1.6628166437,-0.7422149181,0.4549663663,0.9701353312,-1.4671440125,1.0306642056,-0.1751043946,-0.3081717193],[-0.1086230949,-0.3445905149,0.3018523157,0.5331122279,-1.4184136391,-1.3059164286,0.0133221578,-0.4561957717,2.0107197762,0.0168454461,-0.5268836021,-0.5163084865,-0.2862188518,1.9081737995,0.3808764815,1.1053049564,-0.1701475978,1.8028768301,1.3454405069,-0.9716171622,2.0151402950,0.4353454411,0.2021296322,0.3094969988,-1.2130287886,-0.6061766148,-0.8579288721,-0.5894806981,0.1631432176,0.1187847629,-0.4992081821,0.6937823892,-0.3709974885,0.3257789016,-0.3987151682,-0.7485961318,-0.2293497622,-1.0146118402,0.0110471090,1.7328293324,1.0677479506,1.2207168341,-1.9264101982,1.5223257542,0.3796040714,-0.3781650662,0.1991085708,-1.2878689766,1.4412564039,-0.3321222961,-0.0153004900,-1.0780576468,0.9807602763,-0.1083590388,0.3219371140,-1.0011390448,-1.0939098597,1.0125167370,0.2617959380,-1.9774245024,0.8176585436,-0.5400171876,0.8523032665,0.0277114268,2.1040093899,-1.4332144260,0.8153598309,0.4336749613,0.6829534769,0.6602942348,0.4276879430,-2.8726365566,-0.9237739444,-0.4451941550,-0.1291576624,0.0826604590,0.2901602387,0.1507606357,0.4976918399,1.9305351973,0.6905815005,1.1261466742,0.1136901900,0.4927196503,2.1992692947,-0.7340601087,-0.6472967267,1.2486354113,0.4821243584,-2.0340688229,-0.7239329815,-0.0443657637,0.1745765060,-0.6582645774,0.4640335143,-1.2629460096,-1.6618301868,0.1800085753,0.9277454019,-0.2590980828,-0.7413901687,-0.6935709715,-0.5875934958,1.2377283573,0.3266763985,0.0787180141,-0.5542194843,-0.9741730690,-0.4150099754,-1.5515369177,1.0647892952,0.6235165000,1.3300273418,-0.3489451408,1.3718315363,-0.6768189669,1.3851624727,0.8357720375,-0.6882252097,-1.0305742025,0.0712251291,0.3130685091,-0.6692923903,0.9052766562,1.9800167084,0.9550880194,-0.7589914203,-1.5862555504,1.5604054928,1.0711680651,-0.1050480083,1.1862883568,-2.3280029297,-1.6582649946,-0.3555588722,0.8859654069,2.1518859863,-0.7442384362,0.6685908437,-1.1250841618,-2.7138724327,0.0137464041,1.3366407156,-0.4603686929,-0.8399344683,-0.4507792294,-1.4983799458,-0.8027215600,1.6966629028,0.4295882285,1.5084462166,0.4342085719,0.1687584966,-0.2440459728,-0.1066379622,1.1238811016,-1.6822667122,-0.9517872930,0.4881279469,-0.0958632976,-0.4737888873,0.4130543172,-1.2868320942,0.0282274242,-0.0538525246,-0.6100097895,-0.4368116558,0.5414137840,-0.6348329186,-0.6271486878,1.1452444792,-0.3874225318,-1.0906164646,0.3533582091,-1.7489846945,0.8108950853,0.1281350255,0.9760327935,-1.0696296692,0.8331253529,-0.9128801823,-0.5867547989,-0.1854138970,-0.8016400337,-0.0122459978,-0.2811549902,1.5724511147,0.8933423758,1.5236115456,1.1318390369,-2.1643643379,-1.1678924561,-0.5303066969,0.1001020819,1.5827381611,-0.3443536460,0.4299806058,-0.6880388856,2.3481752872,0.8799027801,-0.1208624095,0.3161224425,0.1859720945,1.3103215694,0.5370991826,0.2128050774,-1.6082152128,-1.0610541105,-0.3076724708,-1.3912265301,2.0203952789,1.2839845419,-0.1427117586,0.7247159481,-1.4809105396,0.4151991010,0.7133069634,-0.9822463989,1.0256047249,0.8960664868,0.2456524372,2.2871968746,-1.1332281828,-0.9934768081],[0.0801362023,-1.4279161692,-0.1673499644,1.9552475214,-0.7649120688,-0.1214441359,-0.8505011201,0.0120534971,-0.2664821148,0.1010014117,-0.0249482337,0.4609630108,-0.4709973037,0.7293856144,0.3688291311,-2.2109615803,1.2869336605,-0.3740674257,0.0017372227,0.1274275184,-0.0548788235,0.4981512427,2.0514848232,-0.3548823297,-0.5679026842,-2.2651638985,-0.9933956265,-0.4226258695,0.5724143982,0.6467649937,-0.5059992671,0.6243510842,-1.3668729067,-0.5580800772,-0.2308873683,-0.4146345556,-0.8414208293,-0.8874806166,0.8085399270,-0.7730150223,1.0769597292,-2.2644515038,-1.7992531061,-1.5193790197,0.6511667371,2.0010735989,-0.4351148307,0.5123409629,0.4797986448,-0.3782280385,0.3045934141,1.0081927776,2.1089031696,1.4194885492,-0.7833939791,-0.7446916699,0.1809913516,-1.4201674461,1.5365930796,1.5332006216,-0.0955717042,-0.0791906789,1.2793784142,0.2346884012,0.6085548997,-0.7300610542,-0.7544391751,-0.0971425176,1.7628134489,-0.5690230727,-0.2787840366,-0.2442644686,1.3795033693,0.6344102025,0.4248405397,-0.1316951513,-0.4010356665,0.9959358573,0.4057629108,1.0247602463,-1.6310348511,-0.1843463928,-0.0751169100,1.0483053923,0.8583419919,1.0581882000,0.5709701777,0.1121916398,-1.2642484903,-0.7674326301,-2.1037101746,-0.3202399611,-0.6719353795,-0.8261151910,0.7101277709,2.6124789715,-0.1418309510,-1.3277183771,1.0787343979,-1.2282276154,-0.1210674867,0.2911917567,0.6180588603,-1.4232866764,0.5941316485,1.0013989210,-0.0831873789,-1.4150587320,-0.3144525886,0.2749459743,1.5043956041,-0.8493507504,-0.3435238004,0.1091308668,-1.4348704815,-0.6094563603,-0.3990669847,0.1706257463,-1.0890892744,1.3205559254,0.2972381413,-0.7722620368,-0.3236086071,0.4466733932,-0.4585110843,0.8907366395,0.3113555014,-1.0298800468,0.6594182849,2.1507937908,-0.5677946210,0.3051919937,-0.3771156669,-0.5940546989,1.2152264118,-1.7259300947,-1.0823833942,1.3255735636,-0.8910102844,-0.0269354563,-0.1528990567,1.4183866978,-1.6457661390,0.8868795037,-0.8474878669,-1.3936561346,0.0618900172,-0.5172175765,3.1002268791,-1.4958193302,0.3102186024,-0.9779730439,1.1106261015,0.7437641025,1.4932644367,1.0055906773,-0.7624963522,-0.4558341801,1.2478998899,-0.0513665862,-1.0852686167,2.5063498020,1.4703984261,-0.3500232399,0.6674960256,0.2232310176,0.7430081964,1.4739216566,-0.2805545330,0.0959332883,-1.2647999525,-0.8458613753,0.6121600270,0.3654181659,-0.5369811654,-0.4391386211,-2.1446189880,0.8581434488,1.3331199884,0.0303843282,0.6091219187,1.1128925085,-0.6695650220,-0.2710272074,-1.6208012104,-0.6248922348,0.5111383200,0.4374771416,-0.0601305887,-0.3222166598,1.5219869614,-0.2043878138,0.7294411063,0.5184146762,1.4763427973,0.9772499800,-1.1257607937,0.1171994582,-0.5923586488,-0.3355710208,-1.0741111040,-0.5246708393,-2.2213320732,-0.1392166018,-0.4191494286,-1.0089921951,-0.3403863013,0.5482953191,-1.1640391350,-0.3767908812,0.5043502450,1.2352068424,-0.0104251159,-0.1011670530,-1.4769479036,1.0991642475,0.9631522298,-0.1085720211,-0.3289594054,-0.1785956621,-0.1722016186,-0.9106789827,-0.0757652223,1.6013062000],[2.8825588226,0.0283018276,-1.4420264959,-1.0566796064,0.3184948862,-0.7164399028,-0.6493185163,-0.9150819778,0.1000853702,0.6208550334,-0.1778074652,1.1770408154,-1.3932850361,1.4212722778,0.5371090174,0.4546726048,0.4049543738,0.3981168866,0.0488865748,0.8620789647,-1.1955425739,1.2540414333,-2.0386204720,-0.0258903690,-1.3946413994,1.7492851019,-0.3227463663,-0.0915191621,-1.4765635729,-0.0714700073,-0.4856380224,-0.3160601854,-1.1019278765,-1.4820511341,0.4517401159,-1.3478686810,-1.5971720219,0.4214863181,-1.2817587852,-0.3930203617,-0.1581448764,0.6314710379,-1.2951623201,-0.1740406007,-1.0108921528,1.2809277773,0.6833066940,0.6616382599,1.7153782845,-0.6153312922,1.5842894316,0.7544012666,1.6303297281,0.1301917434,-2.7217433453,0.3398594558,-0.8163784742,0.1465732604,-1.2905712128,-0.4101938009,1.6654448509,-0.2119739801,-0.7760418653,-0.4227268100,0.6563307047,0.2659547925,-0.0037535611,-1.0131603479,-0.2349339277,0.9261472225,0.3318826854,0.3253736198,-2.6372146606,0.2674591243,-0.1940494925,1.3571807146,-1.6915826797,0.5772549510,0.8997503519,-1.3403272629,0.7346234322,1.5062402487,-0.1679347157,1.2141203880,-1.6488654613,0.6915194988,-0.2572011054,-0.1990943253,0.2498833686,1.2450773716,0.3484862745,-0.2589534819,0.7116019726,-0.0218918305,0.5702000260,-0.9457767010,0.6703088284,-0.1766744554,1.3134074211,-0.1710943580,-0.2920600772,0.9888873696,0.6314431429,-0.6614246964,0.9294165969,0.2508786917,0.5414218903,0.2954601943,0.8805912137,-0.0731926933,1.7276763916,0.3145478964,1.1606113911,1.0999108553,-1.5043370724,-1.5866116285,0.0370911658,1.3214834929,0.0746138021,0.9906339645,0.1499240994,0.9241827726,-1.4057812691,-1.2694085836,-1.2315986156,-1.0792061090,0.9238930941,0.3661459982,-1.6559048891,0.3552600443,0.1590854973,-2.0836281776,-0.3751182556,0.8667919040,0.2102769166,1.4756578207,-0.5184097886,-0.0366294533,0.8832123876,-1.0546194315,-0.1811883897,-0.1210824773,-2.1764533520,-1.5692141056,2.0706942081,-0.9752756357,0.1462701559,-0.1786944121,-0.9237043262,0.2103956491,-0.1442062855,1.2261948586,-1.5112875700,1.5062558651,0.0944361016,0.7306627631,0.6643269062,2.2133276463,0.8330996633,-1.4814969301,-0.3682819903,-0.4991627038,0.5526918173,0.3599629402,0.6244876385,-1.0617635250,-1.0455168486,-1.1398391724,-1.3221683502,0.0184009988,-0.5667765737,1.8121285439,-0.9326394200,-0.7341508865,-1.1592438221,1.2299790382,1.7348306179,-0.7335332632,0.8989298344,1.2027136087,1.1966621876,-0.1625650376,-0.1017761528,0.2831718922,0.4695369601,1.1291228533,0.6805573702,-0.0990784019,-0.7106087804,0.2395488918,0.1024907678,1.7224653959,-0.9518821239,-1.7251338959,0.9914779067,-0.9855437875,-0.3009017408,0.9082549810,0.0410030857,-0.0605210140,1.1658284664,-0.9308250546,0.8737416863,-1.2085171938,1.0346195698,-0.5344747901,-0.2332480103,0.6903033853,0.1028862447,0.8828144073,1.8908827305,-0.7360958457,-0.2265916616,-0.5171056390,0.6966750622,0.3447278440,-0.0323995762,-0.6886252165,-3.2709035873,-0.8757738471,-1.4021294117,0.1837808639,-0.9577113390,1.2251466513],[-0.2875821292,-0.2114524692,-1.4256474972,0.8641709089,0.0528381169,-0.0744572952,1.5797815323,-1.1734684706,-0.0783758909,0.2729905248,0.5856707692,-1.0022348166,-1.2545819283,1.0302791595,-0.3580471575,-0.9342455268,-1.9813054800,-1.5998274088,1.3066407442,0.4830575883,1.6324607134,0.1079617515,-0.4973579049,-0.5194500685,-0.3114619255,0.1192606017,-2.0525612831,0.2186866999,-1.5327219963,-0.4967102706,-0.1050102040,-0.0578927137,-2.1460046768,1.3595274687,0.2326336652,-0.1665844172,-0.4899091423,0.7253504395,-0.2241188586,0.0341161974,-1.7404079437,0.5003447533,0.4687319100,1.5334626436,0.2852185965,-1.5014708042,-0.4241717756,-1.2355568409,0.0657551959,1.2892919779,1.2911703587,0.6110507250,-0.0205472484,-1.2269330025,0.0883116126,-1.0893718004,-0.6376819611,0.5561304092,0.0832567215,1.6728909016,-1.2436583042,0.1721850187,0.6242291331,0.3138146996,-2.2066717148,0.2796196043,1.0710690022,1.3156319857,-0.0950455889,-1.5921502113,-0.9913931489,0.1297880560,1.4663813114,-0.1556957811,-0.8660828471,-0.4202181101,-0.9978961945,0.5445381403,-1.5141788721,1.9711934328,0.5910512805,-0.9280182719,-0.6373056173,0.4286117256,0.7815433145,-0.0861902535,0.0154004376,1.5583499670,-0.5532020330,0.2428787202,-1.3022339344,-0.6369246840,-0.1703581214,-0.3426039219,0.8764404655,0.5651657581,-1.1531502008,1.3143377304,-0.6532971263,1.3424408436,-0.5932373405,-0.3148380220,0.3417423964,0.0807463825,0.7385244966,0.1785105616,-1.0377227068,2.5720541477,-1.6021740437,0.3730979860,-1.0089817047,0.2074581236,0.7690361142,0.1414442956,-0.0658927634,1.2981730700,0.5637852550,-0.7915480137,1.1714626551,-1.0327466726,1.2980322838,-1.8191382885,-1.5950011015,-0.0475060828,0.1701105088,-0.1173396930,-1.1642503738,-2.2592539787,-1.0229673386,0.0272143669,-0.0346517377,1.9163516760,-0.3508054614,-0.5195938945,0.7642535567,-2.5424456596,0.0920520052,-0.5108087659,-0.8645259738,0.8696979284,1.5335197449,0.1574706435,0.8634047508,-0.7973005772,-0.7276571989,-0.0082663214,0.6169670820,1.2819828987,1.0705560446,-0.3476598859,0.8035858870,-0.2554123700,1.5247392654,-0.3427659571,-0.2781977654,-0.0387262739,-0.2124831229,-1.0138919353,1.1995728016,-0.3768789768,-1.0625375509,1.2412786484,-0.0552544296,-0.7425494790,0.5503547192,-0.2265021652,-0.5776142478,-1.0426775217,0.0635304376,-1.0789164305,-0.0305424035,-0.4114821553,0.8515140414,-0.8038818240,0.1948237717,0.1175146028,-1.1841410398,1.4999989271,0.7257269025,1.1487137079,1.6531249285,-0.9639879465,0.9420316219,-0.6486244202,0.2358347327,0.2451820672,-0.9946019053,-0.4021207988,0.6609389782,-0.6528478861,-0.7113485336,0.2650088370,0.6931459308,-1.5257225037,-0.5300140977,0.4230892360,-0.1780603528,-0.7085021138,-2.0326261520,1.1543872356,-0.6382230520,0.1267194450,1.1673766375,-0.1194199398,0.5173939466,1.3585667610,1.1757655144,1.2875258923,1.7019704580,-0.3240980208,0.3856199980,-0.5278858542,-1.4395867586,-0.4803539813,-0.7832537293,2.5728754997,0.6120492220,0.1088558584,-1.4145896435,-0.0893893540,-1.1972055435,-0.8456657529,2.5421011448,0.8142038584],[-0.4200177789,-1.3778551817,-0.1695551574,-1.4214105606,0.3386326432,0.4762045741,-0.8144193292,-2.0145246983,1.3744714260,-0.0912032276,-0.5277309418,0.3268625140,0.0133851767,-0.5244994164,-2.6017088890,1.9043643475,1.1018794775,-1.4449181557,0.0988714769,0.1080312133,0.2872729599,-1.0176873207,-1.5658758879,-1.4753624201,0.6132729650,-0.6109262109,0.7954557538,0.3561187685,-0.6416935325,-0.8037799597,-1.0986231565,0.6042654514,-0.0362247378,-0.4225933254,-0.0711886138,-0.5749738216,1.0292119980,-0.0141652646,0.3381133080,0.2150502056,-0.4366036057,1.1847845316,1.2134108543,0.0970221311,-0.3986929655,-0.3410035074,0.5849024057,0.8327379227,-0.3633055687,1.1413456202,-0.3347489834,-1.4785804749,-0.6094909906,0.8936745524,-0.2556134760,-0.1951664984,-0.6637004018,-0.9761881828,0.5172336698,1.0505046844,1.0351791382,0.9528251290,-0.6650516391,-0.5152972937,-0.5114353299,-0.5137189627,0.1342589110,-0.4055825472,0.9870741367,1.1039853096,0.7906116247,0.1031517833,-0.5891082883,0.4176992476,1.1658488512,0.9659720063,0.0746390373,-1.0539230108,0.8605904579,1.0937052965,1.4656609297,-0.0631425753,-0.2436380386,0.5220047235,-0.5257426500,-0.3511520624,0.1896903366,1.1232941151,0.9083298445,0.6321584582,-0.2297795564,0.4393576384,-1.1335648298,-0.1767812520,-0.9877588749,-0.6336392760,0.6270663142,0.5877546072,-0.2975437939,-0.6329882741,0.9075940847,0.1420644522,0.4181271493,-0.8419764638,1.3827379942,-0.2564235032,0.4914464951,-1.2188130617,-1.7116364241,-1.8048491478,-1.8463013172,0.3928754032,1.4446197748,1.7741932869,-0.2339616269,0.8514254093,-0.7865287066,-0.4787963033,0.8111802936,0.5455805063,-1.4866166115,0.5192504525,0.5673210621,-0.2531650364,0.4797608554,-1.7640885115,-1.7594573498,-1.0405248404,-0.6443892717,-0.0463766120,1.5923037529,-0.9849032760,0.9268825054,-0.3332609534,0.1753576994,-0.4664939940,1.1735558510,-0.4948464334,-0.5844563842,0.7905429602,0.9834874272,-0.5397875309,-0.3261406720,0.5392394662,-0.5751277208,-0.2247804999,-1.1403326988,-0.9152319431,-0.3199635148,-1.0563613176,0.4798598886,-1.5495011806,-0.2683282197,1.0509999990,-0.8671765327,-1.7904404402,-0.4655726850,-0.8255794644,1.2549281120,-0.5127987266,0.3980410993,0.5606667399,0.2152752578,-1.6999623775,-0.0291132275,1.8648895025,0.6139222980,0.0244569257,-0.9495642781,-0.0074470057,-0.9770514369,-1.3872021437,0.1106198654,-1.5846104622,-0.4539607763,-1.1245915890,-1.2696129084,1.3264654875,-0.6482177973,1.0197538137,-0.7960487008,0.0526552238,0.1811360568,-1.6001381874,-1.2790617943,-0.7372736931,0.8097754121,1.8652113676,0.6001631021,0.4037535191,-0.0492363907,1.1228337288,-0.6498822570,1.0116053820,-0.2541756630,-0.5277975202,-0.2402507961,-0.1130158454,1.1338608265,-0.7202472091,0.5499478579,-0.9729791284,-0.8757820725,-1.8153737783,-0.6840500236,0.0959931836,0.5253782272,1.0612294674,0.0080760587,1.4577175379,-1.3004442453,1.9005447626,-0.7658629417,-0.0105433399,0.8367369175,-0.6676630378,0.9297543764,0.9685356617,-0.0386456437,-0.4531649053,0.1079624817,-0.0924528167,0.6686045527,1.7595429420],[0.8170405626,-0.6267473102,-2.5353696346,-0.6208639145,0.0412966460,0.5412843823,-0.6714181304,-0.1228801236,-0.3316660523,-0.1653130352,1.4541288614,0.6057051420,0.7982510924,0.1380858123,-1.4818567038,1.8989332914,-0.9859840870,-1.2145018578,0.7614639997,-0.7683216333,-0.4145599902,-0.4076405764,-0.2403526902,-0.5470996499,0.9964763522,-1.8223305941,0.4747248292,1.5665532351,0.9140262604,0.1729218364,0.5444464684,-0.1636365354,-0.8873438835,0.4439423680,0.6227127910,1.7554078102,-0.4988125265,-1.3193113804,-2.5715079308,0.1748979837,-1.1597133875,0.9316927791,0.1183342338,1.0189135075,1.0244257450,-0.5853592157,0.9235587716,1.2721583843,0.8519009948,-0.4364132583,-1.2144123316,0.2642391622,-0.3939892352,-0.0805365071,0.4169847965,-1.6299033165,-1.9063054323,-0.9444448352,0.9864621758,-3.2844877243,0.4394325912,0.5473648310,-0.2011999935,0.6504402757,0.3405825198,0.5267879963,1.7770034075,0.5222274661,0.9281985760,-0.2189456522,0.3734814525,-1.2006289959,0.2925578952,1.1988446712,0.7234704494,-0.4607257247,-0.8953572512,-0.0688983947,-0.1996186823,2.5704195499,0.2287005633,1.5053178072,0.0987073630,0.4207965136,-0.1668317318,-0.3914250135,0.0472337753,0.8962624073,-0.6044977903,-0.2340323478,0.3957251310,-1.4881333113,0.4008922279,-0.2279271781,2.0719885826,-0.5289013982,0.3241880536,-2.6822669506,1.7045650482,-0.1832091957,-1.3438804150,-0.1880180538,1.0730087757,0.3521874547,0.2114675641,0.5612317324,0.9730195999,-1.4720804691,-0.0725681335,-1.4717452526,1.6405420303,-1.4896878004,0.0528693758,-1.7767858505,0.9301193357,1.7837607861,0.2781662941,0.1405596286,-0.3057101071,-0.2079758793,0.5234370828,-0.5637179017,-1.4416309595,-1.0048124790,0.4227478504,1.0247695446,-2.4148521423,1.2287952900,2.0506439209,-0.3796081543,1.4330184460,-1.6261191368,-0.4310197830,-0.3651142120,-0.8131893873,0.1408641189,-0.0862338468,-1.0842740536,-0.4975840449,-0.8313937783,-0.3324953318,-2.9499680996,-1.3614028692,-0.0652090460,1.5336951017,0.6620618105,-1.4137949944,1.0900887251,0.1952648610,0.4613917172,-1.3025218248,1.1172809601,-1.0675442219,-1.5549499989,-0.2244365811,0.5666527748,-1.8053593636,0.2591793835,1.1105011702,-1.9888157845,-1.3004876375,0.9484360218,-0.1045219526,-0.1354653537,0.7844735980,-0.3132534623,-1.7118363380,-0.1989667118,-0.2725273967,1.5450774431,-1.2673224211,1.6919636726,0.2171541452,2.2025887966,1.4983998537,0.5503693223,1.3401776552,1.4544260502,-1.9329987764,0.1200451478,-0.9440642595,-0.3997438848,0.7572956085,-0.0289923865,0.4980324805,0.4981862605,0.6089060307,0.2215690613,2.8459966183,-0.3495443463,0.3898267448,-1.3111178875,-1.4949693680,1.5423781872,0.4355337620,-0.2667261362,-0.6964426637,-0.0315350629,-0.0684453547,1.2804672718,-0.3476506770,0.3679663539,-1.9576956034,-1.5535807610,0.3000622094,-1.0038506985,0.0751238614,0.0237246975,1.8045992851,-0.9453797340,-0.2001632899,2.1614372730,-0.9781127572,-1.0257860422,0.3228924870,-0.0656457618,-0.5376325250,-0.5086116195,2.1152424812,-0.4741887152,0.9335947037,-0.9201823473,-1.5525681973,-0.8249668479],[-0.2767834663,0.3954752684,-2.5128004551,-0.5187504888,1.7483252287,-1.5200672150,2.7166440487,-0.3970314860,-0.8433889747,2.4272284508,0.2976959348,-1.3999366760,0.9331608415,-0.6598845720,0.7047004104,0.4617165029,1.2049220800,-0.4600013494,0.7962696552,1.4818439484,0.4008847177,-0.4036520720,1.8412313461,-0.1604305208,0.0991646573,0.4628383219,-0.9043536782,-0.8916121125,0.4315722287,1.3325688839,1.9264107943,-0.6885731220,0.3321375847,-0.2413543910,1.0912894011,1.1436660290,-1.4943883419,0.4226326942,-0.3977138102,0.6066868305,0.1816943884,-0.0278797448,-0.7122353911,0.7773500681,-0.3370968997,-0.0819773525,0.3859794736,0.0197459012,0.7838968635,1.1054192781,-0.0492523052,-1.1183481216,2.0457713604,1.4440535307,1.4088461399,-0.3506045938,0.9419050813,-0.4516931474,-0.0231885668,-0.2311489135,-0.5307110548,-0.6097031832,1.5688374043,-0.0618275702,-0.3840580285,-0.1462161839,-0.7490805984,-0.4684410393,-1.1734311581,0.2761482000,1.6468441486,-0.4369893670,0.6186989546,0.5075575709,0.2093936950,-0.0073280954,1.4158865213,1.3943082094,-1.1606276035,-0.1925585568,0.7860649824,-0.1255825013,0.7291187644,-0.0382035486,0.8796746731,3.2665982246,-0.1572821438,1.2827008963,-0.2016982436,0.0290385038,1.2065564394,0.5020515919,-0.0758089051,-0.5608183742,-2.4836220741,0.7568008900,1.7811390162,1.6835224628,-1.1110769510,0.3087627888,-0.5306309462,1.4413764477,0.0940743685,-1.1490490437,1.7519093752,1.1665536165,-0.5972436070,-0.9645920396,-1.8831322193,0.3731502593,0.7097339630,0.8633415103,0.0572413802,-0.3672955632,0.0742917284,-0.5524155498,-0.0990694165,1.2753888369,0.1014469489,-0.2720424533,0.7490035295,-0.0318105072,-2.5118894577,-0.7490311861,1.6888785362,-1.5632879734,0.3120557964,2.4351465702,0.9439433813,1.7685650587,-1.7593694925,-0.9471104741,-0.3451783657,0.7697889805,-0.6745629311,-0.1576409042,-0.7649592757,0.7051118612,0.4522475898,0.1596816778,-0.3910192847,1.8914453983,0.1121585667,-1.7256836891,2.5300681591,0.2082690150,-1.8599275351,0.7754518986,1.6154131889,1.3406716585,0.7618990541,-0.1711893827,-0.0310675185,-0.0496421643,0.4200766385,-1.5414303541,-0.3754616678,1.0248707533,0.7819921970,0.8943247795,-1.0348619223,-0.3936333656,2.4675981998,1.1788799763,-2.4421749115,-0.7062406540,-0.4198507965,0.0803024247,-0.6633006930,-0.1075372994,-0.4508400857,0.4074032307,-0.7849459052,-1.2711269855,1.3193122149,0.4826998115,0.5387716889,-0.2636992335,-2.0554578304,0.3182610571,-0.1439537555,-0.0299516935,0.4889170527,0.6923212409,0.6136444807,-1.1940793991,0.4658052623,-0.0280528236,-0.3650117815,-0.5338830352,0.7320910096,0.6896799803,-1.2262221575,0.6613311172,-0.8810014129,0.0679493994,0.3078416884,-0.5542497635,-1.0899014473,-0.1688338518,0.6363798380,0.9226398468,0.2054868639,0.2460445613,-0.9185663462,0.1189117134,-0.0556812920,1.2891730070,-2.4134826660,-1.5054019690,-1.1148625612,0.5536409616,-0.7582384944,1.5371382236,0.5538336635,0.2182800174,-1.1763197184,0.5749715567,0.7819045782,0.8062111139,-0.1452429444,-0.2014859468,0.5254017711,0.1605516672],[-1.7581435442,0.5122259855,-0.6614473462,-0.5915403366,2.0017547607,-1.4677304029,-1.8257400990,1.0064178705,-0.5828084946,-0.4141933322,-1.9140504599,-0.0616672523,-0.0180219132,-0.4038219452,0.6312733293,1.2066689730,-0.4449644983,-2.8706459999,-1.2025235891,1.0925306082,0.7826869488,-0.7611364126,0.3302960694,0.5973567367,-0.0336623602,0.4979988039,-0.6890585423,-1.2695591450,0.9333893657,1.1220725775,-0.7051638365,-0.3467444777,0.9953252077,0.5555455089,-0.0913688913,0.9911448956,0.0564950071,1.1763910055,0.4268105328,-1.6847207546,0.4380587935,-0.8402453065,0.8270261288,-0.0309975259,-0.7158976197,0.2076825500,0.2148397267,-0.6354771852,-0.2590553463,1.9080600739,-2.6474463940,-1.6957372427,0.0676404461,-0.1840924770,0.0144398957,-0.7230440974,-1.4499595165,-0.3431442082,0.7012284994,1.6000750065,-1.1220747232,0.6476452947,1.2227518559,-0.6261531115,-2.0118496418,0.5587783456,0.5587535501,-0.7095311284,0.9690502882,-0.0612876080,-1.0169298649,-1.8414348364,1.4388775826,1.2759331465,-0.5176864862,-0.3065904975,-0.1054771617,-0.8933632970,1.7129597664,-1.0600179434,-1.7203254700,2.1747725010,0.5608677268,-1.0009634495,-1.6312663555,-0.1436923146,1.0822681189,-0.8980425596,-1.1708720922,0.8111722469,1.4318437576,0.7622471452,-0.1665702909,-0.7519943714,0.9354758859,0.4794349074,-0.9928976893,1.2101974487,-1.4867138863,0.6146788001,-0.2739740908,0.7458140254,-0.4276439250,-0.7129654884,1.0291323662,0.1629079729,0.5204790235,0.4500774145,0.6597102880,-2.1476511955,0.7354454994,2.0337862968,-0.4528868794,0.0470402688,-0.6466990113,-2.4321630001,0.9452065825,-0.8999962211,-0.2017476708,0.5209354758,0.1222702488,0.3367359638,-0.6224790812,2.3189260960,0.8680391312,-0.5176272988,0.5175158978,-1.4247558117,0.2031879127,-2.5270433426,-1.1400226355,1.2911773920,0.4571803808,1.5411505699,0.3775106966,1.0139513016,-0.9173130393,-2.1711902618,-0.0480994694,-1.5899727345,0.1245185956,-0.1466777176,1.1491481066,1.8068344593,-1.2763129473,0.2475799173,0.9650402069,-1.5658047199,1.2057271004,-2.1787817478,-1.9162775278,0.3622097671,-0.6148615479,2.2243676186,-1.0941650867,-0.1319183856,0.3140008748,0.1496839225,-1.5112067461,1.5638947487,-0.0783534944,1.1089277267,0.0493666902,-0.3356166184,0.2406343371,-0.2573273480,-1.6920350790,-0.3557884097,0.5366314650,0.9400575161,-0.9624430537,1.0769087076,-1.0523335934,-0.0877942294,-0.0052203424,1.4278577566,0.7861345410,-1.3564819098,0.1132832840,0.3424504399,0.5758148432,-0.7315395474,-0.2296319157,0.7997226119,-1.0046871901,-0.2958320081,1.2034623623,-1.9853528738,-0.0400415435,-0.7096756697,0.5029446483,0.3679005504,-0.6466954947,0.8429231048,-0.3528733253,-0.8913444281,2.4212992191,0.4225657880,1.5877314806,1.0010522604,0.8875393867,0.0978859067,0.9387049675,0.9202107191,-1.2897603512,2.0188601017,1.4211659431,-0.6326606870,0.3057494760,2.0222163200,0.9547148347,-0.1210186332,1.0936723948,0.4895169139,-0.4024842978,1.2918033600,-0.7272326350,0.5280922651,0.9792674184,-0.9152534604,-0.4218613207,0.7013449669,0.7105325460,0.4497777820],[-0.1501760036,0.4193355441,0.0714192092,-0.0715517104,-0.3542770147,-0.3148297369,0.2007195204,0.4526666105,1.2837107182,0.6288003922,1.4994120598,0.1824219525,0.9061366916,1.5625766516,-0.3971055150,-0.2912955880,-0.0010462973,0.6671071053,-0.7881889939,-2.7450766563,0.4060696363,-0.1217147857,-0.0527308248,-0.5595987439,-1.0663646460,-1.6144676208,-0.8451096416,-0.7483228445,-0.4292066395,-0.7167803645,0.3249592483,1.0160164833,0.5160896182,0.5963602662,1.8268053532,-0.2776781321,-0.4084135890,1.1316784620,-0.0133100701,-2.4482481480,-0.0586686917,-0.8878390193,1.3675085306,-0.0189303663,-0.3145028353,-2.3099646568,-1.7913293839,-0.4868177772,-0.9915539622,-0.8610973954,0.5070689917,-0.6919354200,0.4805471301,2.0585455894,0.2822274566,-0.6514480710,-0.1684999913,-0.9540156722,-0.5895755887,-0.4050535560,-1.8935021162,-0.2592824101,-0.5564218163,-0.5321366787,-0.3610735536,0.7515267730,-1.0464690924,0.2622892857,1.9876885414,0.3568264544,-0.7345346212,-0.0195515398,-1.3769825697,0.3296893537,-0.2662521303,0.4305160344,0.8540118933,-0.0390822478,0.3072167933,0.2258054018,0.6492428780,-2.1197884083,-0.7837410569,1.2383978367,0.4710659683,1.1728113890,-0.9507129192,0.2316634953,1.5851820707,0.9761613011,-0.6257829070,0.7709450722,-1.2106201649,-0.1576354951,2.5922064781,0.5946409106,-0.8199970126,1.3616390228,0.8881381154,0.1613833308,-0.7847581506,-0.1245018691,-0.3079818189,-0.2307329029,1.0258018970,-0.3544710279,0.4603793323,0.1158363000,-1.5349887609,0.5325800776,-0.3238184154,-0.7847681642,-1.0883129835,-0.3599952757,0.3478820324,0.8348949552,-0.1920605004,-1.6158021688,-0.4275300503,-0.9156060219,-1.5471988916,-0.0337640494,0.3297310174,0.1512339711,-0.1176083460,0.0440559611,0.0721314177,-0.0469049104,-0.5487977862,-0.6798905730,0.1742166579,-0.9439554811,0.5917685628,-0.0007143845,-0.6769761443,0.9055025578,1.2873252630,0.8089355230,0.1100778431,0.9906618595,0.5761532187,0.3763134181,1.0817846060,-0.2395380288,0.8779850602,0.1564888656,-0.3758312166,-0.9272571802,1.9166927338,-1.0327939987,0.1510976553,-0.5881170034,0.7933108807,-0.5769035220,2.0220353603,-0.7505076528,0.9200173020,0.3916193843,0.2468304783,1.6190906763,-0.0692143887,2.9150190353,0.6242098808,0.6861552596,0.3290536106,-0.0465923660,-0.1690702885,0.2150677294,0.8022250533,-0.4570151865,-0.3585347533,-0.8290402889,0.9220418334,0.3166327477,1.3787505627,-0.4238806367,-0.6832771301,-2.3926756382,1.3433904648,-0.4304220378,0.0106177842,0.0189974494,0.1315878183,0.2400119901,0.3355641663,-0.4072660506,0.9903833270,0.1759019643,1.1644349098,0.1000541449,-0.4680081308,-1.3790748119,0.3504778445,0.0912212506,-0.7095877528,0.4825436175,-1.9707536697,-1.7738105059,1.2781053782,1.0709049702,-1.1580072641,-0.0813724622,-1.3623567820,0.3525542617,-0.7098584771,-1.3302956820,-1.0732815266,0.7031227946,1.6278609037,0.3216923773,-1.6622815132,-0.5861720443,0.7162619233,0.5596861243,0.3960135877,-0.1563261449,0.7563478947,0.8782933950,0.6613606215,-0.9163061976,1.9936851263,0.7046018243,0.5298975706,-0.3517471254],[-0.2529703975,-0.6819351912,0.4511622190,-0.3695062697,0.2835831046,0.0965076685,0.9501286149,-2.5614476204,0.5790864229,0.4087923765,-1.2475892305,-1.3186043501,2.3067893982,-0.5290445685,-0.4532274306,-1.4219975471,0.4242465794,-1.1147121191,1.1617943048,1.3607591391,-0.1647880673,-0.6505087614,1.3001717329,-0.4226404727,0.0101207495,0.8180606365,-0.1478663832,-1.1388643980,-0.9242997169,1.6581678391,0.6156594157,-1.3556060791,1.0397180319,0.0057428153,-1.0736074448,-0.1493596137,0.0751666129,0.4852134883,-0.0637846664,1.2028560638,0.0310445186,-0.1631756276,0.5054315925,-0.6068215966,-0.7260636687,0.9337012172,0.4490439594,0.9612168074,-0.4713724256,0.0757001787,-1.2501882315,0.0985274091,-0.0683198124,1.4502435923,0.1849096119,-0.7560486197,-1.8285669088,0.6093274951,0.0586636290,0.2028598487,0.6563054323,-0.5652397871,2.2766592503,-0.9085285068,0.6356878877,0.5116490126,-1.5929365158,0.1327019781,0.1812831610,1.7507266998,0.9238305688,1.8036668301,0.2698839903,-0.1360591203,-1.8491562605,-1.5269157887,-1.4091269970,-0.4059423804,1.1683697701,0.3950791955,-0.3730430007,0.9432966709,-0.4507664144,2.1472918987,0.4739236236,1.6310644150,0.4206609130,-1.5757130384,-0.9026260376,1.4252194166,1.0862905979,-0.5142824650,0.9228453636,-0.9622536302,0.5519937873,-0.0327038616,1.2062320709,-0.1808272600,-1.4194060564,0.9631287456,-0.6597050428,-1.0749583244,1.4704674482,1.8247721195,-0.7203150988,-1.0254713297,1.4141848087,3.0542407036,-1.0799494982,-0.6734762192,-0.1547389776,0.8099894524,-0.0583977439,-1.0075556040,-1.7363808155,-0.2186141759,2.6054716110,-0.3520674407,-1.4125469923,-0.9866930842,-0.3136315644,0.2758622169,-0.0382532291,-0.2909025848,-1.2334349155,-0.6941742301,0.4460225999,-0.3344380558,0.2541289926,-0.7754428387,0.6982194781,-0.1970705390,-0.4232169390,-0.2842783928,-2.1254146099,-1.6987265348,0.8912936449,0.0369419828,-0.4753980041,0.9474012852,-1.1694549322,0.0957721621,-1.9647138119,0.8450680971,-1.0638986826,-1.7959105968,1.0335805416,-1.0350733995,1.3430973291,0.5613515377,-1.3858976364,0.2772824168,0.2037287652,-0.4732226133,-0.9405746460,0.7366333008,1.3613336086,0.0208984539,0.4093787372,0.0871815830,-2.0470230579,-0.6185591817,0.5369277000,-0.9558705091,0.4795728326,-1.9930492640,1.2474435568,0.0068958555,1.5790319443,1.0980535746,-0.0637846440,-0.5902965665,-0.0061245328,2.0637385845,-0.5407064557,-0.8410700560,-0.3185648620,-0.3438983560,-2.0251550674,0.1584590971,-0.4653836489,0.4902653098,1.1586868763,-0.6380576491,-0.3382273018,0.1718517691,1.8048999310,-0.6133977771,1.1827590466,-0.4546300471,0.5546512604,-1.0758531094,0.3262319863,-1.6125426292,1.1405082941,0.9962644577,-0.3908626735,0.2542061508,-0.5308600664,0.8220796585,1.4729973078,0.6980400681,1.9615049362,1.2876982689,0.2182829082,2.1297807693,-1.1058113575,-0.8929600120,-1.6870913506,-0.9594828486,1.6345727444,-0.2234487683,0.2769846916,-0.4827255011,0.6188679338,-0.7529045343,-1.8513833284,-0.5449662209,-0.2289349586,-0.2275588512,0.1584722102,-0.9139913321,-0.4197747111,0.9362422228],[0.8670089245,-0.0218320917,-0.1137257963,0.8240271807,-1.0934950113,-0.1945286244,0.8003533483,-0.2396968752,1.1544240713,0.1843745708,1.8148233891,-1.2879664898,-0.2209738046,0.8045834899,-0.0956563428,-0.5216675401,0.0386822708,-0.0084566763,0.7532719970,-0.5548394322,-1.2596553564,0.6709458232,0.5227302909,0.3640642464,-0.4185277820,-0.2747395933,-1.3535087109,-0.7166903019,-0.4688091278,-0.7064665556,0.1297899634,0.4820433259,1.6228344440,-0.0645888969,1.1060124636,1.3474090099,-0.0167565756,0.1718845665,-0.7142949700,-1.5984379053,-1.3632293940,0.4917477965,0.2064529210,0.7901336551,0.8137999773,0.7803962827,-1.1908946037,0.7658921480,0.2878961563,-0.4552826583,-0.0192807633,0.2585481405,-0.1401580274,0.4660041928,-0.2457492054,-1.0530674458,-0.1534679681,-1.4978170395,0.5043719411,0.1542381793,-0.5338128805,-0.3461004198,0.1491215080,0.7485570908,-0.0845168531,-0.2989161909,-1.5050033331,-0.2240383327,0.1300648153,0.3475236297,-0.1500207782,0.6453188062,0.0438788645,0.6376432776,-0.5648962855,0.6896584034,2.0232799053,-1.0980631113,-0.1578527689,1.0964521170,0.6350690722,1.6230158806,-1.2539564371,-1.8732732534,0.2195006907,-1.4115343094,-0.3199668527,0.5239178538,-1.4737608433,-0.8933182955,-1.3867191076,0.1345203072,-1.0216890574,0.9537410736,-0.4129306376,-0.2612096369,-2.0409665108,-0.0334131494,-0.0334027857,0.9169976711,-0.7904871702,1.2259902954,1.1751644611,-0.4285097718,0.5169038773,-1.4336477518,-0.8526900411,0.4836533964,0.5627523661,1.2323125601,1.4806730747,0.8469880223,0.4425852597,-0.9853484035,0.0293111708,-1.5114089251,0.9939314127,-0.5136312246,1.8423138857,-0.3224902153,1.3486384153,0.1608252227,2.3281126022,0.3682502508,0.8500390053,-1.2086205482,1.6425739527,1.6567891836,2.5163788795,-0.5516718030,-0.7090018392,0.5536077619,0.1044385284,0.5848913193,-0.5738294125,-1.1334881783,0.4964465499,-0.4429698884,-0.5444251299,-0.0184476022,1.0488680601,-1.3786221743,-1.3408986330,-0.8560641408,-2.0122303963,2.1372659206,0.5076947212,-0.9105145335,1.0034862757,-0.5493667722,0.8107016683,0.5115770698,1.4349923134,-0.2621361017,-0.8416423202,-0.3985565901,1.4450401068,-0.5568643212,-1.9461606741,0.1166291758,-0.0909571052,0.7195361853,0.2246374786,-0.6057894230,-1.2935698032,-0.3281378448,0.0623177253,2.6832988262,-0.9763700366,-0.1302705854,-0.4990547597,0.0406178869,-0.7492877841,0.0462367646,0.1370660067,0.9836137891,-1.3005661964,-0.3430009186,-1.6979012489,-0.6472293139,-0.5646333098,0.3106422722,-0.9889423251,-1.7156691551,0.6313973069,-1.9979158640,0.3836011887,-0.3882793784,-1.7840440273,0.7430712581,0.7994744778,-1.2537626028,1.3493068218,-1.0350950956,-1.1018353701,-1.8689471483,-1.4032249451,0.1971910149,-0.4794034660,0.2162466347,1.1728647947,0.2414074838,-1.8751955032,-0.6058746576,0.3895087242,-0.3630200624,-0.7610980868,-1.3947130442,0.3746354878,-0.7805181742,-0.8008945584,-1.1045049429,-0.8625071049,0.2624949515,-0.8144610524,-1.1403104067,-1.4325137138,-1.2969572544,1.1078927517,1.2386733294,-1.0097072124,0.4666787088,-1.9054232836,-0.3379438221],[-0.9478093982,-0.1815164536,-2.1254286766,0.3806206882,0.3613242805,-0.3807277381,0.6509169936,-1.1140364408,-1.6195075512,0.1372729540,0.8177353144,-1.6663779020,1.6164512634,0.5298832655,-0.6429971457,0.4450341761,-0.3613208830,0.4252066314,-0.3191341460,-0.6303688884,-0.6494750381,0.3941224813,0.7792255282,-0.7178655267,0.5349315405,-0.3798153996,-0.3439167440,0.6115089655,0.8245086074,0.1483690888,0.8659130931,-0.4422810674,0.2366063893,-0.8263459802,-0.2544108331,0.7105622292,1.2936662436,-0.0061172731,-1.3991149664,-0.2684777379,0.5478208065,-0.5861970782,1.2108368874,1.2831014395,2.4623322487,1.7803323269,0.0805140138,-0.1960275471,0.0362248681,-0.1431263834,0.4654183090,-0.7120475769,-1.4517489672,-0.3825183213,0.3767770827,-0.2440060675,0.2647753358,-0.5264542103,-0.1851858348,-0.5366991758,1.2142837048,0.0382215828,0.2983546853,-1.3661375046,0.6581992507,-0.3689630628,-1.1523690224,0.2207235843,-0.3588210046,-0.5084909201,0.4237173200,-0.2514657378,-1.4737418890,0.3986872733,1.4346228838,-0.6114665866,-0.0471816361,0.7091511488,-1.2037036419,-1.2475736141,-0.2429477423,0.0104824426,0.9887840152,-0.5232872367,-0.6540614367,2.9717705250,-0.8098665476,-1.2220925093,0.6366750002,0.3437053561,0.2384938598,-1.5848807096,0.1682298779,0.7390740514,-1.1523597240,0.5150222182,0.2875719666,-0.3649416268,0.5651715398,-1.4896733761,0.8772756457,-1.3336732388,-0.7780230045,1.8497377634,0.2610572875,0.7758993506,0.0206704233,-1.0282015800,3.1966378689,0.2818923295,-1.2109206915,-0.8321417570,-1.3174719810,1.2380001545,-0.2314254940,-0.8635729551,-0.1830273271,-0.8175476193,-0.4133780003,-1.5508835316,0.1624538600,-1.0400195122,-0.2823365629,-0.7531560659,-2.6896286011,-2.2430329323,-0.0545572713,0.1620842814,-0.1392417997,-1.0881642103,-0.2334382683,1.0073410273,0.9778640866,0.0852946490,-0.3693783879,-1.1538172960,0.9188566208,-1.2099559307,-0.6782140732,-0.1138747409,-0.4954129457,-0.7024160624,0.1558332443,-0.1858649552,1.5633962154,0.2435840368,0.2163445055,-1.9303464890,-0.4743229151,-0.9498266578,0.3237558603,-0.2460615784,-0.4553605020,-0.7298208475,-0.3582246900,2.1175317764,-0.8765597939,0.5634529591,0.0625522062,-0.9692933559,1.1842176914,0.4230535030,0.4874381721,-0.5687189102,1.8937847614,-0.1945808977,1.4794397354,-1.3262608051,-0.8606801033,0.2191320211,0.7188069820,0.0009182103,0.6576800942,0.8063944578,0.8810168505,-0.5035043359,-0.7760851979,0.1666544825,0.1153337806,-0.8356602192,-0.5901991129,-0.3215951622,-0.9975553751,-0.6187186241,0.2700315416,1.2824653387,-0.1581376642,-2.1533644199,-0.7170999646,-0.6648142934,0.2680378854,-0.4821653366,0.0698350668,0.6788092852,1.2517094612,0.0046356483,0.0086138453,-0.6417683959,-2.4156284332,-0.4155599475,-0.6338680983,0.7500723004,-0.5130292773,0.5746928453,-0.1184019148,-0.2648290098,-2.0818395615,-0.7137652040,-0.9739657044,0.7537212372,0.2535819113,0.6169744730,-1.4554769993,0.3506465554,0.7433042526,1.3686823845,1.4085261822,0.9369397163,-1.1510248184,-0.8294801116,-3.0858607292,1.1538147926,-1.0315407515,0.4071751833],[-2.3339128494,-2.4373297691,-1.5677690506,0.5979134440,0.9495761991,-1.1302498579,1.5698679686,-0.0112411575,-0.4865964055,-0.6322322488,2.2198269367,-0.1834526360,-1.1265172958,1.3004908562,-0.8718284965,-1.0175803900,0.1230721474,1.2845140696,-0.3991899788,-1.1820335388,-1.5773611069,-1.8915426731,1.0482774973,0.4631630778,0.5453273654,-1.2216370106,-1.2610167265,0.3723307848,-0.7056763768,0.5499272943,-1.4075728655,-0.2471286952,-0.1996609718,-0.6414245367,-2.0822985172,1.0572780371,-0.2430976778,0.9677157402,0.2788408399,-0.0577941090,0.6746733189,-0.7687827945,0.1710018367,0.2308249325,-1.4008908272,-1.5655425787,-2.2899365425,1.1975847483,0.7657328844,-1.6059250832,0.6931754947,0.6015011668,-1.5637310743,-0.3585254550,1.7346768379,-0.4272238016,0.6572923064,0.7208228707,0.4170411229,1.6985064745,-0.6069838405,0.1985742748,1.3401014805,0.1548866481,1.1064743996,1.3651067019,-1.2938311100,0.1128372401,0.4152404964,1.8631957769,0.6998057961,-0.7854279280,0.7744868994,1.1030099392,-0.6815320849,-0.6766694188,0.7262734175,0.1705344319,1.5088578463,-1.4776148796,-0.0586480461,1.1006953716,0.7420361638,1.6587747335,0.1205245256,-0.0020093988,2.2212829590,1.0334048271,0.7167780995,0.7675334811,-0.4229756892,-1.8542928696,-0.1795200706,-0.3950515389,-0.1793856621,0.2677515745,-0.2145903558,0.9883928895,0.6095905304,0.8403566480,-2.7272720337,-2.5840220451,0.4287470877,-0.8445891142,0.5398139358,0.1210710034,0.9378256798,-0.3317985833,-1.0996099710,0.0415950157,-0.3239310086,0.3395366967,0.6336280704,0.7191601396,-0.7759758234,0.1708994955,3.4803307056,-0.9206820726,-0.4048551321,-1.0755497217,-0.8828337789,-0.5957428217,-2.1283802986,-0.5010408759,0.4380800128,0.1451422721,0.0548269637,0.7946581244,1.5738191605,0.4238313138,1.1378115416,-2.3402349949,0.2621157765,-0.5924669504,0.6416893601,-0.0999486968,0.0793482810,1.2093353271,0.6182104349,-0.1222885475,1.1483215094,-1.2530092001,0.8336417675,-0.5853553414,-0.8618545532,-1.0123625994,-0.2709977627,1.1808912754,1.0666790009,-0.5410199761,0.2418011427,-0.0650454462,-1.3687447309,0.6643924713,1.2235996723,-1.2306170464,-0.3710289299,-2.3630490303,-0.0823582411,-1.3592556715,-0.0260777790,0.4334647059,0.2359000891,-1.6590589285,-0.6708953977,1.0374119282,0.5894627571,0.1390295923,0.4612559378,-1.1581498384,-0.6379216313,-0.5233878493,0.2453523725,-0.6686667204,-0.2729541361,0.0582876019,-0.1641405523,2.6693544388,-0.7174984813,1.1088682413,0.5329573154,0.0444557518,-0.5908399820,1.7005236149,-0.1409078687,-1.1669863462,0.6829881668,-0.1745918244,0.2864745855,-1.8335051537,0.4798454344,0.3556045294,0.5267731547,0.4601722360,0.5871078372,-0.0579544082,-1.0574859381,0.9699077010,-0.1474561393,0.9218323827,-1.1894884109,-0.6066743135,-0.8240451217,0.6136225462,0.6406924725,-1.2699052095,-0.7971712947,0.7746338248,-0.9440367222,-0.0133051872,-0.3265692890,-1.1451457739,-2.6992063522,-1.1471670866,1.6526821852,-0.1636654586,-0.7557908297,1.0712401867,0.9058020711,0.6016139388,-1.8284126520,1.7088695765,-0.1438959986,0.9743440151],[1.1755664349,-0.6853446364,-0.9805140495,-0.7326768041,0.6781848669,-0.9718044400,-1.4187847376,1.5915788412,1.6385931969,0.6819905043,0.9449669123,-2.1971836090,0.6113591194,-1.1820850372,-0.3689990342,1.5513277054,-0.5906776786,-0.4767121673,0.8203312755,1.0888025761,1.1581448317,-1.5558147430,-1.1395241022,0.8870739341,0.1858982444,0.2697058022,-0.9096365571,0.3069518507,1.2159620523,-0.2464475781,0.3862953186,-0.3224743903,0.2444050610,-0.6824684143,0.0999120772,-0.6802471280,-1.1838104725,0.5632140040,0.6371393204,1.0253103971,0.3471388221,-1.4211109877,-0.3159162104,-0.5746237636,-0.1463756412,0.5199471116,0.4381284118,-0.9439998269,-1.1170305014,-1.6122260094,1.5769633055,1.3907139301,1.1999559402,1.3022366762,-0.6340661645,-0.0995960012,-1.6399950981,-0.6119499803,0.1380764097,-0.6237627864,1.1016682386,-1.9356443882,0.5634610653,0.4649307132,0.5689449906,-0.6906414628,1.0428044796,-0.9816251993,-0.0944080576,0.9025543928,-0.9932280779,0.7247273922,-1.6060585976,0.6052199006,-0.0746025890,0.2692794502,0.0836235955,-0.6294842362,-0.2354930639,-0.3489116728,1.4909719229,2.2565157413,1.1001173258,0.1191658676,0.2973752916,-0.6802826524,0.4829027653,0.6770628691,-0.8792845607,-0.5022274852,0.4680658877,0.0524122715,1.2018536329,1.2017225027,0.2157837749,0.6835497022,-0.5031975508,-0.8575496674,-0.7981103063,-1.3048182726,-1.0906238556,-2.1067719460,0.2701582015,-0.1355769187,0.1928640008,0.7015531063,-0.2581508756,0.0451877564,0.8290317655,2.4921574593,0.6755216122,0.6239854097,1.9080717564,-0.2598963976,2.0262806416,0.6672372222,0.1927894354,-0.5935207009,1.2906221151,1.1303480864,1.8212866783,0.1627627015,-0.7681046128,-0.2628323734,-0.1882967800,-0.4641387463,1.1165431738,0.0123880208,1.1824625731,-0.5635791421,0.4094505906,0.5499470234,0.6730532646,1.1384918690,-0.0318734385,0.5249695778,0.9564266801,-0.1829735190,-0.8484261036,0.1849524081,-0.6116765738,0.4040069580,1.1256523132,1.3207218647,2.0485391617,0.2022244185,1.0597351789,1.2315956354,0.3742661178,1.4664480686,1.0261442661,0.9445564151,-0.7210229635,0.3330769539,1.1520477533,0.0342893526,-0.1105533689,-1.1973357201,-0.3417419195,0.8220157623,0.7772755027,-1.5686694384,1.6022686958,-1.6342042685,0.9400302172,-0.4801312387,0.1337077469,-0.0188103653,0.0742905065,-0.7465792298,-1.0140510798,1.7433282137,-0.2325217575,-0.2834181190,-0.8204702735,-1.2446351051,-0.2062532604,0.8481276631,0.8852192163,-0.8912778497,0.3083090186,-1.8603940010,0.3590204418,-0.1462968290,0.8163167238,1.3781898022,-0.5198218822,-0.6847628951,-0.8724231124,-0.2871099412,0.0967746899,-0.0191583131,0.5907481313,-1.5904307365,1.3970313072,-0.2175879031,0.7887555361,0.8525711894,0.9093270302,1.1224218607,-2.1383500099,0.8649312258,0.8823870420,-1.1048812866,-0.9562670588,-0.8385741711,-1.4619178772,-0.1579514742,-0.2121369094,1.2227568626,0.5742275119,0.0359639078,-4.6599531174,-0.0978555605,0.8772867918,0.9253914952,-1.5906937122,0.3186887503,0.7967299223,-1.0925660133,-0.1356026530,-2.7123830318,0.7223868966,-1.3690061569],[-1.2025033236,0.7792565823,-1.1973105669,1.7177031040,0.0948203802,2.5072460175,-1.5663551092,-0.6132227182,0.1630306989,0.5378769636,0.5238105655,0.2860624492,-1.1264125109,0.1445035040,-0.5489215255,-0.3247438967,-1.3651691675,-0.6719406247,-0.3707260489,-0.8329691887,1.3739992380,1.4054895639,0.6142627597,0.3801291585,-1.9824923277,0.5518612266,-0.9396815300,-0.4278084040,0.0637271479,-0.8120394945,-0.5692890882,0.9053315520,-1.4505615234,0.7103596926,-1.2559800148,0.0491107069,2.4521374702,-0.9200552106,0.2403309941,0.0198677108,1.2498037815,1.2705478668,-0.3080240786,0.8807117343,-1.7154098749,-0.2565684021,0.1640652120,-1.4016714096,0.7097941637,1.9095629454,0.9995204806,-0.4326931834,-0.6567226648,0.1821094900,1.0034320354,-0.3981125355,2.0536632538,1.8618398905,1.3602993488,-0.4498952627,2.0644013882,0.9212067723,-0.2571714818,-0.3730241656,0.8073030710,1.3182489872,0.1169435158,1.3609436750,-0.1411994398,-0.1029899791,-1.9461628199,0.4719499946,-1.4018038511,1.0566523075,2.1330528259,-0.7859642506,0.0822543800,-1.3732144833,1.7473028898,-1.0639010668,-1.2659695148,0.2229897678,0.4247764647,0.4805686474,-0.8474889398,-0.3383961320,0.8022504449,-0.5559909344,-0.4041362405,-0.5135216713,-1.3163421154,-0.1205931008,1.5122067928,-0.0104675554,-0.2192304283,-0.1263844669,-1.1371041536,-0.5089555979,-0.1946117580,0.8600028753,0.0638399795,-3.5213615894,1.0045546293,0.5567478538,-0.6885315180,-1.4055252075,-0.3676231802,-2.9203844070,-1.1243324280,-0.3046654463,-0.7170249820,-0.6134189367,1.1806969643,-0.0221272781,1.1801662445,0.3245626092,-0.2887108624,-2.0269603729,-1.4936410189,0.1252851188,0.5235942602,0.7694662213,0.1733997315,1.3879876137,0.9336732030,-0.2111702561,-1.0503019094,-2.0826382637,1.4609395266,-0.0856881514,-1.7071745396,-1.1867042780,2.1376116276,0.8348627687,0.2069842666,1.7539217472,0.3535583615,-1.8743047714,-0.0553962588,0.8481658697,-0.9995617270,0.3901985884,0.1343593001,1.6648913622,1.2137875557,0.3217611909,1.4884146452,0.1555677801,1.0586683750,-0.0288448110,-1.7163136005,-0.4568204582,0.6596849561,0.3663463891,-1.0010029078,0.6994525790,-1.2128429413,-0.5362828374,1.0057533979,-1.0551908016,-0.5448315740,-0.3002343476,0.2546850741,-0.2694426477,-1.7030316591,-0.8511922359,0.4108665884,0.8940765262,0.4596008062,-2.0727031231,-1.5894145966,0.6068012118,-1.3510271311,0.8960603476,-0.6474032998,0.2486500740,-1.2714422941,-0.0038882706,-1.2690234184,-0.7396576405,-0.0264970679,-0.3070549965,0.3133223951,-0.0787381977,0.8041164279,0.2555361092,-1.9416064024,-0.9758633375,1.1594858170,0.6093669534,0.9097701311,-1.1062548161,0.8592962623,-0.2846635282,0.6063728333,-0.2664417028,0.3516781628,-0.0238830093,0.6255849004,-0.7902545333,-2.0264768600,-1.1567298174,0.1229565144,0.0750426725,0.3840484023,0.6105062366,1.4286056757,1.5390007496,0.0215927530,-0.6384230852,-0.0326580927,-0.4754877985,-0.0593326017,0.2306943089,-1.5714039803,0.2333770394,1.1175312996,-0.4276227057,-0.2300469875,-0.7984775901,0.5063858032,0.7095662355,0.0707808509,0.3261324763],[-0.5166196227,0.7755348086,1.4209582806,0.4472828507,1.5131568909,-0.5091314912,-0.1463637352,-1.2307108641,0.8167770505,-0.8237021565,1.6269707680,-1.3415946960,0.0487488285,-0.8004634380,-0.6269339323,1.8451014757,0.4949062467,-1.3215615749,0.6344664693,0.2089666575,-0.4782479703,0.3076966107,-1.6594357491,1.3730232716,-0.2208669931,-0.5210494995,0.2851623893,0.8996701241,-2.5622427464,-0.0674514249,0.0234192368,1.0144755840,-0.9546979070,0.8461613655,-0.0649054199,-0.6372078061,1.3974251747,0.3746976852,0.7681015730,0.2594341636,1.7057238817,0.1613429636,0.0854009017,-0.8134281635,0.9436557889,-0.9423817396,0.5801091790,0.1318775266,-0.2848078012,0.1335559636,0.5876556635,-0.3178980350,-0.0128314896,0.0865386352,0.3319094777,-0.2248116732,0.0240327138,0.6187692881,0.3574542105,0.9088279009,0.0697148070,0.4406653047,-0.1967163235,-0.4700434208,-2.1740925312,-0.3500843346,0.1516977102,0.2233592272,0.2852753103,0.6240082383,0.1125972867,0.3403343856,-1.4563639164,0.3733136654,2.1346266270,-0.9266167879,0.3932163715,-2.0109872818,-0.0350480564,0.5640030503,-0.3578226268,0.1307384968,-0.2989596725,-0.7109010220,1.3044710159,0.4014618695,-0.5151721239,-0.0912843123,-0.8773889542,0.3478965759,-1.0700992346,-1.2515821457,0.6234746575,0.5236243606,-1.6252604723,-1.2834419012,-1.0935277939,1.0660840273,1.4242330790,0.1721225530,-0.4695689976,0.3664133847,1.7772021294,-0.9540381432,1.0367292166,-2.3827981949,-0.3921341002,-0.5850293636,-0.2167987227,1.3427859545,0.6053371429,-0.6285265088,0.3417230546,1.3946480751,-0.3851041198,-0.5210492015,-0.8341775537,1.8781659603,0.4993408620,0.9196839929,1.4708603621,-1.3287057877,-1.8750246763,-0.6538562179,-0.4436583817,0.3147097826,-0.2363654673,-1.1563998461,-0.7322943807,-0.8094955087,0.1966855228,1.8921687603,0.1011165977,1.0714277029,-1.1122522354,-0.8491483927,1.0026543140,-1.8699041605,0.4910463989,-1.3007510900,-0.7558706999,-1.2062203884,-1.3015846014,2.2981455326,-0.3842036724,-0.4086307585,-0.5081439614,1.0238132477,-0.0232919119,-0.7040374875,-0.6754306555,-0.5368385911,0.4415324926,-0.2231244147,1.5856546164,0.3726929724,-0.8339967728,-0.3401282132,0.4355895817,0.4122020900,0.5042055845,-1.2808250189,-0.3089379668,0.2095138282,-0.9887799621,0.5872370601,-0.4417540729,-1.7355332375,1.5637019873,1.1506174803,0.0401921943,0.8149108887,1.6987164021,-1.0433235168,-0.4187589586,0.3610968888,-1.5596761703,-0.3735758960,0.2021843344,0.6289901733,0.8797625899,0.5344867110,0.1503546983,-0.7813857198,0.2976354361,0.7332540154,-0.7058255076,0.0368137881,0.3777216375,-0.5616474748,0.0333609879,0.7632266283,-1.1654425859,-0.4225851893,-0.4182069302,-1.7396740913,1.2583700418,-1.4786802530,0.2079833299,-0.6789172888,2.0099575520,0.2491786927,-1.5300163031,-0.3059594631,-1.3193536997,1.9566799402,0.6859662533,0.2039905638,0.1241332442,-0.0059582829,0.4895803034,-2.1932377815,0.4535388052,-1.3756812811,-0.7140263915,0.5834588408,-0.4265936911,-0.7711508870,-0.0634818077,-2.0736818314,0.4877722561,0.7690168023,-2.2140629292,-0.1235852242],[-0.7416536808,1.1636881828,0.7511378527,-0.1915588677,-0.6453274488,-0.4290712178,-0.7228347063,-1.0105632544,-0.5159531832,-0.4688208699,0.6070471406,0.2960442603,0.7811290622,1.2193843126,-0.7364284992,-0.5761831999,0.0020575896,-0.4753360450,-1.2325395346,2.1083066463,0.3523429036,-0.9894808531,0.4212930799,-0.3872063458,-0.7398980260,0.6265519857,1.1915148497,0.5910297632,0.9382076859,-1.6488192081,0.9358170629,-0.0788703635,-0.7320182323,0.4131195545,-0.6173188090,-0.5903555155,-0.2789388299,-0.1472140253,-1.2764873505,-0.1217904240,1.2791328430,-0.6618398428,0.6625029445,0.5320830345,-0.4551331699,-0.5017670989,1.8236455917,0.6914116740,0.3026286960,0.1815621406,0.2498879433,-1.2506207228,-0.0442353934,-0.1216291115,0.5298925638,-1.5443810225,-0.4020614624,-0.0662033781,-0.0818239599,-1.4185363054,-1.3683942556,-0.0888589248,1.7211959362,-0.3998090625,0.0440240912,0.6941738129,-0.0163219403,-0.2769297659,0.3046324849,-0.2503691316,-0.3313120604,0.2744653523,-1.2622573376,-0.2323275506,0.0178010147,-0.8282309175,1.0964192152,0.2451598942,1.5029834509,-1.4286749363,0.5833849907,0.5235205889,1.7843471766,0.3572329283,-1.2430522442,-0.5951647162,1.7783439159,-0.9175342321,1.5678970814,1.7504197359,1.7831438780,-0.7279756069,0.4373401701,-1.2401688099,0.0116553763,0.0059235734,-1.5822329521,0.4546556175,-1.1685270071,0.2259088755,-0.5036152601,-0.4659640491,0.0927771181,2.4530568123,0.6292487979,1.6537941694,0.3002971411,0.3096996248,-0.6645023823,-1.5523566008,-0.4031520486,-1.6061965227,2.7819669247,-1.0583498478,-0.1395788193,-0.1203634888,0.1022626758,0.8720859885,1.4830600023,-0.4962566197,1.0328984261,0.3373303413,2.3986284733,0.1229605749,0.6004591584,-0.7395074964,1.4079188108,-0.0688579455,-0.2888651490,0.0585243143,0.7603533268,-0.7401866913,-0.6543632150,1.1241512299,1.3855564594,0.2501965463,-0.0244971812,0.7574801445,1.2781273127,1.5284435749,0.6185496449,0.5040275455,-0.2804699838,0.1216544211,-0.6845163703,0.7941741347,-0.0366092920,-0.6983697414,-1.8522673845,0.2542896867,-3.2831485271,0.3953566849,-0.5015579462,0.6498671174,-1.1276022196,1.1505984068,0.1903680265,-1.2231880426,-0.6573792100,-0.4833384156,-0.4638567567,1.3860788345,-0.6521995068,0.1385852545,0.3805219531,0.1140052676,-2.4107685089,0.8874932528,1.6550575495,-0.3197214603,-1.3988662958,0.4590135515,-0.2101721168,0.4094855785,-0.9342734218,0.6102930903,-0.0219919235,0.8110104799,-0.1327028722,-0.5929984450,0.0896767303,-1.4792257547,-0.6357883811,-0.1910596639,-2.4750189781,-0.5322038531,0.5241442919,-0.6994981766,0.8529137373,-0.4903007746,-0.8817639351,-1.3482192755,-0.0317793489,1.2985696793,-0.3465938270,-0.5808402896,0.0075849593,-0.7275836468,0.8141966462,0.4521128833,0.2785006464,1.8277621269,-0.5190520287,-0.8914665580,0.2689761817,-0.9662404656,-0.6470789313,0.2914797664,-0.8155297637,-1.3739093542,-2.3848102093,-1.1209721565,-0.8182407618,-0.1182447374,-0.3398118019,0.4466997087,0.2464601994,1.0585348606,-0.2261178344,-1.3365659714,-2.1779890060,-2.3538610935,0.9866024256,0.8527196050],[-1.0528454781,-0.8125745654,-0.3296542466,-1.1564801931,0.7735942602,0.9443894625,-1.1332502365,0.3197281063,0.2241103351,-1.9744640589,-0.9417819977,-0.2481104732,0.3611280024,0.8934960365,-1.1568579674,-0.5745681524,0.0709185824,-0.2200739235,-1.0778328180,-0.1161477938,-0.0324398316,-0.4999149740,-1.7372184992,-0.0684553757,-0.8528540730,-0.5410062075,1.1508429050,-0.1721008271,0.1344915330,-0.5637725592,-0.3995460272,-0.6501408219,0.9465684295,1.9391757250,-0.5684672594,1.5173685551,-0.6282674074,-0.9417642951,0.0144041413,0.0889731869,-0.2330531925,0.2974446118,1.2360913754,0.1447630525,-0.7290937901,-0.7561873198,1.1870754957,0.2414626628,0.2983992696,0.7458506227,2.1857104301,0.8399736285,0.7549082041,0.7326828837,-0.0318742953,-0.6576696038,0.0198269822,-0.6515598297,-0.4805519581,0.6991463900,2.2100965977,-1.7864370346,-0.6630361676,-1.1771378517,-0.6534810662,0.4054700434,-0.6601068974,-0.7464119792,0.2744610012,1.6158492565,-1.7060116529,0.2919547558,0.1256903559,-0.7018199563,0.2004989535,0.3009130657,0.8931428790,-1.2528479099,-0.9481801987,0.7718013525,-0.8815566301,-1.0193971395,0.0920027643,-0.7858209014,0.9181225896,-0.7570737600,-0.0542134568,1.0911778212,-1.1670839787,-0.3656161427,0.6322442293,2.5866792202,-0.9024486542,-0.1006447151,0.0434250124,-0.6026551723,-0.7511234283,-2.1465878487,-1.3147332668,-1.0441503525,1.6845152378,-0.9341364503,-0.3411107957,-1.7455786467,-0.5735567212,-0.3543936908,0.8459254503,-0.0149782002,1.0783048868,-1.4044744968,-0.5981349349,0.7644030452,-0.2567512095,-0.4749291241,0.3467444479,-0.0457928218,0.3171402514,-0.3241792321,-2.8057308197,0.9024006128,-0.5445762277,0.9712939262,0.4558846056,-0.1432908773,0.1366594285,0.5268583298,0.4409561157,0.3136016130,0.3074067235,-0.5648976564,-0.8228408098,-0.9566798806,-0.9248560071,-0.2933781147,1.1406354904,0.9162472486,-0.0694442838,-1.0854730606,0.4244476855,-1.0796084404,-1.1030313969,1.4148253202,-0.4556069970,1.4371075630,0.0990088806,-0.7168421745,-1.4504446983,-0.2324653566,0.7561825514,0.1022730097,0.6021108031,-1.4685246944,-1.6902762651,-0.9539712071,-2.0895330906,0.4726255536,-0.4029881656,0.0007847449,-0.1372971684,-0.0785327554,-0.7824497223,0.3144697845,0.9834409952,-1.3111101389,0.1794102639,-0.5454942584,-1.2275632620,-1.1055543423,0.1894212812,-1.2745862007,0.3700563610,-0.0895574987,0.1387081742,-0.1900572032,-0.8278012872,-0.2102169394,-1.1714949608,1.8205448389,0.1352094561,1.0171357393,-0.7764116526,-0.9255347848,0.3677875102,-0.1336479634,1.0903134346,-0.5522167683,-0.1727647781,2.1905345917,0.0677361563,0.0585403368,-0.5572353601,-2.4835076332,-1.2631917000,-0.9267233014,-0.3717812896,-0.2926731408,0.0466216952,-0.7515469790,-0.5927543640,1.4340451956,0.3440235853,-0.4755963087,0.3029372394,0.4228192568,0.8591195941,0.9187337160,-0.9696948528,-0.8918405771,-0.7458072305,-0.3890754282,-0.7237933278,0.2729220390,0.5538294315,1.1950474977,-0.1350257993,-0.0628644973,-1.0330879688,0.8075287938,0.4006544948,-1.5620900393,-0.2522285283,-0.7478344440,-0.7600804567,-2.5066511631],[0.2466514409,0.6960947514,0.8043972850,-0.7229521275,1.8516392708,-0.5131801367,1.9044151306,0.7484899163,0.3236445189,-0.2882115245,0.7676124573,-0.3992791474,1.0807728767,-0.6194197536,0.1106903628,1.4164921045,-0.4014247060,-0.8979519010,1.2561978102,0.4640975296,1.9639859200,-1.8645833731,0.4402790070,0.2453724444,-0.8920995593,-0.2342789471,0.9694778919,-1.5542093515,0.4818401933,1.1038746834,0.0457208902,-0.3195508420,-0.1857536137,-0.5587615967,0.2741684318,1.5274512768,-3.0374977589,-0.5423014760,-0.0150344083,-0.6761445403,-1.0511331558,-0.4787085354,0.2322977632,0.5701154470,2.6077516079,1.5823068619,-0.4669645429,0.9781920910,-0.9199672341,-0.1703516096,0.4301041663,0.7154524922,0.4585685730,-0.8489269614,1.4372155666,0.8874803185,-0.1794517487,-0.0144406203,1.2646210194,-2.9158418179,0.4403441846,-1.3688344955,0.8333687186,0.5244902968,0.2989941537,0.9304910898,0.6709625721,0.9794834852,1.3098096848,-1.1596137285,-0.8816054463,-0.5656709075,0.9917655587,0.4784300923,-0.5290926695,-1.1244556904,0.3402399421,-0.8361654282,1.4321713448,0.8068723083,2.2013251781,-0.1939378530,-0.4361837804,0.3527769744,-0.8224888444,0.2225948572,-0.0207322240,0.1455113739,-1.2164512873,0.0929658487,-1.1042196751,1.4114979506,-0.6173691750,1.9363077879,1.2560794353,-2.7780065536,0.7533286214,1.3619881868,1.1684308052,-0.8395903707,1.9072697163,0.4078255296,-0.2256548852,-0.5843978524,0.0805882514,1.6368948221,1.3414212465,-0.6055875421,-1.0454299450,1.3621159792,-0.2686824799,-0.0739303008,0.5343942642,-1.3068132401,0.9384565353,-0.3994387090,0.4542995989,0.7322161198,0.7305169106,-0.5614368320,1.1140637398,0.6526744962,-0.0719099790,0.9098330736,1.7640006542,0.4237023890,-0.6691291928,0.5890297890,-1.9160044193,-1.0037022829,0.9969838262,0.8288206458,0.7137361765,0.4376760721,-0.3152429163,-0.1279374659,0.3058860302,2.4879970551,-1.6800762415,1.3998795748,0.1315685362,-1.5002568960,-0.2054777592,-0.5640303493,1.3824599981,-1.5066652298,-0.1466574669,0.0915420279,-1.5763106346,0.3803360164,0.3596437871,1.6621896029,-2.0491542816,0.5765722990,0.1055285856,-0.6198460460,0.1618170589,-0.2169940472,-1.3427215815,2.0164062977,-0.1058017090,0.1578689814,-0.2270162851,-1.2023625374,0.1935603917,-1.2298405170,0.8036319017,0.4344801307,0.8951734900,-1.2423279285,1.8490303755,-1.7592107058,-0.0522167459,-0.4019152522,-0.5006610751,-0.3094775975,-1.4385825396,-1.0389883518,0.1763581932,-2.0889842510,-1.5009762049,-1.7347135544,-0.5499082804,-0.4091817141,-0.5084925890,-1.7259931564,1.1904336214,1.4008835554,-0.7470872998,-1.3493409157,0.7245262265,-1.4676628113,0.5550032258,0.9892581105,0.4554331899,-0.9375141263,0.2234862894,1.4883899689,-1.2857881784,0.6532883048,0.4550784528,-0.4044540226,1.1459242105,-1.0726145506,1.7762796879,0.6335772276,2.1317172050,0.0332975127,0.5179934502,-0.0992914513,-0.7737841010,-1.8695286512,-0.7923610806,0.1761527956,-0.7032017708,0.5412458181,-0.9078052640,0.4937606454,1.1527210474,1.1989228725,-1.6906565428,-0.6281328201,0.2278220356,0.4933513403],[-1.0064896345,-0.0000735259,-0.8934984803,0.1968393624,0.6673569083,-0.5315707326,0.3909243643,-2.2143652439,0.9027023315,1.4943727255,0.9048801661,0.2294406146,0.9573911428,-1.2223769426,0.1271428764,-0.4152106047,1.0905336142,1.6445459127,-1.5889605284,0.1431340277,1.3259994984,1.2536746264,0.8460054398,0.7108342648,-0.1171533167,-0.7995001078,0.7147882581,1.0996558666,-0.8065167665,1.5425846577,-1.9306179285,-0.5112408996,1.7824292183,-0.3771158457,-0.5927338004,1.0465234518,0.9517975450,0.8834025264,1.7332577705,0.4638868272,-1.2697304487,-0.1591738909,-1.7999488115,-0.9415250421,-0.1269199103,-0.0580913350,-1.2965680361,2.1223971844,0.0455871597,0.2907867432,-0.3369020820,-0.6078436971,-1.5405783653,-0.1512064636,-0.6500342488,1.2631211281,1.0848257542,-1.4378325939,1.8252682686,-0.3971527517,-0.4756045938,-0.8820447326,-0.0112926476,1.2548937798,-0.6723657846,-1.2332930565,-1.6561949253,-0.4634982944,-0.6589117050,-0.8096425533,-0.0774505436,-0.9581915140,-1.0015362501,0.3436736166,-0.7924914956,0.4270743132,1.6997318268,0.1629018933,2.6756398678,-1.3755729198,1.0998866558,1.8532304764,-0.2589176893,-1.5839751959,1.2968481779,0.1454011351,1.0266642570,-0.9606741071,0.2631573379,-0.6971349716,-0.7989433408,0.6325493455,0.5290307403,-0.8914008141,-1.4983018637,0.1810698956,-0.0936394185,0.7586063147,-1.2714086771,-0.1377002895,2.1368517876,0.0857330486,-0.9389966726,1.3523417711,1.9735085964,1.8241761923,-1.9587459564,0.3082878888,-1.1944063902,-0.4493401945,1.2215254307,0.1729801297,-0.0349820703,1.2029552460,0.5404530764,-0.0075979042,-1.0764358044,-0.7699571252,1.1834006310,0.5547564626,0.9156490564,-0.9856804609,-0.2785350978,-0.5186696053,1.3056958914,1.0631380081,0.1274959594,-0.2401805520,0.1479017437,-0.0582404770,-0.0790114775,1.0718803406,0.7031837702,-1.9135128260,0.3803895712,0.5894224644,-0.2192968726,-0.6255428195,-0.6426970959,-0.9397515059,0.5840689540,1.3284479380,1.3117339611,-1.5271680355,-0.3960041106,0.6294850707,2.4824326038,-0.9239836931,-0.5315219164,-1.0664471388,-1.2447993755,0.5901226997,0.2070456147,0.9431948662,-1.0233124495,0.6501449943,-0.0389963426,-0.1377603114,-0.1678514928,-1.2874910831,-1.5522481203,0.8630752563,-0.3618498445,-0.2759974897,-0.8493277431,1.7580159903,0.9600577354,-1.8509514332,-0.9705380201,1.6187961102,2.1266665459,0.0816572905,-0.4026440084,-1.1554012299,-0.2847146988,0.2674738467,-0.8082942367,-0.8134348392,1.1221520901,0.0270938650,-2.2677624226,-0.9180805683,0.2363503575,0.2875520289,0.8363741636,-0.0643644705,-1.5559432507,1.6432741880,0.5946225524,0.4202830791,0.0109968660,0.6184870005,-0.4283002019,-0.3038954735,0.2131291479,0.0461715311,1.5333558321,1.9102970362,0.0692449957,1.0393091440,0.1182040423,-0.7626225352,0.6060833931,-0.4008977413,-0.5854520202,-1.5112763643,0.9768438339,1.2555006742,0.3712321520,0.3047838807,0.5041245818,0.1352996081,0.6537587643,-0.1001813114,-0.8632913828,-0.7738704681,-0.8903281093,0.0303989761,-1.5796298981,-1.8807632923,0.4715923071,0.7007640600,-0.4200832248,-0.6220460534],[-0.0537967421,0.6815012693,-1.2996648550,0.7641907334,0.1130914018,0.3476420939,0.8941524029,0.0757900104,1.6656370163,0.4651370049,0.7107572556,-2.1352045536,0.9271841049,1.3995282650,-1.4733744860,0.3333723545,1.2920279503,-0.0935617089,-1.4775958061,-2.2185895443,-1.0155769587,-0.0170312412,0.4430365562,-0.0669785142,2.0955364704,-1.2891235352,-0.3313333094,0.1508097947,-1.0214672089,0.6050590277,-0.3226830959,-0.6393830776,-0.7354826927,0.5948662162,0.0514817126,-0.3487473130,0.2998561263,-2.3917312622,1.9389803410,1.3662700653,-0.2757565379,-0.3776474595,0.0289556049,0.4271129966,-1.4703136683,-0.8241951466,-0.9340753555,-0.0965261236,-0.3129947484,-2.1099956036,-0.4961585999,-0.6232930422,-0.4650352299,0.9700036645,-0.1744728535,1.5180350542,0.6357852817,-0.2957075536,-0.7334786654,0.3546252549,0.1438398659,0.0114881210,2.3138237000,0.6166037321,0.5094236135,0.2026752681,-0.8440943360,-0.5575982928,0.3867180049,0.9236090779,0.2406300455,0.9912159443,0.5896103978,0.3247047365,-0.0631345809,-0.8472291231,0.4703036845,0.5695025325,0.1397066861,-0.2487258315,1.1258966923,-1.0027259588,-1.0338441133,1.1072216034,-0.7892639637,1.6951647997,0.1438161433,0.0723784938,1.4623792171,0.6937738061,0.0695512146,2.1728897095,-0.3971661031,1.0819935799,0.3043802977,1.4986712933,0.6935018301,0.5144372582,-0.7089034915,2.0252778530,2.0352370739,-0.5140570402,-0.4385772347,0.1052131802,0.7070764899,1.1298842430,0.9526279569,-0.3403368294,0.3679586947,-0.5590703487,0.5506619811,-1.2554082870,0.9212560058,0.3789348006,0.4162181020,0.8884142041,0.8954343796,1.3765288591,0.1669536531,2.1947975159,-1.0765970945,-0.4415110648,-0.8056438565,-0.5395519137,1.4391063452,-1.0550506115,-0.3433833122,-0.7339180708,-0.5391919017,-0.2920258939,2.4956951141,1.7541620731,0.3999114931,-0.5657392144,-1.4513942003,0.0673754513,1.3311697245,-0.9032511711,-1.1899926662,0.9078088403,-0.5799974799,-1.0377428532,-0.6368058324,0.3528950512,0.4219335318,-0.1826667786,-0.5943683386,2.2517185211,0.7124376297,-0.5504871011,-0.8670164347,-0.2532586455,-0.9611055851,0.5109337568,-0.9788072705,0.7710443735,0.0522208549,-0.1642433405,-0.2398169041,1.0718778372,-0.2769748271,0.8341576457,-0.9128410220,0.6022919416,-1.1872302294,0.5882807374,-0.7837983370,0.0029770739,0.9149373174,-0.0023518181,-0.3423418403,-0.2342749983,-0.2588711977,0.5861242414,0.2901591957,-1.9919506311,-0.5077320933,-1.0557484627,-1.3903524876,-0.8617141247,1.8194987774,1.4909827709,1.1573592424,-3.5810458660,-0.9443206787,-0.6563444734,-1.9958139658,-0.4275041521,-1.9974306822,1.0515087843,1.4036556482,-0.8239413500,-0.2179917097,0.8843204975,0.4367180169,-0.9152120352,0.5092750192,-0.4325858355,-0.2898980379,0.1608916968,0.0433968008,0.7225730419,-2.3706922531,-0.1152571887,-0.2443416417,-1.7064037323,-2.2502951622,0.9623617530,-0.9202928543,0.5554278493,1.4936690331,-0.8538382649,0.4118251801,0.4199622273,1.0834783316,0.7692968249,0.9061453938,-1.7944680452,-1.6934518814,1.0800925493,2.9148349762,-0.6273124814,-1.0704936981,-1.0964157581],[1.2692940235,-0.0670206621,0.4683865309,-0.2278233916,-1.6969687939,-0.1294624507,1.3910541534,-0.8832511306,-0.5370650291,-2.7819578648,1.0052194595,0.0291601941,0.1069035754,0.1541337669,0.4313090146,-0.0532100685,-0.2221809328,-1.2927292585,2.0467059612,0.6717846990,-1.4643356800,-0.5161830783,0.8706104755,-0.7400351763,1.7521550655,0.7392818332,1.1075246334,-0.9495987296,-0.2971502244,-1.7795100212,-0.3664148152,0.3774253428,-0.3637647033,1.0382292271,0.9270262718,0.9909930229,0.4617742598,-1.7854360342,1.1072719097,-0.4553121328,-0.1896470487,0.7345716357,-0.1584715992,0.1256743520,-0.4396451116,1.7899280787,0.4290977120,-1.1429880857,0.1356596798,0.2704877853,-2.9028491974,0.2635847330,-0.2302935719,1.4411734343,0.7899709344,0.7097681165,-0.1670908928,-1.1633665562,0.8314404488,-0.5223177075,-0.6025066376,-1.0426368713,-1.0971044302,0.0751154199,-0.0060958047,0.5358031988,-0.3496257663,0.1946886927,-1.0601308346,-0.3055567741,0.7915786505,-0.6497223377,0.1561425328,0.0680144206,0.1885385960,-0.0341186747,-0.0913931131,3.2813217640,0.7469103336,-0.7709512115,-0.6652572155,0.6616647243,0.8398932219,0.1677317768,0.2261482328,0.5642980337,-0.9490278363,-0.4119906127,0.8089081645,2.3489282131,0.0059719253,0.1277764440,0.4831945300,-0.1986074001,1.3758893013,0.9559911489,-0.3276863098,-1.9237053394,-0.0273960028,1.2066987753,0.5038545132,2.0547614098,-1.0013649464,-1.2404457331,-1.6175206900,-1.8106907606,-2.2587397099,1.4911020994,0.5077068210,0.1578625590,-0.2871367633,0.2982170880,0.2387153804,-0.8506680131,0.7121008039,0.2630681098,-0.4266733825,-1.2599084377,0.1100775227,-0.7961258292,0.0884582102,-0.4036722779,0.2554541230,-0.7653817534,1.2740881443,-0.9017828703,1.0660438538,0.2381526232,1.3428379297,-0.9337721467,0.5818094611,0.6203938723,0.5277485847,-1.3358016014,-1.0161755085,-0.3920041919,-0.3217634261,0.0977156609,-0.7099808455,-2.1828842163,-0.2492626160,0.5232057571,0.6404082179,-2.8672451973,0.0099505708,0.0627208427,0.5408713222,-0.8533446193,-1.0616909266,2.1865887642,0.7885946631,1.6995860338,0.8987043500,-1.2937207222,-1.3201901913,-0.3269158304,-0.5847586989,1.1706972122,-0.3119771183,-0.8133928776,2.2317814827,-0.0656682402,-1.2794408798,0.7121872902,0.6827929616,-0.9849511385,0.2409691066,1.4361633062,0.8795505762,-0.1011779606,-0.3517198861,-1.3311268091,0.0398764387,-2.5397872925,0.5240123272,-1.1318515539,0.1736470163,2.3500032425,-1.0210710764,-0.9013856053,1.2843419313,0.1951693445,0.2922241092,-0.9190829396,1.3920661211,0.3929746747,-0.2270657122,-0.0221442040,0.5061912537,-0.7442373633,-0.3657039106,-0.6952665448,0.6621089578,-0.1160282418,1.0427018404,-0.4019634724,0.3994499743,1.1279504299,0.1322755814,0.0824102312,0.1473124176,0.7525485158,1.3919463158,0.9325844646,-0.2164360136,-2.0686099529,-0.3924939334,-0.2780714035,0.0101592699,0.6358350515,-1.4313091040,-1.1910624504,-2.2517459393,0.1745254099,1.0222985744,1.6670593023,0.5249434710,-0.1815162599,1.8214089870,-1.4406580925,-0.9919193387,-0.5310760140,0.2919684052,-1.6313235760],[0.3511316478,-0.9309894443,1.1974879503,-1.6794967651,-0.8118848205,-0.8416833878,2.4623873234,0.3009655476,-1.4257361889,0.6312994361,2.2594048977,0.6614672542,0.9749627113,-0.8439388275,-0.6263849735,1.0214941502,1.8435775042,-0.7835627198,-0.5893369317,0.9996767044,-1.7351483107,0.5586682558,-0.9594988823,1.1005119085,0.2692739367,-1.0257135630,-0.7306185961,-0.3707855940,1.0002390146,1.7818371058,0.2506727278,-0.1208279952,0.7990749478,0.4406612515,-2.7838218212,0.3292020559,-1.1683198214,-0.7424931526,-1.6718312502,-2.2292375565,-1.5895670652,-2.1763262749,0.6465926170,-0.2387440056,-1.2814507484,1.5724308491,-0.0866635591,0.3556214273,-1.5556114912,2.2516131401,-0.0788190961,-0.9873566628,-0.5010582209,0.5617690086,-1.3074728251,-0.6975981593,-0.2382981330,-0.8825020194,-0.3499997556,0.8037531972,0.2698097825,1.4658676386,0.8141592145,0.6054009795,0.7439873219,-0.9993648529,0.3399610519,2.7317314148,-1.0810840130,0.1323891431,0.3207344413,-0.6511629224,0.3457941711,-0.7831311226,-0.9879703522,0.3776917756,0.8772977591,-1.7880016565,0.2965702415,-0.9034597278,1.7864934206,0.9782868624,-0.2956964970,0.6477823257,0.0506903753,0.5080415606,-1.3493453264,0.2018167675,-1.4426267147,-1.1808420420,1.0383150578,0.9860555530,1.8076187372,-2.0575242043,-0.6333572268,-0.2041875720,-0.9048840404,-1.3473826647,-0.2191231549,-0.5989145041,-0.9193871617,0.7825738788,-0.6629906893,-0.0044529312,0.4017296731,0.5302216411,-0.6302867532,0.1900966913,0.1169983447,0.6267263889,0.2086966336,-0.3434092402,-0.0818031728,0.2871176302,1.8508354425,-0.4121738076,-0.3695137501,-0.8422460556,1.0362706184,-1.5687952042,1.3191411495,-1.4238070250,-0.1425267160,0.1950446665,-0.0501813479,-0.9971384406,0.5666148663,-0.7734076381,2.1213474274,1.0192799568,-0.2502143681,-2.5300765038,-0.1859777421,-0.5562409759,-2.1014626026,-1.5545033216,-1.9805938005,-0.3614606857,1.0864875317,-0.6382172108,-0.1713171601,1.4588235617,-0.1968735009,0.3457186520,-1.2535402775,-0.4798072278,-0.5499140024,-0.6521920562,0.0745719150,-1.3990981579,1.5811384916,0.9943895936,0.8224885464,-0.9130792618,0.6656284928,0.4408087134,-1.0863834620,1.6354051828,-0.1830042750,2.0229175091,0.9130825400,-0.7718485594,0.5636826158,0.9207437038,0.6272261143,-0.0113689406,0.1329929233,-0.7101846933,0.7689437866,-1.1801084280,-0.2835783064,0.4113340974,0.5327725410,-0.7214313745,-1.5421614647,-1.9424954653,-1.6779434681,-0.0172744263,-0.5212426782,2.3069400787,0.4098668694,-0.3884662688,-1.4270358086,-0.4474042058,0.0626090840,-0.6908984780,1.6261773109,-1.3247954845,-0.9704781771,-0.8681657314,-1.9094009399,0.7300789952,0.0771100372,-1.2461156845,0.3015325367,0.3156217039,-0.0172931459,-0.4923977852,0.8767017722,-1.7975519896,-1.3432596922,-0.1768195033,0.2010150999,-0.0109114358,0.2074261457,-0.0014144401,1.3219326735,-0.1442534029,1.5599341393,0.5865324736,-1.0671136379,0.9432078600,-2.4022483826,-1.6965386868,-0.3520489037,-0.6729676127,-0.4498138726,-0.1366031319,2.2423534393,-0.6495697498,-0.6955966353,0.2690102160,0.0754883811,-0.2480758727],[0.4548644722,-1.6006692648,2.9957056046,1.1254948378,0.1499847472,0.1920881867,-0.5768033862,0.2211746275,-0.2526126206,-1.2733937502,-2.1981089115,-0.1316194385,0.6110743880,-0.4321219325,-2.6817026138,-0.1910612583,0.9476387501,-0.9832076430,-1.1690601110,0.3118586838,0.1494942456,-0.0717568323,0.1027661785,1.6578212976,-0.4804956317,0.4835452437,0.1303046048,0.8426531553,-0.4875191748,1.4141306877,1.9516247511,-1.5209347010,0.1807358265,1.0086404085,0.1693771333,-0.6385271549,0.3150402606,0.2134657651,0.1275645643,-0.2692812979,-0.4176293314,-0.5310031176,-1.2867263556,0.4024953544,-0.6629074216,-0.8900157213,0.0173238255,-1.3357878923,-1.4739190340,1.2099581957,-0.8706635833,-1.1307188272,2.5382192135,-1.3760643005,-0.8997769356,0.0810065046,-0.9796665907,-1.9800330400,-1.3330463171,-0.0266266223,0.6131624579,0.7309258580,-0.4710527658,1.8186960220,1.0128185749,-0.2675081193,0.3097981215,-0.3390848637,1.3907425404,-0.1129092425,-0.1128220260,-0.7463982105,0.3480891287,0.1308476329,-0.8871914148,-1.1553707123,-0.8099475503,0.9398341179,0.6741917133,0.5551754832,2.3883841038,0.0637566000,-0.3006283641,1.7931590080,0.6349275708,0.0974078625,1.9796898365,0.9637216330,1.2067064047,2.6625752449,-2.9752113819,-0.2145902961,0.8895272017,-2.5124297142,-1.1934756041,0.6111067533,0.5439530611,0.9195749760,-0.0145593109,-0.1723228544,0.8032369018,-0.1235672086,0.8890910745,-0.5182160139,0.6868421435,1.3251487017,2.2621772289,-0.9907764196,1.2263659239,-0.7818847299,2.5534036160,0.0250207074,-0.6922243237,0.4500946105,0.8247621059,-1.5440511703,1.0582951307,0.9684923887,0.0219670925,-0.6095378995,0.2694644332,0.6649788022,-1.0967034101,-0.7213160992,1.1593426466,1.0333893299,-1.1508731842,0.6200610995,1.0737234354,-2.6945307255,1.0907959938,-0.4236430526,0.5467333794,0.3633519411,-1.4687079191,0.2660523951,0.4187117815,-0.2689535320,-0.7421191335,0.4190295637,-0.0754861236,-0.9021491408,1.7387038469,-1.0696322918,1.2910215855,-0.9210324287,-0.5588905215,1.5117577314,-0.7016773820,1.5223786831,1.0122696161,-0.4869149625,-2.6730711460,-0.4777221382,0.3872422874,0.1491771042,1.2386612892,0.1987817436,0.7457051277,1.2785915136,0.1476992965,0.2074522674,-0.4072807729,-0.5786799788,-3.3096375465,-0.6840371490,-0.8496714234,-1.0677458048,1.0516594648,0.3639971912,0.0243675150,-0.0318917781,-0.0292874929,0.4616909027,1.9434530735,1.6624674797,0.8748117685,2.0247778893,-1.5770218372,1.6974775791,0.7939242125,-0.1475799233,-0.4257296026,-0.9291113019,-0.1765965670,-0.9489408135,-1.0270024538,0.3457305431,-0.1267700195,-0.8588191271,0.0220551956,0.4580956995,-0.8898931146,0.6698141694,1.1612095833,1.5577360392,-0.0882073417,-0.1198285371,0.5890731215,-0.3089555502,-0.9081842303,-0.1492018104,-1.7106012106,-0.7759175897,-0.4991606772,2.0636332035,-1.2211964130,0.2082914710,-1.8170255423,0.1874208301,0.8306953311,0.1591050774,0.1346683651,0.5078101158,0.3186313212,0.0747938156,1.1232910156,1.1165424585,-1.5080624819,-1.8543579578,0.4946228862,-0.3128373027,-0.8260221481,-1.1768620014],[2.7075240612,-0.8599362969,-0.1943347603,0.3728359938,1.5643889904,0.1224704236,-0.2554540038,-2.4267845154,-1.2763670683,0.1865660548,0.9339870214,2.1913509369,1.0479716063,0.5310890675,-0.4444085956,0.0102695441,-0.4931077957,-0.8591019511,0.3635742068,0.1819271296,1.4570590258,0.6254669428,-0.0766212940,0.7366033792,0.4073336720,0.3157582879,1.2674711943,1.5260124207,-0.1460522711,-0.6169248223,0.7802696228,-1.1895266771,-1.6959145069,1.0615260601,0.7290648818,-0.0039269542,-0.2192747742,-1.2065370083,-0.5926264524,0.4990629554,0.6350317001,0.2500172257,-0.7767462730,0.1204522327,-0.0855962262,-2.9598872662,-0.0594991967,0.0107230311,-0.6957206726,0.6268827319,0.8848359585,-1.0965247154,-0.3538480401,-0.2793050408,-0.6663282514,1.6578052044,0.9893534184,0.6942030787,-0.0255823508,-2.1172115803,0.8453764915,0.0800878629,-0.0446344092,-1.3436617851,-0.3629655540,0.7153369784,1.8806054592,0.5140509009,1.3209331036,-1.5869755745,0.0630695596,1.0406191349,-1.1759856939,0.7265060544,0.5853495598,-0.4912182987,-2.3420684338,0.2360036075,-0.0184030049,-0.2514174879,-0.5149710774,-0.9576873779,1.1475124359,-0.3164398074,-0.6195050478,0.9377791286,0.6614924073,-2.1293914318,-0.8671734333,1.8095625639,1.2773045301,0.7302829623,0.9080862403,0.4408243597,0.1983782947,-0.6720163822,-0.5482321978,0.1505797505,0.5328464508,-0.5543004274,-0.2349988371,0.3065464199,0.2317438871,-0.2143950164,-1.9504706860,0.3647886813,-0.9620810151,-0.5049009919,1.1725252867,-0.2122905999,-0.0769631341,-1.5674600601,0.1936446875,0.9210233688,-0.7128981352,-1.8007926941,0.3849214613,0.9164446592,-1.5944501162,0.4832324684,1.2739291191,0.3470695019,0.7192894220,-1.2342510223,-0.5707786679,0.3982577324,1.6125490665,0.8283737302,-0.4355474114,-0.5516900420,1.8830914497,1.4993001223,-0.4664620161,-0.6440277100,-0.1982753575,1.7335997820,0.7828823328,0.4520802200,-1.0521594286,0.6122381687,2.5034127235,-0.2546629012,-0.0021838548,-0.3127757013,-0.2439358383,2.8342952728,-1.1100224257,-1.8255298138,-0.2851569653,0.6020030379,-0.0797088295,2.1328616142,-1.2060884237,-1.0391825438,1.9026936293,1.5160888433,0.2516080439,-0.7265576124,0.6257002354,0.2562513053,0.3520869315,0.2826059759,0.7028957605,-1.1565545797,3.2492113113,0.6998537183,0.3435056210,0.0474957936,0.0238234904,1.0075225830,1.0553672314,-0.5581072569,1.4102920294,-0.8471719027,-0.3154569268,0.3686653078,0.3602911532,0.4066274166,-0.6914747953,-0.2434081584,0.1288081408,1.6579293013,1.3141508102,1.4821339846,-2.0403368473,0.6059473753,-1.0832898617,-1.4585893154,-0.3728159070,-1.8068834543,0.3847947121,0.5434930921,-0.3806275427,0.8868139982,1.4337956905,0.3013240099,0.3587332964,-1.2386696339,-0.8911620975,0.0859268382,1.1943483353,-0.6367681623,-0.6623798609,-0.2061398029,-0.5292823315,-1.5271534920,0.3254251480,-0.0908186063,-0.7286639810,1.3047002554,-0.0827621073,-0.1901878268,-1.0190961361,0.9524612427,1.5536764860,0.1080959290,0.7212635875,0.1993038505,-1.4570226669,-1.6779817343,0.3893408775,0.3303967416,1.0638159513,1.7132321596],[0.8575462103,-1.6333636045,-0.4927726984,0.7085943222,-0.6649975181,-0.7540698051,0.3705411851,1.5952237844,-0.9408431649,0.8396894932,-0.0682655498,0.1102770716,-0.1911044121,-0.3837050498,0.1033612564,0.4805106521,1.1790075302,-0.3850603402,-1.5772523880,0.1418904960,0.2107743323,1.0745302439,-0.1163458973,-0.2989508212,-0.1717454642,-0.1741078496,-0.5722891092,-1.1931658983,-1.0240820646,1.3909329176,-0.5920624137,-1.3037887812,-0.4021603465,0.4387072921,-0.4330362976,1.8886967897,1.1421550512,-0.6339848638,-0.4418257177,0.3721695840,0.2701981068,1.3258050680,-0.5577306747,-0.0739254504,0.3768513799,0.4234072566,-0.5511019826,1.1147866249,2.3339898586,0.4122173786,-0.5530776381,0.3115015626,0.0289805699,1.5773285627,-1.2268995047,0.4624019861,1.0431036949,-0.2830632031,-0.6748375893,-0.0909511521,-0.0818173736,-0.7964636683,1.1432673931,-0.5838388205,-0.7343836427,0.8128823042,-1.9741328955,1.0655366182,-0.1382042468,-0.4833156466,-1.5031466484,0.0463636853,-1.9227097034,-0.6942155361,-0.0444220677,-2.9480149746,-1.3248479366,-0.7946349978,0.1203350052,-1.8178205490,-0.8409462571,0.0176582448,0.7654770017,-1.1241449118,1.0637383461,1.5732291937,-0.2002674639,-0.3629679680,-1.1078330278,0.3732503355,0.0650348887,1.0154954195,0.5308608413,0.8413332701,-0.8172613978,-1.2249294519,-0.1462770700,-0.8106113076,1.2757968903,0.8820825815,0.8402888775,1.7235717773,-0.2222751230,-0.4631427824,-2.0011272430,-1.3081351519,0.3727766871,0.2681252360,0.9854740500,-2.2283413410,-1.8407127857,-0.8173459172,-0.1097566634,-0.2490268350,0.0852077156,-1.7533066273,-0.0306593943,0.0494587459,-1.7486133575,1.0223621130,0.9939898849,0.8368474841,0.4449765682,-0.9100048542,0.1528287232,1.8625997305,0.2812252343,-0.4299602211,1.2618894577,0.4911216199,-0.2961321175,-0.3500322402,0.9766417146,-0.6072682738,-0.3793948293,-0.2228083014,0.7391654849,-1.3898029327,1.9329074621,-0.9888586998,-1.6468340158,1.3287822008,-0.4032287598,-0.6147022843,0.8088668585,0.2952104509,0.4654255807,0.4560590386,-1.1324355602,0.8947141171,0.4242992103,-0.7234010100,0.1395297050,-0.5978423953,-0.0356286205,0.9698703885,0.5998618007,1.1406219006,-0.3814889491,-0.9062640071,-0.4771881998,-0.2948560417,0.7076987028,-0.9771244526,-0.5480884910,-0.3703163564,-0.2734317183,1.2818864584,0.8690806031,-0.2168426067,-1.0013710260,0.3965295255,-1.3480749130,-1.3271099329,-0.3585632443,0.1892410517,0.3836632073,-0.3862533867,-0.1512429714,0.8624945879,-0.5295600295,0.9678261876,0.5995805264,0.8620498776,-1.7274713516,0.9354030490,0.8139658570,-0.8361479044,-0.1532518119,0.0936337113,-2.4775524139,-1.9780313969,0.2005530298,0.8046504259,1.7265368700,-1.1824663877,1.0006707907,0.2549159527,-0.5395684838,-1.3906004429,1.1599717140,-1.1941506863,-0.0242273454,1.0806976557,-0.6423122287,-1.7502503395,-0.0535725392,1.0014064312,0.3842054307,-0.9795367718,0.0303774159,-0.5324507952,-0.3401973248,-1.1212528944,0.0285949763,0.4956311285,-0.2267297208,-1.3515195847,0.4184161723,0.1721993089,0.7958292365,0.1263063103,0.2711284459,-0.2135034651],[-0.7038419247,0.2187733203,-0.5362712145,0.1083598137,-0.2600317895,-1.2274924517,0.6941085458,-1.5299791098,0.4951930940,0.0788252577,1.3300241232,-0.9726471305,1.7447634935,0.7137358189,0.4610167146,-0.3599465191,-0.1522779316,1.7052100897,1.5464543104,-0.1919985265,-0.2842799425,0.6308130622,2.0006842613,0.4859058261,0.8829320073,-0.1657233387,-1.2302795649,-0.5572073460,-0.8428971171,-0.0465976670,-0.2116975784,-0.1333411336,-1.3210440874,-0.6006979942,-0.4126179218,0.0754443184,0.7098649144,-0.7600352168,-0.6254181862,-0.4199784398,-0.0825512707,-1.0666607618,1.6464910507,-1.1209568977,-0.7917913795,0.1129865870,0.0059932861,-0.4723086357,-0.2796922922,0.5758187175,0.9736551046,0.2789586484,-0.5730145574,-0.4830294251,-0.2489068210,-1.9525696039,-0.6475285292,-0.2628311813,-0.7734012008,0.8395585418,2.5376198292,-0.3365868330,0.5846264958,1.0598243475,-1.1626652479,0.0405477248,-0.9395235181,-0.2485307753,0.0545488633,-0.9857591391,2.4720458984,2.0487780571,-1.2066218853,-0.0878543928,0.9413759112,0.9323689938,-2.0413239002,0.2184711546,-1.9071009159,1.1700785160,0.5026462078,1.8074660301,-1.5341821909,-0.2498243898,-0.4873360395,-0.6551302075,0.8344846964,0.3601314127,-0.4684588313,1.2849541903,2.2285103798,-1.0161749125,-0.0508062355,0.7545269728,1.4866827726,-0.9345136285,0.7692088485,-2.3289158344,0.2548599243,0.2512003779,-0.9487016201,0.2089540064,-0.9035751224,1.4267714024,-1.0731743574,1.7076709270,1.2272638083,0.2197493315,-0.5237355232,-0.1973887086,-0.2108744830,-0.5565663576,-0.6995753050,-0.5322136283,0.3217452466,1.1632359028,-0.6271314025,0.3296809494,1.0473831892,0.4364182651,-1.3696779013,-0.2976042032,0.0399223492,0.7550673485,0.5357517600,0.4226417542,-0.0695470870,0.1780906320,-1.4290182590,-0.4919155836,-0.6635327339,-0.6278638244,1.0992287397,-1.2899185419,0.5348879695,-1.5645805597,-0.6093159318,1.9165036678,1.6862343550,1.2086102962,-0.5938796997,0.2510013580,1.5818654299,-0.0920653418,0.4793538153,0.4611625969,0.4175966382,-0.7902413011,1.3530526161,2.4345374107,-0.2522033453,-0.4852724969,-0.9632862210,-1.0252519846,-0.7647008300,2.6910192966,-0.3028256297,-0.1939556748,-0.6827206016,0.4865101576,-1.1748796701,0.9050163031,2.6230053902,0.1278017312,0.9562465549,1.0719659328,0.7716265917,-1.8127437830,-2.0730187893,0.2582882345,0.6441308260,1.4222255945,0.5335395932,0.4913542569,-1.1710500717,1.2177139521,0.6230660081,1.5083622932,2.5376482010,-1.0855648518,-1.1384391785,0.4204219878,-1.2163747549,1.5409317017,-1.7850402594,0.2881009877,-1.7824188471,-0.6656666398,0.2255385816,-0.2045584470,0.6886078715,0.4971415102,0.4256645739,-0.2858218849,-0.1442836076,0.8134331107,-1.1828783751,1.7522083521,2.3498823643,0.1647088081,1.1515992880,-0.7531849742,1.1517670155,-2.1913006306,0.9838366508,0.7988774180,1.2420282364,-0.3205684125,-0.0106539931,0.1312756538,-0.5737478733,0.6149320602,-1.3822107315,-0.3531156182,-0.5641967654,-1.1561065912,0.8050509691,-0.3803443015,-0.7108378410,-1.6383564472,-1.3067328930,0.8056710362,-0.9770047665,1.4766548872],[-0.5168541670,0.3145562112,-1.1484050751,0.7664395571,0.0075045303,0.0210218653,0.5821520686,-0.0796812326,-1.5467196703,0.1330328286,0.9529478550,0.6269364357,-0.2962416112,0.4320018291,-0.5550686717,0.9274399877,1.1973664761,1.2910717726,-1.4917968512,0.0722649843,0.5383219123,-0.2111446708,-1.4312043190,0.1909869164,-0.2979548872,-0.1053253263,0.4545270801,-0.1573643684,-2.6192853451,0.3299501240,-0.7100323439,-0.5290266275,0.6963036060,1.0016850233,0.3713839650,-0.0537927784,0.5277480483,0.9580436945,-0.3080750406,0.5621162057,1.1065112352,-0.1435552835,-1.2238268852,-0.6453787088,-0.0446425751,2.3824806213,-0.3643109500,-0.8466729522,1.1849539280,-1.9625260830,-0.2382405549,-0.3108474314,0.7180377245,-0.2890521586,-1.0908354521,0.3532983065,-0.9382154346,0.5689409971,-0.3626126945,0.1292094141,-1.3126652241,-0.4163072109,-0.8589738607,-0.0338691622,-1.0064257383,0.1432997137,-1.2719420195,0.4668884873,-0.3053086400,-0.0977246687,0.2234309167,-0.0470772311,1.0469348431,-1.3022555113,-0.8693733811,-0.1281105727,0.8558647037,0.8242882490,1.8926994801,2.1205914021,-0.3344083428,0.7537657022,-0.1441276968,-0.7482367158,0.4418944120,0.0492459051,0.3857703805,0.6495937109,-0.9396342039,-2.2726123333,0.5253503323,-1.3726311922,0.7066520452,0.8941339850,1.0972421169,-1.2170317173,1.4726543427,1.2693320513,0.7312103510,-1.4249899387,-0.4283734262,-1.2901015282,1.9668451548,0.6691100001,-1.5574185848,0.4526828229,-0.8244184256,-0.4453442395,1.3221229315,-0.2892827392,0.2289659083,-1.0478963852,0.4761168361,-0.2551328838,-0.4333049655,-1.1237722635,0.8928756714,-1.5040786266,-0.9676579237,2.4354195595,0.2679666579,0.8277609944,0.3292419612,-0.1868321896,0.6399228573,0.0246101934,-0.5365167260,-2.2954347134,0.9310479164,-0.1232519746,-0.7564478517,-0.4085958004,1.0798666477,0.4010500014,-0.0367867351,1.1169235706,-0.9824579358,-0.0519333407,1.1601675749,-0.5204966664,-1.4007287025,0.3084657490,1.1606221199,-0.2109643072,0.4174713492,0.7629770041,0.9784686565,0.6438628435,1.3034147024,0.0565330721,-0.1181432754,0.2954445481,-0.0077796308,-2.3304641247,-0.6992905736,-0.1077904403,1.6501281261,-1.1473655701,-0.4608682096,0.0156514887,1.2672146559,0.4234745204,0.1822052300,0.4332121909,0.4516955316,0.7653177977,1.6070424318,1.0253183842,-0.2590387762,-1.0479173660,-1.6088930368,0.1823649406,-0.9678751826,-0.7864817977,-0.0212973952,0.5997629762,-1.3160744905,-0.9506019354,0.5000987053,-0.1930724084,-1.2007164955,-0.9722674489,-0.9011637568,0.9087528586,-1.0918318033,-0.0354075357,-1.4514678717,0.9590007663,0.6602175236,1.3757650852,-1.0226017237,-0.8164103627,-1.0483584404,0.3173210919,-0.4989819825,1.5050772429,-1.3538606167,-0.3477463424,-1.2884012461,0.5413200259,-0.0952065215,-2.5451941490,0.8496850729,-0.8102738857,0.3639152944,1.0038706064,0.8044942617,0.2769761086,-0.8043497801,-1.0582702160,0.9640245438,-0.9525019526,0.2505484223,1.7418069839,1.1239496469,-0.2312931418,0.5900598764,0.2988237441,-0.9258888364,-0.7468770742,0.6754369736,-1.1314466000,2.2557041645,0.3076329231],[0.1141875535,-0.7910183668,0.9181339145,-0.2924492657,0.1954403967,-0.0891278759,0.9766722322,0.8747189045,-1.0259243250,0.6990704536,0.5039597154,-0.2485073507,-1.1559615135,-1.5401313305,-0.5223922729,0.7553474307,0.5209439993,2.2820031643,0.6241645813,0.1820227355,-0.8249456286,-1.0354923010,0.0518557951,0.6551367044,1.0242308378,1.2647337914,-0.3354948759,-0.8350704312,0.1489525437,0.8589512110,0.6270372272,-0.5417641401,-0.2549240887,-1.0517722368,-1.3470808268,1.7237402201,0.2793397903,1.0387977362,0.3658451438,1.7909607887,0.1156069040,-0.6626947522,-0.6802643538,-0.3461824656,-0.1924321353,1.7015250921,0.9053295851,0.1271467358,1.0843471289,-0.3697059751,-0.8226702809,0.3727995455,1.2557933331,1.3963470459,0.8964594007,-1.0065754652,1.3490592241,-0.2992320955,1.1706128120,1.8907653093,-1.7348724604,-0.1472353786,0.5626885891,0.1557893902,-1.5522532463,-0.7784453034,-1.1265046597,-0.5841084719,-0.3729529679,-1.0711065531,-0.4288546741,-1.5960400105,1.3952161074,1.3239439726,-2.5381610394,-1.0360237360,0.3437467515,-2.5755796432,0.0326888487,-0.2264865786,-0.7182295322,1.0793272257,-1.1627691984,1.3597364426,-0.2644090056,0.1385675967,-0.3974082470,1.2229468822,0.9545928240,-1.6005281210,0.4387021065,0.3070425093,-0.0559233315,1.7117067575,1.7901859283,1.9087609053,0.2534044385,-1.0119205713,-1.4722032547,0.5966674685,-1.1095695496,1.1928594112,0.0251557846,0.4326552749,-1.1571912766,-0.6874693632,-0.3447971344,0.3170865774,0.7671335340,-1.7947992086,-1.3456940651,-1.4110554457,0.1072896123,0.1770090908,1.9528205395,-0.5405386090,0.5682284236,0.6587491035,0.9563859701,1.0822306871,1.2021546364,0.7378921509,-0.5463986397,-1.7351492643,0.6233202815,-1.3622575998,-0.4995904863,1.4538668394,-0.5640485883,-0.4524624050,1.5386605263,-0.4490965605,-1.7440617085,1.1747021675,-0.8111938238,0.8217701912,-0.5931746364,1.0802215338,-0.2538038790,2.3748760223,0.1568471640,0.6048847437,0.1726618111,1.2744444609,-0.0180251598,1.1177147627,-0.4916356206,-0.3290631771,1.3595130444,1.1628024578,0.6827121973,-0.2205483317,-0.2955637276,0.6364265680,-1.0403375626,0.8137436509,-0.2903918028,0.2760030925,0.4262830317,2.0322773457,1.5663291216,0.3227753341,-0.0340636298,0.0319516100,-0.3528116345,-0.0407276042,-0.2693226039,1.4170272350,1.7600680590,0.7054511309,1.4845144749,-1.2117332220,0.8845197558,-0.4061044455,-2.0150425434,0.0858845934,-0.3021506965,0.2173276544,0.1497766674,1.0825449228,-0.5751865506,0.4710103273,-0.1027014628,-1.6194263697,-0.7232217193,-0.9368378520,-0.2567723989,0.2749138474,-0.6792963147,0.2497180104,-0.1856022477,-0.5589736700,0.4200106561,1.6372632980,-1.2062540054,0.7298827171,-2.3512728214,-0.7061026692,0.6459239721,0.9156780839,0.2385244370,0.7191008329,0.1381670535,0.2389231473,0.8946838975,0.8140711188,0.3687241673,-0.8611537218,-1.2541856766,0.6498557329,1.6523033381,-0.7426189780,-0.2608479857,1.2524713278,-0.7394489050,0.1951705813,-0.5104800463,-0.1717242897,-0.4723890722,0.0514790267,1.2266012430,0.6670045853,-0.2716105282,-1.6211389303],[-0.7709470987,-0.4063224494,1.0450941324,0.6701294780,0.4660073519,-0.3111612499,0.4569470286,-0.3755842745,0.2935816944,-0.7299827933,-0.3773456812,-0.0588846505,0.5191531777,0.6253951788,0.5395801663,-1.1540787220,1.1528965235,-0.7550507188,-0.4361100197,-0.8517587185,-0.2650027573,-0.4558273852,-0.1218817607,-0.1632884443,-1.7337768078,-0.2454500645,-0.4841493666,-0.2076745182,0.9510493875,-0.1895506978,1.7178503275,-0.0648765713,-1.3134737015,0.0220830422,1.5550876856,1.1541197300,0.9463537931,0.8542752862,-0.3284910917,-1.6964749098,0.8325127959,0.5698849559,0.1683153063,0.4132466018,-0.4727652371,0.9073839188,0.3624867201,-0.5237817168,0.5296620727,-1.5915416479,0.7892110348,1.2042807341,-1.2157537937,-0.0659838766,-1.0977131128,1.8772678375,0.0199707374,-1.0153725147,0.9905872345,0.2679038346,-0.6155877709,1.6769979000,2.2211329937,1.2426955700,-1.1432538033,-1.3055896759,0.1563118249,1.5841238499,-2.2452995777,0.4084261954,2.3925390244,-1.5944696665,-0.2753798664,0.7038434148,-2.1634190083,1.5523954630,0.0431083739,-0.6386553645,0.1563305855,0.4483917058,-2.9002535343,-0.2124763727,1.2646445036,0.9671489596,0.1235830262,-0.6239078045,0.1781397164,-1.2477340698,-0.8786101341,0.0340745784,0.9605447650,0.3556161225,-3.0921344757,0.3453056216,-2.2019863129,1.5977934599,0.0587060228,-1.2354182005,-0.5321778059,-0.8846167326,1.3429207802,0.1889484078,-0.7125145197,-0.9847447276,-0.3402684927,-1.9771710634,1.4227645397,1.8663989305,-0.6787713170,0.0083999801,1.4120321274,-0.8796091676,-0.8530539870,-0.2235978842,0.7683705688,0.3531757891,-0.8423203230,0.8098526597,0.2967545092,0.1743304431,0.6617275476,1.5916795731,0.3336613774,-0.3279126287,-1.1561247110,0.8219054937,-0.0077004205,-0.0758062452,0.9670962691,-0.9370991588,0.1727723330,-1.0711120367,1.2559354305,0.6726061702,-0.1603709906,-1.5501614809,-0.0739242285,-0.6497949958,0.3207924664,-1.1628973484,0.4206815660,0.7446953654,-0.5787026882,-0.3373136222,1.4212529659,-1.6684862375,-0.1762335151,0.3530758023,0.4051634967,-1.2940163612,0.9606325030,0.4437426329,-0.2314557731,-0.0697056428,-0.2034419626,-0.7468641400,0.9059106112,1.1647446156,-0.9227508307,-0.7264728546,-0.7138158679,-0.7802699804,0.7355730534,-0.7290205956,0.5469813347,-0.7842488885,0.4940427542,-0.9318450689,0.3016936481,-1.1100772619,0.4054757953,-0.7490972877,0.5455661416,-2.0676968098,0.1622077972,1.2108730078,-0.5942135453,-2.1084134579,-0.3815261126,0.6267691255,1.2911912203,-1.4211307764,-0.0792804286,1.2469061613,-1.2400754690,-0.6282653809,0.6214656830,-1.9671771526,0.9299145937,-0.1225127801,-1.6832170486,0.3292508423,-0.6738933921,0.3709022999,1.6576431990,1.8432923555,0.1324210614,0.0297149438,-0.7616475224,-2.1235435009,0.1847538352,-0.5866488814,-1.1501641273,-0.1941118091,0.2461440712,-0.2032909393,1.0314236879,0.0010392868,-0.8659663796,-0.7367367148,-1.2445702553,0.5127481222,1.2003036737,-0.9545887113,0.3125925362,-1.1271697283,-0.0129429530,-0.7439265251,-0.2158260792,0.9935899973,0.6832865477,1.1826785803,-0.6790092587,-0.7337641716],[-0.3919575214,1.2301563025,0.1421116590,-1.9185277224,-0.1422203928,0.0144021548,-1.2244794369,0.8910320997,0.8001833558,-0.6226376295,-1.9729691744,-0.3718367219,-0.6450943351,1.4727654457,0.4042130411,-0.2300358117,-0.1837593764,-0.3389971852,1.0521665812,-0.5878276825,0.8147978187,0.2012608349,0.5305464268,-0.0704385564,0.6780981421,-0.5457631946,-1.0154840946,0.3954752088,-0.8780980706,0.6862570643,-1.6729224920,0.8030760288,0.4177638292,0.5515066981,-0.8780927062,-0.3644495308,-0.7639865279,-0.2581588328,-0.0097340150,0.1328005195,-1.3428583145,-0.5613111258,0.0030085165,-0.2764977217,0.3225880861,1.0092039108,0.2349234968,-1.1569188833,0.3209208548,0.8061857224,0.9328452349,1.5121496916,1.4083685875,-0.7483514547,1.3963418007,-0.2169407457,-1.0577676296,0.2618527114,0.0820487142,0.2296931744,1.1636767387,-0.6497741342,-0.7683633566,0.3333502710,0.8443777561,1.1035634279,1.1778227091,0.9579150081,-0.8281220198,0.4982230663,-1.7778943777,-0.7316639423,-1.6550662518,0.1353182048,0.4598369896,-0.0059965760,0.6672404408,0.6366089582,0.3507957757,0.2280575633,0.5433624983,0.6431276202,1.0851905346,0.8459165096,1.2322151661,0.9934787750,0.1766625792,0.1074638367,-0.8066105247,1.8972502947,-0.6080517173,-0.4537470043,0.5049105287,-0.0842246264,0.8485350013,0.9383783340,-0.1987407804,2.5588440895,0.9118668437,-1.2429528236,0.4617349505,1.2518606186,0.6015286446,-0.4189657271,-0.3078119159,0.0632796735,1.3473374844,1.3317080736,0.8101310730,1.3187366724,-0.0723575279,-0.6994635463,0.8570897579,-1.5637105703,-0.2297752500,-1.2368377447,0.4984486699,-1.7071242332,-1.1083687544,0.0562235564,-0.4866240025,1.7403073311,0.0683723688,-1.6624240875,1.0369526148,0.4173009694,-0.9422541261,-0.3057643175,-0.0225711260,-2.3352799416,-1.2981127501,0.1158406734,-1.7067408562,-0.8495888710,-0.2003936023,-2.6660759449,-0.1373704821,1.8758572340,0.4224371314,-0.9995551705,0.7457840443,-0.6162922978,-0.6332775354,-0.3346899152,-1.7673785686,0.1534919441,0.5546674132,1.5832332373,0.0081346072,1.0892113447,0.9690399766,-0.3808458447,-1.3704863787,-1.2239551544,1.3384201527,-1.7577083111,-0.4571655095,-0.2167893350,0.2889954448,-1.5132929087,-0.0505164191,-1.0508286953,0.9178054333,-0.5142904520,0.6449629664,1.2783317566,0.0204112101,-0.9757503271,0.8718924522,1.2775079012,-0.8486067653,-0.7375156879,-0.8481579423,0.2910692394,-1.4055770636,-2.4370431900,-0.2421151698,-0.1659164876,0.0634354576,0.8462583423,-0.3317031562,-0.0410253853,0.8783381581,-2.0833046436,1.9415665865,-0.4809909165,0.9779080153,-0.1633502096,-1.0577044487,-1.0115772486,-0.9951361418,-0.8234838247,1.3299692869,-0.3004491925,-0.3629858494,-1.1682218313,0.0332856104,0.0584488660,0.5619188547,-1.4354987144,-0.6348283291,0.5653066039,-0.7393788099,0.4992229044,-0.8277471066,-0.8877035975,-0.4010598660,0.1966238767,1.2567228079,1.7160092592,0.6613878608,0.9422925115,0.6065798402,0.1258662939,0.1002879664,0.3995673954,0.5345766544,1.5022646189,-0.6795228720,1.2247700691,0.3355742097,-1.9241726398,-0.5274044275,0.3859989941],[-0.1902812123,2.0812780857,-0.6521640420,-0.9521286488,-0.1160523742,1.2861061096,-0.9185868502,-0.0956847370,0.7666553259,0.0705903322,-0.9173219800,-0.6386165023,-0.2875458598,0.4759561718,-0.2991779745,1.0411814451,-0.2388623655,-1.9096094370,0.9282014966,0.1911507100,1.0705833435,-1.3557881117,1.5480512381,-0.0446682349,1.5442548990,-0.6466215253,0.8872923851,-1.2207497358,1.0928955078,1.7613279819,2.2592256069,0.0676627234,-1.4281723499,0.0194905587,-0.7902153730,-0.2634108365,1.2093659639,1.7138423920,0.4568839371,-2.7929172516,2.0279846191,1.9827568531,-0.3400610685,-2.5195252895,0.1077408493,1.2092367411,1.3849642277,0.4741785228,0.6720715165,-1.3682155609,1.3626445532,3.0927917957,0.2808203399,-0.0680369139,0.9278790355,-0.3832291663,-1.0694628954,-1.3236962557,0.2294029742,-0.1003467962,0.5837204456,1.4401695728,0.6600863934,-0.1587504148,-1.0510327816,2.8563518524,0.3107630312,0.0793141946,-0.7014832497,-2.2048156261,-0.1639941931,0.8510155678,0.8897792697,-0.2509429157,0.7656675577,-1.8384664059,-0.9290833473,0.8442848921,0.0714782104,0.7431453466,0.7526600957,-0.3000283539,-0.8917927146,0.3781193793,-0.8924313188,-0.7258448005,1.2255562544,0.4353717268,-1.0612089634,-1.0652147532,0.1068682298,1.5616360903,-0.5242556334,0.9643884897,0.7915732861,-0.3687317967,-1.5341099501,-0.6866817474,-0.0766677111,-0.5075598955,0.0492559411,-1.5805724859,0.4028103650,0.5294559002,-0.2617757022,-1.5436277390,-0.4214951396,0.7148287296,-0.7212103009,0.4432360232,-1.1655206680,-1.7276140451,1.6665664911,0.1276424825,-2.0064013004,2.2659885883,-1.1007127762,-1.7148460150,-0.9983851314,-0.1229355112,-1.5004343987,-1.0642398596,0.5147129297,-0.1959828287,1.0724027157,0.5129589438,0.2611027360,1.0203869343,1.8625689745,1.0872251987,0.1776206940,-0.6679985523,-2.3908557892,1.9265812635,-0.0750824064,-0.6875794530,-1.8375434875,-0.1266568005,0.4062012732,0.9031251669,-0.1045055613,-1.1981265545,-0.3047462404,0.7788920403,-0.5198747516,-3.0495319366,0.4344826937,0.5080818534,-0.0456377529,-0.6228168607,-0.3361916542,-0.4655468166,-0.8032031059,0.0986663103,-1.4225085974,-0.7214075327,-0.1927438825,-1.2750662565,0.3828608692,-0.2010356784,-0.3411757648,1.0567926168,0.8308400512,-0.3758359253,-1.7007135153,-1.0153723955,0.7814183831,1.3333725929,-0.0588039719,-0.1536338776,1.7262624502,0.5375443697,-1.4447143078,-1.0294841528,-1.0612380505,-0.1466866732,0.0248447470,1.2826215029,-0.6459833384,-1.5521104336,-0.7346411347,0.4375969768,-0.6678929329,0.2032340467,0.7792050838,-0.1258176714,-0.7511096597,-0.6474458575,0.2450655252,-0.9021635652,-0.5974918604,-1.1855431795,-0.8081023097,-0.3418832421,-0.0899530277,1.9315698147,0.0990892202,0.6180377007,-0.9881890416,0.7278859615,0.2622669935,1.4801809788,-0.4546771049,-1.0858519077,-1.0294415951,-1.2702590227,-0.2850361466,1.2038623095,0.4503741860,0.5454785228,-0.6018747687,0.4178525209,1.3235566616,-1.9712238312,-1.0972834826,1.2963863611,0.1940562278,-0.6993190646,0.1960473210,0.3936943710,-0.6680406928,0.2631404102,-0.0992291048,1.5811232328],[0.5356354117,-0.0902978182,-0.2183861434,0.0835172236,0.2231216878,0.4585287869,-1.0632811785,0.2157344222,-0.7864517570,0.0688186809,-0.8122203946,0.9727475047,0.4813622832,0.9644367099,-0.0104756039,0.2411717325,0.3121369183,1.1740385294,-0.8667576909,0.3110567629,-0.4314704835,0.1743235737,0.5403572917,-0.6680797935,1.0002886057,1.6705429554,1.1789067984,0.5091510415,0.2972907722,-0.3430694044,-0.8662321568,1.3162040710,-1.8510755301,-0.4570961595,0.9468869567,0.5335932374,-3.9929611683,-1.1525392532,0.4048165977,-0.7148792148,-0.0899725407,1.1440914869,-0.5628069043,-1.5965940952,2.3284101486,-0.6404005289,-1.9125361443,-0.8831542730,0.5702723265,0.3274931014,-2.0143942833,0.5962705612,-0.2118565291,0.2940602303,1.0698854923,-0.7329437733,0.5447551608,-0.7397634983,0.8467305899,-0.5462845564,1.9932767153,-0.0706784353,2.3692502975,-0.4771787226,-1.3188631535,-2.1861901283,1.1592558622,-0.2879912555,1.3112417459,-0.1735279113,1.7638275623,-0.1482204646,-0.5775184035,1.6037025452,-0.2173911482,0.8606928587,-0.7163739204,0.5022127032,0.6795402765,0.7757351995,0.9296262860,2.1179580688,-0.1177830398,-0.6922940612,1.8955492973,-0.1037179306,0.8602941632,-1.3067930937,0.0278356988,-1.8676902056,0.8719504476,0.2133634984,-0.4246577621,0.2922644913,0.5142137408,0.7787869573,0.3955132365,-0.0308551546,-0.5254090428,-0.0208215415,0.5863494277,0.2410605252,0.2540073991,-0.0505717807,0.3257252574,0.0478928015,0.0124214934,1.1643067598,1.0866931677,0.1932839900,0.7468745708,-0.4936366081,-0.1967490166,-0.5898027420,-0.3114001453,1.2001738548,-1.1749259233,-3.6432943344,-0.7237185836,2.1023168564,0.3772802353,0.9598003626,0.6345593333,0.9760369062,0.5842297077,-0.4238112867,1.0510704517,1.4212644100,1.2052621841,1.1709578037,1.2481642962,0.1114866585,0.7139697671,-1.6507397890,-1.1089572906,1.0787169933,-2.2450892925,1.2657155991,1.0965223312,0.4065240622,0.4205618203,-0.1879614741,-0.6797156334,0.9142844677,0.0223195404,-1.9562933445,-2.0345225334,-0.3191227913,-0.6532308459,0.7075082064,0.8971603513,-0.2222088575,-0.7548856139,-0.8718494773,0.5969509482,0.4012758136,1.4944665432,-0.0430379659,-2.5413269997,0.2290947139,-1.7839071751,1.7898751497,0.9773983359,0.8138353229,0.3676052094,-0.4572495520,2.3534555435,1.7259155512,1.3094695807,-0.2298201919,-1.5486131907,-0.4831325114,-0.1515334994,1.1911811829,-0.1305567771,0.7306992412,-0.9365200996,-1.0807114840,-0.4267658591,-0.6438264847,0.4536756277,-0.4718679786,0.0732847527,0.9452646971,-1.5916801691,-0.0438594930,-1.6057710648,0.2780228853,0.1078691557,-0.4012733102,0.5433638096,-1.8002210855,0.0649706200,0.1245789826,0.8710531592,0.1115417033,-1.9357055426,-0.5548510551,1.2520089149,1.4839948416,-0.5209493041,-0.9624369144,0.8466321230,-0.5569323301,0.2157223821,0.0956537575,-1.0298379660,0.4497537911,0.1934795082,-1.4493902922,0.3130516708,-1.3260329962,-0.2592625022,-1.8027895689,0.5699852109,0.5679404140,-2.5769262314,0.9336313009,0.7585353851,-0.6365656853,-0.4001408815,0.9239498973,1.9906063080,1.4039293528],[-0.5344263315,-1.6005463600,-1.1294572353,-0.6989622712,-1.0434752703,-1.8072863817,-0.2911771536,-0.3740274906,-0.2669128776,-0.6695302725,-1.0134661198,-1.7485753298,-0.2880801260,-0.0940565765,0.1530825347,1.2911791801,1.3317542076,1.1100609303,0.9411215782,0.6393887997,-1.3498729467,-0.1483654976,0.1755969822,-1.0591218472,-0.4806758165,0.0374209993,-0.2632440925,-0.2745012343,0.6968136430,-1.8665977716,-0.1406579614,0.7593407631,0.4745651782,-1.2644783258,0.6004345417,1.4316047430,-1.1685087681,0.2429454327,0.4121043384,-0.2837245464,0.0339051634,-0.4033333957,0.4898167849,-1.8071894646,-0.0860238522,0.3482603729,0.8476430774,1.7993699312,-0.4857821763,-0.9498022199,-0.0087727476,-0.9771353602,-0.8784105778,1.8610028028,0.4241184890,0.4889109433,-0.5960755348,-0.1157418191,0.2188333124,-0.9271811843,1.2049478292,-0.7487514615,-2.4948208332,2.3248777390,0.1948041320,-0.9396010637,-0.0695591941,1.1261837482,-1.4390196800,0.1074865907,0.2451537251,1.4741477966,-1.3714965582,1.4597207308,-0.2240107805,0.4680463374,-0.7798717618,-0.8509687781,-0.0445612594,1.0413498878,1.0824186802,-1.3900460005,0.8383417726,-0.5728988051,-0.7608184814,-1.4918603897,-1.1130791903,0.9705112576,-0.9052938819,0.0602722466,1.7449322939,1.3911231756,-0.2824020386,-0.0249941666,-1.8760643005,-1.9072006941,-0.2799922526,-0.4541512132,-0.3491178155,1.3265537024,0.3157596588,0.7349253297,-0.3306770921,-0.0793977678,-1.4547114372,-0.9750895500,2.5718612671,-0.4372460544,-0.3933745921,-0.2879850864,1.6932580471,-0.2667824626,-0.4276626110,-1.5534998178,0.5865116119,-1.1775484085,-0.3417048752,0.2226299047,0.8676363230,0.8768593073,0.4840772152,1.3084653616,1.7750610113,0.5020828843,0.8089952469,0.6637986302,0.2111792713,0.3420891464,-0.5921617746,0.2682724297,-0.4276701808,-1.0372554064,0.0703604743,-0.8647756577,0.5419068933,-1.7433658838,-0.3307072222,-0.5725274086,0.6627629995,-0.3101287186,0.5326774120,-0.7019532919,-0.8407902122,0.7224192023,-0.1563667655,-0.5524084568,-1.0491452217,-0.7639257908,-0.2581304610,-1.5388603210,-0.6796113849,0.8086640239,-0.1712522805,0.5022003651,1.1249178648,0.2038956285,0.1236568317,-1.4507499933,-0.4801124036,0.1786764115,-0.2940721810,0.9317100644,-0.3659346402,0.7731112838,-0.8082608581,-1.8103444576,-0.4421080351,-0.9311588407,-0.6190632582,-1.6836090088,0.6225354075,1.8004719019,-0.3675890863,-1.5402535200,0.5061526895,-0.7011044621,-0.2044154704,0.7838950157,0.6873581409,0.3003750443,0.2717193067,-0.7483507395,-0.2657977045,-1.1034845114,0.9093827009,-1.7455307245,0.2808555067,-0.6657431126,0.2908801436,-0.3246472478,0.1302491426,1.6909010410,-0.1852371842,-0.3363511562,-1.5134866238,2.3311779499,-0.0808570161,-1.1184997559,-0.1656611711,-0.2683204412,0.5825760961,-1.1025604010,-0.9419777393,-0.7417781949,-1.4885994196,0.1455330104,0.0818805471,1.8152869940,-0.3648020625,-1.0479524136,1.0624965429,-0.1902331710,-0.5713142157,0.6154579520,0.3105306625,1.3375319242,-1.4385323524,0.4703989625,-0.0446280465,0.2165392637,0.2389579415,-1.2069883347,-1.5069841146,2.8056120872],[-0.4463079572,0.4630065858,0.3852918446,1.0595120192,-0.1924976408,-0.7959891558,0.5518957376,0.1073498875,2.0644276142,-1.3379882574,2.1439268589,0.0684103593,0.1015025824,-0.6148912311,0.0664112791,1.3932160139,0.1381065100,-0.4723600745,0.6620807052,-1.3079439402,-0.9490098953,1.5134844780,1.3218621016,-0.4307395518,-0.0436251275,-1.3971976042,-0.9197613597,-0.1284685731,0.9387473464,0.0908714011,-0.6960137486,0.8947666287,-0.4395277500,0.0078539103,0.0331664793,-0.4104016125,0.2590791285,2.3322181702,0.5433509350,0.3989716768,-2.2112383842,0.9127438068,1.0851579905,0.0210881140,1.1078456640,-0.0505299978,-0.5449906588,1.7212568521,-0.7205293179,0.1527352482,-0.5153023005,0.0899326950,-2.0822553635,0.7137017846,-0.6866122484,-0.3113394976,0.3766130805,-0.3086671531,-0.0307912417,0.9296104908,-0.0212098304,-0.7371267676,-0.3977877498,1.2362489700,1.0222923756,-0.5818612576,1.8817348480,1.1494332552,-0.0183267836,-1.1659201384,1.5057953596,2.7619345188,-0.7941793203,0.0287719052,-0.7107766271,-0.2424992919,1.6478064060,-0.6357446313,0.8620203137,0.1403571814,0.3091052175,0.2087818533,0.9582784176,1.2466231585,-0.0951789767,0.0370424651,0.0931980982,-0.0357219428,-0.9581607580,0.6691345572,1.4618363380,-0.9407261610,-0.1881764531,0.1779287457,0.2282896638,0.0024585791,0.9555747509,-0.0863538906,-1.6428929567,0.4155323207,-0.4702777863,1.0200219154,-1.0292131901,1.9110612869,-0.3506657481,0.3369286656,0.1099356636,-0.8530461192,-0.5134611726,1.2525533438,1.9823640585,0.8887806535,0.1952644587,-0.3287902176,0.6369973421,1.1657168865,-0.3110114336,1.9505547285,0.5987884402,-0.3918232918,0.1895684898,-0.8541156650,-0.1494368315,-0.9396588206,0.5325658917,1.4862364531,0.5386831760,1.0839402676,2.7512357235,1.0610109568,-0.3076118827,0.0839342251,-0.2132758647,-0.4443840086,1.1139613390,-0.5371167660,0.6878733635,0.6211798191,0.1295899898,-0.1400801837,0.3262644410,-1.6013503075,0.3448503315,-2.4370861053,0.0091410428,0.4318372011,2.4131960869,0.7610641718,-1.2417212725,-0.5548300147,0.2673580945,0.4718003273,1.0521353483,1.0287045240,-1.2894179821,-0.0358988978,1.2699348927,-0.1881172657,0.5786081553,0.3217543662,0.5109526515,0.6527564526,-0.1036807373,-0.8156668544,0.0486890860,-0.8705747724,0.0336206816,-0.1434892714,0.0206560418,-0.5102793574,0.0312600732,-0.0512645692,0.3963692784,-0.9462928772,1.7282290459,0.0291529056,1.6862194538,1.0479409695,-0.1493015438,-0.4607142508,0.7110564113,-0.7917346954,-0.3492127657,-0.1107221022,1.2990982533,0.0823019817,1.3518792391,1.6583864689,-0.5439693332,-2.4000995159,0.4834471345,0.2429045290,-0.3895867467,1.5748558044,-1.8866602182,0.5014733672,-1.0549618006,0.6886458993,-0.5591328740,1.7372102737,-1.0586389303,-0.7030788660,-1.7595483065,0.5768519640,1.0110862255,-1.0807354450,0.7039963007,0.2570515871,0.2202721834,0.3636488020,-0.0619608201,-0.4254122078,0.1702727824,0.2321278155,-0.2789296806,0.3148307204,-0.2219217718,-0.0343165696,-0.6693635583,-0.7482005358,0.3152418435,-0.6094266772,0.5564612746,1.0861303806],[-0.3645062149,1.1158699989,0.7505072951,1.6706467867,0.9701129198,0.9791996479,-0.4747344851,0.8000352383,-0.8672477603,0.0212012902,0.1365957260,-1.9201898575,-0.6686220169,-0.2664861679,-0.7977192998,-0.1907032579,1.1660275459,-0.8042742610,0.9925575256,-1.2276042700,-0.7777420878,0.4423106909,-1.0314748287,-1.1646858454,-0.1148602143,-0.0822476819,1.2950305939,-0.1583028287,-0.3801110685,2.1181035042,1.1377567053,1.1714947224,-1.4710574150,0.4374298453,-0.3345782161,-0.2080748528,-0.4310014546,-0.3359941840,-0.6037350297,-0.4509580731,-0.2071041912,0.4953692853,-0.9209408760,-1.6250727177,1.3350207806,0.5784474015,-0.7418002486,-1.0781373978,-1.4611221552,0.8532691598,-0.2607238293,0.5639829636,-0.5600467324,-1.1986761093,1.9563639164,0.9690504670,0.2546267211,-0.3837289810,-1.2347412109,-0.1670784652,2.9372646809,0.8678464890,1.1291265488,1.5747507811,-1.6810820103,-0.2933495343,-0.6225114465,-0.3882344663,-1.3630297184,0.0092936410,-0.5639885068,0.2759515047,-0.2828184068,1.1528820992,1.1894996166,1.1782293320,1.7409573793,-0.3719946146,2.2909574509,3.0704162121,-0.9461196661,-0.2518333197,-0.6309245825,0.5706209540,-0.4163119495,-0.0939932466,-1.1150132418,-0.4613604546,0.0398271419,-0.2534232736,-1.9071460962,-1.8280466795,-0.4866101444,-1.3184527159,-1.4179421663,0.2209537029,-1.1249928474,-1.9038407803,-2.7663104534,0.6664661169,1.0175549984,0.1773152649,0.2420164049,1.5144679546,0.0260805730,0.3917195499,0.2187479883,-1.1952811480,0.0412240624,0.1335472763,-0.3340080678,0.0762820840,1.1268568039,-3.0501861572,-0.8245661259,1.7237170935,-2.8219740391,-1.8330252171,0.7809044719,-0.3621836007,1.7247868776,-0.1848417073,0.3827749491,0.8851286173,0.1156658381,2.0285098553,0.8909919262,-0.7913900018,-0.4023923874,1.4397592545,-0.2121118456,1.3215402365,-0.1021919549,-0.2679600120,-0.6018244028,0.0751775131,-0.2411542386,-0.8728830814,-0.7452493310,-0.9146154523,-0.3126709759,-0.0148961348,0.3722932041,1.4483546019,0.3125064075,-0.4321153164,-0.3628958166,1.5365076065,-0.4864256084,-0.4952484965,-0.4730603993,-1.5365190506,-0.7496447563,1.2180451155,-1.2079240084,0.6674391627,1.0864683390,-0.2232403457,-0.1725623906,-1.0300061703,0.8511503935,1.1540714502,0.2684497833,2.0633459091,1.5850899220,0.9936402440,-1.7767057419,0.2575030923,-0.2837025225,0.9801308513,0.0955806598,-1.3171532154,0.0740581304,0.7199848890,-0.3346158564,0.2480404526,2.2816629410,-0.7901241183,-1.2123996019,-0.7006688118,-3.1003596783,-0.4071684480,-0.2404175103,0.3279474974,-0.1991623193,0.8064158559,1.3032054901,-2.3590478897,-0.0860624835,-0.0414754823,-0.7682408690,-0.6589986682,-0.3123554587,1.5079144239,0.2001926452,-1.2423874140,-0.8025832772,-0.2944001555,0.7860500813,-0.5993912816,0.0452994257,-0.1173223034,-0.3171161115,-0.6040194631,0.4698894918,-0.2821411490,-1.8677010536,-1.0488799810,1.0262153149,-0.9576758146,1.3599735498,-1.7847856283,1.2750602961,1.3729448318,-0.0263605714,-0.9299097061,-0.1553459466,-0.5619153976,-0.2708025575,-0.5707209706,-0.3943482339,0.3243137002,-1.2652244568,1.3428239822],[-0.4716863632,-0.1471398324,0.5845143795,0.7798466682,-0.2970978022,-0.2879599035,1.1741337776,-0.3029748797,-1.0286338329,-2.5272741318,1.8863145113,0.5938357711,-1.5568106174,0.1661666781,1.0948197842,-0.4422018528,0.4641939998,-0.8608870506,-1.7977205515,-0.6338832378,-1.5577857494,1.3675371408,-1.8018686771,-0.4680710137,-1.3529676199,-0.3295443952,-0.0289831124,0.8985692263,0.5144635439,-1.5340685844,0.7415658832,-0.4952560365,-1.2680888176,1.5494275093,0.9896262884,1.1337209940,0.1172252595,0.5003512502,0.3104941547,0.8642488122,-1.9873039722,0.2901904881,-0.8896881938,-0.3933593929,-0.3691793084,-2.2398619652,0.4009266198,1.4610875845,-0.2109690011,0.3212065697,1.1923828125,-0.9447377920,-0.7677613497,-0.0199948680,0.7007796764,-0.9966187477,0.1880163848,-1.5960267782,1.1941541433,1.3885799646,0.3051355183,-0.4914822578,-0.0551109239,0.8271973729,-0.0875257626,0.5913065076,-0.3327606320,-0.6633206010,1.1625896692,0.5635477304,1.5795736313,0.7323051691,0.5140512586,0.7973369956,-1.0491675138,-0.8615369797,-0.0855538324,-1.1763240099,0.6985024214,0.7214810848,0.3039073646,0.7869974971,0.7645076513,1.6823736429,-0.3224197328,-0.6913388968,-1.0375618935,2.2595710754,0.3254954815,-3.2841098309,-0.9829556942,0.6173440814,-1.9146822691,1.6384691000,0.3188497424,-0.9541326165,-1.3458615541,1.1278688908,0.3943688273,-0.2816155255,-0.6670376658,1.7449932098,1.6885739565,1.6031520367,0.1641594321,1.3355803490,-0.5794470310,-1.4504963160,0.0844430998,0.2493409365,0.1463551372,-2.4972960949,-0.4465410709,-1.2534028292,0.9957426190,-0.7179170251,-0.9940697551,-1.5139750242,-0.3549970388,-0.0613587759,1.6371300220,-0.3277179301,-0.7049137950,-0.0345475487,0.8238404989,-0.0779935420,0.1792568564,-1.1482546329,-0.6932830811,0.6941252351,0.0120476391,1.0842584372,-1.1157293320,-0.2156767249,0.1917534322,0.2484143525,1.1325517893,-0.8084028959,1.9061423540,0.2140457481,2.5162801743,0.1694203168,0.2875690758,-0.4746890962,0.2268090844,0.6107716560,0.8992943764,1.2538052797,1.0825774670,-2.5980246067,0.8615135550,-1.2262650728,-1.2255992889,-1.7337546349,-0.0070146662,0.8009208441,1.0482748747,0.1379574686,1.4142774343,-2.7599356174,-2.6668968201,0.5720087886,1.0927113295,0.5224719644,-1.5818355083,1.6267343760,0.8453273177,-1.3154888153,0.7564472556,0.8823671341,1.7488454580,-2.1088161469,-0.3071520329,-0.9865061641,-3.3408041000,0.8878111839,-0.5089591742,-1.7002439499,-1.6869919300,-0.7074144483,-1.5640826225,0.0950985998,-1.5500789881,0.6208086014,-0.3661846220,-0.8483721018,0.1937151700,-1.3323022127,-0.3852874339,-0.2245144695,0.5444493294,2.3293950558,0.1018384323,0.0900604799,-2.0430760384,0.1845244169,1.0585149527,-0.4266940057,-0.2745861709,-0.9895468950,-1.4220205545,0.1086603552,0.7137717605,-0.7071261406,-0.6727264524,-1.1171315908,1.1003943682,1.4516644478,0.2674415112,0.7746139765,-0.4443411827,-1.4839503765,-0.4744228423,0.6212148070,0.1285764873,0.4126030803,-0.8194864988,0.2797495127,0.7431498766,-0.6712838411,-0.5915322900,0.0478469022,-1.9627013206,-0.1429765671],[-1.1604765654,0.7248870730,0.5285183191,0.3912655115,0.8200900555,-0.0187557004,0.3092218041,0.8215553164,-1.2147023678,0.8691491485,0.1782266051,-0.3846947849,0.2185835987,1.0123075247,0.5845950842,0.8014039397,-1.2928036451,0.2337188721,-1.0127472878,-0.1021579877,0.4277488589,-0.4217742980,0.8380681872,0.9014368653,-0.5033906698,0.3400486410,0.0378595553,1.4128333330,0.8601081967,1.4224760532,-0.5014793873,-0.1299059242,-1.0861430168,-2.1957287788,-0.4724318981,-1.2804791927,0.8848083019,-1.0232039690,-1.9286930561,1.9506202936,-1.2104187012,-0.8170687556,1.6399490833,-0.2306459248,-1.4863326550,-0.9082179666,-0.2592566311,-0.6489036679,-1.1253803968,0.2403939068,0.0076182713,-0.6155346036,-0.6273389459,1.2138336897,0.1259267330,-0.4360826612,-0.2855346501,-0.9321278930,0.2889509797,0.0191639476,0.3197908700,0.1481243372,-0.3380688131,-1.4064606428,-0.1481387913,2.0216214657,0.1358122379,0.3295311034,-1.3799560070,1.0495952368,-0.6023465395,0.7122180462,-0.2905054092,0.0740248412,1.8817989826,0.0416676588,0.6724176407,-1.2744238377,0.3802832663,-0.3467430770,-1.1206740141,1.2709239721,-0.0360667482,-0.4205509424,0.8479788303,-0.2158568501,-1.0330330133,0.4496297836,0.2781554163,-1.9734966755,-0.7712466717,-0.1627359092,0.2592321932,-0.5584867597,0.3639693856,-1.8700847626,0.0532161370,-1.0296891928,-0.7521325350,-0.2848433852,-0.1023412347,0.3919997215,-1.0115511417,-1.2037749290,-0.5682365894,1.3685394526,-0.0325128809,0.1605796665,-1.9592183828,-0.7536043525,0.5448845029,-0.4187681377,1.5351369381,0.7247617841,-0.1284294128,-0.7700523734,-1.3904502392,0.2299453765,-0.0612412915,-0.4955629408,0.4037678242,-2.3232903481,-0.5163176656,-1.1535639763,0.1474791318,-0.3062639832,-0.3380345702,0.4434403479,0.1649090350,-0.0253193621,-0.2594115436,0.3455506265,-1.1220024824,1.1439352036,1.9330021143,-0.2664181888,0.6555473804,0.4029106796,0.6844402552,0.5201416612,-1.7882884741,-0.0639681965,0.9145079851,1.1875967979,-0.2303394228,1.4555683136,0.0759437382,0.3192932904,0.1900645643,0.1694294959,0.2299467325,1.1252853870,1.3348319530,-1.4127507210,0.8178436756,0.3104344010,0.0983516201,0.3478089571,-0.8033730388,0.4526213109,-0.2079967260,-0.3332097232,-0.7742238641,1.0865453482,0.3817871213,-0.8886569738,0.9300866723,0.2929508984,-0.1157365143,0.7728258967,-0.6975427270,-0.9684841633,1.3194321394,-1.7829133272,-0.3412713706,-1.2375971079,-1.0775034428,-0.5838529468,-0.7925577760,-0.5509713888,1.3133839369,0.3206467927,1.3325715065,0.4925050139,1.9970691204,0.4951836169,0.1311973631,1.4893194437,0.5594164133,0.4966352582,1.2948005199,-0.8946422338,0.3424288332,-0.4328266084,-0.8383091688,0.3484825790,-0.5664611459,-1.5805536509,-1.2075260878,1.0009816885,-1.1555238962,-1.1081479788,0.2666878104,-0.0843278915,-1.4013813734,-0.0587022007,-0.5409751534,-0.9161397815,-0.0049323305,-0.5609408617,-0.9947663546,1.7990250587,2.1042752266,0.3246460259,0.8142860532,-0.4318092465,0.0749454051,0.5255390406,0.8453169465,0.5342290401,1.3789731264,-0.1871801764,0.1282057762,-0.8915588856],[-0.0542284548,-0.1044168994,-1.5793139935,-1.0756293535,-0.9769575596,-0.3783388138,-0.1792947501,0.1676430255,0.9718354940,-0.3327423930,-0.9197884202,-0.6990206838,-0.7812698483,-0.7872581482,1.0362886190,1.0649689436,0.3260521293,2.1018323898,0.0040883296,-2.0931577682,0.6810662746,-0.8956177235,-0.6246948242,2.6490068436,1.4860161543,-1.1300300360,-0.0098225111,-0.3665522337,-0.1155728847,-0.2216357440,0.0329407454,-0.3452160060,-1.6651244164,1.1996959448,-1.0849312544,0.0575289764,-0.6823420525,0.4595836699,0.4365171492,-1.0625749826,0.6479692459,0.1152886003,0.6351547837,0.8655322194,-0.1900938898,0.6194599867,1.3249210119,-2.9902887344,-0.9829337001,2.3041112423,0.8291086555,-0.6690052748,-0.2973387539,0.2854040861,0.2512688935,-1.0038155317,0.0745838508,0.6835303307,-0.3019574583,0.0656837150,0.2763928175,1.0574886799,0.2528705597,-2.0649085045,-0.2525582910,0.5633801222,0.7122578025,0.6231449246,1.2130430937,-0.3417893350,-0.6274027824,-1.0826811790,1.3845734596,-1.6314278841,0.9325391650,-0.2828492224,-1.1294640303,0.3213407695,-0.1412851214,-0.5739104152,-1.8523945808,0.1375545859,0.8524460196,0.9795963764,-0.4191142619,-0.1105024815,-0.8044444323,1.7718458176,-2.1773252487,-1.9565769434,-0.0790803954,1.5214020014,-0.9446907043,0.2151204497,-0.7931734920,0.1091351062,0.0690722018,-0.2768059075,-0.1884625107,-0.0551569052,-0.8582618237,0.7444183230,-0.3289690912,-0.4206421971,0.6482777596,-1.8037947416,-0.6215838790,0.2966335118,-0.5860925913,0.2072690427,0.3849021196,-0.0163337681,0.7419171929,-0.7282328010,0.3230105639,-1.0278416872,0.2014814466,-0.1945166439,-0.0878563076,0.5807428360,0.4235515296,0.5991556644,-1.6961510181,-1.7235015631,-0.1653775871,-0.0229291823,0.3718184233,-0.5282102823,-2.0017440319,-0.5723828673,-1.0784522295,-0.1570475549,-0.4171837568,1.1093618870,0.8538563251,-0.6578562260,0.2458041012,1.4572660923,-0.4246588945,0.5097835064,0.6360350847,-0.2560229897,0.0219479911,-3.1653001308,0.7525913119,0.1623850465,-1.1981235743,-1.5419058800,-1.2821328640,-2.3250749111,-0.0390541479,1.9492053986,-1.6592499018,-0.2550086081,-0.8584333658,-0.6264243722,0.0905184820,2.3940489292,0.8812188506,-2.6074683666,0.5467622280,-0.2826860547,-1.7907680273,-1.6223297119,0.0945725664,-0.6790642738,-0.0352940187,-0.6122911572,-0.7683660984,-0.5392516851,-0.3982014656,0.8679015040,-2.1680600643,-0.5140717030,0.8265195489,1.5463507175,-0.5618708134,-0.9446499944,-0.9218199849,-0.5487257838,0.5468548536,-0.1935274601,0.2828486264,-0.7948492169,-0.0389313214,-0.7580267787,1.2519108057,2.5005273819,-0.4626162350,0.7376989126,0.7565030456,2.0083138943,0.8451092243,0.8647089005,0.0845709145,0.6045849919,-0.2940496504,-0.7108674049,-1.6182552576,-1.1536600590,-0.5232933760,0.8932416439,0.0766904950,0.2250192761,-3.3111915588,-1.2070434093,-0.5528053045,0.2814783454,-0.0727163851,0.4501400888,-0.5304975510,0.0220623203,0.5409076810,0.5520184040,1.1103562117,-0.7519988418,-2.4101579189,0.8522349000,0.3122522533,-1.3813184500,-0.9066482186,0.0503111966,-0.5682703853,0.8344324231],[0.0455108099,-0.5007439256,-0.0220513381,0.6138712168,-1.5130780935,-0.8231852651,-1.2862572670,0.0757517591,-0.3834818006,0.9912862182,-0.2152566612,-1.8530063629,-0.8569292426,0.8364073038,0.1884991527,-0.8838205338,-1.4345126152,0.4621810317,1.8554944992,1.7764538527,0.2408635467,-0.3988132477,-1.3176436424,0.3313519657,-1.5404409170,0.8066987395,0.6141310930,1.0647925138,-0.5903156996,0.4407163262,1.0937725306,-0.6846204996,-0.8255679607,0.0301491339,1.1726086140,0.5547693968,-1.9036145210,-2.1507112980,0.9439404011,0.8304248452,0.8022865057,-0.0395314060,-1.7726892233,-0.7131135464,-1.3535375595,-0.2295022607,0.6360743046,0.3305475414,0.1700846702,-0.4246495068,-1.0522756577,-1.6014963388,0.8445893526,0.4519396126,-0.7346076369,-0.9541358948,0.0660870895,-0.2796024382,1.5112785101,-0.2288440019,0.7933291793,0.9457479715,0.0437120758,1.0272008181,2.6187674999,0.2938658595,0.0002436232,-1.8856309652,-0.2801553011,-0.8098967075,0.2391258031,-1.5088934898,0.1154030189,0.1677009463,-1.0893034935,0.1279728115,-1.1742339134,0.0815478116,-0.4688488841,1.9693156481,1.1393784285,0.7566793561,0.0332948640,-1.1088514328,0.6060234308,0.3168972135,0.9489172697,0.8915824294,-0.0744909570,-2.9674692154,0.2204231173,1.3969603777,2.8875684738,1.1382696629,0.0757457167,-0.3666677177,0.0659102425,0.0668415278,-0.5749508739,-0.2340170890,-0.5543112755,0.2262243181,0.3229537606,-0.3703001440,1.2843537331,-1.3588908911,-0.4063398838,0.3984783888,0.4746867716,-0.5155292749,2.0688710213,-0.7366701961,-1.2884358168,0.0306869484,-0.6278145313,1.6479524374,1.2733566761,-1.5114120245,0.5874340534,1.4164183140,2.1109287739,0.6936261654,-0.7801271081,-0.1424248666,1.4228651524,0.7660459280,0.0757838488,-0.3184366226,1.6464478970,-1.3185890913,-0.6462491751,0.5449139476,-1.3927319050,1.1904312372,0.5407568216,-0.0330950134,-0.3919748962,0.7456977367,-0.6043021679,1.7865223885,-0.3086243570,1.0165492296,0.2271374613,-0.7343394756,-0.6063373089,-1.8273447752,-0.8988688588,0.3666749895,0.2657340765,-0.2736480534,-0.7589226961,0.2684849799,-1.1223801374,-1.1474984884,-1.2845572233,-0.3152797520,0.3679023683,-0.2113581449,0.5975563526,-1.9758757353,0.2240671962,-0.5090435147,-1.2870079279,-0.0849780515,-1.1679377556,0.4966672361,-0.6675986052,-0.4594515264,-2.6474492550,-0.3685693741,-1.2190847397,1.2057003975,-1.1603654623,1.1231783628,-0.5407581925,-0.5821552873,-1.8120853901,2.3940880299,1.2630169392,-1.4847002029,0.0002179273,-0.2718341053,-0.6148485541,0.6212561131,-0.8127297163,-0.2228314281,0.6530621052,-0.2918465734,0.8982840776,-0.3148573935,0.0192828048,0.1528849155,-1.0955338478,0.1683052927,0.7518684864,0.0263153650,-0.5481123328,-1.0166777372,-0.0326641724,0.6306336522,-1.0569366217,-1.3100726604,0.8450725079,1.1383305788,0.2161744982,1.4873583317,0.1062991545,0.0131904045,-1.0483144522,0.8597632051,-1.7767972946,1.5863535404,2.7823541164,1.6619212627,0.3272190094,0.6373428702,-0.3165480196,1.0289241076,0.0949189141,-1.5440686941,0.3050356805,0.1199226975,-1.2530963421,1.2903825045],[-0.0241483971,-1.6955019236,0.4644385278,0.2589190602,-1.3097108603,-0.5500065088,-1.0812994242,0.6089993715,1.7156662941,0.4168740809,-0.6155125499,-0.1195571795,0.1958514154,2.2196655273,0.7892631292,0.7401278615,-0.2391066402,0.9518038034,0.5104688406,0.7215762734,1.3206062317,0.2645880580,1.1824287176,0.9417513013,0.7604390979,0.1519564837,-1.5020102262,-0.4238476455,1.3813818693,-0.3240005970,-0.9087860584,-0.4296375513,-1.5056200027,1.0380312204,0.9179576635,-0.2080131173,0.4261639714,-1.2480202913,-0.1067449674,-0.1843754649,0.2243936658,1.2510346174,-0.4090640247,-0.3410273790,-0.8742214441,0.0221707560,0.0311127305,1.9829660654,0.4096661210,0.5662220716,-0.6245689988,-1.7555264235,-0.0548473075,1.2680255175,-1.4447710514,-0.7565324306,0.5842719078,-1.8277680874,0.1456336826,-1.0918613672,-0.9649658799,1.3064036369,1.1600183249,1.1379429102,2.1565191746,-0.8689957857,1.5636215210,0.4368431270,-0.6594150066,0.2378106266,-0.0386009179,-0.2020491958,-0.7171027660,0.7523133159,-3.0087399483,-1.2096756697,-1.4830690622,1.1132252216,-0.0736747012,0.2278408259,0.8607809544,0.0230039097,-0.0100709805,0.5663136840,-0.4573984742,-0.3147220016,-0.5916610360,0.3633759320,-1.7309250832,0.7415539622,0.9788953662,0.4454882145,0.9996737242,-0.7172723413,0.4500461519,0.2097088099,-0.1300976425,0.3683418930,1.6021925211,1.3492051363,0.5966399908,-0.2869676948,-2.2014067173,-0.6466356516,-0.4141579270,0.0322575942,0.2764803469,0.0523899347,0.9976919293,-0.4418748617,1.1962385178,0.8795982003,1.3394536972,-0.5127735734,-0.5311194062,0.8626922965,-0.7423558831,0.9309554100,-1.4290853739,0.0858087614,-2.6495206356,1.6978728771,-0.4349122941,-0.3831622601,-0.0684857070,0.7442887425,0.5322425961,2.0612838268,0.1241032407,-2.3649044037,0.3680788875,0.1251933873,1.7623183727,0.3210356832,-0.2840740085,0.7346179485,0.1568394750,-2.1416144371,0.0518188179,1.2533953190,0.8536785245,-0.2809889913,-0.4789661765,-0.7346751690,0.5946739316,-0.7996873260,-1.3966721296,-1.7271291018,-0.7277098298,-0.0455504879,1.6599112749,-0.2437718362,-1.4668591022,0.0005579666,-0.6929865479,0.9924173951,0.1146905273,-0.0723900497,-0.3171693087,0.4661208093,-1.2721763849,-0.9968279600,1.4310706854,-0.1644446850,0.2184965909,-2.4001052380,0.5335850120,0.8570533395,-1.2253437042,2.6228101254,-0.8978168368,-0.4737388790,1.2820608616,-0.2572124600,-2.3622217178,1.6324220896,0.2936998904,0.4136489630,0.4954855144,-0.6417990327,3.0474805832,-1.4472365379,-3.3824377060,-0.5294840932,2.2834894657,-0.2915201187,-0.5834859610,0.6736102700,-1.0685982704,-1.0622173548,-3.0860776901,-0.1529570520,-0.9341332316,0.6026499867,-1.6047893763,0.5600776672,0.2471535951,0.3888663054,-0.5745077729,1.1368119717,-0.1875921935,-0.2457357347,1.0788971186,1.3245619535,0.4084223807,-0.9082864523,1.3701889515,0.5639448762,0.6552087665,0.2393982857,0.5696803331,-1.8782240152,0.1603340507,-1.0327521563,-0.1901682466,0.5955625176,0.8033736348,-0.3817498088,0.4623665214,0.5622440577,-1.9576177597,-0.2124888599,0.0532335043,1.7872124910],[1.0935274363,-0.6834817529,-0.0209047794,-0.2557629645,-0.0322177224,-0.4546183646,-0.6292889714,-1.8742160797,-0.8476565480,-0.0208909176,0.9334092140,1.3976117373,0.7560239434,-0.6076855063,-0.4940192103,-0.2218679041,-0.0247830916,0.4010295868,-0.2099526823,1.6346033812,1.0749132633,-0.7803043723,-0.6348884106,-0.6152335405,0.9498234391,0.2001388222,-0.5263258219,-0.8058626056,1.0469522476,0.3226033449,0.1166816801,0.9418267012,-1.3817039728,-0.2754204571,0.6587382555,-1.1658596992,-0.5008330345,0.6715875268,1.0270637274,0.6949701309,1.0310188532,0.3335841298,-1.1015419960,-1.6726310253,0.0740614757,0.3277696967,-0.3727508783,0.1862053722,0.2605803013,1.8626759052,0.0707811713,-2.1108138561,-0.6218534112,0.4231096804,0.2173334807,-0.7314818501,-0.7883370519,-0.1343939155,0.4867017269,-1.6478499174,0.8052513599,1.0267293453,0.0742363185,0.0273618270,1.4169248343,1.9043926001,-0.3838944435,0.5245365500,0.2650912702,-0.3235317171,-0.8705883622,-1.5474305153,0.6036111116,0.0714829117,-1.1413797140,0.1435114741,-0.6295248866,-0.0765427724,-0.1058588624,-1.4466910362,-0.6485618353,0.0132218618,1.4305149317,0.3642526269,1.3517880440,1.3447993994,0.7358057499,0.7414743900,-1.2427915335,-0.3946131766,0.1648515463,1.1895046234,-0.2594691217,0.8554831743,-1.0946022272,1.1648385525,-0.2886697054,-0.7935397625,0.0376145579,1.3619145155,-1.3953189850,-0.1312556565,-0.4305959046,-0.7347027659,1.2504189014,-1.1138132811,1.8914943933,0.6668202877,0.5314742923,-1.1446431875,0.8470588326,-0.3559998870,1.8328109980,-1.0825778246,1.5532661676,1.4346286058,1.6508401632,0.1166369244,-1.7578734159,-0.2210106105,-0.3382823765,-0.7606144547,0.7037070394,-1.4212046862,0.7512388825,0.1273766011,-0.0193244293,1.8327016830,0.1493382752,-0.0756782517,-0.0355336964,-1.6446069479,-0.8279816508,-0.0862731785,-0.6101303697,0.6401699185,-0.1602170914,-0.9755163193,0.3217153549,0.1617767066,1.2197539806,0.1922034472,-0.4725489616,-1.2373657227,-1.7496455908,0.5991289616,-0.5498912930,0.0673314482,0.8375106454,0.8581475019,0.9310656190,-1.3271347284,-0.1280384958,1.7297261953,-0.2735048532,1.1787731647,2.2917857170,0.8384599686,0.9903156161,-0.2916140258,0.3382817805,1.0331203938,0.3903645873,-0.6004440784,-0.5967748165,0.9446532130,1.4222265482,0.0261503533,-1.4113605022,-1.8482435942,-1.0479794741,0.6841892600,-2.3524160385,-1.1195981503,-1.0631546974,0.3228687644,1.1693851948,2.5934445858,-0.4578936994,0.1776099205,0.0838143975,0.8039101362,0.5364631414,0.1926894188,0.4836985767,1.1132575274,0.1154624745,0.1372138709,-0.2826304138,-1.1644810438,-0.0809413269,-0.4451850951,-1.1100091934,-0.5838759542,0.3142054081,-0.6515339017,-1.3535983562,0.3675374985,-2.6818180084,-0.4444566667,-1.1640266180,0.0140349520,-0.3826368749,-0.7109439969,0.6577385664,0.0257320683,-0.4784716666,-0.6491413116,-1.1920108795,0.7272107005,-0.4145368934,1.3471291065,0.3028367460,1.9280700684,0.8856548667,-2.2657513618,-0.1738021672,1.1121703386,0.8565863967,0.0598084405,-1.1367391348,-0.3166633844,-0.4727935791,0.1086008698],[0.0661494732,-0.5598045588,-0.2818463743,0.7618486881,1.0420223475,-0.0124819009,-0.2460182905,-0.2402713597,0.2803384960,0.6347672939,-0.7395008802,0.3717006445,1.8899036646,0.7324008942,-0.2531517148,2.1485793591,1.1714525223,0.7662360072,-0.7396721244,-1.2587963343,1.2214463949,-1.1060516834,0.2466125786,-0.0663431361,0.5754920840,0.0819447786,1.2410321236,-0.0663276836,-0.8556157351,0.1471997797,-0.1371292323,1.4430338144,0.1633171737,-1.1141107082,-1.4421511889,0.2584901750,0.1109801009,0.5133917928,-1.6274529696,-0.3198641539,1.5091960430,-0.5411385894,0.4407715201,1.6539150476,-0.1996767372,2.6565704346,0.0422839411,-1.1861289740,0.6739498377,-0.0582477152,0.0878551304,-0.0720350966,-0.9809566140,-1.0485097170,0.1298928559,1.5011910200,0.3872312605,1.3922791481,-0.8527609706,-0.1992137879,-0.3257340789,-0.2213805318,-1.5234873295,-0.1974401027,-0.8294673562,1.2558363676,0.7330928445,0.6681995988,0.8794973493,0.6108455658,0.5544348955,-0.6794480681,-0.7796459198,-0.7026695013,-2.1434984207,0.0411658436,0.3642963171,-1.1052713394,0.2658634484,-0.4520622790,1.2095198631,1.5577833652,0.3189541698,-1.2588919401,0.8697476387,0.2926436365,1.0497710705,-1.7970030308,-1.8265334368,-0.0994349569,0.3744624853,0.2839959860,0.5508910418,0.5231243372,0.7695473433,0.2353003919,1.0889656544,-0.0385504179,-1.6064237356,0.2855109274,-0.4909099638,1.2451142073,0.3549577296,-0.1303832978,-2.4352939129,0.5502526760,1.4097541571,-0.8143534660,-0.4583834112,0.5625880361,-0.0530144200,-0.0092057148,0.0317580067,-1.3821700811,-1.1400830746,0.2341952622,2.0570058823,0.6172062755,-1.9741103649,0.9930515289,1.0489962101,1.2614012957,-2.0655992031,-0.6154462099,0.3555959165,-0.2700898647,-0.9979034662,-1.8507244587,0.9184408188,-0.3622143865,-0.1437021047,0.9383545518,0.0250540581,0.5107842088,-1.4865230322,0.6810082197,-1.9815660715,1.1557435989,0.1040510312,0.8280178308,0.8028651476,-1.1024335623,-0.7035324574,-0.0274420045,-0.6655241251,-2.0715312958,0.2726945877,-0.6792376041,-0.1231326684,1.4458146095,0.2314967811,-0.6573718786,0.6712464690,-0.7653050423,1.4972485304,-0.9735265374,1.5854916573,0.2542545795,-0.1565025300,-2.3489050865,0.0153308529,0.7762720585,-1.4138087034,-1.5466600657,-0.2691728473,1.6029889584,0.7616153359,0.9480590224,0.9128962755,-0.7814031243,0.6315351129,-1.1406873465,-0.0793438703,0.7408534884,0.3598624766,-1.6224952936,0.7605880499,-0.4141521752,1.9717255831,-2.0489830971,0.3203971088,-0.5467629433,-2.4523425102,-0.9296472073,-0.2040585130,-0.5218498707,1.4715273380,0.9998838305,-0.4258372784,-0.1646424234,-1.0052925348,1.8012080193,0.7845478654,0.8161594868,-1.8706011772,0.0538863353,-0.5203756690,0.3057786524,-0.3377862573,0.2710774839,0.1396749020,-0.0968563706,-1.6479283571,-1.2091325521,-2.3561117649,-0.0236232337,-0.4046415389,0.0204066318,-0.2489682585,1.7718049288,0.7430356741,0.3051402569,-0.3081843555,0.6530711651,0.6653307080,0.9015319347,1.2185962200,1.2392010689,1.0117390156,0.1708698869,-0.0968572199,-1.0288013220,-1.4395042658,0.4888134897],[-0.1481398195,0.2349804938,1.0387753248,-0.8918272257,1.1819578409,1.1469008923,-0.8863270879,-0.0768607333,1.6103775501,0.2564200461,-0.6540455222,-1.4819751978,0.3392748833,2.3457913399,-1.9290598631,-0.5931926966,3.8038439751,0.0150417322,0.4764592648,-0.7765587568,-0.1422771513,-0.0276196878,-1.0998263359,1.0197985172,0.7770087719,1.2607687712,0.3303723335,1.3598996401,-0.1502128541,-0.9660602212,-0.2033868581,1.8268421888,0.7102911472,-1.2924709320,1.4800399542,1.2848676443,-0.6317282915,-2.0580847263,0.7169486284,0.8882303238,0.9122008085,-0.0191191155,-0.9266655445,-0.7308059335,1.1142582893,-0.4701244533,-1.5005794764,-0.4836395681,-0.5511965156,0.8093124032,-1.2713971138,0.4431983829,0.5598557591,0.1382537782,0.5284568667,0.0855464339,1.0339580774,-0.9627118111,-0.0867765024,-0.5620479584,-0.0449695885,-0.5436484218,-2.0981886387,-0.2393644899,-1.0797698498,-1.1307771206,0.2753635347,0.7339200974,-1.4783803225,0.0846191719,0.4496826828,0.4637614489,-0.5138217807,0.8895533085,0.4025216401,0.4540904760,-0.2553262711,0.0476168953,0.3607804775,-0.9458797574,0.5130068660,1.0887874365,0.0324329808,1.7249981165,-0.5506155491,-0.2265087664,0.1730436236,-0.8672905564,-0.9517143965,-0.6819998622,1.8925633430,-0.5262590647,0.2710691392,-0.0368213281,0.2499300092,0.1145922467,0.4249081314,0.5221466422,-0.6290580034,0.5858543515,-0.7500481606,0.4885399044,-1.0888670683,-0.1751397997,-1.0800218582,-0.5044698715,-0.6824820638,0.3367467821,-0.0662877113,0.0004860878,0.3974931836,0.9878262281,-0.2203817666,-2.3339583874,-1.3713185787,0.7010454535,-0.1792138517,1.6398990154,-1.3528527021,-0.0485312343,1.6457034349,-1.5228041410,-0.9622097611,-0.7212290764,0.1708365530,-0.1943217069,-0.1791012436,-1.5973223448,1.0700832605,-0.0263415724,-0.0783090517,0.5096049309,0.2359365672,-2.0904521942,1.3343380690,0.0267197248,1.0169812441,-1.6574496031,-0.7635102868,0.0978857651,0.8636631966,0.2661770284,-0.9758553505,0.6706265211,-1.1487759352,0.7276757360,0.0401194058,-0.8232668638,0.1121659577,1.8414586782,-0.5288928747,-0.5942555070,-0.0549913421,-0.3535532951,0.4613077641,-0.8000953794,-1.1293758154,0.7078111172,0.2381000668,-0.2260020673,-1.0287990570,0.6059779525,1.0500333309,0.4062401950,0.5371391773,0.0047277003,1.6705203056,0.4563495219,0.2244141847,-1.6428673267,-1.8438435793,0.3145678639,-0.4614353478,0.4700029194,-1.7696510553,0.0000749491,-0.0976910740,0.5977476239,0.2733864784,0.3712363243,0.7553363442,0.3672829568,1.1928751469,1.4054775238,-2.3258380890,-2.9297554493,0.2743716836,-0.7235479355,1.5969778299,-1.8682086468,1.4272466898,-2.3928434849,-0.7951687574,-0.3718027174,-0.2111870050,-0.4080311656,0.4804771543,1.6419860125,0.9514871240,0.6680909395,-0.5509938002,-0.0222789478,-0.1015945151,0.1699242294,-1.2516422272,-1.5921986103,0.0323502608,1.3214668036,-0.0185927413,-0.4885272980,1.0027142763,0.0604401119,-0.5876231194,-0.2673060298,1.1417536736,-0.5216576457,-0.9080795050,0.2505319715,1.5503072739,-0.5802096725,-0.5431979895,-0.8207826614,-1.8523241282,0.7910633087],[-1.5131099224,-0.5346738100,0.8551876545,-0.8079803586,0.3514728546,-1.0119205713,0.1222014651,-2.3043191433,-0.5708674192,0.5543924570,-0.8940060139,0.2141406089,-0.4124077559,-0.0373211876,-0.4836105108,0.9895735383,1.3710926771,0.3094007373,1.0810010433,0.7413520813,0.0272918269,-2.6059725285,0.7118025422,-0.3565561175,-1.5837817192,-1.8069742918,0.0314307399,0.7519264817,-1.6187287569,-0.0656537339,-1.2725286484,-0.0628013238,-1.2660244703,-0.2694690824,-0.9322804213,-1.7178848982,0.3884723186,1.1313183308,-0.8840425014,-1.5170679092,0.8701610565,0.6604436040,-0.3095566034,1.1559892893,-0.0074164211,0.9563302994,0.5253620744,-1.1275501251,2.1611838341,0.2815635204,1.0019308329,0.3967360556,-1.3517037630,-1.6757366657,-0.4909931421,-0.2150374204,-0.4196070433,0.0863581672,-1.1695873737,-0.2097769529,2.7132956982,0.6640202999,-0.3107252419,0.1927692294,-1.1766229868,0.5481172204,-0.0203474443,1.5921030045,0.8253238797,0.2750462592,-0.3395743668,-0.1995105147,0.2876706421,-1.2705643177,-0.4173485339,-1.0307558775,-1.9259363413,0.3010179400,0.5864936709,1.1382341385,2.5246179104,-0.5178236961,0.8482388854,0.9648278952,0.5560377836,-0.2276707441,1.0964823961,-0.2981500030,-1.1091046333,0.2975334525,0.6693788767,1.2441065311,1.2527660131,0.1742370129,-0.2687437832,0.8906718493,1.1863725185,-1.3319261074,0.1384135783,0.9704707861,0.6115167141,0.1373640597,-0.4339033961,-1.5552423000,0.6139218211,2.3210573196,0.5673650503,-1.5066421032,-0.9496950507,-1.8230872154,0.2817157209,1.3412139416,-0.3125414848,-0.7926390171,-1.8839703798,-1.5029090643,-0.4812536836,-0.8076776266,0.1639763117,-0.7107151747,0.9037547708,1.1245650053,0.4608503878,1.4936001301,0.8328832984,-0.1224921197,0.8565199375,-0.8202630877,-0.1175216883,2.6427290440,0.7497962117,-0.1043440327,-0.5497339368,-0.1805608422,0.2127023935,-3.0390629768,-1.0456823111,0.2500087917,-1.0225489140,-0.7522569895,0.1320671141,0.0960535109,0.2754767239,0.7104935646,0.4646618962,0.8646464944,2.8653013706,-2.1563317776,-0.9281902909,0.3972793519,-0.9229031801,-0.2998534441,-0.3409955204,1.2158575058,-1.6193940639,-0.4401818812,0.1239653826,-0.1996781826,-0.8049672246,-0.3598818183,0.2379658520,0.7701300979,-1.6182599068,-0.8581345081,-0.2470516115,-1.7491062880,1.4807767868,-0.5614758134,1.2209767103,0.2904929221,-0.0265204366,0.5695201755,0.1617171317,0.7726086974,1.0486824512,-0.3704072833,1.1730215549,-0.5312774777,1.3397569656,-1.0443192720,-1.3226132393,-0.2041894644,-1.5650993586,-0.0995839760,0.2208478004,-0.4249693155,0.9977360368,0.5591554642,-0.0693533272,-0.9527857304,0.7313643694,-0.1154942065,0.6147226691,-0.5344278216,-0.4461407363,0.6166067719,1.1970794201,0.3441028297,0.9995790720,-0.1399023682,-0.2110286802,-2.0446467400,2.6426510811,1.7442144156,-1.2394208908,-1.1079133749,0.2935610712,0.0725700185,0.3695668578,0.5065841675,-1.0672954321,0.5645725131,0.7228435874,-0.8897636533,0.4221113324,-0.4366683066,-0.4785924852,2.0385699272,2.5463235378,-0.9433593154,-0.9427928329,-0.7519164681,-0.6663823724,-0.3502293527],[-1.4659262896,0.2184054852,0.3883730173,-0.5445824265,-0.4340218902,3.7029385567,-2.9254779816,0.0942560732,0.7164333463,-0.4786452055,-0.1175122783,-0.6979387403,0.2932125032,-2.5538980961,-0.6223775148,0.2257152498,-0.3763697743,0.8137111068,-0.5992847085,1.3243182898,-1.2376658916,-0.0529220700,0.3752489388,-0.2927281559,-0.8624119759,-1.7237237692,1.1811594963,-0.3744150698,-1.1541925669,-1.5252789259,1.3609256744,0.3352670372,-0.7940573692,-0.5847789645,0.0678094849,-0.2327625602,-2.0279872417,0.0251677223,-1.4056166410,-1.6464124918,-0.2422740161,0.8370131850,1.5657683611,-0.4537927806,0.7318181992,-1.0169379711,0.7392541170,-0.3775894642,1.7589201927,-0.9612664580,-1.5072410107,-0.9849436879,0.2215117812,-1.2333002090,-0.1126676127,0.1513326466,1.5155856609,-0.8208940029,1.8061890602,0.1750951260,0.0165317189,-0.0252383649,1.7736938000,0.5884373784,-0.9119014144,-0.6989759803,-1.0164759159,0.0299540386,-0.0303175766,0.5049239397,-0.6621921062,-0.2147170007,0.2130482644,-1.0169265270,-1.0796117783,0.9682517052,-0.2484214753,-0.0983709693,0.4231301248,0.9714114070,-0.3323786557,1.5941475630,0.0305444896,0.4641273916,1.1204073429,-1.7273485661,-1.9588909149,-0.0995916799,0.4214471579,0.2391090691,-0.8550423980,0.5174822807,0.6435247660,0.7362871766,0.2990854383,1.6299250126,0.0470980965,-1.4124903679,2.0001816750,2.4043455124,0.4756488204,0.3924701512,0.3098393083,-0.0730008855,0.0644879714,-0.5879397988,3.1723389626,-1.1309983730,0.4812060297,-1.0504877567,1.6183617115,-0.6508139372,-0.3729372919,0.8800085783,-0.1217752844,-0.7541463971,-2.0912880898,0.0173039250,-0.3026672900,1.9386875629,-0.5450688601,-0.8840935230,-1.0172849894,0.2282519937,-0.7308015823,0.0178454760,1.1036424637,1.0490378141,0.0145709217,-0.6570018530,0.6453847289,0.8068607450,-0.9951968193,1.6353204250,0.2871158421,-0.1186865047,-0.7178679705,-0.2702004313,0.7870482802,1.2354224920,-0.5233058333,-0.9856014848,-2.3766124249,-0.3460740149,0.8292262554,-0.1120812893,-0.5926383138,1.5636755228,0.5033448339,0.0718692541,0.0410754010,0.3172487020,1.0507453680,0.0159926750,0.4478392601,-1.4229664803,0.7920886874,0.1866728514,-0.8899513483,1.5150766373,-0.7372040749,2.3344931602,1.7175177336,-0.2363054156,1.2435344458,0.1209539473,2.1040270329,-1.0785930157,1.0357654095,-1.2035720348,0.6071705222,-1.2331465483,0.0831739157,0.5158026218,1.2514107227,0.5857951045,0.5511865616,-1.0304999352,-0.7450944185,0.8752056956,-0.8220462799,-0.5751981139,-0.3852460086,0.4066129625,-1.4922865629,0.5390688777,1.0325790644,-1.0658810139,0.8860808015,-1.0557252169,-0.3435255885,-0.3831379414,-1.5366101265,0.7045727968,-0.4226921499,0.2454482317,-0.9025840163,0.5418877006,-0.0653132945,-0.6542792320,-0.2471905053,0.6364162564,1.2548311949,0.8018607497,-0.0924706161,1.7474905252,-0.0242728647,0.7594467402,0.2591527998,1.2631505728,-1.0255336761,0.5002287030,-1.4737606049,-1.7188454866,1.5862935781,-0.7704221606,0.2158858180,-0.6859569550,-1.4657487869,0.1420394927,-0.7383154035,-0.6743084788,-2.7707183361,-0.1151084825],[-0.6279640198,-1.6584491730,-1.2487521172,-0.1758100241,-0.2976056635,0.6281543970,-3.0891396999,0.6630514264,-0.9883195758,1.5522327423,0.6034018993,0.2812715769,-0.2210479379,0.0146459658,-0.3543334901,-0.4833302498,0.9713264704,-0.2638811171,-0.6870847940,0.2799275219,0.2579723001,0.0368682183,0.7310587168,0.2262041271,-0.2016255260,1.0671253204,0.0625227988,0.6604328156,0.6529068351,-1.1744426489,1.3701006174,-1.0094944239,0.5174626112,-0.8501479626,-0.3785215020,0.2888900936,0.8252806067,1.7142086029,0.7759332657,-1.7127082348,0.5845321417,-0.3192206919,-0.1254154444,-0.1918734610,1.2321667671,2.6393880844,0.2824049890,-0.6123250127,-0.3582670987,0.9553580880,-0.5618331432,-0.2247440219,-0.8032472134,-0.4080768824,-0.9748763442,-0.2706478536,0.5201159716,0.0428278558,-0.6018859148,0.7460201979,0.0247440431,-0.0502817854,-0.7423041463,-0.8392975330,1.2616049051,-0.9946495891,1.9058818817,-0.1542883366,-0.1210363731,-1.2415443659,-1.2345449924,0.9470774531,0.3019158542,0.8561933041,-2.6018064022,-0.7974361181,0.2222371846,0.4300654829,2.8669002056,-0.5035877824,-1.6985487938,2.0215651989,0.1090312600,-1.6191835403,-0.0101143373,1.0134818554,0.4818102717,-1.9236172438,-0.6230416894,-0.1731863171,1.4452866316,0.1037637815,1.5183800459,-0.4374295473,-2.5311095715,0.2603306174,-0.4341614842,-0.0227584727,0.0439706743,0.5057605505,-1.7936534882,-0.5852804184,0.1672791690,0.8425058126,-0.6111262441,-0.9124225974,-0.2337473482,-0.0449558981,-1.8964269161,-2.0928378105,0.8949176669,0.9105094671,0.9112442136,-0.4834975004,3.5092229843,-0.5729820132,2.0387670994,1.0817768574,0.3173472285,0.1624213457,0.8563386798,0.4270950556,-0.3292297125,1.1796474457,-0.0568520613,-0.6135614514,1.0171507597,-1.0845253468,-0.0473044775,1.4754977226,-0.0622475371,0.7117826343,-1.0041016340,-0.9487560391,-0.0518494286,1.4331901073,0.2149605602,-1.1410475969,-0.6128460169,0.5419374704,0.3689844310,0.8537623286,-0.7307296395,0.4909996092,1.1410055161,-0.7519097328,0.8367847800,-0.4134968519,-0.3831160069,-0.1194213182,0.2132193446,0.8922790289,1.1994528770,1.8140201569,-0.5317217112,0.7226940393,0.4397925436,-0.4289795458,1.2970182896,0.5386045575,-0.3921145797,0.3799457252,-0.2168604136,-1.2815352678,-0.5558221936,-0.6706677079,1.1508843899,0.8142685294,0.0629897863,1.0782709122,-0.3097993731,1.1608967781,-0.0450324751,0.9269484878,0.1099951491,1.0994768143,0.0873215795,-0.6024252772,0.0804315135,1.1702136993,1.2390604019,0.0422957130,-0.2660755217,0.0252159797,0.3653047979,0.6382434368,-0.1962671429,0.5976749063,0.2093169689,1.3053292036,-0.1818927377,-0.8006437421,0.7526998520,-1.2449376583,1.6166766882,1.4126682281,-0.9880897999,0.1460569501,-0.3867262900,0.8620718122,-0.9725794196,-0.2764593959,1.1102123260,-0.8179458380,-0.4496610463,0.1712568700,-0.5782694221,0.7184853554,-0.7871074677,0.1874635667,-0.9851943851,0.9395116568,-1.3483319283,1.3027681112,0.5989563465,0.3927023411,-1.0175455809,0.7853148580,-0.4428822696,1.2068151236,1.2453286648,0.2258458585,1.1729338169,-0.3745219409],[-0.1013393402,0.7370291352,0.2293027043,-0.5496422052,0.4026574790,1.7350859642,0.8466253877,1.6243348122,0.2167896926,0.0440245979,-0.1143867150,-2.1155114174,-1.3466324806,0.3417681754,1.0367302895,-0.0448520556,0.1536931545,-0.1178123504,0.0942037478,-0.2840364575,-0.8906734586,1.0415467024,-0.3474560380,1.7721912861,-0.6977778077,-0.6533368826,0.2706574500,0.4579048455,0.5070598722,0.2563111782,1.0382679701,-2.1179025173,0.1377092451,0.0209780280,0.9947603941,-0.0382568911,0.2360651493,-1.6812630892,0.4709948599,1.8487871885,1.1363065243,0.6794203520,1.6432129145,0.3331112862,0.4847803116,-0.2373197377,0.8562826514,-0.8727971315,-2.7746782303,-2.3559737206,-0.2040310651,-1.7742786407,1.9584763050,0.7933133841,0.6449356079,1.4588258266,-0.0484815538,-1.0195758343,0.2607858479,0.3664327860,-1.2107329369,-1.9957568645,-0.4150539041,1.5697511435,1.9711836576,1.5477201939,0.0248154551,-0.7006254792,0.6331615448,0.5916293263,0.0568252914,0.1661783904,0.2813738585,1.0866055489,-1.0118426085,-0.0859973654,-0.7634933591,2.1033339500,0.3371011317,-1.1005963087,-1.3655861616,-0.8443711996,1.1577920914,-0.6241202950,-1.8529772758,-0.6705492735,-0.3144679070,-0.1968949288,-0.2270013392,-0.1578439176,-2.0221607685,-0.1185465753,0.4237858653,-0.0993085131,1.3173325062,0.1934987605,-2.9459500313,-0.1461422294,0.2651478350,-0.8410601020,-1.7123898268,0.0538811646,0.0466864482,0.0800267011,-1.7519488335,0.2975313067,1.4007811546,-0.9453305006,0.5971914530,-1.0898880959,-0.7380421162,-0.4164339900,-1.2008498907,-0.6115810871,-0.1443822831,1.1616984606,1.1686754227,1.2920237780,1.4396482706,1.3408201933,0.4317942858,-1.3623890877,-0.7680445910,0.5570045114,0.4734558761,1.1375391483,0.5354587436,0.4446857274,0.5467852950,-0.1238997132,-0.6851717830,2.1651611328,-0.4368666112,0.0831865296,0.0198221132,0.4435079098,-1.6825557947,-0.1355063766,-1.6379758120,-0.2781943679,-0.2904914320,0.8691003323,0.2901981473,1.5223109722,-1.3436995745,0.4895364940,-0.6511492133,-0.3522801995,-0.8010972738,-0.3451113701,-0.1639397591,0.6182901859,-0.8302804232,-0.0971979052,0.1578138769,1.5443570614,1.3465884924,-1.1086828709,1.7096700668,-0.1851585954,-0.0696520433,0.8750807643,-2.0553681850,-0.8725947142,0.6036568880,1.8323498964,-1.1572742462,-0.2411930561,0.6965112090,0.6085639000,-0.6458541751,0.1013178676,-1.0230764151,-0.8841358423,0.6133187413,1.1686683893,-0.1828366816,0.1179393306,0.6337798834,0.4901778698,-0.9213451147,1.1282690763,-1.3957413435,-0.9345256090,1.4516988993,-0.3256015778,-1.7746162415,1.1917959452,-1.2848377228,-2.0234019756,0.6253730059,0.5754631162,1.0525467396,-1.4490629435,-0.5101265907,-0.5352268815,-0.1461670250,0.4182570577,-0.8196110725,-0.2917360961,0.1357769370,-1.4317878485,0.5729535818,1.0932507515,0.9999333620,0.9448190331,-1.1056905985,1.2392117977,0.0591306388,1.4897656441,1.6061283350,-0.6775377393,-0.4090647399,0.3424934745,-0.9212122560,-1.3457553387,-1.0322591066,0.2986054420,-0.8338737488,-0.0210380573,1.5235228539,0.1132932231,-1.1214870214,0.2483399361],[0.6102077365,0.1721328497,-1.2462910414,-1.4098676443,-0.0068863430,0.6785459518,1.0933088064,0.0307702217,-0.8330343366,0.0852738246,-1.6568013430,0.0855158269,-1.0569945574,0.3314259350,-0.7124000192,-1.2863686085,-0.0895156786,-0.0622664876,0.4961340427,0.1014336422,-1.3754044771,1.3578120470,-0.0705222040,-2.0296418667,-1.0720357895,0.5115970969,1.0631728172,-1.0660487413,-1.4533889294,-0.2905147672,0.2778032422,1.8945484161,0.6868228912,0.5472155809,-1.9280380011,-0.4031965435,-0.5989333391,-0.2061021477,0.8257248998,0.9506226182,-1.1330982447,0.4963971674,0.5451398492,1.2906789780,-1.5282840729,-0.8242005706,-0.0166608077,0.0335856453,2.2408883572,0.4733282924,-0.7696737051,-0.8335739970,-1.4651881456,0.9389994740,1.8702383041,0.0202532560,-0.4591764808,0.2830611765,2.2367684841,-1.4029413462,-0.1545487493,0.2303007990,-0.1837261021,-0.8145059943,-0.0218058843,0.0932727531,1.0467610359,0.7866660357,0.4435950518,-0.0753492266,0.3163510263,2.2630624771,-1.5913091898,-1.3097940683,-0.8521066308,2.0647218227,0.2911355495,-0.9971901178,-2.6643984318,0.1316338181,-0.7985268235,0.1305931062,-0.2064609826,1.2719758749,0.5087019205,0.2554793358,0.6482380033,0.7646680474,-0.6734172106,0.2700004578,0.0559221543,-1.0332484245,-0.4205034673,-0.9777092338,1.4239164591,-1.2049342394,1.1088242531,1.6507546902,0.2748976350,-1.1399236917,-0.5846256018,-0.4150563776,0.1476821899,-0.7782986164,0.9938122034,0.3498940170,-1.1976610422,0.4588616490,-0.3367350996,-0.4915359616,-0.2937815785,0.7800746560,-1.1308561563,0.7726313472,0.3986551762,0.5029765964,1.6762688160,0.4709570110,2.0376982689,-0.7127743363,-0.1392951310,-0.9906319976,-0.2955588996,0.2931172252,1.2992885113,1.7970376015,0.4209524691,0.9865259528,-0.9265616536,0.5395507813,-0.2611883581,-0.0639136136,-0.0800991580,0.5079711080,0.2602309883,-0.4143621325,-0.9424008131,0.6740956306,-0.3937194943,-1.0629720688,0.5936015248,-0.6519501209,1.1757981777,-0.7144102454,-0.1102670729,-0.3803231716,-1.3536223173,-0.8384763002,-1.8306593895,-1.6056002378,0.2277152538,-0.5050942302,-0.4376785755,-1.3809183836,-0.9032619596,0.9237828851,0.5372219682,0.2870643735,-1.5153040886,-1.6259210110,-0.0451527424,0.2230039835,-0.7418859601,-0.4045087099,-0.1136229634,1.6879923344,-0.9585251808,0.0452248342,-1.4134316444,1.5886644125,-2.2719659805,-0.8550316095,-2.3864667416,-0.4699884653,-1.4324147701,0.2486818582,-1.2270326614,-0.6884349585,-1.8654394150,-1.2348514795,0.4219813049,-0.3581189215,-0.3238225877,0.7833189368,-0.1920600682,0.7204907537,-0.1709788293,2.3878262043,-0.2370357811,0.9252066016,0.4822584987,0.6451716423,-0.6012758017,1.0694143772,-0.1956869662,-0.8577533960,0.4864356816,-0.2107046247,0.6681656241,-0.8437408209,1.2435525656,0.6121364236,-0.5362357497,-2.3874721527,-0.7291497588,0.6766259670,0.3310092092,2.5184285641,-0.9417285919,0.3908450902,-0.9795922041,0.4405451715,1.6861338615,-0.8826841712,-0.3646148145,-2.1359474659,-0.8896039128,-0.8913089633,1.8602352142,-0.0924306363,-2.5201990604,-0.4996497631,-0.7356934547,3.0227034092],[-0.2779353559,-0.9125910401,-0.5229797959,-0.3131267428,-1.1275002956,1.0743371248,-0.5666452646,0.3104890287,0.6701961160,-1.6568621397,-1.3614599705,-0.3111041188,0.1598366499,-0.3078313470,0.1877021044,1.6093233824,1.2444223166,-0.2770603597,0.2751346529,-0.3467116654,-0.6917365193,0.5257941484,0.7689983845,0.3357297182,0.1524437368,1.6918798685,0.3152635694,-1.0607805252,0.6609638333,0.3341110945,-1.0415834188,0.8687285781,0.8544582725,0.1155287027,0.3381574154,1.2941756248,-0.2241853923,0.5553392172,0.8699635267,1.2132816315,0.1029268280,-1.5269670486,0.1696930826,-0.3773308396,-0.1566898823,-0.8203327060,-0.5906712413,-0.5726389885,1.7057299614,1.0661014318,-0.8808380961,1.9740517139,-1.5351676941,0.0559960082,-0.1033231318,0.2611945570,1.6715939045,0.6242154241,-0.9938521385,-0.0939270034,-0.6858149171,0.3675427139,-0.1992009878,0.6083980203,-1.5106667280,0.5414758325,-1.5780515671,1.3849695921,-1.3922752142,0.1662127376,1.1939231157,0.1069043130,0.0348040052,-1.3020936251,-0.7385377288,-0.9714508057,-0.1246909872,-1.2599545717,-0.5691602230,0.0913027450,-0.3601581752,-1.2885582447,-0.6923422813,0.0218632966,1.1481493711,0.3199811280,0.6208224297,0.1437385976,-0.0351644456,-0.2080980837,-0.7999038696,-0.0143128783,1.8700890541,-0.8725613356,1.9158043861,-0.5309497118,0.5074033737,-0.2050864398,1.8342417479,1.3920772076,1.2333165407,-0.0904451087,-1.6107214689,-1.2150744200,1.6213306189,1.3490490913,-1.2004847527,0.4088242948,1.6381580830,-0.0337113850,-2.1315181255,-0.6664378047,-0.3451897502,-0.5859259963,-0.7312455773,-0.8802397847,1.0734250546,-0.2296246886,-1.1476899385,0.0753600448,-0.1252810806,0.5062884688,-0.0693661049,0.0646183267,1.8327145576,0.6647765040,-0.3912880421,2.0176594257,-0.6505314112,1.8023506403,-1.4049191475,-1.0420913696,1.1070687771,-0.5325237513,-0.3083402216,-1.1807949543,0.1285699606,0.1529561281,-0.1071198657,0.6465570331,0.2372447997,0.6004552245,-0.3670341671,0.1296212673,-0.7611243129,-0.0786540359,0.7158681154,-0.4898069501,-0.0783000439,-0.2104012817,-2.9151592255,0.0618991032,-0.8712686896,-0.9809957743,-1.3866854906,0.2422163934,0.7125367522,-0.1848194599,2.3176703453,-0.4459954202,0.5021077394,-0.2677321136,-0.6174522042,0.0102121448,-0.8748270869,-0.5692952275,-0.2101370692,-3.1677110195,-0.9791932106,-0.4656033218,-0.8843027353,0.0018855287,1.6177743673,0.7415726781,1.2843861580,-1.5041314363,0.3272034526,1.0130852461,1.0509754419,0.1386350840,0.1534719616,-0.8860257864,-0.2906185389,-0.4992392659,1.0161581039,-0.3753769994,-0.8682446480,-0.9607298374,-0.9157308340,-2.2028121948,-1.2302342653,-1.0565544367,0.6787397861,0.4913366139,1.1512709856,0.2273500413,-0.3140943348,0.9650218487,1.1202888489,1.4143911600,0.1416300088,-0.6327657104,1.1599075794,0.9035273790,0.3643698692,-0.1275601387,-1.2958214283,0.4469485879,0.1940546483,1.2936275005,0.2017951459,-0.8351511955,-0.0646853894,0.1063179448,-0.7791896462,1.3193002939,-0.0662499070,-0.1504552513,0.5341873169,-0.4398691654,-0.0537598990,-0.8293182850,-0.8617116809,0.7136631012],[-0.3939204812,1.5737469196,-0.0540837012,1.4036841393,0.1378640980,0.9466567636,-0.7069780827,-0.4565291703,-0.1137186959,-0.1762137711,-1.0199981928,-0.2759044766,0.7262592316,0.2267369032,1.0947389603,0.2019783854,0.3519686162,1.2753775120,-0.0727279261,-1.5853526592,2.0832924843,0.9841842651,-0.9553976655,-0.1030969620,0.1650699079,-1.0748144388,-1.8021926880,-1.3359909058,0.3166110814,-0.2559000254,1.7907859087,-2.5822634697,-0.7786031961,-0.8376648426,0.3916146159,1.1735594273,0.1578970551,-0.0654573068,-1.0033574104,0.4908643365,-0.3393624723,0.2656250000,0.5900098085,0.1851583868,1.1900016069,-0.0636883900,-0.6513484716,-0.1623256058,-2.4267354012,-0.3015221357,0.8572072387,-0.3075442612,1.3093880415,0.1426732540,-0.0542080179,2.3609998226,-0.5969130993,-0.9994596243,-2.0351839066,1.2536141872,-0.0141155524,1.1488707066,0.3340318203,2.5600366592,0.3605966866,0.6146651506,-1.4186687469,0.2727081180,0.0324210338,0.1732143015,1.2016236782,1.1515592337,-0.8048486114,-0.4166365862,-0.8107392192,-0.2680552602,0.7969276309,-0.2536909580,-1.2414793968,0.0307318419,-0.7246635556,-0.0614210144,-0.0980322510,-0.1273739636,-0.8733004928,0.8253334761,0.3285685182,0.2195947468,-0.4676854312,-1.7725604773,-0.0706478730,-0.8255471587,-0.8020666242,0.7796689868,0.0053337635,0.5974035263,-0.5471351743,-0.5485429168,-0.0285736788,0.0145894540,-0.8462850451,-0.3286907971,0.5849106312,-0.0293410309,0.3513242304,-0.6039078236,0.6773921847,0.8920910358,-0.3698989749,0.8443539739,-0.9569555521,2.1414206028,-1.4383490086,1.8881260157,0.2543646097,0.1823499501,-1.8379988670,-2.3626091480,1.9306805134,1.1447869539,-0.7892955542,-1.3583947420,0.1193050891,-0.5391947627,-0.8748807907,-1.0574221611,0.1377330273,-0.1817522049,0.3828250766,-0.6502447128,-0.4013140500,1.5177444220,-1.1044421196,0.4088673890,-1.6362469196,-0.0767254755,-2.1334729195,1.6754659414,-1.0141512156,-0.2841005921,-0.7273792028,-0.1547815204,-0.3518456221,-1.6101245880,0.3806446791,1.5120161772,-0.3957258463,-0.3997440934,1.0012264252,-0.5169299841,0.2179621607,0.0726020262,0.8056242466,-0.3302493989,-0.5640383363,1.6068236828,-0.8795584440,-2.0162665844,-2.8408839703,0.0471107960,-2.4121069908,-0.4298157692,-0.2259212136,-0.2872554958,-2.6355597973,-0.0566891581,-0.3242926300,-2.4704289436,0.1055299938,-0.9959416389,0.3086238503,0.9671957493,-0.5396192074,0.6040809751,0.5914162993,-1.0520744324,0.2196249813,1.1837112904,0.7854421139,0.2140046954,0.0103036715,-0.4791815281,1.0802093744,2.2894952297,0.5970520377,-1.0285168886,-0.8583711982,0.5456027985,-0.4148845971,-0.3353210390,0.1510142237,-1.1208082438,0.4330037832,1.5839723349,-0.4517824650,0.7348692417,0.6062211394,1.1804046631,-0.2156865895,-1.9402424097,0.7674831152,-0.6204615831,-0.9424259663,-0.7667592168,-1.7198344469,0.1882339567,0.3915354908,-1.2951434851,-1.4637438059,0.4099708200,-0.3732828200,0.2660309076,2.2915365696,0.8822026849,1.2864061594,-1.0476447344,-0.5300865173,-0.8190235496,-0.7607998252,0.6160025001,1.4188951254,-1.2633959055,0.9278722405,1.2578333616],[2.5271196365,0.2058767378,0.4043089151,0.9949845672,-3.1527338028,1.2946237326,0.9710022211,-0.1420307904,0.5478397608,-0.8206241131,0.3207755387,-1.1367980242,-0.3007896543,1.0672885180,1.1049232483,1.0061985254,-0.6842412949,0.4629309773,-0.6022585034,-1.8163396120,2.4398572445,-0.5302162170,-0.1656367332,-0.4852881432,0.3775443435,0.5550758839,0.1274569929,-0.5471289158,-0.3199679852,1.2949386835,-0.6124832630,-0.1371923089,-0.2731872797,0.6984583139,-0.9401183724,1.2994360924,0.8181514740,0.7731116414,1.0036376715,1.5827988386,-1.2974874973,-0.1434350908,0.6647178531,1.7826393843,-0.5892428756,0.2183041722,-0.9415503144,0.6296509504,1.4211704731,1.8153684139,0.4850921035,-0.8563700914,0.6953250170,0.7074752450,-0.9068002105,-0.7238519192,0.8457453251,0.6591275930,-1.3648034334,0.9885088205,0.9899643660,-1.4283225536,-0.2611402273,0.6861331463,0.6731824875,-0.9871585965,0.3236202002,-0.2243195921,-1.3043832779,-0.9144351482,-0.0288481209,0.3471129537,1.9157763720,1.3639667034,-0.0679871738,0.0099768415,-0.7806731462,-0.3309215307,-0.5239000916,-0.7487209439,-2.0122323036,-0.7074913979,1.0903810263,-0.1148329824,0.7587329149,-0.8750514984,0.1420132518,0.7049495578,0.9195414782,0.0968873128,-0.5733492970,1.3557649851,0.6867563128,0.5227616429,1.4819657803,-0.5568839312,-0.4434660375,-0.6686267257,-0.2037937194,-0.2320554256,-1.2747259140,-0.5040209889,0.1536800563,0.3864961267,0.1177268252,0.4147295058,-0.4290784895,-2.1149220467,0.1111558527,0.5690610409,-0.5074170828,-1.3116228580,-0.3538000584,0.9582650065,-0.4444060624,0.4389349818,0.6684995890,2.0895423889,-1.5145853758,-1.0995954275,-1.9046037197,1.4437774420,-0.3302248716,-0.9572323561,0.3254705071,1.1758222580,-0.3314490020,-1.9969503880,-0.2365534455,-0.0082780914,0.2744103074,0.0309559889,0.4002328515,-0.7249093652,-0.1263285577,-1.1013489962,1.3815333843,1.8019288778,0.0788859054,-0.8572096825,-0.7831746936,-0.6566607952,-0.5514259338,1.1561759710,0.0486259237,-1.4882011414,-1.7478359938,-0.4349352121,-0.5980690718,-0.0105753476,0.6441072226,0.8070641756,1.3366454840,-2.4222583771,-1.7629585266,0.4242415428,0.5227168798,1.2409411669,-0.9768437743,0.5091602802,0.4706636965,0.8562821746,0.5442659259,0.9753083587,-1.0318638086,1.0895196199,-0.3157981634,-0.2498181760,-0.0072620874,0.3074987233,0.7405095100,0.0923283026,1.5314687490,0.4712828100,0.7196969986,-1.2597471476,-0.8728388548,-0.7292687893,0.1062755734,-0.7407863736,-1.3908658028,0.0886164233,0.6232225895,-1.4311265945,0.9013275504,-0.3596685529,0.1025697961,0.0446388386,0.6425230503,-0.2347142994,1.9980522394,-1.0065909624,0.1190448180,-0.2422155142,0.0530792251,-1.4308447838,0.2798071504,0.6393198371,0.4337556064,1.0134214163,-0.7661970258,-0.3400665224,0.6894814372,-0.6211586595,-1.1139371395,-0.4133455455,-0.8308604360,0.1453806609,-0.7555692196,0.6801390052,0.8128879666,0.5958173275,1.0729752779,0.4843373299,-0.4979102314,1.3864185810,0.9193938971,-1.5006879568,-0.4991283119,-0.2648375630,0.4287530482,0.8467736244,0.9204974174,-1.7422208786],[-1.4175399542,1.3389828205,-0.1382659376,0.5003929138,-0.8001933694,-1.1260420084,-0.7873304486,-0.4730960131,0.1702415347,-0.4686396122,-1.4204618931,1.7457896471,-1.9309419394,0.6022258401,1.0969513655,0.2393837273,0.2116515785,0.5585256219,1.0960396528,0.9194073081,0.2306050062,-1.1204822063,0.5107620358,0.7941451073,0.1340462267,-0.8106143475,0.3599536419,0.5154025555,0.0600135475,-0.7232104540,0.3722366095,0.2504935563,-0.0045491043,-0.6716938615,0.2279936522,0.1172791049,0.5441622734,0.7014442682,0.1636167318,1.9247628450,1.3642557859,-1.3011333942,1.4473772049,-0.2376226336,0.6195828319,0.2934015989,-0.5696805120,0.0819147378,-0.0124860499,1.4523462057,0.4789505899,-1.6496356726,-0.0578171164,0.6523512602,1.5945521593,-1.2960835695,-0.6454660892,0.3898889124,-0.2493240684,-0.7321791053,-1.3518134356,-0.8230043650,1.4176563025,-1.4937674999,0.6788947582,0.0714109913,-0.1779415756,-1.9323011637,0.6027720571,-0.2723703384,1.1922283173,0.9429624081,-0.1027375609,2.0723633766,-0.8419672847,-0.7306387424,-1.5093916655,0.6934100389,-0.0410907716,0.7249060273,-0.1354019344,0.7563098669,0.3674355447,0.7718367577,-2.6049280167,-1.3262532949,-1.2036824226,-0.4017079473,0.4643644989,0.3264178038,-2.6617326736,-0.7910155654,-0.3290653527,0.2054002136,0.9232746363,-0.3793618381,-0.9501160979,0.2249486148,-0.6229797602,-0.9222994447,-0.8020499349,-0.1218004450,-0.1192131341,0.1981703192,0.2027932554,-0.5831612349,-1.5477751493,-0.6242799163,-1.9537978172,-1.3120753765,-0.8792384267,-0.2546037138,1.8232898712,1.9963171482,0.4631770551,-0.2157828957,-0.0033036820,0.9925291538,-0.6906385422,0.6020773053,1.8886865377,-1.2071678638,0.0356135145,1.9553414583,0.7453075647,-0.5721513033,0.3352523446,-0.8478366137,-0.7228087187,0.8866829276,-0.9401163459,-1.9854770899,1.0203789473,-0.4592664838,-0.5595240593,0.4180965722,-0.1041441485,-1.3117289543,-1.1925373077,-1.3618315458,-0.2454625666,-1.3739359379,-0.1230784133,1.3046419621,-0.3713450730,-0.1312476695,0.7927514315,-0.8698234558,-0.1680863947,-0.6188244224,0.3212174177,1.1077984571,-0.8575962782,-1.2911204100,-0.1478271186,0.8769837022,-0.4112337232,1.2420055866,-0.2285336703,1.3366562128,-1.6899101734,0.2851883173,-0.9931355119,-0.2546955347,-0.7716149092,1.0934343338,0.0426416881,1.5614147186,-1.0928755999,-1.6933665276,-0.4665859342,1.6828256845,-2.1104826927,-0.0419604629,-2.0320513248,1.1672431231,-1.0663675070,0.4055701494,2.9667816162,0.3135769367,-0.1272612065,-0.0019367305,-1.7865858078,-0.7973675132,0.0983067825,-0.7250729799,-0.1266469806,0.0654597431,1.6557376385,0.1442626268,-1.7148323059,-0.7256135941,0.2546228766,0.9104588032,0.4652565122,1.3994613886,-0.7664901614,0.1988223642,-0.6781148314,-2.5610532761,-1.2301489115,0.5389909148,0.4202900231,-0.1574928910,0.7148252130,0.0710679144,-1.8049743176,0.0158903282,1.6363402605,0.1195532903,1.5923948288,-1.0496184826,0.0179251097,-0.7340781093,-0.0044836081,0.2723886967,-0.1416033208,0.1716030240,-0.4047521949,0.2403499484,1.2817729712,0.3288176060,-1.0603730679,2.0009100437],[0.3797760308,2.0110740662,-0.2068622559,0.7860251069,-0.5047575235,0.7063904405,-1.1786493063,0.3923666775,-1.0244634151,1.8500113487,0.5193219185,-0.9555152655,1.3364324570,1.4916113615,-0.5875380635,-0.2249176949,0.5363004804,-0.4077191949,1.2315989733,-0.0209633969,-0.7538880110,-0.4608057439,0.1677348167,0.6560636759,-1.0530261993,-1.1236027479,0.5489697456,1.1629155874,1.0196343660,-0.0328121632,-0.7612766027,-0.3375090957,-0.0431881398,-1.3949629068,1.6447660923,0.6821311712,-0.2217063755,0.7762494683,-0.1715980470,1.7823339701,-0.4120734036,0.3940093219,0.4754644036,0.0328231864,1.2554707527,-0.5431787372,1.2304182053,-0.8765879869,0.9375509024,1.1339820623,-0.3679790795,0.9494005442,0.9975679517,1.0307329893,2.1102905273,0.7695582509,0.5826106071,0.3467871845,0.0634680912,0.7141479254,0.2264734805,2.0048527718,-1.0984022617,1.2317520380,-0.8595981598,-1.2654389143,-0.6585014462,-1.2234858274,0.8852560520,-0.4570990801,-0.5080608726,0.1632690579,1.0896167755,0.3817169368,-0.0259354860,-0.4530935287,0.2448261827,-0.0249542110,-0.5475708842,0.8203950524,-0.0454053171,-0.1523360014,0.1439412832,-0.6669614315,-0.2771725059,0.3978259265,-0.4932720065,-0.9815487266,0.0112280538,0.0622864105,0.1676025093,-0.3985056281,-0.7404897213,0.8559097052,-0.8888737559,-0.3696354628,0.7236633301,-0.8088818192,0.1112069711,0.8000018001,-0.2828109860,-1.3591109514,0.0597446077,1.1427609921,0.9351323247,-0.5741488338,0.8525617719,0.3621784449,1.7871932983,-0.1630001217,-0.4958277643,0.3296086788,0.1071355641,-1.2501461506,0.2655957639,0.9955238700,0.3480997086,0.5811783075,-2.4376964569,-1.1687315702,-0.0813406780,-0.5605207086,0.8873816133,-0.6016888618,0.6948810816,0.3314882815,-3.5329918861,-1.4900625944,0.9414265752,-0.0608544908,-0.2282129824,0.2961200774,0.2064784765,0.9235963821,0.8165992498,1.0421718359,0.6450840831,0.3433330655,0.4462094009,-1.2916414738,0.2994377315,-0.6783190370,-1.4601011276,1.0403785706,-0.2990740836,-1.3489027023,0.4652444422,-1.8370354176,-0.8784723282,0.9383602738,-0.9043878317,-1.5784652233,1.0723438263,0.2476393133,-0.8018630743,-0.4048002362,-0.0732393563,0.9851552844,-0.7021204829,0.6982021928,-0.6263809204,0.7441608906,-1.2284842730,-0.0449464247,-0.6855147481,-1.4545799494,2.1253712177,0.6591351628,1.0927282572,1.2526097298,-0.6281160712,0.4327580929,1.6550632715,-2.2945992947,-0.0502533279,1.5303075314,-0.5409997702,1.1727341413,-0.5400050879,-0.4364598989,-0.0967092067,-0.4712429643,-0.6747933030,0.5726812482,1.1124210358,-0.0922250971,-1.5018668175,-1.0531485081,0.0086183911,-0.0545431152,-1.5454921722,-1.6756819487,1.5852866173,0.0307595395,1.1762250662,2.2206773758,-1.2185657024,0.5149343610,0.9203640819,0.5588316321,-0.0089860512,-1.4906351566,-1.3091335297,-1.5451923609,0.4636520147,-0.8842917085,0.2086382061,-1.3636584282,-0.5502992868,-0.3902366161,-0.5923804045,-1.5842198133,-0.0137870312,0.0795252994,1.9029623270,-1.1827250719,0.2084009647,0.1835195422,0.4632900953,0.3212157488,1.2559412718,-0.1729954481,-1.1393396854,0.4833806455],[2.6549761295,-0.0624392033,0.2982380688,-1.1094950438,-0.4198685288,0.0068994844,0.3850246668,-0.5606642962,0.3175287545,-0.3674174547,0.4738435745,-0.5514103174,-0.9766073227,0.5305664539,-1.0486661196,0.2787618637,-0.8301656246,-0.1464786083,0.4386092722,1.0080722570,0.1367931962,0.6738983989,0.3898560107,2.6003527641,0.8472453356,0.8475563526,0.4127884507,1.4216381311,0.4300921261,0.4178426266,-1.4710381031,0.2721470594,0.8424003720,-0.2040570527,-0.8711925745,0.6280260086,0.2913725376,0.5029097199,-0.7581620216,0.1996175051,-0.8232809901,1.2004920244,0.5533089042,-0.8662698269,1.6752365828,0.5223246217,-0.0690548792,-1.2152057886,-0.0013798352,-0.4171872437,1.5942960978,-1.9580565691,0.9489240646,-0.5740866065,0.2111291140,0.9112172127,-0.2032184899,-0.8871902823,-0.3309149742,-0.7680382133,-0.2359905690,-0.9237763286,-0.0109956544,-1.9898606539,2.5647330284,-0.6217227578,0.0064217350,-0.1358610094,-1.1649965048,0.6981079578,-0.5372963548,-0.1233563721,0.2467169762,1.5969997644,1.5088599920,0.8951876760,0.3103065491,-0.2586609125,-0.0645775348,1.1478109360,0.7551135421,-1.5989308357,-1.0639597178,0.8607670069,-0.9049039483,1.0084027052,-1.0110167265,1.8463071585,-0.7842986584,-1.8329027891,1.6607512236,-0.1428315938,0.1744260192,2.4861826897,-0.1852011085,-0.3658530414,-1.5189673901,-0.8197247386,-1.4171614647,-0.7043625116,-0.0423857495,1.9210053682,-0.2739804387,-1.6898438931,-0.2022707313,0.5238206387,-1.7040828466,-0.6559479237,-0.2088008374,-0.4546749890,0.6832018495,-0.1399490386,-0.0158994384,1.2031900883,-2.3218762875,-0.8002865911,0.1409174502,-0.8941115737,0.1592972279,-0.7806494236,-2.2411077023,-0.6832921505,-0.2099120617,0.2427671105,0.5994983912,1.7587484121,-0.4734762311,0.9576066732,1.0439326763,0.3215130866,1.1165105104,-1.1664500237,-0.2707373798,0.8627887964,-0.5434683561,0.4097805321,0.8309146166,-1.2992010117,0.8051446080,-0.0502262339,1.2870852947,-0.0100108348,2.0543682575,0.3049731255,1.5535672903,1.2411839962,0.9481426477,-0.8539407849,1.0286288261,0.0647694692,1.0538676977,0.1396375895,0.6957244277,0.2910648584,-0.9009956121,1.1273261309,-0.0948203057,-0.8096640706,-0.4723818004,-0.6017938852,-1.5425246954,-1.3817493916,0.0491434112,1.0174921751,0.2119679451,0.1593198776,0.2184775919,0.0306259133,-0.0538404770,-0.8706930876,-0.2231848836,2.4912929535,-0.5015062094,1.0344709158,0.2444898188,1.4917860031,-0.6608645916,-1.4219645262,-0.2831400335,-1.3320243359,1.7224861383,1.0793416500,-2.1341109276,1.2406805754,-0.7852844596,0.0782570988,0.6343486905,-1.2520850897,0.7410355806,-0.6997129321,0.1451499909,-0.1370168477,0.9041714072,-0.1804198623,0.0712918416,-1.4128985405,0.7510366440,-0.1180425957,-0.1724923551,-1.7639014721,-0.0395201556,0.7621546388,-0.5221664310,0.7228161693,-0.1106093898,0.4198686481,0.8827266097,0.1813685149,-0.6062692404,1.4166896343,-1.0538520813,-1.7832826376,0.1536631137,0.7696893811,1.4886592627,0.2027274966,-0.3299671710,-2.1470396519,0.1910541207,1.2473070621,-0.6160305738,0.9013956189,-1.1003696918,0.0428022183],[-0.2240002900,1.3721675873,0.2639842331,0.9908525348,-0.6108818054,0.6185465455,0.8091144562,-0.0958564207,-2.4742417336,-1.4627320766,1.2571899891,2.1967086792,-0.6468477249,0.4778275192,0.4487043321,0.8870905638,-0.5353265405,0.8139606714,-0.9202001095,-0.9111188054,-0.4403496683,-1.0960507393,-0.7820392251,0.3184137642,-0.8955340981,1.3307368755,2.5789258480,1.6368669271,-0.1470758021,-0.5990946293,1.8969904184,0.0959065408,0.3653283119,-0.8044139743,-0.1963423491,0.6994704604,0.9716321230,-2.4430494308,-0.0891370699,1.6713135242,-0.0061685285,1.3528705835,-0.7780772448,-1.2525608540,0.5983073115,0.7732192874,-0.6087434292,0.9746813774,-1.3182982206,-0.8068457842,-0.0047311722,1.7748740911,-0.6146159172,0.4845753610,0.6594731808,-1.1879068613,-0.3992961049,0.6266974211,0.4242372215,0.9081152678,0.0307728127,-0.8630605340,1.9725608826,0.9605245590,0.5259451270,0.1010104865,0.9442236423,-0.9346194267,-0.3141328990,1.5740138292,0.8825586438,1.2932610512,-2.2646677494,-0.8570740223,2.5405399799,-0.9077283740,-0.0705576167,1.1092141867,1.1942255497,0.1073936000,-0.7081599832,-0.1220684573,-0.1524242908,2.7508671284,1.3673774004,0.2572293878,-0.7890204787,-1.9807367325,-0.0600517057,-0.3794493377,-0.2172252089,-0.1736208946,0.4301603138,1.1614133120,1.8514386415,1.1475257874,0.4402486980,1.6443941593,0.8660095930,-1.0846067667,-0.7280923724,-0.7247751355,1.1563705206,-0.6895301938,-0.2457161993,0.6665600538,-0.1587394476,-0.9480696321,0.6335174441,-0.8753589392,-0.3239900768,-0.4539276361,-1.1788517237,-0.0075284201,-0.3648636639,0.7042046785,-0.8619546294,-0.8080089092,-1.4834641218,-0.5682203174,0.0640545860,-1.3926751614,-0.4957585931,-0.9867656231,2.2077383995,0.8689068556,-0.1288353801,-0.2259429991,-0.3784595132,-0.1942718327,-0.4684728384,1.3292363882,-0.5330204368,0.9046695828,1.6821259260,0.0649134219,0.9101820588,-0.6161113381,0.5321943164,0.1378296018,-0.7966747284,-0.3984784782,-0.7223089933,0.1889540851,-0.3044525981,-0.5599074960,0.2821009755,-1.0995750427,-1.1855823994,0.1406255960,0.1761286408,-1.1918876171,1.7644279003,0.8393975496,-0.1007546857,-1.5601830482,-0.2472936213,-0.6002095938,-0.5832691193,0.5630375743,-0.2304146737,-0.5821666121,-0.8668646812,0.1507098377,-0.8167546988,-0.7604181767,-1.3662077188,-1.5394088030,-1.5751433372,-0.4525258243,-0.3810411990,-0.2660856843,-0.7493767738,-0.8499608636,-0.3706573248,0.0562004596,-0.0263963435,-0.0762054101,0.0347596668,-1.7043470144,-0.3087723255,0.0925105289,0.3055296242,-0.6038087010,-1.3270583153,0.9827371836,0.2591477334,-0.5748916268,0.2182596624,3.1320657730,-0.1381565481,-0.7192556262,-0.0967331678,-1.2724033594,0.1888975799,0.9075344205,0.4408209026,-0.5638342500,-2.3636741638,1.3404798508,-0.4956731200,-0.1394995302,-0.5335831642,-1.9022147655,0.7844576836,-1.7396374941,-1.0470457077,0.0059125903,1.1089094877,-0.9378197789,1.2084882259,0.6415619850,1.1708856821,0.9288407564,-0.4969819784,-1.6088763475,-0.2479011863,0.3180273175,-1.4168658257,1.8286755085,0.1195928678,0.7719916105,-0.7462855577,0.1041880324],[-0.9672582150,-2.6459031105,-0.1165833101,-3.1117084026,1.9623993635,0.8309456706,0.1377648413,-2.3812682629,0.6896886230,-0.1855984628,0.3543067873,0.9753615856,-0.6475732327,-0.8925844431,-1.0086665154,0.0782525837,-0.6183305979,0.6852272153,0.1687427461,-0.8541080356,-0.2065338194,0.4088042080,1.0348843336,-1.8947924376,0.0465237722,-0.4570004940,-1.2192322016,0.8203142881,0.3576781452,-0.2713727951,-0.0683235750,0.2432604730,1.3800150156,1.1533381939,-0.9297128320,0.5261302590,0.9035075307,-1.2131354809,0.2522619963,-1.5607283115,0.7069227099,-1.2718977928,0.5051910281,-1.1626993418,-0.0957609937,1.2475578785,0.4460627437,1.6918580532,-0.3250173628,0.6188752055,0.3587437868,1.6347857714,-0.5673042536,-0.6678325534,-1.5126785040,-2.0843534470,0.9316207170,-0.7939562201,-0.1389047503,-1.0928068161,1.3225532770,-1.9624245167,-0.5996420979,-1.2197872400,-0.9119546413,0.4276641309,0.0259381942,-0.4532708824,-1.7074309587,-0.8331914544,0.7614836693,0.0791674033,-0.6992869973,-0.8943839669,-0.2679222226,0.2194266021,-0.8759810328,-0.5073649287,-1.5110708475,2.2356791496,-1.3427168131,1.5245552063,-0.3868552744,0.1256234497,-0.1761559248,1.0805300474,2.1072673798,0.9131340384,-0.5150635839,-0.5568994284,0.1934793591,0.2294926941,0.0569185466,-1.5624272823,-0.3017155826,-1.1742011309,1.6733289957,0.3321471512,-2.5649945736,-1.0484580994,1.5249539614,0.6937811971,-1.4968557358,-0.0413185954,0.0577384606,-1.4827886820,0.6641077399,-0.4808337986,-0.5935848355,1.5506532192,0.2863505781,0.3010305166,0.7863500118,1.0970948935,1.0758353472,-0.6738184094,0.3312270939,-0.4622066617,1.8852907419,0.0494275317,0.6119221449,-0.9082134962,-0.2488493621,1.5805485249,0.5710447431,-0.0100161182,1.0434415340,2.6215713024,0.6912501454,0.0497087650,0.8760806322,-0.3577581644,1.4436415434,0.0282163415,-0.4429470897,0.0530390367,0.0762220472,1.1364022493,-1.6470911503,0.6877381206,0.8901814222,0.7812303305,1.3263591528,0.0477996878,2.5010333061,-2.1394217014,-1.9078707695,1.3412349224,0.4777317047,-0.0139821116,-0.1222880036,-1.0256145000,-0.3868508935,0.4203441143,-1.0873005390,-1.2217711210,-0.5821444392,-0.8056239486,-1.0757553577,-0.1141380593,0.7272791266,0.2058192939,0.0688363239,-0.2044557482,-1.1473109722,0.4968492687,-1.4591329098,0.1536498815,0.5128298402,1.3853830099,0.3292523623,-1.7917298079,-1.6369482279,0.8037596941,-0.3507469296,-0.9165260196,0.6277545094,-1.8771381378,-1.1168591976,2.0002963543,0.9762254953,-0.2480376661,-0.5302870870,1.9402929544,0.0267861206,-1.3505047560,1.7754563093,0.7663806081,-1.2515767813,-0.4576311707,-1.6291935444,-0.0677957162,0.6024122238,-0.9290607572,-1.2449533939,0.6113210320,-0.1610737294,0.0371984690,-0.2587880194,0.6847470403,0.3342157006,1.4733811617,0.6087473631,1.1020613909,-2.3822624683,0.6188970208,-0.6561591625,0.2269476801,0.2247478813,0.1287886500,1.0781495571,-1.5613390207,0.2265433371,-0.3973863423,0.8339469433,-0.6040529013,1.7165282965,0.7048300505,1.7899045944,0.2359161973,1.1899683475,0.9303969741,-0.5528115630,0.8526928425],[0.4013674855,0.0542528853,-0.9132806659,0.2976969779,-0.4440785944,0.8972502947,0.8655430675,1.1715748310,-0.1977638453,-0.2246027142,0.0634627566,-0.1137804464,-0.7993347645,-1.0984220505,-0.6201526523,1.0808788538,-1.2932568789,1.0611051321,0.5813468099,0.4603781402,0.5116732717,-0.1236280203,-0.3366009891,2.3701324463,-0.2713805735,-0.9819604158,2.4698693752,-2.2890300751,1.6352475882,2.1706984043,1.7595663071,1.3014839888,-0.3608497679,1.2908350229,0.0412799045,1.0770276785,-0.8863621354,1.3306670189,0.6597825289,0.4063103199,0.0115959998,1.3941509724,-0.8768700957,1.2024830580,-0.2052872479,-0.3123206198,-0.1067714840,0.6414546371,0.4451971352,0.8133208752,0.4083485901,0.7939832807,-0.4251754880,0.4888560474,-1.1338469982,0.3686774671,-1.5876305103,-1.1317766905,0.3729013801,-0.0210948307,0.1209617406,-2.4760849476,1.0929088593,-0.2816660404,1.1428662539,0.4579641521,2.0584762096,-0.6132389307,0.2567722499,0.0170761682,1.2307608128,1.3749313354,0.8315657973,-0.4026392400,2.3707995415,-1.1325217485,-0.7696317434,0.9405528307,-0.4137949049,-0.1175819635,0.7805142403,1.8320804834,0.3148280978,0.3389058113,0.4954504073,-0.0342098735,1.2256599665,0.6079592109,-0.3224415183,-1.5470880270,-1.8527430296,0.7457096577,0.0043871086,-0.3397135139,0.6516956687,0.8055987954,-0.2371622622,-0.1931099892,1.1936423779,0.1004671603,1.0995388031,-0.2976596951,-0.1392439753,0.5259703994,-1.7792139053,0.7952412963,-0.2315692008,-1.1396455765,-3.1083474159,0.2638598979,0.1089628115,-0.7766565681,-1.1014711857,-0.1453552842,-1.4054925442,-0.0237163752,0.2195326090,0.5424352884,0.8379007578,-0.2166462690,0.2039635777,0.6850985885,-0.2847076654,0.7809556127,-0.3086790740,-0.3514919281,0.9503691196,-1.7364805937,-1.2029470205,-0.1090737283,2.3331315517,-0.7135651708,0.2003588378,0.1698276401,-0.8578826785,-1.7039403915,-0.1923047006,-0.6373355985,1.4842764139,0.8991172314,2.2821702957,0.2180647701,-2.0747385025,-1.3818309307,1.2608627081,-0.5096740723,-1.3451678753,1.1649746895,1.5543422699,0.6470571160,1.7453366518,-1.4800138474,-0.8186712861,-0.6925798059,0.3354967237,0.0370089076,-0.0598824024,-1.0103763342,0.2989828289,-1.2845501900,0.1014768258,-0.5032425523,0.8546702862,0.4338257015,-0.5709214211,-1.6417970657,0.2938322723,0.7571172118,-1.2316056490,0.5295363069,-0.3947353959,-0.5521669388,0.8162702918,0.0821020678,0.5902984738,-0.0606680997,-0.9304512739,-0.4575162828,-0.5528289080,-1.0609980822,0.1770114452,-0.7643995881,-1.8346927166,-0.3263114691,-0.7260974646,-0.2625757158,0.2421620190,0.8234714866,-0.3505697548,-0.1155276671,2.2433609962,-0.6450213194,0.5869686007,0.1855111122,-1.3059544563,-0.0481009148,-0.5584850907,0.9967713356,-0.1201881021,-0.0914791599,-0.6160403490,-0.5912835598,0.5793107748,0.8839061856,-0.6082174778,-0.1807097793,0.0818651319,-0.3470787704,1.5814223289,0.3732798398,0.0022905916,-1.1152783632,-0.8789777756,0.3264580667,-0.0843966827,2.0768249035,0.3223586380,-1.4704591036,-0.3605157733,-0.0795432106,1.3762940168,-0.3133580089,-0.3769643009,1.0150228739],[-1.6130084991,0.7662905455,0.1537739336,-0.5736240745,-2.0595908165,-1.2742669582,-0.4551030099,1.1831039190,1.2629544735,-0.8028033376,-0.6116998196,0.2680445313,2.1303925514,0.6933071613,0.8110494614,1.6245647669,0.0246582367,0.1792008579,-0.6756305099,-0.2165194303,-1.7439608574,-1.0947178602,-0.5515912175,1.1397629976,-0.1814203858,-1.0873012543,1.9065188169,0.4820602238,1.2316817045,-0.7144633532,0.7393344641,-1.4779632092,0.8479858041,-0.6084827185,-1.4399133921,-1.2956974506,0.2913162410,-0.9487263560,-2.4500293732,1.2459511757,-0.8626870513,1.2747393847,0.8515521288,-0.9107000828,-0.0041014827,-1.0741207600,0.1243414879,-1.6088092327,0.3258252740,-0.5649567246,0.2144881934,0.1281290799,1.3411060572,1.1804949045,0.6733310819,1.5220293999,1.1019228697,-0.6039502025,0.4356611669,-0.4059346914,-0.6738638878,-0.2382538617,-0.3904451728,-0.9828028679,1.2583773136,1.6638858318,0.0410357155,-1.3966635466,0.1359305084,0.0167859215,0.6521763206,-0.2972988784,0.1731460541,0.1411231160,1.0567188263,1.6228011847,-0.5794419050,0.4272536039,-1.5739947557,-0.8606178761,1.0731964111,0.1164292842,1.1253268719,1.0621339083,1.8113048077,-0.2671763003,0.6486790180,0.6535865068,2.1206243038,0.3569327295,-0.3855935633,2.1172261238,-1.2179837227,-0.3329633772,1.1804958582,0.4870519936,-0.0225906167,-0.1032814756,0.2909719646,1.6734399796,0.6995837688,1.6279866695,-0.9171071053,-0.4649128914,-0.7849975228,0.1026836857,1.7261559963,1.4697047472,-0.9684790373,0.0554991364,-1.7402436733,-0.2492827326,-0.0431437008,-1.3587737083,0.6815609932,-0.8693833351,1.2369399071,-1.8863832951,-0.2015711665,0.5212758183,-0.8070146441,0.6563775539,-0.6166521311,1.0231083632,1.2899951935,0.6691089869,0.5692141056,1.3590040207,-1.2761324644,0.2012262344,0.1072357595,0.0752140060,-0.5331254601,0.2735618353,1.3830242157,0.4706042409,-1.1795712709,-0.3537247181,-0.5190734863,-1.1926151514,0.1233324781,0.3509021699,-0.5091461539,-0.0536347479,0.5855531096,0.5949400067,0.2779593468,1.2884949446,-0.9481160641,-0.4234532714,-1.1077837944,1.2787302732,2.0269575119,1.4635083675,-0.9697679877,2.6235475540,1.2805155516,-0.1222272217,0.3515428901,1.0016529560,-0.0150964251,1.6880726814,-0.2948833406,-2.3703005314,1.8436559439,-1.2131029367,-0.2624935508,0.9433795810,1.8394038677,-0.8153470159,-0.9672814608,-0.2367526293,1.2682460546,-1.8260550499,-0.6101045012,-0.8524032831,2.4874143600,0.6338421106,0.7834047079,0.6262134910,0.8847147226,-1.1235461235,-0.9794437885,0.3526352048,0.4729247093,-0.2141827941,-0.7229529023,0.0829834938,1.3534119129,-0.4975259006,1.0918790102,0.5944721699,-1.4659571648,0.8361387849,-0.3421808183,-1.0087279081,-1.2426744699,0.3281867206,0.0251218807,-1.8685960770,0.9150018096,-1.9452970028,-0.8615595698,2.1733384132,-0.9754824638,1.6710414886,0.4690485597,0.8657920957,-0.4328330755,1.2893782854,-1.0481760502,1.3680763245,-1.3068273067,0.9040389657,0.9369824529,0.8598939776,-0.8227170110,-0.3654539287,-0.6739743948,0.3190602660,-0.9016777277,-0.4370117188,1.0818015337,-0.7992702127],[-0.8296168447,0.5022249818,0.3412137330,-0.3512504399,-0.3503727615,-1.8021380901,2.2804696560,-1.1720472574,0.2263271660,0.2877822518,-1.1337782145,-0.3909066916,-1.2376701832,1.0095298290,0.5217128992,-2.2821540833,0.5881370306,-0.2536352277,-2.5339996815,0.0288899709,0.1474454105,-1.5530334711,-0.1875560880,-0.2964601815,-0.1737500578,1.6984698772,0.9925410748,-0.1834912300,0.0390073396,1.2886043787,1.1899611950,1.6747417450,-0.0493365452,0.6665617228,-0.3943178952,0.4373920858,1.0100117922,-0.2434471995,0.4705701172,-0.0913033038,-0.1496414989,-0.1203377172,-0.9248952270,3.0170164108,-0.2714706659,0.4221477807,2.1623134613,0.8745349050,1.3869328499,0.7160037160,0.8574240804,-2.4605793953,1.4067883492,0.7826843858,0.0132020181,0.9728497267,-1.4302023649,-1.5503321886,0.7042499185,-0.0613900833,0.6003428102,0.6352524161,0.4287099242,1.4980370998,0.3880011737,-0.4563947618,-2.7111141682,-0.4926683903,-0.6083946824,0.9790099859,1.3553739786,1.1713775396,0.4362692833,-0.8042359352,-0.7024526000,0.3011744618,-1.4273091555,0.9567792416,0.6883066297,0.4607406855,-0.5390653014,0.6746558547,-0.8313179016,-1.0302064419,-0.0380053222,-1.8747175932,1.3076663017,-0.2346269339,-1.3506914377,-0.4520126581,-0.4520035982,1.7138769627,0.1253602654,0.2660092711,-0.0731366202,0.2919928432,0.0841064826,1.7978950739,-0.3284988999,1.1751443148,0.5153539181,0.2937392592,0.7174130082,0.4991062880,-0.1639733016,-0.3563329577,-0.4395076931,-0.2766478062,-1.0674500465,1.8819682598,0.7912315726,-0.1282933652,1.8472059965,-1.5106694698,0.7845364809,0.4449427724,-1.2240473032,0.5322827697,-0.1730926335,0.9488574266,0.4537747204,0.0691529736,3.4170293808,-0.9167636633,-0.3978354037,-0.5361738801,-0.4047685564,0.7148522139,0.2225925773,0.6639422774,-0.1292048544,-1.1110087633,-1.0102939606,0.2644718587,0.9000436068,-0.7264199257,1.3637973070,-1.2778736353,-0.1999248862,0.5343344212,-0.4080455005,0.1374131441,-0.2867443264,2.0778164864,-0.9607052207,0.9882478118,-0.3585515916,-1.4047431946,0.9531259537,0.2088538110,0.1369061917,1.3903574944,0.4575279653,-1.3510797024,-0.4881621003,0.7812165618,-0.4035978317,-1.0870518684,-1.5355210304,1.0299673080,-0.0183984078,0.1233158112,0.2659848630,-0.9376357794,-0.2414065301,-0.3026311100,0.0925273895,-0.0816137046,-0.0960379541,-0.0317799374,-1.3813295364,0.1347098649,-0.0820175707,-1.2389616966,1.6740804911,-0.2374209315,0.9658219814,-0.7939558625,1.2590600252,-1.5339493752,0.3371162415,1.4419755936,-0.4797958732,0.0273850411,0.7507920861,-0.1079000458,0.9697693586,-0.6925873160,-1.4347451925,0.3423879147,-0.6725934148,-1.7418110371,-0.0797282755,-0.3020401299,-0.3803686202,1.0125890970,-0.9325743318,-0.1049213335,0.0712086931,-0.2040427327,-0.7109842300,-0.7788132429,-0.4137320518,0.5208038688,0.1299502850,-0.1351567805,-0.6129224300,0.0145029593,0.5655912757,-0.8317987919,0.5818741918,0.6020925641,0.1391010433,0.8798360825,-0.3785371780,0.8962466121,-0.1627954245,-0.1554978937,-0.0491830073,1.0199846029,-0.5086528063,0.1819584221,-0.1886064410,-0.5281379819],[0.5540053844,1.2801107168,0.5145618320,-0.2815166414,-1.8504830599,-1.4968154430,0.5677479506,-0.8508284688,0.4678899050,-1.3412883282,-0.1465302259,0.2389729619,-0.6580345631,1.4656655788,-0.5119754076,0.3634024858,0.8378620148,0.6821438670,-0.1599557549,0.8418995738,-1.6232244968,1.2753105164,-1.5744374990,-0.3053387403,-0.7655331492,1.0510780811,0.8004363179,0.2560030520,-0.6166889668,0.0854177177,-1.9206666946,-0.5267802477,-0.7869284749,-0.4115960598,-1.1991744041,-0.5750448108,0.2946201861,-0.7443304062,1.4293489456,-0.4902796745,-0.0483880341,0.6640558243,0.8327726722,1.8407005072,0.7809294462,0.0030927989,-1.5531117916,0.9506762028,-1.3810019493,1.1338344812,1.1597759724,0.1388784498,1.6293661594,0.1739115119,0.2015552223,1.5030477047,-0.4344612062,1.8117796183,2.2257161140,1.4357246161,0.7157769799,-1.1085772514,0.3848017156,-0.8877408504,0.5003702044,-0.8792350292,0.4230170250,0.8809534311,2.0490174294,0.0204322394,-1.2171998024,0.7011641264,0.5943517089,-1.5732725859,-0.7199173570,0.7867372036,3.0451655388,0.1843346208,0.7179120183,-0.1636048704,1.5786978006,-0.7018404007,-0.4428640306,-0.0944580883,-0.4445796311,-0.9633267522,-1.4163391590,0.3170681596,-0.8742064238,0.0117640896,-0.9380337596,0.4167654812,1.0110545158,0.8198906779,1.4507848024,-0.4969290495,0.9055968523,-0.0989162475,0.2817564011,0.1054915190,-0.2824602127,-2.7734065056,-1.1041206121,-0.9647773504,1.5364704132,0.5445757508,0.7791507840,-0.0502580889,-0.9385029078,-0.9562464356,0.5013201833,-1.3731353283,0.6837669611,0.7879750133,0.4322114587,-1.0507004261,-0.4315394759,0.5607248545,-0.3958632052,0.4185329974,1.0252966881,-1.0072593689,0.8607437611,0.4936594069,0.7559320331,-1.0182080269,-0.1923506409,0.5055880547,-0.3494649231,0.4047941864,-0.3316917419,0.1759418398,0.0013057090,1.2570570707,-0.2191038728,0.9023769498,0.7408381104,0.5125377774,0.9699680209,-1.3627812862,-1.5679252148,-0.0293262303,-1.8678886890,2.1712396145,-0.1571822166,-0.3564154208,0.0553524904,1.1599270105,1.2627038956,-1.3459423780,0.6730185747,-0.0750857741,1.0460783243,-0.4429310262,-0.3001503050,0.0078839790,-0.2567608058,0.9423267841,-1.5697605610,0.6863792539,0.9789968133,-0.0081998017,1.7019345760,0.2806222439,-1.4620214701,0.5297275186,1.1651798487,0.8243795037,0.8680918813,-1.7472949028,2.0696997643,-0.2578157783,-0.0249881465,0.6565919518,0.0272306334,-0.8412454724,0.2397221625,-1.7836779356,1.2204616070,0.5548753738,-0.4932431579,0.3310505152,1.0874546766,-0.1795441955,0.9982895851,-0.2980313897,2.0983877182,-0.7227540016,-0.2248633504,-0.1296203732,-0.9494423270,1.4522721767,-0.9315572381,1.8279565573,0.1612818539,0.2126344889,0.6816353798,1.1261162758,0.9266378284,-0.3534713686,0.9973428845,0.4953095913,0.8687413335,0.6141946316,1.4996018410,0.1254296452,-0.7379559278,-1.2608875036,0.3750228584,-0.6231452227,3.0864508152,0.4766899645,-1.1117894650,0.2051137537,-1.7827329636,2.1619787216,0.6303663850,0.5379341841,2.3428905010,0.1024986655,-1.9776982069,-1.9109508991,2.0136795044,0.5614098310],[-1.5266704559,0.1959593594,0.6133632660,1.1159074306,-0.5647493005,0.4011000693,1.7993389368,-0.0339073166,-0.2392009497,1.6570544243,0.7138476968,0.3991074860,-0.8774254918,-0.4240868092,1.0801563263,-0.1920657456,0.3216613829,0.2953280807,-0.4288747013,0.4682311714,-0.9422526956,-1.1518111229,0.7867501378,-2.3118257523,-0.9087553024,1.6632435322,1.0761674643,-0.2712795138,-0.8323598504,-2.2843811512,0.2200830430,0.6697045565,-0.9509277344,-0.8576626182,-1.9374886751,0.8887458444,0.5036352873,2.0676424503,1.0283118486,-0.2918054461,-0.8564800024,1.6496522427,-0.1889360398,-0.3224303722,-0.5871344805,-0.9989820719,-0.0417234078,0.4708833098,0.8459716439,0.3378955126,-1.3688849211,1.9070968628,0.6046031117,2.0323977470,-1.0468131304,0.7598228455,0.1940415055,1.2512792349,-0.3850702941,-0.1691117436,-0.8089761138,0.8820098639,0.2849083245,1.2971107960,-1.0834826231,-0.1189776212,-1.1106203794,-1.3206923008,0.7566649318,-1.9178347588,2.7865297794,-0.5657482147,0.2699114084,0.3090089858,0.6460570097,-2.5136375427,-0.1902826726,-1.3483183384,-0.7845107317,0.8349735141,0.8349094987,-0.9854658246,0.1150932983,-0.3015092909,0.4304955900,-2.6301472187,-2.3760235310,0.2100565583,-1.6220120192,-0.6598060131,0.0390680023,0.9663178325,-0.0312575251,0.3434973955,-0.5051045418,0.2603479624,-1.6800607443,-2.0438206196,-0.7828196287,0.8378771544,-1.0073713064,0.3594104946,-1.0887169838,-0.2854229510,-0.6326141953,-0.7492433786,0.1771231443,-2.0925412178,0.7221005559,1.0629680157,0.8875825405,0.9514683485,1.1898092031,-1.3159422874,0.3789256513,0.0052759089,0.6215171218,-0.1703885645,-0.8301671743,-1.4931890965,2.1389789581,0.0210836176,0.0997546017,0.3395195603,1.5730198622,-2.4545269012,1.4472402334,1.2666022778,0.2692795396,0.7309278250,-1.1675630808,0.1287209988,-0.2159086317,1.9478616714,0.6417044997,0.4954462051,0.2918036580,1.1044400930,-0.0565379076,1.2288534641,0.9528832436,0.0863463283,0.5586382151,2.1897523403,0.1289473176,-0.8775374293,0.5111732483,0.6179109216,-1.8111522198,0.0995398909,1.3215231895,-1.1074717045,-0.8939023018,0.2247423381,-1.0083754063,0.8858939409,-0.2505567372,-0.3213457167,0.8018403649,0.2690301239,-0.5970544815,-0.9925628901,-0.5494341850,0.1522569507,0.1563308090,0.6602344513,-0.0799589828,0.8824433684,1.1563723087,-0.3943536580,-0.8907910585,1.6997411251,-1.1539968252,0.2835911512,-0.9666313529,-0.9075029492,-0.3648546338,-1.1380269527,-1.6632107496,0.0931004658,1.5548486710,0.6206492782,0.0812515914,0.5045549870,-0.0324226506,0.9576783180,0.6753861904,0.0415677503,-0.8783605695,-0.5465931296,-0.5524756312,0.4131335020,-0.6216126680,0.9888315797,-0.3318295181,3.0655612946,1.0167248249,0.0809800029,1.0825935602,2.0115580559,0.3990780115,-0.1417630613,-0.7643346786,0.9107628465,-1.2469594479,0.2185842395,-0.5608953238,0.7581259012,-0.5780990720,-0.3210789263,-0.1193744466,-0.6779866219,-0.5087441802,-0.1424155980,1.1234807968,0.1186617762,-0.0176850781,2.0666704178,1.0098127127,1.3752470016,-1.2679327726,-1.1970074177,-1.0551103354,1.1164258718],[0.3721234500,-0.9255678058,-0.9442209005,0.4268756509,0.3172981739,-0.3741634488,-1.4125268459,-2.3680481911,-0.1069736257,-0.4030350745,-0.4043666124,0.1732717007,0.4453865588,0.0835408270,-0.3626154065,0.2124658823,0.7893753648,0.8026905060,-1.3991522789,-0.3200124800,0.9662565589,-0.9130623341,0.4093767703,-2.2999207973,-0.2824724913,0.8781957626,0.0921964645,2.3364260197,-0.3582146466,-0.2903281450,-1.2173844576,-0.5441349149,-0.8577818274,0.6779240966,0.0227339007,0.9655373096,-0.2647223771,-1.9813855886,-0.5777205229,-1.6076692343,0.3734553158,-0.2309042215,-0.0047361567,-0.1747259945,1.4980055094,2.3268282413,0.0851788521,-0.3865151405,0.6857342720,-1.9992141724,-0.5850980878,-0.8566595912,0.5763618946,-0.4204704463,-1.0309052467,-0.3967194259,1.0643119812,-1.3884699345,-0.2525475919,0.0347681716,-0.4270209372,1.8029408455,-1.7515459061,0.0540627949,1.4515020847,2.2537171841,0.7252607942,-0.3768343031,-0.4213645160,0.3329466283,0.4443221986,-0.1684675515,-0.5755594373,0.6468083262,0.6131626964,1.2769823074,-0.3528927565,-0.2351228446,-1.2301682234,0.4037817121,-1.3678727150,-1.0195753574,-1.1714248657,0.4959396422,0.7116174698,0.3401472270,-0.5646832585,1.3907834291,0.2785235345,-0.8563492298,0.3405274451,1.1643620729,1.2994824648,-2.8496544361,0.0588610545,1.9530397654,0.5605524778,-2.7157382965,0.9060288072,-0.6560640931,0.4947168231,1.2521569729,0.7251318097,0.4450671673,0.1428324431,0.5182865262,-2.0029828548,0.4784449339,-0.3109569848,1.4094113111,-0.4395599067,0.0288746692,0.5327027440,0.9105619788,0.7118870020,0.0834584162,0.6378134489,0.4777068794,0.6714622378,-0.2473972887,0.9334355593,-0.0149712618,-1.2888498306,0.3178930879,0.3878506124,-0.4473615289,-0.0200192370,-1.1375455856,0.2947454154,-1.6181722879,-1.4102656841,0.6471197009,0.0660516992,2.5253288746,-1.4385683537,1.1881508827,-0.5137376189,-0.3100413084,1.8618991375,0.2196703702,0.8764735460,-1.6034924984,0.9608685970,0.7578120828,0.1340373456,-1.2120584249,-1.0811951160,-0.5332429409,-1.2666810751,1.1997977495,-0.0147280395,-0.3411561847,0.0395528637,-1.0613685846,0.8180955648,1.3017721176,0.8937417865,-0.7969278693,1.0919103622,-0.6793391705,0.5171425939,1.9261372089,-1.4655749798,-0.9694035649,-0.2225221097,-1.3643192053,-0.1500839889,0.5843650103,0.7056804895,-1.0301253796,-1.0684210062,-1.0028663874,-1.5143984556,0.0788298324,-1.7967107296,-0.3639734983,-0.7513176203,0.1762141883,1.1550230980,-0.4082490206,0.9812905192,0.4297397435,-1.0552676916,0.4100736082,0.1593143344,-1.5990869999,0.9750310779,-1.0146784782,-0.4160335660,1.6984248161,1.1609507799,0.7490220666,-0.8482598662,-0.6165418625,-0.0323930122,1.6698666811,0.1386055946,-1.1053417921,0.2234233469,-0.0349204689,0.5528994203,0.7125429511,0.0064929891,-1.0954294205,-0.4205610752,0.8501464128,1.0036913157,-1.4143588543,0.4902287722,-0.0818249434,0.1800193787,3.1680388451,-3.1549611092,-1.7138682604,-0.3571625054,1.7889220715,-1.3005938530,-1.1199634075,1.1873326302,-0.3849809766,-1.2680217028,1.4615029097,-0.6191694140,-0.2495485842],[0.2082553059,0.1791836023,0.3728811145,2.1315107346,1.3774552345,-1.4320223331,-0.3597615361,-0.1586332172,1.0218647718,-1.1757861376,0.7682100534,-0.3420245349,0.7694000602,-0.1330372542,1.1873819828,-0.3006245792,0.7143628597,-0.2423644811,-1.5537725687,0.3462584019,0.4230340719,1.0040265322,1.0997759104,0.7828722596,0.4102944136,-1.1098705530,-1.0867903233,0.0346978270,1.1880047321,-0.5926411748,-0.5738988519,-0.4091166556,-0.8370656371,1.0624153614,-0.9222751856,-0.2183016688,0.8524411917,-0.2983358800,1.0201835632,0.5510687828,-0.6143435240,0.0868408009,0.6313892007,-0.6356560588,0.9379420280,-0.9612347484,0.0437515527,0.1527118981,0.0146253034,0.1002622694,1.3769297600,1.8511936665,-0.9757603407,-0.1710399836,-0.3755131066,0.6657829881,-1.4439826012,0.8646513224,-0.5456988215,-0.3876164854,1.1869660616,-1.0903005600,0.2324468642,0.4489884079,0.2146653384,-0.3496420681,0.2460367978,-0.3882636726,-3.5028512478,-0.8601097465,-0.3408805728,1.0436613560,-0.1229604706,-1.1517763138,0.6286780238,-0.5831050873,0.2989288568,-0.8618805408,-1.0957497358,-1.6585500240,0.4104961157,0.3074675798,1.7889269590,1.4891097546,-2.1270682812,-1.0630022287,-1.0487341881,0.6423425674,0.2223631740,-0.6816553473,1.4914273024,0.8645065427,0.4738709927,1.0744345188,-0.0626458526,-0.1355794221,-0.5250419378,-0.5246269703,2.2068521976,0.8553270698,-0.0701120049,0.0589456148,-1.3844010830,0.1336048990,1.4933985472,-0.7632629871,0.0417235643,0.0002870396,-0.5796526670,0.8333060741,1.2923172712,2.1858572960,-0.3396995068,0.5367954373,0.9259405136,-0.2317183018,0.5833922625,-1.8033591509,1.9753354788,-2.3160240650,-0.5039364696,1.1172219515,-0.5671532750,-0.4029072225,1.2319533825,-1.3085190058,0.4285664260,2.4368367195,-0.8638158441,1.3917367458,-0.4627754688,-0.0249045976,-0.6280714273,-0.3255177736,0.6238153577,2.4476258755,0.3078250289,0.2715431154,-1.6325097084,-1.1116009951,1.1871554852,1.6313469410,-0.1071722433,1.0070543289,-2.4956753254,0.1061718240,0.0120313754,-0.2095249593,0.3418090045,-1.2988194227,0.9380907416,-0.1236676201,-0.4573471248,-0.7191598415,0.8948004246,0.4774148464,-1.7359133959,0.6295673251,1.4042685032,0.4560325742,0.2498383373,-0.7552278042,-0.2820581496,-0.9741064906,-0.0094816862,0.4927726090,-1.5904263258,2.0433969498,-0.2484231889,-0.3923215866,-1.2565356493,0.4341219962,-0.2114793807,0.0118124252,0.1367721707,1.8070771694,-0.6738863587,-0.7573686242,-2.5797410011,-0.0804657713,-0.4304643869,0.3704101741,0.0622019991,-0.3475724757,-2.1042618752,0.5381350517,-0.7691090703,0.1761910468,0.1809306294,-0.3777577877,-0.9449252486,0.2289055586,0.2820708156,0.4839225709,-1.6858013868,-0.8856335282,0.9421400428,-0.0791686252,-1.6161797047,-1.0410964489,0.6739453673,0.0878255740,2.3287050724,0.7083165050,-1.3314689398,-0.5426526666,-0.1750532389,-2.0199997425,-1.9016313553,-2.2880182266,-0.6802806854,-0.5845252872,-0.3345496356,-1.3252086639,-1.4987232685,-0.5198655128,-0.8556139469,0.7470584512,0.7125549316,0.0654351264,-0.6348320246,0.7843273282,-0.4739659429,0.2645100951],[0.7582881451,-1.4469571114,-0.8762317300,-0.1305171847,-1.8916240931,-0.2047363669,-1.4006288052,-1.4427716732,0.4075613022,-1.5529645681,0.4010265768,-1.2205121517,-0.5830441713,0.7983804941,1.3810188770,1.2851274014,1.6508965492,0.8096008301,-0.7591167688,0.7297912836,-0.7162643671,1.0841537714,-1.2955800295,-0.7949441671,-1.1049574614,-1.8323066235,0.4911190569,-0.0560565628,-0.7176681757,-1.8872069120,0.5780738592,-0.8776240349,1.1596308947,-1.4674416780,0.7553941607,0.1954972744,-0.5818178654,0.2693494856,-1.0041868687,0.6039539576,-0.6923959255,1.9953961372,0.1902663261,-0.0727277547,1.5628709793,-1.3869262934,-1.2224737406,-0.9052775502,1.8306998014,0.1413167864,1.7664381266,0.7349786758,-0.1183590814,0.8745171428,-0.1398411989,0.2598904371,0.0656923652,0.8621975183,0.8937485218,0.2206461132,-1.2316112518,-0.7846273184,0.7935965657,-0.7879238129,-0.7651246786,-0.1138043851,0.6265539527,-0.1829637885,-1.1814017296,0.0661041215,0.9544205070,0.3241200149,0.1085844710,-1.2307775021,0.3755576015,-0.3521512449,1.3882151842,0.5725190043,-0.0776720494,-1.3191725016,-0.4986121058,0.3813388348,1.4137277603,-1.5064461231,0.0442575030,-0.5795447826,-0.6121605039,0.7674696445,-0.3727608919,-0.1838898957,1.0338747501,1.9497634172,1.0179313421,-1.1559066772,-0.3884924650,-1.3412394524,-0.8717449903,0.7448266745,0.4458397627,1.6077286005,1.6052932739,-0.5146309733,-0.3278999627,0.7633868456,-0.8159669042,1.4002127647,0.1569492966,1.9102462530,-0.1955651641,-0.6491927505,0.2193007469,-0.4692718685,-2.2204196453,-0.3630209267,0.2141387612,1.0269374847,0.7454722524,0.3479391336,-0.2595340014,-0.0004230243,-0.5843359828,1.0068081617,0.5920389295,-1.4571993351,2.0111148357,1.6898576021,0.1999664903,-1.2243306637,-1.9395003319,0.7388504744,1.3946845531,-0.8135850430,-0.8188219666,-0.1027977988,1.2393451929,0.3642773926,-1.1666374207,-0.5038280487,0.1114103869,-1.5347028971,-0.5841313601,-0.8017553091,-0.1922202855,-0.3450426459,0.3865180016,-1.2264298201,0.5880392790,0.2358413041,0.9229251146,-2.5078446865,0.4116154313,0.6169677973,0.3932493329,0.7349784970,-0.1033508405,0.5472589135,-0.8508823514,0.4239557981,1.7755675316,0.0498641804,0.0039831121,-1.1012924910,-0.0702986047,0.0500299931,0.9911147356,-2.1317226887,0.0189382583,-1.2867163420,0.5649483800,0.5770050883,0.6580603123,1.4723309278,0.0125505729,0.7892917395,0.5260727406,0.7284713984,0.9198905230,0.4369905293,0.1269310713,-2.4369902611,0.2138117403,0.6558313370,0.1240368336,-0.6512229443,0.7850050926,0.8441942930,-2.6291081905,0.0859598890,-0.8896072507,-0.6072485447,0.5743405223,1.1331851482,-0.4991896152,-0.6477818489,-0.5778541565,0.0418310277,0.4881654680,-0.9791178107,0.1232551485,-0.8317633867,-0.4315126240,1.2505896091,-1.6251081228,0.3261093497,0.7886345387,0.2484288216,-0.5134832263,-1.5124263763,-1.6093171835,0.5770623088,0.5730774999,0.4694595337,-0.1032461897,-1.6306973696,1.0383374691,0.3179350495,1.6510140896,-1.4021530151,-1.6068631411,-0.9987185001,-2.0106415749,2.4225859642,-1.4711263180,1.0438575745],[-0.6822944880,0.5343873501,-0.2162118554,0.4941727519,-0.0919690430,-0.0362560339,1.7026308775,-0.5808956623,0.2818481922,0.4702845514,-0.7051241398,0.4703357816,0.4183109701,-1.0918869972,-1.1447758675,0.1090682745,-1.0183391571,-0.0740790442,0.8214829564,0.2784597278,-1.2427382469,-0.8619667292,0.9212391376,-0.3883512020,0.0477222577,-0.8693955541,-0.1270150840,0.4906855226,0.0592403039,-0.2530156374,-1.9048664570,-0.0264112484,-0.5457171798,-0.0354791954,1.5265716314,-0.1019395888,0.7486965656,-1.5187047720,-1.0162277222,0.6233072877,0.4643627703,-0.8993694186,0.4188348353,-0.2046624869,0.0254781116,1.2895642519,0.6310182810,0.3446836472,-0.5590118766,-0.2556653917,-0.9896861911,-0.5274890065,0.2978168428,0.0847402886,1.4692370892,-1.3065133095,-0.6991641521,0.3440778255,-0.2718342543,-0.5695889592,0.0174058340,1.0984630585,0.1190534905,0.9798473120,1.8879725933,0.8673732877,-0.9161585569,0.4036022723,0.0035582194,-2.3451409340,-1.0702661276,-1.6849539280,0.5669997334,-0.0257244613,0.0361120626,-0.0581826456,1.0888381004,0.4999228418,0.5219503641,0.4908412993,-0.4648894370,1.2614723444,0.3910034001,2.0438618660,2.3622827530,-1.1208287477,-0.5126113296,-0.9887000322,-0.5191606879,-0.2261203378,-0.1252585500,0.0878077075,0.0369837433,-0.3490452766,1.1845042706,-0.2965066135,0.8207860589,0.1675992906,-0.9627301693,0.2979668677,-1.6763862371,-0.1788655072,-1.2550932169,0.7845742106,-0.0772865936,0.3855877817,-0.7745981216,-0.1232291982,0.7760265470,1.3818191290,-1.0101274252,-0.7251560092,-0.2056710720,0.0522477031,0.1851748675,-0.5892301798,-1.8814272881,-0.8631243706,-0.6360574365,-1.1729818583,-0.9179823995,0.9177124500,0.7739092112,1.4054862261,0.8030353189,0.4492469430,-0.8796300888,1.2875020504,-0.9119425416,-1.7323316336,-1.3451622725,-1.0565350056,0.0448045060,0.3449862301,-1.5712947845,0.5012857318,0.5056712031,0.2673935294,0.2346226573,0.8127566576,0.8522439599,1.3639987707,-1.5733017921,0.4449477792,1.0757219791,-2.1031193733,0.0343645252,1.1781619787,-1.4266718626,-0.0131833591,-0.2667540014,-0.0093084434,-0.5884740949,1.7538708448,1.1161477566,-0.1020128131,-0.2309615612,-1.2694759369,-0.5607418418,-0.4304545224,-0.3253737688,1.4808640480,0.7316930890,-0.2684265673,0.4855044782,0.3031046987,-0.8914096355,-1.4147173166,0.0380460210,-0.7299775481,-0.2678869963,-0.3101365864,0.0350842513,0.2402429581,0.7029187083,2.7874162197,-0.1721138507,-0.6986640096,1.1523491144,-1.3354657888,0.0191495139,1.1237381697,-0.3511058688,-0.4177630544,-0.9081257582,-0.2498989403,-0.1846230924,-1.6133767366,-0.8762238026,1.6221534014,0.2161289006,0.4476101398,0.2794209719,0.8718365431,-0.1944708526,-0.6252462268,-0.7593362927,-1.4583418369,-1.4333637953,-2.1012365818,0.4250461757,0.9667654634,0.2156297415,1.4574354887,-1.2711378336,0.6589086056,0.5638456345,1.1474407911,0.1375724077,-0.2267200947,-0.1479162425,-0.0163359381,1.4957658052,1.5470947027,-1.1362988949,-0.0993754491,0.0944555551,-0.0559778437,0.9665927291,-0.2777977884,0.9807844758,-1.7313894033,1.2457816601,-0.7532542348],[1.5628550053,-1.8272314072,1.1460046768,0.5990837812,0.6483633518,-0.8140592575,-2.5026910305,1.5262742043,-0.5325720310,0.1642222852,0.3184698522,0.9115745425,0.9058817625,1.2242089510,-0.7914894819,-0.5712547898,-0.2330745012,0.7534739971,-1.3289783001,0.2245698124,1.4651367664,-0.6622340083,-1.4486229420,0.4370337129,-0.4643350840,0.5815520883,-2.0180966854,0.6800788641,1.9025919437,0.4159368277,0.3808253109,-0.3558321893,0.2614868283,1.4179209471,0.3718845248,0.9492228627,0.1516134590,0.3136004806,0.0753630325,-0.8924691677,-1.5429160595,0.2965213656,-0.5717129707,-0.0413963459,-0.9464138150,-1.2638531923,-1.4999424219,0.4140037298,-1.3621689081,-0.1094720140,0.1787716299,-0.3219559491,-1.4167441130,0.5873966813,1.1247311831,-1.4185732603,-0.6824306846,0.3787460029,-0.2558441162,1.1735316515,1.0506619215,0.3373606205,-1.7914761305,1.0255435705,-0.4845902026,1.3263299465,-1.1724784374,0.5220450163,0.8054077625,0.0096950140,0.8125115037,0.3299151063,-1.0037217140,0.3730991185,-0.9277126789,1.4822149277,1.5176270008,0.0745661706,-0.9604218006,-0.1578906775,0.2494293302,-0.9300476909,1.5165371895,-0.5894943476,-0.1630633473,0.0795842856,0.3101577163,-1.2102751732,2.5860579014,-0.9084997177,-0.9838130474,-0.4032730162,0.4376574755,-1.1266232729,1.2498075962,0.5042888522,0.2260986120,0.6913132668,-1.5524065495,1.0065914392,1.2285039425,-0.3059667349,0.6399753690,0.1240009218,-0.5298648477,0.7409871221,0.6777750254,1.4533698559,0.7625650167,0.1595736295,-0.1319786757,0.8643478751,1.0831538439,0.0846288800,2.1185903549,0.4222730100,-0.5730257630,0.8299391866,-0.3804766834,1.1045529842,-0.5073445439,0.1418423355,1.2586370707,0.8240965009,-0.0925577134,1.7273045778,-0.3900260031,-0.0798248723,0.8808295131,1.2586567402,0.9773184061,0.9998583198,-0.2488287389,0.2702918947,1.4333872795,0.5639662147,-1.0274194479,1.8915591240,-0.9012168646,-0.3748772740,1.9961894751,0.3488658071,0.2414206564,-1.1801184416,0.8474820256,0.2638389170,0.4284295440,-0.4961136580,0.5766240954,-0.8417623043,0.4841005206,-1.2339916229,-0.7195590138,0.6247155666,0.2961598039,0.1626374722,-0.1993692070,0.4295589626,0.1204819679,0.2459912151,2.3836979866,0.9892594814,1.5916163921,0.0971853957,-0.8566622734,-0.2674368918,0.5222064853,0.9773113728,-0.3150422573,0.8756898046,-0.4676228464,0.2016017884,-0.7167613506,-1.4885647297,1.2574212551,-0.6553381085,-0.6623219252,-1.7356320620,0.2538641989,1.0851593018,0.5627361536,1.1075919867,-0.7186845541,-1.4439043999,-0.8579331040,0.5339972377,-1.0072786808,0.6900771260,-0.8863855004,-2.2532014847,-2.5111033916,-0.4095038176,0.2482353151,-1.2830238342,-0.8127713799,-0.0843408108,0.1111695021,-1.0864380598,-1.5028867722,-0.8787872195,-1.5936022997,-0.1871830970,-1.0399664640,0.0147543494,0.7644811869,0.9782803655,0.0003450581,-0.2709909379,-0.1951631755,0.3221687376,0.3713282347,-0.4911470413,0.7329746485,0.0925365835,0.2304435521,-0.9668128490,0.4846392572,-1.1093183756,-1.3233888149,-0.0245773736,-0.0953248069,0.1685880274,-0.0482572317,0.5383268595],[1.0802992582,0.2973013222,0.9245066047,-1.3101730347,-0.4364936352,-0.4648250341,-1.1351803541,-0.8205749989,-0.8568386436,2.3754029274,-0.9961820841,0.8363527060,0.2676135302,0.8208408356,-0.9117810726,-1.0280570984,-0.9798866510,-1.0432735682,-0.6724710464,-1.4416898489,0.7651339769,0.5950027108,-0.1433894634,0.6269071102,0.8044335842,1.0749381781,2.0385212898,-0.1334894150,-1.8284147978,2.0971255302,0.5444354415,-0.7383652925,-1.5236394405,-0.5464919806,-1.5434194803,0.3095881343,-1.1199891567,2.2750618458,-0.4669438601,-1.8398299217,-0.1546743661,0.0634817109,-0.4727517366,1.8044909239,1.3376697302,1.1377027035,-0.3198766410,1.7948639393,-0.2277033031,-0.3369950950,-0.5130826831,1.2988376617,-0.7731146216,-1.0604265928,-0.2253106982,0.1074710190,-1.8205831051,0.7368350029,0.7032628059,0.9893723726,0.6483827829,1.4742196798,0.0884756595,0.4978652000,1.0682324171,0.6970139146,0.8740690351,-0.9838154316,0.2248925716,-0.5559855700,0.4227550626,1.1992962360,2.3643376827,0.8882084489,-0.5913435221,-0.7380414009,-1.0349892378,0.9173351526,-0.5621873140,-0.0949840769,-0.1317703575,0.0631912276,0.9709743857,0.3090033233,0.9268954396,-0.4655930102,0.2861961126,-0.2644225359,-0.3251528144,-0.8222166300,-0.0960599184,1.4646030664,1.4194988012,-0.4227886796,-1.8527027369,0.2715824246,-0.5044975877,-1.8221150637,0.1931934655,-1.5894869566,0.2379000485,-1.0019147396,-0.2313501835,0.0035350423,0.0740525126,-2.6869721413,1.5776410103,-0.5823276043,1.4910770655,-0.9430855513,-0.0499442630,-0.2272549123,-0.0936595276,-0.1898984909,0.5837596059,-1.1511113644,0.3232372105,-2.0862889290,-0.0450896136,0.1430051774,-0.3782043159,1.2295246124,1.5990449190,0.4077310264,1.0603466034,0.1066178828,0.9763474464,0.5174518824,-1.0154049397,0.1633393317,0.9158678651,-0.4462140799,2.7762815952,-1.7528024912,-0.8473525047,1.3816362619,-1.4776809216,-0.0661653429,-0.5095384717,0.1321265846,0.3022744358,1.0764505863,-0.8410828114,-3.0817403793,-0.0300313104,-0.0764164105,-0.2109280974,0.1390453428,0.4264347255,0.0728728324,0.2737209201,-0.4620195031,-0.0491712280,-0.3356860578,0.4319370985,-0.2530729175,-0.4461062849,-2.0072276592,0.9238764048,0.1594048589,0.0638244227,-1.3263802528,0.1089418977,-0.5390468836,-0.5481974483,-0.1197349951,1.0218429565,-0.7934098244,-2.4163577557,0.7732511759,-0.4387922585,0.6461163759,-1.3936929703,-0.6618784070,2.1668913364,1.0993745327,-0.1202072054,-0.4973158836,-0.6462906003,0.0609707199,-0.9450492859,-1.3187497854,-1.0022757053,-0.1960598081,0.7324171662,-0.4829072952,0.4753527939,-1.4519374371,0.8318330646,1.3754912615,-1.1183576584,1.5637904406,0.3519226611,-0.4841500819,0.3794345558,0.5975720286,0.6764720678,0.5199920535,-0.0438902751,2.0463252068,-0.5165079236,0.0998032242,-0.9830651283,-0.6394013762,1.1262912750,-0.8103187680,1.0153911114,-0.6205669045,0.3662223816,1.1925779581,0.8172106147,-1.0794445276,1.8628367186,0.0463979691,-1.1959922314,-0.0023014785,-0.3665894270,0.6613500118,-0.1001069471,-0.9814707041,0.3034345806,-0.7231702805,0.7263912559,0.8849028945],[-0.0591529347,-0.7766273022,-1.2518745661,-0.6941793561,0.5461867452,-1.4127789736,-1.1363685131,0.4137958586,-0.2157655805,-0.1393927634,2.9335629940,-1.3698664904,0.4863412976,1.7759577036,0.9367753863,-0.5121124983,0.5831712484,-1.5519332886,0.2913210690,-0.5899180174,0.0952703059,-1.8471230268,-0.0709084868,1.4827005863,1.4237951040,-0.5086781383,-1.6200858355,-0.1084599942,-0.3973819017,-0.7838751674,1.9469268322,0.4652900100,-0.6131108403,0.4059070349,0.2010302395,-0.1316693425,-0.0746570900,2.3412575722,-0.1811244637,-0.7504181862,-0.3903382123,-0.2243826389,0.9201111794,0.4877253771,-1.4580579996,0.8116889000,-0.5697486997,-0.4015645683,-0.4854067564,-0.6364825368,0.9034591317,0.6379844546,1.5771756172,0.5850499272,-0.6220704913,0.4983624816,0.4711814821,1.1179269552,-0.2089185417,0.5540845990,1.2248898745,-0.5977462530,-0.5361250043,-0.0362544581,0.8173483014,0.1797466129,0.6411272883,0.5963133574,-3.6270735264,0.3135236800,1.7770348787,-0.5708009601,1.2956660986,2.3631596565,0.6470772624,-0.1276730001,-0.4334301949,1.7504278421,-0.7313968539,1.3321459293,-1.1168040037,0.0644597635,0.6098915339,-0.8870542049,-1.1227594614,-0.1024430543,1.0980104208,-0.1197945774,-0.7725059986,-0.2856721580,-1.4923102856,0.2849843204,1.2012387514,-2.6904499531,0.1604321897,0.0795088485,-0.9636630416,0.4146036506,-0.2131204307,-0.5044959784,0.3896358311,0.4751556814,0.2694335878,-1.3584232330,-1.5450501442,0.2078277916,-1.4040329456,-0.9148842096,-0.5544590950,-1.9398692846,-0.7526990175,-0.2037417889,-1.1072591543,1.9583116770,0.4932885468,0.2148445249,0.7477450967,1.5735576153,-0.0146483416,-0.7617738247,0.5999017954,0.7779759765,0.5772922635,0.2835288644,-1.2251069546,0.0834239051,0.7457368970,0.5876018405,1.0709481239,0.5153665543,0.7868348956,0.9774563313,0.1690707654,0.4484441876,-1.3282804489,-1.5179308653,0.4773723185,-0.2764462531,1.5602777004,0.3877660036,-0.3111630082,0.9336119294,0.9268003702,1.2970597744,0.4492572844,-0.5420606732,2.1234147549,0.4513804018,-0.7160360217,-0.3417129517,-0.3907018900,-0.0474371351,1.1380760670,0.4578434527,-0.2161199600,1.1832199097,-0.5278459191,-0.7826381326,-0.9901517034,-0.2069672644,0.7864428163,0.3632103205,-0.3682445586,-0.4018748999,1.0661147833,1.0399434566,1.4891061783,-1.6929762363,-1.0862518549,-1.6203837395,0.1468081474,-0.3604899645,0.8857752681,0.4906992018,0.0710786209,-1.1928530931,-0.1335477829,0.3187680244,0.0093177333,-0.9354539514,0.2795247138,-0.3294781744,0.4063001871,-0.9572349191,-0.8981993794,0.3415729702,0.8799454570,0.6367980838,0.7841370106,1.1163245440,-1.9475696087,-0.4421205521,2.8911242485,-0.8780661821,-1.0232143402,1.6205269098,-0.7261584401,0.0078559443,0.6348084211,1.1820709705,-0.1163733304,0.8389843702,1.2746939659,-0.1599865109,-0.9582620859,1.6713591814,0.6692322493,-0.6660379171,1.0564323664,0.9545536041,1.2530609369,-0.2479295433,1.6680572033,-0.4401457906,0.7781750560,-0.3038768470,-0.4356323183,0.6398789883,-1.5581569672,-2.0755450726,0.6866158843,1.3195549250,-0.3794221878,0.5103654861],[1.9730210304,-0.7375587821,0.6860241890,-0.9076411724,-0.6199430823,-0.9175467491,-1.3471761942,-0.9156367779,-1.3426939249,-2.0646798611,-0.6329957247,-0.0766262189,-0.5408920050,0.9520366192,-0.8571756482,-1.1599135399,-0.5584220886,1.6461673975,0.1045306548,0.2158393413,1.7292610407,1.2974575758,-1.3764995337,-0.4452150762,-0.1526507884,-0.9902333617,1.5345755816,-0.1589398384,0.9030612707,2.0708162785,-0.9861791730,0.2067744881,-1.9495503902,-0.5985112190,0.9967989922,-0.8435174227,-0.2820928097,0.3777058125,-0.3613773882,-0.5519063473,-0.6273316145,-0.4378284812,0.2569901943,0.9104852080,0.0983464718,-0.2658705711,0.3255655766,-0.5762305856,-0.9602346420,0.2911734283,-0.5379109383,-0.9457253218,0.7973366976,-0.7568753958,-1.7424373627,-0.7140328884,0.7866174579,-0.0874564052,1.0776548386,0.0858798251,-1.2913337946,0.8008134961,-0.3137482703,0.5386188030,0.4172564447,-0.2212724090,1.0290312767,-0.6964595914,1.6124269962,-0.5174486041,-0.6736907959,-0.3899573386,0.1325595826,-1.0444660187,-0.2226642817,0.7306677103,0.5076549053,0.9035003185,-0.4636362791,-0.5478364825,-0.5060825348,-2.3333342075,-0.9416838288,-0.0387029424,-0.1578006744,-0.6142289639,0.8859317303,-0.8811742663,-0.6772906780,-1.1823346615,-1.3328503370,-0.0093972646,0.5491439700,-0.6104214191,-1.2850006819,-0.6465294361,0.1817125827,-1.4026774168,-0.1775815934,-1.2878239155,-0.4031934738,-0.8606334329,-0.7982609272,-0.6417425871,-0.6398243308,-1.3228656054,1.2034947872,0.8919556141,-1.8049753904,0.4476211667,0.2820806801,1.8818633556,-0.9953379631,-0.3909469247,1.3604015112,2.3987371922,0.3861819506,1.0936772823,-1.1391612291,0.8738053441,2.6254327297,-0.1070521250,-0.4421439469,1.9917935133,-0.3215175569,0.6542347670,-1.0363366604,0.5616360903,-0.9750161171,0.5296633244,0.5032150149,-0.8450386524,0.2101103216,-0.8270325661,0.2830041051,-0.9392392635,0.8031272888,-0.8688564897,2.0318152905,1.5412871838,0.3922415972,0.3853695095,-0.1013855189,-1.2406502962,-0.8898990750,1.6640088558,1.4869363308,0.0144009460,-0.0533166006,0.1196023002,-0.0499731563,0.0728715211,1.6635298729,0.1403089315,0.2601655424,-1.2603138685,0.5634941459,0.5055314898,0.7348325253,-0.4419586062,-0.1415433139,0.7567729950,0.9354738593,-0.0609624647,-0.3537159264,0.8630941510,0.5925130248,0.1479303986,-0.5582378507,0.8495017290,-0.1681861877,-0.8794690967,1.5341011286,-0.9373859167,0.3684532642,0.2468343675,0.5768260956,0.1956495643,1.7508788109,-0.2407533079,0.0557939187,-0.3764891624,-1.4740674496,1.0650231838,-0.6109123230,-0.8943587542,-0.2481622696,0.5509840250,0.1316975355,-1.0856409073,-0.5210722685,-0.4651280046,0.0003967007,-0.9472005963,0.3264623880,0.0406349227,0.6955839992,0.6747277975,-0.8340126276,-0.6404592395,-0.4645507634,-0.5293129683,0.1810609400,0.5685809851,1.0409775972,1.8969447613,1.4060031176,-0.9676909447,-1.1406483650,-0.8645566106,-0.9631246328,-0.4495841265,-0.3670941293,0.1041967198,0.9390909672,-1.9684011936,0.2957823277,0.0055266228,1.9068297148,0.2354781479,0.0417030305,-1.7206138372,-1.1816716194,-1.6864529848],[-0.2795744240,-0.4381642938,-1.3464711905,-0.2227810323,0.4033103883,0.2749841511,0.2874121964,-0.0273968987,-1.6804682016,-0.8865109682,-0.2472569197,-1.1258128881,0.9342336655,-0.5807120204,-0.1418465823,0.0790987611,-1.0749927759,2.2134726048,-1.2006694078,0.2691707015,-0.2489442676,-0.8762915134,1.6632809639,-1.0335812569,0.8721647859,-0.1475594044,-1.2235310078,0.5650429726,0.2195928097,0.0491285175,-1.0111253262,-0.8699862957,0.1976990700,0.3421122730,0.4036723971,-0.0111169284,0.1301707774,-1.9352202415,-0.3411969543,1.5464907885,-0.8455371261,-0.2012025714,0.5543738008,-0.3549480140,0.9601488113,0.5847229362,-0.9601101279,0.1428328454,0.7909606099,-0.2072594315,0.5749731660,-1.2849420309,1.2086113691,0.4027648866,-0.9209063649,1.3428764343,-0.0497116633,1.1252496243,-1.4034109116,0.0854413658,0.5364454985,-0.4646377563,-0.2525386512,1.5714304447,-0.3075014353,0.7793367505,1.7497104406,0.1035423651,0.4156560600,-0.7896444201,1.5372807980,-1.0925191641,-0.3885779679,0.7863638997,-3.7716197968,0.3558905423,0.2657609880,-0.0128362346,0.2688941061,-0.3216839731,-0.8197370768,-0.4061123729,0.0722145662,0.6458423734,-0.5728822947,1.5331975222,-2.2644534111,-0.7149686217,0.7342334986,0.0273310989,0.4219648838,-1.8402411938,-0.5089277029,0.5984560847,0.6699381471,0.4165192246,-0.5552244186,1.8353990316,-0.6733164191,0.3321333230,-1.0508931875,-2.3420100212,-1.1035705805,0.2500540316,1.2323869467,0.9139543176,0.7953773141,-0.2368800640,-0.8238989115,0.9296641350,0.1124773398,-0.4949807525,2.1422438622,0.3602790236,-1.2223575115,-0.5173635483,0.2391503006,-1.1317293644,-0.8869965672,-1.9696947336,-0.8611670136,-0.4780994356,0.1703852862,0.0392747857,-0.8513867259,0.5941985250,0.7758460641,-0.2394779474,0.5881170034,-0.3526118100,-0.9439957142,1.4639183283,0.8294926882,0.2655101418,0.2055927962,1.5004353523,0.5081455708,-0.7707537413,1.5194914341,0.9345728159,-1.1830033064,-0.7755748034,-2.1746835709,-0.1002500951,-0.9036408067,1.1387488842,-0.0545942672,0.7622455359,-0.3603307605,-1.1414390802,-2.1771762371,-0.3917343915,0.9779875278,0.0251153242,-2.3134996891,-1.2564883232,0.5975403786,-2.0236263275,0.4483772218,0.4159247279,-0.9448549747,1.5468273163,0.4482175410,-0.6612976193,-0.4731059968,-0.4449138641,-1.9738931656,0.7042369246,-0.0061952700,-0.4430695176,0.7511484027,-0.6052109003,-0.7535775304,0.2447089404,-1.4141345024,-0.6515182853,1.1660512686,-1.8217883110,-0.6141258478,1.2125263214,-0.0212147497,-0.6908379197,-0.2760960758,-0.3090609014,0.0992453471,0.3898756504,0.6739951372,-0.2484564334,1.1012392044,-0.4972129166,0.4972700775,0.1244544089,0.8972615004,-0.9175131917,0.8274303079,-0.4539879858,-0.2072115093,0.5159810781,0.9847689867,1.0576518774,0.4988316596,2.1206722260,-0.2258833945,-1.7573635578,-0.6614010930,-0.5449556112,-0.3869622052,-0.1445140392,-0.1908085942,-0.1185690537,1.7395980358,2.7544355392,0.3639674187,-0.6550902724,0.3023473620,-0.2460299134,0.0533388853,0.6053739190,-1.3006967306,-0.2727494538,-0.8468128443,0.9766883850,-0.6045349240,0.2835266888],[0.2119962424,2.5724422932,-0.1248733848,-1.5877096653,0.4133778512,1.3906018734,0.3611457348,-1.4941810369,0.9097679257,-0.7895452976,-0.0131430048,0.4787768126,-1.4094642401,-0.7671945095,0.4627577662,-0.2127619982,-0.5586658716,-0.0810662135,-0.6329986453,-1.5000852346,0.1569671482,0.4610330462,0.5216055512,1.0951989889,-0.6010218263,0.5522876978,0.4321692884,0.6307811141,2.0530083179,-0.5217092633,-1.4897885323,0.6250126958,-1.5613023043,-0.0225469321,0.3653716445,-0.1923164874,0.4022558033,-0.3239069879,-1.1409021616,0.2954473197,-3.0193588734,-0.4682822824,-1.4773166180,-0.9042945504,1.0454072952,-1.1082838774,-1.0444512367,2.1708672047,-0.4832859039,-0.6777984500,1.5595563650,-0.2915486395,0.0221522450,1.6878607273,0.9222207069,0.5000102520,1.1645658016,-2.7195198536,-1.5850836039,-0.3821529746,1.1708577871,0.5796393752,0.0682036281,1.1589673758,1.6575613022,-0.7662689090,-1.2816127539,0.1619419008,-1.3800725937,1.1070289612,0.9855763316,-0.9594042301,-0.8101201653,1.4170614481,1.6217786074,2.5478603840,-0.9287158251,-0.8941593170,-0.7534239888,1.6682021618,0.3101405501,-1.0944006443,-0.0103884470,-0.6301780343,3.0968067646,1.1102194786,-0.8093426228,-2.0855097771,1.2799286842,-0.7997585535,0.6396434903,-0.7613869905,0.3158714771,0.2002303302,1.6275659800,0.6539255381,0.4313740432,0.8388326764,0.3235431612,-0.3483707905,-1.0462989807,-0.6488026977,-0.1168876439,-0.2802293897,-0.3582161069,0.7303908467,0.0247646552,-1.3823878765,-0.6700788140,-1.7218041420,-1.5610544682,-1.3730332851,0.0397199988,0.0339602269,-0.0213197358,1.0941541195,-0.2352717072,-1.6322890520,0.0799121186,-0.9556933045,0.9889248013,0.2835512459,-0.4392039478,-0.9414988756,0.4072564840,-0.9518223405,1.3833876848,-1.1340763569,-1.8409861326,0.5475594997,0.5772783160,0.0075433361,0.4877021611,0.7304438949,0.3374955654,-0.1783489287,0.5135381818,0.5862373710,0.4374061227,-1.2511309385,-1.0444641113,-1.2574287653,0.8642635942,0.8503614068,-2.4733905792,1.0161190033,-1.7650945187,-1.0526522398,0.9574536085,0.2752321362,0.3666337430,-0.3609675765,0.8975061178,-0.3459243774,-1.4031714201,0.3312491775,-0.6045495868,1.2702319622,1.0439270735,-0.9935119152,0.2147451937,1.1008656025,0.4299190640,-0.1372111440,0.6169565320,-1.5516846180,2.0646662712,0.9198255539,-0.3870903254,-0.2804978192,1.3506839275,1.4641412497,-0.8443898559,0.1116885245,0.3858242631,-1.1577074528,0.0645114258,-1.4962903261,0.5474752784,-0.3548987508,1.0024415255,0.5446375012,0.8305513859,2.6467790604,0.6494945884,0.0595578328,0.3597752154,0.2516870499,-0.1181238145,0.4422335625,-0.1153549701,0.0733081177,-0.2758054435,-0.1951264143,0.1095378771,0.3252094388,0.8930382729,1.0510355234,0.9716653228,0.1504226327,0.5727462769,0.2539176345,0.0424893796,-1.0405927896,-1.4690296650,-0.0724262744,-0.1799088567,0.0264645424,-0.1598746330,-1.4990167618,-0.2105566263,0.1332618743,0.7448723316,-0.9744883180,-2.1181311607,-0.7248430848,2.7197012901,-0.5189481974,0.6716082096,0.1522336751,1.0515916348,-0.3552295566,0.0526108779,0.9851053357],[1.0241189003,-0.5028194785,-0.8301829100,-0.3506364822,2.1013507843,-0.2975785732,-1.1915485859,-1.9310274124,0.0281611960,0.7278511524,0.0044026095,-1.3758076429,-1.2809020281,0.7183985710,0.2994307876,0.9146254063,-1.8804464340,-0.5685744286,-0.1684366763,0.6454023719,-0.1555209905,0.0017470572,-0.0461361818,-0.4751440883,1.6674766541,-0.2610110044,-1.3076088428,-1.0537695885,-0.1325166225,-1.6590558290,0.1996072829,-0.4648935199,1.6472010612,-0.7532877326,-1.2189908028,-1.2708369493,1.7304326296,0.9739592075,-1.6090298891,0.9269093871,-0.5141512752,0.3343603611,1.0514758825,-0.7026612163,-2.2228353024,1.6996566057,-0.0153976502,-1.1894229650,0.1029311270,0.9860951900,-0.0424149409,0.2471846044,-0.6132379174,0.1702657193,-0.9761342406,0.1023765877,0.9823054075,-0.2936163843,0.5247262120,0.2902828455,1.3442021608,-1.2850005627,0.8672465086,-1.5464787483,1.5290105343,-1.3002258539,-0.4517161846,-0.3036861420,0.4617317021,0.4716469944,0.1666602939,0.2115351707,-0.5178134441,0.9480272532,-0.0098326812,-0.2219161838,1.0748243332,0.0413280837,-0.3751921058,1.0054109097,1.0878173113,0.2635469437,0.8403769135,0.2879878879,-0.1068011448,-0.9250847697,0.2024889141,0.1903910190,0.7410366535,0.8781462908,-0.2524499595,-0.1668965071,0.7493336797,0.8561979532,-0.0669981241,-0.2866696715,0.0031576664,-0.3875811696,0.4032322168,-0.0186206941,-1.0397504568,0.5258228183,-2.0523731709,1.4011635780,-0.3832056522,0.2031720877,0.6009286046,-1.0024389029,0.9176901579,0.1213238463,0.4889194071,0.5418915749,0.6136344075,0.1356879771,-1.2291907072,-0.7746384144,-0.1509719342,-2.0120000839,-0.1013733819,-0.3087831140,0.5470914245,-1.7141972780,-0.2830403149,1.5329728127,-1.1248511076,-1.0335241556,-0.5896264315,-0.0678327307,3.0144801140,-0.1954812109,-0.8513020277,-0.8838554025,0.8399129510,0.4587032497,0.6862203479,0.4576050341,-0.9797825813,-0.9266083837,1.2219574451,0.1443065107,0.6248232722,0.2345218509,0.0487408191,0.1535460949,0.3320425153,-0.1576162130,-0.6590068340,-0.0338018723,0.4617294669,0.1008614600,-1.2741967440,-0.3311012089,1.3876305819,0.9555911422,-0.8060911894,-0.4946635365,0.9969357848,-0.1563356966,0.2311514914,0.8098935485,0.1144014150,0.3466364443,1.6067898273,-0.4078532457,0.8250998259,1.1099538803,-0.0130652459,-0.8130413890,-1.2216724157,-1.2805823088,-0.7013052702,-1.1525663137,0.5510089397,1.4384489059,-0.0168083627,-0.2640907764,0.5070714951,-0.6443662643,1.8311312199,0.6864961386,-0.1773331016,0.2223710716,-0.0704963654,-0.9840225577,-0.3356443942,1.0691854954,0.1201403961,0.5101134777,-0.8782663941,-0.5339717269,-0.9799844623,0.7834316492,0.7693622708,0.2057720423,-0.3445561528,-1.4284933805,-0.0017824746,0.9755434990,0.7316427827,0.0141161242,-0.0398816057,-1.2709821463,-1.5649716854,-0.3071387112,-0.5473391414,0.9616869688,1.5145448446,-0.7195694447,-0.3944427967,0.4071395993,0.1094340906,1.9397739172,0.0562009476,0.5139475465,-0.0046406253,0.0536796823,-0.8677092195,-0.7403584719,0.0448231101,1.3504319191,-0.3382182717,-1.2779160738,0.2713087797,-0.4566406012],[1.0716338158,0.2072720528,0.3301466703,-0.2964983284,0.5896486044,-0.4153626859,0.5670648217,1.1750820875,1.4873989820,-2.3126528263,-0.8691516519,0.4574732780,-0.8475249410,0.8546934128,-0.6666801572,0.1129124910,1.2397251129,-0.3301310539,-1.1247928143,0.1654882878,0.2989130020,-0.2470651269,-0.3026075363,0.1921841800,-0.1730351001,0.4377845526,-0.2193670720,0.6114021540,0.6966733932,0.3748733699,-1.4609225988,1.5907051563,0.9739543796,0.8248856068,-0.3926510513,1.5374946594,-1.1393166780,-0.2162578404,-0.7730450034,-0.4251903296,0.5946242213,-1.1560473442,-1.5628254414,1.1010065079,0.2154533267,0.7111494541,-0.0475142524,-0.0388706252,-1.2525222301,-0.5174888968,-0.5561483502,-1.1154675484,0.1784954369,0.8350765109,-0.4459244013,-2.4139649868,-0.8504996896,0.5401726961,1.9785127640,-0.6393229961,-1.3613717556,2.0546019077,0.0595806055,-0.4285869002,0.9154852033,-0.3232243061,1.5594203472,-1.1150901318,2.0952014923,0.5021405220,-0.7830901742,-0.7693719864,-0.1663081795,1.4347996712,0.2960758805,-0.0027330727,0.2682686448,0.0127709676,0.2424747497,0.4638241827,1.7889491320,1.1757079363,-1.0222588778,2.6014802456,1.5808367729,-0.2146649361,0.7575861812,-0.4495947361,-0.5908117294,0.5592653751,-1.3259441853,0.5948846340,0.4069502056,0.8398963213,-0.4906890988,0.6471970677,0.6398313046,1.6471980810,1.4133006334,0.8426824808,1.0106039047,1.0245624781,0.0944321230,1.4529879093,-0.6713413000,-1.5175800323,2.1336538792,0.7717633247,0.6927742362,-0.4853357077,1.3379818201,-1.6785788536,-1.0422717333,0.5675963759,1.6423405409,0.1245919019,0.9202539921,0.0586088188,0.4284663498,0.1830497682,-0.1064387262,2.5498642921,0.1319633871,-0.8298854828,0.2510634661,-0.8929324746,0.6934131384,0.8412130475,-0.3025356829,-1.3898062706,0.7125902176,-1.8900271654,-1.2866756916,0.3507175446,1.0069307089,0.3439614475,-0.4223397672,0.4167747498,-0.6913272738,1.3721678257,0.1986006349,-0.5031130910,1.3957043886,0.4102612138,0.7169515491,-0.6730329394,-1.6226317883,2.0858526230,0.0266461149,1.1714679003,-2.1153361797,0.0740644708,-0.6488339305,0.0858231783,-1.3241528273,-1.0434464216,-0.3803936243,-0.6672549844,-0.1567256898,-2.3133258820,0.1390058398,-0.4018515348,-0.4512442946,0.3724332154,-1.1800881624,-1.3332873583,0.7908762693,-1.3407357931,1.6218533516,0.1306812912,-0.6198000908,0.0341969095,0.6919595599,-0.1018244103,0.3843306005,1.2551382780,0.1839106530,1.3701967001,-1.9215074778,1.5717345476,-1.0818972588,-0.2024257630,0.0068036322,0.7846305370,-0.4962382913,3.0493257046,2.2000374794,1.3815038204,0.5984432697,-0.6466378570,0.1070459485,0.0651153326,-0.4481697977,0.4599678218,1.3020391464,-0.7581306696,-0.2754032910,-0.0371919386,-0.5418404341,0.7452659607,-1.1534259319,0.9381926060,0.6659347415,-1.1937292814,-1.5376166105,0.1310789287,0.2398359627,0.5549139977,0.2483216375,-0.6198633909,0.8630862236,-0.1393041015,1.0174250603,0.3143958449,0.0683760419,0.0315545090,1.3078721762,0.4492118359,-0.4071181715,0.5910757184,0.3741379380,0.2453961819,-0.8749790192,1.2522603273],[-0.3844586313,0.0375177786,-0.6383576989,0.7360663414,-2.3159997463,-1.3596284389,0.4820798934,0.7575388551,0.8861390352,1.0024284124,0.5143792033,-0.6864389181,-0.7257505059,1.7803571224,0.5731166601,0.8511525989,0.8942019343,0.3404513001,-1.7471883297,0.8651315570,0.4115097225,0.9011681676,1.6878750324,-1.6129397154,0.5108485222,-0.2522997558,0.7438052893,0.5033487082,-0.4613233209,0.4820011854,-0.4439652562,-1.4911876917,-0.3602609336,-1.5907628536,0.6927799582,-0.1477584839,-1.0334455967,-1.5674887896,0.4550902545,1.5604319572,-1.4422808886,0.5788357258,-0.2870239317,-0.4004207551,-0.8785344362,-1.4270888567,0.9402923584,0.8992320299,0.4155126214,-0.8877316117,1.7329494953,-1.5584703684,0.2766417265,0.6156333685,-1.3220993280,0.7813851833,-0.6368366480,-0.6957759261,-0.5995090008,-0.3846778572,-0.0360092148,0.2149695456,-1.0402828455,-0.8459938765,-0.9112716317,1.0995310545,-0.9367135763,-1.9902956486,-0.0227747150,-0.4758028686,0.9322866201,0.9273737073,0.8070508242,-1.7838457823,-0.6699023843,-1.5915911198,-0.2805326581,-1.4233908653,-1.4749735594,0.1594439596,0.0329884961,-0.8356202841,1.3274855614,1.0824410915,1.0488785505,-0.4469273984,-0.9992522001,-0.9932340384,0.5085969567,-0.5421534181,1.9483562708,1.1397675276,-0.0622630306,0.1688272506,-0.7671681643,-1.1403740644,-0.7531850934,0.3955881298,0.6842200756,-0.5735817552,-0.5361065269,-0.7205488682,-0.0343498290,0.4714860320,-0.1568353325,0.5368303061,0.7607414722,-0.7243655324,-1.3951404095,-0.8374814987,0.4954386055,-0.0606812388,-0.6688954830,-1.6458029747,0.6280921698,0.2495674193,1.5667924881,1.3704221249,-0.1882616132,-0.3503639698,0.6975366473,0.4451564252,-0.6488027573,-2.6514444351,-0.9436893463,-1.0788868666,-1.0912772417,0.0611912012,-1.4206846952,-0.6495169997,-1.2007901669,-1.3360172510,-0.0519311912,-1.2045214176,-1.1121664047,0.6280339956,-0.5048688650,-0.4457834363,-0.4472849965,1.1495735645,-0.5288956165,1.2007362843,-0.4974692166,-0.3109487891,-0.9244396687,-0.2693705857,-1.0468840599,-0.6721541286,0.5465496778,0.0884050056,-0.4098713398,-1.0514574051,1.4148094654,-0.4750121534,0.6167279482,0.9246222377,-0.4253928065,-0.6544781923,0.7891964912,0.3140968978,0.7648479939,-0.1695642024,-0.1479424685,-0.1064250320,-2.2116746902,2.3312244415,-0.6676529050,-0.9457675815,0.9474021792,0.7973690629,0.1704794019,-0.7035692334,0.7092891932,-1.8698717356,-0.2639527023,0.2181621939,-0.4548698962,1.5478996038,-0.4820409119,0.6898891330,0.6085611582,0.0193841830,1.8152153492,-1.6339917183,-1.3337707520,-0.5393002033,0.1599944085,0.4317499101,-1.7887134552,0.7140734196,-0.8148634434,-0.4388350248,0.6136690378,-0.8982139826,2.4423866272,-0.9562060833,-1.4921243191,-1.7531054020,-1.9354444742,0.1809691191,-0.9431200027,0.1708678603,-1.9138517380,-1.3267153502,-0.8237664700,-0.1747481823,0.8606752753,0.8452942967,-0.3969329000,-0.7858321071,-1.0196638107,0.1269257069,1.0621590614,1.5306878090,-1.5189861059,0.9269375205,1.2335311174,-0.4745801091,-0.6515474916,0.7936541438,-1.3056498766,0.2814712524,1.4409999847,0.8073496222],[-1.8721452951,0.1184589267,0.8932977319,-0.5317001343,1.8536045551,-2.7512083054,-1.3216940165,-1.3283517361,-1.1938714981,-1.2970254421,1.1642961502,0.3630233407,-1.1360250711,0.4634298086,0.2409296781,-0.9782558680,1.8368214369,-0.0928911120,1.2228311300,-0.4003975093,-0.3872050047,-0.2419685572,0.1414249092,-0.3762511015,0.1452577263,-0.2219312191,0.7660420537,-0.7883638740,-1.6409637928,1.1355031729,1.3309619427,-0.9560182691,0.4649753571,1.1633868217,-1.4728108644,0.2295930833,-0.1990584433,0.3435695469,0.8488633037,-0.3607235849,-1.8055906296,0.2481548339,0.7938757539,-1.8822200298,1.2305692434,0.9074109793,-0.3646234274,2.4204812050,-0.8100681305,-0.5929164290,0.8905769587,-0.7271680236,0.3828736246,1.0301723480,0.7428522706,1.0397125483,0.5743683577,0.9277793765,0.3855857253,-0.4105027914,0.5453033447,1.5967614651,-0.2897183895,-0.9443405867,0.9612666965,-0.2591313124,0.7493833899,0.9668287635,2.2179422379,1.4288818836,-0.3036374748,0.3314071596,0.1685507894,-1.0392718315,0.5757080317,1.8846974373,-1.0078357458,0.4577882290,1.0183068514,-0.7346784472,-0.4650122225,0.0640585124,-0.7890337110,0.7123513818,-2.1669125557,-0.7675073147,-0.9449701309,0.8574027419,0.7751913667,0.1898535192,1.6693571806,-0.3673186004,-0.0210592635,0.4653729498,-0.6247548461,0.1980665624,0.7378042340,-1.0381380320,0.8246774077,-0.1759186089,0.6200713515,0.3294046819,0.2967543006,-1.0276694298,0.0227232017,1.1291382313,-0.6281598210,0.8043770790,1.0822321177,-0.1657423079,-0.0776807144,-0.1167014837,0.3684995174,1.9841905832,0.4295887053,0.6405870318,0.6379342079,0.8725138903,0.3877270222,0.8504217863,-0.0968416929,-2.1198146343,1.4159415960,-0.3126500249,1.5506500006,1.0458365679,-0.3016704917,-0.2235870957,-1.2391153574,-0.5851184130,0.5509126782,0.4651997685,-0.4118579626,1.5482922792,-0.0040421048,-0.9912667274,0.0142603684,0.8469740152,0.1916214526,-0.0247817542,-0.0518606752,-0.7906127572,0.6237937212,-0.2116433233,1.8560798168,0.6758502126,0.4323855340,-0.5427485704,-0.2471507788,0.8266699910,-1.2941006422,0.0277775917,-0.0114485594,-0.1420166045,0.4924000800,0.4590295553,2.5640699863,0.3363261521,-1.5822482109,-1.6178342104,1.3771090508,1.1515468359,-0.9595029950,1.7391210794,0.6310431361,-0.8181252480,0.3020314276,-0.8029966950,0.6402264833,1.1647168398,-0.8939806223,0.1350935549,-1.0642356873,-0.5717391372,-0.6346552372,0.5374536514,1.3573206663,-0.3348610401,-1.6466457844,-1.3988031149,0.6637595892,-1.4271485806,0.4709821343,-1.4963316917,1.3809591532,0.3187876642,1.0921761990,-2.2258255482,-0.5045356750,-0.7421919107,-0.2157391161,0.7126739621,0.6691911817,-1.1976441145,-0.8853336573,0.8246211410,-0.9868354797,-0.0254206806,0.0362228155,-0.1663735956,-0.9848557711,0.7515419722,-0.7585852742,-1.0937254429,-0.1821856797,-0.6961805224,0.3334859610,0.4389159083,0.2469871491,-2.0076756477,0.4291914105,-1.2334452868,1.3440158367,0.0627860799,-0.9219018817,-1.2558528185,-1.9622758627,0.1995464712,0.4981423914,-0.5179405808,1.9443092346,-0.4232400060,0.0491162017,-1.3236367702],[-0.6165354848,0.7170464396,0.1112871245,-0.8130300641,0.2534252405,0.5796111226,1.9850246906,0.4960811436,-0.2429900765,-0.4833355546,0.6328579187,-0.6804760098,-0.1585241705,0.3408620358,-0.4281844199,-0.8441763520,-0.8560928702,-0.5648352504,0.0108333835,0.3450529873,0.7727462053,0.5534866452,1.6443219185,0.5270298719,0.2769190669,-1.0014778376,0.5765803456,-0.8796996474,0.5687882900,-0.0461753644,-1.2852069139,0.7221946120,-2.7615418434,0.0332008004,0.2197542191,-0.2995602190,-0.0586884245,1.8317363262,-0.6810992956,1.5084025860,0.3555968106,-0.2981703579,0.9358145595,1.9291380644,1.2278101444,-0.2416273504,-1.0936526060,-0.2135961056,1.5356572866,0.9269886017,-0.4677660167,0.5069100261,-0.5780176520,-0.2702128291,-0.5411428213,-0.3239991665,1.5921620131,-0.3274900615,0.0973749086,-0.1159433350,-0.7867313027,-1.3489117622,-0.6642484665,0.1416206062,-0.3015295565,0.7219636440,-0.5237560272,0.3400763571,-0.1233913451,-0.2009309828,0.8506157398,-2.1546499729,0.3547619581,-1.6757706404,-0.1415503621,-0.8680157065,-0.0768875107,-0.1592646539,-0.5014755130,0.0249998868,-1.8573825359,-0.3155896068,0.5905711651,1.0066084862,-0.5826127529,-1.1200389862,0.1239113286,-0.4321572483,0.2175883204,-0.1679378450,0.6261950731,0.5056493878,-3.1710517406,-0.1413709968,0.6767768264,0.3793789148,0.3022807539,0.7109302282,0.2650648355,-1.0345375538,-1.6468921900,1.0483371019,1.0237673521,0.3251764774,1.2741376162,0.2375988215,-0.7581788301,-0.5037139058,0.0572597869,-0.8450796008,-1.1608557701,-1.0911349058,-0.5320959091,0.9157405496,0.4893115759,1.6712793112,-0.5092890263,-0.9690325260,1.4766815901,0.7314090729,-2.4690501690,-1.3202705383,-0.3301222920,0.8962110281,2.0390863419,1.0254950523,0.1695734710,-0.2835012674,-1.0160237551,-0.2788228989,-0.0651427954,-0.3828991652,0.2328362316,-0.7724130750,0.5774948001,-1.8650261164,0.2756900489,-1.3820643425,-1.4922863245,-0.3723741472,-2.0753431320,-1.7942260504,-0.3470088840,1.7341159582,0.8438899517,1.0441874266,1.4977054596,1.0677449703,0.1652990133,-0.1096124351,0.1542658657,-0.0562523417,-0.0727530718,-1.1196274757,1.4136055708,-0.4164610803,-2.0809350014,0.0478956066,0.9895527959,-0.8603312373,-0.6730191708,-0.5802956820,0.7802674770,-0.9864706397,-1.9121707678,0.6803776622,0.1682149619,-0.7673743367,0.7056384087,-1.0421631336,-0.3231330812,-0.9202150106,-1.7474627495,-1.6890426874,-0.0504816063,-0.7599089146,0.6442566514,-1.4180889130,0.8134864569,0.3692963719,0.3944670260,0.1946570277,1.3176888227,1.3182153702,-1.4753251076,0.2395100147,-2.1885092258,1.0353777409,0.4226379693,2.4927759171,-0.7165281773,-1.2685643435,0.7180500031,-1.0596232414,-0.1318729967,1.2955181599,1.4145418406,0.3703507483,-0.2906166613,-2.1202604771,1.5874541998,-0.4480857253,0.1233151332,1.2725905180,-0.4341225624,0.8674485683,2.0624625683,0.5839276314,1.4544333220,0.3140803874,-1.1686724424,1.4655153751,-0.1854373813,0.0675033107,-0.1820190549,1.3975430727,0.3684431016,-0.5828213096,1.6327118874,-0.5973852277,0.6849465966,-0.5043992400,3.1433739662,1.0341066122],[0.1156917736,1.5954945087,2.6153404713,0.6946935058,1.7668703794,1.1358177662,-0.5453659892,0.3216398060,-0.3820880949,-0.7366890311,-0.5614402890,-0.6733281612,0.5156762600,-1.2313117981,0.5289177299,1.3301786184,-1.7023651600,-0.9341116548,-0.8916813135,-1.1652710438,-0.1437996477,0.9670381546,0.2372026592,2.4822874069,0.5470020175,0.4206858873,2.6281704903,-1.5163463354,-0.5004493594,1.2942957878,0.6173464060,0.2298103869,0.4356625080,0.3032996058,1.1061048508,-0.3733479083,-1.9930585623,1.1398425102,1.1698259115,-1.2241064310,0.7899841070,0.2383679450,-0.7305232286,-0.4423871934,0.7964922190,-0.4245458543,0.3769742846,-1.8120678663,0.5874359608,-0.0313255563,-0.7554463744,0.6911882162,-2.2524123192,0.5145077705,-0.4655631483,-0.5376864076,-1.6770755053,-0.6433816552,-0.0429741144,1.1280231476,1.1490014791,-0.0671586543,-1.2202082872,0.7045454979,-1.0823764801,0.1428896487,-2.9295396805,0.6375863552,-1.7725259066,0.6920745373,0.0506045334,-1.8579263687,1.0407875776,-0.1360074580,-0.8596337438,0.0067301448,-0.1954611391,-0.1238392219,-0.8143445253,-0.6136426926,-0.9348247051,0.4640007019,0.7521933913,0.6640645862,-0.1378400475,1.1553994417,1.0206215382,0.3986735940,-0.4069051743,-0.0785310194,0.4717287719,0.2707085907,0.1270379424,-0.4223307371,-0.3201220334,-0.3389673233,0.4850408137,0.4809367359,0.2361204922,-1.0482326746,1.0621023178,-0.1028745249,0.1858724207,0.3908754587,-0.6750885248,0.2174529731,-1.0515180826,0.2160267383,0.2982191145,0.8590397835,0.0086025409,0.5641648173,0.4481794834,-0.6979264021,0.3443371356,0.3175536394,1.2855532169,-0.7491867542,0.3338579834,0.7577729821,0.3670987785,0.2343304902,-1.9538394213,0.7163391709,0.3700796068,-1.7885614634,-0.9458172917,3.0109028816,1.4560543299,0.6147252321,0.6920373440,0.6372101307,-1.3865582943,-1.7684077024,0.5304525495,-0.0079795672,0.8801140785,0.9465383291,0.1581633389,-0.8426349163,0.9479221702,-0.1354119480,-1.5711832047,-0.3637903631,0.4112898111,0.1602426320,-0.1830785424,0.7868187428,0.4419188499,0.0504920371,-2.2241265774,-0.6185190678,-1.1892626286,0.6705932617,1.0291326046,3.0448400974,-0.8008592725,0.9581548572,1.3260705471,1.1762444973,-0.9842439294,2.0668766499,-0.3960740864,-0.1752684265,-0.7192801237,-1.4173226357,-0.5742908716,-1.1648813486,0.9344018698,-0.1113579199,1.6854794025,0.3993677199,-1.1008958817,-0.5278835297,-0.7492056489,0.7847382426,0.4598958194,1.8640118837,0.4370495081,-0.9409788847,-0.5925995708,1.2170940638,-0.4932170510,0.7076863647,1.2122285366,1.3748151064,-1.1437553167,0.1958882958,0.1235001981,0.0688811988,-2.3859999180,-0.5745288730,1.7441747189,-0.8688986301,0.6938255429,0.2857529819,0.0937574208,-0.6805610657,0.6712647080,-0.2787623405,-1.1043686867,-2.3983387947,-0.8090463281,-1.0024791956,-1.1571393013,0.3918244541,0.3093427420,1.1425877810,0.9271572828,0.7051991820,1.2124754190,-0.4664890766,0.0037588740,1.7412911654,-0.4158358276,-1.5634813309,-1.3250594139,-0.2688057721,0.2384223789,-0.5030126572,0.8553127050,0.4374273419,1.1957253218,0.0462666750],[-1.8070065975,-0.4504835904,-1.3686780930,-0.3058474660,-0.0495945141,0.1645372808,0.1849104613,-1.7716939449,2.3738782406,-0.1378646493,2.4643244743,0.5075989962,-0.1704922765,-0.0185646489,-0.0093221944,2.1728961468,0.9254801273,-0.5384533405,0.9051353931,0.3683229089,-0.8445315957,-0.3709745407,-0.9525077939,0.6499077082,-0.0273034014,-0.4088584185,2.1410124302,0.8542435765,0.9650903940,-0.6568892002,1.6100125313,0.7433479428,-1.6218360662,1.1497720480,-2.6727676392,0.3415644169,-0.4711919725,-0.4807176888,2.3651483059,-1.6804189682,-0.7778606415,-0.5134579539,0.8294714689,0.3388239145,0.5550401807,0.3536906242,0.8353669047,-1.1603729725,-0.9736599326,1.4669877291,1.6135033369,-0.1117185876,-0.3129193783,1.0819211006,-0.7731477618,1.1307908297,-0.6786795259,0.2116177529,-0.0737237260,-0.4597691596,-1.1927083731,-1.3991508484,-0.2123755366,-1.2098524570,1.4069837332,-0.2333389074,0.0752018467,1.4358216524,-1.5128277540,0.4061498940,-0.7061683536,0.2710811794,1.2111703157,-0.4321515858,-0.3260831237,0.0225785729,0.2212020308,1.4104518890,0.2452560216,-1.5462257862,-1.2980943918,-0.1110542938,1.1513741016,-0.3489472270,-0.8876929283,-1.0408627987,0.4383756220,-1.5169585943,0.4438357651,-0.3842916191,2.2116289139,0.2712109387,-1.6035920382,1.0682024956,1.2236803770,0.5410172343,-0.5111415982,-1.3180493116,0.4625207186,-0.0136173591,-0.5168929696,2.2955324650,-1.3852297068,0.6380922794,0.0593690909,1.6699051857,-1.6007295847,-0.3379013240,1.0723598003,-0.9798635244,0.8318992853,-0.8471778035,0.3142752051,0.5284886360,1.1218874454,1.8486001492,2.7345709801,0.4374895692,-1.2230619192,-0.2786168754,1.1966751814,0.7694658041,-0.4521544278,-0.4873460531,0.0760180429,-2.1347806454,0.9052285552,-0.5944761634,-0.3380056322,-0.1664792448,0.5691460371,-0.5731565952,-2.2000503540,0.0652889013,0.9432873130,0.3104563355,0.5494231582,-0.9663535953,-0.5229769349,-0.2445623577,0.5183826685,-0.9060575366,-0.9137417674,-0.1996527612,0.1335192323,-0.4433824420,0.3377778530,-1.6413617134,-1.7930675745,-0.1941461861,-1.8694553375,-0.9999119043,-1.4513466358,0.6723552942,0.1838610321,0.7941121459,-1.1660890579,-0.8116760254,-0.9434131980,-1.1249563694,-1.1655663252,0.6973057985,-0.6014246345,1.2728025913,0.1407245845,0.8612917066,-0.2324358374,1.4373520613,0.8628004193,1.5325776339,-0.1889394820,-0.5824972987,-0.9653490782,1.3446065187,-0.7162488699,-0.1897031218,2.0714435577,-0.6121030450,2.0374407768,-1.4945025444,-0.2567083538,-0.8972054720,0.1291142851,-0.3037604690,-0.8362293243,0.7068982720,-0.4486835897,0.2427803576,1.0932581425,-0.1559365541,0.7241747379,-0.7676721215,1.5684889555,0.7923648953,0.9502861500,-0.3227502108,-1.2624378204,0.2234718055,0.4752887785,0.2963661849,0.2820369303,-1.3083859682,-1.8737363815,-0.3256417215,-0.1815035492,0.3622895777,-0.2257685065,0.1857555956,0.0977337956,-1.1289507151,-0.1029930860,0.0768001899,-1.2031559944,-0.1173039898,0.0196279101,-0.2452402562,1.1445440054,-0.7530807257,-0.6296615005,-1.0584937334,0.7693899274,0.8202177286,-0.3264771104,1.0771063566],[-0.4935064316,-0.5727998614,0.1287579536,-0.9393902421,0.9558954239,0.7553883195,0.6227222681,-0.0084682750,0.2514541149,0.8778777122,1.0188686848,-0.2856886983,-1.8559024334,0.0303529315,0.1430030465,-0.3618414998,-0.0351615176,-0.6785556078,-0.9759765863,-0.4848660827,-0.2929255962,-0.1925849020,-0.7591896653,-0.3228958249,-0.0205045063,0.3483543694,0.2820397615,-1.6995092630,-0.7122277021,-1.3917955160,0.5476497412,-0.3970116079,-0.0744653568,-1.1424483061,1.3950189352,1.2138121128,0.7932989001,0.3546606302,-0.1634015590,1.4944585562,0.7755617499,0.7647002339,1.2824169397,0.3859403133,-0.1856594533,-0.9363436103,0.4315616786,-0.2700640857,0.3939462602,1.4889338017,1.0297901630,2.0167706013,-0.5965355635,-0.8565047979,-0.9398768544,0.2297771126,-0.5574417710,-0.1267592162,-2.3832242489,1.1377506256,-1.1425361633,-0.5246202350,-1.5346717834,0.1314383298,0.0684951022,1.1131851673,0.2731709778,0.6787238121,-0.5446536541,0.0450214595,0.8453890085,1.8095526695,-0.2790301442,1.1491837502,-0.4293262064,-1.3085041046,1.0832661390,1.3500005007,1.1210151911,0.5666023493,-0.4597205520,1.0817568302,0.7697666287,0.8304558992,-0.2385064662,0.8480877876,-0.7320706248,-0.5562512279,-1.0011099577,0.5937754512,-0.8616179228,0.8863510489,1.6711868048,1.0623165369,0.6543915868,-0.1773722172,-0.0344006717,0.5973542929,1.2686825991,-1.5581994057,-1.1329184771,-1.2591944933,-0.6952503920,2.6306145191,-2.2633259296,1.0291699171,0.9703497291,0.7907989621,-0.9672769904,-0.7324648499,0.5958577991,0.2923487723,0.0744326934,1.4750037193,-0.1998525858,-0.1739941984,-1.8254380226,0.3562199175,0.7012470961,-0.2482330352,0.4486927986,0.0901318491,0.7526713014,0.2792355716,0.5251368880,0.3370932341,0.6470039487,-1.2970519066,-1.4007022381,-0.9944904447,1.5340650082,0.7310197949,-1.6445587873,0.4814043939,-0.5434415936,-0.9484261274,-1.4867604971,-0.5797979832,-0.1667503268,-1.4086076021,0.9149457216,0.4772666395,-0.6579295397,-2.5258929729,-1.9978051186,-0.2341778129,1.2802121639,1.7618249655,-0.8002349138,0.3736089170,1.1083867550,0.1761967391,0.4046993256,0.9638820887,0.8057360649,0.8382278085,-0.0438889638,-0.2375431359,-2.0383751392,-1.0941091776,1.5616539717,-2.0328569412,-0.9788680673,-0.6005727053,-0.5285338163,2.7921683788,-0.7145404220,-0.8009266257,-0.2848567963,0.6346355081,-0.4794748127,1.5668632984,-2.4055385590,0.4379455149,0.5932794213,0.6854198575,0.9216139317,-0.3407145441,-1.3855520487,-0.7462887764,-0.3033876419,0.0186376907,-0.0690334588,-0.2632014751,1.6265926361,1.4949783087,-0.6602244377,-0.7783056498,-1.3226950169,1.5882486105,-0.0956622288,0.7664166093,-0.5125342011,-0.7737212777,0.6168346405,0.7540829182,1.1455770731,0.3315310776,-0.1345494241,-1.5078363419,1.8647091389,1.8537608385,-1.0891287327,-0.5876540542,-0.3828836977,0.5380443931,1.4859291315,0.1173877418,-0.0830619484,-0.2604106069,0.9005637169,0.1149029881,-0.8461684585,0.9646666050,-1.3376936913,2.1376085281,2.3744444847,-0.1123782471,-0.6790237427,-0.2712702751,-1.2458635569,-0.2799184620,-0.0585089289,1.2172613144],[-1.3013261557,0.4938486814,0.0018817389,0.1639826447,-0.1096468046,0.1895873547,0.6653810740,1.3794633150,0.3325508237,-0.3157938123,-0.4053605795,-1.2119607925,1.6195567846,1.0380227566,0.0044104871,1.2443182468,-0.5239819884,-3.1816990376,-1.6022835970,-1.0422313213,1.6971408129,-0.1422895491,-0.3386924267,0.0337778777,1.3895584345,-0.6798459888,-0.8735042214,1.4608218670,0.9804942608,-0.0700358227,-2.2575590611,-0.5207765102,-0.5970131755,1.4351145029,0.5342422128,-0.5326063633,-1.2744994164,-1.6573332548,0.1688603461,0.8684266806,-0.4371896982,1.9104814529,-0.1397882998,0.7860920429,0.3916041851,0.1504721791,-1.0214750767,0.2536700964,0.0443325303,0.4851707220,0.0727755502,-0.1179084405,1.6902691126,0.2719678283,-1.1939750910,-0.0545632914,-1.5358672142,-0.0898130834,1.1942884922,0.9992315173,1.0113880634,-1.4582569599,0.6580271721,-0.8810248375,0.7868432999,0.6703838706,-0.4072601795,0.8141466379,0.7604040504,0.2462893724,0.0050235200,-0.7474465966,0.6245143414,-1.0206460953,-0.8153722286,1.1893461943,0.4940040112,0.4967643023,-0.2958534658,1.8670926094,1.3553891182,-1.0535255671,-0.1981259435,-0.3524850905,-0.6183097363,-0.0779046416,0.7972301245,-0.5016496778,0.4480767846,-0.2101167887,-0.0168276727,-0.2014111727,-0.5612958074,1.3728827238,-0.3534809053,1.5735669136,0.9426801205,-0.0221692715,-0.9934756756,1.4621682167,-1.4462176561,-1.5750948191,-0.2356135994,-0.3192728162,1.4097740650,0.3186092973,-0.0515878908,0.4262378216,-0.0413998328,-0.7294147015,0.5786992311,0.5278693438,-0.2732734382,-0.0219900087,-0.5813630223,1.7156662941,-0.3235935867,2.4618074894,-0.6561592817,0.0910400450,1.2659327984,0.8638816476,1.0664783716,-1.2248061895,0.6043577194,1.0140352249,0.4129702151,-0.5681805015,-0.0218119808,1.3225139380,-0.0147479223,0.0736440644,1.1261093616,1.4245935678,0.9596525431,0.5288068652,1.1710895300,-0.7957918644,-0.1596978903,-1.3176041842,-0.3115763068,-0.0672747642,-0.3569861352,0.2234518230,0.3094016016,0.9527436495,-1.2522807121,-1.2509135008,-1.6274449825,-1.2778787613,-1.0911378860,0.2284249663,-0.3604622483,1.1675195694,2.0061020851,0.7697836161,-0.9829636216,-2.2009537220,-0.1629348844,-0.7106205821,-0.4898200333,-1.4288221598,1.0543422699,0.1862355769,-1.5556902885,-0.2575588226,0.5807380080,-1.8932569027,-0.4353618026,1.5906252861,-0.9965630770,-2.0895090103,0.9025256634,0.3491262496,-1.4236516953,0.1656542122,0.2637915611,-2.3229284286,-0.7769226432,-0.4434509277,0.2784052789,-1.7719700336,1.9006904364,-0.3290591836,0.4922101796,-0.5404827595,-0.9461563230,0.6440004110,-0.3489286602,-0.9171313643,0.4032333195,0.5990763307,-1.7341868877,1.2567689419,-0.2003704309,1.5010881424,-0.1889965534,-0.6302958727,0.1775143445,0.2809249461,1.7796574831,0.4534760118,0.1546487063,-1.5418481827,-0.5078651905,-0.8182194233,1.2638050318,-0.1745502055,-0.4214827716,-1.0797538757,-0.1164133102,-0.7410816550,-1.3497550488,-0.1168411151,-1.3411626816,-2.0407903194,-1.2275632620,-1.3231233358,0.0243034586,0.4815312326,1.4609789848,-1.4686429501,0.1677938551,-1.0592018366],[-0.2852744162,-0.0706676841,-0.1581604034,0.9220636487,-0.1272540987,-0.2593966126,-1.7004094124,-0.5371990204,-0.7370152473,-0.0971123576,-0.3767374456,2.3517546654,2.6098873615,0.4989961088,0.0687675998,-0.3759937584,-0.8398221731,0.9545980692,-1.4972980022,-0.9398316145,1.1029561758,0.2521598339,-0.0855884403,-0.9977290034,0.8008356690,1.5740467310,0.4187212586,1.9298514128,0.1145944372,-1.1502786875,0.0434628800,-0.1572289616,1.0625115633,-0.8050823808,0.2032386810,-0.0399338417,0.5518578291,0.8568918109,1.7390502691,0.6615341306,-0.5819038749,-1.2263169289,0.1351081878,-0.1532971412,0.2919063568,0.6675716043,-0.0507290736,-0.6389257908,-0.1211285144,-0.1745242774,-0.2457411587,0.2195800394,0.6483494043,-2.5658056736,0.9271953106,-0.3155171871,0.2342108935,-0.4574358165,1.2141743898,2.0983936787,1.2039935589,-0.1287723780,-0.0756367967,-0.2917146683,-1.5107499361,-0.0836014524,0.7631447315,1.9176464081,-0.5091345310,0.0782249793,0.1212063432,0.6442352533,-0.7711468339,0.5957395434,0.2253505737,-0.9019545317,0.5706453919,-0.5385786891,0.9147205949,-0.7850139737,-0.1509184986,1.2764766216,-0.5555887818,0.5547939539,1.4719022512,-0.2048540562,0.1632129550,-1.7694426775,0.3638873100,0.2493520528,-0.5556089282,0.6050454974,-1.1778032780,1.2873336077,-2.7311754227,-1.8287607431,-0.3213097453,0.2433038205,-1.2711951733,0.2429094762,1.5860184431,1.2297668457,0.9442595243,-0.8065596819,-1.2923262119,0.8501077890,0.3705936670,1.4010832310,-0.4744163156,-0.2130869031,-1.9867215157,1.2959045172,-1.9881888628,0.1270659864,1.1208313704,-0.1007476598,-1.1433990002,0.1744137406,0.2917715907,0.2511937320,1.7150257826,0.9280009270,1.1453393698,-0.3360704184,0.4720384777,-0.3854337335,-0.4944903851,1.6074732542,-0.7994801998,1.4878766537,-0.0383373089,-0.4597311318,0.4766893387,0.7497329712,0.7272310853,-0.5855666399,-0.1302246153,0.3075602651,-2.5834276676,1.1175756454,1.2175463438,-0.4559751749,-1.0002273321,-0.4586575031,-1.1881129742,-1.0883032084,0.4656453729,0.1931448430,-1.4194663763,-0.1684705317,-0.0929002762,1.4055173397,0.1011286005,-0.2336747944,-0.7242429256,-0.4713487029,-1.6743985415,-0.4759764373,0.6226197481,-1.0360772610,-0.7374758720,0.3776315451,0.7032527328,0.3570717871,1.6821124554,-0.7404009104,-0.6392166018,-1.8168293238,-0.2498717159,0.9376166463,0.3249538541,1.8151599169,-0.8932920694,0.8224828243,0.2315905243,-0.4471608400,0.3912411332,0.9867906570,-0.1927725524,-0.5247645974,-1.5280073881,-0.1495466530,0.0160353705,0.0425342433,0.7152678370,0.3655344248,-0.9652132392,-0.8394863605,-2.0853965282,1.5263340473,0.8999857306,-0.9665651917,1.3215459585,1.6256463528,-0.6425731182,-0.3002862930,1.3954046965,-2.0651483536,0.3873505294,1.7838572264,0.8483130932,0.7518290877,0.5456620455,1.5536112785,-1.3005512953,-0.0011652977,1.1804986000,-0.3673217297,0.0885236189,0.8989169002,-2.2710711956,0.1494538188,-0.0522320829,-0.7923706174,0.2892282903,-0.2980457246,-1.0174229145,0.9635899067,-0.8385112286,0.6136938930,0.2540693581,-0.9374857545,0.9380664825,0.1181033030],[1.1129386425,0.9579056501,-0.9558597207,1.0601702929,0.7341737747,0.2400165498,1.3471019268,1.6240410805,0.8954927325,-0.2616872191,-1.5816322565,-0.2222897261,0.1012802795,-0.6242704988,0.4884990752,0.9954836369,1.2131110430,-0.7540560365,-1.0051815510,0.6316062212,-0.3438287973,0.2564253211,-0.3662753701,-0.4446938932,0.6212993264,0.5648611188,-0.1582486629,-0.5771440864,1.2945601940,1.4222235680,0.9416752458,0.6796181798,0.1929067373,-0.8631902337,-0.0065787770,-0.2801783681,0.1711493433,1.6927223206,1.9421662092,-0.6135012507,1.3712947369,-0.6469724178,0.0916923508,-2.0832099915,0.3947067261,-1.0328670740,1.3029398918,0.2731808722,0.1960372478,1.0402327776,-0.0111813899,-0.0176756624,1.6581752300,0.6408863068,0.2431170642,0.1895090193,0.5048427582,0.7047714591,0.9869223833,0.2082526386,-0.1569980681,1.8514502048,-0.1799520552,0.4686902165,-0.7296008468,0.9013362527,-0.1100251153,-1.0055503845,-0.9817690253,0.8043930531,0.5625268221,1.8719913960,-1.1463146210,-0.0346449055,0.1817469299,0.9270682931,-0.2574214339,-0.0812553093,-0.5912104845,-0.3345954716,-0.8301431537,1.0862048864,-0.7808485031,-0.0158352572,-0.3031056821,1.6571618319,-0.8905954361,0.0837379172,0.6826679111,0.5788830519,0.2168190330,0.5412689447,1.2712599039,0.7540125847,0.6827997565,-2.8907747269,-1.5387016535,-1.5241500139,-0.3924700022,0.0892303810,-0.5330017805,-1.7088195086,-0.1490052342,-0.3517466784,-2.0205872059,0.7012131810,0.9212424755,-1.2042565346,-0.8458803892,1.0954351425,0.1554866433,0.6053807735,-1.4512552023,-0.0093505876,0.7159829140,1.6241915226,-1.0060654879,0.5553455949,-1.4876133204,-0.9805587530,-0.6491589546,0.0377813093,-0.4374382496,-0.4266566336,-2.0373737812,-0.3526954651,-1.3506067991,0.7612988949,-1.1326609850,-0.2315673083,0.4062868357,-0.4989110529,-0.0301045813,-0.2450016141,1.1841666698,1.1055570841,-0.3115909100,1.5708198547,0.0280597452,0.0123594552,0.4701385498,-2.1805176735,-0.4992506504,-1.8012614250,0.6283216476,-0.8813948631,0.6601570845,1.7074941397,-0.0323648006,0.8825834394,1.3912378550,-1.6596704721,-1.6631295681,0.8406025767,0.0631161332,0.1085712463,2.0449030399,0.0345824733,1.1145036221,0.4704167247,-0.1383501887,-0.4539908171,-0.0641246885,-0.3558729589,-0.9145979285,2.0010707378,1.6858053207,0.3380937576,-1.3622864485,1.6141508818,-1.9733695984,0.9540565014,-2.1892340183,1.3951656818,-0.5408843756,0.6286481619,-0.7919251323,0.0105986586,0.8478817940,0.0002034874,-0.9074158669,1.5799069405,-0.2770311236,1.0688809156,0.9766724706,-0.7193752527,-0.4294345379,-0.1861296296,0.7368682027,-0.1072331071,-0.8955709338,-0.2778291702,-0.2356924266,0.0711850822,-0.6330330372,0.2691031694,-0.5513830781,-0.7187303901,0.0601756237,-1.5095027685,0.8429980874,0.4193808436,0.0188407283,1.1542465687,0.9463533759,-2.3277208805,3.1905584335,-0.8127261996,1.6442073584,1.6567442417,-0.8463044763,0.1784670353,-2.0980811119,0.8293775916,1.9536508322,-0.7747604847,1.1398042440,0.9690032005,-0.2423246056,1.3939158916,0.1454046220,1.7181164026,0.0453010388,-0.0042198766],[-1.3608663082,-0.5003177524,-0.8680332303,-0.9361079931,-0.3540412188,1.5443038940,0.1527195871,1.9295225143,0.8829023242,1.0667285919,0.7185322046,-0.1005984619,-0.5712147951,-0.2969015241,-1.0879497528,0.1453319192,1.1827607155,-0.6063740849,-0.5514721870,1.3078466654,0.8379271030,-0.6013044119,-0.9620295167,1.0878808498,-1.4823607206,2.0232403278,0.6163513064,0.5846143365,0.1666686833,0.4976269603,-0.1785562485,0.7540314794,-2.4182591438,1.3574743271,-0.5127028823,2.2171654701,0.8127683997,1.0364259481,0.0117332144,-0.7456771731,-1.8012324572,0.5935599804,1.0617488623,0.1362540573,0.5942974091,-0.2413583249,0.8704653382,0.6090841293,0.8453356624,0.2453041673,-0.7767860293,0.4689100981,-0.2965798974,-0.2062119991,0.3263522387,0.2172193527,1.1833193302,0.2821285129,2.3209342957,-0.1554056257,0.1805481464,1.7754137516,1.7161456347,1.2346943617,-0.8394340277,0.3948509991,-1.2444956303,0.7143678665,0.1295471340,-1.6973470449,-0.9740580320,0.9478079677,0.0702200681,0.6023330688,0.3383901715,-0.5129318237,-0.9346300960,-0.2646407187,0.5493919849,0.3167619407,1.0960144997,-0.1926865131,1.1150927544,1.4298577309,0.5634369850,-0.8687397242,1.0241733789,-0.7390829921,-2.4886791706,0.1626223028,-0.0862197801,2.2243971825,-0.4995155036,0.1662081778,-0.3019536138,1.2557128668,0.2725491226,-0.4110666811,-2.0384042263,0.4620322287,-0.9412257075,-0.3955143094,-0.9924059510,0.1638202667,0.3578505516,-0.4322696328,-0.7486923337,1.4323714972,0.4394994676,1.2358547449,0.6357761621,-1.2816650867,0.3573113084,-1.6242008209,-1.2238067389,0.1763255596,0.7219309211,-1.1676596403,0.3330031931,1.0878335238,-1.0375856161,-0.5326374769,1.8756786585,1.5115497112,-1.2856667042,1.1450330019,-1.0966980457,1.1074422598,-0.6321702003,1.5199065208,-0.5065426230,-1.0368436575,-0.0203417726,-1.3390274048,-1.2279171944,-1.0808128119,0.3871849775,-0.3497922719,2.3117475510,0.6136760712,-1.0078067780,-0.8818475604,0.8308844566,-0.2642922997,-0.8865725398,-0.2153000832,-0.2417999208,-2.1490194798,-0.7199160457,0.0679587796,-0.2582596838,-0.8022152185,0.9388174415,-0.9667415619,2.2158970833,0.9668938518,-0.6219958067,-2.9191241264,0.7569295764,-0.3887217045,-0.9876939654,-1.3517054319,1.3579764366,-0.4132209122,-0.3938776553,1.0867658854,1.0614277124,-0.3257678151,0.5012996197,-0.9414650202,0.8639216423,-0.1593751311,-0.8448317051,0.5117835999,-0.0846990570,-0.3059445918,-0.5071154237,-0.5131493211,0.0728861243,-0.0135237006,-1.0407599211,0.1897208542,0.8545592427,-0.5512021184,2.0016546249,2.1962902546,0.6786051393,0.6614071727,0.3245737553,0.7180817723,0.5551066399,0.0685995594,0.1573880464,2.0783345699,0.5659430623,2.3344035149,0.3795775771,-1.1248421669,-0.6414177418,2.1331074238,-1.0174721479,0.3696995974,1.3464163542,0.4752027988,1.4939831495,0.4542889595,-1.4727809429,-0.4702996016,-0.3798926771,-0.0606959425,-2.7277064323,-1.2520660162,-0.1419366747,-1.7190331221,-0.6506875753,0.0628099963,-0.7629907727,0.6080714464,-1.9897142649,-0.3891165257,1.0277502537,1.1713209152,1.4797284603,-1.3836252689],[-0.2505295575,-0.4447467029,0.1925516874,-0.6498235464,0.2901880443,1.6240186691,0.4743281007,2.1063592434,0.5284393430,-0.8047794104,0.2104256451,0.7566365600,0.9716638327,-0.2522500753,-1.2101258039,-0.9453462958,-0.8787621260,-0.0724296495,0.6382610202,-1.0404309034,-0.4261433184,1.5532246828,0.5275081992,-0.3574368060,-0.7761614323,-1.7335830927,1.0692806244,-0.5937536955,0.4269003272,-0.3536632359,0.9446827769,2.5059707165,0.0690293163,0.2944066226,-0.7414646745,-1.0082733631,1.1343686581,0.2710540593,0.7274960876,-0.2651674747,-1.4867268801,0.5070603490,-1.2814332247,-0.3046406507,0.5114221573,0.3708107173,0.1999140978,0.7607744932,0.0477416702,0.4222696722,0.2854658961,-0.9390783906,-0.7843756676,-0.8260797858,1.4266068935,0.8196808696,-1.1903074980,-0.4611110985,-0.4952437580,0.6804166436,-0.6994231939,-1.1552095413,-1.7887793779,1.3153162003,0.6558619738,0.4816769361,-0.0126672778,-1.3285335302,-1.2688424587,-0.1523873210,-0.4339653254,-0.1709483713,0.2231142521,1.4828188419,-1.4240397215,-1.4417916536,0.4451074302,-0.1589459926,1.2505066395,1.3375947475,1.1642736197,0.3594298959,0.7713550925,-0.6801846623,-0.4980467260,0.9504658580,1.6608889103,0.4158179760,-0.2972727418,1.2367286682,-1.0620138645,-1.8265507221,-0.5331158042,-2.1153428555,-0.6209367514,0.4675408602,0.6143893600,-1.7113894224,0.2891764641,-0.6705935597,0.4754212797,0.3198952973,0.7838545442,0.3976619542,-1.1661629677,0.4603621066,0.9248743653,2.1145966053,0.0825881287,0.6749655008,0.8121272326,-0.2189459950,-0.1639946699,0.3587153256,-0.7834233642,0.6059847474,0.5444562435,-1.4124641418,-0.0233196020,0.0880037472,0.1275142431,1.0809987783,0.6402114630,0.7634710670,0.9324608445,0.3846043348,-1.7244272232,0.3992971778,-0.3185180128,0.1687958091,-0.0704131722,0.6610387564,-0.6090794802,-0.4523200393,1.0825520754,1.0088615417,-0.5682323575,-1.3370556831,1.0058012009,1.6292598248,-1.2862094641,-0.6961910725,0.2314819843,0.7249957323,0.6836280823,-0.1757106632,-0.9063949585,-1.3477391005,-1.1346484423,0.3227340579,1.3607194424,-2.8635163307,0.2292383462,0.4060978889,-1.3265417814,0.2751484215,0.7121296525,-0.6921774745,0.6154708266,0.2200543880,-0.5754459500,0.3323008418,2.1421134472,0.7250728011,0.6870709658,0.5515963435,-0.5132011175,-0.1644386500,1.9557372332,0.5852532983,0.7971339822,-0.4506426752,1.4849123955,0.2836766541,-0.6032394171,0.4705572426,-1.0361155272,0.2024379373,2.0730745792,-0.1682272553,0.3044758141,-0.1541289538,-0.6910542250,1.4061708450,0.4611213803,-0.2320941687,-0.4938946962,-0.2043031901,-0.0456889868,0.9397354126,0.9698880315,0.0257637091,-0.2238553017,-0.2935015857,1.0003639460,0.8938386440,1.1174061298,-1.4390172958,-1.3158385754,-0.4139026999,0.4302385151,-2.2730138302,-0.6109037995,0.6164110303,0.3562093377,0.9595016241,-0.5901470184,1.8615894318,0.3402916193,-0.0181768406,0.8151611686,1.2026921511,-0.9299021959,0.6094628572,1.2707104683,0.4547021091,0.9683445096,-1.0731282234,0.7785558105,1.6914439201,-0.2198672742,-2.2547419071,1.2058824301,0.6307622194],[-1.8844203949,-0.3400211334,-1.4175158739,-0.1408693939,-0.2282040119,0.6934574246,1.0840233564,-0.1311188787,-0.4547898471,0.0308669638,-2.9230649471,0.1924819946,-0.0251757074,-0.1361604929,1.3143212795,1.3681744337,2.5020995140,-1.7766501904,2.9016273022,0.4275854528,-0.0154027585,-1.1051633358,0.2060504258,0.1583422422,0.3727222979,0.9876830578,0.0626214370,-1.1221803427,3.4378697872,-2.8637402058,0.9245373607,-0.2257023752,-0.5427214503,-1.8706995249,0.5202276707,1.4523717165,-1.8733434677,-1.2647411823,-0.5458576083,-0.6326601505,1.9794954062,-0.0096601248,2.2758998871,0.3492978215,0.2012365162,0.2095893174,0.6796535254,0.0502694286,-0.0435023643,0.2123025507,-0.4695489109,-0.3350050747,0.2190171927,-1.0715231895,0.3171286583,-0.2802983224,-0.6058686972,-1.5530456305,0.9496842623,0.2735980153,-0.4321633577,-1.6320595741,0.8468621969,0.6839948893,0.2943500280,1.5179771185,-0.3009389341,1.1591215134,-0.4598817229,1.6021217108,-0.2987172902,0.1620407104,0.0022509200,-1.0108187199,-0.9158037305,0.6183546185,-0.0589140914,1.9188567400,-0.0930142999,0.4509327114,1.3160575628,0.1009306461,-1.2808429003,-0.3061289489,0.2605623901,1.1934047937,1.3779098988,-0.8680241704,-1.0800518990,-0.0498448424,0.4656108022,1.0705919266,0.5382865071,-1.0938647985,0.6651583910,0.1397799551,-0.7773083448,0.6970692277,-1.1914598942,0.4689689577,-1.5771685839,-1.7258087397,0.9759751558,0.9858368635,0.9479118586,0.8612651229,-0.4446322322,-0.3175063431,1.2068784237,-0.5929441452,0.1791196018,-0.9298995137,-0.4742261767,-0.4334574640,1.3930308819,1.5312166214,0.5114341974,-1.9366881847,-0.0120311510,0.2527293563,-0.3314227462,0.5194210410,0.1215722114,-0.5180037618,-1.5782446861,-0.3423944712,1.1970065832,-0.0611416213,-1.3710052967,0.8990535736,0.5925429463,1.5193586349,-0.4414147139,-1.0274288654,0.3192766309,-0.2304987013,-0.0761491209,0.0690865219,1.3222659826,0.0257438123,-0.2477024496,0.2511856556,-1.5257942677,1.4712384939,-0.0396799296,-0.0264192075,-1.6288158894,-0.7242621779,1.9021406174,-0.5092082620,-0.0244931318,-0.5751573443,0.9152583480,1.1320323944,0.3048408628,0.7192915082,0.0029749731,1.7288789749,-2.0123455524,-2.4080128670,-1.7593473196,-2.4347224236,1.6136039495,0.4126974642,-0.7342951298,-1.0291619301,-0.2701182067,-0.2743706107,1.5435656309,-0.4356430173,-0.3694144189,0.5576394796,-0.2556772828,0.4563857317,0.2570762634,0.3890536427,-0.8774389625,-0.2383845448,-0.5045922399,1.0469880104,1.6212552786,-0.2120764405,0.6647508144,-2.0476558208,-0.8174135685,-0.8982970119,0.7912128568,-0.7010203004,-1.3018070459,-1.6871378422,0.5631584525,-2.1441884041,-1.6575580835,0.9514712691,1.6940368414,0.0827083439,-1.3294876814,-1.1565612555,-0.1200775355,0.4073521793,0.0284242388,0.0219066888,0.5942062140,-0.4101213515,0.8342248797,0.3034593165,0.2702042758,-0.8202449083,0.8972294331,0.1362715960,0.6182371974,0.3332054317,0.3803861737,-0.0642565563,1.5682134628,0.0266444050,2.2625157833,-0.8821482658,0.9058094025,-0.9818913341,-0.6558931470,2.1423466206,0.8509710431,-0.3466790020],[0.8928981423,1.8331450224,1.0823125839,-2.2121467590,-0.2042878270,-0.8251465559,-0.3494953811,0.5181334019,1.1187945604,0.8085275888,-0.9451194406,-0.1899499297,-1.5023754835,0.5746624470,0.3227597475,0.9226501584,-0.6544494629,-1.2606118917,1.4417942762,-1.2978820801,-0.0916388780,0.2793105245,1.1175570488,1.7324228287,-1.0782588720,-0.0125431186,0.6125292182,-0.2632619441,-1.0598345995,0.2111615837,-0.3366095126,0.9723577499,-0.0001053959,-0.9998166561,0.3668155372,-0.1650307626,-2.6319227219,-1.3921569586,0.5988414288,-1.7351586819,1.4667917490,-0.5279781222,-0.3885532022,0.3108111024,-0.1206910387,0.8990362287,0.1039673463,0.1421874166,0.4403844476,-0.5696710944,-0.2248260379,0.9770126939,1.2786602974,0.8683383465,2.3025348186,0.0278422982,-0.1820368022,0.8733520508,-0.3343454897,-0.5557032824,-0.5515244007,0.5276218653,-0.7114800811,1.7720230818,-0.3970922530,0.4389536083,0.0650350600,-1.1340105534,-0.7053775787,-1.8313711882,-1.6354445219,-0.4521007836,0.3524754345,0.4045782983,-0.1586718410,-1.0496007204,-0.8052915335,-1.4610325098,0.3669247329,-0.9069761038,-0.7325763106,-0.9586330652,-0.1848229915,0.1050765216,-0.1473184377,-0.1583449841,-0.3924146891,1.3609659672,0.5217448473,1.9404394627,1.0777040720,0.3909571171,0.6589066386,0.1673523486,1.0541881323,0.3378158808,-0.7065559030,0.1609089524,1.7601629496,0.2293126136,0.9480530024,-1.2335206270,-0.0344778039,0.7309923172,-1.5902619362,-0.4351943135,0.6615905762,-0.3688139915,2.3102419376,-0.9426203370,-0.5544672608,-0.6343543530,1.0669492483,-0.2379402369,-0.4261119664,1.8326580524,-0.7513182163,-1.4482220411,-0.7471646667,-0.8635528684,-1.2731425762,0.6533293128,0.6602415442,1.2218433619,0.6049968004,-0.5613385439,0.0613612570,0.9906570315,0.5033934712,-0.6237265468,-0.0041219913,0.8261147141,0.2427824140,-0.3063735664,-0.0811720788,0.1534022242,-1.1207720041,-0.6678703427,-0.2377390862,0.8940547705,0.2770272493,1.3696693182,-0.0493086502,-0.0745674670,0.4672318697,-0.3150355816,1.4334459305,-0.1679604053,-1.9706282616,-0.3613967896,-1.6547517776,1.1160451174,-1.6003566980,2.0415852070,2.0366399288,-1.0523581505,0.3376710713,1.0529056787,1.6930022240,0.9688546062,-1.4531956911,0.8070805073,-1.3145008087,-1.4333742857,0.0553366765,0.4915426075,0.3531095684,-0.7884507179,-0.1045462191,-0.5995621085,-1.3206208944,0.6596292257,-0.9884840250,1.9658976793,-0.4340378940,0.0516867973,-0.5761942267,-0.4747160673,2.0759608746,0.9117660522,-1.3595043421,-0.0750775263,1.0767716169,0.8811606765,1.1698845625,2.6263203621,1.5705279112,-1.4361846447,0.8416768312,-0.7291257381,-1.9000356197,0.1692651510,-0.5643911958,2.2524120808,0.3646518886,-0.2983020544,-0.4583005309,0.9823272824,0.0525144637,-0.7168455720,0.5617127419,-1.3276793957,0.0449519083,1.7765147686,0.3622149229,-0.7156533003,2.7877490520,1.1703428030,1.6707651615,0.9258809090,-1.0642477274,1.1153731346,0.5419448614,0.0624054596,-1.0790256262,-0.2572177947,-1.1862916946,0.9607563615,0.1119695157,0.5639559031,-1.7139394283,-1.2567867041,-0.2268253267,0.3858694136],[-0.0999460742,0.4597186148,-1.6977158785,-1.0717468262,1.4142743349,0.2119618356,1.3924894333,-0.9887067676,-1.0531584024,-1.2928825617,1.2426022291,0.5878120065,-0.9949681759,0.8283343315,-0.2635304928,0.4014886022,-0.6090260744,0.5791243315,0.5834465027,1.5511604548,-1.2230901718,-0.6187438369,-0.3775131106,0.9911983013,-1.7839671373,0.1290750504,1.6611936092,-1.9548208714,0.5270512104,0.9121604562,0.4895628095,0.0196124874,-0.4402401149,-1.8178555965,-0.2203289568,1.2370438576,-1.3454525471,2.4535121918,-0.1176287308,-1.0192205906,0.5798735619,0.0242904909,-0.0841123089,0.5548579097,-0.7952302098,-0.8613741398,0.8380604386,1.0689024925,-2.4395861626,0.3448410630,-0.5043354034,0.0641216338,1.1720117331,1.2903999090,0.4812498987,1.1072002649,1.0949929953,-2.3961193562,-2.7901766300,-0.2823035121,1.2701352835,-2.0818095207,2.6965289116,-1.3935769796,-0.2706691027,1.0988687277,-2.1704502106,0.6163380742,1.3522902727,0.7691040635,-3.0363001823,2.5481321812,-1.5075689554,-2.3067941666,0.8177285790,-2.4801552296,0.9572334290,1.1474175453,-0.5582699776,-0.2533708811,0.1953659654,0.0904167295,0.6058718562,0.3594837487,0.5418453813,-1.2951164246,0.4690193236,1.7330023050,0.1667902321,1.1768552065,-0.0910997316,-0.2359167486,-0.0121873543,0.4238246381,-0.0331050605,-0.5837310553,0.5397078991,0.3845285177,0.1106715724,0.1258557290,-0.4165729582,1.0588035583,0.4541542530,-2.6346459389,-1.2074881792,-0.1499014944,-0.3375472128,-1.5428763628,1.2837784290,-0.6475168467,0.3657090068,0.4615589976,1.6139746904,1.0400425196,0.8091872334,0.0503223762,-1.4139139652,0.1132474169,1.5251283646,1.6179705858,1.2431045771,1.7049555779,0.1189959124,-0.2425130904,0.1800673753,-1.4013088942,0.6485371590,0.8376440406,1.1608923674,1.1170085669,-0.8883395791,-0.1524236053,-0.8715451360,-0.1062527671,1.8281518221,-0.6069445014,-0.6816360950,-0.3067930639,0.9655979276,-0.2940889299,0.9251574278,1.1493262053,-0.5966495275,0.5556038022,1.9187790155,-0.2040707618,-0.4193641543,0.2341037840,-0.2179090828,0.7499462962,-0.2418686002,0.5113593340,-1.8201842308,-1.1509025097,-0.7798062563,0.1130120382,-0.3003650606,-1.9360016584,-0.9553108215,0.9728742838,-0.1972751766,-1.7404620647,-0.4687793553,1.1957448721,-0.5498569012,-0.7883052230,-0.1498728544,0.2681272030,-0.2486549169,-0.4183347225,-0.3109455109,0.1175777838,0.7868638635,0.9115644693,0.1073277220,-1.0881794691,0.8313896656,0.4506823123,-0.9034633636,-1.0354187489,0.6935775280,1.1028820276,0.6876826882,2.0347700119,0.5668647289,-0.3477827907,1.6544166803,0.3295305669,-0.1434911638,-0.1425676346,-0.4272623062,-1.2900912762,-0.2381984293,0.6456766725,-1.0504271984,-0.9134973884,-1.0105756521,1.1871837378,0.2558339238,0.9838050008,-0.5840255618,1.6918026209,-0.7227749228,0.5577063560,-0.2009231299,0.9748980999,1.0686962605,0.1406216919,1.4599475861,-1.1134356260,0.4520905316,-0.3960721493,0.9606674314,0.4791612625,-0.5085137486,-1.1975919008,1.2257055044,-0.7944650054,1.0772857666,-0.8134288192,-0.1460259855,-0.7566825151,0.5483761430,0.4096541405],[-1.1490131617,-1.2600245476,0.4720077217,-0.8848083019,-0.3584946394,-0.3885389566,-1.6692073345,1.8913199902,-0.9435986280,-1.6560325623,-0.4170581698,1.4248950481,0.7457003593,-0.2701747119,-1.4329892397,-0.9257540107,-0.5308506489,0.2141064256,-0.7413557172,-0.7860522866,-1.6483081579,-1.3483446836,-1.2292420864,-0.5480573773,-0.5060109496,-0.5933606029,-2.0655674934,1.2769864798,-0.0294818971,-0.5929264426,0.6686656475,0.5973464251,0.2934952974,-1.2685967684,1.0854260921,0.0394030698,0.6842738390,-0.7315492630,1.6883585453,1.6437082291,-0.8642318845,-0.1803095788,-0.1081145182,-1.4840042591,-1.0925523043,-0.0899927989,-0.3712283671,-0.8431726694,-0.9773939252,0.5591737628,0.4611403346,0.0734049380,0.5970427394,1.9101938009,-2.3871018887,-0.4583065808,-0.5112628341,-1.3271329403,0.7063751817,-0.6450681686,-2.1740829945,2.9771606922,-0.7224324942,-0.6503081918,1.5257606506,-0.2274675965,0.4022358656,0.4434968829,-1.0218620300,-0.1315025240,0.0179176256,-0.4155089855,0.4858019650,-0.8543961048,-0.5012856126,0.8567308187,0.2221969515,0.1999620646,0.6042551994,0.6662653685,1.0663129091,-0.4896272123,-0.2026205212,-1.1394944191,-1.2054032087,-1.1226356030,-0.4939437509,0.1943001896,0.6055281162,0.7169389725,-0.3576804101,0.6503378749,1.6525493860,0.5573248863,0.1776629537,-0.4948503077,-0.0657189265,0.1559060067,-0.1242087036,0.0716841519,-1.5930215120,0.0682995692,-1.2083141804,0.9281802773,-0.0046851207,0.9349762201,0.2173268646,1.3025127649,0.0904640332,-0.3832762539,-1.6360230446,0.4152020812,-1.4499077797,0.8201241493,0.8912186027,0.0771826804,-0.5154270530,0.9378012419,-0.4901067615,0.5525051951,1.2330327034,-0.3991430104,1.4831937551,0.7175683379,0.2221147567,0.3608321846,0.2894625068,-1.6595556736,-1.9314495325,1.8751310110,-0.4770455658,-0.5211828947,-0.6297601461,-1.1078292131,1.9312903881,0.2273961604,-0.3932147026,-1.0034306049,-2.2730729580,0.0832397044,-0.5162621140,-0.9493662715,0.7647309303,-1.4768458605,-0.7454246283,0.2828139067,-0.3133463562,0.0699866787,-0.8258736730,0.9094543457,2.2345933914,-0.2716061175,-2.0914297104,-0.8450802565,-0.1864923686,-0.2835510671,-0.1033757925,-1.5680092573,-1.2161769867,-1.6124955416,1.3700754642,0.7851352096,-0.3823657930,1.3108464479,-0.8988699317,0.5083639026,0.3493502140,-0.0577305481,1.0597113371,1.0081275702,-0.6358467937,-0.4504096806,-0.3147639036,1.0732730627,-0.8271607161,-0.6222040057,-0.7991218567,0.7039133906,0.3510959148,0.5554497242,1.4979805946,-2.1750752926,0.2891605496,1.9272480011,-0.9401828647,1.3731950521,-1.7251560688,1.3734343052,0.3356144130,0.9673383832,-1.1017390490,-1.1915374994,-1.7516787052,-1.0826570988,0.6179661155,1.1888171434,-1.0887053013,-0.2124068141,-1.1567455530,-1.0044180155,-0.9566497207,0.7797390819,-1.7934116125,-0.3947986662,-0.4438236058,0.4794886708,1.2552320957,-0.7013828158,1.3951803446,-0.0517238714,-1.4133031368,-1.8694245815,-0.5054339767,-1.2462067604,-1.2365274429,0.3690735698,1.2771483660,0.0578073971,0.3081986904,-0.2649683654,0.5571885705,-0.5460993648,0.5755186081,1.1222224236],[0.0097143548,-0.1952548772,1.3565348387,0.9633554816,0.7406532764,0.4492635727,0.3314831555,0.1872891337,0.5805309415,0.3974446356,-0.5829093456,-0.0477444232,1.0294648409,1.5680623055,1.6048808098,-1.7844184637,1.3980298042,0.5945453644,-0.5468156338,1.2185471058,-1.0988667011,-0.5286205411,-0.5052637458,-0.1807851791,0.1174189076,-0.9720669985,-1.5364803076,1.3941923380,2.1483066082,0.4284505546,-0.1856944412,-0.7900459766,0.0108980974,0.9540181160,1.2205339670,0.4898815155,-0.9313154221,-0.9521727562,-0.5750778913,-0.2174461782,2.1930093765,1.5066055059,-1.0824288130,-0.2641721070,-1.2634152174,-0.1384104043,0.4317926168,-0.6106199026,-0.4305661917,-0.2218575478,-1.1112926006,0.1515053213,-0.3358800411,-0.1186274439,-0.7633775473,-0.2121664286,1.9252479076,-0.5415530801,-0.4101755917,-1.9593467712,-0.1169006303,0.1504407674,1.2334483862,1.0831296444,-0.4920203984,-1.1058235168,-1.3844621181,0.1942979842,-1.2118817568,0.9645847082,-2.0245478153,0.2150682956,-1.8037940264,2.1545493603,0.2532272041,0.3322596550,0.8775139451,0.0413930006,-0.5169070959,0.8127090335,3.4397461414,0.8755329847,0.6949456334,-1.6751369238,-0.7122853398,0.5115014315,-1.2610419989,-0.0193399414,1.1886892319,0.3200702965,-0.4151143730,-0.4470207989,-0.7033379078,2.1508812904,0.2372479588,-0.8369963765,-0.6052927971,-0.6665679216,-1.3898322582,-0.1554756016,-0.9192994833,-0.7518845797,-0.3134998679,-0.2572743595,-1.5091480017,-0.5846824646,0.3287142515,1.6392349005,-0.0196348205,0.5276853442,1.1864744425,0.7721220851,0.6423051953,-0.7868347764,-0.1904338896,-0.0519685000,0.2901331782,-0.0316568166,0.3051229119,1.2303379774,0.2227156460,-1.8869382143,-0.0082769394,0.2376345396,0.8816763163,-0.1637671590,-0.8250055313,0.9399932623,-0.2580229640,2.0155923367,-0.1516564339,0.1265739948,2.1453938484,0.8217768669,0.9791879654,-0.2263262570,-0.4864189625,0.0454811975,1.3710892200,-0.9420208931,0.9791120291,-0.4800106883,0.8337278366,-1.3876479864,0.8446266651,0.8098564148,0.5751532912,-0.9790716171,-0.0171645191,-0.5026745200,0.6194605827,-0.8599231243,-0.3917999566,-1.4034454823,0.8840796947,-1.6543312073,0.0696845874,-0.4744838774,0.8552336097,1.8941549063,-0.4313303828,-0.2739033401,0.3928846419,-1.5880547762,0.9341297150,0.3463015854,-1.3363854885,-0.0499476194,-2.9639329910,-0.5594952106,0.7434971333,0.1999131292,-2.0764930248,-0.5285277367,-0.2470802069,-2.0624265671,0.9572924376,0.3903219700,-0.6605809927,-0.6549845338,-1.0186446905,-0.0800478607,0.8186472058,0.6616176367,0.0518838763,-0.9223122597,-0.5421885848,-0.2825040221,-0.3912544847,-0.2029047906,0.6893842220,-0.8635937572,0.5363397598,0.6943371892,0.0873916373,-0.2490534633,1.2992305756,-0.7386673093,2.0405597687,0.8024853468,-1.4069356918,1.1822263002,-1.3656258583,-0.5556980968,0.1787927747,0.1019315794,-1.3714215755,0.1987746656,-0.3629374206,1.8519080877,0.6735958457,0.4217328429,-0.3506659269,-0.1316977292,1.2174890041,0.3776035905,-0.8396139741,-1.1620602608,0.3407081068,-0.3355799913,-0.7459722757,-1.8262302876,-0.8597204685,-1.4879345894],[1.1369839907,1.1803423166,0.0483212918,-0.3535215557,1.7021528482,0.1325516403,-0.1556667686,-2.0734696388,0.3474398255,-1.1306002140,-0.5391064882,0.0367188156,-0.1031585932,0.5129575133,0.1265309304,0.0982659385,0.4955267012,1.3003282547,0.6503933668,0.3970629275,1.8765217066,-2.2937936783,-1.0489534140,-0.3891129792,2.0643260479,-0.1525699496,-0.3799881637,0.4029478729,0.8279522061,-0.6166064143,1.0913140774,0.1232835948,0.0043251989,-1.2592334747,1.1358189583,1.3957405090,-0.0802572221,0.3133191466,1.0224242210,0.5248175263,-0.6712270975,-0.6747437119,0.0586817972,-0.4509712160,-0.9520729184,-0.7986681461,0.9456746578,2.1470000744,1.4799757004,-1.8066306114,-0.0310747288,0.8656225801,0.5112625360,2.5125331879,0.2004410326,0.2644173801,-0.5642243028,-0.3665525317,-0.8501471877,2.1454489231,0.0146866925,-0.2753212452,-2.2508983612,0.4200601578,-0.5757294893,0.3302652538,0.6833993793,-0.1050571799,1.3361978531,-0.8743016124,-0.1878415346,0.5748300552,0.4990630448,-0.9741004705,0.2689557374,-0.8159407973,1.5243690014,1.1392233372,1.6356216669,-1.5474957228,-0.0423558354,0.0719309151,0.0674327686,-0.4663897157,-1.0062319040,1.1016836166,-0.3533191681,-0.9534746408,-0.5174071789,0.3306112587,1.3137228489,-0.4841048717,-1.5414144993,0.7894215584,0.7925677299,-0.2099342644,1.6130772829,-2.2307887077,0.6066238284,0.4182311296,-0.8662330508,-0.1302055269,0.6588820815,0.7003934383,-0.6792898178,2.0538380146,1.4943599701,-1.2634873390,0.2456917167,-1.6332542896,-1.1893531084,0.7613589168,-0.3110498488,0.1274812669,-1.4536637068,1.2830148935,0.3669683635,-0.6743721366,0.3679584563,-1.8977421522,0.8388814926,1.7857834101,0.3952881098,0.9302375913,0.1583839953,-0.4621100724,-1.2903282642,-1.6294634342,0.7264854312,-0.0749876797,-1.2859839201,-0.3973017633,0.2206417620,-0.6645338535,-0.2155216634,0.5685132742,0.1122510657,-1.3511683941,-0.8059957027,-1.0774103403,0.4130584002,-0.2617705166,-0.1687471569,-0.2825335562,-0.3444823325,-0.9686818719,-0.1723157912,0.2521053255,-2.1374604702,0.4649966657,0.5567176342,2.4056639671,0.2969538569,1.8013700247,0.0617722496,0.4764775336,-1.2795540094,0.6798602343,1.4741630554,0.7309804559,0.3686659932,0.0889440775,-1.4905848503,2.1611554623,0.4314791262,0.1953540742,1.5225317478,0.7390133142,-0.4725934565,1.1161459684,-1.9433223009,-0.1393656880,0.2824656963,3.1519126892,-0.5079812407,-0.8862577677,0.6268798709,-1.2120158672,0.7088871598,-1.6071879864,0.8341999054,0.6533966660,-0.1934999079,-0.1612865776,-0.1745747179,0.6138933301,-0.2608250678,0.0169208851,1.4444025755,0.0534766540,0.2234792262,-1.0242514610,-0.7009769678,0.4972992241,0.1313402355,0.2999706864,-0.9448517561,0.2542484105,1.3423204422,-0.7973331213,0.4570710957,-1.5696889162,0.8125736117,-1.5350893736,0.5826305151,0.2596044540,-0.8103986382,2.3475239277,0.2638120949,1.5259733200,0.5666651726,-0.9071298838,-0.8720138669,-0.9882591367,0.4064132273,1.3104044199,0.1794917881,-0.3529058397,0.8675439954,1.5312627554,-0.1812909395,0.4155770242,1.4033385515,0.1539124548],[-0.1244183630,-1.5379589796,-0.8867199421,-0.6548126340,1.8252388239,0.6651643515,-0.6815838814,-0.0382827222,0.4838942289,0.7248817086,0.0481229275,-0.5961760879,1.7160128355,-0.5742226243,0.4242486656,1.3640313148,-0.0682704076,-0.1149882078,-0.7724986672,0.5856184959,-0.5331230164,0.2725982368,-2.2121868134,-0.5297753811,0.3282147050,-0.2310766876,0.9220627546,-0.4692978561,0.9146265388,1.2192900181,1.1941511631,-0.1149736866,0.9485147595,-0.4857148528,0.6253182888,0.1254618019,-1.3645582199,0.4393002987,-0.9393618107,-0.3738214076,0.3530284464,0.3340075612,-1.0520275831,0.3124981225,0.2227747589,-0.3774113357,0.3376967609,-0.1335430741,-1.4239124060,0.3777493238,-0.6754847169,1.7558667660,-0.3419993520,-0.5435870886,1.5623818636,-1.0864465237,2.7157764435,-1.1701687574,1.0629879236,0.1548271179,0.2290085405,-0.5235656500,-1.2878988981,-0.5667066574,-0.0501692370,1.7568033934,-0.1276576668,-0.8953829408,1.0946452618,-0.3542604744,-1.2604600191,-0.3953779340,0.2435953319,-0.7940179110,-0.5217481852,0.4265375137,-0.3192216158,-0.8071255088,0.5322649479,-0.3526906967,-0.0918452591,-0.6988605857,-0.2911368012,-0.6488785744,-0.6978461146,0.3133989573,-0.7867271304,0.3081840575,-0.1553696841,1.6520466805,-0.8007677197,-1.9127302170,-0.8846508265,0.7461178899,1.0180773735,0.9306701422,-0.1126613319,0.1929772198,-1.0579559803,-0.2422025204,1.8559801579,0.3720010519,1.1825789213,-0.1039613783,1.0414545536,-0.3254982829,0.9571298361,-0.4382165670,1.0003467798,1.5939487219,0.5429488420,-2.0420072079,0.5728467107,-1.8618476391,-0.2918467820,-0.0487218536,-0.9199935198,-1.0266177654,1.6822305918,1.4191178083,0.6947760582,1.0251603127,0.6672874093,1.2495917082,-0.8335829377,0.2156088799,1.1924821138,-0.4395660460,0.6522639990,-1.6726173162,-1.0532317162,2.1165602207,-1.9447240829,-0.0893126354,0.2683942318,-0.1623480618,-0.2299717814,-1.2482939959,-1.4748595953,-1.1440454721,-0.1509122998,-0.5654972196,0.9534760118,2.3310527802,1.1738048792,1.2242695093,-0.1066419408,-0.4332847297,-0.8702556491,2.0648863316,-0.9727774858,-0.3140878677,-0.5490342379,0.6551995277,0.4908427596,1.0332645178,1.1219984293,1.3135952950,-0.4913543463,0.6445788741,1.7172362804,-2.7000076771,-1.9069814682,0.5195291638,-1.2169370651,0.2950018048,0.5568491220,-0.6202567220,3.0056195259,1.6280162334,-1.7385234833,1.4424477816,0.8223228455,-0.0615691058,-0.8732482791,-0.0132957678,1.3641716242,0.8280566931,-0.6633744240,-0.0640525073,0.1176194623,-1.9407721758,-1.8422350883,-1.4371339083,0.1744921356,2.3886828423,0.1898625940,-0.5868138671,0.9374192953,1.4271277189,0.1327908039,-1.2658762932,-1.1797477007,-2.2054107189,-1.2667415142,-1.1865519285,0.3430148959,0.7459679246,-1.5639922619,0.6891250014,0.6442363858,-0.1363123357,-0.3265230358,0.8321313858,-0.0314064920,1.1229662895,1.1464596987,-0.6990435719,-1.4267598391,0.8259562850,1.7598297596,1.2947094440,-0.5276837945,-0.2367414087,0.5686634779,0.6113168597,-0.1315857321,-0.0734574869,0.0436162390,0.1786444336,0.6293717623,-1.1854665279,-1.8017184734,1.1474248171],[-0.2221667469,-0.8318022490,-1.1769648790,-0.6398457885,1.1581661701,1.0694650412,-1.1927752495,-0.3782918751,-0.5945242047,1.5984972715,-0.4153506756,0.9948699474,1.1457722187,0.2799803019,0.0362116992,0.1169509590,1.1984179020,-2.1507964134,-0.2818993628,0.1193391681,1.1043921709,-0.1576315612,0.7816631794,-0.1992943287,-0.7222493291,-1.6471685171,0.5236202478,0.3679266572,-0.2783089876,0.1911206990,1.7007887363,-0.2828701138,-0.6807302237,-2.0343830585,0.0755103827,0.1925597936,1.1212855577,-0.6325791478,-1.4917219877,0.1507131606,-1.4023673534,-1.4923455715,0.2636044025,1.2325544357,0.8019176126,0.7070560455,-0.4439917505,0.3190809786,0.3905167878,1.6310776472,1.1720501184,-1.2261582613,1.4022692442,0.6078658700,0.8062526584,-2.9621202946,-0.3987344801,0.3414399922,1.7619208097,1.5134036541,0.6041021347,0.5934362411,-1.1736227274,1.3398493528,-0.5730978847,0.1356645226,0.2863060832,-0.0154768359,-1.0178600550,-2.3243145943,-0.1675287634,0.6049609184,0.2159761786,0.0666745231,-0.0865136832,1.2352000475,0.0394294672,1.0100305080,0.2578782439,0.6886412501,-2.9555711746,0.3245435357,-1.1066570282,0.0578106418,0.8995000124,-0.5578057170,0.8344925642,-0.3650110662,-1.9382914305,-0.9237997532,0.1845855862,1.0926511288,-1.4047546387,1.5311555862,-0.8047308326,-0.7788903713,0.4715638757,0.4318743646,-1.6874840260,-2.7890527248,-1.5477393866,-0.4640494883,0.4432340562,-1.0020309687,-0.5192461014,0.1724080592,1.3290741444,0.6016322374,-2.4097959995,0.5145076513,-1.1226621866,0.5319163203,-0.9135279655,-0.3886004984,0.2594231069,-0.8287024498,1.3806180954,0.2069475651,-0.8147791028,-1.8920502663,0.3105571568,-0.6040369272,-0.5843714476,-1.3048652411,-0.9555811882,1.6718477011,0.2061201781,0.2449650019,0.6344435215,1.8947550058,-1.1782568693,-1.3522021770,0.5857774019,-1.2353422642,0.5616509914,1.3375346661,1.3691686392,-1.7339766026,-0.5559787154,-0.4727867544,-0.9000478983,1.2659494877,0.4353246093,-0.0236592907,0.9436045289,-0.6716923118,1.0447865725,1.3527976274,-1.0631425381,2.4571130276,-0.1190586910,-1.3206595182,0.2101318240,-1.4100403786,0.7463139296,-1.5983033180,-0.1158415601,-0.3557868004,-0.1035592780,-2.1708722115,0.3732755780,0.8958846927,-1.5255739689,0.0184564441,0.5205706954,1.0403897762,-0.0308587700,-0.9651818871,-0.2936720550,0.0902625993,0.4703084528,-0.3178536892,0.3725000322,-0.5491930842,0.6766549349,-0.5850317478,1.9644398689,2.4331896305,1.3197941780,-0.5910408497,0.5551754832,0.7306241393,-1.0116714239,-0.7693768144,0.1477881372,0.5307281017,-0.5618506074,-0.3604781926,0.5028930902,-0.5547525883,2.3241920471,0.7301399112,-0.0238630101,-0.3112625182,-0.0513433814,0.0332490802,-0.3835499585,-0.3944619596,-1.0770056248,-0.8284639120,-2.2706835270,1.1904422045,-0.7383732200,-0.7856744528,0.2669454515,-0.4751650691,-0.3949821889,2.0143547058,0.1436878145,-1.2537568808,-1.7973768711,0.3158451319,-0.2288792729,-0.6633766294,0.3909885287,1.5668081045,-0.8202961683,-0.2624257505,0.0357421786,-2.8803546429,1.0226624012,-1.6296515465,-2.3747596741,-0.0938686728],[-0.4877255559,0.4899860322,-0.5391430855,1.3695386648,0.6441672444,0.7667090893,-1.3685532808,0.6206942797,1.2448111773,-0.8042119741,-0.5956533551,0.0014253046,-0.8887516260,1.6678035259,2.0496942997,-0.1034380123,-1.0603953600,-0.4802497625,0.1270350516,2.1686303616,-1.5617771149,0.4015067816,0.1245800033,-0.3452774882,0.8196158409,-0.1079464853,0.8407748342,2.0225064754,0.1155442074,0.5847235322,0.4388331473,0.1234409586,-0.2910670638,-0.6586142182,2.0476639271,-1.8610835075,2.2706551552,-0.1068520844,-1.3966004848,0.8132696152,-0.4033440351,-0.1480889022,-1.0043388605,0.3936049640,0.1324205846,0.5566450357,-0.1675183326,0.5736991763,-0.6849721670,0.9904925227,-1.8894196749,1.5981130600,0.7703539729,-0.1610601544,0.9238270521,-0.0515182950,-0.8868377805,1.0417042971,0.6102308035,-1.8677076101,-0.3840411901,-0.1164660156,0.0456374995,-1.8475975990,0.5355637670,-0.6964988112,0.7957055569,-0.1288603097,-0.3235868216,-1.6325315237,-0.5247679353,0.7241875529,0.7006774545,0.6510369182,0.2127076685,0.3019182682,1.7634776831,1.0604306459,-0.0160322711,-2.0777280331,1.1360679865,0.4793742299,0.4298013747,-0.9959617257,-1.0719579458,0.3770346940,1.4366446733,0.7699731588,-0.9030251503,-0.8420140147,-0.2891960144,-1.3440425396,-1.4827768803,-0.8795897365,0.8493244648,0.1213431358,2.4658744335,-0.5044754148,0.8377598524,-1.3034893274,0.4661247730,0.5391213298,-1.0591400862,-0.0326998606,-2.0926170349,-0.3239274919,-0.9163483977,0.7162846327,-0.0685323477,0.4281194508,-1.1246926785,-0.1664071679,-0.6484211087,-2.1521315575,-1.0705305338,-0.8617205024,0.2989324927,1.2873754501,-0.2156222463,-1.4065039158,-0.6083480120,0.0496303663,-0.0799441859,-0.3430354595,-0.5707333684,0.2139833272,0.1441838294,-1.6315569878,-0.4222122729,0.7287571430,0.4752489328,1.4714828730,0.7369390726,-0.5534162521,0.4637760520,0.8952754736,-1.3604298830,-2.1174213886,2.2016246319,-0.2555569708,0.5857450366,0.1036471277,-0.6595315337,0.5226187706,-1.8577984571,0.8471649885,-0.5092989802,0.8184267879,1.6758291721,0.4162685573,2.5203919411,-1.0321208239,-1.4705890417,0.5317592025,0.8014125228,0.9071531296,-1.5176868439,0.4063048065,0.7387344241,1.2877237797,0.3409604430,1.8790210485,0.9491173029,0.2067094594,-0.9716271162,0.3574623168,1.0916240215,-0.2871577740,-1.7678191662,-1.4303933382,-0.2483468205,-0.1160075366,-0.1474684626,0.4271591902,0.7722973824,-0.8035597205,0.2369090319,0.5585380793,-1.6493217945,-0.9682631493,-0.2224204242,-0.3922516108,0.8767914772,0.9763500094,-0.3122895956,0.9651550651,0.6296004653,-0.1026753411,-2.0676610470,-0.7020508647,-0.1088828817,1.3923741579,0.1279061884,-1.0830278397,1.7001254559,-0.7274262905,1.5997858047,-1.7580193281,-0.1680758595,-0.7228416204,-2.0124907494,1.3119337559,0.2128693163,1.7944684029,1.8756083250,-0.3741932511,-0.2608598173,1.0983625650,2.3297736645,0.3219519854,0.4034484327,1.2642672062,0.8221021891,0.4798425138,0.0399289317,-0.2851095498,0.5849317908,0.7732047439,1.2581137419,1.8532098532,-0.6349226832,0.4174361527,0.4140231311,-0.6864744425],[-0.1428144574,0.4959804416,0.7714887261,0.2804813683,-0.5753896832,-1.2983716726,-0.5563231707,-0.0272231661,-0.4999603927,-0.2157250941,-0.7681286335,-2.0508408546,-1.0966194868,1.6365146637,-0.5183550119,0.6196789742,-1.3453786373,0.5836728811,-0.4710648656,-0.5288444161,-0.2772250473,-2.3445336819,-0.6098615527,-0.3629501164,-0.1410931945,-0.4874231517,0.7105670571,-0.5983030796,0.5260640383,-1.0717720985,0.3782913387,0.7160261869,-0.8814377785,-0.2456881255,-0.6174123883,0.2108476460,-0.3944846392,-1.3366764784,0.0207539462,-0.5891013741,0.2604995966,1.6924297810,-1.1274348497,0.5514831543,0.6292963624,-0.7273381352,-0.1221026108,-1.3238046169,-1.8603323698,0.0118839173,-0.0494511090,-0.7719197273,1.1183060408,1.1271860600,-2.0702965260,1.1230894327,-0.5023233891,-0.5238962770,0.1952285916,-0.2513331473,-0.3816655576,0.6158700585,-0.7880089879,-1.8909306526,-0.7469483018,0.0433481671,0.1173256263,0.4212864637,-0.3761186600,-0.9839541912,2.0143651962,-0.3616572618,0.2649833560,-0.1013830006,-0.6988646388,0.6318305135,-1.3341007233,0.5088815093,0.2114373595,0.6875721812,0.2607598007,0.0316194594,0.3898535371,0.7478242517,-0.9930844903,-0.5001050234,0.8352737427,0.3108315766,-0.0316842087,-0.7823688984,0.1578320265,0.9126993418,-0.2352722585,0.9408181310,0.6109023094,-1.9303262234,0.5498456359,-0.6723092198,-1.4504003525,0.5355149508,0.3497037590,0.1889631450,0.1326940656,-0.6038624048,0.5095413923,-1.0285254717,0.2388528883,-0.2497551739,0.4331155121,-0.0697732419,0.6253356934,-0.1688880771,0.9176494479,-1.8235207796,-1.0847314596,-0.5756394863,0.3497035503,0.2342435122,-2.6334090233,0.5881343484,1.7326449156,0.4294790924,-1.3273426294,-2.0741112232,-1.2210987806,-0.1957486868,0.2234403342,0.4510703385,-1.2828766108,0.2367241681,-0.2933946550,-2.7297432423,-0.3880361617,1.3037126064,-0.3852106035,0.5652311444,-0.3321615160,-0.5173915625,1.3139929771,0.8717244864,-0.6854110956,-1.9135602713,0.6436522007,-0.8172343969,0.2767552435,-0.1575471610,-0.5616329312,0.4937283993,-0.3603928089,0.4888756573,0.4363217950,0.7434330583,1.0488585234,0.2988363504,0.8631799817,0.4911931753,0.5605632067,-1.2499432564,-1.8365777731,-1.5436005592,0.2294761688,-0.9752155542,-0.7505861521,0.8158047199,0.8112878799,-1.1278395653,0.1002079770,0.1714561433,-0.0902783796,0.1032009050,-0.4986415505,-0.1339252293,-1.1308209896,-1.2856656313,-0.6171336174,0.0464790165,0.8600177169,0.7068345547,-0.5624985099,0.4751739502,-0.3413746953,1.0102072954,0.0230765510,0.2426536977,-1.8445314169,0.7333210111,0.5530741811,0.2356170565,0.1776789725,-0.2307106107,-0.0811266378,0.7031916976,1.5869663954,-0.3827952743,2.0957953930,-0.0665417612,-0.5297813416,1.3718634844,-0.2815313339,0.2085584700,-0.3606263101,-0.4346255958,0.9304487705,0.6789392829,0.3673276603,0.1621620655,0.4276526272,-0.1493246108,0.8981700540,0.7842456102,-0.7916181684,0.1158784404,0.4618153572,0.3543474972,1.4618315697,-0.0058341846,-0.8900887370,-0.4046399891,0.9123374820,0.5789139867,-0.4968176186,0.1236161813,-0.4920249581,1.8629844189],[3.1188952923,2.1145527363,0.4688876569,-0.9595867991,-1.0293675661,0.6830950379,1.3771774769,0.1248709261,-0.4470041692,-1.2351502180,-0.0911605358,-0.3967476189,-0.0899577588,-1.3996888399,-0.5305298567,0.7499350905,0.5719929934,-1.4103066921,-0.3479000628,-0.2184364051,0.1737931967,0.0222933087,0.5647415519,0.8052543998,-1.0294834375,0.8431189060,1.5975235701,1.1170238256,0.6672995090,-0.8730934262,-0.5672993660,-1.2327417135,-1.4132713079,0.9281820655,0.7359776497,1.0769482851,-0.1153248027,0.2036904693,-0.8216454983,-0.7670912147,1.1390515566,-1.3738186359,0.3531615138,0.4086511433,-1.4268133640,1.0887863636,-0.4670736790,1.5938810110,-0.6779985428,-1.6609425545,-1.1497362852,0.0858812928,0.3081794083,2.7873344421,0.0017950319,0.7034717202,0.0402048379,-0.0539211519,-0.8005341887,1.2653822899,1.1613918543,0.0606069304,-0.9181155562,-0.2290524691,-0.1122236475,0.0549350791,-0.8877182007,-0.3766615987,-0.5949606895,-1.6478073597,0.8510466218,0.2196233720,-0.3607971072,0.8401561379,0.0543761253,1.4959983826,-0.5632653236,-0.2425057590,0.0648398325,0.7764827609,0.7312174439,-1.1121135950,-0.6776387691,1.0912896395,0.9742969275,-1.9399904013,-2.1802775860,2.1129066944,0.8291311264,-1.1868286133,0.5817332864,-2.1304891109,1.0515449047,0.4106768072,-1.2035242319,-0.7135511637,-0.4709822536,-0.5258688331,0.7505949140,-0.8277429938,-2.7819249630,-1.1523625851,-0.8028509617,-0.5026053190,-0.1736410111,-0.3603467047,-0.2416286469,0.1397867203,-0.1340719461,2.9043488503,-1.2880799770,-0.0549019389,-0.6406171918,-1.3738036156,-0.4437952936,-0.6030911207,-1.5600773096,-0.3178840280,0.7513098717,-0.0685902163,-0.5909373164,0.0339408405,0.0979840606,-1.9291602373,-1.1436108351,-0.0531030297,1.4900053740,-0.8120594025,-0.4823431373,-0.6355569363,-0.2808527946,-0.0437979996,1.9277224541,0.9682134986,-0.3746079803,-0.2915026248,-0.4581932127,1.6945850849,0.6297083497,-1.9033675194,1.3773635626,1.8451112509,1.3157194853,-0.4916698933,-0.7966602445,-0.3494751751,0.7045466304,-0.4455379248,-0.3833731711,1.3972334862,-0.5082765222,-0.1657868773,0.0295438245,-0.1786982268,1.2381014824,0.0978887677,-0.6584629416,-0.2192865759,0.9467751980,-0.0771661103,-1.0597406626,-0.1748822480,0.7894371152,-0.4872848988,2.2833466530,-0.5168839693,-1.3883464336,1.3372455835,0.2018319666,-0.2333852649,-0.3624745011,-0.5905905366,-0.7401220202,1.0021631718,1.2818202972,0.2926157117,-1.0420081615,0.4264304936,-0.0421497524,-0.5193667412,-0.8652704954,0.8540445566,1.0223616362,-1.1298987865,1.0555826426,0.0991955176,0.4002920985,1.1874296665,-1.5832803249,0.9813435078,1.2280945778,0.5722233653,-0.9180369377,1.3107286692,-0.2247706950,-1.1927497387,-0.7917345762,-1.1626080275,0.9272098541,0.5459935069,-0.6274787188,-0.0236869771,0.4932527244,1.1298884153,0.0539514609,-0.5518273115,1.1233708858,-0.0687913820,-0.1571341753,0.1179229915,-0.3325984180,0.2601344585,-0.1911605597,-0.4110339582,1.2266099453,0.2822836637,1.2508134842,-0.3510955870,-0.1616224051,-1.6587704420,0.1728632450,-1.3107187748,0.9391930103,-0.0142953722],[0.7546882629,-0.8996400237,0.9940255880,-0.2330982387,-0.4980289638,0.2528294027,2.6544928551,1.5781192780,0.4817900956,1.1860171556,0.3339211643,1.6164300442,-2.9351656437,0.2499151379,0.7760117054,0.7523182631,-0.1247479841,0.3462697268,-1.0257136822,-2.8691601753,-1.1061319113,0.7648423910,-0.9595790505,-0.0078468043,-0.2652287781,0.9949512482,0.5908301473,0.7397772074,0.2819495201,0.9627394080,0.1467755586,-0.7852812409,-0.3648975492,0.7329995036,0.9112494588,-0.6349129677,1.0441013575,1.1536734104,0.7727792263,0.1085980162,0.3048982322,0.2045264691,-0.0964837223,-1.1621612310,-0.1049908698,-1.8086529970,1.1371610165,0.3990722299,-0.2340318710,-0.8070739508,-0.5447965264,-0.9057466984,-0.7056326270,-1.3920232058,-2.2496838570,0.6314688921,-0.0317909122,0.5471078753,0.2917467952,1.1517522335,-0.2557494044,2.4834494591,0.1827259809,-1.1629668474,0.0617253222,-1.1677007675,-0.6916891932,0.1345950365,1.0560717583,0.7259826660,-0.9369471669,-1.2553867102,0.8967444301,0.0060753874,-0.1242959350,-1.3210841417,0.0899388492,0.8379094005,1.1313247681,0.3248853981,0.3055925071,-1.3190481663,-0.4589845836,-1.0276916027,-0.4143671095,-0.2306914777,-0.7529071569,-0.3986451030,-0.4679827392,1.0484802723,0.7876905203,0.3911187947,0.9792061448,-0.1854469031,-1.1137851477,-0.4035208821,-0.8948764205,-1.0624626875,0.4093474150,-2.7420592308,0.5685927272,0.1045850515,0.8899027109,-0.6605038643,-0.0338754356,0.4368726313,-0.5698662400,0.3048413694,-0.9810655117,-2.1477074623,-1.0738272667,-0.4583679438,-0.1335805207,-0.6811560988,0.6706511378,0.0815103278,-0.0946848914,-0.8044914007,-0.1711455286,-0.2423008829,0.8169430494,-0.7940458059,1.6181575060,0.2855857611,0.8950667381,-0.6162665486,0.1139321849,-0.6352536678,-0.0765659958,-0.7508867383,0.1942287982,-0.7451517582,-0.8653050661,-0.2339722216,1.0282814503,0.2243474871,0.0027308404,1.3209453821,0.2174192816,-0.2634245455,0.1675521731,1.2263836861,0.0566299632,-1.2975367308,0.3696041107,-0.9182784557,-1.3032855988,2.0671389103,-1.4618188143,0.4181285203,-0.1183236837,0.4636798799,0.7680442333,-1.1236658096,-1.3899666071,1.2378437519,-0.6141276956,0.0449728668,0.3780954778,0.5110079646,-1.5044722557,0.5645886660,-0.7422678471,0.9555368423,-0.8196920156,0.5401763320,-0.5338007212,-0.0242628828,0.1132424623,-0.6125180125,0.7859786153,-0.0349822044,0.0732879490,1.1536586285,0.1490181834,1.1301728487,0.7858697176,1.2088547945,-1.8960126638,-0.0871836841,-0.8826141357,2.8862614632,-0.8807513714,0.0494962074,-1.1702387333,-0.6589772105,-1.6818491220,0.9417175651,0.7542197108,-0.8279682398,-0.4384127259,-0.1200602949,-1.6220785379,0.1089254692,1.3359794617,2.4555397034,0.0173700582,-0.0874115005,-3.2741274834,0.1969607472,-0.2217420191,1.3908416033,-0.3902492821,-1.8578752279,-0.8916897774,-0.4912000895,0.9974804521,0.2037437856,-0.3271367252,-0.6474327445,-0.1907690018,-1.6302834749,0.6798949838,-1.6188380718,2.0891149044,0.4113075733,1.0672979355,-0.1462323815,-0.1751722842,0.3902901411,-1.0913881063,-1.5157181025,0.9157704711,0.7945489883],[0.7722069025,0.4243962467,0.9691821337,1.4120765924,-1.0809820890,-0.6280459762,-0.6338055730,-0.4000754654,-0.7342044711,2.2752947807,-1.5868879557,-0.5589603782,-1.3043125868,0.3147029877,-0.2022144794,0.8512126803,-1.4063329697,-0.6932082772,1.4493491650,0.0560668260,0.9236955643,-0.5947605968,-0.2050286680,-1.2416573763,-0.6073765755,0.4851435125,0.4411651194,-0.1564729959,0.2195530385,0.3204922378,-1.1177259684,1.0289072990,1.1188204288,-0.0411395468,0.6005723476,1.5676715374,-0.1191437095,-0.5251495838,-1.5412362814,-1.2549991608,-1.7545846701,1.3483644724,-0.9670109749,0.8642841578,0.3205362856,-1.7713440657,1.1340459585,-1.0328261852,-0.1334087253,-0.3448979259,1.2247921228,0.7575573921,0.6027758718,-0.6131908894,0.7743551135,0.5759877563,-0.1214414164,-0.0009457574,0.8447682858,-1.7850353718,-0.1575969607,1.1574027538,0.1245122924,0.6179469824,-0.9759038091,0.1979667097,-0.5714919567,1.1229913235,0.3113834560,-0.4361872375,0.0621054992,-0.8684849739,1.5532866716,0.4511200488,-0.5184523463,0.4275344014,0.4822023511,-0.5793940425,-1.2044733763,-0.0556597859,0.3327965736,0.1595038027,1.5718646049,1.1590943336,-0.7317303419,-1.0252996683,0.3034447730,0.0385221504,-1.4219295979,1.6513744593,0.5207285881,-0.5225039721,-0.5949496031,-0.2466515452,-0.7081933618,1.7684363127,0.2115738690,0.1667692959,0.9263616204,0.5199217200,-0.8082281351,2.0617651939,-1.6870878935,-0.5724027157,-0.0029290456,-0.8472099304,-0.3068249524,-0.7209590673,-0.4516503513,1.2118517160,-0.5764548779,-0.6438353658,0.3038035929,-0.4023731649,0.5642825961,0.1920437813,-1.0712314844,1.2746276855,1.6927058697,0.4486570656,1.3200535774,0.0160080902,-2.3641381264,0.5951035619,-0.1099106371,-0.1813999414,0.6511411071,-1.0492880344,-0.9243508577,-0.5737870932,-0.9042535424,1.7057296038,-0.6014401317,-0.3643075228,-0.2186646163,0.1627379209,-0.8163861036,-1.1460425854,0.9978126884,1.1337602139,-3.2709360123,0.9469668865,-2.5101165771,0.6990010738,0.8786799312,-0.5839535594,-0.6743929386,-2.4371728897,-0.0503076240,0.8705804348,-0.2608596385,0.0898400098,-0.3027840257,-1.2079818249,0.0692925304,-2.0684671402,-0.0220023058,0.3078320622,-0.7879421711,-1.0444283485,1.2165591717,0.1952220052,1.4552105665,1.0593975782,-1.0149605274,1.2875981331,-0.4663090110,1.5024882555,0.9685600400,-0.2157099992,0.2602674961,-0.9132912755,0.0151141891,0.4012723863,1.0047228336,-0.9285416603,-2.0817165375,-0.2928998768,1.5027165413,-0.1005974039,-0.3758979440,2.0350098610,-0.4853464663,2.9407486916,-0.5036264062,0.2076288760,0.2921140492,-1.1655380726,-0.9878220558,1.4906018972,-0.8561943173,1.8389580250,-0.1549417078,-0.2442326397,-0.7554346919,-0.8123953938,0.4190253019,0.6878544092,-1.6918449402,-0.0898365825,1.0332804918,1.4314122200,-2.1892902851,0.0718886927,0.7476850748,1.8579891920,-0.8819519877,0.9258650541,-0.9806548357,-2.4932901859,0.6940491796,1.6708992720,0.1564088911,-0.5294389725,0.7051659226,-0.4034671485,0.8142341971,1.4990714788,-1.3586189747,-0.5893127322,0.2642396390,-1.9446407557,-0.2521522045,-1.6591632366],[0.6953535676,-1.0761489868,0.5135737658,-0.8951653838,-1.4820936918,1.6783659458,-1.3350443840,1.3742983341,1.4251997471,-0.4624463916,0.2494481504,0.8023421764,-0.5172142386,1.5071295500,-0.1262708008,0.1829024106,-0.3753479421,-0.7461107969,-0.9378911257,0.8594884276,-1.3996262550,0.2003267109,1.0123081207,-0.7833893895,-0.3972645700,-0.6808474064,0.4937455356,-0.5152196884,-0.7318217754,0.2788423002,1.9877916574,1.0418436527,-0.6853435636,0.0596736781,1.0391395092,0.7398166060,-0.9924521446,1.1051650047,-1.2520617247,2.8686056137,-0.1694054306,1.9018117189,-1.0746030807,-1.2265579700,-0.8510357141,-2.1456563473,0.6359826922,0.5066269636,0.9674390554,-1.1224880219,-1.0891484022,-0.1846278459,-0.5367618203,1.8058052063,-1.1354886293,-1.9588290453,0.7469393015,0.1361483037,0.1992729604,-0.8409942389,0.1434929073,1.8949650526,-0.5321903825,-0.2382396460,0.0410348624,0.5797135830,1.5907917023,-0.2758657932,0.1504357457,-0.5587373972,-1.4018324614,1.4326224327,-0.6219562292,0.4503026307,0.4008595347,0.3934817910,1.2696865797,1.8898544312,-0.1165970489,0.6457780600,2.2075743675,-0.9847111702,-1.2228060961,-1.0882190466,1.0747212172,1.1284934282,-0.7757837176,-0.1252997965,-0.1738859862,-1.0792155266,1.7511061430,-0.1438600868,-0.9502803683,0.2619571090,-0.2889937162,0.0514364801,0.5962271690,-1.3032315969,-0.3775050342,1.3788658381,-0.5376601815,-0.3634388149,-0.5220135450,0.6863537431,-1.0816643238,0.8089571595,1.1289632320,-0.1704909503,0.0241376441,0.9559123516,-0.2972258627,1.2451685667,-0.7107396126,-0.4264647663,-0.2931824625,0.5185298920,-0.1059233323,0.6447357535,0.7338988185,0.5817427039,-0.6184529066,0.1341575831,-0.4200459421,0.8393083811,-0.3927664757,0.4707723856,-0.0673559830,0.3716157675,0.3322022855,0.0640629455,-0.7132993340,0.0691948086,-0.8344638944,-0.1031417176,-0.4747044742,-1.6690871716,0.6154571176,-0.6817176938,-2.0737211704,-0.3878850639,1.4523063898,-0.6992966533,-0.4581898749,0.1936287731,0.2572551072,1.4544843435,-1.6571730375,-1.7270624638,0.0417574421,-0.9110734463,-0.0972457156,1.5952712297,-0.8988962173,0.5708675981,2.3974201679,-1.5280815363,-0.1108923852,-0.3071365356,-0.9979051352,-0.1661987454,-1.0751403570,-2.5248780251,-0.2498296946,0.9437659383,0.9217237830,0.7171913981,-1.4578920603,1.2408026457,-2.4817552567,0.2420867234,-0.3377861381,0.0360229239,-0.5133728981,-0.3208512962,-1.3301918507,-0.6975618005,0.6498871446,0.8788000345,0.0332687683,1.0863420963,0.9952074885,1.0205569267,-1.5511695147,1.1222990751,-0.2344319969,-0.6389635801,0.2854325175,-0.7309339643,-1.9008569717,-1.2902566195,-1.4089726210,-0.0378140882,0.8407415748,-0.6611420512,-1.2427229881,-0.5391062498,-0.7145795226,-1.0734699965,1.3682132959,-1.0783115625,-2.2720541954,-0.7096360922,-0.2335556746,1.0443377495,-0.4696555734,-0.3875656724,-0.8236765265,0.2266024798,0.1619909555,-0.0944048539,1.6084132195,-0.3988568187,0.1957289577,1.5395137072,-0.1689223051,-1.5008987188,-1.3173402548,1.1269226074,0.3471261263,0.3052436709,0.3154096305,0.0921308771,0.1269512475,1.8258371353],[0.1363988519,-0.9507242441,0.7849888802,0.3052140176,-0.2709810436,-0.8993048072,0.2169116437,0.1867516041,0.7598631382,-0.8012745976,-1.1976956129,0.0239624623,0.3074752390,1.2831456661,-1.3402698040,-0.4516152740,-0.0013061600,0.4203725457,0.3012847304,0.2762047648,-0.9804582596,1.1610437632,0.5745239854,0.6747921705,1.4261837006,-0.5042745471,0.5857908726,0.8970323205,0.1667999625,-0.4011804461,1.7648248672,-0.8087148070,-0.2742171586,0.4606503844,-0.0885505006,0.2718375921,-0.8004333973,0.6516600847,1.3673778772,-1.5335977077,-0.0962385684,-0.8232393861,1.6774146557,0.2271115631,-1.5541958809,0.3077866137,-1.7232172489,-2.4555950165,0.8917973042,1.7451490164,-1.7838643789,0.6180729270,1.0859361887,1.1453979015,0.9061566591,-0.3431972563,-0.0705898181,1.0991281271,-0.2261981666,0.3282406032,1.0704436302,-1.1716351509,0.9848763347,0.2437052131,1.9884727001,0.4174208343,-0.1021214947,-0.3435896039,1.3310985565,-0.5973044634,-1.0328304768,-1.0166631937,-0.2418605387,-1.0737953186,-1.5078314543,-1.9658055305,-2.4816696644,0.0847596675,0.9371824861,0.5712789297,0.1847821474,-1.6016809940,-0.5153174996,0.3281519115,-0.0960697681,-0.7360475659,-1.3864686489,1.0394927263,-0.0865700766,-1.2883691788,-2.0292968750,-2.0399634838,0.6458489895,0.4366787672,-0.2919606566,-0.8011184335,-1.1781026125,1.2439732552,0.7523661256,-0.4787828326,-0.1176917329,-0.3622176647,-0.9125710726,-0.3694320023,-1.2082426548,-0.5479109883,-0.8315254450,-0.7423962951,0.1000797302,-0.2114170194,-0.6813454032,0.8390816450,0.4109891355,-1.2436894178,0.5534462929,-1.4438923597,0.8049284816,-0.8555861712,2.1477587223,-1.3949433565,-0.2616662383,-0.9542909265,0.1196367517,1.5450251102,-2.3118369579,2.7454116344,-0.5705853701,0.0806028023,0.2696255445,1.1088483334,-0.6098455787,0.2533075511,-1.6156879663,0.4766279757,-0.9498989582,1.3911211491,0.0607453063,0.3484802246,0.2540907860,-0.9823598862,-0.6200393438,0.6513434649,0.6371821165,1.2473409176,0.4007479548,-0.6293382645,-0.3995212615,-0.3992503285,2.1516108513,0.8495406508,-1.1839305162,-1.5761988163,-0.3321847916,-0.6933479905,-0.5461527705,0.2077549547,-0.0470405109,0.9233145118,-0.3973110914,0.0615227558,-0.0844099149,-1.7535184622,2.8657610416,1.7541314363,-2.3690114021,-1.1117147207,-0.9374561906,-0.7801645994,1.7342389822,0.5957742929,1.4701598883,-0.4405146837,0.6708141565,-1.2153530121,-0.1325355917,1.0050604343,-1.2139691114,-1.3798935413,2.3798618317,-1.9887228012,0.3868185282,0.8959914446,1.6038486958,1.5363543034,-0.7630540729,-1.4995023012,-1.8495545387,-0.2919710279,-1.5776841640,1.7685606480,1.1297414303,2.3360943794,-0.1433131546,0.0687443092,0.5931475163,0.3375712037,1.2554653883,0.2474237680,-0.9403129816,-0.2061767131,-0.4198792279,0.3141739070,1.7077875137,-1.4152840376,0.1187793538,-0.7630447745,-0.6395958662,0.0102028055,0.1042164192,0.1145903766,-1.4752571583,0.2472184002,-2.1607117653,-0.4515348971,1.2877726555,0.8726791143,0.5221362114,0.0558537394,-0.7062091231,-1.0879805088,0.5266829729,0.1596421599,2.6657764912,0.8143402338],[0.9257330298,2.1734650135,0.5997606516,-0.1436795443,0.3496937156,-0.9546177983,-0.1620016545,0.9732399583,-0.5467224121,-0.2262014300,0.6731067300,0.2159280777,-1.3774738312,1.1656154394,0.6550341249,1.9122684002,-0.9768811464,0.9356287122,-1.2053273916,2.4479525089,0.6917709708,-1.5790313482,-1.4568662643,1.2126783133,0.6390294433,-0.5516815782,0.8256443143,0.4482912719,-1.8841407299,-1.3875689507,-0.1513385475,0.4152318239,-0.6240538955,-0.2882059515,-0.1251111031,0.0627681017,0.2525975108,1.4493167400,-1.2855988741,-0.0614147782,1.2215783596,0.3575677872,0.3007901907,-2.1066825390,0.6250435114,-0.6612740755,-0.1341212541,0.2166793793,-0.9497941732,-1.2139436007,1.2116563320,-0.9395774007,0.3062827587,1.1016186476,0.0750297382,0.1435457021,-0.8186900616,-0.4630457759,0.5266259909,-0.0092119304,-0.5354993343,-2.7169761658,-1.1486746073,0.2088882625,-0.6807827950,-3.0617961884,-0.3313309550,-0.1758318841,1.4389998913,-1.4673558474,-0.7488585114,-0.0289274994,-0.3636911213,-1.2437512875,-1.5300729275,-1.1790323257,0.3857260942,-1.5467594862,-0.1613754779,1.4048904181,-0.5613690019,-0.7192589045,1.7300932407,-0.5238711834,1.0657387972,-0.9189323783,-0.5509738326,-0.5299234390,-1.8563530445,0.7000102401,1.3967370987,0.1085478887,0.9389107227,0.6333724856,1.0824018717,1.2884047031,-0.4879322350,1.5856468678,-1.3458198309,0.4698221385,0.9904260635,-0.7995867133,0.4108642042,0.8107822537,1.3460342884,0.7748122215,0.6732077003,-2.0124661922,-0.1145293713,-0.4353868663,1.0582432747,0.4740674198,-0.8044639230,-0.0220145155,-1.1738559008,0.1407169402,0.1516216695,0.4735494852,0.9077764750,0.9353476763,-1.6902123690,-0.0617531240,1.0522574186,1.1403518915,0.4502256215,0.3180954456,-0.8513272405,1.7308795452,-0.9569784999,-1.1495845318,0.0731476098,1.3367174864,-0.1736793220,1.0153282881,0.7569540739,-0.4068577588,-0.9060294628,-0.1295568198,1.8163871765,-1.3072237968,0.6215203404,0.6559920907,0.6238611341,0.8602724671,1.3712561131,1.7863894701,-0.1854007989,-2.0719249249,1.9330905676,-0.2852199376,-1.1075080633,-0.1679364741,1.5029848814,-0.3086349368,-1.5096340179,-1.3769066334,-1.7379231453,0.3038515449,0.5172749758,-0.6259486675,0.7043590546,0.6408320069,0.4185501933,-0.1646103710,0.3187517822,0.8196350932,-0.4609601498,0.3704506457,2.3777439594,0.5094441772,0.5521851182,-1.3909140825,1.1121610403,0.8421571851,1.1006048918,0.6569547057,-0.8560597301,-1.2483242750,-1.7567801476,-0.8713760972,0.2235589772,-0.2905384302,0.6802270412,-0.0276177991,-0.9667946696,-0.3455171287,-1.4121873379,0.2059963942,-0.6952722073,-0.7459636331,1.0301487446,-0.6903392076,-0.1190134883,0.2581081092,0.2567854524,0.7716189623,-0.2509430051,1.0092092752,-0.5644721389,-1.3445675373,1.3076673746,-0.9773423076,0.3826100528,-2.6880400181,0.4268960953,-1.1517989635,-0.8691557050,-0.6256707907,0.5513302684,-1.1145905256,0.8849994540,-0.8217748404,0.9004893899,-0.5353831649,-1.2508430481,0.7113912702,-2.6005218029,-0.7853864431,0.3203434646,2.5373580456,-2.3225741386,-0.2459121794,-1.1617994308,0.7776864767],[-0.1386852562,0.8962039948,-0.3026582301,0.0292508341,-0.2019753456,-1.3439552784,1.8885324001,-0.2357038409,1.0730520487,1.1235663891,0.1953596920,-0.2643015087,-0.0000123232,0.8602338433,-1.4039980173,-0.5216125846,-0.0435859486,0.3864676356,-0.6059318781,1.2893551588,-0.7495974302,-0.7594406009,-0.2251145393,-0.8615306020,-0.3031273782,-0.7514218092,-0.0263201371,1.0012412071,-0.4263814092,0.3582890928,0.3132919073,-1.7429720163,-1.0658556223,-0.5562067628,0.6188708544,1.2195292711,0.2364767790,-0.7100563049,0.6203096509,-0.9709291458,-0.7118019462,-1.3894826174,0.1876191944,0.7731317282,1.7870222330,1.3483870029,-0.2876165211,1.2424424887,1.0495731831,0.2199662328,-0.6506385803,1.4809036255,0.1745363027,0.5330241919,0.2584872544,1.7948721647,1.2031013966,0.2966628075,1.0648564100,0.2883406579,1.7498236895,-0.3993926644,-0.5972360969,2.3710198402,-0.1631287187,0.6456073523,0.1387225240,0.9679862261,-0.4325176179,0.3984373510,-0.1760327667,0.6858507991,0.1566565484,0.7886711955,-2.6216802597,0.6207202673,0.0285687726,0.6761151552,-1.3356418610,0.7205585241,1.2382751703,-0.6817107797,-1.1695361137,1.7380303144,0.8015923500,0.2585445940,0.3077518046,-1.6452590227,0.4001936316,-0.7441572547,-1.4529861212,0.2506214380,-0.0533802733,1.1744852066,0.7447588444,-0.3415437043,-1.0891363621,0.6034783721,-0.1482316405,-0.4128837585,-0.0329800732,-0.7001284957,-1.2899012566,-0.7876139283,1.2155237198,0.0087155225,-0.0377709866,-1.3937059641,0.7941431403,0.1870978177,0.6264303327,0.8145871758,-0.7728582025,-0.2493467182,-0.5739196539,-0.5565655828,-1.1334906816,-2.2793505192,0.4410243928,0.4123649299,0.0763852969,1.4099802971,-0.7776322365,0.9002335072,-1.7927150726,0.3387904763,-1.5426032543,0.1109750047,-0.1004541144,-0.7143050432,-0.9074044228,0.1946927160,-0.6922547221,-0.5126389265,-0.2491152287,0.8359513879,-0.5823138952,0.1778424084,-0.1264958382,-1.2308934927,0.0432014875,-1.7107870579,-1.2794522047,-1.4112915993,0.2468446046,0.2457828671,-0.3186832964,0.7732830644,0.9469830394,0.5959275961,-0.2578312159,0.3627173305,-0.3071349859,-0.1745721549,1.1378543377,0.7938080430,-0.5381634235,0.4013376534,-0.9682885408,-1.1399817467,-0.9329072237,-0.5803301930,0.1786864400,0.6346725225,0.5647393465,0.0286316723,0.0992199630,-1.0894353390,1.3608300686,-0.4605360329,-0.8283517361,-1.8477090597,-0.7479529381,0.4846981764,-0.8102111220,-2.7513906956,-0.8704450727,0.0546552204,0.9808416963,-0.4524506330,-1.1581110954,-1.8916167021,1.5128071308,-0.2710370421,-0.0432773046,1.0887061357,0.8253607750,0.9289151430,-0.1334707141,-1.1068481207,1.0271012783,-0.4083566964,-0.9534019232,0.9252843857,1.2556301355,-0.3255678713,0.6176265478,-0.1261923760,0.7121532559,1.1629872322,-0.7318741083,-0.0021026048,-0.6439540982,0.7954640985,-0.5480089784,-0.9157254100,-1.1337063313,1.0404101610,-1.9267649651,1.2752271891,-0.6368082762,0.1870581359,0.9406405687,-0.3973135352,-0.0837642848,0.1083026305,-0.1242073551,-0.2883494198,0.0421624370,0.0474995077,0.6321162581,-1.5836554766,-1.1032531261,-0.5887776017],[-0.1524914503,-1.6673083305,0.9382166862,-1.0333873034,0.4561629593,1.1972030401,-0.6820040345,1.9431983232,-0.2159686685,1.8049536943,0.5602540970,0.0748924091,0.5605863333,-1.7343138456,0.6010981798,0.4802421331,-0.8561789393,0.1013424918,-0.4451296926,-0.3943363428,-0.0749234781,1.0080494881,1.6752249002,0.3181442618,-0.2150738686,-0.0598962717,-0.6142386198,-0.4861352444,1.5881255865,-0.0372432843,0.1362068504,-0.6435383558,0.3061073124,-0.1080607101,0.9142783284,-0.6750063896,0.1442564428,-1.9035853148,-1.9769639969,-0.2643598616,-0.1542481184,0.6130289435,-0.9667575359,0.4809058309,-0.8523111939,-1.5576803684,-1.7187989950,0.3377987146,0.7529808879,-1.0509667397,-0.7730994225,-1.2046824694,0.4276761115,-0.8403949738,-0.4825850129,-1.4747754335,0.7216343880,0.4391319156,-0.2775903642,1.8374639750,0.8860115409,0.5017250776,0.1924185902,-0.3279211819,0.5628093481,0.9199988246,0.2608970404,-0.1163030937,0.9977791309,-0.1468842328,0.8259064555,0.8571233153,-1.1635246277,-1.2943441868,-0.6337320209,0.0078612473,0.2800707519,-0.4688892663,-1.5412379503,-1.2810702324,-0.9014613032,0.7189544439,0.6765769720,-1.1631652117,0.4434119165,0.9910751581,0.3494699001,-0.4065314829,-0.3706806600,-1.8773276806,0.2003043443,-1.7980109453,0.4200835824,-2.1893362999,-0.2583111823,-2.0468666553,-0.2437464893,1.0890002251,-0.0624531172,0.3706884086,0.4278266132,-0.1703085750,0.8178662658,1.0959708691,0.5633809566,0.4105978310,-0.4165204167,0.4329338372,-3.3593549728,-0.7902920246,-0.5976426601,0.2041165531,0.7518958449,0.9105246067,0.6583688259,0.8852194548,-1.0034178495,2.1018614769,-1.3160889149,-1.2050503492,1.3702383041,-0.9215651155,1.1412268877,-0.5294218063,-2.0968155861,0.9883624315,-0.4318448901,0.9150150418,0.6191473007,0.9672363997,-0.5683717728,-0.9324068427,2.0865449905,-0.2670659721,0.3249341547,0.2030726075,0.2451879531,-1.6215312481,-0.5928865671,1.1713583469,0.9839522839,-0.0872299895,-0.3621385992,-0.9829427600,0.8884603977,-0.1793822050,0.5179939866,1.3586529493,-0.9791272879,0.3234313130,0.5289196372,-1.9766300917,-0.0034390925,-0.7666292787,0.7933413386,0.8603540659,0.9405009151,0.6027413607,0.5473909974,0.6982603073,0.3897379637,0.3500108123,0.1734568626,-0.6509557366,-1.7006138563,1.1740543842,-0.2860489190,0.7598106861,-1.4153439999,1.2704614401,-0.1627092808,1.0294121504,0.4012169838,2.0456750393,-0.0367478020,0.6690829992,-0.3728141785,-0.2102022916,0.0779081285,-0.9818955064,0.1598586440,-0.7887569070,-1.6699723005,1.1914888620,1.7864699364,-0.5499423742,0.5733178854,-0.5447788239,-1.7513740063,-0.9034187198,0.5524613261,-0.1586202979,0.0848929808,1.8023984432,0.4640448987,0.3638080955,0.2646439970,0.1547241062,-1.4154069424,1.4598560333,0.8310509324,-0.4224171937,-0.0165275075,0.2615670860,-1.1916067600,0.2750308812,0.6544116735,-0.3800196648,-1.8660432100,0.2894992232,1.1039299965,0.1959735453,-0.0791411027,0.1106350198,0.1640786827,-0.1299720556,-1.6187663078,-1.1956890821,1.1165030003,0.6533244848,0.3864424229,0.2913912237,1.0975930691,1.4230506420],[-0.0867393687,-1.6922757626,-0.2748315632,1.2141071558,0.5547963381,0.0646452904,-1.1866117716,-0.0320456326,-2.1211907864,-1.9493031502,-1.0958501101,-1.0897547007,-0.1803523153,0.3207212687,1.2608158588,-0.5926650167,1.2983776331,-1.7294532061,-0.4939606190,-0.8452877998,-0.4836902916,0.6845428348,0.1369161159,1.1284254789,1.4544321299,1.5625711679,-0.1469358951,0.1927613020,-0.7056970000,2.1935093403,-1.0594838858,-0.1013154238,0.1026509628,0.8066827059,-0.2695352435,-0.5729221702,-0.3849198520,-1.0156600475,-1.5466378927,-0.1314595789,0.3548667729,1.6491620541,-1.0673861504,-0.5086348653,0.5900128484,0.5537997484,-0.9028851390,0.3827905357,-0.4903090894,0.1471827030,0.5189530253,-0.3839559853,-0.0064701196,0.0774409026,1.1993261576,-1.8423675299,0.8041459918,0.0865443349,0.9438959360,0.4559169412,-1.9510413408,-0.0187463369,-0.5971686840,-2.1307387352,-1.0078853369,-1.1159837246,0.0679549426,0.3910646141,0.6450982094,0.2970682085,-1.4807668924,1.1750893593,0.6418598890,1.7684969902,-0.5421749949,0.0230772309,-0.5444252491,0.6334091425,-0.1794523001,0.8945975900,-0.3239262700,-1.0767819881,0.3691432476,0.6942480206,-0.0642852709,-0.8203831315,-1.1774146557,-0.2505559921,0.8347317576,-0.1928883344,-0.5916996598,-2.2447366714,1.4082713127,-0.2975973487,-1.0574928522,1.0062141418,1.1215678453,0.6743121743,1.5402876139,0.5045122504,0.4825377464,2.3591935635,-0.1581576616,1.3925447464,-0.4449972510,0.6436666846,0.6480388045,0.6492596865,-0.3070269525,0.5588863492,-0.8348108530,0.7839213610,0.6336535215,-1.4643620253,-0.8206500411,-0.7092571259,1.5029654503,0.9617568851,-1.5836352110,1.0906035900,-0.5501468778,-0.9884999990,-0.1372122318,1.2027401924,-1.1109777689,-0.5006306171,-1.2436493635,0.8456214666,0.9922400117,-0.5578774214,-1.5881406069,-0.0128620062,0.7343695164,-0.6115727425,-1.6536990404,-0.0360801816,-0.2093567550,0.7303119898,0.3931238949,0.9025912881,-0.2072690576,0.2031912059,1.1792848110,-0.8231495619,1.9168200493,0.2706981301,-0.7323203087,0.8667071462,1.5296094418,0.2382735461,1.0494682789,-0.3883500099,0.1169275641,0.9171454310,-0.1054672450,0.0179931466,-0.9391468763,-1.3003677130,-2.8754465580,-0.1241761670,-0.1267907768,-1.4475177526,-0.0179561060,-1.7879973650,0.7654473186,-1.1107738018,0.1945697963,-1.0288122892,1.6509739161,0.1266491860,0.4879723191,-0.2690182924,1.3367944956,0.5753632188,-0.0968106240,-0.9281479716,-1.2006169558,0.0849150419,0.9885011911,-0.5190308094,1.0386395454,0.1456762850,0.4775098860,-0.4643878937,0.6115694642,-1.2517410517,0.8366264105,0.4481097162,-1.3137805462,0.1005480215,-1.1128195524,-0.0262514874,1.1235595942,1.6695141792,-0.1654336303,0.7355045676,-0.8682185411,0.0562831461,-1.4327114820,0.0250446890,0.0975233465,-1.1841610670,0.7637282014,1.5198569298,-0.7049927711,0.8754818439,0.3610333502,-0.6621111035,0.2180406004,-2.0024406910,-0.8863433003,-0.1631226391,-0.3874653280,0.5548259616,1.1022121906,1.0524899960,-1.2551360130,0.9903590679,0.8982601762,0.8245859742,-0.3155473173,0.0082518505,1.3746151924,-1.0334572792],[1.6923905611,0.3971545994,0.2894242704,-0.5514086485,-0.3389030099,-1.0362766981,-0.5482236147,-3.2779073715,2.1528837681,0.1756819487,0.3480467200,1.8974837065,-0.7496224046,-0.3268892765,1.3972829580,-0.3456662893,1.4522365332,-1.6095561981,-0.0210978836,1.5160992146,1.0965744257,-0.2358710319,0.6267479062,1.7261326313,-0.3205836713,-0.1390050799,-0.7758511901,2.0753962994,0.1757676750,-1.7555754185,-0.2505002618,0.2840710282,-0.5393370986,-0.4869838059,-0.1303175837,-1.1189780235,-1.0521354675,-0.0231210198,0.0181847401,0.2078745663,-0.0338686258,-0.4818850160,0.0961250514,-0.9138529897,0.9418380857,1.3300968409,1.5610092878,0.4450918734,-0.1296869665,-0.2519723773,-1.5859483480,-0.9557150602,-0.2045584917,0.0128896115,-0.2095763832,-0.4511358142,0.5498040318,1.8048099279,-0.0429422446,2.2919325829,0.0038200787,0.2721864283,0.4414996207,-1.3005160093,0.1768380255,-0.9363270402,0.4566507041,0.3796249628,0.4649937451,0.7009580731,1.0819793940,-0.5470877290,-1.1549626589,-0.2274688035,-0.4943396151,-1.7293069363,-0.1460026801,1.5145381689,0.1582663655,-1.7069764137,-1.0666083097,-2.0992667675,-0.3120955825,1.5823105574,-0.2526357174,0.6126517653,0.7511574626,1.9334533215,-0.1653770506,0.0007328019,-1.1496319771,1.0879269838,-0.4144269824,-0.1782887280,0.2705714405,-1.3183277845,-0.7594566345,2.8128893375,-1.7517324686,-0.1413636059,0.4365155399,0.8948895335,0.3178620934,-0.6415089965,-0.0946490541,-0.0807673633,-1.0004671812,-1.5345644951,-0.2357628345,-1.5820407867,1.0702680349,-1.8865656853,-0.0901094601,-0.0760608092,-0.1922454238,-0.5639578104,-0.5438117981,-0.2419601381,0.9866610169,0.3484502435,0.2144006640,-0.5236511827,0.2716521621,-1.1190679073,1.0188894272,-2.0297060013,1.5627001524,-1.2868920565,1.2256826162,0.2894935012,-0.7158158422,-1.3701560497,-1.0779329538,-0.0256836489,0.3960071802,0.4327994287,1.2228806019,-0.1757212430,-1.3786426783,-1.9703372717,-0.5863664150,-0.0080045890,0.4041854441,-1.3040728569,-2.0099992752,0.8277711868,0.7524700761,-2.8358297348,0.3762879968,-0.0772758722,-0.1378519088,0.0027876387,-1.0244839191,-0.3540144861,1.7863705158,-0.3465160429,-0.5774444342,-0.2480545044,-1.1455147266,-1.4086068869,-0.7194760442,-0.4547182322,1.7560970783,-0.0071993573,-0.6203646064,-0.1874046624,1.5937808752,0.5413727164,-1.4341344833,-1.6528888941,-0.2458195090,1.1216475964,1.8508564234,-0.0923150182,0.1672917157,-1.5417040586,1.4463324547,0.5009288788,2.8749001026,-0.7061102390,0.0816367939,-0.2914707661,-0.0285660401,0.0983645543,0.2963778079,-0.3110705614,0.8581354618,-0.7723867297,0.8028447032,0.2162812054,1.0721161366,0.0564372241,0.6005313396,-0.1831871420,-0.4489869177,-0.9456906915,-0.0371265896,0.4959995449,-1.7010400295,-0.6790872216,0.0206640214,-0.1886631548,0.1214666665,0.5371381640,-0.6261289120,-1.1847596169,-1.1002988815,1.4077138901,0.3684228063,-1.1791006327,-0.0171466414,0.1485635340,-0.1480452716,-1.2246426344,1.0717194080,0.5924861431,-1.5882186890,-1.8824286461,-0.7987720966,1.6857186556,-1.9873899221,-0.6040264964,0.9996188879,0.7584842443],[-0.5855907798,1.0974962711,-1.0957789421,2.0606269836,-0.5865968466,-0.8793839812,-0.7873229384,-2.5463385582,-0.6912784576,1.2159292698,-1.9346731901,0.3178563118,-1.4861592054,-1.6317461729,-1.2586566210,0.9521486163,-0.2169671655,0.7148294449,0.7958927155,0.9093554616,-0.1415878832,1.3334403038,0.5828847289,0.8194612265,-0.1050379872,0.4659985602,0.6288198829,-1.1417542696,-1.3998000622,-0.3798598349,-0.4254036546,-0.3694149554,0.5209847093,0.8141931891,0.2401541024,2.4843487740,-0.5280447602,-0.9274998307,-0.1653894037,-0.1363191158,-0.8200877905,2.1133174896,0.8898195028,0.8735603094,-0.0686016232,-0.9245443344,-0.7718877792,0.4842652678,-0.2915898860,0.7432041168,-2.1153192520,0.9568510652,-0.2215748578,-1.8422077894,-1.2303696871,-0.1241944283,-0.7307086587,-0.6680089831,-2.3341002464,2.2076082230,-0.2407692969,0.5288803577,0.2479257435,-0.1337349117,-0.5434597731,-1.7410125732,2.0310096741,1.1825866699,-0.5783034563,0.2525941133,1.1653379202,1.0282540321,-1.0832226276,0.1212414354,-1.0867486000,0.2782225609,1.0789091587,1.1128096581,-0.7514941692,0.4164353907,-1.0873526335,0.5424474478,-0.2048412859,-0.2030930966,0.5863599181,-0.8489893079,-0.3559145629,-0.5795920491,1.5493372679,0.4460066557,0.0156894699,0.1472632438,-1.3260676861,0.7402061224,-0.6366640329,-0.3111352026,-0.1385994852,0.0820364133,1.1148211956,-0.0950735360,0.2773782015,0.9351531863,0.1982546598,1.3031610250,-1.2071324587,0.5444293618,-0.4473699927,-0.4108420610,-0.0568518825,0.4197574854,-0.6447360516,0.3293173313,2.5226984024,2.6850180626,1.9191200733,0.0876055658,-1.8289321661,-0.3188391030,0.1750170290,0.7548211813,0.6596356034,0.3386530280,1.7292153835,-0.1219225526,0.6411107183,0.7875508666,0.2660456002,-0.1457971931,0.7125737071,1.7682244778,-1.1409661770,0.6875343919,-0.9086990356,1.6409335136,0.8345350623,-0.9138651490,1.2705948353,-0.2345217317,0.4959183931,-0.6939529777,-1.3865205050,0.0969705433,-1.4421837330,-1.9025647640,-1.1793817282,-1.9167758226,-0.3548213243,-1.7836306095,0.4976726174,0.9922037125,-0.0509538837,-0.7649970651,0.8111050725,0.1019998342,0.2750899792,-0.0343734659,0.9133414030,-1.3589154482,0.5406615138,-1.3692927361,0.7702706456,-0.8848457336,-1.3492389917,1.5141055584,-0.8034302592,1.4701484442,0.7194578648,-0.3280103207,-0.1737902015,0.3155027628,0.8393989801,-0.7880792618,1.4342643023,-0.2792739272,0.7217347622,-1.1199485064,-1.6488090754,-1.1588809490,0.2449669391,0.9734916687,-0.8612310886,0.2776378691,-0.1442465335,0.2899083197,0.1926309317,0.2951425612,0.1448704749,1.0455939770,1.5262594223,-1.8003038168,0.2336754799,-0.9080817699,1.2704061270,-2.1403088570,-0.9260414243,-0.1840798110,-0.3402787745,-1.6307851076,-0.2041860521,-0.1278241426,0.5095170140,-0.6179625392,0.0498966575,-0.4110610485,-0.5157909989,-0.6018150449,-0.4285067618,1.0633392334,-0.7848201990,0.1631879508,-0.2149524540,-0.3020791411,0.6054601073,-0.0504952818,-1.0965297222,-0.7816454172,-0.2012111992,-0.6565100551,-0.1104105487,-0.2088105083,-1.9115939140,-0.2466346174,-0.2519866526,-0.4590988457],[-0.0494616181,-0.6155298948,0.2704988718,-1.8088672161,0.7919092178,-0.1851701885,0.4661017060,-0.3695489466,-1.0320540667,-1.1022082567,-0.6764156222,0.4022864997,-0.2559004724,-1.2627768517,0.2470270246,-1.8833502531,-1.2920002937,-0.5564384460,-2.3891890049,-1.1999177933,0.7154252529,-0.1808043569,0.1423880011,-0.0719283819,1.1965063810,1.0246064663,0.2847369015,0.0961781964,-0.2521937788,0.7456894517,-0.0504549034,-2.1432087421,1.7391443253,-0.0206905380,0.9230058789,0.1440964341,0.3340483606,-0.5639219880,1.1893047094,0.8010678291,-0.0857279226,-1.6603605747,-1.8865714073,-1.5158432722,2.2416851521,1.3034378290,-1.2954757214,1.3497186899,0.6987150311,-0.5378327370,0.9773977399,-0.6292833686,-0.2258862704,-0.0231768172,0.2287255973,-0.0307552647,0.2602588534,-1.1946291924,-0.4701487720,-1.6524150372,1.5998086929,0.8542186022,-0.8980391622,-0.3424322009,-0.7620440722,0.5442741513,-2.6877915859,-0.6870270967,-0.1067433506,-0.5056813359,-1.1792836189,2.6766495705,-0.7369825244,-0.4661085606,-1.5385745764,-0.0672003403,0.3695683777,-0.7869833112,0.4003332257,0.3783661425,0.5367749333,-0.5993347168,-1.6679698229,1.3760275841,-0.1786604971,-0.0802356526,1.0228990316,-1.9961047173,-1.0266917944,-0.5655768514,0.8560996652,-1.4708665609,0.7181494236,-0.1992621869,-0.3848416209,-1.4284293652,0.0078248149,0.0301324613,1.0360671282,0.0559637584,0.7436648011,0.2172421217,-0.4548107982,-1.8581829071,-0.2697221041,-0.7858588696,0.2048793733,-0.1674392670,-0.5469172597,1.3462063074,1.0827282667,1.1228549480,1.0326079130,-0.9947468042,1.4552406073,-0.3257787228,-0.7388832569,-0.7861909270,0.2828303277,0.3056680560,0.2146909982,-0.2772153020,0.3958362639,-1.3230555058,-0.8603905439,-0.2827332020,-0.3374444246,1.5174309015,-1.3406960964,-0.7753478289,-0.6090564132,0.4148904383,0.1143402532,0.6358146071,-1.0189952850,-0.2470776290,1.8384991884,0.2994590998,-1.3782683611,-0.9034779072,1.0411815643,-0.8380994797,-0.0788478255,-1.2593122721,0.0735025629,-1.2232187986,0.0042018746,0.4956058562,1.0793445110,0.3841406405,-0.5286387205,-0.8840914369,0.1363095641,0.2901922762,0.5978170037,0.2368114442,-0.1575534791,-1.1024419069,1.1252269745,1.5449730158,0.3530938923,0.7228652835,0.1168344319,-0.0158468988,-0.1661573797,0.5381613970,0.3136244714,0.0325458795,-1.2324377298,1.0933052301,-0.1038841233,-0.2963826954,-0.9586788416,0.7200127840,1.4924416542,1.7752082348,-0.0550629981,-1.6644073725,-0.5077100992,0.2773285806,1.0041874647,0.4742742479,0.4997110963,0.2690739036,1.5814677477,1.5842721462,1.1153169870,-2.0206856728,-0.6906757355,0.8339030147,-2.4003989697,1.7535443306,1.4520895481,0.5084596872,-0.1650930792,0.1084622741,0.5147604346,-0.7298306823,-1.9664291143,0.6762105227,0.5794365406,0.7264261246,1.2572396994,-1.8034398556,-1.4639929533,-1.3501420021,0.3714019656,0.7283082008,0.5316600204,0.4573111832,-1.6541035175,1.7793091536,-1.6135524511,1.4555981159,0.8226402402,0.7780967951,0.1016966924,1.1728751659,-0.4354915619,-0.4268640280,0.8830320835,-0.7973753214,-0.8927229643,1.0299344063],[-1.4835407734,1.2919406891,0.9132064581,0.3831681311,0.3834024370,0.9610646367,-2.4891815186,-0.8557786345,0.2069403827,-1.5988606215,-0.9463294744,1.0044388771,0.5957759619,0.3084153235,0.2008418739,-1.1726783514,0.2280961126,-2.1500291824,1.9621980190,0.0185094886,0.4452189505,0.9216417670,-1.0115605593,2.1943533421,-0.1640067250,-0.4602047801,-0.2296462357,0.9627048373,-1.0101568699,-1.4202458858,0.4832316637,0.0462592840,-1.5302428007,-0.3245599866,-1.0381706953,1.2537368536,-1.6339751482,-0.1652428210,-0.3389556408,-0.0146003719,-0.7025259137,2.6607224941,-0.6276010871,-1.0365874767,0.4586429000,-1.0496587753,-0.2999071777,1.3842225075,1.0823534727,-2.3419559002,0.8761150837,-1.8273769617,-0.2215786427,-0.3491138816,-1.9336280823,-0.0474423543,-2.4403247833,-1.2015104294,-0.2873283327,-0.8806145191,-0.8555080295,0.0481235869,-0.8469879031,-1.1857453585,-0.9132013321,-0.4592289925,0.0136311175,0.6643366218,-0.0491968133,0.2498437613,-0.8947763443,-0.0383777991,0.0546499304,-0.3900771439,-0.4424458146,1.0053803921,-1.6437406540,-0.2351907790,0.0837585628,-0.8065138459,0.3854508698,1.7295747995,1.0369310379,1.0545397997,-0.1897726953,-0.6609727740,-0.6300903559,1.0252816677,1.3816955090,0.0301201232,0.4552626610,-0.0021572479,-1.2396833897,1.6422270536,-0.9795646071,0.3572402298,-1.4496890306,1.4168978930,-0.8599024415,-0.1259931624,-0.8222708106,0.3623867333,0.1708000451,-0.3181733787,-1.9245707989,0.0009699502,-1.5193839073,0.6653414369,0.2419777066,1.6192805767,0.9414151907,-0.6941750050,0.4210597575,0.2198369801,-0.5225008726,0.5529451966,-0.1377501786,-0.0109302262,-2.6844522953,1.0686687231,-1.0907949209,-1.6353297234,0.2023297548,0.6188598275,-0.2628114522,1.0546277761,-0.1554549485,-0.4642913342,-0.0471217670,-0.0422636569,0.7548336983,0.4991921484,-2.0590631962,0.3868599534,1.4013535976,-0.9825395346,-1.3489254713,-2.1613402367,-1.1077320576,-0.0115719670,-0.1067917049,0.2030382305,1.0208203793,-0.9721820354,-0.8007465005,0.7475744486,-0.1014548764,0.1717854142,-0.8900861740,-1.1787598133,0.2836291790,0.6105752587,0.0714654922,0.2032693624,-0.0388820209,0.0560920835,0.2171260417,1.4609282017,0.3397671580,0.6166418791,1.0261284113,1.1398640871,0.0585809164,0.3839966655,0.1210960820,-1.4486052990,0.9515943527,1.1694599390,-0.4625164866,-0.5766818523,1.3717787266,0.4527533352,-1.0115367174,-0.2389606684,-2.1135818958,0.9200044870,1.2121772766,-0.0131268874,-1.0547879934,-0.4625411332,-1.1228022575,-1.9511053562,0.2523438334,0.0335213430,3.1744377613,-0.3097344339,-0.0796215609,2.3843185902,0.1680177748,1.1310538054,-0.2883731723,1.0993871689,-0.0557040833,-1.3803638220,-0.1418625414,-0.0994756371,-0.3607069254,0.1630599797,-1.8716735840,0.5809935331,-0.0887572840,-2.0139195919,-0.1107501611,-0.1171881333,0.0923445001,-1.1246478558,-0.0643767715,-0.6560503244,-0.0982639194,-1.1385906935,0.3597646356,0.7687802911,1.0907343626,0.9932415485,1.5794206858,-0.4556078315,-0.0845103040,0.1953143030,-0.4470531642,-0.3148096800,-0.5831922889,0.7251600623,-0.3664899170,1.1142308712],[-0.2529630065,0.1108265817,-0.0751002133,1.6626136303,0.1881306767,1.1572419405,-1.5092159510,0.4438718259,1.0644911528,0.0690576583,-0.5484440327,-1.1120855808,0.2062901407,1.7126512527,-0.6370053887,0.4516954422,-0.6429297328,-0.4736404121,1.9491024017,-1.1057304144,-0.0583164655,0.0027293719,0.0433787964,-0.9135349393,-0.7275012732,0.1881026775,-0.6704826951,0.3486644924,0.7248296738,-0.3045236170,0.4539546967,0.5046046376,-0.4356022477,1.3943544626,0.8857178092,1.4404188395,2.6176686287,-0.5996223688,0.9162171483,-0.5749520063,0.0330234617,-1.6783542633,0.2071019560,0.2668921053,-0.2316341698,2.5724213123,0.0771679357,0.0226416700,-1.0104405880,2.0703089237,0.2559503615,0.3081150055,0.4898057878,-0.8981673717,0.1386307776,-2.3951253891,-1.0491285324,-1.4085648060,2.2214100361,-0.0909142420,-1.2632048130,-1.3839141130,-0.4299320579,1.6684657335,2.2021830082,0.5257831812,0.5739874840,1.2139933109,1.1535638571,0.1974461824,0.5228608251,-0.4663076997,-0.8205826879,1.2096979618,0.4772752225,-0.2883346677,1.1037921906,-0.4656058848,-0.0572867543,0.1929015964,-1.2480163574,-0.2613046765,0.3618875444,-1.8665893078,0.4445092082,0.3430060148,-0.3016341925,-0.6712490320,-1.9647588730,-1.4762607813,0.6007431149,-2.1796936989,1.7869907618,0.4041797519,0.7283347845,0.2149192095,0.0791092888,-0.0756125897,2.0927040577,0.6142238379,0.6972085834,-0.2808259726,-0.6297846437,-0.6537150741,-0.6563082933,0.0635813102,1.1474578381,-1.3920365572,0.8224881291,-1.2620323896,2.2960383892,-1.5349198580,-0.7157036066,1.8768742085,0.6480322480,-0.0833342448,1.9466234446,-0.9626903534,0.1901564449,0.2450418174,0.6438478827,0.5414466858,-1.6009700298,1.2528609037,-0.5713541508,-0.1698438674,-0.0962642208,0.7907935977,-0.3675736487,0.2545646727,-0.1731839627,-2.0102469921,-0.0823436901,0.4109261632,0.9124290943,-0.5108228922,-0.1702895910,0.5690754056,0.1842122078,1.1614910364,-0.7973006368,-1.7424471378,-0.1686261892,0.9480429292,1.2066537142,0.9066698551,0.6930658221,0.3705970645,0.5162458420,0.6608072519,0.4866171181,-0.4750882685,0.9606438875,-0.1774173826,-1.1164638996,0.6565501690,0.2430785298,1.3326601982,0.2276160568,-0.7638772130,-0.6724875569,-1.3945460320,0.7054424286,-0.3470198810,0.7814444304,-0.9647708535,-0.8465443850,-0.5008096695,-0.5663548112,1.0299090147,-1.2219821215,-0.9223409891,-0.0413170531,0.7327345610,-0.4150656462,-0.3697193563,1.3278313875,1.1300793886,0.8595128059,-2.1058371067,0.4117957950,1.4069085121,-0.8631376624,0.9940280318,0.0828463808,0.6927403212,-0.5555388927,-2.1278247833,-0.2419712394,0.2147070467,0.8770046234,-0.8252259493,-0.3071250319,-0.4251047075,0.4736202955,1.4847452641,0.7876216173,-2.3725543022,0.8668217063,-0.5885546207,-2.0502035618,0.1267351955,-1.3493905067,0.6182358861,0.1972312331,-0.8721260428,-1.7464439869,1.3869823217,0.1516992599,-0.0960684121,-0.5976011157,-1.6084567308,-0.1104261652,-0.3266306520,0.9532612562,2.8422458172,0.7323229313,2.3837733269,-1.4173332453,-0.2441341728,-1.0598491430,0.0659141317,0.1451799124,-1.3646274805],[-1.3189326525,1.0868996382,1.1407871246,-0.7739912271,0.2757473886,1.0186146498,0.3553026319,0.0994194523,0.4954006076,1.3101518154,0.9655687809,-1.3013892174,1.6685962677,-1.6873700619,1.2311973572,-0.7829509974,-0.5541905165,0.2968224883,0.1373548359,-0.3030631542,2.0867037773,0.2923654318,1.0142391920,-0.3932260871,-0.0352618359,0.5864498615,-0.5295163393,0.6852313280,-0.1643852890,-0.3441419303,0.2067484409,0.1120802909,-1.8413914442,1.0845415592,0.7977465987,-1.6193389893,-0.3356162012,0.5115650296,0.1433262229,0.6185114980,1.3554399014,0.7320787907,0.7957426906,1.4062149525,-1.7205719948,-0.2503417730,0.5847280025,-1.2508269548,0.0660410076,1.6999821663,-1.6192164421,-0.5963765979,-2.1174936295,-0.8630749583,0.3964715302,-0.6267700791,-2.0068068504,-1.7333778143,0.3489642739,-0.3829937875,2.0327324867,1.9875861406,0.2677262127,-1.4474372864,0.3494439423,-0.8642144203,-0.5014429688,2.0740737915,-0.3882977068,-0.4317372143,-0.4059042037,1.0080896616,-0.6950193048,0.5184195042,-0.9137551188,-1.4409933090,-0.7854880691,1.8851081133,-1.7892971039,-0.8158497810,-0.3761253655,-0.2095122784,-0.7854520679,-1.0257658958,0.3675001562,-0.7535496354,1.1216030121,1.0277554989,0.1140085235,0.3061859906,-0.8787401319,1.1658949852,0.1283266991,-0.2367148250,0.6065468788,-0.2742420435,1.1453546286,-0.0845240206,-0.0997853056,-0.4292584658,-0.9366860986,0.7969179749,-0.0418951027,-1.9022171497,1.3403178453,-0.2650252581,-0.1395718455,-0.1130125746,-0.2644657791,0.8363052011,1.4173330069,-0.0139653347,0.0140969520,-1.3867497444,-0.2896922827,1.4394404888,0.0371060893,0.4729910493,-1.6448743343,1.9283530712,-0.0540869758,-1.5597012043,1.3833075762,1.4959639311,-1.2113410234,-0.8035944700,-0.0742011741,-0.0012471536,0.5972085595,-0.5049758554,-0.8977391124,-2.4309682846,-0.7694957852,-1.2447004318,1.8496736288,-0.1600045711,0.3851931393,1.6098812819,1.3233433962,-0.5853709579,-1.4895451069,-1.6313624382,-1.0789774656,0.1750666648,-0.8689029217,-1.2971252203,-0.1602051109,-0.4810235202,0.8413378000,0.4719519913,-1.4436874390,-0.4337729514,0.1231140718,-0.1029697135,-0.2789793313,-1.8605657816,0.9095398188,-0.3967422843,2.4552280903,0.7007195950,-1.0971454382,0.1571447253,-1.6993728876,1.1674585342,-0.6445943117,-0.3044758141,0.6822560430,-0.6443675160,0.2809943557,-0.8231162429,0.3418474495,-0.9646231532,-0.9446858764,-1.1312217712,0.2950759232,-2.0998363495,-0.9623675346,0.5983527899,-0.5706901550,-2.8149156570,-0.9644336700,-2.2830367088,-1.4362345934,-2.5027389526,-1.0116815567,-1.3881716728,-1.0808703899,1.1213512421,-2.9426026344,-0.8580161929,0.8869637847,1.3504574299,1.6489382982,-2.0368595123,0.4648270607,0.4125159681,1.5089983940,0.1617003679,0.2972085178,0.3435890079,1.2503364086,1.6973853111,0.5223636031,-0.6074130535,-1.4800554514,1.0115360022,1.2797082663,-0.9638424516,0.0238593630,0.7001688480,1.0566946268,-1.2123521566,-1.5452622175,-0.7135031223,-0.2995873392,1.3495112658,1.0317986012,-2.1518487930,-0.2608613670,0.4728642404,0.1432889700,-0.5136743188,0.8816134334,0.4160916507]],[[0.9742739797,-0.5911037326,0.5096975565,0.7329760790,0.1781499535,0.4407790899,-0.7507807016,0.9301936626,-0.1136841103,0.2096375674,-0.3831914961,0.1437592804,1.9191459417,1.1668577194,-0.0390432589,0.7552515268,0.7185825706,-0.1992563307,0.2191366404,0.4416317344,-1.7806103230,-0.5829799175,-1.6506224871,-0.3800404370,0.5495235324,-2.4511246681,0.2122054845,-0.6588183641,0.2381319553,1.4138728380,0.1893429458,-0.2653420866,-1.4341834784,-0.0134915356,1.0526467562,-0.4494713843,0.3941705823,0.4576940835,-0.6274823546,-0.4867767394,-0.3145232499,0.7940014005,0.2057987899,-0.9098162651,0.3779762983,0.4768439829,0.5508027077,-0.2443206012,0.3771245480,0.9358108640,1.0886226892,0.9422910213,0.5694215894,0.1278016269,-1.6078619957,0.9920114875,2.0334231853,0.8720038533,1.2157073021,1.0835859776,0.0771418586,0.9433466792,1.1806234121,1.7133400440,0.8349636197,-0.3409295380,-1.4217534065,-0.9918186665,1.3714574575,-2.8138613701,1.5928198099,0.0514569990,-0.3350365162,0.1681042314,2.0830357075,0.1723357737,-0.7525523901,-0.6203945279,-0.4310478270,0.8727289438,-1.6561784744,-0.0651366413,-0.2049668729,-0.6367900372,-0.4473701417,-0.4513550401,1.0316622257,-0.6272398233,0.8326088786,0.6830444336,-1.3253782988,-0.5727469921,1.1235693693,0.3231909275,0.7557680011,-1.1440552473,1.2230265141,0.4768500626,0.2638700008,-0.3666841686,-0.1768282652,-1.6205668449,1.7069422007,0.8102005124,0.3011316061,1.5844978094,1.3711355925,-0.9427987337,1.6457824707,-1.0328581333,-0.2671949267,-1.4802416563,1.9849010706,-0.5626236200,0.0849812925,0.2822172940,0.7228338718,-0.7783156633,-1.2473411560,0.2193592489,1.0145031214,-0.0329789929,0.2228203714,1.1445417404,-0.7094033360,-0.7841330767,-1.0221625566,-0.5994387865,2.6038806438,-0.0352037288,-1.0069522858,0.7614061236,-0.3151522279,-0.4787478149,1.0351151228,0.9559835196,-1.6696609259,-1.5952768326,0.8844750524,-0.1393707395,-1.9339787960,0.5813475251,1.3983411789,0.7789682746,0.6822609901,1.1423463821,-2.8080708981,1.2361961603,0.1793654710,0.2740286887,1.7551431656,0.8033042550,1.5408403873,1.2425165176,-0.3679214418,-0.4071923494,1.0675301552,1.0157678127,-1.6916062832,-0.0623279475,-0.0539634116,-0.8613557220,1.2769105434,1.3883987665,-0.0332604498,-0.3398702443,-0.6552860737,1.4018083811,1.0553525686,0.1255802065,-2.8219633102,0.5146217942,1.4022740126,-0.0046099331,0.8136485219,-0.9195903540,-1.3112980127,-0.4670576155,1.4047838449,0.5764878988,0.3580507934,-1.5716396570,0.6349464655,0.7771478295,1.0435512066,-0.3213168979,-0.8127188683,-0.2205226868,-1.2179155350,-1.9275535345,1.7975726128,0.5448293686,0.7044969797,-0.3992608190,-0.5087509155,2.1094746590,-0.7843719125,-0.2291737050,-0.7270941734,-0.7784538865,0.4416177571,-0.2448096573,0.8384055495,1.4899570942,-1.1972079277,0.3306650817,-0.5491546988,0.3423768878,-0.7401416302,0.6068357825,0.3473240435,0.3175152540,-0.0742286593,0.3980647922,0.7580665946,-0.0345872045,-0.8518533111,-0.4360183477,0.6210014224,2.7143337727,0.7998730540,1.5790160894,-0.4784846306,-1.3615520000],[-1.1311392784,-0.8737077713,0.4509058297,-0.9993168712,-1.9487090111,-1.0683714151,-0.6897867918,0.1995844096,0.6471304893,-0.7248669863,0.5458530784,-0.0195752960,-0.6879757643,0.0733593181,1.4149322510,0.6252916455,-1.2030237913,-0.8833094239,0.4984725118,-0.8347005248,1.1032197475,0.3863393962,0.4053649902,-0.3788195848,0.0324828625,0.3399371207,2.4221336842,0.5677464008,1.5449701548,1.0187703371,1.1051464081,-0.0787986144,0.7276761532,-1.7961450815,-0.7372569442,-0.4961169660,-1.8179187775,0.4991966486,0.4435710609,0.6528558731,0.0615191497,0.5835493803,1.1446419954,0.1317789853,0.4888918102,0.7580172420,1.5825773478,-0.4546769559,-0.4600296915,-0.2378547490,-0.0576866977,0.9007138610,-0.5783562660,-0.9360529780,0.8583307266,-1.1411458254,-0.2024374753,-1.1240649223,0.5279912353,-0.4751847982,0.6631527543,1.6318333149,0.0969477370,-0.4408459961,-0.4316680431,0.5758231282,-0.7765583992,-0.3135903776,0.1644244492,-0.9466973543,2.4841709137,-0.1300010383,0.9978062510,0.7827248573,2.0063924789,1.5243164301,-0.3948339224,0.0537491739,0.7989837527,0.2240976840,-2.4085738659,0.7977293134,0.8477253914,1.0057256222,0.8337087631,-0.4097576737,-0.4895608127,-0.6883573532,1.0892106295,-0.0431515090,-0.6405534744,0.6923466921,2.7715198994,-0.0121520748,0.4013367295,0.2561378479,-2.0776169300,1.7508301735,1.3922433853,-1.0740218163,0.5858798027,-0.8643900752,1.1791176796,0.5617208481,-0.2729750872,0.4421900213,1.9618458748,0.1082156897,-1.0202893019,0.6061573029,-0.9337441325,-0.6937928796,-0.6611673832,0.5429068208,-0.4369900823,0.4420260191,0.2651502788,0.1907901913,-0.1864834428,-0.6720966697,-0.1947799325,-0.4772402048,-0.3070857525,0.5616519451,0.4010860026,1.2816871405,-0.1345330328,-0.3634308279,0.2095918655,0.5743795633,0.5211598873,-0.3771933019,-0.5893455148,0.4899389446,-0.5980780721,-0.2535832822,0.0282106660,-0.7644257545,-0.4702759683,1.3134050369,-0.1494729519,-0.7538666725,0.0136112459,0.0777157620,-0.1643834263,-0.2342753261,0.8124720454,-0.6758717299,0.1944087893,1.5166828632,-1.3424445391,-1.7649502754,-0.8097494245,-0.0012850497,1.4531955719,-0.4727535844,-0.8327094913,1.0155029297,0.3293326497,-0.6450911760,0.3764901757,0.1045299768,-0.1445874125,-0.2936522067,0.9235664010,-0.2783232927,1.3137370348,0.0172006525,-1.3841036558,1.5561493635,-1.8639020920,1.0495033264,-0.5255575776,-0.1408940107,-1.9543750286,-0.1556380987,0.1417855322,-0.2631154954,-0.4426996112,0.5870085359,-0.5000515580,-0.5050081611,0.5463028550,-1.0903365612,1.5628612041,-2.7882788181,-1.3344719410,0.8784140348,-1.6840302944,-1.8166384697,0.0311815683,-0.4051390886,-0.0798487365,-1.5385483503,0.9227674007,-0.5343394876,0.0507932045,-1.4572614431,0.3584845960,0.3457713425,-0.5961431861,-0.6469150782,0.1584200561,0.4568893313,0.3769147992,0.4354852736,-0.5486419201,-0.2239340246,0.6698993444,-1.0490089655,0.5744723082,1.5347912312,1.5397765636,-0.1221618131,0.5470622778,-0.6007783413,-1.2336231470,0.9368577600,-1.1393072605,0.0572789684,0.5984082222,-0.1399062723,-0.9106681943,-0.8067533970],[0.1479307711,0.9215002060,0.3112674356,1.3075610399,-1.7026318312,1.3720930815,1.2918262482,0.5612526536,-0.9967388511,1.2426233292,-2.2951490879,-0.0152674960,-0.9043204188,-0.5141695142,0.4446947277,1.1492751837,0.0521705337,0.6528357267,-1.2749031782,1.0507358313,-1.0188709497,1.7837384939,-0.9868627191,-0.2265499085,0.2017863989,-0.9687949419,-1.9731079340,0.1696194410,0.5233901739,1.4001473188,0.4637394249,0.7948895693,0.0983602554,-0.4224130213,-1.2093416452,1.1387294531,-0.9182177186,0.5414411426,-1.1186860800,-1.8909265995,-3.0632636547,0.4568120241,2.0387444496,0.4237249792,-1.9272114038,0.9853472114,1.3389278650,-0.1318121105,-0.8920201063,-0.2773597240,-0.4517774582,-0.7616629004,-1.1646945477,0.9879109263,-0.3608299196,0.4053466022,-0.4165501297,0.3574571311,-1.2699836493,-1.6300295591,0.5598648787,-1.4674304724,0.4131439328,-0.4219754934,2.0304031372,-1.4413354397,0.4799681306,-0.0420383178,1.6915184259,-0.5232487917,-0.8039526939,-1.0601407290,-1.5404548645,-1.0749971867,-0.2996744812,1.3575615883,1.0603859425,1.0530307293,0.1001334339,0.3675707281,1.1156344414,-0.7672547698,-2.4473090172,0.3544388413,0.7223312259,0.8279894590,-0.6757613420,-0.1550412029,-0.0579219274,0.1946964562,-0.6854659319,-1.0218989849,-1.1483523846,-1.4241236448,0.5684677362,-1.2633242607,-0.5404099822,-0.6181235313,0.8761564493,0.2720177472,0.6743654013,-1.3917897940,-1.3827804327,0.0717442855,0.1175106540,2.1652770042,-1.1354564428,-0.6457980275,0.0317389071,-0.7400742173,0.9619795680,-0.0593260564,-0.6544432044,-0.1337867081,-1.2988967896,-0.3555890620,-0.6045878530,0.2226153016,0.9822761416,1.1462154388,0.2895379364,-0.4104592204,-1.1960166693,-0.1224415079,1.1972869635,0.6265248656,0.3610025048,-1.7972739935,1.3773232698,-0.6522130370,-0.7375943065,-0.4990181625,-0.3867849708,2.1639332771,0.1491455138,-1.3630967140,1.8294426203,-0.8609524369,1.4427920580,-0.4151449502,0.1516723335,0.2960756719,0.3308898211,0.4983998239,0.1312209517,-1.0052082539,-0.8411090374,-0.6049742699,-0.0483746789,1.0027732849,1.0322577953,0.2257779986,2.2114481926,0.2201044112,-1.3947371244,-1.0045163631,0.5725672245,-0.4024380744,-0.1606439054,0.5383403301,-0.9939112067,0.4404547215,0.4778611660,-0.7972090840,1.2639709711,-0.0904689953,0.0119346408,-0.2128958404,0.1294909567,0.6935775876,-0.8069289327,0.0381242223,0.0640896633,-0.8519039750,-1.1961863041,-1.6041916609,0.3708937764,0.4002925456,0.6246642470,-0.3795958757,-0.2372878045,0.5787289739,-0.6773391366,0.1989562958,-0.0726481527,0.6693786979,0.5971757770,-0.1213830337,-1.7431809902,1.4139084816,-0.3425191939,0.2435627282,-0.0741600394,-0.9220610261,1.0343546867,-0.8678551316,-0.3198132217,-0.6899349689,0.4179919362,1.2124581337,0.0375941433,2.2486476898,-1.0220209360,0.6465313435,0.4278988540,0.9193848372,0.4222237170,-0.0933749005,-1.0670326948,0.7879496217,-0.9432157278,-0.9096699357,-0.7628371716,-1.6910816431,-0.1087954193,1.1553153992,-0.2835746109,0.9786539078,-0.1439840198,-0.5912532210,-0.7626441717,0.2139990479,0.8680096269,-2.1934850216],[0.7954643965,1.2878558636,0.3072288930,0.9573891759,0.5701214671,1.2529802322,-0.0815569162,0.8128978610,-0.9811226130,0.2701521218,0.9489287138,-1.9259961843,0.2753039300,-1.3883477449,-0.5270357132,-0.2019829303,-1.1113721132,0.1056974530,0.4965963960,-0.1771866232,2.4821672440,0.8626716137,-0.1475741863,-1.8463534117,-0.1793068349,-0.8907314539,-0.4722703099,-1.5586285591,0.1077410355,-1.9330302477,0.7558904290,0.2947430611,-0.6818034053,0.8157358766,-1.2121231556,0.3502351940,-2.2251851559,-1.0909916162,-0.0607460439,-0.7485980392,0.5062255263,0.2378816903,1.2893656492,-0.6041642427,-0.6925551295,-1.1607208252,-0.2876462042,1.0311911106,-0.0499792546,-1.3574689627,0.8249968290,-0.8970457315,-0.1135150492,0.2143322527,1.3763307333,0.8509597182,1.7496386766,0.6001273394,1.4015005827,0.3860958517,0.5434395671,-0.4725933969,-0.2877027690,-0.6129447818,-1.4196848869,0.3062418699,-0.1553191394,-0.9508764744,-0.0760941654,-0.8062614202,-0.8954066038,0.0484242700,-1.0906312466,-0.1732535660,-0.3397390246,0.3176231682,-1.5844293833,1.2997056246,-0.6607711315,0.1271229833,0.0425397344,-0.4639439285,-0.9103606939,0.3158217371,-0.3238605857,1.6766819954,0.9372931123,0.2473313659,-0.8097632527,0.0890621245,-0.8043243289,-0.3928592503,-0.3789885342,-0.7877340317,-0.7610335946,-0.4013801515,-0.0189523175,2.3299083710,0.8038778305,-1.9815123081,-0.7011249065,-0.9096357226,0.9332553148,-0.2004610598,1.7981244326,-0.0330081321,-0.2495200932,0.8726334572,-0.6514548063,-0.5128629804,1.6354423761,0.3905225694,-0.8390223980,0.8316834569,-0.0595287457,2.0173168182,-0.8562936187,0.6865690351,1.3743298054,0.3907682300,-0.2398041636,-0.1251194626,-0.6344032288,2.2980499268,-0.4630606771,0.2461500913,1.3341190815,-0.3485464156,1.2087879181,0.1016988084,-2.1994295120,0.1064224765,-1.8571059704,1.5093195438,0.1532788873,-0.5053347945,0.3831263781,-0.1220782623,1.1170195341,1.0353970528,-0.7229524255,-0.3387831151,1.6032934189,1.7609171867,-1.2161747217,0.5180673003,-0.6836446524,-1.6527055502,-0.8767384887,1.0462769270,0.2828881741,0.4827982187,0.2307805121,0.7823326588,0.9507107735,1.4595805407,0.6798090935,-0.8676077724,0.3908489645,1.0838125944,0.6227587461,0.0919146538,0.6267312169,0.7369835973,-0.4665488899,1.5379717350,-1.0313144922,1.0398963690,0.8687855005,0.2055855989,-1.7716643810,0.2428264916,0.1570801437,-0.8153182268,-0.2022062540,0.5724145174,0.5329394341,1.6678758860,0.5379707813,-1.2548235655,1.5401520729,-0.2270848900,-1.1637035608,0.7588675022,0.6385750175,0.2570756972,1.0582165718,0.3091140985,1.6547634602,-0.2367554605,1.2576045990,0.0599439889,0.0446613617,-1.8018302917,-0.2758919001,0.6777992249,-0.1838328093,-0.9692851901,0.9890301824,-0.2224184275,0.5610032678,0.7233167887,0.0912953764,-1.4654313326,-0.5956678987,-1.0447202921,-2.0769333839,0.9551025629,1.1798607111,-0.5978704691,1.7572724819,-0.1310936958,-0.2408575267,-1.5555117130,1.5630480051,0.4313979149,0.6554265618,0.0821268037,-2.1948366165,1.2969125509,0.8677711487,0.3639212549,-0.5787127018,1.5105884075],[1.0996694565,-0.5503646731,-0.4026169181,-0.0874640346,-0.4514757693,1.4189465046,-0.4797129035,0.3733192980,0.2657961547,-0.8370972872,0.3150520921,-0.9210009575,0.1015821472,-0.5309605002,0.0664992332,-2.0074801445,2.2297945023,0.0819370598,0.1479992270,0.1758866161,0.8668915033,-1.0901758671,-1.8159155846,0.5403104424,-0.1107598692,1.0183703899,1.9810978174,-0.4993706942,-1.5936735868,-2.0820102692,0.8839993477,1.2424896955,-0.3754470050,-0.4873703122,-1.3391138315,-2.8575394154,0.8945623040,-0.0517046452,1.1353893280,1.0172095299,0.1628982425,-0.3798680305,-0.8960040212,-0.2011121809,-0.0567418709,0.2119727731,0.6210333109,-0.1791775525,0.8862378597,0.8935353756,0.8979471326,-0.8102669120,0.0424867161,-0.6294496655,-1.0798614025,-0.2119414806,0.7185308933,0.8276394010,-1.5393027067,0.7995367050,0.6526834369,0.3812628984,1.6862120628,-0.5823438168,0.5357362032,-1.8718117476,1.8158414364,0.0115653146,1.0090109110,-0.0481402576,-0.0621399991,0.5543729663,-0.0510954261,-0.4064195454,-0.5600697398,-0.1420308799,1.2426292896,0.2519005239,-0.0725310817,-0.8719974160,-0.6856629848,0.0003807768,-0.1563270241,-0.4662569165,-0.0164176356,-0.3071641624,-0.4057198167,1.3780448437,-0.9845128059,1.2129591703,-0.2401970625,1.7031100988,1.1683377028,-0.9992445111,-0.8173805475,0.2220273167,-0.3547341526,0.4088989198,0.3500408828,-0.4819968045,0.3505795598,-0.8347722888,-0.6508687139,-1.4876539707,-0.2163732499,-0.2417284250,0.9192639589,-0.0174811445,-0.6125581861,-1.7423535585,-0.2791055739,0.3562209904,0.2949730754,1.1177265644,-0.5972783566,-1.7783836126,-1.1208477020,-0.2938118279,-0.4967626035,1.0432761908,-1.0701702833,1.0008829832,2.2375409603,-0.0449116081,-1.3642407656,1.4596179724,1.1632910967,-0.5697431564,-1.7598483562,0.0133777484,-0.4007295072,0.7930163145,0.9166504145,-0.4451917112,-0.3771822751,0.5663695931,0.6020166278,0.6256309748,0.3488479257,1.4128754139,-1.1541855335,1.3467293978,0.0763891712,-2.2180795670,-0.8790378571,0.7706360221,-1.0272171497,0.3823670149,-0.4660917819,-0.5811817646,-0.9728909731,-0.3998017013,1.3444324732,-0.2047507763,1.6922044754,0.7689399719,0.7740138173,-0.9281789660,-0.4968115091,-0.7952126861,-0.0446061157,0.2427589148,1.3154057264,-0.9638298750,-2.4491629601,1.8243825436,0.6550926566,1.3024812937,1.2062610388,1.5797829628,1.6227316856,-1.1122174263,-1.0028078556,-0.3198828399,0.6408913136,0.5411697030,-0.3132085502,0.0905704945,0.4082139432,1.0586953163,0.0793558359,0.2908940911,-0.4207540154,-1.0434504747,0.8331912756,-1.2780961990,-1.4766488075,-1.6978293657,-0.0705931783,0.2020439953,-0.0928261653,0.6194707155,0.0804834291,0.3376404047,3.4837553501,0.2669399977,-1.4626413584,0.8958411813,1.1550073624,1.0528649092,0.7435577512,-1.5690900087,-0.4255738258,0.9140344858,1.2361495495,-1.2471395731,0.7626679540,-0.1928639710,-1.2733954191,0.7811949253,-0.4019659162,-0.4094708264,0.0163625069,1.4163241386,0.2105273604,-0.4444972277,-0.5089505911,0.4650335014,-0.2184793502,-0.4025595784,1.0784583092,1.0624145269,0.0769374445,0.0645173937],[0.8455424905,0.9994160533,0.9331492186,-0.7005475163,-0.2368602306,-1.2239063978,0.2130108923,-0.5972399712,1.8470237255,-0.6240378618,0.5833202600,-0.5017367601,-3.0102710724,-1.3837611675,0.1786506176,-0.3187453449,-0.5781022906,0.0315280780,-0.4391703606,-0.2296233028,2.3488371372,0.3843484819,-0.5347214937,0.1268615276,0.8902928829,3.8520202637,0.7244234681,1.1944116354,-0.6678529978,1.0415666103,-0.8658940792,0.9077596664,-0.3378550708,0.9775423408,-0.0754776895,0.6063098907,0.6713734269,0.0194545370,-0.3463670015,2.1391818523,-0.4366911352,1.7306786776,-0.1830381155,0.2859286666,-1.4076973200,-1.0983570814,0.2282403558,-1.1998163462,0.4056214094,0.9875051379,-2.2728385925,-1.3111302853,-1.8743584156,-1.2082616091,-1.5854942799,0.4378720522,1.4147380590,0.3267107308,-0.5729110837,1.2912527323,-0.6939992309,-0.0833639130,-1.6856313944,1.0955225229,0.5547582507,-0.9104179144,0.4392534792,1.9463402033,-1.1000357866,-0.7706585526,1.2468438148,-0.4902793765,0.5345413089,-0.9500870705,0.1477532536,1.2505365610,0.0899634138,0.3677219152,0.8650166392,-0.2937146723,-0.1290635020,0.3401563466,-0.3108875155,-0.7258995771,1.3177022934,-0.9953907728,-0.0467900336,1.0885127783,0.1454632878,-1.2228176594,-0.7175816894,-0.7583139539,1.9346177578,-0.7771400809,-1.0648733377,0.1046056002,-1.6488226652,-1.0661364794,2.0642158985,-1.8892747164,0.1650791317,1.0846154690,-0.1468636394,-1.1351752281,-1.2594205141,0.4703114927,-0.3027814627,0.2559323013,-1.8508899212,-0.9306232929,0.9595783949,0.1062435508,0.8443471789,0.2098277062,-0.8347812891,-0.7564993501,1.0179579258,1.0825403929,1.3440455198,0.1753898412,0.4277119637,0.2193833888,1.4308446646,0.9249140620,-1.7949538231,0.0283557717,1.7641273737,-0.7250720263,0.6685691476,1.1260482073,-0.3413906991,-0.4857299626,-0.0957151726,-0.4632709324,-0.7183370590,-0.0102504687,1.3925209045,-1.1265658140,0.8746911287,-0.9904094934,-0.4190586209,-0.2849096358,0.6538196802,0.7931527495,-0.3543632627,-0.8418226242,0.6238386035,-0.7723533511,1.4631421566,0.1103246212,-0.8867618442,1.6526117325,1.2548710108,0.2880513072,-0.4933074713,-0.5117329955,0.6032633185,-1.3497583866,-0.5111608505,0.6675429940,1.8336050510,0.3924017251,0.5415292382,1.6077739000,-1.6158584356,0.5651822686,-2.9483938217,-0.1493530571,1.1534879208,0.2299067229,0.2548667490,0.3125708103,-0.1421936601,0.0962879658,1.4915516376,-1.4980111122,0.2542267740,-0.3468973637,0.4250290394,1.9612369537,-1.1135908365,-1.4034284353,-1.3214765787,1.4184826612,2.2413375378,0.5875048041,0.0437724106,0.7939956188,-0.6215597987,-0.5815342665,-0.0857137591,1.4495682716,-3.0346090794,-0.5904287100,-1.1807073355,-0.8728227615,0.1840069443,0.5226079226,-0.7583761215,1.5459980965,1.1350790262,1.2372657061,0.2346145958,0.9069361687,-0.1464601457,0.8803369403,1.6036959887,-0.0359401405,-0.5794182420,-0.8478991985,0.4720226228,-0.0823028833,-1.2614284754,0.4648005366,1.1294105053,1.5758064985,-1.3074091673,0.4784797132,-0.0898916945,-0.4759197831,1.2124656439,-0.2539686859,0.0967589170,0.0534103699],[0.2857500017,-1.3306634426,0.4249919951,-1.1950973272,1.0747199059,-0.2170769274,0.0450050756,0.1741701663,-0.2596285939,-0.7920688391,-0.4063050151,0.9191468358,1.7260661125,1.4962009192,-0.2412421852,0.3035558164,1.5768409967,2.0336244106,-1.6690231562,-1.3676552773,-1.2393416166,-0.5456511378,0.8899183273,-0.1883179098,1.1415015459,0.3300242722,0.6155939698,0.7084887028,1.5292983055,-0.3306114674,-0.0788572133,-0.4493339062,-0.2701256275,0.5901252031,1.5623044968,0.3643705249,-1.4534931183,0.5249486566,-1.2966890335,1.1808009148,-0.0224379860,0.3152523935,-0.2782708406,0.8091103435,-0.4187511802,-0.4770900905,-0.3365066946,0.5623541474,-2.0182147026,-2.1983170509,1.1045867205,1.2624795437,-0.0276180673,0.1718224883,0.1991921812,-0.2634349763,-0.0887101442,-0.8009012341,0.3470789790,-1.4324259758,1.4175089598,-1.6810934544,-1.0911394358,0.5939617157,-1.1652457714,0.7605130076,-0.6547212005,0.3845574260,0.7944753766,0.6379547119,-1.1523764133,0.1821886748,-0.5537084341,1.5448677540,0.8237330317,-0.5419313908,-0.7200653553,0.6935390830,1.1008996964,0.9097881913,-0.9642880559,-0.4383548498,0.8910411000,-0.3298941553,0.9730165005,1.2895667553,1.1915460825,-0.6843030453,-0.9302793145,-0.2390873879,0.1579615325,0.2212390304,-0.6818502545,-1.7127672434,1.1771396399,1.1114701033,-0.3492176831,-0.0472809970,0.7482308745,-0.0730901062,-0.5841656327,0.4504536092,-1.1025894880,0.0581932291,0.6327283978,0.1792763323,-0.8223001957,-0.2284287810,-1.2400608063,0.4701416492,-1.1249814034,-0.4419503808,0.4520464242,0.4384505749,0.4458945990,-0.4454752505,0.6874499321,-1.9018042088,-0.8928960562,1.0825363398,0.2178558558,-1.2581878901,-2.8035640717,1.6928415298,0.0167334545,-0.7488485575,-0.6693356633,-1.2117948532,0.1265929341,0.5291302800,0.3899676800,0.4554530382,0.4301682711,0.4348187447,1.0085790157,-1.7501940727,1.1373521090,1.1931743622,2.0794131756,-0.2773678005,0.8379659057,0.6542209983,0.2838761508,0.5350176692,1.3807713985,1.7053838968,0.8826751113,-0.3271561265,-0.3581827283,-1.2562394142,-0.8812279105,-1.0632889271,-0.1390643716,-0.7417252660,0.5785334706,-0.5842409730,-2.0249407291,0.0031048066,1.6786370277,-0.1028214321,1.4051040411,-1.2307188511,0.0745233521,-1.3036009073,-0.4741173089,0.7923251390,-1.6625601053,0.5009847879,-0.4260414243,0.4607705772,-1.4588941336,-0.8828140497,-0.2869668007,-2.0858445168,-0.6617110372,0.9500411749,-0.4903324246,-0.1242794171,-0.6876080036,0.9306303859,-0.2304113060,1.0719676018,-2.1756176949,0.5917387009,1.5272648335,-1.1067727804,0.3803062737,-2.1274337769,0.5029523373,-0.2445702404,-0.0006700782,0.9635633230,-0.2261878550,-1.6574515104,2.6968736649,-1.1250771284,1.0807653666,0.0584954619,-1.4129952192,0.8620643020,0.8713108301,1.6652963161,-1.6135631800,-0.1984029859,-0.2902413011,-0.6188423038,0.7531723976,0.1087239310,-1.6153539419,0.4767957926,-0.4382492602,-1.0736072063,1.2319749594,-1.2424856424,-1.4469009638,0.5112141371,-0.2780409157,-0.9318844080,0.4025871754,-0.7289595604,0.4709553123,0.1000782549,1.2074514627,-0.2669787109],[-0.3595815599,-2.1342899799,-0.4210056067,0.4573920667,0.9248502851,-1.2482802868,0.6569746733,0.4175840020,0.0368951969,-0.0607408360,-1.8906854391,-0.0682709813,-0.5059621334,0.2151026875,1.8872957230,-1.1071970463,0.1205267534,-1.0122386217,-0.6658754945,1.1404993534,-0.5657894015,-0.8201801777,0.6875778437,2.0678489208,-0.6186278462,1.0876693726,1.6158015728,-0.1567347497,-1.2611894608,1.4985532761,0.4219383597,-0.1459040344,-0.6523817182,0.7554470301,0.2956840992,1.1193044186,1.6107659340,-1.5123630762,-0.4555642009,0.7889720201,0.8956725001,-1.3317306042,0.6401042938,0.2166453004,0.8215140700,0.0503329076,-0.0965074822,-0.1113443226,-0.2042803764,-0.6873565316,0.5317636728,1.1151205301,1.5243155956,-1.2777767181,1.7654581070,0.0725174472,1.0041288137,-1.6742922068,-0.1915116161,-0.6583703160,0.0271707252,-0.7335672975,0.1892032921,0.2953191400,0.0094617000,-0.1575515270,0.3323207200,-1.2621731758,0.1978865266,1.3326668739,-0.6343193650,0.9134386778,0.0729104951,-0.1353257895,-2.0858786106,1.0808997154,-0.3004504144,-0.2755873203,-0.7367097139,0.3021540642,0.2700982988,0.4530409575,-0.7007525563,-0.8959960938,-1.1999323368,0.4935321510,-0.5960142612,-0.9639679790,-1.2851704359,0.4559888542,-2.5519254208,1.4413543940,-1.0547283888,-0.4198956788,0.2472599745,1.0853878260,-0.5223800540,1.9698237181,0.5240100026,0.7831386328,0.4234170914,-0.1564492285,0.1359184831,-0.1162117422,-0.4195275009,-1.1759042740,0.3552658260,-1.9931274652,-1.4269702435,0.6328967810,0.4862891734,0.0348587781,-0.0004265035,0.1926831156,0.8282247782,2.8950171471,0.4765744507,-1.0976687670,-0.7158638239,0.7778658867,-0.2204324454,0.0522998534,2.3419752121,-0.2337766886,1.5526149273,-0.2743647099,-1.2966957092,-0.5145607591,-0.6441264153,-0.9436242580,-0.5575792789,-0.2886804938,-0.5300575495,-0.7995063663,0.4609216154,0.0503422990,-0.1202928945,1.2408776283,-0.5033631325,-0.0448194630,0.2108783871,0.0507609658,1.4534044266,-0.9200549722,0.0032031226,0.6261848807,0.2485037446,-0.7718658447,0.2863176465,0.0988020748,-0.7075133920,2.0979690552,0.0536587685,-0.4469783306,-0.6916967630,-1.4713886976,-0.6406098008,-0.9751100540,-0.7251353860,0.7354019284,-0.0255986620,2.1644103527,0.1879548430,0.9535272121,1.1498577595,-0.9122163057,-0.8776962757,-1.5234766006,-0.9056487083,0.4926247299,-0.3897213340,0.4013664424,-0.1390452832,0.0777551010,0.8162872195,-0.3259987235,0.1420471519,0.1295469105,-0.5685203075,-0.0366480462,-0.7493227720,-0.0521788187,1.3544013500,0.9088275433,0.9768580198,0.5170670748,1.2871779203,-0.1010567024,-0.0856445506,2.6273643970,-1.6656155586,0.5624790788,-0.5161991119,-2.1735665798,-0.3417043090,1.9319416285,-0.9397265315,-1.7080259323,-0.7614494562,-1.2761473656,0.4220181406,0.9958624244,1.0256894827,-0.1235863641,1.8414309025,0.0351293795,-1.0325827599,-0.4244602025,0.1708007455,0.5898537636,0.2644017041,0.1715549082,-2.2165124416,1.0189716816,-0.5055457950,-0.5834990144,-1.2355498075,0.8014555573,0.3301530182,-0.8047369719,0.2689402997,-0.8028509617,0.8982989192,1.5976356268],[-0.2384607941,0.5498903394,-0.3153492212,0.0371826515,-0.2799673378,-0.2572603822,0.8187432289,0.8705870509,0.7939828634,1.4407110214,0.0046773427,-1.0408636332,-0.4254775345,-1.0005455017,0.1042390913,1.2375731468,-0.0690675676,-1.8301037550,-0.2057546228,-1.1653020382,-0.9878067374,-0.1361900568,1.1316952705,1.0342150927,-0.6592921615,0.2408573180,0.2436449975,-1.2514510155,-0.5721936226,-0.0520525165,-0.4139078557,1.8763142824,1.1753531694,0.5226545930,2.1683094501,0.6907484531,2.2160840034,1.1130344868,0.4593990743,0.7844545245,0.6997361779,-3.1261579990,1.5720311403,1.7266691923,0.3456651866,0.4389714897,-1.6385536194,0.4049912691,0.2712102234,0.1878780574,1.3390874863,-0.0299388077,-1.3081719875,0.0632864684,1.2408345938,-0.9132910371,0.5147520900,1.1721259356,0.0147358524,-0.3792796135,0.8482502699,0.9654959440,-1.4460035563,0.4597578347,0.1704647690,-1.1115249395,-0.7208780646,-0.3793327212,1.2741154432,-1.6364229918,1.5149157047,0.0631237701,-1.2959022522,0.4598662853,0.0281794146,-0.7976759076,0.3872869909,-0.8583239913,0.1689493805,1.5861971378,-0.8286434412,-0.9093782902,0.0804205164,0.0313321017,-1.8652266264,-0.3101404309,1.6558412313,0.4230325520,-1.1583219767,0.9033274055,-0.4117244184,-1.0604335070,-0.5190912485,-0.5632234216,0.1435869932,-1.5192936659,0.0426138528,1.4924136400,1.2707833052,1.9864619970,0.5108605623,1.2856057882,0.8407601118,-2.0035340786,0.8936482668,-0.3260267377,0.8181855083,1.0708587170,-1.8654093742,-0.4303693771,0.3521697521,0.1649127901,0.2585572302,-0.7880253196,0.2695698440,-1.3194481134,-0.6205232739,-0.0030871087,-1.2652636766,0.5406863093,0.4429259598,0.4298315346,-1.2147737741,-0.1896856129,-0.4850975573,-0.3081047237,2.3780131340,1.2015973330,0.2664550841,0.7288299203,0.7034993768,0.0048138006,-0.3591594398,-0.7174685001,1.7085003853,-1.3032776117,1.1584476233,1.0995483398,0.5414423347,0.9730444551,-1.0727057457,-0.5748537183,-1.3290205002,0.7744051814,1.4045319557,-0.2159052491,1.7948094606,1.6460920572,1.3411537409,-0.4496222734,-0.5609518290,-0.0992444083,-0.0853365064,1.2166137695,-0.0213507153,-1.4752011299,2.1615161896,1.2456383705,0.2582988143,0.2717613876,-0.1682876796,-1.4384615421,-1.0607285500,-0.1907466799,1.6710277796,1.2433642149,-0.7574949861,-0.0647511855,-2.9306302071,1.5923978090,0.0510240272,0.3966749310,1.0205726624,1.1665779352,1.6779688597,-2.3571422100,-0.5891851187,0.3185988665,1.8541604280,-0.0838108212,-0.4632383883,-0.1345472634,0.8540049195,0.7585686445,-1.0865671635,-0.8296186924,0.4694327712,0.3256563544,1.4290058613,0.7655954361,1.2069232464,-0.1459943205,0.8065757751,-0.3280249536,-0.0742716119,-0.4548668861,-0.0853952095,-1.1056786776,0.9523906708,1.7686954737,-3.2922542095,-1.5821778774,-1.3896070719,-1.2549228668,0.3908016682,0.6844573021,1.4914137125,-0.4642134607,1.1045403481,-0.4899351001,-0.9562233686,-0.2807927430,0.0053708674,1.1767382622,0.1583883911,0.2319228053,-0.8078649044,-0.1561281085,0.3615372777,0.4036455154,0.9082906246,-0.0023141834,-0.0452877991,-1.5662910938],[-0.1033781320,0.5418262482,-0.9377847910,1.0195208788,-1.1293481588,0.0335480943,-0.4405394495,-2.2315773964,-0.4775855541,2.0507366657,-1.3063079119,1.6552379131,-1.1819411516,-0.0324509963,-1.4311203957,0.8767514229,-0.1345372945,0.8536100984,1.5587157011,-0.2368998080,0.1880593002,0.2717109025,0.6395956874,-1.2362473011,-0.0319846682,-0.8295824528,-0.2055542320,1.5347025394,0.7827878594,1.2703878880,2.7300498486,-0.2844167948,1.1753647327,0.4245087504,-0.5369454026,1.2073285580,0.0930977017,-0.5196133256,1.0421657562,0.1690351069,-0.4875896275,0.6770972610,0.0044279699,0.3105448484,-1.2954131365,1.3982955217,1.7429810762,3.6478598118,-0.7394582629,-1.1202218533,-0.4658226073,-0.9041526914,1.1250499487,-0.1801972389,0.1294232458,0.7745076418,0.5126174688,1.4980577230,-1.0465172529,-0.4988512099,0.5188845992,1.1863158941,-1.1096326113,0.5208597779,1.2098367214,1.0017185211,0.6673674583,0.5668189526,-0.0521996841,-0.8716872931,0.1681198180,-1.2476809025,1.9259685278,-0.5453615189,-0.8922271729,1.9886724949,-0.3135860860,0.7334384918,0.5198380351,-0.4526397884,-0.9506946802,-0.2661958933,-2.5570335388,-0.0557328612,1.1625821590,-0.4434820712,1.1122316122,-0.0545103103,0.2586460412,-0.6618434787,1.4436861277,-1.3688731194,0.7621035576,-0.2985791862,-1.0211650133,-0.1357472390,1.0373653173,0.3776987195,-0.4136635363,-1.6953018904,-0.1204522103,-1.1567699909,-0.7138453722,-0.3276610970,-0.6435422897,-0.7366944551,-0.1425680071,1.7642153502,2.8425951004,0.0610087626,-0.4151884615,-0.1082942411,0.0857538283,-0.8597403169,-0.9152332544,0.1713453978,0.9422041178,-1.0082613230,-0.7561030388,-0.3161784708,-0.1782092303,0.0957031474,-2.5437932014,-0.6769134402,-1.4552763700,-0.1227465048,0.4310432673,-0.9728448391,0.3342837393,1.1829782724,-0.0065461015,0.5665283799,0.1695615500,-0.0044607525,0.0285913981,-0.8989881277,1.1593964100,-1.4732522964,-1.4600700140,1.7094210386,0.0207724255,2.0963575840,-1.2747777700,-0.6057670712,-0.8993343115,0.2116632611,-0.1160674915,-1.2055330276,0.7836779952,0.5415595770,1.0465161800,-1.6241914034,-0.8055034280,0.5579012632,0.8232837319,1.6447900534,-1.2311408520,0.2137642205,-0.9106723666,-1.9084692001,-0.2605872154,0.2076506913,-0.9598652124,-0.9803130627,1.2351433039,-0.9859101772,-0.3903642595,-1.0345238447,-0.3124050200,0.7462546229,-0.5244422555,-2.4146971703,-0.1631130129,0.5022974014,0.9429028630,-0.8194223046,0.0069113285,-0.2531582713,0.2928455770,-0.6136545539,-0.3634734154,0.8031544089,0.0733277798,0.5689120293,0.9162364006,-1.4241131544,0.6944862008,0.1126557738,0.2768815458,0.0754733682,0.4862255752,0.3217608333,0.3620449007,-1.7405321598,1.1516690254,0.9695023298,1.0705699921,1.1136047840,0.7925577760,-1.7781143188,0.8929021358,2.0690708160,0.5849423409,-2.1492705345,0.5832387209,-1.0383287668,-0.4568869174,3.1900093555,-0.7241430283,1.1837641001,-0.1022715420,1.8591374159,-0.3411387801,0.3150487542,0.0275884047,-1.3723717928,0.0300342757,0.4547230303,1.8566195965,0.2995341718,0.6253018379,0.4939307272,-0.0265730321,-0.3234100342],[0.3595822155,0.6894904375,-0.6438739300,-0.9093788266,1.2942886353,1.2176066637,-0.9077116847,0.1113884673,1.0686095953,0.1878102571,-0.5059827566,-1.6027270555,0.0214889795,-1.3829032183,-1.7231186628,-0.3605511785,0.5595287085,-0.4247933030,-0.3993293941,0.0908166617,0.1786594391,0.7535029650,0.6296449304,-1.5375419855,-0.2698430121,-0.7669034004,1.2986503839,-0.5635685921,0.1221668199,-0.3815145195,-0.4466152489,-1.1692444086,1.6965802908,0.3721182048,1.7709367275,-1.6176935434,0.2316016853,-0.8005692959,0.7475548983,-0.8234938383,0.7365947962,0.7701295614,-0.9460060596,-0.4202794731,0.3530687392,-1.8267436028,0.2953773141,-0.6090833545,-1.0919172764,-1.6189069748,-0.0322289020,0.4412971735,-0.4813476503,-0.6603649259,-0.8035909534,-0.8457508683,0.2187346816,-0.9387763739,0.0936134532,-0.0921785682,2.2268385887,-0.4825090468,1.5371801853,1.9999849796,-1.3969053030,-0.0445936732,-1.4784121513,-1.8323581219,0.3004155457,-0.3889199495,-1.3225893974,1.8505661488,0.0649111494,1.6666123867,1.4079970121,0.2198562622,0.4401271045,-1.1840212345,-1.4271931648,-0.7065832615,-1.2413952351,-1.3692640066,-0.0096538514,-0.6498388648,-0.2452564836,-0.9085732102,-0.1391163319,-0.8336672187,0.5473741293,0.8369570374,-1.1456743479,0.0624656156,1.8641681671,1.6749701500,-0.5735353827,-1.4077433348,-1.9448838234,0.5631722808,-0.6609920859,0.2506439686,-1.1130369902,-1.0673143864,-0.3806003332,1.4953833818,0.6200692058,-0.6979174018,-3.6335635185,-0.8773848414,1.7083442211,-0.7415100336,-1.0148513317,1.2024078369,0.7076079249,-1.2916729450,1.0737441778,1.0685819387,-0.5960540175,1.1803132296,-0.0333345570,0.0980365053,-0.4779715836,-0.8072968125,-0.0824001506,-1.9273227453,0.7178004980,-1.0592756271,-0.9261478186,0.1021981537,-0.2795476615,-0.9590332508,1.0488163233,-0.4690090716,1.6720734835,0.6879519224,-0.0157241132,0.5268683434,-0.5965974927,1.6848707199,-0.8259072304,0.7594338655,1.2602584362,-0.6720221639,-0.0584140681,1.0932010412,-0.5520174503,-0.6187421083,-0.8900236487,-0.8719530106,1.2347486019,-1.3540980816,-1.2299486399,0.6464612484,-0.0974468589,-0.3091480434,0.5283569694,1.0791188478,0.2561168075,1.4158717394,1.7008452415,-1.4289003611,1.3387390375,1.1170512438,0.5130531192,1.0400758982,-0.6153922081,-0.1668823063,0.7639955878,1.0470120907,-0.6455356479,0.7287098169,-0.5354412794,1.2531111240,-0.4258435369,0.3980273902,-1.6516693830,0.5757321715,1.1960839033,-0.4860200882,0.3503985405,-0.0763770789,0.0167850349,0.8753983378,0.5204891562,0.8198471665,-1.4274868965,-0.5994745493,1.2439334393,0.8900793195,-0.3447135091,1.6019076109,0.4453103244,0.3134413362,0.2098031342,0.8220742941,2.1609876156,0.4291853309,1.9126983881,-1.1369169950,0.1236273199,-0.0741692111,0.1741669476,0.1106243953,0.4574111104,-0.7266279459,-1.6765234470,0.8124452233,0.4615250528,0.2150113583,1.4487680197,0.6443644762,-1.3313271999,0.6483889818,0.0551732592,0.0946426168,-0.5147324800,1.3383497000,-0.6886813045,-1.3321741819,1.4576708078,0.3417120576,0.1916114688,-0.0513381623,1.6174348593,0.7993416786],[0.1274137199,-0.8448427320,1.2690343857,-0.5826838613,0.0557822697,-0.7457270026,0.0534833819,-0.5319348574,0.5449818373,0.0617729798,0.1800281107,-2.3204801083,1.0729317665,0.3631948233,-0.4377925098,0.1569657326,0.5283293128,-0.5462140441,1.2347840071,0.5054156780,1.6532857418,-0.6099229455,-0.6667820215,-0.5278279781,-0.4208771884,0.4319642186,-0.1246592775,-1.0448354483,0.0785995051,2.6429173946,0.0422035158,0.2535940409,-0.9304233193,1.0765348673,0.3263046741,-0.6096877456,0.6794693470,1.5073816776,0.3643631041,-0.7833818197,-0.8042606115,-1.1636455059,2.1159131527,0.6903854609,0.1691145897,1.3395061493,-0.3655520380,1.3998910189,-0.4119296074,0.1565577090,-0.9600688219,0.1500866711,-0.4632772505,-2.9622406960,-1.1724597216,0.6507877707,-0.5097853541,0.3659832180,0.4810813665,-0.0095656291,0.4939456284,0.1545771658,2.0188348293,-0.3500584066,1.2580102682,-0.3938673437,1.1757705212,0.5819926858,-0.1100534201,-0.4838815629,-1.1713180542,1.0384311676,-0.4346498549,-0.6087762117,-0.5977138281,1.2044967413,0.5018853545,0.3364687562,-0.2894819081,1.2561717033,-0.1431188136,-0.0513663627,0.1945189983,-0.1383751035,-1.2002265453,-1.5160598755,-0.3494048119,-1.1222406626,-0.1564642340,0.1808167845,-0.3393794000,0.5907009840,-0.9833743572,0.9756684303,-1.3281390667,0.2585115135,-0.3627780676,0.8321187496,-1.8786463737,0.4968658686,0.0689388067,-0.0079110470,0.3008804023,-0.7525809407,-0.3469925225,-1.5925863981,-0.7068521976,-1.3115445375,-0.2572110593,0.1371918172,-0.6321007609,-0.4874186814,-0.0709755793,0.0640492663,0.3276489973,0.3932814300,-0.2793305516,0.4494642913,1.7821687460,-0.9390084147,1.5413779020,0.0079562068,0.1199712157,0.1818806082,-1.7790522575,0.0833725855,0.3141438663,0.2376887649,0.3440466821,0.7322938442,-0.6423465610,-0.9430661798,-1.0507161617,-1.1356643438,-0.9131414294,-0.1980059296,-1.0524215698,2.0543701649,-0.5051992536,-0.2961077988,-0.1411919743,0.2223303318,-0.5895858407,-0.5231840014,0.6206701398,0.8080673814,-1.3365075588,0.3356038332,-1.6119444370,0.8270612359,-2.3478622437,-0.1573795974,-0.8355489969,-0.1855920851,0.2819520831,0.1627157480,-0.4957902730,-0.2017339319,0.0587787591,-0.8486970067,-2.2888104916,-0.9550905228,0.0425533839,-0.8235985041,0.2843135595,1.3483312130,1.6403613091,-0.1185661405,0.2167840004,-0.4161785841,-1.7830762863,-0.5561192036,-0.0708618909,-1.0473921299,1.0332918167,0.1671882719,1.5324164629,-0.6358706355,-0.2964316905,-1.5015150309,-2.2901973724,-0.2640973926,-1.3396550417,0.7526706457,-1.1424431801,0.2186951339,-0.4216327071,-0.4880354106,0.5038363338,1.6515885592,-1.1511272192,-0.6911903024,-0.8230931759,-0.2474271357,-0.3428781927,2.3004641533,-1.2634423971,1.2604916096,-0.4096618295,-0.2799764574,0.7918033600,-0.6636201143,-0.0015233173,1.6005517244,1.3663774729,-1.7475005388,1.0692484379,0.2425226271,0.4911504090,0.6091113687,1.1200417280,0.6362381577,0.1359951049,0.1392289251,-1.5782338381,0.9691632986,-2.3665611744,1.0742968321,0.5516464114,-0.5444959402,0.8318801522,-0.7501646280,1.2486605644,0.3589930236],[-0.3600519598,-0.7901878357,1.5135924816,-0.2313470095,-0.7407914996,0.8029611111,0.2708649933,-1.6062647104,0.5937306881,-0.3154849410,0.1361830235,-1.0001647472,-0.0763395801,0.0344249457,0.3669148684,0.5293010473,0.0763758123,-0.2572541237,0.5138447285,-0.1288889498,0.6439459920,0.5794592500,-0.7824993134,2.2121942043,-0.3496154547,0.4119117856,-0.8420560360,0.4544038773,0.0109467581,0.0010534312,-0.2950676680,0.1750898808,-1.2973902225,-0.0623139627,0.2126978934,0.6877377033,0.3352945447,1.7457261086,-1.0843807459,0.7854700685,0.0600954890,-0.0855508447,0.7329173684,0.2756085098,-1.0048882961,-0.7781894207,0.8054320216,0.0727083459,2.3196663857,0.3006044030,-0.2822029293,0.4725848138,-1.5897241831,0.0941874236,-0.9508988261,1.4989129305,-0.3068813980,0.2385282964,0.4217751622,0.7946755886,0.7492550611,0.5127679706,-2.2465128899,-0.1663811803,0.5605801344,-1.8675581217,0.4418747425,-1.0016011000,0.7059368491,-0.0510153249,-0.2325836271,1.5391352177,0.9156982899,-0.8702345490,-0.5608828664,0.8970757127,0.3895307481,0.6618192792,1.2001011372,-0.1240015626,0.4234113991,-0.7949218750,-1.7483150959,0.1998164356,-1.5106359720,-1.1773667336,-0.6218593717,0.3076383173,0.5031841993,1.5083596706,-0.7131744623,0.2623642981,-0.0309111103,-0.1339277327,-0.1106061265,-0.1045916006,1.0869770050,1.6638536453,0.6696953177,-0.2361247987,0.9860504866,0.4088005424,-1.0221886635,0.0886315182,-0.4157384634,0.1207668409,0.4294110239,0.1377238184,1.2307782173,-0.1232055947,-0.8324245811,-0.3747373223,2.4342443943,-1.1951713562,-1.1546982527,-0.9615500569,1.2484122515,-0.3975964487,0.8206714392,-0.8452029824,-0.2450099438,-1.6266416311,1.1472618580,1.0578019619,0.1282176673,1.1876020432,2.5764918327,-0.0823692530,1.1622513533,1.2577518225,0.7237423658,0.4700427651,-0.7886118889,-0.4397766888,-0.8123567104,3.0631451607,0.5537109971,-0.0900671929,-2.6399536133,-0.1701684147,-0.6035872102,-0.6521043777,1.1507827044,1.2750959396,0.2453898042,0.3872298300,-0.8278653026,0.9271902442,1.6431338787,-0.4256422222,0.5522973537,0.1817890704,0.4363834858,1.0876022577,-0.4799385071,0.9566588998,1.3544300795,-0.1174629778,-1.2165155411,0.8368718028,-0.8922112584,-1.5480458736,0.0400244184,0.3244729638,0.0817489848,-0.9257849455,-0.7675449848,-0.7365167141,1.6259465218,0.6298605800,-1.4840712547,-1.4950245619,0.1297564805,-1.0976947546,-1.1410716772,-0.9285092950,1.0353862047,2.7001140118,0.9804233909,-0.3006408811,0.1535374373,1.3037381172,-2.0147192478,-0.1158716157,-0.6375645399,-0.1695705652,0.6505923867,0.6162424684,-0.8195439577,1.2143636942,1.0763442516,-1.8476772308,0.3398576081,-0.2474824041,-0.9045435190,-1.1834901571,1.6120089293,0.4015118480,0.3753013313,-1.0195120573,0.3743047416,1.5763468742,0.2818703055,-0.0415413007,1.1435390711,1.0312833786,0.5930593014,0.4299762845,-0.7077791095,-2.0817489624,-0.8573213220,0.8970362544,-1.1242753267,2.1089203358,-0.1735206395,0.7554202676,-0.6492454410,-1.1986908913,0.3451207876,1.0742040873,0.2218725234,1.5225878954,0.1871998757,1.1086134911],[0.1162764803,-0.2187245935,0.3414756060,0.1085516512,-0.7073196769,1.2486360073,-0.9194564819,-0.8338657618,-1.0484052896,-0.1623076200,-1.9373788834,0.5603187084,0.5427429080,1.5060900450,0.0874666423,-2.4002182484,1.3652335405,0.5865104795,-0.3230989277,-1.3929996490,1.1904447079,0.3914655149,-0.6465815902,2.4218268394,-0.6179955602,0.0971411094,0.0927094892,1.1953548193,0.6744587421,-0.6237062216,-1.4494575262,0.5976842046,-0.3370549381,0.0000538884,-0.9751046300,-1.4531168938,-1.2553899288,-0.5801687837,-0.0705595762,-0.6704121232,1.3072650433,1.5074343681,-0.1624190360,-1.0658824444,-0.6713882089,-1.2154396772,-1.0532267094,1.3715112209,-1.1906760931,0.8829891682,0.1461430639,0.3025677800,-1.0018360615,0.7597610950,0.4627263546,-0.9727967381,-1.8330810070,0.6842470765,-0.0457889512,0.0743883923,0.1160207540,0.4871888459,-1.3810641766,0.7513743639,-1.6535325050,0.2805307209,-0.6730520725,-0.6538075805,-1.0937461853,0.8920948505,-0.5652847886,-0.2217692137,-0.9390385151,-0.2413975745,-0.0277992953,0.4737479687,-0.0896343440,0.4736139476,1.7478946447,-1.3817198277,-0.5763424635,-0.7620200515,-1.2223913670,0.0114631774,-1.4424446821,0.1032015532,-0.4778788388,0.6391831040,-0.3346210420,0.7208385468,0.6453597546,-0.6174742579,1.3672366142,0.4193812609,0.2484990656,-1.2752428055,-0.4317915738,1.3278837204,0.4312328100,-1.5758079290,-0.8080574274,-0.0961645246,0.2024365366,-0.2257446200,-1.4323306084,-1.1698937416,-0.1322149932,-0.2605111599,0.1435681283,-0.3786023855,0.9958216548,-0.0179624446,0.3827536106,-0.9316131473,-0.9425467849,-0.8002999425,0.9446987510,-1.1967359781,0.5227084756,0.3323342502,0.1467415690,0.3252617717,-0.2421308011,-0.3801048696,1.5932736397,0.8452807665,-0.3431094885,1.2928600311,0.0758657232,-0.8354057074,0.6789767146,-0.0734707043,0.4803312421,1.3606332541,0.8671076894,0.9994120598,-1.1754306555,1.4058384895,-0.9871997833,1.0385127068,-0.4434279203,0.8501870036,-0.6956353188,-0.6379371881,0.8698224425,1.4933327436,-0.9885481000,-0.6133674383,-2.2802202702,0.9688157439,-0.2850471735,0.0442743227,-0.3362223208,1.1406686306,0.5380051732,0.6882110834,1.2516015768,-1.3449673653,1.0782428980,1.6350847483,2.2968215942,-0.7110384703,1.6386603117,-0.9126443267,-0.3877492547,-0.1043018326,-0.0090003163,0.6059634686,-1.0972509384,1.7331362963,1.3298485279,-0.6946768165,0.8652173877,-0.8342711329,0.3025682271,-0.8910675645,0.6248769760,-1.6545723677,-0.9224897623,1.7043474913,-2.1606175900,-0.7243913412,-2.4045393467,1.6853871346,0.8936536908,1.0344368219,0.1707872897,-2.0846495628,1.3154208660,0.2850298584,-0.3303071558,0.9252412319,-0.5945811272,0.2951497138,-0.4839909971,-0.2808298469,-1.2129616737,-0.2385444343,-0.9515303373,1.8470141888,0.4530581832,-1.3355108500,-0.1361455619,0.8572332859,1.6664037704,-1.7786149979,1.1624349356,0.4030810893,1.3191045523,1.9514727592,0.1954828501,-1.1131478548,1.8760913610,-0.4106545448,0.9871078134,-0.3514513075,-1.1491596699,-1.0029109716,-0.4375024438,-1.7376359701,-0.1112331674,0.3778693378,-1.2642121315,-0.9193267226],[-0.4944682419,-0.7798122764,-0.4271022081,1.0683219433,0.1328333765,0.7467706800,1.3215091228,-1.1968958378,-1.1443666220,0.6172171831,-0.0051761493,-0.2794991434,0.3744995892,-0.3662587702,0.4500087798,1.5653235912,-0.2852126658,-2.0067307949,1.3534057140,1.2254853249,-0.3875033557,-0.4426376224,-0.9592947960,0.6190398335,-0.3124010861,-0.5994002223,1.5445845127,-0.0053672073,0.9292419553,-1.4328645468,-0.9411212206,-2.3532216549,0.0921249315,-0.3608577251,1.1292972565,0.7657831311,2.4220712185,0.1606874019,-1.4992787838,0.2326702625,1.0695062876,1.6562827826,-2.1346623898,0.8291113377,1.9936954975,-0.0670990944,0.4760950208,0.6264278293,0.3247528374,0.6127083302,1.3447989225,2.1764240265,-1.0073144436,-1.2598137856,-0.3319583833,-0.9408110976,2.1974444389,1.1426124573,-1.1883490086,-0.6941516399,-0.8429222703,-0.4689262211,0.1896196008,2.4364590645,1.4495763779,0.9550628066,-1.3749413490,1.0702964067,-0.7423433065,-1.6166442633,0.3353677094,0.8143854737,-2.1510691643,-1.4398878813,0.4295713007,-0.6825832129,1.4495866299,0.0767971873,0.8710640669,-0.9394101501,1.2120714188,-0.1885681599,-0.6676308513,0.5666903853,-0.5110077858,0.8325109482,-0.9449007511,0.3893716335,0.0370208956,0.3440933228,0.0902620628,0.1622183025,1.9678138494,1.0434346199,-1.4853209257,-2.2796955109,-1.0654275417,-0.7173523307,-0.8523059487,1.6222326756,-0.6020047069,-0.1935931742,-1.0439174175,-0.8322221637,-1.7539211512,0.6675662994,1.5207853317,-0.7924110889,0.3303536475,-1.4821587801,-0.2574607432,-0.7888179421,-0.2782670259,1.1471936703,-1.1040791273,-1.1629147530,1.2700695992,-0.4843160808,-0.5253778696,0.8253301978,1.6041414738,0.3806031942,0.7163059115,1.0897123814,-0.9337714911,-2.2876377106,-1.0739610195,-0.3832927644,-0.1494626701,2.3470876217,-0.1703584939,0.3322209418,-0.8650518060,0.5197970271,-0.8209413290,-0.6846882701,0.6192402244,2.3423302174,1.3338751793,0.1373818070,0.7219576836,0.9726226330,-0.1536917388,0.9913292527,0.2540746629,0.1626318097,-0.3465512991,0.4595924914,1.2181417942,-1.0088903904,1.2518771887,-0.1303908825,-0.9561480284,1.8820406199,1.2069588900,0.3529352248,-0.8306714892,-1.4193843603,0.1155640781,-0.5738064051,0.6434262991,0.4984490573,-0.1118573099,0.7238074541,-0.5256049037,0.6390516162,-1.1183315516,-0.0976157486,-0.1260827035,-0.5548433065,1.7662122250,-0.2686588168,1.0087834597,-1.4268404245,1.1659404039,-0.9612586498,-0.3340024650,1.5359613895,0.1235429570,2.9805519581,1.0028824806,-1.0232834816,-1.4397224188,1.3039492369,-0.7188103199,0.5913882256,-0.9829553962,0.2619152367,0.0651016235,0.1422494650,0.0322833695,0.7933985591,0.3038003445,0.6963039041,-0.4510253370,-0.0787391812,1.2138137817,2.1563596725,1.1974829435,-1.3827234507,-0.5387663841,0.3749031723,0.3817621768,0.5126075745,0.7817907333,2.8445768356,-0.7148271799,0.1042771265,0.0861898214,-0.2078901976,0.1141170561,-0.3227742314,0.0504558496,-0.5519875884,-0.2047938854,-1.8481737375,1.2803936005,1.1041901112,-0.1439001858,-0.2400241643,-1.0030885935,1.1916694641,1.3330214024,0.0078061824],[0.0833170936,-1.3876982927,-1.3872941732,0.8939967155,0.2273259908,1.3578511477,-0.1441743076,-0.7695208192,-2.5133471489,0.1119075045,-0.4572762549,1.0314948559,0.9591494799,0.3220496476,0.1193715185,-0.0206195451,0.6939818263,0.0383000039,1.0197111368,-1.3477758169,-1.8404810429,0.8510962725,0.8117278218,0.0951100439,0.6611233950,-0.7656561732,-0.9922797680,-0.9179639220,-0.3361779451,-0.5715629458,0.5896957517,0.1407307684,0.7236926556,0.7947764397,0.4836243689,-1.3869879246,-0.1118176356,0.2992152870,-0.4100322425,-0.7881225944,-1.8818029165,0.9634225965,-0.0309839323,0.8337911963,-1.9074735641,0.7431362867,0.4745285511,-1.1356483698,1.0912343264,1.2545942068,-1.4796434641,-0.4746834636,0.3405517042,1.1815891266,-0.8854237795,0.0957699418,-0.2880924642,-0.0463199876,-1.3265682459,-0.1858589202,1.2988238335,0.6762433648,-0.5991950631,0.3327338696,-0.5831449628,1.4332697392,1.9494212866,-0.7886651754,0.5240651965,0.1793854237,0.6290607452,0.4811582565,-1.4723852873,-0.0745147914,0.5363513231,-0.8404342532,-0.4846772850,-0.9714007378,-0.2141099870,-0.2834431529,0.1940501779,-0.5316337943,0.9639087915,1.6239774227,-1.3215353489,0.9414067268,0.7094277143,0.8003622890,-0.8477427363,0.9165262580,1.6716412306,1.2298988104,1.4809641838,-0.4254212081,-0.4077859819,0.3305155039,1.0415029526,-1.1496744156,0.1712993681,1.4701676369,0.4293683171,0.1795604676,0.0516814254,0.2441919297,-0.3086892366,1.1564617157,-1.2884559631,-0.7572581172,0.6286239624,0.5937031507,-1.1899099350,0.3375867009,-0.4291598201,0.5017452836,-0.8552255034,-0.8198128343,1.4330694675,-0.2992851734,-0.4924618900,-0.1553115398,-1.0980613232,-0.1505965292,-1.7112737894,0.5384337306,0.9439564347,-0.4357626140,0.5460928082,-0.8451873064,-1.3298441172,-0.5760731101,0.2821652591,0.7656444311,-0.0708712041,-0.7376880646,0.7219296694,-0.7072268724,-0.2859232426,0.6533915401,-0.8342830539,1.1105629206,1.6008365154,-0.6448624730,2.8344218731,0.6748697162,-0.3423575163,-1.4236814976,-1.1950898170,-0.5678108931,0.7195250988,0.5357619524,0.4457416534,-0.4799274206,-0.1652086526,0.1340560615,0.6014035344,-1.3409739733,-0.5987482071,0.0769548640,-0.3581912220,1.1240202188,-0.2238033116,-1.4019105434,1.6488307714,-0.2175414711,1.9418623447,-2.5849790573,-2.4083001614,0.3356060684,-2.0707116127,0.9840598702,-1.2379816771,1.0672813654,0.2889554203,-0.0074083852,0.4752945006,0.1841205209,1.2986530066,-0.0591709130,0.1742814332,-0.0644250885,-1.1764115095,-1.7949217558,0.7206540704,0.4559027553,0.3954004347,-0.7530158162,-1.8091009855,-0.3017390668,0.2449345738,0.7457346320,-1.5417525768,0.8053346276,-0.5716369748,1.6070737839,-1.2270644903,1.9155560732,1.0043300390,1.8278682232,-0.6639636159,0.6661688089,0.4428050816,0.0892480984,0.2309782803,0.8315519094,0.1459945887,2.3326561451,0.5832629204,-0.2406825274,1.9596458673,-0.6441795826,-0.8311957121,-0.9647058249,-1.5046192408,0.5703581572,-0.5187826157,1.4320012331,0.2922801673,-1.2171894312,-0.6377372742,1.4113386869,0.5109692216,-1.4242004156,-2.1111421585,-1.6401022673],[0.3583278954,-0.0144035704,0.8650972247,-0.0165269319,0.9271504879,1.5814197063,-1.2962620258,1.4669339657,-1.8920297623,-0.5688353777,0.1828519851,2.2016677856,-0.1028954163,-0.9203289151,-0.1486750543,0.3816725910,0.4249047637,-0.5387316942,-1.5233194828,-0.8753087521,-0.2341729850,-0.2465784848,0.0562043898,0.1882509887,-1.0140819550,-0.2454162836,0.8557434082,-1.3931586742,-0.4529600143,-1.4558101892,1.3878477812,2.2907056808,-0.0797444135,-0.4271253645,-0.2662831247,0.5487980843,-1.0244842768,-0.8371625543,0.8822185993,-0.9598054290,-0.0063711633,0.8817792535,-0.8519062996,0.0839718059,0.6059336662,-0.8637058735,0.5584779978,-0.0299070161,-1.4362994432,-2.0078036785,-0.2094628215,-2.6569430828,-1.1245808601,-1.3522570133,-1.3392878771,0.2719299793,-1.0510017872,-0.5297577977,0.0293570850,0.3279094696,0.9607840776,-0.0345142111,-0.0836286321,-0.4210097790,-0.3227880597,-0.4815520048,0.4784446359,0.8643121123,-0.4473959506,1.6125975847,-0.3212073445,0.2659475207,1.1676646471,0.5559692383,0.7779700160,0.6278859973,-0.2657962441,-1.6537842751,0.5440248251,-0.3830282688,0.7568864226,-1.9953464270,0.2166253179,-0.1360926181,0.3883821666,1.4031965733,0.1071888581,-1.0993431807,-1.4928320646,-0.0251265056,0.4121911228,-0.7856203318,-0.3717033565,-1.3687229156,1.2271392345,-1.2881999016,-0.6959694624,-0.8874993920,1.4558117390,0.3226454854,-1.3846375942,-1.0860627890,1.0846112967,0.4653168321,2.5614218712,2.7988910675,0.9378732443,0.0436836891,-0.2922166884,1.7467682362,-0.9949976802,-1.1876443624,1.5535912514,-1.5948337317,1.3830382824,-1.7654812336,1.8867820501,-0.0235525575,1.3374916315,2.3650345802,0.6751262546,-0.6060769558,0.0485105924,-0.1316827983,0.0151644247,0.4798516929,-0.7473517060,-0.9493153691,-0.5720299482,-0.1864403486,-0.9693927169,-0.0035792037,-1.6661548615,1.3482993841,2.0277435780,0.3658308387,-0.8208331466,0.4718213677,0.6236744523,0.4980045259,-1.3271654844,1.3967874050,-0.1287742406,-1.1586636305,-0.2044127733,0.2533079088,1.2015191317,0.1085055918,-1.5884567499,1.0797991753,-0.5118536949,1.7802431583,-0.3157459795,-0.2945337296,1.5226759911,0.0407008827,0.9528471231,0.7287182212,-0.7189680934,-0.0266107712,-2.0186710358,1.2203032970,1.1047537327,-0.8241505027,0.8184968233,-1.7929810286,0.9446066618,0.0339597128,-0.3622136414,0.4354104698,-0.4571644366,1.4052172899,-0.2578752041,-0.1565036923,1.5309950113,1.0972181559,-0.2565720379,-0.4348123670,0.0947441384,0.9954605699,-0.5722588301,0.3652120829,0.5728793740,0.1401024908,-0.1164690852,-2.0019223690,0.2620675564,0.8731961846,-0.0032438957,1.2527974844,-1.0324943066,0.9453257918,-0.8272736669,1.6740643978,-0.9105581641,-0.3780149817,-0.0894450843,2.1181685925,-0.4071816206,0.7425952554,-0.7459664345,2.4063830376,-0.4379978478,0.9473155141,1.5492707491,-0.0160185508,2.9175112247,-0.3762186766,-0.8467391729,-0.0670067072,0.0901584849,-0.4757851958,0.0956222340,-0.3810622394,-0.9159494042,-1.2364341021,-1.1209826469,0.8402428627,1.3614509106,-0.6751222014,-0.3164610267,0.5810042024,-1.0004385710,0.6155641079],[0.8530336022,-0.6594149470,-0.4928688407,0.7073107362,-0.4533206820,-2.0503511429,0.7237859964,-2.3081104755,0.7931211591,-2.2254080772,-0.5442180037,1.7981696129,-0.4281713963,0.3618839681,-0.5407407284,-0.2953153551,-0.2099083960,1.0288715363,0.8415134549,0.2290135175,-0.4205887914,0.0023723538,1.2285922766,1.1784408092,-1.0734616518,2.3597648144,1.1720961332,-0.6565869451,0.7126896977,0.1096705943,-0.0268855970,-1.7138684988,0.7106988430,-1.0446763039,-1.2062015533,-0.3407447040,1.3466372490,0.6082682014,1.3727915287,-0.0571841560,1.0773525238,0.2110953927,-1.5239913464,0.6126305461,0.6644054651,-0.9984656572,1.3315540552,-0.3504710495,-0.1185619310,1.3879070282,-0.9998387098,0.5702096224,2.1872117519,0.7356611490,-0.3437471986,-0.6454073191,0.6962169409,-0.4820179045,-2.4766716957,0.9680387378,-1.1541639566,-2.0597898960,0.1503677368,-0.7456140518,1.4302666187,0.8453559279,-0.9875950813,1.8884847164,0.7621361613,0.6842812896,-2.5535175800,0.1136048585,-1.4359040260,-0.9684360027,-0.2280776203,-1.2567373514,0.4444446266,0.6932767034,0.5163866282,1.1237536669,-1.4660167694,1.9476058483,2.8177776337,-0.6734972596,0.5416264534,-1.4821751118,1.6413025856,-0.6007871032,-1.6072694063,0.3414277434,-1.1667805910,-0.3343241811,-1.9332513809,-0.1265669763,-0.1862681210,0.1386926919,-0.4867524207,0.4340206087,0.0739651024,0.9674845934,0.8658832312,1.0888012648,-1.3989909887,-0.0956165418,-1.4477106333,0.6629640460,-0.2317875922,-1.2718285322,-0.9693523049,0.5948191881,2.1844816208,0.5021012425,0.0804309994,1.8132755756,-1.8225727081,-0.8481187820,0.2896516323,0.6112565994,1.3894894123,0.8772185445,0.5663446784,0.7064805627,0.5084221363,0.0819462836,-0.4970123470,-0.6747379303,1.2025511265,1.6175380945,-0.8847118020,-0.4290374517,1.5003818274,0.4689340889,1.6352735758,-1.1227899790,0.8453468680,-0.2696839273,0.7648316622,1.3724489212,-0.1384429038,-1.1072838306,-2.4854152203,0.1164309531,-0.0418152809,-0.8732971549,-0.3142679930,0.7281127572,-2.0499973297,-1.5354900360,-1.2159559727,1.0636960268,0.0742390528,-0.4086834490,0.6344577670,-0.0591966920,-2.1613221169,-0.1440149993,0.3146380484,-1.0171301365,-1.6388372183,-0.9077087045,-0.2367976457,-1.8067845106,0.7331464887,-1.2680211067,-0.5259135365,-0.2642040253,0.4714785516,0.4978014529,-0.4443696141,0.4366762042,0.9659940600,1.0915999413,-1.3117097616,-0.4258576334,0.2191979438,0.5577743649,-0.3677088618,-0.2587479949,-0.2951894403,-0.1285194010,-1.7410380840,0.2037728727,-0.0169256441,-1.9258970022,1.1891771555,-0.1056173444,-0.2551109493,0.0803601369,0.3078361452,1.4552494287,-1.7418527603,-0.1786758006,1.3843970299,-1.8449815512,0.2047158927,0.7096581459,-0.2328862250,-0.3915565908,-0.6430918574,1.1635156870,0.7798740864,0.3655540943,0.2442674339,0.9248095155,1.3240067959,-0.3801167905,-1.3439861536,-0.6244315505,2.1410539150,1.2676112652,-1.6609078646,-0.8470503092,1.0326119661,1.1942317486,-1.9800231457,-0.7131582499,-0.3807858229,-0.8833439946,-1.1810083389,-1.3195772171,-0.0540605076,-0.0507104397,-0.0006503842,0.2843582630],[-0.1849077493,-0.9837927222,-0.4980651736,0.5068479180,-0.0115457019,-0.1425283551,-0.1853843629,-1.3138334751,0.6789863110,-0.1799073815,0.1523042619,0.8186873794,-2.3586330414,-1.1386350393,-1.8707106113,-1.3997253180,-1.6075724363,-1.7040657997,0.5580219626,0.8402561545,-0.4480331242,-0.8665820956,1.6506772041,-0.8412654996,-0.8535141945,1.0686321259,-1.8647556305,1.6807284355,-1.4227255583,0.7940302491,1.1190499067,0.1854119748,1.0127215385,0.4834828973,-0.0771656111,-0.3799327314,-0.0321431160,0.4283321500,-0.2940807045,0.4353268147,1.0631655455,0.1714275032,0.8127216101,-0.4509384334,-0.1770134270,-0.6772844195,0.1549729407,-0.0158453472,-1.1719727516,1.0151515007,0.5892142057,0.7689551711,1.2890911102,-2.1213917732,0.7964041829,-0.7404226065,-1.7036547661,-0.2855556011,0.8033329248,-0.1386539191,0.5521915555,-1.6757645607,1.1778060198,1.4586355686,1.4529005289,-0.1698000878,-0.6933089495,-0.2682672143,-1.6710566282,-0.0512348413,0.6260530353,0.7085033655,0.7272201180,0.3297210336,0.3958292603,0.2138654739,-1.0701493025,0.9027658701,0.6664137840,-0.4717260897,-1.6510242224,-1.2329187393,0.3288979530,-0.5451109409,0.5936804414,1.3355246782,0.1301572770,-0.2022518814,-0.8950264454,1.2793928385,1.5864177942,0.4107882977,-0.0241710469,0.5844276547,0.5538517237,-1.6992001534,0.6862621903,2.2658751011,0.8761395216,0.3837254047,-0.2647534311,0.4720512033,0.1841666549,0.2001533508,0.0668039396,-0.1149130762,-0.4432745576,1.1688767672,0.3746031225,1.0524545908,-1.1692347527,-1.2809635401,1.3166548014,-0.6610518098,1.4376925230,-1.0611702204,-0.5353015661,0.1217178032,0.6271507144,0.7130964398,0.5698034167,-0.0544330068,-0.0827762783,0.4918554723,-0.5561770797,0.9431702495,-1.2986550331,-0.9600256085,-0.1253585219,0.6193585992,0.2552787066,1.2878757715,-0.5704673529,-0.3831481934,1.0959254503,2.0086066723,-2.1474649906,-0.1529575735,0.8048326969,-0.4581208527,-0.0162758399,-0.2110669166,0.1082363203,0.6949726939,-0.5438286662,-1.1747916937,1.6570202112,0.5229940414,-1.2811999321,-1.8111737967,0.4468028843,2.0589504242,0.7505765557,1.5820052624,-0.9668190479,-0.3587667346,0.3426084518,-0.4487540424,-0.5114123821,1.7574405670,-0.4817022383,0.6514472961,-0.7424803972,-0.5344078541,-0.1580218077,0.3720195889,0.5527791381,0.4424994588,-0.1315764487,-0.4980100691,-0.5082452297,-0.1763692796,-0.4118542671,-0.7443788052,1.9393750429,0.8029169440,1.1061604023,-0.0243216679,-1.8400745392,0.6343280077,0.2006740719,0.5488215685,0.2727405727,1.0560905933,-0.6984013319,0.3588303328,-0.1235076785,0.0187691264,0.5021178126,1.6440634727,-1.8855284452,0.6837254167,-0.3057632446,1.5436513424,-1.0427814722,-0.5129017830,0.6372142434,-1.2669779062,-0.0673971698,0.4885091484,-1.0358808041,-0.4340546429,-0.3351761401,-1.0594669580,0.4214607775,-0.2359634340,0.1719900966,-1.6630712748,-0.7779294848,0.7398859859,0.9763131142,0.9751464725,0.3020714223,0.7720235586,0.8978174329,-1.3763316870,-1.1374385357,0.4448396564,0.0724404156,-0.7544128299,1.1938458681,-1.3248474598,0.0271121897,0.3545929790],[0.1016891822,0.8709347248,1.2646727562,-1.1754066944,0.7666998506,-1.0119038820,-0.3860526681,-0.8156091571,-1.6459562778,-0.5340557098,0.7559775114,0.2526681721,0.3399650455,-1.1079380512,-2.3619008064,0.1570393592,1.3197020292,1.2027004957,-0.4237133265,0.7454201579,0.6623315215,0.1792276502,-1.3496153355,0.3558274209,-0.6212102175,-0.8429967761,0.0761769116,-0.9115606546,-0.1469836384,-1.2940363884,-0.3781982064,-0.3462474942,-0.8051198125,0.9975365996,-0.4378167391,-0.3652188480,-0.4983713627,-0.3541530967,0.4649488926,0.4458468854,0.1212970540,-1.2046818733,1.1245608330,-1.0488841534,1.5659842491,0.3328139484,0.4238919020,-0.0850975588,-0.2209404260,0.2532872260,-0.3511019945,-0.0293736942,-1.6951823235,0.3349150717,-1.3477668762,-1.0378328562,-0.0748831630,-0.6586401463,-1.7394788265,0.2208534479,0.3594329655,0.4248667955,-1.0971237421,-1.8624976873,-0.8188779354,0.5210671425,0.1090878546,-0.7082766294,-1.1870042086,-1.0185188055,-0.0105035771,-1.0859087706,1.0135194063,-0.2151187509,0.8128567934,1.1123318672,-0.2369445115,-0.7196567655,1.3356708288,-2.0030705929,-1.7887951136,0.5355278254,-1.9258182049,-1.0679100752,0.4666056633,0.6799566746,-0.6068207026,0.0376300998,-0.8954614997,-0.1887252182,0.0670508146,0.2367895395,-0.7261124253,0.1988874674,1.1370029449,0.2628233731,-0.1744217128,1.0385823250,1.9289591312,-0.6241374612,-1.0708029270,0.7845878005,0.2213747650,1.2395253181,-1.6699419022,-0.5792590976,-0.7590466142,0.1721247435,0.6796442270,0.2983916998,-2.5116040707,0.4435839653,-1.0932787657,0.3189210296,-1.4347689152,-0.4141550660,1.2627629042,1.2916916609,-0.7968822122,-1.6745941639,0.6268520951,-0.2823364735,1.5425739288,-0.6658401489,-0.2700735033,1.0813314915,-1.7792905569,1.3893454075,-0.2835470736,1.1381509304,-0.5500882268,-0.3804549575,0.2835711241,0.6406458020,-1.9321515560,0.4521977603,0.0565207116,0.0297215879,0.1266185045,-1.1317875385,-0.0913078189,-1.4813172817,-1.6310836077,-0.1334815770,-0.5777752995,2.1017003059,1.8821809292,-1.0327913761,-0.4687115550,0.9024651051,-0.2668906152,-0.3431535661,0.9062344432,-1.4410095215,-1.8423423767,0.1403119117,0.3281984925,1.4122068882,-1.4440245628,-0.2864913046,-1.1767127514,-0.2729947269,-0.3813755512,1.2839496136,-0.3271493614,0.1473392993,0.5373675227,-0.5618190765,-1.8962076902,-0.4639317989,1.9236954451,-1.6677190065,0.1464398950,-0.3466111720,-0.0322197936,-0.4217617810,0.6667121649,-0.6521687508,-0.1843066812,-1.4433963299,-0.0677214116,0.1818892509,0.2523166239,0.0736405179,-0.3877593279,0.8917374611,-0.3416461349,-0.9169380665,0.5122210979,-0.4665381312,0.5767468810,-1.4800413847,1.7983590364,-0.6339222789,-0.2424547374,-0.4834831655,-0.8287428617,0.3135240376,0.8550209403,-0.0855247900,1.5032401085,-1.1833488941,-0.0465497859,1.0746041536,-0.9027399421,-0.5185764432,1.8878662586,-2.1824460030,-0.5935040116,1.2031533718,-1.2403950691,-1.1820745468,0.2421696186,-0.3005659878,-0.5867491961,-0.1843579561,-0.0163780041,0.4539252222,-0.9440818429,-0.3242641687,-0.9195576310,0.1757366210,0.5645468235,0.6839561462],[1.7646182775,0.0367704555,1.0817803144,-0.3612675667,-0.8710808158,0.6322647333,-0.3519022763,0.7156081200,0.2653824687,0.0456245095,0.6150549054,-0.2387576997,-0.9838037491,1.1115764380,-0.7091395259,0.5565314293,-0.1952365786,0.7066562176,-0.1020995602,-0.7186366916,-2.5378692150,-0.2224598378,-2.9098627567,-0.6451359391,0.3340830505,0.3082834482,-1.3182439804,-0.3153700233,0.3725237250,0.6292732954,-1.1076341867,0.1932128668,-1.1732625961,-1.2407459021,0.0289576929,0.4187066257,-0.1023253500,-0.5126762390,0.3325288594,-0.9176725149,0.3762895465,0.4857420325,0.6293258667,0.4774833620,-0.6179270744,-0.7745063305,0.2900193036,0.6556027532,0.7161750197,0.0969872698,-0.2772769630,0.9734714627,0.9022426605,0.5066912174,-1.3773590326,-0.4419030547,-0.9061026573,-1.2645193338,-0.6771960258,-0.7416522503,-0.4033834934,-1.7905031443,-0.7197990417,-0.1203307286,0.1458987892,1.0860456228,-1.3431071043,0.5452275872,-0.8861137033,0.0133407973,1.8350932598,-0.7494224310,-0.9539094567,-0.1490415484,0.4838488698,0.1974841207,-0.1861835718,0.4435926378,-0.8599298000,0.8017390370,0.8421186805,0.1312466115,0.6086190343,2.2241444588,0.9180830121,-0.1491888463,0.3608322442,-1.1499377489,-0.3584724963,-0.3109835982,-0.0412371941,-0.2816070318,-0.3876631856,-0.2826207876,-0.3808020353,-0.8287383914,-1.2302982807,-1.8592271805,-1.9187139273,1.2635271549,0.0478380919,0.5384370089,1.9528542757,0.0862558335,0.0315737054,-1.4575328827,-0.6429215074,0.2418305725,-0.8390690684,0.2132374197,2.1373596191,-0.7756822109,0.7898190618,0.3226329386,-0.6459203362,-0.8996171951,-1.1986262798,0.5406119227,0.3710613251,0.0836729556,-1.3073230982,-1.7027620077,-0.7785076499,-1.2514255047,-0.7407965660,-0.6227626204,0.7248088717,-0.9243300557,-2.0425086021,-0.1531879455,-0.8591698408,0.2854239345,-0.8992351294,0.6543528438,0.2941486835,-1.4519603252,-0.4732797444,-1.6374208927,-0.0668385923,-1.9343371391,-0.7832041979,-0.9100412130,1.6664811373,-0.2630090714,0.0338229686,0.2412451953,0.6096515656,0.3015289009,-0.1897550225,-1.3486781120,1.5449596643,-1.4853315353,-1.6852138042,-0.1775777191,-0.1915055960,-0.2492433488,-0.5336878896,-2.4988777637,0.8858756423,-0.8840622902,3.1992139816,-0.1212008819,-0.2876134515,0.0497608893,0.4297440648,1.5458735228,-0.1417457461,0.2272661030,0.4704810679,-1.0789257288,-2.4185669422,-0.8468856812,0.4660403132,-0.5138173103,1.7465744019,-1.1475670338,-0.1167239025,-1.4896283150,0.5956840515,0.9185001254,-4.8521175385,1.9427913427,0.1110671237,-0.8844205141,-1.5047377348,-0.0306469593,0.4130496979,-1.3862971067,-0.1759836525,-0.0979491323,2.1978659630,-0.3581565320,0.0429174267,-0.4899995327,1.6953723431,1.6068502665,0.8885279894,-1.8128461838,0.8054482937,0.3749721348,-0.1091318503,-1.4783062935,0.4781311750,1.1914623976,0.8886550069,0.9224624634,1.4349462986,0.6760825515,-1.1191790104,0.9839367270,0.5592536926,-0.3300490081,0.9047982693,0.6609485149,-0.0711592138,-0.9454471469,0.9492801428,-0.0225257389,1.1091521978,-0.3238544762,-1.5080913305,0.8890247941,0.1120769978,-0.3998112082],[-0.0686101541,-1.0087525845,1.1175528765,0.7182284594,1.0960410833,-1.1370993853,0.5957803726,-1.2771836519,1.3723937273,-1.0925372839,-0.0115856826,0.5942286253,-0.4304260015,1.0643801689,1.0266114473,0.5688973069,-1.3226926327,-0.0613019504,0.1347569972,0.3081897497,2.7131412029,0.4377599359,0.6334095001,0.3660160601,0.0500705987,-2.2779371738,-1.4221690893,-0.6358492970,0.1561469287,-0.9243812561,-0.2980656326,2.0219075680,1.4666409492,-2.5164093971,-0.3836316168,0.8046503663,-1.5332547426,1.2595324516,0.9782047272,-0.6557393074,1.0916570425,0.5810801387,1.0126934052,1.5139583349,-0.3119343519,-1.6407750845,0.7864783406,-0.8549495339,-0.7213819623,0.3934730887,-0.7415456176,0.3942766190,1.1725586653,-0.3623433709,0.6115031838,0.3008793890,1.5449849367,0.3181261420,-0.6514180303,0.2031024098,-0.2640877068,1.7698686123,0.6486811638,0.4432561994,-1.1322586536,-0.0290353186,-0.0781910121,-0.8639219999,-0.1526577324,0.1938848794,-0.5969279408,-0.4834834635,0.1573596299,-0.0592252091,-0.2702434063,-0.1515217274,-0.6052051783,1.9567449093,-0.4848371446,1.4323050976,0.0834007561,1.0406645536,0.0405718312,0.4692253768,-1.5332688093,0.2604510486,0.1545928121,0.5367847085,-0.3339368403,-0.4093825817,2.1708376408,-0.7797004580,2.4333009720,1.3753080368,0.2541124821,-1.9115686417,-2.1534113884,0.7856022120,-0.3899005949,-2.4047720432,0.1866433322,-0.6678485274,-1.3123197556,1.2337210178,1.6666272879,-0.1272123754,-0.9279864430,0.7762946486,0.9160736203,1.7719123363,0.7667344213,-0.1590543240,0.6486815214,-0.6689631939,0.5103842020,0.4520286322,0.1566184610,0.3792171776,1.0502470732,0.4308493733,-0.3508707881,0.7412344217,-0.3274298310,0.9234266877,0.9893772602,-0.8493971825,0.0270343702,0.4415463209,0.6724033356,0.1209470704,0.5693858862,0.0050581717,-2.1161911488,-0.6223433614,-0.5580531955,0.2278435379,-2.5865576267,-1.7090963125,-0.0353317894,0.5161625147,1.2569897175,-1.4674524069,0.3912377357,1.5026664734,-0.4857432544,0.0476039536,-1.3667999506,0.2982648015,0.2895137668,-1.2085553408,0.1725535691,-0.1539426297,0.6676985621,0.1313693672,0.3164666891,-1.0863517523,0.0785836726,0.7611423135,-2.1127955914,-1.4819056988,-1.0567213297,-0.8317061067,-0.4102806449,1.3269690275,-0.4393536747,0.0698533207,-0.6893197298,2.1084895134,0.2268670052,0.8511615396,0.3854487240,0.4431305826,1.0205435753,0.4819956422,0.5102502108,1.5450580120,0.8968815207,-1.0848166943,0.7952970862,2.0239479542,-0.7240387797,0.0220920593,-1.1563844681,-0.1014404744,0.3656550944,0.5663932562,1.0644277334,-0.5995697379,-1.2106146812,-0.8576260209,0.5674060583,0.6007801890,1.5416058302,-0.6908056140,-1.0867762566,0.2649148703,1.4798949957,0.7393624187,0.2967948020,-0.2771933377,-1.1488587856,1.6003367901,1.6553497314,0.6626985073,-0.6280882955,0.0936584100,0.4981217682,0.9492636919,-0.4618748128,0.2677331269,1.2989481688,0.0265531186,0.3216917515,-0.2674317956,-0.0111542614,0.2487320751,-0.3743169010,-0.9511997104,1.6651052237,-1.6196660995,-0.5902080536,0.3614725173,-0.4152885973,-0.7183656096],[0.3520698249,1.4739658833,2.4271173477,-0.5691303015,0.9813193679,-0.5925389528,-0.5128179193,-1.1542140245,0.5358704329,1.3890676498,0.5996650457,-0.2719255090,-0.0948925391,-0.7808255553,-0.6444920897,0.9755622745,-0.2208294421,-0.3493718505,0.2699482143,-0.7280527353,1.3917051554,1.4498575926,-1.2113507986,0.3349868953,1.3251229525,-0.9171952009,0.4963794351,-1.7986804247,0.0789784789,1.2199368477,0.3269993365,-0.1948697865,1.3435184956,-1.0425250530,1.3683353662,0.2499956936,-1.4721306562,-0.2956948280,0.5207143426,0.4645217061,1.4431642294,1.2225159407,-1.3223319054,1.0886017084,0.9286890626,-0.1376378387,-0.5915322900,-1.0464158058,0.5867432356,-0.8797436953,-0.7235412002,-0.6804904342,-2.0942678452,1.0294381380,0.7672358155,1.0971862078,0.6071217656,0.6025097370,0.8277061582,1.1057502031,0.5251725912,-1.1081231833,-0.4811174273,-0.3702764511,-1.3508657217,0.2259329855,-0.0718035176,0.1823831201,-0.2373090535,1.1184384823,1.3183101416,-1.9148426056,-0.4826577306,-0.4699207842,1.2780392170,1.4454777241,-0.2684786618,-0.2248070687,-2.3880140781,0.4959293306,-0.3267105520,1.3605651855,-0.5306251645,-0.0932557359,-0.5050152540,-0.7892034054,1.0203313828,1.2451971769,-0.4140982628,-0.3513950109,-0.9037069678,-0.0241906717,-0.9075904489,-0.6557211876,-0.1932841986,2.3157627583,0.5288925171,-0.6291821003,0.9991913438,-0.4957025051,-1.1099767685,0.0842455551,0.0248599015,0.0334466957,-1.6484893560,-0.8135964274,-0.0473386273,-0.7118042111,0.2833947837,-0.0167807993,0.2662077546,0.6260454059,-1.7930265665,-1.1567467451,-0.5512157679,0.6015375853,-1.2865725756,-1.9408442974,-0.1325186938,0.4100147486,-0.8295570016,0.1938338429,0.0979138017,2.5477495193,-0.4140664637,-1.7858626842,0.6429503560,-0.3502498269,-0.2588729262,-1.0729311705,-0.3541617393,0.3570177555,-0.6643162966,-1.4513443708,-1.6973505020,-0.0602100231,0.2496821880,-0.9163889885,-1.5405261517,-0.4512024224,-0.6341095567,-0.0097503057,-1.2270138264,-1.3736895323,0.1918285489,-0.9191774130,-0.6565604210,0.6039719582,-0.1332715899,-0.8285505176,-1.0401451588,-2.4385795593,2.2394487858,-0.2572565675,-1.8150984049,0.4597521126,0.4808021188,0.5467767715,0.6092917323,1.1429659128,0.2309983224,1.5645856857,-1.2469551563,-0.2577306330,-1.1540452242,-1.3649905920,0.6702425480,0.2360917628,-0.1505737454,0.3084661663,0.9137232900,-0.8299762607,1.7046604156,0.4507624209,1.6921821833,0.0341390707,1.4222145081,-0.5597399473,-1.0490896702,0.6614012718,0.7688275576,-1.1840931177,0.3911654651,-0.4891320169,-0.7211934924,0.8481540084,-1.0097805262,-0.3961494267,-0.4591137469,0.1611833125,0.1956343353,-1.2476583719,0.4416362345,1.1361719370,0.0931860134,-0.9278345704,-0.7907865644,0.6913827062,-1.3883576393,-2.3996989727,-0.3636757135,1.1655604839,-1.4245017767,1.2208950520,-2.0990407467,-0.9538810849,-0.5822237134,0.1402315944,0.0464295745,-0.5008851290,-0.2975231707,0.8082782030,-0.3997581005,-1.6819230318,-1.0130381584,1.7739200592,-1.0784034729,-0.1895323843,-0.7823113799,0.8008701801,-1.5388237238,-1.5723116398,-0.4203703105,0.1166450381],[1.0583444834,1.5439826250,-0.5834670067,-0.9337796569,1.9155551195,0.1372870356,0.7505593300,-0.8319081664,0.6255533695,-1.4047561884,0.1378288567,-0.8832240701,0.5066967607,-1.3870570660,-0.2949680090,-0.5365743637,0.7241915464,-0.0380382910,-0.4298193157,-0.3633871377,-0.1892514825,-0.9280431271,1.6019604206,0.9871795177,0.4851090908,1.4912133217,-0.0962753594,0.2499185055,-0.5457340479,-0.4101860225,0.2294923216,2.2696387768,-0.3302621543,-0.2712452114,1.5598413944,0.2520926595,3.1268947124,-3.2329590321,-0.2350266725,-0.1970554143,1.6455147266,-0.4833549261,1.8614071608,0.1210813671,2.4485468864,0.3696423173,-1.1890643835,1.1097162962,-0.9812495112,-1.5117832422,0.4652085900,1.0399012566,0.4068359435,1.6711966991,0.4793457091,0.3833797872,-1.2494162321,0.1649812311,-0.4304262102,-0.7297959924,0.8276520371,0.7847200036,0.5139238834,-1.9722926617,1.0811411142,0.1848393828,0.0775561258,-0.2357782871,0.1912350357,0.5073704123,0.0564742349,-0.8233909607,-0.8220529556,0.1913980395,0.8005524874,-0.1441638619,0.7258716822,0.2305646241,0.9593602419,-0.1887491792,-0.6131626964,-0.2583982646,-0.7700645328,-1.6128127575,-0.6146055460,1.2405661345,0.4227060676,-1.0741027594,0.3241637647,-1.4035487175,-0.0658114403,0.6804963946,0.5681362152,-1.3921498060,-1.0300476551,-0.2371842414,1.8360530138,-0.9216187596,0.9207280278,-0.5574217439,-0.4712163210,0.2855410874,0.3498257697,0.5446513295,-0.0277582034,0.9110357165,1.0241811275,-0.4904206991,0.0496969819,0.6285252571,-2.9861686230,-2.1029310226,-0.3938883245,-0.2817675471,0.2416146547,0.9426638484,0.3875665367,0.1224803925,1.0414155722,-0.3351289928,-0.4107232392,-1.5650410652,-0.8893459439,0.0849630237,-1.7995682955,0.9912497997,-1.8885445595,-1.1762053967,-1.2450914383,2.7380778790,-0.3616687357,-0.2209551036,-1.7513599396,-0.8957287669,-2.2897720337,-0.0876673013,0.6963365674,0.3551538885,0.9297693968,0.0609674230,2.4712345600,-0.4208989441,0.5249027014,-0.2286021709,0.2329783738,0.1197842136,-0.9520625472,-1.8651957512,0.2399929166,-1.2985645533,-0.6591146588,0.2112485319,-0.6970937252,-0.6887322068,-2.0674235821,-0.0465840735,-0.9805367589,0.4654271901,0.5672482848,0.7053048015,-0.6611399651,1.5472348928,-0.9092199206,-0.0546701252,-1.0351397991,1.3140008450,-1.5395072699,0.0404680595,0.9997606874,0.6848905087,0.4523021281,0.4604639709,-0.1260150522,0.1083656922,0.5990031958,0.7269591093,-1.7155878544,0.6685690880,-0.6821148396,-1.3120325804,-0.5852013230,-0.3806896210,2.0486714840,0.6823633313,1.0913326740,-1.6782044172,1.5952193737,0.6944016814,-1.1871331930,0.9472473264,0.5641186237,-1.0589094162,-0.8352508545,0.8253346086,0.5406584144,1.0564606190,0.6108237505,1.3554942608,-0.0652598366,0.3243359029,-0.4841716290,1.2476578951,0.2305612564,0.9184203744,0.7574568391,1.0057890415,-0.4710326791,0.0801935941,0.0023979940,0.6727606058,-1.4283015728,2.0315153599,2.1044881344,0.8079706430,0.2700591683,-0.1420193762,-0.4140471816,-0.8383685350,0.3478310406,-0.1102372110,1.3906400204,-0.4889595807,-0.1187727004,-0.8679751158],[-0.0362039842,-0.0926602483,-1.0390424728,-0.5257984400,-0.0478054583,-0.9813799858,1.6373766661,-0.0322443843,-0.4711576700,-1.7374867201,-0.3807227612,-0.1615162790,-1.1044647694,0.1208630577,0.8739864826,0.4768305421,0.6930779815,0.3838321269,-1.5314662457,0.3809277415,0.1716790795,0.3956367671,0.9040478468,-1.4133884907,0.8503904343,1.1443232298,-0.0336308964,1.1579585075,-0.3186286688,-0.1953814030,1.1942832470,-0.5953484178,-0.9550271630,0.2698577642,0.1056953147,-0.0633471906,0.3749876022,0.1402860582,1.2944872379,-0.4121560156,0.4817930460,-0.0993803889,-1.5939522982,-0.4357390404,-0.4619060159,-2.3683354855,0.2805862129,-0.3046934605,-1.2935652733,-1.1554673910,0.1633427739,-0.5986722112,1.2634762526,0.9150058627,-1.0899602175,-0.5049229264,-0.0143814981,1.5127053261,-0.8898098469,0.3804249167,-1.1658823490,1.2145162821,0.1391834766,0.9084006548,-0.0797242448,1.7454173565,0.8128961921,-0.3380572498,-0.8773782849,-1.1081911325,0.7548649311,0.2150626332,1.1833186150,-0.3489762843,2.1793832779,0.3019517660,-2.0800480843,1.3498222828,-0.6469517350,-1.0101433992,-0.0867938995,-0.3999692202,0.5472961068,0.4918823838,0.9543326497,1.3563923836,-0.8218570352,0.0116919894,1.2118961811,-1.2016319036,0.4808796048,-0.4284862280,0.2089256644,0.8705192804,-0.3592517376,1.0025163889,-1.3868917227,0.2231241167,1.6292181015,-0.7550464869,0.4189785719,1.3076460361,-0.3605156541,-1.1180034876,-1.1126116514,-0.6930063963,0.3895854354,-1.1811661720,-0.7325514555,0.1279658824,-0.5388088226,-0.3408534229,-0.2591611445,-0.1481301636,1.1229610443,2.1432387829,-0.3806581497,0.8855195642,-0.5941774845,0.8604633212,1.0511958599,1.0734173059,-0.2655522227,-0.3933710754,0.3349715471,-0.8550754786,-2.2061393261,0.9727731943,0.7498283982,1.6936372519,0.5487921834,0.0691117644,2.7273132801,-1.8114776611,1.1478933096,1.0795128345,-1.0359307528,2.3957827091,-0.7863538265,0.5545166731,-0.2966851890,-0.2659127712,-1.2508859634,1.1771451235,0.4923931360,1.2278065681,-0.7286225557,-0.0930251777,-0.5998708010,-0.2299500108,-0.4579644799,0.5729542971,2.3201682568,-0.8726349473,0.3805692196,0.8630654216,1.6897252798,-0.8851308227,-0.3429644406,-0.7624091506,1.3988976479,-2.7959020138,0.4372125268,0.0636285543,1.8906742334,0.0905186236,-1.0840857029,2.0455691814,0.9350713491,0.1695361584,1.2804718018,-0.6724713445,-0.3662121296,2.0804431438,1.7761652470,2.8678522110,-0.0991531014,-0.3440111876,-0.9633069038,-1.3825284243,-1.4679977894,0.2173433304,0.0366753228,-0.7773872614,-0.0060850084,0.2425571680,-0.3767814338,0.8276780248,2.0392413139,-0.2837807238,1.2338672876,-0.3843937814,0.5756478906,1.6414058208,-0.3666916788,-0.9714407325,-0.2356366962,-1.2913995981,-0.8960181475,0.1272795498,-2.2407054901,1.1917226315,0.4333340824,-0.3930523992,1.1191014051,0.6791875958,0.4361049235,1.0977270603,1.0344529152,0.3062209487,-0.0446278788,0.0910232663,0.5822839737,-0.1521469802,-1.1190607548,-0.2476054430,-0.7367978096,-0.4856684804,0.6659203172,-1.2921715975,1.8827733994,0.5838482380,-0.2108717859,-0.0243280269],[0.1365130842,1.4073339701,0.9856215715,0.3627391756,-1.3479270935,1.1825037003,-0.2711353302,-1.2824739218,-0.6591538787,-0.5253095627,-0.4016566873,0.4823636115,-0.6241627932,0.4012881219,2.4049458504,1.8389296532,-0.8160229921,-0.8917958736,0.2054117322,-0.5678172708,0.0877340585,-1.3987337351,0.3651104569,-1.0884479284,-0.0357880071,-0.7008450627,-1.4183855057,-1.0605516434,-0.5557442904,-1.4653967619,0.3975590765,0.9693913460,-0.5058177114,0.1420748681,-0.1394046992,-1.1864151955,0.5770588517,-0.4497047067,0.6133778095,-0.2091179043,-0.8822066188,0.9266041517,1.9369000196,-0.9379522204,-0.5523123741,1.6441167593,2.0208413601,-0.5548222065,0.0124798529,0.5058183074,1.1044448614,1.3189389706,0.1909236014,-0.3080491424,0.5630736947,-0.0837698281,-0.3891959786,-0.9374899864,0.6200323701,0.7414302826,-1.2315765619,1.9131008387,0.8843234777,-3.2155885696,1.7159427404,0.2663838565,-0.0736449733,1.1531600952,-1.0004495382,1.8306009769,0.4198108017,-0.5446417332,-0.3730129302,-0.5347300768,-1.3494056463,0.5697371960,0.3700086176,-0.8028948307,0.1239058226,-1.1565580368,-0.9970475435,-1.4502310753,-0.5761058331,-1.4261182547,-1.5865499973,0.8991324902,0.6901183128,0.3546708822,-0.3055213392,-0.1828400046,0.0213740449,-0.7595441341,-0.1451109946,0.2397838533,1.0029948950,0.7565121055,1.3437296152,0.9344487786,-0.0852555111,0.8469684124,0.5529686213,-0.2060049176,0.1530812085,0.6497737765,-0.7188387513,-0.6574992537,1.2068393230,1.4107749462,0.5801166892,-1.9565601349,2.3294208050,1.6649837494,-0.1143305302,1.1704581976,0.1861006618,-0.3432528973,0.7738602161,-0.7633075118,1.6620934010,-0.0148074497,-1.5225170851,0.4448566735,-1.0804424286,-0.2595146596,-1.2039898634,1.1865116358,0.1244517788,-1.7411261797,0.6327655911,-1.4254326820,0.6072432399,2.2639195919,-1.7797319889,0.1546354294,-1.6730787754,-0.6163903475,-0.3629196286,0.1454897076,0.6794376373,1.9794061184,-0.4524740875,1.8563537598,-0.1063185409,-0.9688279629,-0.1407084316,-0.5214707851,-0.2364092022,0.1224080771,-1.4739046097,0.2961368263,-1.8272217512,-2.0639176369,-1.3346837759,0.5843850970,-0.5584329963,0.0578550063,1.0357207060,1.7963436842,1.2943576574,-0.7003925443,0.8481504321,0.6891615391,2.0482206345,-1.2533652782,0.2402896881,0.6737480164,0.4490898848,-1.6458679438,-0.9338932633,-0.3591641188,-0.0254199263,-0.5341939330,-1.2297190428,1.7889677286,-1.4703223705,-0.1667094529,-0.8333431482,-0.8809360862,-0.0065073199,-1.0043187141,-1.0234842300,0.3805767596,-0.4275160134,0.5769917965,0.9083298445,0.7803912163,-0.8485795259,-0.3394566476,0.6788250804,-0.5510795116,1.0690231323,0.6100328565,-1.1385766268,-1.2801609039,1.3604898453,-0.1692980230,-0.9945854545,0.1831673235,-1.6491280794,-1.4920870066,0.5627005696,-0.2753271461,0.1657936871,-0.5342975855,0.2055897117,1.9000588655,1.7738777399,0.0581240840,1.7368767262,2.5812046528,0.3220744729,-0.3783294261,-0.3094934523,0.8187251091,0.9548506141,0.9769791961,2.0029132366,1.7150564194,0.0704670697,-0.3864088058,-0.6378495693,-0.0315747187,0.3025371432,0.4778320789],[0.3217836022,0.6492074132,-0.9109751582,-0.7691422701,1.4026347399,-0.9658105969,-0.1002218947,1.5368416309,-1.2793962955,-0.1795984358,-0.3374651372,-0.8700996637,0.4942047596,0.1025737524,-0.9360316396,1.4573398829,-1.1856033802,0.3710526824,0.5413604379,-1.3624647856,-2.2412455082,-1.4766739607,0.7899505496,-2.7598965168,0.4047561288,-0.6828455925,1.9673783779,0.6626994610,-1.0715668201,0.2047873735,0.0702401102,0.2123592794,-1.3596889973,-0.6493543983,0.7024531960,-0.7617487907,0.2701660991,0.5384673476,0.4484993517,1.4303277731,-0.2872730196,0.2255844325,-2.0188221931,1.1860226393,-0.0449120253,1.4178494215,1.1506255865,1.9027528763,-1.0151681900,1.4701216221,-0.3491511941,1.9576206207,1.7094520330,0.8521990776,-0.6261596084,1.6764523983,0.7147725224,-0.4068088233,0.6308442950,-0.0767498091,0.6154634356,-0.5383847356,-0.3225371540,0.4823666215,-0.3547261953,-0.2532891035,0.6753273606,-0.5546728969,1.0751554966,-1.3161867857,-0.9454118013,-2.4545342922,-0.0518297330,1.0433616638,-0.5003373027,-0.3093115389,1.7492716312,0.2976246774,-0.8289340138,-1.5835789442,-0.3375042379,1.9913307428,1.0772083998,0.0367077291,-0.0472101569,0.4658238590,-2.6238150597,0.9969777465,0.0754188821,0.3507779837,0.4585555196,-0.0999287292,-0.3928496540,1.7388290167,-0.8560393453,0.0992961749,-2.1302506924,-1.3153241873,0.8314374089,0.0599960461,-0.1368577778,-0.3394154906,-0.5296584368,0.2793669999,0.7099635005,-0.0835562423,0.6021608710,-0.1196800545,-1.3378901482,-0.3641176224,1.2721914053,-1.4890229702,-0.6869053245,-0.4686065614,0.6426337957,-0.5083370805,0.2268973142,-0.1974736899,0.2527569532,1.7184084654,0.8922979236,-1.5970692635,-0.4582478404,-0.4076239169,-0.1977892369,-0.7555869222,-0.3449466527,-1.2325458527,0.1456603408,-1.2386804819,0.1323075742,-1.8673850298,-0.0677953959,-0.8390944600,-0.2440842092,-1.1514731646,0.3661558926,1.9396024942,0.5101702213,-1.1429055929,-1.1398406029,-0.5165202022,0.3982322812,0.5776135921,-0.2317762971,-1.4540951252,-1.3447998762,0.8522616029,-0.5578137040,-0.2578280866,-0.0890806839,0.0349463075,0.6758262515,1.1467132568,-1.6023062468,-1.1934000254,0.7290937901,1.1782319546,0.3880271018,0.0174299795,0.7336113453,1.0139441490,0.5473438501,1.4955612421,-1.4451948404,0.6801576614,-1.3201237917,0.6385329366,0.0782859474,-0.0863106698,0.6152905822,1.3551502228,-0.0890230834,-0.0486132428,0.8171393275,-0.1492211223,-1.7590962648,0.9003215432,0.1985468268,-0.2245959789,0.6709069610,-0.9071885347,-0.0217190348,1.0644416809,0.2005730122,2.5758602619,-0.1752953529,-0.4584687352,1.0486868620,-1.1132804155,1.2848652601,0.0694471598,1.1292618513,0.9819657207,0.8070746064,-1.1838014126,0.9127116799,-0.0829647705,-1.8536667824,-0.0619276203,0.0663916543,-0.9791166782,3.1375920773,-1.5782490969,-0.0379903316,0.9323066473,-1.2534710169,-0.5725334883,-1.3511468172,0.7215948105,-0.5781445503,3.3989739418,-0.2205531001,0.3763498962,-1.2892030478,0.6944087744,-0.0149582736,-1.7329707146,-0.1396349519,0.8098288178,0.1786913425,-2.2183401585,-0.0800998956,-0.0742241070],[0.3845080733,-1.6463791132,0.7107392550,0.9891936779,0.7867619395,0.0749255121,0.2001665980,-0.1341113001,0.0893330351,-0.9379463792,-0.9581400156,-1.2719728947,-2.7973825932,0.5311394334,0.5251041651,-0.4277549386,-0.2632635534,-0.0099409046,0.7350210547,0.4192803800,0.0018239388,-0.0443116762,-0.0689535663,0.7293576598,-0.4967725575,-0.5072520971,-0.1359933764,-1.2276622057,0.6094608307,-1.6442064047,-1.6363502741,1.2297830582,0.0999848545,-0.0986485630,-0.0178937595,-0.2287111878,-1.5501198769,0.2100454867,0.8902381063,0.1387624741,-1.6011765003,-1.2235500813,0.3127393425,0.3142060637,0.8466959000,-0.4800876677,1.5377589464,0.2422349155,-0.9983308911,-0.0291883871,-0.1622420102,1.0121430159,0.6871752739,-0.4247593284,1.0550050735,0.3513199985,-1.4561393261,2.1552989483,1.3274356127,-0.5354955792,-0.6845734119,0.4479947090,0.3242218196,-0.5768624544,-0.3779707849,-1.0639026165,-0.2439878583,0.0086092334,1.6724859476,0.0878778622,1.1755329370,0.1444046646,0.6706321836,1.2580265999,-0.4368670881,-0.3676288128,-0.1864498854,-0.5201572180,1.1724785566,0.2010924965,1.6923856735,0.2290286273,1.6569188833,-1.7776489258,0.4573445320,1.3080205917,-0.3747129440,1.2378357649,-0.5824361444,1.1357333660,-0.9598888159,1.1630618572,-1.5749995708,0.7833538651,0.1442527771,-0.2554296255,-0.9792585373,0.4596745968,0.6203821301,0.5021705031,0.9194006324,-1.0334367752,1.6282829046,0.8093135357,-0.2234846950,1.2612539530,-0.9625234604,-0.6355591416,0.4715775549,0.0068701040,-1.0172108412,0.2494076043,-0.5978446007,-0.7592504621,0.8392009139,-1.3911912441,0.4128962755,-0.7735784650,0.0836772174,-1.5262907743,2.5723426342,-0.1008686349,-0.2837621868,1.0058437586,-0.7359182835,1.1496107578,-0.8255521655,0.7245010138,-0.8101114035,2.0787091255,0.8559275866,-0.5351945758,0.6291060448,0.0856874585,-0.0063943989,0.3543403745,-0.5699222088,0.2983883321,0.7868961096,-1.0871948004,-0.2908948958,0.0930618271,-0.8280441761,1.1082425117,0.2538062036,0.2027538866,0.1541826278,0.2480404079,-1.4028227329,1.1533126831,1.6422873735,1.0832636356,0.8322021365,0.8517474532,0.7841898203,-1.7440323830,0.4765495658,1.0487160683,-0.7914233804,-0.4405639768,-0.2160148919,0.1842859089,1.1629821062,-0.0767243430,-0.7312394381,-0.6774958372,0.3769921958,-0.5758907795,-1.5134738684,1.4181641340,-0.7427386642,-1.0262612104,-1.0112211704,-0.8954152465,1.0703066587,-0.3507716954,1.0855424404,-0.3428264856,1.1746728420,0.3893014789,0.9072918296,-1.5879071951,-1.7463401556,1.5358440876,-0.2369948775,0.2931667566,0.4586500227,0.2925986946,2.3936650753,-1.6313507557,-1.3064545393,-0.2756645977,-0.1694798321,-1.8411900997,0.1854819059,-1.0566730499,-0.1226647571,0.0523388982,1.1009833813,-0.8073753119,-0.0405311659,-0.1907524914,1.6200549603,-0.8034185767,0.9933840036,0.8365669847,-1.0088229179,1.3915550709,-0.6660532951,-0.4306976795,-0.2386864126,0.0843213871,0.5830325484,0.1325431019,-0.7190958261,0.4042440355,0.1904487908,-0.3511888981,-0.0677566752,0.7692873478,-0.9150775671,-0.2478805482,-0.1723368913,-1.7062830925],[0.1571991444,-1.2567185163,-1.2486581802,-0.3304790258,-0.1429319084,-2.0155272484,0.1789071411,-0.6599000096,0.2713014483,-1.5153937340,0.5429274440,-0.5614190698,2.1147344112,0.1305354685,2.3193845749,1.0932368040,-1.4592151642,0.8998706341,-0.5619531870,0.1160965636,-0.1549596637,-1.3603447676,-0.1224236861,0.1485626996,0.2863123417,-0.7079705596,1.2752975225,-1.8051139116,-2.4880578518,0.5680893064,0.7016727328,0.1439006329,0.4267070591,0.1790274084,-1.6500613689,-0.8564890027,-0.9328461289,-0.1378492862,-0.9564430118,-0.2885970175,0.2648217380,0.2335142493,0.8945885301,-1.3390487432,0.6258945465,1.6428123713,1.6459735632,0.5991826653,0.1086787358,-0.7074826956,0.9272280931,0.4744967520,-0.4278449118,1.0480604172,-0.4526143372,1.3277521133,-0.0564636923,1.3267620802,0.2104748785,0.3823283613,2.0361435413,-0.5682298541,-0.9197977185,0.2950363457,-0.0598035455,-0.3787401319,0.7625000477,-2.9526107311,1.0240620375,-0.1243128777,0.9883049130,-0.2431208044,-0.5595453978,-2.2868940830,1.2129172087,1.8089084625,1.2684593201,0.5851132274,-0.2312829345,1.8950910568,1.1503691673,0.6162592769,1.1002618074,-1.1529587507,-1.5807942152,1.0720236301,-0.8970527649,0.7691181898,-0.3447100222,-1.0881081820,0.6037220359,0.1806760728,-1.5382072926,-0.6406788230,1.2235890627,0.6681796908,0.5669136643,-0.9254377484,0.4890148640,-0.3914958239,0.3539678752,0.9485787749,0.3038055897,-0.1886343509,0.5120302439,-0.1569281220,2.2453000546,-0.4241433740,0.7208776474,0.3618130088,0.5907598138,1.0482422113,-0.7412730455,-0.1428454220,0.0379181169,0.2372295856,-0.2962428927,0.6757543683,-0.6153737903,2.5128779411,0.1799121350,1.3834085464,0.5711925626,0.9366635680,-0.7422040701,-0.5731911659,-1.0898987055,-1.5423468351,-0.4778312743,1.5007041693,-0.7780981064,0.2048971504,-0.5669568777,0.3746558130,-0.4002239704,0.6586911678,-0.3613056540,-0.9264248013,-1.0720075369,1.2189369202,-0.3600791097,-0.7939486504,-1.1968816519,-0.6835036874,-0.2224785089,-0.8625183702,1.2658147812,2.2403526306,0.8670475483,-0.9484440684,0.6970045567,-2.1961770058,-1.6428921223,1.0440677404,2.1168622971,-0.0151213622,-2.2564005852,-0.6929907799,0.2016869783,2.4280624390,-0.0699802339,-0.5621324182,-0.7554766536,-0.6078110337,0.8083912134,-0.8453896642,2.1699969769,-2.6128845215,-0.0212052483,0.4015052915,-0.5356868505,0.9977837801,0.1122059673,0.0778343603,1.3476768732,0.2835381627,-0.0142331179,-2.0367105007,-0.7371923923,-1.2246629000,2.2353093624,-0.7961319685,-1.9554548264,-1.2512631416,-1.0218428373,1.2124526501,1.1355682611,0.6193997264,0.2487773597,-0.2672690749,-0.4545343220,0.7911229730,-0.3282171786,-0.1699972153,-0.6668633223,-0.4596285522,1.5497270823,-0.0206197053,-1.1669906378,0.6233358383,0.7453544140,-0.4479801953,1.0420253277,0.1992836148,0.5260112286,-0.7913386822,0.5561696887,0.9695681930,0.4419442713,-0.2964356244,0.1331102252,1.4653676748,-0.4023957253,-1.9205198288,-0.1914123297,0.5629822612,0.4740603566,-1.2222735882,0.2560369074,1.1243366003,0.3371331096,1.7473640442,-0.4876609147,1.2396174669],[-1.2924531698,-0.8329555988,0.8106189370,-0.3956554234,-0.5539463758,-0.2932499051,1.0425081253,-0.3843310177,-0.2666507065,-1.8152126074,-0.6074080467,-0.0640053749,0.3151269555,0.6183727980,-1.3044564724,0.8262851834,0.3810087442,1.1846222878,0.3196219802,-1.6368120909,-1.0149875879,0.5286067128,0.2212347835,-1.3540037870,-0.2165237516,-0.0980249420,-0.1607593447,0.6369425654,-0.1619250774,-0.5857174993,2.6423957348,0.9990339279,-0.3168399930,0.3381016850,-0.3765596449,-0.3019950092,-1.2485268116,0.0268364325,-0.4783943295,0.1249814257,-0.4447103739,1.1281132698,-1.7312458754,-1.7791578770,-0.5942288041,-0.5394236445,-0.3606801033,0.0106915636,-0.0157427508,-0.8622555733,-0.0226995777,-0.0577265061,-0.1762067825,0.2407553047,-1.0976681709,-1.3848628998,-1.8822962046,-0.0017791095,-0.5731220245,1.9252378941,-0.4404511452,-0.4575495124,-1.0234162807,-1.6612168550,2.4114139080,0.2933978736,1.0817257166,1.0481520891,0.3456228673,0.0791619942,1.0115976334,3.3180394173,-1.0060584545,-1.0057104826,2.2909924984,-1.4247964621,0.7731633782,2.4017169476,-2.6709401608,0.1828530282,0.2281860262,0.2734143734,2.3842585087,0.7326384783,1.1197537184,-1.3578523397,0.9588159919,-1.0450409651,0.7155377865,0.1096529886,1.2337069511,-0.2804929018,-1.8858106136,0.3575464189,-0.0844301283,-0.8718315363,-0.9173175097,-0.6598443389,1.4191524982,-1.0103415251,1.1741003990,1.8414045572,-0.3743493259,1.0813381672,0.4680810869,-1.5805219412,1.5340808630,0.7638397813,2.2288279533,0.1072080508,-0.7255489826,1.1193445921,0.1728860289,-0.6225542426,1.1737477779,0.5976502299,-1.7421793938,0.2293184102,0.8057854772,-1.6511553526,0.4416771233,0.3466319740,-0.0454043299,-0.1209124625,1.6247614622,-0.7286786437,0.5841590762,-0.1831379086,0.5785822272,-0.4785403013,0.9208111167,0.3044312298,-1.4587692022,-0.1678542942,-0.6752638817,-0.2496262640,-1.6186631918,0.7310032845,-0.4667387307,0.0839912370,0.0778162628,-0.8575924635,-0.2125936151,-0.6107373238,-1.1801326275,0.6009773612,-1.1573953629,0.7150562406,-1.4650510550,1.7268512249,-1.0357637405,-0.4485883117,-1.4579374790,0.8677504659,-1.4565110207,1.5130494833,1.5456116199,0.0075432993,0.0928202122,-0.4449352622,-1.2877678871,0.8085286021,-1.3784750700,-0.0561535470,-0.4697008133,1.3949416876,-0.4398849010,-1.9523414373,-1.8618255854,-0.4945134521,-2.0245842934,-0.7269943357,-1.2706829309,-0.8815476298,-2.0346591473,-1.1727267504,-0.7687079906,-1.7004654408,-0.2370119691,0.4915896654,-0.7950913310,2.6508188248,0.6726636887,-0.4534219801,1.8084821701,1.0631934404,-0.7404429317,0.1299059689,0.2656057477,0.7115128636,-0.0633414313,-0.6535018086,0.7916250825,0.3691915274,0.0040150597,-0.6620145440,0.2596068382,0.4594883025,-0.5084525347,-2.4214904308,-0.9198400378,0.5192216039,0.4900861979,0.8938479424,1.7531205416,0.2292024791,0.2872477174,2.1692998409,-1.0889079571,0.8446975350,0.0144895865,-0.4923832715,0.2821266651,-0.1231191456,1.4335643053,-0.2643232346,0.8737241626,0.9721201062,-1.1100924015,-1.3487612009,-0.2477868050,-1.1834433079,-1.2005623579,1.0191519260],[0.7395281792,1.1104048491,-0.0219590571,2.0232183933,0.5489047766,-0.0274477396,-0.7132084966,0.2385355234,-1.7889448404,0.7535198927,-1.0275415182,0.0347306617,1.3003791571,-0.1093231812,-1.3040199280,-0.2029657960,0.5065885186,2.0733292103,1.3696361780,0.9754089713,-2.0454306602,-0.0382960923,-0.5714712143,0.2167275846,0.6750763059,-0.3483577669,-0.7746589780,0.6165201664,1.3618383408,1.2122330666,1.3800063133,0.5365054607,-1.9177306890,-0.5251973271,-1.0988540649,0.0278137494,-0.7636909485,0.6048128009,-1.1625010967,-0.7021710277,-0.6840939522,-0.2839722633,-0.9208801985,-0.3885153234,-0.9178700447,1.5394451618,1.4378056526,-1.4322452545,1.8636082411,0.4008088708,-0.4350753725,-1.1601147652,-1.1638344526,-0.0322148390,0.4595857859,0.9118782878,-1.5388293266,0.6242665052,-1.0804662704,1.2000719309,-0.1710569859,-1.0350323915,0.4459679723,-0.1440854073,1.3036524057,-1.7022899389,-0.9515473843,-0.6042004228,-0.5069737434,-0.2445824593,-0.6166288257,-0.1103350669,0.0383932628,0.3011859059,-0.5186234117,-1.4050905704,-0.3328391314,1.2396959066,1.4162265062,-0.3398074210,-0.1771123707,-1.0423727036,0.9640633464,-0.4511846006,0.9080410600,0.5163629055,0.9133321047,0.2308627367,0.9364045858,0.4994279742,-0.9036644697,-1.0831034184,-0.4359258115,0.0862119645,-0.6307891011,-0.0972120389,1.4832203388,-0.3554448485,0.2293844819,0.7009916902,-1.2562806606,-1.1880155802,0.8979741931,1.0003473759,0.8966107965,-0.1492953151,-0.1308869272,0.2263855487,-0.0130055640,0.4222289026,0.4016860127,-0.4904920459,0.7437108755,1.9387946129,1.6307559013,-0.5204173326,0.3517319560,0.0115969433,-1.4235920906,-0.4553895593,-0.1439766139,0.7038559318,-0.2786969543,1.5749616623,-0.0669014901,-0.8004282713,-0.7990862727,-0.4329472184,0.2307552397,1.6242434978,0.5721014142,-0.7301663756,-1.3531529903,-0.8805804253,3.4693908691,-1.3049875498,-0.4333794713,-0.2324779332,-0.7562892437,-0.0847490132,0.1238018200,2.0294234753,1.1775977612,-0.7600325942,-2.2919991016,-1.6500588655,-0.9651675224,-0.0731742159,-1.8575273752,-1.3492054939,-0.1087381393,-2.6830763817,-1.9446489811,-1.1404054165,-0.3651441932,-0.5104267001,0.8189151287,0.6377196312,0.5000778437,1.4043917656,0.2186368108,0.5616023540,-0.7406401038,0.7284206152,-1.1032539606,-0.1957814097,0.6142774224,-0.1744652241,-0.8923316598,0.5519254804,0.0332063623,-0.6735925674,-1.1467906237,0.5041069388,-0.0882616341,0.5556857586,1.6077281237,-1.5892181396,3.5689439774,-0.5289149880,-1.3064787388,0.1786384881,-1.1435884237,0.4807915092,-0.4079792202,0.0394755080,-0.6162922978,-0.6694833636,0.8787062168,-0.8178895116,2.6429357529,1.2247593403,0.0740591511,0.5969020724,0.4470560551,-1.7656972408,-0.9531686306,0.6847508550,-0.7460402250,-0.0438507125,1.1078300476,1.3620145321,0.3313216567,1.3880150318,0.4393956661,0.8094944358,0.4581452608,0.4303210080,-0.1151056662,-0.8882232308,1.1301658154,1.4888341427,-0.5208579302,-0.5673535466,0.0692926571,0.2035409361,0.4037210345,-0.4218095243,1.1342378855,-0.4237312078,-0.3344818950,-0.0618350804,-1.1918430328,-1.4559003115],[-1.0172894001,-0.8198907971,-1.1291286945,-1.2913620472,1.1161898375,0.9469137788,1.3321481943,0.6899978518,0.6809046268,0.2032615840,0.0857793987,0.1699928343,-0.1400194466,0.4252460599,0.1930265427,-0.6893171668,-0.0112007195,0.5478432178,-1.6128880978,0.5252740979,1.9394230843,0.2731349170,-1.6321368217,1.1112415791,-0.9279717803,0.1051486731,-2.1612613201,0.8048269153,-2.2167458534,0.2070944011,0.3835910261,1.3284368515,0.1795494854,0.3918175399,-1.3203768730,-1.4642324448,0.3817415237,0.5377549529,-0.2622683346,0.6836646795,0.3972084820,-0.4240120351,0.0491578691,0.1181937605,-0.7205049396,1.6013940573,0.1794056743,-0.3333885372,0.8673188090,0.7273941040,-1.3895088434,0.8160765767,1.0607300997,2.6599748135,-1.0260976553,-0.4237062633,0.6025332212,-0.3183956444,-1.0793176889,1.1519999504,-0.4337845445,-0.2645676732,-1.4520622492,0.3616208136,-0.7904537320,1.5489475727,0.0706833676,-0.7320204973,0.5462500453,0.9109166861,-0.2380175889,0.8746619821,-0.5622177124,-0.7081284523,0.1509067416,-0.9439134002,-1.5680643320,2.3613348007,-0.4723900259,0.2858115435,-1.2789443731,0.4329091311,-2.0931222439,-0.1401796043,1.3160349131,-0.5561891794,2.8609473705,-0.9401915073,-1.1080731153,0.3751921356,1.6322127581,0.6731638312,0.0600589253,0.4372738302,0.4696765244,0.9339655042,-1.3887944221,-0.2863088250,0.7338685989,-0.1506843418,2.5223164558,0.8215770125,-0.2824140787,-0.1048718765,0.8483038545,-0.8062709570,-0.2444159091,1.4299122095,-2.1488876343,0.4855590165,0.7521313429,-0.0362242311,0.9838863015,-0.9353536963,0.0884423181,2.2920429707,-0.1700564772,0.2196080536,-0.7678773999,0.1562586427,-1.7027182579,0.6213787794,-0.9202602506,0.6407473683,-0.1986954063,0.6446263194,-1.6563129425,-1.0442481041,1.4251111746,0.0319977812,-1.0316035748,1.1146814823,0.9316073656,0.4386779368,0.3499840796,-0.3451722562,0.1396140605,0.6647171974,0.8718655705,-1.3025929928,-2.4178709984,-0.2907205820,0.3312182128,-1.2822395563,-0.1600407064,-0.1754976213,0.9503936768,-0.2703380287,-0.1025897637,-0.9934663773,0.0249258950,1.2744728327,-0.8118573427,1.1149960756,0.1397129893,-0.7670947313,-0.6668843031,1.3282564878,0.5447962880,0.5307455659,1.3833870888,-0.0055647413,1.0632278919,-0.5771430731,-1.2756719589,0.1890316457,2.1345887184,-0.8619757295,-1.0646456480,-0.5164789557,0.5170358419,-0.2600931823,-0.4217748642,-0.4166208506,-1.8200933933,0.1230064183,-0.0741620585,-1.4303330183,-0.4159705937,0.3553825617,-1.4155803919,1.0580601692,-0.1156407297,0.5948669910,2.3360552788,0.2601666749,-0.0094588660,-0.9592040777,0.6389791965,-2.1392457485,-0.7021299601,-0.5663596988,-0.7259340286,2.0994577408,-2.1976509094,1.4211015701,1.4037487507,-0.5544083118,0.4942214787,-1.3881099224,0.5758545995,0.5089108348,0.7930374742,-0.9582809210,0.7822327614,1.4710839987,-1.6592271328,0.6535981297,0.3041735291,1.0828449726,0.4052551985,1.3086524010,-1.1172833443,-0.6074464321,0.3001531959,-1.0475873947,-1.2174046040,0.0880611092,-0.3901859820,-1.6000955105,1.0908169746,-1.0064556599,-0.0125739658,1.0592347383],[0.4535114467,0.9690465331,-0.9640182257,-0.8309549093,-0.0977261811,0.3498162031,0.6293475628,-0.2722138464,-3.0948634148,0.9942316413,0.7735679746,-0.2930199504,0.3510787189,-0.7872864008,0.3636801541,-1.6712719202,-0.5944473147,-0.7780117393,-0.0843283460,-1.4538215399,1.6016918421,-0.7877728939,-0.3877354562,-0.1187206879,-0.0575074181,0.7608479857,-0.7298144102,-0.8270982504,1.1454995871,0.9177365899,2.6103880405,0.0275212266,0.3144996762,-0.1462692618,-0.9897634387,-0.5029512048,0.6674743891,2.3210890293,1.0447092056,-0.4356173873,0.7188963890,0.3660807908,-1.3855001926,0.8256156445,-2.0150454044,1.1064456701,-0.6010576487,1.2496664524,0.1507250816,0.6486197710,1.2184954882,-0.1510147303,2.0641293526,-1.4498168230,-2.0317718983,1.2131361961,0.8736751676,0.1112035960,0.0250869896,-0.3772203326,-1.6222440004,-0.5841495395,-0.4699107707,-1.3665386438,-0.2026459277,1.0634649992,-1.0488432646,0.2799955606,0.5228345394,0.5676972270,-0.7177141905,0.8343642950,-1.1448087692,0.4393841922,0.5103766918,1.5104135275,0.0295173712,-0.8243343830,-0.7828230262,-0.6596770883,0.6153479815,-0.2708790898,-1.9587074518,-0.3796555698,0.2415509522,0.5919256806,-1.1046775579,0.1845662445,0.2122246623,0.6119584441,-0.3259581625,-0.3889915943,-0.7639250755,0.6078098416,-0.0667440817,-0.3067077100,0.5324385166,-0.4453656077,-1.1343411207,-0.5743299723,-0.2151301205,-0.2456267476,1.0301148891,0.4816589355,0.2701259255,0.0924869999,-0.7086822987,0.3224029541,-0.8674822450,0.1477069706,-1.4585555792,-0.7812615037,0.7472718358,-0.2691109478,-0.2376997769,-1.3751879930,1.0826475620,-0.4113312662,0.2539284527,-0.1638429761,1.3389561176,-1.0764062405,0.5501658320,0.7915554047,1.5759218931,-0.6999486089,2.9310317039,0.1950336695,0.4499127269,-0.5996899605,1.4481815100,1.6616878510,1.1369335651,0.2755799592,-0.4852996469,1.3345595598,0.3228714168,-1.3306968212,-0.4727328718,-0.6471650004,-1.1132016182,-0.0713217631,0.3562543690,-0.3879558742,-0.8892503381,-1.2379481792,0.8939266205,0.3848524690,-0.0012620862,-1.1376874447,0.5002755523,-3.1173944473,-0.3279280066,-0.8613476753,-0.2599486709,-0.5934633017,-0.5903510451,0.8368667364,1.4305448532,-1.0941609144,-1.2185326815,1.2903679609,-0.1296792477,-0.4799862504,-0.5509805679,0.4149641097,-1.7689479589,-1.5645645857,0.7041118741,1.1712943316,-1.5331878662,1.7426551580,-0.9360130429,-0.0077561252,1.1543563604,-0.4171964824,0.8241519928,-0.5553875566,-0.1821218431,0.5545003414,-0.9405351281,-0.7519527078,-0.1459754258,-1.1258925200,-0.0645399243,-0.0919869691,0.4392310381,-0.3822971284,0.9538373351,2.5823388100,-0.0202496629,-1.1847659349,-1.6862163544,-0.3795819581,-0.3198311329,0.2588284314,1.2887588739,-0.1577174366,0.3362316787,-0.3093566597,2.0915386677,-0.1336469650,-0.7076271772,-0.2241379768,-0.9879319668,0.3666142523,0.5691226721,0.8202407360,-1.7439919710,-0.5116857886,-2.3288261890,0.0181529243,-0.2852157354,-0.6766232252,-0.6026852131,-0.2991698682,-1.4322966337,0.4033516049,0.4984943569,-0.6238273978,0.6512020230,-0.1294459999,-0.2563645840,-0.0850734860],[-1.6635020971,0.1540960521,-1.0692788363,1.2856413126,1.3886764050,1.2394807339,-0.3786866367,0.0405688733,0.0261330679,0.7440382838,1.1676402092,0.9192047715,0.3690302074,0.6918146610,1.0192283392,-1.1002117395,1.5993460417,0.0842537582,-0.2882339954,-1.6641160250,-0.2254856080,0.2086762488,0.1024906188,1.9029095173,-0.1601699591,-0.1618199199,0.0761273950,0.7380160093,-1.0263552666,1.2613258362,0.7727131248,-1.4276200533,0.2427285612,0.9233678579,-0.6908003688,1.1374328136,-1.5583342314,0.3230153918,0.5732840896,0.5374802351,0.0478959084,0.6080489159,1.9260646105,-0.1249577850,1.1183328629,-0.2025958151,0.9080084562,-2.0576729774,-1.0507560968,-1.0327750444,-0.2893113196,-0.5023987293,0.6811234355,-1.2514544725,0.3200329244,-1.2652056217,0.3359721005,-0.0252995491,-0.7358947992,1.0601046085,1.8516906500,0.8605123758,-0.3745788634,0.5313572884,-0.7876136899,-0.6817808747,-1.7882242203,0.7149001956,-0.8047255874,-0.2163498551,-2.7289826870,0.0985979512,0.5790400505,0.2990364432,-0.3108656704,0.4906449318,-0.0220372509,0.3153430820,-0.6366631985,0.5582022667,-0.0725143254,1.6017705202,0.3281800449,0.0548433438,-0.6032658815,-0.0313083753,-0.6846083999,0.8272629976,0.3764564991,1.2219976187,0.3424511850,-1.9520853758,0.5699878931,-0.2838295102,-1.2962739468,-0.3520423770,-0.0343406647,-1.7270351648,-1.5421413183,0.1779328436,-0.9161617756,-0.4136451185,1.1171588898,-0.4643569887,-2.5890705585,-0.0978169665,0.7960562706,-0.8719828725,-0.5912626386,0.6984806061,-0.5227940679,0.4376514256,-1.0884710550,-0.2189821005,-0.1773026586,-0.2215059102,-0.3727942407,-1.6285271645,-0.9006853700,0.0758861899,0.5086434484,0.3348540664,-1.3667118549,1.2991203070,-0.4577853084,0.2003161907,-0.0515099727,-0.8061420918,1.6344506741,0.0430850051,0.3335125148,0.4028064907,-0.2435404509,1.1255035400,-0.3624939919,1.3392149210,-1.1797457933,0.3497534692,1.7610549927,1.1116669178,-0.1403288543,0.5325508118,-0.4736567140,-1.1872699261,-0.8908538818,0.5292066336,-0.6279459596,1.1363736391,0.0741886124,-0.0612685420,1.8719406128,-0.7305373549,-0.7488399744,1.4618368149,1.0680894852,-0.5133447647,0.3789470494,0.6920509338,-0.0438605398,0.6057320237,2.3371551037,1.6273084879,0.4970181286,0.1759300977,-0.4805933237,2.0895617008,0.2881127298,0.1409717500,-0.9227795601,0.2344621420,0.1607084423,-1.3211661577,0.2385593355,-0.5965131521,-1.2087808847,0.2794382572,0.1888812184,1.9271669388,-0.6863006949,-0.7454478145,1.3094059229,0.0873842910,-1.5320618153,0.5683317780,-1.1412267685,-0.0090721669,0.4966467023,-0.9476978183,0.3827875555,-0.1609680057,0.6752560735,-0.7131679058,1.4439325333,0.8276761770,0.4239243269,0.0850653648,0.4328988194,-0.4906337261,0.0462160818,-0.0169979334,0.7315130830,-2.4731819630,-0.3385759592,0.8918678761,-0.3003723025,1.4887953997,-0.5745051503,0.1477561146,0.9416487217,-3.4703557491,-0.5359448791,0.7355026603,1.0419493914,0.2170319855,-1.0533621311,-1.3572306633,0.2210417986,-0.2556262016,0.8509920835,1.1125905514,1.2063499689,1.5522272587,-1.2215622663,-1.2061839104],[-1.2950669527,0.2025153190,0.8298991323,-1.6604008675,0.6038448215,-1.3068019152,-0.0830953941,-0.1207791269,0.1019082442,-0.4354915023,1.0499731302,2.1721775532,0.8277935386,-0.8255026937,-0.3824424446,-0.6085035801,0.4934102297,-0.3934786618,-0.3122988045,-0.6388264894,-0.4257539213,0.1983288080,0.1787884980,-0.6914410591,0.3757758439,0.3676111102,0.0018327866,-0.3925342262,0.0072206180,-0.7803044915,0.7397019863,-0.3460148573,0.6108574271,-0.1799765527,-0.5952509642,1.7402403355,1.2058306932,-1.7820718288,-0.4921284318,-0.1624327749,-0.4435231984,-0.9929452538,-0.2207475901,0.9404109120,0.5551370382,-0.2299775779,-1.3604174852,0.6192497611,-0.1024679542,1.5234620571,-1.3118313551,-0.4637701213,-0.3026833534,-0.6559758782,0.7063180208,-0.1408954412,-0.2510201633,0.9387018085,-1.2084950209,0.9204330444,1.0249712467,-1.1086518764,-0.6219455600,1.0969376564,-0.9379596114,-0.8758180141,0.6695581079,0.4155954719,0.1114221439,-1.5905003548,-0.6057395339,1.1781516075,0.5552685857,0.0532497801,-0.8565030098,-0.0965542048,0.4256413877,0.5131956935,0.7127240300,-0.9340515137,-0.2032660544,-0.7067037225,0.8651694655,0.4279882610,0.0549685322,-0.3155857027,0.9081064463,-0.7040295601,0.5488665700,-0.7164933681,0.6457732320,-1.0947531462,0.5694273710,0.2458718419,-0.9906666875,0.2122940868,-0.1173533797,-0.4974184334,-0.6052320004,-1.2996211052,1.6052107811,0.0167007670,-0.8896363974,-0.1235675514,1.2180057764,-1.2930305004,-0.0934745148,-0.8663709760,0.5964619517,0.0963961259,0.7674152851,0.8012778759,0.5961014032,1.0097689629,-0.7147139311,-0.8749305010,-0.6941998005,0.9761347175,-0.4838216305,0.8083631992,0.0955365077,-0.7880268097,-0.0426202081,1.8439741135,0.4282551110,-0.0741682723,1.0584558249,-0.6867261529,0.8829703331,0.5617801547,0.3434916139,0.8344439864,0.4797274470,-0.3557169139,-0.3400082290,-1.2465457916,-0.0580307133,0.3834264576,-0.2598493397,0.3235251904,1.4750258923,1.1747642756,0.3843635023,-0.6985956430,1.5027586222,1.4956634045,0.8067368865,0.1969479024,0.0862167776,2.6182115078,1.2178329229,0.4351801872,-0.2157924771,0.0267683528,-0.2018048316,-0.2664071023,2.3273704052,-0.2287864387,-1.2879660130,-0.2599785924,-0.9273023009,0.5641449690,0.7914847732,0.9565588236,-0.8911844492,-0.0868590698,0.8247932792,1.1331245899,-0.7190009356,-0.5144978166,-0.2154463977,0.4985496998,0.9098943472,-0.2854295969,1.0122947693,-1.0168125629,-1.7378758192,1.9295582771,0.2372324765,1.4199649096,-1.4571065903,0.9987430573,1.4982053041,-0.1646643132,-1.0409728289,0.2996204197,-0.3438556194,0.1763267219,-1.2381007671,-0.5269424915,1.0394705534,-0.5188234448,-2.2872128487,-0.8193461299,0.7512777448,0.5032290816,-0.2287727147,0.0983647630,-1.3239839077,-0.3555992246,0.3286261261,0.3280561864,-0.0815626234,-1.0128548145,-0.6927438974,0.8964392543,0.8159549236,-0.3390179276,-0.3015466630,-1.4631524086,0.4497293830,1.3367420435,0.3859525323,0.7919450402,-0.7560143471,-0.4574146271,-1.0860054493,-2.2208795547,-1.6294802427,-0.0413829871,-0.2428517640,-0.4857134819,-1.1662257910,-0.9579188824],[0.7621256113,1.3296331167,0.3138369620,-0.4716928005,0.6125260592,0.3406782746,0.0522802398,1.0202609301,-0.0340798423,-0.9259276390,-1.2390106916,-1.5818425417,1.1560853720,1.0905010700,0.7594812512,-0.5976432562,-1.1142151356,1.5844764709,-0.3622460365,-0.1226565912,-1.9871875048,1.4768576622,0.2379653603,-0.9133228064,-0.7880818844,0.9237326980,1.8276497126,0.7280939817,-0.5917063951,1.3107105494,0.7642640471,0.8204598427,0.5962021351,-0.3783559799,-0.4161889851,-1.4495459795,0.0745454207,-1.3453316689,0.9977020025,-0.4342275560,1.0422127247,1.1708983183,0.4311306477,-0.6837021112,2.1621861458,-0.5827428102,0.4477067292,-0.0211558007,0.1824309528,0.4264740944,0.1203467995,0.9909948707,-0.1859768480,-1.0240132809,-0.3639334142,-0.0936418921,1.0512423515,0.3733913898,0.6839901805,-1.9282292128,0.5622627139,-1.2091028690,-0.1106549725,-1.2581288815,0.8361244202,-1.1004523039,-0.1258943379,-1.5989625454,1.8102275133,0.4441590011,-0.5214574933,0.9797497988,1.2888114452,-0.8276115656,-0.0220992491,2.4498775005,-1.8377934694,-0.8514962792,0.9489679933,-0.0830500126,-0.0359301940,-0.1420495808,1.0556294918,-0.5497905612,1.0126923323,-0.7473862171,0.4052971900,-1.0751425028,0.1719949245,0.3107260466,0.4252511263,1.0308989286,2.1891362667,-0.7951731682,-0.9288254380,-0.8406194448,-0.5870738029,1.4166525602,-0.0508782193,-0.5263801217,1.7727472782,0.1618817300,-1.6156774759,0.9979695082,-1.5955677032,-0.1549735814,1.2778108120,1.1898374557,1.2261581421,0.1232592538,-0.4701025486,0.8268409967,1.4406825304,2.0723600388,0.6214050651,0.1583388299,1.0687154531,1.2600852251,-1.7723824978,1.7461088896,1.7796293497,0.1533010453,-0.4502541423,2.8840515614,-0.0278098118,0.2774529457,1.2356098890,0.9059458971,1.3719466925,-0.1427477598,-1.5313657522,-0.1988708526,0.0918548554,-0.7983700037,-0.7351155877,0.0807933062,-1.1822363138,0.4204095602,1.6773831844,-0.6493386626,-1.3810855150,1.8829346895,2.2381758690,-0.3106609285,0.1599945575,0.5009009242,0.2571044564,-0.2493799925,1.4916465282,-0.0642954409,0.1002008095,0.0911449566,-0.2840108871,0.0648470521,-0.5927429795,0.6111506224,0.2633831203,0.6087102294,-0.1314200610,-1.9108045101,0.3572774529,-0.2191513479,0.4376700521,1.1833018064,-0.8609423637,-1.1047463417,-1.0791573524,0.6086138487,2.6700644493,-0.7331061959,-0.4262737930,-0.6036394238,-0.7591724992,2.0492615700,-0.4318020344,-0.4356160462,0.4063673317,-0.0951606557,0.1538648903,-0.7921331525,-0.5645645261,0.6250701547,0.2017248124,1.6619211435,-0.0490566455,0.8955783844,1.0819174051,-0.8749611974,-0.5747096539,-1.3129776716,-0.8326802254,-0.5609090924,1.6952314377,1.0350174904,1.0616799593,0.9478091002,1.2279448509,-0.0191568751,-0.1780776232,-0.5881857872,-1.0479393005,0.2833366990,1.5692890882,0.0149694970,-1.1679285765,0.6675914526,0.2725379169,0.5069898963,-0.8566572666,0.2730761468,0.7228215337,-0.7045961022,-0.8416188359,0.4508380890,0.1944348961,-0.4541225731,0.7563244104,-0.1131819710,0.7128592134,1.4402623177,0.1528645456,1.0313960314,0.6298319697,1.2467131615],[-1.1085821390,-0.9003415704,-1.4343361855,1.0231429338,0.0072061568,-1.8835331202,-0.7989682555,1.0520348549,0.3447473049,-2.5748610497,1.6703809500,0.0138393026,1.1409097910,-1.1738827229,0.7495995164,0.5710754991,-1.0748022795,-0.6461317539,-0.2453123629,1.1448947191,0.8853188753,0.7727425098,-0.0282142591,-0.1457471848,2.4659855366,-0.7119729519,-0.1440470964,0.3802301586,0.0594930463,-0.7933204770,-1.5470353365,0.9903058410,-1.0986013412,1.3785964251,-0.6182013750,-0.2940633595,-1.6926203966,1.8205765486,-0.0862483159,1.7647430897,-0.7206363082,-1.1045497656,-1.6156738997,-1.0962505341,2.8444337845,0.5576008558,1.6882987022,-1.1970424652,0.4513413310,0.9049255848,0.2253855616,-0.6835092306,1.8480138779,-1.3671431541,-1.0080400705,-0.9925702810,1.2446583509,-0.5805303454,-0.1164868474,-0.5888524652,0.2782382667,-1.0922057629,0.9598587155,-1.0899872780,-0.7082154155,0.3065707684,-1.5383666754,2.1593325138,-0.5230810642,1.0688405037,0.5502248406,0.2891297638,0.3945007026,-0.7157661915,-1.4161725044,0.5869718194,0.1101675332,-0.9291577339,0.9818926454,-0.2781111300,-1.0967657566,0.1139538214,1.1031048298,1.0086518526,0.6278707981,-0.1493199915,-0.4495257139,0.2201409191,0.4300591052,1.2368929386,0.6283149719,2.0061483383,0.7451803088,-0.2425652146,-0.2525975406,1.1054331064,-0.5970991254,0.9908251166,0.4830693007,-2.2409613132,-0.8885762095,-0.0143143935,1.0882319212,0.2041956186,-0.9607824087,-1.1832842827,-0.3721539676,-0.3367851079,0.2151497006,0.0878699198,-0.6686208844,-0.3243316412,-0.5673260689,0.2042728662,-0.0433752351,-1.7372738123,-1.4927453995,0.2947275341,1.8073326349,-2.2004306316,-0.1771846712,0.5147547126,1.0263893604,-0.3667627871,-0.3732538819,-0.8672274947,-1.5185539722,0.1291330457,-2.3146927357,-0.0381119587,-0.6768706441,-1.1104094982,0.1284427047,0.5935358405,-1.7596523762,1.6137292385,1.4737104177,-0.3111882508,-1.1401060820,1.3278982639,-0.7087484002,-0.5006707311,0.9336168766,-1.2285281420,1.1852614880,-0.6464233398,0.3168686628,1.0129629374,-0.9842917323,-0.8322681189,0.3770053089,0.4055832326,-0.7535272837,-0.5763852000,1.2844207287,0.2033589929,1.0656037331,-2.3893010616,0.2218722403,0.5269311070,-0.5230823159,0.2871558368,-0.6817102432,1.0522680283,-2.2263123989,-1.4442704916,-1.7066681385,-2.1534831524,-1.1033091545,0.5186153054,0.9830374122,-1.4134610891,0.7257140875,-0.3652280867,-0.8273479342,-0.0123440940,-0.7055724263,-0.0065825772,-1.4281263351,1.0454120636,2.1973111629,2.1394906044,-0.2185653597,-0.7726922631,0.0578657836,-0.1241003796,1.3615926504,-0.4700821936,0.6276119947,0.7610633373,-0.3084032834,-1.5049169064,1.1694749594,-0.3002104461,-1.7592983246,-0.7689751983,-0.6482440233,-1.3943746090,-0.6685357690,-0.2740963697,-0.1009262577,1.3430944681,-0.9573187232,-0.4756442606,-0.1120443791,0.4514842033,-0.1808938533,0.0360276587,0.7046375275,0.5217391253,0.1769850850,-0.3516299427,-2.1484513283,1.0110119581,-2.1996293068,0.8975638151,-1.2163029909,-1.0508220196,0.6954064369,-1.2345478535,0.6365551353,-0.1546197087,0.5181404352,-1.8113756180],[-0.5626225471,-1.3982005119,-0.4505920410,-0.2327025235,-0.0388276242,1.5718796253,-1.1048280001,-0.1993524432,0.4389081597,0.7646227479,-0.5730227828,-0.5616265535,-0.3296748400,-0.7627649307,1.8219025135,1.0186827183,-0.7380231023,-0.8434823751,-0.6264718771,0.7264119387,-0.5833688974,-1.6604254246,0.0730206072,-2.4509689808,-0.5503947735,0.6281988621,0.3269348443,0.0925165340,-0.4955527484,-0.2733169198,-0.5889083147,1.7769594193,-1.1998845339,-1.7700980902,-0.8073721528,0.0494673774,-0.8524341583,-3.2187163830,0.5676971078,-2.2943770885,-0.1579331160,-0.7956486940,2.0401656628,-0.1461989731,2.1896297932,1.5324863195,-0.5835031867,-1.5549209118,-0.4209410548,-0.0279977787,-0.0782606602,-0.7426626682,0.0121981045,-1.5553230047,0.5819070339,0.0326453373,0.6056950092,-0.4712967277,0.4049454927,-2.1062376499,-1.1619187593,-1.2910219431,-0.6611953378,-0.4365321100,1.7293006182,0.4786488712,-0.2935953736,-0.4926992655,0.7536605597,-0.3894644082,-0.7081032395,1.0595657825,-1.0791609287,-2.6619422436,-1.6152977943,-1.1396865845,-1.2950086594,-1.2873415947,-0.1941757053,0.5377328396,0.8336847425,1.3057926893,0.7935741544,-1.1920603514,-1.2580881119,-0.1752467602,1.4270538092,0.3531105220,-0.2798886299,0.2382898629,0.5952281952,-1.3373407125,-1.5224821568,0.6214637756,1.1671440601,0.1557538509,-1.3565181494,1.1534074545,0.0552476197,0.8227424026,-0.2473755926,1.7100614309,-0.6010099053,0.0952295437,0.7519122362,0.6125386953,-0.1340970695,-0.2649757266,1.2478978634,1.3848875761,0.2351858020,-0.7321636677,-0.7952386737,0.4714525342,-0.7676478624,1.6104269028,1.1910309792,1.6014332771,0.8937367201,-0.5093985200,0.7270994186,0.1354524046,1.1415119171,-2.6321196556,-1.8495267630,-0.5692487955,-1.5690562725,-0.2997058034,0.9866194129,-0.0548178814,0.1174306124,-0.1186611429,-1.4897632599,-0.0651347712,1.4272356033,0.8654854298,0.3896121979,0.0904301852,0.0244684536,0.9569140673,0.5948505998,-0.6984776258,-0.2093743682,-0.1100684255,-0.4356186390,0.9230844378,-0.0802567452,1.0543168783,0.8095011711,-0.9505899549,1.5887154341,0.9821254015,-0.4611521661,-1.1893174648,1.1560828686,2.1422307491,-0.3830331862,0.2881630659,-1.0069626570,0.9505732656,-0.6370865703,-0.5005722046,-0.5218068361,0.0570890643,2.1391377449,-0.3942484856,-0.0125554344,-1.0029050112,-1.6481519938,-1.3115295172,-0.6975313425,1.4723504782,-0.5125975609,-0.7702323794,-0.0430547260,-0.0723921731,0.3706169128,0.9811360240,0.0741774514,-1.2132822275,-3.3102068901,0.2480957657,0.1500856727,-1.3044787645,-0.7662258148,-0.5189638138,-0.7032186985,2.0692028999,-0.4092837572,-1.9459711313,-1.0947782993,-0.8841983676,-0.0839635059,-0.8565853834,0.7222669721,1.4330573082,2.0029449463,0.6141778827,0.4747509956,-0.5246588588,-1.0898851156,1.2181375027,-0.7382065654,-1.0789153576,-0.4607920051,-1.2100571394,-1.8802586794,0.2697679102,-0.0484813824,-1.1286646128,-0.0754335970,-0.4993629456,-0.9535911083,-1.2571040392,-0.3249886036,-1.7524542809,0.1891974956,0.5681154728,-0.1350892484,-0.1838773489,-1.8538595438,2.1625046730,1.0804477930,0.8050185442],[0.0583708286,-0.8403818011,0.4379777312,0.2320236266,-0.5907236934,0.5305605531,-0.2355878800,0.8831157684,-0.3832046986,0.1206885651,-0.2441270649,1.4652829170,0.1251680106,1.6854983568,-0.5830694437,-0.8190350533,1.2951843739,1.1042377949,2.4435899258,0.6359903216,1.0345590115,0.0918972865,1.2627774477,0.6608592868,-2.0900311470,0.3947992027,1.1637389660,-0.5153884292,-0.8478496671,1.4432096481,-1.4082688093,-0.2457654625,0.3125251830,0.4924857020,-0.0458401740,1.0880835056,2.3190202713,-1.3728237152,0.9819077253,1.4168225527,-0.8269272447,-3.1057460308,-0.2784600556,0.0084761279,1.1402667761,-0.1192869022,-1.1153726578,0.4603817463,-0.4175572991,0.5515987873,-0.8892507553,-0.9335702062,0.1421700418,0.8622007966,-0.1144793853,0.7831965089,-0.8715204000,-0.0539018773,0.7396158576,-0.2736086249,0.1636993736,-0.3604790866,0.6992456913,-0.4008758366,-0.5472824574,-1.3291764259,-1.1045629978,1.3297357559,-0.8264792562,-0.6032384038,0.1439969242,-1.2441627979,0.7882626057,0.2286830693,0.4731123149,1.1700091362,0.3139848411,-0.3568086028,0.3434693217,-0.7872848511,0.1237977520,-0.9416929483,0.4094843268,-2.0613017082,0.8186947703,0.2799305320,1.6776723862,-0.9916943312,-0.6852881312,0.8760497570,0.3148263097,0.6045241356,-0.8686185479,-0.6712832451,0.0449492224,0.1750403196,-2.0695581436,0.2488401681,-0.4023043811,-1.9713195562,0.9366938472,0.3973978460,-0.8258979321,0.2593599260,-0.4171347618,-2.0138168335,-0.2195019126,1.3088977337,-1.0805025101,-1.2335897684,1.7145081758,-0.7687286735,-1.5577871799,-0.7377884388,-1.0902771950,1.3249225616,-0.0432096124,-1.5101686716,-1.4007518291,-0.0030049440,0.5537450314,1.1865566969,-0.5845212936,1.2676486969,0.6573771834,0.4485799372,-1.3348814249,-0.2957971096,0.7416982651,-1.2257262468,1.2528117895,-1.4885023832,0.5322642922,-0.6019110680,-1.2413516045,0.3678843975,0.5308324695,-0.1434124857,-0.0195947830,-0.3229777813,0.5714412928,1.2527539730,-0.1473772079,-0.0419942997,0.0068792035,0.9574351311,-0.2035091221,-0.0903420448,-0.3691366613,-0.5072920918,-0.5398548245,-1.3546386957,-0.6282287240,0.5655155778,-0.5688884854,-1.9287480116,1.2996890545,-1.3261818886,0.2849215567,0.6106883883,-0.1060630158,-0.4809936285,0.0340173580,-0.9125748277,0.7783385515,-0.1310227066,1.2201029062,-0.3814059794,1.3374848366,-1.2288342714,-0.1551678479,1.0072926283,-0.6144304872,0.0839847699,-0.6462374330,-0.0756285638,0.7238970995,0.2455556691,-1.4122244120,-1.8843190670,0.2838617563,1.0761822462,0.2097860724,-2.2004213333,1.2754834890,-0.4360707998,0.8155106902,-0.0288391188,-1.7672243118,0.2476938218,1.0521885157,0.7191450000,-1.6686151028,0.0634164885,-0.3962253332,-0.9131394625,-0.3257185817,0.9493992329,-0.1860646755,0.4772316515,0.7308128476,-0.3646406233,-0.1680960655,-0.6723977923,1.8579225540,0.3541227877,1.3605973721,0.8575440645,-1.5376170874,1.1852170229,-0.8388038278,0.6530769467,-0.5707948208,0.1520410031,-0.5276001096,-0.6008660197,-0.7531970143,-0.1710914224,-0.0899885967,0.5186805725,1.0073469877,1.0555337667,-0.9564317465,1.0060201883],[-0.3865070343,-0.2376815975,-0.2118640542,0.9394926429,0.6976744533,-0.2015943229,-0.9899131656,0.5405406356,0.8071326613,-2.0545885563,0.4489324391,-0.7159293890,0.7658372521,0.8533259034,-0.2951918840,0.5808369517,0.2168655097,-0.5482198596,0.3273206055,-1.5042330027,-0.4117659926,0.0382566266,1.1967242956,-0.8289934397,-0.5600231290,1.0414994955,-0.1932150424,0.0411262065,-0.4130541384,-2.0425164700,0.2735721767,-1.5550554991,-0.7226807475,0.7977443933,0.1485011876,0.9446786046,-0.4927309155,0.6173900962,-0.3243460059,1.2881433964,2.2453174591,-1.1089458466,0.6956653595,-0.9906356335,-0.0505813882,0.3978154063,-0.9886980057,-0.2622124255,0.0275835395,-0.3144522607,0.9326926470,-0.3414547145,0.4837387204,-0.2374359816,0.3685643673,-0.8963376880,0.6249161959,-0.2738130987,-1.0066910982,0.5856098533,-0.2894586325,-0.0821105018,-0.3100988269,-0.4248235822,0.2665741146,-1.7740272284,-0.1976617873,2.3445346355,-0.7961371541,0.3188365102,0.3922545314,-2.0804495811,-0.6469821334,0.7504340410,1.1765389442,0.9240966439,-0.3588912785,1.2186727524,1.1245067120,-1.7694383860,-1.4732788801,-0.2784310579,1.3830229044,0.1213222444,-1.1693680286,-0.7890093923,0.2241205424,0.0043442440,1.6878660917,-0.1504489183,-0.2639863193,-0.0475605465,0.4061010182,0.6480402946,0.9678143263,0.8842444420,-0.2457452416,0.9848177433,0.4075931609,-0.6050483584,-2.1982643604,-0.9470968246,0.6483018994,-0.9678875804,-1.0634075403,0.1625384986,1.2762326002,-0.8872547746,0.1652131081,0.5170665979,-0.0231691115,0.6141547561,0.3632443547,1.0437520742,-0.4099225700,-1.5331041813,-0.1598736197,-0.8773027062,1.1364567280,-1.7034946680,0.9508250356,0.0975447968,-0.6070606709,1.7351464033,0.4743512869,1.2093871832,1.2255852222,-1.2209358215,-2.1459023952,-0.5637799501,-1.3911374807,0.4593183398,0.5979510546,0.4527249634,-0.0445172526,-1.0204105377,0.6146193743,0.1513276845,-0.5046305060,1.2186865807,1.6874295473,1.6179510355,1.0376192331,-0.0943819210,2.2098233700,-0.5462614894,-0.8779112697,1.2222505808,1.2055162191,-1.5389888287,1.5902979374,-0.4417829216,1.7803531885,-0.5176959634,1.8613972664,0.7545554042,0.6129772067,0.4468424320,0.9953684211,-1.3956910372,2.0089464188,-0.4462093711,1.5784149170,0.3510382175,-0.0703873858,0.2534176111,-0.1998426467,-0.8111057281,0.3384461999,0.6583834887,0.8084661365,-0.9903705120,0.4196079671,0.3588507473,-0.5640988350,-0.1225297973,0.2798833847,2.6040446758,-1.2985949516,1.3740561008,-0.1327347904,-0.3478104770,0.6357547045,-0.8267017603,-0.0101069696,1.1574569941,-0.5891892910,-1.2464402914,-0.3473787606,0.3087954521,0.2835386992,-0.4206701517,0.2591592669,-1.2195488214,-0.1933093965,-0.5647271276,-0.8968281746,0.1931795776,1.3121608496,0.0971596316,0.9895777106,-0.4723748267,-0.5463014245,0.0892185643,-0.0195480473,1.8247439861,1.5372071266,0.0110428948,1.2742878199,-0.2674755454,-0.3730522692,-1.5067985058,0.4618104398,-0.1699099392,0.7135494947,0.6154643297,0.4335562885,-0.2648655772,-0.0513244420,0.5973302722,2.1014957428,0.2783280611,-0.5136555433,-0.7486355305],[-1.0634144545,-0.5323330760,-1.5201735497,0.5931404233,0.4664847553,-1.2523989677,0.6027503610,0.0915953219,-1.3840013742,-0.8534832597,-0.3615029752,-2.4294033051,0.1112238094,0.0172027890,-0.2286780924,-0.8285278082,2.2321007252,1.9687647820,0.2778130472,0.5375308394,-0.8006395698,0.2198387384,-0.3095584214,0.8869483471,-2.8386275768,-0.9168944955,1.2239718437,-1.0722330809,-0.5158202052,0.6645996571,0.8560354114,0.1393965185,0.9551382661,0.4548152685,-1.9653131962,0.8772224784,-0.2593773305,-1.1952117682,-0.3621657491,0.0389585756,1.1604454517,-1.0822688341,0.5837280154,0.5754579306,-0.5814859867,-0.2461313754,0.0065278327,0.4200200737,-1.5244244337,-0.2629826665,1.3117327690,2.0171401501,-0.8675763011,0.2327602357,0.1637782604,0.4368565083,0.9518216848,-0.7166954875,-0.9328627586,0.8369590044,0.3547633886,0.1785100549,-0.0237964932,0.8859897852,-0.6042553782,-0.3762065172,-1.6474996805,-0.3082104325,1.0396336317,-1.5162066221,-0.9067734480,-1.0850670338,1.2581267357,-0.2214823216,-0.8220285177,0.4123277068,0.7663524747,-0.8339322209,0.6646717787,-1.0077667236,-0.5644736290,-0.9657685161,-0.0941603929,0.4566901326,0.3407334983,-0.7013095617,1.0366376638,-0.1873145849,-0.3645239472,-0.5983661413,-0.0358053558,0.9808912277,-0.3805266619,2.4766175747,0.9397842288,0.6496300101,0.0903372765,0.1319412291,-0.3573524952,0.7131528258,0.7749145627,1.8421769142,-1.1503663063,1.0779078007,2.4074707031,-1.2194875479,2.3397269249,-1.4365357161,0.4038117230,0.2114246339,-0.7000735402,1.7133610249,-1.6811267138,-1.3434336185,0.3275044262,-0.9965205193,0.6525971889,0.1109063923,0.5218975544,-0.2513494194,-0.1506602168,-1.0675710440,0.7060869336,-0.2939028740,0.7074757814,-0.1937825084,1.8526563644,-0.6828480363,0.1911466569,0.1817938238,-0.8393192887,-1.2712761164,-0.2967023253,-1.5580698252,1.0618728399,-0.7327790260,-0.0453399830,0.7231896520,0.2260579467,0.0045189788,-0.0292946324,-1.5553299189,0.3317606151,0.7949090004,0.6328572035,0.4006032944,2.1425490379,-0.6258561015,-1.3994067907,-0.5250332952,-0.5599845052,1.0373660326,0.8478179574,0.0953366831,-0.6932147741,-1.3818480968,1.0562552214,-0.4130301178,-0.1007573754,-0.6099736094,0.2333106846,0.9275327921,-0.7962839007,-0.6180340648,-1.5996370316,1.8704158068,0.8759252429,1.3884956837,2.4303166866,0.4895366132,1.3197805882,-0.1183772013,-0.1369920820,1.3485703468,0.6114919782,-0.0323543325,-0.6448656321,1.2038643360,1.1820948124,-0.2048023045,0.6790464520,0.4966803193,1.5222219229,-2.9318621159,0.4659831226,-0.9096521735,-0.1747777015,-0.0647855178,-0.8331367373,-1.5714011192,-0.2310294956,0.4662609696,1.3658882380,-0.5742117167,0.4292911887,0.5996888876,-2.3400518894,-1.5466567278,0.7606943250,-0.7578786016,-0.2359795421,-1.0300453901,0.1930342764,0.0036071180,-0.0514582843,-0.4616616070,1.7474654913,-1.2599040270,0.9778512120,0.3718870580,2.0573425293,0.0001507712,-1.3301340342,1.2774391174,1.6158695221,0.9101704359,-0.3199186623,-0.4357638061,-1.6184130907,0.4685378373,0.7183168530,0.1964053959,-0.0726887211,-1.1159377098],[0.5207815170,-0.9250515699,-0.8146038651,1.5330342054,-0.6654334664,1.2355058193,1.2149068117,-0.8081433177,-0.1008972526,-2.6701030731,0.1122501418,-0.1568444520,0.6637252569,0.8165430427,0.2798295319,-0.7888636589,-1.4502432346,0.2165211886,-1.3123550415,-0.0796058998,0.9572609067,-0.3779307306,0.8503680825,0.4904717207,1.5245455503,0.1286518425,-0.3477151990,0.2938793302,-0.6671968102,-0.0250478294,-1.3159838915,-1.5637085438,0.7293962836,-0.0600265227,1.3219997883,-0.7478125095,1.6180843115,0.7206456661,1.0755368471,-0.6887121797,1.2116334438,0.1095357239,1.8574286699,0.5284230113,-1.0005639791,-1.7658504248,0.8115992546,0.1519138515,-0.8634884953,0.0946448296,-0.6255034208,-0.5461331010,1.1926347017,1.6944109201,0.7403153777,-0.4044455886,0.7798045874,0.7493108511,0.6431380510,0.1134475395,0.2037412971,0.2755376995,1.1141532660,0.9121242166,1.5169768333,-0.9620696306,-1.9707722664,0.8506955504,0.3520783484,1.6420823336,1.1664842367,0.0395719223,0.5504347086,-0.5065367222,-0.7530210018,-0.1424470097,0.3533922434,-0.5853402615,0.2046187520,-1.9142156839,0.9022826552,-2.0429213047,-1.1913815737,0.2445087582,0.6097036600,0.3708125651,-0.1089679971,-0.4501316845,1.0040342808,-0.7449307442,0.3174760342,0.1585830599,1.1094741821,0.2051526755,0.2611257434,-0.9332591891,0.5158112049,-0.0843781233,-0.5690991879,0.9467744827,1.3892228603,-0.1915261894,-1.4694588184,-0.6829240918,1.8040553331,1.4982194901,-0.6782018542,-0.2488974929,2.0247976780,0.7601909637,0.2023352534,-0.0089879557,-0.3011207283,1.2983838320,-0.6964088082,1.2567225695,1.4973249435,-0.0151675921,1.1256455183,0.5787013173,-0.7115837336,0.4392588437,0.6161403656,-0.3426590860,0.4325609505,-0.0037368375,-0.2693090141,-1.4960060120,1.1586462259,0.7610878944,-0.5264061093,-0.7910407782,-0.3750860989,-1.5029484034,0.7323586345,-0.9829524159,-0.6052945852,1.6173005104,-0.3960254788,-0.3167100251,1.4021726847,0.1614335328,-1.7691595554,0.4891624153,1.7525867224,-0.6651031375,-0.2736934125,-1.5221942663,-1.3538130522,0.9329830408,1.7201684713,-2.1362659931,1.3983808756,1.0712672472,0.8699352741,1.9718719721,2.8391821384,1.1305669546,0.1621609330,-1.0223643780,0.1640578508,0.3482623994,0.0992343724,0.9226043224,-0.5990541577,-0.5642827153,-0.3903325796,1.5199590921,1.2961598635,0.4010730088,-1.3141818047,0.6916372776,0.0341012813,0.2534821630,-1.5459098816,0.7012271285,-0.9198668599,-0.3922315240,0.5712463260,1.0828225613,-1.4264631271,1.8115274906,-1.2499611378,0.8060184717,-0.5210914016,0.2651344240,-1.2977297306,-0.3151990473,-1.4129822254,-0.3059104383,-1.2792009115,-1.3503084183,0.3274895549,-0.8965561390,0.1319619566,0.2559824288,-0.8015326262,-0.8983778358,-0.8235607147,-0.4976842105,0.9133901596,-0.9979527593,-0.0663572326,-0.1228386909,-0.3515924215,-1.6459765434,-1.1646555662,-0.5235185623,-0.3147287667,0.8435072899,0.6037076712,0.1352328509,-0.9873789549,-1.3439757824,-0.5656504035,-0.7461355925,0.1942629516,0.4915701151,0.9518358111,0.3652679920,-0.8218551874,0.0957542583,-0.4124514759,-0.3918253779],[1.0724167824,0.1508688480,-0.1930464804,-0.3708569407,-0.2351885140,0.2533005774,0.3928980827,0.1800833941,-0.1871147901,0.1421864182,0.8545504808,-0.2546977401,1.2093913555,-0.0946916267,0.4536883533,1.8203556538,-1.4522353411,1.0122228861,0.3969878256,0.8579048514,-2.4725697041,-1.1170048714,-0.3419044316,0.2210522890,-0.0589853376,2.0522193909,1.3139799833,0.0032415872,0.9737606645,-0.3109631836,0.6522809863,0.1454812586,0.3584970832,-0.6686333418,1.0057399273,-0.4018355012,0.4271542430,0.7752248645,0.7583116293,-0.0558623113,1.9829217196,-0.0791059658,-0.1452606767,0.6852729917,1.0473220348,-0.4100103974,0.8070753813,-0.5444563627,0.5194547772,-0.7474954724,0.0800675452,-0.4973312616,0.7176632881,0.3519951701,1.1283892393,1.4414153099,0.7251556516,-0.2250899971,0.7314395308,-1.5019141436,-0.7311096787,-1.1236633062,1.5846997499,0.3824317753,-0.8789307475,1.9165765047,-1.2047196627,1.9510960579,0.1713950783,0.2518723309,-1.0073115826,0.0976815745,-3.2116572857,1.0625407696,0.3055147827,-0.1818564683,2.4210405350,0.2135395557,1.0038594007,0.2944985330,-0.1435895413,-1.9485757351,-0.3004073203,0.4409326613,0.6442545056,0.5157012939,-0.5109453201,-0.3380121291,0.4624716043,2.5077555180,-1.0816066265,-1.3965083361,0.6884524226,0.2567251027,1.2027677298,0.2761328220,-1.8016265631,0.4354678392,-0.4259582162,-1.4475284815,-0.4208916426,0.9427179694,0.4259283841,0.0079847854,1.1683820486,0.6213946342,1.0260797739,-1.8386904001,0.5446011424,-2.1929266453,0.3169703484,0.1466955394,0.2231619656,-0.5830193758,-0.9312384129,-0.4481034577,-0.4660280049,0.0486355685,-0.0424672253,1.1460434198,1.0676603317,0.2260905653,0.7005288601,1.4022908211,0.0001941096,-0.3831669390,-0.0770072937,-2.1831009388,-1.5603170395,0.4208289385,-0.9125955701,-0.5532537699,1.8200123310,1.6134670973,2.5783603191,-2.3645694256,-1.2364205122,-2.1010518074,-0.3773110211,0.3891334534,-1.8309109211,0.1862813532,-0.5885137916,0.5469305515,-1.2611180544,0.2711682320,-0.2295575142,-0.8624431491,-0.2658065259,-0.5441913009,-0.6111053228,0.3815867305,2.4132184982,0.2860789001,-0.9591971636,-0.2013577223,-0.2921943665,-0.5352737308,0.2526368499,1.0676268339,-0.0170707032,-1.2185895443,-1.6883766651,-0.0279599130,-0.4728796780,0.4600687027,-2.1366546154,-0.6944203377,0.4398227036,0.8040372729,0.6955130100,-2.2377107143,-0.1955517530,-0.8615193367,0.0634703040,-0.3660758734,0.1347871870,0.2589198947,1.4126286507,0.6336156130,-0.7764823437,1.5431591272,-0.7278642654,0.6717827916,-0.5883294940,-0.9275021553,-0.5374195576,1.4585071802,0.4842028618,0.7848344445,0.1956220865,-1.4329868555,1.4823243618,0.9601696134,0.3370892107,2.0254490376,-0.1215440184,0.3890198767,0.9085683227,2.1455557346,2.4146547318,-0.2953370810,-0.1564641893,-1.0599550009,0.9626013637,0.0099062221,2.0847289562,-0.6303656697,1.3723542690,-0.7465828061,0.3475526869,-1.6720502377,0.4044115543,-0.0562933162,-1.5195279121,0.7755575776,-0.8021618128,-0.2751158774,-0.7458245754,0.1956940740,-1.0238747597,-0.9840406179,1.0258538723,2.0173861980],[0.1315372139,-0.5138539672,-0.5669877529,0.5406587720,-0.3071814775,-1.9967687130,-0.4333548844,-1.9966330528,-0.4374782145,1.7048588991,0.3344865143,0.3502452672,-1.5361242294,-0.2268669605,-0.4311481118,1.0138232708,-0.5507349372,0.5594305992,1.0659108162,1.4947098494,-0.7976589203,1.1181552410,1.3706774712,1.2994995117,-0.1372869313,-0.0259376168,-0.3461629748,0.4131499827,1.1723839045,-0.2778978348,-0.5928153992,0.0675194412,-0.8313975334,-1.1776564121,-0.4460479021,-0.6112349629,0.4268786907,0.8870689869,-0.6189897060,-0.1257660389,-0.9239557385,-1.8860009909,1.1535906792,-0.8335864544,-0.6902356744,-0.1756162494,-0.7561747432,0.5024242997,0.5363253355,0.0824064240,0.2639508545,-0.6364569664,-0.3769787550,-0.4377987981,-0.5338432789,0.5736504793,0.2441345602,-0.9038761258,-0.6077290177,-1.3279690742,-0.4976146519,-0.4994116426,1.3304718733,1.0966929197,-1.5022242069,-0.1590254903,1.6550925970,0.7573397160,1.3145182133,1.0289083719,-1.0945633650,-0.8774670362,0.1911343187,-1.0744937658,-1.0281189680,-0.4112767577,1.8617081642,0.2614100873,-0.1120274514,-1.1018739939,-0.5599132776,-0.9639128447,1.4452749491,-0.2379340380,0.0965326279,1.2240991592,-1.1137310266,-1.5898432732,-1.2747541666,-1.2285566330,-0.2418234199,-0.5687929392,0.6784836650,2.7191262245,1.0439400673,-0.5211651921,-0.1766644418,0.2213129550,-1.2737783194,-1.0919522047,3.2457404137,-0.5371184349,1.0131412745,1.3599733114,-0.1014137343,0.9044458866,-1.5049918890,0.3647313118,-0.7347620726,-0.7936519980,-0.4339566827,-2.1577720642,0.3033457994,0.6737972498,0.2448626608,0.6937257648,2.6209950447,-1.6615227461,-0.8978943825,0.3889890909,-0.4596033096,-0.7707110047,-0.1208278388,-0.5201724768,1.3842766285,-0.4282974005,0.7007796168,-0.4204046428,0.1363129318,0.4809338152,-1.1034513712,-0.4319312572,0.4005286396,-0.6485356688,0.2712681592,-0.7505175471,-1.4745311737,-1.1564862728,0.5677739978,0.6589148045,-0.8751621246,0.1891540736,-0.5418350697,1.8105150461,1.7355394363,0.2601662576,1.4141949415,-0.4384791851,-0.7842199802,0.4371353090,0.6943714023,-0.5380707979,0.4983506203,-0.5410212874,-1.1434106827,0.9545881748,1.3708180189,0.5112654567,-0.2825934291,0.8243518472,-0.3324177563,-0.9993035197,-1.6045049429,-1.7449423075,-0.8640743494,1.4113116264,0.3710772991,-0.3376179934,0.5454092026,-0.9610939026,-0.6323919296,-0.3632386029,-0.7752295732,1.8468760252,-0.0391475186,0.0493343733,-0.5033078790,-0.4727648795,-0.5759646893,-0.1179095358,0.1768748015,-0.0848587006,0.9794515967,-1.0092216730,1.5041515827,-0.2412413508,-0.3741708994,-1.3928172588,-0.1496126205,0.8959793448,-2.1168699265,-0.1839516461,-1.0776592493,-0.8422918320,-0.2578078508,0.7120755315,0.5457919836,1.2602188587,-1.3638870716,0.3252347112,0.3103461266,-0.6843565702,-0.8641714454,-1.3515841961,-1.3709787130,0.8442222476,0.6913160682,1.1721013784,-3.0055868626,0.1678446978,0.1083558276,-0.1259460449,-1.4384572506,-0.4379824996,0.6088418961,0.3323244452,0.1043943688,0.0842534676,-0.7801011801,-1.0713446140,-0.5946297646,-0.0363405943,0.6251944304,0.8738729358],[0.8417578340,-0.1302471012,-1.7174865007,0.8975894451,0.7157564163,0.7714152336,-1.2969895601,0.2700895965,0.6959099174,-0.7589926720,-0.1790614128,-1.4486862421,-0.0582850389,3.2882657051,-1.0190426111,-0.2596707344,1.2057558298,-0.9189699888,2.0075521469,-0.6953464746,0.5769160986,1.0798561573,0.3362890780,-0.9042438269,-0.6373935342,-1.2408188581,0.5856894851,-0.6166279912,-1.0748144388,-1.1937690973,0.2725262344,1.7097991705,1.2231884003,-0.1864460707,-0.3886844814,-1.1401408911,1.1001895666,0.0698222071,-0.8169481754,-0.2381762117,-1.0990309715,0.0363042206,-1.0470657349,0.5295069218,-0.8829963803,0.6988388300,0.0920695812,0.2125919312,0.2035269290,0.4504480660,0.5238382220,-0.4745627046,-2.6972100735,-1.2137219906,0.8994110823,-0.5333787799,-0.2330003232,1.0217889547,0.1832010299,-0.1263355166,0.6309781075,-0.9253399968,0.1927347630,-1.1983563900,2.1006891727,-0.6179254055,-0.7291156650,1.5111485720,-0.9217118025,-1.3076606989,1.3268530369,-0.0251848493,0.7743625641,0.3491370082,-0.2782400250,1.5292787552,-0.9830396771,0.0843514204,-0.6756237745,0.4932007790,1.2645338774,0.4777994752,0.4024190307,-0.4145161808,-1.9489889145,-1.1476491690,0.6487477422,-0.5789185166,0.5087463260,0.6426893473,1.1313562393,-0.1236293092,-1.0429756641,-0.1709813625,-0.0481598303,0.7611688972,0.6133790016,0.3980072141,-0.2242967188,-1.0123447180,-0.6344790459,0.9931277633,-0.5333514810,1.0943918228,0.0513154417,-0.0190032981,0.6046923399,0.6036578417,-0.7517727017,-0.5252866149,-1.1066845655,0.9470067024,0.0642272905,0.5968157053,-0.2527898848,-1.1888705492,0.5340593457,-0.0894497484,0.4148004949,0.1358723789,-0.1815864891,1.2741400003,-0.4312350452,-0.5020402670,0.1918521374,-0.3785336614,0.3328542411,-0.9043675065,-0.1148370057,0.2523318231,0.5270742178,-1.7881046534,0.9077923894,-0.6153175235,-0.8822738528,-0.8405439258,-0.0023670495,-0.4354154170,0.0477021970,0.8350443244,1.4262235165,0.6257942915,-1.5644793510,-1.2305992842,-1.1948616505,-0.3577411771,0.9498245716,1.3089897633,-0.1266740113,-1.2482118607,-0.7585697770,0.3164147139,-1.6025328636,1.4389153719,0.8901625276,-2.9274833202,-1.3023848534,-0.1057398170,0.1712914407,-0.9100197554,0.9962213039,-0.3809780180,-3.3738229275,0.8629325628,-1.3146237135,0.6146615744,-0.7009440660,0.7499541044,2.5462694168,0.2149894536,-1.3944144249,-0.2389547080,-1.4857633114,1.2554066181,-0.7945663929,0.1306199729,-0.4769009650,-0.2954279780,-0.4177624583,0.7645667791,0.0706775561,-0.0036673227,-0.1686885506,-1.2878538370,0.9614015222,0.9980438352,1.6333302259,0.3764960468,-1.7786319256,-0.1212472469,-0.5497505665,-1.6330521107,-1.0370134115,-1.8117271662,0.1047022566,-1.6284438372,0.1362413913,-0.6433253288,0.5122199059,0.8732233047,-1.8397277594,0.3793027401,-2.2091073990,0.1155540869,-0.2137276232,0.8290930986,0.4650845826,-0.8942251801,0.4178222120,-0.6271676421,0.0223560892,-0.9844577909,0.5359445810,-1.5864883661,-0.5337475538,-0.0250616819,-1.2855857611,0.8331471086,-1.4578803778,-1.2561945915,0.7204806209,0.5493844748,0.9813920856,0.6658095717],[0.2176280916,0.2478718907,-0.6187527180,1.7733623981,-1.0680766106,0.3115855157,0.9599409699,-0.6318791509,-1.7253450155,-0.1581967920,-0.2847787440,0.7510251999,0.6722778082,-0.5759375691,0.3976946175,-0.2699123919,1.1467082500,1.2894232273,-0.7950692773,-1.2313041687,2.5918538570,1.4108102322,-2.5290796757,-2.7064659595,2.0493443012,-0.1575528234,2.5390417576,-1.0978960991,0.6092931628,-1.7137644291,1.3925527334,-0.0829210207,0.9157499075,0.2611907423,-0.5531715155,-0.2612552643,0.1815842986,-0.2074785680,-1.3900825977,0.0094462279,-0.5616672635,0.6902035475,0.3310068846,-0.4598512352,-0.3554174304,2.5179774761,1.7027835846,0.7137155533,-2.5243444443,-0.0778169781,-0.7383882999,-0.4915468097,-0.4458041489,-0.1211165935,1.6788327694,-0.5277331471,-0.6074772477,-0.1313184202,-2.0344195366,0.6652619839,1.1473989487,-1.1869680882,-1.2858363390,0.6587207317,1.9654532671,0.7096003890,-0.7670360804,0.3632927835,0.1158852875,0.6583322883,-0.1609952003,0.1934109926,0.3165058196,-0.5613026023,0.1166221797,-1.3770916462,1.3263069391,4.2417716980,0.0803468376,-0.2049070597,1.5271331072,1.3036043644,0.5373516679,-1.9896894693,0.5142652392,0.6343153715,0.0603445619,0.5120731592,0.4221506715,-0.4696768820,0.0600340888,-0.4477773011,-0.0307193249,1.6819882393,0.2639681995,0.2140350789,1.3125169277,-0.3825906217,0.0022184977,0.1268048286,1.5481674671,-0.5468621850,-2.6106348038,-0.1282699555,-0.4177289009,-0.2894179821,0.9866001606,-1.2609921694,-0.9886783957,0.7946244478,0.8661491871,-0.4253971875,0.5601807833,-0.7023891211,-0.5714130402,-1.4841006994,-1.3401839733,0.0336159281,-0.0244027600,-0.1347808540,1.3022232056,-0.4897793531,0.6417346597,-0.8937379718,0.1637545079,0.2157087028,0.5360102654,-0.1275391430,1.7299357653,0.8968551755,-0.8804937601,-0.3912613988,0.8144260049,-0.4776334465,0.0370308645,-0.5445691347,-1.5247189999,-1.0484591722,1.0209940672,-0.0507922173,1.0969961882,0.0081091505,-1.0735074282,-0.4637985229,0.5490329266,0.7707652450,-0.9320597649,-0.5035763979,-0.3571883440,0.2904200852,-0.1153651178,-2.4413213730,0.5403788090,-0.8432013988,-0.7396709323,0.1283033639,-1.4762566090,-2.9589779377,1.3033297062,-0.2565463483,-0.3692474365,-1.1774467230,0.5953117013,-1.0849800110,-0.5788462162,-0.9446442127,-0.9873202443,0.8887941837,0.5191269517,-0.0704374313,-0.5656064749,-0.2987744510,0.4930670857,-0.4615590572,-0.2847803533,0.4016345441,-1.1605231762,-0.1913904548,-0.1202806160,0.2625747919,1.4581007957,-0.5056554079,1.7628393173,-1.4981478453,-1.3307580948,0.0612610243,1.5295269489,-0.3462225795,0.8649011254,0.4291520715,0.4492061138,1.0830732584,0.1221628636,-1.0343058109,0.3313332498,-1.0550051928,-0.3262061775,-0.2889346778,1.5725835562,-1.2517917156,0.0634046644,1.1232205629,-0.2517988682,-1.6302595139,-0.2371418178,1.7263067961,-0.7344884276,1.6452118158,0.2126683742,-1.0070602894,-1.1667169333,0.1198763698,-0.1139037609,-3.2478482723,0.1158405244,1.2873246670,-0.2967154384,-1.9113676548,-0.3062439561,0.2123287916,0.7192662954,0.2661412656,0.7092028856,1.0256335735],[0.4500994980,1.9605486393,0.0655291229,-0.6529865861,-0.1378134042,-1.7298293114,-0.1140466258,-0.0287944637,-0.3820255995,-1.4283573627,-0.2104176134,0.8270501494,1.6666053534,-0.2273907214,0.4807750881,-1.1465576887,-0.7090556622,-1.8215582371,-0.1210410222,-0.3808588386,0.0259706769,0.1462689787,0.2766871750,-1.3793306351,0.2024340034,0.4679293633,0.7783885598,0.2412308306,1.4358551502,-0.5162473917,-0.3459844589,0.3041249514,-1.2383747101,-0.7322098017,0.1910422891,1.8059240580,0.9277132154,-2.0171482563,0.1951866299,0.6639137268,1.2373590469,1.2164707184,0.1519757211,0.6530285478,0.9477738738,0.2688260674,-1.2582738400,-0.9600851536,1.0390130281,-1.0648518801,-0.8012365699,-0.2251798809,1.1142963171,0.9341740012,-0.4841320217,-0.0355923399,0.3871592879,0.5604581833,-0.4087530673,2.0429780483,-0.4825588167,-0.6649580002,0.8005616665,-0.3599886000,0.1193309203,2.4247162342,1.1964256763,-0.0806443915,1.5196145773,-0.4052409232,-1.2675876617,-0.6781725287,1.5818355083,-0.9401290417,-0.5591271520,-0.6824262142,0.4200727046,-0.0844191387,-0.2632116973,1.2377724648,-1.8268915415,-2.0282392502,-0.7599870563,0.4133173823,-0.3004525900,0.3071993589,-0.0633047372,0.4281289279,-1.2683299780,-0.4179551601,-0.3186133206,1.1701862812,0.9885575175,-0.1080873460,0.2283486128,0.3161881268,0.6529545784,0.7720674276,-0.1712777019,-0.0354263894,-0.2102745920,1.9657728672,1.0205694437,1.4390608072,-0.5366148949,-0.7892500162,1.9705640078,0.3855912387,0.0629414320,0.6265225410,1.1438483000,-0.9780292511,-0.6935688853,-0.9762239456,-1.4685491323,-1.4372947216,-1.0730566978,1.4005103111,0.5869930983,0.0406822041,-0.3796490729,-1.4310114384,-1.5541208982,-1.1360594034,0.1388325095,-2.1530423164,0.1147642508,0.0471051075,0.3919199407,0.7114022374,1.1730850935,-1.3012977839,0.3481751382,0.2530570626,1.7267895937,1.5392260551,-0.9095569253,-0.5668856502,-0.4846865833,-1.9252877235,0.0159935709,-0.2737687826,0.8173170686,-1.4598239660,0.4550220966,-0.4915318787,0.3947442770,-1.4720059633,1.1339253187,1.3018774986,0.6073305607,-0.0294520799,-0.0245533902,0.8299852610,-0.2093410045,-0.8937062025,-0.2104247361,0.4905937016,0.9270766973,-0.8489860892,3.0901689529,-0.9478366971,-0.7347039580,-0.0914564133,-0.1481568217,2.1250960827,-0.6915543675,-0.7773598433,0.7704299688,-0.8429703712,1.2658998966,0.1026027575,-0.9573163986,-1.0858606100,0.2484246790,1.5868668556,-0.5566447377,-1.2599195242,-1.3841241598,1.1235691309,1.7500667572,0.0562867485,-0.2367135286,-0.5502758026,0.8646404743,0.2884162962,0.2858077288,0.3000690043,-1.2110253572,1.4334803820,1.3211159706,0.1077606380,-0.5899074078,1.2182687521,0.9392931461,1.7326339483,0.6656506658,1.1504017115,2.5321750641,-1.3975093365,0.1026985720,0.0797936320,-1.2145488262,-1.6663728952,0.1640934795,-0.7657446265,-0.9649801254,-0.2156695276,-0.3802717328,-0.0361323729,2.2739071846,0.5935225487,-0.7992947698,0.5043234825,-0.0635420382,0.0635460094,-0.4986944795,-1.2183550596,0.8497512937,1.3682345152,0.0592218079,1.8176462650,0.0380014367,-0.1648121178],[-1.1811887026,1.6452852488,-1.1771593094,0.9260467887,-0.2866005898,-0.3300427198,1.0535910130,-0.4557752013,0.9618560672,0.3766882122,-1.8635056019,1.7812006474,-0.6835395694,0.2296070904,0.0775303841,-1.5565257072,-0.9657327533,-0.1612064391,-0.9848763943,-0.7570252419,-0.5434094667,0.0870989487,-0.3932954669,-1.4176454544,-0.6030968428,-0.5313537717,-0.7281560302,0.2537410855,0.8596052527,0.5377355814,0.0889668092,0.6040338874,-0.8646016121,0.4936359227,-1.3471070528,0.1851646155,0.5584734082,2.0683081150,-0.6238692403,2.4669466019,-1.0006500483,0.4998898208,0.5908782482,0.1898878664,0.9036785364,-2.6795573235,-0.6285718679,-2.2651124001,-0.0100735221,1.6246125698,1.2570762634,0.9649873972,0.4480108619,0.3538306952,-0.6394305229,0.6310080886,0.2673046887,1.3926157951,-0.5519245267,-0.6487637758,0.6073168516,-0.4114406705,0.3761738241,-1.4606273174,-1.4759228230,1.7613903284,1.2103283405,0.6520544291,-0.0654612854,-0.8724196553,1.2886662483,-0.9957754016,-0.2996768653,-0.5182425380,1.1075606346,-1.5807965994,0.1618001610,0.0243788250,-1.6595828533,-1.7371256351,0.1204244867,-0.2156693637,-1.6937848330,-1.0714200735,-0.2341331691,-0.3915313184,0.2276837975,0.2442232966,0.6898937821,0.8300570250,1.6709381342,-1.4706568718,0.6646685600,-0.7181670070,-0.0028348088,1.4022594690,2.1807963848,0.6828091741,-2.4582383633,1.1281207800,-0.6115419865,-1.1212936640,-1.2312363386,-0.4915806353,0.5487788320,-0.4697555602,-0.2441457063,0.4054579735,-2.1404464245,1.5693150759,-0.5173771977,-0.4552463293,1.1641325951,-0.4407202303,2.2006056309,0.2322905511,0.1107065305,0.2862088978,-0.2815102637,-0.3408988416,0.2058108151,-0.1996606290,1.8439724445,0.8755335808,-0.2496846020,1.5357745886,0.2453305572,0.6203836203,0.8212997913,1.3025162220,0.1074199975,0.8545994759,2.0262827873,1.6857354641,-0.5158991218,-0.9814931750,-2.5545742512,0.5971880555,-1.2477747202,-0.8447640538,-0.2711225450,-0.1848308742,-0.6540018916,0.2878918946,0.3621478677,0.6957350373,-1.6501054764,-1.1784461737,-0.8048467636,1.2036799192,-0.7421025038,1.2379450798,1.7582762241,-1.5390353203,0.4638553858,-1.3758928776,0.1921562254,-0.2725992501,2.1471478939,-1.2595988512,-0.3549736738,0.2839905918,-0.1939975917,-0.0150380591,0.9684531689,-0.8801771998,-0.6602508426,1.1853135824,-2.7470576763,0.5793197155,-1.2005586624,0.9979766607,0.1249912754,-0.7145936489,0.0190785658,-0.0678644404,-1.1773444414,-1.3656818867,-0.9308581352,-1.5273888111,0.6262164116,-0.9275572896,0.6307232380,-0.3932364285,0.3676517010,0.1095421910,0.4901863337,-1.3060173988,1.4643207788,1.2106331587,0.3033938110,-0.7580705881,-0.7045764923,-0.2487854362,0.7929059267,0.2234160602,-1.1094235182,0.1523469090,0.6313382387,-0.3104008734,1.2057654858,1.2911474705,0.1881976724,-0.0363846235,-0.6520819664,0.7624713182,0.9030723572,1.4012763500,0.9809569120,-0.1520080268,-1.1902999878,2.4968516827,0.5791797638,-0.5331690311,1.8849302530,1.2531266212,-0.5741626620,0.6255322099,-1.2660032511,0.0748630986,1.6345337629,0.2266604304,0.6084037423,-0.7702188492],[1.9156914949,0.4047849774,-1.2006857395,-0.1776676178,-0.1083059758,1.7594144344,-0.1029189825,-0.5665433407,0.6066942215,-0.5196262002,-0.2490082681,-0.9327793121,0.3236896098,-0.3677179217,-0.1818056256,-0.5839886069,-0.2038229257,-0.0597114004,1.1402399540,-0.0260799173,-0.6193695068,-0.3960160315,-0.9552299380,0.6448193789,1.4817017317,1.5544289351,0.9933441281,0.4653750658,0.3257007897,0.1263968647,0.1049721017,0.7257955074,-0.3700866103,-0.5341292024,2.2867617607,0.2005052119,-1.9354718924,0.8988186121,-0.0689575225,-0.6440743804,0.5772926807,-0.4547063410,1.4799640179,0.0647191256,-0.4898588955,-0.8096262217,-1.0249881744,1.6329885721,-1.0937043428,-0.6664443612,-0.2176919729,1.8012778759,-0.9251882434,-0.5116018653,-0.1121688411,-0.2361086607,1.7645183802,-0.9666455984,-0.8906248212,2.5078215599,-0.3365430832,1.4512392282,0.2457870543,-0.7407913804,-0.5314186215,-0.2175210267,1.6160784960,0.1030726433,-0.2955012023,0.3930938840,0.3691558838,-1.4468399286,1.4991459846,0.7861236930,1.4582768679,1.0606905222,1.4576917887,0.4594173431,0.9900460839,0.0367541350,-0.0919883996,-0.4150067866,1.1121563911,-1.8991169930,-0.0029766932,0.9736584425,0.5819031000,-0.3772553504,-0.3100667000,-0.2848064303,0.2640193999,-0.3516303599,-0.4092839956,-2.0935854912,-2.2584433556,0.3784409761,1.2266635895,-0.6566455960,-0.0365018025,0.3758405745,-1.1574118137,-1.6608725786,0.3360662758,-0.4174785316,1.7872010469,-0.5762202740,-0.9449684024,0.1880954653,-1.2142983675,-1.3987449408,-1.0537490845,0.0820185915,-0.5293899179,-0.5560578704,-0.5497375131,0.2450578064,0.4957640767,0.8905921578,-0.9192040563,0.5279426575,-0.0151319457,-0.3531607687,-1.6664785147,0.5860685110,-0.1222635657,-0.4696131647,-0.8306789398,-1.7213633060,0.4148206413,-0.9287552238,0.5041882396,0.6945232153,-0.7018371820,-1.7570469379,0.3231323659,1.0354024172,0.1899425387,1.2799785137,0.5467972755,1.1144099236,-0.1551059335,-0.8696814179,0.6797798276,-2.4373333454,0.6826940775,0.5466501713,0.8491961360,0.5381039381,-1.3631504774,0.6576964855,-1.6812366247,-0.1420921236,0.0644702837,0.3568619490,-0.4917500317,0.0291998181,0.5068673491,-1.1943705082,1.3364574909,0.0296131652,-1.1348272562,0.3270067573,-0.7610291839,0.1490225941,0.1008909866,-1.4996412992,-0.6174682379,-0.6078441739,0.2064402550,-0.8942359090,-2.6421942711,-1.3883210421,0.1245091781,0.6594743729,-1.2438187599,0.7344162464,-0.3856297135,1.3615497351,-0.6648976207,0.1783328503,-0.0513411611,-0.3603737056,0.3984665871,-0.4077322781,1.3554097414,0.2356388569,-0.2174219340,-0.1714425683,0.4619775414,1.7577792406,-0.9965714812,1.1256047487,-1.0318431854,0.0069990228,0.1054399237,-1.0228053331,-0.3113851845,0.8585723639,0.2263864875,-0.6888257265,1.6379286051,0.7351904511,-1.3622685671,-1.4697754383,0.6048600674,-1.0446178913,-1.6048693657,-0.0697253868,0.7481107712,-0.8034046292,0.2074136585,-1.5064373016,1.5586475134,0.0487470403,0.2746212780,1.3123952150,-0.0130932564,-0.5733627677,0.1082773730,-1.0324797630,1.4006639719,0.1419160068,-1.2351351976,1.4697282314],[-0.4961551726,0.3234536946,2.5705366135,0.6191286445,0.2001783103,1.8302137852,1.4770355225,1.2808552980,-2.0362477303,-0.7708904147,-2.2161989212,-0.6043381095,0.1801552922,-1.8412429094,0.2060479224,-0.2533358932,-0.8370944858,-0.7637611628,0.3168261349,0.5018517971,-0.2618725598,2.0702135563,2.1030116081,-0.0092163961,-0.9284299612,0.5606794357,-0.2881985605,-1.1079137325,-0.0594206415,0.2303173840,0.0974219963,-1.1072945595,0.6925656199,-0.3535038233,-0.6043095589,1.5273171663,1.1504018307,-1.5695350170,-0.6234639883,2.2262411118,0.5078104138,0.5744566321,0.9775137901,-0.2812677026,-0.6070799828,-0.6489948034,-0.2179257274,0.9815560579,-1.8020489216,-1.3939456940,0.2427239269,-1.1940877438,0.7539755106,0.2131537497,-0.9054090977,-0.9812750816,0.2375390530,-0.7263105512,-0.4075903893,0.4848812819,0.7323713899,-0.2195205986,0.7746742368,-0.5940266252,0.5959823132,1.5027459860,0.7207845449,-0.9153077602,0.7144644260,-0.4492384791,-0.1792533398,0.4524716139,-1.4900296926,0.1651769876,-2.2085511684,1.5511286259,0.7659050822,-0.3244543970,-0.4670624137,0.2195701599,0.7060580254,-1.4186925888,-0.3362497389,-0.8520848751,0.8867446780,0.3390224874,1.0442818403,-0.9417509437,-0.7377603650,0.0769798011,-0.7142660618,-0.7156736851,-0.1892230809,-0.5361557007,-1.3213405609,0.2317953408,-0.6825730801,1.0135622025,0.5309364200,0.2421779037,0.4017920494,-0.4618558586,2.3104119301,1.6827931404,-0.8894057274,-0.3898538053,0.8753790259,-0.6747983098,0.3014372587,-0.6322235465,0.0239859428,0.9638617039,0.9551012516,0.7515984774,-0.0033408087,-0.9071614146,0.3515233696,0.5865153670,-0.2021835446,0.4500505626,0.9413334727,-0.5805366039,1.7458322048,0.0380769074,-0.0990866572,-1.0987000465,0.0226462241,0.5436627865,1.1500978470,-0.4444086850,1.5100064278,-1.8368489742,-1.3223434687,1.0303223133,0.6480669379,0.4582530856,0.7627970576,0.1246709898,1.7402290106,-1.6799961329,-0.5588530898,0.7539991736,-0.6544169188,0.0672161505,-0.3272388577,0.5595880747,1.2202001810,-0.3627043962,0.8311170936,0.3439565897,-0.1743943244,0.2378960103,-0.4759786129,0.0559345298,-0.4318015277,0.3085387945,0.7614166737,-0.6752606630,1.1796420813,0.3769035339,-0.0317898244,1.9001022577,0.9514611959,-0.0556233376,1.3591566086,-1.2247810364,0.1007181108,-0.7172840834,-2.5895702839,0.4835788608,-0.9515837431,0.4092848897,-0.5652714968,1.6664245129,0.1597400755,0.3711966276,-0.6338654160,2.0866260529,0.5391490459,0.1859727055,-0.5546110868,-0.5572327971,-0.6505861282,-2.2647030354,0.4426457882,0.1652098149,-0.2662654519,-1.3499914408,-2.2734627724,0.5519658923,-0.0034194007,0.3603643179,-0.2471415550,-0.1274028271,-0.0114346435,-0.9155646563,0.1687090397,1.8955256939,0.2796881795,-0.5321885943,-0.7036546469,-3.1717319489,-0.7830032110,0.9552480578,-1.7417746782,-1.4782881737,0.2072343230,-0.3422027230,-0.0000039180,-0.7132492065,0.4970744252,0.2872682214,-0.0240418799,-0.0924070626,0.2005609572,-0.2416967005,-0.4522222579,-0.7911132574,1.1676659584,1.0429853201,0.4757418931,0.6947637796,-0.4414412677,-0.3858654201],[-0.0315578692,-0.0620186105,0.7743752599,1.6831254959,-1.5434107780,0.2417434901,-0.4039699435,1.0505112410,-0.0232733302,2.4460079670,1.8306061029,-0.2217491418,-0.8391377330,0.4859440923,-1.8701001406,1.6040929556,-0.7600015998,1.1947399378,0.8387134075,0.2391007692,-1.1038827896,1.5782461166,1.1515152454,0.8457059264,0.5228255391,-0.7674822807,-0.2895836532,-0.6218723059,-0.5329485536,1.9727894068,-0.8545120358,-0.0130121699,-0.2113794982,1.2866762877,0.4343684018,-0.2603058219,0.4808883667,-0.7769899964,-0.2809680998,0.6315764785,-0.2057825625,0.6242778301,0.3901982903,0.5251529217,-0.3042872846,1.0285104513,0.1606976837,0.1045264378,2.1321480274,-0.1765216440,-1.4004218578,-0.4953492284,-0.3176076710,1.2840528488,-0.0970956907,-1.2646052837,-2.8805596828,1.0095061064,-1.6003195047,1.3290346861,0.1850828230,-0.4360282421,0.7396296263,-0.9351475835,0.3767962754,1.6771639585,-0.7094474435,0.6432923675,-0.2963850200,-0.9941192269,-0.4628743827,0.8353748322,0.6730687618,-0.1000754759,-1.0190597773,1.6968350410,0.6228394508,-0.8952073455,-0.7015125155,1.4203540087,-0.0927628949,-0.2419510335,1.0769065619,-1.0362827778,0.0588853620,0.4759929478,0.1233649552,-0.8965017200,0.2282161117,-1.5211969614,0.6221343875,1.4138190746,-1.0571297407,0.7189363241,0.6118679047,1.3299641609,0.8587504625,0.0770226419,-1.5501102209,-0.6436623335,-0.1981956661,0.5344657898,0.0764187127,-0.2980852723,0.1402880549,0.4510383606,0.1825833172,-0.6221678257,-1.8053606749,0.3611282408,-0.8057336211,1.4318394661,0.5070827007,0.5489898324,0.9191237092,0.6925198436,-1.1693946123,-0.9724825025,1.0652650595,-1.3838489056,-0.4112027287,-1.3346097469,-0.5550889373,2.1472911835,1.1219021082,0.3011842966,1.1503249407,-1.3700573444,-0.2416723967,-0.8239528537,0.2148872912,-0.1503032893,-1.6401677132,-0.4880145788,0.1143725961,0.0351575762,-0.2061207592,-0.7204528451,-1.8366031647,-0.1839381754,-1.5259851217,-1.1155805588,0.1875498742,-0.9459524155,0.1837504357,0.2436221540,0.1203817278,-0.0044349479,-0.6723968387,1.5836912394,-0.0804802477,1.4106041193,-0.6819021106,-1.0202736855,0.9404947162,-1.2492835522,-2.0210082531,0.0438118316,-1.1920198202,0.2237372845,1.3341157436,-0.4678844810,0.5081447959,1.5526033640,-0.8240080476,-1.7554879189,0.2668016553,0.0097025707,-0.1617574394,-0.3676709831,0.5528894663,-2.0743434429,-0.5731478333,0.1601055115,0.3278245628,1.2261792421,0.5595084429,0.8565822840,-0.0552693568,-1.5954004526,-1.1731684208,0.2836863399,0.0900228396,0.3806756735,1.6710040569,0.4169368446,0.0130567746,-0.3834365010,-0.6937999725,0.7352904677,-0.4927332103,-0.1502003819,0.6302086115,1.2963200808,-0.4325444400,-0.5431685448,0.7643723488,-0.3315337896,1.4310835600,0.6894853115,1.2302139997,-2.7370042801,-2.0758631229,0.8194510341,-0.4422242343,0.4619798958,0.4433602989,-0.4337116778,-0.9711326361,-0.7581747770,0.1246169358,1.2777051926,1.1372745037,-1.4670459032,-0.4377576411,0.5218424201,-1.3545349836,0.4560755789,0.5561665297,-0.2870936692,-0.9668313265,-0.5786038041,-0.4808922112,0.6795433164],[-0.9512211084,-0.0520349778,0.6492468119,-0.4023033082,-1.4939315319,0.1896193027,-1.4484800100,-1.0535236597,0.4588146806,0.8064754605,-0.6888238788,-0.2907008529,-0.2247964591,0.5738497972,-0.3463048637,0.7758319378,-0.8346418142,0.3204473555,0.0675710440,0.6141188741,0.0013366932,-0.7410218120,1.6322492361,-1.2700757980,-1.0345075130,-1.0681058168,1.1929841042,-1.1688381433,-0.5785624981,-1.2230278254,0.5554084182,-0.4128984511,-2.2871227264,-1.1279395819,0.8433239460,0.1518727094,2.0980987549,0.4355578423,0.8607432842,1.7625917196,0.0613590404,-0.1310444772,-1.1538994312,-0.2084873766,0.6174628735,-1.7004872561,-1.0596629381,0.8424559832,0.6678346395,-0.7544751763,0.0714253187,0.6278148293,0.4992212653,1.0457350016,-0.1810946763,2.0991773605,-0.8590780497,0.2861403227,-1.4618811607,1.0282397270,-0.5337978601,-0.1446364969,-0.9851919413,-0.0448441878,-1.3567535877,-0.4009573758,-0.5194885135,1.9878838062,0.6508784294,1.0800508261,0.7166387439,0.2542805076,-0.1705901474,-0.8281711936,-0.2093469799,-0.3811714649,-2.3445951939,-0.3831149042,-2.7244744301,-0.8163167834,1.1056801081,-0.7393755913,-0.0141836274,0.7436538935,-0.7740547657,0.3735834360,-0.2184483260,0.9375358224,-1.2539305687,0.5076622963,1.3608239889,-1.1145510674,0.3206633925,-2.1811640263,0.3985752165,-1.3613519669,1.0203813314,-0.0525459312,1.2631360292,2.2825152874,-0.8123476505,-1.4406666756,2.0423271656,0.1090389937,1.4997259378,1.4937498569,-2.8995282650,-0.6637921333,-1.2237437963,-2.7321140766,-0.8622496724,1.9248254299,-0.0118800914,1.1111546755,0.4902781546,-1.1836172342,-1.7035608292,-0.6081807017,1.3975087404,-0.7155966163,0.2222000808,0.1491065621,0.5913746357,-0.5024318099,-1.7523351908,-0.8049672246,0.0088898782,0.5498868823,1.2858682871,-0.8413038254,-1.2562233210,0.8540031314,2.7723174095,-0.0660058931,-0.0704578236,1.4507570267,-1.7885080576,0.0580867492,0.0526900925,-1.2424575090,-1.1932390928,0.0536607280,-0.5483915210,-1.1195905209,1.2937419415,0.4144541919,0.5000180006,0.2320733964,-1.2249164581,1.0522677898,1.1093354225,1.0206264257,1.8482019901,1.0249714851,-0.6673058867,-0.4649356008,1.1188815832,-2.9661209583,1.1700307131,-0.8108969927,1.0020540953,0.1462125331,0.7842738628,-0.7974629402,0.2370982617,-0.2903953195,-2.3647882938,-0.4633262753,0.3321042359,0.0682333335,-1.5955102444,0.4221783876,-1.0460327864,0.0833548754,-0.0800244212,0.0916623548,0.8428997993,-0.0078278603,-0.0038369715,-1.8239731789,-0.6578385234,0.5915154815,1.1527293921,-1.2100425959,1.4766799212,0.2016633749,-0.1270681322,0.2116021514,-0.7006193995,-0.3207107484,-0.5239910483,-0.2711711824,-0.0808895975,-0.2620672882,-0.5259278417,-0.2087262869,-0.0847980306,-0.6684075594,0.1318341643,-0.3069368303,-0.3217082024,1.2094240189,0.3053298891,0.0364941768,0.4884267449,-0.2790943980,-0.9987446070,0.0164896939,-1.4116181135,-0.0203813892,2.0271334648,-0.4562243521,-0.7777702212,-0.0108297598,0.1521138549,-1.5504790545,-0.4843142927,1.1425955296,0.9327201843,-0.2901050746,0.5929782987,-0.7860559225,0.2315905243,-0.1417619437],[0.0613079369,-0.0808374882,-1.5937153101,1.0773460865,0.2746188343,-0.7450028062,-0.6539436579,0.8430441618,-0.8579425216,0.8792074919,-0.1914065927,0.8935930133,-2.3076658249,-1.2254799604,-0.9316254854,-1.3976552486,-0.3991250396,-0.6948458552,-1.5662037134,-0.0899952948,-1.2958993912,-0.2588024437,0.0242907070,-0.1605933458,0.1624411196,0.6484341621,0.0786128640,0.1670193970,-0.7773826718,-0.7471885085,0.2092164755,0.2134766579,0.5705438852,-0.4578116536,0.5587601662,-1.0892243385,1.0605726242,-0.3094975352,-1.4312770367,-0.1730458140,0.4337505400,1.0741657019,1.6160172224,0.1272350997,-0.1938657910,0.6725165844,0.5144274831,-0.5641947389,-1.0385379791,0.2151829004,-0.4242037535,-1.5490713120,-3.3838853836,0.3964085877,0.4213456511,-0.3092971146,0.5519814491,-1.0131312609,0.1652939320,-0.7183796763,-0.2181751877,0.4323822856,-0.4072811007,-1.3852882385,0.0991055146,-0.9232534170,-0.0023331596,0.5881130099,0.4912368953,0.5116588473,-0.4789597988,-1.3557976484,0.8674719930,0.8575125933,-0.1776522994,-0.1775404066,-0.1856035888,1.3404828310,-0.9878146648,1.6222991943,-0.2595064640,-0.5972997546,-0.5425067544,0.5912622809,0.3834866583,1.3222781420,-0.8192518950,-1.0907922983,-0.6246671081,0.6052445769,0.4081635475,0.2373645455,0.1690674126,0.5802407265,-0.5613754392,0.2456480712,-0.9186141491,-0.2769204080,-1.7706328630,0.4763954878,1.5354692936,-0.0598922297,-0.5450186133,-0.1883676797,-2.5338580608,0.4641127586,0.3048112392,-0.6876582503,-0.4315545857,0.3876003921,-0.8581926823,-0.6158384085,1.0436244011,-0.3341654837,-0.0467620641,0.7010356784,0.1533711851,0.9586766362,-0.0557326339,0.3174902499,-1.2664761543,0.1697869450,-0.8404021859,-0.4100980163,0.2164599746,-1.5345907211,-0.3343243003,0.1733925790,0.6227089167,-0.3400717676,0.2302381098,0.7133636475,1.0837763548,0.6489784122,-0.4140874445,-0.4357428849,1.0704756975,-0.1154135317,0.4830461740,-0.2553482950,0.6317064762,1.4388637543,-0.6082205772,0.6572952271,0.4606111348,-0.4769994318,-0.6360312104,-0.0991303176,1.0446186066,0.9986180067,1.2831765413,2.1639351845,1.1235325336,0.2174135298,0.1962851286,0.7651981115,-1.2645888329,2.4407944679,0.0277731884,0.3911207616,0.3047181964,1.5580196381,-0.7807807326,0.1302722842,0.9293032289,1.0180013180,-0.1589470506,0.9012888670,2.0110130310,0.4158630371,0.0888017491,-0.3296423256,0.2352096140,-1.3284860849,-1.0254509449,1.5939886570,0.1282520890,0.4186613858,0.3527747393,1.8500404358,-0.6223271489,-0.6476612091,-0.2049963474,-1.3311686516,-0.1923779994,0.9806227684,0.0004467586,-0.6268636584,0.0827400163,0.9971995354,-0.9771696329,-0.2499272972,-0.3006271124,-0.3148871660,1.7323715687,1.7476186752,1.2388221025,1.5113986731,-1.3835494518,-0.7499847412,0.6338880062,0.5752592087,0.4825147986,-1.0033938885,0.3850116134,-1.2665108442,-1.2748463154,-0.0880842432,1.2344012260,-1.1817024946,1.1284167767,-0.3691191077,0.0791317150,0.3606567383,-1.3921650648,-0.8959398866,1.5028594732,-0.6054318547,0.2944380939,-1.0308992863,-0.0682621300,-0.9367830157,-0.4790829122,0.3819047511],[-0.0056958185,-0.0107525261,-0.1028386652,-0.0407153480,0.8296846747,-0.1679052711,0.7480251193,-0.2704329193,-1.6717312336,-0.9499000311,0.0938736945,0.0792115331,0.9397433400,0.4047400951,-1.5396602154,-0.6735089421,-0.1467912346,-1.1075301170,-1.2755281925,-0.2567361891,-1.1061099768,-0.7344524860,-0.3856717646,-2.1764931679,0.0527051352,-1.4594818354,0.1187220141,0.0322499759,-0.8149708509,-0.0294837058,0.8411209583,0.5211537480,-0.0760316700,0.5130376816,1.3642598391,0.0968394950,1.3154854774,-1.4872413874,0.9520645142,0.3005496562,0.6140743494,0.4355062246,-0.5848545432,-2.5305237770,1.3488068581,0.3234812021,0.0880915597,-1.1336556673,0.0930403545,-0.1541861147,-0.3016662002,-1.4424922466,1.5437906981,-0.6499189734,-2.3192703724,2.1626596451,-0.9241775870,-1.1621884108,0.9141823053,-1.0259217024,1.8104969263,-0.3519425690,-0.9066295028,0.5648496747,-0.8370807171,1.4850924015,-1.0320893526,-0.5918660760,0.1176175773,0.1124671847,-0.5896363258,-0.5832690597,1.6353310347,-0.7493469119,0.0465250574,-1.0367507935,-1.4493075609,0.7984340191,0.0820988566,0.5954594016,-0.3189447820,0.5058848262,-0.7114982605,0.5402135849,0.9257064462,-1.4224647284,-0.7928759456,0.0240264982,1.6763641834,-0.3487647474,-0.1829279959,-0.3225548267,-0.3663921356,0.1824751198,1.0975593328,0.1602485627,-0.6861212850,0.7449244857,-0.1191331446,-1.4907772541,-0.0717278123,0.5842505693,0.2929826379,0.0632381737,-0.1342022121,0.4378673136,0.4098827243,0.8411206603,-0.0789814442,-0.2399690449,0.5405627489,0.6278280020,-0.0108255902,0.0505150706,0.9220871925,0.4489594996,-1.0202002525,0.0912503898,0.9841045141,0.9010125399,-1.1630496979,0.0682095513,0.8780198693,0.0305589475,0.3894029856,-0.7775303125,-0.0655512884,-1.6206985712,-1.1411087513,0.6450503469,1.2038553953,1.4712681770,1.2446136475,0.2164686620,0.2948817015,0.0351893939,-1.1672149897,1.1487790346,0.4665854871,-1.8246629238,-0.4153800309,0.1016899049,0.9141763449,-1.3728073835,-0.1546313763,0.8691310287,-0.7926211357,-0.1904437244,-0.6742334366,1.5098134279,-0.7134387493,-1.3717981577,-2.2027521133,0.1625714153,-1.5810571909,-1.4009828568,-1.3369977474,-0.9206066728,0.5045040846,1.0087817907,0.7085490823,-1.8204631805,0.3505395055,0.1601599306,-1.2087808847,0.4851200879,0.6961211562,-0.8115411401,0.6887942553,-1.4158912897,-1.6351356506,-1.5797746181,0.6322211623,0.7213320732,0.4606862962,0.8185957074,0.4297682345,-0.0461610332,-0.4848355055,0.0079860473,-0.2723667622,-1.7874740362,-0.9821856022,-0.1429887712,-0.0252348650,1.9776024818,-0.0204313695,1.2319833040,1.4091806412,2.0734572411,0.7006293535,-1.7150832415,0.2069371343,-1.6790082455,-0.2211100459,-1.1749776602,0.0365302674,-0.3868663907,-0.0528737232,-0.9732688665,-0.3719446659,1.6559091806,0.3440146744,0.3946353495,-1.2677277327,-0.1625610441,-0.9379303455,0.5799641013,-0.0261510685,-0.7596936822,-0.7694777250,-1.0063365698,0.2434116006,0.4409281015,0.7339676619,0.8679358959,1.1445062160,-0.8765382767,1.2840873003,0.0781262815,0.6481414437,-0.9263640642,-1.4005607367,-0.5750855803],[-1.0007141829,-1.0111794472,-2.0524334908,-0.6442151070,0.7671940923,-0.1145124212,-1.1316365004,-0.4282675087,-0.5749440789,0.1833458543,-0.1921777874,-1.4095654488,-0.8744406700,-1.5981025696,-0.5336658955,-0.9894477129,-0.4347616434,-0.5107789040,1.3670536280,1.3113130331,-1.3499810696,-0.3468316793,-0.0047309161,-0.7323764563,-1.0284543037,-0.0138079068,0.1132538766,-1.5646991730,0.0751108974,0.8303509951,-0.1938802451,-1.9379101992,0.0315455496,0.6615291834,-0.3474954963,0.0518144444,0.2939739525,0.0353117324,-0.0989409313,-0.3005047441,0.1867303848,-0.2230330557,0.1489258558,-0.3088251352,-0.8473169208,-2.1500792503,1.2460508347,-0.5631950498,-0.0580349751,1.0495109558,-1.5226836205,-0.3884546161,-1.3195962906,-0.4702957571,0.9313479662,-0.6823721528,0.2000008970,-0.5764540434,0.8537899852,0.9402247071,-1.2445976734,1.5359268188,2.0948321819,0.0036094575,-0.8047494888,0.6427945495,-0.2275648564,0.9242594838,0.5359516740,0.4261472821,0.1482180059,2.3247148991,-0.0195343848,0.1890586466,0.5467409492,0.1936537623,0.1615436673,0.7448188066,-0.1855718195,-0.0946201459,-0.7014371753,1.3129265308,0.2720538378,-0.8413616419,-0.4604212642,-0.4750130177,0.1501253098,0.5798549652,0.6560567021,1.0006505251,-0.0545246713,-0.8020230532,0.1682970226,-1.3182489872,-1.1121965647,0.4462005496,0.6017032266,-0.9011870623,0.0598643757,-0.4081072211,-1.7386128902,-1.3758144379,1.4552366734,-0.4543621540,-0.0994410738,2.2818775177,0.5025957227,0.1134583354,-0.7030552030,-0.4476456046,0.2547734380,1.2222807407,-0.9206188917,0.2277347744,0.0041218023,-0.5322578549,-0.6025855541,0.7921522856,0.7033787966,-0.5238165855,-0.3242940903,0.2282759100,-0.7225562334,0.2189976722,1.8640956879,-0.2014419436,-2.0383541584,0.7670707107,0.0946207866,-0.9718177915,0.5874319673,0.6024810076,-2.0453674793,0.3536788225,0.9645302296,-0.1440635771,-0.5780105591,0.4737685323,0.5964793563,-0.6031044126,1.7813662291,0.2994776368,0.0040594037,-0.4686329961,2.2406263351,-1.0148507357,-0.1007227823,0.5790359974,-1.3669410944,2.4542148113,-1.0502914190,-0.3570873737,-0.5959746838,0.1286456734,-0.2735119760,1.1831076145,-1.9064669609,-0.0173027087,-0.8793388605,0.9526354074,1.7210581303,-0.5359464884,-0.4232902229,-0.7482769489,0.3254696131,-1.7439950705,-1.3261831999,0.6918687820,1.3905178308,-0.2113317549,-0.6825257540,-0.3879581392,1.7752315998,-1.2752637863,-0.1166420951,-1.2915318012,0.1837764680,-0.1601903141,-1.2379313707,-0.0902454183,-0.2519405782,0.5891952515,-1.2630279064,0.6976277232,-1.1750415564,-0.5748518109,-0.6612970829,-0.2794839144,-0.5347990394,-1.3255223036,-1.1630715132,-0.1496529430,-0.2865560055,-1.6683949232,-0.9315761328,0.7481259704,1.2013155222,-1.5000940561,-0.6404548883,0.7500379086,-0.4412990510,0.5010219216,-0.5410768986,0.0552483760,0.4714125097,0.0664171502,-0.5142256021,1.9464106560,0.3117389381,0.1133240163,0.0764206946,-2.1993992329,0.5718014836,1.3705084324,-1.5698078871,-0.5611788034,0.9989655614,-1.3109512329,0.2550405562,0.2179651111,-0.2458942831,0.7524243593,-0.8888767958,-0.9764984250],[1.2953237295,0.4097941518,0.1875261217,-0.6974471211,0.5086981058,1.6450030804,1.9019715786,1.1486008167,0.7689781785,0.5361904502,-0.6677723527,0.6889824867,0.9869290590,1.0165002346,0.5444405675,0.5614240766,1.5270713568,-0.8393536806,-0.9333688021,-0.0409526937,-0.0998738036,-1.0882169008,-0.3863084316,0.3301858604,-2.1117844582,0.4842156470,1.3418143988,-0.5743961930,0.0791617855,-1.6336181164,-1.3499776125,-0.0084255254,-1.4727298021,0.7257235050,1.0187690258,0.8479738235,0.0947717726,-0.4799260199,0.0153171076,1.7028288841,-2.4398531914,-0.7412115335,-0.7696051002,1.3584727049,0.5375123620,-0.4170902073,-0.3068646789,-0.3780648708,-0.6893613338,-0.3763635159,-0.5504732132,0.6834804416,0.1133769155,-0.8619261384,1.0548527241,-0.3079719841,1.3956831694,0.2459279150,-1.6134513617,0.0425924845,-0.2693678439,0.4709191024,1.9377353191,0.7869468331,-0.0102772806,0.7438308001,-0.9149633050,0.0896701813,-0.4285502732,-1.3545906544,0.3714318871,-0.9662501812,0.0663880482,0.0755720139,0.8606848121,-0.3789045811,-0.5214172602,-0.0249759257,0.3855774105,0.7961698174,-1.1770154238,-1.4023742676,-0.2600174546,0.4639261067,0.4731233418,-1.4157626629,-0.8307561874,-0.4470773637,-0.3219674826,-1.9117525816,1.8478777409,0.4157984257,0.2571049929,0.3644884229,2.1297881603,-0.2326315641,0.4425879419,1.5757837296,0.0731884092,-1.9210509062,-1.3611035347,0.1254009157,-0.7626289129,1.3449469805,1.7275245190,-0.5753748417,0.7083702683,-1.1395683289,0.9009451270,-1.1356049776,0.1987769753,-0.0978872925,1.1407341957,0.9220844507,0.4430917501,0.6003719568,-1.2657593489,-0.3270651400,-0.5956538320,0.1456205398,0.6889383793,0.1033993214,-1.3690428734,0.7404351234,-0.1215081289,-0.1990689635,0.8810068369,0.3059888184,0.4480573535,0.0953786969,-0.1013018936,0.4117246568,0.3502552509,0.0194713138,-0.2473152578,0.0101099778,0.9885329008,-0.8361511827,1.9361438751,-1.5811871290,-1.1002520323,0.9397823811,-0.0545123816,0.7610992193,0.5216606259,-1.0363000631,-0.7652194500,-0.0382754430,0.9650835395,1.8669173717,0.1112473533,-1.2015788555,0.6443081498,0.6186507344,0.0816031545,-0.4066605866,-0.4410473108,-0.5908294320,1.1830356121,0.2105588764,-0.1489932090,-1.3678106070,0.5889723301,0.3990076482,1.5867253542,-1.5419884920,-0.3135329485,1.2928620577,0.5041116476,-0.9222351909,-1.7613097429,-0.4756738544,-0.7624484897,0.4504918158,-0.4556908309,-0.0724610984,0.7097553611,1.1203337908,-0.3484529257,1.1659314632,0.1397081167,-1.5629338026,-0.7076761127,1.4508572817,0.5759756565,-0.9922260046,0.2997192740,-1.5346330404,-0.9537140131,2.3760414124,-0.0492447056,-0.2149013877,-0.7276610732,0.0869092569,-1.3743280172,0.0617098734,-1.3765540123,-0.3937661052,0.5421155691,-1.4106866121,0.4895611703,1.1866769791,1.5226407051,-0.4637456834,-1.9443510771,2.0126781464,1.2011687756,0.4509884119,0.4688838422,0.1416492164,2.0351710320,0.4427682161,0.4046050906,0.5470671654,0.1857168674,-0.1370263547,-0.7957639694,1.1775459051,-0.2308127433,-0.3254511952,0.6586336493,-0.6313273311,-2.5197975636,0.6868247986],[0.9503057003,0.5775336623,-1.1326794624,-0.1656296253,1.3421090841,-0.0567202605,-0.6803632975,-0.4050382078,0.6355462670,-1.5059229136,-0.2022885233,-0.2591747940,0.3714067638,0.8712681532,-0.3979974091,-0.9245500565,-1.0143244267,-0.4404507279,0.5970969200,0.2319602817,0.5535842180,-0.4240028858,0.5902436972,-0.5997185707,0.1466287971,2.3729963303,3.0086288452,-0.5136756301,-0.4350767136,0.9363536835,-0.0597128756,-0.8972352147,-0.6416056156,1.2984263897,1.8004604578,-0.7545091510,-0.3338066936,-0.2864007652,0.4252581596,0.5166036487,-1.0704407692,0.3727474511,-0.4662492871,-0.8777085543,1.0085009336,0.4567803144,-1.4343152046,-0.0693903938,0.6386763453,-1.2341254950,-0.0431989394,-0.1082422808,1.1039363146,-0.9819817543,0.6192163229,-1.2208844423,0.5299162865,-0.1968284845,0.7186149955,-0.2716750801,-1.0598012209,-0.6523534060,-0.7307460904,-0.1305984706,0.4185395837,-0.1593158543,-1.0495679379,-0.3906728327,-0.2174431086,0.1957010031,-0.8210510612,-0.7583345175,-0.6436064839,0.6527817845,-2.2995562553,-1.4217740297,2.6572241783,-2.4107866287,0.2148666978,1.1167837381,0.0689454675,-1.3653887510,1.1535874605,-0.4121600389,0.2929527760,0.3599597514,-0.6850985289,-0.4406382740,-0.5719017982,-0.1960382611,-1.7868933678,-1.3133436441,-1.1885946989,1.0874711275,-0.5319527388,-0.6841382384,1.2883000374,0.0816733018,0.5675491691,-0.3255595267,0.3317215741,1.0300211906,0.5762638450,0.8731001019,0.8163179755,0.7308938503,-0.6628636718,0.3519839942,1.2436153889,-1.1196715832,-1.0711348057,-0.1543116719,-1.0863668919,0.4676058888,-0.6846093535,1.3797492981,0.3526509106,0.1648032069,0.0237876102,0.2023786604,-1.1174907684,2.2319457531,-1.5312778950,0.6684376597,-0.8544011116,0.6476187706,-0.7095187306,-0.5701030493,0.2349684983,0.7085313201,-0.2691114843,0.3251794875,-1.1168712378,-0.3028653264,-0.2788650990,-1.0868864059,0.4872075617,-0.5189303756,1.1145070791,0.4584412575,-0.5998378396,-1.4327603579,0.1275960505,0.2901370525,0.5808152556,-0.3115820587,0.8584854007,0.8452209234,0.3459075093,-1.5614558458,1.6772382259,0.3585601747,-0.8467258811,-0.4606466293,1.7624496222,-0.6848064661,1.2684752941,0.8637610078,0.4877776504,0.8418605924,0.1541614383,0.8735924363,0.3841915727,1.6905543804,-1.1739689112,-1.3215125799,0.1166850626,-0.5511772037,0.1038629040,1.0686504841,1.3229821920,-1.6473169327,-0.8972117901,-1.7887773514,-0.7082397938,0.0563377440,-0.7301474810,-0.6179089546,-0.7773608565,1.2667914629,-0.5679482222,0.7532033324,1.0075652599,-0.5667064786,0.8302335739,-1.4011286497,-0.0043775220,-0.7475999594,-0.7050182223,-0.8238686323,0.0595666282,-0.9293041825,0.3678091466,-0.5593569875,0.8309092522,1.4071900845,-0.5506002307,-0.8832412958,-0.4212633371,1.7079013586,-0.2129941732,-1.2215648890,0.9746375680,0.5255259275,0.1676276326,-0.7460688353,-0.3985492587,-0.2419589013,1.0022572279,0.3096496165,0.0151867429,-0.9989675879,-0.8324916363,-0.2698731124,0.7522588968,-0.7222785354,-1.3158954382,0.2006363124,-0.5149350762,0.0061105140,-1.7563093901,0.0626748428,-0.5142523050,0.9311300516],[0.2424508929,1.6043307781,-0.3289819360,-0.0494096093,0.7316811085,-3.0088012218,-0.3715510070,0.6926851869,0.0006539202,-1.5461733341,0.0995478183,0.2765178084,1.4613583088,-1.3338665962,1.1406615973,0.3171734214,-0.4835242331,-0.2373184413,-0.0232107006,0.8536342382,1.6586548090,-0.5782043338,0.4537277818,-1.6588265896,-1.2349920273,0.0675332099,-0.2628900707,0.3805647194,0.8191167712,-0.1620512605,-0.6835185885,-0.8386046886,-0.4493160546,0.1507066637,-1.9121794701,0.5574362278,0.7430413961,-0.2474150509,1.0876133442,0.4239715040,-0.3730142415,-0.0829822570,-1.4674832821,0.2024282217,-0.8522486687,2.2314176559,-0.5353853703,-0.1779713780,-0.2383777201,0.5534749031,-0.7674822211,-0.3580855429,-0.8130614161,-0.4385259449,0.4330780208,-1.6723484993,0.4503363967,0.2866947651,-0.3895815313,-0.2773104012,-0.2627112865,0.8851501942,-0.4606986940,-0.6227043271,-0.2819042802,-0.1450483352,-0.1862329692,-0.9947724938,0.1286654323,0.8603655100,0.8581410646,0.0987485275,0.5210778713,1.4589275122,0.6336409450,-0.8249817491,1.1077028513,0.0143080968,2.6831395626,-0.4208842516,0.1807337701,0.8042305112,-1.6470074654,-0.5645323992,-1.4007233381,-0.1503272057,-1.0351790190,-1.1351163387,0.3737889528,-0.4462286234,1.0661048889,0.3744573891,0.9195076823,-0.4215391576,-0.5308960676,-0.2440245748,0.2437709421,1.8744490147,0.0601186305,-0.2889816463,0.6083256006,-0.5951191187,-1.1753454208,-0.4446639717,1.6190202236,0.2731115520,-1.2823290825,-0.6856404543,1.0023931265,-1.0596644878,0.0099540399,-0.9112805724,-1.9877477884,-0.3559651077,0.6992651820,-0.5291158557,-0.2777605951,-0.6945871115,0.4263310730,-0.4606139064,0.5313073993,0.3477262855,-0.6290251017,1.0485340357,-0.4984591007,-0.7014091611,-0.1268873066,1.7174379826,0.3756468594,0.6778792143,0.6780077219,-1.9288370609,2.1379055977,-0.3503865302,2.6057658195,1.4180355072,-0.8483371139,0.4836727679,0.2986367643,-0.4045870006,-0.3841612935,0.4206042886,-0.2121454924,-0.7215571404,-0.9440875053,-1.8669323921,-1.2715229988,-1.3166813850,-0.2099368423,-0.4971982241,-0.6165689826,-0.8177767992,0.7123737931,0.0018365026,0.2189357430,0.4399895966,-1.4153739214,-0.4745371640,0.8956039548,1.8533095121,-0.6692598462,0.8224695325,-0.4222893119,0.0309825372,-0.0985280126,0.0038888101,-0.6720025539,-1.0864828825,-0.5846391916,-0.8959212899,0.7892262340,-0.5727711320,-0.2901080549,2.3455419540,0.5100870132,0.6484789848,0.2203816473,-1.2419773340,-0.2932655215,0.0602538697,-1.2372095585,-1.3545566797,0.7277938724,-0.5110531449,1.8365730047,0.1518082768,-1.2500865459,-0.8315503597,-1.6715674400,-0.6968267560,0.1409083009,-0.4922994673,-0.7246807814,0.0654167756,-0.7244126201,-0.1597559899,-0.5303936601,0.0489419326,1.0108239651,0.6513858438,0.5965271592,-0.8281033635,-0.4502198994,1.3260207176,0.1846620142,0.3190106750,2.7237520218,-0.2407226712,-2.0167696476,0.0360962264,-0.6404058337,2.0376145840,0.4976940155,-0.1475096196,-1.6457666159,-0.3195464015,-0.4240814745,-0.7984828353,-1.7798746824,-0.5305024981,-0.9451339841,-0.3695907891,-0.6193536520,-0.8740698695],[1.6382918358,0.8859256506,-0.6783981919,-0.6647996902,1.8402383327,-0.4664635360,-0.2869035602,0.8843571544,-0.6794208288,-0.0783729777,2.0344007015,-1.1569439173,0.5343900919,-1.2190382481,-2.7440197468,0.3725993037,-0.9437746406,-0.8705223799,-0.2391330153,-0.3168485463,-0.1414447427,1.3143306971,-0.0398241282,0.9941142797,0.5481469035,2.4456527233,1.1136090755,0.8380909562,0.8857203722,0.4825400114,-0.4374621511,-0.3491562307,-1.1849820614,0.9566195011,-0.6884231567,-0.6400103569,0.4962954223,-1.2663766146,-0.3680319488,-0.4868077934,-0.5154715776,-0.2976514101,-0.8406806588,-0.0521839485,-0.2806237936,0.2266739756,-0.2517160773,1.0988744497,1.2061139345,0.1976041198,-0.5565751195,-0.4206511378,-0.6106412411,0.6980303526,0.8038991690,-0.5697152615,1.1649491787,0.7628110647,1.1644734144,0.9731919169,-0.7703890800,2.5272583961,0.2993666232,0.9850234985,0.7090337873,1.2726600170,-0.9547067285,-2.2828862667,1.8040145636,-0.3618632853,-0.2424062788,-0.0360498950,0.6348895431,-1.1561373472,1.6774686575,-1.1213855743,-2.1498403549,0.4187663794,-0.8002408743,-0.7238284349,1.1115944386,1.5962899923,-0.8676059842,0.7034949064,-1.2499570847,0.1972162724,-1.4184181690,-0.1329640746,-2.0752580166,-1.3371758461,-1.7525701523,-1.4658756256,-1.5818002224,1.6401294470,-0.4202954471,0.6047040820,0.3399446011,-0.8635126352,2.1736896038,-0.6485803127,0.7760526538,-0.5920584798,-0.1532118171,0.0984346718,-1.1254554987,-0.4605285823,1.1066592932,-0.3280683458,-0.2730957568,0.0873861089,0.3761961758,0.3713440597,0.1002379730,-0.3835586011,0.5256533027,1.5230630636,1.6018843651,-1.2051194906,1.0897098780,0.7993661761,0.0984100774,0.5448901057,-1.2182017565,0.2122826129,-0.0533185825,-0.8356292844,-0.0336980745,1.2712229490,-1.2643465996,-1.6775107384,-0.9985657334,1.0351241827,-1.5880872011,0.7903474569,-0.0167097095,-1.2681533098,-1.6248126030,0.6737528443,1.4385950565,0.2779538631,1.9929624796,-0.1199937686,-0.4443750381,0.9745465517,0.4621928632,-1.5144137144,0.9511830211,0.8978304267,-1.1884974241,0.0929282978,-0.5976312757,-0.9062644839,-0.4685940742,0.0498959236,0.1588799804,1.4182624817,-0.0872410312,-0.4276543558,0.0140459212,-0.8659215569,1.4454176426,-0.3340324461,-0.0349760242,-0.5698040128,-0.9010173678,2.1993246078,-0.8341708779,0.8671241403,1.1048197746,1.0847196579,-0.2546294630,1.2404844761,0.4475292861,0.2714076340,-1.4715112448,-1.2379156351,-0.2507206798,-1.3513888121,0.2730809152,-0.0653217137,-0.1397357732,0.5785194635,-1.1722716093,-1.1036603451,-0.0131531553,1.6345367432,1.2507511377,-0.5973930359,-0.0155039188,-1.9891339540,0.3615313172,-1.4075814486,-1.5032497644,-0.1248913184,0.6577493548,0.0023436123,0.2874972820,-0.8602231741,-0.0809271559,0.1523856521,-1.0670419931,-1.0369086266,0.2998365760,-0.6676939130,0.6504074335,1.3342704773,0.5072284341,-1.4595576525,0.7439997196,-0.9475765228,0.2244775295,0.4849691689,-0.3844098151,0.1123687103,-0.6872268319,0.3756891489,0.6597490907,0.7952565551,0.4855103195,0.3044109941,0.1490117610,-0.5080056787,1.2712522745,1.2272012234],[-0.3299134672,-0.0199540183,1.8290190697,0.5677607656,-0.5968216658,-0.2418765873,-1.9487468004,-0.2772829235,-0.5381264687,1.2586739063,1.5063394308,-1.3124256134,-0.2960510552,-0.7850624323,-0.6174619794,1.1621369123,0.5564032793,-0.7371488214,-0.3468227684,2.3980824947,-0.1111919731,-0.2277705222,-0.6066004038,1.5664755106,0.8420552611,-1.3558931351,-1.0245736837,0.8999151587,-0.5651986003,-0.8465386033,-0.4856346548,-0.9645081162,0.1725240499,-0.7497085929,0.9174042344,-1.7963420153,0.0882364661,0.5786577463,-0.0313097350,1.8228253126,-0.1195402592,-0.3144756854,-0.7695646286,0.6737496853,1.7059468031,0.1110388115,-0.1290749758,-0.7515364289,1.1298259497,-1.9274809361,0.2750416994,0.9833395481,0.0931785628,0.5955570340,-1.3676586151,0.5912148952,-0.6878173351,-0.2396046817,0.4571484029,0.3076282442,-1.1353645325,0.5450010300,1.2716612816,-0.7891782522,-1.1442798376,0.5127852559,-0.3669523299,-0.8581065536,1.3917527199,-1.8237631321,0.1568117738,0.6871304512,-0.8506605029,-1.0436689854,0.7592877746,-0.4107601345,-1.4867564440,0.6242988706,0.4792506993,-1.1623834372,1.2963851690,1.8081043959,2.4934203625,-0.2422652245,-1.7136925459,-1.8513816595,-0.8949121237,1.1286860704,0.3440145850,-0.4634765983,-0.0613686703,1.7311720848,-1.2469245195,1.2263395786,0.3082263768,0.2303123176,1.1889343262,1.2646365166,0.0945713371,-0.1564330161,-0.1214945167,0.6576443911,-0.8273627162,0.1350538135,0.2720624208,-0.1212793067,0.7692606449,-0.7661889195,0.6638042927,1.2159714699,-0.6703962088,0.4307232499,-0.5342184901,1.0875924826,-0.5991690159,0.3697001636,1.0889072418,-0.2163190544,-1.2570856810,0.6826377511,1.3048549891,0.8470559716,0.3935773671,0.7606564164,-0.9260808825,-0.3522996008,0.4411866665,1.0888419151,1.8529471159,-1.0159670115,0.1586700678,-0.2941544652,-0.7677201629,0.6702014208,3.0485541821,-0.7110427618,0.0087685026,0.2127590925,-0.6027239561,1.8547546864,-0.9267640710,-0.9728785157,-0.5564861894,0.1114080474,0.6857814193,0.5599179268,1.0410881042,0.2579667568,0.9110988975,-0.9568061233,0.1526732743,-1.4196509123,-0.7998716235,1.2206411362,1.0975108147,0.8231616616,-0.0641085878,-0.6946839094,1.6023092270,-0.6245922446,-1.9093415737,1.3474690914,1.4625424147,1.6814835072,-0.1330091953,-1.6498638391,0.0761528090,-0.4358590245,0.2756622732,-0.8825276494,-0.3384377956,-1.3949745893,1.0857973099,-0.4014986753,-2.3074398041,1.1703572273,2.3323669434,-1.2561129332,-1.0790011883,-0.7676408887,0.3213644028,1.0556669235,-1.2354131937,0.6065539718,-0.6644827127,-0.8000290394,0.0954485089,0.3187395930,-0.6467474699,-0.3926055431,-0.8689112067,-0.3884165287,-0.4145347476,0.0493419617,0.2418716103,-0.3547745347,0.6621610522,0.8859947324,-0.7657930255,-1.9621956348,-0.6089704037,-1.1985352039,0.7472391725,0.6400617957,-0.4264973700,0.2084112614,-0.4279962480,-0.1868031621,-0.1573167592,-0.1913782507,-1.0503474474,0.1423900723,-0.7872502208,1.3039880991,0.2708427310,2.0053474903,0.8214874268,1.5796195269,-0.7677533031,0.6795278192,-1.0888278484,0.7510353327,2.0150184631,-1.5912657976],[0.8089207411,-0.1970883459,0.8027809858,-1.2548944950,0.5105749965,0.4831301868,-1.5909609795,0.5785495639,1.3184320927,0.0695535466,-0.2157524079,-1.7929224968,1.2852239609,0.4923533201,0.3263441026,-0.9354546666,1.4339323044,0.2739963531,-0.0995752290,0.2425071448,0.5357111096,0.4545810223,0.5341140628,0.0805591345,1.3591556549,0.1434901059,0.2359346896,-1.2574553490,0.3136060238,-0.4058042765,-0.8217685819,1.3852506876,-0.9113008380,0.7037917376,-0.7449426055,0.9320917130,0.8437697291,-1.9806908369,-0.0670251027,0.5133256316,-0.4757016003,-1.2597001791,-0.4355548322,1.3151249886,1.1659435034,1.6419327259,-0.6083964109,0.4444638789,-0.4170851111,-1.5645383596,-1.3208391666,-2.3012502193,-1.0070270300,-1.3298571110,0.1743980348,2.5718021393,-0.6891092062,-1.6248441935,-0.5891755819,1.6138558388,-0.0993815511,0.6511113048,-0.2385976166,0.6658123732,-0.7941064239,1.3413614035,-1.2250225544,1.1066839695,-0.7866426706,-0.7299267054,-1.1541323662,2.5609581470,0.5317540169,0.3519812822,-0.6689489484,-0.1380787790,-1.1013877392,1.0932749510,0.9128538966,-0.9880322218,0.7098346949,-0.3154917657,0.6754376888,-0.0766175464,-0.3399885595,-1.0069338083,-1.2817411423,0.1082873270,1.1880060434,-0.3071840405,-1.3175830841,-0.3208683729,-0.7482139468,1.1091876030,0.0395907052,-0.0068238205,-1.8176137209,-2.2272694111,-1.0444130898,1.7971816063,-1.3809732199,-0.8909279704,1.7548052073,-1.9525624514,-0.4300846159,1.0891311169,-0.3463200033,-0.1930101216,0.8065233231,0.1266611665,-0.4590313137,1.0066936016,0.9566790462,-1.7197459936,1.0318695307,-1.4141339064,-0.0097696707,-0.1190411523,2.0956239700,0.4579995573,-0.6332737207,-1.4304943085,0.9629821777,0.3824220598,-1.4023016691,-0.5606012344,1.1431181431,-1.8398458958,-0.0627954900,-0.6901521683,1.6339272261,0.2412263602,0.1537550688,-1.1139428616,0.8870545030,-0.1872967929,0.5684547424,-1.0961941481,-1.1251753569,1.1886631250,-0.4543816447,-0.7960464954,0.9073171020,0.6863700747,-0.4552413225,1.5016720295,0.0206240546,-0.5202254653,0.5245888233,0.3571237922,-0.9080306292,1.0753819942,1.7178027630,0.7182201147,-0.5630775094,-0.0887095183,0.1610158384,-0.9418390989,-0.3211406469,-0.5363221169,0.6615352035,0.7728872895,0.5593809485,1.5761929750,0.4294976890,0.4844272435,-0.8199804425,-0.6379592419,-0.8718496561,0.4785815477,-1.6744025946,0.7056298256,1.1006934643,-0.3888773322,0.6314990520,-0.3749686182,-0.5715098977,-0.9321445823,0.6626054645,-0.1253063679,-0.6241140962,-2.5891406536,0.4141782820,-0.6123164296,0.5643692017,-0.7155497074,0.5605821609,1.6867932081,0.1108570546,-0.4897870123,-0.6119366884,-0.0666864291,0.4495974481,0.0085499445,-2.1522080898,-0.7636897564,0.2133529484,0.4067107141,0.8969731927,0.7151698470,-0.4244272411,1.2850111723,0.7844523191,-0.1529647559,-0.1405523121,0.3252138495,0.2649473548,0.5022157431,0.0850721747,-1.4688378572,-1.3880082369,0.6504700184,0.6187646389,-1.3998051882,-1.2288914919,0.1766103208,-0.4094978273,-0.2433090359,2.1143550873,1.4301925898,-0.4404291213,-0.0626156554,-0.4196424484,1.0478527546],[0.9831517339,-0.4679268897,0.3133228123,-0.9002415538,-0.7844398022,-0.3946162462,-1.5743603706,0.6414580941,-0.2618705630,0.6462222338,-0.6590774655,1.0195286274,1.0634303093,0.0653074756,0.7577853799,-0.0516320206,0.6106615663,-0.4302142560,0.8421626687,-0.2350383252,-0.8129963875,0.5303506255,0.7106423974,0.0993085951,0.2506384850,0.9918402433,0.8889755011,1.3261528015,-0.2485965937,1.1315611601,-2.1970417500,1.4103869200,-0.4955347478,-1.8461620808,-0.0154520124,0.2505555451,-1.5311112404,-2.5265429020,-1.0538671017,-1.6120213270,-1.6950901747,-0.5707432032,-0.3110536635,0.0624820627,-0.1097832322,-0.5843027830,1.3145704269,0.2922441363,-0.4884521663,0.1748099029,-0.8954635262,0.8497131467,1.3345705271,-1.5859323740,0.1828643680,0.0375167504,1.4893962145,-0.8841351271,0.1239212006,0.1301903278,-0.2170444280,-1.4712740183,0.2646017671,-0.1447753906,0.9968265891,0.0993815586,-0.6090484858,-1.3570221663,0.7227269411,-0.0219510533,0.7023817301,0.0429944210,1.1319042444,-1.0029493570,-0.9856308103,-1.9643776417,-0.6212342381,-0.5674418211,-0.2722060084,0.5762308240,-1.7308312654,-0.1668733805,-0.3293681741,-0.3582380116,-1.1411564350,1.1042435169,1.6444514990,-0.4203042686,-0.7046890855,1.9541190863,-1.5065824986,0.1620152444,-1.4102603197,0.4765975475,-0.2991090417,1.2118850946,-0.3334017396,0.3071345687,-0.6667149067,1.2665424347,-1.2411324978,-0.3781921864,-0.1119469181,0.0351102464,-0.1019369289,0.0713379160,1.8285521269,1.1005022526,0.3116882145,0.7197256088,1.1275570393,-0.1129998118,0.3050722182,0.1734147966,0.2428309172,1.0162949562,-0.5538579226,1.0453534126,-0.2064165026,0.2954626679,1.1894056797,1.6967473030,-2.0918354988,-0.5620151162,-0.6162993908,0.1899436861,-0.1020427197,-1.0234582424,0.4789095521,0.0332386084,-1.4767955542,-1.9037156105,1.2354108095,-0.0018077961,-0.9863997698,-1.2346477509,1.8885161877,0.2936889231,0.8233779669,0.0891269967,0.4238564074,0.4432381392,-0.8548453450,0.3422487080,1.9250447750,-0.8454698920,-0.6730789542,0.1415164173,0.1184886545,1.3533905745,-0.2967815101,1.1134816408,1.2478499413,0.1484726369,0.7938684225,-0.0986338928,-1.1890305281,0.2958755791,0.1388797909,-0.6163766384,-1.4716683626,0.8207072020,-1.1493715048,-0.2178318650,-0.9332431555,0.0315085016,1.5580168962,-0.3341762424,-1.3798252344,1.9283100367,-0.2898777425,-0.9559110403,0.7655615211,-0.4317200780,0.4280168116,0.7353171706,1.4805815220,-0.2730192840,1.7999408245,-0.6273670793,-2.6943187714,-0.6990944147,1.1060009003,0.8578431606,-0.0840007663,-0.5585980415,-1.2817488909,-2.4378654957,1.6060805321,0.8997866511,0.6551699042,-0.3740907013,1.3194372654,-0.4168642759,-0.8414853215,-0.7929033041,0.3561357856,-0.3539623916,0.2779755890,0.4590013325,0.5880220532,-0.2773589492,-1.8300926685,1.2118127346,-1.0504593849,-0.9004604220,-0.0812280998,0.2696877420,0.0787571594,0.1149807945,1.3142724037,-0.1294342428,-1.7291231155,-0.0914452523,0.1939240694,0.6503858566,0.3664641976,0.7926014066,-0.2039321363,-0.4512749016,-0.5041785836,0.1370467991,-0.3436326385,-1.7019400597],[1.3645384312,-1.0790003538,-0.0885153338,0.7273120880,-0.0962846130,-0.3853226304,0.9186987877,1.6411539316,0.0192694329,0.9914398789,1.0116108656,-0.2402564734,-0.2873272002,-1.0821862221,-1.9422683716,-0.5636457205,-0.9914990067,0.7986121178,-1.1720125675,-0.8651517034,-1.0168634653,1.3929049969,-0.5326869488,0.2972522676,-0.7363356948,-1.0323220491,0.2945378721,1.2807190418,0.2211526334,-1.8246880770,0.2985337377,-0.1622590274,-0.5336527824,0.5530636907,0.9003993869,1.0602345467,0.5563013554,-0.4820825756,-0.3741382360,1.2548416853,0.5277336836,1.0442218781,-1.0164369345,0.9119832516,1.8069756031,-0.8453161120,-0.6892279387,-1.9548838139,0.5399494767,0.5839894414,0.6357702017,0.4068482518,-0.8281987309,0.7952207923,-0.5682151318,0.6527551413,0.4371876717,-0.8736358881,0.7888579369,-0.2305625677,0.3886904716,0.4261684418,-1.7041398287,0.1460140944,-1.3941833973,-2.3413891792,0.3794504404,-1.5065003633,-1.0899549723,0.5814636946,-1.8431659937,-1.9167656898,1.0604321957,-0.6192001104,0.1313293129,-1.9399279356,-2.5218393803,0.8841423988,1.4313941002,-0.0698332638,-1.7999757528,-0.5638673902,-0.0389924608,0.8049888015,-0.9703688622,0.3512552083,-0.2308388054,0.1044155210,-1.2669880390,0.4026793242,0.9971115589,-1.0468974113,-0.1251971573,0.1129145920,-0.6320928931,0.7838298678,0.4312362075,1.1978577375,-0.6947177052,-0.2477532178,0.4473966956,1.3458278179,-1.3238114119,1.2147200108,-0.3629972637,-0.1120925844,-1.6106210947,-0.6715474129,-0.3695737123,-2.1063737869,0.3464091718,-0.1429023743,0.0154739022,0.2679836154,-1.3591574430,-0.4414421618,-0.5401853919,1.2818124294,-0.8664167523,1.2607386112,-0.5515830517,0.0223816186,-0.9567921758,2.1468987465,0.4018232524,0.3902551234,0.2271844745,-0.7034388781,-1.1465339661,1.3632919788,0.7479792237,-1.4447779655,-1.3149640560,1.6014412642,0.8386735916,-0.9777460694,0.0764696151,0.8212373257,0.7903966904,0.2094957680,1.1794633865,0.3876317143,0.2945517600,-2.2883436680,-1.4445925951,-0.0050359364,-0.0154747693,-1.0497438908,-2.1858596802,0.3199483752,0.4688971639,1.3384841681,-0.1049553305,-3.5104098320,0.0139802890,-1.0582283735,0.0157511942,-0.1317526698,0.5070823431,0.2266791016,-1.0741680861,-3.3275530338,-0.1029340774,1.3333551884,0.1533930302,-1.3400937319,-2.1410186291,0.6519215107,0.0122133074,-0.0897119120,0.7088866830,1.3721456528,0.7374876738,-0.1000189185,1.3357746601,-1.4139446020,0.6488098502,0.5202904344,0.3015293181,-2.3389220238,0.8091678619,0.6978511810,0.9705064893,0.3247336447,0.0050228359,-0.8070387840,1.5278664827,-1.6689298153,0.7235268950,0.3391357362,-0.4543726444,0.7102225423,0.0603245422,-0.2386469692,-0.1376174837,2.2262902260,0.7978293300,-0.1623223573,1.3508191109,-1.1919469833,0.5105102658,-1.2463465929,-0.7253215909,-1.5177099705,-1.8005180359,-2.0953040123,-1.4494000673,2.0138463974,-0.9422380328,0.4424597621,-0.1397095770,-1.1371566057,-0.0423600972,0.1930373460,0.2709112763,-1.1953920126,0.5570302606,-1.5900931358,0.4856736660,-0.5310302377,0.1510527581,0.3505887687,0.6495220065,-0.3104924262],[-0.2100895643,-0.6556578875,0.0484873876,-0.4949260950,-0.5762988329,-0.0790554658,-0.8858715892,-0.7482577562,0.4565393925,0.8495110869,0.5410699844,0.2003258765,1.1066104174,2.4233675003,1.1743931770,-0.3243298531,-1.0906335115,-1.0359903574,1.1506870985,0.8038827181,0.0065249666,1.2016972303,-0.4348216951,0.4592243135,0.9285517335,-0.9519800544,0.9002231956,-0.0417470746,-0.6550382972,0.3652853668,-0.4817460179,0.4155755043,0.2305550873,0.0383341387,-1.3802094460,-1.0148710012,-0.4687513411,0.2466093600,-0.0788045675,0.3197855353,0.0394861288,-0.9513927102,-0.0540379882,-0.9681669474,-0.6324351430,0.1304977834,-0.1444966942,0.4706237316,0.3779866993,0.2013960183,0.2234681100,0.4384725988,1.8610857725,0.1288777292,-0.5215864182,-1.0086157322,0.7050257921,0.7817319036,-2.3805580139,-0.2110639066,-0.3800990283,-1.5330798626,0.8230214119,0.2022407502,1.2536604404,-0.3143420517,-0.3846403956,0.8717328906,-0.2717157006,0.2214356810,-0.7705176473,0.7046155334,0.6591244936,0.0442096666,-0.8060303926,1.8267966509,-0.2754566073,-0.8256037831,-1.5357751846,-1.0315228701,-0.8241909742,-1.0270508528,0.3138917387,-1.6600333452,-0.3888295591,0.0759017095,-0.5470116138,0.9134837389,0.9181766510,0.6794993877,1.8896580935,1.2877893448,-0.5754200220,0.1308063716,0.6196119785,0.2646083534,1.3591319323,-0.1376973838,-0.4108622372,1.0915858746,-0.4022491574,-0.2787691951,-1.4625638723,1.5055365562,1.3839673996,0.3047131300,-0.1136100814,0.0511076674,0.9380768538,1.2948635817,1.8470802307,0.0492418595,-1.2036857605,1.1663081646,0.4418370724,-0.2655751109,-0.9233930707,0.8207384348,0.5034034252,0.5235258341,0.9936625957,1.0459955931,-0.2367948443,0.4002315700,1.7345780134,-1.6753051281,0.9616335630,-0.5974237919,1.1064232588,1.8339977264,-0.2873453200,-0.2717796862,0.6485736966,-0.7970598936,1.2907695770,-0.1626946926,1.3069441319,-0.6765155196,-0.8155764937,-0.4989302754,1.5502585173,0.6216536760,0.3171115816,-0.9816764593,0.2284350544,0.1353972405,-0.7532781363,0.5987361670,0.2846145630,2.6697471142,0.8273726106,0.3656917810,0.3345196545,1.3987786770,-0.6768500805,0.9154971838,0.9169610739,0.6782941818,-2.2053909302,-0.2868286967,1.4951024055,1.3020434380,-0.1167226881,0.5561181307,1.3928959370,0.8201711178,-1.3544884920,0.7482158542,-0.9934462309,-0.7618706226,1.5368840694,-1.0926820040,-0.7544658780,-0.7657099366,-0.2162702084,-0.6240744591,-0.6869540811,-1.3244251013,-0.0615085214,-0.2740594745,0.7680806518,0.5850219131,0.7697795033,-0.8452374339,-0.2424748093,0.8591217399,-0.1297827959,-0.3272397220,0.5281672478,2.8159592152,0.0707529411,-0.0019932245,0.7365709543,-0.1870024353,-1.0456169844,-0.0245733839,0.3742526472,0.0466749147,-0.9359126687,-0.5803662539,1.2145210505,0.2210874259,0.8019478917,-1.6911070347,-0.8182746172,-0.7799111605,1.5378501415,-0.0670324489,0.5352994800,-0.9907878637,0.0810974091,0.0742879882,-2.1321122646,-0.2070179731,1.0230563879,-1.4774166346,-0.0569191352,-1.1915849447,1.2854273319,0.5736584067,-0.3161622286,0.0788122043,-2.6321368217,0.7516192794],[-0.6747380495,1.7880039215,0.0600294322,0.8084431291,0.8202488422,0.7850489616,1.7138910294,0.8183184862,1.3371344805,-1.0704429150,0.7927608490,-0.7731685638,-1.5480746031,0.9363201261,-0.8637107015,-0.5787624121,0.5742868185,1.2831314802,1.1431300640,-0.1921418905,-0.1575302482,0.7531200051,0.1107663810,-0.1392740458,0.1642324924,1.1048551798,-1.0657318830,1.2152526379,-0.0280989539,1.4245631695,0.3038326204,1.0398333073,0.5144931078,1.3959500790,-1.4316306114,-0.4561814964,-0.5418317914,-1.2976498604,1.6599001884,0.6544582844,0.4120349586,0.1889998317,-2.0116407871,0.0190515742,-1.0315045118,-0.2084806114,-0.9521855116,1.3717107773,0.8201537132,1.2848557234,1.0412549973,1.1097702980,-0.7424427867,-1.3213529587,1.4162770510,-0.2218238860,0.1572814584,0.0420207120,0.8158637285,0.2443248034,1.5342217684,1.2111642361,0.7858352661,-1.0876142979,-0.3700317442,-0.2657727301,0.4492015839,0.0319982916,-1.0240249634,-2.5414311886,0.1361544430,-0.2829069793,0.5869057775,-0.8466492295,1.4590594769,0.6824162602,1.3178961277,0.1956192702,1.9053285122,0.2270766795,0.8468378782,-1.1426655054,0.1628837287,-2.0766322613,0.3954115212,-0.7994513512,0.8160585165,0.2688303888,0.6566452384,1.4310846329,1.2181339264,-1.1336773634,0.7861024141,0.6040832400,1.0225002766,-0.0625130981,-0.9539570212,-0.5392764807,-0.9254257679,-0.8865107298,1.5495288372,0.3416455984,-1.4592988491,1.1174447536,-0.8456580639,-0.2513967156,1.6444369555,0.4698045254,-0.3425119817,0.4703255594,-1.1068733931,-0.6331372857,0.0833904520,2.0509891510,1.1461318731,-1.4606683254,-1.3410766125,0.0828015357,0.8186067343,0.3983367383,0.0698770657,0.4072398841,-0.0359241553,-0.1619289368,0.8406882882,1.8046275377,0.1601787359,0.6465020776,1.1391314268,0.0097278478,-1.3131502867,-0.8829632401,-0.5586873889,-0.8721792102,-1.7141357660,-1.6847779751,0.9566531777,0.0900609344,-2.1673758030,-1.5872809887,-0.4245087802,-0.6903084517,-0.3588181734,1.3054128885,0.7735430002,-1.5612035990,0.0641486570,1.7226607800,0.4291057289,-0.8341003656,0.6271658540,0.8506008387,-0.6141491532,1.2932060957,1.0017577410,0.9150823951,0.9628162980,0.2699362934,0.0677945018,0.6238765121,0.8336200118,-2.0658693314,-0.8160885572,0.6131067872,-0.6992746592,0.0867898762,-0.9291868210,0.3126922250,-1.0321285725,1.1907221079,-0.8579597473,0.3683812320,0.0951150134,1.4797778130,0.1138498932,-0.1657631844,2.1404304504,0.6909690499,1.0141973495,1.3627005816,0.9600053430,-0.2231254727,0.8661641479,0.6439171433,-0.3375835419,-1.4438884258,0.5471693873,0.2623979449,1.4910924435,0.1808727682,0.0957317352,1.1674259901,1.0773286819,-0.8375552893,-0.1509543806,-0.3889933527,1.2889469862,-1.6503235102,-0.6901810765,1.3719030619,0.2788748145,0.2398491651,-0.1003404036,0.2019098103,0.8351427317,-0.3547764421,0.0477411337,-0.2419873178,1.1113913059,-2.3828821182,-0.3928278089,0.2063228041,1.4378747940,0.6241122484,0.6654278040,-0.8381136656,-0.1943056434,0.8262708783,-0.8781173229,0.7365460396,-0.2867861390,1.0262185335,-1.0862541199,-2.2211563587],[-0.0351699404,-0.1053142026,1.0903381109,0.0490656979,0.6290349960,-0.1448164582,-1.0651451349,-0.2358892411,-1.5510635376,-2.0036041737,-0.8184807301,-0.7312480807,1.6621870995,0.0102839842,0.0925677046,1.7449682951,1.3801565170,-0.6567270160,0.5415236354,-0.2259770483,-0.6260111332,1.5308971405,1.0322775841,0.9775959849,0.1011903435,-0.1734205186,-2.5078294277,1.1471949816,-1.0667153597,-0.1465885788,-1.1242318153,-0.6100597382,-1.3902848959,-0.9408340454,-0.5747480989,-0.7050228119,0.2044974118,0.1679736227,-1.5893906355,1.6384850740,0.0945908651,-0.5734204650,-1.4962567091,0.8425856233,-0.7955751419,-0.7523640990,-1.3928989172,-0.6434774399,-0.0267408527,-1.3192676306,-1.1446229219,0.8909816146,0.7566283941,0.4858905971,-0.4653729498,0.2230161875,-0.3106475770,-0.9960952401,-1.9371960163,-0.0624998100,0.2280373722,-1.4546812773,0.7684928179,-1.3647917509,-2.4256286621,-0.8073800802,1.0723882914,-0.9055548310,0.8978029490,0.3257782757,0.6538295746,-0.2891353965,-0.1892136186,2.2979393005,1.4039591551,-1.9162375927,-1.5704979897,-1.2596937418,-0.5759724975,-2.3971576691,-0.5725220442,-0.8249366283,-0.3085795045,0.5204046965,-1.2902706861,2.2250847816,-1.3025299311,-0.5453331470,0.4720706940,-0.6122683883,-0.6175191402,1.1907383204,-0.6027181149,-0.1497769952,0.3475134373,0.0696816221,0.4372898340,0.6333782673,-0.1903668344,-0.5238707662,-0.2407862246,-1.7575255632,0.6175248027,0.4117534757,0.5725713968,-0.1537111849,-0.2685767412,-0.1302557737,0.6916129589,-1.7661845684,-1.2384231091,-1.5353415012,-0.8544256687,0.8948006630,-0.8968293071,0.4587621093,-0.6570677757,-0.6455622911,-0.3647587895,0.6582919955,0.0151019888,0.6143096685,-0.4849007130,-0.0224776696,0.8556644917,2.1596455574,2.0912363529,0.6864846349,1.9000794888,0.3535133600,-0.4887687564,0.2296750844,1.7741783857,0.8200288415,-2.3570518494,-0.6121400595,1.1244128942,-1.9312944412,0.2823550999,0.2266518772,2.0878908634,-1.7262382507,0.3788723648,1.0462868214,-1.2126829624,-1.5116736889,1.8529467583,-0.1404440552,-0.1386186481,-0.7274141908,-0.9291960597,-0.7300670147,0.5603982806,0.3851436377,-0.1989250630,3.1476488113,-1.4545503855,0.2226756215,0.6318311691,-0.5034347773,-2.0605225563,-0.2405045331,0.7324491143,0.2608508170,1.1732854843,0.3615217805,1.1910451651,1.1258134842,0.8236389160,-0.1661823541,2.0450708866,0.1467454135,-2.3809611797,0.4373816252,-1.4531396627,0.8588591218,-0.1570577621,1.0039044619,0.2207947969,1.0857644081,2.3349044323,-0.4538405836,-1.0556931496,-0.1135712489,-1.3338731527,2.0147054195,0.5743921399,-0.0406258553,1.0636233091,-0.7739616632,-0.4744435549,0.6629287601,0.6593925357,0.2389109284,0.3244543970,-2.4939632416,0.2376910746,0.2401821762,0.1714852750,-1.6680035591,-0.2254704535,0.1158540174,-1.5890207291,-0.0886749774,0.8084253669,0.2941642106,-0.2246614844,-0.6902972460,-0.9486091733,0.1864258349,1.3744177818,1.2856972218,0.0242699496,0.7165759802,-0.7738999128,-1.5016100407,1.3694467545,-0.5406073928,-0.5955885649,0.8310559988,0.7758463025,0.0332392007,0.0597205423,0.7009345889],[-0.1214630902,-1.2265552282,0.9551652670,-0.9825314283,-0.0588454418,1.3474606276,0.2749947906,0.7564658523,-0.9267882109,0.2939739227,-0.7862483263,0.3147085011,-1.0929533243,-0.4037205279,0.1726106852,-0.9854048491,0.8076828122,0.5334472060,-0.5711045861,0.2277940810,-0.3884009719,0.0958303660,0.4074799716,-0.1783480793,0.6015118957,-0.2043264210,0.6595416665,0.5929055214,0.2067304552,-0.4159007668,0.0456653237,-1.0317988396,-0.0439903848,0.2529656291,-0.5262223482,-0.0891434103,-0.5803508162,0.2372836471,0.1403294653,-1.1127064228,-0.7271862030,-0.3711755276,0.4906193912,-0.9802835584,-0.4931763113,-2.4230916500,-0.6911710501,0.2911610901,-0.3813099563,-0.9066234827,0.4782035351,0.0682695433,-1.2720994949,-2.0620915890,-2.0388426781,-0.8398178816,0.1875489354,-0.6239607930,0.1227937043,-0.8136839867,-0.4940828085,-0.2377815545,-1.1428968906,-1.3174897432,0.2686184347,0.2657438219,-0.0437186994,1.7247594595,-1.2954945564,-1.5489299297,-0.0903909281,-0.2665699422,0.8504773974,0.9000029564,0.1670958251,0.2365684360,-0.6508274674,0.6521868706,-1.2457232475,-0.4461270869,0.0367440879,0.8848665357,0.4277692735,0.4192968607,-1.8041940928,-0.8850283027,-1.4226913452,-0.1611105055,1.5754060745,-0.7368183732,0.8704988360,0.5925328135,0.3334204555,2.3063228130,0.5068302751,0.6421422362,0.2884857357,-0.6937939525,-0.4742089808,-0.5211635232,0.5846737623,-0.6871022582,-0.5160235167,0.4397167861,-0.1327747852,-0.3461024463,0.8904767632,-0.3413029611,-0.7286113501,-0.3106502593,-1.1513842344,-1.4070318937,1.4902381897,0.7819204926,-0.5711467862,0.9897950888,1.0421978235,0.0064553972,0.2257383913,-0.5662776232,-1.6784616709,0.1300082803,0.3907846808,-0.1842308342,-0.5939362049,0.4409166574,-0.4065026641,-0.2743496001,0.2813808620,0.3920689821,-1.1994462013,-0.4680491686,0.3003554046,0.8930982947,1.3606835604,-0.8178442717,-0.9329233766,0.8158513904,-0.2013928294,-0.7077201009,1.0480406284,1.7602568865,1.0767135620,-1.1549855471,-0.4318770170,0.2878724635,-0.1249971166,0.0026210824,-1.9303877354,0.2099825144,-1.8180750608,-0.5248698592,0.5573206544,-1.3221145868,2.1918184757,-0.1596627235,-0.7318202853,-0.3619192839,-0.2752758861,-1.1489162445,-1.0136946440,0.8739019036,-1.1305433512,-0.6433794498,2.3157281876,0.4867565036,0.2869711816,0.7111119628,-0.7375450134,-0.2039852291,-0.2768211663,1.3463346958,-0.3390847147,-0.6786736250,-0.7262747884,0.8902787566,0.9133289456,-0.2505047619,-0.0415848196,-0.3829526007,0.5145270228,0.5090532899,-0.8748589754,0.5908644795,-0.8097217083,-0.6209503412,-1.6769145727,-0.7063034773,-0.0019511768,-0.7479312420,-0.4937320054,1.2623711824,-0.0325878747,-2.1162991524,-1.3957357407,-0.4022807777,0.2463667393,-0.2883108854,-1.0714792013,1.3542736769,0.3475864828,1.1947330236,1.4123615026,0.4734445214,-0.0291386191,-0.7498828173,1.8264600039,0.8586798906,0.8935324550,-1.2798057795,-1.2434326410,1.4402679205,-0.3074516356,-0.3824082613,0.9012572169,1.3327422142,2.0704622269,-0.7559067011,-0.6273303628,-1.3138438463,-0.5460834503,-0.1313429028,0.3516915143,-1.2709006071],[0.2318837494,1.3678478003,-0.5695576668,0.9467203021,-1.0783817768,1.8729202747,-0.0971904844,0.0755688325,0.0776100382,2.0316021442,0.1603736281,0.5459352136,-0.9726063609,-1.1684632301,-0.6058312654,-0.8898193240,0.2162799835,0.2188819796,0.6266080737,-0.9264500737,-0.3930732012,1.2791380882,-1.4856559038,0.6038267612,-1.5345782042,-0.8310987353,-1.1575284004,-1.6151446104,-0.3164165914,1.6876577139,0.3008544147,-0.0813643858,2.1535699368,0.6083881855,2.0528795719,1.0854481459,-1.8040492535,-0.8503279090,-0.9169733524,-0.7187844515,-0.0188348703,0.9217076302,-1.1328121424,-1.0641452074,0.6557211876,1.6046416759,-0.8734141588,-1.1367805004,0.7530630827,-1.2990561724,-0.0882783160,0.0793868899,-0.1091432571,-1.5902625322,0.3879348934,0.0378575362,-1.3857095242,0.0273315851,2.5557720661,0.2602879107,1.6669652462,0.0515464433,0.2788645923,-0.4719827473,0.6808572412,-0.5266553164,1.5635540485,0.6531999111,0.5617042780,0.3659709394,1.2030426264,-0.3417057991,-0.3226774931,0.9697617888,1.1391057968,0.6558498740,0.2402107716,0.1402628124,0.0865132883,-1.1623040438,1.4959161282,0.4355273247,-1.5472781658,-0.1442041993,0.4357748032,0.3796449900,-0.1988835186,1.2006313801,-0.3266404867,0.4909732342,-0.1155063435,0.7805097103,-1.5917422771,0.2558749318,0.0028779260,-0.9973745346,0.3408148885,0.6907517910,1.7726098299,1.2931444645,0.0731726885,0.0033546537,-0.5160992742,0.9919461012,-2.0193617344,0.0540464595,-1.0469379425,1.8422071934,0.2026942223,-0.2223332375,-0.8785908818,-0.4768011868,-0.6508824229,1.4654546976,1.9984804392,-0.3018554151,0.5743958950,-0.9103906155,2.3126950264,1.1369340420,-1.1978269815,-0.5591164827,-1.3324406147,1.4774297476,0.4085949659,1.0875927210,1.2534742355,-0.2782524824,-1.2699588537,0.2338881344,-0.1154674813,1.3444168568,0.5542261004,0.5446771979,-0.7692145109,-0.8970652223,-0.4507069588,1.7428719997,-0.4185885191,-2.0615775585,0.5384839177,-0.1812524796,0.5966222882,-0.0884481892,-1.3890218735,-0.5953192115,-0.8984060884,0.2904509902,1.3895179033,-1.2992860079,0.3779320717,-0.0900823697,0.1684620082,-0.2983181477,0.7044838071,-0.3086249530,-0.4766749740,-1.6568388939,0.3046866953,1.1033426523,0.9387074709,0.5648691654,1.1102232933,-0.5370892882,1.6050131321,0.3298608363,0.8434912562,1.5994970798,0.3545402586,-0.9816116691,0.5793850422,1.8203120232,0.6507356763,-0.6160635948,-0.4756502509,-0.1226160377,-0.4512326717,0.9120926261,-0.0639616176,0.6028991938,-2.1917812824,-0.7067122459,-0.1262857169,1.8095263243,-0.6735818386,-0.0236892402,-1.2794020176,-0.3109898567,-0.4389147758,0.1079273149,1.0574131012,-0.3642730713,-0.5820379853,-0.0673111752,0.6866626143,-0.9802171588,-0.4072963893,1.5017372370,-1.3176370859,0.0456849113,0.6719582677,-0.6977844238,-2.4139885902,0.9166274071,1.9176933765,0.0757182539,0.2686379850,1.2910649776,-0.5725409389,0.0763535500,-2.0364468098,0.0009822916,1.2169647217,0.5885798931,-0.9567729235,0.0846289247,1.2598075867,-1.2678459883,1.2660022974,0.5127959251,-0.3129904270,-0.7242843509,-0.2958310246,0.2941564023],[0.3908843100,0.1993192434,0.8601083755,-0.0118097821,2.9144937992,0.5598602295,-0.7121825814,0.6126078963,-0.4042938352,0.2696480155,0.7432195544,-0.3382290304,-0.3175045848,-1.0074145794,-1.1684331894,0.0796425790,-0.4121067822,0.4740624726,1.3114098310,1.0700200796,0.1490948498,0.6387857795,-1.7708073854,-0.9917168021,-2.5573399067,0.6741188765,-1.3366583586,-0.5455180407,-0.2197612971,-1.4311057329,-0.6745821834,-1.5028071404,1.0440351963,0.7176719308,-2.6579515934,-1.0510629416,-1.2848021984,-1.3853023052,0.4791358113,2.5045385361,-0.4729622602,-1.2650476694,-0.7059856057,-1.0171307325,-0.7169947028,0.1002878100,-0.3277071118,-0.2671995163,0.5931706429,0.2614943087,-0.1734369695,0.0991713181,0.1256472766,-0.3000682294,0.7157314420,1.0789563656,0.2905321717,0.0915650576,-0.2459096760,0.9417273402,1.1695841551,-0.4629750252,0.6636813879,1.2860013247,0.1320593655,-0.4724427462,-2.4939298630,0.8686680198,0.7980304360,0.0212606862,0.0895598605,-0.6142240763,0.2135329247,1.5683747530,-1.3909196854,1.9122843742,-0.1695288122,1.0792137384,0.2949759364,2.9196517467,0.1250986755,-0.9956979752,0.8259578943,-0.4130439162,-1.5986818075,-1.3872983456,-0.3145367205,0.1179488599,-1.7167025805,-0.3398543596,-0.2405494601,0.1653799564,2.2529299259,0.1527817100,-0.0522611253,1.6368466616,1.5746954679,-0.6853815913,0.7834398150,-0.4030088484,-0.9495356679,1.5865054131,0.0136783160,-1.1354285479,1.2099617720,-0.7505401969,-0.3670379817,-0.9019443393,-0.0718230233,0.3455559909,-1.0303230286,1.2956593037,-1.3617696762,0.6880182028,-1.8390884399,1.1746094227,0.2042525560,1.8721073866,-0.2429490089,-0.9037572145,0.4598532319,1.4084200859,1.4145408869,1.2590812445,0.4491093755,-0.2917197943,-0.1804877669,0.3747444153,-1.2223923206,-0.0042179539,-0.0286030844,-2.1629972458,0.8808540702,-0.4634902477,-0.4097259641,0.1188818812,0.8841651082,-0.7411962748,1.3089855909,0.2433037907,0.2072167397,0.0030417852,-0.4322900772,0.0580277666,-0.8421820402,-1.1227341890,-0.4095312059,-0.6351722479,-0.9240295887,0.1515339166,-2.7080621719,-0.8268998861,0.6793233752,0.0793776214,1.5474833250,0.0869547799,0.4061647654,-0.1170388460,0.7532392740,0.2103636712,0.2250218242,-0.3768270612,0.3614977598,0.9480288625,-1.6463083029,0.4892472327,-1.7122375965,0.5919700861,-1.8297494650,-0.3791226149,-0.4474929571,-1.0941311121,-0.6403060555,0.1746183336,-1.7660150528,0.0863045081,-0.5001140833,0.6139918566,1.6994225979,0.1999011487,-1.9983192682,-0.7136126161,1.4435380697,-1.1255981922,1.4526222944,0.5327004194,0.0478152595,-0.1422421634,1.5422103405,-0.4843623638,0.5077458024,-0.4235121012,0.4101985395,0.2985644937,-0.4474204183,-1.4437913895,0.0946096405,-1.0242272615,0.3150271475,-2.1535305977,-0.0214283168,2.2918503284,1.6116689444,-2.0501370430,0.4185002446,-1.2304670811,0.5945562720,-0.4853458703,1.2488714457,0.2259249687,-0.1908799410,0.0583935194,0.4459559619,0.4773775935,1.6011041403,0.7079492211,0.4374597669,-0.5841044784,0.9688305259,0.8779850006,-0.0789694786,-0.1578722596,-1.8777650595,0.0557074770],[0.2137283534,-0.0753122121,-0.5718395114,-0.6800060272,1.3663361073,-0.3309047520,0.4557212293,0.4449327290,0.9501424432,1.0727024078,-0.9129831791,0.7791337371,0.8639063239,-0.7943357229,0.9940410256,-0.1175013930,1.6619648933,1.3453713655,0.1633823812,-0.1596832424,1.5186958313,0.9925870895,0.7669295073,-0.2227122933,0.9052174687,0.1742046773,-1.8046705723,-0.3167853653,-0.4060903490,-0.3562828302,0.9901017547,0.5220876336,1.0828599930,-0.9335242510,-0.8265603781,-0.5601838231,-0.1037390828,-0.0347839370,0.8033310771,-0.7626441717,0.1769310981,-0.8562979102,0.6401655078,0.4897463024,1.2890980244,0.3583827019,1.1658328772,-0.0718992352,-0.8952594399,-1.3356628418,-0.9950582385,-0.6626952887,1.5818070173,-1.6137483120,1.6730548143,1.0554708242,-0.7988551855,0.1265448183,1.4124367237,-0.4724154770,-0.9337536097,0.8365861177,2.3900251389,0.8229386210,-1.1517981291,0.6948511600,-0.1482112110,-0.5633614659,-0.3132559359,-0.1994884610,-0.8831216693,-0.1983086616,-1.2630894184,0.5556268096,0.9699519873,-0.1344390661,0.5338910818,-0.8431673050,0.2585937381,-0.5014234185,-0.9196692109,0.6794525981,-0.4677133858,-0.3580030501,-1.0047782660,-0.4246050119,-0.8890936971,-0.1875797808,-1.3680918217,2.4937102795,-0.2065069526,-0.3412998319,0.9275464416,-0.6399300098,-1.0326942205,-1.2631584406,-0.7057926655,0.2132282406,0.1349920183,-0.0767279118,-0.0180238225,0.9167872071,1.2965246439,-0.4996848106,-0.1523783654,1.3595687151,0.5616595745,-1.2922233343,-0.2067446113,-0.0955479667,1.0408117771,0.2260160744,1.5209286213,0.7850476503,1.2328828573,0.4577864110,-1.0894739628,-0.5421277285,1.8188209534,-0.4353050292,1.0453416109,0.9334639907,-0.8626065254,1.1428796053,0.4184126854,0.8542534113,0.2786234617,2.2667417526,-1.3422918320,0.0365474224,-0.6399766803,0.2765336931,0.6240792274,0.9419114590,1.2190315723,0.9540512562,1.8149379492,-0.5880147219,0.5229048133,-1.8234910965,0.5798783898,-0.4019498825,-2.4706156254,2.2076125145,0.0733084604,-0.8490743041,-0.1897683442,-0.0908034369,1.3684400320,0.3928494453,-1.5093868971,-0.9352346063,-1.6818646193,-0.3056127131,-0.9864490628,-0.1267329901,-3.4295580387,-0.8379977345,-0.2364562154,-0.0146869794,0.2902716100,-0.3721041679,-0.5221943855,0.1014696434,-0.0193115585,0.1473674327,-1.5494464636,-0.5399037600,-0.0246628616,-2.0717842579,1.0729697943,-1.1848657131,0.4384807050,-1.3108595610,-0.8152757883,-0.1412737817,1.5141129494,-0.1638685614,0.8585687280,-1.1072340012,-0.8543540239,1.0696147680,0.1962230355,-0.1166522354,0.4228607118,0.3876480758,-0.2862882912,-0.9716600776,-1.0824446678,-0.5052465796,-0.0500417948,1.0491191149,0.8568240404,-0.6495133638,-0.3671525717,0.0186419897,-1.5158751011,-0.0458600409,-0.4317787886,-0.0550946295,-0.8744986057,-0.1637722999,-1.1921287775,-0.5725089312,1.6396950483,-0.1935610920,-1.8141798973,0.5524504185,0.0224844627,1.7025898695,-1.1939163208,0.0611830056,0.4557733834,-1.8053938150,-0.1088439003,1.3869347572,-1.3557333946,-2.6577615738,-0.5621011853,-1.5449957848,-1.4346992970,1.6745887995,-0.7087044120,-1.1405794621],[0.8468211293,-0.6259702444,-1.0013346672,0.8264898062,-0.0107424371,-0.0377501473,-1.6358803511,0.1592251062,-1.2914931774,0.4267860949,-0.0591404252,-0.3762255311,0.7003886104,0.3074934483,2.0389971733,1.0119352341,-0.8226264715,0.1964271665,0.6776129007,-1.9366357327,0.2702591717,-0.7943894267,-0.5161536932,-0.9527539611,-0.1551645100,-0.3698361218,0.5130124688,1.6964564323,-0.4065807760,-1.2760372162,1.2567688227,0.0966834277,0.5076339245,0.3789902031,0.8715313077,1.5521655083,-2.2539381981,-1.0728610754,1.2374900579,0.6128914356,-1.1944071054,1.9067157507,1.6737689972,1.1818349361,0.3899155259,0.4014538825,-0.5103981495,0.1652175784,-1.9009075165,-0.2527178824,2.0375125408,1.6334018707,0.0990877450,-0.2214143127,-1.2417927980,1.4322711229,-0.8110809922,0.6237356067,-0.8600297570,0.9621957541,-0.1469984949,-0.7951014042,-1.2351981401,0.3188866675,-1.0440903902,-0.9646935463,-0.4229103625,0.4964722693,0.1531331837,2.7971539497,-0.3278644085,0.5300644636,-1.1382143497,1.2793962955,0.2531116009,0.8383103013,-2.0147705078,0.6147590876,-0.2501372099,-1.6798156500,-1.1679533720,1.9104994535,0.7335022688,-0.6242956519,-0.1997241825,0.6580860019,-3.5438032150,2.2302262783,0.2623988092,0.4263668358,-0.5162287951,0.9116119742,0.7537800670,0.8147564530,2.0275216103,-1.0469170809,0.0447648279,0.4066671729,1.4574295282,-0.5511431098,0.1064528897,2.2086167336,1.0268939734,-0.6577744484,-0.1552726179,-0.0653077587,0.8309588432,-0.2720114291,-0.4248123765,0.4521262944,1.8212699890,-0.2710549235,-2.6737802029,0.8627644777,0.4140576720,0.7305349112,-1.1009577513,-0.0405872762,-1.7268170118,-0.9778028727,-0.3233233988,-1.1608653069,0.0106323780,0.0213840399,0.5337206125,-0.5303680301,-1.8097007275,1.0664070845,-0.0200915802,-0.8092259169,-0.6417803168,-0.2247069329,0.2727496624,-0.0173603650,-1.0712687969,0.2251258940,0.0471394435,-0.0443964601,0.7301927209,0.3327662051,-0.1669893563,-0.0900482908,-0.1879023314,-1.1513732672,-0.3616277277,-1.6583201885,-0.1897500455,-2.0509023666,1.4391367435,-0.3599698842,-0.6990902424,-1.3491034508,-1.0897932053,-2.0706076622,0.7112004757,0.0375067219,-0.6403739452,0.4059659839,0.6470164061,1.8503149748,-1.6849238873,-1.3065844774,1.0666915178,0.9946438670,-0.6309731603,0.2481388897,-0.6018763781,-2.1274530888,-0.4725838006,1.3387097120,-0.3620197475,0.4139246047,0.5800728798,0.0942340717,-0.3209901452,0.0889385194,1.1412287951,-2.3286304474,-2.0180230141,-0.7359105945,0.4361290336,-0.3935990930,-0.9925333858,0.5121065974,-1.0553890467,0.0176953413,-2.1810755730,0.4622453153,-0.2540527582,-0.7596648932,-0.0581332184,-1.7947191000,1.0270496607,-1.2321225405,-0.3689308763,0.6238868833,0.1727206409,-2.8847465515,0.2620530725,0.9489654303,-0.3782636225,-0.2526799142,1.0940079689,1.0400370359,-0.4539102912,1.4904272556,0.3527225256,1.2626172304,-0.8670571446,-0.3333664834,-0.3730029464,0.2025904655,-0.3326246440,-1.2971646786,0.9655994773,-0.1379029453,0.7102864981,0.8918810487,-0.9970547557,-1.4230805635,0.8162118793,0.7776215672,0.1969143003,-1.3174074888],[-0.5844110250,2.4673445225,-0.5167017579,0.6806137562,-1.1483550072,-1.0401893854,1.7456251383,-1.6846861839,-1.0622183084,-0.0755550563,0.6051344872,1.4599839449,1.8236228228,0.0921196342,0.3357533216,-0.5934019685,-0.3730606437,-0.0822857097,0.6652532220,0.9473050237,0.1076403409,-0.1182667017,-1.6469464302,1.0899176598,-0.7156873941,0.5861592293,-0.8050276041,-1.7994321585,-0.5858718157,1.1953302622,0.8592127562,0.5409115553,0.5861743689,0.9212389588,-0.5880656838,-1.4463483095,-0.1815917939,-0.4911633432,0.5372625589,0.7450032830,-0.7984898686,1.0677664280,0.8281040192,-1.9189672470,1.7005119324,0.1223435998,-0.8320993185,0.2740614414,0.7001248598,-0.1688963920,0.5390168428,0.7814664841,-0.6030362248,2.4221360683,1.2155414820,0.1503422856,1.1967055798,1.4468495846,-0.4375343025,-1.6593545675,-0.2992757857,-1.3680181503,-0.2705932260,-0.8648747206,-1.6523784399,0.6376203895,-0.4645305276,-1.8024930954,1.7780036926,0.6793705225,-0.3502464890,-0.3452021778,0.7236866355,-0.0813378990,0.4103395045,1.2789365053,-0.8342955112,0.2764317095,0.1251170337,-2.4885394573,2.0236468315,0.4303716719,-1.5654964447,-0.2343400568,-0.6242378950,-0.0403054543,-1.2115743160,-0.1085954681,-1.2910075188,-1.7384176254,-2.4113729000,-0.0978536010,0.6314791441,2.3066551685,-0.3984035254,1.6925071478,-1.2291654348,0.4835025668,0.5373200178,-0.6602330804,-0.3534172177,-0.8657686710,-0.0444904864,0.1402515024,-1.3653799295,-1.6854676008,-0.6546943188,0.7114022970,0.3269849718,0.7538878918,0.8704966307,-0.2834851146,-0.3685553670,-2.0719401836,2.0501718521,1.5302041769,0.1144342571,0.0451931544,-0.3070267141,-0.4794501960,-0.6803899407,1.5648218393,0.6181089282,-0.7039632797,1.2172878981,-1.4121437073,0.5843098164,-0.1053195447,0.5416721106,0.3049403727,-0.6772118211,-0.9164791107,1.6576575041,0.3724136651,0.6646793485,0.4738456309,-0.6636233330,-1.9127037525,0.4893687069,-1.0458182096,1.0213255882,1.1951941252,-0.0156067144,-0.6898881197,-0.3947982192,-0.3666029871,-1.5148080587,-0.8758003116,0.0948660448,-2.5608656406,0.7091881037,-1.9902096987,0.2046894580,0.7916052938,-1.7560980320,0.3696931303,-0.7017374039,-2.2834138870,0.0468963534,1.3962758780,-1.0620392561,1.0181268454,-1.5523796082,0.8203259706,-0.3706235588,0.2112499028,-0.8861809373,0.9715576172,0.0313248895,0.3653339744,0.0832739323,-0.4625202119,0.3381362259,-0.2845149934,0.0662649721,-0.9863808751,0.0047100959,-0.0938413143,1.2307792902,-1.4328401089,-0.2935240567,0.2290964574,0.8445306420,-0.2453528792,-0.2585487962,1.1594229937,1.3921138048,-0.6235087514,0.7814606428,-0.7613408566,0.0034697903,-0.2389897704,-0.3329078257,-0.2609840333,-0.2696022391,-0.9208405614,0.2603013515,0.9995867610,0.0641251057,1.4203132391,0.7948572636,-0.6019089222,2.1973364353,0.9349791408,0.3494509459,0.1695573479,1.1671073437,0.6311008334,-0.5451691747,0.0396844149,-0.1348823607,-0.7637632489,0.9957833290,0.3538493514,-0.9998304844,-0.3069563210,0.0764340535,1.2341115475,-0.4459015131,-0.8853935599,-0.1158692837,0.3361707628,0.5471341014,-0.7158341408],[-0.7225044966,1.5734320879,1.5906124115,2.3091294765,0.5050844550,-1.6723904610,0.1712281853,-0.2585093677,1.0001472235,0.5437983274,-0.0820056647,-0.0909900889,0.0230511352,0.7223117948,-0.7696962953,-1.0347841978,-1.2189891338,-1.9741156101,0.9746890068,1.5432999134,-0.9736856222,0.4056926370,0.4569702744,0.8798183203,-1.0623204708,1.8637923002,0.3763478100,-0.1089622602,-2.6937398911,-0.3388677537,-0.3510338366,1.0932720900,-0.4909926355,-1.4203675985,-0.1821767986,-1.2154457569,-0.7117774487,0.4508901238,-0.7958429456,0.2701161504,-2.1759605408,-0.6888856292,0.1048809215,0.3557719886,1.4286407232,-0.9892821908,-0.9702765942,-0.3957818449,-1.2025412321,-1.6958194971,-0.4899291694,1.2200242281,0.2535707057,0.3699500263,0.1447062790,-1.5934619904,-1.1716135740,1.2759325504,-0.8569272757,-0.6799893379,0.8412258029,-0.3783779144,0.6456711292,0.7831376195,0.2905963957,-1.3606551886,-0.8964964747,-1.4888651371,0.7162798047,1.0289067030,-0.8015437126,-0.1136556342,-0.2551667094,-0.6595488787,-0.2111678421,1.4372746944,-1.0267357826,-0.5512586236,-0.8835692406,1.0588902235,-0.1691202819,0.4261890650,-0.9193620682,2.5551671982,-0.7390261292,0.8471983075,-0.4127860069,0.3208262622,-0.6716014147,0.8648295999,0.0461724885,-0.8361611366,0.4000117183,1.2253652811,-0.8352629542,-0.4399746954,0.2978938818,-0.6611793637,0.4268187284,-0.3053519130,0.8972523808,-0.3825258315,-0.2013663799,-0.3409064114,-1.4009368420,-0.0572384521,-0.8031398058,1.5172399282,0.6763442755,1.4432848692,-0.2264365107,-1.4626077414,-0.1477045119,-0.3010220528,-0.4390170574,1.0616732836,-1.6102832556,-0.2703692913,0.1356078684,0.2474282086,1.6711789370,-0.1386670470,-1.0705966949,-1.0397030115,-1.6174699068,-0.1875222772,0.3125074208,0.7058945894,1.6676181555,-0.8247151375,-2.6883125305,-0.0254399497,0.2976902723,0.7683712244,-0.2267342806,-0.0494219884,-0.6753277779,0.7129440308,0.3559135199,-0.3811956644,0.9569373131,2.0430958271,1.7159265280,-2.7088499069,0.4763430059,0.6436210275,-0.3214740455,-0.0530350506,0.8985423446,-0.5783354640,-0.6321326494,1.0727838278,-0.5652558804,1.9493138790,0.1443732232,0.7849040031,-0.9399734735,-0.0713980570,0.8070620894,1.4234772921,0.6374298930,-0.2836876512,1.9719129801,0.2325410694,0.8821958899,0.3814108372,-0.2397420704,0.1884869784,1.9887346029,0.9581700563,0.6938206553,0.8666102886,-1.2881469727,-2.5677328110,-1.0640289783,0.3348835409,-0.0858363882,0.4353652298,0.1455067396,-1.4057297707,0.2277472913,-1.1727318764,-0.6120919585,1.2771542072,-0.8003570437,0.6429291368,1.9732037783,-0.7130243778,-0.1332945228,-0.2530682981,-0.2103462666,1.7769855261,0.3102103472,0.1589248031,0.8054945469,1.4725772142,0.6171528697,-2.3603472710,0.2448849529,-0.0658212975,-0.4911062121,0.3746935725,-0.7771610618,0.5732238293,-0.2027808279,-1.3332324028,1.3494966030,-0.3272345364,1.5416948795,0.3958894610,-0.5925759077,2.2976679802,-1.9548562765,0.7169862390,2.1682713032,1.2674481869,0.0668742955,1.0733947754,0.7876006365,-0.4546663463,0.0326386094,0.0436380059,-1.1151885986,-0.2926286459],[-0.2934510708,1.1921510696,1.5980594158,-0.8130311966,1.1409503222,1.3313578367,0.8444764614,-1.2648280859,0.9731772542,-1.1700046062,1.0548166037,-0.5340769291,-1.8655465841,0.1589542329,1.6472337246,-1.1182059050,1.2130138874,1.1797075272,0.5020463467,-0.7686650753,-0.1314456314,-0.7434322238,-1.5818883181,-0.6749847531,-0.0076208692,0.7187939286,-0.5591735244,-0.6265111566,-1.3190778494,1.0906132460,-1.0671056509,0.0342655107,-0.4741897285,-1.1158161163,-0.2803910077,0.5985125899,0.5886665583,-0.1324573904,-0.8394958973,0.0530796871,1.2963345051,0.0079052718,-0.9098715186,1.1129853725,-1.6644388437,-0.6847685575,1.1993128061,2.5089540482,-0.3476056159,-1.9554597139,0.5798164010,-1.9721982479,1.0853871107,-0.3682597578,2.2866818905,1.1455234289,-0.0026167410,1.1169599295,1.3755278587,0.5090430379,-0.6836943626,-0.4428147078,-0.0736249387,1.4757091999,-0.8413479924,-1.1724884510,-0.7022691369,-0.3636963964,-1.0440857410,-1.0499132872,-0.9251657128,0.4470546544,2.1343755722,-1.0937464237,-0.3145466745,0.0791752636,1.8015995026,-1.1708307266,-0.3710770905,0.0021474429,-1.2492738962,1.2540581226,-0.3887788653,-0.0979342535,0.6381052732,0.5091582537,-1.3069186211,-3.2205343246,-1.6814134121,-0.2490135878,0.6927216053,0.3460028768,0.4161698520,1.7263311148,1.3670994043,-1.2537178993,0.8223969340,0.1540791988,0.4974704087,-1.4824185371,-1.3710247278,1.9001389742,2.0547459126,0.4271369278,-0.3725138009,-1.3944122791,0.2006481439,-0.2336889803,-0.9436457157,-0.6381720901,0.3321398199,-0.2137336731,-0.6376451850,-2.6190822124,-1.4501197338,1.0656617880,0.6423869729,0.3586390316,0.0616030321,-1.3555532694,-0.6935083270,-1.4561247826,-0.6963708401,-1.9475837946,-0.1860286593,-0.9210773110,0.7253800035,0.7981371284,0.4483519793,1.6668821573,0.1371351928,1.3965914249,1.5389333963,-0.6738855243,0.7440813780,1.6378698349,2.2377979755,-1.6630915403,0.6449002028,0.0226498768,-1.0483651161,-2.1157052517,0.7034059167,1.0860549212,0.6332697272,0.5163491964,-0.7699266076,1.0532444715,-0.2817136347,-0.1954266131,0.6466432810,-1.1061758995,0.4786778092,0.6414473653,1.2876604795,0.5732532144,0.3575452566,-2.2497246265,1.2791950703,0.5004061460,-1.9408650398,-0.5976858735,0.3575026095,0.2238463461,1.0250608921,1.5798437595,-0.7968079448,-1.5439547300,1.2245321274,-1.3009462357,-1.3025891781,1.0694528818,-0.6745738983,0.0826234743,-0.1688699126,1.3309227228,1.3732751608,0.7255052924,0.2665914893,1.3750485182,0.0715328604,-1.1448044777,1.1675745249,0.2949744761,-1.1843786240,0.1423856020,-0.6588349342,1.2551316023,-0.1594646573,0.6699225903,0.6179960966,-1.7483422756,1.3339719772,0.3029723167,-1.1223556995,-0.4683272839,-1.6160479784,-0.3782319129,-0.7951239347,1.4735752344,1.6484818459,1.9880499840,-0.1242350116,0.1590626389,-1.8769822121,-1.0277003050,0.3765939772,2.0874798298,-0.9381197691,-0.2639953792,-1.9528026581,-0.3390138745,-1.9990794659,-0.6516773105,0.9337078929,0.6412097812,0.4076411426,0.0087392740,-2.6370751858,-1.2452684641,-0.3792372048,2.0947167873,1.2018176317,0.9371799231],[0.2011674345,-1.8033559322,0.5050754547,-0.5637676120,0.0107321963,0.6286424398,0.0492702387,-1.6498469114,0.0104927039,-0.1102788448,1.2789262533,-0.2689329088,-0.9935606718,-0.8252071738,1.4261518717,-0.1880145818,-0.9265387058,0.4540956914,-1.2168729305,1.8058356047,-0.1168055758,0.5073953867,0.3392245173,0.7426418662,0.5837747455,-0.7182095647,0.3108665347,-0.0449238829,-0.6104008555,0.5157542229,-0.9972209930,1.6061866283,-0.2779401541,1.4404076338,2.3598353863,-0.0747601092,0.8856165409,-0.4305108786,2.0704896450,-1.2632654905,-0.1376118213,0.3399257362,0.4172295034,0.0091022998,-0.5716640353,1.1888803244,0.1341378689,0.7014256120,-0.5142672062,0.2883620858,-1.7981014252,-1.5133697987,0.6216871142,0.1071367636,0.4802518785,0.7183364630,-0.5698022246,0.8356449604,1.3484818935,-0.5819205046,0.2602949739,-0.1543977708,1.7979315519,0.4334428012,0.3003485203,-0.4186342061,-1.8175272942,0.1426157504,0.2628866732,0.3613359332,-0.2564287484,-1.0144606829,-0.8692834973,0.2546459138,0.5684943199,1.2995219231,-1.7172603607,-0.2453630418,-1.2629826069,0.7414638996,0.2217936367,-0.9192535281,-1.1625478268,-0.7427448630,0.6232370138,1.4456577301,-1.3260273933,-1.3074417114,0.3686677217,-1.7282638550,0.1614782363,-0.8077552915,-0.5116407275,0.9091966748,-0.8138747811,0.8174273968,-0.2035060823,0.1387581825,-1.0483995676,-1.1789760590,-0.1418992430,2.0072953701,0.2946895659,-0.1563377231,1.9827004671,-0.5443348885,-0.6208754182,-0.0153734796,-1.0330175161,1.5375193357,2.0169827938,-0.8606604338,0.7027775645,-1.1580990553,-0.8982186317,0.4647661448,0.1343112588,-0.4590297639,0.1715912819,1.5083426237,0.0586618483,0.7613887191,-0.6067987680,-1.7963507175,1.2548172474,1.1760858297,1.3762103319,-0.8332884312,1.9168746471,1.2767934799,1.7126957178,-0.8065787554,-1.2705796957,0.5567392111,0.6912044287,-0.1580438316,2.1341280937,-0.5214887261,0.4928900898,0.4675154984,-0.1240027398,-0.6425836682,-0.0069333049,-0.6486710906,-0.6996839643,1.3014633656,0.2614509761,-0.2373386174,-1.3424488306,-0.4017966986,2.2234640121,0.6983535886,-2.3241012096,-0.9949077964,-0.2743484080,0.5673701167,-0.8014136553,-1.0267834663,-0.3481470346,0.2476348877,0.0547130778,1.3874765635,0.7338072658,-0.3730239868,-0.4653834701,-0.6865920424,0.3276156187,-0.5956894159,-0.0898129418,-0.8099248409,-0.9437165260,-0.1692812443,-1.0413445234,-1.7066810131,-2.0119111538,1.3718327284,0.2448530644,-3.1781678200,1.8929358721,-0.2453606278,0.7510632873,-0.2731828988,-0.0230197310,-0.6737012267,0.8937125206,-0.3046015501,0.3742146790,-0.1818317771,-0.4224560559,0.4293473661,0.0446226597,0.2028374076,0.6543111205,2.0454571247,-1.4786034822,0.3524400592,-0.4629359841,-0.3411388695,1.7129424810,0.4731816351,1.0813529491,1.3048049212,-2.1584177017,0.0670639798,-0.5799887776,-0.0091375699,-0.3846576512,1.0352602005,1.5867995024,0.9555609226,0.3972346187,0.1691245884,0.7903192043,-0.7628286481,0.7735763192,3.3371245861,0.5317189693,-1.3681658506,2.1529917717,0.4164772034,-0.4630190432,0.3562625945,-1.3067346811,2.7096297741],[1.1794174910,1.2390013933,-0.3905018866,-1.2902466059,-0.6624063849,1.3344436884,-0.6159383059,-1.1763250828,0.3439395428,-0.6466949582,1.2733309269,0.5567304492,0.1190240979,0.4002852142,-0.1299002916,-1.4366146326,0.5557325482,-0.3819403052,-2.2948584557,0.7538915277,0.4088020027,-0.5706181526,0.9400187731,-2.2233648300,0.2392009497,-1.4521886110,1.4968353510,-1.1174561977,2.2601320744,-1.8304297924,1.3039175272,-0.0048594456,0.7367036343,-0.3838936687,0.5286387205,0.1551837325,-0.4323429465,-0.6561080813,0.5545196533,0.5781207681,-0.3996987045,2.3530459404,-1.3647236824,-0.1340306848,0.1753463745,0.0249666255,-0.3483884931,-0.0333038643,-1.9605727196,-1.5999488831,-1.3416832685,0.2788104117,-0.7476895452,0.8272824883,0.9861788750,-0.6861533523,0.1893069595,-1.6422375441,0.5166320205,-0.3142879903,2.8894968033,-0.3574573696,-0.7443041205,1.9324642420,-0.6711962223,0.8798464537,-1.0091282129,-0.5488491654,1.5701944828,0.2705782056,0.8730167150,0.9876908660,-0.0415039994,-0.6048167348,0.2426076084,0.1948920041,-2.5501427650,-1.7648794651,-0.0415776409,1.1606342793,0.3051763773,1.4745252132,-0.3646067381,0.1290424615,-0.2389613539,-2.2449805737,1.3227287531,-0.3557299078,-0.2465161532,1.0724699497,-0.0132042440,-0.6840516329,-0.4048824012,1.0555580854,0.2012488693,1.4333781004,0.5459110141,1.4253919125,0.1036763117,-0.8666305542,-2.1211733818,-1.3604879379,1.2255884409,1.0984667540,-0.2646973431,-1.2453637123,1.2485244274,-0.4639970064,-0.1485658139,0.4927731454,-1.1315159798,0.6742309332,0.1616810262,-0.2924068868,-2.2380015850,1.5050231218,-1.1828123331,-1.0715100765,0.2920324206,0.4176152349,-0.2618497610,0.3900544941,1.2369613647,2.2400150299,-0.6591055989,-0.3432904482,-0.9165864587,0.3899152577,-0.8128656149,-0.6911546588,-1.8697581291,-1.9872611761,-0.8085523844,-0.0751557872,0.2399503887,-1.3846840858,-0.1106751859,1.8108611107,0.2479185611,-2.0952146053,-0.0888442099,-0.1570831537,0.0162983797,-0.7323496342,0.9778578877,-0.4015278518,1.5470517874,-1.1965001822,0.5501039028,-0.3973948061,-0.5402927995,0.1135616377,-1.1342197657,1.5787736177,0.7312040925,1.3263958693,-0.9830656648,-1.5166749954,1.2743954659,-1.6303781271,0.5302878618,0.4884847105,-0.0564946122,1.4196146727,0.0485945493,-1.6874896288,0.0264888797,-2.2721157074,0.5597230196,0.0342372134,0.3025023639,-1.1467289925,-0.1418615282,0.4835216701,-0.4516725242,0.2854830325,-1.2665178776,1.5576566458,1.0947488546,0.2143646628,-0.2931555212,-0.6761689186,-0.5095722675,0.1104367077,-0.9266905785,0.1388486475,-1.7587862015,1.1457693577,-1.7809176445,1.7472233772,0.2431917340,0.9257518053,-0.1105109155,0.7692781687,0.8367552757,0.7457382679,-1.5059551001,-0.8665480614,-0.8388282657,-1.1409242153,-0.6999360919,-0.3496262133,0.2825672925,-0.7251719236,0.3119471371,0.6827612519,0.8235883117,-1.2512964010,0.2607354224,-1.1948734522,1.6742203236,0.1868904084,0.8913527727,-0.0905617997,-0.4602533579,-1.3090718985,0.2370790094,-0.5549578071,-0.3860284686,-0.3769266605,-0.3289939761,1.2747739553,-1.1576405764,-0.7707039118],[-1.1006671190,-0.0534144416,-1.0460067987,-0.0220385697,-0.0187210329,1.0596003532,0.4855110049,0.9563537836,0.6240862012,-0.2181169987,-0.1297433525,1.3240571022,-1.3595471382,-0.0788813084,-0.4473232031,-0.8702023029,1.2506643534,0.9834125638,-1.0927263498,-0.6073045135,0.3495458961,0.1213660464,-0.4297527075,-0.3331126571,0.7537350059,-1.9193907976,-0.8438121080,-1.9194872379,-0.2119607925,1.8239436150,-0.2754066885,-0.5554115176,-1.0301826000,0.7643379569,-0.0227805395,1.3933587074,-0.1937872171,0.9693816304,0.6298752427,0.2584379315,-2.3164453506,-1.8396610022,-0.3373851478,-0.4343889356,0.5835508108,0.5079726577,0.7551559806,0.8957632184,-0.5572155714,1.4874306917,-0.5413039923,1.7688095570,-0.1760699153,-0.4086333811,-0.0682857335,1.2715420723,1.3316938877,-0.6478120089,1.7098898888,-0.2466237396,-1.8732783794,0.1492802203,0.8813462257,-1.7876430750,-1.2078424692,-0.5233929157,-1.0958852768,0.1002101004,1.2844595909,1.1891831160,-1.2014337778,-2.2963385582,0.9300718904,0.7659554482,0.7081581950,-1.4520958662,1.1127053499,0.2289859056,-0.7110806108,0.4880263507,1.5001052618,1.5714887381,0.7933399081,-0.3028346896,-0.1017167792,0.6995674968,-0.0198632907,-0.7587066293,0.7734092474,-0.3478929996,0.0377644747,-0.6838935614,-0.7128896117,0.5029635429,1.9088362455,-1.1404583454,-1.6888059378,-1.8423115015,-0.0389658883,1.1948564053,2.4157738686,-0.9063307643,0.6310094595,0.9897720814,0.7520170212,-0.4465702176,-1.6844987869,-1.9101190567,-2.3421952724,-0.4606425762,-0.5885396004,-1.0535298586,0.6714470387,-2.0614287853,-1.5895788670,-0.9668141007,-1.8850803375,1.0163460970,-0.4743680656,0.9278936982,-0.6074044108,0.8148276806,-0.1381165832,1.0583986044,-1.5783662796,0.7843630314,0.3421233296,-0.8151394129,0.8304324150,-0.5892474651,0.0996279344,-0.1991706789,-2.5094156265,-1.5909996033,0.2353384942,0.6018269062,-0.8568947911,0.2557218075,0.6682499051,-0.1879249215,0.0074428706,0.4798038006,-1.7355819941,1.9003592730,1.5346397161,-0.6947375536,-0.6060379148,1.2424333096,-0.4308054745,0.6093084812,-1.5997244120,-1.5050390959,-0.3377222419,-1.2019016743,1.1436249018,0.9114562869,0.1259697229,0.4765327871,0.3271493912,1.5688660145,-0.0834053159,0.6043325067,0.0358087867,-2.0696275234,1.1063140631,0.7426227927,-1.2348821163,-0.2855467498,-0.1477694511,2.1614141464,-0.7326849699,0.5784502625,-0.1212657541,-0.6860406399,0.2312044650,0.7677243948,1.4630963802,-1.4831662178,0.3842955828,0.3330720067,-1.4914379120,-0.6227321029,-0.6727771759,0.4612464607,2.0365641117,-0.8002539277,-1.1648292542,0.1289562881,0.4260367751,-1.7791209221,0.3649482727,-0.7779896259,-2.4400405884,-0.0435852557,-0.3500059247,1.1347149611,0.3358536363,-1.8796350956,-0.6165884733,-0.3348190784,-0.5539978147,-0.6812824011,-0.5439455509,-1.9715577364,0.3464953303,0.8795771003,0.1512504965,0.0113260550,-0.7210302949,0.4954958558,2.0371608734,-1.3999308348,2.0279750824,1.0738673210,0.7125077844,-1.7527254820,1.9098694324,-0.4230732620,-1.3026645184,0.4003872573,0.2172527611,2.9619445801,-0.8010261655,1.4232937098],[2.3403263092,1.3790491819,0.1906539947,1.5853165388,0.1980482191,0.5038337111,1.2534674406,1.1705026627,-0.9915919900,0.7790826559,0.5936439633,-0.7537182570,-0.6383305788,1.0891114473,-0.5446549654,-0.2027605176,-1.1356508732,0.8898881078,-1.1803863049,0.0760592073,0.3940411508,0.8132078648,-1.2225940228,-0.2539710402,0.9250649810,-0.4512603581,-0.0236611646,0.2419966459,0.5349957347,-0.2259923965,-0.7846670747,0.4239006341,-0.0066497908,0.8761689067,-1.0638122559,-2.5720088482,-0.5335486531,-1.0095447302,-1.7223300934,-0.6535322070,1.1655402184,1.1832163334,-0.2269054651,1.7193808556,-0.1380582601,1.5151145458,0.2277833372,0.0959253907,-0.5170901418,-1.0309077501,-0.4735969007,-0.0485035852,-1.1286866665,-0.7307674885,-0.5463230610,-0.0409587808,2.0975720882,0.3434947133,0.9940972328,0.3789162636,1.0542325974,-1.8494417667,-0.4977275729,0.7369968295,0.5268564224,0.1865140498,0.8466647863,-0.6316319108,0.1136949435,-0.6331836581,0.3402028978,0.5535160899,-0.2889496684,-0.7305825353,0.4190462232,-1.0515617132,-0.4801098704,0.0261303224,1.5960645676,0.1610219181,1.7804647684,-0.3047503829,-0.8188756108,-0.3621890247,0.3808596432,0.2823227644,1.5666717291,0.3734123111,-0.8837423921,-0.7322314382,0.1986591965,-0.0118880589,-0.7151897550,0.0778959244,0.8643652201,0.6962469816,-0.5422630906,-0.9411893487,-0.8344305754,1.1522097588,1.3389163017,0.0177985504,0.5214101672,0.6627396941,-0.0895537585,0.3205173910,1.1086225510,-2.7512409687,0.7842624187,0.2301861048,0.4868739247,0.1914014667,1.4830135107,0.0156363230,0.6475189328,1.8308417797,0.4691707790,0.3148375750,0.7041759491,-2.6743814945,-1.5270642042,0.2455077767,-0.5976601243,0.9597811699,-0.4113830030,0.7531632185,-0.4996159077,0.6813071370,0.7660171390,-1.4191707373,0.3065551817,-0.1291158944,-0.4976983070,-1.2929699421,-0.1371014863,0.7562644482,1.0583063364,-0.4212669730,-1.9234799147,-1.6065359116,-0.2558904588,-0.4901143014,-0.0949113965,1.1031668186,-0.5381546617,0.2360364795,0.0465177372,-0.3361516595,0.1274346560,1.9163759947,0.6101396084,-1.1195522547,-0.7814033031,1.3373019695,0.2333422899,1.0861701965,-0.0382700153,0.8989006877,-0.0218960550,0.1018365696,-1.5967543125,-0.0113325082,1.1269840002,0.0552202910,1.4069075584,-0.9980571866,0.0247818157,-0.7040320039,0.3176784813,1.6605193615,1.0455915928,-0.8923811316,0.9219143987,1.1073287725,0.7103539705,-1.4154820442,1.0690660477,0.1309919506,0.0383858383,0.9466379285,1.3524271250,0.8901629448,0.4715458155,-0.1681349725,1.7018618584,-0.4245993197,1.5963537693,0.4084403515,-2.2818739414,-0.2058727890,1.1010887623,-0.7389399409,-0.2773337960,-1.1447142363,0.5750554800,1.7250024080,-0.9898491502,-0.4104329646,-1.6079535484,0.2768232226,-1.6983526945,-0.5670870543,2.1507933140,-0.2975871563,2.1437575817,1.3332302570,-0.3730902672,-0.1122499704,-0.7847885489,0.9248625040,-1.3942272663,0.6543221474,1.0042756796,-0.9440436363,-2.4518766403,1.3270406723,-0.1127001345,2.2061531544,-0.3989170194,-0.4466468096,-0.5355738997,0.0526149832,0.6064239740,0.5198518038],[-0.8466613293,-1.5090380907,-0.4535782039,0.4313685894,-0.1462003887,0.1641013771,2.1521577835,0.3630537689,0.2070374936,1.7983050346,-0.0564059503,0.5928288698,-0.1816093922,-1.5518671274,-0.3621249795,-1.0239580870,1.0249418020,-0.0258395690,-1.7892510891,0.6218369007,0.6613118649,2.2723629475,-0.4432788789,1.0698333979,-0.8403570652,-1.4873845577,1.5990829468,0.3635856509,-0.1802307367,-0.3384521902,-0.2343396097,-1.4471864700,-1.3438107967,-3.1347074509,1.3278874159,1.4307368994,-0.2085703611,1.5691628456,-0.0372373499,1.5322495699,-0.8938466907,-1.6205263138,0.9487670660,0.0607111529,-0.0718811527,0.4896041751,-1.2187587023,0.8618543148,-0.2260780036,-0.4962435067,-0.8952035308,0.1894100606,1.9990708828,0.6714808941,-0.3252743185,-0.3147354424,-1.2198890448,-1.1942043304,1.2294464111,0.9353618622,-1.7695857286,0.3260254264,0.8139547706,0.1962550431,0.1314067394,0.5124615431,0.7095163465,1.3730001450,0.7314730287,0.8850992322,0.5249949098,-0.0936656371,1.0094020367,-0.4246652126,-0.0342488736,0.5519418716,1.0054379702,0.2137652487,-1.7681372166,-0.9254562259,-0.4854847193,-0.6538339853,-0.2760174870,-0.2612164021,0.9103474021,1.3952193260,-0.3264450431,-0.0597991198,-0.0280044656,-0.7869258523,-2.5064163208,-1.2314569950,1.4557712078,1.2016288042,0.4458014369,-1.4862548113,0.9897671938,0.9338101745,0.7926711440,-0.1426515281,-0.9763342738,-1.1071892977,1.8700821400,2.0336098671,-1.5667554140,-0.5214136839,1.1199978590,0.7776375413,-0.1399716139,-0.0623685531,1.1362810135,1.0089765787,-1.3147529364,0.6011255383,2.0578691959,0.2968917489,1.1136337519,-0.6109186411,-0.6683439016,0.9962469935,-0.8060427904,0.3595867455,0.7470183372,1.0633322001,-0.9276279807,0.3000597060,0.7063956857,1.0950361490,1.4885321856,1.4787135124,-2.1218326092,0.9264213443,-0.1775084585,0.9011278152,0.2840862274,0.0977649391,-1.0456649065,0.1681640744,0.0973941088,-1.7393004894,0.2261245102,-0.0660531223,-0.3362882733,0.8595430851,-0.4416444898,0.0228420068,-0.7061185241,-0.1840548813,-1.2924098969,-1.0702615976,-0.1731543988,0.9668654799,0.7544121146,-0.5534548759,-0.1589929909,-2.7193152905,-0.5930914283,-0.4942319989,-1.1334577799,-0.1847745329,-0.8339127898,0.1843493879,-0.3052302599,0.5634783506,-0.0959699973,-0.9182578325,-1.9924126863,0.3829368651,0.2503584921,0.2333265692,-0.3078962266,0.5468981266,0.1616246402,-0.4544194043,0.2356493622,1.1045236588,-1.3996573687,0.5584018230,0.5636180043,-0.8096659184,-0.1284091473,0.4092279077,0.2285577953,-0.3103590608,-1.3424509764,-1.2082949877,-0.9283201694,2.0955545902,1.1210683584,-0.8812590837,1.2313210964,-0.3197748065,1.7898057699,0.0698545128,0.7923595309,-1.0690809488,0.6286915541,-0.1670196503,0.9845463634,1.0878648758,-0.1941708624,0.5016475320,-1.1764376163,-0.7215655446,0.3653467000,0.7173801064,0.3487474024,1.1008827686,0.6648312211,-0.7699224949,1.8959935904,-0.1441157013,-0.2147849053,-0.7026571035,-1.2497316599,-0.7573789358,-0.5733095407,-0.4854148626,-0.7615327239,-0.5698633194,2.8097834587,0.8722813725,1.2352868319,-0.9432064891],[-0.0974869877,1.7055737972,-0.1014531478,1.0176905394,1.5702327490,-1.7046865225,-0.3277420104,-0.3331027627,-2.1215126514,-1.7561151981,-0.2295200676,1.6113495827,-0.2443063259,2.5411062241,0.3537022173,0.4745986760,0.6965101361,0.1384816021,1.1178592443,-1.1784735918,-1.1290528774,0.2061873525,1.0603878498,-1.1349711418,1.2598598003,1.7623924017,0.0733724684,-1.3333655596,-1.0912177563,-0.6205294728,0.7761052251,0.1271678060,1.2515132427,-0.5780774951,0.7281842828,-0.7465434670,1.1616883278,-0.6196599603,0.3022282720,0.8347775340,-0.0490818210,1.5939271450,-0.9503580332,-0.4347720146,-0.7656218410,0.7085364461,0.9605593681,-1.0164953470,-0.2222494930,-0.8314278126,0.4303712249,-0.7721483111,-0.3019029796,0.5900394320,0.0245243497,0.1932301819,-0.0898328945,0.4563172460,0.6518187523,0.0078258719,-1.2128522396,0.5727971196,-1.3422386646,-0.8164447546,-0.0112239150,0.4942510426,-1.7097381353,0.6157289147,1.2700591087,-1.0838717222,-0.4467307925,-0.2466057837,-1.0219964981,-0.3065997958,0.3694855869,-0.0713111088,0.4026225805,0.6370279789,0.8591992259,1.0705020428,1.8353241682,-0.4750539660,0.6103578806,-0.5137580633,0.7581811547,-0.1050673276,1.5978858471,0.6333515048,0.0460980870,-0.7376776934,0.2096036524,-0.4071459174,0.7349792719,-0.4330402315,0.7938969135,-0.0671691522,-0.3470112383,1.8795491457,-0.1271699071,0.9610880613,-1.4886234999,-0.8252080083,1.3477144241,1.5563865900,0.1703135967,-0.9434504509,0.0534256622,0.1484180987,-1.2368420362,-0.1398315430,0.1595588923,1.5863084793,1.3536993265,0.5934506059,1.1358667612,-0.1774367094,0.9255916476,-0.1539941430,-0.6215142012,-0.6159935594,-0.4488615990,-1.4912188053,-0.5317133665,-0.2063709795,-0.4096370935,0.9561219215,0.5284463167,-0.8182538152,-0.5056212544,-0.1462823451,0.0752253309,-1.2333616018,-0.5615788102,-1.0774196386,-1.4043681622,1.4619164467,-0.6012640595,-0.2459688783,0.3324500620,1.0232548714,1.4352263212,0.1944244057,-0.1463443935,-0.2790006101,-0.6674974561,-1.3180520535,0.3201041818,0.3282475471,-1.4190750122,0.3450450897,-0.1171793342,0.1504306048,-0.7891467810,-1.0476018190,-0.3834323287,-0.1902684718,-0.4965841472,-1.3784673214,-0.1844371557,-0.4675067067,-0.6985540986,0.4536924958,0.3140238225,0.0564287789,0.7583971620,0.6013430953,-1.8449608088,-0.1788474470,-0.0510619581,0.5304078460,-0.4194923639,-0.7391867638,0.1276831329,-0.6782696247,0.1311978102,2.7735209465,-0.2934664488,0.5471537709,0.0393295288,1.0524917841,2.4084842205,-0.1510079354,-0.0755359530,0.2978080213,-2.5932602882,-0.1261765659,-0.0178737193,-1.1064251661,1.6046415567,-1.8980734348,-0.2849923372,0.4307873547,-0.7775550485,-0.9093212485,-2.1181476116,0.5486614108,-0.8518691063,0.1899482012,-1.2635831833,0.0362131745,-0.0570133552,-0.2774310708,0.5166860223,-0.6961334944,-1.1381798983,0.0194949433,-0.2258192897,-0.0000388383,1.0765069723,1.1303399801,1.8657202721,-0.4969576895,0.5364794731,-2.0242605209,-0.8213713765,-1.4851415157,0.8490946889,-0.0753547922,0.3361081779,0.0728952736,0.0231189914,-1.2837551832,0.9499085546,0.1681417376],[-1.4466546774,-0.2972466350,1.1298607588,-0.2603873909,-0.9126073122,-0.4502425492,-0.5704441667,-0.9064111710,-0.8553234339,-0.4430517554,0.2436235696,1.4905276299,-1.0585029125,0.4943399131,0.7875627279,-0.3641681075,1.1554255486,1.2991687059,0.1732139587,-0.7331561446,0.1793219149,-0.1800402552,0.5449168086,1.3240280151,-0.1309091151,0.4784817994,-0.4267643988,2.0781397820,-0.7498555183,1.6521127224,-0.1664374918,-0.7460601330,-0.9441272020,-0.4832925498,-1.5331821442,-0.4067618549,-0.7364491224,-0.9086607695,0.3413515687,-0.8198691010,-1.7870553732,2.4673192501,-0.2311452627,0.3308919966,-0.1159708798,-1.7726674080,-0.9126517773,0.1084937230,-0.2664954662,1.1111640930,0.1229815185,0.6907859445,-0.1474541426,0.9019960761,0.2317667753,0.8363870382,-0.3399315774,0.7041403055,-0.1264661849,1.1791597605,-0.9016402364,-0.1811943799,2.3676764965,1.1302993298,0.3110736012,1.5677846670,-0.7050484419,-0.1518790126,-1.7081288099,0.6985465884,0.7089632154,-1.0309268236,-0.1347338855,-0.0365764089,0.1824831814,0.5456715226,-0.0108612115,0.1843760610,-0.0527592488,1.1165183783,2.0685019493,0.0236146916,-0.1301397979,-0.1713315994,-0.7619674206,-1.6910823584,-0.5223678350,-1.7465475798,-0.2006603628,0.7275930643,-0.4736891985,-1.4242618084,0.0546379611,-0.3088788986,-0.1335759610,1.9706972837,-0.2325765043,0.2274934202,2.2387323380,1.4907130003,0.3462500870,1.3234370947,-1.9889664650,0.7430348992,0.6448816061,0.5218971968,-0.1231725663,1.5066103935,0.8738765717,2.7089717388,-0.8890984058,0.0795818791,-0.0192832518,-0.9809086919,-1.2750723362,0.2867516875,0.2137882411,-0.9008516073,1.5888748169,-0.4282222986,0.4386356473,-0.1085125059,0.6985945106,1.9744687080,0.4545195699,1.3060090542,-1.3972746134,-0.2608897984,0.6921256185,-0.2758881450,1.0756412745,-0.1134573221,-0.0426482968,-0.9878318906,1.5196024179,0.6308060288,-0.9928121567,-0.4773582518,-0.5678960085,1.1409707069,0.0252071396,-0.9686629176,-0.1847696155,-1.1870638132,-0.4968124926,0.7360694408,-1.6462267637,2.3052799702,-0.8024062514,-0.6898394823,1.2083338499,-1.2181578875,-0.8410550356,-1.2503207922,-1.3341989517,0.5452713370,0.0004595034,-0.1027127653,0.5961635113,-0.5245649219,-1.8084018230,1.0589570999,-0.7794679999,0.6349820495,0.3862176836,0.8346452713,-0.8713117242,0.8153094649,-0.5405928493,0.2869177163,-0.6896800995,0.0557697900,0.1182901114,-1.7857323885,-1.0742902756,-0.4063031971,1.7793726921,-1.7563652992,0.6893250942,0.8844146729,-1.0142682791,-1.7650473118,0.2160642594,-0.7887262702,-1.1181463003,-0.1376193762,0.1754952371,-0.2765376568,-1.7079218626,0.9067323804,-0.0235241614,-0.3631467521,0.8013179302,-0.1235235482,-0.1538935751,0.5224968791,0.8662751317,0.2375682592,-0.8459567428,0.3390309811,0.6798582077,-0.9715719819,-0.1630507708,-0.1906574965,-0.3275502622,0.2688953876,0.4053089321,0.1773621142,-0.9372261167,-0.1691677719,-0.3724011481,0.8112956882,0.1267861128,-0.5620681643,0.2927173972,-0.8813909888,0.0762910396,-0.0483648069,2.1920011044,1.7463549376,2.3207309246,0.2887925506,0.7549450397,-1.4306452274],[0.3228493333,-0.0288458075,-0.2087575942,-0.8381208777,-0.9550005794,0.7841243148,0.2179069519,-1.0709420443,0.3135614693,0.0914136097,0.0320585743,0.3876436651,0.3842196763,0.8502239585,1.0612168312,1.8604735136,0.0568598658,0.2732332349,0.0278589707,-0.8741342425,0.3974182904,-1.3401019573,1.6500184536,0.5345094800,0.4902154505,1.3779866695,1.4287627935,0.5794705153,-0.4708151221,-1.2481455803,-0.9796716571,0.1646438241,-0.2908467948,0.1688180864,0.2806825936,0.3941893280,-0.5539347529,-0.0067497804,-0.2968336046,-0.4971628189,1.9962546825,0.2005653828,-0.6932837367,-1.6317993402,0.3809179664,-1.5123542547,0.1992826313,0.3300321996,-0.3034650087,-0.1181315854,0.0882374570,0.1668884158,0.3019710481,1.4882793427,-1.1674114466,1.4780186415,-0.3395954370,1.6217408180,1.3903936148,0.0755978227,-1.1194069386,1.7365732193,0.1781414896,-0.9232194424,1.8039987087,-0.8838068843,-0.4083239436,0.3007299900,0.1699735075,-0.8590050936,-1.7392672300,1.5597413778,-1.3260040283,0.2854619622,-1.9159833193,-2.3440728188,0.4461363852,-0.0687278062,1.9588936567,-1.1906564236,-0.7276904583,0.2522965074,0.9506995082,-0.0913084596,0.7821321487,-0.0195057746,1.2213931084,-0.1069381684,0.7629193068,0.1845369637,0.3870324492,-1.0695841312,1.0488708019,-0.5519348383,0.5487302542,-2.5772011280,-0.2137518078,1.2179993391,0.9156683683,0.3357705176,0.5025701523,0.3491545618,0.7370117903,0.3278817534,-0.7132780552,-0.5390459299,-0.7518448234,0.5354685783,0.0184270069,0.2321900129,-2.1776525974,-0.0709908977,-0.3209287524,-0.2798088789,2.0468642712,0.9956209064,-0.2023959607,1.3509565592,-0.1920386255,-1.5777788162,1.6549317837,0.4675681591,-1.9365965128,0.9573156834,-0.7165775895,0.9248271585,2.1137826443,0.0793551803,-1.1837171316,0.7460383773,0.6538752317,1.3912616968,-0.1166813821,-1.7531785965,0.2474077493,-1.1580266953,-0.7121775150,1.8413614035,1.9593071938,-0.2458700389,-0.5498188734,1.1876565218,-0.3367114961,0.8950569630,-0.8747928143,-0.6351636052,2.0111460686,-1.3398618698,-1.7623288631,-0.3162357509,-0.3991983831,-0.1123126596,0.4193961918,-0.8965550661,-0.3267290592,-0.8449197412,-0.9209045172,-0.2399897277,0.4476865530,-0.7418269515,0.5197879672,0.0386816151,1.1803443432,0.2628131509,-0.2394232601,-0.3722534180,0.6683313251,1.3366475105,-1.4936909676,-1.4146258831,-0.2376010120,-0.2817085683,1.8941197395,0.4649803042,0.8323036432,0.4888179898,1.1489104033,0.6556180716,-0.7000198960,-0.6820146441,0.4848237932,-0.5177137852,0.0824175030,-0.8797442317,1.1639926434,1.1476025581,-0.8527011275,2.0083818436,-0.2703168988,-1.2097411156,-0.2276968956,0.2739947438,-0.5334571600,-1.5935918093,1.2445474863,-0.3134967685,-0.2985264361,-0.7968905568,0.8554915190,1.6267149448,0.6216568947,1.5030903816,-0.1151232570,0.8909834027,1.8018869162,-0.0360328667,-0.8202154040,-0.2220973074,-0.3517731130,-0.5957280397,-0.1514147520,-0.4198016822,0.2551528811,-0.1152595282,0.2171547413,-1.4629511833,1.1438225508,1.2084295750,-0.8150461316,-0.7321365476,0.3776405454,-0.9554858208,-0.4959684014,-0.2907817662],[-1.0054425001,-0.4517457485,0.1696795672,-0.6147547364,1.1529808044,0.0660743788,1.5753021240,-0.4344066083,-1.7833483219,0.0277798567,-0.5330213308,-0.9183149338,-0.6086820960,-0.4047652185,-0.6416500807,-1.0837899446,0.0532945246,1.1404436827,-0.0453740098,0.7457920313,-1.9460426569,1.7987947464,0.9324730635,-0.8482402563,0.6843633652,1.3739305735,0.3718565106,-0.1377663314,-0.9630939960,-0.3043234050,0.0718399286,-0.2703020275,-1.1004296541,-1.3805667162,-0.5695853829,0.4196306765,1.9971634150,0.8231883645,0.2833084464,-0.2101227045,-1.8163676262,-1.7911117077,0.1537543535,1.9449908733,0.1513811201,-1.1334222555,0.5895084143,-0.5986622572,0.3019756973,0.7390938997,0.9933456779,0.6807634830,0.9933760166,0.5551660061,-0.8488301039,-0.5885685682,0.4313100278,-1.3862833977,0.7359342575,0.9195241928,-0.0853480026,-0.1070248708,-1.2493082285,0.5755701661,-1.0534989834,0.0406493060,-0.5918547511,0.0460599251,0.7082794309,0.7456382513,-1.0815974474,-1.2848150730,0.7570005655,0.9823464751,-0.4332475960,1.9694807529,-0.5031172633,-0.8139947653,-1.9751011133,1.2804644108,0.5563730001,-0.4628264606,1.1694854498,-0.4353146255,-1.6423184872,-0.5444234014,1.8026432991,-1.5473425388,0.3108020425,1.2917841673,-0.3807676435,1.7654125690,-0.6993806958,1.1027238369,0.6636332273,-0.6843606830,-1.4899740219,0.2734349966,1.2600288391,-0.2286336422,0.2327901125,2.4362354279,0.1817132533,-0.7989073992,1.1250274181,-1.0452672243,-0.8149611950,-0.2601708174,-1.3295027018,2.2031581402,1.1800258160,0.2624831796,-0.0770635083,0.7740607858,-0.3567553759,-1.1034004688,0.1847368032,0.6796195507,-1.0074751377,1.1585245132,1.4195199013,-0.6195434332,0.7116128802,0.6401365995,0.5280639529,-0.9293158054,-0.0831103027,0.5697512031,1.0032013655,1.0931603909,-0.4489105642,-0.2979501188,1.0878436565,1.6233495474,1.4550083876,-1.1979384422,-0.9113218188,-1.2620170116,0.1533333361,0.3634153008,-0.3303910196,0.1722114384,0.1782008410,0.9415059686,-0.7712236643,-1.5482803583,-1.4313429594,-0.3579024374,2.3041453362,2.1099629402,1.6484801769,0.1672390848,-0.0169916637,1.7207933664,0.2674842775,-0.4923659265,-0.6619585752,0.8079488277,-1.6243458986,-1.7960036993,-0.7675559521,-0.0231633689,-1.0103166103,1.2741718292,-0.6419000030,-0.7730162740,-1.4375553131,1.1895669699,0.3485536277,-0.4232673645,1.2198886871,0.4842070043,0.0227463860,0.5305594206,-0.8054476976,-1.7734887600,0.3406913579,-2.7052669525,-0.0847135931,-0.6385948658,-0.5110730529,0.3484685719,-1.3105498552,-2.5472373962,-0.3775078654,-0.0988472104,0.4710415006,-1.2057213783,-1.7653845549,-0.0680527538,0.7711173892,-0.7321217656,1.0039491653,-0.7371895313,0.5678637624,0.3336027265,-0.5274707079,-0.9774408937,-0.3786778450,-1.4249675274,-0.7147436142,0.3129166365,-1.7783929110,-0.4209950566,-0.5415373445,-1.2222980261,1.3970868587,0.8098871112,-0.2767983675,-1.8499927521,0.1502419710,-1.5492517948,0.7092196941,1.4950456619,-1.1154440641,-0.5161579847,1.2957165241,-0.5796651840,0.6269744039,0.9531516433,0.2816024125,0.4732829630,-0.7617855668,1.6947957277],[-0.0578788295,-0.4390718937,-1.1987463236,-0.4341277480,-0.0426968820,0.7203280926,0.3357464373,0.8612458110,1.0400627851,-0.0488214567,-1.2987916470,0.8273304105,1.0139622688,-0.0649127662,0.7446480393,-0.1540967673,1.9260265827,1.2363505363,-0.4862475693,0.4868167043,1.7191879749,-0.0682080835,-0.9640083909,0.3089770079,0.9302484989,0.9873762727,-0.5898210406,0.4689652324,1.5302232504,0.7320433855,-1.3513690233,0.7897057533,0.2824765146,0.5834311843,0.1131865606,-0.8406014442,0.4050630033,-0.4101101756,0.5337298512,-1.3036297560,-0.6484665871,-1.1359941959,2.0571515560,0.6670948267,-0.4131903946,-0.5690752864,1.4522426128,-0.9800069332,-0.1403042823,-1.0196304321,0.9959092736,-0.2827495635,1.9643141031,-0.6609107256,-0.7525837421,0.5954794288,-0.3523457646,0.3810907602,-1.0135425329,1.4329707623,0.8080863357,0.8896982670,0.2208904177,0.3672073483,-0.6885126233,0.4308250546,1.1822276115,-2.0149857998,1.9600487947,-0.5171279907,-0.1703243852,0.8568869233,1.2518591881,0.2865099013,0.8115671873,-1.4276633263,0.5710371733,1.0781027079,2.2357771397,0.7279140949,-0.2229265273,0.5872340798,1.4881185293,2.3195126057,0.5637220144,0.6914354563,0.2569299936,-0.2285066694,0.4804783762,-0.2613047361,-0.4131895602,-1.4262596369,0.4228245020,1.0623100996,0.2604335845,0.6574383378,-0.0409760214,-0.1292385161,-0.6623541117,-0.6476683617,-1.6716749668,-0.1830161214,-0.3365137577,0.6030265093,-2.0935566425,1.2513388395,0.3918113708,0.7372437716,-0.0178348534,-1.1537516117,1.0530462265,-0.8401392698,1.8309344053,-0.9686001539,-0.3691553771,-1.4921547174,0.0017514684,-0.4581293762,-0.6935526133,1.0855137110,-1.0821048021,0.7962128520,-0.1281221062,-0.8133049011,-1.2505276203,-2.2208952904,-0.2130862474,1.0099068880,-1.4203333855,-0.2492176443,-1.3636775017,0.1963888705,1.9370468855,0.5962476134,-1.6595464945,0.1022451743,-0.5378738046,-1.8219336271,0.0530054718,-0.3030683696,-0.0812117681,-1.8513782024,0.3097834587,-1.5337352753,-0.2538897991,1.3182882071,1.1397893429,0.4829955697,-0.9046905637,-0.8380981088,-0.2948737741,1.9916062355,0.0728184804,-0.1014356986,-0.1235939339,-0.5081471205,1.2604811192,-0.1343659312,0.2073383778,0.3723934889,-0.4487771392,-0.4661149383,-0.3359813094,-1.3195090294,0.3540134430,0.3601185679,0.0083168242,0.3976401091,-1.6742360592,0.2504509389,-0.6635281444,0.8641865849,-1.1257250309,-0.3801716566,-0.1833992749,0.3565971553,-1.3327323198,0.6035583019,-0.5465292335,-1.5566965342,0.6932677031,1.6228110790,0.6801456809,-1.5924171209,-1.5193947554,-1.0084333420,0.4458001256,-0.8410766125,1.5464582443,0.6618500948,1.3115892410,-0.8060818911,-2.0020639896,-0.7427703142,0.9982657433,0.0429815277,-2.2929155827,0.7949628234,-2.0755562782,-1.1538249254,2.2420399189,0.3406197131,0.0048286184,0.2839885652,-0.4365164936,1.3366078138,-1.8391609192,0.3603280187,-0.0859675929,0.5570289493,0.5718832016,0.6106733680,-0.3083177805,0.0144931190,-0.2672089338,-1.2353001833,3.0229272842,0.3395426869,0.5264291167,1.2634595633,0.1770529449,-1.4849345684,-0.6991217732,-0.7283034325],[-0.6939549446,1.5446032286,-0.2754387259,-0.2167219073,-0.7424051166,-0.9166474342,1.5751912594,-1.6181910038,-0.8253030777,-1.0706801414,0.8698484898,-1.1994905472,-0.8146388531,1.2920223475,-0.4625878930,-0.3836516738,1.0654515028,1.6666485071,-0.1208937168,-1.4995495081,1.2321619987,-0.6949200034,-0.5739510059,-0.1775606424,-0.9272752404,0.5301542282,-1.1949777603,-0.4780461788,-0.5277665854,-0.9728552699,0.4626088738,0.4503042698,0.9948613644,0.2734299004,0.1685026735,-0.6112354994,-0.9500792027,1.8824837208,1.7905997038,-1.1740034819,0.0983359888,-1.1613261700,-0.3503814936,-0.8911704421,1.0788980722,0.5437412262,0.6564418674,-2.4974653721,0.4571970403,0.0749873742,0.0620500483,-0.2306726277,0.1126581952,0.6336370111,-0.4393058419,0.6253045797,-1.3444340229,0.3629311323,0.9891084433,1.3607655764,-1.6233907938,0.3538567126,1.2099943161,0.7256388068,-0.4102138281,0.4880381823,-0.6662564874,-0.3059380352,-0.1223294958,0.9289811254,-0.9578050375,0.7637707591,-2.6466319561,-0.8069033027,0.2090090364,-2.1910057068,0.8204702735,-1.6458770037,0.1837251335,-0.4417119026,-0.3663473129,0.7258780003,-1.0395754576,0.3824823201,-1.3599922657,-1.3667659760,0.2267869860,1.0660798550,0.7252836823,-0.0433388688,-0.8327400088,0.5989152789,-0.1539460868,0.8072755933,-0.1370912343,-0.3414518237,0.6859472990,-0.0732531026,0.4887581468,1.4984587431,-0.8649017811,0.1688008904,-0.1289001107,-0.9218314290,-0.3603636324,0.9271414876,0.0079463189,-0.3383709490,0.8675869703,0.1249827370,0.4651188850,1.1462744474,1.2493416071,-0.4183758795,-2.5614197254,0.5680629611,-0.6206008792,1.7465558052,0.3847604096,1.4685661793,-1.1315544844,1.1918085814,-0.6606284976,0.4041602910,-1.4739680290,1.1007773876,-0.4919222891,-0.1771818995,-1.2822349072,-1.0434724092,0.3237209916,-0.7618917823,0.3099120855,-0.5639561415,-0.0917711481,-0.5850586891,0.6370205283,1.0494756699,-1.9202592373,1.2494323254,0.5849095583,-0.8469260335,1.3565355539,-0.0963294879,1.2453507185,-0.1400534064,0.8450857997,0.4133973122,0.7155808806,1.4432688951,1.5716166496,0.6924489141,1.4065362215,-0.4133162200,1.0888381004,1.3608398438,-1.0978585482,-0.2764387131,0.5563209057,0.8842734098,-0.3509296477,-0.9768342376,-0.7402306199,0.2903321981,0.1414962411,-0.4799614251,-0.3082216084,0.0267795082,1.2097295523,0.2476162165,1.1630425453,-0.0845261142,-0.2282030135,0.2696264684,1.2454012632,-0.1312718689,1.4098299742,1.0410118103,-1.6086004972,-0.5174929500,0.8974530697,1.0401508808,2.2852160931,-1.0753257275,0.0802447274,-1.2287327051,-0.0713328272,-0.8569629192,1.1176137924,0.5114846230,0.6936994195,-0.8175045848,-0.6783365011,-0.7737123966,-0.3651148975,0.6318411231,-0.6266897321,-0.6617985964,-0.5669463277,0.2499994934,1.1892088652,-1.9240505695,-0.3544670045,-1.0515387058,-1.8288736343,-1.2691622972,-1.3089474440,-0.0297324564,1.2603071928,-0.4449788332,-1.7621899843,-0.8053673506,0.5204708576,0.2721701562,0.2346938401,-0.6873981953,0.2655944824,-0.0325214528,0.5738306642,-0.5787928104,-0.8237167597,-1.2874125242,1.1350760460,-2.1356105804],[0.4515221119,-2.1500139236,-0.4114108384,-1.1628191471,0.1297886521,-0.2495429516,-0.0260325857,0.9392570257,-1.3046522141,0.6903501153,0.6977592707,1.0701305866,-0.2887674570,-0.4104785323,0.1533241421,1.3199142218,0.3096901178,0.4101806581,-0.7352973223,-0.9126303792,-0.7222605348,-0.4227232933,-0.2477057576,0.5480054617,-1.1432491541,0.3385492861,0.2351884991,0.0385535322,0.8181906939,1.5828475952,0.7214038968,0.0897340551,-0.3189024925,-0.3974009156,0.9097927213,1.9650508165,0.1053364798,-1.2227591276,-0.0153107131,-1.2919523716,-0.7455282807,0.9781656861,0.6193071604,0.4077815115,-0.7046490908,0.1537861079,-0.0643131956,1.2573903799,0.9574343562,-0.5834718347,0.1090049073,0.5478889346,0.3504471183,-0.2250039279,0.8269471526,-0.0169237554,0.6220702529,-0.2975426614,1.2149496078,1.1151980162,-0.4507243931,2.2206854820,-1.8293071985,2.6430711746,-0.1439989805,-0.0610520355,-0.2829388976,1.0629047155,-3.1820971966,0.4401958883,0.7499151826,2.5593225956,-0.9967869520,-0.2721943557,-0.0896866843,0.6282408834,-0.0261002425,-0.9709675312,0.0230096858,-0.0140103968,-1.6569482088,-0.1382927299,0.7253227234,-0.7418099642,0.9728586674,0.4095517993,0.0483989120,0.0979183540,0.3018720746,-0.3589890599,0.5353608727,0.5360045433,0.7536387444,0.7505097389,0.5256783366,1.3380362988,-0.2981445193,1.8530151844,-1.1281915903,-0.0498322956,-0.7330253720,-0.6458021402,-0.5024451017,-2.2123041153,-0.9805191755,0.8008458018,-1.0077247620,0.6239641905,-0.8552334309,-0.2744834423,-0.1157714501,0.0847591385,-0.3774202764,-0.3264375031,-0.0360224955,-1.0819702148,0.3266488910,0.5532034039,-0.3854060173,-0.1618631631,-1.9549201727,-1.7788834572,-0.1174470335,-0.2493743002,2.1404275894,0.5593461990,0.6229158044,-1.8816431761,1.4607295990,-0.3706546724,0.0042911167,0.3046770096,-0.0710433796,-0.1237470731,0.4302512109,0.4529004395,0.6425746083,-0.1122784540,0.9161322713,-1.1342129707,-0.5450671911,0.2877930403,-0.2188682407,1.2186625004,-1.1504718065,0.5568408370,1.3841053247,-0.1620552093,0.7049739957,-1.2413967848,-0.9855633974,1.3243196011,0.3625219762,0.2983141243,-1.2990237474,-0.2681641281,0.1363256872,-0.1956966519,0.2655187845,-0.1531553864,1.2532037497,1.8852713108,-0.0882719830,1.7805033922,-0.4716087878,-0.9298340678,-0.3180843890,1.2743237019,1.0903788805,1.0813274384,-0.6996761560,0.7500818372,2.1777994633,2.2021887302,0.2121428996,1.6765725613,-1.2105097771,-1.5527480841,-1.3053249121,1.9286055565,-1.3529680967,1.0474323034,0.2311239541,-1.3959530592,-0.6062406301,-1.8592184782,1.0289849043,-0.1385550201,2.3706924915,1.1001138687,-2.1009137630,0.6809428930,-1.6257436275,-0.6237348318,0.6981055737,-1.8643096685,0.4661882818,-0.2186380774,-1.3467729092,-0.9527578950,0.2725820839,1.3400031328,-0.5603945851,2.5774965286,-0.5632506013,-0.3768528998,0.0353212729,1.4043915272,0.0230951086,-0.0061584190,0.6321358085,-1.9023759365,1.5614373684,-0.1971298605,-0.7250073552,-1.0256675482,-0.4578739703,-0.6443290710,-1.0963523388,-0.0895931944,-1.4758558273,1.6121660471,1.1630821228,-1.7796010971],[0.0145237353,-0.5974840522,1.0503715277,0.3695131540,-0.3743128181,0.0213968288,-1.0481902361,1.0413702726,0.3212760389,0.3309848011,-0.6946014166,-0.7083328366,0.9349833131,-0.1450703889,1.1152725220,-0.8652854562,-0.4565080702,0.3007369041,0.9256248474,2.0625565052,0.2747337222,-0.2963839471,-2.6546571255,1.2803245783,0.1107093021,-0.6835398674,0.4723945856,0.9344331622,-0.7295447588,-0.2803449929,0.1233135983,0.6120019555,0.5841313004,1.7533856630,-1.0543075800,-1.2622847557,0.4116769433,-0.6446053982,-0.2465761304,-0.7146756053,1.3107023239,0.6113011837,-0.1636191308,0.4882713854,-0.3482097387,0.1603152454,-0.4162207544,0.1293164194,1.7988963127,-0.8851790428,0.3992467523,-1.7737257481,-0.3428349793,1.8912366629,0.0722259954,-1.6537472010,-1.6307649612,-0.4667405784,1.1868702173,-0.7915568352,0.4578755200,0.2519705594,-0.9535153508,1.2609884739,0.4392801523,-1.3256343603,-1.0367951393,-0.2201278061,-0.6169109941,-1.0181512833,-0.4103388190,0.1084682569,-0.6541967392,0.7603313923,1.3501311541,-1.6931246519,-1.5221095085,-2.0480751991,0.4730706215,1.3266686201,-0.0894597024,0.3304823041,0.1989917606,-0.4702607393,0.2213799506,-1.5739086866,1.7835437059,0.3327397406,-0.4354090691,-0.1039255261,1.8387240171,0.3680784702,0.6319974065,-1.5963237286,0.0214680173,-1.0597387552,0.1692428887,-0.1024794206,0.3460634351,2.1404290199,-2.7088286877,-3.3357634544,2.4715435505,0.5048002601,-0.0024718831,-0.3783724308,-1.8512169123,0.0197386872,0.5260105133,0.0301570203,1.1527439356,-0.6599699259,-0.7668974996,0.9182029963,-2.0533874035,-0.5711094141,-0.8333142400,0.5297877789,0.1696676463,-1.1874402761,2.6695325375,-0.3015936315,0.0863558426,0.6479152441,0.4200901091,0.0836594850,1.3033480644,0.6622453928,-1.5946129560,1.1357042789,-1.2606321573,1.0469983816,1.1802233458,-0.1032438353,1.1054823399,-1.2712923288,0.0872885138,1.3110666275,-0.3147211671,0.7948974371,-0.6520865560,2.3930139542,-0.6381352544,-0.1903901398,0.7538375258,-0.6007530093,-0.4748378098,0.4931529164,-0.3945479691,-0.7058361173,-0.4707331657,-0.6427865028,1.1003127098,1.2145934105,0.6543659568,0.1378059983,0.2667719126,-0.4505184293,0.7668704987,0.3069391251,-1.0402356386,-1.0768088102,-0.3584763110,1.4544060230,-1.4216730595,0.3472038507,0.2086229771,0.5839641094,-1.4341008663,-0.6708384156,0.2508373260,0.8319552541,0.5872681737,-0.4998468161,-1.1823883057,-0.3054107130,-1.0352274179,-0.6697105169,-0.8784491420,-0.8758939505,1.3824001551,1.1037901640,0.5120603442,0.2342875302,0.7024837136,1.1357293129,-1.0685719252,0.1360779256,-1.2616283894,0.1958422810,-0.7124678493,0.4309418797,-0.8070568442,-3.6666617393,1.2247881889,-1.8534743786,-0.8791117072,-1.1550799608,1.3362749815,0.6916596889,0.1911562085,-2.1766350269,-0.2334984988,0.1585360169,1.7308087349,0.2101554424,-0.4219810665,0.6413149238,3.6585171223,-1.6798986197,2.1572251320,-0.9573594332,2.1295564175,1.6554572582,1.4173837900,-0.1184742749,-1.3791389465,-0.1836668104,-2.5507338047,-1.8902881145,-0.2113366872,1.7934159040,-0.4378238618,-1.6616495848],[-0.5122920871,-1.7967692614,-1.2768464088,1.4960579872,-0.4098713994,0.2792552710,-0.6689916253,-0.2977707684,-0.5372231603,0.6325138211,-0.3380914032,0.1361745149,0.5060076714,-2.7306387424,0.5758006573,0.0367980748,0.4948950410,1.3093698025,-0.6927099228,0.5162115693,-2.8285930157,1.4057217836,-0.8556907177,-0.5703662634,0.1480660737,0.2890440822,-1.7777055502,0.6198061109,0.9095965624,0.2691595852,0.0894911662,-0.0680072829,0.8367866278,1.0884227753,-0.7422149181,-0.3737275600,-1.6788115501,0.1495314837,-0.9467903972,-0.1744271517,-0.7147907019,0.5539358854,-1.1186755896,0.6251395345,0.6120824218,0.1489314586,0.7221364379,-0.6673222780,0.5348725319,-0.0781799033,-0.4313751459,-1.4768824577,0.6255025864,0.5036774278,-2.0770540237,0.8228256106,-0.2019738853,-0.0633517280,0.7215458751,2.4272737503,-0.4849633276,0.7702387571,0.7469590902,-0.1873421669,-1.3352429867,0.8034677505,0.5133347511,0.3616528809,-0.0800107121,1.6482689381,-0.2358781248,-1.1339635849,-1.1484259367,-0.2858934999,-1.7178094387,0.0422221310,0.7282291651,-0.5086132288,0.2948485017,2.0248792171,0.9847933054,-2.2134039402,0.1210705861,0.9005804658,-1.7938214540,-1.1630740166,0.6061529517,-1.3917492628,0.0659918785,-1.5254364014,-0.8436492085,-1.4030380249,-1.1273025274,0.6605858207,0.5244283080,-0.5535778999,0.5154033899,-0.0494096726,1.9339607954,1.0602059364,-0.2275834680,0.1745944023,-0.3988296390,0.2007772624,0.4210469723,1.1622320414,-0.1475005299,1.6775492430,0.3825560212,-1.4785602093,-1.0011734962,0.9199843407,-0.3728865385,2.1775677204,1.0654608011,-0.3552238345,1.2858337164,-0.0408678167,0.7288712859,-0.6996934414,-0.9515546560,1.4982056618,-0.8414525390,1.5781154633,-0.8477112055,-0.2554596066,0.7263152599,-0.8979797959,1.9918254614,-0.0547553524,0.2416841537,-0.0709533319,-0.8204307556,-2.0357275009,-0.1745917797,1.0008513927,-1.1479756832,0.6434391737,0.3278440535,-0.8116710186,-1.0752303600,0.3474791348,0.0802315548,0.1050836816,1.6445997953,-0.6685065031,0.2889428735,0.1904595792,1.9781349897,0.0672670230,-0.0199888926,-0.3196633160,1.0699506998,0.5876069069,1.4938871861,-0.2294600457,-0.6761348248,-0.9675368667,0.3901038170,0.9939051867,0.1740347892,1.2410316467,0.2331166565,-0.4013488293,0.0780516416,0.1232619882,0.4011231065,-1.1274694204,1.7268871069,-0.3178609014,1.1595098972,0.5392588377,0.5934121609,0.0710647479,-1.4837100506,-1.5382093191,2.3655643463,0.4086492658,-1.4912976027,-0.2096750140,-1.2068072557,0.1913460344,0.9109123945,0.3377050161,-0.8817027807,-1.7224570513,-0.7705970407,-0.5596790910,-1.0413635969,-0.4826701283,2.7578041553,-0.7895061970,0.2393582165,-0.1677271277,-1.5210280418,-0.9905596375,-1.2625994682,1.8897997141,-1.3356403112,-0.2018123269,1.1257551908,-0.6290640831,0.6600694656,-0.0313108787,0.3371075690,0.8491163850,0.9079491496,0.7914938331,0.5391131043,-0.4028544128,-0.3595976532,0.9391801357,-0.9340050220,-1.5900048018,1.0865169764,1.5760774612,-0.9845061302,-0.4753736556,-0.4715046287,-1.0118333101,-0.3843485117,-0.0267880801,0.0341033973,1.6630090475],[0.7196737528,-0.9001411796,-0.1763302237,-1.8384820223,-0.5711111426,-0.6256062984,-1.0332602262,-0.2247431427,-1.4039026499,-0.6790704131,-1.2205200195,0.5722601414,-0.8030456305,-0.4784704745,-0.2942892313,0.6278070807,0.1514698267,0.4336849153,-0.3661702573,1.2590053082,-0.3364170790,-0.2155324370,-0.0285432246,0.7176227570,0.2905178368,-0.4122419655,-0.0120573184,-1.0927163363,-0.4791768491,1.0728253126,-0.4578598142,-0.1820637584,-0.6342881322,-1.2003618479,-0.5084941983,-0.1252528578,-0.5457894206,0.3418527246,-0.2050891072,-0.3844342828,0.8712399602,-0.0385699309,0.0265888497,1.3547886610,-0.3125000596,1.2956840992,0.9276458025,0.3030017912,0.9541946054,-0.7833327651,-0.9277109504,-1.5816023350,1.3954637051,-0.0899096355,0.9990671277,0.7146677375,0.2361557186,-1.3013103008,-0.1263361424,-0.2064883858,-1.6868683100,-1.2527776957,-0.4654269218,-1.3014801741,-0.1094226167,-0.1444881260,-1.4770342112,-1.9824503660,-0.3436704278,-0.8657111526,-1.2764846087,0.4240189195,0.5265209079,-0.7517572045,-0.8044191599,0.8929079771,-0.3144601882,1.0459331274,0.3026503921,-0.8141273856,1.7526532412,0.5301589966,-2.1711890697,0.2059477270,0.8248731494,-1.6328703165,0.4597865939,0.8475491405,0.2477189749,0.0925393179,-0.4595969319,-0.1853267998,0.3266243339,0.6708236933,0.6769939065,-1.0828754902,0.2627280354,1.3661319017,0.1279993355,0.9695571065,0.1432794780,1.1143633127,1.1418329477,0.1405775994,-0.4752911925,-0.0177135766,1.0221664906,-0.3322848976,-0.5167743564,-0.7432743311,0.3890959322,0.1556181014,-1.8614654541,1.6840809584,-0.7621278167,0.1582335532,-0.2684752047,0.5648133755,-0.1776323318,-0.6139817834,0.1006056443,-1.0148520470,-0.0685680658,0.9740679264,-1.0096333027,-1.2929003239,-1.1497299671,-0.2576673329,0.2650620937,-0.5298980474,0.1767422408,0.3737693727,0.0116174156,-0.8848586679,-0.2841534019,0.2786668837,1.2182451487,-0.8785102963,-0.6624893546,-0.2506814599,-1.6583607197,-0.0440883674,0.7125759125,-1.9737573862,-0.4872403741,-0.1577950418,-0.8622025251,0.1490759254,-1.1009280682,-0.7130812407,2.9556460381,1.6436719894,0.6758390665,1.0099107027,0.0647838414,0.3598891795,-1.5434740782,0.2607182264,0.7817473412,-1.9403500557,1.4225147963,-2.1519820690,0.5616424084,-0.8769695759,-0.2707855403,2.4196352959,0.2859359682,0.0165289156,-1.7499842644,0.3692513406,0.5031059980,-0.1664648801,-2.0056574345,-0.7654278278,1.0434603691,-0.6375411749,0.2848365307,0.0735133290,0.8452553749,-1.4876236916,-0.2486355007,1.4121173620,0.7177804112,-0.7401977777,0.3545803130,-0.5262724757,-2.8600194454,-1.9175211191,0.4451819360,-0.0762447342,-0.1314585060,0.8150295615,0.1831329763,-0.0879756361,-2.0604195595,0.2432845235,-0.1897565573,2.0000879765,0.2861734331,0.0464978926,0.1709866971,1.3898837566,-0.3738046587,1.2233188152,1.0814232826,-2.1520938873,1.0503945351,0.7663048506,-0.7665901184,0.5697636604,0.7073093057,-0.7511551976,-0.9094853997,-0.4420854747,0.9772966504,-0.1575781852,-0.1842168272,-0.3205026686,-0.6077862382,-0.0882692859,2.2927906513,0.9862863421,-0.2564762235,1.4920821190],[-0.2900837064,2.2671697140,-0.0333347134,-1.1995899677,-0.3519096971,0.0096649854,-1.1579176188,0.1301806569,-0.3239947259,0.2254819572,3.1275162697,-2.1915900707,1.6642601490,0.3813494444,1.1953544617,-0.1923963130,-1.3483207226,0.6332259178,-1.1693799496,-0.8710323572,2.1890964508,0.5400183797,1.4145956039,0.7931981683,0.0518532507,1.3498840332,-0.3464334905,1.8504086733,0.2446140349,0.6482831240,0.9984034300,1.3537937403,1.1133658886,-0.3245791793,1.3398704529,0.4796682596,-0.4233224094,-0.6885091066,0.1215857938,0.6505106688,-0.4377563298,0.6642935276,-0.7825295925,-1.0611126423,-1.0111138821,2.4128932953,0.8924791813,-0.4122083485,1.2046082020,0.5568127632,0.7432289720,1.3462983370,-0.4105149806,-0.0584207624,0.8289735913,0.0149429040,-0.9931199551,1.9934371710,-0.4420913756,-0.3624625206,-1.1244492531,1.4691061974,-0.0499073863,0.2002401203,-1.1248832941,-1.2726184130,0.9452376962,-0.4676527977,-2.2370703220,-2.2448759079,0.9237034917,0.4242494106,1.7204461098,-0.3976499140,-0.0906097963,1.4850599766,-0.1435298324,0.3990937471,-1.3640803099,0.1675154865,0.9451347589,0.1672854722,1.1541965008,-1.3738102913,-1.6310482025,0.4398053586,0.9419565201,0.4221067429,0.8890461326,0.9389849901,-0.5344372392,-0.2121012062,0.1934045106,-1.1489433050,-0.1732483953,1.6298263073,0.7644013166,0.0068043680,-1.4946343899,-0.1486235559,1.0825355053,1.0867308378,-1.5739576817,-1.1025074720,0.3009252548,0.1753404289,0.1530693769,1.4307221174,-0.9575708508,1.3203545809,-0.6723961234,0.4719530344,-0.2558166087,-0.4151659906,0.2092033029,-0.3377778232,-1.7142255306,-0.0780475810,0.1520982683,-0.6602406502,0.8542318344,0.0446649306,-0.1880837232,-0.8102146983,0.1999365538,-0.8471016884,0.6609727740,0.0869688913,-0.8234723210,-0.0038673137,-0.7962244153,0.4736574292,-2.1708366871,0.5728909373,0.0155894784,0.1019155160,-0.9545924664,-0.0417357050,0.0830023661,0.3825495839,-1.5725542307,-0.5406516194,1.8002802134,-1.1717652082,-0.1948368400,1.2039487362,0.5995358825,2.4622888565,-1.4696532488,-1.8450510502,-0.8768398762,0.0656154454,-0.6295272112,0.0151507873,0.9739693999,-1.3466656208,1.7466659546,-1.2967122793,-1.5379266739,0.0025664088,-0.5259938240,-0.7475365400,0.1608507335,0.2720134556,0.6598511338,-0.2686775327,0.2295012027,0.1086877584,0.2343160212,0.4769563675,-1.0597660542,-1.9808396101,-0.7161743641,0.8676317930,-0.5259642601,-1.6665190458,0.2771702707,-0.5351759791,0.5666056275,-0.0272630919,-1.2385288477,0.6036389470,0.5278122425,0.0797868595,0.0228599031,-1.9098942280,1.5135689974,-0.2744904459,0.3447202146,-0.2918914258,0.0123673119,-1.0660989285,-0.7806712985,0.4060901105,-0.3038170338,-0.5685398579,0.7254636288,-1.4262027740,0.3587500751,-0.0250621587,0.4670973122,1.7571234703,1.2794920206,-0.7825925350,1.7953877449,1.4440474510,0.2549488842,-0.5291935802,-1.8028976917,0.2191513032,0.2555356026,0.0594101250,-0.2747816145,-0.8839666843,0.5966591239,-0.2426848710,-0.2062047273,0.6629035473,-1.1385248899,0.0354150161,-1.4973627329,-0.4372212291,0.1723114401,-1.4453716278],[-2.5439577103,1.0714870691,-2.3216881752,0.6471449137,-0.2535232008,-2.4768803120,1.8743288517,0.3274333775,-1.4912362099,-0.4837066829,2.1048803329,1.4479202032,-0.6346545815,0.2597027719,-0.5986030102,-0.9634928703,1.1574920416,0.4726140201,-2.7617619038,0.4451379776,1.5339012146,0.6199848056,-0.6614613533,0.4931938350,-0.7092412114,0.4458418787,-0.7853662372,-0.2501262426,0.1611269265,-0.5285350084,-1.6091034412,-0.1904814690,0.5117421150,0.3120868802,-0.4000595510,-0.7666798234,0.9289351702,-0.2531622946,1.1696226597,-0.4199027717,0.3557711840,-0.4890812635,-0.5624075532,1.2869091034,1.1910554171,-0.3527726829,1.6044161320,1.2182590961,-0.0506892353,0.2301639915,2.1127395630,-1.8038836718,-0.1752304137,0.3366157413,0.1188599542,-0.3038607836,-0.6614480019,0.6676669121,0.8247133493,1.1330426931,-0.3428809941,1.3835335970,0.7142785192,-0.2841734886,0.8137839437,-0.9763744473,-0.5805509686,0.8007320762,0.7538213730,0.6759210825,0.6667888165,0.1327999085,0.5388278365,-2.5514461994,0.2968763709,0.0856194794,-1.2785900831,1.3809236288,0.1545603722,-1.6245212555,-0.9858641624,0.0683231354,0.9755828381,-0.1573059112,-0.5441203117,1.4863932133,-0.3820349276,-1.8454742432,-0.9734540582,-1.0773816109,-1.5850419998,-0.7565042973,-0.8189355731,0.4546409547,0.1463429332,0.2750940919,0.3259109557,-1.6890685558,0.6469773650,-0.5459377170,-0.3998305500,0.8339851499,2.4585688114,0.1916992962,1.0748101473,0.9357467890,-0.0929414183,-0.6584252119,-0.3570420742,0.0665325448,0.7298585773,-0.8224898577,-0.6889728904,1.1683588028,0.0241510794,0.6844079494,1.1921299696,2.1824836731,0.4918388128,1.9142444134,0.4802091420,0.7324468493,1.3169702291,-0.0847907364,-0.4453571439,0.3699448407,-1.0258462429,-0.3116862476,-0.0516151153,0.7953120470,0.6979629397,2.1893255711,0.3988084495,-1.1665880680,0.6708530784,0.6878688931,-0.8248539567,-0.5847064257,0.5866395235,-0.3840931952,1.4220393896,-1.1018649340,-1.1234416962,0.4223946333,-1.5349240303,-0.0431141518,0.2294328213,-0.6951954961,-0.4299486279,0.9124315977,-1.1123651266,-0.0226851795,-0.6936195493,-0.0072121215,-1.1652355194,0.1803177744,0.5420455933,-0.1974164546,-0.1034566686,-1.1659656763,1.4620234966,0.0676573440,-0.1860376447,-0.0255707297,0.5868535042,-0.7124443650,1.8730525970,-1.4280098677,-1.2111260891,0.0161925871,-1.0231288671,1.1533496380,0.0843438879,1.8062423468,0.4847639501,-0.1865802705,-1.1985164881,-0.7404387593,0.4595836401,0.3563097715,1.2884920835,-0.1245478317,-0.0213925559,-0.2822938859,-0.7291942835,-0.8705769777,0.9221906662,1.9151195288,0.2899247408,0.1849166900,0.0617027730,0.9693223834,1.1434739828,1.3673071861,1.3153882027,0.0179758314,-0.6445850730,0.1999620199,-0.6685073376,2.1232419014,-0.5719731450,0.6747733951,-0.0902681053,0.5864809155,-1.6446373463,0.9344949126,0.1072397977,0.9640066624,0.2265440077,0.1202564687,1.8057010174,1.2321231365,0.1216363013,-0.8128648996,-1.1337174177,-0.4002026916,-0.3188920021,-1.5820558071,0.6039871573,-0.9864867330,0.6012089252,2.4563450813,-0.5732899904,0.9521592259],[0.3260547221,1.8853037357,-1.3204982281,2.0496821404,-2.2340407372,-0.6887130737,-2.2608993053,2.0778744221,2.7424345016,1.0892106295,2.5908606052,0.7477040887,-1.4859695435,-1.3533649445,0.5172604918,-0.2675825357,-2.2346560955,-0.7207927704,0.6210254431,0.4351610541,-1.5440757275,0.7776510715,0.8183146715,-1.9388126135,0.6442348361,0.1557279527,1.6979936361,-0.4870493710,-0.2069469690,1.2790093422,0.0868159607,1.1734422445,-2.3761756420,-0.2298842072,1.1400439739,-0.6254145503,-0.4515355527,-0.3749254644,0.8611729741,0.4854235649,-0.1822100878,-2.0930531025,0.2222319990,-1.4868695736,-0.3692798316,-1.3354876041,-1.3781586885,-0.2719875872,0.9334027171,0.5190452933,-0.8337520361,0.4389167726,-1.2404805422,-1.3586823940,0.8238016367,0.0736838952,-1.8239998817,0.6703809500,0.2388516814,-0.8363975883,-0.5493038297,1.1531976461,-0.2203984410,0.8369795084,-0.3004584610,1.3316445351,0.2204501927,-1.3941072226,0.5796142817,-1.4743403196,0.2360322624,-0.0470898226,3.3544440269,-0.0383296460,0.5620162487,0.3400013149,1.6698427200,0.3058384955,1.0403901339,0.7896620631,2.2139503956,-0.8617098927,0.0116197933,0.2294577211,-1.6858422756,0.8578336239,0.0620743819,-0.1545120627,0.3085938692,0.7593377829,0.6065549850,-0.5367869735,-0.5062915087,0.9420825839,0.3617729843,1.5440033674,0.1615864933,0.0659199730,2.2409822941,1.7344744205,0.8405517936,1.2798552513,0.2383038104,-1.1642180681,1.2421092987,-0.6393301487,-0.7457547188,1.6431002617,-1.1643340588,3.0044844151,0.3767578304,0.5810221434,-1.0409605503,-1.0302780867,1.0567331314,-0.5488164425,-1.0528279543,-1.1218669415,1.9058232307,-0.3550024033,0.3868098855,0.3766503632,2.7125041485,0.2050943375,-0.1935515255,-0.2608304918,-0.7655370235,-1.3006209135,1.7410725355,0.3444109559,0.9625317454,-1.5197042227,-2.2566416264,0.1158713400,-0.3307718933,0.0338330679,-0.2279780209,-0.6273221374,0.5892687440,0.0315760486,-2.4211452007,-0.5741143823,-2.9741172791,1.8141866922,-1.2270373106,-1.7089072466,-1.2472568750,0.2550107539,-0.2996766865,0.6542093158,1.1454129219,0.3152971268,0.0964322165,0.3907919824,-2.0905296803,-0.6558496952,-0.3131229579,-0.4145734608,-1.6045930386,-1.1430335045,-1.1225223541,0.5742155910,0.2588180602,-0.8291150331,0.3271596730,1.0199117661,0.4195581079,-1.2312644720,1.2604261637,-0.1361154765,0.9647939205,-0.8266542554,0.4887631536,1.7701907158,0.7247890234,0.4453731775,-0.8973620534,0.3199851513,0.0539685972,-0.8106490970,0.0733280629,0.0133526186,-0.6579945087,-0.3815068007,1.3540554047,-1.3347084522,0.4231979549,-0.3235702217,0.0383079089,0.2308514863,-0.9960447550,0.0932694674,0.1048167944,-0.6200649142,1.2804870605,-0.0674211308,0.6397436261,-0.1728785187,0.1322377771,0.6113134623,-1.3866189718,0.2125978023,0.0523288436,-0.3402370811,-0.2536427975,1.3289234638,-0.4033775330,0.6017901897,-0.8228362799,-0.7473247647,-1.4831603765,0.3024454117,0.1503181756,-0.6277262568,0.4740160704,-0.1196713299,1.7544775009,-0.0974104479,-0.1266327649,0.9487879872,-0.4919475019,-1.5392595530,0.6162174344,0.5011700988],[0.9101887941,0.8777204752,1.2595125437,0.4018152654,-1.7166287899,-0.8698635697,0.4974375665,-0.0144640859,-1.7648240328,0.9489676356,-0.4182487428,0.4941950142,-1.1213845015,0.2394562066,-1.8312802315,-1.4354618788,0.9216833115,-0.8544840813,1.0875998735,-0.9171810150,-1.1399039030,1.3363617659,1.1059136391,0.8108481765,0.2172601074,-0.6312165856,0.6373825073,0.4367862642,-0.8210284114,-0.7013685107,-0.4239511788,0.5677524209,-0.2321190834,0.2549104989,-0.9880585670,1.1797896624,-1.8412686586,1.3705574274,-0.4650182724,2.6413793564,-0.4931051433,0.0091245165,0.2830466032,0.3595472872,-0.4690435529,-0.6955348849,-1.0818399191,-1.2531290054,-1.5354416370,0.6554519534,1.2130562067,0.1278675348,1.0614162683,0.0805788711,0.1987828314,0.7905623913,-1.1146622896,-0.7665066123,-0.4313217998,0.2837044001,0.1385461688,-0.6566702724,-0.7525277734,-0.3352095187,-0.0488350540,0.6274835467,-1.3588238955,-0.6190199256,0.0382770114,0.0188390687,-0.3117430210,0.7832928896,-0.0662060976,0.0303651243,0.4428240955,1.1008049250,0.4989125729,0.3406849802,-1.3838136196,-2.5580713749,0.1072607487,-1.7010232210,0.5015914440,0.3309070468,-0.2240238488,1.1814460754,1.0675925016,0.2472229749,-1.1753770113,-1.4115841389,0.6599791050,-0.8903461695,-0.7513751388,0.4441344738,0.0984115377,-1.6328358650,-2.7814333439,0.0352627784,-0.1952062249,2.5094332695,0.2350155562,-0.7036231756,-0.6226969957,0.1006892025,0.2735290527,0.3248717487,-0.9859005213,0.5884060860,-1.6339645386,0.6399517655,0.0949723125,-0.8603503108,0.7330551744,-0.4018296599,0.0621890053,0.9158736467,1.1411347389,0.4787520468,1.3159184456,0.6555212736,-0.1050648540,0.5479673743,0.1422740966,-0.0826835856,-1.1075340509,0.2643381357,-0.1198114604,-1.2939515114,0.4103282094,0.9645326734,-0.5595097542,-0.3438761234,0.9068605900,0.7637982965,1.2814041376,-1.3839570284,-0.5809727311,-0.3262540698,-1.5268710852,1.2061607838,0.9503488541,0.0565067790,0.5170537233,0.8136867881,0.9898092151,-0.5544456840,-0.2870925963,-1.6100064516,0.2155759931,-2.0440261364,-2.6521677971,0.6410305500,1.6874045134,-0.6198192239,0.6183931231,-0.3552034199,0.3808440566,-1.5048646927,-0.7682568431,-0.3286943138,-1.2153249979,-0.8321967125,0.6857488155,-0.7768926024,0.4683583379,1.4393649101,-2.0580306053,0.2417555600,-0.1287895143,0.9849979281,-1.2673319578,1.4937902689,0.6139543056,-0.7011052370,1.6358752251,-0.5591640472,-1.8699896336,-1.2037640810,-0.4650449753,0.4130979478,-0.4781028628,-0.3773052096,0.4896984100,0.5291000605,0.2909781933,-0.1239582077,-0.2593092024,-0.7551556230,0.0570368096,0.7843011022,-1.3545408249,0.3017587662,0.3814600408,0.1865427494,0.3018524945,1.1108403206,1.4385936260,1.2737234831,-0.1687798649,-0.2700041831,0.0939677283,-0.5982590914,0.0442983992,-1.2631759644,2.1559460163,-1.0446652174,1.4373403788,-0.5136585832,0.5201707482,-0.1930320710,1.4882138968,0.1718825847,-0.6136702895,0.1616206765,-0.1911924034,-1.3606525660,-0.6224917173,-1.8901438713,2.1093156338,-0.5344481468,0.5858753324,-0.1767421663,0.5121246576,-0.4941667914],[0.4248593450,0.3161675930,-0.8331077695,-1.4626350403,-1.7189588547,-0.2488492578,-2.3246755600,0.8617739081,0.4979015291,0.7135391235,1.3816496134,1.7785661221,0.1185933501,-0.0692301467,1.3845703602,0.9387071133,0.1926213652,0.1305808425,-2.9852871895,-0.5626637936,-0.8844199181,-1.3015922308,-1.2187414169,-1.1000576019,-0.3812155724,0.7491910458,-0.7949401736,-0.1017555743,0.5167959929,-1.2674736977,-1.3182394505,0.3694709241,1.0132818222,2.0066077709,0.0588135272,-1.0413175821,0.4451495111,-1.5961176157,-0.5127106905,-0.0736058503,1.2943279743,0.9867642522,-0.3749952614,0.0219509210,-0.4264596999,0.6405826211,-0.2793408334,0.3446552455,0.2953101695,0.4174514413,-1.3424669504,-1.2077305317,-0.5029827356,-0.9704607725,-0.5791391730,-0.3410248160,1.2203289270,0.3139457703,-0.6450718641,1.9737294912,0.0937890559,-0.7589262724,-0.0004526210,-0.1392761469,-0.5184856653,-0.6445096731,-2.0933535099,3.6174440384,-1.4014939070,-0.3285408914,-0.0851450935,-0.4968545437,0.8777781129,1.2304735184,0.9676017761,0.2992458940,-1.0884692669,0.7705311775,1.2788481712,-0.4678036571,0.8469130397,0.5697970390,0.3013136983,0.7260673642,0.2109190971,0.9467196465,-1.5477573872,-0.7547515035,1.0165404081,-1.2013443708,-1.0009258986,-1.0381124020,-0.9545734525,1.0061957836,-1.3585246801,0.2498244196,-0.6100196242,-1.9319077730,-0.3646067679,-1.4917006493,0.7829834819,0.7965960503,-0.8203605413,0.7800762057,-0.1055379063,0.3846244514,-0.9033451080,0.9355486035,1.3010201454,1.5083156824,1.0601898432,-2.1822016239,0.0641560182,-0.1708367616,2.2009952068,-1.4505349398,0.6033412814,-0.3475028276,-0.0245241802,0.9902088046,0.7871859670,-0.2820515037,0.6837606430,1.7600520849,1.7086303234,-1.3337498903,0.5148547888,-0.1411535144,-0.5787584782,-0.1753298044,0.7662317753,2.2890543938,-0.3373456299,-0.7940471768,0.0870957598,0.2046343088,-0.3655799031,0.1403742582,0.7487806678,-0.0493908785,-1.0613024235,1.6651679277,-0.0721103400,-0.9661730528,-0.0118546663,-1.2443343401,-1.8370332718,-1.1943985224,-1.3811161518,0.0265875105,0.7113240957,0.4174372852,0.7321901321,0.3165853620,-0.8200309277,0.9104493260,-2.0742957592,-0.8789162636,-0.6044740081,1.7389211655,0.3027465940,0.8017164469,-0.1329290718,1.1411834955,-1.2318832874,-0.2818616629,0.3668227494,-0.1943957955,0.1467296183,-0.9400892258,0.6216479540,-0.3850940168,-0.2756251991,-0.0756106824,0.6708564162,1.7877844572,0.2348715514,-1.1913688183,0.1658644378,0.3144343793,-1.8043081760,0.8749025464,1.1378152370,-2.5943167210,-1.2024546862,1.2210763693,1.1203122139,0.1245990917,-0.9790798426,-1.1268764734,-0.4267218709,-1.6712464094,-0.2931473255,0.9772987962,-1.3596971035,0.0209441744,0.3558107018,1.1550899744,-0.5986868143,0.3185078800,0.6811698675,-0.3164308965,-0.1549912542,0.0994456187,-0.1038790345,-1.7121478319,0.2452230304,-1.0489326715,-0.2093195915,-0.4916971028,0.8829078674,-0.1305598617,0.0435559079,-1.4980833530,-1.2615302801,0.0852286965,0.5818434954,-0.8840133548,-0.4041324556,-1.0049831867,-0.5677416921,-1.8189404011,0.6888154745,0.4762073457],[-1.6676757336,-0.0117815798,0.9636470675,0.2832666636,-0.3563674390,0.9574931264,1.1856490374,0.1930582970,1.1372985840,1.4829488993,0.4178263247,-1.0189751387,1.3939164877,-1.5043624640,-0.2485114038,-0.4893157482,-1.0204097033,1.2906845808,1.1540821791,-0.4610202312,-0.4744220972,-0.0785018057,0.5601623058,0.2905760109,1.2008045912,-0.9055359364,-1.2546924353,-1.3627620935,-0.4191326201,-0.2417927384,-1.6857676506,-0.6772042513,-0.7309972644,1.0813498497,-2.3207206726,0.3441968262,-0.6235502958,1.6584020853,-1.7576291561,-0.8164684176,2.6233417988,-1.4983351231,0.0849152654,-0.2290281355,-0.6555573940,-0.1725483537,-0.3922250867,1.2119016647,-1.9553462267,-0.6105675101,1.1522092819,0.6248836517,0.0559881888,0.4049755335,0.0688843802,-1.0096157789,0.3500415385,-1.3688995838,-1.1148210764,-0.2586264610,0.0292662345,0.2617202699,-1.1088030338,-0.2185048759,0.1528363377,-0.1823088825,-0.7560430765,-0.8633227348,1.8211431503,1.2482187748,-1.0431731939,-0.1619359106,-0.2505675554,-1.1278392076,1.4434571266,0.1162277162,0.8356947899,0.2004559189,2.7481603622,-0.6122629046,-0.1336817294,0.9449667335,-0.2098537385,-1.0220044851,0.1911619157,-0.6354675889,-2.6473999023,-0.4538244307,-1.0034236908,-0.4797924757,0.3036291897,-0.7502538562,0.3391863406,-0.3267144859,-1.5663520098,1.5844787359,-0.1541496515,0.0080233496,-0.2457676530,0.5802427530,-0.2375820726,0.4829804897,-3.2749388218,0.6998037100,-0.8014594913,-0.9856346250,-1.3622802496,0.9920208454,-0.8667515516,-1.1467056274,0.8337497115,1.4432774782,-0.4017332196,-0.1752787232,-0.1312951446,0.5203475952,1.6061456203,-0.3765571713,1.2273452282,1.0681328773,0.5741702914,1.6537784338,-0.1392436624,1.5222073793,-0.4666440189,0.8264855742,0.2139969468,-0.7227970362,0.1318987459,0.6431787014,0.0269627497,2.9194891453,-1.3445217609,-1.3357516527,-0.3646216989,-0.3339246511,-0.9061585069,-1.7167688608,-0.2931183875,-1.2002370358,0.0027208761,-0.8410323262,-0.8513520360,-0.8169867396,1.7177858353,-1.0786051750,-0.9716501236,0.7862368226,0.4703477919,1.5872616768,-0.6943163872,0.5795159340,-0.4428988099,-0.0336347744,-0.0675015002,-1.1188058853,-0.0059059928,0.3079329729,-1.5865097046,1.1954914331,0.0107388850,0.2328452766,0.3771139979,0.1556452364,0.5254788399,0.7656033635,1.8018838167,1.3587571383,-0.1060156971,0.5899567604,1.1444541216,-0.3648647964,0.3766710162,0.0900015309,-1.0156743526,1.4167870283,-1.1703493595,-0.6448116302,-0.1051116288,0.0020963072,-0.0745361373,0.8696280718,2.8341960907,-0.3513678610,1.3280119896,0.1946055442,-0.5135858655,-0.5002872944,-0.3048385382,1.4506249428,0.0044972659,-0.0088738408,0.6910936832,0.6415634751,-0.6344572902,0.2697565854,-1.0585714579,-0.5347191095,0.4512789249,-0.7875731587,-0.1061345711,-0.3123532832,-0.6738702059,0.7054605484,1.8631844521,0.6854217649,-2.4223566055,-0.3866092861,0.4214096665,-0.6705033183,0.6293103099,-0.9606555700,0.4024822712,0.9286620617,-0.2251343727,0.0375308469,-0.6891617179,-0.4474659264,-0.4347704947,-0.2488876730,-0.6395900249,-0.6902248263,-0.8352594972,-1.1157217026],[1.1579307318,-1.0021015406,-0.9917744398,-0.5251912475,0.3097694218,0.9646864533,0.4288187623,-1.4920967817,-0.5031630397,2.0002958775,-0.8010430932,-0.1773892045,0.7078548670,0.7890493274,-0.7094715238,0.0394469835,1.5130943060,1.0473628044,0.7980496287,-0.2446120083,-0.8357499242,0.3035597503,-0.0023617276,1.8378235102,1.0127995014,1.0019253492,-2.0041329861,0.5986630321,-0.8298110962,-0.4977650940,-0.7103406191,1.1140469313,-1.0463049412,-1.1186000109,-0.6619963646,0.4847993851,1.1088175774,0.1798587739,0.6712608337,1.7970820665,-0.7250568271,0.5614169836,-0.3647833765,0.2266235948,0.4138481319,-1.9311686754,-0.9302152395,1.2558653355,-0.9853758812,0.7505119443,0.8275774121,-1.0908306837,1.4267879725,2.1317381859,-0.0587358475,-0.6239714026,0.5322213769,1.2944272757,-0.5415900350,0.3425603509,-0.0551128164,1.2260411978,-0.3073526919,-0.0158004314,0.0446104370,-0.2874431610,-1.3118320704,0.3087975979,0.4541290104,-1.3672093153,1.0977481604,-1.3857321739,-0.3321656585,-0.7112305164,1.7794216871,0.7596542239,0.9350003004,-0.6077075601,0.3117588460,-1.4983228445,1.2511718273,0.2724351883,-0.1407505125,-0.5702785254,1.9123885632,-0.0837195367,2.3620803356,0.5817832947,-0.6662903428,-1.4938615561,2.1392271519,-0.7084274292,0.3544390500,-0.1004240811,-0.7890850902,-0.6086986065,0.4826420844,0.5797073841,-0.9093092084,0.5444802046,-2.0700707436,-0.4536957741,-1.0652302504,-0.5660454035,-0.6453261375,0.5172896981,-2.2897562981,-0.8132987022,0.9573528171,0.1642129719,-1.1041778326,-0.2960029244,0.8012885451,-0.6051740646,0.2041897923,-0.2072525769,1.1589853764,-0.4625106752,-0.4647492766,-0.1737691313,1.1656908989,2.6465435028,0.7356799245,-0.6035447121,0.7761955261,0.1258596480,1.3203146458,0.0428092740,-0.5732164979,0.5334172249,-0.4383377433,-2.6443896294,-0.4882341325,-0.4131014943,-0.6506836414,-0.0589487515,-1.1589503288,0.0197763573,-0.4185853302,0.7655056715,-0.3029468656,0.7333875895,0.5726686716,-0.2836067080,0.0208763424,-1.9726077318,1.9458686113,0.8031986356,-0.5565538406,0.3064076304,-1.3078083992,1.3880399466,-2.4986119270,0.2955805659,0.4166693389,0.3991940618,-0.3132807612,1.9816641808,1.2151714563,1.7478718758,0.2781778276,-1.2538698912,0.3590858877,-1.6911876202,-0.3242392540,0.0751941875,2.0730650425,0.8241178393,1.5471822023,0.4346613884,-0.2523943782,-1.6861196756,0.3959576786,-0.3940330148,2.2799172401,-2.1057817936,-0.6796046495,-0.2385020703,-1.6790004969,0.6090376377,-0.2106702179,-0.5197761059,0.8246834278,-0.6630619764,-0.6153048873,0.3878263831,-1.6378360987,0.2738104463,-0.6956075430,-0.6798497438,-0.0288074967,-1.8039088249,-1.1410199404,2.5300855637,-1.3932182789,1.1908702850,-1.0227982998,-0.9062312245,0.8252307773,1.0381580591,-0.2450716347,-1.2224311829,-0.5532920361,0.0643740818,0.4217404425,0.9086120129,1.4833407402,1.8077830076,0.1879525930,-0.2831076384,0.8077802658,-0.0759647638,0.7265745401,1.4762508869,0.3692570925,-0.8803898692,0.5382549167,-0.4830761254,1.4964990616,-0.4836547375,0.4294926524,1.5498590469,-1.8430325985,-2.7364418507],[1.1671922207,-0.4623150527,-0.0125561962,1.8133157492,0.4415445030,1.3514368534,-0.6283529401,0.9533058405,-0.9504081607,-1.3233529329,0.0559342392,-0.3043320477,-0.6132194996,0.2827467918,0.0211385582,0.6981754899,-0.3219046593,-1.1821082830,-1.6019649506,-0.7358622551,-1.5021909475,1.3134816885,1.0149642229,-1.4239060879,-0.4501751363,1.4480149746,-1.8850461245,-1.0855251551,-0.0552835427,1.1126552820,-0.3457433283,0.3627731502,-1.2939021587,0.0297653861,-1.0057828426,1.3407930136,-0.0574732423,-0.0764587745,2.4194180965,-1.4760574102,-0.5405495167,1.3480896950,-0.9823193550,1.7727591991,-0.7762312889,-1.1459913254,-0.9690763950,0.3205409050,0.2968346179,-0.0867063925,1.4222966433,0.0621957928,1.0828131437,0.5762202144,1.8638826609,0.0221075136,-0.4786196649,-0.5847828388,-0.0353326574,1.8716874123,0.3440319300,1.1833447218,-0.8264019489,-0.4332354665,-1.6377058029,-0.2957936227,0.4296649694,0.0330034830,1.4408249855,-0.9070200324,-0.2233535200,-0.5568699241,-0.0201630127,2.8606996536,0.1218447164,-1.1993663311,-0.1101554483,-0.0428591706,0.5230171084,0.9578502774,0.3164121807,1.4629195929,0.7499135137,-0.6314778328,-0.8180010915,0.9594846964,-0.0734098107,0.3534791470,0.2100719959,-1.3744202852,0.9371819496,1.8197331429,0.0385162905,-1.4419538975,0.4320627749,-1.0061644316,0.3089143634,-0.1311210543,-0.8641072512,-0.7125442624,0.9392834306,-0.6008676887,2.2745432854,-0.0422098339,1.4438676834,0.7602827549,1.7542577982,0.3981570899,1.5356993675,0.2983169258,2.0158410072,-1.3415718079,1.8989692926,1.7546209097,1.2540712357,-1.6169677973,-0.8762540221,0.9787555337,-0.6943745017,-0.0487157442,-1.8384969234,-0.0759068280,0.4606977403,0.4347637594,-0.4993920028,0.2148143649,-2.0123169422,1.6227184534,-0.2299919575,1.4253363609,-0.3838140070,0.2162957191,1.7839958668,-0.6574246287,0.2178135365,0.4388503432,0.5152842999,2.0197911263,0.3589576781,-0.1519799829,-1.3294918537,0.3256805837,-0.3265584409,-1.4352842569,-1.0348974466,0.6977605224,-1.4681086540,-1.1425429583,0.6803500652,0.4048239589,-1.0909260511,1.1290786266,0.8217989802,-0.9025979042,1.1533937454,1.0704193115,-0.4884494543,1.4770652056,0.0373433903,0.0077836541,0.6826797724,-0.8579919338,0.2251290381,-0.9385836124,0.7698971629,-0.1225737706,0.1218759790,-0.3805133700,0.0967485979,-1.8844478130,2.3024020195,-0.6849756837,-0.3304445446,-1.2974334955,-0.6761263609,-1.4177010059,-1.3832147121,-1.3262499571,-0.3938881457,-0.6269910932,0.2355906516,0.5818787813,-0.0156290065,-0.4996424615,1.8249180317,-0.2702373266,-0.0378957987,-1.3491808176,0.8316279650,-1.3577632904,-0.7994315624,-0.3274593353,-0.8930548429,0.6743643284,0.5958985686,0.0216204133,1.6101511717,-0.9585517049,-0.3917265534,-1.8435053825,-2.2900974751,-0.6511493921,2.3449501991,0.1335494220,-0.1154850870,-0.0498552248,-0.5847663879,-0.7291586995,0.2750283182,0.7288957238,-0.2256766409,0.7525697351,0.9032928348,1.1697378159,0.3740186095,-0.6911426783,0.3261582851,-0.2271312475,0.4647946060,0.0876392424,-1.1337498426,0.8469501138,0.7095554471,1.2220104933],[1.0301719904,-0.7379205823,0.9471271634,0.3801421523,1.1546025276,0.5475896597,0.0115259765,1.3954277039,-0.7114993334,0.9818466902,0.1740345955,-1.6722321510,-0.5188888907,-1.2580188513,-0.0774400756,0.3143561482,-0.5942593813,0.4429920316,0.6551231146,1.8638199568,-0.8231512308,0.9583299756,-2.8441357613,-0.4623622894,-1.1016976833,-0.3292089701,-0.3490474820,0.7918364406,0.1010522470,-0.5955529213,0.8099021912,-0.6571288109,-0.5177548528,0.7080375552,0.4156391323,0.6487434506,1.0146337748,-1.5045164824,-0.4959926903,0.1978851706,1.0364204645,0.0211838745,1.3547075987,1.0349222422,0.2016938329,-0.7771456838,0.3141413033,0.4111408889,-1.7946816683,-0.8617743254,0.4630215764,-1.4796572924,0.2156282216,0.0648754835,-1.1287294626,0.8153375983,0.8462402225,-0.9069088697,-0.2586428821,1.5503832102,1.2572857141,-0.8906383514,-0.1220265105,-0.3822940588,-0.1107923314,0.3378605247,-0.4759807289,0.8223170638,0.5999978185,-0.5032872558,0.7468966246,-0.5116183758,-0.6416741610,-2.3637897968,0.5536193252,1.0693031549,-0.1984776855,0.5212368965,-0.1010358110,0.1642877162,2.5420272350,-0.8539747000,0.0189713798,1.4053024054,-1.0632971525,-0.9598603249,1.3750352859,-0.4135048091,1.8846297264,1.1503682137,1.0024423599,-0.0212843530,-0.7975702882,-1.5567365885,0.1432193369,-1.2462441921,0.1648118943,-1.9669402838,-0.0362720527,1.8989335299,-0.7362707853,-0.0263065547,-1.3074949980,-1.6733379364,0.4589193165,-1.5560327768,-0.2851602435,-1.1524580717,-0.7894440293,0.2129414976,-0.0265334882,-0.9173115492,-0.1233007684,-2.1116597652,-0.2548934221,-0.1000629812,-0.2341962755,-0.5075214505,-1.0097604990,1.6759552956,-1.1805329323,0.3522910178,1.1721547842,-0.8795605898,-0.2225883007,0.3419721127,-0.5531402230,0.5414720178,1.1503173113,0.8090146184,-1.1797549725,-0.4781074524,-0.4606604576,-0.6850347519,-1.0958294868,-0.8130276799,-0.4801674187,-0.5496889949,0.1198312268,0.9770823121,1.0006096363,1.5605578423,-0.2776606977,1.5436390638,1.3431284428,-1.0694115162,0.0319580436,1.0235497952,-1.6327320337,0.2094062716,-0.2779122591,-0.2549938858,0.9235299230,-1.7200658321,1.3078080416,0.3141606450,-0.9713298678,0.6507166624,0.5240988731,-0.8743537068,0.2245029956,-1.6577179432,-0.8995621800,-0.5963401794,-0.3028064370,0.6373192072,1.0106426477,-1.1714172363,-1.1551836729,-0.1290144324,-0.4418173432,-0.5398723483,0.7920086384,-0.1006233692,-1.2925938368,0.4630675614,0.5318786502,1.0925804377,-1.8792281151,1.2360200882,-1.0882061720,0.2990363836,2.0535075665,-1.3437252045,1.7961989641,1.0472359657,-0.6580789089,0.3637872040,0.8887823224,-1.0567698479,-1.4380348921,0.3630021513,1.0801453590,0.0848029777,0.3457524478,-0.5923113227,-1.1642155647,-0.6858039498,0.2838497758,0.1613612771,0.3430170417,-0.0750150234,-2.2998411655,0.1167422906,-2.1216418743,-0.7399117947,1.0929222107,-0.8083111644,-0.4524880946,-0.5343817472,-0.2154788822,0.5326138139,-0.4642713368,0.3502049446,0.2950757146,-0.3348174989,0.6952251792,-0.7663141489,-0.1191459745,1.8016871214,1.1503630877,0.0879920721,1.6557919979,-0.9713310003],[-0.0952440724,0.0566643141,-0.4121611416,0.3589834869,-1.2417886257,-0.3981787860,-0.0955326259,-1.1746367216,-0.0903539136,1.8228377104,0.1594277769,1.3675057888,-0.8630368114,1.4712395668,0.1054009870,-0.2368630320,2.3829298019,-1.6929466724,-0.7913834453,-1.4271126986,0.2466607392,0.1234057769,-0.3978918791,2.7214033604,-0.6303295493,0.0198053010,-1.1812092066,-0.1202231199,0.3901976645,-1.0169278383,-0.3118689060,-1.2093325853,1.5801656246,-0.1262614280,-0.2772842050,1.7430753708,0.7829131484,-0.9974278808,0.5059066415,0.2753480971,-0.8170706630,1.4233903885,1.3587424755,0.3708223104,0.2664794624,0.0544346385,1.0021533966,-1.4790313244,0.5714965463,0.5185027122,1.2404508591,-0.3423627019,1.0433201790,-0.5620381236,-0.7999502420,0.3974117339,1.3396379948,0.5925657749,0.5062606335,-0.9846734405,-1.2468835115,0.1999926269,-0.7213277817,0.5002495050,-0.7886040211,1.4698174000,0.5244092941,0.2752928734,0.4098138809,1.4521061182,-0.7069355249,1.2058763504,1.6311770678,0.2563337386,0.5263522267,0.1898272932,0.8347653151,1.3658587933,1.1031242609,-0.0868932381,-1.0478790998,-0.6240808964,-1.3151839972,-0.6831984520,0.8852385879,1.6934367418,-0.3677411377,-1.4583725929,-0.6255928874,0.8111643195,2.1374673843,0.7400379777,0.1656147391,1.7086645365,-0.0734900981,0.8953284621,-0.6813739538,-0.0614439286,-1.2261948586,2.0596497059,-1.5261852741,0.6271134019,-0.3758432567,-0.9482771754,0.0166640133,-1.3925417662,-0.6260951161,-0.0622643456,-0.3813502789,0.5994868875,1.6862668991,0.5603588223,-0.0449138060,-0.1375849396,-1.6965512037,0.3292586803,0.3680608571,1.5089581013,-1.2935491800,-1.4540292025,2.0181722641,0.1847225428,-1.5040644407,0.8418682218,0.3571032286,0.1277844757,0.6452940702,0.8302066922,0.3316073716,-0.4937435985,0.1727012694,0.3508908153,0.4942966700,-0.1890991032,0.1464315653,1.2672983408,-0.5420016050,0.3489682674,-0.1656922847,1.8272436857,-0.0872897953,0.6986168027,-1.1467864513,-3.0869462490,-0.6514915228,1.3085490465,-0.1464149803,0.5489206314,0.3809079528,0.1223399192,-0.2563851178,-0.3303251863,-0.8524194956,-1.0438624620,0.9942628741,-0.7164640427,0.2523762882,1.8409081697,-0.0395202041,-1.9294208288,-0.4429220259,-0.2262780219,-0.6784824133,-0.4743759632,0.1318203062,0.1481006593,-0.0424164087,-0.5459465981,-0.7982281446,-0.4270841181,0.8304693103,-0.3444064856,0.0296886452,-0.0255151410,0.2025533468,0.6004497409,-0.5635874867,-1.3864352703,-0.5710804462,1.3912957907,0.1319333613,1.6396538019,-0.8336297870,-0.6034071445,-1.9802199602,-0.0118512791,-0.1108769551,-0.0129075134,-0.2374236435,-1.5954982042,-1.3275305033,-0.4772657752,-0.2762099206,0.8561545610,-1.0112084150,-0.6464535594,1.1537920237,0.2840941846,0.1511519849,-0.1263342947,-0.3606207073,-0.9840847850,1.4167820215,0.6281455755,0.8761416674,0.0240065102,0.0428546667,0.7045249343,1.3199563026,-0.2455899715,-0.4333233237,0.0839606822,-0.5357459188,-0.5189141631,0.0773424804,-2.4033913612,0.5579027534,0.2220277339,-0.1305405945,1.1741777658,0.1842390150,-0.1543905437,0.2372447699,-0.0578634739],[1.1268147230,-0.6168632507,-0.6207220554,-0.3247307837,-0.6050617099,2.4619061947,-0.5523837209,1.7028896809,0.0241982024,-0.3090949655,-0.1183786169,0.6874657273,0.5977433920,-0.3424105048,1.0477715731,0.0600404032,-0.4835692644,0.2377821803,-1.9360288382,-1.1095243692,-0.2601641417,0.7298178673,-0.9078957438,-0.7949860692,0.5281986594,-0.0193301439,-1.4828484058,0.4641183615,-0.6966081858,0.6504861116,-0.7225518227,-0.0420571826,-2.2333300114,-0.6292539835,0.3642581701,2.1744987965,-1.4005479813,0.6594906449,0.0190161932,0.7265573144,0.4081478715,-2.0283536911,-0.2277533710,0.2533299625,-0.2692781985,0.8519235253,-0.9961074591,-1.3291726112,1.1561888456,-0.1695564687,-0.6412171721,-0.5963202119,-1.2543619871,0.3706256449,0.2367728949,-0.8329231739,0.1212827340,-0.1572631747,0.4193502367,-0.5477141738,0.4441763759,-0.1068404242,0.6890825033,1.2757037878,-1.3237390518,-0.7205080986,-0.2300838381,1.9304093122,-0.8222891092,2.3237411976,-1.5258569717,-0.2129818201,1.1545015574,0.1849959046,-0.4873098731,0.5797542930,0.4350897372,-0.4880842566,1.4818840027,0.7041978240,-0.0506795347,0.4129129052,1.2196661234,-1.2100301981,-0.6907811761,0.2639520168,0.6720523834,-0.1672959328,-1.3558725119,1.5750267506,0.5315433145,-1.3783342838,-0.5600621104,0.1594666988,-0.1252618730,0.1798083037,-0.3277032673,-0.8285724521,-2.2868967056,-1.3793725967,-0.7088655233,-2.1242513657,1.8299680948,-0.0487202220,-1.1619849205,-0.7737154961,1.1845551729,0.0657233149,0.5641449094,1.8346886635,-1.1153488159,-0.8920660019,0.1958157718,0.3952474594,-0.2617920339,-1.6110399961,-1.3485546112,-1.3979731798,-0.6749272346,-0.2331542522,-1.0221537352,-1.5719534159,0.0787544250,1.4476664066,1.0816694498,0.5359908342,0.5086901784,1.3539590836,2.1905260086,-1.3281347752,-1.5861293077,1.0052930117,0.2073535621,-0.2195786089,-0.6525210738,0.1171994656,0.2404762059,1.7054648399,-0.0166308656,-0.9980779886,-0.3845568895,0.2131299078,0.8598623276,-0.2320811450,1.2193504572,-0.7898730636,0.5503804684,-0.6646599174,-0.4484573305,-1.3752053976,-0.0077703991,0.4516531229,0.5158145428,1.5019677877,2.3400790691,-0.1689572185,-0.7131705284,0.0862102732,-1.8778115511,-0.5737338662,0.3488899767,0.0695413202,1.4736529589,0.1018898711,-0.3780619502,-0.3608337045,-0.3679491580,-0.5431326032,-1.0214086771,-0.1403140426,0.3082767725,0.1509161294,0.6119741201,0.7313207984,0.8829240799,1.8567088842,0.5742434263,0.4984557033,-0.4186875820,-0.9313029051,-1.6411021948,-0.5562921762,-1.4994580746,-1.5915665627,-0.0610120445,1.5467461348,-0.7506592274,-0.4631983638,-0.1926139444,1.7024631500,1.6659804583,0.5192429423,-0.5472186208,0.0565663762,0.2284213901,1.5348349810,0.2286630720,0.0394604020,-0.8655991554,0.8045617938,1.2748690844,0.5713320971,-0.1608036757,0.7733766437,2.1223337650,-0.9744271040,0.7814424634,-0.5671689510,0.4748123884,0.9607690573,-1.0613287687,1.2581557035,-0.7631398439,-1.7736225128,0.0356541537,-0.6820560694,-0.7282853127,-0.1053154469,-0.9815471172,0.4679682255,0.1102410853,1.7661479712,0.9696770906,-0.6157441139],[0.0832946599,0.0154042337,1.4200258255,0.1912794560,0.8820790052,-0.7564318180,-0.9813390970,2.7474064827,-0.4309709370,-0.5363450050,-0.1029144824,0.4400387704,-1.0086545944,0.8715492487,-0.4610286951,1.2774317265,-0.3406333029,1.0627976656,-0.0177198425,-1.4871954918,-0.3110130727,-0.1273606718,-0.5646942258,0.3632155955,1.1745493412,-0.5503141880,-1.6987376213,0.5262299180,-0.5710070729,-1.2713433504,0.8851079941,-0.6829869151,-1.9171721935,0.7940423489,0.5627143979,-0.9600135088,-0.2346548587,-2.2996087074,0.8724299669,0.0616741218,0.4606336057,-1.6986548901,-0.1236542985,-0.8901436925,1.6886969805,-0.6276292801,-0.7237911224,0.5590992570,-2.4663658142,-1.0565769672,-0.4022848010,1.6344102621,-0.4803332984,-0.4069189131,1.0568474531,-1.2874617577,1.5985314846,0.8697004318,1.5187076330,0.3975264132,-1.4097440243,-0.0381349325,-1.1354163885,-0.4750063717,-0.2226876616,0.7452670336,0.2666867077,0.1722257733,0.4386547804,-0.4073028862,-1.2752696276,-0.3434637189,-0.2251634151,-0.8087444901,-0.7511361837,0.2216085494,-0.5121274590,-1.8204050064,0.7472478747,0.5685710907,0.1281473041,-0.8394502401,-2.5131106377,-0.9077122808,-0.6943283677,-0.5469990373,0.7462623715,-0.3083970845,-0.8384279609,0.8198192716,0.4979269207,0.0270626321,-0.1213922799,0.7247827053,0.3348164260,-0.2508352697,0.7643464804,-0.4294463694,-0.1170479879,0.0250977706,0.7436251640,-1.5160830021,-1.5851063728,-1.5032289028,-0.4924083948,0.6285675168,-1.1842933893,1.1135993004,1.2736849785,-0.0435120836,-1.3845635653,1.7388744354,1.7109934092,0.6870909929,-2.3529503345,0.5297210217,0.0182195548,-0.8128156066,-0.1349581033,0.5240533352,-1.1022468805,-1.2241221666,-0.2381365448,-1.0415886641,-0.1002316251,-0.4005219340,-0.2848261595,0.0950556099,1.7183667421,1.2604315281,1.8246142864,-0.2740058899,1.5894554853,0.2125496417,-0.8694303632,0.3586742282,-0.6369240880,-0.3130733967,0.1877637058,-1.3034642935,-0.5413097143,-0.9775701761,0.3182064295,-1.2245349884,1.3321487904,-0.1348018348,-0.0079483762,0.5871071815,1.6216561794,-2.3066439629,-0.3211205304,0.0831940025,-1.0155971050,0.3312295079,1.2365626097,0.4100286365,0.4295566678,-0.4415833354,1.0282262564,0.7797985673,-2.6591732502,-1.3471249342,0.7747009993,0.7138849497,-0.1456113309,0.9523909688,1.2731977701,-0.9705725312,0.6225118637,-0.6567438841,0.1674103588,-0.1905986071,-0.5853870511,0.9797718525,-0.5967161059,0.4017387033,-1.3391740322,-0.7596985698,0.9018880129,1.2323490381,0.6116399169,-1.2004809380,0.7040430307,1.9253069162,-0.1669967473,-0.4774456620,-0.1453061104,2.7242598534,1.0761202574,-0.7772324085,-1.8398270607,1.1820985079,2.2291193008,0.1210172623,0.5169967413,-2.0369746685,1.3334836960,0.9594944715,1.3979049921,0.0152120637,0.3863286078,0.0464983992,-0.8423812985,-0.4377827644,0.6164746284,-0.6307107210,-0.4342782497,0.2298891991,-2.0079159737,-1.4043928385,0.1458393335,-0.6708733439,-1.6447790861,0.0703957826,-0.2523188591,0.3924876451,1.0161181688,2.0849382877,0.8516621590,0.9680776596,0.1318573952,-0.7458209395,0.1732912064,-0.0171494074],[1.5169693232,0.3570988774,0.9281348586,-1.3707672358,-0.0427510217,0.5875717402,0.0559275597,1.6824957132,0.8721467257,-0.8102717400,-1.0185278654,-0.8793839812,1.4426631927,1.5465738773,-0.8688929677,-0.1653910875,0.3350679874,0.9537213445,-0.9668954015,1.6751704216,-0.6430878043,0.3188067973,-0.3906269073,-1.2710647583,0.3388320804,-1.1807581186,0.9608909488,-2.0899810791,0.7109305263,-0.9204555750,-0.9144626856,-0.6868638396,-0.1635092348,-1.7601587772,0.1764658093,0.1667043418,-1.5872178078,-0.0366247445,1.0080679655,0.4422006309,-2.4767754078,-0.9045382738,-0.7697851062,-0.1585956812,-0.2367852628,1.1754802465,-1.1316013336,1.7630548477,0.2800961733,-0.6288664937,-1.2302342653,0.3262609243,0.2660154700,-0.0530349985,-0.4598177373,1.6693603992,-0.1160294041,0.4715109169,0.9127645493,1.6281094551,0.9660769105,1.7898586988,1.6077495813,-0.9647401571,1.1253098249,-1.2583026886,0.8288027048,-0.2743187547,-0.4070868790,1.4383150339,-0.7290006876,0.1344997436,-0.1674019545,-0.3628393710,-0.4540843666,0.6296550632,-1.8585166931,-0.1117752790,-0.1778820157,2.4086344242,0.1753031015,-0.4950847924,-2.2426016331,-1.4452265501,-0.2448479533,-0.1808102429,-0.2288896441,2.1536924839,-1.3813554049,0.2147695422,-0.0196479950,0.0244122334,-0.8968064189,-0.6828643680,-0.1684126258,-1.0011633635,-0.7751793861,-1.1394294500,-0.8886438608,-1.8494819403,0.1862669736,0.3815818429,-1.2124689817,0.4131552875,1.2002992630,1.5946217775,-0.5457906127,0.4128240943,0.2786887288,0.2461876273,-1.3450722694,-1.1776552200,-1.2061234713,0.2029906958,-1.4223995209,0.5777701735,1.6634222269,-0.2870059907,0.2412759215,0.3775524497,-0.9314845204,0.6768042445,-0.4252702296,-0.7470455766,0.2802719474,-0.5464428067,0.6813836694,0.7286756039,-0.5409637690,-0.7058429718,-0.1084262133,2.0533525944,1.2147784233,0.1359319836,1.0503759384,1.0613586903,-0.2144398689,0.1920922250,-1.0345438719,-0.5036805868,1.6150988340,1.6786446571,-0.4663047791,1.0681966543,-1.3721605539,0.1843423843,-0.5531916618,-0.8321005702,-1.0742633343,0.3251988888,-0.7832528353,0.7555800676,-0.1313978136,-0.0314237624,0.5373218656,-0.7306606770,0.6202054024,1.9043105841,-0.4875988960,0.2367156595,-0.4076660872,1.0004101992,0.3512436152,1.2557044029,-0.5128421783,-0.4611277282,0.1007170677,0.0925083235,-2.1999547482,0.1508994251,0.6993888617,0.7741886973,-1.5526758432,0.7084724307,0.1943209916,-0.4375484288,0.7226394415,0.1368113905,0.0707129464,-0.0306051858,-0.0026378226,-0.1765782833,-0.5268459916,1.9544335604,0.2153434902,-0.5700322390,3.2406566143,-0.1734665334,0.2326574624,0.2949871421,-0.8500581980,0.4187697768,-1.0791419744,0.8719388843,-1.4023494720,0.4674481452,0.4940485954,-0.4561758637,-0.2247166485,0.7698090076,1.1066019535,0.9135959744,1.8430663347,0.3033389449,-1.0240417719,-0.5697768927,-0.5450455546,-0.7129164934,-2.1384212971,-0.4906723499,1.0716369152,1.0448619127,1.1970314980,0.0851596743,-0.7071413994,0.4134632349,0.5083128214,0.4324565530,0.4515452981,-0.4726723731,-0.4617720246,-0.0498823039,0.1511984169,-0.1242134199],[1.7733763456,0.6548894644,-0.9969044924,-1.0999536514,-0.4729947448,-0.8252610564,-0.3246595860,-0.2494323254,0.8223640919,0.5732952356,1.1071861982,-0.9319651723,0.3545930386,1.3546044827,0.3564473987,0.8952826262,-0.0207289252,-1.9850325584,1.6648896933,1.0103242397,-0.8624767661,-0.5156413913,-0.5986755490,-0.5133580565,2.7740614414,-0.0843275264,0.5370041132,0.0354410969,-0.4008655846,1.6665250063,0.0965174660,-0.5183625817,-0.9195118546,0.3387037218,1.1137673855,-1.5738148689,1.6057540178,1.3714587688,-2.3321993351,0.3463236392,0.7981258631,1.7536504269,0.0942437500,-0.3396389484,-1.2540856600,0.7180249095,-1.1979964972,0.5125359297,-0.8158826828,0.1937242448,0.5850766301,-0.2552036047,-1.0627428293,-0.0171058923,-0.3047343791,-0.5120671988,0.6329017282,-1.2476128340,0.6401041746,-0.8136311769,-0.0626717582,-1.4570958614,0.2782136798,-0.8262432218,-0.5279885530,-0.2686576247,0.0438584201,0.6822136641,-1.4326016903,0.5378295779,-0.7443700433,2.0603389740,-1.1167420149,0.3034130335,-1.0690333843,-1.0751850605,0.0629330426,-1.8225570917,-1.5243411064,1.2396709919,1.9703440666,-0.4888984561,0.4765799642,-0.3094685376,-0.9791085124,-0.1147989929,-1.6355508566,0.4577118158,0.2286115885,-0.1039451063,0.7752788663,-0.3030622900,-0.0726639628,0.2677658200,-1.6213132143,-0.4331697822,0.1601847857,-0.7287859917,0.3674771488,-0.1830540895,-0.8931375146,0.4875233173,0.9322209358,-0.1905431449,3.1566889286,0.3486742377,1.0539091825,0.4353290200,0.6332775354,-0.7265618443,-0.7866077423,2.5195691586,-0.0017073880,-0.6077119708,1.2275959253,0.7180952430,-1.3357245922,-1.0510390997,-0.7974286675,-0.4727028608,1.5297054052,-0.4044028521,-0.7877702117,1.6482814550,0.8431814909,0.2029867172,-0.6682254076,1.6193115711,-0.5954051614,-0.3906408250,1.9035891294,0.6554541588,-2.8776378632,0.2603771389,0.7601162195,0.5754011869,0.7990735173,1.1752868891,0.4685513079,0.9743111730,-0.2307738513,-0.0264357254,0.4110282660,-0.3004532456,0.2818160057,-0.5710403919,1.0028342009,0.5024909973,0.4595133066,1.4817398787,0.2799629867,0.5637354851,-0.6241150498,0.8133111000,-0.0602380522,1.1828916073,-0.1272625327,-0.8460607529,0.3959697485,1.4310905933,-1.0955809355,-0.2613841593,-0.1580035388,0.3657860160,1.2321866751,-0.6503368020,0.0420608893,-0.8093909621,0.5546837449,0.2265381217,0.3556352556,0.7656990290,-0.3249478340,-0.1872305423,-0.8028757572,1.8795111179,1.1952743530,0.8303111196,-0.1781547517,0.5080417991,-0.1168814152,1.1812840700,1.0313570499,-1.7196556330,0.2446504086,0.1371421814,0.0973915532,-0.4585001469,-0.8280373812,-0.4593352675,-0.6757261753,-0.0793135092,-0.4180409610,1.2445529699,0.2011100203,0.9248659015,-0.4210970700,-0.8320764899,-0.1646859646,-2.5782234669,1.1942738295,1.5204107761,1.2211294174,0.7107045054,-0.1187882051,-0.8583485484,-1.0267019272,0.5830163360,-0.1584341824,0.6857743859,-1.6815145016,0.2011470497,0.1775249690,0.4979387820,0.8372054100,0.5836708546,-0.1584636122,0.2616901696,0.5922822356,1.0962525606,-0.2246936560,0.3583438396,0.5079059005,2.4989459515],[0.7612602115,-0.6034088135,-0.6274229288,-0.0049325931,0.4828433096,-0.1417066455,0.5503855348,0.9566833377,-0.4469841719,-1.8436908722,0.0132681746,0.6910766363,-0.9844883680,0.6257191896,-0.0121382996,0.9449425936,0.4866577685,-0.1009661183,-0.1784197092,0.7964317203,0.1096719578,-1.0981303453,0.1085784137,-0.2215659022,0.6406838894,-0.7363584042,-0.1355459243,0.1792862862,-1.5051630735,-3.0252084732,2.3134453297,-0.1067946255,-0.2164448351,0.5656723976,0.1987833977,-0.4938872755,0.3542607427,0.7917511463,2.0004119873,0.1138286367,1.0661466122,-1.2925044298,0.0914441869,0.6103685498,-0.9954180121,-0.4100271165,-0.4298735261,1.2885445356,-1.7387659550,-0.5108031631,0.1936938167,-0.2972007990,-0.5979477763,1.5313448906,0.0219627060,-0.7502336502,3.1279857159,-2.1839592457,1.0440800190,-0.6610156298,-1.2552039623,0.4473415315,-0.9450294375,-0.2153885067,1.1292688847,1.4167747498,0.0083199078,-0.7387729883,-1.5310351849,-1.2331342697,-0.0405933820,-0.4422261715,-1.0382130146,1.3626192808,1.1903202534,-0.1882884353,-1.8005816936,1.5561032295,3.1526033878,0.4462724328,-0.5650293827,0.2617058754,-1.4923347235,0.1959274411,1.6606786251,2.1215653419,0.4635251164,-1.6947836876,0.5675776005,-0.0769856125,0.8061911464,-0.7121078372,0.3398130536,0.2747018933,-0.9738521576,-1.0838885307,1.6086019278,-0.1070212498,-2.4784710407,1.2567377090,-0.9551062584,0.8460628986,-1.0296751261,-0.3630988300,0.4910036623,1.3033028841,0.6093656421,-1.5684084892,0.2002652586,1.8276238441,0.1794807613,0.7145086527,0.4715159535,-1.3110510111,0.3708418012,0.5671648383,-0.1211298183,-1.9422543049,2.7631845474,-0.0019290138,-0.9331898093,0.2469065487,0.9991958737,0.1478086561,0.5288000107,-0.2993927896,0.6493500471,-0.5947993398,0.0689935088,0.3103188276,-0.5490578413,0.5102764964,1.0045766830,0.4634752870,0.3118042052,-0.1784106791,-1.2324748039,-0.2715279460,1.3155361414,0.6285444498,-0.7697691917,0.3814707100,-0.7372010946,1.6317893267,0.0496188216,1.8298569918,0.2486533225,0.2665860057,-0.5764042139,-0.1545846164,0.5823374987,-1.2349112034,0.9288624525,0.2240541875,-0.8932236433,0.6563537121,-1.9804890156,0.2280142903,-0.9330589771,0.8488634229,-0.0275313407,0.7628030181,-0.5166394114,-0.0639637858,0.5297335982,0.0976115614,-1.2929050922,-0.2473904490,0.0017580237,-0.0445541181,-1.2251801491,-0.9520522356,0.3250408471,-0.9763738513,-0.3191157579,0.0639646500,0.3773847818,-1.7942626476,-0.3388437927,-0.7155359983,-0.9516233802,-0.8122008443,-1.0962382555,0.8137555122,-1.6893570423,-0.7392467260,-1.0180219412,0.3281373978,0.8430277109,0.6054344177,0.9184139967,-0.4063994288,0.0855568647,-0.1104435921,-0.7943481803,0.2338262349,1.0993648767,-1.3479268551,0.3901562393,-0.3676171601,-0.5882406831,-1.7047501802,0.0156326983,0.3451995254,0.1704274416,-0.5342517495,0.1036691368,1.9168593884,-1.0093951225,-0.2074879557,-0.1844485551,-1.1339180470,-0.7229844928,0.3331287503,0.3014786243,1.4839015007,-0.2690161169,1.0232335329,0.7889760733,-0.4552712739,0.3501521647,-0.2285785526,0.9389357567,0.0447394811],[0.1851441115,-1.2355756760,1.2031196356,0.0116550243,0.9264087677,0.3385564387,-0.5845701694,-0.6950528026,1.0220985413,-0.7311565280,-0.9815580845,0.8116582036,-2.7727127075,0.1836193055,-0.8430696726,-0.9484882355,0.3477781117,-1.2728915215,-0.9775258899,2.0550086498,-1.1804845333,-1.6777393818,-0.2583949864,1.9016064405,-0.2666935325,-1.0596487522,1.1103088856,-0.5217511654,-0.1671878248,-1.3836013079,0.5602372289,-0.7295765281,-0.0698853359,-0.5959703326,-0.6792480946,-1.0985150337,0.4957344830,1.3550078869,-0.3935103118,1.1229667664,0.8153097034,-0.6519557238,-0.4303318560,-2.8154788017,-0.3087665737,0.7753482461,0.6616540551,1.1300858259,0.6900553107,-1.8309692144,-1.0626448393,-0.7698283195,-0.8680559993,-0.7421313524,1.8048762083,-0.0947011113,0.4935582280,-0.1392302960,0.1740383208,0.1270922571,-1.5548484325,-0.0411530435,-0.6951826811,0.3269871473,-1.4882113934,-0.3722811639,-0.3547021151,0.1275018305,-0.4833881557,-1.8390951157,0.1966317892,0.5947086215,-0.0949502289,0.2317803800,-0.9899123907,-1.4969880581,1.9577594995,-1.1843186617,1.5552152395,0.5649688244,-1.7155886889,-0.6562070251,-0.8054563999,-1.6628246307,0.7222976089,1.0663632154,-0.1094901860,-1.5984297991,-0.5933257937,-0.0686090812,1.2584441900,-0.3986067474,-0.4922445416,0.2149075568,1.6925339699,0.4467121065,0.0721166655,-0.1689761132,-0.4769457579,0.1210901141,-0.1196763217,-0.3491147757,-0.6679611206,-1.1081523895,0.8642325997,-0.3652341068,-1.4192044735,0.5283550024,-0.4291144013,-3.9799249172,2.5193560123,-0.0001015828,-0.3741996288,1.1290762424,1.6821254492,1.0898627043,0.0934686586,0.3777738214,-0.5232481956,-0.5119860172,0.8392738700,-1.3982032537,-0.5136648417,0.2465056181,1.1420283318,-0.4828293025,1.1170063019,0.2153869420,-1.0433691740,0.7852531672,-0.3490794599,-0.5112475753,1.0232989788,-0.0248605367,0.6658914089,-0.0275805313,-0.3348940313,1.8200131655,-1.0081651211,-0.2610522807,1.3033430576,-0.4144646525,-0.0354455411,0.8998509049,0.5341755748,-0.4204404354,0.2261348367,0.2233097106,-1.1665488482,-0.5937159657,-1.4060828686,0.4686772823,0.0984703451,-0.7751576304,0.3141351938,0.8974028826,-0.1393257678,0.7376810908,-0.3796384633,0.7480918169,0.2854343355,0.4962151349,0.0067210966,1.3005772829,1.0714277029,0.6444802284,-0.5154283047,0.0870009586,1.0096105337,-1.8330725431,0.1132716388,-0.0708507001,0.1738734245,-1.0237518549,0.2748146057,-0.2124217898,-0.1312001646,-0.7777321935,-0.8523564339,-0.5532600880,-0.4056203365,2.1330664158,0.2694247961,0.0929921642,-1.1530809402,0.5591195822,0.1892250180,0.6476491690,0.0823355317,0.4193497598,-0.6377058029,0.2406051308,-0.6672044992,2.6748664379,-0.5676947236,-0.3393431604,-0.5449929833,0.2807078958,0.9032731056,-0.6467321515,0.8295060992,-1.8630459309,0.4991222322,0.4042034745,-0.0883236080,-0.0470565073,0.9142104983,1.0030266047,0.2447029352,0.6141192913,-0.8892118335,-0.7647240758,-1.1047742367,-0.8095286489,-2.0631930828,0.2100882679,0.2161284685,0.1404330134,-1.7657244205,0.2368792892,-0.1763023734,-0.4331485033,0.1648295075,1.7212885618],[-1.1766096354,0.1771905273,0.0255289283,0.0521093681,0.2013487816,0.8420627713,0.9915324450,1.5613195896,0.6682843566,-1.4083592892,-1.2373524904,-1.3657908440,0.5665727258,0.1114991754,0.5536442399,1.0835049152,-0.0504580699,0.2503991425,1.9299334288,-1.5645395517,0.4178295732,0.1182594225,-0.2803184986,-0.0412648320,0.1382355541,0.7969234586,-0.1608946025,-1.3431527615,0.2850189209,0.2608601153,1.9069812298,-0.3040925264,-1.0234804153,0.8924683332,0.9798862338,0.7105198503,-0.5560311675,-1.9237110615,-1.5857894421,-0.0867523253,0.7510196567,-1.6821100712,1.7742727995,-1.1292693615,1.2592233419,-0.5400198102,0.4569769502,-0.1657794416,0.5541830659,1.0131599903,1.1774101257,0.3166932166,1.1890761852,0.4440013170,1.7804197073,-0.8230453730,0.0510413684,-0.8502710462,0.2226673365,-0.8931104541,0.7461739779,0.5200843215,2.4808294773,-0.4306486249,0.7000415325,-0.6627852321,-0.6666034460,0.6417964697,-1.3115973473,-0.4170653224,-0.0155315036,0.2604958713,-1.9038473368,0.8179932237,-0.8048866987,0.1132161394,0.6200553179,0.8735640645,-0.8754689097,-2.9979059696,1.8664747477,0.2899555564,-0.3376359046,1.4218786955,0.0991161391,1.5541207790,-0.0542410761,1.2513942719,-1.3293449879,-1.5720999241,0.2153496891,0.4149667025,-0.3295794129,-0.2767211795,-0.0946821421,1.7693696022,0.0545876808,-0.6418469548,1.1964954138,0.4129462242,0.3027030230,2.1559007168,-0.6829319596,-0.3651222289,-0.0442247801,2.2708928585,-0.7823866010,0.4403840601,0.1520252228,-1.2476013899,0.5838144422,-0.5700789690,0.0327484533,-0.8868691325,2.3026103973,-0.3992435634,0.4614451826,-1.6609683037,-2.1689975262,-0.7229250073,2.8979327679,0.7089292407,-2.3434951305,0.1656176299,0.5335359573,0.0808340311,0.3161955476,-1.3906550407,-0.6597616076,-0.3027254045,-0.3078902364,-0.3183090985,0.2484171093,-1.8363064528,0.1134862378,0.9222687483,1.4002178907,-0.6588960290,-0.5069347620,-0.1029141247,0.0792785361,0.8484690189,0.5411299467,-0.2156495005,-1.4614349604,-0.0294835921,-0.0679289103,1.7664188147,-0.3468401432,-0.9617422819,-1.0435235500,1.4455068111,0.4401999414,0.0200419500,0.0252560489,1.3682826757,-0.7683274150,-0.8309261799,-1.9147306681,0.1773295254,-0.1637866497,0.8641271591,-1.4150470495,1.3570704460,-1.2145545483,0.1236755922,-0.0837656856,-2.5834338665,0.3076185882,0.2000334412,-0.2788176537,-1.8437790871,-0.7380565405,0.6632315516,2.2895617485,0.6810886264,-0.4265203774,1.3332300186,0.9933365583,1.0180410147,-2.1297695637,2.2985951900,-1.2535015345,-0.7859506607,0.3115945160,-1.4633288383,-0.2197153121,0.1873781085,-1.5290776491,-2.2650036812,0.4986795783,-0.4243268967,-0.4248904884,1.2769355774,-0.3564965427,1.3732653856,-1.4011700153,-1.5260056257,0.6965202093,0.1845299453,-0.7892647982,-0.1872230172,0.4882991016,0.1650603563,-1.6113953590,0.2592717409,-0.1188143864,1.2228955030,-0.8016638756,0.6876689196,0.7840186357,-1.5337510109,-1.1500921249,1.2201095819,0.5806935430,-0.8960040808,0.8627079129,-1.1348323822,-0.9670757055,0.3248357773,-0.4929454625,0.1369240880,-0.5122059584,-1.0240994692],[0.8177993298,-1.5714014769,-0.9127979279,0.3507647216,0.5462158918,0.0155025600,0.2486607134,0.4373335540,-0.0667797402,0.3301390707,0.5512520075,-0.0004508996,0.2623131573,0.8791959882,-1.3433330059,0.0341191590,-0.1302566528,-0.6102820635,-1.9780943394,1.0417385101,-0.1486586779,1.3084896803,-0.1083629355,0.2558193207,0.6630388498,-0.7649769783,0.0660102144,0.6541601419,0.1619965583,-0.2343661040,1.1309125423,0.9972749352,0.3769946396,0.9053672552,0.0170451328,-0.2916249335,-0.0637479946,-0.8857896924,1.0207086802,-1.3605722189,1.5603965521,0.4978734851,-0.0114495857,0.2822905183,-0.0966921449,0.4231345654,-1.5016077757,0.5884187222,-0.8756873012,0.5964536071,1.5745950937,1.0954363346,-0.8237411976,-0.2152891606,-0.2709789574,-0.2081480324,2.2277188301,0.3763382137,1.2043631077,1.1044359207,-1.3858469725,1.2034906149,0.0602937639,1.5043460131,-0.4531169236,-1.3994597197,0.9959958196,1.4748657942,-0.6896502376,0.5163658857,-2.1955311298,-0.7591708302,-2.2448000908,0.4303465486,-1.8543956280,1.4948334694,-0.3116846383,0.4466024935,-0.0118529536,0.7244918942,-1.6666995287,0.0236608721,-0.4630951583,-0.5484100580,-1.1173944473,0.4898535311,1.6996483803,-1.8060977459,-0.4497342110,0.4933337867,0.0872113928,1.6383810043,0.2239727825,-1.2708234787,0.3466352224,-0.4335897863,-2.5197424889,0.7378028631,-0.8447279334,-0.0559481047,0.6839315891,-1.1025488377,0.2024087608,2.9150803089,-0.3840768039,-0.0326391011,-0.0803115442,-0.8246007562,0.3065483272,0.0389659554,0.8100610971,-0.3687470257,1.0887268782,0.0036491603,-1.0394093990,-1.0970540047,-0.7277394533,0.2685888410,0.7355921268,0.9661358595,0.5190473795,0.0508830100,1.2438833714,-2.2499885559,0.8982744217,-1.2954920530,0.8556146026,-0.4744929671,-0.6942690611,-0.1644901037,-0.2620332837,-0.1451232731,0.0785102472,1.0016654730,-0.2739449739,0.8009263277,-0.1702002585,-0.8886307478,-0.1733903736,-1.1252379417,-0.4982013106,-0.5461481214,-0.7811477184,0.1073800698,-0.7631527781,-1.3092365265,-0.1617819816,-0.0008865069,0.5713884830,-0.1136347950,0.1617997438,0.1273338348,-0.7498638630,0.8821802735,-0.0337238349,1.3859173059,0.5330554247,0.1874614954,1.6891919374,2.0251679420,-0.4745604992,0.0918694958,0.1943952888,0.6713520288,-2.0351748466,2.6817276478,-0.1689636558,-1.2043520212,-0.8034988046,-0.6250877380,1.4457765818,-1.4667338133,-0.8495544195,-0.3539527953,-1.0773371458,-2.4111745358,0.5679433942,-0.2081842870,0.9879924655,1.2360608578,0.7949596047,0.4199788272,-0.6940913200,-0.8832488060,0.5228080153,-0.2757284045,0.2805615962,-0.4129777253,-1.1507169008,1.8317346573,0.1299459189,-1.3232452869,-0.2197397053,1.6015273333,-0.7579594851,0.5799200535,1.2151949406,-0.2547016442,0.1068254113,0.0516264960,1.1671109200,1.3634036779,0.0900574327,-1.0950243473,0.8401091695,0.6654623151,-0.3086810708,-0.7375400066,-1.0307489634,0.1048286334,-1.5460332632,1.8186627626,0.8098951578,0.9708928466,0.7102554440,0.9713087678,0.3875834644,-1.1385778189,0.5401791930,0.4986961782,0.2263549715,-0.2634185255,-1.5598323345,0.3776816428],[-1.7435592413,-0.6064981818,-1.0359470844,0.7043692470,-0.5080588460,-1.9649239779,1.1656636000,-0.8278691173,-0.3729766309,-1.5564032793,-0.8267266154,-1.1934009790,0.6669597030,-1.0290228128,1.0644114017,2.3462307453,-0.2566364408,1.2686136961,0.2236939818,-1.1905844212,-0.8348674774,1.0824284554,-0.8072878718,-2.0340442657,-0.1297576725,0.1229395643,0.4321765304,-2.7568418980,-0.5900364518,-0.2194618881,0.9851743579,0.5149931312,0.7409499884,0.9859756827,-0.7990174294,-0.9565790892,-0.6254098415,-0.5258603692,-1.1526063681,0.5984534025,1.7542530298,0.5825209618,-0.0108615747,1.1506799459,1.6939212084,0.6322537065,-1.0341697931,-1.4992163181,-0.9192206264,0.2501222789,-1.6166472435,-0.3118333817,-1.5592268705,1.2363440990,-0.2899449468,0.2497586608,0.8226853609,0.0091154482,0.5934258699,-1.0807846785,0.6175705194,-0.2989517748,0.0668154061,-0.3421589136,-1.2376976013,0.2341058552,-0.2281744033,0.9363278747,0.7818625569,-0.5937199593,-0.2155658305,-0.2269501239,0.0548860207,1.2610039711,-1.1864619255,0.2939851582,-0.1575176418,0.3959463537,-0.5379269719,0.0852746218,-0.1205217764,1.1457941532,-0.6422573924,0.6109707355,-2.3347873688,0.8817616105,-0.4318682849,-0.2186090648,2.3368148804,-0.8696759343,-1.3311178684,-1.4940352440,-0.1126572788,-0.1654232144,0.6756837964,1.0262820721,-1.5286720991,-1.7455735207,-0.0086124055,-1.3048062325,0.7480476499,-0.1624615192,0.8573369384,-0.2427486479,1.5718251467,-0.4005960226,0.1661429852,-0.3388045132,-0.8788685203,1.1277475357,-1.8028851748,-0.9683129191,-0.2205161005,-0.3632950187,0.9993163347,-1.1250728369,0.2273952663,-1.6806476116,-0.5850839615,0.4633552432,0.5404006243,-1.1591656208,-0.4043771923,-0.5775872469,-0.4467805326,-1.3486293554,0.1647246778,0.5355979800,0.6689383984,1.6987106800,1.0737128258,1.5600349903,0.5412341952,-0.2705770433,-0.5515835881,0.4355035424,-0.1320427060,-1.2384092808,-0.3141611516,1.3099900484,0.6195878386,-0.0317559093,0.4572177231,-1.0924277306,1.0926088095,-0.3314957321,1.8907419443,-0.4943482280,-0.0626017973,-0.0550512932,-0.0231284834,0.7892349958,0.3658465743,0.4307425022,0.1753020287,0.4385994077,0.9858914018,-0.7419704795,1.7252422571,-1.0884103775,0.9333866835,-0.4368148446,-0.6445410848,-0.7094064951,0.5206435919,-1.9676936865,-2.0886249542,-0.0491473936,1.7515029907,-1.2138712406,-1.0226154327,-0.5423930287,0.7038816810,-0.8431472182,0.7065833211,0.2752345502,0.0155479768,0.4675925374,0.9834657311,1.3818086386,-0.0449825265,-0.8379833102,0.6575056911,-0.9417086840,-0.9865073562,-0.8108641505,0.4895179570,-0.2885791063,0.6517304778,-0.0437750444,0.9180089235,1.0069895983,0.5761268735,0.4679514468,0.0979376882,-0.6210292578,0.2446375191,-0.2866552174,-0.4697014689,-0.6670892835,-0.2212103456,0.1083477437,-1.3208975792,0.2513886690,-0.6949612498,1.3961971998,1.0441313982,-0.8038179278,-0.6545550227,1.0894525051,-0.8547613025,0.4289109409,-0.9605471492,-1.3116114140,-1.4649935961,2.2020881176,-0.7328217626,0.2469987720,-1.4316263199,1.0094193220,-0.5877754092,-1.2018294334,-0.4497393668,-2.0317924023],[-1.3781358004,0.2250173539,-0.2393929213,1.0711579323,1.0380489826,-1.6511549950,-0.3789921403,0.9080212116,0.8696871400,-1.6957534552,0.3965432644,0.0837741494,0.0010892110,-2.4933798313,2.0837154388,-0.2696723044,0.7178445458,-1.4326989651,-1.0441292524,0.9024427533,-0.2075345218,-1.0056159496,0.8656446338,0.0663467273,-1.0112538338,0.4815161228,0.6747781038,1.7164748907,0.0540352389,0.0186312608,0.6076316833,0.5149070024,-0.6382617354,-0.2592832446,-0.1123041362,-2.3297955990,-1.2326625586,0.1074425578,-0.0259655807,-0.4965038002,0.4682292342,-0.7256876826,2.5770611763,-0.5546184778,-0.7165570259,-0.5988902450,1.3805335760,-0.9438036084,1.8827584982,2.2911372185,-0.3109315038,-0.4171719849,-0.9282898903,-1.2628384829,-0.8774039149,-0.1966391057,0.1648458242,-0.0454749390,-0.4523634613,-0.8597000837,-1.8105161190,-0.5551453233,-0.4640676081,0.0456892885,0.9407829046,-0.4700866640,-0.4446507394,-0.9410798550,0.1821057349,-0.8256917000,-0.5830290914,1.5628407001,-0.1577423513,0.3348858356,-1.0220993757,1.5164207220,-0.4169793427,0.0705198646,-0.9684671760,-0.7755367756,0.8577855229,0.7357011437,0.8049449325,-0.5722336173,2.4884157181,-1.0409582853,-0.5645226836,-0.8657156825,0.8756935000,-0.0816876218,1.6169995070,-0.4836786091,-0.5187353492,-0.1280231029,0.3786218464,-1.0286502838,0.9747040272,-0.4097430408,-0.0414662100,0.1704852134,1.2174123526,-0.4135402739,-0.9620710015,-0.6299701929,0.7686001062,1.6596417427,0.1165930480,1.5519549847,-0.0122310994,0.6999790072,-0.3696699142,-0.6388676167,-0.3132658899,-0.3769345582,0.4913886487,0.9003452063,-0.6527366638,-0.9828684926,0.0031222911,0.3023118973,1.1393951178,0.0686765835,0.8947140574,0.1511452794,1.2494877577,0.2830680609,1.0252991915,-2.0512757301,1.0091516972,0.8578301668,-0.4312323630,-0.0823090598,-0.6327573657,-0.6221244335,0.1637613624,-0.4898373783,-1.0175414085,-0.1421252489,0.7623457909,0.6941013932,0.8787897229,1.0183321238,-1.3048707247,0.6710067391,-0.5081533790,0.7300412655,-1.0180243254,-0.6606916189,1.6511225700,-0.3809156120,-0.8625769019,0.1733328998,-0.3080062270,-0.4351976812,0.6185950637,0.1875175387,0.2302682847,1.6791144609,0.6098973155,-1.7957258224,-1.9296709299,0.4632092416,0.9872512817,-0.2040626854,-1.6726781130,-0.7090438008,0.1458017230,-1.1318336725,-0.2968883812,-2.6211783886,0.1238219291,-0.3862033188,0.1050975546,0.4096537530,0.3024674058,-0.8510958552,0.7929092646,-0.2956811488,-0.1781456769,-0.4592222571,0.6618961692,0.7615174055,-0.3307532370,0.0029948372,1.0480338335,-0.9730817080,-0.7652286887,0.1419316083,0.9494506717,-0.6568839550,-1.5343198776,0.7911569476,-0.0277832802,-0.0588698722,-0.9085233808,0.0808649138,-0.0074584479,0.2145736217,-0.0665548891,-0.3323315680,0.0640165284,-0.9991938472,1.8833183050,-1.0895928144,0.8137967587,-0.4096502066,-0.2939639390,-1.3181260824,-0.7032576203,-0.8461838961,-0.0827098563,-1.9362289906,-0.3094807267,0.0122073861,-0.4892766178,0.8326879740,0.8380709887,-0.5376721621,-1.5581637621,0.4174664915,-1.3119213581,-1.4869244099,0.8268741965,-0.0097436709],[-0.6998175979,-0.6954088807,-0.2588140666,1.1530423164,-0.8297799826,1.8891391754,-0.7696596980,0.0943683162,-0.0774358884,0.7215057015,-0.5595492125,1.3661303520,0.7425178885,-0.0379569791,-0.5930621624,0.3185641766,1.4783425331,-1.1458002329,-0.9837580323,1.0978168249,0.8367465734,0.6999016404,0.6078990102,0.6545804143,-0.9899599552,0.6728334427,-1.1816710234,-0.8178920746,1.1061612368,0.3675162494,-0.5634465814,-0.2326633036,-0.6539015174,0.4468860626,-0.9113517404,-0.3453916311,0.1714687198,0.2547176480,-0.2261466980,-1.6652747393,1.0987795591,0.7416546345,-0.5794039369,1.2014195919,-0.4735201597,0.7171515822,-0.3532316089,0.5127348900,2.0128924847,0.9054009914,0.0433341525,-0.2319250405,1.2205764055,0.7686790824,1.4405000210,-0.3859676123,0.6124944091,-0.7336784005,-1.2352366447,0.7807996273,-2.3257813454,-0.3928811252,-0.2172492445,0.6651454568,1.7893996239,-0.3957181275,1.1372895241,0.7369244099,-0.8327875733,0.6632367373,-0.0397752263,1.4086065292,0.4437205195,1.4935532808,0.0716269463,-1.0937371254,0.4778392613,-0.4515686929,-1.5649410486,-0.0638999194,-0.4206497371,-0.5077050924,1.3147307634,0.2272905707,0.0551176965,0.3449913859,-0.6504779458,-0.0079302043,-0.6291233301,1.4332242012,1.2907489538,0.3631163239,0.5036269426,0.5870404243,-1.5838334560,1.8516979218,-0.1510378718,0.0084957005,1.7752424479,0.3407488465,-1.7290748358,-0.1309578121,0.1811762899,0.9766051173,0.8586809039,-1.4115585089,-0.5120242238,-0.9084070921,-0.6975479126,0.2765782475,0.5603165030,0.8318665028,0.1136660054,-1.2513357401,0.3720861077,0.6408976912,1.5998200178,1.0436722040,-0.3570035696,-1.7902721167,0.0918675587,-0.1112808436,0.3380652964,0.3301384747,-0.6644927263,0.6445029974,-0.6061426997,1.9779821634,0.1567000747,0.1088435352,-0.8300416470,0.1776271611,-0.3211027384,-1.1594855785,-0.4041348696,1.4324586391,0.2450418323,1.3597365618,-0.2198139727,0.7452784181,0.5204197764,-0.1552402973,-0.0877770483,0.6449214816,-2.0652017593,-0.4502398968,1.5628504753,-0.2074006945,0.5728432536,0.5312701464,-0.5582386255,0.4903011024,0.4000514448,-0.3402361572,0.1078620851,0.1054711565,0.0762601420,-0.8976479173,-2.7556171417,0.6805824637,-0.8755759597,0.5861801505,1.0099159479,-2.0829658508,0.8546541333,0.4041488171,1.4322775602,1.9776724577,2.1114580631,0.2207719088,1.9176034927,0.9244675040,1.6649698019,0.2163754702,0.0035872904,2.6416580677,-1.4437009096,0.4020408988,-1.1299657822,2.0063018799,0.3601942062,0.0552108772,-1.6590025425,-0.6537028551,0.6874204278,1.6122521162,0.2168688625,0.9429037571,0.5354552865,-0.9631315470,0.9434006214,1.5907050371,0.2731074691,0.9513245225,-0.2791555226,1.1296684742,0.7820475101,0.1685398072,-1.9420115948,-0.9730284214,1.1653062105,-1.4316834211,-1.3316079378,-0.0116488738,-0.1106063351,-0.3327346146,-0.1284098923,-0.4964047372,-0.3758531809,0.4843265712,1.5204824209,1.1139789820,-0.3578812182,1.8609610796,0.9948360920,-1.3479914665,0.8106604218,-1.0499941111,-0.6323813796,-1.4502238035,-0.7411080599,-1.1738317013,1.0985153913,-1.7316554785],[0.5024092197,0.6399983764,1.8322696686,0.2193556577,-0.2059195042,1.5266751051,0.8948894143,1.6918249130,0.5572015643,-1.3783254623,1.2419376373,-0.1381992251,-0.3036934733,-0.4471315444,1.3595137596,0.5195784569,1.0076969862,-1.0152332783,0.4372532666,-2.2233533859,0.3535255194,1.5505034924,-0.5713678002,0.0189786758,1.5610871315,0.6443794966,0.3136890829,2.5183229446,-0.1112032235,0.4580637813,0.4732072651,-0.6347094178,-0.1455913633,-0.3562349081,-0.0463523492,-0.2493555695,0.7978110313,1.3423910141,0.2120787650,-0.9567505717,1.3303672075,-0.3680039048,0.7403077483,-0.2130557746,-1.3168724775,0.9083569646,-0.4936971068,1.0476306677,0.5149489641,0.5401264429,-0.3668811321,0.5347924232,0.4814083576,-1.0775488615,0.9366270900,-0.3202053010,-0.3182254136,-1.4235715866,2.0460758209,-0.6138587594,1.4505295753,-0.0747881457,-0.8538289666,0.0534842983,-0.2949616313,0.3182134926,2.0032804012,0.2364036888,0.9947002530,-0.5805375576,1.5063704252,-0.9826948047,-1.6039320230,-0.7312178016,0.2631585598,0.0081778783,-0.2120543271,1.4288504124,0.0310508497,-0.4847147167,-0.3707052171,-0.1414912492,-0.1123282909,0.5229693055,-2.4375815392,-1.9124066830,0.1451980472,-2.0455174446,-0.4165692627,1.6499553919,0.0982848108,0.9086429477,1.1524801254,1.1450995207,-0.8715981841,0.2962348163,-0.0864251330,-0.7070257068,-0.0059363027,-0.0952185914,0.9882470965,-0.2431317419,1.0272905827,-2.2877068520,-0.0339677818,0.2500078678,-1.6668512821,0.3891637921,0.5523136258,-1.7448424101,0.6041775346,1.7510638237,-0.5160238147,0.7353807092,-1.2460041046,0.0510762073,-0.1354765445,-0.8791379929,-1.5496599674,0.1134271175,-0.2463372648,-0.0203294978,1.5303599834,-1.0988429785,0.3622785807,-0.7890076041,1.4901959896,-0.2522631288,0.6070600748,1.0430524349,-0.1125693843,-0.4349509776,0.7843998075,0.6623550057,0.0715067312,-0.2036272436,0.0255890638,1.2453645468,0.1566366106,-0.4086671472,0.8515979648,0.4264606535,0.0430518128,0.1734429896,-1.9741030931,0.6002647281,-1.0860610008,-0.4001491666,0.4027245641,-0.4328278005,-0.6476949453,-0.0458221361,0.4308312535,0.8188905120,2.1767871380,0.5373866558,-0.9187107086,1.3414472342,-0.8143109679,-0.4182454348,-0.9356334209,1.1228146553,1.3687461615,-1.2463581562,0.5486357808,0.3156320751,2.7218301296,0.4282638729,-1.6033006907,-0.2760243118,1.5372369289,0.3407447040,0.8939761519,0.2236253321,-1.2835608721,0.5933554173,-0.8735864162,-0.1540386975,1.7338123322,-0.2981918752,0.6927968860,-0.5642343163,-1.1485286951,0.7087337375,0.7157472372,0.1084082723,-1.2599521875,-0.1718678474,-0.1303246766,-1.6611258984,-0.1089953035,-0.8575206399,1.2285900116,-0.5913127661,-0.8356810808,-0.8497319818,1.6891064644,0.9818716645,-0.7118170857,-0.0595040582,1.1489208937,-0.1327075511,-0.7262676358,-0.6392194629,-2.7723453045,-1.0678972006,1.4211807251,-0.5049116611,1.0029976368,-1.7353630066,-0.0004422014,1.3126903772,2.6282687187,-0.8856875300,1.2361621857,1.1301437616,-0.7847929597,-1.4580696821,-1.9722865820,-0.3570451140,1.5762296915,0.6130509377,-0.8741570711,1.8246018887],[-0.4657349586,-1.2615884542,-0.4083859324,0.3294874430,1.1362862587,1.5787801743,-1.5691125393,1.1296247244,-0.8173444867,0.0448391438,-0.2004632354,-0.1206745356,2.1760234833,1.4095352888,1.7142325640,0.1296830177,-0.0240912717,0.1981743574,-0.5931743383,-0.6481423378,0.2112238854,0.0259035602,-0.7714331150,0.4130635262,0.4231062829,-0.1897049546,-0.2473408580,-0.5636284947,-1.4844306707,-0.5463525057,1.6333186626,-0.4062886238,0.6718820930,-0.1990920752,-0.0755890235,-2.4272682667,-1.0069460869,0.2937736213,-1.8922125101,0.0387951620,3.0649490356,-0.2021356225,-1.7997900248,0.1907552779,0.2254720032,-1.4364736080,1.7227665186,-1.0507551432,-0.3714186549,0.2494156957,0.1630332470,0.5737173557,0.3798969388,0.6762976050,2.1938946247,-1.3434695005,-1.8401567936,-0.4973259270,0.2549467981,-0.4350290895,1.7625454664,0.2511341572,2.0254280567,1.4629654884,-0.2105290741,0.1842921525,-0.8437554836,-0.1074019298,1.3886370659,0.6858979464,-0.7213442922,0.7010446191,0.2083124965,-0.5340302587,-1.8905137777,-0.2197514623,1.8318955898,0.5987322330,0.5050485134,-0.0544060431,-0.5782997012,-0.2975089848,-0.1296807826,-3.1675622463,0.4436289370,-0.8021137118,0.6727771163,-1.4421380758,1.5984972715,0.7499288321,-2.0262806416,-0.2081878334,-0.5387793183,-0.0523785017,1.5313670635,0.7524619102,-1.2980279922,-0.4864748120,-1.9687265158,-1.1095793247,-1.3467646837,0.1210370436,-0.4014209509,-0.3103523254,-0.5346525311,0.2820588350,-0.1820099205,-0.1331241280,0.2891933918,-0.2156425118,-0.4738779366,-0.4227135181,-0.3396925926,1.0411024094,0.5079514980,2.2190010548,0.1058152467,1.4273592234,-1.4032130241,-1.1333082914,0.4115984738,-1.2759627104,-1.7923833132,-0.9866417050,0.0260736253,0.2983654141,-1.0883109570,0.1540415883,0.7682629824,0.1405778527,-1.7093214989,0.8520715237,0.3354863822,1.0088881254,0.9830012321,-0.1587079316,0.2750813067,-0.5337952971,0.5333610773,0.5658074617,0.1133866310,-0.5827506781,-0.1758695394,-2.3300492764,1.3145494461,1.5094611645,-1.1486440897,-0.9897459745,-1.2299499512,0.4605937004,-0.7130784392,0.9297976494,0.1480864882,-0.6089158058,-0.8862819076,-0.0294247605,-2.0403492451,-0.4883415103,-1.0245062113,0.2432617396,-1.2315342426,-0.3076558113,-1.0241663456,0.4212306142,0.1273297518,-0.8465504050,-0.7393798232,0.0760505572,0.3653028309,0.1351646036,-0.0163296815,2.1622841358,1.6225196123,-0.9306966066,0.9292284250,0.3719254434,0.1162487194,-0.0589749292,0.2543700635,-1.6679431200,0.4688638747,1.8470935822,-2.0336110592,0.2948815525,-0.5645554066,-1.2335046530,0.6282685399,-0.3543842137,-0.7103201747,-0.4193950295,-0.7470821142,-0.7791161537,0.9769921899,1.0773099661,0.6259647608,0.8145341873,0.2484787703,-1.5553941727,0.2921783328,-0.8426584005,-1.1621177197,-0.0484674945,0.8237134218,-0.8495957255,1.6625444889,-0.5229098201,1.4123066664,1.4815165997,-0.5415441394,0.0697141588,-1.3264377117,-0.2319623828,1.2776615620,1.8105273247,0.4019567072,-0.3616358936,-1.6378117800,0.3181915879,0.9920549393,0.4044793546,-0.6421396136,-0.7581509352,1.0275832415,1.5470706224],[-0.7398016453,0.6584480405,1.1294622421,-2.4815869331,-0.8624873757,1.3350468874,-2.7070226669,-1.4999361038,0.1586216390,1.8153672218,2.8114330769,1.5783907175,-0.1268175691,0.7761794925,-0.0535948388,1.4350873232,-0.3202698529,-0.6328254342,0.6196979880,0.9736643434,-0.8059627414,-0.0634836704,-0.4779267907,0.8468277454,0.9047194123,-2.0163111687,0.8780838251,-0.4427151680,1.5682283640,-0.0647773743,1.9337058067,-0.0634923875,-0.8278262615,0.1841171682,-1.2062327862,-0.1774312556,-0.3325543106,0.5048562884,0.1616335213,-1.3787573576,0.2442163378,-0.4408593178,-0.1357885301,-0.5383255482,-0.9141966701,-0.1598681062,-0.3700854778,-2.1424248219,-0.7458830476,0.1234179586,0.9672073126,-0.1455941498,0.4572854638,0.9470093250,-0.1653638631,-1.2481099367,0.6318738461,-0.0626924559,0.5399544239,0.8403316736,0.9862011075,-0.3642502129,-0.6327438951,-1.3328721523,0.6068063378,0.8446694613,-0.6332240701,0.5425485373,0.7139165998,-0.0305935647,-1.4514397383,-1.3369790316,0.5513197184,-1.5224503279,0.5453975797,0.8725185394,1.4529868364,0.5624259710,2.0542755127,1.7089699507,0.9274854064,0.4328777492,0.7026924491,-1.3295015097,0.2746844888,0.2033832967,-0.4538957775,0.1762087941,1.7823884487,-1.4706496000,-1.2204672098,1.4729512930,0.1214189082,1.0871387720,-0.3612067699,1.7784131765,0.9309613705,0.2597780526,-1.4110622406,0.2922568917,-0.6586437821,-0.0512520969,-1.3286775351,1.6530493498,-0.7626387477,0.9156857729,1.9149209261,0.2932203114,-3.0808136463,-1.0101010799,0.6143974662,0.2382051498,-0.0467596762,-0.0474947765,0.9090418816,-0.2340825051,0.5424991846,0.4172537029,0.4624036252,0.4270164073,1.6592540741,-1.0672186613,-0.1578507125,0.0752028897,-0.9551435709,0.7953690290,-1.1591863632,0.7421058416,-0.2349259704,0.5437689424,2.1095736027,-0.0088392152,0.8944605589,-1.1591901779,1.0720796585,1.0443638563,-0.4178381264,-1.0055652857,0.2256257236,0.1818790883,0.8913904428,-0.2827850282,0.5382263660,0.5552778840,0.5970648527,-0.3059228361,-1.1920317411,1.1359806061,-0.6876258850,0.9734085798,-0.1243617684,0.7628061175,-0.8110380173,-0.7298203707,-0.0372581482,1.0960302353,0.8897184730,1.0013744831,1.2954847813,1.0943894386,1.0337970257,-0.6451089978,-1.1438543797,1.6907882690,0.6538919806,0.2325033396,0.7079269886,-0.4387639165,-0.8021144271,-1.0609641075,-0.5739922523,-0.0819813982,-1.1191276312,-1.4699013233,1.2190533876,-0.7642792463,3.6206753254,0.2141495645,0.5528623462,0.1748252213,-0.6519388556,-0.6073905826,0.3652521074,-0.9204075933,-0.7603232861,1.0630009174,-0.9325689077,-0.5900637507,-1.1881865263,1.2027990818,-0.8413612247,0.2781607509,-0.2080128193,0.1023407057,-1.2445551157,0.6239708662,1.8129197359,1.0736453533,-1.8964418173,0.0017032215,-2.1157252789,1.3135426044,0.5474115610,0.2074623406,-0.4299831986,0.8374266624,1.6306321621,-0.3326140046,-1.8029530048,0.1280977726,0.6147022247,-1.1669268608,-0.2353281975,1.2309724092,0.2109920084,0.5916821361,1.1949048042,-1.6691788435,2.2587573528,-0.8347861767,0.7055326104,-1.2176433802,-1.5963435173,-0.9394806027],[1.1430879831,-1.0527108908,-0.2372030914,0.5021895170,-0.1742438227,-0.6610577106,0.9951126575,-0.5641317368,-0.9586183429,-0.4543020725,1.5888733864,-0.5602775812,-1.4585738182,0.2766699195,-0.2458266318,0.0311310496,0.3165929019,-0.4411346018,-0.7168897390,1.6787896156,-0.5344859362,0.1416471153,0.1917462647,0.5545337200,-0.9684610367,-0.2502814233,0.8965836763,-0.3063308001,1.5524247885,-0.2281495482,0.5966132879,-0.0518924855,1.4180803299,1.1030684710,-0.5906302333,-0.4166026711,-0.1812176853,1.2438272238,0.0661688223,-0.4422826469,-0.0799380690,0.1919439882,-1.7107120752,1.0774523020,1.3768286705,-0.4153911173,0.5705907941,-0.0298591200,-0.6760396361,2.0611596107,0.6797024012,-1.9420374632,0.1937771142,-1.4224283695,2.3359291553,1.5439224243,-0.1088899821,0.3045460284,-1.9224047661,-0.8653616309,-1.4767225981,-1.0221436024,1.1010134220,1.6597355604,-0.9422587752,0.2524269819,-0.5980673432,-0.8463162184,0.1441775709,-0.0865074918,-1.4574160576,-0.6760374308,-1.1465848684,0.6518920660,0.3316673040,-2.1897077560,-1.5613869429,1.1978230476,1.8515207767,-0.1663241088,1.5600320101,0.6311469674,0.5801112652,-0.6303533316,-2.2313139439,1.1076989174,1.4743758440,0.0909270272,-2.3126523495,0.0237566549,-0.5718167424,-0.2207095027,2.1772885323,-0.4963387847,-1.0081264973,-1.5023629665,0.7746482491,0.5341107845,-0.5666756630,-1.3972266912,-0.6807674766,0.9254412055,0.5504223704,0.6605623960,0.8508396149,1.3720737696,0.1541136503,0.8757134080,0.5460981131,-0.3596907854,0.1870216429,0.0117444582,0.6488144994,-0.5489287972,-0.3713762760,-0.7516688704,1.4095368385,-0.9506681561,1.2551702261,-0.2971719801,0.7826270461,-0.1850647628,0.0467204042,-0.5125409961,-0.4186106026,0.6029446721,-0.1644520611,-0.7338615656,1.0119118690,-0.9809918404,0.1098827720,0.0214030836,-0.0427952483,1.5712121725,-1.2539938688,-1.1002814770,1.1327694654,0.7072475553,0.3967822194,-1.5087244511,-0.7145330906,-0.3722834885,0.6798279881,-1.1214778423,1.6043719053,-0.5292537808,0.6880461574,1.8885463476,-0.0577552915,0.2836357951,-0.5647696853,0.2792871892,0.8215509653,-1.3532710075,1.7211893797,-0.0139528438,1.8308835030,0.7426056862,0.7111511827,-1.6903294325,0.1072090045,-1.0851238966,1.3005352020,0.0922427997,0.3189244568,-0.2317275256,-1.6285777092,0.2906110287,0.5595755577,-1.5583685637,-0.5369408727,-0.2023313344,-0.4653785825,0.2014144212,-0.7816708684,-0.3592549860,0.8583226800,0.8177946210,0.5941096544,1.9991146326,-0.1164116636,1.6596740484,-0.1440827549,0.4968259335,0.6306084991,0.6978434920,-0.7464147806,-0.2029277682,0.0740947649,1.0179958344,0.3487094343,-0.9844735861,0.5505439043,0.2724183202,1.6677926779,2.0806522369,0.4009736776,-0.5573444963,-1.5428788662,0.3109628558,-0.1744219512,1.1310318708,-1.1780905724,0.2007820308,-0.7422007322,-0.9340967536,0.5350461006,0.0442560837,1.1695052385,-1.3321918249,0.6491150260,-0.4747289419,0.5556890965,0.7429620028,-2.0583868027,1.9008646011,0.3469739258,0.0191483814,1.1078541279,-2.2555680275,-1.9164475203,0.2035084963,-1.1060945988,0.3864742815],[1.8186519146,1.0955337286,-0.1587141752,1.2405792475,1.1290600300,0.1848345250,-1.3021864891,-0.5888038278,0.5011674762,-0.2290555686,0.1941754520,-0.0028979329,0.1620416641,-0.1024544761,1.3428152800,0.5823216438,0.5299244523,-0.4311875105,-3.9000253677,0.1455959678,0.1971248835,1.5607292652,1.1137356758,-0.4552126825,1.4894210100,-0.2594603002,1.3953379393,0.0575970933,1.1616679430,-0.5125772357,0.2737622857,0.6855733395,-2.2379720211,0.0204583369,-0.4642761946,0.5056542158,-1.1518577337,-0.0480799004,0.8714804649,-0.4403902292,0.9177758694,-0.2967298031,-0.1093246862,-0.3690237701,-0.8198360205,0.8055944443,-1.2107851505,0.3110791147,-0.1688169837,-0.3388747573,-1.0188363791,0.5383563638,0.2944939733,-0.4924473166,-1.0070977211,-1.6481902599,1.4362820387,0.6015846133,0.5694240332,1.6722046137,-1.7027004957,-1.3625906706,-0.1457322389,-0.5113507509,0.2680238783,-1.7027720213,0.7206459045,-0.5888085365,-0.6532110572,-1.0642852783,2.3046946526,-1.4179235697,0.2645181417,-0.8454362750,0.4878188074,-1.3037599325,0.7964784503,-0.1442486495,0.3195949793,1.4605239630,-0.6226558089,-1.4823346138,-0.4593512416,-0.9331763983,-2.1541390419,0.6264881492,0.0523369238,1.1512045860,-0.4332700968,-0.9674804211,-0.1992558837,1.5613337755,-0.3819993436,0.6795611978,1.0330165625,-0.6436550021,-0.0710420236,-0.8683775067,1.9464271069,1.7365958691,0.8267059922,-0.2149627805,0.3993192315,-1.1981478930,0.6476678252,1.6629068851,0.1144966111,-1.2598162889,1.8402992487,0.1836557686,-2.4232974052,1.7620264292,-1.8486828804,-0.9964275956,-1.1717942953,0.6456684470,0.2873594165,-1.3820856810,-0.7862964869,-0.2583193183,-0.8816519976,-0.5542247891,-0.6523939967,0.5725956559,0.4938937128,-0.7512942553,-0.0124218957,-0.9071369171,1.4349648952,1.1218518019,-0.1233512685,1.0522289276,-1.6539301872,-0.1291158497,-0.2237756848,-1.1045513153,0.4521828890,-0.9502961040,-0.1316948235,-0.7792828083,0.8250408173,-0.8836473823,1.2532879114,0.7888385057,-0.8419920206,2.7643673420,1.0004105568,-0.4344625175,0.2776719928,1.3631216288,-0.7419697642,-1.4716395140,-0.1492524296,0.0200016722,-0.5150440335,-0.4084215462,1.4968609810,0.4010550976,-0.7996406555,-0.3596092463,0.5699978471,-1.6187551022,-0.2355066538,0.4505786896,1.0382069349,0.1090804636,1.1643670797,-0.9958294630,0.8406545520,-0.6185593605,0.4861762524,-0.1435554177,-1.1838173866,0.6962852478,0.8449316621,-0.5324366689,0.0854075029,0.3886263371,-1.1088260412,0.0637926683,-1.5521410704,-0.8580669165,0.5527657270,-1.3862566948,-0.3908568025,0.9630132914,-0.7347410917,1.8605796099,1.0593957901,-0.3667660952,-0.6616054177,-0.5578662157,-1.0552052259,1.4580582380,1.1121358871,0.9594700933,0.2913878858,-0.1537316144,-0.5127632618,-0.9405737519,-0.6017516255,0.2945948243,-0.3719144464,-0.9353292584,-1.3446847200,0.9422808290,-0.6510933638,0.8299900293,-0.3449546695,0.6160442829,0.4052671790,-0.6935716867,1.0162554979,0.6377590299,-0.4352501035,-1.4257146120,-1.2524747849,0.7853822112,-1.5440671444,-0.1379649192,-1.7845451832,-0.3356660903,1.3356161118,-0.1294857413],[1.7964646816,0.0304738451,0.7768535614,3.1488924026,-0.2558109760,-1.0679833889,-0.7244794369,-1.2258969545,0.6932333112,0.0211398620,0.1776879132,-0.5041844249,-0.2590360343,1.1173756123,-0.4724331498,1.3155813217,0.4622018635,0.6426978111,-0.5433134437,0.4678083062,-0.3611486554,0.8685443401,0.8873092532,0.9602596760,0.2302647084,1.2853378057,0.5569596887,0.0047805822,-0.6112685800,-0.6782135963,1.8684548140,-0.0579861216,-0.5789955854,-0.5955775976,-0.0142703950,-1.9806233644,-0.3151291013,0.1911353171,-0.7444159389,0.3003838956,0.3280220032,0.3322880864,-0.4767027795,0.2453860193,-1.5312238932,0.4211049378,1.9572474957,-0.2563537657,-0.7224268913,1.6377799511,-0.7879798412,0.3240051568,0.9042937756,0.7157511115,-2.7979767323,-1.2734445333,0.9677686691,0.9257705808,1.2029485703,-1.3839800358,0.7151804566,-0.2824457288,1.1473516226,0.2747390270,-0.7925121188,-0.7553021908,1.7981562614,0.6151124835,-0.2200139612,0.2743393481,0.9763574004,-0.3318104446,0.3161179721,-1.2327262163,0.7387780547,1.0659815073,0.5092077255,0.4646466076,-0.1616929471,-0.3656149805,-0.9359837770,-1.6265611649,-1.7118582726,1.0020611286,-1.0089641809,-0.5619198084,0.0407424457,0.4716152847,-0.7087352872,0.2111394554,-0.0356240310,0.1253914833,0.3591753542,0.6934660077,0.6154317856,0.5487660170,-0.6675722003,1.3222116232,-1.3435800076,-0.3621238470,0.6056843400,-1.4640836716,-0.1939837933,1.3377373219,1.3163279295,-0.2840470374,0.1725505888,-1.0558664799,-0.5492330790,-0.2993440926,-1.0710476637,0.0941991359,0.1093792766,-2.4019165039,-0.7428352237,0.9365871549,-0.1508529186,-1.0279309750,-0.2757891715,-0.3821061254,-0.1312558502,-0.9542873502,-0.9391035438,0.7704802155,2.5137343407,-1.8576669693,0.8858221173,-1.0607217550,0.8156251907,0.7083178759,-0.2483287305,-0.3928205669,1.5658286810,-0.4305017591,0.7362316251,1.7274297476,0.6333814859,0.7388486862,-0.0623305477,0.3513028622,1.0972409248,-0.4740725160,0.1975655705,-0.0587218367,-0.9982358217,-1.9500787258,0.6160622835,0.8414084315,0.1016046256,0.6070315838,0.1194629371,-1.4341905117,-0.1593689024,-0.6355468631,-2.6104624271,-2.0266444683,1.4117767811,-0.3176827431,0.9845973253,0.0557049438,0.4061246812,-0.8875346184,0.6713454127,1.0031460524,0.3599224389,0.9649901986,0.0233377106,2.1343255043,-3.0345256329,0.0627000332,1.7538636923,1.6268380880,0.3056345880,-0.3472956717,0.4976058006,1.6890598536,-2.3212182522,0.3656421304,0.1159397736,0.8868479133,1.6109799147,-0.6043145061,0.0901816487,-0.5041403770,0.1524684727,0.4275237024,0.8015805483,0.1325070113,1.7922909260,0.0355416387,-0.0134482626,-0.1337601840,-0.4896154702,-0.4701613188,-1.1700284481,1.0802490711,-0.3751786947,1.8959558010,0.7834692597,-0.0290208533,0.2484013289,0.5301579237,1.0153846741,-0.1129957139,0.0102426130,1.1725256443,1.3199801445,-2.3764221668,-0.0671974123,0.1287298054,-0.0941253453,0.4301971197,1.1682566404,-0.8586180806,-0.0630043671,-1.1315516233,0.8523266912,0.8010767102,-0.5508226752,-2.2161388397,-1.2388525009,1.3747489452,-1.3621162176,-1.3913899660],[-0.8186293840,-0.4310851693,-0.3265868723,-0.6595625281,2.3377609253,-1.4617418051,-0.3850235641,-0.5001681447,-0.7882581949,-0.6133599877,-0.7521061897,-1.3350371122,0.8898956776,-0.1522801816,-0.1465325207,-0.3226323426,-0.4473139346,-0.5643959045,-0.9613133669,0.8519867659,0.3872238696,1.4215050936,0.3426579237,-0.9598645568,1.6452548504,1.7389665842,-0.6744043231,0.3472268283,0.2202254236,0.2865630090,-0.6780747771,2.7566978931,-0.8731807470,-0.7633216977,0.6871228814,-0.6211286187,-0.4921645522,0.0778925493,0.2215663344,-0.2430664897,-0.3674600422,-1.3664894104,-1.6891640425,-0.6190827489,0.6096336246,0.4023823142,-1.6844097376,-1.2404732704,1.2487276793,1.0760413408,0.7050176859,0.2361517102,0.0590851456,0.6469197273,-0.0474967249,0.4412597120,0.5177218914,0.2961484194,0.6438617706,-1.9988317490,1.8926148415,0.1096209288,-0.1981773376,-0.4132628739,0.1424571574,0.2664468884,-1.3339275122,-0.5177214742,-1.3208273649,0.3496943712,-1.0425800085,0.9637414813,1.1737784147,0.4498517811,-0.0644505918,0.5101744533,2.1366763115,1.4451804161,-0.4666092992,-0.4894590378,0.3287404478,0.3134740591,-1.2987185717,1.3868933916,-0.2306082249,-0.6893220544,1.3180077076,1.3273700476,0.4140501618,0.7191207409,-1.2703672647,-1.2914323807,0.7756549716,-0.5866320133,-1.0754280090,1.3346775770,-1.1625427008,-0.0874402970,0.3637840152,-0.7374979258,0.0501301140,2.0589954853,-0.1709011495,1.1750373840,-0.7916604280,0.6599710584,-0.1943487674,-0.7274425626,1.4040251970,2.2787554264,1.5396624804,-0.0670272559,0.2121014595,1.2338556051,0.1547452658,0.0879635513,-0.7265217304,-0.2062857002,-0.5102232099,-1.6904525757,0.6855853796,0.4538678527,0.6846259832,0.2891429365,-0.3029614091,-0.3062827587,0.7371539474,-0.2199067771,-0.1700534672,1.6199733019,1.2885725498,-0.7799361944,0.1090501919,0.4491660595,-0.0288286079,-0.0628983006,0.8857010007,0.2705981135,0.3398106098,1.2366462946,2.3887815475,0.2669627666,1.2540435791,-3.3561549187,1.2307398319,-0.9899719954,-0.8115420938,0.3570990562,-1.3176939487,-0.5587753654,1.2871930599,0.5928227305,-0.1087322533,-0.9933822751,1.3560775518,0.1624943316,0.0694571063,-0.0019136409,0.7378163934,1.8368467093,-0.5628502965,-2.1004903316,0.1066910550,0.9081601501,-0.0222555753,-0.3945684731,1.4343575239,-0.5552595258,1.7829042673,0.0356468856,-0.2347169667,1.4369341135,0.8930010200,0.4602114558,-1.9390178919,-0.4052565396,-1.1254340410,-0.2811224461,-0.5239998698,-1.0315814018,0.1420727074,0.3060365021,0.0850403309,0.0480993278,-1.2978804111,1.0131076574,1.3917434216,-1.8141626120,-1.2823776007,-0.1916575879,-0.3455839157,-1.2886086702,-0.8586668372,-0.6362670064,-1.4075217247,-0.9333543777,0.1762785763,-1.3649258614,1.2995932102,-1.1005318165,0.4033268690,-0.3083316386,0.0538614839,1.3773829937,-0.0019546524,-0.2184900641,0.2411379069,0.8984738588,-0.2639446259,0.2973072827,-0.1650132984,1.6599066257,-0.5840214491,0.2017857730,1.3713948727,-1.0020972490,-0.4293357432,1.4016997814,0.1269748211,0.9718328714,0.1917524785,1.1341800690,-0.8607709408,-0.6005527973],[-1.3925638199,-0.3704949021,-1.1400412321,-0.1199970022,0.3022707105,-0.3858364522,0.2210528702,0.0352988578,-0.2549173236,0.2986409068,0.4361649752,-1.0413330793,0.3294892013,1.4552787542,1.5862495899,-0.0227510352,-1.0812748671,-2.4277453423,1.2509670258,0.2331276834,0.4919901788,1.9083909988,1.1447935104,0.7911493182,-1.0623035431,1.0173264742,-0.5506180525,1.3086786270,-1.1312993765,-0.2539741099,-0.7769720554,-0.8569401503,-1.0207500458,-0.4745619595,-0.0598216392,-0.8905045986,0.7787166238,1.3677231073,0.1858465374,0.4457555711,0.7118093371,0.2962641120,3.0088768005,-0.4178437889,1.5481282473,0.4459330738,-0.9167686105,0.8193830848,0.7471025586,1.5379596949,2.2011966705,0.3746042550,-0.7496331334,-0.9353886843,0.0856559277,-0.2704099715,-0.2366278768,1.7826231718,1.6654078960,-0.1156392843,-0.3040034175,-1.1702992916,0.3079847693,0.2505033910,-2.8715040684,-0.2713453174,-0.6889963746,0.9085056782,-0.4304555953,-0.8376456499,-0.5170556307,-0.3983263373,-0.1063844338,0.5968197584,-0.8843860030,-0.5697482824,0.9696322083,0.4466669858,2.6486330032,1.0537792444,1.1315171719,-1.6960200071,1.4721640348,0.9752472043,-0.7270838022,-0.0512396209,-1.2781599760,0.0176169593,1.8218171597,-0.7481163740,0.5921722651,-0.3314355314,0.5687924027,-0.0314355269,-0.1451000571,0.0710702613,-0.9201598763,-0.7968252897,1.6790449619,-1.2463930845,-1.1037224531,0.2885740697,1.1588948965,0.6854236126,0.2255977839,0.2921647429,-0.6920856237,-1.1240699291,-0.3644988835,-1.7979272604,-1.1228899956,1.2272027731,-0.5487995744,-1.3136093616,-1.0297425985,0.7001258731,1.7890787125,-0.7551582456,-0.8016943336,0.0867572725,-0.5807051063,-1.9119019508,0.5016669035,-0.7634596825,0.4129818976,-0.6169055104,0.2314937115,0.0343069099,-0.3614818454,-0.0572039559,-0.0302359294,-0.4526371956,-1.9301646948,1.3086107969,-0.2420634627,-0.4892685413,-1.1492084265,-1.3499071598,0.6945834160,-2.6993854046,0.1622554362,1.8245487213,-2.3775310516,1.2132954597,0.4981803596,-0.4224981964,0.6930828691,0.7331420183,-0.8765794039,-0.1939884126,-1.3364939690,0.9969375730,-0.7817802429,0.1041869819,0.5689641833,-0.1894062161,1.7746828794,3.4010379314,1.7501906157,1.1160522699,0.5270024538,-0.5151191354,0.7056614161,0.3733058274,-2.1757833958,0.7578504682,1.2810604572,-1.3550686836,-0.0728973374,-0.8941698670,2.7244987488,0.6562985778,-1.3424791098,-1.4879219532,-0.4622645676,-0.8685433865,-0.8274912834,1.8957636356,-0.2304519713,0.7528501749,-0.4163318872,1.2387114763,0.4693082869,0.5144113302,-0.7112178802,-0.5271394253,0.2068586797,0.3400960863,0.4101056755,-0.7695389390,-1.8163501024,1.2588369846,-0.7120572329,-2.1530566216,-0.3808211088,0.1169305518,-1.2409464121,-0.4366751611,0.2634794712,1.7055178881,1.6841148138,1.5426491499,-0.9426856041,-0.3110214472,-0.6535518765,-0.9700681567,-1.0188583136,1.0102300644,-1.3473342657,-0.9368382692,0.2325300127,-1.4934231043,-1.2163499594,-1.3792191744,-1.7362242937,-0.4512777925,-0.5170749426,0.4252825677,0.7567505836,-0.5467865467,-1.1445674896,-1.5563454628,-0.2236038148,0.1928770691],[0.7539950013,-0.2075280994,0.5398771763,-0.9476968646,-0.9016711712,0.2438880652,-0.0737104565,1.0209364891,-0.9426563382,-0.4375885427,0.4001968503,0.2803052068,-0.2396769971,-0.5233165026,-0.0707564503,0.8131171465,-0.3948679268,1.0977296829,0.9422441721,0.0974572301,0.7161006331,-1.0236706734,0.3119589984,-1.7411181927,-0.1571304351,0.9331872463,-0.3369457722,0.4094284177,0.4208583236,0.4179211259,1.1235785484,0.7556237578,-0.3103762269,0.8064317703,0.0034907553,0.0354942977,-0.5652838349,-1.3458013535,0.3002234101,-2.1501505375,-0.0334357657,-2.6188161373,-1.2006733418,-0.0334124565,0.6445909739,-0.6677893400,1.4247175455,1.0710269213,0.4332047105,1.3313224316,-0.4722385705,0.6907301545,0.1094585806,-0.2851660252,0.3637922108,1.1852264404,0.1758918315,-0.7584307790,0.2633218169,1.0621172190,0.2851718664,-0.6434780955,-0.1335996836,0.1274035126,-0.5857658982,1.0504328012,2.0061223507,-0.4886282980,0.0543444939,-0.2162028104,-0.6167577505,-0.7822800875,0.9975462556,-0.0760741457,0.8697055578,-0.5896819234,1.4034289122,-0.6588875651,0.1298187226,0.3436029553,0.2405471802,0.8691177368,-0.2007995248,-2.4776923656,0.1163100004,-0.3875218034,-0.1351628304,0.4279449880,0.0200409684,-1.2814878225,-0.0035458973,0.5171042085,-0.7989640236,-0.5903617144,0.7532331944,-1.8532392979,-0.7018445730,1.1726995707,0.8857141137,-0.6251412630,-1.4943467379,0.8706994653,0.1993800104,0.6574698091,1.3126466274,-1.0984554291,-0.3523005247,1.7237235308,-0.4371421337,-0.2530871630,-0.3125239313,-0.1090791225,-0.2442880720,-2.3474428654,1.0265904665,-1.1621482372,0.5418043733,-0.1000503302,1.6922651529,0.4630069435,0.1329090595,0.6170014739,0.7838420868,-0.5633745193,-0.8491060734,-1.1731932163,1.7671117783,0.2489637136,0.3668170869,1.0373156071,0.4797249138,-0.3064503670,1.5058437586,-1.7131545544,0.5895314217,-0.1014877111,0.0677429512,1.6819930077,-0.5216151476,0.2966107726,-0.6731379032,-2.9088072777,-0.1597213149,-0.8009475470,-0.3718868494,-0.5039060712,-0.8859555721,-0.1667399555,0.8699729443,0.2995379269,-0.0808590502,0.1149650365,-1.2343654633,-0.8192324042,-0.2976688147,0.6526411176,0.8078739643,-2.2390789986,-1.3788408041,-0.0203030016,1.0580019951,-0.1675913930,-0.3040016294,0.4077051580,1.6256030798,-0.4806309640,1.5464959145,-0.2882610261,-0.6253515482,1.1862369776,0.2702769339,-0.0170092937,-1.1964570284,0.9761407971,-1.3807101250,-0.0267551281,-0.7193158865,0.1454325020,0.8919139504,1.2186563015,-0.5411350727,-0.0417981744,0.4048648477,-2.4425880909,1.4544825554,0.9795028567,-0.9254541397,-1.0124781132,-1.2881475687,0.0540674552,0.0661320314,0.2954331338,1.0461391211,-1.0540885925,-0.4005309641,-0.1517889798,0.4216340780,0.0530298762,0.1122975871,-0.3990632594,-0.7042989731,0.1255299598,0.3206685483,1.1385912895,-0.4752325118,0.6582928896,-0.1538144350,0.6831755042,1.7123292685,0.5671478510,0.6366145015,-0.3570763469,-0.0657979697,0.4211172163,-2.2837688923,1.0036052465,0.2398010492,1.0281908512,-0.4071367979,0.9219275713,0.0722149760,-2.6470177174,1.3578057289,0.1536241025],[0.4696274400,0.8234762549,-0.4083556831,0.3090576828,-0.6590387225,0.2341024876,0.1462763995,0.2002189904,0.2424708307,2.4530985355,0.1110666841,-1.9045937061,0.6433965564,0.4160172939,-1.7808061838,-0.1339179873,0.8255435228,-0.2238264829,0.4268227518,1.0249240398,0.8471112847,1.3706696033,1.1877846718,-0.9383885860,0.3343340456,-0.4794832766,-0.4944494367,-0.1750551313,0.3527951539,-0.8698111773,0.1937713474,-0.1925773323,0.9743717313,0.6055154800,1.2223517895,0.3614206910,-0.2067714334,-0.9952375889,0.3734723628,0.6979523897,-0.4219244421,-0.5955637097,0.9482497573,-0.7519682050,-0.7860441804,1.3740886450,0.9647676945,1.7791903019,-0.7668116689,1.3360780478,-0.6458644271,-0.1355869919,2.8971319199,-1.1330628395,-0.6012628675,0.5005002022,-0.4042484164,0.4834759235,0.3865878284,-0.3293069601,0.8250538111,0.2771557271,-0.5596065521,0.9277037382,2.3185040951,-2.0182914734,-1.8434972763,1.8513425589,-0.2051957250,-0.3890289664,0.3600519001,0.9884135127,1.2457922697,-1.3364551067,1.7537808418,0.2910317779,-0.2233881950,1.5224063396,-0.4635525346,-1.0450754166,-0.4129071236,1.3809294701,-1.6089481115,0.3078214824,0.9460044503,-0.4184794128,-1.7909796238,-1.0131839514,-0.3165536225,-2.1457698345,-0.9870025516,-0.9171952009,-0.1917024702,-0.7953843474,0.6767178178,-0.3210875988,1.3021671772,-0.4075162709,1.5203324556,0.3550294042,-1.4809036255,-1.2452964783,0.1944308579,-0.9956462979,-0.3705863357,0.5113051534,-0.5084799528,0.7373731136,-1.0020056963,-1.2280149460,0.0769010037,-1.0816389322,0.6120415926,0.5528837442,0.9699832797,-1.6074030399,-0.3329996169,0.2555321753,0.7930832505,0.5816390514,-0.6598396897,-0.2890957296,-0.7990741134,-1.4835071564,1.6605551243,0.8653522134,-1.0595493317,0.9085577726,0.3510276079,0.5514123440,0.8528275490,0.0775258243,-0.0942507908,-0.5741663575,0.4272858500,0.4936881065,-0.4890993237,0.8531798124,-0.5760458112,-1.8211534023,-0.6887232065,-1.4060014486,0.7858404517,-0.7537473440,-1.0371315479,-1.1446654797,0.8815869093,1.2654161453,-0.6383075118,-0.8257868290,-0.9015462995,0.1437274814,1.3519058228,-0.1625126898,-1.9967465401,-1.8057687283,-1.2218840122,-0.4923840463,0.6176373363,-0.7052366138,-0.7661755681,-0.2294089645,-0.0140243210,1.3474508524,0.9664565325,-0.5530868173,0.5022634864,-0.4175859094,1.2790845633,0.3473213315,-0.5481665730,0.1712707430,0.5872876048,-0.2753806710,-0.7414954305,0.4202188551,1.1896193027,0.8167036176,0.0596480407,0.9580199718,-1.3611825705,1.7055370808,0.0063230065,1.8501033783,1.4732143879,-0.0210349392,-2.0528321266,-0.0954066589,1.0599880219,0.7781971097,-0.4838570356,0.4811122417,-1.8912461996,0.3263382018,-0.2272473574,-2.0044629574,1.1873849630,1.5935777426,-2.1323070526,0.3736905158,1.1431974173,2.2025403976,0.5080515146,0.7770988941,-1.9484568834,1.8847243786,-1.0159929991,-0.3248388767,-0.1660611182,0.4894922078,0.1777799129,1.0510669947,-2.4797639847,-1.1504589319,-0.8618552685,0.9332055449,-0.4171830714,0.1696135104,0.4340079725,-0.0387988202,-0.3269042671,0.1180056855,-1.0725272894,-2.8303813934],[0.2598698735,-1.9996215105,0.7559278011,-0.9798888564,1.3117982149,0.0267896838,0.7077409029,-1.3086125851,0.4044201374,-2.3094742298,1.6848480701,0.1338439882,-0.6248208880,1.6932362318,0.0373727605,1.5558153391,-0.1714841425,-2.3042349815,-0.4278883338,-0.9955630898,-0.5958486795,-0.0447826646,-0.1579531431,1.5243920088,1.9227956533,0.0402987376,-0.5851413012,1.5017855167,2.1803050041,-3.0837733746,-1.4648975134,-0.3116869926,1.9565404654,-0.3280645013,0.4791809916,-0.5352160931,1.1737276316,0.4787496030,0.4083316028,0.3431856334,-0.6605724096,2.7108654976,0.1147972420,-0.4463745356,-0.5844126344,-1.1667927504,0.5431652665,-1.9812204838,2.9475674629,2.6613872051,-1.2210031748,-1.1133307219,1.1255795956,0.5629199743,2.0009100437,1.5682934523,0.3448376358,1.2724161148,1.6458244324,-0.6888008714,0.2147175968,0.7798329592,-0.3226068914,1.0111198425,-0.5555141568,0.0550549068,-0.4487963021,-0.0010088464,-0.2460303754,0.3926325738,-0.4658337235,-0.0550008826,0.9606356621,-1.3934432268,0.2840017378,0.4586493075,1.6391483545,0.2773736715,-0.4988467097,-1.2484349012,0.6145244241,-0.2343735695,-0.6073554158,0.1037323400,-0.4924787283,-0.8200624585,-0.8451112509,0.3545969725,-0.9189939499,1.3295829296,1.4435455799,-0.4522539079,-0.6177167892,0.4639691710,-0.1216813028,-0.2014279813,-0.3888003230,-0.7420611382,-1.0614147186,0.1071903482,-0.1916001439,-0.8809502721,1.1320753098,1.8845841885,-0.6628830433,-1.5071902275,-0.1309629083,0.3128623366,-0.8688427210,-0.6582813859,-0.2740010321,-0.7021917105,-0.0590858348,0.5003613234,0.4435341060,-0.2532876730,-0.8129278421,0.0408533216,0.9117563963,0.1363046020,0.2125895768,0.2870524526,0.4323830903,0.8272413611,-0.8387100101,-0.5534974933,-1.4483233690,0.5585784912,-1.7981764078,0.2371001542,2.4801561832,-0.3021345437,-1.9256451130,-1.3350820541,-0.2666696012,1.0199847221,0.4717119038,-1.3698065281,0.4468599856,-0.0044480967,0.1817408204,-0.4527272880,-0.1334051341,0.3836607635,-0.4943971932,-1.4046680927,0.9504975080,0.0615183525,-0.5815150142,1.3562990427,-0.6733461618,-0.1815000921,1.1351847649,-1.8547165394,0.0427258536,1.2927519083,-0.7696661353,-0.8274409771,-0.2532393634,-0.0308168735,-0.7520497441,1.3307759762,-1.3623781204,-0.4073944986,1.5803796053,2.0406692028,0.5565537810,0.2322279662,-1.0841896534,-1.7789659500,3.2291426659,1.6942933798,0.9701338410,-1.2909091711,-1.8891059160,1.4711436033,0.4616388381,3.1894185543,-1.5284318924,-0.4539346397,1.7737467289,-1.0063527822,-1.0183699131,-0.7966518998,-0.6973011494,-1.8228462934,-0.6940974593,-1.1141934395,-2.0748908520,-0.6731943488,-0.1411335617,-0.5918131471,0.3981718421,-0.4185449481,0.6575914025,1.2034171820,0.7911134362,0.4504041076,-0.1689087003,-0.6241304278,0.1604327261,1.1599665880,-0.5865268111,1.4177463055,0.7862163782,-0.9866256118,1.3610484600,0.1662872732,0.7623544335,0.4777636528,-0.4978094697,-0.9205987453,-0.6019160151,-0.5025660992,0.2650274038,1.7796533108,-0.6799931526,-1.1500374079,2.1551897526,1.2656960487,-1.8137335777,0.8788180947,-0.1766525656,1.0173312426],[-0.3976323307,-0.4874837101,1.3260022402,2.5298173428,0.3581320345,0.6151618958,-0.9057765603,0.1278074235,0.4101307094,2.0343420506,3.1579160690,-0.2835023105,-1.6285960674,0.2310625762,-0.1825703532,0.7172837853,-0.9383327365,0.7119765282,-0.1827139258,-1.0331274271,-1.5335197449,-0.0797828063,0.9077846408,1.8982933760,1.4115979671,-0.4417716563,1.4773440361,-0.0987058356,0.1569772065,0.1336328238,1.4706661701,0.3962785900,0.3535576165,-1.0396783352,-0.8096060753,-0.9630953074,-0.7505313754,-0.1620675176,-0.0327635854,-0.5966364145,-0.0887803882,0.2664997876,0.6335312724,1.0054351091,0.0889114439,-0.5581123829,1.3313112259,-0.0271414258,1.7692632675,-0.3311189711,0.2609533072,0.4870017171,0.5392374992,1.2806769609,-1.3460060358,1.5021263361,-1.1109247208,-1.3770757914,-0.5652141571,0.5334227085,-0.3018633425,-0.9164126515,-0.4580081999,0.1177384034,0.8895198107,0.8998060822,1.0108145475,0.6404081583,1.3359845877,1.2474398613,3.1175408363,-1.2394305468,1.5618791580,-0.8826349378,0.3026507795,-1.4841973782,1.9171879292,0.6733652949,0.2914884686,-0.0425878204,0.7053133845,-0.2756962776,-1.1116207838,0.2003324777,0.4464299381,-0.4472725391,-0.5560350418,0.5862920284,-1.1646692753,-0.7891066074,0.0170574617,1.8001844883,-0.3394494653,0.5136883259,-0.5891038775,0.0483943895,0.4620130360,0.4717587233,2.6874232292,0.8009564877,0.0847277343,0.5359853506,-0.5240750313,-0.2925489545,-1.8751555681,-0.0490361750,0.9783039689,1.7588734627,0.6002600789,-1.5460512638,-0.3047045469,0.1539754272,0.0915840715,-0.5992405415,-0.8042455912,1.4659844637,0.1012221351,-0.1497529149,1.3030563593,0.1541244388,1.2717978954,0.8867399096,-0.4318663478,0.0770084262,1.6258184910,-0.6968182325,-0.3166991472,-0.7483909130,0.5786557198,0.0780947804,0.2426024377,0.0229945462,0.6200746894,0.2310543954,-1.1079101563,-0.4020124376,1.2420753241,-0.6679664850,0.2542519271,1.0629830360,-0.2348063737,-0.0438308194,1.3501619101,-0.7894948721,-0.5703001022,0.3089890480,1.5118911266,2.1752941608,-0.4885034859,0.8232432008,-1.0765361786,-0.0186190307,0.3972360492,-0.6259902120,-0.7424067259,-0.0163867436,0.1629671752,-0.0893045589,0.4563429058,-0.3665952981,-0.0754317641,0.5265722871,-3.5460734367,-0.6457802653,0.7653337717,-2.1300947666,-0.6183684468,-0.1406880170,0.5203972459,-1.0640109777,-1.0158450603,-0.4575025737,0.2972842455,-0.8521395326,-0.7556797862,0.9271347523,0.7557883263,0.9758871198,-1.1998525858,0.7437992096,1.5684306622,-0.5971830487,-0.5554358959,-0.2583679557,1.1155749559,-1.0676947832,1.0032776594,-1.8513797522,-0.3517214060,0.6328516006,1.7444175482,-0.8807442784,1.4790461063,-0.1968304813,0.4180056751,0.7282510996,-1.6500500441,0.3640683889,1.1129844189,0.3351568282,-0.0096940594,-0.8879166842,-0.2117554396,-0.6859401464,0.3963009417,0.4607585669,-0.2736138701,1.7940083742,-0.1557259709,0.4512560666,0.7477252483,-0.5220136642,-0.0076781982,0.2178294808,-0.6283373833,1.9523649216,-2.1755685806,0.3284509480,0.0459350087,1.3447287083,-2.9746835232,-0.4574049115,-0.4559212029,-0.7927967906],[0.9932083488,0.5941705704,-0.0860538781,0.4814716876,-0.5481458902,-1.3757659197,-0.6130162477,-0.9668075442,-0.2767145038,1.1864753962,-0.2110439092,0.0695132017,-0.1974823922,0.2356964946,1.3622299433,1.2369188070,-1.0267732143,-1.3356039524,0.4908373058,-0.4819887877,-0.4039094150,-0.6929343939,1.2614177465,0.1905747056,1.1455303431,-2.1158614159,0.2583775818,0.4038105011,1.1752716303,0.4600075483,1.1974021196,-0.8707522750,-0.2817451358,0.0584771670,0.0356177837,-1.0699961185,-0.8630678654,-1.3466277122,-0.6549223661,1.1585097313,-0.1777383983,-0.2447415143,-0.0686794594,-0.4224056900,0.1100068390,-0.2732768059,-0.0694849864,-1.6778137684,-1.1774704456,0.2881167829,-0.3823485672,-0.2653631568,1.6647901535,-0.6484985948,-0.3476527035,0.0990425199,1.2877132893,0.2625562549,-1.7195298672,0.1339644045,0.9347825050,0.6773042679,1.3379974365,-1.1952675581,-1.4742416143,0.4088601172,0.5837417245,-0.8978670835,0.9447718263,1.2271732092,-0.1702542156,-0.7172225118,0.8332493305,-0.9608859420,0.6667991877,0.1675264537,-2.4551825523,-1.3160276413,-1.3327628374,-0.3683435619,0.2090760469,0.2522863746,-0.3697250783,-0.6468358636,-1.3560963869,-0.2139449418,-0.0944914892,1.3924498558,0.8243910074,1.6508191824,0.0590403564,-0.2889220715,-1.5590000153,-0.1294312924,-0.6941680312,1.7140065432,0.3213033080,-1.2664537430,-1.2225905657,-0.4669961035,1.2360336781,-0.0267958753,0.7786743045,0.3648271859,-0.2769484818,-1.7265735865,-2.1605618000,2.6050536633,0.5951356888,0.1564329863,-0.4526445568,0.3408089876,0.7182588577,1.1662245989,1.1812388897,-1.6080279350,-0.6149919629,0.0630393922,1.1043940783,-0.3757201433,2.8040859699,-1.2027796507,0.4753766060,-1.0247412920,0.0444119908,-0.0440445766,-0.9484578967,-0.9130100608,2.2063055038,1.8155006170,-0.4957681298,-0.0989568830,-1.0796426535,0.7822895050,-1.5829148293,-2.0142185688,-0.2487754971,-0.5554159880,-0.4700220823,1.3454676867,1.0648335218,1.0868573189,-1.7082974911,1.4550259113,-2.0609056950,0.1190925390,-0.3908568621,0.7174708843,-1.6193270683,-0.8048923016,0.3411201835,-0.4839953780,0.2841969430,1.0325852633,0.4843482375,-0.3298141658,0.8892413974,-0.3105459511,-0.9782136679,0.9867196679,0.3042748868,-1.2289379835,1.2453569174,-0.9088758230,3.6785209179,-0.8134478331,0.1131959110,0.1259584725,0.1765042692,-0.4288792610,-0.1435288638,-1.2279988527,-1.8669244051,1.1203285456,-0.7590832114,-2.0584406853,-0.3421135247,-0.6633723378,0.3519513607,-0.0436158255,0.2055463493,-0.3620091081,-0.6658552885,0.8952859640,0.4214867353,-1.3065363169,0.5173885226,-0.2587612569,-1.5498518944,-0.7420065403,-1.0491241217,-1.3573287725,0.8090244532,1.1611310244,1.3288847208,-0.1811762154,1.1714485884,-2.1874475479,-0.5001651645,2.5885710716,0.0558268018,-0.2819325328,0.3446589708,-0.4038097858,-0.1715606302,-0.1005059630,0.0135818729,1.7682633400,-1.9547574520,-0.6286581755,0.5792235732,-0.7384209037,-0.5796393156,-0.0029503813,1.5651570559,2.1308877468,0.3770836890,-0.2992276251,-0.4751704931,0.9646815658,0.8811886907,1.2541283369,-1.8058665991,-1.2195588350],[-0.2300940603,-0.2072250694,0.3269861937,-1.7808606625,0.9240894914,-0.2287798822,-0.7127582431,2.2856419086,-0.9465098381,-0.1695698649,-0.2585377395,-1.0501010418,-0.7296738625,0.5356189013,0.0110701267,-0.4649605453,-0.1267327815,0.3069141507,-0.2413745672,-1.9641823769,-0.4948186278,-0.3253749609,-1.1279431581,-0.2000069767,-0.0331251286,0.9191393852,0.2092423439,-1.7219876051,0.2334833145,0.5516022444,0.1393381953,0.8058815002,-0.6283417940,0.5322570205,-0.1071301252,1.4491105080,-0.4998187125,-0.8268104792,0.0094867535,-1.2482749224,0.4494889379,-0.0147008393,2.0177221298,-1.5565807819,-0.9726316929,-0.7886752486,-0.9195621014,-0.3232328296,-1.6523182392,-0.6833917499,1.1239200830,0.1679050326,0.3240798414,-0.7736946940,-0.5426825285,0.5655519366,-0.5882941484,-1.1837854385,1.1089788675,-0.5024980307,0.5236488581,1.0887473822,0.1232571155,-2.0052480698,0.4221430421,0.4283335209,-0.4680225849,0.8110977411,-1.0689373016,-0.5722622871,-0.9641005993,0.1786365956,0.4109344780,-0.2174949795,-1.2787374258,-0.2087196559,0.5712300539,0.4369576275,-1.6268904209,-2.0347113609,1.1750626564,0.3646789193,0.1261532754,-1.7006433010,2.5722634792,-0.4614850283,0.4970600307,0.4907185733,-0.0615386628,-0.0598403178,-1.1332823038,0.3685927987,-0.9918488264,0.2320727259,-0.3870346844,-0.2509341836,0.0421222299,-0.3261132240,0.1328040659,-1.4239165783,0.4410168231,0.8100617528,-0.0148307206,-0.5331758857,1.3579202890,-0.4871828556,-0.5692438483,-0.0303262174,-1.2603951693,0.3815560937,-0.2728674710,2.5167510509,-0.0868564248,-1.0303136110,0.7523828149,-0.0511697941,-0.8479565382,-1.6042995453,-0.7471022010,0.1003591493,0.9704436064,0.2647618651,1.9099409580,0.7866731882,0.1858402044,0.1325393170,0.9469413161,-0.4874721766,-0.0009565527,0.8496548533,-0.6209245324,0.2415722907,-1.9862576723,-1.4180949926,2.0506248474,-2.1965746880,-0.0406296737,-0.3715413809,1.2954125404,0.6550036073,1.6960650682,-0.6649290323,0.6570203900,-0.0266126283,1.8540788889,-1.2913211584,0.3372913897,-0.4722357988,-1.3040305376,0.0621721260,-0.0789120272,0.0933005512,1.3070337772,0.4582025707,0.7568929195,-0.0107256742,-0.1645700037,-1.1813340187,-0.2204653174,0.2581693232,0.8963260651,0.0670128316,0.8854981661,0.1641241163,-1.2728308439,-0.9655811191,-0.4384520650,1.6505980492,-0.5964840651,-0.2924530804,0.1418313682,-1.1281623840,-0.1900935769,-1.4205986261,-1.0110018253,-0.8043029904,1.2034075260,0.2360412180,0.7618456483,2.4332990646,-0.2489451468,-2.5425071716,-0.7711074948,-0.4524628222,-0.8661909103,-0.7690161467,1.0050204992,1.8060809374,2.0945205688,0.6686277986,0.6050437689,-1.2227077484,0.4604870081,-0.7087969184,1.8917765617,-0.0848279819,-2.1807665825,1.7026932240,0.1401837468,-0.4388398528,0.0585128926,1.5256692171,-0.2458079308,-1.3954952955,-0.8067660332,0.4578137696,-0.0693819672,-0.5427391529,0.5748937130,1.9503580332,-0.9398818016,1.0572952032,0.7284172177,-0.4724606872,0.7204971313,0.7258913517,-0.2430327237,-0.3732158244,0.8272249699,0.4962197840,-1.8620917797,-0.1028145254,1.1597520113,-0.4121288359],[-1.2027361393,0.6976327300,-2.4462964535,-1.7038713694,1.3121316433,-0.5465721488,1.1759943962,-0.0005320666,0.1727978289,0.4999870956,1.2192925215,0.0659877881,1.3122242689,-1.5925445557,-1.8933696747,-1.5688316822,0.7063481808,-0.0540814176,-0.4093583226,0.5415523648,-0.2810840607,1.2379930019,1.8455535173,-1.9942268133,1.9342218637,0.2190882415,0.2802676260,-0.6883954406,-1.4517098665,0.7597011328,1.3933690786,-0.4069502056,1.2743493319,0.7263096571,-1.1906430721,-1.7904323339,-0.7842006683,0.0496981777,1.1656367779,1.2140377760,0.5458866358,0.1088893190,-0.7287024260,-0.3762235045,-0.0407645404,0.0176126137,-0.3848236799,-0.3484652638,0.4573885798,-0.7914643884,-0.5282921195,-1.6176098585,-1.3753209114,-0.7362214327,0.2440520674,-0.5911464691,0.0714291632,1.3157017231,0.8958864212,0.7456697226,0.5918785334,-0.8514651656,1.1709183455,-0.0985445157,-0.7902092934,-0.8238563538,0.6776641011,0.6242868304,-1.7112609148,1.1113905907,0.3983590603,-0.3072124124,0.9468228817,1.0867260695,0.0427968390,-0.2378879040,1.1288268566,0.1591834277,-0.8023855686,-0.0328822881,0.9165640473,1.8892006874,-0.5545433760,0.2005936652,-0.1401738226,0.7532416582,-0.7191939354,0.1173081622,-0.9961249232,-0.0623728819,0.8635931611,-1.3111095428,1.0600906610,0.3490823209,1.0388455391,-0.1214415208,-1.2302621603,-0.3991114795,0.6279140115,-0.7386919856,-1.3221846819,-0.8256247640,-1.4266526699,-0.1054395214,0.1213134825,-1.1871228218,1.2489768267,-0.6177254319,-1.7859925032,-1.0651925802,1.6506885290,-0.0826005265,-1.7071485519,2.9751720428,0.7023817301,-0.5732915401,-1.4239773750,-0.4566698670,-1.0727081299,-1.4537141323,0.1222363412,-1.3030611277,-1.7162252665,0.1611646712,-0.0376786999,-0.5798072219,-0.8318155408,0.3107450306,0.3105241358,0.1700328141,-0.7377840281,0.1470898688,-0.0813722014,1.3564113379,0.1340189725,-0.6625532508,-0.0410240777,-1.3023015261,-1.0368205309,0.5293307304,-0.5021158457,-1.0542820692,-1.1481188536,0.1947138011,0.3171689510,-1.0173329115,0.5187175870,0.1057950556,-0.7455425262,0.4904182851,-1.0010027885,0.3528049588,-0.0746711195,1.0803209543,2.0294268131,0.4842899740,1.8676234484,-0.5828618407,0.7272090316,0.0941341445,0.9208273888,-0.4615549743,0.3433523774,1.0225626230,-1.3546812534,1.1875417233,-0.1132918969,-1.1203707457,1.2187744379,-0.4989327192,-0.9112264514,1.1047421694,1.0056560040,1.5598826408,0.6722717285,-1.3777923584,0.6470066905,0.2687519193,-0.2030589879,0.5464590192,-0.0320212431,-0.2359107584,-1.0255286694,-0.3290923834,-1.6468341351,0.0409779586,0.3232469857,0.1093000770,-1.0682225227,-1.3022935390,-0.1504627466,-1.0588222742,-0.3715017438,-1.1889908314,0.2288609296,0.2993321717,0.4552179277,-0.5519235134,0.0778525472,-0.1051733196,1.2543287277,-2.6627442837,-0.5184801221,-1.8136134148,0.1367356926,0.0721394643,-0.7105522156,0.7686612606,0.3240852356,0.8016172051,-0.2594587207,-1.3651039600,0.7902607918,-0.0750836357,-0.0406575575,-1.5975726843,-0.9080748558,-0.8948983550,0.7329773903,2.6953494549,-0.8728044629,-0.4765469730,1.2926892042,-0.8532977700],[0.2273871750,0.8500044942,0.8162901998,0.3053658307,0.1576725543,-1.2945351601,-0.2502624691,1.2744140625,-0.7250322700,-0.2637364864,-1.2683051825,0.9549968839,-1.9983717203,-0.9407818913,0.2056222409,0.9628034234,2.5071811676,0.4945580959,-0.2393418849,-0.4995599389,-1.2183736563,0.6175827384,0.3530254960,1.8969041109,-0.7900554538,1.6417257786,0.5082850456,0.7210969925,0.1719370484,-0.8770158291,1.2253770828,0.0307714008,-0.5999941826,-2.0702910423,0.1392152160,1.5867739916,-0.2774418890,-0.3968944550,-0.6462703943,-1.8079203367,-0.1435433030,0.2059432119,1.6489461660,-0.2428348064,0.8678718805,-2.1709468365,-0.0419791229,0.5046077967,-0.6470251679,-2.1490387917,1.5061701536,1.2396297455,-0.4780521691,0.0269159973,-0.7107182145,0.2932947576,-0.0467100739,-0.6241620183,1.5232892036,2.2675340176,-0.1043779999,2.0489764214,2.2696111202,1.2227553129,1.6606631279,-0.5712388754,0.0546040423,0.2590203583,1.2644896507,0.2683848739,2.7493901253,-0.9891241193,0.4144333899,0.3645968735,-1.1913316250,0.0431974605,1.7358802557,-2.4658632278,0.0826712474,0.4959769845,0.5519586802,-0.2668684125,-0.9466699958,1.6705639362,0.7358431220,0.7760394216,-1.1162991524,-1.0943129063,0.5646335483,0.5407114029,-0.5756079555,1.8289971352,0.8284856081,-1.6793274879,-0.0482726954,-0.3428310454,0.4674871862,0.4687924385,-0.7644754052,2.9350652695,1.5090997219,-1.2739346027,-0.8650171757,1.0715694427,0.7076815367,1.1324560642,-0.7596591711,0.4681403041,1.6337355375,0.3445463777,-0.3208348751,1.2135955095,1.3381587267,-0.1650899053,1.0665588379,-1.1150864363,1.0701800585,-1.3052628040,0.0422085188,1.9998065233,-1.7469418049,-0.2975669801,0.5028178692,0.8254718781,-0.4733499885,0.4095201790,-0.4338210821,-0.4989029765,-0.9439502954,0.1678249538,1.9921586514,-1.6073846817,-0.8297287822,0.0909254551,-0.6161510348,-0.5609905124,-0.2859306931,0.6985344291,-0.6712636352,-1.4290097952,0.5024514794,-0.6953155994,-0.8284509182,1.5240137577,0.3114162683,-0.5688574910,-0.5349234939,-0.4808453023,0.1530985087,0.2676455081,-0.2223212123,0.1744278967,0.2862839401,-0.6046664119,0.4904140234,0.0750555247,0.7289562225,0.6942309737,-1.0967652798,-0.7154685855,0.4513160884,2.0987625122,0.9161058068,-0.9617047906,-0.0101944050,-0.3066106141,0.0998871475,1.0319311619,0.2301389426,-1.0832765102,1.8243718147,-0.9366260171,0.1783067882,-0.1649248004,1.3017970324,0.6445472836,1.7660093307,1.3376543522,-1.1191633940,-0.1635971516,-1.6489555836,0.0993696302,0.6865174174,0.9365585446,0.2498519421,0.1184616387,0.6225427985,1.2603803873,-1.4978742599,2.1375963688,-0.1365803182,-0.1895365268,-0.2252049595,-0.1031861082,0.4588972330,-0.1754950583,-0.7891124487,-1.4106558561,-0.2436112761,1.3145904541,-0.2643491924,0.2929087579,-0.2023253739,-0.3809172213,-0.2256442457,-0.7095449567,1.6255832911,2.3382439613,0.4938969910,-1.9099097252,0.4098784924,0.9579098225,1.1237193346,-0.0052647912,0.0314161777,-0.2493994832,-0.8705860972,-0.8726430535,0.7882552743,0.7331334949,1.7048001289,-0.8281053901,-1.4328604937,-0.6085548997],[1.3891786337,-0.5192667246,0.4646701813,0.1095870212,0.7222380638,-1.0051553249,0.9021429420,-0.3742950559,-1.1925488710,0.6312025785,1.0276609659,-1.8227084875,0.1302038729,0.5725678802,0.7566816211,-0.6131682992,1.6283639669,0.5575514436,-1.5517416000,-0.3973201811,0.3933172822,-0.6209436059,2.2299447060,0.9194506407,0.1127791330,-1.3650295734,-0.3201951385,0.3234144151,0.8839156032,-1.6540412903,-0.6980065107,-0.6564571857,-0.7529318929,-1.6853379011,-0.8302558064,0.9642930627,-0.3338047266,1.1360625029,-0.1617259681,0.8039368987,-0.7213765979,0.9166084528,-0.2778778970,-0.1063275859,0.2079641819,-0.0063563455,0.2102287859,-1.0888133049,0.4468370378,1.1374158859,-0.5253376365,-0.4778427184,0.2243154347,-0.6749815345,1.0322270393,-0.9930639863,-0.4859096110,-0.3069780767,0.4539842308,0.7087320685,1.3813514709,0.0334968083,-1.5020490885,0.2795063853,0.9115028381,0.0795381293,-0.9493045807,0.0057000839,1.8318384886,0.2391988933,-0.2994287610,-0.1245497540,0.3006445765,-0.4049408138,0.0507116504,0.1041309908,-0.4472372830,0.2908937633,-1.3848526478,0.9086578488,0.3378473818,1.4619905949,-1.5845800638,-1.1525335312,-0.6863923073,-0.1426220536,-0.0538095944,-0.9843934178,0.7708935142,0.3634274900,0.7222071290,-1.4486782551,2.1479396820,-0.9167398810,-1.0138820410,-1.2438874245,1.6870024204,0.0624750219,-1.7277748585,1.0282175541,-1.0794452429,-0.9668543339,0.6164269447,1.0389000177,0.6272831559,-0.0910195485,0.3704117835,0.0858492926,-0.6236388683,0.6556786895,-1.1343281269,-0.6852283478,0.4187905490,-0.3736714423,0.4399876595,0.4863699675,-1.2893429995,1.2970787287,-0.9376946092,-0.0708927140,0.4519757032,-1.2560162544,-0.7564775348,-1.2853606939,-1.4764347076,0.0187401716,0.5181655288,0.1931071728,2.0165812969,-0.6870343089,-0.5474315882,-0.9243340492,-0.0350501649,0.4463641942,1.0694886446,-0.3940467834,0.8290531635,2.7378354073,0.6988120079,1.1682595015,1.2565833330,2.2958664894,1.0740492344,-1.1948891878,0.7294963002,-0.8258320689,-2.4308962822,-0.2404142767,0.3294350803,0.5455643535,1.1034344435,0.1110701412,0.3185352981,0.4520530403,-1.4553079605,-0.5798675418,0.3087005615,1.6675961018,-0.4612086117,-0.8659790158,-0.8224008083,-1.1954746246,2.3250622749,-0.0474405810,1.1397987604,0.4894039333,0.6380180717,0.8720409870,1.3701871634,-0.6702001691,0.1365573555,0.8938791752,-2.2082872391,-1.5051922798,0.4718648493,0.4459237456,1.0747903585,-0.3032374978,-1.9131064415,-0.9425461292,-2.1948966980,-0.5122132897,-1.1340646744,1.0650100708,-0.0310556535,-0.3919205070,1.9453985691,-0.9041585326,-1.1375190020,0.5593029857,0.3512958884,-0.2719665766,-1.2004096508,-0.9829906225,-0.5594509840,0.7834479809,-1.0296547413,-2.9886469841,-1.3362042904,1.3961710930,0.0321802497,1.6879734993,-1.9495279789,-1.3146616220,-0.6059178710,-0.1589753181,-1.5342533588,-0.0959366262,-1.0127555132,1.6585209370,-1.0970559120,-0.0442612171,1.0565162897,-0.4931052327,-2.3154871464,0.8893052340,-0.5490179062,-1.0743147135,-0.7600001097,1.4808275700,-0.6100429893,-0.0965769142,-0.1723432541,-0.3031660020],[-0.9866677523,-1.1144934893,-0.0944731832,1.2081950903,-0.5706077218,1.4837689400,1.4544682503,0.3421855569,0.5417101383,0.1148955226,-0.1729538888,-0.6006920934,-0.8390375972,-0.8688585758,1.9264659882,-0.5148521662,0.0482680723,1.4802362919,0.0934116393,0.1687042266,-0.5380268097,-0.1822315454,-0.6247577071,1.0951794386,-0.5520597100,1.1341189146,-0.9410278201,-0.6713935137,-0.7394921184,0.4502103925,-0.6198716164,-0.6839472651,0.1225380972,-0.2928015292,0.5678059459,1.3917983770,1.9239299297,-0.2305802852,-0.5392155647,1.9057199955,0.1113132760,0.2179827541,-0.3769581318,0.5038686395,0.5441030264,2.1462361813,-0.1754538417,-0.2272842228,-0.7777884603,0.3594806194,1.0193614960,-1.1204891205,0.9485693574,-1.2363845110,0.6520394087,-0.2308014482,1.6299146414,-1.1982326508,-1.4894257784,0.1250338554,1.6823540926,-0.0901871324,-1.1973161697,-1.0665248632,-1.6871834993,-0.8226858974,-0.2290683836,-0.2399158925,-1.1904006004,-1.1496270895,0.4385099411,-1.4581204653,1.4406924248,-0.3567754328,-1.2256165743,0.6519488692,-1.6003174782,1.6349955797,-0.4684672058,-0.8184659481,-1.4700213671,0.9492305517,0.9033931494,0.0192416534,-1.0961804390,-2.7277271748,-0.6291338801,-0.4750710428,-0.9310365915,-0.4051821530,0.6260236502,-1.2584766150,0.1520262361,-0.5098328590,-0.5226279497,0.5744693875,1.2434821129,0.0292647406,0.1323002875,2.9719250202,-0.2966171503,0.2232382149,-0.8948405385,1.9650298357,-0.4943556488,0.2295539081,1.1664227247,1.3640789986,1.3521430492,1.5934813023,-0.1320769340,1.0461800098,-0.6914286613,0.5779567361,-0.5560915470,-1.5874744654,0.2247067690,-0.3761298060,-0.6025452018,-1.5737394094,-0.8176524043,0.1051435396,-0.5349892974,-0.1311412752,-0.2104492337,1.1861252785,-1.3101953268,0.0564993545,-0.6865065694,-1.7655328512,0.8406751156,0.9682949781,-0.3303029239,1.2768537998,0.5624393821,-0.2575190067,0.1811937392,-1.0224636793,0.0395928435,-0.6465151906,1.3817548752,-0.8498784900,0.9605399370,-0.5322082043,-0.5752568245,-0.2549653947,1.0302226543,0.1062326059,-1.2118384838,0.1191878691,0.0014828539,-1.1837663651,1.0874757767,-2.8219051361,0.3327438831,0.3325736225,-0.9217550159,0.7638429999,1.3450644016,-1.1933127642,0.5198947787,-0.5931326151,-1.2922502756,0.1143693775,-0.0845904127,1.0581725836,0.9305245876,0.4070013165,-0.7341075540,-0.1060848758,2.1465601921,0.1787221581,1.7992449999,-1.2903478146,0.1223225221,1.4852894545,0.2455696911,-0.8079850078,-0.9959421754,1.3645406961,0.8025732636,0.2571508884,0.4182025492,0.7682970166,-0.2245483398,-0.9194589257,-0.4008037746,-1.7039490938,0.7321809530,1.0105509758,0.8314722776,-0.9105592966,-1.0720477104,-0.4215989411,0.7376125455,-2.4036571980,-0.3848825991,0.8627403975,-1.0709260702,0.9025353193,-0.2341202199,1.4916242361,1.0881841183,-0.4007686973,-0.3403339386,0.5308698416,0.7718102336,1.6546257734,2.7061583996,0.3537329733,-0.0448146500,0.1036679819,-0.7673404217,1.4605641365,-1.6149725914,-1.0541573763,0.0763381571,-0.8486981392,0.0386477970,-0.4559693038,-0.8339561224,-0.4026286006,-0.4288522899,-0.3586120307],[0.1533448994,-0.1136549860,-0.8437499404,0.6916508675,-0.4252757728,-1.9518626928,0.4714142382,0.5827259421,-1.2929222584,-0.3891405165,0.4851530194,0.5295843482,1.3490113020,-0.5542256832,0.0399686582,-0.6475527883,0.7677624822,0.1303908378,1.0854752064,1.7191438675,-1.0001112223,-0.6612618566,-2.0385735035,0.9662070274,-0.9222030640,0.2547031045,-0.1173657700,1.5544829369,-1.2023463249,0.2614167929,-1.9884563684,-0.0514406934,0.4254758358,0.2848817408,-1.1909495592,0.4748627841,-1.1672532558,-1.2991082668,0.2625207603,-0.2106069028,0.1302159131,0.6869119406,-1.2869962454,-0.4398226440,0.1023867726,1.9681735039,-0.3628941178,-0.3845193088,-1.8106294870,-0.3800882399,-0.3710056543,0.0102245891,0.5141667724,-0.9211145639,-0.1126588136,-1.1501650810,1.9226061106,1.2949194908,-0.2108839899,0.7516497374,-0.6861072183,0.4273181260,1.4809700251,-0.2807698846,-0.6138682961,0.1348924786,-0.1911712438,-1.2088664770,-1.3775349855,-1.5261063576,-0.3294872344,0.9192034602,0.6944690347,-0.2111102492,1.3722932339,1.0175527334,0.1142073944,-1.1281518936,0.5902181268,0.8330125809,0.6166064143,1.2236114740,0.4062391222,1.2697188854,-0.8363413811,0.1143634394,-0.4184566140,-0.3388504982,-0.0837198272,0.2457090765,-0.2588721216,-2.6595571041,-0.2221146077,0.2140495479,-0.8823900223,-0.4630803168,0.2623826265,0.4191049337,0.8740573525,0.9312805533,1.6630910635,-1.1229330301,-0.3988712132,0.3089698553,0.6604155898,1.0093617439,-1.6651716232,-0.7364777327,0.3452317119,-0.4719576538,0.0853512287,-0.6963783503,-0.2385855615,0.4833333492,-0.0953106210,1.8122999668,0.3582427204,-0.7948063612,0.4761179686,0.2154228389,0.6026530862,0.3933068812,0.6928840280,0.2977566421,0.6478030682,1.0729994774,0.4346579313,-0.9511593580,0.9497818351,0.8753393292,-0.3284352720,2.1367778778,1.6212879419,-0.6599621177,0.4766357243,-1.4166018963,0.7105426192,0.1458755434,-0.1393823624,0.1677254587,0.3930626810,0.0387067795,-1.0487891436,-2.1851563454,0.4409761131,-1.3168141842,-1.2756685019,0.4949780107,-0.0053673140,-0.3223702908,-0.6519203186,0.5039027929,0.2957015634,1.2806518078,-0.7631545663,-0.7418268323,1.3974416256,0.4859984517,-0.2325882018,-1.1511465311,-0.0982916951,-0.2832758725,0.0918571278,-0.1249718368,-1.4539741278,-0.2667857409,-0.5932501554,1.5480179787,0.1944077015,-1.2783476114,-0.1039233878,0.2209801972,0.5416983366,1.8491382599,0.7789201140,1.4660717249,-0.7854235172,0.7227502465,-0.0031879030,-0.3045358062,-1.3084830046,0.7555395365,-0.3509569466,0.8722602725,0.2640450895,-0.0512377284,-0.3850245178,0.2748367488,-0.8619273305,-0.5983606577,-0.2911280096,-1.1638661623,-0.1225519106,-1.1247522831,-0.7245661020,0.3172141910,1.0484007597,-0.6781160235,1.6836032867,1.7057974339,-0.7776958346,-0.7127845287,-1.7799401283,-0.9424460530,-0.2567984760,-1.7354787588,0.5987724662,0.6738944054,-0.7204313874,1.4037210941,0.9783397317,0.1899294406,0.8302225471,-0.8875858188,-0.1858076006,-0.8468185663,-1.0604618788,0.0126434006,-1.1486490965,-0.5749979019,-0.4174935520,1.3039525747,-1.1012734175,1.0699433088],[-1.4980803728,1.2565586567,0.4683552682,-0.4248012006,0.5665093660,-0.8894879222,-0.9242884517,0.5567165017,-0.2315455079,0.6107094288,0.0718754381,-0.4053229690,-0.1032371223,0.1640097946,0.0202594232,1.6416261196,-1.9592071772,0.7429304123,-0.7617667913,-0.9404531121,-1.0880265236,-0.4952817559,-0.1792133898,-1.0586580038,-0.2037551254,-2.7976267338,-0.2782055438,-0.3884856105,1.4003995657,-0.7439337969,0.1906057298,-1.2973126173,-0.5760190487,0.0010075861,-0.3164952993,0.4198083282,0.5122329593,0.6419844627,1.5902930498,-0.9253531694,0.4934574664,0.1158690378,-1.2287720442,0.8400771618,0.8152031302,-0.9727423191,-0.5200578570,-1.6350430250,0.7669951320,-0.7027513385,-0.2783911526,1.2756760120,-1.0061872005,0.5207966566,-0.1774211377,0.1268888563,1.1396467686,0.2380308509,-0.2564617991,1.5399690866,0.1576723307,0.7258236408,0.9801822901,-1.7694141865,-0.4557214975,-0.4337657094,1.1572785378,0.5312779546,-0.2003088146,0.6846209168,1.2052086592,-0.2296351045,1.1039553881,1.3138203621,2.8796958923,1.0311387777,-1.8142951727,-0.3141567707,0.2261027247,0.5151367188,3.0695443153,0.2548579574,-0.9201954603,-0.3639743030,1.4982109070,0.2461914569,-0.2713237107,-0.9528974295,0.7874175906,0.3267644346,-0.8727814555,0.1782025397,-1.1037790775,0.1650889963,-0.1240965798,-1.1618554592,0.1651325077,-0.6387265921,0.2335938811,1.3918951750,-0.9376947284,1.0478419065,-0.3835449815,1.6917550564,1.0707421303,1.5578179359,-0.4176296294,0.8883716464,-0.0825842023,1.0968365669,1.4495837688,0.3236922026,-0.0004275652,-0.0215025973,0.0394743606,0.8524720073,0.8768084645,0.8922595978,0.2185391933,0.4017852545,0.8576724529,0.8088468909,-0.3175498545,-0.4993957877,0.4370413125,0.0653270409,-0.5538098812,-0.8825448751,-0.1844616681,0.0126108462,-0.5641058683,0.2735268176,0.3900638819,-1.4210958481,1.3800935745,1.7626299858,0.2206579298,0.2094408274,0.1169288829,-0.0439985879,1.6005190611,1.9413633347,-0.9205535054,1.0154896975,0.5654467344,1.1234375238,-0.0849402025,-0.5356362462,-1.8453865051,-0.5912247300,2.3808486462,0.6380431652,-0.0779390782,-0.5372304916,-1.0714815855,-0.9567815661,-1.9812661409,-0.8885300756,0.3465723991,-1.5381587744,0.5294940472,-0.8159481883,0.1816445738,0.5993745923,-0.0688724965,0.4917052090,-2.1673610210,-1.2983940840,0.6069096327,-0.1220805272,-0.3988957107,1.2535202503,1.5654284954,0.2309513092,1.1095582247,-0.2400104105,-0.3079902530,0.2744587660,1.0943692923,0.2639151514,2.7983736992,1.5682145357,0.2655566633,1.0198340416,-0.1856144965,0.0719759017,0.6974091530,-0.3650955856,-1.0268156528,-0.0775278434,0.3819551766,-0.2707291245,0.4601927996,0.1960742921,-2.8733341694,-0.7153529525,-1.5700154305,-1.1252595186,0.8889979124,-0.1681040972,0.4190223217,-0.1275238544,-2.1446073055,0.4539523423,-1.7821577787,-0.7988014817,0.9233134389,0.0920516923,1.9260506630,1.1535536051,0.1917728931,-0.8712940812,-1.3023717403,0.2304571271,-0.2696764469,1.7969834805,-0.2386470139,0.3282249868,-0.6356702447,0.5387446880,0.8461692333,-0.2104572505,-0.8630337119,0.7488220334],[0.9744762778,-1.9335272312,0.2642577291,-0.7576102018,0.1086469367,1.0653381348,0.5454222560,-0.3583743274,-0.2401515096,0.8858023286,-1.5596271753,1.0870817900,0.4798313379,0.9968619943,1.0761747360,0.0166343134,0.7263668776,3.0349943638,0.2674559653,1.0315163136,-0.6091318130,1.3296933174,1.1060048342,0.1791629940,0.7824376822,0.3643795848,-0.0688682124,0.3397498429,0.0978621468,-0.6770811677,0.6139251590,-0.5844218731,-1.0284618139,0.8847949505,1.6860802174,-0.1588840932,-1.4511556625,0.9644990563,0.1010972485,0.7338010669,-0.1568467021,-0.9706460238,-0.6732884645,-1.0313397646,-0.8497923613,-1.4788682461,0.7079484463,0.3125767708,0.0050403099,0.4007194936,-1.1395459175,2.0679659843,-0.0851880386,-0.5869293809,-0.4736531079,0.7341102362,-0.3778316677,-0.5041046143,0.6410043836,0.0255725123,-0.6730322838,-0.4380423427,-0.6478739381,-0.0050521214,-0.3049477041,-0.8393627405,0.7373325229,0.3903099298,1.0192760229,0.7384288907,-0.3072429597,-1.5440155268,-1.2261928320,-0.6180872321,1.3727256060,-0.3824479878,-0.0834695697,0.7977703214,-0.7664447427,0.3092956245,-0.8667956591,-0.0599133223,-1.5001881123,-0.5292657018,-2.3725910187,-0.4090963602,-0.3331638277,-1.2272868156,0.8172550797,1.5381202698,-1.1170098782,-0.8876941800,0.1524083763,-1.9791057110,0.3771382570,-0.5076460838,1.2998002768,-0.8550341129,-1.0795487165,-0.1310768127,0.5569698811,-1.9104845524,0.6803012490,0.1322748959,0.3465833664,-1.1199702024,-0.3691485822,-1.7317962646,-0.8497638106,3.6870191097,2.2949135303,0.5006017685,1.6048829556,0.9236067533,0.5055042505,-0.6939823627,1.1900153160,-0.1262883246,2.3308765888,0.1819272786,-0.5119418502,1.1648005247,2.2314093113,0.6448895335,-0.6988681555,-0.0384671949,1.5574637651,-0.6268103123,-0.4058221579,0.4050198793,1.0012731552,0.7926679254,0.1292645335,-0.0604107268,0.3225554526,-0.4775817096,0.8939678073,1.1663762331,-0.5386640429,0.9164229035,-0.1918735504,1.4974851608,-0.6885712743,0.0340448506,-1.2865372896,0.7451160550,-1.6031799316,-0.4937097132,1.3747198582,-0.5014317036,0.8943790793,-1.3215848207,0.5586235523,1.5413256884,-1.4550051689,-0.9163423181,-0.5487048626,0.1736886799,-1.1643437147,-1.8016388416,0.9181058407,0.4756911993,0.0509872176,0.1600379348,0.1163054481,0.5942176580,0.2267961651,0.4649511874,-0.2198812366,1.7069232464,0.0407197364,-0.0010570940,2.6112511158,-0.8208794594,-0.8140652776,0.9350974560,-1.2511539459,1.4331966639,-0.6402844191,0.6937068105,0.5366545320,-1.0673049688,-1.0099349022,-0.7698056102,-0.1967914104,0.4551593065,0.6804695129,-0.2212719023,0.1560983211,0.2077799588,-0.2559347749,-0.6590756178,1.8647357225,1.4707669020,-1.7711633444,-0.0389776789,1.2798991203,-0.2931283712,0.4195695221,1.2389466763,1.4354935884,0.8050691485,-0.7181984186,-0.5427057147,-0.0143227829,-2.0996563435,-0.2624518871,-0.0271753389,-0.4848319292,0.1522143483,-0.3941994607,-1.3695958853,0.8463138938,0.7625041604,-2.6974196434,0.1153355688,0.9856745005,-0.1841914952,-1.1387770176,0.5851966143,0.9297739267,0.7000173926,-0.7721249461,-1.0423361063],[0.6085122228,-0.1843611747,0.2995264232,1.6914433241,0.1621672064,-0.6695007086,-1.5838277340,0.1919981539,-0.2652933002,1.2727167606,0.7557719350,1.3111314774,1.0354379416,0.0457438529,1.8368517160,0.5285716057,-0.0358069725,-0.3260233402,-0.0109968940,1.7571246624,0.8397163749,0.0457721874,-0.2342656553,-1.1719472408,0.4112628996,0.4550276399,-0.2209479064,-0.3792432845,-0.5196670890,0.6338948011,-0.4687461257,0.3789885342,-0.6974380612,0.6244715452,0.6647689939,0.1989351660,0.9490326047,0.9538723230,-0.3232532144,-1.2485311031,-1.1898362637,-1.3543343544,-0.1478466094,-1.7027926445,-0.2219823599,-1.3504548073,-1.4105155468,1.4693017006,-0.9841487408,-0.1953340322,0.8112298846,1.3687382936,0.2290214747,-0.8213371038,-0.5722841024,1.8054988384,-0.2536306977,0.0729738325,0.4649703205,0.0422217771,-0.3143117428,1.3984079361,-0.0015608553,-0.5787816644,0.5725606084,-0.3906839788,0.0330548063,-0.5729920268,-1.3764246702,1.0837403536,0.8048295379,1.1178059578,0.9453338385,0.0617002957,-0.6828507781,0.8997985721,1.0651228428,-0.1373525262,-1.5338754654,0.1621236652,0.2117942125,-0.9399450421,0.9874948263,-0.6034541726,0.1361092329,-0.3756698966,-0.0457318723,-0.6480113268,-1.4728063345,-0.3526277840,-2.5419843197,0.2226303667,-0.0023718607,-0.2957418263,0.4968234003,0.7623850107,0.4839349389,0.0289388001,-1.1439795494,1.4620765448,1.2326036692,-0.9133926034,0.4964179397,-1.0620882511,0.7359864712,-2.2167191505,-0.7982082963,2.1331815720,-0.9304955602,0.1575941294,-0.7669993639,-0.3956198692,-1.8070354462,0.3882973194,2.8525314331,0.9374451041,0.9887226820,-0.6674115062,0.2041293830,-0.6884194613,1.2304478884,-0.2918583751,0.3572739363,-0.4392817020,0.7722851038,-0.2283833772,0.2234459370,0.6006508470,-0.5135402679,1.1010036469,0.0592561252,0.9372741580,1.2490446568,0.0454107709,-1.2143180370,1.2831765413,-0.2877854705,1.1066776514,1.7132198811,-1.5615347624,1.2326842546,-0.6733837128,0.5636820793,0.2575648725,1.9115114212,-0.6896926165,-0.9419258237,-0.0117741860,0.1123029664,-0.2908022702,-0.3897866011,1.2393567562,-0.5410919189,-0.0594159551,0.5686154366,1.2410963774,-0.5654718280,-1.1199505329,-1.6332823038,-2.3909821510,0.7570685148,-0.1796463877,-1.0146815777,-0.6059055924,0.8664308786,-1.8528240919,0.6743731499,-0.6671823859,-0.8835974336,-1.0965219736,1.5275717974,0.7300167680,-0.2790175378,0.7463711500,0.1410504282,-0.8136600256,0.7124322653,0.6585643291,0.4684002995,1.6137416363,1.1167794466,-0.6949005127,-0.1380605549,-1.5743788481,0.1458098441,0.1653126776,1.1618109941,1.2810046673,-1.1934560537,0.2221141905,-1.3247392178,-0.9758756757,1.7848703861,-0.4239293635,-1.5496392250,-1.0143722296,-0.9373916388,-0.0033001278,0.4032337964,-0.0522733368,0.5052266717,-0.3572324216,-2.1708812714,-0.8520315289,1.2847210169,-0.5304069519,-0.4453616142,-1.2954775095,-0.3580422103,-1.8790763617,0.9117733240,-0.7384397388,0.3458448648,-1.1715149879,-0.5691010952,-0.3036805391,-1.4098423719,2.2985846996,-1.0172985792,0.4456582069,0.1196669713,-0.1597136408,1.3163573742,0.6440555453],[-0.4741426706,-2.1343107224,0.7861979604,1.0434623957,0.1084718406,-0.0941078067,-0.1117874235,-1.9170396328,0.3132521510,-0.3455376625,-0.5181628466,-1.0792073011,1.1498949528,-0.3038112223,0.3882457614,0.5555512309,0.8595563769,0.9541366696,1.4233583212,-2.9718353748,-0.7919180989,1.3287876844,0.6337758303,-2.0233647823,0.1994745880,-0.0833952278,-0.8845870495,-0.2299005687,0.3188984990,0.4911392033,1.5346169472,-0.2562862933,0.3037653565,0.6775609255,-0.2594868243,0.9171200991,-1.1272230148,-0.0771622583,1.9170622826,0.0905564949,-0.1810506284,0.0498206317,0.1376063973,0.8997771144,1.0737489462,0.0820906535,1.3511236906,-2.0293605328,-1.1584099531,0.2168816030,-0.0768081844,-0.5279827118,0.7440552115,2.0216920376,-0.1949700266,1.0104807615,-0.7675292492,0.1056777388,0.4347531796,0.7299080491,-0.7642510533,-0.0106105693,-0.2643010318,-1.8072508574,1.3940663338,0.3064333797,0.4452610612,0.6079790592,0.3127636611,0.5288425088,0.0853652284,0.1245446801,-1.2992873192,-1.6149802208,-1.0670560598,-0.6736964583,0.6929746270,-0.7262415886,0.9602357745,-1.0777587891,-0.2623558342,0.2148114890,0.8735753298,0.0214401037,-1.0976064205,-0.4945487976,0.7895449996,0.2681889236,0.7866907716,1.1217195988,-0.0987510309,0.4238417447,0.6273287535,0.4225327671,-0.8348941207,1.6016408205,1.5246837139,1.4992862940,1.3512203693,0.9674935341,0.0298249535,1.2098042965,0.8318428993,-0.2006926239,-1.1404662132,-1.2379252911,-0.3444960713,-0.5028432012,1.1103632450,0.7753596306,-0.6887470484,-0.8987953067,-0.0610145554,1.9189139605,-0.1576679349,0.0352119133,-1.6327770948,-0.4427270293,0.7562611103,-0.2466283292,-0.0400478803,0.9635592699,0.5934650302,-0.8815698624,-1.0712459087,-0.6394456029,2.0791168213,-1.5084906816,0.1330436021,0.5975986123,0.3697303534,-0.6510151029,-1.2109297514,-1.4117516279,0.4338284135,-0.0864954367,0.0024174221,0.9538350105,-0.4261922538,0.4513969421,1.9549031258,-1.6480233669,-1.2982463837,1.6858726740,-0.8666954041,-0.3963478506,-0.8370561600,-0.6677888036,-0.0802014470,-0.9439086914,-0.4302936494,-0.6549695730,0.2192734480,1.8352367878,1.7390035391,-0.4004605711,0.3200043440,-0.2482670993,0.8343432546,0.9010576010,-0.6557250619,-0.3174437582,-1.4815505743,0.3341366053,1.3810726404,0.3408678174,0.2780138254,0.4463123679,-1.0640423298,0.5948966742,0.9701595902,-0.3354811668,2.1228146553,-0.3853601515,0.8973402977,-1.6078014374,0.4986139536,1.5752862692,1.0740270615,0.5252494216,-0.9065878987,-0.3239499032,-0.8208144307,1.4252241850,0.7748389244,1.6811554432,-0.4814466238,-0.4224803448,0.3014148474,-0.4887431860,0.5419479012,-0.2225292623,0.3341558278,1.3281743526,-0.7061454058,1.1406314373,1.7456609011,-0.3581234515,1.8958735466,-0.9984493256,0.9563930035,-0.5737659335,0.8312965035,-1.0460712910,0.2084058076,-1.1228160858,-0.3064347208,0.0921172574,-0.1723273247,0.2616755962,1.8572127819,1.2639824152,0.0297166966,0.7248386741,-1.0261186361,0.6415510178,0.5504319668,0.0855867118,-0.2484924197,1.0947587490,0.6734647751,0.0738114491,-0.6452680826,1.9789581299],[0.3102551699,-1.3629002571,0.0274566971,-0.0190395452,0.2973560095,0.1557310671,-0.1468652487,1.4216049910,-1.3144340515,0.8182174563,1.2978197336,0.6167443395,0.2798875868,-1.1098065376,-0.4280781448,0.2754967213,-0.0831788480,-0.6103398204,0.4578257203,0.8553571105,-0.5358012319,-0.1420752406,0.3383913934,-1.3604153395,1.0594539642,0.6419425011,0.3380404115,1.0207705498,0.6028524637,-1.0487345457,0.3747584224,-0.9873114228,0.5715779662,0.8900516033,-0.3863794208,-0.3268699944,1.1698690653,0.8926932216,0.2617246211,0.1278757006,-0.8974625468,-0.4251092672,0.7840434313,0.2114642859,-0.4872782528,1.1878559589,-0.0447122194,-1.6312956810,0.9626309872,1.5028502941,-1.1069815159,0.5493133068,-0.0670908988,-0.9915824533,-0.5925250053,-0.9678844213,0.6036627889,-0.1400649548,-0.9178651571,-0.0356261842,1.5151066780,-0.1383053213,-0.3510780334,-0.4292304814,-0.5429307818,-1.5706568956,0.1042975038,0.7102490067,1.8552374840,0.3347614706,-0.7081097364,1.1634874344,2.1036181450,-0.7794679403,-0.5313645601,-2.0222191811,-0.6882897019,1.2757804394,0.9143882394,-0.7780158520,1.6167298555,1.0925157070,0.7997589111,-1.6580874920,-0.3671254814,-0.1214561462,0.5333232880,1.8171951771,-0.5902642608,-0.7296072245,1.4643578529,0.6638869047,-0.5257128477,-0.1212170571,-0.9509884119,-0.6017122865,-0.3032298088,0.3008897901,1.7487587929,-1.4716196060,-0.4065019786,0.1656065583,-1.1072632074,0.7372105122,-1.1134939194,0.6655398607,0.5232836604,-1.3998881578,-0.2136789113,-0.3169328570,-1.0540878773,0.0523890071,-0.3415353894,0.2425192446,0.2672145963,-1.0849430561,0.1197969317,1.6511855125,0.8567194939,-0.4198167622,-0.0061530750,-0.2666466236,-2.3022949696,-0.1626988351,-1.1963500977,0.0330941156,-0.2138646841,-1.1792644262,-0.8235653639,-0.5738009810,-2.5206964016,0.9328417182,1.1077494621,-0.1811641604,0.2459245920,-1.6585836411,-0.5189835429,-0.2116440684,-1.9794597626,-0.3549507856,-1.1766176224,-0.1413746923,-0.9840304852,0.2118366063,0.1956791133,0.0256031454,-1.7828342915,-0.9711146951,0.9384088516,-0.9248518944,-0.1011454016,0.3087741137,-0.8797972202,0.3986951411,-1.6176860332,1.1128709316,-2.2385592461,0.6260812283,0.1871027201,-0.0254999250,0.3770545125,0.3706595004,-1.7061337233,0.3555938900,0.3026677072,2.3950519562,-0.8477203250,-0.2803919017,-0.5236788392,1.4797307253,-0.2061602622,-1.2524269819,-0.4490454495,0.3581798375,0.4895872772,1.5629991293,0.8327241540,-0.3907397389,-0.6531038880,-1.0658258200,0.8748501539,-0.7687430978,-0.7552185655,-1.2149530649,1.1354509592,1.5423836708,-0.5928421021,-0.6696397662,-0.8266085982,0.5995821953,1.6138184071,-0.3615201712,-0.8327620029,0.4664305151,0.9396260381,-0.6726012826,-0.3961285055,1.1466346979,-1.1610549688,-0.3767644167,1.1379449368,0.1347682774,-0.0495987386,1.2985827923,-0.7667092681,-0.2375340462,-0.7819229364,0.4346813858,0.9445708394,-0.5190451145,0.9228600860,-2.0988132954,-0.2211748511,0.0915240720,0.5281891823,0.5730536580,0.3545318246,0.9084903002,0.7303026915,-0.6365323067,-0.2045491636,1.2701267004,0.0289865974,-0.0711520687],[1.9361691475,0.1886532605,1.2679781914,0.7062788010,0.2168968320,-0.2863503993,0.2347160429,-0.6281237602,0.0405537225,-0.2235246599,-0.2665000856,-0.8118832111,1.4879057407,0.3624395132,-0.2154130787,-0.6729236245,-0.9089426398,-0.1334685832,-0.1044594646,-0.6322259903,-1.7956866026,0.1443674415,1.1078892946,0.1278653294,-1.4188404083,-0.8868185282,-0.3755210340,0.6156448722,0.2973950207,-0.3261060119,0.3075840473,-1.0595344305,0.6999576688,-0.1738804132,1.1124271154,0.0801854208,0.5716896653,-1.0374674797,-0.0721909776,1.9828381538,-1.3460509777,-0.5868654251,2.1197776794,-0.8811689019,-0.5918867588,1.3096985817,0.5816102028,0.6569513679,2.4478139877,0.0861110836,0.5811345577,0.4229134023,-0.3605940342,1.5369105339,0.4504128993,0.3325541914,-2.6560053825,0.0780802891,-0.5812519193,-0.1294345409,0.2331547439,0.5030118823,0.5096856356,1.6935214996,-0.5959262252,0.2305141240,1.0273890495,0.2305488288,0.8759493828,-1.8008491993,-0.7317401171,1.7506126165,0.4269411564,-0.0429548435,-0.1476143003,-0.2250723988,1.7548139095,1.6987363100,0.4552177787,1.5571171045,0.6796240807,0.2440751046,1.0473498106,0.5143606067,1.4031444788,-1.2559754848,2.0805716515,-0.0157355964,0.3213089406,-0.2585878968,-0.9314686656,-0.2365136594,-0.5920739770,-1.4044491053,0.3690233231,-0.7666178346,-0.3709909916,0.2319030166,-0.9386507273,-1.5684202909,-0.9699392319,-0.1636213660,0.1689850539,0.2926900685,-0.6215610504,1.4952127934,0.7245684266,0.4298216999,-0.6791428924,-1.4297268391,0.2240037620,0.8946802616,-0.6121846437,-1.0005315542,-0.9501293302,-0.2909827232,-0.5344090462,2.3198983669,-2.2879838943,-0.9288844466,-0.0034081170,1.0812544823,-0.8310899734,0.8552720547,-1.0219707489,0.2177965343,-0.9733440280,-0.0174913909,0.4193081558,-0.3332124054,0.4880933762,-1.5816885233,-1.4639986753,-0.6394988298,0.2832532227,0.1952934414,-0.8858428001,-2.3023381233,0.0895171762,1.8412590027,-1.1203542948,-0.8483638763,0.2196273506,0.3113439679,0.1000621319,1.1735875607,-0.8860414624,-0.8599038124,-0.6715561748,0.1867471635,-0.2337619066,-1.1266306639,0.9036043286,-0.9000715017,-2.0205488205,-0.9581099749,0.7905055285,-1.2348706722,0.4671658576,0.4668186605,0.3785419464,-0.0996996388,-0.1534485519,0.4217807949,-0.4923402369,-1.4948923588,-0.1059649363,-1.4504761696,-1.2151101828,-1.5624502897,-1.6301554441,-1.0333613157,-0.6717498302,0.1625663489,-0.9796378016,0.2504408658,-0.4498012662,-1.3315489292,0.5227864385,-0.4967396855,-0.7746353149,-3.2565047741,-0.7625097036,-0.2435188144,-1.6264389753,0.2406946421,0.3750441670,-0.5929222703,-1.7938333750,-1.2085139751,0.0064371475,-1.7667037249,1.7038869858,0.4753684402,0.5051966906,-0.2259977311,1.6209121943,-0.4586403966,0.4619432092,0.7081202269,-0.1198732406,-1.9040656090,2.5461783409,-0.5098463893,-0.5149093866,-0.6470791101,0.0598744750,-0.0804928392,-1.1862183809,0.7540088892,0.5231956244,-0.7941627502,-0.5351686478,0.1464129686,-3.2503798008,-0.9900084138,0.0208800603,-0.0939341038,-0.7024986148,-1.4116461277,3.3354561329,-1.6807198524,0.7628878951,-1.1663935184],[-0.9231708646,0.5788217187,-1.5420737267,0.3719091415,0.4810803533,1.2950202227,1.1624336243,1.7827768326,1.2083466053,2.1456758976,0.0785913095,-0.7186422348,-2.0778264999,-0.7815722823,0.2554871440,-0.2430197746,0.3738383651,-0.1691911221,-1.5751345158,-0.6850867271,0.9226065278,-0.4163829684,0.2016043663,0.2430605590,-2.0118079185,0.6240944266,0.4783047438,-0.2266190797,-0.0495828427,1.2547620535,-0.7426257133,0.0852039978,-0.3938572705,0.3231543005,-1.2266924381,-0.7905484438,0.7264937758,-0.1436624080,0.6927309632,-0.1799502820,0.7293747663,0.2032651603,0.9094336629,-0.4490212798,-0.5067827702,0.2291303724,1.0174266100,-0.5093548894,-1.4442439079,2.3562941551,1.0144230127,0.8205124140,-0.3604024351,0.5040035844,-0.0327425338,-0.9325399995,0.7155613899,0.1095617339,-1.8171060085,0.1261477470,0.8790358305,0.9427734017,0.8065360188,0.9868785143,-0.6248038411,1.1574034691,-0.4969314933,-0.9548960328,0.2056397945,0.6338357329,0.7086056471,0.3203323781,-2.4203219414,0.5915226340,0.7968711853,-0.7548759580,-0.7817936540,1.4170876741,-0.5229552388,-0.2530987263,0.1287870258,0.1348793209,0.3662788272,0.9481594563,1.5518367290,-0.1748871803,1.5927872658,-0.5910189748,-0.2056038827,1.4960036278,1.5196142197,0.6698520780,-1.1990022659,0.4433546960,-0.9905802011,-0.8024225235,-0.3416491747,-0.1046441644,-1.0017228127,0.4857330620,0.3659887016,0.1782243699,2.0004279613,1.5760009289,1.3417978287,1.2599104643,-0.5527366996,2.0692360401,0.4941767156,2.0470237732,-1.0875812769,0.7576946616,1.7907387018,-2.9291129112,0.8575381637,-1.3554950953,-0.4839210510,-0.0655705929,-1.1057314873,-1.3319424391,1.8408344984,-1.6634352207,-0.4494746327,-0.1036276221,-0.6675081253,-0.5608288050,0.0666391999,-0.5574812889,0.2405882627,1.4979455471,-0.3060584068,0.0120285843,1.6143784523,1.1359167099,-0.9106744528,1.2418780327,-0.9291626811,-1.9791833162,-0.9713079333,0.3243151903,-1.4647428989,-1.2395029068,0.0382350758,0.8031321168,-0.8530051112,0.4882732034,0.8558873534,0.6460304260,-0.5682061315,-0.9189545512,0.8990505934,1.2603000402,-1.0781060457,-0.7336502671,0.8740951419,0.7658302784,-0.2238070220,0.6157169938,0.6769030094,1.2593274117,-1.8752324581,-0.0772253424,-0.2439778447,1.8605688810,0.3804485798,-1.8733338118,-0.5691103339,0.5537204742,-0.6699575186,0.4479410946,1.0299412012,0.7603972554,-0.4966503382,0.3634217978,0.8501811028,1.6847851276,-1.2144894600,-1.4988577366,-1.1940227747,-0.5858005285,0.6833459735,0.4837764502,0.8482929468,0.9312326312,0.5123922825,-0.9934224486,0.7389886379,1.5904425383,0.1515328288,1.8001233339,-0.5165967941,0.6810074449,1.1736761332,0.4009504914,-0.1953288466,0.5581561327,0.1997068077,2.1620819569,-0.0908827558,1.0417481661,1.6549855471,-0.7760302424,-0.3316659927,-0.2115495950,0.5966792107,-1.0887472630,0.2982098758,1.5184670687,0.4310917556,-1.4303095341,-1.6879490614,-0.5256771445,2.3723435402,-1.1201488972,-1.6246181726,-0.7182115316,2.2989356518,-0.7916751504,-0.1085834578,-0.0711057559,0.3724485934,0.3015347719,0.2816698551,-0.5369709134],[-0.7400909066,-0.4385055602,0.4664757252,0.7917868495,-0.0000355124,-1.0004467964,2.1101815701,0.9932739139,0.8110388517,1.0303093195,2.1443128586,0.2792387009,0.8437840939,1.1218912601,-1.3918541670,0.4506911635,-2.0793647766,-0.0121497940,-1.0440405607,-0.9657757878,-0.5087488890,-0.0789047405,1.4459255934,-1.5358483791,-0.4629052877,-0.3632760942,-1.0295374393,0.1215865463,-1.5990295410,0.9378693700,-1.1836384535,-1.0631419420,0.6355529428,-2.4377219677,-0.0904419720,-0.4806424975,0.2118658572,0.4217100441,0.1558341831,-0.9841300845,-2.7493028641,0.9413385987,0.3539286554,0.6233276129,-0.9138478041,-0.5024273396,-1.6548289061,1.0006732941,1.9455966949,0.0933097750,-0.1363495439,0.0907459259,0.0056755515,0.3079532683,0.9652287364,-1.1859726906,2.0238280296,0.2781577110,0.5562726259,0.7591621280,-0.1311170906,0.9709328413,-0.8000256419,-0.3964389563,0.3583988249,-1.2603985071,-0.6288074255,0.1997476220,1.5333764553,-0.9561508894,-1.1526873112,-2.0702159405,0.2626678348,2.6558797359,0.3057968020,-0.0556512699,0.4347166121,-0.2524664700,0.5330020189,2.1679821014,-0.0680899248,0.0938287526,0.3151662648,1.3177949190,0.2194497287,1.4449121952,-0.2030343860,0.3360009193,-0.9564311504,-0.8714737892,1.4866508245,0.6670100093,1.0294823647,-0.8069849610,0.7517868876,0.6363819838,-0.2490818948,-0.2208531499,-0.7862807512,-0.0412395597,1.8023437262,0.8893542290,0.8283433318,1.1427662373,1.0990648270,-0.1380199790,0.0603353344,-0.6218895912,0.6183639169,1.6191837788,-0.2276409566,-1.7644103765,-1.5434100628,0.5247124434,0.5242751837,0.7079523802,0.1106121019,-0.4096989632,-0.9592539668,1.7692834139,-0.4390759170,0.9115946293,3.0482261181,-0.0887733400,0.4471243024,0.1182788983,-0.0481228195,0.2647619545,0.1651706994,-1.0986613035,-0.2626098692,0.8710223436,0.2023158371,0.0185922198,-1.6146917343,-0.7584802508,-1.8991247416,0.1000484303,0.0444046557,-1.0100754499,2.0759148598,0.6520631909,2.0937871933,-0.1748518348,-0.0945110694,1.4562783241,-0.3171853125,-1.3235573769,-1.0152374506,-1.1026420593,0.2675947845,0.6974791884,-1.4816414118,-0.5089939237,0.8205317855,-0.6032758355,1.1610811949,0.2848510146,-0.2113544494,-0.0693584830,0.4601627290,0.2715222836,0.3543163836,1.1241178513,-1.0009111166,0.4233255386,0.8821615577,-1.2673573494,1.0599933863,-1.9858329296,1.7473354340,-0.8925181627,-1.5392049551,0.3679618537,-0.9061989188,1.6234297752,0.3026809692,-1.2521638870,-0.9746393561,-1.6885068417,0.1270830184,-1.1797159910,-0.1793961972,-1.6843693256,1.1783792973,1.2475259304,-0.2204871029,0.7822663784,-0.7288885713,0.5897322297,1.5894587040,-2.0798897743,-0.9088625312,1.0042586327,-0.4040690362,1.0435874462,1.0213680267,-0.5034691691,-2.8381204605,0.5957017541,-1.7376792431,-0.3987622857,-0.5514874458,-0.3678982854,-0.1919072121,0.9552351236,-0.2671971619,-0.7120574117,0.7922269106,0.7821462750,-0.8510670066,0.4528163671,-2.8724989891,-1.7404062748,-0.0358726680,-1.0349525213,0.9958155155,0.7042294741,1.8772702217,1.2644282579,0.5117878914,-0.7683165669,-0.3760642409,0.3992835879],[-0.6319594383,-0.1234124750,1.3657724857,-0.4243614376,0.3409079909,0.8824703693,0.3096629381,1.1415899992,-0.3601651490,-0.4110813737,1.4603716135,0.2897032797,-0.6120595336,-0.6877930760,-0.0798668489,-0.9543554783,1.1657077074,-0.7224388719,0.5840330124,-0.4707277417,0.9872319698,-2.4506506920,0.0268113595,-0.4014832675,-0.9910459518,1.8364913464,-0.1808601767,-0.6913096905,-0.4763865769,-0.8570129275,0.6800124645,0.2375830114,0.2993802130,0.6163609624,1.0929050446,-1.2402175665,-0.9390913844,-1.3111484051,1.5088058710,1.3480857611,0.5016481280,-0.2476610988,0.7036614418,-0.0966909081,1.3176031113,1.0656447411,0.2396778166,-1.9526786804,0.8595772386,-0.2595057487,0.9080550671,0.0765460432,2.1173491478,-1.0088042021,-1.7454794645,1.2033076286,0.4952604175,0.1535671651,1.0042064190,-0.2208959013,0.5841389298,-0.3922788501,-0.3614006042,-1.8522236347,0.2663365901,-0.4901424944,-0.7449171543,-0.3399548233,-0.3400003314,1.3681870699,1.0906018019,1.0649048090,1.2599071264,-0.1997389942,-1.5190547705,0.9784672260,-0.8858841658,-0.7331888676,1.7371307611,-0.9528511167,1.2391638756,-0.6803607941,-0.8854863644,0.5288819075,0.0407255515,-1.7051877975,-1.1911907196,-0.6170998812,1.8401297331,1.2869774103,0.7416225672,0.1012784541,0.9506714940,-0.6890847683,1.1691758633,0.2125397325,-0.5651474595,1.1363813877,0.3282095492,0.2325947732,0.8333733678,-0.3224644661,-0.0435054190,0.8779395223,-0.1092095077,-1.3758436441,-0.8854057789,-2.1870152950,1.4359880686,-0.5213032365,0.3130353987,-0.3004281521,1.5077716112,0.7161954641,0.4850605428,-1.3312293291,0.1999393255,0.7143920064,-0.2711220086,1.0470339060,-0.5351563096,2.1122450829,2.2901554108,0.1676862389,-0.3756659031,0.2262535393,-0.4149861634,0.7650973797,-0.7531386018,1.2070817947,1.2787936926,0.1595245451,0.1642178148,0.2609710991,-0.7997441292,-0.1890465617,0.4980888069,-0.8788653612,-1.4926164150,0.2842349410,0.1910644174,1.9113937616,1.1137232780,1.0943866968,0.9748689532,0.8634682298,-0.2590862811,-0.0192987844,-0.8789740801,-0.7913516760,0.6180747747,-0.7121517062,0.3878835738,-1.3793500662,-0.1918176562,0.0958151296,3.1094701290,0.3986200392,0.2136057466,1.1821939945,-1.1471401453,0.8588299155,-2.0046691895,-0.1928652674,-0.6466591358,0.0150989927,-1.0306770802,0.9040096998,-2.1417479515,-2.7447814941,-1.1748067141,0.5511487126,0.0278319884,0.3978701234,-0.4533764422,0.6338236332,0.2078187764,0.6372888684,-0.6748731136,-0.1286127865,-0.5645707846,0.4383159578,-0.7623828053,0.7904329300,-0.1006841809,-0.4627493024,1.0368299484,0.4300449491,-1.7973104715,0.7144021392,-0.4045287371,-0.8542004824,-1.1299022436,0.7849571109,1.1772171259,-0.8059206009,-1.1383796930,0.4343900383,-0.2633340955,-0.4062776566,0.1033182815,-0.5581795573,-0.0883774608,1.0320638418,1.4132361412,1.6514214277,0.1695428044,0.2159408629,0.8343988657,-1.1636316776,1.0585942268,-0.1515603811,0.8289833069,-0.0556437746,-1.2486417294,-0.3393330276,-0.3956955671,-0.1733885854,-0.6103331447,-1.6589229107,-1.3826439381,0.8436100483,1.1876813173,-0.5175749063],[-1.0741704702,-0.2104587704,0.8762942553,-1.0773400068,-0.2500136793,1.3766224384,-0.4211629629,1.6774389744,0.4335694313,0.1595920026,0.7415378690,0.6724870205,0.3878264725,-2.0489420891,0.8812448382,0.1120989919,-2.1312644482,-0.3068031967,1.4072669744,-1.8038721085,-1.9736406803,0.0415054634,-0.0308436342,0.9076535702,0.3400208652,-2.2132844925,2.3224930763,1.3219075203,2.3032946587,0.6108966470,-0.1145442501,-1.5732045174,1.1812853813,0.2263955772,0.4435280859,-0.4413294494,-0.2191576660,-0.4763602614,-1.1287884712,0.9238072634,-0.2126406729,-1.4508607388,-1.0319195986,0.1644870937,-1.5002006292,-1.2616881132,-1.5085011721,0.1622222662,0.4869650006,0.5460504889,0.1898836493,2.0961461067,-0.3474938273,1.8628062010,-1.1646004915,-0.9452017546,-0.5434473753,0.2812504768,-0.5410401821,1.7726508379,0.5803670883,1.0367780924,-1.6005064249,-3.0092163086,0.1392125040,2.4194037914,0.3260611594,-0.9767702222,0.5747098327,0.1420334429,0.2335471809,-0.0718801618,1.0421361923,0.5592299104,0.5113453269,0.2498618364,-0.9558623433,-0.3218068182,-0.1402815431,-1.6505886316,0.0835994780,0.9163863063,-2.8156573772,-0.7224086523,-0.2607764006,-0.7745448351,0.7839495540,-0.7016636133,-1.2114204168,-1.3935538530,-0.1395538896,-0.5087388754,0.8473874331,1.3882595301,2.1699202061,-0.2208291292,-0.3559795022,-0.7718069553,1.0199204683,-1.1986038685,1.2818330526,-0.8575738668,0.7588049769,-0.3571656942,-0.8376092315,-0.1244146153,1.0222516060,-0.2545220852,1.8373160362,-1.4680243731,1.3336762190,0.2705392838,-0.7611726522,0.6549308896,-0.0792744756,0.9494455457,1.1246320009,0.0557211973,0.8274643421,-0.0141875586,-0.4157760143,-0.2442740798,-0.1689703465,-1.4288641214,0.2172202319,0.7753660679,0.6355167031,-0.6720003486,-3.3398942947,-0.3721321821,0.7748588324,0.2912347913,-0.0677062869,0.2498321682,-2.0455522537,0.3599575460,0.8134485483,-0.7892555594,-0.9455010891,-2.1506998539,-1.1472947598,1.6073971987,0.3767109215,-1.2686046362,-0.3529419899,2.0551450253,0.9534871578,-0.3973580897,-1.4923707247,-0.5456050038,0.0431090258,-0.2758043110,-0.2455203533,2.1469547749,-0.9647199512,-2.1054956913,-1.6152031422,0.6804012656,0.7725505233,0.6639801860,-0.5187103152,-0.6598019600,-1.0491077900,-1.5657329559,0.5533331037,1.2845594883,1.1052020788,0.9238629341,-0.4194052517,0.5052201748,0.6276740432,0.4903647304,0.5449354053,0.8436488509,0.5930407643,-1.2337615490,-0.7152583599,-0.7161428332,-0.5219246745,1.1452192068,0.6020815969,-0.6623034477,-0.3540771306,0.2188376784,0.2990494668,-0.3308923244,0.3350869715,-0.0748877674,-0.0690385029,1.8809734583,-0.6586229801,0.3756047189,0.4063786864,0.9886311293,-0.9031563401,-1.2128840685,-0.5440398455,0.8093140125,0.1141676530,-0.3551281095,-1.1269905567,-1.5992106199,-1.0362365246,-0.3802249730,-0.0893609747,-1.0580019951,1.2933925390,-1.9946708679,-0.7844277620,1.2969027758,0.4970777929,0.8003904223,0.2300457060,1.2736281157,-1.0021532774,0.4828888178,-0.1670626998,-1.8273473978,0.2340641320,-0.6276967525,-1.6696863174,0.0048749070,0.9814209938,0.6604838967],[-0.5587424040,-0.5823022723,0.1978456527,0.8078650832,0.8510124087,-0.6664382815,0.8562269807,-0.6985203028,-0.3025896847,-0.7559245229,-1.0003705025,1.5326434374,-0.0137318987,-0.6858326793,-0.3156582415,0.4162408710,-0.6101112366,0.4867403209,-0.3826802969,-1.2199368477,-0.4200175107,1.0042294264,0.3964896202,-2.3662467003,0.9675166011,-0.3118633628,0.0873739272,-0.4918978512,0.6635258794,0.3169205189,-0.3862748146,-0.7754532695,0.7976115346,-0.9872649908,-0.9692402482,-0.2291807681,-0.7056533694,1.3953976631,0.3147648871,0.4875667989,0.6540688276,-0.9398791194,0.7224289179,1.1777349710,-0.9915122390,0.9375629425,-0.5861776471,0.4290914237,-0.8124514222,-0.2063853294,-0.9410168529,-1.1546858549,-0.1660162508,-0.2981323004,1.5257365704,-0.6887041330,0.4322051406,0.0187902357,0.3851979673,-1.5070592165,-0.0877329856,1.6083472967,-0.7919498086,1.7719179392,-0.5008546710,-0.7702132463,1.5552997589,0.0259561371,1.3426429033,0.0170335490,0.5972861052,-0.3192002177,0.6835110784,0.4572851360,1.8811956644,-2.5470488071,-0.4798873067,-0.4972526431,-0.4892331660,-0.7172479630,-0.0641170889,-0.0721785799,-2.2250163555,-0.7003098726,-0.9463029504,0.5387721658,0.1171965152,0.7794079781,0.9873548150,-1.2788540125,2.0102021694,0.1099134684,-1.4208669662,0.1849313974,-0.2944383323,1.1591405869,-0.2540302277,0.5338650346,2.5842995644,0.4451530874,-0.3505676389,-0.1859664619,-0.1787521392,-0.0542083494,1.6706476212,0.9630910754,0.6391386390,1.6442874670,0.0071885237,-0.5470194221,-0.0029828497,-0.0276721809,-1.6777534485,-0.8231400847,-1.2297281027,-0.7362014651,1.1312052011,-0.3959090114,0.2928639650,0.0256736502,-0.9436841011,-0.8668084145,0.1719511896,1.7600250244,-0.3097595572,-0.7261739969,-0.0894311666,2.0879607201,-1.0827295780,1.1696344614,-1.8928852081,-1.4066766500,-1.6461080313,0.0934822261,-0.0529018976,0.1754294932,-0.4250826538,-1.6005401611,0.2298563123,-0.5026178360,1.5759580135,1.0479354858,-1.3923493624,1.5411509275,0.5450043082,-1.5957812071,0.0078355968,-1.0235074759,1.4455271959,0.2516079247,-0.0059909956,0.5103890896,0.6152398586,-0.1133571044,0.0452356748,0.8750831485,-1.5508222580,-0.8830829859,0.2080126405,0.0176873859,0.0009350795,0.3643147647,-0.3425868452,-1.1389331818,0.2679989040,-0.2351417392,-0.3743028641,-0.2967391312,-0.3949703872,-1.2074548006,-1.5331141949,1.0537494421,-0.6973699331,-0.3979943097,0.9710779786,0.1486331820,0.7276759148,-0.3589974344,0.1361308545,0.5942563415,-1.1173524857,0.1162814796,0.1708787680,-0.9081650972,-1.2323310375,-0.2043872625,-0.9505392313,-1.7209446430,0.0769074261,0.3739518523,-0.9559456110,0.6215204597,0.5835911036,2.1823852062,-1.1971487999,0.4832358956,0.6317840815,-0.8399761319,-0.8457883000,-0.2453269064,-1.3662616014,0.2259954810,-0.8711085320,-1.0088886023,-0.0795122460,-0.2749667168,-1.4504717588,-1.1735156775,-0.1123355925,-1.2269114256,-0.8250812888,0.0331604481,1.8441269398,0.6902368665,0.6502194405,0.4802848697,-0.4361600578,-1.9481081963,-0.9514036179,-1.0143113136,0.4540541172,-1.0325177908,-0.1135502532,0.8384748101],[-0.1321249306,-1.8348624706,-0.9096595049,-0.0661195368,-1.1773709059,-0.2655218244,-1.0226361752,1.7931406498,-0.5638654232,-0.8700715899,0.1031277925,-0.3818658292,0.9453787804,0.2722228169,-0.4883063138,-0.7425808311,0.3954738379,3.5413982868,1.1422576904,-0.1177065670,-1.3207576275,-0.4200673103,-0.9338312745,0.4874849916,-0.7146152854,0.5772513151,-0.2517969310,-0.5349732637,0.9662621617,-0.6398757100,-0.2928552926,0.3574216366,-1.0193123817,-0.5647442937,1.3439898491,0.9467440844,-1.8994320631,-2.3763782978,-0.0155165065,0.9333648086,1.2560585737,-0.5713263154,-1.1139186621,-0.6125965118,-1.5679283142,2.0549027920,-0.7713843584,-0.5557987094,-0.5585767031,0.1265952587,0.9497575760,-1.6862744093,0.4159049392,1.6351566315,-1.0996099710,0.1305427402,0.2132267654,0.2193509191,0.1171146780,-2.9186856747,-1.6857429743,1.2380404472,0.1488410085,-0.5628337860,1.2209196091,-1.0582157373,-0.4046570361,0.1282916665,-0.8336083293,0.6628674865,-0.6343099475,0.4895289838,-1.5728354454,0.1174273640,-1.0989309549,0.7623879910,0.6364852190,-1.3394111395,-0.9577367902,1.2531023026,-0.0068191397,-1.0997195244,0.5822053552,-0.8811379671,-0.9198532104,-0.0741320997,0.0454016849,0.2210511118,-0.3389450312,0.2945863605,1.6904438734,0.2295662761,0.3174210787,0.8400560617,-0.5025610328,0.4230135083,2.0097754002,-0.6062899828,-1.3862564564,-1.6698477268,0.5712623000,0.1846959591,1.3817620277,-0.4764849842,0.0659929290,0.3075664937,2.1360414028,-0.3807438016,-0.2041121721,0.4065637887,-1.4117066860,-0.2051517218,0.6111496091,-0.4619669914,-0.3576725721,0.0207850803,0.5707749128,-0.5527526736,-0.1327189803,1.0202493668,0.6725012064,-0.1995927095,0.4116949141,1.3201146126,-0.8511425257,-0.4216827452,1.0985009670,0.5239596367,1.6738646030,1.0952597857,-0.4176313579,0.6746681929,2.0459949970,0.9361134768,0.5849192739,-0.8187872171,-0.8112853765,0.6149257421,-0.0066334433,-0.8003438115,0.0657561496,0.3939533532,0.1361585706,-1.2089350224,-0.2615604103,0.9463102221,-1.8061428070,1.6406424046,1.3682647943,0.6324747801,2.7865254879,-1.1303302050,0.3894205093,-0.0782135278,-1.0114086866,-0.6274721026,0.5245721936,-1.5063163042,-1.9922429323,0.3584777415,0.5166468620,-0.1857368648,-0.8542599678,-0.9650046229,0.4234408140,-0.3074883819,-1.7931255102,-1.5989216566,-0.3567943275,1.4546153545,0.6844121814,0.5107967257,-0.0391141847,-1.4022755623,0.3946054280,0.6230074763,0.3575500250,0.6456065774,-0.8244645596,1.6284826994,1.3106117249,0.5425267220,-0.0468281060,-0.1053166837,1.5829575062,2.1437420845,-2.6331455708,-0.8834359646,-0.0580697283,-0.2573893964,-1.0412811041,-0.9003218412,-2.2390277386,-1.4651108980,-0.1499534696,0.3560865223,-0.8711300492,0.4428383112,-0.7426754832,1.9199404716,-0.1389176399,0.8840380907,-0.0846645460,1.1519436836,0.6512896419,0.4417738318,0.5237726569,-0.0788306445,-1.0378000736,-0.0144220442,3.2748272419,-0.4563313723,0.1812780350,1.0921074152,-1.0814076662,-0.7821266651,-0.3949780762,-0.3073712885,-0.4651679397,-0.1078325585,0.8263167739,1.2388310432,-0.0984496921,-0.9567881227],[-0.2131212503,0.4681727588,-0.8755455017,-0.3477372825,-0.3609948754,0.2883796096,-0.1543822289,1.0686825514,-1.2054982185,-0.2616977692,-0.2440364957,0.3487615287,0.3591597080,0.8647454381,0.0454029441,1.3971421719,0.4091089666,-1.1374706030,0.3737558722,0.0758722574,-1.3365100622,0.3250429630,-2.3053030968,-0.0668937638,-1.0083193779,-0.7527424097,-0.0073583261,0.4523725808,1.3834358454,1.5274974108,0.5206997395,-0.1829435676,1.9268072844,1.7041747570,1.2395449877,0.8001048565,-0.6456826329,0.1719383746,-1.1980223656,0.1657909304,-0.3878783286,0.1869106740,-0.5592908263,-0.5458331704,-0.7450099587,-0.0316931196,-0.0618060678,0.3835448027,0.4217584133,1.1206994057,-0.4465995431,1.7511516809,1.3176025152,-1.5822561979,1.2087854147,-0.0655139759,-0.2714920640,-1.0172853470,0.8885841370,0.6089287400,0.7097648382,-1.0400695801,0.7560692430,-0.1527360380,-0.0720087662,2.0048029423,0.9776907563,1.8429362774,0.5951318145,0.9638172984,-1.4097329378,0.9143978953,-0.6318609118,0.3108898103,-1.0307670832,1.5938689709,0.1314638257,0.1371712685,2.2998940945,-1.1544933319,0.9436885715,-0.1268677711,2.5043098927,1.9020754099,0.0107692750,-0.6744157076,0.4011289477,0.3623802662,1.7662875652,-0.9019196033,0.2089881748,0.7885096669,0.1734537929,1.3288722038,-1.6579455137,1.5910657644,0.4592954218,1.3853743076,-0.4122804701,-0.6964764595,0.1028952003,-0.2327424735,1.3980441093,-0.2943899035,-0.1668646485,0.7435558438,0.1972083002,0.6509979367,0.2048090845,0.6409730315,0.0838581249,0.9075406194,-1.0100653172,-0.8159732819,1.1770539284,-1.1766952276,-0.4897406101,1.5740053654,-0.5039058328,-0.0056537632,1.2350809574,1.9353353977,0.7992894650,1.1764726639,0.5102623701,0.1331552863,1.6801360846,-1.4638708830,-1.4318088293,0.7316586375,-0.0776355043,0.1326100230,1.3219194412,-1.2926105261,0.5352171659,0.2413900346,-0.5880570412,-1.6252239943,-0.2039538324,-0.2277193218,-0.7866405249,0.4781764746,-0.2251786292,0.4531523883,-1.5962477922,1.4734230042,0.5247167945,1.3522722721,0.8866689205,1.1717010736,0.4088672400,2.3660075665,0.4089567363,-0.2065753043,1.2208466530,1.6604276896,-0.8562346697,0.3405393362,0.0647998601,0.8287518620,-0.6682997942,0.9128044844,0.8960058093,-0.5783953667,-1.8637040854,0.9895142913,0.1827133447,0.3518616557,0.5791466236,-0.6660627127,-0.6760500669,0.2832719982,-0.3868727982,-0.6564489603,-0.5221094489,1.2264560461,-1.3206096888,-2.1526074409,0.3476690948,0.3220793307,-0.7065263391,1.2328867912,-0.0474791229,0.6002050042,-0.6559346914,-1.1499810219,-0.2271601856,0.0775771588,1.4095139503,0.3245908320,-0.6881970167,-1.1337358952,-0.4698321223,0.0208422132,-0.2039146572,0.6458313465,-0.1945850253,-0.2234966606,-0.4036347568,-0.1006233916,-0.5893812776,-0.3851362467,0.6804109216,-0.2284515202,1.5971822739,-0.5365006924,0.3082237542,0.5073539615,0.1667853743,0.5216529369,-0.5112001896,-2.3981661797,-0.5508834124,-0.4447124004,-0.1626759470,0.1065026373,0.2490579486,-0.6605647802,0.1807012260,0.9926020503,-1.1432589293,-0.6874733567,-0.0878316462,1.0072650909],[1.5095521212,-1.6570911407,-3.3662109375,0.3644823432,0.9877869487,1.3811259270,0.1469745040,0.6534839272,-1.9952636957,-0.6510771513,-0.7774798274,1.4520498514,0.2092368752,0.4919971526,0.1677229851,0.6899916530,-0.3618898690,0.2964192331,-2.1922900677,-2.8469946384,1.2499256134,-0.3056958020,0.2330211699,1.8649996519,0.6184995770,0.0053699506,0.3032593131,1.4434729815,0.7936376929,0.1969661117,-1.3081755638,-1.2562611103,-2.0427565575,0.8932399750,-0.4132222235,1.3045953512,-0.9867917299,0.1995005459,-1.7860189676,0.7441645861,-0.3486237228,0.0734504163,-1.0310095549,2.6190226078,-0.2775891423,0.9633615017,1.5110583305,-0.8518043160,-0.3761514425,-0.1686878651,0.2582244277,-0.1115956157,-0.7016026974,-0.4886007607,0.2954337895,-0.7916867733,-1.0443811417,-1.2603008747,0.9914335608,-1.0003927946,1.2466765642,-2.2859148979,0.9926853180,-0.6228358150,0.2328247130,0.1546843797,-1.4545462132,-0.3578131497,-0.7590592504,-2.1569862366,-0.8818256259,0.6978793740,-2.1174297333,0.7130247355,-0.3263854086,1.4276014566,0.9877415895,0.5409177542,1.0951485634,0.1220243201,-1.7321188450,0.4447348714,0.6616247296,-1.3816518784,0.8129363656,-0.5154392123,1.1574823856,2.6286344528,-0.5050146580,3.4021975994,1.4216302633,-1.0183618069,-0.3884378374,0.8395509124,-0.2105975747,0.4740676880,-0.0842759758,-1.9411085844,0.4352273345,-0.1354239732,-0.9024779797,0.7749136686,-0.4862678647,-1.8972977400,-0.4917300045,-0.0748297572,0.9854032397,0.2509278655,0.8058610559,0.1157606617,0.7863579988,1.4600559473,1.1236486435,2.3348710537,-0.5767028332,-0.6702319980,-1.2023346424,0.5142666698,0.2149520069,0.6355566382,1.5429649353,-0.2103935927,0.3804515302,0.6273841858,0.3169680238,0.3265507519,-0.3104707897,1.5206472874,1.2834477425,-1.0403629541,-0.7318644524,-0.5680179596,-1.3175101280,0.0111222519,0.0148605723,-0.0837772042,-0.1081169844,-1.1673394442,0.5008950830,-0.8864564896,0.4724477530,-0.0436774604,0.1715297550,-0.3525592387,-0.9352099299,0.3690447807,1.9065877199,-1.3537904024,1.1824322939,-0.9419884086,0.1806846112,-0.2388978302,-0.4788640440,0.9938432574,-0.4293144345,-0.9021407962,-1.0721393824,-0.3390442729,-1.4578844309,1.4132852554,0.5697637796,-2.3599929810,-0.2966747582,0.9824299216,-0.3056213856,-1.2084267139,-0.4508726001,-0.6144681573,-0.2360159010,1.3597880602,-0.6465095282,-0.3063315749,-0.3036143482,-1.1422830820,-1.6548137665,0.3659083247,-0.6401305199,-1.0536401272,0.8842850327,1.0639809370,-0.7625054121,0.7067868114,-1.8095438480,1.6055014133,1.9578210115,-0.0146530466,0.1607527882,0.0634327382,0.9684318304,-0.6058360338,1.0671213865,-0.1546684355,-0.4690584838,1.0974758863,-0.3586915731,-0.5239579678,2.1667027473,-0.6392484307,-0.5902347565,-0.5282523036,1.4030483961,0.5651695132,-0.2470132709,-1.3528504372,0.7387406826,-1.3053969145,0.4459773898,0.6248256564,-0.4102109075,-1.4706453085,-0.1719833761,0.6154183149,-0.1918688715,0.7560840845,0.2232549936,-0.4626301527,1.8654347658,0.4972397983,-1.4983967543,2.1595275402,0.4665803611,0.0884945244,-1.1619651318,-0.2765117288],[-0.1715537906,-0.3518653810,-0.9461241961,-1.1457321644,1.2026308775,2.4595475197,0.4014116824,0.6902657747,-0.2663109899,-1.3605632782,-0.5054963231,-0.4462118149,-0.6951832175,0.6664215326,-0.1848514378,-1.6823832989,-0.3947536945,-1.2504912615,0.6512746811,0.1747264266,-0.0348788761,-1.1168770790,-2.1164686680,-0.6573825479,-0.4557669163,0.4483180940,-0.3280870020,1.1538000107,0.6581790447,-0.1787044406,-0.6257719398,-0.2563052475,1.0647335052,-0.6776366830,-0.0441894680,0.8303117156,-0.6585288048,-1.7951769829,-0.4860822260,-1.2639374733,0.8270537853,-1.5556081533,0.4140720665,-1.3718259335,-0.4767510295,-0.7637899518,0.2245294452,-0.2684869766,0.9843692183,-0.5093818903,-0.8165298700,-0.0653958842,1.1830229759,-0.5425726175,2.0562729836,-0.5610432625,-0.5977123380,0.9381351471,-0.2507341206,-0.3926568329,0.6979966760,-1.9392474890,-0.5521243811,-0.9458277822,-1.1004375219,-1.6574097872,-2.4388780594,0.7281301022,1.3262432814,-0.3952213824,-1.2761065960,-0.4463176131,-0.1996756494,0.0460203737,-1.8395447731,0.8358345628,0.0776033774,0.3844222426,0.2353907377,-0.5447943211,0.9198400974,-1.0590447187,-0.7494023442,0.7968676686,-0.1726324707,1.8498902321,-0.1682839394,0.1879395097,-1.7048875093,-0.4969268739,-0.1124747172,2.0928378105,0.1882959604,-0.1514537334,2.7660396099,0.4743964076,-0.1937914789,2.7596483231,-0.7834467888,1.0347872972,0.0152278300,-0.7610386014,-1.9078309536,0.3805552125,0.3727011681,-0.0694252625,-1.4736522436,0.2938594818,1.3245332241,1.7787688971,-0.0070120962,1.6617984772,0.8333523273,-1.7519874573,-0.2134312093,-0.0100032743,-1.4998146296,-1.8115137815,0.9965004921,-0.3041839004,0.4738995433,-1.0041937828,0.5569674969,0.8938269019,-0.8305382729,0.0593917184,0.3627528250,-2.4793670177,-0.0537039153,1.3435860872,-0.1680946201,-0.2323862910,0.3036299050,-0.5398108363,-1.0809445381,0.3632449210,0.6647271514,0.7428128719,-0.1369682550,-0.0496443063,-0.1413195878,0.9341860414,1.8201687336,1.4392632246,-0.4426941872,0.5988597870,-1.0540181398,1.5845466852,-1.8445963860,-0.6951264143,-0.7396306992,0.6659613252,0.5334790945,-0.1635098904,0.0039233668,-0.6267506480,-0.6743814945,-1.3609571457,2.2019507885,1.0629549026,-3.3453154564,-0.4029050469,0.1154041514,0.7916713953,-1.0441731215,-1.8813159466,-0.8272339106,-0.0320983604,-0.6287982464,0.2015765011,0.4855963588,1.6028063297,0.8580584526,1.3570153713,0.7573254108,-1.0525548458,-0.8210426569,-0.3900167942,-1.5697445869,-0.6533695459,-0.0810939893,0.5408882499,0.7513781190,-0.4506444633,0.9149255753,1.4994388819,-0.0303670596,1.0417119265,-0.2023945749,0.8270350099,-0.1792833060,-0.0137997605,-0.6589713097,-0.3070053756,0.0456016734,-0.0929416865,-0.8315680623,-0.9879750609,-1.9678112268,0.4647029638,0.3763283193,1.1709362268,0.8068950772,0.3331092000,-0.2832021415,0.1535404325,0.0047497978,-0.6976765394,0.0749599189,-0.1627242714,-0.6791628599,0.5772633553,1.5324664116,-1.1410440207,1.7173656225,-0.3408393562,0.9652553201,0.2716450393,-0.7487616539,-0.1968285292,0.1606321335,0.6696608663,-2.2959189415,-0.8271072507],[-0.0054756412,-0.2375295013,0.0163776018,-0.8759958148,0.5110145211,1.9992854595,-1.2928618193,0.5799829364,-0.7743336558,0.9561066628,0.0935266614,0.3667050898,-1.1060663462,1.1112999916,0.8064696789,-0.7583508492,1.5314108133,1.2391928434,0.0666859671,1.9098252058,1.2984734774,0.4810087383,0.2941254973,-0.0169484559,0.4986978769,-0.3291710019,1.6480085850,-0.9981449246,0.5191267133,-1.3664671183,0.2130313367,-1.1082155704,-1.0726953745,0.0305134151,-1.2059307098,0.3432687521,0.2289526612,0.8410323858,0.9339575768,0.9122983813,0.1333952397,-1.7370053530,-0.3468855321,0.2191209644,0.5838090181,-1.1512639523,-2.1934931278,-0.4012824297,-0.5996697545,-2.5864920616,-2.8078482151,-1.9428218603,0.5865498185,1.0440528393,0.2802296579,0.2726215720,-0.7898737192,-0.5721588135,0.9211881757,0.9148751497,0.1910182387,1.4056829214,0.4825052023,0.3429075181,-0.5708855987,0.3786260784,-0.9315177202,1.0208618641,-0.9725951552,1.0935645103,-1.2232342958,1.0409260988,0.6748644114,-0.8847046494,1.7760386467,-0.0125171226,0.8016899228,1.8771826029,0.2736357450,-0.6259844303,-0.9930151105,0.3236394823,-0.2595492899,-0.1605339348,-0.1315153837,0.6235751510,0.6957799792,1.6935269833,0.4405800402,0.0049002608,-0.5700479746,0.2159934789,-1.4336683750,-0.3797477186,1.4713785648,1.8703402281,0.0075660832,-1.4935919046,-0.8821055293,-0.6234281659,0.9677629471,-0.7328437567,-0.2575615942,1.0310820341,1.8457337618,0.0857489705,-1.4493061304,-1.2911778688,-0.0060188952,-1.2698920965,-0.5432638526,0.1579181850,-0.7544699907,-1.0726186037,1.2530040741,0.0434547551,0.6073220968,-0.6502922773,-0.3982699513,-0.2588134706,0.0140502499,0.7154316902,-2.1796958447,1.3421326876,0.9615692496,-0.5703505874,2.5212097168,-0.0208987631,1.0348778963,1.9658333063,0.3773489296,1.4068762064,-2.7714827061,-0.5579730272,1.2298541069,0.6545353532,0.8766176701,1.4339731932,1.1497399807,-0.5089753866,-0.5128499269,0.5389691591,1.3455309868,-0.8251724839,-0.8975332975,0.8422950506,0.8494792581,-0.1416618824,-0.6297848225,2.3947386742,0.1711565703,1.2305666208,-1.0552343130,2.2050673962,-1.1041059494,0.0757807940,0.5562387109,-0.3192164898,-0.7332569361,0.2332962155,0.8301983476,-1.3935382366,-0.1565247476,0.2479268014,0.0560801663,1.7070498466,0.7718081474,-0.7962634563,0.7645039558,-2.1454570293,0.6315026879,0.4187367857,1.1163557768,1.2457189560,1.1284046173,-0.0940276459,-2.3251667023,0.5586260557,-0.3234356046,-0.3314171135,-0.7101279497,-0.1197030023,-1.7641125917,0.6344687939,-0.3916080594,2.1762027740,-0.9665816426,0.0275986735,-1.0265176296,-0.7908039093,-0.2755824327,-0.1384730488,1.0248076916,0.8314927816,-2.0504336357,1.2907943726,-0.2341943830,-0.6094402671,0.5007750988,-0.3549800217,0.1185565591,-1.1103465557,-0.2405512780,0.4953965843,-0.0646155775,-0.4671186805,1.1927411556,-0.6523253322,-0.5857589245,-0.2545292974,-0.1239542216,-0.0203897934,-0.0034907917,-0.6640003920,-1.3702145815,-0.8654106855,0.0326672718,1.3138712645,0.4208591878,-0.9769559503,-0.8214844465,0.1297749430,-0.1297391802,1.0146135092],[1.7940646410,0.2197928280,-0.1319720149,1.0318607092,-1.8993104696,-1.7712563276,0.2810631990,0.4633574188,1.6809766293,-0.3751383126,0.7755542994,1.0474252701,0.2971350551,0.5951340795,1.6116594076,0.1424589455,1.2979151011,-0.4820315242,-0.1617800742,0.0396738835,-0.8928088546,1.1956772804,0.3390346169,-0.4060118496,2.4652819633,0.8918821812,-0.0780620202,0.7143768072,0.3408457935,-1.7165350914,-1.1632332802,0.3031347394,-0.0014043999,0.3238669932,0.6181593537,0.0723976716,1.0229631662,1.1097664833,-2.3270466328,1.3248306513,-0.2984437943,-1.4610385895,-0.0516262911,0.5079306960,-1.0296018124,0.0228735339,1.1103858948,0.0523694046,-0.0578812994,0.1035495922,-1.4611114264,0.3591971695,-0.9242418408,0.3373346329,0.5315905213,-1.1278209686,-0.1850767732,0.0478607044,1.8553361893,-0.1041360721,-1.0382961035,0.5020739436,0.0162114426,0.4946565330,-0.3373891413,-0.8022236824,1.7341907024,-0.0187101774,0.7622405291,-1.3417707682,2.0679044724,-1.0047054291,-1.5931515694,0.6079227328,0.6830322742,0.1589563638,-1.3811666965,-1.2644420862,-1.9886835814,1.9554864168,1.1086750031,1.0685912371,0.0946351141,-0.1953218877,0.3039997220,0.4832267761,1.2116614580,-0.5229575038,-1.0833784342,-1.9380421638,-0.1046403423,-1.2439750433,-0.1244251058,1.0000693798,0.9575945139,-1.6811643839,0.2811778784,0.8408188224,-0.7864719629,-2.2276020050,1.8322992325,-0.4967197478,-0.7136280537,0.0266603082,-1.4942921400,0.8555268049,-0.7208786607,-0.8286910653,0.0459153466,-2.0120952129,0.3852713704,0.0434668995,-0.9490215778,-1.5576071739,1.1132159233,1.4050958157,-0.4533327222,0.4062691331,-0.7710663080,0.4899501204,-0.3605414927,-0.7879081964,-1.4783047438,-0.6426596045,0.0291680340,-1.8412855864,0.3601514101,-0.1602088660,1.1215077639,-0.6920269132,0.4370068312,-0.6374166012,0.9822850823,0.0243866947,0.1872158647,-0.0480282828,-0.2081749886,0.2828293145,-2.1054990292,-0.4270705283,-0.7249308825,-0.9879758954,0.0718391016,-0.5815575719,-1.1283096075,0.1646392792,2.2088565826,-0.8298310637,0.1257507503,-2.0224769115,0.6178207994,-1.2828671932,0.4831278920,-2.5446667671,0.0235447213,0.6328905225,-0.2720804214,0.2828456461,-0.2629654408,0.2357223034,-0.2633309066,0.3823617697,-1.0612810850,0.8905143738,0.2787717283,-2.9553375244,0.4997931123,0.3578976393,0.5189617276,0.3165110052,0.1713823676,2.0032389164,0.5444811583,-1.5100010633,1.6583312750,0.4102133214,2.2827858925,-1.4170377254,1.0672702789,-1.0848878622,-0.9437910318,-0.7768186331,-0.7263257504,1.1376026869,0.1669677347,0.1908601671,1.6095134020,1.1411566734,-0.1959963888,1.8457901478,2.7180738449,0.0899037942,-0.4714163244,1.5490199327,-0.0940073505,-0.6156017184,-0.8397650123,-1.5853828192,0.6788233519,-2.0469789505,-1.0404422283,0.6478714943,-0.5592913032,0.7406380773,-0.6209233403,-1.7690067291,-0.3744283617,0.5275703669,0.1906655729,0.1695944369,-0.1484553218,-0.5499172807,0.6520490646,0.1473938972,-0.8528057337,-0.6439852715,-0.2095579505,-0.1239432767,0.3361501396,0.2245066762,1.9661922455,-0.0664852038,0.4860768616,-0.1386903822],[0.9536022544,1.0937627554,0.1194592714,-0.6387403607,-0.5548762083,-0.5451083779,1.4894568920,-0.7443156242,0.7375551462,0.3067426980,-2.0427732468,-0.4271575809,-1.3816052675,-0.5549044013,-1.5412914753,0.9646945596,1.0175510645,0.8287231922,-0.6588662267,-0.8034091592,-1.0398589373,-0.1990352720,-0.2321966738,-0.8626244664,0.7902417779,0.5957292914,0.0277387053,-0.0978775173,-1.7130082846,0.7651479840,0.6190039515,-1.8823497295,-2.8895819187,-0.3021363914,0.6384620667,1.1293169260,0.8372123837,1.8616006374,-1.3465864658,-0.9666275382,-0.8175053000,-2.2610628605,1.1906888485,-0.8104630709,0.0317354426,-0.2019810528,2.2502446175,-0.2259142995,-0.2013495713,-0.4183434844,-1.7510526180,2.2945125103,-0.7594260573,0.3623003066,0.2451833487,-0.5534833670,0.8765788078,-1.2913513184,-1.5042854548,0.7121409178,-1.0445988178,-0.9468733072,1.0692250729,0.2658461034,-0.5082159042,2.3868613243,0.3205399811,-1.0530476570,1.8602855206,1.8128275871,0.6259533167,-0.7656450272,0.3985169232,0.3867060840,-0.1872877032,0.4220613837,0.8917585015,-1.4155749083,-1.2021927834,2.2409737110,1.1562517881,0.3536393642,-1.7080003023,1.5424200296,0.9600786567,-0.0872785896,0.7514562011,0.9496883750,-0.6002808809,1.6835938692,-0.2155499607,0.9516291618,0.4370562732,-0.4840465486,0.1115614027,0.2884726524,1.3234615326,1.0874466896,-0.5007370710,-1.0555545092,-1.3426496983,-0.3183835149,-0.0679995045,0.3516796231,-0.7929608226,-1.4250048399,0.3407011926,-0.5564637184,-0.5598407388,-2.0848259926,-0.3684155047,-1.5428141356,0.0320821367,-0.7246804833,-0.5784972310,-0.0024182077,-1.0722943544,0.6343519688,-0.2621089816,0.2908956707,0.8434097767,0.7944604754,-0.0688873604,0.2049677670,-1.0564239025,-2.9353809357,-1.1070784330,0.8343291283,-0.2105655819,0.8942486644,0.1070115343,2.8148813248,-0.1915903836,-1.2917917967,0.6141395569,0.2729116678,-0.0089775594,-0.9163486362,-0.3805558085,0.4296191335,-0.5568725467,0.0454204902,0.3127158284,-1.2098184824,0.5218726397,-1.8089548349,-1.7078270912,-1.0075867176,0.2304183394,-0.6987965107,0.6294561028,1.4788361788,0.2974897623,0.9382640719,0.6637336016,1.1047103405,-0.1571140289,-0.9894824028,-0.5497223735,-2.0647106171,-2.5246694088,-0.3880341649,-0.2562047243,2.0650627613,0.2814728320,1.5140551329,-1.5470811129,-0.5152064562,0.2855536342,-0.7161605358,0.1456434429,-0.7985317707,-0.1925467849,0.7353378534,0.8663336039,0.0977301821,-1.6826072931,0.3238239288,1.8810269833,0.0977288559,-0.6490523815,-0.7416719794,-0.4745566249,-0.4449836314,-1.5103685856,-0.0510294065,0.3497232795,-1.3869361877,0.1665868908,1.4021207094,0.9121657014,-0.4573785961,-0.7126724124,1.1894726753,-0.7016700506,-0.1637506485,-0.1672487110,-0.2228942066,0.7889564633,-1.7819786072,-1.0300257206,1.5504140854,-1.2054677010,-0.5473297238,-0.6004752517,-1.4439833164,0.7139390111,0.3109297752,-0.3902191520,0.9313625693,-0.0723820999,0.5176621675,0.1685560048,0.6693775058,0.4388497770,0.4009196460,-0.7448923588,0.1883382797,-0.0165072586,0.1768764108,1.0786801577,-0.4834615886,1.3717924356,-3.5873508453],[1.6853481531,-1.1144967079,-0.6112350821,0.4245436490,-1.7847050428,-0.1747217029,0.1853655428,0.1908560395,-0.6552680135,1.0942722559,-1.4203670025,0.0901684761,-0.8357254863,-1.3958472013,-0.6363635659,-0.3363048434,1.3161414862,2.3062155247,0.2548306286,-0.5943790674,0.1230811551,1.3223766088,1.0881372690,-1.4070535898,1.0152657032,-0.4547869563,-2.4484944344,-0.7348918319,-0.1146792397,1.8282408714,0.4467991889,0.5553500056,1.0960659981,1.1745150089,-0.6352566481,0.9568100572,-0.1503251493,-0.6595150828,0.4656138420,-0.2699167728,0.3962543607,0.0893835947,0.9601736665,-0.2942351103,-0.0805045590,-0.1939863265,0.7092486620,0.7895736694,0.5779416561,0.3850767314,-0.4842936695,0.6331396103,1.1849069595,-1.2239178419,0.6399813890,-0.1655349433,-0.2754193842,-0.4561723173,-0.0982396007,-0.7950536609,-1.3405947685,1.4434165955,1.4510871172,-0.5008798242,2.3326554298,1.7171764374,-1.1214941740,-2.1384959221,-1.4371384382,-1.2622073889,0.3689718246,0.3668732345,-0.1287642419,0.4816688895,-0.7438533306,0.7954478860,-2.9465436935,0.0280521810,0.5243663788,0.8543690443,-0.5550677776,-0.9916635752,1.0783920288,-0.2361106575,-1.0624667406,-1.0621988773,-0.3515598178,0.0059884796,-0.2377974391,-0.2549857199,-0.5997722745,1.4417372942,-1.2261022329,-1.3927828074,0.3786606491,-0.8822529912,1.9552049637,1.3698495626,-0.7608377934,0.4118162394,0.3870486319,-0.1944354028,-0.6407830119,-0.4655601978,0.3296374083,0.3511639833,0.6457598209,-1.6749877930,0.3648128510,-0.3509193361,1.3478113413,-0.6998479962,0.1091958135,-1.2648110390,0.6919533610,0.8096422553,0.4792396128,2.1586918831,-1.0145262480,-0.4494864345,2.1004250050,-0.8175516129,-0.1288737357,0.6267175078,-0.9657980204,-0.3166680932,1.2221369743,0.6191287637,0.8959646225,0.2814862728,-0.3159213364,1.3342118263,-0.9356304407,-0.7985250354,2.5087397099,0.2187114060,-0.3500434756,-0.5621758699,-1.1118510962,2.3423166275,-0.8084353805,-0.6880099177,0.9092693329,-0.4087174237,1.1080206633,0.9253252149,-0.9031172395,-0.6476074457,-1.5008963346,0.4107389748,0.5725745559,0.0407793745,-0.8929830790,0.7959543467,0.6446728706,1.5393860340,0.2424186319,-0.7756873369,-0.3757202029,0.9156033993,1.5491391420,-0.2052761167,-0.7376536131,-1.8269327879,-0.6759383678,1.0747044086,0.3764349818,-0.0615763552,0.0539816134,1.3830044270,-0.2138948143,0.6070758104,2.2050757408,-0.5970668197,1.5818897486,-1.7589093447,0.2374013215,0.6174883246,1.5783377886,0.2767152190,-2.0720968246,-2.1631407738,1.1022502184,0.9568095207,-0.7925844789,-0.0129932240,-0.4571443796,1.1118091345,-0.0802769810,0.5072802305,1.4994150400,0.9595949650,-0.5461329818,1.2634196281,-1.2565197945,0.0333052836,0.4604368210,-0.5963924527,-0.4048521519,-0.0185353886,-0.4433542490,1.0058584213,-0.6991336942,1.2530970573,-1.8591336012,0.1965983361,-1.3790326118,0.6991924644,-1.1815651655,0.5590909123,-0.5781280994,-0.2810287178,0.9894128442,0.5948468447,-0.2116240114,1.6549823284,2.1434628963,0.1538797021,-0.2101962715,-0.2762024403,-1.0265514851,1.2561662197,-0.0961916596,1.8024312258],[-1.0695019960,2.2832481861,1.1988962889,-1.3095893860,-0.6166658401,-1.0923980474,-0.4167923927,-1.1538887024,0.6146585941,-1.0539424419,0.2147403657,-0.9397309422,1.2403391600,0.6924793720,-0.1568285823,-0.2127067894,-0.4619454741,0.4466876090,-0.8593928218,0.3885909915,0.5803668499,0.3129754663,0.5921312571,0.2810893059,-1.0458991528,-0.3443108499,-1.3744816780,-0.3332820535,-0.8077567816,0.4288160503,-1.7361116409,-0.1163452491,-1.3128036261,0.0584229752,-0.1550195813,0.5011024475,0.9355930686,0.5465435386,0.4411371946,0.1784846634,0.4730665684,0.3743475676,0.3358268142,0.3381077945,-0.0342798606,-0.8083887696,-0.3979845345,-1.1604353189,-0.0066471668,0.1081213802,-0.0444181934,0.1806796342,-0.7541006804,-0.7393528223,-0.9072701335,0.2659946382,0.8296968937,-0.2166975737,-0.0818051770,-0.1148054823,-0.7135875821,1.2623482943,0.9717949629,0.6293833852,1.6772850752,1.0530080795,-2.5207023621,0.0990564898,-1.2991740704,1.4384398460,-0.5041558743,-2.4514865875,-0.7514006495,1.0182894468,-0.4337368309,1.8021686077,-0.4107547402,0.0077835210,0.2561873496,1.2580149174,-0.2291361839,0.1945354342,-0.7990037203,-0.4128118157,0.3472887278,0.6855760217,-0.0667856112,0.5450399518,-1.0061272383,-1.0191876888,0.8211130500,-1.1527760029,-0.5438255668,-1.2694709301,0.2857372463,1.1246901751,-0.3271239996,-1.5309205055,0.9586626291,0.7188817263,1.3707944155,2.0927855968,0.9110329747,-0.4163654447,-0.2413679957,0.1141957939,-0.3054531515,-0.1258421391,0.4647783935,-1.0801485777,-0.4261832237,-0.9162673950,0.7924376130,1.0909430981,-0.6960784197,-1.6909022331,0.2807614803,-0.9859634042,1.3558187485,0.4552947879,-0.6819797754,-0.5628001094,-1.1365072727,0.0257628281,1.3203532696,-1.7315850258,-0.2278526872,1.1781377792,2.3463416100,0.1275173873,-2.5863084793,0.6135267019,-0.8700727224,-0.8105805516,1.5111283064,-0.3862929642,-0.4356092811,0.0636376143,-0.5927600861,-0.5895982385,0.1878833175,-1.9611967802,-0.3901644945,1.0398082733,1.2147284746,-1.1976518631,2.3415234089,-0.7130673528,0.2361291051,-0.3015916646,0.5939913988,-1.4054470062,-1.0503392220,0.6779701114,0.5623335242,0.0663100109,0.4308245182,1.0027855635,0.8165775537,0.4989019632,-0.0002661952,1.5691658258,-0.7713332176,0.1407069266,-1.0722569227,1.5917398930,0.6614595652,0.2007022649,-1.0523856878,0.0822489485,-0.6039589047,0.4681268930,-0.5547893047,-1.6950765848,-0.6791362166,0.1981026828,1.9102047682,-0.5541394949,-0.9890259504,0.7984296083,0.0003176213,0.0550370589,0.4107954204,-0.0231982525,-0.7725265622,0.6694859266,1.2758612633,-0.1744779348,-0.0360984839,-1.6794584990,0.4439693987,1.3904777765,0.5694403648,1.7294931412,1.5751372576,-1.9432759285,-0.3927997947,0.9067459106,0.5820047259,1.2948682308,0.1216424406,1.1153415442,1.5315512419,1.1464210749,1.7029515505,-1.0568026304,-0.0944661796,0.3944582939,-0.1618760973,0.0606435351,0.6873081326,1.0236778259,-1.4742099047,-0.4439935684,0.6182740927,-0.0559816100,0.4584692419,0.9900916815,0.5068622828,0.1548262686,1.0944187641,0.4731070399,0.9248341918,-1.4233351946],[-0.8062060475,0.1411530674,-0.7675983310,-0.8649474978,-0.0317371860,0.7700589895,0.7849109173,-1.3150564432,-0.1650596410,0.1696455181,-0.0934922174,-0.6370592713,0.5575283170,-1.4024553299,-0.2018749118,0.2528863251,0.1751198769,-1.1257034540,-0.1473133713,1.0206111670,1.2502821684,-1.1858514547,-0.4971847236,-1.9140037298,0.1850125492,0.6661189198,1.7885531187,0.4540091157,-0.5431872010,0.5952700973,0.3110172749,0.7823548913,-0.3349719644,-0.3264983892,-0.7418109775,-1.7509810925,0.1852183491,1.0977188349,1.5520267487,0.3697099686,1.7792718410,-0.1528856754,0.2824684978,-2.4584684372,-0.2338568270,-0.1353407651,-0.7570917606,2.0325019360,0.3802107573,0.0842941403,0.1059182584,-0.0346744955,-1.6290546656,0.5493621230,-0.3867263496,-1.4424030781,-0.6036384106,0.0387400873,1.5152940750,0.1151001677,-1.0722410679,-0.2107968777,-0.4670877755,-0.1375235170,-0.1439756453,-1.0298039913,1.0332676172,0.8170245290,-0.1887505949,-0.0494192839,-1.0711476803,-0.8699306846,0.4188597202,-0.3706445992,-1.1961505413,-0.7853118181,-0.4474318326,0.4565109611,-0.3028509021,0.0739333481,0.5674366951,0.1040408462,-0.0226907004,-0.2704212666,1.5229213238,1.6927081347,-0.4871771336,0.7022641897,1.0417212248,0.6593301296,0.6112542152,-0.9994509816,0.6857087016,-0.1322730035,-1.0452054739,-1.3029451370,1.2493374348,0.0080011049,-1.2995612621,-0.0949502289,-0.3533111811,-0.8815700412,-1.4849035740,0.6015857458,0.3696425855,0.1216674820,0.1807393730,0.3852241039,-0.8649283648,-0.2445823848,1.3928322792,0.8241620660,-0.1049988940,1.2019417286,-0.8042196631,-0.1875663549,2.1202955246,0.5893884301,0.6650680900,-1.4238115549,-0.4695525169,-1.4288089275,-0.3693912327,0.8288694620,-0.1480082422,0.8329437375,-1.5620113611,1.4933506250,-1.9748326540,0.3324063122,-0.8589459658,0.5188101530,1.1035842896,-0.2568058074,0.8886639476,-0.7919666171,0.4869928360,1.8516781330,-1.1125978231,-1.1663391590,-0.2702799141,-0.9070065618,-1.8252818584,-0.3983536363,0.9135220051,0.6276210546,-1.2996852398,-0.8030201793,-0.8869251013,-0.1006307080,-0.1849675477,0.4097519517,1.1918514967,-1.1746904850,-1.5114037991,-0.0913629159,-0.5989208817,0.9410465360,-0.8433012366,-1.2563877106,1.0473858118,0.9126477838,-0.5827721953,-0.5856482387,-1.7571837902,1.3324482441,0.4234672189,-0.6457720399,-1.0114756823,-0.0575404130,0.4846711755,0.5327658653,0.0827659070,0.2714155316,0.4643567502,1.9540315866,1.3832021952,-1.0131411552,-1.0373722315,0.8032507300,0.2130549699,-0.8439952135,0.9219255447,1.1542927027,-0.0212097038,1.6666793823,-0.5585919619,0.2945807576,-0.0132242423,0.5881583691,-0.6531445384,-0.2318715155,0.0456866808,0.0064122831,0.4922307730,-0.6649486423,1.2209202051,-2.3153216839,-0.8188648820,-0.7945204377,-1.2346620560,-1.2860282660,-0.8244218826,0.7727444172,-1.7617459297,0.0262414347,-0.3023742139,-0.0454284027,0.5600838661,0.3081323504,-0.6677283049,-0.0851749256,0.4230430126,1.1981501579,-0.5143482089,0.1208445281,-0.3302524090,1.4326996803,-0.4619111121,0.8303595781,-0.2305427641,0.4566539824,0.0284742471,1.1725553274],[-0.6778904200,0.7474908233,-1.4357063770,2.0686724186,-0.6995326281,0.1032049581,-0.9478116035,-0.1820008606,-0.8253668547,1.1953212023,0.3244872391,0.6627004743,1.0053045750,0.2691727877,0.2567492425,0.6107055545,-0.7879512310,0.9819718599,0.6871561408,-0.8443669677,0.0294165127,0.6981332898,-0.4156535268,-0.4417705238,-1.1484087706,0.4935772717,0.9644652009,-2.2177762985,1.2895581722,1.9430149794,-0.1687173247,0.2524688244,-0.0231861938,0.2763607204,-0.8558312058,-1.3785704374,0.9749888182,-0.0315952413,1.2336918116,0.8569123745,-0.1242811829,1.0706355572,0.0651801452,-1.0987329483,-0.0845275149,0.3220182061,1.3133853674,-0.3922113478,-0.8745101690,-1.6545143127,1.3099434376,-0.3884691894,-1.0207983255,0.9145443439,0.6784790158,-0.4679728150,-1.8396611214,-0.6501001120,1.0174416304,-1.2393320799,-0.0569090955,-1.5452183485,0.4984608293,-1.0995006561,0.4160440862,-0.2431684881,-0.0810485184,0.1673014760,-1.2867910862,1.1799737215,-0.2366669923,0.2107269764,0.1286269724,0.7097382545,-0.7196391225,1.0831000805,0.2915651202,-0.8028673530,-0.4497925341,-0.2961663902,-0.3938611150,1.3102715015,0.1274573952,-0.3415600657,-1.3402360678,0.3736258149,-0.1554016620,-2.0350890160,-1.6971189976,0.2403908372,0.2295648903,-1.4248719215,2.1251647472,-1.4979672432,-1.2219846249,0.3304457068,0.0374567807,2.1840059757,0.5777245164,0.1611160338,0.8961535096,0.3680781722,-1.9025293589,-2.9646284580,-0.6679293513,0.0742716938,1.4247859716,-1.4310384989,0.6191782951,-0.2454420179,1.8304854631,0.5033181906,1.5404642820,2.0885241032,-1.3709269762,1.6574317217,-0.5738112330,0.4851473868,-0.5364739895,1.9374666214,0.5323956609,-0.7937283516,0.7213377953,-1.9963128567,0.5997330546,-2.5905666351,0.4053169489,1.0426921844,-0.2191019654,-0.5558608174,1.5116729736,-0.8256971240,-0.2864851356,-0.5010135770,1.4569622278,-0.4448128939,1.1936591864,0.6703691483,2.1985344887,-0.6182096004,-0.7770881653,-0.7601132393,0.8608002663,1.1269603968,0.0908432677,0.1105712205,0.9951350093,1.0663802624,-0.6953923702,0.0459341668,-1.2705870867,-0.4319158792,-0.7496430278,0.5289142132,-1.6569365263,-1.0187835693,0.9269422889,-1.4888494015,-0.4321640730,-0.1888191402,-0.4988208711,-0.7500018477,0.5415527821,1.2704626322,-0.5392972231,-0.5853786469,2.0392589569,0.8061009049,0.4328102469,0.1999326795,-0.1828068048,2.0369491577,0.5376186967,-0.9456459880,0.2697933614,1.6897407770,-1.3979105949,0.2796856165,-1.2426117659,0.6467071176,1.0034134388,-0.5081450939,-0.1762165129,-0.9470931292,-0.9216216803,0.0103546353,-0.9105349183,-1.5967898369,1.6333882809,-1.1184823513,0.7191648483,-0.1556680799,-1.1775581837,0.6844239235,-0.0761843994,-0.8918274045,-0.9387372136,-1.4445216656,-0.8445951939,-0.1898192763,-1.2781102657,-1.6075053215,0.6881842613,1.5046265125,-0.6197365522,0.2504410148,-0.2626022100,-0.5783621669,0.9059876800,-1.2948360443,-0.2569444478,-0.8420674801,-1.3039224148,0.5142272115,0.4427238107,0.1815748215,0.4946224391,-1.3786232471,-0.2891066074,0.4617694318,-0.8750137687,-0.6624167562,0.2299284190,-0.3934533298],[0.1057336032,-0.5486158133,-0.7816815972,1.4666053057,1.4891735315,1.4241904020,-1.1920310259,-0.6360286474,-0.4250755310,0.0566578321,0.7702762485,0.1289665252,-1.5876783133,-0.0016417794,0.4250714779,1.0069613457,0.9110912085,0.6452820897,1.2802551985,0.8736882210,-0.1176333576,1.3896194696,-0.8711903095,-0.8319627643,-0.2103565633,-1.5452733040,-0.2017096281,0.0036522455,0.3344618678,-0.4068647623,-0.7161149979,-1.2258971930,0.5839748979,0.6048594713,-1.1431585550,-1.7302323580,-0.0968581066,0.1390904784,-0.3141165972,2.1717050076,1.9471942186,0.5632810593,-0.2581654489,-0.8188413978,-0.7301347852,-1.9451324940,0.1461771280,-1.3031450510,-1.2256352901,0.2075386643,-0.3776623309,-0.6919597387,0.7050512433,1.5114662647,0.1402460635,0.9202152491,0.4513795972,0.7388632298,1.0826296806,-0.3678231835,0.3671057522,1.1094573736,0.3479338884,-0.6016224623,0.0073265564,0.4310854077,1.7144032717,-0.3383898139,1.0131033659,0.2747268379,0.7387757301,-0.4018067122,0.6781775355,-2.1581020355,0.2278467268,1.7021756172,1.3773645163,-2.0554974079,-0.6872193217,0.1226048097,-1.9852946997,-0.3543234169,0.1991903782,-2.1077291965,0.9524010420,-0.8265848756,0.0515594371,0.7495928407,-0.5910230279,-1.9738554955,-0.4401849806,1.2871997356,0.3963395059,-1.3662760258,-1.0966602564,-0.1497679204,0.2017935365,1.5538893938,-1.4132012129,0.8584647775,0.0906719267,-1.0717674494,0.1683136374,-1.6558039188,0.5440172553,1.2324019670,0.5137600899,0.2472419590,-0.0660363659,-0.3757608831,0.5444558263,-1.7358322144,-0.0051293653,-0.2503592670,-0.4951822162,1.2275048494,0.2725945115,0.3664792776,-0.6326197982,0.0382645912,0.7879148126,-0.1890475899,0.6260702610,0.6560769677,-0.8796280622,-1.0355696678,-1.1185706854,1.8544151783,0.7461084127,1.7544984818,-0.6223727465,-0.4497933388,0.8481234908,-0.3922756314,-0.4858475626,0.0968091264,0.0066539571,1.0293912888,0.2278260440,2.0721802711,0.4783202708,0.7750809193,0.3829811513,-1.0834962130,-0.6774073839,-1.5615712404,-1.6606448889,0.3167343438,-0.4906793833,0.0483574681,2.2943816185,-0.0707739368,-0.4499971867,-0.1112928838,-0.2830072939,-2.6263105869,-2.1136035919,-1.1006478071,0.7729149461,1.4741439819,-1.2733880281,2.3177292347,-0.5778938532,0.0909742787,0.4740823805,0.5157712102,0.5300472379,-0.8119449019,1.2384713888,0.5614928603,0.2497849166,1.2373347282,0.1882596016,-0.1199569330,0.5125343800,-0.4821256101,-0.5073617697,-0.3014651537,0.6116203666,0.1299709827,0.6942937970,1.1415054798,-0.2746116817,0.1055720225,-0.5667106509,0.4699902534,0.3701259792,-0.4664997160,-0.0554599166,-0.9459332824,-0.7991710901,-0.0254140403,-0.1795408279,0.5957148671,-0.9031828642,-1.5857150555,1.0605216026,-1.5644696951,2.2349119186,0.2240771055,0.9779759645,-0.3761295676,-0.6323259473,-1.1920045614,1.5147486925,1.5310157537,-1.3137631416,0.0094543528,-0.2456318289,0.5364572406,-0.9972238541,1.2738656998,-1.3176814318,0.3259899318,1.0292425156,-1.3389029503,1.4183219671,-0.4529511333,-1.0125648975,2.5792348385,0.6706108451,-0.4555190206,-0.8167887926,-0.2441108376],[1.4933758974,2.3693647385,-0.0329419933,0.2555276155,0.1001068726,0.4355405271,0.7977789640,-0.3352335989,0.1421558112,-0.2073267847,0.0726978779,0.4240096211,-2.3663616180,-0.8102269769,1.0085755587,0.2688770592,-1.0159114599,-0.1881680489,1.8253681660,0.8918188810,0.0901473612,-0.1998090148,0.6020976305,0.9907689691,-3.2210643291,1.6036646366,-1.0785632133,-0.4379940927,-0.7611535192,-0.0714974627,-0.6375125051,-0.3485542536,0.3473993838,-1.2568857670,-1.3050128222,0.0136722717,0.7082781196,0.8472917676,0.8057292700,-0.1659359187,-0.5133125186,0.3766638637,-0.2679806054,-1.4026973248,-1.0416235924,0.7020118833,-0.1503003240,-0.5658562183,0.1204976216,-2.6717855930,-1.9358696938,-0.8094495535,-0.6051175594,-2.0002884865,0.7022117376,2.1806385517,-1.7201800346,-0.0582118407,-0.1393259019,-0.1016402170,-0.2271406353,-0.0488020889,0.1912101656,1.2248835564,-2.2081222534,0.0591728874,-1.0548201799,0.2093756348,0.7086314559,0.4830908477,2.0755898952,0.0622914173,0.9292717576,-0.4380646348,-1.6233458519,0.1461371183,-0.5482233763,0.7338901758,0.0389567427,0.6998253465,1.0116010904,-0.3618020415,1.8255698681,-0.9448072910,0.1480549872,0.9181790352,1.1242004633,-0.7497963309,0.5569706559,-0.3830719590,-2.8234889507,-1.1781505346,1.4321072102,1.2316467762,0.2388788015,0.0512132496,-0.4228218794,1.1188286543,0.3400440812,-0.4093388915,-0.3253835738,-0.5566400290,1.2506176233,-0.5119938850,-0.7112380862,1.1241469383,0.2668454647,-1.7113490105,0.2358381599,1.1011041403,0.8496887684,-1.2886971235,-1.3477289677,-2.8305227757,-1.0628187656,-0.2909052968,-0.1643280834,0.0555402003,0.5318995118,0.5488132834,-0.3408282399,2.4728338718,-0.0778830498,0.2826209366,0.3594366014,1.1223710775,-0.2526025772,-1.8722442389,1.2654098272,-2.6454839706,-0.4887316823,-0.4427509308,1.2608346939,2.1664521694,1.1900120974,-2.3902049065,-0.6655550003,-0.1544005424,0.5118381381,0.0688272491,0.3676896989,0.1625170857,1.4110800028,0.0116869053,0.6106952429,0.0455630608,-0.1926245689,-0.2303574234,0.8177344799,0.1466995478,0.0532553457,-0.6141367555,0.5387769341,0.0295939930,-0.0525368638,0.5687243342,0.6703144908,-0.6449947357,0.3593075573,0.7457771301,-2.0517790318,0.0016169018,-0.0653042272,-0.1865885556,-1.4693999290,1.1893911362,1.0076799393,-1.4579092264,1.3490563631,-1.6361253262,-0.5041018128,-2.0277647972,-1.4027677774,0.4853744805,1.1966117620,-1.9860762358,-1.1433326006,-0.4991798997,0.8099403977,0.9741308689,-1.1231752634,1.2916612625,1.2223510742,1.1004766226,0.1277393401,0.9839468002,0.0020423641,-1.3422156572,1.1989815235,1.0622923374,-1.0130317211,0.0652642846,-0.5992264152,0.4082034826,0.0806819052,-0.3798908591,1.2465872765,-1.0993893147,-0.5118710399,-0.2986849844,0.5246425867,0.6732012033,0.0628702417,-0.9138420820,-0.9044297338,0.0181843303,-1.6680605412,-1.1724907160,-0.8832670450,-0.2367167175,-0.0361848995,1.4360933304,-0.2679459751,-0.8920229673,1.0423166752,-0.0651038066,1.7097898722,-0.9543641210,-0.7677184939,0.1742484719,-0.2672248483,-0.4275380373,-0.1414251775,0.9008878469],[0.8739157319,-0.3322092593,-0.5973111987,0.2851661444,-0.1468812525,-0.2907929420,-1.0881502628,-0.6024230123,-0.5802869201,-1.2372754812,-0.4289747477,-0.5685067177,0.9802915454,0.0765238106,0.9593495131,0.6458209157,-0.6384899020,0.6581201553,0.4170886278,1.3364971876,0.8242484927,1.4422096014,-0.4826723039,0.2676834166,-0.1370688677,-0.3821313083,-0.9061530232,1.1765571833,0.4614237249,1.9140841961,0.0065699657,0.4581893384,2.2392342091,-2.1551437378,-0.6505623460,0.1773225814,0.1001090482,1.1466196775,-0.5088146329,0.9910753965,-0.6102650166,-1.6703640223,-0.5117351413,-0.6359770894,-1.0455538034,0.3068882227,-2.8139216900,-0.7243748307,2.3602602482,1.1162489653,-0.5636335611,-1.3498125076,2.0970036983,-1.6311956644,-0.3248442113,-0.7526709437,-0.6360383630,2.2367043495,0.8790027499,-0.4985416234,-0.4463484585,-0.7190703750,-0.3197208345,-0.1645631343,1.6601192951,-0.0426488407,2.3123860359,-1.5778800249,-0.0058000362,0.1096969396,-0.0175224114,0.9351284504,0.9580059052,-1.3353056908,0.1352744251,0.7962580323,-0.7297627926,0.9289662838,0.7584497333,-1.0704288483,-0.1412631720,-0.2476228029,-0.6682301760,0.0307615045,-1.4234244823,-0.7623709440,-0.9366347790,2.0120165348,-0.5410431027,-0.4628232121,-0.1133372635,-2.6215143204,-0.1878034025,-0.1376096606,0.2618097961,1.3662329912,-2.2204368114,0.8991046548,0.7960936427,0.4082904160,0.3135900199,1.0021790266,2.2778322697,-0.6833299398,0.2828281820,0.4902434647,-1.6065645218,-1.2876976728,0.4138703048,1.4634865522,-0.8037552238,-1.8263963461,-0.1062341779,-0.4714684784,-0.2142926008,0.8328797221,-0.2681073248,0.4731663764,-0.6831670403,1.0777174234,0.3799136877,-1.1204904318,-0.1482727826,-2.2189519405,1.4719551802,1.0429495573,0.2535926402,-0.9038581252,-0.2051886171,0.2231454849,-0.9873346686,-0.1028130278,-0.9490976930,0.5669447184,1.3465336561,0.5354964137,-1.4442919493,-1.3446625471,0.1368197799,0.3121242821,0.6082185507,-1.9605544806,0.7699592113,-0.9161998630,1.6719350815,0.0406786539,-0.2060671747,-0.3644065261,-0.4067469239,0.6496877670,-0.4569232166,-0.4409435093,0.1681868136,-0.8629373312,-1.1391000748,-0.5719874501,-0.6937422752,-0.3806343377,1.2496166229,0.1289080679,0.9666724801,-0.5838910937,0.1186358780,-0.7255809307,1.3304361105,0.2326592952,0.8757634163,-0.7049357295,-1.0692155361,-1.2293150425,0.2959873080,0.0319055319,0.1620924175,1.4239611626,0.7566784620,-0.3976113498,-0.4200448394,1.8963367939,-0.5100295544,-0.1136146113,-0.7116947174,-1.1979464293,0.0628819540,1.6701619625,1.3206782341,-0.3954262137,1.6946469545,-0.1842424870,2.0869755745,0.4656747282,-0.9359200001,1.0937289000,-0.2609280646,0.8597438931,-0.4747339189,-1.7674329281,-0.7370138168,1.6124833822,0.8634741306,1.2746123075,0.4331390262,-0.2857427001,-1.2503889799,-0.4103792012,1.2073097229,-0.2294325531,0.1557102054,0.1661618203,0.1306845248,-0.3510292768,-0.7382842898,-0.1517895609,-1.5608966351,0.0113451760,-2.5729622841,-0.2821547389,0.2182861120,-0.0131761897,0.4549012780,0.2538626492,0.8881043196,-0.2298544347,-0.0876468346,-0.3201176524],[0.2374811918,0.8972510099,0.4730105698,-0.6594987512,-0.0209075324,1.3408588171,1.4259743690,0.6906683445,1.4695892334,-1.7932443619,-0.3990622759,0.3883207440,-0.1653114706,0.7263434529,1.0113916397,0.7325154543,1.0358587503,0.2038902938,1.6042402983,0.0524286889,1.0023922920,1.1868354082,1.6180857420,0.8422029018,-0.3649192750,0.6122750640,0.4988044798,-0.5457030535,-0.4046074748,-0.2460051328,-2.1963894367,0.2778351903,-1.2490707636,-0.5998269320,0.4359167516,0.9332191348,-0.2432841212,0.1132514253,2.0727136135,0.7468464971,-1.4363973141,1.6554725170,-0.5935822129,-0.0301943701,-0.0096276281,0.9447058439,-1.1380391121,-0.3193050027,0.3661886156,0.7435558438,-0.1640613824,1.1098930836,-0.3936080635,-0.4477424622,-0.3403979838,-2.5856881142,0.9082188010,-0.8674103618,1.4339728355,-0.8392519355,-0.5988446474,0.1905030608,-1.3781613111,1.0868411064,1.0486859083,1.1246263981,-0.7191772461,-1.0842479467,-1.6448795795,0.8328526020,-0.9384015203,-0.4217948020,-1.3769412041,1.9037914276,-0.5599355698,0.1158174872,-0.1574285477,0.4571927786,-1.7404035330,-1.8275996447,0.7204089761,-0.2950896919,1.4025130272,0.2427419722,-0.0037648869,1.3575364351,-1.1893963814,-1.1178938150,0.3041092753,-0.8188020587,-1.1610109806,0.2950367033,0.0649628937,-0.4260863960,1.3116868734,0.5123817325,0.3020256758,0.1874590069,-1.5938930511,-1.7766227722,-0.2056816667,-0.1434836537,-0.6319875121,-1.3853851557,1.7522269487,0.1705312729,1.9834896326,0.0428273194,-0.2813680470,-0.0961206928,-0.6106506586,-1.2427818775,1.4810787439,-0.6395637393,1.4544156790,-1.0006572008,1.0430054665,0.2761348486,0.3180278540,-0.5181749463,-0.0602431931,0.0929836407,0.4769335091,0.3015520573,0.5975839496,0.6896985769,-0.1892197728,-1.0921987295,-0.4295704961,0.4336581826,0.3824291527,0.1809328645,-1.0461263657,-1.4477847815,0.6859626174,-0.7645117640,1.0940294266,1.4802038670,-0.9201808572,-1.0781209469,0.9212292433,-0.3049114645,-0.0715369731,1.0498589277,1.0016895533,1.4182119370,-0.5187898278,-0.1311899126,0.9670145512,0.5622414351,0.4447497129,1.1318860054,0.7751047015,-0.3009999692,-0.4801048636,0.0647643581,0.9612099528,0.8127611876,0.6242893338,-0.0070319353,1.0763334036,0.3677817285,-1.3627214432,-1.0657974482,0.8311042190,-1.3935128450,-0.2092558742,0.4993962944,-0.9601699710,1.0647552013,2.2043251991,0.3423570991,0.1696930826,-0.2994166017,1.0733228922,-0.3528176844,0.4736300111,-0.5256693363,-1.5099061728,-0.2726048827,1.8399487734,1.0151726007,-0.4483799636,1.0994441509,-1.8038136959,0.4645211697,0.3011919260,-0.7694646716,0.9885981083,-0.3152141571,-0.8028303981,-0.1730686873,-0.6892525554,-0.9899894595,0.6263012290,1.6653257608,0.3095535338,-0.5246341228,0.0186689477,1.6799933910,-0.5161258578,1.8456840515,1.9402329922,-1.6643905640,0.7645756602,-1.0416288376,1.8423246145,0.5390047431,-1.9061768055,0.8301377892,1.0769793987,0.6682855487,0.9632487297,0.6451629400,0.0955077112,0.6230443716,-1.1125954390,-1.1393494606,-0.2665373385,2.2958316803,-1.8586397171,0.1168465689,-0.0136828171,1.4055063725],[-1.3613090515,1.2799606323,-0.8697892427,2.4187178612,-1.5788184404,0.5349386334,0.4891916215,0.6985910535,-2.4071002007,0.6912989616,-0.1731216162,-0.3386763334,-0.6762531996,1.8666601181,-0.0154876197,1.4744048119,0.8798419833,-1.8359698057,-1.4010410309,0.1690005064,0.4600860775,-0.2688275576,-0.1073251069,-0.6607923508,0.0372763202,0.1201879010,-0.1509473771,-1.0607008934,-1.1784601212,0.0837295130,-1.1999645233,-0.7227930427,-1.1312910318,1.4536602497,-0.3281475008,-0.2820889950,-1.9610961676,-2.2980158329,1.6167727709,-0.9151739478,0.7914054990,1.4067481756,0.5194048285,0.8777590990,0.5513826609,0.2247539461,-0.0472301021,0.9914721847,2.1558043957,1.8301255703,0.1900593936,0.2965781391,-0.0984601229,1.0304844379,0.4766424894,-0.7833252549,-0.5008711219,-0.5573242903,0.4439268708,-3.2847006321,0.4752749503,0.5018771291,0.2221382260,0.7174639702,-0.7699761987,0.3723175824,0.5160827041,0.4284012020,-0.8318826556,-0.3510650694,0.6850978136,0.6224149466,0.1465418190,-0.3880501390,0.1311548799,-0.3739495575,-0.0982051417,0.0241817795,-0.2103700042,2.3744344711,-0.5839608908,-0.1087427288,-0.2498640120,-0.0379346982,0.3635368645,-1.4021166563,-0.3856891692,-0.5266512036,0.5923498869,0.5811980963,-0.1913233995,0.2475237846,-0.6341478229,-0.4015057981,-0.2930896282,-1.6198561192,-1.5492149591,-0.4098072052,-1.1009782553,0.0696988776,2.2207129002,0.3850325644,0.1232785955,-0.1360888630,0.5543779135,-0.0335183479,0.1661666334,-0.9750319123,-0.3781283796,-0.6851177812,-1.3397881985,1.8235151768,0.7576472163,0.4756061435,-0.1756661981,-0.7518137693,0.8833954930,-0.1723460406,1.4779287577,1.1817060709,0.6576637030,1.7607759237,0.3329871595,0.3841156960,-1.6105310917,-3.4629151821,1.5630116463,0.1894923598,1.1110873222,-0.4295303226,0.0688258111,-2.6791574955,-2.3411090374,-0.6443280578,-2.5961453915,1.8770301342,1.6789271832,1.1087728739,0.4900375903,1.0108305216,-0.7254547477,-0.8507792354,1.0462187529,-0.5669525266,1.7076981068,-0.2117697448,-1.2549630404,0.7878744006,-0.6559647918,2.6261720657,0.0378729068,-0.9183604121,1.5393013954,-0.1139022857,1.5634179115,-0.0301232282,0.0856970623,0.2333744764,-0.3873595893,1.9680051804,1.4474378824,1.2159805298,-0.0786215439,-0.8479925394,0.3741477430,-1.3048787117,-0.6006184816,-0.1725184321,0.3311712146,-0.2673325241,-0.3190022707,0.5827477574,-0.7233198881,-1.2103803158,-0.0765999481,-1.9542196989,0.3852671683,1.4858161211,-0.0310763177,-1.7405352592,0.8110358119,0.1677609235,-0.3663225472,0.4820788801,2.0430974960,-0.2657999992,-0.4802150428,1.6891940832,0.6045791507,-0.2234309465,0.8571979403,-0.2686817646,-1.5307800770,-1.0786039829,-0.9064611197,-1.1571456194,-0.0541361533,-0.5406714082,-0.2749684751,0.1239802092,-0.5381905437,0.6174876094,-0.9688651562,0.9094956517,0.2015793473,2.0235207081,0.2642515600,-0.7107654810,0.6661833525,0.7415869832,1.9120591879,0.9989323020,0.4370322526,0.5388150215,0.2214304805,-0.4388776720,0.0455226041,-0.9726260304,1.4213956594,0.7784917951,-0.8738764524,-0.6479167342,0.9478707910,1.3753795624],[0.0296977572,0.8786270618,-0.5038436651,0.1264273673,-1.8705068827,2.3100185394,-0.8938060999,0.3092682958,-0.2958122790,2.1298017502,0.2210101187,0.8279167414,-1.3819217682,1.0468441248,0.7928908467,0.6151984930,-2.1460978985,0.0850671232,-0.2827851772,-1.4269472361,-0.7818253636,2.2002530098,-0.9858335257,0.5671643615,1.5863406658,-0.0924895555,-1.0210804939,0.6836526394,-2.2170433998,1.1493161917,0.3413541317,-0.2154352516,0.5662803054,-0.7847980857,-0.4791492820,-0.8131125569,0.8970845938,-0.4572354853,-0.7225107551,-0.5756103992,-2.2434556484,1.0185506344,-0.5351034999,-1.3657653332,2.0675470829,-1.0386697054,1.4762983322,-0.7982075810,2.0018270016,0.2635194659,1.2353080511,0.4287237525,0.0174695142,0.9563567638,1.6330941916,1.6011214256,-1.2320027351,0.1984476298,-0.9122123718,0.7111468911,0.7488287091,-0.3852639496,-0.1148628294,-0.2698810697,-0.3712545931,0.2244575918,-0.0854918435,-0.0593928993,0.1030734554,-0.4511945546,0.5479831100,-0.6317163706,0.6489620805,-1.0175821781,-0.1811434031,1.5832920074,-0.6779805422,1.3851190805,-0.3345347941,-0.1046513170,-1.6282833815,-0.9392483830,0.2728080750,-0.0925905555,0.2160407156,0.6644548178,-0.2666618228,2.6946973801,0.0434410423,-0.6470096111,1.7980229855,-1.2989336252,0.3984160423,0.6778487563,-0.0981895402,0.9798938036,-1.2538865805,-1.4360401630,-0.5902845860,-0.8581837416,-0.2199495137,-0.3045339286,-0.5929993987,-1.2001744509,0.4193471968,0.4130449891,-0.0643541962,-0.3918884695,-1.9008547068,-0.6400154829,-0.0310537405,0.3388310373,0.7863161564,-0.7796239853,-0.6914700270,-0.8911319971,-0.1382533163,1.1118685007,-0.6569674015,-0.1794133633,-0.5717086792,-0.4314243793,0.7357690930,0.2798558176,0.8829190731,-0.3237231374,-0.6520662904,0.2529103756,-0.3372556865,0.5908755064,0.5626513362,-2.2970151901,-0.2403149754,-0.3434230983,0.6380447745,-0.5477387309,-1.4343986511,0.8501843214,0.6045268774,-0.5088319778,0.1553441733,0.7751315236,-0.5564307570,0.0922820196,1.0430363417,-0.0166171789,0.0004913778,-0.3558220565,-1.0550065041,0.5611869097,0.5342825055,-0.0304869320,0.2164323777,-0.1864604950,0.0301791150,-1.0917770863,-0.1840087622,-0.0184875894,-0.6151685119,0.3353316188,-0.1206564009,0.0764194801,-0.2989043295,-1.5547710657,-0.6537920833,1.8045568466,-0.1564288735,-1.1329820156,0.5988346338,-1.8747942448,1.9920927286,-0.2383167297,-0.0050372910,1.7662442923,0.4502291381,-1.6901876926,0.2092062235,-0.3719630539,-1.4031659365,0.8186979890,0.3760634363,-2.2954051495,0.5926311016,0.0171566214,0.2273662835,-0.4911201298,0.2143833786,0.5893830061,0.1842756271,0.9189839959,1.7500014305,-0.0224729050,-0.0380397290,0.3788199723,-2.2946538925,1.7362905741,0.5558519959,1.1632210016,-0.3035589755,0.7126848698,0.2978425324,1.0315711498,0.4382598400,1.0446311235,0.1455101073,-0.7554751635,0.1581776291,-1.1658589840,-1.2868412733,0.7790217996,1.1975550652,0.5179507136,0.4167861640,-0.1070448756,0.8847790956,-2.1269643307,-2.1746633053,0.9809335470,-1.8558607101,-0.0589242019,-0.3949842155,0.1342334598,-1.9917581081,2.0874900818],[0.2646040916,-0.3620138764,-0.4476041794,-0.0092560723,2.7727761269,-0.9875227213,-1.1261665821,0.8644894958,0.0200013202,-0.5129029155,0.4734310210,-0.0488065369,-2.4048309326,-0.1564999670,1.1135571003,1.1397062540,-1.4627355337,0.7660913467,1.1386791468,-1.7050300837,-0.8706223369,1.3747351170,-2.1854331493,-0.7849680781,0.5419611335,-1.3487257957,-1.3122342825,0.1711963266,0.6263733506,0.1525226533,1.4732654095,-0.0650783703,-2.6287243366,1.1032720804,-0.2081544995,0.2773850560,-0.8050147891,1.7604209185,-0.4776040614,0.3857266903,1.1773180962,0.5346040130,0.7014524937,-0.2645107806,-0.1771366298,-0.8814995885,0.1855238378,0.5930702686,1.3211059570,0.3621626794,0.4415043592,0.9820915461,-1.1620858908,-1.5575093031,2.3486135006,-1.4105807543,0.5415712595,-0.3308941722,-1.2917026281,-0.1719277054,0.1799248606,-0.0432348475,0.2673465908,0.1946123540,0.8462187648,-0.8374105692,0.1203589514,1.9621579647,1.5041606426,0.6325783730,0.4140176177,-1.5472197533,-0.8060904741,1.3864337206,-0.7399890423,0.3658272028,1.8032287359,-1.0524270535,0.2010727376,-0.5947147608,-0.4959990084,0.5370004177,0.4044150114,-0.5982482433,-1.7088147402,-1.0533634424,1.2031201124,-0.1458389908,0.5778471231,0.0965054780,-0.5388431549,0.8363574147,0.4032280743,-0.2996425927,0.0294992831,1.9186033010,0.9316451550,1.4993157387,-1.7577295303,1.2611371279,1.0125068426,-0.2110064179,0.6125735044,0.7863415480,0.5930656195,-0.4154962897,-0.7229909897,-0.7670542598,-0.4934742451,-1.7241714001,0.9628956914,1.3955906630,-0.4624428153,-0.7550997138,0.2275715172,0.8299591541,0.9396640062,0.9205953479,-1.9562321901,0.5347333550,-0.3697306812,-1.0418801308,0.7764693499,-0.5618635416,-1.3113641739,-1.2335938215,0.7353008389,0.1957557499,-0.7724125981,1.1139376163,1.3518786430,0.0805120543,1.5585693121,0.5469241738,-0.5058984160,0.8069629669,-0.7987787724,0.1504934281,-1.6120203733,0.0503893793,0.7792242765,-0.3512535393,0.0884182081,1.1663037539,0.6744574904,-0.3506546021,1.3014886379,-1.0340659618,-0.3779429495,-0.9123903513,-0.3398987949,1.0012029409,-0.2418974638,0.8322359920,1.2388926744,1.4504948854,-0.0846828520,-1.9022624493,2.3461015224,1.5761616230,-0.4713822901,-0.2990472019,1.4670183659,-1.4660875797,0.1758549064,0.5109443665,-1.3963633776,0.3641264141,0.4785705507,-0.7390723825,-0.1571729332,-0.0894001424,-0.8202293515,-0.0035593198,-0.3045049012,-0.6465057135,-0.5934047699,0.8569859862,-0.4925061762,-1.3001301289,0.1930694282,0.0667027608,1.0578979254,-1.4051775932,0.5545969009,-1.9266557693,-0.3564795852,1.1653716564,-0.6970770955,1.0272433758,0.1583375931,-0.3123612106,-0.9408250451,-0.0537493266,0.4240416884,-0.1350065917,0.4084877074,0.0649588257,1.2394356728,0.0960153639,1.5031138659,0.2776180506,0.7171580195,0.8699058890,-1.8727498055,0.3917263150,1.8253725767,0.2035031319,-0.3448802531,0.8481007814,-0.4494965672,0.8984521031,-0.3145972490,0.1103251651,-0.2218849212,-2.0152313709,1.2233344316,0.6462559104,-0.1851824820,0.5452057719,-0.1294742972,-1.7381348610,1.4482873678,-0.0049621151],[-1.7968306541,-0.4029459059,-0.3125780225,0.9872809052,0.7336838245,0.1305475235,-2.8211119175,1.8661111593,-1.2324781418,0.2694059908,0.9574196935,-0.4235167801,1.3931517601,-0.5280845165,-0.7973154187,-0.1673012674,-1.2300739288,-0.1831893921,-0.9138897657,0.5805856586,-0.0222635027,-0.4146025181,0.9250200391,1.1654540300,0.1428458989,2.0506296158,0.8355020285,0.6833344102,-0.6771549582,-0.1392180920,1.3074415922,0.6128761172,0.9603426456,-0.9988200068,1.3938060999,1.4483420849,2.3093905449,0.8961781263,-0.4832390547,-2.5290327072,0.6685495377,-0.9248668551,0.5174524784,1.5624305010,-1.0747776031,0.0684774816,1.2940604687,-0.2854608595,0.8098345399,1.0701078176,0.0960196704,-0.7887099981,0.3971018493,0.3612840474,0.0241373852,-0.2990406454,2.1660528183,1.1026490927,-0.9874019027,-0.0388727337,0.2446159571,-0.5563523769,0.2279099971,-1.0013686419,1.0143290758,-1.5527062416,0.8506965041,1.2558175325,-0.2148320824,0.8770825267,1.1198774576,0.3611235023,-0.2671338916,0.0430668853,-0.1653925180,-0.9529352188,0.0276208688,0.1230975538,0.8873025179,-0.1752104014,0.2513716519,-0.2348491400,2.1551887989,0.7347248793,-0.4261212051,-1.4302924871,0.9446163177,2.5085418224,0.4724510610,-0.1860275865,-0.5170155168,0.8182741404,0.5141733289,0.0327660330,-1.5501236916,-1.9620072842,-2.0062484741,0.9044618607,-0.1077708378,0.6405754089,-0.5198460817,-0.1346986145,1.0445538759,-0.0471237898,0.6623798609,-2.9060950279,-2.2693784237,-0.9272943139,-0.7652207613,0.6250887513,1.0728448629,-0.3229994476,0.6231277585,-0.8498978019,-0.7956701517,0.2162239701,0.3675494194,-1.1451079845,-0.9653976560,0.2594099939,1.9920861721,-1.0531497002,1.0164002180,-2.6073150635,0.1700922847,-0.6301968694,0.2218909413,-1.0393652916,0.1635822654,-0.8220946193,1.2132215500,-0.2270285189,-0.0932268277,-1.9150704145,0.5888198614,0.8222286105,-0.9247884750,0.5923997760,1.2341030836,0.2066896558,0.0557046607,-0.4123021960,-0.5475937128,0.0371494219,-0.4496034384,-0.3558986485,-0.0194285549,-0.2425739467,1.0339695215,-0.9851918221,-0.5976397395,-0.8841294646,1.6174160242,-1.9489552975,0.7389916182,-2.6600670815,0.4461476505,-0.8088853359,0.8462277651,-0.6602464914,1.6914891005,-0.2079225481,0.1770420671,-0.1914528757,1.8777939081,-1.0028346777,-0.3571747243,-0.5823912621,-0.3817023933,-0.7646320462,0.3977974951,2.1370313168,0.0228019860,1.0142006874,-0.5090188980,2.4213712215,0.2384048551,0.4153284132,-0.5815684795,-0.3279374242,2.0014894009,-0.2259496450,-0.7761495113,-0.7924335599,-1.6703323126,0.2091366947,1.5131661892,-0.2143556774,0.4151538014,1.4834935665,1.0899714231,0.3293718398,2.0831754208,0.1978849769,0.2735405862,-0.6502671838,1.2972633839,0.1335616857,-0.3772461116,-0.2008029372,0.0285819676,-0.5361480117,-0.2193754762,0.2928190529,0.1224274337,-0.9885019660,0.0658708662,-1.2322801352,1.8402957916,0.6772176027,-0.7635117769,-0.2598209679,-0.9752302170,-0.9368657470,0.5052095056,-0.5493775010,-0.7561277747,1.1606968641,-0.8948866129,0.7956269383,-1.0711044073,0.4195824265,0.4358394146,-0.5332286954],[-0.7573457956,-0.6774635911,-1.6076130867,1.3072478771,-0.5927714705,0.1389958411,0.7185947895,3.7669422626,-0.0290491264,1.9280542135,-0.7334471941,-0.6078340411,0.8206052184,-0.7893049717,1.1871837378,0.5642030239,-0.4392837882,-0.5427054763,1.4480323792,0.9012516737,-1.4478220940,-0.2742679417,0.3881469667,-0.7936916351,0.9068234563,-0.2690926790,0.6635046005,-0.8194814920,0.8768583536,-0.2403390408,-0.4118672311,-0.1585797518,0.7974650264,-1.6701654196,0.2306298316,1.2565150261,0.7238920927,0.2698653042,-0.1637378335,-1.1882640123,0.1818947494,-0.7447010875,-1.3373509645,2.0189678669,-0.2577935159,-1.8308736086,0.5547665954,0.5500036478,-1.5290178061,0.2087140232,-2.4184942245,0.1614990234,-0.5703580976,-0.3850896358,0.3657353520,-0.6663386226,-0.5914227366,-0.1594028920,0.5978386998,0.6462613344,-0.5100587010,0.9332790375,0.1504412591,2.2288496494,-0.0264447983,-0.3791185021,-1.0874207020,1.3430011272,0.1849463731,0.1412463486,0.4610200822,-0.4325022697,-0.1002900004,0.4237989783,-0.0037172488,-0.7439040542,1.3982913494,0.6422342658,0.2297436744,-0.3174774945,-1.8848919868,0.4019989967,-1.0464760065,-1.4771206379,0.9808022380,-1.1731636524,-0.5929113626,1.4529060125,1.8952001333,0.8750221729,-0.8161361217,-0.8773684502,1.5960000753,-0.0505642667,2.1601340771,-0.5453072190,0.0826762095,-0.7931586504,-0.4187310934,0.8851934075,-0.0452485234,0.7194694877,-1.1667468548,-0.8115680814,1.0736287832,-0.0052045393,0.2035568357,-0.2131816298,-1.4077930450,0.4754085839,-1.6127541065,0.9811051488,0.3119699657,3.0805301666,1.3608472347,-0.4485848546,-0.5215060115,2.0166110992,-0.3327984214,1.2407314777,0.6968078017,0.0539997965,0.0699510276,-0.1169878095,0.8041012883,0.1101086438,0.0337015390,0.6410145760,1.6915745735,0.0526624918,-0.2966575623,-2.1406083107,-0.8456923366,0.7050642371,-0.0481108874,0.1677255780,-0.3968006372,-0.2574980557,1.8308932781,0.7751365304,1.7864623070,0.4759546816,0.3503513336,1.3485484123,-1.1443794966,0.4006808996,-0.9962391853,-1.1875445843,-0.9584485888,0.6679587364,0.8440365791,-1.6700847149,-1.6379746199,1.4682731628,-0.1765896976,0.0353947654,0.1196492165,2.1720237732,1.5219050646,0.2371326536,-0.2789554000,-0.6909232736,-0.1486644447,-0.7018318176,-0.6372700930,0.4759524167,-0.6893990040,1.1211477518,-1.2002559900,0.2479523867,-0.1926452816,1.7607023716,-0.5664526820,-1.9813816547,1.9055066109,-1.4104237556,-0.5574155450,-1.0331728458,-1.3837974072,0.2761999965,-1.9508929253,-0.0858562961,-0.4836938679,1.6077718735,0.7905129194,0.2117312104,-0.4057725370,-0.2354757786,-0.2259673476,-0.3836088479,0.3403173387,-0.2910141647,0.2482576072,-1.5559599400,0.3220538497,0.7779824138,-0.2298941314,0.7137753367,0.0410445072,0.8990941048,-0.0591654740,-1.2160205841,0.7276846170,1.3633673191,0.2227507532,0.0442474857,-0.0720062032,-0.4402479529,-0.2421229333,-0.1403012723,-0.8662195802,0.7516864538,-0.5260761380,0.3029883206,-0.1204497144,-0.8628455997,0.6711946130,-1.5811458826,-1.0912187099,1.5749882460,0.8000087738,1.4918595552,-0.0239492729,-0.6851808429],[1.6194691658,-0.6268472075,0.7402077913,0.3828008473,0.2770670056,-1.6399253607,-0.7085242867,-0.4483987689,1.1894177198,0.1778671741,0.7329412103,0.2275713235,0.9885006547,0.3365898430,1.0390247107,-0.3508904874,-0.7135981321,-0.5375814438,-1.8914824724,-0.8740919828,-1.0973818302,-1.7287472486,1.2013589144,-2.6296367645,0.7503966689,0.1960070133,1.0780261755,0.5065403581,-0.5458502769,0.3345984817,0.1483134329,0.9152549505,0.1478965133,1.5200810432,-0.7546271682,-2.1418476105,-0.2708940506,-0.2287595272,0.6522660851,-0.2509295642,2.2269825935,0.5684635043,1.0577745438,0.7382932305,-0.0012409586,1.4664787054,-0.1449816078,-1.0437244177,0.6076418161,-0.4137753546,1.8593870401,-1.4749391079,1.8089412451,0.0769915134,0.5412765741,0.3424822688,1.3275287151,0.1800303906,0.0366178341,0.3738724887,-0.7425840497,-0.0967953280,0.6599724293,1.0047409534,0.9127472043,0.5290979147,-1.2235962152,1.8528863192,-0.3413074315,-0.3027526140,0.4114413261,0.9746417403,0.1835276186,1.0362179279,-0.6989639401,-1.2286016941,-1.2848297358,-0.9485433102,1.5757834911,0.2003630102,0.3445673585,1.0008212328,-0.0349921770,3.7082657814,0.8298263550,-0.6615275145,1.3744132519,-1.6828603745,0.5283031464,0.6071580648,2.1209676266,0.9397947192,0.5499040484,1.2169115543,-1.5446734428,0.0012747148,1.7734676600,-1.0784699917,-0.9334080815,-0.3895108104,-0.4134129286,2.5833759308,-1.1176213026,-0.5504791737,0.8675289750,-0.5212416053,0.4319742918,1.1117717028,-1.8729016781,-1.6570641994,-0.9629516006,1.2397440672,0.0037459065,1.0992046595,0.0913702697,1.1402840614,-1.0717334747,-0.9748516679,-0.2703653872,-1.2980529070,0.6257337332,0.6659516692,0.3891419768,0.8163871169,1.4753880501,-0.2891272604,-0.5772545338,-0.3888491094,-0.7375437617,0.1387289315,-0.0836863145,0.1055305451,-1.0039715767,-0.2395810485,-1.6525434256,1.6949305534,0.3663100302,-0.7106671929,0.3623050153,0.1398703009,0.0709123462,0.4497546256,-0.4463917315,-1.8335397243,0.1431326866,-0.5487794876,0.2520763278,-0.4617098868,-0.6493166685,0.8026529551,-1.4981682301,1.4404788017,0.2838287055,0.7307789326,-0.5151305199,0.7872204185,0.8811286688,-0.1221335679,-0.0872639865,-1.3045835495,-0.4357743859,2.2551152706,0.2483468205,-1.9551292658,-0.4188409150,0.0355005972,0.7761195302,0.5764486194,-0.9114215970,0.4186478853,0.0519385114,-1.3720073700,0.9566460252,1.0162568092,-1.2849221230,-0.1405265629,1.1921300888,2.2774107456,0.6626639366,1.2861585617,-0.4691481292,-0.5023907423,-2.0549042225,-0.5954032540,0.6965740919,-0.0113108791,0.9470972419,-0.4827988446,-0.7364525795,1.5049593449,-0.5338101387,-0.5999995470,0.2748598456,1.2512134314,1.3383157253,0.8594609499,0.2186160684,0.7017844915,-0.4627053738,0.0034496156,-0.1348692179,0.3913027644,-0.7634017467,1.2350702286,-0.2394351661,-0.3701135516,0.5651393533,-0.0991172194,-0.0025406210,1.0179543495,-1.3121225834,-0.4948684275,1.2599962950,0.1616281420,0.6271054745,0.3267571032,1.7896078825,-0.1520906836,0.7437182069,0.1602447033,-1.1462504864,0.7538537979,0.1994169801,-1.5422637463],[-0.1115837023,-1.1347322464,-1.5603486300,-1.3231898546,-0.5864630938,0.2438378185,1.0190917253,-1.4585080147,0.2916617692,0.1212151349,1.8103116751,0.0997496024,0.8982300758,-0.6879515052,1.4924706221,-1.4564908743,-0.7765069604,-0.9859484434,0.2000730485,1.2155988216,0.0159493014,1.4958109856,0.1811317950,0.5959467888,-1.3172075748,0.2131920159,-1.2470915318,-1.2693005800,-0.5841072202,-0.0466538183,1.4094572067,0.7706953287,1.0238623619,0.5501586795,-1.9324756861,0.8537841439,0.0228950474,-0.9649195671,-0.1006398126,3.7234966755,-1.4024811983,1.4509285688,-0.0061165541,-2.3573803902,-1.7164319754,1.4367122650,0.4027778208,2.0568888187,-1.4577573538,2.1596512794,0.0112678483,0.4823514819,-1.0151923895,-0.2748779655,-0.1498049647,-0.9562303424,-0.4943629801,0.9176456332,-1.1512298584,-0.6620446444,-0.7779641151,0.3872365654,-0.9894063473,-0.7422857881,-0.3691743314,-0.7506567836,-0.1978930682,-0.8630726337,1.1947299242,-0.6482266784,0.6255560517,1.0509376526,-0.1812707335,0.3463947773,1.2976157665,-0.3504452705,-0.4335976541,1.2804650068,-1.0233523846,-1.5230047703,-1.8323204517,1.0930193663,-0.9281481504,-0.5940221548,-0.7265738249,-1.2965354919,-1.0185232162,0.7093110681,1.1391676664,-0.7207520604,0.0884170756,0.2882946134,0.5193282962,-1.4298005104,-1.6081708670,-1.2717075348,0.4682482779,-0.0170615129,1.7580053806,-0.9173281789,-0.4693002105,-1.9680835009,-0.3299243450,1.0726827383,0.8806637526,-0.7417538762,-0.6689109802,-1.7082813978,0.4313643873,0.0347285047,-0.9227795005,-1.4209941626,0.7686933875,1.9805835485,-0.8032012582,1.6413334608,1.3727482557,-1.0299570560,-1.0851159096,1.5018138885,-1.3383134604,0.9689275622,-0.9780363441,-0.4031761587,-0.3334401846,0.6129492521,0.1054987162,1.3304780722,-0.1858203262,-0.4620821774,-1.9098011255,-1.3124620914,-0.7047039866,-0.7252709270,-0.1312124729,-0.9187059402,0.6097781658,0.2633499205,0.6997118592,0.8780248761,-0.1637497842,0.2091602236,0.5623223186,2.0462377071,-0.8705199957,-1.1179158688,-0.5697802305,-1.0332211256,-0.3013561070,-1.1498878002,-1.2747392654,-0.5573811531,-0.4904537797,-0.7212755084,0.3405832052,0.2075432539,2.3557858467,-0.3203197718,-1.0107301474,-0.4470166564,-0.8744552135,0.9583758712,-0.3872155547,-0.3068799078,0.2179597914,0.5574518442,1.4925589561,-1.0170717239,-0.7130813599,0.1082530543,0.7955113649,0.4275724590,-0.7770849466,0.3904846013,-2.1188902855,-0.8035373688,-1.8661222458,-1.3831884861,1.5261973143,0.9782083035,2.0999190807,-0.9385385513,-0.7025492191,-0.5098134279,-0.0868382528,0.6528349519,-1.4956269264,1.7219619751,0.6622504592,0.0382152312,0.4614329040,0.5472880006,0.5798900127,0.5045868754,-0.6498355269,-1.9459872246,1.3929944038,-0.6809290648,-0.2962422967,0.0108508496,-0.9638430476,-0.9484646916,0.5165789723,-0.1640670300,-1.7002995014,0.8765365481,0.1269982159,-1.2878123522,0.6062132716,-1.5158621073,1.2426468134,0.0607079677,0.4924314320,-0.1854637265,-0.1353620142,0.2781190872,0.2474597394,-0.9971805215,-1.0900882483,-1.0746543407,-1.8340034485,0.3812614679,-0.8598755598,-0.1040497944],[1.3925131559,0.3986272812,-0.0415031575,-2.3675251007,-0.0305877812,1.6660884619,-0.1070648730,-0.5445333719,-0.7710431218,0.3810178041,0.2547812760,0.0593349338,-1.8837507963,0.5730901957,-1.0793691874,-0.4145066440,0.8921239376,1.0510808229,1.2406498194,0.1838686913,0.2870498300,-1.5952960253,0.3279765248,-0.4117076099,-1.3453356028,-1.2739416361,1.5920748711,0.5261922479,-0.0942941159,-1.0936295986,-0.6000131369,1.0554364920,1.1460963488,-0.8146736026,0.8606137037,1.5689191818,-0.5961349010,0.0758933052,-0.3385274708,1.3630273342,-0.5600120425,0.9728752375,-2.6392047405,0.3673083484,0.9900626540,0.3304453790,-0.1471444517,-0.2834138274,-1.0353995562,-0.5525274277,-0.5576259494,0.1510661095,-0.2260467261,0.2252246886,-0.6449528933,0.5770357251,0.6935384870,-0.1395560056,0.2972361147,0.1321164519,0.1647407860,0.3600349426,0.2307815105,-2.8135678768,0.0207150485,1.1604722738,-0.6801362038,0.8473317623,-1.2867816687,0.9362281561,-1.0985594988,0.3639356196,0.0338978171,1.3219554424,0.1933799237,-0.4506433904,0.8257185221,-0.2247683406,-1.3679728508,1.0385216475,0.9364742041,-0.6909128428,1.5373262167,0.6860220432,0.4178827405,0.6212495565,0.4344584346,-0.8601510525,-0.3218546510,-0.8268820047,-1.5539178848,-0.0130258258,2.2126760483,1.7009296417,1.0740090609,-0.3324290514,-0.8985465169,1.4576840401,0.4782959223,-0.3509194255,1.1262203455,-1.2219204903,-0.2200453132,1.3531450033,-0.1954530925,2.3975813389,1.8100578785,-0.5661913753,0.1861822009,1.3492763042,1.3133230209,-0.6185108423,1.3017319441,0.9068949223,-1.0815957785,-0.4340805113,-0.3503829241,1.4641578197,-2.0569789410,1.0820784569,1.7439060211,1.8248600960,-0.7774598002,-0.4939185083,1.0127602816,0.7717211246,-0.3367897570,-1.2080198526,-0.6399074793,1.1590770483,-0.1210201681,1.1723109484,0.8146342635,0.1039312407,0.8836551905,-0.5924564600,0.1072652787,-0.3974935710,-0.2205309421,-1.1659206152,-0.8362118006,-0.8025512695,-0.2125316709,0.7141345739,0.6295037270,1.1987661123,0.1260221899,-0.1531082541,-1.0310268402,0.0731412917,-0.1965200156,-0.4568440616,0.8053567410,0.2883935869,-0.6044726968,-0.1260431856,-0.0165450405,0.6680634618,-0.2518533468,-1.0752733946,0.3414058983,-0.7781500220,1.3908762932,-0.4280652702,0.3502120078,1.5740196705,1.5462254286,1.3005847931,0.2207820117,1.2206280231,1.2158526182,-0.4125004411,0.3144003749,-1.4685095549,-0.0741400197,0.5391044617,1.2650680542,1.1636080742,-0.6286639571,0.3842039406,0.4505725801,1.3273755312,-0.7348456383,0.7658395767,-0.4287714362,0.3170690536,-1.6846287251,1.2574837208,2.2304196358,0.1366726011,1.1004315615,-0.1765469760,-1.1436213255,-0.4769022167,-0.0776945502,-0.5553138256,0.6878874302,0.7855275273,1.6607589722,1.0358875990,0.4534233212,-1.6446279287,-0.1976266950,2.1636257172,-0.7871826887,-0.2852320671,0.4802152514,-1.8419892788,-0.2016286850,-0.0867040604,-1.0645453930,-0.8415598869,1.5475199223,-0.3730809987,-1.4787642956,0.2056966722,-0.3578105569,-0.1109743640,-1.2149100304,1.2062696218,1.3023071289,0.9274048209,-0.7516996264,-1.3269295692],[0.3645735383,0.2758433521,1.9448007345,-1.4305678606,-0.0758651718,0.0477598943,0.7821953893,-0.5015829206,-1.6125231981,0.1059435233,0.6842874885,0.9025937915,-0.6946648359,-0.2183943540,-0.3142189384,-0.0116561214,1.1320439577,0.6315576434,-1.1208635569,-0.3343107104,0.5499612093,0.8175386786,1.5596295595,0.5662127733,1.5119391680,-0.1291352212,-0.1208290309,0.5914816260,-0.8418577313,2.0425741673,-0.2564697862,1.2515211105,0.6368747950,-1.1409413815,-1.0605794191,-0.1664215922,-0.9259193540,2.5111291409,-0.0861898288,-2.0625565052,0.6647928357,0.0328417681,0.5346663594,1.9692085981,0.4651908278,0.0526132919,-0.4012923241,1.6575239897,1.1805459261,0.5314475894,0.7792928219,-0.5494493246,0.3818164170,-0.0567653663,0.4041859508,1.4742921591,-1.0135000944,0.4748797119,0.7541258335,-1.0927159786,0.0926267952,0.9003172517,-0.5001785755,0.5012148023,-0.6418183446,-0.9861421585,0.9376791716,-0.5699564815,-0.6094684601,-1.0713261366,1.9327700138,-0.9455596209,-0.9719607830,0.6471062303,-0.0367334001,-0.5560221672,-0.3756830692,2.0556206703,-2.3344113827,0.3166075051,-0.0935946330,0.9930018783,0.5013293028,-0.1114913821,1.7273072004,-1.0956972837,0.1108219996,0.0784082487,-0.1691263020,0.2886268198,1.7784488201,0.1265564561,-0.5149015784,-0.2519254982,-1.0531100035,0.7891155481,0.4315459132,-1.5275148153,1.4729213715,-0.3302557170,0.0881882012,-0.8817841411,-0.7540366650,0.8779074550,-0.8743508458,0.2878976464,-0.5434914231,1.0279676914,0.9953140020,-0.8819229007,-1.3638242483,0.3675503135,-0.3905891180,-0.7033363581,1.2411277294,-0.3832196891,-0.5711125731,-1.4024157524,-0.3557167053,0.3662314713,-0.6938498020,0.0533289164,0.3695681691,0.1856286973,0.0036913755,0.1313543767,0.8270304203,1.3178888559,-0.3671407402,-0.2499872595,0.3193106651,0.3666144907,-0.4536692798,0.0637796372,0.6246864200,0.0064440314,0.4970657527,-1.6432136297,1.0414912701,0.5748828053,0.1528365761,1.0264930725,0.0289772060,-0.7684674859,1.8304229975,1.9216315746,-1.1279611588,-0.2746619284,-0.1850537807,0.7062392831,0.3555676937,1.0713543892,0.4431940615,0.1439880580,0.9895448685,-0.0911555588,-1.7978384495,0.5766109228,-0.0966744050,0.0376248136,-0.9085863233,1.7782652378,1.0522457361,0.6517040730,-0.8900560141,0.6581432223,0.6606887579,-0.4885232449,-1.1564961672,1.9684470892,2.1824321747,-0.3049057424,-0.8701323867,2.3631441593,0.3133211434,-1.1149035692,-1.3495191336,1.9155670404,-1.0757849216,-0.6060916781,0.3758997321,-0.9581190348,0.6761481762,-1.5759015083,0.0877688527,0.9830061197,0.5539374948,1.0762681961,0.2258782238,1.5641163588,-1.3022426367,0.4803126752,0.6107187271,-1.0131540298,-1.5784831047,-0.8989061117,0.4761588275,-0.9230661988,0.5590661168,-1.0126085281,-0.3330108821,0.4091787934,0.8562009931,1.3837891817,0.9529459476,0.0378294997,1.7116134167,0.1376933604,0.4267888963,1.4159948826,1.1380883455,-0.8711742163,1.3790419102,-1.1110154390,0.0842593387,-0.2383049726,0.3928242922,-0.0319515020,0.6350234747,-0.5745252967,-1.0377408266,-0.5247514844,-0.3461388052,0.2433131039],[1.9716480970,0.8219295740,0.6601530910,-0.9692701697,0.3216661215,1.2229069471,0.9288674593,0.3407813609,-0.0449075103,-0.6433902383,-0.3510982990,0.4039245546,1.4285668135,2.1478550434,0.1891006231,0.5928854346,-0.0630526319,1.3296774626,-0.2089577764,1.1789846420,-0.8879231811,-1.1200343370,0.5014985800,-0.6400913596,1.3707021475,0.8418042064,-0.0632963851,-1.1129459143,1.2401936054,-0.2269525230,1.8008857965,1.1790270805,0.5483096242,-1.9282461405,-0.0091208350,-0.0526960641,-0.5831608176,0.7987315059,-1.6418938637,-0.9462875128,0.3626517355,0.7144135833,0.0920099840,-0.1160614640,-2.1590559483,0.8512916565,-0.9435035586,-3.0365452766,0.3538107574,-0.9204629064,-1.2248587608,-0.4802761674,-0.0648367479,0.3905614316,0.5931641459,0.4110111594,1.9805357456,-0.5985766649,1.4174280167,0.2611472011,0.4249542952,0.8081253767,-0.4066265523,1.3087968826,0.3670448959,-0.4409090877,0.1955190897,0.8306531906,1.0105479956,-1.2028720379,1.6242797375,-1.0599883795,1.1703038216,0.3944687247,-0.0508274548,-1.7612975836,0.9173631072,0.1352318525,-0.9668301344,0.9786379337,0.9339951277,-1.3411684036,-0.8798589706,1.2675123215,-0.3759378195,1.0647661686,1.0037868023,-0.5811907649,-2.7897138596,-0.1828412116,-1.3186894655,-0.6859114170,0.3702635467,0.1879754215,-0.2166812122,-0.4224203527,-1.1869359016,1.4768501520,-0.1624441296,1.3615174294,0.0276617203,1.2067369223,1.8776559830,0.4511460364,-1.2373962402,0.7331387997,1.7680873871,0.9055304527,0.3431780040,-0.7190120220,-0.8236030340,0.3431842923,-0.2256057709,0.1887262464,0.7279495001,0.1026036218,0.0037209450,0.2864089906,0.1306706518,0.3032578826,-1.9690222740,-0.3773420453,-0.3418140411,-1.1602931023,0.6077716947,0.9177352190,-0.7473893762,-0.3076132238,-1.1850211620,0.1065358967,1.5186152458,0.7725510597,-1.0014253855,-2.1937179565,0.3250127137,-0.6440759301,-0.1811335981,-1.1045614481,-2.1818692684,-0.2243866771,-0.0173453316,0.3536208272,-0.0725474954,1.8809698820,-0.2315787226,0.8793317080,1.1665952206,0.0993929505,-1.4947397709,1.0953342915,-2.2650103569,1.1013573408,-0.3124994040,-1.2784965038,-1.7349416018,-0.7721288204,-0.9405128360,-0.0988894254,0.9771395326,0.6489016414,1.6465309858,-0.1293012649,0.0067455000,1.6675785780,-1.0477951765,0.4693665802,-1.3260034323,1.6122599840,0.8219315410,-0.2401393354,-0.4720887244,1.1021941900,0.4860531688,-2.6774544716,0.7096341848,2.5483486652,-0.2957075238,-0.6031370759,0.1926307380,-0.4034855664,0.1463181674,0.5350070596,2.0819504261,1.8139798641,-1.0643401146,-0.3128781021,-0.1442638040,-2.1851909161,-1.5748522282,-1.0711203814,-0.4753435254,0.0661187768,1.1281665564,0.1130188406,0.8959493637,-0.0683412403,-1.3801412582,-0.9871893525,-0.7403144836,-0.8445615172,1.6470441818,1.0460041761,0.6926950812,-0.7393996716,-0.9453443885,-0.2465624362,-1.2683348656,1.9672598839,0.9712939262,-0.7164001465,1.2384980917,1.3134400845,-1.2783517838,-0.9249799252,-0.9393361211,-0.6579603553,-0.2182013690,0.9198998809,1.2773807049,1.1855596304,-0.6381624937,-1.0942879915,0.4838123024,-0.4306229949],[0.1810165793,-0.1996294707,1.2885905504,-0.1678638160,-0.1881410182,1.3225277662,0.0953734741,1.8875477314,0.0770172328,1.6916360855,-1.7926950455,0.4905884266,1.0937842131,-0.0595675372,0.9946762919,2.0324313641,-1.0170233250,2.0547215939,1.6750184298,-1.0087105036,-1.6904520988,0.3167631328,0.6377083659,-0.7585461140,-0.6136844158,-1.8868082762,-0.1847042292,0.8395951390,-0.1964798272,0.0806577355,1.1836705208,0.6784030795,0.8249893785,-1.2950379848,-1.1152267456,0.6911618710,-0.4725620151,-0.6914433241,-0.2887046039,0.8318896294,0.5734977722,0.7145007849,1.7885981798,0.0377317369,0.7398540378,-0.6245130897,0.7582843900,0.8068283796,1.8076828718,0.6233335733,0.6361873746,-1.2487183809,0.2187319547,-0.6943225861,0.3531611562,0.8272674084,-1.9904072285,-0.9367148280,-1.0429241657,0.6981034875,-1.5150380135,1.4994618893,-1.4663890600,2.2427587509,-0.3023525774,-0.6387372613,-0.8940060735,-0.1322064400,-1.0280362368,-0.4068548679,0.2255519480,0.9413877130,0.0257321727,-0.1329892278,-1.6131180525,0.8192076683,0.1125036925,1.3352439404,0.6392394900,-3.1156716347,2.0869877338,1.0112729073,2.4250719547,-0.1496810317,0.7387430668,1.2361007929,0.0684093386,-0.9277183414,-0.1533320844,-0.7304266691,0.6191169024,-0.7954546213,-2.1412179470,1.2499122620,-1.3871273994,2.1028695107,-1.0435626507,-0.5314572453,0.2357278168,-0.7836205363,1.6709499359,1.4669779539,3.1746845245,1.3023787737,2.3591947556,-0.1380295604,0.9297934175,-0.5155693889,-0.1036393195,-1.1186670065,-0.1851234734,-0.2512214482,1.1714904308,-0.2232470512,-1.4790050983,1.2210992575,1.7563029528,0.4102286100,0.0601698495,0.2168567479,-0.8762237430,0.9252765179,-1.7906960249,-1.3530749083,0.0037942464,-0.6046807766,-0.4491837025,0.1465787441,-0.2187786400,2.6423318386,1.8395512104,-1.2164244652,-0.0062938496,0.7514982820,-1.7471815348,1.3979443312,0.4519581199,1.2722970247,0.0478293486,0.0334270969,0.5252046585,0.8231018782,0.4410232306,-0.2736770213,-0.6498092413,0.6456122398,-1.5126706362,-0.9126222134,-1.0825484991,0.8857665658,-0.4664177299,1.2132717371,-1.1514225006,-1.0739794970,0.1377086192,1.9164336920,-1.2203828096,-0.7694478035,1.3315281868,-0.7050071955,0.8304207921,0.6928241253,-0.4450389445,1.1498022079,-0.6999378204,-0.6708523631,2.8639185429,0.7906737924,0.6672154069,0.7954690456,0.1095909402,0.1944302619,0.0010998135,-0.4005476534,0.2124238312,0.4286309779,0.2146281302,0.6275988817,-0.0620207116,-0.4323121309,-1.7828309536,1.5191432238,-0.3685704768,0.3279248774,0.2474352568,0.7215018868,0.7299074531,0.0056346082,0.6240631938,-0.7521722317,0.7414829135,1.3525691032,0.2047773600,0.8658626080,-0.2844838798,0.5375939608,0.0502211004,-1.9202650785,0.6328778863,-1.0243452787,-0.3291671872,1.2854202986,-0.3071571589,1.1487460136,-0.6724426150,0.9937167764,0.0755469427,0.0181107987,0.3345252872,0.0915125236,-0.1587377340,0.9943240285,1.0686699152,-0.0836843699,0.8532247543,-0.1283108890,0.5830957890,-0.5074785352,-0.8843220472,0.9858810306,0.4600271285,-0.9244024158,0.2015917897,-2.2051551342],[-1.3684561253,-0.0070179529,0.9507652521,-0.8961302042,1.3040012121,-0.4097203612,0.5474057198,1.3817955256,-1.2054364681,0.5918625593,0.2912769020,0.3815670609,-1.7695758343,0.2565854192,0.6678588390,-1.9968492985,-0.4847605526,-0.4586411417,-0.2693426907,-0.1771221757,1.7575697899,0.0226126239,1.3108720779,-0.1741402000,0.5192768574,1.5016207695,-0.3459168375,0.8827842474,-1.0666747093,-0.6622051001,0.2035106122,-0.0336035080,0.4757933319,2.3770163059,0.7890660763,-0.4296334386,-0.6260621548,0.8593677878,0.9754520059,-0.5508008003,-1.9562290907,-0.4794707298,2.1985228062,0.7364495993,0.2542623580,0.2583675385,-0.0006999811,-0.0937647223,0.0307967123,2.1079499722,-0.9306992292,2.4908797741,-0.0683824047,0.8416444063,-1.4192576408,-0.2434727401,0.7902024388,-1.5382992029,0.9410758018,-0.7255437374,-2.1972353458,1.4353498220,0.6533204317,-2.7115106583,0.6271278262,0.3922728002,0.1899161339,-0.2647828162,2.3061597347,-0.3845082223,-0.2433109134,0.4348921478,-1.5491979122,0.1541192085,-0.4593312144,-0.9138321877,-0.1297478825,-0.2293144763,1.6444797516,0.9206303954,0.2121412903,-1.3027334213,0.8136668801,-0.7190756798,-0.3822887242,0.2437139153,-1.1598749161,-0.2366128266,0.9961916208,0.9275527000,1.0301316977,0.8716277480,1.1247831583,0.7137174606,-0.0485413894,0.5693393946,-1.9486206770,-0.4884814322,1.6192941666,0.8910787106,0.0180339478,-1.7390911579,0.2681433856,-0.1390723884,1.0840507746,-0.3824338019,-0.5290316939,-0.6295898557,-1.5379174948,1.1584496498,0.6609149575,-0.6455150247,-1.4774763584,1.3007858992,-0.8344992399,0.2422126979,-1.0668669939,0.7216842175,-0.2747242153,0.6934828758,-0.3771536648,0.2721296549,1.6877368689,-0.3436425626,-1.3526052237,0.6161730289,-0.3104036152,0.4038379490,-0.8696197867,-0.0881191045,0.8002542853,0.2152355313,-2.1688718796,-0.1544931531,0.5612537265,-0.7872266173,-0.6685209274,1.3880770206,-0.0438574255,0.6691686511,-1.1393299103,1.5715401173,1.7690942287,-0.1770737022,-1.2838332653,-0.1482763737,-1.1807324886,-0.1246396676,0.5556858778,-0.5587546825,1.9595792294,-1.0273351669,-0.2892897129,2.1519374847,1.4404784441,-1.6324875355,0.9220415354,-0.8804201484,0.9086288810,1.3144823313,0.1263292283,1.1237426996,0.1560661346,0.9178130031,1.0348485708,-0.0764593482,-0.1803882867,0.4778284132,-2.6791322231,-1.4770958424,0.0729591846,-1.4535052776,0.1024222225,-1.0942270756,0.3704869747,0.4322766662,0.2548621595,0.6323297024,-1.3469249010,0.9590857625,-0.7356510162,-1.4300709963,-0.8788210154,1.3715491295,1.1217643023,-0.5817310214,-0.6780789495,0.1847612560,-2.2621667385,-1.3844285011,-0.4063298106,-0.4010620713,1.0016220808,0.0181291588,-0.3874219656,-0.6327757835,-0.6283560991,-1.4798924923,0.0586887263,-0.3952905834,-1.0938864946,0.9593856335,-1.2852833271,0.5518447161,-1.1919951439,0.4875165224,0.4057821333,0.9937305450,-0.9238799810,2.6145756245,0.4748609662,-0.3027443588,0.2600146830,-0.2662793696,0.5243310928,-0.7458996773,-1.3094813824,0.2878949642,1.0456981659,-0.3525495529,-0.8172661662,0.2186447978,-0.5194003582,-0.2409159392],[1.5012190342,-0.3814423680,-0.2719659209,-1.6522637606,0.8976294994,1.6063685417,-0.3337241113,-0.7124217749,0.8000403643,0.0348635353,-1.2435899973,0.2742917240,1.2672042847,1.3071616888,-0.3089774847,-0.5630499721,-1.6949775219,-0.3910141885,-0.4686586857,-0.4318181872,-0.4553795457,-1.5290528536,0.7596347928,-1.2247896194,0.7929475904,3.2648231983,0.0765919611,-1.7044337988,0.5527252555,-1.2542990446,1.0776225328,0.7700014114,0.3957244158,1.0734199286,-0.3395042419,-2.0320532322,-0.7339397073,1.7088943720,-1.7991150618,-0.3706928492,0.2754571736,0.2273407429,-1.0307626724,1.1256923676,-0.8246914744,0.3956819177,-0.9917507172,0.7536754012,0.6936249733,1.8786382675,0.5164870024,-0.5184623599,0.1207001433,0.2281433642,-0.9246230721,-0.0446610264,-0.3871702552,-1.5042041540,-0.0066607548,-1.5743414164,-0.2049447745,-2.2182295322,0.2431137711,0.5944489241,0.5098665953,1.4379987717,-0.4044253528,1.0512136221,-0.6692307591,-0.5159274340,-0.5512489080,0.5259061456,0.2661730647,-1.5106946230,1.9273676872,0.5774011612,-0.4199075699,1.3673183918,-0.5175925493,0.0277764630,-1.0282031298,-1.2392059565,-0.3175444007,0.0651226565,0.1687745154,-0.7958122492,-1.1371675730,1.8399970531,1.0105547905,-0.5101677179,-2.0513987541,-0.2365717590,1.0404173136,0.8387218714,-1.5863597393,-0.2953415215,-0.3548167348,-0.8969500661,-0.5921047926,-0.4963432848,-0.7090011239,-1.4159350395,0.0659552068,0.6083223820,1.4280581474,-2.7973577976,0.2793337703,0.4457551241,-1.1316871643,0.7324762344,0.0465258770,1.2708383799,-0.8990916610,0.9488687515,0.9713152647,-1.1941430569,-0.3017987013,-0.9432987571,0.7103397846,0.1358730197,0.1415656805,0.5150106549,-1.7309122086,-1.9883096218,0.7167133689,0.0778244287,-2.1529717445,-0.9673052430,0.3667652607,0.1836275011,0.5833052397,-0.4578369260,-0.4256786406,-0.9554052353,1.3549624681,-0.4652006924,-0.2363625318,0.0994006470,-0.6568480134,0.0348669849,0.3077637255,0.3547363579,0.8136194944,-0.6794636250,-1.0101652145,0.7936237454,0.7654732466,-1.3967742920,2.0011563301,-0.8323894739,-0.1461954713,1.0044622421,-0.6343291402,0.2242664993,1.7496575117,-0.4828544855,2.1647851467,0.7960585952,0.0863240808,-1.8356450796,-0.1754117906,0.9070207477,-1.2061300278,0.6664801240,0.2954353392,1.9272356033,1.0407719612,0.4945245683,0.5699928999,-0.1160124764,-0.4865684211,-0.0170620270,-1.6850682497,0.6151177287,-0.6133508086,-1.4380831718,1.6350381374,-0.2955036759,-0.4080443978,0.6777896285,-0.7060639858,-0.4976985455,-0.9679428339,0.0706002116,0.0416285954,0.2848752439,-0.8778271079,-0.1663546264,-1.4794622660,-1.9278879166,1.6978425980,1.7794172764,-2.9531173706,-1.2205433846,1.5037174225,0.5455800891,0.6231148243,-0.1637870520,-0.1267165393,-0.6366458535,0.4950649738,0.3843171895,0.3920426071,-0.8354402184,-0.2788834572,1.0194483995,0.0986561626,-0.5203149319,-0.0823765323,0.0222817399,-1.2570695877,0.0349741466,0.2028528899,0.2815420926,-0.1766107976,0.2764383554,-1.3478579521,1.4862694740,-0.4602116346,0.1060000360,0.6406856775,0.6851381660,0.9093105793,0.5848791003],[0.1880259067,-0.9595815539,-0.2873590291,0.5662950873,-3.1337368488,1.4183876514,-0.3170253038,-1.9341831207,-0.7753921151,-0.5650481582,0.9571687579,-0.0570281744,-0.2114823312,-0.1237578094,-0.5994918346,1.2810485363,2.4784626961,-1.4082845449,0.6607084870,1.2495912313,-0.8925880194,-0.8004648685,-0.1284520179,-0.9761826396,-0.3523084521,1.5950284004,1.3361389637,-0.4479740262,1.9581365585,-0.1791668683,-0.7813001275,0.9748125076,1.9654608965,2.1079697609,-1.1303747892,-0.3680176139,-0.1541169137,-0.4737612605,0.2893137038,0.0942446738,-1.2750767469,-1.6875716448,0.0291447006,0.9104262590,-0.7338591218,0.1654359996,0.4833254218,0.1837635785,-0.0783622116,2.0477280617,0.4575521946,1.4642084837,1.8992501497,1.3101515770,-0.8264897466,1.1015996933,-1.8358174562,-0.5262581706,-1.3427133560,-1.8201973438,-1.3885704279,-0.7330588102,0.9829092622,-1.1346485615,-1.8879934549,-0.4256469011,-0.5990062952,2.0774192810,-0.2175528258,-1.3139890432,0.8544276953,-0.8505318165,-1.5247815847,1.0540881157,-0.7676452398,-0.0154721728,0.0645280629,1.1965137720,-0.7829078436,-0.7461743355,-0.9474745989,-0.4311519861,-0.1322558373,-1.4352858067,0.3022035360,0.8314113021,1.5980254412,-1.2497850657,-0.4789477289,-0.4042346478,0.9194238782,-1.9574488401,-1.5083035231,-1.0556405783,0.3304715455,0.4454304576,0.7204989195,-1.4876512289,2.3934466839,-0.3657112420,-0.6892605424,0.7944124937,1.1503497362,-1.1436711550,0.4530783594,0.1073921546,0.2563025057,-0.2650616765,-0.5858703256,0.3914135396,-0.0370595977,0.6986816525,-0.6059454679,0.7438039780,1.5383099318,1.5709109306,-0.0677266866,1.0556417704,-1.0941773653,-1.9463193417,-1.5928672552,0.7118583918,-0.1078235880,-0.1363062114,-1.1848173141,1.0763760805,-2.1852428913,-1.0170569420,1.0829125643,0.2023819089,0.3322030008,-0.8313293457,-1.2125116587,-0.7498506904,-0.6480713487,-1.0642518997,-0.0535658970,-1.5605756044,-0.3149810433,-0.6010142565,-0.4871916175,0.8242281079,1.8240424395,-0.0996669456,1.5988140106,-0.5990390778,0.0666736066,0.0582151599,-0.4938335717,0.1771529615,-1.6437077522,-0.5018564463,-0.3862938881,0.2370947152,0.4438247681,1.8506821394,-0.8399331570,1.2295398712,-0.0405258946,0.5014107227,0.3826455772,0.0491048954,-1.0085386038,-1.4686864614,0.6855700016,-1.0127818584,0.3318201005,1.3520195484,-1.1127836704,1.1357103586,-0.9928646684,0.8556365967,-1.0570096970,-1.6995784044,-0.9159128666,0.6399106979,-1.0972138643,0.7753602862,0.4758385718,2.3228681087,-0.0455329157,0.8702580333,2.2876014709,2.2667026520,1.0052275658,-0.5927371383,1.3670648336,-1.1437461376,0.3765378594,0.7944192290,-0.1012149379,1.0979666710,-0.3328878284,-0.5348561406,-0.0316795930,0.9100095034,0.7350540757,-0.7042039037,-1.0100349188,-2.6791682243,1.0962806940,0.8025962710,-0.0379705727,1.4441269636,-1.5085335970,0.0003205369,-0.0880573839,1.1053018570,0.1118759215,0.9201207161,3.1936144829,-0.5558418632,0.4877890944,0.0970641673,0.4611878693,0.2145508379,0.0961457565,-0.0012075664,0.4158501625,-0.0128498999,2.2061691284,0.3245992959,0.5149602294,-0.8647653461],[-0.4778534770,0.1767767519,1.1144613028,0.7067543268,0.2953458130,0.4151735902,1.3802677393,0.4514097869,1.4310191870,0.1890203953,-0.0607744493,-1.0145285130,0.4822328091,0.0662695393,0.5966459513,-0.3585332334,-0.4946506321,-0.5472767353,-0.0253793262,1.2802063227,-0.5363033414,0.4587896466,0.2496608347,-0.7562409639,0.6417131424,-0.3292468190,0.4829611182,0.0107420720,1.8334723711,-0.8969556689,-0.5865179896,1.4124860764,0.9335811734,0.6803567410,-0.7399356961,-0.9453975558,0.0589627735,0.9332634807,0.2295327932,1.2144051790,-0.9396538734,0.7404459715,-1.4058126211,0.3758881092,-0.5415832400,0.3609177470,0.4216291308,0.0672683716,0.4954783320,-0.3137744963,0.0770393088,-0.7609525323,0.6436778903,0.6758462191,-0.2501106262,-0.1796221882,-0.4029569924,0.2886023521,0.1287038326,1.7206531763,-0.2571421862,1.5516285896,0.1779292226,1.8721263409,0.4874676168,-0.3857848346,0.7131745219,-0.1883796006,-1.1051175594,1.0015405416,-0.0806301087,0.5015828013,-0.7649217844,1.2179169655,-0.6834611297,-0.3220885992,0.1348572373,0.3637213707,-0.0528043061,1.0957769156,0.2416266203,0.3221211135,-0.1729038656,0.4899542034,-1.3926836252,-0.8075534701,-0.4263211191,-0.3185393810,1.8184756041,-1.4812299013,0.4924119711,0.1793278903,-0.3062870204,0.7527695894,0.6813973188,-0.4136724770,0.1930301487,2.4636559486,1.0544172525,0.5969682932,0.3538553417,-0.7953445315,0.4444134831,-0.9988148808,0.2708710730,-0.2231728882,-1.5424858332,0.0671981946,0.7165861130,-1.7390002012,0.1175277680,0.5987126827,1.2449854612,-0.4765720367,0.9063929915,0.5871992111,-0.2316968739,-0.4770857990,-2.0407674313,-0.8889138103,-0.4778654575,1.0535147190,-0.8856458068,-0.0943486169,-1.9650886059,1.5634150505,0.9272034764,0.8704501390,-1.2399034500,-0.0045439149,0.0171208791,1.5940405130,0.5065575838,-1.0628005266,0.6307820082,-0.0456372090,-1.1207317114,0.0359663069,-0.6097961068,0.2768594623,-0.9132888913,-0.8086346388,-0.5995755196,1.0584326982,-1.3386423588,-0.0910859630,1.2630821466,-1.1991431713,-1.0497963428,0.6557671428,-2.8756728172,0.0049800058,1.1699956656,1.7030316591,0.5046100020,-2.2967424393,-0.4492062628,-0.4919267297,-0.8143878579,-0.7950159311,-0.5516642928,-0.1837606877,2.1352300644,1.1462190151,2.0455040932,-0.5048813820,0.8189655542,0.1658494473,0.3154302835,1.2556283474,0.7457754612,0.3385994136,-1.6795654297,-0.6067926884,1.2469490767,0.5089952946,-0.0530367568,0.5155069828,-0.6019174457,-0.1175310239,0.3800863326,1.4116129875,0.4822998345,0.6734839082,-0.7265940905,0.9609754682,-1.0695693493,1.2773134708,-0.0383228734,-1.1684441566,0.6060315967,1.6832801104,0.7533211708,1.5771530867,0.4684437513,-0.2532663345,1.5824003220,-2.6677989960,0.7695742846,0.4340322316,0.6323025227,-1.0616002083,0.6552160978,-2.0156300068,-0.2300008535,0.6324152350,-0.8441838026,-0.7557621598,-0.1748217940,-0.6759279966,-0.8832311034,2.3019340038,-2.3870930672,0.0982676297,0.1205674782,1.3445794582,-0.2255969346,0.6980847716,-0.4308630526,0.8111328483,0.9373266101,-1.6052649021,1.4621964693,-0.2540515065],[0.8156651855,1.0076911449,-1.7301025391,-1.6204510927,0.7440069318,0.0341309048,1.5476413965,-0.8900391459,-0.3766621947,-0.6074256301,0.3939062655,-0.5218016505,1.2918245792,-1.3143782616,1.5072596073,1.0231168270,-0.2359389961,-1.1725305319,-0.9327100515,0.6893956065,-0.2224030793,0.1603256017,-0.4459508359,-0.1136763841,-0.1959608495,0.7789053321,1.0510292053,-0.0700007603,0.5266662836,-0.9966610074,-0.9432050586,0.7437862754,1.3008739948,0.3434124291,0.0258266907,0.0869140998,0.6960690022,0.7120946646,0.0515276678,-0.0726527721,-0.3411781490,-1.3804582357,-0.2881645858,0.5035392642,-1.3463668823,-0.6638413668,-0.5775796175,-0.5389131308,0.4343502820,-0.2203941643,1.6989192963,0.5641354322,1.6838576794,1.2410887480,0.1104821563,1.7673959732,-0.8824916482,1.3487502337,0.5774065256,-2.2027637959,0.3938376009,0.2434094995,-0.4469590783,-0.1877329051,-1.0865863562,-0.6238707304,0.1701171249,1.0134552717,-2.2195823193,-0.5561084747,0.0621400401,0.2244819254,1.0336713791,-1.3963009119,-0.2985310555,0.3228592873,1.3612934351,-1.4880374670,-1.6463325024,0.4461197257,-0.3382137716,-0.4326903224,1.2684216499,1.2643234730,1.0895330906,1.2925497293,0.1350784600,-0.4523508847,-0.8258852959,-1.7149457932,0.6874905229,-0.7296146750,1.2389507294,0.9146369100,1.3814866543,-0.0028754754,2.3910377026,0.1634847671,1.5939153433,0.1600411683,0.3318611383,0.4221729338,-0.9187315702,0.3347060084,-0.4368286729,-1.2759553194,1.1613256931,0.4264028370,-0.2972605228,-0.0180641264,-0.6553592682,-0.5903983712,-0.5169649124,-0.4399635792,0.0741655752,-1.2233260870,0.2017238438,0.9079591632,1.0736171007,-0.4128850102,-1.1717312336,0.6035646200,1.5451973677,0.6227061152,1.2984673977,1.6092420816,-0.9485616088,1.8946329355,1.2957694530,1.0307075977,1.4255412817,1.7190335989,-1.3016798496,-0.9480474591,1.1068694592,-0.8514854312,0.3590765297,0.4884167910,-0.2413618565,-0.2864337265,1.9658122063,-1.2297976017,-1.3506499529,-1.0592341423,-1.2962996960,0.6311379671,2.3245151043,-0.8188385367,0.6870383620,0.0071533509,0.4189999998,0.5118589997,0.9282453060,0.1976849586,-0.2832446694,0.4292204082,-0.1534999758,1.5900549889,-0.2832624912,0.9434610009,0.7822171450,0.7615031600,-0.3012326658,-0.1083905846,-0.3890056610,-0.5637946725,0.0940228626,-0.1532761008,0.4407454431,1.1213968992,-0.7484930754,0.4968990684,-0.8124283552,0.8374453187,-0.5514011383,0.7271595001,1.1901440620,0.4425831437,0.7223262191,0.0920367613,0.1827518195,-0.9553514719,2.2778782845,-0.0951565281,0.6599346399,-0.4985764623,-1.2338274717,-1.0685179234,0.5681238174,0.4908271134,0.2003903836,-0.1043929160,-0.6512681842,-0.0636266619,0.0732113644,0.4932597578,-0.9207543135,-0.1076298058,-1.6838189363,-0.2428099662,0.7228756547,-0.4640580118,0.6245583892,-0.7864403129,0.9854980707,0.1483794004,-0.1238286272,-1.9773298502,-0.6761142612,-2.0313007832,-2.1170983315,-1.5734508038,-0.8779585361,-0.0909841582,1.4897482395,0.6401346922,-0.4880248904,-0.4623247981,-0.2730410695,0.2345179766,-0.2584047318,-1.0733059645,-0.6118106246,1.8113634586],[2.0679802895,-0.9476329684,0.7383434176,0.0674313679,-1.6945581436,0.3529974222,-1.2724621296,1.0977891684,-0.8179125190,-2.8983523846,-1.2201880217,-0.1745642573,0.0033349809,0.2662228346,0.7162650824,-0.4651221633,0.5385454893,1.1434221268,-0.4105019867,-1.6019604206,0.6570641398,0.3827970028,-0.8969772458,-0.6783589125,0.4421870112,0.3902520239,1.2652155161,0.8713499904,-0.1916929632,-0.1945413202,-0.4036102295,-0.0592381246,-0.7947420478,2.4768147469,-0.5596383810,0.0513080209,2.2183132172,0.7428569198,-1.1224207878,1.2905070782,1.6988288164,-0.0087889144,-0.3099016845,-1.0261619091,1.2815786600,-0.3887642324,-1.8203357458,0.6565842628,0.8482696414,-0.3657097220,2.5571284294,0.9646287560,0.5014576912,-0.7785434723,-1.0838786364,-1.0007174015,-0.6832125783,0.6869196296,-0.8909994364,0.7105440497,-1.5718376637,-1.0136063099,-0.4651289284,-0.9581088424,-0.4869917929,0.8937326074,2.2701437473,-0.5487756133,-1.2599574327,0.2497866601,-1.0012441874,-1.0906326771,-0.0996188372,-1.5389875174,-0.3022162914,2.1879057884,-0.7253212333,1.0624971390,-1.2916557789,-0.3524868786,-0.9313794374,-0.7848801613,1.5815815926,0.1106806770,0.4807272553,0.6900759339,1.2145121098,1.1435748339,-0.7682368159,-0.1096613705,0.6715409160,0.0713482127,0.2610453367,-0.2741608322,-0.2731364667,-0.6862300038,-0.4789772928,-1.3881918192,-0.6710447669,-0.2162043303,1.2445957661,1.1760710478,-0.5608954430,0.2300017178,-0.5237638950,-1.4979652166,-0.4840055704,-1.0160231590,0.0633943900,0.2442258596,-0.1619214714,-0.1240627095,0.2674188316,2.1347377300,0.7680336833,0.7131571770,-2.2288031578,0.3683202863,1.3970509768,0.2402867228,-0.3467687964,1.0369762182,0.9858167768,1.7241109610,0.3450718224,-0.3579460382,-1.4150452614,0.3929713666,0.4741328359,0.2615441382,1.3558253050,-0.1826655865,0.7189281583,1.1006554365,-1.2778162956,-0.6737187505,0.2032980323,-2.3750617504,0.5275279284,0.0699259490,0.0522095188,-0.0179084744,0.7750706077,0.3565326333,0.2300423533,-1.4687960148,-0.0426535085,-3.7455356121,-0.7975733876,-0.2791737318,1.7593774796,0.0835426897,0.8876459002,1.1769098043,-0.8923800588,0.6272987127,-0.8653988838,0.5507037640,-0.1309112310,-0.7270892262,-0.5536189675,0.1476335227,0.9601663947,0.4076004028,-0.8485045433,-0.1765632182,0.2317116857,0.0372135900,1.3644959927,1.3756741285,-0.2507399023,-1.8323187828,1.4180147648,0.3964681327,0.2041271180,-0.3245566487,-0.4420765638,-1.3968116045,-1.3434318304,-0.9108263850,0.6881069541,0.6643660665,0.2395091802,-1.3126059771,0.9548830390,-0.2024228126,-1.4638279676,-0.4380912483,0.1316522658,-1.5384185314,0.2050340176,0.6454075575,-0.9426968098,-0.5764552951,0.6355412602,-1.1904445887,-0.0340364538,-0.8278380632,0.5902572274,0.3412825763,-0.1369599551,-0.0623027273,1.5578794479,0.7105544806,1.0725549459,-1.9698865414,0.8213605285,-1.2326803207,-0.4292006195,-2.2102420330,0.7281572819,0.3447653949,-1.3590006828,0.3094686866,0.2325506210,0.1171084866,-0.0100068245,-2.9082498550,-0.0394074060,-0.7112797499,1.0732913017,1.0485405922,0.8922514915,0.0037649970],[0.3218860924,-0.2153628469,0.1521495283,-0.9304068685,-3.3970711231,-0.5037402511,1.1651905775,0.7021167278,-0.7433487773,0.3981168568,0.9380275011,-1.5853767395,0.0831920207,0.0841076896,-1.0592501163,-0.9714565873,-0.7586913109,-0.6948418021,-0.4219135642,-0.8064761758,0.7126872540,0.4738564491,0.3092348576,1.3096922636,2.0170652866,-0.8423181176,-0.0324107222,0.6601939201,-0.5238676667,1.1682136059,1.5572181940,-0.7529825568,1.6637852192,-0.4154839218,-1.3779836893,-0.1081061214,-1.5806005001,1.8208845854,-0.3819041550,0.5165681243,1.8766587973,0.8654612899,-0.9601491094,-0.3456733525,-0.6894811392,-0.7221453786,-0.6066665053,1.0309771299,0.8200556040,-0.7337735891,-0.1614723206,0.9861069918,1.4574701786,-2.1444618702,0.3525089025,1.7056667805,-0.9310776591,0.6707212329,-1.3782510757,1.1024730206,0.8600340486,-0.2841083705,-0.4831273854,-0.1850067079,0.1130082682,0.2305518389,0.3413489461,-1.0034881830,-1.0885648727,-0.9350720048,0.1794212759,0.4361413419,-1.8060265779,-0.1309453547,1.4649746418,-1.3094321489,-0.3479111791,-0.8606664538,0.3332029581,3.0923016071,1.3583788872,0.0134683866,-0.8860229850,0.2574064434,1.1946133375,-1.1781209707,1.3553206921,-0.6204507351,-0.6351287365,-0.0614457428,-0.5441697240,-1.1895301342,-0.1810240746,-1.1449270248,-0.1968882829,0.5040771961,-0.4403439760,-0.4055637121,-0.4553186297,-0.9085854888,0.0759990811,-0.1860614270,-0.2720640302,-0.1599345505,0.3134514391,0.4481948316,1.9346771240,-2.1949660778,0.8848531246,0.6827428937,-0.9667618275,0.1164045185,0.3335077763,-0.9087823629,1.8471758366,-0.6295967698,-0.8704705238,-1.4105505943,-0.7595064640,-1.2473758459,-0.3623605371,1.0579477549,0.0246553812,0.5469079018,1.0893458128,0.4062927067,-0.3033017218,-1.2926683426,1.0524569750,0.2503056228,-0.3500367701,0.6117320657,-0.6270340681,-0.2911286056,-1.0969469547,1.0318131447,-2.5585491657,1.3501057625,-0.2241195887,-0.4697246253,-0.5922012925,0.3770401776,-0.1585283130,0.8999029994,0.2463238984,0.3050529659,0.0677323788,-0.6213287711,0.5435842276,-0.3985887468,-0.9317046404,-1.2294517756,0.0828140751,1.2535772324,0.0256637987,2.8696305752,1.4507285357,-0.4330395758,0.8850038052,0.4619824290,0.5875666738,-0.3392009139,-2.4040269852,-1.2309052944,0.8450149894,1.1948267221,-0.4159831703,-1.7273284197,-0.4782824218,-0.0359106883,-0.4035047591,-0.1818247736,-0.6230558157,-1.1475069523,0.4307565391,-0.7822400331,-0.4539892375,-0.5645713806,0.7916404605,-1.2734173536,1.1932449341,-0.3262642026,0.1361260116,0.4524342418,1.8424292803,0.2679949403,-0.7157343030,0.9898098707,0.4705828428,-0.5157236457,-0.0149328671,-0.3967804015,0.2779393494,-0.6822986007,0.1624890715,-2.0525116920,0.0262571834,0.2762793601,0.4157950878,-1.2589560747,-1.0236248970,-0.8549474478,0.4833740294,1.2150045633,0.3890248537,-0.0069801174,0.9563995004,0.1099641994,-0.3607292473,-1.0780054331,-0.1775603443,-1.5771030188,2.8553757668,-0.6350323558,1.4035706520,-1.5484172106,0.4918170571,-1.0095106363,2.1850802898,0.9022112489,0.4329253137,-2.1208972931,0.9020392895,0.5670707226],[0.2932744026,-1.7365777493,0.3774078190,0.7109208107,-0.1204144135,-1.3939466476,-1.0072985888,-0.1054091528,0.6280057430,-2.4682083130,0.2006152719,-0.9499309659,-0.8038668036,0.5076775551,-0.9942888618,1.0489151478,0.1733068526,0.0147161372,-1.7972850800,-0.1931378543,-0.4037202895,-0.7168515921,-0.3700949550,0.2424467951,1.3536716700,0.5044452548,-1.1023502350,0.1263127476,0.1913612634,0.4485045075,-0.0773018971,2.0170025826,-1.0194779634,0.3922486901,2.9099795818,-0.1763591021,-0.5611923933,0.2001597732,0.4984797239,-0.4362840652,-1.5781536102,-0.0572126582,-1.6782734394,0.1083617732,-0.4752859175,1.9072747231,2.0408916473,-0.8214539289,-1.2110806704,-0.0955808163,-0.7157170177,-1.5102883577,1.8471789360,-0.6968463063,0.5167600513,1.8500298262,-0.6309816837,-0.0008967539,0.6174471974,0.2788871229,-0.8685209155,-0.1644606888,-0.6464869976,-2.1174771786,-1.7138003111,-0.1598047316,-0.3160598576,0.6028695107,-2.0008893013,0.5322968960,1.4469785690,0.6468976140,1.1313585043,-1.0836691856,-1.6371321678,0.1979190111,-0.4971550107,0.3789435327,1.2085425854,2.3498613834,0.2326178402,0.6639389992,-0.2816003561,-1.1688593626,-0.7957587242,2.1797263622,-1.2281219959,-1.0931417942,2.4704101086,0.3591396511,0.3633414209,1.7679173946,-1.5819662809,0.5613010526,0.8031804562,-0.9179223776,0.3878885508,-0.1153258309,0.2231075317,-0.1290087104,0.7834047079,-0.2219590694,-0.0516575351,0.7322729826,-0.6734268069,1.3610804081,0.7266793847,0.7462473512,-0.9291234016,2.1887683868,-0.9468863010,-0.3495929539,-0.0428985730,1.3444181681,-0.0642006993,-0.4990572035,0.9972393513,-0.1914417893,1.2694454193,-0.6293994188,-0.1585151702,-0.2537268102,-0.0725091919,0.7362402081,1.1210161448,-0.4297934473,-2.9094355106,-0.1323205382,-0.2460999191,-0.7158421278,2.1082756519,1.1742449999,0.9783309102,-0.2963083088,-1.1627272367,-0.0309837069,0.8325194120,0.3749943376,1.0111228228,0.1109903753,0.0558381490,0.2111046016,0.0354266241,0.9828637242,0.5850839615,0.0928330123,-0.9776461124,-0.8418869376,0.5141755939,-0.1413520724,1.1167169809,0.8080258369,0.8365648985,-0.6641654968,-1.2707079649,0.1409545690,0.9196810126,-1.2050665617,-0.6511962414,0.4858438671,0.5146605372,1.5368059874,-1.4482113123,0.7256953716,-0.4815798998,-0.4271739125,-0.7080703378,-0.7192265391,0.7303133011,-1.1464543343,-0.0006264384,1.9445780516,0.0367070660,-1.9715096951,-0.9793316126,0.9981327653,0.4227823615,-0.3801508546,0.8276200891,-0.3732515275,-0.5242275596,0.7299136519,0.9283463955,-0.9516673088,0.0363494493,-0.4172412455,0.6668561697,0.7712953687,1.2420126200,-1.0641741753,1.0731617212,-0.3424093425,0.0912520811,0.8670433164,-0.2390884310,-1.2840259075,-1.0081032515,-0.4506661594,0.0318231210,0.9985192418,-0.4479965866,1.9789948463,1.1024099588,-0.8284841776,0.9265839458,0.8247851133,1.1524419785,0.6611723304,0.8174999952,-0.0465653911,0.7161703110,-1.5024360418,-0.6508199573,0.2619676888,0.8435713053,-0.8028890491,0.7184874415,-0.9584380388,-0.5889627934,1.1051890850,1.0269550085,1.1142108440,0.8566098809,-0.1889745146],[2.1708762646,0.3211729527,-0.7073199153,0.2713794112,2.1762907505,-1.6629705429,-1.5230276585,0.2944081128,-1.8447911739,-0.4856126010,0.1250060797,-2.6290686131,-0.7232851386,0.4715391695,1.0080659389,0.0524155572,0.4955901802,0.0261317994,-0.4716481268,-0.0053549618,0.9015104771,-1.0712935925,-0.8792697787,0.2951650620,-0.7993769050,-0.7291873693,0.1309042275,0.8772374988,-0.8512756228,0.6310116649,0.2843870819,-0.6464396119,0.4193823040,-2.7665982246,-0.6511185765,-0.6155664325,-1.3428583145,-0.3334512115,-0.3145128787,-0.3880962133,-0.0138087487,-0.0732112080,0.8911177516,1.0543974638,-1.6987104416,-0.9329395294,0.6918064356,0.4133537114,0.9851148129,0.5021405220,-0.4187388122,-0.3045681715,1.2685724497,1.5336577892,0.4741467834,-0.4501512349,0.4564183354,0.3882172406,0.5919806957,-0.4965659082,0.3365334868,1.6907614470,-0.7521701455,1.6877909899,0.6436438560,1.6518549919,1.0253766775,0.9109950662,0.7273064256,0.5286173820,1.1155961752,1.8448877335,-1.5188112259,-1.7974079847,-0.9361957312,-0.5099250078,-0.1010021567,0.8071030974,0.1876153201,-0.0064344634,-0.0497765839,-0.5475576520,-1.0387189388,-0.3532833457,-0.5067877173,-0.9035395980,0.7844749689,0.9171721339,0.8255385160,1.1045073271,0.9061017632,0.8320184350,-0.3664151728,-0.6954748034,-0.3559719622,0.1738903522,0.9105868340,0.2641392946,0.5053906441,0.4349961579,0.8912056684,-0.1841396093,-0.7150786519,-1.3447219133,-0.4978642464,-0.0933635756,-1.1518702507,-0.6728532314,-1.6219307184,0.3163990974,-0.7288520932,-0.5077275038,0.3367995322,-0.7175356150,0.4415037036,0.7889863849,0.6716121435,0.9814752936,1.2690737247,-0.6693392992,-1.1873300076,1.4108334780,0.3188565969,-0.2024255991,0.7412149310,-0.4959668815,0.0802198648,-0.7489406466,-0.1091844365,0.9267287254,0.0638568923,1.1230969429,-0.2486508340,-0.2947012782,0.2912669778,-0.3035437167,1.0478081703,0.9640623331,0.7009201646,-0.8394269943,-0.4298658669,-0.0208310578,-0.9664135575,-0.4727172256,0.3689211309,0.8614180684,0.8856139779,-1.0648728609,-0.4798837602,1.6359280348,-1.5494107008,-0.2393066734,1.6723638773,1.0869294405,-0.1503750980,-1.3417633772,0.3851701319,0.8203778267,0.2636116445,1.2380123138,-0.2437708676,-1.6408116817,0.9459151030,-2.6160981655,0.1274162382,-0.8868399262,1.2920694351,1.2345162630,-0.1554797590,-1.2826472521,0.3236345053,-1.3748104572,0.1620595008,-0.5028721690,0.7731192112,0.0074709491,1.0263241529,0.3440847397,0.4407630861,0.2580718100,0.4818874002,-0.9904689193,0.3456379175,-0.7643799186,-0.5512456894,0.6247019768,-0.0195069723,-2.3036928177,0.4286721349,0.3994229734,-0.3317076266,0.9082452059,1.3464412689,0.8724412322,-0.2232536077,-0.9121289849,-1.0593949556,-0.9844374061,0.4946773946,-1.2179546356,0.8431268334,-1.0629291534,0.9424785376,0.6598713994,-0.5156948566,1.2651425600,0.6092521548,-0.9670048356,0.5924138427,0.8206498027,-0.1055323258,-1.3936834335,-0.2509214580,0.4976290762,0.7790489793,-0.6068552732,0.7373483777,-1.0231674910,0.7975035310,-1.8134320974,-1.6039186716,0.0184440892,-1.5174918175,0.1511792094],[2.4891557693,-0.7333879471,0.5969586968,1.1847419739,1.4520896673,-0.1143778041,-0.4433355927,1.7384566069,-0.3925912380,-0.0808646306,-0.4252757728,0.8303156495,-0.0310105458,-0.4736564159,0.0218356308,-0.5272080898,-0.3808960319,1.7675691843,1.5582160950,-1.0983140469,0.5440816879,1.2187278271,-0.4388285279,-0.2428838164,-0.4820106030,1.0899262428,-1.1954586506,0.8806062937,-0.0880520642,0.1150913313,-1.0539691448,0.6986237168,1.8885309696,0.9215537906,1.8435418606,-2.1448302269,0.3851986229,-0.7131193280,-0.3901919425,-1.3624395132,1.5063632727,-0.4614166915,-0.3000406027,-0.1013559550,0.0810948461,1.5957373381,-0.5460469723,-1.3816664219,0.5379842520,-0.6066759229,2.2901933193,-0.7612150311,0.5551570654,-0.9203477502,0.7038357258,-0.6596065164,-1.1327278614,-0.7435246110,1.0183490515,0.6444005966,-0.6632887721,0.1449518502,-0.9932124615,-0.7061646581,0.6948489547,1.6899902821,-0.3036556840,-0.7680572867,0.4792656898,0.4494149387,1.9874323606,0.1310714483,0.5772669911,0.4271465540,-0.3796473444,-0.5137271881,1.3119901419,-0.6844062209,-0.5254566669,0.0645294040,-0.2315711081,-0.9539589286,-1.5046663284,-0.3089866936,1.3342427015,-0.7204396725,0.3384650350,0.0651963726,1.0387514830,-2.1739177704,-0.7222473621,-1.8249527216,0.5022072792,0.0201506056,-2.2221145630,-1.7429068089,1.2421858311,-0.2116023302,-0.7000771761,-1.1120636463,0.2420840859,0.5119414926,-0.9397857189,-1.3358485699,2.8444836140,-0.9619747400,-0.8597752452,-0.3271184266,-0.5138223767,0.0424317084,-0.0836657956,1.0398344994,0.1050056592,0.7023622990,0.0821461901,0.9920848012,0.6409959793,0.3868029118,0.1679462940,0.4380161464,-0.8118830323,-1.0857101679,-0.8440061808,0.8092541099,0.1587680727,-0.5558717847,-0.2662746608,-0.7828058600,-1.8595489264,0.6069732308,-1.0864323378,0.0739274174,-3.2514317036,1.3766409159,-0.4158675373,-0.6441162229,0.5918098688,-0.3352716565,-0.6861219406,0.6173312068,0.8340231776,2.1576929092,0.7559766769,-1.9395177364,-1.5068037510,-1.6640449762,-1.0494740009,0.3865607977,-0.8776586056,-0.3444928527,2.2688899040,0.1278932095,1.4168598652,0.5408000350,-0.4373477995,0.7047700286,1.6746137142,0.0112737603,-0.4115543067,0.4049486816,0.3209570348,-0.0363627002,0.0535618663,0.9652701616,-1.0335603952,-0.8917180896,-1.0212895870,-1.2253913879,1.3610954285,0.5986154079,1.2577533722,-1.1704236269,-2.7986218929,-0.2227247208,0.6540288329,-0.5734651685,-1.3834633827,-1.4270046949,-0.7254546285,-0.6367143393,1.8409622908,0.0058467253,1.4262659550,0.1178915799,0.5660122037,0.9328692555,-0.3919641376,-1.2900167704,0.5978199840,1.0307321548,-0.6817595959,1.5728329420,-0.9726041555,-0.3166731596,1.4570447206,-2.8043234348,-0.1258659363,0.1362825036,-0.2868614495,-0.8499239087,0.7297567129,2.0664722919,-1.0674616098,-0.0646936595,-0.4705346525,0.8866371512,0.0606195629,-0.2033787519,-1.0553119183,0.2734993100,-0.1094169468,0.2509588599,-1.0711268187,0.2432459742,0.7302077413,2.0591175556,0.9453964829,0.7122648954,0.1843675375,-0.8156602979,-0.9698758125,-0.3812840283,0.5082962513,-0.6116515398],[-1.4949227571,-0.8798989654,0.1354723424,-0.3616631031,0.4147650898,-0.3228835464,-0.5915602446,1.3671255112,-0.5830543637,1.1036515236,0.0076271435,-0.8378448486,-0.9426112771,1.0647289753,1.1454114914,0.6047657728,-0.5769218802,-0.3423543870,-0.0147391697,-1.2193109989,0.8734396100,-0.5175681114,-0.8919221759,0.6767447591,-0.3358008564,-0.5971326232,1.1393744946,0.1059352309,-1.0928663015,-0.0023749452,-0.8992219567,0.5955408216,-0.1915006489,0.9519176483,-1.6528651714,0.4768061042,0.2691729665,1.0863753557,1.7779698372,1.6851940155,-0.5937855244,-1.2471388578,-1.4843024015,-1.6690483093,0.8079186678,0.6009854674,-0.1532800347,-0.4540135264,0.8403998017,0.0214106571,-0.6578392386,1.5833458900,0.3572075069,0.0490770936,0.3193014264,-0.0974152535,-0.2502352893,0.7973176241,-0.7291021347,0.0530070215,1.3097430468,0.3382050693,-0.4680342078,0.5104855895,0.3980289996,2.7617027760,2.8023548126,-0.9081757069,0.0257167872,-0.6720604300,-1.0278261900,0.5908465981,0.1360614896,0.0536954999,-0.6864715219,0.2259481102,-1.7827547789,-0.4555638433,0.4290686250,1.1622430086,-0.4761041105,-0.1950169206,0.2584655583,0.9476106167,-1.2855743170,-0.1756140739,-0.1077017039,0.9727046490,2.2783825397,-0.7615939975,-0.0914252028,-0.4492025077,-0.1590934247,0.3942250311,-1.8457733393,2.9138922691,-0.6026506424,-0.6460301280,-1.1496130228,0.2545246184,-0.1576082557,0.4011459947,-1.0190671682,-1.5612306595,-0.8178557158,0.1179975867,-0.1237922013,-0.1505875736,-1.5492738485,0.5640113950,0.4910785556,-1.4756758213,-0.0605845712,-0.7968961596,-0.3090079427,-1.4476984739,-0.0446362980,-0.7001911998,0.9888365865,0.0728237554,0.5284053087,0.1921956390,-0.0171768777,0.6343825459,1.2951797247,0.6299999356,0.5879244208,0.2705804408,-1.6909981966,2.4678122997,-2.3150990009,0.7861059308,1.1524168253,0.0536516793,-0.7031561136,2.6576049328,-1.3356398344,0.4453574717,-0.0873836875,0.8940993547,1.7681417465,-0.9471057057,0.7181583047,-0.9346099496,-0.8268125057,0.8860337734,-1.6342958212,-0.9823036790,0.2693607807,1.3396952152,0.0485697836,-0.3400638998,0.7500931025,0.3225327730,2.6488277912,0.3102334738,0.8130268455,-1.1668528318,0.9298003316,0.3419047296,0.3981050849,-0.7024795413,-0.0838392451,2.1503913403,0.3166868687,-0.4655977488,-1.0422612429,0.5906660557,-0.3481139243,-0.0162188802,-0.1142532825,-1.6149265766,0.0337558053,-0.8138551116,-1.9581266642,1.8291319609,-1.5578610897,2.1736629009,-0.7472221255,-1.7286211252,0.7932048440,-0.5200570226,0.1771980226,0.7093726993,1.9725366831,1.6122074127,1.6713001728,-0.2878654003,0.9476270676,-1.9848715067,0.6367865801,-0.8831424713,-1.0610102415,0.6461561322,-0.9139950275,0.2092022151,1.7103219032,-0.3820849359,-1.5692948103,-1.8255572319,0.3962948620,-0.1347055435,-1.3692386150,-1.3420454264,-0.0727353171,0.6803857684,0.2496832758,-0.7072216868,1.5534228086,0.5460235476,-0.1979304999,0.4633287787,-0.3761905730,-1.1127810478,-0.2559715807,0.0235608853,-0.6353579164,-0.6384484768,0.6815594435,-0.4563060701,1.2554326057,-0.4534022510,-1.7506196499,-0.5014964938],[-0.2295739204,-0.1924448907,-0.4733704925,-0.2241870314,-0.8210021853,0.5572533607,0.2077593505,1.3226039410,0.8503068686,0.3388375938,-0.5830029249,2.0334277153,1.0397149324,1.2391461134,1.0362479687,-0.5698149204,-0.6259616017,-0.2362915277,-0.3661133647,-0.8876191974,-1.6699805260,1.5485368967,0.0311318263,-0.7217914462,-1.2424839735,-0.0392079912,0.3139655590,-1.0111051798,0.7933912873,0.3894520998,0.2327877879,0.6048811674,0.8615930080,1.2502133846,0.3906288147,-0.5203686357,-0.1070114747,0.8577687740,-0.7629805207,-1.5210009813,1.3381309509,-1.6148548126,-0.2387400120,-0.1157045662,0.1873877794,-0.9930270314,-1.7484232187,-0.9100748301,-0.2736967206,-0.8384456635,0.0644456372,-0.6747201681,1.2409303188,-1.3965411186,0.5551806688,-0.2347596139,-1.5017775297,-0.4084243178,-0.6948349476,0.6206647754,1.0374354124,0.8068956733,-0.9258612990,-0.3243401945,0.3644148707,-1.1015172005,0.7313768268,0.8892873526,0.1480387300,-1.0648075342,-1.4160317183,-0.9014066458,-0.4222130179,0.2445816100,-0.5626324415,-0.4851743877,-0.3403742909,2.9591856003,-0.0881588683,0.5897176266,-0.2818697095,-0.4818490148,-0.0522636548,-0.7201578021,0.8085952997,0.7062283158,1.3574514389,3.1008806229,2.4156870842,1.3525607586,-2.0091259480,-0.0851096436,0.9077156186,-1.2800422907,-0.8717706203,-0.0972527489,-0.5018403530,-0.3444746137,0.7191011310,0.1668957919,0.5495115519,0.8817923069,-0.9869719744,-0.4999747574,0.4192822874,2.0949213505,1.0182723999,-0.9415771961,2.1363947392,-0.1266851574,-0.1686273217,0.1555329263,0.2692514956,0.0107193282,0.6864767671,-0.3113112748,-0.6079178452,1.0384211540,0.3583306968,-0.0989136025,-0.1000434235,0.1119473800,0.9623701572,-1.6869838238,-1.7702426910,-0.5314478874,-0.1993282437,-0.2009946853,1.0972038507,0.1889088154,0.7660380602,0.6959185600,-0.0903240070,0.4758541286,0.4996165335,0.3103198409,-0.2523146272,0.0269732401,0.7140724659,-1.4831380844,1.7946801186,-0.2277995795,1.5964488983,1.7150933743,-0.1319871843,-0.3674093485,0.7914146185,-0.3072839379,-0.8771335483,1.3631716967,0.6096215248,0.0368815847,1.0796775818,-1.5741424561,1.8392524719,-0.1673394144,-0.0709104463,1.3131312132,0.1592441201,0.7896984220,-1.4784001112,-0.2822154462,1.2148447037,-0.0967611820,-2.2488405704,-0.1258734167,0.1380659342,-0.7507057786,-1.7866828442,-0.1211231351,0.4181524813,0.3321387172,-0.1422798783,0.5991910100,1.8145097494,-0.2987695634,0.1185233369,1.7562841177,-0.0889143497,-0.8222903013,-1.4978560209,1.5606752634,-0.7209838033,0.9876718521,0.7995440364,0.3725219369,-0.1506541669,-1.4730818272,2.0495085716,-0.4954752922,1.1434779167,-0.5692482591,-1.4061460495,0.6421436071,-1.8218369484,-1.2275563478,-1.0374466181,1.6718900204,0.3865244985,-0.7263987660,0.5036387444,-0.1251687109,0.0561733358,-0.0534702614,1.3122061491,0.3336322904,0.4826311469,1.7704149485,0.1591441482,1.5266855955,1.2783801556,-0.6986181140,1.0752340555,0.4369335771,-0.4910381734,-1.2765337229,-1.5475559235,0.1101098582,0.4548444748,0.4179526865,3.5148334503,1.3793034554,1.2272132635,-0.7775411010],[0.1388985366,-0.9853470922,-2.1588702202,1.4759497643,1.0479900837,-0.2329456359,0.4264881909,1.6505819559,-0.6207741499,-1.3017786741,0.8918532729,-1.8384761810,1.6479315758,1.4943436384,-0.2758189738,-0.4083052278,-1.7543193102,-0.8631698489,-0.3464847505,-1.0863679647,-1.0439672470,0.7048197985,-0.6013585925,-0.5195314288,-0.4424979389,-1.3421670198,-0.1133228242,-0.1815241724,-0.4386858344,1.2019025087,-0.5518383980,1.4825443029,-0.1924960464,-1.3046330214,0.8492895961,0.3339430988,-0.4778641462,-0.7398019433,-0.1873675436,0.9931180477,0.1758742630,1.1829441786,0.5680868030,-1.5838301182,0.1504982114,0.6607076526,-0.0231101718,-0.1185555682,-0.1340669245,-1.0390249491,1.7121312618,-0.0431025848,-0.3924097121,0.5211667418,-0.8721234202,1.0714064837,-0.2642340660,-0.4705439508,1.5070914030,0.0275213439,-0.8787099719,0.2398306876,-0.9827113152,1.2846596241,1.2042113543,-0.2682869136,-0.0602881312,-0.1187253222,0.1935069561,0.6068314314,-0.9962551594,0.1362517178,-0.5295902491,-1.4795718193,-0.3208876252,-0.0433504656,1.3398160934,-0.5696412325,-0.1615510434,2.2393598557,-0.4070459902,1.1034712791,0.8850687742,0.4269449711,-0.0309651364,-0.3605507016,0.7600423694,-0.7894233465,-0.1974647790,1.5508267879,-0.8302157521,-0.3430190384,0.0971933901,0.7561179996,0.1921995878,0.6613284945,-1.3063372374,-1.8925228119,-0.5819588304,0.0686352998,0.7780308723,-0.1821022481,-0.2784351110,0.9828819633,-0.3580800891,-1.4925870895,1.1827394962,-1.2048211098,0.0015660966,-0.7056044340,0.3516597152,-1.3311002254,0.1076234132,-0.9575064182,-0.5319263339,2.2485411167,-0.8890992403,-1.5095508099,-1.4435843229,1.2062375546,-1.0373508930,-0.9600437880,-0.2674483955,-0.8744071722,-0.8395624161,-0.2602224052,-0.0716149807,0.1754660159,-0.0509066172,-1.3418159485,0.9362290502,2.0501604080,-0.8092800975,-0.2566420734,-0.9628132582,0.4812192321,-1.7650332451,1.6551828384,0.2969235182,1.9384123087,-0.7422243357,-0.8657963872,1.9764961004,0.2587822676,-1.2811017036,0.8559010625,-0.2178717405,0.5513018370,0.6288032532,0.3456377685,0.0518872105,-0.2260817885,1.3474808931,0.4960836172,1.1231657267,-0.2785956562,0.1864937544,-0.9686026573,-0.4685764611,0.0343641862,0.8239734769,-0.0382086560,0.8648043871,-1.5192743540,-0.3801605999,-0.7757578492,0.6177380085,1.1645232439,-0.3902976811,-0.4834443033,-0.5099754930,1.1100440025,1.9989259243,0.0120947240,0.7738932967,-0.1626637578,-0.2748828530,0.1500775963,-0.1423937976,0.7558955550,-0.2406939566,-0.8073249459,-0.8291538358,0.5597535968,0.6038845181,-0.4026309848,0.6417809129,-1.0559430122,1.4719893932,0.5502031446,0.4619366527,-0.3297341466,0.9770031571,1.1077107191,0.6487148404,1.0592761040,2.2452018261,-1.4517990351,0.3241731226,0.5708991289,-0.2178623378,-1.4555898905,0.8619402647,-0.7125315666,-0.4648037255,-1.1987832785,-0.3875809610,-0.8089615107,-1.6652204990,-0.6737309098,0.4696156383,-0.9500765204,0.5246605277,-0.3600152433,1.0752583742,0.3630267680,-2.2392160892,-0.4884197116,0.0897057503,0.5078607798,0.2434695959,0.6669661999,0.1939474046,1.1956789494],[1.2731165886,-0.2553974092,0.5463133454,0.7415204644,0.8253844976,-2.2207424641,-0.5939807892,-0.5405517220,0.5992439389,0.5527491570,0.2647654712,0.3965257108,0.7488173246,-0.6501255035,0.7335247397,2.3412547112,-0.2428223044,1.3753275871,-1.7682363987,-0.0811214820,-0.1482257694,0.0008704733,-1.1559543610,0.4070131183,1.3749901056,-0.3521507978,-0.5838592052,0.7910311222,-0.8108868599,-1.7694545984,0.6860965490,0.6280745268,-0.0189835038,0.7401046753,-0.0957614034,1.5742759705,0.6285890341,0.6735109091,-2.2882628441,0.9412603974,0.8516026139,-0.2982982099,0.7832838893,1.8826241493,-0.6292356253,0.5742823482,1.9699435234,-0.9881187081,-0.6871194839,-0.7571073174,1.7136073112,0.0445418991,-1.3317672014,-0.0278986376,0.1168358251,-1.9115582705,-0.3436582088,-1.2878416777,-0.4543692172,0.4965259731,-0.3469325006,-0.2261100411,0.7186712027,0.4477775395,-1.1620095968,0.6727824807,-0.0883801803,-1.8939939737,-1.2754259109,1.0608910322,-0.8201976418,0.2681608796,-1.5161629915,2.1384537220,-0.0212910455,0.6318595409,0.0462469086,0.3144905567,0.3767325878,-1.1331812143,-0.2977827787,1.0825668573,0.5229936242,-0.8243591189,0.2687449157,0.4557905197,-1.6796916723,0.0562063754,0.5043262839,1.0077110529,0.6924237013,0.9910323620,-1.1296902895,-0.8210905194,-1.4609053135,0.9427023530,-0.6383852959,0.1021938547,0.5141855478,0.0296744183,1.2136639357,0.6528182030,-0.4292894602,1.3012942076,0.8445175290,-1.8041859865,-0.0520955026,0.6637779474,0.8166362047,0.4969393611,0.1612919420,-1.2245982885,0.8383113146,0.1584201306,-0.6001465321,-0.6403214931,-0.2489199936,-0.8059918284,-1.7376863956,0.2539567053,0.9096975327,0.9964336753,1.1921913624,1.0994590521,0.0866026059,0.1853996962,0.3626529872,-0.6009641290,-0.8141295910,-0.1610737592,0.0605895333,-0.6976833940,-0.6364158392,-1.6901087761,-1.4714292288,-0.8151713610,-1.0487115383,0.6838766336,-0.5414502025,-0.5800019503,0.4200898111,0.8287611604,0.2556479871,1.0243107080,0.6832750440,0.9061375260,-1.0559997559,-0.5638802648,-0.0439743996,0.1173364073,1.6787053347,0.3498084545,1.0181390047,-0.5923074484,0.3546623290,-2.6732180119,0.7222507596,-0.9528815150,0.6282278299,0.6948962212,0.7244352698,-0.4419042766,0.6062676311,0.4268949032,1.2982889414,-0.0492773838,0.1728408486,-1.8239735365,0.0842023790,-0.1465672702,-0.5029066205,-2.5519418716,-0.8701648712,-0.3346950412,0.8195809126,0.9252887964,-0.4902196527,0.7730194330,-1.9046716690,-1.5051252842,-2.4048972130,-0.2875409722,0.8639270067,0.7805999517,1.4741975069,-1.3624743223,-0.8319619894,1.3560540676,0.3230776787,-0.8606194854,-1.7348588705,1.6464912891,0.9878881574,0.6745097041,-0.7136139870,1.6001394987,0.1300480217,-1.5410406590,-0.5384830236,-1.4236744642,0.5284870267,0.1341856122,0.2507516742,1.0759317875,1.0543617010,-1.2375645638,0.1014309525,-1.2856109142,1.5152353048,1.0129420757,-0.5068862438,-0.0818387121,0.8557261825,0.6897322536,-0.6936151981,0.8198645711,0.5045341253,1.4971163273,0.7517387867,-1.0433621407,-2.2742910385,-0.5024763942,-1.5331497192,-0.6632850170],[0.1812437326,2.9718608856,1.1071363688,0.1692442596,2.0369729996,0.4972907901,1.4878046513,-0.1821295917,1.4015305042,-0.8876495957,1.7378798723,-0.8951967955,1.0968136787,0.7477356195,0.5420276523,0.8818691969,0.7651853561,-2.1793744564,0.1379532367,-1.5338999033,-1.6719213724,0.8710910678,1.4140412807,-0.8285297155,-0.4181405008,1.0015592575,1.0259017944,-1.0662993193,-1.5625612736,1.5425119400,-1.5811508894,0.0358812883,-0.4583835900,-0.2951638699,0.6391587853,1.2410137653,-1.0080453157,0.0030940096,0.8546661735,0.5233162642,0.1683357954,-1.0702836514,0.6986927986,-2.6348586082,-0.2744528651,1.0474078655,0.6944044828,0.4282088280,0.5804508924,-0.2621594965,-0.1507897526,-0.6044706702,-0.5499491096,-0.8979150057,0.6255866289,1.9500479698,-0.3619381189,-2.0538542271,-2.2297744751,1.2849069834,-1.6552209854,2.3148140907,0.7873542905,-1.1823868752,-1.3611557484,1.6840388775,0.8037516475,0.1811046898,0.1025839970,-0.0465708077,0.9594902396,-0.5949816108,-0.4117023051,-1.0228437185,-0.0251171757,0.8824501038,0.7591711283,0.6231270432,0.5917374492,0.6219696403,-0.0495552123,0.4270074368,-1.1682341099,-0.8261880279,-0.5186452866,0.9526984692,0.2365787178,0.7776110768,0.7848405242,-1.3479496241,-0.6416763067,-0.3390326500,-0.5502781868,-0.9254025221,-1.2017199993,0.1044468060,0.5422565341,1.1859241724,0.0673230439,0.3402942121,1.9644356966,-1.2449916601,2.0009057522,-1.7514717579,-0.1355766803,-0.0298841372,0.0156432688,-1.6379914284,0.0300169382,-0.1188428104,0.4865311384,-0.5542267561,0.2956261337,0.8642213941,-0.3620271087,0.5115072727,2.4552910328,0.1400547177,1.0980756283,-0.9341176748,-0.0969975144,1.0263452530,-1.5575366020,-0.6205061078,1.1078485250,0.9087997079,0.0788982362,-0.2932177484,-2.6274516582,-0.5674450994,0.8807153702,1.8894174099,0.5257295370,1.7268207073,-0.0833458751,1.6114909649,0.5052893758,0.8463229537,0.4828009903,-0.0895236433,-0.2982712388,0.4179650843,-0.0710343793,-0.5111294389,0.3934566081,1.1269727945,0.5989762545,-0.1793426871,-0.5627002120,-0.3595992625,0.5753319860,0.3222674429,-1.3311716318,-0.5409787297,0.1170743182,0.1916641593,2.1233403683,-0.3985538185,1.1539492607,2.1156904697,-0.2566215396,-1.5207226276,2.0685455799,-1.1869761944,-0.8740059137,-0.4242081940,1.5217876434,-1.5788558722,0.5758740306,0.3150611818,-0.7949121594,0.5349891186,0.4776623845,0.4408864379,-0.5021479726,-0.2214682847,-0.4992967248,-0.0067575639,-1.8457427025,0.9280284047,-0.6053181887,0.7470316887,-0.0616565011,-1.2198575735,0.0267692748,-0.6104433537,0.0101288538,-0.3280124068,1.1101564169,0.3608153462,0.4453727007,0.9636499882,-0.8304093480,-0.4685767293,0.9103437662,-0.6982381940,1.1686404943,1.1610959768,-0.1441449672,1.0166361332,1.8670653105,-0.1394449472,-0.4043854177,0.5049604177,-0.6021443605,0.2187743187,0.3758377731,-2.1905653477,-0.4398531020,0.1216888949,1.1135267019,-1.3173891306,0.5598527193,-0.0590468831,0.6931667328,2.4740109444,-2.0966489315,0.1619170755,-1.5034503937,-0.6038945317,0.5554865599,-1.4479967356,-0.8917023540,-0.6455559731],[-0.4626631737,0.4900054634,-0.3706803024,0.5256991386,0.1678807884,-0.5502416492,-0.3375319541,-0.3431487679,2.1183767319,-0.7996915579,-0.3146005273,0.3703980744,-1.0030088425,-1.9712535143,-0.2197760195,-0.2830720842,-0.1667875350,0.5648902655,-1.6655410528,0.7426096201,-0.1408229321,-0.3646937609,-0.3127844632,-3.0347666740,0.1353932172,0.6488391161,0.5635000467,0.6422799826,-0.5255937576,1.3670353889,-0.1994901747,0.2347858846,-0.5399846435,-0.2052275538,0.7761011720,0.3517878354,-2.4709758759,-0.5613272190,-0.0516717657,-1.9915071726,-0.8145096898,-0.1598192304,-0.4733167291,0.3067771792,-0.4550936222,0.4928416610,-1.5474539995,-0.5095470548,1.3454902172,0.6438094378,2.3404154778,1.0623267889,0.2608698010,1.3165721893,0.3916930854,1.2871226072,-1.1001724005,0.4859510958,-1.0528469086,0.0048558200,0.5584079623,0.8512579799,-0.9370487928,-1.2440121174,1.4365291595,0.0317177959,0.2058206797,0.0645339563,0.2769898772,0.0672572851,1.3564913273,-0.8004226089,-1.0108019114,0.2683846056,-0.3449343443,0.5715727806,1.8105565310,-0.4251840413,0.0933929682,-0.0318874568,-0.0917476565,0.2792342305,-1.3499195576,-0.3130746782,0.1089106649,0.7911434770,0.1674464196,-0.5621147156,0.0358672328,0.3739733100,0.1566600949,1.6247915030,-1.3720308542,0.0518179424,0.2691105306,0.1931755841,0.0992060304,-0.0557342619,0.6415981650,0.2318139821,0.1195221543,-0.7894995809,0.7086682320,-0.2938629985,0.4510083795,-0.2968471646,0.9600773454,0.8050220609,0.9683947563,0.5477582812,2.1819415092,0.2233613133,1.3260169029,-0.3495138288,0.0223538298,0.1567016244,-0.7688352466,-1.0201549530,-0.0924670175,0.0193462633,0.3662928641,0.5027229786,0.4168736339,-0.6279435754,-0.4090660214,-0.2738506794,-0.7319854498,-0.7716166973,-0.4655268788,0.0913858637,0.6756829619,1.0749363899,-2.9845871925,-2.3411133289,0.7401157618,-0.5512769222,1.1327184439,-0.3372319043,-1.0306859016,-2.1232624054,0.1218878105,0.2254609168,-1.8603311777,0.1255722195,0.1211966723,-1.5937081575,1.2448704243,-0.4653194249,-1.1432752609,-0.8448824286,0.4323055148,-1.5212147236,0.4646244943,1.2543433905,0.6006751657,-0.5522754788,0.6215143204,-1.3684450388,-1.2913494110,-0.7055885792,-0.7935898900,-0.2950033844,0.8652331233,-0.3805498779,-1.6659548283,-1.2524651289,0.0942175016,0.4701784253,0.4993986189,0.3036863804,0.3379638195,-1.0666376352,0.9350460768,0.0076516639,0.8255095482,2.0182163715,-0.2377916873,0.4250042140,-1.2792454958,1.1763788462,-2.4107418060,0.0559295639,-0.8411914706,0.9756729007,-0.2996968329,1.1991971731,-0.0036011224,1.5130709410,-0.7111702561,-1.3614077568,2.2163901329,1.9284011126,0.6578730345,-0.5633271933,-0.5350072980,-0.6862948537,-1.6068059206,-1.5318435431,0.6679149866,1.0064121485,1.1902654171,0.2330749482,1.5748063326,1.1538707018,-0.9454938173,0.4218579233,-0.6603856087,0.5557044148,-0.3310923576,1.2670267820,0.6889605522,0.5879198909,0.8793924451,0.5530157685,-1.3612712622,0.6004394293,1.5862592459,1.7088806629,-0.3228707016,-1.5339814425,1.3532996178,0.1337690353,0.4033794701,0.9362673759],[2.5070328712,-1.4719178677,1.1299579144,-1.2589129210,0.8528534174,0.8711306453,1.6040962934,1.7402243614,-0.6684724689,-0.8084316254,-1.1939450502,-0.2090555429,-1.5653285980,-1.6939487457,-0.7268587351,-1.8237613440,0.1104834676,-0.5062472224,1.1678825617,-1.4700238705,-0.8294458985,-0.4625340104,-0.2558137476,-0.5204338431,0.5611996055,-1.4619174004,-0.6257733107,-0.7569713593,0.7612380385,0.6510775685,0.4265863895,0.8792460561,0.0824565366,-0.6086297035,-1.2662295103,-0.5894017816,1.4452910423,0.6569434404,-0.3121954501,-1.8380763531,2.0977277756,1.1606415510,-0.8928475976,-0.2812048793,-0.7891488075,-0.5135432482,0.1404644102,1.7727484703,-0.7247069478,-0.4047034383,1.1784008741,-0.0882060081,0.4330866337,-0.2398931980,0.6297299266,-0.7113015652,-0.2888081670,-0.6917343736,-1.0160704851,-0.0501448251,-1.7557926178,0.8874514103,0.6896318197,-0.9837778211,1.9327976704,-1.7024279833,1.1284931898,-1.7392871380,-0.1856941730,1.3723155260,1.3488906622,-0.2660662234,0.9858436584,0.3364506662,0.4291782975,1.2164518833,1.5936523676,0.5462658405,-1.1891343594,-0.3331855237,0.0216330234,-0.3720126450,0.6676066518,-0.1610603034,-0.6453488469,-1.2660588026,0.1885916442,-0.1913194507,-0.2533226907,-0.4697803855,0.0147942333,1.9812374115,-0.0728941634,0.4952958822,0.0687560663,-0.6622215509,0.0422153100,0.0999927744,1.8953807354,-1.4996274710,-0.5814818740,0.7663285136,-0.6812976599,-0.1066205800,0.2546959221,1.2007323503,1.3327698708,0.2031353116,0.7176383138,1.2447898388,0.4891205430,-0.9856312275,0.8673879504,-0.6054337621,0.4199489355,0.1581125259,-1.4761838913,1.1224590540,2.9868655205,2.0953581333,-0.6189078093,-0.7180119157,1.2323864698,-0.4477436841,-0.4142808020,-0.1765896827,-0.9405736923,-0.3397848606,0.6380889416,1.3421542645,-2.2330129147,0.2280873209,0.7671604753,1.4022625685,-1.3546260595,-1.7892752886,-0.9778614044,0.2651600242,-0.7413028479,0.2085521221,0.6731303930,-1.5684994459,-1.0101099014,-0.5023919940,-0.6700144410,0.7095361352,0.8025005460,-0.2038030177,0.4760249555,0.4454511702,-0.5939602852,-0.7164628506,-1.5171319246,1.4800959826,-0.0717306584,0.9497123361,1.4563376904,1.5433458090,2.4010305405,3.3334319592,0.4822435677,-0.5614119172,-0.9017120600,0.9639977217,-0.2467245311,0.6018739343,0.6744873524,-0.7295719981,0.2263079584,-0.0595782287,0.4756217003,-0.1380271614,1.4944998026,-1.1411798000,0.4533945322,-0.5606918335,0.6212491989,-1.1574071646,-0.2401100546,-0.5401071310,-0.8303443193,0.3478609622,-0.8683995605,-1.3129228354,-0.1908754110,-1.8695154190,-0.3507457078,-1.6960946321,-1.4109058380,-0.0006375873,-1.9163777828,2.3475019932,-0.9438844919,0.9024425149,1.2787234783,0.2669394016,1.0342036486,-1.1763765812,0.2994127572,-1.5218186378,-0.6456666589,-0.0896530226,-0.3566700816,-0.5822973251,0.2027257532,1.2007070780,-0.9537147284,-1.7942665815,-1.6502112150,0.3794765770,-2.1909353733,0.3905528784,1.6661679745,-1.2986001968,-1.2059516907,-0.9303827286,-1.6575475931,0.2639370561,0.2804372013,-0.1337700784,0.4575373232,1.4409017563,-1.3529820442,1.1944533587],[0.1142227128,0.1391180605,1.3320208788,-0.5805708766,-0.4466572404,-0.8611194491,-0.1826741546,0.3247959912,-0.4794469774,-1.5512284040,-0.6987625360,0.7923681736,0.3309248388,-0.5274670720,1.7341567278,-0.5651237965,-1.1047180891,-0.8757085800,1.9002363682,-0.5654552579,-0.1503885388,2.3177163601,-1.0674678087,0.4179449975,-0.8482495546,0.1508035660,-0.2906613052,1.7630039454,-0.4032920003,-0.8586524725,-0.8267387152,-1.0485448837,0.4945226312,0.8998538852,-1.2160298824,-0.4785618782,-0.3782664239,-0.1571599543,-0.7503957748,-1.5480009317,0.2291377932,0.6442772746,1.8228875399,-0.0811340883,-0.9858629107,0.1007367000,-1.0231248140,0.8747323155,0.7531514168,-1.6565316916,0.9526207447,-0.0294028688,0.2464197874,0.1333850771,0.5864880085,1.0717082024,1.4090615511,0.8718538284,-0.0508552119,-0.0712610632,-0.1940155625,-0.7806551456,0.5808362961,-0.0453044698,0.9831092954,-0.0407605805,1.6183751822,-0.4711570144,0.9464051723,-0.1902614683,-0.5595943928,0.7917163372,0.8945364356,2.0407261848,-0.0378948264,-0.5753856301,0.7319956422,0.7065110803,-0.1146564707,-0.3133381903,-0.6908829212,1.3543372154,-1.2197914124,1.0682621002,-0.4357069135,0.4142681360,-0.8415632844,0.0666707754,-0.1261732876,0.4309399724,1.7831349373,0.2475240082,0.5416277051,-2.3900408745,-0.0873001814,0.4318037629,-0.0842064768,0.0386220776,0.0086299293,0.4828562737,-0.3126920760,0.6790862679,-0.0472784191,1.0052809715,-0.7735071182,0.4777276814,-0.1993416548,-0.7341573834,2.2931449413,-0.8017128110,-1.2624527216,0.4948807955,1.9594209194,-0.2333582640,-1.1403502226,0.7428901792,0.4308326542,-0.0369284041,-0.8011110425,0.2558170557,-0.8303340673,0.4260979295,-0.7176937461,0.3255190253,0.2934720814,0.0140975080,-1.6804726124,-0.3438348174,-1.4502503872,-0.9531344771,-0.1847796291,0.1688913107,0.6571382880,1.5159125328,-0.1185081005,-0.9134144187,2.3134839535,0.9210193753,-0.1931522787,1.1115853786,-1.5775060654,-0.4333670139,0.9147577286,-0.8540613651,0.4533645511,-1.5374147892,0.3218281269,-2.0726091862,-0.2654472589,0.4145475924,-2.7371721268,0.0493369773,-0.8245742321,-0.7906406522,-0.4056213796,-1.2431070805,0.1490028054,-0.5710224509,-1.1668577194,0.3953489661,0.7068749070,-0.3533940911,-1.9965847731,0.8758977056,0.4603376687,1.2675856352,-0.7088031769,-0.6654269099,-0.9213801622,0.4947441518,-0.0715525448,-1.0028457642,0.3830953240,0.4520997703,-1.2615735531,0.9056553841,0.3861218393,0.1425308436,-0.1260088682,0.9447017312,0.6559101939,-1.8171663284,1.4004011154,-2.0061945915,-1.8255103827,0.1024764478,0.1558491588,1.2420189381,-0.4764030576,0.3463437855,-1.7062416077,-1.5446808338,-0.3215472400,-0.4196771085,1.8047077656,-1.8244951963,-1.2479240894,0.8842428923,-0.7375934720,-0.4316256344,-1.1759667397,0.3775776029,0.3401539326,-0.7903994322,0.4455604553,0.5148267150,1.0482620001,-2.2607221603,0.0069156969,-0.9996435642,0.4997765124,-0.8113646507,-0.2739197314,0.7623291612,1.9041544199,0.0038511909,-2.2380661964,1.2144907713,-1.3699985743,0.5216916203,-0.9953954220,-0.0678855330,-0.9681578875,-1.0197651386],[0.4510965943,-0.9384776354,0.4080997109,0.2205782682,1.4572985172,-0.0716064721,-1.3010287285,1.0668503046,0.5560036302,0.0169553459,1.3414565325,0.3216943741,0.4991309047,1.4584230185,0.2209969759,-0.6442297697,1.2691410780,-0.0391359441,1.1090129614,0.3669699728,-1.2093535662,-0.2478190958,1.0072946548,1.3478624821,1.4052371979,0.7991831899,-0.5470215678,0.1076327264,-1.2209036350,-0.2053703666,0.6416805983,0.8527611494,0.8009208441,-1.1441555023,0.5289140344,-1.3059980869,0.8233359456,-0.9994716048,0.4629640877,-0.9534198642,1.9949338436,1.0057520866,0.4678486586,-0.4734914899,0.0677830651,0.3521122932,-0.1107401550,1.0441912413,-0.4773075581,1.3873674870,0.5615459681,-0.7976940870,-1.3771500587,0.8169613481,1.3486270905,0.5091228485,-0.3491946459,-0.4665772915,0.2233534604,0.6446563601,0.9125355482,0.9257678390,0.8465470672,0.0578327812,1.3858836889,-0.2434680015,1.6562070847,0.9416037202,0.5632153749,-0.5659677982,0.5274534225,1.1877981424,1.4489836693,-0.7451183200,-1.5661666393,-1.1835601330,-0.9559128284,0.1651993841,1.1368753910,-1.0665529966,-0.4836068749,-0.8145627975,0.3149281442,-0.1883910000,0.4216847420,-1.1827241182,-0.0994306877,-0.3438635170,-1.9663894176,0.2089749277,-1.4949587584,0.3206232488,-0.6770358682,1.1477482319,-0.2624453902,1.4652047157,-0.0285696778,-0.0104226936,-1.3233613968,-0.5543357134,-0.1505497992,0.2329614758,-0.5039017200,0.2137409151,-1.2017041445,-0.6092761159,0.5041880012,-2.6073977947,-0.5180581212,-1.2378791571,-0.7136105895,-0.7491711378,0.8616729975,-0.6471394300,-2.4227328300,-0.1132138893,-0.6853970885,0.3431085646,0.8768229485,0.2715342343,-0.3802629411,-0.8611323833,0.3513724804,0.5549112558,-1.2833085060,0.7726469636,-0.4660710394,2.5595307350,-1.7120479345,-0.4161309004,-0.2351716012,0.3630282879,0.9382425547,0.6695708036,-1.7179670334,-0.5959632993,3.7025876045,-0.2213580161,-0.1287264526,0.6192150712,1.0531377792,-0.2467512041,1.4417458773,-0.3152952492,-0.7391575575,0.1590342820,0.8058029413,-0.1784851551,2.3034143448,-0.4181099534,1.3932461739,-1.1737822294,0.7538549900,0.6269468069,-0.7819752693,-0.7092577219,-0.5887012482,-1.0844168663,0.9721927047,1.2547427416,1.7547675371,-0.7578601837,-0.3038023710,0.6221553683,0.2105609030,0.6844446659,-0.0726488233,-0.8142358065,1.5024870634,-0.9599415660,0.8041617870,1.2129323483,0.2786012888,-0.4806325436,-1.5507179499,-0.6431815624,-0.9598134756,-0.0414928049,0.9750754833,0.2134624720,-0.7340024114,-0.9867333174,-0.9676715136,-0.3153736889,1.7369050980,0.1728345603,-1.6357651949,0.5001330972,0.0361833759,-0.3400045931,0.6917695403,-0.6486070156,0.5865759850,0.6650483012,2.3782908916,-0.7281708121,-0.1784517318,-0.6568906307,-0.0668870881,0.4001824856,0.5983645320,1.4374731779,0.8282849193,-0.2202301174,1.4063117504,-0.2658022344,-1.0646981001,0.2660436034,0.2665685117,0.8515069485,0.0041442462,0.3150144517,-2.1227982044,0.7639856339,-1.4190670252,0.7797390819,0.9185681343,2.0322315693,-1.4458156824,0.1613995284,-0.4561886489,-0.8995773196,-0.2119886428,0.1786645055],[0.1776530445,-0.3201810718,0.1161821634,-0.5768741369,0.6252367496,-0.1033955812,-1.0653221607,-2.6090035439,-1.9755235910,-0.0643296540,0.3567645550,0.1918041110,-0.2986488640,1.0968306065,-0.4158258736,1.2320274115,0.6485459208,1.6077394485,-3.1362752914,0.3075769842,0.4591005147,0.3467891812,-0.3982659280,1.1793015003,-1.7780370712,-1.3126175404,0.4909677207,1.6737562418,0.0124732330,-0.3899201453,-0.1232452691,-0.6100804210,-0.9292196035,0.8099145889,0.6836316586,1.4735952616,-2.2071619034,1.9934275150,0.6117762327,-1.5051028728,-2.4555799961,-1.0627012253,0.9527903199,1.5368833542,0.0395082273,-1.3321162462,0.5243310332,-0.1798415333,0.5129242539,-0.4493110180,0.0150246527,-0.9672916532,0.9956399202,0.2072731555,0.5924943686,-0.5087363720,-0.1341209561,2.0133500099,1.2745864391,-1.7311307192,0.8783261776,0.2382151335,-0.5218320489,-0.7136888504,0.9078969955,0.1662100703,0.6880350113,0.6426402926,-0.1699856371,0.8449928761,-0.2059611827,1.2312691212,-0.1832074076,0.7024871111,0.0767141655,-1.0377204418,-1.2589073181,1.2890216112,-1.6496952772,1.3731424809,-2.5840425491,0.8842554688,0.7858142257,1.2424460649,-1.1142414808,-1.6087180376,0.5588724613,-0.3181532919,0.1282300502,0.3226558864,1.7192137241,1.8144487143,0.0832170099,1.0794529915,1.0780323744,0.1856908500,-0.6103316545,-2.1893501282,1.1166290045,0.7507947087,2.0451416969,-0.0583368354,-0.7521981597,0.1606955081,0.9020569921,0.2915190160,2.0094237328,-0.5653720498,-2.0505139828,0.3951239884,-0.1658412069,0.1319901049,-0.8073076010,1.1072100401,-1.6461715698,-1.6490888596,-0.7021602392,0.1291684657,1.0130834579,0.4326122403,0.0710866079,0.6522153020,0.9110949039,1.2549139261,0.3565925658,0.4046155214,0.4020137191,0.3690601587,-0.4672718048,1.3912384510,-0.4798047543,1.5850034952,0.2715241313,-0.0765904635,-0.1810461581,-1.1558505297,-0.3667913377,-0.3131943941,1.6469752789,0.1343065500,-0.6560097933,0.5086302757,0.4044632912,0.6018323302,1.7113457918,-0.1906548142,0.0425776057,0.0373008773,0.1381624639,-0.4156305790,-0.7981631160,0.0582099520,2.6061866283,-0.7722873688,-0.0470007211,1.5131361485,-0.0309810881,-1.0579044819,0.8854081631,0.7483155727,-0.2261427641,-0.7398789525,-1.0611298084,-1.4868919849,-0.4077556431,-0.5551923513,-0.3160439432,0.7666338086,1.4771444798,-0.1199424937,1.2570329905,-0.6283643842,-0.6694485545,1.3534792662,0.1671885401,-0.9664984941,0.1351937652,-2.2209308147,-1.7459772825,0.8721058369,0.4668622911,-0.9920146465,-0.3378461897,-1.5689595938,0.0369126610,-1.8374615908,-0.9583182931,-0.0569251254,0.0614638180,0.3806518018,-0.9281975031,-0.3982549608,0.6343744397,0.5360945463,0.4420194328,-0.1513541192,-0.1408912539,-1.2138690948,0.9535938501,-0.1115452573,-1.2844252586,0.2494014949,1.4498769045,0.8719550371,0.0237522461,-1.6092516184,0.1648471653,0.5427899361,-0.4420486987,-1.4820035696,-0.0689525232,-0.7968429923,-1.6705019474,0.7698445320,-0.1265096515,0.7726453543,0.1753320694,-0.5076130033,-0.1544819772,0.2653084695,-0.0847224519,0.6755714417,1.0591664314,-0.0757832155],[0.0805232376,0.0671821758,1.6652826071,0.2529174089,-1.0575348139,0.0438446961,0.6698242426,0.5310724378,-0.4435330033,-0.3427950442,1.0558329821,-0.9846804738,0.5860679746,1.3339457512,-0.8269437551,0.9077727199,0.1672893316,-0.8939241171,0.3120681345,-0.6364495158,-0.7267069221,2.6667218208,1.1729916334,1.2637227774,-0.3764693141,0.1535074413,-0.2704104781,0.0951040611,-0.7366018891,-1.2720896006,-0.6093081832,-0.2956297398,0.1220130399,0.9173392057,-0.2534488142,0.4520286620,-1.8082159758,0.7244623899,0.7396792769,0.6426346898,0.0797319785,-0.5158020258,0.5891546011,-1.4976643324,-1.2477073669,-1.7936444283,0.8376747966,-0.7089396119,-0.1811517924,-0.7688514590,-1.4071519375,0.2784728110,-2.5875277519,0.9116972685,-0.5199452043,0.1479708701,-0.8809148073,1.8356801271,-0.8060521483,0.4518039227,-1.6225919724,-2.0098841190,0.1720021516,-0.4275765419,0.1004268378,0.1585029215,0.8063095808,-0.1378083378,-0.0826101378,0.6346687078,-0.3701512814,0.4816154242,0.3261772096,0.7613685131,0.6079658866,-0.6945112944,-0.2238053679,0.0034116469,0.0198742934,-0.6288275123,0.3040928245,1.0772180557,1.1524056196,1.1629301310,-0.3494142294,0.5766470432,0.3408490717,0.4503265023,-1.0144721270,-0.0498800389,-0.5482375026,0.3489536345,0.7263273001,-0.9238932133,-0.3699231446,-0.8713849783,-0.7335543633,0.8255431652,-1.5280839205,0.5197145939,1.0876455307,-0.2757599950,1.5692808628,0.7538465858,0.9047079682,1.1547479630,1.0163842440,-1.1070152521,-0.2055583447,-0.9297230840,-1.8402305841,-0.0856739655,-1.4860048294,-0.5312367082,-0.2028853595,-2.2556784153,0.3083379269,-0.6643678546,-1.4581902027,0.1362811774,-0.2612127960,-0.3958458602,0.2158193737,2.3531103134,0.5730254650,1.5000845194,-1.6561217308,-0.9346296787,-0.0294601489,-2.1529641151,0.3335566819,1.1823263168,-0.5181627870,0.6104934812,-0.7329621315,-1.1848534346,0.4069354832,0.0345559642,1.9944534302,2.3979656696,-1.1336600780,1.2679555416,-0.4079175293,0.0792932883,-0.4174874723,-0.2649638653,0.9532611966,-0.3875336945,0.6469477415,-1.6494807005,-1.5241768360,-0.9117911458,0.9734886289,0.2089319974,0.5428323150,-1.9180088043,1.2095471621,-0.2803136408,-0.8650236130,0.3140363991,0.2163758725,-1.3617240191,-0.9943023324,0.6911141872,0.7621330023,-1.2232488394,1.6510376930,0.2612263560,0.1864361614,-0.0706841946,-1.0403263569,-2.0321514606,-0.7808988094,-0.8977251649,1.6592231989,-1.3123985529,-0.5749298930,1.1344422102,1.6615151167,0.1504340172,-0.0827697366,1.0395177603,-0.7313262820,1.0351142883,0.2846813202,0.3578853905,-0.8776550889,-0.0276330616,-0.2406794727,0.2422078401,0.2753272355,-0.9000979066,-2.1857159138,0.2421487421,1.9852120876,0.6831824780,-1.3812738657,0.2342664003,0.7960075736,-1.0990927219,-0.6038036942,-1.5455021858,-0.2055137306,-1.2040677071,0.1401914656,-1.1692546606,-0.7095637321,-0.2126831561,-1.3855490685,-0.2365769148,-0.3908159137,-0.0181708690,-0.4164932966,1.7714226246,0.8974324465,0.5611874461,0.8519725204,0.9009156227,-0.1347933710,-0.3339865208,-1.3960329294,1.1774704456,0.2161447108,-0.6004638672],[-1.0433120728,1.2550470829,-0.3633813262,-0.5906378031,-0.0185135938,-0.1028014421,-1.7445312738,1.6993048191,-0.6432486773,-0.9254992604,0.5182227492,-1.2138725519,0.8419089317,-0.6030687094,-0.1839088202,0.4044023156,0.1647076309,2.3706243038,-0.8371855021,-0.3153517246,-1.0361464024,-0.1055759564,-0.8192484975,-1.3087396622,0.8839029074,0.8903806806,-0.1596081257,-0.3812036216,-0.8509666324,-2.1042964458,-0.1953709871,0.7087575197,-2.1301450729,-0.0047883722,1.4961576462,-0.6381576061,2.3768637180,-0.9427425861,-0.1516778916,-2.2839558125,0.5090414286,-0.2221545130,0.4502406120,1.1437622309,-0.1963302344,0.3094167709,1.3509380817,-0.5044822097,-1.6174205542,1.5378000736,1.3241184950,-0.2176084369,1.0739580393,-1.3880198002,2.6526143551,1.3435379267,0.9719251990,0.1656426191,-0.4877803326,-0.9836961031,-0.7674241066,-2.3378081322,0.3354743123,-1.1810867786,1.2881627083,-0.2078816146,-1.2664439678,0.5565501451,4.0197734833,0.8854044676,-0.5009924173,1.0933628082,-0.4822470546,1.1983449459,-0.7168338299,1.0617597103,1.4151138067,0.2771531641,0.7030990720,-1.2996754646,0.5242429376,-0.5445105433,0.7130293250,-0.3225224316,-0.6599298716,1.4526163340,-1.0813671350,-1.3591839075,0.8215834498,-1.0169330835,0.1585911661,1.6827707291,-0.9877544641,-0.8410540223,-1.2568205595,0.6768353581,0.3739871085,0.2208058387,-0.3934979141,-0.4662604630,0.5557742119,-1.4550930262,0.6333280802,0.2107170522,-0.0745086297,0.4422843456,1.3056447506,0.2425055355,0.0369345658,-0.1842762232,0.6566666365,-0.2707042992,1.6456437111,0.3681226671,0.6295041442,0.9925812483,-0.3826599419,0.1413362175,0.4062003493,2.3438417912,-0.3850591481,1.5470700264,-0.8475514054,-0.6330932975,0.4426223338,1.1135199070,0.0156516992,0.9530420303,-0.2288440615,-0.5985863209,0.8398433328,0.3938765526,1.7500195503,-0.6720721722,0.2872419059,0.6539419293,0.1856593937,0.1378008425,-1.3960168362,-0.1134697646,-0.2837387621,-0.8654352427,0.3288089037,0.0301165637,1.1624650955,0.0011483978,-0.6361009479,0.5182092190,-0.0406220965,1.7414946556,-1.6548664570,-0.6029675603,-0.5882458687,1.4716166258,1.1737931967,-1.1955555677,-0.1590917110,0.2381560504,0.9164401889,-0.1090425402,0.5899898410,-1.2650150061,0.5999732614,-1.0951660872,-0.0711921379,-0.7832251191,-0.6987342834,-0.0219315849,1.1395345926,0.2375022620,-1.7151470184,-1.5024656057,1.7313393354,-0.5030255914,-1.2392387390,-0.1868620217,0.9655178785,-0.5649629831,-0.6893165112,-0.7033599615,-2.2348511219,-0.0525545701,0.3070724905,0.6070462465,1.1996613741,-0.5176995397,1.0299232006,0.8695846200,-2.1147446632,0.4078527093,-0.8786563873,0.4362991750,1.1177811623,0.5423116088,0.9149170518,2.4281659126,0.2401718497,-0.5961187482,0.0141318729,0.3701690435,0.1808289289,1.2599830627,0.2183745056,-0.3465143144,0.8355005383,0.2451741993,-0.1894374192,0.6959276199,1.5179269314,0.2976191342,0.5701390505,1.4372801781,2.0795273781,-0.3756849170,-3.5909032822,0.0716620237,0.6304141879,0.0261821225,0.8042483330,0.9069849849,0.3464861512,-0.5794659853,-0.5438749790,0.0289033912],[0.9105071425,-1.4992305040,-0.4793045819,0.8542497158,-1.0811475515,0.2171867788,-0.5041297674,1.7892602682,-0.0283070989,-0.0441253670,1.2115254402,-0.7234268785,-1.1590812206,-0.1522644460,-0.9474188685,0.1717792600,-0.2749402225,-1.0617312193,0.6321954131,-1.3946050406,-0.5197392106,0.8903101683,1.0188506842,0.0586597063,0.3688014150,0.7033883929,0.5548697114,-0.9274749160,-0.6818982363,0.4796386361,0.2311522514,1.9238237143,-0.6357836723,-0.1479309797,2.6409428120,0.5992001891,0.0839401186,-0.3585545421,1.6161761284,-1.1302142143,1.3481178284,0.5973325372,0.5636794567,-0.0097401869,-0.3516861498,0.8041421175,2.3142130375,1.7594890594,1.5309214592,1.8990132809,-1.8272774220,0.8267484307,-2.2523925304,-0.1968108118,-0.2863500416,0.4057592750,-0.7191252112,0.2487123907,-0.6566113830,-1.3493041992,0.1164432839,0.1086965054,-1.4072995186,-1.7230648994,-1.9000509977,-2.4684510231,1.2062046528,0.4911671579,-0.0894651636,-0.3995863199,0.0991533101,0.0014761668,-0.7353255153,1.0571197271,0.3556884527,1.1477404833,1.5356293917,2.0558340549,0.2317831367,-1.1191799641,1.3284108639,-1.4650167227,1.5791805983,0.2670737803,2.0296282768,0.7951829433,0.4688805938,0.0782255381,-0.3811810315,-2.0412206650,0.9424459338,1.4058378935,2.0227613449,-0.4164577723,0.1712241471,-1.2878608704,0.2662688792,0.4602661729,-1.9420094490,-1.0917539597,1.0351030827,-1.3396145105,0.4247620106,0.4522379935,-0.1753645241,0.3741957843,0.6832326651,0.2857375145,-0.7751410007,0.2919989228,0.3030637205,-0.7733544111,0.2354594618,-1.6997842789,0.8201661706,-0.2567533255,-0.4506866932,-0.3111549914,-1.0207248926,-0.5771256089,-0.9786616564,0.1340002865,-0.8608180285,0.5694761872,-0.4418152273,-0.1610226035,0.2886717916,1.9575083256,0.7234182358,-1.0948028564,-0.2901492417,-1.3798342943,-0.6869062781,-0.5215283036,0.2655810118,-0.2533085644,0.2296121866,-0.6031663418,-0.8315091133,-0.5224909186,-0.4015553594,0.2074885517,1.6709539890,0.8634210825,1.5834872723,-1.7893948555,-1.3206269741,-0.6768729091,2.1363854408,1.2582120895,1.0345571041,-0.1696926504,-0.7326226830,1.0371384621,-0.9781098366,-0.2509648204,1.0754765272,-0.5080704689,-1.7204457521,1.3550037146,-0.7376111746,0.2516528964,0.0143696088,1.4959465265,0.9013447165,0.0666519925,1.2546199560,-0.6876458526,0.1289302558,-1.2125207186,-1.3062785864,0.1477067918,-0.9687612653,-0.6614953876,0.4936519265,-1.0251137018,1.5563279390,-0.0668517277,-0.6016780138,1.2578229904,-1.6711659431,-1.7495858669,-0.6962295771,1.7163276672,-0.6159476042,0.8581643105,-0.0230642557,-0.1786852777,-0.5286377668,0.2368431985,-1.6006636620,1.2182400227,-0.9850192070,-0.7063335776,-1.0004578829,0.2919587493,-0.1049265787,-0.7563028336,0.4225782156,0.0824065655,0.2042005062,1.2740459442,0.1095527783,1.1274046898,-0.6521725059,0.0609520078,-1.1165735722,0.9280680418,0.1987430453,0.4761828184,0.1669905335,1.0124512911,-1.2776417732,-0.3814474642,0.7289512157,-0.3052847087,0.7282732129,-0.5393836498,1.2419661283,0.5911603570,0.3793136477,0.1260632575,-0.1371300519,1.8880878687],[2.8134269714,-0.2924009860,0.3732911646,-2.3748250008,0.3623911738,-1.5661566257,1.3808929920,0.9725285769,0.2446972132,0.7711442113,0.3922756314,-2.1892728806,-0.0823605731,-0.2208417803,0.4101850986,0.2744546831,-2.2105195522,-1.3529791832,-2.0533165932,0.7840827703,-1.2388772964,-0.0091828620,0.2710341811,0.7293034196,-1.9300572872,0.2674436867,-0.6061401963,1.2984786034,1.4081494808,-0.7296541929,0.4568593502,-1.4688404799,0.3038083017,-2.5448539257,-0.9092229009,-0.3816877902,0.7162517309,0.7011740804,-1.4708878994,1.0043964386,-0.0533178635,0.1780886203,-0.0656738058,-1.8832274675,-0.7870368361,0.2258797288,0.8136019707,-1.6132806540,-1.6500467062,-0.7262764573,-1.8060714006,0.1261256337,0.0837224349,0.0827596188,0.7586573362,0.6176240444,-0.3255368769,1.4318530560,0.0577022061,-0.2958157361,-0.6132124662,-0.8152903914,0.8220098615,0.2450569719,1.2573981285,1.5494660139,-1.2309507132,-0.9299182892,-0.6454734206,1.0468801260,1.1484909058,0.5869446993,-2.1177897453,0.4626471102,1.0198746920,-0.1138288900,0.4247196913,1.1570379734,-1.4135748148,0.0725923181,-0.2679024637,2.7661833763,0.2679988146,-0.5027480125,-0.0181943942,0.6711435914,-0.0941974670,0.1855315417,0.2543438077,-1.4764968157,-0.9377606511,0.3014186025,0.2370459586,0.7084241509,0.1626368910,0.3276515603,0.1585559845,0.2503229976,-1.1399179697,-1.0920814276,-0.6667899489,-0.8548904657,0.7033684850,-0.7761536837,2.1346483231,-0.2393279821,1.3535968065,-0.9506493211,-0.4076920450,-0.5581548810,0.1968198419,-2.0824365616,-0.6504297853,-0.7817162871,1.5999169350,-0.4707145393,0.2664442956,-0.0279191937,-2.3622436523,-1.2645890713,-0.0324613638,-0.4157408774,-1.9002436399,0.8337079883,-0.5889626145,-0.5031498671,-0.0618600398,-2.4662024975,0.0627399012,0.8152358532,2.1090805531,-0.0630077720,-0.1326286197,0.1434296221,0.8538603187,-0.1592878252,1.6691845655,-0.4810115099,0.4189854264,0.2810520232,1.4511069059,0.1108039841,-1.6083573103,-0.2608066201,0.6873603463,1.0323524475,0.3345609605,1.4037721157,-1.1845630407,-0.2300915718,-0.1948073506,0.5498065948,-0.2714291513,-0.7518230081,-1.1877788305,0.3030099869,1.2471328974,-2.2817478180,0.5460087061,0.8085818887,-1.4102207422,-0.7649542093,1.1998691559,-0.3109331429,-0.5489453673,-1.1739296913,-0.7573813200,0.3622814119,-0.1803682745,-0.5011548400,-1.0009831190,0.1868225932,-0.6029005051,-0.8547886014,-0.1348072588,-1.1864544153,0.7664079070,-1.4393889904,-0.9475734234,-0.5573213100,0.8925701380,0.5009971261,0.7198488712,-0.5231234431,-0.1421491206,-0.6183432341,0.0882850811,-1.6847449541,-0.5839746594,0.2101413608,-0.6959885955,-0.4180793464,1.7377842665,0.5398337245,2.4762804508,1.5915863514,-0.7944422960,-0.3500894308,0.1223903745,1.8400007486,-0.0912151709,-1.1437829733,-0.3499307334,0.8078536987,-1.0026249886,0.2237447202,-1.0386419296,0.2516576052,1.2548787594,0.0778929368,-0.4539818168,-0.5510498285,-0.0578782186,0.4725125134,1.8870012760,2.4323177338,0.2010755539,1.1056728363,-1.8366775513,0.9536247849,0.7939686775,1.1003919840,2.3458998203,1.4692678452],[0.6490464211,1.9670273066,0.4042147398,0.6566967964,-0.7697871923,1.5224796534,-1.0281040668,-1.0950347185,0.8894008994,-0.7297667861,0.6717054844,-0.2670283914,-0.5510465503,0.0973181725,1.1938539743,-0.0754729062,-0.4562895000,-0.8172528744,-0.0423791744,-0.5283162594,-0.3101071119,0.1037842855,0.3862933218,-2.5298702717,0.0148019399,2.0022518635,0.4043906629,-1.8893733025,2.0781323910,-0.2799007893,1.5677926540,1.0496910810,0.2109147310,0.7038741112,-1.3184523582,0.7989603281,0.2703145444,-2.6894779205,0.5198162794,-2.2797818184,1.5013480186,0.9218204618,0.8572191596,0.8756232262,-1.7740607262,-0.4072049558,-0.0811734796,1.3906067610,0.8266698122,-1.7439453602,-0.0620697737,0.1208140701,0.1889812201,1.7714536190,2.0588123798,-2.7417387962,1.1500811577,-0.4305594563,-0.8185398579,2.0702614784,0.5266123414,1.0601838827,-0.5946053863,0.8639861941,-0.1706417352,0.6222539544,1.3736292124,-0.7181694508,0.0831693858,-1.1598395109,-0.4952351451,1.2962372303,0.0690178201,0.1662487537,0.3988175988,0.1684028208,1.8821505308,-0.5882483125,-1.0507441759,1.7254124880,0.1957102418,-0.3709122241,1.4042637348,-0.2510891557,-0.4316080213,0.0685362518,0.6092994213,0.7451376319,-0.4354406893,1.3262227774,1.4801108837,1.3739769459,0.1878964752,0.4108370543,0.9605338573,-1.5134578943,-0.1964982003,0.6435061097,0.0736133829,0.8728601336,-1.3197245598,-0.1270552576,0.0426341854,0.8820625544,1.1046253443,-0.2755198777,0.2538744807,-0.9900706410,1.3647599220,-0.8029752970,-0.9458059072,1.7950054407,-0.3628331125,-1.4039831161,-1.1162095070,0.4925628901,-2.0292842388,1.5619550943,-1.8280488253,-0.0055643721,-0.5366774201,-0.7328761816,-0.8478962183,0.0121869650,-1.6183778048,-0.4681866765,0.2380838990,0.3942469358,-0.0017471110,0.6457191110,0.1618220210,0.3286625147,0.1949937940,-1.8389343023,0.0252901614,-1.1302721500,0.5937021971,-0.1768487543,1.4522219896,-0.4664727151,-1.4856642485,-0.9684375525,-0.6833172441,-0.4522916675,-1.3983139992,0.2946139574,0.7114067674,-0.4360235631,1.3628817797,0.3709173501,-0.1819907874,-0.1708724201,0.6615654826,0.5053963065,-1.2913050652,-1.1293932199,-0.3762286007,1.4005522728,-0.3681354821,0.2439703345,-1.9376685619,0.0061770533,-0.3084800243,0.0341180898,0.6517161727,-1.0235981941,0.3485942483,-0.8717853427,0.1752543598,-0.8681605458,0.5755802989,-0.4126141667,-0.7222152352,0.9932783842,-0.8108548522,-0.8651838303,-0.4406715333,1.2634201050,0.3228797913,1.7299368382,-0.2031331807,0.7388085127,-0.3652395010,-1.0800300837,-0.4905096591,-0.3866543770,2.2770588398,-0.6903892756,0.9328139424,0.8298351765,0.0020154689,-0.4209246635,-0.6973096728,0.0860553086,-0.5382195115,-0.3424751163,-1.3047096729,3.0107023716,-0.2132201046,-1.7896158695,-0.2584707439,0.7357210517,-1.6454079151,-0.1919890344,0.8854669333,0.5473588109,2.1477603912,-0.3015838265,-0.3280054033,-0.5345777273,-0.0564358309,-1.0776684284,-0.0625013635,-0.8091907501,-0.9522696733,-0.0291139688,0.8151841760,0.4640530646,-0.4346605241,0.7263224721,-0.9717104435,0.0229827929,1.1674512625,-1.3929512501],[1.2918885946,-0.2988780737,0.2283933908,0.3405737579,0.3639356792,-2.1020929813,0.8405295014,1.0143632889,-0.1635631919,-0.2963094711,0.1203462780,-0.5817037225,1.0243686438,-1.1446744204,-0.1163422912,1.0779199600,0.7442581654,-0.5289862752,0.3235702515,0.8104526401,-0.6944915652,0.7105034590,-0.4998841286,1.3602881432,0.9865704179,0.9607505798,-0.5986943245,0.3859920204,0.8108100891,1.3603370190,0.1453486830,0.4959866405,1.0660791397,-0.3927115500,1.5472337008,-0.9704090357,0.6275779605,-0.1618680060,0.5084855556,-1.0714122057,-0.8444173336,2.5354545116,-0.4143082500,1.4401415586,-1.8746173382,0.0717201754,0.9315521717,-0.1924583465,0.2826135755,-0.4956521988,-0.5171129704,2.2835865021,1.2014240026,0.4965412319,0.6352142096,0.2207820266,-1.5204813480,-0.2159054577,1.4612492323,0.4836098850,0.7791002393,-0.7924147248,-1.9963259697,-0.9082238078,-0.0739156827,-0.1735779345,-0.6885651946,-1.6898833513,0.1898370534,1.3742725849,0.4149084389,0.9179326892,0.1505246013,-2.1365816593,0.6585453749,0.0833003744,0.1664917320,0.2179910988,0.1975180507,-1.2237678766,-1.4798632860,-1.5096775293,-1.6420594454,-1.0052191019,-1.4127585888,0.2579631805,-0.4421169162,-0.0934734941,0.4567604363,2.2722504139,-0.3473166525,0.1571355462,-1.1206731796,0.4965228736,0.5498935580,0.1304169148,-0.7870458961,0.8281716704,0.4544727802,-0.0937828198,-0.8460122347,-0.0507329628,-1.3765958548,2.1486096382,-1.7733092308,-1.1574308872,-1.3433600664,-0.3764662147,0.2813237309,0.2202699929,0.6045076847,-0.1821339726,1.2025723457,-1.3828216791,-0.9747838378,0.2388301194,-0.8707327247,-0.1252780706,-0.6416910291,1.3086551428,-0.5332537889,0.0063522253,-0.6723710895,-1.6051152945,0.3130868971,-0.1302162409,1.4862173796,-1.0393095016,-1.2276927233,1.7093112469,-1.7208932638,-0.3173751235,-1.1881891489,-2.3398308754,0.2897851169,-0.5540105700,0.3708947003,-0.6902725697,0.4827075303,0.5152086020,-0.8001226783,0.5289755464,0.6751053929,0.1977580190,-1.8228195906,-0.5684699416,1.7509375811,-0.7015925646,0.6359348297,1.3688038588,1.0106269121,-0.6654335260,1.0200300217,0.0526707098,0.4489641190,2.0376715660,-1.5846176147,1.3363882303,0.4211618304,-2.3855931759,0.4183519781,0.7159678340,0.1074562818,-0.2730627060,0.3309553862,1.5496168137,1.1344338655,0.9619284868,-0.8347564340,1.0374566317,1.4408031702,0.1390461475,1.7395902872,2.9890756607,-0.3393157423,-1.5076727867,1.4666700363,0.2921347022,0.3553029597,0.9788808227,1.2384502888,0.3073142171,-0.6225633025,1.0165023804,-0.7128052115,-0.0668653473,-0.5550617576,0.4804181457,2.0454192162,0.0330334306,-0.2663131058,-0.7029079199,-0.2169049233,-1.0697338581,-0.8404394388,-1.1840552092,-2.3508896828,0.1685397476,-0.1027170047,0.5997434855,-1.9493203163,-0.8024225235,-0.4092237949,-0.4173745215,0.2873328328,0.4117684960,0.2399783283,0.2981419265,-1.6334201097,-0.9804607034,-0.3757415116,0.6015896201,-0.7534538507,-1.1112320423,1.6509275436,-0.1514259428,-0.4161066115,-0.3126336038,1.3812828064,0.8161883950,-0.0387242921,0.7398338914,0.0209227726,-0.4717726409],[0.1601802260,0.7710543275,0.1447940767,-0.0283841789,-1.6753015518,-0.8868216276,-1.0859651566,-0.6444486380,-2.2279274464,1.3281667233,0.0351902060,0.0617455207,-1.5774900913,-0.5398665667,0.7255911231,-0.4908719063,2.2670347691,-1.9021373987,1.2350738049,-0.9386199713,0.7089691162,0.7957819104,-0.9514808655,1.2497242689,-0.3672064841,-1.3906115294,-0.5919629335,1.7621365786,-1.4293707609,0.3684018254,-0.7765325308,-0.1114501879,-0.9075161219,1.2576494217,-0.8669949770,0.4935116470,0.1877672523,-1.3370648623,-0.7650950551,1.1577278376,-0.6422299743,-0.6276326776,0.1233862042,0.2219813317,0.0170367789,-0.7312810421,0.6946656704,-0.7900857925,0.5711570978,-0.6337569356,-1.2564939260,-1.9684580564,0.7597616911,-0.6197546721,1.0595633984,-0.1620984077,-1.4307351112,-1.2425879240,0.3334033489,-0.8551697135,0.7142262459,0.2552497387,-1.4737071991,-1.2400416136,1.2253839970,-0.4654192924,-2.1136255264,0.6339798570,1.6607668400,0.7691323161,-0.0107232919,-0.3207734525,-0.8382658362,0.3205773532,0.0976580307,0.2763164639,-0.4415012300,-0.7790068388,-0.6566889286,1.4151773453,0.4216744602,-0.8975771666,-0.4491711557,1.3207405806,0.9556534886,-0.2062499523,-0.4255151749,-0.5410502553,0.7635242343,1.0545524359,-0.4757571518,-0.3823243380,0.3522265255,0.6711670756,-1.7673711777,1.4733440876,-0.5324310660,-1.6187958717,0.6993540525,-0.9426357150,-1.1725157499,-0.3552233279,0.5078729391,1.4298133850,0.2571879029,1.4719634056,0.5778535008,-0.2065822333,2.3589992523,0.2131077051,-0.3065221906,0.4680456221,0.1980422735,-0.8443328142,-0.2465468198,2.7550952435,-0.0163592007,0.0708196610,-0.3586863875,-0.1086756513,1.2919684649,0.9084424376,0.7995363474,0.3683059812,1.1267640591,-0.0028436235,0.1808570176,-0.1837261468,-1.2601875067,-0.2753442824,-0.0100960284,-0.1514759511,1.3774771690,0.2780973613,0.1607485414,-0.3234707117,-0.6531004310,-0.4414724708,1.2662265301,1.3459706306,-1.0425080061,-0.2603740394,0.0437238626,0.2833698094,-0.0748640820,-0.3282703161,0.6336637735,0.0157164261,-1.1069000959,0.0525508635,0.2700309157,-0.3761922717,-0.4791140258,0.4749079943,-0.2400727868,-1.6324510574,0.1481562406,-0.2036515921,0.5730488300,0.3368732631,0.6147609949,0.8890558481,-0.7400008440,2.0510735512,0.0945650861,-0.3494800329,-0.6844366193,0.1977184117,-1.5378758907,1.7580767870,0.4568970799,0.7572392821,1.4373345375,0.2000212222,0.0556829832,-0.2715509832,1.1920673847,0.8942832351,-1.2035263777,0.4194567502,0.4005192220,0.5703665614,0.5575336814,-1.3979532719,1.0963864326,0.0250807907,-0.0674761832,-1.3193664551,-0.3257948756,-0.1157574356,0.1749069393,1.5000028610,1.1391853094,-0.9547190070,1.0541182756,-0.2260276377,0.7414301634,0.9308434725,0.1472192556,0.2028086632,1.0589454174,-0.3633924425,-0.3617959023,-1.1625291109,2.2101442814,0.2819685638,0.1859583557,-0.2687607408,0.0440233089,-1.0015631914,0.6896772385,-1.3712488413,-1.0608513355,-1.1808810234,-0.4728468060,1.0679525137,0.9100573063,1.6535276175,-0.5239212513,0.5605090261,0.9432093501,-1.9324355125,0.8190538883,-2.1978337765],[-0.2977302969,0.6257901192,-0.1263576597,0.9088796973,1.5055203438,0.1521550417,-2.2133748531,-0.1194746345,-0.5715026259,1.5380450487,-1.0065170527,-0.5844327807,2.0164713860,0.0378994644,-0.1642175913,0.5951735377,-0.7040801644,-0.2119091898,-1.3055820465,-0.0513718873,-0.3308599591,0.0339758396,0.8075446486,0.1187833175,0.6131482124,-0.4814960957,0.0764794052,0.9820422530,-1.7076627016,1.3723251820,-0.7145006061,-0.5938451886,-0.0183429793,0.7288543582,0.8106418252,-1.5039621592,0.3948372304,1.0601956844,1.0729640722,-0.2177887261,-0.3614816070,0.6474339366,0.4636586905,0.1736662537,0.7257980108,-1.3463358879,-0.1328872144,-0.4435177147,0.2674212754,0.8134050369,-0.8270956278,-0.7177349329,0.6141757965,0.6915240884,-0.4994766414,0.5097733736,-0.1994809657,-0.9650039077,-1.3885537386,1.5378166437,0.5882808566,-1.0985815525,-0.8555923700,-1.3664350510,-0.5528749228,0.2774377763,0.6846002936,-0.6543120742,-0.9933399558,-2.6504812241,0.3394016325,-0.2626033425,0.4674544036,-0.2633951604,-0.2068358809,-1.0403506756,0.4053127766,1.1273319721,0.1939062625,1.3327549696,0.7872596979,-1.1306591034,-0.3493178189,0.1557210684,1.1165831089,1.3770676851,-0.7949782610,-0.6838003397,0.6526792049,0.6445679665,-1.1254947186,-0.2173337489,-0.8494017124,1.1666642427,1.2532933950,0.9628742337,1.0837029219,-0.6331955194,0.8385151029,0.3361526430,0.3915138543,-0.0933211073,1.1968898773,-0.5742031932,-0.0500956103,0.9402880669,0.7142255902,0.5690819621,0.7714705467,-1.5002479553,0.3130998313,1.1639127731,0.6727718711,0.0101976758,0.9451437593,0.5655166507,1.6275627613,0.8402985334,-0.3411326110,-0.3251051605,-0.7312197685,-0.2371633053,-0.1245433688,-1.7427054644,-0.6266471148,1.6334964037,-0.0915187076,-0.3417382836,-0.0914409012,0.6394451261,-0.9623674154,-0.2364961803,-1.0329637527,-0.1231606677,-0.2556661069,1.1051532030,-0.3228335083,-0.9258052707,0.6563531756,-0.4999623001,-1.6209917068,-0.1153591797,-0.8811206222,1.0089468956,1.2612160444,0.5947718620,-0.2692162991,0.2056351900,0.5102504492,1.7642725706,-0.5403218269,1.0996212959,-1.2504529953,1.0675100088,0.5051488876,-0.8187961578,-0.8085387945,0.0946609080,-0.1569521129,0.7736035585,-2.0229930878,-0.5482857823,1.0867170095,-0.1797525287,-1.0244921446,-0.3236004710,1.0927515030,-0.2026444674,0.4056845307,-1.0475788116,1.4953615665,2.0108199120,-0.3379966319,-0.3342337012,-0.4520601332,0.9962898493,-1.3105881214,1.2578663826,0.2881660163,0.5212141275,2.0398442745,0.8010252714,-0.7030718923,-1.5192036629,-1.2763825655,0.7861399651,-0.5948548317,1.0182152987,-2.0379059315,-1.1396664381,-0.4048861265,0.0318119638,0.8137828708,-0.8330386877,-2.0578346252,1.3120894432,-0.2637521923,-0.4078529477,0.2702302933,0.7227398157,-0.8606963754,-0.5009006858,0.4711194634,1.6186574697,1.4121378660,-0.0002848467,0.6598795652,1.2394162416,-0.5023958683,0.4566825926,-0.8102485538,-0.0185096450,1.5662649870,-1.0101697445,0.5655524731,-0.2134593427,0.0219880417,1.3343462944,-0.9169719815,-0.1619948745,1.0586124659,-0.3841233253,0.1626750678,2.1752021313],[1.5525988340,-0.2626132071,-0.6253407598,-0.3021035790,0.5044687390,-0.1912412196,-0.0795418695,-0.6908488274,-0.4781500995,1.0784960985,-0.4422915280,-0.4586786628,-0.1436938047,-1.0484225750,0.3351688981,0.9214617610,-0.1563158184,0.3626534641,0.6382313967,1.2679969072,-0.0452971682,0.7486544847,0.5606036186,0.9569472671,-0.0700276718,-0.3959464133,0.1904693246,-0.7078367472,-0.0979526788,1.9592298269,-1.1198773384,0.4000999331,-0.3875292540,1.4898318052,0.6270580292,0.5690717697,-0.3728976846,-0.5502703786,-1.1284275055,-0.0041970341,-1.6097604036,1.1527045965,1.6746832132,1.5408614874,2.0273838043,-0.5349593759,0.7125822902,0.7540348172,-2.8705582619,-1.8638910055,0.5038121939,0.6373669505,-0.5574759841,-0.5050622821,-0.8507771492,-0.5713256001,0.8568009734,0.5190439820,0.0653787851,-0.9294126034,-0.6523821950,0.4704572558,-0.8607774973,-0.1529620588,0.0826679990,-0.1757801771,-0.5495941639,0.7834180593,0.0671116933,0.7145394087,-1.3842253685,-0.7460412979,-0.6277463436,1.0182669163,0.0907923281,0.9149090052,1.0882586241,1.7943143845,0.4608672261,-0.0809834972,-0.8602779508,1.1938617229,0.6517693996,0.0092181843,-0.7635804415,1.2017304897,-0.8502430916,-0.1846573204,-1.6849205494,-0.2419226319,0.4188693166,1.2817186117,-0.4473271072,-0.1569877714,0.7320977449,-0.0141397016,0.4355955422,0.4715900123,0.6773759723,-0.4215749204,0.3264049292,2.4344141483,-0.0604711324,-1.0446228981,1.5301111937,1.3959794044,-1.4553323984,1.7583258152,1.5257213116,-3.0210578442,-0.9142537117,1.7957469225,1.1200355291,0.5840203762,0.5999845266,0.0407632738,0.6290143728,0.2058549970,-1.1277829409,-0.8097259998,1.6846119165,-1.2507945299,0.1569381058,0.7785435915,0.7929396629,-1.8754302263,-1.1775150299,-0.9298060536,1.7393478155,0.4945442080,0.4163727760,1.5628179312,0.2502279580,1.3238953352,0.4312912524,0.0220150761,-0.2147385180,-1.8523584604,-2.0134761333,1.4432740211,-0.5076486468,1.3906644583,0.5807424188,1.2673971653,0.2372799963,-1.5070856810,1.0928071737,-1.5873508453,-0.1693418473,-0.4200076759,0.2688405514,-0.3601242900,-1.2954751253,-0.7431051731,-1.2575507164,-0.2012025267,2.2574408054,0.1104647666,-0.1512969583,0.0174025111,0.4458542466,-0.7248215079,0.9082215428,-1.3589265347,-0.9368333817,-0.7312052250,1.0721185207,-0.7326382995,-0.5639464259,0.6095701456,-2.0346000195,1.0954475403,-0.2311936021,1.4384429455,0.5641620755,-0.2077282071,2.3088765144,0.9453276396,-0.1790513992,-0.6986765265,0.0849507675,1.8025000095,0.4350953996,0.7734685540,-0.3476094306,-0.3814012110,0.4308768213,-0.9413963556,0.5490882397,-1.2022879124,-1.0516980886,-0.9482901096,-1.6367952824,1.8386996984,-0.7118865252,0.1055972502,-0.7026752830,-1.0756582022,-0.1422663033,0.3937110305,-1.8437414169,-0.2732995152,-0.2043423504,0.5992166400,0.6439359784,1.7993170023,-0.7580192089,-1.3359285593,1.4962481260,0.1730103940,-0.6356427073,0.0779678747,-0.3810184300,0.4025068283,-1.0337539911,-2.1191465855,0.2678066492,-0.5662587285,-0.5005620718,-1.0199508667,0.2415617257,-1.9484695196,-0.5845148563,2.9574551582],[-0.1329732090,-0.2781083882,2.0026571751,-0.6654509902,-0.9315814972,-0.5124284029,-1.0220663548,1.5914081335,0.7247942090,-0.8515052199,0.6342797875,-0.4960220754,0.5928196907,-0.9952784181,0.8246176839,1.0320022106,-1.9398835897,2.0664064884,-0.9641517997,-1.8633273840,-0.0337739438,-1.3035702705,0.1331416517,0.1698399037,-0.1870108247,0.3671292365,-0.4189546108,-0.4695348442,-2.6624681950,0.5423961878,1.5010020733,0.3459192812,0.0675365776,-0.8721294999,-0.4595796466,-0.5068891644,0.5873795748,0.3420045674,-0.0998604968,0.4836119413,-1.3472015858,1.6239326000,0.6348768473,0.0179399494,-0.4565315545,-0.2640585005,0.8355404139,1.1033011675,1.5110306740,1.1274770498,-0.7093459368,-0.0744580925,0.2659905851,-0.1485776901,-1.6719353199,0.7489042878,0.0443355255,-0.9490557909,-0.8815689087,-0.6441028118,1.2353124619,-1.6454527378,0.3230223954,1.0776292086,-0.8387464285,-0.5173830986,1.4620894194,-0.0286578313,-0.7814294100,1.4929542542,-0.7920092344,0.3892004192,-0.4025924206,-0.9289677143,0.8831150532,-2.4916441441,-0.7269167304,-0.3389181793,0.0518149436,2.3087661266,-1.0090583563,0.9877404571,0.1650268286,0.9553805590,-0.7569466829,0.7209989429,-1.1299874783,-1.3067044020,1.1002798080,1.8736176491,-0.1802768111,0.9645875096,0.7066676021,-0.1683974713,-0.9512957931,-0.8372158408,-0.3797715008,-0.2867600620,1.0953788757,-0.4406148195,-1.3737680912,-0.8403751254,0.7122806311,0.6732238531,-1.2772501707,1.4470598698,-1.6477565765,0.8578764796,0.9182669520,1.3492974043,0.6436387897,-0.1199597120,-1.4940854311,0.1026622728,0.7803411484,-0.5787874460,0.9917155504,0.4909535050,-0.5574346185,-0.6866763234,0.2321932614,-2.1183209419,0.4316413403,-0.4042158127,0.8837202191,-0.1353183091,-0.7172310352,2.7005834579,1.1152646542,0.8556065559,0.3109346032,-1.0019065142,-0.1889768243,0.6665068865,1.8737750053,0.5609683990,-0.4682036340,-0.3340584040,1.0639505386,0.1632687896,-0.2742658556,1.0798113346,-0.6057754159,-0.2749799490,-1.1890770197,-0.2930025756,1.8735110760,2.2746007442,0.8219443560,1.9643771648,1.3195849657,0.1458032131,0.5102447271,-1.6404271126,0.6829751134,-0.6760859489,-0.7591422200,1.1757338047,-1.2378255129,0.8542348742,-1.6906770468,-0.5590405464,-2.0516254902,-0.7075557709,-0.6136775017,0.7814898491,0.3277451098,0.7363686562,-0.7694044113,2.5541281700,-0.9461388588,-1.0866278410,-0.6960321069,0.3335900009,1.2417874336,1.3287516832,0.9328956604,-1.0503815413,0.2631334960,0.5054531693,-1.5257029533,-0.2071426213,1.1814788580,0.2966313064,-0.9725993872,0.3309081495,-0.3167668879,0.6853064895,-0.2060380578,2.7066907883,1.5249216557,1.2280702591,0.5085874200,-0.3147315979,-0.1998735368,-0.7020621896,-0.7529008389,-0.6573159099,-0.4857104123,0.2419129610,0.4221928120,-0.8887619972,0.4519138634,-0.5847700834,1.5164148808,-0.3560203314,-0.9716092944,-1.1448581219,0.1373418719,-1.6014651060,1.3189338446,0.2547961473,-1.4622094631,-0.1789774597,0.8794451356,-0.3040826917,1.5860936642,-0.7125411630,-1.5092651844,-0.4747304022,0.4026572704,-1.7841160297,1.4312634468,0.6950429678],[0.4651306272,0.8870432973,0.0887216479,0.4096937478,-0.5062332749,-0.2962773740,1.1971416473,1.3719564676,1.9563847780,0.8283421993,1.7993576527,-1.2152526379,-0.0858739465,0.2326095849,-0.2911576033,1.0595628023,-0.5787106156,-0.4047865868,-0.5017714500,0.0693776011,-0.4276338816,-1.3622661829,0.4445276856,-0.7927712202,0.9814260602,0.5316746831,1.5762796402,0.8111576438,1.9288363457,-2.2223255634,-1.3883488178,-0.3999559581,0.1310270727,-0.3544170260,0.9905593395,0.2306950390,1.9439210892,-0.5027143359,-0.0143833691,0.3165681660,-0.1115248650,-0.0917987972,0.2415666580,-0.0787839368,0.4116333425,0.5505738258,1.1692513227,0.7651976943,1.2653208971,-1.2472923994,1.1245467663,0.4015773833,-0.2108438611,1.5653553009,-0.0090202373,0.3653865457,-1.0638052225,0.3883264959,1.2481312752,-0.1546858996,1.3234074116,-0.8675174117,2.0376651287,-0.7730429769,0.4148543179,0.2054794282,-0.8716959357,0.5587248802,-1.0899597406,0.5063377619,0.2100979537,-1.6693204641,-0.4869732261,0.8901972771,0.7649075389,1.4790863991,0.1984126270,0.2106683105,1.2196710110,0.4870115221,-0.8584403992,-0.6259924769,-0.4319318235,-0.2345139086,1.1433793306,1.2349792719,1.1137609482,0.3417917192,2.7702353001,0.5551794171,-0.7779662013,-1.3389432430,0.3691887856,2.0352487564,0.1049689651,-1.3625838757,-1.2054003477,-0.1510334313,1.4791287184,1.5493704081,-1.2234296799,1.1098680496,-0.3641369641,0.2689851820,1.0093431473,-1.0931015015,0.8034202456,-0.1910215169,-1.2416673899,-1.4777083397,1.0865781307,-0.2192025781,-1.4991483688,-0.3896990120,0.7066301703,0.6640823483,-0.2975024283,-0.1652295887,0.2889887094,-2.8535153866,0.6054579616,-0.0322866999,-0.8152453303,-1.0316278934,-0.4888164401,-1.3081847429,-0.1466890872,2.9084668159,2.2889723778,-0.4269299209,1.8235203028,0.6112687588,0.1520540267,0.2729745209,0.4491570592,2.5501518250,1.4125373363,-0.3937563598,0.6757388115,0.1855499148,0.8891111016,0.4965530932,0.6705762148,-1.0051658154,0.4118530452,-0.3306933641,0.7756954432,-0.1614267975,0.3098185360,1.0678449869,0.8791055083,1.5784460306,0.1212876365,-2.1881835461,0.2398423702,-0.6166628599,-1.0495116711,-0.2046049982,-1.5786684752,-0.4201811254,1.0646731853,0.6574370265,-0.4980138242,-0.9568085670,-0.3948968053,0.5613503456,-0.2417133749,0.0204319879,0.4374487698,0.9105584025,-0.1001281738,-0.0798568353,0.0035209819,1.7699304819,-0.8864417672,1.1483435631,0.9128133655,2.3566586971,0.0609659180,-1.8903415203,-0.9600875974,-0.0770855024,-0.2616542280,-1.2499434948,0.5119954944,-1.7796143293,-1.5820428133,0.8977500200,-0.7375071049,-0.7488090396,-2.2484743595,-0.8623650074,-0.7743161917,1.3640680313,0.4618594646,-0.4605868459,0.1531334072,0.2061138004,-0.7647488117,-0.0337435305,-0.7385146022,-0.5625520945,-1.2569856644,0.2129893452,-0.9553509355,0.0948104262,2.5250504017,-1.6878075600,0.6934902072,0.6625792384,-0.5822758675,-0.7410813570,0.0621082224,-0.0299436729,1.1133687496,0.4249279499,0.5673895478,0.1886189878,0.0354875848,0.1282968819,-0.6992625594,1.1561349630,1.9655958414,0.4005120397],[-2.1143636703,-0.5863888860,0.7493094206,0.5219535232,0.8459887505,0.5666259527,0.1399505585,-1.2476874590,-0.4998947680,0.3239963055,2.3878009319,0.4304938018,-1.2388671637,-1.0990287066,-0.3986670673,-2.1690292358,0.5610336661,-0.2620897889,-0.7111989856,0.2920810878,-1.6883717775,-0.1379259974,-0.9189206958,0.4376925528,1.5696545839,0.6147102118,0.5508036017,-2.5583667755,0.3781401813,-0.5038151741,-1.7127869129,-1.4350970984,-0.1412201822,-0.0816503242,0.4081589580,0.7809235454,0.6255474687,-0.2427212447,-0.2043065876,-0.6814991832,-1.8603205681,-0.5025548339,-1.9000107050,-0.1304477006,-0.6851273179,1.0265516043,-0.6661650538,-2.3975861073,-1.7486999035,0.0605677813,0.6212993264,0.4197052121,-0.3831737936,1.3025627136,1.1073590517,-0.8643981814,1.3570622206,-0.4238911569,-1.2117905617,0.1408114433,-0.1960694492,0.6899011731,-1.3563886881,1.4247977734,-2.2074992657,-0.5906651020,0.6465181708,-2.1144294739,0.8956786990,-0.9129745960,2.0979588032,-1.1935856342,-1.2795675993,-0.8881188035,1.0661122799,1.6177762747,1.1079305410,-2.0232996941,-1.2102463245,1.5625468493,1.1707061529,0.2248326093,-2.2631869316,0.3136663437,2.0882434845,-1.1118528843,0.8269920945,2.2001664639,-1.0525869131,0.1732597351,-0.6301019788,-0.2861230671,-0.7035486102,-0.6158868670,0.6836638451,1.2540742159,1.1350761652,1.1168063879,-0.0671314672,-0.5278354883,-1.6708488464,-1.8426793814,-1.4898799658,1.5074266195,0.8699860573,0.4585454762,-1.2405214310,-0.3520327508,0.2623713315,1.2189782858,1.9564009905,-1.8678971529,0.2386119515,-0.3934918940,0.0209807158,0.0772104561,-1.7392042875,-3.5995438099,0.9020640850,-0.1195738018,-0.5754822493,-1.1097879410,-0.1072186753,-0.9343097210,1.1105796099,-1.1826517582,-0.1520720869,-1.3386286497,-0.6448604465,-0.0941165313,-1.3919219971,-0.0296481308,0.1762068868,-0.2783095539,-1.6376761198,-1.0517594814,-0.6948207021,0.0998614058,-0.6569946408,-1.8207826614,-1.1987988949,0.8665890098,-1.1092697382,-1.3847301006,-0.3812083602,-0.3967207372,0.7238845229,0.1558206677,-0.3647925258,-0.5851247311,0.4368338585,-0.7570932508,0.0958884582,0.7646985650,0.6894189119,-1.3329033852,0.0163381044,-0.3504721224,-0.7617780566,-0.8003528714,0.9703143835,0.2425616533,1.4180577993,0.8548147082,-0.4770228565,-2.0487129688,-0.4198290110,-0.5687003136,-1.3870117664,-0.5220409632,1.4553519487,0.5901443362,-2.1908631325,0.0048135603,-0.9869437814,-1.1687397957,-0.6182921529,0.3242558539,0.1612989306,-0.5659643412,-0.6204778552,-0.6357564330,-2.4414963722,1.7249587774,-0.4532583356,0.7349098921,0.6877690554,1.6072475910,1.3938087225,-0.1504776627,0.1785371900,1.7748429775,-0.2517042160,1.1803649664,-0.4024221003,0.2015507370,0.2357416600,-0.1340533942,-0.1057958901,1.6359223127,0.3880910873,-0.3377050757,-0.4242796600,0.1702330261,1.4504141808,-0.2115017474,1.4328291416,-0.5159255266,0.0500065312,-1.7297106981,1.3677421808,-1.0256661177,0.7512863278,-0.5840461850,-0.9218277931,-1.3074322939,-0.7596263885,-0.1303110570,0.6998056173,1.6276044846,0.9300687313,-0.6382806897,0.6924938560,-1.4637494087],[-1.0807045698,-0.7944297791,-0.2523753047,-0.9769495130,0.0696765333,0.0790520534,0.4146587253,-0.3838759363,-0.8621741533,0.5710561872,0.5936986208,0.4419238269,-0.2544811368,-0.0549358837,0.2354987264,-0.5314922929,-0.5649043918,-2.2214460373,0.5590788722,1.1864672899,-0.7585781217,-0.6145302057,-1.5558300018,-1.4099539518,0.8280416727,0.9231147766,0.7842741013,0.9364667535,0.1571408212,-0.0711527541,-0.3171508014,0.5082898140,-0.4674690962,0.2489854246,-0.6435582042,0.1705356985,0.0941540748,2.0618970394,-1.3521838188,-0.6742211580,1.9083428383,0.3393023908,-1.2060261965,-0.3309186101,-0.1274856180,1.2129625082,0.0350924097,0.8869325519,0.4675163925,0.3396925926,-0.6079598665,0.8370874524,-0.9059494138,1.4852802753,-0.2368526459,0.5075811744,1.3175470829,0.9353399873,-0.5524255037,0.1175686345,-2.5573158264,-1.6868667603,0.2753569782,1.0403324366,-0.6371220946,-0.2227719426,-0.2769407034,-0.5384197831,-1.2360104322,1.6299774647,0.3417989910,-0.1112552434,0.3643215597,1.7209881544,0.5956207514,0.1271493584,-0.3662891388,0.8358281851,-0.4693106115,-1.0826193094,1.5552965403,-0.3990328312,0.3532243371,1.3296083212,-0.1214048937,1.4217294455,0.2872732282,0.0609251931,-1.2650548220,-0.2146121860,-0.5631961823,0.9530636668,-0.5803635716,1.0396372080,0.0411144495,0.4864621758,-0.6175361276,-0.0901167020,-0.4780751169,1.1069477797,-0.1862807423,-0.8247175217,-1.6179413795,0.7025146484,0.3208766282,-0.1970423460,0.7278860211,1.6373358965,0.6480888724,-0.8358708024,0.6518102288,-0.1171416640,-1.0953273773,0.3416704834,0.2952704728,0.0544999093,0.1129286736,-0.2682801485,-0.7968363166,-1.3915562630,-1.0005905628,0.0861226767,-0.2342195213,-0.2970535755,0.3015836179,0.6606718898,-0.3637237549,0.2571111917,0.4363758266,-0.2371700704,1.7235711813,-0.2948478460,0.4730113745,-0.1083832458,0.9850977063,-0.3806095421,-1.6842314005,-0.3037002683,-1.1704878807,1.0020384789,0.3830957711,-0.6013025641,1.1467078924,0.0981238931,1.7418649197,0.6734358072,0.3837615550,0.2948663235,1.7496265173,0.6189327836,-1.7904872894,-0.4210635424,0.9242000580,0.6071117520,0.6344060898,-0.7756280303,0.8779548407,-1.7009711266,1.8138630390,0.3526122868,0.1569912732,-0.2284603566,0.4965924919,0.1646198183,0.5903906226,0.6860850453,0.9792392254,0.8350661397,0.2694632113,0.5770038366,0.6414442062,-0.0007064881,0.1891670227,3.3849787712,0.3436114490,1.0508141518,0.8994019032,-1.4829807281,0.5896140933,-0.7613607645,-0.2228245735,-2.2727417946,-0.7928166986,0.8713169694,0.7702066302,-0.2003716528,-1.1520656347,1.3296504021,0.1453818381,0.1850291640,2.1702175140,-0.0070129517,-0.9936382174,0.7370295525,0.6775777340,-0.3520190716,0.2269107550,0.3641117215,0.2620734572,-0.3037872612,0.7699797153,-0.5900830030,-0.0836289078,1.4655891657,-0.4953507483,0.9964833856,0.6668241024,1.6373746395,-1.0739023685,1.5138012171,-0.9158896208,0.3178542852,0.1864966750,0.7493636012,-1.6645890474,-2.7454783916,-1.2306253910,-1.6678169966,0.2192802429,-1.4617450237,-0.8441195488,-0.5457106829,-0.5780082941,1.8181878328],[0.0347079597,-0.3133533299,-1.1121459007,0.7268883586,0.3151122928,-0.5868066549,-1.7393980026,1.4476518631,0.4207965732,0.6836621165,1.8285927773,2.0717456341,1.6458361149,-0.2653712630,1.2877157927,0.4153696597,-0.8352056146,-2.2649061680,0.0756139308,-1.1362289190,2.3471145630,0.6214581132,2.1959691048,-1.1375454664,-2.3548367023,0.2680402696,0.2878903747,-2.3156638145,0.7746370435,0.5787066221,-0.5267652869,0.7834333181,-0.6082605720,-0.2690319419,0.3098383546,-2.3032941818,-0.5287114978,-0.4524430037,0.5710779428,0.2465500385,-0.0645666569,0.3519504070,-0.5015529990,0.3956600130,1.3372106552,0.8897716999,-0.4648024738,-0.2677031755,0.2189188153,0.4962541163,-0.4791700840,2.3462817669,-0.5984350443,1.2644433975,0.1036138311,-0.6658110619,0.0362412110,-0.2366671264,0.1581058204,1.3862762451,0.6495568156,-0.2329453528,0.0977031738,-0.5559794307,-0.8441994190,-0.0818201452,-0.1046767831,0.1273758411,0.4763332605,-0.1459098011,-0.1045340598,0.7774121165,-1.0574012995,-0.3140885234,-0.1191266701,0.8644908071,-0.0673007965,-0.6035963893,-0.5750159621,0.1821621656,0.6175658703,-0.7133141160,-1.4616454840,-1.5156626701,-0.5073814392,-2.4346702099,-0.7047346234,0.9782006741,0.8120400906,-0.4308604598,-0.4996556640,-0.5441598892,-0.2324918956,-0.7524700165,0.4344911575,0.1049314365,-1.2540309429,0.8012621999,1.1741023064,0.1364507526,0.0647116527,1.0092266798,-0.9059830308,-2.1721153259,1.1897959709,-0.1393132359,-0.8553556204,1.7093845606,-1.1029998064,-0.1866924912,1.3107669353,1.1628562212,1.0969941616,-1.2107185125,0.5536623001,1.1172728539,-0.2946329415,1.9040652514,0.0030044951,0.0876778588,-1.6686580181,0.0381268524,-0.3123620749,-0.5600023866,-1.0734754801,-1.3627321720,-1.2353109121,-0.0829710588,-0.1355157346,-0.9162235856,0.3678059578,-0.2945487499,-0.4046669006,0.8659502268,-0.5342887640,0.8775099516,0.2651894093,-0.0377184190,0.7347764373,1.7639116049,0.0837946832,1.2767306566,0.5433607101,-1.1277854443,-0.4092319906,0.3871065974,-1.3732262850,0.0484902039,0.3826854825,-0.8100257516,0.5566810369,-0.0652687475,-0.5655643344,-0.7411604524,-0.2269089818,-0.8638867736,0.9950011969,0.2840652168,0.5898693800,0.1182757393,-0.6138235927,-0.4195107520,-0.6143563986,0.2658936679,-2.7899060249,2.2320573330,1.3054738045,0.5085213184,-2.8490052223,0.5092891455,-2.4805524349,-0.7394009829,0.6821433306,1.6671719551,0.4611730278,-1.3810774088,0.7877162099,1.0247899294,-0.4438346326,1.6174777746,-0.5054686666,0.1760485619,-1.3114635944,0.4309389293,0.4800304770,-0.1087075472,-0.5780231953,-0.7047863603,-2.1476862431,-0.2139531970,0.8352521062,0.8809152246,0.2713364661,0.3603095114,0.3651247323,-0.8552098274,-1.1180651188,-0.6216140389,1.9746416807,-1.6354407072,0.6205993295,-0.3181858361,-0.5416350961,-0.8335084915,-1.0118873119,0.4987774491,-0.3509887457,-1.1358166933,-1.7522422075,0.6429727077,-0.1995020509,-1.5381036997,0.1544527858,-1.2883707285,0.5439504981,-0.0020331419,-0.2399383932,2.7470574379,0.5867281556,-1.3848453760,-1.5909763575,-1.3414326906,1.2320175171,0.9906088114],[1.5391710997,1.7031075954,0.1598866582,0.5140388608,0.7190224528,0.2349463552,0.8574712276,-1.1896023750,-1.2586706877,0.2605868578,-1.1289823055,0.5695583820,0.5411376357,0.3698288500,0.3675069809,-2.2430713177,-0.6659237146,0.8024030924,-1.3820905685,1.0470803976,-1.4281877279,-0.3887173235,0.3055648804,-1.5657507181,-2.1697707176,1.4391497374,-1.5002782345,0.8511645794,0.5123485923,-0.0351729281,-0.1442015767,-0.3420092463,1.5379468203,0.9169666171,0.0578503907,0.2137975246,0.1961234212,-0.5389728546,-1.6089800596,0.4754639268,0.2233437747,0.1733196378,1.2684612274,-0.2865017951,-0.1468806267,2.4246311188,-0.0136099840,-0.1019738093,-0.6110169291,-0.8038156033,-0.0695337504,0.4967861176,1.9961833954,1.2340877056,-0.0171101298,0.0730238631,1.2438621521,-0.5428292155,0.5201815963,-2.0732119083,1.0970755816,1.7306132317,-1.7191520929,-1.0230888128,-1.6075240374,0.6967043877,1.0618695021,-0.3732878268,0.6697552204,-0.9198107123,0.7653980255,0.6345857382,0.3547610641,0.7504929900,-0.3338160217,-0.5197306275,0.9472274780,0.2227191478,-0.1130388081,-0.0185331739,0.5474200845,-2.2555432320,0.4503259957,0.9500397444,-1.8120361567,-0.9349394441,-0.0216028914,0.5007613301,-1.4258812666,0.9653455615,-0.2123300284,-0.9972632527,0.4599822760,1.4621566534,-0.9623032212,0.0821985975,0.7254318595,-0.8844993114,-1.1279693842,-0.0905174986,2.1163721085,0.5569891334,0.4357909858,0.1275379509,-0.3727503419,-1.5048179626,-0.9216700792,2.6693191528,1.8621063232,0.0340112634,-0.2996329963,0.1685769856,-0.0407568179,0.2233879268,-1.7338620424,0.4193114340,-1.7407432795,-0.3416656554,1.9146230221,0.7406899929,-1.3047709465,0.4453069568,-1.8995198011,0.1776600182,-0.1196291074,0.6559990048,0.4963692725,-0.4981673658,-0.2931072414,-1.5256278515,-0.3368673921,-0.0629800633,-0.3711330593,-0.4460727274,-0.1216611713,-0.7857735753,1.7829257250,0.5151084661,-1.0354783535,0.3958749175,-0.7858343124,0.3499429822,-0.4504361749,1.8571742773,-0.5264532566,0.8362266421,-0.6549919248,-2.2250118256,-0.3075691462,-0.7183296084,1.3063586950,0.2724380791,-0.7761501074,-0.3567362130,0.0337915644,-0.6230852008,1.2878644466,0.8142136335,0.7467980981,0.0545031875,-0.7823157907,-2.0375680923,-1.5598759651,0.7258884311,0.2529396415,-0.1873801351,-0.2569098175,0.9535322785,0.3202666640,0.9372274280,-0.7806515098,0.0057575130,2.1948187351,2.1471142769,-0.3790979385,1.2801339626,0.6664164662,0.8390047550,0.2793636620,0.3506715596,-0.8946903348,-0.1892145276,-0.2091954798,0.8307814598,0.7596340775,-0.6366750598,-0.2397034466,0.0246799253,-0.7540863156,0.6899034381,0.7105519772,1.9139291048,-0.9092925787,1.0814040899,-1.6866900921,1.2264014482,-0.0857531503,1.1177709103,0.0718116313,0.6214321852,0.8597166538,2.5825061798,-0.0952331051,0.2181185037,-0.2722103000,2.0228359699,-0.4731457233,-0.4897209406,-0.8466237187,-0.7927527428,0.2535848320,-1.7146792412,1.4753947258,0.1289210320,1.0652494431,0.8515291214,0.4173741341,-2.0765564442,1.2504419088,-0.0443437658,1.5116868019,1.8558517694,0.3491135240,1.6937165260],[-1.0338196754,-0.4067380130,0.8262225389,1.4143344164,0.8692837954,-0.2377124280,-0.3093731403,2.4539537430,1.8264842033,-1.1981890202,-0.0252850149,0.4000261128,1.0661538839,0.8577992916,1.7236268520,-0.3746747077,-0.6475515962,1.0553491116,1.0886489153,1.0331463814,1.4672757387,-0.2489519864,-1.0970305204,0.9412765503,1.0543107986,-0.0536026321,-1.0659681559,-1.0156000853,-1.5400930643,0.4246522188,-0.0187172014,0.8591113091,0.5056961775,1.0604348183,0.4011115432,0.6088192463,-0.4373769760,-0.4630948007,-0.1633478701,-0.0268928893,-0.9721285701,-0.0090062944,-1.8252615929,-0.8835651875,0.8288798928,0.9291929603,-0.6319452524,-0.7508799434,-1.2358655930,-0.2250781059,0.8632010818,-1.3284317255,-0.1848070771,0.9850524068,2.7465870380,0.3367196023,-0.1066792235,0.6596503854,0.5575050712,1.2563635111,1.6511420012,-2.0832810402,-1.7445597649,1.0956785679,-0.3525366783,0.7681734562,-0.8345808983,-0.4927855432,-0.7883468866,-0.7326194048,-1.2440841198,-0.0345288664,-2.7205607891,0.6942610145,0.7805992961,1.9402210712,-0.3773797154,0.6861529350,1.4874285460,-1.1206152439,0.4860053062,-2.2346808910,1.6082966328,-1.4920960665,-0.0897716880,-0.9688981175,0.2630314231,-0.2303248346,0.7283701301,-0.2108328491,0.8876338601,-0.0404112227,0.3710825443,-1.7251782417,1.6026358604,-0.5407242775,-1.5847551823,-0.2540806532,1.0504814386,-0.6451629400,-0.2521845698,-0.7335251570,1.4679886103,0.9870139360,-0.2953719795,-0.6325708032,-0.2677554488,-0.6401776671,-0.9570604563,0.7377889156,-1.1807512045,0.2154497951,2.0238044262,-0.0856862068,-0.9026969671,-0.7318025231,-1.8274935484,-1.2555172443,2.3201255798,-0.0519613996,0.1016197801,-1.6232725382,-0.1265503317,0.3801241219,1.2250181437,-0.8491208553,-0.0472059064,0.8237924576,0.0123016173,2.5513939857,0.4294205010,0.9571111202,-1.0746427774,0.7808172703,-0.0097871711,-1.0527443886,1.9884264469,1.4375798702,0.1371785998,1.1927306652,0.5296163559,0.7076142430,0.7082402110,-0.0101034651,-0.9877721667,-0.7654364109,0.8536295891,2.0100181103,1.7068982124,-1.0156911612,-0.3960444927,1.1031227112,-0.5324784517,-1.4763936996,-1.0825276375,1.8315578699,0.2433653921,-0.8337994218,0.4364992380,-0.0291025154,-1.1196365356,-0.6079728007,0.3150257468,0.9560776353,-0.4641246200,0.0915727168,1.4087785482,0.5882574320,-1.0789182186,1.1352727413,0.3621936738,0.7451221347,0.9481421113,1.1590278149,0.3194058836,0.2874425352,2.1013104916,0.5960393548,-0.9192075133,2.1953134537,-0.2614995241,-1.1423128843,-0.6494238973,1.2167494297,-0.4384835362,-0.4017018974,0.9502454400,1.7171781063,-0.3878672123,0.1367998719,-0.2946698964,-0.3433934152,-2.0157806873,0.3329694569,0.1684727818,-0.6111791730,-1.2262228727,-0.3766339123,-0.0963002965,2.6844952106,-1.4412274361,-0.3843735456,-0.6183712482,0.2814792395,0.4891471267,-1.6365194321,-0.5198187828,-0.4364172220,-0.1316254586,0.7158564329,0.9850757122,2.1309149265,1.7007933855,0.4094660878,-0.4367023706,0.1017921567,0.9821092486,-1.8584427834,-1.8049651384,0.3871385157,-0.7662376165,0.8493132591,0.8043100834,-0.7121513486],[-1.0511052608,1.5637362003,-1.4827083349,-0.5826970339,-0.1382329762,-1.4593697786,0.8752971292,1.5086021423,0.0051624221,-0.6862916946,-0.0043434026,-0.1981390715,1.6246167421,-0.7251117826,1.4283695221,0.3859604895,1.3972984552,1.5540999174,2.5077965260,0.6106916070,-0.2925674021,-0.6402983069,1.1182700396,1.3853303194,-1.2084366083,-0.0332357697,1.1627514362,0.3403298259,-1.4742140770,0.1268104017,0.0190874953,-0.2759317756,1.9947268963,0.3381502628,-1.2353051901,0.1884480715,-0.0715656877,-0.4184769392,1.6307207346,0.8701319098,1.6656786203,-0.2166256756,-0.8495945930,-0.9305421114,-0.2624989152,-1.0245912075,-0.3905347288,-0.8387540579,-0.7649152875,0.7450212240,-0.1490360945,-1.2048982382,0.0052565383,0.2848755121,0.2007502466,1.8125782013,0.6278348565,0.3501205146,-0.7780155540,0.2294713855,0.2744221985,-0.7625269294,1.2621465921,1.7222411633,2.0579175949,-2.1386463642,0.1349654347,1.3889241219,1.0150574446,-0.1360161752,1.1493418217,0.9397243857,-0.2867267430,0.1947177649,-0.4196768999,-1.1011104584,-0.4494042397,-2.1123704910,0.8957493901,-1.3408659697,-1.5817927122,0.5287882090,-0.4619643688,1.8015409708,-0.3952269852,-1.9611263275,0.7645410895,0.0038085454,-0.5192066431,0.6467928290,-1.2985756397,-1.0088950396,0.0489339791,0.7028077841,-1.9100688696,-0.2068070322,1.0011336803,1.4111698866,-1.7828242779,1.2758779526,1.0975503922,0.1588366926,1.1677552462,-0.1960469037,-2.4964635372,-0.5020163655,0.4433894455,1.3550033569,-1.8140020370,1.3322262764,-0.0272594821,0.5605358481,0.6819627285,-0.3883218765,-0.4051531553,1.9179229736,-0.1898309290,-0.4401783049,0.3061546683,0.5527454019,-0.0615562722,1.1755992174,-0.3579424620,0.6841399074,0.8924797177,1.0648961067,0.3632277846,0.5677139759,1.2489463091,-0.1964088529,0.6993002892,-0.3229619265,0.2260445952,0.9134680629,-1.1335862875,0.8838149905,-0.2615242898,-0.5182797909,-0.7718955278,0.6097289920,-1.0060567856,-2.0341730118,2.3055205345,-0.0929920897,0.8716686368,-0.9269929528,-0.2685537636,1.5324709415,-1.0759367943,0.1769119054,0.6477789879,0.6450597644,-0.0311753843,-0.3823924959,0.9851641655,-0.0099959550,0.3076856434,0.8313928843,-0.9556804299,0.9590794444,-1.6914818287,0.1404424012,0.1872563809,0.6698793769,-0.6741681695,-0.6608606577,-0.5262241960,1.7734227180,0.5527629256,1.2757896185,0.1524516791,-1.0622953176,-1.8591362238,0.6258513927,-1.0650826693,0.1954589635,0.0974331871,-0.5126291513,0.7299199700,0.3523078263,-0.3958586156,0.3189874291,-1.6703007221,0.1114622653,2.7657816410,0.3965610564,-0.3496138453,-0.9666339159,-2.0601592064,-0.5482038856,0.7389851809,0.7662830353,0.6029362679,-0.3516857326,-0.2998183370,-1.4955811501,-0.3838070035,1.3036487103,0.5547218919,0.3298881352,1.1064683199,0.9430986643,0.3839015961,1.4636781216,0.5495438576,-1.0143105984,-0.1891726404,-2.5100672245,-1.1935496330,0.2183128446,0.9599092007,1.5681704283,0.8648939133,-0.5642094612,-0.3421497941,1.4679930210,0.1009292603,-0.1126660332,-0.0972139761,-0.9727141261,1.3819562197,-0.2575739920,0.2227812111,1.6426393986],[0.5266095400,0.3665766418,1.4545422792,0.8404076695,-0.8013294339,0.9210820794,-0.6172357798,-0.4117927849,-0.7112870812,-0.2017239034,-0.0910126418,-0.8601288199,0.0719281286,0.7666918039,-0.0444614887,0.2221361250,-1.0537574291,0.6026690006,0.1948057115,-0.5820829868,0.9025380611,-0.1448483169,-0.1274862140,0.0873810202,2.6046392918,1.4802066088,2.0170738697,-2.2271463871,2.0235662460,0.1746399105,1.1856992245,0.3476963341,-0.1823394597,-0.9577738643,-0.6233409643,-1.0143755674,0.8863685727,1.2911611795,-0.7955857515,1.9125989676,-0.7551600337,-2.2882008553,0.6161273718,0.1451403648,-0.7093462944,1.2617323399,-0.6054198146,0.2747418582,0.4786858857,0.6184700727,0.0912145898,-1.2048335075,0.3290942907,-0.2360955030,0.5835716724,-0.9597758055,1.7403790951,1.5781999826,-0.7483271956,0.5618293881,1.2583650351,1.1227834225,-0.4328562021,0.1996002942,-2.6929018497,-0.8850705624,0.3370223641,1.0766296387,0.2751036286,-0.0249971207,0.3484513462,-1.4661811590,-0.5508713126,0.7615688443,0.2132720649,-0.5305895209,0.5651596785,-2.1194589138,1.5398962498,-0.1179601401,-0.0704612285,0.8395457268,3.8448247910,-1.6787865162,1.2805011272,0.5926063657,1.0369732380,-0.9114980102,0.7235304117,-1.1771003008,-0.8924003839,1.5978136063,-0.9991945624,1.4228422642,-0.2267849296,-0.8862344027,0.7951184511,-1.0446032286,1.5943152905,1.0219110250,-0.5544071198,0.0632716194,1.3749527931,-0.3679391444,0.8503480554,-0.5020872355,0.6588804126,-1.8511382341,1.3213031292,1.8603336811,0.5004802942,0.3071632683,-0.2847483456,0.2404857427,-1.5341200829,-0.3211204112,2.2744309902,-0.1365518570,0.8923602700,-0.3667230308,-0.5734806657,-0.3486274481,-0.1186143011,0.8307850957,0.6011481285,-0.7103462219,-0.6508090496,-0.8464833498,0.6043699980,0.2763639688,-0.1737786531,-0.0291063003,1.4368228912,-1.5170634985,-0.9887233973,1.1650506258,-0.8643849492,1.2665343285,-1.3787747622,0.3148807883,-0.8181608319,1.1779116392,-0.7715457678,1.0275965929,-0.1303185225,0.1004789919,-1.2106908560,-0.0878942758,0.1088947505,0.6320188642,-0.9480621219,-1.3948607445,0.0503652915,-0.3959770799,-0.1595087647,-1.2462719679,-0.8731278181,-0.3450042605,1.7942147255,0.4963313341,1.0791658163,-0.0937815085,-1.0531029701,-1.2803995609,-0.3876163363,0.3790248632,1.1142001152,-0.4398929477,-0.8452531099,-0.3301915526,-0.3507989347,-0.9891392589,-1.3544497490,0.9459623694,0.2598392963,0.5682955980,-0.9399594665,-0.2605684400,-0.8619480729,-0.5104118586,0.5194774270,-0.9169836044,-2.6900908947,2.7366669178,-1.4323606491,1.6291747093,-1.4959498644,-1.1548160315,2.5480496883,0.2900446653,1.4242911339,-1.2634080648,0.1963342428,1.4480313063,0.3778089583,-0.6415191293,-0.2120681107,-1.3008939028,0.3545155227,-0.2053026557,-1.9529192448,-0.0525757596,-1.0794268847,0.6703519821,-0.4758709371,1.1867605448,-0.3449138403,0.3547198176,1.6804623604,-0.9776315093,-1.9091552496,0.6955775023,0.9033918381,-0.1566496491,1.4611063004,-0.8640043139,-0.9245055914,-1.2249439955,0.7369412780,1.0058851242,0.5710602403,0.1333852112,-1.2250558138,-1.7569497824],[-0.5837014914,1.0665876865,-0.8548253179,1.4985412359,-1.5736359358,1.4790285826,-0.5080267191,-0.4287172854,-1.9432892799,-0.6099470258,-0.2015367150,-0.8354988098,0.1913418621,1.0483424664,-0.2596043646,-1.7170075178,1.5210067034,-0.6716116071,-0.0163926631,-0.4215934277,-0.0219555702,-0.7757620215,-1.5904581547,-0.2381323576,-0.9930461645,-0.7130764127,-0.0591275357,1.0727508068,-0.3543343246,3.0782167912,1.2767871618,0.7470397949,-0.9672718644,1.4354689121,-1.2770084143,-1.4952831268,0.6237402558,-1.3215762377,-1.4172451496,-1.2935512066,0.5550708175,1.3703895807,-0.5524062514,0.4399808049,-0.4097305834,-0.3165017962,0.3495762050,0.7616158724,-1.1521909237,-1.0199534893,0.7851632833,-0.7123095393,-1.3383508921,-0.6731940508,1.0400739908,-1.7057328224,-1.6736978292,1.1462876797,0.6484333873,-0.7210823298,-1.1814935207,-0.0193868037,1.2531992197,0.1176470742,0.1281784475,1.0190414190,0.3068197668,0.2843888700,0.6067525744,-0.4399292171,-1.8810433149,-1.7169803381,0.1544450819,0.3690387607,0.1635134518,1.8912612200,-1.0243716240,0.8578745127,-1.1951819658,1.0820078850,-0.0740952268,0.4703108072,-0.4310616255,-1.6714330912,-0.6286044717,1.5786759853,-0.2143421322,1.0809478760,0.4545911551,-0.6744666100,0.0244885739,0.5634146333,0.1647529304,-1.3875241280,0.2178342938,-0.0701511055,1.8721164465,0.2166976780,0.5953648686,-0.3440511525,-0.3522693515,1.8573738337,-1.2786791325,-0.6153733134,0.7561426759,-0.5053442717,-0.0918319449,0.5894739628,0.1524143517,-1.6383651495,-0.4843915999,-0.2537766397,1.3841521740,-0.7449327111,0.0421048775,1.2323815823,-0.7230997086,-2.1666941643,0.0070414278,-0.3546690643,-0.6129224896,0.5168722272,0.0161491167,-1.3249355555,0.2833559513,0.2064211667,0.6386342645,-0.2798440158,1.6235107183,-1.9121247530,-1.0733910799,0.7745764256,-0.2634502351,0.1581316441,-0.2726663649,0.2395518273,0.0382109508,-0.9823060632,0.2630215287,1.1353650093,-0.9041807652,0.3829956353,-0.0649039596,-1.3555053473,0.7701690197,-0.8610923290,0.8052645326,-0.8781912923,-0.7311577201,-2.9039270878,0.0947883651,0.4915755689,0.6707758307,-0.3843032420,-0.0357921906,-0.5924118757,0.9842528701,-1.4029403925,1.2588713169,2.0025391579,-0.3116514683,-1.9610450268,0.6682580709,-1.6717112064,-0.5060207844,-0.3277585804,1.4305614233,-1.0951304436,0.2470769286,-0.0456573442,-0.3003408611,0.5671326518,0.0664090812,3.3978607655,0.1929901093,-0.3330067396,0.0479260944,0.6180526018,1.0526547432,-2.1311256886,0.1027629226,0.8785018921,0.0608107410,0.4887917042,0.6694629788,0.3460426033,-0.4522186220,-1.4803252220,0.0475699566,-1.0768768787,0.2929372489,0.2556186914,-0.4252198935,-1.8915692568,-0.2100862265,-0.0845040902,-0.7917732000,-0.4677768350,-0.4222314358,1.2808054686,-0.3017964065,-0.1799859554,-2.1371099949,-1.6400431395,0.5384729505,-1.4935162067,1.4454184771,0.4328987002,-0.4091602862,1.2936794758,-0.7945672274,1.1871507168,1.5372850895,-0.8579943776,-0.7523997426,-1.3768255711,-0.5170516372,0.5266361237,0.5500448346,0.2686732411,1.1339052916,0.1505801380,1.0301640034,-0.5608243942],[0.5116033554,-0.1938695461,-0.2318379730,-0.6758121252,0.6146999598,0.7851496339,-1.3886620998,-0.3124918342,-0.1945108175,1.3500403166,0.5355877876,-0.6446334124,-0.3353732824,0.1246494949,1.4483478069,-3.0476114750,1.8138883114,-1.7988743782,1.6526031494,-0.1701273024,1.5951917171,-0.4962559342,0.5138649940,-0.1693724394,0.2989767194,0.9507718682,0.1399330646,-0.2367779166,-0.5848656893,0.0408673994,1.1533704996,-0.1025816724,-1.8772822618,0.7537982464,-1.0198754072,0.0566727556,-0.8193967938,0.6501716375,-0.6657895446,-0.7310741544,-0.4085451663,-0.9486625791,-0.0811254904,-0.2931586504,-0.2771906257,1.2176634073,-0.7611948252,0.5342000127,0.2206405699,-0.3270626366,1.6594908237,-0.9000232220,-1.8313491344,1.1096508503,-0.1459466666,-0.6582320333,0.3887246549,-1.8797705173,-1.1226110458,1.8616946936,-2.1927931309,0.0301647671,0.0473821871,0.6955246329,-0.5873502493,1.2350827456,-0.7400141358,1.6582258940,0.8974621296,-0.8383839130,-0.1603887081,0.4101619422,2.2152121067,-0.6797272563,-0.2635640800,-0.3556258082,-0.7574476004,-0.1034703329,0.2876872122,-1.0428086519,-0.5722566247,1.1083568335,0.5434480309,1.1623055935,-0.1919558942,-1.3977537155,0.3964772224,-0.3976658583,1.0213907957,-0.6081345081,-0.1984753758,0.7719884515,1.6116856337,-0.5647926331,-2.0718548298,1.2286254168,-1.5091139078,-1.1186282635,-0.2798033059,0.0749905184,0.3844977021,-0.0438706018,-0.2827286422,1.2734726667,1.2332676649,0.5621038079,-0.1019901857,-0.4522658288,-1.5339000225,-0.7295792103,-1.2359548807,-0.5382317901,1.6926423311,0.0574849769,0.5758835077,1.9828463793,0.5544055104,0.1886905134,2.5439355373,-0.3392686546,-0.6245530248,-0.4217739999,-0.1226378530,-1.1134990454,0.3403982222,-0.2209866643,-0.0216140505,0.8030695319,-0.5919985175,-0.0856432617,2.3269383907,0.2625982463,1.1156468391,0.2020006925,1.2754621506,-0.8897251487,0.9669903517,-0.5493559241,1.4233219624,1.3478831053,0.4568857253,-0.0930040702,-1.2832940817,0.2640421391,0.6426716447,1.6632509232,-1.4980686903,-0.5740880966,1.1556472778,1.3272485733,-2.1616599560,-1.6268901825,-0.5095205903,-0.1517010182,0.0697079748,0.6218604445,-1.7932162285,-1.5523602962,-0.2364832610,-0.2116670161,-0.9998782873,-2.7679753304,1.1789580584,1.2540689707,0.2540247738,-0.4325055182,-0.3744247854,0.5653214455,-0.2487467080,0.9951139092,-0.3737806082,1.4783589840,-0.4989001155,0.1346275359,0.4608412683,1.2888094187,-0.4664494693,0.9327733517,0.6842293739,0.4536570907,0.7156196237,-0.8881968260,2.7164895535,0.3652488589,0.6087380052,-2.6499264240,0.4615913928,0.5398000479,0.6077411175,0.1409384161,-0.6123158336,-1.3663132191,-0.8288936615,1.7153819799,-0.5907678604,0.9421477914,1.1129484177,-1.4384629726,0.5338668227,-0.7236166000,-0.3594409227,0.3317614794,-0.0182189792,-0.9622032046,1.2848628759,0.0738611221,1.8387027979,0.9678576589,-0.5062652230,-1.4060206413,0.7187169194,-0.0001753581,-1.2740622759,0.3961002231,-0.4440979064,0.2215769291,1.1455968618,-1.5288711786,-0.1165431365,0.1862287372,0.4621602297,-1.2539870739,-1.9266433716,-0.3532809019],[-1.2628905773,0.6617218256,-0.4527367353,-0.2476387024,-0.3344658315,-2.1655943394,-0.0272593759,1.5402753353,-0.8876109123,-0.5740729570,1.4609684944,0.8156918883,-0.6739629507,-2.1460916996,0.1425817907,1.2147607803,-1.1786000729,-0.6290073991,-1.2190272808,1.9812610149,-1.8409180641,-0.5344301462,0.0046728416,1.4701166153,0.0316930152,0.6217797399,0.3374963999,-0.0923807099,-1.0196141005,0.2970777452,0.5648963451,0.0846193135,-0.5526602864,-1.0114645958,0.0666290522,0.7835562229,0.2549487650,-0.5999336243,-0.9358597994,-1.2035146952,0.0867796391,0.2015919834,-0.6562302709,0.5852459669,1.0319597721,1.6525110006,1.0681165457,-0.3666232228,0.4162625074,-1.4362779856,0.9617007971,0.2182102948,0.4531165659,0.0359582119,0.2627207041,-0.6436700225,-2.5046739578,-0.1782875061,0.6615344882,-0.8519901037,0.2686338723,-0.6474667192,-0.1683191508,0.6466437578,1.4409619570,-0.2617304027,1.9130183458,-1.5077996254,-0.9462426305,-0.2838377357,0.0526938327,-0.6236193180,-0.4251126945,-0.0550731048,-0.8132697940,-1.2440909147,-1.0892269611,0.3136571050,-0.0490135439,-0.6237413883,0.6349942684,-1.0433175564,-2.0065948963,-1.9424735308,-0.6845101714,-0.6202213764,-0.3914147615,2.2720336914,0.0739588216,2.2880032063,0.5641351938,0.4740304947,-1.8853971958,0.4114308357,-1.1214060783,0.4880087078,0.4406731129,-1.2576910257,-0.6754882336,2.2553937435,-1.3919758797,-0.6938131452,-0.0537123717,-1.1831597090,-1.1901445389,0.6069281697,0.8246663213,0.3340826333,-0.2523243427,0.0757556856,-0.2096677721,0.7798483372,-0.8629242182,1.6895529032,0.1870408505,0.3471640348,1.1528530121,-0.5211679339,-0.3400958478,-1.8149099350,0.1189164966,0.4202474952,0.2755539119,0.3063024580,-0.1489593983,-1.3223544359,-0.1619068384,0.9510185719,-0.0225095376,-0.8107722998,-0.3870677650,1.5546485186,0.0615551658,1.6332503557,-0.7618871331,0.9830802679,1.2946121693,-1.3857856989,-1.5067552328,0.2777404487,-0.4606186450,-0.4907822013,1.3923071623,0.6343035698,0.9364335537,-1.7004997730,-0.4155659676,-0.1267781258,0.3403526843,-0.5315839648,-0.7173085213,-0.0580628328,-0.4464083612,1.3359794617,0.0476373993,-0.2086803317,-0.1372452974,-0.0591464713,1.3616489172,0.0903304443,-0.3211252391,-0.9177269936,-0.9550664425,0.3909903765,-0.2268156558,-0.0082257940,1.2424693108,-1.3258881569,1.4352700710,0.7880265713,0.1414163709,-0.2054441571,-1.1014525890,0.6595254540,-2.3181002140,-0.0876651481,-0.6974890828,-1.4716603756,0.7876362205,0.1344156265,-1.3554036617,0.6036042571,0.1190322861,0.9254586697,1.2508783340,0.2331293225,0.4055940509,0.7998869419,-0.5757550597,-0.9135111570,0.7675039172,-0.7781230807,2.4180750847,2.7852742672,0.1045434698,0.6787592173,0.4007455409,-0.3564361036,1.5347712040,-1.4705452919,-0.7479464412,-0.9566254020,1.2649875879,-0.1646696180,-0.5155603290,-0.7206394076,2.1404452324,-0.7790234089,0.7116395235,0.9158282876,-1.4436525106,-0.4211129844,0.7956809998,-0.9565706253,-1.3517844677,-0.3119147122,0.2453809679,1.4581965208,0.7543629408,1.6049604416,-0.4158651829,-1.2043303251,-1.0034831762,-1.1723743677],[1.0647040606,0.2321242392,1.0505148172,0.8940162659,-0.1205676943,0.3933456540,-0.4981866777,0.8221628666,-1.1024289131,0.9230841994,0.1395951062,-0.2751727700,-0.4984712899,-1.3625754118,-0.4306615889,-0.5384787917,-0.9555833936,2.1937851906,-1.1307214499,0.4357461333,-0.8771927953,0.8578191400,0.3055385947,0.2204068154,0.6302809715,-0.0163004622,-0.8319802284,-1.3126729727,-1.3170480728,0.5569218993,-0.3531013131,0.3641794920,0.3810200691,-1.4238847494,0.1009590551,-0.3800279498,0.7649206519,-0.6368266344,1.7794678211,-1.1880887747,-1.4316164255,-1.6969466209,0.5561639667,1.1471374035,0.9899902344,-0.1633966267,0.6074290276,0.5961022377,-2.8764624596,-1.0727797747,-0.8768104911,-1.0192153454,2.8190350533,-0.2256239206,-0.3943177760,0.8255993724,-1.1556965113,-2.1723108292,0.7178436518,0.6824865341,0.5922962427,-0.3120145798,1.0898365974,-0.3011100590,-0.1291559339,0.4131253660,-0.3478083014,0.3667017221,0.3330966830,0.3752444685,-1.4574064016,-0.2556218207,-0.3968566358,0.5036401153,-0.9107782841,1.3879652023,1.2289830446,0.7992539406,0.6849098802,0.0707288608,-0.0696446821,-0.1699579954,0.4269837737,-0.5689053535,1.0672354698,-0.7523844838,-0.6921916008,0.7129670978,0.0622992143,-2.1855492592,-1.0172541142,0.6609465480,-0.2331690043,0.2233446985,1.7745434046,0.0969029590,1.6294189692,-0.6170974374,-1.2387092113,-2.4741020203,-0.0289155487,0.2518738508,-0.1491267532,-0.3006697595,-0.9428971410,0.3010059893,2.3177731037,0.1611628830,0.6096910834,-0.5344636440,-1.1325211525,0.5708059072,-0.2217664272,-0.1881082803,-0.2953444421,-2.7062401772,-0.0275921281,0.9700416327,-0.5384776592,1.4731795788,-0.6910365820,-0.8083460927,-0.2849730253,-0.9540877938,-1.0074419975,-0.3939363658,0.8493119478,1.3113639355,0.1255604029,-0.5570324659,-1.1712133884,-1.6394242048,1.0471559763,-0.6554031372,0.3733699918,0.8540666103,0.1184398606,-0.1343704611,1.3181189299,-0.0528781712,1.9076501131,-0.1207094342,1.0920281410,0.5713807344,1.3109537363,0.7840664983,-0.7930327654,0.7855635285,1.5472874641,0.6432893276,-0.0242837500,-0.3870694935,-0.5971293449,-0.2899690568,-0.7334938049,-0.0300292429,-0.9743447304,1.1392588615,-0.9915491343,-0.8986487985,-0.8678231239,-0.6919215918,0.4460268319,0.7036303878,-0.3525555134,0.2887427211,-0.7344346642,-1.1392246485,-1.1486465931,-1.1256231070,0.2172904909,2.2096951008,-0.2768126130,-0.4194982052,0.0895349309,-0.4994725883,-0.4294939935,1.5925871134,0.9207655787,0.1646861136,-0.3742833734,1.3894538879,-0.4878883362,-1.2071138620,-0.1191699728,-1.4390913248,0.2861081958,-0.1579031646,-0.3974367380,0.1752463430,0.3748835325,-0.3617827296,0.8169750571,1.3362919092,-0.1484031230,0.8671530485,-2.8536005020,-0.0906554833,0.0088043632,1.6732629538,2.1423466206,-0.7370496988,-1.2664498091,0.6641715765,1.2869904041,0.6079743505,1.3022139072,1.3500844240,-1.5029324293,0.6907622218,-0.4061397612,-0.8483211398,-0.3681233823,-0.8578148484,-0.1548754722,-0.5499112606,0.2328155786,-0.6580201983,-0.0703464821,0.5699104667,0.1111837327,0.7781289220,-1.5467842817,0.5690723658],[0.4452523887,0.0673167929,-0.0463277884,-1.8743733168,-2.8230288029,0.5497969389,1.9723832607,0.5564820766,-0.3615648150,0.0729349479,-0.0699678808,-0.7118794918,1.3303546906,0.0751053691,0.5226886272,0.9767999053,-1.7849285603,1.5602053404,1.0742962360,-2.6806015968,-0.4691926837,-0.5361781120,-0.2826193571,-2.2090110779,0.9901263714,-0.0261297319,0.8651404977,-0.2917624712,0.4948405921,-0.7538119555,1.5201109648,-0.7561060786,1.1786170006,-1.1826119423,0.7336524725,0.9188173413,-1.9997373819,0.4963880479,-0.8404771686,0.4128607512,0.4337054193,0.2035663426,-0.7117573619,-1.7409338951,-0.5598160028,-0.9008379579,-0.4741972387,0.0416170508,-0.7776945233,-0.2547941804,1.1234530210,-0.3588818014,-1.5244874954,1.1329869032,1.7894061804,0.7349005342,0.1345884353,-0.5785191655,-0.1267298460,-1.1764208078,1.3696135283,0.4106918871,-0.0976169482,-1.2190376520,-0.9109613299,-0.0955943689,-1.0170726776,1.0108891726,0.1099565029,-0.7064110637,0.2942879200,-0.5495336652,-0.4723607302,0.5777610540,-0.7016943693,0.7423120141,0.0810565427,0.8770937324,-0.1352943182,0.6857259870,-0.5274185538,-0.3148475289,0.4312712550,0.8180384040,-0.1410973966,0.5621446967,-0.8120179772,-0.0752384365,-0.0236309376,0.9926939011,-0.5143352747,0.1516641676,0.7576454878,0.1885048598,0.8092653155,-1.4317138195,0.7512933612,-1.7853584290,-0.3403401673,0.5114456415,-1.0479772091,-1.7917723656,-0.7522316575,0.9491505623,-1.5872008801,-0.1898578852,0.7266498208,-0.9619154930,-0.1413860023,0.9052723646,-0.3882459998,0.1237693802,-0.2326326221,0.3206429183,-1.3629474640,1.1452333927,0.7874519825,0.4181374013,-1.5147526264,0.0730068237,1.3311444521,0.7093572021,-0.9712299705,2.1352322102,1.0231037140,-1.1636049747,1.0286316872,-0.5036383271,-1.6746673584,0.4073672295,-0.0182898492,0.1493997425,-0.7960625291,-0.4407243431,2.3877747059,0.4815349579,0.5813381672,2.2279050350,0.9356135130,-0.4409787059,-0.8286496401,0.7311822176,0.2700185478,0.5507662892,-2.5871624947,0.6772342324,0.2074013352,0.4615665972,-0.4190964699,0.4770605862,2.4217619896,-1.9182502031,-0.8544636965,0.8381118178,-0.4485734701,-1.0919756889,-0.2648122609,1.0098994970,-0.5472998023,1.2348033190,-1.4701932669,-1.9155840874,-0.1090629399,-0.1855992675,0.1095966473,-0.6782081723,0.1180862114,-1.2295197248,1.5274536610,2.3235061169,0.2913398147,-0.3180084825,-0.9371327162,-0.5686488748,1.8743900061,0.0845806673,1.3588453531,0.1544833183,-0.4382197857,-0.6617422104,-0.2704090774,-0.0473123975,-0.1375723928,0.5586237907,-1.4751049280,-0.3098645806,-0.8870976567,1.3249599934,-0.2478749007,-0.4527670145,1.0433957577,-0.4988560379,2.0353519917,0.1222634986,0.8268195987,-0.2070628256,0.4064068496,0.7379081845,-0.1530139744,1.1486705542,0.7987783551,-0.0146313990,-0.8424440026,0.2351379693,0.5839104056,1.9307159185,-0.1338217556,-1.2729601860,-0.9533520937,1.4415456057,-0.0951500610,0.3972956836,0.2098439634,0.5354406834,0.8776354790,0.5473576784,-0.2294035405,-0.8871148229,1.8933590651,0.1261401772,-0.5996055007,-0.0468349196,-0.4060138166,0.0416313224],[0.0498172343,0.5637643337,-2.2129881382,-1.1236852407,-0.1304418892,-0.9155464172,-2.0547239780,0.6958734989,0.0104737310,-0.0415819399,0.4621022642,0.0875720531,-0.5839158893,-0.1784689277,-0.4025821686,0.9536439776,-0.1893357337,0.4697566330,-0.8019378185,0.4339478314,-0.4863450527,-0.7643893361,-0.4413174093,0.7450090647,0.4705619812,-0.5967475176,0.2144325823,-0.0487319790,0.4239455760,-1.8733527660,-0.6309217811,-1.3656258583,0.5353654027,0.3123120070,1.3560433388,0.0231874250,-0.7732568383,-1.4160184860,1.1718209982,-0.6379296184,0.9241999984,1.1896660328,1.4317412376,-0.1303557307,0.5576506853,0.7538163662,0.4692847729,-1.0503007174,2.2192955017,0.7662305236,1.4199377298,-1.1396961212,0.6984863877,0.8586908579,-1.4875825644,0.1292233914,3.3051817417,0.7467623949,0.8142894506,0.5744169950,1.0687381029,-0.8679308295,0.2648360729,-1.8214925528,-1.5498822927,1.0336124897,-0.3167379498,1.6213614941,0.0680482686,-0.0182913449,0.0870022327,-0.7723815441,1.0164847374,-1.1489291191,0.3398660421,0.4741336107,-0.1662758589,0.0326907784,-0.3206206262,-2.1174454689,0.3675934374,0.3425858915,-0.5936652422,-0.7317870259,-0.8706053495,0.3625514805,-0.7185558677,-0.3813236952,1.6017394066,-0.4086554348,1.1893210411,-0.3677315414,-0.8534657359,0.2419654280,-0.2601073980,-1.7017725706,-2.0002150536,-1.2726695538,-2.8272268772,-1.3180217743,0.2695805430,0.8193210363,-0.0601939671,0.2057047337,2.1258807182,-1.0387762785,-1.0656303167,0.8474498391,-2.0752680302,-0.7518011332,-1.0824229717,-0.0520494208,0.8673002720,0.7077292800,-0.6896764636,0.5508205891,-0.5087099671,-0.7953282595,-0.7552286386,0.3783166111,0.1480294019,-0.4080043733,1.3215683699,-0.2240578383,-0.9652453065,1.6936491728,-0.5004338622,-0.4313458204,1.8143146038,-0.8686926961,1.2914714813,-0.2357162833,1.4363687038,-1.2961885929,0.1975327134,-0.8971417546,-1.4519535303,-0.4755281806,1.1832439899,0.5823754072,-0.7990812063,-0.1475500166,-1.5841269493,-1.0030679703,1.5403479338,0.8109509945,-2.1969048977,-1.9337809086,0.9102554917,-0.2021962404,-0.8042155504,-0.3982497156,2.4267811775,0.6462002993,0.5270777345,0.2776626945,-0.3879787624,-1.2539374828,-1.0608892441,0.8798408508,-1.0080276728,-1.2297723293,-1.1807438135,0.7539607286,-0.3988789618,-1.2075831890,1.9517722130,-0.0741142184,1.6240398884,0.2602418065,1.8968186378,0.9252575040,-0.6450889707,1.4229569435,-0.1587112993,0.8568664789,-1.0669229031,0.6148740053,0.7432900667,-2.3107280731,-0.7096374631,0.1439763308,-1.7630770206,0.8053269386,1.0544917583,-0.2164501399,-1.1995971203,-1.1014965773,-0.6774827838,-2.0162796974,-1.3040640354,1.7306530476,0.4679165184,1.5393271446,-1.1890480518,0.1019532531,-0.9641910195,-0.6676917076,0.3440614343,-0.2543410659,0.5562074184,1.3031641245,0.3274115324,-2.5846605301,1.2806965113,1.9282523394,-0.1295706928,0.4334105253,0.8793066144,-0.5240471959,-0.1192059442,0.2685222328,0.6581997275,0.3720701039,1.0289461613,0.6471891999,-1.3104859591,-0.7705357075,-0.6850705743,0.2172274739,1.4807068110,-0.2180891037,-0.6771414280,1.5424208641],[1.0833897591,-2.8045880795,0.4607556164,0.7872321010,1.3725421429,-1.5756440163,-0.3378772438,1.1134512424,-0.0857403502,-0.7604584098,2.4517266750,1.1246706247,-1.8834089041,-0.3396808505,0.4468193650,-1.3754568100,0.2401239425,0.2437317073,-1.6011885405,-0.6616942286,0.3145923913,-0.5798115730,1.6361236572,-0.8283223510,-0.0387554541,-1.4568865299,-0.9266756773,-0.6654282808,-1.0409642458,0.1891269684,-1.0281019211,-0.6576402783,-0.2294373065,0.3463232517,-1.1547359228,-1.8444094658,0.9560483694,-0.6993877292,0.3474283516,0.2573563457,-0.5045565963,0.6014789939,0.0792550221,2.0729916096,0.5574637055,2.2566828728,-0.7126176953,-1.8690760136,1.1867282391,-1.2382060289,0.5142056346,0.9849852324,-0.9336848855,0.1257813275,0.0927541032,1.3971587420,-0.3119037747,-1.1926451921,-1.2881635427,-1.3171620369,0.4146163762,-0.4774722159,-0.3868423402,-1.0103545189,-1.1745402813,-0.2091901600,1.3277802467,1.1256504059,0.2741110325,-1.3244829178,1.5284399986,-1.0842255354,-0.8799241185,0.4729141891,-1.9505088329,-0.4351518452,-0.8081485629,0.3269541264,-0.7062594295,1.6481834650,0.4895448983,-0.9180828333,-1.4240669012,-0.1890579164,-0.9585830569,-1.0736232996,-1.2009921074,0.0086217457,2.2900185585,0.9107310772,0.3459389806,-0.2933109701,1.7081165314,0.7132967114,0.1095507592,1.3954622746,-0.0916409269,-0.4428087771,0.3978556991,-0.9287421107,0.6775330901,-2.0309498310,-0.2861202955,-1.3187100887,0.6868452430,-0.1280173510,0.3954797983,-1.4868066311,2.6867311001,-0.2837397456,-0.4692958593,0.8649511933,-0.4000707567,-1.3636385202,-0.4336688221,0.4807041883,-1.1696223021,0.4139298797,-0.3658573031,-0.8346003294,-0.6913728714,0.6472249031,-0.9495275617,1.0565907955,-0.1551950276,0.0402307771,0.5411900282,1.9618538618,0.3323952258,-0.1880543530,0.3495495617,-0.2323200107,-0.0774672851,1.1608976126,-0.6110397577,-0.0833180025,0.5343641639,-0.1402324885,0.7121111155,1.1572567225,0.6025365591,-0.0661304966,0.9814050198,-2.0910024643,0.6004716754,-0.8897766471,0.5802534223,1.2903648615,1.0397731066,-0.8031101823,-0.8996082544,-0.2524068058,-1.2267825603,-0.9092355967,-1.0889654160,0.8441944718,0.8110027909,-0.0405239128,-1.0715160370,-0.1124812588,-1.5997368097,0.1420539320,0.6181820631,-1.4807455540,0.2736857831,-0.0874448717,1.4611581564,-1.0566525459,0.4870162010,-0.0450987816,-1.0364639759,0.7334445715,1.5605502129,0.7317325473,-0.8760612011,-0.3242733181,0.5894088149,0.5113537312,-0.2457536161,-1.0911314487,1.1978468895,1.4237911701,0.2477284819,-0.2938754857,0.5031723976,0.0124401199,-0.4126176238,-0.7320761085,-0.4052883387,-0.2108996511,-0.7781730294,2.1468038559,2.6817145348,0.2516969740,-1.0822778940,-0.0529402457,0.8459635377,-1.2501021624,-1.1299482584,-0.7857146263,0.1984169632,-1.6308257580,0.9140481949,-0.2152125388,-0.2536766827,1.7120695114,-1.6373603344,-1.5643184185,-0.4590578377,0.4787595272,0.0577515550,-0.8291638494,-1.4593600035,-1.9532253742,-0.7408310175,2.6602454185,1.3658504486,-0.4614217579,2.1605172157,-1.6152997017,0.0586317293,-1.2505980730,1.0217580795,0.0332866982],[-0.1231489927,0.8719854951,0.3502269387,-0.3011886775,-0.3683407009,-1.3110753298,-0.5376300216,0.3520507216,-1.2554615736,-0.3561529517,-1.4860554934,2.0891554356,0.4023406506,-1.2871041298,0.4452855885,-0.3306802511,-1.0909045935,1.2342441082,-1.3968569040,1.2671681643,0.3808578551,0.1622464359,-0.8135055304,-0.2238749415,0.1271271706,0.3600368202,0.1486675888,-0.3422999382,-0.9236380458,-0.1554073095,0.7903410792,0.1860257089,0.1308351606,-0.2578380704,1.5718585253,-0.7012122273,1.0408076048,-1.0936492682,0.8577693105,0.4340395927,-0.2430948913,0.2685880661,-1.8785663843,0.9641314149,0.3043458462,-0.6693259478,0.6128401756,-0.5462137461,2.3161671162,0.0049466984,0.0762785673,-0.8627022505,-0.1909276098,-1.7148029804,0.8741716743,-1.0369155407,-0.9876949787,0.0601231158,-0.9681298733,-0.8266518116,-0.0769524872,-0.5266578794,0.6762766838,1.3591698408,0.6121618748,-0.3233613670,0.0347821042,0.8331773877,1.1339416504,-1.0403983593,0.1039372236,0.3434944749,0.7209177017,-0.6197436452,-0.6891433597,-0.7516368628,0.9467796683,-0.2635918260,0.6381617785,-0.5782182217,-0.3497321010,0.9256712794,-0.6284171343,-1.8411649466,0.2277844101,-0.4046567082,-0.2888169885,0.7736723423,0.7876480222,0.9130166173,2.5497612953,1.3560099602,0.5737367272,0.2998895347,-0.7253481150,1.6977071762,0.1428591907,1.6922923326,-0.7899493575,-1.6153711081,0.8779649734,1.6200410128,-0.7291763425,-1.0820019245,0.7858555317,0.2406725436,-0.2910236716,-0.0595621802,-0.4121816158,0.3560032547,0.9713022709,-0.3304999173,-1.7688531876,1.1638083458,-0.9774751067,0.4061269462,1.4668464661,0.2371457219,-0.1761687845,-0.2912491560,-1.8847373724,-0.6073658466,-0.5818338990,-0.3286408484,-0.6674704552,0.3853674531,-1.2600563765,-0.0117033292,-1.2790175676,-0.0959654301,2.4882016182,0.3786404729,-0.7029803395,0.7567839026,0.1165871844,-0.1981320381,-0.4176677465,0.7999218106,-0.3878305852,2.3828504086,0.4363039136,-0.3107819259,-0.3244489431,1.2963690758,-0.5949802399,-0.3192541301,0.1184017360,-1.0748354197,-0.2322329283,0.0255772974,-0.1782385856,-0.4525050223,-0.9569429159,1.7335696220,-0.5961167216,-0.9838139415,-1.5097740889,-0.5643052459,-1.0459479094,2.0024902821,1.1414196491,-0.7664759755,-1.2552784681,-0.1251813024,-0.2468750924,-0.2969430983,0.2811274230,1.1297291517,-0.4285236895,1.6837493181,0.1570269614,-1.3796472549,-0.0221588574,-0.9108480811,-1.4940923452,0.3765714467,0.1872464269,1.0197986364,-0.4883226156,-0.3771733940,0.0635235235,-0.3010345995,0.2305924147,0.3498499691,0.6820290089,0.7376207113,0.2440648973,0.2061410099,1.6231516600,-0.4384661913,-1.0267047882,-1.4744080305,-1.1771376133,0.1827095300,-1.0081615448,1.2591556311,-0.1388425082,0.4758206606,-0.2766729593,0.5053372383,-0.0261183511,-0.1096292585,0.3588334322,-0.1530779451,-0.1198340133,-0.2816464007,0.1439685822,-0.2829401791,1.0409681797,-0.0788520947,-1.1657164097,2.2557945251,2.0186963081,0.2609864473,0.6588216424,-0.0292505622,-3.0369510651,1.2542692423,0.5731019974,-0.0044440515,-0.4187056422,0.6035118103,1.5395250320,0.9933514595],[-0.5339363813,0.7204960585,-0.2520633638,0.1129418686,0.3106788397,-0.1949516982,-0.7873948216,-0.9607906938,0.3240345418,1.3103933334,-1.2638764381,1.3863197565,1.8211445808,-0.3672892451,0.4053455591,1.7521601915,1.6447889805,0.7403008342,-0.8374760747,1.8835134506,1.4278148413,3.0550069809,-1.2181924582,-0.8227337599,-0.1327370256,0.3542967737,0.6749712229,-0.3170649409,0.9784115553,0.3086686730,-0.1767462045,0.0843835175,-0.6492677331,0.3701886237,-2.1043498516,-0.0279286541,-0.8700038195,0.6007041335,-1.9832814932,0.2308917642,-1.3054196835,-0.3247613013,0.2073209584,-1.3216863871,-0.1985991448,0.9323601723,1.6048812866,-0.0951557308,-0.2890453935,0.6215823293,-0.0441857241,0.9499992728,1.8083887100,0.1037293598,0.8567683101,-1.0534691811,1.9651103020,-1.9569315910,-0.4348264337,0.6502805948,-0.9168347120,1.1094492674,-0.9395704865,0.8174906373,0.9895029664,-0.1846548319,0.4872570932,0.3263651729,-0.6341340542,-0.1032819226,-0.7346142530,0.1689246595,-0.1534329206,-1.5958054066,0.2307239473,-2.1755774021,0.4595609605,1.0350043774,1.1936333179,1.2886185646,0.5671181679,0.0456784628,0.4965856373,-0.8134568334,0.9582699537,-0.0257120635,1.0541176796,0.1182258353,1.0523356199,-1.8147698641,0.7733538747,-1.3099697828,-0.9093947411,0.5400250554,-0.3680446744,-3.2436428070,0.1594742090,-0.6183111072,-0.5293322206,2.0307416916,-0.1469859183,1.3975157738,1.4203033447,0.5249443054,0.7566241026,-1.0997810364,1.1431112289,0.8553681970,1.1881549358,-0.3714130521,0.1984697133,-0.5287451744,-0.3881629407,0.8084456325,-1.3102331161,0.2904533148,-0.6494663358,0.3399093449,2.3053667545,-1.9359093904,1.7290390730,1.5100733042,-0.2926071882,0.3544541597,1.9625288248,-0.6443831325,-0.4173543453,1.1030114889,0.3282997310,-0.0072064842,0.4895175695,-0.5674248338,1.6213970184,-0.4362982810,0.3757034838,-0.8909569979,0.5226764083,-0.6865759492,0.8143765926,1.2320840359,-0.7808359861,0.0008738514,-0.8514294028,0.4705692232,-0.3333962560,-0.3492262959,0.2567463517,-1.1781015396,-0.4792070985,-0.3375956416,0.8531564474,0.8517883420,-0.1833593547,-0.3668282926,0.4279274046,-1.1330124140,0.6293390393,-0.1875369698,0.1160475165,2.1682090759,0.8720598817,1.4349334240,1.2770864964,-0.1004557535,-0.5276843905,-0.1601087302,1.3145697117,-0.0181776304,0.5276513696,1.3746075630,-0.8591616750,0.0153728090,-0.1448028535,0.6654556394,-0.8472108841,1.1001250744,0.4171614349,-0.0177354254,-1.1216690540,0.7256075144,0.1507796347,1.0267204046,0.6718088388,0.2002222389,-0.8297312260,-0.8101356030,2.3435692787,-0.4786757231,1.0909048319,-1.7758495808,0.2627611756,0.2653958797,-0.5927811265,1.5539902449,-0.9272117019,-0.2532606721,0.0267162919,0.5292546749,0.3089748025,-0.3464745879,1.7746504545,2.9259448051,-0.6218938828,-0.3926011324,-0.3428994119,-1.1926394701,1.5049992800,-0.8424344659,0.7986744046,-0.7885806561,-1.1607004404,-0.0135042230,0.9922061563,1.9919799566,-1.2437075377,-0.4395283461,-0.5311597586,-0.0297754165,-0.5898804665,-0.7635480762,-1.5769889355,1.0587168932,-0.0034756528,0.1553152949],[-1.9455130100,0.2943831086,1.3622395992,0.4102196097,-0.4908825755,0.4014503062,0.0278171897,-0.0536309853,-1.2789026499,-1.2796289921,0.4953480065,-0.4660226703,-0.9548496604,-0.9615236521,0.9491387606,0.3494040668,-0.9034793377,-0.8746790290,0.3864723444,-0.7408620119,0.8857078552,-0.9054695368,0.3242028058,-0.9857577682,-1.7794227600,0.7796273828,-0.2548395693,-0.9350469112,-0.0115598422,-0.0557284057,0.4348230958,1.1245038509,2.6359810829,-0.2186294347,-0.0949227884,-0.2960619032,-0.7614319921,0.6499288082,0.0611613914,0.4410744905,0.7678567767,0.0930860490,-1.0300735235,0.3483096361,0.7597416639,0.0011209270,-1.1556539536,1.1331659555,0.9468092322,0.2521147132,0.1224794090,-0.5496848226,1.5198215246,-2.1303267479,-2.2218475342,-2.5806076527,0.0837756246,0.4271501005,0.8224028349,1.1306557655,0.3919700682,-0.0115966676,0.0475504175,-0.6004253030,-1.2305142879,0.5945131183,1.0072363615,0.3247313201,-1.2197523117,-1.1868721247,-0.9118242264,0.1778739989,0.3262349665,-0.8597450256,0.0205650963,1.7530291080,0.7721672654,-0.1331358552,1.0669914484,0.8614687920,0.9448447227,0.4930521548,-1.0212017298,0.7986354828,-0.7440475821,-0.1944002360,0.3420980573,0.5632216334,-2.4997861385,-0.4994763136,0.6333222389,-2.0620503426,0.8145817518,1.4459090233,0.9552916884,-0.7975873947,-2.1032149792,0.0418324843,0.5111634135,1.7133549452,-0.3090857565,-0.6058284640,-0.8708721995,1.2175831795,-0.2389851511,0.4239602387,-1.5448274612,-0.3783124983,-0.0029062431,-0.6120293736,0.2799707651,-0.0570160225,2.0717158318,-0.3598082066,1.4251867533,-0.0134970779,-0.9720110893,-0.2252669930,0.5460293293,1.5419213772,-1.5147491693,-0.3293933272,-0.0275527146,-0.9521688223,-0.3834776282,1.4057289362,-0.7203474641,-0.9101272821,-0.1940222532,1.3746294975,0.4223842025,1.1517481804,-0.3333800137,0.0154191507,2.1487598419,-0.9099595547,1.2981448174,-0.4607750475,0.7508655787,-1.1511445045,0.6852880120,0.3033760786,1.1724207401,0.0635844842,1.9570583105,0.8574123979,0.5666555762,0.7369867563,-0.3596095443,0.0712693930,-0.2050676346,0.7476635575,-1.5979381800,-1.6696869135,-0.0033412154,0.2548270524,-0.4833313227,1.0923733711,-0.2952604592,-0.8811125159,-0.4041282833,1.2807453871,0.1407522559,-0.2607857287,0.8139761686,0.2080699205,-1.0974228382,-1.0547447205,-0.5768285394,1.7892030478,0.4703820050,-0.7589623332,0.4739449620,0.3060959876,1.4044001102,-0.4405074418,1.5767858028,0.2276372015,-0.0085633667,0.1122219488,-0.1186968386,-0.7556450367,-0.2551757693,0.2367250472,-1.3682280779,-0.0868984833,-0.9292290211,0.4767979980,1.2660095692,0.7017489076,-0.0994677246,0.4604631066,0.5536651611,-0.6214803457,-0.1921214759,-0.9230517745,0.2861795425,1.9730838537,1.5194038153,-0.2218993008,0.0676458031,0.8111193776,0.5127606988,-0.0850184634,-0.3213681579,-0.2438399494,-0.1359906793,2.0709638596,0.3468191326,-2.5300788879,-0.7631182075,0.2191569060,0.0090567889,-0.7353634834,-1.5984612703,-2.1198539734,-0.1774644852,0.7098138332,0.8213640451,-0.0107929576,1.2051004171,0.0632010624,-0.3080267012,0.9183343053],[0.0939481556,0.8973917365,1.3931572437,-0.2151860446,-1.0463820696,1.2186774015,0.4525365233,1.2896789312,0.3542840779,-0.4334890842,0.0643699095,0.1865993589,0.5796870589,-0.7730751634,0.4719973505,1.7717124224,-1.5757242441,0.2662694454,-0.4041916132,0.4621180892,0.6618397832,0.1943725944,-0.9175354242,-0.0798297524,-0.7954104543,0.4863788784,-1.4996377230,0.8618410826,-2.9106974602,0.3565086722,1.5209159851,0.5370289087,-0.1094295308,2.1429982185,-2.2595870495,-0.4742313921,0.4032390714,-1.3632237911,2.1153512001,-0.8269336224,0.4764483869,-0.3866333365,0.4835948348,-1.0510087013,0.9150114655,-1.4388750792,-0.3364607096,0.0429121219,-0.1839305013,0.3848934770,1.5128203630,-1.1400394440,-1.4442991018,-0.4257532358,0.8354291320,-0.2160586119,0.9489045739,0.3786438107,-0.6682091355,1.0235065222,-0.8382804990,-0.1399486065,-0.9926248789,-1.2264190912,-1.3117069006,-0.1271556914,0.0079912404,-0.3760568500,0.5664597154,-0.8595212698,-0.1183861569,0.8529757261,-1.9854843616,0.2800190151,-1.7601602077,0.6664977670,0.6719502211,-0.7664304376,1.0983656645,0.2911262214,-1.1661839485,-0.8158940673,-0.2933756411,1.4010317326,1.6039435863,-0.5165402889,-0.1408138573,1.1347715855,-0.4194507301,-0.5617449284,0.5707189441,-0.0715572983,1.7359468937,-0.6072747111,0.2172731906,-1.6416498423,1.1322650909,-1.7049622536,0.2442215234,0.7345991135,0.9624508023,-0.7012826800,-0.4934635758,0.2794271708,1.1199611425,0.0138948634,1.3469048738,-0.5083768368,-0.3632907867,-0.4739925861,-0.8815130591,2.1984884739,2.9767618179,1.0400345325,0.5973387957,0.6966767311,0.3020858765,0.7583796382,-0.7667793036,-0.0830394998,-0.6338041425,0.4115710855,-0.0886307508,1.4750047922,0.6065263748,-0.6084166765,0.4203138649,-1.1035499573,-0.1466142833,0.0930199549,-0.4346646070,-1.3677084446,1.7228507996,0.7114614844,-0.7142797112,0.3464543819,-0.6122346520,0.7374718189,-1.2237185240,-0.4646267891,-1.4108309746,-0.4445342124,-0.4988172054,1.8301521540,-0.7865855694,-1.0325416327,0.3002009392,0.1759860516,-0.3930571973,-0.0841103196,0.6128829122,1.1748423576,2.0594334602,-0.1751677841,1.9157706499,-2.0087039471,0.0548449084,0.9300742149,0.7866141200,1.1222592592,0.1036242396,0.6618089676,0.0268117934,-1.9981175661,0.9294196367,2.2584624290,1.0756754875,1.1363080740,-0.2930243909,-0.0277448576,0.8949297667,-1.3616064787,0.3977112472,-0.7987060547,1.2334747314,0.2763086557,-2.1072409153,-0.2869181931,-1.5621356964,0.7045946717,0.8275618553,-0.4897731841,-1.0612022877,-0.2257146686,-0.4813244045,0.8550677896,0.4653412998,-1.9319355488,0.0123473499,0.6307618022,0.1360304952,0.3419711292,0.2482180744,1.0112982988,-2.8517372608,1.0242309570,1.3946468830,-1.4900366068,1.3303142786,1.1665626764,-0.0065014083,-1.8222254515,1.1139158010,-0.4737008810,1.4344037771,-1.1761858463,-1.6211301088,1.0432368517,1.0384088755,1.0714992285,1.5375100374,-0.7071874142,0.0438755527,-1.1680675745,-0.1133877560,-0.8412772417,-0.7477636337,0.6549735665,-1.6748837233,0.1416903734,1.8518723249,0.5723341703,-0.2291723639,0.4399379194],[0.8588612676,1.9773676395,-0.4936513007,0.2287907153,-1.0384471416,-0.1597029120,0.1248741448,0.7425230145,0.5512924790,0.7303834558,-0.8165476322,-0.7640091777,-0.1144194901,0.8935913444,1.6052113771,-1.2538806200,0.2855737209,-0.4613464773,1.2086780071,0.4985507429,-0.3405464590,-0.8937238455,1.2616927624,-0.9317969084,-1.0411241055,0.8100412488,0.7059498429,1.3697428703,0.9094738960,-1.9717726707,0.2420305610,-0.9077606201,1.6671775579,2.3022015095,-0.1790511310,0.8325935602,1.3162128925,-0.2899599671,-0.4036357999,-1.3132460117,0.1316954643,0.9602703452,-0.1307659447,-1.0357890129,0.5832430720,-1.4192433357,1.2996268272,-0.1998086125,-0.6707410812,1.8675396442,-0.2962323725,0.9742482901,0.2611752450,-0.1437780410,-0.6985815167,0.6297942400,-0.9077713490,-0.3890576065,0.4322726727,0.1401295662,-0.0165128782,-1.4022303820,0.3342009485,-0.0561159886,-1.2076241970,0.0269585140,0.0797272325,0.1862457544,-2.2476606369,0.9444752932,-0.2633162439,0.5973744392,-1.3259795904,-0.4369872510,-0.6111346483,0.4619466960,0.1574052423,0.4900255203,0.1863169968,0.2730141282,0.0945252627,0.6730496883,1.2070502043,-0.7237103581,-1.5864273310,-0.5616668463,0.8356405497,-0.0633497909,0.1185418442,-2.5133163929,-0.2127508223,0.9225611091,-0.4511755705,-1.3210129738,-1.5038857460,0.3074418604,-1.4114911556,-0.9896835685,0.4621183872,-0.1546157897,-1.0632112026,-0.6646524668,-0.2710254192,-0.0559235252,0.2106004804,0.3330169618,0.2115703970,-0.1708970219,-0.2765752971,-0.3595123887,-2.1213698387,-0.0695509166,0.9423075318,-0.6175131798,-0.5573751330,0.0207770932,0.8922007084,0.4503358305,0.0453439653,-0.9838991761,-0.1429029554,-0.0101154400,-0.3242063522,0.8762276173,-0.1310543418,-1.3132834435,0.1048069224,0.9953035712,0.0019832081,-0.5763996243,-1.5993032455,0.2357865274,-0.5236660242,1.2498191595,-0.3972760141,0.5428637266,-0.3759145141,-1.0959819555,0.4462662935,-0.4387346506,-1.1529084444,-0.0285182241,-0.1020001173,0.5932555795,-0.5795496702,-1.0888371468,-1.7439676523,-0.2361356318,-0.0777337477,0.7394383550,0.0782520548,1.6987942457,1.7381364107,1.1641181707,1.4951943159,0.1576152593,-0.6602454782,-1.6606831551,-1.8109960556,-1.5184327364,-0.6278802752,1.0903507471,1.0761795044,1.2518355846,-1.6058274508,-0.3573199511,-0.9369678497,1.4489934444,-0.0281910524,1.0515197515,1.3035051823,-0.7908535600,1.5493855476,0.1785422266,-0.2503399253,0.5442001820,1.4513047934,0.2108101994,-0.7557499409,-0.8626450300,-1.3776224852,1.0434657335,1.1152778864,-0.5381469727,1.1608798504,-1.1071029902,0.7692725062,1.7361100912,0.6548727751,-0.2329286039,-0.2109164447,0.8669708967,-0.5419610739,0.3878645599,1.0662782192,-1.9696036577,0.8957763910,0.0708186775,0.8453901410,0.0108403713,1.4044237137,0.6272301674,-0.7871339321,0.7302926779,0.5642443895,-2.1129529476,-1.1371386051,-0.0070197177,0.9867774844,0.0979930758,0.2202437371,-0.3924337029,0.5245616436,0.9684751630,-0.3208032548,-0.6954401731,0.2925103903,0.6001781821,-0.2031298876,2.5664210320,-0.7845811844,-0.0679450184,0.0334720835,1.3929741383],[1.5113071203,0.2075864077,0.6620129347,-0.6452699900,0.7408705354,-1.7304400206,-0.1776096076,1.5553163290,-0.1056940630,-0.6364824176,1.5327748060,-0.9357856512,-0.2761799097,0.4746173322,-0.8172100186,-0.6547473669,0.5243036151,-0.0618794486,-0.2851223946,-1.0006905794,0.7876641750,-0.8730727434,-0.6232350469,1.6723668575,0.5514597893,-1.0533039570,-0.3889478743,0.4371635914,0.9450132847,-2.1121516228,0.7922255397,-0.3742645383,-0.1040292904,-0.1706566215,0.8582338691,-1.1603547335,0.6076740623,0.9498484135,-0.8680899143,-0.8084941506,-0.5008729100,-1.4534268379,1.1655235291,-0.9484149814,-0.0762682483,0.0358931646,-0.2384106368,0.9499883652,-0.9772548676,-1.0445420742,0.1405598968,0.5456544757,-0.5860677958,2.0359721184,-0.7959108949,1.8235174417,1.8872886896,1.1418535709,0.3126354218,0.3064375222,-0.2798117399,-0.4689850807,-1.6476761103,-1.5992029905,-1.2629358768,-1.7541141510,1.1695880890,-0.0457241274,0.4733829498,0.0854842886,2.0468397141,0.2154959887,0.0350889787,0.2971332371,0.9285229445,0.9239901304,0.4604280591,-0.5467797518,0.4883889854,-0.9885988235,-0.5728197694,0.9264045358,0.7810805440,1.0949702263,0.1750663072,1.4096968174,0.2180642933,0.7436999083,-0.1266501397,1.3566181660,0.3130574524,1.2390726805,-0.1934143156,-0.7612330914,1.0759313107,-0.3289625347,0.1660896689,0.0740403831,-1.1717393398,1.1129521132,-1.1576689482,-1.7789555788,0.9600282907,-0.7468833923,0.5374038219,0.9328088164,1.5861493349,0.8423115611,-0.4052751660,1.0446491241,0.0374817997,-1.1891011000,-2.3277878761,0.3875864446,0.2966091335,-2.2180492878,1.0663291216,0.2598075867,1.9161059856,-0.7282618880,-0.6266215444,-0.6739975810,-0.6415785551,-0.6179529428,0.6047268510,-0.2844314575,-0.4305409491,1.9450337887,-0.1466508955,0.4638362825,2.0536992550,-1.2035881281,0.2433279604,0.3226449490,0.2997124195,-2.0240993500,1.5049318075,2.5287919044,-1.6473007202,-1.8275324106,0.7434623837,0.3066866100,-1.1217963696,1.3457329273,0.6340570450,1.1217298508,-0.0488672554,-1.2668669224,-0.4204682112,1.2064194679,-0.1509321928,2.8461978436,-0.0451924764,0.4221574366,1.7345610857,-0.5981693268,0.7031428814,-0.0062330775,0.8340855241,1.6620643139,-2.1220414639,0.2365728617,-0.1516558230,0.5030402541,-1.7541236877,1.3960247040,-1.2516578436,0.9715057611,-0.1300617158,0.2552434802,1.9452192783,-1.0372388363,0.5538521409,-0.3709153831,-1.3116137981,0.6361788511,0.9690408707,-0.9008204341,-1.6484543085,-0.0956472978,0.3216061890,-0.4470887482,-0.7181816697,-0.4564741850,0.0436974354,-0.8783827424,-0.2153731734,-0.7284212708,0.0316979624,0.5592319369,0.3584336936,0.3528334498,0.7266294956,1.2041256428,0.6806427836,0.3226889074,-1.6938356161,1.0152925253,-1.2883669138,1.5473442078,-1.6940101385,0.7440575957,0.5449872017,-1.5002492666,-1.1375263929,0.4404759109,-0.9481218457,1.0618398190,0.3651114106,-0.1956363469,-0.0600685142,0.2220172286,-0.2049867213,0.7872170806,0.5148497224,-0.4739772379,1.0607751608,-2.2600011826,-0.2448202819,-0.5458866358,-1.6628972292,0.6307308674,0.6686230302,-0.8363482356],[-0.5782997608,-0.6807925105,-0.1028094068,-0.3188337684,-0.2476856112,-0.8180071115,0.1356047243,1.5968712568,-0.8921045065,0.7458104491,-1.2947020531,-0.0416521616,-1.5273494720,-0.4211159647,0.4188219309,0.0392890759,0.6555977464,0.9114678502,0.3619571626,-0.5588787198,0.2664752603,-0.3256011307,2.4297068119,-0.5743183494,0.2944850326,0.9484425783,0.9825353622,0.3974099755,-1.0577732325,0.6464989185,-0.2026707530,0.3146737516,-1.5660982132,-1.0093059540,0.9294809699,0.9666360021,0.3079728484,-0.0658802390,0.5567436218,-0.4141434431,-0.9001114368,-0.2555243075,0.4303688407,0.6368939877,-0.3189966381,-0.1767177880,2.1670389175,-1.1297926903,0.0641836300,0.1511009932,-0.8907510042,1.9690883160,0.8412009478,-0.9752152562,-1.5275386572,1.0577732325,1.3252483606,0.8089088202,1.4796154499,-0.2566390634,-1.3153479099,-0.8288609982,-0.2103146762,0.1020352095,0.9253606200,1.2352576256,0.5293609500,0.4743773043,-0.8591988087,-0.1014773473,0.5960752368,-1.3404738903,1.1582435369,1.2524724007,-0.6086274385,-0.7040278912,0.8515852094,2.5486783981,-0.6445094943,0.4089204371,0.3661203384,0.1784386188,-1.1837905645,-1.6509048939,1.1145668030,0.2557139695,-0.0395323709,0.0242085606,-0.4647791684,-0.2464168668,-0.5249590874,-0.0612839907,-0.1260320395,-1.0659192801,0.7829201221,1.1975088120,-1.8215229511,1.2382860184,-0.5807151198,-0.2608066201,-0.0539864413,-1.5416053534,0.7714299560,0.5916702747,0.5156317949,1.1404858828,-1.3494912386,2.1378667355,0.3047274649,1.2195792198,-0.3578601480,0.0984548479,0.7212519050,2.6335790157,-0.4315360487,0.5843237042,-0.0755212381,-0.2289883345,0.9112005234,-1.0294852257,2.1142601967,1.0679942369,0.5550442338,-0.1199776754,0.5793484449,1.8686089516,-0.6061916947,-0.0292256158,-0.1874776036,0.4629638493,0.4734807909,-1.0361982584,-2.4138829708,-1.1330380440,-0.5306935310,-0.6419487596,-0.5470802188,0.1643695980,-1.6450990438,-0.2671853602,-0.1303537935,0.0227936525,0.9756251574,-0.0457911752,-0.4323091805,-1.2599148750,-0.1694476306,0.4077905118,1.0774275064,-1.1228400469,0.2199149877,0.4889201522,-1.2091072798,0.4924613833,-1.7255488634,0.9194186926,1.4995775223,-1.0543442965,0.4016973972,0.9137138724,1.4287455082,-0.7213507891,0.5033758283,-1.1633116007,-0.7225621343,-1.4303677082,-0.2655269504,-0.8245450258,-0.6388269663,1.1980402470,-1.7719768286,-0.5820679665,1.0126674175,1.2914206982,-1.1844006777,0.3093956411,-1.2403331995,-0.4625792503,1.7663393021,0.6885182858,-0.1327944398,-0.8755931854,-1.5265610218,0.2222946286,-0.3382628858,0.6167198420,-1.0755565166,0.0484146737,-0.9836882949,-0.3923490345,0.5466002822,-0.3768848777,-0.7443243265,0.3661516309,-0.1037210673,-0.6928908229,0.7701465487,-0.2680769861,-0.1214702353,-0.6213663220,0.2482643723,1.5134902000,-0.0739834234,-1.4493509531,1.1660684347,0.1881067455,-1.2092013359,1.2742975950,0.3987053037,0.3768714368,1.4138619900,-0.8284464478,-0.0801821947,-1.1233460903,-0.6918120980,0.5208147764,-0.0682908595,-0.9105952978,0.1875498742,-0.5142341852,-0.5274975300,0.7494561672,-0.7152031660,0.4310196340],[-0.4823740721,-0.2510137260,0.1944553405,-1.3107248545,-0.1923373193,-0.1888710707,0.2006835639,0.8063291907,0.7279618979,0.2301612198,0.5022042990,-1.6600764990,1.0828719139,1.8728606701,0.6034314036,-0.4063622355,0.0974950045,-0.8997538090,-1.5935285091,-1.0190300941,1.0629904270,0.0831695795,-0.4738741815,-0.4485514760,0.3416296840,-0.1248249412,1.1692199707,2.2329859734,-0.3595784605,2.1218981743,-0.0728280321,0.4060373604,0.5449692011,0.8206561208,-0.4518414736,0.2907862067,-2.1724636555,1.1614646912,2.5593364239,0.7427705526,0.9906321168,2.0261092186,-0.0148052406,-0.2360571027,0.0363243632,1.2760270834,2.2971293926,-1.1610804796,-1.7893850803,-0.3227211535,-1.8779487610,-1.3441138268,-0.8956089616,-0.0985141471,0.4440960586,-0.9870734215,-1.1733098030,-0.4680004716,0.0355096310,-0.9436638355,-0.1876551807,-0.2832317352,1.1657112837,1.5924739838,1.0328596830,-0.4097533226,-0.4622452855,-0.5567474961,1.1893956661,-0.3589572310,-0.8171840310,-0.3599012196,0.5284134746,0.6601926684,-0.2853067815,-1.5502932072,0.6783145070,-1.3337225914,0.9995742440,-0.2768847942,0.7775377631,0.6332675219,-0.1205610335,-0.9084852934,-0.1494361758,0.3467653990,-0.8398606181,0.4216800332,-2.1474878788,1.0741838217,0.5535035133,1.4500633478,0.5962111950,-0.3862569332,1.0310724974,-1.6946784258,-0.4748837650,0.9095218778,1.2876939774,-1.8530343771,-0.6701546907,1.8218663931,0.6484426260,-2.1453423500,0.1260983646,0.6405251026,-0.1364991963,-0.5790331960,-1.1921503544,-0.0568375923,0.2709504068,0.9835256934,0.8560069799,-1.0371228456,0.1756413430,0.4604872167,0.1207154393,0.0869964808,0.1099092290,0.7736787200,2.4697232246,-0.8884851933,-1.8836847544,0.4033220410,-0.9149255157,-1.0660659075,-0.1941040903,-0.9507152438,-0.6525584459,-0.2845203578,0.5010703206,-1.1385391951,-2.5650165081,-1.0482982397,0.7601513863,1.0343254805,0.1622663587,0.4804539382,0.4948939681,-1.1310155392,-0.4456132054,-0.0148028927,0.6583849192,0.8705469370,-1.4775637388,1.5122822523,-2.5054395199,0.6433568597,1.1670595407,-2.4150571823,-0.7000072002,0.4520107210,0.2669504285,0.0745046884,-0.0961903110,0.8950932026,-0.0153444046,2.3216211796,-0.0087271752,-0.7285081148,0.8591826558,-0.0213805623,-0.9274692535,0.8245937228,-0.0500126071,0.9498620033,0.0939764380,0.2396673709,0.3366479874,-0.0664266646,1.4263436794,0.2282274365,-0.6703722477,1.6196011305,1.5359443426,0.8145934343,0.0678382739,0.0837390274,-0.2199727595,-0.4387906492,0.3200174570,1.2475085258,-0.4494763911,1.4635165930,0.7197579145,-0.1485113055,0.0235467963,-0.0737998858,-0.1217009351,1.1287385225,0.3565534651,0.0661123544,-1.4903062582,-0.1748026311,-0.3600355685,0.4882857800,0.7067445517,1.7316313982,-1.5993480682,1.1871122122,0.0806279257,0.1233804226,1.4097801447,0.0073844674,0.4032472372,0.1401428878,1.4815948009,-0.3940936625,0.9161916375,-0.1888489723,-0.2546358109,0.1415531337,0.2554948926,0.5829980373,0.5339160562,0.1090676412,1.0032021999,-0.1615457684,-0.6464024782,-0.5283038616,0.3620505929,2.9733176231,-1.0681328773,0.7487841249],[0.8224366307,1.5359381437,2.7004611492,0.6946715117,0.9688691497,-0.2185768485,-1.7118986845,0.6170027852,-1.4078933001,0.2380145192,0.9218633771,0.8282817602,-0.2167852372,1.2319794893,1.1953899860,-0.7662591338,-1.2329901457,-0.6564483643,-0.7983369231,0.4234818816,0.4227693379,0.3662743270,0.3555461764,0.3225057721,0.0377100483,0.7185549140,0.1276012510,-0.8430910110,-1.1099169254,-0.7077657580,0.9883331060,-1.2329790592,-1.6483401060,0.0518403128,0.4276541770,-0.2361366451,1.1640164852,1.8784108162,1.5808711052,-0.7807156444,-0.8628751636,-1.4693769217,1.1178039312,-0.8612729311,0.3775274754,0.0553550571,-0.2481898218,1.1510961056,1.4155236483,0.4141672552,-0.1486062407,0.1500177085,0.7754682302,1.1698340178,1.0434730053,-1.3811140060,0.4370983243,0.5352761745,-1.8544534445,1.2646948099,0.2929232419,0.0995730236,1.7987678051,1.3470214605,1.2089182138,0.1961900741,0.1137968972,-1.6796602011,-0.0636957809,-1.4094570875,0.2016211897,-0.8527985215,-0.3630916178,0.8872061968,-1.0532958508,0.3649423420,0.8411453366,-0.3478016555,-0.1131411865,0.4069535732,2.3565943241,-0.6662643552,-0.9734390974,-0.6908809543,0.9773233533,-0.1573923081,-0.7164798379,-0.4443368018,-0.1431108266,1.3523162603,1.8951483965,0.3178819716,-0.5887733102,0.8732778430,-0.6213280559,-1.3105956316,-1.8258597851,0.3260253668,-0.9911643267,-0.7561984658,-0.2424144000,-1.4406697750,-1.8866000175,-0.4525777102,0.4981562197,2.9362752438,-0.4877498746,-1.0544553995,-0.3383483291,1.2915300131,-1.2027258873,0.3907198012,-0.0700768232,0.8171388507,2.8624243736,1.7433649302,0.0142839653,-0.7587402463,-1.0983080864,-0.1429131776,0.9946195483,0.2118909806,0.2142890394,0.7555453181,-0.7680301666,-0.7556154132,0.1945812553,0.9759526253,1.1216096878,-0.0747907311,0.7729748487,1.2828271389,0.2414376140,1.2615263462,0.6379812360,-0.4018948674,-1.7069851160,-0.4766686261,1.3859781027,0.9483999014,-2.4435849190,0.9283960462,0.1634889543,-0.3825234175,1.8444286585,0.2194792330,-0.9868534803,0.5217725635,1.0495344400,0.8497440219,0.6917409897,1.0312966108,1.2406227589,-0.3547858298,1.9061973095,-0.7241984010,0.4865470529,0.6772283316,-0.7669066787,0.3942653239,-0.8978323936,0.8897718787,0.2239701748,2.4911932945,0.3996796012,-0.5503692627,-0.8688344955,-0.4812298417,0.0351235606,1.9813669920,1.5126305819,-2.0804917812,1.2661659718,0.0236697476,-0.6080989838,-0.9334201813,-1.1945544481,2.2148840427,-0.0787384063,-0.4709452689,0.3463014662,0.2351722717,-0.6441224217,0.7173498273,-1.0471272469,-0.9710624814,0.3333386481,-0.0502374433,0.0522217862,0.7412327528,-0.9452767968,0.4947839677,1.9146962166,0.4911620319,0.6288027763,-0.2933082283,0.0901089683,0.7404099107,-0.1309597939,-1.9170587063,1.6293636560,-1.7506906986,0.4758150280,-0.6292086244,-0.1218387485,0.2569647133,-1.7762563229,1.1726000309,-1.1609822512,-0.4973379970,1.3849647045,0.2108934671,1.7501076460,0.1977767199,-1.5061140060,0.2052410543,0.8849083185,1.0420837402,-1.5378690958,-0.9945856929,0.3950080276,-1.1667097807,0.7336266637,1.0541245937],[-0.7604739070,1.0298671722,0.4831345081,-0.9305076599,2.3098583221,0.8978148699,-0.7780020237,-0.0763000399,0.1405724138,-0.9309204221,-1.2948073149,-0.9992104173,-0.7626069784,-1.1946477890,-0.0887558684,-1.1694561243,1.4181926250,-0.5647112131,0.1377933472,-0.8132561445,1.9780479670,-0.2693868876,2.5489280224,-0.4812024236,2.1438763142,-0.6436809897,-0.2257929593,-0.1043324471,-0.0241426136,0.3608437479,-1.3132336140,-0.2473047823,1.9679007530,-0.0772148073,-2.0944638252,-1.2689850330,-0.3390084207,0.0193167645,1.2736999989,1.4182841778,0.7875558734,-0.5130640268,0.0768273100,-0.9786463380,0.2218512148,-0.9420463443,-0.2778419554,0.2273272276,-0.4655575454,-1.0925114155,-0.1088153049,-0.8575080037,-0.7718771696,-2.1138706207,0.1061449572,-1.2214846611,-0.4387984276,0.6525202990,0.7284477353,-1.1624752283,2.0951714516,1.4666159153,0.5122377276,1.6254560947,0.5767518282,-1.4740482569,-0.3197341561,0.0622204207,2.2914700508,0.5785361528,0.5546657443,0.7368097901,-0.2011972219,-0.3459620476,1.4479689598,0.7183705568,-0.7114289403,-0.2445436120,0.9475785494,0.1670360416,-0.1054515019,1.5192409754,-0.2389630973,-0.3069087863,0.3849918544,-1.9849205017,-0.8939456344,0.1685190648,0.6565925479,-1.5131690502,0.3487373590,-0.4754433334,-0.5820451379,0.9503807425,-0.6067024469,0.4588730037,1.6100572348,0.5035930276,0.4674057364,-0.3812543452,0.4779359996,0.5542572141,0.8604867458,-0.0800580233,-0.5947277546,0.3877373040,0.9289138317,-2.0369527340,0.8429705501,-0.8200830221,-0.3597954512,0.1188161224,-0.8078942895,-0.6227641106,-2.1208841801,-1.0918895006,-1.2143417597,-0.9062018991,0.2425887883,1.0418492556,-0.8915598392,1.4448651075,0.5658872128,-0.6691966057,0.0779437646,0.3703321815,1.0202715397,0.5159808397,-0.3667911887,0.1917651892,-0.0155972950,0.6114939451,-0.5516857505,2.0143985748,-1.4417407513,0.1094853356,1.6910899878,-0.5058087707,-0.2841334641,-0.4427628219,-0.7798770070,0.3184933960,1.5184088945,-0.3982077539,-0.3256989717,-1.9115608931,-0.4265502393,1.9067032337,-0.5795803070,-0.9464166760,0.1618944108,0.2046693861,2.7043745518,-1.1141778231,-0.2270487696,0.0677588880,-0.8169810176,-0.0516205356,0.2156272531,-0.0458132848,1.1395378113,-1.9288947582,1.0327042341,-0.6658371091,-1.0002037287,0.7938078046,-0.5091620684,1.1180939674,-0.8911152482,-0.6446077824,0.1085245833,0.1545051485,0.7162056565,0.0998768061,1.5463054180,0.3392734230,-0.1677765995,0.0178067535,2.2100381851,0.1243686378,-0.4336081743,-1.1434708834,-0.2915322185,0.2938709855,0.1741888076,-0.9570773244,0.2976694703,-0.8821976185,-0.0567142293,1.8015016317,-0.0966258571,-0.8744275570,-0.3556698859,-0.2422062755,-0.0088157170,1.0579258204,0.6251289845,2.1922950745,0.8293181062,1.1037898064,0.1161758080,1.3735295534,1.8517149687,1.2776745558,1.1805377007,-1.1309832335,-1.4316225052,-0.7249827385,0.4579662681,-1.0276892185,-0.9429395199,0.7407530546,-1.1371870041,0.8070212007,0.8429144025,1.3206818104,0.6324065924,1.9122589827,1.9089622498,0.9503044486,1.8724390268,-0.3448031545,-0.0643757060,0.5141147375],[-1.0741199255,-0.6546777487,-0.2585322559,-0.4660071731,0.2123210430,-0.6828444600,-0.3794018924,1.5062401295,0.1076194942,1.1968497038,1.3637975454,-1.5723030567,-0.2526699901,-0.8103145957,-0.7729591131,-0.3791863918,-0.5915085673,-0.0657836497,0.5478895903,1.4897552729,1.2893457413,0.3316682577,-1.2735449076,0.3352213204,-0.1605404168,-0.6274160743,0.0444982313,0.4427059889,-0.1887391508,0.0424868092,0.6873580813,-0.5684332252,-1.8209024668,-1.5618481636,1.1418203115,-0.1679338664,-0.1575482935,0.8625968695,1.7374920845,-0.6851282716,-0.7020391822,-0.5403820276,0.0701741204,0.6840214729,-1.0883448124,-1.0662660599,0.3783966303,-1.2807009220,-0.6299644113,1.1646709442,0.6178892255,0.2659975588,0.5481295586,0.8629608750,0.3323490322,-1.7330682278,0.7170284390,1.3835469484,-0.1560150534,0.6267575026,0.0135885328,0.5174806714,-1.3381071091,0.2403987944,-0.6325420737,-0.3140203655,-0.5688653588,0.0899808258,0.2951270342,-0.3967520297,0.8154203892,0.6676385403,-1.0550323725,-1.7451119423,1.2019145489,0.8572792411,0.5403226018,-0.3976074755,-0.5785681009,-1.1575276852,-0.7872358561,-0.7130684257,-0.6214036942,-1.0228501558,0.0767670125,-0.4886718988,1.1801239252,-0.2055412382,-0.6231933236,1.3826036453,-0.8254976273,-0.2961417437,1.4713494778,0.2379731238,0.8579406738,0.3441528976,-0.5790320039,-1.0394935608,-0.3038409948,-0.4149534404,-1.0315365791,-1.7151025534,-0.5805560946,1.2495222092,-1.4802988768,0.2671444416,-0.6707246304,-1.2410545349,0.1354297549,0.9997917414,0.4972026944,-1.0840744972,0.6837875843,-0.3927797079,0.3305899799,0.6962628365,-1.0900052786,-0.6791628003,0.3703728318,-0.3813745379,-1.2939769030,-0.4187581241,0.2346390933,-0.2559554577,-1.0008970499,-1.3414121866,-0.4729372263,1.4051200151,-1.5527420044,-1.2766270638,-1.0660550594,-0.5740276575,1.2553514242,1.3542720079,-1.4046258926,0.3287573159,-3.0902717113,-0.8251926899,1.3747756481,-0.5134324431,-0.0683978125,-0.5844672918,2.3122715950,-0.1144952849,-0.1771259159,0.6621459126,-0.9766175747,0.9550002217,-0.2011980265,1.0007474422,0.2201173604,0.5645438433,0.1237144545,-0.7314903736,-0.0032809498,-0.6201736331,-0.3656804562,0.4783496261,-0.1402065754,-0.8107504249,0.4245607853,-1.9665911198,0.0242772475,-0.2220647484,-1.0378764868,0.8202825189,1.1987079382,-0.3498147130,-0.1208228320,-0.7644951940,0.0111101391,-0.5653514862,0.5157155395,1.1558949947,0.1688937247,1.6910996437,-0.8339630961,-0.7835124731,-0.0265033338,0.7289951444,-0.0396559015,-0.6672050953,-0.9940689206,-1.0564441681,0.9359007478,-0.7483726740,0.7249618769,0.7593082190,0.7683255076,-1.2546075583,-0.1201548800,-0.1855690032,-1.3772150278,-1.6465450525,-0.4068576992,1.6283794641,0.1573139578,-0.4508792460,0.4044491053,-0.5097197294,1.2035160065,0.4231516421,1.4437040091,-0.1109154448,0.2013794035,-1.0002640486,-0.3121077120,-0.6141272187,0.1143021211,-0.3780872822,1.0046535730,1.8508744240,0.5487187505,-0.7260112166,2.1805567741,0.3423650563,-0.8168572783,-0.4555746317,-0.3628090918,-2.2633161545,-0.6448135972,-0.7452431321,0.2334146351,0.8234783411],[0.5006471276,0.4734275937,0.6055223942,0.5627831221,-0.4720727503,-0.1004450172,-0.2453504503,-0.3004291356,0.2103607357,0.8166112304,-0.9881518483,0.6973307133,1.4465472698,0.7168634534,0.5701670647,0.9973092675,0.7807003856,0.0557702519,0.8964763880,-2.2280788422,-1.1141041517,0.7094801664,1.4749150276,-0.1043331474,1.5968731642,0.6388558745,-1.8664846420,-0.9645528793,-1.4716819525,0.1554246396,-0.7584255934,-0.6129170060,-0.1678290069,-0.2931369245,0.8345580697,0.4046899974,-0.2836818397,-0.1639537364,-1.3903989792,-0.4709609151,-0.1503601223,-0.4822956324,1.0982751846,0.7710070610,-0.7754664421,-0.2240831107,-0.0304530412,1.0826807022,-0.5305797458,0.6023666263,-2.1382906437,-0.0566323511,1.1664443016,-0.7720962763,-0.9089879990,-0.4142166376,2.5208303928,-0.1248839349,0.0180679932,-0.9986228347,-1.2627335787,-0.5651069880,-0.4552381933,0.5557269454,0.9943681955,-0.2865622938,-0.5457049608,1.6967656612,-0.5081860423,0.0036205626,-1.0185145140,0.2515618503,-0.3381818235,1.0141428709,-0.5825280547,0.6387898326,-0.3677871227,-0.5683578253,-0.7212477922,0.1982797086,-0.2174220383,0.9147565365,0.4437566400,-0.3062036335,-0.9490559101,-0.4480834901,0.4285705388,-0.1649990529,0.8649626374,-0.6476624608,0.7200577259,-0.3377147615,-2.0285484791,0.7261819839,-1.1678304672,-1.2852076292,-0.4837974906,1.2880570889,-0.1298787147,-0.1980783939,-0.3344875872,-0.3914431930,-0.6124061942,-0.6765239835,1.3272296190,-0.4486954510,-0.3164072931,0.0308305565,-0.3133567274,-0.1732591838,-0.3273687065,0.9443682432,1.1220173836,0.1123387292,1.3723402023,2.0625617504,-2.4485590458,-0.5619884133,0.3550988138,0.6343781948,-0.3924500942,-0.8131564260,-0.2328732610,0.2468012124,-0.5624132156,-0.8416020274,-0.9421200156,0.9453729987,0.1409107000,1.7535657883,-0.3472236991,-1.4894933701,-0.5080354214,1.9926500320,0.5554619431,-1.6960769892,1.7523661852,-0.5975508094,-0.4664914012,-1.0947922468,-0.6454249024,-0.2021548301,0.2050734758,-0.2703858316,-0.3102994859,0.3622126877,-1.8181655407,-0.1004303396,-1.0705139637,-2.0765106678,1.7234400511,-0.1522772014,-0.3638286889,0.3420642316,1.4070423841,-1.3161656857,-2.1506977081,-1.5808782578,-0.3616223037,1.1524333954,-0.4334156513,0.7738724351,-0.8342115879,-0.7282395959,0.6749752760,-0.4777715802,1.4923013449,-0.6583909392,0.5604837537,-0.5135775208,-0.4712557197,1.6082854271,-1.3844133615,2.2784225941,1.3843681812,-0.9408109784,0.0214956999,0.1947668046,-1.2912441492,-0.3801298738,-0.8062320352,-0.7280040383,-1.0826486349,1.0985850096,0.1326422393,1.8456447124,2.1873707771,-0.7083841562,0.6465144753,-0.5518972278,0.2373841107,-0.0164395105,-0.0291097742,1.6453253031,-0.5479516983,0.1853130013,0.6488700509,-0.3811258376,0.0817243606,1.0087109804,-1.1935225725,0.2637767792,-0.8994972110,0.3363832235,0.2713788450,0.5522467494,-0.6908838749,3.2648844719,1.3637685776,-0.5434085727,-0.8160778284,2.5939521790,0.5945166349,0.0170803964,-0.8449776769,-0.8151623607,-0.6342274547,0.9066578150,-0.6744567156,0.2777738869,0.2134030014,-0.6038596034,0.5620812774,0.4132402539],[-0.9826997519,1.0715109110,0.4687927365,-0.1042525694,-0.4841112792,-0.2296179980,0.9663137794,-0.5415360332,-0.3242196739,-0.0827071369,1.3674101830,-1.2061275244,-1.1793091297,2.2461023331,-1.5597070456,-0.7396000624,-0.1240326911,-0.0979401693,-0.8028639555,-0.9611197710,-1.2690706253,-1.4909585714,1.5873177052,0.1216608360,-1.4530825615,1.1822804213,0.6036565900,0.6341340542,0.6647869349,1.0432860851,0.1756239235,-0.1246411651,-0.0421862118,0.7873358130,0.0727108419,-1.8087786436,1.8381971121,1.0782800913,-0.3556542397,-0.9404379129,1.1988965273,-0.3806185126,-1.1939523220,0.2033593506,1.0147236586,-2.7281723022,0.7938317657,0.1125748605,0.5684316754,-0.6089876294,0.2789086998,0.9851705432,1.7537573576,0.2681056559,0.8519150019,2.4660148621,-0.2989487350,0.0854466408,-0.4715458453,-0.5302187204,-2.3346014023,-0.4460463226,-0.5775428414,-0.6922178268,-0.3119456172,0.3294471204,-1.3128342628,0.3397966027,-0.2910468876,0.9310884476,0.5599357486,-0.0645182207,-0.0022406222,-0.5151447654,0.5345206857,0.3004088998,0.2320223451,2.1810531616,-0.1001604870,-0.6051329970,-0.5047010779,0.0176231600,-0.5135729313,1.3429766893,0.4130504131,1.0463596582,0.8617561460,0.0478619747,0.0463153683,0.3848271072,1.6161820889,0.8716033101,-0.7510064244,1.6390513182,1.7839026451,0.5109132528,-0.4672897458,1.1222280264,0.9888499975,-0.9561136365,0.5663571358,1.4470038414,-0.7865455747,-0.4490454197,0.1197444052,-0.0093231238,0.5896588564,-0.2719801962,0.1982195675,-0.2045348287,0.4862102866,-0.9825719595,0.2153809816,-1.1695139408,1.4600381851,-0.4786804616,-0.2161083519,1.3162412643,-1.0952185392,-0.2379431874,0.0441102535,-0.4496622980,-0.5129234195,-1.9090203047,0.5068237185,0.2302060127,-0.7661902905,0.0725760013,0.9885923266,1.0863655806,-2.9028239250,-0.5360904932,-1.9112136364,1.8043782711,-1.2130129337,1.3801456690,-0.3277495205,0.2450582385,-0.7642333508,-0.3404273093,0.1624981314,1.7061364651,0.8927001953,0.8685998917,-0.5195849538,-0.0039161691,-1.3448412418,-0.3250283897,-0.7400012612,0.1771302372,0.6164767146,1.7528487444,-0.3680059314,-1.0127396584,0.7688501477,-0.2467727959,0.7563650608,0.9357182980,1.0622066259,2.3305165768,-0.5720127225,0.5467406511,0.5635195374,-0.9249114990,2.7588467598,0.8314345479,0.1939035803,0.7628485560,-0.4104726315,-1.9406030178,-0.3032291830,0.4363195598,1.2151136398,2.1058390141,1.1828629971,0.3339031935,-0.8795049191,1.2889955044,-0.8493562937,0.2111147940,-1.2545552254,-0.6699841619,0.1545667648,0.9607257843,0.9349159002,0.8397400975,0.6369614005,1.0919691324,1.8135930300,0.9556264877,-0.1990593225,-0.8042902946,0.6528923512,-0.0553436130,-0.9927421808,-0.5951382518,-0.0551041886,0.6262081265,2.3066689968,-0.2077073157,0.7109154463,-1.1168620586,0.5868679881,-1.0205169916,-0.4096455276,0.5808160305,1.0296586752,0.6482652426,-0.2194568217,-1.4834964275,-0.1184953302,0.4194943607,2.1360015869,-0.4918103814,-0.7194541097,-1.3561576605,0.2670113444,0.6546127200,-0.2104466856,-0.1670428365,0.5703315735,-0.4559410810,1.0440343618,1.2773948908]],[[-0.5565972328,1.6024007797,0.2661893368,-0.4304800034,0.2327338457,-0.6957453489,0.8529608250,-0.1704961210,1.0020779371,-0.4330396354,-0.1417545974,-0.1366312355,1.1569111347,1.1959494352,-0.6120578051,0.3761216402,1.3039016724,-0.6344389319,0.5409040451,-0.2106793523,1.7126660347,-0.1599847823,1.6979123354,-1.2285583019,-1.1358175278,2.6169531345,0.8238010406,-1.1758143902,1.1397869587,-2.3761627674,-0.2448349744,0.1050523967,1.4465601444,0.3278667629,-0.9939356446,-0.9700256586,-1.2412242889,1.1203885078,1.4587260485,-1.9321591854,-0.2354511768,-0.0673358142,0.4001037776,0.2343670577,0.7390849590,-0.2944712937,-0.3466485143,-0.1584270895,0.3662830889,-0.6116271615,-0.2264357954,1.5210326910,-0.6150326133,-1.5202091932,-0.5913832784,-0.4052457213,-1.4865701199,-1.9483112097,-0.5207968354,0.4274829328,0.3046709895,-2.0096416473,-1.5396296978,-0.4011812806,1.5112646818,-0.7303193808,-0.5129240751,-0.8286438584,0.0695798472,1.7603245974,-0.3305148482,-0.8455023170,-1.4774661064,1.2175356150,0.3046442270,1.5573654175,0.2028430849,0.1601100266,0.9338047504,-0.1322722286,0.5623129010,-1.1053166389,0.3339771032,0.7388482690,-0.5675597191,1.4587401152,0.9002025127,0.7754671574,1.2447451353,-0.9453841448,-0.0203836951,-0.7514674067,0.2120767683,0.2850379646,0.1254609972,0.2035339177,-0.3764954805,-0.9387801886,-0.1428789049,0.5332627892,-2.6065831184,-0.8216148615,-0.7328824997,-1.4360533953,0.2980169356,1.7235522270,-0.1267851740,1.2048579454,-1.2153682709,-0.4437190592,0.7440560460,0.9141807556,1.5864833593,0.6928018332,-0.9534311891,0.6793599129,0.5651528239,0.2193024904,-1.1105041504,-1.0860613585,1.1356219053,-1.1252977848,-0.3664375544,-0.0169660207,-0.5677758455,0.5198579431,-0.4156208634,-1.2413494587,-0.4517956972,1.2629374266,-1.9606316090,-2.0649142265,1.2586480379,-1.0318564177,0.6451462507,-0.0639000982,0.3058444858,0.3714894652,3.2189693451,0.8671776056,-0.6356307268,-1.2256245613,-0.6236977577,2.0042676926,0.3193460405,-0.2986065447,-0.0811628103,1.1276055574,0.4359809458,-0.8637416363,-0.8411214352,0.0813471526,-2.5876824856,-0.4164364338,-1.0778590441,-0.4280856252,-0.1837351769,-0.4342537224,-2.1249547005,-0.7090559006,0.4856309593,-0.2251652330,-0.4225297570,0.7283768058,-0.4008677304,1.7381820679,-2.5891318321,-0.0358014069,0.7630400062,1.2215707302,0.5623279810,-0.2767444849,-0.7788773179,0.6425501108,-0.2790244818,-0.0601394959,0.9226582050,-1.2856285572,0.5124117136,0.6543196440,0.6985861063,-1.1863965988,-0.7743391395,-0.8036898375,0.1511544138,-0.2061739713,0.2215616107,1.4245580435,0.6433502436,0.2171405405,1.0231777430,0.8845224380,1.2237645388,0.7285240889,-0.2870639265,-1.1345304251,0.4285645187,-0.9715670943,0.5583679676,-1.2529093027,-0.0554292351,-1.1577489376,-0.8337481618,-0.9095815420,0.5952699184,0.4276467860,0.4741272628,-0.4778809249,-0.5791444778,2.0132668018,0.0643128380,-1.7063057423,0.2727234066,0.1687286496,0.6589840651,0.2412974089,0.3876089454,0.1535113156,-0.7328495979,0.6013470292,-0.8155531287,-1.5449550152,0.8408113718,0.0877092257],[-0.5942927003,1.5020433664,0.5605551004,1.1409633160,1.9613232613,1.1202725172,-0.3546351194,2.1160829067,0.5874300599,-0.6321915388,-1.2108942270,0.1716439277,-0.2549261749,-1.2034045458,1.0263698101,0.9422594905,0.2088890970,-0.1047871783,-1.2705695629,-1.1836208105,-0.7867333889,1.7524260283,-0.7368371487,0.3624839783,-0.5364046097,-1.8370509148,0.1137978658,0.6024752259,-0.2508539855,-0.2528652251,1.6051051617,0.0342720747,1.5153194666,-1.0420209169,-0.4630776942,0.1474878788,-0.9713253379,0.0588521026,0.3362514973,0.2530673146,0.3107353151,0.1848080754,0.2764378190,-2.1524662971,2.7178449631,1.1053645611,0.8582649827,-0.1122928783,1.1051639318,-0.7339845896,0.0696830601,-0.2674444020,-0.2647155225,1.1227550507,0.2607275844,1.8020198345,-0.6892679930,-0.0933775827,-0.2251587808,-1.3664307594,0.3179258406,0.0487223379,0.4735718071,0.0241525117,0.8647046685,0.6103027463,-0.2139733285,-0.4484854937,1.4012920856,0.3160155714,-0.3197103739,1.7200762033,3.0469586849,-0.2641649544,-1.4224961996,-0.7580975294,-0.9873674512,0.1957168132,-0.4102197289,0.2144638300,-0.7272182703,-1.2818526030,-0.3538502753,0.5934340358,0.0904515535,-0.5685857534,-0.7609325647,0.4194073081,0.8674047589,-1.3904449940,0.0177442655,0.8740746379,-0.5924335122,-1.7502161264,-0.0792060792,-1.3580681086,1.1974959373,1.0274500847,-0.8471487761,1.3548214436,-1.0140715837,0.9863840938,-0.9353408813,-0.8694415092,0.3170388937,1.4484037161,0.6906737685,0.8037082553,-1.4583121538,-0.3531333804,-0.3408255875,1.5303416252,0.5340528488,0.6250340343,1.3767137527,-0.5115512609,2.1108081341,1.5340073109,-0.0016632290,0.0813266411,-1.1583846807,-0.7179369330,-0.7177065015,1.1551008224,1.6540576220,-0.4208811224,0.4496422112,-0.9338514805,-0.5616552830,0.6697068214,0.3913882077,-0.3269231319,0.2208180726,0.5489981771,-0.8624801040,0.0121480618,1.1665678024,0.1535249054,-0.1479054093,-1.0810278654,0.5306854844,-0.4219224155,-0.4571480155,-1.6084355116,0.9218780398,-0.2359026372,0.2739748061,0.2933753133,2.9100708961,-0.9172291756,1.7333257198,1.4234604836,-0.3695767522,1.8046455383,0.5196769238,-0.0223775655,-0.0609745122,0.4093607068,-1.4313142300,0.4745879769,-0.2658665776,0.1218895316,0.0092011830,0.6915896535,-1.2806612253,0.4025485814,1.1575285196,0.4067355394,0.9415536523,0.4428307414,-0.3000774086,-0.9426550865,-1.0320085287,0.7782731056,0.6642094254,0.8447062373,-0.7672371268,-1.0206696987,-0.1935468763,0.6972218752,-0.8549840450,0.0458985455,-0.0316515081,-0.9947295785,0.0599444099,-1.6356289387,1.4886174202,-1.1804332733,-0.5649894476,-0.3141291738,-0.5540065765,-0.0268782862,0.1907479763,0.3300601244,-0.0094888257,1.2847651243,0.7848905921,-1.8625837564,-1.5045292377,0.7757042050,-0.5151198506,3.4884924889,0.6410551071,-0.8507729769,0.2530885935,0.4635319412,1.5692179203,1.8518171310,-1.3878117800,0.6403602958,-1.1527994871,0.5967409611,0.1180173531,0.5744929910,0.2153960019,-0.4859421551,0.2957971692,0.4451091886,1.2154741287,0.3185328245,0.1403361112,0.0406874493,0.5657766461,-2.6195344925],[-0.7967351079,-0.9214567542,-0.7220990062,1.1061993837,-1.4456450939,-0.2186345011,-2.0822770596,-1.3654150963,1.8305156231,-0.0043937741,-1.7268742323,-0.4598450661,2.7442691326,-0.7711686492,0.2119555622,0.6657399535,1.8725705147,0.8713087440,-0.0707813352,-0.2834530771,-0.1813057214,0.8837855458,1.0758835077,-0.0943579450,-0.2998608947,-1.2143330574,0.1564082950,-0.6589041352,-1.5153027773,-0.4579784274,1.2980852127,0.2589216232,-0.6551944017,0.5270182490,0.7990094423,0.5013885498,-0.5969684124,0.9251852036,0.1497516632,0.3835472763,0.2245106548,0.3399053216,0.9988560081,-0.2961629331,-1.2441223860,-1.0278291702,0.6651023030,-1.1447552443,-0.9833480716,-1.8347622156,0.7757753730,0.4770247340,-1.5743138790,-1.1904482841,-1.2004079819,0.4256134331,1.4869117737,-0.0860315859,-0.2575958669,-0.0120158978,0.4862571061,0.0525990017,0.4884842336,0.2983732820,-1.5491154194,-0.9905139804,0.6269329786,0.1086431965,-0.2456014603,-0.1656596661,-0.9914613366,-0.2911224961,0.3206650615,-2.0059871674,-1.4480019808,0.7958505154,-0.9722334743,0.0591765344,-0.6312087774,0.2814213037,-0.4718524516,-0.4777416587,-0.8816665411,0.5701669455,-0.8234921098,-1.6473771334,-0.2114145905,0.7337220311,1.7068860531,0.1092660576,0.5084820390,-1.0156943798,-0.6823235750,0.4121778905,-0.0907771587,0.0038814421,2.6090002060,1.1189103127,-1.9125928879,-0.3776756823,0.4361971617,0.0557256676,-0.0821820721,-0.7631011009,0.4519874156,-2.4382345676,0.1086765230,0.7393144965,-0.0481885150,-0.3635574579,-0.3686617911,-1.4666473866,1.1164562702,-0.9669422507,0.7040798068,1.2670004368,-1.7580957413,0.0831064582,1.8570041656,-0.5323807597,0.7726399899,-0.5263054371,-1.6371046305,1.4653873444,1.3102437258,-1.8714300394,0.7401259542,1.4640793800,-0.3641062081,0.5393807888,-0.4796035886,-1.0403650999,-0.4575759172,1.8171423674,1.2859230042,0.8897191882,1.0549505949,1.3600300550,-0.3504465818,-0.6137373447,1.4029437304,0.7413427234,1.4288338423,1.3940635920,1.4501439333,0.7185511589,0.4196183681,1.1800745726,-1.0972715616,1.0779728889,-0.3046537638,0.1432365626,-0.7692264318,0.4684534371,-0.4527581930,-0.0037421312,0.9633156061,-1.6476392746,-0.6606883407,0.7935672402,1.4174605608,1.2771332264,0.1160060540,0.8817456961,1.2700384855,0.5530409813,-1.2621611357,-0.3488104641,-0.0174375270,0.5478835106,-1.4185255766,0.0217602048,-1.1966099739,-0.6348711252,1.1889677048,0.0976005048,2.5740497112,-0.3773222566,-0.4613899291,-0.0986436456,0.0541701391,0.2876920104,-1.7548836470,0.9253419638,0.2831991613,1.9411002398,1.4710776806,-1.4093031883,-0.5841726661,0.6404128671,-0.8920534253,-1.5517531633,0.4828912020,-0.2645669878,1.1087429523,-0.2828297615,-0.0158070270,1.5413781404,-0.8813383579,-0.4438351989,1.0622408390,0.9529253244,0.6253203750,0.9075292945,0.9945212007,-1.6742419004,-1.6183062792,-0.2126778513,0.6402559280,-1.2418143749,1.5323899984,1.8869425058,-2.1036224365,-0.2886890471,-0.2596866786,-0.2835617661,-1.5854977369,0.6081105471,-0.1809121221,1.2577123642,0.2052631229,-0.8163881898,-1.1063590050,-0.0912579298],[0.3450472057,-1.5712219477,0.8036522269,1.0187789202,0.4720780253,-1.2311170101,0.5175920725,-0.4290986657,-0.3666216731,0.6965180635,-0.1685263962,-1.5898982286,0.0220934488,-0.1997891963,-0.0530815497,0.8660960197,1.2025402784,-0.2280056924,-0.4191038013,0.9244895577,0.3037866652,-1.0225591660,-0.3245607018,-2.1816282272,-1.6213059425,-0.0621134415,0.5913257599,-0.1323050410,1.2850565910,-0.9554300308,0.2916534841,-0.1600199640,-0.0251585990,-0.8426494002,-0.2910325229,-0.0537761226,0.9424139857,0.6624855995,-0.5286464691,0.7076581120,-0.6090992689,1.5335679054,0.0755734891,-0.1748557091,-0.2534921765,-1.8603761196,1.1692599058,0.6042902470,0.9103702903,-0.1232990026,-0.4136359692,0.4350080490,-1.5793688297,0.6769706011,-0.4317162931,-1.4103405476,2.2864291668,0.0951946005,-1.4713804722,1.1892334223,1.1890237331,0.4429376125,-0.8387430310,0.0935311168,-0.7773849964,-1.8710905313,1.7024439573,-0.3445017934,-0.4553095698,0.7642885447,0.9821681976,-0.4583004117,0.0066860346,-2.2333476543,-0.1283345819,-0.5246235728,0.9105899334,-0.8629112840,-1.2106477022,-0.4193559587,0.0832006633,0.8583486080,-0.7869291306,1.7270901203,0.6822558641,-0.7230071425,-0.9777082205,-0.3344391882,-0.3701584637,-1.7645907402,-0.4649545252,1.0170435905,-0.0217694566,-0.9039642215,-1.6144140959,-0.7381874919,-0.6702867746,-1.3525016308,-0.4430875778,0.7287974358,-0.2385735363,-1.9752480984,2.0346944332,0.7689108253,-1.3528065681,0.6410573125,1.3192390203,0.3270509541,1.3804658651,0.2953875363,0.2034599483,-0.0645830110,-1.0361405611,0.5138510466,0.4093931317,-0.1102928594,0.0347731411,1.5495173931,0.4664959610,-0.3283931017,0.8254403472,-0.5107836723,-1.5615645647,-1.3665698767,0.3007652760,0.2336972654,-2.3706405163,-1.4505566359,0.3662475348,-2.1614456177,0.1082526296,0.0599631332,-0.4569309950,-0.0464944579,-0.8626391292,-0.5210534930,-1.3991318941,-1.0450747013,1.2357361317,0.7472088337,0.0294626709,-0.0323381051,0.8234859109,-0.6799733639,0.4621934593,-2.5489547253,-0.9245632291,-0.7865313292,0.6090202332,-0.3597594500,-1.5277130604,-0.7501274347,0.0724710599,0.3506417572,-0.0656960309,-0.4374740124,-0.4982353151,-0.1413731575,-0.6359506249,0.5705936551,-1.5379847288,2.0364022255,-0.4719463289,1.2827460766,-0.0661825612,0.2151302248,2.1430366039,-1.1678416729,-0.7821418643,-1.0774587393,0.0208406113,2.9643104076,-0.7014997602,-0.6581448317,0.8276968598,-1.2026946545,0.2845969498,-0.2574639916,-2.1206636429,0.2223912179,1.0973002911,0.5591518283,0.4466454387,0.2330162525,-0.7495750189,-0.0029619916,-0.4267501831,-1.6162149906,-2.1914863586,-0.6577326059,0.4989123344,-0.6556199193,-0.0895094723,-0.0112790493,0.0701447576,0.2679291070,1.5603930950,-0.9185328484,1.2158505917,0.6496582031,0.9756454825,1.3745002747,-0.8222441673,0.2244423032,0.1343432516,-0.3055539429,-0.9232123494,-0.1694074422,-1.1765798330,-1.1712293625,0.3721295893,-0.6939271688,-1.7612162828,-1.1052387953,-0.8033535480,-0.2181822509,-0.8658139706,0.2206589878,0.6181307435,0.5405413508,0.5935147405,-0.4217613637,0.4479676485,1.5241919756],[-1.2515456676,0.3748424947,2.0765490532,0.3203200698,0.8813945651,0.8924462795,-0.1924320161,-1.5929996967,-2.3032474518,0.1078558564,0.6173242927,-0.8837097287,-1.4029815197,-0.5150201917,0.1820349246,0.8237689137,-1.0956190825,-0.6264654398,2.0228900909,0.5752149820,0.0658197179,1.3183259964,-0.6357634664,0.8268523812,-0.3430708945,-0.0185604282,-0.7815250754,-1.1980940104,-2.0872871876,0.4336906970,-0.4385131896,-0.5323140025,-0.3113310635,-0.3358350396,-0.8802808523,-0.1359132677,-1.0820661783,0.6475699544,-0.2714052200,0.3617736399,-0.2199012190,-0.2596733272,-0.9010024071,0.7449892759,0.4877879322,-0.9734915495,-1.0764521360,0.6607186198,1.9671131372,-1.5323868990,-1.6164841652,-1.3796384335,0.9423570633,-0.2951205373,1.3519340754,-0.2381277531,-1.0184181929,-0.8733766675,0.4789345264,-0.8836178780,1.3279944658,-0.9801152349,-0.1271719635,0.2661816180,-0.9542296529,1.6332507133,-2.5235753059,1.0538122654,-0.7360922098,-0.3780718446,1.5566005707,0.2872518003,0.8437467813,0.3021098971,-0.5063041449,-0.5947359204,-1.4161554575,0.5075585842,1.0832319260,-1.0159112215,-0.7424957156,0.4478953183,0.6075967550,1.0550105572,-0.1584615558,-0.7829976082,-0.7657365203,-0.1861712188,0.2837609649,-0.3579142988,-0.1085585579,-0.2172659487,-0.5554970503,1.2288670540,1.9843388796,-0.7815829515,1.4802480936,-0.8235428929,0.8025032878,2.2790677547,0.9906148911,-0.6867852211,-0.7537479401,0.1955792606,0.0749062449,1.9827543497,0.9090567827,0.9319398403,-0.7689310312,0.2998859584,0.0846697241,-0.3599868119,-1.5779273510,-0.6001316905,0.7877557874,0.0443977341,-0.5797268748,0.2533348501,-0.2036195248,-1.3632600307,0.8793420196,-0.4388233125,-0.7233103514,0.5692442060,0.4922772646,-0.1630162150,0.3571025729,0.1961718649,0.2645778954,-0.7818415165,0.7580390573,-0.0369912013,-0.7612245083,-1.0428425074,-1.6256395578,1.5037935972,-0.1167479455,-2.5956823826,-0.7661041617,0.3693991005,0.4504629374,0.3117121756,-0.5613887310,-1.1832345724,1.2855526209,-0.6030053496,-0.6560151577,0.1852611452,-0.5990884304,2.4433631897,0.0511516444,0.4812693000,-0.0832996145,0.2482223809,-1.9432893991,-1.6592720747,-0.4858410358,0.5000698566,0.1994387358,-0.1384883970,-0.9286952019,1.1757085323,1.1086024046,-0.8868203759,-1.0367414951,-0.1277745217,-0.8232452273,0.4371286631,-0.1243734136,-0.3757143617,0.1735518128,1.7695791721,0.0277947448,-1.3792670965,0.3176753223,1.3203235865,-0.3832557797,-1.4734238386,-0.2980696857,-1.0584447384,-0.5828962922,0.5312491655,0.3283676207,0.9943018556,-0.7106354237,-0.9073405862,-0.5333749056,-1.0341758728,0.1330567151,-1.0525498390,-0.2770160139,-0.2599232197,1.0627063513,-0.0400909074,0.4328006506,1.7536995411,1.3945586681,1.4096744061,0.2681570947,-1.3833025694,0.3761285543,1.2858004570,0.6580190659,-0.1391668916,-2.1075122356,-0.6969650984,-0.4540668726,0.2670806050,-0.2743560374,-0.0209443346,-0.7997515202,-1.1989635229,0.0803686678,0.7327975631,-1.4023789167,0.1931434423,0.9743856788,0.8532840014,1.1824666262,-0.7256034613,1.2055327892,0.5179441571,0.4593677819,-1.0894452333],[0.9806094766,-0.2745089829,-1.4509284496,-0.4190232158,0.7034731507,0.5885317326,-0.8590492606,-0.1408797503,-0.1429534405,-0.1686502695,1.5083961487,0.0664177239,1.6594139338,1.6427066326,-0.3360191584,1.2271697521,-0.6477968693,-1.8092172146,-0.9437155128,0.2422796041,0.9709712267,1.1938385963,-0.2580324411,0.2235572189,-0.0080338093,-1.0137414932,-1.0294257402,-1.2949119806,0.4745123088,2.0134837627,-0.3586051464,-0.7135395408,-1.3953684568,0.3262714148,-0.9641054869,1.4204113483,-1.4024853706,2.1189253330,-0.5846406817,-1.1655416489,-0.8957322836,-1.1574605703,-0.1748732775,-0.4009687603,1.1054528952,1.9881278276,1.3893102407,-0.7504013181,0.7380410433,0.2013456821,-0.4352020621,-0.1771009415,1.5475465059,-1.0222883224,0.6397832632,0.8793185949,-1.1897361279,2.3307409286,-0.0984229222,-1.2100880146,-0.3881158829,-0.5199695230,0.4241195917,-0.0116548957,-1.4528745413,0.6298332810,-0.6663973927,-0.4127940536,1.3213293552,2.1988401413,-0.1545394957,0.6720430255,-1.5908136368,1.1947685480,0.6126347780,0.2747550309,-0.6444818377,-0.3922859728,-0.2779766321,-0.5158533454,1.0888648033,-0.4051799178,0.0381805673,1.0520000458,0.4421851337,0.7295815349,0.2826419771,0.3008019626,1.0624518394,0.0210728142,1.0455101728,0.0507967658,1.4170062542,0.6779723763,0.1280576140,1.0865745544,1.4637603760,0.8787689209,-0.3339741528,-0.9834817648,1.2178926468,0.2774706781,-0.3207702339,2.3072702885,-2.0267996788,-1.0228863955,0.2627939582,-0.6198943853,0.2880648971,0.1017346680,0.0353729799,-0.5069180727,-1.6419990063,-0.7368588448,-1.0442868471,1.7887995243,-0.0079993038,0.1718894988,-0.6887407899,-0.3678089082,-1.5573518276,-0.4891292155,-1.2914336920,-0.5052575469,-0.6315882206,-0.9011530280,0.1310378164,1.2318689823,-0.7219327688,0.8740233779,2.0904352665,0.2487895340,-1.0377489328,-0.4964048266,0.8279573917,-0.3556987345,-0.4572505951,1.0447623730,-0.7082029581,-0.1460758001,-1.6420407295,-0.5102956891,-0.6610686183,0.6488505006,-1.1323249340,-0.3774205446,0.6173532009,-0.2405151129,-0.0985287949,-1.2055841684,-0.2576752603,-0.3729512990,-0.9217440486,-1.7550470829,0.4352207780,-0.4867249429,0.5257884264,0.6555952430,-0.1585087776,-0.3430123627,-1.8562735319,-0.3113770783,0.5841275454,0.5621509552,-0.3464486003,-0.5415142775,0.0917498544,0.2995837927,0.3927473724,-2.8245084286,-0.3212784827,-0.3785587549,-0.7113556266,-0.6022580266,-2.6119797230,-0.0254186876,1.0269163847,-0.5338106155,-1.4207351208,0.8609966040,0.9684557319,-1.1536953449,-0.1923975646,0.7043721676,0.8859874606,0.4263644814,0.8193140626,-0.1051984951,0.1981133819,0.6787722111,0.4743790627,0.4614576697,0.4642948210,-0.9053800106,0.2947457731,-1.3450095654,0.4394025803,0.4799166918,0.4150917530,-2.1966297626,-0.0429796278,1.6986823082,-0.5377891064,-0.6587689519,-0.8812151551,-0.5194984078,0.0733721107,0.5582324266,0.3964279592,0.2120293975,0.3054826260,-0.2871125340,0.7834923267,0.3903766870,-0.0846089721,-0.1705707610,1.2015510798,-0.6100164652,0.5426404476,0.0102491574,-1.3187043667,-0.3852408826,0.4882010818,-0.6264355183],[0.6432433724,2.6491410732,-0.7734444737,-2.0508220196,0.3473735750,0.4667849839,-1.8796936274,-2.5656268597,0.2532318532,1.7867356539,1.5840818882,-0.9908537865,-0.1217880398,0.1486516148,0.5512774587,-0.2852463126,-0.7899572849,-0.3199961782,0.0614964701,0.0901733115,-0.3096054196,-0.1612850279,1.2000919580,1.2057310343,0.1887879372,0.4619420469,2.2673599720,-0.0099240970,-1.8448913097,-0.8179546595,-0.8862370849,0.2955748141,-0.1304718554,0.9787134528,0.8472347856,-0.1288524419,0.5706324577,1.2143545151,-0.5389542580,0.1138983741,0.7284115553,2.3312344551,1.2056900263,-0.9562098980,2.7902858257,0.1274458468,-0.5790623426,0.5577117205,0.6300501823,-0.1559722424,0.5293494463,1.9820027351,-1.3815310001,1.7998952866,0.7246081233,-0.1543004215,0.6131431460,-0.6942011118,-0.3525010645,-0.5144149065,-1.2278832197,1.4018414021,1.7004867792,0.6758140326,-1.8631590605,-0.0994367823,-1.4539935589,-0.8443304300,0.0253531449,0.1577637196,-0.8598079681,-0.1473255455,0.8919650316,-0.7243660092,-0.1229981408,-1.6209475994,-1.6732126474,-0.8255846500,1.2784086466,-0.3470903039,-0.1857288927,0.4499431551,0.5625912547,1.1732017994,0.1230131313,-1.8694161177,1.3007078171,-0.3756277859,-0.2890734673,0.0999187604,0.0808265805,1.2099618912,1.0885607004,2.2550585270,0.8889443874,0.7035228610,0.8246964216,-0.6405169368,-0.3325651288,0.0771268159,-0.6547657251,-0.8956646323,1.3768603802,-1.4912687540,-0.2551057935,-0.9616905451,0.4821599722,-0.1796675920,0.2831281424,-0.1340553463,1.0645695925,-0.6067219973,0.8581216931,-0.8533557057,0.1947769821,-1.3405125141,-0.0545026809,-0.0804695264,1.3081027269,-0.3672732115,-0.9950604439,-0.1673544943,-0.6271149516,1.0561723709,-0.8029696941,0.1047154367,-0.6536308527,0.4128457010,-0.3468890786,0.2208393216,-0.3895644844,-1.8310519457,-0.2431140691,0.1881937236,-0.7013317347,0.0772262961,0.3569529653,-0.5789977908,0.8872517347,-0.5732261539,0.4532896876,1.1999888420,1.3070069551,0.5669031739,0.6928222179,0.1783494949,-0.8095983267,-1.6969149113,-0.2449482530,0.8840457797,-0.4317087829,0.2946268618,-0.7384603024,0.8370640278,0.2964679301,0.6661210060,-0.0482210517,0.2334891409,0.2837832868,0.5236741304,-0.4918816090,-1.4834574461,1.5049879551,0.3132415712,-0.0655054599,0.2337341458,0.5793153048,0.4204308987,0.1527949274,-0.5522327423,-0.3710898757,-1.4416307211,0.3492567837,1.9169337749,-1.1547886133,0.1601953954,1.6751732826,-0.0111167338,0.3223155439,0.7473372221,-0.8906780481,-0.5188031793,-0.2556838393,0.7918527722,-0.1100981310,0.1180559546,0.8671013117,0.9141765833,-0.2264145017,0.9463275671,1.2277970314,-0.2910000980,0.3737040758,0.0451757126,0.0928840712,0.1828407347,-0.7114292979,-0.7026841640,-1.1814341545,-0.4881572127,0.0155485887,-2.7573442459,-0.6148686409,0.0310324896,-0.7967752814,1.1301997900,0.8300131559,0.2524907291,-1.0185047388,0.6041662097,-0.6465168595,-1.6469916105,-1.1552380323,-0.2510072887,1.3071713448,0.2094026506,-0.8110154867,1.0590174198,2.1924555302,-1.1202188730,-0.8820242882,-0.2305018008,0.4774096608,0.4056183100],[0.3316685855,0.7246740460,-0.2441137731,-1.1167860031,-1.3302159309,2.1725409031,0.8256790638,0.8306998014,0.0433921330,-2.6837403774,0.1183398664,-1.5742039680,1.3130691051,0.5735969543,1.3894512653,-0.2349755168,-1.4868464470,-1.5704554319,1.2836638689,-1.4032113552,0.0997329354,-0.3284315169,-0.7569980025,0.4101175666,0.2967536747,0.0999957025,-1.7917844057,0.1166789830,0.9223045111,-0.5208253860,0.3779269159,0.3759181201,-0.9334786534,-0.4578769207,0.3737753034,-1.1865646839,1.4694766998,2.1069328785,1.4723044634,0.8738681078,0.8008302450,-0.6091771126,0.6025853157,-1.7750355005,2.5449066162,1.9469372034,-0.0892248526,-0.7289734483,1.7631760836,0.3075451553,-1.0003085136,-0.4178240597,0.8397405148,-1.3949640989,1.2439329624,-1.2816982269,0.5767304897,1.1978013515,-0.0095124440,1.4640440941,0.9479848742,0.4466987848,-0.4483757317,-0.1509431601,1.0327948332,-0.4387851655,-0.7053198814,-1.7758510113,2.3635933399,-1.2871538401,1.2330980301,-0.0103951599,-2.0432476997,1.1372212172,-0.2039745152,1.1895712614,2.1988720894,0.4431562126,-0.6325745583,0.7957141995,1.5974450111,0.2394031286,-0.7119681835,-0.5758239031,0.2845700085,1.1215269566,0.3454354703,-1.1157178879,0.7630261183,-1.0396157503,-1.2194751501,0.4127437174,0.8683571219,-1.4072741270,-1.0504204035,1.4434605837,1.7263585329,1.0688557625,-0.6204966307,0.2786242962,0.0499957725,1.7350540161,1.0227462053,0.0887895748,-0.1811278611,0.0582000278,-0.4657486677,-1.7719552517,0.8368805051,-1.1324976683,0.3575865328,0.5382347107,-0.8788229227,0.5577881932,-1.4245994091,-1.3215843439,1.1583495140,0.6064372659,1.8785479069,1.2669287920,-1.6431622505,1.4687012434,0.1747233868,-0.7408152819,-1.7918393612,0.0706328824,2.3624007702,1.3255273104,-0.8967608809,-1.8747004271,0.0701705366,-1.8117138147,-1.1072317362,0.4218684733,0.9634417892,0.2764050663,-1.0511800051,1.2283619642,0.5277022719,0.9080645442,0.8044661283,0.7120621204,1.1845030785,0.6737482548,0.2570166290,1.1729450226,0.9283907413,-0.2583913505,0.2896552980,0.4090211391,-1.5187941790,-0.1357347518,-0.4136520922,-0.9496188164,-0.7845684290,-0.6877620220,1.4111977816,0.7903782129,0.2329047620,2.4450430870,-1.2297238111,-0.3623290360,-0.1056897566,1.4517501593,0.0444726944,-1.6526907682,0.1572376639,0.8969747424,-0.4539208114,0.3395579457,-0.7769137025,-1.1884061098,1.7207614183,-0.2107622027,-0.3655122817,0.7336954474,3.9021322727,-0.1486244053,0.0116885044,-1.4245262146,-0.8629705906,-1.4355378151,1.3567553759,-1.2594320774,1.1831315756,-1.1354767084,2.2162899971,1.4533467293,-0.6889243126,0.6834006310,0.1057725549,-0.8852538466,0.1678477079,-1.3591594696,0.5192753077,0.3029523492,0.7172673941,0.9541074038,-1.1572390795,-0.2347368300,-1.7002626657,0.2449831665,-1.1469874382,0.8641168475,1.4774310589,-0.0520692803,-1.3869181871,-0.2344624996,0.1029674485,0.8140225410,-1.2963079214,1.6603183746,0.0933022127,-1.0267788172,0.7393246293,1.4649448395,0.3346224725,-0.4709796906,0.3763844967,-0.2554529607,1.2848366499,-0.1008833647,-1.4221745729,1.3263037205],[-1.5512310266,-0.0674574524,0.1974201351,1.9573367834,-0.5765719414,2.5311250687,0.0921669155,0.5961879492,-0.5765373111,1.1144328117,1.0982080698,-2.1409747601,1.4250926971,0.9062033296,-0.4229671955,-1.7703527212,-1.4376366138,-1.2039889097,0.4187853932,-0.9995331168,0.0991715044,0.5590418577,-0.3176335394,-0.7645334005,0.1076866239,-0.2528138459,-0.5841336250,-0.4563258290,0.6959413290,-0.3995080888,-0.2161872089,-2.4993913174,0.1716826558,1.9722093344,-0.1077965498,-0.2359120846,-0.6924694180,0.3745718896,-0.6471085548,1.1404736042,1.0013725758,0.0168315694,1.0300970078,1.0599291325,0.0489125997,-0.7580710649,-1.4751648903,0.6038507819,0.0657226518,-0.1146567091,-1.2612990141,-0.1839995682,1.0401122570,-0.4304883778,0.1605222672,0.3470155001,-0.2415146828,0.6425074339,-1.8182214499,0.5159596205,0.1558146328,0.0451287888,-0.0311791990,1.4505674839,-0.9712655544,0.7230480909,-0.5097474456,0.1868300736,-0.7318282127,-0.4006540775,0.0385716334,-1.0778014660,0.9501018524,0.2879245579,1.0167944431,-0.4306978285,0.3810772598,2.1066005230,-1.9030117989,1.2764573097,1.3993682861,-1.0606230497,-1.3270933628,0.8174833655,-0.8962209225,2.0776956081,-0.2050100565,0.9393951297,1.3354527950,0.9743983150,-0.3406037092,-0.5477521420,2.6748812199,0.4349660575,2.3099956512,0.7869509459,-1.4177607298,0.9141976237,0.6136193871,0.6456041932,-0.6823461056,-0.4874903858,-0.1170524731,1.9651896954,0.0810671002,-2.3023612499,1.1835066080,0.8766115904,-0.9631479383,-2.1853756905,-1.2428277731,0.0720189661,-0.4796108902,-0.1266144216,-0.9395406246,0.8748876452,0.4869423807,1.9948581457,-0.6947699189,0.2270700186,-0.3317213953,0.6422330141,-0.4053525925,0.6357408166,0.1552224904,0.1389403045,0.9798236489,1.6588976383,0.6248997450,1.3167870045,0.1932007372,-0.8761605620,-0.5672311187,-0.2326073647,0.3890257478,0.2928788364,-1.3203250170,-0.9401980042,0.3664203882,-0.5633349419,1.0236297846,-0.0916399956,-1.0432430506,-0.7348905802,-1.4612178802,0.0253731403,-0.0812676325,-2.3844251633,2.1396183968,-0.3534555733,-0.5941804051,-1.4541046619,0.3458630145,-2.0107643604,-0.4030187726,-0.5923594832,0.2931079268,-1.4421180487,-0.9520923495,-0.9846225381,-1.5098954439,-0.7086515427,-0.0347315706,0.2972258031,1.1616653204,0.1510649323,-0.5060365200,0.0093106842,-0.0073053623,1.0132656097,-0.3046438992,-1.3726711273,0.0134564815,0.8839237094,0.9101039767,-1.4731736183,0.2599923015,-2.1118168831,0.9287536740,-0.9762788415,0.2646948993,-0.3983414769,0.8249574900,1.6295404434,1.0062322617,0.1468868256,1.0601128340,-0.1219236851,1.4774035215,-0.1063908115,0.3392222524,0.2582385242,-0.8551683426,1.0945085287,-1.0423394442,-0.4286342561,-0.9640535116,2.5036330223,-0.0431590155,0.9850620627,-1.0826802254,-0.0051798890,-0.8156575561,-1.6702399254,0.6556189060,-0.6202479601,0.8388648629,1.3374948502,1.1133385897,-0.6639132500,0.8211407065,0.7570949197,0.6676589847,-0.1204034463,0.2454028726,0.6149377823,1.1963888407,-0.5744004846,-0.7452492714,0.1945770830,-0.9737981558,-1.4163233042,-0.4329152703,0.0736613274],[1.3573538065,-0.1728477329,0.2341295928,-0.1851958036,0.8775706291,0.1613099128,1.1886209249,0.9928414226,0.5726636648,-1.0495097637,-0.7697876692,-0.0529748313,-1.2455561161,0.8108440042,-0.8752728701,0.4357014596,-1.5372251272,-0.7729937434,0.2567974627,0.4485482275,1.8845136166,0.2564440370,0.1844030470,1.7274739742,-2.3057599068,0.7775979042,0.5623003840,0.2061211616,-0.5265920162,-0.3165593147,-0.7587300539,0.5017391443,-0.0206113029,-0.5766850710,0.9890679717,-0.6054541469,-0.4035939872,-0.5519093871,0.3898775876,-0.8180458546,0.8589459658,-0.4786759317,1.0392462015,0.2402326465,-0.3096106946,0.2151809037,-0.6961288452,-0.4223067760,1.2795596123,-0.2458327711,0.1790923178,-0.7446967363,1.5123780966,1.8332709074,-0.1309396625,-0.5887723565,-0.1028404683,-0.8091694713,1.5737030506,0.0129924612,1.7106083632,0.3016880155,1.5860959291,-0.7288613915,-1.2153372765,-0.2323260754,-1.5699255466,0.4336345196,-1.5333116055,0.0587876514,0.6924380064,0.9327794909,-0.1513533443,1.1799548864,-2.2671871185,0.1525774598,0.8967125416,0.6222859025,-0.2084840834,-0.1403931379,0.1346957684,-2.0565283298,0.4950318336,-0.4319967330,0.9162702560,2.7992014885,1.5372692347,0.8877406716,1.0195660591,-0.4253476262,0.5052951574,-0.8647077680,0.2749290466,0.1537522674,0.5428519249,-0.6285081506,-0.4747620225,1.7780978680,0.5352087021,-1.5661109686,-0.5283264518,0.0906709209,2.0831224918,0.7579495907,0.5779919028,-0.2779310644,0.1220835596,0.6336193681,-1.0817463398,-1.1549593210,-1.4704773426,-1.1104469299,0.3906410336,2.1418421268,-0.4694980979,1.9122390747,1.2186927795,0.2590028644,-0.0949972644,0.5684119463,-0.9523119330,-0.1190907806,-0.6136029363,-0.0960134193,0.6986835003,-0.9704755545,-0.6268044114,-0.5610452294,0.6267525554,-0.4279605448,-0.3012984395,0.0593986809,0.9925241470,1.3790812492,1.0318965912,1.0415772200,0.2689839602,0.3476128876,1.5841513872,-1.3008955717,-0.6530005336,0.2141337395,0.9915444255,-0.1424658149,1.3862416744,-0.9926421642,1.7042853832,-0.8251213431,-1.5188190937,0.4262528419,-0.4792524576,-0.9867950678,1.0061618090,0.5120648146,0.5273358226,-1.1984391212,0.7163698673,0.6981152892,0.1147046685,-1.1462678909,-0.7785617113,0.9478487372,1.5295920372,1.4200506210,0.3724110425,-0.5420010090,-0.4563983381,0.6563023329,-0.4331812859,0.0931108072,-0.1951667070,-0.3696748018,-0.5538032055,-0.6306383014,0.8567004204,-0.2205395103,-0.8111694455,-0.8887060285,1.2530621290,-0.3134068251,2.2097487450,-0.7303216457,-0.8344842196,-0.4096687436,-2.4196662903,-0.0106813936,0.4485253394,1.8782069683,-1.8911976814,-0.1464397758,-1.3518520594,0.7725373507,0.0136093041,-0.6495004296,-0.5794745684,-1.0514913797,-0.2449943721,0.5634900331,2.0310289860,-0.7452318072,-1.1391593218,0.1187069863,0.6543679833,-0.0734154284,-1.2560356855,0.7408869863,-0.4925400317,-1.0945181847,0.0686069354,0.3952773809,-0.1935277283,0.4949653149,2.5847566128,-0.6973088980,-0.1285281926,1.4140560627,-1.1363401413,-1.0524957180,-0.5918847322,-1.4523485899,-1.9308159351,0.0701744705,-1.3957195282,1.1885744333],[1.2093832493,0.2166377157,-0.3030340970,0.0597176440,-0.5784593821,-0.1520032585,0.0584271438,-0.8972605467,0.6736978889,0.6967254877,0.4667727053,0.0286635868,0.2837365568,0.4581332207,0.3056941926,-0.0116381422,0.9084653854,-1.3365603685,2.8502945900,1.0096521378,-0.8975444436,-0.3106841445,-0.1088178307,-0.8972753286,0.9920865297,-0.0723804459,0.5263554454,-1.0233750343,1.2341985703,-0.3395036757,-0.0130303865,-1.0015542507,-0.3587833345,-0.5115778446,-0.9865487814,-0.1158994958,0.8093836904,1.4579401016,0.3082989156,0.2266820669,0.9964211583,0.0073694857,-2.0784134865,-1.7294149399,-0.4265480638,0.4939056933,0.7890198231,-0.4054355621,-0.5412698984,2.1344156265,0.7287756205,-1.2158207893,-1.0318105221,0.6429752111,2.7805309296,1.1857364178,-1.5435954332,-0.1381753832,-0.2538266480,1.1751824617,0.0941778049,-0.4272075593,-0.4489406049,-1.2893304825,0.3536476791,-2.1066591740,-2.2367551327,-0.5366755128,-0.5199564695,-1.3183445930,0.8450385332,1.8122462034,-0.1849884093,-0.5415422916,2.2770223618,1.5768443346,-1.4913719893,0.6609851718,0.9181861281,-1.9391117096,-1.4061818123,0.2801476419,-0.3844480813,-1.1504819393,0.3682684898,-0.6189881563,1.1590030193,2.6743111610,0.4546943903,0.2690356374,-0.1853425950,-0.2681950033,1.4984539747,0.0044779256,-0.3983008564,0.1030340418,-0.0268113017,0.1630320698,-0.4327528477,-0.7730445266,-1.0588603020,1.5333759785,1.5113227367,-0.3076021969,0.1370654255,0.0941180363,0.8745601773,1.1456735134,0.4186946154,0.3277677894,1.2020785809,-1.2507486343,-0.4188847244,0.6161471605,0.3914790452,-0.9654144645,0.2758603990,-0.0457552262,0.6747549772,0.9733185768,-0.6183296442,0.0650520474,0.7692757845,0.5387182832,-0.8589666486,-1.7312017679,0.0673047602,-0.0327091403,-0.1677000821,0.6749193072,-0.8480613828,-0.4110155404,0.1744018346,1.9193584919,0.2909437716,0.2042945921,1.2015007734,-0.1746187657,2.0067448616,2.2541844845,-0.9878281951,-1.7081015110,1.0036889315,0.8530163169,-0.0468556173,0.9961929917,-0.9784662127,0.4863100350,-1.1680382490,-2.0089857578,1.6244140863,0.0916079283,-0.3821515739,2.1537449360,0.3477901518,0.4862693548,-1.6556663513,-0.9747734666,-0.6179747581,-1.8434002399,0.2441639751,1.2477802038,1.0027505159,0.7601565719,0.4941864014,-0.8031545877,-0.1345568299,1.3695721626,-0.6639223099,0.3247958720,0.0043942472,0.1913578659,0.0745675713,-0.6932394505,0.0181727502,-2.3302776814,-0.7034760118,-0.3380787969,-0.1769035012,0.7856540680,0.1328079104,-0.3098344803,-0.9356276393,-0.0556659177,0.2006022036,3.4013428688,-0.1138542965,-0.7904957533,-2.3656320572,-1.6644995213,-0.0752599388,1.2415909767,0.4848107994,-0.2712499797,1.0699157715,0.6268231869,0.6100659966,1.6827350855,0.2057249993,0.4664883018,-0.8904610872,-0.6790813208,0.4590415955,-0.0912852958,1.3914935589,-0.7131012678,0.2295179665,-1.3685091734,0.4473334253,-1.7683360577,-0.5036785007,1.3914653063,0.5441012383,-0.5553440452,-1.2935613394,0.8311233521,-0.6089043617,-0.1083713174,-1.0062488317,1.0542385578,-1.4220438004,1.4223126173,-0.5420668721,0.7962802052],[-0.4968725145,0.7224124670,-0.9014311433,-0.0173226800,-1.8692378998,0.7442633510,1.1637054682,-0.9639398456,-0.1304800659,-1.3041782379,0.1481969059,0.1446544379,-0.3365060687,-0.5480625033,-0.5852605700,-0.5312329531,0.2836891413,0.4132503867,0.4652761221,-0.3472861648,0.1682231873,2.2558338642,-1.2319662571,0.5227214694,-0.4734502733,1.7072815895,-0.3913713694,-1.9416004419,0.1609739214,1.0534284115,0.6118378043,-0.8480007648,-0.6930434704,2.4445428848,0.3759610951,0.0456087664,0.3976513147,0.1482326239,0.0915998146,-0.1041516364,-1.3221958876,-0.8823789954,-0.9341890812,1.6293078661,-0.2838116288,0.0902046338,-1.2891747952,0.9338698983,0.0727711618,-1.2760754824,0.3651596606,-0.1811099648,1.4663398266,-1.3555800915,0.5154141188,-0.6398217082,-0.9735441804,0.9723172188,-1.4711934328,1.0343658924,1.1213243008,-0.2257323861,-1.5045515299,1.3349263668,-0.2323711216,-0.1518178135,-1.8108577728,1.8254790306,1.2762485743,-1.0044380426,-0.8060824275,-1.1536781788,-0.0594115667,-1.5260562897,0.6320920587,0.5781828165,1.8738344908,-1.7082090378,-1.2055032253,-1.6750184298,0.9773060679,1.7373665571,2.0059866905,2.3705921173,-0.9223784804,-0.6740148664,-1.4445827007,0.5108569860,0.4014914334,1.7079445124,-0.9667310119,-0.0251678526,1.2655253410,0.5475633144,-1.3689777851,0.9426730871,-1.2328010798,0.9257682562,-0.2959577143,-0.6005290151,0.5494224429,2.0729205608,-0.6353592277,0.3192138970,0.6131855845,0.1523712426,0.0587622598,0.2394811511,1.7130912542,1.2879834175,1.1557379961,-0.3637089729,-0.2145810127,0.9560777545,-0.0508564450,-0.1313996017,0.7307595611,0.5367805362,1.8643679619,-0.9024128914,-0.2627148032,2.0112290382,-0.5909074545,1.4932234287,-1.0802291632,0.6225048304,1.3895535469,-1.4506711960,-1.3074595928,0.2020994872,0.3432753086,0.2278840840,0.6360409260,-1.0779962540,-1.0535291433,1.5352840424,-0.2123942673,1.2253196239,-0.8935346603,-0.9941996932,0.3262534142,-1.1904532909,-1.7328804731,-0.2980934978,0.1107948944,0.5599442720,-0.4482952654,1.2198443413,1.1387603283,-0.1491803825,0.1041010097,0.4254328609,0.3654618561,0.9308492541,-0.3119447827,-1.3580926657,1.0101203918,-1.5451142788,0.2026536763,0.3477017283,2.3403952122,-0.0724377260,-0.1430427432,0.3848078549,-0.1609339267,-1.2239019871,0.0277106408,1.5609391928,-1.3043521643,-1.1645221710,-0.2278364003,0.7276721001,-0.1473096013,0.2433789521,-0.9658976197,-0.2630626559,-1.3427623510,-0.3987060785,0.6780408621,0.3158074319,-1.4230879545,-0.2691633701,-1.5128043890,-1.4413479567,0.0664281994,0.2924026251,-0.0307732169,-0.0908246115,-0.6818583608,-0.3328570724,-0.8593428731,-0.4004930854,1.3459855318,1.1941367388,-0.5967947245,-0.0030466982,0.7044922113,0.5484733582,0.1524624825,0.1162964776,-1.1390522718,1.6581176519,0.0612685196,-1.6157741547,-1.9226813316,0.7732533216,-1.3893855810,-0.9835121036,-0.4904064536,-0.7418014407,2.0079159737,-0.5394204259,-0.4050091803,-0.8270653486,-0.3081693947,0.2111128122,-0.2667162418,0.2949900031,0.1134409904,-1.0153115988,0.0493623950,0.2900373936,0.7097696662,0.7082123160],[0.3717222214,0.2401635796,0.0780070871,-0.9557723403,-0.1404657364,0.0148379868,-1.7255346775,-0.7337841392,-0.8126275539,-0.2047343999,-0.6537588239,-0.1437726617,-0.3824236989,-1.8198882341,-0.3698104620,0.7416635752,0.7381506562,0.3432342708,-0.6773697734,-0.5082986951,0.0642926022,-0.0629437044,1.2554669380,-0.1163146794,0.5489455462,0.6377999783,0.4149386287,-0.7792364359,2.1353731155,-0.5781716108,-0.8798395991,-0.9598072171,0.9588785768,-2.1271765232,0.6209558249,0.1527183652,0.3872463405,-0.5138054490,-0.0146184405,-0.2630323172,-0.9618073702,-0.3712114990,-0.8843356967,1.9723078012,-0.4195107520,-1.6190768480,-2.0020143986,1.8401244879,0.2102288306,0.0879932866,0.0495191440,-1.1159087420,-1.1406750679,0.5224584341,-0.2365093976,0.3528335094,0.5825575590,-0.0848055333,-0.1709183455,0.3926368952,0.7267042994,0.9824547172,2.0305740833,-0.3654267192,-1.7048155069,1.1908239126,0.5972874165,1.1509351730,2.3433780670,0.5069172382,-0.3373184502,-1.2413910627,-2.4824178219,-0.7841681838,-0.2749174833,1.4181344509,-1.1041259766,-1.2695769072,0.4413174987,1.4447129965,3.0843203068,1.1354111433,0.9596952200,0.0692582726,-1.8266723156,-0.1291946471,1.1994426250,0.1244117692,0.5631824732,2.2121365070,0.4624514878,-0.7957055569,-0.0160569772,0.1778953522,-1.3429569006,-0.0165645760,-2.1498680115,0.4199761152,-0.3032145500,-1.1267306805,1.4630862474,-0.0667405128,-0.1823576540,0.8276292682,0.2431415319,-0.0560194775,1.2737932205,-0.1540903896,0.5906959772,0.8791603446,-0.3834092021,0.0394203626,-0.5252980590,-0.2408296615,0.7998421788,-0.6245598793,0.8748078346,0.6978314519,1.8217324018,1.1725424528,-0.5058450103,-1.1825522184,-0.1523513198,-0.9517591596,0.3418970406,0.8446737528,-1.1665372849,0.3165706992,0.9696199298,-1.2162469625,-0.5531826019,-0.3029631674,-0.1280330271,-0.2350587249,-0.9241914153,-0.0322184227,0.0353361554,-0.5270760655,-0.2926283181,-1.0359057188,-0.5217298865,-0.0545432195,0.9752314091,0.4386629462,-0.1634233445,0.3504447937,-1.6003768444,1.5135135651,0.6002835631,0.8065099120,-2.4900567532,0.8633902073,-0.6167137623,-0.3883438408,-1.3907675743,1.3066443205,-0.7658559680,-0.5670421124,1.6960184574,1.9653801918,0.6198975444,-0.7111523747,1.6282720566,0.4226017296,0.2790413797,-0.9668534398,1.1160160303,0.6634340882,-0.2094600797,-0.4581094086,-1.0761604309,1.4729379416,-0.6079205275,0.6071959138,0.3631769419,0.0164318662,-1.3065373898,0.6455099583,0.1708582193,0.0014818548,-1.8358229399,0.8673004508,1.3817694187,0.7489470243,0.1228798330,0.7249348164,-0.2352134287,0.4495169818,-0.9030604362,1.0610578060,0.8425629735,0.6626394391,0.7027125955,-0.8264446855,0.4003860652,0.1967862993,-0.5250782371,-0.0714196265,-0.6816290617,-2.0005903244,-0.1332174838,-0.9352809191,-0.0467789285,-0.3418357372,-0.5217453837,1.4358524084,-2.8482091427,-0.1477034539,-1.8174880743,-1.4565387964,0.5249419212,-1.3412015438,1.0536248684,0.6991011500,0.9847779870,0.0895238072,-0.2042881399,-0.1511867195,-0.1911108941,-0.8483758569,-1.1394416094,-0.6263279319,0.3262504339,-0.4187115729],[-0.9000268579,-0.5546446443,-0.8508963585,-0.4535124004,0.0920558199,0.0759566426,0.2565580606,-1.8011915684,2.1070511341,-0.3016174436,0.5993249416,-0.6456158757,0.2734361589,0.2095994651,-0.9430122972,0.0983353779,-0.7922813892,0.6698214412,1.5299458504,1.0799561739,-0.4676948488,0.8438532948,1.4016919136,0.5662866235,0.6506659389,1.3225227594,-1.1737947464,0.0492078066,1.6108115911,1.2239793539,1.2101303339,-0.0170063470,0.1157550141,-0.5027875900,-0.5176725388,0.8887032866,0.5435528159,1.5120570660,-0.1064564437,0.6360615492,0.6020301580,0.8799878359,-0.0621630587,2.2621920109,2.6340875626,-0.5016446114,-1.1367765665,1.3416843414,0.4375230372,0.7510082126,1.6931953430,0.9512465596,-0.8695816398,0.9354771376,0.5091691613,-0.4463708401,-0.0330573730,-1.3820322752,1.2793287039,1.1265002489,0.3270427585,1.1429853439,-0.3402656913,0.3560847044,-1.0059175491,0.2028916031,-1.1150841713,-1.3619811535,1.0493425131,1.5576870441,-0.5290608406,1.1438901424,-1.5659375191,0.2392479181,-0.3553093076,0.0694101900,0.8125998974,1.6412402391,0.0621431805,-0.2580104470,-1.6815711260,1.6350235939,1.7907772064,-0.3013070822,-1.3498184681,0.9766227007,0.8020770550,0.1571407765,-0.3929232657,-0.3565121293,0.8001793027,0.7584725022,-0.9664206505,-2.0399837494,-0.7562393546,-0.0191347543,-0.1659532487,0.2461894006,0.0534501150,-1.4511636496,0.8290471435,1.4202630520,-0.6461420059,-1.1758359671,-0.9557312131,1.4096255302,-1.0442581177,1.8160980940,-0.3281961679,0.1080035195,-0.4689996839,-0.4098793268,-1.8007936478,-1.4797694683,1.2166144848,1.3121565580,-0.6322200298,-0.3365573585,0.7352458835,-0.2050330043,-0.4896292984,-2.1675162315,-0.0871381313,-0.6976063848,1.2434237003,-0.2033266723,-0.0167122856,0.1015309244,2.7080247402,0.2532948852,-0.8676658869,0.0866448730,0.9485441446,0.1438010484,0.1264382154,-1.1578149796,0.5896145105,1.3902870417,1.6312388182,0.3860552907,-1.3228954077,0.6822083592,-0.8747211099,-0.1168340445,-0.0816032961,0.0949813724,-0.2675711513,-0.2850045860,0.6082111001,-0.3288976252,-0.7892969251,0.8910753727,-0.4391539991,0.4801815152,1.5932966471,-0.7188422680,-1.0578689575,0.4190469682,-0.3070425391,0.4496214390,0.3089589775,-0.6861286163,1.1116139889,0.5191278458,0.7216537595,1.0761941671,-0.1724572033,-0.2763061225,0.1145950407,-0.0036275154,0.1950309277,-0.4134352207,0.2700786889,-0.4687091112,0.3374997079,-1.2691653967,-0.9487433434,-2.3327836990,0.4507500529,0.9267107248,-0.3292925954,1.1686035395,0.2126315683,0.1942957640,0.5142114162,0.8292499781,-0.2244424075,0.4714294672,1.2915810347,-1.1542066336,-1.3901450634,-1.0315078497,-0.5843418241,-0.3625900745,-0.6811479330,0.0109514073,0.4034639895,-0.7629212141,-0.6994753480,-0.5534796715,2.0600101948,0.9628931880,0.0250039492,0.1217478886,0.6854715347,-0.3456495404,-0.6771116853,3.5718197823,-1.3524976969,-0.7330499291,-1.1353269815,-0.6868650913,-0.1242099255,-1.8488539457,-0.1784221828,-0.5338147879,-1.4749981165,-0.9773634672,0.1918828785,-1.6867041588,-0.9012171626,0.9544080496,0.3894421458,0.8154364228],[-0.5928393602,0.7729503512,-1.2539441586,0.9535008669,-0.4108581543,1.1859742403,-0.7260681987,0.0907370001,1.5868970156,0.1683713347,-0.4438912570,-0.3875832558,-1.8504856825,-0.8048030138,1.2810553312,-1.0632429123,-1.1179351807,-0.6259436011,0.2457038760,0.7051655650,0.5252198577,-0.7441893220,0.3223232329,-0.8422292471,-1.6582146883,0.2706842124,0.5295950770,-0.6153351665,0.4503405988,-0.6227647662,0.5138043165,-0.2224375010,-0.1313886046,-0.9083277583,1.6377744675,-1.3036310673,0.9719820619,0.6772412062,0.5541941524,2.0696039200,2.1902463436,-0.9359971881,1.5101017952,-0.8271532059,1.1484556198,-0.7858132720,-0.8876923919,0.9454967976,0.4654867053,0.8108379841,0.9280185103,-2.0502116680,-0.2926620543,0.4862718284,0.8723092675,0.6239894629,1.6616897583,0.4626489580,-0.6597118378,0.7611908317,0.4519271851,0.1392660737,0.3012660742,-0.1104904786,0.5151509047,-0.4535919130,-0.5744910240,0.1589653343,0.6228471994,0.6753728986,1.4053241014,-0.3785912097,-1.3496842384,0.0238295868,1.2929308414,-0.7779426575,-0.0595474653,-0.0276483335,0.2683479786,0.3438584805,0.5323349833,0.0969401300,0.7111643553,0.6105911136,1.3095695972,0.3804032207,-1.2960375547,0.0383879021,0.4270890057,-0.9619238973,-0.7499616742,0.1058324948,0.6600143909,-0.1229051799,0.8982565999,0.6313751936,-1.2400015593,-1.3406934738,-0.7260510325,0.1453833431,-0.2229319066,-0.7966413498,0.0715794265,0.8668941259,-0.2490485162,0.9793452621,-0.4966464639,-1.3573478460,0.2178915888,0.0893864110,0.7565530539,-0.3809821308,0.3454945385,1.0413463116,-1.5530769825,0.6615539193,-0.6907436252,-1.0928965807,-0.3175019026,2.4032316208,0.1700476259,-0.2063445449,0.4634450078,0.9587469697,0.1602792293,0.4382282197,-0.6603542566,-0.1854121238,-0.7110162377,-1.0502065420,0.3033640385,-0.0090368362,0.9971413612,0.0437622704,-1.6310248375,-0.8269742727,1.7787942886,-0.7838104367,-0.0933950394,0.7300506234,0.8639712930,-0.7480256557,1.2835432291,-0.9399489164,0.7385509610,-1.0714541674,0.4192541242,1.2823365927,0.4535315037,1.6193950176,-0.8308219314,0.6912407279,-0.2242636681,1.3361150026,0.6657164097,1.0476800203,-0.3936673701,0.3589158058,-1.9339230061,0.4487364888,2.0723960400,-1.0328774452,0.7694115043,0.2298396081,-1.9235595465,0.8481762409,-1.0481160879,-1.7839934826,1.8505026102,-0.1471647471,1.4117530584,-0.4248147607,1.0637511015,-1.6170943975,-0.1204279289,-0.8135151863,-0.9916515350,-1.3610657454,0.2157028317,-0.4446689188,0.7377965450,-1.4382053614,0.8609056473,0.4337145686,0.4578846097,-1.0002191067,-0.2169005722,-0.3418893814,1.4400826693,-0.6663337350,-0.9091744423,0.3776062727,1.3034206629,-0.6550186872,0.0032101073,-0.8028377891,-1.3056483269,-0.1703900844,0.6938927770,-2.5968184471,0.6109979749,-0.1903804541,-1.7227438688,0.0623724312,-1.3457913399,0.3999651670,0.4954300821,1.3090932369,-0.4598393142,-0.6244497299,1.5473086834,-0.1307564676,1.5282249451,0.0975896195,1.3858814240,-0.4262283444,-0.0787541643,0.0616688319,0.5138702989,-0.3498030901,-1.3233053684,0.4028116465,0.3263849616,0.5358191133],[-1.7932841778,0.9994078875,-1.3617011309,1.1623231173,-1.7890124321,0.6985490918,-1.1428382397,0.2147274613,0.1396360397,-1.1729005575,-0.6266290545,1.2859871387,-1.0260407925,-0.5826871395,-0.1028865278,-1.7940053940,2.0454301834,1.0318255424,-1.1083452702,-0.4782355428,3.4777047634,2.4325971603,1.7189693451,-1.4776905775,0.5881302357,0.5027113557,2.8661079407,-0.1098669693,-0.1802531779,1.1672160625,-0.0681959763,-0.5658228397,0.3528211415,1.2640615702,-0.4119260609,0.4474448264,-0.6144564152,-0.3673212230,-0.0331863575,-0.2325524390,-0.5525498986,0.4451538920,0.3916987777,0.0216711983,0.8088118434,-0.2224570215,0.5602177382,-0.8138303757,0.0004522767,1.0792688131,-2.1361529827,0.9336476922,1.6088411808,1.8209517002,-0.8839716911,-0.2459857762,-1.2426134348,-0.1604559571,0.8543289900,-0.8487345576,0.7471719384,0.0556920283,0.4832492471,0.7741348147,1.4826248884,-0.9840801358,0.7220378518,1.0715200901,-0.4242345691,0.0419250503,0.8400822878,0.0718306378,-0.2302375734,-1.6227520704,1.2532012463,-0.6595457196,1.4351598024,0.4085188508,-1.9150269032,1.5044716597,-0.0597980022,1.1048128605,-0.4878143370,0.3595606387,-0.8408216834,-0.6972226501,-1.1642152071,0.9077614546,3.1851241589,-1.2781811953,-0.9502065778,1.4101806879,1.1578328609,0.5755327940,-0.3855011463,0.1394493580,1.5561326742,0.3201376498,-0.3746150136,-3.3304371834,1.5891109705,-0.4015725553,1.0671237707,0.4332732856,0.5515481830,0.8586447835,0.1360096633,-0.0430612415,0.8356793523,-1.8970340490,0.5594943166,0.8892461061,1.5231126547,-1.9689277411,-0.4791465700,-0.1135723814,-1.3317320347,1.2973388433,-0.7804918885,1.8386099339,-0.4639483690,-0.1304014027,0.4642805159,0.4224203527,1.5378631353,-0.6809688807,-0.7566910386,-0.8838561773,-1.6089427471,-1.5269782543,-0.5458895564,-0.0781098157,2.1578493118,-0.1167186201,-0.7636818290,-0.5750817060,-1.4051170349,0.8595709205,0.2047751695,0.5402824283,0.0225952975,0.5965875983,0.0429086573,-0.8807626963,0.2503642142,-0.2607371807,0.7006023526,0.1024034098,0.2675665021,0.8997215629,-1.3800114393,0.6278210282,-0.6009278893,0.0229070876,-0.8209497929,0.6459999681,-1.3651671410,-0.9422188997,0.8445447087,-0.5556010604,-0.2866741419,-0.6988923550,-0.2834370732,-1.5584450960,-0.3284163177,-0.2235734910,-1.2259905338,0.5285528898,-1.0578813553,0.3370731175,-0.1701500416,0.2106822878,0.1954178661,-1.2928462029,-1.0101048946,-1.0162965059,0.7101376653,1.1929751635,0.2930506766,0.3283641040,1.8373104334,0.1537463218,1.6340095997,1.5657008886,-0.1655842960,1.2367951870,2.0852239132,-0.3394646943,-1.6677970886,-0.1699476838,0.1307020038,1.3757307529,-0.1364614964,0.5837905407,-0.6506878138,1.0382732153,-1.1101716757,2.2023341656,-0.5978506207,1.2223081589,0.0622529648,1.0693994761,-0.1176077351,-0.5627841353,1.4076569080,1.7500611544,0.6664909124,0.2402775288,0.5856772661,0.1869566590,1.1902452707,-0.3557678163,-0.2289677262,-0.5311463475,-0.7184414268,0.8175365925,0.2107467204,-0.5429624319,-0.1168661341,2.7603781223,0.7776145935,0.1007762924,0.1165065393,1.5572880507],[0.3648286164,2.0247426033,-0.3560695946,-0.0204400718,-0.7988411188,-0.9985968471,-0.5758203864,0.2325834781,-1.2333204746,1.3030353785,-0.7864224315,-0.5838130116,-1.3188313246,0.2502593100,1.2273973227,-0.2414512187,-0.8900685310,0.4520162046,0.5013954639,-1.2339198589,0.5234656334,1.8722342253,0.2422109544,-1.3943357468,1.5739291906,0.3023069799,-0.1267141402,-0.4534452260,0.7923907638,-2.1231665611,-0.2892504334,-1.2547494173,1.1192709208,0.1841019541,-0.5269345045,-0.3538885415,-0.3898421228,-1.2952687740,-0.7927324176,1.1888612509,1.3829047680,1.0012235641,0.1509179920,-0.5995969176,1.0717393160,-0.6755110621,1.8635855913,1.7817964554,-1.5924193859,0.7871559858,0.1822643280,0.6299715042,-0.4798375964,-0.2566851377,-0.8125913739,0.0916184112,0.3510672748,0.7465915084,-1.2982705832,0.4566431642,-1.4501985312,-1.0551677942,1.1857924461,0.2242699265,-0.2082041055,0.1747808009,1.4196221828,-0.8689292073,-1.5527167320,0.0466211066,0.3829758465,1.4496682882,-0.6000432968,-0.2342979163,0.9764523506,-0.1101170778,-0.2743099630,0.5450106263,-0.6299778223,-1.0088422298,0.5787731409,0.6363965869,-0.9875950813,0.4465334415,1.6913824081,0.8224819303,0.6353470683,0.6847267747,-0.8218975067,-1.7893120050,-0.3205640018,1.1568148136,-0.6012837887,-0.0253048092,-0.8224580884,0.4368734360,0.5922030210,0.2948924601,0.4711373150,0.2339244038,-0.8675713539,-0.1623468995,0.2907239795,0.0648862347,-0.7679846883,0.4781250954,-0.9584442973,1.2950092554,0.4279106259,0.1744094789,-0.2890577018,0.4377071559,-1.0612857342,0.6737443805,1.4462653399,-0.0192514602,0.4609615207,-1.7821609974,-0.3013353348,-0.3335744143,-1.1731162071,1.3955233097,-1.2123579979,0.2251206934,1.1979632378,0.0889767930,0.5139861107,0.9769540429,0.3754872680,0.7002015114,-0.4718477726,-0.9561746120,1.3625870943,0.5210314989,-1.7651660442,-0.5209172368,1.1049113274,0.0181649495,0.4430385828,0.0258633532,-1.1045073271,-1.0388607979,-2.2845268250,0.8429036736,-1.1021239758,-0.2099017054,-0.6958201528,1.9615095854,-0.3146707714,0.7117929459,-0.9054128528,0.3786839545,1.2535066605,-0.5864236355,-0.6019744277,0.3552685678,-0.7152664661,-1.2186499834,-0.8128777146,-0.1254124641,0.5544507504,0.8033244610,-0.0788029656,-0.3765355647,-0.1530294567,-0.0445062034,-1.0340093374,2.1138308048,-1.4381253719,-0.3641769290,-0.7461310029,0.5013814569,-0.4317185283,-1.5411124229,-0.0315037817,0.1206132025,0.3322246075,-0.3178224564,-2.4873731136,0.9532303214,-0.2243309021,0.8475925922,-0.1884335577,2.1482484341,0.1499400139,0.5643538237,0.4252497256,-1.0068868399,-0.1776017845,-0.8614460230,-0.3621585071,0.3836019039,0.9026619196,0.9432954192,-0.3222702444,-0.6358183622,-0.6205145121,-1.2305152416,-0.1459206790,0.9727852941,-0.6101299524,-1.1514422894,-0.5444167256,0.2890828252,-0.1478310674,-1.2646784782,-0.7198884487,0.7411797047,0.3236621618,0.6102467179,1.9273406267,-0.7630124092,-0.8950296044,0.1038967371,1.3054035902,1.1023359299,2.0166347027,1.2310870886,0.3453706503,0.4032297432,-0.3820646107,1.4327986240,1.2221440077,-1.3450983763],[-0.6113471985,-0.0736481622,-0.4696511924,0.0348328836,0.2735553682,-1.7579873800,-0.2284794301,-0.2847909033,-1.1847519875,0.9780002832,-0.0279450770,1.1050629616,0.5406913161,-0.4539557099,0.3432274759,-0.5386041999,-2.3486204147,1.0494381189,1.0514642000,0.1160023734,0.4832822382,-0.0234396216,0.0183158182,1.3490015268,-0.5867910385,-0.5934441090,1.6418461800,0.8020620346,0.3180578947,-0.0109635685,-0.8504745364,-0.3389580846,-0.4799004793,0.2578036189,0.4269354939,-1.3231627941,-0.5754043460,0.1770341992,1.9017140865,0.4532558918,-0.7606176138,0.7406585813,-1.0882092714,1.8031427860,-1.3217358589,0.4403271973,0.3200185895,-0.0844190046,-1.0422834158,1.3751612902,-1.3435511589,-2.2676715851,0.1229116991,0.4220400751,0.8152249455,-0.5771619678,-1.5192883015,-0.1184898242,0.0107239718,-1.6067227125,1.5956927538,0.0263506193,0.9669717550,-0.8889227509,-0.4053593278,1.3851294518,-0.4103517532,-1.2005337477,0.8040002584,0.0697512850,0.6278099418,1.3422721624,-0.1635767967,-0.1250185519,-0.3778596222,-1.0309101343,0.5122585893,0.3585168719,-0.3668381572,0.7736611962,-0.5901617408,1.0048699379,1.6593602896,0.0736119524,-0.5228585601,-1.1776105165,-1.2693563700,0.7466336489,-0.4147734344,-0.6222800016,-0.4369660020,1.1900970936,-0.0045385598,-0.3916423321,-0.4234701991,0.4515590668,1.6646592617,-0.7858273387,-1.0369464159,-0.8603183627,1.8155955076,-0.9686090350,0.2055028230,-0.0671775788,-0.1541100144,0.1235477552,0.6986100078,1.5323183537,0.3527896404,0.5008907914,2.1498534679,0.3520805240,0.8896867037,2.5267851353,1.0206768513,-0.8351369500,0.4967375398,0.8105723262,-0.7931277752,0.2910664976,-0.7967676520,0.2498087585,0.3448672295,-0.1164075732,0.0368722863,1.3828399181,0.4213814735,-0.9003941417,0.5536766052,-0.4180638790,-0.4147278368,-0.9282360077,-0.7860546708,1.1377804279,0.4159342051,-0.7434610724,-0.7378995419,-0.0775714889,-1.1062474251,-0.1626705676,-0.0320903845,-0.6710093617,-1.9599610567,-1.0496373177,0.4051431119,-2.0050468445,0.3016331196,1.1520593166,1.2890785933,0.6122150421,0.4631689787,0.2665107846,-2.0319108963,1.7829045057,-0.5972828865,-0.2010582238,1.9431426525,-0.1263195872,-2.0065937042,0.9118897915,-0.4932822585,-1.6785792112,0.7633743882,-1.4086085558,-0.7630506754,2.4220027924,-1.4050699472,0.3170969188,0.3332512081,0.5363234282,1.2329015732,-1.1386669874,0.7243200541,-0.5739594102,1.1940883398,-1.3103320599,0.0189716071,-1.0934224129,-0.6630554199,0.6524068117,0.4000760615,-0.4352589548,1.8864926100,0.1805183440,-0.5843188763,2.0612359047,0.2389136702,-1.0740573406,-0.3438945413,0.5787284374,0.6672785282,-0.3425982594,-0.2763700783,-0.0430908315,0.3072268665,0.4272223711,0.7463321090,2.3031678200,1.1491247416,0.7463536263,1.6505769491,-0.7130047083,0.4484150410,2.5032739639,1.5361375809,-0.0852901042,1.7603957653,-0.7834419012,-1.5477027893,-1.2571730614,1.3132101297,1.6227086782,-0.1173110008,0.7305071354,0.3547957242,1.1275627613,-1.3428343534,0.1347589195,-0.2234453708,-1.6528010368,1.1485866308,1.3285199404,-2.2826619148,0.2097360492],[0.7691443563,-0.3663109541,-0.9105324745,-2.1862375736,0.4142344594,-1.2459869385,-0.5919961333,-1.1263803244,-0.7219035625,-1.2844762802,-0.6319314837,0.4255679846,-0.9710791111,-0.1546536833,-0.9671045542,0.8408493400,0.4944689572,-0.9499318004,-0.9606438875,-1.6236588955,-0.0765073076,0.1454095095,-0.6725363731,1.5028852224,0.3382230401,0.9947757721,-1.5230675936,0.3305849135,1.0095059872,-1.2323187590,-0.9129067063,-0.6490970850,0.7627770901,-1.5042232275,1.9205620289,0.6990779638,0.3981493115,1.7094851732,-0.8547058105,0.6071660519,0.3208642602,1.3405076265,-0.8251224756,0.7542603016,1.5279054642,-0.0644014403,0.0308947098,-0.4144746959,-1.2328224182,1.3450195789,0.5506920218,0.7623227835,0.4588803649,2.4532387257,0.4284676611,-0.6673672199,-0.7856223583,0.6798230410,1.3754965067,1.2486560345,0.8557427526,-0.6578801870,2.8154249191,-2.0665171146,-0.8970668316,0.8705096245,-0.5161067843,-0.8774737120,-1.3747942448,-0.0044820942,-0.0629592091,0.8383932114,0.1018836051,0.2315552235,0.8842407465,1.9258217812,-0.3826536238,-0.7059099674,-0.9088483453,-0.0775755346,-0.8568040133,0.6119330525,-1.1394847631,-1.0729565620,2.5123028755,-1.1320472956,-0.0720144957,-1.3061287403,0.0605295226,0.5201559067,-0.7216553092,-0.0817440078,0.4655059576,-1.0059340000,1.6584295034,1.3138953447,-0.1685477644,-1.7573693991,-0.5177462697,0.2565937340,-0.7753594518,-0.3755358458,0.9658132195,-1.6226822138,2.0549082756,1.1139340401,-0.4460828304,0.3770333529,-1.0984185934,-0.7914294600,-0.7182283401,0.8725944757,1.4336870909,0.1247830093,-2.4568533897,0.8807242513,1.4794603586,0.5188699961,0.1264361292,-0.9099682570,-0.3541872203,1.0382727385,-1.1467587948,0.0937129781,-3.0879652500,-0.7070456743,-0.8758177161,0.7819408178,-0.7893208265,0.3356752098,-0.3563073277,0.4893903732,-1.0363898277,-0.9055857062,-0.7337564230,0.1480658501,0.9461278319,0.3862171173,-1.0495040417,0.9934561253,-0.2928678691,2.2781779766,0.4375535846,1.4886761904,-0.2329892665,1.1606227160,-1.1704735756,-0.8371050358,0.4440965652,0.0336679667,1.2123416662,-0.3476849496,0.5047299862,1.0939006805,-0.1639223248,-1.1171123981,-0.8182387948,0.5606721640,-0.4868011773,-0.1622991860,-1.1784236431,-0.0316150524,-0.6546902657,1.9915369749,-0.1884418726,-0.1821149737,-0.5000926256,1.5627294779,0.5795538425,1.7219066620,0.3575980067,0.7396416664,-0.7974137664,0.1111887917,-1.3111822605,0.1688636690,-1.2101415396,2.0626072884,-0.1517615914,-0.1261129677,0.5988250971,0.3151803315,0.2292727828,1.5950623751,0.7607192397,-0.6130860448,1.2753077745,-0.0473849252,1.0907490253,-0.5991789103,1.1112835407,-0.7966700792,-0.9286405444,1.1475811005,0.7448164821,-1.0517472029,1.8815093040,0.4007554352,0.5530241728,-0.3934260905,-0.0581542328,-1.2024121284,-1.1866624355,0.4203956425,0.9341752529,1.2243565321,0.8899989724,0.3466911018,-0.7389758825,-0.9940132499,-1.0827097893,-0.6211391687,0.0189784765,0.6920895576,-1.3577948809,-0.1852061152,-0.3256109953,-0.3312273026,2.7818863392,-0.6032305956,-0.5881502032,-0.2108903974,0.9609686136,-1.7761542797],[0.1638919264,-1.8034422398,0.3438547850,-2.0366148949,1.2626676559,-1.0085458755,0.8398509622,1.1671663523,-1.6237409115,1.2003210783,-0.6494108438,-0.3809467852,-0.8366717696,1.0897408724,0.7005752921,1.8595799208,0.7068119645,0.2377082705,-0.4191710055,-0.0960142314,-0.4157134593,-1.4508297443,0.6521006823,2.3216221333,0.6430325508,1.4736698866,-0.4491530657,1.0629155636,-0.9583804607,-0.6451090574,-0.6868199706,-0.0433918200,0.9207156301,1.4857054949,0.3920950890,-0.4053091705,0.8437995315,-1.5179128647,0.1888910085,0.0703740716,-0.7147838473,0.2936773896,-1.1163834333,0.4440954924,-1.5620125532,-1.4867972136,0.9728970528,0.0287208781,0.2518751621,-0.8563625813,-0.0962809995,0.9173402786,0.1593800485,0.3545618653,-0.5275490284,-0.1848464012,1.1037555933,-1.7516676188,1.1288124323,0.4776148796,-0.0878350437,0.8084073067,-0.5648819804,1.1433855295,-2.9104449749,0.8359450698,0.0287205018,-0.3140828609,0.2962246537,2.6138598919,1.9440122843,0.2640841007,-0.9142881632,-0.5235984325,-1.0183387995,0.2797614336,-0.6935437322,-0.2669579685,0.8215245605,-0.2807157934,-0.4905641377,0.4833571315,0.2829817235,0.5185962915,0.6601308584,-0.3351995051,1.0649538040,0.1437657326,-0.1996699721,0.1168802828,1.0141957998,0.1364440620,0.6416521668,-0.0714805797,-0.1246618554,0.6482216120,0.7308468819,0.2386373132,1.0028398037,0.5547579527,-0.5872223973,-0.7867373228,-1.1379771233,0.0934178531,-1.1745666265,-0.7475016117,-0.5521036983,0.1610332578,-1.0222641230,-1.0741324425,-0.0681060180,-0.5505965948,-0.0340255834,-2.0789461136,1.2974512577,-0.2434522808,-0.6320980787,1.6777137518,0.2020441890,0.7013676763,-0.9742774963,-0.4643732905,-1.7642954588,0.2217412889,-0.8830285072,-1.1129447222,-0.6823812127,1.3527050018,-1.2479234934,-1.7996045351,-0.8925056458,-0.0426734425,0.5319902301,0.2391276062,1.0110605955,-1.4177103043,0.4677368701,-0.4861982763,1.6502017975,-0.2858231962,0.4730424285,0.4991550446,1.8836057186,-0.4640003443,-0.0351408608,-0.4028859138,-0.2978712618,0.0881469771,1.7128058672,0.0741602257,-2.0762367249,0.6053909063,-1.3824832439,0.5659181476,-2.0131797791,-0.3630668521,0.6377583742,1.1255798340,0.1482716650,0.0555110313,-1.0066103935,-0.0993747562,-0.5288520455,1.3735204935,0.4501582086,0.9103261232,0.5915299058,0.8719922304,-0.1844073534,-0.4059509635,-0.7417193651,-1.1776884794,-0.2048167139,-0.7765574455,-0.3557467163,-0.4653972089,0.1155521870,-0.0342244320,-1.6729363203,1.5951782465,0.3936387300,2.1581971645,0.0029275578,-0.2868529856,-0.9359773397,1.2794549465,1.4388250113,-0.2805157900,0.1168797389,1.3458018303,1.0483114719,-0.6029849052,0.7616730928,0.5633711815,-1.0731381178,1.0358542204,-0.7741640210,1.0627392530,-0.4310510159,0.1969968379,-1.5273967981,0.9693744779,0.4679141045,-2.0622494221,-1.4941822290,-0.2936666906,-1.3504440784,-0.4829608202,-0.4429127276,0.8962433338,0.4110181332,-0.7450675964,0.3003093004,-0.1053996682,0.5656231642,0.0190361906,-0.5192342401,0.3490352929,-0.2058995962,1.3725743294,-0.8968298435,0.5423145294,-0.8962261677,-0.1491219550],[-2.0275385380,-0.6608038545,-0.9174212813,-0.3269410431,-0.6984471083,1.6944645643,0.4648699462,-2.1589312553,0.0426462181,-0.4002126157,-0.9844839573,-0.9373098612,0.3330574632,0.8121154308,0.1654017717,0.0265124217,0.8958481550,1.0384033918,1.3312896490,0.2691825032,-0.5273260474,1.1580491066,0.9136486650,-1.8261531591,-1.6294533014,-0.4599237144,1.3166241646,1.2717188597,0.3420768678,0.1444726288,-0.9680743217,0.4697493017,0.8182046413,-1.4236667156,0.7308368087,1.0863107443,-1.4559316635,-0.0180493183,0.0013391395,-1.5703587532,-2.0040662289,-0.1724464595,-0.7469660044,-1.5486024618,0.9595509768,-0.9743630290,-0.0639142990,-0.3483835161,-0.5184774995,-0.1823514253,0.2897956967,-0.2570497990,-0.0183025580,0.1666794717,-0.4107077122,0.3243245482,-0.6374425292,-1.3727964163,-0.7992866039,1.1616351604,-0.0549143553,0.0692056417,0.4817972779,0.3330696821,-0.3786138296,-1.9377900362,-0.8723055124,-1.9915663004,-0.7240679860,-0.5480913520,-0.0218152720,-0.1139964089,-1.6564424038,-0.9848123193,-0.5869332552,1.3057488203,1.0492138863,-0.3129112720,-0.9047678113,-0.7177250981,-1.2049781084,1.3314702511,-0.6231637597,0.7844091058,-0.0176781621,0.4116369486,-1.3127673864,-0.2670100331,-0.0723978579,1.0006428957,-1.9221795797,-1.2662967443,0.5329766870,0.0479580536,1.1082743406,-0.0548312664,-0.8269295096,1.1963032484,-0.2248819619,-1.2792636156,0.4042378664,0.0060892166,0.2407629043,-0.2863627970,-1.7510541677,-0.8826270103,-0.1805585325,-0.8399214149,-0.3694585264,0.6691579819,-0.6126208305,0.5864830017,0.3455734253,1.0982719660,-1.2218780518,-0.7412814498,0.6595743299,0.6852849722,-0.4938407838,-0.8290623426,1.8494502306,0.7113271952,-1.3318347931,-1.2668334246,0.6168457866,0.0138242068,2.3361804485,0.0133694801,-0.9318174124,1.0154223442,-1.8149859905,0.1063761935,-0.4912840724,0.0053320178,0.1622054279,-0.3720568120,-1.7061535120,-0.3411099017,1.8075840473,-1.4172304869,-1.3801350594,-0.1177887693,-0.4706535637,-0.5100216866,0.0099745048,1.0853533745,0.3263893127,0.0064063426,1.8728448153,-0.5153175592,-0.0248616636,1.0729645491,-0.9651572704,1.2770721912,-1.1947435141,-0.4683704376,-1.5828979015,-0.2075870931,0.7596347928,1.1891981363,-0.2871383727,0.7696259022,1.1756484509,-0.2555998862,-1.1806615591,-1.5777649879,0.7371015549,-0.8200165629,-0.3479668200,1.5302962065,-0.8369179964,1.9417058229,0.3632695973,-0.5283840299,-1.4799474478,-1.0299297571,-0.3006336093,0.7646048069,-0.7382662892,-0.0858026668,-2.0297603607,0.7612405419,0.4500703812,1.1386395693,1.0415931940,-0.6529315710,-0.1752320975,-0.0934556127,-0.3674602807,1.1684161425,1.7406518459,0.2140564770,0.1849590987,-0.4445253611,0.9117878675,0.6770923138,0.8259384036,0.0250431057,-0.6785590649,1.5031665564,0.4678463638,-0.2961201370,-0.8332222700,-1.7620338202,-0.5066124797,-1.4213819504,0.0325794555,0.4598464966,0.8975044489,0.7793223262,0.2708227634,0.3689299226,-0.2866435945,-0.4612148404,0.2356104404,0.4840772152,0.0035614318,0.5616604090,-1.1055561304,2.8946905136,1.2669999599,0.4352328479,-0.6449033618,1.0168118477],[0.1424993426,-0.6114996076,0.1845914721,-1.2095587254,0.1187560558,0.1327622831,-0.9325071573,-0.0071629351,0.8303387761,-0.0023879316,-0.4078947604,0.5116908550,0.5126677752,-1.8852918148,-0.6545630097,-0.8021213412,1.7759143114,0.0790987164,-0.0372745730,-0.5091186762,0.3631534874,0.1563995630,-1.1762232780,-0.2444291711,-0.9540668130,0.8484722972,-0.2094481140,0.2584297061,0.4090232253,2.4077796936,-0.6442091465,-0.3512670100,-1.6794028282,0.6157962680,0.9134595394,-3.0511646271,-0.1610290557,0.0838241577,-0.2765303552,0.5775899887,0.5077181458,0.6141081452,-0.2548968792,-2.2971007824,-0.4871230423,-0.7298045158,-0.2915547490,-0.8250328302,-2.1933531761,0.5092428923,-0.9196913838,-0.2856418192,0.4852777123,-0.5238398314,-0.8249314427,-0.2213008106,0.4922359288,0.0046398826,0.0410495177,2.0978188515,-0.6671367288,-0.6185826659,0.9899863005,-1.0268276930,0.9838751554,0.1415936202,0.9602013230,-0.4019086659,0.0917325616,-1.7731301785,-0.6001939178,1.4872521162,-2.1950342655,1.4718630314,-0.4139128923,-0.9561069608,-0.8113186955,0.3326387703,0.8214855194,-0.2038237303,0.1721137315,-1.4140814543,0.8006810546,-1.1539144516,-0.4913604856,0.0820094123,-1.4642955065,-0.1441533267,0.3502728939,-0.5427587628,-0.1549838632,0.9814139605,1.6057529449,-1.7895412445,-0.5796220899,0.1910269558,0.1117242649,-0.8288728595,1.0306375027,-1.0849578381,-0.6961807609,-0.8261684179,-0.1974864304,-0.8071389794,0.9624972939,1.9254288673,0.3915759921,-1.8992302418,-0.6777492166,-0.3872167468,0.3622063696,-0.8133463264,0.4327911437,0.5757126212,1.9948111773,-0.8023193479,1.4258333445,-0.3936565518,-0.7623519897,-0.1646928489,-2.3818755150,-0.3824919164,1.1486035585,0.4143595994,0.5045929551,0.2677341700,1.3893061876,-1.6509420872,-0.1595229954,0.5341352224,0.5419927239,0.5675623417,0.0678644925,-0.4693374336,-0.2899969816,-0.2941041589,-0.3687418997,1.0642687082,0.3990949392,0.2112073898,2.1691763401,2.5314116478,1.1707247496,0.7853136063,0.8281221390,-2.0882747173,0.2841241062,0.3198890388,0.1984391659,-2.4289112091,1.0632315874,1.9254181385,-0.2166331410,-0.5206612349,0.3861836493,2.1343843937,-0.9316899180,-0.4525576830,0.0293869525,1.5127466917,-0.5325924158,0.2071255594,0.6559854150,-0.4750492275,0.6517692208,-2.0260581970,0.5454990268,0.9007616639,-2.1234264374,-0.6924746037,-0.2750160992,-0.7721719146,1.6402645111,0.1960601807,1.0715852976,-0.8867476583,-1.2654846907,0.8506244421,-1.0026184320,0.4239800870,0.0143493805,-1.0331511497,-0.2458608299,-0.2627170682,2.2830805779,-0.4507263899,0.4234290123,1.9715867043,1.0045151711,-0.8947780132,1.0483188629,-0.2448585927,0.9080960751,0.6717668176,0.7670707107,2.6760594845,0.2887377441,-0.5765333176,-1.3535066843,-0.1563234329,-1.6242492199,0.8874552846,0.4395448565,0.2620094121,0.8149400353,-0.7656787634,-0.1232830957,-0.6337431669,0.0293763019,1.4243355989,1.0572086573,-1.8027933836,-0.5571980476,-0.3124030530,1.2649337053,2.0032932758,0.9319395423,0.1501328796,0.1547182649,0.0764016360,2.4216063023,0.7080580592,0.2585445046,-0.2951288521],[1.1590728760,-1.4281768799,-0.6427973509,1.0778914690,1.3738418818,-1.0305179358,1.2834491730,-0.1111884490,0.0416006520,1.2782590389,-1.3953489065,-0.2880483568,0.2030022740,-1.4335784912,-0.8353443146,-0.7202455997,0.5462498069,1.4243865013,-0.1032523811,-0.0441531800,0.1812570095,0.6535425186,0.7359706759,-0.3278496861,1.6425333023,2.3137261868,-0.1360592395,0.5812934637,-0.6448676586,0.0685226098,0.8798252344,-0.7275282145,-0.6837694049,1.2179878950,0.5052300096,-0.1817120165,2.7077167034,0.5897852778,0.0560361706,-0.2262458354,-0.1569620222,-0.8156437278,-2.8309299946,-0.0347006284,2.3700158596,-1.4517055750,-0.7827415466,0.2200433016,0.6763939261,0.5266736746,-0.3205535114,0.4175000787,-1.4401388168,-1.2638221979,-0.0063566575,-0.4596130848,2.0484826565,2.6739189625,0.1921385825,0.8656622171,-0.7989099026,1.2951737642,0.5948476791,-1.1622223854,-1.0827726126,-0.6607543826,1.0801876783,1.7353651524,0.2039402127,0.4578662515,1.4513297081,0.6782011390,-1.0781478882,0.9227527380,0.8389589190,-0.3721253276,1.6220294237,-1.5711258650,0.5092188120,1.6772679090,0.2103672624,0.3513631523,-0.6458117366,-0.1160704643,2.5399868488,0.2986964285,0.7530531883,2.7717247009,1.3016248941,-0.5547922254,0.8124800920,-0.5440471768,2.9482123852,-1.9910891056,-1.2753846645,-0.8022624850,-1.4422289133,-0.5684272051,-0.1325278580,0.6433197260,-0.0040892162,-2.0867819786,0.3624523878,1.7563529015,0.1436444223,-0.7461550236,-1.4902784824,0.2751983702,0.2716933787,-0.5269842744,1.3409408331,-0.5586332083,0.5571196079,-0.4478425086,1.0341912508,0.5109900236,-0.9873884916,0.0708701387,0.3049991429,0.5304082632,0.1405647397,-1.0756629705,1.1812801361,1.6754041910,1.5391957760,0.1541727930,-0.3437381983,0.8482227325,-0.1725202948,0.8681633472,-1.2652460337,0.9696620107,1.3187575340,0.1618376225,-1.4064910412,-0.4977366924,-2.9630827904,0.2724413574,0.5373820066,-1.1703877449,0.2719387710,-1.0549979210,1.4473180771,1.9692100286,-0.1396738887,1.2972766161,1.3983153105,0.7203134298,-0.4064352512,-0.5069192052,-0.7329676747,0.4564698935,1.4903695583,-1.5688288212,-0.2548439503,1.0869340897,-2.3693585396,-0.0668379068,-0.2766051590,-0.5685678720,0.1007213444,0.4818153977,0.3051174581,-0.6083016396,-0.4800639451,0.2737148702,1.9641357660,0.9281412363,0.3207195699,0.2103534937,-1.6923233271,1.1626750231,-2.2308335304,1.4662967920,-0.0578541830,1.0219566822,0.8956917524,-0.4066852629,0.6047781706,-0.2009098828,-0.0700096786,1.5064666271,-0.0695334747,-0.8002229333,2.0655736923,-1.8748968840,1.4135584831,-0.1391232759,1.5362677574,0.2097577006,0.2475161254,0.9883912206,-1.5831539631,0.4373769760,0.3146182895,0.6461459994,-1.4481978416,0.4126372039,0.2698713243,0.2971876562,0.6936231256,-1.7687585354,-0.1024047658,1.8644307852,0.1501193047,1.4666852951,-0.0993625149,0.8320145011,0.2680338025,0.9386171699,-0.4379406571,0.0308498964,-0.5973451138,0.1201213524,1.8185228109,1.2896347046,-0.7865766883,-1.1671062708,1.0264364481,-0.4385924637,1.1909672022,-0.2365868241,-1.2961139679,0.7524974346],[0.2822886109,-0.8649634123,0.0341063626,-2.8012323380,1.8497134447,-1.4406055212,-0.8967682123,0.4737245440,1.4397727251,-1.7417879105,1.3006533384,-0.5665988922,-0.8679270744,1.1672435999,0.8426421881,0.5777239799,-0.0019734991,0.2628566623,-0.6765850186,-0.9454251528,0.6963109970,-2.0113916397,1.2220158577,-0.3644271493,0.3940259516,-0.4764729738,0.6744356155,0.4133459032,0.7129043341,-0.1390728951,2.4798042774,0.4449064732,1.0364000797,0.3874957561,-0.2127955854,-2.7277810574,0.8964817524,-1.1030762196,-0.9547590613,-2.6085407734,0.8240185976,-1.0024412870,0.5572627783,1.2201204300,0.6980441809,-0.3182870150,-1.0811100006,-0.8984940052,-0.9392712712,2.1252901554,-1.0693789721,0.6708747745,-1.2933703661,-0.4726088941,1.1888222694,0.2114872038,-0.4543708563,-1.3946108818,-1.3129746914,1.8129559755,-0.6399911642,-0.3761229515,1.0799931288,-0.8454584479,-1.1660637856,1.2091180086,-0.0841020271,-0.6888543367,-0.2568129599,0.1405620426,0.3394407630,-1.4461472034,-1.1847535372,1.1745417118,-0.5321114063,-0.0360228755,-1.2600207329,0.2216902971,0.3386104107,-0.4858584404,-0.2366298586,-1.5974326134,1.6561534405,-0.0659240410,-0.1518712789,-0.2975880206,-1.6202889681,-0.5111511350,0.4328710437,2.2999274731,0.1036824062,1.0497152805,-0.6470173001,1.2935286760,-0.8573095202,0.2456389517,-1.6096886396,-1.5166975260,2.2229051590,1.7827144861,0.0722996220,1.2876883745,3.4461617470,0.7335449457,1.2204228640,0.0536431260,0.0680795535,0.1912591606,1.3810964823,1.6709645987,-0.5143932104,-1.0899485350,-0.7233867645,-0.6821279526,1.3069764376,-0.2939797640,-0.5712470412,-1.4585288763,0.6952814460,-1.2076721191,1.0499768257,-0.0916740969,-0.0482324064,-0.5674934983,0.5605602264,0.7756689787,-1.1728376150,-1.5867017508,-1.2057219744,-0.2544875145,0.2107366025,0.5100553036,-0.3370763659,-0.9028173089,0.3074392378,0.0235144868,0.4365579486,0.2447894067,3.3895692825,0.7273297906,-1.8519378901,-0.7392191887,0.6342939734,-1.1659489870,0.2778316140,3.2687973976,0.8361719251,1.5480146408,-0.3752118051,-1.6580158472,-0.1478787810,-1.5146595240,0.6179021001,0.7550742626,-0.0943882689,0.7755956650,-1.5603001118,1.3169560432,-2.0728368759,-1.0825216770,0.6124401689,-0.2606959045,-1.1539610624,-0.7913170457,1.8059349060,0.4528495371,0.1427119225,-1.5344995260,0.1410109401,1.0175800323,-0.6247666478,0.9975888729,0.1294607967,0.1843679994,-1.1771456003,0.9625459909,0.9107032418,-0.7806379199,0.3773173392,-0.6000570655,0.5929875374,0.6998752356,-1.2417523861,-1.2792830467,-0.0448341966,-0.0431805402,-0.4964704514,1.9926270247,0.7569426894,-0.7253473401,0.1321403235,-0.0756466463,1.7732292414,-0.3561931551,-0.3286891282,0.7018263936,-0.8119922280,-1.1326178312,1.7631685734,0.4812170267,-0.1956695765,0.4439469278,-0.2367948741,1.3929479122,-1.0372287035,-0.0327260606,-1.0332548618,-0.6647860408,0.3720833957,0.1972508132,0.4465264380,1.3022334576,-0.3980585933,-1.0992186069,-1.1671767235,1.2631903887,0.3971188068,-1.5323983431,-0.4032920003,-0.6760450006,-0.2094847560,0.3652860224,-0.3081654310,-1.8190023899],[-0.1701465398,2.2619700432,0.2899451852,-0.7703804374,-1.1064257622,1.2357764244,-0.2967175245,-0.2518014610,-1.5689984560,-0.0665395111,-0.0862559229,-0.9216248393,1.2766802311,1.5393574238,1.0558137894,0.0327471532,-1.6642035246,0.8495980501,-1.7295979261,1.3477027416,0.1795098633,-0.2721120119,0.7435341477,-0.4873876870,0.5400074124,0.4466554821,-0.1522121280,-1.0432454348,-0.0845364034,1.7754360437,0.3426231146,0.4469730258,2.3664221764,0.6919433475,-1.1116325855,0.8461518288,-0.2383031100,0.8409350514,-0.7595874071,0.3608250618,0.8970346451,0.2595388591,1.0047931671,0.2540408075,-0.3400612473,-0.0989332497,-0.0843240842,-0.3543157279,-1.7633622885,-2.1017308235,1.2253223658,-1.1555352211,-1.7978022099,-0.3463512659,-0.0117409742,0.5840808153,-1.0323102474,-0.4460191131,0.0180029366,0.5165215135,1.0082906485,0.4918971956,-1.4555004835,0.4997964203,-1.1232254505,0.9987378120,-0.8648818731,0.1568584144,-0.7785245776,-0.3282842934,1.8889665604,0.1762571484,0.1230012551,-0.1905523986,-1.9035918713,0.6908513308,-0.4261215627,0.1221275926,1.0988082886,0.0977977887,-1.7845711708,0.7103321552,-1.1468501091,-1.2720302343,1.1274603605,-0.8638828993,-0.8295898438,-1.3732126951,0.1212136671,-0.7930681109,1.4917933941,-0.4714035690,0.2895286381,1.6895601749,-0.8141436577,-0.8679010272,0.1823181361,1.4328541756,-0.4204801917,0.2739147842,1.2020156384,-1.8207857609,0.0096109519,-0.8583530188,-0.9887015224,-0.1826705188,-0.6696085334,-1.5090953112,-0.1405836046,1.4303534031,0.3717813790,-0.6468277574,0.6109654307,-1.4839514494,-1.3144893646,-0.7689852715,0.6979426742,-1.9050546885,0.5673009753,1.0644997358,1.4237437248,0.8367416859,1.4612578154,0.9356698990,-0.5868975520,0.8214159012,-0.3272928298,0.4603173733,0.0749733821,0.1287813038,-0.2579364479,-0.1763682663,0.3552583456,-0.6590710282,0.8675601482,0.1804513335,0.0760865286,-0.2730111480,-0.7868251801,-0.1163439304,0.0811429247,0.1231359839,0.5234337449,0.1796350628,0.9489487410,1.6153148413,0.9410284758,1.0284470320,-0.9127117395,0.2013400793,-0.4045197666,0.7818467021,-0.3640500903,-1.8174712658,0.4952382147,-0.3121580482,-0.2593118548,0.1520194411,-0.0602648407,1.1699638367,-0.4944905341,2.1621308327,-1.3320986032,-1.6290991306,-0.7247669697,1.2197005749,-0.4413643777,-0.3647455573,-1.7348531485,-0.3638899028,0.0509795621,-0.5163988471,1.0410149097,1.0232046843,-2.2675664425,1.1889684200,0.4370892942,0.6450393200,0.7288362384,0.1710212827,1.3312356472,1.3694489002,-2.7367355824,-0.8831370473,0.4819488823,-2.1779723167,0.5980419517,-0.5120624900,-0.3950772583,-0.9564168453,0.3127147853,-0.3291445673,-0.9316582084,-0.6530855894,-0.5801360607,-0.8719137311,0.6725584865,0.3189466000,-0.9949298501,2.0379827023,-0.3290540576,-1.4721562862,-0.5032038689,-0.9637928605,-1.2094340324,0.7158033252,-0.0927066430,-1.2442517281,-1.0542521477,-1.1061631441,-1.2389072180,0.5120587349,0.2246490270,-2.1872861385,-2.1402411461,1.5952260494,-0.0812475979,0.6093641520,-0.5080390573,-0.2121757418,0.2089391798,-1.5400040150,0.3724039495,-1.1847088337],[1.5130584240,0.7076172829,-1.5021262169,-0.6401950717,0.4719422758,0.0465151928,0.5896236300,0.1516773552,-0.7396040559,1.5477974415,0.4146561623,-0.0829181895,0.0063156439,1.1727257967,1.2963408232,-0.4674797058,-0.1541069448,-1.4523546696,-2.4150819778,0.3667570949,2.6291785240,-0.4988396764,2.0458302498,-0.0747351050,-1.2224282026,0.2986982167,-2.0429217815,-0.9583296180,0.8893783092,0.2481311560,-0.5463197231,-0.1864299476,0.8800899386,-2.0384750366,-0.5730707049,2.2102062702,-1.3429448605,0.9100778103,2.2369208336,2.9532630444,0.2449691147,-0.0423857160,-1.2589607239,1.9544759989,-2.0986361504,1.1010017395,-0.6761249304,-0.5158253312,2.1491205692,0.6918260455,0.4865469933,-0.0197637565,0.3024237752,1.2770613432,1.5790829659,-1.3644719124,0.1178247705,-0.2447064966,2.2056901455,0.5842333436,-0.8112859130,-0.8845480680,0.5018767715,-0.9628272653,-0.6917787194,0.5183368325,0.2223260701,-2.4080696106,-0.6258693337,-0.2220265567,-1.1970661879,-0.4511078000,0.1818592101,1.0391007662,0.1656878889,-0.1195623577,-0.4278938174,-0.3380023241,0.7096031904,-1.2523366213,-1.0001627207,2.1837749481,1.1153163910,-0.3457722962,0.2794967890,-0.3424712420,-0.4181531668,0.5340260863,-0.9294694066,-2.6755039692,-0.5858883858,1.1506539583,1.0913171768,-1.0706495047,0.0423627459,-2.0051174164,-1.3628343344,-1.1141849756,-0.0660549477,-0.9681855440,-0.6248205304,0.6539865136,-0.7431252003,1.1930276155,0.0813564435,0.4269303679,-2.9126818180,0.6502090693,-0.5792543888,-0.0636761934,-0.2226995528,0.2611485124,-0.1963765770,-0.7510466576,-0.4036752284,-0.7355467677,0.8295586705,0.2841912508,-0.5753408670,-0.4433884919,-1.4987837076,0.7639639974,1.1248340607,-0.3037438393,3.0878508091,-0.6452888250,0.1875440478,0.0079111867,0.2569694221,-1.0926113129,0.2290089428,0.3807959557,-0.0390078202,-0.0512206107,-0.0093297828,1.1520316601,0.3930533230,-0.6344077587,0.0940024257,0.8543282151,0.3782372475,0.9214416742,-0.3992048502,0.2794377804,-0.0629065409,-1.1470851898,1.0025026798,-1.7029709816,-0.3982113898,-0.1038763076,1.7992355824,-0.7881760001,-1.4537860155,-0.4431024492,-0.0188008342,0.8453229666,0.3609706759,-0.1380304694,1.3005646467,-0.7697758675,2.0646855831,1.5479987860,-0.6123995185,-0.7303442955,1.0699342489,1.2121047974,-0.2541789114,0.4415258467,-0.2636577189,-0.3476258814,2.0897309780,-0.0466064326,-0.9225341678,1.2898052931,1.8792136908,1.5108665228,-0.1160245165,-1.3695697784,-0.2568242252,-0.8984883428,-1.3459892273,0.3001153171,0.2522737384,0.4287333488,-1.4466867447,0.4010054767,1.7777253389,-0.0489532314,-0.3533559740,0.1734737754,-1.0124557018,0.0805245712,-0.1957098097,0.1044771150,-1.0432385206,-0.3111307025,-0.6315842271,0.2103316486,0.9604236484,-1.4323203564,0.0213550087,0.2500723600,1.5048394203,0.8332036138,0.5828694701,-1.1305952072,-0.5224618316,2.7289142609,0.5754151940,-0.1905027777,-0.7241643667,0.1696766764,0.3804264367,1.3739337921,-2.0588717461,-1.2422560453,-0.2515717745,0.0229164604,0.0536120906,-1.9283407927,1.7888227701,-0.1094532013,1.1283302307,0.0836060122],[-0.1131427437,-0.1455607265,-1.7971167564,0.4484315515,-2.7939643860,-1.3108272552,-0.3902270198,2.8373355865,-0.2132748961,0.9357783794,-1.0359768867,-0.1403545290,0.7936987877,0.7273046374,-1.9549911022,0.0507963784,-0.6482225657,-0.0779463053,1.0098884106,-0.4915691316,-1.7953245640,1.0999864340,0.3593125641,-0.2098834813,-0.6835131645,-0.0330367573,0.1991713047,-0.6592006683,-1.2774186134,-0.9763864279,-0.4536683857,-1.0885910988,1.1693351269,-1.0799335241,-0.7590987682,-0.8095877171,0.2647846639,-0.3310623765,1.2105258703,1.8539288044,-2.3340659142,0.1285315156,-0.3244792521,0.8191900849,0.9074296355,1.1251484156,-0.6740199924,-1.7290600538,-0.2522689700,-1.2518934011,-0.3544958234,0.3565763831,0.5419793725,0.2216321081,0.0744594485,0.4009789228,-0.9030396938,1.1985112429,1.3050527573,0.2644869983,1.2111719847,1.4938938618,-0.3135474324,1.6932300329,-2.1511843204,0.8760904670,0.3790178001,0.1553075314,-0.7556515932,0.4605605006,0.1281558573,-0.5230821967,1.4032703638,-0.2235340029,0.7719467282,-0.8938195705,1.8719172478,1.7318835258,-1.2508921623,1.4505107403,-0.7890445590,-0.1284002364,0.5220596790,0.3965850472,0.3776007891,-0.8109610081,2.5998513699,-1.2116770744,0.6401030421,0.0995526016,0.2845474482,-0.3137201369,0.5138025880,0.5688924789,0.0490821227,-0.5854231119,-0.1241295859,-1.2401731014,1.7278945446,1.3531194925,0.8374717832,0.7225297093,0.7827263474,0.6421630383,1.3519514799,1.1704369783,0.1243656427,-0.4696682394,-0.5816171169,-0.0667436421,-1.6302013397,0.2320342809,1.2981059551,-0.3229315579,2.4780550003,1.4481487274,-0.8240926862,0.6625467539,0.4279925823,1.1662126780,-1.0683534145,0.6251668334,1.0923655033,-0.3942586184,0.9659493566,-0.9605253339,0.9597508907,0.2220281959,0.0806372091,2.1207196712,-0.9710009098,0.1539418548,1.8333917856,0.5665449500,0.3243786395,1.5739785433,-1.0496633053,-0.3769021332,0.4410339892,-0.3513799310,-0.2858914435,-1.0758993626,-0.2779846191,-0.9377691150,-1.5190898180,0.1464475989,0.5293513536,-1.0593185425,-0.7463225126,0.6748778224,-0.9203764200,-0.8092591763,-0.6285470128,0.6317136884,0.5938884020,-0.2016290873,-1.0690140724,0.8877389431,1.0266020298,1.7298130989,2.1594829559,-0.0914283618,1.1662026644,1.1089171171,-0.5123221874,-0.3319963813,-0.3509967923,-0.8236360550,1.2866132259,-1.0179052353,-2.3028659821,0.2056182623,-0.5770489573,0.6231693029,-1.2684029341,-1.0089328289,-0.6858816147,0.2580859959,0.5197789073,-1.2355266809,0.8098257780,-1.1747789383,1.2804471254,0.4993601441,1.1285796165,-0.1324673593,-0.3163600266,-0.1993279010,1.1616035700,-0.4632444382,-0.3264450729,0.3072538078,0.1358029395,1.4254649878,0.1142488122,-0.0371584930,0.4293601513,-0.9944052696,0.6163930893,-0.1263526529,0.6469143629,1.0459427834,0.6449648738,0.3584982455,-0.3914584517,-0.8582537770,1.0382779837,0.4811164737,-0.7321038842,0.8452780843,-0.8298352361,-1.4297674894,0.8056114912,-1.3645051718,0.2932583988,-0.4535759389,0.8559157848,-1.0738148689,0.2676266730,0.0838424116,0.1785915345,-0.0274679642,-1.0839809179,0.1531840116],[-1.6444801092,-0.5802404881,0.6161803603,0.4401788712,-0.0861002505,-0.1942696422,-1.3705750704,-0.3469303250,0.4158354700,-1.9293687344,-1.3931224346,2.0027859211,0.7087925673,0.5379871130,1.9546526670,0.7514656186,-1.3095040321,-0.0490565225,0.5774003863,0.0596319139,0.1235771477,-0.8998639584,1.1102211475,0.3734438121,-0.1030125245,-0.0789468586,-1.3038878441,0.0609854460,0.3799828291,-1.2476456165,0.6118445396,-0.3204786479,-0.4230053425,0.5840607882,-0.6879864931,-1.7932746410,0.9541391134,-2.3621644974,0.4547835588,-0.2293474674,0.9833710194,0.1935296506,0.1290696412,-1.5336457491,-0.7804637551,0.8936599493,-1.0555058718,-0.8772373796,1.6407520771,-0.3166005313,1.5166699886,1.1298456192,-0.3969652653,0.7104807496,-0.9392107129,-0.1705227792,-1.2460235357,1.4000566006,0.2025852352,0.2420491725,0.6935530305,-0.4681382179,-0.7721756697,0.9676588178,0.1198359430,0.1922744066,1.8341497183,-2.7488372326,-1.0611772537,0.1251877993,-0.3508950472,0.8396126032,1.1544494629,-2.1873486042,0.1149946302,-0.6635305285,-0.2694257200,0.6697966456,0.4052609503,-0.4259499311,-0.0662061721,0.0750824437,-2.0473349094,0.5177432895,0.4888772666,0.9255126715,-1.5766785145,1.2748899460,-0.7883622646,0.2249861956,2.6988117695,0.3158188462,1.5834808350,0.1978325099,-0.2180462778,0.5772840381,-1.1148155928,-1.2709002495,1.0019147396,0.1085478067,0.0644170269,-1.2529100180,-0.0105301021,1.6947497129,-0.2921498418,0.6520886421,0.7268299460,-0.5517195463,1.1416976452,-0.8316757083,-0.1236225069,0.7180331945,-1.7250076532,-0.5391926169,0.7702272534,-1.8290551901,-1.0975006819,0.1231060103,0.1836741120,-0.2079072148,-0.3733049035,-0.2890796661,0.1109581962,0.0841258988,0.4721401334,-0.5287404060,-1.0826221704,-0.0720474869,-0.5139278173,0.6535246372,-0.1900931895,0.5698981881,-0.2145587355,-0.1457729191,1.0641998053,1.2511655092,0.8574934602,0.8291519880,0.4578750432,-0.7188019753,0.0051001688,0.1767217517,-0.0890806839,-1.6345053911,0.6368270516,1.6084293127,-1.2352643013,-0.0038276801,-0.9700303078,1.0549601316,-1.4505219460,-0.4465314746,0.8876798153,1.0627869368,0.4874746203,-0.5734726787,-0.5883601308,0.5869722366,0.2755430341,0.0495398827,0.6863536239,-0.8936468363,0.9656808972,-0.0415965989,-0.8703505993,-1.5649480820,0.7539380193,-0.8508267999,-0.7657111287,0.9372233152,-0.5270434022,0.3940381706,0.9115097523,-0.6412190795,-0.7600647807,0.4493492544,1.0575815439,1.2218286991,0.2423966229,0.5654261112,1.1719374657,-1.3715431690,-0.1721050143,0.2998908460,-0.6642464995,-3.1117630005,0.3729720712,-0.3114005923,0.2432861477,-0.3534573615,-0.1880740076,0.5819046497,-0.3220902979,-0.3700904846,1.7209761143,-1.0574181080,-1.2687592506,-0.5830300450,-0.6538046002,0.6858478785,1.8021428585,-0.0014441964,-1.8535352945,1.3842341900,0.5036410093,0.8910794258,-0.2037441581,-0.8835229874,0.2358111292,0.8903995156,-0.6490919590,0.1825971007,-0.4593852758,-1.3794941902,-0.5826588869,-0.2057380974,-0.1274713874,-0.0318536498,0.5356215835,1.3765367270,0.6221526265,-0.5182808042,-0.0778893232,-0.1184563860],[-1.5069004297,1.8594160080,-0.6981487274,0.9367927909,0.4855525494,-0.1547571570,-0.5993320346,1.1967276335,-0.2093621343,-0.0519363619,-0.8732603192,0.1621766239,-0.7586136460,1.8687325716,-1.0110545158,1.7136559486,0.2294831127,-2.1291050911,-0.0054081036,1.2772713900,0.4153493345,1.1322646141,1.2953122854,-0.4021786153,-0.8659353256,1.3614841700,0.4887880385,-2.2888154984,0.7517309785,0.7804059386,1.4343571663,1.1280318499,0.4125819206,1.6644282341,1.0138138533,1.1136579514,0.0347738303,-0.0656888932,1.3539829254,-0.8064793944,1.7992246151,-2.0517346859,1.1155378819,0.3399031460,0.1162784547,0.0452033058,-1.8890718222,1.8050925732,0.7577112317,1.9084178209,-0.2905920446,-0.8593422771,0.5828835964,-0.0455377772,0.5747262239,-0.8365653753,-1.5725414753,0.6745752692,0.4211437404,-0.1733942330,-1.3247904778,-0.3337387145,-1.0581274033,-0.1005994231,1.0474176407,-2.9184198380,0.3160714209,0.6893534660,-0.7181006074,-1.0671374798,-1.9399932623,-0.8578689098,-1.2643039227,-1.0543541908,-1.3231947422,2.0581784248,-1.4635936022,-1.8555091619,0.5931416750,-0.0641021878,-2.0411744118,1.4487619400,-0.9831895232,1.1797959805,-0.9788149595,0.5968145132,1.6935912371,-0.7220646143,-0.5078238249,0.8034408092,1.7207052708,0.6739747524,-0.0758223757,0.5789842606,0.9166283011,-0.2725763619,-0.3547967076,0.9579622746,-0.4331351519,-1.4609791040,-1.0095167160,0.1861312240,0.0965674669,-0.5023012161,0.7978777289,1.1768301725,-0.2873516083,-0.3553848565,0.7831780910,-0.3031035066,1.0905922651,-0.3384682238,-1.5104107857,-1.4142198563,0.5335105062,-0.9249476790,0.3325738907,-0.6441063881,0.3561105132,0.1303441375,1.3840107918,-0.7485288382,-0.9498573542,0.4881732464,-0.0074557588,-1.2329013348,0.8200548291,0.0354016982,-1.2749636173,-0.7796729207,-0.9187532067,-0.7172759771,0.4123795033,-1.2273064852,1.5586960316,1.1174397469,-0.2219770998,1.9217811823,0.6616954803,-1.4222782850,-2.0599975586,0.0867804214,-2.2925868034,1.3697549105,0.8076735735,-1.4938260317,0.0309710633,-0.4543955624,0.9160174131,-1.3730944395,-0.7710475326,-1.1737235785,-1.4549480677,0.3577015102,-1.7589148283,0.0929171890,-0.4369859695,-0.3473596573,-1.5033946037,-0.0985980779,0.0101886811,-1.2029874325,1.4330537319,1.2072029114,-0.0131212957,0.7697699666,0.1444311142,-0.0711760968,1.4660404921,1.2070227861,0.2769919932,0.7466009855,0.4890920818,-0.8605375886,-2.1168172359,1.1051851511,-3.1067254543,0.4607944191,0.5312582850,0.7263253331,-0.1775484681,-0.4622008502,0.3566424549,0.8464894891,-0.6093367934,0.3593332469,0.2175244391,0.7817227244,-1.7723919153,-1.4126907587,-0.3926410079,-0.5199172497,-0.5307080746,1.0303177834,0.2988476455,-2.0974349976,0.5404635072,-0.0937601328,-0.1606501788,-0.1147610173,-0.8684297800,-1.1027407646,0.5567104220,0.6773251295,-1.6910611391,-1.4039698839,0.0599403195,1.2236547470,-0.1871442646,0.0593416244,0.8148764968,0.3711216450,0.6319756508,-0.0623319261,1.5143686533,0.9789410830,-0.7867607474,-1.0358027220,-1.3657990694,-1.8243933916,0.4192204773,-0.2765286565,0.3474358320,0.4425532222],[-0.2964217961,1.0173045397,-0.0976274237,0.6084778309,-0.5599411130,0.2848835588,-0.2352688760,-0.1809513718,0.1363043934,0.1136598513,0.4099833667,0.8103279471,-0.4263535440,0.4786857963,-0.4269832969,-1.9123014212,-0.8065711260,0.5510292053,-0.0150344782,-0.3349220753,0.0880308002,-0.8002303243,0.3176805675,0.8337778449,-1.5033525229,0.5691682696,0.0770459697,1.5434830189,-0.0361299738,-2.0033481121,-0.7982869744,-1.1101838350,-2.1667561531,-0.8929808140,0.2909214199,-0.0600078702,1.8678102493,0.6645435691,-0.5375238657,0.3015767038,0.8310984373,0.8583566546,-1.2841429710,0.2107521594,0.9149329662,-1.4058991671,-0.7324476242,-1.6634695530,-1.3212348223,-0.8275483251,-0.1830658913,-0.9442102909,0.6262663603,1.0551890135,0.8133642077,-0.4543934464,0.9712342620,-0.3185590208,-0.1727422923,1.0257860422,-0.2464818507,0.5092759728,-0.3984328508,-0.7305040359,-0.5991800427,-0.7956777811,-0.1845896840,-0.0305629540,2.0064487457,0.9583771229,0.0363172144,-0.0191859677,-0.3577366471,-1.2744368315,-0.0630775541,-0.0684572831,1.8726197481,-0.1023565009,1.0510714054,0.4325604439,-0.8658845425,-2.0781784058,-0.0383642949,-1.1078237295,0.5350592732,1.4228900671,-1.1049482822,-0.0801641643,-1.0005115271,-0.4297798574,-0.0559677742,0.2867806852,0.3633767962,-0.2082037479,1.1467409134,0.7982702851,0.0084525729,0.1348223388,-0.0987749547,0.4950430095,0.2327910811,-1.5314847231,-0.8182778955,0.2929429114,0.5952238441,1.5427302122,0.8737509251,0.7998639345,-1.2590091228,0.4758330584,0.5333705544,-0.8768574595,0.3324282169,0.0279628281,0.3362767398,-1.4143075943,-0.2139455825,-0.8838997483,-0.4397753775,-1.3462122679,-0.0181162562,0.5175273418,0.3846479952,1.2185842991,0.6779496670,0.0439935140,0.0856824592,-2.0083913803,0.2829786539,-1.2908842564,0.9698345065,-0.1842066050,-1.7637593746,0.8919836879,0.4382879734,1.2267940044,-0.1170026511,-0.2494598031,-1.7412210703,0.6522830129,0.0244456157,-2.2340483665,-0.4830648899,0.1403207183,-1.3167569637,0.1011837199,-0.6489614844,1.0936598778,0.1544923484,-0.9260298014,-1.3525762558,0.7894437909,0.1807664037,-0.6345968843,1.0378496647,-1.1073594093,0.5320475698,0.1034940556,0.7239211798,-1.7146366835,-0.9290454984,0.5460405350,0.6197391748,-0.6808706522,0.5027135015,-1.1057575941,-0.2300989777,-1.2203605175,-1.9491988420,1.2034761906,-0.4060276747,0.4978751242,0.1822032332,-0.6505709291,-0.3541910648,0.2075365186,-0.0094287647,0.2557637393,2.1727738380,-0.8296252489,-0.3163543642,-1.2971352339,-2.0529043674,0.3074778914,-0.2195957899,1.0099247694,-0.0117843701,-0.1268891692,-0.9868423343,0.3460584581,-2.3466484547,1.3982958794,-1.0281090736,0.6707727313,1.4497638941,-1.6454523802,0.3650892377,0.3026635051,-0.4742977619,-0.4233018160,-0.6639545560,2.2567322254,1.9358763695,-0.4846771657,0.6917907596,-0.0628971905,-1.6291610003,1.3567776680,-1.1375604868,-1.7353007793,0.1414480656,2.2786235809,-0.5899149776,0.1783048213,0.7491911650,1.6066801548,1.2810688019,1.7628934383,-0.9193839431,-0.1078255475,-1.4992743731,0.1968777180,-1.1903437376,1.4254845381],[1.0927671194,-3.0272002220,1.9271652699,0.5826958418,0.9965778589,-0.4075433016,0.4611845911,1.0407460928,-0.8860122561,0.8657599688,-0.1595807970,0.4464957416,-0.4966609478,2.9157238007,-0.6837469935,-0.2011281997,0.0069960882,-1.0792711973,-2.0461034775,-0.2763215601,-0.1097814068,-0.2376969159,0.8770941496,-0.9377326369,0.6465483904,1.5600159168,0.4526254237,-0.8150980473,0.0279371645,-0.6981883049,-0.8752954602,-3.2129752636,-0.6134070158,-1.5156295300,0.4352482259,0.1551049948,-1.2635306120,1.9354977608,-0.6287520528,0.5079177618,-2.1001939774,0.3900247812,0.5513725877,-0.5999241471,-0.7899345160,-0.7181108594,0.1940894574,0.6138250828,-0.4738227427,0.8976388574,-0.8623569608,0.7657828927,-0.0611073673,1.1355913877,1.4761710167,-1.0551246405,-0.7247058153,-1.9813483953,-3.0640888214,0.1663405299,0.0124569135,-0.2628451884,0.8943400979,-0.2469936311,-1.9407814741,-2.2188706398,0.4381474257,0.1332459599,-1.7151442766,-1.0381941795,1.9939316511,0.8085494637,-1.1738445759,1.0872621536,-0.1853503287,-1.1145530939,0.4168367386,1.0869874954,-1.6734377146,-1.0069420338,1.4226447344,1.3044387102,-1.2753057480,0.2289368361,-0.3836416304,0.0978016034,0.2819712460,-0.1764324158,0.8378456831,0.2236027420,-0.2701498568,0.6950197816,-0.0077380328,-1.1970260143,-0.3734492660,2.0470554829,-0.4651390612,0.9548599124,-0.2230303735,-0.8019583821,0.3957392275,-1.2221180201,0.4072573483,0.3880814612,-0.6441312432,0.6174584627,1.0846157074,0.6076990962,1.3721759319,-0.0749501362,1.0494583845,0.1591409296,1.2146483660,1.2344856262,0.1549460143,0.9441838861,0.8796211481,-0.5252056122,1.1343315840,-1.0764510632,0.2324069738,-0.1777675897,-1.1545977592,0.7995833755,1.0012146235,0.0061407532,-0.3563856184,-0.2577395439,0.7515842319,1.1220419407,1.0445507765,-0.9168112874,0.6538673639,1.4195749760,1.0493332148,-1.3196792603,-1.7301003933,-0.7242970467,0.6112730503,-1.0061122179,0.7732378244,0.5763732195,0.1579078883,-1.6517345905,0.7781819701,-1.1136115789,0.0968965143,1.1728216410,-1.3469579220,-0.0480953678,0.8859339356,0.1404125839,0.4784304798,1.3342672586,-1.2530455589,-0.5268494487,0.7873428464,0.4130172133,0.1780058444,-0.3267858326,-0.0826260373,-0.3123799562,-0.9598277211,-0.0987945795,0.7427921295,-0.7078848481,0.4165566862,1.3543208838,0.4051081240,-0.8848313689,-0.2286397219,0.2070284635,-1.6439064741,1.2955049276,-0.2987741232,0.9125178456,2.1810588837,0.2293576598,0.8749178648,-1.3403472900,-0.5969886184,-0.4366630316,0.0945704505,1.6732033491,0.4051296711,-0.4160036445,0.9374626279,-1.2810299397,1.7304445505,0.0030098860,0.4869622588,0.8563162684,1.0231707096,0.3627135456,0.2392707467,-1.9643951654,-0.7161080241,-0.2974090278,0.2324644774,-0.3435841501,-0.6919144392,-2.9642279148,0.4640024602,0.0311744493,-0.1404165179,-0.4386716485,0.4086935222,0.1824422330,0.7120854259,-0.6734026074,-0.8858430386,0.3498681784,1.5669513941,0.9241248369,-0.8324046135,1.1146484613,0.9041340947,0.2956512868,-1.0480246544,2.8630208969,1.4399596453,0.5258901715,0.5015463829,-0.3171377778],[-0.4555572867,-1.0919431448,0.5997264385,-1.7279002666,-1.0438963175,-0.0601046048,-1.2033896446,-0.1048328951,1.7255532742,-1.2785211802,0.8037407398,-1.1056637764,-0.8321886659,-0.6981978416,0.6783164740,-0.6690093279,-0.4299170375,-1.9805632830,0.3072091937,-1.4346629381,0.2722908854,0.3559779823,0.5817649961,0.9715235829,0.9819499850,-0.3409855068,-1.3291027546,-0.8595953584,-0.2314220369,0.5113080740,0.1259804070,-0.0081106750,1.8190455437,0.8666212559,-0.1718907505,1.0014348030,-1.0544644594,0.0663620755,0.8409745693,0.1343867183,-0.4915670455,-1.1927989721,0.4487790167,-0.3024437428,-0.9536862969,0.0255348217,-0.2934833765,1.4988203049,0.1637242138,0.1977759004,1.1715903282,1.3605438471,-0.7619611025,-0.5757738948,-0.2096434981,-0.7292847037,-1.3548737764,-1.3679894209,-0.4481071532,-0.6871653199,-0.1607779413,0.3172212541,0.0824580118,-0.0072252350,-0.8245114088,0.8536661267,-0.6266992092,-0.1128967926,1.2639377117,-1.9712094069,0.5339668989,-0.2932064533,1.5378162861,1.1618890762,-0.8614155650,-1.0626561642,1.5011287928,0.0631416887,-0.6804838181,-0.1782898456,2.5135641098,0.4825144708,-0.4274825752,-1.5566039085,-0.1402496248,2.8049387932,-0.9801715016,0.5661875606,0.7939589024,-1.0236939192,1.9633581638,-0.1391828954,-0.2312912196,-1.1576309204,0.5281303525,0.9900379181,0.2896926105,0.0827321708,2.2535164356,0.1212359220,0.5381548405,1.5913234949,2.0978007317,-0.7286148071,-0.5753691792,0.0828324556,0.0549454726,-3.2569141388,-1.8817790747,2.0503520966,-0.2591432333,0.7470239401,0.3248349428,-0.4882176816,-1.3225115538,0.4865359664,-1.0414841175,-0.7895792127,0.8525590897,0.7897837162,-0.6213835478,0.0934152529,-1.9919574261,-1.2485752106,-0.9487129450,-0.5130205154,1.1634507179,0.3678902984,-0.7489489317,0.8876307607,-0.1656009853,-0.4229520857,0.7181720138,-1.0354518890,-0.3245650828,-0.0019222775,1.8520420790,-1.1361961365,-0.7627169490,-1.5301635265,-1.6102468967,0.3663013875,-0.2587484717,-0.3138886690,1.6451128721,-0.4497563839,-0.1718848050,-0.4313556850,0.2476296723,-0.4489573836,2.0903165340,0.7821831107,-0.0666693375,0.7762039304,-1.5657635927,-0.3966723382,0.6482068300,1.2220047712,0.2189141363,-0.3889785111,0.1967448145,0.0520267896,0.4590506554,0.5041958094,-0.1697325110,-1.2416745424,0.1428915411,1.6692961454,-0.0450778902,-1.6467303038,-0.8072274923,-1.3155603409,-0.9456545711,-1.0992310047,0.6658183336,-0.5641582012,-0.3236243725,-0.9900243282,1.3859374523,1.8923567533,0.1954327673,-0.5008239746,-0.8863476515,0.0526065789,2.1536448002,-0.2155486196,-0.8135938644,1.8556420803,1.4562864304,-0.0517918803,0.6744711995,-1.7987359762,0.9923582673,0.3135000169,-1.9504567385,-0.1242371649,1.5248242617,-0.1120917201,0.2942447364,0.4463695288,1.0370858908,-0.2582800686,-0.5030901432,1.0489833355,-1.9220529795,-1.3229802847,-1.1880118847,-0.6967065334,-0.1832891852,0.6462819576,-0.9041520357,0.8098919988,0.8147661090,-1.4799150229,-0.3029147983,0.3950447738,-0.3467047513,0.9894205332,0.9808540940,-1.3005290031,-0.9212888479,0.8779946566,-0.5321686268,0.4928537011],[-0.1064901054,2.0072026253,0.2842397392,0.0219647326,0.2706302404,0.4453656971,-1.0214610100,-1.1261841059,0.5503368974,1.0498505831,-1.1867488623,-0.3131522238,0.2150351256,-0.7758720517,-1.2624981403,1.5144072771,0.5820170045,0.1279897690,-0.4706283510,-1.0356974602,-0.2843975723,-2.1004853249,-0.1418736279,0.3538064361,1.2380549908,-0.0495617092,-0.2308800519,-0.6746603251,-0.5816107988,0.1941962689,-0.7019876242,1.4929260015,-0.8704730272,-0.3817136288,-1.3759821653,1.1188722849,-0.6615345478,1.7322149277,-0.6132860184,0.0772217736,0.7857915759,0.5138965845,0.8683391213,1.0741430521,1.1444586515,0.2174246758,-1.2567451000,1.2492321730,-1.2271503210,0.2029304504,-0.4345974922,-0.5613096952,0.9899421334,-0.7794497013,0.9226804972,-1.4347459078,1.8226745129,0.2517725229,1.3048853874,0.2598344386,0.8284549117,-0.1853842437,0.9909932017,0.8433710337,0.2945882678,0.1580699831,1.1083993912,-1.2196359634,1.0195169449,-0.7478786707,-0.7492168546,-0.7852820158,0.8240640759,-0.1690352857,0.3865447342,3.0318000317,-1.3104873896,-0.2106412500,-0.0911137834,0.1675297022,-0.4070737362,-0.5210168362,-0.6016403437,1.5758588314,-0.4493618309,0.5540955663,1.4807208776,-0.4585899711,1.5710061789,0.2264640331,0.7540287375,0.2100552619,0.1138534173,1.2335205078,-0.1995334029,1.2598119974,1.9921194315,-0.1144331172,0.1608610004,-1.0974568129,1.5505696535,-0.4277614951,-0.7436270714,-0.5684842467,1.5652070045,-1.9541970491,0.1121145114,-2.2796773911,-0.4204390347,0.3072674870,-1.1443328857,-2.1288378239,-1.4380398989,-1.1068999767,0.7632105350,0.8052682877,0.0307863411,0.5012815595,-0.7604537010,-0.1332142353,1.9456522465,0.9724579453,-1.0146543980,0.4536070824,1.9216108322,-0.9443646073,-0.4025148451,0.5432344079,0.2296927720,0.8836902976,-0.4197762311,0.2263710350,-0.4735622704,0.2265483439,-1.0051529408,1.0260937214,0.0039349413,-0.6438735723,-1.5596687794,2.2677609921,-0.0115529187,-1.9361091852,-1.2214096785,2.3677110672,-0.2835217118,-0.4986249208,0.0987454504,-0.2859016657,0.5878900886,-0.2068942785,-0.4712097347,-0.0293408167,0.1004185751,0.5754767060,1.2288657427,0.7598702908,-0.3827791214,0.2701179087,1.9307528734,0.3766086996,-1.2595345974,-0.1427586228,0.2123726457,-1.0897004604,0.9504085183,0.0233728476,0.0613599457,0.1773449779,-2.6466653347,0.4382828176,1.0714106560,-1.0919705629,0.2308921069,-0.5418452024,-0.6811359525,-0.0733657703,-0.1108241528,-0.6205454469,1.3386415243,0.3169844449,-0.0132978726,-1.6194648743,-0.1214035675,-1.2176237106,2.3920612335,0.2677967548,1.9048371315,-0.1090479195,-0.2812086046,-0.8757746816,-0.1215230674,-0.8371799588,1.3561232090,-0.5343367457,-1.3576158285,-0.0527018160,-0.8965148926,-0.1940020174,-0.1157381758,1.3223857880,-0.0518980287,-0.5180981755,0.7235001922,-0.5034645200,-0.9930452704,0.5033494830,0.5044299364,-0.1784189790,2.0653004646,-2.3807713985,0.2423002571,-1.1575675011,-0.8651419282,-1.2484507561,-1.4254765511,2.6023020744,0.5604815483,1.2961738110,0.7792427540,-1.0478221178,-0.4755157530,-0.2933375537,-1.2906724215,-0.5171805024],[1.1661533117,-3.0772411823,0.4945068955,0.8361061811,-0.4539458454,0.6520614028,-0.0636517256,0.1846643686,0.4559874535,-0.7601983547,0.7996939421,-1.3556103706,1.4006017447,-1.2842981815,0.0201086011,-2.4067430496,1.0017011166,-1.2208684683,1.1259891987,-0.1002016664,0.4338685572,0.5633941889,-2.0205998421,0.7079035640,-1.3134320974,-0.7481291294,-2.0741927624,-1.3709870577,0.2138299495,-1.3161886930,-0.5571706891,0.5849643946,0.8160891533,-1.3734140396,0.1693484336,0.4666670263,-0.1848581731,0.6021742225,0.7658632994,-0.2800158262,-0.8746697903,0.6097396612,-0.6713853478,-1.1039898396,1.4990520477,0.1935194135,0.6815602779,0.9476210475,0.0892999396,-0.3394016922,1.2948087454,0.2439201474,2.0538434982,-0.0148545131,-2.3645238876,-1.2313162088,1.4056997299,0.3840621412,1.3106213808,-0.3852375150,-1.1932288408,-0.1657445431,0.0863959864,-0.2528146207,-0.2925288379,-1.4965672493,1.0718343258,1.5679070950,2.0106396675,-0.0910225138,-0.4950281084,0.9399472475,1.1115689278,0.4600949883,1.2392388582,-0.0673111305,-0.8725827932,-1.6198756695,0.1760318428,-0.0825519711,0.2308563888,-1.0177457333,-0.1170732826,-0.8819388747,0.2377493382,1.0289160013,-0.8309114575,-0.8098101616,1.5898245573,1.0980596542,0.3096333444,-0.4698989689,0.5669177771,-0.0059816218,0.4326222241,2.6160898209,-0.5428299308,-0.5957408547,1.2545629740,-0.7029317617,-0.0110392943,-0.3283358514,0.4800703526,-0.4944664538,-0.3524605036,-1.0188369751,0.1142610535,0.4262864292,-0.7915928364,0.4737672210,0.6954224110,1.0050495863,-1.1403791904,0.0269745216,0.8302710652,-0.7582543492,0.0169747900,1.6924520731,1.5741794109,0.1987948418,1.1307752132,0.6800326705,0.3710531890,-0.7827582359,-0.8127122521,1.0480321646,0.3776837587,-0.8211373091,-0.1559403688,1.6256326437,1.0607466698,-0.3728995621,-0.5707845092,-0.3217938840,0.4106550217,-0.9551732540,0.0835519657,0.2593034208,0.7858951092,-0.0355189852,0.4143232405,0.3124325573,-0.6817540526,-1.0727124214,-0.8389334679,1.4706026316,-0.7480475903,0.1054771543,-1.3369356394,-0.0947246999,-2.0778944492,-1.3957155943,0.2285256088,-1.2759611607,-0.3343304098,0.0893022716,0.7450934052,-1.3093935251,2.0228989124,1.3606815338,-1.8446570635,-0.5683281422,0.2735865414,-0.4449757934,-1.9793847799,-0.0671787336,-0.4893623888,-0.6754632592,-0.7179415822,0.2092989981,-1.4558817148,-0.0806349814,1.6811379194,-1.4438048601,-0.7252404094,-2.2249646187,-0.8503048420,-0.1294687390,-0.4447180629,-1.4720023870,0.9454625249,0.5489059687,-0.0052853036,0.4739248753,-0.6366956234,-1.6935415268,-0.7258015275,0.1571865827,-0.6548541784,1.8914365768,1.0652371645,0.4324242473,0.8018820286,-0.5653391480,2.6766371727,0.1969307512,-1.0498875380,2.2293496132,-0.6575648189,-1.5168567896,-0.3831220865,0.0663847551,-0.0127926245,-0.0289653186,0.8270576000,-1.2334251404,-0.1683523655,1.0262917280,2.0759630203,2.6905012131,-0.3199998736,-1.9919358492,1.3505409956,0.4858950078,0.4985644221,-0.0941948369,-0.1956441253,-1.8708121777,0.5297023654,-0.2530247569,-0.7849372029,1.8887374401,-0.1904240698,-1.0549265146],[-0.9172002077,2.4479074478,0.0858606398,-1.1978263855,-0.4034654200,-0.3708377481,1.3119311333,-1.9337878227,-0.2400770187,0.7398082018,-0.7001578808,0.2794561386,0.1471390873,1.7028954029,0.4503670931,0.6144703627,1.8867530823,0.1782859117,0.5037370920,-0.8585119843,-0.9124016762,0.9520156980,-0.4813594818,1.0137593746,0.5279722214,-0.4412348866,1.2565177679,-1.2011139393,-1.2171599865,0.3179094493,2.3057527542,0.0162226334,-1.0044425726,0.8488297462,0.8104820848,-0.3958480060,2.2574179173,-0.8400303125,-0.0290416647,-0.8049724102,0.1110918969,0.1712533683,-0.3029280901,0.9060761929,0.4949313998,1.5811786652,1.1496472359,1.7891368866,1.6989605427,-1.2844060659,-1.4256250858,1.9526252747,1.4928870201,-1.0814081430,0.9162069559,2.1888005733,0.4808399081,-1.2371330261,-2.3750650883,-0.5375353694,-1.4709631205,-0.9133584499,0.0837792382,-1.8881543875,0.6336411834,-1.2488851547,1.1549003124,0.9698441625,-0.1542241722,0.0810905918,0.1434110105,-1.2068035603,-0.2149506211,2.2364194393,-0.7714393735,0.1282031387,-1.6601262093,0.2144560665,0.2410408109,1.5626209974,-0.1163768768,0.9992430210,-0.8798793554,-0.5838215947,0.3413982987,0.1006489098,-0.0714010820,0.0203747340,0.4074044526,1.3556555510,-0.2700893581,-0.9345957637,-1.3730533123,1.3425372839,0.5145732760,-0.7645043731,-0.7455899715,-3.1266086102,-0.0076880553,-0.8621249795,-0.9780056477,-0.4648793340,0.1431073397,0.8786411285,0.0829805061,-0.6876224875,0.9151754379,-0.4527014196,-0.3110263348,0.1074373797,-1.1986666918,-0.6184980273,-0.9270381927,-0.2079146504,-0.4372631609,1.1997834444,0.1383371651,0.2398205400,0.4831244349,0.6147862077,-0.6656706929,0.8235280514,-0.5021594763,0.9050056338,1.0267746449,-0.0839773044,0.2934941649,1.4526919127,-0.5227711797,-0.6307641864,0.7450272441,-0.6373867989,-0.2456774414,1.1337466240,-0.9645944834,-0.5025370717,0.2286235988,-0.4299673736,-1.4565142393,-1.0908288956,-0.6438586116,0.9676718712,-0.7037091255,0.0187808182,-0.8907798529,0.9865032434,-0.2691493332,0.2577497959,-1.9395964146,-0.6774106026,-0.9088381529,-0.0304178633,0.2201902270,0.8568982482,0.7380009890,-0.5199701786,-0.7361801267,0.6977995634,1.9890062809,-0.2889457047,-1.9138393402,-0.9784683585,0.7392961979,0.3418764472,-0.1733308434,0.4325277507,0.1451773793,-1.2198106050,-1.4039171934,1.1004782915,0.1187280118,0.6469854116,1.0802356005,-0.1612346619,0.4949934781,-0.0387254246,-1.1470397711,0.9689227343,-0.6782378554,1.5090565681,-0.1389405578,-0.2824213803,-0.1344258338,-0.1288654208,0.1623164266,-0.1821363419,1.1691294909,0.8031123281,-0.5740057826,2.0071382523,-0.1318964064,-0.4972498715,-0.4756318033,0.9943248630,-0.6209115982,-0.6978612542,-1.5944736004,0.3212108910,-2.1554422379,1.2142683268,0.9055288434,-0.3738659322,-1.6954593658,0.8965313435,0.9389824271,0.9087389112,-0.5691789389,-0.5289321542,1.2088074684,-0.8341738582,0.5690646172,-0.8160740733,-0.2697964609,-0.8304342031,-0.7724876404,-1.3361012936,-2.0136146545,-0.7321896553,-0.5210469365,0.4472853243,1.3969060183,-0.0323589183,-1.2332663536,-1.2633476257],[-0.5402527452,-0.8164409399,0.8692822456,1.5394263268,-0.6449768543,2.7324981689,2.2829384804,-0.0243462957,-0.4708231091,0.7802814245,-0.8517262340,-0.5438752174,1.3492876291,-2.1100623608,-1.0599167347,-1.5305445194,0.9629749656,0.3840721250,1.6027978659,0.6158267260,1.7967692614,0.6978132725,-0.3588051498,0.5557240844,2.0012843609,0.4117252231,-1.2055360079,-0.8306830525,0.7206227779,0.1442007422,-1.0898617506,-0.5330889225,1.3166472912,0.7317251563,0.7280780673,1.6997103691,0.1930397451,-0.1861350387,-1.4679410458,-0.6894759536,0.5577561855,-2.7302639484,1.7028381824,0.4612136781,-0.4558180273,-0.3445228040,0.0529292449,-0.3675781786,0.8226752877,0.4586767554,1.3503264189,0.8677229285,-1.7889962196,-0.9069355130,-0.1365360618,-0.8261981606,-0.0284881555,-0.7588346601,-1.0901699066,-0.9001386762,0.9422703385,0.5553904176,-0.2856693268,0.4512157142,0.1980567575,-0.6102285981,1.1556385756,0.1393267065,-0.5263919830,-1.1312519312,-0.3653927445,0.8627732992,1.0686572790,0.3649345636,0.0013756203,0.1762937754,0.1261652112,0.0962752104,-0.7591469288,0.0422343500,0.1832052916,-0.1315644085,-2.0348117352,-0.5923904181,-0.5324992537,1.1435881853,-0.9107784629,-1.7445204258,-0.4858583212,1.5137287378,0.4766238034,-0.1806454360,-0.9680721760,-0.3028369248,-0.3065006733,-1.1280874014,-0.7446612716,0.8452233076,0.3190322518,0.8402855992,-1.6233057976,-1.4034744501,0.4287578166,1.1428534985,-1.5632708073,-3.8884437084,-1.2155498266,1.5331667662,0.2305153459,0.6118981242,-0.5459505320,-2.5060958862,0.9057552218,-0.0377277546,-0.2225462347,0.5662169456,0.7287843227,0.8154967427,-0.8809786439,0.6194652319,0.2452322841,-0.1912601143,-1.1317753792,0.6059587002,1.3791689873,1.5434559584,-0.3077296615,-0.0549331531,0.6082119346,-0.4465995431,-1.6164356470,0.7707487345,0.3566562831,-1.0651414394,1.2344359159,0.2838335931,-1.4992698431,0.9605651498,1.3660390377,0.4354725480,0.1872466207,0.6108676195,1.2492525578,0.1859040558,1.0766217709,0.2355428487,1.1754379272,0.0057427166,-1.5887515545,0.3769720495,-0.2890666425,0.1181629524,-1.0365852118,-0.9767994285,-1.2941398621,0.4882920086,-0.9798254371,1.8120832443,-1.2130056620,0.2267473191,-0.1387804300,-0.3132337034,0.4717891514,0.2309700698,0.0566052124,0.1241219193,2.1627066135,-0.6082924604,1.2542783022,-0.3301045597,-1.0780427456,2.4032220840,-0.2280689329,-0.2645161450,0.1353555322,2.1393721104,1.4293290377,2.6570754051,-0.7824953794,0.6957845688,-0.2106664479,1.4867365360,-0.6972081661,-0.0462230630,-0.6337217689,-2.0922307968,-1.1119211912,-0.0705752820,0.3467822969,2.1771509647,1.1771267653,-0.3236551881,-1.8883998394,0.4537965953,0.2125834823,-0.3116625547,-0.0193837136,-0.7874299884,0.0644968003,0.4301182330,2.2707767487,1.0474052429,-0.5718730092,-0.6267511845,1.4229493141,-0.5729410052,0.8310288191,0.5821508765,-1.0076065063,-0.1922570169,0.0469058715,-0.1675707102,1.0838055611,-1.3790389299,1.3507223129,0.6134766936,0.0176201425,0.8428995609,-1.4992263317,0.0356088988,2.2975597382,-1.0299751759,0.2906056643,-0.5603120923],[-0.5430343151,-0.5503050685,-2.0349340439,0.3507160246,0.0798382014,1.8635390997,1.4128968716,-0.3009924293,-0.6323676705,1.2433440685,-2.6133000851,1.0574719906,-2.4669671059,-1.0075842142,0.7337694168,0.4244298339,-0.5422949791,-0.5591009259,0.1210958511,1.4105414152,-0.2632221580,0.7265729308,-1.6804473400,-0.7905883789,0.5711280704,0.6042720675,-1.0270519257,-0.5709877014,0.4242979884,1.0850281715,0.5821233392,0.2940220535,1.1921521425,-0.3279542625,-1.0906670094,0.3363183141,-0.4115214050,0.2423718274,0.9718721509,1.3256907463,1.3731383085,-0.3152098656,0.2616109848,0.7625992298,-1.1046898365,0.7405341864,0.3569501936,-0.0304454751,0.4198069572,-1.1654562950,0.3820678294,1.9267227650,-0.4919791222,0.2134685665,-0.5913387537,0.3960608840,-0.3911811709,1.5581468344,-1.7947453260,-2.1093726158,1.1553223133,-0.4462725818,0.3671998382,-0.2049529552,-0.3518499136,1.1037057638,-0.8849042058,0.6334500313,0.8141261935,-0.3845141530,-0.9080251455,2.6200330257,-1.3253928423,0.2427710891,-2.1761622429,0.3425274491,0.4839399457,-0.1631598324,0.8371133208,-1.6910206079,-0.5188969374,-0.5083132982,-1.0096994638,-0.0340005644,1.6234545708,-2.2919981480,-0.8237686753,-1.6870236397,0.2045819759,2.6661920547,-0.3744850755,-1.3754044771,-0.5262570381,-0.1487215459,0.3899958730,-0.3349808455,-0.2147201151,0.8382616639,0.0128131611,0.2007857412,0.4351505935,1.2793378830,0.4645488858,-1.1116894484,-0.2591662705,-0.4701961279,1.0315747261,-0.1856772453,0.9001066685,-1.3104590178,1.3114802837,-1.0081175566,0.3077223003,-0.6241681576,1.4459326267,-2.0826654434,-1.4616817236,0.5706226230,-0.0821249560,-0.4760001004,0.5327296853,1.5234986544,0.1988669932,0.6964995861,0.0093918424,0.7568479180,0.5697199702,-1.8994301558,-0.6445646286,-1.0114517212,-0.3073485792,-1.0358322859,1.1701234579,-0.7295694947,0.1950688511,-0.0414369553,0.7513110638,-0.3562017083,-0.3402531445,1.0202345848,-0.1407631487,-0.7549223304,1.6024886370,-1.5254733562,2.2856884003,1.0988514423,-0.2321708053,0.5067417622,-0.8190221190,-0.9867876768,0.9519258738,-0.6780868769,0.1301680803,0.8194435835,-0.4938240647,1.2405018806,0.6402520537,0.1182226539,0.7945907116,0.0175350085,1.5766327381,1.1100355387,0.5462289453,-0.2211159617,0.5913339257,0.5664343834,0.5663946867,-0.3709756434,0.3571896255,0.7610222697,0.1248397306,0.0199591443,-0.4644696116,0.4498883486,-0.7748786807,-0.2893411219,-1.7340050936,1.2877976894,2.4911108017,0.5305491090,0.7266719341,0.1545901895,0.9030400515,0.3245202005,-0.1829281300,-0.3200109899,0.3289081156,-2.9458432198,0.8093186021,0.3413748443,0.5557262897,-0.1546023935,1.4225945473,1.1587522030,0.2325889766,0.5621083975,-0.9386121631,0.0797256827,0.5854457617,-0.8971057534,0.2960605621,0.4373312593,0.2856488526,-0.2587785721,0.8635243773,0.1729522645,0.4400544167,0.6298682094,-1.7541642189,0.0078843795,-0.4985109866,0.1519829929,-0.4746786654,0.6637639403,1.1471449137,-0.2405385375,0.5008134246,-0.4119012356,-2.0247213840,0.6055507064,-1.0243568420,0.3559515476,-0.8437631726,3.1102230549],[0.7065474987,1.5697489977,0.2100787759,0.3337875605,0.3581123650,0.4796786904,-0.1394378543,1.7159233093,0.9836455584,-0.4811730683,1.2921022177,-1.4980825186,0.3234030604,-0.9263525009,-1.9753719568,-1.2800205946,-0.6501467228,-0.4534287751,0.6986515522,-0.6066743135,-0.3495893776,-0.4151754379,-0.1613209397,-0.3423712254,-0.4407419860,0.3743268549,0.0228571836,-0.2950489223,0.4956834912,-1.0063159466,0.8071094751,-1.4763947725,1.2184668779,-1.2583113909,0.4009124339,-0.9315493703,0.3108949065,-1.0812360048,1.2314478159,-0.0053389100,-0.2175952047,-0.3266625106,-0.6629793048,-0.6949992776,0.1832964271,-1.3664089441,-0.7330182791,0.0111999176,0.3048385084,0.3065909445,-0.3205359280,0.3547713757,0.2387059480,0.0288662184,0.3407653272,0.0688019916,-0.6346371770,-2.9236137867,-0.0657542571,0.2381646484,-1.4751336575,-1.0005645752,0.3903593123,-1.5890965462,-0.5685404539,0.7253620028,-0.4060972035,-0.7793660164,-0.9706671238,-2.1348261833,1.4878499508,1.6540946960,0.0158814453,-1.3564432859,-1.2082302570,0.5394089818,0.4960101247,1.3246908188,-1.0439869165,-1.4069757462,-1.2993481159,2.4255983829,0.9048818350,-2.4917924404,1.1100304127,-1.5330778360,-1.1706732512,0.1931117922,0.2511897683,-1.3754106760,0.3725407124,2.4211819172,-0.4166229665,1.7748960257,-0.3293336928,-0.3670873642,0.2008082420,-0.7868642211,-0.7763853073,-1.8592094183,-0.5754011869,-0.4025988281,-1.4596866369,0.6588805914,-1.7689900398,0.5031039715,1.2791875601,0.1997129023,0.0951175317,0.7077681422,-0.3682105243,-0.6120201945,0.2989490330,0.4668666124,-0.0516404063,-0.0072434973,-1.4731035233,-0.6742974520,-0.1071094051,-0.0481653474,0.3348087370,0.7697380781,1.0444390774,0.6389073133,0.7965302467,-0.5480006933,2.1861627102,-1.7186702490,-1.0547710657,-0.7335004807,-0.4895069599,-2.5456478596,0.4805270433,-0.2115930915,0.6833215356,-1.2567358017,0.5541188121,1.4430524111,-1.2658931017,1.3923492432,-0.5163353086,-0.8217576742,0.5181081295,-0.0065804077,-0.1548570544,-1.4562957287,-0.6649558544,-0.2081828266,0.7567382455,-1.9596194029,1.1570088863,2.8178348541,-0.9480741024,0.3081744909,1.0048877001,-1.0370769501,-0.9271890521,-0.5718141198,0.5977467895,-1.5901451111,-0.5971243978,0.7271687388,-0.3104576766,-0.7801139951,-1.4054712057,0.5915648341,0.6452729702,-0.5297361016,0.0908126980,-0.5574408174,-0.8353986740,-1.0422720909,-0.3353568017,0.7226265073,2.0564503670,-0.5431280732,0.6647246480,0.1695094258,0.0573868677,-0.9498914480,-0.3513489366,-3.0019948483,-0.0066395034,-0.4072309434,-0.9640353918,-1.3792288303,1.2849546671,0.4020608366,0.5306547284,-0.1897467226,-0.9025427103,0.2592071891,-0.2822673023,-1.5999060869,0.8814782500,2.0872652531,-1.8682709932,1.2595149279,-0.7809629440,-0.6400290132,-1.4980249405,0.2437685728,1.3451523781,-0.0861069784,0.0169560835,0.8107209802,0.0504279770,1.0673788786,-0.3041729927,0.7378834486,-0.2011504918,0.2117267698,-1.3680343628,-0.7541540265,-0.8309316635,0.0250439066,1.0769543648,0.1214284599,-0.0662044212,0.4549642503,-1.0856857300,-1.6837859154,-0.1463906467,-0.4956187010],[0.0208592359,1.2435624599,0.5489192605,0.2411355078,0.3825907111,-1.6864269972,0.6340034008,1.2933397293,-1.2414149046,1.3236659765,-0.4560811222,0.0856558979,-1.0383340120,-0.0595628954,1.1350011826,0.8011539578,0.3942179084,1.4639997482,-0.4066986144,0.4662700593,0.9133059978,0.9119114280,0.2083911151,-0.2895689011,-0.6915959716,0.5094326138,1.2007409334,-2.1352896690,0.2560206056,0.2516901791,-1.3380346298,-1.4319194555,-1.0343267918,0.6933841705,0.0523918718,-1.6278096437,-1.7473222017,-0.5706251264,0.3746893108,-1.4037268162,0.6328114271,1.1285928488,1.0005843639,-0.0091215968,0.2692708373,0.5430057049,0.6359573007,0.6018438339,-0.9794576764,0.4686716497,0.1191736385,0.6245647073,-1.6013067961,-0.3516278565,-2.0193760395,0.8631431460,0.6426948309,-1.2152938843,-0.7574124932,-0.1954472661,-1.4739801884,-1.7282948494,0.7948954105,-0.3243267536,0.4970909953,-0.9655341506,1.0447359085,-1.6254166365,-2.4415161610,-0.0110456059,0.6514555812,-1.2094271183,0.8311422467,0.7294400334,-0.7013913989,-0.9180595279,-1.5018708706,-0.2198044360,0.3838777542,-1.1265043020,1.1347600222,0.8210456967,-1.1263074875,1.6014512777,0.1510402262,2.6066834927,-0.8219428062,-1.2476803064,0.6389467120,-0.4005217850,-0.7135859728,-0.6143370867,0.6910892725,0.2512392700,1.4253145456,1.7152156830,0.3454139829,-0.9362812042,1.2939212322,0.5791168213,-1.5516303778,1.2336236238,1.2780562639,-0.2307296097,0.5151617527,2.3914289474,-0.8676834702,0.1678168923,-1.8745352030,0.0887532830,-0.0148388622,-0.5805263519,1.8545913696,-1.0860618353,0.3407716751,-0.0286480375,0.8063637614,0.1843805462,-0.2964666188,0.2472796142,-0.4235060513,0.4835164249,1.4205632210,0.7961882353,0.1411911696,-1.9163612127,-0.6891729236,-0.6566789150,-1.4901683331,-0.6937078238,1.4161351919,-2.0617113113,-1.0494184494,-0.8323073983,0.8514946699,-0.3380451202,-0.8639111519,-0.8786121607,-1.7560200691,-0.7598963380,-2.0147390366,2.4679574966,-0.2002087235,0.3709254861,-0.1325344741,-0.7265365720,-0.1155025512,0.0221462883,-0.5978046656,1.7831364870,0.3680877686,1.3986713886,0.2212256342,0.8435522914,1.4509996176,0.2756426930,-0.0882249475,1.1780829430,-1.2781015635,0.9661692381,0.0849523470,1.4569551945,0.1979640424,0.0714182109,1.2154024839,0.6585696340,-0.3421055973,-1.0114430189,1.7284212112,0.1652764082,0.0440218449,1.9963196516,-0.2177901864,-0.3685661256,0.3754687011,0.7273710370,-0.0082708513,-0.8066374063,1.3471446037,0.9464960098,1.4551038742,1.7335010767,-0.7727061510,1.2335720062,-1.0976076126,-1.2013162374,0.8994884491,0.8080055714,0.0058319694,-0.7987020612,-0.6682382822,0.3353258371,0.2434836775,1.0560492277,-0.6121442318,0.4928761423,-0.3694842756,-1.8778828382,-0.0021931289,0.4397992492,0.0179698374,0.6603020430,-1.2249333858,0.3797037899,0.0017936496,-1.1764934063,0.1773265749,0.2379674315,1.8248631954,2.0317924023,0.5488117337,0.6360759735,-0.0691221654,1.2608212233,-0.0978059471,-0.4273525476,1.1747417450,1.1928589344,-0.7382788062,-0.1795980632,0.3162508309,1.9766232967,-1.1666280031,-0.3134149909],[0.0035257055,0.4764416814,-0.4459216595,-0.0344706252,0.9892446995,-0.7527502179,0.1697129011,1.0399791002,0.8508217335,-0.1022319868,-1.0322203636,-0.8765253425,1.6454482079,0.2713077366,0.0806060657,0.6563324928,-0.3451429307,0.1096281931,-0.4809369147,1.7522681952,0.2002703995,0.1564730555,-0.9563501477,-2.6347720623,1.6463611126,0.2392531037,-0.2259557843,0.3380222619,1.8624340296,-0.1859715879,0.0106931049,0.3789506555,0.1891666949,0.2392414361,1.6173669100,-1.2887517214,0.4338289499,-0.0806743056,-0.4152863920,-0.4792827368,-0.3852518499,1.1886054277,0.2293532044,-0.0757466853,-0.1865629554,-0.6457681656,-0.1063916758,0.2182568908,1.9584716558,-1.3472338915,0.7995253205,0.1375773102,-1.4171931744,-0.0779861510,1.7121382952,0.3812084496,0.1072792262,-0.4451794624,-0.4806461930,-1.0805552006,-0.7667006850,-0.2458909303,-0.5287251472,-0.1539849043,0.4048659801,0.4522774518,0.9267548919,0.0533838384,-0.5965754986,-0.7000019550,0.6710319519,0.8343346715,-0.7737452388,-0.6778567433,0.8769618273,1.4523413181,0.3435254991,-0.9673869610,0.1777562052,-1.7617540359,1.0451806784,-1.2181231976,1.2019933462,1.7208796740,2.4286599159,-1.6599638462,0.6040220857,0.6516717672,-0.2622128725,-0.7239272594,-0.8162904978,1.4671630859,0.2137639672,0.3574137986,-1.0039931536,-1.0461215973,1.1023905277,1.2569910288,1.0668817759,-0.0154753542,-0.9374274015,0.4624779820,-0.1082559749,0.6684598923,-0.5393890142,-2.1180541515,-0.9959486723,0.2782751918,0.2944386601,-1.7939523458,0.4444533885,-0.1882642359,0.5147325397,0.2929969430,-1.8092688322,0.1214284822,-1.0645792484,-1.6098444462,-0.5044631362,-2.3047981262,1.7936631441,0.5146069527,0.9426302314,-0.6366305947,-0.2010298967,0.6305668950,-1.2798579931,0.1300441027,0.1669493467,0.7836005092,0.3279234469,1.2725434303,-0.0735878423,0.5170953274,-0.0156838894,0.2086202949,-0.3578739464,0.5611712337,-0.1845562160,-1.3503365517,-0.2838805020,-1.0690042973,-0.1829974055,0.7866693735,-0.2664945722,0.8257446289,0.4535391033,-2.3860180378,2.7660934925,-1.6022106409,-1.6800652742,0.1000329405,0.2255189270,0.4775266945,0.9800982475,0.0695611909,-0.1636302322,0.7325497270,0.9747005105,0.7435159087,1.4804545641,1.2320071459,-0.8016436100,0.4638025165,-1.3180469275,0.9796977043,-1.6520098448,2.0815844536,1.7063485384,-1.1672502756,-0.2736607194,1.4327979088,-0.1547749192,0.7636130452,0.3382462263,-1.3285471201,0.9728714824,1.3283112049,0.5871783495,-0.3835515380,1.2146409750,1.5588822365,-0.4628805518,0.1570315212,0.7060463428,1.1901321411,-0.6649191380,-0.7949004173,-0.2459549904,-1.0208480358,0.6603729129,-1.2617355585,0.2647471428,-0.2705723345,-0.1136778742,-0.2293915749,-0.2497108281,1.0774041414,0.9895689487,0.7714352608,-2.4308023453,1.0879162550,-0.0287893172,-1.7348122597,0.8923826814,0.2280374914,0.5680179000,0.0501384847,1.2335151434,1.3422961235,-0.3221537471,0.1614855081,-0.6970582008,0.8141419291,-1.5891063213,-0.2522450089,-0.0983667821,-0.1677764803,0.3889777958,-2.3217234612,2.2559692860,0.9853873253,0.1528448313,0.1335994452],[0.0960316658,-1.1012336016,2.1527287960,-0.7543258667,1.4281525612,-0.5952357054,-0.6522353888,0.0763411149,-2.4124116898,0.1143976673,3.1385512352,-0.6567717791,0.5438843966,0.2478468269,-0.8372309208,-1.0154943466,-0.0634565204,-1.0248450041,-1.1556870937,-0.7422597408,-1.4936764240,-1.8463222980,-0.9808491468,-0.7588642836,-0.1710610092,0.0920800194,-0.7544130683,0.6935185194,-0.3152893484,-1.2180192471,1.5641496181,0.2795628011,0.5534697771,-0.3952704966,0.7293699384,-0.4047926664,-0.0643341914,-1.7703485489,0.1241352558,-0.9134722948,0.3001177013,0.7077474594,0.8252443075,-1.1510851383,-0.5846912265,0.2272734344,-0.1831062436,1.2051370144,-0.1876637787,-0.5765454769,-1.0280808210,0.3914548457,0.9839670062,1.0020608902,-1.4458919764,1.3140779734,0.0528018363,0.2362228930,-0.4548761249,1.6524966955,1.7901469469,-0.2248424739,0.1754400581,0.9779208302,0.5776523948,0.9024821520,1.1635148525,-0.3024394512,0.3784768581,-1.7918648720,0.3735060096,1.0666931868,-0.4633731842,-0.5668267608,0.9597026706,-0.7974790335,2.2758290768,0.9152048826,0.9018004537,1.3983670473,-0.9318888187,0.3431453705,-0.2198560387,-0.6310589910,-0.5225726962,1.9764684439,0.3742882907,-0.4727636278,0.2065766305,-0.2785574496,0.2065834999,0.3348022997,-1.0158301592,0.0557430349,1.2103167772,-0.8823601604,0.5185846686,0.2634567320,2.1054568291,0.3655582368,1.0914030075,1.0806419849,-0.7831611037,-1.0495216846,0.4184381962,-0.0781125873,1.3796223402,-2.1329607964,-0.6260906458,0.1810334921,0.5751695037,0.2468598932,-0.1141770110,1.3287128210,-1.1890742779,-0.4207553864,0.7122940421,-0.9073532224,-0.3651553094,-0.4226305783,-0.0830838680,-1.0728796721,-0.1376402676,-0.7822508216,1.2550245523,1.8266563416,0.7291188836,0.8544880748,-0.2885992229,-0.6249886155,-0.1918189228,1.0057471991,1.3464815617,-0.0350044779,-0.7850279212,1.2537226677,-0.7549847364,-0.7763412595,-0.5457606316,0.8749659657,-0.1705278009,-0.1407030970,0.1812559962,-0.2123323083,1.0182181597,-1.2406775951,-1.7901406288,-1.7229549885,-1.9905031919,-0.5597758889,-0.6072100401,-0.2092664540,-1.9350706339,1.2083387375,-0.7741381526,0.7506406307,0.9133481979,-0.7715125680,0.6030091047,0.5493309498,0.4792211056,-0.2988275588,0.9531223774,-0.0966653600,-1.4549390078,-0.5645372868,-0.3757342100,1.3526890278,0.7144587040,0.2461370528,0.0577381253,1.6676069498,-0.5236766338,-1.4429916143,-0.7393116355,-1.2088468075,0.0605625138,-1.5278388262,-0.9210611582,-0.2517165542,-1.9440078735,0.6988514066,0.1982673407,0.3086035848,0.5408528447,-0.9238742590,1.0745069981,1.2209699154,-0.5641684532,-1.5028772354,1.5196889639,0.2270999849,0.5806286335,-0.1388664693,0.2372200936,0.6519583464,-0.1114310697,-1.2452012300,0.8265328407,-1.1701641083,-0.9429509640,-0.0014479960,-0.0517590232,0.7875321507,0.2763226330,-0.3950987458,1.4887293577,-0.0903558433,-0.6510680914,-1.0548905134,-0.1655217260,1.2616093159,-0.0127523160,-0.1354828477,-0.2113614231,-0.6180521846,-0.0597930215,-0.2407428026,-1.5077199936,-0.1971070170,-0.6861489415,-0.0931828097,-0.8914435506,0.1905655414],[-0.8744983077,0.0098695075,-0.3797313571,0.7740601301,0.0845667273,0.2132222652,2.2988576889,-0.1039733142,0.7457541823,-0.8987375498,-1.3461180925,-0.8154997826,-0.5025764108,-0.3514509797,-1.0883080959,0.8634200692,1.5210120678,-0.8854886889,-0.7128968239,1.0747821331,0.7041067481,0.8238183260,0.4915956557,-2.5970296860,-0.5477147698,-0.9649059772,2.5200729370,-0.3640636206,0.8815293908,-0.9766589403,0.4918274581,1.3674652576,1.0179756880,-1.2977014780,-0.8691310287,0.3466084301,-0.6623682976,-0.3450996876,0.6613887548,1.5178869963,0.4877279997,0.9343072176,-1.0097370148,0.6456452012,0.3174564540,0.7257652283,-1.3117456436,2.4636328220,-0.6551507115,-0.2571562529,-0.4730632901,1.5332181454,-0.7421475649,-1.1132484674,-0.5764532685,1.0589469671,-1.1869277954,-0.5670490861,0.6283428073,-1.1403493881,-0.9345607162,-0.4807460606,0.3763595819,0.4279716611,1.1986782551,-0.2374680340,-1.2819801569,0.1377163678,0.4830263853,-0.5013511777,-0.6719954610,1.5218192339,-0.1730282307,3.6219646931,-0.1248321682,-0.1220605448,0.2830874920,1.6604256630,0.5359668732,1.3583760262,1.6301083565,0.6225463152,-0.9911639094,-0.8537741303,-0.6471802592,1.8175919056,-0.1935535520,0.0851218626,-1.0140290260,-0.1194371432,-0.7439462543,0.3609744310,0.1996553689,0.4417337179,0.0744811743,0.6526217461,0.5125229955,-0.2765215933,1.2034013271,-0.4930281639,0.6316803694,-0.8473200202,1.1323636770,-0.9325317740,-0.1085536703,0.2190643996,0.4257893264,-0.1137780547,-0.8615435958,-0.6558609605,1.7273952961,-1.1156474352,0.4715724885,-1.2323323488,-0.9245020151,0.2340586185,0.4171338379,0.5103055835,-0.8030940890,1.3934131861,-1.7207546234,0.8516317606,-0.7090355754,1.3007886410,-0.2409402430,1.5606890917,1.2331166267,1.1565121412,1.1419471502,0.4737171829,-0.0646478087,0.7538887858,1.1985429525,-0.8085051179,-1.7302092314,-0.7214023471,1.0235490799,1.9192208052,-0.0632185340,0.5950103402,-0.0955924690,0.0445052236,-0.2363787144,-0.6596962214,0.9489222169,-1.8176273108,0.3249565661,0.5321244597,-0.2519031167,-1.0410438776,0.3578720987,-0.2318406850,-0.4318566322,-1.0400800705,0.0432879440,0.2405186594,0.6484411359,-1.4198623896,0.5731082559,1.6903151274,-1.2536605597,0.8934561014,-1.7532277107,0.3545506299,0.3714213669,-0.4971423745,1.3633073568,-0.8850327134,0.5577733517,-0.0002178800,-0.7989585400,1.0268219709,-2.0480480194,1.0952467918,0.4634559453,-0.0991200507,-0.2166755944,-0.1800895333,0.2274161875,-0.1490253955,-0.1709231734,-1.1723672152,-0.7563461661,-0.4348413944,0.8389985561,0.5278283358,-0.0855753422,0.6674067378,0.1275337189,0.1393093020,0.0813582242,0.1333002299,-0.5479602218,-0.8135802746,-0.7446947694,1.0566511154,0.8022787571,-0.8659976721,0.3931536376,1.5148967505,1.2328475714,-0.4744767845,-0.0396147296,-0.3586524427,0.0859779865,0.8946546912,-0.2486164570,-0.3365946710,-0.6286838055,2.3325791359,-0.0751589090,-0.1462642252,-0.6692479849,0.1297315210,-1.5226391554,-0.3211878538,-0.5852881074,-1.2526824474,-1.5365287066,0.4731831849,0.8354712129,0.1848156005,1.1968663931,-1.0546797514],[-2.1675684452,-0.7382207513,2.6727535725,0.5967551470,2.2287185192,1.3105413914,1.5683557987,0.3387235701,-0.5780242085,-0.9694200754,-0.0542619228,-0.7023670673,-0.3473659456,0.1283979267,0.9860328436,1.8086926937,-0.2518500388,-0.7601784468,-0.6687451005,0.5814293027,1.2638016939,-0.0553046763,-1.8108512163,0.1957877129,1.5420058966,-0.6303429008,-0.0445778482,-2.4625091553,-0.2039029151,-1.0149819851,-0.3121074140,0.7373507619,0.0069484706,1.5634709597,-0.4883809388,1.6265809536,0.4967319071,-0.6871525049,0.5922659039,0.8526945114,0.6075148582,0.2839967012,-1.9847686291,1.2767001390,1.7508671284,0.2804496586,-0.0903932750,0.7302295566,-1.0770375729,-2.1788797379,0.0533087961,0.5184302330,-0.9022838473,0.7812830806,0.7872949243,-0.8752070665,-0.7053655386,1.6697065830,1.4777274132,0.0331411175,0.4019483328,0.0795291141,-1.7472738028,-1.6027665138,-0.8463315964,-1.3971626759,2.2727987766,-0.7152912021,1.2079150677,-2.5324273109,-0.2201086730,-0.8586494327,-0.6715086699,1.0920449495,-1.4367811680,-1.1334613562,0.8020817041,-1.4378912449,1.9518941641,1.0430494547,-1.9164535999,-0.7914465070,0.4349558949,-0.6983420849,0.8788402677,-0.6938580871,-1.6382540464,-1.3454945087,0.3314038217,0.7301472425,-1.6229919195,2.4571003914,0.9816461205,-0.1934306026,-0.6380050182,-0.6916678548,-0.0730158985,-0.6435752511,0.2499887049,0.9054161906,-1.9165015221,-1.7656427622,-0.9856516719,1.3381596804,-0.0236687176,-1.7433477640,-0.4669364989,-0.3746806979,-0.6021181941,-0.3435127437,0.5429405570,1.6806927919,0.3754731417,-1.7427126169,1.1184897423,-0.0914018527,-0.2547079027,0.0444798470,1.2944244146,0.9571075439,-0.9411146641,0.0178249963,-0.2201198488,0.2704386711,0.1415091902,0.7426109314,-0.1449626833,-1.3681911230,-0.9975286722,-0.7867771983,-0.1971043646,1.1083836555,-0.2619640827,-2.1241817474,0.5822571516,-0.5158762932,1.4686164856,0.4476768076,0.1179724336,0.8681359887,-1.6180802584,1.3157556057,1.3680702448,1.2255792618,-1.2426224947,-0.5846403837,1.3134173155,0.8510391116,0.0566169582,0.5327958465,0.5105850697,-0.0755836368,-0.1719992310,0.5381946564,0.0138440710,1.1242705584,0.3068951666,1.7257516384,0.9957590103,0.8762136102,0.1220595390,-0.5659295917,0.2277036905,-0.5829985142,0.2336221337,0.5245922804,0.6979594827,1.2982851267,0.6850818396,-0.5728030205,0.5248136520,-0.3277309239,0.4528121650,0.4383815825,-0.9423355460,1.1154732704,0.1451929361,-0.7689108849,0.2972296774,-0.5694436431,-0.1806611866,-0.9186413288,0.7959067225,0.4119114578,0.5434252620,-0.0524524115,0.5843896866,-1.5658318996,0.2693421543,-0.2651713192,-0.6462969780,0.9390318394,-2.3189029694,-1.9477659464,0.9899591804,-0.3546177149,0.1199787855,-0.4854714870,-1.6318863630,-0.1745455414,-0.4783552885,0.1253313422,-1.0133267641,-0.1568865627,-0.9031783342,0.0745390579,-0.1635649502,1.6315912008,0.3370809257,-0.5317426324,0.1358670890,-0.7163832188,-1.0768035650,-0.9230828881,0.2219968587,-0.5246748924,0.6829332709,-0.2877226770,0.2317267507,0.4880829155,0.6605032682,0.7699363232,1.2062032223,-1.3281532526],[-2.0703158379,-0.3716563880,-0.3761352897,-1.0077327490,0.0220914204,-1.0736358166,-0.5218760371,-0.6671406627,-0.3167509735,0.2668158710,0.7138624191,1.6762847900,0.5364630222,0.7131706476,-0.4521187544,2.2354354858,1.0070247650,-1.4814945459,-0.9544055462,0.5616211295,-0.4388044178,-0.5384815931,2.5188238621,0.3578646779,1.0072035789,-2.3006868362,-3.0283837318,0.1582179517,0.0786268860,0.2975986898,0.3020519316,0.2252492458,-0.1169260144,1.1698973179,-0.2377756834,-0.9876517653,0.9444678426,0.7366877794,-0.2140249312,0.6662899256,-0.1698795259,0.1597381830,-1.1117084026,1.5774035454,0.7038061619,-0.8269827366,-1.2444053888,0.6046368480,-1.2741203308,-0.0075483327,0.1214978695,0.1185389608,0.1672204137,1.4469048977,0.9831426144,-1.2359778881,-0.0607702211,1.0580685139,2.1470263004,-0.6063710451,0.3545796573,-2.0619568825,-0.6264045238,-0.6515289545,-1.4760322571,0.6560636163,0.1312911958,-0.6359328628,1.0950239897,0.8734268546,-0.0087732999,0.2270132303,0.2920324504,-0.5871611834,-0.6498407722,-0.4679347277,0.7037794590,-0.3263746202,-0.6026700735,1.5626196861,-0.3909978867,1.2148580551,-0.1074242890,0.2417563349,0.8551151156,-1.1614420414,0.4012592733,-2.6826231480,0.2988246977,1.0284411907,0.9653552771,0.0241382923,-0.6744859815,0.0252455268,-0.5691433549,0.9945526719,-0.7929928899,-0.0188295841,1.7620083094,1.1326615810,-2.1382300854,-0.4516732097,-1.7331445217,0.9578317404,-1.9982173443,2.0854375362,-1.4002338648,-1.2631605864,-0.2547115088,-1.1759921312,-0.1266184151,-0.1415224522,-0.2777317166,1.8104367256,-0.8818674088,-0.1876362264,-0.6606785655,-0.8318985701,-1.3102446795,-0.1048089117,0.2003894299,-0.4948415160,-0.6170853972,0.3395398855,1.1125634909,0.8719718456,-1.1671817303,0.1546787918,0.1532687992,-0.7465642095,-0.3725242317,-0.5178562403,-1.1987986565,0.6836966276,-1.4780657291,1.3997263908,-0.7279686332,-1.4971139431,-0.7506303191,-0.8950587511,0.3868414760,0.0534542501,-0.2117811590,-1.5080431700,-0.7876279354,1.1501566172,-0.3116948903,-0.3763196170,-0.9797936082,0.5246469975,-1.2769556046,-0.6016128659,0.0543154590,-1.8809392452,-0.8625527620,-0.4563853145,0.5321665406,-0.2617306411,-0.6068289876,1.8533170223,-1.9562035799,-0.5140955448,0.4745437205,1.1698094606,-0.5805649757,-0.3693213165,-0.9228043556,-0.7345765829,0.5629547834,0.3122575879,-0.3497559130,0.9135907888,-0.7911636233,0.1847207397,0.3809929788,-0.8879823089,-0.1338323057,-0.3490095735,2.1165955067,-3.1589138508,-0.5436285734,-1.7455108166,-2.1049289703,-0.1302325428,0.4640696049,-0.7451955676,0.5188206434,-0.4138991535,-1.8001177311,1.1008683443,0.4474474490,-0.1786603630,-1.0551677942,0.5523467660,1.5565564632,-1.0508276224,-1.2913016081,0.4513285160,-0.7465084791,-0.2419379652,2.0543141365,-1.3258612156,0.8564865589,-0.7004728317,0.9789661169,0.2529447079,0.3756442666,-1.1552598476,0.2251007259,-0.6217958927,-1.2075531483,-2.1160759926,-0.5377485156,-0.4078000784,1.2833286524,1.2621468306,1.9884413481,-1.9587829113,-0.3365916312,-0.3414989412,-0.1485837251,-0.5208674073,0.9343284965,-1.1504461765],[-0.7001681328,0.5539194942,-1.4810436964,-0.6036912203,0.9199162126,-0.3080572486,0.4597660601,0.0606138296,-1.4523596764,0.0182523839,0.3977748454,0.2504182160,-0.0457268581,0.1204058975,0.8092479110,0.7734143734,0.7709316611,0.4113011956,-1.3937188387,0.4055718184,-0.4176865220,-0.3609451056,0.2942406535,1.7711349726,-1.3721432686,1.8080185652,-0.6679579020,-0.6259421110,-0.4016775787,1.0835027695,0.9990584254,-0.1049466804,0.0027226934,-0.8598833680,0.5965404510,0.4464733899,-0.9998130798,0.3802746236,0.1903662384,1.0069195032,1.7607945204,-0.8770285845,0.0015625909,0.1690426022,0.4705689251,0.2795690894,-0.3947535753,0.2917557359,0.4942769408,1.7180832624,1.8849114180,1.0528432131,0.0112085836,1.9515383244,-0.8529167175,2.5300376415,0.9883424044,1.5475788116,2.0330502987,-0.4147629738,-1.7989500761,1.3826758862,-1.0269834995,0.5999916196,0.9867404103,-0.2702056170,0.3600501716,-0.7583336234,-0.5758518577,-1.0197910070,0.6279180646,0.5060213208,-0.1114388481,-0.7055656314,-0.1577177644,1.3694399595,-0.1905371100,-0.6570346951,0.3182444572,0.6559875011,0.1073881984,-0.8325852752,0.7134782672,-0.0001436342,1.1665621996,-0.9761604667,0.4922949076,0.1762838662,0.8394795656,-0.9334070086,-0.6946863532,1.1474127769,0.0837756693,1.0693693161,1.5215953588,-1.8261275291,-0.5862078667,0.8109973669,-0.0215907376,-0.3971729875,0.1443639547,0.7105217576,-1.5630483627,0.8906173706,-0.0771133453,-0.7070862055,-0.6657230258,-0.2032502890,0.4337565005,0.6398509741,-0.0961007103,-2.0606822968,-0.3801790178,-0.7287710309,-1.1679856777,0.8636924028,2.0345959663,0.8056660891,0.0790649727,0.1292844862,-0.2441989928,0.6843965054,-0.0466990806,0.2574798167,1.7390024662,-1.0807328224,0.3902039528,-0.0904696211,-0.6639072299,-0.9807626605,-0.5844775438,0.7836155295,0.0463408977,-0.5736125708,-0.2846724689,-0.8746067882,0.1992656440,-1.1177204847,1.5649669170,0.1965961903,0.8586657643,-1.4847395420,1.4379744530,-0.3827043474,1.0768358707,0.5697392225,-2.5409808159,-1.1086584330,-1.4590468407,1.9061291218,-0.4039606154,-0.3141342402,-0.6788002849,0.4221759439,-0.6185811758,1.2265267372,1.3419264555,0.9365577102,-0.8626218438,0.0714093968,0.3579553068,-0.7517531514,-0.1326746047,0.4824095666,-0.1637495905,-0.2193266600,-0.7112823725,1.6637583971,0.6090463996,-0.3620583117,0.7486864328,1.4172860384,-1.3887069225,-0.4702116251,-1.8121628761,-0.4067336619,-0.4508442581,1.6809742451,-0.6856459379,-1.0960397720,0.6169106960,-0.5126711130,0.7367286682,-0.1227271408,-0.9381112456,0.3682388067,0.5237234831,-0.1901168376,-0.7895736098,0.0390522219,0.1888903528,-1.9014731646,1.7833452225,0.3512957990,0.0422377065,1.1961449385,-0.4127797782,-1.7096159458,0.4396434426,1.6618039608,-0.2889024615,-0.4047232866,-0.3430258334,0.4502983093,-0.8993715644,-0.2819147706,-0.6409164667,0.3729336262,-0.7371287346,0.4244229794,-0.9481826425,1.7409205437,-1.9509413242,-1.0926223993,-0.7474913001,-0.3169195354,0.2738073468,1.0230799913,0.5049025416,-0.1530090719,-2.4460172653,0.3451049626,-0.7874396443,0.4234744608],[0.8010879755,-0.7266035676,-0.0287357550,0.5261215568,0.4974059165,-2.2859344482,-0.5524745584,-1.6442023516,0.2132236212,0.4708622992,1.5447111130,-0.4303492308,2.1815609932,0.3391433060,-1.6993885040,0.5555101037,0.1248709261,-0.5245776772,-0.8237133026,0.0852307156,-0.6021905541,-1.9891923666,-0.2042742521,0.0665240660,-0.5630695820,1.3211286068,2.6150777340,-0.8884028792,-2.7250652313,-0.7953874469,0.4453403652,-0.0466313288,-0.8660322428,0.9148148894,1.3114209175,0.0853873789,-0.0865378082,-0.8240836263,1.5242363214,-0.1447027177,1.0685709715,0.6887947917,0.3693505526,1.2384396791,-0.1528254151,0.4717794359,0.2557708025,-0.2342842966,0.2985180914,1.4603908062,1.1556718349,-1.0860780478,0.8485520482,0.8186067343,-0.4889854193,-1.1824482679,-0.0552532971,-2.0472691059,0.3136096597,0.0956232697,-1.2475279570,-1.0659813881,-1.4372975826,1.3939661980,-1.5543537140,-0.2287389338,0.8398764133,-1.0875582695,0.3782944977,0.5431341529,0.8844070435,0.0712101758,-0.7438293099,1.4641083479,-0.2514543831,-0.7803747654,-0.7884621620,-0.5040911436,0.1236531809,1.3709342480,-1.2878456116,-0.0993014053,-0.5816023946,-0.1617263407,1.4658393860,-1.1843436956,0.1996217370,0.7093799114,0.6478058100,2.4174399376,-0.9946103692,0.9256981015,0.2354338616,-1.5637323856,-0.4097500741,0.9532274008,-1.7446402311,0.3983721733,-0.9129959345,-1.8641510010,-0.0650398284,1.8328038454,-0.3282188177,-1.4489845037,-1.2304841280,0.3586666584,-0.2042628974,-0.3145161271,0.9496209621,-0.3764860332,0.7279493213,-0.8917263150,0.3433779478,1.1824033260,0.0408701040,0.4175057113,-0.6440492272,0.9012810588,-1.0648696423,0.9149550796,-0.5418872237,-0.3558420837,0.8348129988,0.1753689051,0.0140529331,1.4487305880,-0.0789629817,-0.0814047679,-1.5064017773,-1.0320729017,0.5171748996,-2.2602293491,-1.7476472855,-1.5101404190,0.3992056251,-1.5515506268,0.7638925314,0.6154313684,-0.2482495755,0.0194012262,0.0098772850,0.4108774662,-0.0947879553,0.4412504435,0.1092329770,-0.7233854532,-0.5602306724,0.9036995769,1.2431905270,0.0285293981,-1.6290500164,1.2356460094,-1.9054615498,-1.5224015713,0.4625911415,0.1904840916,1.5625308752,0.4856654108,-0.0246778559,-0.1548504829,1.2992460728,0.1943925023,1.2544627190,0.6112693548,1.4020404816,0.8133472800,-0.7494078279,1.3856695890,-1.6867324114,0.1892388910,1.2696735859,-1.3791120052,2.2082250118,-0.9735414386,-0.2286798209,1.3652757406,-0.8393858075,-0.6100654006,-1.1962810755,-1.3717755079,-0.7170159221,-0.8828419447,-0.3372710645,-3.3247058392,-0.9084119201,1.6302831173,0.8176118731,-0.0111802975,-0.2251095921,0.5649040937,1.4012663364,0.7089836597,1.2665133476,-1.1005299091,2.4844765663,0.1761873364,-0.0088386908,-1.3534541130,0.5137736797,-0.5706036687,0.8757398129,0.0449838303,2.4435112476,0.2079927921,-1.9996918440,-0.6087223887,-0.6385773420,1.6442676783,-1.4658410549,-0.2207080126,1.0008113384,-0.7584369779,1.4786957502,-0.4342084229,-0.0399412848,0.0586465672,-0.5101655722,0.0631559715,1.0645006895,0.9889351726,0.7960072160,0.9740015864,1.7200036049,1.4311993122],[0.3660484254,-0.0995860323,0.2606077790,1.2410806417,-0.9831237793,-1.6199501753,-0.1432815939,1.0276182890,1.9477783442,-0.1715546250,0.9028071165,0.9455989003,1.7721807957,-2.3770511150,0.0422005840,-0.2649149299,0.1626689434,-1.7165741920,2.0142836571,1.2888416052,1.3182411194,-0.2988808751,-0.9702196121,-1.3273938894,-0.1208585128,0.1018750072,-0.3974203765,-0.9312421679,1.5495072603,-0.2257322222,-1.5345994234,-1.2101567984,0.5442845821,2.3244996071,0.6885232925,-0.3893218637,0.5857572556,0.8583837152,1.1273128986,1.5923551321,-0.6475256681,-0.4328983426,0.3193134367,0.3736357391,-0.0583607033,0.9926229119,1.1097198725,0.0237506758,-0.3030325472,-0.4242801964,0.0150921475,0.8205795884,-0.1523793340,-0.1743751317,0.1002234742,-0.5741091371,0.4680297673,1.3265185356,0.7591851354,0.7968487740,2.4510006905,1.0455299616,0.1616481096,0.4591381252,-0.5118848085,1.3974906206,0.0444282293,-0.2367054075,-0.5850853920,-1.7926813364,1.0850917101,-0.0153463064,0.8085225224,-0.0650879070,-0.8191810250,-0.3907743394,-0.8836235404,0.8422560096,-0.9372207522,0.1334853172,-1.4340825081,-1.0172991753,-2.9318859577,1.7403756380,0.6695041060,-0.5234505534,-0.4980879426,-2.2248792648,-0.3270122707,1.4429141283,0.0257454775,0.9544253945,-0.1030614227,-0.0222805012,0.8595652580,0.7901018858,0.0139517374,-1.5680670738,-0.7101589441,-0.9396982193,0.5569794774,1.6559907198,-1.0252135992,-0.5993455648,0.7724023461,-0.0052897935,-0.7790496349,-0.5721173286,-0.1611474603,0.1147837266,-0.4912993908,0.2682353854,-0.1408393532,1.1909285784,0.1201085225,-1.1754007339,1.0866947174,-1.3127019405,0.5852900147,0.4494962990,-0.0917720422,1.5772259235,-0.0028221165,-0.9338590503,0.4315549433,-0.7911572456,-0.7792272568,0.4389078021,0.2131284922,-0.4663459361,0.0148027567,0.6259170771,-0.8518497944,0.0935833678,0.2924852371,0.3590088189,-0.5718174577,-1.1496263742,1.2068468332,-0.1130583733,-0.2927685678,-0.6665418744,1.2084698677,-0.5074796677,0.6037199497,0.5330540538,0.0671800524,-3.1931815147,1.3412168026,-0.0472922698,1.2909542322,-0.0951048583,-0.3293198049,-0.3368616104,-0.6205972433,0.1633470356,1.0400021076,-2.4417200089,-0.1430457532,-0.3949324489,1.5079751015,-0.0933669582,0.7111688852,0.4588423967,1.0893750191,0.6711891890,-0.1723787934,0.3056735396,-1.0339529514,-0.5147407651,-0.0625401884,-0.4944783449,-0.2281164825,1.8994734287,-1.1122488976,0.8064043522,-0.9070198536,-0.7515420914,0.4880173504,0.5601146817,-1.6990242004,-0.5601556897,0.0330446623,-1.1386178732,-0.0413848385,-0.4364368320,-0.0484491810,-0.8381305933,-1.9084656239,0.3641952574,0.4897067845,0.2050378919,-0.5196628571,1.1518422365,0.2143942863,0.6736453772,1.3390365839,-0.9382547736,-0.5802168250,-0.2404362857,0.0388144962,1.2864723206,-0.7813917398,-1.4558441639,-0.5873885751,0.9965207577,0.0476341769,-1.6072156429,-0.6474640965,0.7038937807,-1.2242337465,-0.3836533427,-0.3961207867,-1.0149700642,1.0830006599,-0.2466066629,1.1158556938,0.6452320218,-0.1450219601,-1.9907687902,0.2957869768,0.7098045945,-0.0348816589,0.7518092990],[-0.5935727358,-1.2528318167,-0.5177393556,1.5163943768,1.2004730701,-0.9606300592,0.4015153646,0.4090261459,-0.0634377599,1.2732558250,-0.3282780945,0.9231778979,0.2382800877,0.8408981562,-0.7702533007,0.4584795237,0.1061263978,0.0769923702,0.4656309485,1.1552935839,1.3841668367,-1.3653978109,1.5498673916,-0.3918735981,-0.0810078084,0.6211736798,0.7627735138,-0.0829695016,-0.9319149256,-0.5280153751,0.2751726806,0.3330031931,-0.7518762946,-0.1446366012,0.1988864541,1.2466605902,0.4027922451,-1.2030483484,0.4533600509,0.7313265204,1.0936702490,0.0037894470,0.0050270087,0.0251088440,-0.0352713205,-0.5701832175,-1.9719990492,0.4366341829,-0.1345392913,-0.1814393252,-1.8553251028,0.0617607608,0.6062008142,-0.4244881868,-1.0817292929,-0.6807661653,-0.6261473894,-0.0540766194,0.2786253989,0.8372100592,0.0017704318,0.5310544372,0.6352730989,0.9221379161,0.4457738400,1.5647474527,-1.6837626696,1.2410209179,-1.0796965361,2.2747447491,0.5723548532,-0.5651129484,-0.1718373448,-1.6550757885,-0.3474877179,0.6547319889,-0.7872471213,-1.5143712759,-0.3837069571,1.4854574203,1.3285131454,-0.2988612354,1.3719160557,-0.3424156010,0.4441322386,0.3561630547,0.3292013407,-0.4663069844,-0.3964741230,1.4079635143,-0.0168015026,0.2487754226,-0.6136851311,-1.5734347105,-0.8163513541,-0.2254180610,-0.0962739587,0.2816110551,-0.9428220987,0.1577043533,-0.2643011510,-1.3468902111,-1.6785992384,0.6566977501,-0.1757939309,-0.4243683219,-0.8836905956,0.1485837400,-0.7256431580,0.0940819830,-0.7429986596,1.3145816326,1.5680030584,-1.3079791069,1.0795658827,1.6322503090,-1.3270734549,0.2825237513,-1.8363772631,1.4280408621,0.8779270649,0.1488070041,-1.1796215773,-0.7179731131,0.7257335186,-0.0448335893,-0.4180825651,-0.9488744736,1.3562891483,0.6680294275,-0.4531681240,-1.1939545870,-3.9058244228,1.6860084534,0.0763802007,0.8831697106,-0.0188103337,-1.6418547630,-1.3217287064,-1.7891827822,0.3362942636,0.3228811324,-0.1052464694,0.3700760007,-0.2983788848,-1.7491821051,0.0411379486,-1.1203415394,0.6413006186,0.0217642933,-0.6283631921,0.5370006561,0.0651604682,-0.6565572619,-0.4752445519,-1.2263764143,-0.6557531357,-0.1826541275,2.3952448368,0.1607136875,1.0272328854,-0.8275399208,1.0438835621,0.7619448900,-1.1307396889,-1.2205406427,1.3907771111,0.6221396923,-1.0409410000,-0.7856572270,-1.1721094847,-1.8064625263,-0.3921511769,-0.2310555279,1.5131512880,0.4917284548,-0.0432878397,1.6977798939,-1.3908723593,1.4940390587,1.8423306942,-0.1109176725,0.2542709708,0.2416193187,0.1212384477,0.2414720356,-0.4855381846,-0.6720389128,-1.0101215839,1.1480742693,-0.0514740795,0.2556264997,1.1470924616,0.9148079753,-1.1060256958,0.4811087251,1.2506557703,1.3353089094,1.7559952736,-0.1265173256,-0.8010993600,0.4566311240,0.3428300023,-1.7524900436,-0.9317060113,-0.8634976745,0.3061068654,-0.1965166926,1.0003229380,-0.1569621265,0.1014276296,0.8939095736,0.9408544898,0.2834113538,-1.3292179108,0.9167075753,0.2990117371,-0.6239182353,0.9428245425,0.2198539376,-0.4146504402,0.7951261997,0.2012115568,0.6641840935],[1.9132390022,0.0612025335,0.3415098488,-1.0602945089,0.8366987109,-1.6226406097,1.1113557816,-1.3292145729,-0.3143319786,0.3570109010,-0.1764928848,-0.1717350632,1.6343352795,2.5383000374,-1.0527589321,-0.7758113742,-0.3192665279,-1.0465342999,1.9890724421,0.3994129896,-0.5866478086,-0.8957120180,-1.6844896078,-1.3510472775,0.3692949712,-0.2429165691,-2.3788287640,0.5294952989,-0.9295908213,-2.0615482330,-0.6898310184,0.0278328750,1.6711131334,-1.6555784941,1.4271929264,0.0273575801,0.2971141934,-1.1789822578,0.9930096865,1.0365034342,-0.1519527435,0.4682734311,-2.3577785492,1.3565287590,0.9537298679,-0.2706686854,0.2951039374,0.3883643150,-0.0566962734,-0.3984859884,0.2041887492,-0.9976084828,0.7897940278,1.4099681377,-0.3404171765,0.3241531551,-0.4855938256,-1.2421112061,1.5360037088,-0.9463829994,-0.3573916852,-0.0232634433,-1.2513146400,-2.2355549335,-0.7908390164,-0.4839990735,-1.6056691408,0.0734652206,-0.2260978222,-0.5341376662,1.0534939766,1.5730623007,1.0264832973,-0.9042997956,0.7410283089,1.2227886915,-0.9809672832,0.8328786492,2.7288522720,-1.6969711781,1.2512172461,-0.3696545064,-0.2950987518,0.0518392660,0.8288123608,1.2777222395,-1.2569802999,-0.3112655282,-0.5696928501,-0.2377070040,1.1983821392,-0.2934773564,0.8378570080,1.7270666361,-0.8205666542,0.3335210681,-0.8726528287,0.6561540365,1.0646369457,-1.4809577465,-0.3941823542,-1.0625725985,-1.1441496611,1.1643037796,-1.7063095570,0.1826176941,-0.3895122707,-0.5045616031,0.5522103906,0.0773333758,-0.0574592091,-1.5021427870,-1.7225257158,1.7434196472,-1.8855978251,-0.1019827873,0.4856334627,-1.9518294334,-0.1191142574,1.0445772409,-1.1645052433,1.6964392662,0.5589596629,0.6847245097,0.6614169478,0.4820757210,0.3499038517,0.6179351211,-1.9481719732,0.2576283216,-2.2164435387,0.2630864084,0.0669203550,-1.1707545519,0.6772699356,0.1684508771,-1.2875515223,1.0062065125,-1.1552302837,-0.8029338717,-2.2737193108,-0.7485864162,0.6612769365,-1.1619497538,2.1173989773,-0.3294250369,0.3562559187,0.2107373476,0.8565757871,0.6712851524,0.5338254571,-0.8010561466,-1.6697434187,0.3606756032,1.2867676020,-0.2094346732,0.5576734543,1.0525827408,0.9210794568,-0.2470230609,-0.0034603099,1.6976779699,0.6332430840,0.0199560001,0.0755996332,-1.0569541454,-0.6732931733,-0.4159931839,-1.5330858231,0.5702850819,-0.1691996306,-1.8038196564,-0.9590678215,-1.5392922163,0.3994826972,-1.0821446180,0.3027870357,1.0220326185,-0.3363763094,0.0301486459,1.5134326220,0.5621951222,-1.2312422991,0.6054644585,1.1600813866,1.3731253147,0.5823098421,0.1793181896,-0.9241508842,-1.3068791628,1.0078873634,-0.0847553387,0.6938104630,0.1029771790,0.1699392349,-0.5331149697,-0.5968712568,0.8790861964,-0.2934698761,1.2495779991,0.1272739172,-0.7125212550,-0.4451971948,0.4075957537,-1.0298527479,-0.2791715860,0.3461883962,1.6816877127,-1.6031706333,0.0056235124,1.0078634024,1.3490788937,-1.5436534882,0.1696927845,-0.5270277858,1.6223733425,-0.5162442327,0.3828381300,0.4506391287,-0.9998950362,-0.6521401405,-0.3246061206,0.2585060298,0.9276508689],[-2.0880565643,0.8682402372,0.0620451719,0.2503739893,-1.0289133787,0.1270450950,-0.8162886500,1.4642271996,-0.3115217388,3.0074081421,-1.2791824341,-0.2708213627,1.7376896143,-0.5052874088,0.6811109781,-0.7712689042,-1.5408807993,1.1143568754,-0.4985656142,0.7032669187,-1.3779296875,0.0579764359,-0.3669286370,0.5524052382,-1.1774955988,-0.9593952298,-2.5553278923,0.3986499012,-1.8606998920,0.5338377357,0.6182241440,-0.3920603395,-0.9350994825,-0.6044998169,-0.3034831882,-0.4526685476,-0.5963338017,-0.7564249039,-0.2662839592,-1.2210955620,-0.6124317050,-0.5627401471,-0.4255528152,2.0560801029,-0.8246173859,-0.9876614213,-0.6199842691,-0.4691637754,-0.1785079837,0.5794753432,-0.6497477293,-0.3748328388,1.7727497816,-0.4439648986,-0.2879000008,1.4938969612,-1.4431997538,-0.1966993660,-1.5506589413,-0.0824496299,-0.3735535145,1.8474872112,0.4143075943,-1.3707956076,0.4110006988,0.1122452915,-0.8783571124,0.1543612927,0.0001337494,1.3457558155,0.1956868619,0.0120426994,0.4659405947,1.0559220314,0.4118642807,-0.2827820480,-1.0537084341,-1.7089247704,-1.2605371475,-0.6627028584,0.0284964610,-1.5919424295,-0.8879739046,1.3372726440,-0.3084809184,0.2604015470,1.5567058325,-0.9774645567,0.9206609130,-0.3910360932,-2.0741565228,0.6043616533,0.8186147213,-0.2919317484,-1.8189321756,-1.3367571831,0.0049841022,1.1277798414,-0.1117683128,-0.7106814384,-0.0921612084,0.4363644421,0.5508530736,0.2112184912,-0.4217892885,-0.3625360429,-0.5210886002,-0.7911268473,0.6036680937,-0.9205818176,0.9471231103,1.3281573057,0.5098133087,-0.2765147686,-0.0400175266,0.9152051806,-0.0791764036,-0.4207233191,-1.4149441719,0.1290478557,-0.0053360481,-1.1818076372,-1.2097790241,-0.6358364820,1.0997394323,-0.3663605452,-1.3995124102,0.5268882513,-0.0040545431,0.8065931797,0.5818359852,-0.7355961204,0.2951637805,-1.8553963900,0.7826838493,0.9054294229,1.3948525190,0.4250205755,0.3838944435,-0.6590161324,-0.5383474231,1.9159451723,-0.3733713031,2.2129595280,1.8961585760,1.6835546494,-0.5816136003,-2.1392905712,-0.9826152921,-1.5338691473,-0.4559504688,-0.1861871928,-0.4478240609,-0.9198418856,0.6889845133,-0.7898256183,-1.5680773258,0.6147295237,-0.5621627569,-0.3072946072,1.9393631220,0.9291343093,-0.6737373471,1.3822253942,0.2642171383,0.2372154593,0.3773427904,-0.4426511824,-1.9745779037,0.8025727272,-1.0189781189,-0.6997631192,0.2891789675,-1.1383354664,-2.0471360683,-0.1752144843,-0.0870466754,0.2797400653,0.2387488484,-0.1481952071,-0.2302711010,0.8003568649,-0.6089862585,0.0534715429,-0.5716961026,-0.9841057658,1.2828774452,-0.1688161492,1.2798538208,-1.0940895081,-0.2482675761,-0.7160670161,-0.1450303942,0.3953606784,0.6148098707,-1.0022808313,0.3334307969,-0.9138579369,-0.8279646039,-0.2285257876,1.1132116318,0.6443887353,-0.1507997662,0.8884394169,-1.0409892797,-1.2017220259,0.4876228273,0.6594662666,1.0187479258,-0.8132072091,0.4418626726,0.6325352788,1.2463976145,-0.1135608032,-0.9164980054,0.7859613299,0.4463684261,0.6968076229,0.6227259040,-0.6469994187,0.9220806956,0.7630829811,1.0957705975,-0.4319012761],[-0.4152270555,-0.2044944763,0.9506226182,0.5847282410,0.2539539337,-1.7795450687,-0.6887845397,0.6866399646,-0.0558012277,1.2429003716,-0.5696885586,-0.1877678037,-1.1906847954,-0.4526516199,1.0057507753,0.7095575929,-1.6089472771,0.6746912599,-0.0422934592,-0.1314605325,1.1723577976,1.3062764406,0.0529669598,-0.3620713353,-1.5355778933,1.1157461405,0.1266220212,-0.3168752789,-0.2654770017,-1.4859975576,-1.1428658962,-1.8808702230,0.7916238904,-1.0978664160,-0.3868590593,-0.0748722702,-0.5582896471,-0.3408991992,0.0357785411,-2.8540141582,0.1198951602,-0.3569850326,0.6131075025,-1.6270140409,-0.2847393453,0.0791950300,0.7125725150,0.9216774702,-0.5362963676,1.0582467318,1.0025253296,-0.4564027488,-0.9334833026,1.2776427269,1.3159724474,0.9056265354,-0.0394703001,0.2864100337,-1.8664575815,0.1179925725,0.2843076587,0.6929876804,-0.8222893476,-0.7008691430,-1.1756200790,0.0781022459,-0.9581169486,1.2042080164,1.2288545370,-1.2023644447,-2.1400723457,0.8589876294,0.9031086564,-0.6311286092,0.2257191837,-0.2904500663,2.5082805157,-0.7749772072,0.4404409826,1.5937997103,-0.6869628429,-0.8768021464,1.1542279720,0.3378003240,0.7085487843,-0.0843881667,1.3874114752,-0.8350510597,-1.8387274742,0.0471214205,-0.6551354527,-0.8090748191,0.6171238422,-0.1551373601,-1.3722908497,-0.7655444741,1.6287691593,1.5615321398,0.3192647994,-0.9870176911,-1.4553993940,-0.6677874923,-0.0301759038,1.0221860409,0.2797620595,1.9955043793,0.2484874576,0.0276262611,0.2522889674,-0.8351356983,-1.0892126560,-0.1633870602,-0.4688637555,0.2814452946,0.6965830326,-0.8071414232,0.4933938980,1.1570245028,0.5350809693,-1.7146855593,0.3662984073,0.6917946935,0.4543409348,1.3346126080,0.8517884612,-0.8048650026,1.4301743507,0.5187414289,2.5650734901,0.5571698546,-1.4397175312,-0.0305997543,-0.5573413372,-0.3947639763,-1.7622674704,-0.5569675565,-0.9407226443,-0.4493946731,-1.6613689661,-0.8826978207,0.3443049788,-0.1960539669,1.6036450863,0.2909748852,-0.1233137473,-0.2173997611,0.3150077462,1.0498563051,-0.6585019231,0.0957669914,-0.2687816024,-0.6586936712,-0.3349936903,0.9559844136,0.2193197906,1.3311969042,1.7547625303,-0.4384146035,-0.5910891891,-0.0369080007,-1.7037577629,1.1412252188,-0.1880388260,-0.6701797843,-0.2139253467,-0.4461108446,0.4609785080,-0.6667152643,-1.0494136810,-0.9170075059,0.4246536493,0.4500367045,-0.4323231876,-0.7817713022,-0.8252133131,0.9703499675,0.5069776773,-0.5588415265,-0.5356715918,-0.2954487205,-0.6099143028,0.0746136606,1.6815915108,-0.3075112998,-0.8263124824,-0.7620713711,-0.8745070696,-1.0211056471,-1.0041357279,-0.7308058143,-0.2631337643,0.5446914434,0.6488231421,-0.5848956704,-0.2323754430,0.6658927202,-2.3170225620,1.2412644625,0.7278534770,-0.1591682732,-0.0863246992,0.0435153991,1.1766989231,-1.3432213068,0.7669236064,-0.8524904251,-3.0434315205,-1.0095849037,-0.0935433209,0.4370863438,-0.2978182733,-0.8723636866,-1.6289815903,-0.4529634416,-0.2159748822,2.2257893085,0.5159680843,-0.0077975448,0.6277788877,1.8291573524,0.7833284140,-0.6949352622,-2.3123188019,0.8569595814],[-0.3096266389,-1.2400895357,-0.3410450518,0.5242138505,1.9794309139,-0.1311748475,-0.6779469252,-1.0015225410,-0.7074307203,-0.6888180971,1.0624884367,0.6066510081,0.8800684214,-1.3701462746,1.2867304087,0.5365315676,1.2999631166,0.6271314621,0.3918847144,1.9875098467,-1.5191190243,0.4580776691,-0.0840576738,-1.1077204943,-0.3676539361,-0.5172212720,0.4895897508,0.6502180099,1.4189645052,0.4343042374,-0.3245664239,-0.3949351311,0.3454012871,-0.0033489121,1.3653379679,0.7686623335,2.2122874260,-0.2360703945,0.5532864928,-0.1661921591,1.6751831770,0.0132257845,1.4176884890,1.7192947865,1.6324951649,0.3114824593,0.7976205945,0.4467043281,1.4926853180,-0.2878831625,-1.6661862135,-0.0044908966,-0.2455088496,-3.2745137215,-0.2838715017,-0.1518843770,-0.8731604815,0.1849374026,1.0501680374,0.2308520377,-0.1487602592,1.1539299488,1.0388782024,-0.7927692533,1.0399427414,-0.3865675628,-0.3344063163,-0.7397027016,1.5190846920,0.0405279025,0.9089628458,0.3263829648,-2.2702953815,0.2929112017,1.2355471849,0.7129567266,0.5659303069,1.5859659910,0.3280636072,-1.5915364027,1.1417033672,-0.4248448312,0.7930880785,0.0883681923,1.8440197706,0.1610373408,0.5732743144,0.0075075594,0.4905359447,0.6674293280,-0.0319283381,1.1117783785,-0.8937884569,-0.5062664151,-1.0149229765,-2.0016167164,-1.0205111504,2.2229752541,0.0113997580,0.8003988266,0.2961736619,-1.6971895695,0.8078061342,0.2169072777,0.0451913550,0.5088980198,0.2208980471,0.6954385042,0.8863599300,-0.3556869626,0.5780002475,1.1718003750,1.1406605244,1.0717451572,0.4423497319,-0.5674478412,0.2898722291,-0.9614549875,2.5144340992,2.3160614967,0.8769077659,-1.0370091200,-2.4022161961,0.6049721241,-0.4953610301,-2.3281629086,-1.4473972321,-0.9300103784,-0.9152269363,0.0122897141,-1.7381287813,-0.9521546960,1.2463902235,1.2603173256,1.0538812876,-0.4877656996,1.4871590137,0.7009145617,0.6607995033,-0.5052556396,-0.3830589950,0.2948637903,0.6955531240,1.4680861235,-0.8497928381,0.3967521787,0.3792177737,-1.8099541664,0.8800607324,-0.2358582616,-0.0895350128,0.4097339213,0.1439082474,-0.8039250970,-0.5788547993,0.6057286263,-0.0667697638,0.5830227733,0.9660890102,-0.2354748845,0.4906316698,-0.2285811156,0.6608774662,1.3144094944,-2.1506252289,1.1890547276,-0.0107625481,0.5143793821,0.6493283510,-1.3720718622,1.7808263302,-0.5661188364,-0.1417411417,0.3248441517,0.5483885407,-0.3008512259,-1.2304807901,-0.3662090898,0.1400451660,-0.0959511250,-2.2191183567,1.1093850136,-0.0595243648,0.9957280159,-1.1562790871,0.7081853747,1.2150986195,-0.1032100692,0.6972517371,0.7193431854,-1.0294797421,-0.1768095046,0.9343592525,0.2372649163,0.7474091649,-1.5894510746,0.7022696137,0.8446638584,0.7430752516,-0.6214725375,-0.4367303252,-0.4456193745,0.8655875325,-0.5149858594,0.3280645013,-0.1024664417,0.7322508693,-0.6337894201,-0.4929722548,2.4898169041,-0.8463423848,-0.5593562722,-0.5450578332,0.8441916108,1.4418475628,-1.0300234556,-0.6466807723,0.2288228869,0.0954682603,-0.5915262103,-3.0876836777,-2.0283315182,0.5751449466,-0.6132372022],[-1.1095269918,-0.0689608827,-0.0798729211,-1.4768925905,2.1811816692,-0.2500698566,-0.5732578039,1.0652742386,-0.8546937108,-1.1292870045,1.2695087194,-1.2510520220,-0.8316658735,-1.6696878672,0.4545242190,0.1797860861,0.0970116705,1.2030805349,-0.7623143792,0.8427641392,0.2079904526,0.7502828240,0.2300081998,2.2469732761,0.9968439341,0.6118012071,0.4577815235,1.2006202936,0.4174729586,-0.2497812212,-0.2987058163,1.5730762482,-0.7681244016,-0.4798147976,-1.6172803640,-1.7546819448,0.2139769793,0.3619492650,0.0847462118,-1.7089012861,-0.1561836153,0.3462845981,0.6182725430,-1.8326427937,1.2170871496,-1.0436891317,-1.3648991585,-2.2823894024,-0.5620194077,-0.5828886628,-1.7087174654,0.6809772849,-1.0947186947,-0.2100659460,-1.0447046757,0.5272279978,-1.5686168671,-0.1761078984,1.0267621279,-0.9264883399,-0.5448895097,-1.1664751768,0.9067573547,-0.1943168789,-0.9029732347,0.3409390152,-0.3423842192,-0.4933809340,-0.1203116477,0.6469014883,1.4632275105,-2.2619640827,-0.4403368235,-0.2874864936,0.3995994627,0.0293659437,0.4649489224,0.5351887345,0.5258300900,0.2087713480,1.2079454660,-0.0458886884,0.5520437956,1.7916995287,0.3296727836,-0.2645019591,0.1053925529,-0.3000160456,0.5894111991,-1.7004368305,-1.1702919006,0.7144211531,-0.3652415574,-0.4975321591,-1.2789793015,-1.5159935951,0.9037019014,0.9386519790,-0.2034149766,0.0920588151,-1.9900647402,-1.4680039883,-0.8426797390,0.0067214174,0.9794570804,0.5074473619,-0.4286257327,-0.6218476295,1.3840569258,2.5538566113,0.6628007293,-0.2189400643,0.4221946299,-0.7699494958,-1.4606316090,1.2698231936,-1.3451348543,-0.1210730821,-0.9649879336,-1.0809884071,0.5695738792,1.4865843058,-1.7563409805,-1.7095904350,-0.8119915128,-0.5803133249,0.2327409685,-0.6860216260,1.4256445169,0.5897830725,-1.0782427788,-1.7356255054,0.8439821005,-1.0922048092,0.3640676439,0.1694461256,-0.2845337391,-0.0015914980,-0.6929503083,-0.3809933960,0.7967649698,0.3464201391,-0.8488762379,0.0735314563,0.3796829581,-0.3305215538,0.1394914091,0.2664026618,0.5161311626,-1.1287683249,-0.2626350820,0.5476616025,0.3476051390,-0.6108629704,-1.8545712233,-0.7726867795,-0.1765599549,-0.6290928125,-0.2056999356,0.6018232107,-1.4811753035,0.9063700438,0.5083891749,-1.2830897570,-0.4046699703,0.7701845765,0.0314547308,0.4233820736,0.9221630096,1.7657654285,2.3693754673,-0.1023481190,0.2427469939,0.3453752100,1.0699369907,0.3138153553,2.0743460655,-1.7169541121,-0.4115565419,-0.1496352702,1.1596267223,-0.7840048075,-0.7558175325,0.8825569153,0.6057924628,-0.0026302291,0.6777153611,0.7162336111,0.8861199617,0.9713836312,1.0570205450,0.1008922532,0.2519199550,-1.7796982527,1.0196323395,0.1301587075,-0.3126945794,-0.5350653529,0.2415903062,-1.2447651625,0.8951871991,0.1416310817,-0.5771242976,0.3480791152,-0.5791205168,0.5143607259,0.6251235008,0.4787263274,-0.5682345629,0.5818405747,0.5476638079,-0.2344136238,0.4926259518,0.0020974108,0.4648458064,-1.2469049692,0.9685260057,0.2488428503,0.8673420548,1.3650695086,-2.1453146935,0.0148590002,2.2118170261,-0.4790977240],[1.8335345984,-0.4013042450,1.8113731146,0.8131791949,-0.4838180840,-0.5870887637,-0.3348998725,-1.5233494043,-1.4847657681,0.4049981236,-0.3059670329,0.1298778802,0.7187888026,-2.7429368496,0.4560085237,0.3684889376,0.2669047713,0.5020672083,-0.0811798126,-0.0078832060,-0.2715966403,0.2410754412,0.0067627095,-0.8049398661,-0.3467500806,-1.1980543137,0.3195302486,-1.2405413389,-1.0926430225,1.2750275135,-0.2405105084,-0.1525734663,0.5744650960,-0.3584517539,1.9834860563,-0.0039220667,-0.2727905512,0.8813328147,-0.8601807952,-0.0736739039,0.3197454810,-1.5756418705,0.5200622082,-0.4795474708,2.6535255909,-0.9987680316,1.2702453136,0.3841578662,-0.4805614948,1.2172632217,0.7606619596,-1.5414643288,0.1976353973,0.7160565257,0.4035159051,0.6984516382,0.1601044387,1.0915095806,0.3534328341,-0.1184051558,-0.4150804579,-1.8912463188,0.4045003653,-1.8975316286,-2.2630052567,-1.4463518858,0.3579699993,2.1300139427,1.2352734804,-0.2789600790,1.3526080847,0.3938015997,-1.3408533335,1.7260075808,-0.0077996096,1.5074975491,1.8468036652,0.6475793123,-0.3252647221,-0.8967847228,0.9662185311,0.1558922529,-0.1998231411,0.6230989695,-1.4783391953,0.6823682189,0.8574786782,-0.8982612491,-0.0487452298,2.1404359341,-0.7900241613,0.2257759124,-0.4074427187,-1.0557363033,-0.9133280516,0.3864759505,-0.2905539274,-0.7860685587,0.7287256122,-1.2111824751,1.0915997028,-0.5741703510,-0.8632349372,1.8330698013,-0.2431361377,-1.3883429766,0.5145610571,0.4060795903,-0.3183524907,-0.0028131553,0.1949463636,0.1193532422,-0.5316559672,0.8666998148,-1.0010826588,-0.5808494687,0.8329043984,2.0336697102,-1.1984018087,0.7143980265,0.1848467290,0.6511341333,0.5259750485,-0.3475879431,-0.9058694839,-0.5467703342,-1.4293168783,-1.7306935787,0.6718448997,0.5225819349,1.1162972450,-1.5372996330,-0.7658146024,-0.6932194233,-0.5045186281,0.7642450333,-0.5672447085,-0.3762857020,1.3930920362,-1.0889377594,-0.9572563767,-0.0917122737,0.6868191957,-0.4004735351,-1.3633692265,0.6427147388,-0.7196304798,-0.8512632251,0.5361620188,-1.1725454330,0.1028381735,0.5060002208,-1.3521600962,-0.8992948532,-0.2606385052,1.1963212490,0.4467527270,0.6490583420,0.6809411645,1.4820773602,-0.7598612905,0.7339666486,1.4499570131,-0.1634774804,-1.6448104382,0.0809721798,-1.1142492294,-0.2901205420,-0.1146303713,0.2195672244,0.4080210030,-1.0791637897,-1.8717880249,1.6003428698,-0.1694111526,-0.3783018291,-1.6030508280,0.2283682078,1.0475113392,0.7307556272,-0.4210613668,-0.3517955542,-0.4050840735,-0.5490471125,-0.3479705453,1.9079108238,0.3872929811,-1.1613664627,-0.1516494900,0.3707110584,1.4472540617,-0.2746278346,0.4029833674,-1.2051960230,0.2465004772,-0.7065382600,0.0371126421,0.8646813035,-0.6913815737,-0.3071915507,-0.1538460851,-1.7917917967,-1.2598940134,-1.8016839027,0.9368255734,-1.3379613161,0.5263180733,-1.5782839060,-0.4033615589,1.0468375683,0.8125714064,-0.4790661335,1.0822108984,0.5306308270,3.2135152817,0.6620602608,-1.6667072773,0.0143323876,2.9585022926,1.2215119600,0.4183119833,-1.1557128429,-1.1458501816,1.0365495682],[-0.2553441823,-0.4749603271,-0.4489164948,-0.1246030629,0.9618656039,0.0579839200,-0.3051787019,0.1202649400,0.0543751307,-0.6086128950,0.0081361765,0.4306674302,-0.0972862765,0.3353385031,-0.2733195722,-0.1604502201,0.9772319198,-0.1361977905,-0.5129996538,0.6338104606,0.1387738585,0.1811870933,-0.4650717676,-0.2434235364,1.3702030182,-1.0426175594,-1.5735248327,-0.5793191195,-0.3952281177,0.6459346414,-1.9756667614,0.3489671350,2.3008513451,0.0771680027,0.6087843776,-1.4710805416,0.3890483677,-0.5295361280,0.9832375646,0.1260338128,0.8573789001,0.2447784096,0.3204292953,-2.5175051689,-0.7122991681,-0.8745144606,0.0141174290,0.9317763448,-0.9803977013,-1.5740404129,-0.3517892361,-0.8231508136,0.0262164883,0.6211560369,-2.0268793106,1.0612952709,-0.3552500010,0.9536997080,-1.2097529173,0.4185555279,0.2573874295,0.6497887969,0.8740681410,0.4048483372,-0.4238315225,-1.9540976286,0.4655889571,-0.0313888192,-0.2161873132,-0.2317292094,-0.0759339109,0.0675848275,0.0878292397,-0.9489600062,-0.1432806849,1.1445298195,-2.7369377613,-0.5840117931,-0.5899749398,1.2879940271,1.3466374874,1.0962083340,0.2304671705,-1.7148902416,1.3411163092,0.1041158214,-0.0788779110,-2.2844712734,-1.0280727148,-1.2645610571,1.1588771343,-0.4571908414,1.2539838552,-0.2162689269,-1.7553088665,0.2741356492,0.7430927157,-1.1741378307,0.3358246982,0.0238060430,0.0031688462,-1.3877177238,0.5490756631,1.1530783176,1.6128815413,-0.3410646319,-1.2937667370,-1.6469665766,-0.7334685326,0.1037245989,0.8387489319,0.0148075577,0.5645905137,-1.3039965630,-0.5846779943,-1.5839500427,-0.2638729811,-0.0072477343,0.8843932748,-0.3676111102,-1.5938299894,-0.8872205019,-0.4249498248,-0.1557797194,-1.3146603107,-0.1697295159,-0.2765368521,0.4457674026,0.9021023512,0.1397904456,1.7429642677,-0.4599343836,-0.6597484946,0.4328436255,1.4983532429,1.2432944775,-0.4208336174,0.4489599764,0.1689072102,-0.4926060140,-0.0422246605,-2.3662722111,-1.8588395119,0.3812257349,-0.7578253746,1.2569688559,-0.3351842761,-0.7692585588,0.4607789814,0.8273998499,-1.1147665977,0.8162701130,-0.2671040297,1.4073350430,-0.7888681293,0.1561743319,-1.2401427031,-1.2775328159,-0.7646373510,-0.6475517750,2.0057315826,1.0876049995,-1.1954290867,-0.1807238609,-0.8135599494,-1.2610627413,-0.9346448779,0.3905242085,-1.4709957838,0.6816416979,-0.4965876043,-0.1572820246,-0.4188430607,1.1873232126,0.9308753610,-1.9551130533,1.0496579409,-1.0345253944,0.8236737847,0.5495346189,0.5497707725,2.3413565159,1.6484090090,-0.7264957428,0.3208186626,0.2824824750,-0.6003243327,0.4457696080,1.7169154882,0.0159275644,-1.6579325199,-1.0831270218,-0.5027672648,-0.6477096677,1.4063690901,0.3528015018,1.1226716042,-1.5352687836,0.3197790682,-0.2060278803,1.3496638536,0.6235291958,0.0421331450,-0.3542109728,-0.6579025388,-0.6250056624,0.6840335131,1.3491804600,-0.6798034310,1.0311497450,0.3708248138,2.3034310341,-0.8823706508,0.2779397368,0.2376018465,-1.0999842882,-2.3625836372,2.5004153252,-0.0031369352,-0.5761529207,-0.9552867413,-1.1486686468,0.2252877206,1.3544378281],[1.1923971176,-1.0452986956,0.4250714779,-0.6534120440,-0.9170339704,-1.8241276741,0.6940741539,1.0891760588,1.1354733706,0.8429929614,-0.5788779855,-0.3875228167,-0.5509646535,-1.9952298403,0.8218613863,0.6798110604,1.1460310221,-1.0449521542,-0.5703985691,0.5750790834,-0.7848528624,-0.5177422166,0.2719036937,0.5763064623,1.7560209036,0.7867377400,-0.8005876541,-1.3305873871,-1.3257182837,0.7724911571,-1.8970799446,0.9184693694,-0.5014103055,0.5121781826,-0.2338661253,-1.9732248783,-1.2685248852,0.5421708822,0.3566212654,-2.7719295025,0.0120019102,0.2266089767,1.5233207941,-0.5277122855,0.8992790580,0.9162186384,1.3245271444,-0.3380609453,-0.2191164792,0.1802497059,0.7791525126,0.3603124619,0.6356837749,-0.0949871168,-0.1510977745,1.0524718761,-0.8924908042,0.1134970784,0.7762354612,-0.1041747034,-1.2717193365,0.1911706775,0.2469402403,-0.4907810986,-0.0449020043,-1.1582369804,-1.8383409977,-1.4759110212,1.8879642487,1.2204946280,0.7254795432,-2.1488111019,-1.9087783098,0.8363744020,0.1254971474,1.1508957148,0.1561222225,-0.1453067958,-1.0493863821,-0.3168115318,0.5137270093,-0.6062394381,-1.0988081694,1.2591933012,-0.1684797853,-1.2144109011,1.3161044121,-0.2794598639,0.6305717230,-1.1718168259,-0.5043300986,1.3296680450,0.6659186482,0.0582517385,0.0609587096,-1.5843622684,0.9506537914,1.3065859079,-2.2422685623,0.5747953057,-0.1019831151,0.1581490189,-1.2338908911,-0.7650310993,-0.4184915423,-0.9221194386,0.6965978742,-0.8460552692,0.1583629102,0.8735681772,-0.3400534987,-0.1723876595,2.3087937832,0.6242477298,-0.3329307139,0.1324429065,-0.5053774714,-0.9988173246,-0.9100036621,0.0229380503,-1.2823932171,-0.3242506385,-0.6225237250,0.7587878704,-1.6966271400,-0.3292137086,-0.9888989329,1.8447217941,0.6552241445,0.5138090253,-0.7122284174,-0.6896860600,0.2713268697,0.2952925265,0.1329986453,-0.2784538269,-1.2005258799,-0.4423667192,-1.3582398891,2.2271566391,1.7319980860,0.1650353372,0.6235536337,0.9041349292,-0.7492755055,-0.6408665776,0.5311431289,1.1350789070,-0.0610747412,1.5564525127,-1.1107958555,-0.9086764455,0.4045865536,-1.2979991436,-0.3478908539,-0.6960718632,-1.7867015600,-0.3869774938,0.3975443840,0.3818458319,-0.9753320217,-0.2342271954,-1.3683166504,-0.2547301948,-2.3559172153,0.3542802930,0.6389033198,-0.2198933065,-1.5690653324,-0.4407686889,0.8176122904,-0.0587791465,1.2063684464,1.1797443628,-0.3437459171,-0.4739488363,-1.5506465435,-0.3452513516,0.8360907435,-0.8022053242,1.5924566984,-1.5707868338,-1.0464003086,-1.2346476316,-1.5263248682,0.8003885746,1.1033037901,-0.1054809988,1.4715224504,-1.7682429552,-1.1850502491,-0.8126041293,0.8842704296,0.9425088763,0.0330735892,-0.2409315258,1.5489830971,0.6468773484,-0.1781851649,-2.7890059948,-0.0014940457,0.4603638053,0.1230962947,1.5464887619,0.3794255853,-0.7880982161,0.7934423089,-1.5169560909,0.0380557850,-1.4559270144,0.5339881182,0.3679506779,-1.1586297750,0.0873472542,-0.5471687913,0.0661044717,2.1989641190,0.2341248393,1.7468538284,3.7643828392,-0.9423438311,-0.0853690207,1.9483058453,-0.2876728475],[-0.1539317667,1.6529641151,1.3316450119,0.9574147463,-1.1216984987,-1.0181498528,-0.7338421345,-1.3882849216,1.0699387789,-0.6837269664,-1.2928775549,-0.1253340095,0.4996267855,0.9240604639,0.5986737013,1.7402137518,0.3486900628,-0.5245723724,-1.1849862337,0.2726819515,-1.6281638145,0.2654599845,-0.4116245806,0.4574624300,-0.1537874192,-1.9013284445,1.4959928989,-0.0532272644,-1.2966313362,-1.3777812719,-0.1085461006,-1.1366568804,2.4738366604,-0.6625595093,-0.3700547814,-0.6848813891,-0.9847663045,-0.3399543166,-0.7798151970,1.1458569765,1.2416856289,-1.1501663923,-0.6315066814,1.1676069498,-1.4100657701,-0.5687700510,-0.5040619969,1.4950553179,-0.7586460710,0.3278019726,1.1096271276,-0.3833582699,0.2235788554,-0.5815556645,0.0966813415,-0.6273540854,0.6732571125,0.0662461072,-0.1287237704,-0.8700714111,0.8890676498,-0.3600372672,1.6472837925,-1.6016446352,0.8628495336,0.4772661924,0.0748748258,-0.3812872469,-0.7134984136,0.6940374374,-1.6486783028,1.4736716747,-0.0254137758,0.0967500806,-0.1661897451,1.1823667288,-0.6112408042,-1.5404152870,1.2952648401,0.7924426794,-0.4812881052,0.8054927588,-0.2136539519,-1.7206912041,-0.3016053438,0.2895011902,2.0912322998,-1.9579486847,-0.8064804673,-0.3919655383,1.6497608423,1.0967392921,-0.0155962491,-0.0089084962,0.5606082082,-0.7090340853,0.7310581207,0.0730704814,0.6230039597,0.4864089191,0.4549812078,-0.2649352551,-1.1511001587,-0.9840397239,-0.2999227345,2.2628390789,1.1271271706,0.3714485168,0.2622233927,1.4365916252,2.0530672073,-0.0107169645,0.6136042476,0.0226049926,-0.0324297585,-0.7721080780,-0.5181171894,-0.3055936694,1.3205901384,-1.5773636103,-0.3167185187,-0.1485596001,-1.1861242056,0.1441131532,0.7637791038,-1.9861276150,-0.2936843038,0.8477237821,0.0517439954,0.0406466722,-0.9705188870,-0.7261933684,-0.0312301386,0.3390150666,-1.0390940905,0.2602939308,2.0953760147,0.0473578013,1.2825541496,-0.3004450202,0.5410876870,-0.1447229385,0.4664852023,0.3632920980,0.5882064700,-1.5549762249,0.0268624183,-1.0717278719,-0.1177723333,-0.4627850652,0.3465367556,2.2943594456,1.7421190739,0.0722834393,0.5992326736,-0.7836269140,0.5283228755,0.0322488360,0.5551918149,0.2175752223,-0.4546332657,1.4639328718,-0.6801419854,-0.0971610919,0.6066857576,1.3741084337,0.2518433928,0.6852304339,0.2889463305,-0.1472174823,0.2218355238,-0.0487602502,-1.0502138138,-2.2987830639,1.5475156307,0.3123504221,1.0007822514,-1.7533344030,0.3010531962,-0.1631305069,0.4981911182,-1.0523788929,1.5803315639,-0.5132887363,0.2444366068,-0.7612324953,-1.0226480961,-1.5267606974,1.2850830555,1.5239545107,-1.4156295061,-0.8883552551,1.0721236467,-0.7571452856,-1.2760622501,-0.0873958692,1.1551344395,-0.2928124964,0.1826197952,-0.1817886829,0.0830894485,-1.1084549427,2.3446435928,-0.1304149628,0.8465356231,-0.0870315060,-0.2174553275,0.1470153332,0.6346374154,-0.2693883181,1.3204683065,-0.3679193556,1.1488084793,-1.1435592175,-0.4943830669,-0.2291122079,0.2982085347,-0.9264073372,0.9086833000,0.8751946092,0.3901867867,1.0084059238,0.2700186968,0.4779162705],[0.2833504975,0.0971349180,-1.3063620329,-0.7672454119,0.1625791639,0.1736915410,-0.8138611317,-0.3182066679,-0.5231506228,-1.3166358471,-0.2597752213,-2.8546957970,0.2950502336,-2.2095010281,-1.1995346546,0.1285841912,-0.0192146059,-2.2865560055,0.8165867925,-1.3509417772,0.7002499104,0.3727229536,-0.4820721447,0.8639489412,0.8133208156,0.0887968391,-0.3125910759,3.0393307209,1.4080055952,0.2269381732,-0.0300363377,0.7396939397,0.0493804477,-0.4246927202,0.1304106563,0.9637285471,-2.9354915619,-0.3544369638,1.2836763859,-0.8559589386,1.8721460104,0.9731093049,0.5492524505,-1.0334913731,1.3251916170,1.3108726740,0.5195982456,-0.4179671109,0.6543501616,0.4693245590,0.8555210829,0.0848020539,-0.1515882611,-1.7304179668,-1.0883326530,1.0102947950,-0.2386721522,0.8745925426,1.3899519444,0.7876927257,0.6158488393,0.5215525627,1.4409991503,-0.0144396611,2.2355680466,0.4180262387,1.0790933371,-1.4335969687,0.7017084956,-0.0270159505,-0.0620286353,1.0418047905,1.2607593536,-0.8740250468,0.3189888597,0.2929864824,-0.8016648293,1.1566418409,-0.3863493800,-0.6131212711,0.4015072286,-0.3994033933,0.5563696027,-0.3870293200,-0.7484621406,1.9729208946,0.3187820911,-1.0970444679,-0.8086558580,-2.5966191292,0.6420378089,0.6515240073,-1.5672519207,-1.1887092590,0.1352421641,-0.1588215977,-1.1035181284,-0.3870321214,-0.1285940260,-0.0611574203,0.5199727416,-0.6207996607,-0.8858322501,0.2921783328,1.7415742874,1.1214604378,0.4722931385,-0.0375859216,0.9042145610,1.5266809464,0.9645444751,1.1944296360,0.3133407533,0.0609702654,-0.7147462368,-0.9949486852,0.0263745487,-0.9406294823,0.6160652637,2.3014707565,-1.1834588051,-0.8900075555,-0.2471520454,0.4672086537,1.9268618822,-0.6749277711,0.6205279827,-0.2686580122,-0.7099538445,-0.9639384151,0.7790831327,-0.5402551293,-0.0003102511,-0.8768327236,-1.0687489510,-0.4675288796,-0.2242846638,0.8382201195,-0.3525581658,-0.7518466711,0.4557478726,0.2760375738,0.6195541620,0.3318205774,0.8327250481,0.0537931845,-0.3553763926,-2.0381026268,1.6230796576,-1.5148580074,-0.1860478520,-2.0974383354,-0.2627943754,-2.4067049026,-0.4767214060,1.3863887787,-0.7196408510,-0.1204650104,0.7024926543,-0.8471753597,-0.4811016917,-0.2101222873,-1.5753133297,0.0518422872,-0.6725054979,-0.0771305710,0.2476974279,1.1442660093,0.8213924170,-2.0979964733,0.0083712144,-2.8003196716,0.5200743675,1.1690288782,1.6796238422,1.7359300852,-0.3625329137,0.8729770780,0.0395113789,0.4805897474,0.8971570730,1.0508792400,-1.1386889219,0.2386064231,-2.1639914513,0.1895284206,0.7108013630,-0.1431214809,1.1821506023,0.4952346385,0.0679816455,1.9600552320,-2.0621733665,-2.3717496395,-0.0206102040,-1.3767124414,1.6856360435,-0.5215978622,-0.1548906118,1.1633713245,0.1333243102,-0.9365465641,-1.1232398748,-0.1756432503,-1.5008443594,2.0137960911,0.5293657184,0.4739171267,-0.0677911416,2.7306401730,-0.8007552624,0.8647549748,0.1554780453,0.2040397525,-0.1762325615,0.3810936511,-0.2150106728,0.1929219216,1.3094671965,-1.0703794956,-0.6601840854,0.1119203046,-1.0493140221,0.6418649554],[1.0578992367,0.3493487239,0.0387381762,-0.7578765154,1.7252588272,-1.1950592995,-1.0095664263,0.8863735199,-0.9332340360,1.0328786373,-0.3667743504,-0.6750990748,-1.0239139795,0.6232453585,-0.4233544767,-0.6697765589,0.3974470198,0.1741041839,-0.2741079032,-0.0697648525,-0.2788359821,0.0967028365,0.4162475765,0.0730348527,1.6830052137,-1.7050038576,-0.9182111025,-0.0687336549,-0.4993008077,-2.6333050728,1.2867982388,-0.8157199621,1.8138675690,1.3882590532,-0.6029289961,0.0287784822,1.1772009134,-0.2755854428,-1.2099580765,-0.2019699514,1.2501586676,-1.0892593861,2.4252128601,0.8154894710,-1.0352143049,-0.2517317533,-1.3882302046,0.9432877302,0.7587483525,1.0898754597,1.0566821098,-0.8384352922,-0.3188551664,0.1994140446,-0.7795777917,2.1310243607,-1.7256475687,-0.3948467374,-0.7480053306,0.7033433914,0.5914207101,0.0639219061,-0.3472526371,-0.2467501760,0.1587737948,1.3566329479,-0.1416103244,1.3385957479,0.9539729953,0.0049228636,0.9412178993,0.6962132454,0.8853695393,0.0664721727,-2.7985961437,-0.9155045152,-0.0428128801,-0.6666034460,0.9059969783,-0.3675763011,-0.5666050315,-0.6845122576,1.0391764641,-1.1581093073,-0.0269874949,-1.3750796318,0.9732193351,1.2611368895,0.0356923081,0.2246537358,-3.0932531357,2.3984165192,2.2979116440,-1.7785514593,1.3218277693,0.1280211210,2.0275268555,-1.0245759487,0.8668836355,-0.8607371449,-0.7002310753,0.0155158956,-1.4728592634,1.7080414295,-0.7406947017,0.8791487813,-0.4585764706,0.3052116632,0.0468687899,0.5164999366,2.0355801582,0.6521666646,1.8466851711,1.7116943598,0.6319134235,1.3719582558,-1.6670839787,0.4300492704,-0.6846798062,0.9375258684,0.3836639822,-0.0728900358,0.0684805885,0.3087892830,0.6202475429,-0.7178317904,-0.2522747517,1.1459041834,2.0370550156,-0.2767766118,-1.3960233927,-0.9338543415,1.1877619028,-2.6348557472,1.9516296387,-0.1824237257,-2.0092451572,-0.9741954803,0.4679149389,0.9008089304,-0.5722278953,0.6044968963,0.6506953239,-1.6755926609,1.5076848269,-0.5969001055,-1.0222656727,1.9562164545,-1.2689777613,-1.7846150398,0.1661764532,-1.5170683861,-0.7610163093,-0.8992091417,-1.6415470839,1.3204326630,-0.2228368372,0.7749367356,0.4024796188,-1.3226431608,0.2422006428,-0.4162628353,-0.5555692911,0.0719668418,-0.1871663779,0.5951982737,-0.5845536590,-2.1090695858,-0.1772703975,-0.0548733100,0.4509508312,-0.9313350320,-1.7773886919,-0.3563153446,0.2669277191,0.2997245789,-0.4658081532,0.5417457819,-0.2356391251,0.4109375477,1.5283123255,-0.3351726830,-0.3406950831,1.2914118767,1.3758453131,0.8519734144,0.4109028578,-0.3435279429,0.8422434330,-0.0266405381,-1.5175741911,-0.1276775450,-1.5115371943,0.3101183772,-0.6066270471,1.0376282930,-0.0093769059,-0.0544994585,-0.5122949481,-1.2871826887,-1.0043041706,-0.8428524733,0.1637625247,0.1111728325,0.0281228572,1.3473829031,0.0723849237,-0.1692962050,-0.8452322483,0.5937932730,-1.5416975021,0.5983489752,-0.2016807795,-0.4967584610,0.1199091077,0.4734321535,-0.0088558868,1.2080721855,-0.3648279011,-0.6955690980,-0.4260819256,-0.7084145546,0.0491476320,-1.5102821589],[1.0569622517,0.5366889834,-0.0563135706,1.5393899679,1.7595765591,-0.3918105364,0.9197299480,0.3016016781,-0.1586421728,0.0058817188,0.4183733761,1.5566524267,-0.6086767316,-0.5249670744,-0.8945267797,1.9990261793,1.0346226692,0.9244108796,1.8088890314,0.4348047376,-0.7111964226,-1.3284702301,-1.1754543781,0.3776286244,0.4044982493,1.7677334547,-1.0616512299,2.2402262688,-0.1550719440,0.1358435005,-0.1329102516,-0.7924153805,-1.3541342020,-0.0541118197,0.7407044172,0.1474341452,1.3359956741,0.7409295440,-1.4758354425,0.0812779739,1.5620260239,0.1671449095,0.3341617882,1.3199590445,-1.4782437086,-0.5552400947,-0.2017293274,-0.4659951627,0.2432397157,-0.6974768043,-1.2883409262,-0.7465677857,-1.1168094873,0.4712827206,1.6376899481,0.6912307143,-1.3686044216,0.1189413965,-1.4354121685,-0.1401813477,-1.1164814234,0.3037294745,-0.3992435634,0.2656225264,0.3130260110,-0.8659200072,-0.4726854563,-0.0152176367,-0.0930923000,0.3841016293,0.9208854437,-1.3471815586,-0.2411450744,1.6320923567,-0.6548956037,-0.4167873561,-1.5803978443,-0.1135653034,-1.2573654652,-2.0530848503,0.9897403121,-0.4235695899,1.5027476549,-0.6127468348,0.2498405725,-0.1621945947,0.5481315255,-1.0397114754,1.0767316818,0.6571477652,0.8535944223,0.3218203485,1.0353807211,0.2963234782,-1.5258238316,1.6403651237,0.4474198818,0.5159975290,0.0112749394,-0.4295137823,-0.0053783916,0.8010345101,0.0232382063,-0.5535251498,-0.7191053629,-1.7811300755,-0.3628551364,0.5310444832,2.4867811203,1.9475063086,0.4156304598,0.6378083229,-1.8176249266,-0.4333888888,1.0649061203,-0.7765557766,-1.1090966463,0.0572323352,0.2367252856,-0.1627569199,-0.0342329107,1.1516398191,0.0576944612,2.5441424847,0.4243032932,0.6389544606,-0.7369576693,-0.4054327011,-0.1950183809,1.2195286751,-0.7707237601,0.3494344354,-0.8499855399,-0.4434051812,-0.8354090452,0.6398029327,0.0468047112,-0.0552285872,0.5982164145,1.0310800076,1.2464448214,1.5467491150,1.9671247005,-0.7465594411,-1.2460572720,-2.2472889423,0.7259353995,0.6860581040,-0.9589980245,0.7132995129,-0.4153469801,-0.3921484649,1.2021461725,-0.0087545915,0.3043210506,0.5953631401,0.1601187736,-0.7451269031,0.2441005409,0.3973066509,0.2535099089,-1.4309951067,-1.6279504299,-1.1323072910,-1.2013375759,1.3001255989,1.8920334578,-1.2963359356,-0.4052232206,-1.9754989147,0.7988344431,0.4170355797,-0.4807196558,-2.0655698776,-0.7232215405,-0.8220478296,-0.4829567671,0.0755685493,0.6312243342,0.5088500381,-2.4266726971,-0.6307898164,0.1054759845,-0.3693656921,1.7723187208,0.6556220055,-1.3453546762,-0.6720908880,1.1442575455,-1.4109114408,1.2649291754,-0.8008374572,0.0384092443,-0.2519807816,-0.2426828742,1.2038187981,0.4667410553,-1.0764840841,1.2552109957,1.3426625729,1.5059294701,-0.9154779315,-0.0498299412,2.0281126499,0.6835604310,-1.8105126619,0.4737175107,-0.8432004452,2.0750002861,0.2878215015,-1.1522181034,2.0514914989,-0.7687619328,-0.6637822986,0.6724092364,0.6839038134,0.0562897585,-0.3016141951,0.4957450032,1.0637838840,-0.6770959496,0.2365283668,-0.2895208299,-0.8617882729],[-0.9594106674,-0.0157391503,0.8399378061,0.8110836148,0.4611687660,-0.7466703057,-0.0806623921,-0.9150440097,-0.9413416386,0.3335480094,-0.3684301674,0.4436233044,1.0678399801,0.5630221963,1.3437930346,-1.7708089352,0.3103902936,0.6368618011,-0.4932948351,-0.2312989533,-0.3020412326,0.0923235640,1.3628299236,0.7628677487,0.7790932655,1.5329159498,0.2183266133,-1.2446256876,1.0062347651,-0.0709650144,-0.7959281206,1.2688463926,-0.8175793290,-1.0273636580,-0.6656317711,1.1612483263,-0.5840453506,1.4325695038,-0.8769946098,-0.5628446341,-0.2621915340,0.6548326015,0.0389738902,1.7369393110,2.3276939392,0.2912586033,-0.1409829259,0.8625109196,-1.4595911503,0.2316573858,0.9324889183,-0.6206426024,0.5359648466,-0.3008208573,0.0612839088,-1.7155638933,0.9352427721,0.1170468032,1.0926513672,-1.5903171301,1.6055085659,-0.1270006150,2.5916931629,0.3588993847,-0.4123798013,-1.4389734268,-0.4076504111,-0.3596978486,1.8169152737,0.3336016238,2.4945390224,-0.6098358631,-0.3644248843,0.8882802129,0.4999442995,0.5015850663,0.6149253845,1.1175642014,1.8079290390,0.2978225946,0.6117570400,1.7117923498,0.4619747698,-0.2899212837,0.7065873146,0.1721730530,1.0118405819,0.7275610566,-0.6780998707,-0.1265953183,0.9778655767,-1.3090860844,-0.4225935042,-0.1998132616,0.0992397964,0.9909891486,-0.8389635086,-0.2723429799,1.9301704168,0.9522368312,-0.8642147779,0.6867908239,1.2381422520,0.8961113095,1.7952609062,-0.2819908857,0.7153847218,0.0273373686,-1.3802032471,-2.0942184925,1.0688688755,0.2658931017,0.9404011965,-0.0069303596,0.5577315092,0.2194401324,0.5258272290,0.5876083970,-1.1635379791,-0.5140573978,-1.4443894625,-2.1708698273,-0.1201778799,-0.6402384043,0.2971675098,0.1304511130,0.1133562103,0.6318150163,0.1640680581,1.5028864145,-1.7749533653,-0.7292850018,0.1292218417,0.1633696705,0.5901848078,0.9318404198,0.0090533029,0.5410732627,0.6497133374,0.0382000655,0.2489194572,-2.0864832401,-0.4515062571,-1.2362923622,-0.2087910324,-1.2355883121,0.5569347739,1.3323907852,0.5438510180,-0.1150219962,0.0459278598,0.4252981246,-1.4181237221,-1.7148958445,0.6399292350,1.3488681316,-0.5249696970,-0.7036726475,0.3729901612,1.3503198624,-0.6562246680,-1.8335790634,-0.7283580303,0.4336759746,-0.0378222503,-0.2591528594,0.6934357285,0.2126318961,-0.2649644017,-0.5409034491,-0.1988546699,-0.0906299800,0.1306391060,-0.6612063646,0.3144090772,-2.1315474510,0.5624923706,0.6593207717,-0.6201909781,-0.5569365025,0.3504077792,-1.3898166418,-0.3267312050,-1.5478287935,-1.0569232702,0.4089502990,1.0574078560,0.3555086255,1.3506741524,0.7100051045,-2.4160699844,0.2878858447,0.1611437201,-0.5060824156,-0.5658532381,0.0795270875,-0.5061843991,0.2112802863,0.1915567666,-0.1671071202,-0.4966573715,-0.3014395535,1.4349435568,-0.0081969770,0.8687512875,1.7044713497,0.7172669172,0.1723873615,0.7683315873,0.8068082333,0.7374470830,-2.3381924629,1.4708920717,-0.6122639179,1.1907120943,1.3497942686,-1.3654749393,-0.3542251587,0.6783241034,0.7457907200,-0.1441514641,1.7073394060,0.1043201610,0.4953304827],[0.4898969233,-0.7908129692,0.3112324178,1.2854849100,-1.1699900627,0.0307697803,-0.0978933945,0.2470248342,1.5309244394,0.0981095359,1.4879208803,0.6778614521,0.1257022619,-0.6889806986,-1.7376300097,1.6067085266,0.2240911871,0.3167164326,-1.0415329933,0.6432635784,-0.1780188829,0.2714662552,0.5061727762,-0.0400048196,0.6168119907,1.1573162079,-0.6214919686,-0.0559455715,0.2033772618,-0.0524638593,0.4651843607,0.7974411249,0.9521196485,-0.8156989813,-0.7175847292,0.0430429354,-1.1026031971,0.7501711249,-0.0570429079,0.2406162471,-0.1664299071,-0.4878765345,-0.8518081903,1.0820053816,0.2024823874,-0.6305008531,0.5867401361,-0.1111482680,0.9543295503,-1.5515456200,-1.3526494503,-1.2814865112,-1.2083516121,0.6300700307,1.2377527952,0.1157994419,-0.1324049979,0.8532496691,-0.2272783518,-1.9050981998,-1.4512780905,0.5527977347,1.9561060667,-0.5117322803,0.2994587719,-0.8818569779,0.7253333926,-0.2888867855,-0.0488190539,0.6111413240,0.8727059960,1.0785720348,0.6621670127,1.1721650362,-0.5383167863,0.7041527033,0.8776189685,-0.1341997683,0.5267609954,-2.2195818424,-0.0641778633,-1.4765670300,-1.6097735167,-0.2215120941,1.0128235817,0.3994425237,-0.6306587458,0.8556500673,0.5664199591,-2.7598052025,-0.5542398691,0.9546697140,0.2972911000,-0.5923198462,0.8346678019,-0.5135295391,-0.2254577875,0.3647433221,-1.1742988825,-0.0221651979,0.2011860758,-1.1618353128,0.3587828279,-0.3569490016,-0.6757637858,0.1543741524,0.5523005724,1.1413083076,0.9759478569,1.3029022217,1.5852334499,0.5111664534,0.9931082726,0.9090556502,-0.8942973018,-0.8136970401,-0.3231808841,-2.9619100094,-1.7260572910,0.6305112243,0.9046978951,-0.6565795541,1.1913491488,-0.6926817298,0.8507476449,-1.4126199484,1.5484261513,-0.3532052040,-0.1554211229,-0.5210480094,0.5052769184,-0.5785877705,-1.8058321476,0.4119521677,1.2402284145,0.5937663317,0.9732915163,-0.3283392489,0.1560894847,0.2356980294,0.0013753889,0.4547128677,0.2108037323,2.2417321205,-1.5689145327,1.9793784618,0.6024902463,0.1141058430,-1.3041629791,0.5927038789,-1.4569309950,2.2926297188,-0.7402815223,-0.0437542982,2.4251124859,1.0239573717,-0.9977756739,-0.7223476768,-0.6301113963,1.4681988955,1.1038920879,0.2033595741,-0.6396667361,-0.1190782934,-0.0005420816,-0.1301973015,1.5639034510,0.0902540758,-0.1481139511,0.6896650195,-0.5098571777,-0.3564096689,1.5234655142,0.8508958220,0.8980773091,0.5878921747,0.0211714022,0.1310488582,-1.7261478901,0.4319587052,3.3127100468,1.2555725574,-2.4129700661,0.1548143625,-1.3937963247,-0.0669993684,-0.4945868850,0.6425649524,0.4099414349,-1.2230517864,1.1344254017,0.2476155311,-0.0826025009,-1.7094839811,-1.3960864544,0.0779310912,1.0671143532,-0.7367090583,-0.8694255948,-2.1651918888,0.2056596577,-0.6978781819,-1.3931280375,1.1689006090,-1.0735945702,-0.1320720315,0.3157189786,-1.1467095613,-1.8069293499,0.3965733051,0.6804736853,1.1677420139,0.1507781297,1.5749038458,-1.6790566444,-0.4240026474,-0.0486205667,-0.4190273285,0.0764597952,0.2051582783,-0.1262133867,-0.0686233491,0.9195166826,0.2387665063],[-0.1942841411,-0.9263069034,2.2984178066,1.1526581049,0.8616171479,0.3174169362,-1.7027870417,-1.7472238541,1.0208576918,-0.3556275070,-0.1951083988,0.7903609872,1.3327525854,0.8751086593,0.2799882591,0.1954333037,-0.3937276900,-1.7281918526,0.0852415413,0.9325007200,-0.1659661531,0.7091153860,-0.4390468597,0.0979182422,-1.1979483366,0.4096597433,0.1257333606,-0.6742489338,-1.5028485060,-0.7201532125,1.8562434912,0.0316390693,-0.5786090493,0.8689778447,-1.4990413189,1.0552029610,-1.6764425039,-1.5236849785,-1.5653573275,0.4090256691,0.1788931787,-0.1072124392,-0.7379006743,-0.0220090747,0.2177771926,0.6614961028,0.3497148156,0.2820779681,-2.2396645546,2.3719971180,-0.4914728701,0.3381384015,-0.9512355924,-0.4113211334,-0.1319223195,-0.7636014223,-0.0705542788,0.0945843682,0.5141342282,-0.2185685933,-0.5152382851,0.0872560367,1.3440091610,0.6572620273,0.4007388651,-0.5317484140,0.7063996792,0.9795297980,0.9454191923,0.4927286804,0.2224742323,0.0633541867,0.1593330353,-0.7465167642,1.2386281490,0.2575991154,-0.4987689257,0.2045920640,0.6867196560,1.4968909025,-0.3125460148,-0.9242553115,-1.4892401695,2.4101431370,-0.0921221748,-0.9982684851,-0.1168525666,2.1869966984,-1.4745687246,0.5221762657,0.1889460534,-0.8488870859,-0.9581833482,0.9912944436,-0.4461146593,-1.0186274052,0.8116633892,-0.7028707862,-0.2395506054,-1.3628484011,0.6342271566,0.4812361300,-0.8224328756,0.1313161552,1.5504002571,-0.9111961126,0.2188706696,1.0305371284,-0.0790315494,-0.9508096576,1.2417500019,0.4015571475,0.2897460163,0.1677240729,-1.4348064661,-0.5332908034,1.5793312788,-0.1691692472,0.1776634902,-0.2642582059,-0.5794829726,0.8309162855,-1.6052863598,-0.9368858337,0.9558969140,-0.0408717506,0.3745765686,0.6062545776,0.6897696257,0.6617829800,0.6259799600,-2.4486322403,0.3745160699,1.1456580162,1.0062994957,0.3313815892,1.0699118376,-0.6684012413,0.5727875233,0.6733343601,-0.0626238659,-0.4325711727,-1.7898069620,-0.7044024467,0.2529786825,-0.1920649707,0.3323691189,-1.1825512648,0.5753129721,-1.4658842087,-2.1032445431,1.1631325483,-0.5351141095,1.3923389912,0.8280168176,-0.7771325707,-1.1401008368,-1.7066495419,-0.6760823131,0.4116258919,-0.6975342631,0.6350879073,1.9184994698,-1.0803020000,0.3992839456,1.0008097887,0.1050618738,0.6947386265,0.9541548491,-0.4823937416,0.0798426941,2.0793166161,-1.0948320627,-0.7919138074,-0.3036155403,0.2675095499,-0.3363015354,1.2337770462,-0.0565242618,1.1298168898,0.2538796067,-1.4344059229,0.7543268800,-0.4848037958,0.1588084549,-1.9065899849,0.7605556846,-0.9614201188,-1.6162747145,-1.3088313341,0.7706814408,-1.2203983068,1.0250405073,0.1936548799,0.6911200881,-0.9923204184,-0.7455840111,0.4099641144,0.4822587371,1.1336737871,-0.3551127613,0.3467658460,-0.2646628320,2.0391478539,0.3230980039,0.8732517958,0.3943471909,-0.7295206785,0.6893681884,0.6827322841,2.2047653198,0.2522270083,1.0462615490,0.3333738148,0.9523355961,-0.5945228338,0.4556433558,0.6373681426,0.7313504815,1.3079185486,2.0483045578,0.9635858536,-0.7311703563,0.8531662822],[0.1186441779,0.2476326227,0.7424806952,-0.1205264479,2.3297095299,0.2212003767,0.0787303224,0.4282938540,-0.7846913934,0.7370166183,0.1164571047,0.4009823501,-1.5018119812,-0.7576122284,0.2048009932,0.4468692541,-0.7943142056,0.9658300877,-0.0924994797,-0.5092731118,0.1271789968,-0.5681103468,1.2777352333,1.3446389437,-0.4176205099,1.1254996061,-0.1987000555,0.3697093427,0.6449429393,2.1140766144,0.0287610460,-0.2087999135,-2.2224323750,-0.1076841950,-0.5155770779,0.3130055368,-1.2204751968,-1.6488747597,-0.7013992667,0.9725614786,1.8311501741,-0.0420313030,-0.4865702391,-1.2558275461,0.0240110978,0.0409290753,0.5420319438,-1.2074099779,-1.4793547392,0.0677488744,2.0287189484,0.1211663187,-1.0394985676,-0.2092629373,-0.5419185162,-0.2526287735,0.6307049990,0.7881709933,-1.3417301178,0.5147671700,-0.1236193851,0.3639915884,0.0800566152,-1.4012484550,-1.3367093801,-0.9948487282,-0.2640236318,0.0116868606,-0.1084337905,-0.8910443783,-0.2605855167,0.7310494781,0.2275884598,-0.4247497022,-0.3506610692,0.5176494718,0.8392418623,-0.1413768381,0.4017984271,-0.8027892709,0.0070667868,-0.1382342279,1.1400476694,-1.2793195248,0.9133566022,-0.4161378741,-1.3781870604,0.1653951555,0.4276550412,0.1956777126,2.8762161732,-1.1102243662,0.1171223372,-1.6822745800,-0.9032545090,1.2693028450,-2.2613315582,-0.8250658512,-0.2805194855,-1.4849926233,-1.2322430611,2.0261244774,0.0272019934,-1.8280910254,-1.6233180761,-0.1065211222,-0.2862212360,-0.4451537132,-0.2557170689,0.4339169562,-0.9887716174,1.2868506908,1.1453354359,0.8280850053,0.1992067993,0.2173833102,-1.7708069086,-0.2707109451,0.7886084914,0.0197867677,1.5219205618,0.4495322108,-1.8151134253,0.1503618360,-0.8707200885,0.1044267043,-0.1727676690,-0.3222731054,2.0104050636,-0.2804686129,-0.0797644407,2.4352912903,-0.3499047756,-1.8723024130,-1.0576988459,1.5366865396,0.2987216115,-0.5741385818,0.7206164002,-0.5140460730,1.1396903992,0.4905533195,-0.7538064122,-0.0299488585,0.6763259768,1.1116752625,0.0127552878,1.0710752010,-0.6004653573,0.2281350344,-0.9140645266,-0.3691907525,-0.3375107050,-0.8282238841,0.7861650586,0.3188356161,-1.5241420269,-0.7093237042,0.6100569367,-0.1316596866,0.3417309523,-0.4438876510,-0.8978509307,-1.0393338203,-0.8868865967,1.9541177750,1.0689802170,0.0547712296,0.5502784848,0.0277117081,0.6421114206,-0.6505494118,0.6176762581,-0.9987822175,-0.1520087272,0.1977556646,1.4567295313,0.9476689100,0.3120063245,0.1578296274,1.3490449190,0.9076947570,-1.5802649260,0.0284059606,0.3208372891,0.6578550339,1.2572970390,-1.0926563740,1.2521040440,-0.6177709699,-1.0928300619,1.1918451786,-1.0383467674,0.2914623618,0.5544426441,-1.9991108179,-0.6491318941,-0.1269466877,-1.3599926233,0.2060403973,1.1349284649,0.5194821358,0.8900290132,-0.7164104581,0.5118864179,0.2188488990,-0.6501572728,-1.0021990538,0.9629164338,-0.6489469409,0.1203286722,-0.3681460321,0.0381681658,-1.0675135851,0.2147252262,-1.4677051306,0.9678803086,0.5567008257,-0.4400278926,0.5860223770,0.7764663100,0.7028333545,-0.2341640592,-0.5149731636],[0.7935917974,-0.5220533013,0.6459440589,-0.2909165323,0.6664276123,0.5232625008,0.9840770960,0.6887192130,-1.1589708328,-0.1337791681,2.1475710869,-1.1999230385,-0.0397767313,0.2972322702,-1.1360603571,-1.0197287798,0.0860073194,-1.0629270077,0.7163521051,1.6431040764,0.8835981488,-0.8887983561,-0.4056135714,0.5577527881,-0.5359670520,-0.6085321307,-1.0848847628,2.3708302975,-0.3000078201,0.9129256010,-0.8528723717,0.7104384899,-1.4673618078,-0.0468426906,1.5498622656,-0.6497247219,0.1055054367,-0.6367157102,-0.7790260911,0.1303004473,-0.8807298541,1.2931112051,-0.0242201276,-1.3028495312,1.0931696892,0.8489127755,0.5620557070,0.3824717402,-0.8211005926,-0.2894999683,-0.3295023143,1.9161242247,-0.3875193894,-1.5950083733,-1.2387349606,0.9860461354,1.1215429306,-1.4169552326,0.7774617076,1.3781566620,0.4257713854,0.1041262969,0.6187925935,-0.1452303976,-0.1544398069,-0.3350002766,-0.7903438210,2.2526700497,0.5864151716,-0.0685864314,-0.6246374249,-1.3993204832,-0.4303893149,-1.2822613716,0.1559318602,1.5798255205,-1.0765495300,0.4107350409,1.3614157438,-0.7937434912,-0.1587989628,-0.0826069266,0.1638277918,0.3035634458,-1.0530942678,0.3941660225,-0.3778450787,0.5913537741,1.7228548527,-0.2308357060,-1.0750149488,1.7766871452,-0.2589304149,-0.6578678489,-1.3033595085,1.0156477690,-2.2127144337,-0.7063292265,-1.1299027205,-0.8751794100,0.0943318456,-0.8357192278,-0.6747906208,-0.1819349229,1.2726923227,1.1324168444,-0.0653668642,1.7812215090,1.4365423918,1.2911921740,0.6556486487,-1.8097792864,-0.4941692948,1.0009518862,0.9566744566,0.1753100008,0.7363275886,1.0797368288,1.5626415014,1.5590084791,1.1770696640,1.8266727924,-0.8828266859,-0.5780462623,1.1614030600,0.0819185227,1.1126433611,0.2359033823,1.5360554457,-0.7368761301,1.1381391287,-1.1523531675,0.4186245203,0.4560465217,-1.5258060694,-1.1486253738,0.0944953263,1.1487392187,0.4432187080,0.4212090075,0.1783284396,-1.0721273422,-0.6282792091,0.1184149534,0.1077746376,1.6965463161,-0.0277585164,-0.1918672770,0.5346242189,1.8567492962,0.9038385153,-1.7754197121,0.0008006179,1.3203227520,-0.9212475419,-0.1841087192,-0.1914251149,-0.2672573626,-0.7262297869,0.5493069887,-0.0501594692,0.4068301320,-0.4298945665,-2.2442214489,-0.7497539520,2.2230682373,0.1048255190,-0.7576667070,-0.2962541580,-1.4607772827,0.6312759519,0.2765136063,-1.7077609301,-0.3536328375,2.0978047848,-1.0209709406,-1.9278744459,-1.0835400820,-0.6693260074,-1.7807040215,-0.0103057222,-1.2683321238,-1.4392585754,0.5706706643,0.2299688011,-0.8338006735,0.0692932904,-0.0159100499,-1.2371246815,1.5750194788,-0.4039532244,0.0204617586,-0.3619193733,0.6404605508,0.3293938637,1.2330603600,1.0708794594,-0.2778260112,-1.0820939541,-1.1345940828,-0.9574249983,0.1622495651,1.4215382338,0.4582139850,0.4866745770,-1.1338460445,2.6203932762,0.3832954764,1.0140392780,-0.9759644270,-0.1634153575,-0.7954094410,-0.3030864894,-0.3183051348,0.4918797612,0.8558522463,0.3166932762,0.5860859156,-0.6056419611,0.1423945576,1.3445808887,3.1942925453,0.3613812327,0.7262311578],[-0.0039374535,-2.8172578812,1.2778655291,-0.5762477517,0.8407689333,-0.7480339408,-0.7672458291,-0.1195229515,0.2324158102,-1.8714499474,0.5519493222,-0.5039342046,-1.7590987682,1.6869199276,-1.9298254251,-1.3470934629,-0.6425828338,2.0217587948,0.6767479181,0.3098316789,-1.4022588730,0.6783840060,0.8065295219,-1.1054807901,0.5464166999,-0.3851571679,0.3955411017,0.3208754361,1.5504038334,-0.2144103050,-1.5548576117,-0.2614115775,-1.4425181150,-0.2540749609,0.3945147991,-0.7667369843,1.4624339342,-0.0706093833,0.2780528069,0.8718714118,-1.7170612812,2.0644669533,1.3981959820,-0.0241681747,0.0153257139,0.1551327109,0.8673528433,0.4098148942,-0.4804344475,0.4392388463,-0.6017239690,0.0909047872,-0.6866038442,-0.8969714642,0.9651571512,-0.6932944655,0.6336821318,0.8397692442,2.6203660965,-0.6895518303,-1.7960164547,-1.6501295567,1.8551411629,0.1128157750,-1.7513470650,0.8404824138,1.3223236799,-0.5824505091,-0.0671811923,-1.2150722742,0.9815346003,-0.3406918347,-2.1729745865,1.5029435158,1.4895166159,-1.0849428177,0.0340151675,0.3322142959,-0.5848049521,0.1907823086,-0.8129360080,-0.0226928703,-0.0279446561,-0.2016942650,-1.1095571518,0.8078178167,-1.1353489161,0.2967233658,0.5234735012,-0.7270941138,1.1792708635,0.6389978528,0.3985453844,-1.2015575171,0.7863968611,1.1112177372,-0.5595420599,-0.5727396607,-0.8466938138,-1.9493989944,-1.0869103670,-1.0681582689,0.3008961976,-0.5245221853,-1.5388499498,0.6930798888,0.5540941954,0.9507279992,0.8663878441,-0.4860620499,-0.1471722573,-0.1502821445,-0.4803615212,0.6171793938,0.5383598208,1.2860487700,-0.1998111606,2.9812784195,2.1534991264,0.6020771861,-0.2183421254,0.5264843106,0.2851968110,0.4971400797,-0.9605002403,0.9666528702,-0.0700143129,-1.3566243649,0.3706783354,0.4595982730,-2.5560221672,0.3511317670,-0.6403661966,-1.2884974480,-0.6789479256,-0.6717274189,-1.2106477022,-0.0163370986,0.3457172811,1.6352626085,-0.4889360368,0.5581252575,0.3349420130,-0.1122701392,-0.9621224999,0.8743726611,0.5053303838,-1.8802504539,1.1275217533,-1.1256831884,1.3232104778,0.3116884828,0.9008846283,1.4898868799,0.0042057778,0.1580962092,-0.9577461481,0.0220026318,-0.4892366827,0.1239973605,1.1565297842,-0.4127237499,-0.1431506574,0.0075374050,-0.2322238684,-0.3758442998,0.4405148923,-0.2959225178,-1.9549883604,0.0191233903,-1.1686453819,0.3244661689,-0.7563298345,-0.9059374928,0.2940572798,1.7263318300,-1.4125913382,0.8717287779,0.8929125071,1.0991194248,0.7424063683,-1.1658024788,0.1678114682,-1.3147382736,1.0513582230,0.8278679848,0.9144086838,-0.2248248011,-0.6977533102,0.7616482973,0.1096616462,1.9541888237,0.2071506679,-0.0072634853,1.6672128439,-0.2180464268,0.2142009735,0.6916992664,0.8198504448,0.2656022012,0.4191500843,-0.1563543826,-0.0766619965,-2.0684545040,-0.5974961519,-0.3178389370,0.9397915006,-0.2236629725,-0.6539345980,-0.1499699354,1.3334772587,-0.2018540204,-0.8706596494,1.8530994654,-1.2432664633,0.0694535226,-0.6399093866,0.5563447475,1.1774454117,0.3720839918,0.0705601349,-1.5265114307,-1.3808004856,-0.7226364613],[-0.0671254769,-0.7904212475,0.1652627289,1.6826713085,0.9410175085,-0.1707097888,-1.1935794353,0.5031608939,0.7144062519,0.3102995157,0.0893526301,0.4119528830,-0.6044661403,0.7267200351,-0.5432944894,-0.4802464545,-1.2587251663,-1.6022361517,-1.0864841938,-1.4701102972,-1.3144232035,0.9932833910,-0.4609676600,0.4559967816,0.9519225955,1.9400188923,-0.9149392843,-0.5548046827,-0.0177294016,-0.7559466958,0.4510883987,-0.0255159680,-1.1708710194,-0.7565687299,1.3700623512,-0.8828909993,1.1024614573,-0.4919537604,-0.8594636917,-0.1812487990,-0.5608417988,0.7042586207,1.0489093065,-0.0738557726,-0.3878282309,0.0897080898,0.6622738838,0.3745981753,-0.2529806495,-0.2844143212,-0.0747609586,0.3734287322,-0.3683740795,-0.9679385424,-1.5396726131,0.1518522203,1.1448954344,-1.1830432415,2.3214013577,-0.2190132141,-1.0381678343,0.4893311262,1.2217268944,0.4482461214,-0.7320929766,-0.0737974867,-0.4775159955,-0.2962387800,0.9918540716,-0.2520774901,0.7377212644,-1.1158239841,-0.3469292819,0.7313711047,0.6559993625,-0.9429005980,-0.9391338825,0.8579667807,0.0583344996,-0.0099074114,1.0966665745,0.1336360723,-0.1326661706,-0.9921869636,0.5340189338,-1.0867534876,-0.4550673962,1.3559694290,-0.1116294712,1.0091249943,0.1412471086,0.5255767107,-0.3881579638,1.3969186544,-0.1676418036,-0.1452837437,0.9945383072,-0.2506922185,1.1986724138,-1.7370616198,-1.3467944860,0.8549793363,0.2784049809,-0.6807608008,-0.6913851500,-2.2432849407,-2.3831932545,0.7839045525,0.0188699197,1.0645942688,0.1286784858,0.4382385910,-0.6687325239,0.5978592634,1.1087765694,-0.1641141921,-0.3620709479,-1.7388533354,1.3200438023,-0.2533108592,-0.4184384346,-1.5299417973,0.5822222233,-0.5063092709,-0.1981868595,-1.3021932840,1.2743741274,0.0432033651,0.4569822550,-0.5047770739,0.1699194163,-1.3279764652,0.3524358571,0.1321837306,0.5692359209,0.6053616405,0.2480343133,1.6470834017,0.3169894814,-0.5630028248,-2.4546029568,-0.3362979591,-1.3944190741,0.7804767489,-1.0859674215,0.4189290106,0.8998404145,-0.6749925017,-0.7698157430,-2.2536473274,-2.3058500290,1.2095606327,0.1959027052,0.3622468412,0.0256938580,-0.9933300614,0.4945476353,0.9161879420,1.4648804665,-1.8006176949,0.7049255967,0.8880855441,0.4263364673,0.2365619391,-0.3710543811,-0.7134770155,-1.3234868050,-0.0384293981,-0.1907331049,0.2600144148,-0.8479497433,0.5255256295,-0.5529755354,0.1325170100,0.2094378769,0.3922344744,-0.2805377841,1.1552339792,0.0220841095,0.7056201100,-0.8617077470,0.5466896892,0.8280608654,1.7718749046,0.5791893601,0.3954809308,0.6250145435,1.8468208313,-0.5212256312,-0.7269707918,-0.0188041087,0.2731238902,-1.7006938457,1.2754198313,0.4935974479,-1.1783540249,0.6881445646,-1.0047477484,0.8587782979,-0.4211801887,-1.1423093081,1.8459129333,0.4986789227,0.3107667565,2.1952350140,-0.2329679281,2.2344813347,-2.3799045086,-0.6172450185,0.9939242005,1.6428675652,-3.8169021606,0.8213685155,-0.2383841127,0.8085193038,0.8943603039,-0.7287612557,-0.8341719508,-1.6358424425,-0.0163045768,-0.3030619025,0.0600189455,1.0085133314,-0.7476360202],[-0.5483891964,-0.9263272285,2.6085176468,-1.2780238390,-0.9766176939,-0.5389821529,1.3002591133,0.9012833238,1.6520648003,1.8740491867,2.7003669739,1.3751206398,0.7383854389,0.4513852000,-0.0785458460,0.9784024954,-0.9420097470,1.0711314678,0.1228365153,-0.4089529514,-0.0938136727,-0.4688596725,-0.0395304710,-1.4214097261,1.9610992670,-0.8136855960,1.5312091112,0.7010276914,-1.3808984756,-0.2077739984,-1.0954723358,-1.5649642944,1.6708961725,0.0262108147,0.4416314065,-1.0797450542,-0.4252988994,-1.1980069876,-1.4782294035,-0.9501484632,0.0933405459,0.8175611496,0.4934800565,-0.0887110531,-0.2513252497,0.2430290729,0.0722308606,1.1764454842,-0.5189903378,-0.4726536572,2.1911048889,-0.4614456296,-0.1552190930,0.6430007815,2.3364167213,0.8803315759,1.2821724415,-0.1783988476,0.6398929954,-0.6916404963,0.0685556680,0.4741430879,1.7757326365,1.3293614388,-1.6313576698,-1.1642336845,0.3845895231,1.0874531269,0.8030692339,0.9298844934,1.1026318073,-0.4465782642,-0.9066677690,1.6973699331,0.3255869448,0.4437576234,0.6298837066,-0.1388432831,0.4877704084,-0.5768527985,-0.6939834356,-0.6422843933,3.4686851501,-0.8099491000,1.0799697638,-1.1004024744,0.4231131375,0.1130894870,-1.2025954723,-0.4120817780,-0.9697688222,-0.8534077406,-0.5020948052,1.3400472403,1.7938925028,0.6689025164,0.9888962507,0.2849170566,-0.0868029594,-0.2804565132,0.2670521736,1.1490031481,0.2609162629,1.0914986134,-0.1469503343,-0.1726162434,-1.0124996901,1.6172940731,-1.4287543297,0.0207366310,0.4133826494,-2.1272759438,-1.0434956551,0.3534581065,1.8912388086,-2.0706202984,0.7521951199,0.1733450145,0.3660833538,0.4582214057,-0.0752727687,1.0993863344,0.7004218102,-0.2836722732,-0.3275241554,-0.9340184927,1.6282463074,0.6356199384,0.0225061178,-0.3679436743,-0.3817060590,0.6974017620,0.0895482525,0.4078374803,-1.0503004789,-1.8791811466,-1.1724802256,1.0532171726,-0.0058436240,0.2517630458,-0.7201429009,1.2702769041,-0.0458424352,0.4193159938,1.5528337955,-1.1905040741,-1.3259557486,-0.5397362113,0.7404883504,-0.9480809569,0.7349120975,0.2989366651,1.1667926311,-0.0275125522,-1.9530737400,-1.0072118044,0.5982735753,-0.7819896340,0.2409633547,1.0072343349,0.5993404388,0.1452574730,0.4981725514,0.4348387420,1.0504794121,0.9790147543,0.6895599365,0.1251991391,-0.8385702968,0.8198758364,0.3724953830,0.5099990368,-0.4112640321,0.1107642874,0.3995991051,-0.3792087734,-0.1735995412,0.0200130008,-1.1897200346,-0.0422304757,-1.0703550577,-0.0569381416,-0.0801360905,-0.8206369281,0.7945466042,1.3026648760,1.1943631172,0.4166095257,-2.3133356571,-2.0431711674,-0.0859138668,-0.7850488424,0.4934567809,-0.0740535557,0.5806821585,1.1419591904,-0.5928174853,0.3389694393,0.9335060120,1.5325789452,0.2682217956,0.6445912719,-1.1811283827,-1.3357715607,-0.2605241239,-0.1114650816,-1.0182495117,-0.1397437006,0.4789614975,-1.3894425631,-0.8259648085,-0.9689841270,0.7458997965,0.2991378307,0.8106826544,-0.3790820837,0.7313722372,0.6382530332,2.0013265610,0.2483182549,-2.0251369476,0.1511827558,-1.8691866398,0.3484821618],[-1.3629926443,0.0344463214,0.2344449162,0.4313440025,0.2729780376,-1.5452675819,-0.0550522581,-0.6612935066,0.5417645574,2.0425043106,0.2200262994,-0.1124477014,-0.3067903817,2.6385672092,0.0758075565,0.9185811877,-1.1661971807,0.0555665419,-2.0132887363,-0.0612099841,-0.0498628020,1.1586894989,-0.4992930293,-1.0344942808,-2.0566396713,-1.7438385487,0.4182908833,1.9617835283,0.3961114585,1.9309381247,-0.4613923132,0.1689138561,-0.1322667748,-1.2334487438,0.9377317429,0.7998334765,-0.6520880461,-0.1707964838,-0.5642157793,1.2804228067,0.0208851863,-0.5527569056,-0.4434795380,0.0876937509,1.7762702703,0.8643834591,0.7251709104,-0.2246964127,-1.1765624285,-0.8831675053,-1.3195488453,-1.7548000813,-0.6290677786,0.4152307808,0.5666602850,-0.2163264304,-1.6369053125,1.3941433430,-1.7514255047,-1.9723445177,-0.7031470537,-0.9993395805,-0.8704144359,-1.2276570797,2.4672963619,1.1786454916,0.2572571933,0.2190767229,0.6950595975,0.7149975300,-0.0045571011,0.5289281011,-0.6817349195,0.7423099875,0.0222218968,-0.3022816181,-2.5626525879,-0.8660374880,0.5463693738,0.1478862166,0.4148804545,-0.2576435804,-1.3388051987,-0.9884957075,-0.1858710945,1.3034764528,1.2491785288,-0.2166267633,0.3619436622,-0.5870447755,0.4298285246,0.4556368589,1.4775171280,-1.2154983282,0.4434569776,1.2889420986,-0.1644225568,1.5875742435,-1.2383381128,0.1369165182,0.8227247000,-0.2720991671,1.3772487640,-1.0866830349,0.5779238343,-1.3674609661,0.3796960711,-1.2340633869,1.5236074924,0.8480984569,0.8815364242,-0.2750021517,1.0931401253,0.3164792061,-0.6241439581,1.0555865765,-0.3296152651,-0.9498324394,-0.6139930487,0.0078044580,1.9462395906,0.1260036528,-0.0267669503,1.5000147820,-2.2058892250,0.2527365386,1.4516202211,0.0328859240,1.4210357666,-1.8086668253,-1.2372293472,1.3431107998,-0.3939828873,0.3295665383,0.0458708517,-0.8294584155,2.3301115036,-0.1783146113,-0.3502511084,-0.8924494982,1.0734086037,-1.8520869017,-0.9093677998,-1.1956225634,-0.4636309147,-0.2347074598,-1.0941042900,0.7002073526,-0.0075926357,-1.6863496304,1.6743515730,-1.3652117252,-2.9964680672,-0.0707102716,0.1781398654,0.4942721725,-1.4362123013,-0.7788179517,1.6206996441,0.2547658384,-1.1822535992,0.9864816666,0.3597769141,-1.1920862198,-0.5226871967,-0.6126774549,-0.7237797976,-0.1754818857,0.2489106804,0.8328000903,-1.1851280928,-0.5307146311,-0.6191227436,0.7989680171,0.1682512611,-1.1042748690,1.3994650841,1.0414515734,0.0627358332,0.6830793619,-0.0892262608,1.5641775131,-0.1822139621,1.6051907539,0.8997560143,-0.0619025528,-1.2406721115,1.5553014278,0.6757091284,0.2351159900,-3.2029783726,-0.2392832935,0.0999327451,-1.0192216635,-0.5507404804,-0.4247370958,-0.1092737615,-0.4355481267,2.0317144394,-0.2517910004,1.1875435114,1.0763266087,-0.6341527700,1.2765887976,-0.6087099910,1.2905139923,0.2440712750,-0.2550278902,0.6374529004,1.1602060795,-0.5558677912,-0.7458577752,-0.1815477461,0.1130491644,-0.1535087526,0.8310692906,0.2297729254,1.5552196503,0.3676000535,0.1164899766,-0.2793305516,0.0842381567,1.1744824648,-0.5380910039],[1.1350588799,1.6307651997,0.5054000616,0.3415246606,0.1714084893,1.2019101381,0.0502630584,0.5542773604,0.9386487603,-1.0577836037,-0.2838217020,-0.6229845881,0.0756353810,-2.2039809227,1.0031237602,0.8466241360,-0.4926395118,-0.3171692491,-2.5975430012,-0.9221938848,-0.4696512222,-0.1379987448,1.6596388817,-0.7340003252,-1.3171730042,0.3707151115,-1.3485159874,-1.1177152395,0.8394469023,1.7678155899,-0.5140664577,0.3884877563,1.1650243998,-0.7375556231,0.0720298588,-1.3064266443,0.8903331757,-0.4982529581,-0.2980082631,-1.1973793507,0.4307799637,1.4663467407,-0.4940899312,-0.1184043512,0.6036401391,-0.6061331630,0.1731641889,-0.4107045531,-0.2922571301,-1.0013037920,0.1425794065,-0.4769991040,-0.8599395156,-1.5917048454,-1.4707101583,-0.5162295699,0.0312709026,0.9078719616,1.0566560030,-0.0541705601,0.4534251690,-0.3871499896,-0.1667667329,-1.3632845879,0.8195942044,-0.7354663014,-0.6091674566,0.8308277726,1.1752266884,0.7835314274,1.3266550303,-0.0654770806,-0.3728080094,-0.8336992264,1.1622248888,-0.3846301138,1.0055280924,-1.2250599861,1.1725132465,-0.1270800084,1.9019786119,1.5142729282,-0.1225099191,-0.1151846126,-0.1686504632,-0.0648666248,0.1656596959,-1.8448135853,-0.4019766450,-1.2433512211,-1.1153597832,0.9793652296,1.3056718111,-1.3503453732,0.5518975854,-0.2453743219,0.0945341364,0.9384822249,-0.4939763248,0.1662771553,-0.0385320112,0.4536897242,1.6621969938,-0.8693246245,0.5161611438,0.3339156210,1.2932506800,0.8945618272,-1.8559041023,-1.0705528259,-0.6615146399,0.0996204913,0.4900225997,-0.2379509211,-0.0018551836,-0.0410420187,-0.6998842955,-0.6224127412,0.7980476022,1.6843181849,-1.3955712318,-0.0199635271,0.6951984763,0.5448355675,-1.4337213039,-0.6472737789,0.5337479115,-3.0123813152,0.1207514852,0.3387083411,1.0219434500,0.3195720017,0.4922348559,0.4318786860,-1.5374253988,0.2995702624,0.6266925335,-0.1475751251,-0.9628657699,1.3692935705,1.7332062721,1.0504539013,1.6112185717,0.3021770716,-1.5211652517,-0.0542259924,0.7082381248,0.2385789454,-0.5415235758,0.2251457572,-0.7766765952,-1.1232200861,1.4787577391,0.0738270581,0.9700493217,1.4055027962,0.2390944362,2.5265150070,1.1471470594,-1.9478795528,0.3010020554,-1.1164835691,0.5054375529,-1.1840339899,0.8414674997,-0.2105533332,1.0772610903,0.1378324628,-0.2625436783,-1.2023355961,-1.2959524393,-1.9612358809,0.4218974710,-1.6659665108,-0.5268239975,0.9753533006,1.0594240427,-0.2273750156,-0.7501193285,1.5347290039,0.6678853631,-0.0581064932,-0.1379352659,-0.2763449848,-0.2625544667,0.9622181654,-0.5553058982,1.4604641199,-1.2602964640,0.0350733362,1.3494240046,0.0647262409,-0.2970011532,-0.1286823303,-0.4147179127,0.3933965564,1.0783635378,-1.1027158499,0.9179580808,-1.1704337597,-0.1262967438,-0.6603304148,0.2892201245,0.3097912669,-0.4064918160,0.6283999085,-1.0411736965,-0.0379688144,0.0183309894,0.4731063843,0.9111999869,-1.3404321671,-1.8525221348,0.4234216213,-0.0911732167,-0.1249522865,0.3678356707,0.5566195846,-0.0299841147,-0.0175340120,1.2085926533,0.6181845069,0.7668458223,0.2914898992],[-0.3310023546,0.5573409200,1.2957259417,-1.1349855661,-1.5247069597,-0.1056646109,0.8937266469,-2.4366216660,0.4809564352,-1.0557091236,-2.0977416039,2.2568325996,0.8655489683,1.5234045982,0.7148271203,0.9997466803,-0.3668726683,1.5002795458,0.5295903683,-0.0688808635,-0.3074559867,-0.5956582427,-0.7585545778,-1.7481904030,-0.7318330407,-1.0182477236,0.2035604119,1.8202784061,-1.6629431248,1.1729696989,-0.8748952746,0.4623175859,1.7992956638,-0.2273533642,0.8438876867,-0.5174407363,-0.3450641334,-0.2517184317,-0.4905411601,0.6074772477,0.8887094259,-1.1760468483,0.2600905597,-1.2200479507,0.8933705688,2.1258146763,-0.5572912097,-1.8600933552,0.7779752016,2.5773346424,0.1733140200,0.1253862232,0.5314847827,0.2223214656,-2.9037373066,-0.9753293991,-0.8351879120,0.0942138582,-0.6895788908,0.7909314632,-0.9552357197,1.1238473654,0.5297656655,0.7592256665,0.2255097628,-0.8313544989,0.9092111588,-0.3085229695,1.8506548405,-0.2688187361,-0.0903450549,1.1477758884,-0.8157768250,0.5765594840,-0.6213289499,-1.0924533606,0.3988735378,-2.0458335876,-0.5939063430,0.9805639982,-1.0290400982,0.0781027302,0.2894194126,-0.0519753769,0.8758471012,-0.3554590642,-1.8353921175,0.0100867106,-0.4113383293,-0.2478044927,1.4757231474,-0.1460000873,-0.1348475516,-1.9121975899,1.1605008841,0.4398345947,-0.8364595175,-1.1910685301,-1.2645142078,0.6124418378,1.1392834187,-1.2183645964,0.0038925342,-1.0627678633,-0.7069594264,0.4486484826,1.2478134632,-1.8462580442,-0.6957844496,-1.5319015980,0.6585696936,0.7199642658,-0.2957348526,0.6642016768,1.3873531818,0.9918613434,0.2797018588,-0.4639515877,-1.5886434317,-0.4204191566,0.8649352789,0.9510036111,-0.9391480088,2.0292148590,-0.1282894462,-0.7238674164,-0.1788964719,0.0195331033,1.0771375895,-0.4234253764,1.4801914692,-1.7666741610,-0.5446242094,0.6746259928,0.1982609779,-1.1595536470,0.9801321030,1.8839025497,0.2379632145,0.5130231977,-0.6771757603,-0.3835451603,0.6720554829,-0.5601616502,-0.2693561912,-0.7150922418,0.9604054689,-0.1766158491,-2.1141669750,-0.1035743877,-0.1573709249,-0.6674705744,-1.6187994480,-0.3529086113,0.0866112337,2.2035353184,1.0520107746,-0.4452825189,-0.3899340332,0.1289973855,-1.2968221903,0.3952779174,-0.2544807494,-0.4893265963,-2.1965639591,0.2055465132,0.7529613376,-0.3973491192,1.2315872908,1.6232351065,-0.0800956413,0.0522243381,-0.6569432616,-0.7473552227,0.1069898829,-0.9588980675,0.2506351173,-0.0965848640,-0.0504180305,0.5609652400,2.0323703289,0.0770998970,-2.0426630974,-1.7376391888,0.7450774312,1.9501069784,0.6154040098,0.9178984165,1.4370058775,-1.7092440128,0.5272009969,0.5203163028,-1.1656514406,-0.5942999721,0.1818185747,1.5111366510,-0.0468265265,-0.5543540120,-0.1215622202,-0.8664098978,-0.2492751628,0.0694557428,0.1134820506,0.3137679100,-1.2232038975,0.8229790330,-0.7367380857,-1.8020118475,0.0884521306,-0.8212783933,0.2604509294,2.0971336365,-1.0609384775,0.7672310472,0.5581717491,0.0284287650,-0.7300484776,1.6223838329,1.8920705318,0.7585506439,1.2013385296,-0.6682579517,0.5687101483,-0.0458690971],[0.7914537191,0.6461397409,0.5958306789,0.3313758969,0.5736564398,0.1297451556,0.6420159936,-0.3512704968,-1.6068793535,1.0904586315,-0.1694664806,0.8543040752,0.4681234062,0.6300991774,-0.5073465705,0.0981798470,0.5250322223,0.2505944371,-0.7383431792,1.5344400406,0.8768033981,0.9915237427,-0.1485308558,2.7769432068,-0.7796960473,-1.1073904037,0.6106311679,-0.1776532382,-0.7403364182,-0.0547751002,0.1981050968,0.4532022774,0.9392779469,-0.8190399408,-0.7111040950,-2.2401287556,0.0680859536,-0.5838615298,0.2028556317,-0.4420544803,-0.5240891576,0.1586313397,-1.1596050262,2.0908944607,-0.6640832424,-0.0376775339,-1.3674635887,-0.0834356248,-1.1543424129,-0.2867710888,-0.2558217049,-1.2492008209,1.9734691381,1.3248716593,1.6726819277,0.1237697229,-0.5251456499,-0.0927522108,0.9765563011,0.4406595230,-2.0358545780,-0.0322042890,0.2834545076,-0.2984555364,2.5113687515,0.6123604178,-1.7609668970,0.1541956365,-0.1179339066,-1.6794335842,-0.0388058573,-0.3712675571,-1.3224407434,0.8247251511,0.1571902931,-0.4441169202,-0.0230798498,-0.0669803545,0.8565628529,0.4665855765,-0.7799400687,-0.8101128936,-0.5259451866,-1.4127696753,-0.9107655883,-0.0380207002,-1.1302808523,0.2475105673,-1.0421621799,-0.2518622875,-0.6521123648,0.9967294931,0.3215339780,0.1610585898,1.0579377413,1.0964981318,1.0885875225,1.5951294899,-1.4319165945,0.5318467021,0.3024968803,-2.2370159626,0.3017950356,-0.3017114103,-0.6175892949,0.8086286783,-1.0254920721,-0.7773222923,0.3564308882,0.8522893190,-1.5144516230,-1.2158509493,-0.0528383479,-0.2945184410,-1.1897754669,0.1364751011,-0.3321232796,-2.0284500122,-0.6192065477,-0.9456180334,-0.1601357013,0.4249179065,0.1432779431,-1.2779705524,-0.1355749816,-0.5009277463,0.6132161021,-0.2418944091,-0.2416060418,-0.2200713307,1.8523660898,-0.1587850600,-1.9892187119,0.9731972814,1.1787183285,-2.0948641300,-0.8218100667,0.9060323238,-1.3545004129,0.5676369667,0.1284798980,-1.0084096193,1.4587696791,-0.5944750905,-0.5187753439,-0.6267364621,0.0450868420,0.2594417930,0.8670076728,0.4846119881,-1.4180843830,1.0617189407,-0.2597389221,0.9757429957,-0.9346953630,-0.4718877971,-0.1700360179,-0.2180638164,1.1518630981,-0.5961496234,0.1541005969,1.2585749626,0.1049211174,0.1370392442,-1.1218445301,-1.2547951937,-1.3814834356,0.9695735574,-0.1086363643,-0.8288313150,0.5897889733,0.1763927042,-1.6123449802,-0.3767241836,1.0482236147,-0.6818988323,-0.8554486036,-1.0561922789,-0.9120386839,-1.0435223579,0.7464059591,1.5055519342,0.5351999402,0.1048421785,-1.5796937943,-0.1231038421,-0.1131892130,0.0296576433,-0.1823898405,-1.0289748907,-0.8570144176,0.3856546283,1.2521356344,0.4814043343,-0.6368389130,-0.4543634653,0.7679942846,1.6410802603,1.9045054913,0.1016824543,-0.9051506519,-1.0915925503,0.5618121028,-0.0764838457,-0.1261715442,-0.2757646739,-1.6904895306,0.1201585308,0.4685803950,-0.7667537332,0.5887155533,1.0020669699,-0.4119359553,0.5783369541,2.0256936550,1.3819185495,0.9228142500,-0.8997434378,-2.1333208084,0.5421010852,0.5343675017,-0.1246316880,-1.4611852169,-0.3485037684],[-0.1807983965,-2.3376026154,-1.5892693996,0.0738651678,1.8555306196,-0.3728599250,-0.7936410904,-0.7694765329,0.1537291110,-1.2192952633,1.0588021278,1.7063795328,-1.4852479696,0.7988227606,-1.5708323717,-0.0804250613,-2.3823585510,-0.7975266576,-0.6703687906,-0.1327716857,2.0303711891,0.4827000797,-1.0179685354,-0.1614174098,0.3180173337,0.4515491724,1.3316066265,0.6386704445,-0.2509791851,-1.0985279083,-0.5632628798,0.0922431201,-0.0308225118,-0.3874062300,1.0752867460,2.0628256798,-0.0924772248,0.5879930258,0.2235952467,1.9383288622,0.2087660283,1.0126709938,-0.9500865936,0.8185034990,-0.6236126423,0.2088777721,0.1101473123,0.3424575925,-0.1206818819,-1.3793892860,-0.4113827944,0.9614917040,-0.9937870502,1.7580798864,0.0080232378,1.3879599571,-1.9699418545,-0.0177614167,0.3617301583,-0.0824940875,0.4315568209,0.5943727493,0.3909710050,-0.2284035534,-0.0388397276,0.7088179588,-0.8342286348,1.0389921665,-0.4749735594,-1.2656178474,1.6008704901,0.0157479066,-0.8812649250,-0.2172022909,0.5726627707,1.1687834263,0.7123550177,-0.2866460979,0.9056574106,-0.1671238095,0.7700747252,-1.4749193192,-1.9875162840,-1.9177916050,-1.0036017895,2.3900575638,0.1375515759,-0.6110166311,1.5173581839,-0.5640364289,1.3913329840,-1.1090196371,-0.4102989435,-0.4230274558,-0.8586001992,-0.0699972957,0.5191432238,-1.7878763676,-2.2792520523,0.4116008580,-0.5971567035,0.6625709534,-0.1563580632,0.0519339070,1.3340390921,0.3207374215,0.7841516137,-0.1863603294,0.8826067448,0.8950135708,2.3930306435,-0.1827965528,-1.4979075193,0.6857637763,0.4625678360,0.2142915875,-0.6569813490,-0.3911370039,-0.0136631001,-0.4137517214,1.3745381832,0.9061549902,-0.6315728426,0.3884846270,0.7096084952,0.2195208967,-0.9828910828,-0.6491459012,-2.3690044880,-0.1559623629,-1.0859808922,-1.3553333282,0.2614978254,0.3400335610,0.7255932689,0.0967543870,0.3278695047,-1.0589563847,-1.6091902256,0.5406217575,2.1439728737,0.6192141771,0.9494862556,-1.4554423094,-0.0448695049,-1.4860472679,0.5595998764,-0.9465503693,-2.0627651215,0.4471147358,-0.6683236361,-1.2262574434,-0.2228455693,-1.3789155483,-0.5255289674,0.1673466712,0.5638135076,-1.4176610708,0.5824736357,-1.4399206638,-0.2605627477,-0.5131645203,1.5817571878,0.9342992306,2.0273759365,-0.9074780941,-1.4963614941,0.2098528743,-1.3330277205,-1.0364831686,-1.1096246243,1.2138468027,-0.6777035594,-0.9634899497,-0.2676663101,1.7159404755,-0.5196566582,0.5119627118,-0.3616420925,-0.3848065436,-0.2256528139,0.3118752837,-0.3490059674,0.1993868798,-0.9058728218,0.4433694780,-0.3112775981,-0.0912733003,-0.6117324829,-0.5883981586,1.7798914909,0.0368696302,0.5935785770,-1.6113942862,-1.8226302862,0.7137543559,0.3825588524,-0.0519888066,-0.5621618032,1.2645081282,-1.4425507784,-1.0361926556,0.5223254561,-0.3062395751,-0.6406183243,-1.5627833605,0.7695292830,-0.3185529113,0.1646706760,-0.3141778708,0.6940912008,-1.3859407902,0.2595100701,0.9433202147,-0.9889213443,-0.7876140475,1.5091377497,-0.0022732059,-0.9895153642,-0.7657221556,-0.3768515289,-0.7495852113,0.9069278836,0.8732688427],[-0.6578397751,-1.8318632841,-0.5395345688,0.3903403282,1.3636548519,-0.5079500675,-0.4028911889,-0.9917718172,0.4559382498,-0.0439116731,0.6223576069,-1.9363251925,0.2596221566,-0.4668768048,-0.1926385313,-1.1734788418,-1.9986387491,1.9325946569,-1.3068764210,1.1438997984,2.0557627678,1.9167425632,-1.3739305735,1.8942967653,-1.3069814444,2.5766277313,-1.2845826149,2.6390814781,0.0117756752,-0.8140695095,-0.0189126283,-1.7700136900,0.1316619217,0.6881023049,-0.0800242648,-0.3123969436,-0.1349624991,1.1459685564,-0.3061838448,0.8619158864,-1.3219367266,1.3408268690,-0.2174276561,0.7989725471,0.1819919050,0.3068866432,1.5338616371,2.0315186977,0.7892776132,-0.7427718639,0.5892996192,-0.0023908424,0.2674568594,0.0645150989,-0.4013667107,0.8512451053,0.6213958859,0.8364474177,1.9893764257,-0.1589120328,0.4406788349,-1.8999271393,-0.0583184175,2.8087897301,-0.6425076127,0.5418109298,0.3948427141,1.2454727888,0.4545042217,-2.7257065773,-0.8574002385,2.5853521824,-0.3502472937,-0.5604780912,-0.6599280238,-1.1559455395,1.3983881474,-0.8947708011,-0.3445873260,0.0215723384,-0.5061492920,-0.1462244987,2.1528968811,-0.8219048381,-0.6240154505,0.9768301845,0.6534996629,1.4312934875,0.5451650620,-0.1355044693,-0.4385142028,1.1829334497,0.8356802464,-0.5545064807,-1.4946864843,-0.9651259780,0.8002617955,1.6885603666,0.4508632421,-0.5405079722,-0.7677236199,2.1276350021,-1.5085190535,1.0470719337,-0.3742038906,0.4816255867,1.2239198685,2.2002627850,-1.1459487677,-1.9393211603,0.8348519802,-0.8973862529,-0.0270348825,-0.4584574997,0.2868529260,-0.0345946364,-0.9449191689,1.6783432961,-2.5124733448,0.2714243531,-0.7285017371,-1.1114827394,0.4374779165,1.2229865789,0.1188821420,0.6417844892,-0.0339858830,0.1788398623,-1.2683531046,-0.8834310770,0.0883188769,-0.7413329482,0.4079362750,-1.5605533123,0.6710539460,-0.3997884691,-0.8656931520,0.5296225548,-0.1787042618,-0.6306082010,0.1262859553,-0.0460716784,0.9800592065,0.6499250531,0.2291618437,1.3460974693,-0.4842807353,-1.3417708874,-0.3950604796,-1.0232306719,2.1392402649,-0.4437207878,0.3694180548,-0.8835060000,-0.7907400131,-0.6032630801,-0.4702125788,0.0179922599,-0.4642401636,-1.3101488352,0.8742898703,-0.7117217183,0.1661468446,-0.5677089095,-0.8415204287,0.5891016126,1.4322475195,-1.4599040747,0.4364635944,0.7957346439,0.4348514676,0.2555656135,0.7927491665,0.1485411376,0.5828484297,0.8439933658,-0.5871391296,0.0046702735,0.2771545649,-1.2712996006,0.2197576612,0.0016449381,0.9348805547,-0.1721846610,-0.2281569988,0.4430972934,-0.7958605289,0.9351830482,0.1677804291,-1.0998831987,-1.4856913090,-0.7987915874,-0.0081814257,0.3317958713,0.0209362600,0.9280778766,1.3373113871,-1.8254497051,-0.4517202079,-0.4237102866,0.4339829683,-0.0882978663,-0.8270207047,-0.5583283305,1.0447891951,0.3826203048,0.6272954345,-0.4586470723,0.5723845959,0.7396498322,1.7548961639,1.2826986313,-0.7351671457,-0.8461158276,0.0586707927,0.4825576544,1.0927946568,-0.8203950524,-1.0296913385,-0.1650319546,1.3045974970,-0.0735537857,-0.5020622015,-0.2131796926],[-1.9353233576,-0.0282856803,-0.0519692637,1.2257838249,-0.3289196491,0.0804954693,0.6277905703,-0.4614386857,0.8901169896,0.6736842394,0.2361200452,-1.7104322910,1.4444857836,0.6761213541,1.8072758913,0.5737406015,0.1480455250,-0.5947783589,-0.0989431441,1.4684959650,1.1539797783,0.2363432348,-0.1957866400,-1.4308358431,-1.0022342205,0.6599704027,1.3189892769,-0.0529710427,0.7598709464,1.4087154865,-0.1161145121,1.4070295095,0.5922804475,-2.0745668411,1.0892683268,0.6900909543,0.2436967641,0.5517532825,1.2490776777,0.1645147800,0.5463775396,-1.2008447647,-0.6278283596,0.5528708696,1.0359449387,0.0826989859,-0.2087807506,-1.0988688469,0.8048434258,-0.8141828775,0.3546221852,-0.2129131258,0.2873154879,1.7163056135,0.3986051679,-2.2500936985,1.6220793724,0.1039571762,-0.6963388920,1.6507164240,-1.1810199022,1.0891958475,0.4307694137,1.2198220491,-0.9328013062,-0.7104882002,0.2273416668,1.2003076077,2.2550117970,-0.9103827477,-0.5597643256,-0.4322056770,-0.4192131460,0.7286592126,0.0349851511,0.8051463962,-1.2571941614,0.2547564805,0.2583035529,-0.4347037077,-0.0791249797,-1.6814187765,0.5205547214,2.3092405796,0.7493299246,0.9001592994,-0.1037041694,0.7711718082,0.3438462913,0.4822458625,-2.0766065121,0.0221872889,-0.0421145484,-0.7678053975,0.3982978165,1.2170741558,-0.4540759325,-0.7831273675,-1.0605846643,-1.6356533766,0.2026299387,-0.1641400456,1.0439503193,-1.0182771683,0.1616994590,-0.2499357611,0.7871867418,0.1861835867,0.5747888088,-1.7889155149,-1.9064621925,-1.0385804176,0.7762565613,0.3348379731,0.2335910946,-1.0033197403,0.7750243545,0.6567839384,1.4885655642,-1.0188703537,0.4090184867,-0.0310165752,0.5363311768,1.5438086987,0.6926738024,0.9071796536,-0.1791453212,-0.2743999660,0.0375481956,-0.0360734574,-0.0225227624,0.7798106074,-0.3104155660,-0.5394257307,-0.3660118282,-1.2562537193,0.1709382087,-1.0761003494,-0.0932355300,-0.1811507940,0.5199954510,-0.5508889556,-1.0994771719,-0.4694336057,0.7088356018,-0.0675911307,1.3022845984,-1.4211859703,1.2567162514,-2.3688504696,0.7742468715,-0.4273701310,0.1674273014,-1.3203834295,-1.8911298513,0.5281402469,1.3740572929,0.3885111809,1.0885382891,-1.7151808739,-0.3717588782,0.0876440331,-0.5725683570,0.0155486111,-1.0659922361,0.4667593837,-0.0940362737,-0.4041534066,-0.1656819880,2.3223657608,-0.7614206076,0.9490546584,-1.4844082594,0.7458498478,-0.0060643018,0.7887188792,-1.1180588007,-1.0001761913,0.7314289808,0.2357216030,0.3824612498,-1.7703744173,-0.2727081478,-0.1264617443,0.6515104175,0.5096127987,0.6367074847,-1.0184148550,1.2223472595,1.3987852335,0.7798677087,0.6547874808,-0.3030628264,0.0454207473,-0.3884822726,0.3867644668,-0.9542946815,-1.8427627087,0.3048293889,1.9156039953,1.2040926218,-0.2316040397,-0.5864921212,-0.2658803463,-0.4716253281,-1.5087419748,-0.0129999518,-1.6092796326,0.7645639181,0.7615116835,1.2913290262,-0.4876889288,-0.4697682858,0.3820580542,0.4084323943,-0.6136229038,0.6022604704,-0.3676231503,-0.7209435105,0.5586885810,-0.5583328605,0.4996970892,-2.0316460133,0.0585249402],[-1.0543115139,-1.0559836626,0.3332637548,0.0095748026,1.8420743942,1.0140199661,0.0083231404,-1.5318380594,0.6986145973,0.6373190880,-1.8509548903,0.2487269789,-1.2360033989,-0.3390091062,0.4469988048,-0.1896353811,-0.3229845464,0.4780968726,-1.3126137257,-0.2899254262,-0.1851799935,-0.3376224339,0.4259453416,-0.4082024992,-0.8978323936,-0.6690765023,0.2631308734,0.5530337095,0.1995165348,0.2926838398,0.8894163966,-0.0055672377,0.7018143535,-0.8826447129,0.0749236047,1.2906317711,1.0046035051,1.0409386158,3.0276360512,-0.0133288782,-0.0506551601,-1.8908251524,-0.2714906931,-1.3030624390,-0.2462271601,-1.8104127645,0.1279558837,0.1793313920,-0.7023834586,-1.0055930614,-1.4191739559,-0.4562571943,0.4048222899,1.3166447878,1.1341559887,0.7166735530,0.5539758801,0.4354004264,0.3141281605,0.6505777836,0.2128535211,1.6090195179,0.1628310382,-0.6526899934,1.2286890745,-1.2203204632,0.4666250944,0.7341588736,0.1583673805,0.1684402227,0.1241835728,0.7640827894,1.2957168818,0.7631736398,-0.6601703763,-0.0008463254,0.0685535297,1.1067372561,-0.4385348558,1.7345260382,1.9808120728,-0.6242682338,-0.0815576091,-0.1650884449,0.4348444045,0.5211656094,-1.6433618069,-1.7573856115,-1.4892438650,-0.4256701171,0.2377105355,1.5365910530,0.3241511881,-0.5333830714,0.5189927220,-0.3648990393,-1.9341444969,-0.4795877635,-0.9685363770,0.3621700704,-0.4226962328,1.9258793592,0.5076354742,0.2610789239,-1.2897127867,1.4333382845,-1.6838537455,-0.7709188461,0.8705440760,0.1586957872,0.2956838310,-2.0230813026,1.1002036333,-1.1002752781,-0.5147218108,0.1019927114,-1.0212808847,-0.5737935305,-0.7874208689,0.4350974560,0.1936040074,-1.3571660519,-0.4171037376,-1.6063512564,-0.2483906001,-0.2693999410,-1.5144617558,-0.1226147190,1.5742284060,-0.2891094089,1.2006629705,-0.3703657985,-1.6033035517,-0.4120010436,0.2618357539,0.0800257102,-0.0225607231,1.0052714348,0.2401950359,-0.9681993723,1.6243599653,1.7711733580,-0.4288421571,0.8745658994,-0.7261585593,-0.7740636468,-0.3783999681,1.0468926430,-0.1909528077,2.1233911514,1.6014511585,-1.4648680687,-0.2578746378,0.5449219942,-0.8318146467,0.2432440072,-0.5521076918,0.4936685264,-0.0429933108,-0.7929572463,3.4898121357,0.6794958115,-0.6419728398,-1.2206847668,1.3815455437,-1.5034804344,-1.4729547501,-0.7631402612,1.4460397959,1.1735513210,1.1392821074,-0.3697892427,0.1053928807,0.1785945147,0.1319828779,-1.2402095795,-0.7929094434,0.0239162408,1.2175018787,1.2581635714,-1.1096943617,-0.7467578650,-0.1821890175,1.7982856035,-0.9656202793,0.0464049689,-0.4019823372,0.8537561893,-0.9632617831,-1.3833892345,-0.3336651325,-0.6310629249,-0.6288160682,-0.6705541015,1.1295614243,1.2133406401,-0.1866526157,1.8917940855,-0.2755779922,-0.6542409658,-0.3929320574,-1.2494572401,0.3466140926,-0.5797623396,0.8853974938,-0.2189182490,0.6220045686,-0.3418531418,-0.9100940228,0.8505608439,-0.5887028575,-0.7855650783,-0.3946730494,-0.2303701043,1.3030629158,1.7425899506,0.1629963517,-0.1453389823,-1.2816724777,-0.6563048363,0.0003808588,-0.6638936400,0.8562603593,-0.1051234752],[3.0289349556,-1.4466205835,0.1836807728,-0.5522636771,0.5423675179,2.7219474316,0.8910558820,0.6465635896,-0.0759895593,0.1817708462,0.3457686603,0.7239876390,0.7132585645,-0.4586695135,1.0333595276,-0.2189355940,-1.1411225796,-0.8677021861,-0.9688470960,-0.6339297295,-0.0074916431,0.0354148932,2.4297785759,-0.8822140694,-0.9148740768,0.8378556967,-0.1476698518,0.6026245952,-0.0974400416,-0.6325020790,-0.8441191316,0.5103759170,-0.7266747355,-0.7533026934,1.4277828932,-0.7310283780,-0.0555546284,-0.9097919464,0.9007401466,1.1238573790,0.0252667442,-0.7156252265,-0.2277295440,-2.1477272511,1.5932493210,-1.8628616333,-0.3776150644,-0.1090053990,-1.5743252039,0.9123138785,-1.3972257376,-0.0600600950,0.3769355714,-0.4396723211,2.1093904972,0.3233568668,1.2451854944,0.7502902150,0.8297964334,-0.0762333572,-0.0797017440,-1.1705890894,-1.7427301407,-0.5095266700,0.3880380094,-1.5055037737,1.0071586370,0.5120072365,-1.0015641451,0.6942522526,-1.1639013290,-1.5982449055,1.4353106022,1.3250426054,-0.7664409280,-0.2600101531,-0.8542203307,-0.8446595669,-1.4417269230,-1.6894496679,-0.7739345431,-0.4729284644,0.6903986931,0.5380149484,-0.6231607795,0.1010175794,0.6848499775,-0.6755223274,0.3829651773,1.0227255821,0.0714008585,-0.0293932445,1.8250042200,0.2873611152,1.8083481789,0.5629999042,-0.8379108906,0.4814782143,0.9358680248,0.3034250140,0.4835818708,0.6362583637,-1.9723367691,0.1129239202,-0.9730765224,-1.2520319223,-0.6632204652,-0.5693321228,0.0858491659,-1.6128265858,2.4252493382,-1.2824521065,-1.2813147306,2.0825462341,0.9443973303,0.3961180151,-0.6054111719,-2.1568701267,0.8435489535,1.5585157871,0.8996023536,-0.1726988107,-0.2633880973,-0.3633803427,1.7543785572,-0.5090419054,-0.1032537743,0.2715066075,-0.4920640588,-1.5715212822,-0.9911488891,-0.4591449201,0.1021347195,-0.1885639131,-0.1514699608,1.2429906130,-1.2038488388,0.1201531067,0.6239452362,0.8239920735,-0.8424552679,-2.4820933342,0.0825293586,0.7349550724,1.4615254402,-1.8211261034,0.7355304956,0.0824137405,-0.0821021348,2.4546854496,-0.6104809046,-1.4369820356,0.3347872794,0.0953120217,-0.1711449176,-0.2775689960,0.2559177577,-0.6192325950,-0.0518681258,0.7144017816,0.1674056649,-1.9963171482,-1.2033586502,0.1521248519,2.5265574455,-0.6133159995,-0.3060063124,2.1903588772,-0.0703226253,-0.3838198483,-0.9298690557,2.1403698921,-1.1335151196,1.8761792183,-1.1630308628,0.7195526361,-0.9941395521,0.2390076518,-0.5244604349,0.2172025740,-0.3227881193,0.4246652126,-0.7177217007,-0.1237884983,1.0893511772,0.3389956653,0.6737839580,-0.4500505924,1.9005181789,0.3046309054,1.0528272390,0.6181700826,1.2530003786,0.1073710024,0.6634507179,0.6261904240,0.9524511695,-0.3014968634,1.7174031734,-0.7578988671,-0.4663391113,-1.4782775640,-0.2628045380,0.5235355496,2.8543355465,-0.7934467793,0.2137494832,1.1492207050,0.7139776349,0.2760613859,0.4369187057,-1.4526598454,1.1321744919,0.4283018112,0.6453626752,-0.3679089546,0.8216310740,0.2988089919,-0.6367230415,1.3324908018,-0.2185408473,0.4428556859,-0.9793181419,-2.3876283169],[0.7414361238,-0.5878328085,-0.1686553657,-1.0306122303,-0.5140874386,-0.6933035254,0.8230751157,-1.2295557261,2.2675883770,-0.0020896329,-1.5306205750,-0.2049885243,-0.6176739335,-1.0552675724,-1.0267729759,0.7805772424,1.4680932760,-0.6374821067,-0.0440820456,0.6838778257,0.1650949717,-0.6097522378,-0.9951433539,-1.1233109236,0.2587343454,-0.6517683268,0.6634348631,-0.1877324134,0.0308681466,0.0288206376,0.5415495634,-0.5201122165,0.4827626944,-0.1858599484,-1.3083807230,0.2003160715,1.1300369501,2.4712626934,-0.2974512577,0.7705679536,0.0965008363,-0.8537307978,1.1313229799,-0.4410588145,-0.3641335070,-0.4061614573,0.2904089391,-0.2795744836,-0.6002675891,0.2109877020,-0.0930366442,-0.0394295789,0.2261722982,0.8024736047,-1.8428796530,-0.6576650143,-0.3111714125,0.8684728742,-0.5444998145,-1.9280499220,-0.1636284292,2.2250468731,-0.1079393625,0.6490951180,-0.6822468638,-0.1159646735,0.9660808444,-1.0916345119,1.2806700468,-0.6005136371,0.4279638827,1.6419439316,-0.7441155314,-0.5071685314,0.2477568686,-0.6498341560,0.8419561386,-0.7465654016,0.8345745802,0.7719214559,0.0275003444,-0.8203216791,0.7621380091,1.8413269520,1.9317811728,0.7190065980,1.2122613192,-0.5506513715,1.2079052925,-0.5110875964,-0.9887308478,-1.4687793255,0.7604703307,0.8583361506,-1.1693745852,0.9667693973,-1.0014497042,0.9113854766,-0.8501291275,0.1033337414,-0.2851611078,0.4838745594,0.0306431390,-0.9261587858,0.5031406879,-0.7204946280,0.0854555964,0.5342746973,-0.2392912656,-0.3483181894,-0.7283043265,-0.1952078938,-0.2268493474,-0.5073602796,0.4071733952,1.7896782160,-0.7856964469,0.1932238191,-0.8698413372,-0.2894476950,-0.4371841550,0.6789008379,-0.5095900297,-1.1314054728,-0.9796065688,-0.9307839274,-0.2411198616,-1.8260837793,-2.1603901386,0.7138137221,0.4682048559,1.8073990345,0.3030933738,-1.5624271631,-0.1223475486,-1.4457436800,-0.1992979646,-0.4670667946,0.5985659361,0.0050599673,1.0435233116,-0.3494057655,-0.1419804990,1.0799071789,-1.1984112263,-1.6463519335,0.9736518264,0.2760215700,0.4218560159,0.2312800139,0.0733595192,1.0696476698,-1.1329557896,-0.2230795324,1.1043237448,0.3311566710,-0.1949632615,0.4872407019,-0.5673235059,0.3815425336,-0.5976793766,0.3133822381,1.1188403368,0.1023709923,-0.7640777826,0.5128422379,-0.9548774958,-0.7417376637,0.8232577443,0.6041470766,-1.5107276440,-0.6352846622,-1.6938943863,-1.4230499268,0.8208574057,-0.1627612412,-0.3808040321,2.1364395618,-0.9052449465,-1.3772284985,-1.6807270050,-0.5424402356,0.9703767300,2.0392134190,0.0691738948,-0.1845367402,1.0785111189,2.0295569897,-1.2068397999,-0.4048538506,0.3063526452,0.0183172151,-0.7925714850,-2.4151265621,-0.5461390615,0.5046216846,-1.1571860313,-1.0568937063,0.0951267779,-0.2945894301,0.6409581900,-0.5636038184,-1.2628661394,-0.5963248610,1.0266063213,-0.1695082039,-0.8970633745,-1.5442864895,-0.6414470077,-0.6634951234,0.0513509884,0.4815643728,1.3058356047,0.4562318921,-1.1240118742,0.8064363599,-0.2292824686,-0.9644187689,-0.8825575113,0.4957913756,0.4160384238,0.7359398603,-1.0344359875,0.6110323668],[0.0616292097,-0.8699955344,0.9175645113,0.5920697451,0.2377380133,0.1208677292,0.1229188591,-1.1193079948,-0.5608552098,-1.0286526680,1.4120683670,0.2869732082,-0.7669487000,-0.4248317182,-1.6202235222,1.1397325993,-1.1340955496,1.7128037214,-0.0321023352,2.4242587090,-0.3608433008,-0.3228870332,1.5381108522,-0.1577602327,1.7750029564,-0.7555779815,-0.2008749545,-0.7190867066,-2.0994808674,0.6139115691,-1.2123483419,0.8159266114,-1.8295583725,-3.5421888828,-0.3941495419,-1.0236091614,0.2525290549,0.5810935497,-0.1076203361,0.4633731544,-0.4760022163,-0.0666669309,0.9731261134,-1.4544069767,0.0888629779,1.7643368244,-0.1727118343,0.5579123497,-1.1497603655,-0.3366352916,0.8751723170,-1.4169456959,-0.8309580684,0.3659218550,-0.3941937983,-0.3234885931,-2.1415123940,-0.2322276533,1.7834504843,-0.4405650198,-0.1744079888,-0.4390559494,-0.8015988469,-1.1638343334,-0.5664111972,0.2518135309,-0.6597012281,0.2770966589,-1.0669745207,0.2963372171,-0.0308781285,-0.3477644324,-0.8671655655,-1.2455052137,1.2770495415,0.1706280708,0.1064133495,-0.7048476338,0.6347349286,-0.7335917354,-0.8754142523,1.1479344368,1.5040304661,1.3156964779,-0.5855301023,-0.1134957299,-1.0665155649,0.2300466001,0.9959614873,-1.3386521339,0.4310475886,0.4835477769,-0.5265149474,1.6178389788,-0.0525867380,-0.9931297302,-0.2980194092,-0.8178262711,-0.7970831394,-0.5230959058,-0.2952790558,0.1519235820,-0.5994967222,0.8724045157,1.0875824690,-1.2039092779,1.0752569437,-1.4313342571,-0.0718548223,0.5625308156,-1.0406832695,-0.5584868789,0.6927548647,1.2421534061,0.2440869510,-0.3585351706,0.6971446276,-0.0839696378,0.1699494272,1.3358668089,1.2711428404,-0.5545876026,-1.3880506754,2.2152044773,-0.8038774133,0.5055804849,1.7750318050,-0.0108200340,-0.4221525490,-0.0152974008,-0.7703688145,0.0354482979,0.3172935545,0.6298582554,-0.8587480187,-1.3135603666,-0.5014687181,1.8155835867,-0.6328458190,-0.3373099566,-1.5931397676,-0.2987871766,2.0653538704,-0.3638643920,-0.9729212523,0.7749390006,-0.7497231364,-1.8904943466,0.8607858419,0.1471386701,-1.5295069218,0.5777686834,0.8261218667,-1.0119875669,0.0897994936,-0.4430561960,0.0323836058,0.2219475061,-1.2818908691,-0.2708592713,1.4573109150,0.8804024458,-1.5916829109,1.0492521524,-1.1633678675,0.8123183846,0.4321608245,-0.8351872563,-1.6270632744,0.5444349051,1.7567660809,-1.4698419571,-1.3057603836,0.6106351018,-0.6856634617,-0.0027157622,0.7870089412,-0.1850146800,2.8850343227,1.5643398762,-1.0507323742,0.6868153811,1.1048454046,0.1835930198,0.5201048851,0.7623929977,-1.4696066380,0.0583677441,0.4692377448,-0.2277192771,0.9734441638,-0.0795424357,-0.2654288709,-2.2028191090,-0.2876582742,-0.0065556336,-0.0743332133,0.5298947096,-1.4907051325,-1.0873161554,-2.1639416218,-0.4339273572,0.3658332825,-0.2206286639,-2.2076971531,-0.4790238142,-1.8818588257,0.4069420695,0.2041137069,-0.2685712576,2.1940827370,0.3173443079,-0.5103394389,-0.3893362880,-0.4261973202,1.2880473137,1.3300783634,1.0021003485,-1.7409325838,-0.1299262196,1.4271765947,0.1755771786,-0.1361971200,2.6091077328],[-2.1213953495,-1.4876091480,0.4027919471,-0.7992261648,0.8391885757,0.6907860637,0.0751595125,1.9657648802,0.5034283400,-0.5381843448,2.5232529640,-2.2388384342,0.5314210057,2.4042863846,-0.1817212552,0.4993635118,0.2077349424,0.8697637320,0.4360201657,2.3369359970,0.2676490843,1.1903718710,0.8799911141,-1.1188682318,-0.9714317918,-0.9767854810,-1.3006596565,-0.2873510718,1.4437073469,0.9303485155,-0.3495375216,1.2017552853,-0.9662505388,-0.7631691694,1.6332331896,0.8243235946,-0.4884332120,-0.0201347098,-0.7768230438,-0.7631254792,-1.2737504244,0.4627576768,0.1729757488,-0.3228688538,-2.4766850471,-1.1090526581,-0.6864037514,-0.8046090007,-0.7138791084,1.6377116442,0.6567335725,-0.0047580036,-2.2416980267,0.1491056532,-0.0847144574,0.2720263004,-1.6646059752,-1.1872817278,-1.5886445045,-0.2330412865,0.2284714878,-0.9844371080,-0.1336304843,-0.5829446912,-0.2540107667,2.2281854153,-0.5904042721,-0.5926827192,-1.3128408194,0.1390305161,-0.1672947854,-0.7939913869,-0.6221416593,-1.5166251659,-0.1779445559,0.0959212556,-1.6828132868,-0.7401673198,0.5231422782,-0.3322941959,-0.1783479750,0.4485481381,-0.3100477755,0.0970409885,1.1743543148,-1.8261427879,0.4978103042,-0.0108789131,0.5395866036,3.3376805782,0.5630101562,-0.4743687510,-1.9228357077,-0.0009492107,1.3383150101,1.5428452492,-0.2962970436,-0.5699378848,-0.4364693761,0.2970700264,-0.5584040284,-1.9995604753,-0.7018766999,0.7856997252,1.6371961832,-1.1782302856,0.5775887966,1.2874151468,0.6118171215,-0.5430070162,1.0252311230,1.9605115652,1.1515276432,-1.1050601006,-0.3522197604,0.3429833055,1.9247678518,0.1838604361,-0.4415161908,-0.7715782523,0.1724594831,0.7692704201,-0.5029193759,-0.2041465640,0.6117989421,0.2968843579,-0.9996942282,1.7067486048,0.6819940209,-0.6585046053,-0.1444742084,-1.0080267191,1.4498666525,0.3252133429,-0.8631876707,-0.0172087178,-0.9234526753,-0.1951841712,0.8916925788,-0.4279014468,-0.4757214785,1.1231888533,0.5568086505,0.5508191586,-0.1973502934,-1.0614956617,-0.9130429029,-1.2311202288,-0.1490184516,0.6185660958,-1.0488156080,0.4432644546,0.8579077125,0.6526910067,1.9192267656,0.6657885313,-2.7699515820,-0.8622987866,-1.5329411030,0.5477091670,-0.3008966446,0.4089683890,-0.2756463885,0.2297792137,0.7346068621,-1.2895530462,-0.6784411073,-1.2107373476,-0.4332395196,-0.0472933054,-2.2021653652,-0.2635502517,-0.7713882327,0.9504997134,0.5086578131,0.8237007260,0.6284089088,2.4425268173,0.6307875514,2.4813482761,0.1044523641,0.6090664268,1.5078846216,0.6996105909,0.5948370099,0.2660599947,-0.2112916857,1.5669039488,0.9084113836,-0.6771945953,-0.1638096422,-0.4631693959,1.1034935713,-0.1055989787,0.0793656483,0.6271271110,0.7019786835,0.3838609755,-1.0738798380,-0.6967238188,-1.4907166958,1.2653003931,2.0242471695,-0.3331431746,1.3018358946,0.2629842162,0.0544529743,0.5886738896,0.7199140191,-0.2402157187,-0.3608579338,0.7160065770,1.2522007227,-0.8780410290,1.2056716681,-1.2108603716,-0.2572793365,-1.8514751196,-0.1546487361,1.5449539423,-0.2368666679,0.0098899864,-1.0413780212,0.2774117887],[-0.2326158434,1.7425477505,0.9690331817,1.6429095268,-0.5832452178,-0.1323042661,-1.0757516623,0.5345213413,-0.3577066958,-1.2440371513,-1.1071386337,1.0463796854,1.0088303089,-0.0133500062,-0.3407829702,-1.4332808256,1.2677539587,-0.6842421889,-1.5895107985,0.1031334102,0.3260625005,-0.8146274686,-0.5703064799,1.5360972881,1.1497735977,-1.0545229912,1.1621849537,-0.0205927342,0.0773499012,1.5707740784,2.3440666199,-0.0833226591,1.2771021128,0.6326636076,0.3524564505,1.0547939539,1.2561269999,0.9254574180,-0.4101853073,0.7814167142,0.5550388098,-1.8132902384,0.5165917873,-1.6028478146,0.1231929883,-0.8797899485,-2.0823435783,0.3851499259,-1.3730741739,0.3431421518,-1.2870882750,-1.0696678162,-1.9551162720,2.1060843468,0.5244371295,-1.8513286114,-1.8891903162,1.6993538141,0.4485278726,0.8354133368,-0.3720458746,-0.5648186207,0.2110529244,-0.4747396111,-0.7716717124,-1.2227087021,-0.3932309449,-1.0957214832,-0.3843332529,-0.4385857284,-0.2289460748,0.1591261774,1.8326412439,0.3320563436,-0.3171687424,1.8090025187,1.0536510944,1.5990201235,0.6480306983,0.9056814909,1.0043029785,-0.2869877517,0.1809553057,0.9514772296,-0.0422179550,0.7730164528,0.5366948843,-1.3359624147,-1.2184035778,0.5566676259,-1.3658822775,-0.4281991422,-1.7386028767,0.4616151452,-1.1900107861,-2.4713108540,-0.4954342842,0.2345984131,-1.8488981724,-2.1743729115,-0.6402022839,0.0303491205,-0.7284300923,1.9178283215,1.6713473797,0.4037494957,0.2346402109,2.0459208488,0.3015230596,0.7130191326,-0.6341357231,1.4106793404,0.2356214225,0.1327618361,-0.2230495960,-0.4091452658,0.2005991936,-0.0833660811,-0.8700286746,2.7954144478,1.7317944765,-0.6571676135,0.6018900871,-0.0520828702,0.3507285714,-0.5154905915,-0.1296486408,-0.5092809200,1.2773561478,1.0116991997,-0.6150847673,-1.4773579836,0.7563268542,0.8471134305,-0.4720822871,0.4313528836,0.0375151336,0.8350439072,0.4294562638,-0.3305866718,0.4861770868,0.6976563931,0.6272200346,-1.6637167931,-0.2475887835,1.0828968287,1.9375406504,0.0171681941,-0.2749762535,-0.7067273855,-1.8384919167,-0.6727235913,0.4818950891,0.8734824061,1.1389626265,-1.3718824387,1.0223542452,0.4262120128,0.2025973946,0.1423050314,1.4298902750,-1.3199853897,0.4568197131,1.5345735550,0.5935459733,-0.4387113154,-0.1879886091,-0.3233093619,0.4590752423,-1.5666143894,-2.3096513748,-0.8251853585,-1.0219471455,-0.0014929610,0.4029170573,1.0825065374,1.4356482029,0.6444023252,-0.1995886266,-0.2893678248,-0.0991581753,-0.1651691198,1.0189162493,-0.1947890669,-1.2835777998,-1.7958240509,1.9413176775,1.4828227758,-0.8889564276,-0.5395320058,-0.5694774985,0.0775188878,-0.7718951106,0.8657597899,-1.5139957666,-1.2930766344,-1.7993720770,-0.2752868831,0.9181541800,-0.7155758739,1.0465465784,1.2638782263,2.2021529675,-0.9961844683,0.0974123031,0.0123674851,-0.2089405060,0.9067828655,1.2267822027,0.3396924734,0.1184439957,2.0369150639,0.3253271878,1.8651583195,1.4540983438,0.5370510817,-0.7455057502,0.2601398528,1.7201685905,-1.7035471201,-0.2939456701,-0.3423474729,0.9926267862,-0.7185393572],[1.6899878979,0.0963688120,1.5225541592,1.2421973944,1.1041691303,1.5041767359,0.4050821960,0.5243073106,-0.1115907133,-0.1680535376,-0.0969657302,0.0803446621,1.2012168169,-0.4391567707,1.2089624405,-0.7686284184,0.2084608078,0.5243440270,-1.6587768793,0.4990506470,-0.6774152517,-0.4325406551,0.6804471016,-1.4003276825,-0.1406010389,-0.6582599282,1.6774594784,-0.6680455804,0.2592495978,0.8286221027,-0.5804581642,0.2537096143,0.9645189643,-0.3157314360,0.5982891917,0.8521072268,0.5036994219,0.0840232447,-0.1938900799,0.0138512980,-0.2024308294,-1.0314016342,0.0266645644,-0.6234315038,2.1404976845,-0.3796528280,0.8915473223,-0.7924473882,-0.2630101442,1.2968854904,1.0652110577,0.4945122898,0.4680919051,0.4252710938,0.0305738803,-0.7643531561,1.3036431074,0.1892702878,1.0127239227,-0.4154120982,0.1690457165,0.2398145199,-0.2753032148,-0.0394092537,-0.1768219471,-0.6261203885,-0.4798114896,2.0902595520,2.4536385536,0.2569797635,-1.3700937033,0.2429075092,-0.0302436706,0.0957037061,-0.5650098920,0.4821854830,1.5858329535,-0.3087330759,0.0644669831,-0.0620031543,-0.0495426171,0.6991023421,-0.1768588275,0.2576231658,1.0498266220,1.1218752861,2.0640258789,0.2962799668,-0.5372537971,0.0778273121,-0.6351861358,1.9494534731,-0.7365251184,0.2154163718,-0.8799818158,1.1496148109,0.3462239504,-0.7713207006,2.7176063061,0.6472253799,1.5203940868,0.0201782063,1.1566685438,-0.6539087892,0.2394919395,-1.7418583632,-1.1859192848,1.3754141331,0.5204149485,-0.1954313666,0.4072748423,1.2499068975,-0.2092721015,1.2599943876,0.4698984027,-0.1057309136,0.6285521388,-0.1283890307,0.2182361186,0.5527539253,-0.5691096783,1.1039172411,-0.0255232845,1.2810142040,-0.3833185732,0.1088434681,-0.0079321684,-0.6392134428,-0.3211532533,-0.6840104461,2.2359540462,-0.7377978563,-2.1295697689,-0.8262901306,-0.2741989493,-0.7584269047,0.2667566240,-0.1170935258,1.2125899792,1.3901118040,-0.1025477350,-0.5680760145,0.5732792020,-0.6226810813,1.0726773739,-0.7555589080,1.3011289835,-0.7514100075,-0.3121113777,0.1953123808,-0.7970280051,-0.0469507203,-1.1711391211,-0.5039377809,-0.1742249131,1.7650339603,1.0166648626,-2.3796470165,2.1885800362,0.6374771595,-1.4910366535,0.5055052042,0.2328806818,0.8630269766,-2.2999992371,-1.5238486528,0.5972530246,0.1055067182,-0.5324211717,-0.5370553136,-1.2598272562,1.6097830534,-0.7878819704,-1.7461416721,-2.0719652176,-1.1854764223,0.3427107036,0.5295251608,1.7611126900,0.9704126120,0.5100552440,-0.6306912303,-0.4613394737,0.9809895158,-1.7130533457,-0.7521519661,0.1366459578,0.5631435513,-0.1118881255,-2.4540958405,-0.0765665099,0.7642238140,0.6514412165,-0.2037922889,0.9903054237,0.3785459697,-1.6247358322,1.2173683643,0.2166759223,-0.8612121940,0.4859002531,-1.0364907980,-0.4721792936,-1.6349135637,-1.4085305929,-0.8389165998,1.2321178913,1.3330185413,0.3066735268,0.9116269946,0.6791136265,2.8482429981,-0.4379735291,-2.2370553017,-1.5435577631,-0.1815133095,1.2109670639,0.5453848839,0.4780768752,1.5647675991,2.2001993656,2.1024439335,0.8078509569,-0.3416246474],[-0.6584426761,-0.1349043101,0.3547728360,-2.2675919533,-0.2931696475,1.0089668036,-0.6153686643,0.1034570783,0.6448723078,-0.8765313029,0.4534460008,-0.3699859381,1.2004622221,-0.0722925514,-0.0325831361,0.9020727873,-0.7152183652,-0.8456809521,1.1576007605,0.6171180010,-0.0472845919,-0.9105609059,0.1960259825,-0.2576677501,-1.9657860994,0.9828581214,-0.9305165410,0.8664217591,1.2800661325,-1.1278365850,1.8148369789,0.2521477938,0.8906397820,1.5773588419,0.4439918995,0.0060209022,0.8361096978,0.4923763573,-1.1642525196,0.7681359649,0.2648225427,-1.1226508617,-0.2650801241,-1.6352653503,-1.3773202896,0.6504764557,-0.9845471978,0.5976469517,-0.0603680871,0.3551152945,0.9604079723,1.4429877996,-0.9509329796,0.4106124043,-0.8905282617,1.4084507227,-1.2862893343,1.1871342659,-0.2408435643,0.7524929643,0.9669978023,-0.0317020342,0.1478836387,0.4667394161,-1.1262389421,0.4174720943,-0.4278752208,0.0893254876,0.3123355210,-0.6510843039,3.6935560703,0.3481149971,0.9524300694,1.2629102468,0.9691338539,0.8425455093,0.4645085037,0.8258065581,-0.2780044377,1.0713196993,-0.7075175047,-0.1032134444,0.5142312646,0.5319851041,-0.0098116705,-0.1779374182,-0.2449193746,0.8509523869,-0.6144673228,-0.4817785025,-1.2756514549,-0.6126600504,-1.1293950081,-0.1908688098,1.0651584864,0.3274653852,-0.8781397343,-1.2023971081,0.5885425806,0.7167418599,0.0696014911,1.5573089123,-0.5623143911,0.3726352453,-1.0666670799,0.7341268659,-0.7441030145,-0.5995749831,0.4551900327,-0.1493398994,-0.0821552277,0.1749963611,-0.5736764073,0.6820808053,0.1590649784,1.7094728947,-1.2093794346,0.1417449266,0.0874756873,0.0445520170,0.1454997957,0.6226649880,-0.7579197288,-0.8653601408,-0.8565465808,0.0884546340,-1.1490837336,1.2844370604,1.7234747410,1.4080694914,-0.3957767785,-2.0336396694,4.2858557701,0.1105877906,-1.0418227911,1.2223359346,-1.1886203289,0.1480172127,-2.7951111794,-0.5097351670,-1.1575583220,0.9877367616,0.2623551488,1.2971171141,-0.9021095634,-0.7333549261,0.3958832324,0.0376789942,-1.4897017479,0.7928319573,0.3467888832,-0.2384968996,0.5339655876,1.0606970787,0.6961131096,0.5275928378,-1.0800560713,-0.6828938127,0.7291389704,0.4540881515,-0.9229554534,-0.2239529192,-0.8285161257,-0.7912678123,-0.0725147203,-0.1320558935,1.2266693115,-0.1467476189,-0.5098367929,0.1437175274,0.0615985021,1.1454741955,-1.6718358994,0.2883616388,-1.0447431803,1.8523039818,0.0715519041,1.2537438869,-2.2294402122,-0.9726954103,-0.8541699648,-1.2510904074,-0.2680608928,-0.0982486233,-2.7065048218,0.4315496087,0.4952709377,2.3173794746,1.7902907133,1.0368921757,-0.4934299886,-0.6587244868,0.2156995833,-0.7816968560,0.1607544422,-1.4151940346,-0.4447843432,1.6773577929,0.3867525756,1.1128669977,1.8861371279,0.9591953158,2.4224495888,0.8388727307,-0.8159924150,1.5163911581,0.6435745358,0.4092240930,1.7183226347,-0.0800361559,-1.0487719774,-0.2141771913,1.0178449154,0.7592806816,0.5728220344,0.6055765152,-0.8523482084,-0.8808542490,-0.4607036412,0.7181840539,0.5790495276,-0.3860639036,-0.6235859394,-0.3401636779],[-1.3322459459,-0.3735280037,-0.9619418383,0.9170179963,-0.0155261103,-0.7936813235,0.2904969156,-0.2934159935,1.5998722315,0.2245161682,-0.6810392141,-0.7174949050,-0.0229964331,-0.6433348060,0.8996737599,-0.1399249732,-0.9068041444,1.2186594009,-0.4671718776,-0.0194043443,-0.5684084296,-0.8678084612,0.5519516468,-0.9348483086,0.4797087014,-1.7733447552,0.5873536468,-2.0052974224,-2.0336976051,-0.2158455700,-0.2398948669,-1.9766912460,-1.7725622654,0.7912243605,0.6274369359,0.0156383421,-0.4388458133,-0.2509407997,-0.7350332141,0.0592636503,-0.2649568617,-1.1821020842,0.1051000580,-0.4178300500,-3.0862278938,2.1433157921,1.8003720045,-1.3339478970,-0.4993157387,-0.0595764108,-0.0815746859,-1.8172090054,0.9544544816,0.0266386792,-0.7247435451,0.4710936844,-0.0355378725,0.5382333994,-0.2017002255,-0.2736326456,-1.0850501060,0.4361391664,0.7275456190,-1.5133862495,-1.0869604349,-0.4892189503,-0.2241794765,-0.7801714540,-1.8050248623,0.4219087958,0.0985516831,0.7518414855,0.0099756178,-1.3204857111,-0.7362210155,-1.3625191450,1.4659570456,1.5852022171,0.9222944379,-0.6225005388,-0.4121412635,-0.1012708023,1.0839953423,-0.1868243515,1.3171703815,0.7383409142,0.7114495635,-1.3378200531,0.4559955299,-1.0297137499,-1.0663639307,1.8599836826,0.2518027425,0.3776589930,1.6198328733,-0.1385766864,0.0779930502,-0.9513822794,1.6096773148,-1.1503460407,0.2158631533,0.9789539576,-1.6471439600,0.3016359508,1.4386115074,0.7306500077,-1.8357973099,0.6295026541,-0.4132591188,-1.5983790159,-0.0675861537,-1.8248832226,0.5449636579,0.6511113644,1.1925294399,1.4464730024,-0.6434930563,2.7133228779,0.9117081761,-0.4531750381,-1.2206915617,-0.3705402911,1.2997782230,-0.8172878623,1.0327092409,-1.2872046232,0.3879989088,-0.3668969572,1.4443743229,0.0700865909,0.1676746309,1.4242259264,-0.8452426195,-0.4409632683,-1.3721363544,0.9033271670,1.2147731781,-0.7221063972,0.3519815505,-1.0255091190,0.2319916338,1.2535445690,0.1925023496,-1.2753477097,-1.1186707020,0.8844894767,0.0683436990,-1.2587802410,0.2794378698,0.8379051685,0.2669829130,0.3449877799,2.3400497437,1.0804201365,1.4391826391,0.7339784503,-0.8790752292,0.0026909518,-0.8914278746,0.4804829061,1.4641373158,0.6813607216,0.8553986549,1.7090407610,-0.5028632283,1.4325996637,0.2645716369,1.0826915503,-0.0516118817,-0.3066533506,1.0699689388,0.1768194139,-1.2282488346,0.2069570571,1.3318647146,-0.9424896240,-1.0964196920,-0.4572708905,0.4500499368,0.1980328262,-1.9181954861,-0.0174330324,0.0999595150,0.0267650168,-0.5508563519,1.4106292725,-0.2244167775,0.6556907296,-0.0054974109,-0.5443715453,-0.1919281334,0.3812180161,-0.4736113846,0.6195918322,0.7323521972,0.9373790622,-2.6295375824,0.3766013682,0.5224608183,-0.8840683699,-0.7412922978,-0.5194513202,-1.3004968166,-0.0051153982,-0.7945897579,-0.1301139295,-0.0011560072,-1.0379506350,-0.6288818121,-0.0096841361,-0.2014763355,0.6976454854,0.1354101002,1.5714323521,1.0308767557,-1.3190690279,0.8940302134,-0.5858420730,2.8178141117,-2.2886848450,-0.1681453437,0.5785136223,-0.8217912912,0.3408507407],[-0.6147227287,-1.1310694218,0.9431957006,-0.6122029424,-0.6093533635,-0.2190532982,0.2843426764,0.6691458821,-0.8678124547,1.5971509218,-0.1540083289,-2.2375392914,-0.5721747875,-0.3298413455,-1.0728195906,0.5367963910,2.1423006058,1.4925087690,-0.1332240850,0.7596751451,-0.9900237322,1.6975897551,-1.1039967537,-0.2365738451,-0.8477472663,0.5158218741,0.9538124800,-0.1393971145,0.2098560929,1.4214780331,-0.6133620739,-0.3443203270,0.7987253070,0.2965416312,-0.6252503395,0.0912849978,0.3837498128,-1.9019155502,-0.9714212418,0.8765338659,0.2010715902,-0.3521747291,1.3726044893,-0.2925174534,-0.6309738755,-0.1950662732,-0.5796111822,0.2951392829,0.3845759332,-0.1556660980,-0.2773683071,0.2328125387,0.7338165045,-0.1784263700,0.1823998541,0.7299521565,0.1716358215,-0.0308698844,0.2138171941,0.3089756966,0.7322928309,-1.0055240393,-0.4655968249,1.4510535002,0.8053041101,0.8307817578,0.5806190372,1.4012625217,0.4137353301,2.2663216591,0.9053751230,0.2676714361,-0.2638470531,-1.1524258852,-0.2055503577,2.0920567513,-0.0389869474,-0.5045543909,-0.1704713106,1.7036591768,2.3972051144,-0.3624917269,-0.0865611285,1.2785141468,0.6021286249,0.2753924131,-1.7304373980,-0.9938117862,0.2796024084,-0.3657841682,-0.0245826654,1.2244658470,0.2047881931,-0.3773700595,0.8032777905,-1.9749366045,-0.1891875565,-0.0975503027,-0.0583613701,0.5487793684,0.3212392628,0.7469420433,0.4802857339,-0.0953975916,-0.0016161953,0.4400027096,-0.3889231980,-0.2829461396,-1.2563219070,0.6948705316,-0.8324922323,-0.4046234488,-1.0711333752,1.3630745411,-0.4418192506,-1.3101440668,0.5938716531,1.8128287792,-1.2612177134,1.0351736546,-1.6114523411,1.9638142586,-1.2056689262,0.3683494329,-0.4289750457,0.5871503949,-0.4308027625,-0.4693752527,0.4237685800,1.0735077858,-0.4528256953,-0.4611294568,-0.4493314326,0.2348384559,1.4725769758,0.8286721110,-0.8574910164,1.1002789736,0.2178244442,-0.0682177767,-1.1096816063,-0.8833054900,1.1900689602,0.8638876677,-0.7919912934,-1.1155849695,-0.0368701331,-0.8000605702,0.0575532131,-0.6001101732,1.7475066185,-0.1024022102,0.7054311037,0.8884329200,1.4535988569,0.8446703553,0.6319045424,2.1480684280,-0.3688235581,0.6120865345,0.6029073000,-0.9015443325,1.8531484604,2.4086322784,0.5047215819,-0.4523896277,0.0560955927,0.0015819527,0.7949462533,-0.4127202928,-1.2027662992,-0.4916600585,0.9999223351,-0.6061544418,-1.1319777966,1.1700640917,0.6804245114,1.5015945435,2.0653812885,0.0969319269,0.2641748488,-0.0362505354,0.6237525344,-0.8102051616,-1.5553128719,0.2032443285,0.2953858972,0.5768323541,0.1966675222,0.9187663794,-0.9852730632,-1.7754678726,0.5656357408,1.5518661737,0.1246112362,0.7218389511,1.1248434782,0.7300990820,-1.7761151791,0.4202294946,-0.6030195355,0.4817877710,-0.3983127773,-1.2921808958,-0.7133482099,-0.0695136040,0.0268135220,-1.8523842096,0.3018036783,0.5316495299,-0.1636061519,0.3650557697,-0.2680281103,0.0313414261,-0.6150335670,-2.5063366890,0.8658521771,-0.8896884918,-1.4359755516,0.7071300149,-0.8117466569,-0.8193208575,-1.6796098948,0.2288485318],[1.1684442759,1.3867622614,0.3720430732,-2.2209444046,1.0519523621,-0.3739921451,1.1328148842,0.4190898538,0.5840306878,0.3622587621,1.3552360535,-0.6779456139,-0.4645721912,0.7942563891,2.9545433521,0.1648689657,0.2273791283,0.8398232460,0.1645840257,1.2705302238,1.9689812660,0.5604508519,0.2973692119,-0.3421580791,-0.7549017072,-0.9641358852,-0.0671323538,-0.2883839011,0.9323253036,-0.4751592577,0.2069311291,-0.9538527727,-1.8852221966,-0.1730320305,0.0738568977,0.7191287875,-0.1032047048,-0.5027757883,-0.6713217497,1.1065231562,-0.4910871089,0.3194410801,-0.3272990882,-0.5864269733,-1.6019154787,-1.2150155306,-0.4830319285,-2.2106018066,0.5272894502,1.1339933872,-1.6582157612,0.1045286432,0.9769923687,0.6356340647,-0.0837953761,0.8529375196,-1.1835395098,-0.0311263781,0.1229125187,0.1845661104,1.4570873976,-1.0240747929,0.2515321076,-1.5033500195,0.1378750950,-1.1685484648,-0.4524605870,1.0256450176,-1.7534796000,-0.7314636707,-0.9282721281,-0.0200786255,0.3444993198,1.2372258902,0.2347012162,0.0017177498,0.7588968277,-0.8208673596,-2.9165871143,-0.5337390900,1.1897857189,-0.4590893388,1.1943334341,0.6033994555,-0.9690102935,-0.4756906927,-0.5146775246,-0.6181944609,-0.4739496410,-0.8306716084,0.7850691080,1.0141893625,-0.1255545616,3.0548119545,-0.5503615737,0.7137859464,-1.8925294876,-1.6286408901,0.1578745246,0.2897020578,1.1416177750,1.6445769072,0.0266005080,-0.6002950668,1.8851222992,0.5006518960,0.8041281700,-0.4171819687,0.9126561284,-0.9665496349,-3.0324594975,0.1772412509,-0.7743625641,-0.7814670205,0.5761278868,0.1479105800,-0.3336394429,0.4467318654,-1.1148625612,-0.7398648262,-0.1982903183,0.2771024704,0.0037587737,-0.7214294672,-0.7772885561,0.3695692420,-0.8413255215,0.4420565069,-0.3661336303,0.2388688028,0.7574204803,1.8069114685,-0.3563021123,-0.6996699572,0.4083276093,0.9007179141,0.9037804604,-0.1237754449,1.1282645464,-0.3496119678,0.7314071059,1.5345368385,-0.2547729909,-0.8683311343,1.1891893148,1.1465867758,1.3951969147,-0.3881577253,0.5991099477,1.2029726505,1.2379013300,0.1110376492,-0.5402413607,-0.4763735831,-0.6523308754,-0.4530446529,-1.2403632402,-0.4820930958,1.9094609022,-0.8305430412,-0.5602543354,-0.3380995393,1.2139499187,-0.6730381250,1.8736811876,0.5619761348,1.2849764824,-0.7612153888,0.6084617972,0.5260947943,-0.3612978756,-0.0199031364,-0.0179262515,2.7262132168,0.1001558080,0.0873316899,0.4253403842,-0.2572331727,0.3998314738,0.5780014396,0.5289971232,1.1858935356,1.5703359842,0.6258378029,-1.3216575384,-0.3723444939,-0.7417381406,1.0053710938,0.2518371046,-2.2449259758,0.7364364266,-0.7436603308,0.2557405531,-2.1114578247,1.4893753529,-0.1315749884,0.0937679037,0.3575634956,1.3543114662,-0.5280987620,0.0921488330,1.1167320013,-0.1133646667,1.3830109835,-0.3453722000,-0.0628339872,-0.6401051283,-0.6774563193,0.0201832801,-1.3557044268,-1.7727380991,-1.3186904192,-0.8905112147,0.6380245686,-1.9156514406,-0.6751238108,-2.3304250240,-0.0731968582,2.6341967583,-0.1446866840,0.9468255043,0.1266835779,0.5929878354,0.2527090013],[-0.7906475663,-0.1088170782,-0.0180691257,1.7883660793,-0.7638135552,-0.3626271486,0.7353001237,-1.9343894720,0.9877597690,-1.2362815142,0.3096195161,-1.3171862364,-1.2322047949,1.4253273010,0.0599365123,0.4724906981,2.3247697353,0.3512053192,0.4358356893,-0.2270360738,-2.0736320019,2.1042101383,0.6262249351,2.0178065300,-0.4194451571,0.1140522435,1.6493346691,0.5421503186,0.4119198918,1.8837475777,-0.1114307493,-0.4739866257,-0.1118430570,0.3266671002,1.0846471786,-0.7667149305,-0.3911579847,0.7709604502,0.9599793553,-0.6258073449,0.4235780239,0.2291588485,-0.5419658422,-1.3261935711,1.4792988300,-0.5172199011,-1.8417431116,-0.9530127645,-0.5386762023,0.5477579832,-1.3143244982,1.2823202610,-0.8449356556,1.7781059742,0.2892118096,-0.6741756797,-0.2984076440,0.3891500831,0.0456696413,-0.7156158090,0.1342495084,-1.8112068176,0.9137851596,0.0664710850,-1.2650182247,0.2943314910,0.2052179128,-0.0621892735,1.0672193766,0.6473845840,0.4740798473,-2.3851220608,0.1769553572,-0.8922551274,0.3362334967,0.9756082892,1.1115776300,-1.6051492691,0.3320700526,-0.5360628963,-0.4085548818,-0.1155323014,-0.9104030132,-1.3911604881,-0.6406959891,-0.7174149752,0.2314108014,0.9844489694,-0.7310201526,-0.7337293625,0.1070518643,0.1595049948,-0.2646666467,-0.6786507368,-0.1914107949,1.4713796377,-0.9440070391,1.6280828714,-0.9151198268,-0.3460160792,0.0264972281,1.2194769382,0.9777076244,-0.9459745288,0.2136550546,-1.1340179443,0.6967772245,-1.0132279396,-1.6084231138,-1.2107805014,-0.7951170802,-0.9152131081,-1.5939664841,0.2302491367,-1.0035600662,0.5383255482,0.3087547123,0.0042029740,0.5027353168,0.4750975668,0.0975661203,1.3311247826,-0.4950006306,-2.6560649872,-0.9025616050,0.8317630887,-2.5742125511,-1.1284319162,-0.0401342697,-1.3941005468,-1.0090293884,-1.0960578918,1.2098025084,1.3029168844,0.0013557881,-0.2298772782,-0.7874842286,1.1899797916,-1.7895762920,0.6294350624,0.2206836939,0.1826732308,-1.4839683771,-0.6505553722,0.6233601570,-0.8338543177,0.2824832499,0.3885993659,1.0900565386,1.8766107559,0.8075032830,0.1507251561,0.8691424131,0.3986098170,0.5120561123,-0.0635230392,1.1407741308,0.9157882333,0.4682558179,-0.5680037141,-1.6270081997,-0.6732350588,0.9897184372,-0.1337077469,-0.4015781581,-0.5095635653,0.4383413196,0.1368908882,2.3518426418,0.2311750799,0.4614313841,-0.6706026793,-0.9506882429,0.6988849044,1.1486055851,-1.3775639534,-0.1598063409,0.2637604177,-0.1926568598,0.8263894916,-1.2977201939,-0.8600575328,-0.6402958632,1.6776019335,-1.9129065275,1.3457714319,-1.0432997942,0.3238283992,-0.1716978103,1.4572438002,-1.0432966948,-1.5072041750,0.0903891772,-1.2257448435,-0.6195468903,-0.8568475842,0.2043592334,0.2641375661,-0.8464022279,0.2488178313,1.0052314997,0.4492959678,-0.0295740031,0.2290060669,-0.1275434643,0.3373597860,0.1626713276,-0.6524628401,0.7707580924,0.3476366401,-0.0531140119,-0.0087814219,1.6692032814,1.6984108686,-0.1209266335,-0.8074813485,-0.1444534808,0.6372165084,-1.3039739132,-1.8125053644,-0.1793181598,0.4690181315,-2.4916374683,-1.8491213322],[-0.5576793551,1.3404135704,1.0666805506,-1.5593063831,-0.9231505394,0.9490910769,0.4438292682,0.2828915715,-0.1954534054,0.0018107980,-0.1907095164,-0.1145968288,1.9409518242,1.0856090784,-0.9538422227,-0.8621790409,-0.4659977257,-1.7791017294,-0.5010645986,-0.6775425076,0.7356759906,-1.7972897291,1.5602645874,1.2374145985,-0.0871757418,-0.6793118119,0.3375435770,0.8931931853,-0.2421072423,0.8488683701,-1.2290204763,-0.2693973780,-0.1366267353,0.0585083216,-1.7161861658,-0.5805538893,-0.9939773679,-0.2650007010,0.4637797177,0.7559719682,-0.7087855339,0.1012896299,1.1064854860,0.6333464980,0.2938173711,0.2699801624,1.7075775862,0.9169011712,1.4294006824,0.3349779844,-0.6877442598,1.5435626507,0.8220308423,-0.4070742130,0.7013475299,0.1943705380,0.0186969098,-2.3783740997,-0.3056056201,0.2207050323,0.9657750726,1.0608078241,-1.1249374151,-1.8694682121,0.7967307568,-0.1668088734,0.8929074407,0.5497352481,-2.2809424400,-0.0579407215,-1.5476003885,0.8956011534,2.7000272274,-0.2422973365,-0.0058167744,-2.7817559242,-0.8345428705,0.5110921264,0.2318991423,-0.2348987013,-1.1566245556,-0.9918349981,-1.8085412979,-0.5614330173,-0.5285215378,1.1941301823,1.4853051901,0.3416150510,1.3896061182,0.8218294978,1.3636242151,0.2253000736,-0.5104549527,0.3647124767,-1.9561990499,-0.8729413152,0.0701007172,-0.2375694811,-1.7378988266,-1.0403276682,-0.8218318820,0.3926897645,0.1339692175,0.1656624079,-0.2436903417,-0.5954623818,0.1061979011,-0.4252826273,-2.0241460800,0.2748028040,-0.0702078119,0.3288494349,1.0060845613,0.4409485757,-0.6922742724,0.2370910496,-0.5651920438,1.8939553499,1.2779444456,0.7899533510,-1.6924171448,0.9689711332,1.3714234829,-0.1442247182,0.5031644702,-0.2282197922,0.1537615359,1.8720436096,2.1535537243,0.0067972033,0.1998646408,0.2613457739,-0.1279868037,-0.2985032201,-0.3642401695,-1.3701349497,0.7696627378,-0.5171828866,0.0796941370,1.0161266327,-0.1516731530,-1.4251996279,1.0709221363,0.9388007522,1.3731763363,-0.5839059949,-0.2922695279,-1.2067660093,-1.0474648476,0.0075881016,-0.0970899835,0.7804442644,0.2210815251,0.0727487653,-1.4848616123,0.4621598125,1.5490251780,2.5317053795,-0.0357220359,0.1732001901,1.8565897942,-1.5049402714,0.9150194526,-1.2461922169,-0.6568403840,0.8576791883,-0.1446761936,-1.4876954556,0.0227297749,0.6311181784,1.8708921671,0.5190147161,0.9861489534,1.2462815046,0.1173469424,0.9257962704,-0.9750080109,-1.2085790634,-0.4388735890,-0.3076315522,-0.0882748291,-0.7719042897,1.8123188019,1.5279533863,-0.8639286757,0.7459006906,0.2987340391,-1.1948617697,-0.3614636660,0.7074557543,1.8297299147,0.5339940786,0.8260959983,1.7657806873,1.2677180767,0.1828723848,0.0288342815,0.8607107401,-1.5287158489,-0.1524372548,-0.6711853147,-0.3347103596,0.0846325532,2.3551928997,1.0067535639,0.3421706259,-0.5376572013,-0.1644151658,-1.3603549004,0.2899155319,0.6658204794,0.7675340176,0.6614913344,-0.0999178067,-0.6015158296,-0.2123781145,-1.5590256453,-0.3314684033,-0.3496010900,0.0747257844,-0.3252461553,-0.8702195883,1.9684408903,0.8418162465],[-0.3862409592,-0.8120633960,0.7752045393,-0.3333698511,0.5466673374,-0.1422412992,0.7192386985,0.4905242622,-0.1880952865,0.2143013477,1.3246078491,-1.2478592396,2.0077860355,0.5948082209,-0.2080378234,-1.5965000391,-0.2735635638,1.3294361830,-0.7973580956,1.4204609394,-1.1398384571,-0.0142109981,-1.5199676752,1.1904004812,0.3158183992,-3.0604200363,0.1238618642,0.9671947360,-0.2805726528,-2.0813367367,-0.3729003072,-0.1396138072,-0.5271163583,0.1263424456,-0.4353207052,-0.2924160659,-1.1753132343,-0.1308605820,1.1438410282,0.1119249091,0.3604819775,-1.0717647076,-0.2294389606,0.1563594043,-0.6368703842,-1.2792086601,0.0777032673,0.0775017738,1.3998067379,-0.6097219586,0.4177627861,0.4011989236,-0.5329536796,2.3956975937,0.1811924279,-0.8800264001,-0.4670289159,-0.3687879145,-2.8602409363,-0.6590387821,-0.2655385137,-0.3368494809,-1.9276156425,-1.5855481625,1.0563932657,-1.3409841061,0.3984279335,-0.2940606773,-0.5712441206,-0.0067037856,-0.6415054798,-0.4785518944,-0.1218238324,0.2206111699,-0.4235476553,-1.3250722885,1.7152470350,-1.3964438438,0.2865219414,-0.4178703427,-0.6859941483,1.0773335695,-0.4156297147,-2.6461470127,0.4012067020,1.0373991728,-1.4278997183,-0.2466971874,0.0849356949,-0.8817490339,0.5450678468,0.1807463765,-1.0566569567,0.1781830043,-0.0064033647,0.3077405691,0.6102537513,0.0652264729,0.2542940974,-0.2305352241,-1.6249768734,1.2014248371,-0.2054571807,1.1181812286,1.1214034557,1.1425615549,-1.2870421410,-0.1241030917,0.5297012925,-0.0382554494,0.3882488012,-0.6190707684,-1.5623250008,0.7568140626,-1.6866751909,0.0804848298,-1.4858338833,0.2297888249,1.4524667263,-0.5968704820,-0.0833020732,-0.2826476097,1.5136855841,1.4851976633,-1.1327924728,-0.8760012984,-0.6094191670,-1.4564394951,-0.7331520319,-0.9318554997,-0.8895848989,0.8252238035,0.1218120083,0.0662026778,0.6619882584,2.0303890705,1.0554903746,0.7169943452,0.1783896387,-1.1470445395,-0.3398191333,1.3697793484,-0.0253568981,1.1636499166,-1.2545666695,-0.1280414611,0.6935864687,0.1563157439,-0.1740721315,1.1867508888,-0.0169510152,-0.0078435130,0.7718135118,0.1622573286,0.2028463036,1.2996282578,1.9638586044,-0.8999834657,1.1722446680,0.4890742600,1.1593204737,1.3701916933,-0.2957761586,0.7029923201,-0.2342174649,0.3841736317,1.0627040863,1.0547840595,-0.3150112927,1.1945139170,0.4034687877,-0.5125347376,1.0919033289,0.4950831234,-0.5558901429,0.3558287621,-1.0803325176,1.0500856638,-0.6788198948,0.5517796278,0.7566818595,0.5830651522,0.6025416851,1.1403605938,1.9247280359,0.5235610604,0.4997765124,0.7582163215,-1.1293796301,-0.3010092080,-0.9889057875,0.6745214462,-0.8895672560,1.0809191465,0.8366023302,0.4471575320,0.6851827502,2.1633660793,1.8103870153,-0.3494542837,1.1950857639,-0.3678962290,1.7989248037,-0.7417379022,0.5473921895,1.0161283016,-0.3572591841,-1.0351872444,2.6292521954,-0.4950847328,1.0285687447,0.0339635722,-1.6448830366,-0.5516107082,0.0174649954,0.4552769065,-0.1754626930,0.1529568881,-1.2097570896,-0.8168189526,2.3861827850,0.6213911772,-0.8158392310,0.5483052135],[0.0878597200,-1.0948483944,0.2889161110,0.3388940394,-1.2642379999,-1.5323306322,-1.0071874857,1.0103991032,-2.4863183498,-1.1070255041,-1.3683352470,-0.4224180877,0.7639471889,-0.5177192688,-1.0074887276,0.1551092863,1.3369476795,0.5466856360,0.3232005239,-1.6532018185,-0.7173513174,-1.0652648211,-1.3265746832,-0.5660467744,-1.3132909536,-0.8130620122,-0.1815343946,1.2690964937,1.1019312143,0.9516880512,1.7226719856,0.4446351230,0.5170425773,1.0413635969,0.4556803405,0.1972223371,1.8935484886,-0.2192615122,-1.4354544878,1.5602838993,-0.6460107565,0.8395658135,0.8136681914,-1.7072492838,-0.7649660110,-0.2882774472,0.8429179788,-0.2447516322,0.9172566533,-0.2275722325,0.2997744977,-1.0302278996,-0.1052929237,0.3555447757,-0.0440536961,0.2784536779,-0.5446269512,-0.7629913092,0.0162314177,0.5546040535,0.9613997936,0.0616658404,-0.2609883249,1.0197192430,-0.6264053583,-0.5688632131,1.5122493505,-0.3373487294,-1.6997095346,0.5993637443,-0.9272513390,0.0842745379,0.8615155816,0.4361516237,0.0753251612,0.2608761787,-0.9517005086,0.9069390893,-1.0981204510,-0.8780823946,0.7374699712,-2.0681431293,1.6813424826,-1.0838361979,-1.1617941856,-0.2509556115,-0.5042505264,-1.1139525175,-0.2048496157,0.1186023057,-0.3202336729,-1.1404109001,0.2558881640,-1.7212871313,-1.0247824192,-1.7450982332,0.6630205512,-3.1837379932,0.4456369877,-1.1326383352,-2.2103080750,0.1183590889,0.6692360640,1.2288067341,-1.9078927040,-1.5564306974,-2.0014181137,1.4565919638,-0.3936092556,0.1973627359,-0.0117214527,0.2743053138,2.0025804043,0.1375686824,0.4163160026,-1.0364779234,1.3307567835,-1.9111024141,-1.5484838486,-0.1650904715,0.8083508611,1.1169708967,-0.3402090073,0.9838241339,-0.1346430779,1.8263845444,-1.7422721386,1.0641328096,-0.4452352822,-0.0513705388,0.4752159119,1.6146486998,0.5102171898,-0.8978606462,-0.8856185079,-1.4149117470,-1.1215506792,-0.7647787929,0.2960679531,-1.1323281527,0.6728130579,0.2450481653,-1.2277650833,0.2975145876,-2.0824947357,1.9949676991,0.1659021229,1.0105559826,0.2631520331,-1.9276918173,-2.7682528496,0.4685620964,0.2694377601,0.2202383578,-0.0813253671,-0.9782060385,-0.6891325712,0.8377320766,0.0159451328,1.3094940186,0.4035644233,1.0255818367,0.5153917670,-0.1808453053,0.1277346462,-0.0179174189,-0.9238844514,-0.5562965870,-0.3335781991,-0.1112420335,1.0880632401,-0.9739040136,-0.6805738807,-1.2655941248,-1.1422700882,0.3249793351,-0.0527833514,-1.7724030018,-2.8175151348,0.8032355309,2.1276588440,2.1617305279,-0.0527869426,-1.0185065269,-0.4677870572,0.2683751881,0.0304780379,-0.1207695752,0.0642614439,-1.6406875849,1.1186835766,1.4654172659,0.7769879103,-0.5351511240,0.3237947226,0.2958904505,-0.1936709285,1.3598756790,1.1916985512,2.2793455124,-0.0956685990,-1.3434432745,-0.1309389323,0.2118412554,-0.6874347329,-0.8510482907,0.0485115610,2.1660628319,1.8733015060,-0.4796108902,0.7988507748,-2.6731781960,-0.4153742194,-0.1474187672,0.9681109190,-0.4934926331,-2.7047929764,-0.3549762070,-1.0457595587,-0.5838010311,0.0829716697,0.0265663285,-0.9133974910,-0.0864409208],[0.2548206151,-1.3221129179,-0.9357287288,1.3260126114,-1.9737194777,1.4048113823,0.4354310632,2.7296981812,0.4620052874,0.1094514206,-0.8226955533,-0.8031890988,0.0278037265,-0.7649024725,-1.1145186424,-0.0622494183,-0.4565858245,1.1405748129,-1.0133868456,-0.5143136382,0.6725979447,-0.0492859147,1.0113819838,1.1048659086,0.6603216529,-0.3727340400,0.7112011313,0.7985103130,-0.2034807801,-0.3387661576,-2.1463022232,-1.1241475344,0.7797137499,-0.2611505687,0.9198236465,0.8049122691,0.3930318952,1.0530589819,-0.3481652737,1.6241984367,-0.2310409844,0.8785673976,-0.5285487175,-1.6051872969,-2.2492547035,-0.2764416635,-0.5564856529,-0.1424106061,-1.6674304008,-1.1508232355,-1.1337341070,0.2225996703,0.9500737190,-0.8307194710,0.2994698584,0.6526410580,0.3251613379,0.0689228550,1.1470413208,-1.4659957886,-0.0308823474,-1.1581293344,1.3566701412,0.0564573184,-1.8906033039,-0.9888930917,0.8690288067,-0.5142025948,-0.6776256561,1.2080899477,0.0295389090,0.5598149300,-0.4604727030,-1.0156501532,2.0301139355,-0.8149390817,0.4358777702,-2.2680025101,0.7438141108,1.9459930658,-0.1377315372,-0.0557070971,0.6126846671,0.4535076916,1.3437209129,-0.4962809682,0.7099696994,0.7285888195,0.5127623081,0.1487104893,-0.4102285206,0.1795057654,1.2648516893,0.8477721214,-1.3445240259,1.1553347111,1.1411762238,-0.1662096381,0.9143199921,-0.0289703812,1.4069156647,0.3577559888,-2.8750307560,0.5885147452,2.0400743484,0.7269452214,-2.0255157948,-0.7129239440,-1.8075281382,-0.0634173900,-1.1268689632,-2.0150179863,0.8664655685,-0.5846761465,-0.7784549594,1.7345633507,-2.3513545990,-1.0365632772,0.8950498104,-0.2481182218,-1.4468442202,0.0471748933,0.3714194298,-1.3437989950,0.1846590042,0.1699056774,0.0043113870,-0.2608861625,-0.6190173030,-0.6328819990,1.0290852785,-0.0337157957,0.4794288874,2.3265194893,0.6430379152,-2.3729488850,0.9642911553,0.1792990863,-0.7555418611,-1.6815297604,-0.7509630919,0.5920519233,0.5271238089,-0.3613370359,-0.8814773560,-0.4798953831,0.4807034135,1.0450673103,0.4689154923,-0.8064729571,-0.1262860000,0.5163205266,-0.0193388909,1.3125473261,-0.0543386117,-1.2151954174,0.2638210654,-0.0364308245,-0.5281606317,1.2426241636,-0.3205830455,-0.1426075846,-0.4687280953,0.1643962413,-0.3802402318,0.3136390150,-1.4041841030,0.5760033131,0.1114661992,1.5098890066,0.6955231428,0.1370026469,-0.7484128475,0.8863014579,0.6602393389,-1.9834080935,-0.5476236939,-0.1973284334,2.7404897213,0.3918314576,-1.0367541313,3.3678669930,1.2490388155,0.4868883491,-0.1266106814,-0.3812051713,-0.0928058252,-0.6301029325,-0.2929019630,0.8924237490,-1.1051132679,-0.5074141622,0.8979958296,0.6536089182,-1.9230524302,-0.1654659212,1.8929178715,1.2809594870,-0.0103674419,0.3907116354,0.7662822604,1.8371510506,-0.3340227604,2.5882060528,0.2204035223,0.5281216502,-1.7063302994,0.9104158282,-1.6062641144,-0.7288155556,0.2892820239,-0.9528056979,-1.0483649969,-1.4120820761,-0.4685563743,-0.9451056123,1.8808521032,-0.7579481006,-0.1606787741,0.5887008309,0.4289551675,0.3829198778,-0.3382299244,-1.3812276125],[0.4090642035,1.3374681473,1.2288401127,1.5761895180,-0.8516318798,-0.2723531723,-1.2398227453,0.2986592054,-0.1322129071,0.0139572183,-1.0466662645,-0.2781076133,-2.0794694424,-0.0169874635,-0.2712719142,0.0039244746,0.4328824282,-1.1713631153,-0.7448069453,-2.3252754211,0.8434809446,1.0628112555,1.0627806187,1.3218361139,-0.1161545292,1.3281513453,0.2959490418,-1.1790206432,1.4481866360,1.1699938774,-1.1192673445,0.6433444619,0.2693232000,-0.1480689347,-0.1910275519,-0.8702706695,-1.4614882469,1.8815187216,-1.4435530901,1.4433736801,0.3612532616,-1.4280672073,2.8515341282,0.6932016015,-0.9333375096,-0.3010189831,-0.5288223028,-2.0452833176,0.0003303100,0.0397834890,-0.2597131431,0.8163148165,1.0304787159,1.0927882195,-0.3876484334,0.6974347234,-0.2137241662,0.0889874473,2.0182070732,-0.6308559179,-0.4557030797,1.8793838024,1.0392211676,0.3013942242,1.0411463976,-0.7474292517,0.4135686755,-1.2054290771,-0.4202263951,0.7777327895,-0.4293861985,-1.3665663004,-0.6443052292,0.9349673390,-1.2924108505,1.2160620689,0.3794503212,-0.5429808497,1.1152054071,-0.0982226357,-0.2433270663,-0.5958116055,0.1453744769,1.3489847183,-1.0879634619,-0.7124319077,-1.0658148527,0.6417328715,-0.9150858521,1.7110761404,-0.1847843677,-0.5252919793,-0.1499496102,-0.6333470345,0.5339637995,-0.0101217143,1.5284255743,1.4889923334,2.0363731384,0.2519434094,1.1972477436,0.1169527769,1.0913250446,-0.7577394247,-2.2312619686,2.0685327053,0.9227512479,-0.4438139200,-0.7689087391,0.5014011860,-0.0705506951,0.8355823159,1.4453328848,-1.8012226820,1.9324140549,-0.3285877109,0.3485644460,-1.0819559097,-1.6279023886,0.3250808120,-2.0706045628,0.3532761335,0.1403892487,-0.1772062927,-1.8101980686,0.4101774693,-0.0899073407,1.7163796425,-1.1482729912,-0.9280102253,-0.9309043288,0.7067950368,-1.5563870668,0.7764381766,0.4183218181,-0.5949311256,-0.5479576588,-0.3660928607,0.1221372858,-0.8077557683,-0.7920037508,2.3722608089,-1.5027210712,0.0144450190,0.1528333724,0.7655932903,0.4014534056,-2.3351640701,0.4352296889,0.4105211794,0.6907932162,-0.9530596733,-0.6473321915,-1.4276818037,-0.2839665115,-0.6459219456,1.0340801477,1.1909698248,0.4201194346,2.0286815166,1.7382160425,1.2285472155,0.3203367889,-0.1061791256,-0.0371501744,0.1596977413,2.9247212410,0.4545726478,0.1455145329,1.1376500130,-0.6597639322,-0.7237402797,1.7943882942,-1.0624425411,1.0409113169,0.7168018818,-0.7416293025,1.6516425610,-1.2652876377,1.5150113106,-0.4677824974,0.3317876756,-0.1702616215,1.0125669241,0.6174751520,0.0796523988,-0.3447691798,-0.8938841224,-0.2084598839,-1.6965242624,-0.3470969200,-0.5429576635,-1.2105997801,-0.8608617783,-0.9803560376,-1.0367319584,-0.2582563162,-0.0054417914,-0.9185775518,0.0305522084,-1.5969980955,1.0509661436,0.4743795395,0.3633412123,2.1253325939,0.2701029480,0.7550885081,-1.0644488335,-0.6703055501,0.5664753318,0.2437788546,0.9583575726,-0.7638131380,-0.0029742755,-1.6142610312,-2.2065219879,-0.6328999400,-1.1739649773,-0.1321678013,0.4885062575,1.3792656660,0.9682490230,0.0666629449,-1.4227808714],[0.9389460683,0.6142101288,0.4952200651,-0.3560309410,-0.4492895603,0.8873738050,0.4932395518,-1.5655148029,-1.1107832193,1.0344661474,-0.6064270735,2.0916547775,0.8920361996,-0.9328433275,-0.5210302472,-1.3629417419,0.6946575642,1.2764936686,-1.5056940317,0.1522615552,0.5383676291,-1.5878702402,-0.4938810766,0.8755227327,0.9683454633,-1.1522424221,-0.0225333683,0.9137845039,-0.5549679995,-0.7432829738,0.5882457495,1.3948264122,-0.5563979149,-1.2109256983,0.6127566695,0.7170585394,-0.9698428512,-0.0640512407,0.9629530907,-0.6542029381,-0.4348601699,1.3914189339,0.0691725537,-0.7639746070,-1.0185455084,-0.6848862767,-0.8079086542,1.0332250595,2.0122599602,0.2306910008,-0.1354916990,0.6538707018,-0.5724883676,-1.4610589743,-1.1935203075,0.2267056406,-0.8769122958,-1.8110815287,-0.0701249689,-2.1590735912,-1.7739510536,0.7410432100,1.1597032547,1.0969405174,1.9822311401,1.7125688791,0.4174836278,0.7853913903,0.5818963051,-0.2479056716,-0.7231642604,-1.0702124834,-0.3364195526,-0.1131938845,1.1898450851,0.3929497898,-0.9904890060,1.3256819248,-1.4747922421,0.8332212567,-0.3751766980,-0.5573330522,1.6865506172,0.4411950111,-0.1255817413,2.5350151062,-0.0580064729,-1.1477249861,-0.9363329411,1.0586061478,1.6392625570,-0.5135805011,0.5801268816,-0.0548072122,-0.9464817643,0.0946878791,-0.7109422088,-0.3775828481,0.6923879981,-0.1164232716,-0.0274552815,0.9514126182,-0.4382908940,-1.1965507269,1.2529064417,0.2980563641,1.3691887856,-0.0335965566,-0.5210404396,-0.0635088459,-0.6337655187,0.4338080585,0.0755116045,1.0962269306,-2.0093507767,-2.4804511070,-0.4190257192,-1.1947860718,0.7652868032,0.1273288131,0.8648121953,0.1008700207,-2.7048275471,-0.9510456920,-1.4726403952,0.0437071584,0.0826288909,2.0108227730,-0.6876509190,1.0703777075,0.3906943798,0.5436729193,0.9461514354,0.4150171578,-1.2524062395,0.3780267239,0.4788390398,0.1191774979,-0.2176552713,0.6323692799,1.2053620815,-2.3339111805,-0.6712306142,-0.2902439833,0.9778840542,-1.2306065559,0.6382020712,1.4408342838,-1.4152365923,0.7717177272,1.0857385397,-0.0721675083,1.7905182838,0.2215190679,0.3466854990,0.4562369287,0.5969560742,2.9985487461,-1.3164579868,-0.3309495449,-0.1202616245,-1.3430130482,-0.3943373263,-0.2237624377,-1.0122376680,1.0934064388,2.1965367794,-0.9623855948,0.4112857878,-0.7151125669,-0.2869275510,0.2724553049,-0.4762646854,-0.4325575829,0.6262726784,-0.8784438968,-1.5952491760,-1.9096953869,0.3377545774,-0.7852398157,-1.2339268923,-0.6910353899,-0.6821838617,-0.4460454285,2.0578155518,-0.8237828016,-1.1603840590,0.4606530964,0.7554635406,-0.4197842777,-1.7064237595,-0.4686983824,-0.8817868829,0.2867316008,1.2538535595,-0.1737776101,-0.2247789651,-1.5630481243,0.4511615634,0.6216961145,0.0750527158,0.3924145699,-0.3754163980,2.8608353138,1.5530757904,0.0425318517,-0.1855948865,1.3114998341,-1.9979593754,-0.1392807513,-0.5736815333,-1.1196737289,-1.2200103998,-1.2405149937,1.6866083145,-0.1392551512,1.4387949705,-1.4744572639,-0.1115004197,-0.1053058803,1.5830054283,0.5361398458,-1.1185491085,0.1301326603],[-0.1323783994,-0.6407990456,-0.4196328819,-1.3846974373,1.3309514523,-2.7046353817,0.0045477180,0.8607456088,-1.1245253086,1.2274123430,1.0389893055,-1.9995344877,0.8867872953,-0.7012874484,0.4385780990,-2.4129841328,-1.8402497768,-1.1964808702,-1.0597640276,-0.4711083174,0.9958802462,-0.7689711452,0.4363094270,-0.3747068644,-0.8480607271,1.2542796135,0.2656555772,2.2228662968,-1.8119665384,-0.9371265769,-1.4262866974,0.1876880974,-1.0126883984,-2.0700709820,-0.0803233460,-0.7962924242,-0.0414651409,1.7242867947,1.1771477461,-0.8121748567,-0.1645865291,0.9184431434,-1.0123202801,-0.3763069212,-0.1460562795,0.2211588472,1.1201910973,-0.0191150960,0.7085775733,1.4897265434,-0.0689364448,2.1542754173,0.4996797740,-0.4819819331,-1.2270764112,0.9048069715,1.1497319937,0.1581166685,0.9835773110,0.8247444630,0.0351414941,-1.9824454784,0.2739833295,1.2348875999,0.7725579739,0.2573614717,0.5969483852,0.2761405706,-1.0226633549,1.7934087515,0.1439145803,0.9556096196,1.9798411131,1.5616306067,0.4073262513,0.8567109704,-0.5908481479,-0.4551822841,-0.6076927781,3.1823654175,-0.2167487592,0.2382182330,-0.1999024898,1.5419658422,0.5953208208,-0.5438700318,-1.2184040546,1.0318024158,-0.3316418827,-0.8802697659,-0.8947135210,-0.0105047869,-3.3704369068,0.4810324013,-1.9140502214,0.3950212300,1.6429020166,0.8465449214,0.4476325512,0.4394526482,0.6456232071,-0.1141211912,0.3965323269,0.4823193252,1.1300865412,-0.1235338151,-0.5488389730,0.5645562410,0.2612779737,0.3265740275,0.6936011910,-0.2548986077,1.5862183571,-0.9947738051,0.7125069499,-0.2931319177,1.1054304838,2.4689922333,0.8357470036,0.0130731547,1.4690824747,1.7760980129,1.7123028040,0.7116003633,0.0806625262,-1.2783840895,-1.2734508514,-0.5681645870,-0.6348869205,0.4688507617,0.6399840117,-0.1546345800,0.0401923917,-0.4833675027,0.7989139557,-0.5271605849,2.2883720398,0.9557285309,-2.1976850033,-0.9270659089,-0.7950120568,-0.0018317529,0.1464039981,-1.2294088602,1.3516594172,0.9153459668,-1.5380679369,-0.8945394158,0.4787234664,-2.2154150009,0.4985513687,-0.3055807352,0.3027708232,0.7394517064,-1.1887978315,-0.8729275465,0.0136809060,0.2144620121,0.1222232655,2.0651092529,-1.3659356833,-0.6718088388,-0.3926681578,1.7661414146,1.9525667429,0.5477364659,0.4586538374,-1.6999478340,-0.9211118221,-0.7009931207,-1.2132214308,0.1789101660,0.1699265093,-0.5791007280,-1.1458439827,1.6576943398,-0.0798066929,-0.5319992304,1.1932622194,1.2408850193,-0.9731871486,-1.6201738119,-0.4493438900,-0.4226911962,-0.0897471532,0.7771114111,1.4575105906,-1.3590695858,-0.2422187775,-0.0717459098,-1.5813609362,0.3929780126,-0.6107853055,1.0224369764,0.9272652268,0.2391461581,-0.7454996109,2.5418419838,-0.0836168155,0.0664021298,-2.0217483044,0.0662804097,0.8211984634,-0.5589304566,0.2068057358,0.0640166700,0.9859380722,-0.9006294012,0.8219537735,0.1385826170,0.9545242786,-0.0526152104,0.2759426534,1.2005162239,-1.2941392660,-1.3588281870,0.0705319345,0.7191971540,0.9778149724,-0.8361923099,-0.0056291926,2.0522983074,1.5833069086,-2.3987114429],[1.3274357319,0.0064704483,-0.6484159827,0.9853000641,-1.3528258801,0.9159484506,0.2959030271,1.7370500565,1.2756021023,-0.0047396058,-0.2977020144,-1.0068558455,0.3262749016,-0.2303540409,0.2995219529,-0.3161655068,0.0587377883,2.1757032871,0.7780051231,-0.6155528426,-0.6651592255,-1.9044895172,1.0170710087,-0.2350578159,0.8182260394,-0.8092999458,1.7549093962,1.1859247684,1.2325001955,-0.4970070422,-1.8003635406,0.0549518168,-0.8563482165,1.9319586754,1.1410189867,1.4234589338,1.0574718714,-0.3678317964,-0.6011454463,0.9331198335,0.4304300845,0.9622161388,0.7820424438,0.5289376378,-0.1422957182,-0.0972463861,-1.1529253721,-0.3259295225,-0.7833554745,1.3779989481,-0.8609403372,-0.1602535546,-0.0599776134,-1.4946795702,0.2707850337,-0.5179989338,0.8004550338,-1.2054712772,0.5004020333,-1.8651032448,0.5174418092,-0.9989871979,0.3034828305,0.3723188043,0.3597621620,1.8270949125,0.4796812832,-1.3419852257,-1.7499210835,-0.1308304220,-0.8859434128,-0.4169880450,0.1174671724,-2.3178379536,-1.3677680492,0.3057938218,0.4547121227,-0.3513031304,-0.2731293142,0.0115191638,-1.1899770498,-0.7389093637,1.1789582968,-0.8494232297,1.5048207045,-1.4718470573,0.0784921274,-0.8367189169,-0.7885698676,-1.6579880714,-0.3294337988,-2.6300072670,-0.4856718183,-0.2715861797,0.2706677616,0.3428797126,-1.0286426544,-0.4359491169,-0.8634329438,1.7400062084,-0.6620771289,-0.8842269182,-2.4243388176,-1.9477770329,-0.2599925101,1.3951880932,1.7454518080,-0.8353650570,0.9496917725,-1.1362177134,2.7656846046,-0.3285786211,0.1080850959,-0.3550520241,0.5351243019,0.0626995116,-0.9387552142,-0.3435013890,0.7270622849,0.6199610233,-1.6607036591,-0.6297575235,0.7958660126,0.2775168121,0.1991933733,1.5361841917,0.0672869310,0.3522060812,-0.6162402034,-1.7315293550,-0.0968044847,1.3629524708,0.0375886112,-1.2578284740,0.4546712935,-2.2027161121,-0.7068659067,0.1386501193,-0.5204744935,0.8022690415,0.1464944929,-1.0731513500,-0.0275365952,0.3999854028,0.6108346581,0.7355435491,1.8661667109,0.4773959816,-1.0730679035,0.8396475315,1.1034216881,-0.5024297237,0.7762069702,-0.0361183025,0.6873787045,-1.6265043020,0.0699692369,0.3768470585,-0.9322768450,0.0477588251,0.9751135707,-1.9907674789,0.1195630580,0.3530760705,0.3995818198,1.1568886042,-1.2285298109,-0.2479773462,-0.0089573311,0.7797752023,0.3299199641,0.9332191348,-0.4034706950,-0.3111452460,-0.4305924177,1.3256598711,1.1262835264,1.1973778009,0.1951432228,0.2563470602,-0.2257311791,-1.0581756830,0.5406070352,1.2327010632,0.3645146191,-0.9438908100,-0.5273874998,-0.0965472907,-0.7064593434,1.3101346493,-0.6243762970,0.5321087837,0.8991381526,-0.0843996927,-0.9281385541,-0.0536656491,1.8132817745,-0.6429740191,0.2113274634,-0.2402074486,1.0136674643,0.3831409812,-1.0056535006,0.6262342334,-0.0874363706,1.3416323662,-0.4109298885,-0.0141236670,-0.7411656380,2.0815262794,-0.7702627182,-0.3985530734,-0.6752582788,-0.9599013925,-0.3416412175,-1.3532416821,1.8661503792,0.1309303790,0.1339515895,-0.5823944807,1.7389822006,1.0031000376,-1.0844048262,-0.4892614186],[0.5706951022,0.7267456055,-1.5972158909,-1.3844085932,0.1251437813,-0.1611333787,1.2971414328,-0.4890547395,-0.2959007025,-0.2359022498,-1.5221960545,1.6418451071,1.8629655838,-0.6008669734,-0.6448215842,-0.6749278307,-0.2803279161,2.2114477158,-0.2965689003,-0.3168765903,1.3606915474,-0.1464030147,-1.2688233852,-0.8596234918,-0.2300802022,-1.5350259542,-1.1681234837,0.0532827750,-0.6676311493,1.1589723825,-0.2325885743,-0.1550059468,1.2478905916,0.2482963949,1.3774282932,-0.8201007247,0.7307916880,0.3846693635,-1.3935933113,-0.5091059208,0.1876038760,1.2699453831,0.0560138673,0.8535129428,0.5293108225,-1.3195542097,-0.3675530255,-0.5306439996,-1.3628910780,0.4709477127,-0.4559128284,-0.2349215001,-0.9671003222,-1.4965330362,0.3901738822,1.0340222120,-0.1132831424,-0.8353191614,-0.2977720499,0.4513333738,0.8387823701,1.0269012451,0.7130354643,-0.3210241795,0.6128639579,1.6848592758,1.6609615088,-0.0710362568,-0.4100494087,-1.2127778530,0.8047491312,-1.7605817318,2.9120903015,0.9917885065,-0.5696196556,1.1640443802,-0.3759628832,0.5348631740,0.8605149984,-0.4322288930,-0.8272349834,0.8973447680,-0.9950327277,-0.0557409078,1.0292906761,1.1893057823,-0.2426627278,-0.4762936532,-0.3401097953,1.9560794830,-0.9409772754,-0.1492427140,-1.4113221169,-1.7686339617,1.7728070021,1.4844536781,-0.6800952554,0.8788170815,0.8911567926,1.2230223417,-0.8783909678,-2.2849061489,-0.5757948756,0.9280388951,-0.7595967650,0.0587394610,-0.1624921113,-0.2155980915,0.2433040589,0.0557866134,-1.1757665873,-0.1818657964,-0.2644596994,-0.9477189779,-0.9844954610,-1.3387968540,-0.0450486951,-0.6944232583,-1.0707694292,-0.7842970490,0.6350764632,-0.7319315672,-1.6097750664,-0.4874561727,-1.2056317329,-0.3934192061,-0.9384541512,-1.2435036898,0.7331660986,-0.1061901152,-0.2536563277,-0.4899016917,0.2329620421,-0.8986451030,0.2935437560,-0.6980230212,-1.5478031635,-0.6342258453,1.7035830021,0.1028883606,-0.7894119024,-0.1351793706,0.5689413548,-1.0633289814,-0.0086180344,1.7042182684,-0.6117020249,1.2448474169,-0.6777845025,-0.2346623689,-1.9456766844,-0.0306932554,2.1096534729,-2.2844188213,0.0728366897,1.2431104183,-0.5718863606,-1.0966272354,-0.9830226898,0.4332626760,-0.6825378537,0.8575298190,1.3153325319,1.9670081139,-1.3972771168,-0.1857836246,0.1219161525,-0.5779262781,0.6101424098,0.2377535701,0.2397469133,0.2591325343,-2.0442194939,-0.6908274889,0.2541438639,1.8209025860,-0.7531303763,0.8614359498,1.5990792513,0.8102792501,-0.6263534427,0.0995728076,0.2731429636,0.7282792330,0.0114688184,-0.4514726102,0.4547981322,-1.7314876318,0.9617160559,0.2156383395,2.5990610123,-0.8828431368,-0.1147982627,0.1282157302,0.4696044922,-0.5134382844,-1.5148773193,1.6586340666,-1.0218036175,-0.0250433777,-0.2382687032,0.2340601683,-0.7567276955,-0.3305819333,-0.3038704395,0.8338783383,0.1608898491,0.4396271110,-1.8737742901,1.3429384232,-1.3435566425,1.1009212732,2.0582981110,-0.9400506020,0.7694303989,1.4895060062,1.1229710579,-0.4763310552,-1.1265016794,0.4342498481,-1.0764023066,0.1756174266,0.2279533893,0.4279670715],[0.2260316908,-0.9843562841,0.6740505695,-0.8490674496,-0.3237790465,-0.4662958682,0.1734868139,-1.4428119659,0.6804466844,-1.0618598461,-0.7360239625,-1.2143082619,0.4160400033,-1.0800274611,0.3315010965,-2.1402599812,1.2378108501,-0.7899955511,-0.1041101590,2.0045559406,-0.4333100617,-0.4725388587,0.1273653358,1.4571205378,1.3495620489,0.5959379077,0.2545638978,0.3057740331,-1.0019536018,0.5784690976,0.2518575788,0.6918970942,1.5384705067,-0.2609039545,-0.2850797176,1.3235375881,1.1674975157,-1.4240127802,0.6997848153,-0.8154026270,-0.3378392756,-1.8234149218,0.6829866767,-1.2062163353,1.6120146513,-0.7018817663,-2.2123842239,0.8546330333,-0.4547747076,-0.3642478585,-0.3706068397,-0.2988089323,0.6899989843,-0.6115986109,1.2595679760,0.2240811884,2.6741890907,-0.5876898766,1.5658192635,0.6845703125,1.0701469183,-0.3652437329,-0.2096356004,0.3791528642,1.5315600634,-0.1469570398,-1.5674178600,0.7937241197,-1.5094797611,-0.4463437796,-0.4219004214,-0.3260750473,0.3198117614,-0.9380064011,-1.2634960413,1.2424186468,1.4660786390,-0.4180395603,0.1620767266,-0.2839062512,1.7759104967,-0.8857923746,-0.5912535787,-0.5251594782,0.3382563591,-1.5086387396,-0.3405946791,-1.8012282848,-0.0873163119,-0.5842204690,1.2199001312,-1.4205188751,-0.2968263030,-1.3448718786,0.6785737872,0.7722478509,-0.2272520810,0.0527814403,0.9790414572,-1.3844666481,-0.7844890356,-0.7075823545,-0.3223336637,1.5800389051,-1.0154743195,0.3925264180,0.5471366644,0.2460810691,0.1381963342,-1.3894932270,1.9471534491,-1.5014412403,-1.0413231850,0.3324744999,-1.6873404980,-0.2129056156,1.4749987125,-1.4523178339,-3.1190896034,-1.2718390226,2.4424221516,0.9950337410,-0.0496420674,0.9837993979,1.9826014042,0.7137126327,0.3330027759,0.0999203622,-1.2657548189,-0.4869438410,0.9282532334,1.1198828220,-0.3685782850,-0.3691154420,0.2586824596,0.4915171266,-1.1458977461,0.8304197788,-0.2539549768,-0.1996763498,-0.5544936657,-0.0025407872,0.1407305300,0.4582334757,-0.4711025953,-0.2920936346,0.6538417339,1.5560307503,1.7619452477,1.2360332012,-0.1315362304,0.0920973346,-1.4905031919,1.2301188707,-0.4421279728,-0.6929170489,-0.7973194718,0.0634542629,-1.2857584953,0.0631699786,-0.0861530378,0.2839331925,1.2970875502,0.2933334410,-2.3788189888,-0.6523681879,-0.0305581670,-1.1570004225,-0.7451606989,-1.3816602230,-1.4926953316,1.2165389061,0.4167850018,-1.4080522060,-1.4262783527,1.2082084417,0.6638391018,-0.7060305476,1.1353538036,1.3180701733,0.8578649163,-0.8731014729,-0.9829578400,-0.5651094913,0.6818859577,-0.6653128862,-1.0673677921,0.8933325410,1.2560358047,-1.1704576015,1.4296464920,0.6040555239,-1.8464752436,1.3270595074,0.1488941312,1.6056982279,0.5017446280,-0.8903104663,0.1706050634,0.8819519281,-0.0030436872,1.6358895302,-1.2797946930,1.2676116228,0.1627141535,0.7670778036,1.4692165852,0.0669638589,-0.3648871481,0.4743653536,1.0513372421,-1.1020389795,0.7753785253,1.0108804703,1.0408326387,-0.5209835172,0.6818500757,0.0904824138,1.0239518881,0.6365932822,0.4699539542,-0.7060739398,-1.0360990763,-0.4540962279],[-1.3326988220,-0.5124089122,1.0929511786,0.5940018296,-0.4353291094,1.6788744926,0.7090277076,0.6970493793,0.6935103536,-1.5174489021,-0.6674986482,0.3959916234,0.0314268805,1.1867270470,0.1292989403,0.2488492429,-0.2043378651,-1.3287620544,-0.7390286326,0.8793639541,0.5443399549,0.4859343767,1.2453608513,-0.3818763494,-1.6809335947,-0.3598356843,-1.3391313553,0.6595156789,0.0735542253,-0.0190615188,0.3803745508,1.8210678101,-0.0958271772,1.8411321640,-1.3670797348,0.5427723527,-0.7703274488,-0.6727721691,2.0768125057,-0.8670269847,-1.2214968204,1.0898650885,-0.0725031719,-0.0864036158,-0.4280133247,0.4899945259,-1.1253395081,2.0763638020,0.1353816539,1.4642211199,-1.5274518728,0.9599250555,-0.7167305946,-0.1460455060,-1.6847262383,0.4852546155,-0.2923988700,-0.4610815942,1.1864080429,-0.3434349298,-1.2183084488,-0.4544598758,1.3759692907,-1.1702294350,-0.8205097318,0.0596311018,0.2823673487,-2.4523892403,0.1126904637,0.4746390879,-1.6707005501,-0.9480802417,-0.1634900272,-0.4575511217,0.6539748311,1.3743329048,-0.2066034228,1.9918875694,0.0467024818,-0.5654628277,0.5026557446,0.7354540229,-0.2884488404,-1.1007701159,0.3083924353,1.0963255167,0.1424771547,-0.1030726954,0.8625275493,-0.7215863466,-0.9913249612,0.6632069349,0.4783222377,0.3580340743,-0.1725240052,-0.6392794847,0.2011111081,-1.6484777927,-1.0174485445,-0.0872349590,1.1921215057,0.5025959015,0.2836691439,0.8681715727,0.3389258087,0.5296366215,0.6545069814,-1.9461959600,0.5430054665,0.5104394555,-0.7187666893,0.4541369677,0.9566402435,-0.0594963208,1.6902428865,0.8972370028,0.2773114443,-0.4134947062,-1.2408342361,0.4204489291,-0.4516286850,-0.2110308409,1.0829126835,-0.9779353142,1.0323063135,-1.2429711819,0.6037898064,-0.3205806017,-1.2381771803,-0.3923279941,-0.0805774033,0.4635975063,-0.2898182571,-0.2014379799,-1.4034485817,1.1136484146,0.0691273436,0.2618164718,-1.8693686724,-1.2310501337,-2.6746325493,-0.6870961785,0.4724680781,-1.7013285160,0.7355017066,-0.2741308510,2.2835941315,0.7250497937,0.7654329538,-0.5647279024,0.1644261926,0.5307115316,0.5746467710,1.9586399794,-0.5265694857,-0.4879961312,-1.4865173101,-1.1455962658,-0.4128835797,0.6693022847,-1.5804158449,1.2603547573,0.7857226729,1.2067354918,0.4648757875,0.9003841877,0.5806150436,-1.7880207300,0.5372216702,0.7575263977,-0.5589818954,-1.0746775866,-0.0021043734,-1.6893839836,1.7304996252,-1.1568057537,-0.0617405735,1.0687190294,0.5711197853,-0.4428283572,0.1552816480,0.2527465224,-1.1368750334,-0.5381591916,1.5881439447,0.2860608101,0.3805446029,-1.6672601700,0.7602344155,0.4308459461,-0.5576328635,1.4369031191,0.5004381537,0.6119653583,0.7021585107,0.3857577443,-0.7897416353,-0.6196731329,1.0033321381,0.5967955589,0.6828916669,-1.1994363070,-0.9904943109,-0.3934526742,-0.4732136130,0.2426480353,-2.0425484180,0.8007996678,0.5071797967,1.5479482412,-1.9100352526,-0.9601728916,0.9510821104,-0.0823726431,1.3400896788,0.0834527016,0.6438699961,-0.9652031064,-0.4854627550,0.1482844353,-0.9956216216,-0.3950155675,-1.4705321789,-0.6609185338],[-1.8514585495,0.1311995983,1.6940715313,-0.1337316930,0.5798290968,-0.2847125828,0.3875262141,-0.0368419625,-0.2958053052,-0.2373793572,0.3187682629,-1.4628130198,1.7015430927,0.3156693578,-0.0703716800,-1.4071842432,-0.5739857554,-0.6005715132,0.0037154576,0.3062636256,1.8498947620,1.4188458920,1.4881933928,-0.9052850604,-0.1264310330,0.3862965405,-0.8140952587,-0.3810549974,0.6935589314,0.2257216722,1.0393418074,-0.1589675099,-0.1574358940,-0.8825321794,-0.1856806725,-1.1158484221,0.0572987758,-1.7708715200,-0.1011177674,0.7603165507,-0.8440169096,-0.2493463904,1.7614215612,-1.5872883797,-0.1093700677,-1.3668683767,0.0597839355,-0.4041580260,2.0201151371,-0.8874104619,1.0089551210,1.1857368946,0.8527296185,-1.5749006271,0.4625138044,0.0153149329,1.2475622892,-0.1801164150,-0.1654497087,0.2987264693,-0.0666087642,-0.5388320088,0.5844370723,-1.0518535376,1.1758712530,2.0751616955,0.0651127324,1.0435625315,1.2183630466,-1.1029225588,-0.3894551992,0.4537330270,2.0570921898,-0.6587897539,0.7516878843,-0.3802327514,0.6569559574,0.3319684267,-0.7577400804,-0.6714701653,0.6479204297,-0.0799216107,0.8066828251,0.1515915394,0.4205271602,-1.3345781565,1.2583372593,-1.6809546947,2.8598580360,0.6408307552,1.4548523426,-2.0656576157,0.9247038364,0.1503020823,1.2612864971,0.1509641409,0.4554774463,-0.3452061713,0.7462652922,-0.1073673740,0.9654961824,-0.1182610542,0.4436274767,-0.9892361760,-0.7423120737,0.0164196324,1.1263713837,-0.9633116126,2.3793332577,0.0003982415,1.7510664463,1.4233546257,-1.7395235300,0.7623349428,0.3986560702,-0.3507902622,-0.9979709983,-0.2359415144,0.1348109394,0.1125656813,-0.6708725691,0.3356072605,-1.2268385887,-0.1155574620,1.5565766096,0.4939498603,-1.1511844397,-0.2640740573,1.2304129601,-0.6129906774,1.4380507469,0.5469411612,-0.2832261324,-1.2993297577,0.9013664722,0.2450891584,-1.5840128660,-0.9167362452,-0.8300950527,0.1757986993,-1.6784344912,0.0636152625,-0.4752668142,1.0027318001,-0.4386060238,0.4709543586,0.2046563327,0.7120774984,-1.0557335615,-0.1954088658,-0.3462091386,-1.4930646420,-0.1731721908,-0.0131932432,0.1934194416,0.6658257246,0.6255763173,0.7684777379,-0.2945936620,-0.6855698228,1.6016274691,0.4373116791,0.2726550102,0.8263636231,0.8291152120,-0.3335766494,-0.8821152449,-0.2419444174,0.3788249791,0.7710736990,0.1886449307,-0.7076955438,1.0469366312,-0.1463537216,2.6089265347,-0.6633446217,-0.2353220284,-0.2212995142,-0.7482500672,-0.2117432356,-0.6482862830,1.7328232527,0.2244552821,-0.6652676463,0.3964279890,0.5592606664,-1.9637831450,-1.8922384977,-2.4780626297,-0.1493539512,-1.4736962318,-0.5816019773,0.9401268959,-1.4714045525,-1.1942760944,0.3668976128,0.0830259994,-0.8098945618,-0.2031483948,0.7353839874,0.0058557466,0.1347817332,-1.1094521284,0.2028308660,0.2542048991,-1.1559451818,0.2734256089,0.2041602880,0.7460332513,-0.0899389461,-0.2154787630,-0.2122800201,0.9198871851,1.8418318033,-1.5593268871,0.2610054612,-0.7024561763,1.7462798357,1.4986221790,-0.8376168609,1.0504339933,1.7258865833,-1.9487813711,1.1911542416],[0.2755780518,-0.2877275944,-0.9894776344,-0.3523977399,0.3716616035,0.7340951562,0.0465161428,2.0946755409,-0.4287497699,1.2823368311,-0.1708106399,-0.3339754343,-0.3745219409,0.0234510638,-0.5941810608,-0.6525736451,1.0548883677,0.2085448205,2.1524362564,-0.2876541615,-0.8876055479,1.3150926828,0.5178136826,1.6735327244,0.2665548027,-1.4265080690,-0.4362727702,-0.3537387848,-0.4326723218,-0.2196337879,0.1769788712,-0.5016754270,-1.9468058348,-0.8031057119,0.8652901649,-0.1611579061,0.2561255395,1.7606273890,-2.1001949310,-1.3642786741,-0.2298175246,0.2183056921,-1.8855527639,-0.4925869107,-0.2614548802,-0.8557839394,-0.1850143373,1.1688899994,-0.6215562820,-0.7275066376,-0.3990510404,-0.0416359417,-1.7970087528,2.5328645706,0.2876232266,-0.9678805470,-1.0853585005,2.2810258865,0.5573899746,0.7743658423,0.6814321280,0.8439531326,0.8247879148,-1.6703401804,-0.6495918632,-0.5389128923,-1.3334633112,0.5207111835,0.6436123848,-0.6462132931,0.3580046296,1.6225868464,-0.4871959984,0.6803805232,-0.2100579590,0.1405427754,-0.5697072148,2.1073331833,-0.9879789948,-0.5540517569,1.3131885529,-0.5549680591,2.8242397308,-0.5965105891,1.5914705992,1.4811695814,0.3065227866,-1.2098796368,0.0168109611,-0.1309269071,-1.2207504511,0.6678852439,-2.1338050365,-0.6641706824,-0.3057371974,-0.0899235457,0.1791422814,0.8580896258,-0.8787598014,0.4738532007,0.6654763222,-0.8824122548,-1.6115596294,0.5279802680,0.0964465886,0.9635578394,-0.3902786672,-0.3802396059,-2.2409791946,-1.0808991194,-0.7525261045,0.2250808775,-0.3743494749,-0.1169842705,0.0614006370,-0.3293505013,0.3097124100,-0.7995985150,1.8154002428,-1.5777002573,-0.5507161617,0.6225386262,-0.3113619685,-0.9763633013,1.4898904562,0.9300459027,-0.5811542869,1.2458498478,-0.2211971879,-1.5934373140,0.0132478783,0.9656499028,-0.1373882741,-0.0728489086,-0.5899522305,0.1944191307,0.4169383943,0.2829756737,0.1644549668,-2.1832423210,0.6266455054,-0.5795871019,-0.9039760232,-1.7385983467,0.2067382336,-0.2576915324,-0.4712277055,-0.5019993186,0.1468517184,-0.9671674371,0.0666483492,0.9021647573,-0.8958581090,0.8980016708,-0.1871165037,-1.4828082323,-0.5323514938,0.0343699008,-0.0326057225,-0.0458269492,-2.2640006542,-0.2263824195,-0.0426686592,-1.0875192881,-0.0749526247,-0.1680147648,-1.7522611618,-1.0544925928,0.5329548120,-0.4137636125,1.4957363605,-1.1262655258,-0.9431248903,1.1091076136,0.3187142015,-0.4425956309,1.9129506350,0.1657711565,-0.1740225405,1.1547033787,-0.4135216177,0.0847586095,0.6157559752,-0.2677897513,1.0562356710,-0.4857402742,-1.3100956678,1.5738106966,0.3299025297,-1.1636645794,-0.5389083624,0.3641978502,1.1415319443,0.3955402672,0.1479743123,-1.5591546297,-0.2825756967,0.9462650418,-0.6458616853,-1.2211798429,-0.6016332507,0.7985215187,1.0275365114,-0.0160687435,0.4915259778,0.1202829033,1.1910266876,1.3510931730,0.9124286175,-1.5193203688,-2.6604340076,1.6008164883,0.1068808660,-1.2210118771,0.8822169900,1.0649386644,-1.4589542150,-0.7992488742,-0.1713563204,0.4389387369,-0.2636434436,-1.6290132999,1.1303752661,0.1080426127],[0.8657753468,-0.9457086325,-0.5738193393,1.2187113762,-0.5768360496,-0.8167025447,1.2510757446,0.3992554247,0.2217383087,-0.6826551557,1.1338157654,-1.2801879644,0.9553549290,0.2114500254,-1.3725106716,2.0755689144,0.3925169110,-0.4047083855,0.5372302532,-1.0917904377,0.1228743196,0.1735812575,1.3295997381,-0.9710007906,-0.0175675768,0.6346111894,0.5424829125,0.4861537218,-0.5748277903,-0.4538476467,-0.6988639235,0.3595512807,-0.3014974296,0.4904040098,-0.4762947261,-0.2976291776,0.2722819448,1.8400909901,1.1112102270,0.3468321562,-0.7620072961,0.4823642671,-0.0431677178,-1.1301577091,0.7625041604,1.0072216988,-0.7397775650,1.2135487795,0.4348278940,0.2891020775,-0.2582736611,-0.4398835003,-1.1159187555,-0.4120377302,-0.3862161338,-1.0398606062,0.5330690145,0.0991028398,-0.3123979270,1.4060606956,-0.2284294963,-0.2856896520,-0.3457508683,-1.8428200483,0.7488943934,0.1384963244,-0.2578194737,-0.1127146631,1.1032286882,0.5590403080,-0.1274744719,1.1027128696,-0.3745149970,1.4169546366,1.0180193186,0.3566385806,-0.2235454619,-0.9186764956,-1.8063689470,0.6170316935,0.8247284293,0.1426244229,0.1843836755,-1.2132909298,-0.3551035225,-1.0385468006,-1.0946146250,-0.6682890058,0.0346868075,0.7390833497,0.6645213962,-0.6030436158,-0.5423620343,0.4705986083,-1.2859250307,-0.0684260353,-0.3965751231,0.0891710445,1.4725625515,-0.5956141353,0.2861987054,0.1781372875,-0.1194024906,-0.2217310071,0.9313843846,-0.6242418885,-0.0836989135,-1.6565767527,0.7462492585,-0.2159588784,0.2116937339,-0.2267422825,-2.0483932495,-0.0378425568,0.7382618189,-0.8901466131,-0.1636219770,0.1239845902,0.3381330073,-1.2129957676,1.0842293501,0.9588112831,-0.4454301894,-0.7255096436,0.5139003992,1.0043842793,0.4047205746,2.6048500538,-0.7641742229,-0.1398930699,-1.2959104776,-1.8625189066,0.8397760987,0.0908519179,1.7395669222,-0.4368506372,-0.4642670155,0.8948456049,0.0972507000,-0.5477635264,1.8051584959,-0.6752122045,1.8902174234,-0.2260762304,0.0208178144,1.9998450279,-0.6578727365,0.1301973611,-1.1438729763,-1.0635344982,0.3119478226,-0.4038535357,1.0320700407,0.0470034257,-0.3322484791,0.8958112597,-1.0423126221,-0.2093100846,-1.0931046009,-0.1595915258,-1.0094487667,-0.2817739248,1.2365338802,0.1999276727,-1.5332267284,-0.7521827817,-0.0498312488,-0.1254887134,0.2784412205,-0.4777686596,0.1808462292,1.7320671082,0.6485695839,-1.1794214249,-0.0287689120,2.1534762383,2.1387667656,-0.1862127781,1.3411992788,1.2107347250,0.9686575532,-1.3244681358,-0.3789440989,-0.5815922618,0.3594802916,0.5171622634,-0.6009837389,0.3175195158,-2.2224929333,-0.6074861288,0.2733509541,0.0802273899,-1.0692006350,-0.9597536922,0.0113089839,-1.3741334677,0.2265929431,-0.6278473735,1.2340617180,0.0882001072,-0.3142201900,0.2227886170,0.8545964360,-0.0908002779,-0.2843574286,2.4294245243,-1.6052583456,-0.2354816645,-0.7199215293,-0.6534951329,-0.1538714468,-2.6277620792,0.9282026291,0.4786596000,0.7063364983,-0.9769585729,0.5844084620,-1.4787604809,1.5954923630,-1.5470777750,-0.0792327374,0.7591784596,-1.0835412741,-2.8589627743],[-0.0519933216,1.1929847002,0.1627516150,0.9921510220,-0.1297453046,-0.5478489995,-1.8640022278,0.2769920826,1.2735897303,0.7470535040,-0.0304872170,-0.0428298526,1.0499415398,0.4667961597,1.2053909302,-0.0714367554,-0.5911626816,3.0160570145,-1.3844703436,-0.1397884190,1.4430978298,-0.6048903465,-0.7319226265,0.0735818520,0.1856525242,-0.3724898100,2.3749258518,0.4259150028,-1.3816146851,-1.0676062107,0.0282882024,-1.3178880215,-1.5067243576,1.7033506632,-0.3256493509,-0.3136684895,1.1504480839,0.3979339004,-0.8979489803,0.6665918827,-0.2177965790,1.7094330788,-0.0203922130,0.5821586847,1.4792309999,-0.0451435968,1.9510873556,-1.4385012388,0.6818209887,-0.6088625789,1.1092604399,0.2422867864,-0.4770748615,1.6948552132,1.1704604626,0.4564086497,-1.0953822136,-0.7527635098,-1.5625915527,0.1791021079,1.8587647676,0.0916229114,0.4258229136,-0.9329550266,-0.0671339333,0.1869338453,-0.1333879828,0.6261107326,-1.5891088247,-0.2642643452,1.1548100710,-1.3183220625,0.6512417793,0.9680756330,0.9950332046,-1.0700780153,-0.2943134904,1.7982032299,0.4225844443,-1.5542390347,0.2829584479,-1.2698465586,-0.8019006252,2.0648713112,-1.2976447344,-0.2495082319,-2.9228045940,0.5553191900,-0.6955707669,-0.0912849903,0.0168256983,-0.3094053566,1.1338710785,1.4277110100,0.6253098845,1.1908096075,0.2107233107,-0.7185921669,-0.2519544363,1.7649414539,0.6261631846,-0.2532690167,-0.4259355664,-0.3002924323,0.7058870196,-1.5119395256,0.6891105771,-2.2670624256,-0.3775601983,-0.7435219288,0.6637009382,-0.6702621579,0.2118441910,2.0264430046,0.2950162590,-1.9733177423,-0.6014866829,0.7231934071,-0.5663225651,0.2073399425,-2.2557966709,0.0920352787,-0.1102471575,-1.1182092428,0.8314582109,-0.6517904997,0.5727631450,0.6002388597,1.0145868063,1.2000041008,-0.3893697262,-0.2220023125,0.0439428948,-0.0033232714,2.1105318069,-0.7621703148,-0.1955248266,-1.0612522364,0.5935337543,-0.7442343831,-0.4376539886,-1.2502030134,-1.1209059954,0.6724943519,-0.6831957102,-1.3959691525,-0.3748416901,-1.3967071772,-0.6097630262,1.0784497261,-0.0811343193,-1.4515217543,1.7069116831,-1.4116988182,0.0189672094,0.0360026099,1.5870809555,-0.8664822578,-1.3608570099,0.5752806664,0.0327789560,-0.4369576275,-0.1481619924,0.5401315689,-0.5292395949,-3.2350654602,1.1074713469,0.9541392326,-0.6092355847,-0.6445279121,-0.1951347739,0.6513268352,-0.7532553673,0.3051186800,0.9178270102,-2.5410294533,-1.6669428349,0.4259666800,0.6748510599,-0.4968488812,1.1697164774,-0.1687025577,-0.6268527508,-1.2338035107,-1.3931992054,-0.3564191163,1.4220254421,1.0918205976,1.2700427771,1.5146982670,-0.3999033272,0.0118509382,-0.3351794183,-0.4617187977,-0.2531777620,1.2234048843,0.2188955843,-1.2126897573,1.0774984360,1.5508300066,1.3986558914,0.4946941733,2.1584038734,2.4771494865,0.5729361773,-1.1950562000,0.5807843208,2.3439888954,2.1803681850,-1.5197203159,-1.3456399441,0.0533661768,-0.7593796849,-0.4989078045,0.0440334193,-1.8984165192,-0.4864933789,0.6840640903,0.2822242975,0.7824342847,-1.0771510601,-0.5577027798,1.4288663864,1.0138487816],[0.4544593394,1.0341511965,-1.6534334421,-0.5851719975,-2.5957257748,0.6972800493,-0.2410648167,-1.1962827444,0.6448838711,-0.4857121408,-0.3649063706,1.0979299545,-1.5546585321,0.1901791841,-0.4854544401,0.0275086965,0.9509277344,-0.7999033928,-0.1402400732,0.0964038074,-0.1028219312,0.5689355135,0.3108150065,-0.3040729761,-0.4496122599,0.4359531999,-0.1692104042,0.0689164475,-0.7930421829,0.8997336626,0.7421265841,0.2194436193,1.0569325686,0.0017485305,1.0466918945,0.3023291528,0.3153903782,0.8032365441,-1.1121325493,2.6082203388,-1.3207836151,-0.3867725432,-0.3457061648,1.4187725782,-0.6497259736,0.7847685814,0.2821341455,-0.1602710783,0.0889612883,-1.4676092863,0.6674157977,-1.0673069954,0.2699068785,0.1724976599,-0.3715167940,1.0822068453,-0.7078635693,0.4691339135,0.3721350431,1.1494282484,-0.0420525447,-0.4953653812,-0.4252645075,-1.8390085697,1.0680310726,-0.0759820119,1.0406930447,-0.6761109233,-0.2665938139,0.3006088734,1.1225448847,-0.9760149717,-0.0417896174,0.0025571589,0.4015204310,0.7486844063,-1.3603100777,-0.8001003861,-0.4906891882,-1.0416312218,-1.0722851753,2.4610190392,0.2289647162,0.3938124776,-0.3345311284,0.5383934975,-0.2748134732,-0.6480213404,2.1349983215,0.1050079092,0.8258822560,-0.4955825508,1.7874985933,-0.3892707229,-0.7922250032,-0.3951547444,-0.0935009569,0.1839891970,-0.8543516397,-1.0464440584,-0.7042970657,-1.1806395054,-1.2209229469,-0.2441308498,0.6465824842,-1.5547673702,-1.7660431862,-0.4821627736,0.1407624334,-0.1678617150,0.9397349954,0.2312805504,-0.0280464403,1.6767039299,-0.9470214248,-0.9351007342,-0.9674376249,0.5599937439,-0.7846594453,-0.4185580611,0.1562634110,-0.5272597671,-1.4458227158,1.7874987125,-0.8945755363,-1.2059798241,0.2898784280,-1.5602114201,1.3168905973,0.4170904458,-0.5218951106,0.0991009474,1.1913528442,-1.0922662020,-0.8239323497,-1.3484790325,0.5477755070,0.6710631847,-0.4247452617,-1.6761363745,0.4697081149,0.0555586666,1.6238911152,-0.6668378711,-0.7418172359,-0.7404046655,-0.2535289526,0.7424023151,-1.4185131788,0.1802333444,0.1009450033,0.3009157479,0.1991927475,-2.3257699013,0.6526278257,-1.6105198860,0.7812666297,-1.1860648394,0.3073795140,-1.5439826250,1.0786151886,1.2680554390,1.1521686316,1.1935445070,-0.1329169720,-0.3085395694,0.0761978850,1.8443210125,-0.2638729513,-1.5977312326,0.8382647038,0.0859807804,0.1356368512,1.1917499304,-1.0192261934,-0.3402422965,-1.1250737906,0.2373281419,-0.5993966460,0.4661963880,-0.5152217746,-0.5071100593,0.5282868743,-0.1788586527,-0.1281480938,-1.3527747393,1.0704989433,0.5577857494,0.0059412615,-0.3464069664,-1.0306385756,-0.1926615536,1.6605831385,1.3970597982,1.7178465128,0.5896280408,-2.2436287403,1.4201089144,-0.6560614705,-2.3737287521,0.0326512717,0.2078212202,-0.0501976386,-0.3021792769,1.1992337704,-0.4503636658,-0.2515912652,0.1691752970,-0.0690513998,-1.7739503384,-0.0254569277,2.1342687607,-2.2958354950,0.4292617738,1.1680651903,-0.0823156610,0.6136156321,-0.1594337076,0.3683827817,-0.0694643259,-0.4721207917,0.3415882885,-0.6719304919,0.0745203495],[0.0926815197,0.7809058428,0.8021600842,1.2536708117,1.2047463655,0.8457165956,-0.2689283490,-0.0406811796,0.7122931480,-0.0020184009,-0.7400744557,1.5476503372,0.7627727985,0.6692178249,-0.5025582910,-0.3137260675,-0.2285013795,0.3727241158,0.3642551601,0.6032511592,0.8335736990,0.1891970634,-0.3147652745,1.5904812813,1.0627577305,-0.6074346304,-1.1208091974,0.5797903538,-0.5553388000,-0.7626169920,-1.4770214558,2.1922209263,-2.2505557537,1.1951607466,-0.6032582521,1.1439125538,0.7489135265,0.8095982671,-0.3108332455,-0.3310345113,-0.8971514106,-1.5992891788,-1.4045076370,-1.4514647722,-0.5385984778,-0.9099396467,0.7821102142,0.3216437697,-0.7002050877,-1.6994129419,-1.0463931561,1.0386457443,0.1906262785,0.0693022907,0.3480342925,0.6630804539,-0.9648827910,0.1472559869,-0.0458867997,-0.9605195522,0.2654193640,-0.4124167562,1.2029577494,0.1682241857,0.1582312435,0.6462152004,0.7105493546,-0.5076053143,1.2874636650,0.7914259434,-0.3501081765,-1.3159191608,0.9087712765,0.3375329971,-0.0894550458,-3.3134696484,-1.0573728085,-0.4799038172,0.1022162735,0.0140804453,-0.4237525165,-0.0029144140,-0.4657649696,-1.2013605833,-1.8574804068,-0.7658002973,0.2583065629,-0.7350707054,0.5241965652,0.5456067324,0.5849019289,-1.0737682581,-0.3010232449,-0.4734374583,-0.3524091244,0.4543142021,-0.4802360535,1.5676404238,-0.0989632905,-1.2484837770,-0.8684524894,-2.6856575012,-1.2519989014,-0.3295105100,1.1998020411,1.5729202032,0.9265779257,1.4141265154,-0.2236066461,0.9494342804,0.8590515852,0.1530281156,1.1916043758,-0.1836208552,-1.3414688110,0.8105289936,0.2061579078,-0.5158743858,-1.1998797655,0.8480063677,-0.9648107290,1.0120599270,0.1300519258,0.1588593423,-0.1458848864,0.4267154038,1.5206574202,-0.2062469721,-1.0819133520,-0.9290098548,-1.7997145653,0.7159577608,-0.5012516975,-0.9157263041,0.1787528396,0.0459793061,0.5049911737,-0.8993741870,-3.0976221561,0.7384113669,1.5854134560,1.1512668133,-1.4262179136,-1.2774196863,1.7157492638,0.8839167953,-0.6359609365,0.8234870434,0.8796422482,-2.3434367180,0.5187589526,1.3172487020,0.2944667935,0.2227121741,0.5968134999,1.2236213684,-0.4995519519,-0.5660246611,-0.6994220614,-0.3106907010,-1.2275322676,0.7521572113,0.3444543183,1.5920861959,0.2251924723,2.2333335876,-0.0396176800,-0.1746587604,-0.1322690248,-0.4289151430,-0.7689406872,1.2304511070,0.2261408120,0.6992877126,0.8450301290,-0.6699578166,-1.3206387758,-1.4513704777,-1.8766469955,1.6361585855,-0.3785747290,0.8501096368,-0.0773385093,-0.1470464617,1.6808183193,0.3025913537,0.2117206156,0.0503673851,0.2844220996,0.3684289753,0.9366687536,0.6129627228,-0.6902558804,-0.3202367425,-1.7368016243,0.1083192900,0.2266173512,-0.2204572260,-0.8055815101,-0.5609047413,-0.5874257684,-1.4272302389,-0.4451175928,0.4001646340,-0.5392788649,0.0617882907,1.2349941730,-1.5404955149,0.6322777867,0.8958539367,-1.2067916393,-1.0665080547,-0.6116001606,-0.6186541319,-0.4733894765,-1.3370724916,-0.4977751374,0.6712226272,0.0883457363,2.0432891846,0.0867749825,0.2891819477,1.6255686283,0.5093495846],[0.8634660244,0.7550491095,-1.4651743174,1.2748829126,1.3423838615,0.0865078941,-0.7244066596,0.8706378937,-1.7124751806,0.5211428404,-0.5917902589,0.5395089984,0.0224716794,0.4386703372,-0.8138235211,-0.2059317082,-0.9192074537,-0.0604479648,-0.4525226951,-1.9594230652,-1.5452584028,-0.2895898819,-0.0253855549,0.6934217811,0.3090851307,-0.2218995094,-0.0529302284,1.2924019098,0.9603215456,0.3587602675,-0.9552350640,1.1490943432,-0.0335796215,1.6779471636,-0.3259856403,-0.1739709228,1.5214538574,-0.6064954996,0.1615271866,0.6834179759,0.3536682725,1.0816570520,0.3024944663,-0.6590800285,-0.8109408617,-0.1566258967,0.4882825315,-1.2851366997,0.6194458604,-1.6078065634,0.0334874280,0.9258344769,0.3397276998,-0.0351016968,-0.8456002474,-0.4561108053,-0.8913973570,-0.1025945768,-0.2888323367,-0.2083857954,-0.1048913971,1.9896476269,-0.4569800794,0.0361020789,1.0713363886,-0.7877628803,-1.0399837494,1.3942734003,-0.4270233214,0.3786351383,0.9357964993,0.9731655717,-0.7636753917,-1.6540989876,0.8779069185,-1.4263557196,-0.0950654224,-1.5323438644,0.0343191847,-1.1123816967,-0.2708704770,0.1493589282,0.5749483705,1.4909499884,0.0755190328,-0.9530516267,0.6965177059,-0.6865991950,-0.9001799822,-1.1961323023,0.1245036572,-0.3526382148,-0.4453476369,-1.8010739088,0.5374770164,0.2574934065,-0.3085855544,-0.0640777200,-0.8079072833,-0.5141606927,-0.5505889654,-0.1982488185,-0.3675305247,0.4208337367,0.0965360552,-2.0656948090,-0.1829688996,0.2506989241,0.2846823037,0.3001767695,0.2364291400,-0.7418822646,0.5268235207,-1.3270870447,-0.8376921415,-2.0307159424,-0.4796929359,0.7163648605,-0.4497404099,0.0741514638,1.1273583174,-1.0137497187,1.3068083525,0.0346848592,-0.8894021511,-0.9632061720,-1.5303732157,-1.1514923573,0.8990085125,0.9828225970,1.2495281696,0.5501637459,-0.7472378612,0.8517258167,0.4913127124,1.1090178490,-1.0322618484,-1.5959858894,-0.3413833380,-0.1173982024,-1.6070528030,-0.2624657154,-0.6415647864,-2.6126191616,-0.5836005211,-1.0906150341,-0.8369953036,0.2202195823,1.8754501343,1.1174076796,1.0884023905,-0.2120194435,-3.0440869331,1.4379463196,0.1025658697,-1.5558474064,0.6945379972,-0.3602610528,0.0620819256,-2.5303983688,-1.4525141716,-0.0484901033,1.0190861225,-1.1029773951,-0.1772495806,-0.0186824985,-0.4372339845,-0.7663955092,-0.2264468819,-1.8660597801,-0.0174309351,1.2895250320,2.8766248226,0.5093661547,0.6372798085,0.3782185614,-1.2775412798,-0.9037016034,0.7714995146,-0.7808824182,-1.0733382702,1.5326451063,1.6875671148,1.1831212044,-0.3538039029,0.4219298065,1.1515035629,0.1067457125,-0.6593474746,-2.3907163143,-1.6215455532,0.0232353490,-0.4400199056,-0.0634604543,0.2586629093,-2.2663216591,0.6431171298,-0.6066202521,1.8293699026,-0.6815040708,0.5783654451,-0.8462299705,-1.7036626339,0.2688787878,-0.1152593791,0.0679218322,-0.4225247502,-1.4610382318,0.5065841079,-1.9560542107,0.4685810506,-0.6444819570,-1.4162796736,0.9209262133,-0.9962162375,-0.8262240887,0.9954686165,-0.6650307178,0.9970404506,-0.1991078705,-0.6442340612,2.0496962070,0.0618126243,-1.0454064608],[-0.6637465358,1.9695299864,0.1250195652,-0.7327752709,-0.1359323561,-0.4412856400,-0.3675518930,-0.2591361403,-2.5769243240,-0.5419714451,0.2259461731,-0.1193021312,0.7141500711,-0.6071059704,0.3218783438,1.9557578564,-0.2104028761,-0.6749036908,-0.4633462429,-0.5990371704,0.9030663967,0.6569451094,-1.5796002150,1.3999508619,-1.0496129990,0.3546217382,-0.0530159920,-1.1967575550,-1.5856877565,0.0531225428,0.8695508242,-0.3861188293,-0.7244033217,-0.1855974942,1.0887128115,2.3390982151,-1.7254819870,1.5232543945,0.3753961921,0.9166776538,0.2828798890,-0.3334253728,-1.2294572592,0.3655229807,2.0233829021,0.2879462540,-1.4131339788,-0.7401593328,0.6440915465,-2.2614407539,-1.4203243256,2.6897394657,1.0188958645,0.8922799230,1.5838291645,-0.3026787937,-0.4696189463,0.7466695905,-0.6594412327,1.2971545458,1.1126279831,0.0633644760,-0.5827593803,-1.2563552856,0.1899198294,0.8947163820,0.4806729853,0.0808907971,0.8153989911,-0.5255755782,1.4518349171,-1.5716581345,-0.5564303398,-0.1252691299,0.8958023190,0.7547018528,0.6371154785,-0.1164987236,-1.7575292587,-0.8542814851,-0.7686762214,-1.0782096386,0.6460056305,-0.7257015109,-0.3034015298,0.6748173237,-3.3844981194,0.1262123883,-0.1035634950,1.4074908495,0.5247564912,-0.3472731113,0.8271560073,1.0331898928,-0.6009578109,1.1177165508,0.5174444914,-0.2727499604,1.3153707981,0.4109996557,-0.1952133924,0.3007640541,-0.5758360624,-0.5602527857,0.3082463443,-1.3120938540,1.7105407715,-1.5808700323,-0.8947734833,1.3084439039,0.5117423534,-1.4647477865,0.0225376338,-1.7247672081,0.4059568346,-0.5297921300,-0.4586297572,0.5186428428,1.1221458912,1.1941851377,0.7090333700,-0.1947846115,1.9539064169,0.2526306808,-0.7416341305,-0.3133064806,-0.3473016918,-0.0026028380,-1.0973331928,-1.1440614462,0.5836762190,-0.4917782247,2.3554165363,1.5738666058,0.5428704023,0.0142914858,-1.2510061264,-2.2476670742,2.6870701313,0.8242061138,1.8147623539,-0.3681277633,0.8724951744,0.4893138111,0.4355538487,1.4844454527,-1.4029507637,0.2526011765,0.1869145930,-2.3647630215,-0.4800653756,0.9991058707,-0.5212966800,0.1787936091,0.2968055010,-0.9514818192,0.7149151564,0.4679512978,-0.2365171462,-0.0139078815,-0.1421744227,1.4185181856,-0.8405714035,0.1105919480,-0.5550268292,1.3210300207,0.3122994602,2.4679713249,-0.6052124500,1.2478696108,0.0954830274,0.9173564315,-1.9533030987,-0.9786008000,-0.3256475627,1.0131014585,0.1920856088,0.8135458231,0.5758315325,2.8805861473,-0.5539932847,-0.1677597016,-1.1043391228,1.4870136976,-0.3989987969,-0.7816363573,0.9050074220,-0.6455032229,-0.4766314626,1.0585227013,0.3848082721,2.5556023121,-0.7868890166,-1.9005753994,-0.7796630263,1.6994067430,-0.8025813699,0.5371515751,0.4411693215,-0.1469948888,0.5937262177,-1.2914032936,-0.0362044945,0.7089365125,2.9356086254,0.0261324681,-0.0169340111,-0.0850357786,-1.1811912060,0.5618770123,0.1779046506,1.4448437691,0.1256714612,-0.9184864759,-1.1742694378,1.0258781910,-1.4049621820,-1.8019059896,1.6122846603,-0.9469008446,-0.9792156816,-1.0730874538,-1.9244215488,0.1242880449],[-1.3025438786,0.9926157594,0.0525438972,0.1790150851,-0.8310004473,-0.1794541776,-1.1104866266,-0.2170728147,-0.4860383868,-0.0960857049,-0.4243250787,-0.3056462705,-0.0738886967,-0.2791997492,-0.1564156264,-0.8399229050,1.1458208561,0.6837478876,1.6160286665,-0.1099802181,0.1030385345,-0.4512418509,0.8318041563,0.9506053329,-1.7757173777,-1.9854574203,0.3565828502,-0.3700554073,-0.6152406931,-0.9687829018,1.3810563087,0.2320157290,-0.0397417285,-0.4070185125,1.0224226713,0.6736516356,-0.1206513643,0.3238360882,-0.7017626762,-0.0897023901,1.4467226267,-0.7196639776,-2.5757844448,0.3970977664,-0.2733588219,-0.3398293257,-0.2725647688,-0.4187522829,0.3255429864,1.5826820135,-0.5754086375,0.0943736285,-0.4439761937,-0.1710792184,-0.4510376751,-0.0273558944,0.2152557671,1.4247077703,0.0805678144,-0.6716182232,-0.9144644737,1.0538321733,1.6822096109,0.2016815394,1.2589432001,0.4971260726,-0.4490357339,1.7315534353,-0.4510636628,0.1474699676,0.9106471539,0.1625524759,-1.4341237545,-0.7749769688,-0.3517271876,0.8197718859,-1.1186920404,0.6920174956,-0.4592382312,-1.0409634113,1.7210446596,0.1002967730,0.0334235467,0.9438540936,0.5847578645,0.4156909585,-0.7136794329,0.5987677574,-1.1312581301,0.4730856419,-0.9133135080,-0.4913589358,0.1282602251,0.2038312554,1.4765167236,0.5084848404,-0.2917088866,0.3517220020,0.6517812610,0.6369057298,-1.2051205635,-0.6937068105,0.0933035389,-1.1256998777,-0.8713313937,0.6497979164,1.6654264927,-0.2665706575,0.9316011667,1.0475007296,-0.4163180590,-0.2547314763,2.2949197292,-0.7823335528,0.4881402254,-0.5845407248,-0.2045453191,0.1332875192,-1.0624183416,-0.7866970897,-1.6789759398,-1.4453302622,1.1875790358,-0.9976070523,0.7905008197,2.6770126820,-1.5829631090,-1.4054129124,0.2432076484,0.5539929271,0.9461371303,-1.2753396034,-0.4961832166,-0.1548965722,-0.4539283216,0.2310051918,0.1122577861,-0.3159879148,-0.1546914428,0.9573659897,-0.9810982943,-0.0313256383,2.7361254692,1.1906349659,-0.1153252274,1.1036010981,0.1202579215,-1.8402842283,-0.9810041785,1.6234278679,1.7131366730,0.9054279327,-0.0411528088,-1.1291643381,0.9142575860,-0.1347265393,0.7485822439,-1.1155381203,0.8338834047,0.0044410489,-0.1199790686,-0.2514708042,0.2934603393,0.0102223698,-0.9591860771,-0.4407611191,2.4567036629,-1.4015727043,0.3956102133,-1.2079521418,-0.3189548254,-1.0416499376,0.3006887138,-0.2559400499,-1.5648876429,-0.5479289889,0.9294615984,-0.7143161297,-1.3360306025,1.2542421818,1.6548587084,-0.7020130157,-0.0768703371,0.0540100858,0.7866579890,1.0969753265,1.6252093315,-0.5440499187,-0.7300460935,0.5121057630,-0.1335268468,2.5226573944,-2.0427842140,0.1997319609,-1.1326947212,-2.2240474224,1.7012556791,1.6482611895,-0.0981576741,0.6183930039,0.7943008542,-1.0404632092,-1.2540209293,1.0974723101,0.3400676847,-1.7999849319,-0.5149138570,-0.1294233352,0.2476343811,0.8727270365,0.0723300055,0.2251117826,1.4213521481,0.1532366574,2.6486074924,0.1167246997,-0.4805921614,0.3937732577,-0.9335401058,-1.4705386162,1.1323963404,0.0321465358,0.9102234840,1.3990534544],[0.4905488491,1.0244057178,-1.6630067825,-0.3994784951,-0.3401413262,-0.3454761505,-0.0475850701,0.4195607305,0.1365256459,0.0498067103,0.4276661575,-0.5091152787,1.7417982817,-1.5653539896,-0.1948224455,1.4137815237,-0.3796446621,-0.6055162549,-0.4563442171,0.9318701029,2.0537283421,0.7551569939,0.1563235372,0.4450795949,0.2379143834,-0.8415094614,0.1656335741,2.3153066635,0.6370079517,0.6397714615,1.8445942402,-1.9889305830,-0.8778341413,-0.2747113407,1.4435391426,0.3957173228,-0.1096090600,-0.8891395926,-1.2789076567,0.0116699347,1.0117725134,0.3835023642,-0.5092669725,-0.0680650771,0.9934516549,-1.1740164757,0.1203645915,-0.2318346053,1.2257432938,0.1388643533,-1.1649215221,0.5175055265,0.2016887665,-0.3390520215,0.6372504830,-0.8617182970,0.3922643661,-1.1405359507,0.2569530010,-0.4474111497,0.0448078550,-0.2087467760,1.6296164989,-0.0296883136,-1.2102688551,0.5677373409,0.0009645627,1.3742536306,1.1661435366,-1.1338286400,0.7868994474,0.4527561665,0.2778838873,0.6215717793,-0.9352455139,-0.3557741642,-0.2322310805,-0.9921100140,-1.4324650764,0.6023430228,-1.1864603758,-1.0976997614,0.8242604136,0.1729867607,-1.1119036674,0.2921111584,0.0507399924,0.6631184220,1.2487870455,1.4756360054,-0.7475041747,0.5366319418,0.1717128307,0.4651950300,-0.0540710352,-0.2115193903,-0.2900735438,0.7111625075,-0.1079788059,-1.6012848616,0.3743604422,-0.0169442687,-0.9762680531,0.3801927269,0.1165735647,0.4455650449,0.1271342188,-0.5844639540,-0.3766897917,0.5482994914,0.2216329426,-0.5788456202,1.3263802528,0.6550196409,-0.7106993198,-1.1114065647,-1.2377024889,-0.9838712811,-1.2531896830,0.4703316987,0.0795856193,0.1470217109,-2.2242867947,-0.6280842423,0.3862711787,-0.2358767688,-1.0647182465,-0.2574432194,-1.4735280275,1.1477354765,0.8785506487,0.6193646193,0.0978459045,-1.0593994856,-1.1262053251,1.2516567707,-1.4200671911,-0.3345801234,0.2187118828,-0.1080835685,-1.2882615328,0.7958784103,-0.6218714714,-0.2906329036,-1.4599126577,-1.1279275417,-0.9053555131,-1.0980588198,0.1504440308,1.1394290924,0.3642742634,-0.1579071581,-0.9825789332,-2.1468319893,0.0083792591,0.7393897176,1.4622263908,0.1394752264,-1.5417640209,-0.2361054122,0.7392612696,0.8448073268,0.5637169480,0.0669474825,-0.0490082353,-0.1221956983,-0.0868485868,0.0036756690,0.0283785351,0.9145377278,1.3506923914,0.2616867423,2.4223361015,0.3500064015,0.5549456477,-1.2616983652,0.2947146297,0.8523635864,1.2621779442,1.0800333023,1.5876103640,-1.9408265352,0.2570088506,1.7748100758,-1.6168836355,-0.9866812229,-0.0237126984,1.4818791151,-0.2154493928,0.0660840422,-0.6549353004,0.0116032315,0.4643039107,1.3120938540,1.0078448057,0.4175274968,1.7760876417,-0.2337876260,-1.1625869274,-0.5908479095,1.1094672680,-0.5619422197,0.2336943001,0.5299299359,0.7005144358,0.9736207724,-0.4927281737,1.4668943882,-1.0239930153,-0.0770049617,0.7348200083,0.2523647845,-0.2476083785,0.9233145118,-0.0127188815,0.7690323591,1.3147650957,-0.0070816535,-1.9878875017,-1.5712317228,0.7105522752,0.5278979540,0.2327578664,-1.2572900057],[-0.1001996845,-0.1817302555,-0.3798861802,2.3356816769,-1.6914775372,-1.1774146557,1.2655371428,-1.5351748466,0.7914337516,-0.6201122403,-0.3057745695,0.1126926094,0.6709082723,1.1037712097,0.1790883690,0.6736957431,-0.4431216717,0.8462934494,-1.1090414524,1.5360138416,-0.6972581744,-0.4059306085,-1.8871546984,2.0013945103,0.0334238783,-0.2949000001,-1.3512518406,0.6522342563,-0.7653886080,2.2844948769,0.8337475061,0.3221344352,-0.5305902362,-2.4431941509,0.8988071084,-1.0610204935,0.8465533853,-0.1923000813,-1.3131548166,-1.5066918135,0.6025505066,-0.8746331334,-0.3928633332,1.1593921185,0.1034096554,-0.3026155829,0.4849913418,0.1135137603,1.6720339060,-0.3694227338,-1.4470843077,0.7193400264,1.0495063066,0.0959530175,-0.6709577441,-0.3056570888,-0.6415513754,-0.5251059532,-0.6818318367,0.3147960603,-0.4887236953,-0.9638143778,-1.1264886856,-0.4230669439,-1.3753056526,1.4559468031,1.4765875340,1.2504413128,-0.4936591685,0.3515748084,0.2022776455,0.9292532802,0.8853971958,-0.6428562403,0.2419474721,0.6410183311,-1.7333700657,0.9614998698,-1.3203852177,-1.3492267132,0.1798286587,-0.5843139887,-0.3930367231,0.0276744086,1.1229655743,1.2574152946,0.5836015344,-1.8359272480,0.0817378908,1.3239192963,-0.4715490639,-1.7266557217,0.3238314688,-0.2207290232,1.0489692688,0.9688036442,0.4010276198,-0.3819698393,0.1248324960,-0.0166277848,-0.9547690153,-0.4959850311,-0.5902844071,1.8655183315,0.0865403786,-0.6640671492,0.5868545175,-0.2740457356,0.6969856024,1.4407171011,-0.1463461518,0.8920519948,0.3390126824,-0.0442162864,0.4553397000,-0.6127408147,0.3977233469,-1.7088475227,-0.1218002737,0.1388552785,0.4274210334,-2.0180900097,-0.1601694822,1.6350325346,-0.4815888107,1.4442528486,0.1157166064,0.2844043970,0.1153378934,1.5692323446,-0.2419142425,-0.5717042685,-0.1694380194,0.8772971630,-0.6982527971,-1.3658602238,-0.7143605351,-1.2969588041,0.3269412220,0.8335607648,0.4132811427,-0.0035941750,0.5502625108,-1.2089297771,-0.4677336812,0.4932954907,-0.2773734927,0.3793223500,-0.2142858654,1.4164570570,-0.0279256701,-1.4179240465,0.9715839028,0.3082867861,0.3752784431,0.9423430562,1.3675347567,-0.6815214157,0.5373745561,-0.4753558040,-2.2577373981,-0.1867559999,-0.4563869536,1.1202806234,1.1302267313,-0.9389406443,-0.7574245334,0.5267103910,-0.8715178967,-0.1430505812,-1.9246406555,1.8750369549,-2.5997474194,-0.8381543756,1.5143324137,-0.5030655861,-0.5764772892,1.0302103758,-0.3347219825,-0.0474841893,0.0351716802,-0.8788820505,-0.7732199430,0.1133688316,-2.5959587097,0.0701987743,-0.9390360713,-2.1080806255,1.2983478308,-1.7246512175,-0.6754827499,0.1538365036,-1.6606847048,-0.6705852151,0.4778603315,-0.4059040546,-0.4880833328,0.2577357292,-0.1147083938,-1.2455481291,-0.5508422256,0.3719415665,-0.5658921003,0.5245736241,-1.1117680073,0.0928273723,0.0218124297,-0.3010913432,0.8775908351,-0.2169012874,-0.8590472341,-0.3291174471,-0.5996320248,0.1607422233,0.4372586012,1.5891768932,-0.0624184571,0.7525455356,2.5343019962,0.7479329705,0.8544104695,0.4639654458,-2.2253143787,-1.0417768955],[-0.0010615842,0.2537246644,-1.0748190880,-1.3404649496,-0.2640745640,0.3361747563,0.2933667600,0.7682360411,0.1233829632,-0.7152925134,0.7858465314,0.2019590288,0.5854577422,0.9132457376,-0.3124257922,-0.6049257517,-1.7231265306,0.7158805132,0.7621665597,0.4707882106,-2.9688909054,1.0587520599,-0.1631650478,0.0793522522,0.1336528510,1.0867087841,-0.5280027986,0.2855147421,-0.0667707622,-0.0035638765,0.4216847718,0.5443183780,0.2461861372,-0.1809778661,-0.8805108070,-0.1438997090,0.3478353322,0.8228884339,-0.5667278171,-0.4600711167,0.6176699400,2.3540959358,-0.8628973961,0.9088322520,0.6932885051,1.0956834555,1.0052500963,-0.4408296347,2.0047576427,-0.9447867274,0.9418581724,-2.8230834007,-0.3410270214,1.3397841454,2.1751554012,-0.0430340357,0.4309962392,-0.0155594079,-0.6336702704,-1.5577777624,0.1310043186,-0.4537871182,2.4866731167,1.1449908018,-1.5187022686,2.0298993587,-1.5842804909,0.2514736354,0.9716297984,1.7878285646,-1.2653026581,-0.1265586317,-0.1470164657,-0.7129474878,0.4800057709,2.8610293865,-0.1240404844,1.0638287067,-1.5669780970,0.8731978536,0.3589456379,0.2653459311,-0.5578699112,-0.6540279984,-0.2132600099,0.0867705345,1.2787603140,1.3769998550,1.2984076738,-0.0174389817,-0.0729143023,2.0917215347,1.5554282665,-0.5141366720,-1.5810707808,1.3129007816,-0.3244799376,0.7690106034,-1.3313903809,-0.1335199028,-0.1562579721,-1.2194261551,-0.0974660292,-1.2154504061,-1.0153028965,-0.9581129551,1.4123982191,-0.1402374953,-1.2557382584,0.6547852159,-0.9383400083,1.0114247799,-0.7177949548,0.8543280959,-1.2188569307,-0.4752506912,-0.6735640764,0.0805893838,2.9027316570,0.0258577615,-0.2556987703,0.0209448729,0.2968816459,1.4511637688,-1.0195590258,0.3860342205,-0.2076390535,-0.7492733598,-0.2612061799,0.2322777659,-0.0773505419,-0.8092628717,-0.3922980130,-3.0591294765,-0.7700003386,-1.5647056103,-1.1685365438,-1.1834771633,-0.7745651007,0.6517621279,-0.6666411161,-1.2065341473,0.8831382990,-1.3153542280,-0.0880349576,0.5406996608,-0.2709668577,1.2457705736,-0.4655426145,0.5110310316,1.6702328920,0.0507460684,-0.6514158845,0.4647938311,0.5530887246,0.1307623982,0.0550314337,0.3614056408,1.2339098454,-0.5118358135,-0.1313847899,-2.1826581955,-0.1817007065,0.0118881213,1.1555757523,-0.5332305431,-0.7610717416,-1.1782858372,-0.1647985727,0.6561669707,-0.7847946286,0.1190716103,1.7935775518,-2.1962480545,-1.1013448238,-0.8153190017,2.0948555470,-1.3238399029,1.0909674168,0.7659215927,-1.5969687700,-0.5738921762,-1.4300869703,0.7761298418,0.1169549376,-0.2567743063,-1.3555506468,-0.0286894329,0.3896198571,0.1592593044,-0.1276924163,0.7255066633,0.8067373037,0.9591949582,-1.6959720850,-2.4361855984,0.6476137042,-0.7000209093,-1.7535645962,0.0522507057,-0.9426590204,-0.1567277461,-0.2495714128,0.3145912290,0.7321646214,-0.2300930619,-1.1882681847,0.7567175031,-0.7333874702,1.4053243399,-0.3211281300,-1.7157675028,-0.7250712514,-1.2069982290,1.3195250034,0.2129811645,-1.8306376934,0.0801075250,-1.1997556686,-0.3797031641,-0.1439069211,-1.3755490780,-1.9295804501,1.0497505665],[2.1053328514,1.3196328878,-0.0395816118,1.2614026070,-0.0005954857,-1.2713522911,-0.1604739726,0.4342354238,-1.0333087444,2.0633223057,1.0521577597,0.3827154636,0.7366225719,0.4496108294,0.0940372646,-0.7754995227,0.1078162566,-0.7633399963,0.4329693019,-0.7188794613,0.6669287682,1.4806238413,-1.2925173044,0.3922462165,-0.1607157439,1.3540416956,-0.2693899870,-1.8579820395,0.0982898772,0.9795159101,-0.0613298379,-1.5851001740,2.1930634975,0.8165221810,-1.4409997463,-0.8420747519,0.8619980812,0.3620576262,0.0261950083,0.1866548657,1.8038645983,-0.2240944952,0.7091770172,0.7871082425,-0.6826594472,0.8542404771,-1.8168696165,-0.7128713131,-0.0538153574,0.1904992610,1.2892400026,-0.9406535625,-1.7142122984,-0.4913613498,0.6450148821,-1.1126114130,-0.2288775891,-0.4744396210,-1.3762102127,0.5254834294,-0.5287557244,-0.5262985229,-0.1062486768,-0.6348058581,-0.9365763664,-0.1497449428,-0.1949767023,0.2172982693,-0.3022725284,0.8047858477,1.3903118372,1.3883538246,-0.7042567134,-0.1121719480,-1.3455646038,0.3870194852,-0.0965950713,0.9518412352,-0.1091744900,-1.1043176651,0.6279343963,1.4129952192,0.5325927734,-1.6628987789,1.5105077028,-1.2438029051,0.8583120704,0.2078500837,0.5437254906,1.4735174179,0.2169694006,-0.1485404968,-0.7130903602,1.4223324060,-1.0756270885,-0.9242228270,-0.7942515016,-0.6728418469,-0.6729523540,2.3121747971,-0.0744164959,-0.2179881036,-0.4866858423,-0.8224667907,1.1551148891,0.7958337069,-0.3938231468,0.2431697249,-1.9529821873,-0.2154482901,-0.2813555598,0.0592441037,0.2069272697,1.7651711702,-0.3149431348,1.1901835203,0.9350205064,2.1286346912,-0.1006811112,0.0827367827,-0.6280285716,-1.2208617926,-0.6480045915,-0.6513798237,1.3888345957,-0.0509894826,-1.5703502893,0.5715800524,0.9913051724,-0.1888523996,-0.3661287725,-1.8164740801,1.6539181471,0.2706396580,-2.3285901546,-0.6339191198,1.0078198910,-0.2167435288,2.8713746071,-1.0747338533,-1.0448617935,0.4725149870,-0.4177502692,1.1482754946,0.7925708890,1.6601495743,0.0192887355,0.2908635139,0.2582688928,0.7782739997,1.9146323204,0.7430780530,0.0700267404,0.0750506967,-0.8305863142,0.3035894930,0.2465319186,-0.2787216604,0.5338457227,-0.6506549716,0.9350888729,-0.1797624230,0.7599012256,-0.3284780979,-0.8421573043,0.1916376352,1.1291347742,1.5996108055,0.8662851453,-0.1602092385,1.8883359432,0.3574674428,0.2051007152,1.5985630751,0.8055735826,0.7127355933,-0.3533496857,-1.4823664427,-2.0089395046,-0.9939237833,-0.6550972462,-0.0262048952,1.2165206671,0.1078934073,-1.3689708710,0.1513076723,-0.4456702173,-0.0554245599,0.7030171156,-1.2767373323,-0.7908762097,0.2892021835,-1.2196539640,2.0823202133,1.0406847000,-0.2916181982,-0.3322509229,-1.2407140732,-0.1473717541,0.5597736239,-0.3266303241,0.8758299947,-1.4328693151,1.1302080154,0.9464007616,1.0138847828,1.8907960653,-0.7582595944,0.1726988256,0.1735392958,-0.5599799156,1.8208374977,-0.7746500969,-0.6911838651,0.0445401333,1.2483352423,-0.7943223119,0.0519079305,-1.5848653316,0.8621872663,1.2930183411,2.7784893513,0.3236986399,1.4388288260],[0.7258377075,-0.8048277497,0.2800495028,-0.0499605834,1.3089578152,-2.4130134583,-0.2753242254,-0.7478988767,-0.6888538003,-0.8414610624,-0.8301863074,-1.1255892515,-1.0282196999,0.5207557082,0.2261744589,0.3474826515,-1.0355030298,2.4439811707,-0.8515596390,-0.1384771913,-1.0451205969,0.6859315038,-0.9893190861,1.7343578339,0.1348031312,1.1066228151,-1.0953452587,-0.1847550869,0.2814736664,-0.6099718213,1.6427118778,-0.1525585204,-1.4401333332,0.5616949797,0.7036054730,-0.3839696050,0.0347902104,1.3739222288,-1.3430659771,-1.5973757505,0.3135956228,-2.0644445419,-0.5595622063,1.2138389349,-0.0590524115,-0.2654310167,-0.0230365172,2.3398439884,-0.4614900947,1.2380152941,-0.2797059715,1.4746888876,0.2444090098,1.7653659582,-0.4687747955,-0.3881003559,-0.3427613378,-1.1154838800,-0.9173331857,-1.2664217949,0.8875035048,0.6352165937,-1.1627904177,-0.6139476895,0.5397555828,-1.8344066143,-0.2994301021,0.2945496738,-0.2418930382,-1.9963649511,0.3640803993,-0.1844701171,-0.6950638294,-1.4857170582,1.1242765188,0.7085320354,0.2723022401,-1.5546014309,0.1953181177,-1.5684660673,-0.6086485386,-1.3542300463,-0.4276607335,1.4777511358,0.0993386284,-1.8139836788,-0.7365110517,0.6629828811,0.2414772660,0.5044490695,1.0643922091,-0.1175867841,0.0788819045,1.0972431898,0.2484007776,-0.0112973209,0.7241719365,-1.8626170158,0.0347400159,0.1793256998,-0.8870456219,-0.0319259167,0.8181232810,0.6993884444,1.2714951038,1.4938253164,1.2755190134,-0.7280711532,-0.0053603342,-0.6611225605,1.2958151102,1.1125426292,0.0500253364,1.4127457142,1.2092069387,-0.6113958955,0.5983562469,-0.7948383093,-0.0255965069,0.5676263571,-0.4503745735,0.8968400955,-0.5737510324,-0.8267991543,1.5909577608,-1.5717008114,1.0053443909,-0.1096132845,0.4106061757,-1.1032598019,-0.0910399705,-1.2157471180,0.9933787584,0.3237740099,0.1134621352,-0.1503658146,1.2782334089,-0.7552730441,-1.2234134674,-2.0203626156,1.0978006124,0.7394206524,1.2319457531,0.5734107494,0.4472402632,-1.9491572380,1.7882524729,0.7341278791,-0.3616535962,0.6778038144,-0.5595866442,-1.1381254196,1.4615677595,1.6242483854,-0.5140312314,-0.2640652955,-0.0112548228,-0.3889789283,0.6068689227,-2.8274755478,-0.0085445140,0.4904544950,0.1616977453,1.0789258480,-0.0618563630,1.7721537352,0.3887664080,0.9643759727,1.4623794556,0.1064803451,-0.8805568218,0.2923339605,0.9559921622,0.3187932968,1.0508491993,-0.2954924107,-1.0405685902,-0.0598033965,0.9603899717,2.1053576469,1.3079080582,1.2512524128,-0.6764339805,0.4047291279,-0.8339336514,1.8518990278,0.9332556129,0.7688630223,1.9514070749,-0.3171782494,-0.4678969979,-1.0559155941,2.0887012482,-1.8049836159,0.0430028215,0.3988151848,0.4500122964,-0.2574366927,-0.3913790882,-0.6121475101,1.0972211361,-0.0593496226,0.3291324973,-1.7988839149,-1.0083388090,-1.6112517118,0.1174412668,-0.3404005766,-0.4726842642,-0.4491213262,-1.0615438223,-0.2107420564,-0.3817735910,0.2995887995,-0.9482329488,-0.2399285883,-1.4830079079,1.4051352739,0.8220074773,1.0181708336,0.7705841660,-1.4086661339,0.1062365994,-0.3218088448],[-0.0925237611,-1.0620304346,1.6652784348,0.0058519463,1.4551757574,-1.4438506365,0.2192535847,0.3335919678,-1.2820657492,1.2886893749,0.3952491581,1.1653424501,-1.0537530184,-0.0074237022,-1.2312995195,0.4738241136,0.2731259465,0.7465436459,-0.6233760118,-0.2805426717,1.5110511780,1.1256532669,-1.1806602478,-1.3495339155,0.3354279399,-0.1391374320,-0.8745224476,-1.7112647295,-1.4185328484,0.1687638760,1.9717078209,1.1529743671,0.6822891235,-0.4956736565,0.0578047819,-0.7183652520,0.5551437140,0.3473408520,0.4653676450,-1.8341126442,2.2764511108,0.4477431774,-0.6344168186,0.5948296189,1.5689496994,-0.1255524755,0.9398848414,-0.4700069427,-0.0290891398,0.9742378592,-0.8186925054,0.4190974534,-1.6216185093,-0.3389663696,0.5089208484,1.5303941965,0.3098924458,-0.7619076371,0.3587762415,-1.3091235161,0.8170292974,-0.6985152364,-0.1834008396,-0.1097707674,1.0684025288,1.3673287630,0.0024511134,-1.0490896702,-3.0076134205,1.4514784813,1.2511434555,-0.0860626996,0.5120401382,0.6248494387,-1.5006062984,1.8949669600,-0.7988373041,0.0659881309,-0.2443456352,-0.5105406046,-0.2946806252,0.8995938897,-0.6719914079,-0.9645525217,0.4793780148,-0.3133210838,-1.4314814806,-0.9944772720,-1.4454166889,0.0671993122,-0.5245435238,-0.3248140812,-0.0862735212,-1.2868779898,-0.2069968730,-0.8418411613,-0.8098770380,-0.0245797615,0.2288850546,0.4888761342,-0.9548777342,1.2134466171,1.6297017336,1.0540798903,-3.5775067806,-0.5092924237,1.5152270794,1.3439284563,-1.3064225912,0.5239573121,0.6371377110,-0.5815712214,1.4705703259,0.2761515379,-2.2635154724,0.8119996786,0.2082707137,-0.3286717832,-0.2771964967,2.2118780613,-2.1735358238,0.1249354333,1.4037474394,-1.4499368668,0.3003337979,-0.5304105282,-1.7573840618,-0.6856645346,-0.5914506912,0.7638018727,-0.2675991952,-1.0980226994,-0.4089368284,1.1221373081,-1.0884951353,-0.6319181919,-1.2549594641,0.7170437574,0.0045071896,-0.6356463432,2.3410551548,0.8706154227,0.5790865421,-0.8792712092,-1.1271067858,-1.8982055187,-0.6447094083,-1.5107764006,-1.7840856314,-0.5934742689,0.6970037818,0.8130237460,1.6075373888,-0.1049850583,-0.2260528356,-0.8203536868,-0.2570720613,-3.0439462662,-0.8317464590,-0.0221328903,-0.7338784933,-1.1955124140,2.2275626659,-0.6152293086,1.0770606995,-0.8655061722,1.4497618675,-0.2080348730,0.0465875715,-0.6258221865,1.1146214008,-0.4552742243,-0.5624456406,0.8417857885,-0.5142645240,-0.2130188048,0.2848901749,-0.4078902900,1.4543926716,0.9406331182,-0.4867027402,0.0767036080,-0.2662543356,1.5841546059,-0.0371828750,-0.3927924335,0.5836333632,0.7966336608,-1.4907039404,-0.7028891444,-0.7504969239,-0.7478713393,-0.9128293395,1.1214281321,-0.9350854754,-0.4669199884,2.3918759823,-0.2142266184,1.8552978039,-0.9463956356,0.4829190373,1.2582088709,0.4328311682,1.1292302608,0.3805397451,1.2985144854,0.6846257448,1.6851937771,1.7153955698,0.8266667128,0.3266970515,-0.9034817815,-1.9895290136,-0.1561795175,0.7916913629,1.2313185930,-0.0866586864,0.8667936325,-0.9921289086,0.2599111497,-0.5031987429,0.2345968187,1.1069636345,-0.6858910918],[0.6509312987,0.8513160944,-2.5334670544,-0.0781184584,0.6531443000,0.2202208340,1.1924302578,-0.0056578512,0.9925175905,0.4677110612,0.4470631778,-0.7054750323,-0.3973045945,-0.4325635731,-0.6860013008,1.0001733303,-0.9480412006,-0.8789937496,0.6653664112,-0.6843322515,0.8633398414,-0.5817058086,-0.6112054586,0.8448106050,0.6149102449,-1.7258707285,-0.2807382941,-0.1708089411,1.5577105284,-1.2645984888,0.9760804176,0.5516930819,-0.5967000127,-0.3854245543,-0.1739615947,-0.0207591373,-1.2616332769,-0.4005171359,0.3221673667,-0.2869417071,1.0295534134,0.1294983327,0.6947427392,1.2161408663,0.9053342938,1.4678857327,0.9029391408,0.5970921516,0.7944867611,0.4798160493,0.5198258758,1.2812861204,-1.1480890512,-0.0881863534,-0.2476950139,0.4717128873,-0.6225486398,0.2505447567,-0.6700253487,1.4420573711,-1.6963062286,-0.7611740232,0.6617693901,-0.0086765150,0.5258134604,-0.1621825248,-0.6918660998,0.7783997059,0.0281068888,-0.1641781330,-2.5166752338,-1.4063049555,-0.4924475551,-0.6959780455,0.9172568917,-0.5700587034,-1.9835411310,1.5791540146,-0.4925122857,-0.0458450317,-2.0372576714,-0.9087883830,-0.6633548141,-0.7705920935,-0.3299379647,-1.0611827374,-0.1742917597,1.0259675980,1.1829209328,-0.9411280751,-1.0046323538,-0.7954505086,-0.4666687250,-0.1476753652,2.2875802517,-0.2079681754,-0.2876192033,1.2012648582,0.6918829083,0.0862621292,-0.7218672037,-0.0940898657,1.8701169491,-0.0919047073,0.4451364577,0.5785190463,1.0394281149,0.4340084791,0.6246455312,-1.7443118095,0.0707430169,-0.2417886257,0.6761582494,0.2193481177,0.0652367175,0.2281389534,-0.1589985937,-0.0161593799,-1.1481174231,-0.8527640104,-1.9987218380,-2.2161722183,-1.1947005987,1.8645793200,-0.5195894241,-0.7349023223,0.1043193266,-0.7930381894,-0.5610723495,0.9559897780,-0.1279041469,-0.0052697291,-0.2292125374,-0.8965917826,0.1681797057,-0.5898171663,-0.3136174977,0.3532792330,-0.8734504580,-1.3299645185,-0.7033492327,-2.1503171921,-1.2717120647,-1.0888218880,-1.3648161888,-0.1491083056,0.5447972417,-1.7824438810,0.6282010078,0.1593699604,-2.5262157917,-0.7317644358,0.6677094698,-0.2354516834,-0.7564174533,-0.0117295058,1.3500658274,0.6012318134,0.6721284389,0.4980325699,-0.7003022432,0.5756027699,-0.5126936436,-0.0571699105,-0.7423155308,-0.8981233835,0.4272961020,-0.0257868152,0.8800768852,-0.8634132743,0.4586406350,-1.4525709152,1.0027289391,-0.2363725603,0.7175219059,-0.8529829979,-0.4932558239,0.1452093124,0.3464656770,-0.5518008471,1.5215642452,-0.3498942256,-0.8275839686,-1.7511603832,-0.5457261801,0.8822903633,0.0703879967,-0.9963659644,1.2804361582,2.6976993084,0.4033238292,0.8038191795,1.2802057266,-0.2588358521,-1.1650398970,1.1707892418,-0.5047108531,-1.0936511755,0.5469737649,0.0263632089,1.0759764910,-1.0194599628,-0.3734528720,-1.9364116192,0.0224650707,1.0462511778,0.4847756624,0.1901340932,1.2200636864,-0.5608323812,0.0520195700,0.0540600084,-1.3133441210,0.3604013920,2.1903557777,1.2696301937,-1.7450065613,1.4612462521,1.1277128458,-0.9614953995,0.1591152400,-1.0061297417,-0.3439558744,0.5855348110],[-1.4501737356,1.5458877087,-0.6284275055,1.8623217344,-0.2068486512,0.0354135484,0.3395684361,0.2642773390,-0.7321042418,-0.5713635683,1.0891259909,0.3595553041,-1.1960923672,-0.3504222929,0.4999573231,-0.5098363161,-0.8443207741,-0.3192293346,-0.4329998493,-1.4102517366,-0.2385727018,1.1790127754,0.2475124747,-1.0066397190,0.9068338275,-0.5423150063,-0.3165667355,1.6968829632,-1.1119974852,0.4216455221,-0.8087496161,0.1220505908,0.1811276525,0.2846711278,-0.9815238118,0.0810643435,1.3072894812,-1.5053919554,0.8824597597,-0.9424833059,0.5472854972,0.1866397262,-0.3782856166,-0.6489279866,-0.1960052550,-0.7824270725,1.1514835358,0.9470567107,0.6862918735,0.4898491502,-0.2240593582,-0.5082946420,1.5246375799,-0.7205083966,-0.7252399325,-1.2068340778,0.6562124491,0.8725742698,-1.3504531384,0.2655869424,-1.1543271542,-0.3576900065,-0.3631928563,0.0644443333,1.0960409641,-1.2485544682,-1.3851460218,0.4644928277,1.5118932724,-1.6827667952,-0.1037741601,-2.8227734566,-0.4538195133,1.8951827288,0.2947148085,-0.4407174885,-0.6552804112,0.8836204410,-0.7145662904,0.2673783898,-1.0053929090,0.0218670834,-1.3796296120,0.3507533967,-0.2442570925,-1.0691617727,-0.7232266665,1.7802512646,1.4755240679,0.2832114100,-0.1803559959,-2.4272570610,0.9166278839,-2.0436441898,-1.2705802917,0.5730054975,1.5823352337,1.0499491692,-0.0399254858,1.3576145172,0.5052609444,-0.0344939381,0.7515062690,0.9314097762,-1.3302785158,-0.2806506157,1.6494643688,-0.6945965886,1.4947701693,0.5611131191,0.1360976994,0.1329657882,0.6428858638,-0.8688668013,0.5326808691,0.1221128181,-2.0173878670,0.2001792789,-0.4099375308,-0.6947799921,-0.2808746397,-0.1188127324,-0.5621849895,0.1748513132,-0.6217819452,-2.3000860214,1.0840605497,-0.7496613264,-1.5219525099,-1.7810281515,-0.8950442672,-0.0703199431,0.3249710202,0.4448671937,0.0597899482,-0.3028272986,-0.8436756730,-0.9046571255,-0.3266875148,0.4347041845,0.3949435353,1.1973904371,-1.1560224295,-1.0398107767,-0.0699188486,-1.4301743507,-0.0146584222,1.3570151329,-1.6681015491,1.2011176348,-1.0629158020,-0.2667770386,1.9444168806,0.8705314398,0.1332209855,-2.6169438362,1.2772220373,0.4033893347,-0.6000118852,-0.9993430972,-1.0177826881,1.3402198553,-0.4007156491,-0.1191185564,0.4988934398,0.0814124867,1.3495470285,0.2988345027,0.5763264298,-0.2183794379,0.4989469051,0.3067570627,-0.2668756247,0.0877958536,0.5085925460,1.4055000544,0.6140383482,0.0896825865,1.3617206812,-0.3267118037,0.6306826472,0.6890478134,1.1796201468,1.0196118355,-0.5822272897,-1.0949366093,0.0220815465,1.0912129879,0.8769557476,0.9900035262,0.9397202134,-0.0994466692,1.6669824123,1.4517176151,1.3211481571,0.2491372228,0.0155243585,-0.6893531084,-0.1140175387,-1.8416621685,-0.2381248474,0.7401183844,0.3654841483,0.2882166505,-1.0167877674,0.7473244667,-0.1891875267,0.9808660150,0.9236414433,0.6881241798,0.0554025508,-0.0573162176,0.5692203045,0.7792938352,0.6142871976,-0.6288341880,0.2108489275,-0.3727882206,-0.4873711467,-0.3820485771,-0.3673192561,1.5540332794,0.2947099507,-1.5285689831],[0.3804260194,-2.0671718121,0.0550171919,0.8531175256,-0.6018936634,-0.8865386844,-0.5568744540,-1.4733337164,-0.8281208873,0.3987645507,-1.1531097889,-0.1092760414,-0.9467664361,-0.7997373939,0.3220396340,-0.3594762981,-1.1803499460,-1.3892723322,1.6811511517,-0.2709219158,-1.3034292459,-1.0962494612,0.1192781776,0.3181969225,0.6427437663,-0.4708162844,-1.8047537804,-0.1646451503,-0.2703527808,0.6318225265,-0.4536555111,0.1989347637,0.8032560945,-1.3178848028,0.6166307330,0.2845878303,-0.9893379807,1.2858288288,0.7459064722,0.9172755480,-0.4152849317,0.6127344370,-0.1464545727,0.3386917412,-1.1799244881,0.5358649492,1.8083715439,-0.9430469275,0.4336068034,0.8175193667,0.6452999711,0.8691123128,0.2442887574,1.8094760180,-1.6751033068,0.0677766576,-0.9925397635,-1.0871496201,-0.2437263578,-0.4215831161,-0.1992474645,-1.3104773760,-1.4547755718,-1.1687017679,0.3781206310,0.5475878716,0.9851617813,0.4662987888,0.3638120294,0.2394635975,0.9616629481,-0.0387977324,0.9407829642,0.0011507085,1.8726496696,0.1954480559,0.4769034386,-1.0875762701,-0.7599243522,-0.8566319942,-1.9922207594,-2.2812983990,-0.2973401546,-0.5694825053,0.0664883927,0.4117467999,1.1683961153,-3.0562281609,1.7612978220,1.7013229132,0.2869657576,0.0255660973,0.3154593110,-0.2584732175,-0.1614930183,2.3610413074,0.4873589873,-0.5125331879,-0.3015183806,0.3679594100,-0.3626063764,1.0938476324,-2.5488677025,-1.3180434704,1.4499435425,0.7009496689,-0.1576300114,0.4049410820,0.1778909564,1.3396008015,-0.2206843495,-0.2286064029,2.0505385399,-0.4772749841,-0.0627959520,0.0254990458,-1.3595477343,-1.3190525770,-1.0695114136,-0.6161350608,1.0919641256,-0.3321318030,0.7260578275,1.1332423687,-0.2503430545,1.1677489281,0.7886906266,0.2164149135,0.7647698522,-1.3076148033,1.1607283354,1.8137441874,2.8209869862,-1.1829620600,1.2793636322,-0.4161130786,1.4560747147,-0.8382224441,-0.2064589411,-0.3601056635,-1.9282368422,0.2948355377,0.6405059099,-0.9806084633,0.0274974220,-0.4893453717,2.7329032421,1.4204442501,0.6902386546,-0.4438398480,0.8475421071,-0.8242413998,-0.2056256086,1.2381081581,1.8283888102,-1.7836816311,0.7437072396,-0.2981782556,0.4307036698,1.2446408272,-1.5674799681,1.7146155834,-1.9383628368,0.3435726762,-1.5765607357,0.0504746065,0.8115888238,-0.3472369909,-1.1612019539,0.3186337650,-0.6062232852,-0.7585208416,-0.5607734323,0.5222164392,0.7790774703,2.6089880466,0.6626429558,-0.2798112035,1.1184885502,-0.1381485909,0.0895912275,0.7735747695,-1.2175766230,-0.2237391025,-0.1026925892,2.0417473316,0.7975260019,1.5041879416,1.0822825432,0.7253676057,-0.8952884674,-1.8488601446,0.1530598551,-1.9314616919,-1.6300439835,-0.8163452148,-0.5146376491,0.5783777237,0.0899859294,-0.2165891677,-1.5324651003,-1.3432455063,-0.5275440812,-0.6659939885,0.0604406931,-1.3241438866,-0.9218361974,-1.9925515652,0.2451740056,-0.3967302442,0.1276450008,1.2230695486,0.0030670676,-0.2247872502,-0.0633816347,1.4691959620,0.5458295345,-0.5807732344,0.9379773140,0.3422805071,1.5076049566,0.6746079326,-0.1290075630,0.5062861443],[0.1298588663,1.8845261335,0.9324017167,-1.0008316040,0.1478935778,0.5123214722,-1.5427743196,-2.1174151897,0.0422072709,-0.3248087168,-0.4121863544,-1.5678561926,-0.7964280248,0.7105121613,0.2873294055,0.6519631743,0.5355852842,-0.6085993052,1.1440550089,0.9712348580,1.3347989321,0.2167384177,-1.2789026499,1.2023334503,-0.4959734082,-1.8249573708,0.3743883073,0.0709712729,0.2931367457,1.6866817474,0.1300894916,-0.1589677930,0.9902415276,0.1049473360,0.6208986640,-1.7520277500,-1.6000163555,-0.9802316427,0.0425251871,2.1325788498,-0.9563222528,-0.4039032757,-0.6859712005,-1.1440685987,2.8847551346,-1.4577374458,-1.6533602476,1.5437217951,-1.6310477257,-0.3676165342,-0.5928305984,1.3807523251,-0.5501603484,-0.5219319463,-0.0825198963,-0.3111062944,-0.7352650762,1.1713025570,-0.0221288241,0.6046815515,0.2135297805,0.1448385417,0.9258138537,0.6199473143,0.8385660052,2.1954982281,1.4447660446,-0.2872459292,0.2114256471,-1.0960211754,-0.8197528720,-0.2152145207,-0.1540306658,-1.6386111975,-2.5233354568,1.2656054497,0.9129872918,-0.6085659862,-0.6617404222,0.6074315906,-1.9679963589,0.7266260386,0.2714606822,0.1947142482,0.5195682049,0.8111485839,0.3197706044,-2.0950548649,0.5762258172,-0.8431512713,-1.2846280336,0.9035053849,-0.3686189353,0.8176126480,0.6898447871,1.2300739288,3.2548313141,0.9898021817,-0.7532128096,-0.9045040011,0.5490559936,0.8705734015,0.7328756452,-0.8265963197,-1.5101081133,0.3340276182,0.5381116867,0.6491934061,-0.1450466812,1.7645809650,-0.7833189964,-0.5206372142,-0.3905070424,1.1084893942,0.5302432775,-0.7692154050,1.2692584991,0.9881228805,-0.5279560685,-1.0720012188,-1.6803950071,-1.5877646208,1.9102419615,0.3213319182,-1.6464880705,-0.1613440216,0.3271096349,0.5641773343,-0.2901338339,-0.6286158562,1.0368025303,1.4191545248,0.3205574453,0.8432882428,-0.4522081912,1.1206735373,0.9856461883,-0.1103538498,0.3788451552,0.8254184127,-0.9615396261,-1.7476845980,0.3499236703,-0.8517538309,0.7227854729,0.9791315794,0.7726531029,0.6913672686,-2.0398874283,-1.3073374033,1.6918557882,-0.8667309284,-1.0342205763,-0.3692653477,0.5241845846,-0.4410466254,2.1805980206,0.5975066423,-0.6366894841,-1.4811692238,-0.3947469294,0.1086023152,0.1037365198,-0.5710659623,1.2462812662,1.2071619034,-0.6543464661,0.3778489232,1.4369262457,0.8684048057,0.9227760434,1.9248412848,-1.0600451231,0.5242900252,2.4939887524,-1.4329419136,0.5927869678,-0.0053195218,1.4319832325,-0.6679624915,0.1154298410,1.2070343494,0.1147793829,-0.6715605855,0.0334689096,1.6597380638,-0.9170382023,-0.5053157806,0.1027670279,1.1581686735,-0.5237666965,-0.4403850436,-1.6496279240,0.7673044205,1.1748074293,-1.4838258028,-1.6586328745,-0.6399556398,-0.5451463461,0.2890629768,-0.3449325264,0.5661980510,-0.2247092426,-0.3704339266,0.4762049317,-0.5505967736,-0.3622225821,0.8294441104,-1.0188047886,-1.3211250305,-0.5238136649,0.0719858110,0.5811215043,1.3551915884,0.9022732973,0.8348205090,1.7601321936,-1.2035567760,-0.1610284895,-0.6985741258,-0.8050824404,1.6160351038,-0.2635310590,-1.2230753899],[0.0442702919,1.4457842112,-0.2831886113,-0.5522205830,1.8225749731,0.5256469846,-0.0092465924,-1.1489993334,0.7133934498,0.7125946283,0.4428358972,-0.2494825572,-0.8214784861,-0.3348659575,1.7764191628,0.3546428382,1.4700632095,1.0542569160,2.1549079418,-1.7109314203,-0.4482811987,-2.1268780231,2.0661754608,0.1601428241,-1.0449494123,-1.0156296492,0.6733638048,0.9290160537,-1.0412443876,0.3149002790,0.7868452668,0.8227619529,0.1526217610,-0.1421793401,0.6412929296,1.3321976662,1.2902623415,0.0031562971,-1.4625291824,0.2808877230,0.5652723312,1.1053085327,-1.2109456062,-0.3344887495,0.9515791535,-0.9712296724,-0.4897898436,0.5153864026,-0.5969720483,-0.2086329162,1.0944644213,0.1678696126,1.6910209656,-1.6885894537,0.8107523918,-0.1487727761,0.8241059780,2.5300707817,-1.1119675636,-0.0557166263,-0.9830359817,-0.0218584090,1.2924665213,-1.5760532618,0.2903743386,-0.0360350795,0.1165624261,0.5244975090,-0.1452632844,-0.7145482302,0.9750678539,0.4187014699,-0.1647199988,0.0827688649,2.8011438847,1.1906450987,0.5398116112,-0.6014398932,0.4039287567,0.2727628350,1.3142924309,-0.5986911058,0.3755066097,0.3850517273,0.5240681767,0.1509334147,-0.2982423306,-0.9690389037,0.1482082903,-0.5794170499,1.1058530807,-0.6659970880,0.5248563886,0.6544551253,-0.1016000733,1.7019340992,0.1335662603,-0.3516343236,0.7148193121,1.2314994335,-0.9258267283,0.0363452137,-1.1703430414,1.7012137175,0.2636000216,2.5404133797,-0.0304493718,0.4889936447,0.7376376390,-0.3831209540,1.7555421591,0.2273544669,-0.8490986824,-0.1827587038,-0.7989795804,-0.8379164338,-2.0694205761,-0.5316456556,-0.2463656515,-1.3631684780,-0.7051585317,-1.6138539314,1.3198417425,0.3702345788,0.0753858536,-0.0132740363,-0.9950203896,0.6775140762,-1.5192737579,-1.1931970119,1.0480457544,1.0526509285,-0.2511567771,-0.2708320022,-0.9000599384,-1.0730860233,-0.8332417607,0.6814241409,-1.1012221575,-0.5161744356,-1.9944025278,0.5473926663,0.2162017673,0.6118625402,1.6970700026,0.9290080667,-0.6630474329,1.9621838331,0.6916124225,-0.5143244863,1.3465787172,-0.1014270186,1.6098065376,-0.4065241218,0.2456360906,-1.0644011497,-0.6824391484,0.9376380444,-0.4972450137,-1.3954156637,-0.7931784987,0.3863891661,-0.5077642798,0.1169658527,-2.2643694878,1.2981832027,-0.0722546279,1.3718563318,-0.9511761069,1.1634547710,-2.4079461098,-0.2811202109,-1.4747558832,-0.7960328460,1.6813354492,0.1580020636,0.4906089008,-0.6557968259,0.1914050281,0.8196141720,-0.5575736165,-0.1627945155,0.2597014308,-0.4878998101,-1.5364331007,0.8920765519,0.1526391953,0.2662674487,-1.8847845793,0.3603782356,-1.0658118725,3.2456190586,0.2545815408,-0.0740487799,1.0587692261,0.4594543278,-1.2144246101,0.8521980643,0.1920867115,-0.6966613531,0.9356654286,1.4123430252,1.9032716751,-1.1575750113,1.2239912748,-1.3523010015,1.3675932884,0.0028783996,-0.1164716333,0.0636274517,-0.1263964623,0.6503781676,0.4069949985,1.0436509848,-1.4743063450,0.1385995448,0.1789027303,0.0112371799,0.2460647076,-1.1011744738,-0.5300943255,-1.6115192175,-1.9061876535,-2.5654923916],[0.0496643037,1.0157072544,-0.2320692241,-0.2537081838,-1.9836074114,-1.2122704983,-0.8500339389,0.1666402519,-0.5580587983,-0.7058231831,-0.6347995400,0.2047363073,0.2284586877,-1.1299531460,0.7173625231,1.1171007156,1.0772426128,0.0177433770,-1.2954846621,-0.5299506783,-0.5749281049,0.5966389179,2.7219719887,0.6849224567,0.8479590416,2.0062556267,-0.0862625539,0.9452161789,0.8045234680,-0.3388015628,-0.2429846227,0.6233448982,1.2148296833,0.1562897563,-0.3656148314,0.1104548946,0.9932194352,0.5684428215,-0.9437820315,1.0924483538,-1.5989911556,-2.0403063297,0.8947772384,0.3354680538,-0.8832319379,-0.9793969989,1.6826566458,-1.0103543997,-1.2948708534,0.0603040457,-0.3768114746,0.6062066555,1.3397293091,0.4506835639,0.4326622188,-0.4533344805,-0.3971926570,-0.6595103741,-2.4956014156,1.7504210472,1.3513900042,-0.6080017090,-1.0249046087,-0.1988066733,2.1393291950,-1.8256448507,1.2898792028,-0.9707172513,-0.6307196617,-0.3292931020,-1.3354488611,0.3118260503,1.2624188662,-0.3525214195,0.0506781153,1.5740100145,0.9165746570,-0.3342539668,-0.2621529996,-1.2329554558,-1.5453443527,2.1995866299,0.6019462943,-0.3987810612,0.5707879066,-0.4209889472,-0.0048142010,-0.0332821496,-1.5761977434,-2.1236410141,0.4128189087,-1.1925446987,0.4046672285,-0.3922690153,0.9129861593,-1.2042829990,-2.5270962715,1.1464778185,-1.0409102440,-0.2294565439,-0.1564095914,0.2098131180,0.4423882961,0.6708213687,0.0011970691,1.3835297823,-2.1146876812,0.4195026755,-0.5974859595,-0.7789434195,-1.1835699081,-0.4481227994,-0.5926176906,-0.5601764917,-0.2906090021,-0.0740258768,-0.4800468087,-1.0527486801,-0.7800763845,0.3000288010,-0.3290872574,0.9112530947,0.1389426142,1.1165764332,1.5068144798,0.5325995684,0.4274400175,1.1449474096,0.7173746228,1.2336890697,0.6168799400,-0.5105078816,-0.4883999825,0.2610085905,-1.3890395164,-0.2097653002,0.1856197417,-0.8605422974,-0.3273102641,0.2982771695,-0.0288848169,-0.9086425304,0.3567273021,-0.5856960416,0.7619949579,-1.1993529797,-0.4374361038,-0.0871637315,-0.8370314240,0.2818605304,1.3145585060,-1.1522951126,-0.1624574959,2.4466850758,-0.2131539136,-0.3643190265,1.1231597662,-1.4967659712,-1.3696490526,2.5906090736,-0.8502928019,-0.0154352458,0.0708632097,-0.0892392695,0.9884635806,-1.4502840042,1.9888542891,0.0451550037,0.2025863230,-0.3004295826,-0.0864960775,0.2483033687,-0.1879534423,-0.3177562356,-0.5728502870,-1.3048775196,-0.5676960945,0.9221876264,1.3602030277,0.2550102472,0.5108075738,0.6118447185,-1.2540539503,0.7811601758,0.1337841600,-0.5227097869,0.6001027226,-0.5171794891,-0.4444061518,-0.9055517316,0.3879831135,-0.3776228130,-1.8549275398,-0.4994918406,-0.5876417756,-1.6503987312,-0.2694072723,0.9541515112,-1.4059816599,0.6579794884,0.8008347750,0.8226274252,-1.1246813536,-0.0056366222,0.5829159617,-1.5171155930,-0.2439792901,0.2839211524,-0.7109612823,-0.7272608280,1.2907973528,-2.4604752064,0.2687241733,-1.1931895018,0.1179150194,-0.2157852799,-0.2656730711,-0.8884975314,0.7275415659,0.2283545136,1.6116662025,0.7428886294,-0.0892703235,-0.0904256403],[-0.1552596092,0.4787852466,1.3613634109,0.1748903543,-1.6370717287,1.0790226460,-0.1739098728,-1.0114746094,1.4560087919,-1.9859833717,1.3460286856,0.1526861787,-0.4367708862,-0.4582619667,0.9152957201,2.7462913990,0.1391252726,0.7504043579,0.7596225739,0.5596187115,-0.0583412908,1.6963244677,-0.3965150118,0.1869936138,1.8145555258,-1.3269416094,-2.2988479137,-0.5298420191,-0.8341919184,-0.8106333017,-2.5397062302,-0.0860494003,0.4506426752,-1.2943652868,1.0642453432,0.9367136955,-0.1269537657,0.5206781626,-1.1788055897,-0.0925211310,1.0476361513,-1.2335727215,-0.2922436893,0.0496872887,-0.2639459372,0.3845275939,-0.1538703591,-0.1169964671,0.1776241362,0.6594055295,-0.5636582971,-0.9561285377,0.4923869669,0.2409788370,1.6332099438,1.9169098139,1.1394318342,-2.5501418114,-0.3268522024,0.9455291033,-2.0710549355,-0.1209253818,-0.9449245334,1.5542948246,-0.2254855931,0.5607367754,-0.4980247319,1.2718555927,-0.9843304157,-1.6917517185,0.3631607592,0.2028281391,0.0599766299,-0.0586545616,1.1578779221,-0.1312680691,-1.6256289482,-1.0176999569,-0.1003823131,-1.0121793747,-0.4727144837,-0.5200366974,-0.2728419602,-0.4269649386,0.7875617146,0.9885934591,-0.4017859101,-0.0173298735,0.6428719163,0.4834312797,-0.2587702572,-1.1848987341,1.4192327261,0.1049401015,-0.3738996387,0.7806690931,-0.6665983796,0.0297318734,1.0643459558,-0.3537979722,-1.3078233004,0.7726968527,2.1866788864,-1.1002861261,0.1728644818,-0.3756933510,-0.9906257391,-0.8670617938,0.2038979679,-0.4590037465,-0.9317319393,0.1539378166,1.7603360415,-0.7288731337,-1.2079353333,-1.7485420704,-0.2674083114,-0.7923513055,0.8678576946,-0.8161421418,-0.5595346093,0.5180754662,-0.5601761937,1.9338699579,-0.2529518902,-1.1536580324,0.8240326643,-0.2938970327,-1.0393499136,0.0384243019,-0.1298382133,0.6317511797,0.9699698091,-1.2614362240,-0.3775847554,-0.3486490548,0.2697852552,0.7138586640,0.1024048105,-2.2221372128,0.0392785184,0.5322803855,-0.3420674801,-0.3462099135,1.7591598034,1.8691898584,-0.1903523058,-0.5461303592,-1.8728995323,-0.3152549863,-0.1878576875,-0.8418833017,0.4303056002,-0.0654961094,0.3263684511,0.5223084092,-1.4092383385,-2.7141454220,-0.6595085263,-0.6816119552,-0.3690533638,-0.3258565962,-0.6590272784,-1.3374688625,0.2712288499,-0.5335494280,0.3359538317,-0.5624163747,-1.7754486799,-1.3497921228,1.7255154848,2.0891745090,-0.1147161946,-0.4825686216,0.1295297593,0.3011428714,0.9685909748,-2.6775097847,2.0935235023,0.5985561609,-0.5186195374,1.1494662762,0.0178137068,-1.7103466988,0.0695706159,-0.4838339686,-0.6910316944,-0.3891272545,0.4077689052,0.1103762239,0.3082531989,1.6875544786,-0.8012143373,-1.2962436676,-1.1299370527,0.8236520886,-0.2105285227,-0.0350589901,-0.1153961793,-1.2768372297,-0.4285729825,1.0847262144,0.7416942716,-0.3530087173,-3.1282429695,-0.9469532371,-0.6899254918,0.8435842395,-0.2861720920,-0.3562988639,0.2356502563,-0.5722347498,0.5325630903,0.2162115574,-0.2787752151,1.9393185377,0.2387497127,-1.0231500864,-0.1833423823,-1.3151319027,-1.1783853769,-0.8859948516,1.2054790258,-0.5636700988],[0.4386736155,-0.6196897626,0.8047269583,0.1394855082,1.4760224819,-0.7151355743,-0.3518398702,-0.6956925392,0.4250043929,0.1587561816,-0.8085920811,0.9429353476,-0.4326693714,0.8953675032,0.9709054232,-0.9297704697,-0.1016962975,-1.1935923100,-0.9966936111,0.2610221803,0.0391289629,0.8596969247,-1.1022953987,-1.3328871727,-0.3936585784,0.3379068673,0.3390775621,1.8913849592,0.1280214936,-0.1877435148,0.7314977646,0.8108587861,0.2118465602,0.5429373384,0.2095481604,0.0367294289,1.3247373104,1.1130683422,-1.1776549816,-0.1899321526,0.9921732545,1.9340195656,-0.7474932075,0.5622502565,0.2565377355,-0.5058898926,-1.1801614761,1.1046266556,-0.2420715690,1.0296734571,-0.4392384887,-1.2390613556,-0.6452093124,-0.3723985553,-0.0273250490,-0.8667244315,1.2947559357,0.6697645187,0.8879160285,0.1957534999,-0.1988423914,1.2720395327,0.2759205401,1.7492319345,-0.2943438888,1.1912611723,-0.0809823349,0.0359159783,-0.4873678982,2.9271998405,-1.1786690950,1.0902714729,-1.0070717335,-0.0559710450,0.5503617525,-0.2708517611,-1.4462407827,-0.0244226474,-0.9263724089,0.3695653975,-1.0758725405,-1.0543339252,1.0808296204,-1.3966062069,2.2895793915,1.8400338888,-0.5700233579,-1.0836747885,0.7417201400,-0.2145137787,0.5379716158,0.3843903542,0.0347256139,0.3680998087,0.3415450454,1.0920809507,1.0413075686,0.0939790681,0.7268617153,-1.1950058937,0.4225138724,0.7281823754,-2.2234709263,1.2858834267,-0.9244327545,1.6425834894,-0.8119661808,3.3916356564,-0.2394975424,-1.2916408777,-1.2434978485,0.1182641983,-1.0540636778,1.0878865719,1.0936306715,2.7451908588,-2.4534094334,0.0842225254,-0.7882358432,-0.2610311210,-0.1405747086,-1.2278044224,0.4670206010,-1.5480242968,-1.0367826223,0.1100453958,-1.6257070303,-0.6632278562,1.6480108500,-0.3857504129,1.3263442516,-0.7813788652,-0.2630614638,2.2347497940,0.0668686181,1.6011503935,-0.2215524614,2.5348920822,0.6628156900,0.6786739826,1.7699764967,-1.7675346136,2.8429641724,1.7460889816,0.1622756273,-1.2240581512,0.9790400863,-0.4723766446,-1.2521955967,0.1068915650,0.2467359900,0.4016210437,-0.4117667079,-0.4852364361,-0.0951911286,0.6827812195,-0.2690612674,0.6709222794,-0.4353961647,0.8948027492,-1.9490350485,-1.0914882421,-0.3209503293,-0.2963206768,0.4339114130,-2.2697405815,1.2201591730,-1.8413844109,-0.3661860824,0.7219944000,-0.9503106475,0.6586489081,0.0980350003,1.0423717499,-0.4973531961,-0.5744463801,-1.3544663191,2.0338106155,0.0617796816,0.5912122130,-0.7075412273,0.0460111462,1.2361793518,1.7257440090,0.7506306171,0.5277210474,0.6417139173,0.3805473745,-0.9311766624,0.9438769221,-0.8593149185,-1.7188237906,-1.4166681767,0.7779759169,1.1453680992,0.2659092546,-0.7271529436,0.1307234317,0.7076218128,-0.6337662339,0.8103998899,-0.7279103398,0.2717936635,0.5227202773,-0.8961473107,-0.1444521397,-0.7144158483,-1.4684110880,-1.8797703981,1.4193205833,0.2078864872,0.7992144823,0.0320600830,0.8656871319,1.2021867037,-1.2167718410,0.8329434395,0.1805313081,0.3732955158,-0.6465490460,-0.9415496588,1.2734313011,-0.8172774911,-0.9249973297],[0.1996483356,0.4317770898,-1.0799595118,1.4785776138,-0.0096156178,-0.8270172477,-1.0379046202,0.2136165649,1.1392287016,1.2750461102,-0.8241134882,-0.2134141624,1.7874548435,1.2697911263,-0.7215440869,-1.2200564146,0.7094538212,0.2863227427,-0.0210470576,0.3863923252,-0.3042067885,1.0375249386,0.6831886172,-0.8644843698,-0.3198086619,0.4823707640,-0.2265983522,-0.4702896476,-0.5963839889,0.9628387094,-0.3546466827,-1.3467926979,1.0188089609,-0.8363088369,-0.0953657553,-1.3766548634,0.5656586289,-0.4472826719,-0.6763988733,0.9349191189,-0.8283550143,0.5968904495,-0.5544840693,-0.1959493458,0.0332162529,0.4148295224,0.0375486575,-1.0577303171,-0.9384614229,-2.0667753220,0.4751504958,1.0487740040,1.3003197908,0.3228363395,0.2529530525,-1.1515924931,-1.2190105915,-0.9456254244,-1.1880462170,2.6874439716,-0.3045670986,-0.7014102340,1.5996931791,0.3379813731,0.9407749772,1.1743144989,0.9814895988,-0.3301385045,0.7717459202,-0.2328396291,0.9203398228,-1.0166190863,0.0462331660,0.0152617535,-0.0070052808,-0.6647570729,0.5429480076,-0.5906264186,-0.0308912378,1.0884922743,-0.0040217643,-1.1533929110,-0.1063377857,-1.0902687311,-0.0371520258,-1.8188394308,-0.7905772328,0.1596994698,1.1226239204,-0.6813367605,-0.1751530617,-0.0979226381,-0.8810763955,1.1648347378,1.3413946629,1.1284626722,0.5128263235,0.0187418163,-2.0666255951,1.0853357315,1.0145881176,0.6455498338,0.3209685683,0.5687319636,1.0641225576,1.4271905422,-1.3905673027,-0.9891104698,0.0020802424,2.3130781651,0.6045076251,-0.8978734016,-0.3523695469,-0.9741426110,0.3125937879,-0.1361470819,0.7558745146,-0.6038464308,2.2138009071,0.2676672935,0.4894275367,-0.1646461040,0.2514430285,-1.4151527882,-0.1414795518,0.3838063478,-0.8808335662,1.9905422926,-0.3512460887,-1.5725287199,0.7633091211,0.0000131410,1.6761398315,-2.1059546471,-0.3404865861,-0.5632696152,-1.5203379393,-0.5280867219,-0.7296812534,0.6876379251,1.4581366777,-1.1583836079,0.6405873895,-1.0695174932,-0.1391310841,0.7101902962,-1.3572121859,0.1433014721,0.6498459578,-0.2396032214,0.4949466884,-0.0583442226,1.3597508669,-1.1603624821,0.9705311656,2.1088654995,-1.7841472626,-1.5925489664,0.4765196443,0.7140438557,-0.2969127893,-1.0004702806,-1.4487510920,2.0203588009,-0.4833310246,-2.3372316360,-0.5886305571,1.6995788813,0.0083491514,-1.0568113327,-0.1681756973,0.7782428265,0.6581347585,0.0151447775,-1.6557891369,0.8034425974,-1.0629384518,0.7985153794,2.1845128536,0.5347851515,1.6842222214,-0.6536903381,0.9090065956,-0.6756687164,1.9171681404,-0.0657464713,1.3801714182,-0.5541518927,0.2358659059,-0.1883283705,1.0601769686,-0.2588506639,0.4196306467,-1.2001607418,0.5803898573,-1.4160250425,1.7342479229,-1.1105753183,0.0719698370,-0.0096054645,-0.6200602651,0.1690012068,1.2434685230,1.4427556992,-0.1093287393,0.1923421919,1.8308950663,-0.6960420012,0.1550203562,1.0692973137,0.2098853588,-0.3324133456,-1.2730194330,0.6015979052,0.6712191701,0.2805779576,-0.5934511423,0.9138393998,0.0384552255,-0.0258487891,-0.8879203200,1.0867121220,-0.2558456957,1.3400003910],[0.8616962433,0.9091723561,0.3765018284,-0.0544774011,-1.1591389179,-0.1942774951,0.9726710916,-0.9714103937,1.3434149027,-1.1338416338,-0.7891272902,-1.9806178808,1.4280047417,-0.3443213403,1.2054544687,0.9668977261,-0.4083234668,-1.2918647528,-0.6399744153,-1.3426039219,-0.5441907644,-0.0543023236,0.1115766466,-0.4973062575,0.0533691123,-1.0189346075,0.0490081087,2.0632190704,0.9324282408,1.4523895979,0.5729588866,-0.4189784825,-0.3286801577,-1.1514205933,-0.8807796240,0.2938400507,0.7655924559,2.1366088390,0.0733786449,-1.6295683384,0.1775692850,0.8521859646,0.4896481931,0.5674389005,0.2375758439,0.0813688114,-0.6042775512,-0.5440589786,-1.2309296131,-1.4950492382,1.4114691019,0.4880595207,-1.9063181877,0.3281845450,-0.3974549770,0.2618822157,0.7912273407,-1.7842062712,1.4989305735,-0.2391272038,3.0528898239,0.3499550223,-1.4347679615,1.0079482794,0.4206511974,0.6000564694,0.1201554537,-0.5846073031,0.3514090776,-0.8588541150,-1.0708264112,1.9690668583,-0.0061912374,-0.7901587486,0.9585415125,1.9639558792,-0.4290848672,0.5445072651,1.2167263031,1.4984652996,-0.3440700471,0.3850102127,1.1216851473,0.6268048882,-2.3510096073,-1.3539459705,-0.0425043851,1.0418293476,-0.2806175649,0.4425379932,-1.3477407694,-0.9427426457,-0.0101525290,0.9552149773,0.2844652534,-0.0253380742,0.4472802281,-1.0420781374,1.8255304098,-0.1858208627,-0.9919012189,0.4774014056,-0.1065087095,0.5237557292,0.1012536585,0.4370443225,0.4235878587,0.6120776534,0.6691703200,1.0026834011,-0.3116049469,-0.0213360675,1.8255027533,-0.4455459118,-0.0459221825,-0.7012569904,-0.1852261573,0.5536851883,2.1449239254,-0.1896124780,-0.3305495679,-1.0523796082,0.5392750502,0.7831043005,0.3730578721,-0.1366892159,-1.0765298605,-0.4404587150,0.9915053844,-0.0562079400,-2.5473208427,1.1598278284,-0.7649890184,1.2011524439,-0.0606509931,-0.0063281334,-0.7450198531,0.6501030326,0.2470915914,0.3507081568,-0.8862549663,-0.6400193572,0.8826782107,1.1692306995,1.7553559542,-1.0607961416,0.6163920164,0.2505285740,1.2890933752,-0.5036508441,-0.5445509553,1.3842186928,0.3668037355,0.2400822192,0.4661004543,0.4137141705,-1.2596691847,-0.8630129099,-0.2596816719,-1.3117531538,2.5076625347,0.8216830492,-0.8308066130,-0.2402574420,-0.3923582733,0.2755875885,-0.8734244704,0.0743628964,0.1546310931,-0.0208904427,1.4379118681,1.2219953537,0.2602660060,1.0976809263,-0.0790601820,1.4060403109,-1.7074382305,0.1989279836,0.7320793867,1.0455595255,0.8985303640,0.5794932842,0.7482882142,1.0181425810,-2.1934535503,-0.1195093766,-0.5527552962,0.9271039963,0.0382056795,1.3007956743,0.2055685967,1.7423490286,-0.0785754547,-0.0049815332,-0.0163416192,-0.4962151647,0.1003125310,-0.8750685453,1.3657735586,-0.3143217564,0.1486933380,-0.6111947298,1.1490167379,-0.5325989127,0.0778353959,-0.8201375008,0.9185136557,0.3281737566,-2.0297646523,1.1189615726,1.0363020897,-0.2897744775,0.3120375574,0.7430668473,-0.1351819932,0.2756820321,0.2833786309,0.1408464015,-0.7921109796,1.6592693329,0.8912046552,-0.2330278158,-0.7310743332,-0.6375401616],[0.7888815999,-0.5637577176,1.5878105164,0.5654777884,-0.7989003062,0.6526470780,-0.5024884939,-1.4605481625,0.6715288162,1.0051063299,-2.2796263695,-0.0421458483,0.5515695214,-1.6218701601,1.2987296581,0.1798663735,-0.5212946534,0.5497487783,-0.9817274809,-0.7634006143,-0.4252837300,-1.6081527472,-0.5749193430,-0.0845824331,0.3179959953,-0.8117060661,-1.2607413530,-1.7419383526,-0.8477881551,-0.7736322880,0.0881378502,2.0652372837,-0.6772291064,-0.2555656135,-0.1456658989,1.2135969400,0.6333103180,0.6250420213,-0.5774396062,0.2879560590,0.8352848291,0.4350744784,-0.3484890163,0.0795898214,0.8474707603,-0.1157193184,0.4633275568,-0.3415514231,-0.6989926100,0.6122351885,-0.8388199210,-0.7292807698,-1.0647550821,0.4956947267,0.1355170608,1.2068425417,0.2432746291,-0.2817509770,0.2291960120,-2.0235092640,1.2210216522,1.1453696489,-1.3078383207,0.5784826875,-0.1166832522,0.4518258870,1.1240378618,-1.8123457432,0.6662940979,-0.9611188173,0.7314282060,-0.2969223261,-0.3550669253,1.4163671732,-0.9020425677,0.0199772660,-1.1304079294,0.1079952568,0.4287727475,-1.6183984280,0.6845467091,2.8033049107,-0.2084693313,0.1491068602,0.2246538848,-1.1328402758,1.1970536709,-1.1747390032,0.6738021374,2.5621573925,-0.6892337203,-1.5325818062,-0.8725388050,-1.8731801510,-0.8512567282,1.2584059238,0.6210688353,1.8926613331,0.5126205683,0.7967803478,-0.4531759024,0.6966472268,-0.9069048762,0.0096370932,-0.6075945497,-0.7283154726,0.5826425552,0.2152446657,-1.2975306511,-0.5846914649,-0.4268600941,-1.0764298439,-0.7263673544,0.9428517222,-2.0351703167,1.5888681412,1.1721462011,-0.7234143615,-0.6291493773,2.0337893963,0.2800778449,0.0618867502,-1.0652469397,-0.1269562542,0.0394340493,-0.2710518539,1.1494917870,0.7313186526,1.4539731741,0.4317398369,0.1555168629,-0.6679779291,-0.6185320616,-0.4576709569,0.8619299531,-0.3402082324,-0.0162530318,-0.9234033823,1.1426278353,-0.4658351541,0.4760331213,0.1220721081,-0.0449118540,-0.7347796559,0.4099939167,-0.4844644368,0.8656507134,-0.0493310839,-1.6068956852,0.6101850271,0.3961558640,-0.5334519148,-1.2353732586,-2.1528379917,-0.6461086869,-0.7687655091,1.0920033455,-0.0443054624,0.9873304367,0.1205482185,-0.0901331529,0.2483174056,-0.0232608952,-0.9315232038,-0.4337712228,-0.7576855421,-1.4757883549,0.4191368818,1.0992763042,-0.5225518942,-0.3033358455,1.4686160088,0.2693915963,-0.5548349023,-0.0744934976,-0.9019102454,-0.4306880832,-0.4611977935,-0.8646348715,-0.9281977415,-0.2186366916,-0.7222694755,-0.5996643305,-0.7126638293,0.9143796563,-0.1718460768,-0.7571513057,-0.8897672892,-0.2077621669,0.3679377139,-0.6376763582,0.6803770065,0.1529729366,1.6237425804,-0.7013158798,-0.7598114014,2.4127118587,-0.0613814630,0.5617071986,-0.4261292517,1.3765552044,-0.1034885272,-1.5822913647,0.2727092505,-0.4394279420,0.0855139270,0.5925198197,0.9543700218,-1.3967151642,1.0356595516,1.2689288855,-0.0545611903,0.2998319864,0.2138415426,-1.5542737246,-0.5578077435,-0.0780339167,-0.7181679010,0.4998109639,-1.1115146875,-1.6112943888,1.5544286966,0.7367937565,-1.1129119396],[0.7780324817,0.4822565317,-0.8536413908,0.4397369325,-1.3432642221,-0.9365406632,-1.2382225990,-1.2970274687,-0.6214320660,-0.4893566370,0.1583636999,-0.2710636258,-0.1458918750,0.6112800837,0.6005087495,-0.7187950611,0.8169072270,1.6199951172,-0.7268152833,0.2404350340,0.2810488343,-0.4852738678,0.9925027490,0.0956963748,-2.8951740265,-0.3318829834,1.3719614744,0.5818605423,-2.0010683537,-0.4226765037,1.0245097876,0.7852441072,-0.4349224865,1.4291239977,-0.4860503674,0.3763881922,-0.4843531251,-0.3523619175,0.7469093204,0.6595010161,-0.3882180452,0.9919152856,-0.2797157764,2.4180109501,0.0411199592,-1.8055504560,0.5286077261,0.1346055269,0.3227880299,-0.3051802516,-0.3027456105,-0.0010736089,0.4762355387,2.2697131634,-0.3243034482,0.1008412465,-1.4537488222,-2.0363309383,2.1591031551,1.5303931236,-0.7778717875,0.2341108471,1.3119387627,1.3180574179,-0.8847883344,0.5251903534,0.2385541797,-0.3019807041,1.0521885157,0.0388435014,-0.1639668941,1.7076059580,-0.6597477198,1.3579747677,-0.7429273129,-0.0546745248,-0.1375604272,-0.9347351789,-1.1826816797,2.0394957066,-0.4085650742,0.4151297510,-2.3462462425,-0.3660867512,0.2620778680,0.4003766775,-0.0485555828,0.8496025205,-0.8509175181,-1.4193309546,0.8738920689,-1.3105102777,-0.1310589164,-0.0740571916,0.1930714250,-0.9857059717,-1.6849098206,0.4512572885,0.1277438253,0.3671044409,-0.1350663900,-1.0645258427,-0.0029378785,-0.0521377474,-0.0649472773,-0.6643292904,0.5142106414,-1.7635880709,1.6859691143,-1.0984498262,-0.7941613793,1.1221563816,1.3669830561,-0.0905384943,0.2765818536,0.6256956458,-0.3617914617,-0.2852146924,0.7979028225,0.2126920968,0.8998239636,2.2797007561,0.2099189162,-0.3656977713,1.4928067923,-0.4475935102,-0.4616768956,0.2791537642,0.2556216717,-0.6943846345,0.4273623228,0.1741427183,0.1316052079,1.1936930418,1.5984715223,1.0562391281,1.0298056602,-1.2818797827,-0.8192679286,-1.9437376261,0.6600036025,0.0188401472,0.4665989578,1.4290597439,0.7851303220,0.2261232734,0.7731981277,0.5813441277,0.5581420064,1.3034487963,1.0300573111,2.0724573135,-0.7095886469,-0.8896782398,0.1834590137,0.6919991374,-1.0479919910,-0.9896458387,-0.2847566903,0.7601591349,0.5769826174,1.3861632347,-1.7981691360,0.5600209832,0.1504398137,0.6467001438,-0.3333204985,0.2885289490,-1.5847134590,-0.6807080507,-0.4037251472,-2.4674272537,0.1292898357,-0.1086133048,-0.3344806433,-0.4761407077,-0.4093746841,0.8121746182,0.9868445396,-0.4382044971,-0.7254872918,-2.4136452675,-0.3176845610,1.9473218918,0.2170838863,0.2626359165,0.1446496695,2.9867980480,-1.6840299368,1.3493431807,-0.6842327714,0.2324342579,0.8585680127,-0.6588760614,-0.3588640988,-0.3767255545,-0.5748527646,0.2612748146,0.1919667572,0.9821797609,0.4179340303,-2.2200431824,0.2395582497,-1.1190239191,0.7308257818,-1.2480312586,0.7303984165,-0.6898256540,0.3308303952,-0.6568310857,-1.0560706854,0.3887234926,0.5087570548,0.7185772061,-0.2959088087,-0.2250892967,0.4436039627,0.7100724578,-0.3117805421,-0.7210696340,1.4668381214,-0.6216652989,0.4426654875,0.4567816257],[-0.8505325913,0.8974496722,0.8174543977,0.4282577932,0.8738188148,-0.3808510900,-0.5091495514,2.3714761734,0.1522615403,-1.2650282383,1.3257935047,-1.4895132780,2.3313984871,-1.2792991400,0.7764997482,-0.7210823894,-0.1681635827,0.1529988945,0.4695449471,0.1967510283,0.2921490669,1.3127886057,-1.2193900347,0.2852847874,-1.2616995573,-0.7584261894,-0.1306209564,0.9959502816,-0.8483362198,-0.6962991357,0.3551653624,-0.2280855030,-0.7159309983,-0.3786600828,-0.4753340483,-0.1977416724,-0.9485037327,0.3293333650,1.2257366180,-0.1455384195,-1.4984012842,-0.1919256151,2.1786687374,-1.0019141436,1.0610024929,0.6823122501,0.0800496340,0.9403626323,0.5023782253,0.3214206100,1.6189458370,-0.1983540952,1.7617611885,1.1387735605,-0.8849478364,-0.8603631854,0.5942631960,1.4637345076,-1.2320071459,1.5357743502,-2.1328830719,1.7089049816,1.1534118652,-0.0069381776,1.1776837111,-1.5097141266,0.2115106434,-0.1186552644,0.1261516213,-0.3330809772,-0.1185161993,1.1600736380,0.3577325046,0.7345383763,0.7769987583,0.7655192018,1.6940709352,0.6339361072,1.0973848104,0.4087703526,-1.7375053167,-1.0483746529,-0.4302817583,1.5430914164,0.0006288328,-1.2928329706,0.3590690494,-0.3027823269,-0.9697774649,-1.5566012859,0.5141104460,1.1575758457,-1.1170324087,0.0305965990,-0.1459143758,-0.7981154919,0.4145491421,1.4571155310,-1.3967839479,-0.0655855462,-1.0104351044,-1.1136292219,-0.1907861084,-1.8058137894,-0.2570574582,-0.6587927938,-1.3274798393,-0.1549696177,0.2305874527,-2.0742232800,-0.0295557063,1.3538379669,0.3438618183,-0.2380728573,-0.2246685326,1.5747584105,0.6863652468,1.3693779707,-1.2951009274,1.3435353041,2.0449006557,-1.6192568541,1.7248616219,-0.1750580519,0.3598082364,-0.8229542971,-0.9378738403,0.1098855510,-1.0314471722,0.0641856417,0.6701291800,1.5685896873,-1.6793966293,0.4545374811,-0.0591463894,-0.0386027545,0.1191953644,-1.2499835491,0.9295192957,-0.5419933200,0.1619770080,0.1354142129,1.5632513762,-0.3869238198,1.2132548094,0.0554801188,-0.6512631774,-2.6216135025,-0.2254151106,0.4330559969,-0.5367842317,-0.8901445866,-0.8257043958,0.5760731697,-1.5437245369,-0.7159849405,-0.9570081830,-1.3737111092,1.6673197746,1.3580203056,0.2791488171,1.0237982273,0.9078584313,-0.8839287162,0.0231411308,-0.3945089281,-0.2564010918,-0.5195775628,2.2790925503,-0.9839748740,-0.1140683740,0.7237284184,0.1794247627,0.9464429617,-0.4277381003,2.1282601357,0.0217097141,-1.7380595207,1.3669571877,-0.2642624676,1.5720680952,-1.8166810274,0.5242646933,-0.8335030079,-2.0098788738,-0.6536083221,0.2945873141,-1.4782997370,-0.1516771913,1.1408663988,-1.4657688141,-0.3113844097,0.7662166953,-0.4563453794,-2.4438807964,-1.4533135891,-0.4236075282,0.5318500400,-1.8602722883,-0.6261865497,2.0996797085,-0.8635161519,-1.2204080820,0.4388450384,-2.0069413185,0.2024406493,-0.8838145733,0.0309155099,0.7476049662,0.8999215364,-0.8639245033,0.0988463908,-0.6415423751,-0.6678134799,-0.0940411165,-1.7194350958,-0.4066540897,-0.4778663814,-2.6078004837,-0.9008126259,-0.2668288946,0.7076120973,1.3418669701,0.4082632065],[-0.4423490763,2.5291156769,-0.0829270333,-0.9171121716,-0.7369276285,-1.6547844410,0.3231929243,-1.0614206791,0.3161898851,-0.3398891389,-0.0626130924,-0.9866225123,1.2956130505,-1.7446339130,1.8742061853,-0.0142730968,-0.3598657548,-0.6989034414,1.3945087194,-0.0364240184,-0.6663540006,-1.7180625200,-1.2895328999,-1.8650425673,-0.2848217785,-0.6872664690,0.6471204162,1.3397916555,0.3441358507,-0.6507552266,-0.0044669984,-0.5204245448,0.3537707329,0.6218783259,0.8653511405,-1.9395599365,-0.3520051837,-0.3968527317,-0.1872099787,2.5866026878,1.3923414946,0.4628205299,0.0473618843,1.5894398689,1.1269013882,0.3289514780,0.0482734181,-0.6642089486,1.3771688938,-0.0996530503,0.3009798527,1.1523993015,-1.0771466494,0.8700125813,-1.5626024008,-0.1853292584,-0.4131501615,0.3208904266,0.7904941440,0.1825299263,0.0197186228,-1.1914066076,-0.9240799546,0.8452393413,-0.1513461173,-1.2526690960,0.1262753606,-1.6897244453,-0.4013103247,1.9904233217,-1.3785617352,-0.0342551842,1.2655111551,0.2004050314,-0.4384508431,-0.9500871301,-1.6665233374,-0.7320570350,1.6763832569,1.4295755625,-0.2407578230,0.8036060333,-1.8491408825,2.9870183468,-0.6388540268,-1.8830124140,-1.6932355165,2.1526920795,0.1216804460,-1.4847382307,0.1124377400,-0.0760384947,1.3658516407,1.2392852306,0.0693709552,0.2876286805,0.0789157599,-0.8022515178,-0.2305098027,1.9006695747,0.9412038922,0.3962628841,0.6482299566,-0.8974066377,-0.4834775031,1.2487920523,1.1562736034,0.6536567807,-0.3170304596,0.2463564575,2.4179408550,-0.4878221750,0.8989579082,0.4149280190,-0.3923274279,1.2863267660,0.7034213543,-0.7199124694,-0.4644466341,1.4654096365,0.3432749510,-0.9306016564,-0.7491942048,-0.1750261039,-0.7223092318,0.1536332071,-0.3865046799,0.5169742107,-0.4115245044,1.6728409529,1.1052545309,0.2504644990,0.6183803678,0.1082001925,-0.6866397858,-0.1700413972,0.4843541086,1.3254795074,0.1514006704,-0.2798740566,0.4530131221,-0.1875061393,-1.5138655901,0.2696782053,0.3310975432,-1.5183287859,-0.4288371205,-1.0433771610,0.3195703924,0.2241828144,-1.7004652023,0.8005896211,-1.3455075026,-0.7087361217,-1.0131795406,0.4448340237,0.8383929729,0.7657484412,-1.0465499163,1.9463306665,1.6850688457,1.6361572742,-0.4113813639,0.8208792210,-1.3112231493,-0.4735149145,-0.9699966908,0.8628758788,0.9517716169,-0.9734086990,-0.6572603583,0.5387002826,-0.2033254206,0.6137370467,1.5639606714,0.3512731194,0.2677857578,0.1491088122,-0.6859575510,-0.1748508811,0.0057693808,1.2078301907,-0.2483086884,-1.5793319941,-0.1355962008,0.5014082789,-1.3090702295,-0.9902474880,0.1836945713,0.3580158055,0.1922743022,0.0095682470,-0.2284177840,-0.5888280869,-0.9356576204,0.5763784647,-1.0042114258,-0.3705872595,-0.2505876124,-0.7080980539,0.1050026119,1.1257755756,-0.4953170419,0.9546896219,-0.6923274398,0.3183096647,-0.5688853264,-0.6310497522,0.2750060260,0.0254878420,0.2571436465,0.5700058937,-1.7250375748,-1.4157315493,-1.4600043297,-0.8703076243,0.0653536320,1.1396144629,1.4283490181,-0.6810770035,0.5877169967,0.1974487454,0.0439355373,-0.5478091240],[-0.8501861095,-0.1452676207,-2.0167658329,-0.1970911473,0.0701859370,0.7894083858,-1.7675129175,1.2736612558,1.4884271622,0.5635340214,0.6796386242,1.2423328161,0.0540213957,1.3500727415,-0.2239092141,1.4953960180,0.2478047758,0.5561262965,0.2338157147,-0.1017286703,-0.3675415218,-1.7073664665,1.6565083265,1.6814539433,-0.0360563397,0.4497372806,0.0980281234,1.4531240463,-0.6414846778,0.1412290037,-1.0449951887,-0.4438725412,0.9779627919,0.7688708305,-0.5568965077,-0.9761307836,0.7911739945,1.4035484791,-0.5847625732,-1.4764374495,-1.2948927879,0.7119658589,-0.8156015277,-0.2008070648,0.8081845641,1.5236290693,-0.3609860241,-1.0343962908,1.7678860426,-1.1862468719,-1.7334928513,-0.8210963607,0.3756120205,0.2281047553,-0.8125753999,1.4891103506,-1.8475048542,-0.1294121295,-0.6484277248,-1.0389952660,-0.2641805112,-1.4201703072,0.9795006514,-1.3405352831,1.4375221729,2.0169596672,1.1405482292,0.7233772874,0.5449270606,-0.7012047172,-0.2516741455,0.1970013082,-0.7681846023,0.5858729482,-0.3477722704,0.2033023685,-2.0873692036,0.9893494844,-0.5800985098,0.6182278395,-1.9546772242,-0.6086624265,-0.7773972154,-1.2600343227,1.8167859316,-0.0987552106,-0.1810840815,1.2722183466,-0.3335888088,1.5315827131,-1.3085680008,-1.2660052776,0.9042910337,0.3279466331,0.3045103848,-0.3153513670,-1.5934838057,0.4933577776,-0.3654850125,-0.5737106800,0.2879352868,-0.4627600908,-1.0108826160,-0.6777721047,-0.8984144926,0.2504751086,0.8091649413,2.6401121616,1.7911562920,-0.8305351138,0.8344239593,-1.4438226223,0.3508011997,0.2592615485,-1.5862669945,-1.4240231514,1.5163314342,-0.8924900293,-0.0843262747,-0.7113832831,-0.2796256840,-1.0860145092,-0.3189810216,1.4416177273,0.5681290627,-0.3779550195,0.4941016138,0.2960478067,-1.2442852259,0.0931274667,-0.0518829748,0.1845656782,-0.7338954210,-0.5237256289,0.8684477210,0.6828365326,0.5501427054,0.8220844269,0.7082092762,1.0605940819,-0.2782573998,1.0126662254,1.1477307081,-0.0826153383,-1.4017388821,-0.7767349482,1.1191364527,-0.4544364214,2.3977463245,0.5580090880,0.8164888024,-0.7969041467,-1.2697274685,1.5049589872,0.1810080558,-0.2684362829,-0.3502424061,-0.4023967385,1.2367614508,2.0550661087,-1.4493792057,0.1685050875,0.4869134128,0.8313385844,0.6722729802,-0.5500596166,0.2919599712,-0.0776337832,0.3386179805,-0.5368257761,1.8082138300,-1.5536627769,-0.2598360777,0.4972334504,-0.5991903543,0.1945166141,0.2947003841,1.3485603333,-1.8906611204,-2.3424110413,-1.2688889503,0.0344363041,0.2485752255,0.9743855000,0.5960764289,0.3309855461,-0.5189491510,-0.8919455409,1.1610465050,-0.5045477152,1.3423473835,1.5131756067,-0.6779407263,-1.7057740688,1.3286938667,-0.2861714661,0.3773959279,-0.0487306230,0.9769710898,0.0155381076,0.2035691142,0.2230139375,-1.5775175095,0.3229602873,-1.2306278944,0.0244359765,-1.4809737206,1.2955615520,-0.0120471921,-1.2050281763,-1.8280923367,-0.1050783247,0.9775385261,0.7549197078,0.0924711451,-0.1297639459,0.2104711384,0.2063230872,0.0318034813,0.6998687387,-0.0475344919,0.0653456599,-0.4324573576,-0.2389912307],[-0.0385269783,-0.3596199453,0.9366604090,0.2594126165,0.8580753803,2.4031043053,0.6737225652,0.6189863682,0.3409185708,-1.7746075392,-2.3424885273,-1.2195376158,0.0443152338,-0.3048034012,0.4648899436,1.8452289104,0.3149746358,0.0522343889,0.3195095360,-0.7438256145,0.2410807163,0.5562276244,0.2206500024,0.3171322346,-2.2169415951,0.5470215678,0.9518302679,1.1646939516,-0.4791328311,1.3888785839,-0.3748681843,0.7778351903,0.8615957499,-0.8165302873,0.5390430093,-0.6619489193,-0.1029545069,0.0851025358,0.9509700537,-0.5260720849,0.8613053560,-0.9844242930,1.3661401272,0.5948407054,0.1009211168,0.0307289679,0.1981360763,0.8672862649,1.1369253397,-0.1251385957,0.1661457717,-0.9388157725,-0.2449741960,-2.4094924927,0.0623265356,-1.2366584539,1.8895184994,1.3102403879,0.9462187886,0.3420774639,0.3925039768,1.2095966339,-1.4361178875,0.5183697343,0.3537858129,1.2890650034,-0.2607053518,0.2743812501,-0.6387026906,0.6534357667,0.1286210418,0.5044686198,0.1814749390,-0.4254333973,1.5863105059,1.7321563959,1.5264887810,0.1210952625,-0.5436092615,-0.9529995918,-1.6551265717,-0.1553141028,0.2441254854,-0.7731710076,-0.2470936179,0.2712610662,-0.6617076993,1.0395247936,0.6841502190,-0.9874541759,1.3303834200,-0.6565662622,1.1300296783,-0.1293801665,2.3106272221,1.4594422579,-1.0526741743,-1.0220749378,0.0577553660,-0.5097522736,0.3840633333,-1.2838438749,0.7513307929,-1.6153922081,-2.3277418613,-0.6914343238,-0.4368054569,0.3361124098,0.8448717594,0.4833911061,-0.4330910444,0.6864856482,-0.7480033040,0.5588150024,-1.5406918526,0.8605827093,0.0865264311,0.6239672303,-1.1139910221,-1.5116748810,-0.7817136049,1.5633437634,1.6206901073,-0.1340485662,0.5526981354,-0.0745343417,-0.5309984684,1.1839113235,0.3718740642,-2.9650082588,0.6733871102,0.3198423088,0.6197563410,-0.0403066762,-2.3952093124,1.2700175047,-0.8881990314,-0.7429559231,-0.8290517330,-1.1331794262,-1.8599679470,0.7857927084,-0.2451763600,0.3910550177,0.1410217583,0.7675020099,0.7578459978,-0.3758762479,1.2606920004,-2.8472449780,1.4555310011,-0.2381768674,-2.2845680714,0.1599343717,0.7724515796,-0.1817385405,0.8262177706,-0.7928229570,-0.8502887487,0.1392326504,0.4767868519,0.7469817400,0.4203766584,1.9672571421,1.5491867065,0.0936574414,0.1747819185,0.1045727581,2.1363508701,-1.2314023972,-0.6329893470,-0.4213211536,0.8636973500,-0.3888166547,-0.4820105731,0.4317566156,-1.1235409975,1.3373652697,0.7710515857,-2.7142024040,-0.2182231843,1.6929919720,-0.0195358247,1.3989555836,-0.3875274360,-1.0358554125,-1.1316094398,-1.5382269621,0.9663111567,1.0785235167,0.3704659343,0.2607283294,-0.2777065337,1.8606232405,-0.6065781116,0.3053328991,0.6207325459,0.0930356830,1.2091175318,0.1401986331,0.8139212132,0.0959253460,-0.8038667440,-0.9163048267,-1.9060095549,0.1097709686,-0.9959290624,-0.3285111785,1.2623658180,0.2582879066,1.3890106678,0.1220844612,-0.4873121381,0.5140516758,0.3178030849,0.5869648457,-0.3526543379,-0.3959585726,2.3274693489,0.1709618419,-0.9874722362,0.3988903761,0.3557094634,0.9122059941],[-1.7059065104,1.5565053225,1.3327462673,-0.3147378266,0.1606792957,-0.5181043744,-0.0744991302,-0.1525735259,0.8446281552,-1.9320089817,1.4879050255,0.1854588836,-0.1515202969,0.2979521453,0.5038195252,0.8791267872,-2.6462893486,0.1011591628,0.5374528766,-0.0020966744,-0.0970841497,1.9185301065,0.5449617505,-0.4810386002,-1.5802356005,0.3312494159,-0.8791120052,0.3050711453,1.0962580442,0.5063979626,-0.0172869842,0.9996516109,-1.5001869202,-1.8311299086,-1.1142653227,1.2140845060,-2.3504610062,0.3383857608,-0.2453690469,1.0279306173,0.0342444330,1.4575858116,0.7556669116,-0.2027938962,-0.3228623867,-0.1250157803,-0.2761982381,-0.0912716240,0.6139395237,0.1320572197,-1.7598555088,3.8252158165,2.3801681995,-0.5304226279,0.4853073955,0.4158081114,1.9410279989,1.3093279600,0.4837481976,1.0330768824,0.1908234954,0.8460777402,-0.4605898261,0.4409380257,-0.0441601798,1.8047442436,0.9391747117,-0.9743002653,-0.6022531986,-0.9103568196,0.7067342997,1.0348250866,-0.1873424798,-0.4120945930,-1.6666873693,0.1700548679,0.8848538995,-0.5189235806,0.6518719196,-1.4336264133,2.1102340221,0.6158944368,2.0755994320,0.9486870170,1.4196146727,-0.6370040178,1.4725530148,-0.2676182985,-0.1825240552,2.1773166656,1.9086070061,-0.1137179285,1.5164897442,-0.2023674399,0.2179028988,1.7951279879,0.3171834350,0.4730903208,-0.9259622097,-0.5089650750,0.7982944250,1.1375516653,0.3526133299,1.3780834675,-0.0489090681,-0.1768110543,0.0780580640,0.0144599462,-0.0084795738,-1.2281516790,0.9156061411,0.2464201152,-0.0167766493,-0.1420723051,-1.5855610371,1.6599810123,-0.0597732589,-0.0273799822,0.8296191692,0.8847360611,-0.5626728535,-1.7483476400,1.1403344870,-0.0690325201,1.3960663080,0.6275839210,0.5832805037,0.0118617360,1.2410900593,-0.6006539464,0.2910424471,-0.5646578074,0.4662118554,-0.5337926745,0.3868106008,-0.4784782529,0.5543790460,-0.4056519866,0.7447139025,-0.0469487794,2.4892342091,1.3938800097,-0.1264778972,0.4362581074,-0.8975135088,-2.5608577728,-0.7694765329,0.6344984770,-0.1369070113,0.4036124349,-0.2340947241,-0.9804391861,0.4205627739,1.2156971693,0.1035736427,-1.7539309263,1.0600017309,1.2703372240,-1.2950500250,-1.5065121651,-1.3031721115,-0.1997547597,-0.5540348291,1.3706039190,0.3994727433,-0.2424000353,-0.5245046020,-0.0897789225,-1.1353756189,-2.1022062302,1.2222563028,-0.2435424030,-0.0676702186,-0.8102147579,0.3558183014,2.0535726547,1.0213717222,-0.2912064791,-0.7373991609,0.3800884783,-1.0866839886,-0.2113780975,-1.0243973732,-1.0861248970,-0.1851570606,0.4320114851,0.5590879917,-0.3961796761,0.9062599540,1.5587188005,0.7504378557,-0.4613761306,-1.9730011225,-2.1190068722,0.4325800836,0.0537056699,0.8598451614,0.6354776621,-1.6893638372,-0.2476013154,0.9518808722,1.5663367510,1.6191905737,-1.6549032927,-0.3134933412,1.4668923616,-1.8313214779,-0.3367227614,-0.5707110763,1.6618161201,-0.3145177364,0.0156628042,0.1667782664,0.8142173886,-0.3663666248,0.2926310003,1.5102934837,-0.8687940836,-0.5283280611,-0.5114175677,0.6322653294,-0.1110791191,0.0009631776,2.9895756245],[1.4097995758,-0.5420150161,0.9210902452,0.5834612250,0.7994394302,-1.3424725533,-1.0264796019,-1.3861973286,0.1247647181,0.1029771194,-0.4731315672,-0.5921694040,0.7910331488,0.6274262667,-0.1472722888,2.5348424911,1.7897282839,0.6568221450,1.7179471254,0.5924511552,0.5724430680,1.4984492064,-0.5010377765,0.0174681451,-0.5767454505,-0.4235750437,-0.3686167598,0.1869814843,0.8584616184,0.0341167487,-1.2190752029,-1.8360806704,-1.7777047157,0.3351447582,0.2758966386,0.2172588408,-0.1049501747,0.4783955216,-0.3389906883,-1.5148092508,-0.2531677485,1.5903692245,-0.2204401195,0.6823341250,0.6670246124,0.3315284848,1.3474885225,-0.6062452793,0.0387766436,0.3242726922,1.6093249321,1.7843703032,0.5993479490,0.9222139716,-0.7880330086,2.2105967999,-0.7617409229,1.2312707901,0.6148130298,-1.4690411091,-0.4746180475,-0.6721571684,1.0992940664,-0.0940861106,0.6958556771,-1.2895159721,0.2084652185,-0.7831521630,2.0253167152,-2.4375717640,-0.0784258023,-0.2919144630,0.9683870673,-0.2354955524,0.5750846267,-0.3641513586,0.8403220177,0.5809874535,0.6007576585,0.0460261889,-0.3048984408,-0.7436544895,-0.1375458986,-0.0768285319,0.6623673439,0.4524120688,0.3483710289,0.6208192706,1.7647602558,0.4740043879,-0.2359790057,-1.0643504858,0.4502330124,0.4655013680,-0.9948694706,-0.2585768700,1.3167483807,0.1989564151,0.2323360294,0.7958525419,0.3262020648,-1.3898649216,0.1671944559,-0.9119162560,0.2252446115,-0.9267496467,1.1519112587,-0.5130313039,0.1882673055,0.6804435849,-0.4746681154,-0.4878197908,0.1800938845,0.1051743627,-1.0580087900,1.4008100033,0.5489974618,0.5481874347,-0.5832469463,-1.2305303812,-0.3625142276,0.0120345866,0.2264911383,1.2822241783,-1.2907737494,-0.8618292212,-0.9438347220,0.4134423137,-1.0703722239,1.4725133181,1.4156024456,-0.4514758289,-0.5120148659,-0.6924622655,0.3693538606,0.8347567320,0.6099922657,0.8409538865,-0.2589316368,0.1839996278,0.3036150634,-0.0842262655,-0.2866791785,1.1813234091,-2.3766260147,-0.9739966393,-0.9959408641,-0.7160070539,0.3816682100,-1.7369004488,-0.7823039889,-1.6728433371,0.5592993498,0.6593230367,-0.6877341270,1.5265905857,-0.0742725432,0.8444102407,-0.7459825873,-0.0516187772,0.2302361876,2.4707098007,-1.7039589882,0.1352397501,0.4142930210,0.1444460303,-1.1740654707,-1.2802608013,0.1036694646,-0.3136852682,0.0832345337,1.2571145296,1.8054370880,0.7833987474,0.2159276009,1.3131119013,1.0496573448,-0.2494902462,-0.9532986283,0.8174754977,1.8209816217,-0.4751147926,1.5540887117,-1.6372287273,-0.5182297230,-1.2041864395,-0.7944150567,-0.2081774175,-1.7688667774,0.9372014403,2.3383221626,1.2419533730,-0.1160321012,0.8310201764,0.3418564796,0.0724793375,1.4042508602,-0.0997533277,-1.5711517334,-0.1969444603,1.4441918135,-0.7084956765,-0.4584809840,-0.4085344374,-1.7429348230,-1.4977319241,-0.2693070471,0.9721183181,-0.4137994051,-0.6265276074,0.5437030792,-1.2866847515,-0.7732895613,-0.8717803359,-0.0661167651,0.0868368670,-1.4115611315,-1.1569627523,-1.9649760723,-0.1715185195,-0.9807068706,0.1386085302,0.4682126045,-0.7425556779],[-0.6112797260,-0.9979614019,-0.3386137486,-0.1766004115,-1.7047775984,0.1936862469,0.1481569558,-1.5413073301,0.3674076796,0.0811279714,-0.9216552377,2.4965655804,0.8719742894,-2.0750610828,-0.8497855663,0.5540226102,0.2907184958,0.6172452569,-0.4571862221,0.4977155328,-0.9748178720,0.5232276917,-0.0748741329,-1.2881364822,1.3667689562,0.1484711468,0.9896100163,0.3713888526,-0.4138089418,-1.0227590799,0.3866163194,-0.9294429421,-1.0626591444,-0.0199987087,0.0961680189,0.2672078311,1.2036789656,0.7388323545,1.2408499718,0.6849725246,-0.2830341756,1.7859346867,-0.6817632914,0.6170616746,0.7021603584,-0.0052990373,0.0670303106,-0.3759177029,-0.5266339183,0.5763165951,-1.1446790695,1.0514925718,-1.2716163397,-0.3700241148,-1.7133955956,0.4637143910,-0.0871691108,0.4271546900,0.7751936316,-0.4274554849,-0.4503265619,-0.5333029032,0.5149218440,-0.2053171545,-0.6556538939,1.2024029493,-1.4963507652,-1.0015004873,0.2347760946,0.5991263986,-0.5835256577,0.0645779371,0.4332801104,-0.8684640527,0.6871785522,0.2869982421,0.0550472997,-0.6587293148,-0.5711126924,1.2947942019,-0.7356818914,-0.3393787742,-0.3914661109,-1.1938683987,-0.0813956112,-1.5593749285,-0.3439086378,1.2804453373,-0.9656952024,1.6998059750,0.3437785804,1.2053562403,-1.3430669308,-0.2202706486,-0.2032942921,-0.4607440829,0.6888646483,0.9494631290,-0.5992566347,0.6588060856,-0.8213128448,0.3451172411,-0.1872069687,0.5875008702,0.6419531703,0.8461623788,1.4980093241,1.5428068638,-1.0534813404,0.1087473035,-1.9668846130,-0.4938659072,1.0318287611,1.4068181515,-0.1060440391,0.1013416573,-0.3797146380,-0.0020411925,-0.4893866777,1.2483414412,-0.9816740751,1.0710183382,0.3810484111,-0.2293544710,-0.9001345038,2.1937174797,-1.2616010904,-1.2372906208,-1.6419368982,0.2051976770,-0.3173463643,1.1138305664,0.0690412819,-1.4165972471,-0.9509050846,-0.4597766995,0.2946996391,-0.7381731272,0.9412861466,-0.6347444654,-0.2386984080,-0.5287588239,-0.2509096265,1.6997805834,1.9507800341,-1.3863464594,0.0064332704,-0.7473371029,-0.5901803970,0.7893643975,-0.0761919171,0.6964703202,-0.6725970507,0.2126092315,0.7568253279,-0.9808959961,0.1224446669,1.5959967375,-0.4214015007,0.7471168041,0.4177250266,2.1514711380,-0.2517547309,-0.5861119628,-0.7115761042,-0.2423496544,0.4907200933,1.3639013767,-0.0233906917,-1.2201986313,0.5158403516,0.2809210420,0.1244746000,0.2069127262,1.4315224886,0.3899920881,-0.2881478667,-0.4212123156,1.2691807747,0.1441033632,-1.9708248377,-0.3076345325,0.6847977638,-0.2083214968,-0.9662112594,0.5662410259,-0.4710945487,0.6587586999,0.4371346533,1.2881650925,1.8142797947,-0.2653055191,-0.8061235547,-0.2660584152,1.1144868135,0.4511666894,-1.0467418432,-1.2955060005,0.4813936055,0.0717912763,-0.2718241215,-1.4206292629,0.5015422106,-1.5696884394,-0.7510741353,0.5357149839,-0.4761382341,0.3027769625,1.0439693928,-0.2968443632,0.4115947187,0.1417464614,-1.7686996460,0.5488359928,-0.4652283788,-0.2023715526,2.0392110348,0.9476948977,-0.5790069103,1.3782595396,0.0463112034,-1.4538191557,1.0799579620,-0.4495364726],[0.6995849013,0.9958032370,0.1410770565,-0.7974423170,0.7335600853,-0.8051303029,-0.5576012731,0.6688166261,-0.6395130157,2.0262405872,1.8946053982,0.4527872503,0.3898622692,-0.6865835190,-0.8698310256,-0.1018036604,-1.2498087883,-0.9337173700,-0.6132207513,0.1670824289,0.5964810252,-1.0693768263,1.1424970627,-0.4055011570,-0.0484616831,1.4438123703,-0.3742063642,1.9146231413,-0.8712691665,-1.6244757175,-0.9451323748,-1.8532702923,-0.4922710955,-0.7652184367,-1.5269926786,1.0519231558,-0.7186142802,-1.1369289160,0.1738671213,-0.8412277102,-1.2510764599,-1.5560952425,-1.2814587355,0.2802022398,-0.1892165393,-0.3990669250,-0.4418022037,-0.1641026884,-0.0208841823,1.4467082024,-0.1773938984,-0.2656075656,0.9284580946,0.8337346911,1.3918434381,-1.5541487932,-2.3029310703,1.2056473494,-0.7153151631,-0.6681523919,-1.8057876825,3.0629129410,1.0158607960,-2.2026958466,1.0273900032,0.9297968745,0.7835995555,1.2467330694,1.9681586027,1.1947727203,-0.2920994461,-0.2260138839,-0.4726005197,0.7220215201,1.1053067446,0.6730644703,0.2965455055,0.5883324146,0.9228509068,0.1861357391,-1.1552476883,-0.1277422905,1.7361689806,0.5216132402,-2.2289402485,-0.5762023330,-0.3333890736,1.0773743391,1.5460308790,-0.4793303311,-1.2360799313,0.8915560842,0.5807973146,-0.9052379727,1.8003457785,1.5324009657,-1.0009678602,-1.3704814911,-2.0461034775,-1.6933704615,-0.3919988871,1.0435934067,-0.1997463107,-0.2654405534,-0.3796259463,1.1171686649,-1.1622449160,0.2430635393,1.2648487091,0.2460895479,0.7015656233,-0.2465724051,1.4588391781,-0.3708950579,0.1133627743,-1.8181365728,-1.2764829397,0.1797037721,-0.9096319079,0.1833432168,0.6413455606,0.2378626913,1.0471014977,2.1745960712,-0.2824437916,0.6198886037,1.4454858303,-0.6241021156,-0.0874009505,-1.2409427166,0.6960406303,-0.4450871348,0.4783905447,-0.7847806215,0.3396017849,0.8402874470,-0.3396469653,-0.5243570805,-0.4028583765,-0.4860184491,1.6486966610,0.9750620127,-0.4632399082,-0.0831132308,0.4595403671,-0.3360282183,1.2276799679,2.9337465763,-2.7567064762,-1.8911068439,-0.5368624926,-1.2443982363,-1.9356986284,-0.4737740457,1.1805994511,0.4548642337,-0.3293735385,2.1251163483,-0.1460131258,0.5125373006,-0.6775860786,-0.3787424862,-0.4837478399,0.1269359440,-0.5968005657,0.0434482470,0.0882226899,-0.4975458682,-0.8245134950,0.5286591649,0.0518661812,-0.1529803872,0.2797895372,-0.2412377894,-0.3942654729,-1.4258618355,-0.1979169846,-0.4607566595,1.9837756157,1.3194955587,0.4016983807,-0.1167668849,0.0360824801,0.7133179903,-0.0223064348,1.4471542835,-0.4008077383,0.9587165117,1.2700641155,0.2602819502,-0.1262758076,-0.7956354618,0.9213441014,-0.8105702996,1.2343528271,0.1792832315,1.0454692841,1.8160818815,-0.2565951049,0.6567875147,-0.9121094942,0.0882756636,-2.3321690559,-0.2221315503,0.4362458885,-0.4040724635,-0.9728463888,1.8061887026,2.8223960400,1.9238047600,0.4281252325,-1.0113406181,-1.0569467545,0.7463124394,0.7058548331,-0.2373242229,-1.2442312241,1.4772787094,-0.1161244661,-1.1747477055,0.0377737507,0.3676355183,1.0512756109,1.0939421654],[0.3265855610,0.5304086208,-0.4535779953,-0.1297783256,0.9609361291,-0.6423221827,0.1481865197,-0.1527392268,0.2322229594,0.6704090834,-1.1654951572,-0.3747846484,0.7840104699,1.1033495665,0.5778945088,0.5689970851,-0.0741880238,0.8559090495,-0.6340079904,-0.2442087978,-1.2459036112,-1.0313720703,-1.2667640448,1.8217892647,0.8577761650,2.7689638138,-1.1735712290,-0.5874087214,-0.7750111222,-0.9776890874,-0.9262546897,-1.0075428486,1.1539785862,-0.2632883787,1.9082475901,0.7450637221,-0.4940591156,0.6841157079,-0.5536890030,0.6943057775,0.6760548353,0.9306396246,1.3879038095,-2.4359529018,1.0196198225,-2.3955872059,1.2872189283,0.4549034834,-0.0617699176,-0.7158759832,-0.4316509962,-0.6489620209,0.5559383035,1.0439940691,-1.3500230312,0.6517568827,2.4761118889,-1.4052060843,1.4435579777,-0.4848107994,0.0389968641,0.7329373956,0.0906467363,1.3932663202,1.6153687239,-1.0152901411,0.3226601481,0.3640216887,-1.2530380487,-0.3811959624,2.3525457382,-0.3899695873,-0.3235520124,-0.5818331838,1.0788103342,1.2848529816,-0.5897274613,1.8995616436,-0.0095527787,0.4356729984,-1.2555909157,-0.7084031105,-0.5457687378,0.3907106519,0.4804974198,1.0105402470,-1.1896303892,-0.9705888629,0.7118433118,-0.6612381339,-0.3134835660,-0.6854033470,-0.1347203255,1.0188876390,1.8438681364,-0.5356816053,0.8939056396,-0.2387982011,0.3773968518,-0.5067309737,0.3475973010,0.1116873249,-0.3291538656,-0.4861176014,-0.1610072404,-2.0568583012,-1.5355416536,-0.7158253193,-0.9671429992,0.3337282538,-0.5167418122,1.8286303282,-2.3692855835,1.1425521374,-0.4168982506,1.1942421198,-0.5849003196,-0.4810088575,-0.2370169908,-3.2313024998,0.1890334487,0.2700953782,1.3857370615,1.1544367075,-1.5452423096,-0.7408530712,0.3481022120,0.4148260951,-0.3747694790,-0.6615316868,1.5172027349,-2.1836273670,0.0456692502,-0.0833318532,-0.9288641214,1.1208633184,-1.5847899914,0.8110761046,0.2517983913,0.3271596134,-1.8162002563,0.4868193269,0.9116107225,-1.9339040518,-1.3720557690,-0.6756509542,-0.1909210086,-1.2321888208,-0.0097519904,-0.6435344219,1.5714240074,1.3172254562,0.7528502941,0.3804392219,0.7566663027,0.3085334003,0.4007671475,0.3258664608,-0.9499855042,1.4211901426,1.2636379004,-2.1053454876,1.0966620445,-0.2560794652,-1.1866722107,-0.1082164645,-0.9384649992,-0.4065252841,-0.3620557785,-0.6806376576,0.6351390481,-1.2025653124,-1.6817848682,0.4998220205,1.2873026133,0.6916964054,-0.1651842296,1.1613186598,-0.5508473516,-1.3952718973,-0.7212381959,-0.2760586739,-0.9742767215,0.9609985948,1.8242496252,2.2517738342,-0.8537518978,-1.2370250225,1.6758298874,-1.2859879732,0.8290636539,-0.0007398854,-0.6669198871,0.7239009738,0.4930982590,2.3166418076,0.2593319416,-0.4904332459,0.9100781083,-1.5406177044,1.0337108374,-0.0417323932,0.1815911382,0.8316265345,1.1683583260,0.2908100188,-0.8651617765,0.1349878907,0.5809453130,0.6313925385,0.9107890129,-2.0079622269,2.2667987347,0.1362239718,-0.4542431533,0.8893927336,-1.3563278913,-1.4928052425,-2.5350847244,-2.5609161854,-0.5255851150,0.0438330211,-1.6740049124,-0.3690219522],[2.2857151031,0.9468349218,-0.3893817365,-0.0589064620,0.4195207953,-1.5684384108,-0.9302867651,-0.9671948552,0.6613282561,-0.8047224283,-1.7897052765,-0.2297337502,0.0934528559,0.2821525931,-2.0221195221,0.5018086433,-0.1606549770,-0.0830269158,1.6817605495,-1.3330610991,0.2334875464,0.9362577200,-0.8867135048,0.6424555779,0.6285135150,0.8862875104,-0.7600226998,-0.1048611850,-0.1568768024,-0.1049152017,-1.1293969154,0.4765573442,-0.4045842588,1.0629390478,-1.0136929750,0.1548542231,-0.0260496736,0.7368959785,0.6437106729,0.8629876375,0.9513613582,0.5345712900,-0.3428974450,-0.8951515555,-0.7072141767,-0.8290418386,-1.9441916943,0.3352486789,-1.6981418133,-0.6064324975,0.9298272133,-0.0455442220,0.4545024931,-0.6905018687,1.6557446718,-1.8656061888,-0.0764164850,0.1352482140,-0.9649400115,-1.1107592583,0.8871802688,0.3857004642,-1.3479410410,0.2313199341,1.1432842016,0.9774471521,-0.8249517083,0.9975424409,-0.4786781669,0.9934655428,-1.5249751806,1.4200520515,-1.3656040430,0.1201347858,-1.4304561615,-0.5459815264,0.3317955434,2.7635042667,0.5319283605,-0.4035831988,0.1656674147,-0.0348275118,0.9128665924,-1.0659881830,1.5218554735,0.4108059406,-0.8818579912,-1.8898864985,-0.8056340218,-0.2248883545,-0.3115158677,-1.9761034250,0.2347872108,0.0435139611,0.8959118724,-1.4635934830,-1.4139857292,0.5881743431,0.4636629820,0.4652281404,0.1577140689,-0.4330151975,-0.6331516504,2.2929763794,0.1627715528,0.4049665332,-1.0525262356,1.3194918633,-0.2121516019,-0.8326429129,-0.1556195468,-0.2294962704,1.7370721102,0.0576815419,-0.3365804553,-1.9751281738,0.3499570787,0.8850831985,0.6897427440,0.3805027306,0.8950417042,0.5748343468,0.9603128433,-1.5605746508,-0.9406257868,0.1844527274,0.4658717513,0.5011162758,-0.2088306248,1.1758521795,1.1635521650,0.4772557020,-0.7171283960,-1.7251302004,-0.7932379842,-0.7980470061,0.2869731784,-0.9468921423,-0.0588519983,-0.9246168733,-0.4363086224,-0.7276189923,-0.0040393011,0.1989732385,0.8241624832,0.4181136787,1.7991360426,1.6254104376,-1.0304411650,-0.8917178512,1.1133543253,-0.8911256790,0.3151792586,0.9218831658,0.9283659458,-0.7275673151,-1.8838251829,0.8151659966,-0.3029778004,0.3363099098,0.7451679111,0.6459904313,1.5648649931,0.4582104683,0.6197059751,-1.5433871746,-1.1824321747,1.0177524090,-0.7003726959,0.1140275300,0.5453913212,1.4336909056,0.2446464747,0.3940807581,1.9681952000,0.6926275492,0.2997940183,-0.5876314640,1.6486163139,1.3615334034,0.3137921095,0.7396493554,1.2286649942,0.6699882746,0.6917827129,1.3034600019,0.4483648837,-0.1363343149,0.9878246188,-1.0063430071,-0.3920277953,-0.2899178565,0.5260047913,0.6267307997,1.3453166485,0.2274762243,-0.0629633963,-1.1903443336,-0.6259784102,-1.4462623596,-1.6484795809,-0.9891631007,-0.6025660634,0.4475584924,0.2266521007,-0.0169600137,0.3846054673,0.3575282097,0.6688492298,-1.5007214546,1.0728921890,-0.9921048880,-0.8433125615,-0.2272939533,-0.5822410583,-1.4138011932,1.1975219250,0.6921180487,-0.2646894753,-0.7491828203,-0.4853253067,-2.2885367870,0.1974256933,1.4541417360],[-2.2559049129,-0.7094967365,0.4081711471,0.0805931389,-2.1951055527,-1.2974768877,-1.3876715899,2.2653400898,-0.9290544391,-0.0399216227,-0.6982507110,-0.5406550765,0.2805226445,0.2198031545,-0.5062025785,0.1667801738,-0.1336404383,-0.2559153736,-1.5829130411,1.7256107330,0.3249048591,0.7238846421,-0.0049705473,0.1070211902,-0.0440496467,-0.5861712694,-1.2517702579,0.9920070171,0.0184310395,-0.2521992922,1.0478001833,-0.8511126041,-1.0877283812,0.8057253957,0.5307689905,-0.2180033922,0.8828327656,-0.4397656024,-1.9520184994,0.7932648063,0.5082966685,0.3833715320,-0.9113702178,-0.3640641570,0.7479833961,0.6907197833,-1.7358108759,-1.0114843845,1.0597826242,1.4337526560,-1.9482045174,0.1001625508,-0.9070497751,0.6013016701,-0.3366676569,1.5244055986,-0.4733226001,-0.4377292693,-0.6595449448,0.9301330447,-0.4823285341,-0.4746858478,1.0638920069,1.8176459074,-0.0334640481,1.5302972794,-1.4440655708,0.3651750982,-1.0116682053,-0.1984334737,-0.1631214619,1.5133635998,-0.7044069171,2.1420927048,-0.1501505226,0.0840540752,0.5596846938,-1.3633971214,0.6780669689,-0.2560397387,-0.3821427524,-0.0927638486,-0.6855667830,2.3247995377,0.0162809864,-0.8881989717,0.5334579349,0.1402356327,-0.9644435048,0.2259918451,0.4266486466,0.7260495424,0.3118670881,-0.6226944923,2.6717684269,-0.5756723881,-0.4234935641,0.3955070078,0.6205416918,-1.3463940620,0.3289612532,1.3118835688,-1.3313224316,1.2080878019,-0.4787815809,0.1577800959,0.0663677827,-1.1360330582,-2.5606052876,-0.6011534929,-0.3796326518,-1.1565338373,0.1681053191,-1.2738326788,0.8146806955,1.1786562204,0.3972067237,1.4141077995,-0.8077833652,0.2641517222,-0.6645264626,-1.1914525032,-1.9402074814,-0.1699234396,-0.1685755104,1.9913597107,-0.5442168713,0.7998579741,-0.2674789727,0.2076923251,0.8118554950,3.0703585148,0.6901968718,-0.7633002996,-1.9414342642,-1.8525712490,-0.3263784945,-0.7097280622,-0.3387582898,0.5192115903,-0.0137907621,1.2764714956,-1.2416212559,-1.1341321468,-0.2834493518,0.2832931876,-0.8440576792,-0.3899221122,-1.2861567736,-0.0039009314,-0.3991547823,-0.1507866830,-1.0784094334,0.6873516440,0.2398162633,-0.8915411830,0.0160620846,0.1190035343,1.3595669270,2.2851662636,1.2656277418,0.5647453666,0.4386366606,-0.2786923051,-0.0450435877,-0.9073960781,0.8157382011,-0.2528479397,0.1099881604,-0.5259025693,0.9864002466,-0.3098907769,0.1118538901,-0.1846686453,-0.4830997586,1.3937410116,0.6692713499,-0.2855373621,-0.3015303910,-1.2393440008,1.0436235666,-1.3757392168,0.3743044734,-1.0209194422,-0.9742291570,0.4092015922,0.2553415895,-0.6028506756,-1.3848016262,-0.2793629766,-1.3943332434,-0.2918968201,0.9001582265,-0.4785775244,-0.6464923620,-0.0132998796,-1.1716102362,0.2067309022,1.0480587482,-1.4701331854,-0.2972986698,0.7241115570,-0.4221785963,0.3793757260,0.5570320487,0.8278382421,-0.5132340789,0.4097019732,0.1758598834,2.0416235924,2.2838354111,-0.4602448940,0.4201773107,-1.0172317028,-0.3338675797,-1.9052309990,0.0349817537,-0.3715404868,1.1575589180,0.4315404296,-0.6823685765,-0.2479079217,-1.3237612247,2.5582401752],[0.6202364564,-1.0272048712,-0.7202998400,1.8100709915,-0.2270940989,0.4384859204,-0.4506998360,0.4351607263,-0.1438863873,1.5774437189,-0.4889984727,-0.7328954935,-0.9708195329,0.1281945407,-0.1182762161,-2.0853116512,-0.9564751387,-1.6145592928,0.5195460916,1.4585510492,-1.0158007145,0.0924392417,1.1844977140,1.8504533768,-0.3173408806,1.0457521677,-0.8087899089,1.1489765644,-1.3961595297,0.2893914282,-0.2376532108,-1.2573392391,-0.8585844040,-0.4192205966,0.6898103356,-0.1702734679,0.7137411833,-0.1038140580,-0.5801587105,0.3293689787,-0.1780804843,2.1423149109,0.5859926939,0.8150727153,0.0960851610,-0.0801161528,-1.5072997808,0.3067455888,0.9733224511,-0.6691715717,-0.0896074921,0.7155662775,-0.8350681067,-0.8200449944,-0.2977332771,0.4310656786,-1.8563512564,1.2392419577,-1.5641089678,-0.0520131513,-0.3065806627,-0.1996008605,-0.1993302405,1.4793595076,-0.6917536855,1.1229088306,0.0617509261,0.2330419570,0.9543156028,0.3231183290,-0.3006335795,-0.5459107161,-0.3147141337,-0.0181828812,-0.2925636470,-0.1001829058,-1.5876607895,0.1932607740,-0.6446776390,-1.1246975660,0.7368519902,2.3475573063,-0.1616914719,0.3552694023,0.0810829923,-0.2951989174,0.3914378285,0.3118802309,0.6587822437,1.0558242798,1.5984497070,-0.2858626544,-0.9805120826,-0.7844312191,0.2835466862,-0.6526885033,-0.2016610503,0.8101894259,-1.3438426256,0.1658969969,-0.5662457943,1.3452924490,-0.2444649488,1.6518733501,-0.9791072011,-0.7222418189,-1.0671290159,0.4526766241,0.4453455806,1.2471383810,1.1432536840,-1.8908467293,-0.4724683166,0.4400090277,0.4122885764,0.7332125306,-1.7321538925,1.9177784920,-0.2359334975,0.7171579599,-1.1484898329,0.2975558341,0.1324217170,0.8498703241,2.0220580101,0.0642231554,2.0187501907,0.6166564226,-0.3392534256,0.8539775610,0.3858638108,2.0248098373,-0.1607054621,0.4017896354,-1.4190493822,0.3904750347,0.2559507191,-1.5319329500,-0.1452561766,-0.4501852691,0.3744405806,0.7547407150,-0.3966326714,-0.1949160248,-0.9319941401,0.5192106366,0.1668777913,0.7877882123,-0.5676600933,-0.5376181602,1.7631980181,-0.4717208445,-0.2970747948,-1.6887958050,0.8346431851,-0.2908143401,0.4495654404,1.1724085808,0.3258660734,-0.3408461511,0.9547800422,0.7426418662,0.7121489048,0.0964861736,0.3108291626,1.7955456972,0.6886371970,0.6476287842,-0.9715218544,1.6482530832,-1.3654054403,-1.4465010166,1.8486927748,-1.2850440741,0.1249407083,-1.0885308981,-0.2485388815,0.2764838040,0.5420431495,-0.4004425406,1.9209617376,-0.6851661801,1.1391372681,-0.1613515317,0.4473843873,0.4043840468,-0.2895264924,0.5565383434,0.7464129925,0.6821542978,-0.6891804934,1.7582477331,-0.0945984796,1.8338278532,-1.2337061167,1.6532959938,0.5494328737,-1.1798790693,1.4739180803,-1.6404334307,-0.5586791635,-1.6484837532,2.4162697792,-0.3091666996,0.0423371978,2.1097922325,1.0761109591,-1.7526555061,0.1723993570,-0.3163443506,0.3997488320,1.1593134403,0.2288526297,0.9422805905,0.4081701636,-0.6158750653,0.4793061912,-0.7150249481,1.4441509247,-0.4498966634,-2.6479845047,1.2745366096,1.1473503113,1.8102370501],[0.1164595559,-0.6753442883,-0.6330440640,-0.3043009639,0.6173820496,0.4520477951,-0.9501532912,-0.4605540037,0.0309729315,0.2326854914,0.4442227781,0.2579292059,-0.1524424553,-0.2695141137,0.6597102284,0.3571333289,-0.9604148865,-0.1528694034,0.4026290774,0.8940368891,-2.1126663685,0.6282519698,-0.9926656485,1.4733834267,0.6488913298,-0.9213027358,-0.5731467009,-0.6374110579,0.6401329041,1.3533540964,-0.3756834865,2.6135437489,0.1940688789,0.0630342886,-0.1559038311,-0.8732242584,1.2136697769,-0.4508092403,1.0960631371,0.3103138506,1.1365141869,-0.8697562218,1.2769855261,-1.9234557152,-1.1386402845,1.4014794827,-0.9552711248,-0.0660765618,1.6641439199,-0.8903948665,0.0384727232,-0.9314847589,0.7644599080,1.3544265032,-0.8408039808,1.2758435011,1.6310828924,0.8584744334,0.7314692736,0.2873908281,0.0925639942,-0.1756385863,0.7188848853,0.1778239012,0.0075021056,-0.3179973066,-0.8962370157,0.5907183886,1.3393832445,0.1291782856,0.2163106054,-0.1918319762,-0.4468945563,0.1792847812,-0.2239666730,1.4377971888,1.4812185764,0.4640325606,1.1958353519,-1.6090112925,0.7192595005,-1.8846025467,-0.4219196141,0.8524013162,-0.3068114221,0.7166014910,1.0810377598,0.0509062372,-0.8165169954,2.6991014481,-0.4963667691,1.1587369442,0.6737996340,-1.4490827322,0.6106519103,0.3198771179,0.0267605837,1.4674906731,0.4409443736,1.1011567116,-0.4959523380,1.0790488720,1.3844223022,0.1370456964,0.4587925076,1.1820701361,1.2755533457,1.1850805283,0.5455582142,0.2740237713,0.3107175529,-1.3007590771,1.2341132164,0.5541449189,-1.4907549620,-0.3145397305,-0.8253541589,1.0124372244,0.4967445433,-0.1998725533,2.3858799934,-0.2239804566,0.6783373952,0.6622564197,0.5010342598,-0.8637469411,-1.2166178226,-0.7888168693,-0.4862669706,-2.1496996880,0.2434635609,2.2176091671,0.6138218641,-0.1121539772,-0.0708785951,0.2982955277,-0.8135273457,-0.1176461875,-0.3864695132,-0.3397121727,1.9965733290,-0.4525742233,-0.7801883817,0.1617140174,2.0445573330,-0.6421329379,1.0683028698,1.4681972265,0.4830242693,-0.7713213563,0.9028714299,-1.1294329166,1.1345814466,-0.5910274386,0.6970822215,1.0826804638,-0.8912727237,2.1045033932,-0.1427501440,-0.0198568925,0.4012412727,0.5061278939,0.6106132865,0.8168576956,0.8838734031,1.8924901485,0.1710574180,-0.7019839287,0.3490320146,1.5296567678,-1.1244053841,-1.1642284393,-0.4193575084,0.5746083260,-1.1689153910,0.2877218127,-0.1881029308,-0.3931901157,0.4175453186,0.7864643931,-1.7730191946,1.7171610594,-0.5239853263,-1.6591190100,0.2596905231,2.7576310635,0.3869003057,-0.9029765129,-1.6944363117,-0.6992952824,-0.1057624966,-0.2710611224,-1.0353348255,-0.3324691951,-1.1543858051,-0.8658341169,0.1334683746,-0.1229031235,0.5339189768,0.6564360261,-0.4034191966,0.4229221046,-1.0462310314,0.4336838722,-0.2808940113,-0.6553801298,-0.0031763159,0.1415689141,-1.3357542753,0.0714949444,-0.4073644578,0.4600543976,0.3203541636,-0.1955058873,-1.4824770689,-0.4383046925,0.8585121036,0.0071368990,1.4751278162,0.6233286858,0.6207359433,-0.5246831179,-0.0414664410,1.7709965706],[-0.0709905550,-0.2821904719,0.1701576114,0.8789879084,0.8838764429,-0.3726766109,0.7503628135,1.8942296505,-0.5893586278,-0.9376010895,-0.4995022118,2.0414099693,-1.1939601898,2.3895537853,-0.5800492167,-1.4941869974,0.3914678097,0.7654280066,-0.3711814880,-0.1482408494,-0.8229711652,1.0125423670,1.9014531374,2.1342532635,0.8377076387,-0.5943709016,1.0816322565,-1.4450104237,-0.9806501865,-0.0139226625,-0.0261511095,-0.1476468891,-0.1782859713,0.0654901490,-0.9970036745,1.8520280123,0.2998270392,0.4222496748,0.3607931137,-1.9088587761,-1.3702299595,0.1112247109,-0.3712612391,-0.8905547857,1.5990093946,1.0519772768,-1.6142657995,0.9828546643,1.1408561468,1.0751584768,-2.1395509243,0.8739467859,0.8484569192,0.4239789844,0.7949110866,-2.2604801655,0.2834377289,0.1236173958,0.6627572775,-0.1966073513,-0.8987166882,0.1549092829,-1.7537448406,1.3848297596,-0.0972593948,0.2573923469,-0.3925730288,-1.3261764050,0.6730273962,-0.0823751241,-0.2015870810,1.3150098324,-0.4746868014,-0.9555039406,0.0567918830,-0.2349265516,-1.2320506573,1.6329518557,-0.3194123209,-1.8545962572,-1.5797518492,-0.7913144827,-0.5306677818,-0.7435374856,0.0374403261,-1.5283485651,1.6451562643,-1.7834663391,-0.1449975967,0.9374970794,-1.1378582716,-0.4348377883,-0.3451203704,1.6907564402,-0.0922324732,-0.2340613306,0.3029639721,0.6703021526,-1.2160876989,-0.1070491970,-0.8925184608,-0.9153434634,1.5998293161,-0.2169963121,-0.0921376273,0.7837740779,0.3923157454,-0.4793142974,-0.6740366817,1.2255915403,2.2740068436,-0.3493728042,0.7215506434,0.4427873790,-1.2372415066,0.3400833905,0.3696044385,-1.1532834768,-1.3326276541,-1.3978134394,-0.7902693152,-0.0793885812,-0.0871508643,-0.8185797930,0.3032013774,-0.9155107737,-0.4498315752,0.0133975549,1.2002675533,-0.0543997772,-0.5718390942,-1.1672619581,0.2706866860,-0.6121038198,1.5340633392,1.3675135374,-0.3314547539,-0.6407935023,0.1904756725,-0.0176693890,-0.1542854309,-1.0666533709,-0.8065037131,-0.5172939897,0.1125126705,-0.8618041277,-1.0027049780,0.1088037044,0.8530612588,2.4807577133,-0.4043349624,-0.6118181944,0.4480062425,0.1232022196,0.2338018715,0.9920058250,-0.8674075007,0.3546648920,-0.7531710863,-1.1711133718,0.5628138781,-0.4150498211,0.3105407059,-2.4759883881,1.6793575287,1.7537196875,0.1912416220,0.2514575124,-0.5988570452,0.0741735473,-0.2699765563,-1.2148730755,-1.7194465399,0.2601687908,0.6379894614,-0.1272419095,-0.5413345098,-1.0075840950,0.4337921739,0.4659135342,-0.3689009249,0.0981860682,-0.5749191642,-0.5014486313,0.3198375106,-0.7704556584,0.9145529866,-0.4085680246,1.4911134243,1.4047791958,1.1867433786,-1.6885709763,-0.3014551997,-0.1118771732,0.7182075381,1.0829957724,-0.8519905210,0.1300453693,-0.4334434271,0.4877637625,0.8054375648,-1.4181320667,-0.4921872914,0.4618836343,-0.6419098377,0.4662364423,-0.5979802012,0.3456205726,-0.3643333912,-0.2508301735,-0.2602106929,-0.9041038156,-0.7807335854,1.4818336964,-0.0963200405,0.9439044595,-0.0088408943,-1.5938279629,-1.6104105711,0.1701638550,0.1085248068,-1.1758224964,0.3988420963,-0.4401808977],[1.3763401508,-0.4395669699,0.4183611572,-0.8105944991,-0.6222549677,0.2567108572,-1.8000807762,-0.3589302897,-1.5436463356,-0.1948490441,-1.3861801624,-1.2604641914,-0.4003143311,2.4831156731,0.9226174355,-0.3849322796,0.2759652436,1.2357858419,0.5366231799,0.2259910405,-0.2262105793,0.9908719063,0.6953120828,-1.4158452749,-0.0114128450,1.1648558378,0.0026331618,-0.6417639852,1.7007479668,-1.0409736633,1.2814551592,-0.6953024864,-0.6436872482,0.5956673622,-0.8900166154,0.8791947365,0.4307781756,-0.5199248195,1.3481355906,-0.3190667331,0.5465198159,-0.1034876481,0.2821509540,-0.2298322320,-1.2568703890,-0.5725135803,1.8433989286,-0.1977019906,1.4076530933,-0.4182282388,-0.0286837257,0.4214601517,0.2952394187,-0.1422851980,-2.3230524063,0.5700261593,0.8659446836,0.4792743027,-0.0860511214,-2.4152247906,-1.1893260479,0.6089113355,-0.3519750535,-1.0636354685,-0.9764991999,1.4291633368,0.8169809580,0.7607301474,-0.8915486336,0.2099952251,0.7656538486,-0.9530802369,0.9028717279,-0.6837580204,-1.1406611204,-0.0980555266,-0.8678302169,0.0618282855,-0.9474887848,-0.4634743035,-0.8415288329,-0.2886526287,0.3071372211,0.2244463861,0.3965576887,1.1935093403,0.7316077948,-0.6506578326,0.2081241310,0.6839416623,-0.9692512155,2.0871205330,0.5729706883,3.4624316692,0.5383459926,0.3396901488,-0.5575518608,-0.5723373890,-1.7078907490,-1.6338896751,2.9699966908,2.5188298225,0.0883686095,2.0530881882,-0.6892683506,1.7217940092,0.8135957122,1.0825539827,-1.3073396683,0.2213107646,-0.8297741413,0.8229870200,-0.2925997078,0.2938055098,-0.2028401792,-1.0501542091,0.3662391901,0.0436582863,2.8220024109,1.6359175444,-0.2000506669,-0.6892738938,0.6414157748,0.1814173460,-1.2759850025,0.7353927493,0.9251579046,1.1306307316,-0.8013715744,-0.3105247319,-0.2614181936,1.4937690496,-1.1773779392,-0.9231160283,-1.9470348358,-1.2480545044,0.1362063885,0.6162065864,-0.4116608500,0.4412190020,-1.4277477264,0.7063032985,0.8237750530,0.4653929472,0.2365056872,0.0285083521,0.9960690141,-0.8011170626,-0.5131413341,-0.4017004371,-0.2410633862,-0.6117011905,0.5667151213,-0.4457055628,-0.7197325230,-2.7344152927,-0.2064120770,1.0151226521,-1.3595682383,1.3549883366,-0.1983778775,1.3015027046,-0.1049090922,0.7218281031,-0.1797719151,0.7467459440,-1.6058382988,0.8894509077,0.2281947583,1.1969484091,-0.6350207329,0.7831915617,0.5507146716,-1.3282835484,-1.1195918322,1.0832135677,0.4004996419,2.0190610886,-2.1272804737,0.5968118906,-0.6430553794,-0.9771640897,0.4279308915,-0.8297137618,-1.1755281687,1.0167371035,-1.0099674463,-0.3996248245,0.8061579466,0.5100678205,0.6430982351,-0.0116527425,-0.0569755547,-0.8373223543,-0.1155486926,0.8459715247,0.3501706421,-0.7146452665,-1.3728239536,-0.6906177402,-0.8787280321,0.3087691367,-2.0644254684,-0.8531473875,-0.7981085181,0.9267256856,-0.2735356688,1.6844605207,-0.3576886952,0.4597080350,0.5911514163,0.9546337128,0.7858938575,-1.0806646347,0.0153782917,0.8166173100,1.3586717844,0.6759258509,-1.1599477530,-0.9371449351,1.9049314260,0.4763910472,-0.0252852160,0.5366802216],[0.1893805712,0.0235877261,-1.2631338835,0.1262231022,-0.4659309089,-2.0677807331,-0.2172206342,0.2418677211,0.2863640487,-0.5541240573,-0.4875922203,0.9178158641,0.8355688453,-0.3356583416,-0.4909035861,1.3364617825,-0.5274421573,0.0552913696,1.2746754885,-0.4226909876,0.6258471608,0.8982531428,0.7511479259,-0.2033134550,-1.7190890312,1.2277430296,0.3163192570,0.2681086063,-0.3645800948,0.3647791445,0.5008106828,-1.0483267307,0.0229089204,-1.1120997667,0.4731096923,1.0203477144,0.0132737607,0.3177182376,0.1482971162,-0.9006447196,0.7418464422,0.8549311161,-0.4601553679,0.4142823219,-0.5291186571,-0.2832478881,-0.9400891066,2.0045392513,0.9907512665,1.2803305387,-0.8706706762,-0.0438755862,-0.1437354237,0.1352631003,-0.2147992402,-0.4306062162,0.7384632826,0.1026570722,-0.9193673730,-2.8592047691,0.5031278133,-1.6948381662,0.0623850375,0.4504397213,-0.2354303449,2.3214147091,-1.8391581774,-0.0035846911,-2.3443999290,-0.3101183176,1.7663384676,0.3807913065,1.7143090963,0.2600724101,-2.4273955822,1.1260516644,-0.1763039678,1.6863145828,1.0932096243,-0.2558988035,-1.1969647408,-0.1792815775,-1.8289457560,1.5591448545,0.8391063809,1.2969179153,1.6576949358,-0.4130849540,-0.7088075280,-1.2217174768,0.1126335263,0.0402188487,-1.5738127232,-2.1769027710,-0.2445770651,-0.3699555695,0.3325831890,-1.2605797052,-0.0567351133,-0.0740334541,-1.6811589003,1.3763895035,-0.4706969857,-0.9843922257,-1.1051831245,-0.4698985219,0.8479211926,-1.0044732094,0.2730768919,0.8285003901,-1.1652021408,0.7067195177,0.5655376911,0.4849031866,-1.0408544540,0.2165114880,-1.6860473156,2.1083023548,0.6807785630,-0.3152302206,0.8114103079,0.0277315732,0.8226593137,-0.7260422707,0.0740356222,1.3696192503,0.4227825999,1.0678523779,-0.6593800187,2.4530041218,-0.5130468011,1.3998401165,-1.1113494635,2.8427562714,-1.2515159845,1.7911391258,1.4213436842,0.1319545954,0.9271252751,-0.5539256930,-2.1196947098,0.5384703279,1.7564214468,0.2037496120,-0.4946107566,-1.3231574297,0.6391705871,-0.2147297114,-0.9286428690,0.3146469295,0.3201300204,1.6130293608,0.9120296240,0.2281010896,-0.9520408511,-1.7568795681,-0.0860790312,-0.3216405213,1.5971487761,0.2164636552,-0.8063322306,1.4201835394,-0.9114590883,2.0482690334,0.4544996917,-0.9965992570,1.7234151363,0.4029419422,-0.0605051927,1.1774067879,-0.1526352167,-1.3627843857,1.0119982958,0.5999497771,0.1030077115,0.0237347633,-0.9818653464,1.2113623619,-1.0080310106,0.6341429949,0.3184710443,-0.6536894441,-0.3419989645,0.8524641991,-0.0375635810,-0.4750849605,-1.1283820868,0.0493629947,0.1907393783,1.8657075167,1.6624958515,0.4755136073,0.3853852451,-1.7855322361,1.5675200224,-0.3439013660,1.4749809504,0.4404431283,-2.3438589573,-0.6509875059,1.7350389957,0.5399621725,-0.7919213772,-0.7674368024,1.3210825920,1.8883692026,0.7461881042,-0.1613261849,0.9186052084,0.9145761132,0.4075089395,1.4331464767,0.7037459612,0.5462380648,-0.7761021852,1.0948152542,0.9146957994,-0.7808047533,-0.9392088056,-1.2161570787,-0.4258204401,-0.0622376651,0.6151955724,-0.1300780773],[-1.2082298994,0.1191710830,-0.4793582261,0.3698245883,0.5303626657,-1.4388060570,0.3178140521,0.2438377589,0.5394156575,0.0692542270,0.7180391550,0.0523633435,0.5539424419,0.2073472142,-0.2396894544,0.8435296416,-1.1016947031,1.1617139578,-0.2411237210,-0.2124344260,0.0503696166,1.1276808977,-1.8850216866,0.9905751944,-0.1987105459,-0.0699739307,-1.5369192362,-0.4018937349,1.0958739519,-0.5575975776,1.7045712471,0.7433077097,-0.5954273343,0.6866076589,0.9778364897,0.2530172765,-0.1848416775,-0.4636569917,0.2392144501,-0.3474380970,0.8023971319,-1.1096651554,0.8329088688,-1.2316823006,-1.0217108727,-0.3689428568,-0.6006485224,-0.6255954504,-0.6185607910,0.2374453992,0.7998460531,-0.4682604969,0.4123714864,1.7706925869,1.1507540941,0.9583311677,-0.4108321667,-0.1481068283,0.0342794694,-1.1916863918,0.8081879616,-0.9666482210,1.0906995535,-0.1639183015,-1.1320599318,-1.5155814886,0.3599909842,-0.9653348923,-0.7701853514,0.7542246580,0.0639560819,0.2171070725,0.0094398977,0.4814985693,0.6337531805,1.2952977419,0.0742636099,0.6645027399,0.1214422286,0.6358388066,0.5883584023,-0.6672433019,-0.5090835094,-1.8338547945,-1.2418490648,-1.8587424755,-1.2738869190,-0.4197821021,-0.1913201660,0.8862715364,0.2564136982,0.2612314522,2.2699978352,0.9013975859,-0.6366124749,0.4804432690,-1.0621896982,-2.1655814648,-0.9202276468,-0.6696566343,-0.5409637690,0.7462758422,-0.3009483218,1.7442085743,0.2338610142,0.0837169215,0.3850113153,1.1419742107,0.5510025620,0.8352980614,0.4338721633,0.4724246562,0.2723473907,1.7705891132,-0.2735621929,-0.6433638334,0.6840257645,-0.2410134673,1.8150671721,-1.8267018795,2.3056502342,0.3518390059,-0.3171675801,1.3055222034,0.8078785539,1.3020491600,-0.9885943532,0.8765869141,-1.5162200928,0.2587037385,-0.6520469189,1.3673703671,-1.5630866289,0.9796712399,0.2590078712,0.4991450906,-1.1541076899,-1.1035073996,-0.3751842976,-0.9048180580,0.3524917662,-0.8524733782,-0.6322894692,-0.4022717774,0.6911383867,-1.1806863546,0.0684107095,0.5824618340,0.4333196282,-1.2544018030,-2.2081205845,-0.9804613590,0.1319895834,0.9541259408,0.3229891062,0.0549656115,1.7258254290,1.1441347599,0.7627381086,-0.3379273117,-0.0155780120,0.1145182028,0.0303633697,0.4696642160,-0.2231311798,-0.1135704070,-1.4808541536,-1.3070743084,-0.8091095090,0.2676424682,1.4319150448,-0.1194170862,0.3829805553,-0.4280880392,-1.2592542171,0.5967583060,0.0245815534,-0.5553333163,-0.0784272552,-0.9085083008,-0.2988561690,0.0019547611,1.6142442226,0.0618199073,0.1645226181,0.6071572304,-0.9973375797,1.0976777077,1.4667928219,-2.1746215820,0.3314406872,0.0927470177,-0.2048122436,1.4143673182,-1.1663620472,-0.2159975767,1.9280686378,0.7685463428,-0.1528172344,0.0260222536,-0.7631306052,-0.7862910032,-0.2226518989,0.5308263302,0.8197928071,-1.6154203415,0.3681326807,-0.7986581922,-1.6969778538,-0.8310033083,-1.1390073299,0.4657533169,-0.0444163941,-0.0632248893,-1.8191641569,1.0638192892,-1.3606250286,0.6245934367,-1.3430202007,-1.1037573814,0.2160263807,0.8564948440,0.1884693950,0.1738087684],[0.0949733108,-0.9783391953,2.1088376045,0.8860807419,-0.1600560844,-0.9818916917,-1.6848745346,-0.4343046248,0.6651635766,1.9885319471,-0.2870945036,0.9635787606,0.0447185487,-0.4152676761,-1.3084661961,0.5204074383,0.5992215872,1.9145693779,-0.8939692974,1.0495287180,0.2583296001,-1.0402622223,0.6333838105,-2.7544460297,-1.5477565527,-2.1013078690,0.0633282438,-1.1671156883,0.2825079858,0.6212230921,0.2638114393,0.5210747719,0.2484542876,-0.3608361185,1.8238667250,-0.1757961065,0.4721578658,-0.6213430166,0.2073867470,0.0125586027,-0.5172837973,1.1994928122,0.4062629342,-1.2676839828,-0.5387573242,1.6995825768,0.4816310704,0.8849418759,0.2497922778,-2.0610547066,-0.8003373742,-0.2149155587,-0.2607664168,0.0678154752,-0.1214217842,-1.7614785433,0.4538424313,1.5986852646,-0.5152428746,-0.9982887506,0.2431342751,1.0021549463,0.7107254863,1.2005827427,-1.3013552427,-1.7587054968,-0.8021962047,0.9416404366,-1.0701202154,1.7020624876,0.8600663543,-0.0366093032,1.1781638861,1.1968842745,-0.5840277076,-0.0184764396,-1.8357027769,0.4158545732,0.3273654580,-0.3873716891,1.6004836559,-0.2800341547,0.7892519236,-0.1054666266,0.5070289969,0.3303568065,-0.4675278366,-1.6173273325,1.1189441681,0.3584862351,-1.4084955454,1.0983003378,-0.1029772460,-0.9113822579,-0.6339533329,0.0714032203,0.6224611402,-0.5115113854,1.3892023563,0.5634469986,-0.6348069906,0.2799148262,0.2578309178,-0.7327276468,-0.6066296697,0.5482274890,-0.9846584797,-1.4991323948,-2.6056971550,0.5932595730,-1.4395385981,-0.7592237592,-0.9646526575,0.6389320493,-1.6647413969,0.7618963122,0.5511775613,0.5638883710,0.0919290781,1.2376972437,-0.4143681824,1.6608420610,1.1134301424,-0.3549555540,1.1900693178,-0.2941959202,2.0036094189,0.1889229864,0.2768132389,0.3631108105,-0.4527046084,-0.0876503065,0.4518642426,1.2216837406,-0.5853601098,-1.2460826635,1.4365702868,0.3918838799,0.4344147444,-0.3103542626,-1.4794760942,-1.1437619925,1.4064294100,0.8123748302,-0.8287097812,0.0644502342,0.5039708614,0.0464049242,0.6796227098,0.2334593832,-0.0366540141,-1.4885165691,-0.5560668707,0.0822257921,-0.2212876827,1.6971707344,0.1416028738,-0.3375744522,-0.2809337080,-1.0013574362,-1.1856667995,-1.1524541378,-0.0177265145,-0.1572738290,-0.4203666151,2.2288005352,1.6894711256,2.2608036995,0.0578856580,-1.6283227205,0.5768634677,-0.1903793067,-2.0627646446,-0.4917488992,0.8252843618,-0.9127709270,-0.4938484430,0.9456370473,-0.2809567451,-0.3177897632,1.0033947229,0.2217080295,0.8518112302,1.9643433094,1.8874276876,-1.5707550049,-0.8626135588,-0.2519744337,-0.1130561605,1.0470582247,-0.1848319024,-0.7244028449,1.4268809557,1.9367127419,-0.5947025418,1.2139691114,-1.1336416006,0.4106018245,-1.2320458889,-0.0817621946,-0.6174464822,0.8128951788,1.5221180916,-0.2531063259,-1.6988849640,-0.2455919683,-1.8654962778,-0.8597924113,-1.6687711477,0.0930599421,0.9343118668,-0.5308892727,0.9207680821,1.6391845942,-0.0630735978,-0.5165376663,-2.3341619968,0.9599221349,1.2102273703,0.7616304159,0.0274917819,-0.0581777953,-2.3147866726,2.0967600346],[1.4149196148,-1.0977212191,-1.6241617203,0.0720105991,-1.2375843525,0.3066580296,1.5010496378,0.2146603018,-0.1769921929,-0.2609245181,0.0112806298,1.3085396290,0.1515700370,-0.1343057156,0.5356020331,-0.3263483346,-0.8350747228,0.9119567275,-0.5826714635,0.1920842826,-0.7925053239,-0.3949542046,1.8459690809,0.3976787925,1.0924711227,-0.2221397012,-1.7229070663,0.4710488021,-0.1177469268,0.7858834863,-0.5207809806,-1.7956728935,0.2063074410,0.1493661255,-1.9783360958,-0.6310718060,-0.5640435219,-0.7965235710,-1.4358403683,-0.9247434735,0.4182759821,-0.7810117006,-0.1456603557,-1.8540166616,0.9421648383,-0.9929693937,-2.0774114132,-0.7155947685,2.6200604439,0.1311487406,0.1741613895,0.1489846259,1.5386333466,0.7290978432,0.3322045505,-0.8202380538,0.5158880353,-0.6750556827,1.3344002962,-0.4489007294,-0.1453179717,-1.0834563971,1.0381031036,0.2068814933,-1.0295704603,-0.3696275651,-0.6469484568,0.3838300407,-0.4384109974,-1.5211050510,0.6330561638,0.1468900144,-0.2644926608,-0.0833066702,-0.1666891873,0.1699847132,1.7822586298,1.4239218235,-0.4095877409,-1.2577289343,-0.2261657417,0.1536862552,-2.4841010571,-0.1690602601,1.1236718893,1.3120075464,0.4703026712,0.7917539477,-0.0024559440,-1.0582803488,-0.8924607635,1.9422979355,0.1470964402,1.5199949741,-1.4546091557,-0.2491374910,0.1628889889,-0.0223796498,0.0273962896,-0.8624451160,-0.3553950787,-1.1562964916,-0.5350723267,-1.1832722425,0.0762151852,0.8057109118,-2.6641361713,0.9860156775,-0.3996573389,1.4996862411,1.6512048244,0.9693275690,0.9770048261,0.5826814175,0.7136456370,0.0839540139,0.2687855661,0.7190995812,0.3238495588,-0.0822469965,0.5498287082,0.8601994514,0.4141937792,0.1921910942,-0.5373055935,-0.0399440303,-2.9793040752,1.1761871576,-0.5453553796,-0.2182899714,-0.3996349275,1.3795903921,0.6229868531,-0.1509124041,-0.4743267298,0.3084315658,0.6045871377,0.2694589198,0.8859691024,1.3523759842,-0.3754374087,-0.0136016058,-0.5773574710,-0.1826155484,1.0299216509,0.9875227809,0.3665340245,-1.8713101149,0.4778993130,1.3742740154,0.8267615438,-0.4324137568,2.3989701271,1.5465426445,-0.7949463129,0.8532446027,0.9856741428,1.3061232567,0.8775233626,-1.6787419319,-0.3526584208,1.4827687740,0.3022454977,-0.1343125403,0.8234794140,0.0794778913,1.1625801325,1.5019149780,-1.1354607344,-0.6904141903,2.9148166180,-0.5319117308,0.8360420465,-0.8598858714,-0.3846859634,-0.8635641336,0.7971241474,-1.5385718346,-0.9597117305,1.4342877865,0.0260773115,-0.6131992936,-0.0396655016,0.0818362907,1.3144561052,1.8665846586,-1.1760950089,1.0667315722,-0.2050069720,-0.5495911837,-0.5504364967,0.3499258161,1.7730232477,1.0658493042,0.0046089194,0.5828529596,2.8937945366,0.7601087093,0.4959368408,-0.3398263752,-1.1179248095,-0.0759203956,-0.7791621685,0.5058390498,0.3832107782,0.6314857602,-1.2179418802,-1.2486593723,0.2801901996,1.0210988522,0.3753940463,-1.4467526674,-0.6495608687,0.0395036824,0.2830541134,-1.3890956640,2.1286034584,1.4011929035,-0.0469790399,-0.0941228271,-0.0834299549,-1.4725753069,0.7305154800,-0.2272632569],[0.2950935662,-0.7175719738,-1.7210181952,-0.2352459580,1.8049403429,0.2901862264,0.2384487092,-0.4335539639,2.1697747707,0.5837168097,-2.0065824986,-1.5619883537,0.5572479367,0.6204081774,-0.6503604054,0.3235428333,-0.6237254143,0.0254766438,-0.9556758404,0.9256320000,0.1110662892,-1.3950949907,1.6628248692,-0.5550198555,0.2506259084,-1.2280722857,-0.8849002719,0.2853524089,0.7186892629,0.3485122323,0.4830940366,-0.9126710296,0.0648288727,0.5861564875,-1.1012845039,0.8903179765,-0.5021155477,0.2146847993,1.5970772505,-1.6596252918,1.7038996220,0.5897356868,-1.9231088161,0.1841837913,-0.8422321081,-0.2690632641,0.3078442216,-0.6378785968,-0.2733365893,-0.4018662572,1.1151080132,0.1203839406,0.5301672816,1.9052218199,0.5232292414,-0.5509648323,0.1532472223,-0.6852358580,-0.0168571491,0.2829575539,0.9693676829,-1.3901803493,-0.8579546213,1.6857061386,-1.8683937788,1.1424872875,-0.1706787050,-0.9109064341,-1.0370491743,-0.5145337582,-1.4502618313,-0.8660738468,1.1110023260,0.4196021557,0.2588478923,0.2073096335,-0.7804108262,-0.7706721425,0.2005842775,-1.2953560352,-2.0662872791,-0.1776489168,-1.5487222672,-0.4867807925,0.9630966783,0.0749432296,-0.4267623425,-0.8447555900,0.5929803252,-2.1236004829,-0.0424731523,2.4609448910,-0.9759138227,-1.7832497358,-1.6214101315,0.5515912771,-0.7159795761,0.4605422914,-1.0591150522,-1.1180940866,-0.1050870642,-0.4247130156,0.8251236677,-0.6458255053,0.0270585772,-0.7671965957,-0.6085761786,1.5552158356,-0.6036732793,-0.9812249541,-0.8282076120,-0.2138131559,-1.4566339254,0.5174087286,2.0507590771,-0.1476765126,1.0797405243,1.3560801744,1.0424228907,0.3342956603,1.2495017052,0.7392379045,1.1074708700,0.1809802502,1.7764987946,0.2816728055,-1.9343605042,-0.0202827193,0.3671101928,0.0815860778,0.0512436032,-0.5376491547,0.4952427447,1.7752697468,-1.6656461954,0.1051459163,-0.6446504593,0.8385035396,0.6940054297,-0.5207248926,-0.1721844971,1.7069374323,-0.5153077841,-0.5635568500,1.5965918303,0.1318876296,0.3199602365,0.8840111494,-0.2084402442,-2.4241857529,0.9587558508,0.4362331629,-0.9273656011,-1.2563002110,0.1305574328,1.1341838837,-0.4569153786,0.0596981235,-1.3988307714,-1.5971320868,-0.4953704476,0.7124653459,0.3637164533,1.4194965363,0.6654574871,0.6962931752,-0.0628936514,0.4574303627,-0.8212164044,-1.2626589537,0.7945659757,-0.1253741384,0.2529629171,-0.5118837357,-0.5959960222,-1.4390065670,-0.1195924282,-0.1827707291,0.3372287452,2.5050127506,0.5503866673,-1.8325817585,0.2706216276,-0.2371074110,-0.1965096891,-0.6216363907,0.4986001253,-0.0241942480,1.1589165926,-0.5790826678,1.8832604885,0.2265321314,-0.4650239050,0.8145852685,1.2787189484,1.2368496656,0.2810806036,0.5590633154,0.0165573135,-0.1671055108,0.2981031537,-1.7313406467,0.0702708587,0.9237394333,0.7955741882,-1.0611020327,0.1046951190,-0.6292627454,-1.1767753363,1.6472417116,0.9162951112,2.4193027020,-0.4519712627,0.1546483189,1.0768247843,-0.2482834458,-1.0319919586,0.9817290902,-1.2656478882,0.3102760911,-0.6407822371,0.2745261490,1.4232051373,0.0962205753],[-0.5079040527,-0.2026175261,0.4734267294,0.3398221433,0.1162773520,-0.3965824842,-0.0524844863,-1.3785188198,-0.5009530783,1.3419600725,0.4458302557,1.6235131025,0.5704522729,-0.5884652734,0.1655925512,-0.5934273005,0.8961495757,0.1395948231,1.8811582327,-0.4704824090,0.6056877375,-1.8371515274,-0.3883416951,-0.2261543870,0.1075822636,0.8845273256,-0.2323669195,1.0182594061,-1.2435951233,0.1850990653,0.6556530595,0.7260800004,-0.0777541026,0.6657271981,-1.5444598198,-0.4093579352,-0.1104109064,-1.2403335571,-0.7037749887,-0.0475286730,-0.0110759111,-0.3843451142,0.9995355010,0.2956174314,-0.1288992912,0.8820962906,-0.7885088921,-1.3096528053,0.1360455155,0.6605646610,0.4773285687,0.1000278816,-0.2009449899,0.9680691957,0.9661629796,0.3475872874,-1.0521339178,-0.2399311066,-0.6726133823,0.6403880715,-0.2364929765,-1.4091043472,-0.5644134879,0.7580716014,-0.1551982909,0.0443324931,-0.6542751789,0.1354361475,0.1713111103,0.3366525471,-1.1363209486,-0.8197965026,0.9096105099,-0.3897396922,-1.1618636847,0.2323047370,-0.1626732200,-0.3283954561,2.0063948631,-1.6640892029,-0.2251854539,-0.5884788632,-0.7459271550,-0.3304202259,0.5145342350,0.0916930512,1.8832346201,1.8666164875,1.2778718472,-0.6346173882,-0.6968966126,-1.2571614981,-0.0928414315,-0.8907172680,0.4955960512,2.0003273487,0.3299041986,0.5431902409,0.9838431478,-0.6571618915,1.1528811455,-1.0470914841,1.3171378374,0.2243078053,0.3119038641,-0.3337868452,2.0032920837,-1.0856931210,-1.9538714886,-0.1341106296,1.0241699219,1.2208647728,-1.6466290951,0.2385760546,-0.1349076927,-0.9472653270,0.3611437976,1.8478543758,-2.3713274002,-0.9970322847,-0.0145900678,0.6273401976,-0.2074701786,0.3642231822,0.8874557018,0.6856731772,1.0776827335,0.4891940057,-0.6078050733,-0.3043553829,1.4372410774,-0.9145619869,-0.1745103747,0.8110514283,0.5476842523,0.6497424841,0.5543809533,-1.0524549484,-0.6980483532,0.3797794580,-0.8720604777,-0.0382295512,0.1213534698,1.2860654593,0.6915104985,0.9888854623,-0.4740723670,0.3520619273,0.6744676232,-0.4462754428,-1.4237420559,0.2925428450,-0.4372058511,0.7580588460,-0.1709490865,0.2753054500,1.5881859064,-0.8896048069,-0.9051228166,-0.0149402972,-0.0148151759,-1.5531495810,0.0139444396,0.5084567070,-0.8614171743,0.5642536879,-0.5657557845,-0.1187821403,-0.4563186765,0.3088122904,2.5398862362,-0.0289198011,1.4139448404,-0.8513980508,-0.1655536145,1.7412998676,-0.0283007547,-0.6911320090,0.6281622052,0.6107546091,0.6955740452,0.9466020465,-0.4255324900,1.3051526546,-0.6912388802,0.7697520256,-1.6392782927,-0.4025473893,-0.1954599470,-0.0032584365,-1.1283636093,0.8464527726,0.1064395979,-0.3436134756,-0.9426211715,0.5683631301,0.4941053391,-1.6244103909,0.4565205276,0.7516614795,0.6100686193,0.2671549320,-0.4684554636,0.4916934669,2.2799537182,-1.0597896576,-2.0844984055,-0.0247393716,-0.1110131443,-0.0192019120,-0.8450926542,0.2091219872,1.3058104515,1.6750828028,0.6700471044,-0.2928491533,1.7783085108,1.0262509584,-3.0318641663,1.6394308805,0.4991608560,1.4980392456,-0.1323428154,0.6211748719],[2.0379743576,-0.5224591494,0.3273642361,0.1952998042,-0.4680666924,0.4209490120,0.5322787166,-0.7942463160,-0.5132656097,2.5295221806,-0.4084399343,-0.2231356502,0.1975568682,1.2763007879,-0.6899693608,1.0642337799,-0.5653979182,-1.2869952917,1.2466315031,-0.1241233274,1.0194280148,0.2496183366,-0.1709581017,1.7310829163,-0.1135782599,-0.1945802718,-0.4631730318,-0.1747877598,0.2316061854,-0.2133623809,-0.7444867492,1.7741721869,-0.5213237405,-0.0052383416,0.6835482717,-0.2477262169,-1.1038888693,0.3005446196,-0.9216147065,0.0160131194,-1.3955249786,1.8256142139,-0.2967949808,-0.6049112678,1.0554323196,-0.0910923630,-1.7532095909,0.6685379148,-0.2274804860,-0.7936704755,-0.6907336116,-1.1049113274,-0.4015493691,1.2235264778,0.0144734150,0.1218120903,-0.3425313532,-0.0198845416,0.4693568945,-0.1412683278,-0.1441329271,-1.2095980644,0.3068394065,1.4111222029,0.1158419624,-0.9842948318,0.3015630543,1.1471153498,0.7848933339,-2.0293517113,0.5302294493,0.2043271512,-1.1321178675,0.6940064430,0.8048830032,-0.0286177434,0.1219166294,0.6416437030,-0.4542204440,-0.4865248501,-0.0323937610,0.2326822728,0.7127761841,-0.7575730085,0.7263642550,0.0730553716,-0.2330107689,0.6749507189,-0.0529052541,0.2778891921,0.6335431337,-0.9314393997,1.5075997114,0.0337050445,-0.5642749071,-0.7681024075,-1.2881439924,1.7528229952,1.3125530481,0.1685075015,-0.4345751405,0.6745186448,0.4473234117,3.4154362679,0.0939195305,1.4169496298,-1.7073221207,0.1352761835,0.6206020117,0.1212979928,-1.6183773279,0.2478966564,-2.0915911198,1.3072786331,0.0073723211,0.1456651837,-0.5119955540,0.6050021648,1.0283262730,-0.7437700629,0.6922110915,1.8326694965,0.3287743926,0.9887950420,-1.2720338106,-0.8566733599,-0.4071277976,0.6455544233,-1.4670102596,0.5637975335,0.0863739476,1.1576341391,-0.5866075754,1.1902070045,0.4292430878,-0.6513916254,-0.2522836328,-2.7996482849,-0.0108595164,-0.3560985923,-0.0908001214,0.9481230378,-0.9476252198,-1.5085779428,1.0062683821,-0.8160352707,-1.3991876841,1.4791408777,-0.1391293257,-0.2057078928,-0.8553714752,0.5186762214,0.5040826797,-0.8638767600,-0.9191851616,-1.3979185820,-0.6784217358,1.2306538820,-0.9332259297,0.7083811760,-0.4577125907,-0.0620121956,-0.0351362042,1.0652267933,-1.1760458946,0.0446300060,-1.5669469833,0.4134811461,1.5374922752,0.0544624180,1.0646126270,1.5449422598,-0.4001969397,0.2695350945,0.8955142498,-0.1994645745,-0.8738275170,1.7183253765,1.0541538000,-0.6865471601,0.8660798669,-0.9972230196,-0.0987603590,-1.1042673588,-0.1311311722,1.1802910566,-0.4166555107,0.1959019303,0.1618142575,1.2000273466,1.7318041325,-0.7448762655,-0.6215144396,0.8590826988,-0.9501357675,0.2647697330,-1.0188965797,0.1645448953,-0.5019816756,-0.1122976840,-0.7125408649,-1.6597757339,0.0543932766,0.6145295501,-0.9701308012,-1.8752157688,0.5507056713,-0.3657512069,-1.3984634876,-2.3039720058,0.5814130902,0.9320523739,-0.6677701473,0.4140245020,-0.2230456620,1.1848533154,-0.2327275574,-2.3117311001,1.0083969831,2.0880084038,0.7807578444,-0.2806711793,0.3659831285,0.3532699645],[-0.1186335459,-0.2383485287,-0.2226555645,-0.1092954203,-0.7526459098,0.0635053217,-0.4191754162,-0.3093976378,-0.3990128934,-0.5901996493,-0.6269081831,0.5943827629,0.9215744734,0.6849617362,-1.1751044989,-0.1513048857,-0.4341301620,0.8947159648,0.0362598374,-0.6948597431,0.9275332093,0.5545841455,-0.4738935828,-0.1504694670,2.4883825779,1.1122415066,0.1489479244,-2.0799446106,-0.3141900599,0.5396690369,-1.3634753227,-0.9709356427,-0.3622013927,-0.3671687841,-0.1727541834,-0.7378624082,0.5973001122,-1.1517913342,1.4300465584,-0.7156851292,0.8373429775,0.2585847974,0.1721901596,-0.0018438506,-0.8097422719,2.1313259602,0.7968180180,-0.2775542736,0.8631947637,-0.8420573473,2.7444453239,-0.6136727929,0.3800564408,-0.2119310200,-0.4017619491,-0.4494703114,0.4593496323,0.5182594061,1.0070230961,0.3051910102,-1.7294261456,-0.3718368113,0.8044009209,-0.5573958755,1.2480124235,1.8836368322,-0.1714086086,0.0244216826,0.2147856355,0.6537339091,-0.6575426459,-1.7292504311,0.3809339404,-0.5554701686,0.8253012896,1.0695204735,-0.8808111548,-0.4070812464,-0.5109980106,-1.3897756338,-0.0042871847,0.4558518827,0.4185920954,-0.2674489021,-0.3960486948,0.8053077459,-0.1883954406,2.6233503819,-0.1203388870,-1.6325961351,-0.5712729096,0.0376099385,-1.0410552025,-1.2952389717,1.7780630589,0.2537894547,1.2101197243,-0.5471873879,-2.1763610840,1.1458199024,0.3720822334,1.6003818512,-0.4095530510,-1.9625519514,-1.5254100561,1.0795557499,0.6412642002,-0.2453266233,-2.3478925228,0.6769649386,0.3459262252,1.0892885923,0.0059136492,-0.4414263368,0.0774223357,0.4073279500,0.9646092057,1.3409656286,-0.5068500638,-1.1570055485,0.4112512171,-0.8605152369,-1.1234155893,-0.9586995840,-0.6890590787,-1.5872890949,-0.4864371419,0.0497915000,2.1994481087,-1.7377170324,0.6416237950,-0.7923098803,0.1304780692,0.3397642672,1.4351961613,0.2319086790,-1.0006706715,-0.3280332685,-1.0791562796,0.9721171260,0.4853863418,-0.9484644532,0.3009895980,0.3293378353,-1.5128226280,0.9338934422,1.1475496292,0.0169203840,0.3695321083,0.0343017913,0.5545833707,-0.5545576215,-0.3831988275,0.8395510316,1.2106777430,-0.3786276579,-1.2317663431,-0.2997151911,0.9177377224,0.9696442485,0.3414999843,0.8652888536,2.6955928802,-0.3250256777,-0.9790326357,-1.0462377071,1.8175184727,-2.4941484928,-1.0403255224,-0.5982604027,0.2539756596,-0.7462789416,-0.9838350415,1.3585087061,0.1825647354,-2.0929007530,0.4379278719,-1.1159633398,0.8405852318,0.5710203648,0.6748986840,-0.3078625500,0.2797708809,0.1595890224,0.1293483973,0.2212487906,-1.0180786848,-0.6851283312,0.5324478745,1.7520087957,0.7664175630,0.6118838191,1.4230185747,-0.0700611845,-0.3144167364,2.0584418774,-1.1153713465,-2.3753750324,0.0852801129,1.0693234205,0.3637231290,0.5887388587,-0.4191713929,0.1261838675,-2.0334804058,-0.4840212762,0.2973466516,-0.5006974339,0.9281368852,-0.1265035123,0.4268484414,-0.1641852856,-1.4899976254,0.7738882303,-0.1064092964,0.9350059032,0.7224861979,0.6837131977,-0.2813137174,-0.2018849254,-1.4496183395,0.2822894156,0.6741409302,0.5833388567],[-0.3834654987,0.9216980338,-0.5402904153,0.0449905135,1.6210678816,0.4038809836,0.8260396719,0.9907310605,-0.1848691553,-1.2419217825,0.9926421046,0.6334283352,0.8985478878,-0.5619766712,-0.8210536242,0.1006678417,-0.0309645627,-0.0446182489,0.7577447295,-0.6273584366,2.8296339512,1.2464458942,-1.1661547422,-1.5976299047,-1.6750463247,-0.6959130764,0.4801782668,-0.3721280694,0.2400489748,0.3327723145,0.3250359595,0.5707746148,-1.3501437902,0.2438921332,0.8684121370,0.0373390540,-0.4162268639,-1.5516031981,0.4893818200,-0.9510130882,0.4297042787,0.3235373199,0.1469300836,0.7171718478,-0.2045047581,-0.2003860474,-0.9752749801,-0.8899661303,-1.1855645180,0.3116814792,-1.2083834410,0.3561777174,0.3181398809,-1.3351159096,0.2889765501,-0.3798525333,-1.2264633179,0.1558524966,-1.3657807112,0.0333562158,1.3740893602,0.2170120925,1.4955347776,0.0376551785,1.0529077053,0.9597856402,0.1183744967,-0.0969234258,0.6593038440,2.1102674007,0.8496427536,-0.9985044003,0.6058217883,-1.0085011721,-1.0398656130,-1.0336683989,-0.0988367200,-0.6505918503,0.9194741249,-0.7106096745,-0.8037624955,-0.6252217293,-0.7874242067,0.9015535116,0.0489723794,-0.3165658712,0.4750823081,-0.8362310529,0.5985621810,0.0831746012,-1.9769351482,0.4109248817,0.9616567492,-0.1925200224,-0.6872479916,-0.0291667711,0.4817531705,-1.2618397474,0.8041153550,-1.0452064276,0.9234446287,2.3404836655,-0.5770075917,0.1780190021,-0.6452456117,0.1780572832,-1.1206645966,-0.7152308822,-0.9273412824,1.5543338060,-0.9651429653,-1.2783868313,0.3413947523,-0.3924708366,-0.6756599545,-0.1602455676,-0.1887249947,-1.0889669657,-0.3058025539,-0.9298772216,0.9752748013,-0.1857045293,0.3902679384,2.4783458710,0.6216580868,0.8140361309,0.5323762894,-0.1220407784,-0.1911884248,0.9695015550,1.7107357979,-0.8332494497,1.0158033371,-1.3568849564,0.3202181160,-0.4249622524,1.0535852909,-0.0648548529,-1.1206247807,-1.3313488960,0.2760379314,-0.8660236001,-0.2355240434,1.1059592962,-1.0518497229,-0.8911786079,0.4755552411,-0.4749928117,0.8618794680,0.8533287644,0.0164538156,-0.1435441524,-0.2964879870,-1.2023267746,-0.0720503479,-0.9484385848,-0.2501189113,0.6416484714,0.7757827044,0.7773790359,-0.2722290456,-1.0522669554,0.5333345532,-1.8526387215,0.3252865672,-1.0955603123,1.5631155968,0.6736435890,0.1962209195,-0.0650429949,-1.5892469883,2.2871513367,0.5928719640,0.5669403672,0.2304450274,-0.4885673523,2.0260953903,-0.5616394281,1.2725027800,-1.2598658800,-0.7209414840,-0.8498359323,-1.6869456768,-1.4535824060,0.6325818300,0.8368428349,-0.0728568211,-0.8834574223,1.3237099648,1.5845621824,-1.4369770288,0.1884049624,-0.3132542372,-1.3461782932,-0.1914197803,-1.0135910511,0.4060253203,0.8252695799,0.5558263063,1.1091421843,-1.1206122637,2.4767508507,0.5643259287,0.9157356024,0.5002053976,0.7908154130,0.8274473548,-1.7692761421,0.7141329646,0.4376039803,1.3221139908,0.1192799956,0.1986379474,-0.6428962946,-0.8024330139,-0.6532709002,-0.6049788594,-1.2296186686,0.6375156045,0.9582261443,1.9821717739,0.9358143210,0.8946135044,0.0913714394],[-0.6747932434,1.0523104668,0.0653479770,-0.7234938145,0.1506555378,0.2563973963,0.6422882080,-2.1984939575,0.1423068494,1.3988722563,0.9099609852,0.4305366874,-0.0558515005,-1.2119708061,-0.4905354083,0.4168024957,-1.1127213240,0.2744130790,-0.9005704522,-0.2425050139,0.9652739763,-0.1550932974,-2.7414751053,2.7117772102,-1.4189043045,-1.0313910246,1.4430273771,0.3495107293,-0.1600436270,-0.4971172214,-0.6566462517,1.7511360645,2.6167964935,-1.6863998175,-0.8562343717,-0.9014871120,-0.5291117430,0.0483883694,-1.5449298620,2.5941178799,1.1498541832,-0.8130968809,-0.9091265798,1.7938588858,-1.9932663441,0.1483443230,0.5673434734,0.3636609614,-1.4151256084,0.9914378524,0.3327944577,-0.7365987301,-1.1241701841,-0.3251023293,-0.1126096621,-1.0407176018,0.2644003928,-1.2129006386,-0.6472233534,2.0263800621,-0.2055637240,1.1558588743,-0.5569451451,1.0327348709,2.9583289623,-0.0690718144,-0.1696743220,-0.2801229656,-0.1217102259,1.6591004133,1.0041955709,0.7745190263,0.4571551681,0.2016106397,0.6540086269,0.6006768942,-0.6057759523,-2.4989545345,-0.4389500022,-1.1566992998,0.2245195806,-0.1928787529,1.0130279064,2.8678760529,3.4200766087,-1.1019076109,0.7968271971,0.3168827593,0.9232071638,1.5975202322,-0.4766547978,0.1227640808,1.5033007860,-0.1045565084,-0.5908822417,-1.4120235443,0.3343747258,-0.6939716339,1.3138922453,1.7582516670,-2.0948252678,-0.3867600262,1.4881784916,-0.2614794075,-1.0096803904,-0.8288538456,1.1242891550,0.8932532072,-0.7381454706,-1.5702298880,-0.6367726326,1.0390914679,-0.6599599719,0.1851871461,0.2441084385,0.1874911785,-0.2299124449,-0.0557615831,1.2884424925,-0.4906115234,1.0648677349,-0.7498172522,1.3501983881,1.2616695166,0.9143901467,0.1215201989,-0.1740020961,-0.8299691081,-1.3438180685,0.6896972060,0.2989670038,-0.9304523468,2.1257295609,-0.3191212714,-0.9800383449,0.9961398840,-0.3433336616,-0.9051725268,-0.5238730311,-0.1880698651,2.2426862717,0.3711197674,0.4215188324,1.9264063835,-0.8985811472,0.9724038243,0.9724006653,0.5963121653,-0.1313816905,-0.9208067656,0.0731222332,-1.0595364571,1.7986855507,-1.4451892376,-1.6628885269,0.1031904817,1.8886716366,2.1079497337,-0.0436575674,1.6163161993,0.7489252687,1.0819302797,-0.6431999803,2.6041235924,2.1054654121,0.1228456721,-1.3445225954,0.2740955949,0.8061972260,-0.7437826395,-1.1212561131,1.0593879223,-0.7465335727,0.6351316571,-0.1259510219,0.7468909025,0.9277933836,-0.6623082161,0.9495425820,-0.3312621713,0.3156095445,-0.0621979199,0.2871080637,0.5507833958,0.7471680045,-0.1918139458,-1.0566934347,-0.3854509890,-0.9094166160,0.3269947469,1.6029323339,-1.3439395428,-1.0700266361,0.8704138994,1.7585107088,0.8698053956,0.1614292413,-2.1836338043,0.4575574696,-0.3837276399,-0.7936035395,-0.7575538754,0.7196499109,0.3993514776,-1.1120111942,-2.4084584713,1.1744660139,0.6642578840,-0.1801867336,-1.1095536947,1.8082687855,-0.5784759521,0.7034670711,0.0047463123,1.9643903971,-0.6102766991,-2.0074377060,-0.0763465017,0.2575736046,1.8768074512,-1.2388337851,-0.7563742399,0.5345876813,-0.8474777937],[0.0104724141,-0.0560626648,-0.2254574150,0.7342847586,-0.1823126227,0.9410361052,0.1615654975,1.0111799240,-0.9020431638,-1.2960655689,0.5210303068,-0.0611365028,0.1001681834,0.7305082083,0.5266997218,-0.6311574578,-0.1671253741,-1.4037280083,-0.2443351150,-1.2517610788,-0.6421838999,1.4461832047,0.2268501073,0.0399844833,0.7946876884,-1.2721235752,1.2676312923,0.3134599328,-0.3629707396,0.5874877572,1.3795229197,-2.0228421688,-1.0742843151,0.7796605825,1.8933137655,-1.9481340647,-0.7232157588,0.6457300782,1.0992538929,1.2602574825,1.2173357010,0.3225525320,-0.8072761297,1.9048274755,0.2093349546,0.0960786119,-1.5297433138,-2.1362874508,1.6078487635,-0.0083657373,-0.4866380692,0.3900962174,-1.9635704756,0.3925728798,-0.9276992679,0.9480867982,-0.4484598935,-0.2534022331,1.1133691072,-0.4018307030,1.1105887890,0.0697086081,-0.0130785871,-1.2711316347,-0.4720092714,-0.6093285680,-0.3815086186,-0.2593854666,0.6868306398,0.2982982695,-1.0888745785,-0.0356637649,-0.7768434882,-0.8902729750,-0.7600454092,-0.4218118489,0.3945529759,-1.7036703825,-0.6038737297,-0.6489652395,0.7995288372,-0.2697657645,-0.2542925477,0.1636188477,1.6742730141,1.2430789471,-1.2469968796,-0.3124547601,-0.1815218925,1.0640125275,1.8174276352,0.1209504902,-0.2399097234,-2.0230894089,-1.1016820669,-1.4689011574,0.1792734712,-0.1051847711,-0.5941409469,0.1108767465,0.5730710626,-1.2479622364,-1.0237392187,1.3488510847,1.1845982075,0.3580740094,0.9250638485,-0.2230274081,0.1302635074,0.6828508377,0.7219886780,0.4433879852,-0.5854877234,0.7401592731,0.8713111281,0.2033105642,-0.5434444547,0.3103612363,-0.2429424971,0.3660286367,-0.9809601307,-1.3397420645,0.1668384820,-0.2828427851,0.4434334338,0.5371432900,1.9583348036,1.2049523592,0.2372088730,-0.3331832886,1.1592470407,0.2292878628,-0.7659143209,-0.3226299882,1.0478326082,-0.0462466627,-0.0130897462,-1.5684120655,-0.3709748387,0.9952441454,0.5843552351,-0.5555575490,-0.9959812164,-1.1858104467,0.3064933121,-0.3683856726,0.9699641466,0.9785125852,0.0002325818,-0.4245741367,0.3247098625,-0.1118231490,-1.9157896042,0.2442909628,-0.5570378900,0.1214372441,1.2457281351,2.3516080379,-0.0346703567,0.2227616310,1.0316162109,-0.3360175192,-0.1249095425,-1.2381279469,0.5380402207,0.2509429753,1.4545207024,0.3763480484,1.1317685843,0.1574275047,-0.4038485885,1.0350737572,1.5813468695,-1.2058871984,1.8397190571,-0.5369747281,-0.1675504595,-1.6808416843,-0.3973738253,0.8955420256,-0.9474033117,0.8863657713,-1.3639428616,-1.1407256126,0.0768821016,0.1626100093,1.1859166622,0.8617249727,0.0092263343,0.5938783288,-1.6519448757,0.4657230377,2.0321743488,-0.6253198981,0.4228032529,0.1740853935,-1.1310549974,-1.1406570673,-0.3442859054,1.0565874577,-0.2801092267,1.8965030909,-2.4183242321,-0.1742828339,-0.7835971117,0.2747937143,-0.6171855330,-1.6374839544,-0.0998052806,-1.5593715906,-1.2759281397,-1.4176751375,0.7536286712,0.8720196486,-1.2819939852,0.7087141275,1.9042751789,2.7620675564,1.1157127619,-1.0292673111,-1.2326740026,0.4892507195,2.1081411839,0.4370924532],[-0.5593727827,-1.4354031086,1.0570830107,0.4167571962,-1.1469852924,0.4386380911,0.4627600610,0.0152175874,-0.5675275326,-1.2706890106,0.3094842136,0.3353318870,-0.5180909038,1.4484337568,-1.8907704353,-0.0004836629,-0.3969363868,1.0633287430,0.0843856484,-0.1612736881,-0.0583505481,-0.0020569484,0.0918041766,-0.5388286114,0.4628135860,-1.0131388903,-0.4627156258,1.8314408064,-0.5217072964,-0.1163138002,-1.9294489622,-1.0067454576,0.5082025528,-0.3337702453,-0.2269964814,-2.1366028786,0.1338530332,-0.3698782027,-1.0692406893,0.3903188705,-0.0497018024,1.1616665125,0.9621710777,1.4915152788,0.1182670668,1.5708578825,0.7640832067,-2.0470445156,0.2740463912,-1.6569459438,-0.2085611820,-0.9933959246,0.7703897953,0.1919436753,-0.1551560611,0.5901817679,-0.7839456797,0.5055692792,-0.6600452065,-2.1255404949,2.5530233383,-1.4889702797,0.6624996066,-0.8144878149,1.2584605217,-0.6314144731,-0.9057261944,0.0311285201,-0.8325083852,0.3457152247,1.0724612474,-0.1779902279,0.6224679351,1.1843028069,0.3990680873,-0.3444859684,2.2503702641,0.5720024705,-0.6524844170,0.8113818169,0.6207444072,-0.2953625619,-0.2325160801,0.5621783733,-1.1866376400,-0.2479910105,1.3240972757,0.4694396853,-0.8371084332,-1.1386457682,-1.5861579180,-0.1940881610,0.5150237679,-0.7020881772,-0.1579501629,-0.3900207281,-0.9069029689,0.2974373698,1.0060987473,0.3582396507,1.3689446449,0.8325985670,-0.6259139180,-1.0219625235,0.8717699647,1.5553936958,1.9892309904,1.7129807472,-1.7524341345,-2.4156734943,0.3734627664,-0.9791836739,-1.1838378906,-1.6001518965,-0.2521800399,-0.9588496089,-0.0467710644,1.4703460932,-0.1281604916,1.9166052341,-0.7853434682,-0.8153104782,-1.2059153318,-0.0041961134,0.8652752042,-0.1252185553,-0.3175365925,0.0003913178,-0.8116727471,-0.1722526997,0.2703862786,1.1430435181,-0.0179268271,0.5940791368,-0.0597756766,-0.7731306553,1.7648396492,0.6532765627,-0.0256540347,-0.2082199454,-1.4125479460,-0.2929659188,-0.0797623321,0.4935064018,0.6900241375,-0.8711327910,0.2377964407,-1.3421390057,-1.0594630241,0.8106474876,-0.9261490703,0.0168073885,-0.1170370802,0.4168663323,1.3247480392,1.9524668455,-0.7113932371,0.8084810376,-1.1566987038,-0.2625491023,-0.1377955228,1.8415939808,0.2191815525,-0.3335691094,1.3335675001,0.6392911077,0.2304753959,0.1254295856,0.4711110294,0.5567567945,-0.1583541334,-0.1846426278,-0.3851806819,1.0271519423,-0.0131389173,1.4698441029,-0.5619645119,-0.0572414175,0.5521336198,-0.3570716977,0.4461381435,1.5787917376,-0.2964460850,-0.6109013557,0.5612183213,2.4743282795,-2.0427081585,0.7436702251,0.5880488157,0.1136561483,0.4198854864,-1.5047591925,0.5283017159,0.4352711737,-0.8595504761,-0.1644246876,-0.3773303032,-0.6257966161,-0.6975237131,0.4315404594,0.1575031728,0.5167097449,-1.4270300865,0.4468098283,-1.6486895084,0.4399907589,-0.2159845680,1.4771642685,-0.0137156723,-0.0051958985,-0.0996746719,1.8157832623,1.2126486301,-0.8141968250,0.0797947198,-0.3740171492,-0.0235440154,-0.4075264931,-1.5856205225,-0.8143793941,1.2097389698,-0.6563789248,0.0917219445,-0.2388767302],[-0.5819990039,-1.9098675251,2.0859701633,0.4343566298,-0.4059936106,0.4935959578,1.1469419003,-0.8147927523,-0.0642666221,1.0216310024,0.2484840751,-0.2188244909,-0.4161131978,0.9300234914,0.8221677542,1.3278201818,0.7848719358,0.7350556254,0.3275857270,-0.9404194951,0.8318222165,1.3112833500,-1.7765289545,-1.7815247774,-0.3423713744,0.4871589839,-1.4034453630,-0.3804571629,0.4861087799,-1.2812838554,-0.0835635960,1.6544206142,-1.2440884113,0.8436793685,0.7396528721,1.1865768433,0.0844733939,0.1610046029,-1.5059672594,-2.0161046982,0.4744549394,-0.1027829498,-0.8932421803,2.6681511402,0.8733780980,-0.6771495938,-2.1171841621,0.4179076850,-1.5294778347,0.6122385859,-0.5188607574,0.0253527220,-0.5863035917,-0.6137494445,-1.3239506483,-0.1866202801,1.4357578754,-2.6238973141,0.9697104692,1.3150304556,-0.1740820110,-1.2808016539,-0.7015659213,-0.3883333206,-0.0155479601,-0.5180083513,-2.6574621201,1.2562646866,1.7780845165,-0.8762903214,0.4460975826,-0.5522505045,-0.1025809869,1.6547645330,0.4384832084,0.8500418663,-1.0074121952,0.8443838954,0.3932457268,-0.2968220711,-0.8347958326,1.3171423674,0.9378548265,-1.2623959780,0.7586356401,0.2216345519,-1.1635854244,-0.2694393396,0.7349928617,1.1965305805,-0.4751051366,0.2625339031,-0.5044584274,0.3648485243,0.6711106896,0.8469839096,-0.0794413686,1.4676808119,-0.5880222321,1.2756909132,2.1141371727,1.2257579565,-3.1443977356,-0.8633837104,-0.4782708287,0.7528486848,0.0728427917,-0.1478959173,-1.8716983795,-1.2341648340,1.9306218624,0.4773486257,-0.7664555907,1.0502686501,-0.6320495605,0.7806579471,0.5659831166,-0.7911544442,-1.2014272213,-0.4031826258,0.3939342797,2.0412855148,-0.5606901050,-0.3510407507,0.4227169454,-1.3822602034,-0.2495191246,-0.4121963382,-0.4797803164,-1.6132888794,-0.1541571766,0.5172279477,1.8714196682,-0.8915430307,-1.0249711275,-0.4603399038,-0.8396726847,-0.2851386666,0.4890405834,2.7800784111,0.2114791274,0.5173261762,-0.4926160276,-0.5128554702,0.3635650575,-1.2499142885,0.1577661932,-1.1878240108,-0.1372671872,0.2076225579,-0.9449370503,-0.6277039647,0.0855376199,0.9168802500,-1.4615525007,-0.8221495152,-0.0306883696,-0.3794527352,0.6988317966,-0.3404355347,2.0155558586,-0.9869990945,2.3109531403,0.8517220616,-2.0357089043,-1.3717461824,0.4539968669,-0.3938590884,-1.1531527042,-0.6132876873,1.6227045059,1.5988278389,-0.4305523634,-1.5203722715,-0.2126695216,-0.7949711084,1.0062519312,0.1088095903,-0.1143707111,-1.5254942179,-0.5971631408,0.0266929027,0.1798896790,0.6165832877,0.1774144769,-1.1522259712,0.3164618611,1.3300391436,1.5971671343,1.4609972239,-0.1620344818,-0.0415391847,1.7246509790,-0.3874904513,2.1607668400,0.3433647454,0.3206478953,-0.6261118650,0.5304078460,0.0695448145,-0.0012609762,-0.5348551273,-0.4886779487,-0.1492735595,-1.1154803038,0.2743631601,-0.8677986264,-0.8241101503,1.0104525089,0.6254281998,-0.9113668203,-1.6015820503,1.7713805437,-1.2572556734,0.1901635826,2.1025061607,-0.0653650835,1.8718448877,-1.3963491917,2.3231360912,1.5577259064,-1.4847933054,0.1091064811,0.1189436316],[-1.8201619387,0.9907189012,0.7037143707,-0.1040920019,-0.3882758319,0.6091170907,-0.3306565285,0.2832741141,-0.2206534594,0.8937234282,-0.1409606785,0.9197987914,0.9479706883,0.4320091307,0.8930882215,-0.2672516704,-0.2465402484,0.1139625460,0.5704907179,0.3211796284,-0.7757544518,0.2262502760,-0.6387628913,-0.5648298860,-0.1335153878,-1.9806940556,0.8351049423,-0.6341540217,0.2234416455,0.7767619491,1.0302518606,1.2044047117,-0.6560946107,-1.1046437025,-2.5660791397,0.7642706633,1.7174856663,-0.2373072505,-1.4493148327,-0.0723819956,1.2432292700,1.9301974773,-0.0527655259,1.5347336531,0.7458348274,-0.3054525852,0.2830949426,0.7241360545,0.4799636900,-0.4515165687,-0.9522143006,-0.2386518270,-1.5671073198,1.1002331972,0.9565708041,-0.0753502250,0.3522382975,1.5541101694,1.0659878254,0.2211657315,0.7039881349,1.3312822580,1.1425888538,-0.8043620586,0.1118339002,0.7051146626,0.2405101657,-1.2077591419,-0.7707319856,-0.4111624956,0.8228642344,1.3478077650,1.3317564726,1.2580941916,-2.1362700462,-0.6044573784,1.3983079195,-0.2267834991,1.1038742065,1.8309376240,2.5118257999,-0.7461085320,0.4706372619,0.3283693790,0.5749509931,0.0096055260,1.5103399754,0.4314630032,0.8736536503,1.0169622898,-1.6128311157,-1.1360261440,0.5721042752,-0.6031692624,0.1187613904,1.1165670156,0.1368981600,-0.9850955009,0.6313478351,0.7843387127,1.2135647535,0.3128623366,0.5757420659,-0.1311307251,-1.4725977182,0.2902924418,-1.6542005539,1.3504993916,-0.3761866093,-0.0340541266,1.3819690943,-1.8941106796,-0.2743933499,-0.2576075792,-0.4365912676,0.4990610778,0.9072874784,1.0279914141,-0.2289631665,0.5705673099,-1.8134893179,-1.4707256556,-0.1991883665,0.1905540079,-0.1706116498,-0.4919151664,0.8360471129,-0.4966766536,1.3094147444,-1.3342524767,-0.8660213947,0.4778424203,-0.3906137049,-0.2144182622,0.3572425842,0.2118220329,1.1288861036,-0.0217239484,1.0320818424,-0.3704183996,0.1348206401,0.0780876055,-1.1623704433,0.0471805632,1.8262777328,-0.1326367855,1.2541949749,0.4579695463,0.6132331491,-0.6418583393,0.8945944309,1.3853417635,0.6127995253,1.7553474903,-0.8952273130,0.5639309883,1.1167125702,1.1573514938,0.6090118885,0.4605760276,1.3694016933,0.3081550598,0.4055664539,-0.4725796282,1.7730118036,-0.0457734875,0.0639425218,-0.6374250054,0.5051111579,0.5209814906,-1.3633556366,0.6776292324,0.5936003327,-0.9643524885,1.3238570690,-0.9794591069,-1.8232733011,-0.9560294151,-0.9466639757,0.4216081202,0.0111554582,-1.2830356359,0.0037587516,0.2466487288,-0.7067456245,-0.1197778061,1.3785123825,-2.2226142883,-0.1396190971,1.3109309673,0.0336780511,-0.5899109840,1.8874821663,0.4395863414,-1.2137041092,0.9638198614,0.5589596629,0.1619139463,-2.0408895016,0.0560152270,0.7049338222,-1.8372228146,0.0895555690,-0.8652178645,-0.5274806023,-1.4498169422,1.8824826479,-0.7552757263,0.8409298658,-1.4074972868,0.2751502097,-0.0361059606,1.3069361448,-0.4460712075,0.7948856354,0.2241593003,-0.0092800902,0.1511366814,-0.5700770020,0.9738789201,1.4050635099,0.7358398438,0.3554930687,-0.5356953740],[0.6154323220,0.9979453087,-1.3390628099,-0.8583290577,-1.7452572584,0.1104434729,0.3890730739,0.0756772608,-1.6073110104,-0.3842329979,-0.5743327737,-0.5342398882,-0.5943794250,-0.6329166293,0.1348872036,0.0209345836,-0.6921874881,1.2580894232,-0.1487711817,1.4409718513,-1.9829041958,-0.3945930004,-2.0652801991,1.9432833195,0.8998360634,-0.2134810537,1.7895114422,1.2492611408,-1.2575070858,0.2093546093,-1.7946289778,-2.2766561508,1.5815219879,-0.6217867732,-1.5840293169,1.0552384853,-1.4108628035,0.0078826984,0.0647610202,-1.2973845005,-1.4553879499,-0.6940916181,-0.0104205608,0.2180850804,-1.1110692024,-1.4055019617,0.6910184026,0.1690434515,-0.5238924026,-0.9038918614,0.1617193222,-1.5668437481,0.5746982098,0.7966463566,0.3421330154,-0.1778702289,0.0627256557,1.2577672005,0.7419780493,-0.4220174253,-0.5558896661,0.5636611581,-0.5715605617,1.2305231094,-0.6542611718,0.5123797059,-0.5756513476,-1.2421853542,0.9539251924,-0.7216297388,0.2888312638,0.0398992859,1.9420181513,-0.9858289957,0.1439062804,-0.1981014162,-1.3955296278,0.1860184371,0.5092500448,-2.2733435631,0.4448447824,0.7383634448,0.9620494843,0.9712896943,-0.5859873295,-0.4633949995,-1.1635951996,-1.0255144835,0.9860011339,0.3003800213,-0.6828405857,-1.2880755663,-0.0181694888,1.3963208199,0.6787196398,-1.3546409607,1.0398122072,-0.0002420847,0.4149239361,-1.3920470476,0.4387226701,1.7786114216,0.0279869679,-0.5754590034,-2.8383243084,0.3669659793,0.3989785910,-0.3432056606,-1.4067237377,-0.5664580464,-0.7988797426,1.9535807371,0.9568747282,-0.2729994059,2.0905108452,-0.9002789259,1.7589235306,-0.2031082958,0.2045028061,-0.3391548991,-0.0187342484,0.6179451942,-0.1164130643,-1.0554273129,1.1348347664,-0.5240642428,-0.0533059984,1.1834615469,1.4722721577,-0.3002305627,-1.1545814276,-0.0434754863,0.3521986306,1.1709691286,0.2513211071,-0.4330072105,0.3237198889,0.2494965345,0.2712747157,0.1401934177,0.0521037690,-1.1359419823,-0.8886265755,-0.4292696416,-0.6217500567,-0.2999503016,0.2239636630,-1.4546821117,-0.4629549384,0.2481509894,1.1152031422,1.2186678648,-0.8037263155,-1.8307890892,-0.9193554521,-1.9347736835,-1.7906540632,1.4603732824,0.6034803391,-0.7582035661,-1.4562839270,-0.4283702075,0.2467783093,0.8750617504,-0.5399234891,0.0219904687,0.1347843856,-0.6984317899,1.2807254791,0.6972069740,-1.1246480942,1.2445069551,0.0746976510,-0.3634368479,0.9292349815,-2.1193280220,1.9793330431,0.9654034376,-0.8357360363,2.3896911144,0.5750907063,-0.6683084369,0.4982488751,-0.2650178075,0.3913143575,-1.9028433561,1.2230871916,-1.6067589521,-2.1641135216,-0.6277225614,0.8032400608,0.6450325251,-1.3358204365,-1.5891611576,-0.3048976958,-1.1766319275,0.9003731608,-0.2710793614,0.7207944989,0.2262839079,-0.2502329051,-0.2332939208,-0.3703612685,0.4698289037,-0.7305675745,-0.2357319444,1.5106195211,1.0950709581,-0.6133319139,0.5051881671,1.0278711319,-1.7371171713,-0.2370272130,-1.0186152458,-0.5381708741,0.6036685109,0.0080919098,1.3370512724,0.3532626629,-0.7991682291,-0.2605423033,0.9962968826,0.3493235409,0.4128884375],[-0.7413421273,1.0753172636,-0.1606328785,-2.4201703072,-0.7577661276,0.9301439524,0.3775624931,0.3262410164,0.8824862838,0.4710078835,-0.8032360673,-0.5725351572,-0.0582918525,0.7470893264,-0.4320694208,-0.4195062816,0.7359803915,0.9830491543,0.1097064689,0.5909559727,-1.5239113569,-0.5554866791,0.1916795075,-0.2884576321,0.4674206376,2.1317734718,0.8106547594,-1.1282305717,1.1774668694,0.5297613740,-1.5435117483,1.0490826368,-1.4259819984,-1.2291139364,-0.0065258709,-0.2216066718,0.3152341843,0.3517228663,1.0046916008,1.1837831736,1.6787571907,0.5730426908,1.1133731604,-0.7634158134,-0.4686076939,0.7512238622,2.1902413368,-1.1080244780,-0.0866195410,0.7111128569,-0.0059136329,1.6001064777,0.2407293469,0.7014157176,2.2532317638,-1.1692081690,1.0892179012,-0.1247122213,0.3142854273,-0.0842473879,-0.1432422101,1.1002390385,0.5898386240,-2.9799461365,-0.5483270288,-0.3715531826,-0.3583878279,-0.0210139882,-1.1149870157,2.4004268646,0.1063649058,0.7702438831,-0.4968264401,0.0172923505,0.2284315526,-1.0430088043,-0.9970272183,-1.6039516926,0.2978511751,0.3246447444,-1.1764000654,-1.0792928934,-0.7115822434,0.9049133658,-0.1356133074,0.7902173400,-1.8608924150,-0.0342043675,-0.2184225768,0.2755118310,-0.5773473382,1.9079804420,-0.9805600047,-1.7185441256,-0.2441821545,-1.5625159740,-0.0510449111,-0.6830695271,1.4691959620,0.3331987858,-0.1861740500,-1.2373799086,-0.2372961491,-0.1537933350,-0.1583372951,0.9213557839,0.1104177907,0.7021344304,-2.2697911263,-0.9931976199,1.3266222477,-2.0886709690,0.9214906096,-0.4929367900,0.7904263139,1.5942453146,-0.5685650706,1.1563068628,-1.7574599981,0.3507694304,0.9195406437,2.1849930286,0.4669177234,0.0332165174,-0.9302831292,-1.1052159071,0.5699044466,2.7418849468,-0.8644914031,-1.1204868555,1.0906944275,-1.8325343132,1.2423360348,1.0562793016,-1.8252551556,-0.1053487584,0.6263771057,1.5337488651,0.3819944263,-0.3788675964,0.4347829521,1.0719298124,-1.9757854939,0.4538616240,0.9466922283,2.0393373966,0.8154541850,-2.2556941509,0.2045707256,1.3789530993,0.4618313015,0.6645997763,-0.1469683647,0.9429551363,0.3144655824,0.5231790543,-0.3446117938,-0.9457993507,0.7486024499,1.3672702312,-0.7518332005,-1.3094704151,1.2875912189,-0.6445900202,-0.4857350588,2.7472012043,-1.5506581068,-1.7447063923,-1.6849077940,-1.1065803766,-0.0241784956,-0.0330930389,0.8838341832,0.7734587193,0.5042662621,-0.6122062802,0.7282840014,0.9780203700,0.7290042639,1.1951694489,-0.0285034478,-1.7318187952,2.8072602749,-0.8139628768,-0.6801139712,0.3368071020,0.6303592920,0.0890115127,-1.2492383718,0.6210021377,-1.3981107473,-0.0684296265,-0.8358044028,0.7615279555,1.1252840757,-0.2928672731,-0.1489151269,-0.2545097768,1.7402290106,1.0581798553,0.1409235895,1.3134722710,-1.2648737431,0.3441012800,-0.9336190820,-1.2345448732,-0.1902813017,0.8853290677,-0.5469426513,-1.2303242683,-0.2285664976,-0.2023745328,-0.4210486412,-0.1622483581,-0.6331458688,0.6673703790,-0.8797009587,0.1470505744,-1.6115312576,-1.7434673309,-1.1627935171,0.2992941439,-0.7151190042,-1.2173782587],[0.2558501363,-0.5031922460,-0.1038606390,-0.1561047882,-0.8796012998,1.0886113644,0.3119497001,-0.4022977650,-0.1012258232,1.7479351759,1.3647730350,-0.0543092415,0.2226750851,1.6659430265,-0.5892763734,-0.3167293370,0.7162985206,2.7947816849,-0.2856855094,0.3759886622,0.6820854545,-1.9602570534,2.3843588829,-0.8123612404,0.6058959961,0.7391664982,-0.4000688791,-0.9828444123,0.7060558200,-0.7857539058,1.1159673929,0.2286228538,-0.4661549330,1.4146889448,-1.5722010136,0.8475800157,-2.1850621700,0.4052006304,0.9268170595,-0.5286844373,0.3858979344,-0.9194214344,-0.8946732283,0.0659045056,-1.2474843264,0.4395863116,-1.1277976036,0.7343308330,2.2652714252,0.2736219764,0.8193798661,0.4121380746,0.9677261710,-1.4630614519,-1.0397695303,0.3285118639,1.4500451088,-0.1098707393,0.8462182283,-0.6662430167,0.5424329638,1.0663647652,0.3155307770,-0.6716760993,-0.8097543716,1.6464439631,0.5302342176,-0.3151896596,0.2689172924,-1.2359784842,-0.1069266498,-1.5323109627,1.3193717003,0.6514467597,1.5726830959,-0.5646753907,0.8270943761,0.0647617579,-0.5997740030,-1.9733934402,1.2835294008,-0.7041236758,0.2832516134,-1.2470406294,0.0342308544,0.5632337928,1.7738374472,1.0023999214,0.5243485570,0.4966398180,-1.9411910772,-0.5846640468,-0.1156490222,-1.8383711576,-0.6315868497,0.2895682454,-2.7668647766,0.1243848652,0.5403708220,2.5971651077,0.9537038207,-0.5311449766,-1.9729572535,0.1686208397,-2.2282452583,0.0519388616,-0.8657557368,2.1935203075,0.9435275793,0.3583879769,-0.7221446037,0.8322559595,-0.1130532399,-1.4182372093,1.7974859476,-0.6891320348,-0.2459898293,0.3112476468,0.2605895996,0.4272195399,0.5657240152,0.1126472205,0.5242422223,-0.5227910876,0.8922997117,-0.4843709171,0.3712804914,-1.6282891035,0.1348460317,0.3993672132,1.3468558788,0.7090018392,0.4272810817,0.3625778258,-1.5106115341,1.8112719059,-1.1968251467,-0.1324630231,-0.7746610045,-0.1489947736,-0.2863193154,-0.7271369696,0.4484030902,0.4039295614,-0.6630720496,0.2465493828,-0.1913393289,-0.5363966227,0.0706059188,-1.9758604765,1.2545558214,-1.2935758829,0.1870326698,0.5147288442,0.2047286779,-0.9113526344,-0.3674494028,-1.3195163012,-0.5636649728,0.4023266137,0.3050526977,-1.3710098267,0.7285389900,-0.2600562572,0.9642859101,0.7061238289,0.0804846361,0.3684956729,-1.3706005812,1.7705264091,-0.0568827242,-1.1363366842,-0.5154197812,0.4832886755,0.1776915193,-2.1355118752,-0.2538057566,1.1987390518,-0.9549040198,1.4837807417,-0.3976415396,0.2713989019,-1.3256140947,0.0872604623,-1.8873546124,0.4834865928,0.7455752492,0.0950498506,0.2772811353,1.2658462524,1.5551054478,1.7185579538,-0.2580177486,-0.5866049528,0.3638573885,0.4724708498,-1.0699458122,0.3302752972,-0.5717364550,-1.9355258942,-0.4169055820,-0.3732923865,-0.4488600194,0.1205200925,-0.6611413360,0.3152336776,0.5914426446,0.5891466737,-0.4318289161,0.5248982906,-3.0162057877,-0.7288278341,0.5338007808,-0.5691620111,0.0026917364,1.3950619698,0.9638216496,1.2086063623,0.5523138046,-0.1347372085,0.4186432958,-0.0125459330,-2.3909592628,1.2674869299],[0.8219062686,-0.9020726085,0.8937808871,0.5497428775,-1.8376889229,-0.3232720494,0.9673829675,-0.1577348262,-0.8051065803,-1.4380515814,0.5605085492,-1.6228570938,0.8229484558,0.6320642829,0.8324034214,-0.7958431840,-1.2507016659,0.3585999310,-0.2646588385,1.5732150078,0.3994229436,-0.6318780780,0.2118939310,0.4288383722,0.1017487049,0.3347709477,1.3281470537,0.5289729238,2.2454297543,-1.1805934906,-0.1550690830,-0.1861247271,-0.6157492399,-0.2086265832,1.8791502714,-0.5473974943,1.2587167025,0.0400549732,-1.2131153345,0.4959487915,-0.6701539159,0.0377192609,2.0914919376,-1.2017105818,-0.5418961048,0.0715248883,1.1804654598,0.0372791365,0.5071865916,-1.1190364361,0.1947530657,-2.2451906204,1.8115687370,-0.3882434666,0.4428933263,-1.3330305815,0.0781882405,-0.1043641046,1.0043901205,-1.0324101448,-0.4917244315,-0.5101444721,0.7722641230,-0.5372172594,0.3941535652,-1.0564483404,-0.1142743900,0.2367779464,-0.4811623394,0.8114396334,0.7052668333,1.3426921368,0.4971502721,-0.4514482915,0.1606788486,-1.0752604008,-1.5647202730,0.3447371721,0.2896904051,0.6959680319,-0.0190260354,-0.1225974485,0.6586026549,-0.3244411349,0.7685539126,-0.1682316661,-0.9780043960,1.9468239546,-0.1223541796,1.4135782719,-0.3331348896,1.5832772255,-1.2763670683,0.3524271250,-1.0963369608,-1.3600704670,1.3364082575,0.8700780272,-0.3765659034,-2.6367402077,-1.0929170847,-0.0520115234,-0.9117823839,0.5583176613,-0.8233562708,-0.1356477439,-0.5791643262,0.2063644528,0.0742090866,-0.9129800200,-0.5781509876,-1.3166632652,0.5495023131,0.6567738056,0.5245886445,-0.5641840100,0.1216162369,-2.2587895393,-1.1103811264,-0.2331566811,0.4222166240,1.4316504002,0.8986779451,0.2228398025,1.3977748156,1.0694563389,-0.8107503653,0.7228331566,0.2350391448,-0.1462830901,1.1465845108,3.0110960007,0.1884022951,-0.2710390687,-2.4918198586,-1.3116731644,0.6081784368,-1.8274118900,0.3237744570,-1.0996891260,1.6525988579,1.7144389153,-0.7473198771,0.1106017306,0.0607738420,-0.4209593534,0.5615531802,1.1473829746,1.3023934364,-1.0771951675,-0.0043456485,1.1957966089,-0.6975986362,0.6175321341,0.0798140764,0.3990066051,-0.1930454522,-0.2801606953,-0.6360430717,1.5740067959,-0.6150626540,-0.6657387018,0.6328179836,1.5874658823,-0.8234479427,-0.9082185626,-0.9049493670,-0.1048155203,1.4692788124,-1.4920828342,-0.2135273218,0.1324931532,0.3250708878,-0.5595640540,-1.6266405582,0.6432067156,1.0292233229,0.4580583870,0.1775811911,-0.0012351129,-0.4039856195,-0.8623089790,-0.4011097252,1.3185437918,2.4721014500,1.5110800266,1.1565639973,0.1544111222,-1.4040644169,1.1386544704,0.4006298184,0.1019146889,1.1850543022,0.5945206881,-0.3560276926,0.8948068619,-0.3113974035,-0.5629203916,0.8216427565,-0.8694606423,2.0849277973,0.4553703964,0.2011788487,1.6399581432,1.1543406248,-0.6563993692,-0.1601136774,0.3659283817,-1.5479089022,-1.6454467773,-0.5885052681,0.4701368809,0.8731890917,-1.0473310947,-1.6058140993,0.5154587626,0.0192641634,-1.1761554480,0.3215034902,-1.0969194174,-1.1431628466,0.9230197072,-0.8684829473,1.2271327972],[1.0552119017,-0.4195388556,0.7144517303,-1.5553835630,-0.5773956776,0.9839149117,1.9320902824,0.4686033130,-0.5418919921,-0.2432720214,-2.3585970402,0.6843538284,-0.5589225292,1.4080122709,0.5970636606,-0.6819794774,-0.7745859027,-0.5762728453,3.3780813217,0.9202184677,-0.5481423736,1.3764128685,-0.3434719145,0.3977651894,0.3125531971,-0.3766749501,-0.9650712609,-0.6414336562,0.2432779819,-0.0810078681,-1.4610075951,-0.5868371129,0.6019322276,-1.1608246565,0.4770873189,-1.1839425564,-0.9647042155,-1.2911399603,-0.7851390839,0.0474583209,0.5524968505,-0.4648094177,0.6342483759,0.2583498955,-1.6864966154,0.0093507320,0.6306871176,0.3638943732,-0.5551549792,1.9378409386,0.4792116284,-1.3169698715,0.3789154291,0.5505794883,0.3445309401,-0.2305428237,-0.5491927266,1.4039272070,-0.1311127692,1.2018909454,1.4131186008,0.6897209883,2.8037290573,-0.9207084179,0.3258124292,0.5043047667,0.0766622126,0.6969939470,-0.2453171611,-0.0269711744,0.1035700291,0.1819505543,-0.5836421847,-0.6208786964,-0.8684943914,-1.0072883368,2.5586364269,1.1484230757,1.4538383484,1.0172985792,-1.1306617260,-1.1712599993,1.4082340002,0.1663869172,0.8057622313,0.0210015643,1.0863547325,-0.4501363635,1.1340610981,0.3947748840,-0.9205060005,-0.1627232879,-1.3064237833,0.7684119940,-0.2114374936,-0.4287933707,-0.4915237129,-1.6247475147,-1.9873917103,-0.0473227836,0.3810624182,2.1965646744,-0.4863922298,-0.2136739194,0.2542577088,0.6342818141,-0.6232274175,1.3386460543,-2.8707752228,-0.8340262175,1.6175956726,-1.5389409065,1.0650939941,1.5738481283,-0.2559369802,1.8801785707,-0.6821094751,0.5025829077,2.0232660770,0.7300717235,-0.7082805634,-1.3114689589,0.5112272501,1.6427576542,1.4437679052,-0.4480295777,1.2616261244,-1.0043283701,0.0544764996,-1.6036715508,-0.1986960471,0.7933214307,0.6842283607,-0.9457077384,0.3122385442,-0.3216295838,-1.5933513641,-0.9579170942,0.3627172709,-0.8289911747,0.6612448692,-0.1123323664,-0.1781046540,1.4285470247,-0.7790344954,2.9845044613,1.1620213985,0.8558660150,-0.2367271185,0.3755499125,1.6477419138,0.1121707782,-0.9373643398,-1.3632234335,-1.5604901314,-1.6682239771,-0.2411317378,0.1953369826,-2.2244782448,0.4661431313,-0.6613999605,1.1189903021,0.4893060029,0.8133631945,0.1535202712,1.6659163237,-0.6708444357,1.8384982347,-2.3238289356,-0.0551512279,-0.9679136872,0.0428756699,-0.7018769383,-0.1918913126,0.7778669596,0.8716066480,0.6106519103,-1.2859187126,-0.4537215829,-1.1330381632,1.0173296928,0.6480510235,1.4370362759,0.1523553282,-1.6252946854,0.6880706549,-0.4669993222,-0.1189476922,-0.0044371127,2.1690118313,0.2642358840,0.2700255215,1.5732755661,-0.3509848118,-1.8715351820,-0.0526743382,0.3962967694,-1.5842945576,1.2396576405,-0.8333008885,0.1840726137,0.7715519667,-0.3682126701,0.3607615530,2.0035834312,0.1541242748,-1.3954974413,0.0325314477,-1.6350804567,-0.9458541870,1.2335181236,0.3805272579,1.0706945658,1.7222070694,-0.6014351249,-0.0275845937,2.3488023281,-0.9489246607,1.0424493551,0.5509977937,0.5912781954,-0.3194181025,-1.8344695568,-1.5199538469],[-0.2534407377,-0.6822112203,-1.5001311302,-0.5672664046,-0.0814578012,-0.7873686552,-1.0422301292,-0.3025351763,0.4241429865,-1.5904347897,0.1101631746,1.0006101131,-0.6859989166,0.9753202200,1.0051900148,0.5923214555,-0.4300889373,0.7324078083,1.2622574568,0.0162067059,1.2897980213,-1.6987493038,-0.7873823047,-1.1084727049,2.4867002964,-1.0069047213,-1.1833405495,-2.6878020763,-0.4799250960,1.2554645538,-0.5688960552,0.0881453305,-0.4376583397,-0.6516036987,-1.0665112734,0.1584681869,3.1608436108,0.5720539689,-1.5262326002,0.4792236686,0.2597243190,-0.0070801945,0.4739401639,0.1779567897,0.4280982018,0.3195990920,-0.8183332682,1.4376466274,-0.3180052340,1.3227945566,0.3346593678,1.6663300991,2.1549162865,0.5529408455,2.3215246201,1.6306368113,-0.2287237942,0.3626566231,0.6241252422,0.2874783874,-0.5562575459,-1.0645240545,0.3074939847,0.9751630425,0.4938282371,0.4887374043,0.0105593298,0.6010999084,0.1875414848,-1.4502644539,-0.5887443423,-1.1079670191,-0.7399981022,-0.5830377936,1.9610450268,1.9490827322,0.1687097102,0.6483928561,-0.3631643653,1.2609117031,0.5470111370,0.3083211482,0.9779278040,-0.2285307497,0.6608920097,0.4434262514,2.3462135792,1.2344197035,-2.0416243076,-0.5078871250,0.6721323133,1.3655580282,0.6412608624,-0.1009380966,0.8257616758,-0.8150367737,-1.0085000992,-0.2928051651,-0.6150723100,-0.4936757684,-0.2183852941,0.9689996243,1.4067459106,0.2134040892,-0.4075092673,-0.4920285940,-1.7855901718,-1.0941070318,-0.2059654444,0.6229280829,-0.7334624529,1.2113645077,-0.4594071805,-1.4258073568,0.5498878956,-1.5167644024,-1.0164055824,-0.9841329455,0.4166430235,-2.2984161377,-0.7978627086,-1.9428042173,-0.4716677964,-2.6861188412,0.2464052886,-0.7541234493,0.0767890364,2.4924387932,1.0294005871,0.4078120291,0.0231720693,-0.2605399787,1.0949954987,-0.4825623930,-0.8047713041,-0.9314247966,0.1557550281,-0.2464975119,-1.7501392365,-2.1452300549,2.0917215347,-0.5078848600,-1.1502003670,0.8368268609,-2.2316017151,-0.6839256287,0.8083926439,1.5101549625,1.7092049122,0.3911790252,2.3782274723,-1.8589886427,-0.3594858348,1.6014767885,-1.1541914940,-1.0710375309,-1.0111941099,-0.3062404394,-0.2918458283,0.4250952005,-1.4448601007,-0.6035043001,1.5781127214,-1.5689852238,-0.8176691532,1.5150188208,0.6178444624,1.6630421877,0.4960637987,0.2892667055,-0.4297368526,0.3063223064,-0.3640555441,-1.3150182962,-1.5254424810,-0.1548462957,0.1717245430,-2.3199658394,-0.6687420011,-0.8737539053,-2.1310255527,0.1657736599,1.0154526234,-0.8788712025,0.4433715641,-0.6407283545,-0.1064218804,0.9291561246,-0.4667377174,1.0356763601,-0.2875490189,1.7318445444,-0.1297231019,0.2333392501,0.1582544744,2.0523178577,-0.7429432273,-2.1222071648,-1.4895672798,0.3120341897,-1.4317234755,-0.6542963386,1.3853491545,-0.8467838168,-0.7864772677,-0.7140951753,0.7617847919,2.6022748947,0.4526357949,-0.1344853640,-0.7801381946,-0.3832502067,-0.0866161734,0.4906598032,-1.0615791082,0.8940999508,-0.3497473001,-1.2624950409,0.1075066999,-0.0659220442,-0.6305398345,0.2686221302,-0.1823195964,-1.4534528255],[1.9937129021,0.1483629793,-0.2734782398,-1.5328689814,1.9309226274,0.0419135392,-1.9509328604,0.3718533218,-1.2233750820,-1.3658379316,-0.2958777845,-0.4734799266,0.7543007135,0.2856521606,-0.4374417663,0.8792541623,1.3735964298,-1.4071791172,0.1599424481,0.2732090056,-0.1884759814,1.5184983015,-0.3780666292,0.1182340458,-0.3417127430,-0.6154448986,0.9049147964,-1.8434865475,-0.6616224647,-0.2895358503,-0.4325897992,-0.6810391545,-0.1942910403,1.0399843454,0.7656468153,0.8919506073,0.8561698794,0.5976582170,0.8780407310,0.2451337427,-1.2113918066,-1.2457233667,0.5290142298,0.8976787925,-1.4235173464,-1.2115622759,-0.9235364795,-0.5710926652,-0.6926459074,-0.8372098804,0.1629962623,0.7053339481,-0.0904416740,-3.0984549522,0.6686881185,-0.7333278656,-0.6111493707,0.7304154634,-0.8588292599,0.8808065653,-0.6583173275,0.7451996803,-0.2612974644,-0.7302530408,0.4351638556,0.9296533465,0.6713768244,0.3829293251,1.4064166546,-1.2013900280,0.1457719803,-0.9313827753,2.6163449287,0.5473232865,1.4893498421,1.4373891354,-0.7347108722,-0.8683974743,1.4311701059,0.2228661329,0.4464642406,1.1559196711,0.0410200059,-0.1286565661,0.2647444606,0.6956012249,1.3178719282,1.0041838884,2.1021845341,1.0517970324,1.4445554018,-1.1228381395,1.5592223406,1.2020138502,-0.0028192063,0.5974342823,-0.8048190475,-0.9230668545,0.3318743408,-0.8511945605,0.0609234236,2.2532401085,0.2028496712,1.1774448156,-0.5526928902,-1.9647001028,-1.8609727621,1.0291814804,1.0270837545,-0.1857776791,-0.8119634390,1.0355705023,0.2473882139,1.0378277302,0.5087451935,-0.9540367126,0.8271316886,-0.2665016651,-1.7770938873,-0.6815657616,2.0232789516,-1.2229034901,0.2241365314,-0.0978290513,1.2142428160,-0.3724579513,-0.3604663312,-0.4075214565,-0.1006050706,0.7029295564,0.9857544303,-1.3855587244,-0.7235353589,0.1705339402,0.2272443473,1.5756491423,-1.0199034214,-1.4712222815,-0.1937682778,-1.4571503401,-0.4014287293,1.7913830280,1.4787840843,-0.8528799415,0.8736353517,-0.1563538909,0.1690182835,0.4658602774,0.1371072829,0.2078346610,-0.4466711581,-0.9114081264,-1.0166107416,-2.6908288002,-1.5399973392,1.4427289963,2.1239728928,-1.7548342943,-0.4824391305,-0.5262094140,-0.5427834392,-0.6966464520,0.4204410613,0.4263037443,-0.4591600001,0.2637559772,-1.9718030691,1.6417837143,-1.1051946878,-0.3639093637,-0.9486865997,0.4432385862,0.3481712639,0.4060375094,-0.5423926711,-0.3527583480,-0.4264259338,-0.9308899641,1.5440005064,1.5066274405,-0.1226757243,-0.3284087479,0.1419685334,-1.1529750824,0.2755541205,0.7012867928,0.0936654210,0.8950644732,-2.6700437069,0.2992852926,0.6254771352,0.1787260175,0.9335781932,-0.5372442603,-0.9831199050,0.8023634553,0.9252654910,0.0591549464,0.6226165891,0.2587251365,0.8355274200,-0.7327592969,0.6793955564,-1.4561997652,0.7465295792,-0.7011393905,1.5439178944,1.1826778650,0.2308337688,0.3512136638,0.1431451440,1.1339086294,-0.2589430809,-0.2183725685,-0.1566379517,-1.6944813728,0.3345756829,1.6899503469,-0.0316444114,-1.1810456514,-0.3684019446,0.3207881749,0.7920659781,1.6422004700],[0.2716714144,-0.5603635907,-0.3599757254,0.1677870452,0.0078198100,-0.4939026535,-1.7546492815,1.0226032734,-0.0200631488,0.5507588387,0.4915273488,-1.8573807478,0.8740385771,0.0427967496,-1.6065392494,0.7721800208,-1.1217762232,0.6055000424,-0.9973308444,0.6799186468,0.1740625948,0.0296238065,0.5217496157,0.1923450679,-0.5879488587,0.5928028226,-0.5710805058,-0.7600550652,1.1986136436,0.8760007024,-1.8550120592,0.7245780230,0.0701237619,-1.0581716299,-0.1062726602,-1.1632649899,2.3888280392,0.0001293403,-2.5444760323,1.6522936821,-0.4627757967,1.1748318672,-0.1822272688,-0.7353000641,-0.9973748922,-1.2355535030,-0.4939834774,-0.9968354106,-1.7728964090,-0.0487909168,-1.1636132002,0.7202252746,1.9825910330,-1.6155759096,0.5454311371,-0.7507312894,0.6600294113,0.0698041469,-0.8771593571,0.4602817297,-0.5655164719,0.0821633413,-1.6148333549,-0.2856899202,-0.9010733962,-1.5962631702,0.0608109497,1.7148712873,-0.1972455084,0.7661412358,1.7797145844,-0.7024210095,1.1253442764,1.9075862169,1.8911184072,0.2929170728,-0.4096447527,-0.0711998194,-0.7124832869,0.2756503224,0.8862338066,1.5982501507,-0.8595206738,-0.8688374758,-0.6461665630,0.7948951125,0.2192122340,-0.0730743557,0.1788961291,0.0994319618,2.5646259785,2.1651239395,0.6786749363,0.8574656248,0.1200354621,2.7303805351,-0.4879829586,0.8700460196,-1.2592363358,0.5971306562,1.3760768175,1.4629197121,1.4793893099,-2.2686212063,-0.3273313642,0.4973333180,-2.2781610489,1.1125961542,-0.3504355848,2.1755020618,1.2387313843,-2.1263351440,-1.7304564714,0.6072626114,0.8712618351,0.1535709053,-0.8973566294,1.4609305859,0.0118550872,1.2186610699,-0.3652283251,-0.0985303745,-0.4372721910,-1.1246514320,-1.9076437950,0.0822057426,2.1654286385,-0.5866769552,-2.5962011814,0.7465396523,1.0702034235,0.3408767581,0.8843899369,0.9880162477,0.3733038008,0.7469571829,-0.3654667139,-1.1835292578,0.0639124513,0.9607874751,-0.6897494197,-0.1239338443,-2.1009445190,-0.1710033417,-0.0617053173,-0.0188370664,-0.2777236104,-1.4326847792,0.8153862357,-0.0844529122,-0.1488251239,-2.0507738590,0.6880062222,-0.5953730345,0.8839527965,-1.1588287354,-0.8564046025,1.0334230661,-0.8303106427,1.0943323374,-0.1897011697,-0.7996029854,-0.3756458759,-1.0631002188,0.6210219860,-0.8794413209,-1.2669420242,-1.6641253233,-0.9473111033,-0.1794932634,-0.7986416817,0.4215012491,-0.3709406555,0.1757474840,0.9827396274,0.0277207959,-0.1375899166,0.9105586410,-0.6598195434,0.6502033472,-0.9171069860,-0.0821832120,-1.2704133987,-1.4377149343,0.8652183414,-1.9529612064,1.0661842823,0.9509752989,0.7923091650,-1.2883765697,-0.1132074520,0.0863654539,2.7263689041,-0.2265016735,0.2427878529,-0.7156814337,2.1491582394,1.4603886604,-1.8682442904,-1.1941982508,-1.4207702875,0.2143935561,0.6873126030,-0.8676314354,0.8319645524,1.4122692347,0.3956587911,-0.3590189219,1.4693763256,0.0357302763,-1.7043617964,-0.5215309858,1.2631664276,-0.5264201760,-0.1750185192,1.4218360186,0.2109541893,-0.8578353524,0.0341349170,-0.8854100108,0.6071174741,0.2590273321,0.6369572878,0.2007258683],[-1.5683698654,0.0528304949,0.2185943723,-2.0792095661,0.3473087549,1.5566731691,-0.9429470301,0.3542098105,0.1842521578,0.0781935975,0.9226818085,-0.1423786730,-0.4596142173,0.4551231563,1.8416955471,1.6079021692,-0.8521170616,0.5756788850,-0.5148565769,-1.2641240358,0.1978517026,-1.4766743183,1.1515684128,0.1763038188,0.2640692890,-0.8323398232,0.0230683852,0.6586131454,1.5933395624,-1.4179666042,-0.7774670720,-0.4964284897,1.4293264151,0.2270773351,-0.0167941060,-2.0283677578,0.0885260925,0.3764227331,-0.7332847118,0.9534940124,-2.1248283386,1.4294171333,0.7428011894,0.3662443757,-0.0806344151,-0.0672872812,-0.1971713156,0.2399710268,-0.5236250758,1.1410593987,-0.0949220136,-0.1947147548,1.1396582127,-0.1080093160,0.3194012046,0.2399757802,1.7890506983,-0.1936757118,0.8872783184,-0.1072830781,1.3524184227,-0.4339081645,-2.6261727810,-0.5521342754,-0.3567785025,-1.3351668119,-0.5639197826,-1.2869068384,0.4983268678,-0.3498931229,0.0223497171,-0.6007211804,-0.0052249120,-0.0424862429,-0.4661367536,0.1067087874,-0.0690322369,-0.8722267747,-1.4753071070,1.1081073284,-0.3371511996,0.5770972967,0.4407486618,-1.3237779140,-0.4519773126,-1.7085222006,-0.8972691298,0.9348768592,-1.0095119476,0.8008344769,0.5961164832,-0.3867681026,0.0516395830,-0.5375924706,0.7138876319,-0.9433989525,1.0816320181,2.1809961796,-1.1811590195,0.3935625851,0.3030322790,-0.6758750677,-0.9937831759,-1.3757630587,-1.3634648323,-0.8317195773,1.4114764929,-0.3476401865,0.2501884401,1.3920596838,-1.5477883816,1.7170302868,-0.8295890689,0.6617835760,-0.5374920368,0.1471432447,0.3597033620,-0.1635111123,0.8404725790,0.1525148302,0.7160716057,-0.8264759183,-0.5981168151,1.0916924477,0.9427590370,2.3003776073,0.5416697264,0.4278137088,-0.3079363406,0.1193876937,-0.2164104432,-0.0375090204,2.0504729748,0.2739741802,1.0871514082,1.1895177364,0.0692935809,-0.2275115550,1.4736508131,0.6942522526,0.1104047671,-0.8769634366,0.7775915861,-0.3444183767,0.0806557834,0.7060059905,0.7481377125,-0.9507731795,-1.2001460791,-0.2362423986,-1.4188394547,1.1630134583,-1.5362975597,-0.5948074460,1.5993782282,-1.2341548204,-1.5226598978,-1.5064285994,-0.7967886329,-1.6548575163,-0.7267010212,-0.2619487047,0.8821213245,0.9663928747,1.4349234104,2.9031307697,0.6623058319,1.7527112961,1.4296424389,-0.1722200811,-0.0848347172,0.3168185651,1.6467040777,0.2583158612,0.6035906076,0.8671565652,-0.5262527466,0.3319886625,-0.5497463942,0.4200908840,1.6204705238,2.8567569256,0.6891232729,0.3979914188,0.5920624137,-0.7559434772,-1.0975944996,1.1700562239,-0.4009858072,-0.7570546269,-0.8614937663,-0.8159902692,-1.5159968138,-2.3159945011,0.0993413925,-2.6141450405,-1.0592906475,-0.3929703832,0.5702621341,0.2737638056,3.2217895985,-0.4519669116,1.0404402018,-2.1363830566,-0.0241962317,-0.0399352834,0.8125180006,0.9310963154,-0.2669395208,2.5663955212,-0.6801775694,-1.4735100269,-0.8673360348,-1.0219819546,1.7570893764,-0.4413214624,1.6531904936,0.0096711023,-1.3588843346,-1.0291464329,-0.2288335562,-0.2092395574,-1.4058778286,-1.1912089586],[0.3284628987,-1.0475353003,0.3501040041,-0.0520703569,-0.4176031649,-0.1001003236,0.3600634038,0.1698016822,-0.5551374555,-0.0693989471,1.8276351690,-0.4938232303,0.9474299550,-0.3534296453,0.8308686018,1.2829548120,-0.9371561408,0.2828264236,0.8204187751,0.6815150380,-0.6391969919,-0.9756089449,-0.5878043175,1.2785863876,-0.4788477421,-0.9972364902,0.5297797322,-0.4876992702,-0.9401187301,0.1255615801,0.2447793335,-0.2591195703,0.2090267986,0.7089790702,-1.8272619247,-0.1440818161,0.8749162555,-0.2225515842,0.4995204806,0.0576416738,1.8956968784,-1.2364346981,-0.7038460970,1.0908890963,-0.4718849063,1.5693057775,0.4428057373,-1.1780397892,-0.7212728858,-1.7846194506,-0.1182599962,-0.7501379848,0.2489561588,1.8320907354,-0.6475961804,0.3235092163,0.1731548309,0.0735771284,-0.5053666234,0.7075278163,0.6965507269,0.5890418887,1.1716029644,-0.0515752882,0.3394494951,0.7700596452,2.5715339184,0.6072596908,-1.3007305861,0.0350546315,1.1603693962,-0.4538965821,-0.9783593416,0.0265502688,-1.2858632803,0.8167668581,0.1688446403,1.0662164688,-0.1026733741,-0.2595639229,1.0778741837,-0.4531909525,0.7623999119,1.2975021601,-0.0175253283,0.5181298256,0.2790040970,0.2507131994,0.0025475521,-1.6279962063,-0.1111964807,-1.8634821177,0.6435146928,1.0264328718,-2.0913350582,-0.5856692791,-0.1057725400,0.9844291210,-0.5320448875,-0.3761405051,-1.4245996475,0.6602067947,0.6311514378,-1.3559193611,0.3137751818,0.0596246235,1.0330663919,0.2284503877,-2.2971618176,-0.2615343928,1.0323364735,0.8715969324,1.4434144497,0.5397428274,0.6451210976,0.1844959557,-0.5831304789,-2.3598949909,-1.2856405973,-0.8049598336,0.3824418187,0.3135951459,0.5857704282,-1.8370773792,0.0837134793,-0.7865746617,-0.0764674395,-0.9077297449,-0.0669379085,-1.1441036463,0.5880070925,0.8309264779,0.4098588824,0.6063267589,0.1993841529,0.5601068735,0.0649905801,-1.5127788782,-0.6029203534,-2.0260241032,-0.0003807083,-0.1481437981,-0.8094789982,0.0633503646,1.7339324951,1.2790162563,-0.9811885953,0.9675010443,0.9853165150,0.5567837358,-0.3229420185,-2.2369308472,0.8412714005,2.2719078064,0.6248325109,0.0663437992,-0.4816880822,-1.0125662088,0.3245003521,0.8556165695,0.9044156671,-0.4809673429,1.5824408531,-0.6986125112,0.1831489205,-0.3048543334,-1.6329776049,-1.0484548807,-0.6673472524,-0.8022212386,-1.0551776886,0.3699105084,-0.9840784669,0.1571353376,1.7181296349,-0.1393129677,-2.6459677219,1.3768317699,-1.6159496307,-0.0081165750,0.4292753935,-0.2024205625,-0.9072603583,-0.2654633522,0.9579659104,0.1673599780,-0.8797431588,-0.5267537236,1.4190117121,0.6785064340,-1.1449259520,1.6081007719,-0.0469748788,-0.1846182495,0.2468699813,-0.2540178895,-0.7599125504,-1.4848744869,0.3800463378,-0.3932013512,0.1033055782,-0.8647897243,1.3579629660,-0.6633557081,-0.2434872836,0.5606272817,0.6669448614,-0.8341302872,0.3966738284,-0.3269149959,1.6633266211,-0.1126521826,1.4816939831,1.4156904221,-1.1166630983,-0.0606553070,0.0546188205,0.5445508361,-1.1936712265,-1.6309795380,0.8516145349,0.5265679359,0.1179899126,-0.5334897637],[0.5507861972,0.7433545589,-0.7290908694,0.0266585760,0.2498948872,-0.4962490797,0.9441946745,-0.5713911653,-0.0213359054,-0.7809194326,-0.3588345647,1.6766144037,-0.6612223983,-1.1081395149,0.7486165762,-1.3002353907,-0.0150593566,0.5272737145,0.7198087573,-1.7407058477,-0.8895874619,-0.9602290392,-1.2390866280,-1.4357074499,1.9074325562,-0.7550415397,-0.6781569719,0.4313832223,-0.8729441166,-0.1306226403,-1.0444041491,-0.3175023794,-0.0994322449,-1.2217093706,-3.0405607224,-2.0784544945,-0.3157816529,-0.2450707406,0.1222063676,0.6923543811,0.4367699325,2.0453593731,-0.1379508525,-0.2453946471,1.9804742336,0.8290540576,1.7626791000,1.0024492741,0.0583503172,0.4870532155,-0.5370153189,-0.3089361191,1.2510309219,0.3513412178,-0.9523450732,1.0798521042,-1.5914729834,-1.3701014519,-1.4990744591,0.9432504773,-0.8200966716,-0.7967817187,-0.1315291226,0.1992424875,-1.4237283468,-1.0715271235,-0.3152555823,0.6245658398,-0.5692147017,0.6681869626,1.0679280758,-0.0746023059,-1.1953831911,0.1121772081,0.9854134321,0.1716614068,0.7160137892,0.8524336815,1.9281487465,-1.0551660061,-0.0363917984,0.8865175247,-1.9527336359,-1.4990662336,1.0684392452,-0.6240852475,-0.3335091770,0.0973984450,-0.1852093339,0.5368331671,-1.4069367647,2.1364684105,-0.7955299020,0.8053387403,-0.2109974325,-0.2769947350,0.9727430940,-0.8858503103,0.2244490236,1.5067592859,0.9171571136,0.0369015187,0.2488748580,-0.4096579254,0.1632209867,-0.1903329939,-0.0624251887,-1.4613902569,0.3192446530,1.0520952940,0.1580966562,0.8568762541,0.0363764167,0.0138445962,1.7657972574,0.0549255759,0.9261724949,-0.2794926465,0.2829901874,3.4744529724,0.6240840554,0.0514252074,0.3288902938,-0.9306465983,-1.0744475126,-0.7351902723,-2.2935986519,0.7545247674,-1.1019783020,-1.3874216080,0.7956593633,-0.2586257756,0.8067504764,0.6845870614,1.1044504642,-1.3624535799,-0.0332132876,2.5060169697,1.7505385876,0.6042174101,2.1284039021,-1.6996500492,1.5683763027,-0.3298414052,0.6837784052,-1.2028763294,-1.4863746166,-0.4961459935,1.9162024260,-0.7335478663,-1.2341134548,-1.7722808123,0.8240976930,-0.0894157216,-1.5428103209,0.0126349842,-1.2780302763,1.0979908705,0.0222587101,0.4325967729,0.9945772886,-0.5362792611,0.1228516474,0.2050954252,1.4675050974,-0.5703064203,2.1974587440,-0.3568199277,0.2641554475,-1.1423335075,1.1377191544,0.0303159729,-0.2573347986,1.0057265759,-1.4090272188,2.4242238998,-0.6121561527,-0.0515848696,-0.1749645323,0.3523877859,-0.1276333034,-0.1103901342,-0.2866850197,-0.0464149900,-0.3077701330,1.4960175753,-1.3631552458,-1.7209351063,0.6491242051,1.4050065279,0.2676798403,0.2793314755,2.0455369949,-0.2602713704,0.6774649620,0.2130657732,-2.2492074966,-1.3942852020,-0.8942496181,-0.4105454981,0.8247628212,-1.2053675652,-1.0555409193,-0.4163510501,0.9191569686,0.0591878742,0.8970375657,0.0131746996,0.3099085987,-1.1245883703,-1.4458359480,-1.0214380026,-0.4532108903,-1.0915752649,-0.2660917342,1.9693787098,0.0519596413,0.3412489295,1.4262259007,0.1354945898,1.2808616161,-2.3592946529,0.2309911996,0.6223339438],[0.2537646294,0.3454696834,-2.0824322701,-0.1931213439,-0.3224272430,1.1653181314,-0.2260784060,-0.4215541780,0.4131685793,-0.3051755428,-1.3343261480,0.1708700806,-1.3212068081,-0.4973501563,2.0076658726,0.4784404337,0.0108408937,0.6487086415,-2.2793097496,-1.5167176723,-1.1210008860,2.0984489918,-0.9964933395,-0.2537392676,-0.6362712383,-0.6566887498,-0.4824739695,0.9052526355,-0.9712961316,-0.1638876498,-0.0615659095,-1.1938039064,-0.2663016915,1.2792323828,0.7501752377,0.3072949052,-0.3662904799,-0.8046880364,-0.0044827433,0.4288261235,-1.3418343067,-0.8763881922,2.3424198627,0.4666526914,0.4213204086,-0.4127517939,-0.7668270469,-0.8369839191,1.0337042809,1.5320605040,0.9207615256,-0.4779024720,1.1084336042,-0.6088128090,2.3399221897,-0.8646865487,0.2438831180,-0.5149421692,-0.5303933620,-0.5218394399,-1.6868261099,-0.0578323007,0.0224060658,-2.2675683498,-0.0580441840,0.7456520796,-1.2059559822,-1.2292213440,-0.2225447744,-1.3295269012,0.0634491891,-1.0590066910,-0.3777595162,1.3155578375,1.4310772419,-0.5424057841,0.1443112940,-2.4531741142,-0.8215950131,-0.1773539037,0.8877364993,0.1353154629,-0.6411932111,-0.5869292021,-1.4347361326,0.0225070901,1.1605197191,0.6720536351,-1.4405599833,-0.5473454595,0.7145345211,-1.1814160347,1.8402242661,0.3987374604,1.1327170134,-0.0116247721,0.3855871856,2.2576406002,0.3361343741,-0.6971358061,0.0903513953,1.9358855486,-0.0837212652,0.7940992713,0.6294739246,-0.0230218563,-0.4086762071,-0.7781276107,-1.5784299374,1.9371912479,-2.1188192368,0.3531058133,1.1220483780,-0.5264663100,-0.1083615795,-0.0701917931,0.1257195175,-0.3426410854,1.4279152155,0.8725541234,0.2656742334,-1.4155806303,-0.5311722755,0.5770264864,-1.9181660414,0.2317260057,0.6019353867,0.9077202678,-0.4718322754,-0.5740461946,-0.1232953146,0.9156028628,0.4621352255,0.4633850753,-0.7176672220,-0.9824862480,-0.1898862273,-0.2449781001,-1.5985085964,0.9512001276,0.8413665891,0.0192009527,-1.0423125029,-0.9620074034,-1.1568783522,0.0290435944,1.1404398680,-2.0211308002,1.6410949230,1.1472212076,-1.1088819504,0.8519145250,-0.5757724047,0.0184833016,0.5957829952,0.5147176981,-0.7896971703,-0.9568346143,0.3608250320,-2.3002521992,0.9077754617,-0.6996142268,-1.3030370474,-0.4180361331,-0.2474314123,-0.0635405704,-0.4888866842,1.1069529057,0.4685696661,0.1609025151,-2.3913598061,1.0130462646,-0.2157368809,-1.0683450699,-1.5078005791,-0.5091286302,-0.1590702087,0.3359768391,0.5903284550,-0.0485109314,-1.3737658262,0.5323999524,0.1814595312,-0.9770289063,-0.0009856647,-0.5759761333,-0.8987975717,0.5400237441,0.9631063938,0.7029232979,0.1237453818,0.6921179295,-0.9819133878,-1.1649929285,1.9168785810,0.0260932688,0.5961437225,-1.0539348125,-0.1854109466,-0.4541092813,-0.8012022376,-0.8627537489,-0.3639629781,-1.5960626602,1.1273057461,2.2912368774,-1.5819373131,0.0768916383,-0.7541574240,0.9644643068,0.4963940382,-0.3556044698,-0.1880870759,0.2791007459,0.9201265574,-1.6708345413,0.0320946388,-1.5955154896,1.6061308384,0.0217349976,0.3895410895,1.7398505211,-0.1265877634,0.4094446003],[-0.6966269612,-0.6312574744,0.4016741514,0.8904861808,0.6256996393,-0.4016552866,0.4941623807,0.4165548086,-0.1249488592,0.7846920490,-0.4188575745,-0.1406167001,0.5847015977,-0.3898477852,-0.9432166815,0.0343675576,1.8963041306,1.1666419506,1.9057749510,1.6193721294,-0.5440173149,-0.2137131691,0.7101011276,0.6619938612,-0.5311143398,0.6764826179,-0.9523943663,1.3086831570,-0.3232148290,0.3837288022,1.1706738472,-0.1583350450,1.3312753439,0.2241350561,-1.6838308573,0.5691562891,-0.4549115300,0.3088451326,-0.1045141220,-0.0612401441,-0.7073890567,-1.0161758661,0.5968431234,0.8028610945,-0.2996942997,0.2950571775,0.4948324561,0.4771435261,0.2741064727,-0.2335633188,0.2663442791,-0.0397798195,0.5287874341,0.3386964500,0.0797383338,-0.0556877255,-0.8294295669,2.1340575218,-1.6558796167,0.3286934495,1.6617243290,1.0909644365,-1.7356817722,-0.3916744590,0.6007047892,0.0267706569,-0.6286586523,-0.8454390168,-1.2340822220,-0.5262479782,0.2183321267,-0.7545596361,-0.9005653262,-0.5328521729,-0.8601483703,-0.7806350589,-1.0522665977,1.8136574030,0.9027171135,-0.7261156440,-0.0490528308,0.5775359869,0.5649013519,-0.7593803406,0.3464302421,0.6238087416,0.6746863723,1.7811609507,0.0354143009,-0.7106010914,-1.6929748058,1.2977504730,-1.0984780788,1.1117619276,0.1678503603,-0.5709893703,-1.7195818424,-0.9556397796,-0.9034484625,-0.0908293873,-1.0700818300,1.3671873808,-1.8064297438,0.7554394603,-0.5026817918,1.7806242704,1.0917088985,0.8868745565,-0.0075888634,-0.4598862231,-0.7556013465,0.0549330562,0.2221430540,1.3729677200,-0.9540419579,-0.3789446354,-0.1505113691,0.7726684809,1.7614964247,1.0252877474,-0.3415622711,-1.3802000284,0.0690797642,-1.1985107660,-0.5829547644,-0.0029087467,0.6864536405,-0.7441490889,0.9273821115,1.1335115433,-0.0438062884,-1.3134324551,0.2154727727,0.3803556263,-0.3672275543,-1.3546211720,-1.5402970314,0.0953702331,-0.4527512193,0.2291198820,-0.7780883312,0.3277203739,-0.0930765718,-1.0328152180,-0.8990030289,0.9657142162,-1.7825680971,1.2948658466,0.8651069999,0.5233416557,0.6785563827,0.1730827838,0.6709386110,-0.2403714061,-0.3969297111,-0.0562244244,0.9086267948,-0.0806159154,-1.3319112062,0.4982792437,0.0500809923,0.6967548132,1.0448199511,-1.1299753189,-2.7560486794,0.1868395656,1.3268688917,0.5976457596,-1.8544654846,-0.6114086509,2.1996905804,-0.0069061555,0.9068157077,0.0332768932,0.5828394294,-0.3923009932,-1.1152610779,-1.0476948023,-0.0980769023,-1.9471046925,0.3602553010,0.6499018669,-0.3639253378,-0.6350816488,0.1268273890,0.4119163752,-1.2266123295,1.4781122208,0.3314739168,-1.3378620148,0.2472490221,-0.8766486049,0.0906429887,-0.6079481244,-0.6427965760,-0.8671147227,0.8152963519,-0.0899893939,-0.0517604873,0.2418456525,0.3122246563,-0.7654765248,1.3197525740,-0.1157104000,1.7435361147,-0.6417414546,0.6864396334,-0.8152928948,-1.3649516106,-1.5759655237,-0.1911571473,0.9851586223,-0.1836736351,-0.1282128096,1.4109607935,-0.3864745796,-1.0339030027,-1.3751579523,-0.5158736110,-1.0271582603,1.2922033072,-0.2954572439,0.4869286120,-0.2285408676],[-0.0162027627,-1.9017342329,1.6025589705,-0.4845978022,-1.2267167568,-1.0501669645,-0.6654804945,-0.9559248090,-0.6065716743,0.0243174918,0.7494941950,-1.8077092171,-0.1077916324,0.8796631098,0.0062837857,0.5151043534,-0.6206137538,0.1735114008,-0.9676117897,-1.3759051561,0.0525254011,-0.5183722973,-0.1062775850,-0.8050181866,1.2755728960,0.0143800322,1.2767688036,0.9166401029,-0.3082312644,1.8080588579,-0.6768823862,0.3165327907,0.8092871904,-0.4501982927,1.8188048601,-0.9475402236,-0.0415148437,-0.8145132661,0.2790819407,-1.1835186481,0.7343748808,-0.5032110214,1.4122532606,0.2738788724,0.8091881275,-0.0099356575,-0.6308447719,1.1316365004,-0.3096952140,0.2567524314,-0.3351007700,0.8001866937,1.6676236391,-0.8846474290,-0.2603313923,0.1370266527,0.0493109673,0.9733078480,-0.3513320982,-0.8878127337,-0.7262001634,0.2246757895,3.0239856243,0.2840051949,-0.7080864906,0.4759110510,-0.0502727516,-1.0398198366,0.4720818400,0.3196711242,1.2657147646,-1.1127299070,1.2408133745,-0.3051033914,-0.1297694147,0.6046590209,-0.4969549775,-0.7131097913,-0.9040707946,0.6418151259,-1.3217216730,0.0984766409,-0.9235069156,1.1407945156,0.2827794254,0.2023810148,-1.7084463835,-0.1919159442,-1.6301255226,-0.4870570004,-0.0667773858,0.4956967235,1.3667733669,1.2660340071,-0.0160720777,-1.4300978184,1.5479718447,-0.5026729703,1.6225996017,-0.4625676870,-0.9789949059,-0.6550973058,0.4892952740,-0.0918960720,0.4369480312,-2.7726590633,-0.3816050291,1.4429813623,0.7978885770,1.0307193995,-0.2216360867,-0.0587479360,-0.1162854955,-0.3884234428,0.0990402028,0.2731604576,0.1190148816,0.0518279746,-0.1054238603,0.4520575404,-2.0491440296,-0.3809584081,0.3005086780,0.3459797204,0.3234552145,-0.1599316448,-1.6980955601,1.7123942375,1.4055458307,-0.8293477893,0.1827742159,1.4559041262,0.0637789965,0.0475166366,0.6155928373,-0.9956529737,-1.4131263494,-0.1182635501,0.8190299273,0.5125033259,0.0770660043,-0.7871782780,0.4206069112,-0.0446137898,-1.1966786385,-0.2150852829,-0.2339724153,-0.7401009202,0.5566579700,0.6771113873,1.4938442707,1.3028098345,-1.3230277300,1.6066470146,-1.3189405203,-1.7869796753,-0.8453440666,0.1779462099,-1.0992565155,0.8223231435,-0.3773572743,-1.4823824167,1.8274016380,-0.9582762718,0.8690104485,-0.1457026154,0.4907839894,-0.4807015657,-0.1869495213,-0.9083141088,-1.5402960777,0.0317795537,0.0290756747,0.2935406268,-0.0085515557,0.8803539872,0.3078594208,1.2030415535,-0.8128335476,1.6358748674,1.1200956106,0.4681635797,-0.2939386666,0.9326472878,-0.6292216778,-0.7046082616,0.2003094703,-0.8784189820,0.2474409193,0.5440397263,0.1008563042,0.5313048363,-1.4959849119,0.2413581014,-1.0873416662,-0.7164362073,1.2811007500,-0.7253606319,-0.3197246790,-0.9055584073,1.0824757814,-1.0967745781,1.4039075375,-0.8226977587,-1.5650757551,-0.1779900938,0.3357891142,1.6376110315,0.2842205167,0.0611847453,1.0054154396,1.5567678213,-0.4624824226,1.7156288624,-1.6495491266,-0.9886603355,-0.5040734410,2.1461417675,0.4368059337,0.2772457898,-1.4140040874,0.0257757753,0.3154803514,-0.2752482891],[1.2735975981,0.9660745859,0.3129498661,-0.9183510542,-0.0528000295,-1.4535831213,-0.4218615294,0.2285152674,0.4834912121,-1.0394859314,1.5353862047,-0.6263337731,-0.3788434863,-1.7689143419,-0.9368940592,-0.4472614527,0.7502661347,1.1232037544,-0.2082219273,-0.3345964253,1.1510394812,-1.1813848019,0.5503615737,0.5231403112,-0.8894044757,-1.1481416225,0.3089981377,-0.0089807743,-0.3435496390,-1.4222484827,-0.6001983285,-0.2960148454,-0.6301397681,-0.2517532110,0.5226249695,1.6253954172,0.6440001726,0.6713175774,-0.9343277812,-0.4876067936,0.2829792798,1.1756912470,-0.6696252823,-0.0702424943,-1.6962444782,0.5822570324,-0.3008073270,-0.7424628735,0.4427735806,-1.6259888411,-0.2963338494,-0.9052641392,2.0158989429,-0.0409712046,0.0850246251,1.2611733675,-0.5877819061,1.5001118183,0.3374356329,-0.8833405972,-0.8129408956,1.0231391191,0.2039081007,0.5219868422,2.1710143089,-1.5364967585,-0.9825080633,-0.4576882720,0.1181099117,-1.3131301403,-1.3747745752,0.6685074568,-1.1598657370,0.1115264669,1.2732436657,0.9837774634,1.0079425573,1.2042628527,1.4757161140,1.9824581146,-0.0836161450,0.6380442381,0.6169739962,-0.6086562872,-0.5009913445,-0.5547283292,0.9836614132,-0.1071657091,1.2167388201,-0.7316679955,-0.1268628389,-0.5843551159,-0.3292862475,1.3337472677,1.3200297356,0.7653281689,0.2057382762,0.2780999839,1.9284055233,-0.4090837836,0.1546208113,-0.2178405672,1.8453789949,-0.0609560572,-0.8356684446,-2.0992302895,-1.6502887011,1.8192211390,0.5958074331,0.6264472008,-0.6886128783,0.0464336015,2.2116112709,-1.5227183104,0.2625297606,1.5642783642,-0.5167988539,-0.1535461694,1.0193160772,-0.4149324000,-1.3377883434,0.2785515785,-1.4212867022,0.5633701086,-0.5416283011,-0.5272933245,0.0087975692,-0.5657374859,-0.5035797358,0.5643779635,1.5762873888,-1.0485948324,-2.0079891682,-1.6455342770,0.7365643382,-0.0382613279,1.7036528587,1.3061082363,-0.6127634645,-0.4571321309,0.9667922258,-0.2397709042,0.7455832362,-1.3719718456,0.1176320612,-0.9625248313,-0.3331040144,-2.3206830025,-1.1050237417,0.0505683310,-1.6521037817,-1.1555784941,-1.8600842953,-0.1436365545,-1.3095002174,0.5672355294,2.2935447693,0.2653402388,-0.4710046947,0.0420597941,0.6235096455,-0.3959354758,0.6133328676,0.8685811758,0.3623816073,0.8595988750,-1.2475121021,-0.8317956924,1.3300708532,-0.6564030051,-0.3196218610,-0.9565773010,-0.9911345840,-0.7629100680,-0.9317533970,0.4946089685,-0.0070229843,0.5263068676,0.1096296310,-0.7030386925,1.3341176510,0.0588235408,-1.3557839394,1.3701558113,1.3979157209,-0.0241335556,-1.5421690941,1.3926768303,-0.0572788082,0.0924422741,0.1306796372,-0.4506987333,1.0594346523,0.2627052665,-0.7358296514,-1.7386627197,-0.8884651661,-1.5768609047,-1.2932181358,-0.3603904545,-0.5076446533,-1.8667639494,1.0743008852,0.7913588881,1.0181564093,0.6479498148,-0.0823470876,-0.0806593820,-0.4157681167,2.2641456127,0.4007044137,-0.3426760733,0.3771518767,0.5061627030,1.5559914112,-0.4237650633,-1.1088336706,0.7923083305,0.6938353777,-1.3250257969,1.5535944700,0.0049045268,1.1246401072,0.0097699733],[1.5038679838,0.5279566050,0.6128170490,0.3623874485,-0.0392698757,0.3612433374,0.6001952887,-0.7416902184,-0.3430719972,0.1613178700,-0.6219967604,0.2432402670,1.3867510557,1.4010163546,-0.6260728836,-0.2785092890,-0.8145068884,-2.1860978603,0.4015526474,-0.5687218904,0.0735837519,2.3719646931,-1.2226122618,-1.7341774702,0.5211261511,-0.7689599395,-0.4643093646,-0.1880650520,1.8296451569,0.1179625243,-1.2873387337,0.2452962250,0.7529872656,-0.4656670690,-0.7779188156,-0.9317938089,0.2511917353,-0.8010789752,0.3875940442,-0.2854766250,0.2135651559,0.9496786594,0.4434437156,-0.2921417952,-0.2893070579,1.8729990721,-1.3912776709,0.8823206425,-0.7693408728,0.8297691941,0.6176412106,-1.6273237467,1.3451734781,0.7532227635,-0.9126815200,1.1919921637,-0.1777725518,-1.3362666368,-1.0282952785,0.7495453358,0.7746673226,0.5302920341,-1.1779735088,0.4359086156,0.9370511770,0.6625077128,-0.2525390983,-0.1961964667,0.6796283722,0.1560912132,2.0079288483,-1.3467699289,-0.9276216626,-0.0741150156,-1.6958822012,-0.2605181336,-1.2530063391,0.2627429664,0.7165470719,-1.8141559362,-0.9000178576,0.7370605469,1.4289104939,-0.6930576563,1.7424666882,1.6449006796,0.5382955670,-0.7510215640,-0.2363715321,1.0369923115,0.9657538533,-0.7431786060,-0.8046639562,1.4332410097,0.1665688902,-0.6746644378,-0.3744138181,-0.2142071873,0.3782551587,-0.0284325425,-0.5066626072,2.2650198936,1.4231873751,-0.0743084699,0.4235864282,1.5384422541,1.3585460186,-0.0352351889,-0.2189443856,-0.8031013012,2.0303540230,0.5452772379,0.2255565971,0.0925863236,-0.4921898842,0.2845351994,0.2062941492,0.3464781344,-0.0111541692,-1.7251620293,-0.7196274996,1.0001733303,0.1533291489,-0.7373467088,-0.4225189388,0.1528119892,-1.0180543661,-0.2779996693,-1.5780826807,-0.4613038599,0.5342951417,-0.1558797210,0.0374767967,-0.3804990351,-0.4957786202,-0.2577657402,0.5751443505,-1.0979690552,0.0594886243,-1.0828264952,-0.1919242889,-0.3676927984,0.5205117464,0.1584236622,0.1191426143,-0.8184567690,2.6164002419,1.8043608665,-1.4470647573,-0.7504413128,-0.0128210383,1.7690523863,-1.0356879234,-0.8855569959,-0.8161084652,0.3014318943,0.0012616556,-0.5786834955,-0.5376681089,1.7274968624,0.7023185492,0.0531656817,0.0197559502,0.0192194246,-0.4238943756,1.0916885138,0.3940524459,0.6978895664,-0.5300963521,-0.1750378609,-0.7282319665,0.3309630752,-0.7242958546,0.8023917079,1.2333136797,0.0600263327,1.8488427401,-0.7148504257,0.4385460615,0.9688469768,0.0299088582,-0.9750109315,-1.0483665466,0.0277658887,0.2368310541,-1.2071895599,1.2570800781,2.0445125103,1.6918454170,-0.6865190268,0.2112120986,1.5974457264,-1.1287384033,-1.5620183945,-0.9195592403,-1.1469166279,-0.2893614471,0.6627973914,1.0217088461,-0.3918666542,2.0864372253,-1.0320661068,0.3124392033,1.3697936535,-0.3738217652,-0.1886551231,-1.0122405291,0.8189871907,-0.3247868717,0.3251466453,-0.2431050241,-0.1908031404,0.0159445219,-1.1716156006,0.4313590527,-0.4181154966,-0.5454003215,0.3107697368,-0.4312063754,1.6988054514,0.3427794576,0.5752515793,0.3136224449,0.0746937171],[-0.2343776822,1.3644443750,1.7880711555,-0.8330552578,-0.6701244116,0.0977286845,0.6908764243,-1.6960651875,2.1555716991,-1.0845538378,-1.0772690773,0.9238383770,0.1551842541,-1.7129218578,1.0820883512,-0.6496840119,-2.1078553200,-0.8854192495,-0.0315521695,0.1816882342,-1.6160395145,-2.0274572372,-2.2314357758,0.3817986846,1.3415185213,-0.7032774687,-0.6273907423,-0.5772133470,0.8161108494,-0.3537312150,0.8900743127,0.7412501574,-0.1986007243,-0.0569934621,-2.0071425438,0.8271296024,-0.2330507785,0.3007969260,-0.4647093117,0.4949223399,-0.4368942380,-0.0580027997,-0.2060611546,0.5465889573,-0.7176531553,0.2781758308,-0.0540548712,-0.8538475633,0.7752673030,-1.1576044559,-1.6881654263,0.2807030082,-0.8262535930,-0.0240261238,-0.9551573396,-0.2277644128,-0.2805421650,-0.2276842594,0.8241704702,1.1559234858,-0.7621541619,0.1560282558,-1.3470283747,2.0022385120,0.4317237735,1.2540224791,-0.1634758413,0.8219774365,1.1389209032,1.5196316242,-0.1628244519,-1.0992746353,-0.9544483423,-1.6510956287,-0.6569509506,1.0500578880,0.6824148297,1.5513504744,0.4548636675,2.7451868057,1.0425418615,0.1934823841,1.9569149017,-2.2074351311,0.9075617790,2.0576922894,0.8540446162,-0.9392482042,0.0225526020,-1.3599926233,0.0982640311,-0.4961391985,-2.2542142868,0.7189049721,0.5635581613,0.7037670016,0.3800044656,1.0090733767,0.7960253954,1.5524606705,-1.0578310490,0.0954674557,0.6344092488,0.2032889277,-0.2713959813,0.8102875352,-1.6120643616,-1.3914624453,1.3278867006,0.7482947111,0.8191539645,-0.2149544358,0.5363849998,-0.9918892384,0.8326933384,-0.4563343227,0.9957206845,-0.3911642730,-0.5761083364,-1.6257008314,0.3507175446,-0.6471960545,-0.6633731127,-1.1012862921,-0.1357158273,-0.4140460193,1.1256126165,-0.2547148466,1.1853294373,-0.6736295223,-1.2852391005,-0.6020596027,1.0396854877,-1.1734688282,1.0483592749,-0.1930020899,1.4001460075,0.6863822937,0.1874726713,0.1960242987,0.3801824749,1.2780096531,0.0715012625,-0.5726922750,-1.1324484348,0.3092543483,-0.1041374058,-0.7648612261,-0.6558532715,-1.1425851583,0.3871780038,0.9160779119,-0.2528611720,0.3229587674,-1.3621455431,1.0407729149,-1.5615122318,-1.7674455643,-0.1657711864,0.3561912477,-1.0535864830,-1.3808097839,0.0066822111,1.7632035017,-0.5058322549,0.4460215271,1.8662033081,-1.4253808260,-0.0295527857,1.3246070147,-0.4824059010,-1.3144742250,1.6353757381,-0.7058399916,0.6552345157,-2.3853127956,2.0654399395,-0.8080892563,-0.7019706368,0.4695746005,-0.5398542285,1.9267045259,-1.6482932568,-0.7667036057,-0.0920951143,-0.3394153118,0.4104304612,0.0820508972,-1.0529474020,-0.1035258546,-1.2655054331,-0.8173121810,-1.5338441133,0.7748808265,-1.3071223497,0.7862274051,1.2642416954,-1.6465983391,0.7156240940,0.0004608648,1.3903490305,0.7563246489,0.7296988368,3.1193788052,-1.5182796717,-0.4620943666,0.6188848615,-0.6592494845,0.1667702049,0.1148974299,-0.4236586988,-0.6178045869,0.5739185214,-0.7387426496,-1.9529211521,1.4813373089,0.9259881973,-0.3275189400,-0.5661215186,1.0159183741,0.3718490899,1.7064806223,1.0632951260,0.2158806324],[-0.0558133088,0.9810810089,1.0780202150,1.5175112486,-1.0642248392,0.8798881769,1.0489073992,0.7985528708,-0.2425360382,-0.5692839622,-0.4421700239,1.1829074621,0.1835972518,0.4506121576,0.9762408137,1.1348419189,0.5359632969,-1.2192471027,-2.3011717796,-0.2138852626,-0.4985000491,-0.1059588790,-0.0988596678,1.0034368038,-0.5149103403,-1.8779270649,-0.6865703464,0.1736295819,0.8265163302,-0.5225960612,-1.3647763729,-1.0260533094,0.2010704130,0.6330992579,-0.8404617310,0.3592998385,0.3706113994,0.5354285836,1.0670536757,-0.1831074059,-1.4682694674,0.0098087247,1.5302503109,0.1058414206,-0.1238831207,0.4766884148,-0.4761192799,1.2386989594,0.8832945824,-0.0387233086,1.3760423660,1.6366331577,-1.9266943932,0.2121179402,-1.6006190777,0.1265319735,-0.9488639832,0.3272991180,1.2126785517,0.6757702231,-0.9969081283,-0.8930254579,-0.6828251481,2.1810488701,0.1021359116,-1.7115709782,0.7546977401,1.5872236490,0.2561224401,-0.6409754753,-1.3000752926,-1.5575027466,-0.3797564209,-0.2285472006,1.1859147549,0.0516077168,0.9234830737,-0.8794897795,-0.2575468421,-1.9377614260,-1.5830955505,-1.5876456499,1.1509450674,-0.4737126529,-1.1898517609,1.3928337097,1.6262747049,-0.4686792493,1.0509574413,1.5402649641,-0.7110273242,0.7389134169,-0.4548214078,-0.8832143545,-0.7396560311,-0.8138209581,0.4599758983,0.0499489568,1.7926896811,1.0037494898,-0.0817498639,-0.9807576537,0.1917232126,-0.7828310132,-1.9145452976,0.4286828935,0.9644827247,0.0961133540,-0.3806517720,-0.5880751610,0.0334974229,1.4758988619,0.0888777971,-0.6958780289,0.1805043221,-1.7607322931,0.2534747124,0.0275088325,2.2588088512,0.7110636234,-0.1102534533,-0.0299374927,-0.5157200098,-0.4786728323,-0.7900257111,0.4293768108,0.3290570080,0.0805050284,0.6053853631,-1.1858336926,0.1329994649,-0.8362782598,-0.4463221431,0.2802517414,1.5189036131,-0.9790352583,-1.0094639063,0.5211237669,-1.6937114000,-0.1594283283,0.6029412746,-0.0587336123,0.0071533583,0.0523125641,-0.2256063074,0.1295899749,0.1687475741,0.6759440899,-1.6508245468,-0.6614670157,1.0911030769,-0.4605453908,0.4364682734,1.3217781782,1.1893281937,0.1233901903,-0.9958592653,-0.3711001277,0.5379791260,-0.0070412341,-0.3989662826,-1.0785380602,0.1701700389,-0.3741233051,-1.2407640219,0.4376882315,-1.1232664585,0.4728543460,1.7027870417,0.3569022715,1.6514447927,0.0602583177,0.3123974800,0.1193226576,-0.5213288069,-1.4381340742,1.4283020496,0.5926224589,0.1629691124,0.5686670542,-1.0246697664,-1.1791366339,-0.9171492457,-1.4742364883,0.7465749383,0.4492428005,0.9631430507,1.0591726303,0.1239316314,-0.3596034646,-1.0907340050,-0.6030093431,-1.1057633162,-0.8140255809,-2.1808836460,0.8547969460,-1.1567357779,1.3341215849,1.3263473511,-0.6043748856,-0.6644125581,0.8303882480,-0.7894672751,-0.5509001613,-1.1912413836,-0.4110831618,0.0052479375,-0.5508015752,1.2821806669,0.8930993080,2.6927654743,-0.0384279191,-3.2195563316,-1.0180802345,-0.8228209019,1.4848512411,0.1804568619,-0.0416801386,-0.4823452234,-0.3673158586,-1.7843573093,-0.6924807429,0.5369516015,-0.6816094518],[-0.7692908645,-0.4790070057,-0.4438000023,0.0430159196,-0.1231807619,-2.3414075375,0.4058639705,-1.1069461107,0.9379891157,0.5567139387,1.6205422878,-0.0333523005,0.4900620282,-1.5986202955,0.5159187317,-0.8220391870,0.5268859863,-0.1108082831,0.9905723333,0.6257894039,1.1551108360,0.2970907986,-0.0550662689,-0.3417685330,0.4716485739,-1.9665927887,0.6737215519,-0.3018049300,0.1630557626,0.4377413690,1.6650433540,-1.2340313196,1.0803861618,-0.9714391828,1.0315043926,2.7991528511,-0.9023184180,0.3342669308,0.8662059903,0.4187904596,1.6631202698,0.9461517334,0.0383105762,0.3874709606,-0.9331662059,-1.5427381992,-1.2080690861,-0.0268997028,-0.3367968798,0.8820804358,0.8696154952,-0.9525085092,-0.1397367120,-1.0950832367,0.9954438806,0.1454400718,-0.1136238873,2.0428493023,1.1829013824,1.1000951529,-1.1066437960,0.1360276341,0.7086342573,-1.2057681084,0.1230445802,-0.0300690066,-1.3471719027,-1.1712793112,0.4815954268,-0.0332318284,0.0187704470,0.8783824444,-0.2106922418,2.4618752003,1.7784832716,0.2102992982,-0.2804616690,1.8727797270,-1.0322837830,-0.6775320172,0.5041838288,-1.9068819284,-0.4035941362,-0.4970909655,0.4764173329,0.7262157202,-1.3696603775,-0.2180223465,0.9146516323,-0.6380504966,-0.6836457253,-0.9392985106,2.5385532379,0.7848135829,-0.4739396274,-0.1609451324,-0.3473201990,0.4547399580,0.0022027995,0.3719330728,-0.2290373445,-0.8378288150,-0.5939671993,-0.4501929879,-0.9564500451,-0.7512276769,0.3331740499,1.7069742680,-1.8661465645,1.6704733372,-0.7440801859,-0.5513499975,-1.3664151430,0.4946258068,0.6614918113,1.0545921326,0.7613997459,-0.1494040340,0.1473492235,1.0284640789,0.1794055253,1.3030784130,-1.0664352179,0.8017191291,1.0707521439,0.4280396998,0.7229521275,0.6674688458,-1.4753994942,-0.1161597595,-1.5815646648,-1.1041642427,0.2921075225,-0.6162883043,-1.0617330074,0.0474488698,-0.2354591340,-0.3827132881,0.0709788352,0.9242456555,-0.7282121181,0.5422099233,-0.7606021762,-0.6083860993,-0.0204535741,2.1217327118,-1.1327162981,0.5107544661,0.0726600289,-0.0524952970,-0.1610554010,0.5236122012,0.4384548962,-0.2347787917,-1.5538160801,-0.2386575490,-0.0072724433,0.5456941724,-0.1086401194,0.7346530557,-0.5908902287,-0.1918531358,-1.8580100536,-0.5037835240,2.0335335732,-0.9577164650,0.0651107430,0.9060782790,-0.5345256925,0.1751286089,0.4575206637,0.3239696324,0.0477281995,0.8175007701,0.9586806893,0.2785039842,0.5604625344,1.0212132931,0.4966633320,-0.0258158334,0.9687435031,-1.3536084890,0.8649412394,-0.8730837107,-0.8804031014,-1.6783584356,-0.5945875645,1.2743442059,-0.6184026599,0.1897069663,0.0141431382,0.2350640744,-0.7193506956,0.1511185616,-1.5639774799,-1.2819161415,0.0610125959,-1.1097881794,-0.8521714807,0.5019956231,0.0409483574,0.1502437443,-0.7988834977,-1.8650946617,0.5831124187,-0.7960220575,0.2591686845,-0.0523419008,0.6130117774,-0.4703146815,0.8446735144,0.9293216467,-0.7193233967,-0.3462554812,0.0266267434,-0.2921901047,-2.0190968513,0.8755489588,-0.1333952248,2.0068006516,1.4492278099,0.1540557742,1.7440308332,-1.0899720192],[-0.3867556453,0.1545641422,-0.7254714370,0.7313339710,2.5498569012,-2.4311244488,2.2926156521,1.3991469145,1.1989866495,1.3550738096,-0.9609283209,0.3282889426,0.8142353892,1.5970354080,-0.2387523353,0.0792709738,-0.5485795140,0.4264426529,0.2009727061,0.3822948635,-0.3160178065,0.9924373031,-1.0522011518,0.6819784641,0.1449483186,0.7345592380,-0.4123094082,-0.8291899562,-0.1473677456,-1.9918057919,1.2308152914,-0.4426414371,-0.3787471354,-3.0473160744,-1.6859333515,-1.3871006966,-0.0958412439,2.0981168747,0.5395963788,0.5177062154,-0.6460278630,2.3688600063,-0.7596668005,0.8309730291,-1.3052463531,-1.1465224028,-0.2534221113,-0.0777573884,1.5662372112,-0.8439161181,0.5423812270,-0.4233368933,-0.0301173329,-0.1885342002,-0.7636304498,0.6386103034,-0.3293936551,-0.5986554623,1.6507289410,0.6110025048,-1.1714577675,0.2430572808,1.4613201618,1.2600826025,-0.5669208765,-2.4710955620,0.7282733917,0.0715143606,2.3573234081,0.2791848481,0.3507069945,-1.4286094904,-1.3957622051,-0.6828049421,-2.1207742691,-0.5035426021,-0.0339629166,0.7257949114,0.6201557517,1.2286385298,0.9077768326,-0.3393974602,-0.8418522477,1.1805078983,0.0826563761,1.4109766483,-0.1218153611,-1.1622179747,0.6720657349,0.8871431947,-0.6016659737,-0.2201759368,0.5747567415,-1.4447933435,0.4896268249,0.7228952050,-2.4493446350,-0.7032957673,-1.5452859402,-1.4874507189,1.7770699263,-0.5654845834,0.9598588943,-0.2666018903,-0.1979894638,-1.8093239069,0.5844339132,0.0779933855,-0.7883568406,2.2556648254,0.2092601806,-0.0397630557,1.2062901258,1.1393060684,-0.5246899128,-0.6015848517,-0.1014802530,1.8655210733,-1.2383139133,0.0829883963,0.3803620636,-0.4085579813,3.0920526981,-0.8149982691,0.2523723245,-0.3126744330,0.2925846577,1.2427134514,-0.3900679052,-2.2280981541,-2.0701429844,-0.5498985648,-0.3402158022,-2.2202494144,1.0159556866,0.8057820797,1.4112379551,0.4200439751,-1.3517037630,-0.4663037062,0.8230061531,1.4041137695,0.0515968874,-0.3207317591,-0.9438809156,1.0704278946,0.8321710229,-0.0838056281,0.1407217234,-0.4895201027,-0.1014306247,1.3761303425,1.7014065981,-0.2475039661,1.0797243118,0.2581546605,-0.0555630252,-0.7183734179,-0.4093923867,-0.4314723313,1.6992679834,-0.5810607672,-0.4299448729,-0.1757195890,0.4817943871,0.8706088662,1.1988463402,-2.6215486526,-2.0343215466,0.4191960394,1.6816884279,0.2876927257,0.9413163066,0.7395244837,0.4954246581,-1.6228507757,-0.6659519076,0.6042791605,-0.3217880428,0.1207242757,-0.2647857964,0.2765117884,-0.1759828329,1.1069335938,0.4527967274,1.9177540541,-1.4181673527,-0.5517150760,1.4401162863,1.2062914371,0.1064064354,1.1001257896,0.5693667531,-0.0823822916,-0.8558875322,-0.3975574374,0.8107486963,-0.7338398695,-1.6810228825,0.2814816236,2.0815069675,-0.8396160007,0.7340915203,-0.1046710759,2.3806743622,1.1243662834,-0.7335973978,1.3209965229,-0.0835166797,-0.4221691787,-0.2683244646,-0.2047377080,-0.8506658077,0.3369838893,-1.3512299061,0.5600090027,0.4778011739,-0.9195017815,1.3022620678,0.0175112654,0.5426177382,0.1755894125,1.0299211740,0.7771252394],[-1.3532475233,-0.3722300828,-0.1029285714,0.7554801106,0.5151497126,0.3328151405,0.2304541320,0.1953039616,-0.5153769851,0.4043092728,0.1988413036,-1.2689567804,0.4159680605,0.3145648837,1.0199477673,-1.7042839527,0.8625801802,-2.2885460854,0.4905623198,-0.5468719602,-1.4397684336,-0.2191286832,-0.3316198587,-0.0544967875,2.1939680576,-1.3659399748,-0.3305815756,-0.7713046670,0.9388977885,-0.7135739326,0.1881475300,0.7682553530,-1.1076250076,2.2674283981,-1.1733065844,-0.0992131382,-0.0065833130,-0.7474772334,-2.5292699337,1.3511906862,-1.0686225891,2.1275341511,-0.7644865513,0.7966610789,-0.0352354646,1.6254175901,-0.4221188426,1.3619796038,0.6780332923,0.3214618266,1.1740883589,0.0146083022,-0.9775192738,-0.8737570047,-0.3637194037,-2.3226566315,1.0262193680,0.3523386419,-0.8140775561,-1.0497467518,0.1893426478,-1.9934339523,-0.4920479059,0.0709401369,-0.2049642354,1.6265887022,1.4623658657,-0.3380526900,0.6974494457,-0.6214071512,0.6323260069,-0.1815361679,-0.2832997441,0.4412190616,1.0376160145,-0.0181217846,0.2928813994,-1.3679971695,0.6803232431,1.3326506615,0.5246801376,-0.4922191799,-1.0667096376,0.5750662684,-1.2777228355,0.4103414118,0.0826276764,-0.0194279980,1.1380794048,-0.7974613309,-0.1227394193,-0.4622454047,-0.9462473392,-1.0382105112,3.0540573597,0.3594521284,0.5559291244,0.7019115686,-0.1078845412,-0.8671279550,0.3546288610,0.7513562441,0.1686596423,-0.2308986932,-0.6836241484,-0.9934930205,0.0928095505,-0.4998281896,1.3151831627,0.7407268286,0.5519161224,-1.9649449587,1.5386253595,1.6443663836,1.4250090122,-0.4822165370,-1.0676637888,0.3187181652,0.4153170884,-0.8883006573,-0.2742860317,0.2505654395,0.3850674331,1.8112043142,-0.6003881693,-1.2677028179,-0.0657587349,0.5034014583,-0.1584416926,1.0296255350,0.7896546721,-0.1844998002,-0.8324058652,0.7251090407,0.3456537127,-0.1612031162,0.5519640446,-1.2285329103,0.4051709771,1.7382738590,-0.7669152617,1.3024536371,-0.2334417999,-0.8495087028,0.0963180736,-1.4320399761,-0.4009811580,-0.0825446695,-1.6872545481,0.3303807080,1.4178494215,-0.7337759733,0.9069579244,-0.9080467820,-1.1837409735,-1.2833039761,-0.7491318583,2.5821473598,-0.5403444767,-0.4830243587,-0.2678615451,0.8788146377,-0.0934342444,-0.6283229589,-0.1921992749,-1.9643006325,0.2488131225,-1.1531807184,0.3248987794,-0.2471186668,-0.7058179975,0.5422291160,1.1079134941,1.4611216784,0.8116730452,-0.0140767116,0.4220570624,-2.0194842815,-1.1250903606,-0.9013854265,-0.1820059419,0.3428933024,-1.2460854053,1.4754422903,0.9102709889,-0.7721157074,-0.0860439911,1.6187556982,0.5236514211,0.6717063189,0.8059553504,-2.1456305981,1.8775573969,-0.8045200706,-0.7242151499,-1.2274599075,-0.3879765868,0.0832767859,-0.9325447679,-0.5355188251,0.7970010042,-1.0122157335,-0.2169611752,-0.4314942956,-0.7536115646,0.9737334847,0.5086837411,-1.4131776094,0.0424655750,-0.5832741857,0.7751629353,0.6793202758,0.4981907010,0.6098150611,-1.0555416346,-0.4501825869,1.2009423971,-1.1430513859,1.2606869936,-0.2908736765,-0.3532292247,0.2760093510,0.0486681461,-0.9852678180],[-0.5179210901,2.9281203747,1.8839042187,-1.2333828211,-0.5978958011,1.6062750816,-0.3856277764,0.3353347480,0.9008058310,-0.4921723604,0.4860668480,0.1493294090,-0.3788942695,0.3776074946,-0.1087439433,0.8221364617,-0.0039486336,0.0350735076,-0.7033908367,-0.3409045637,1.2027546167,0.0638573021,-0.4985602200,-0.3220447898,0.8035058379,0.0312587582,-1.0049045086,0.0859072581,0.4213287532,1.3841396570,-1.9018831253,0.2462813258,0.3927054703,0.8714232445,-0.5415591598,0.9047995806,1.5702162981,0.5778482556,-0.3848070502,-1.4516043663,-0.1781129241,1.0578336716,0.1071197763,0.4987775087,0.4444977641,0.1904703826,-0.0500574149,0.8652390838,-1.3767344952,0.6599958539,-0.1546868533,-0.4128006697,0.4206206501,0.7529597878,1.6279267073,1.6883649826,0.3809396625,1.4642118216,0.0701802224,0.9007957578,-1.0695656538,0.0756013691,-0.2558471262,0.6178277135,-0.1640721709,0.5888280869,-0.0450142436,-1.9780269861,1.5108190775,0.0201296490,0.3490985334,-0.8458254933,-2.2395009995,-0.1078830585,-1.2979288101,2.2681913376,1.1489452124,0.5595183969,0.6483691335,-0.8494003415,-1.2000247240,-1.7947679758,-0.3354310393,-0.8372511864,0.4137484133,0.4951325357,-1.3353157043,-0.4037382603,0.1360496730,1.7940540314,0.5145080090,-1.1026799679,0.3898757398,-1.3311768770,0.6445837617,3.1182668209,0.7394161224,0.5409551263,1.4241337776,-0.4103944302,-0.9392827153,-1.0108066797,-0.9068341255,0.0709434673,0.9242113829,-0.2061729133,-0.5818979144,1.0988814831,-1.0460087061,-1.2680871487,0.4011656642,0.7063562274,0.5195142031,0.1841870248,0.4354915619,-1.1544827223,0.7519261837,-0.8682871461,1.2139902115,1.8845286369,-0.3824562132,-2.9315984249,1.0975605249,-0.6656762958,-0.3710460961,-0.4581338763,-0.1719742119,0.6240089536,1.4918496609,-1.3402218819,0.5351183414,0.3458021283,0.4096876383,0.6907798052,1.3452779055,-1.4073811769,-0.7307488918,0.4452004731,0.9288562536,-1.3342907429,-0.9971263409,-0.2311260700,-1.9997789860,-1.1380411386,0.5919696689,-1.6074502468,-0.5450325012,0.1557483375,0.4483186901,0.7535129189,0.3043299913,0.7550746799,-0.7720427513,-1.0777661800,1.1778817177,-1.8641024828,0.5344553590,-0.0597600453,-0.5732216239,0.9112659693,0.1287456751,0.8489707112,0.4421401024,0.6027107835,-0.1285692751,-2.4745180607,0.1951732039,0.7277007699,-1.4732520580,-0.4286610782,-0.1154345125,-0.0769550353,-1.2591086626,-0.1017499194,-0.0210780762,-1.1737729311,-1.1453151703,0.3725708425,0.8576273322,-0.4034656584,0.1232902855,0.4509639442,0.3320712149,0.2331693470,-0.6623501778,1.1165158749,0.3908354342,-0.2755193710,0.7766165137,2.2021074295,0.7839115262,0.1298831701,-0.8191106915,-1.3449116945,-0.2769561410,0.0589544252,1.4789159298,0.6346148849,-1.5675116777,-1.2349410057,0.0871513784,0.4487423897,-0.2801265121,0.8739984035,-0.6375115514,0.2110447139,-0.3336372674,0.9330590367,-0.5646930933,0.5073488355,-0.0384628698,-0.3513286114,-0.8750590682,-0.9063674808,-0.3104023635,0.4549559355,0.0239758920,-0.5857896805,0.0329510607,0.6853071451,-0.9402374029,-1.0898227692,1.1488786936,0.3411648273],[0.7200110555,0.1597515345,-0.3885217905,1.3693848848,-0.3649668992,1.1565055847,0.0767926425,-0.2566693127,0.4097830355,-0.1426362097,0.5309285522,-1.0245457888,-0.0216459669,0.5516811013,0.1327580810,-0.2019371092,2.0907456875,-0.6933072209,-2.1148211956,-2.0299284458,0.3427873254,-0.0116709834,0.4638992548,0.6261547208,0.5721890330,2.9009780884,-1.1405460835,0.1877879500,-0.1944549531,-0.1844120920,-0.5303893089,-0.4590159655,-0.1941795200,-0.1404060721,-0.7878784537,-1.3694969416,-0.0244819149,-2.6898519993,-1.0086953640,1.1471830606,0.5464369059,0.9971829057,-0.0929342508,0.7358595729,-0.2085779309,-0.9564427137,0.2623963654,-0.7120863795,-1.0331848860,0.2814248204,0.2566384077,2.1200895309,-0.7847632170,-0.9394590259,2.1373596191,0.4847322106,-1.0768883228,0.0257185530,-0.3627569079,1.1377524137,-2.3272347450,-0.7009713054,-0.3798924983,0.5566964746,0.2665447593,0.2588733137,-0.4382838905,-1.5288709402,0.7454527020,-2.0549416542,-0.7336265445,0.7902216911,1.0842144489,0.8565483689,-0.5160650611,-0.2388527393,0.1325882673,1.2349766493,-0.5192912817,0.6799807549,0.3111981750,-0.1828193814,-0.6931616664,1.0630458593,-0.3297859728,0.3573651314,0.1918357015,1.0749788284,0.8603224158,-0.5975485444,-1.3843562603,0.9666813612,0.2949757576,-1.0964585543,0.9764108658,0.0149889840,1.0665789843,-0.8470121622,0.8645345569,1.3421913385,0.7372649908,-1.0292098522,-0.7991414666,2.0994555950,-0.0386573002,-0.1209427640,-0.1018707603,-1.8904802799,0.2138135731,0.5829991698,0.1209255829,0.2682153881,0.1597340852,2.0694198608,-0.8160514235,0.9405336976,-0.2612581849,-0.7488277555,0.2938060462,-0.4454466403,2.0709934235,1.0594415665,-2.0218315125,0.1861803085,-0.3184541464,1.0500028133,-0.7971460223,-0.4653572142,-0.1952562183,-0.3340694904,-1.5124847889,0.8677157164,0.1723863482,1.2384344339,-0.1841048002,-1.2000898123,0.4072071910,1.3923242092,-0.7195712328,-0.9771297574,0.7300099134,0.0627533272,1.9138554335,-0.0584848188,0.8246739507,-0.4379732609,1.9184089899,-1.5187605619,-0.8122234941,0.8544716239,-0.3368932903,0.4989197552,-0.6576798558,-0.5772010684,1.0729271173,0.8953892589,0.4057393372,1.2137151957,-0.8393722177,0.0525206178,1.3459855318,2.1250915527,0.1242673770,-2.0543036461,-0.7189696431,-1.6973289251,-1.5774748325,-0.0314979739,-0.2412691116,0.2239166349,-0.5194889903,1.0742644072,0.1609522700,1.0746810436,0.1200929657,-0.2726342082,1.4211553335,0.0857824907,-0.2053765953,0.3187888861,0.5143779516,-0.5992152095,-0.6816874743,2.9358263016,2.0151743889,0.5622567534,-0.0935387388,0.1492368132,0.5007673502,0.5098925233,1.1149957180,2.1785109043,0.6689604521,-0.9561447501,-0.7765575051,-1.2729759216,-2.5599219799,1.2151373625,0.5340461135,-0.6273633838,-1.3690152168,-0.7996125221,-0.2076777518,0.5959313512,0.0312181860,-2.0051684380,1.3749988079,0.5573381186,-1.8676750660,-1.1792148352,-0.4203329086,0.0985771790,0.6573711634,1.1733105183,0.3345836997,-1.2246304750,-0.5836922526,0.4716092944,0.4792221189,-1.2958402634,1.8630772829,0.2784361243,1.0937455893,-0.2099616975],[0.4214741886,-0.5144574046,-0.2383110076,1.2241157293,1.1667163372,-3.0629582405,0.2755071521,0.9796141982,1.1982212067,-0.7435212135,0.0888084620,-1.1646779776,-0.8349477053,0.1836696565,1.0464693308,1.4091336727,0.3512982130,-1.1093584299,-0.0148777813,-0.7249640226,0.5148887038,-1.1883825064,0.7954053283,0.0648263469,-0.3492616117,0.6866639853,0.1444717646,1.5906834602,1.1906172037,1.1974231005,-0.2618243992,-0.7974213362,-0.7960732579,-0.3851014674,-0.9355871081,-0.6335410476,0.1873655766,1.1265091896,-0.0838515311,0.1801059991,1.3423806429,0.8518787622,-0.6987745166,-1.1183056831,0.0339035653,1.2098129988,1.0811673403,-0.1526868492,0.5863786340,-0.6288794875,-0.2684370577,-0.1966408342,-1.8438454866,-0.7123699784,-1.7283849716,-0.9004950523,0.4246796668,-0.3308639824,-0.0020428838,1.0246882439,-0.8577266932,0.6733812094,-0.9730071425,0.5584130287,0.3536276817,0.5739370584,0.5155097246,-0.0672519654,0.1801947504,0.1836582124,0.5291321278,0.5086742640,0.0655369684,-0.8849287629,0.6907548904,-0.0488187782,3.3209657669,-0.0501876213,0.8470934629,0.1091856286,0.5762174726,0.1049321070,0.4455679953,-0.8758905530,1.8857425451,0.7774903774,-0.0076641762,0.8670355678,-0.3381590247,-0.8557353020,-2.0405881405,-0.2047748268,0.5438323617,-0.9618906379,0.7381688356,0.3258624077,1.4617965221,-0.0028879009,-0.2676487267,-0.8923782706,-1.8217835426,-0.0186384395,1.7530089617,-1.1791179180,-0.4352858365,0.9670229554,-0.3449574411,-0.3940675259,0.4946045578,0.2536029518,-1.0679191351,-1.2482924461,-0.1159172207,0.8719373941,-0.3312026560,-0.3494094908,0.6839279532,-0.5084854960,0.7218002677,-0.2618828714,0.3507729471,0.2381303906,0.7938268185,0.7570793033,1.0622991323,0.6175446510,-0.9916316271,1.6769894361,-0.3215295076,-1.0892727375,-1.4173982143,-0.5432204604,-0.2167730033,1.6159602404,-2.2631447315,0.9030470252,0.6519251466,-2.4697923660,-0.8317915797,0.0972767100,-1.2725318670,0.6019590497,0.0795885250,-1.1497843266,-2.4003005028,-0.9797340035,-1.4273698330,0.6937153935,-1.8866250515,-1.2132887840,0.2422034889,0.9020537734,-0.6387479305,-1.2040439844,-2.5717265606,-0.9823888540,-0.8603173494,-0.1113521829,-1.5185772181,0.3711701632,0.7167190313,-0.2066891938,-0.6412582994,-0.9086228609,1.2789639235,-0.5552526116,-0.2739999890,0.6681884527,0.2850576937,1.1664259434,-1.3593558073,0.3325866759,-0.2983123660,-0.1339031905,1.3430116177,-0.1880369335,1.9697154760,-0.1896455884,1.6410228014,-1.5998033285,0.4156921506,0.5480934978,-2.6319608688,0.3182291687,-0.4545027018,-0.0404811949,0.5020309687,0.4129009247,-0.9264763594,-0.8234125376,0.6390948296,-0.0019334590,-0.5494630337,-1.3261779547,-0.7078437209,-1.0352882147,0.8757877350,0.3657563031,1.5296053886,0.6850796938,1.3457660675,-1.2615861893,0.0619928725,1.7164348364,1.1516953707,1.2520174980,-0.0763857737,-0.9964673519,-0.6538204551,-0.7642871737,-2.7673952579,0.0269780625,-3.5376372337,0.2615314126,-1.8532675505,-0.3508562446,0.3244189620,0.4992468953,0.2832391262,-0.8766116500,0.3057894111,1.6057144403,2.1141822338,-1.0781127214],[-0.9287056923,-0.8534093499,-0.2685569525,1.3060340881,0.1535290927,-1.2112205029,-0.4509466588,1.3873354197,-2.5044455528,0.5372287631,-0.6981587410,-0.4939994812,-0.2896173000,-0.2453738302,0.3176817298,1.4238258600,0.7012366652,0.2226518840,0.8875376582,-0.2895725965,-0.5405652523,-0.5801152587,-0.6099621058,-0.1794458330,0.3522992730,-0.3782028258,0.5201066732,0.7829122543,0.1191232353,0.1265148669,-1.2816922665,0.5576660633,-1.7459415197,0.2186397910,-0.5908021331,0.9046570659,-0.6815707088,0.2523507774,-0.1003115773,-0.6361647248,-0.2780383527,0.8642255068,0.9515980482,0.9820371270,-1.4303667545,-0.0983356610,1.4673168659,1.1516301632,1.9530824423,1.2597833872,-1.4902018309,0.3255969882,0.7496146560,-0.0849756375,-1.0007236004,-0.5960429311,-1.0154364109,1.2511830330,1.2626070976,-0.5875334144,1.4028255939,0.4695758522,-0.0255566500,1.8417292833,-0.0399172194,1.1069836617,-1.2826505899,-0.1472184211,0.9856489897,-0.1729861200,-0.3876519799,0.6959972382,0.2856332362,-1.0207430124,-0.9289630055,0.0632748380,0.4820849895,1.5037152767,0.4867504835,0.0797959343,-0.1955450326,0.7896358371,-1.1532626152,-1.1208647490,-1.6553353071,1.8035339117,-0.0838049203,1.3239436150,-0.4971853793,0.8107408881,-1.1551069021,-0.1953871846,-0.7771847248,-0.7813466191,0.5065687299,-0.1820823550,0.2684529126,0.9292242527,0.7593740225,-0.9009737968,0.4765752554,0.1643548906,-0.3488487899,-0.3658882976,-0.2314924598,0.4069265723,-0.9488435984,0.7248171568,1.4139008522,1.5861266851,-0.1279027164,-0.0553276762,0.7255465984,0.1429184526,0.1820705831,0.2178043425,-0.0077814292,1.4773465395,-1.1627372503,-0.6053957343,0.4770019352,-1.0118191242,-1.3295882940,0.7362958193,0.7915869355,0.3331216276,0.5359749794,-3.0860996246,0.3936527073,-0.7873288989,0.1485450566,-1.2330771685,-1.2370146513,-0.5632748008,-0.4914303720,1.0046528578,-0.7574470639,1.2790346146,-1.9569309950,-0.6932571530,0.3972877562,-0.1010534316,1.6112462282,-1.3199983835,-0.5487880111,-0.5936339498,-2.0641529560,0.5989998579,0.8124501705,0.3341778815,-0.7117627859,-1.7005486488,-0.7905029655,-0.7358307242,-0.0822679624,-0.2129524499,-0.0102335811,0.2852599919,-0.2062803209,-0.5369126797,0.9724613428,-0.3041396439,-1.0042915344,1.0672525167,-0.2289439440,1.3452187777,0.5357630253,-0.4374930859,0.6523374319,0.0650269166,1.2034343481,2.5041522980,-2.2119021416,-0.0325773247,-0.2486386299,-1.2621959448,0.9550855756,-1.0251768827,1.3576012850,-1.0082174540,0.4790438414,0.7539923787,-0.2292823493,-2.0603711605,1.2167100906,-0.0632605180,0.6282228827,-1.2875719070,1.2758817673,-0.0361953229,-1.1444544792,0.0995380729,2.0643885136,0.0476284251,0.3599074483,1.3913724422,-0.8276139498,0.4267857373,-0.9940422773,-1.6722874641,-0.4820932746,0.9585770369,-0.3583551347,2.2475891113,0.6795741320,0.0427838154,-0.0337097496,0.2812221050,2.2352130413,1.5787955523,-0.0525405966,0.7612043023,-0.2395873815,-1.1629893780,0.9450313449,-1.5587869883,-1.0885256529,1.2096309662,-0.6176126003,0.2794592083,0.4000840187,-0.2106546909,2.1949429512,-1.7044574022],[-0.2265886366,0.5426079631,1.1125903130,-0.5519072413,-1.1224460602,0.8363450170,-1.5743186474,-0.5271378160,0.6508792639,0.5543196201,-1.1395976543,0.1506068110,-1.3732875586,-0.5603883266,-0.3320862949,0.0500928350,1.4821004868,1.3670789003,1.0397155285,0.9093261957,0.2789103091,-0.4556762576,0.9363845587,0.5859700441,-0.8330316544,0.9558687806,-0.2014526874,-0.0478864685,-0.8698826432,0.2509627938,0.1928938031,-0.5245307088,-0.7604442835,-0.9027975798,-0.8322229981,-0.2535308897,0.2865951061,-0.8358671069,-0.0175675061,-0.4595412612,0.2229304463,0.2540488839,0.1446684897,-0.5048643947,1.2162379026,1.4006818533,-1.8685104847,-2.1495907307,0.3018081784,-0.9322087169,-0.6951150894,-0.3476719856,-0.8317773938,0.5383685827,-1.4129815102,-0.1823303849,1.0443432331,1.1775488853,0.0203446262,0.6534597278,0.1316889226,0.9037830830,-0.8564657569,-0.2451224774,0.5288985372,-0.3417075574,-0.4940801263,1.2461600304,0.1253131926,-0.1442823708,-0.2593114078,-0.7173941135,1.5848474503,1.3283705711,-0.4759116769,0.5617510080,-0.2803081572,-0.1529893428,0.7441819906,1.0778692961,-1.6746370792,-0.0191578642,-1.6233841181,1.3962148428,2.4005887508,-0.7849439979,1.6532175541,0.8943389058,-0.7218570709,0.0025848437,0.1503669173,-1.1533824205,-0.6082597375,0.5895243287,0.6123408675,1.0880017281,-0.8497795463,0.2664059699,0.1507609338,-1.1018321514,-0.7293382287,-1.2999854088,-0.5079394579,1.9566550255,-0.6045329571,-1.1847981215,1.3479480743,-0.4842537940,0.4533431828,0.2714575231,-0.0489397943,-0.0387133919,-1.2142137289,-0.5738778710,0.4517196119,-0.0616835095,0.7554846406,-1.2030466795,0.2224065214,0.8758687377,-1.5098052025,-2.5789873600,-1.3044232130,-0.9159380198,0.3439801037,1.0308520794,-0.5655797124,0.6416946054,-1.5301198959,-0.2333142459,1.3705869913,0.5391677022,2.1740953922,0.6560552716,0.4826991856,-1.5273411274,0.6695680618,0.4246776700,-0.1707382351,-1.0786836147,0.3128767908,0.5183787942,0.5231448412,-0.7981930971,0.0068109701,-0.9226770401,-0.7949106693,-0.1485601366,0.5388772488,1.1333990097,0.2188962549,-1.0067234039,-0.3539472222,0.7378255129,-0.2585050464,-1.1470578909,1.2719910145,0.6277831793,-0.6327692270,-1.3171602488,-0.6405596733,-0.4403110743,0.5331921577,-0.0618506931,0.1133509055,-0.2367716134,0.0172743201,-0.9363154769,0.0419243276,1.2715209723,-0.6324470639,0.4981867671,-1.4744064808,0.9432847500,-1.3852499723,-0.0248926673,-0.1987091005,0.9218626022,0.3695135117,0.6386186481,0.9051753283,-0.6882691979,0.1940152645,1.0115746260,-0.1039159670,0.0934223235,0.6569828391,1.5856434107,1.3781589270,1.1807434559,-1.2077684402,-0.0713830218,-0.6892064810,0.7983953357,-0.1320799440,0.9095534086,1.0258624554,-1.0350824594,-0.1084372476,-0.7068145275,1.0966942310,0.2121248394,1.6011934280,-0.1354372799,-1.6426924467,1.9219787121,1.9771164656,-1.2754851580,0.4517096877,1.3728480339,-0.5387133956,-0.5070499182,1.0105915070,0.7020894289,-1.4419972897,0.0110328682,-0.1098614261,-1.4754793644,0.3179679811,0.3669794202,-0.9281835556,1.2898329496,-0.9199350476,-0.9801771045],[-0.9031108022,-0.3989333808,-0.3263574541,0.1047135815,-2.0553271770,-0.9773147702,-1.4125535488,-2.2337026596,-0.9101406932,-0.2648207545,0.5565056205,-0.6932720542,-0.8804928660,-1.4365410805,0.0302592870,-0.1010962129,-0.3346745968,0.3059233427,-1.3067972660,-0.0088376841,-0.3915492594,-0.6771771908,-0.5156219602,0.1084053442,-0.7851771712,2.0248520374,0.0768793225,1.6671743393,1.1379973888,-0.0037817969,-0.1691045761,0.1110712513,0.0237912405,-0.2556541264,-1.9002453089,-0.7983082533,-1.8317855597,-1.2898895741,0.0141506037,0.2845778465,1.1239186525,0.4390246570,-1.0800429583,0.9424997568,0.0881775469,-1.6444958448,-0.8998572230,-0.9650136828,0.2967081070,-1.0165929794,-0.5798018575,1.3800915480,0.6454607844,-0.0140456948,-0.5817818642,-0.1011662409,1.0112067461,-1.8833494186,1.3017792702,-0.7716539502,0.5666996241,0.2956871092,-0.2049203664,1.9108538628,-1.3607987165,-0.0717590377,0.0956634954,-0.5079314113,0.3618172109,-0.3595963717,-1.4459455013,0.2162457108,-0.7750922441,-0.2253048420,-0.6305879951,-0.6282056570,0.1342151314,0.9079183340,-0.3357201517,-0.9684320092,-2.0011854172,1.0298330784,-1.1672452688,1.2127306461,-0.5632342100,-1.1428827047,2.3399078846,-0.8843747973,0.5320113897,-1.1435959339,-0.8313342929,1.1637715101,0.8253864646,-0.9990708232,0.3957966864,0.4645176828,-0.6763690710,-0.2837693691,-0.4347972870,-0.7891721129,0.8290168047,-0.3206014335,-0.3041936457,0.7272228599,1.2169837952,0.7897850275,1.2175595760,0.9864512682,0.3498557806,0.9575622678,0.1763862520,1.5062904358,-1.4601914883,0.8869418502,0.4095101357,-0.0545322858,-1.0357341766,0.0918669775,0.7407202721,0.4786416292,0.6654282212,0.2292959541,0.2076932490,0.2183427811,-0.7750036716,-0.8375609517,-0.8351103663,-0.9253761768,0.0818053335,0.2907602191,1.0790300369,3.0961189270,-0.4322378933,-0.5297884345,-0.7185241580,-0.0790990219,-0.5318148136,0.3011080623,-2.3876340389,-0.6010844707,-0.6294558644,0.9789693952,-2.1652061939,-0.9091853499,-1.5246109962,1.6593085527,-1.1628981829,1.6147589684,2.2718012333,-1.5532001257,-2.0527155399,-0.4091469049,-1.0820819139,1.5153158903,-0.4930484593,0.8512840271,-1.1784952879,1.0569536686,0.0160085466,-0.0003073825,-1.4744670391,-1.2892800570,-1.2932577133,0.6164194345,-1.7043364048,0.2677577436,0.6191200018,-0.2715763450,-1.5342735052,-1.0618538857,-1.2137271166,-1.9104957581,-0.5562978387,2.0095219612,0.4219472408,0.2807143331,0.2929550111,0.0925148204,-0.7408627868,-1.2786219120,0.5953165293,1.1873878241,0.7985800505,1.1434313059,0.1420213878,0.3704443574,2.0830335617,0.8558987975,-1.0863219500,-0.6328001618,-1.0930259228,-0.7155646086,0.2989422679,-0.1714941859,-0.2596023083,1.0251544714,0.4840430021,0.5426319242,-0.0317259394,-0.5928928256,2.3365657330,0.8119526505,-0.3559478223,1.2486777306,0.3161210120,0.3295256793,-0.2986513674,-0.8083889484,-0.1058205962,-0.2012898922,0.1898895204,1.2744652033,-0.5763427615,-1.7786310911,-0.4619722664,-0.0343275331,-0.0804154128,0.0896555558,-0.2375993878,-0.6510353088,0.5634438992,-0.2756451070,-1.1919237375,0.0106554357],[1.0845719576,-1.4242180586,-0.7915678620,-0.4528565705,-0.5867254734,-0.3834092915,1.3105416298,-1.1719135046,-1.8663834333,-0.4044742584,-1.0571340322,2.1624724865,-0.3283438981,-1.6711210012,-0.2128783464,-1.1012598276,1.2401621342,0.1001905724,-1.6413034201,-0.4412343502,-1.6417485476,0.5818608403,1.0299129486,2.2170233727,-0.7258693576,0.3411789238,-0.0758841783,-0.1068310887,-1.5048621893,-0.7476377487,-1.0252985954,1.2971277237,0.9907476306,-1.0029420853,-0.8703005910,1.2154511213,0.1695734859,-0.6100532413,-0.2701125443,-1.8159172535,0.0744715631,1.4320803881,0.8319642544,-0.3548873365,-0.4736134112,1.2513190508,0.7541353106,-0.7968253493,-1.1330760717,-0.4850624204,1.3151828051,-1.8339599371,1.2641755342,0.6874196529,-0.1858444214,0.2456031442,1.2378497124,0.3065797091,0.1393100023,0.8186387420,0.5621598959,-0.2971632183,-0.9280245304,0.5608353615,-0.5151640177,1.3092679977,-0.1564584374,0.0101362988,-0.1147716045,-0.3308030069,0.5426612496,1.0478041172,-0.7588689923,0.9058098793,-0.6638476849,0.0271008201,-0.5996529460,0.9415428638,0.3311151862,-0.6850024462,-1.6973967552,0.8252288699,0.5447505116,-0.5340859294,-1.0320228338,-1.9213482141,-0.3236763477,-1.1210227013,-2.0754590034,0.9175598621,-0.1582947224,0.0355038270,-1.8584797382,0.8211516738,0.5859157443,0.9854813814,1.0377193689,0.8087520003,0.0553114042,-1.1089491844,0.2617270350,0.7748783827,-1.3112766743,-0.2853017449,-0.8751548529,0.4537158906,-0.4724691808,1.9114776850,1.1201875210,-0.6998970509,-2.2755661011,0.6124466658,-0.4858125746,0.1044009998,0.6781061888,1.0112067461,-1.0053843260,0.5832854509,1.3902379274,-0.3954279721,1.4138845205,-1.1117755175,0.0675625056,-0.7510002851,0.1992310286,0.1564755589,1.4238003492,1.1698158979,-2.0644772053,-0.7029905319,0.2314308137,0.2684699595,0.7005752325,-0.1916425675,0.1933514178,-0.3600777090,-0.5549749136,-0.6029216051,-0.7787041664,-2.5662279129,-0.7273243666,0.6202687621,-0.7350562215,0.2137556970,1.7911796570,0.5588005781,0.0419100970,-0.6966994405,1.0583099127,-2.8149473667,-0.2643944621,-2.2827417850,-0.7935805917,-1.1352584362,-0.2278170884,-0.8056265712,0.7685412169,0.3496689200,-0.3449663222,0.3916354775,0.0579748861,-1.4494156837,-0.1893656999,-1.6724202633,1.0861943960,0.5076280832,-0.0133326286,-1.2809752226,1.3494324684,-1.2507998943,0.7153228521,0.2078970075,-0.6573593020,0.7824165821,-0.5148826241,1.3713448048,-0.0101848487,-1.4278615713,0.6900991797,-0.7380038500,1.6913758516,1.6196644306,0.9960871935,-0.6944152117,1.6037415266,1.3314589262,0.2126102298,0.8190729618,-0.3324639797,-0.4693499804,-1.2738293409,-1.9276175499,1.8065828085,0.6967000365,0.1457846165,-0.8485788107,-0.0902344882,-0.4437417090,-0.8114491701,0.6733375788,-1.0148701668,1.2667295933,-0.2555393577,1.0527654886,0.0487716459,-1.0939552784,1.0991251469,-0.9288706183,0.6616157889,-0.1064211130,0.5533397794,0.4668231606,-1.0746521950,0.5423960686,1.0720642805,0.8632643223,0.1105926335,-0.4526108205,-1.0977783203,-0.8176969886,-0.2230648398,-1.1865390539,-1.0723834038,-1.0440925360],[0.9188759327,-0.1689279526,0.3107523918,-0.1185576096,0.3864873350,0.5850405693,-1.3451581001,1.1591099501,-1.7213473320,-1.1616033316,-0.3630917370,0.9260309935,1.9034808874,0.7039475441,0.8226540685,-0.2596986592,1.2070049047,-0.9120343924,-0.1701831520,-2.9533367157,1.2373387814,0.7368846536,-1.1026964188,0.7084195614,-0.5575756431,0.6083027720,-1.9660063982,0.0480993725,1.7596119642,-0.2430192381,0.5924292803,-2.4542522430,-0.1702857912,-0.8102533817,0.3578571081,-0.3168414533,-0.7957355976,0.8105921149,1.2186963558,0.2567883730,0.9143864512,-0.0124621931,2.1438231468,-0.7441107631,-1.0311625004,-1.0583885908,-0.3230136037,-1.2296295166,-0.5679237843,-2.8217861652,1.2539693117,-0.6278217435,0.2693663836,0.5483108163,-0.1207809523,-1.0742874146,-0.6069313288,1.5203852654,-2.7174751759,0.3162423968,-0.9960474372,1.1332710981,1.0375409126,-0.0346815698,-0.2378142923,-1.8274402618,-0.7565240860,0.3035017252,0.6736583114,-0.3378650248,1.4695996046,0.8090537786,-1.8708773851,0.9925187230,0.3028483391,1.4664566517,-0.6074188352,1.0674394369,0.1149998233,-0.0621099845,-0.1732854992,1.2566143274,0.4596324861,0.1921480447,0.6472542882,-0.0070908745,-1.1495875120,-1.5454713106,-0.8810761571,-0.9932910800,0.4973982275,-0.0830497742,1.3150237799,-0.8044264317,0.3923761845,-0.8494360447,-0.5066404343,0.4646987021,0.1593044102,-0.0949085504,0.2508442402,0.8306478858,1.5014774799,0.6009095907,-0.2514753044,0.5792323351,3.3733355999,-1.7200666666,1.0272295475,-0.0590384156,-0.4829144776,-0.7770543694,1.1213577986,0.2282679379,0.3859090507,-1.0854517221,1.3076217175,-0.0168237090,-0.7895203233,-1.0785923004,0.0915632099,0.9873991013,0.6320911050,-0.6186110973,0.6905904412,0.6880133152,1.5512328148,0.9121033549,0.5840374827,-0.4260689318,1.5989313126,1.5102443695,2.8731038570,0.1307470798,-0.0351151600,0.4154373407,1.0803951025,-0.7689779401,0.2349200249,0.2127694488,0.1637901515,0.2540091574,-0.5789597631,-0.2085085064,-1.4617660046,-1.2019745111,1.3285419941,-0.5355551839,-1.0052437782,0.8944669366,0.6750456095,-0.3523706496,0.5630317926,-0.2512718141,1.2605104446,-0.1540245712,-0.7847375274,-0.5049604177,0.5393610001,-0.1454569995,-1.9208295345,0.1988556087,-0.5276432037,-1.9898313284,1.3303182125,1.1966954470,0.8466907740,-1.3549520969,0.1534716338,-1.6951408386,-1.0888959169,-0.7846040130,0.0208831783,-0.5135362148,0.1377027631,-1.1420041323,-1.0348410606,1.0945382118,-1.6735677719,0.8930306435,1.2586365938,-0.4186193049,-0.5548886657,0.7520841360,-1.2721277475,3.2311539650,0.4641966224,-2.0481486320,1.9270210266,0.4723309278,0.3861116469,-0.5980855823,2.4221987724,1.1139601469,0.7150088549,-0.5016779900,-0.2652854025,-0.5569392443,0.5937147141,0.7529249787,0.9008342028,0.4063133895,0.1512439102,1.2978001833,0.2491936237,0.2895989120,0.8108627796,-0.2134086490,1.6224154234,-1.1592159271,0.2234223038,0.1212291792,-0.9138811827,-0.4205074310,1.5794224739,0.1617780775,-1.6059589386,-0.4434491992,0.6146141291,-1.1083737612,-0.3466961682,-0.4387931228,-0.2230413556,-0.1678518355],[-2.6264977455,-0.1727611125,1.0355259180,0.3443305790,0.5022798181,-0.5143413544,-0.4986781478,-0.8272845745,-0.0147204055,-0.5727688074,0.0031921251,0.3674898148,2.0267584324,0.5630730391,1.0627936125,0.2569692135,0.3916804790,-1.0073970556,1.2933391333,0.0424646772,0.5398423076,-0.5044786334,-0.5983367562,0.7175628543,0.4539347291,-0.8410576582,0.2607088089,-0.2576574683,0.1949142963,-0.2623168230,0.6356629133,-0.1043076217,0.1324877888,-1.0602743626,-0.2008478791,1.2160608768,-0.0292315781,-0.1890411377,-0.8686054349,-0.9382474422,-1.0121945143,-1.4328993559,1.1785588264,1.5638674498,0.1427608132,2.0439651012,0.9626225233,0.2164034098,-0.3151763380,-0.4807561934,1.9710719585,-0.4390293062,1.7081115246,-0.3730249107,-0.8752344847,1.0402947664,1.0650632381,2.2234606743,-0.2990089655,0.1338202506,-0.0058257026,0.8717163801,0.4131239653,1.7113682032,-0.3095157444,0.4589198530,-0.6647123694,-1.9829771519,0.2398172468,0.7303733230,0.0065799267,-0.2386463732,0.7785472274,0.1026007161,-0.9697332978,0.9232811332,-0.3040901721,-0.3353936076,0.4633235931,-0.3067607284,-0.1129981503,0.0151031548,1.4305889606,0.2822506726,-0.3901175261,0.2373402417,-0.3301363885,-0.4907036722,0.3101215363,1.6566424370,-0.1352761388,2.0479047298,-0.8596891165,1.3939336538,0.5532745719,-0.1697778255,1.0746655464,-0.6602277756,0.4712713957,0.8705766201,0.2959489226,-0.1473409534,-1.7874796391,0.4816951752,0.0301115476,-1.0984210968,0.3432916701,-0.6739798784,-0.8314334154,-0.0407379083,0.4170181155,-1.1367658377,0.7947146297,1.3205595016,0.4236109555,1.4152296782,-1.2873961926,0.5734673738,-0.1166800484,-0.4258343875,0.7109622359,0.9978282452,-0.7153664827,-0.1323070973,-0.1627074182,1.2122015953,0.7887623906,-3.2201621532,-0.1679748148,1.3945446014,0.7399389148,-0.2768034637,0.4165839553,0.8265034556,0.9802414775,0.4776431918,2.0166103840,-0.0249875244,0.4829532504,-0.7448346615,-2.0533394814,0.9954663515,0.5559939742,0.1557032168,1.2560248375,-0.0910743177,0.7448199987,1.5151149035,1.0682363510,-0.3936732411,-0.4404679239,-1.2789002657,-0.7345156074,-0.8073781729,-0.2055337727,0.8966993690,-0.0118007520,-0.7772536278,-1.2498084307,-2.0847239494,0.4443413317,-0.9075754285,-0.0707240850,-0.3365614414,1.1858187914,-0.7117362022,0.9870203733,-0.6684255600,1.4725475311,0.1366336048,0.7424795628,1.4361493587,-0.2917276919,-0.3563250899,0.6698901057,0.8398872614,-0.8428919911,0.7355796695,-0.0806103125,-0.4388633668,0.4438080788,-0.6025252342,0.1011375487,0.4766345918,-2.1126976013,-1.3606692553,-1.0401051044,-0.0643365011,1.7286696434,0.7811232209,1.2519133091,0.1582637727,-0.1840040833,-1.0580223799,2.1860694885,0.7420462370,-0.6358695626,0.3461574614,0.0260392353,0.2544353902,2.4137811661,0.5811787248,1.4372696877,-1.1470407248,0.0356767289,-1.9682257175,-1.6825907230,0.7045783997,0.9582905173,0.4860132933,-0.4299276471,0.4619325399,-0.3726094365,-0.4181045592,-0.6713925600,-0.4985078275,-0.6091927290,-0.6080089808,0.3473231494,-0.0035682917,1.2791190147,-0.2127529532,1.0742793083,0.7116464376],[-0.6488238573,0.2859148681,-2.0417239666,0.4539825618,2.0291399956,-0.5095365644,-0.9993411303,-1.1150147915,-0.6259767413,0.1875912845,-0.2886156142,1.0134998560,-1.1804151535,0.1758203655,-1.2308334112,2.5587472916,-0.9636149406,-0.0713531971,1.5979670286,-1.5108559132,0.1449113637,1.1441174746,0.2312457860,0.2274304181,0.1459915936,-0.6081528664,0.0709826797,-0.0125633776,-1.3180460930,-0.3260259330,0.4805007279,-3.2030875683,0.4306159019,-0.5212286115,0.8575561643,-0.8960441351,-0.1932385713,-0.7063098550,-0.9964183569,0.7934882045,-0.4802501500,0.3568688333,-1.0943177938,-1.6923885345,0.7779947519,2.6947827339,-1.0811645985,-1.4731924534,0.0109302318,0.3299140036,-0.1448013633,2.4050495625,0.1636990309,-0.1965565681,1.5348142385,0.2748448253,-0.9842527509,-0.4361257255,-0.5287380815,0.7196028233,0.2204200625,-0.5418440700,1.3916079998,0.7451648712,0.2643483579,0.6969248652,0.2191476822,-0.3153337836,-1.1294671297,0.2236620039,0.8282073736,-1.0530409813,-0.4204666913,-0.2162797153,-0.7607799172,1.0307723284,0.7421389818,0.4817281663,0.3772906065,-0.2255851179,0.2419869155,-0.2462873459,1.2878042459,1.2064702511,1.9481141567,-0.3638069034,-0.2458624840,-0.8163990974,-1.3361914158,0.1759304553,0.8073284626,0.2768845856,-0.1631110460,0.7782697082,0.2213267684,0.6580312848,-0.8218443394,-0.5532718897,-0.5647550225,-1.2183798552,0.5782849789,0.5329989791,-0.6715250611,0.8244873285,1.0194652081,-1.1014153957,1.6967984438,-0.0429516323,-1.0793919563,-1.1583365202,1.1959785223,0.8431320190,-0.4192778170,-0.3327940702,1.9657697678,0.3050765395,0.8305168152,0.9101333618,0.5969888568,-0.2304041535,1.8251065016,-1.1460005045,-0.2217972875,-0.5881155133,-0.4704960585,0.4888032973,1.3412656784,-1.0570495129,0.4420121312,1.2955129147,0.8408915997,0.7732490897,-0.6764742732,-0.0356562771,-0.6429589391,1.8658261299,0.5035269260,1.2822653055,0.9182057381,0.4520080388,-0.5673279762,-0.4225480556,-1.2345860004,0.9104699492,-0.6104401350,0.5325096846,0.2594264150,2.1227545738,0.6246936321,0.3436859250,0.4797151685,0.5576152802,0.4796886742,0.6413477659,0.1015166044,-0.5291261673,0.0093640964,1.1927584410,1.5470068455,0.9478160739,-0.8584091067,1.4589502811,1.0034278631,-2.0495164394,1.0458933115,-0.6270156503,-1.6971756220,-0.4960333407,1.4561544657,1.0882210732,-0.0976122990,-0.7173095345,0.1641712338,-0.6962606907,-1.3123685122,0.7208009362,1.4251564741,-0.9525543451,-1.6627132893,-1.3450037241,0.9273759127,0.3668547571,0.0847202390,-0.8547070026,-0.1450030655,-1.0790885687,0.6394917965,-0.7484422326,-0.0812485516,-2.3238573074,-0.8937191367,-0.2410787642,0.7658015490,-0.1973870993,0.1931478530,-0.5485792756,-0.3950333297,0.3372679055,0.7965800166,-0.1148310974,1.0055104494,-1.7035870552,-1.2542206049,-0.0020835998,-1.0969371796,-0.6635608077,-0.6708840728,-0.1306800544,2.0890042782,0.2768505216,2.3234868050,-0.1149376929,1.7064332962,-0.0357143879,0.0917023197,0.2306758016,-0.8789747953,0.6359471083,1.0145800114,-1.2079186440,0.7884379625,0.5274050832,0.1402386427,0.3174589276],[-0.5563088655,-1.8163865805,1.6633933783,-0.1732443124,0.1822638363,-0.8089761734,0.6185746193,1.4769704342,-0.8918659687,1.0805387497,-2.2777967453,0.0982830897,-0.9784666896,0.1409724802,-0.6910139918,-0.2581627965,-0.3227139115,1.2393940687,1.4970542192,0.3486480415,0.6801899076,-1.2906173468,-0.8694254160,0.6916615367,-0.3949315548,0.4967114031,-0.9639406204,1.0947629213,-0.5677868724,0.1172745824,0.1036742851,0.8766137958,-0.5469971895,0.0912009105,-0.8483621478,-0.1651878506,-0.8982597589,1.8080214262,-0.1094860435,-0.6009634733,-0.3665615022,-1.1674404144,-0.3560737967,0.4168348908,1.2016643286,-0.5615233183,0.6447062492,-0.2535068691,-1.1192104816,1.1256830692,-1.6240452528,0.9639360905,0.4438114166,1.8228126764,1.9016685486,1.3340772390,-0.9280285835,1.7382543087,0.5836077929,2.3370118141,0.5108041763,1.4469654560,0.4597733915,1.0623366833,0.0035629983,0.0866807178,-1.3463780880,1.1956572533,0.3716119528,0.0912752375,0.5642582178,0.7687278390,-0.0072249584,-0.4405217469,1.2104653120,0.0289006699,-0.6262733936,0.2481508404,1.7004103661,-1.0955580473,-0.9726043940,-0.9737624526,0.2870694399,0.5214788914,1.4835518599,-0.2087257504,0.0239505731,0.2233554721,0.2403004915,-0.8426833749,-1.2004982233,0.7131180763,-0.6364238858,2.0007157326,-0.3677787781,0.5677682161,-0.5169160366,0.7849178910,-0.1691215634,0.3505437374,1.2626199722,-0.3145025969,0.0967818573,-1.1018123627,2.0346674919,-0.5044198632,2.0918335915,0.5418912172,0.1253499985,-0.4790270329,0.9885153770,-0.2244774550,-0.7532069683,-0.2590924203,-0.3724360764,-0.5268012881,0.0099415304,0.9852399230,-0.9677956104,0.9610522985,-0.4307636321,0.9771789312,0.4913146794,-0.1979523748,-1.2591580153,0.1725802869,0.3992780149,0.2902160883,0.6774837375,0.1915705055,-0.5545125008,-1.6164081097,-0.4995488822,0.1495799124,0.7770783901,-0.6658815742,-1.3216475248,0.7363036275,0.4968822896,0.5971414447,-0.6678465605,0.1934313029,0.4205310643,0.5923535228,0.0337040164,0.4831328690,-0.3680884242,0.3568235636,-0.3181792498,-0.8712489605,-0.8680101037,-0.5461862683,0.6563332677,-0.3947722018,-0.2092790902,0.0315701514,-0.4167560935,0.0570355020,1.4990587234,-0.8132793903,-0.4818907976,0.4890195429,0.6042798758,1.5244840384,0.4387151301,0.3640931845,1.5731611252,0.5005360842,1.0017102957,-0.4325468838,-0.7100720406,-1.3056461811,0.3504725397,0.2902453542,0.3730077446,-1.8038679361,0.3144704103,1.1459712982,-0.9655853510,0.4122045934,1.5242187977,-0.2994287908,0.0495324992,-0.8841370940,-0.9477353096,-0.5417408943,-0.0676480904,-0.4663005173,0.9185313582,0.0686773732,-0.6691791415,0.3122829497,-1.1253046989,-0.1556600630,0.5729102492,-1.0837713480,0.0669136345,0.4971903265,0.0897533596,-0.1047038808,-0.6622824669,-0.7894172668,0.8324003816,0.7106827497,0.3483260870,-0.7259815335,0.6386246681,-0.5196985602,0.5204224586,0.0865338221,-0.7810475230,-1.2443351746,0.3589504063,0.4397775829,-1.1927919388,0.9172086120,-1.2508444786,1.7453820705,-0.3246877491,-0.4532547295,-0.0731760189,-1.3975434303,-0.7634076476,1.5434905291],[-0.4080433547,0.4650013745,-1.4238005877,0.2563230693,1.7736479044,-0.7137615681,2.2522010803,-1.3089007139,1.1525106430,-0.1405597478,0.5320132971,-0.6810653210,-0.4485176206,-0.3747822344,0.4281465709,0.8007715940,0.0123932073,0.1402721852,1.9548217058,0.3309944570,0.3134448826,-0.8624955416,0.1731562018,-1.1307023764,-1.2180870771,0.3798313737,0.1041512787,-1.0410223007,-0.2304405719,0.0916122720,-1.1846817732,-1.1531682014,-0.4354380965,0.4104143679,-0.4816984534,0.4900153279,0.7041178942,-0.7546210289,0.5832744837,1.2078104019,0.4386845529,-1.5632371902,-0.8126469851,0.1477559507,-0.0461850464,-0.3673082590,-0.4501196742,-0.1551027894,-0.7583582997,-0.0730031282,-1.9887934923,-0.8846876621,0.6732758880,-0.0811193660,1.0592772961,-0.5077518821,-0.3238532543,1.3735461235,-1.4465440512,0.1821355820,0.0676640049,-0.4560262859,-0.3496088684,-0.6075907946,0.6537126303,-1.7022186518,0.0755707473,-0.1904866546,-2.1203536987,1.7095605135,0.7199447155,1.7510561943,0.8169274330,0.8673193455,-1.2237163782,2.6919562817,0.3033113480,0.2355719060,0.4901683927,-0.7814581394,0.8745526671,0.2152803242,1.0555243492,0.3528210223,0.3482930064,-0.0836526528,-0.8183871508,-1.3136165142,0.5067337751,0.9809406996,-0.3727931380,0.1372788996,1.2173398733,1.0570574999,-0.1092743352,0.4346898496,-0.7528158426,0.8488703370,-0.8521622419,-0.8124353290,-2.1803562641,-0.7766513228,1.1283991337,0.3751853406,-1.3530814648,1.3838658333,0.4173292816,0.7445374727,-0.0002430229,-0.9045234919,-2.6463234425,-0.3570845723,0.2908712029,0.3473463953,0.5702397823,0.7549381852,1.4791334867,0.9874882102,-0.6921150684,2.3449747562,0.6544520259,0.3682841957,1.9040485620,0.4765006304,1.1654456854,0.7307538390,-0.4611986578,0.6390597820,0.3741174340,-0.5228999257,1.6194049120,-1.0346127748,-0.5952832699,-0.8493008614,0.8470287919,-0.5518957973,0.2897983789,-0.0187721644,-0.3662812412,-0.3504948616,-0.3711920381,1.7615973949,-0.4802156985,-1.8253751993,-0.1795722097,-0.3333802223,0.4528505504,0.7250844240,-0.1168686971,-0.2354764342,-1.0015610456,0.5932791233,-0.7151992321,1.1872414351,-0.6099489331,-0.8342801929,1.0395519733,0.4641788900,-0.1812096685,-0.2494318336,0.1936044991,-0.3829216361,-1.2802710533,-0.0509898625,-1.1535187960,2.3395931721,0.9372795224,-1.0627675056,-2.2381188869,-0.0003268473,0.1890586317,1.9689356089,0.4381758869,0.6209080815,-0.0069314591,0.6565827131,0.9101958871,-0.8087453246,-1.0782167912,-0.4138877690,-1.1793274879,0.3645130098,2.4339489937,1.2218075991,-0.5169935226,0.3686336875,0.3838019669,1.0012456179,0.0827877894,0.6399840117,-1.1393259764,1.1500494480,0.1755813956,-0.4248540998,-0.9612060785,1.2496811152,0.6430417895,-0.8739003539,-0.6526424289,1.1801552773,-1.3901848793,0.6684985757,0.7047442794,-0.0454604439,0.9043201208,-0.0832923055,-0.0246616453,-0.7082564831,-0.6657384038,1.1625177860,0.7331817746,-0.8065531850,3.0549848080,-0.6254408956,-0.9780409336,0.3719343543,-0.7863800526,0.2642794549,2.1103971004,-0.2257129401,-1.3434938192,-0.6517789364,0.9215201735,-0.5927384496],[0.9416487217,0.0437185653,-0.3107615113,-1.8002775908,-1.5352293253,-1.2877420187,0.2497640550,0.0889306590,-0.3507105410,-0.5227136612,-0.5778329372,-0.7980781794,0.2110481560,-1.3815633059,-0.6894747019,-0.2972003222,0.8221981525,-0.2503884733,0.7942054272,0.9488335848,-0.1884921640,1.0855621099,-0.6987811923,-0.2730012834,1.4528568983,-0.6969278455,0.5462369323,0.4251119494,1.1552058458,0.3428002000,0.8982730508,-1.4678502083,-0.5434156060,-0.0516024604,1.3498048782,1.7462068796,1.0836268663,0.8198674917,-0.4759975374,-1.0495749712,-0.9996607900,0.3143595755,-0.8451169133,0.9353263378,0.9249797463,0.2017759383,-1.9286822081,-1.1570881605,0.5033632517,1.1476683617,0.2429254949,-0.6879550219,0.9943078160,-0.2067762166,0.6849880219,1.4044768810,-1.4275686741,-0.8624293804,0.7384086847,-0.5561550856,-0.8264702559,0.7136894464,-0.3004029989,0.8249571323,1.3415811062,-0.1680783927,1.0953153372,-0.8835517168,-1.0230761766,-0.2327672988,1.2700972557,-1.1392649412,-0.4342262745,-0.7223634720,0.5530021787,0.5501202941,0.3454295397,-1.0601592064,0.3098625839,0.3677841425,0.9607761502,0.6529376507,1.4435209036,1.0172544718,1.3574310541,-0.0099963387,0.0024282169,-0.4879335463,-0.4324880540,1.4777532816,0.2670559883,-0.4485886097,0.6798025966,0.6807366610,-1.4864747524,0.2098133266,0.4501485527,1.2317266464,-0.2362726778,-0.4584268034,2.8149886131,0.1683895439,0.2099463046,1.2645759583,1.2813298702,-1.3021427393,-0.0147156315,-0.4178023934,1.9107762575,-0.6563686728,-1.0691428185,-0.4230622947,-0.0132462140,1.8421353102,1.7196012735,0.5972613692,0.3515547812,-0.5387497544,-0.0692220926,-0.3597345054,-1.0044300556,0.5427931547,-0.8332656026,-0.5242043734,0.8488559127,0.1747637987,-0.3172430992,-1.2048152685,0.6520953178,0.4357360303,-0.5587982535,-0.4179782867,0.4264180362,-0.0548314825,-1.1908137798,0.4734176993,-0.6122033000,0.4030275941,-0.2191144228,-0.3238929510,0.6194945574,-1.2550044060,1.5398229361,-0.4604188502,0.7527544498,-0.3760625124,-0.6907941699,0.6619018316,0.3901435733,-0.9779542089,0.2812705040,1.3486820459,0.8936386704,-1.0297094584,-0.7766075730,0.5017185807,-0.9354196787,-0.4815663397,-0.0947765708,-1.3963850737,0.2590472400,2.3648326397,-1.9255220890,1.3736010790,-0.2367615402,1.2566763163,1.9916495085,-0.5050504208,-0.6587370634,0.1339435875,-1.5667083263,0.4359713495,0.0420700498,0.0357943885,-0.0433804691,2.2872498035,1.2421827316,-1.6219403744,0.2979706526,0.5869822502,-0.6416541934,0.3410756886,-2.2526934147,-0.8089647293,0.2043059468,-0.1011173427,-0.4876793921,-1.3751064539,-0.2896009982,-0.3614101410,-0.4906738997,2.8373856544,0.2476367205,0.0999422297,-0.9083200097,1.2840486765,-0.9753172398,0.0642992556,-0.8871639371,0.3426522911,-2.1190073490,-0.7735089660,-1.3035955429,-1.9201433659,1.1583014727,-1.3426201344,1.3124009371,0.4313886762,0.0344963595,-1.2075973749,1.2700092793,0.6499128342,3.3665959835,-1.6459656954,-1.3581641912,-1.8216447830,1.4461368322,-0.1574293673,0.1464605033,-0.7976961136,0.1109806001,0.1583217680,1.0877591372,-0.0723408088],[-1.0505943298,0.2393178642,-1.4363778830,0.3752524555,1.2342381477,-0.8808445334,-0.2301429659,0.3931305408,-0.4250062704,-0.5529979467,0.6862195730,2.4019544125,-1.3151100874,-0.2301423550,0.4676499367,0.5954039693,0.4348103404,-0.0355928615,1.3828746080,-0.2989363372,-0.8919566870,-1.3971213102,1.7783776522,-0.2302737534,-1.7624164820,-0.6463377476,1.1267231703,0.4442473948,0.4791038632,-1.4811384678,-2.1068751812,0.5931427479,-0.1246527433,2.1817100048,1.6543698311,-0.1731125265,-0.1042543724,-0.0780078098,-0.7192997336,-0.4809856117,-0.1577761769,0.4041523635,-0.0076569053,1.1993273497,-0.3402960598,0.7366626859,1.3511939049,0.4218282104,0.1720520854,1.8980066776,0.2961442173,1.1082334518,-0.7020488977,1.1318085194,-0.6802171469,-0.6433147788,-1.0897530317,-0.1884802282,-0.8276434541,-0.7988906503,0.3866085112,-0.0302876048,1.0736109018,0.0648761988,0.8674221635,0.4764135778,-1.1911543608,0.7603228092,1.4816516638,-0.8416389227,0.1508948207,-0.4492126107,-0.5611734986,1.6207737923,0.8324872255,-0.5176488757,0.5176472068,3.0010211468,-0.5873510242,-0.7627546191,0.7660671473,0.6905959249,-0.2636062503,-0.4258366227,-1.2875888348,1.4470908642,0.0458003469,-0.5081302524,1.5856559277,-0.5258768201,0.1839931756,-0.4267107844,-0.7687863708,-1.9748845100,-0.7904258966,-0.3710695207,0.2210629135,0.1301920861,-0.8979573250,0.2273373753,0.7875413895,0.7730968595,1.2694029808,0.2548620105,-0.8165200353,2.0619447231,0.0528843738,-1.4856121540,0.5538668633,0.4425628781,-0.2612531483,0.2037329376,1.2159631252,-0.1642043442,0.4371479452,-0.2363016307,-1.6688332558,-0.8167054057,0.2186635584,-0.2214595377,-0.2448555678,-0.3115261793,0.9378111362,0.1662903726,-0.6850562692,-0.8621481061,-0.8770141602,0.2879326642,-0.7475222945,-0.7961471677,1.0106245279,-0.2903393209,0.1513967961,-1.1147676706,-1.2387049198,2.6652097702,-0.1084619164,1.3810008764,0.6159277558,-0.5572487116,-0.4658169150,-0.2820259333,-1.1114195585,-0.3663528562,-0.4245306551,-0.1668689102,1.3447589874,0.0826546401,0.5931445360,-2.0726764202,-0.6349374056,0.1409984976,-0.8965637088,-0.2465610951,-0.7543249726,1.3068647385,0.5193663836,1.2673950195,0.0530828945,-1.3074204922,-0.9408559203,-0.3132842779,-0.3520522416,0.2639456987,-1.6094528437,0.2965346277,1.1979731321,-0.6135300398,-1.2268162966,0.1642137766,0.9970766306,-1.3188539743,1.0603978634,-1.1148290634,-0.0663175508,-0.7761843204,-0.8369712830,1.0939637423,-0.1550624073,-1.6109610796,-0.0460047275,0.6645396352,-0.2567320764,0.0598233193,0.7579881549,1.9713249207,0.3284783065,-0.5257778764,0.9011698961,1.1173251867,-1.0251053572,-0.0538991652,0.2657756507,0.6858976483,-0.0486010723,0.1332474202,-0.5991258025,-1.6124672890,0.0241628140,0.0723342896,0.6667144895,0.3458332419,-1.0246504545,0.0992837548,-0.8964792490,0.0723521560,0.2398944497,-0.7628162503,0.4543907344,2.3749954700,-0.1462941468,0.0003235369,1.0902603865,0.5219897628,0.3533296883,-0.7274948955,0.3543509543,0.3556120694,0.8606618643,-0.6099217534,0.6097067595,-1.0419270992,0.1097846627,-0.1417976171],[-1.1922610998,0.0365081467,1.3653956652,-0.5518307686,0.3183691800,0.1170945540,-0.1883369386,0.9821451306,-0.0275580082,0.6468264461,-1.5413923264,1.6071134806,-0.9581078291,0.7696560621,-0.1862657368,0.4136942625,-2.6068975925,0.8061500788,-0.7720109224,0.1059482396,-3.1292214394,0.0776122808,-1.3263397217,0.3734543025,0.9427181482,0.1115776300,1.9515994787,0.7113924623,-0.2507297397,1.5339950323,-1.1204895973,0.3802853823,0.5562388301,0.4393317401,0.3805435896,-0.1796236187,-0.0068756798,0.3063318431,-0.8758431673,0.0523967259,-0.9203234911,0.4118686318,-1.1167979240,-0.8827121854,1.2052417994,-2.1289260387,0.5620676279,0.9299391508,1.7372515202,0.9391069412,0.1322578937,1.6648646593,0.6191270351,-0.4869742095,0.0202155150,-0.0032952912,-0.7536495328,-0.0577502102,1.4988473654,1.1853531599,-1.1569428444,0.4523760378,-0.4761290252,0.4584850967,-0.0814426169,1.1522524357,0.5754417777,0.9752656817,-0.6391966939,0.8959468603,-0.5191081166,0.4095037282,1.2617907524,1.3778895140,0.9799590111,1.6725335121,-3.0943217278,0.9498860240,0.8754180074,-0.3936840296,-0.4376997054,-0.2135491520,-1.3722800016,0.3389026225,0.1849378496,0.1413071156,0.1889852732,-0.9622039795,-0.6044231057,0.8672102094,0.0209429972,0.6589286327,-0.1708002537,0.1002857089,-0.8388501406,-0.1158685908,-1.9335631132,-0.2489332408,-1.6315416098,0.5039172173,1.1015135050,0.5260147452,0.3572916090,-1.0680633783,-0.7171474695,-1.0039732456,-0.6688591242,-1.7563149929,0.3410589695,-0.0830375180,-1.8177490234,-0.1606295109,1.2918634415,-0.8302479982,-0.1817802340,-0.2532959580,1.9223433733,-2.0590584278,-0.7776770592,1.4901905060,1.2545901537,0.8289508224,0.1364070326,0.0048171692,0.4144740701,0.1384837925,-1.2213534117,0.4205737710,-0.1566273570,-1.3302257061,-1.0808893442,-0.3223316371,2.3164737225,-0.6410893798,0.7436528206,-1.2338428497,-1.8407684565,-0.9513223767,-0.5103200674,-0.5803955793,-0.3061780930,0.4369329214,-0.9844542146,-0.3322239220,-0.5980868936,2.0401651859,0.7489195466,1.0855104923,-1.0209662914,0.9438039660,1.3219096661,-0.7857168913,-0.7631080747,0.3792274594,1.9473079443,-0.4503182769,0.3687690496,0.5508050919,-0.0530326739,0.3569258451,-0.2823581696,0.1155722141,0.0433711931,0.0135188764,-1.3273210526,0.2165891230,-0.6997963190,-0.0507710278,-1.7662363052,1.1606320143,-0.0638204664,-0.1239236444,-1.5254353285,0.8884639740,-0.0218050219,-0.4060950279,1.0596505404,1.6093878746,0.6677417755,-0.8113313913,0.2368712276,-0.7918953896,1.3205071688,-1.1475688219,-1.0701115131,1.3642249107,-1.0599621534,0.0186037924,0.5424012542,-0.7976353168,0.5737085938,-1.7811143398,-0.8941566944,0.6157245636,-1.4740045071,-0.0339163728,-0.2008125037,0.4697420597,0.7060800195,0.3085568547,1.4709632397,-1.4332449436,-1.3343772888,-1.9902968407,-0.3875651062,1.7211320400,1.4731040001,-1.1057852507,0.8125382066,1.5489228964,0.2021583468,2.3115398884,-0.4626514018,-0.0988822505,-0.3713912368,-0.1346644759,-1.1158742905,-0.1639193296,0.0890217721,-0.4499882460,-1.1167421341,0.4916646779,0.2613768578,0.2538814545],[0.2545026839,1.0541639328,3.0064599514,-0.7094870806,0.7198755145,0.1062187478,-0.7458360791,1.3711556196,0.9849750996,1.1028059721,0.2148303241,0.2942796648,0.3881600797,-2.2538924217,-0.0313082747,0.0384150855,1.4320955276,-0.9805296063,-0.8168244958,-1.0708725452,-1.3232696056,-0.0492793471,1.1589303017,0.9985959530,0.1406734884,-0.2315083295,-1.3020762205,1.6821084023,0.0845298097,-0.0012636767,1.3425643444,-1.1528156996,-0.4846612811,-1.1822725534,-0.6761192083,-0.0896490365,0.0943385810,0.3186304867,0.7720057964,-0.2023819238,0.2094048411,-1.2419801950,-0.3391025066,3.1322660446,0.1629785895,1.3776082993,-0.4834138453,-1.4403148890,-0.9649205208,0.6832093000,0.3958668113,0.5880941153,0.0307263527,-0.8174967766,-0.0751677454,0.4808998108,0.1181297526,0.2792820930,-2.1766908169,0.3593552709,1.2126170397,-0.7078419924,-0.1993245333,1.2566301823,0.3003346920,-0.4112148881,0.4869786203,-0.0476831608,0.6511651874,-1.5540522337,-0.4369316995,1.0390468836,0.9457051158,0.2111051232,-0.3961375952,0.9423094988,-1.1002054214,-0.7350199819,0.0146141937,0.5745733380,-2.6855244637,0.5434151292,-0.1790276021,0.7951074839,-1.7326436043,0.0923662931,-0.5095089674,0.5917858481,0.8422880769,-0.1820607781,-1.6822385788,0.9818657041,0.5777400136,-0.7624858618,-1.0306129456,-0.2130795270,0.9831990600,0.8381908536,0.4186193347,0.2878623307,3.3586564064,0.6768281460,0.7891095877,-1.5845159292,0.6934824586,0.1463712752,0.1077918932,-0.0776537731,-0.9601588845,1.5597449541,0.2852105498,1.1821978092,1.4600056410,-1.0947434902,-0.5802459717,-0.3190021217,0.2932513952,-1.2169111967,-0.1552423239,0.2548938096,-0.6420126557,0.9541916847,-0.4548354447,0.2853848338,1.0490132570,-0.1054982617,-0.8711935282,0.0640761554,2.0955986977,2.2802312374,-0.7062234282,-0.9325771928,-2.7064745426,0.5550458431,-0.8726333976,0.8819828033,0.4131059051,-0.4404965639,0.5863825679,0.9154740572,0.2949877381,-2.6654002666,1.2274175882,0.1260483563,1.0107161999,0.4425277412,1.2145001888,2.3537797928,0.3502727151,0.4897087514,-1.4731489420,-0.6829451919,0.7784183621,0.0769444928,0.8899565339,0.6568589211,-1.4877281189,0.8486710787,0.2527044415,-1.2365397215,1.8013707399,2.1016540527,-0.0500311032,-1.3022519350,-1.9501366615,-0.5360963345,0.3994601369,0.5160616636,-0.8235284090,-1.7060825825,-0.7978700995,1.7503622770,-0.1441345215,-0.9500775933,0.5486234426,0.1444375366,-0.7123869658,-0.1756013632,0.1871870011,1.1465924978,-0.3897903562,1.8086135387,1.1280059814,-0.0083743073,-0.3285886049,0.2197051197,0.7689400911,1.0436525345,0.2513706982,2.6073219776,-0.3112938404,0.2724854648,-2.2912137508,0.2034903765,1.1328512430,-0.0851072371,-0.1136390269,-0.2366341352,1.2237306833,-0.2443148643,-0.1149372980,0.8067960143,0.7126696706,-0.3885196447,0.8958705068,0.6093592644,0.7420414686,-1.1293281317,0.0535399579,0.1800847501,0.9242040515,1.0947157145,-0.4837334454,-0.8320499063,-0.0283501279,-0.7321954966,-0.4648154080,0.5936284661,-0.1790808439,0.6123202443,-0.3115934134,-0.6220819950,0.3708849251,1.4454218149],[-0.1824720055,-1.4103887081,0.2352046967,1.5774737597,0.8297368884,1.1899679899,1.4147301912,-0.1582700610,-1.1319595575,-0.5135700703,0.7510833144,-1.5830452442,-0.6762850881,-0.6799480915,0.4081239104,0.8614497185,-1.4354953766,-1.4303246737,-0.3100005388,0.6109189987,0.3791231215,-0.7022027969,1.0591596365,-1.4798945189,-1.0763273239,-0.6885050535,-0.5565457940,-1.9920260906,-0.5430017114,-0.2988832593,-0.4576859176,0.6534484625,0.0158297122,0.0252145696,-0.5811895132,1.6511220932,0.0293052960,0.8481232524,1.3957945108,0.9147311449,1.9018837214,0.0225104298,-1.0985888243,-1.5755480528,-1.9768010378,-0.4817121029,-1.4537837505,0.7854406238,-0.3301056623,1.2437915802,-0.0110594444,1.3535820246,-1.1919013262,0.6878582835,0.4309915602,0.0344051160,-0.4072600603,0.1183931530,-1.1715111732,1.1703085899,1.4365798235,-1.1344217062,-0.5182979107,0.8712841272,0.8400304317,1.4912987947,0.3017890751,0.2879858613,1.5178281069,-0.3468378782,-0.1891960204,0.3158127964,0.2165258229,-0.2097465992,0.6558660865,-1.6493350267,0.0418719389,0.2596367896,-1.1072080135,0.1650775224,-0.6635113955,-0.7450125813,2.1797268391,0.7675815821,1.6386095285,-0.7342829704,-0.8826609850,0.8525941968,0.0779680610,-0.2251982987,0.5225286484,1.7051614523,-1.0837510824,-1.2669957876,0.1919908673,0.8322687149,-0.7541927695,0.6820940971,-0.0351917483,-1.4053186178,0.2227966338,1.3239190578,1.3002049923,1.4524662495,0.3714876473,0.2805170417,-0.0919749141,-0.5110696554,1.1964526176,-0.0531565435,-1.1680847406,-0.6295111775,1.6872906685,-2.5313727856,0.2121602297,-0.8588676453,0.5005645156,-1.4779620171,-0.4762757719,-0.2537456751,0.5073809624,0.6816993356,0.5533781648,0.9636012316,1.4837093353,1.5566785336,1.2752995491,-1.1811940670,1.4324865341,0.2819110155,-0.4524187148,0.7230597734,-2.5340185165,0.0218055099,-1.2718653679,-2.7177481651,0.5364590287,-0.0088465754,-1.0927875042,-0.2092958838,0.3976587653,0.1919134855,-0.5732107162,-0.0421538688,-1.5942142010,-0.3043152690,-1.3545215130,-1.0652045012,2.5165457726,0.8235344291,-0.8325254917,0.7410484552,-0.8188034892,-0.3710550368,-0.1469221711,1.4208518267,-0.6055439115,-2.3149800301,0.0533712320,0.9889836907,0.5201842785,1.0583690405,0.0227681696,1.2871819735,0.8295145035,0.1761369258,-1.2930803299,0.5198744535,0.3537299931,0.9064239860,0.3577017486,-1.2018020153,1.5393267870,1.1606206894,0.3462671638,1.0494418144,2.0337548256,-0.6492514014,-0.2536161840,1.1055673361,0.7026463747,0.8310782909,1.0652977228,-0.2652146518,0.6976580620,-0.1085451543,-1.5246456861,0.3379697502,0.6745842099,0.6699867249,0.7807105184,0.3559247553,-1.4706202745,0.2427008599,0.1194189042,-0.9471550584,0.5363782048,0.7715194225,-0.7956759334,1.3846803904,-0.0059837541,1.6803729534,0.4307575822,1.3302416801,-1.9971408844,0.9966135025,0.6510790586,0.3689449728,0.3054286242,-0.0868136510,-0.4496328235,-0.4203562438,0.0446175151,0.0464101844,-0.6030803919,0.9469773769,0.8830232620,-0.6825973988,0.0063548535,0.9545045495,1.7176597118,-0.4867136478,0.5350939631,0.2192623615],[1.4326761961,-1.8599370718,-1.3125212193,0.1645046771,-1.1676248312,1.4429554939,-0.5579243302,0.4837940633,0.9947431087,-1.2013341188,-0.4283203483,-1.7186342478,0.0546005592,1.0362206697,-0.9954963326,0.0477595143,-0.0268921070,0.6498132348,0.1365755051,0.1145898923,1.2277548313,-0.7165322304,-0.3180812597,-0.9940687418,-1.9625616074,0.7006590962,-0.0359061249,-0.3734052181,-0.2956706583,1.2970714569,-2.0354444981,1.5242056847,1.4134936333,0.6256355047,0.6216192245,0.5148414969,1.6640155315,0.0376209542,-0.2404932976,-1.3875989914,1.0557792187,0.9804635644,0.2911759019,1.6869058609,2.4566636086,1.6708139181,-0.6846850514,-0.0785990059,-1.9915647507,-2.2253570557,1.7390940189,-0.0221805908,-0.5199558735,-0.4286879599,-0.8465876579,0.1239822507,-0.3726306856,-0.8203553557,-0.0182525702,0.5630733371,-1.8752394915,0.4155042470,0.5997116566,-1.9111266136,-0.1225134805,-0.9004780054,0.5674170256,0.9721233249,-0.5763071179,1.2280766964,0.0869190246,1.5497611761,0.0690125600,-0.3725866973,1.1458171606,0.7421871424,-0.2898759842,0.4488764405,-1.1340010166,-0.6257569790,0.8099257350,-0.5755894184,-0.3711819947,-1.2420692444,0.5750556588,1.2347095013,-1.3155068159,0.5820550323,-0.1800024062,-0.7291352749,0.9901236296,1.6943480968,0.3219739199,0.4525323808,-0.8705807328,1.2545053959,0.0177927706,-0.9325863123,0.5151397586,-0.0098660281,0.4695507288,-1.9551370144,0.1708181202,-1.2006139755,-0.6353746653,-0.3414449990,0.6378657818,1.6013050079,1.3553612232,0.3463516533,0.4996110201,0.5760862827,-0.3864963055,0.6062389612,-0.9762124419,3.0961830616,-0.3332355917,-0.6250987053,-0.3858212531,-0.0016092911,0.1147701219,0.1219455898,3.0642769337,0.9969142675,-0.3090046346,-0.6291542649,1.7868961096,0.3111509979,2.1643211842,0.2989636660,-1.6422638893,0.0548259765,0.0236156844,0.6794649363,0.6361483335,-0.2971331179,0.1430229992,0.3408798873,0.1371335685,-1.1627069712,-1.6395305395,-0.7312899828,0.8405165672,-1.4277316332,-0.6171786189,0.9497544765,0.1589775980,1.0119231939,1.3794178963,-0.4360499978,-0.0422094576,1.5773836374,2.0762181282,0.7507163882,-0.3115755916,0.7139906287,-1.1984947920,1.6259472370,-1.8023258448,-0.2536619902,-0.3082579374,0.0895618051,-0.2826513350,-0.7755914330,0.1211613193,-0.5160586238,0.3996881247,2.6127579212,0.7454584837,-0.5768119097,-1.1571289301,0.2560498416,-0.8254200220,-2.6075785160,0.8396331072,1.7745233774,-1.1578730345,-0.2286801040,0.1026872993,-0.4894656837,-1.0386230946,0.6681470871,1.1531586647,-0.2183658928,0.3109624088,-0.6623362303,-1.8053112030,0.3087073267,1.0570780039,1.0746499300,0.5073927045,-0.8723320365,0.6104628444,0.4474971294,-0.7325591445,0.2552275360,0.4838557839,-1.0039037466,-0.1963803470,-0.6780635118,-0.4196233749,1.5498566628,-1.6033440828,0.3494834006,1.0629042387,1.0952367783,-0.9827461243,1.2384337187,-0.7188549638,0.0698258132,-0.5090520978,-1.3466193676,0.2113907784,-1.9054464102,0.7425320148,0.0713237897,-0.3361792564,-0.1544833332,1.2751290798,0.9343932867,-1.1927865744,0.1609488577,-0.0817530602,-1.6265294552],[0.0792161152,1.0164269209,0.0274453927,0.5830265284,0.4806797206,1.7057460546,0.3446226120,0.1727063209,-0.5576907992,0.5787950158,0.2685793638,-1.1247400045,-1.1956520081,0.7687556148,-0.8483432531,-0.1325481385,0.6503481269,1.2317855358,2.5165569782,0.9462372661,0.7442957163,-0.1037486866,0.8058280349,1.2004961967,-0.2012088895,1.2878060341,-1.6287704706,1.1572757959,0.2970125377,-0.8433011770,0.6941140890,-0.3813734949,-0.7616585493,0.2744237781,-0.0081879422,-2.0205621719,-0.4257437587,0.0422513634,-0.0920020416,-1.5582008362,-0.0928666592,-1.2283902168,-0.1833214313,-0.3190955520,-0.0863488317,0.9307079315,-1.9677612782,-1.5400369167,1.0738964081,-1.0055810213,1.1223951578,0.5734156966,0.4714262486,-0.0594240949,-0.8080806136,0.2278677970,1.0362339020,-0.2495359480,-0.7958166599,-1.1433826685,-1.1669473648,0.2910191119,-0.2219422013,0.1698516011,-0.5612007976,1.1821777821,-0.9412812591,0.3577224612,-1.4375506639,0.6173115969,-0.0753971860,0.4404014051,0.3479704261,1.7728350163,-2.4903116226,1.9318493605,-0.1871684641,-0.0486194454,0.5669991970,-0.1999413967,0.3919151723,-1.0137641430,0.4496327043,1.2215998173,-1.3674079180,-0.3145882785,-0.6682460308,1.7609888315,0.4312697351,0.4117654860,-0.8673826456,-2.2342579365,0.8778496981,1.7627274990,0.1801376790,-0.9213985801,-0.7761957645,1.5885344744,0.3222749829,-0.7482187152,-0.0693738833,-2.5485017300,-0.0396765657,0.9220715165,-1.1876281500,-0.2048502117,-0.8791155815,-0.2799369693,-0.7904237509,0.7771903872,0.0180385746,0.2614985704,0.1531894207,0.4526953697,-0.7578824162,-1.0335227251,1.0775671005,0.2108330131,0.2002221793,0.4302573204,-0.9542171359,-0.1206719950,-2.2216730118,-1.1611789465,-0.8942284584,0.3513225317,-0.4492657483,1.7326287031,0.8167504072,-1.3099080324,1.0597997904,-0.0260978416,-0.0922782496,-0.2445535958,-0.8588480353,0.0233809687,0.8380134702,-0.8886740804,0.0055783619,0.8041345477,0.5155995488,1.3099330664,0.0607562996,-0.4853078723,-1.2040776014,-1.8143754005,-0.9156990051,1.1903975010,0.3376074731,0.4168293178,0.2208614945,0.0823195726,0.3248101473,1.0169280767,0.3660759330,0.7446341515,-0.6754641533,-0.9308695197,-0.9214447737,-0.9902174473,0.0467686094,-0.4429434836,-0.2878521383,0.4659559727,-1.5754804611,0.7648607492,0.6077049971,-0.2009276748,0.1410205662,-0.7378753424,1.5970349312,2.4544296265,-1.7936341763,-0.6051549911,1.5142178535,-1.1078259945,0.3795293868,1.7709084749,0.4984016716,-0.2436304092,-2.2577059269,-1.2458858490,-0.7088252306,1.5962058306,-1.9355672598,0.6987320781,0.3166677952,-0.8615584970,-0.1896579862,-0.6173701882,-1.0844488144,0.5718342662,-0.5237622857,-0.4043986797,1.4371533394,-0.5186876655,-0.8964678645,0.9899360538,-1.3119916916,-0.7182676196,1.0389785767,-1.1774294376,0.6187053919,-0.4993627965,-1.1766394377,0.0940822363,-1.0231488943,-0.4538815022,-0.6294190288,0.5030683875,-0.2208803594,0.9774012566,-1.0498013496,1.1215320826,-0.2495442778,0.3619700968,-1.0616446733,1.0567066669,1.3423643112,0.7119333148,2.0674512386,-1.6267881393,0.6471201181,0.2160703689],[1.3485805988,-0.1709989458,-1.2881512642,-0.3260785937,-1.8688436747,-0.1713963598,-0.2822999358,-0.7701655626,0.1642793864,-0.4175589979,-0.6222335696,0.2136606127,0.4244953096,1.0451035500,-1.6823191643,0.0403030105,-0.4710847735,-0.2437952608,1.3693777323,-1.1758570671,-0.0542855076,-0.9699545503,-0.5364821553,-1.7010054588,0.2163471431,-0.2669135332,-0.3398927152,-0.0248095077,-0.4684832692,-2.0178093910,0.4001567960,-0.6108651161,0.5783423781,0.1649526060,0.8978267312,0.7421654463,0.4261821806,-1.0519768000,-1.4087666273,-0.8967926502,-0.6909884214,-1.0733422041,1.3887276649,-1.5292494297,0.6003373861,-0.3892106116,-0.0517769493,0.0179459751,0.1571146250,-0.5726046562,0.0190555137,0.5697455406,-0.8141471148,-0.3097047508,0.4373070598,-0.3019420505,0.1362971067,-1.0670320988,-0.0000817540,0.8114110827,0.4083184600,1.8282098770,0.1381276697,-1.9155849218,0.0719738081,0.2208820730,1.2493988276,0.3405856788,2.0780653954,-1.2551661730,1.0017586946,-0.5354287028,0.8841439486,-0.5767771602,1.1559445858,0.3145794272,0.1051806211,0.6909182072,0.7290110588,0.1197143495,-0.6287478209,-0.6085677147,-0.8197476268,-1.5091049671,1.0208386183,-0.0439724736,1.7148467302,-0.0321364142,1.3864095211,-1.0215556622,-3.0513429642,-1.1673578024,2.1856009960,0.2588728666,0.3434796929,-0.8460996151,-1.9793503284,-1.1928198338,0.9098986983,0.4554537833,-1.0023721457,-1.8009947538,-0.6367038488,0.9891266227,0.0203659870,0.3414032459,-0.4097161591,0.2278744727,1.2209628820,1.1159952879,-0.9623272419,-0.4903887212,0.4621538222,0.2723306119,-0.9540486336,-0.2404303402,0.3517660499,-1.4416646957,0.4990543127,-0.7126743197,0.3536555171,-0.9577149749,-0.3402226269,-0.8734661937,1.0238445997,-1.2684786320,0.2645486295,0.2472764254,1.1231633425,0.8844839931,-0.5912511945,-0.3987582028,-0.6692000628,-1.7035512924,1.1779968739,1.1480795145,-1.9619866610,-0.6322578192,0.4063148499,0.8685815930,0.1516608745,-1.1480426788,0.0343636200,-0.2617740929,0.8340689540,0.8000271916,-0.4284797311,-0.0311497673,-1.1331477165,0.1529792249,0.7315949202,-0.4204881787,-1.7638682127,0.6233843565,0.1415092200,0.2131320089,1.0914654732,-1.6148589849,0.4392548501,0.3568493426,-1.3790297508,-0.1812242717,-0.4438568354,-0.5541661978,-0.4909126163,-0.9593079686,1.0866227150,0.0711556524,-0.6503744125,0.3807986677,-0.0300570186,-1.6283313036,0.6702497005,0.2368342578,0.6066207290,-0.6736672521,-1.0897010565,-1.1997205019,0.2919110060,0.5207223892,0.6099629998,0.6813975573,0.4636356533,0.0687451810,-0.4502599537,0.1851959527,-1.4338147640,-0.4437556267,1.4200047255,-0.6315543652,-0.7744462490,0.6045300961,-1.2394708395,-0.2095934153,-0.3702944219,-0.6200354695,-1.3909845352,-0.3433581889,-0.9693596363,-0.3587476015,0.3847161233,-0.2518064678,0.4357206225,-0.9091473222,0.6839234829,-0.0295625720,1.4972803593,-0.1460575759,-0.0939826742,-0.2217037976,0.0946994647,-0.2410782129,0.9098507762,-1.3256295919,-0.0817464665,1.3583462238,0.7781915665,0.0069436296,0.5164428353,-0.0651496351,0.7075184584,-1.4202232361,0.2231310159,-0.3650387228],[0.1650255471,-1.3219217062,1.0231046677,0.7444481254,0.5359328985,0.5666531920,0.3261220157,-1.3667412996,0.1605643630,0.2437143475,0.4470121264,0.3345755637,1.5117784739,-1.4013890028,-0.3585534990,-0.2405987382,-1.2795827389,0.3206909299,-0.4760054648,1.4264730215,0.2798275650,-0.0359120257,0.4687083960,-0.4645172954,-0.1489082575,1.6419786215,0.8950193524,-0.2498975992,0.0254452918,0.7679093480,0.3977237940,-1.2860594988,-0.3435108960,1.3776507378,-0.2499962747,0.5837259293,-0.3566249013,-1.5012432337,-0.0366155952,0.6415373683,1.7652125359,-1.3682287931,-0.0861081928,0.0383691862,1.2417683601,-0.1283382177,-0.1103384346,-0.7885918617,0.0254839975,-0.7123177052,-1.1390565634,-0.0729266852,0.2223683745,-2.5684566498,0.4270484746,-0.0046181614,0.0415315256,0.6546118855,-0.9823287725,-1.3519179821,1.2908511162,-0.5305488706,1.7785286903,0.8555805683,1.3566535711,-0.4564712942,-1.4938685894,-0.9814547896,-1.3854124546,-1.5733133554,-0.8129420280,0.3343071938,-0.2484793961,0.7299104333,0.6596966982,-0.7074251175,1.6392796040,-0.7114089131,0.3561543822,0.1788314283,-0.0102422200,-1.5114012957,-0.0173943099,-1.0720241070,-2.3937253952,-1.3889380693,0.0471616276,-0.4784292877,-0.8774627447,-0.7773531675,0.5064339042,1.4271920919,0.6994634271,0.3179759979,2.0188155174,-0.6257854700,0.4875732064,0.9742701650,-0.6120517254,-1.6027874947,-0.5442904234,-0.2421242595,0.5398359299,1.3619903326,0.7030728459,-0.4203302264,-0.0356382467,0.4759574831,-0.3176399469,0.9230871201,-0.7705651522,-0.8988927603,1.8517060280,0.6137269139,-0.1375556439,0.7560847998,-0.7871061563,-0.8101333976,0.9718251228,-0.2960273027,-0.8531348705,-0.7449046373,-2.3861715794,0.5155227184,-1.4067013264,0.3101865947,-2.1656732559,-0.4105100334,0.4387335181,0.6727346778,0.2653877437,0.9309439659,-1.1482447386,1.7589145899,0.4293440580,0.6618949175,0.6083560586,-0.7772499323,-0.2988936305,1.9642839432,-1.3446459770,0.3042408824,-1.8033968210,1.1037830114,-1.7535353899,-0.5489560366,0.9455407262,-0.8433424830,-0.1801532805,-0.4749929905,0.2693964243,0.4919607341,1.2626482248,-1.0708043575,0.1030553579,0.1843638122,-0.7110225558,-0.0900483355,2.1595232487,0.1592033505,-0.9272395968,0.9635785222,-3.7438845634,-0.7730277777,-0.9164265394,-0.2171987295,0.0299481545,0.4585601985,-2.6037380695,0.2609812319,1.3040894270,0.9559362531,-0.0413816720,-0.6852220297,0.4935119152,0.7921469212,0.1276420653,-0.1001019925,-0.0484789126,-0.5497506261,-1.6832387447,-1.5431174040,-1.1059116125,-0.1819771081,0.1616688073,-0.5764904022,-0.9880220890,0.0783481523,1.4437611103,1.9365442991,-0.3549702466,-2.2470059395,-0.4686661363,-0.9983262420,1.7535051107,0.6303505898,-0.7734314799,2.6364636421,0.0392718092,1.5254333019,-0.6479625106,1.6099628210,0.3199017942,0.3757773638,-0.4623837471,-0.1848932803,0.3996022344,-0.0442292541,1.3251973391,-0.4424172640,0.0063154404,-1.4659434557,0.5882980824,0.6704141498,0.9195390344,0.5987154841,-0.6095600128,2.0387501717,1.5419700146,-1.3648542166,-0.9416725636,-0.5326945186,-0.6290355921,0.2064234316],[-1.1643296480,1.3079528809,0.4787613750,-1.1614365578,-1.2261186838,0.6149030924,1.1901096106,-1.7869648933,-1.7015125751,-0.0369055569,-0.1572385281,-1.8780126572,-0.1200147271,-0.4242606461,-0.3192552626,0.0104762856,-0.6464591026,0.8706829548,0.5600137115,2.6534612179,1.3147951365,-2.0153663158,-1.6875379086,-0.6459116340,-0.4426926076,0.1987432837,0.4108413756,-0.4009189308,0.9362496138,-0.7191836238,0.6938658953,-1.7758921385,-0.1403557509,-0.0228065960,1.0006980896,0.8325053453,0.6643543839,1.0108941793,-2.3972070217,-1.1385048628,0.4741702378,-0.6687298417,0.4963333309,-0.7473987341,-0.3865411580,0.2450307906,-0.8685840368,-1.5923013687,0.0233582482,0.8655768037,0.9436566830,-0.7575323582,1.0158644915,0.6127064824,-1.7185145617,-0.4814927578,0.0290120076,-1.8096933365,-0.8135496378,0.5268061161,-1.3745579720,0.8449429870,0.7756015658,0.2198592126,0.1512816399,0.7091155648,1.1741852760,-0.5143500566,-1.6707895994,1.2580791712,1.5318725109,-0.5002031326,-0.2448760867,-0.2390441149,-1.4903237820,0.6865671277,-0.3201767504,-0.0794070512,1.9951273203,-1.4163293839,0.9918632507,1.7666332722,0.1555749178,-0.3725374937,-2.1025719643,-1.2156616449,-0.8836985230,-0.3397898972,0.4215038717,0.4887259007,-0.4905745983,-1.3774695396,-0.6583486795,0.0738252997,0.7343323231,0.4211414754,0.1247600242,1.7488490343,0.2251959741,-1.0532218218,0.4716623127,0.5744453073,-0.0024379981,1.1637272835,-1.9481000900,1.4603103399,1.5550012589,-1.2255028486,0.6156579256,0.6069850922,0.8603722453,-0.5207680464,0.8478476405,-0.3461050093,-0.6320928931,-0.7256397605,-0.9042200446,0.7228765488,-0.0928905234,-0.7883547544,1.6062039137,0.0710300654,0.5437248349,-0.4915961325,-0.1041649207,-0.3031794131,-0.4425407946,1.1392168999,-2.0505964756,0.4079806805,-0.0800484493,-1.3515720367,-0.9124820828,-0.8583441377,1.2894556522,0.5908492804,-0.5798185468,0.0520669743,-0.1212026998,0.4420488775,0.6347669959,2.2531759739,0.4614149332,2.1107082367,-0.6364678741,0.6796067357,-1.2275124788,-2.1137130260,-1.4560955763,-0.1521469504,-1.1474013329,-0.0936479792,-0.4342494607,-0.3847503066,0.9745326042,-0.8003863692,-0.1759050786,0.4681669176,-0.2825888991,0.5702939630,-2.0764098167,0.0134140672,-0.7991058230,1.5231801271,0.3928492367,0.8087913990,0.1406501085,0.2782938778,-1.2058972120,0.7704486251,-0.6398083568,1.8859362602,0.8982129097,1.1188243628,0.0342440680,0.1872243583,1.8391888142,1.3006812334,0.3444184959,0.1678676009,-0.2385491282,0.1145463064,-1.5685496330,-0.4964090586,-0.3101575971,0.3440920413,0.1293883473,0.5067656636,0.3827549219,-1.0888370275,-1.4858487844,-0.9661257863,-0.7745360136,-1.4245500565,0.1393938810,-1.4260451794,-0.5082685947,0.8763096929,-0.6662014127,-1.2093539238,0.7315164804,-1.1380810738,0.0281808823,-0.9322289228,1.2290621996,0.7308822870,0.1425308883,1.4224876165,0.7261432409,1.1260491610,-0.8474053741,-1.1105213165,-0.8556407094,-0.7260117531,1.1177033186,1.1724927425,-0.1795573980,1.0290118456,-1.1191790104,0.0396585315,-0.0047235782,0.2664375901,-0.5832667947,-0.8921521902],[-0.8755141497,-0.9598096013,0.4069607854,2.1447017193,0.3723194003,0.3736964762,-0.6830319166,1.3127006292,-0.6640972495,0.7430377603,1.0540186167,2.0303764343,0.8735508919,-0.6355731487,0.5069352984,-0.9557527900,0.9533885121,-0.4849050343,-0.1885131001,-0.6945106983,-0.6633606553,0.2577122450,0.6968512535,0.0700863674,-0.8694693446,-0.9572583437,-0.1153896451,0.2002226561,-0.3849981725,0.2202363312,2.0618686676,0.3926514387,-1.5631883144,-1.2940770388,-0.5507266521,0.7313650250,-1.7164524794,-0.7593462467,0.3257178962,1.1041924953,-0.6681020856,-1.2724117041,-2.1357870102,0.3565237820,-0.0748446658,2.0107169151,-0.8546269536,-1.1847459078,1.8804349899,-0.1341653168,-0.5399327278,0.6568149924,-0.6100254059,-0.0584002621,-0.3279354274,0.0255116895,-0.4765267670,-1.3591419458,-0.1435545087,0.5789028406,-1.7496489286,1.4535236359,-0.6286903024,0.2937216759,0.1816752106,0.3662569225,-0.8155812621,-0.7177008390,-0.6351452470,-0.5987464190,-0.7643153667,-0.0924506560,0.0002173648,0.4418594539,0.2845681310,0.1451430768,-0.9784609079,-1.6784154177,-1.2964258194,1.5535236597,-0.7936209440,-0.5138964057,-0.9194343686,-1.3460687399,2.6226949692,0.8296312690,0.6559735537,0.5372175574,-1.2835997343,-1.2348910570,-0.9327340722,1.3215962648,0.4506095648,-0.1519411802,-0.1253918707,0.1870906502,1.6497941017,0.0869149938,-1.0887746811,0.7199364305,0.4501723051,0.2700583041,-0.4041815102,1.0006670952,1.3125792742,0.6810870171,-0.1800761074,2.4339106083,0.1763603538,0.8710671067,-0.3742243946,0.4617210627,0.3632403314,-1.1402145624,0.8772246838,-0.2340421081,-0.8595511913,0.0905576646,-1.4307808876,0.5439644456,-0.0927907899,-0.6408239603,0.6136070490,0.5124572515,-0.6934475303,-0.6896564960,0.5295175314,-0.1667579859,0.2426082194,0.0189859830,2.2494912148,-0.2460813224,-0.2038732916,-0.6019621491,-0.0026605215,0.1960952580,-1.1902012825,0.4211482704,-0.4473219514,0.4534853399,-0.3401862979,0.1513924450,0.3664656281,0.2547777295,-1.0560954809,-1.9132366180,1.4109679461,0.6244448423,0.4116883576,0.4919942617,-0.6390546560,-0.0430185869,1.6317751408,0.2780894935,1.6450288296,-0.3439806104,-1.5669082403,-0.0530644134,1.5441375971,-0.4733426571,0.6746113300,1.0411593914,0.9855762720,-1.7978531122,-0.9804763794,0.6059101820,1.1089842319,-0.8183146119,-0.9320350289,-1.1980115175,0.1660242826,-0.7525303960,1.5922327042,-0.8294901252,0.9423011541,-1.8002551794,0.0447457209,1.4288845062,0.5406267047,-0.1151995510,0.8838722706,-1.7344129086,0.5595968366,0.4175852537,0.6582928896,0.7738341093,0.7438519001,1.0156049728,-1.3063168526,-0.9007453322,-0.5195229053,-0.0660142750,1.7721335888,-0.5381574631,0.0451898575,-0.8444939852,-0.0029482909,0.9423898458,-0.8007932305,2.1239731312,-0.2745014429,-1.9707047939,-1.2178057432,0.7433405519,0.2328127027,1.4669008255,0.1985478550,0.8833305240,0.4210582674,-0.6766483188,0.7286726236,-1.7391421795,0.4351043105,0.6838805676,0.1998222470,1.5584337711,-0.7463527322,1.0878604650,-0.5838196278,0.3922228813,-1.2806158066,0.0529088899,0.3663826287,-0.1530306935],[0.0078097042,0.3014991581,-1.7583640814,0.5166698098,-1.0459570885,0.3360862434,-0.3021540940,1.6351559162,0.2339851409,1.3475879431,-0.2810580730,1.2364726067,0.8393220305,-1.1669842005,-0.9311234355,0.3072465360,-0.7316344976,0.6209425330,-0.4304316342,1.0347040892,0.3869737089,-0.8412460685,0.9965993166,0.2931939960,-1.5351545811,-0.2541728318,-0.5098071098,-1.9362185001,-1.0679433346,1.2295447588,-0.6092607379,-0.2998788953,-0.2101449221,1.5696338415,-0.9006688595,1.9490916729,-1.1761932373,1.4566138983,0.7829972506,-0.9729502797,-0.6038942933,-0.3258980215,0.3648591340,0.6088670492,0.5756407976,0.1295090169,-2.0584166050,0.7405277491,-0.0565741882,1.6712781191,-0.0073989523,-0.7615351081,-1.7625714540,-1.6687066555,0.2233912647,1.7904453278,-0.2203775346,0.3719655573,-1.2596836090,-0.1352416873,1.6784518957,0.9244667888,0.6470136046,0.1950820833,0.2686503530,-0.3164552450,-1.4811400175,-0.4500595033,0.9400957823,-0.8417010903,1.2031250000,0.1384577602,0.7147232890,0.8561578989,0.8689596653,-0.7192590237,-0.4665651917,-0.1493928432,-1.8951672316,0.9723851681,-1.4815393686,-0.3151030242,-2.1755998135,0.3345840275,0.4477573931,-0.8480839133,1.4286578894,-0.5938410759,-0.3036641181,-0.1643898785,1.4175655842,-1.2077329159,0.5321143866,-0.3946001530,1.0930041075,0.8004240394,0.0015130466,-1.2983140945,-0.2006176710,-1.0674712658,0.6152316332,1.3683578968,0.5303028822,-0.3842371404,-1.3599123955,1.2266104221,0.7833281159,1.6608939171,-1.1905647516,-0.5298700333,0.9390087724,-0.7273864150,-3.0896890163,-0.9112982154,-0.2719693184,0.4283858538,2.2000403404,1.2904229164,-2.8206224442,1.5973047018,-0.3084849417,-1.2416521311,0.3893934488,0.9960338473,-1.6180120707,-0.7857273817,-1.6723463535,-0.1164723262,-1.4589574337,-0.2019911110,0.2670248449,0.9341351390,-0.5450060368,-0.9017308354,-1.1602119207,-0.9146684408,-0.4528957009,-0.7189208269,-0.8239266276,-1.4149652719,1.0071654320,-0.7333939075,-1.2482929230,0.3368687928,1.5294657946,2.1932830811,-0.0322217532,1.3643707037,-0.7223550677,-1.0060669184,0.4713765979,0.0374758020,-0.2954056263,-0.5196225643,-0.3418472409,0.8694543839,-0.0366826206,-0.3693021238,-0.0720364004,-0.3977731168,-0.2981844246,0.3481273651,-2.2333576679,0.1124164909,-0.5860416293,1.2649736404,0.4698347449,-1.6995165348,-0.6768918633,0.5547665358,0.1519433111,0.4794569612,-2.0995452404,0.1126223803,-0.5789268613,0.7632908821,0.6681728959,-0.1237390935,-0.2411789000,-0.8700891137,-0.0535563789,0.1604736447,-0.2040840238,-0.3879729211,0.3112916052,0.7895922065,-0.0806431398,-1.9122922421,-1.2963095903,0.9608740211,-0.5996281505,-0.2180650234,-0.1556250751,0.5601133108,0.5118433237,-0.6297109723,-0.2477449030,0.4377675653,0.1655773968,-2.1302938461,-1.1546865702,1.3413959742,-0.7347362041,0.8989260197,0.4398216605,-0.8333153129,-0.5954267979,-0.4634758532,-0.0472172499,-0.1780685633,-0.6798447371,-1.1439967155,-0.0540322363,-0.7338622808,-0.3657207787,-1.3311915398,-0.0009648965,0.8482714295,-0.3821905851,0.5203379989,-0.4978805780,-0.6867644191,-0.3023690581,-1.7219276428],[-0.0706413612,-0.9097322226,-0.6544209719,-0.6343010068,0.1573838145,0.8664366603,0.7233697176,-2.0078625679,0.2135981321,0.4005902708,0.3631705046,-1.0551964045,0.1154454425,-0.6361332536,-0.7936405540,-0.8201953769,1.5267411470,-0.3820387125,0.8344710469,-0.4640203118,-0.1984673291,1.1495521069,-0.2665902674,-0.3317551613,-2.1129162312,-0.9538705945,0.7641342282,-1.5903077126,0.8508225679,1.7642945051,-1.1222983599,0.5385093093,0.4542241693,-1.7904034853,-1.0321934223,3.2340333462,0.3957667649,1.8712517023,0.1348981857,0.3125168383,0.2443830669,0.4996775389,0.2636229098,1.9391344786,-0.4150807559,-0.5804721117,-1.0143501759,0.1284436435,1.1321600676,0.3555642068,-0.1672841161,0.6568336487,1.0217926502,1.1322041750,-0.7075734138,-1.3878320456,0.4098901749,0.2341521829,-0.1374025643,-0.0444334671,0.4156866968,0.9274237156,-0.5546074510,0.6505785584,-0.1995093673,-0.0293728597,-0.3378144503,0.5294931531,-0.8031910062,2.0809404850,-1.1738935709,0.7713395357,-0.7774733901,-0.5353500247,0.3451361656,-1.3199455738,-0.3418870568,-1.0163743496,-1.1710753441,-0.2852978408,1.0949852467,1.3117394447,-0.8261899948,-0.6163251400,0.0279291794,-0.4021877348,0.5516725779,0.5848386884,1.0316517353,0.3473063111,0.6851702929,0.8027648926,0.5897445679,-1.0397065878,0.8500480652,-1.5022568703,0.5201315880,1.5309634209,-0.0968860835,1.6017364264,0.3336514235,-0.0345757678,1.5534172058,1.7362976074,0.6443283558,0.8172535896,-2.2950198650,-1.5091173649,-0.1997069120,1.0716969967,-0.1073218584,-0.7956750989,-0.6962655783,0.4508700073,-0.3004487455,-1.1373023987,1.7564595938,-0.1807855815,-0.3760594130,-1.2967089415,-0.6098486781,0.4165080786,-0.8964832425,1.7748460770,-0.7647678256,-1.5128406286,-0.3202617466,-0.2043563724,1.0446738005,-0.7297920585,-0.0262035001,-0.1444185227,0.3840039670,0.2060625702,-0.2181037962,-1.1294969320,-1.0868163109,0.3352339864,1.9930715561,0.5076326132,1.3730456829,-0.0023539420,-1.5736697912,-0.3311335146,0.9903514385,0.0521080866,-2.4414179325,-2.5822451115,0.1565915942,1.4188001156,0.7368789911,0.2345260978,1.0152734518,2.2886154652,-1.0148390532,-0.3122152388,-0.2702304125,-0.5222117305,1.4889755249,0.5128302574,-1.7104084492,0.1142061651,-0.2926707566,-0.8836719394,1.8603035212,0.1356332749,0.8755128384,-0.9803749323,-0.2293416560,0.2006159127,1.7615724802,0.3264065087,0.6724275351,-0.0469790995,3.1880333424,0.4425591826,-2.3789470196,-2.4320871830,-1.5881433487,-0.4209427834,0.4819173217,-0.5247082114,-1.5172423124,0.8938916922,-1.6846462488,0.2446191460,-1.0680624247,-0.3203495741,0.0754074827,0.9786596894,-1.0031156540,-1.4559154510,0.3935267925,-0.3842234910,0.4690336883,-1.0834065676,1.4030132294,-1.4369273186,1.1485811472,1.4488285780,-0.9468003511,-0.5519796014,-0.4509304166,-0.4519639015,0.0976786837,-1.9000874758,0.7958002090,0.1177812293,-0.3470512629,0.0172776580,0.8916116953,0.1383883506,0.1076137498,-0.1316533536,-1.9620102644,-2.0274169445,1.5324951410,0.7383908629,-1.1564779282,1.5768051147,-1.4502425194,0.9668387771,0.3540835083,3.4085419178],[1.9100345373,-0.8650856614,1.4155201912,0.5690464377,0.5973732471,-0.8678830266,0.8286939859,-0.2351627350,0.9233389497,-0.5177605748,0.4496781528,0.0800872520,0.2445594221,-0.2652956247,2.1359424591,0.1638048589,-0.7652742267,-0.8763234019,0.3231536448,-0.6016790271,0.2801153660,0.8367733955,0.5688657761,0.6027054191,1.1826387644,0.0957681388,0.1674305946,-0.1956560016,0.5108280778,1.0816714764,0.6208047867,-0.1561427861,0.8358264565,0.9209582806,-0.8191552162,-0.1509196013,0.8101909161,1.2557537556,0.2925243676,-0.0419176333,1.1210883856,-1.0228914022,-0.1098661944,-0.8591143489,0.2410670966,0.9973797798,-1.4664530754,-0.0980027616,1.8604106903,-0.6354730725,0.2107730508,0.5374374986,-1.1003706455,0.8711256385,-0.6260376573,0.2263965756,-0.0102861263,-0.1871914864,0.8020673394,0.5475625992,0.2951956689,1.2020192146,-0.4511943161,0.6142093539,0.0216287971,1.3507753611,-0.3723137379,-0.1512116939,-1.7927660942,-0.4930469692,0.6896463633,-1.8515053988,-1.6282465458,-1.2689808607,0.3326217532,2.1312882900,1.7744779587,-0.8050757051,-1.5977858305,-0.3082677424,0.1215020418,1.4367951155,1.2268460989,-0.0081329355,-0.5573827624,0.1035985723,-1.2760721445,-1.1780765057,-1.2280064821,1.1160129309,1.7673761845,1.7334949970,-1.7602155209,0.5039381385,-0.3720277846,0.3963406086,1.0447177887,-0.8420517445,-0.0861799866,-0.1530893594,0.2599826753,1.1811122894,0.5532831550,-1.3378735781,-0.8013755083,-0.8412433267,1.0781079531,-0.0028998200,-0.6660804749,-0.7412942052,2.3283331394,-0.4305628836,2.7005894184,-0.6134419441,-1.0312664509,1.0619452000,-1.1738936901,1.2846373320,1.7262413502,1.3725773096,-0.7013794184,-1.3772867918,0.4172764719,0.4020915329,-1.1206417084,-1.2978309393,-0.0802910849,0.9490975738,0.0574220717,0.1611342430,0.6615610719,-1.6403830051,0.2258825600,-0.2138030678,-1.7920988798,-1.5084474087,1.6496242285,-1.0713797808,-1.8907377720,0.0772534534,-0.2337310910,0.0140136406,0.5051288605,0.4664832950,0.8642095923,-1.8167469501,1.0395417213,-1.1637365818,1.1356519461,-1.7042986155,-0.0973814875,1.1059724092,0.6865385175,-0.0097261248,-0.0064713955,0.2397290915,0.4610339105,2.3903193474,0.5248602629,-0.8353090286,1.6538916826,-0.5744239688,0.4300434589,-1.4513535500,-1.2397629023,-0.9852696657,0.1993034035,0.0661344454,-0.5709816217,-0.0929837227,0.2432375997,-0.0583449788,0.0009275326,0.8209046721,0.0518882200,0.8052539229,-1.6794208288,0.2178734690,0.7934340835,-0.8915417790,-0.9696401358,0.0556026287,0.7279754281,0.4486949742,1.3063867092,0.0655736998,-0.5748510361,0.0804484934,-1.2538518906,1.0344552994,0.0385405272,0.1673927009,-0.4514338970,-1.3391981125,-0.3614654243,0.9321334958,0.4562244713,0.2335039377,0.1003533304,-1.6953806877,0.1109338477,0.4388502240,-0.1420196295,-0.5670718551,-0.4239734113,0.6679923534,0.1954314113,-0.7743871212,0.7897585630,0.2001348287,-1.4090903997,-0.6529703736,-0.5033426285,0.8819975853,-0.8331340551,0.4697431326,0.1163358986,0.7791193128,-0.4779369533,0.3048064709,-0.5940655470,-0.4552982152,-0.4650419652,0.0361061431],[1.5899168253,0.0063533471,1.6877865791,0.0515951440,-0.9018155932,1.2319109440,-0.6424393058,-0.2568293810,0.6929336190,-0.3502320945,1.4825685024,1.3803684711,1.1104739904,0.6726384163,1.6660864353,-1.1913731098,0.4051263034,-0.3095752001,-1.2381680012,-1.6911195517,1.1668710709,-0.6490058303,0.4584015310,-0.9608908892,0.0266461428,-1.2123868465,-1.4942896366,0.1524978429,-0.6395063400,-0.3380450904,-0.9216715693,-0.0510670282,1.1863936186,-0.6247690320,0.3869996667,-0.5245535374,0.3170322776,-1.0178136826,-0.1742559820,-0.1370685846,0.8142974377,0.7342265844,-1.9699764252,2.2751204967,-0.3226063848,-0.5424374342,0.6009609699,-1.7587728500,-1.5937917233,0.9980217814,0.4808895290,0.5623027086,0.3096584976,-1.2413680553,0.0726389959,-0.2213159949,2.1800575256,-1.4602934122,0.0018683164,-0.1173354015,-1.0309307575,-1.3381360769,-0.7076413631,-2.1738369465,0.6837378740,-0.7091113329,-2.7662103176,0.7608048320,1.0504329205,-1.4428687096,-0.7895605564,0.7236565351,-1.1905863285,-0.6919526458,0.8573622108,1.5371603966,-0.9198681116,0.7844922543,0.6338860393,-0.4500072896,-2.0559315681,-2.0172803402,-1.2834616899,-0.3556898534,0.6204849482,2.3478055000,-1.3467277288,0.1222354546,-0.2214947045,-0.2462989539,-0.0867773294,0.6118643284,0.6414287686,0.4616425931,-0.1879353225,-0.3813198209,1.3266900778,0.4950915277,-0.8527886271,-1.9915175438,-0.0268161800,0.2655802965,0.3149074912,1.0536828041,0.8504412174,-0.7472555041,-1.5911126137,1.3834769726,-1.4433643818,0.6882103086,-1.4758917093,0.8962612748,-0.6082796454,-0.8149115443,1.3695408106,-1.9525241852,-0.5935515165,2.1671218872,-0.9892462492,-0.2221836448,-0.2219780684,0.9403944612,0.1978086531,-0.7817671895,-0.4124997258,0.3687230051,1.0968439579,-1.8594348431,0.0029549592,0.1101333052,1.5394915342,-0.0038964562,1.0981504917,-0.0400299132,0.2016086131,-1.3140232563,0.2941615582,-0.7528982162,-0.0452784672,0.7580948472,-1.3945977688,1.3409752846,-0.3101814091,-1.2464029789,-1.1343907118,1.7912290096,0.7197954059,-0.5015527010,-0.1126659065,1.1124942303,1.3593585491,-1.0211582184,-0.7661721110,-0.6488007903,-0.9884676933,0.7920768261,0.5087880492,1.3648190498,0.6742984653,-1.6158971786,-0.8027673960,0.2757093608,0.2329666018,-0.6251592040,0.0634803176,-0.1018172055,1.5068968534,0.6900904775,-1.7025721073,0.3553458452,0.3871901035,0.9579328299,0.7569516301,0.7862011194,1.3912926912,-0.1798933148,-0.3273005486,-0.3807692528,-0.6068280339,-1.0164443254,0.7183098793,-0.3550116718,-0.0208350029,-1.5521923304,-1.1644581556,1.0302523375,1.1733825207,0.2134277225,0.0292529687,-0.5484084487,-0.3583379388,-0.8931321502,-0.1975669116,0.2763205469,1.1366759539,-2.0308594704,0.3366666138,0.3184001446,0.4373573661,-0.8740926385,1.0525960922,-1.2406895161,-0.5679717660,-0.5852907300,1.1207735538,0.4359603524,-0.8621008396,0.2411166579,-0.1937544495,0.6315796971,-0.4637977481,-0.8728907704,-0.3243234754,0.3536755145,-0.2431924790,-1.2344973087,-1.1073101759,1.7516149282,-0.4201321900,-0.6599223614,-0.6240724325,0.0641648844,0.8778529763,-0.7815441489],[0.8508049250,-0.8690682650,0.9813565016,-2.0868430138,-1.1037631035,-0.1131165251,0.0414229818,-0.8837306499,-1.4935542345,1.1044009924,-3.1893181801,0.5092691183,0.4799568355,1.3159337044,-0.0970403627,-0.2791163325,1.7901148796,1.5886393785,-0.3469516039,-0.9854775667,1.3777000904,-0.4683577418,1.6312767267,-0.5955067277,0.6797708273,0.6795912981,0.5310679078,-0.8304234743,0.7193893194,-0.0236661416,2.5271956921,-0.7473905683,-1.0720716715,-0.1273948997,-1.3011174202,-1.6902400255,-0.4170007110,1.1451983452,0.8875586390,-1.9904561043,-0.6132237911,0.0021884013,-1.0157800913,-1.2629107237,-1.6704556942,-0.9015039206,-0.7487605810,-0.0734295398,-1.4195762873,0.9139781594,1.6136165857,-0.3155820668,-0.5592501163,1.7205772400,0.1843333095,-0.2459741086,0.7138553858,0.9178287387,-0.3423841894,0.6098057628,1.5959429741,0.6128612757,-0.6838819981,0.6335126162,-1.2277721167,0.0185308941,-0.8606790900,0.1154111698,1.4965243340,-1.1527279615,-0.6795991063,-0.9856798053,-1.2678070068,-0.4434808195,0.0805942714,0.2353551388,0.0121626621,0.5561687350,-0.0758090466,0.1448151320,0.2649245858,0.7526773214,-0.2883499265,3.2922325134,1.2451647520,-0.6571530104,0.7045717239,1.0751260519,0.2896658480,-0.0684312060,-0.7031256557,-0.7291713953,-0.4636211991,0.5477852225,0.4574085772,-2.8637101650,-0.3515471518,-0.6543455124,1.0587671995,-1.0377566814,-0.1081912965,-0.7057687044,-1.2898498774,-1.6401636600,-0.0102215903,-1.6310093403,-0.6036027074,-0.3954981267,-1.0030182600,-0.6082266569,2.5081093311,1.6789019108,0.2176964432,1.2520422935,-0.4151733816,-1.2843372822,-1.0400305986,0.3440242708,-0.4483019412,0.6690663099,-0.7175396681,-0.1458407640,2.2244884968,0.1947859079,0.1055293828,-0.6040461063,0.8576923013,-1.5997809172,1.8376449347,2.8909277916,-0.5177230239,0.2298939377,1.8100821972,-0.5709523559,-0.5115388036,1.2967069149,-0.5621591806,-1.1462398767,-0.9591106772,-1.6882694960,-0.7014160156,0.6921666265,-1.1444196701,-0.1640264839,0.4573389292,0.5338413119,0.8252105117,0.1567942351,1.4680320024,-0.2286765873,-2.0002055168,-0.6564577818,1.9003719091,0.4424801469,0.4283055365,0.6784015894,-0.9360645413,0.5108249784,-0.0562780872,0.0913101286,-0.8051065207,-1.8382200003,1.4490640163,0.5964669585,1.4912917614,-0.3538664877,1.1519131660,-1.2095607519,0.6885273457,-0.0599056147,0.8238558769,-0.0441347621,-0.9308155775,-0.7122085690,-2.2147786617,1.1632386446,0.6279171109,-1.3401852846,-0.9253047109,1.6279819012,0.5150529146,-0.6503204703,0.1271840185,-0.1690019965,-0.5984864831,0.5934988856,1.3468483686,-2.3228442669,0.3811374903,0.0807556435,0.4608016312,1.1064083576,-0.9534149766,-0.5410642624,1.6721212864,0.3387215137,-0.5848330259,0.1684350818,1.1708821058,-0.0112401946,-0.0421408676,-0.0110325404,-0.0399069786,1.8626705408,-0.3295717239,0.2178692967,0.3128709197,-0.6914327145,-0.0489474349,-0.4623219073,0.6858608127,0.4037072062,-0.4837623537,-1.6117572784,0.0941191539,-0.6782262921,1.2615525723,-0.2805505693,2.6711010933,1.3798089027,-1.1973799467,-0.4020443559,-0.5639002919,1.5455411673],[0.6903855801,0.9709954858,0.0853693932,1.2079653740,-0.1136911586,1.1985101700,-0.8023996949,0.8236210346,-0.7027028799,-0.5922062993,-2.2783963680,-0.3009352088,-0.7893054485,-0.2613692284,1.8687489033,0.1451711506,1.1809588671,1.4143635035,0.2772610784,-0.3749769330,0.6385892034,0.7431727648,-1.3622537851,1.3739572763,-0.9690158963,1.0907326937,-1.2023899555,1.2161438465,1.5248703957,-0.1389713138,-0.3804358244,-0.1093302220,0.5503053069,-1.3575862646,0.4582281709,-1.1639021635,-0.3647545874,-0.7115260959,1.4336994886,0.2221960723,2.0658452511,-0.2943881452,-0.7061772346,0.6280207634,0.5662272573,0.3857337832,-0.4284617901,-1.3776395321,-0.6398698092,0.4830209613,0.9864068627,-0.7793473601,-0.4055784643,-0.5901540518,-1.5036782026,2.4848642349,-0.3595902622,0.4562184513,-1.2775880098,-0.4926495552,-1.4293605089,-1.3942863941,0.7871758938,-1.9546544552,-0.9582464099,-0.8009232283,-0.3317953348,0.5323982239,0.4935196638,0.8553711176,0.1533751488,0.9288468957,0.1826938540,0.8980613947,0.4672872722,-0.0300932415,0.3488621712,2.1993985176,-0.3082002103,0.6237960458,-0.5904917717,-0.5733503699,-1.4770421982,-0.3424594402,1.8895410299,-1.0194392204,-0.7236206532,-1.3778822422,1.0129799843,1.1487203836,-1.0671883821,0.8054333925,-1.6088992357,-0.9679784179,0.4632321298,1.2936443090,0.6811715961,0.0065504359,-1.3251696825,-0.5823521614,0.3334719539,-1.1553093195,0.6754829288,-0.6283892393,0.0070697991,-0.1301544905,0.0000159070,1.6803572178,1.0456736088,-1.2322541475,-1.7359474897,0.8063043952,-0.3912761509,1.4771649837,-0.3736453354,1.5715459585,1.1237367392,-1.5193575621,0.5520567894,0.5154913664,-0.0857104436,-0.9226989746,-0.1511355489,-0.1532225013,-0.4101878107,2.1041259766,-1.2773672342,-0.4122236669,-1.2621785402,-0.6445494890,0.2983115911,-0.4472069144,0.0983226672,0.2284848690,0.4287489355,-0.2973191440,-0.8167265654,1.2091214657,-0.4534170032,0.9312779903,-0.3093512654,-0.3528625667,-0.0741263479,1.0887343884,-1.2669620514,0.3354581296,0.0044241310,0.3022244275,1.1900383234,0.2975670993,1.4324852228,0.6217397451,-0.5294061899,0.4970742464,-0.5087625384,0.1304465979,-0.1666449457,0.5193110704,0.8491689563,0.5580003262,-0.5960376263,2.3947246075,-0.5828321576,-1.2493512630,0.7954925895,-0.5065029263,-0.5937213898,0.9579040408,-0.0251192842,-0.5465407968,1.1391458511,-0.4674501121,-0.1967873126,0.2859996259,0.9837669730,-0.9668199420,-0.1133453920,-0.2571675479,2.5318741798,-0.5155376792,0.8728454709,0.0778044462,-3.1305611134,-0.4914603829,1.1896528006,-1.4559942484,0.3755713403,-0.5113667846,-0.4030328989,-0.8139418364,1.3834196329,-0.1454866529,0.2846169770,1.1209895611,0.8712860346,-0.0718860626,-0.6956607103,-0.4459330440,-0.4794711769,1.5473222733,0.0038444141,-0.9808133841,0.2412267625,0.9256925583,-0.5301594734,-0.4780997336,-0.6812683940,0.5242022872,0.1786995083,0.8638377786,-0.3517528772,0.5365935564,0.5407190323,1.1510950327,-1.0237739086,0.6385636926,2.2565796375,0.7124504447,-1.9051090479,0.5709059238,1.9324462414,-0.8584792018,0.8580051064,0.2227552831],[-2.1613814831,-0.2940069735,-0.1814101785,0.7617862225,0.6633594036,0.1448772103,0.4526188672,1.6508759260,-1.7932395935,-1.7927554846,-1.2987227440,-3.7009484768,1.4530611038,0.2657386363,-1.1115087271,0.3476705849,0.9187952876,-1.2118660212,0.9229725599,1.2597514391,-1.9153442383,0.4390523732,-1.1919585466,0.2263162732,0.8045522571,-0.8975812793,0.1102162153,-1.7435343266,-1.5475306511,0.1274931878,2.3257565498,0.2937721014,-1.1089141369,0.5591292381,-0.3039319515,0.7630262375,0.2723177671,-1.6435691118,-1.1236854792,0.1609627306,0.6847237945,-2.4690279961,0.0209366865,-0.5310238600,0.4430907667,-0.3956150711,2.5748353004,0.9656668305,0.5004914999,0.4974248707,0.1869066209,-0.3793111145,-0.5843949914,1.0160140991,1.0422772169,0.0353021547,-0.2838065624,2.4264302254,0.1781947762,-1.4493917227,-1.2831250429,-1.2559127808,0.8513959646,-0.4708440900,-0.9816243052,-1.4453076124,1.0640504360,-2.4597752094,2.4951465130,-1.3210877180,-0.3973611891,0.6279271245,1.2438503504,-0.8696066141,-1.6361762285,-0.0915040895,0.5303424001,0.1272949874,0.6879992485,0.6507039070,1.6750425100,-1.7682852745,0.4391796291,0.0041834023,-0.2630576193,-1.8533036709,-0.8588609695,0.1978735924,-0.7241343856,-0.4095613658,-0.6204192638,0.2809041440,0.0266254898,0.5631239414,-0.0666484684,-1.3585243225,0.1720818728,-0.9857338667,0.6432245374,0.4694710672,0.3492645621,-0.5567824244,-0.4697691798,-0.4117129147,2.1153721809,0.5442541838,-1.6541137695,2.1774716377,-0.1963181496,-0.8331007361,0.7331581712,-0.0479035601,-0.4869109094,0.6976183653,-0.1689549834,0.6171053648,1.2108881474,0.5635103583,-0.8354082704,1.2755782604,0.9359452128,1.2179498672,0.2265103161,1.6606619358,0.5011734366,0.1251170188,0.2605842352,-1.5394775867,0.4511664212,-0.2313937992,1.1197661161,-0.1140006483,0.3820356131,1.3873690367,0.6571901441,0.5457989573,-0.6308066845,-1.1354978085,-2.4026119709,-0.3027979434,-0.5020709634,-0.2321306318,-0.0201238710,0.9409797788,0.7256790996,1.3103921413,-1.0225174427,0.8154883385,-1.1430218220,0.5008414984,0.3574302495,0.0495680012,1.5883718729,-0.1069783717,0.1745323092,-0.0496963039,1.0220882893,0.4006874859,-1.0913971663,-2.3291058540,-0.6900204420,-0.9169560671,-1.1179387569,-1.0036914349,0.2173535079,1.0790767670,0.0816364065,0.2541635036,-0.8011046052,0.4594263136,-0.6509816051,1.1677899361,-0.3477513790,0.5618755221,1.1921788454,1.5922267437,-0.2455349416,-1.0317528248,0.2817627192,-0.8351853490,2.2162134647,0.3737241030,-0.9866889119,-1.3697648048,-0.1978318542,-1.4279685020,0.1537464261,1.1027641296,-0.4549758434,1.3040695190,0.9517257810,1.3341605663,-0.5766361356,0.4542465210,0.6612707376,-1.2149206400,0.7250176668,1.0353204012,-0.3413012028,-1.3613491058,0.4005934596,-0.1110523567,-0.0788928047,1.1674675941,-0.3902478814,0.8044789433,0.7681032419,0.0321332254,-0.2218496799,-0.8551516533,-0.4030755162,-0.0721132234,-1.2544465065,1.4844565392,-0.5042630434,1.1852766275,-1.2150322199,2.2204818726,-1.1828657389,0.7846149802,-1.9904559851,0.1393995136,0.6561467052,0.6043061018],[-1.0907354355,-0.7484697104,-0.7126812339,0.8230614066,-1.3216540813,0.1326915920,1.2171856165,-0.4447000027,0.8073532581,0.6256611347,-0.0607976690,0.7808696032,0.3649677634,0.5778753161,0.4065142274,-0.5495318174,0.4771209359,2.5231893063,0.6165393591,-0.5275948644,0.5816348195,0.5120906830,-0.7348868251,-0.4592056274,0.4749583006,1.1358423233,0.6339197755,1.2536251545,0.0191743113,1.3371372223,0.6908062100,-1.5238791704,0.7582221627,-2.0854206085,1.0246943235,0.3916818202,-0.3441234529,-0.3393303752,0.5980219245,1.7809047699,-0.0322760791,-0.5638660192,0.7236633301,0.1639522463,1.5012469292,-0.3719091415,1.1622062922,2.8788545132,0.0829108357,-0.0003996070,1.1071796417,-0.9313130379,0.1179228798,-0.6248297095,-0.6864803433,0.3658074439,-0.1976805627,-0.7697430253,1.1433596611,0.4378886223,1.0920422077,-0.1473139822,-0.4550720155,0.8509523273,-0.4368683994,0.1370265186,0.9994106889,0.0470677242,-1.2024868727,0.7732259631,0.7193604708,-0.4950317442,-0.6972939372,-1.1340928078,1.4360136986,-1.0777835846,0.6327699423,-0.7165985703,-1.7260651588,0.7511957884,-0.8365247846,1.4378730059,-0.2593916655,-0.7531266212,-0.0018816765,0.2780501246,0.3728078902,0.5840899348,1.1903779507,-0.7974692583,0.6395269036,-0.1412059069,0.5012361407,-0.1481624842,-0.0530277118,0.2532734871,0.7361589074,-0.5505169630,1.3500739336,-0.7229986191,0.5972332358,1.5588985682,-0.9929527044,0.6563365459,2.0401813984,0.2912432551,-1.0593951941,-0.7136363983,-0.4101233482,-0.8217135072,-0.9647257328,1.5899295807,1.4830676317,-2.0082218647,0.6201471686,0.0921110585,1.0209305286,-2.2872138023,0.2827239335,-2.0732119083,1.1393668652,-0.8216959834,0.2826004326,0.0577638298,1.0376316309,-1.2894252539,-0.5568271875,0.3712979555,1.0854558945,1.6734513044,-0.2837608159,0.7808902860,1.2088090181,-0.3496015966,-1.8581924438,-0.3988831639,-0.4860283136,0.5964819193,-0.1913231909,-1.8144276142,-0.9861682057,-0.9875866175,-0.3508193493,-1.0703758001,0.9739978313,1.7451748848,1.1622909307,1.2399636507,0.1838206500,-2.1057317257,-0.3868809640,-1.4580152035,-1.9583094120,-0.5214228630,-0.8149047494,-0.2598052323,-0.0679654256,1.5236089230,-1.8181490898,0.7602464557,0.2036284059,-0.6710840464,-0.4183948934,1.6409194469,0.0934851244,-0.9035621285,-0.1599463671,-0.2006375492,-0.1909182221,-0.9695960283,0.5366060734,0.0499979891,-0.5023054481,1.6040503979,-1.2374203205,-0.2361531258,-1.3638973236,1.9122620821,-0.8687421679,0.5375559926,0.4242394269,-0.5340645313,0.7021860480,0.2912759781,-0.4078489244,-0.0786173418,-1.5945433378,-0.3502337337,2.0544710159,-0.1928384900,0.9104574323,-0.0862179101,0.6111246347,-0.3534440100,0.5634724498,0.4857985377,0.2631002367,1.4814050198,1.1656149626,1.1763496399,-1.2619029284,1.0174994469,0.8297733068,-0.5920416117,1.5649130344,-0.7932690978,-1.9636081457,0.0073007345,1.7777563334,0.4939030111,-0.8862183094,0.5388250947,2.3175821304,-0.6583781838,0.0406383574,-0.6774467230,-0.0744541883,-0.6628022194,-0.7723457813,0.8113358021,-0.0109207612,-1.4630174637,0.8977974653,0.0017468558],[0.8730773926,1.8256112337,0.9446329474,1.9473477602,0.5063759089,-0.2418232411,1.6875723600,0.5409396887,-1.7663731575,0.7723557353,0.7904483080,-1.1151059866,0.6893996000,0.0964727402,0.3555314541,2.8903863430,-0.2023495734,1.1651166677,-0.1074468717,-0.4425264895,0.3537662923,-0.5301949382,-0.1266164333,0.1925617903,1.4034749269,-0.4170812666,0.2439171225,0.6235732436,0.0569755919,0.4600847065,1.2430694103,-0.2150766850,0.5950499773,0.1923431158,-0.1020159572,-0.9160646200,0.8229887486,1.0482641459,0.5558660626,-0.7286831141,1.7109791040,-1.3896939754,0.7022241354,0.7677382231,0.8328258991,-0.7697517872,0.6147429943,2.0839247704,0.7161874175,-0.3919250667,0.9053443670,0.0932821408,-0.7115653157,-0.2126543075,-0.8772642612,-0.1721208841,1.3694014549,0.2511317730,0.1415636986,0.0112932771,-0.6726903915,-1.6037570238,-0.5378059745,0.0845245272,0.7646923065,-0.2101514488,0.9166517854,0.4953337014,-1.5049014091,1.1143501997,1.6012240648,0.6021217704,-0.9676260948,0.5672973990,-0.0398906879,-0.2725612223,0.6513987184,0.5207162499,-1.9729640484,0.8367348313,1.0528236628,-0.4983050823,0.5411375165,-0.6223208308,-0.5388869643,0.9614911675,-2.0136291981,2.0114109516,0.2237331420,0.7709437013,0.1642037779,0.9776116610,-0.2966648638,-1.1799397469,0.7032052279,0.0894018561,0.7504652143,0.3706242144,0.5047916770,0.6028758287,0.4940436482,0.8593719602,0.5868232846,0.1310377866,-0.3590817153,-0.8620396852,0.6448566914,0.6952871084,0.9641913176,0.1289675683,0.9945142865,-0.3077834547,-2.1486518383,1.8851884604,2.6745195389,-0.3878851533,0.3051368296,-1.3988252878,-0.6323857307,-0.8161635399,-1.3922702074,-0.6827468276,-0.2562257946,0.5322270989,-0.7437602878,-0.2868638039,0.7098515034,-1.4383655787,-0.6667670012,-1.3716266155,-0.1098348424,0.7378181219,-0.0879292637,1.1370954514,1.7925657034,-0.2012682259,-0.2694388330,-0.4670928419,-0.5821979642,-0.7644075751,0.6695444584,0.0306869764,-1.6800422668,-1.6340225935,-0.4223179817,0.7865099311,0.1305941641,-0.7552067041,-0.0254859049,1.9027297497,-1.3717647791,0.7811303735,-0.2215402573,2.1291007996,1.1312514544,-0.2409127802,0.5062365532,-0.5534172058,-0.3004655242,-0.5346511602,-0.8108330965,0.3282672167,-1.0185667276,-0.3533863127,-0.8215651512,0.2005594671,1.8420407772,0.3635246158,0.7861365080,-1.0597175360,-0.1873623729,0.7724383473,1.2314850092,-0.7266256213,-0.2569041550,2.0182263851,0.4881033897,0.3779078424,-1.1142752171,-0.5767669082,0.1944286525,2.3854238987,0.7869626880,0.8664147854,-1.8784008026,-0.5955214500,0.6883810163,-0.6199914813,-1.1991696358,1.2152301073,-2.4070537090,-1.0307068825,-0.6643091440,-0.5183378458,-0.4899657965,-1.8663284779,1.7655661106,-0.7547045350,-1.4823474884,0.0760180503,-0.1749891788,-0.3796023130,0.0475712158,1.7384783030,-0.2914048135,-0.1437018216,0.1325060278,-2.3676879406,-0.2258290648,1.6418195963,-0.7734972239,0.7461740971,0.6074157953,0.4117653370,-0.6067041159,-1.6472831964,1.7200328112,0.0232646484,0.4374954998,1.2455363274,0.5894487500,-1.7584183216,1.0120782852,1.0967348814],[0.0198814012,0.3985802829,0.6743020415,-0.8145176768,-0.7799610496,0.3777892590,1.0087720156,2.0344851017,0.0204913002,-0.0241909269,-1.8100771904,0.7795275450,0.2824576795,-0.6737608910,1.0618784428,-0.3735115230,-0.1141227409,0.0508635230,0.1573044062,-0.7637298703,0.1603048593,0.1740360856,-0.0722323060,1.0070261955,1.3884991407,0.5985660553,-0.5918496847,-2.4490859509,0.3502865136,-0.7734159231,-0.7966606617,1.0198645592,-3.6166062355,1.3282010555,-0.9895777106,-0.0881013423,-0.5323204994,-1.1653339863,-0.9234332442,0.1528675258,0.6556922197,0.2687607110,0.2779617310,-0.4949930906,-1.0904607773,0.9958393574,0.6449023485,-0.0139402952,-0.6758316755,0.0389272645,0.2130638659,-1.5142279863,0.5219237804,0.6738876104,0.0340359621,1.2885986567,0.3472016752,-1.3768179417,-1.1709387302,-0.1141757742,0.3047106266,0.3039871454,-0.5379764438,-0.0836184621,-0.8551713824,-1.5300897360,-0.6361967325,-1.9284130335,1.1020956039,-0.2246899605,0.5269551873,-0.9494376183,-0.9701120257,0.7764113545,-0.2850833535,0.5099344254,-0.1375637800,1.1417900324,-0.2920076251,-2.0219731331,-0.7772913575,0.0708815753,-0.3063801825,-0.4740124345,-0.2298611403,0.7288610935,0.2877394259,0.1773329675,0.4523156583,0.6529164314,-0.7750585675,1.6021527052,0.8939141631,-0.0089290179,0.2659767866,0.1753713489,-0.5379931331,0.2873095870,2.2902197838,0.9296517968,0.3582972586,-1.6173806190,1.1959173679,0.3798225224,0.3370334506,-0.6950842142,-0.1877526492,-1.0495693684,-1.9517418146,0.9486764669,1.3959928751,-0.1565124840,-0.8272699118,-0.6829131842,-2.2610342503,0.2792412639,-0.8317098618,-0.0329254083,-0.0633183643,0.0907527655,1.6180012226,0.1789626926,-0.5308724046,-0.1154055297,2.2669715881,1.0755400658,-1.2605806589,-1.3714374304,0.6071957946,0.6227938533,-0.2147837281,-0.3323607743,1.2683321238,0.0792714432,-0.7269679904,-1.4837813377,0.1482709199,-0.8914306164,-0.3136545420,0.8342787623,0.5124506950,0.3868113160,0.6767075062,-1.5070546865,-1.1736109257,-0.4428335726,-1.2228515148,1.6584920883,0.8860940933,-0.3657595515,0.2537226081,-1.5250725746,0.7131353021,1.0954420567,-0.4071093798,0.6125251055,0.2761376500,-0.2350142002,1.3670639992,-0.3232049346,-0.7421479821,1.3723272085,0.1204612628,-1.4851864576,0.3227433860,-0.8716908097,1.0561125278,1.1365944147,0.4882908463,-2.3186235428,1.2475165129,-1.0494215488,-1.1936898232,-0.8012831211,-0.1370985806,0.7666711211,0.1403384358,0.3451739848,0.7921397090,-0.0043549207,0.4684241712,1.0302612782,-0.1617009193,0.5881261826,-2.0054824352,0.3759513497,1.6423543692,0.8723756671,1.5837312937,1.2748030424,-1.9394344091,-1.3062525988,0.8021035790,1.2067923546,-0.6591643095,1.4958468676,1.4278705120,0.2870118320,0.2882831693,-0.9594781399,0.1612415165,1.7277601957,0.8202464581,1.2654410601,1.6296563148,-0.4528054595,1.2124234438,-0.9555593729,0.7217754126,1.3135437965,-0.9282189608,1.1860563755,1.3845413923,-0.0372342989,-1.0035890341,0.7181358337,2.2787775993,-1.8576666117,0.1417283565,1.1989032030,-1.0669670105,-0.5698820949,-1.3290554285,0.0011848081],[-0.6872947216,-0.0984507203,0.0053924564,1.8844401836,0.6896256804,-1.1454101801,0.2944171727,0.5787634254,-1.6376229525,1.6078158617,-0.6052874327,1.0842406750,-0.6343843341,1.1234267950,0.6614363194,-0.5274475217,-1.6983145475,0.1424814761,1.5018779039,1.6019635201,0.2406056076,-1.3378336430,0.3889307678,0.7279734015,-1.4437752962,0.2451101989,1.3260084391,0.4043070674,-0.6334703565,-0.0905047655,2.2889361382,-1.6737362146,0.4014829993,0.9645870328,-0.3940742314,2.2201044559,-1.1043834686,1.0660539865,0.6407681108,-2.9392998219,-0.1941942126,0.0725432858,-1.6567014456,-0.9918910265,0.0521232933,0.2113792449,1.2878824472,1.0472283363,-0.0297677517,0.6040073633,0.8790017366,0.5193853974,1.0043952465,0.1468016207,0.2064592689,-0.1191236898,-0.7948093414,-2.2682187557,0.3695141971,0.0789367929,-0.2442376614,2.2216601372,2.7800517082,-0.0808964223,-0.9864557981,0.1934024394,-0.1443898380,-0.5223660469,2.2616927624,-0.0752847716,0.6587827206,0.4554153383,0.4489373267,-0.7989477515,-1.1259423494,-0.9596457481,1.1747367382,0.0612658411,-0.9934889078,-1.4369478226,0.4548711479,1.2188794613,-0.8638844490,-1.6974016428,0.3228982687,0.4121471941,-1.0056366920,-0.4581864774,-1.6678930521,-1.3405390978,0.4931705296,-0.0594159402,-0.9209959507,-0.5359746814,-1.2050573826,-0.2416513860,0.0205140915,0.8051583171,-0.3039194643,0.4501983523,1.6569852829,0.8124082088,-0.2882144153,-0.0287091434,-0.8580284119,-0.5605448484,1.5844684839,1.6319321394,0.2433615923,1.8722686768,1.3541325331,0.2546465099,0.7951636314,0.4162734747,0.7153671980,-0.5670320988,-0.4803200364,-0.4373582602,-0.8799190521,0.5002053976,-0.4195810854,1.4329137802,0.7451282740,-0.2605960965,-0.1553190053,-0.1571091563,0.5476624370,-0.5343143940,0.3311653733,0.8613076806,0.4169470966,0.1481061727,-0.9566766620,0.3370350003,-1.6706675291,0.6576215029,1.1566696167,-0.2248429358,-1.4964891672,-1.4325475693,-0.4104234874,-0.0051225480,0.2184714079,0.8567622304,0.0752849728,-0.5590589046,0.0686668381,1.1503990889,1.2555845976,0.8356750011,-1.2136638165,-0.6048707366,-0.0624855757,0.0132966116,-0.0085763428,-0.8367882371,-1.0246938467,2.9996769428,-0.2250829041,-1.3752338886,0.4163417816,1.5430538654,0.8278558254,1.0882321596,0.6639254093,0.0291860960,-0.6185283661,-1.1871991158,0.5030259490,0.8486614823,-0.4215537012,0.5474107265,-0.1873566657,-0.5774893165,0.9267744422,-0.6131139994,0.6064761877,2.2037541866,1.8484804630,0.2465688884,0.4237998426,2.8766086102,-0.6650875807,-0.2597856522,0.4639374614,1.1446344852,-0.6116654873,-0.0218625776,-0.4358089566,-0.5646321774,0.5649581552,0.7810892463,0.7661207318,-0.0211674664,1.7043101788,2.2350108624,-0.1355992109,-0.4077370465,1.7714235783,-0.8152344227,-0.3246816397,0.2365811616,-0.8649201989,-0.8507444263,-1.2081509829,0.7433677316,-1.2326159477,0.6326366663,-0.6106397510,0.0574978702,-0.7415626049,0.9871259332,2.2205648422,0.6921710372,-0.8000081778,0.9471917748,-2.2180683613,-1.0951820612,-0.4547932744,0.7424654365,1.8997622728,-0.5095831156,1.2237663269,1.2456455231],[-1.2396967411,-0.3688427806,1.2640622854,-1.0342392921,-1.4049681425,-1.5210585594,-1.0076230764,1.3294607401,0.6158753037,0.7308965325,0.0848912075,-0.1969574690,-0.0819591358,-0.3404512405,-0.6221250296,-0.7510188818,-0.4768586755,1.2083463669,0.4217658043,1.4242808819,0.0852572173,-0.3053416908,0.9350500703,-0.4598187804,1.8786497116,1.3874312639,0.5443605185,-1.4051083326,0.1745427251,1.4360253811,-0.9045711160,0.0467361361,0.5182443857,1.4112025499,1.2951264381,-0.7757521868,-0.2411681116,-0.4241921604,-0.5597399473,-0.6066641808,-1.1611486673,0.2278152853,-0.0413084850,1.1212860346,0.6273789406,-1.0726985931,0.7191551328,1.1449415684,-0.5668084621,-0.8096838593,0.2828166783,-0.6697654724,0.0814880952,0.4383083880,-0.3998520970,0.6673691273,-2.2718055248,0.4157564938,-1.9803010225,-0.4021820128,-0.9504793286,0.5314682722,1.3823496103,-0.1732390821,-0.3971373439,1.3028131723,-0.8584709167,0.7786294222,1.4142960310,-1.2077190876,0.3213968873,1.8558900356,0.2112760097,-0.3709301651,1.1207209826,1.1596553326,-0.5884460211,-1.1001182795,0.3257922828,-1.1892958879,-0.7330425382,-1.5030636787,0.2551620305,0.2792249322,-0.3615036905,-0.6149381399,-0.5369812846,-1.1995630264,0.5322521925,-1.3133509159,-0.6877193451,-0.0793563500,0.7657913566,1.0005102158,-0.1015864313,-0.6221787930,-0.6660726666,-1.0681372881,1.1551531553,-0.1612300426,0.4519759119,-0.8223438859,0.7364921570,2.1589651108,-0.6829474568,-1.0340731144,-1.4623531103,-0.8314996362,0.9340252876,-0.1944855750,0.0987237468,0.0290169753,-1.8448588848,-0.7846073508,-0.0732088313,-1.7333260775,-4.0291380882,-0.5306329727,0.7409014106,0.2832567990,-0.9075359702,0.1050713211,-1.2936125994,0.9663115740,-0.5140835047,-0.5231055021,0.4298277795,0.8415327668,0.3716501296,-0.0323132835,-0.6347941160,1.3319286108,-0.8133419156,-0.9136698842,1.3833984137,0.8047218323,-0.8619208336,-0.9462947249,0.1797300130,2.6600546837,-1.1597884893,0.4171127379,-1.4330574274,-0.2685092986,-0.5563176870,-0.1373858452,0.2979919016,-1.6659470797,0.9449476004,-1.2770717144,-0.1901160926,-1.1533769369,-1.0791780949,0.9957335591,1.4861640930,1.0744752884,1.2719250917,-0.4285410643,-0.7043931484,1.0934284925,0.3522428274,0.3764531016,-0.3339338601,-1.4555534124,-0.4577522576,0.2179024220,1.1649961472,0.9591401219,0.6655773520,0.3939776421,0.5124578476,0.2843569219,1.3237973452,-2.1637573242,1.0497915745,-0.0111749861,0.5166006684,-1.2268714905,-0.1555034965,-0.0298048109,-0.8875045776,0.1314747632,-0.3267731369,-0.5512117743,0.6083897948,0.6339176297,-0.8001993895,-1.3872141838,0.4129168987,0.6903015971,-0.8204827309,-1.4886994362,-0.3166840076,-0.0989368260,-0.5029759407,-0.2094892561,-0.2207775265,0.6260372400,2.3226847649,-0.8254155517,-1.2173273563,0.4000590742,-0.1477404535,-0.2987808287,0.7859148383,0.5849666595,-0.2389931232,-0.3345387280,-0.4545707107,0.0136067895,2.3920662403,-1.5474188328,-0.0663272217,-1.2421488762,0.7367187738,-1.2968086004,0.4109822512,1.8854326010,-0.3313354552,-0.7640479803,-0.1868978590,-0.2272865027,0.1732350737,0.0523185283],[0.1169780940,-0.2806419432,2.2475368977,0.2511340380,-0.5130397081,-0.5074169040,1.0990157127,1.2466640472,0.3869113028,0.3809750080,1.1946322918,-0.3047500253,-0.5252722502,0.0020444905,-2.1285502911,0.2623390555,0.5451027155,1.8167289495,-1.1404950619,0.5926309824,0.5969594121,-0.0148727661,0.0002652447,0.4641184211,-0.4436666667,-0.6841523051,0.5644008517,0.1762598157,-0.3095212281,0.9205504656,-0.0046450230,-0.4804340601,0.5649686456,-0.3629473150,0.1853141338,0.6242626309,-0.6242429614,0.3195052743,-0.2832978666,-0.7747485638,0.6012946367,-0.1605652124,-0.4696919918,0.7488157749,1.0230870247,0.5574153662,-0.6297346950,-1.2690787315,-0.6727031469,-1.1326258183,-0.9907282591,-0.4507670999,0.4559905231,0.9178841710,-0.6517288089,-0.4528707564,2.0510644913,-0.4466638565,0.3832630217,-2.3713276386,-0.7958715558,1.5951139927,-0.6162417531,-0.7024154663,-0.7358942628,-0.0514691435,-0.2618040740,0.3440458477,-0.2452831417,-1.1451121569,-0.2014667243,0.4369078875,0.3151161969,0.4121712446,0.2650465369,0.9768393636,0.9651262760,-0.1851613075,0.7949992418,1.2735664845,0.9264928102,-0.3236273825,0.4478474557,-0.3741316497,-0.1214300245,0.0958605856,-0.4238029718,-0.4743156135,-0.8259438872,0.7757176757,0.3365464211,-1.4995009899,0.7254462242,0.8834500313,-0.6244534850,-1.6591092348,-1.0582569838,-0.4764253795,-1.7733248472,0.9393494129,0.1969945282,0.1639864594,0.1929222494,1.4174526930,-1.1465125084,0.8462514281,0.5178537369,-1.0700050592,1.4103268385,0.2136932462,1.4934782982,1.8879579306,-0.7683445811,-0.8442191482,1.9063091278,-0.6955130696,1.6429194212,-1.1053818464,0.1641834229,0.1306985021,-0.1009309143,-0.1554664522,0.6429009438,-0.3547316790,-0.3141651452,-0.7517613769,-1.2651112080,0.4944086671,-1.7528989315,-0.0695738867,0.3000741899,-1.3996940851,-1.2547707558,-1.1402056217,-0.5491457582,-0.1278792620,0.6167907715,0.4023316503,-0.2770686746,0.8350918889,-1.2336196899,-0.8996614218,0.8900222778,-0.1889329702,1.2801160812,0.6478344202,-0.6443380117,0.3230400383,0.5593442917,1.1015638113,-2.2827229500,-0.5276812911,-0.3871492147,0.8352118134,-0.2477561980,-1.7110137939,2.6469244957,0.3301985562,-1.3280184269,-0.8027424216,-0.7288699746,-0.8320157528,0.1406433880,0.0071848044,0.6772457361,0.7906083465,1.5656526089,1.8632999659,1.4107118845,-0.1728944927,1.5014334917,0.5151723027,-0.5827595592,-0.6851282716,-1.1576546431,-0.7370390296,1.1400220394,0.1113723516,0.8906426430,-0.1768481135,1.7654592991,-0.0840447322,0.6247481108,-0.9882045984,1.0037338734,0.0724427253,0.1596299112,-0.5648909807,-1.0839179754,-0.5491756201,0.8615249991,-2.6529829502,-0.5975295305,-0.2969869673,0.8291987777,0.9888102412,-0.0241858158,0.0042725853,-0.6767337322,0.4008062184,-0.1150729880,0.6527836919,-1.3201049566,0.5537902713,-1.0040954351,-0.8293229342,0.4181952775,0.2711547315,0.7081342340,-2.2100968361,0.4100460112,-1.2368681431,0.0858222991,-0.6589938402,-0.0142544145,1.3100295067,0.4607637525,-0.3612421453,2.7282576561,-0.4968651235,-0.4072162211,0.4305685759,-0.9169256687,1.4734205008],[-0.6685152650,-0.8737441897,-0.5534303188,0.5200399160,-1.1848164797,0.3177312613,0.0415630788,0.5504212379,1.3617442846,0.5164669752,-0.0615786836,-0.6427606344,0.1296616346,0.3920331001,0.2797629535,-0.7505016327,0.0564294457,-0.1437963694,0.2603136599,-0.9009698629,-0.2277601510,-0.7939273715,0.3479827642,1.3618816137,-0.5092176795,0.3851687908,-1.3853656054,0.7346545458,0.5953921676,0.6145576239,0.5108073354,-0.7996370196,0.2773365974,0.2169533074,1.1366590261,-1.0889068842,-1.0152673721,0.7809972167,0.7224917412,-1.3904248476,0.2733822465,0.6747696400,-0.0467090495,-1.1458916664,0.7344310880,1.2709969282,1.0786265135,1.3192250729,-1.7286123037,1.0670614243,-0.0010136778,-0.5207394958,-0.3519857526,-0.0657513589,1.2044017315,-0.4093344510,-1.0834403038,0.0430439971,-0.1649459302,-1.7983412743,0.6118800044,-0.7838023305,1.0087105036,-1.5372934341,-1.0053912401,0.7615340948,1.9778778553,0.9270530343,1.1736884117,0.0775798485,-0.4577569664,-1.3965622187,1.8189158440,-1.9773858786,-0.7674823403,1.9165551662,1.1599441767,1.7834349871,-0.1174152419,0.9704736471,-1.1975587606,-1.5968848467,-1.7459049225,0.5906978250,1.1235213280,2.0732831955,-1.0014953613,1.6119662523,-0.4356044829,-0.1917356253,0.7411944270,0.5390760899,0.4717051387,-0.6492152214,0.8157361150,-1.8203171492,0.2394956350,-1.0516985655,-0.1302240044,0.4464549720,-0.8567866087,-1.8950551748,0.0301094223,-2.3413836956,-0.9489056468,-0.9709588885,0.1027731523,0.6144888401,1.4732381105,1.7237547636,-1.2133656740,0.1944560856,-0.0028214678,0.4646196365,-0.4514550865,-1.0959115028,1.0776283741,2.0255107880,1.2845118046,-0.0877527371,-0.5224267244,1.7399580479,0.7644951344,-0.8440381885,0.5369064808,-0.7958087921,-0.0592793599,-0.6017829776,1.6896573305,-0.6590547562,-1.0762586594,0.4362809658,0.0668928027,2.0457201004,0.5018579364,0.5019522309,-1.1521987915,-0.2816174030,-1.7676477432,0.0686004385,0.5971288681,1.5740783215,-1.5005308390,0.2959667742,0.0992885381,-0.4789198041,-0.4450925291,1.6048885584,1.9666018486,-1.5215646029,0.6102808118,-1.0968976021,1.1952879429,1.0654774904,0.0481246337,1.0292104483,0.6108282804,-1.9799226522,-0.1822997779,-1.0944604874,0.3795200884,0.6275728941,-1.0811973810,0.5579342842,1.4074038267,0.2341215163,-1.9398125410,0.1486764103,-1.7583855391,-0.8627243042,0.8016073704,0.2244357616,-2.3210380077,-0.1822131872,0.2372455597,-0.0160580892,-0.6138968468,0.2037782669,-1.0922092199,1.6620970964,-0.1207506806,1.4912840128,1.9458825588,-1.0110816956,-0.3563345075,-0.7647183537,0.3050349653,-0.0155192744,0.8703370094,-0.2007468790,-0.0845041946,-0.2028828859,-0.7978484631,0.1054977104,0.3118167818,-1.6269580126,1.1409548521,0.9135065079,0.1632136852,-1.0845395327,-0.8973467946,-0.1036828682,-0.3324064910,1.1789542437,-0.3770294487,1.5620265007,-1.3538585901,0.1679062247,0.1029169038,-0.4538665116,0.0385230370,0.7132785320,1.3319801092,0.2151621729,-0.6791537404,0.0503540374,-1.7433155775,-0.2129102945,2.3486294746,0.6435081959,0.0417948477,0.4045048654,0.5760819316,0.2136069536],[1.5347999334,-1.6873236895,0.6115571260,-0.0327798501,-0.5411595106,-0.4453768730,-2.0417563915,1.9538553953,1.5702822208,0.9667896032,0.0068834368,-1.4329059124,1.7121027708,-1.2107728720,0.8673722148,-0.0087148799,-0.2151390463,0.1782071441,-1.7905542850,1.1115261316,1.0436437130,-0.8217792511,-0.8795315027,-1.5809032917,0.3619697690,1.7345861197,0.5589722395,1.2194062471,1.2847113609,0.7623335123,-0.4450575113,-0.5541297793,2.3819768429,1.0045148134,-0.2210227102,0.0310857967,-0.3054305315,-0.2346151024,-0.2350860834,-1.0135045052,-0.4001455605,-0.4735128582,-2.3997213840,-1.6969120502,0.6540305614,0.7910987139,2.6275141239,-0.5053266883,-0.6437256932,0.4331637621,-0.1179857254,1.3676888943,-0.3311014175,0.0152680846,-0.0236807577,0.9302479625,0.6600639820,-2.4037752151,-0.9464588761,0.0438113771,-0.0725328401,1.0720617771,1.0761135817,-0.2737096846,-0.5588893890,0.2598156333,0.9769091606,-0.6441979408,0.7905815244,0.8603966832,0.2026889175,1.3673669100,-1.3858262300,0.2233661562,-0.1094158739,1.4453797340,0.0151959099,-1.6613028049,0.7575907707,-0.1962304562,-0.1996283978,-0.7416828871,-0.7916600108,1.4287346601,-0.4484005272,0.6876532435,-0.6248690486,0.3298631310,0.9127324820,-0.9684745073,1.3250454664,2.2452054024,-0.9814113975,0.0123218549,-0.0707478821,-0.0363855138,1.0943500996,-2.1510896683,0.5944954753,-1.0275869370,-0.7499034405,-0.9592984319,0.5362375975,-2.1820952892,0.6188629866,0.7973918915,-1.7052952051,-0.7358623147,1.4108181000,0.3214335740,1.3040961027,0.6007806063,0.6384257674,0.5477932692,-0.0138832955,-0.8589347005,0.8096865416,0.3102856278,-0.2026515752,0.9718768001,0.6564921737,0.1603172570,1.0024086237,-0.9002149105,-0.8006245494,-0.9411728382,-0.1576966792,1.1415449381,-0.0577506274,1.1866910458,0.3541076481,0.6679896116,1.3162442446,-0.2557232678,-0.3763893545,0.2416353524,0.7076978683,-1.8903992176,1.1509604454,-0.5034546256,0.2765678763,-1.8116631508,0.0736504048,-0.5345338583,-0.2187395692,0.8730805516,0.2851711512,0.4014697671,0.5744179487,-0.9496762156,-0.9646509886,-0.2687695622,1.8421496153,0.8335016370,-0.6742351651,-0.3898480535,-1.3010454178,1.2313911915,2.5872275829,1.0136504173,-0.1823874265,-1.9427940845,0.0121092172,1.0977327824,-0.9339600801,0.0772805214,-0.7465289831,-3.4953188896,-1.5749645233,0.0567488112,0.4932927489,0.4036336541,1.5102025270,-0.3151389062,0.4847653508,2.2489926815,1.2595604658,0.4640718997,-0.2695495784,-0.7470452785,0.7966135144,-1.2423455715,1.0008935928,0.1510142535,-1.3154373169,-0.5294527411,-1.4460440874,0.0734137148,-0.9246321321,0.0041351891,0.3118188381,-0.4110965133,-0.4202423096,0.1939986348,-1.1355917454,1.0640664101,0.1575757712,0.2295729369,-0.5353895426,0.2008222491,-0.0894518197,-1.2742060423,-1.2324142456,-0.0447078608,-0.1281779855,-1.2925449610,-2.0690896511,1.6433898211,-0.6176621914,-1.8369152546,0.8185908198,0.3368025124,0.2383316606,-0.3955019116,1.0942413807,1.0905435085,-0.6878349185,-0.8977550864,0.5481200814,-0.8589039445,0.2464670092,2.2294669151,0.3704622686,-1.6462700367],[-1.0096625090,-0.7288820148,0.0720273480,-1.4444570541,-1.4314428568,-1.0962486267,-0.6008301377,0.6626305580,0.3884952664,0.1886538416,-0.0186148025,0.9334484339,0.5448017716,0.1605713814,-0.5728644133,2.3174045086,-1.1572582722,0.3675760925,1.2822412252,0.6096727848,1.0833481550,-1.4809343815,0.6521091461,0.4051893651,-0.0447794273,-0.4249199927,0.9600380063,-0.1428283155,0.1274694651,1.0010346174,0.9665576220,0.7157662511,0.1009588018,0.9976038933,-0.3180441856,-0.7530906796,0.5247619748,-0.1527070850,1.2734057903,0.4071713686,0.9690848589,-0.7642352581,1.0456936359,0.6039264202,-0.6427201629,0.2151586413,-0.3214791715,-0.4597530067,0.5271275640,-0.6031562090,-0.3741852939,1.9624742270,1.3004586697,-0.9262704253,-0.5459587574,1.9289519787,-0.1279011965,-0.8018730879,-0.3704701662,0.1392263919,2.2456700802,-0.2131865770,0.2803228796,-0.2014343739,0.4005430639,-0.7303941250,-0.6890636683,-0.1513775736,0.8517774343,-1.3185310364,-0.8762566447,-0.1303947270,0.9660004377,-0.4019751847,0.4808919728,-1.3781931400,1.8092302084,1.2631082535,1.0598397255,1.1649284363,-0.5299757719,-0.2302153260,0.4181659818,1.0113337040,-0.0762433708,-0.1666911095,-0.0848165229,-2.1126856804,-1.9476577044,-0.3496899605,-0.4967019260,0.5956168175,-1.2740421295,1.6090040207,1.1050789356,-0.3681654334,-0.4480681121,-0.2363000661,-0.2702892125,0.8397852182,0.4432892799,-0.6930931807,0.8435317278,1.0990096331,-1.4728732109,0.3252948225,0.1570708901,-0.2961833179,-0.0988946781,-0.6090566516,1.7864310741,0.3255374432,-0.7772111297,0.8540177941,-0.3736964464,-0.1458571851,-0.4800638556,0.6023507118,1.3466943502,-1.3057725430,-0.6299296618,-0.1963314563,-0.6180570722,-0.6634348035,-0.9157820344,-2.1389577389,-0.5336833596,1.3248667717,-0.6429237723,-1.1559734344,0.6903609633,1.2517040968,0.9739649892,1.2991584539,0.2655103207,0.3484174013,-1.2541536093,0.1894829720,0.8233504891,-0.0764275566,-1.2177714109,-0.8417980671,0.8581524491,0.0035927813,0.6365355849,-0.8042198420,-0.1388953626,-1.6326507330,-0.6042947769,-0.9669717550,0.2938333750,0.9714415073,-0.1223727614,-0.8723503947,-1.1460812092,-0.2133648992,-1.1128445864,0.9363123178,0.1991896331,-0.8822627068,0.1223424897,1.3291049004,0.0668822005,0.1119093895,1.6684954166,-0.9703634977,0.8244425654,-0.3810819983,0.6499918699,-0.4449295998,0.5241637826,1.8430259228,-0.1138890088,-0.7317150235,1.5404244661,-0.6431032419,1.0746787786,0.9530599117,0.2035676390,1.7534618378,0.4639586806,0.0214881431,0.3588767946,-0.4447282851,1.6261456013,1.7030267715,-1.4624485970,0.6691627502,-0.4120528400,-1.2016181946,-1.0315819979,-1.8414787054,0.7353565097,0.0985109657,-0.5989940166,0.6289370060,-0.7668067217,-1.0768667459,1.1415388584,0.2427966297,-1.5039368868,0.2121663541,1.8831692934,1.7689586878,-0.0031755727,0.1881536543,0.0798226669,0.4581492841,0.2642267048,0.9306536317,-0.4983684719,-0.1663818955,-0.9144891500,0.9846673012,0.2117036432,-0.6294923425,2.2421128750,-0.1303592473,0.4997661114,0.9499601126,0.7253001332,0.2831318378,-0.4090300798,1.1107546091],[2.2380344868,1.3806735277,-0.3407057524,0.4970670044,1.3422850370,0.1040709913,-0.1962797344,-0.6731407642,0.3965022266,1.6650975943,-0.6868150234,-0.9100295305,1.5729589462,-0.1079028100,-0.8686639071,0.2137991190,-0.8141530752,-1.2586628199,-1.0130791664,-0.1064461023,1.3744591475,-0.5557163358,-1.4183386564,0.4970445335,-0.1264280081,-0.6952810287,0.7875682116,0.9634685516,-0.0873802453,0.4830078781,-2.0697469711,-0.2951569259,1.9275074005,-0.2655444741,-0.5246491432,0.6961337328,-1.1402201653,-0.8014924526,0.4705885053,1.3276981115,0.9487622380,-0.1226340607,2.2388117313,0.9282962084,0.8358730078,0.5729625225,1.3172289133,-0.7303490639,0.1824570298,-0.6966341138,-0.8805764914,0.7411084175,0.2305240929,0.0980034173,0.8006091118,0.2612165213,1.0207033157,-0.0486149713,-0.1296018958,-0.2046490759,0.3058603108,0.1007243693,-1.8249213696,0.5788246989,0.5505513549,0.8283489943,2.3347523212,-0.3820807934,-0.6665652990,-0.6986245513,0.5523877144,0.5897316337,-1.7852385044,1.5092515945,0.5711547732,0.3464311659,0.2958600223,2.0925042629,-0.2321517169,-0.1434054524,0.5728091598,-0.2187698931,-0.4759446084,0.1177005395,0.2013361752,0.5981849432,-1.9799973965,0.2529041767,-0.5476834178,-0.1730593890,-1.7854814529,-1.1705741882,-0.3581634760,0.3817416131,0.7512064576,-0.1027551740,0.6353870034,1.3960314989,-1.2177100182,-1.2762954235,0.2829504609,-0.3222350478,-1.9788143635,1.1852393150,-0.8120482564,0.2017181963,-0.7778249979,0.1324888170,-1.7530630827,-0.3840102851,0.5514950156,-0.9082639813,0.2709457278,-0.2724392414,1.7407710552,1.0251507759,0.5038827658,0.0599922501,0.2778967619,1.1140643358,0.5126416087,0.9094967842,-0.2608771026,1.0770685673,0.3795000911,-0.1550950557,0.4098328948,0.0182807632,1.7432143688,0.0442269817,-0.0733183324,-1.7420239449,-1.0372033119,-1.4505765438,-1.0695979595,0.3520697057,-0.1199650243,0.1623314768,-0.4059309363,0.2590332925,0.3168719411,-0.2985204160,-1.1141160727,0.4796123803,0.5619376302,-0.0879992098,-0.2896755636,0.0345673114,-0.6017387509,2.2942187786,-0.4575514793,0.4433886707,-0.6393599510,-0.7018532753,-2.0840408802,0.2744977772,-1.7085973024,2.0885903835,-1.0704900026,0.6227512956,-2.3643672466,0.4737828970,1.8228685856,1.2083232403,0.5655582547,1.1943204403,-1.6110794544,-0.8104713559,0.6233947873,1.4821898937,-0.3512924016,-0.9832256436,-0.3109658360,0.2102089971,0.0661011934,-0.7598971128,0.2737687528,-0.3565664291,0.0649320632,-0.8309890628,0.8115738034,0.5818820596,-1.2470250130,1.5066984892,0.0355780944,0.5166119933,2.2526850700,-0.8431701064,-0.0671550483,-1.1754525900,0.5172681212,-1.4026856422,1.2922587395,1.1752586365,0.4308859110,-0.1489560753,1.2196234465,0.2266667634,0.6856338382,-0.0718977973,-0.7315303683,-0.7587987781,-0.6444988251,1.0108252764,1.7422361374,-1.0566121340,-0.1381981969,-2.4956552982,-0.6139257550,-1.2556135654,3.1750850677,0.2591003776,0.2450738251,0.3005379438,-0.9752496481,1.7764663696,-0.1214622557,0.5261299014,0.2342363894,-0.0667102411,-0.9946687222,-0.7998950481,-0.1393250823,0.5439608097],[-0.3280489445,-1.6486464739,0.0773164555,-0.6094115973,0.7776465416,-0.7698317170,-0.2485616952,0.7960718274,0.2313920408,0.0460020714,-1.7142246962,0.6232043505,0.2541713715,-0.4534405470,1.7899509668,0.1362471282,-1.9164524078,1.3635959625,-0.8852525353,-0.3090792894,0.3931639791,-0.9311147928,-0.3980593979,1.9160156250,-2.4146127701,0.9591246843,0.0804828331,-1.1393139362,0.5533338785,-0.0600930825,0.2490953207,0.3519489169,0.7660022974,-1.6150187254,0.6105470061,-1.2359455824,-0.8698206544,0.3892658055,-0.7539950013,0.6950904131,-0.9604725242,0.4308103025,-2.3256311417,-0.7111156583,-0.4337917566,0.8061956763,-1.3608552217,0.4315817952,-1.3107632399,0.5574386120,-2.2791669369,-0.9615024328,-0.0104346648,1.8257387877,1.4231183529,0.2945812345,-0.1324300915,0.2946366966,1.6782826185,0.6541477442,0.4029373229,0.4747828245,1.1353145838,0.3953845799,0.1936890781,0.4062623680,0.4179663360,-0.1990364641,0.2323853821,-0.1627434790,1.2631576061,1.0686953068,-0.2658118308,0.6809504032,1.5407712460,-0.6695981622,1.8600367308,0.4357393682,-2.5560443401,-0.5719974041,-0.8085638285,-0.2786338627,-0.7596287727,-0.7966396809,-0.6347700357,0.6346846819,-0.1229087114,-0.4061551392,0.4961775839,-0.9298028350,1.7950794697,-0.4195333719,-0.6384252906,0.4334599376,1.2472872734,-0.2829909325,0.1914219707,-2.3444113731,0.1594360918,1.2696460485,0.4959804118,0.2380602211,-0.1448186785,-2.1736934185,0.0322666392,-1.1135591269,0.1910399944,0.5964226723,0.6934137344,-0.3309519887,0.5232272744,0.6514256001,0.5427439809,-1.2049249411,1.2035021782,-2.1379439831,-0.0123552503,1.6907293797,1.3622356653,0.0854423568,-0.1239003465,-1.1248239279,0.3118157685,1.4227129221,1.3931421041,-0.5201041102,-0.6520749331,-0.1951945126,-0.0238409471,-1.0981628895,0.1196731627,-1.0028203726,-0.0433323942,0.1475336701,-0.2034588307,0.0879188478,-0.1838414073,0.8159422278,-0.6409673691,0.0307073314,-0.8225424290,0.1864877790,-1.2164173126,-1.0154621601,-1.4576314688,-0.7777158022,-1.0057930946,1.5576810837,1.0352222919,1.8638725281,-0.0097064106,0.8054191470,0.1223727986,-0.0801445544,-1.4936922789,-0.4619750381,0.0596807078,0.3278731108,-1.0393821001,-0.4951561689,-0.3413927257,0.1904700100,-0.1210603565,-0.1594241560,-0.5338973403,-2.1179163456,0.9923138022,1.6756576300,-0.5539448261,-0.4309539795,1.2551391125,-0.9311563969,0.6251273155,-0.0564246289,0.8008189201,-1.9777007103,-0.3098180890,0.7816771269,0.6166989803,-0.6998443007,0.4654302001,0.0343916118,2.4334328175,0.5027511716,-0.5978142023,-1.1934173107,1.1687659025,-1.3383669853,1.0300269127,-0.0677551478,0.3900418580,-0.4028481245,1.5647825003,0.4914836287,-1.0139048100,0.7788290381,-0.9364992976,-0.4769464433,-2.2719848156,-1.2316024303,1.4936300516,1.1475230455,-1.6251858473,-0.1341409087,-0.6324136257,0.9583027363,-1.8046439886,-0.6657356620,-0.2849379182,1.1019287109,0.3203172982,0.8305735588,0.8801663518,0.2541929483,-0.7876218557,-0.4848353863,-0.6038257480,-0.3463531435,-0.4620301723,0.3399290442,0.4784194231,-1.4630919695,0.5666418672,0.6698990464],[-0.1886400282,-0.0589710884,-0.8409474492,-0.5538551211,1.1771206856,0.0073921932,-0.0275823288,0.4827136993,2.1394298077,1.0037149191,1.3728575706,0.5932582617,0.6228179932,2.2393188477,0.6417520642,-0.6442270875,0.3314913809,0.4365530312,0.9575701952,0.1253282130,-2.6363835335,-1.1195566654,0.1741666943,-1.3815630674,-1.3405553102,-0.9259150624,0.7431330681,-0.5993356109,-0.9352449775,-0.2250264287,-0.3261758387,-0.9840246439,0.8795168996,-0.0585330613,0.4813308418,0.6932357550,-0.6614339352,-0.6948084831,-0.5029956698,1.0634011030,1.9179027081,-0.1568887830,1.2873591185,0.7845811844,-0.9608737826,-1.5719852448,-0.3008854091,-1.5339412689,-0.2134107649,0.4248637855,-0.3359931409,-0.1320641041,0.8435388803,-2.7100458145,-1.7172553539,0.8296487927,-0.9835475087,1.5917860270,0.1721152812,0.4304965138,0.7664323449,-1.2311582565,-2.3880450726,-0.4940323532,-1.0391459465,-0.7237001657,1.2727987766,1.6193475723,-1.0287649632,0.7299798131,-0.6349011660,-0.0274767578,-1.5900055170,-0.4706409872,0.7383552194,-1.4446251392,2.9851384163,-0.5311147571,0.0925131291,1.0023488998,0.9239675403,-0.3110629916,-0.3885660768,-0.5063413978,0.2894129455,0.8276406527,-0.4842105806,-1.5078349113,1.2256025076,0.8286691904,-0.3395346701,-0.0520976894,-0.6069356203,1.4886635542,0.8925319314,0.5489603281,0.4804161787,1.3660881519,-0.4378204644,-0.7837976217,0.3279939890,-0.4669124186,0.5884266496,-0.1041898355,-0.7743563056,1.9065687656,-0.1401365250,0.0404757708,-0.4654856622,0.1470849812,-1.2212413549,-0.2502074838,-0.5511439443,0.1273302287,-0.8361667991,0.5060011148,0.4187401831,-1.0274453163,0.1833582371,0.1491656005,0.8520369530,-1.0309195518,-0.2227566093,0.0463354513,-1.7734726667,0.2360962629,1.2328624725,1.3344942331,-0.6834138036,-0.9213144183,-0.2375682145,-1.0676547289,-0.7755971551,-0.1519525945,-1.1810057163,0.6070850492,-1.2324193716,0.7946123481,0.4432314336,-1.2385253906,0.7860576510,-1.2088580132,-0.6392730474,-0.8253847957,2.0636448860,-0.3773825765,0.2946148515,1.2927073240,0.3535074592,-0.4379634857,-0.0179980081,-0.0072805942,2.2701530457,-2.1468765736,-0.3751948774,-1.4295772314,2.2897188663,0.7464865446,0.2147173584,-1.1608482599,1.5358866453,0.5187512636,1.8804030418,1.5796387196,1.0487182140,-0.8676830530,-0.0556595251,1.3028850555,-0.5795240998,-0.5162953138,-0.0655466840,0.9422415495,0.7499783039,-0.9180640578,0.4498133957,1.1590911150,0.4048582017,-1.7504346371,2.2532100677,0.0281716362,-0.6220969558,-1.5183184147,0.9161355495,-0.8907647133,-1.3098375797,-0.3318792582,0.2209775299,-1.2912184000,-1.5370779037,0.2036247104,2.0972721577,-1.0611443520,0.9589093328,1.1591913700,1.0168168545,1.2855957747,-0.0241216458,-0.8847688437,-0.2365839183,0.0415988378,-0.1515546739,-1.0070446730,-1.1214296818,-0.3417044580,-1.3171265125,-1.5975716114,0.8810922503,0.4943160117,0.0729970559,-0.0358922035,1.4424666166,-1.5872232914,-1.3994543552,-0.4739158154,-0.4085120559,1.6491034031,0.3564356565,-0.5753759146,2.0592122078,-1.0333622694,2.1544709206,1.3682296276,-0.7064908743,1.7868995667],[-0.9723177552,-0.7266247869,0.9715178609,0.4088996947,-1.7122020721,-0.4308840632,0.3252696097,0.2743642926,-0.8869826794,-1.0270365477,-0.5657429695,0.0832255706,0.0655450523,0.4747250974,-1.3726384640,0.4130788743,0.6846900582,-0.5520215034,-2.0530512333,-2.1558058262,0.1264140904,0.3746607900,1.8946000338,0.1379344761,-1.2841641903,-0.3011165857,-1.3064757586,0.6817041039,0.9905909896,-0.3322680295,-1.0966534615,-0.5721865296,1.1613034010,0.4173618853,0.2521701753,0.0170840435,-1.1218461990,1.3891819715,2.0246393681,-0.3689926863,1.2998167276,-0.0202535987,0.4367121756,1.0193365812,0.9042968750,0.6680484414,-0.6065965295,-0.5022114515,0.5810588598,-0.3901543915,-1.1834454536,0.8150207996,-1.1420608759,-0.3598107100,-1.6044889688,-2.4384107590,0.0117071439,1.9975863695,-1.3817124367,-0.3072476387,0.0048268959,-1.7112979889,0.9031231999,-0.2966916263,-1.0396625996,-0.1289494932,-0.7083650827,-1.4692304134,-0.5094682574,1.9392372370,-1.6139283180,-1.1871839762,-1.1809390783,-1.9714508057,-0.9087123275,-0.2333478034,-0.7533107996,-0.9897646308,1.4758378267,0.1605499536,-0.0507054329,-0.2238054425,0.8934520483,-0.0074833194,-0.3475907147,0.4808534384,-1.3428298235,0.0425792374,-1.4493671656,-1.1105960608,-0.3286893964,-1.1615190506,0.7743390799,0.5156641006,1.4899449348,0.7932764888,-0.3827163875,0.9504209161,-0.1228832677,-0.5156506896,-0.2268660665,-0.6296691298,1.0760799646,-0.4033052623,-0.7914856672,-0.2877828479,1.5812536478,-1.1596530676,0.0779661462,0.6545679569,-0.2202242166,-0.7564415932,-0.0417143740,-0.3171341121,-0.4582436979,-0.9056224823,-0.9524829984,1.6070226431,-1.2132416964,2.2914252281,-1.1252080202,-0.6691984534,-0.2136556059,1.5424201488,0.3239894807,-0.1936365366,-0.3805269301,0.3153375685,-0.0132743232,-0.7174763680,-0.6745864749,0.5147384405,0.2804582715,0.8614385724,1.0986897945,1.7333666086,-0.7799022198,-0.9911209941,0.9604320526,-0.8721246123,-1.6883252859,-0.8845354915,0.7923492789,-0.5049350262,2.0076103210,-0.0851265863,0.7332333326,-0.1772662252,1.5707787275,-1.1044410467,-0.1525815576,-0.9408708811,0.4330949187,-1.2818670273,-0.3055057526,-0.1675800681,0.0284603387,-0.8147291541,-0.4210108519,0.7458789945,-1.1510759592,-0.5196728706,1.2945172787,0.3134264946,-0.5981710553,-1.7199571133,-0.9573861957,0.3585983515,0.1577864438,0.0976585224,0.9235061407,-0.4882372022,-1.2233790159,-0.3873362243,0.1936537027,0.8459808230,-2.3967981339,-1.7979650497,-1.3401989937,-0.6894354224,0.0410711132,-0.7186352611,0.0600788631,1.3476153612,0.3278820813,-1.3045172691,-1.3771083355,-1.7876477242,0.2890119255,1.3028764725,-0.3252322376,-1.6646826267,1.0996216536,0.8495755792,1.1889733076,-0.6801342368,-0.9674876332,-0.5559958220,-0.8111504912,1.2361888885,0.2234009504,1.1607664824,-0.7407107949,-0.2627224922,-0.6727229357,0.6591702700,-0.1328006536,1.9366884232,-0.0906984061,0.7591729164,0.1739566326,0.7143741846,0.5020965338,1.7826042175,0.6204472184,-0.5581421852,0.2132278383,0.3122832775,-0.9909564257,1.2599604130,-0.1934482753,-0.9713167548,1.1571723223,1.5756896734],[-0.5407086611,-0.6157051921,-1.6384927034,0.9595841765,1.1959745884,-0.2396019995,0.5296090841,0.0760802999,-1.4119911194,0.2633403540,0.8130000234,-0.0615768097,-2.6958682537,-0.0774267390,0.2242079079,-1.1902222633,-2.2081933022,1.3072851896,-0.6697205305,0.3599147201,-0.5324732065,1.8647314310,-0.6539912224,-0.0811203197,-1.6695041656,-0.1677595228,0.1529489309,0.0326982960,0.3603679836,0.2729178965,2.6728754044,0.0517382957,-1.1903650761,1.9886676073,0.3085786998,1.2711433172,0.3996826708,-0.9335381389,-0.3373581767,0.5985429883,-0.7188912630,-0.5423603654,0.8109322190,2.1182680130,-0.2123235017,-0.2050291598,-0.3468085825,-0.0957891718,-0.2557451129,0.1651563942,-0.9804444909,1.7935678959,-0.5420064330,-1.2521762848,-0.4070931375,-2.2685718536,0.2797066867,-0.1671731770,0.7808704972,-0.2584797740,-1.3019461632,-0.8495772481,1.1621996164,0.7994099259,1.4648393393,0.0249336269,-0.5417373776,-0.1430224478,0.3782896399,0.4983956516,-0.3447601497,-0.0361413844,-1.3818486929,-0.1909462810,-0.6342868805,-1.5501580238,0.4438783824,1.4791367054,-0.2694898248,-0.5962350965,-0.4852928221,-0.9123769999,1.4600878954,0.0140110198,-0.0810736865,0.4739871919,-1.6347879171,0.6561161280,0.7903645635,-0.0381236821,0.7106814384,-0.4605905414,0.0920624137,-0.7547672391,0.0979229510,0.3171488345,1.1117020845,1.4691916704,2.1095216274,0.0467688963,-0.2855694592,-0.9627265930,-1.6467630863,0.4463484585,0.1725912839,-0.4279518425,-0.1798031181,0.3101399839,0.0048362794,1.4784452915,1.5026966333,1.0328475237,-0.4919162393,-0.1434764564,0.4667615294,0.9142318368,-0.9285969734,-0.2069311589,1.4059398174,-0.5289928913,-0.4967575073,-1.3956668377,1.4277200699,0.8334128261,-0.0185762569,-0.1923116297,0.0418641306,-0.4321692884,0.8290914893,0.8341650367,-0.2333217710,0.7934345007,0.3587965965,0.5614849925,-2.0616495609,-0.1924656183,-0.0459085889,0.2538861036,-0.4057802260,-0.0318090059,1.3345776796,1.9516458511,-0.4307067394,0.8678566217,-0.1097779796,-1.6316813231,0.1627230048,0.6683475971,-1.0535581112,0.3491037786,0.4147782922,-1.2055585384,0.4886077940,-0.1130808294,0.7219112515,0.0661294386,-0.1400840580,-0.0657013953,-0.6607044935,-0.6015189886,0.7106990814,1.7140800953,0.9367416501,-1.2773195505,-0.6420869827,-0.6175547242,-1.8307830095,-0.4264650047,0.5967303514,1.3804618120,0.8144572973,1.5835609436,-0.3536988199,0.7703401446,1.4535437822,0.7620157599,-0.2588397861,1.8452669382,0.2829564810,-0.8681214452,-1.8005297184,0.4997319281,-0.4752902091,0.8748758435,-1.3430117369,0.5314849615,-0.5759868622,1.2641209364,0.8554402590,1.5168594122,0.2480181605,-0.2194981575,0.5211862922,-0.6440048814,0.7792780399,1.4497418404,0.7278845906,-0.5463015437,-0.0357734151,2.2245075703,1.7500543594,-0.1910633743,0.6573238969,-1.3362874985,-0.7534814477,1.2886375189,0.8610209227,-1.0121858120,0.9795084000,-0.1356090903,0.5075626969,-0.8574798107,0.5879177451,-1.7946065664,-0.0130303819,2.6269860268,-0.2716670632,-0.0657275766,0.1854525954,0.6998822093,0.2578449249,-1.4977544546,-0.6380822659,0.3094310462],[0.9322613478,1.0821434259,0.0982655212,1.2655304670,0.8077735901,0.7658033371,0.0170805342,-0.6523284912,0.4967969060,0.2543117404,-0.1472297460,-1.0754979849,-0.3788460195,-0.6500695348,-1.2020435333,-0.7062480450,-0.1719639748,0.1413584501,1.7674720287,1.4245822430,-0.6928767562,0.4070508480,-0.8351898193,0.6124445200,-1.4726996422,1.2751882076,-0.0078987619,0.7250511050,0.4638235867,0.2765693367,1.0345190763,0.6211076975,-0.2134134769,1.0013273954,-0.3205123544,-1.9580650330,-0.1975866854,0.5886839032,0.2872526348,1.8216292858,-0.8485202789,-1.1175162792,0.0909620300,-0.1545620412,0.6551841497,0.7736989856,-1.3568792343,0.7647265792,-0.2998824716,-1.3783611059,-2.0769608021,-2.3308551311,-0.1665841937,1.6644699574,-1.1846499443,1.8231101036,0.4575906098,-0.5072680116,-0.3290798664,-1.7656143904,-1.0600577593,-0.2223261595,0.4002893269,0.7339606881,-1.7777433395,0.1461729258,0.7029541135,0.7349690199,-0.4175417423,-0.6678780913,-1.4948253632,-1.2591323853,0.7002172470,0.4057020247,-1.8846118450,-0.4096996486,0.3572451174,-0.5729187727,0.3632394969,0.8388002515,0.7888129950,2.0745422840,0.1977431327,0.1684732288,-0.5357504487,-0.7693734169,0.4129380882,-0.3638237119,0.6388210654,0.0842956081,-0.1233994961,0.0458561741,0.0440238081,-0.8346993327,1.1818025112,0.3015581667,0.2385077327,-1.2280052900,-0.0870673805,0.3921283185,0.4252742529,-0.0002660285,0.4988144040,-0.3004822135,-0.2015036196,-0.5291435719,1.3097598553,1.4225721359,0.6904790998,-0.7867959142,1.5844599009,-0.2980011702,1.2988511324,0.2470097095,-0.4910517037,-0.4043954611,1.0191758871,0.1660731584,1.6221412420,0.0662123859,0.1093840599,0.3859095871,-0.1514603198,0.7722839713,-1.8401905298,-0.4582791924,-1.5480029583,0.8383934498,1.5307412148,-1.0848832130,-1.4832700491,1.3966538906,0.7267266512,0.5375882387,-0.7886016369,1.0743261576,-1.1768319607,0.1318906993,-0.3701928556,0.5601051450,0.3761105239,-0.0058422196,0.8287389874,-2.5389571190,2.1728942394,0.3999422789,-0.0519637764,-0.0581969433,0.1575311124,-0.7968834043,1.0298998356,0.3352723718,1.8463332653,-1.3961603642,-0.1399134099,0.8294527531,0.8428064585,0.1062459201,-1.6589927673,-0.5416683555,-0.3484183848,-1.3727669716,-2.1644611359,-1.1050168276,-0.6006038189,-0.6543050408,0.4155762792,-0.4915748537,-2.1399559975,1.0393067598,-0.1879214644,0.8151748776,-0.5987551212,-1.0111064911,-0.1526869535,-1.9934095144,0.0915879831,0.0305229817,0.0214052256,1.4779964685,0.4579023123,-0.1609235853,0.8058716059,-0.1193252355,-1.4209824800,-1.4432802200,0.3448107541,0.8484117389,0.3917312622,-1.3221226931,0.2110161185,-0.1322266310,1.2812737226,0.2105053812,0.5784762502,1.2395321131,-0.1125992611,0.6148221493,0.1073464751,3.1926834583,-1.7169188261,-0.4264818430,0.5589302778,0.0098558068,0.0445020609,-1.0925434828,0.5685714483,1.8520839214,0.2249426991,-0.1640461534,0.4741087854,-1.8028807640,2.5509712696,-0.3230691254,0.1120029092,1.4169816971,-1.1021708250,-1.3753912449,-0.6458590627,0.1685565859,-0.2544992864,1.2977367640,-0.6434763670,-0.4687041342],[-1.5133686066,0.5543805361,-1.6351692677,-0.5480977893,-1.8672090769,1.8669744730,0.1007032543,-0.7519266605,0.9288895130,1.0849411488,0.8286650181,0.2248411477,-1.4469972849,-0.8432237506,0.4922606349,1.0674357414,1.0139129162,1.8805615902,-1.0198467970,-1.4214346409,0.0867412463,0.7238012552,1.2070678473,-0.2821899652,-2.5294377804,-0.1533146352,1.5075769424,0.1515903473,-0.7290561795,-0.3938293755,-0.3148902059,-0.0655002147,1.0328027010,0.9855728745,0.0142235737,0.1424133629,2.3653128147,-0.0937726349,0.6720957160,0.3193032146,-0.0347761326,0.8808904290,0.6379359365,0.5533177257,1.1247152090,-0.0711953193,0.0823010355,1.5953356028,-1.1492849588,-0.2697221935,2.5629601479,-0.4678767622,0.3748208582,-0.3017410338,-1.1595700979,0.9848414660,-1.5886949301,2.1473009586,-0.4320541620,0.3351209760,1.1572041512,-1.6510055065,1.4763158560,-0.8780969977,0.7521370649,0.7664933801,0.6427647471,0.6748605967,-2.1987223625,-0.7755331993,-0.6144747734,0.1226312146,-0.0433201976,-0.9534757137,-0.7053830028,-0.2736400366,-1.2137296200,0.6272516251,-0.1973339170,0.1871281117,-2.7975583076,-0.1932645291,-2.1384193897,0.0670402572,-1.4602125883,-0.1883114576,-1.9900075197,-0.1766163856,1.3107856512,0.8267203569,1.0474450588,-0.4156937301,1.4552435875,-0.9403783679,0.6154562235,0.9041082263,-1.4211113453,0.1542359740,0.2054781765,1.2778793573,-0.1201702952,-1.3128867149,0.3029603362,-0.7522927523,-0.9131040573,0.9282863140,0.5733796358,-0.9316766858,0.2135121375,-0.7382605076,-0.5023554564,-0.9460375905,-0.3469051719,0.3816500604,-1.3926203251,-0.7923706174,-0.8669332266,-0.5555337071,0.2142671943,0.5651071072,-0.6646097302,-1.0670479536,0.0496083200,-2.2859947681,-0.5420932174,-0.8602784276,1.1403759718,0.5368918180,0.4115375876,1.4746530056,-0.2967051268,-0.4382048249,-0.7692515254,-2.1632175446,0.2171597481,-0.4031183124,0.8962494135,0.9963235855,-0.8190985322,-0.3271569610,-1.8990769386,0.3499003649,-2.3797082901,0.6820656061,-1.0557612181,-0.1580118388,1.3777353764,-0.9384363294,0.2778090537,-0.7599855661,0.0350417122,0.3391051590,1.0693094730,-0.4610579610,-0.7098724842,0.2182071954,0.8414065838,0.8761809468,-1.2011101246,-0.0909584537,1.6314455271,-0.3418768048,-0.6599820852,0.0627719760,1.9397410154,-2.6317727566,-0.9709402919,0.2281691581,-0.1328415275,0.3824462593,0.9747131467,2.1028826237,-1.8011776209,-0.7046427727,0.1276893169,1.2597452402,-0.2805773318,1.7098058462,-0.5087597370,1.8857795000,-1.8405008316,0.1865903288,-0.4646588862,-0.8164331317,-0.6990193129,-1.1148481369,-1.4104843140,-0.5799752474,-0.4449983835,-2.0007512569,-1.5721967220,-1.9587359428,-0.6990004778,0.0577918403,-1.6206052303,-0.1327113062,-2.6066644192,0.3853771985,0.7883592248,-0.4743609726,0.3637579083,1.8269271851,-0.3550509512,-0.3871345222,1.4265280962,1.3550875187,-0.0144073600,0.0320234708,0.2594887316,-1.8753159046,1.2990872860,1.1652233601,-0.4094679356,0.6926257014,0.4886541367,-0.3665470183,0.9978932142,-0.4778836966,-2.4469411373,0.8460370302,0.2013505697,-1.1429011822,2.0269160271,-0.3998430967],[0.2654949725,-1.0957469940,-0.7791578174,0.7616731524,1.3665670156,-0.8782943487,-0.8286035061,-1.1778832674,-0.8636802435,-0.0061646109,2.2168614864,1.0119706392,-0.2279826999,-0.6041381359,-0.6085149050,-1.2976219654,-0.2709690034,-0.0114500197,-0.4823000729,1.2648714781,-1.3784844875,0.6856546402,0.9222300053,0.6966942549,-1.8633707762,-0.1454697251,-1.3210371733,-0.1145494431,1.1239894629,1.9604538679,-0.4812983572,-0.5667022467,-1.0941706896,-0.7513188124,-0.4788049757,-0.9990747571,-0.7967398167,-0.9379817247,-0.9230493903,1.3206555843,-2.1616094112,0.9471166134,1.4199587107,0.6445659995,0.4961482286,0.4532108903,-1.3316901922,0.6221421361,1.1318142414,0.5100553036,-0.1593082994,-0.7789161801,-2.7416522503,1.1332864761,-1.0270265341,-0.2043064237,-0.8484025598,-0.1263597310,-0.1946774721,0.6200547814,-0.6168873906,-1.3963674307,-0.6660103798,0.0967090577,-1.6318593025,0.2810838521,0.6856861115,-1.1412400007,0.3920578957,0.3876029253,-2.0110924244,-0.6093295813,0.8308763504,-0.6013477445,1.4756437540,0.3031979799,-0.5353441238,-0.5915081501,0.9075561166,1.1504007578,0.6115757227,-2.3925230503,-1.2824686766,-1.3351628780,1.1867614985,0.2219233513,-0.8080751300,0.5201536417,-0.1467103064,1.5671451092,1.9779373407,-0.2780325115,0.3241499662,1.5429707766,0.3520541489,-0.0514822975,-0.1410155594,-1.3119984865,-0.3331850767,-1.3207861185,-0.0688070208,-0.1959996521,-1.2634282112,0.4501601756,-0.6468436122,0.6385756135,0.4816366732,0.8377054334,-1.1855635643,0.5892620087,-0.6773741245,-1.5393686295,1.6592665911,1.1991732121,-0.8384900093,-0.0617964156,-0.3647063375,0.0483825915,0.6977452040,0.8587312102,-0.5782601237,0.6752809882,1.8073829412,-0.4694879651,2.6854341030,-0.7661315799,-0.1617361754,-1.6490541697,-0.3861537874,1.1943726540,0.4325670004,0.9196245074,1.1767194271,-1.1586842537,2.2081480026,-0.8941178322,-0.1457774490,-1.3551640511,-2.0796351433,-1.5357860327,1.3167635202,-0.4685901403,-1.3264200687,-1.1433360577,0.5723722577,-1.3153508902,-0.0003914612,0.4272823632,-0.6007682085,-0.9464733005,0.9466275573,-0.3957338631,0.3242906034,0.3518322706,-0.9805750847,-1.4119974375,0.9316169024,-0.0752965063,-0.3946371078,1.0344431400,0.5991471410,0.7365334034,-0.8374543190,-0.0171916950,0.5245986581,-0.2836663425,-0.7435886264,-0.5999352336,0.5393315554,-0.9004894495,0.9330707788,-1.0388084650,0.5326125026,-0.2157468051,0.3797125518,-1.6414234638,-1.5240204334,-0.1943216473,2.4012343884,-0.1102185398,1.0404075384,-0.5839586258,-0.6843239665,-1.0446710587,1.3564828634,-1.1882144213,0.4350872934,0.2158848494,-0.3997232020,1.3127151728,0.7766410112,0.0745480806,0.8701055050,0.5700027943,-1.9929957390,1.7805351019,0.0368345529,-0.2137983143,0.9791098237,0.3465301096,-0.1790597886,0.7861873507,0.2161410749,1.9109491110,0.0643576086,1.4099440575,0.5143269300,-1.2256814241,0.3933446407,0.9667174220,-0.0542557761,-0.6940347552,-0.1299740523,-0.6604279876,0.0753633454,-0.4645026326,-0.9675666094,0.6419441700,1.0465860367,0.4986249506,-0.3357897699,-1.8439725637,-1.3316335678,-0.4952035844],[1.5557930470,0.6939328909,-0.9675522447,0.1987546235,0.0348333046,-1.2145023346,-1.0559561253,-1.6981366873,-0.6445525289,-0.4339305460,1.0280379057,1.7661401033,-1.4261125326,-1.3287851810,-0.8939813375,1.1151138544,0.4468123913,0.5887181163,-0.0540989488,0.8927987218,2.1387660503,1.7823724747,1.7096697092,1.8147362471,-0.1693580598,-1.0753818750,1.2679872513,-2.0095827579,0.6978594661,-1.2827425003,0.3344699740,-0.0405597240,0.1785971820,-1.7131185532,0.8015590310,-0.7889513373,0.4382426143,-0.2685180604,-0.4353355765,0.8980778456,1.4425247908,-0.6797144413,-0.5390772223,-1.2332694530,-0.8271765709,1.2962095737,-0.6857874990,0.4101184309,-1.0391228199,0.4739780426,0.2037101090,1.2740825415,-1.7347189188,1.1883305311,-0.2816534936,-0.0767895505,-0.1394811720,-0.5756545663,-0.4448460937,-0.9648231268,2.6027748585,-1.0214937925,-0.8026785254,0.1019904837,0.0143244155,-0.4809729457,-1.2737265825,-0.5467684269,1.2570265532,0.7116396427,1.6483726501,0.4744225144,-1.3267292976,-0.0733655021,-1.5249861479,-0.0464469939,2.2251887321,0.2355747223,1.2292549610,-0.8253579140,-0.1976063699,-2.2327098846,-0.2294290513,-0.8787239790,-0.4248799682,-0.9848347306,1.1690719128,-1.8133636713,-1.8084857464,0.0479786545,2.0665719509,-2.6742475033,0.5344931483,0.0683459938,-0.4919559062,-0.9514514208,-1.2987574339,-2.8886623383,-0.0731047243,-1.9328001738,0.9243254066,0.4220359325,-0.7849935889,-1.3458760977,0.3226920068,-0.6062769294,-1.6259837151,0.6885620952,1.7813590765,0.0121325897,1.6976926327,-0.3455361128,1.0504717827,0.1753101647,-0.0284505151,-0.2338887602,0.7708004713,0.4418765306,1.2308294773,-0.5319422483,-0.4612456560,0.3082421124,-0.5902484655,-0.2054267824,1.5642417669,0.1995749623,-1.3234007359,-0.6998391747,-0.6464892626,-0.3926881850,0.1428338438,0.2082134038,-0.3770585358,0.2527660429,2.2772750854,0.6177937388,-1.6407494545,0.0468776226,-0.9865619540,-0.4993889332,0.8691691160,-0.2714325190,0.9959207177,-0.4606694579,-1.1768029928,1.0289868116,0.7226113081,-0.7540940642,-0.7165128589,-0.5451242924,-0.4123264253,-0.8378821611,1.1969192028,0.2376316488,0.6620379090,-1.2662197351,-0.2605038881,-0.4416389763,1.6299383640,0.0120546557,0.1751725078,2.0253489017,-2.0747103691,0.7142991424,-0.8726304173,0.1750395596,0.4198315442,0.8568505049,-0.7676183581,1.2397425175,0.6619054079,-0.8535833955,0.9861800671,-1.0405623913,-0.2410328388,1.1349664927,0.2311729193,0.1398157328,0.7055028081,-1.5441834927,-0.2105818391,0.8658529520,-1.5761580467,-1.0042510033,-0.6100757122,-0.1319507957,1.9316471815,-0.3888913393,1.4629999399,0.3790009618,-0.7812678814,0.2428106815,1.7966394424,0.2591165304,-1.0846961737,0.4639006257,-0.3702415228,-1.3542098999,-2.5225086212,2.1170349121,0.2919540107,1.5617704391,-0.5720227361,0.1448655128,0.8777965903,-0.7311284542,0.6961936951,-0.2438761741,0.2898792624,-1.0157407522,0.1055957079,1.5368494987,-0.2627255023,0.5305185318,-0.9444933534,0.9583199024,-0.4513328671,-1.7957193851,-1.9104524851,1.5604406595,1.0587686300,-0.2916118801,-0.8341334462,0.6081914306],[0.5385888815,0.2649672329,1.9931534529,1.7101278305,-0.3339462578,-1.6201473475,0.4781301022,1.6592991352,1.8929585218,-0.5796220303,0.1620042324,-0.9191796780,0.2226093709,-0.3510306776,2.5146825314,1.3014285564,-0.5516109467,-1.7102047205,-0.8960490823,-0.9496671557,-0.4185546935,-0.6126506329,2.3513891697,-0.5045138597,0.2073156387,-0.0403192304,0.2158572525,1.6789017916,-0.0491207652,-0.0926134512,-0.9717449546,0.3141212761,-0.1822118014,1.5598946810,0.1269149184,-1.7584841251,-0.2879258394,-0.5144426823,0.8904726505,0.0560838953,1.0901510715,-0.3769854307,-0.6356492043,-0.2561465204,-0.2323470414,0.8959264159,1.5302934647,0.9458421469,-0.7589567900,1.3199381828,0.2553999126,0.0001370502,-0.9755280614,0.1034968719,-0.5387631655,1.1641650200,1.5335197449,-0.0891886353,1.0386070013,0.3652443886,0.4225503206,-0.9608186483,0.2354464382,0.8191389441,-0.0019620592,-2.2598648071,0.2033568025,0.9071549177,1.6220998764,0.0620047227,1.1191365719,-0.6640503407,-0.2967987657,-0.1196853444,0.4721096754,-1.5424871445,2.5747241974,-0.9410291314,0.2411335409,-0.5882598758,0.5838999152,1.2239956856,0.0473095626,0.0269213933,-1.7474669218,-0.1254788041,0.6390517950,-0.1682768017,0.1148226485,-0.8987408876,-1.1397955418,1.4998438358,1.4500621557,0.1748585701,-0.1256774962,0.6514495611,-1.3195337057,-1.3019288778,0.1573456079,1.1976250410,-0.6434811950,-0.0719533190,-0.0295303278,-0.9087253809,0.6040388942,-0.3942040503,0.8656547666,1.3265830278,0.1610490084,-0.5010558963,-1.4810063839,-0.2897340357,-0.6645922661,0.2873976231,0.0032383415,-2.2635951042,0.7036095262,0.2958571017,1.3461945057,0.4321117997,-1.0315165520,0.9034367800,-1.0506396294,0.3633961380,1.2920157909,-0.1767129153,0.9583020210,1.9570393562,-0.0153314872,0.2814647257,-0.3974483609,-0.2976866663,-1.3961068392,-1.6619836092,-1.1750111580,-0.3427028060,0.8588997126,-1.0289971828,-0.1670207977,-0.2226491570,0.5565141439,0.3041157424,0.1601631343,-0.6869891882,0.0558025204,-0.1517806500,0.2210749239,-1.2610491514,0.4965585470,-0.8622374535,-0.2092900723,1.9283560514,2.0117387772,1.2364124060,-1.2820222378,-0.5927175283,-1.2122610807,-0.2559721172,0.3717762232,-0.4502603412,1.7123154402,0.3875474632,0.6138361096,0.4190999269,0.7165051699,-0.9974407554,0.6195259690,-1.8539083004,-0.8086301088,-1.7899594307,0.5038084984,-0.9629930854,1.0737767220,-0.0268682484,-1.5845243931,1.5086864233,0.6216324568,1.9804166555,-1.0086367130,2.2385258675,-0.5203742385,-0.8196234703,1.1165432930,-1.2808620930,1.2763589621,-0.9164367914,0.1268753260,2.0863809586,-0.7245088816,-1.2106389999,1.2202898264,-0.8158519864,0.8383903503,1.5530946255,-0.1538568288,0.2419621795,-1.1090013981,2.5024588108,0.1697666496,1.0527145863,0.6301341653,-1.3145409822,-0.5830605030,-0.2038480788,-0.1603022367,-0.6039440632,0.6921498179,0.5524765253,0.8396931291,0.4275787771,0.5381883383,-0.0318626240,2.2095665932,1.4112273455,0.0706845075,0.3475209475,0.3752060533,-0.5024744272,0.4471912980,0.3937501609,-0.4151968062,1.5611482859,1.7722326517,-0.4225720763]]]] \ No newline at end of file diff --git a/csharp/CSharpUsage/testdata/squeezenet.onnx b/csharp/CSharpUsage/testdata/squeezenet.onnx new file mode 100644 index 0000000000000..810cf8f88bf73 Binary files /dev/null and b/csharp/CSharpUsage/testdata/squeezenet.onnx differ diff --git a/csharp/Nuget.CSharp.config b/csharp/Nuget.CSharp.config new file mode 100644 index 0000000000000..5fe5cc9e5472f --- /dev/null +++ b/csharp/Nuget.CSharp.config @@ -0,0 +1,7 @@ + + + + + + + diff --git a/csharp/OnnxRuntime.CSharp.proj b/csharp/OnnxRuntime.CSharp.proj new file mode 100644 index 0000000000000..2b8ff6e89ffa2 --- /dev/null +++ b/csharp/OnnxRuntime.CSharp.proj @@ -0,0 +1,49 @@ + + + + + + + Debug + AnyCPU + bin\$(Platform)\$(Configuration)\ + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/csharp/OnnxRuntime.CSharp.sln b/csharp/OnnxRuntime.CSharp.sln new file mode 100644 index 0000000000000..7e1aeeb52924d --- /dev/null +++ b/csharp/OnnxRuntime.CSharp.sln @@ -0,0 +1,37 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.28010.2003 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.OnnxRuntime", "src\Microsoft.ML.OnnxRuntime\Microsoft.ML.OnnxRuntime.csproj", "{584B53B3-359D-4DC2-BCD8-530B5D4685AD}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.OnnxRuntime.InferenceSample", "sample\Microsoft.ML.OnnxRuntime.InferenceSample\Microsoft.ML.OnnxRuntime.InferenceSample.csproj", "{1AA14958-9246-4163-9403-F650E65ADCBC}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.OnnxRuntime.Tests", "test\Microsoft.ML.OnnxRuntime.Tests\Microsoft.ML.OnnxRuntime.Tests.csproj", "{50173D13-DF29-42E7-A30B-8B12D36C77B1}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {584B53B3-359D-4DC2-BCD8-530B5D4685AD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {584B53B3-359D-4DC2-BCD8-530B5D4685AD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {584B53B3-359D-4DC2-BCD8-530B5D4685AD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {584B53B3-359D-4DC2-BCD8-530B5D4685AD}.Release|Any CPU.Build.0 = Release|Any CPU + {1AA14958-9246-4163-9403-F650E65ADCBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1AA14958-9246-4163-9403-F650E65ADCBC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1AA14958-9246-4163-9403-F650E65ADCBC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1AA14958-9246-4163-9403-F650E65ADCBC}.Release|Any CPU.Build.0 = Release|Any CPU + {50173D13-DF29-42E7-A30B-8B12D36C77B1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {50173D13-DF29-42E7-A30B-8B12D36C77B1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {50173D13-DF29-42E7-A30B-8B12D36C77B1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {50173D13-DF29-42E7-A30B-8B12D36C77B1}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {C3DBDA2B-F169-4EDE-9353-858904124B75} + EndGlobalSection +EndGlobal diff --git a/csharp/OnnxRuntime.proj b/csharp/OnnxRuntime.proj new file mode 100644 index 0000000000000..344455b69ecc3 --- /dev/null +++ b/csharp/OnnxRuntime.proj @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/csharp/OnnxRuntime.sln b/csharp/OnnxRuntime.sln new file mode 100644 index 0000000000000..ae96af7512211 --- /dev/null +++ b/csharp/OnnxRuntime.sln @@ -0,0 +1,31 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.28010.2003 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.OnnxRuntime", "OnnxRuntime\Microsoft.ML.OnnxRuntime.csproj", "{584B53B3-359D-4DC2-BCD8-530B5D4685AD}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CSharpUsage", "CSharpUsage\CSharpUsage.csproj", "{1AA14958-9246-4163-9403-F650E65ADCBC}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {584B53B3-359D-4DC2-BCD8-530B5D4685AD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {584B53B3-359D-4DC2-BCD8-530B5D4685AD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {584B53B3-359D-4DC2-BCD8-530B5D4685AD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {584B53B3-359D-4DC2-BCD8-530B5D4685AD}.Release|Any CPU.Build.0 = Release|Any CPU + {1AA14958-9246-4163-9403-F650E65ADCBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1AA14958-9246-4163-9403-F650E65ADCBC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1AA14958-9246-4163-9403-F650E65ADCBC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1AA14958-9246-4163-9403-F650E65ADCBC}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {C3DBDA2B-F169-4EDE-9353-858904124B75} + EndGlobalSection +EndGlobal diff --git a/csharp/OnnxRuntime/Exceptions.cs b/csharp/OnnxRuntime/Exceptions.cs new file mode 100644 index 0000000000000..411bece218165 --- /dev/null +++ b/csharp/OnnxRuntime/Exceptions.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.ML.OnnxRuntime +{ + + /// + /// Enum conresponding to native onnxruntime error codes. Must be in sync with the native API + /// + internal enum ErrorCode + { + Ok = 0, + Fail = 1, + InvalidArgument = 2, + NoSuchFile = 3, + NoModel = 4, + EngineError = 5, + RuntimeException = 6, + InvalidProtobuf = 7, + ModelLoaded = 8, + NotImplemented = 9, + InvalidGraph = 10, + ShapeInferenceNotRegistered = 11, + RequirementNotRegistered = 12 + } + + public class OnnxRuntimeException: Exception + { + public OnnxRuntimeException(string message) + :base(message) + { + } + } + + + public class CoreRuntimeException : OnnxRuntimeException + { + private static Dictionary errorCodeToString = new Dictionary() + { + { ErrorCode.Ok, "Ok" }, + { ErrorCode.Fail, "Fail" }, + { ErrorCode.InvalidArgument, "InvalidArgument"} , + { ErrorCode.NoSuchFile, "NoSuchFile" }, + { ErrorCode.NoModel, "NoModel" }, + { ErrorCode.EngineError, "EngineError" }, + { ErrorCode.RuntimeException, "RuntimeException" }, + { ErrorCode.InvalidProtobuf, "InvalidProtobuf" }, + { ErrorCode.ModelLoaded, "ModelLoaded" }, + { ErrorCode.NotImplemented, "NotImplemented" }, + { ErrorCode.InvalidGraph, "InvalidGraph" }, + { ErrorCode.ShapeInferenceNotRegistered, "ShapeInferenceNotRegistered" }, + { ErrorCode.RequirementNotRegistered, "RequirementNotRegistered" } + }; + + internal CoreRuntimeException(ErrorCode errorCode, string message) + :base("[ErrorCode:" + errorCodeToString[errorCode] + "] " + message) + { + } + } + + +} diff --git a/csharp/OnnxRuntime/InferenceSession.cs b/csharp/OnnxRuntime/InferenceSession.cs new file mode 100644 index 0000000000000..fc44ee730b14a --- /dev/null +++ b/csharp/OnnxRuntime/InferenceSession.cs @@ -0,0 +1,217 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Runtime.InteropServices; +using System.Collections.Generic; +using System.IO; +using System.Linq; + + +namespace Microsoft.ML.OnnxRuntime +{ + + public struct RunOptions + { + // placeholder for RunOptions + } + + /// + /// Represents an Inference Session against an ONNX Model + /// + public class InferenceSession: IDisposable + { + protected IntPtr _nativeHandle; + + + internal InferenceSession(IntPtr nativeHandle) + { + _nativeHandle = nativeHandle; + } + + #region Public API + public InferenceSession(string modelPath) + : this(modelPath, SessionOptions.Default) + { + } + + public InferenceSession(string modelPath, SessionOptions options) + { + var envHandle = OnnxRuntime.Instance.NativeHandle; + IntPtr outputHandle; + + IntPtr status = NativeMethods.ONNXRuntimeCreateInferenceSession(envHandle, modelPath, options.NativeHandle, out outputHandle); + + _nativeHandle = IntPtr.Zero; + NativeApiStatus.VerifySuccess(status); + _nativeHandle = outputHandle; + } + + public IReadOnlyDictionary InputMetadata + { + get + { + return null; // TODO: implement + } + } + + public IReadOnlyDictionary OutputMetadata + { + get + { + return null; // TODO: implement + } + } + + public ModelMetadata ModelMetadata + { + get + { + return new ModelMetadata(); //TODO: implement + } + } + + public IReadOnlyList Run(IReadOnlyList inputs, RunOptions options = new RunOptions()) + { + var inputNames = new string[inputs.Count]; + var inputTensors = new IntPtr[inputs.Count]; + var pinnedBufferHandles = new System.Buffers.MemoryHandle[inputs.Count]; + + for (int i = 0; i < inputs.Count; i++) + { + inputNames[i] = inputs[i].Name; + + // create Tensor fromt the inputs[i] if feasible, else throw notsupported exception for now + inputs[i].ToNativeOnnxValue(out inputTensors[i], out pinnedBufferHandles[i]); + } + + IntPtr outputValueList = IntPtr.Zero; + ulong outputLength = 0; + IntPtr status = NativeMethods.ONNXRuntimeRunInferenceAndFetchAll( + this._nativeHandle, + inputNames, + inputTensors, + (uint)(inputTensors.Length), + out outputValueList, + out outputLength + ); //Note: the inputTensors and pinnedBufferHandles must be alive for the duration of the call + + try + { + NativeApiStatus.VerifySuccess(status); + var result = new List(); + for (uint i = 0; i < outputLength; i++) + { + IntPtr tensorValue = NativeMethods.ONNXRuntimeONNXValueListGetNthValue(outputValueList, i); + result.Add(NamedOnnxValue.CreateFromOnnxValue(Convert.ToString(i), tensorValue)); // TODO: currently Convert.ToString(i) is used instead of the output name, for the absense of C-api. + // Will be fixed as soon as the C-api for output name is available + } + + return result; + } + catch (OnnxRuntimeException e) + { + //clean up the individual output tensors if it is not null; + if (outputValueList != IntPtr.Zero) + { + for (uint i = 0; i < outputLength; i++) + { + IntPtr tensorValue = NativeMethods.ONNXRuntimeONNXValueListGetNthValue(outputValueList, i); + NativeMethods.ReleaseONNXValue(tensorValue); + } + } + throw e; + } + finally + { + // always unpin the input buffers, and delete the native Onnx value objects + for (int i = 0; i < inputs.Count; i++) + { + NativeMethods.ReleaseONNXValue(inputTensors[i]); // this should not release the buffer, but should delete the native tensor object + pinnedBufferHandles[i].Dispose(); + } + + // always release the output value list, because the individual tensor pointers are already obtained. + if (outputValueList != IntPtr.Zero) + { + NativeMethods.ReleaseONNXValueList(outputValueList); + } + } + } + + + + /// + /// Runs the loaded model for the given inputs, and fetches the specified outputs in . + /// + /// + /// + /// + /// Output Tensors in a Dictionary + public IReadOnlyList Run(IReadOnlyList inputs, ICollection outputNames, RunOptions options = new RunOptions()) + { + //TODO: implement + return null; + } + + + #endregion + + #region private methods + + + #endregion + + + + #region destructors disposers + + + ~InferenceSession() + { + Dispose(false); + } + + public void Dispose() + { + GC.SuppressFinalize(this); + Dispose(true); + } + + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + // cleanup managed resources + } + + // cleanup unmanaged resources + if (_nativeHandle != IntPtr.Zero) + { + NativeMethods.ReleaseONNXSession(_nativeHandle); + } + } + + #endregion + + } + + public struct NodeMetadata + { + public uint[] Shape + { + get; internal set; + } + public System.Type Type + { + get; internal set; + } + } + + + public struct ModelMetadata + { + //placeholder for Model metadata. Python API has this + } + +} diff --git a/csharp/OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj b/csharp/OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj new file mode 100644 index 0000000000000..49df9a66d6350 --- /dev/null +++ b/csharp/OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj @@ -0,0 +1,15 @@ + + + + netstandard1.1 + true + true + false + OnnxRuntime.snk + + + + + + + diff --git a/csharp/OnnxRuntime/NamedOnnxValue.cs b/csharp/OnnxRuntime/NamedOnnxValue.cs new file mode 100644 index 0000000000000..0a0fa404248c3 --- /dev/null +++ b/csharp/OnnxRuntime/NamedOnnxValue.cs @@ -0,0 +1,322 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Numerics.Tensors; +using System.Buffers; +using System.Collections; +using System.Diagnostics; + +namespace Microsoft.ML.OnnxRuntime +{ + public class NamedOnnxValue + { + protected Object _value; + protected string _name; + + public NamedOnnxValue(string name, Object value) + { + _name = name; + _value = value; + } + + public string Name { get { return _name; } } + public Tensor AsTensor() + { + return _value as Tensor; // will return null if not castable + } + + /// + /// Attempts to Pin the buffer, and create a native OnnxValue out of it. the pinned MemoryHandle is passed to output. + /// In this case, the pinnedHandle should be kept alive till the native OnnxValue is used, then dispose it. + /// If it is not possible to Pin the buffer, then creates OnnxValue from the copy of the data. The output pinnedMemoryHandle + /// contains a default value in that case. + /// Attempts to infer the type of the value while creating the OnnxValue + /// + /// + /// + internal void ToNativeOnnxValue(out IntPtr onnxValue, out MemoryHandle pinnedMemoryHandle) + { + //try to cast _value to Tensor + TensorElementType nativeElementType = TensorElementType.DataTypeMax; //invalid + IntPtr dataBufferPointer = IntPtr.Zero; + int dataBufferLength = 0; + ReadOnlySpan shape = null; + int rank = 0; + onnxValue = IntPtr.Zero; + + if (TryPinAsTensor(out pinnedMemoryHandle, + out dataBufferPointer, + out dataBufferLength, + out shape, + out rank, + out nativeElementType + )) + { + } + else if (TryPinAsTensor(out pinnedMemoryHandle, + out dataBufferPointer, + out dataBufferLength, + out shape, + out rank, + out nativeElementType + )) + { + } + else if (TryPinAsTensor(out pinnedMemoryHandle, + out dataBufferPointer, + out dataBufferLength, + out shape, + out rank, + out nativeElementType + )) + { + } + else if (TryPinAsTensor(out pinnedMemoryHandle, + out dataBufferPointer, + out dataBufferLength, + out shape, + out rank, + out nativeElementType + )) + { + } + else if (TryPinAsTensor(out pinnedMemoryHandle, + out dataBufferPointer, + out dataBufferLength, + out shape, + out rank, + out nativeElementType + )) + { + } + else if (TryPinAsTensor(out pinnedMemoryHandle, + out dataBufferPointer, + out dataBufferLength, + out shape, + out rank, + out nativeElementType + )) + { + } + else if (TryPinAsTensor(out pinnedMemoryHandle, + out dataBufferPointer, + out dataBufferLength, + out shape, + out rank, + out nativeElementType + )) + { + } + else if (TryPinAsTensor(out pinnedMemoryHandle, + out dataBufferPointer, + out dataBufferLength, + out shape, + out rank, + out nativeElementType + )) + { + } + else if (TryPinAsTensor(out pinnedMemoryHandle, + out dataBufferPointer, + out dataBufferLength, + out shape, + out rank, + out nativeElementType + )) + { + } + else if (TryPinAsTensor(out pinnedMemoryHandle, + out dataBufferPointer, + out dataBufferLength, + out shape, + out rank, + out nativeElementType + )) + { + } + + //TODO: add other types + else + { + // nothing to cleanup here, since no memory has been pinned + throw new NotSupportedException("The inference value " + nameof(_value) + " is not of a supported type"); + } + + + Debug.Assert(dataBufferPointer != IntPtr.Zero, "dataBufferPointer must be non-null after obtaining the pinned buffer"); + + // copy to an ulong[] shape to match size_t[] + ulong[] longShape = new ulong[rank]; + for (int i = 0; i < rank; i++) + { + longShape[i] = (ulong)shape[i]; + } + + IntPtr status = NativeMethods.ONNXRuntimeCreateTensorWithDataAsONNXValue( + NativeCpuAllocatorInfo.Handle, + dataBufferPointer, + (ulong)(dataBufferLength), + longShape, + (ulong)rank, + nativeElementType, + out onnxValue + ); + try + { + NativeApiStatus.VerifySuccess(status); + } + catch (OnnxRuntimeException e) + { + pinnedMemoryHandle.Dispose(); + throw e; + } + + } + + internal static NamedOnnxValue CreateFromOnnxValue(string name, IntPtr nativeOnnxValue) + { + NamedOnnxValue result = null; + + if (true /* TODO: check native data type when API available. assuming Tensor for now */) + { + NativeOnnxTensorMemory nativeTensorWrapper = new NativeOnnxTensorMemory(nativeOnnxValue); + DenseTensor dt = new DenseTensor(nativeTensorWrapper.Memory, nativeTensorWrapper.Dimensions); + result = new NamedOnnxValue(name, dt); + } + + return result; + } + + private bool TryPinAsTensor( + out MemoryHandle pinnedMemoryHandle, + out IntPtr dataBufferPointer, + out int dataBufferLength, + out ReadOnlySpan shape, + out int rank, + out TensorElementType nativeElementType + ) + { + nativeElementType = TensorElementType.DataTypeMax; //invalid + dataBufferPointer = IntPtr.Zero; + dataBufferLength = 0; + shape = null; + rank = 0; + pinnedMemoryHandle = default(MemoryHandle); + + if (_value is Tensor) + { + Tensor t = _value as Tensor; + if (t.IsReversedStride) + { + //TODO: not sure how to support reverse stride. may be able to calculate the shape differently + throw new NotSupportedException(nameof(Tensor) + " of reverseStride is not supported"); + } + + DenseTensor dt = null; + if (_value is DenseTensor) + { + dt = _value as DenseTensor; + } + else + { + dt = t.ToDenseTensor(); + } + + shape = dt.Dimensions; // does not work for reverse stride + rank = dt.Rank; + pinnedMemoryHandle = dt.Buffer.Pin(); + unsafe + { + dataBufferPointer = (IntPtr)pinnedMemoryHandle.Pointer; + } + + // find the native type + if (typeof(T) == typeof(float)) + { + nativeElementType = TensorElementType.Float; + dataBufferLength = dt.Buffer.Length * sizeof(float); + } + else if (typeof(T) == typeof(double)) + { + nativeElementType = TensorElementType.Double; + dataBufferLength = dt.Buffer.Length * sizeof(double); + } + else if (typeof(T) == typeof(int)) + { + nativeElementType = TensorElementType.Int32; + dataBufferLength = dt.Buffer.Length * sizeof(int); + } + else if (typeof(T) == typeof(uint)) + { + nativeElementType = TensorElementType.UInt32; + dataBufferLength = dt.Buffer.Length * sizeof(uint); + } + else if (typeof(T) == typeof(long)) + { + nativeElementType = TensorElementType.Int64; + dataBufferLength = dt.Buffer.Length * sizeof(long); + } + else if (typeof(T) == typeof(ulong)) + { + nativeElementType = TensorElementType.UInt64; + dataBufferLength = dt.Buffer.Length * sizeof(ulong); + } + else if (typeof(T) == typeof(short)) + { + nativeElementType = TensorElementType.Int16; + dataBufferLength = dt.Buffer.Length * sizeof(short); + } + else if (typeof(T) == typeof(ushort)) + { + nativeElementType = TensorElementType.UInt16; + dataBufferLength = dt.Buffer.Length * sizeof(ushort); + } + else if (typeof(T) == typeof(byte)) + { + nativeElementType = TensorElementType.UInt8; + dataBufferLength = dt.Buffer.Length * sizeof(byte); + } + //TODO: Not supporting boolean for now. bool is non-blittable, the interop needs some care, and possibly need to copy + //else if (typeof(T) == typeof(bool)) + //{ + //} + else + { + //TODO: may extend the supported types + // do not throw exception, rather assign the sentinel value + nativeElementType = TensorElementType.DataTypeMax; + } + return true; + } + + return false; + } + + // may expose different types of getters in future + } + + internal enum TensorElementType + { + Float = 1, + UInt8 = 2, + Int8 = 3, + UInt16 = 4, + Int16 = 5, + Int32 = 6, + Int64 = 7, + String = 8, + Bool = 9, + Float16 = 10, + Double = 11, + UInt32 = 12, + UInt64 = 13, + Complex64 = 14, + Complex128 = 15, + BFloat16 = 16, + DataTypeMax = 17 + } + +} diff --git a/csharp/OnnxRuntime/NativeApiStatus.cs b/csharp/OnnxRuntime/NativeApiStatus.cs new file mode 100644 index 0000000000000..bc4626d54987d --- /dev/null +++ b/csharp/OnnxRuntime/NativeApiStatus.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Runtime.InteropServices; + +namespace Microsoft.ML.OnnxRuntime +{ + class NativeApiStatus + { + private static string GetErrorMessage(IntPtr /*(ONNXStatus*)*/status) + { + IntPtr nativeString = NativeMethods.ONNXRuntimeGetErrorMessage(status); + string str = Marshal.PtrToStringAnsi(nativeString); //assumes charset = ANSI + return str; + } + + /// + /// Checks the native Status if the errocode is OK/Success. Otherwise constructs an appropriate exception and throws. + /// Releases the native status object, as needed. + /// + /// + /// + public static void VerifySuccess(IntPtr nativeStatus) + { + if (nativeStatus != IntPtr.Zero) + { + ErrorCode statusCode = NativeMethods.ONNXRuntimeGetErrorCode(nativeStatus); + string errorMessage = GetErrorMessage(nativeStatus); + NativeMethods.ReleaseONNXStatus(nativeStatus); + throw new CoreRuntimeException(statusCode, errorMessage); + } + } + } +} diff --git a/csharp/OnnxRuntime/NativeCpuAllocatorInfo.cs b/csharp/OnnxRuntime/NativeCpuAllocatorInfo.cs new file mode 100644 index 0000000000000..bc9c38fe33d55 --- /dev/null +++ b/csharp/OnnxRuntime/NativeCpuAllocatorInfo.cs @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.ML.OnnxRuntime +{ + internal class NativeCpuAllocatorInfo : IDisposable + { + // static singleton + private static readonly Lazy _instance = new Lazy(() => new NativeCpuAllocatorInfo()); + + // member variables + private IntPtr _nativeHandle; + + internal static IntPtr Handle // May throw exception in every access, if the constructor have thrown an exception + { + get + { + return _instance.Value._nativeHandle; + } + } + + private NativeCpuAllocatorInfo() + { + _nativeHandle = CreateCPUAllocatorInfo(); + } + + private static IntPtr CreateCPUAllocatorInfo() + { + IntPtr allocInfo = IntPtr.Zero; + try + { + IntPtr status = NativeMethods.ONNXRuntimeCreateCpuAllocatorInfo(NativeMethods.AllocatorType.DeviceAllocator, NativeMethods.MemoryType.Cpu, out allocInfo); + NativeApiStatus.VerifySuccess(status); + return allocInfo; + } + catch (Exception e) + { + if (allocInfo != IntPtr.Zero) + { + NativeMethods.ReleaseONNXRuntimeAllocatorInfo(allocInfo); + } + throw e; + } + } + + ~NativeCpuAllocatorInfo() + { + Dispose(false); + } + + public void Dispose() + { + GC.SuppressFinalize(this); + Dispose(true); + } + + private void Dispose(bool disposing) + { + if (disposing) + { + //release managed resource + } + + //release unmanaged resource + if (_nativeHandle != IntPtr.Zero) + { + NativeMethods.ReleaseONNXRuntimeAllocatorInfo(_nativeHandle); + } + } + } +} diff --git a/csharp/OnnxRuntime/NativeMethods.cs b/csharp/OnnxRuntime/NativeMethods.cs new file mode 100644 index 0000000000000..6475182472f9b --- /dev/null +++ b/csharp/OnnxRuntime/NativeMethods.cs @@ -0,0 +1,236 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Runtime.InteropServices; + +namespace Microsoft.ML.OnnxRuntime +{ + /// + /// NamedOnnxValue type, must match the native enum + /// + + internal static class NativeMethods + { + private const string nativeLib = "onnxruntime.dll"; + internal const CharSet charSet = CharSet.Ansi; + + #region Runtime/Environment API + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /* ONNXStatus* */ONNXRuntimeInitialize( + LogLevel default_warning_level, + string logId, + out IntPtr /*(ONNXEnv*)*/ env); + // ReleaseONNXEnv should not be used + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ReleaseONNXEnv(IntPtr /*(ONNXEnv*)*/ env); + #endregion Runtime/Environment API + + #region Status API + [DllImport(nativeLib, CharSet = charSet)] + public static extern ErrorCode ONNXRuntimeGetErrorCode(IntPtr /*(ONNXStatus*)*/status); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /* char* */ONNXRuntimeGetErrorMessage(IntPtr /* (ONNXStatus*) */status); + // returns char*, need to convert to string by the caller. + // does not free the underlying ONNXStatus* + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ReleaseONNXStatus(IntPtr /*(ONNXStatus*)*/ statusPtr); + + #endregion Status API + + #region InferenceSession API + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /* ONNXStatus* */ONNXRuntimeCreateInferenceSession( + IntPtr /* (ONNXEnv*) */ environment, + [MarshalAs(UnmanagedType.LPWStr)]string modelPath, //the model path is consumed as a wchar* in the C-api + IntPtr /* (ONNXRuntimeSessionOptions*) */sessopnOptions, + out IntPtr /**/ session); + + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(ONNStatus*)*/ ONNXRuntimeRunInferenceAndFetchAll( + IntPtr /*(ONNXSessionPtr)*/ session, + string[] inputNames, + IntPtr[] /*(ONNXValuePtr[])*/ inputValues, + ulong inputLength, // size_t, TODO: make it portable for x86, arm + out IntPtr /* (ONNXValueListPtr*)*/ outputValues, + out ulong /* (size_t*) */ outputLength); //TODO: make it portable for x86, arm + + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(ONNStatus*)*/ ONNXRuntimeRunInference( + IntPtr /*(ONNXSession*)*/ session, + string[] inputNames, + IntPtr[] /* (ONNXValue*[])*/ inputValues, + ulong inputCount, /* TODO: size_t, make it portable for x86 arm */ + string[] outputNames, + ulong outputCount, /* TODO: size_t, make it portable for x86 and arm */ + + [MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 5 /*index of outputCount*/)][In, Out] + IntPtr[] outputValues /* An array of output value pointers. Array must be allocated by the caller */ + ); + + + [DllImport(nativeLib, CharSet = charSet)] + public static extern int ONNXRuntimeInferenceSessionGetInputCount(IntPtr /*(ONNXSession*)*/ session); + + + [DllImport(nativeLib, CharSet = charSet)] + public static extern int ONNXRuntimeInferenceSessionGetOutputCount(IntPtr /*(ONNXSession*)*/ session); + + //TODO: need the input/output names API + //TODO: need the input/output shape/type API + + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ReleaseONNXSession(IntPtr /*(ONNXSession*)*/ session); + + #endregion InferenceSession API + + #region SessionOptions API + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*ONNXRuntimeSessionOptions* */ ONNXRuntimeCreateSessionOptions(); + + //DEFINE_RUNTIME_CLASS(ONNXRuntimeSessionOptions) + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ReleaseONNXRuntimeSessionOptions(IntPtr /*(ONNXRuntimeSessionOptions*)*/ sessionOptions); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeEnableSequentialExecution(IntPtr /* ONNXRuntimeSessionOptions* */ options); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeDisableSequentialExecution(IntPtr /* ONNXRuntimeSessionOptions* */ options); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeEnableProfiling(IntPtr /* ONNXRuntimeSessionOptions* */ options, string profilePathPrefix); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeDisableProfiling(IntPtr /* ONNXRuntimeSessionOptions* */ options); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeEnableMemPattern(IntPtr /* ONNXRuntimeSessionOptions* */ options); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeDisableMemPattern(IntPtr /* ONNXRuntimeSessionOptions* */ options); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeEnableCpuMemArena(IntPtr /* ONNXRuntimeSessionOptions* */ options); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeDisableCpuMemArena(IntPtr /* ONNXRuntimeSessionOptions* */ options); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeSetSessionLogId(IntPtr /* ONNXRuntimeSessionOptions* */ options, string logId); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeSetSessionLogVerbosityLevel(IntPtr /* ONNXRuntimeSessionOptions* */ options, LogLevel sessionLogVerbosityLevel); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern int ONNXRuntimeSetSessionThreadPoolSize(IntPtr /* ONNXRuntimeSessionOptions* */ options, int sessionThreadPoolSize); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern int ONNXRuntimeEnableCudaProvider(IntPtr /* ONNXRuntimeSessionOptions* */ options, int deviceId); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeDisableCudaProvider(IntPtr /* ONNXRuntimeSessionOptions* */ options); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern int ONNXRuntimeEnableMklProvider(IntPtr /* ONNXRuntimeSessionOptions* */ options); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeDisableMklProvider(IntPtr /* ONNXRuntimeSessionOptions* */ options); + + #endregion + + #region Allocator/AllocatorInfo API + + //TODO: consider exposing them publicly, when allocator API is exposed + public enum AllocatorType + { + DeviceAllocator = 0, + ArenaAllocator = 1 + } + + //TODO: consider exposing them publicly when allocator API is exposed + public enum MemoryType + { + CpuInput = -2, // Any CPU memory used by non-CPU execution provider + CpuOutput = -1, // CPU accessible memory outputted by non-CPU execution provider, i.e. CUDA_PINNED + Cpu = CpuOutput, // temporary CPU accessible memory allocated by non-CPU execution provider, i.e. CUDA_PINNED + Default = 0, // the default allocator for execution provider + } + + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /* (ONNXStatus*)*/ ONNXRuntimeCreateAllocatorInfo( + IntPtr /*(const char*) */name, + AllocatorType allocatorType, + int identifier, + MemoryType memType, + out IntPtr /*(ONNXRuntimeAllocatorInfo*)*/ allocatorInfo // memory ownership transfered to caller + ); + + //ONNXRUNTIME_API_STATUS(ONNXRuntimeCreateCpuAllocatorInfo, enum ONNXRuntimeAllocatorType type, enum ONNXRuntimeMemType mem_type1, _Out_ ONNXRuntimeAllocatorInfo** out) + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /* (ONNXStatus*)*/ ONNXRuntimeCreateCpuAllocatorInfo( + AllocatorType allocatorType, + MemoryType memoryType, + out IntPtr /*(ONNXRuntimeAllocatorInfo*)*/ allocatorInfo + ); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ReleaseONNXRuntimeAllocatorInfo(IntPtr /*(ONNXRuntimeAllocatorInfo*)*/ allocatorInfo); + + #endregion Allocator/AllocatorInfo API + + #region Tensor/OnnxValue API + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /* ONNXStatus */ ONNXRuntimeCreateTensorWithDataAsONNXValue( + IntPtr /* (const ONNXRuntimeAllocatorInfo*) */ allocatorInfo, + IntPtr /* (void*) */dataBufferHandle, + ulong dataLength, //size_t, TODO: make it portable for x86, arm + ulong[] shape, //size_t* or size_t[], TODO: make it portable for x86, arm + ulong shapeLength, //size_t, TODO: make it portable for x86, arm + TensorElementType type, + out IntPtr /* ONNXValuePtr* */ outputValue); + + /// This function doesn't work with string tensor + /// this is a no-copy method whose pointer is only valid until the backing ONNXValuePtr is free'd. + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeGetTensorMutableData(IntPtr /*(ONNXValue*)*/ value, out IntPtr /* (void**)*/ dataBufferHandle); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeGetTensorShapeDimCount(IntPtr /*(ONNXValue*)*/ value, out ulong dimension); //size_t TODO: make it portable for x86, arm + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeGetTensorShapeElementCount(IntPtr /*(ONNXValue*)*/value, out ulong count); + + ///** + // * Generally, user should call ONNXRuntimeGetTensorShapeDimCount before calling this. + // * Unless they already have a good estimation on the dimension count + // * \param shape_array An array allocated by caller, with size of shape_array + // * \param shape_array_len the length of passed in shape_array. + // */ + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(ONNXStatus*)*/ONNXRuntimeGetTensorShape( + IntPtr /*(ONNXValue*)*/ value, + ulong[] shapeArray, //size_t[] TODO: make it portable for x86, arm + ulong shapeArrayLength); //size_t, TODO: make it portable for x86, arm + + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(ONNXValuePtr)*/ ONNXRuntimeONNXValueListGetNthValue(IntPtr /*(ONNXValueListPtr)*/ list, ulong index); // 0-based index TODO: size_t, make it portable for x86, arm + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ReleaseONNXValue(IntPtr /*(ONNXValue*)*/ value); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ReleaseONNXValueList(IntPtr /*(ONNXValueList*)*/ valueList); + + #endregion + } //class NativeMethods +} //namespace diff --git a/csharp/OnnxRuntime/NativeOnnxTensorMemory.cs b/csharp/OnnxRuntime/NativeOnnxTensorMemory.cs new file mode 100644 index 0000000000000..55d1079d93da9 --- /dev/null +++ b/csharp/OnnxRuntime/NativeOnnxTensorMemory.cs @@ -0,0 +1,170 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; + + +namespace Microsoft.ML.OnnxRuntime +{ + internal class NativeOnnxTensorMemory : MemoryManager + { + private bool _disposed; + private int _referenceCount; + private IntPtr _onnxValueHandle; + private IntPtr _dataBufferHandle; + private int _elementCount; + private int _elementWidth; + private int[] _dimensions; + + public NativeOnnxTensorMemory(IntPtr onnxValueHandle) + { + //TODO: check type param and the native tensor type + if (typeof(T) != typeof(float)) + throw new NotSupportedException(nameof(NativeOnnxTensorMemory)+" does not support T other than float"); + _elementWidth = 4; + + _onnxValueHandle = onnxValueHandle; + // derive the databuffer pointer, element_count, element_width, and shape + try + { + NativeApiStatus.VerifySuccess(NativeMethods.ONNXRuntimeGetTensorMutableData(_onnxValueHandle, out _dataBufferHandle)); + // throws OnnxRuntimeException if native call failed + ulong dimension; + NativeApiStatus.VerifySuccess(NativeMethods.ONNXRuntimeGetTensorShapeDimCount(_onnxValueHandle, out dimension)); + ulong count; + NativeApiStatus.VerifySuccess(NativeMethods.ONNXRuntimeGetTensorShapeElementCount(_onnxValueHandle, out count)); + ulong[] shape = new ulong[dimension]; + NativeApiStatus.VerifySuccess(NativeMethods.ONNXRuntimeGetTensorShape(_onnxValueHandle, shape, dimension)); //Note: shape must be alive during the call + + _elementCount = (int)count; + _dimensions = new int[dimension]; + for (ulong i = 0; i < dimension; i++) + { + _dimensions[i] = (int)shape[i]; + } + } + catch (OnnxRuntimeException e) + { + //TODO: cleanup any partially created state + //Do not call ReleaseTensor here. If the constructor has thrown exception, then this NativeOnnxTensorWrapper is not created, so caller should take appropriate action to dispose + throw e; + } + } + ~NativeOnnxTensorMemory() + { + Dispose(false); + } + + public bool IsDisposed => _disposed; + + protected bool IsRetained => _referenceCount > 0; + + public int[] Dimensions + { + get + { + return _dimensions; + } + } + + public int Rank + { + get + { + return _dimensions.Length; + } + } + + public override Span GetSpan() + { + if (IsDisposed) + throw new ObjectDisposedException(nameof(NativeOnnxTensorMemory)); + Span span = null; + unsafe + { + span = new Span((void*)_dataBufferHandle, _elementCount); + } + + return span; + } + + + public override MemoryHandle Pin(int elementIndex = 0) + { + //Note: always pin the full buffer and return + unsafe + { + if (elementIndex >= _elementCount) + { + throw new ArgumentOutOfRangeException(nameof(elementIndex)); + } + Retain(); + + return new MemoryHandle((void*)((int)_dataBufferHandle + elementIndex*_elementWidth)); //could not use Unsafe.Add + } + } + + + public override void Unpin() + { + Release(); + } + + + private bool Release() + { + int newRefCount = Interlocked.Decrement(ref _referenceCount); + + if (newRefCount < 0) + { + throw new InvalidOperationException("Unmatched Release/Retain"); + } + + return newRefCount != 0; + } + + private void Retain() + { + if (IsDisposed) + { + throw new ObjectDisposedException(nameof(NativeOnnxTensorMemory)); + } + + Interlocked.Increment(ref _referenceCount); + } + + protected override void Dispose(bool disposing) + { + if (_disposed) + { + return; + } + + if (disposing) + { + // do managed objects cleanup + } + + // TODO Call nativeMethods.ReleaseTensor, once the corresponding native API is fixed + // Currently there will be memory leak + + + _disposed = true; + } + + protected override bool TryGetArray(out ArraySegment arraySegment) + { + // cannot expose managed array + arraySegment = default(ArraySegment); + return false; + } + + + } +} diff --git a/csharp/OnnxRuntime/OnnxRuntime.cs b/csharp/OnnxRuntime/OnnxRuntime.cs new file mode 100644 index 0000000000000..eb9a1f005fbe0 --- /dev/null +++ b/csharp/OnnxRuntime/OnnxRuntime.cs @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Runtime.CompilerServices; +using System.Collections.Generic; + + +namespace Microsoft.ML.OnnxRuntime +{ + internal struct GlobalOptions //Options are currently not accessible to user + { + public string LogId { get; set; } + public LogLevel LogLevel { get; set; } + } + + internal enum LogLevel + { + Verbose = 0, + Info = 1, + Warning = 2, + Error = 3, + Fatal = 4 + } + + /// + /// This class intializes the process-global ONNX runtime + /// C# API users do not need to access this, thus kept as internal + /// + internal sealed class OnnxRuntime : IDisposable + { + // static singleton + private static readonly Lazy _instance = new Lazy(() => new OnnxRuntime()); + + // member variables + private IntPtr _nativeHandle; + + internal static OnnxRuntime Instance // May throw exception in every access, if the constructor have thrown an exception + { + get + { + return _instance.Value; + } + } + + + private OnnxRuntime() //Problem: it is not possible to pass any option for a Singleton + { + _nativeHandle = IntPtr.Zero; + + IntPtr outPtr; + IntPtr status = NativeMethods.ONNXRuntimeInitialize(LogLevel.Warning, @"CSharpOnnxRuntime", out outPtr); + + NativeApiStatus.VerifySuccess(status); + _nativeHandle = outPtr; + } + + + internal IntPtr NativeHandle + { + get + { + return _nativeHandle; + } + } + + + ~OnnxRuntime() + { + Dispose(false); + } + + public void Dispose() + { + GC.SuppressFinalize(this); + Dispose(true); + } + + private void Dispose(bool disposing) + { + if (disposing) + { + //release managed resource + } + + //release unmanaged resource + if (_nativeHandle != IntPtr.Zero) + { + NativeMethods.ReleaseONNXEnv(_nativeHandle); + } + } + } +} diff --git a/csharp/OnnxRuntime/OnnxRuntime.snk b/csharp/OnnxRuntime/OnnxRuntime.snk new file mode 100644 index 0000000000000..6ebb3cffb0679 Binary files /dev/null and b/csharp/OnnxRuntime/OnnxRuntime.snk differ diff --git a/csharp/OnnxRuntime/SessionOptions.cs b/csharp/OnnxRuntime/SessionOptions.cs new file mode 100644 index 0000000000000..16daea4cf1b54 --- /dev/null +++ b/csharp/OnnxRuntime/SessionOptions.cs @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.ML.OnnxRuntime +{ + public class SessionOptions : IDisposable + { + private static SessionOptions _defaultOptions = new SessionOptions(); + private IntPtr _nativeHandle; + + public SessionOptions() + { + _nativeHandle = NativeMethods.ONNXRuntimeCreateSessionOptions(); + } + + internal IntPtr NativeHandle + { + get + { + return _nativeHandle; + } + } + + public static SessionOptions Default + { + get + { + return _defaultOptions; + } + } + + #region destructors disposers + ~SessionOptions() + { + Dispose(false); + } + + public void Dispose() + { + GC.SuppressFinalize(this); + Dispose(true); + } + + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + // cleanup managed resources + } + + // cleanup unmanaged resources + } + #endregion + } +} diff --git a/csharp/sample/Microsoft.ML.OnnxRuntime.InferenceSample/Microsoft.ML.OnnxRuntime.InferenceSample.csproj b/csharp/sample/Microsoft.ML.OnnxRuntime.InferenceSample/Microsoft.ML.OnnxRuntime.InferenceSample.csproj new file mode 100644 index 0000000000000..ba07300312e54 --- /dev/null +++ b/csharp/sample/Microsoft.ML.OnnxRuntime.InferenceSample/Microsoft.ML.OnnxRuntime.InferenceSample.csproj @@ -0,0 +1,36 @@ + + + + Exe + netcoreapp2.0 + ..\.. + $(OnnxRuntimeCsharpRoot)\..\build\Windows + $(buildDirectory)\$(Configuration)\$(Configuration) + + + + + Always + false + + + Always + false + + + Always + false + + + Always + false + + + + + + + + + + diff --git a/csharp/sample/Microsoft.ML.OnnxRuntime.InferenceSample/Program.cs b/csharp/sample/Microsoft.ML.OnnxRuntime.InferenceSample/Program.cs new file mode 100644 index 0000000000000..b5e89074a66a7 --- /dev/null +++ b/csharp/sample/Microsoft.ML.OnnxRuntime.InferenceSample/Program.cs @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Text; +using System.IO; +using Microsoft.ML.OnnxRuntime; +using System.Numerics.Tensors; + +namespace CSharpUsage +{ + class Program + { + public static void Main(string[] args) + { + Console.WriteLine("Using API"); + UseApi(); + Console.WriteLine("Done"); + } + + + static void UseApi() + { + string modelPath = Directory.GetCurrentDirectory() + @"\squeezenet.onnx"; + + + using (var session = new InferenceSession(modelPath)) + { + var inputMeta = session.InputMetadata; + + // User should be able to detect input name/type/shape from the metadata. + // Currently InputMetadata implementation is inclomplete, so assuming Tensor of predefined dimension. + + var shape0 = new int[] { 1, 3, 224, 224 }; + float[] inputData0 = LoadInputsFloat(); + var tensor = new DenseTensor(inputData0, shape0); + + var container = new List(); + container.Add(new NamedOnnxValue("data_0", tensor)); + + // Run the inference + var results = session.Run(container); // results is an IReadOnlyList container + + // dump the results + foreach (var r in results) + { + Console.WriteLine("Output for {0}", r.Name); + Console.WriteLine(r.AsTensor().GetArrayString()); + } + + // Just try some GC collect + results = null; + container = null; + + GC.Collect(); + GC.WaitForPendingFinalizers(); + } + } + + static int[] LoadInputsInt32() + { + return null; + } + + static float[] LoadInputsFloat() + { + // input: data_0 = float32[1,3,224,224] for squeezenet model + // output: softmaxout_1 = float32[1,1000,1,1] + uint size = 1 * 3 * 224 * 224; + float[] tensor = new float[size]; + + // read data from file + using (var inputFile = new System.IO.StreamReader(@"bench.in")) + { + inputFile.ReadLine(); //skip the input name + string[] dataStr = inputFile.ReadLine().Split(new char[] { ',', '[', ']' }, StringSplitOptions.RemoveEmptyEntries); + for (int i = 0; i < dataStr.Length; i++) + { + tensor[i] = Single.Parse(dataStr[i]); + } + } + + return tensor; + } + + } +} diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Exceptions.cs b/csharp/src/Microsoft.ML.OnnxRuntime/Exceptions.cs new file mode 100644 index 0000000000000..411bece218165 --- /dev/null +++ b/csharp/src/Microsoft.ML.OnnxRuntime/Exceptions.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.ML.OnnxRuntime +{ + + /// + /// Enum conresponding to native onnxruntime error codes. Must be in sync with the native API + /// + internal enum ErrorCode + { + Ok = 0, + Fail = 1, + InvalidArgument = 2, + NoSuchFile = 3, + NoModel = 4, + EngineError = 5, + RuntimeException = 6, + InvalidProtobuf = 7, + ModelLoaded = 8, + NotImplemented = 9, + InvalidGraph = 10, + ShapeInferenceNotRegistered = 11, + RequirementNotRegistered = 12 + } + + public class OnnxRuntimeException: Exception + { + public OnnxRuntimeException(string message) + :base(message) + { + } + } + + + public class CoreRuntimeException : OnnxRuntimeException + { + private static Dictionary errorCodeToString = new Dictionary() + { + { ErrorCode.Ok, "Ok" }, + { ErrorCode.Fail, "Fail" }, + { ErrorCode.InvalidArgument, "InvalidArgument"} , + { ErrorCode.NoSuchFile, "NoSuchFile" }, + { ErrorCode.NoModel, "NoModel" }, + { ErrorCode.EngineError, "EngineError" }, + { ErrorCode.RuntimeException, "RuntimeException" }, + { ErrorCode.InvalidProtobuf, "InvalidProtobuf" }, + { ErrorCode.ModelLoaded, "ModelLoaded" }, + { ErrorCode.NotImplemented, "NotImplemented" }, + { ErrorCode.InvalidGraph, "InvalidGraph" }, + { ErrorCode.ShapeInferenceNotRegistered, "ShapeInferenceNotRegistered" }, + { ErrorCode.RequirementNotRegistered, "RequirementNotRegistered" } + }; + + internal CoreRuntimeException(ErrorCode errorCode, string message) + :base("[ErrorCode:" + errorCodeToString[errorCode] + "] " + message) + { + } + } + + +} diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs b/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs new file mode 100644 index 0000000000000..ae4e5ad309948 --- /dev/null +++ b/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs @@ -0,0 +1,297 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Runtime.InteropServices; +using System.Collections.Generic; +using System.IO; +using System.Linq; + + +namespace Microsoft.ML.OnnxRuntime +{ + + public struct RunOptions + { + // placeholder for RunOptions + } + + /// + /// Represents an Inference Session against an ONNX Model + /// + public class InferenceSession: IDisposable + { + protected IntPtr _nativeHandle; + protected Dictionary _inputMetadata, _outputMetadata; + + + internal InferenceSession(IntPtr nativeHandle) + { + _nativeHandle = nativeHandle; + } + + #region Public API + public InferenceSession(string modelPath) + : this(modelPath, SessionOptions.Default) + { + } + + public InferenceSession(string modelPath, SessionOptions options) + { + var envHandle = OnnxRuntime.Handle; + + _nativeHandle = IntPtr.Zero; + try + { + NativeApiStatus.VerifySuccess(NativeMethods.ONNXRuntimeCreateInferenceSession(envHandle, modelPath, options.NativeHandle, out _nativeHandle)); + + // Initialize input/output metadata + _inputMetadata = new Dictionary(); + _outputMetadata = new Dictionary(); + + // get input count + ulong inputCount = 0; + NativeApiStatus.VerifySuccess(NativeMethods.ONNXRuntimeInferenceSessionGetInputCount(_nativeHandle, out inputCount)); + + // get all the output names + for (ulong i = 0; i < inputCount; i++) + { + _inputMetadata[GetInputName(i)] = new NodeMetadata(); //TODO: fill the shape/type when C-api available + } + + // get output count + ulong outputCount = 0; + NativeApiStatus.VerifySuccess(NativeMethods.ONNXRuntimeInferenceSessionGetOutputCount(_nativeHandle, out outputCount)); + + // get all the output names + for (ulong i = 0; i < outputCount; i++) + { + _outputMetadata[GetOutputName(i)] = new NodeMetadata(); //TODO: fill the shape/type when C-api available + } + } + catch (OnnxRuntimeException e) + { + if (_nativeHandle != IntPtr.Zero) + { + NativeMethods.ReleaseONNXSession(_nativeHandle); + _nativeHandle = IntPtr.Zero; + } + throw e; + } + } + + public IReadOnlyDictionary InputMetadata + { + get + { + return _inputMetadata; + } + } + + public IReadOnlyDictionary OutputMetadata + { + get + { + return _outputMetadata; + } + } + + public ModelMetadata ModelMetadata + { + get + { + return new ModelMetadata(); //TODO: implement + } + } + + public IReadOnlyCollection Run(IReadOnlyCollection inputs, RunOptions options = new RunOptions()) + { + string[] outputNames = new string[_outputMetadata.Count]; + _outputMetadata.Keys.CopyTo(outputNames, 0); + return Run(inputs, outputNames, options); + } + + /// + /// Runs the loaded model for the given inputs, and fetches the specified outputs in . + /// + /// + /// + /// + /// Output Tensors in a Dictionary + public IReadOnlyCollection Run(IReadOnlyCollection inputs, IReadOnlyCollection outputNames, RunOptions options = new RunOptions()) + { + var inputNames = new string[inputs.Count]; + var inputTensors = new IntPtr[inputs.Count]; + var pinnedBufferHandles = new System.Buffers.MemoryHandle[inputs.Count]; + + int offset = 0; + foreach (var input in inputs) + { + inputNames[offset] = input.Name; + + // create Tensor fromt the input if feasible, else throw notsupported exception for now + input.ToNativeOnnxValue(out inputTensors[offset], out pinnedBufferHandles[offset]); + + offset++; + } + + string[] outputNamesArray = outputNames.ToArray(); + IntPtr[] outputValueArray = new IntPtr[outputNames.Count]; + + IntPtr status = NativeMethods.ONNXRuntimeRunInference( + this._nativeHandle, + inputNames, + inputTensors, + (ulong)(inputTensors.Length), /* TODO: size_t, make it portable for x86 arm */ + outputNamesArray, + (ulong)outputNames.Count, /* TODO: size_t, make it portable for x86 and arm */ + outputValueArray /* An array of output value pointers. Array must be allocated by the caller */ + ); + + try + { + NativeApiStatus.VerifySuccess(status); + var result = new List(); + for (uint i = 0; i < outputValueArray.Length; i++) + { + result.Add(NamedOnnxValue.CreateFromOnnxValue(outputNamesArray[i], outputValueArray[i])); + } + + return result; + } + catch (OnnxRuntimeException e) + { + //clean up the individual output tensors if it is not null; + for (uint i = 0; i < outputValueArray.Length; i++) + { + if (outputValueArray[i] != IntPtr.Zero) + { + NativeMethods.ReleaseONNXValue(outputValueArray[i]); + } + } + throw e; + } + finally + { + // always unpin the input buffers, and delete the native Onnx value objects + for (int i = 0; i < inputs.Count; i++) + { + NativeMethods.ReleaseONNXValue(inputTensors[i]); // this should not release the buffer, but should delete the native tensor object + pinnedBufferHandles[i].Dispose(); + } + } + + } + + + #endregion + + #region private methods + private string GetOutputName(ulong index) + { + IntPtr nameHandle = IntPtr.Zero; + string str = null; + + IntPtr status = NativeMethods.ONNXRuntimeInferenceSessionGetOutputName( + _nativeHandle, + index, + NativeMemoryAllocator.DefaultInstance.Handle, + out nameHandle); + try + { + NativeApiStatus.VerifySuccess(status); + str = Marshal.PtrToStringAnsi(nameHandle); //assumes charset = ANSI + } + finally + { + if (nameHandle != IntPtr.Zero) + { + NativeMemoryAllocator.DefaultInstance.FreeMemory(nameHandle); + } + } + + return str; + } + + private string GetInputName(ulong index) + { + IntPtr nameHandle = IntPtr.Zero; + string str = null; + + IntPtr status = NativeMethods.ONNXRuntimeInferenceSessionGetInputName( + _nativeHandle, + index, + NativeMemoryAllocator.DefaultInstance.Handle, + out nameHandle); + try + { + NativeApiStatus.VerifySuccess(status); + str = Marshal.PtrToStringAnsi(nameHandle); //assumes charset = ANSI + } + finally + { + if (nameHandle != IntPtr.Zero) + { + NativeMemoryAllocator.DefaultInstance.FreeMemory(nameHandle); + } + } + + return str; + } + + #endregion + + + + #region destructors disposers + + + ~InferenceSession() + { + Dispose(false); + } + + public void Dispose() + { + GC.SuppressFinalize(this); + Dispose(true); + } + + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + // cleanup managed resources + } + + // cleanup unmanaged resources + if (_nativeHandle != IntPtr.Zero) + { + NativeMethods.ReleaseONNXSession(_nativeHandle); + } + } + + #endregion + + } + + public struct NodeMetadata + { + //TODO: currently shape and type is not available in C-api, so this struct may change based on implementation + public uint[] Shape + { + get; internal set; + } + public System.Type Type + { + get; internal set; + } + } + + + public struct ModelMetadata + { + //TODO: placeholder for Model metadata. Currently C-API does not expose this + } + +} diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj b/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj new file mode 100644 index 0000000000000..cbdaff3ce4a50 --- /dev/null +++ b/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj @@ -0,0 +1,15 @@ + + + + netstandard1.1 + true + true + false + OnnxRuntime.snk + + + + + + + diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NamedOnnxValue.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NamedOnnxValue.cs new file mode 100644 index 0000000000000..829b63cdbcdec --- /dev/null +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NamedOnnxValue.cs @@ -0,0 +1,323 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Numerics.Tensors; +using System.Buffers; +using System.Collections; +using System.Diagnostics; + +namespace Microsoft.ML.OnnxRuntime +{ + public class NamedOnnxValue + { + protected Object _value; + protected string _name; + + public NamedOnnxValue(string name, Object value) + { + _name = name; + _value = value; + } + + public string Name { get { return _name; } } + public Tensor AsTensor() + { + return _value as Tensor; // will return null if not castable + } + + /// + /// Attempts to Pin the buffer, and create a native OnnxValue out of it. the pinned MemoryHandle is passed to output. + /// In this case, the pinnedHandle should be kept alive till the native OnnxValue is used, then dispose it. + /// If it is not possible to Pin the buffer, then creates OnnxValue from the copy of the data. The output pinnedMemoryHandle + /// contains a default value in that case. + /// Attempts to infer the type of the value while creating the OnnxValue + /// + /// + /// + internal void ToNativeOnnxValue(out IntPtr onnxValue, out MemoryHandle pinnedMemoryHandle) + { + //try to cast _value to Tensor + TensorElementType nativeElementType = TensorElementType.DataTypeMax; //invalid + IntPtr dataBufferPointer = IntPtr.Zero; + int dataBufferLength = 0; + ReadOnlySpan shape = null; + int rank = 0; + onnxValue = IntPtr.Zero; + + if (TryPinAsTensor(out pinnedMemoryHandle, + out dataBufferPointer, + out dataBufferLength, + out shape, + out rank, + out nativeElementType + )) + { + } + else if (TryPinAsTensor(out pinnedMemoryHandle, + out dataBufferPointer, + out dataBufferLength, + out shape, + out rank, + out nativeElementType + )) + { + } + else if (TryPinAsTensor(out pinnedMemoryHandle, + out dataBufferPointer, + out dataBufferLength, + out shape, + out rank, + out nativeElementType + )) + { + } + else if (TryPinAsTensor(out pinnedMemoryHandle, + out dataBufferPointer, + out dataBufferLength, + out shape, + out rank, + out nativeElementType + )) + { + } + else if (TryPinAsTensor(out pinnedMemoryHandle, + out dataBufferPointer, + out dataBufferLength, + out shape, + out rank, + out nativeElementType + )) + { + } + else if (TryPinAsTensor(out pinnedMemoryHandle, + out dataBufferPointer, + out dataBufferLength, + out shape, + out rank, + out nativeElementType + )) + { + } + else if (TryPinAsTensor(out pinnedMemoryHandle, + out dataBufferPointer, + out dataBufferLength, + out shape, + out rank, + out nativeElementType + )) + { + } + else if (TryPinAsTensor(out pinnedMemoryHandle, + out dataBufferPointer, + out dataBufferLength, + out shape, + out rank, + out nativeElementType + )) + { + } + else if (TryPinAsTensor(out pinnedMemoryHandle, + out dataBufferPointer, + out dataBufferLength, + out shape, + out rank, + out nativeElementType + )) + { + } + else if (TryPinAsTensor(out pinnedMemoryHandle, + out dataBufferPointer, + out dataBufferLength, + out shape, + out rank, + out nativeElementType + )) + { + } + + //TODO: add other types + else + { + // nothing to cleanup here, since no memory has been pinned + throw new NotSupportedException("The inference value " + nameof(_value) + " is not of a supported type"); + } + + + Debug.Assert(dataBufferPointer != IntPtr.Zero, "dataBufferPointer must be non-null after obtaining the pinned buffer"); + + // copy to an ulong[] shape to match size_t[] + ulong[] longShape = new ulong[rank]; + for (int i = 0; i < rank; i++) + { + longShape[i] = (ulong)shape[i]; + } + + IntPtr status = NativeMethods.ONNXRuntimeCreateTensorWithDataAsONNXValue( + NativeMemoryAllocatorInfo.DefaultInstance.Handle, + dataBufferPointer, + (ulong)(dataBufferLength), + longShape, + (ulong)rank, + nativeElementType, + out onnxValue + ); + try + { + NativeApiStatus.VerifySuccess(status); + } + catch (OnnxRuntimeException e) + { + pinnedMemoryHandle.Dispose(); + throw e; + } + + } + + internal static NamedOnnxValue CreateFromOnnxValue(string name, IntPtr nativeOnnxValue) + { + NamedOnnxValue result = null; + + if (true /* TODO: check native data type when API available. assuming Tensor for now */) + { + NativeOnnxTensorMemory nativeTensorWrapper = new NativeOnnxTensorMemory(nativeOnnxValue); + DenseTensor dt = new DenseTensor(nativeTensorWrapper.Memory, nativeTensorWrapper.Dimensions); + result = new NamedOnnxValue(name, dt); + } + + return result; + } + + private bool TryPinAsTensor( + out MemoryHandle pinnedMemoryHandle, + out IntPtr dataBufferPointer, + out int dataBufferLength, + out ReadOnlySpan shape, + out int rank, + out TensorElementType nativeElementType + ) + { + nativeElementType = TensorElementType.DataTypeMax; //invalid + dataBufferPointer = IntPtr.Zero; + dataBufferLength = 0; + shape = null; + rank = 0; + pinnedMemoryHandle = default(MemoryHandle); + + if (_value is Tensor) + { + Tensor t = _value as Tensor; + if (t.IsReversedStride) + { + //TODO: not sure how to support reverse stride. may be able to calculate the shape differently + throw new NotSupportedException(nameof(Tensor) + " of reverseStride is not supported"); + } + + DenseTensor dt = null; + if (_value is DenseTensor) + { + dt = _value as DenseTensor; + } + else + { + dt = t.ToDenseTensor(); + } + + shape = dt.Dimensions; // does not work for reverse stride + rank = dt.Rank; + pinnedMemoryHandle = dt.Buffer.Pin(); + unsafe + { + dataBufferPointer = (IntPtr)pinnedMemoryHandle.Pointer; + } + + // find the native type + if (typeof(T) == typeof(float)) + { + nativeElementType = TensorElementType.Float; + dataBufferLength = dt.Buffer.Length * sizeof(float); + } + else if (typeof(T) == typeof(double)) + { + nativeElementType = TensorElementType.Double; + dataBufferLength = dt.Buffer.Length * sizeof(double); + } + else if (typeof(T) == typeof(int)) + { + nativeElementType = TensorElementType.Int32; + dataBufferLength = dt.Buffer.Length * sizeof(int); + } + else if (typeof(T) == typeof(uint)) + { + nativeElementType = TensorElementType.UInt32; + dataBufferLength = dt.Buffer.Length * sizeof(uint); + } + else if (typeof(T) == typeof(long)) + { + nativeElementType = TensorElementType.Int64; + dataBufferLength = dt.Buffer.Length * sizeof(long); + } + else if (typeof(T) == typeof(ulong)) + { + nativeElementType = TensorElementType.UInt64; + dataBufferLength = dt.Buffer.Length * sizeof(ulong); + } + else if (typeof(T) == typeof(short)) + { + nativeElementType = TensorElementType.Int16; + dataBufferLength = dt.Buffer.Length * sizeof(short); + } + else if (typeof(T) == typeof(ushort)) + { + nativeElementType = TensorElementType.UInt16; + dataBufferLength = dt.Buffer.Length * sizeof(ushort); + } + else if (typeof(T) == typeof(byte)) + { + nativeElementType = TensorElementType.UInt8; + dataBufferLength = dt.Buffer.Length * sizeof(byte); + } + //TODO: Not supporting boolean for now. bool is non-blittable, the interop needs some care, and possibly need to copy + //else if (typeof(T) == typeof(bool)) + //{ + //} + else + { + //TODO: may extend the supported types + // do not throw exception, rather assign the sentinel value + nativeElementType = TensorElementType.DataTypeMax; + } + return true; + } + + return false; + } + + // may expose different types of getters in future + + } + + internal enum TensorElementType + { + Float = 1, + UInt8 = 2, + Int8 = 3, + UInt16 = 4, + Int16 = 5, + Int32 = 6, + Int64 = 7, + String = 8, + Bool = 9, + Float16 = 10, + Double = 11, + UInt32 = 12, + UInt64 = 13, + Complex64 = 14, + Complex128 = 15, + BFloat16 = 16, + DataTypeMax = 17 + } + +} diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeApiStatus.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeApiStatus.cs new file mode 100644 index 0000000000000..bc4626d54987d --- /dev/null +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeApiStatus.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Runtime.InteropServices; + +namespace Microsoft.ML.OnnxRuntime +{ + class NativeApiStatus + { + private static string GetErrorMessage(IntPtr /*(ONNXStatus*)*/status) + { + IntPtr nativeString = NativeMethods.ONNXRuntimeGetErrorMessage(status); + string str = Marshal.PtrToStringAnsi(nativeString); //assumes charset = ANSI + return str; + } + + /// + /// Checks the native Status if the errocode is OK/Success. Otherwise constructs an appropriate exception and throws. + /// Releases the native status object, as needed. + /// + /// + /// + public static void VerifySuccess(IntPtr nativeStatus) + { + if (nativeStatus != IntPtr.Zero) + { + ErrorCode statusCode = NativeMethods.ONNXRuntimeGetErrorCode(nativeStatus); + string errorMessage = GetErrorMessage(nativeStatus); + NativeMethods.ReleaseONNXStatus(nativeStatus); + throw new CoreRuntimeException(statusCode, errorMessage); + } + } + } +} diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMemoryAllocator.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMemoryAllocator.cs new file mode 100644 index 0000000000000..d0e836c981256 --- /dev/null +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMemoryAllocator.cs @@ -0,0 +1,154 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Runtime.InteropServices; + +namespace Microsoft.ML.OnnxRuntime +{ + internal class NativeMemoryAllocatorInfo : SafeHandle + { + protected static readonly Lazy _defaultCpuAllocInfo = new Lazy(CreateCpuAllocatorInfo); + + private static NativeMemoryAllocatorInfo CreateCpuAllocatorInfo() + { + IntPtr allocInfo = IntPtr.Zero; + try + { + IntPtr status = NativeMethods.ONNXRuntimeCreateCpuAllocatorInfo(NativeMethods.AllocatorType.DeviceAllocator, NativeMethods.MemoryType.Cpu, out allocInfo); + NativeApiStatus.VerifySuccess(status); + } + catch (Exception e) + { + if (allocInfo != IntPtr.Zero) + { + Delete(allocInfo); + } + throw e; + } + return new NativeMemoryAllocatorInfo(allocInfo); + } + + internal static NativeMemoryAllocatorInfo DefaultInstance + { + get + { + return _defaultCpuAllocInfo.Value; + } + } + + internal IntPtr Handle // May throw exception in every access, if the constructor have thrown an exception + { + get + { + return handle; + } + } + + public override bool IsInvalid + { + get + { + return (handle == IntPtr.Zero); + } + } + + private NativeMemoryAllocatorInfo(IntPtr allocInfo) + : base(IntPtr.Zero, true) //set 0 as invalid pointer + { + handle = allocInfo; + } + + + private static void Delete(IntPtr nativePtr) + { + NativeMethods.ReleaseONNXRuntimeAllocatorInfo(nativePtr); + } + + protected override bool ReleaseHandle() + { + Delete(handle); + return true; + } + } + + + internal class NativeMemoryAllocator : SafeHandle + { + protected static readonly Lazy _defaultInstance = new Lazy(CreateDefaultCpuAllocator); + + private static NativeMemoryAllocator CreateDefaultCpuAllocator() + { + IntPtr allocator = IntPtr.Zero; + try + { + IntPtr status = NativeMethods.ONNXRuntimeCreateDefaultAllocator(out allocator); + NativeApiStatus.VerifySuccess(status); + } + catch (Exception e) + { + if (allocator != IntPtr.Zero) + { + Delete(allocator); + } + throw e; + } + + return new NativeMemoryAllocator(allocator); + } + + static internal NativeMemoryAllocator DefaultInstance // May throw exception in every access, if the constructor have thrown an exception + { + get + { + return _defaultInstance.Value; + } + } + + /// + /// Releases native memory previously allocated by the allocator + /// + /// + internal void FreeMemory(IntPtr memory) + { + NativeMethods.ONNXRuntimeAllocatorFree(handle, memory); + } + + public override bool IsInvalid + { + get + { + return (this.handle == IntPtr.Zero); + } + } + + internal IntPtr Handle + { + get + { + return handle; + } + } + + protected NativeMemoryAllocator(IntPtr allocator) + : base(IntPtr.Zero, true) + { + this.handle = allocator; + } + + + protected static void Delete(IntPtr allocator) + { + NativeMethods.ONNXRuntimeReleaseObject(allocator); + } + + protected override bool ReleaseHandle() + { + Delete(this.handle); + return true; + } + } + +} diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs new file mode 100644 index 0000000000000..2eeac4da6c0ed --- /dev/null +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs @@ -0,0 +1,288 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Runtime.InteropServices; + +namespace Microsoft.ML.OnnxRuntime +{ + /// + /// NamedOnnxValue type, must match the native enum + /// + + internal static class NativeMethods + { + private const string nativeLib = "onnxruntime.dll"; + internal const CharSet charSet = CharSet.Ansi; + + #region Runtime/Environment API + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /* ONNXStatus* */ONNXRuntimeInitialize( + LogLevel default_warning_level, + string logId, + out IntPtr /*(ONNXEnv*)*/ env); + // ReleaseONNXEnv should not be used + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ReleaseONNXEnv(IntPtr /*(ONNXEnv*)*/ env); + #endregion Runtime/Environment API + + #region Status API + [DllImport(nativeLib, CharSet = charSet)] + public static extern ErrorCode ONNXRuntimeGetErrorCode(IntPtr /*(ONNXStatus*)*/status); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /* char* */ONNXRuntimeGetErrorMessage(IntPtr /* (ONNXStatus*) */status); + // returns char*, need to convert to string by the caller. + // does not free the underlying ONNXStatus* + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ReleaseONNXStatus(IntPtr /*(ONNXStatus*)*/ statusPtr); + + #endregion Status API + + #region InferenceSession API + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /* ONNXStatus* */ONNXRuntimeCreateInferenceSession( + IntPtr /* (ONNXEnv*) */ environment, + [MarshalAs(UnmanagedType.LPWStr)]string modelPath, //the model path is consumed as a wchar* in the C-api + IntPtr /* (ONNXRuntimeSessionOptions*) */sessopnOptions, + out IntPtr /**/ session); + + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(ONNStatus*)*/ ONNXRuntimeRunInferenceAndFetchAll( + IntPtr /*(ONNXSessionPtr)*/ session, + string[] inputNames, + IntPtr[] /*(ONNXValuePtr[])*/ inputValues, + ulong inputLength, // size_t, TODO: make it portable for x86, arm + out IntPtr /* (ONNXValueListPtr*)*/ outputValues, + out ulong /* (size_t*) */ outputLength); //TODO: make it portable for x86, arm + + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(ONNStatus*)*/ ONNXRuntimeRunInference( + IntPtr /*(ONNXSession*)*/ session, + string[] inputNames, + IntPtr[] /* (ONNXValue*[])*/ inputValues, + ulong inputCount, /* TODO: size_t, make it portable for x86 arm */ + string[] outputNames, + ulong outputCount, /* TODO: size_t, make it portable for x86 and arm */ + + [MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 5 /*index of outputCount*/)][In, Out] + IntPtr[] outputValues /* An array of output value pointers. Array must be allocated by the caller */ + ); + + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeInferenceSessionGetInputCount( + IntPtr /*(ONNXSession*)*/ session, + out ulong /* TODO: size_t */ count); + + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeInferenceSessionGetOutputCount( + IntPtr /*(ONNXSession*)*/ session, + out ulong /*TODO: size_t port*/ count); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(ONNXStatus*)*/ONNXRuntimeInferenceSessionGetInputName( + IntPtr /*(ONNXSession*)*/ session, + ulong index, //TODO: port size_t + IntPtr /*(ONNXRuntimeAllocator*)*/ allocator, + out IntPtr /*(char**)*/name); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(ONNXStatus*)*/ONNXRuntimeInferenceSessionGetOutputName( + IntPtr /*(ONNXSession*)*/ session, + ulong index, //TODO: port size_t + IntPtr /*(ONNXRuntimeAllocator*)*/ allocator, + out IntPtr /*(char**)*/name); + + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ReleaseONNXSession(IntPtr /*(ONNXSession*)*/session); + + #endregion InferenceSession API + + #region SessionOptions API + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*ONNXRuntimeSessionOptions* */ ONNXRuntimeCreateSessionOptions(); + + //DEFINE_RUNTIME_CLASS(ONNXRuntimeSessionOptions) + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ReleaseONNXRuntimeSessionOptions(IntPtr /*(ONNXRuntimeSessionOptions*)*/ sessionOptions); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeEnableSequentialExecution(IntPtr /* ONNXRuntimeSessionOptions* */ options); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeDisableSequentialExecution(IntPtr /* ONNXRuntimeSessionOptions* */ options); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeEnableProfiling(IntPtr /* ONNXRuntimeSessionOptions* */ options, string profilePathPrefix); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeDisableProfiling(IntPtr /* ONNXRuntimeSessionOptions* */ options); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeEnableMemPattern(IntPtr /* ONNXRuntimeSessionOptions* */ options); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeDisableMemPattern(IntPtr /* ONNXRuntimeSessionOptions* */ options); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeEnableCpuMemArena(IntPtr /* ONNXRuntimeSessionOptions* */ options); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeDisableCpuMemArena(IntPtr /* ONNXRuntimeSessionOptions* */ options); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeSetSessionLogId(IntPtr /* ONNXRuntimeSessionOptions* */ options, string logId); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeSetSessionLogVerbosityLevel(IntPtr /* ONNXRuntimeSessionOptions* */ options, LogLevel sessionLogVerbosityLevel); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern int ONNXRuntimeSetSessionThreadPoolSize(IntPtr /* ONNXRuntimeSessionOptions* */ options, int sessionThreadPoolSize); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern int ONNXRuntimeEnableCudaProvider(IntPtr /* ONNXRuntimeSessionOptions* */ options, int deviceId); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeDisableCudaProvider(IntPtr /* ONNXRuntimeSessionOptions* */ options); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern int ONNXRuntimeEnableMklProvider(IntPtr /* ONNXRuntimeSessionOptions* */ options); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeDisableMklProvider(IntPtr /* ONNXRuntimeSessionOptions* */ options); + + #endregion + + #region Allocator/AllocatorInfo API + + //TODO: consider exposing them publicly, when allocator API is exposed + public enum AllocatorType + { + DeviceAllocator = 0, + ArenaAllocator = 1 + } + + //TODO: consider exposing them publicly when allocator API is exposed + public enum MemoryType + { + CpuInput = -2, // Any CPU memory used by non-CPU execution provider + CpuOutput = -1, // CPU accessible memory outputted by non-CPU execution provider, i.e. CUDA_PINNED + Cpu = CpuOutput, // temporary CPU accessible memory allocated by non-CPU execution provider, i.e. CUDA_PINNED + Default = 0, // the default allocator for execution provider + } + + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /* (ONNXStatus*)*/ ONNXRuntimeCreateAllocatorInfo( + IntPtr /*(const char*) */name, + AllocatorType allocatorType, + int identifier, + MemoryType memType, + out IntPtr /*(ONNXRuntimeAllocatorInfo*)*/ allocatorInfo // memory ownership transfered to caller + ); + + //ONNXRUNTIME_API_STATUS(ONNXRuntimeCreateCpuAllocatorInfo, enum ONNXRuntimeAllocatorType type, enum ONNXRuntimeMemType mem_type1, _Out_ ONNXRuntimeAllocatorInfo** out) + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /* (ONNXStatus*)*/ ONNXRuntimeCreateCpuAllocatorInfo( + AllocatorType allocatorType, + MemoryType memoryType, + out IntPtr /*(ONNXRuntimeAllocatorInfo*)*/ allocatorInfo + ); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ReleaseONNXRuntimeAllocatorInfo(IntPtr /*(ONNXRuntimeAllocatorInfo*)*/ allocatorInfo); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(ONNXStatus*)*/ONNXRuntimeCreateDefaultAllocator(out IntPtr /*(ONNXRuntimeAllocator**)*/ allocator); + + /// + /// Releases/Unrefs any object, including the Allocator + /// + /// + /// remaining ref count + [DllImport(nativeLib, CharSet = charSet)] + public static extern uint /*remaining ref count*/ ONNXRuntimeReleaseObject(IntPtr /*(void*)*/ ptr); + + /// + /// Release any object allocated by an allocator + /// + /// + /// + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeAllocatorFree(IntPtr allocator, IntPtr memory); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(const struct ONNXRuntimeAllocatorInfo*)*/ ONNXRuntimeAllocatorGetInfo(IntPtr /*(const ONNXRuntimeAllocator*)*/ ptr); + + #endregion Allocator/AllocatorInfo API + + #region Tensor/OnnxValue API + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /* ONNXStatus */ ONNXRuntimeCreateTensorWithDataAsONNXValue( + IntPtr /* (const ONNXRuntimeAllocatorInfo*) */ allocatorInfo, + IntPtr /* (void*) */dataBufferHandle, + ulong dataLength, //size_t, TODO: make it portable for x86, arm + ulong[] shape, //size_t* or size_t[], TODO: make it portable for x86, arm + ulong shapeLength, //size_t, TODO: make it portable for x86, arm + TensorElementType type, + out IntPtr /* ONNXValuePtr* */ outputValue); + + /// This function doesn't work with string tensor + /// this is a no-copy method whose pointer is only valid until the backing ONNXValuePtr is free'd. + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeGetTensorMutableData(IntPtr /*(ONNXValue*)*/ value, out IntPtr /* (void**)*/ dataBufferHandle); + + //[DllImport(nativeLib, CharSet = charSet)] + //public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeGetTensorShapeDimCount(IntPtr /*(ONNXValue*)*/ value, out ulong dimension); //size_t TODO: make it portable for x86, arm + + //[DllImport(nativeLib, CharSet = charSet)] + //public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeGetTensorShapeElementCount(IntPtr /*(ONNXValue*)*/value, out ulong count); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeGetTensorShapeAndType(IntPtr /*(ONNXValue*)*/ value, out IntPtr /*(struct ONNXRuntimeTensorTypeAndShapeInfo*)*/ typeAndShapeInfo); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern TensorElementType ONNXRuntimeGetTensorElementType(IntPtr /*(const struct ONNXRuntimeTensorTypeAndShapeInfo*)*/ typeAndShapeInfo); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern ulong /*TODO: port for size_t */ONNXRuntimeGetNumOfDimensions(IntPtr /*(const struct ONNXRuntimeTensorTypeAndShapeInfo*)*/ typeAndShapeInfo); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeGetDimensions( + IntPtr /*(const struct ONNXRuntimeTensorTypeAndShapeInfo*)*/ typeAndShapeInfo, + long[] dim_values, + ulong dim_values_length); + + /** + * How many elements does this tensor have. + * May return a negative value + * e.g. + * [] -> 1 + * [1,3,4] -> 12 + * [2,0,4] -> 0 + * [-1,3,4] -> -1 + */ + [DllImport(nativeLib, CharSet = charSet)] + public static extern long ONNXRuntimeGetTensorShapeElementCount(IntPtr /*(const struct ONNXRuntimeTensorTypeAndShapeInfo*)*/ typeAndShapeInfo); + + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(ONNXValuePtr)*/ ONNXRuntimeONNXValueListGetNthValue(IntPtr /*(ONNXValueListPtr)*/ list, ulong index); // 0-based index TODO: size_t, make it portable for x86, arm + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ReleaseONNXValue(IntPtr /*(ONNXValue*)*/ value); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ReleaseONNXValueList(IntPtr /*(ONNXValueList*)*/ valueList); + + #endregion + } //class NativeMethods +} //namespace diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxTensorMemory.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxTensorMemory.cs new file mode 100644 index 0000000000000..b5f66e94658b2 --- /dev/null +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxTensorMemory.cs @@ -0,0 +1,233 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; + + +namespace Microsoft.ML.OnnxRuntime +{ + internal class NativeOnnxTensorMemory : MemoryManager + { + private bool _disposed; + private int _referenceCount; + private IntPtr _onnxValueHandle; + private IntPtr _dataBufferHandle; + private int _elementCount; + private int _elementWidth; + private int[] _dimensions; + + public NativeOnnxTensorMemory(IntPtr onnxValueHandle) + { + IntPtr typeAndShape = IntPtr.Zero; + try + { + NativeApiStatus.VerifySuccess(NativeMethods.ONNXRuntimeGetTensorShapeAndType(onnxValueHandle, out typeAndShape)); + + TensorElementType elemType = NativeMethods.ONNXRuntimeGetTensorElementType(typeAndShape); + + Type type = null; + int width = 0; + GetTypeAndWidth(elemType, out type, out width); + if (typeof(T) != type) + throw new NotSupportedException(nameof(NativeOnnxTensorMemory)+" does not support T = "+nameof(T)); + _elementWidth = width; + + _onnxValueHandle = onnxValueHandle; + // derive the databuffer pointer, element_count, element_width, and shape + NativeApiStatus.VerifySuccess(NativeMethods.ONNXRuntimeGetTensorMutableData(_onnxValueHandle, out _dataBufferHandle)); + // throws OnnxRuntimeException if native call failed + + ulong dimension = NativeMethods.ONNXRuntimeGetNumOfDimensions(typeAndShape); + long count = NativeMethods.ONNXRuntimeGetTensorShapeElementCount(typeAndShape); // count can be negative. + if (count < 0) + { + throw new NotSupportedException("Symbolic dimensions in the tensor is not supported"); + } + + long[] shape = new long[dimension]; + NativeMethods.ONNXRuntimeGetDimensions(typeAndShape, shape, dimension); //Note: shape must be alive during the call + + _elementCount = (int)count; + _dimensions = new int[dimension]; + for (ulong i = 0; i < dimension; i++) + { + _dimensions[i] = (int)shape[i]; + } + } + catch (Exception e) + { + //TODO: cleanup any partially created state + //Do not call ReleaseTensor here. If the constructor has thrown exception, then this NativeOnnxTensorWrapper is not created, so caller should take appropriate action to dispose + throw e; + } + finally + { + if (typeAndShape != IntPtr.Zero) + { + NativeMethods.ONNXRuntimeReleaseObject(typeAndShape); + } + } + } + ~NativeOnnxTensorMemory() + { + Dispose(false); + } + + public bool IsDisposed => _disposed; + + protected bool IsRetained => _referenceCount > 0; + + public int[] Dimensions + { + get + { + return _dimensions; + } + } + + public int Rank + { + get + { + return _dimensions.Length; + } + } + + public override Span GetSpan() + { + if (IsDisposed) + throw new ObjectDisposedException(nameof(NativeOnnxTensorMemory)); + Span span = null; + unsafe + { + span = new Span((void*)_dataBufferHandle, _elementCount); + } + + return span; + } + + + public override MemoryHandle Pin(int elementIndex = 0) + { + //Note: always pin the full buffer and return + unsafe + { + if (elementIndex >= _elementCount) + { + throw new ArgumentOutOfRangeException(nameof(elementIndex)); + } + Retain(); + + return new MemoryHandle((void*)((int)_dataBufferHandle + elementIndex*_elementWidth)); //could not use Unsafe.Add + } + } + + + public override void Unpin() + { + Release(); + } + + + private bool Release() + { + int newRefCount = Interlocked.Decrement(ref _referenceCount); + + if (newRefCount < 0) + { + throw new InvalidOperationException("Unmatched Release/Retain"); + } + + return newRefCount != 0; + } + + private void Retain() + { + if (IsDisposed) + { + throw new ObjectDisposedException(nameof(NativeOnnxTensorMemory)); + } + + Interlocked.Increment(ref _referenceCount); + } + + protected override void Dispose(bool disposing) + { + if (_disposed) + { + return; + } + + if (disposing) + { + // do managed objects cleanup + } + + NativeMethods.ReleaseONNXValue(_onnxValueHandle); + + _disposed = true; + } + + protected override bool TryGetArray(out ArraySegment arraySegment) + { + // cannot expose managed array + arraySegment = default(ArraySegment); + return false; + } + + + internal static void GetTypeAndWidth(TensorElementType elemType, out Type type, out int width) + { + switch (elemType) + { + case TensorElementType.Float: + type = typeof(float); + width = sizeof(float); + break; + case TensorElementType.Double: + type = typeof(double); + width = sizeof(double); + break; + case TensorElementType.Int16: + type = typeof(short); + width = sizeof(short); + break; + case TensorElementType.UInt16: + type = typeof(ushort); + width = sizeof(ushort); + break; + case TensorElementType.Int32: + type = typeof(int); + width = sizeof(int); + break; + case TensorElementType.UInt32: + type = typeof(uint); + width = sizeof(uint); + break; + case TensorElementType.Int64: + type = typeof(long); + width = sizeof(long); + break; + case TensorElementType.UInt64: + type = typeof(ulong); + width = sizeof(ulong); + break; + case TensorElementType.UInt8: + type = typeof(byte); + width = sizeof(byte); + break; + default: + type = null; + width = 0; + break; + } + } + + } +} diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/OnnxRuntime.cs b/csharp/src/Microsoft.ML.OnnxRuntime/OnnxRuntime.cs new file mode 100644 index 0000000000000..f2adfdc41f5c9 --- /dev/null +++ b/csharp/src/Microsoft.ML.OnnxRuntime/OnnxRuntime.cs @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Collections.Generic; + + +namespace Microsoft.ML.OnnxRuntime +{ + internal struct GlobalOptions //Options are currently not accessible to user + { + public string LogId { get; set; } + public LogLevel LogLevel { get; set; } + } + + internal enum LogLevel + { + Verbose = 0, + Info = 1, + Warning = 2, + Error = 3, + Fatal = 4 + } + + /// + /// This class intializes the process-global ONNX runtime + /// C# API users do not need to access this, thus kept as internal + /// + internal sealed class OnnxRuntime : SafeHandle + { + private static readonly Lazy _instance = new Lazy(()=> new OnnxRuntime()); + + internal static IntPtr Handle // May throw exception in every access, if the constructor have thrown an exception + { + get + { + return _instance.Value.handle; + } + } + + public override bool IsInvalid + { + get + { + return (handle == IntPtr.Zero); + } + } + + private OnnxRuntime() //Problem: it is not possible to pass any option for a Singleton + :base(IntPtr.Zero, true) + { + handle = IntPtr.Zero; + try + { + NativeApiStatus.VerifySuccess(NativeMethods.ONNXRuntimeInitialize(LogLevel.Warning, @"CSharpOnnxRuntime", out handle)); + } + catch (OnnxRuntimeException e) + { + if (handle != IntPtr.Zero) + { + Delete(handle); + handle = IntPtr.Zero; + } + throw e; + } + + } + + private static void Delete(IntPtr nativePtr) + { + NativeMethods.ReleaseONNXEnv(nativePtr); + } + + protected override bool ReleaseHandle() + { + Delete(handle); + return true; + } + } +} \ No newline at end of file diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/OnnxRuntime.snk b/csharp/src/Microsoft.ML.OnnxRuntime/OnnxRuntime.snk new file mode 100644 index 0000000000000..6ebb3cffb0679 Binary files /dev/null and b/csharp/src/Microsoft.ML.OnnxRuntime/OnnxRuntime.snk differ diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs new file mode 100644 index 0000000000000..16daea4cf1b54 --- /dev/null +++ b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.ML.OnnxRuntime +{ + public class SessionOptions : IDisposable + { + private static SessionOptions _defaultOptions = new SessionOptions(); + private IntPtr _nativeHandle; + + public SessionOptions() + { + _nativeHandle = NativeMethods.ONNXRuntimeCreateSessionOptions(); + } + + internal IntPtr NativeHandle + { + get + { + return _nativeHandle; + } + } + + public static SessionOptions Default + { + get + { + return _defaultOptions; + } + } + + #region destructors disposers + ~SessionOptions() + { + Dispose(false); + } + + public void Dispose() + { + GC.SuppressFinalize(this); + Dispose(true); + } + + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + // cleanup managed resources + } + + // cleanup unmanaged resources + } + #endregion + } +} diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs new file mode 100644 index 0000000000000..40e655ee9a45a --- /dev/null +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Numerics.Tensors; +using Xunit; +using Microsoft.ML.OnnxRuntime; + + +namespace Microsoft.ML.OnnxRuntime.Tests +{ + public class InfereceTest + { + [Fact] + public void CanCreateAndDisposeSessionWithModelPath() + { + string modelPath = Directory.GetCurrentDirectory() + @"\squeezenet.onnx"; + using (var session = new InferenceSession(modelPath)) + { + Assert.NotNull(session); + Assert.NotNull(session.InputMetadata); + Assert.Equal(1, session.InputMetadata.Count); // 1 input node + Assert.True(session.InputMetadata.ContainsKey("data_0")); // input node name + + Assert.NotNull(session.OutputMetadata); + Assert.Equal(1, session.OutputMetadata.Count); // 1 output node + Assert.True(session.OutputMetadata.ContainsKey("softmaxout_1")); // output node name + + //TODO: verify shape/type of the input/output nodes when API available + } + } + + [Fact] + private void CanRunInferenceOnAModel() + { + string modelPath = Directory.GetCurrentDirectory() + @"\squeezenet.onnx"; + + using (var session = new InferenceSession(modelPath)) + { + var inputMeta = session.InputMetadata; + + // User should be able to detect input name/type/shape from the metadata. + // Currently InputMetadata implementation is inclomplete, so assuming Tensor of predefined dimension. + + var shape0 = new int[] { 1, 3, 224, 224 }; + float[] inputData0 = LoadTensorFromFile(@"bench.in"); + var tensor = new DenseTensor(inputData0, shape0); + + var container = new List(); + container.Add(new NamedOnnxValue("data_0", tensor)); + + // Run the inference + var results = session.Run(container); // results is an IReadOnlyList container + + Assert.Equal(1, results.Count); + + float[] expectedOutput = LoadTensorFromFile(@"bench.expected_out"); + float errorMargin = 1e-6F; + // validate the results + foreach (var r in results) + { + Assert.Equal("softmaxout_1", r.Name); + + var resultTensor = r.AsTensor(); + int[] expectedDimensions = { 1, 1000, 1, 1 }; // hardcoded for now for the test data + Assert.Equal(expectedDimensions.Length, resultTensor.Rank); + + var resultDimensions = resultTensor.Dimensions; + for (int i = 0; i < expectedDimensions.Length; i++) + { + Assert.Equal(expectedDimensions[i], resultDimensions[i]); + } + + var resultArray = r.AsTensor().ToArray(); + Assert.Equal(expectedOutput.Length, resultArray.Length); + + for (int i = 0; i < expectedOutput.Length; i++) + { + Assert.InRange(resultArray[i], expectedOutput[i] - errorMargin, expectedOutput[i] + errorMargin); + } + } + + } + } + + + static float[] LoadTensorFromFile(string filename) + { + var tensorData = new List(); + + // read data from file + using (var inputFile = new System.IO.StreamReader(filename)) + { + inputFile.ReadLine(); //skip the input name + string[] dataStr = inputFile.ReadLine().Split(new char[] { ',', '[', ']' }, StringSplitOptions.RemoveEmptyEntries); + for (int i = 0; i < dataStr.Length; i++) + { + tensorData.Add(Single.Parse(dataStr[i])); + } + } + + return tensorData.ToArray(); + } + + + } +} \ No newline at end of file diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/Microsoft.ML.OnnxRuntime.Tests.csproj b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/Microsoft.ML.OnnxRuntime.Tests.csproj new file mode 100644 index 0000000000000..8f58977d3623a --- /dev/null +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/Microsoft.ML.OnnxRuntime.Tests.csproj @@ -0,0 +1,38 @@ + + + + netcoreapp2.0 + false + ..\.. + $(OnnxRuntimeCsharpRoot)\..\build\Windows + $(buildDirectory)\$(Configuration)\$(Configuration) + + + + + + + + + + + + + + + Always + false + + + + Always + false + + + + + + + + + diff --git a/csharp/testdata/bench.expected_out b/csharp/testdata/bench.expected_out new file mode 100644 index 0000000000000..403588c4e2167 --- /dev/null +++ b/csharp/testdata/bench.expected_out @@ -0,0 +1,2 @@ +softmaxout_1 +[[[[3.30038e-05]],[[0.00240706]],[[6.37481e-05]],[[0.000747601]],[[0.00150019]],[[0.00483676]],[[0.0225981]],[[1.00831e-06]],[[1.09255e-05]],[[4.99784e-07]],[[3.56863e-06]],[[2.39458e-07]],[[4.05555e-06]],[[4.125e-05]],[[2.7419e-05]],[[7.0522e-06]],[[3.13915e-06]],[[1.43166e-06]],[[3.89503e-05]],[[9.59126e-07]],[[4.04121e-05]],[[1.44285e-05]],[[3.57193e-06]],[[1.79007e-06]],[[9.95244e-07]],[[8.06105e-06]],[[0.000350968]],[[1.53663e-05]],[[6.4513e-06]],[[0.0012472]],[[5.49246e-06]],[[1.79511e-05]],[[3.62395e-05]],[[0.0108226]],[[0.00336278]],[[0.000252014]],[[0.011735]],[[0.00017365]],[[0.00218219]],[[6.12092e-07]],[[2.211e-05]],[[3.50974e-05]],[[8.72451e-06]],[[1.4989e-05]],[[0.000234527]],[[0.0068162]],[[2.21972e-05]],[[0.00151148]],[[0.000432405]],[[0.00197073]],[[0.00484609]],[[2.82015e-07]],[[0.000228804]],[[3.39269e-05]],[[0.000382118]],[[4.70206e-06]],[[1.1913e-05]],[[2.23097e-06]],[[0.00685201]],[[3.31187e-05]],[[0.000336639]],[[2.79269e-05]],[[0.000199309]],[[0.00317999]],[[0.000113043]],[[0.00128387]],[[0.00155218]],[[0.000923709]],[[0.000240535]],[[0.00253241]],[[2.83529e-05]],[[0.00133016]],[[2.0673e-06]],[[0.000300064]],[[2.02181e-05]],[[0.00039521]],[[0.00180333]],[[0.00526178]],[[0.00672168]],[[0.00165962]],[[1.2716e-06]],[[2.40267e-05]],[[7.0667e-05]],[[8.33176e-06]],[[5.46878e-06]],[[0.000184338]],[[0.000245671]],[[9.80053e-07]],[[2.81819e-07]],[[4.73917e-06]],[[2.19647e-07]],[[2.98639e-06]],[[7.11458e-07]],[[8.84598e-07]],[[1.8719e-05]],[[2.70032e-07]],[[1.23598e-05]],[[3.72205e-07]],[[1.26912e-05]],[[2.47628e-05]],[[1.79942e-06]],[[0.000126682]],[[4.94588e-05]],[[0.0102678]],[[1.58766e-06]],[[2.19647e-07]],[[7.29778e-05]],[[0.00677497]],[[3.8126e-05]],[[0.000435307]],[[2.3404e-06]],[[0.132055]],[[6.02585e-05]],[[5.89881e-05]],[[3.03346e-05]],[[1.3844e-06]],[[5.24638e-05]],[[0.000825554]],[[0.00500985]],[[0.000136619]],[[1.46635e-05]],[[7.55287e-05]],[[3.5211e-06]],[[1.12535e-06]],[[8.07075e-05]],[[0.000272862]],[[0.00321429]],[[1.22513e-05]],[[2.93262e-05]],[[1.48032e-05]],[[5.36753e-06]],[[7.58634e-05]],[[0.000112646]],[[5.02625e-05]],[[3.24783e-05]],[[1.71106e-05]],[[4.81191e-06]],[[0.000141718]],[[7.72931e-05]],[[2.35613e-06]],[[0.00012713]],[[8.392e-05]],[[2.60958e-05]],[[3.85835e-06]],[[1.14518e-05]],[[6.8281e-06]],[[1.35586e-06]],[[6.49379e-05]],[[1.18678e-05]],[[0.00341779]],[[2.61765e-05]],[[2.91871e-05]],[[2.34715e-06]],[[0.000145097]],[[4.2643e-05]],[[0.000197889]],[[7.91037e-05]],[[9.83346e-06]],[[3.39639e-07]],[[3.03579e-06]],[[1.17122e-05]],[[2.06673e-05]],[[0.000150339]],[[0.00020328]],[[1.04667e-05]],[[6.91003e-07]],[[4.05582e-06]],[[5.44345e-07]],[[4.66852e-05]],[[1.18521e-05]],[[3.95729e-07]],[[5.05007e-05]],[[7.4885e-05]],[[6.45486e-06]],[[1.28401e-06]],[[2.01996e-06]],[[5.05181e-06]],[[4.2517e-06]],[[3.03471e-05]],[[2.77055e-05]],[[2.64213e-05]],[[1.88586e-05]],[[1.32463e-05]],[[6.96098e-06]],[[2.90003e-06]],[[3.06301e-06]],[[1.12795e-06]],[[9.59877e-07]],[[2.86446e-06]],[[1.51577e-06]],[[2.72648e-06]],[[6.72556e-06]],[[6.54428e-05]],[[4.4694e-07]],[[1.67012e-06]],[[1.66503e-06]],[[1.55137e-06]],[[1.84327e-06]],[[2.53061e-07]],[[1.05396e-05]],[[1.86334e-05]],[[7.70901e-07]],[[1.40602e-05]],[[0.000601871]],[[0.000196814]],[[8.07645e-07]],[[6.08658e-06]],[[3.00776e-06]],[[2.60375e-06]],[[2.38763e-05]],[[5.27953e-06]],[[1.07436e-05]],[[2.97076e-05]],[[2.28515e-05]],[[7.25094e-06]],[[0.000152125]],[[1.37768e-06]],[[1.51765e-05]],[[2.42037e-06]],[[6.10985e-07]],[[1.8458e-06]],[[7.37962e-06]],[[1.46182e-05]],[[2.51198e-05]],[[3.27596e-07]],[[1.4108e-06]],[[6.25375e-06]],[[2.68018e-06]],[[2.69143e-05]],[[6.06063e-06]],[[0.000106133]],[[1.46003e-05]],[[4.83793e-07]],[[1.34876e-05]],[[2.31457e-06]],[[6.68377e-06]],[[4.43177e-06]],[[7.89233e-05]],[[6.51787e-06]],[[3.42215e-06]],[[2.56579e-06]],[[2.39804e-05]],[[2.38996e-05]],[[7.296e-06]],[[2.30304e-07]],[[6.76266e-05]],[[1.74024e-06]],[[1.99373e-05]],[[6.47423e-07]],[[3.43456e-06]],[[1.10051e-05]],[[1.23886e-06]],[[2.60883e-05]],[[3.96144e-06]],[[4.10274e-05]],[[9.1267e-07]],[[5.01223e-06]],[[0.000132038]],[[6.39193e-05]],[[0.000112808]],[[1.90401e-05]],[[8.00349e-06]],[[7.82643e-06]],[[2.27094e-05]],[[1.38722e-05]],[[4.99394e-06]],[[2.65734e-06]],[[7.1897e-07]],[[3.4084e-07]],[[2.5075e-06]],[[9.08597e-06]],[[1.05238e-06]],[[2.37228e-05]],[[8.48681e-05]],[[0.00015608]],[[4.96326e-06]],[[9.76552e-05]],[[0.00012748]],[[1.16663e-05]],[[0.000474339]],[[0.000125924]],[[9.32651e-07]],[[6.14761e-06]],[[1.84157e-05]],[[2.0661e-05]],[[2.15305e-06]],[[1.90258e-06]],[[6.07655e-06]],[[2.72655e-06]],[[5.12465e-07]],[[3.07574e-06]],[[5.70555e-06]],[[3.70938e-05]],[[3.89949e-06]],[[5.05176e-06]],[[1.89544e-06]],[[3.55921e-05]],[[0.000115676]],[[8.98462e-05]],[[5.97214e-05]],[[2.12628e-06]],[[6.78752e-05]],[[4.29522e-05]],[[1.56884e-06]],[[1.50015e-05]],[[0.000283533]],[[6.39846e-07]],[[9.70085e-05]],[[1.88628e-05]],[[6.64936e-07]],[[0.000356803]],[[8.39299e-06]],[[5.74065e-05]],[[7.27516e-05]],[[0.00261427]],[[6.00777e-05]],[[7.12263e-05]],[[1.26491e-06]],[[0.000868939]],[[0.000397627]],[[2.35373e-05]],[[2.28251e-07]],[[3.06452e-07]],[[2.19647e-07]],[[8.73208e-07]],[[9.02118e-07]],[[1.382e-06]],[[0.000156077]],[[0.000121389]],[[8.77277e-05]],[[0.00316587]],[[0.000807538]],[[0.00204002]],[[0.00032277]],[[0.000302543]],[[4.31282e-06]],[[2.19416e-05]],[[0.00177341]],[[1.06765e-05]],[[1.11829e-06]],[[2.17968e-06]],[[1.53142e-05]],[[6.17556e-06]],[[1.55956e-05]],[[0.00168962]],[[5.31761e-07]],[[7.22402e-05]],[[6.69545e-06]],[[3.47428e-07]],[[9.00044e-07]],[[0.000113894]],[[2.07959e-05]],[[7.21104e-06]],[[6.67658e-05]],[[1.03238e-05]],[[3.54474e-06]],[[0.000450102]],[[0.000364276]],[[0.000164161]],[[0.000321661]],[[7.09733e-05]],[[0.000874399]],[[0.000532035]],[[0.000946064]],[[1.63541e-06]],[[6.66925e-07]],[[1.00801e-05]],[[9.33275e-07]],[[2.26388e-07]],[[2.65757e-07]],[[3.81374e-07]],[[3.49317e-06]],[[5.70276e-07]],[[1.79782e-06]],[[3.96168e-06]],[[3.79932e-07]],[[9.98164e-07]],[[2.68421e-05]],[[7.04427e-06]],[[2.14382e-06]],[[5.87577e-05]],[[8.82346e-07]],[[1.65695e-06]],[[4.0358e-06]],[[3.4606e-07]],[[4.24055e-05]],[[6.4875e-06]],[[7.15376e-07]],[[2.83752e-07]],[[0.000187299]],[[5.81318e-05]],[[0.000161363]],[[1.87876e-05]],[[1.81743e-06]],[[0.00169564]],[[0.00922038]],[[2.4518e-06]],[[2.37628e-05]],[[1.56076e-06]],[[4.16376e-05]],[[2.22112e-07]],[[2.74136e-07]],[[8.87607e-07]],[[2.37552e-05]],[[5.59479e-05]],[[0.000238186]],[[4.14247e-06]],[[2.62818e-07]],[[5.15268e-06]],[[2.44908e-05]],[[5.29226e-06]],[[0.000488674]],[[0.000196068]],[[4.08526e-07]],[[1.51241e-06]],[[4.3665e-06]],[[1.00193e-05]],[[0.00267409]],[[0.000107653]],[[0.00136346]],[[2.21079e-06]],[[1.27526e-05]],[[2.54319e-06]],[[1.4435e-06]],[[2.92003e-07]],[[1.88393e-05]],[[2.84691e-05]],[[2.419e-06]],[[2.81245e-06]],[[4.00504e-06]],[[1.22796e-05]],[[0.000329972]],[[7.76645e-07]],[[8.17329e-05]],[[0.0075691]],[[0.000272345]],[[3.47348e-07]],[[1.72498e-05]],[[0.00059095]],[[2.19856e-07]],[[2.97259e-05]],[[5.70946e-05]],[[3.50248e-06]],[[0.000505348]],[[2.6645e-07]],[[1.84202e-05]],[[0.0419757]],[[1.19775e-06]],[[0.000125246]],[[3.40366e-06]],[[1.28719e-06]],[[7.57776e-07]],[[0.000119444]],[[4.55271e-06]],[[5.57566e-06]],[[1.85926e-05]],[[4.23977e-07]],[[4.13716e-07]],[[0.000238636]],[[0.00234118]],[[2.00847e-05]],[[8.80801e-07]],[[6.20842e-05]],[[0.000201885]],[[4.35091e-07]],[[0.000212658]],[[7.93442e-07]],[[2.56245e-07]],[[2.86597e-07]],[[4.9724e-05]],[[0.000595791]],[[3.17517e-07]],[[8.28939e-07]],[[2.65091e-05]],[[0.000116553]],[[1.81658e-06]],[[2.7075e-07]],[[1.80121e-06]],[[0.00671099]],[[2.88147e-07]],[[3.31486e-05]],[[0.000103751]],[[7.2855e-07]],[[2.59299e-06]],[[8.29641e-06]],[[2.8954e-05]],[[3.82065e-07]],[[1.58906e-05]],[[7.5345e-06]],[[1.88725e-06]],[[2.20567e-07]],[[2.64769e-07]],[[0.000704843]],[[7.48047e-06]],[[7.07716e-06]],[[3.24475e-06]],[[0.000229562]],[[1.04762e-05]],[[2.75673e-07]],[[5.60927e-05]],[[0.000670869]],[[5.25422e-06]],[[7.67442e-06]],[[3.99764e-05]],[[2.02051e-05]],[[4.53167e-06]],[[4.54816e-06]],[[8.61861e-06]],[[1.01519e-06]],[[3.966e-07]],[[4.79176e-05]],[[2.37593e-07]],[[4.46226e-07]],[[2.95312e-07]],[[3.49675e-07]],[[5.54914e-06]],[[0.000167629]],[[2.35371e-05]],[[9.46391e-07]],[[0.000218146]],[[0.000695242]],[[0.000177932]],[[2.13105e-05]],[[1.37545e-05]],[[4.23788e-07]],[[0.00279564]],[[1.56444e-06]],[[2.2925e-07]],[[1.14547e-05]],[[0.000838637]],[[1.94783e-05]],[[2.22737e-06]],[[0.00269667]],[[0.000587703]],[[0.000242132]],[[2.39822e-07]],[[7.88235e-07]],[[2.19647e-07]],[[3.96023e-05]],[[0.000942217]],[[0.000199963]],[[2.98115e-07]],[[0.000114011]],[[2.78705e-06]],[[1.49864e-06]],[[5.44075e-05]],[[6.16242e-06]],[[2.25314e-07]],[[2.90298e-07]],[[0.0563553]],[[2.19647e-07]],[[0.00041908]],[[2.45128e-06]],[[0.000216731]],[[0.000220765]],[[7.10003e-06]],[[0.00314294]],[[4.27621e-06]],[[1.21116e-06]],[[3.57691e-06]],[[2.19866e-07]],[[2.77421e-05]],[[1.33204e-06]],[[3.23081e-06]],[[4.16899e-07]],[[3.00326e-05]],[[2.23953e-07]],[[5.92595e-06]],[[2.59079e-07]],[[3.95766e-07]],[[6.23917e-05]],[[2.3895e-07]],[[2.59321e-05]],[[8.72774e-07]],[[1.46551e-05]],[[2.37166e-07]],[[4.85887e-07]],[[5.91673e-06]],[[1.5925e-05]],[[5.48621e-07]],[[1.29689e-06]],[[2.42972e-07]],[[2.58339e-07]],[[7.07611e-06]],[[9.51068e-06]],[[9.89523e-05]],[[4.30986e-05]],[[6.61681e-05]],[[0.000377943]],[[3.49729e-06]],[[5.08567e-06]],[[0.100853]],[[1.0899e-05]],[[1.52226e-06]],[[4.74296e-07]],[[2.98692e-07]],[[1.48701e-05]],[[2.72196e-07]],[[8.15167e-07]],[[0.000854144]],[[4.81532e-05]],[[7.4703e-05]],[[2.33504e-07]],[[4.01518e-07]],[[4.65201e-05]],[[0.000614875]],[[2.85956e-05]],[[7.15136e-07]],[[7.42455e-06]],[[6.68006e-05]],[[0.00338469]],[[0.000373694]],[[3.92498e-07]],[[8.70474e-07]],[[1.46042e-06]],[[2.39024e-05]],[[4.56457e-06]],[[7.64493e-05]],[[2.58002e-05]],[[0.000290927]],[[5.9998e-05]],[[1.0992e-06]],[[7.49084e-06]],[[0.000270476]],[[1.07873e-05]],[[8.33809e-07]],[[0.000378689]],[[2.31458e-07]],[[6.9372e-06]],[[3.28015e-05]],[[1.66907e-05]],[[0.000668885]],[[3.65539e-05]],[[4.492e-06]],[[6.63593e-07]],[[1.4175e-05]],[[5.3893e-06]],[[1.25146e-06]],[[8.96112e-06]],[[1.56599e-05]],[[0.000639026]],[[5.25124e-07]],[[4.33042e-07]],[[9.55997e-07]],[[7.20381e-05]],[[1.8769e-06]],[[0.00591436]],[[0.0142333]],[[5.78227e-05]],[[5.58047e-06]],[[1.15346e-05]],[[0.000191445]],[[2.50651e-06]],[[5.37143e-05]],[[1.61384e-06]],[[2.10579e-05]],[[2.82325e-07]],[[0.000562215]],[[7.31107e-05]],[[0.000185458]],[[9.19092e-06]],[[2.24806e-07]],[[0.00122146]],[[3.34874e-06]],[[3.1904e-05]],[[2.49545e-07]],[[0.000527636]],[[5.26276e-07]],[[4.21786e-06]],[[0.0981208]],[[2.38566e-07]],[[8.18549e-06]],[[7.68801e-06]],[[1.79197e-05]],[[0.000124411]],[[2.81175e-07]],[[2.37748e-07]],[[0.000151226]],[[1.34168e-05]],[[1.33137e-06]],[[0.00140345]],[[4.45635e-05]],[[0.000123665]],[[4.43648e-07]],[[3.71324e-07]],[[2.56964e-06]],[[6.44212e-05]],[[3.52327e-06]],[[0.000145309]],[[2.28818e-05]],[[8.69461e-07]],[[7.64946e-06]],[[0.00124899]],[[1.95925e-06]],[[1.82611e-05]],[[0.000128566]],[[6.53273e-05]],[[6.7991e-05]],[[6.96028e-07]],[[6.85518e-07]],[[0.0999919]],[[0.000171717]],[[2.45555e-07]],[[2.20557e-07]],[[6.95378e-05]],[[2.36021e-07]],[[6.83477e-05]],[[2.71391e-06]],[[0.000297223]],[[6.14753e-05]],[[3.98032e-05]],[[0.000202539]],[[0.0211731]],[[0.000109838]],[[0.000204545]],[[9.54438e-07]],[[0.000132366]],[[3.13755e-07]],[[1.7683e-05]],[[3.05367e-06]],[[8.13365e-05]],[[8.62639e-05]],[[5.79236e-05]],[[1.72338e-05]],[[5.51936e-06]],[[2.37853e-05]],[[2.38583e-06]],[[7.3413e-05]],[[0.0365768]],[[0.000350949]],[[3.97942e-07]],[[0.000166126]],[[2.62828e-07]],[[0.000119214]],[[2.59006e-07]],[[0.000244027]],[[2.20833e-07]],[[2.69368e-05]],[[2.08583e-06]],[[6.67744e-05]],[[1.15667e-06]],[[0.00231503]],[[2.33152e-05]],[[2.7366e-06]],[[3.53167e-05]],[[2.03656e-05]],[[6.22545e-05]],[[1.2622e-06]],[[1.95008e-05]],[[0.000707449]],[[0.0022835]],[[2.14179e-05]],[[6.99508e-06]],[[7.92588e-05]],[[6.41414e-07]],[[0.00251094]],[[2.4398e-05]],[[6.75114e-07]],[[8.26408e-07]],[[4.115e-07]],[[3.32577e-05]],[[3.77059e-05]],[[2.97349e-07]],[[1.01808e-06]],[[1.14929e-06]],[[2.92594e-07]],[[3.70965e-07]],[[0.000210903]],[[7.57253e-06]],[[0.000671497]],[[2.57833e-05]],[[4.24406e-05]],[[0.000115908]],[[1.37939e-05]],[[1.4135e-05]],[[8.69344e-05]],[[1.36076e-06]],[[5.66509e-06]],[[9.69823e-06]],[[2.20374e-07]],[[3.79278e-06]],[[3.20583e-07]],[[3.48438e-05]],[[2.23965e-05]],[[3.01125e-05]],[[3.28908e-07]],[[1.8647e-06]],[[3.01314e-06]],[[3.23302e-07]],[[0.000202211]],[[1.46115e-06]],[[0.000135461]],[[1.81886e-06]],[[0.00220625]],[[0.0100632]],[[4.68942e-06]],[[5.13535e-06]],[[0.000709504]],[[6.95215e-05]],[[1.05843e-05]],[[2.19647e-07]],[[5.92654e-05]],[[7.72615e-06]],[[1.30853e-05]],[[0.00035334]],[[2.60654e-05]],[[1.70955e-06]],[[2.0768e-05]],[[1.40579e-06]],[[5.68283e-05]],[[2.28711e-07]],[[3.1642e-05]],[[0.000870124]],[[0.000250426]],[[8.21169e-06]],[[0.00041412]],[[5.17313e-05]],[[2.19648e-07]],[[2.58391e-05]],[[4.21913e-07]],[[2.75305e-07]],[[2.02621e-06]],[[5.78907e-06]],[[2.06269e-06]],[[1.4204e-05]],[[2.70143e-06]],[[1.57078e-06]],[[6.74661e-07]],[[3.99983e-05]],[[5.39646e-06]],[[2.84113e-05]],[[1.13468e-05]],[[2.61936e-05]],[[0.000348203]],[[1.3984e-05]],[[5.74484e-06]],[[4.46801e-06]],[[3.07966e-07]],[[0.000679563]],[[4.93455e-07]],[[0.000668054]],[[0.000694075]],[[0.000109974]],[[2.86653e-07]],[[0.000163258]],[[0.00102639]],[[2.56474e-05]],[[5.7945e-06]],[[3.08543e-06]],[[2.04542e-05]],[[6.7593e-06]],[[1.35856e-05]],[[1.80622e-05]],[[1.42013e-06]],[[2.31614e-05]],[[9.83816e-06]],[[2.23705e-07]],[[8.88647e-06]],[[2.99608e-07]],[[2.06588e-06]],[[4.90268e-07]],[[0.00990751]],[[5.02635e-05]],[[2.991e-05]],[[6.99187e-06]],[[2.19762e-07]],[[4.38414e-06]],[[1.94361e-05]],[[0.0118964]],[[4.71439e-07]],[[3.48044e-06]],[[3.06884e-06]],[[8.46049e-07]],[[2.84214e-06]],[[2.61806e-07]],[[8.08323e-07]],[[0.000378208]],[[1.44979e-06]],[[2.48622e-07]],[[2.63819e-05]],[[2.31254e-06]],[[3.73906e-07]],[[7.06266e-07]],[[8.55171e-05]],[[0.000463059]],[[0.00519056]],[[2.3448e-07]],[[7.43225e-05]],[[1.92467e-05]],[[3.02221e-07]],[[3.01564e-06]],[[1.00334e-05]],[[0.00138754]],[[5.73591e-05]],[[4.46816e-06]],[[0.0001871]],[[0.000772637]],[[5.25714e-05]],[[0.000222698]],[[0.00011766]],[[6.29192e-05]],[[0.00104611]],[[0.000150696]],[[2.85862e-06]],[[0.0213807]],[[7.83114e-05]],[[1.22906e-05]],[[2.89221e-05]],[[0.00558292]],[[1.80053e-05]],[[6.28303e-05]],[[0.000187814]],[[2.22076e-05]],[[0.019121]],[[1.44428e-05]],[[3.48743e-05]],[[4.95461e-06]],[[2.01822e-06]],[[0.000154636]],[[3.96885e-06]],[[1.15581e-05]],[[0.0217395]],[[0.00106286]],[[1.47636e-05]],[[7.78406e-07]],[[0.000182114]],[[3.64317e-05]],[[2.45914e-05]],[[0.000218702]],[[1.77705e-05]],[[3.88213e-05]],[[6.38537e-06]],[[6.49358e-06]],[[6.16064e-07]],[[9.00705e-06]],[[6.60956e-06]],[[3.62144e-05]],[[3.51255e-06]],[[3.13264e-05]],[[6.02226e-07]],[[9.67834e-06]],[[1.9771e-06]],[[8.10685e-06]],[[4.62282e-06]],[[1.3246e-06]],[[7.31311e-07]],[[7.34991e-06]],[[2.95626e-05]],[[3.87789e-05]],[[1.1141e-05]],[[2.79194e-06]],[[3.32728e-05]],[[8.18433e-06]],[[2.76908e-07]],[[2.60412e-07]],[[4.92324e-06]],[[2.33574e-07]],[[3.27539e-06]],[[1.97259e-05]],[[2.95824e-06]],[[2.76007e-06]],[[0.00158116]],[[5.84331e-07]],[[1.81017e-06]],[[1.5555e-05]],[[7.36356e-07]],[[0.000136843]],[[1.42929e-05]],[[0.000111963]],[[0.000735147]],[[0.000271685]],[[0.00143252]],[[6.72216e-05]],[[0.000145032]],[[0.000286367]],[[0.000266569]],[[0.000258938]],[[0.00629227]],[[0.000196766]],[[0.00144487]],[[0.000658506]],[[6.19422e-06]],[[7.89064e-06]],[[0.000167934]],[[3.46261e-05]],[[2.56693e-05]],[[1.69383e-06]],[[1.93635e-05]],[[1.4758e-05]],[[1.72688e-06]],[[2.75854e-07]],[[7.37269e-07]],[[6.86722e-07]],[[2.20309e-07]],[[3.00458e-07]],[[8.82893e-06]],[[5.07031e-06]],[[3.30098e-05]],[[5.39999e-06]],[[0.0041139]]]] diff --git a/csharp/testdata/bench.in b/csharp/testdata/bench.in new file mode 100644 index 0000000000000..b28238dad41ee --- /dev/null +++ b/csharp/testdata/bench.in @@ -0,0 +1,2 @@ +data_0 +[[[[1.7640523911,0.4001572132,0.9787380099,2.2408931255,1.8675580025,-0.9772778749,0.9500884414,-0.1513572037,-0.1032188535,0.4105985165,0.1440435648,1.4542734623,0.7610377073,0.1216750145,0.4438632429,0.3336743414,1.4940791130,-0.2051582634,0.3130677044,-0.8540957570,-2.5529897213,0.6536185741,0.8644362092,-0.7421650290,2.2697546482,-1.4543657303,0.0457585156,-0.1871838570,1.5327792168,1.4693588018,0.1549474299,0.3781625330,-0.8877857327,-1.9807964563,-0.3479121625,0.1563489735,1.2302906513,1.2023798227,-0.3873268068,-0.3023027480,-1.0485529900,-1.4200179577,-1.7062702179,1.9507753849,-0.5096521974,-0.4380742908,-1.2527953386,0.7774903774,-1.6138978004,-0.2127402872,-0.8954665661,0.3869025111,-0.5108051300,-1.1806322336,-0.0281822290,0.4283318818,0.0665172189,0.3024719059,-0.6343221068,-0.3627411723,-0.6724604368,-0.3595531583,-0.8131462932,-1.7262825966,0.1774261445,-0.4017809331,-1.6301983595,0.4627822638,-0.9072983861,0.0519453958,0.7290905714,0.1289829165,1.1394007206,-1.2348258495,0.4023416340,-0.6848101020,-0.8707971573,-0.5788496733,-0.3115525246,0.0561653413,-1.1651498079,0.9008265138,0.4656624496,-1.5362436771,1.4882521629,1.8958891630,1.1787796021,-0.1799248308,-1.0707526207,1.0544517040,-0.4031769335,1.2224450111,0.2082749754,0.9766390324,0.3563663960,0.7065731883,0.0105000203,1.7858705521,0.1269120872,0.4019893706,1.8831506968,-1.3477590084,-1.2704850435,0.9693967104,-1.1731233597,1.9436211586,-0.4136189818,-0.7474548221,1.9229420424,1.4805147648,1.8675589561,0.9060446620,-0.8612256646,1.9100649357,-0.2680033743,0.8024563789,0.9472519755,-0.1550100893,0.6140793562,0.9222066998,0.3764255345,-1.0994007587,0.2982381880,1.3263858557,-0.6945678592,-0.1496345401,-0.4351535439,1.8492637873,0.6722947359,0.4074618220,-0.7699160576,0.5392491817,-0.6743326783,0.0318305567,-0.6358460784,0.6764332652,0.5765908360,-0.2082987577,0.3960067034,-1.0930615664,-1.4912575483,0.4393917024,0.1666734964,0.6350314617,2.3831448555,0.9444794655,-0.9128222466,1.1170163155,-1.3159073591,-0.4615845978,-0.0682416037,1.7133426666,-0.7447548509,-0.8264385462,-0.0984525234,-0.6634783149,1.1266359091,-1.0799314976,-1.1474686861,-0.4378200471,-0.4980324507,1.9295320511,0.9494208097,0.0875512436,-1.2254354954,0.8443629742,-1.0002152920,-1.5447710752,1.1880297661,0.3169426024,0.9208588004,0.3187276423,0.8568305969,-0.6510255933,-1.0342428684,0.6815944910,-0.8034096360,-0.6895498037,-0.4555324912,0.0174791589,-0.3539939225,-1.3749512434,-0.6436184049,-2.2234032154,0.6252314448,-1.6020576954,-1.1043833494,0.0521650799,-0.7395629883,1.5430146456,-1.2928569317,0.2670508623,-0.0392828174,-1.1680934429,0.5232766867,-0.1715463251,0.7717905641,0.8235041499,2.1632359028,1.3365279436,-0.3691818416,-0.2393791825,1.0996595621,0.6552637219,0.6401315331,-1.6169559956,-0.0243261252,-0.7380309105,0.2799246013,-0.0981503874,0.9101788998,0.3172182143,0.7863279581,-0.4664191008,-0.9444462657,-0.4100497067,-0.0170204137,0.3791517317,2.2593090534,-0.0422571525,-0.9559450150,-0.3459817767,-0.4635959864,0.4814814627],[-1.5407969952,0.0632619932,0.1565065384,0.2321810424,-0.5973160863,-0.2379217297,-1.4240609407,-0.4933198690,-0.5428614616,0.4160500467,-1.1561824083,0.7811980844,1.4944845438,-2.0699849129,0.4262587428,0.6769080162,-0.6374370456,-0.3972718120,-0.1328805834,-0.2977908850,-0.3090129793,-1.6760038137,1.1523315907,1.0796185732,-0.8133642673,-1.4664243460,0.5210648775,-0.5757879615,0.1419531703,-0.3193284273,0.6915387511,0.6947491169,-0.7255973816,-1.3833639622,-1.5829384327,0.6103793979,-1.1888592243,-0.5068163276,-0.5963140130,-0.0525672957,-1.9362797737,0.1887785941,0.5238910317,0.0884220898,-0.3108861744,0.0974001661,0.3990463316,-2.7725927830,1.9559123516,0.3900933266,-0.6524085999,-0.3909533620,0.4937417805,-0.1161039397,-2.0306844711,2.0644929409,-0.1105406582,1.0201727152,-0.6920498610,1.5363770723,0.2863436937,0.6088438630,-1.0452533960,1.2111452818,0.6898181438,1.3018462658,-0.6280875802,-0.4810271263,2.3039166927,-1.0600157976,-0.1359497011,1.1368913651,0.0977249667,0.5829536915,-0.3994490206,0.3700558841,-1.3065268993,1.6581306458,-0.1181640476,-0.6801782250,0.6663830876,-0.4607197940,-1.3342584372,-1.3467174768,0.6937731504,-0.1595734358,-0.1337015629,1.0777437687,-1.1268258095,-0.7306777239,-0.3848797977,0.0943515897,-0.0421714522,-0.2868871987,-0.0616264008,-0.1073052734,-0.7196043730,-0.8129929900,0.2745163441,-0.8909150958,-1.1573553085,-0.3122922480,-0.1576670110,2.2567234039,-0.7047002912,0.9432607293,0.7471883297,-1.1889449358,0.7732529640,-1.1838806868,-2.6591722965,0.6063195467,-1.7558906078,0.4509344697,-0.6840109229,1.6595507860,1.0685093403,-0.4533858001,-0.6878376007,-1.2140773535,-0.4409226179,-0.2803554833,-0.3646935523,0.1567038596,0.5785214901,0.3496544659,-0.7641439438,-1.4377914667,1.3645318747,-0.6894491911,-0.6522936225,-0.5211893320,-1.8430695534,-0.4779739976,-0.4796558022,0.6203582883,0.6984571218,0.0037708890,0.9318483472,0.3399649858,-0.0156821124,0.1609281749,-0.1906534880,-0.3948495090,-0.2677335441,-1.1280113459,0.2804417014,-0.9931235909,0.8416312933,-0.2494585812,0.0494949818,0.4938367903,0.6433144808,-1.5706233978,-0.2069036812,0.8801789284,-1.6981058121,0.3872804642,-2.2555642128,-1.0225068331,0.0386305526,-1.6567151546,-0.9855107665,-1.4718350172,1.6481349468,0.1642277539,0.5672903061,-0.2226751000,-0.3534317613,-1.6164741516,-0.2918373644,-0.7614921927,0.8579239249,1.1411018372,1.4665787220,0.8525519371,-0.5986539125,-1.1158969402,0.7666631937,0.3562928140,-1.7685384750,0.3554818034,0.8145198226,0.0589255877,-0.1850536764,-0.8076484799,-1.4465347528,0.8002979755,-0.3091144562,-0.2334666550,1.7327212095,0.6845011115,0.3708249927,0.1420617998,1.5199948549,1.7195893526,0.9295051098,0.5822246075,-2.0946030617,0.1237219125,-0.1301069558,0.0939532295,0.9430460930,-2.7396771908,-0.5693120360,0.2699043453,-0.4668455422,-1.4169061184,0.8689634800,0.2768719196,-0.9711045623,0.3148171902,0.8215857148,0.0052926461,0.8005648255,0.0782601759,-0.3952289820,-1.1594204903,-0.0859307647,0.1942929327,0.8758327365,-0.1151074693,0.4574156106,-0.9646120071],[-0.7826291323,-0.1103892997,-1.0546284914,0.8202478290,0.4631303251,0.2790957689,0.3389041126,2.0210435390,-0.4688642025,-2.2014412880,0.1993001997,-0.0506035425,-0.5175190568,-0.9788298607,-0.4391895235,0.1813384295,-0.5028166771,2.4124536514,-0.9605043530,-0.7931173444,-2.2886199951,0.2514844239,-2.0164065361,-0.5394546390,-0.2756705284,-0.7097279429,1.7388726473,0.9943943620,1.3191368580,-0.8824188113,1.1285940409,0.4960009456,0.7714059353,1.0294388533,-0.9087632298,-0.4243176281,0.8625960350,-2.6556191444,1.5133280754,0.5531320572,-0.0457039624,0.2205076516,-1.0299352407,-0.3499433696,1.1002843380,1.2980220318,2.6962239742,-0.0739246681,-0.6585529447,-0.5142339468,-1.0180418491,-0.0778547525,0.3827324212,-0.0342422798,1.0963468552,-0.2342157960,-0.3474506438,-0.5812684894,-1.6326345205,-1.5677677393,-1.1791579723,1.3014280796,0.8952602744,1.3749641180,-1.3322116137,-1.9686247110,-0.6600562930,0.1758189499,0.4986902773,1.0479722023,0.2842796743,1.7426687479,-0.2226056755,-0.9130792022,-1.6812182665,-0.8889713287,0.2421179563,-0.8887202740,0.9367424846,1.4123276472,-2.3695869446,0.8640522957,-2.2396039963,0.4014990628,1.2248705626,0.0648561046,-1.2796891928,-0.5854312181,-0.2616454363,-0.1822447777,-0.2028968334,-0.1098827794,0.2134800553,-1.2085736990,-0.2420198321,1.5182611942,-0.3846454322,-0.4438360929,1.0781973600,-2.5591845512,1.1813786030,-0.6319037676,0.1639285684,0.0963213593,0.9424681067,-0.2675947547,-0.6780257821,1.2978458405,-2.3641738892,0.0203341823,-1.3479254246,-0.7615733743,2.0112566948,-0.0445954278,0.1950697005,-1.7815628052,-0.7290446758,0.1965574026,0.3547576964,0.6168865561,0.0086278990,0.5270041823,0.4537819028,-1.8297404051,0.0370057225,0.7679024339,0.5898798108,-0.3638588190,-0.8056265116,-1.1183118820,-0.1310540140,1.1330798864,-1.9518041611,-0.6598917246,-1.1398024559,0.7849575281,-0.5543096066,-0.4706376493,-0.2169495672,0.4453932643,-0.3923889995,-3.0461430550,0.5433118939,0.4390429556,-0.2195410281,-1.0840365887,0.3517801166,0.3792355359,-0.4700328708,-0.2167314738,-0.9301565289,-0.1785890907,-1.5504293442,0.4173188210,-0.9443684816,0.2381031513,-1.4059629440,-0.5900576711,-0.1104894057,-1.6606998444,0.1151478738,-0.3791475594,-1.7423561811,-1.3032428026,0.6051200628,0.8955559731,-0.1319086403,0.4047618210,0.2238435596,0.3296229839,1.2859840393,-1.5069984198,0.6764607430,-0.3820089698,-0.2242589295,-0.3022497296,-0.3751471043,-1.2261961699,0.1833391935,1.6709430218,-0.0561330207,-0.0013850428,-0.6872990131,-0.1174745485,0.4661664367,-0.3702424467,-0.4538040459,0.4032645524,-0.9180047512,0.2524966300,0.8203217983,1.3599485159,-0.0903820097,1.3675972223,1.0344098806,-0.9962126613,-1.2179385424,-0.3049636483,1.0289355516,-0.0722870082,-0.6006575823,1.5522432327,0.2869044840,-2.3205943108,0.3171606362,0.5200406313,0.2256086618,0.4497120976,-0.0672756061,-1.3183958530,-0.3707039952,-0.9456157684,-0.9327409267,-1.2630683184,0.4524890780,0.0978961438,-0.4481653571,-0.6493379474,-0.0234231055,1.0791947842,-2.0042157173,0.3768765330,-0.5457119942,-1.8845858574],[-1.9457030296,-0.9127835035,0.2195095569,0.3930629194,-0.9389815927,1.0170209408,1.4229835272,0.3960865736,-0.5914026499,1.1244192123,0.7553957105,0.8674074411,-0.6564636827,-2.8345544338,2.1167910099,-1.6108783484,-0.0357680731,2.3807454109,0.3305767477,0.9492464662,-1.5023965836,-1.7776669264,-0.5327028036,1.0907497406,-0.3462494612,-0.7946363091,0.1979672909,1.0819351673,-1.4449402094,-1.2105430365,-0.7886692286,1.0946383476,0.2348215282,2.1321535110,0.9364457130,-0.0350951776,1.2650778294,0.2114970088,-0.7049213648,0.6799748540,-0.6963266730,-0.2903971076,1.3277827501,-0.1012814865,-0.8031414151,-0.4643376768,1.0217906237,-0.5525406599,-0.3868708611,-0.5102927685,0.1839254946,-0.3854897618,-1.6018360853,-0.8871809244,-0.9327890277,1.2433193922,0.8126740456,0.5872593522,-0.5053583384,-0.8157915473,-0.5075175762,-1.0518801212,2.4972004890,-2.2453217506,0.5640085340,-1.2845523357,-0.1043434888,-0.9880019426,-1.1776289940,-1.1401963234,1.7549861670,-0.1329884231,-0.7657021880,0.5557869673,0.0103493147,0.7200337648,-1.8242566586,0.3036039174,0.7726948261,-1.6615983248,0.4481952786,1.6961815357,-0.0148577038,0.8214059472,0.6705704331,-0.7075057030,0.0397667363,-1.5669946671,-0.4513030350,0.2656879723,0.7231004834,0.0246121250,0.7199837565,-1.1029062271,-0.1016972736,0.0192793850,1.8495912552,-0.2141666561,-0.4990166426,0.0213512238,-0.9191134572,0.1927538514,-0.3650552034,-1.7913275957,-0.0585865527,-0.3175430894,-1.6324232817,-0.0671341568,1.4893559217,0.5213037729,0.6119272113,-1.3414967060,0.4768983722,0.1484495848,0.5290452242,0.4226286113,-1.3597806692,-0.0414008126,-0.7578708529,-0.0500840954,-0.8974009156,1.3124703169,-0.8589723706,-0.8989421725,0.0745864064,-1.0770990849,-0.4246633053,-0.8299645782,1.4111720324,0.7858038545,-0.0574695170,-0.3912170529,0.9409176111,0.4052040875,0.4980524182,-0.0261922367,-1.6882300377,-0.1124659851,-0.5324898958,0.6450552940,1.0118424892,-0.6579510570,0.4683852196,1.7358789444,-0.6677127481,1.6819217205,-0.8525858521,0.0229597557,-0.0111456122,0.0114989001,-0.8376780152,-0.5911831260,-0.6677202582,0.3269625902,0.3300351202,2.2259442806,1.3709889650,-0.5098432302,0.3248696029,0.9971179962,0.0306018237,-0.0696415752,0.0515749417,0.8672766089,-0.8483205438,-0.3256694674,0.4704331458,0.3114470840,0.2395827621,-0.3698011637,0.9725357890,2.1338682175,0.4064154923,-0.1931767017,0.7557402849,-0.5391326547,-0.7496903539,0.0328087471,-2.5827965736,-1.1539503336,-0.3479618430,-1.3533889055,-1.0326430798,-0.4367483258,-1.6429653168,-0.4060717821,-0.5352701545,0.0254052076,1.1541839838,0.1725044101,0.0210620221,0.0994544551,0.2273927778,-1.0167386532,-0.1147753224,0.3087512553,-1.3707599640,0.8656529188,1.0813760757,-0.6313759685,-0.2413377911,-0.8781903386,0.6993804574,-1.0612223148,-0.2224770039,-0.8589199185,0.0509542786,-1.7942292690,1.3264616728,-0.9646064043,0.0598946847,-0.2125230432,-0.7621145248,-0.8877801299,0.9363985658,-0.5256406069,0.2711701989,-0.8014968634,-0.6471814513,0.4722471535,0.9304084778,-0.1753164083,-1.4219198227,1.9979560375],[-0.8565493226,-1.5415873528,2.5944244862,-0.4040322900,-1.4617327452,-0.6834397912,0.3675448895,0.1903115511,-0.8517292142,1.8227236271,-0.5215796828,-1.1846865416,0.9606934190,1.3290628195,-0.8174930811,-1.4013472795,1.0304383039,-2.0473237038,-1.2266216278,0.9674461484,-0.0553525463,-0.2639373541,0.3528166115,-0.1527744234,-1.2986867428,1.2760753632,1.3250139952,0.2053325623,0.0451340154,2.3396248817,-0.2764328420,-0.2595769763,0.3644812405,1.4713219404,1.5927706957,-0.2585726380,0.3083312511,-1.3780834675,-0.3119761050,-0.8402903676,-1.0068317652,1.6815767288,-0.7922866344,-0.5316058993,0.3658487797,1.2978252172,0.4811151326,2.7593550682,-0.0746679753,0.2587164342,0.2756006718,1.4350494146,0.5072389245,-0.1162296981,-0.9474886060,0.2444434613,1.4013447762,-0.4103817940,0.5289435983,0.2461477816,0.8635196686,-0.8047537208,2.3466470242,-1.2791610956,-0.3655510843,0.9380925298,0.2967331707,0.8299861550,-0.4961023331,-0.0748049840,0.0122319832,1.5692596436,0.6904290318,0.7966721058,-0.6579260826,0.9688826203,0.2255816609,1.3891453743,2.0140602589,-0.3067657650,-0.4063031375,-0.8640449643,-0.1435795128,-0.3820254505,0.3595044017,-0.1445668191,-0.3615992665,1.0645850897,-0.9378802180,0.4331079423,-0.4059417248,0.7243685126,1.3852615356,-0.3030982614,0.4410329163,0.1787928641,-0.7994223833,0.2407875061,0.2891204953,0.4128708243,-0.1983989030,0.0941923037,-1.1476109028,-0.3581140637,0.5559626818,0.8924738765,-0.4223148227,0.1047140285,0.2280533314,0.2014799416,0.5407735705,-1.8180776834,-0.0493240692,0.2390335947,-1.0003303289,1.6739857197,0.1615592688,1.5634047985,-0.7905229926,-0.9073001146,0.2242522240,-1.6786884069,0.2149655968,0.0972192287,1.0156652927,0.7010413408,-0.4174773395,-1.0974966288,1.7123051882,-0.7921150327,-1.0455245972,-1.0848560333,1.1173052788,-0.5189002156,-0.7537044883,0.1376898289,-0.2069447041,-0.6780954599,0.7539914846,1.0653154850,0.9853175282,0.7669196725,0.4026255310,-1.7758879662,1.6692508459,0.3019891977,0.6081564426,1.1149623394,1.4333524704,0.4183980227,0.4355461597,-0.5992242694,0.0330897495,-0.8541612625,-0.7199405432,-0.8935744166,-0.1560238898,1.0490932465,3.1709747314,0.1894996315,-1.3484131098,1.2649832964,-0.3007838726,-0.6606085896,0.2098494768,-1.2406245470,0.2224631608,-0.0883755237,0.0983779058,0.3814162612,0.0674922541,0.0163380839,0.2843145132,0.4154006243,-1.0314824581,-1.4299912453,-0.0616380535,-1.4327354431,0.0875314698,0.9387468696,0.6071116924,-1.0481704473,-0.8602624536,0.3283012807,-0.4012978077,-0.3166553080,0.5969064832,-0.9872866869,-0.4012347162,-0.8000825047,-1.0431294441,-0.8570781946,0.6774621606,0.0518203899,-0.8791606426,-0.2311016023,-1.6388072968,-0.7333127856,2.1495745182,-0.0902438462,0.7316589355,-0.0654883757,0.3481692374,0.6632580757,-1.1046166420,-0.0309362579,1.5788651705,-0.7955005765,-0.5664398670,-0.3076912761,0.2690240741,0.5249178410,1.2674117088,0.4994982481,-0.0620531254,1.2591670752,0.7041110396,-1.4956794977,2.5263681412,1.7699214220,-0.1682142168,0.3779101074,1.3243587017,-0.1722007990],[0.7303518057,1.1045784950,-1.0148259401,-0.6023318768,0.9214084148,0.4608144760,0.9237965345,-0.1325680166,-0.2890052199,-1.9986394644,-1.1460003853,0.0470660962,0.8245572448,0.5311783552,-0.1282419711,-0.2717715800,0.2171796262,0.0782111809,1.4045455456,0.1464407742,-1.4812459946,-1.2725580931,1.5187593699,-1.1711604595,0.7644974589,-0.2683727443,-0.1697582901,-0.1341327876,1.2213850021,-0.1928418279,-0.0333192833,-1.5308034420,0.2066905051,0.5310425162,0.2391455770,1.3978962898,0.0551713556,0.2989774644,1.6485040188,-1.5500141382,-0.4558253586,1.4261587858,0.9361291528,0.6783800721,0.8326507211,0.3270662129,1.6315973997,0.3777591586,0.2398671061,0.1589586735,0.1928639561,-1.1570172310,0.7706730366,-0.1304397285,1.8219151497,-0.0756504685,0.4209182858,0.2466021925,-0.6255570054,0.9921368361,1.9050636292,-0.0147772199,-0.3004787862,-0.3550287187,-1.8923618793,-0.1778131425,0.2509981096,1.0547579527,0.9600477219,-0.4164990783,-0.2768229842,1.1239053011,-0.1734638959,-0.5100295544,1.3925184011,1.0375856161,0.0187917911,-0.5937774181,-2.0118803978,0.5897036195,-0.8963696957,-1.9627319574,1.5848205090,0.6479678154,-1.1390081644,-1.2144013643,0.8709617853,-0.8779706359,1.2961498499,0.6164593101,0.5365965366,0.4046954513,0.1914508790,0.8805112243,-0.4540803730,0.0859519765,0.7519465685,0.5629897118,-1.1949868202,-0.5004096627,0.2528035045,-0.4080147147,1.7746585608,-0.3931531906,-0.1622184515,0.7694301605,0.3305327296,-0.1452744603,-0.7564935088,0.3015140593,1.0390964746,0.4790952206,-0.7781835198,1.7367749214,-1.4465779066,-1.5826855898,0.9605572224,0.2258404791,-0.5494985580,-1.0985707045,2.3207998276,0.1170908734,0.5342011452,0.3178851008,0.4348079562,0.5400944352,0.7324240208,-0.3752224147,-0.2916419804,-1.7410228252,-0.7803044319,0.2711127996,1.0450233221,0.5990395546,-0.3406923413,-1.2631728649,-2.7773592472,1.1517339945,-0.5892289877,-0.4484650195,0.1315739751,-1.4055600166,-0.3497821689,2.0234718323,0.5053869486,0.3592491448,-1.5824944973,2.2436017990,-1.4227949381,1.9223247766,-2.1150560379,1.4053654671,1.6180542707,-0.8244091272,0.4225803614,0.5474805832,-0.8137944937,-1.4491176605,-1.3177173138,0.5410082340,-0.0851156041,-0.5643010139,0.9667680264,0.5080679059,-0.7554627061,-1.2012015581,0.5232617259,-0.5375833511,0.0992048606,1.5762989521,0.5023282170,-0.8622670174,0.1606611907,-0.9526449442,1.6085221767,-0.5615787506,0.2072707415,0.3077325821,0.1592504680,-1.9585489035,-1.4464210272,-0.4523502886,0.3194318414,-0.1377792209,-0.9571474791,-1.3484243155,-0.4015575349,-0.4684760571,0.5128364563,-0.3263184726,0.6027076840,-0.5946497917,-0.2559576631,-0.3480463922,-0.7823669910,0.6251186728,-0.8135960102,-0.5216414928,-0.0731196478,-1.2973796129,-0.3249349594,-0.7113063335,-0.3881541789,-0.0599280037,-0.7999136448,-0.2200757861,1.3086687326,-0.0257985573,1.1452621222,0.3464944363,0.7741606236,-0.7744589448,0.1049071625,0.1339129210,-0.6126257181,-0.8228283525,-1.4902653694,1.4961396456,-0.9724028707,1.3462210894,-0.4674931765,-0.8624932766,0.6225191355,-0.6311919689],[0.5684589148,-0.3328117728,0.4804244936,-0.9681860805,0.8313510418,0.4879726768,-0.9196506739,2.6429357529,0.5401230454,2.2904670238,1.6002677679,-0.1888347864,-0.4122717679,-0.4034591913,-1.8300285339,-0.6958351135,0.2467660308,1.5259575844,-0.7727718949,0.8820565939,-1.2525933981,-0.5863200426,-0.4576405883,0.3718110919,0.4573096335,0.9623417258,0.7708369493,0.2431682199,0.3903649449,1.5885306597,-0.5109261870,0.7747282982,-1.8081439734,0.4113342464,-0.4832495451,0.0025711823,1.0400862694,0.1646438092,0.8851875663,1.4737647772,0.3890939653,1.1710410118,-0.3265609741,-0.0082098823,-0.5226194263,1.0429775715,0.4140913486,-0.5072344542,0.1546688378,1.0415683985,-0.0392679907,-0.9489328265,0.1319117546,-1.9805655479,0.7687706351,-0.4213275909,-0.4693107307,0.8756957054,-1.3651628494,1.9470986128,-0.4802420437,-0.5232509375,1.0212247372,0.7086952925,2.4512298107,-0.2112059891,-0.1204066351,-1.4793159962,-0.3321022689,-0.7214313149,-0.4487670064,-1.7441877127,1.6606075764,-1.4166034460,-2.8022027016,-1.1884244680,-0.6038395762,-1.1495540142,1.0983035564,-0.1378391832,0.0253856052,0.6103917360,0.2860125303,0.9785673022,-1.1094775200,-0.5475180745,0.6659671664,-2.5345544815,-1.3751845360,0.5009922385,-0.4802490473,0.9361075759,0.8091803193,-1.1980929375,0.4066570997,1.2016978264,0.1474343985,-0.9774648547,0.8793899417,0.6354245543,0.5426107645,0.7159388661,-2.9946129322,0.8809375763,1.8081318140,0.4366384745,0.1927289963,0.6964386702,0.3382254839,0.6517812610,0.0014710003,-0.7667048573,-1.0043227673,-0.9981917143,-1.3730425835,-1.0677419901,1.7612661123,0.7540956736,-0.6250274181,-0.3903926909,0.1125575304,-0.6555450559,0.0675168559,0.7776041627,-0.0357427336,0.3360157311,0.8864915371,-0.2721317708,0.2847906053,-0.3093775809,-0.0285288692,-0.3247302771,-0.5288698673,0.1737118512,0.5665453076,0.1463044435,0.4987269640,-0.7379317880,-1.2037352324,0.4170435071,0.6878814101,0.0498572662,1.3480358124,0.9076988101,2.6805708408,-0.2008085102,-0.9988487959,-0.7401368022,-0.5654978156,0.4760313928,-2.1580686569,1.3185510635,-0.2392965853,-0.2467935532,-1.0793431997,-0.1142255515,0.0132397674,-0.1219449267,0.3390592635,-0.5896320343,-0.8958157897,0.5483281016,0.0986674502,0.1971810609,1.0590271950,-1.0225644112,-0.8552404642,1.2572196722,-1.4828833342,-1.3094121218,0.8178618550,0.2382001877,0.1052321345,-0.0916594118,0.0312675461,-0.0921121165,1.3554426432,-0.3981481194,-0.1613735408,1.7944488525,0.0275097024,2.2320163250,-0.1049797013,1.3674149513,-1.6553440094,0.1536444575,-1.5844736099,0.8444542885,-1.2128678560,0.2837695479,-0.2821958661,-1.1582032442,-1.6193599701,-0.5110404491,1.7406294346,-0.2934850454,0.9172215462,-0.0570428669,0.8767267466,-1.8269113302,-0.4031883180,0.9494055510,-0.1632549465,-0.0864552855,-0.4304619133,1.1493793726,0.2975143492,0.0440222770,0.6430545449,0.5882249475,0.2125870436,1.5470315218,-0.0602875352,0.2780810595,-0.6429525614,0.1501152217,1.5877615213,-0.6432576180,-1.1335928440,0.9967596531,-0.1487661451,0.0960042030,-0.0451133028,0.0791217238],[0.8505306840,-0.8391242027,-1.0117740631,0.0849681348,-1.6064397097,-1.3730535507,1.8666831255,0.7574683428,-0.0100564715,1.2380069494,-1.0405992270,-0.3156031370,0.6234536171,0.8906716704,0.5129168630,-2.5412387848,-0.9680821300,0.4770680964,-0.3559514880,2.5402317047,0.9265583158,0.5580818653,-1.1169495583,-0.0352967381,0.2412039638,1.1277836561,0.8811311126,1.0329891443,-0.9239119887,1.4121516943,-1.3804306984,-0.5359145403,0.4307711422,-0.1498915851,-1.0060368776,-0.8215498328,-1.5482543707,0.5319746137,1.2605688572,-0.1003935039,-0.4003488123,-1.4723229408,0.9132019281,2.2113044262,-1.7974557877,-1.0634329319,-0.6795930266,-0.5643178821,0.2273459435,1.6142495871,1.0085972548,0.5275973678,-0.7239286900,-1.1196281910,-0.7967752814,1.5480668545,-0.0617433004,-0.4468362629,-0.1837557256,0.8246182203,-1.3128496408,1.4148740768,0.1564762592,-0.2163439840,0.4428461194,0.2183970660,-0.3441964686,-0.2527106702,-0.8688625693,0.6563907266,-0.5319938064,-0.9562584162,0.1658635288,1.3291412592,-0.0483446233,-0.6081012487,0.4038960338,1.9367125034,-1.4519054890,0.3822027743,0.2050866187,1.1615338326,0.9909091592,-0.1867091060,-1.6845172644,0.8065637946,-0.8351926804,-0.9467403889,1.1483505964,-0.9108504057,1.4028447866,0.3358447254,0.3191184103,0.3072647750,-1.6384236813,-1.7763886452,0.2155530602,0.5680073500,0.0826110318,-0.8215345144,0.0189221036,-0.0820341557,-0.9571580887,1.0139721632,-1.7302761078,0.5887424350,0.3843234181,1.0097118616,-1.0053118467,0.1014071479,2.1711649895,0.6620742679,0.1005812064,0.5391612649,0.0861768425,2.1908979416,0.9836362004,-0.0856149569,0.2523314357,-0.3907980025,1.2098500729,-1.4061048031,-1.6047384739,1.4587147236,2.1531198025,0.4683049023,0.1127379388,0.6572676897,-0.6470535398,0.1712435484,0.0389087051,0.6265642643,-1.5579985380,-0.5070347786,0.8449956179,-0.6755938530,-0.9933613539,2.0420720577,0.0381180011,-0.5789181590,-1.6923704147,0.7293463349,0.6991361380,-0.2987596095,-1.1022301912,-0.0245494228,-0.8358560801,-0.9420936108,-0.1032127514,-1.0513904095,0.2466489524,0.6079925299,-0.8396324515,-1.3682451248,1.5612796545,-0.9402702451,-0.6599426866,0.2130171657,0.5993693471,-0.2563169003,0.4607943296,-0.4009861648,-0.9711706638,1.4263168573,2.4884417057,1.6959695816,0.1418066323,1.8334354162,0.3557035029,-0.4772862792,0.4663795829,-0.0943925083,-0.9831181765,-0.8983219862,0.8020517230,-1.8465319872,0.6041367650,-1.6295835972,-2.1211764812,-1.8388465643,1.9667639732,-0.1962339580,0.0865831822,1.4192550182,0.9341797233,-1.3915052414,0.8690063357,0.1841812581,-0.3416780829,0.0242909137,1.2798119783,-0.8859664798,0.4008856714,-0.0096572367,-1.7971645594,-0.8022531867,0.1932135522,1.2973420620,1.0013309717,0.5972124934,-0.8152756691,1.8012139797,0.2152404636,-1.0063655376,-0.1829049736,0.8962484002,0.0076174983,0.8868646622,1.1036939621,0.4005306959,-0.8577026129,0.1354546696,0.0451658554,1.8593463898,-1.6263219118,-0.1348224580,-0.5840935707,0.3351056278,-2.4375643730,1.1149245501,0.0137484875,-1.8447011709,-0.3611131310,0.6089623570],[-1.5914478302,0.0032222164,-1.0574736595,-0.5559850335,0.0267383829,0.1834502518,-0.4707424939,0.2727963924,0.8179776073,-0.2789142728,1.4315677881,1.4622141123,-0.4287020564,-0.6378405690,-1.6641730070,-0.1265693307,-0.3634377718,0.7790512443,-1.5096615553,-0.2773914039,0.9687443972,-0.7303571105,-0.7623615265,-1.4469403028,2.6205737591,-0.7474731803,-1.3003468513,-0.8038504124,-0.7742950916,-0.2693897784,0.8253722191,-0.2983231544,-0.9228233099,-1.4513385296,0.0218573585,0.0425390750,1.5309323072,0.0924477354,-0.0990083143,-1.0506538153,-0.3059525788,-0.4384744465,-0.3701641560,-0.9592553973,0.5383296013,-0.1424454153,-0.2003534734,-1.7140461206,0.4936440885,0.4870153368,-0.8391293883,0.9901213646,-1.3647582531,-0.0218699090,-0.2712073326,-1.3171747923,0.1897026151,1.7025702000,0.0676342323,-0.4630217552,0.4470241666,0.1057199985,0.0277621318,-0.4255422056,1.4219756126,0.4563633502,-0.5286706686,-0.1080038399,-0.7408667207,-0.6082911491,-0.6407257318,-1.1343115568,0.7772769928,-0.2910414636,0.5541275740,-0.6701259017,-0.0603624955,-0.7110406160,0.7196681499,-0.2484192997,-0.7308735847,-1.6417032480,0.2756665349,-0.7083850503,-0.0157792177,-0.4917300940,0.9541895986,0.5441447496,0.4472121000,-0.6161211133,0.4662900567,1.7148315907,-0.8321860433,0.1723391414,-1.6492170095,1.3985620737,-0.3979120851,0.7825788856,-1.7232282162,1.7975393534,-0.3568715155,0.5456573367,0.1508181989,-0.2554707825,1.6857923269,-1.6480462551,0.2987136543,0.9106456637,-0.0298561212,-0.1181707829,-0.1426877081,-1.2276364565,0.0381273851,0.5127175450,0.0685992241,-0.2722761035,-0.4897250235,-0.2792966664,1.2577441931,-2.0866348743,0.0400714576,-0.3277549148,1.4558079243,0.0554922260,1.4849256277,-2.1238899231,0.4595848918,0.2800578475,1.3905339241,-1.6413486004,-0.1550358087,0.0660602599,-0.4957954884,1.2165777683,-0.3386821747,2.0347626209,1.0541778803,0.9508336782,0.5592989922,-1.0636955500,-0.4310963452,0.5727513433,0.6775570512,1.3071838617,-0.4674410224,-0.8601533771,0.8591042161,-0.8096265793,0.8733118176,1.1997361183,0.4561530352,-0.3575790226,0.0410822257,0.5934659243,0.0101855183,2.1982963085,-0.9906709194,-1.0026686192,-0.9768953919,-0.5895799398,-2.1789314747,-0.6296504140,-0.6532847285,0.0785140246,0.4178005755,-1.2402163744,0.9000542164,1.8022422791,-0.2082851082,1.5743712187,0.1989894956,1.9887318611,1.1172834635,-1.5639046431,0.0186273698,1.0543249846,0.0305465814,-0.0368835293,1.2697647810,-0.7098541856,0.0175156128,0.3236257732,-0.3337909579,-0.0201291032,0.7750232816,0.4328376353,-0.8087175488,-1.1041239500,-0.7891021967,0.0012484558,-0.1599397808,-0.8319575191,-0.5981504321,-1.5200393200,0.4178537130,-0.0400187261,-1.2597873211,0.0286205038,1.3426220417,-0.7399358749,1.3151376247,-0.3234574795,0.1978281736,0.0977508053,1.4015234709,0.1584338397,-1.1419013739,-1.3109704256,-1.5329210758,-1.7119702101,0.0461350605,-0.9583745003,-0.0808116123,-0.7038590312,-0.7707843184,-0.4808453321,0.7035855651,0.9291451573,0.3711725473,-0.9898225665,0.6436312795,0.6888966560,0.2746472061,-0.6036204100],[0.7088595629,0.4228185713,-3.1168565750,0.6444520354,-1.9137426615,0.6635615826,-0.1540724039,1.1936116219,-0.0981612131,-0.8866142631,-0.1473536640,1.0598063469,0.0262466185,-0.1143351570,0.7435535192,0.2103593647,-0.0059274058,1.3660600185,1.5551140308,0.6133262515,-0.2859591544,1.4969110489,1.1831195354,0.7188971639,-1.2160766125,0.1406719089,-0.7436721921,-0.1590122581,0.2400569320,0.1001594067,-0.4751751125,1.2729537487,-1.6961312294,0.7301835418,-1.8574832678,0.3825981319,-0.8869042993,0.8783037663,0.0864525214,0.2477063835,-1.0182793140,-0.6545701623,0.2072173953,0.5835699439,2.9290962219,0.2228583246,0.9760375023,-1.5569338799,-1.3298919201,-0.3554947674,-1.1974277496,1.4863992929,-0.4102186859,1.3821818829,1.4867824316,0.0427797213,0.5017997622,-0.0560994744,0.5384370089,0.4833418429,-0.1236496270,0.5049699545,1.7236962318,0.7130162120,0.3257996142,0.1247695237,-1.0126731396,-1.0272969007,0.3233565390,-1.3693910837,-0.7663276196,1.2815113068,1.9142297506,-1.6659560204,1.6266496181,-0.2114382982,-0.0150050875,-0.1134116277,1.0805441141,-1.6076766253,0.4561636150,-0.9448701739,0.5707885027,1.5427963734,-0.0004173264,0.3741550744,0.4095517695,-0.7995935082,1.5116393566,1.7064682245,0.7017833591,0.0732854307,-0.4618938267,-0.6264902353,1.7108365297,1.4144150019,-0.0636614859,-1.5799305439,-2.8320119381,-1.0834267139,-0.1306203902,1.4006890059,-0.6516562104,0.5048154593,1.3031809330,0.1285363138,-0.1424478740,-1.3087635040,-1.2024753094,0.4160996377,-0.2009075284,0.1225313172,-0.0472777151,0.6641440392,-0.7846873999,-0.3355806470,1.8961821795,-0.7997861505,-0.2815754414,-0.5893867016,0.4447813630,1.0223922729,-0.4982116222,-0.4314143360,-0.2789815962,0.5298337936,-0.7393953204,-0.3759599626,-2.3721938133,-1.3817449808,-0.1124437526,0.8978641629,0.2950757742,-1.0987684727,-1.4002561569,0.1746800989,-1.6528036594,1.0659267902,0.0638961941,-1.6073201895,-0.9659538865,-0.7243112922,-0.7731925249,-1.4899330139,-0.8746625185,-0.6844015718,-0.7112857699,1.1279566288,0.1048278064,-0.9932572246,-0.3346216083,-0.8795570731,-0.3000066578,0.8755091429,0.2522707880,2.2856011391,0.3759274185,-0.9135944843,0.8097407222,1.0799312592,1.0941669941,-1.0942409039,-0.1476374120,1.1318119764,-1.6847289801,-0.4994167686,-1.4269376993,-0.9325702190,-1.0124571323,1.2505698204,-0.2345380336,-0.8633555770,-1.0356057882,0.1416671723,-0.0111356275,1.3440743685,0.5000166893,-1.4317977428,-0.6289806962,1.0700725317,-0.6210827231,1.7345721722,-1.0982894897,0.5726133585,-0.8612155318,-0.5095951557,1.0985816717,-0.1270671636,0.8134522438,0.4732905924,0.7538656592,-0.8881881833,-0.2215743959,0.4242526293,-0.8490728736,1.6295000315,-0.7772280574,-0.3000035882,-1.0065590143,-2.1433081627,1.7969185114,-0.2043389380,-0.4479148388,-0.1987150609,1.4198639393,-0.9651066065,0.6795678735,-0.4237882495,-0.5966708660,0.5670582056,0.9882405996,-0.5139029622,-0.7688491344,-1.1690957546,1.1035038233,-0.5752559900,-1.8491307497,1.4099521637,-1.3698594570,0.7794605494,0.1834286451,0.2879154384,-0.5843752623,0.3655914664],[-1.6677799225,0.5880377293,1.5570100546,0.8840271831,-2.0195400715,-0.9842090011,-0.1877949238,0.4869373143,-0.1066526771,-0.4932143986,0.5953003168,1.1641517878,-0.2322940081,0.7289298773,-2.5790507793,-0.9375093579,-0.3212589324,-0.4885662198,0.3327982128,1.0137505531,0.5066690445,-0.6222254634,-1.5227681398,0.5569640994,-1.8381767273,0.6530373096,-0.1884490848,-1.1758350134,0.2872573137,-0.0028761027,-0.0365972929,-0.0842233002,0.4195241034,0.9244340062,0.4966152012,1.0121332407,-0.0441397205,1.6184593439,0.5711098313,-0.5436940193,-1.0938950777,0.2057968080,-1.3065215349,-0.9733759761,0.2390870750,-0.6078874469,-0.9333162308,-0.0344750471,0.0726778954,-0.2058340311,-0.3775469065,0.8546427488,0.3424273431,-0.2234261185,2.4643218517,0.1938317418,1.1320050955,-0.5609809756,-1.3629409075,-0.7917565107,-0.2680097818,-0.4966081977,1.3363862038,-0.1200411245,0.4614688754,-0.0464811549,-0.4335543215,0.0379960127,1.7140514851,-0.7679485679,0.7669904232,-1.0260072947,-0.4596264362,0.0035832059,0.3263750970,1.4831286669,-0.0500826426,-0.8436155915,0.6500419974,-0.3641698062,0.2386815697,-0.1162224412,-1.9434568882,0.5082991719,0.5833680034,0.9266047478,1.8004627228,-1.1951037645,0.5165074468,0.4092949927,-0.4190819860,0.3971062303,0.4996469617,-1.2186838388,0.2462227643,-0.9179843068,-0.6518564820,-1.7747448683,-0.4733609259,-0.2035706788,0.5498568416,0.0008999266,-1.5422881842,0.8621480465,-0.1185866222,0.4883705974,0.9659361243,1.4226047993,1.9612269402,-0.0722387582,0.3111244440,-1.0783610344,1.0616002083,-1.1848874092,-1.8052517176,0.8303859830,-0.5216965079,0.7776072621,0.4080746472,-1.6300026178,-2.7196793556,-1.0966017246,0.0164914876,-1.2217763662,-0.6527614594,-1.4589407444,0.1698779613,0.0908259302,-0.4813926220,1.3970652819,1.4977149963,0.5652672052,-1.7997711897,-1.1046901941,0.4071303308,-0.6285575628,-0.4870914221,0.8989673853,0.5108748078,1.3141543865,-0.4292092919,1.3752254248,-0.5541312695,1.4994914532,0.1058346406,-0.8605097532,-1.6312195063,-0.3014723063,-0.2562327087,0.8576619029,-0.1105905026,-0.4324319661,1.0770374537,-0.2248265594,-0.5762417912,0.5746089220,-0.4898282290,0.6588021517,-0.5969170928,-0.2229591757,0.1521769762,-0.3741263151,-0.0134514691,0.8154719472,0.4106017947,0.4809698462,-0.6354304552,0.8528297544,0.6695623398,1.0044192076,-0.7263658047,-0.1724586040,0.6335338950,-0.6088151336,-0.2261224687,1.9258056879,1.9517610073,1.2399405241,0.9385851622,-1.0192511082,0.5125622153,-0.3591165841,-1.0585719347,-0.5090058446,0.1156650707,-0.5473555923,-0.5507994294,0.7920414805,0.1441064924,0.2334580868,0.1118723974,-0.6757031679,-1.3705719709,0.3105646968,-0.5070366263,-2.0107822418,-0.3925672472,-1.0922179222,0.6986502409,0.5216252208,0.4968931377,-0.6650416255,0.7315515876,0.3196497858,-0.4098545313,-0.4533374310,0.8927081823,-0.4736040533,0.3036564589,1.0339570045,1.9093426466,1.6638730764,0.9008227587,-1.5059113503,-0.6890484095,-0.5480871797,1.6531498432,-0.6993179321,0.3861663640,0.1008670628,-0.9351271987,0.3818240166,0.3982960880,-1.2557748556],[1.2228775024,-2.0865099430,-0.5907571316,0.9719703197,-1.1932578087,0.3502659202,-1.2963603735,-0.0930241421,-2.3137731552,-0.8425716758,-1.5429214239,-0.4017637372,-0.4152314067,-0.6736641526,0.7979131937,-0.8868796229,0.6343866587,1.6292757988,0.1390641481,-0.8576701880,-1.2493385077,-0.7097851038,0.7046427131,0.1555907279,0.9367952347,0.7703309059,0.1408106536,0.4734882712,1.8552461863,1.4156562090,-0.3027460277,0.9896794558,0.5858508348,1.1363880634,0.6716165543,-0.9741674066,-1.6196845770,0.5726270080,1.9026181698,-0.7756640911,-0.1880897433,-1.0357477665,1.1778295040,-2.3051669598,-2.2636601925,0.3750199080,-0.0823436454,-0.4796230197,-0.3010948002,0.5369879007,-0.4138039947,-1.0969250202,-0.9273629189,0.8883388638,-0.5247419477,-1.3852776289,0.1021783277,0.5049947500,1.3289607763,0.2179033905,-0.6597112417,0.4740078747,0.7271748781,-0.0389053077,-0.0445993915,0.2601329088,-0.0698564947,0.2501139045,-1.0219132900,-1.1504377127,-0.8361113667,0.6422109604,0.2587975562,1.0402389765,-0.1866909266,-1.1436413527,1.1445535421,-0.0187670551,1.2834550142,0.5979464650,2.1886186600,-0.2197729796,0.9007239342,0.8913640976,-0.5551263690,-0.1724823117,-1.4617383480,-1.5487961769,0.1265687943,0.7930070758,0.6380240321,0.3400245905,0.8630171418,-0.5896977782,-0.2725327611,0.7375215292,0.4331187308,-0.2101888210,1.3207943439,-1.2920012474,-0.5186786652,-0.2833977640,0.8165348768,0.0023851979,-1.2614917755,0.5140041709,1.0875463486,0.7393045425,0.6191549301,-1.8743134737,-0.8998864889,0.4820806086,-0.0548881851,0.5225576162,-1.2663426399,-0.0614947639,-1.3897809982,-1.9536786079,0.2957790792,0.8425887823,0.2456164211,-0.0329964794,-1.5620143414,1.0061070919,-0.0440448970,1.9595620632,0.9423143268,-2.0051255226,0.7550497055,-1.3965352774,-0.7594954967,-0.2507566810,-0.0940624475,0.3975652158,-1.0228550434,-1.1506919861,0.6006051898,-0.0132502681,0.1743730456,-2.1936833858,-0.1771373898,-0.8907291889,-0.9206264019,0.9219347835,-1.0956711769,-1.0928965807,-0.3310106099,0.4502888322,-0.8840147257,1.2341440916,1.4498475790,-0.8814470768,-0.2450817525,-0.7786754966,-1.6853821278,0.3030110598,0.7335948944,2.0118641853,-0.8974094987,1.3362350464,1.3423537016,0.1978533119,0.6021634936,0.8732730746,1.9740999937,0.4778085649,-0.0601378866,-0.8661687970,0.3053207695,1.0241649151,0.2446103543,-0.7799232602,0.0890762061,-0.1291534752,0.2647387683,-1.6618484259,0.5507886410,0.5954231620,0.4448534250,-0.0037628172,-1.8059362173,-0.0193227921,1.0607149601,-0.8601288795,-1.9892694950,-1.5405579805,0.3140257001,0.3728760183,0.8862931728,-0.0552589968,-1.5003284216,-0.8185041547,0.8188393712,0.1404959112,0.6498296261,0.4347887933,-0.2049605548,-0.1740068346,1.8571022749,0.4146742523,-0.1285875440,0.4554199874,0.2229058146,-2.1573562622,0.6500844955,1.8209393024,-0.7802798748,1.4540357590,-0.2568697035,0.2934713960,1.0703600645,-0.7200014591,1.2424938679,-1.2142173052,-0.8751547337,-0.5935203433,0.6620053649,-0.3408744037,-1.5199744701,-0.2165328711,-0.7842214108,0.7312936187,-0.3432350457,0.0707740784],[-0.4054724574,0.4339389801,-0.1835907698,0.3251987100,-2.5933885574,0.0972508788,0.4139136672,-0.1992800534,0.6693924665,0.7386070490,1.3042138815,0.1048116088,-1.9138007164,-2.2854993343,-1.6018409729,-0.0379070602,-0.1573052853,0.2762398422,-0.6252459288,-0.7364911437,0.5550479293,0.6559244394,-0.2566501498,-0.0384766571,0.4043143392,0.5043435693,-1.1439807415,-0.7195738554,-1.2305459976,-0.5069066286,0.8123335838,0.5462718606,-1.0980979204,0.5122667551,0.0858431086,-0.4939267039,-1.4064596891,-0.1748233736,0.6799439788,-2.1630976200,-0.3961232007,2.2542836666,0.6726367474,0.2598325014,-0.7371851802,-0.6783298254,-0.0832883939,1.6028636694,0.4655891955,-0.8721584082,1.1767870188,-0.2925941944,1.6973464489,-0.5666030049,-1.0032657385,0.1746295840,0.9823269844,1.0374448299,0.1591917723,-0.9880967140,-0.5053406954,-2.0182819366,-0.9131215215,-0.1784568131,0.3890021443,-0.3394543231,-0.0569790564,-0.3961854577,0.7510253191,-0.8991129398,0.8375478983,1.9608807564,0.4727896452,-0.5270916224,-0.5362701416,1.2098371983,-1.1265894175,-0.9538044333,-1.1644484997,-1.2785137892,-1.0448163748,0.7899049520,1.1022825241,-0.6970731020,0.2073340416,0.7591567039,0.1005642042,-0.9549427629,-1.4704017639,1.0104275942,0.4961794019,0.5769559145,-1.1076469421,0.2349771857,0.6289995909,0.3140338361,-0.7450231910,1.0122605562,-1.5276319981,0.9287419319,1.0810559988,1.5723303556,-0.3424921930,-0.9994300008,0.7938803434,-0.6992152929,0.0439955108,-0.3174622059,-0.9020719528,0.3209994733,-1.3920159340,0.5922057033,-0.9669311047,-1.7317312956,-0.0501074605,0.4316338599,0.5769345760,0.8183537126,-2.3536403179,-1.0051444769,0.1066522971,1.5190032721,0.7837445140,1.9013400078,-0.5249394178,0.2744169831,-1.0999708176,-0.4043522179,-0.7352957129,-0.6339886785,-0.3934491277,0.0027175399,0.0222126637,0.5434534550,0.1399884671,-0.3440456390,-0.5225785375,-0.3071317077,-0.4490371346,0.4909710586,0.8655251861,1.2740445137,-0.7977027893,0.4693722129,-1.3946796656,0.3731747270,1.0826722383,-0.1495895088,1.0726360083,-1.1385679245,-0.8886452913,-0.1358098388,1.0222103596,-0.4174294472,-0.4535531104,-0.9916283488,0.2028810382,1.2466951609,0.7006801367,0.6966506839,-0.2069744766,-0.5633093715,0.6772459149,-0.0319110751,-0.1736082286,0.8982406259,-0.1977874488,-0.8377762437,0.9091885090,0.0807198882,-1.0370293856,-1.1129058599,0.0954118744,2.3374097347,-0.3928205967,-0.3362738490,1.5237711668,-0.0572811998,-1.4484668970,-1.5727964640,1.2266639471,0.6663545370,0.8261256814,-0.0577565581,-0.7267120481,-0.2171631157,0.1360312104,-0.8383111358,0.5614498854,-1.2595961094,-0.3327587545,-0.2040078789,-0.6910198331,-2.2055053711,0.4478696585,-0.7557507753,1.3257079124,-0.3419822752,-0.5413596034,0.0915219486,1.0534397364,-0.5634076595,1.0147377253,1.4403036833,0.9903228283,1.6264314651,1.2926460505,1.5148823261,1.6043263674,0.2080695331,-0.4292238951,-2.2622437477,-1.3227331638,-0.4482828081,-0.3817350864,-0.1527944654,-1.0007604361,-1.5957776308,-0.1302231699,-0.1894179285,-0.8075540662,-0.7421521544,-0.9401565790,-0.3965237439],[-0.8563027978,1.2598752975,0.2409967333,-0.9723179340,-0.2804477811,-1.1802855730,1.0121682882,1.3841867447,1.2520020008,-1.1446926594,-0.0912670195,-0.4015706778,0.5620130897,-1.0079097748,-0.6758916974,-0.4132170379,0.1532884687,0.6941286922,-0.3287276924,0.6639650464,0.8220763803,-0.2132152319,-1.2456581593,-1.1711903811,0.5917269588,-0.4762244225,-1.7126293182,0.6129523516,0.1295545250,-1.4059671164,1.1794199944,0.8366360068,0.1387452483,-1.2743194103,-1.4023305178,-0.3070684969,-1.7139153481,0.4050802588,-1.4108233452,0.1649127305,-0.2881314456,0.7117852569,-0.9379475713,0.2737294436,-1.3948402405,0.7955495715,-0.1149617657,0.4958506823,-1.3205252886,0.4990842640,0.3062033951,0.3636978865,0.3126339614,-0.1934638768,1.2412992716,-0.1558979899,-0.7391691804,-0.0587261915,-0.9505179524,-0.4639964104,-0.1772466153,-0.3795541227,0.1993970722,1.9457614422,0.5709498525,1.0723006725,-0.5037094355,-0.5870162845,-0.3781780601,0.8528891206,-2.1481184959,-1.0331647396,0.1023358479,-0.2240923643,1.9677296877,0.4476832151,-0.6621914506,-1.5776070356,-0.3405600488,-1.3032200336,0.4667506516,0.1611063182,0.3200319409,2.0791766644,-0.9074659944,-0.1924042106,-1.2125157118,-0.0805985183,1.5932736397,0.5687224269,-0.1144870445,0.2516302466,-1.2108556032,-0.3937337101,0.0852525756,0.0994219854,-1.5306162834,0.3276231885,0.2791965008,-0.3770512044,0.0041749990,-1.4834915400,-1.4797955751,0.1346872598,-0.6677231789,-0.0115555199,0.8394906521,-0.1739299297,-2.8106679916,-0.1506536454,-0.4810440242,-0.2346943617,0.8997308016,-1.5785301924,0.2439566255,1.5703039169,-0.6259431243,0.4723278880,0.9663057923,0.2102314383,-0.6850969791,-0.7095209956,0.7438001633,0.5921490788,-0.7864683867,-1.1764731407,-1.2808066607,1.6616518497,-0.0679451227,2.3602285385,0.5555456281,0.4395223260,0.3062724769,0.9991498590,-0.9660632014,2.1600131989,-0.1003017053,-0.7034000754,0.3025610149,1.0923389196,-1.0075548887,0.5668693781,-0.7164441347,-0.5062735081,-0.4894824326,0.7635414600,-1.1090726852,0.1926161051,-0.3434178531,-0.8472101688,-1.2135236263,-1.2028883696,-1.6337959766,0.8961672187,-0.2416531593,0.1586519331,1.1781893969,-1.2201172113,-0.9415456653,0.2547155321,-1.8240795135,-0.5787085295,-0.9248930812,0.3295224309,-0.4258158803,2.0081493855,0.9378913641,-0.8532384634,-0.3873134255,-0.3475845158,3.3065743446,-1.5101996660,0.2035396993,-2.0844321251,-0.0069374414,1.9098905325,-0.4084554315,1.1045544147,-0.0661152229,-0.4224987030,-0.2516563535,-0.5869026184,-0.6260582805,-1.3301943541,1.5068007708,-0.3930763900,0.2937743068,-0.8765318394,1.1169905663,-0.2735557854,-0.0910326689,-1.8289766312,0.3959762156,1.8115056753,-0.8690775633,-0.4582291543,-1.1383239031,0.1291621774,0.0640241951,0.7050811052,0.5514735579,-0.8125160336,0.2249480486,-0.3283011019,-1.0910329819,-0.1268558800,3.8016602993,2.3151705265,0.1398265958,1.7388571501,-0.0453833640,-0.0531383380,-1.9495717287,-0.9601055384,-0.7834992409,0.1075190306,0.0139845349,-0.5789423585,-0.5888131857,-0.1661531329,-1.3814116716,-0.6126385331,-0.3812898695,-1.2489489317],[-0.3302378953,-0.8348071575,1.2353824377,-0.2438037992,-0.1895456612,0.4280281067,0.5569683313,-1.7362418175,-0.3767841160,-0.9090323448,-0.1451702416,-0.5363325477,0.1570694596,-0.9804592729,-0.5677672625,-0.5911596417,1.0825914145,0.3680036664,0.3688887060,-0.2863182724,-0.3847178519,0.5610029101,0.7774339318,0.0151467854,1.1416479349,1.2741550207,-1.6646980047,0.4303788841,-0.0426019281,0.3882888258,1.1159765720,-0.9205381870,-1.6202740669,1.1061915159,-0.9984846711,-0.6862195134,0.2046208978,-0.6861017942,-1.5922106504,0.0341897681,-0.7814846635,0.5978598595,-0.5060765743,-0.6884461641,-0.2100005448,1.0521534681,0.9079040885,-1.0932261944,2.7997076511,-0.3257763386,-1.1524157524,0.8882319927,-0.3616724610,2.1537194252,0.8474083543,-0.1987198442,1.5753068924,0.8491151929,-1.2288951874,0.8883941174,-0.5164874196,-0.0833262876,0.1310544461,-0.8790960312,-1.3333423138,0.3677840233,-1.3882335424,-2.5752027035,-0.8361055851,0.3310924172,-0.2698811293,1.2671309710,0.1837534904,-0.7663096786,-0.4395835400,-1.4365413189,1.0857971907,-1.3811000586,-0.9204076529,-0.1602862179,0.0023532645,-1.5026503801,-0.9055358171,0.2650406063,1.1297233105,0.3490035534,-0.0258097611,-1.5624086857,-0.6173423529,0.5214942694,1.0809466839,0.8893759251,0.1380716413,1.2046004534,2.8814606667,-0.5938619375,-0.7631158233,1.5184829235,0.2354645282,0.1123076901,0.3923743367,-0.6544864774,-1.0347952843,-0.7771475315,1.2459462881,-1.4366406202,0.4986546338,-0.5576874614,-0.3533668816,0.7429509759,0.8439888954,0.3429765403,-1.8731197119,1.5709646940,1.3101965189,0.0914368331,0.0102578169,1.8014491796,0.9472242594,-0.0292944033,-0.2923386693,-0.1935371161,1.1772320271,1.0399917364,-1.6134229898,0.4646424055,0.8641213179,-1.5064631701,-0.0029647513,-1.7770435810,0.1294928342,-2.0832345486,-0.6817455292,-0.6110659242,-0.7088498473,1.4515280724,0.5355105400,-0.3995688558,-0.9330778122,-0.2387763113,-1.0291129351,0.9730799794,1.9967659712,1.0531998873,0.3316903412,-0.1656287760,-0.4051062763,1.7452845573,-0.5759356022,1.5610985756,-1.1315392256,-0.2962316573,-1.7140566111,0.1592341959,-1.2637276649,1.6650494337,0.4122722745,0.5373967290,0.2826784551,-1.0925408602,0.1241182908,1.8370807171,0.0085549261,-1.0170161724,-1.8523426056,-0.7133269906,-1.7622288465,0.8305173516,0.7811672688,-0.8756818175,0.6139813066,-0.5764546394,-0.0456142835,0.3719555438,-0.4439600110,0.4182033539,-1.6857280731,0.1174749881,-0.0349520221,-2.0463931561,-1.8096902370,-1.8595236540,0.4143068194,0.1239596233,0.2739575803,-1.3263784647,1.1389738321,0.9828411937,-0.7669630647,1.1760603189,-0.2509224117,-1.7762050629,-1.6326947212,0.7337234616,-0.1040488109,0.8812249303,-0.0883731246,0.2676708996,2.1235263348,1.3968490362,-0.4328272641,0.3749687374,0.4944454432,0.7613911629,0.0710088089,-0.4935318530,-0.0036228173,-0.4802871048,1.6833672523,1.2407262325,-0.2036150247,0.4282922745,-0.1654592603,1.1932411194,1.0488804579,0.5686108470,0.8712642789,0.6605708003,1.1740618944,0.5311313868,0.1519005299,-0.5772256255,-1.5717507601,-0.0278483797],[-0.7410555482,0.0600090660,1.1404883862,0.1728246808,-0.4150016606,-0.8531286120,-1.4301352501,1.3328052759,-1.7766909599,-0.9347830415,-2.3132019043,-0.3161415756,-0.3422845602,-0.4042944312,-0.0631299019,-0.8212651014,-0.9136556387,1.8178263903,-0.3340629339,0.9076586366,-0.8367710710,1.6127285957,1.5141820908,0.2310186774,-1.0995316505,0.0870013833,0.0473044999,0.2396239191,-0.9782206416,-1.5230001211,0.1623630375,-0.0102913165,0.0020750219,1.0268006325,-1.4751604795,1.0106936693,-0.7432275414,-0.3952220678,-0.8257793784,0.0896198601,-1.9058178663,-0.5680857301,-0.5157565475,1.2639302015,0.1506981403,0.6955183148,0.0059388145,-1.0489003658,0.9072048664,-0.8454413414,-0.5262433290,0.1820997745,0.9455388188,-0.2013845444,1.5105247498,-0.5714784265,0.6655892730,0.0036163009,1.5466718674,0.2144060135,-1.8773127794,1.0883351564,-0.0815485120,-0.5530619025,1.2229647636,-0.3313086331,0.5999867320,-0.7683833241,-0.8361301422,1.8105818033,-0.7870327234,-0.5847709179,-1.7083207369,1.6299822330,0.3979983330,0.2377796322,0.9751384258,-1.3293365240,-0.5410467982,-0.0914377347,-1.5484710932,1.3114271164,-0.0184290502,-0.3232886493,0.2362254858,-0.7525823116,0.0451130047,3.4275386333,0.6046820283,1.6683111191,-0.3550831079,-0.7515689731,0.3097035885,-1.3417049646,-2.3069577217,0.7315924764,0.6413381696,0.8338512182,0.0281698741,1.9783726931,-0.0873281881,-0.5539647341,-3.0064988136,-0.0471658669,0.8318777680,0.0068611987,1.1242216825,2.2948811054,-0.1733502150,1.2312535048,-1.5858526230,1.0313191414,0.0634904802,-0.2213905007,-0.1633989215,-0.1563034654,-0.3088029027,0.1986729652,-0.1742921323,-1.1557924747,0.4170538783,-0.6078679562,1.0479866266,-0.0338269658,0.1270239502,-2.0492320061,-1.2566801310,0.9396144152,-0.7338167429,-0.5324376822,-0.2779399753,1.3637425900,0.3741379976,1.3102645874,-0.2677477896,0.2131762654,-1.2032434940,1.1780312061,0.1086482033,0.0441290997,0.3383155465,1.4467921257,-0.2144951075,1.6630389690,-0.8515225649,0.4221846163,2.0092184544,-0.4898147285,0.2452558577,0.8775050640,-0.1378996968,-1.5003532171,-1.0559593439,0.5809326172,0.8915153146,0.7845553160,1.1464320421,0.0719851926,0.2082331777,-1.5188686848,0.3173289597,0.6126807928,-0.5832113028,0.6440017223,-1.8158888817,0.7510995865,0.3002843261,2.1106085777,1.4130855799,1.5069802999,0.8173971176,0.6466156244,-1.1816313267,-0.3350912929,1.8267284632,-1.4561644793,-0.4502818286,-1.4192340374,1.4509518147,-0.5657813549,1.5445343256,-0.4137624800,-0.5041320920,1.2785291672,0.9388317466,-2.7162802219,0.4511407912,0.6001668572,0.2098069340,-0.6576579809,0.0284084454,-0.3980614841,0.2113230228,-0.2023942620,-0.6219281554,0.1637704521,0.8024389148,0.2890059054,-0.5536423922,0.3362540305,1.0697923899,1.5954041481,1.2075525522,0.5373802185,-1.0091240406,-1.3655527830,-0.2023812085,-1.4091848135,-0.7847847342,-0.1701223105,-0.4842104316,-0.3279180527,-1.3280045986,0.2314667553,0.9965080619,-0.5481374860,0.7257553339,2.6627266407,-0.0918110311,0.6512100101,0.1967700869,0.9696237445,-1.7186498642,-1.0569567680],[0.1434639245,0.8869625926,0.1305240244,-1.6645731926,-0.8236132860,-0.7947061062,0.3889919519,-0.7620389462,-0.6808071136,1.0847475529,1.3353163004,-0.4132747948,0.4249026775,-1.8814837933,0.1983270645,1.1899780035,0.5267816782,0.1010608450,-0.3886429667,-0.6467919946,-0.1790824085,-1.5514411926,1.6104589701,0.5642106533,-0.1024370342,-0.6198047996,-0.0703396201,0.7977949381,1.0114479065,-0.9036909342,-0.9735287428,2.0782299042,1.1365928650,0.7085199356,-0.1386462450,0.9240203500,-1.2732331753,1.5317684412,-0.0357722230,0.7908614874,0.6462177634,-0.1315709054,-0.1753663570,1.2215831280,1.0264973640,-1.7722311020,-1.6924057007,-0.9462206364,-0.8935453892,-1.1185258627,0.2721106410,-0.4637044370,1.2061246634,1.4528777599,-0.0286832377,1.6834579706,0.0242143106,-0.4347906411,0.0647857413,0.9448599815,-1.6144609451,-0.2085993290,0.2974056602,0.3630846739,-0.3684363365,0.4887857437,0.2921216190,-0.5919080973,2.1815986633,0.4395501912,-0.3311833441,-0.5717190504,1.0294089317,0.1020597890,2.5481126308,-0.4359241128,-1.2426069975,-0.0276984591,0.1750674546,-2.1184506416,-0.3091685176,-0.3684154451,-0.3687635362,-0.6302257180,-1.3431925774,0.7580380440,-0.5838408470,-1.0237014294,-0.7599342465,-0.4723232388,0.1086471230,0.6683390141,-0.9531794786,-0.4792973995,-1.3455077410,-3.3922998905,0.1557939351,1.5200035572,0.5220832825,-0.5070599318,0.0964791402,-1.1748200655,-0.1222923547,-0.4277092516,-0.8527142406,0.4056522250,2.5998671055,1.6654495001,-0.0720730200,0.8841146827,0.8627074361,-0.6475380063,0.6439040899,-1.4409921169,-0.8052983880,0.2387528718,-0.4147876501,1.7564786673,0.6480404139,-0.3820381165,-0.4705797136,0.1869706064,-1.0555312634,0.5956119299,-1.3753019571,0.6230102181,-0.1645947248,0.4146135151,-1.0125858784,0.2449852079,2.4123477936,-0.4572167397,0.3173998594,1.5055669546,0.7617041469,0.4318854809,-1.0136892796,-1.2775883675,0.0534324422,-0.4632358551,-0.0190582145,0.2056566775,-0.6764278412,0.4941030145,1.8585561514,-1.0093410015,-0.4695463479,-0.0496106595,1.1404596567,-1.1863820553,-1.0651481152,-2.1636610031,-0.4403622150,0.6801457405,1.0652247667,0.3571536541,-0.6009569764,0.7064716220,0.2043185979,-1.9207055569,-1.2280948162,1.5118652582,0.3222050965,-1.3747943640,0.8199530840,1.0614349842,-0.4350340366,0.6576821208,-3.7401006222,0.9735767841,1.1751554012,-1.1247026920,0.2820853889,-0.3381205499,-0.1025294811,-0.4248804450,-1.3322954178,1.8904037476,-0.3103108406,0.1047550440,-1.0094006062,-1.0368671417,0.4125984013,0.5263921022,0.8779241443,1.1037739515,-0.2102075368,-0.4442030787,0.7468138337,-0.6374391913,0.8717585206,0.3745002747,1.1550264359,0.6703916788,-1.0544458628,-0.8656336665,0.7324852943,1.9070558548,-1.3228117228,0.0232116040,0.2816745639,-1.5257774591,0.4781250060,-0.0931227952,-2.0965573788,1.6217279434,-0.8632082343,-1.2825033665,0.4201416075,0.5574867725,0.7364113927,-0.3860033751,-0.0109143378,-0.7308067679,-1.3101973534,1.0791306496,-0.1027626842,-0.1823142618,-1.9992675781,-0.1783711016,-0.8424944878,-0.1746136546,-0.2192441523,-0.4464647770],[0.9388386607,0.4470541477,1.1271544695,-1.3248273134,-0.6489559412,-0.0402808189,-0.4066389799,-0.0792577267,-1.1821033955,-0.7161780596,-1.6415539980,-0.8900255561,0.6941767335,-0.2142068893,1.5057532787,-0.5955338478,0.1190710813,-1.2132523060,2.6006717682,-0.1786205918,0.8296298385,0.4133850336,-0.5838788152,-1.3309012651,0.1561431289,-0.5567897558,-0.1555043012,0.6513020396,0.0782411546,0.3771162927,0.1500465721,-1.4672492743,1.3960622549,1.1758522987,-1.1361649036,0.5053006411,-0.6620242596,-0.7469163537,-0.0048416615,1.7476682663,1.0579575300,0.6052213311,-1.1506056786,2.5544493198,0.8737310171,-2.3488373756,0.3994743228,-0.4886947274,0.4099823534,0.4006403685,-0.9185190797,1.8258857727,0.1997846216,0.9413478971,1.3514236212,-0.7381575704,-0.9117684960,1.1219073534,1.3928374052,-1.3770185709,2.0112431049,-0.2355033159,0.6917845011,0.5643882155,-0.9713423252,-0.8640481234,-2.0835924149,-1.1511501074,-1.4824759960,0.0401905142,1.3694021702,-0.0271449313,0.3388541639,0.7780035138,0.6797094345,-0.3858315051,-1.4633450508,-0.4298055172,0.0629593581,-0.8716452122,0.3619607091,-0.2927120924,0.6218215227,-0.8032394648,-0.9219676852,1.7740563154,0.0287562385,0.5529638529,-1.0984222889,-0.3772644699,0.6821694970,1.5656158924,-0.7244850993,-0.8029174209,-0.0226689409,-1.5243951082,-0.0301333070,-0.0647283867,0.7247491479,1.4146097898,0.5698443055,0.7415510416,0.0522789508,-0.3597445786,-1.9959769249,-0.8862208128,0.2172666788,-1.6455937624,0.2428898215,-0.4008347094,-1.0215598345,-0.4700243175,0.7287815213,0.8855010867,-1.9370213747,-0.1494840086,0.9138846397,-0.2578948736,0.1088152677,-1.4954109192,-0.4800336659,1.8287754059,-0.7880680561,-1.4406323433,0.1494717598,0.7886293530,1.1938104630,-0.5177226663,0.2224755734,0.5443548560,0.6492105126,-0.5472027659,1.7127249241,-0.6872970462,0.7078721523,-0.0219112355,-0.5872185826,-0.6428512931,-0.5863469243,-0.4468710721,-1.0188856125,0.6974096894,-0.7035152912,-0.6150208712,0.4886905849,-0.1079619452,-1.4219036102,-0.9360095263,-0.1965572387,-0.5749878287,0.7504825592,-0.7644020319,-0.9671270847,-1.0105462074,0.4066572487,0.4834717214,-1.6724445820,0.6220752001,0.8609732985,-1.6909977198,-0.6904314160,1.4288923740,1.0061017275,0.0247926600,0.5012493730,2.1120195389,0.5027969480,-1.2208088636,1.3649389744,-0.8709388971,0.9939022064,0.6562706828,0.8895135522,1.5409330130,-1.4659143686,-0.0695885569,1.9460494518,0.9763817191,0.1771583408,-1.0231730938,0.1067204922,-0.9118813276,-1.4683669806,0.5764787197,0.0653056055,-0.7735127807,0.3949481845,-0.5038898587,1.7795591354,-0.0305724442,1.5770882368,-0.8128020763,0.6133491397,1.8436999321,0.2710909843,1.1364476681,-1.7383319139,0.7071347237,0.0303861313,0.7650019526,0.8676652312,-2.2562501431,-0.4436027408,-0.6700232625,0.1521641910,-1.9405333996,-1.0905086994,1.0019207001,0.1768924445,-1.0880144835,-0.2532173693,1.0982730389,-1.8395668268,-0.2114286125,-0.2296632379,0.1869794130,0.5037794709,1.9103424549,0.5537812114,-0.5874814391,1.2579499483,-0.8586683869,0.4361870885,1.5714631081],[1.0773148537,0.8110896945,-2.2315375805,-0.1010025144,-0.5873750448,1.3248683214,0.8406484723,0.2611061931,0.7944416404,-0.6496164799,0.6342844963,0.0950026661,-1.6832067966,0.3440461457,0.7071580291,1.1934145689,0.5273885131,1.0067039728,-1.7323273420,-0.3734121621,-0.1425103992,-0.3297423720,-0.0890421197,-0.5773962736,0.7361654639,-0.9912055731,0.1251746118,0.0731527135,0.1439372301,-0.9477243423,1.3992298841,-0.2261237204,-1.4388542175,0.8013010025,-0.0033314459,-0.0969415605,-0.0958714485,0.3954369724,-0.0532430410,-0.7734996080,-1.4191855192,0.3034519255,-1.5182067156,1.1197077036,-0.9538609385,-0.8496140242,-0.9818996787,-1.3630776405,-0.7725985646,-0.2836254537,-2.3276040554,-2.4452273846,-0.7158649564,0.8833968639,-1.3004398346,-0.0763390809,1.4305567741,-1.3234086037,-0.4383561611,-0.7431524992,0.8919675946,0.4638727605,0.6176608205,2.4964170456,1.6294752359,-0.0990447029,-0.2019919604,-1.4488258362,-1.7141647339,-0.0496415719,-1.2993954420,0.6253554225,-0.7917193174,-0.5829433799,-1.5526804924,2.1101534367,0.7588295341,-0.7099302411,0.1511470377,1.3230912685,-0.9278250933,1.9065986872,-1.0321749449,-0.1773614883,-1.6503783464,-2.5385110378,1.0100908279,0.0857020915,-1.7338609695,-1.6406011581,1.1453614235,-0.1505951136,1.4314432144,0.6365867853,-0.0665628463,0.0323299803,-0.5550736189,0.0977861062,-0.0609849803,0.8375166655,-0.1634184420,0.8235554695,0.9206323028,0.1807626039,-1.3129683733,-0.1604766995,-1.9060746431,-1.2066216469,0.7304183245,0.4948927760,-0.0032088785,-0.3024331629,-0.7394009233,-0.5128120780,0.9652515054,0.4766792655,-1.3712165356,0.1988528371,0.1399628371,1.6486734152,-1.7575517893,-0.7831295133,0.9736258388,-1.1109322309,2.3856215477,-1.1789442301,0.0291221458,0.5595475435,0.8810371161,0.7152084708,-0.4620775282,0.9132069945,-0.7546525002,-0.5349771380,0.4566424489,1.5095769167,-0.2281712443,-0.8903415203,1.2097718716,-1.2648000717,1.8381814957,-0.9840826988,0.6409484744,0.9266912341,0.7850340605,0.2270026356,0.0495295525,-1.6531804800,-0.7808196545,0.7246446609,0.6633691788,-1.0378812551,0.3469790220,0.2520309985,1.7509188652,-0.4184011519,-0.5198572874,-0.9234429598,-0.9992784858,0.3749483526,-0.7043411136,1.0747038126,-0.6272990704,1.5339956284,0.4177254438,0.2583836317,-1.1504285336,0.3291141987,0.0455219783,0.6320825815,-0.5108472705,-1.4536298513,0.2752215862,0.1397872418,0.2438923568,1.0056501627,-0.9396406412,-2.3817501068,0.4750269949,0.4059125185,-0.4770562947,0.1705982089,-1.0477807522,-2.1061971188,-1.6929115057,0.0423611403,1.3827104568,-0.3895183802,0.8139379025,-0.5943319798,-0.0554390214,0.7965607047,0.1331798881,-0.5416783690,-0.8653023839,-0.0925281271,1.1821020842,-1.5706546307,0.8593307734,0.2836470008,-0.9691122770,-0.0183737297,-0.2040354460,-0.9477403760,-0.5394350886,-1.2562873363,-2.0715236664,0.1512364447,1.0444895029,1.6333490610,-1.1113785505,2.1473650932,1.5263067484,1.4234750271,-0.7856664062,-0.5622516274,-1.9383574724,0.1911592335,-0.3936029375,0.1617906839,-0.8345180154,0.6728704572,0.3431881070,-1.1441230774],[-0.0458879471,0.2846845090,2.0084414482,0.0957808495,-0.9404090047,-0.3162630498,-0.0312234666,-0.1335952878,-1.8414719105,-0.3315768838,-0.6933088899,-0.2605129480,2.1209623814,-0.8322905898,1.4379409552,1.1606181860,0.6834978461,0.0031104183,0.6543934345,-0.4499197900,-0.5464591384,-0.7478616834,0.2739015818,-0.2097705752,-0.2395850718,1.4202288389,-0.7047485113,0.7353649139,-0.5219275951,-1.5921950340,-1.4259504080,-0.4915523827,0.6296114922,0.6417863965,-0.2406871468,0.6184217334,0.0762631074,-0.2614903152,0.8547312617,1.1878938675,1.0160274506,-0.3068729639,0.5677075982,-2.1292457581,0.1950669289,0.3610023558,0.1519751996,-0.2222727239,1.3047732115,-0.0932376981,-0.1395379305,-0.2409367412,1.0341054201,-0.2924596369,-0.8343493938,-0.1087585390,1.7077137232,-0.3006861210,0.6157716513,-0.2781107426,-0.2767468095,-0.5608413815,-1.2163040638,-0.1097816303,0.7184557319,1.5761929750,0.4418694973,-0.8168610930,0.7455046177,0.4540291727,1.3983632326,2.0896103382,1.2146077156,-0.3927581906,-0.1592295468,1.1579405069,-0.5076931119,-0.1404834688,0.6343402267,1.0706061125,0.2229107618,-2.8925197124,0.3393659592,-0.3120492101,-0.9759209752,0.0241301656,1.1204642057,-1.1298772097,1.5899230242,0.9787300825,0.9341636300,-0.8147085905,-0.2512424588,-0.3828348815,0.0003409579,-0.0622048005,-0.3173488677,-0.1779273450,-0.2116203159,0.2674838603,-1.6518170834,0.4865783453,1.6972717047,1.0725008249,-0.9458972216,1.0446079969,2.2896277905,-0.2425562590,0.7210319042,2.2993803024,1.0302956104,1.5507205725,-1.0952255726,0.2356208563,0.3919691145,0.5642871261,0.7895075679,0.1561750621,-1.3123941422,1.0662723780,0.3788603246,-1.1786166430,0.7589958310,0.5529800057,0.5370345712,-0.2868351936,1.6344056129,0.5712266564,-0.7899540663,-1.5427579880,-0.7673321962,-2.3696231842,-0.2605598569,0.1289570481,-1.2076663971,-0.5454435945,1.5532826185,0.1072913334,1.5353435278,-0.1849514097,-0.1767827123,-2.1992540359,0.7292887568,1.1005582809,-0.3423680365,-0.3103042245,0.8130363822,-0.7362970710,-0.7823243141,0.3170981705,-0.6062513590,0.6110578775,-0.9203095436,1.5095089674,0.6564636827,0.0658511817,-0.8736306429,-1.0577539206,-0.8972370028,-1.4322032928,1.3404302597,0.3420748413,1.0706282854,1.7390650511,1.3066060543,0.4496167898,-0.4110504389,0.7492995262,0.2111377716,-0.2606511414,-0.5151983500,1.0792167187,1.4774627686,-0.2751087248,-0.4370077252,0.6981697679,-0.6351764202,0.6036766768,1.7369296551,-0.4178344309,0.0245132316,1.9896930456,-0.3731683195,0.1410367340,-0.9418154955,-0.7028346062,-1.3788369894,2.9252495766,0.5717520118,-0.0458755493,-1.0570642948,-1.8246930838,0.2417382896,-0.1451518238,0.5019926429,0.6923288703,1.0122750998,-0.0421771109,0.6220104694,-0.4075361192,-0.8514463902,-0.3381832540,0.7689677477,0.6064976454,-0.0126131419,0.3224277198,1.6075083017,-1.8024101257,-1.2550679445,-0.0130337598,0.7003573179,-0.2152732313,0.9030884504,-3.0074372292,-2.3304672241,-0.5678032637,2.6673221588,-0.0000683526,-2.2414488792,-0.3600297272,0.6174938679,0.9622223973,0.4705488980,-0.2358042002],[0.6787893176,1.1536179781,-0.5547327995,-0.1236007586,-0.1360976994,-1.7055424452,-1.2570964098,-2.3186249733,0.0843379274,-0.2976732552,-0.3390328884,-0.0344037488,0.5192691684,-0.6725831032,1.1736112833,-1.5922294855,-0.0136492010,-0.0983454883,0.8498770595,-0.4949856997,1.0687308311,-0.2338664085,-0.1018051058,-0.1490771621,0.6980962157,0.8531481624,-0.6047407985,-1.4088909626,-0.9811016321,0.1651830673,-0.0166458562,0.1409326643,-0.7251721025,0.7957408428,-0.1764385402,-1.6334345341,0.1553497761,0.5474259853,0.1437603086,0.1302209795,0.2271719575,-1.0537291765,1.5317530632,-0.3264935911,1.4668451548,-0.5541288853,0.9666754007,0.2401796430,0.1003144309,-0.9387914538,1.2058185339,0.4189720154,-0.0902500674,1.2073749304,0.0885801986,2.2098054886,1.0166771412,-0.3307090700,1.3583818674,0.0608563833,2.0058913231,0.0816583633,-0.0792093650,0.0939683020,0.3995066881,1.5493112803,0.2928569317,-0.1067149267,-0.4934600294,0.7856813073,-0.5543451309,-1.1688302755,-0.9422464371,-0.4649276137,-0.1244343743,1.0531721115,-0.5772435665,1.1971284151,-1.6377062798,2.6346027851,-1.5864238739,0.1980870366,-0.2175800204,-0.7853147388,0.6764087081,1.0572141409,-0.1721861064,1.2093849182,0.0640576929,-1.0637799501,-0.0791875049,-0.5274960399,0.4602147639,-0.3991668820,-0.0741831958,-1.0853574276,1.0693564415,-0.3556284308,-1.0694770813,2.6314742565,-1.4129743576,0.5074303150,-2.1636655331,0.3085227609,0.1881135255,-0.2523779571,-1.3635439873,-0.8585309386,-0.8065607548,-1.1741236448,-0.9480458498,1.4351259470,0.3912090063,1.3308275938,-0.9888421297,-0.4313032329,-1.7370495796,-1.1904556751,-0.2911131382,-0.4206063449,1.7773537636,0.1080465764,-0.2650998831,0.4594625831,0.0268785525,-1.0857034922,-0.6268877387,1.3989568949,1.3037877083,0.2572353184,1.0194852352,1.6428092718,-0.4417257011,0.1843148470,0.2210908979,-0.8268135190,0.4464316368,-1.1436780691,0.1022599861,-0.2689138949,0.4295471311,-1.3548612595,-0.5210994482,1.8831641674,-0.1348460466,0.0681441575,-0.6695713997,0.0453709252,0.3797713816,-0.4755639732,-0.2494962960,-0.1664597243,0.2957374454,1.1305173635,-0.6693257093,0.8059806824,0.8046057224,0.2548323572,1.0816518068,0.0990683660,0.1078162864,-0.1735524535,-0.2703637779,-0.2104981542,-1.8608211279,0.6002736688,-0.3222564161,-1.4688658714,0.9703220129,-0.9755422473,0.8594921231,-0.7415559888,0.9588875771,1.8989210129,-1.5950626135,-0.2549622059,0.9568085670,0.2796890736,0.2816965282,0.9156034589,-0.7888602614,-1.2830469608,-0.7138097882,1.3335521221,-0.7639064789,-1.5349581242,1.1567809582,-0.5121814013,-0.4490087628,-1.3938723803,0.5303887129,0.5679660439,-0.5504558682,-1.1937325001,-1.8920999765,0.5680583119,2.5964953899,0.4043229222,0.6083219051,0.1465612203,0.3495765924,-0.2330164611,0.2941886485,-0.8010068536,-1.2406933308,0.1112386584,0.0981622413,-1.1744223833,-0.5514314175,-0.3500730395,0.7157834768,0.1586675495,-0.1676100791,-1.5922158957,0.2704105675,0.5291992426,2.1981124878,0.4463334382,0.9241107106,0.9336605668,-1.8127021790,-0.1246180162,-0.1696453393,-0.4599695206],[0.8565337658,1.6267207861,-0.9244696498,-0.1559941918,0.4249975085,-0.7795513868,-2.1165361404,-0.2150630206,-1.1040906906,-1.9384589195,0.9018970132,1.6354812384,0.5289262533,0.4159639180,0.4340830147,-0.0776079670,0.4491689801,-0.1313019097,-0.7509349585,1.2776517868,0.7487615347,2.6572730541,-0.8186072707,-1.4451512098,1.3417631388,-0.1051895544,1.3567872047,2.0140140057,-1.3726296425,-0.8574283719,-1.6812884808,-0.1505986601,-0.7210639715,-1.9408824444,0.0999050587,1.2818363905,-0.9700794220,1.2453765869,0.6968713999,-0.0645520166,-1.0578403473,-0.4692222774,2.1283955574,-0.3892408013,0.5696129203,-0.2742994726,-1.0984168053,0.6897208691,1.6492482424,1.0651326180,1.6090450287,0.1430166811,-0.0025570951,0.5276804566,-0.2226212621,-0.4914850295,-1.5938395262,1.0183731318,0.0496852174,-1.6018487215,-1.8184107542,1.7716566324,1.0756032467,0.3834027946,-1.4669975042,-0.2647817731,-1.0320789814,-0.3644350767,-0.2168742120,-0.7252820730,-1.3937695026,0.7387053370,0.2493469417,0.5485517979,1.1348981857,1.1029473543,1.6632301807,0.9258661866,0.4591159523,-0.4123454392,-1.2897313833,0.5902590156,-0.2920606136,1.9976263046,-0.5045529604,0.2800785005,0.2846641541,-1.4732339382,-0.7650971413,-1.9866602421,0.2745036185,0.1866807640,0.7437629700,-0.7251886725,-1.0595256090,-0.3219626844,0.4132559597,-1.0829941034,0.3818486631,0.4902530909,0.8203164935,-0.3574656546,0.0871377513,1.2285019159,-0.6999685168,-0.0382436216,2.0060265064,0.3040466607,0.9432737231,0.3004783690,1.0069338083,0.4846802950,-0.2194198221,-0.2402843833,-0.2284196168,0.6577883959,0.0243720971,-0.0167382881,0.0804451779,0.2744990885,1.5632476807,-0.9265797138,1.0785713196,-0.4262202978,-0.3350518346,-0.1482925713,-1.3545845747,-0.6348443031,-0.1358720660,-0.5967708230,1.2970815897,-0.8241472840,-0.1672525704,-0.6040331125,1.2512923479,-1.3428666592,0.4152507782,1.1968697309,1.4077038765,0.1768872589,1.7283319235,-0.9516835809,-0.5515766740,-0.5391194224,0.2572713494,-2.1139318943,0.9485329390,1.2312760353,-0.2881865203,1.2581969500,0.5737562180,-0.9768746495,-0.6910295486,-0.5072914362,0.3287530839,0.9405872822,0.6720070243,-0.6202755570,0.1128496304,-0.6132038236,1.5822707415,-0.3844380975,-0.0673749968,-0.2834100127,-1.1995179653,-0.0790346786,0.8218730092,-0.1707502753,-1.6510307789,-0.2086256593,-0.7109470963,0.0523468554,0.9653596878,-0.7002084851,-0.1555075496,0.0463614166,-1.4276753664,0.3529338241,-0.2941586375,-0.9563120008,-1.3457732201,-0.0424981080,0.8205020428,0.4909951687,-1.1780422926,-0.3410792053,0.4586801231,1.1928086281,1.5076155663,0.2159299105,1.0762003660,-1.3993560076,0.5086227655,1.5924257040,-0.0414426811,1.3759416342,1.3556041718,0.6253252029,1.6755017042,-0.2901961207,-0.2289954126,1.5246464014,0.3891948462,-1.2429443598,0.0270989761,2.9799761772,-2.3688035011,1.3221940994,-0.5086752772,0.5284387469,0.8252167106,-1.3484315872,-1.5581241846,-0.8698654771,-0.0978383347,-1.6825011969,0.6562595367,0.5922316909,0.2983350754,0.2247846574,-1.3188779354,-0.3159044385,0.0416775160,0.7735174298],[0.1309816986,1.6671591997,0.6244473457,0.0545613840,0.4523994327,1.6147964001,-1.1268408298,-0.3600841165,-0.3496910334,-0.6707585454,-0.9565799832,0.5979391932,-0.4567743838,0.6110959649,-0.4604537189,-1.2620191574,1.1980472803,1.7984120846,0.2246788591,0.5230852962,1.7367670536,0.5086534619,1.1650388241,0.7131203413,1.3199769258,0.3111094236,0.2989323735,0.6086726785,-1.3495463133,-0.8733163476,1.3610419035,1.4206547737,0.4303741753,-0.9519916773,1.2027678490,0.0799262673,-1.1702611446,0.6195428371,0.8924890161,0.8761147857,-0.3095809221,-0.5196042657,-1.3068716526,1.5195988417,0.2128613889,1.5664979219,-0.2589054108,1.7114889622,-1.8208161592,0.1634945124,-0.8131170273,-0.6053546071,-1.3275238276,-0.6441715956,1.9088834524,-0.5635452867,1.0824725628,-1.9519108534,2.4412162304,-0.0172850862,0.9122820497,1.2396584749,-0.5733674169,0.4248894751,-0.2712600231,-0.6835675240,-1.5374375582,-0.1013744250,0.7466657162,0.9291818142,0.2294180095,0.4144058824,0.3097238243,-0.7374562025,-1.5369198322,-0.5622548461,-1.5995111465,0.8243899345,2.4914863110,1.0118551254,-0.2812379301,0.0167065077,1.1539196968,-0.6009150147,0.1329372674,-2.1699743271,-1.1165589094,-0.5693680048,0.6362667084,-0.7676492333,0.6184794903,0.1831612140,1.8593512774,-1.0755146742,-2.0192196369,-2.4374644756,-0.1485635042,1.1511514187,0.1853233874,0.6584609151,-0.0495361015,0.2373988479,-1.5115803480,1.9323703051,0.4824220240,-1.3157830238,-1.9636150599,0.5330681801,-0.1093232185,-0.2831834257,1.4337438345,-0.5686172247,-0.8498464823,0.7244464755,0.6058983207,-1.2902585268,0.7894331217,1.9609137774,-0.3355398178,-0.6076121926,-0.7272418737,0.8514086604,0.8884165883,-1.1113233566,1.1378443241,1.2755492926,0.2945994139,-2.1856794357,0.7923774719,-1.6250413656,0.7398307323,1.0142691135,0.9582080841,-2.6153709888,-0.7281308174,0.7424984574,-0.0821439624,-1.5249018669,-0.4381790459,-0.0053916587,-1.7120848894,1.7055863142,0.8111898899,0.1493609101,-0.7354556918,-0.3129665256,-0.6534162164,-0.9912257195,0.1489663869,1.1033235788,-0.1900487989,-0.4122638702,0.1786202043,0.7816811800,-0.5246403813,-1.3810967207,0.9454761744,-1.3616911173,0.5349876285,-0.0959592983,-0.8964068890,1.3871814013,0.9973930717,0.5764676332,0.4400008321,0.4754700959,-0.9835773110,-0.5619937181,-1.6410032511,0.2922827303,1.4331048727,-0.3837113082,0.2374925911,-0.5327551961,-0.4711798429,-0.3400845230,-1.4632056952,1.1385982037,-0.0050248299,0.9715877175,0.3023412228,-0.8062357306,-0.3169119358,-1.5868651867,0.7886871099,0.1372247785,-0.7851805091,-1.5065702200,-0.5764098167,-1.1072884798,0.1352825314,-1.0527776480,2.0522935390,0.9853089452,-0.3728401065,-1.4862358570,-0.1116613746,-1.9596153498,1.0607012510,0.7912979722,1.4486447573,-0.5566545129,0.8055643439,1.3840409517,-0.4356646836,-0.7338418365,1.3481131792,0.1454764307,0.4252978861,-0.0880398229,-1.2484735250,1.1045972109,-2.0293858051,0.9979759455,-0.2792384624,-0.1876324564,-0.5325407982,0.5268504620,0.4087450206,1.6114984751,-0.1840722412,-0.1820611060,-0.3537951708,0.1039311066],[-0.4489805400,1.1552664042,1.3704001904,-1.9223861694,-1.2888606787,-1.0187835693,0.6616163254,-0.8920385242,0.5518894196,1.3713300228,0.7708854675,-1.0448182821,-1.1999044418,-2.6680612564,0.4900120795,-1.2550389767,0.0423943661,0.2307881862,-1.3097743988,0.9433927536,-1.0359666348,-0.5941559672,-1.1822756529,0.0490396470,-0.6646436453,2.2700588703,-0.6435940862,0.1863738149,-0.2803147733,0.8866586089,0.0039090207,-1.4617713690,0.1988759339,0.4399749041,-1.2791353464,-1.0427937508,-0.6556071043,0.2571541965,-0.3568092585,-2.2522468567,-0.7602754831,1.1351559162,0.6326811314,0.6538631916,-0.0560684986,2.1729958057,-0.0687307790,-0.2313192934,-0.4723461866,-0.3986878395,0.7822806239,-2.0840001106,-1.1852180958,1.4007776976,-0.7757635117,-1.1275815964,-0.1672484130,-0.7482985854,0.2928707302,-1.1963318586,1.4991011620,-1.3109302521,-0.0839465857,0.0558208153,0.2463919818,-0.0606065616,-0.0057793492,-0.0700023621,1.3056834936,-0.1253987551,0.9635531902,-0.7322648168,-0.0024187912,-0.1301703900,1.0004709959,-0.5866019130,-0.6917225122,-0.1560804099,-0.0212033875,-2.1023504734,-0.8584808111,1.1101720333,-0.2545737624,-0.2416257262,0.5702453852,0.5405352712,1.7392754555,0.6312438250,-0.4402846098,0.0550365932,0.7741043568,0.9616689086,-0.9026238322,0.6697832346,1.7671315670,-0.1734208614,0.0113102626,-0.9254624844,-1.0618740320,0.0115344757,0.7880923748,-0.2987428606,1.1056511402,0.9078025818,-0.3661470115,0.7300938368,-0.1293408722,0.0929499343,1.0738584995,2.4386839867,-1.3962153196,-0.3303312063,1.1602600813,-0.2442411035,-0.4293623567,1.6676892042,-0.0608364716,0.7872980833,-0.3093358874,-1.5586376190,-0.5111013651,0.6837660074,0.1929822713,-0.7978610992,2.0638716221,0.1178953871,-0.9953600168,1.2995123863,0.9166028500,-0.5599343181,0.6541744471,0.7905471325,0.4623413086,0.1298543960,-1.7680779696,-0.3233806193,-0.0529145338,-0.5833628178,-0.3562459648,0.7015071511,-0.4987028837,-1.6961719990,-0.6258285046,-0.5355837345,-0.1618913412,-1.2267602682,-0.3106380999,-0.8768603802,0.7795471549,-0.1935388297,0.5475685596,0.2768119276,-1.2845706940,-0.0423595309,1.6698843241,-1.6108504534,0.9478467703,0.0986463651,-1.5106642246,0.5734161735,1.0034544468,-0.0609908327,0.3071651459,1.8223924637,-0.6054177284,1.8280153275,1.6417883635,-1.3252933025,0.1235409901,0.0669323280,1.2934104204,1.1690789461,1.5101593733,-1.6193233728,1.3771556616,0.5679165125,-1.8386728764,1.3130825758,-1.1070921421,0.7729588747,-0.5860818624,-0.2058396339,0.4436039925,-0.2096983939,-1.4521727562,0.3058028519,-1.1525199413,0.5443555713,-1.2522698641,0.1393335462,0.3642078936,2.0137174129,-1.9375973940,0.3161602020,-1.2603361607,0.4786128998,-0.1122477651,0.7691808939,1.4161857367,-0.3374640048,-0.0133701405,0.9041422606,-0.5291411281,0.4031642973,0.8388589621,-1.0761878490,-1.5249725580,0.1757129133,0.8767262697,-1.5049668550,0.4910938144,-0.3653927147,-0.1716035455,1.0316590071,0.1405045390,-0.8550847769,-0.0122549757,-0.2336222380,-0.2811496854,1.1793352365,-0.6546878815,-0.7931123972,0.2504535615,0.6389125586],[-0.1338374466,-1.2315516472,0.1087464318,2.4242548943,0.5663132668,-1.6622200012,-0.9516454935,0.0935006887,0.5418270230,-0.6956735849,2.1641728878,0.5378525257,0.3326438665,1.2569468021,0.2456189245,-1.9654610157,-0.3942227066,-0.6392866969,-0.3373064399,0.1140286326,0.6925070286,-0.7737633586,-0.7652257681,-0.0776397437,0.5756703019,0.5510339737,0.6002933979,1.1095582247,-0.3609852493,1.1201286316,0.3345442414,0.1578962952,1.8014444113,-0.3187448680,-0.7662656307,1.4651508331,-0.0448592342,1.1052986383,0.3984775245,-1.7524188757,0.2810229659,0.6246092319,-0.0725462735,0.6233510375,1.1921753883,1.4715322256,0.7919710279,-0.8447570801,0.1241444349,-0.7139037251,-0.4604227841,-0.4756816626,-0.0533989593,0.2218795717,2.0838932991,0.2475150228,1.7827460766,-0.0134890378,0.7679715157,-0.3554916978,0.0362120308,-0.7166433930,0.6927425265,-0.7791339755,-0.2974212468,0.2203360796,0.4349841774,0.5249035358,-0.4737904668,-1.3593201637,-2.2599756718,-0.8934810758,-0.6567152739,-0.6929903626,-0.8416537046,0.4606203735,-0.2749021649,-0.1371530890,-0.7862843871,1.6317793131,-0.2760257423,-2.3511528969,-0.7772335410,-1.3283784389,-1.4011971951,0.4626928866,2.3816864491,-1.2089858055,-0.4188674390,-0.7853214145,-2.1645514965,1.1445811987,-0.9505017996,0.7641232014,1.3194897175,-0.1203246713,-1.0543094873,-2.7589275837,0.6597554088,2.1434054375,-0.4508769512,0.4651074708,-0.3877440095,-0.6067355871,1.5365588665,0.1711568981,0.0135632185,0.1285218298,0.1237851009,-1.6126866341,-0.1699062586,1.7187911272,0.7952760458,-0.5621861815,0.1939502656,1.2603075504,0.3326231539,2.4959945679,0.3348725140,0.2440851331,-1.0814745426,-1.6712106466,1.0294514894,-0.3955523074,-0.7647819519,0.5640015006,-0.6139293313,0.7056366205,0.2410723269,-0.6252645254,0.2575959563,-1.4764999151,0.4396305084,-0.5134104490,-0.0369065218,1.2340645790,0.0310939327,-0.6546210647,0.9864290953,-0.0455574021,1.2032821178,-0.7458623052,-0.2978456020,-0.9822353125,-1.3218252659,0.1633568257,-0.6249672174,-0.7417123914,0.1362227798,-1.0594494343,-0.0809739381,-0.5245481730,1.8382658958,-0.8360715508,-1.3591579199,-0.5534932017,0.1547365636,-0.8778612614,-0.5545231104,-1.2758891582,-0.8472756147,0.3394901454,0.8867839575,0.3970008194,0.9629596472,-0.0797268972,0.6448428035,0.4046718478,-0.0212829933,-2.4381325245,-0.1657105386,0.9223551154,-0.8054891229,1.8176586628,0.8265794516,1.5185177326,1.7634532452,-2.1006796360,0.9573124051,0.2478752732,1.1825718880,0.9785193205,-0.5731644034,-0.9844460487,-0.9941863418,0.1693080962,-0.9317142963,-0.9543631673,-0.6126065254,0.3264756799,-0.8015695214,1.0485463142,-0.7815785408,1.3674834967,-1.0214259624,-0.2009140849,0.1025521085,-0.9636093974,-0.7346326113,0.2468834966,-0.4661616087,-0.5672651529,-0.4167720973,0.1828022152,-0.8394173384,1.6634211540,-1.8836456537,0.0390705056,-0.1487848461,-0.3561755717,-1.4013719559,0.6198907495,-1.6715638638,0.5220121145,-0.1027402878,-0.5389942527,-1.2783858776,-0.7213203311,-2.3513112068,0.6171727777,0.1380944103,0.5125414729,-1.2011198997,-0.5964149833],[0.5951352715,0.2339069694,1.1006313562,0.4247358143,0.3372429609,0.1389364749,1.4059437513,-0.4783534110,-0.6449186206,0.0131500335,-0.0374759957,-0.2436141372,0.8794044256,0.3872634768,-0.5042549968,2.1577014923,-0.5692856908,0.5889863968,-0.8519099355,-0.5406209826,-0.3512584567,1.8592058420,0.1071278453,0.1881214976,-1.4387773275,0.2278479040,-1.0301718712,-0.6761962175,0.4439955652,-0.4711591303,1.5134398937,1.2920711040,0.1850506514,0.0724349618,-0.1809644252,0.3238140941,0.4700443745,-0.2543262243,-1.1642212868,0.2567294538,0.1768714786,-0.6741221547,-0.0544025823,-0.8167592883,0.6026630402,0.1571673602,0.1848359108,0.7246237993,2.1916341782,0.0395292416,0.5885300040,1.0862364769,-0.0933451578,0.8974225521,1.2546049356,0.0124170966,2.8417673111,-1.7472499609,-0.7727107406,-1.1648753881,1.1488866806,0.2784065306,-0.7488279343,0.7987616062,-0.2916001379,2.6300568581,-0.5040947199,1.0890872478,-1.0070320368,0.5389986038,-0.4499021173,-1.3006299734,-2.0500237942,-0.2852370441,0.9818706512,0.0392058752,-1.4161587954,1.4106215239,0.4247548580,0.7822275162,0.4471131861,2.5369141102,0.1805239916,-0.8006153703,1.0798196793,-2.1828672886,0.2883101404,-0.3787364364,1.2829811573,0.0463522710,-0.7552426457,0.1391619742,-1.3081711531,1.8474140167,-0.2298999429,0.6889461875,-0.6292168498,0.4396181107,-0.0836468861,1.9265546799,-1.4717415571,-0.4133013189,-0.5089668632,-1.2310500145,-0.1285655946,0.8536369801,-0.7277516723,-0.0126291113,0.4544115961,0.5498104095,-0.5179764032,-2.4123616219,-0.4469912946,2.0694754124,-0.9394243360,-1.0521241426,0.2036963105,0.0071311919,-0.6870504022,-0.0155820046,-0.1647448540,0.3724843860,0.4417797029,-1.0829602480,0.6957066059,-0.2200885266,-0.6384267211,1.0450456142,-0.0275342409,-0.3599655032,-0.2629359663,0.5287644267,0.1829177290,0.6768316031,-1.8263455629,1.1303802729,-0.0842294767,0.5546888709,1.3973606825,1.3698474169,-1.3730745316,1.0151197910,0.2647194564,1.0315226316,-1.5784586668,-0.9907632470,0.2567261159,-0.8996278048,-0.3867754042,1.4097727537,2.2299740314,-0.1896184236,0.9524666071,0.1702094078,0.9398848414,0.5321618915,-1.2387745380,1.2894320488,-0.8582060337,-1.3007978201,1.1981871128,-0.3366500735,0.8419474363,0.5433722734,-0.9629876018,-1.6009552479,1.7573373318,0.6935952902,1.4654803276,-0.1797384024,1.2437579632,1.7125695944,-0.2490044981,-0.2686403692,0.0930381417,-0.0375618339,-2.6600220203,-0.9509128332,0.2806393504,0.2936806977,0.2567313313,-0.3014256954,-2.0332870483,0.1230432019,0.2137270868,-0.9075357318,0.3526922762,0.2563580275,-0.5407764316,-0.1236735359,0.8370244503,0.2930872738,-0.9313671589,1.5671607256,1.1615507603,-0.8167613149,-0.7182096839,-0.6333760023,0.5856393576,0.6780885458,0.5890313983,0.7123593092,-1.2848955393,-0.5609303117,-1.9114166498,0.8378915191,1.0157173872,1.0953134298,-0.4000841677,0.3343574703,0.3863209784,1.5166685581,-1.9124687910,0.7424203753,-0.3382374644,-0.3509818017,0.6303770542,0.4310825765,-0.8497053981,0.5221378803,0.4046062231,1.6502557993,0.2492208332,0.7841585279],[-1.4824200869,-0.4545456767,-0.9722738266,-0.0509347804,1.7074534893,-1.7254183292,-0.3090017140,-0.0453792103,-0.5058189034,1.3871597052,1.1207715273,1.0124270916,-0.3037402928,-0.1054088175,0.5237683654,0.1429743320,-0.8187479973,0.7458306551,-2.1103670597,1.0728235245,1.7312847376,0.3593048155,0.8765829206,-0.0282392837,-1.0374023914,1.3495305777,-0.8798107505,2.2148194313,-0.3535363674,0.0634836853,0.0064468719,0.4102476537,-0.5776757598,-0.3282352984,1.2123632431,-1.4314142466,-0.6823901534,-0.8864627481,-0.1065689325,0.0673301294,-0.2074949145,0.2912158370,0.9534795880,0.7978639603,-0.1701406687,0.2286783755,-0.8701025844,1.3222193718,0.1815055162,-2.4394092560,-0.3474123180,-0.9069854021,0.9881507158,-0.5943987966,-0.6776933074,-0.5054681301,-1.7560210228,-1.4633255005,0.5323156714,-0.8000277877,1.1683307886,-1.0447676182,0.0635707676,-0.3356823921,-1.1476477385,0.1429802775,-0.6482725739,0.0072258324,0.4359221756,-0.8878523707,2.4653964043,-0.5865577459,0.7874836326,-0.0772186443,0.0906665623,0.1741325259,-0.7149444222,-0.0434522107,0.0239767432,0.1674236953,-0.3820293546,-0.0326141678,-1.2816491127,-0.2897256017,-0.5946949720,0.0430568755,0.3749506772,0.0121182492,1.7634828091,-1.3510473967,1.1881501675,0.4915170074,0.0009339975,-1.8913590908,0.4536544681,-0.3723362088,0.2312607020,-1.4136934280,-0.5840505362,-0.1064425334,1.3762027025,-0.4251186252,-1.9075818062,0.2827370465,1.4952538013,-0.2975981832,-0.7461140156,0.3224439025,-1.4082685709,0.1460434049,-0.1548788100,-0.2577507496,-0.2077642530,-0.6747339964,-0.5161724687,-0.7427644134,0.2661053836,0.0695588514,-0.4860402048,0.5244890451,0.4730707407,1.7623431683,0.8991122842,-0.0184322987,-1.0504052639,0.2551665604,-1.1870924234,0.6593571305,-0.3136428595,1.2176970243,-1.0874655247,-0.0515389442,0.2061298043,-0.6265270710,1.3921668530,2.8257706165,1.0335572958,-0.8086478114,-1.0901274681,-0.3106675744,-0.1632628441,-0.4603413045,-1.3341411352,0.5154522061,-0.4899453223,0.3032837510,2.4529964924,-1.8736343384,-0.0681086928,0.0572270155,0.0044212798,-0.0653590336,0.2338360548,-1.1391998529,-0.0103998687,-0.5209394097,-1.0892212391,-0.3462033272,-1.8454072475,-2.5989222527,0.1982216686,-0.4841802418,0.2070287764,0.7549870610,1.3594202995,0.6960720420,0.6822013855,-0.0112141585,1.3447604179,-0.8314924240,-0.4079718292,-1.3308038712,0.3525986075,-0.5378847718,0.3934444189,0.2865182757,2.0425362587,-0.9194611907,0.1146700308,-0.1374237090,1.3655269146,-0.2616383433,-1.2862409353,-1.6546574831,0.0372327417,1.2703013420,-0.1432766169,0.7952552438,-0.0720724761,-0.7424240112,1.2256538868,-2.2053880692,-0.5306376219,0.1682143807,-0.9688461423,0.3096540272,0.9401078820,-0.1633903384,-0.3208823204,-0.4034307897,-0.6640667319,-0.1162874550,-0.4164044857,-0.2106245607,2.6117999554,-0.8170961142,0.2296934575,0.1352863908,2.0153095722,-0.9835947752,-0.0398398787,-0.7099856138,0.2519035935,1.0100510120,-0.7469947338,-0.2271399647,0.8424672484,-0.0028453339,0.0573192686,-0.7911548018,1.5115075111,-0.4292039275,-0.2814901173,-0.2986505032],[0.9713628888,1.1659340858,-0.1845057160,0.0065851845,0.5065482855,0.1957827955,0.7852159739,1.0992507935,-0.6381587982,-0.2258110791,-0.0518908724,-0.4910358787,-2.2459900379,-0.7985108495,-0.8093937039,0.2404313385,0.4376957715,0.3021634221,0.2768120170,-0.9123110771,-1.4636436701,0.8011099696,-1.4775968790,1.9602862597,0.2919983566,0.1149028838,0.3892951906,-0.4618502259,0.7220209837,-0.1333760023,-0.8379451632,0.4440839589,2.7235734463,-1.1158251762,-0.2277544439,1.3317129612,-0.4921458066,1.3590461016,1.0847727060,0.0156160211,-0.1734282523,0.9402840137,-0.7774642706,0.7608965039,1.4017525911,1.2316533327,1.0011581182,-0.9780515432,0.6779560447,-0.0694640726,-1.0573610067,-0.1401209533,-1.4334952831,-0.1790812165,-0.1230426356,0.5381953716,1.0654780865,-1.4013202190,-1.3276036978,-2.0463159084,0.8385336399,-0.6869310737,-0.5620132089,-0.0442883112,-0.0047367625,0.6841859221,-2.1602709293,-0.1973593086,-1.2325326204,1.6918396950,0.5312877893,0.3519688249,2.1720921993,-0.1462069154,0.8961949944,-1.4612011909,0.2014119923,2.2746560574,0.4980108440,-0.7518550158,-0.9755960107,-1.5738663673,0.4436374903,0.8610516787,0.1271061748,-0.8113466501,0.2418205291,-0.6867427826,1.2207347155,-0.4108690321,-0.8827690482,-0.5456284881,0.3868001401,-1.2304912806,0.6670293212,-0.2712611258,0.0818201751,-0.1299658567,2.2640588284,-0.1695053726,0.8452254534,2.4532289505,0.4257523119,1.8475958109,0.0974547565,0.7967066169,-0.5887081027,-1.1121789217,-0.9687710404,-0.0241551120,0.2857384086,-1.1169031858,-1.3783563375,-1.8767281771,2.1267092228,0.1075932160,-2.1452682018,-0.0090910634,-0.1592708379,-1.3771734238,1.1593115330,-0.4573624432,0.0412932597,-0.2843123376,0.0463854149,1.0175329447,0.6056706309,-0.4518868327,1.5754882097,1.5401930809,-1.4632025957,-1.1145495176,-0.6111642122,1.5250842571,-0.1827131957,-0.5719213486,-0.0339936987,-0.2241427749,-0.1559786499,-1.0696374178,-0.7931587696,-0.2671611905,1.3815757036,1.2891613245,0.9156867266,0.3633261919,-2.6118755341,1.7803673744,1.1180121899,-0.3693193495,-0.6483833790,0.1860720217,0.0447100475,-0.6926733851,1.4939805269,-0.6550534368,0.2498712093,-1.2042515278,-0.2552892864,1.0822662115,0.0425696447,-0.5869133472,-1.9481761456,-0.3409913480,-0.3546536267,0.7333459258,1.3222188950,-1.2185709476,0.8016905785,-0.9687362313,1.1493378878,-0.4367955327,0.5572791100,1.5488294363,-1.7036701441,0.1468585730,1.5773946047,0.7791216373,2.1659414768,-0.4962480664,1.0852104425,0.7184401751,0.0505882911,-1.0123195648,0.2237323672,1.9337600470,1.2498807907,0.4884160757,-0.4939172566,-1.4075125456,-0.1990298480,-0.9089871049,-0.3559097946,1.1026279926,1.2252410650,1.0871775150,0.5441542268,-0.5297963619,0.4389980137,-1.7267080545,1.1040461063,-1.0993429422,1.3341366053,-0.6335446239,2.2331142426,-0.2638579607,-0.1156971976,0.7624002099,-1.3418661356,0.2185997516,-1.0052831173,0.6768507361,0.4997563660,0.2243302017,1.2361823320,0.1434970349,-2.1947665215,-0.7532163262,-0.9595815539,0.6633858681,-0.2023599595,-0.6430290341,-0.8134525418,-0.3416174352],[0.4218400717,0.8895066977,0.0375063010,-0.4428788722,0.0392513834,-1.2110996246,-0.4579227865,1.4286129475,1.0722044706,2.2364633083,-0.7332018614,2.6002426147,-0.5068512559,-0.6531524062,0.2969976068,0.4913036525,0.1942404211,-0.0775747150,2.0618264675,-1.1979434490,-1.0965492725,0.2128379345,-0.0973059759,0.0509134494,2.3178815842,1.6716791391,0.6432677507,-0.7658655047,0.2551034093,-1.7418290377,-0.8488428593,0.7497497201,-0.9394742846,-0.6140592098,0.2242153883,-2.9374637604,0.2126290649,0.3135212660,0.5678259730,1.6826602221,-0.5162865520,-0.2058548182,-0.7859453559,-0.6208847761,-0.7223044038,-0.7193372250,-1.9608685970,-1.1417084932,0.7958766818,-0.3401833475,-0.4921501875,-0.4349618852,0.3137913048,-1.3271640539,0.9106199145,-2.0794861317,1.0177873373,0.4266745448,-2.0590798855,0.0865664855,0.3999687433,-1.1941734552,-0.0655964836,-1.5167351961,0.8678767085,0.7040355206,-0.0141630126,-2.5513372421,-0.0591046214,-0.1782632917,-0.6721311212,-1.4577360153,-0.8415274024,-0.4924280941,-0.1054591313,0.6587065458,0.6277593970,0.2027057856,-1.0286164284,0.1998594999,0.6571788192,0.6904782057,0.9589516521,-2.3220672607,-1.4431536198,-0.3931055367,0.7838268876,0.8431079388,-0.6274953485,2.0802054405,-1.8623497486,-0.3251871765,0.6918599010,-0.4644671977,-0.9389165044,1.2123388052,-0.1305605322,0.2123090029,1.2012878656,-0.7994313240,0.2028683871,-0.4153151810,-1.1983784437,1.0681349039,0.5389846563,0.2846174240,0.0825485736,1.4062007666,-2.5803592205,-0.2879975736,-1.4541510344,0.2424149364,1.2237983942,-0.8512282968,1.5265872478,0.3694509864,0.7817362547,0.8075862527,0.3952428102,-0.9362884760,-0.7247197032,-0.0328013301,-0.4716132581,0.5637686849,-2.2679274082,0.3426539898,-0.0979672372,-0.2402446121,-0.0963036716,-2.1943864822,0.6942761540,0.0291213989,2.4530353546,0.7187931538,0.2133912891,-1.0874562263,-0.8873418570,-0.9403048158,1.9221187830,-0.8266971707,-1.5223639011,0.9220241308,0.8729629517,-0.1005499288,0.9404553771,0.2749381661,-1.2735220194,2.1069967747,-2.2149572372,-0.3422880471,1.4668304920,1.9702908993,-0.9250925183,-0.0060714148,-0.9882918596,1.4887702465,0.2804108858,-0.3489108682,0.5542525649,-1.5422412157,-1.2575305700,0.2715023756,1.2902284861,-0.2465406358,0.3424791992,-1.0304328203,1.7649421692,0.8475595117,1.3346787691,1.6747332811,-0.2368809879,-0.3557014465,-2.0324966908,-0.4428349435,0.6714438796,1.0843613148,0.2764685154,-0.1155481040,0.9883731008,0.4112939835,-0.6270067096,-0.1460562497,0.2729308009,-1.0989434719,-2.7572638988,0.7561358213,-0.9827682972,0.9108489156,0.6604135036,-0.1289324462,1.8280123472,0.6759609580,-0.3519555330,1.6215261221,-1.0882657766,-2.5271363258,-0.3571330607,0.9479765892,0.4368820190,-0.2717285156,0.6993663907,-0.7780221701,-1.1863687038,-0.5583704710,-0.7359765768,-1.0251901150,-0.1638904214,1.2510455847,-0.3172542751,-0.9209412932,-0.5378237963,-0.2243570387,0.5573434830,0.7788372636,1.7665655613,-0.5648117661,0.5073568821,-0.1877584457,-1.0030902624,0.0020168158,-2.2090637684,-0.3919711113,-0.0892934874,-0.2768075466],[-0.7382922173,0.4781686962,-0.5271230936,-0.8565163016,-0.0114886239,-0.3409681022,0.9220369458,0.4548239112,1.1105103493,-0.8540458679,-0.5342381001,-0.1587235928,0.3786576390,1.6225638390,-2.4055118561,-0.9667271376,0.7057707906,0.2168771327,-1.8539066315,-1.3120892048,-1.1653203964,-0.5374454856,-0.4951444268,0.6106092930,1.4650698900,0.2833453417,0.0034100253,-0.4601604939,-0.7077453732,-0.1619891524,-0.3710734546,1.2191070318,1.4806418419,-1.0868707895,-2.0648705959,-0.9124275446,-1.9892864227,0.8951463103,0.8109661341,0.7963710427,0.7626373172,0.6699028611,1.3099942207,0.1779273599,0.3980103135,-0.2223629951,0.7420293093,-2.4057607651,0.7425398827,0.7739811540,1.8070101738,-1.3846660852,0.1880014688,-0.1909513474,-1.5174038410,1.2443370819,0.4424822032,0.7484818697,-1.8359574080,-1.0961536169,-0.5201159716,-0.4773720503,-2.1213893890,-0.7915535569,-0.1687713414,-0.3899119198,0.2895978987,-0.3511215150,-1.3487813473,-0.0930239484,0.3696572185,-0.3927856386,0.1512555778,2.9449841976,1.6288179159,-0.3464185596,-1.0187661648,0.1346620172,-0.1407055557,-0.2609769702,0.8170487881,0.0737538487,0.5982130170,0.1496603042,2.7200851440,-1.6137839556,1.5230054855,-2.3850162029,0.9167109132,-1.0846986771,0.1835428476,1.8487994671,0.3989471197,0.8919633031,1.2315006256,-0.0234141331,-0.7482023239,0.1487952620,0.4381481707,0.3836542070,1.5491355658,-0.7265835404,0.9751958847,-1.2847875357,2.1041853428,0.2272570580,-1.1291662455,-0.2813319564,-0.7394167185,-2.5843210220,-1.1766476631,0.9712014198,-1.2400223017,0.6785711646,0.7806016803,0.9310101867,0.2308028489,-0.9521868229,-1.0569379330,0.0101044821,1.2908110619,0.8434351683,-1.0314128399,-1.3817199469,-0.0570717938,0.4098027349,0.7228524089,-0.1619416177,-0.1127954870,-1.3605376482,1.6959846020,-0.1019152254,1.0857456923,-0.1930623353,0.5130358338,0.2026667893,0.7818601727,-0.5955975056,0.1939072609,-0.2461180836,-0.2952910960,1.1024920940,0.4419107437,-1.1785167456,-0.1340370774,-0.3758517504,0.1165554896,-1.1369833946,0.9722950459,0.1699508727,-1.1497621536,-0.5577647090,2.4325559139,2.0300633907,0.3172180355,1.0420550108,1.3970026970,-1.3860317469,0.6756493449,-0.2088022530,0.0416123271,-0.3647925854,-1.3826622963,0.0327142030,-1.7300449610,-0.6120365262,1.1658670902,0.7848781943,-0.5333843231,1.3701926470,-0.6063991785,-0.9086737037,2.2848854065,0.6883058548,0.6623967290,0.8888964653,0.5443738699,-0.7265207171,-0.6869468093,0.3359093666,0.4884953499,0.9215206504,-0.1096334979,1.8479670286,-1.3251534700,1.9301359653,1.4032901525,0.3269095421,0.3069120944,-0.2451367527,-0.8169812560,-0.4496933222,0.4065041840,2.2406930923,0.8854416013,-1.4401488304,-0.9977304935,0.9404314756,0.0276801270,-1.0297243595,-0.9591946602,-1.1910470724,1.7555153370,-0.0457942411,-0.0153770149,-0.3424043953,-0.6024621725,-1.5076104403,0.3168259561,1.2989704609,-0.3437852859,0.3600607812,1.1474263668,0.8276595473,-0.7948626280,-0.3214859068,0.1499467790,1.6915059090,1.2442883253,-0.1117230430,-1.8866000175,-0.0652765781,-0.8484117389,0.9157932997],[-0.8810158968,-0.6766892672,0.0717540085,-0.0943662152,-0.8810151219,1.5139248371,-1.0652790070,-3.0692067146,-1.7233544588,0.2145391554,-0.0761685669,-1.6524933577,-1.8591082096,0.0064446605,1.4435199499,0.4387893379,-0.8295137882,0.4456272721,-0.8297645450,-2.2846164703,0.3067142367,0.2384975702,0.9313440919,0.7382943034,-0.0273835473,0.7648501396,-0.8317270875,-1.4108572006,-1.7982788086,-0.6981588006,0.6069019437,0.8501870036,-0.8509014845,0.4383669794,0.6248463988,0.8739450574,0.3806868494,-0.9634509683,-0.1053313464,-1.0504844189,-1.2173187733,1.4710108042,0.6670583487,0.0257385876,-0.9211400151,-0.5374246240,-0.0370455347,0.9016004801,0.6242956519,-0.0050148740,0.8383492231,-0.6565697193,1.8286083937,0.5789019465,-2.3177831173,-0.5858536363,-0.3403774202,1.7211940289,-0.0403645262,-0.0005720349,0.9837250710,1.1704604626,-2.0550563335,0.5990749598,-1.3058104515,-0.1033330113,-0.3316901922,0.9040453434,2.9563987255,-1.5847589970,0.4970963597,1.2656636238,-0.1820234358,-0.6379039288,1.7642017603,-0.3535215557,-0.3904646635,0.0819236487,-0.4960451722,0.6401004791,-1.1532803774,-0.7206152678,-0.4024859965,0.7426589727,0.6694399714,1.4347842932,-0.3065627217,0.3679106236,1.2681542635,0.0654527098,0.8345687985,-1.1156513691,0.8476576805,0.2385712713,-0.4635876119,-1.1457544565,-0.0187508408,0.5387160778,0.2548680007,-0.0915771574,1.0684789419,1.0852134228,-0.8394083977,-0.1809697300,-0.3924306929,0.1450937837,-0.0587084554,0.6864700913,-1.0417679548,0.2963407636,2.9507956505,-1.9250681400,-0.1433570832,-1.1863735914,0.1866655797,0.9594634175,0.3174482286,1.4533431530,0.5670638084,1.3095489740,0.1332997233,-0.9662900567,0.3597651422,0.1602782160,0.2071735710,0.8496370912,-0.7209706306,0.1189359799,-0.7544926405,0.8743216991,-0.6681683660,0.3383395374,-1.1891311407,-0.0207240880,0.3600969017,1.0823884010,0.4390854836,0.6673156023,-0.1929547191,-0.0451419055,-0.4575970769,0.2687020898,0.5414130688,0.5603662729,0.4634231329,1.2978063822,-0.2309095711,1.7121770382,0.2484276593,-0.7386830449,0.3304316700,-0.0578057617,-0.0305827893,0.9412934780,-0.5984822512,-0.9803550243,0.2070207894,-0.6550219059,0.1320174783,0.7900218964,-1.3689039946,0.9380484223,0.4847313464,0.2433396876,-1.4307051897,-1.0672821999,-0.7570881248,-0.4750105143,0.0807691738,1.2956811190,0.5300829411,-0.4246348143,-0.6007465720,1.3769870996,0.4140109718,1.0277614594,-0.8248614073,0.2571107447,-0.7096899748,-0.1520026624,-0.6228904128,-0.1400981396,-0.1943059713,-1.3681163788,-1.1639925241,0.4308236837,0.1339069456,-0.8116710186,-0.5282785296,0.4628006220,1.3132369518,0.8331753016,-0.2018923014,0.0933108181,-1.0099724531,0.4170527160,0.4332081079,-0.2000631541,-0.4819248021,0.0526755936,0.5904321671,0.4194876850,1.4380538464,-0.2091670036,0.1405492425,0.8455461264,-0.1576862335,0.7914052606,0.5603117347,0.6225966215,-0.6495121121,1.6057735682,-1.4565155506,-0.1592467129,0.9452735782,-1.1509324312,1.1862341166,-0.9232009053,-1.4868230820,1.2887068987,-0.5039800406,-0.9293571115,1.4114164114,-1.3048022985],[-0.5259607434,0.1474925727,-0.6388627291,-0.4544603229,1.6006817818,1.8981077671,1.2952227592,-0.6770508885,-0.0343132466,0.0424407683,0.0593665354,0.1125282869,0.1721530408,0.2323742956,1.0071390867,1.1618844271,2.0738105774,-1.5517368317,0.5792925358,-1.5301368237,-0.9412553906,1.5477662086,-0.1588866711,-0.5141246915,-0.1607577950,0.5885755420,0.7499521375,-1.0819777250,0.2867581248,-0.8426262140,-0.4466435015,-0.7166017890,1.5151894093,-0.3630546331,-1.2952378988,0.2233130634,0.7022958398,1.2997992039,-0.2185886055,-0.3559314013,0.9324097633,-1.1729716063,0.8698990941,0.3515312672,0.5347394347,0.5843375325,-0.9044497013,0.2044183016,-1.1293611526,1.0726300478,-0.5822884440,1.2378178835,0.2893677652,-1.0045837164,-1.0325323343,-0.9010717869,-0.5148781538,0.4178542495,-2.0488326550,-0.9897443652,-0.3382941782,1.5038264990,-0.2582089603,-0.1545956433,-1.6558270454,-0.0935545862,-1.0900809765,0.7780081630,2.1689538956,0.5874821544,-1.1014442444,0.6476220489,-0.8646059036,-0.3191539943,0.1379110068,1.2473257780,0.1051613837,0.5986577272,0.0665234625,0.6185915470,-0.1027515382,1.2281802893,-1.4339200258,-0.7613863945,-1.1129082441,-0.6932907701,-0.9792848229,0.4943833947,-0.6510017514,-0.0364130028,-0.5350659490,-0.7569305301,0.1001948863,-1.3082510233,-1.0891143084,-1.6640188694,-0.7002798915,-0.0759273991,1.1599098444,-0.8365039229,1.5821044445,-0.1271798313,0.4336561263,0.1694084853,0.5724648237,1.8298932314,1.4186511040,-0.9776277542,-0.4195668101,0.4297143519,-1.0633279085,0.3911823630,-0.0747859403,-0.9993607402,0.3444029987,0.9736811519,-0.9430867434,1.6025952101,0.2376157790,-0.1281956732,0.3923225701,-2.5928940773,0.7752914429,0.2649167776,-0.4567572176,0.3371549845,0.4049337804,-1.0609476566,0.0936940014,-1.5922925472,1.8138062954,-0.1034576967,-0.3052410781,-1.4047080278,-0.3657502234,-0.0972745717,0.2751365006,-1.5588892698,-0.3687875867,0.7465646267,0.3279536366,0.1500969678,0.6712778807,-0.3980213404,-0.8001051545,-0.5537888408,0.7133117318,0.8906948566,0.0045022634,-0.9853961468,-0.1782789230,0.9238131642,0.7145440578,-1.0212540627,0.2322989702,-0.1549165547,-0.3999933898,-2.6583871841,-1.0034289360,1.3892837763,-0.0713517517,0.1388881058,-0.0967622548,0.4031153023,0.6281492114,0.5679972768,1.1819807291,0.4587058425,1.7037755251,1.5106276274,0.0188147873,-1.2132136822,-1.1595399380,-0.9960440993,0.1820143163,-1.2581967115,-0.0109062744,1.2203755379,0.6200798154,-0.2033996284,-0.0597796105,-1.1121650934,0.3698798716,0.2429883182,-1.4556866884,0.2045957595,0.2362975627,-0.2086562663,-0.8186010122,-0.2720654607,-0.5458545089,-1.2913902998,1.2412921190,1.4953358173,-0.2214281559,0.5088971257,-1.5477359295,-0.7341091037,-0.0617847852,-0.9167001843,1.7482943535,0.6113579273,0.1244715825,-1.7654695511,-0.8131557107,-0.2266721576,-0.2433164567,-0.9586618543,-0.3598257303,-0.1786702126,-1.8549596071,0.4828202128,-0.2374148220,-0.2234849483,-0.7928118110,-0.0088003566,-0.1138385981,-1.1613034010,0.5579746366,-1.1964149475,1.2761023045,-0.5428506732,-0.7676082253,-2.0380215645],[-0.8374180794,1.8076927662,0.6725515127,-0.4143821895,0.0190587919,0.6254171133,-0.2820325494,0.8748767376,-0.5163024068,-1.0898655653,0.1275738180,-0.1395269185,-1.5575122833,0.4249118268,-0.1607739925,1.3152391911,-0.7662281394,-0.1419369578,-1.1442737579,0.5419403315,-0.0603652373,0.0115801683,-0.4260250032,-0.8693335056,0.3321054876,-0.2232301235,0.1859180033,0.0755599067,0.4812561274,0.0805540904,-0.1881784499,-1.3111925125,-0.0887244120,1.5127700567,0.5737081766,-0.5410040021,0.1011774912,0.9945519567,0.0187130310,0.2556715012,-0.7111772299,1.1038419008,0.7498760223,-0.6901777387,-0.6161143184,-0.8754807711,1.3833988905,-1.4642595053,-1.2905743122,-1.1369878054,-1.8163366318,-2.0593073368,-1.5773495436,-0.1809103638,2.6330025196,-0.8738535047,0.4626492262,0.5717197657,-0.5768548846,-0.3781278729,-0.1874432266,0.1967199296,0.3621532023,-1.7878540754,1.1232843399,2.2314839363,-0.2592034340,0.4197042882,0.3647200167,-0.4982883036,0.8533704281,0.6698650122,1.4574915171,0.1867191195,3.0060515404,0.0711427107,-0.2154966444,0.0293234326,-0.0889864117,0.1243936792,0.0645327345,-0.2294944078,-1.0487772226,0.1695384383,-0.4823871255,-0.2830505073,-1.2472976446,0.1283144057,0.4478857815,-0.6975439787,0.1654880643,-0.3362832665,-0.1657265425,-0.6748378277,-0.6607167721,-1.2857440710,0.2204591036,-1.4501774311,-0.0336493291,0.8193687797,-0.7773550153,1.2970961332,-0.5298593044,-0.2872518003,-0.0863314644,0.3339187801,0.3005096018,0.9379919171,-0.4672140777,1.9484413862,-0.3087449372,-2.0272562504,0.2302280813,0.1488207579,3.2926943302,-0.5023747087,0.9368308187,0.4223787189,-1.2146153450,-0.3633534908,-1.0163754225,0.8161545992,-2.6420648098,-0.9995900393,-0.6842966676,-1.3786201477,-0.1166622937,-0.5009272099,1.3049273491,-1.1700606346,0.4273369908,-0.4868766069,-0.9399681091,0.1936713308,-0.1304665208,1.6364108324,-1.2473168373,0.4555480182,0.7874873281,1.0548788309,1.4028953314,-0.3403153718,-0.7662428617,0.7910608053,-0.5680841804,-1.6864377260,1.7951066494,-0.5170757771,1.3853578568,-0.4875607193,0.9538254142,-0.4127383828,1.2545660734,0.9811357856,2.5652039051,0.4855013788,-1.6319627762,-1.9827576876,0.7196250558,0.3306773603,-0.3879871368,1.6143678427,-1.7628905773,-0.2992368042,-0.2816053331,-0.4913335741,0.1062109917,0.6110439897,0.4482290745,0.8110507131,-0.0504925326,-1.0317854881,0.5823349357,0.8444985747,-0.3849413693,-1.3550829887,0.6306875348,0.3171112239,0.0095400224,0.8568302989,1.8019857407,-0.9952394366,3.2118475437,-0.1661012471,1.7766364813,-0.3340893090,-0.2285335511,-0.0810918510,-1.7271569967,0.4434600174,-0.7047592402,0.5324918628,0.4588414729,0.6069088578,-1.1168786287,-0.4205290377,-0.4082236588,0.9159150720,0.0197826736,-0.9659548998,-0.0031386658,-1.1092762947,1.1993969679,-0.9169333577,-0.3250979185,-0.6554597020,1.0403012037,-0.8468415141,-0.0201100614,1.0483227968,1.3847842216,0.2876734436,0.3542454243,0.4986132979,-0.2432114482,-0.1372609735,0.5234652758,-1.2656041384,0.4806742072,3.0031232834,-0.1512721330,-0.7243952751,0.0387897491,-0.1198186129],[0.8208485842,-1.0074971914,-0.6677929163,0.0483032688,0.1750379354,0.2083159089,-1.9190596342,0.9766510725,1.8913180828,0.4273438454,-1.3285946846,-0.1469343305,-1.3562725782,-0.3669628799,0.1919582635,1.3829925060,-1.0648114681,-0.2708503604,2.7610263824,0.6004490256,1.3111655712,0.9766030908,-0.1186602041,-0.4448900223,-1.0978698730,-0.4263503253,-0.2740243077,-0.1068669632,1.0883152485,-2.9813721180,-1.1545959711,-0.0348202735,0.2651986182,-0.4033118188,-0.8410776854,1.0309700966,0.4036042690,0.1645698398,-0.4990791678,0.8270534277,-3.1262013912,0.2744820118,0.8101330996,0.7581610084,-0.7181271911,-1.3808708191,-0.5076599717,0.0838778764,-0.8853391409,0.6123282313,-0.1626137942,0.2677460611,-1.4126522541,-0.4442411065,-1.2625015974,1.5032956600,0.6110239029,-0.4633914828,-0.2613438368,1.8707574606,0.6762051582,-0.3888108432,-1.0406956673,-2.4245371819,0.4552400112,-0.4199117124,2.0087494850,0.6389307380,-0.3146655858,-0.0531506911,-0.1081308648,-0.8942368627,-0.2720023394,1.1744161844,-0.5152111053,-1.5052090883,-1.3972624540,-0.0511602834,-0.0412393324,-0.2354413718,-1.2756769657,0.1774501950,-0.1988486648,0.8698195815,0.4150623381,-1.1867893934,-1.2666206360,1.5865522623,0.0610986762,-0.1770948023,-0.5854319930,-0.4385347962,0.0175962076,1.3314625025,1.5840754509,-0.3236637712,2.3411216736,-0.6135569811,0.9249237180,-0.2237814665,0.8911208510,0.1451560408,0.2448153049,0.3776126504,2.0296990871,0.6324076653,0.3589432240,0.4496875703,-0.3058456481,1.1094084978,0.4872805178,-0.3038507998,-1.4573172331,-0.9205236435,0.6826651692,-0.3554729223,0.3927389085,-0.3810762763,-1.7063821554,-0.1602554619,-1.6169706583,0.6162632704,2.4065866470,0.3152024150,1.2607896328,0.3310032785,-1.3002210855,0.5527735353,-0.7645601630,-1.6030761003,0.2826044559,0.1884256005,-1.1052546501,-0.0256252103,-1.4617702961,2.3069312572,0.1014108062,-0.5887804627,-0.0482289307,0.7604926825,0.7126061320,-1.1288833618,0.3887622952,-0.2175828516,1.4924032688,-0.1203889251,-0.4008507431,-0.4998865426,-0.9092680812,-1.4348453283,-1.2464061975,0.4905832410,-0.4584877491,-1.7624422312,-1.2848756313,0.1869234592,1.6680623293,1.4857335091,-1.3092820644,-0.4001376033,-0.6902616024,-0.2851842344,0.5611698031,-0.2224484831,0.1025903299,-1.3632470369,-0.5505031347,0.9418570399,0.4167126417,1.5588051081,-0.4624279737,-0.3520548344,0.6923726201,-1.3459591866,-1.3967781067,-1.2523069382,1.1929949522,-0.8578463793,0.2459804118,1.1848936081,-1.6676170826,0.3219023943,0.1425884366,0.8878026605,1.3843916655,-2.0635306835,0.4181311429,-1.6780023575,2.8656015396,-0.6755149364,-1.2139745951,-1.7235435247,-0.0115590412,-1.2834458351,0.6609151959,-0.1157040447,0.3007107079,-0.9618673921,0.6788420677,0.4106647968,0.7691630721,-1.8603491783,-0.2602683008,-0.6554810405,0.8151291013,-1.1044774055,-0.0258444790,-0.6846947670,0.7430288196,0.0927735269,-1.8108327389,1.4642354250,2.1097369194,0.5113024712,0.9214703441,-0.7961116433,0.1557842940,1.6264904737,1.6839746237,-0.1017777249,-0.0270002671,-0.5560973883,-0.3349419832,0.4305644631],[-0.3148517013,0.5323333144,0.4808635712,0.6366700530,1.1908731461,1.2183773518,-0.0871770829,0.1331706196,0.0532641746,-1.2944794893,-0.6270956993,1.0332086086,0.8411585093,-1.1188211441,0.9106353521,2.4634940624,1.3054764271,-0.5078490973,0.6401654482,-2.0949027538,-1.5359004736,-2.1329395771,-1.2214189768,-0.0510937348,-0.6269537807,0.9085376859,-1.7701388597,0.1271123588,0.2025357783,0.2193867266,-0.1419527531,0.4712587297,-1.1216319799,0.2348901033,0.2660713494,0.1221213117,-0.0339588486,-0.4119123518,0.2396080047,0.4450393617,0.1456137896,0.1783765703,-0.8795627356,0.0829178467,0.2220709324,-0.2598603666,1.2970924377,-1.6507456303,0.6115096807,-0.3253150582,0.1117031053,0.7479269505,0.4571745992,0.1007617787,1.4383536577,-1.4001889229,-1.9547203779,-0.7588565350,0.1194262281,0.7364101410,-0.6658715010,-0.0521111190,0.1420148462,-1.2008237839,-0.0141287912,0.1722817719,1.0185024738,0.3625552952,-0.2192810923,0.6853411794,1.0109658241,0.0720389709,-0.5317717195,-0.2604891360,-0.2174306065,0.9860923290,0.3199743629,-0.0393218994,1.5142209530,-1.8734128475,-0.8563058972,0.7394201159,0.9219983816,-0.8890936971,-0.5466315150,0.5607649684,0.8172806501,-0.2346559316,1.4355891943,-0.5468201041,0.1247638464,-0.6282752156,-0.2512498796,0.0405594669,0.4966886342,0.7192645073,-0.5121807456,0.2786832452,2.3277153969,1.7324640751,-0.1355056465,0.9407029152,1.7785062790,2.0651509762,1.1749883890,0.0644653961,0.3345230818,-0.2759347260,-0.9038693905,0.0737935454,1.7145204544,-0.0049464516,-0.4580802917,0.8939317465,-1.5609588623,1.1997606754,-1.3571116924,0.2147751749,0.6041502953,-0.4827028811,0.4263130724,0.3580539525,2.0956006050,-0.2793421745,-1.6219590902,-0.2702989280,1.0025880337,0.7011756897,0.1722511351,-0.1773850620,0.7790995240,0.8636000752,-1.0049010515,0.6323234439,-0.7638198137,0.3079225421,-0.3288014233,0.3946300745,0.8784893751,-0.1306656599,3.0949795246,-1.4146577120,0.3331373334,1.1519114971,-0.1283607334,-0.6941428185,0.1147115529,-0.0274392385,-1.7293875217,2.3312771320,1.6542543173,-1.7632293701,-1.9415524006,-1.1907383204,-0.0048493538,0.4222418964,0.0341976583,1.5213152170,-0.1766054034,0.2243996412,0.7282629609,0.1159321964,-1.4154875278,0.3165683150,0.8783222437,-1.1561028957,0.5572648644,1.5004160404,-0.8511629105,-0.5939493179,-1.1936300993,0.6819807291,-0.0313347243,-0.0286470875,0.8888990283,1.0018740892,-0.1425251812,0.2556823492,0.7652249932,0.1631320864,-0.2498634607,-1.5008529425,-2.9273338318,0.6607398987,0.1327477843,0.5937239528,0.1125463098,-1.6437536478,1.5341486931,2.1285750866,-1.0796158314,-0.2879979610,-2.2914717197,-0.8244517446,0.4066144824,0.6147168875,-0.5193148851,-0.7325679660,0.4611653090,-0.4268031120,-1.1843461990,-0.1097135618,-1.1662861109,1.3328620195,-1.4928250313,-0.2382298410,-1.2711412907,-1.0696352720,0.6976164579,0.3925410211,1.2439258099,-0.3567233384,1.2630944252,0.0982297659,-0.5787029862,-1.2956535816,0.8188961744,0.4646765888,1.2062302828,-0.0755145028,0.2233940512,0.0219676215,-0.2262375951,0.8005526662],[0.2583341599,0.0134167904,-0.9566358328,0.6107801199,-0.3282876909,-0.5326413512,0.3580334187,1.6714169979,1.4230730534,-0.5663484335,-0.7258203030,0.5634755492,-0.4356320798,-0.1045525447,-2.3212728500,-0.2140943706,0.5434054136,-0.5374541283,-0.3802753389,0.2654729187,-0.1454317421,0.3900747597,-0.1429263800,-0.0141180474,-0.5412923098,0.9153400064,0.7681113482,0.1016348004,0.8094421029,0.0004556435,-0.2267173529,1.2817173004,-0.0734994039,1.0696345568,0.7920151949,0.3397075832,0.6335131526,-0.3126899004,0.3656824529,0.3227659762,-0.1339740306,0.3025370538,-1.6606614590,0.1812739521,-0.3153503239,-1.0073664188,-0.7484238744,-1.7166824341,-0.1572054476,0.8618949652,-0.9956087470,-1.1582169533,1.3176295757,-1.6233000755,0.1798952520,0.6858589053,0.0602968559,-0.6038802266,1.0489137173,-2.2571678162,-0.0584337749,-0.1138860956,0.0855550766,-0.7688744068,-1.2541327477,-1.4902909994,0.1294953823,0.0926831588,-0.5188183784,-0.7847260237,-1.0042928457,-0.0707495660,-1.5369973183,-0.8811527491,-0.9041975737,0.7040268183,1.1990603209,1.8376708031,1.3268336058,-0.1728139222,-0.1250518411,-1.5261913538,-1.8103522062,0.3975400925,1.7494088411,1.7238782644,-1.3281983137,0.1908317953,-0.8550479412,0.1436198354,-0.3308296800,0.1347103268,-0.5981942415,-0.9036156535,0.8448509574,-0.5984261632,-0.3971627057,-0.1978619695,-1.2565414906,-0.4385175109,-0.6725533009,-0.5639172792,0.0460369922,-1.9070044756,0.4588394463,-0.4961534441,0.5328682065,0.2758805752,1.0665974617,-0.1897082031,-1.0725839138,0.9363661408,1.1084721088,-0.3609683514,-0.6044198275,0.7308124304,-0.2895660102,0.9214672446,1.1936705112,0.1420208365,0.9903038144,1.0248546600,0.7819829583,-1.5155851841,-0.1204732060,-0.2646034360,-0.4781559408,-1.2578017712,-0.8879620433,0.8834035993,0.3680997789,-0.3455525041,-0.1083529815,0.8362635970,-0.8118629456,-0.8376584053,1.3152875900,-0.3645234704,1.9935706854,1.5848784447,-2.1046626568,-2.5531182289,-1.2426662445,0.2019866556,-0.3053317666,-1.1955868006,-1.5779031515,0.8499122262,0.3275901377,-0.0016695884,-0.0355634280,-0.4892515540,1.9304982424,-0.2626447082,0.8259323239,-0.6432670355,-0.8289811015,-0.2027345747,-0.2578664720,0.0708145201,0.9978447556,0.2600807250,0.9250656366,1.4760758877,-1.8792523146,-0.7825285196,-1.4022051096,-1.4552857876,-1.1523354053,0.0965485498,0.2922802567,0.8867214322,0.1132206246,-0.0294816475,0.2782558203,0.9704330564,-1.4312628508,-1.4022347927,-0.1946246922,-0.5354971290,0.6042767167,-1.4157117605,-0.9723172784,1.0312830210,1.4538744688,0.2337075174,1.3080239296,0.7617688179,0.5921661854,-1.2361630201,-0.9084081650,-0.7632186413,-0.8020383716,-1.3764209747,-1.5502327681,-0.1663934290,0.7598895431,-0.2875463068,-0.3408353031,-0.4380674958,0.1424137503,-0.2995167673,-2.1005253792,2.4057154655,0.3382180035,0.3632414639,0.3133047223,0.4243564308,0.1828026623,0.4043556750,2.5423345566,-0.3791434169,-0.1495744288,-0.8346999884,-0.7742741108,1.2149647474,0.7606167197,0.3171077371,-0.1672189832,0.1230465323,-0.5340057015,-0.0623281896,0.4993795156,-0.6112615466],[0.2279778421,-1.2632958889,-1.0773020983,0.3607445359,-0.2576623857,0.7818472981,-0.4763545990,1.3054454327,0.6843120456,0.4527694881,-2.4472017288,0.6361515522,-0.5922811031,-0.1407441050,-1.1186494827,-0.2712420821,0.3353216648,0.9042813778,-2.1890122890,-0.3902392983,-1.2158573866,-1.4680877924,-0.2748048902,0.8750464320,-1.0083405972,-0.4647849798,-0.1865984648,-2.0889463425,-0.5600870848,-1.3486815691,-0.3641701937,1.1793979406,0.1997187734,1.6865342855,-1.0752303600,0.5811572075,0.3023730814,-0.5949453712,1.3834298849,0.3260892630,-1.5045257807,-0.6118640304,0.1986764967,1.0946094990,0.9827951193,-0.0680601895,-1.0457265377,-1.7469846010,-0.5958772302,-0.3935099542,0.7819852829,0.1315787882,1.3365695477,1.2137441635,0.4329839945,0.7754896283,1.2031998634,-0.4921071529,3.0196695328,0.6024559140,0.2277309895,0.5269126296,-0.1516455263,-0.2588847578,-0.5245341063,0.2717128098,-0.8874869943,-0.8964639902,-0.1968666464,0.0911976174,-0.8873035908,-1.8681821823,-0.0760981515,-0.7805968523,0.4802442491,0.0320365019,-0.7067071795,-0.4631549716,-1.1735436916,0.5445239544,-1.3017100096,0.3699762821,-0.0850021094,-0.7237022519,-0.7464592457,0.2509557009,-2.1246905327,1.6723703146,-0.7169879079,-0.5348271132,0.3575956225,-0.2395388484,1.9923259020,0.1982047707,-1.4630272388,-0.5053290725,0.8650518656,-1.3065795898,-0.6528154612,1.7014002800,0.2201984078,-1.6176035404,-0.5858163238,0.3461747766,1.7925175428,-0.8658657074,0.0194235109,-0.2472789586,-0.9830157757,-1.2460169792,-1.4011884928,-2.3403863907,-0.7892500758,-0.1878778636,-0.0241079796,-1.2374361753,0.4444930553,1.4464956522,-0.6280738115,0.0667637363,0.6859157681,1.8527070284,1.5816912651,0.0943878442,-0.6667955518,0.6660448313,0.5320387483,-1.0508078337,-0.5289185643,-1.9789937735,-0.0936506018,-0.5537081361,0.1577619612,0.3714364469,0.4536406696,1.1362136602,0.4127557874,0.0746068135,1.4729266167,-0.0521894582,-0.3334014714,-0.2249729484,-0.5179290175,-1.0466347933,0.9848658442,-0.0994437635,1.7418688536,1.1313471794,1.5630669594,-1.0602076054,1.6669305563,-1.4789799452,-1.9057705402,0.3552731574,-0.7447243929,0.1306759268,0.9287368655,-0.1092540473,0.3579307199,0.0620544627,0.9341294169,-1.3217412233,-0.1887846887,0.5450025797,1.1946133375,0.1155479774,-0.0156681519,0.0818186924,0.3128174245,-0.5175796747,-0.9493309259,-0.5009193420,0.3581947982,-0.6916647553,0.7086075544,1.2116622925,-0.5839709044,-0.1023268700,-0.4031613171,-0.0775697455,-0.1116259247,0.5571227670,0.6536220312,0.1175153852,-1.2265955210,0.2845886052,-0.3341941833,-0.9034729004,0.3816067576,-0.0520795807,1.3590296507,0.2401977628,-0.5103403926,1.4039888382,1.0099010468,-0.4682047069,-0.1903148890,0.2275711149,0.3296707869,1.3139348030,0.3545247018,1.6756818295,-0.4598955214,0.0968695879,0.9012082815,-0.6129329205,0.5607587099,1.7570564747,0.6849796176,-0.1868782192,-0.4102973640,-1.4443739653,-0.0522584990,2.2450199127,0.1798616350,0.7548625469,1.1856045723,0.2843368649,-0.4107406437,0.4364345074,1.2590898275,-0.3064253330,0.7047862411,0.4805850387],[-0.1725681275,-1.0563880205,1.0208398104,0.9267752767,0.1891755462,-0.3822020590,-1.4817459583,1.2379530668,-1.5762476921,-0.8821061850,1.7232716084,-0.5654813051,-0.0660265014,-1.1681851149,-0.6628458500,0.7722485662,0.2607228458,2.2022979259,-0.7688165307,0.0033735237,-0.1173261702,-0.9136849642,0.3267530501,-0.1441042274,-1.2224164009,0.8834843636,-0.6588438749,-0.2248045206,-0.8135863543,-1.1237211227,-1.0655337572,0.4619101584,-0.7503932118,0.3477369547,0.2655709982,0.8278338909,-1.2869812250,0.3188256621,-0.8562581539,0.7813947797,0.6154556870,0.6575708389,0.1797037572,-2.2783794403,-1.3994863033,0.7637216449,-0.6022380590,1.0573459864,-1.2417126894,0.8444338441,0.4190975428,0.5408490300,0.1426850855,-0.9931777120,-0.3794707060,0.5953972340,-0.8911392093,1.2549008131,1.0709406137,1.3681180477,0.7540420294,-1.3617898226,0.2148898542,0.7666696310,-0.5721467137,-0.9326041937,-1.3463188410,-1.0583540201,0.3082196116,0.1263058931,1.4746549129,0.0198656376,-0.1533403397,0.5427250266,-0.8982430696,1.0728341341,0.9212007523,-1.0459827185,0.2398848087,-0.2399872392,0.2015265524,1.0953737497,-0.7641457319,-1.0590814352,0.9709772468,-2.7245488167,0.0263428781,0.0311356522,-0.1787093282,0.4194993079,-0.8683885336,0.4747060835,-0.4976926148,-0.6131805778,-0.8993081450,-0.6951372623,0.2548579872,2.4627816677,1.2880226374,1.3663511276,-2.0841400623,1.0545402765,-0.6584460735,0.7016707063,-1.4842098951,0.7018342018,-0.7367026210,-1.1353683472,0.8034266233,1.9471383095,0.2818152010,0.6163467169,1.6901090145,-1.7596832514,-0.0940287039,-1.5356237888,-0.4820639789,-0.2813972235,1.9129413366,0.1789698899,0.2698389292,-0.4756282866,-0.0582442805,0.6104027033,-0.1537651569,-0.3071576655,0.0031420253,-1.1825238466,-1.8446222544,-1.3169915676,0.1479340047,-0.6539691091,-0.3660846055,-0.5772585273,-0.5850701928,-0.2924168110,-1.8485302925,-0.1489338130,-0.7512722015,-0.7153009772,1.0778415203,0.6231737137,-1.6968379021,-0.8516989946,-0.9225254059,0.9233241081,-0.3950044215,0.7099353671,-0.0429341830,-0.6496396065,-0.3185375035,-1.8720908165,0.0721271336,0.2912057340,-0.2783482969,0.6049549580,0.6706092954,0.7283976674,1.3359290361,-0.8727504015,-0.1820479333,-0.2766488492,0.5385487080,-1.2421909571,-0.6489337087,-0.8941696286,0.1014679298,-0.0520993546,-0.6106041670,-2.9797456264,-0.6892374754,-2.2069187164,-0.4327895939,-0.6774798036,0.1662548184,-0.1374444216,-0.7766288519,0.4023796916,-1.8384613991,-0.7337769270,-2.0104799271,-0.8689224720,-0.0495265275,-1.3442329168,0.0602338761,-1.7688345909,-0.8403044939,-1.0115302801,1.9035303593,-1.2843977213,-0.4898892343,0.7581721544,1.1472157240,-1.3688549995,0.6080808640,-0.5300948024,-0.6094548702,1.0624125004,-0.0401498191,-0.2764210701,1.5775593519,0.2636820376,1.2670412064,1.1847480536,0.6004807949,0.6627605557,-0.9631214142,-0.0651835725,0.3965368867,1.2247297764,-1.5966616869,-0.9161520600,2.0568938255,0.1804197133,-0.0667092502,-0.0283539798,0.4848047495,0.8173012137,0.8953057528,-0.7625162601,1.4375944138,1.1974359751,-0.6648783088,0.4911331534],[0.4361787140,2.4836254120,1.2508749962,-0.8652247190,-1.1523828506,-2.5317647457,0.6768011451,-0.8235494494,-0.0016548608,-0.2926580310,-1.7133356333,0.2775128782,-0.4684120715,-1.0212373734,0.3111077845,-0.5543714166,-0.5277050734,0.0934340209,0.0553637892,0.1839600801,1.0744355917,-0.8088902831,0.0177089591,-1.0553754568,0.0537255295,0.8920047283,-0.1436830014,0.5428864956,2.6151807308,0.9089220762,-0.6701074243,0.1462989897,-0.4177500904,-0.3071745634,0.2703181803,0.0061454624,-0.0414125137,1.2520478964,-0.5853372216,0.0183674395,-0.3882516623,1.1016137600,1.8874223232,-0.6500365734,1.4510560036,0.2006217688,0.1075251549,0.4110045135,0.2201270163,-0.9558433890,-0.0729107559,-0.2200553864,-0.1955879778,-0.3888328075,-0.1492266357,0.2064908445,-0.4907214046,1.3380523920,1.0333486795,-0.3924434781,0.2761882544,-0.0500014015,0.2783436775,-0.2208845317,0.4076091349,-1.6868382692,-1.1322995424,-0.4491987824,-1.6285598278,-0.0747513771,-0.9900171757,-0.5021103621,-0.0169003569,0.6617447734,0.5219373107,0.0710919797,-0.1033318341,-0.1241831928,0.5050722957,1.7465324402,-0.9384755492,-0.2050182074,-1.4408466816,2.0659956932,-1.2176458836,1.0436911583,-1.6367999315,-0.8311728239,0.0383930989,-1.4327731133,1.6502213478,-0.0957067087,2.2228686810,0.3076825440,-1.7243221998,-1.6951988935,0.2783026695,0.9383012056,-0.8741029501,-1.1462081671,-0.6457691789,-0.8208730817,1.1436293125,1.4125794172,0.2804195881,-1.6822501421,-0.4242359996,0.6605323553,-0.0331386216,-0.0384374969,-0.8116567135,0.1741831154,1.4778373241,-0.2192885429,-0.5614511967,-0.8825250864,-1.2342511415,1.1353919506,0.8536889553,0.6619521379,0.1940427870,1.2519853115,1.4540354013,-1.4876911640,-0.5286277533,-0.5613620281,0.4594193399,0.4614850581,-0.8415557742,-2.4047565460,-1.4944550991,-2.2047286034,-0.7104699612,-0.6865041256,0.4871855676,0.9703326821,0.0191650800,0.0305574350,2.1260392666,0.3825403750,0.1017763391,1.2437696457,0.0841875896,0.0260246377,-0.3963061869,0.2071161419,-0.4882599413,-0.1168544292,-0.9286617041,1.1061260700,-0.2454181761,-0.0635446012,-1.3290656805,-0.2628045976,1.4840424061,0.5852842927,-0.6792443991,-0.4082862735,-0.3425306380,-0.5310598016,-0.2525795102,1.2393785715,-0.6161656976,-0.8743874431,-1.1418819427,-0.5788629055,2.3537743092,1.3391598463,0.7086358070,1.0133312941,-0.6402965188,-0.4752223790,1.2483383417,-0.0412226953,-0.3494254649,1.2869471312,-1.3261907101,1.0926253796,0.3833552301,-0.6908642054,-0.0721437111,0.4729478657,-0.4645186067,0.8151687384,-0.7450098395,0.3585434854,-1.3225007057,-0.5362467766,0.0556739420,-0.5106480718,-1.4202176332,1.1659399271,0.7442878485,-0.6453229189,0.4733515680,-1.0344504118,1.7793974876,-0.7470325828,1.7667423487,1.0275626183,0.0522043034,-1.9739670753,-1.1114044189,-0.3246097863,-0.3506036103,-0.3710768819,-1.9017586708,2.0592153072,0.1685639471,0.0427768752,-1.3544977903,0.5274385810,-0.8166235089,-0.6331622601,-0.0177748725,1.5376009941,1.5840349197,-1.5073336363,0.0665762797,0.3779613674,0.9021579623,0.0178753231,-0.1060614064,0.9497752786],[1.1455736160,1.1762216091,-0.8378641009,-0.5064931512,-0.5005347729,0.0742226392,0.5423977971,0.6719839573,0.3955279887,0.1929187477,-0.0416823812,0.5823065639,0.4075008333,0.9344403744,1.6156861782,-0.4030669630,-0.5414798856,0.3706814051,-0.4143368900,1.2245029211,-1.7612340450,0.0220842361,0.0963661671,-0.9343460202,-2.0305268764,-0.6361606717,-0.4195115566,0.7116051316,0.6106399894,-0.2644272745,0.4211940169,-1.1107473373,-0.6500771642,-0.1556589901,1.8477299213,-1.4575940371,0.4544501007,0.5361469388,-1.8324573040,0.0428999849,0.5673006773,-0.3812290132,0.1586058289,-0.7432076335,-2.3003587723,-3.0178782940,-0.4192528725,0.3673498631,-0.3779532313,0.3696709871,0.7667483687,1.1323468685,0.1713690609,1.9920309782,1.0371128321,0.7972601652,-1.5348200798,-0.2240379602,-0.7211647630,1.2391200066,-0.5251198411,0.3672321141,-0.2873863578,0.1083054766,-0.3605248034,0.0712789819,0.9799603820,0.6624241471,-1.7817859650,-1.0848448277,0.5504264832,-0.4166675508,1.6126602888,-0.2049243599,-1.0211094618,0.3455152214,-0.5317375660,-0.7377064824,-0.4188673496,0.8521258235,0.2819546163,0.8450640440,-1.0280054808,0.9755204916,-0.0629693344,-0.6170272827,-1.3914250135,1.1874107122,0.1367413551,0.4460196495,-1.2611466646,0.0386255011,-0.9642368555,0.4145136178,-0.1752219051,-2.5754177570,-1.6540249586,-0.0349275842,0.0448959023,-0.4018982351,-0.7180095911,-0.0794271529,-1.0193690062,0.0466088690,1.0369007587,-0.4155933261,-0.6224948168,0.2363770604,2.0348365307,-1.3155070543,1.4027558565,0.7323297858,1.2334948778,1.3659120798,-0.0808410868,0.5328992605,-0.0881688148,-1.8309184313,-0.5434551239,0.6239242554,0.8248231411,-0.5365520716,-1.2415540218,0.7438274026,1.1258167028,0.1619975269,0.2171468437,-0.0945205837,-0.7705362439,-0.2652542293,0.3964420557,-1.0244829655,1.4870411158,-0.1595176905,-2.5220484734,0.8388492465,-1.2317235470,-0.9562405944,-0.6581323147,-0.2054009885,0.0864707008,-1.5467650890,-0.0167100113,0.0975208133,-1.6225047112,-0.6531865597,1.1103789806,0.1780981272,1.4125519991,0.6659147143,0.7821815014,-2.0617392063,0.8627902269,-0.3878584504,-0.1652761549,0.0142101552,0.0475623906,0.8808799386,0.0930731744,1.9853578806,0.2075347602,0.0492352843,1.8051396608,-0.5307197571,-1.1836994886,2.3241052628,-0.3693919778,-0.0356154442,1.7207887173,-0.0219147708,-1.0829179287,-1.5767006874,-0.1409223974,1.3477866650,-0.1534931064,-1.1728582382,-0.7144800425,-0.2241252959,0.3540037870,-0.1153135300,-0.9265352488,0.1685252041,-0.6477338076,-0.5638371706,0.1049879938,-0.0841720998,-0.5061149597,-1.2376722097,-1.2302114964,-0.2418429554,-0.0475620590,0.3470554650,-0.7907449603,0.0829786956,-0.9652466774,-0.8953530192,0.7561290264,0.4937201738,-0.3596648276,0.9256682992,1.1828033924,-1.1108970642,0.6139956713,1.3882287741,-0.2518001795,-0.7563911080,-0.9352231622,0.9392328858,0.5460475087,0.9603745937,-0.9488556385,-0.4981226325,-0.4026987553,0.5920459032,0.0090565272,-0.8863218427,0.0490998067,-0.1781470776,0.2800677419,-0.9707691669,1.3887768984,-0.6077256203,-0.2242699414,1.9329805374],[-0.8913836479,0.0187778343,1.3015447855,1.3718682528,0.5478003025,-1.0495789051,-2.1708903313,1.3192451000,-0.9026982784,-1.4011514187,2.0441510677,-2.0233173370,-0.0080679497,-1.6217567921,0.4429032803,0.0473358221,0.9135802388,-0.0456274971,0.0288143251,-0.5566142201,-0.1602125764,0.7894438505,-2.3435871601,-0.8936960697,-0.6324608922,1.4873902798,-0.7355272770,0.0185255464,0.2910588682,1.8088326454,-1.2172501087,-1.4750502110,-0.5276563764,2.0352544785,0.4579901695,0.0531067401,0.1881678998,-0.8126240969,-1.0810470581,0.6841163635,1.4170793295,0.4437287152,-0.0769727081,-1.0915343761,-0.4374568164,1.6740102768,-2.9241530895,1.6825302839,-0.1471130699,-0.3678592145,0.1373748779,-0.2201071382,-2.5014238358,-0.6038212180,0.7263562083,1.2062644958,-1.0378527641,-0.2458584905,-1.3896532059,-0.0786310136,0.9795127511,-0.2071323991,-0.4947267771,-1.6725231409,0.2150016129,-0.6723101735,-0.2168543935,-0.9003168344,-1.2477769852,-0.1771844625,-0.6590757966,1.1977955103,-1.5937381983,-0.7925133109,-0.6221466660,-1.6597610712,0.4350447357,-2.5738360882,-0.6248568296,0.3362247050,1.0735864639,-1.4852718115,-0.2946455479,-0.3111856580,1.2795861959,1.4509670734,-0.9087712765,-0.8750560880,-0.6453881860,-2.5647492409,0.2437494695,0.5734773278,1.1027342081,-0.7475714684,2.0953369141,-1.3439731598,-1.2973996401,1.1596117020,-0.9503767490,0.0014978965,0.1826970875,0.1884649992,0.3975955248,-0.6645811200,-0.1333314031,-2.8415510654,0.4180964828,-2.7764372826,0.2658073008,0.2275587916,1.4011553526,1.5555111170,0.8428824544,0.9028973579,0.3212300241,-0.5996940136,-1.0242420435,-0.6158038378,1.6927067041,0.2687541246,-0.3995000422,-0.3005733490,0.2216373682,-0.2204890847,0.6902763247,-1.4169993401,-0.8151863813,-1.9904801846,0.4423972666,0.6706801057,0.7185264826,-0.2972773910,1.1108764410,-0.8177000284,-0.5754866600,1.1977572441,-2.6281495094,1.5867403746,1.4161043167,0.7671688199,0.1658380479,0.8150725961,-1.6277327538,0.3003188074,1.3336789608,0.8292962313,-0.4848930836,0.1618258804,-0.5849781632,-0.6967284083,-1.1470516920,0.7196304798,-0.9395693541,-1.2485228777,0.9772579670,-0.9749694467,-0.9194549322,0.1552649140,-1.3407961130,1.0570300817,-0.0778691396,1.7038300037,-0.2739207745,-0.3102475405,-1.7548904419,0.4191471636,1.2825931311,-0.5296666622,1.0294588804,0.1527706832,-0.8580433726,0.4660653174,0.2710532248,-0.7573392987,1.7085819244,0.2593532801,0.2727315128,0.0305096321,0.6543181539,0.7176415920,-0.2725006044,1.1646317244,-0.1886187494,0.6548362374,0.7484856248,1.5596144199,-1.2608844042,-0.6304160357,0.6321176887,0.2912154496,-0.8312640786,-0.2090908885,-0.7051216364,-1.5571593046,1.7400833368,-1.1747038364,-0.3725011945,-0.3637978733,0.4852464497,-1.6679120064,-0.4092452228,0.4874619842,-0.0685126632,-0.2348806113,0.1828349084,1.0253461599,-0.3952368796,-1.0767440796,-0.4530940652,-0.5046754479,-0.4909371436,-0.3833635449,-1.8254991770,-1.1022430658,0.3569259942,-0.2243722230,-1.7885184288,0.9822217226,-1.4025393724,0.8213406801,0.3869772255,-1.1860983372,-1.2052828074,-0.1533375829],[1.8314754963,0.8154265881,-1.0724186897,0.8384286761,-2.0999834538,0.2436810583,-0.0349125080,0.7932613492,1.1110419035,2.4974799156,-2.8381979465,-1.0605124235,1.0589070320,0.9585606456,0.8247618079,-2.5230290890,1.5672498941,-0.2026850730,2.0308806896,-0.0637612268,0.5324293375,0.9650161266,-0.3561647236,-2.2441608906,0.5721330643,-0.0420632027,0.8333802819,-0.1294799000,0.2182023227,0.3625458479,-0.3400902748,-1.3537571430,-1.2952094078,0.7723783255,-0.0648406297,-1.2400768995,1.6827946901,1.1401575804,-1.5258969069,0.8159262538,-1.1959249973,-1.5505919456,-0.6034004092,-0.6083353758,-1.0715056658,-0.1194637045,0.0140260626,-0.9964612126,-0.1172712520,-0.5439452529,-1.6995549202,-0.0008625090,-0.9978842139,0.3596127033,-0.7026296854,-0.5324466825,-0.7600529194,-1.3232053518,-0.5340525508,-2.1644289494,0.4067259431,-1.6405332088,-0.3487933874,1.1609245539,0.2904104888,-0.1469300389,-1.5347036123,0.2856205404,0.0134263178,-2.1771950722,1.0045795441,0.8746539354,1.1157619953,-0.0907416940,1.4753232002,0.5067855716,-0.7191897631,-0.3575884700,-0.1088900194,-0.4578658938,1.6910647154,-0.3512211740,-0.3477337062,-0.0016170363,1.6011307240,0.8124353886,0.1603957415,-0.3600437343,-0.2487288564,-0.5773665905,0.8298076391,-0.4896481931,-0.7197877765,-0.0610746816,-0.3374556005,-1.5635737181,-0.6105449796,0.5172725320,-1.2310928106,0.6385520101,0.2386399955,1.5264911652,0.5826241374,-0.6799334288,-0.5346289873,-0.2036975175,-0.7439308763,0.4147465229,0.0078077312,-1.0435984135,-1.2097308636,-0.3265855610,-0.8725820184,1.7610948086,-0.9714349508,1.5349853039,0.2814086676,1.2379722595,0.4718837142,1.1676279306,-1.2804001570,-1.5776510239,0.4372701347,-1.1033475399,0.6475730538,0.4473706782,-0.1467706263,0.7794570923,-0.6803262234,0.9622517824,-0.8587225080,1.2174996138,0.2897004187,-2.5981965065,-1.5418102741,0.0819696561,1.5873248577,-0.7349403501,0.7290760875,0.9269818664,0.7080751061,-1.2577210665,1.1770211458,-1.9203807116,0.7047927976,-1.0936950445,-0.2829663157,0.6887729168,1.2502661943,0.1848202497,0.4263668954,0.0182026662,-1.5792089701,-0.3927980363,0.8779292107,-0.4019509554,0.4690700173,-1.5016077757,0.3696859777,-0.9097312689,0.7302269340,-0.7808380723,0.5710436702,-0.6982168555,1.8683104515,0.0536289178,2.7342181206,0.9965337515,-1.3732835054,0.5279285312,-1.5946539640,0.5114824772,-0.0093094138,-1.9116069078,0.5795866251,-1.3237437010,-0.9147019386,-0.5839483142,-0.1468760073,0.9125059247,-0.1213563755,0.6303017139,-0.4348902702,1.0431065559,-0.3461530209,1.0725315809,0.9376847744,-0.6338280439,-1.7862892151,-0.8356691003,0.4631983340,-1.3741754293,-0.1512538940,0.4590940177,-0.7534619570,-0.8041709065,-0.1798563600,-0.2793759406,1.3496509790,0.9117848277,0.6284425855,-0.1181348413,0.5227888823,-0.8071405888,-1.4427751303,-0.5251013637,-0.4712083340,-0.4328814149,-0.0625080392,0.2701307237,-1.2395265102,1.9819504023,0.5889578462,0.7082515955,0.1354319900,-0.4342207611,0.8350552917,-1.5380432606,2.2015075684,-1.2920140028,1.6503757238,0.4370865226,-0.6922412515,-1.8327370882],[-0.1175616831,0.0189046878,-0.1070931181,-1.7652977705,0.5713495612,-0.8332467079,0.3077534735,-0.5451762676,0.5432214737,0.9431089759,0.2331315875,0.4595543742,0.6053746343,-0.5241636634,0.6633955836,-0.5136868358,0.3695436716,2.5949618816,-1.8505828381,1.3736778498,-0.2972381115,-1.7225773335,0.3269241452,-0.9890488386,1.9549521208,0.5292824507,0.6432071328,0.2097698003,-0.3138158023,0.0110373385,-0.8368542790,0.6338981390,2.1725356579,-1.3096066713,0.5576665401,-0.8026767373,-0.0024915286,0.7201132178,1.1928230524,0.2271231711,-0.0062309671,-2.3676698208,-0.2402453274,-0.5720393062,0.8346250057,0.1938629746,0.6820136905,-0.1138827205,0.0992246643,0.6323102713,-1.8433687687,-0.2471316308,0.8616644144,0.6367298365,0.6539236307,0.5544044971,-0.3177534342,-0.7643328905,0.5900797844,0.7913125157,0.7577337623,0.7785899639,-1.0440396070,-0.6633660197,1.5012997389,-0.0091627706,0.7560212016,0.8975538015,-0.9736179709,-0.2276934981,0.9973939657,-0.3334585130,-0.8088312745,-0.3517300487,2.3488607407,0.6854371428,0.3064183891,-0.4391704798,2.6296787262,0.2384973019,-0.9323630333,-0.1401946098,-0.8720086813,0.7077708244,-0.3136989176,-0.0932615995,0.6460273266,-0.4569582045,1.7222473621,0.2486722022,-0.5947818756,-0.5724456310,-1.2928011417,0.3518567085,-0.2768443227,0.1744876951,0.7715813518,-0.3155222535,-0.6012001038,-0.0778921768,0.6547526121,-1.5268802643,-0.8698882461,-0.2165810913,1.1484020948,0.6043649912,-0.7420395613,-0.0446573719,-1.3837007284,0.5973103046,-1.2017232180,-0.4692558944,1.0996962786,0.4843816459,0.3924929500,0.5177378058,2.1317279339,1.7797539234,1.8023210764,0.5402734876,1.6492481232,-1.1998282671,-1.1383824348,-1.3544220924,1.0511277914,-0.3663887382,-0.0666134506,-1.7126017809,-0.1563147753,-1.6851294041,-1.6602188349,-1.3355588913,-0.2164773196,1.3588712215,-0.7749710679,-0.5971747041,-1.5134319067,0.6159718633,-1.0240488052,0.4399787486,-0.2938184738,1.3824003935,0.9063398838,1.7439777851,-0.8023254871,0.7868092656,0.0004580946,0.4446972311,1.3736970425,-0.3319808543,1.6903849840,-0.8114499450,1.7070233822,0.3595496714,-0.1166067645,0.6463969350,2.5515398979,-0.9076561928,-0.1777824014,-2.1957619190,-1.9981634617,0.4334587753,-0.7519865036,-1.0104613304,-0.7528412342,-0.2588641346,-1.2371978760,-0.0698891729,0.2212488353,0.4128143191,0.5448367596,-0.3559421003,0.1989956051,-0.7013852000,-0.6775157452,-0.4674378335,1.0175814629,-0.9423191547,-1.4644925594,1.0119210482,-0.5379062891,2.3790464401,-0.7766230702,-0.2486738861,-1.7352688313,-0.0490255542,0.3822934031,0.7362163663,0.7763320208,-0.1527945995,1.2485646009,0.2239975333,0.6150395274,0.1295638382,0.0960831046,-0.1421104372,0.6133358479,0.8356537819,0.7742197514,0.1762249768,0.1703704149,-0.3257652521,0.5471190810,-1.5459822416,-1.5104633570,-0.3404698074,1.1019839048,0.8252863288,-0.2277154028,-0.4301207066,-0.0688877627,0.8002319932,1.4292235374,0.4345422983,-1.4420714378,0.1606875360,0.3235762119,0.1411382407,0.1357653439,-0.4222933948,-1.0847389698,1.8023097515,-1.3667072058,1.5778913498],[-0.0131876403,0.3190685511,0.8450273871,1.1641961336,-0.7162372470,0.3938809037,-0.3219540715,-1.7239831686,-1.1850391626,-0.2674075663,1.1804103851,-0.9732410312,0.1530347764,-0.5131613612,0.5818021894,-0.1796519756,-0.3842805624,0.4687068462,-0.3080357015,0.6460461020,-1.9881218672,-1.7341786623,0.0405684933,-1.2557553053,1.1959751844,-0.2807148993,-1.1054583788,-0.9994213581,1.3731906414,0.5248828530,1.1866965294,-0.8613355756,-0.2546068728,-0.4799605608,-1.8969695568,0.2036823183,0.0273817368,-1.9169287682,-0.9908143878,0.1783454567,0.4995328784,-0.3635697961,-0.0450246893,1.6701726913,-0.1013248339,1.0435426235,-0.8696965575,-1.1632086039,0.1831671149,0.6457328796,0.2006639540,1.5764163733,1.4348359108,0.4867582619,0.0998035371,0.2122234851,-0.0686026067,-0.2263608724,0.3516403437,0.7742586732,-0.3687585294,-1.5238984823,-1.5085883141,0.2777234614,-0.3668000698,0.1228307188,-0.8946448565,-0.5748972893,-0.0552437901,0.1306430250,0.4406910539,2.4174592495,-0.9714004993,-0.5191168785,-0.5188544393,1.3178100586,0.1041980833,1.2463740110,-0.0790428743,0.6186703444,1.2864513397,-0.8152043819,0.4917009175,-1.1067799330,0.0927638784,-0.1811284125,-0.4697842598,0.3500574529,0.3294185996,0.1947860867,0.8966708183,1.9381089211,-0.9195051193,0.7260957956,0.8662853837,-0.5145770311,0.1757200509,-0.8238931298,0.1056567132,-0.1374659091,-1.2861822844,0.5849096775,1.8646203279,-1.6820768118,-0.9834036827,1.3807744980,-1.2435959578,-1.6130899191,-0.7388973832,2.2242629528,-0.4154294133,-1.7480731010,-0.2249363214,-0.7993203402,0.6460397840,0.1766046137,-1.6437238455,-0.0110550970,-1.4476402998,-0.2804417014,1.2617326975,-0.2355776876,-1.6873000860,0.8121863008,-1.6272295713,0.7626906633,0.1414132714,-0.9910776615,0.8963002563,1.4895867109,-0.7519250512,-0.1140206978,-0.2397381216,1.8317588568,0.9710619450,-2.1389555931,-1.1381371021,-0.7534425259,-1.2349971533,-0.9838739634,0.7869290113,0.7393561602,-0.9010815024,-0.1183883175,-1.5193796158,1.0234062672,0.4929726720,-0.5983226299,-0.8546621799,-0.9928486943,0.8580879569,0.8566496372,0.3198273480,-0.2430842817,0.2529213130,-0.4792161882,-1.4864950180,1.4555321932,1.3306303024,1.2000240088,-1.2556482553,0.4722168446,0.7385764122,-0.3788935244,-0.4737539589,0.9722071290,-0.9189504981,0.6632404923,-0.1334913969,-1.5663703680,-1.7486513853,-1.1225919724,0.7985507846,-0.0880931318,-0.2178262472,0.3675931692,-2.1636779308,1.5768371820,0.0332911499,-0.6151152849,0.7404027581,0.7260823846,-0.8314467072,0.6558555365,-0.3027761579,-1.3855255842,0.0839681327,-0.8914278150,-1.5680286884,-0.6328111887,-0.1193251312,0.2832503319,0.2091138959,0.0765040293,1.0707458258,1.1313108206,0.6235074401,1.1784594059,-1.4363331795,-0.4905205369,0.1248268560,0.3911684453,0.2099477947,-1.4990891218,0.5412989259,-1.1710778475,1.2187296152,0.6991159916,1.2686791420,0.0984257534,0.1025270298,1.6023763418,0.3297880888,-0.9317253828,-1.2777259350,1.5729357004,0.7261555195,-2.7456653118,2.2690129280,-0.4050147533,1.7808687687,1.9016879797,-0.3218570054,-0.2043680102],[0.1489765644,-1.4593832493,-1.5017405748,-0.8670356870,0.2126237601,1.3351058960,1.6312758923,1.6304445267,-0.4380942583,-1.3650913239,-1.4511771202,1.0316030979,0.8142898679,-1.6095172167,0.7330580950,-0.3812020123,0.5772187114,-1.0423914194,-1.7273218632,-0.0568555333,1.2256610394,1.6008268595,0.4206520319,-0.5724737048,-0.2653441727,0.5704436898,-0.2628526688,-0.7718463540,-0.1277451515,1.1820430756,0.4724800885,1.4606692791,0.0491502918,0.4095941186,-0.5304942131,1.3007168770,-1.6952812672,-0.3756690323,0.3484024704,-1.6185001135,-0.1921009719,0.0708475932,2.4357285500,0.9716811776,-0.9329621792,2.8652036190,-1.7920479774,0.5449343920,2.4494490623,0.0170834791,-1.3960881233,0.5912993550,0.7908234000,-0.2101195306,1.0908360481,0.9109295607,0.1053518280,0.9273505211,0.8930205107,0.5378038287,-0.6903609633,-1.1878529787,-0.4941872358,1.1770657301,1.0582617521,0.5110420585,0.5188018084,1.3615533113,-0.2138238251,0.4488541186,1.4647057056,-0.3835515380,0.7276459932,1.1939560175,0.2122646570,-1.0129065514,-0.5653158426,0.1305658966,0.2739067674,0.7412998080,0.0615668334,-0.4668753147,2.5491123199,1.0991567373,0.6903136969,0.6747993231,-0.4939589202,-1.8001532555,-0.2043026686,0.0035368698,-0.0057333200,1.0612637997,-1.4626442194,0.4315978885,-0.6228010058,1.2404770851,0.1704462916,1.9868983030,0.1113011688,1.2096550465,-0.4573297799,0.9231835008,1.3255188465,-0.9531663656,-0.6781297922,0.3340995908,-0.7549282312,-0.3883825243,-0.6265756488,-1.3064211607,-0.0384958498,0.1435673833,-0.7890736461,-0.0985037014,-1.4528664351,-0.9478659630,-0.8695787191,-0.3055172861,-0.6051473618,0.2248205394,-1.0384724140,0.7718191743,-0.4800092876,1.8347450495,-0.1087654233,0.4544749856,0.0987093523,-0.4583054483,-0.8570440412,-0.7744029760,0.7087873220,0.0184736028,-0.1040805802,-0.2348577380,-0.2509063482,-0.9627502561,0.4934051633,2.1330499649,0.9838882685,-1.8092817068,0.0423588157,0.5168721676,-0.0329206921,1.2981114388,-0.2021170259,-0.8332309723,1.7336002588,0.1906490028,-0.1778104007,-1.0850250721,0.9722623229,1.4314717054,-0.2450730503,-0.6232538223,-0.3959988654,-1.3585224152,1.3226025105,0.0034439384,-0.0001967701,1.5337039232,-0.7730822563,-0.4012549520,-0.1153567657,1.0115554333,1.5132355690,0.9170686007,0.7288931012,0.5520321727,-2.8032341003,-0.3960531652,-0.7131436467,1.3008337021,-0.5528658032,2.0708973408,-0.3659083247,-0.8976300359,1.4449365139,0.1729418486,0.2826884985,1.0908695459,-0.2013481557,-0.6806857586,0.8032750487,-0.3421638310,-0.4724888504,-0.6922569275,-0.0466712639,-0.6195800900,0.8695260286,-0.1468828470,0.3280917704,-1.3630435467,0.8078343272,0.6606640220,0.7788500190,1.5073202848,0.3795762062,0.6519295573,0.3520001471,-0.4837388992,-0.1357428730,-0.1349714249,1.0053361654,-0.8543254733,-1.7947654724,0.9829106331,0.8027749658,-0.4673963487,-1.4142329693,-1.1199934483,0.8871966600,0.0762820542,1.2231287956,-0.6656363010,-0.3412737548,-0.5046559572,0.3275733590,1.1423867941,0.7465314269,1.7910561562,-0.0725371093,1.2476819754,0.0711429790,-1.8477734327],[1.9659148455,-0.2410738766,-0.1578042954,0.5254576206,-1.9116709232,0.9348053932,0.8459461927,0.1184646636,0.5029144883,-0.1982770860,0.7655358315,0.9073389769,0.1973911673,1.3738481998,-0.9893300533,-0.9891023636,-2.4769854546,0.7702542543,-0.0861265808,-0.8576679826,0.6391736269,-0.2472003400,0.2333795726,0.8708141446,0.6738551855,1.6272670031,0.7516965270,-0.7883411646,-0.8082208037,-1.3736037016,-0.4045992196,1.0280630589,-1.5187823772,-0.4838001728,0.6150732040,0.8216204643,0.1266588718,0.5752500296,2.1062376499,0.8944895864,-0.7871508598,-1.3972195387,0.3273196220,0.7946544290,-0.4160199165,0.7112743855,-0.0405380018,-1.7044556141,0.5837534666,-1.0075877905,0.7251850963,0.9467189312,0.5067328811,1.6449491978,0.5131101012,0.3260095119,0.5165187716,0.1619718522,1.9142762423,0.6852089763,2.6714634895,-1.2615832090,0.0633755252,-0.6122508645,0.7064356208,-0.1490675062,0.3214509189,-0.3426703811,0.3457258642,-0.4682802856,0.2918916047,0.4650507569,0.1249749437,0.5599335432,0.6786236167,-0.4045546353,0.6692442298,0.4938224852,1.2209396362,0.1036879346,0.6975646615,-0.6695911288,-1.2909108400,0.1948612928,0.1942252070,-0.6748269200,-1.9970334768,2.4057705402,0.6200784445,0.3772981763,1.5525032282,-1.4655271769,0.3644940555,-0.0155226057,1.1985623837,-0.4178135395,-1.6951625347,0.8210743666,-1.7736728191,-1.0594503880,0.7284531593,1.1591846943,0.2815538645,-0.3945144117,-1.9853501320,-0.4423422217,-0.9395880699,-0.3203176856,-0.9223166108,-0.5297619700,1.6634757519,-2.5264670849,-2.2690434456,0.5216883421,-0.3797107339,-1.0727113485,-1.2990864515,0.1797483265,0.2679230273,0.7701867223,1.3195123672,-0.4258531332,0.0932302922,-0.1008663177,-0.0527959503,-0.2420887053,-1.2358770370,0.2188131213,0.2474349886,0.0911240950,0.9356385469,0.6642363667,-1.7988439798,-0.9775348902,1.0068738461,-0.4412638247,0.2630724311,-0.2938269973,1.2398545742,0.1530414075,0.0919402540,-1.4946030378,-1.6424170732,-0.3924821019,0.9380593896,-0.7755195498,1.4306209087,-0.5973093510,0.4153765142,-0.4966987967,0.4551673830,1.3969449997,0.9603327513,1.6694878340,-0.7250531912,-1.1356847286,-1.2331088781,-0.2271185815,-1.2854925394,-0.9318059087,-0.7606769800,3.2205018997,0.2209428847,-0.0629122630,-0.6298869848,-1.0732094049,0.0010558150,-1.0177214146,-1.5743277073,-1.6354320049,1.7043139935,-0.6617698073,-1.6599169970,-0.3333908916,0.6777613759,-0.0104675246,-1.1323215961,-0.0904404595,-0.5879105926,1.0276099443,1.4621688128,0.2452373505,-1.8568937778,-0.2075275332,0.0924542099,-0.5434443355,-0.5015999079,0.3858227134,0.5888552666,1.1586396694,-0.7475170493,-0.2434626371,-0.1310633272,0.2667567730,0.5259755254,0.1121224090,-0.5332795978,0.8407949805,1.0488711596,1.6149709225,0.9318397045,-2.2712433338,-0.7382249832,0.0473290794,0.1856919378,-1.3058172464,-0.7064233422,-0.1791879386,0.5038546920,-0.6263046265,-0.0282821339,-0.7154991627,0.4878412187,-0.0843407884,1.7737895250,-0.4433949292,-1.0820724964,-0.5127086639,-0.4460237324,1.8900141716,1.3373774290,0.3039647639,-0.1128789261,-2.2114961147],[-0.7659804821,0.0121604325,-0.5553606749,1.8555424213,0.9179546237,0.2352409810,0.8584582210,-1.1751790047,1.6242084503,2.2770774364,-0.1804874986,0.5590753555,-0.1992742270,-0.5453976393,0.5759513974,1.1652381420,1.6872155666,1.8794616461,-1.1923128366,-0.3455305696,-0.0926098600,0.0298121963,-2.2340764999,-0.0128461355,1.7809996605,-1.7698057890,-0.1657604724,-0.2962538302,0.3984484971,-0.2784538865,-0.7657407522,0.5881029963,-0.0268511921,1.2491877079,0.5969085693,0.8053591251,1.2250876427,-1.6654818058,0.7458550334,0.1767183840,-1.0294940472,-0.3047012985,-0.8348066211,-0.5481224656,-0.2275418341,-0.3919983208,1.7537639141,-0.4548853040,-0.2024148703,-0.7957806587,0.7550222874,0.5483199358,-0.6040002108,-1.0742310286,-0.2339738905,-0.7312679887,0.6031473279,-1.9090847969,-0.4644692838,1.8127008677,0.5727068782,-1.1307305098,-0.2126340717,0.9141866565,1.0289263725,-0.9336540699,0.9685837030,-0.4468255639,-1.3155292273,0.9664657712,-0.1305926740,-0.1409974098,-0.2777955234,0.1934855133,0.2790493965,-1.0827175379,-0.6172372103,-0.9062939882,1.6249476671,-0.8238903284,0.6574448943,1.3339018822,-0.7396215200,0.0353739858,-0.9580649734,-0.1081325933,-0.5253806114,0.8399999738,-0.6203387976,-0.1352863312,1.8578811884,-0.9981376529,1.3206781149,-0.0126401605,0.5569839478,-0.1270247847,0.4880434573,0.5574302673,-0.4547669888,-0.6072672606,-1.0674308538,-0.5023055673,-0.4857284427,-0.0919217393,-0.9623599648,-0.0035295901,-0.0758933797,0.0933261737,1.2281299829,-0.0178553499,-0.4720449448,0.2214139700,1.4468604326,0.4306001961,1.0677331686,-1.6240931749,-1.1872859001,-1.1063474417,1.1893283129,-1.3409972191,0.7507525086,0.2144597620,0.3168707788,1.2522690296,0.9644386172,2.1303524971,-0.7632300258,1.7955607176,-0.3486187458,-0.3859466910,0.2825184762,-1.0904860497,0.2479778975,-0.3009818196,0.4486035109,0.9551592469,1.4568217993,-0.6479675770,-0.3713506460,1.0168209076,1.2009193897,1.0800970793,-0.3941484392,-0.1019895002,-2.3731226921,-0.3524095416,-1.0711951256,0.5553830862,-1.7090671062,-0.6871978641,0.0623640977,0.0408379324,0.4368767738,-0.9474046826,1.3001302481,1.1587718725,0.4988572299,0.2281088084,-0.6101936698,-1.8242050409,-0.0441021472,2.4481766224,0.9330115318,-0.1531505287,0.0430765450,1.4130680561,1.1317970753,-0.8838879466,-0.4641161561,0.8401603103,-0.1360720992,0.7551409602,0.9949775338,-1.5624421835,-0.3360073268,0.2582748532,0.6022772789,0.1784233600,0.7269070745,1.0488575697,-0.3621068895,0.6744479537,-0.4786558449,-0.2334593087,0.4071067572,0.2839189470,0.7738007903,0.4134279788,-0.1320126951,1.0825728178,-0.8175003529,1.4486014843,1.0853774548,-0.2358137816,-1.6589474678,1.1178635359,1.0709910393,-0.3829950988,-0.1981175393,-1.3988100290,-0.0638643652,-0.1147952601,-0.3831200600,-0.8857003450,-0.8296794295,1.4530539513,-1.6511634588,-2.5654265881,-1.1252266169,0.3472560346,0.9780961275,-0.7914997339,1.9477938414,0.3580872715,0.1118140593,-0.2804694772,0.4610228837,0.6023619175,-0.5335633159,0.0542477593,-1.7664861679,-0.0508076660,1.9950895309,1.5672857761],[1.4353256226,-0.5908765197,-1.2080715895,-0.6752406359,-0.7683525085,0.2406216562,-0.8646903634,-1.6352556944,1.7879844904,0.3776165843,1.5704457760,-1.4466234446,0.8425059319,-1.2013957500,-0.3469410837,-0.7660391331,-0.6346268058,1.6559984684,-0.6233503819,0.4907566309,0.8447885513,-0.9813486338,-1.4458115101,0.6337310076,-0.5561428070,-0.5065686107,0.1102068424,-0.5101391673,-1.1948239803,0.7441099882,0.0857115984,0.0515982620,1.1483319998,0.7804873586,0.5419780016,1.2188420296,2.0769410133,-2.4862742424,-0.4223719239,1.0099729300,-0.7526311874,0.0810160041,0.5616210103,-0.4127331972,0.1325272769,-0.5238465071,-0.3826581836,0.0631827638,0.2585388422,-1.0420076847,-0.4863687158,0.0387250669,-0.4188231528,0.0140505927,-0.9987033606,-0.8189831972,0.2218448222,-0.5424128175,-1.5457696915,-0.4346087873,0.4255478680,-0.2396979183,-1.6252154112,0.3310910463,-2.3621444702,0.2520245612,-0.2275166214,0.3297084570,0.9491755962,-0.0028857999,0.3958771825,-0.3808728158,-0.7215411067,0.9620351195,-0.9630186558,-0.4433543384,1.2981798649,0.0506372303,1.3202215433,-0.0137817934,-2.1586008072,-0.2851150930,1.0693598986,-0.4877946377,1.0574339628,-0.5105757713,-0.5830003619,1.8261824846,-1.2577497959,2.4384016991,-1.2908158302,-1.2552080154,-1.0708709955,0.3558755517,2.2702765465,1.6279611588,0.9271426201,0.4194103479,0.6996855140,0.2764927447,0.0600915737,-0.2719833255,-0.0216664709,-1.0817744732,0.2898022234,-0.7958546281,-0.5543126464,0.0587645993,0.2073607743,1.6387189627,0.9563472271,-0.1144778430,0.4083743691,0.3125553429,-0.0443716496,-1.4478203058,-1.4528867006,-1.0498052835,-1.0768823624,-0.9817907810,1.0057474375,2.1739764214,-0.1706229895,-0.3973425627,-0.8106067777,-0.3117626309,1.7385970354,0.4636293352,-0.3924411833,-0.5573529005,-1.3127901554,-0.0500530154,-0.3318820298,0.6395338178,0.1124192551,1.6188535690,1.5975137949,0.7731382847,0.1306872517,-1.3235602379,-0.4722246826,-0.2985983193,-0.4397483170,0.9028589129,0.7685422301,0.9230428934,1.6842801571,1.4418077469,-2.5131144524,0.4086238444,0.0655701682,-0.3320867121,1.7466781139,0.3734170496,-0.2728291750,-1.6017853022,0.7018960714,0.4604117274,2.3683567047,0.4982431233,-0.3625016212,-1.7402305603,-1.1604846716,-0.0124342721,-0.0771125332,1.2629164457,0.2355280519,-1.0599642992,-1.4503680468,0.1968804300,-0.3745819032,-0.6852248907,1.8013814688,0.4930929840,0.7483561039,2.1840589046,0.0830427632,-0.6732596159,1.0215715170,-0.5686770678,0.8424426913,-1.9740310907,0.6989012957,0.0312862769,1.7320979834,1.0707975626,-1.0670249462,-0.2167563587,1.7714269161,-0.2345068604,2.4865136147,0.2414270341,1.7394777536,0.4267886877,-0.6875585318,1.1192632914,2.2618951797,-0.1167902946,0.4676852822,1.3006483316,2.9003543854,1.5146406889,-1.1470533609,-1.2451536655,-0.5879987478,0.5812782645,2.1253869534,-0.0506866947,0.4011137187,0.7882096171,-0.6160338521,-1.2647396326,-0.2363180667,-1.5727478266,-0.0531756170,-0.5144679546,0.2395006865,-0.8833888769,0.2919378281,0.2528843284,1.3525061607,0.1802900583,-0.9587054849,-0.1820952743],[3.0571014881,1.5167911053,-1.0126098394,-0.1719368100,0.4079130590,0.1754129082,0.3528579772,-0.2692727149,-0.8049977422,0.0460437089,-0.0798596963,1.0721807480,-1.1028991938,-0.6041131616,3.4571788311,-0.4846538603,-0.6901526451,1.0774567127,0.1411880702,-0.4632414579,0.1769790202,0.8287273049,-0.7865691185,-0.8834164739,0.8981291056,-0.3418011665,0.4376186132,-0.8193335533,0.7731699347,0.3516902030,2.3642060757,-1.3887287378,-0.3825967014,0.6642029285,-0.8499180079,0.8090372682,-0.0887294859,-0.7657515407,0.4094007313,-0.2589460313,-0.0710417926,-0.2994653583,0.1474562585,-0.0265487880,1.6107178926,-0.8812941909,0.0249902233,-0.3142261505,-1.5077643394,-0.5018021464,1.0165740252,-0.0183646958,0.3615891039,0.7975444198,1.2121458054,-0.6506839395,0.3548581898,0.2484272867,-0.2789663076,0.5858780742,2.3015351295,1.2746146917,1.0363812447,0.7774479985,1.1028864384,-0.1068114564,0.1970036775,0.1932538748,-0.3952465057,1.1583861113,1.4059976339,2.1219327450,-1.8358495235,-1.2085362673,-1.3249143362,0.4205504954,1.1037262678,0.9093187451,1.4752906561,-0.8287338018,1.0169249773,-0.6972437501,-1.4810217619,-0.7408619523,-0.6186295152,0.5369817615,-0.2822094262,-0.1443003863,-0.7923364043,-0.6148132086,-0.0632172227,0.1186423898,0.3373407423,-1.3994134665,0.5843821168,0.1835361719,-1.3693171740,0.6581259370,-2.2536697388,-0.7097249627,-0.2993446589,-0.7744182944,-0.7150697708,2.7478237152,-0.4464194179,-0.5393986702,0.5048452020,0.0309016295,0.7893177271,0.8253213167,-0.1216339916,0.8358047605,-0.5530250072,-0.6232783794,0.1180848405,-0.5877562761,0.3572616875,0.6083747149,-1.6882966757,0.3013758063,-0.9361636639,0.6878786087,-0.5113450289,1.3853739500,0.2846948504,0.4329951406,0.1250179708,-0.8994542956,0.2352239043,-1.2032506466,-1.7193822861,0.5931987762,-0.9778930545,0.2653068602,0.5067004561,0.5289206505,-0.8424483538,0.3913740814,-1.1664303541,1.4980700016,-2.3028693199,1.2116063833,-0.2439048588,-1.2052580118,-0.2917331755,0.0616047978,-0.6883229017,0.6314316392,0.2147449553,-0.4651604891,0.0913353115,-0.6877609491,2.5887622833,2.2688219547,1.6409702301,0.7080410123,-1.6533527374,-1.6032719612,1.0296443701,-0.1468841136,0.7079297900,0.7750738263,-0.5503200889,-0.1952609122,2.2069025040,-0.3737757802,1.3544288874,-0.1811147928,0.1393500119,-0.1114728898,1.0114481449,-0.6935762763,0.2787633538,-2.0546996593,-0.5709084868,-0.3062367737,-0.3156863749,-0.8897069693,-2.3231439590,0.4797623754,0.0417126454,0.0786912814,0.8310974240,-1.6643717289,-2.1256644726,1.2952638865,-0.9766326547,0.1436393559,-0.7110436559,-1.1480141878,-1.1452629566,-0.8019243479,1.5351486206,2.1707282066,0.7657800317,1.5620862246,0.1756275594,-1.3503173590,1.7917487621,0.7160843611,-0.1938566267,-0.4526286423,0.2835141718,-1.2592515945,0.3630310297,0.3490190506,0.4931244850,-0.8836598992,-1.2861914635,0.6278840899,0.5298740864,-1.4413779974,-0.2835259736,-0.5337915421,-0.5376721621,0.2081073076,0.9799979329,2.2777760029,1.2127616405,0.5642753243,0.0354485251,-1.0789268017,0.1125210300,-0.2152985483],[1.7759253979,0.9840850234,-0.0767186433,0.6679525375,-1.0115307570,-1.5240343809,0.1609255522,0.8578897119,-1.6950411797,1.2214843035,-0.1197355837,0.9035636783,-0.2301980704,1.8057299852,0.0134482170,-0.8037886024,0.4452606440,0.1248501688,-0.5301916599,-0.8451347947,1.1502391100,-0.1114130989,1.0182909966,0.1798265725,0.0725538880,-0.0725682154,0.6086733341,0.6504692435,-0.0388825051,0.6453584433,1.5192260742,-1.3471193314,0.3118595779,-0.6958822012,-0.8889675736,-1.8208668232,0.3896815777,-0.2915764749,-0.0712015107,0.4959938824,0.5245856643,0.1248565987,2.2854576111,-0.3773034513,-0.4099958241,0.2611141205,-1.1319282055,-0.8118616939,-0.0755043477,0.8247570395,-0.2658351064,-0.5308529735,-0.5981125236,0.5713994503,0.1442613006,-0.0215171184,-1.5180338621,-1.9154884815,0.9877078533,-0.2374628782,1.3302844763,0.1467256844,-0.2150432318,0.6785774827,-0.7292082906,-0.2542716265,0.4539424479,-0.0867115185,-1.2298870087,-1.1490353346,0.9752449989,-0.1623965055,-2.7027933598,1.0214067698,-1.6213696003,0.4340440631,1.3481116295,-1.2926946878,0.3220206499,-0.6569662094,1.6419132948,0.1643116474,-0.0132283932,-1.8333109617,-0.9346026182,-0.9406163096,-0.9235872626,0.6504324079,0.1044687778,1.2268142700,-0.2209423184,-0.2384083420,-1.2015172243,-0.5316178203,0.1752220094,0.6893568635,0.2345945686,0.4606382847,-0.6025770903,-0.8063659668,-0.5693377852,-1.8322895765,-0.4588719010,-0.4438908696,1.3813488483,-0.7166424394,-1.3205441236,1.6986227036,-0.9610489607,0.7402990460,0.5968779325,-2.0013959408,1.5075762272,-0.0389403030,0.4936818480,-0.3687821925,-0.3875725567,1.2237678766,0.3262037039,0.8998846412,2.1900782585,1.2703543901,2.1951560974,0.2156772017,-1.5663990974,-2.2078025341,0.0685291588,-1.5581027269,-1.0036827326,-1.3702378273,0.1694180816,-0.9137530327,1.0961093903,-0.1972991228,-0.8759288788,-0.5241637230,0.4103074372,0.1449525803,-0.7451390624,0.2333553135,1.1782257557,0.1412797421,-1.4361449480,-0.3597810268,-0.8911457658,-2.2995569706,0.0269727893,-0.5035428405,-1.4560674429,1.7810405493,-1.3310171366,-1.8869807720,-0.2242967784,0.7271280885,0.7559499741,1.4708317518,-0.0475333333,-0.5505061150,-1.2091144323,1.0972720385,0.1798228621,0.0975974947,0.6015082598,-1.5747113228,-2.3804230690,0.2242878824,0.4144230783,-2.0694725513,0.0689833388,-0.8785431981,-0.2263758332,0.2870897055,-0.1257645935,-0.5280503631,-0.6401610374,-0.9018902183,0.6883198023,-0.2870998979,2.0648100376,2.5321288109,-0.1234981641,0.3702547550,1.3969179392,-0.2872636020,-0.5175419450,0.1475308388,0.0770351812,1.0055582523,1.4378954172,-0.5518070459,-0.0454276241,-0.9547314644,-1.7417740822,0.5516366959,1.4494178295,1.2299993038,0.0344757847,0.8050377369,0.1586563140,-1.4417908192,0.8038492203,-0.4616938531,-1.5563237667,-0.2434473038,0.8458123803,-0.3156723976,-0.9523938298,-0.3382590115,0.1520356685,0.3631040454,1.6726390123,-0.6849400997,1.6337143183,0.3757461905,-1.1149408817,-0.1292464733,0.5845023394,2.7107896805,0.1954807341,2.0079009533,-0.3136621714,-1.4966130257,-0.4827790260,-0.5490916967],[0.8668015599,-0.4425382912,-0.3470218480,-0.0106614679,0.1458479911,0.9882314205,1.5065393448,1.0266677141,-1.6985048056,-1.2251684666,-0.0537470914,0.6280582547,-2.1100039482,0.0921582580,2.0108947754,0.1368566453,0.2666280866,0.5259639025,-0.5141676664,-0.1254386157,-1.9846127033,-0.6376038194,0.1830519587,-1.3448629379,1.7952604294,-0.6953235269,-0.4445840120,-0.2850975990,-0.6821460128,-0.6116260290,-0.0420884788,-1.1017171144,0.6697148681,-0.3835142255,-0.1298492849,-0.4696798623,1.4395703077,0.0283602029,0.4346496761,0.9028067589,-1.2745651007,-0.8085855246,0.2151665688,1.1069250107,1.0683196783,-1.1012074947,-0.2108737677,1.0316549540,0.0220382344,0.1956191957,-0.0308300909,0.9818850160,-0.9788653851,-0.4604631364,0.7173712850,-2.1900360584,-0.0521114245,0.0403597429,1.2453624010,0.5835828185,1.4235873222,0.9640865326,0.2892068624,0.2281533778,0.1888966411,-0.6644412875,-0.3662932515,-0.4654232264,-0.1265579760,-0.5279335380,-0.2821264863,0.5741770267,-0.9894772768,-1.6030364037,0.5362505317,-0.0642012283,-0.5990480185,-0.3981341720,-0.4884864092,0.8829444051,-0.9191870689,-3.5971634388,-1.1088981628,-0.2825691402,0.0801857784,2.0061070919,0.5381712317,-0.6989403963,-0.6342703104,0.6783562303,-1.7275750637,0.1030801684,-1.1218008995,-0.8425818682,-0.2069326192,-1.2354469299,-0.6567828059,0.1083241701,0.8044376969,0.2656084299,-1.1283918619,-0.7726039886,-0.2678033710,-0.8238725662,-0.7397587299,0.3420245945,0.6743623018,1.0771563053,-1.1560052633,-0.3988094032,-0.2347034961,-2.5536952019,-0.4352763295,-0.7365463376,0.6908099055,1.7416143417,-0.8961228728,1.1003487110,1.5549330711,0.7204515338,0.2944574058,-0.5237665772,-1.2406834364,0.1910576522,-0.5132317543,-0.1078897566,0.2912856638,-0.1775024086,-0.4434978366,-1.6129906178,-0.8831277490,2.1223089695,0.5924318433,-0.1580628008,-1.5093832016,-0.1052221730,0.4946533144,1.3044494390,-0.5286353827,0.5293801427,-0.0658370033,-2.1142232418,-1.6638370752,0.1901883334,0.0722268820,1.6248866320,-0.1596909165,0.9091033936,0.0457334258,-0.1473118365,-1.2650078535,1.2293517590,-2.2262835503,0.8036817908,-1.5079095364,-0.3022215068,-0.5339203477,-0.0028147821,0.9604151249,1.0207607746,-1.7765924931,0.3125726283,-0.5199762583,-0.7035210133,-1.5594059229,-0.9711892009,-0.8094975352,-0.2992601693,0.1038063467,0.4899407625,-0.4866145849,-0.4270956218,-0.1204021871,0.0000380513,-0.7106295824,-1.1571166515,1.2584540844,-0.9098292589,0.4606101811,-0.8967497349,-0.7487707734,1.1330285072,-0.2487686425,0.0485689640,0.9908419251,1.7194275856,1.1388652325,0.4050374627,0.7197656035,-0.0095453719,-1.7850903273,0.2807362080,0.3113456666,-0.4148780704,-0.5144474506,2.3370497227,-0.9018598795,-0.0432650372,0.6294213533,-1.2214990854,-0.3480480611,0.4941302836,-0.8427708745,-1.0920269489,1.1331650019,0.5445849895,-1.9988650084,-0.5136639476,1.5669808388,-1.4852890968,1.1697503328,0.1714032292,-0.1498004496,-1.7167567015,0.2077475339,-0.2835475206,0.6810171008,0.1799054295,-0.3234174848,1.2285729647,-0.2494829297,0.4951613247,1.2835344076,-0.2153073847],[-1.5655417442,1.0642517805,-0.4465774298,0.6258028150,-0.4518454075,-0.6958141923,-0.4009466767,-0.7438731194,-1.0151065588,0.2284471691,-0.5200065970,1.7738699913,-0.9808174372,1.6659055948,-1.2655656338,0.1619434059,0.5714436173,0.8214023709,-0.7151944041,-0.7368196845,-0.5527368784,-0.4281289577,-1.3470569849,0.2309796363,-1.1804646254,-0.1970812082,-0.6844750047,0.1222466752,-0.5097717047,1.0388318300,-0.0032982610,-0.9455435276,0.0521331877,0.9406508803,1.4224059582,0.7051896453,-0.0485493839,0.0030716613,-0.0828417838,0.3621097505,1.1506960392,-1.2411286831,0.6016132236,-1.8681024313,-0.2948068678,-0.5459722877,-1.3271028996,0.0146063436,0.0820118934,1.1872184277,0.0226242170,0.5577337146,-0.6835659146,-0.3828615844,-0.0172030423,-0.3059270680,-1.1533372402,0.2937498689,-1.1304335594,-1.5362212658,1.1067888737,-1.7159109116,0.1627099216,3.4915504456,0.8252794743,-0.6337646246,-0.6024153829,0.2472287565,0.0888550505,0.3549385667,0.6885216832,0.0299285725,-0.0664129481,-0.6327346563,-0.1739193201,-0.8258158565,0.1949875653,-0.2163696438,1.0570889711,-0.7573488951,-1.5297939777,0.3537744582,0.3407221437,1.3309781551,0.7501005530,-1.1971751451,0.3842400312,0.1106487289,0.9332572222,1.2433928251,1.0489282608,-0.7144296765,0.3993691802,-1.1924867630,-1.6388714314,0.6282942295,-0.3221462667,2.2317204475,-1.0032519102,2.5989425182,0.7262270451,1.8382234573,1.0665451288,0.6297884583,-1.7966475487,0.2026394159,-0.6314821839,-2.0691819191,-0.3227912486,-0.6972043514,-0.0853688121,-0.7535147667,0.3925627172,-1.5193345547,-0.2938063443,0.3562227488,-0.9631296992,-0.3119919896,-3.0219933987,-1.2542766333,0.3058491051,-0.0131095555,-0.0122315139,-0.0387849845,-1.0967736244,0.5564718246,-0.6666582227,0.3884274662,1.9603561163,-0.5804669857,0.8530720472,-0.3293070197,-0.7811429501,0.5486994982,-0.2156431228,0.4974171817,1.8149178028,-0.0831253380,-0.2076419592,0.4824213386,0.2035827041,0.2575459778,-1.2080625296,-0.9822015166,-0.8012655377,1.1425778866,0.7922763228,1.2311615944,-1.5688765049,0.7047674060,0.2851555347,2.2098674774,1.0570685863,-0.0321040228,-0.1283721030,0.3590631783,-1.3702954054,-1.4281027317,-0.2610388696,0.6530545950,0.4184088409,0.2936683297,-0.1556836814,0.9377830029,-0.1359964013,-0.8170792460,-1.2400767803,-0.4732416570,-1.1670972109,-0.8413016200,-1.4027423859,1.0371077061,-0.8688614964,-0.5326256752,0.5555218458,-1.4648450613,-1.0836731195,1.8800867796,-0.8621342778,0.5107352138,1.9502209425,0.6216194034,0.4190502167,2.2297959328,-0.8208199739,-1.7991086245,-0.6422991753,-0.7526358366,-0.0102676786,-0.6219030619,0.9146863818,0.6332815886,-1.3110525608,0.8213313818,-0.1096853018,-2.4599967003,0.6964056492,-1.6533551216,1.4545164108,-0.0317682959,-0.2366403639,0.8078718185,0.1043894738,1.0441939831,0.5735760927,-0.3592137694,1.6438568830,-2.3127036095,-0.6342550516,-1.0091156960,-0.3238717020,0.1599443555,0.0075996816,1.2957876921,0.1138323396,0.8152285218,-0.6953651309,-0.7397205830,0.2354884744,-0.0152770123,1.1379036903,-0.4955999851,0.6735898852,0.1157289371],[1.3253082037,-0.0473974869,0.0749195814,1.1126670837,-1.3454822302,0.5657912493,-0.3173732460,-0.6623814106,0.9390728474,-0.3503127098,0.5224634409,1.7018532753,1.1917303801,2.5286018848,-0.9827714562,-1.7515311241,0.3258644044,1.1894116402,0.2793818414,-2.5151178837,0.2461755723,-0.0248270202,0.8610759377,-1.4125248194,0.9993839264,-1.4585593939,-1.2222012281,-2.5958955288,1.7636417150,-2.3849341869,0.8637584448,-0.1760935038,-0.3307088912,1.3145556450,0.5149357915,0.5830872655,2.3947978020,1.3304606676,2.7539777756,-1.1339077950,-1.7897496223,0.0805313960,0.2522512376,0.2491337359,0.6630885005,0.2543898225,-0.1769705266,0.5460646749,-0.1675920635,0.6434882879,1.2286797762,-2.2995398045,-0.2927011251,0.6443910599,-1.0067539215,1.6835182905,0.9931317568,0.4729190767,-1.1432638168,-0.7436600327,-0.2560135126,0.9444711208,0.7911326289,1.6346222162,-0.1302263737,-1.0289372206,0.0085418718,-1.3331294060,1.1780107021,-0.9500072598,0.6754372716,1.0067199469,-0.8243820071,-1.0483652353,-1.9091670513,0.0497869141,0.2428957224,0.0930123553,0.1747187376,-0.2940579355,-1.8441371918,-0.6564555764,-1.3747949600,-0.2950574458,-0.5911638141,-0.9996911287,-1.0452963114,-0.8234270215,-1.3071545362,0.4519526660,1.1589044333,-0.5272555947,0.4144105017,0.9806813598,1.7847551107,-1.0145297050,0.5712207556,-0.5980527401,0.5029620528,-0.0612854436,0.7624505162,-1.3226306438,-0.4346112013,0.6056466103,-0.0498499461,-0.3783754706,-1.5018903017,0.0628009811,0.6265262365,0.8333978653,0.3140480220,0.0424722657,-1.9600057602,1.5053578615,-1.9780138731,-0.5857676268,0.6841420531,0.6114683747,-1.3019052744,0.2540067136,0.9127832651,0.4417865276,1.2836414576,0.1039965823,1.2140318155,-0.5549297333,-0.4985508621,0.6292501092,0.3685869873,1.1157144308,1.5523375273,-2.3205018044,0.5883346796,-1.6743589640,-1.3808029890,-1.1486417055,-0.4540960789,-0.2932817936,-1.0938034058,1.2470657825,0.1483083218,-1.3925426006,-0.8331165314,-0.2666631341,1.5057089329,-0.8068805933,-1.0885897875,2.3916010857,-0.4042384028,-0.4770764112,-0.1564542800,-0.6575467587,0.2871340513,-0.0035539989,-0.1058869883,0.1488221437,0.3605379760,-2.4916563034,-1.5001876354,0.4150368869,0.8202406168,-1.4984824657,1.2506990433,-0.9239646792,-0.9689198732,-0.6747028232,-0.9625166059,0.0919773877,1.7888215780,-0.3183748126,-1.5616273880,0.7808475494,-0.0725234970,0.7236689329,-0.8480882049,0.4332070053,-0.7184300423,-0.5073258877,-0.7155088782,0.1922626346,-1.8225835562,-1.5416775942,-0.4609018862,-0.4596909881,0.2107827514,-2.0516586304,1.5195232630,0.8270009756,-0.3949391544,1.2474613190,1.3624265194,-2.0977537632,0.3367308080,1.3497856855,-0.1149254441,-0.1106928810,-0.6452971697,0.4695451558,0.0963989571,-0.2132138759,1.3956539631,-0.4724814594,-0.8594051600,1.8760187626,-0.1349168271,-0.3604379296,0.4938758314,0.2165178359,-1.3356113434,1.1783431768,0.1670271158,-0.0458714627,0.2959707677,-0.8548012972,-0.2733450532,0.4309072495,0.5981340408,-0.6337695718,0.1154701039,-0.1036763787,0.8955789804,0.3470619619,0.3223627806,0.6703640819],[0.3695249259,-1.2931357622,2.0881593227,-0.3163700700,-0.0996335223,-0.0413534530,-1.4175074100,-0.5930390954,0.4457131326,-0.1496128440,-0.1687347889,0.3874080181,-0.8589089513,-0.5768976808,1.7151972055,-0.6996167898,-0.2785257995,-1.0260729790,1.3125190735,0.2768625617,0.7143705487,-0.2726789117,-0.0571496300,-0.0628784820,-0.7734673023,-1.7554893494,1.3222324848,0.8918345571,1.3041300774,-1.3633157015,0.0353974849,0.3286035955,0.6159794331,1.3449649811,-0.0227691717,1.6639068127,-1.9228399992,0.5120229125,-1.0251792669,0.6372367144,1.5930398703,-0.6409438252,1.4217171669,0.0658135042,1.8625446558,0.6787316203,-0.4409461915,0.1394979209,-0.7030583620,0.7338191271,-0.7291454673,-0.3782072663,-2.8154554367,-0.8627194762,-0.8333143592,1.2222462893,0.8411992788,0.3509971201,0.2359188348,0.9166232944,0.0066510001,1.5729032755,0.4003543556,-0.4913519323,0.8445338607,0.0200802702,1.0654255152,1.3981710672,-1.6055946350,0.0228539519,0.8410942554,-1.5818178654,-0.3087835014,-0.7839787602,0.8243343234,-0.9782735705,-0.1291076690,-1.1661448479,1.2579156160,0.3912162185,-0.1826521456,-1.5388045311,0.0681304187,0.7006773353,0.0295840409,0.1992514282,0.2526450753,-1.8144961596,-1.1816760302,-0.9918892384,0.6096387506,0.2016691118,-0.6762703657,-1.3589177132,0.7350046039,-0.1993409395,-2.6887750626,-1.2836247683,1.9510684013,0.1622150093,-1.4023469687,1.4098094702,0.0630933121,1.0961402655,-0.8833422065,1.8702621460,-0.1089891195,-0.3058429062,-0.7081920505,-0.3911435902,-0.7089096308,1.9077292681,-0.5598803759,0.9991017580,0.7150085568,-0.3869202733,-0.9710565805,-0.0675847158,1.1329613924,0.6529337764,1.2304496765,1.1190041304,-0.9004789591,-0.0169585124,1.5388683081,-1.9974298477,-0.0319209136,0.2239657640,0.9455083609,0.4229235351,-1.1756800413,-0.2048513144,0.9564954638,1.5861339569,0.7912840247,-0.5508337021,-0.6716780066,-0.6968273520,-1.3961554766,-0.5600498915,0.3246770799,1.4076087475,-0.0798354521,0.2515286505,0.4909705818,0.3263372183,1.4774889946,1.0507353544,-0.5025584698,-2.5827465057,-1.3963918686,0.3938339055,0.1324110627,0.6092009544,-0.6405442357,1.0710233450,0.9779106975,0.9465399981,-0.9084601998,-1.0478686094,-0.1336563677,-0.9601236582,-0.2246695161,-1.3933295012,1.6808741093,-0.8732835054,-0.4315283000,-1.2740129232,0.4324110150,1.1807173491,2.5027995110,-1.1592706442,2.4850172997,1.2996827364,-0.1811547577,-0.4567832351,0.0583423376,0.4886063039,1.9107813835,-0.4104271531,0.7122089863,0.1199572161,0.7052668929,0.1382817328,1.1929559708,-0.1740968972,0.9283349514,-0.7982419729,-0.6423220634,1.1604852676,-1.0541092157,0.7567608356,-0.1418243349,-1.3702803850,2.9446048737,-1.4440181255,-0.0868617892,-0.4741066694,-0.8283151984,0.9016938806,-0.1963388175,-0.6778028607,1.1174887419,0.2317363173,-1.0282225609,-0.2005127370,-2.2192344666,0.2017867267,0.0493783355,1.3375517130,0.4729900658,-0.7144607306,-0.9983495474,0.6037179828,-1.8254740238,0.2400026470,0.2330179662,2.5256221294,-1.4175472260,-0.3735436201,1.6245576143,0.2052359134,0.6497414112,0.4077384174],[-0.2230141908,-0.7564809322,-0.0916129127,1.0873105526,-0.2665374875,-0.5684608221,-0.7968423367,-1.6703445911,-0.8607365489,0.3183925450,-0.9658346176,0.0435120389,-0.5150961280,-1.1097168922,0.1698560864,2.9419577122,-2.3477473259,-1.3247072697,-0.4962736666,-0.1773999929,-0.0164038111,2.5538542271,1.7659779787,-0.0989361107,0.3340937793,2.1344652176,-0.4907274842,0.1163907647,-0.2446371317,-1.2983027697,-1.7936493158,-0.6849700809,0.8144246340,0.3181484938,1.9092911482,0.8085999489,0.7802957892,-0.0886350796,-0.2588721514,0.0695181787,0.3667381108,-0.3762246370,0.2074953020,-0.3414714932,0.1384399831,-0.7856284976,0.5178771019,1.3630260229,-0.6483896375,0.8072376847,-0.3309074640,0.2732403576,0.5953650475,0.4950374961,0.4725801349,-0.8133680820,0.4031637311,-0.8463309407,-1.2494440079,-0.7790490985,-0.3001246750,0.7606703043,-2.6036124229,0.2312413156,0.0621792115,1.3174229860,-0.5900319219,1.1107158661,1.0482529402,-0.2892859876,0.2211965621,-0.8559523225,0.0115584433,-0.9336676002,1.9462053776,-2.1260342598,0.2948924899,1.2464601994,-1.7223078012,0.8161273599,-1.0551950932,-1.0623240471,0.6164644957,0.6140427589,0.7598878741,1.3047753572,1.5473181009,-1.1017062664,0.3287614584,0.5281797051,2.5536885262,0.2077630758,-2.2922198772,-1.4933623075,-0.2370071262,-0.7327952385,2.1606707573,-1.1783447266,-1.3211505413,-0.7479308844,-0.5706911087,-2.5351402760,1.1879647970,-0.6937165260,0.0836460069,-0.5761220455,0.9647812843,0.3790502846,-0.5369278193,-0.9889282584,-0.0713329986,-1.9731627703,0.0177028272,-0.0183720831,0.3783574998,0.0163121391,-1.3108023405,1.7357823849,0.8464573026,-0.3903010786,-0.3368534148,0.4967666566,-2.7908391953,-0.3965183496,-0.3624084890,0.4881130457,-0.1702686101,-1.1690906286,-1.8261866570,-0.4230371714,0.4788505435,0.4658548534,0.2310337573,0.0985355750,0.4738362134,-0.7344447970,1.2384089231,-2.6306266785,-1.5320723057,-1.2753394842,1.5467416048,1.6379555464,1.0030064583,0.9898139834,0.0424625278,-0.4771284163,1.1298083067,0.6710648537,-0.3274264336,0.7314128876,-0.8347561955,-0.2209829986,0.8305916786,-0.1107638553,-1.0064861774,-0.7868315578,0.2856790125,-1.1872032881,-2.6854472160,1.1837202311,-0.5017939806,-0.7630453706,-0.7599319220,-1.7406020164,-0.3569377363,-0.3295323551,0.1544053108,-1.6152344942,-0.7816680670,0.9302698374,-0.4331679344,-1.4746901989,-1.2758826017,-0.8955430388,1.1119370461,-0.4404303133,-0.2563897967,-1.3007929325,-1.0545949936,1.7923588753,0.5344469547,-0.1055984795,1.2540366650,-0.1275109500,0.1339087486,-0.2487187982,-0.4006001949,-0.2114236802,1.1732668877,0.5091586113,-0.1582694799,0.6916762590,-0.1102267802,0.2098225951,-0.2684221864,-0.2210887969,1.4088248014,-0.4681519568,-1.2454816103,-0.7584915161,-0.5442646742,1.2987605333,-0.0903712809,-0.7038058043,-1.8618180752,1.3913886547,-2.0506701469,1.0668116808,1.0024168491,-0.9046353698,-0.3053961098,-0.2314878851,-0.1263826787,-0.8304456472,-0.8061316013,-0.2422748208,-1.9520103931,-0.7236518860,0.4473903477,0.1257790476,0.6045052409,-1.0181647539,-0.2301247269,1.0518794060],[0.2211459726,-2.1039614677,0.4767338634,-0.5758100748,-0.7040908337,-1.3656538725,-0.9187939167,0.1141875982,-0.0630888268,0.9285331964,-2.3116347790,-2.1702721119,0.6343227625,-1.6013319492,-0.2154222429,1.5275949240,-0.6772807837,1.9274326563,0.7861020565,-0.2652913034,-0.8883908987,0.2253436595,-0.5352017879,-0.5620777011,0.5740081668,0.3111010194,1.5274264812,-0.3963585198,-0.2029903829,1.3392060995,0.4060575068,-1.0550361872,-1.0509568453,-1.0039232969,-1.6214109659,-1.1161359549,-0.5431999564,-1.6879911423,2.1472158432,-0.5342582464,-1.1732395887,0.2828506827,0.6968113184,-0.3553364873,-1.3209646940,1.4841766357,1.4951782227,-0.0843770579,-1.1510633230,0.2676264048,-1.9190896749,1.3373588324,1.0322799683,0.4293705821,-1.0973135233,1.1228461266,-0.1259989738,-0.3956427276,0.5731127858,-0.7413690686,0.0254712421,-0.1747244149,-0.6090978980,-2.3484940529,-0.8988916278,1.0013848543,-0.6178749800,3.2145085335,1.7016453743,-0.1834490895,0.3850836754,2.8780665398,-0.3341577053,-0.3734532297,-0.1130664498,0.1813558340,-0.5761479735,-0.5341535807,1.4379769564,0.0405320339,-0.6183389425,0.4018271565,0.3525454700,-0.6058232188,-0.5648207068,0.0922305062,0.1551928073,1.5452309847,-0.9681372643,-0.5784638524,0.3552555740,0.4516860843,-1.9098737240,0.2499908358,-0.8666939139,0.8051800728,-0.8144145608,-0.2480166554,0.1575364619,0.2941561341,-0.1956454664,-0.3580013514,-0.9503346086,-0.6282243133,-1.3779340982,1.5391281843,0.0974658281,-1.5649412870,0.6169933677,-1.5744905472,-0.8908445835,-0.0382696539,-1.0136801004,-0.2685077190,0.4745731056,-1.8531562090,1.5317523479,0.6800500154,-0.6525881886,2.0080859661,-1.0958909988,1.0522122383,-0.3234107196,1.9245126247,0.3289987743,-0.1973149925,-0.1600109935,-0.1019958928,-0.1369468868,-0.3338777125,-0.4217321277,-0.6276172996,-0.0088310521,0.2495359033,0.6251870394,0.2042882591,-0.0481986813,-0.4357247949,0.1891266406,0.1527542472,-0.3866901100,1.0209299326,0.6823812127,-0.6945874095,-0.7691562772,-0.5777331591,0.0130191892,-0.8398479819,-0.2155789435,0.4451689422,-0.2670453787,0.1741062254,-0.5216994882,-0.8992028236,0.3179417849,-1.7559511662,2.4414460659,-1.0115315914,-0.7715908289,-0.3075437844,-0.6755710244,-0.6566454768,0.3051428497,-0.1559470147,-0.4731896818,-1.5133440495,-0.9692866802,0.1147779524,0.1330191642,-1.4187124968,-0.2106103599,-2.4852890968,0.3849193454,-0.4459509850,1.0283155441,0.4236266315,1.1467685699,-0.4037198126,-0.0717719048,-0.8492287993,0.8758452535,0.1124814972,-0.1776066273,1.2060687542,-1.8546257019,1.0201659203,-0.8215023279,0.0630850345,-1.8618688583,0.1551971585,-0.7597349882,-1.1710904837,1.4031120539,-1.3759285212,-0.4247080982,0.4434662163,0.4800909758,-1.3437353373,-0.3794100285,1.7216287851,-0.4607748687,1.8423278332,0.3057335317,0.5179230571,-2.2636806965,-0.4333676398,1.2896549702,-0.4285718203,-0.8006920815,-0.4375852048,0.5202440023,-0.9587842822,-0.5118048191,-0.9421377182,-1.1184242964,1.6118689775,-1.2746709585,0.7071536779,0.1218064204,1.1981039047,0.6363534331,-0.0728155077,0.9757786393,-0.4173601270],[-0.0015435452,-0.7620249391,-0.5343719125,0.4405129850,-1.3240416050,1.4426378012,0.4065026343,1.5632677078,0.0156175345,-0.4524750710,1.0752927065,1.2035952806,-0.8711138964,0.2041112036,0.2468991578,-0.1774842441,-1.1503459215,-1.2382020950,2.4137120247,1.1750917435,1.5967836380,-1.6567083597,-2.3314807415,1.3394186497,-1.0232579708,0.9828591347,0.6385476589,0.6664152741,-0.1034429669,1.0297343731,0.2680951357,-0.5350618362,-0.4283976257,-1.2094867229,-0.1973341703,0.5106055737,1.0990395546,-0.3337288201,-0.4588372409,-0.8280749917,-0.2634945810,-0.7192752361,-0.0046136426,-0.2803740203,1.8313490152,-0.2697892189,1.5935767889,-0.6038568020,-0.1862212420,-0.2486424148,-0.1683999300,-1.2148817778,-0.4478370547,0.3814966083,0.3098362982,0.3672737181,-0.5108301640,-1.1260827780,0.4522577822,-2.3199803829,1.2764129639,0.0071071084,0.8953126073,-0.1732992828,-0.8343470693,1.2000719309,0.6027942300,0.0065583684,-0.5235270262,1.4073104858,1.5511393547,-0.6028784513,-0.6788135171,-0.4255207181,-1.2978883982,-0.2149679661,0.1982887536,-1.8132313490,-0.7148556709,0.9376966953,0.0934126228,-2.2183358669,0.2135851234,0.9191421866,1.0439063311,0.9127506614,-0.5701822042,-0.5769571066,-0.3889191747,0.8824122548,-1.0307428837,-0.6692697406,0.4454301298,-0.3818296790,-1.5339218378,0.3329823315,1.0650533438,-0.7414758205,-0.1798739582,-0.3133811057,0.0330814943,0.8718146086,0.3320226669,-0.5496357679,-0.6352506876,0.4852578044,-1.3856655359,-1.0167318583,0.3205580413,0.4294673502,-0.7857626677,-0.1906618029,1.4213653803,-1.3946044445,0.0078004524,-1.0760178566,1.2182247639,-1.1788051128,-0.3671139181,-0.1223488450,-0.6729784608,-0.3291174173,-0.0389075093,0.5921736956,-1.6977338791,-0.2491133809,-0.4006183147,-0.3961207867,1.5913404226,-0.5850166082,0.9393358827,0.4974382520,-0.4705432653,0.4906757176,0.0837168992,0.3925708830,-0.5851889253,1.0661809444,-0.4824724197,-0.2261067629,1.2158709764,-0.1601240784,-0.0983043760,0.5352964401,1.6688851118,-0.5516088605,0.1419655532,0.3875458241,-0.3246272802,-0.5487880707,1.0352808237,-0.3842982948,1.4187676907,-1.3117232323,0.2935632169,0.6130684614,0.4549064040,-0.8751249909,-1.3332929611,-0.8147768974,1.0529824495,1.2756927013,-0.5867362618,2.0169563293,-0.2070715725,0.8094360828,-0.0061388025,-0.2174635977,0.9481768608,1.2745907307,1.1427247524,0.0107475081,0.3241678476,0.3622403741,2.0765511990,-0.8151247501,-0.0743439496,-0.9019891024,-0.0548304245,-0.4506755173,0.7100703716,-1.0151324272,1.8718290329,0.6454366446,-1.4716508389,-0.7614647150,-0.6045665145,-1.5807316303,1.3683742285,-0.7087847590,-0.5360591412,-1.1137635708,0.0546837524,-0.2155482918,0.0912592858,0.5711795092,-0.5264383554,-0.2848584652,-0.9425686598,-2.4303109646,-0.1181594357,-1.4667463303,0.9932658076,0.1965706199,0.3686066270,-0.3189594448,-1.3668688536,1.0229911804,0.3237343132,-1.1159375906,1.0948339701,-0.7159814239,0.9033344984,0.6435657144,-0.1155977696,1.0557695627,0.6256161332,-1.8282810450,0.8863859773,2.2530357838,0.9894797206,0.0942059904,0.0559607707,-0.6437538266],[-0.5194265246,-0.1113528311,-0.5954051018,0.0305511039,2.0396575928,-0.9323299527,1.1999788284,-0.1505632401,-0.0716161281,-0.1202129871,1.6836475134,-0.3819015920,1.0422122478,1.2395704985,-1.1008722782,0.3359590769,0.3583621979,0.3088277578,-0.7779789567,0.2468801737,-0.0547442622,-1.1857970953,-0.7717460394,-0.6387051344,0.5602406263,1.7625153065,-0.0663885921,-1.3087264299,-0.6310637593,-0.0381813124,-0.6173859835,0.3789446056,-0.2139604241,-1.1707614660,1.3018363714,0.7095224261,-0.0035795420,0.1458305418,0.0410517007,0.2223228961,-1.1324160099,1.3681198359,1.0462281704,-1.5376925468,-1.6998412609,1.1517218351,-0.2828924060,-0.1268702596,0.0175452475,0.6152723432,2.9246551991,0.0550507084,-0.4084461927,-0.4082174003,-0.0883197784,-0.6834801435,0.1403120756,-0.0701343417,-0.5488973260,-0.7441422343,0.0780278295,-0.5865127444,-1.9822689295,-0.5372510552,-0.1041933000,-1.2280911207,0.8918114901,0.8050571680,0.7902606726,0.1689276099,0.8103792071,-0.3139070272,-1.0741906166,0.8703773618,1.0449312925,-1.3065792322,-0.3290812373,0.0967386067,1.6248611212,1.0085247755,0.7625691891,-0.6881162524,-0.0375259668,-0.1837271601,-0.5998256207,-0.6495984197,1.8604087830,-0.3011618555,0.7878006101,0.3329676390,0.0409335904,-0.9390720725,1.2049945593,-0.1606714725,0.6641917229,1.1545108557,-0.4691017270,0.9510985017,0.3453378379,0.6326152682,0.8411407471,3.3795402050,-0.5579387546,-2.2784161568,1.2684965134,-0.1679872721,0.5747269988,-1.2267564535,2.6995685101,1.2447136641,2.0237715244,-1.2390452623,-1.2545794249,1.7316627502,-2.1253314018,0.3927900195,0.7085506320,-0.8432416916,-0.0852408782,1.6294378042,-0.6161226034,0.9238883257,2.0524621010,0.8227902055,0.4947157800,0.8637558222,-0.2730913460,0.8136610985,-0.3739970922,0.4879109561,1.6569792032,0.3815992177,-1.2899910212,1.9640835524,-0.0580297746,-0.1557988226,-0.6913321614,-1.4017077684,0.3727024496,-0.6369473934,-0.6303749084,-2.6259300709,-0.0493691675,-0.9832947254,1.0024470091,-0.3762393594,1.3069745302,-1.2268050909,-0.7823822498,-0.0879970342,-0.1371357441,-2.0683543682,1.3137185574,-0.5420494080,-1.8690586090,-0.4229525030,1.2428137064,-2.0199379921,1.1582741737,0.3339900672,-0.1540900767,-0.3291948736,0.5431203246,0.5548845530,0.8049129844,-0.9914619923,-0.2426926792,0.4396611750,1.1291645765,1.5539902449,0.0206491835,3.1093180180,-0.6115761995,0.6421800256,0.3549148440,-0.5587056279,1.3602957726,0.8574593067,2.1869783401,-2.0839545727,-0.1410952508,-0.2207788825,0.3456647098,1.7722004652,0.2104642242,-0.0937629715,-0.6939895749,-1.4438390732,-1.4316949844,1.3473109007,1.1719737053,0.1000234857,-2.5027976036,0.5600686669,-1.7411983013,0.6763187647,1.1642938852,0.1111669838,1.0283162594,-0.9822398424,-1.5453110933,-0.8225803375,-0.6440979242,-0.0711455941,1.9552521706,-0.3010765314,-2.1804435253,-0.4159669280,-0.2086515576,0.0323335044,0.7367733121,-0.1938336194,0.0218441971,-0.6198917031,-1.2360179424,0.0325217918,-0.1815406382,-0.8679505587,-1.2398601770,2.7161452770,0.8557375073,0.0299651753,-0.4124097824,1.7006686926],[-1.1124067307,0.0908856541,-0.1516654342,-0.2011841089,-0.8780253530,0.1487515718,-0.1081603616,-2.6986460686,-0.6502318382,-1.3820240498,1.3549199104,0.2082520574,1.0158964396,-0.3602983356,1.0314747095,0.7706522346,-1.4677193165,0.4941012859,0.4933527708,-0.7369694710,0.5938832760,-0.9388635159,0.3698066771,0.1977473944,2.0577509403,1.8244630098,-0.6621942520,0.9831089973,1.8373091221,-1.7026886940,-0.0270229913,-0.7254310846,-0.5342456102,-0.2495466620,0.9207409024,1.1019066572,1.4010708332,-0.4718470275,0.7765082717,-0.2912778556,0.3498292863,-2.7129755020,0.3319345415,-0.4187539518,1.3095924854,1.1868669987,0.8806769252,0.3419238627,0.3971645832,-0.5310337543,1.2703276873,1.2536746264,-0.2384586930,0.7138925791,-0.6958540082,1.0730539560,1.0104210377,-0.2720271051,2.4342498779,-0.1827040464,-1.1780284643,1.5289424658,0.2870815396,1.0993279219,1.0624066591,0.9699524045,-1.5428931713,-0.3731743395,-0.9524514675,1.6522585154,-0.6506668925,1.8939784765,0.7598444223,-1.4891389608,0.2364653498,-0.4037335515,-0.3316952884,0.4112060368,-0.4684414566,-1.5352321863,0.7067044377,-0.2318485826,-1.1240425110,1.5368109941,-0.1835157275,-0.4744376838,0.3413875699,-0.6717487574,0.7119252682,-0.2505222857,-1.0557770729,-1.3723297119,-0.3597370386,-0.4294839501,0.7981411219,0.1716908067,0.7857249975,-0.2756601572,1.3481935263,0.1696635783,0.4440221786,-0.8391209245,0.4513442814,-1.5586638451,-0.3483391404,-0.8522367477,1.5907986164,-0.3074297011,0.2480690479,0.7581233382,-0.5760253668,-1.8630077839,1.0422192812,-1.7398986816,1.2222512960,0.5329426527,-0.5442413092,-0.4942600429,-0.8551141620,0.0436583534,-0.5721877813,0.6107054353,0.3239576817,-0.7967873216,-0.6653596759,-0.9299156070,0.5174583197,2.7006502151,0.0948338807,0.5206369758,-0.3405960500,-1.6355158091,-1.1801077127,-1.0089060068,0.8826825023,0.8409978151,0.5379786491,-0.8869519234,-0.6959516406,0.8970693350,1.1439201832,0.2837764323,0.6520831585,1.5538183451,0.9409436584,1.6984336376,-0.2489234060,-0.4934352338,1.1103788614,-0.0425555967,-0.2637329102,2.3317098618,0.1342730075,-1.1002831459,-0.3531531394,-0.9949698448,-1.2437766790,-0.3254702687,-0.2078438252,0.6164678335,1.4852756262,-0.6047641039,-1.6532808542,0.4184491634,-1.6963517666,-0.2338368893,-0.7228124738,-0.1625349373,-0.4308318496,0.0099216374,0.4700077772,-1.2281514406,-2.7309982777,0.3110759258,0.4845410585,-0.2558689415,-0.7635210752,-0.3534634411,0.9853637815,2.0035877228,0.1427997351,0.3458914757,1.1226363182,-0.4557880163,-0.7759872675,1.3420548439,0.0752204135,-1.5103042126,-0.0915874243,-1.2788894176,-0.2196186185,0.2062564492,0.2331690490,0.2155169100,-0.5119232535,0.2016098946,-0.2278685421,-0.6937284470,-1.1732790470,-0.2774077356,-1.6016391516,1.0906006098,-1.1602990627,1.1262484789,-1.0025730133,0.0271053445,-0.2579020560,0.4614878893,-0.1589870751,0.1840162426,-0.6898531318,0.0125418641,-0.4625565112,-0.8035131097,-0.2153994441,-1.3669756651,-0.3434249461,0.2921564877,-1.1370795965,1.9588086605,0.8582231998,1.3342782259,-0.1519452184,-0.1291483045],[-0.6357197165,-1.1396441460,0.8062340021,-1.2459180355,-0.9046903849,-1.2425501347,-2.1115393639,-0.4970543683,-0.8733558655,1.1578514576,1.0069839954,0.4968858361,0.3666149080,0.6556609273,-0.3875073791,-0.5488761067,-0.0836496875,2.0249290466,-1.0677149296,0.3628509939,0.3246399462,-1.1875450611,0.2687779069,-1.0852653980,0.7550567389,-0.4066241086,1.2095012665,-0.1736595035,-0.3596734703,-0.6551199555,0.3217178881,0.3877495825,-0.5331507921,-0.5281913877,1.2084648609,-1.3032892942,0.7628052235,-0.1079162583,0.4455998242,1.1350136995,-0.1589067131,-0.6414845586,-0.6046472192,0.9702689648,0.7563495636,1.1022398472,0.9599217772,0.3824656308,-0.2513045669,1.9816982746,-0.1709746271,2.5292274952,-0.0726553649,0.8528701663,-0.2167818695,-1.1923350096,0.7861623168,-0.1764873415,0.5312441587,-0.6337814927,0.5093117952,-0.6839939952,0.5782969594,1.6045950651,0.8131929040,-0.7173815370,-0.3780921400,1.3238222599,-0.5112685561,0.0101344641,-0.2775341272,-0.5660531521,1.0679335594,0.1818200201,0.0210486110,0.2730336189,0.9373193383,-0.5658476949,-2.9819197655,-0.7312950492,-0.6392739415,0.3130269647,1.3852910995,-1.7225570679,-0.1778837144,-0.2917789519,2.5960144997,-2.0717620850,-0.5658675432,-0.1349422485,-0.6983388066,0.0349466354,1.1256330013,-1.4770747423,-0.5363630056,-0.2403480560,-1.1767562628,0.1195749193,-0.2448243648,0.5246302485,1.8328498602,-1.3843655586,-0.0110265408,-0.0536559224,-0.0643325523,0.1795994788,-0.9816644192,0.3176301122,-0.1802364588,0.9853456616,-1.1633157730,0.0661375895,0.1401054561,1.1376334429,-0.4585750997,1.6170740128,-0.8622001410,0.0746508464,-2.0770630836,0.9686267972,0.2395104170,0.7997642159,-0.7137439251,-0.4591530561,1.0764036179,0.7686495185,1.0426416397,-1.1665842533,-0.0190958213,0.2914540172,-0.7461980581,-1.0264773369,1.1318042278,-0.6412966847,0.4828600883,0.4435411096,-0.2497335821,0.7211852670,-0.9790833592,-0.1474432647,0.5849481821,0.3871085644,-1.6994421482,-0.9294052720,-0.4046594501,0.0394828208,-2.2656846046,0.8610637188,-0.5293418169,-0.9234024286,-0.7983475924,0.3869788647,-0.0125233578,-0.6018054485,-1.5917366743,-0.6404822469,-0.8488659859,0.3502156138,-1.9618258476,0.4609496295,0.7993075252,-1.2686504126,-2.0727105141,0.4813488722,2.2085037231,-0.1328210086,-0.1219409853,0.7389209867,-1.2908295393,-0.6629701853,0.0689686611,-1.8911678791,-0.0095606102,2.2362875938,-0.2920475602,-0.4304375947,0.6180891395,0.3166204989,-0.2677041590,0.7496351600,-1.5953862667,1.6120036840,-1.2561944723,-0.2658331990,-1.6532055140,0.3250307441,0.2280231416,-1.7944575548,0.9863104820,0.9655084610,-1.2732337713,0.8295410872,1.7446888685,0.4547727108,0.9031069279,-0.0009527904,0.1082891300,0.0627697557,-0.7638112903,0.1184428260,-0.6439966559,-0.6934063435,-0.6369379759,0.0757776573,-0.3329806030,-0.8138238192,0.6588058472,0.0405100137,-0.7962877154,0.1916862875,1.6392009258,1.5568352938,-0.2725393474,0.4475654662,-1.8609107733,0.2686832845,0.5630605221,-0.8181628585,0.7510074973,0.5397127867,0.1188118681,-1.5726583004,1.1463209391,-0.1599459797],[-1.5908477306,1.1402555704,0.0534986258,0.6712822318,-0.2257112414,0.2104907334,1.2600893974,-0.8423901200,0.4301144779,0.0580476858,-0.1085274071,0.2783560753,-0.1843588799,1.1259652376,-0.9875113368,-1.3340691328,-0.2070790976,1.1330128908,-0.5045336485,-0.0493099913,-0.9911766052,-1.0161695480,0.6145914793,0.2172343433,0.8915202618,0.7863522172,-1.5050923824,-0.4608986676,-0.9159309268,-1.0463711023,-0.4841423333,1.4623806477,0.9499843121,0.1457133293,-0.0732988268,0.4994755089,0.0105162933,-0.9202109575,-1.7985740900,-1.8778783083,0.1385161132,-0.1998910755,-0.6571277976,1.3705222607,-0.0393945649,-0.6115819812,0.1122607738,1.7633579969,1.1450352669,0.0145209311,0.4134386480,1.0941917896,-0.7587047219,0.1511386931,0.8067439795,0.3308694363,0.8130151629,-0.4325229824,-0.3996813297,0.7124310136,0.5596027970,1.2407661676,-1.5103613138,-0.7653915286,0.4409960806,-1.3893051147,0.2526411414,-1.5413987637,2.1944396496,0.6292828321,-0.8632075787,0.2486867309,-0.4590431452,-0.9867677093,0.3626977801,-1.0730080605,-1.6345175505,0.2953622937,0.2162745744,-0.2574618161,-1.3549469709,-0.6115554571,0.7602116466,1.9480760098,0.9903740883,-0.2730256915,-0.9925105572,1.0541404486,0.2880633175,-1.1912264824,0.8099787831,-0.1467864662,-1.1243214607,0.2368906140,-1.0483739376,1.5535750389,1.7349170446,1.7200393677,0.4215571284,-0.5391463637,0.7642809749,-1.4886449575,0.7880499363,-0.1383967400,0.2078452259,1.2732071877,-1.3002576828,0.4559438527,-1.1586076021,-2.5392279625,-0.1181192771,-0.1082064211,0.5446758866,-0.0360974520,0.7261715531,2.2089526653,0.2038448453,1.1848708391,0.0747086108,-0.3506720662,-1.0089811087,0.4365611374,0.2272382379,-1.2056691647,0.1030338556,0.8712006807,-1.4377597570,1.3927700520,-1.1165030003,0.1298575550,-1.0141154528,0.0596608967,-1.4940531254,-0.2160215676,-0.0512839481,0.9271569848,-0.4844455123,-0.2318405509,0.6599657536,-1.7151789665,0.7706661224,0.0119470004,0.9035109282,-0.1867048591,0.0665158778,0.8442751765,-0.0485997386,0.5673190951,-0.2960724831,-0.1498709619,-0.4024744630,-1.2394433022,1.5663323402,-1.0304254293,-1.6466841698,-0.8810069561,-1.1049762964,-0.4639779627,-1.5220855474,0.2335274220,2.7803463936,0.0539885126,0.9135933518,-0.9690374732,-0.1095990017,0.1958942860,1.7529040575,1.5590376854,1.5509411097,0.8768229485,-1.7840468884,-0.1732115000,-0.6301497221,0.3956625164,2.3877656460,-0.0268936399,0.5478615761,-0.5379067659,2.1843235493,-0.5380644202,-0.5254244804,0.5818068981,-0.7079021335,1.3369965553,0.8060417771,-1.6658140421,-0.5134732127,0.7067216635,0.8297515512,-0.4131700397,-0.2021587193,-0.5879874229,-0.7992437482,-1.0584437847,-0.8227606416,0.3689611852,-1.1542803049,-0.0566550530,1.3873462677,0.6274378896,1.1806343794,-0.0127347568,0.7829507589,-1.7611320019,-1.8452315331,-0.4864417017,-1.2627916336,-0.9706924558,0.1326923072,-0.0447083339,-1.2672939301,0.5071989894,0.1937571317,-0.4156983495,-0.7045055032,1.5209288597,1.0463624001,0.2856897712,-0.2598487735,0.3661739826,0.5295231938,-1.4221961498,-0.8767566085,0.6296004653],[-0.0801178962,0.6200874448,-0.5689187050,0.1718118489,-0.3422384560,-0.9554112554,-0.2749791145,1.5549792051,0.0348670818,-0.6264494061,0.4486315846,0.1790567935,-2.0919644833,1.0272736549,-0.3224200904,1.3721137047,0.0656904429,0.1180860549,0.1220767125,-0.1829499155,0.5675747991,0.8053595424,1.9218912125,-0.7440114021,0.0348815545,-1.8749556541,-1.4344085455,-1.3828809261,-0.0116490088,-0.9548451304,0.2953458726,-1.4136312008,-0.9416632056,-0.5594043136,0.6049207449,1.0211660862,0.2590726018,0.9106184840,-0.2311110795,-1.2518192530,-0.7415366769,0.4421019256,1.1234837770,1.3203134537,0.6332851052,0.5463161469,0.2642716169,-0.5631567836,0.2989800274,0.3328503370,-0.4342570901,-0.8483982682,-1.7796927691,0.0795050189,0.7186124921,-0.6157069802,0.1147888228,0.0838786140,-0.2524175644,2.8837597370,0.8444882035,-1.9462414980,0.7658838034,-0.4896616936,-1.7648249865,-0.3483499885,0.8363187313,1.1675939560,0.7815184593,-0.5237451196,0.0220253151,0.6432757378,0.2756863832,-1.3450036049,0.0794675127,-0.4446147978,0.4111537635,-2.1214573383,-0.7913644910,0.6275324225,-0.1632787734,0.3176547289,1.3256460428,1.9135844707,-0.3551263511,-0.1786167324,0.6787138581,0.2391502708,1.0172373056,-1.8531953096,0.3468162119,-1.4673253298,-0.2697222829,-0.7077001333,-0.6697863340,-1.4380449057,-2.5307970047,0.8736237884,1.1434020996,0.6140899062,-0.1727177799,0.5524320006,-1.1338095665,0.3060199618,-0.3793222308,1.4749680758,0.8436607718,1.7927627563,-0.3670867682,0.2940903902,0.6996473074,0.1865092516,0.2714602351,-1.1878365278,0.0535032712,-0.3654344380,-2.1078500748,-0.8450355530,0.7202682495,0.6847658157,-1.5380316973,0.1071202233,-1.4377793074,0.5285930634,1.1309717894,-0.2409960181,1.1711474657,0.0241652615,0.1474795640,1.0787435770,-0.2041695267,1.1821421385,0.7134022713,-0.1815098971,0.6097062826,-0.7310802937,1.3213235140,-0.4690505266,-0.1206440702,-1.9621025324,0.0792470798,-0.9037244320,-0.6727975011,-0.5813845992,0.7638904452,-0.7982986569,-1.1170951128,-0.0278148465,0.7436719537,-0.0349298790,-1.5607745647,-3.6942853928,0.2392628640,1.6242785454,0.1078562513,0.8389151692,-1.0812578201,-1.3334944248,-0.3950431943,0.1764431298,-1.5741150379,-0.2641482949,-0.5769725442,0.0579043850,1.6284952164,1.3821072578,-0.1604852676,0.5396218896,0.3823760748,-0.5413639545,0.3832670748,0.3760714531,0.6530348063,0.7082789540,1.6361300945,-0.8255808353,0.4770345092,0.7754092813,0.5997623801,-1.0650105476,-0.1005972773,-0.6563813090,-0.4330505729,0.2436604351,-0.1934807748,0.4373670518,-0.1382869184,-0.0957607403,-0.9223588109,0.2641588449,0.4486901760,0.7179651260,-0.2507312000,-0.3429058492,2.7186050415,-0.0031434130,-1.7863855362,-0.1491491944,-2.4144194126,1.8584879637,1.0816385746,-0.0691355243,0.0412137210,-0.1214653924,0.8756392598,0.6263403296,-0.2714163363,2.5999414921,0.0031115902,-0.6093174219,0.8194519281,0.2598125339,-1.5010932684,-0.3467047811,-0.5145673156,0.8833550811,1.5528534651,1.1378787756,0.4535325766,-0.2141902596,-1.0113316774,-0.0475542098,-1.9863569736,-0.0898115486],[0.8721480370,0.6683921218,1.5094119310,-0.5674828291,-0.2722056508,1.1545054913,0.9233204722,1.7845782042,-1.1065726280,0.8899226189,1.1991003752,0.3537445962,2.0520317554,0.4479960799,-2.4655239582,1.1212525368,-1.6055749655,-0.5201081634,0.5299128890,2.2476568222,-0.7108030319,-0.0680756792,1.3571964502,-1.8943730593,0.6292427778,-1.0711674690,-1.9200879335,-0.0095262108,-0.0612300560,0.3710441887,0.9317353368,0.2323822975,1.5968242884,1.2942872047,-0.7865222692,-0.0996802449,-0.2432516068,-1.8798357248,1.6190136671,-0.4560818374,1.0802834034,-0.3977110982,0.9913378954,-0.4810326695,1.1207609177,0.3215666711,0.6868842244,0.7012643814,0.3027723134,-0.5669187307,-0.8218886256,2.1448388100,1.5178095102,0.3204951584,1.2383538485,-0.6073427796,0.1993925571,-1.6000400782,-0.7605104446,-0.1507879049,-0.1515130103,0.1344979405,-1.3684259653,0.0471347682,0.0774593130,0.2506715953,0.7475439310,-0.5684504509,-1.4915621281,0.4500528872,0.3627835214,0.2907096446,1.4360669851,0.2089084089,0.7776738405,0.0575551018,0.1220124662,-0.5838123560,-0.0324958488,0.2039363533,0.3325089812,1.3271270990,1.4291518927,1.6142344475,-0.2566379309,0.0891720057,1.0746363401,0.0611726567,1.3872996569,-2.3101968765,1.5936372280,-0.6907869577,-0.0694889277,-0.2972775698,0.8311193585,-0.1777330637,2.0184845924,0.5656145811,-0.2051413357,0.9329389334,0.6280627251,-0.1737667471,1.2492796183,0.8439475298,1.2072235346,-1.0259077549,-0.2974030375,-0.4043617547,0.8485488892,1.0105415583,1.0167632103,-0.3439735472,-0.7064131498,-0.3029172421,1.4635292292,1.8002523184,0.4651216567,-0.3715837598,-0.2971130908,0.3571931124,-2.6784129143,0.3649374545,-0.3777738512,-0.7590556741,-0.0151699381,-0.8821194172,0.5951886773,-0.0852708519,0.9750209451,-2.0264630318,0.3239995539,-1.5468266010,-0.6036015749,1.2733149529,-0.5170837641,1.3886284828,2.0128860474,0.6841310859,0.5642735362,-1.7078286409,-0.1220685244,0.8022543192,2.9402222633,-0.5440009236,0.5440354347,-0.3028250933,-0.1236471310,1.0711059570,0.1711559743,-1.1677023172,-1.5948584080,1.1498628855,1.7074403763,-1.7957524061,-0.3149690032,0.1568339318,-0.3114894927,0.0211651828,0.1890159994,0.9555293322,-0.2745038271,2.7662432194,0.4147194326,-0.4034470320,0.3713617623,0.3691652417,-0.8164408803,-0.0419991501,0.8501044512,0.4174660742,-0.4661606550,0.7105901837,-0.1279045492,0.3523364067,-0.2073855847,-1.1988571882,-0.0351843536,0.1673676521,2.1629037857,2.3720927238,0.5707538724,-1.2859030962,-1.7754752636,-2.2168619633,2.3724796772,0.0648849458,0.4216980040,-0.4210457206,-1.1706908941,1.2849756479,0.0288783815,0.0029941904,-0.2512137294,1.0792390108,0.0430316254,-0.1150320619,1.3237221241,1.2466740608,-0.7737157941,0.3568372130,-0.7242242694,2.0552656651,0.5782729983,-2.1194407940,-0.4371688962,0.5964887738,-0.1009557024,-0.2599818408,-0.5893468261,1.4360679388,-1.2596794367,1.6888058186,0.1015370265,1.0656791925,0.5520688295,-0.1842226237,0.3630548120,0.3245004416,-1.7664282322,-0.6189075708,0.8789561391,-0.0498849228,0.1388183832,-0.6309426427],[-1.2514204979,-0.9320270419,-1.3813583851,1.6924980879,0.4783075154,-0.2157701105,-0.4409082532,1.7098357677,-0.0066842828,-0.6266974807,0.3062791824,0.8604946733,0.3906947374,0.6621009707,-0.3415372968,-0.1327738911,0.1090027690,0.5395166278,1.3441191912,0.3890197575,-1.5003491640,-1.6765406132,1.0947563648,-0.4194751680,2.2247760296,1.4415625334,-0.7291014791,-1.1591322422,0.0448737144,0.1642017961,-0.4120277464,-1.0800826550,0.0302338228,-0.1979196072,0.2417976409,-0.4217292070,2.8807368279,-0.2916334867,-0.1119498610,1.4483487606,1.3079737425,3.1414964199,1.9990832806,1.3666808605,-0.4089882970,-1.9430245161,0.1383289248,-0.2756088376,-0.3302395046,0.3767584264,-0.1456294358,-0.0736739188,-0.2632665634,-1.0246253014,-0.0997517109,1.4146862030,0.3474251032,-0.3105725646,-0.6983625293,0.1704372913,0.5889595151,0.3045978248,-0.9002690315,-1.0737453699,0.9776275754,-0.8216294050,-1.4059617519,-0.8584246039,-0.9430308342,0.0863835663,0.4300649464,1.0046936274,-0.1960627437,-0.9061456323,-1.2908878326,-0.8379617333,-1.2833580971,-0.4425460398,0.8820519447,1.7542774677,-0.9453480244,0.4118882418,-1.1179608107,0.1755656004,-0.6019179821,-0.2438964695,1.4608323574,0.6031800508,0.4058443010,1.3704522848,-0.8749694824,-0.2395764142,0.3866791129,1.1813465357,1.0303496122,0.4329538345,0.6799361706,0.1898843944,0.5167849064,1.3488928080,-0.9842775464,-0.9970665574,0.4313078225,1.4899312258,0.2684113979,-0.8607005477,1.5387905836,-0.2373507470,0.1182171926,-1.9649727345,0.1285471916,0.7161492705,0.0362120271,0.9534409046,-0.9433271289,-0.9471279979,0.4125443995,1.5807524920,-0.6251024604,0.0839564353,-1.3108118773,0.8444495201,-0.9436219931,0.2464914024,0.1501429230,-0.9166177511,0.8560885191,-1.7652616501,1.0727841854,-1.1108458042,2.2795565128,0.9886731505,0.7996646166,-0.7582955360,0.5232353806,-0.5892561674,-0.4959479570,0.5991611481,-0.9851461053,0.5707060695,-1.1462754011,0.0037861473,0.3931219280,0.6216490269,0.6596783400,0.8396199942,0.1782273948,-0.4145593047,-0.1143532768,1.4198349714,0.4542442262,1.0489712954,0.1573176235,1.6916421652,0.2117508501,0.6056104302,-0.8419842720,0.8234154582,0.9980833530,1.0220810175,0.6612729430,-1.0162453651,-0.6721482277,-1.0239974260,0.7389633656,0.6064419150,0.4327298105,0.4807211161,0.1548499912,-0.1756894737,-0.4154034555,-0.5601374507,-0.6286499500,1.0796920061,0.5231323838,1.2025828362,2.7605485916,0.9448372126,0.4897510409,0.3005301058,-0.3569986224,1.4894896746,1.0769951344,0.9457442164,1.3836812973,-1.2388128042,-0.9194841981,0.6592172980,1.1811634302,0.1504609138,1.1520256996,0.7040884495,-0.7302854657,-0.7238515019,0.6783299446,0.5535770059,-1.1894803047,0.1862907112,-0.7157256007,-1.6877734661,0.2565253377,0.9417484403,-1.1264679432,0.3717091382,-0.6848784089,0.8496810794,0.5500252247,1.2373111248,0.5267019868,-0.3370111287,1.7402325869,0.6208246946,0.1496523172,-0.0145279448,-1.0990328789,1.0914890766,0.3146514297,-0.2327844501,-0.9927524328,0.2801242769,-0.8725679517,-0.3057530224,-1.4110640287,0.0511594787],[0.0009446123,0.4381587207,1.0378230810,-0.8866230845,0.1709326804,-0.2147371024,-0.5751985908,-0.4123259783,0.3231201470,-1.2464450598,-0.1151144356,1.0316728354,-1.7826942205,-0.2277909219,0.2456097901,-0.1247552261,1.1718108654,-0.9280160069,-0.8763661385,-0.9531322718,0.3050051033,0.9160202742,1.1899218559,-0.3139192462,-0.5719921589,0.2692719996,0.6695338488,-1.4176100492,-0.1810361296,0.8919016123,0.3253822625,0.2102392316,1.1510844231,-0.2731625140,0.9161008596,-0.3529846966,0.7246912122,0.3719465435,-0.6608036757,-0.7025949955,-0.2617565691,-1.5062785149,-0.2983382642,0.7757444978,0.5407574773,-0.8948160410,-0.3278140426,-0.2843683064,0.3198806047,0.7847097516,1.0833867788,-1.1843409538,0.9079430699,0.7242349982,-0.1339356005,0.4209992290,0.3426757455,1.3023025990,-0.1115856692,1.1961381435,-0.6273810863,0.5569947362,-2.0751523972,0.4874547124,0.0112005575,-1.9131480455,-1.6525598764,-0.9860039949,-0.4189216495,-0.2969622314,-0.4724214971,-0.2920680940,-0.6366106868,1.4640799761,0.0001000688,0.6696881056,0.3056806624,0.2199319601,0.5476730466,0.2989024222,-0.3060296178,1.5866267681,-0.1923424900,-0.7535434961,2.4642388821,1.4206904173,-1.3298442364,-1.6729069948,-0.2089457661,0.1286960840,0.2165742069,-1.2509105206,-1.2517259121,0.0144478939,-0.5250476003,-1.4943634272,1.7899401188,0.7980135083,-0.2467044145,0.1137525737,-0.8360635042,-1.0790261030,-1.4593545198,0.6329082251,-0.1951130778,0.2109565586,-0.6452851892,1.3813667297,0.3914685845,2.4400918484,0.8632235527,0.4989744127,0.4575488269,0.6004958153,1.4023209810,-0.9147142768,0.3939240277,-0.2197887301,-0.9745028019,-0.8095402122,-0.1595830917,0.3253418505,-0.9514853954,-1.2631294727,-1.9905785322,-0.1589462757,-0.2686427534,-1.0362379551,-0.5929874182,2.1278758049,0.6741823554,-0.3402375579,-0.8575452566,0.1285680085,-0.1974782944,1.3091429472,-0.0491636656,-0.8006293774,-1.9171167612,-1.6133759022,0.6724346280,0.2764676511,-1.0947827101,0.7140973806,-0.4589699209,-0.5040537119,-2.6571507454,0.4938603342,-0.1112184599,-1.1413344145,0.3065828979,-0.0351181589,-0.2077741474,-1.2007638216,1.3442721367,-0.2882195115,0.7794566751,-0.0357597359,-0.4914976358,-0.6510698795,0.4476221204,-0.2434361875,0.3303336501,-0.1247829944,0.9456844330,0.3387642205,-0.0858657435,0.0836824849,0.3197313249,-0.5224598050,0.0667929351,0.9599102736,-1.2944271564,-1.6121833324,-0.4610219300,-0.0686687902,-0.6209889054,-0.5094255805,-1.3296127319,-0.6519948840,0.3874265552,0.2548698485,0.6057342887,-1.1156878471,1.7558230162,1.3769574165,-0.5067620277,-0.2540018559,0.1169501916,1.4090871811,-0.2061688751,-0.3826217651,0.0744053125,-1.0701073408,0.2666794658,0.0159707237,-1.0801004171,0.3020006120,-1.1221276522,-0.0603687689,0.4302231073,0.8030367494,0.3813634813,-0.5177665949,-0.5745231509,-1.2133439779,1.0359526873,-1.0773680210,-1.0581761599,-0.7194543481,0.7911873460,0.3575147688,0.4734067023,1.3755928278,-0.3413253725,-0.7216416001,1.4161067009,-1.0657318830,-0.5758734345,-0.0554632656,-0.5676805973,1.0491763353,-0.5460031033,0.0000377406],[-0.1134769619,-0.1902130544,1.1122241020,0.0361957811,-0.5202575326,-0.9287207723,0.4673231840,0.2076914012,-0.5268574357,-1.4158695936,0.9209938049,-1.2451177835,0.3177010119,1.3631247282,-0.6694192886,-0.5273765922,0.1168278307,-0.1416368484,1.0902903080,-1.5505670309,0.8457047343,1.2713711262,0.8615369797,-0.6803045273,-0.9436883330,-0.8042429686,1.2764884233,-1.4393085241,0.6460635662,0.1760231256,0.6022271514,-0.6289021969,0.2052420974,-0.9054312110,-2.2867314816,-0.5080083013,-0.3702924848,-0.3562673926,1.6888591051,1.0248645544,1.2868249416,-1.4498550892,0.4269231856,0.7690059543,1.2068730593,2.2509765625,-1.3853080273,-0.0437916964,0.5083934665,-0.1405894905,-1.9240996838,-0.0695686266,1.0290793180,-0.6863524318,1.5921109915,-0.5149906874,0.4313142598,1.2325224876,0.7381823063,0.0506481752,0.2718752027,-0.0458972342,-0.2813476324,0.1470736563,1.1240963936,-0.4729631841,0.3032023907,-0.0378687717,0.7551904321,0.2726889551,0.8062175512,-0.9914822578,-1.3461425304,1.3953132629,-0.3963041306,-0.1348937005,0.1861299574,-0.1282923222,1.3191727400,-1.3927209377,1.6524684429,-1.4570862055,0.1970746368,0.7873681784,-1.5872268677,0.7789785862,0.7257158160,0.9104884863,-1.0458092690,0.1311151236,0.0523036309,-0.5617845058,0.6315040588,-0.8047713637,-0.7324396968,0.5472825766,-1.1248081923,0.7605649233,1.9306453466,-0.8998404741,-0.9145870209,0.7737373114,-0.7555955052,-0.0085799778,0.6102102995,1.0703949928,0.0122935213,1.7418897152,-0.8565835357,-0.1485811919,2.5863671303,0.2296967655,0.4921732545,-0.2157451808,1.7708336115,1.4435811043,0.3088599741,0.3821006119,0.7828531265,0.8288940787,1.2654770613,0.5320999622,1.6035956144,0.0555833168,-0.7808041573,-0.4768952131,0.0609117076,-0.4685634375,0.0401823297,-0.7980151772,-0.1980319172,0.3622185290,-0.0124242250,0.0019212270,-1.8690266609,-0.6661810875,1.2603762150,0.3925990760,-0.9471579790,0.9259031415,-0.6989381909,-0.5957267284,0.8059161305,-0.0204536505,-0.7605403066,-0.8714606762,-0.0785574615,0.2176924050,-0.0952641219,-0.6237257719,0.8928356767,0.0207476560,-0.8050456643,-1.1606603861,-2.1986162663,0.5743509531,-0.3747348487,-0.9865067601,0.6025411487,-1.6260415316,0.4241379201,-1.5761916637,-0.5231423378,-1.2368144989,1.3236354589,-0.2628894150,0.9006096721,-0.5883111954,0.9315660596,0.3901229501,0.3613577783,-0.9036842585,0.6531979442,-0.5237332582,-0.4377007186,1.2412061691,-1.8077656031,1.0896279812,-0.5850965977,0.9993878603,0.9793956280,1.7526859045,-0.1627811193,0.0789898112,-0.6008516550,0.6423956156,-1.3718004227,0.8177475333,0.1333660483,-0.7116602659,-0.7372015715,0.0193281416,0.5816512108,0.6322121620,-0.9512215257,-2.3198320866,-0.1225626543,-0.7548299432,1.7849982977,-0.4172134101,1.1810698509,-0.6830892563,0.9388549328,-1.2363265753,-0.6732035279,-1.1739046574,-0.3762327731,-2.2701492310,-1.8741184473,-0.9467853308,0.1845830977,0.0290970095,-1.1332253218,1.5481897593,0.6677289009,1.6565377712,-0.1813921779,-1.4472157955,-0.9493680596,-2.3203413486,-0.1047000363,-0.5362039208,0.1503630877,-0.2119492143],[0.1951975077,0.3594886661,1.1071165800,0.5541626215,-2.2391679287,-0.0982199907,0.1499948204,-0.2038379759,-0.3641252220,1.2327725887,-1.0428520441,-1.1857684851,-0.3758674860,-0.7437927723,-0.5258880258,1.6090214252,-0.5477022529,-0.5831058025,-0.7693243027,0.9592191577,0.7612723112,0.6288821697,0.4854650795,-1.0746076107,1.0054817200,0.0261112470,-1.3465268612,0.8569097519,-0.8402596116,-1.0034821033,0.0340053290,0.6326579452,0.0624723062,-0.8272801638,0.9872714877,-0.6493257880,-0.0124687888,-0.0510785058,-1.0020663738,1.7653633356,0.6531934738,1.7806649208,-1.2149158716,0.1973568499,1.2248632908,-0.6891977787,-0.5806148648,-1.4091354609,-1.4452899694,-0.1621054560,0.2553484142,0.5607942939,-0.3715134859,-1.2819964886,0.1613192409,0.7283805609,0.9018242955,-0.4699330330,0.7522509694,0.0473475978,-1.4565970898,0.2873678505,0.6105089188,2.3087093830,-0.4378469586,1.5861054659,-0.0902395621,0.2450557947,1.6665794849,-0.2875549495,-0.6267389655,-1.1200077534,-0.1176927686,-0.9884122014,0.0998817012,-0.3104116321,1.6640614271,1.8047343493,1.7079073191,-0.2945924997,1.4536072016,-0.9042026997,-1.5250431299,-1.2416226864,0.9511591196,1.6640285254,0.9149685502,0.5348483920,0.0271247104,-0.4100965559,-0.4923259616,-0.5284524560,0.7326970100,-0.6320443749,-0.1361745298,1.7820904255,-0.4102703035,0.1149579212,0.4176089168,0.5615858436,-0.1241502464,-0.2711972296,0.8339188695,-0.3911842406,0.8035120964,0.4347385466,0.3256336153,-1.0912731886,-1.1064262390,-1.0023913383,-0.0014343157,0.6572046876,1.1768506765,-0.1612158120,0.2430538386,-0.4206996262,-0.6818819046,-1.5458599329,-0.4415317476,0.8601318598,0.5682681799,0.5093677044,0.6739342213,0.2802112103,1.0092612505,0.5739693046,0.4783741236,-0.7188705802,-0.7555210590,-1.2773789167,0.9673935771,0.7169851065,0.6318838596,-0.3510349393,-0.2784935236,0.9245770574,0.1456117630,0.8596948385,0.5870320201,0.3222205639,-1.1058843136,0.1310071945,0.1351862252,1.4976658821,0.0642576590,0.9805966020,-0.2625622451,-2.0229084492,-0.2577911317,0.8459518552,-0.9816684127,0.0494478457,0.1633282155,0.8872380853,0.2346172631,-1.4553668499,-1.2075823545,0.9899796247,-0.7265192866,-0.1070689112,0.4315021932,1.5511058569,1.0648895502,1.2513504028,-1.6179898977,-0.1960540712,-1.1174563169,-0.0627279058,0.3648060262,0.4943988621,0.4699738920,1.5897046328,0.4787073135,-0.2594979405,0.9839239120,-1.1296446323,-0.2151689082,1.8668168783,1.2359091043,-0.7876203656,2.2410216331,-0.6001842618,0.6790406108,1.9386615753,-0.2716152966,-2.4987998009,0.7988651395,-1.7352396250,-0.8488154411,-0.2405159175,0.0022603455,-0.8565176725,-1.7655363083,0.0227796603,1.1678682566,1.5977461338,0.6137763858,-0.1560899615,-1.2249263525,1.0691320896,0.8002671003,0.9079485536,-0.5812396407,-0.6096565127,0.8414008021,0.8574656844,0.8194988966,1.9166227579,-1.2481331825,0.1276202202,1.2027217150,0.6284421086,1.8388808966,0.7530941367,-0.5810328126,-0.1983797401,2.4699819088,0.5870167613,-0.3973526657,0.4335496426,0.5002970695,1.2368345261,-1.0192248821,-0.4722243547],[0.1429932714,-0.6643745303,0.9991702437,-0.7230226994,-0.9242684245,-0.6563588977,1.8147925138,1.0481163263,0.5963919759,0.3161043823,-1.5705043077,-3.0482473373,-1.4911568165,-0.5620017052,-1.5762267113,-0.2792216539,-0.0183045566,-0.3427428901,0.4325659573,0.0841940790,-1.5596717596,-2.3954472542,0.6650484204,0.9164831638,-0.6995766759,0.7592606544,1.9653995037,2.1778728962,1.3679281473,0.3184723258,-1.3753741980,-0.5408719778,-1.3242262602,-0.0486250222,-1.4743169546,-1.9059720039,0.4585015178,-1.3563024998,-2.1027057171,-1.6160247326,-0.8111845851,0.6833175421,0.3431516886,1.0679210424,0.3967325985,1.3747622967,-1.9252409935,-1.5621434450,0.3567034006,0.1609219313,-0.4597774446,-1.5141679049,0.2331478596,-1.2618752718,-0.7512057424,-1.0166230202,-0.5066205263,1.9142640829,1.3055176735,-0.3582405448,0.3410398662,0.7159633636,-0.9026330709,0.9061701894,-1.3593796492,0.2564473748,0.6371505260,0.4507238567,2.0558533669,0.4674500823,0.4008921087,1.1421283484,-0.7805560827,0.8754662871,1.1091470718,-0.4740432799,-1.0471091270,0.5696887374,1.4610911608,0.2937921584,-1.3595267534,0.5948380232,-0.5259676576,0.6979746819,2.6188032627,1.7860373259,1.3816629648,0.3645930588,0.6413857937,-0.3179526329,-0.6694340110,0.8531759977,-0.5612838268,0.6540765762,-0.7173810005,0.7171044946,-0.3337262869,-1.2696685791,-0.3191621602,-0.0951602459,1.6437281370,0.4104022980,-0.6791217327,-0.5343052149,0.1236637682,-0.6867526770,0.4327032268,-0.9899078012,0.3520361781,-0.0372290350,0.0954944342,0.8924535513,-1.3897480965,0.2908154130,0.1433742046,-1.2002483606,-1.1578544378,0.2018009126,-1.7330788374,-0.3900980949,0.3442434072,0.5612210035,-2.3735797405,0.4677275121,-0.8755277395,0.3840200901,0.7140026093,-0.0161184259,-0.2327160239,-1.3661100864,1.8017098904,0.1095980778,-0.9140314460,-2.1166992188,1.5349193811,1.5171842575,-0.7831214070,0.8759396672,-0.3350074291,-0.6258761287,1.1088163853,0.1659310460,0.3859133720,2.1467437744,-1.4819864035,-0.1044765934,0.0227515679,-1.3247855902,-0.0460397489,-0.2199831605,0.3223781884,-1.8705461025,-0.2860034704,1.3835725784,0.0756484866,0.7798700333,1.1855169535,1.0793471336,1.5322893858,0.4833815396,0.7111821175,1.2975721359,0.5630246401,1.6598074436,-0.3058865666,1.0004166365,0.2399135530,2.0036790371,-1.0272872448,-0.2898159623,0.3397452831,-0.5536059141,0.1857142448,-0.8085384965,-1.5434528589,-0.0007162808,1.0881972313,0.6845315695,-1.8424415588,0.6105918884,1.3732216358,-1.0601470470,1.1996201277,-1.2230579853,1.1186140776,-0.6385557652,1.1005522013,-1.8498262167,1.9758620262,0.3748421669,0.9024031758,0.6665591598,-0.5407711267,-1.1386766434,0.8666719794,-0.7272105813,-0.5572866797,0.3335200846,0.2419228256,0.1356631666,0.0634459481,0.5071093440,-0.1976352483,-0.1633360833,3.0307579041,-0.6317493916,-0.2643266618,-1.2277452946,-1.0420066118,-0.2836993039,0.8368556499,-1.4368258715,-0.4121444225,-0.1726893187,-2.3277754784,-0.5735732317,0.4645721018,-0.1024918482,0.9124009013,-0.9102084637,1.3875967264,0.7080764174,-1.5910292864,0.1935648620],[0.8600574136,0.3366048932,0.4506984651,0.6514768600,-0.0764857233,-0.5673771501,-0.2962603569,-1.1983201504,1.0930721760,-0.4548245668,0.9074430466,0.0770448744,0.8746569753,-0.2766015828,1.2379782200,-0.0664854795,-1.2437325716,-0.4574719965,0.7507360578,0.6556258202,-0.2704447210,-1.5917396545,0.9173870087,-1.8183697462,-0.6614665389,1.0413696766,1.2564733028,-0.5287645459,-0.7868911624,0.4760374725,1.4004799128,-0.3912179172,-0.3319635987,-0.1274384409,-0.3486401141,-0.5874551535,-0.4041832685,-1.2626019716,-0.2735873163,-1.8302440643,1.1272053719,-0.9103027582,0.9363489151,1.7044332027,-1.4405498505,0.1213527992,-0.7432028651,-0.3949774206,-0.6219618917,-1.0270428658,0.3988615274,-1.2037241459,0.0879646614,-1.0123245716,2.2703430653,-0.1986519694,1.0142803192,-0.5765676498,-0.1850306988,0.9218153358,-1.4215303659,-0.3045988679,0.8781455755,0.4839467108,0.3406828046,0.1385729313,0.5530541539,-0.2178639174,0.6572006345,-0.7660921216,-0.1504787654,-0.6955124736,-0.8039906025,0.1935697198,-0.6107688546,1.4775750637,2.1482009888,1.2706098557,0.8919548392,0.5517945290,0.4745905995,-1.3749998808,-1.9628106356,-0.2441599369,-0.2756695151,2.3750700951,-1.0125036240,2.3627867699,-1.0484228134,0.0202945489,-1.6291437149,1.5460455418,-0.2626701891,-1.3716934919,1.7448372841,-1.9003964663,0.7340160608,-0.9719927907,0.4878770411,0.8426557183,-0.0036995597,1.1608890295,-0.1446119249,-0.3867310584,-1.2060709000,-1.8427013159,-2.1762900352,0.5401457548,-0.5232191682,0.6863697767,1.7252759933,0.9453377724,-1.2271872759,-0.5287625194,-0.5573931336,1.1400207281,0.8531771302,-0.7874594331,1.4860290289,-0.3966833353,-0.8653481603,0.6682871580,-1.3460118771,1.1101393700,2.5374398232,1.5781009197,-0.2074164003,1.1810634136,1.3876578808,0.4343869984,1.0187358856,-0.3469872475,-0.0697899982,0.6849715710,-0.7938639522,1.0810682774,-0.7873340249,0.4875499606,-0.5890196562,0.6428114176,-0.5371803641,-0.0320537537,-0.6355226636,0.2840048969,-0.7231734991,1.6757985353,-1.3749076128,-0.1871342212,-0.5509265661,0.2247584015,0.0963228941,0.5343322754,0.4177618623,-1.0538265705,1.7033421993,-0.1041369513,-0.5689711571,0.1449021250,1.5101170540,-0.8077566624,0.1635077000,0.3681793809,-1.3697288036,-0.9026447535,0.1419233531,-0.4355032146,1.4709450006,-1.2994558811,-0.1398774087,0.4056711495,-0.9279340506,-0.4008336663,-0.1832318157,-0.4932772815,1.1810588837,-0.4294885397,0.2586955130,-0.6026366353,-1.6949310303,-2.4281458855,0.3360414505,0.0007651557,0.6452424526,0.0013740991,1.2619608641,-0.6294914484,1.0755714178,-1.6946045160,0.4138219059,-0.9013427496,-0.9153943658,0.2750778496,-1.7745633125,1.2423049212,1.4853067398,0.6582362056,-0.4944946170,0.8580268025,-2.5256597996,-0.4416845739,0.0358854383,0.2637412548,0.1396094710,-0.2113924474,-1.0850139856,0.8667337894,0.6863631606,-0.7880696058,0.2563993931,0.6276249886,0.5765470862,1.0376647711,-0.8950424194,-0.7374470830,-0.5733580589,1.5468299389,1.0180302858,-1.3552303314,1.5083237886,-1.5015984774,-0.4331097603,-0.7236114740,-1.2652302980,1.8355219364],[-0.9491904974,0.7845528722,-0.1004951149,-0.9171906114,-0.6248505712,-0.5434099436,0.2936888933,0.7476300597,-0.1759679466,0.7415206432,0.1636239588,0.3913640082,-1.0406374931,1.5966686010,1.7157511711,-0.9193367362,-0.5155612230,0.3866383433,0.3502348959,0.6012752652,-0.2728933096,1.0108480453,-0.3607590199,-0.9352738857,0.0113944160,-1.0877426863,-1.2035589218,-0.5796896815,0.0864668638,-0.7101565599,0.0531113632,-0.7170058489,-0.0952148288,0.3925267458,-1.0388872623,1.1641366482,0.4681651592,1.2697256804,0.5855969787,0.8054180145,0.5459367037,-0.0193008948,-0.0799392536,-0.7031532526,-0.3332782388,0.8763495684,-0.3275536895,-0.0899465531,-0.7804068923,1.5976791382,-0.0845693126,-0.8380930424,1.2919481993,0.3801421523,1.0650464296,-0.4915513992,0.3796888292,0.3082422614,-1.2515425682,0.8574884534,0.0453626625,-0.8994646668,0.1817809045,-0.3982611299,-0.4245830178,0.4759723842,-0.5196942091,0.9084286094,-0.2574448586,0.6669890881,1.4542843103,0.6965520978,-0.4026068151,0.4368975759,-0.1450076252,-0.7213081121,-1.9050275087,0.1893024594,2.1357374191,1.5123866796,1.4541821480,-1.6385452747,-1.5910087824,-0.6939245462,-0.8785220981,-0.8075929880,-0.3812121153,0.8093725443,0.2798313498,0.5270179510,-2.3664381504,0.5130461454,0.0916117877,0.6262785792,-0.7436412573,0.3277203441,-0.7760006785,-2.5068039894,0.4759555757,-0.9010194540,0.0145629812,0.8987622857,0.3353192210,0.9606677890,-0.2874678075,-2.9939599037,1.2504578829,-1.1720881462,-1.0356649160,0.6958131194,0.4948087037,-0.4572609365,-1.6435623169,1.5744616985,-0.0300552286,-1.0009658337,-0.9311393499,1.0732966661,-0.2361930311,0.2371501774,-0.5839691758,0.8537157774,-0.2181455344,1.7659276724,0.2091579586,1.5832922459,-0.1612836719,-0.6309607625,-1.7614916563,-0.5587900877,-1.0242021084,0.0377286524,0.4732848704,-0.1918084174,-0.0163135994,0.2189601362,-0.9728784561,-2.5277113914,-2.9586434364,0.6248936653,0.9954282641,0.9903995395,2.0193979740,0.4239855707,0.8024560809,-0.2874479592,0.2707243562,-0.7016677856,0.3606193960,-0.3674240112,1.8943160772,2.4282639027,1.5565392971,0.8108466268,-0.4795611799,-1.8067957163,1.3225959539,-1.4695440531,-0.4730064571,-0.1111553162,-0.1633212417,0.7373108864,0.0697204918,0.1390757263,-0.5723037124,0.1945779175,0.4934951663,0.8671396375,0.0400937945,-0.8754099011,0.3754470050,-0.1142536923,-0.4761653841,1.7966411114,-0.2497129291,-1.3456119299,-0.2531846464,-0.0173894931,-0.8581447005,-0.8182291985,-1.2103476524,-1.8957642317,-0.5462412238,-0.3278253973,0.4181403518,-0.8027678132,-0.1248840615,0.3669672012,-0.4320286512,-0.6431820393,-0.3669171929,-0.4180790484,-1.1114394665,1.5662413836,0.4966558814,-1.5271258354,1.6809345484,0.9452425838,0.7926274538,-0.1260183305,-0.3414977193,1.1150146723,0.7628927231,0.7186394930,2.0608656406,-1.1427087784,-0.6720955968,0.4238626957,-0.2674430609,-0.1200173497,1.1119060516,0.2046580315,-0.1742334217,-0.6899335980,0.4895804822,0.5928009152,-0.6894159913,-0.7927412987,-1.3092082739,0.6599462032,1.1592344046,-0.6471576691,-0.8530796766,0.2530079484],[-1.2349803448,-0.4698171914,0.6378022432,0.0646075532,1.2665129900,-0.0035302599,-0.4264021516,1.9557579756,0.3368331194,0.1942038685,0.2976615727,-0.2447934896,-1.1720229387,0.0022187224,-0.5681318045,-0.0083157700,-0.2423404306,0.4079694450,-0.4025951028,-1.6668678522,1.1076716185,1.1742789745,0.8371176124,-1.2049195766,0.8989720941,0.5069872141,1.2650052309,0.5479833484,-0.6105635166,-0.7324912548,-0.7606137395,2.3526127338,0.3232249618,-0.0974526554,0.4910657406,-0.3990236223,1.5860863924,0.8555051684,-0.1618283838,1.5360722542,-0.0187115595,0.4534055293,0.5044534802,-0.2699193060,0.7831186056,-0.0310493931,-1.5355266333,0.1740212291,-0.5708485842,-1.2032914162,1.0133572817,-0.9925648570,0.6193821430,-0.5009004474,-0.0948746130,0.5210202932,1.4905010462,-0.0342098996,-0.7431033850,-0.1009846702,1.3803369999,2.7175147533,-1.6663825512,-0.6393872499,0.2210241407,-1.6114358902,-2.2916140556,-0.1636691839,-0.5086622238,1.3264782429,-0.7173157334,-0.6772971153,-0.3950392008,1.3992792368,0.4134455919,0.9333790541,0.8993307948,0.2048672885,-0.2898763120,-1.4870306253,-0.0224908516,-0.6100093722,-0.0982251316,1.4243254662,-1.5467219353,-0.3993415534,1.2259448767,0.1247667298,-1.8018198013,0.4228524268,-0.2830335796,0.2086140662,-1.8642035723,0.2794917822,0.8912532330,0.4764015675,1.1169859171,-0.8097556233,0.6917906404,-0.5510755181,-0.1711165160,0.7242781520,-0.1862911284,-0.8004393578,0.1734851003,-2.3327710629,1.0729398727,-0.0054144766,-0.6906583905,0.1949700415,1.1526728868,0.6277722120,-0.4168083966,-0.4985950589,-0.4415447414,0.4759062827,-0.9995189905,-0.0556426719,-0.8905441165,1.5150352716,-0.4012076855,-0.5104394555,-0.3579500318,-0.8876761198,-0.1260623485,1.2100745440,0.3163279593,0.1174208298,-1.0488599539,0.8006420732,0.8193587065,-1.4236061573,0.0530269407,0.8419333100,-2.3811285496,0.3123259842,2.0316591263,-0.1334241331,0.9566749930,0.3105371296,-0.0280005373,0.1698162705,-0.6434794068,1.4687440395,0.8223428726,0.5253647566,-0.9414321780,0.2501220703,-1.4306063652,0.4417558610,-1.3391911983,-2.1790165901,-0.6563906074,1.6905251741,-1.0324158669,0.0105966255,-0.4265025556,0.1774287373,1.1015805006,-1.2504085302,1.9962170124,-0.7548727989,-0.4495831132,-0.1260795742,-0.9168621898,-0.5386515260,0.4052873552,-1.6897302866,-1.1016709805,0.3024735451,1.5943657160,-1.4092345238,2.6607553959,1.1071485281,0.2390423715,0.4573163390,-0.1807434261,-1.9098607302,-0.3015142679,-0.5953611135,0.1685141623,0.3814010024,-0.0196545552,0.1171996817,0.5379750729,0.0300317835,-0.7584958076,2.2050728798,0.7647278309,1.4125244617,-0.6500667930,0.7842895389,0.6472234130,0.1368263811,2.1448404789,-0.4679664671,-0.3690626025,-0.8305638433,0.4829550087,-0.4260619879,2.0036153793,0.8908524513,-0.4578307271,-0.4667037129,0.5653272867,-0.2103756368,0.0654499233,-1.0563793182,0.8389202952,-0.9145900011,-3.0074131489,0.8634305596,-0.3165632188,1.3181333542,0.7771042585,-0.3728661537,-0.9975465536,1.3015630245,0.5809165835,-0.1991985440,0.4615896344,1.6888397932,-0.9476919770,-1.0568165779],[0.6325836182,-0.2203704268,1.1994246244,0.2282462716,0.7648639679,-0.6254203916,0.0122858062,-0.1240646094,-0.3559269309,0.5451278687,1.2897644043,-0.1418277174,-0.0237832293,2.4164583683,0.3128513694,1.2583760023,-0.0183320828,-0.2904647887,1.1263755560,-0.1271803826,-0.6991423368,0.4350317717,-0.9930130839,-0.0732888058,-0.0096755922,0.6082351208,-1.4622994661,0.3734008372,-0.2516731918,-0.5549135208,0.2704532146,0.8839444518,0.9480336905,-1.7732236385,-0.3400033414,1.0801545382,0.4076742530,1.1092234850,-0.5090354085,0.0154662654,0.8081864119,0.1444585472,0.4838244617,0.4002939761,-1.8577429056,0.0833002552,0.1059778705,0.7952242494,0.0375305712,-0.0192010757,0.5708048940,-0.6749990582,0.6472918987,-0.5778470635,0.5768222213,2.3813226223,-0.8882810473,-0.0370802470,-1.5818849802,-0.5403636694,0.9223807454,0.8768610954,1.6823604107,0.4641687870,0.5547270775,-0.7476307750,-0.1079692096,-0.5736877918,-1.1933667660,0.2192575037,-0.1325137913,0.0294413287,-0.7202627063,0.0728821531,-1.4440402985,1.1075712442,-1.7380994558,-0.3122649789,0.5126811862,0.0520603210,-1.6752490997,-0.4572330117,-1.4403076172,0.3279295862,-2.2079944611,-0.2930724025,1.5250643492,-0.7436814904,0.1235063225,-0.9559509754,-0.7831512690,-1.1331845522,2.1102159023,1.4193875790,0.7471066713,-0.1063024849,-1.9626525640,-0.8340319991,1.9938367605,-0.6666597724,0.9762901664,-0.8030210733,1.0035098791,0.4384184480,1.1932302713,-0.2671444714,1.6537593603,-0.0300932750,1.7405285835,1.1974927187,-0.2730903327,-0.4148294330,-0.9199048877,0.9715379477,0.2495393604,1.9860477448,0.3632284403,1.0493472815,-1.7863451242,0.4960638881,-0.5253154039,1.2082391977,0.7166950703,-0.1228279471,0.5477110147,-1.1976144314,-0.4063251615,0.1708601564,-0.5023768544,1.9816210270,1.4944957495,1.3486838341,0.1301545054,2.2598488331,1.0608019829,0.6828438640,-0.1969961673,-0.0942310393,-0.1965968609,-0.0145343514,-0.4977135956,1.4621585608,-0.5365141630,-0.1809267700,0.0045570266,0.5690778494,0.7586371899,0.2046857178,-0.2727931440,1.0885236263,-1.1514253616,0.0665233359,-1.5880743265,0.8120266795,-0.0526654460,0.0728608370,-0.0096358024,-0.1773585379,0.8013668060,1.6797046661,-0.2765279710,-0.5227367282,0.4652282894,1.0062574148,-1.0238008499,-0.3171005249,2.1590240002,0.1793651879,0.2974379659,-1.7805523872,0.2365886420,0.5838900805,0.8689589500,-0.1619296968,-1.8251862526,1.0525997877,-0.7366934419,0.2232283652,-0.2124611437,1.1224877834,-0.5253707170,0.8429335952,-0.4547739327,0.2139290422,-0.3529743552,-0.4005251527,-0.3588286340,0.5342850089,1.0697213411,0.3627375960,-0.6180936098,0.7206230760,2.3563475609,0.7881179452,-0.1104493737,-0.1020342410,0.9928730726,0.8416631222,-0.2109418511,0.0594552308,0.1714573205,0.9736061096,-0.2536408901,-0.5090725422,0.2091951966,-0.3952316642,0.6734473705,1.8492509127,0.5406158566,2.1666073799,-0.4017258286,-1.0504158735,-0.5410004258,-0.1275609881,-0.8653998375,-0.7439379096,1.7201974392,-0.3531041741,1.3364866972,-0.6355758905,1.0752155781,0.1554413140,0.8499841690,0.2878955901],[1.3831596375,1.0869204998,-1.9145219326,-0.6446622014,-0.6476334333,0.5867015719,-0.1330263019,0.3355934322,-1.0635930300,1.1624379158,-0.2087804228,0.9965528846,0.4709334671,-0.7888488173,-0.9004927874,-1.5506350994,-1.1885224581,0.0462546945,-0.2447022200,0.3237609267,0.5815305710,-0.0116930585,1.9005151987,1.5096068382,1.0625275373,-2.6801900864,-1.4928581715,2.7504143715,1.5735440254,0.1196702197,0.2730636895,-0.6364715099,-1.6727474928,-0.6743252873,1.0582365990,-1.6658549309,1.0274500847,0.8210723996,-0.3279028535,-0.5612089634,-0.8480519056,-1.2279438972,-0.6800284386,0.9341332912,-0.0853653327,-0.6074674726,-0.5700142980,1.1554375887,-1.6881010532,0.8331784606,1.5076262951,1.9043989182,1.4181431532,0.6705791354,-0.2154797763,-0.1985649318,0.8634696603,-0.2186039984,-0.7260090709,0.8744863868,1.0869740248,-1.0211362839,-0.4984046817,0.0550192371,0.1830883920,0.8126010299,0.8161264658,-0.8569269180,-0.1854367256,0.9796576500,-1.7851219177,-0.4285949469,1.2304253578,1.0196049213,0.1956699491,-0.9608377814,0.0585650690,0.5979899764,-1.1106897593,-1.0379048586,-1.7021956444,0.2371710539,1.3618772030,0.1308800131,1.5423973799,0.0521697253,0.3295921087,-0.3451442122,0.0585741326,0.2526168227,0.1740647852,0.0335522965,0.6994304657,1.5105129480,-0.4503269196,-0.4363505542,1.3006982803,0.8859831691,0.6783366203,0.7363558412,-1.7234025002,1.4570312500,0.8774670362,-0.6389059424,-1.0063931942,-0.4606885016,-0.7491079569,-0.4581098258,0.0056097927,0.3033953905,-0.5941976905,1.0164443254,-0.1951033771,-2.0242991447,0.9950196743,-1.6479471922,-0.2919598818,-0.4742799997,0.8809318542,2.0386807919,-0.1976701617,-0.3874321282,0.1774139851,-1.5380210876,0.7691081166,0.2360260338,0.1340331733,-2.0685832500,-0.4424262941,-0.2943103015,0.7442401648,0.5301663876,0.0796818957,0.8675574064,0.1207659021,0.4363836944,0.3957124650,0.4611773193,0.6354420781,-0.6371799707,-0.3056533039,0.4651468992,-0.1246628910,1.1826097965,-0.0713501945,0.4811518490,0.0363793969,-0.4016169012,-0.2674632967,0.3393284678,0.5643074512,-0.9851466417,0.5367643237,0.8051172495,-0.7623518109,0.3541601598,1.1396274567,0.7583637238,0.4819799662,0.5493212342,-0.2779793739,-0.5416448116,-1.0729044676,2.4494299889,-0.4952209592,-0.9134637713,1.0599906445,0.6519761086,-1.1102234125,0.3736817837,1.1824181080,1.0432199240,-0.1811021417,0.3057804704,1.5537210703,-0.3224816918,0.3417883515,-0.7627717257,0.3877569437,-0.5905166268,3.1262331009,-0.0636707246,0.2948612869,0.8857660294,-1.3919001818,-2.0759520531,2.0139601231,0.5181505084,0.2192502469,-0.4590292275,-0.7326875925,-0.0743509084,0.2777031064,-0.0589995608,0.7715876698,-0.1587277502,-0.5620646477,0.3577314913,0.3553095162,1.7608978748,0.3887453675,0.2869249582,-1.5057801008,0.1353569478,-1.1386996508,-1.7352130413,-1.0972082615,-0.8925606608,1.0090011358,0.4416737258,-0.1895652562,0.5086488128,1.2149100304,-0.5647054315,1.0612516403,-0.0578660406,0.3708175421,-0.0541841723,1.8225225210,0.3027867377,1.0238395929,-2.0085389614,0.0481917150,0.3897301257],[0.2223383039,1.1534072161,0.8207922578,0.0908544362,-1.3841704130,-1.4576156139,-1.2129918337,-0.9277918935,0.1623429805,-1.0262908936,3.5983099937,0.6017745733,0.5672770739,0.6544624567,-0.9301851392,0.1682607383,1.4109365940,-2.0551912785,0.9204959273,0.0109659592,-0.0355977714,0.1439855695,-0.2296297550,0.9706581235,1.0231167078,-1.0316967964,1.8201168776,-0.2633411586,0.4973664284,-0.8761262298,-0.7698000073,0.9660003185,0.2249306291,0.2809285223,-0.9330500364,0.4547836483,0.5424904823,0.0041835858,0.8023304343,-0.3718265891,1.3409324884,-1.7307722569,-0.0929673240,-0.1464788914,0.6054271460,-0.3584529757,1.1562112570,0.3190774918,0.9031167030,1.1098406315,0.3319199085,-0.7225655913,0.0292326510,-0.6798599958,-1.0965301991,0.8027890325,-0.2196585387,-0.6784861684,-0.6861836314,-0.9749602675,0.6559276581,-0.2036371380,-0.5718295574,0.2653274238,0.3244380057,-0.0364570580,1.8394060135,-0.8206712008,0.5605004430,0.3706909716,-0.8948367238,-0.1099870428,0.9826195836,-1.8920612335,0.8355630040,1.1913090944,-1.7892980576,0.8185464144,1.1664609909,-0.6384063363,1.4365177155,1.8991087675,1.4480994940,0.5353578925,-0.1014933810,0.2317328602,-0.0489891469,0.2788106799,-0.3530018032,0.7007697225,0.8251960874,0.5446463823,-0.2403146476,-0.2228565663,0.6680720448,-0.9187930822,-0.0626189634,0.3346115947,-0.0144286985,-0.9312236905,-0.1410451829,-0.1779760569,-1.4948040247,-0.8491395712,-0.6012837887,-1.1472615004,-1.0493052006,0.9051018953,-0.8528327942,-0.8717535138,-0.8167239428,2.2011229992,0.5912480354,0.7781169415,-0.8884767890,-1.3819385767,-1.1633492708,1.2937792540,0.3712132275,0.1289652586,1.7088830471,-0.1212667525,2.0375425816,0.2414857298,0.5944365263,0.6591821313,1.2509437799,0.8774040341,0.0764976963,1.6224048138,0.2065740824,-0.8375450373,0.3608026505,-0.7311544418,1.8801552057,-1.2250365019,0.7259657383,1.4483156204,-1.5710409880,0.8928659558,-0.1776217222,0.5828669071,-1.1092414856,1.0693781376,1.7471616268,-0.0638344735,-0.2843283713,0.1209072024,1.7735556364,-1.4085712433,1.3586884737,-1.7795503139,-0.5770805478,1.8389601707,1.5333522558,2.2969801426,0.6170276403,0.5831583142,0.0016300381,-1.4345458746,-0.3312624991,0.7281045914,-0.0612430722,0.1704209596,-0.7399250865,-0.9588273764,1.4042122364,2.0758225918,-0.7547942400,0.0572688095,-0.9924984574,1.1447558403,2.0738089085,0.4144458771,-0.4910028577,0.8720424771,2.1747357845,-0.3411389589,1.1494208574,0.3893336654,-0.2615806162,-0.1336776763,0.3566448390,-0.1948368698,-1.1085442305,0.0452541150,-1.6467444897,-1.0986956358,-0.4581389427,0.9293973446,-0.9444404840,2.2482736111,-1.8937467337,-1.0940096378,-1.5336623192,0.9742526412,-3.2672441006,0.5969296098,-0.4229683280,1.9410246611,0.3461483121,-0.1929189414,0.8508393764,-0.8942077756,-0.0658058375,-0.9040490985,-1.6763790846,1.9827840328,0.4141021073,0.7216985822,0.3340985775,0.4529460371,0.3948349059,-0.3916791081,-0.6501089931,-0.1761995107,-0.4910579920,-0.9518061280,0.5322595835,-1.1581377983,-0.7828429937,-0.8784741163,-0.7512039542,0.2515104711],[-0.7926872373,0.9031017423,-0.6620030403,0.2099865377,0.1103555858,-0.3409232199,0.3346741498,-0.5610801578,-0.6902984977,0.6454124451,2.1148784161,-0.1916747689,-0.5036607981,1.3881807327,-1.2380584478,1.7191636562,0.3221949041,1.5209794044,0.3738695681,-0.7806591988,-0.4377743006,-0.3996760845,0.9527902007,-0.1983201355,0.2340738177,-0.3172970414,1.4159215689,0.7529594898,0.7795801163,-0.2162986398,-0.1358234584,-0.9613436460,-1.3732218742,1.2092061043,0.6537166238,0.0339915529,1.4929498434,1.3207374811,-0.7652394176,-1.0177652836,-0.6362911463,1.2755696774,0.0088382503,1.0899994373,1.2225722075,0.7833430767,0.4043075442,1.7909801006,-0.3479894698,0.7831335664,-1.4179023504,-0.1100483984,1.2900805473,0.2464116067,0.9515271783,-0.3098686337,-0.3669249713,0.0978010669,0.8864583969,-0.1877962053,0.5702041388,-0.1159990877,-0.3501803577,1.9029144049,0.3099996150,-0.5302504897,-0.6566003561,-0.4687570930,0.9224168062,2.4704847336,0.0035063766,1.6607170105,-1.1333860159,0.1573023349,0.1733097583,0.3055340052,-0.1416791677,0.3056150079,-0.5185316205,-0.2780053914,1.1015346050,-0.9035406113,-0.4951810241,0.8598588109,1.0429813862,0.2664982677,-0.0298852306,0.8533706069,0.0039714552,0.9565080404,-1.5921889544,-0.8913908601,2.3364353180,-1.1469471455,0.0677223578,-0.9267605543,-0.2041192949,0.3063176572,0.2214103341,1.0146539211,-0.2558603883,-1.3943849802,0.1468503773,1.3297446966,-0.8142588139,-0.2132356614,2.4995143414,0.0899943933,0.0429980233,-0.1537914723,-0.4266037941,0.8056005836,1.1891802549,0.2974522412,-0.1454759538,0.1457174271,-0.0381532684,-1.4634613991,-1.3462141752,0.0664003119,-0.7865975499,-0.9306924939,0.3232910931,1.3516124487,-1.3557263613,-0.5764961243,0.2937407792,0.7960445285,0.2003103644,0.3335437179,0.1145793125,-1.4473557472,-0.3551094532,-1.2691313028,-0.1487769932,0.1636830419,0.9360257983,-0.7665287256,0.5842079520,1.2309166193,-0.8302861452,1.6384637356,-1.6886717081,0.7259944081,-0.9984105229,1.0805730820,-0.4680268466,-0.3358856142,-1.3471429348,-2.3989200592,-0.8478084207,0.1898977607,0.7270219326,1.8451247215,-1.4518831968,-0.1755070686,-1.7972141504,-0.3209051192,-0.2535520792,-1.1991846561,-1.0366666317,-0.3719597459,-1.2335456610,-1.3254798651,1.1153312922,-0.5936127305,-1.2354867458,0.4393461049,1.6991671324,1.6203832626,-0.0244100485,-0.3054488301,0.9963783026,0.5320534706,1.1051325798,-0.3606890440,0.4809477031,-0.6204119921,0.6224191785,-0.9839615822,-1.0425922871,0.8285560012,0.6690504551,-0.5170633793,0.1227343306,-0.5366504192,-0.4230706096,0.4494740963,0.0801548958,0.3285671473,-1.4552829266,-0.0545614995,0.3951006234,0.8749583364,0.7818165421,-0.7438398004,-0.3983262479,0.0770372972,0.1963945776,0.9916386604,-1.0397281647,-2.1135330200,0.2499190867,0.8048771620,-0.0107534034,0.6105771065,-1.0989814997,-1.4957785606,0.5590358973,-0.4976819456,0.0038733780,-3.1699786186,-0.5023010969,-1.2277863026,0.9048526287,-0.3307199180,0.3236357868,-0.3506893814,-1.8465671539,0.2423798144,-0.8711999059,0.5552427173,-1.0252454281,-0.9851835370],[-1.0336141586,-0.3072377741,-0.3015651107,0.9300719500,-1.1634535789,1.5237438679,0.4603967071,0.9846302271,0.7602651715,1.2697321177,-0.3905704618,-1.4983217716,-1.4723336697,-0.7304906845,-0.2598758936,0.8250431418,0.7544211149,0.6444281936,-0.8383622766,0.2967797518,0.7523125410,-0.7764243484,0.7958608270,1.2056927681,-0.9662641287,-0.9335420132,-0.1679661572,0.2740947604,-1.5847927332,-0.7598161101,0.0663753822,0.4885937572,1.4334609509,0.1223380715,1.2751848698,-0.5594834089,0.4535132945,1.4081346989,-0.4110103846,-0.4689775407,0.1422861069,-0.0778675601,0.6395888329,-0.5474281907,-0.1878197789,-0.5766133666,-1.8732328415,-0.1212131009,-0.0612858348,-0.1527992487,1.8027352095,1.4275883436,0.4526786506,-0.0216174275,0.7622801661,-0.1540553421,0.4228228629,-1.2467342615,-0.5674841404,1.2695670128,-0.8308476210,-1.9346109629,-0.5754521489,0.2582443655,1.0578256845,-0.4269259870,-0.4187313020,-0.5974100828,0.7509816289,1.0372412205,1.2170906067,-0.0723076239,-0.9454078078,-0.2535687983,-0.0566087402,-0.1577985138,0.7568653226,1.4414137602,0.4087726176,-1.5048319101,0.4968555868,1.2993911505,-0.9232976437,-0.4792942405,0.6598887444,1.9238653183,0.0458117649,1.7594987154,-0.2377487421,1.2578010559,0.0903284103,-1.4757454395,-0.7747513652,0.1500543505,0.5281760097,-1.5053366423,1.3702644110,-1.5342292786,-1.8576005697,-1.5551402569,-0.8589060307,2.2405879498,1.4200407267,-0.2279152572,1.5842866898,-0.5206685662,0.5317901969,0.8036642671,-1.5581449270,0.5541169643,0.3074723184,1.3566111326,-0.6996374130,-0.2088235170,-0.2496134192,-1.2829433680,1.9111602306,-0.0551519729,2.0919880867,-0.6781550050,0.1923809499,-1.1128199100,0.3574016094,-0.1978608668,0.6969423294,0.9032924175,0.1589456499,-0.4350588024,-0.3512411118,-0.0519459359,0.9904433489,1.2984777689,0.6284416914,-1.1503527164,-1.1220369339,0.3831738234,1.0778528452,0.3843923211,0.4374722242,0.7809922099,-0.0230867900,0.9145608544,-0.5255103707,-0.8014039993,-0.5153968334,-0.8046190143,0.9278858304,0.6122110486,-1.8956546783,-0.9318542480,-0.4475701451,-1.4089059830,0.5044367313,1.0466452837,-0.4882206023,1.6315196753,-0.6935045719,3.4683833122,0.6756458282,0.4363811016,-0.0252611358,0.7786549926,-1.3182072639,0.5751082301,2.3058328629,1.3533002138,-1.3967939615,-0.3641127944,-1.0209269524,0.2126713246,0.4525144398,0.2537213862,0.1484848112,-0.8377807140,-0.3130927086,-2.0773918629,0.8348540664,0.4066103995,-1.7121460438,0.4686309993,0.9460710883,1.7069529295,-0.6233937740,-0.8569788337,0.0303167701,-0.5756971240,0.1500440240,-0.2254502177,0.5234786868,0.7500634193,-1.2132166624,0.2510019243,0.2570083737,1.0357878208,-0.6433942318,1.4982335567,-0.9077165723,1.4716104269,-1.7842148542,1.1447430849,-0.6266376376,-2.2693505287,0.3182332516,-1.0613924265,0.6442693472,-0.5000963807,-2.0452585220,1.4310381413,1.8100054264,-0.6701334715,1.5482307673,-0.5137092471,0.5081887245,-1.3406304121,1.3682693243,1.2737818956,-0.3865059316,-0.4780690670,0.7779576182,-1.7476885319,-1.1008824110,0.0263148081,1.8815983534,0.3323176205],[0.2674674392,-1.3087997437,2.1118044853,0.2675665617,-0.4922035635,1.7720580101,-0.7110042572,0.1589149386,-0.4881309569,-1.5783791542,-0.0925537944,0.5986972451,-2.1295366287,1.7461127043,-0.9785813689,0.3955598176,0.6764844060,-0.9934566617,0.7253823876,2.6917986870,-0.5455944538,0.4043146968,0.3411338329,-0.3855164051,1.6743670702,0.5616671443,1.5067958832,0.2787322402,0.8779776096,-1.8535884619,-1.3119099140,-1.4514750242,-0.3689820766,-0.6068240404,-2.3265423775,-0.2324558496,1.0002081394,-0.6160846949,0.0170312542,-0.2363373041,1.9653319120,-0.2599737346,-1.2773052454,0.4339766502,-0.6928628683,-1.0053579807,-0.1896031797,0.6886450052,-1.9172238111,0.4927026331,0.5296412110,0.2830281556,2.3585255146,0.1015405208,0.1898712814,-0.3793027103,0.2264746577,-0.1147489175,-1.2718250751,0.5322838426,-0.1509367973,-1.0796653032,0.2211570591,-1.6349580288,0.7289609909,-0.0924888924,1.2654665709,-0.7704129219,0.3740957081,-0.6158271432,-0.6735936403,-0.7281390429,-0.1193343252,0.7776176333,0.4423677027,1.3741822243,-0.4074907601,-0.1957744956,0.4908902943,-0.7967771888,-1.5564359426,0.6080636978,1.1600704193,-0.3984555602,-0.1730147898,2.0433011055,0.9244021177,1.0907392502,-0.1569561809,-0.9651011229,-1.5164184570,-0.0046005156,1.8511489630,2.0630350113,-1.8583544493,1.2922915220,-1.0781641006,0.3176600039,-1.4096624851,-0.6563232541,-0.1117633730,0.6297587752,1.2154203653,0.2888377905,0.7956738472,0.3510543406,-1.1075544357,2.0214886665,0.9916204810,0.5647026300,0.5993618965,0.5231040120,-0.8116585612,0.6569921374,-1.8315339088,-0.6885490417,0.3351188302,-0.0013942909,0.8267314434,-1.9799656868,-0.6828018427,0.7968899012,-0.2808510363,0.0512608960,0.1743437499,-0.7116479874,-1.9787137508,0.5628860593,-0.6114267111,-0.4011185169,-0.2396005392,1.0169463158,-0.8091300130,-1.1353332996,0.0133572351,1.0202590227,0.2226665467,1.1409810781,-1.5810728073,-0.3684642911,-0.1554839909,-2.3100559711,1.0542657375,1.6269677877,-1.7038469315,-1.2879344225,1.0403683186,2.2401287556,0.1703069061,-0.5179620385,1.0845952034,0.4455762208,0.2167074829,-0.4839757085,0.9151929021,-0.5111671090,-1.0387377739,0.8055750728,-0.5076743960,0.0328205563,-1.5801910162,-0.4398359656,-1.1315128803,0.2544636726,-0.6623916626,-1.1471446753,0.2642677724,-0.1311631501,0.1489771158,0.8975483775,-0.2278074473,0.1106519401,-0.4545907676,0.0902820975,-0.3923905790,-0.7976117730,-0.0126832854,-0.3523421884,-0.1040960327,-0.2254287302,-0.4944473207,0.5934621096,1.1226073503,-2.1177215576,0.6347402334,0.1774308234,0.1942605078,-0.0195572022,-0.1038614362,-1.2880922556,-1.8354381323,-1.0152963400,-1.4097815752,1.7686097622,0.6368556619,-0.1358798593,0.7228752971,-0.9184829593,0.2026240230,-1.6902302504,-0.7515771389,-1.4603588581,0.1438691318,-2.3376917839,0.8028458357,-1.7040096521,0.8609021902,1.7478538752,-1.3853265047,-1.2477948666,-0.5389990807,0.7780463696,1.2358132601,-0.9333853126,-0.4348754883,-1.2053111792,0.2426356375,1.0397968292,-1.2316269875,0.6959838867,-1.7544223070,-0.6464222670,-0.4856535196,-0.3731232882],[-2.0051338673,0.2533751428,-1.3090395927,0.8500843644,0.5513440371,1.5281857252,0.9997274876,0.7820481658,-0.4565566182,-1.1126084328,0.5382915139,-1.1646991968,0.3164305091,-2.3938493729,-1.0044484138,0.4640351832,-1.2155511379,0.1764370948,-0.8172885180,-1.2739762068,-1.3600958586,0.8626067638,-0.9916674495,-1.2624299526,0.9874475598,-0.2145728320,-0.5082772970,-1.6458400488,-0.3702574968,-0.9768534303,0.2904407978,1.3255532980,1.2100210190,0.1227659956,0.2595930994,-0.4679057598,0.4274121821,-1.3154900074,0.1236572117,-0.9854791760,0.5440257788,-0.5402190685,-1.1039479971,-0.9907112122,-1.3060643673,-0.5652656555,0.9776216745,0.2473392487,0.8499594331,0.7057187557,-1.3080376387,1.0187844038,-0.0227408782,0.6088210940,-0.6433678865,0.6544898748,0.9204548597,-2.7137091160,0.7325530052,0.0752336457,0.8366841078,0.9279086590,-1.1596821547,0.3568362594,-0.2196850330,-0.2727072835,0.5928372145,-0.1771852970,-0.2883036137,1.1079210043,-1.0769922733,0.4753883481,0.3450099528,-0.3523810506,1.4889559746,-0.2710278332,0.2396248132,0.6586426497,1.0482380390,0.3379995525,-0.4739660025,1.6513723135,-0.8296175599,-1.3067449331,-0.6117827892,-1.7295495272,-0.9451754689,-0.6887831688,0.4632112086,0.0821003988,-1.1309273243,-0.1603619158,-0.5950310230,0.3923758268,-1.5252598524,0.4210940599,-1.5164136887,-1.2393224239,-1.7814563513,-0.2897252738,-2.1949856281,2.8120102882,-0.8257092834,0.1937599629,1.0681219101,-0.8102371693,1.1713981628,0.3316555321,-1.2564464808,-1.2102553844,-1.8110905886,-0.3118630052,-0.6994720697,-1.5547536612,2.2070877552,-0.5162882209,1.3778606653,-0.7898684740,0.2299599946,0.1487733722,-0.8022645116,0.5625656247,-0.8308770061,-1.2218902111,0.7719486356,1.6793210506,0.6410716176,0.5552347302,-0.6866862774,-2.3685967922,-0.5928680897,-1.2760992050,-2.5390007496,1.2587535381,0.4895009398,0.1038385183,0.5110086799,0.3381683230,0.7548806071,0.1276102215,0.6869242191,1.2325774431,-0.6868656874,-0.8387863636,-0.8901911378,-0.8299873471,0.6570521593,-0.8497098088,-1.9196294546,0.1074195951,0.9909681082,1.0813125372,1.2957479954,0.7512951493,0.6021413803,-1.1488620043,0.6352616549,0.0198759958,0.9322074652,-1.4032728672,-0.5972887278,-1.1572170258,-0.1391252577,-0.0026114394,-1.2078139782,-0.6075099111,-0.1559657305,1.2921199799,-0.6285111904,-0.5723410845,-0.0440929458,2.2640471458,1.1017858982,0.1785074621,-0.4585222602,-1.4878432751,-0.4616599381,2.7842957973,-0.8562133312,-0.8348052502,0.5125629306,-1.6848424673,1.6984417439,0.4297846258,0.6659538746,-1.0364086628,-0.8324087262,0.2216985375,0.5052349567,-0.4887041748,-0.1160030439,0.7263070941,1.2204111814,0.1790825576,1.0658636093,0.2977754474,-0.5876668096,-0.7762431502,-0.5158827305,0.2565769553,0.8809493780,0.8431380987,-0.4027089179,-0.1782636046,-0.1587935835,1.3338432312,-1.4606239796,-2.0378668308,-0.1473344862,1.1285791397,-0.4220150411,1.2866398096,-0.2262062728,-0.6299557686,-0.7837253809,0.7857016325,-0.2874922752,-0.7144582868,-1.5693187714,0.4702760875,1.3465423584,1.0922623873,0.3292343020,-0.3397429287],[-1.1707133055,-0.8345708847,0.1053590998,-0.9551098943,-0.0158422422,-0.6733519435,0.5780305862,0.7031232715,-0.0393282436,-0.9383910894,0.3805969954,0.7196177244,0.2210744172,0.5715443492,0.2615196705,1.4784181118,-0.0361505449,-0.9886692762,-2.5481803417,1.3661826849,-0.3472016156,-1.6215758324,-2.0042810440,0.2201344520,-0.1717495918,0.6369396448,1.3920063972,1.8210952282,-0.6172340512,0.1448362023,-0.4834059179,1.1414402723,1.8423522711,1.4584639072,2.5065975189,-0.4695457816,-0.2602095008,-0.5626283288,1.9504405260,0.4526340365,0.4930631518,0.6368220448,-0.7115779519,0.9143373370,1.0674004555,-1.4705325365,-0.1610876769,0.0818736106,0.1498768330,1.6491193771,0.0072635375,0.7203763723,1.1762765646,0.2638449371,1.9991233349,1.0177111626,0.3286144733,-1.2706656456,-0.1446818113,1.7641743422,0.8927143812,0.4896416962,-0.9942132235,0.9834810495,0.1882795244,0.7997056246,-0.6269438267,-0.7839465141,-0.7389545441,0.7751408219,0.7793362141,0.5096878409,0.2314013690,-1.4001361132,-1.2783879042,0.3915791214,-1.6315677166,1.2537792921,-0.7252321839,0.9186237454,-0.5066030622,0.5364947319,0.3846390545,-0.7608844638,-0.9879293442,1.0536998510,0.1239340007,-0.6024615765,0.7722412348,0.6044166684,-0.1003129780,0.7670496702,-0.5458506942,-1.4519466162,0.2771151066,0.3518697321,0.0249520000,2.6057703495,-0.9137113690,0.5218591094,0.0153903477,2.6444854736,1.3167178631,0.3327669799,0.6593927741,0.2390110046,-1.4895582199,0.6340134144,-1.6390534639,-1.2675498724,1.0433050394,1.2519178391,0.3263645172,-0.1726632714,0.5519032478,-0.1143158376,0.2162200809,0.9341704249,0.3033621013,-0.5463930368,0.8753668070,1.3833848238,0.8291720152,-1.2218199968,0.5906565189,-0.7905145288,-1.7524110079,0.3663987815,2.1609265804,0.8671112657,-0.5826405287,0.1804178357,0.6935610771,-0.6312169433,-0.2545692325,0.1374749988,-1.4491913319,-0.0883082896,-0.0105298012,-1.2318902016,-0.8289853930,-0.3602082729,-0.8703395128,-0.7262350917,-0.6620940566,0.6471897364,0.0936652049,-1.3356847763,-1.9162960052,-0.1310082227,0.8927040696,-1.8789820671,-0.6941149831,-0.8801056743,-2.4179081917,-0.3510309160,1.0250413418,-2.0253155231,-0.2171654552,-0.6943956614,0.4614755809,-0.2940581143,1.0993890762,-0.4676396549,-1.0508223772,1.7287107706,-0.6683145165,-0.1298807859,2.1324386597,0.9666262865,0.8443111181,-0.6013779640,0.7276886106,1.0302677155,-1.2724297047,0.9751001000,-0.1170990020,-0.2573932111,0.0356474034,-0.5298755169,-0.9047068954,-0.1097842306,0.8076342344,-0.3035034239,0.7364196777,-0.1532740593,-1.5517117977,1.3448683023,0.9069454074,-1.1535201073,-1.1836688519,0.0531910323,-0.9787717462,-0.8477154970,1.5395288467,0.1366031617,1.1396311522,2.3474342823,-0.0828050002,0.5587758422,-0.1701462269,0.2563884556,0.4576572180,-0.5936612487,-0.0666432232,-1.2495532036,-1.7759221792,-1.0503594875,1.4247668982,0.3914461732,0.0761216953,0.2842491865,0.6038943529,1.2931584120,-0.2688145936,0.6839090586,-0.0863594636,0.1521898508,-2.1591539383,0.3447727859,-0.4143142998,0.3156183362,0.1544964314,1.3025699854],[-0.0636318251,0.6497617364,0.5397651196,-0.2517716885,-0.9968451858,0.2427535653,-1.6707130671,0.4256668091,0.4201525450,1.6644111872,-0.1637274027,-1.0096265078,-0.7501395345,0.3889355659,-0.3280121088,0.0683555230,-0.2293541282,0.0059699183,-1.2834771872,-1.4719812870,1.0352134705,-0.1490982920,0.2533560693,-1.1749132872,0.5028715730,-0.2666368186,0.7678294182,0.5390011072,0.1143793166,0.0369447507,-2.1100959778,-0.1035841182,0.1734765768,-0.5206263065,-1.5388579369,0.2944219708,-0.5542598367,0.3960934281,-0.5660918951,-0.5781425238,-0.4664564431,0.0804147124,0.4390437007,-0.0616029091,0.7003066540,0.6478360295,-0.2125269622,0.9341881275,-0.9784104824,0.4180209339,-0.6914352179,0.7919614911,-1.1545484066,-0.0029162788,0.8164563775,-2.1450593472,-0.7141282558,-0.1273373812,0.9063172936,0.7292864323,-1.4445300102,-0.5211780071,0.7087510228,1.8131198883,-1.2041488886,1.4940724373,-1.3090559244,1.7167582512,-1.7583614588,0.3108649254,-1.5711164474,-0.6901423931,-1.0158981085,0.4401728809,1.6992042065,1.7186495066,0.5683168769,0.7231294513,0.0740922242,-0.3422453701,-0.4967070222,0.7054743171,1.3211479187,0.2724389136,0.6408790946,-0.9621573091,-0.3153292537,-1.3749207258,0.6791490316,-0.7940768600,-1.0095255375,-1.3924701214,-0.2026451379,0.0677531809,-0.1508983523,0.5414183140,0.7144338489,-0.6705932021,-1.3298588991,-1.3412828445,-0.5634455681,-0.2916662991,0.7729437947,1.5142472982,-0.0278050378,-0.5018460751,0.6177078485,0.0424504094,0.2765903771,0.6403551698,-0.5025407672,0.5923534632,0.6478619576,-1.5444386005,-1.2579747438,1.2809257507,0.6353840828,0.4131743014,0.3158742487,1.1801162958,-0.4564170837,0.4091522694,-1.4798978567,1.2729399204,-0.8004751205,-0.4432139397,0.3686844409,0.7507426739,0.9426909089,1.6492742300,-1.5455055237,-1.2340234518,-0.9009966254,-1.2735397816,-0.0615484640,1.8640985489,-0.3493136168,0.2173888832,-2.4374730587,-0.0383728780,-0.2510394156,-0.7139562964,0.8909270167,1.3366739750,-0.0830413997,2.5130395889,0.0980813131,-0.8486409783,0.4998627305,-0.4094617665,0.5458596945,-0.4862162173,-2.0455002785,1.0393195152,0.1168493107,-1.7165883780,-1.1070877314,0.2518376708,-1.2448687553,0.5589826107,1.7770667076,-1.5959331989,1.5407427549,-0.5897306204,-1.8142468929,0.1698196530,-0.1295103729,0.5277545452,0.2446128577,-0.3235076964,1.2561858892,0.7550396919,0.1513881832,-0.4214838743,0.8148363829,1.4301729202,0.1300896555,0.4603887796,-0.8699390888,-0.0237311311,-1.1238359213,0.1377335340,0.2441793680,-1.7290854454,-0.4972402155,1.3003982306,-0.5680015087,0.5945675373,0.7442512512,-0.2569022775,-0.9850519896,0.2363358736,0.3354244232,-2.0168704987,0.6541719437,0.1009543091,0.0797480121,1.6220499277,1.1278858185,1.5643399954,1.7326484919,0.0972388163,-0.0682537928,0.5457283258,-1.9053533077,1.7635300159,-1.5515018702,-1.2203807831,-0.1801347584,-2.6934776306,0.1715345532,0.6029487848,0.0573038310,2.3573427200,-0.2083336562,-0.4516532719,-0.2417075187,0.4663243592,-0.6326389909,0.1208775267,0.2149387449,1.3439264297,1.2751500607,-0.4505316317],[-0.5756392479,1.4063438177,-1.4878020287,-0.5038126111,0.9840235710,-1.0844315290,-0.2784172595,0.4415511191,-0.1162850186,0.1757821590,0.7411425710,-0.3668819070,-0.7205744386,-0.5024018884,0.2062256783,0.7241781950,1.5902775526,0.0932773799,-0.6890472174,-1.9990137815,-0.4262224734,-1.1738171577,1.1300529242,0.4181618392,-2.3131039143,0.5922805667,-0.3001381159,1.7097831964,-1.1595673561,-2.1293334961,-0.3307192922,1.2602394819,-0.7109816670,1.7607816458,-0.6528120637,1.6467469931,-0.9482585788,-0.8347230554,-0.6540571451,-0.3115817606,-1.6141192913,-1.0731090307,0.0426733084,-1.0998947620,1.0489722490,1.4080376625,-0.3871131837,0.2852624655,1.3540599346,0.5000309348,-0.0193734169,1.1629503965,1.7707066536,-0.7773700953,0.8161155581,-0.6074299812,1.9348500967,-0.7318091393,0.9646677375,0.4912351668,0.5005277395,-0.1886811703,0.6468701959,-0.6115835905,-1.4020545483,-0.2524648607,0.0008449311,-0.3852461576,0.2591911256,-0.2682701349,-1.5723091364,0.3320451081,3.2006881237,0.6155536175,0.8023455739,0.1190806627,0.1179326773,-0.0133221503,-0.1462981254,0.0558719449,0.0060142102,0.7527711987,0.2324405760,0.5915004611,-1.0211442709,0.1468193084,0.9947727323,-0.1153474972,-2.2038846016,-0.3664562106,0.8300547600,0.7540668845,-1.2107092142,0.2661699057,1.3505961895,1.9395216703,-0.1923337281,-0.5798934102,1.1088138819,-0.5727385283,0.6018813848,1.7457513809,-0.7539621592,0.3392607570,-1.8162858486,0.4703977704,-1.6360388994,0.5180320740,0.3072411418,-0.3721626103,1.7841976881,1.7781287432,0.7256065607,-0.5469256043,-1.1149164438,-1.2010357380,-0.0934013724,-0.4959301949,-0.2157954574,-2.6348752975,-0.7417191863,-0.5041836500,-1.9211819172,-0.4793742895,-1.3138118982,-0.6303048134,-0.8648637533,-1.5271095037,1.2077393532,1.3537611961,-1.0298615694,0.2516765296,-1.1857849360,-1.2759015560,-0.1323886216,0.0077520325,-1.1594430208,0.1734642833,-0.8420557976,-0.1126872674,-0.4606709182,-1.1694540977,2.2664411068,0.4393515587,0.8670080304,-0.6363406777,1.0128105879,0.5352993011,1.2980554104,-0.3366172612,0.7446729541,0.0706164017,0.5329166055,1.5777875185,-0.7912862301,-2.4273786545,-0.0999521613,0.4401312470,-0.2123771459,-0.8119220734,-1.1485599279,-0.4177002013,-0.1898192316,-1.5691518784,0.7123882771,-0.4033349156,0.1992595643,-1.0107963085,-0.7487183809,-0.3474726081,0.5490239859,-1.3117270470,0.4529985487,1.2649879456,-1.3282953501,2.2694823742,-1.9726030827,-0.1417467296,2.0489106178,0.6848044991,-1.1864144802,0.0095336009,0.5513510108,0.4967232049,-0.5022209287,0.7636085153,0.7493492961,0.9887427092,1.0534604788,-0.5208474398,1.5930726528,1.1946299076,-1.1940947771,-0.6941130757,-0.2753409147,0.3418467939,-0.3992665112,0.0144357430,-0.1594496369,-0.4563882053,1.4027935266,0.2411439270,0.0072475886,-0.8765354753,0.4561424851,-0.0980236828,-1.0085912943,-1.3533272743,-0.8581139445,0.8061968088,-1.0639472008,-0.1058177575,0.4276168644,-0.8239603639,1.8089847565,-1.5798636675,-0.0302213319,-0.3193961382,0.5277996659,-1.4658607244,0.5939242244,-0.8696262836,0.3683566749,0.4510522187],[0.5968563557,-1.1456565857,-0.0630941987,-0.0725090280,-0.6521866322,1.9407103062,-1.9997001886,-0.4733425677,-0.6821983457,-2.0067911148,-0.1969090551,0.0117811309,0.8634021282,0.5490431786,0.5036259890,0.8777194619,-0.5009964108,-0.2476846427,0.7691996694,-0.4389582276,0.7626179457,-0.9946994185,-0.9002199769,-0.2358198315,0.5558941364,-0.5835582614,0.0424835086,0.5476747155,0.4497909844,2.0062038898,0.5630492568,-0.4302353561,-0.3927549124,0.6127817035,-0.3549419045,-0.0465914086,0.7972400188,-0.0291995928,-0.1873065531,-0.2666181028,-0.5908914208,0.4490607083,0.4655361474,1.3770449162,0.3744198084,2.0639455318,-0.5035088062,2.2640516758,0.8200385571,0.3762246370,0.1349862367,-0.7485954762,-0.6919678450,-1.5081450939,-0.8550764918,-0.3346778750,-0.2704392374,0.7448489666,0.6368254423,1.1909401417,-0.3652415872,0.0266696922,0.1857023090,-2.1289024353,-0.0612011552,-0.1647111028,1.5982892513,0.3783888221,-0.8879693747,0.1596832424,0.9702088833,-0.4322770238,0.7622994184,-0.9912603498,-0.0206081625,-1.8675857782,1.3297318220,-0.2855650485,0.4054629803,-0.6809203625,-0.5133872032,-1.8930710554,-0.0781741962,-0.4591271281,1.4383069277,0.9537320733,-0.9098660350,-0.5606174469,-1.1290600300,1.0253957510,-1.3523454666,0.3395365477,0.7096471786,-0.4674813449,-1.6451238394,0.8107517362,1.7173854113,-0.0553687960,0.3889346719,-1.2086811066,-0.0176033434,1.2727808952,-2.3618505001,-0.0911023468,-0.6099271774,1.4351190329,-0.2916943431,0.2546273768,1.4699568748,-0.0347926132,1.3543151617,0.0582686812,0.7119902968,1.3041653633,-0.2682042718,-0.3871650100,1.2333360910,-1.1758619547,-1.5743108988,0.2008384019,1.6558495760,1.3954097033,0.2431987077,0.6873512268,-1.5398448706,-2.1012260914,-0.8874242306,1.0868637562,-0.8933115005,-2.3817610741,-0.2322047204,0.5744912624,1.1864047050,2.0981147289,-1.0142970085,-0.2725763023,1.1405632496,-0.3188109994,-1.6895825863,-0.9098422527,-0.9992399812,0.2970742285,0.4739212394,-0.3946261406,0.9657480121,0.1714322120,-0.4725795090,0.0793206245,-1.2109538317,-0.9714989066,-1.4903097153,0.5481170416,-1.3274215460,0.1033664346,0.3240928352,-0.0927641466,-0.7859684229,-0.4100660086,0.1349842697,-0.6123028398,-0.4821889997,-0.0113635045,-0.6305410862,-0.6447528601,0.9588113427,2.0857424736,1.6361919641,-0.0809310004,0.4175556898,0.5576475263,1.1547952890,-0.9088178873,-1.3321971893,-0.6993666887,0.8384341002,-0.0056715696,0.0963145494,-2.0406420231,-0.5699337721,0.2109227180,0.1758637279,-0.1132779494,-0.1388639659,-0.5674633980,-1.0927742720,0.4664836824,1.3453660011,0.9642984271,0.1735391319,-0.6066111326,-1.8372893333,0.0989775285,-1.0595743656,-1.7484507561,1.3843749762,0.0083545428,0.3311144710,-0.2330202758,-0.4341944754,-0.3969227970,-0.7791808248,1.1586873531,-0.7870743871,-1.3215303421,0.5297895074,-0.5042585731,-1.6013978720,0.6434103251,-1.1106889248,1.0323313475,0.0253233518,-0.5326119661,-0.7091501355,1.1146080494,0.3246107399,0.4328122735,0.1061507687,0.9968009591,-1.5700018406,0.9111320972,-0.1880164742,0.0720187649,-0.1722674072,-1.2956403494],[0.1623233408,0.9533376098,0.8453432918,1.0247024298,0.8314163089,0.2169153392,-1.1605566740,0.6202650666,0.7922874093,-1.1443600655,-0.1555470228,0.2054934651,-0.0696882978,1.5672497749,-0.6392518878,0.6004613042,0.2808816433,0.9697634578,1.3994048834,-0.9369864464,-1.0598493814,0.6915630698,0.0257297009,0.5673299432,-1.9622377157,-0.7675710917,0.3625481427,0.7994940281,-0.0520695336,0.0688188747,0.7390265465,0.2610666752,-1.5965384245,0.3458299041,-0.7521305680,-1.3208951950,0.5133634806,0.6769243479,0.8414777517,1.7870870829,0.5016800761,0.6279888749,-0.0259170141,0.0858539417,-0.1149455160,-0.9918748140,1.3966687918,0.5319203734,0.0872511491,1.2238022089,-0.6457444429,0.6148810387,-0.2943848968,-0.7676346898,-1.9957956076,0.0006206976,-0.4282977879,-0.3669142425,-0.2334992588,-0.1813053340,2.3843736649,0.0274813995,-0.2801706195,0.0149513474,-0.2649163604,0.2022246122,-0.3847317100,-1.1576153040,1.6478676796,0.5820747018,-0.2079120427,-0.1404580623,0.9750667810,-0.6656797528,-1.0515507460,-0.5181403160,-0.0723514110,0.9246936440,1.0558516979,-0.2617908716,0.0100078853,1.4384540319,-1.5206338167,0.7696095109,-0.7830299139,-0.6658165455,0.8587329984,0.0302361734,0.7325266004,0.6763762832,-0.9670042396,0.5436664820,-0.0928966329,0.2728548050,0.7476511598,0.8182846308,0.0401502326,0.0703607574,0.7100747228,-1.1809411049,0.1525679231,-1.4360591173,-0.9362903833,0.6512863636,2.5559821129,-0.4898336828,-0.5570479035,0.3082244098,-0.1268011928,-0.1690266579,1.1079031229,0.5577710867,1.2079291344,0.3057589829,1.4704600573,-1.5299013853,-0.4372470081,-0.1838792264,0.4103325903,-0.0079925116,-1.4350907803,3.0050628185,1.9440249205,-0.5308479667,0.7670062780,-0.1695310771,-0.4937400520,-1.0065364838,-0.3335910738,0.2164574564,2.0567500591,1.0385917425,-0.9172860384,0.1360678822,-1.4758296013,0.2109170258,-0.6283934116,0.4619432986,1.2396088839,-1.2901101112,-0.4787948728,-0.0482119285,-1.3737345934,0.3587618470,-0.1970571727,-1.1884554625,-0.3847505152,-0.4059247375,-0.8413720131,-0.2346312255,-0.5250424743,-0.8797713518,-2.2436685562,-0.1204453632,-0.9047447443,0.1623364985,-0.1249333546,-0.4604585767,-0.1806154251,-1.5818452835,0.7564077377,-0.3526729941,-0.5011270642,-0.4888699055,0.2328237146,-0.8190051317,-0.8744320869,-1.6677303314,1.3656442165,1.3829454184,-0.1329779178,0.6396308541,-1.6524884701,0.8636744618,-0.6694845557,0.8651663661,-0.9834919572,0.9883199930,-1.2616648674,0.2136829346,-1.4948030710,-0.5576620698,-2.1199064255,-2.6051251888,0.8021505475,0.2092306316,0.8647927046,-1.0634721518,-1.7716604471,0.1749333441,-1.3047918081,0.8585357666,-0.0653507933,0.8114531040,-0.8026093245,-0.2348457277,-0.1972251832,-0.1731291860,-0.9763115644,-0.4425777495,0.1876048744,0.2390181720,0.5495560765,-0.4430482984,0.5191142559,0.3313212991,-0.2732692659,-0.1305333972,-0.3552298546,-1.2423542738,-0.4154689610,-0.5192009211,-1.0146884918,0.0872632042,1.0132828951,0.7355144620,-0.8626582026,-0.9035837650,-0.1659100503,-0.4304459989,0.8710014820,0.2093449235,1.7577981949,-1.4495869875],[0.3389512897,0.3277698159,-1.3833289146,-1.0309791565,0.5938034654,0.3381133080,-0.7506350279,0.5864959955,0.6818603873,0.9190769792,-0.9943068027,0.7560220361,-0.2823075950,0.4619690776,-1.0569438934,-1.6896722317,2.2968416214,0.1863039136,-0.4671124518,-1.7164677382,-0.0640422776,-0.9713795185,0.9030221105,-1.0318793058,0.7022774220,-2.5937387943,0.9347560406,-0.0407111496,-0.0842949972,2.1857056618,0.8122920990,-0.1597754359,-1.2797119617,-2.5321006775,-0.9836844802,-0.3681376874,1.4671288729,-2.6063854694,0.1413185447,1.1272912025,-0.3143338859,0.2970275581,-0.9995012879,0.3840419352,-0.8541126847,0.1262938529,0.8874447942,3.3216011524,-0.0434766375,-0.2402396351,-1.7334960699,-1.1922539473,0.0395632051,0.3123318255,1.5156829357,0.1709397286,0.4083366990,-0.2023052424,1.5522542000,-0.4802343547,1.6968504190,1.4456083775,0.2222046554,-1.3635166883,0.4523965716,-0.1486860216,-1.3813056946,0.2926163673,-1.4815069437,0.0460004359,0.0685515702,0.8657517433,-0.4926210642,0.3616339266,0.2384285331,-0.5062540174,0.9793159366,-0.3106910288,1.4108459949,1.4620524645,0.2490852475,1.3678778410,-0.6981250644,1.0042148829,0.9979116917,0.4013634026,-0.5294713378,-0.9604782462,0.7616765499,1.0867910385,2.4451165199,0.5009768009,0.5941057801,2.1568233967,-0.2864334583,-0.7203481793,0.2201959938,0.0211936776,0.2499561757,-0.5913699269,0.4207766056,0.5149331689,-2.2674925327,-1.3170949221,0.0806484967,1.6441719532,-1.6811925173,-0.7950924039,-0.5914666057,-0.4738128781,1.3944123983,-1.1955771446,-1.4249742031,0.6435713768,-1.5451512337,-0.0101473620,2.2087171078,-0.9800808430,-1.4776380062,0.7782774568,-0.7789295912,-0.0102034928,-0.0880911723,-0.4840727150,0.7514572740,0.0549944900,-2.4430112839,2.1914079189,-1.0618168116,1.4040321112,-0.2890518904,-1.8754194975,-0.4907815754,-0.0630276427,-0.7582322359,-1.1303482056,-0.7832784057,-0.5160414577,0.0700457245,0.6622155905,1.1924040318,1.4414311647,0.6065171361,-0.4135439694,-0.1386700422,-1.0272814035,0.4866032004,0.1530758739,-0.3026266694,0.2846779227,0.0424417257,0.5401003361,-0.7115367651,1.5831142664,-0.2675635815,-0.0213743374,-0.0043467307,-1.5642577410,0.9680214524,-1.3408775330,0.1488857120,0.2985931039,-0.5775466561,-0.7951164246,-0.4007186294,-0.5162972808,0.4733025432,0.2995747328,0.8256702423,1.2272002697,-0.0659088045,0.4785670638,-1.1270323992,-0.5993215442,-1.5867478848,0.1878347248,-1.2973229885,2.1392738819,-0.5237012506,0.4061898589,0.1321331561,0.1819093823,0.3166199625,0.1830487847,1.1566855907,-0.9360304475,-0.1669008881,-0.5197641850,0.7517486811,0.0689731315,-0.3164053559,0.2283985317,-0.4720411599,0.3385706842,0.4731729925,0.4821418226,-0.7884815335,0.3731731176,-0.7758998275,0.7502189875,1.0179754496,-0.2132595032,-0.6256816387,0.6563635468,-2.0351250172,-1.6866027117,0.7681083083,-1.0538374186,0.0967316404,-1.3432598114,-0.2516896725,1.4454326630,0.1537240297,0.7852683067,0.1986494809,-0.2295962125,0.3377442062,0.9981122017,0.7033475637,0.8179015517,2.8854703903,0.1449834108,1.5653721094,-0.8830209970],[0.2447814792,-1.6657313108,-1.0418515205,-0.0914706290,-2.2588729858,0.2607210279,-0.6904564500,-0.2766918540,0.4198768735,0.6650484800,1.2362228632,0.2318393737,-1.7664381266,0.2203081250,-0.6439874768,1.3943929672,-0.2212222815,0.5396524668,-0.9326766729,-1.1483886242,-0.5559464097,-2.1605238914,-1.5237659216,-0.8512112498,1.4548479319,1.4710714817,-0.0255904552,-0.0392868333,0.8066657782,0.2605836093,-0.0717089623,-0.4921136498,-0.7088971138,1.3106839657,-0.7427183390,0.5950650573,0.4102731943,-0.7450366020,1.2448486090,0.8819051385,1.2929413319,0.7543784380,1.5012474060,0.5012357831,1.6025744677,-0.1244885102,-0.7437287569,0.7625656128,1.5684475899,-0.8715553284,1.5655452013,-0.4011086524,0.3054696023,1.0253591537,-0.6708995700,0.3336648345,-0.7808531523,0.2032589763,0.4629889429,0.8305130005,1.3792150021,1.0666061640,-0.5386751294,1.8384336233,1.6489225626,-0.5408121943,1.1491128206,0.3872258961,0.7378184795,0.5292190313,0.1566731632,-0.8639983535,-0.4543217719,-1.1172450781,-0.1373617202,0.0493386127,-0.6242133379,-0.0747201517,-0.0418696329,0.1286173165,-0.2732076347,-1.2780323029,0.0635503307,-0.4641813934,-0.2046342939,-1.7704610825,-0.6292266846,-1.5954760313,-0.6125565171,1.3553122282,-0.6214655042,0.3244607449,-0.1392270476,-1.7893319130,-1.6337937117,1.2505484819,-0.3817860782,1.0203552246,-1.7707964182,1.5794938803,0.9360316396,0.6239099503,-0.5742901564,0.3519190848,1.6146284342,0.4477892816,-0.3142979741,0.4781792164,0.6606804729,-0.0335218385,-0.5427355766,-2.5328576565,2.1415202618,-1.8951349258,-0.2973324358,1.1050219536,1.2794615030,-1.4703121185,-0.4997144043,-1.1340399981,0.5188848376,1.5461864471,0.6832302809,-1.0181527138,-1.8280473948,0.0570660420,0.2686289847,-1.2347159386,0.6066080928,0.8530008197,0.3568423390,-0.3545398414,0.3357949257,-1.1221921444,0.1614009887,-1.2751108408,0.1513240039,0.6052228808,-0.0645731688,0.3507217765,-0.3744005859,-2.7758972645,-1.1287082434,-0.0116145117,1.3980032206,-0.0176374651,-0.1791180372,-0.4948559999,-0.4348399043,0.9603962302,-0.9801307917,-0.6622002721,-0.0265429523,1.6936491728,-0.0890061557,-0.1063767895,-0.6456767321,-0.6210477948,-2.4629504681,0.5300250649,-1.1274645329,0.1202629134,0.7539504766,-1.6694158316,0.8432223797,-0.6346231103,0.5511178374,-1.6847029924,-0.3897745609,-0.1799215525,-0.3502997756,0.7974809408,0.7187066674,0.5191764832,-0.1159625053,0.9157214761,-2.1128296852,0.5010185242,-0.4072211087,-1.2765816450,1.9876989126,1.3443856239,0.2349775285,0.6620453000,1.3393974304,-1.2028733492,-1.1018798351,-0.2779258788,-0.5651109219,0.4557473958,0.4294156134,0.3529987633,0.7264522910,1.4002771378,-1.0009080172,0.0965322256,0.9219058752,0.3410715461,-0.7426944971,-1.0091520548,1.4181830883,1.5620485544,0.4852831960,0.4569746554,-1.5205644369,0.8606964350,-0.0159703139,-0.1952470392,-1.7966439724,1.2196379900,-0.2568109930,0.1493610740,0.1471642554,-0.1334927678,-0.0951507837,0.8027544022,1.8461946249,0.5527061224,1.3992179632,-0.4277961552,1.2142347097,0.0890079811,1.1724892855,0.8421200514],[-2.0531296730,-0.3815230131,-1.2456768751,-0.2631421089,0.3498506546,3.3013672829,-0.3395608068,0.9844453931,0.7795133591,-0.3801021874,0.4742499292,-0.5515266657,0.3347168565,0.4604980946,0.8324033022,-0.0461973809,-0.1935420334,-1.0834821463,-0.5094074011,1.4313070774,1.1016554832,-0.2138853818,0.4686948061,1.3630285263,-0.5100106001,-0.1556339264,0.3429749906,-0.3238503933,-1.0426410437,0.1445600837,0.1049434692,-0.2222164124,0.4424103796,-0.1254242659,0.5358566642,1.5508053303,0.6763385534,0.6407976747,0.1865347624,-1.2624466419,-0.4919446707,1.0839577913,0.8675064445,0.2160370052,1.4639556408,1.1394736767,-0.7107564807,0.0310300067,0.4451051354,1.2646448612,0.6961842179,0.3952502310,0.8698752522,-0.8570802212,0.2444748431,-0.3380377889,-0.3063204885,-0.6506505609,1.0126060247,-1.2486658096,-0.4531163275,1.3865495920,0.8400343060,0.8887103796,0.1331875622,-0.4949502647,0.2909305990,1.3711076975,-4.4466323853,-0.3461068869,-0.4214496017,-0.3191107810,1.4321105480,0.5082851648,-0.4887852967,-0.2842614055,-0.4167383611,-0.5153347254,0.1215999797,-0.9909867048,0.2912752628,-0.0309490580,0.0686672628,0.3116374612,-0.9446466565,-1.4318141937,-2.3070585728,0.1934504807,0.0642832592,1.6173255444,-1.4001069069,0.9930692315,0.4986160100,-2.1691753864,0.3272755146,-0.5290307999,0.3029612601,0.7106252313,-0.5628217459,0.3532442451,-1.2171857357,1.0224491358,0.0653357655,1.4942281246,-0.9784055352,0.9602708220,-0.2665057480,0.4682205617,-0.6420611143,-0.7896304131,-0.2709776163,0.8798081875,-0.2442070842,0.1087937206,-0.3002286255,-0.5277259946,0.4140106440,-0.2964256108,1.4545035362,0.4128693938,-0.8971816301,1.0674988031,0.1346273124,-1.0002290010,-1.5374671221,-3.4322154522,-2.1081695557,1.6781947613,0.0430080853,-0.2318437546,0.4126449823,1.4264055490,-0.5483173728,1.1825624704,0.0128414826,0.3367869854,-1.4143692255,0.1019542515,-1.7843068838,0.2959744334,0.6726202965,-0.4266151190,0.1067341939,0.2738974392,-1.5534147024,-0.8846866488,0.4381192923,0.8158077002,-0.0652576610,-1.3206679821,0.6635075212,2.6380484104,-0.6187989116,-1.0136746168,1.8584141731,0.6533104777,-0.4259442091,-0.1774014086,1.4402084351,-0.1898958385,-1.1205271482,-0.5485401154,0.1698873043,0.0687155649,0.0311105829,0.3088907599,0.3787349463,0.3809392750,0.0914585218,-1.4571095705,0.9450003505,-0.7444524765,-0.7551084161,0.9248359203,-0.2947985828,0.9806703329,0.6765164137,-0.4960588217,0.8475109935,0.7173162103,0.7837095261,0.4317607582,1.0102860928,2.1032080650,0.1576048732,0.0173003376,0.7079991102,0.6061809063,-0.6047534943,-0.1859838217,-0.9648866057,0.3602762520,1.5325679779,0.0651429519,1.0650039911,2.5460152626,-0.6730990410,0.2153566033,0.9968594313,-0.4184798002,0.3913473785,0.2801388502,-0.1887837499,1.6170681715,0.3825345039,-1.9914906025,0.4941851795,-0.1115392447,0.0824190080,0.2398149073,0.7439398170,0.2233340293,-0.7043433785,2.8407576084,0.3171818852,-0.0780908912,1.0117206573,-1.3970212936,0.2704565227,1.1053961515,-0.8552007079,0.1994792670,-0.7967630029,-1.7464600801],[-1.4587714672,-0.9694950581,0.4347965121,0.7123994231,2.5555636883,0.2164577097,0.4949950576,0.8548553586,0.5338478088,1.8773450851,1.8840982914,-0.6057417393,-0.2921206653,0.3764616251,-0.8626120687,0.6312596798,0.1972151399,0.8950186372,2.1432530880,-0.5009615421,-0.0870208442,0.2992241681,0.1273721606,-0.5634948611,0.8877972960,-2.2128298283,0.4280473888,-0.2503516078,-0.0595523603,-0.0990573093,-1.3535153866,-1.0699493885,0.9600315690,0.3506530225,-0.0972179323,0.6288717389,-1.3509629965,0.1111599132,0.2762899995,0.8108143210,-0.1749507338,0.6180928946,0.0537807047,0.4083303213,0.7938087583,1.3735338449,0.1129539311,0.7087584138,0.2543578446,-0.7406477928,1.1616770029,-0.4258893132,0.5054414868,-0.4079273939,-0.8973106742,1.5773397684,0.0967438072,2.5411305428,1.1561323404,-0.3894842565,-1.9563227892,-0.3720968068,-0.4485727549,1.4407653809,2.2971429825,-0.1074938774,-0.1706637144,0.2393594086,-1.2897298336,-0.2838257849,1.4366940260,0.5021362901,-1.3120448589,1.3362712860,-0.9349203110,-0.1795876771,0.2796979547,0.3275337815,-1.5682973862,-0.1581210196,-0.4314252436,0.5257216692,0.5361158252,0.9633139372,0.0385950543,-0.2250328809,0.0394976810,-0.6151430607,-0.4402093291,-1.7778775692,-1.0987372398,-1.1117931604,-1.4587020874,-0.5599415898,-0.8956044912,0.5017939210,-0.3279520869,0.1389401406,0.3314218521,0.9030095935,-1.1500355005,-1.9140057564,-0.4541152418,1.1774168015,0.3544850051,1.1513677835,-1.0737146139,-0.2920063734,-1.4028655291,1.5057044029,0.3650828898,-0.8197543621,-0.6998680234,0.3804240525,0.4712976813,0.5005731583,0.2035487741,0.9716621041,0.5432396531,1.3573894501,-1.0096626282,1.6208111048,-0.5277240276,0.8372216225,1.8990112543,0.0555527322,-0.7414680123,-1.0119155645,-0.1866538525,0.8219956756,0.2626342177,-0.0517387129,1.0967673063,0.4631964564,-2.2763576508,-0.9005764127,0.4502929151,1.2675552368,1.4029026031,0.0410192646,0.4613499939,0.8826645017,-1.8186287880,-0.7178622484,-0.1207559109,1.4767705202,-0.0755815879,0.7688866854,0.4810449481,0.6946743131,-1.5904605389,-0.7692117095,-1.4830653667,-2.5812559128,-0.9567979574,-1.4820371866,-0.6474533081,-0.4920964837,0.6667062640,-0.0434637107,-0.4990424216,1.0992600918,0.7525292039,-1.6032221317,0.5581198335,1.3076291084,-1.6271916628,-0.9884853959,-0.3797832131,-1.3973912001,-0.3615023196,-1.9054090977,-1.2950693369,-0.2564415634,-0.4667300582,-1.2958893776,-0.5529566407,0.0508099534,-0.5288753510,-0.7055816650,1.2113600969,0.3943111897,0.2387282550,-1.2374818325,0.2701815963,-0.3978709877,-1.1572402716,3.0878937244,0.8014529943,1.8457676172,-0.0689089596,0.9194198847,2.2617175579,0.6498615742,-0.7515003085,-1.0202875137,-0.3537638783,0.3832994401,3.3604583740,1.0071740150,-1.1935818195,0.2676483095,0.3094398081,1.2086896896,-1.2866652012,0.7168079019,0.2362437248,-0.5765281320,0.7539612055,1.2162506580,0.2970718741,-1.2590700388,-1.1305162907,0.6088635325,0.4285758138,0.8790427446,1.0297600031,-0.0150263263,0.5672517419,1.9943733215,-0.2904198468,2.2879085541,-0.6683076024,-0.9014266133],[1.1198561192,-0.3908662796,-0.7119098902,-0.1672258079,-1.8771710396,1.4835146666,1.5566693544,-0.5456945300,-0.3992888927,-1.8576182127,-0.0591329224,1.1905106306,-0.7533163428,1.3647427559,1.4910933971,-0.8707145452,-1.2028700113,0.1947313249,1.4388641119,-0.7994881272,-0.1219984218,1.3156124353,0.4465557039,-2.8038535118,-1.6329493523,0.4930388033,1.1889687777,0.9552270770,-0.8060735464,-1.2271238565,1.5816148520,1.0718077421,0.0845048428,1.0317893028,0.3432446420,0.4551599324,0.2916109264,-1.2060781717,-1.1404657364,0.1325116307,-1.1530866623,1.4436823130,1.3028328419,0.4774630666,-2.2254891396,-0.4252972901,-0.3897869885,0.3375076354,0.8366788030,-1.5183181763,-0.3942394257,0.3418150246,-0.9899711609,1.7337936163,-0.6356321573,-1.4572966099,1.1070363522,-0.3804219961,1.2175976038,0.1318281144,0.3227569163,-0.3193053603,1.1206564903,-0.9782357216,2.5470190048,-0.0272714552,1.3424437046,1.3656324148,0.1930369586,0.4632759094,-1.6178817749,0.5364732146,-0.5179991126,0.8289573789,-2.2828421593,-0.7208089232,0.5070265532,0.0186515842,0.0308151282,-0.5973554254,-0.3513200283,-0.2779895365,3.2370834351,0.0105593828,-1.0757038593,0.9850459695,-0.0400293693,-0.2541744411,0.2447330952,-0.1110943928,1.3795638084,-0.2299806774,1.0006225109,0.5272231102,0.8905154467,2.0872952938,0.6532689929,-0.3763174713,0.6781877279,0.3656844199,0.9306739569,0.2416982949,-0.3654942513,1.8575373888,0.1327817738,1.4138320684,-0.7374867797,-0.6212695837,-1.1035248041,-1.7859196663,-0.3236762583,0.2797504067,-1.7126622200,0.3749800622,-0.3638271987,-0.2331794649,0.4900467694,-0.1618707776,-1.4507075548,0.4063182771,-0.1040311977,0.7966028452,-0.3308908939,2.0540111065,-0.8186672330,0.0951378867,0.3161187470,0.3237067461,-0.2970733643,0.0833072290,1.5167295933,0.5328765512,0.3028484285,-1.4206554890,-0.4329741299,1.3647885323,-0.0198237337,1.1204465628,-2.4348003864,0.6366173625,-0.3564464450,0.6139984727,0.0905427411,-0.0487639718,-0.7035763860,0.3491133153,-1.6848409176,0.2357283682,1.1717361212,-0.8481423855,-0.2195025980,0.3186892569,0.2741073966,0.3729113638,0.2715400457,-0.4458433986,-0.3774275780,0.6609749198,-1.0435019732,-1.1683075428,0.2231950015,-0.1027064696,0.5333796144,-0.4214748740,-0.8696775436,-0.5515634418,-0.0729684234,-0.3588608801,-0.7902694345,0.2904454768,1.5837885141,-1.0763326883,0.0376655497,-1.5118037462,0.0069348137,-0.3564294577,0.6037105322,0.0964799896,1.5362436771,0.2995141745,1.2214409113,0.7883645296,-0.0879966095,0.6117028594,-0.3020986915,-1.8245121241,-0.2724688649,-0.1164358258,-0.5263771415,0.2055741847,-1.6351301670,-0.1721231043,0.4942176640,-0.6478079557,-0.7820081115,1.2534043789,-0.9469304085,2.0706932545,0.6533148289,0.5774577856,-1.9339703321,0.7606521249,1.7944775820,-1.0832855701,0.7078484893,-1.5409542322,-1.5427272320,-0.6692368388,1.0229578018,1.4193599224,-0.9133544564,0.4840072691,-0.9753501415,0.2186499238,0.1535844058,-1.3288750648,0.3165779114,-0.8043466806,-0.7737863064,-0.0225233603,-2.1306989193,-0.0007973393,-0.1462996751,0.3758448362],[-0.5482758284,-0.2904385328,-0.2203367203,-0.1360739022,0.9308471084,0.5719411969,-0.8307803869,1.6780029535,-1.1897013187,-1.1363524199,0.5443426371,0.6116832495,0.9746663570,-0.3817377985,1.8900529146,-0.7435458899,2.4997756481,0.2572008967,-0.1710775048,1.5151948929,0.4258111119,0.7934648991,0.6231244206,-0.4840977490,-0.9201597571,-0.4832959473,1.1362259388,-0.7039580345,0.8110657930,0.6199864149,-1.2173436880,0.7433096766,-1.2307972908,0.8163778782,1.2898740768,-0.9440120459,0.7128977180,0.3834163845,-0.2448080033,-0.4208971560,1.0776454210,-0.9177082777,-1.3809242249,-0.0549273714,0.4324325323,0.8900623322,-0.1035629958,0.6625125408,-1.1518876553,0.2836413383,1.2821938992,-0.2739796638,0.4010577202,-0.3207268715,1.7261518240,0.5400056243,0.1708711684,0.6841276288,0.5838326216,-1.1475074291,0.9617252946,-0.5272442102,-0.3061755896,-0.2802444696,-1.2695152760,-1.6422095299,0.3058715761,-0.0128703928,-2.6293754578,-0.3578370512,-0.4003086686,0.9074838758,0.9345986843,0.9679060578,0.3018855453,0.0361836366,0.0973763913,-1.6076494455,-1.4042066336,-0.7118547559,0.5561019182,-0.0364919156,-1.5172846317,2.4007532597,-0.1681730747,-0.9137525558,-1.4355233908,0.7524472475,2.1874260902,0.5050536990,-0.4081895053,-0.2869866192,-0.7814391255,-0.9182626009,2.3791458607,-1.1824381351,0.0357667170,-1.0054905415,0.2820036411,-0.2671211064,-0.1642470062,1.4117362499,-0.2460403889,-0.7910364866,0.9709326029,1.2122645378,0.3736355901,0.9546735287,-0.5189890862,0.8345152140,0.1865234822,0.9267693162,-0.7936452031,-0.1931727082,-1.2181763649,-1.1549400091,-0.3107994795,0.3828459978,-2.1461999416,0.6490299702,2.1667606831,0.5197604299,-1.0844227076,-0.5551661849,0.5941517949,0.8355113268,0.0238041356,-0.1456642449,-1.7534958124,1.0213720798,-2.0443327427,-0.8794829249,-0.8769536018,-0.2262274176,2.2516703606,1.2893469334,-1.1498097181,-1.9233324528,0.4795956910,-1.0476869345,-0.1603702307,-1.3869994879,-0.4466053545,-0.1670096517,-1.2587105036,-0.6671476960,-0.1938714534,1.3857107162,0.3265101612,0.3107939661,2.0093312263,0.8311404586,-0.3633439541,0.9202536345,1.6982054710,1.0785430670,-1.1998401880,1.0437088013,0.9129307866,-0.1036761850,0.9082722664,-0.1097550467,-1.1004810333,-1.0039086342,-0.5625741482,-0.7018459439,1.0801191330,0.2250289023,0.6520404220,-0.2414115965,-1.3095985651,-0.2076982260,0.1262160689,-0.5799643993,0.1192129403,-1.1732382774,0.9104181528,-1.1093168259,1.3726590872,0.5925967097,-0.5359659195,0.7879594564,-0.5177975297,-0.1359538883,0.8724327087,1.2267098427,-0.2726623714,-1.2737469673,0.3153249323,-0.3683904409,-0.3502188623,-0.2695860863,-0.3834236562,-0.7484129071,-1.8965317011,0.4718063474,0.8372075558,-1.8106716871,-2.5100286007,1.0228426456,-0.2571284175,0.4561365843,0.1014745757,-1.9939613342,1.5270358324,2.7336032391,0.5241632462,0.0706301928,1.5364835262,0.4563824534,-0.2476233393,0.9482578039,0.4184074104,0.4652556181,-0.4185591042,-1.7924579382,-0.0816290453,0.1231039241,-2.1013731956,0.3075747490,0.9703420401,-1.1419631243,-0.2391041815,0.1807586849],[1.3525646925,-2.1389710903,-1.8200509548,1.4221153259,-0.8525925279,-0.1797548085,-0.8431107998,-0.9901415706,1.7518792152,-0.6028918028,1.2872824669,-0.2191602588,0.9963245392,-1.0853056908,0.5814145207,-0.2415531278,-0.2949104905,-0.1386898309,0.9096865058,-0.2297956049,-0.5122372508,0.6257262230,-0.5922887325,-0.7369153500,-0.0611394197,-0.4971470535,0.4321023226,0.9778766632,0.1208294109,0.2494894415,0.0673965141,-0.1767411530,0.9871430397,0.2811503112,1.9028134346,-0.2427451164,-0.9117431641,-1.3278282881,0.0232584253,-1.6634082794,0.5697232485,-0.7888104916,0.2057036310,0.3429163396,-0.7559013367,0.9190707803,1.5341836214,1.0631592274,-0.7089481354,1.8540906906,0.6824290156,-0.8776482344,-0.9802616835,2.3940370083,1.6527678967,2.0181610584,0.0669684559,2.7364752293,-1.0129966736,0.2716617286,-0.1089972779,-0.0572592542,-1.0589312315,-0.3265284300,0.3300458789,-0.0004799794,0.8181158900,0.4282137156,-2.5039472580,0.1204811335,0.8078928590,0.6021209955,-0.8651900291,-0.1533203572,-0.2404905558,-0.0607633479,0.5328661203,0.7644009590,-0.8560833335,-1.6823405027,0.6530694366,0.9302550554,0.4051826000,0.2040024847,-0.8227655888,-1.2768743038,-0.2264368683,-1.0160527229,-0.6572948098,0.2939909101,0.6200525165,0.7796336412,0.4549840093,-1.3034356833,-0.5644497275,-0.6971324086,1.0378397703,1.5601544380,-1.4602061510,-1.9396758080,0.5265857577,1.6603387594,-0.3638090491,-0.0310451910,-0.3636223972,1.2418016195,-1.6686720848,0.6132101417,0.0434691086,0.1165482178,-0.5786048174,-1.2646787167,-0.1174873859,-0.4851864874,1.0230669975,0.1483554542,-0.0294206236,0.8758327961,0.1348077953,0.0840644166,0.8931159973,-0.1384363919,-0.2515294552,0.5490271449,1.6555695534,-1.3272736073,-0.1325270534,-2.0754849911,0.6027877927,0.4861270487,2.0449700356,-0.6894755363,0.3789910376,0.3894831240,1.2121812105,0.1136562079,-1.0378799438,1.5067687035,-0.4767748713,1.2916351557,1.2961734533,1.1144324541,0.1773619503,-0.2875421047,1.4568520784,0.8501911163,-0.3830829263,-0.6213039756,2.2090291977,1.0792033672,1.1268278360,0.1464464664,-1.0442385674,0.3112875223,0.9527289867,-0.1369006485,0.1579798460,0.9468470812,1.1619044542,0.1117127314,-1.2547981739,0.0041935598,-1.8870812654,1.8109247684,1.2467620373,-1.1706187725,-0.8198552132,-1.0388509035,-0.3795161247,0.2471856773,0.0978375450,-0.3687669337,1.4281040430,-1.2668262720,-0.7314748168,-0.2361954153,-0.0640455484,1.5850062370,0.5684098601,-1.0112850666,-1.1607600451,1.6386686563,0.0010163832,0.6599608064,-0.5040879846,1.7607916594,-0.2987151444,-0.0699679181,-0.2390695810,0.0480869710,-0.0561540872,-1.4012265205,-0.4074691832,1.0456372499,-2.2115867138,0.1003632694,-1.1718195677,1.9943586588,-0.7404477596,-1.8472335339,-0.3663715124,1.6781787872,-0.3644346595,0.2316042483,-0.6349893808,-1.0591756105,-0.3665927351,-1.3370912075,1.3054594994,0.0549971089,-1.1565880775,-0.1878262907,-1.7153264284,-0.2808305323,0.3602373302,-0.7212537527,0.5203909278,0.6193087101,0.0597162433,1.3419901133,1.7610182762,-0.4293690622,0.7105035186,-0.2299219370],[-1.1954588890,1.5433597565,0.2379141748,0.7672137022,-2.1098141670,0.6415581703,-1.4304503202,-0.5032683015,-0.1808628291,-0.9698618650,0.9286208153,1.2245706320,-0.8532410860,0.3880198300,-1.2975735664,-0.7356634736,0.1969772279,1.2833096981,-0.2106740475,-0.1530257016,-0.1989236623,0.6560999155,0.1675494611,0.4150153399,-0.2940008044,-0.4093782008,-0.8216152191,-0.4769310653,0.9040605426,1.2138260603,0.8652194738,-1.1292105913,1.0701618195,-0.1425126493,-0.8724672794,-1.6130620241,0.0751923323,0.4281897247,0.1331243962,-2.1410734653,-0.9063301682,-0.3643827438,-1.4411940575,-0.8983129859,0.4188375175,-0.6246270537,1.0137139559,-0.0459714942,-1.2153115273,-0.8810849786,0.9074236155,-2.5559902191,2.1576261520,-0.4590293169,-0.9930305481,1.8669599295,-0.4173523188,0.9735121727,-0.5194160342,1.8311252594,-0.2143068463,-0.3267007768,-0.7139260173,-2.0381011963,-0.5063050389,0.6719281077,1.2255204916,-1.0252617598,0.2913454175,-0.5917704105,0.7488190532,1.6724610329,0.8395947814,1.7179481983,-1.6458688974,-1.7235046625,1.6781872511,0.6279851794,-1.2300579548,0.2063685060,1.7433674335,0.2346530110,0.7607139349,-0.3385611773,-0.2102975249,0.0059588552,1.4260768890,0.0142158372,-0.4406615496,0.2704971731,-1.0747419596,-1.3426505327,-0.1716880351,-1.2444826365,-0.8974450827,-0.0273043569,1.3485187292,-0.2789664567,1.0251704454,0.1180881709,-0.3778805137,-0.2141260207,0.4078404903,0.3173700273,0.3696385920,-3.5518553257,-0.0249717198,1.0215418339,0.6116573811,-0.8735608459,1.9158036709,1.0758155584,1.0407550335,-0.1262805015,-0.1615180820,-1.1025336981,0.7169651389,1.9114420414,-0.7175180912,-0.2838610411,0.1815304458,0.0509980470,0.1601001769,1.3206021786,-2.0074746609,1.1405119896,1.0774667263,-0.3010169268,-0.0158685166,0.2949124277,-0.9155491590,0.0511295386,-0.9183694124,0.4059500396,-0.9847790599,-0.3063639998,0.2768558264,0.6799152493,0.1374379843,-1.7768172026,1.0458937883,-1.5003950596,0.2859581411,0.4456777871,2.7115471363,0.0316340886,0.0237582047,1.3859419823,-0.2100570202,0.5256478190,-0.0925751105,-1.4566472769,1.6074087620,-0.4122121632,-0.8000978231,-0.4829004407,0.7124879360,2.1299157143,-0.0259689819,-0.2674769461,-0.5260988474,-0.1544209272,0.8619582653,0.1432756037,1.1954751015,-0.4071525037,-1.1334422827,-0.1827943474,-0.1172728464,0.1429066360,-0.6256256700,-1.1532642841,-0.7034975290,0.5680677891,0.2888707519,0.1756383628,0.5803961158,-0.5129320025,0.0149797229,0.9116480350,-0.1075092405,2.3236284256,-1.1821206808,1.3360717297,0.5781747103,0.2877194285,0.4588770568,-0.1316705644,0.0050465642,0.5311812162,-0.7386845350,1.6503698826,0.8862819672,1.2689412832,1.4950553179,0.0704524592,0.9109133482,0.0946484506,-1.0497871637,0.7867195606,0.2191394717,0.8150247335,0.0630580559,0.1004397869,1.4962004423,0.3818966448,-0.3216921687,1.6232447624,0.4632739425,0.3479743302,0.2854761779,0.0905378759,0.0078849401,1.1418205500,-0.2064098269,0.1464061737,-0.1620241702,-1.1185684204,0.0551032908,0.0955945477,0.1914982051,-0.2445030063,1.0773905516,0.7771847248],[-2.0026953220,-1.3879367113,1.8168145418,-1.3441421986,-0.1909300387,-0.2537579536,-0.4333573282,-0.5043718219,-1.1934229136,0.8883563876,1.3796744347,-0.8058362007,0.6302874088,-1.4562950134,-0.2413089424,-1.1239010096,-0.8434281945,-0.6111188531,-0.3214215040,1.7906646729,-0.2953863144,-1.0996087790,-0.0280012544,1.5195906162,0.4385769963,-0.0715610012,-1.2215670347,-1.1551032066,-0.5126031041,0.5079829693,-1.5987869501,0.6407077312,1.2264078856,-0.5521286726,0.5383154750,0.9565272927,-1.1502499580,0.4597047567,0.3260488808,-2.5554530621,0.0539155044,0.9961486459,-2.3894460201,-0.3041664660,-0.2683731318,-0.8905698657,-0.0944725424,0.5605733991,0.7430984974,-0.6170213819,1.4618774652,-0.4896700978,-1.6775000095,0.9276668429,-0.3350292444,-0.2878435254,0.2499760538,-0.8336799145,1.0695707798,-0.8759759665,0.8804914355,-0.4413791895,-1.4477114677,0.3207118511,-0.6660116315,-0.0039797751,0.2596048117,-1.3988531828,0.0217267051,-0.3165823221,1.4104597569,-0.8589660525,0.3823044598,0.5117325783,0.5799040198,-0.7965207100,-1.0794136524,-0.5460695624,-1.4399230480,0.9373409748,-0.6249065399,-0.7326609492,1.7352846861,-0.7039022446,-0.8619210720,0.2775456607,1.8297880888,-1.3504639864,0.1678970009,1.6588230133,-1.4726707935,0.2412999719,-0.0883939713,0.8553674817,-1.1602106094,-0.0671377331,0.2164906412,-1.9931809902,0.0053256475,-0.2441381961,1.7421977520,0.2616285682,0.6127197742,-0.6840193272,-0.3926095366,0.4958389103,-0.4157512188,-1.8020431995,0.5348650813,-0.4459702671,-1.0132100582,1.6183987856,1.8912308216,1.4364794493,0.4951191545,0.3030346632,0.6038584709,0.6475570798,-0.2035506368,0.4782030582,0.8421400785,1.6720960140,-0.8565639257,-0.4995645881,-0.0788188726,2.6190998554,0.0172765702,0.8030301332,1.9209679365,0.6042689681,-1.6229704618,-1.5398095846,1.2159051895,-0.0491375811,0.6701576114,-2.0786049366,0.2396868467,-0.1663447022,0.4873751402,1.6635489464,-0.7188113332,-2.3959128857,1.7327207327,-0.5349614620,-0.0728966668,-0.2948603034,-1.4774221182,0.4834442735,0.3681408465,0.9823635817,0.5640633106,-0.0505462214,0.3036163151,0.9760785103,-0.2052964270,0.0603897423,-0.0227735676,0.4560826123,-0.7479364872,0.4386453032,1.5767741203,1.2626835108,0.5652157664,1.5275896788,0.0146089382,-0.8305668831,-0.0492977872,-0.4252792895,1.2362649441,0.0874866694,0.4226440191,0.0723808184,-0.0925615653,-0.5521541238,-0.5364710689,-0.5070555806,0.1121965498,-1.1298216581,-0.7371627092,0.8932670355,-0.6637581587,0.4557986557,-0.6397003531,-0.3043342829,-1.1985019445,-0.5069193840,0.5818718076,0.9713091850,-0.5977748036,0.0146248564,-2.4917397499,-2.0943746567,1.2473956347,-1.9712914228,-1.3773270845,0.1807457060,1.2754870653,-0.6485044360,2.5466191769,-1.3412190676,0.1961534172,1.7364504337,0.1384211183,-0.2164301127,1.4340209961,-1.1674877405,2.2805016041,0.7271856070,-1.6405873299,-1.7613757849,1.2478111982,-0.7066447139,0.4562221169,1.0106225014,0.6289210916,-0.2545123994,0.8075569868,0.9933471084,-0.7091018558,2.4793047905,-0.1058447361,1.4626127481,-1.1512520313,2.2819995880],[0.1980908364,-2.0391972065,0.4623443484,-0.4528153837,-0.3034771681,0.6253050566,-0.9960688353,0.1848763525,0.1992302835,1.3969924450,-2.1879806519,-3.5258898735,-0.3500127494,0.1608139724,0.5911024213,0.6979445219,-0.4094097912,0.1062802821,1.5368102789,-1.2191261053,-0.1408000141,-0.3725816011,-1.0421752930,-0.1061916351,-1.8223229647,0.6922587156,0.0466201380,2.0245492458,1.8404192924,-0.5316298604,1.8038771152,-0.9891858101,-0.7872034311,-0.9601790905,1.5571957827,-0.0051668053,1.6011846066,-0.1465292424,1.2160656452,0.4363673925,0.0321315080,0.7290571928,-0.5699969530,-0.1844813824,-0.0893535689,0.4522967935,1.4669539928,-0.2522238791,0.4661012590,1.4432981014,-2.7434496880,0.3867551684,-2.0132300854,1.6866211891,0.7507246137,1.0172561407,-0.2487859428,0.7585685849,0.5755932927,1.2352458239,0.8978250623,1.2966716290,-0.6571815610,1.4671807289,0.7853207588,0.3756480515,0.6194713712,0.3272837400,-0.3676315248,-0.0267670508,-0.2408681363,-0.7468521595,0.6240339279,2.1852478981,1.1214398146,2.0001266003,-1.5732886791,-0.0183822736,-0.8087029457,0.9497622252,-1.0744439363,0.9904448986,1.4737164974,0.1133142486,0.3727804124,-0.5018841624,0.6833029389,0.3201475739,0.2428314686,-0.8037180305,0.2282627523,-0.6574258208,1.8786723614,-0.7279502749,0.5236622691,0.3721244335,0.7680162191,0.7290078998,-1.6448494196,-1.3555030823,-0.9520250559,0.7258281708,-0.2495432496,-0.3343724608,-0.1035859659,-1.5604089499,-0.0533355586,1.2266353369,-1.9780186415,-1.2835073471,1.0977239609,-0.0809845626,-0.1254453063,1.2240229845,0.8162678480,1.1728838682,-0.4857140183,2.0652470589,-1.0620183945,0.4987939894,0.1875279099,-0.1605241150,0.2976987064,-1.7253868580,-1.2821964025,-0.5454786420,-0.6711188555,0.4872035384,1.3012408018,-0.0574941002,1.0852347612,0.2164134383,-0.8638213873,0.0029188166,-0.3997426331,0.8708361983,1.0494933128,1.2122185230,1.0808790922,0.0889089853,0.3659733832,0.6266746521,-0.7211550474,-0.0947615877,2.5941832066,-0.6472509503,-0.0892662406,-1.1688027382,-0.7755636573,-0.1888562292,0.4613487422,-0.1133504286,-1.0579370260,-0.3661361039,-0.9344208837,1.4105546474,0.3662054837,-0.1800113618,-0.8287143111,-0.4588594139,-0.5348659754,0.1927637607,-0.1688094139,-1.0985659361,1.7208666801,-0.9949175119,0.0071640117,0.2181097269,-0.5293247700,-1.2477115393,-0.4222748876,-0.8102868199,1.5723097324,0.7872313261,-1.7233170271,0.5807046890,0.1063605845,-0.5338957906,-2.1653342247,0.4575393498,-0.5575693846,-0.8172625303,-0.3406208754,0.0603416599,-1.5120033026,1.1769851446,-0.2431340963,-0.8860734105,0.2002744973,0.0624899231,-2.2787888050,-1.7242875099,-0.3660179377,-1.5788301229,-0.9279945493,-1.8265781403,-0.8975494504,0.2838334143,1.9209350348,-0.1576115489,-0.5317687988,-1.6259893179,0.0153308073,-0.9110952616,-0.1653829366,-0.4306715727,-0.6055718064,-0.8026126623,1.3090204000,-1.2156563997,-0.0519316867,-0.6713653803,-0.7353394628,1.4131215811,1.1170443296,-1.1881471872,0.2349975854,-0.2238769233,0.2461022288,0.1120637581,0.2336885929,-0.5243279338,-2.1437044144,-0.2431624234],[-1.2464995384,0.4675020576,-0.7527608275,0.5624199510,1.3365873098,0.7289454341,1.7958503962,0.2928060591,0.7216563225,0.9394800067,-0.6092904806,0.5123190880,1.6707273722,1.7536890507,1.2125877142,0.8943855762,-0.8880759478,-1.3625150919,-0.2121257335,-0.6096383333,-0.1692983806,0.1635778695,-0.1416323781,0.3522885740,-1.0253274441,0.0980968177,-0.1966700852,0.7525386214,0.0875271410,-0.4656350613,-0.4846600294,1.0480329990,-2.1517102718,0.4131481051,0.2350990921,0.8666293025,-0.0862590522,-1.3615747690,0.3523576856,0.6853317618,-0.3136028051,0.1880191416,0.4099854231,0.5680887699,0.5689167380,1.6204892397,-0.1595118791,-0.1066793799,-1.1540112495,0.6110669971,-0.7147044539,-1.7258614302,0.7982810736,-0.7267000079,0.1675357521,-0.8346014619,1.2481591702,-0.9113322496,-1.5161044598,-0.1560480595,-0.3392308652,-0.1593530327,-0.1827998012,-0.5442073941,1.3976205587,1.0422940254,-0.2430956662,2.9464750290,-1.0710835457,-1.2913874388,0.4633493125,1.0320210457,-0.1943645924,0.4055425823,0.8175639510,-0.6497596502,1.2451591492,0.3004733324,1.1675512791,-0.6333433390,-0.9965176582,1.0963352919,1.3911732435,-0.2135262489,0.0029021860,0.0531421266,0.3033952117,0.5173051953,-0.0102700051,-0.0946862027,0.2938453853,1.3135150671,0.4863734245,-0.8989269137,-0.7194559574,-0.1590253711,0.4843160808,-1.1027262211,0.0849133283,0.5688287616,0.2679769695,0.4811837673,-0.7134509087,0.2008953393,-0.8335529566,-0.7201511860,0.5242791772,0.0110169752,0.4427189529,0.4736892581,-0.5382049680,1.6526933908,-0.2942489386,-0.4903966188,1.2739274502,1.1805559397,-0.1975703090,0.4837063253,-2.3744602203,0.2897094488,0.6084982157,-1.6204299927,1.3142671585,0.6344429851,-0.4190022051,-0.4888667166,0.6072349548,0.9508922696,-0.6400114894,1.4980759621,-2.1872255802,-0.9253449440,-1.6960446835,1.5540304184,1.1076310873,0.4102774858,0.3571720719,0.0244482420,1.1432708502,1.6378223896,0.1038910747,1.1907352209,-0.7008403540,-0.4064160585,1.1399818659,0.8482441902,-0.8278619051,1.2165918350,-0.1842192709,0.5383604765,0.0136463754,-1.6233984232,-0.3791120946,-0.8511069417,-1.2866246700,-1.8059158325,1.1002595425,-0.6931541562,-0.2090755552,0.0067911642,-1.0231842995,-1.4700142145,1.5429402590,0.5305857658,0.7299745679,0.4535187185,0.1681299061,1.6024149656,-0.9856672287,-1.6728117466,1.1804482937,1.7814029455,0.0313383043,0.1245605722,1.5340962410,0.1487563401,-1.3175044060,0.5722743273,-0.8558244705,1.0446487665,-0.9774014950,1.4014579058,-1.2876857519,1.9653085470,-0.2574622631,0.8741406202,0.1923532635,0.4771126211,0.5219726563,1.0568250418,0.1275015920,0.3534825146,-0.4791073799,0.0526588932,0.8323126435,0.7390776277,1.3197847605,0.3206358850,-0.8508493304,0.1079496369,0.0510789007,0.8233492374,-0.2772503793,-1.0964258909,-1.0618132353,0.0157843046,-0.9365561008,-0.7326688170,0.1077768654,-0.2073359340,-2.3172490597,1.3560752869,1.5103831291,-0.1958469301,1.0512095690,-0.0394407101,0.3858150244,-0.0032381886,0.9143221378,0.9421868324,1.6064001322,0.4654370844,-0.2540049553,1.4979096651],[0.6909629703,0.4411643445,-1.0131149292,-0.1733857840,-1.7086960077,-0.3259657919,-2.0024752617,-0.6706923842,0.6668214202,-1.9155961275,-0.3144879341,-1.2456922531,-1.2796248198,-2.2090945244,0.7396605611,0.0296590049,0.9433910847,2.1231267452,-1.0864417553,-0.6030449271,0.3438302875,-0.6608654857,-1.0862615108,1.3887225389,-0.7988653183,1.3075400591,0.9552127123,0.3432019651,-0.2984606922,-0.1288973540,0.7399637103,0.0231899377,0.6450478435,-1.3013180494,-1.3024494648,-1.0995938778,-0.2466881722,-0.3775995970,0.4923250675,0.1694402844,0.9104561806,0.6665503383,-0.5168366432,-0.3762043715,-0.9129214287,-0.2788790166,-0.4600901902,-0.8824477792,0.5361903310,0.6113188267,1.3454030752,-0.8275414705,0.4882041514,0.8205421567,0.8798395395,-0.6477968693,2.4074959755,0.5788549781,-0.0437356532,-0.7579018474,0.9680426121,0.5040457845,1.1636078358,1.8771742582,0.6200423837,1.5920174122,-1.8497363329,-0.8436110020,0.3149974346,-0.4593172967,0.8473721743,0.0478686653,0.4404908121,0.3380852044,-1.1299993992,-0.1233110130,-0.9066924453,0.4301193058,-0.3466370702,-1.0565040112,0.2993363738,-2.2224280834,-1.1136168242,-0.5317560434,-1.9586926699,1.2876089811,0.5533798337,0.8821557760,1.1840175390,-0.1996572614,-1.7286516428,-1.8491935730,0.5178694725,1.7988730669,-0.2523056269,-0.4970327318,-1.4278050661,0.3399576545,0.7023395896,-0.7998167872,0.7301787734,0.5362419486,0.8577725291,-0.9817480445,-0.9755565524,0.3299261034,1.4980843067,0.7955431342,0.8185396194,0.0570088066,-0.1452435553,-0.4598515928,-0.4803588092,-0.6821164489,0.4632911086,-0.0423791558,1.8142098188,1.3901891708,-1.0697890520,-0.0289973561,-0.6405655146,0.0753127635,-0.5270310640,0.1859117001,-0.1663290560,0.5048159957,-1.7322213650,1.6413090229,-0.7120727897,-0.4916498661,0.8107194304,0.9926199913,-0.1052645668,-0.0880601779,0.6687352657,-1.4801020622,-0.7137554288,-0.3793790638,-0.1603921801,0.7068908215,1.2170107365,-0.4102213979,0.4785642028,0.3500928879,0.4325503111,-0.6816869974,0.2588504553,-2.3803224564,0.0774483383,-0.5226783156,-0.3486301601,0.1307638288,-0.4308033586,1.0323166847,0.7308865786,-1.0509271622,-0.3705799282,-1.4526411295,0.7582592368,-0.1112433150,0.2123344541,-0.1703404486,0.5684955716,1.2879304886,0.4955765605,0.9762223959,0.5969173312,1.0294094086,-0.4016108215,1.4354219437,-0.4525334239,0.9244177341,-1.8113485575,-1.8544753790,0.6716580987,-0.1637547463,-1.0912935734,1.0384893417,-2.0325262547,0.6750365496,0.1859313697,1.4355543852,0.9272583723,0.3481788635,-1.7764997482,0.6461026073,0.3343574703,-1.1932055950,-0.7114098072,-1.1107187271,0.2314597964,-0.3236814439,-0.2080765665,-1.4484943151,0.4726499617,1.2705886364,0.0462085642,-0.8196042776,-0.4398490787,-0.5813989043,0.0109049398,-0.3579022586,-0.3998372853,1.3548592329,-1.9293205738,-0.6319025159,-0.7778295875,-2.0569281578,-0.7499248981,0.9562718868,0.7526506782,0.3659623861,-2.2221159935,-0.5005191565,0.0293950923,0.0451079011,0.2878231704,1.1077967882,-2.5320174694,-2.6052191257,0.5428759456,-0.3326431513,-0.2977385223,-0.3886250556],[0.3837181628,-3.0844507217,-0.0326572843,1.9840731621,-0.0990928710,-2.0813033581,0.9364513755,-1.5018221140,-0.7127351761,-0.4184252620,-1.6085178852,-0.5712712407,0.3170658648,1.3702354431,0.2474628687,1.6732935905,0.3019446135,-0.5106189251,-1.6296691895,0.0827088729,-0.2414676398,0.0117205493,-0.3977093697,-0.8551368713,0.5991954803,0.4162049294,1.5830416679,-0.1344192475,-0.8080347776,0.8270174265,-0.2177505791,0.7213084102,2.7542774677,-1.9037790298,-1.1765846014,0.9574951530,-2.2370584011,-0.1641188562,-1.5068861246,-0.5019809604,0.2476276606,-1.3522347212,1.5981073380,-0.0016236352,0.0727113858,-1.4785469770,-0.8532192111,1.3951283693,-1.0664182901,0.4442366064,-0.4066230357,0.2653382123,1.0576475859,-0.2093126327,0.0964964181,0.1187895015,-0.1235642135,0.1212674677,-0.6063463688,-1.7976351976,-0.2944546044,0.5414992571,0.5756024122,-0.4685805142,-0.7462616563,-0.3464683294,0.2230277807,0.9854597449,0.0767948404,-0.4874532223,-0.1219522431,0.8327487111,-0.1716076732,-0.3945173025,-1.1791951656,-0.4452678263,1.3629028797,-1.1066451073,0.3573346436,0.6323838830,-0.3071047366,1.6411852837,0.8643095493,-0.2245212346,1.4297059774,1.3671625853,0.2288126200,-2.0948269367,0.9121999145,0.1160216853,-0.0257257298,0.2509836555,-0.1600100994,-0.6034733057,-0.1384988427,-0.0839320645,-0.6945816278,1.9426094294,0.0452554338,0.3266722560,1.2575550079,1.1093206406,0.8573603034,-1.2389947176,-2.0883388519,2.1227445602,1.1516443491,0.2675343156,-0.3369225860,-0.4755612016,-1.1894302368,-0.2873914242,-0.1477799714,-0.1865244955,0.9614298344,0.1122136638,-0.1507401168,-0.7496002913,-0.8663163185,1.8104102612,0.1771599203,-0.7490789890,0.0366497710,2.0983855724,-1.0290652514,-0.5826900601,1.1658000946,1.2506613731,-0.1822900325,0.0264755636,-1.3294763565,0.1857622564,0.1860099584,0.6812687516,-0.0765349269,-0.5551304817,0.2183326185,1.0382562876,-0.7524948716,-1.2267206907,-0.5154917836,-0.9957588315,-0.8302957416,1.1273059845,-0.7931482196,0.0290784836,-1.8754261732,0.6611202955,0.7537636161,1.4504109621,0.4328519702,-0.1413241476,-0.3728679121,0.6295785308,-0.2826263309,0.1017487198,0.1004890427,-0.7599399686,1.1586389542,-0.4124349654,-0.9641899467,0.9970339537,0.6460083127,-0.4832358658,-0.6841650009,-0.0303084180,0.5566793084,1.2560545206,1.2719508410,-0.0263155978,0.0070461296,-0.8024127483,0.1088609323,-1.8889316320,1.7075667381,0.4763677418,0.8923625946,-0.1581884325,0.3295147121,-0.6005237103,-0.7768769860,-0.4446044564,2.0271074772,1.3566521406,0.0894624814,-2.5341632366,2.3792328835,-0.2505889535,-1.3664253950,-0.5644386411,-2.1931164265,1.1769444942,1.0361630917,1.8065854311,1.3853068352,0.8310387731,-0.6718800068,0.0839435682,0.5932793617,-0.3277965784,0.4227188528,-0.9974304438,0.1251470149,-0.4001867771,0.2866265476,-1.7253122330,0.4782764316,0.7806475163,-0.5707991719,0.4938725829,2.1363813877,-0.6386837363,-2.1948287487,1.2262840271,-0.0510553345,-1.0744620562,0.5451381803,-1.9778690338,0.9243052006,-0.2363083810,-0.8462770581,-0.4339647591,1.5218746662,-0.2496304214],[-1.2015702724,0.4348722398,-0.7702450752,-0.3616005480,-0.9668240547,-0.7775117755,1.6584910154,1.2157908678,-0.0582093261,0.4874089956,-0.4351541996,-1.2981131077,0.4196226597,-1.1504046917,-1.3714263439,-1.1537238359,0.2113851160,0.9948299527,-0.0589868426,-0.2105388194,-0.1264684796,-1.0223534107,-0.1095140055,0.3037421107,-0.2303951383,-0.3564314842,-0.2918756604,0.3242609799,1.4333026409,1.8529117107,0.2137945294,1.9478644133,0.6609766483,0.2107582986,-0.0890568718,-1.0639896393,1.5600599051,-1.1038756371,-1.2550096512,0.1083938032,0.5406846404,-0.2521128654,0.4976765513,-1.5165120363,0.7626850009,0.2167731673,0.6663960218,0.4201067984,-0.5433947444,0.3073146641,0.7519953847,-0.5402916670,0.5032405257,-0.7342537045,-0.9444994330,-0.4982891083,1.3228744268,-0.1457812637,0.5063264370,0.5172562003,1.5089118481,1.8802934885,-1.1349375248,0.1486520320,-0.3159368038,-0.9561010003,-0.1483890116,-0.6002311110,0.8183563948,0.9246582985,-0.2708984911,-0.1344445795,-0.3975899816,0.2588158548,-0.8930866718,-1.6686511040,-0.5920096636,-0.0261793677,-1.0039272308,0.5324293375,-0.7660702467,-1.5267822742,-0.0449126177,-0.5578223467,0.4411966503,2.8361527920,-1.0847097635,0.5951077342,0.6513732076,0.5490559936,0.2389404327,-0.6742807031,0.2803015709,-0.5785861611,-0.4543779492,-0.6440399289,-0.9178314805,0.3433896899,-0.0781365111,-0.0110655092,1.0301516056,0.2924049199,0.9450477362,-0.7342799902,-0.4853533804,-0.0152373305,0.3297406137,-0.9591660500,0.6535556316,0.6672778726,0.5473683476,-0.0690325350,1.7266031504,-0.6833693385,-0.0771031976,1.3326834440,0.6728719473,0.8581472635,-0.7994928956,1.7931659222,-0.2488678694,-0.9435475469,-1.3550130129,-0.1657632738,1.8103445768,1.6421945095,-0.5307902694,0.3177648783,-0.8274542093,-1.5274112225,1.0199569464,-0.8939806819,-1.1168608665,1.3539602757,0.2608950734,-1.5025458336,-0.0350464173,1.6370078325,-0.2023550272,-0.9884869456,-0.7876907587,0.3030428588,-0.8543554544,-2.0759766102,0.8074741960,0.0092417505,-0.7486258149,0.9306184053,1.5228734016,1.0625956059,-1.0638818741,0.9000504017,0.9773253202,1.4914863110,-0.9541801214,-0.1188048348,0.0520063490,-0.7230827808,0.4520131350,-0.6191820502,-0.9545143247,0.1884564161,0.3061656952,0.4476604462,0.0418105461,-0.1309568137,0.4180471897,1.2613639832,-0.7609384656,0.3685899377,-0.2012770772,-0.4936135709,1.1598070860,0.3737017810,-0.1421750039,0.2495179474,-0.8559209704,-0.6964498758,-0.2374830842,-0.9256697297,-1.7266376019,0.2695619464,-1.4074134827,0.8068033457,0.6471933722,-0.5924095511,0.1117355973,0.2366000563,-1.4429482222,0.8711776733,1.5175033808,-0.2762643397,-1.2968262434,-1.2299476862,0.9790905714,-0.6197896004,0.2056121975,1.1886742115,0.6697474122,0.1087135598,1.2664176226,0.1910816878,-0.3005948663,0.6051598191,0.1337996721,2.2987043858,-0.2989132106,0.0533511490,1.4336971045,0.3602710962,-1.4897760153,1.9669363499,1.7474601269,0.7100765705,0.2949651182,-0.0514402352,-0.3578273058,0.3582149446,-1.3475846052,0.0255273320,-0.7661463022,0.6330846548,1.4332324266,0.9169499874],[0.8839311600,0.7519271374,-0.1242360175,-0.3691127002,-0.8278617859,0.3434691429,1.4831691980,0.8366777301,1.0739248991,1.0072058439,-0.6175259948,-1.3308913708,1.1121007204,-0.8164995909,0.8835690618,-0.9938970208,0.9998544455,0.4616194069,-1.1370314360,0.9062680602,1.1257185936,-1.9194012880,0.2764996290,0.5737554431,1.4669361115,1.4267357588,1.0081113577,0.7518727183,0.9374237061,0.6022189856,0.9636828899,0.6104877591,1.2044204473,0.9677898288,-0.5647366643,1.0105677843,-0.3122237325,0.3334992528,-1.9033569098,-0.9630427361,0.2652240694,0.5199043751,-1.1540907621,0.7361152768,1.2738237381,0.1880766749,-2.2973453999,0.5291508436,0.0009351905,-0.6024293900,-0.2233156264,1.5331068039,-0.9884901643,1.5078092813,0.7398353219,2.3887550831,-0.3120129704,0.5433498025,0.0288962387,-0.2012146115,1.0563759804,0.0455531701,-0.1074009314,-0.9435821772,0.8474586010,2.1150949001,-0.5216084123,0.6223477125,-0.4835988879,0.0323945619,0.1332448572,-0.6117167473,1.2821873426,0.6829779744,1.7952840328,1.4735364914,-0.0281432271,0.3344296217,0.0498930812,-0.7253366709,-0.1882753372,0.7500153780,0.3084684014,-1.2596098185,0.4998308122,-0.2986431718,0.7846732736,-0.5215204358,0.4127131701,-1.4233584404,0.3591822088,0.8100859523,0.6673883200,-1.4757587910,-2.2996366024,-0.5354201198,-0.6260387301,0.0607966073,-0.9204249382,-0.3795864880,-0.0245653503,0.4642808735,-0.2157164365,0.8455864191,2.2098424435,-0.4032137394,1.2565199137,0.3011314571,-0.4515376091,-2.2521011829,0.2975798845,-0.6609811783,1.2324833870,-0.2599743605,0.6994997263,-0.9482492208,-0.0442115664,-0.1683008969,1.7778540850,1.5128661394,1.1165469885,-0.9578713179,0.5861684084,0.4876947403,-0.0258086044,0.8223834634,1.0231527090,1.5395146608,0.5583790541,-0.5228635669,0.8827819824,0.4835412502,0.4577231705,0.5126315951,-0.7414305806,-1.8140503168,-1.3544577360,0.8708856702,1.0170553923,1.0010755062,-0.6828664541,-0.0983215198,0.2329484820,-0.5799430013,-0.9481555223,1.3907268047,0.6413714886,0.3823638558,0.9870128632,-0.4725233614,-1.5783783197,-0.6470431089,-0.9037789106,0.0999176055,-0.6165252924,-0.5567381978,0.6479242444,0.5634599924,0.4559148848,-0.7018021941,0.5007602572,-0.7826791406,0.3516834974,-0.5544981360,0.9681010842,-0.6531418562,-0.0493310317,-0.3987984359,-0.0768239871,1.2028565407,0.9053220749,-1.5396063328,-1.5210251808,0.0095656775,0.4015446603,0.4952757657,-1.9695576429,-0.9904660583,0.0043475674,-0.3832959235,1.5924043655,0.1618719697,1.1238229275,-0.3456900418,0.7589526772,-0.2558067739,-0.1103068292,0.3348076940,-0.2165825069,-1.7158200741,-2.2370626926,0.7226798534,0.1446095556,-0.3551731110,0.7914757133,-1.0397866964,0.3701458871,1.5328626633,0.1711807400,0.0900413617,-0.1533249617,0.7270753384,0.1712650359,1.2431062460,0.2759212554,-1.4751939774,0.1763630956,3.6361017227,-1.5242985487,-0.0279763695,-0.9603900313,1.1799722910,0.2726994753,-0.2330887020,-0.6826753616,1.0471214056,1.1989165545,-0.4046612084,-0.7465475798,0.8356605172,-1.0643728971,-0.3423130512,1.3552594185,0.0537857786],[-1.5534422398,-0.5955978036,1.1433280706,0.2265868187,-0.2054847330,1.4488072395,-0.9428060055,0.2496462762,-0.5333206058,0.9197281003,0.0167320166,-0.5308976173,-0.3981518447,0.1968156695,1.2565236092,-0.1500016153,1.7518627644,-0.5710981488,-0.7607555389,-0.5640133619,-2.1015295982,0.2339061052,0.3560619652,-0.2133811563,1.6464656591,0.8782684207,0.6951352954,-0.8930280805,-0.5173587203,-1.0700474977,-0.6147041917,-2.5534784794,0.4594208002,-2.9251899719,-0.6844087839,-0.1010654792,-0.9624158740,-1.0140864849,-0.0757387206,1.1583821774,-0.5384712815,-0.9480490088,0.9694117904,-1.3669112921,-1.2707827091,0.7595877051,0.9194093347,-0.9817193747,-0.7923594713,-0.9515684247,0.1903454214,-0.6589611173,0.2844074667,-0.0866462439,1.4720683098,-0.1156150550,-3.1793930531,1.5431214571,-0.6267422438,-0.5408167839,-1.4073745012,0.0020372916,-0.9605144858,0.0653555542,-0.0072504482,-0.2048006058,-0.8758507967,0.6794900894,-0.5280171633,-0.2964494228,-0.6805481315,0.7795145512,1.9110471010,-1.3809238672,-0.0489975736,1.4007698298,0.4083929360,0.2260774076,-0.6497412920,-1.0933283567,-0.2311290503,1.1378812790,-0.7990297079,-0.7805021405,2.1594064236,0.0872589722,-0.3067907393,-0.5868695974,0.4651990831,0.0277806502,0.3595694602,-1.7833522558,-0.9235187769,-0.1305120736,1.2421206236,0.7602189779,1.3953794241,0.3049729168,-0.5557566881,-1.0600878000,-1.5434368849,-0.4848669171,0.6762301326,0.3975794613,1.4395723343,0.6811670661,-1.5078858137,-0.2627333403,0.4361654222,-0.0008911950,1.0187122822,-1.0375159979,-1.6047658920,-0.0847408697,-1.3486804962,-1.8990767002,-0.7349280119,-0.1065333709,0.6665663719,0.5214542150,-0.0061568609,0.9659379721,0.7524757981,-0.0891807601,0.1273224354,0.8268703222,-0.1176565886,-0.7092362046,0.2952494621,-0.2299559861,-1.0425001383,1.2988185883,0.5426243544,0.4739787579,0.8060304523,1.5848567486,-0.0154908923,-1.9596747160,0.0608664788,-0.8224123120,0.2057334930,-0.3345183134,-0.7199540734,-1.0148650408,1.2570089102,1.7332231998,2.7068395615,0.5335254669,-2.0145659447,1.3850419521,-0.8592329025,0.6600543261,-0.4218591750,-0.0107094571,-0.6457693577,0.9560267329,0.5790085793,0.0617983118,0.3226754069,1.0685539246,-1.2620589733,-0.2347259670,-0.2421554327,-1.3379868269,0.9540162086,0.5375425816,-0.1855080128,-0.8155971169,1.2428810596,-1.7945463657,-0.0921459422,-1.4434609413,0.1425899863,0.0288435649,-0.7485082150,-1.0032165051,0.7188972831,-0.9643496871,1.1487607956,0.7042000294,1.0104765892,-1.3018821478,1.7104345560,-0.2635222673,-1.3582605124,-1.5040088892,-1.0269665718,-0.6213265061,0.1832721084,-0.3864678442,0.4071185589,0.4705953002,-0.4236857295,-0.1801551133,0.7529713511,0.8858818412,0.2697959542,1.4118680954,2.3722064495,-0.9191628695,-0.9307866096,0.3029454648,0.9904204607,1.2289518118,0.4089305103,-0.8350077271,0.5419706106,-0.5536469817,0.1028060094,0.3641077876,-2.9135649204,-1.1976279020,1.7268692255,1.0548107624,0.9957117438,-1.4481039047,-2.2000696659,0.4746692479,-0.3013611734,2.5931897163,0.1785014421,-0.9042234421,0.7220194340,0.5479753017],[0.8089565635,0.2825019360,1.6301807165,-0.9289301038,0.7758326530,1.7873525620,-1.7252171040,-2.6020746231,0.5115439296,3.2497107983,-0.5192397833,-1.0423642397,0.0324750915,0.4287697673,0.8982563019,1.3349349499,0.3121462464,-0.3937326074,2.0630812645,-1.4304995537,-1.3159050941,-0.0241598841,-0.7293787599,-0.3655036390,1.4027618170,0.1256354004,-0.0866579339,1.6698479652,0.6147372127,2.1514334679,-0.0860073492,0.5820814371,-1.2603700161,0.0365599617,0.1094137803,-1.0534101725,0.3607339859,-0.9958238006,-1.4315663576,0.7793776989,0.1461087465,-0.1628145576,0.3107595444,1.3687645197,0.5772598982,-0.4336755574,1.1175009012,-0.2714384198,-0.1034518629,-0.2754325569,0.9069437385,-0.0283357389,-0.1800245643,-0.1651595682,-0.5861871243,-0.4601516128,-0.0606546029,1.5763812065,-1.0784891844,0.9564955831,0.9455194473,0.7763608098,0.4902954996,-0.2564931214,-1.0015263557,0.3603806794,-0.7381950021,0.2104441971,-0.2726085186,-1.4666761160,-2.0567760468,1.1105402708,0.3748869598,-0.8810894489,-0.0609646738,-1.1932916641,0.0129567189,0.5431383252,0.6572669744,-0.6381964087,0.6827172041,0.2055616975,-0.4754789472,2.6849081516,0.9908787012,0.4223089516,-0.9532756209,-1.4316188097,0.4283581078,-1.3681161404,-0.0958754867,0.5441727042,1.0810705423,-0.0415074416,2.5692317486,-0.3227604330,0.4241096675,1.4338536263,-0.4114264250,0.0011840090,0.3396936655,-0.7032704353,0.0369640626,1.9964704514,-1.0196714401,0.2410306334,-0.6645501256,-1.9832742214,2.1286649704,-0.2816694379,0.6200950742,-2.5297253132,0.2783054113,-2.2589633465,1.5027683973,-0.6791571379,0.7946748734,0.5594283938,-0.4154709876,0.0868373662,-1.1731896400,-1.0965510607,0.4667754173,0.3479546905,0.5917227268,-0.1208579317,-1.9895873070,0.0767334402,0.0827785879,-1.5488442183,0.4962759912,0.6963671446,-0.6112600565,-1.9577354193,0.9674578309,0.7537819743,2.4563322067,-0.0252350532,-1.5132838488,1.2217682600,-0.7690975666,-0.2130514085,-1.2223173380,-0.5371199846,-1.0274674892,1.1752959490,0.0587048046,-1.2375228405,0.4799143374,-0.8488432169,-0.6043387055,1.1303751469,-0.3233990967,-0.6648087502,0.9539896250,-0.3432818949,-0.5456913710,1.2928091288,0.6582537293,-0.3367332220,0.1220468283,-1.5532795191,-1.7195379734,-1.0341162682,0.2998998463,0.8046797514,0.4961921871,-0.6684690118,-1.5009387732,1.2318353653,0.3643293083,0.4558056891,-0.7469726801,-0.1064849347,-1.1267255545,0.2651161849,-0.3827407956,-0.8086588979,0.0461993329,1.4123934507,0.1515855044,1.5572073460,1.3240995407,-1.5372346640,-0.2648573220,-1.5860991478,0.0319794863,-0.9175467491,-0.8246659040,-0.2483655810,0.1255396158,-0.5857534409,-1.9942765236,0.0182692204,-1.7602024078,0.3323732615,-1.4235423803,-0.0769748762,-0.6448416114,1.1782761812,-0.8038555384,0.5822150707,-1.4353785515,0.1170133278,-0.8189358115,-1.2466543913,0.6469664574,0.9749919772,1.2244031429,-0.2720718384,-0.0915584937,-0.6139442921,2.7357959747,0.6321662068,-0.4468442202,0.0670683905,-0.0152648930,1.4001054764,0.8164184093,0.2930732667,-0.6081119776,-0.9192171693,-0.1245781928,2.3893449306],[-2.4778037071,0.5976509452,-1.4283977747,0.4154342115,-0.4454664290,0.3893599212,0.2045798749,0.1197161973,-0.0726990327,-0.0892046988,0.2428955585,0.6287866831,-0.8733956814,0.1548168063,-0.8002090454,-1.3946036100,0.5990428925,-1.1735966206,2.5494780540,-0.7590735555,0.1940673888,-0.4431787431,-1.4156470299,-1.9468938112,0.3451087773,0.5958864689,1.1357485056,0.8258775473,-0.5660755038,1.9902119637,1.3801629543,-1.7834206820,0.8641321659,-2.3902966976,0.6164762378,-1.0545002222,0.6365401149,-0.6782473922,0.5563191175,0.7363837957,-1.8891561031,0.0214900393,-0.7824299335,-0.2060978413,0.3763463199,1.4707410336,0.4039954841,0.6742861867,-0.6028607488,1.2719975710,-0.7390785813,-2.4224550724,-1.2123428583,2.3222174644,-0.4605011046,3.1352612972,0.2794305980,-0.5493283272,1.0512810946,0.1013304591,0.3027843535,0.7218084931,0.2959215641,0.6120178699,-2.3468084335,-1.1663336754,-0.8147548437,-0.4591985345,1.3389530182,-0.0083134025,0.0621182621,1.2398926020,-1.1599477530,0.0201061536,1.5943176746,-1.4953566790,0.0830490291,0.2470071912,-1.9120575190,-0.1153577715,-0.4227010310,0.8099350333,0.0726526678,0.8789698482,0.6540535688,-0.6828325987,-0.4248169661,0.3941554427,-0.6652766466,0.0736672878,0.4986302853,1.2773303986,-0.2276820540,-1.7044115067,-0.9628150463,2.3503878117,-0.9085405469,-0.1369735599,0.4999858141,-2.4511468410,-0.1574627161,-1.2132976055,0.7054603696,-0.8242201209,0.9384362698,0.0222000740,0.5847182274,1.7617032528,-0.7561365366,-0.0969338343,2.0506048203,-0.6521903872,-0.5247980952,-0.8815501928,-0.2154593319,-1.5231957436,0.1173115671,0.8983911276,-1.5951839685,-1.7393609285,-0.4271378517,0.2468763441,-0.6876968741,-0.8700183034,-0.3693754673,-1.0298929214,0.4024917483,-0.4774999917,-0.8142039776,1.4894051552,0.6848152876,-0.6682641506,0.5156447887,-0.0836239308,-0.1595827490,-0.9284272194,1.6347107887,-0.5399640799,1.0173338652,0.2623037398,-0.4373755455,1.2972157001,0.9749194384,0.9706473351,0.6446200609,0.8166325092,1.1084922552,-0.1151099503,-0.0970648900,-1.6904464960,1.3225225210,-0.7746454477,-2.2212016582,1.1361017227,-1.3346679211,0.4757602215,-0.9677015543,-0.4630626142,0.0695191398,1.3389387131,-0.1383368075,-1.3319382668,0.6405091286,1.2340289354,-0.6869156957,-0.4758132398,-1.9471237659,-1.2886620760,1.2084196806,-0.5310587883,-1.3227537870,-0.7771701813,-0.0368499942,0.4158807993,0.3814026713,1.0844006538,3.4120693207,1.7345339060,-0.0976254642,-0.2757738233,-0.8358352780,0.1355589926,0.7318641543,-0.3644649684,0.3209478557,0.2786023319,0.7931556106,-0.5511992574,-0.4364145994,-1.2345582247,0.3531896770,-0.1258494258,1.9726265669,1.5069609880,1.0744990110,0.7512360215,0.5726920962,0.0459811017,0.6249359250,1.7143579721,0.9048161507,-0.1954852939,0.0158866066,-0.4643318653,-1.1683137417,-2.2276227474,0.8133713007,-0.5477312207,1.1287671328,0.4268198907,-0.4344830215,0.5964285731,-0.5707772374,0.1042475775,-0.2392384261,-0.3934238553,2.3036134243,-0.9504826069,-0.6469936371,-1.0541242361,0.0337269567,-0.7810605168,0.9689751267,1.9031298161],[0.2525197864,0.1838586181,-0.2395608574,-0.5237963200,-1.1532931328,0.4638881385,0.4851041436,2.1061365604,-1.5883505344,0.6773467660,0.9099257588,0.4675730467,-0.5751948953,0.8763380051,0.7442619205,0.0406341441,-2.2689523697,0.7088384032,0.2085940242,-1.6460503340,-1.2184493542,-0.8154140115,-0.2806397378,0.2383890152,0.7780022025,0.7270069718,-1.3557515144,1.2708170414,-0.1392763555,-0.3877072334,-1.4174968004,0.3400408626,-0.8829170465,-1.9422985315,1.4105211496,0.4198240936,0.3245728314,0.0032399215,-0.4103808403,-0.1612507552,-0.3893881440,0.5924805999,0.0768460557,0.2591560483,-0.7614800334,-0.4818507731,0.3876158595,-1.8072429895,0.6448305845,-0.4991023242,0.2990922332,0.7611392140,0.2680123746,1.9994397163,0.3640748560,-0.2594336271,-1.1131106615,-0.8219708204,1.1891075373,-0.1939040869,-1.7173801661,0.2022092044,-0.5829202533,0.5355664492,-1.4946354628,0.9865855575,-0.8716754317,-0.9377834797,0.3892740905,0.6675186753,2.0363080502,0.9215611815,-2.0991868973,-1.0165593624,2.0159192085,0.4246093333,-1.3649692535,0.8177539110,0.1407611221,-0.7992921472,-0.2672897279,0.7641906142,-0.3375808597,0.2430761158,-0.2130232900,-0.4740942121,1.6659946442,1.5688807964,0.1841378659,0.4639534056,-0.5424900055,1.4749709368,-1.3607293367,-0.6117712259,-1.2864717245,0.7173013687,-0.8206724524,-0.8166086674,-1.1200137138,-0.1569412053,0.4703750908,-0.7065168619,0.4275971949,-0.3475115299,-0.7177425623,1.2331171036,0.1298312247,-0.9110089540,-0.1160564497,-0.1007026881,0.1473488063,-0.0265515819,-0.1134301201,-0.6335337758,1.0367230177,1.2019270658,0.2903762460,0.8871808648,-1.3211030960,-1.2637345791,-0.3478819430,2.6312584877,-0.0779268518,-0.0033409558,-0.6762089133,-0.2439787388,0.8177816868,1.4604249001,-1.5679810047,-0.1339834630,-0.1631641537,1.5747507811,0.1084988192,1.4038569927,-1.6337524652,0.0177575890,1.0167950392,-1.6949776411,0.0757225081,1.4419057369,0.2556048930,-0.6593601108,-0.2549015582,-0.9439916611,-1.8636031151,0.3080304861,0.4328792691,-0.7933393717,-0.2392593324,0.4455813468,-1.1091521978,0.6833079457,-0.1153917611,-1.4299768209,-2.4306547642,-0.4589000940,-0.9767031670,-1.1525596380,-1.1114211082,-0.6227270365,-0.3940295875,-1.1870974302,0.3540764153,0.8213087320,0.5123792291,-0.0097675789,0.7527971864,0.4728479683,1.2856576443,-1.5182529688,0.7346980572,-0.1847112030,-0.7216877937,1.3291749954,1.1263566017,-0.7484042645,-0.2636847794,0.0217362791,-0.9375768900,-0.7247433066,-0.0415024720,0.0492623039,-0.7459357977,1.1220898628,-0.5154392719,-0.4776915610,1.5804840326,0.8040207028,2.2452106476,1.7335001230,-2.4033691883,-0.2907517850,1.0620511770,0.9767885208,0.2632334530,0.5237601995,-0.5357385278,0.4868034720,-2.4270212650,1.2570142746,0.2010706663,0.6235077977,0.7071534991,-1.2176045179,-0.5152527690,0.3981330395,0.2556486428,-0.5797071457,-0.6267167330,-0.9684706330,0.2104416192,-0.8124510050,0.7867467999,0.5026560426,-0.0581949949,1.6213607788,-0.0828385055,1.9328351021,1.7954280376,0.2441668212,0.4763677716,-2.6320970058,-0.0609419681,0.4138411880],[-2.8224081993,0.8544120193,0.3406496048,-1.3216235638,-0.1166282445,0.0615011118,-0.9227725863,-0.7776309252,0.6261429191,0.6446788907,-0.2852533460,0.4866887033,-1.7207942009,0.3032245636,-0.7486038804,0.2438063323,-0.2671179175,-0.3899340332,0.0800387189,1.1771892309,-2.8977830410,0.5417762399,0.5842677355,0.5015832186,-0.4940728843,-1.4268690348,1.7689621449,1.7392749786,-0.3849787414,-0.9646254182,-0.1681044698,0.1822709441,0.6294948459,-0.4607347548,-0.4217625558,0.4491379857,-1.9762825966,-0.8236123919,-0.6104609966,-0.3675303161,-0.7454993129,2.4073290825,1.1063375473,2.4108457565,0.2904265821,-0.0319271833,-0.1898043603,-1.1604915857,1.1043064594,-0.4222970903,0.1301062554,0.3959206343,-0.1617540270,1.0322108269,-1.7450871468,0.2481280863,0.5010092854,-1.0896248817,1.2918280363,0.1254145205,-0.9438206553,0.5197620988,0.8137003779,-0.5376303196,0.2467346787,-0.6972445846,0.9036880732,-1.7012495995,0.1490175277,1.6514542103,-0.3002925217,0.6282120943,-0.0789799169,-0.5649306178,1.1362684965,-0.1409714818,0.7433593869,-0.4661725760,1.0111490488,-0.1314715445,-0.2794724107,-0.5579557419,-3.0807359219,0.6628274322,-0.4908138514,0.2899478078,-0.0956695154,0.4372552633,-1.0438014269,-0.1414678246,0.0443393439,-0.0560218208,-0.4559345841,-1.9249411821,-2.0573368073,-0.0334194154,0.0930603072,1.3541816473,-0.0132205859,0.1044306085,0.2731702626,-1.0667283535,-1.3606183529,-0.3784544766,1.4905642271,1.3429602385,-0.7809277177,-0.3124582767,0.1603041440,0.7163369656,-0.5278331637,-0.8617659211,0.0707306340,0.5078257918,0.5343469381,2.7920517921,1.1080427170,0.6272335052,-0.2741323411,0.5719816685,0.7550273538,0.8943145871,0.1229175404,-0.6765108109,1.5254110098,2.3936822414,0.8236930370,-0.1658976078,-0.5570540428,0.1447302103,-0.0118487366,0.2371482849,0.2607527077,-0.3373178542,0.0334553272,0.4224862456,2.3579299450,-0.2408792973,-1.9752658606,-1.0364269018,3.8317902088,-0.9200903773,-0.6679349542,-1.6793940067,1.1850756407,0.0062380657,0.9284568429,-0.1546033323,-0.8065575957,-0.3873867691,0.6424305439,0.2068197876,0.1537742168,-2.0683746338,-1.7794313431,-0.9078343511,0.3276721835,0.0317510068,-0.9162718654,-0.9382501841,0.8591877818,0.0205972660,0.3160986602,0.4791103005,2.3585975170,1.2266488075,0.0675020814,-0.0706233829,0.1332499087,-1.2125068903,0.9365451932,0.1137873232,0.2121182531,0.9187198281,-0.9210603833,-1.3618185520,-0.8186307549,0.2531672716,1.0069762468,0.8433437347,1.1979171038,-0.5553349257,1.0772843361,0.3243967593,1.4009695053,-0.7212144732,0.2164977342,0.1635417789,-0.4878661633,0.2016631365,0.6704857945,0.1343654990,0.9387943149,0.0494832285,-0.1038764939,1.4206699133,0.1582160592,-1.6772825718,2.0133848190,0.1406247765,-0.4359038472,1.9959181547,0.0242856946,-0.9826890230,-0.8497070670,0.9738262892,-0.9124195576,1.5310759544,1.1907900572,0.8818747997,-0.2512267828,-0.4001435935,-1.6142051220,0.7161724567,0.1099669784,-0.7687640786,0.1460857093,-1.2593784332,-1.1577454805,-0.8739498258,-0.1982318014,-0.4073867500,-1.3021171093,-0.1136029884],[-1.5907045603,-1.1428781748,0.1213455573,0.3092034757,0.0334026329,-2.4200618267,3.3238034248,1.1879281998,1.9430119991,0.2992740571,1.6709127426,-0.3273898959,1.4901436567,-0.6158698797,0.9194712043,-1.7540516853,0.4838142991,1.2830245495,0.5275840163,0.8925089836,-0.4407205284,-0.0098689431,1.5410522223,0.2787226439,0.6298201084,-1.2696967125,-0.1690566242,-1.3529920578,0.4250549376,1.9450334311,0.9914284945,-1.6382533312,-0.4501300156,-0.0474638417,-1.6697709560,1.3145624399,-0.8170112371,-0.8111792803,-1.0784217119,-0.3456541896,0.4999735057,-0.4648705721,-0.7333679199,0.0848675817,-1.6081210375,0.0173648894,-0.0103555061,0.8468238115,-1.1412696838,0.3661522269,-0.4450252354,0.8290936947,1.1968002319,0.5475606918,0.0861864313,0.5601829886,-1.5211524963,-0.5370119214,-0.5964001417,-0.5213727951,1.0056358576,0.3153733909,-0.8991996646,-0.5396741033,0.9581548572,-0.5757839084,-1.0513538122,1.1611233950,1.8634634018,0.4477458298,-0.2449746579,1.5770003796,-0.2315210402,-0.3703002334,0.0056428425,-0.4994624555,0.0295562055,-0.0041788672,0.8292986751,-0.0069422261,0.4954382181,-0.7247942090,1.1682143211,0.0745441839,0.0941772833,-0.7176083922,0.8280134201,-1.6582189798,0.6430417299,0.1367926598,0.7049418092,0.0571377203,1.1631231308,0.6296694875,-0.0647290796,-1.2796778679,-0.5895133018,0.8610818982,0.3743764460,-0.8071950078,1.3061199188,-0.8803336620,-0.0079116700,-0.5633738637,0.3912031353,-0.1816237271,-0.1595174670,-0.8321972489,0.2964517474,0.9229819775,1.4701541662,0.6736425757,0.5532526374,-0.1107472777,1.7293556929,-0.5153669119,-0.6763880253,0.2341462821,-0.5053901076,-0.1485505253,1.2039222717,0.4330215752,1.6672991514,0.3798584044,0.5994235277,-0.3695307672,0.6090422869,-1.8857115507,1.0357469320,-0.6409605742,0.5081863403,0.8336287737,0.2464233786,0.1892288029,-1.5773829222,0.7635652423,1.6710473299,-0.3723566532,-0.0935106426,0.4004655480,1.6516311169,-0.0239151251,1.3460192680,-0.0556113571,0.7014779449,-1.1409986019,-1.9849413633,0.7863185406,0.1040583923,-0.5113148689,-0.6147990823,-0.1334390640,-0.8577879071,-1.6896690130,0.0303357765,1.0349303484,-1.3887677193,1.4465106726,1.4462199211,0.9695534110,-0.9796079397,1.0599244833,-0.3817900717,0.5725802779,-1.0808739662,0.6993675828,0.0584711172,-0.5152519345,-0.3773689270,-0.9231536388,-0.0607250929,2.4555163383,-2.2077910900,1.5051277876,-1.2260414362,-0.2329563051,-1.2366898060,-0.3229627013,-1.4195483923,-0.1831594706,1.7906260490,0.2295542806,0.2179629952,0.3471575677,0.2313944250,-0.0609209612,-0.1078507155,0.4665036798,1.1243994236,0.9174999595,0.2093791664,1.6390047073,-1.3195793629,-0.1680346727,-0.5173400640,0.6296047568,1.0929300785,-1.5066785812,1.2682543993,-0.3669987917,-0.0628513321,-0.3588504791,-0.8302126527,0.2542398572,-0.3541894555,0.8507209420,-0.6196182966,-1.1007553339,0.5014449358,-0.8606661558,0.4498639703,0.2191863060,-1.4525839090,-0.1573316753,-0.3670399189,1.1949836016,1.2781003714,-0.3769445121,-0.8998956680,-0.1330458224,-2.1117086411,-0.3949528337,1.4143472910,0.0138987098],[2.0456752777,-1.7744398117,-0.0091569005,-2.6830213070,-0.2830215991,0.5834649801,2.3459873199,-0.9336617589,-1.0544435978,0.9978377223,0.8136739731,0.8705741763,-0.4490517080,-0.4460271597,-0.8419141173,0.2584621310,0.2546221018,0.0829493180,0.1972437054,-2.2637302876,-0.3299877644,-1.9285436869,1.0270245075,-1.5507107973,-0.4767905176,0.3876853883,0.9809426665,-0.6491833925,0.2031453401,0.8613374829,0.9279358983,0.3166069984,0.3596234322,0.0556599312,2.6604681015,-0.3559695184,-0.8140570521,1.2609890699,0.7009537816,1.6023476124,-1.6866204739,-0.3539562225,0.2053398639,-0.1173354909,0.0861274227,-3.4895308018,-0.4967344999,-0.0145240240,0.3744366169,-0.9175367355,-1.4568159580,0.7877227068,0.6968756914,-1.1660602093,1.4751977921,-0.1938352287,-0.0264743380,-1.3262991905,0.0237702597,-2.4104363918,-0.6477180719,0.5059946775,0.3510814011,0.3875126243,1.2187185287,-0.2408093661,0.6088964343,-0.3309341669,1.8326859474,0.2369977683,-0.3282163143,-0.2274724394,-0.2073322684,-1.0838879347,-2.1840865612,-0.4663523138,2.0218403339,0.9449817538,-0.9978976846,1.3662493229,1.7149190903,-0.2701505125,-0.8451486230,-1.6304798126,-0.7960706353,1.2277041674,-1.7852749825,1.2480875254,-0.3981883824,-0.7367050648,-0.4135238230,0.5079085231,0.0900633857,-1.2990952730,-0.6961169243,0.3780362606,0.9619675279,2.1818091869,0.4237046242,0.6840161681,0.6984956861,-0.2514249980,-1.7293785810,0.3667742610,1.1325265169,0.7683765888,-1.1529784203,0.4323818088,2.3151834011,0.3081058562,-0.6248998046,0.5047844052,0.6498021483,1.7978770733,0.3899771869,-0.4997722208,-0.1578063518,-1.9672470093,-0.3175988495,-0.0856336206,-0.9187674522,0.9462944865,-0.1890290082,0.0526048094,0.7671905756,0.0052128769,0.6099808812,0.5507966280,-1.3356704712,1.1185127497,-1.0913438797,1.2091479301,0.7399811745,0.1125142574,-2.0963685513,-0.2536096871,-1.4916297197,0.6988286972,1.0503125191,-0.7035870552,-0.4649752080,0.3794374168,0.7661656141,-0.5972264409,-1.0623397827,-0.0284309927,1.3018770218,-0.9428182840,-0.2252209038,-0.3813332319,-0.9502642751,0.0767621696,0.7689592242,-0.0794237703,0.2420342267,1.4336521626,1.5250978470,-1.3550601006,0.5420228243,0.7595084310,0.1850525141,0.9257733226,0.2702014446,1.1696821451,0.8106309175,-1.0838518143,0.4734386802,-0.4565702081,1.3057682514,1.4480742216,-0.5279594064,0.6467399597,1.6548110247,-0.0588355176,-1.4721015692,-0.9004341364,0.0964303613,-0.6494836211,-1.2516726255,-1.5430279970,1.6699434519,-0.2119133621,-0.3817618191,0.3938360512,2.3915538788,-1.0785287619,1.3833017349,-0.5712346435,0.0911998898,1.0019682646,0.4382897615,1.1474125385,1.1693954468,0.1031880304,0.7955002785,1.3405262232,0.0209042039,1.6260020733,0.3945917785,-0.3832558692,0.7924461961,0.0312300827,0.7102786899,-1.7634069920,-1.3943989277,-0.7769746184,-2.2571027279,-1.1994607449,1.0450494289,0.3268254697,-0.6890351772,-2.0280230045,-0.4204472303,0.4531336129,0.1239523813,-1.0549724102,-0.4190306664,1.0953723192,0.2018714398,0.7987915874,-0.2929423451,-0.1676543951,0.5970343351,-1.2773684263],[1.2236326933,0.9840433598,0.1225740537,1.1968276501,-0.3415886462,-0.6405214667,0.7638618350,1.5436466932,-1.0006358624,0.1138499379,-1.0741703510,-1.5411993265,-0.7750322819,-0.9824083447,1.0209515095,0.7557222247,1.0524106026,0.2535609007,-0.3112972081,1.7932996750,-1.2139261961,2.0946898460,-0.0090470267,0.6447563171,-0.7594442964,-0.1354953796,0.0660691410,-1.0660305023,0.8187485933,-0.3587822914,0.4471886158,0.8411183357,0.5693242550,-1.4816662073,-0.7689180970,-0.7788417339,-0.9212349057,0.7576090693,-1.2389307022,0.6051481366,-1.1756383181,-1.2799351215,-1.8131713867,-0.0544976145,-1.5947587490,0.1935119927,-0.3740875423,-1.1885846853,-0.0792386606,1.1125651598,0.3793795705,0.5841192603,0.0571746901,-0.1136863083,-0.5652914047,0.7779823542,0.4327304661,-1.0822675228,1.0263322592,-0.4865728915,0.0487550497,-0.2343792617,-0.2356239855,0.0258828122,-1.5499720573,0.2839837670,0.0145347295,-0.1443930119,1.1493870020,-0.6262537241,-1.7196328640,-1.7135055065,0.4295227230,-1.3662779331,-0.4895913601,0.2493050396,-3.3102571964,0.4204231799,-1.0910575390,-0.9758739471,-0.1436772048,-2.1989107132,0.3001672029,-1.7423200607,-0.5652357936,1.4203227758,0.3454657495,-0.8482355475,-1.0011060238,0.8571538925,-0.4992506206,0.7553210258,-0.8703880310,0.7454122305,1.4743127823,-0.3983528912,0.8992680907,-0.8724070191,-0.4594956636,-0.2891397178,1.6464035511,-0.5800548196,0.1192378998,0.3251797855,1.5719088316,0.2433685362,1.6238752604,0.9168971181,-1.7127541304,0.4310311675,-0.2766275108,1.5945236683,0.7060499787,-0.2566100061,1.0110602379,1.4094588757,0.9080002904,2.1751139164,1.6054965258,0.9616665244,2.2810950279,0.4121274054,-0.0153397154,-0.3080755472,1.4366900921,-0.4038511515,0.7201999426,-0.7327430844,2.0949237347,0.0393173620,2.2166905403,1.2483799458,0.4711368680,-0.4010816813,0.7753511071,-0.5172150135,0.3040603995,-0.2500374317,-1.0669492483,-0.0891368464,0.0055941949,-0.9773228765,1.1023194790,0.4938498437,-0.5543298721,0.0906332210,0.0505249016,0.5267651677,-0.5476112962,-1.3049540520,0.8578358293,-2.0298936367,-0.1674510688,-1.0671063662,-1.4121545553,-0.9709029794,-0.3150966167,1.9244461060,0.4939165413,-0.0465229526,0.4197863042,-0.4554887116,-0.8888947368,-0.4829266965,-1.9815404415,-0.1060535312,-1.4770417213,0.6611261964,-0.8000780344,-0.0008512373,0.0904324129,-2.3193302155,1.2738482952,0.8960350156,0.3338158429,-0.8541143537,0.3179859519,-0.2948468626,2.4262568951,-0.6027595401,-0.1137509346,0.3913904130,-1.4957355261,0.4632006288,-1.5061677694,-0.0406272337,1.3672878742,0.9088773727,0.0280201621,-0.3759260476,-0.5266292095,-1.6783390045,0.7885782123,0.3884814978,0.5505877137,-0.2127685249,1.5057326555,-0.3375179768,-0.4418465793,0.7669808865,0.4645462036,-0.4733898342,0.0666616634,0.6061386466,-0.4868551195,0.4049857855,-1.6582262516,1.4912316799,0.1251794696,-1.2685055733,0.3584597707,-0.0061725895,0.1182529852,-0.4459925294,0.0649669096,-0.5827793479,-0.9905433059,-0.9163452387,1.7737463713,-0.4419859648,1.9480772018,1.4152547121,0.9885281920,-0.7954316735],[-0.6662418246,-1.1790033579,-0.2741505206,-2.0678997040,0.3266300261,1.3303706646,-0.2867909372,-0.9021807313,0.2714713812,-0.5843097568,1.2097675800,0.2966309488,-1.0622887611,0.6333710551,0.3833867610,0.2917357981,-0.5103815198,-0.3244171441,1.2576582432,0.9913121462,-0.1317096502,-0.5059637427,1.1085257530,0.1522968709,1.1645395756,-1.2047892809,2.2415204048,0.1009635627,-0.1189174354,-0.3255894184,-1.5088373423,-0.1761455089,-1.2958545685,0.4631999731,-1.4431842566,-0.0815225467,0.0264114328,-0.0134794405,-1.7359671593,-0.8405629992,-0.0171891171,-1.5212892294,-1.3272206783,0.3295044005,-0.3159745634,-0.4405059218,-1.2775102854,-2.0660405159,-1.3951530457,-0.3871063888,-1.9635423422,0.3878398240,0.2240091264,-0.6311438084,1.1141709089,0.2829698622,0.1319483519,-0.9153652787,1.4351322651,-0.6812025309,0.1347745210,0.8289664388,-0.1638492942,-0.8752292991,0.6741952300,-0.8481402397,-0.4189682901,-0.8234933615,0.2876498699,0.0553242527,-1.3581985235,0.1725343168,1.4257600307,0.3076897562,-0.6096960306,-0.6603984237,0.2657020390,-0.3375091851,1.0193190575,0.3068545163,0.3049698770,-0.8007757664,-1.4549874067,-0.7384971380,-1.3470081091,-1.8314106464,0.0719065815,-0.4571023285,1.0792492628,2.2859106064,0.2279076427,-0.9891816378,-0.6099649668,-0.6531402469,0.4977306128,0.8263016343,1.8157972097,-2.3561737537,0.7034680843,0.3216913640,-0.2272416651,-0.8027136326,-2.7809767723,-0.7257278562,-0.2273125350,0.2905314267,-0.3215314150,-1.8055938482,1.4343173504,-0.9048855305,0.1905162185,0.0468965657,0.8480873108,1.7085329294,0.4332759082,-0.3628672361,-0.6735289097,-1.1163346767,1.3994438648,-0.4484830201,-0.9012812376,-0.4600181282,-0.4331746101,0.1538828015,2.2690322399,-0.1937780529,0.1015669405,-0.5000990629,0.1091853082,0.6234668493,-0.4246721566,-0.0613945201,-0.8901450634,-1.1397343874,-0.8808919787,-1.0332076550,-1.1599528790,0.0582559519,1.4459686279,0.1238117516,-1.9827600718,-1.5158689022,-0.6710330844,-1.8118439913,-1.6237982512,0.0679273456,-1.3040943146,0.3018227518,1.1907341480,-0.0415180326,-3.0691003799,-1.3719874620,1.0663380623,0.4812533259,0.4310096502,1.4828522205,-0.5797268152,0.4582100511,-0.6201152205,0.1096500233,-0.6994256377,0.3923276663,1.5864090919,-0.9220345020,-2.6383047104,0.1972300112,-1.7775818110,1.6898244619,0.5709993839,-0.5445369482,-0.2049824893,-0.4456205368,0.4843392372,-1.4021123648,-1.1148446798,1.3989207745,-0.1107301638,1.9977177382,0.0048229685,0.6804937720,-0.4943865836,0.0055954242,-0.8179009557,0.6151177883,1.7471002340,0.5628360510,-0.6135786772,0.1982985437,0.3813651800,0.1579486132,-0.7328537107,-0.0750542954,-0.1824423671,1.0054148436,-0.0228289776,-0.3487198949,-1.0940510035,-0.7222530842,-0.6091982126,-0.6036486626,-0.2050597668,1.1312741041,2.0681664944,1.1864436865,-0.2236890197,-0.8625605106,-0.7263569832,0.3035775125,0.0642245412,0.4337040782,-0.2739275396,-0.3962904513,0.8403483033,1.0489102602,-0.4937454462,-0.5222024918,2.6116652489,0.1352985650,0.0618634447,-0.3024633825,-0.8286788464,0.1451219916,0.5880844593,-0.6094891429],[0.3219236434,-1.3695703745,-0.0585177056,0.1700653136,0.4964697361,-0.9321046472,-0.1507416666,-0.3361282051,-1.9466036558,1.1729111671,-0.2739106417,1.6313178539,-0.1329207718,0.2038348913,-0.7290941477,0.8423547745,-0.1522175819,0.0712911412,0.9698452353,-0.7045807838,-0.3561033309,0.0883658156,-0.4302078485,-1.2478718758,0.4989421368,-1.7669965029,-0.4198913872,0.6975441575,0.4738608301,-0.0651089847,1.0222551823,-0.5519806147,-0.3389316499,1.5764091015,-0.7053538561,-1.2888039351,0.3234673142,-0.1417339593,0.9278524518,-0.2274197787,0.7508451343,1.4466332197,0.6488268971,-0.9954628348,-0.6652079225,0.2864868939,-0.3673034012,-1.4958170652,0.2327007651,-1.5480067730,-1.3630514145,-0.4443548620,1.6024609804,-1.0053298473,0.7348178625,0.8800488114,0.8419062495,0.0995379835,-0.2248645723,-0.5294179916,0.3160320520,0.0924004465,-0.4043036699,0.8951402903,-1.2198828459,-0.0668722764,0.4917221963,0.5889586806,-1.8725498915,-1.7988634109,0.3151299059,-1.3311706781,-0.4863022566,-0.3315199912,-0.1942166537,1.7517579794,0.2368437648,0.5656731129,-0.0282656793,1.7823673487,-0.1720479727,-1.3562507629,1.0161684752,-0.9181568027,-0.6771322489,-0.2678036094,-0.6415093541,-0.9906380773,1.3346472979,-0.7320179343,0.6933484674,-0.3105112314,0.0937701985,0.5368575454,0.0130084530,-0.9673388600,0.5386470556,0.1005189866,2.0324866772,1.2055453062,0.8220839500,-1.5560021400,0.1882465482,1.6134934425,-0.0579883754,-0.4294087291,2.3164374828,1.3468019962,0.7022433877,0.2779477239,-0.5003404617,0.8945329785,0.2074485123,0.0166519284,0.9358554482,-0.3249890804,1.4698566198,-0.4166862071,-0.5823774338,0.6684930921,0.0280649178,-0.3356093764,0.5318160653,-1.4786239862,-1.0148028135,1.1849619150,0.2391745150,-0.6188251972,1.0286505222,-0.7010456324,-0.5719571114,0.1553662270,1.0689032078,0.0141669363,-0.0817926750,1.2871496677,0.8973370790,1.0335407257,0.5311743021,0.2168038338,0.3711017966,-0.7261734009,-0.6836606860,-0.4303300679,-0.9623488784,-0.2232476622,-2.8081274033,-0.1310659349,-0.1354033500,1.3199428320,-0.4372418821,0.3999006748,-2.7013614178,0.2619364262,0.1507003158,0.0715506971,0.8289863467,1.2401783466,-1.0547454357,-0.0138927512,-1.1025797129,-1.3504421711,1.4697799683,-0.4074902534,-1.1207348108,0.0986788496,0.3246071041,-0.8392274976,1.0582296848,0.2235145271,-0.5397490263,1.1688777208,0.2009891868,0.1039046645,-1.0009261370,-0.8749092817,-0.2685181201,-1.0310175419,-0.4049829543,-3.1205077171,-0.3098204732,0.5935285687,0.1903157085,1.1804158688,-0.3473457992,0.8149939179,-0.7612107396,-0.8262937069,0.6924495101,-1.3365644217,-0.7892350554,0.1343418360,0.7926063538,-0.1322494149,-0.4287471771,-0.3642910719,0.1303860843,-0.1418085694,-1.5912516117,-0.0276062898,0.4361262321,1.0095407963,-0.0749416277,0.2285602689,-0.1831205785,-0.0460781157,0.2784950733,-1.8076405525,-0.3249053955,-0.7071861625,-1.1327867508,-0.0700493380,1.8727740049,-0.9032890201,-0.4349606037,0.4104333520,0.6316285133,1.0076644421,-0.8771318197,-0.9682021737,0.4976361096,0.8091021180,-0.0814546943,0.1587979943],[0.5097995996,0.5882976651,0.5308931470,-0.0849480703,1.5427222252,-0.3993819952,-0.6722313762,0.6797321439,0.3277135193,-0.0085896691,-0.3661646247,-0.9072237015,-0.4623303115,-0.2023209929,0.6599558592,-0.7539873719,0.8019641042,-0.1096544489,0.0187816937,-0.2943032682,1.1049185991,0.8500798345,-0.8090435266,-0.9830206633,-0.2533683479,1.5624762774,-0.8042505980,0.0995938256,-1.0303261280,1.0731709003,0.5126503110,-0.3870714307,-1.0472843647,0.3077230453,-1.7716023922,-0.1703357995,-0.3809374273,0.2320100963,-1.0177683830,1.2246047258,-0.6436703205,-0.2181359231,1.5420596600,-1.2188315392,0.5913078785,-1.0695223808,-1.0774040222,0.4448950589,1.1289564371,-1.2322168350,-0.9961451292,0.1253382862,-1.2857888937,0.8030779958,0.5583163500,-0.6780901551,-0.0292556807,0.6156740189,0.6637461782,-0.2593209445,0.1577128321,-0.6255224347,-0.5165698528,1.3434090614,-0.1373769790,0.8557762504,0.7699329853,0.0583121888,-1.5436519384,1.2966266870,0.5823848844,-0.3058349490,-0.1300800890,-0.3919242322,-1.1922485828,-1.0443159342,-1.5529685020,-2.7964434624,1.2522928715,1.2292847633,-0.2496839911,-1.6875931025,-0.2166756839,-1.1366854906,1.1348092556,1.0292310715,0.2931405902,-0.0401573479,-0.9168517590,3.1208748817,-3.0247054100,0.1153336465,-0.2590649724,-0.5816124082,1.8359223604,0.8361560702,1.3724067211,1.9320363998,-0.8699535131,0.3323044777,0.9254535437,0.9190768003,2.0381422043,1.0914663076,0.7399589419,0.6411070824,0.1178541556,-0.8540045023,1.2046840191,-1.7498428822,0.1689570099,-0.7534065247,-1.4080204964,1.3100963831,2.1610531807,-0.6405741572,-1.1049610376,-0.5449877977,-0.2860692143,0.3250898123,-0.9200837612,0.9884458780,-0.5333630443,-0.6851798296,-1.0003677607,-1.5293353796,0.0621513054,-0.0578085035,0.6272996068,2.0618607998,-0.2794499397,-0.4480098784,0.1708033234,0.8488475680,-0.0460077003,1.1200356483,0.3858909607,-0.1463131756,-0.8500095010,1.1876649857,0.8287144899,-0.3788069487,0.2699967027,0.7841120958,0.9621093869,1.0886086226,-1.4207915068,0.8679859638,1.1370193958,1.1478097439,-0.0892617479,0.4420532882,0.4368574619,-0.1616665870,1.2207653522,0.3205839992,-1.3781871796,-0.5439456105,0.6632543206,-0.2686441541,-1.0038115978,-0.1014877558,0.5503932238,-1.3899176121,0.0010357754,1.5733733177,0.4335486889,0.0456469469,1.9128178358,2.6157686710,0.3550651968,2.7038369179,1.0162559748,0.6567167640,1.4713921547,-1.3117628098,1.0215899944,-0.9033085108,-0.1266045272,-1.4393845797,1.2639056444,0.7589184046,0.8724140525,0.3952097595,2.1992070675,-0.5563782454,1.0330309868,2.2867393494,-0.6047636867,1.7792675495,-0.8073445559,0.4877997935,-0.2104053199,-0.7016320229,1.4434002638,0.0233583841,0.4562973678,-0.4751683176,0.4907547235,-0.1294404715,0.5382491946,-1.3764859438,-1.3180493116,0.6598469019,-0.9768490791,-0.5608139038,0.7176390886,1.1750555038,-0.1087838933,0.8607848883,0.0236848835,-1.5650117397,-0.8263857365,0.2376575172,-0.3216416240,-0.0618195906,0.5945374966,0.3223091960,1.4338846207,-1.7596757412,0.0521411188,0.6781387925,-1.2394995689,-0.9775453806],[1.7855298519,0.4754302204,1.0716226101,-0.0919560567,1.0172238350,-0.3709467351,0.1175377220,0.3524052501,-0.0831708163,-1.4916843176,-0.0503998026,-0.6599224210,-0.2826690972,1.7006320953,-0.6317148805,-0.3335561156,-1.8706378937,0.4418658316,-0.9441236258,-0.3783243299,-0.9927918315,-0.3547174335,-0.6230579019,0.2012146562,-0.4615501761,0.6221498847,-0.2511247694,-1.4008331299,0.3153209388,-0.1327913404,-1.9306480885,0.2888429463,-0.1532785892,-0.7431642413,0.3893904388,0.5003846884,0.5587242246,-0.7005661726,-0.5150581598,0.6435684562,0.3248820007,-0.8303997517,-0.6451705098,-1.5633621216,0.4436821043,-0.5834668279,1.4078401327,0.9660906792,-0.2825075686,-2.9486505985,0.4625095129,0.3063019514,-0.4668817818,-0.8732379675,-0.3178085983,1.3156703711,1.1733013391,0.0139046516,-0.7732798457,1.6154451370,-0.7379982471,0.2065338343,-0.9543405175,0.8040487766,-0.4187107980,1.3101384640,-0.0562577173,1.0397031307,0.8087363839,1.0376975536,0.6207315326,-0.0367724709,0.1032700464,-0.7664158344,0.7779768705,-1.7194195986,-0.9962170124,-0.4599902630,-1.0737614632,0.4924404621,0.5062992573,-2.3763055801,0.3256783783,1.0295050144,-1.0963584185,1.0508966446,2.2131047249,-0.3020640910,0.3280752301,-0.0142530557,0.0741175264,0.3232976496,1.8511095047,-0.0932176560,-0.1747188568,-1.9773966074,0.8212793469,1.3709951639,1.3017138243,0.5213615298,-1.8491699696,-2.2682118416,0.7733955383,0.2187506855,1.3515372276,-0.5978830457,-0.1860491633,0.7750968337,-1.1614238024,0.5150699019,-2.3115155697,-1.1966111660,0.1801932007,0.2726337612,-0.1692290753,-0.5952160954,0.0005237133,0.6088415980,-0.3638955355,-2.2119069099,-0.6087537408,0.8907001019,-0.7753039002,0.1969347894,0.4207064807,-0.8330823183,0.0483885109,1.9549907446,-1.6341890097,0.1474251449,0.3637137711,0.9511996508,-1.1052323580,-0.9953976274,-1.4866085052,-0.2924993634,-0.0756657571,0.8346943855,-0.0225741137,-2.0840604305,0.4949536622,-0.3814873397,1.1613310575,-0.3354426920,-1.0039441586,-1.1599848270,2.5265002251,-0.3191393018,0.3884355426,0.1033144817,1.4330396652,-0.0158385038,-0.5452295542,-0.1095836759,-0.7352720499,-0.9227036238,0.0774430260,0.0953032821,-0.3727390468,-1.0648605824,-0.2965101898,0.8064007163,0.8640318513,1.0614836216,-0.4732660949,-0.9734390974,-0.4409328401,-0.4907247722,-0.4260222316,-0.3214211166,-1.2692657709,-0.6843496561,-1.0450330973,0.9267991781,1.0043290854,-1.9433656931,-0.1234595403,-0.0358733200,-1.0778970718,0.8430383801,-1.6297807693,0.4672100842,0.1302661300,-1.3048177958,0.7166115046,-0.4405407906,-1.6556293964,-0.7538868785,-0.8053990006,1.7398165464,0.8470565677,-1.0320606232,0.6498590112,0.6128501892,-0.2097587138,1.7268390656,-0.4334447682,-0.9774934649,0.0173344910,-0.8519948125,-0.2736777067,1.1822780371,-0.7529136539,1.2937862873,-0.4342118800,-0.4783442914,-0.1367436051,1.1648869514,-1.3607287407,-1.2247972488,0.5062522888,-0.3501808345,-2.3892455101,-0.3553076386,0.0785791352,-0.4684511423,-0.9138480425,-0.7515913844,0.3910466731,1.8633239269,0.0983788744,-0.5529601574,-0.8192595840,0.0123285549],[0.9051536322,-0.5296494961,0.6193763018,-0.7575354576,-0.6588627696,1.4482965469,0.1804517806,-0.0969546363,-0.8561577797,-0.1636443138,0.7998893261,0.4571678042,-0.0476388596,-1.4426069260,-0.1742287725,-0.5233820081,0.7883758545,0.3852189183,0.3091126084,0.5430634022,0.4737339020,-1.0917273760,-1.0901318789,0.4070112109,-0.9084571004,-0.7717982531,0.0192359034,0.9061601162,0.5126354098,-1.3967828751,0.7774495482,0.5376854539,0.1111765280,-0.8687179089,-0.6647869945,-0.0788185447,-0.2937527895,-1.9742597342,0.9528505206,-0.6390838027,1.2707260847,-0.6682310104,0.4715318680,-2.0942647457,1.7908589840,0.2074149996,-0.4311299622,0.4696592391,-0.6548035145,0.4858731925,-1.1523069143,0.5596313477,-1.4902211428,-1.7995990515,1.6622927189,-0.2245040238,-0.5269060731,-0.0428438373,-1.2792727947,-0.7476548553,0.0938511789,0.8331119418,1.3060201406,0.3938763738,0.9923769236,-1.6811056137,-1.1295176744,0.2734489143,-1.6697459221,-0.6384560466,0.9949476719,-0.5062407255,1.0034281015,0.0994017720,-0.0723804981,-0.0227089208,2.1757483482,0.1033674479,-1.2962023020,1.6298233271,0.0310833752,0.0043407893,-0.7852146626,-0.7798393369,-0.7882912159,-1.0835288763,0.3038004935,-0.0921095908,-1.4625827074,-1.5864539146,-1.0401488543,0.3578006923,-0.6439752579,0.6123004556,-0.8602662683,-0.0679839328,-1.9755679369,1.0582910776,0.0997800231,-1.1577682495,-1.5413347483,-0.3838081360,0.4210002422,0.4642764032,-0.0339855291,0.1298860610,-0.3884651065,-0.8865935206,-1.6143238544,3.6056587696,0.1462464035,1.5213328600,-1.0632134676,-0.5663945079,-0.1952020824,-1.3035510778,-0.0436232910,-0.4883011281,-1.0581581593,1.6403815746,-0.2546814978,0.3874523938,0.2777298093,1.1789630651,-0.2963931262,-2.3299434185,-1.0283929110,0.4161494374,0.7006410360,1.7180471420,1.0294718742,0.8499308228,-0.1957364976,-2.0873782635,-0.9732530713,0.3517284989,0.6282029748,0.7819412351,-0.6832048893,-1.0266739130,1.5057219267,-0.9331673384,-2.0270714760,-0.7710177898,-0.6412399411,-0.2176401466,1.9960709810,0.1312145889,0.7394256592,1.3804028034,-3.1236875057,0.8488565683,0.7830206752,-1.2917408943,0.4925467372,0.1587381214,-1.1164747477,1.5651832819,-1.0977840424,-0.6612092853,1.8947798014,-0.7653524280,-0.3012660742,0.4719370008,-0.8919570446,0.1150531024,0.2762588859,-0.4841531515,1.3143765926,0.2696977258,-0.2131957114,-0.6583449841,-1.3425199986,0.2763188183,1.1737834215,-0.8113678694,0.4808428288,-0.8791851997,0.2634968162,-0.6410830617,-0.0812414438,-0.6814115644,1.5104262829,0.5177908540,-0.6783463359,-0.1256770492,-0.2949545383,-0.7587720752,-0.8653795123,0.3860862255,1.2165676355,1.2671173811,0.2852470577,-0.5262190104,1.2160490751,-0.2968252599,0.6153306961,-0.6760441661,-0.0796759948,1.0532268286,1.3543426991,1.5206708908,-0.2734349668,1.6068272591,-1.2159032822,0.4170197248,-0.4496528506,0.5826635361,-0.1970722526,-0.2496076673,-1.2450256348,-0.1965055168,-0.5271847844,0.4371919930,-2.1151013374,-0.2608734667,0.0152701726,0.5723733902,0.0681537613,0.0613985993,0.3411239684,-1.2768346071,0.3332662880,0.5430427194],[-0.0432091579,1.6785727739,-0.7715849876,-0.1714796424,-0.3674465716,0.6206015348,-1.6828806400,1.1599649191,0.4741328657,0.8494206071,1.0655183792,0.8664425611,0.8542308211,0.2530686855,1.4608001709,-1.4641044140,0.9866617918,1.7032502890,0.9210450649,0.8014882803,0.3322421610,-1.5023698807,-0.3505462706,0.0389856398,0.0831000507,-0.3857187629,-1.5359857082,1.2615644932,1.2758626938,0.1480398178,1.2954460382,-0.7860206962,-1.1860078573,-2.2061941624,-1.0470296144,0.3352247775,0.5302492976,-0.0405291393,1.4120001793,0.4016290307,-0.1539154351,0.9125192761,-0.5446004272,1.2182688713,-0.8864365220,1.1918206215,1.0045500994,-1.6376503706,0.9360725284,-1.2072974443,-0.7660700679,0.5314910412,-0.8932678103,1.1890400648,0.3187058568,1.6210167408,0.4157439768,-0.4446620047,-0.6844602227,-0.8145654798,0.6531211734,0.8158108592,1.4798234701,0.4006395042,-0.7666428089,0.9254111648,-0.3484019637,-1.1469308138,-0.7036637664,-0.0442226976,-0.2382203043,0.4113468826,-0.5789852142,0.7120308876,-0.6015210152,-1.1392376423,1.1177431345,0.4594758749,1.5914516449,-1.0618811846,-0.8215261102,-0.7494154572,0.7032563090,-0.1234351471,0.0701818541,-0.5553544760,0.0761191547,0.3339363635,-2.1919014454,-0.3116528094,0.8165085912,0.2899995446,0.2868409753,2.1124336720,-1.1487253904,-1.4357818365,-0.3476918936,-1.3994435072,0.8405013680,-0.9114860296,0.4391860366,-1.6877720356,-0.2196552604,1.3988096714,-0.6408332586,-0.1922979355,-0.2654271424,0.0860480741,0.2465860546,0.2520399392,2.9076354504,-1.4887257814,0.6890859604,0.4454374611,0.6194879413,-0.0967055187,1.1441223621,0.1449663192,-0.5741001368,0.0515831709,2.8111915588,0.2500806749,-1.0168375969,0.0459030941,0.5006407499,1.2243378162,-0.5594944358,1.5234234333,-0.5856988430,0.8465883732,-0.1062590182,0.7699720860,0.7508335114,-0.5606443286,-1.7602663040,0.4370567799,0.3021217883,0.1467457712,0.6101540327,1.5521664619,0.0121918637,0.0001930556,-0.7419973612,-1.3801542521,-0.0754940435,1.1674489975,-0.0283292662,-1.2254366875,1.6937772036,0.7008804679,0.1472029239,-0.2088548094,-0.9180703163,-0.9908519983,-0.8559581637,0.5744030476,0.2971726060,0.7592765689,0.5797044039,-0.3228662908,-0.3427666128,-0.1186992005,0.2182934582,0.9786879420,-1.4661244154,0.7656714916,-0.3914667666,-0.4707569182,-0.0234617610,-2.1558485031,-0.8340327144,-0.4761915803,-0.1583889872,-1.3076349497,0.6191383600,-0.3673848808,0.4211728573,0.0395293981,-0.2179098874,0.1381019950,-0.0317963026,-0.4639281929,-0.0641642138,-1.1540219784,-0.9627858996,-1.6770368814,-0.5858719945,-0.0825671777,-1.5955755711,-0.0500167795,-0.0760848224,-0.0237301663,-0.0268115401,0.2086068839,-1.6112877131,0.0350952968,-1.4237899780,-1.2756197453,-1.6744543314,-1.3942058086,-1.4011231661,0.5572247505,-1.6251987219,0.0624367483,-0.9734458923,1.9991265535,0.3975417614,1.9408439398,-0.4244121313,-0.7449286580,1.5648454428,-0.1315009445,-1.1770961285,-0.1874196529,-1.0828311443,-0.4274128377,-0.3763251603,-1.8707411289,-0.3534707427,-0.0243204199,1.9254937172,0.5000748634,-0.0090672439,-0.1229096130],[-0.1555079669,1.1247876883,0.6508572698,1.3430335522,0.3828526735,-0.7241315842,-0.0204289500,-1.2060266733,1.8055220842,-0.0309408773,1.6533294916,0.9167777300,0.6583053470,2.4935271740,0.1820680201,-0.4153838158,-1.7359157801,-0.9003521204,1.3195201159,1.1472709179,-0.3788455427,0.3824750483,0.5139021873,1.7121081352,1.1626836061,0.1476384997,-0.5816458464,0.2407868803,-1.5572460890,-0.9878185987,-1.6791173220,-0.1426104307,-0.6590248942,-0.6210706234,0.9594504833,0.2867678702,-0.0755748749,-0.0945067704,1.8625407219,1.0528297424,-0.6334456205,-0.5078506470,-1.3274567127,-0.7107422948,-0.2926018834,-2.0036118031,0.1115319952,-0.0476051159,-1.3324609995,-0.7805432081,0.3220021725,0.9446145296,1.6483644247,-0.2507813573,-1.4725340605,-2.6068005562,-1.0014157295,-1.0183038712,1.2119328976,-0.8830393553,0.5760194063,0.4326705337,-0.6804663539,1.8935378790,1.0359658003,-2.4166157246,1.1839493513,-0.2268783003,-0.5061920881,0.9377138019,-1.2114729881,-0.7506918907,0.4980424941,0.8761253357,-0.4148249626,1.1402990818,0.1848191768,-1.8540214300,-0.8729506731,0.8739518523,1.9581677914,2.4186632633,-1.1326782703,0.5931650996,-1.1454510689,0.1306309998,-0.7470979095,-0.5057881474,-0.9316557646,0.0993926078,-0.7553661466,0.1219220161,-0.2455041856,0.3656617999,-0.8616326451,0.1999523640,0.1532318890,-0.7006615996,0.4613142610,1.3785855770,0.6721458435,0.3734226227,-1.3855634928,0.4933677316,-2.0586693287,-1.8312758207,-0.5378765464,0.1734742075,1.1764724255,-1.6062233448,0.9842114449,-0.1152400449,-0.0080436319,2.1545655727,0.9554353952,0.1760829836,-0.4261439741,-1.0313150883,-0.4449267685,-1.6515427828,0.8663641810,-0.3967720568,-0.3075744510,-0.8735423088,0.6620267034,-1.0369021893,0.5346903801,-1.5483399630,-0.3606949151,-1.5275582075,-0.1260908693,1.1827498674,-1.3001586199,-0.8946741223,-0.4685489535,-1.0407261848,0.2360029519,-0.1681945175,0.7322675586,-0.1572569311,1.2127138376,0.0375163071,-1.0115327835,-0.0792735964,-1.0087019205,-0.2862460911,0.9906303883,1.2327841520,1.3395975828,-0.1617410332,-3.1198022366,1.2752115726,-0.9587536454,0.0452648103,-1.0152710676,-0.5220885873,1.2208900452,-0.0604511015,0.6024882793,0.4948946238,0.0399792418,-0.2535513341,1.5232809782,-1.2082173824,0.1494268030,0.7835847139,-0.2103703767,-0.0482308120,0.1537257582,1.2463809252,0.6994246840,0.5315209627,2.1850254536,0.4130679369,-1.9432587624,1.1612490416,0.5338376760,-0.0464589708,1.1988745928,-0.9518498182,0.1815148890,-0.7856477499,-1.3307493925,1.2344479561,1.0216497183,0.6948999166,1.3199999332,-0.3467248082,-0.6526562572,-0.2523265183,-0.4008188844,0.8922727704,-0.4961834848,-0.2049113363,1.6276949644,-0.1190614551,0.1953179687,-0.0473487377,-0.3821465373,1.0792948008,-0.5341405869,-0.7568749189,-0.2242150456,-1.9106316566,0.4949327111,0.8403900266,0.4573747814,-0.8882073760,-0.5454280376,1.3229922056,-0.0953648686,0.1615354121,0.3111441433,-1.2016353607,-1.1228525639,1.5913999081,0.4555059075,-0.0846310630,0.0696771741,1.3984402418,0.8977686167,-0.0743149444,-0.8610787392,-0.0034345402],[0.4643716812,0.6500335932,0.3972373605,0.6056055427,-1.1137615442,1.2621937990,-1.0029988289,0.7454153299,-0.9797261357,-1.0222493410,0.2281863987,-0.2279123515,1.2973057032,0.7009657621,-0.1237428635,0.6601970792,1.6168614626,-0.1405349374,0.4543885589,-0.9073397517,-1.3353450298,-1.4128642082,-0.8668837547,0.2140561193,0.6704117656,0.2983856201,0.1711228490,0.0040384457,0.2274931222,-0.5629349947,-0.2145272195,-0.6500881314,1.0181735754,-1.0987073183,-1.1695632935,1.4457358122,0.9086647630,0.5250781178,1.1210864782,-0.5814261436,-1.9068636894,1.2717565298,0.3779475689,1.1875561476,2.1112318039,1.3692845106,0.4907457232,-0.1053989232,1.7720201015,0.3179871738,0.6021529436,2.0522155762,-0.8119975924,-1.2906919718,1.9221531153,0.2619466484,1.4544030428,-0.0270028636,-1.0798466206,2.4531943798,0.4816752672,2.3576691151,0.8212106228,0.6885333061,-1.1239271164,-0.3848465085,-0.5353651047,1.2736561298,-0.4609678388,0.1968630999,0.5500658154,2.0417609215,0.5277529955,-0.4230228662,-1.5167474747,-0.4852030277,1.0694479942,0.3578872681,0.3670755923,0.3847409785,0.5441589952,-1.1518552303,-0.3627217412,-0.2236316949,-0.1309448630,0.3189993501,0.7243591547,0.6128135324,0.6833281517,0.1058774963,-1.0476717949,1.2348972559,0.1849824041,-1.0603389740,2.7461400032,1.0114446878,1.3566025496,-0.6985847950,0.9919859767,-0.1195081174,-0.5770586729,-1.6611901522,-1.4490962029,-0.1540308297,-0.0394946598,-1.5549813509,0.4014787078,-1.4460828304,0.3835073709,-0.2769566774,-0.1825580746,-1.3795892000,-1.0845396519,0.2838324010,-1.9333698750,-1.5066390038,-0.8090303540,-2.3525457382,0.3431841731,-1.8326971531,0.4958780706,-0.6096104383,-0.1282366365,0.0555926375,0.8264662027,-0.5319551826,-1.3494676352,0.0178227182,0.2657372057,1.5358214378,1.4378138781,-1.5428961515,-1.0020878315,-0.1664336324,0.3720614016,-0.9722996950,0.4478465617,-0.5604529977,-2.8267419338,-0.1732289642,-1.3194926977,0.7245579362,-0.4114677310,-1.0185942650,0.0171113219,-0.3274767399,0.8993369341,0.4165078700,-0.4409069121,2.3318879604,1.0301353931,0.0106273629,0.5165274739,0.3694303930,-0.7169600725,-2.0160875320,-1.5899252892,-0.7279121280,1.0887588263,0.2646250725,-0.2574935853,0.2369214147,-0.1405727863,1.1408565044,1.3749165535,-0.3872929215,0.3741849363,-1.1731528044,-0.8771096468,1.7028417587,0.7422641516,0.7952908874,-0.6359503269,0.0625767857,-1.4387636185,0.7526410222,0.5943829417,-0.6834931970,0.4352502227,0.0361339785,-0.3459370732,-1.3282161951,-0.3925054371,-0.3875874877,-0.4427610338,0.6347910166,0.7603368759,-0.4462294281,0.2494061291,-1.7198584080,-2.5241043568,1.0394392014,-0.8128814697,-1.3440935612,-0.9509823918,0.5185998082,0.6612313390,-0.3765857816,1.2585207224,-0.2729130089,-0.1594479978,-0.8824411631,0.3794788122,1.0085316896,0.3316577375,0.5210019946,-0.2534552217,-1.6504944563,0.1905167997,0.3426152468,-0.4149730504,0.4229606390,0.5006102324,-0.6884571910,-0.0085293930,0.2035993785,1.6628166437,-0.7422149181,0.4549663663,0.9701353312,-1.4671440125,1.0306642056,-0.1751043946,-0.3081717193],[-0.1086230949,-0.3445905149,0.3018523157,0.5331122279,-1.4184136391,-1.3059164286,0.0133221578,-0.4561957717,2.0107197762,0.0168454461,-0.5268836021,-0.5163084865,-0.2862188518,1.9081737995,0.3808764815,1.1053049564,-0.1701475978,1.8028768301,1.3454405069,-0.9716171622,2.0151402950,0.4353454411,0.2021296322,0.3094969988,-1.2130287886,-0.6061766148,-0.8579288721,-0.5894806981,0.1631432176,0.1187847629,-0.4992081821,0.6937823892,-0.3709974885,0.3257789016,-0.3987151682,-0.7485961318,-0.2293497622,-1.0146118402,0.0110471090,1.7328293324,1.0677479506,1.2207168341,-1.9264101982,1.5223257542,0.3796040714,-0.3781650662,0.1991085708,-1.2878689766,1.4412564039,-0.3321222961,-0.0153004900,-1.0780576468,0.9807602763,-0.1083590388,0.3219371140,-1.0011390448,-1.0939098597,1.0125167370,0.2617959380,-1.9774245024,0.8176585436,-0.5400171876,0.8523032665,0.0277114268,2.1040093899,-1.4332144260,0.8153598309,0.4336749613,0.6829534769,0.6602942348,0.4276879430,-2.8726365566,-0.9237739444,-0.4451941550,-0.1291576624,0.0826604590,0.2901602387,0.1507606357,0.4976918399,1.9305351973,0.6905815005,1.1261466742,0.1136901900,0.4927196503,2.1992692947,-0.7340601087,-0.6472967267,1.2486354113,0.4821243584,-2.0340688229,-0.7239329815,-0.0443657637,0.1745765060,-0.6582645774,0.4640335143,-1.2629460096,-1.6618301868,0.1800085753,0.9277454019,-0.2590980828,-0.7413901687,-0.6935709715,-0.5875934958,1.2377283573,0.3266763985,0.0787180141,-0.5542194843,-0.9741730690,-0.4150099754,-1.5515369177,1.0647892952,0.6235165000,1.3300273418,-0.3489451408,1.3718315363,-0.6768189669,1.3851624727,0.8357720375,-0.6882252097,-1.0305742025,0.0712251291,0.3130685091,-0.6692923903,0.9052766562,1.9800167084,0.9550880194,-0.7589914203,-1.5862555504,1.5604054928,1.0711680651,-0.1050480083,1.1862883568,-2.3280029297,-1.6582649946,-0.3555588722,0.8859654069,2.1518859863,-0.7442384362,0.6685908437,-1.1250841618,-2.7138724327,0.0137464041,1.3366407156,-0.4603686929,-0.8399344683,-0.4507792294,-1.4983799458,-0.8027215600,1.6966629028,0.4295882285,1.5084462166,0.4342085719,0.1687584966,-0.2440459728,-0.1066379622,1.1238811016,-1.6822667122,-0.9517872930,0.4881279469,-0.0958632976,-0.4737888873,0.4130543172,-1.2868320942,0.0282274242,-0.0538525246,-0.6100097895,-0.4368116558,0.5414137840,-0.6348329186,-0.6271486878,1.1452444792,-0.3874225318,-1.0906164646,0.3533582091,-1.7489846945,0.8108950853,0.1281350255,0.9760327935,-1.0696296692,0.8331253529,-0.9128801823,-0.5867547989,-0.1854138970,-0.8016400337,-0.0122459978,-0.2811549902,1.5724511147,0.8933423758,1.5236115456,1.1318390369,-2.1643643379,-1.1678924561,-0.5303066969,0.1001020819,1.5827381611,-0.3443536460,0.4299806058,-0.6880388856,2.3481752872,0.8799027801,-0.1208624095,0.3161224425,0.1859720945,1.3103215694,0.5370991826,0.2128050774,-1.6082152128,-1.0610541105,-0.3076724708,-1.3912265301,2.0203952789,1.2839845419,-0.1427117586,0.7247159481,-1.4809105396,0.4151991010,0.7133069634,-0.9822463989,1.0256047249,0.8960664868,0.2456524372,2.2871968746,-1.1332281828,-0.9934768081],[0.0801362023,-1.4279161692,-0.1673499644,1.9552475214,-0.7649120688,-0.1214441359,-0.8505011201,0.0120534971,-0.2664821148,0.1010014117,-0.0249482337,0.4609630108,-0.4709973037,0.7293856144,0.3688291311,-2.2109615803,1.2869336605,-0.3740674257,0.0017372227,0.1274275184,-0.0548788235,0.4981512427,2.0514848232,-0.3548823297,-0.5679026842,-2.2651638985,-0.9933956265,-0.4226258695,0.5724143982,0.6467649937,-0.5059992671,0.6243510842,-1.3668729067,-0.5580800772,-0.2308873683,-0.4146345556,-0.8414208293,-0.8874806166,0.8085399270,-0.7730150223,1.0769597292,-2.2644515038,-1.7992531061,-1.5193790197,0.6511667371,2.0010735989,-0.4351148307,0.5123409629,0.4797986448,-0.3782280385,0.3045934141,1.0081927776,2.1089031696,1.4194885492,-0.7833939791,-0.7446916699,0.1809913516,-1.4201674461,1.5365930796,1.5332006216,-0.0955717042,-0.0791906789,1.2793784142,0.2346884012,0.6085548997,-0.7300610542,-0.7544391751,-0.0971425176,1.7628134489,-0.5690230727,-0.2787840366,-0.2442644686,1.3795033693,0.6344102025,0.4248405397,-0.1316951513,-0.4010356665,0.9959358573,0.4057629108,1.0247602463,-1.6310348511,-0.1843463928,-0.0751169100,1.0483053923,0.8583419919,1.0581882000,0.5709701777,0.1121916398,-1.2642484903,-0.7674326301,-2.1037101746,-0.3202399611,-0.6719353795,-0.8261151910,0.7101277709,2.6124789715,-0.1418309510,-1.3277183771,1.0787343979,-1.2282276154,-0.1210674867,0.2911917567,0.6180588603,-1.4232866764,0.5941316485,1.0013989210,-0.0831873789,-1.4150587320,-0.3144525886,0.2749459743,1.5043956041,-0.8493507504,-0.3435238004,0.1091308668,-1.4348704815,-0.6094563603,-0.3990669847,0.1706257463,-1.0890892744,1.3205559254,0.2972381413,-0.7722620368,-0.3236086071,0.4466733932,-0.4585110843,0.8907366395,0.3113555014,-1.0298800468,0.6594182849,2.1507937908,-0.5677946210,0.3051919937,-0.3771156669,-0.5940546989,1.2152264118,-1.7259300947,-1.0823833942,1.3255735636,-0.8910102844,-0.0269354563,-0.1528990567,1.4183866978,-1.6457661390,0.8868795037,-0.8474878669,-1.3936561346,0.0618900172,-0.5172175765,3.1002268791,-1.4958193302,0.3102186024,-0.9779730439,1.1106261015,0.7437641025,1.4932644367,1.0055906773,-0.7624963522,-0.4558341801,1.2478998899,-0.0513665862,-1.0852686167,2.5063498020,1.4703984261,-0.3500232399,0.6674960256,0.2232310176,0.7430081964,1.4739216566,-0.2805545330,0.0959332883,-1.2647999525,-0.8458613753,0.6121600270,0.3654181659,-0.5369811654,-0.4391386211,-2.1446189880,0.8581434488,1.3331199884,0.0303843282,0.6091219187,1.1128925085,-0.6695650220,-0.2710272074,-1.6208012104,-0.6248922348,0.5111383200,0.4374771416,-0.0601305887,-0.3222166598,1.5219869614,-0.2043878138,0.7294411063,0.5184146762,1.4763427973,0.9772499800,-1.1257607937,0.1171994582,-0.5923586488,-0.3355710208,-1.0741111040,-0.5246708393,-2.2213320732,-0.1392166018,-0.4191494286,-1.0089921951,-0.3403863013,0.5482953191,-1.1640391350,-0.3767908812,0.5043502450,1.2352068424,-0.0104251159,-0.1011670530,-1.4769479036,1.0991642475,0.9631522298,-0.1085720211,-0.3289594054,-0.1785956621,-0.1722016186,-0.9106789827,-0.0757652223,1.6013062000],[2.8825588226,0.0283018276,-1.4420264959,-1.0566796064,0.3184948862,-0.7164399028,-0.6493185163,-0.9150819778,0.1000853702,0.6208550334,-0.1778074652,1.1770408154,-1.3932850361,1.4212722778,0.5371090174,0.4546726048,0.4049543738,0.3981168866,0.0488865748,0.8620789647,-1.1955425739,1.2540414333,-2.0386204720,-0.0258903690,-1.3946413994,1.7492851019,-0.3227463663,-0.0915191621,-1.4765635729,-0.0714700073,-0.4856380224,-0.3160601854,-1.1019278765,-1.4820511341,0.4517401159,-1.3478686810,-1.5971720219,0.4214863181,-1.2817587852,-0.3930203617,-0.1581448764,0.6314710379,-1.2951623201,-0.1740406007,-1.0108921528,1.2809277773,0.6833066940,0.6616382599,1.7153782845,-0.6153312922,1.5842894316,0.7544012666,1.6303297281,0.1301917434,-2.7217433453,0.3398594558,-0.8163784742,0.1465732604,-1.2905712128,-0.4101938009,1.6654448509,-0.2119739801,-0.7760418653,-0.4227268100,0.6563307047,0.2659547925,-0.0037535611,-1.0131603479,-0.2349339277,0.9261472225,0.3318826854,0.3253736198,-2.6372146606,0.2674591243,-0.1940494925,1.3571807146,-1.6915826797,0.5772549510,0.8997503519,-1.3403272629,0.7346234322,1.5062402487,-0.1679347157,1.2141203880,-1.6488654613,0.6915194988,-0.2572011054,-0.1990943253,0.2498833686,1.2450773716,0.3484862745,-0.2589534819,0.7116019726,-0.0218918305,0.5702000260,-0.9457767010,0.6703088284,-0.1766744554,1.3134074211,-0.1710943580,-0.2920600772,0.9888873696,0.6314431429,-0.6614246964,0.9294165969,0.2508786917,0.5414218903,0.2954601943,0.8805912137,-0.0731926933,1.7276763916,0.3145478964,1.1606113911,1.0999108553,-1.5043370724,-1.5866116285,0.0370911658,1.3214834929,0.0746138021,0.9906339645,0.1499240994,0.9241827726,-1.4057812691,-1.2694085836,-1.2315986156,-1.0792061090,0.9238930941,0.3661459982,-1.6559048891,0.3552600443,0.1590854973,-2.0836281776,-0.3751182556,0.8667919040,0.2102769166,1.4756578207,-0.5184097886,-0.0366294533,0.8832123876,-1.0546194315,-0.1811883897,-0.1210824773,-2.1764533520,-1.5692141056,2.0706942081,-0.9752756357,0.1462701559,-0.1786944121,-0.9237043262,0.2103956491,-0.1442062855,1.2261948586,-1.5112875700,1.5062558651,0.0944361016,0.7306627631,0.6643269062,2.2133276463,0.8330996633,-1.4814969301,-0.3682819903,-0.4991627038,0.5526918173,0.3599629402,0.6244876385,-1.0617635250,-1.0455168486,-1.1398391724,-1.3221683502,0.0184009988,-0.5667765737,1.8121285439,-0.9326394200,-0.7341508865,-1.1592438221,1.2299790382,1.7348306179,-0.7335332632,0.8989298344,1.2027136087,1.1966621876,-0.1625650376,-0.1017761528,0.2831718922,0.4695369601,1.1291228533,0.6805573702,-0.0990784019,-0.7106087804,0.2395488918,0.1024907678,1.7224653959,-0.9518821239,-1.7251338959,0.9914779067,-0.9855437875,-0.3009017408,0.9082549810,0.0410030857,-0.0605210140,1.1658284664,-0.9308250546,0.8737416863,-1.2085171938,1.0346195698,-0.5344747901,-0.2332480103,0.6903033853,0.1028862447,0.8828144073,1.8908827305,-0.7360958457,-0.2265916616,-0.5171056390,0.6966750622,0.3447278440,-0.0323995762,-0.6886252165,-3.2709035873,-0.8757738471,-1.4021294117,0.1837808639,-0.9577113390,1.2251466513],[-0.2875821292,-0.2114524692,-1.4256474972,0.8641709089,0.0528381169,-0.0744572952,1.5797815323,-1.1734684706,-0.0783758909,0.2729905248,0.5856707692,-1.0022348166,-1.2545819283,1.0302791595,-0.3580471575,-0.9342455268,-1.9813054800,-1.5998274088,1.3066407442,0.4830575883,1.6324607134,0.1079617515,-0.4973579049,-0.5194500685,-0.3114619255,0.1192606017,-2.0525612831,0.2186866999,-1.5327219963,-0.4967102706,-0.1050102040,-0.0578927137,-2.1460046768,1.3595274687,0.2326336652,-0.1665844172,-0.4899091423,0.7253504395,-0.2241188586,0.0341161974,-1.7404079437,0.5003447533,0.4687319100,1.5334626436,0.2852185965,-1.5014708042,-0.4241717756,-1.2355568409,0.0657551959,1.2892919779,1.2911703587,0.6110507250,-0.0205472484,-1.2269330025,0.0883116126,-1.0893718004,-0.6376819611,0.5561304092,0.0832567215,1.6728909016,-1.2436583042,0.1721850187,0.6242291331,0.3138146996,-2.2066717148,0.2796196043,1.0710690022,1.3156319857,-0.0950455889,-1.5921502113,-0.9913931489,0.1297880560,1.4663813114,-0.1556957811,-0.8660828471,-0.4202181101,-0.9978961945,0.5445381403,-1.5141788721,1.9711934328,0.5910512805,-0.9280182719,-0.6373056173,0.4286117256,0.7815433145,-0.0861902535,0.0154004376,1.5583499670,-0.5532020330,0.2428787202,-1.3022339344,-0.6369246840,-0.1703581214,-0.3426039219,0.8764404655,0.5651657581,-1.1531502008,1.3143377304,-0.6532971263,1.3424408436,-0.5932373405,-0.3148380220,0.3417423964,0.0807463825,0.7385244966,0.1785105616,-1.0377227068,2.5720541477,-1.6021740437,0.3730979860,-1.0089817047,0.2074581236,0.7690361142,0.1414442956,-0.0658927634,1.2981730700,0.5637852550,-0.7915480137,1.1714626551,-1.0327466726,1.2980322838,-1.8191382885,-1.5950011015,-0.0475060828,0.1701105088,-0.1173396930,-1.1642503738,-2.2592539787,-1.0229673386,0.0272143669,-0.0346517377,1.9163516760,-0.3508054614,-0.5195938945,0.7642535567,-2.5424456596,0.0920520052,-0.5108087659,-0.8645259738,0.8696979284,1.5335197449,0.1574706435,0.8634047508,-0.7973005772,-0.7276571989,-0.0082663214,0.6169670820,1.2819828987,1.0705560446,-0.3476598859,0.8035858870,-0.2554123700,1.5247392654,-0.3427659571,-0.2781977654,-0.0387262739,-0.2124831229,-1.0138919353,1.1995728016,-0.3768789768,-1.0625375509,1.2412786484,-0.0552544296,-0.7425494790,0.5503547192,-0.2265021652,-0.5776142478,-1.0426775217,0.0635304376,-1.0789164305,-0.0305424035,-0.4114821553,0.8515140414,-0.8038818240,0.1948237717,0.1175146028,-1.1841410398,1.4999989271,0.7257269025,1.1487137079,1.6531249285,-0.9639879465,0.9420316219,-0.6486244202,0.2358347327,0.2451820672,-0.9946019053,-0.4021207988,0.6609389782,-0.6528478861,-0.7113485336,0.2650088370,0.6931459308,-1.5257225037,-0.5300140977,0.4230892360,-0.1780603528,-0.7085021138,-2.0326261520,1.1543872356,-0.6382230520,0.1267194450,1.1673766375,-0.1194199398,0.5173939466,1.3585667610,1.1757655144,1.2875258923,1.7019704580,-0.3240980208,0.3856199980,-0.5278858542,-1.4395867586,-0.4803539813,-0.7832537293,2.5728754997,0.6120492220,0.1088558584,-1.4145896435,-0.0893893540,-1.1972055435,-0.8456657529,2.5421011448,0.8142038584],[-0.4200177789,-1.3778551817,-0.1695551574,-1.4214105606,0.3386326432,0.4762045741,-0.8144193292,-2.0145246983,1.3744714260,-0.0912032276,-0.5277309418,0.3268625140,0.0133851767,-0.5244994164,-2.6017088890,1.9043643475,1.1018794775,-1.4449181557,0.0988714769,0.1080312133,0.2872729599,-1.0176873207,-1.5658758879,-1.4753624201,0.6132729650,-0.6109262109,0.7954557538,0.3561187685,-0.6416935325,-0.8037799597,-1.0986231565,0.6042654514,-0.0362247378,-0.4225933254,-0.0711886138,-0.5749738216,1.0292119980,-0.0141652646,0.3381133080,0.2150502056,-0.4366036057,1.1847845316,1.2134108543,0.0970221311,-0.3986929655,-0.3410035074,0.5849024057,0.8327379227,-0.3633055687,1.1413456202,-0.3347489834,-1.4785804749,-0.6094909906,0.8936745524,-0.2556134760,-0.1951664984,-0.6637004018,-0.9761881828,0.5172336698,1.0505046844,1.0351791382,0.9528251290,-0.6650516391,-0.5152972937,-0.5114353299,-0.5137189627,0.1342589110,-0.4055825472,0.9870741367,1.1039853096,0.7906116247,0.1031517833,-0.5891082883,0.4176992476,1.1658488512,0.9659720063,0.0746390373,-1.0539230108,0.8605904579,1.0937052965,1.4656609297,-0.0631425753,-0.2436380386,0.5220047235,-0.5257426500,-0.3511520624,0.1896903366,1.1232941151,0.9083298445,0.6321584582,-0.2297795564,0.4393576384,-1.1335648298,-0.1767812520,-0.9877588749,-0.6336392760,0.6270663142,0.5877546072,-0.2975437939,-0.6329882741,0.9075940847,0.1420644522,0.4181271493,-0.8419764638,1.3827379942,-0.2564235032,0.4914464951,-1.2188130617,-1.7116364241,-1.8048491478,-1.8463013172,0.3928754032,1.4446197748,1.7741932869,-0.2339616269,0.8514254093,-0.7865287066,-0.4787963033,0.8111802936,0.5455805063,-1.4866166115,0.5192504525,0.5673210621,-0.2531650364,0.4797608554,-1.7640885115,-1.7594573498,-1.0405248404,-0.6443892717,-0.0463766120,1.5923037529,-0.9849032760,0.9268825054,-0.3332609534,0.1753576994,-0.4664939940,1.1735558510,-0.4948464334,-0.5844563842,0.7905429602,0.9834874272,-0.5397875309,-0.3261406720,0.5392394662,-0.5751277208,-0.2247804999,-1.1403326988,-0.9152319431,-0.3199635148,-1.0563613176,0.4798598886,-1.5495011806,-0.2683282197,1.0509999990,-0.8671765327,-1.7904404402,-0.4655726850,-0.8255794644,1.2549281120,-0.5127987266,0.3980410993,0.5606667399,0.2152752578,-1.6999623775,-0.0291132275,1.8648895025,0.6139222980,0.0244569257,-0.9495642781,-0.0074470057,-0.9770514369,-1.3872021437,0.1106198654,-1.5846104622,-0.4539607763,-1.1245915890,-1.2696129084,1.3264654875,-0.6482177973,1.0197538137,-0.7960487008,0.0526552238,0.1811360568,-1.6001381874,-1.2790617943,-0.7372736931,0.8097754121,1.8652113676,0.6001631021,0.4037535191,-0.0492363907,1.1228337288,-0.6498822570,1.0116053820,-0.2541756630,-0.5277975202,-0.2402507961,-0.1130158454,1.1338608265,-0.7202472091,0.5499478579,-0.9729791284,-0.8757820725,-1.8153737783,-0.6840500236,0.0959931836,0.5253782272,1.0612294674,0.0080760587,1.4577175379,-1.3004442453,1.9005447626,-0.7658629417,-0.0105433399,0.8367369175,-0.6676630378,0.9297543764,0.9685356617,-0.0386456437,-0.4531649053,0.1079624817,-0.0924528167,0.6686045527,1.7595429420],[0.8170405626,-0.6267473102,-2.5353696346,-0.6208639145,0.0412966460,0.5412843823,-0.6714181304,-0.1228801236,-0.3316660523,-0.1653130352,1.4541288614,0.6057051420,0.7982510924,0.1380858123,-1.4818567038,1.8989332914,-0.9859840870,-1.2145018578,0.7614639997,-0.7683216333,-0.4145599902,-0.4076405764,-0.2403526902,-0.5470996499,0.9964763522,-1.8223305941,0.4747248292,1.5665532351,0.9140262604,0.1729218364,0.5444464684,-0.1636365354,-0.8873438835,0.4439423680,0.6227127910,1.7554078102,-0.4988125265,-1.3193113804,-2.5715079308,0.1748979837,-1.1597133875,0.9316927791,0.1183342338,1.0189135075,1.0244257450,-0.5853592157,0.9235587716,1.2721583843,0.8519009948,-0.4364132583,-1.2144123316,0.2642391622,-0.3939892352,-0.0805365071,0.4169847965,-1.6299033165,-1.9063054323,-0.9444448352,0.9864621758,-3.2844877243,0.4394325912,0.5473648310,-0.2011999935,0.6504402757,0.3405825198,0.5267879963,1.7770034075,0.5222274661,0.9281985760,-0.2189456522,0.3734814525,-1.2006289959,0.2925578952,1.1988446712,0.7234704494,-0.4607257247,-0.8953572512,-0.0688983947,-0.1996186823,2.5704195499,0.2287005633,1.5053178072,0.0987073630,0.4207965136,-0.1668317318,-0.3914250135,0.0472337753,0.8962624073,-0.6044977903,-0.2340323478,0.3957251310,-1.4881333113,0.4008922279,-0.2279271781,2.0719885826,-0.5289013982,0.3241880536,-2.6822669506,1.7045650482,-0.1832091957,-1.3438804150,-0.1880180538,1.0730087757,0.3521874547,0.2114675641,0.5612317324,0.9730195999,-1.4720804691,-0.0725681335,-1.4717452526,1.6405420303,-1.4896878004,0.0528693758,-1.7767858505,0.9301193357,1.7837607861,0.2781662941,0.1405596286,-0.3057101071,-0.2079758793,0.5234370828,-0.5637179017,-1.4416309595,-1.0048124790,0.4227478504,1.0247695446,-2.4148521423,1.2287952900,2.0506439209,-0.3796081543,1.4330184460,-1.6261191368,-0.4310197830,-0.3651142120,-0.8131893873,0.1408641189,-0.0862338468,-1.0842740536,-0.4975840449,-0.8313937783,-0.3324953318,-2.9499680996,-1.3614028692,-0.0652090460,1.5336951017,0.6620618105,-1.4137949944,1.0900887251,0.1952648610,0.4613917172,-1.3025218248,1.1172809601,-1.0675442219,-1.5549499989,-0.2244365811,0.5666527748,-1.8053593636,0.2591793835,1.1105011702,-1.9888157845,-1.3004876375,0.9484360218,-0.1045219526,-0.1354653537,0.7844735980,-0.3132534623,-1.7118363380,-0.1989667118,-0.2725273967,1.5450774431,-1.2673224211,1.6919636726,0.2171541452,2.2025887966,1.4983998537,0.5503693223,1.3401776552,1.4544260502,-1.9329987764,0.1200451478,-0.9440642595,-0.3997438848,0.7572956085,-0.0289923865,0.4980324805,0.4981862605,0.6089060307,0.2215690613,2.8459966183,-0.3495443463,0.3898267448,-1.3111178875,-1.4949693680,1.5423781872,0.4355337620,-0.2667261362,-0.6964426637,-0.0315350629,-0.0684453547,1.2804672718,-0.3476506770,0.3679663539,-1.9576956034,-1.5535807610,0.3000622094,-1.0038506985,0.0751238614,0.0237246975,1.8045992851,-0.9453797340,-0.2001632899,2.1614372730,-0.9781127572,-1.0257860422,0.3228924870,-0.0656457618,-0.5376325250,-0.5086116195,2.1152424812,-0.4741887152,0.9335947037,-0.9201823473,-1.5525681973,-0.8249668479],[-0.2767834663,0.3954752684,-2.5128004551,-0.5187504888,1.7483252287,-1.5200672150,2.7166440487,-0.3970314860,-0.8433889747,2.4272284508,0.2976959348,-1.3999366760,0.9331608415,-0.6598845720,0.7047004104,0.4617165029,1.2049220800,-0.4600013494,0.7962696552,1.4818439484,0.4008847177,-0.4036520720,1.8412313461,-0.1604305208,0.0991646573,0.4628383219,-0.9043536782,-0.8916121125,0.4315722287,1.3325688839,1.9264107943,-0.6885731220,0.3321375847,-0.2413543910,1.0912894011,1.1436660290,-1.4943883419,0.4226326942,-0.3977138102,0.6066868305,0.1816943884,-0.0278797448,-0.7122353911,0.7773500681,-0.3370968997,-0.0819773525,0.3859794736,0.0197459012,0.7838968635,1.1054192781,-0.0492523052,-1.1183481216,2.0457713604,1.4440535307,1.4088461399,-0.3506045938,0.9419050813,-0.4516931474,-0.0231885668,-0.2311489135,-0.5307110548,-0.6097031832,1.5688374043,-0.0618275702,-0.3840580285,-0.1462161839,-0.7490805984,-0.4684410393,-1.1734311581,0.2761482000,1.6468441486,-0.4369893670,0.6186989546,0.5075575709,0.2093936950,-0.0073280954,1.4158865213,1.3943082094,-1.1606276035,-0.1925585568,0.7860649824,-0.1255825013,0.7291187644,-0.0382035486,0.8796746731,3.2665982246,-0.1572821438,1.2827008963,-0.2016982436,0.0290385038,1.2065564394,0.5020515919,-0.0758089051,-0.5608183742,-2.4836220741,0.7568008900,1.7811390162,1.6835224628,-1.1110769510,0.3087627888,-0.5306309462,1.4413764477,0.0940743685,-1.1490490437,1.7519093752,1.1665536165,-0.5972436070,-0.9645920396,-1.8831322193,0.3731502593,0.7097339630,0.8633415103,0.0572413802,-0.3672955632,0.0742917284,-0.5524155498,-0.0990694165,1.2753888369,0.1014469489,-0.2720424533,0.7490035295,-0.0318105072,-2.5118894577,-0.7490311861,1.6888785362,-1.5632879734,0.3120557964,2.4351465702,0.9439433813,1.7685650587,-1.7593694925,-0.9471104741,-0.3451783657,0.7697889805,-0.6745629311,-0.1576409042,-0.7649592757,0.7051118612,0.4522475898,0.1596816778,-0.3910192847,1.8914453983,0.1121585667,-1.7256836891,2.5300681591,0.2082690150,-1.8599275351,0.7754518986,1.6154131889,1.3406716585,0.7618990541,-0.1711893827,-0.0310675185,-0.0496421643,0.4200766385,-1.5414303541,-0.3754616678,1.0248707533,0.7819921970,0.8943247795,-1.0348619223,-0.3936333656,2.4675981998,1.1788799763,-2.4421749115,-0.7062406540,-0.4198507965,0.0803024247,-0.6633006930,-0.1075372994,-0.4508400857,0.4074032307,-0.7849459052,-1.2711269855,1.3193122149,0.4826998115,0.5387716889,-0.2636992335,-2.0554578304,0.3182610571,-0.1439537555,-0.0299516935,0.4889170527,0.6923212409,0.6136444807,-1.1940793991,0.4658052623,-0.0280528236,-0.3650117815,-0.5338830352,0.7320910096,0.6896799803,-1.2262221575,0.6613311172,-0.8810014129,0.0679493994,0.3078416884,-0.5542497635,-1.0899014473,-0.1688338518,0.6363798380,0.9226398468,0.2054868639,0.2460445613,-0.9185663462,0.1189117134,-0.0556812920,1.2891730070,-2.4134826660,-1.5054019690,-1.1148625612,0.5536409616,-0.7582384944,1.5371382236,0.5538336635,0.2182800174,-1.1763197184,0.5749715567,0.7819045782,0.8062111139,-0.1452429444,-0.2014859468,0.5254017711,0.1605516672],[-1.7581435442,0.5122259855,-0.6614473462,-0.5915403366,2.0017547607,-1.4677304029,-1.8257400990,1.0064178705,-0.5828084946,-0.4141933322,-1.9140504599,-0.0616672523,-0.0180219132,-0.4038219452,0.6312733293,1.2066689730,-0.4449644983,-2.8706459999,-1.2025235891,1.0925306082,0.7826869488,-0.7611364126,0.3302960694,0.5973567367,-0.0336623602,0.4979988039,-0.6890585423,-1.2695591450,0.9333893657,1.1220725775,-0.7051638365,-0.3467444777,0.9953252077,0.5555455089,-0.0913688913,0.9911448956,0.0564950071,1.1763910055,0.4268105328,-1.6847207546,0.4380587935,-0.8402453065,0.8270261288,-0.0309975259,-0.7158976197,0.2076825500,0.2148397267,-0.6354771852,-0.2590553463,1.9080600739,-2.6474463940,-1.6957372427,0.0676404461,-0.1840924770,0.0144398957,-0.7230440974,-1.4499595165,-0.3431442082,0.7012284994,1.6000750065,-1.1220747232,0.6476452947,1.2227518559,-0.6261531115,-2.0118496418,0.5587783456,0.5587535501,-0.7095311284,0.9690502882,-0.0612876080,-1.0169298649,-1.8414348364,1.4388775826,1.2759331465,-0.5176864862,-0.3065904975,-0.1054771617,-0.8933632970,1.7129597664,-1.0600179434,-1.7203254700,2.1747725010,0.5608677268,-1.0009634495,-1.6312663555,-0.1436923146,1.0822681189,-0.8980425596,-1.1708720922,0.8111722469,1.4318437576,0.7622471452,-0.1665702909,-0.7519943714,0.9354758859,0.4794349074,-0.9928976893,1.2101974487,-1.4867138863,0.6146788001,-0.2739740908,0.7458140254,-0.4276439250,-0.7129654884,1.0291323662,0.1629079729,0.5204790235,0.4500774145,0.6597102880,-2.1476511955,0.7354454994,2.0337862968,-0.4528868794,0.0470402688,-0.6466990113,-2.4321630001,0.9452065825,-0.8999962211,-0.2017476708,0.5209354758,0.1222702488,0.3367359638,-0.6224790812,2.3189260960,0.8680391312,-0.5176272988,0.5175158978,-1.4247558117,0.2031879127,-2.5270433426,-1.1400226355,1.2911773920,0.4571803808,1.5411505699,0.3775106966,1.0139513016,-0.9173130393,-2.1711902618,-0.0480994694,-1.5899727345,0.1245185956,-0.1466777176,1.1491481066,1.8068344593,-1.2763129473,0.2475799173,0.9650402069,-1.5658047199,1.2057271004,-2.1787817478,-1.9162775278,0.3622097671,-0.6148615479,2.2243676186,-1.0941650867,-0.1319183856,0.3140008748,0.1496839225,-1.5112067461,1.5638947487,-0.0783534944,1.1089277267,0.0493666902,-0.3356166184,0.2406343371,-0.2573273480,-1.6920350790,-0.3557884097,0.5366314650,0.9400575161,-0.9624430537,1.0769087076,-1.0523335934,-0.0877942294,-0.0052203424,1.4278577566,0.7861345410,-1.3564819098,0.1132832840,0.3424504399,0.5758148432,-0.7315395474,-0.2296319157,0.7997226119,-1.0046871901,-0.2958320081,1.2034623623,-1.9853528738,-0.0400415435,-0.7096756697,0.5029446483,0.3679005504,-0.6466954947,0.8429231048,-0.3528733253,-0.8913444281,2.4212992191,0.4225657880,1.5877314806,1.0010522604,0.8875393867,0.0978859067,0.9387049675,0.9202107191,-1.2897603512,2.0188601017,1.4211659431,-0.6326606870,0.3057494760,2.0222163200,0.9547148347,-0.1210186332,1.0936723948,0.4895169139,-0.4024842978,1.2918033600,-0.7272326350,0.5280922651,0.9792674184,-0.9152534604,-0.4218613207,0.7013449669,0.7105325460,0.4497777820],[-0.1501760036,0.4193355441,0.0714192092,-0.0715517104,-0.3542770147,-0.3148297369,0.2007195204,0.4526666105,1.2837107182,0.6288003922,1.4994120598,0.1824219525,0.9061366916,1.5625766516,-0.3971055150,-0.2912955880,-0.0010462973,0.6671071053,-0.7881889939,-2.7450766563,0.4060696363,-0.1217147857,-0.0527308248,-0.5595987439,-1.0663646460,-1.6144676208,-0.8451096416,-0.7483228445,-0.4292066395,-0.7167803645,0.3249592483,1.0160164833,0.5160896182,0.5963602662,1.8268053532,-0.2776781321,-0.4084135890,1.1316784620,-0.0133100701,-2.4482481480,-0.0586686917,-0.8878390193,1.3675085306,-0.0189303663,-0.3145028353,-2.3099646568,-1.7913293839,-0.4868177772,-0.9915539622,-0.8610973954,0.5070689917,-0.6919354200,0.4805471301,2.0585455894,0.2822274566,-0.6514480710,-0.1684999913,-0.9540156722,-0.5895755887,-0.4050535560,-1.8935021162,-0.2592824101,-0.5564218163,-0.5321366787,-0.3610735536,0.7515267730,-1.0464690924,0.2622892857,1.9876885414,0.3568264544,-0.7345346212,-0.0195515398,-1.3769825697,0.3296893537,-0.2662521303,0.4305160344,0.8540118933,-0.0390822478,0.3072167933,0.2258054018,0.6492428780,-2.1197884083,-0.7837410569,1.2383978367,0.4710659683,1.1728113890,-0.9507129192,0.2316634953,1.5851820707,0.9761613011,-0.6257829070,0.7709450722,-1.2106201649,-0.1576354951,2.5922064781,0.5946409106,-0.8199970126,1.3616390228,0.8881381154,0.1613833308,-0.7847581506,-0.1245018691,-0.3079818189,-0.2307329029,1.0258018970,-0.3544710279,0.4603793323,0.1158363000,-1.5349887609,0.5325800776,-0.3238184154,-0.7847681642,-1.0883129835,-0.3599952757,0.3478820324,0.8348949552,-0.1920605004,-1.6158021688,-0.4275300503,-0.9156060219,-1.5471988916,-0.0337640494,0.3297310174,0.1512339711,-0.1176083460,0.0440559611,0.0721314177,-0.0469049104,-0.5487977862,-0.6798905730,0.1742166579,-0.9439554811,0.5917685628,-0.0007143845,-0.6769761443,0.9055025578,1.2873252630,0.8089355230,0.1100778431,0.9906618595,0.5761532187,0.3763134181,1.0817846060,-0.2395380288,0.8779850602,0.1564888656,-0.3758312166,-0.9272571802,1.9166927338,-1.0327939987,0.1510976553,-0.5881170034,0.7933108807,-0.5769035220,2.0220353603,-0.7505076528,0.9200173020,0.3916193843,0.2468304783,1.6190906763,-0.0692143887,2.9150190353,0.6242098808,0.6861552596,0.3290536106,-0.0465923660,-0.1690702885,0.2150677294,0.8022250533,-0.4570151865,-0.3585347533,-0.8290402889,0.9220418334,0.3166327477,1.3787505627,-0.4238806367,-0.6832771301,-2.3926756382,1.3433904648,-0.4304220378,0.0106177842,0.0189974494,0.1315878183,0.2400119901,0.3355641663,-0.4072660506,0.9903833270,0.1759019643,1.1644349098,0.1000541449,-0.4680081308,-1.3790748119,0.3504778445,0.0912212506,-0.7095877528,0.4825436175,-1.9707536697,-1.7738105059,1.2781053782,1.0709049702,-1.1580072641,-0.0813724622,-1.3623567820,0.3525542617,-0.7098584771,-1.3302956820,-1.0732815266,0.7031227946,1.6278609037,0.3216923773,-1.6622815132,-0.5861720443,0.7162619233,0.5596861243,0.3960135877,-0.1563261449,0.7563478947,0.8782933950,0.6613606215,-0.9163061976,1.9936851263,0.7046018243,0.5298975706,-0.3517471254],[-0.2529703975,-0.6819351912,0.4511622190,-0.3695062697,0.2835831046,0.0965076685,0.9501286149,-2.5614476204,0.5790864229,0.4087923765,-1.2475892305,-1.3186043501,2.3067893982,-0.5290445685,-0.4532274306,-1.4219975471,0.4242465794,-1.1147121191,1.1617943048,1.3607591391,-0.1647880673,-0.6505087614,1.3001717329,-0.4226404727,0.0101207495,0.8180606365,-0.1478663832,-1.1388643980,-0.9242997169,1.6581678391,0.6156594157,-1.3556060791,1.0397180319,0.0057428153,-1.0736074448,-0.1493596137,0.0751666129,0.4852134883,-0.0637846664,1.2028560638,0.0310445186,-0.1631756276,0.5054315925,-0.6068215966,-0.7260636687,0.9337012172,0.4490439594,0.9612168074,-0.4713724256,0.0757001787,-1.2501882315,0.0985274091,-0.0683198124,1.4502435923,0.1849096119,-0.7560486197,-1.8285669088,0.6093274951,0.0586636290,0.2028598487,0.6563054323,-0.5652397871,2.2766592503,-0.9085285068,0.6356878877,0.5116490126,-1.5929365158,0.1327019781,0.1812831610,1.7507266998,0.9238305688,1.8036668301,0.2698839903,-0.1360591203,-1.8491562605,-1.5269157887,-1.4091269970,-0.4059423804,1.1683697701,0.3950791955,-0.3730430007,0.9432966709,-0.4507664144,2.1472918987,0.4739236236,1.6310644150,0.4206609130,-1.5757130384,-0.9026260376,1.4252194166,1.0862905979,-0.5142824650,0.9228453636,-0.9622536302,0.5519937873,-0.0327038616,1.2062320709,-0.1808272600,-1.4194060564,0.9631287456,-0.6597050428,-1.0749583244,1.4704674482,1.8247721195,-0.7203150988,-1.0254713297,1.4141848087,3.0542407036,-1.0799494982,-0.6734762192,-0.1547389776,0.8099894524,-0.0583977439,-1.0075556040,-1.7363808155,-0.2186141759,2.6054716110,-0.3520674407,-1.4125469923,-0.9866930842,-0.3136315644,0.2758622169,-0.0382532291,-0.2909025848,-1.2334349155,-0.6941742301,0.4460225999,-0.3344380558,0.2541289926,-0.7754428387,0.6982194781,-0.1970705390,-0.4232169390,-0.2842783928,-2.1254146099,-1.6987265348,0.8912936449,0.0369419828,-0.4753980041,0.9474012852,-1.1694549322,0.0957721621,-1.9647138119,0.8450680971,-1.0638986826,-1.7959105968,1.0335805416,-1.0350733995,1.3430973291,0.5613515377,-1.3858976364,0.2772824168,0.2037287652,-0.4732226133,-0.9405746460,0.7366333008,1.3613336086,0.0208984539,0.4093787372,0.0871815830,-2.0470230579,-0.6185591817,0.5369277000,-0.9558705091,0.4795728326,-1.9930492640,1.2474435568,0.0068958555,1.5790319443,1.0980535746,-0.0637846440,-0.5902965665,-0.0061245328,2.0637385845,-0.5407064557,-0.8410700560,-0.3185648620,-0.3438983560,-2.0251550674,0.1584590971,-0.4653836489,0.4902653098,1.1586868763,-0.6380576491,-0.3382273018,0.1718517691,1.8048999310,-0.6133977771,1.1827590466,-0.4546300471,0.5546512604,-1.0758531094,0.3262319863,-1.6125426292,1.1405082941,0.9962644577,-0.3908626735,0.2542061508,-0.5308600664,0.8220796585,1.4729973078,0.6980400681,1.9615049362,1.2876982689,0.2182829082,2.1297807693,-1.1058113575,-0.8929600120,-1.6870913506,-0.9594828486,1.6345727444,-0.2234487683,0.2769846916,-0.4827255011,0.6188679338,-0.7529045343,-1.8513833284,-0.5449662209,-0.2289349586,-0.2275588512,0.1584722102,-0.9139913321,-0.4197747111,0.9362422228],[0.8670089245,-0.0218320917,-0.1137257963,0.8240271807,-1.0934950113,-0.1945286244,0.8003533483,-0.2396968752,1.1544240713,0.1843745708,1.8148233891,-1.2879664898,-0.2209738046,0.8045834899,-0.0956563428,-0.5216675401,0.0386822708,-0.0084566763,0.7532719970,-0.5548394322,-1.2596553564,0.6709458232,0.5227302909,0.3640642464,-0.4185277820,-0.2747395933,-1.3535087109,-0.7166903019,-0.4688091278,-0.7064665556,0.1297899634,0.4820433259,1.6228344440,-0.0645888969,1.1060124636,1.3474090099,-0.0167565756,0.1718845665,-0.7142949700,-1.5984379053,-1.3632293940,0.4917477965,0.2064529210,0.7901336551,0.8137999773,0.7803962827,-1.1908946037,0.7658921480,0.2878961563,-0.4552826583,-0.0192807633,0.2585481405,-0.1401580274,0.4660041928,-0.2457492054,-1.0530674458,-0.1534679681,-1.4978170395,0.5043719411,0.1542381793,-0.5338128805,-0.3461004198,0.1491215080,0.7485570908,-0.0845168531,-0.2989161909,-1.5050033331,-0.2240383327,0.1300648153,0.3475236297,-0.1500207782,0.6453188062,0.0438788645,0.6376432776,-0.5648962855,0.6896584034,2.0232799053,-1.0980631113,-0.1578527689,1.0964521170,0.6350690722,1.6230158806,-1.2539564371,-1.8732732534,0.2195006907,-1.4115343094,-0.3199668527,0.5239178538,-1.4737608433,-0.8933182955,-1.3867191076,0.1345203072,-1.0216890574,0.9537410736,-0.4129306376,-0.2612096369,-2.0409665108,-0.0334131494,-0.0334027857,0.9169976711,-0.7904871702,1.2259902954,1.1751644611,-0.4285097718,0.5169038773,-1.4336477518,-0.8526900411,0.4836533964,0.5627523661,1.2323125601,1.4806730747,0.8469880223,0.4425852597,-0.9853484035,0.0293111708,-1.5114089251,0.9939314127,-0.5136312246,1.8423138857,-0.3224902153,1.3486384153,0.1608252227,2.3281126022,0.3682502508,0.8500390053,-1.2086205482,1.6425739527,1.6567891836,2.5163788795,-0.5516718030,-0.7090018392,0.5536077619,0.1044385284,0.5848913193,-0.5738294125,-1.1334881783,0.4964465499,-0.4429698884,-0.5444251299,-0.0184476022,1.0488680601,-1.3786221743,-1.3408986330,-0.8560641408,-2.0122303963,2.1372659206,0.5076947212,-0.9105145335,1.0034862757,-0.5493667722,0.8107016683,0.5115770698,1.4349923134,-0.2621361017,-0.8416423202,-0.3985565901,1.4450401068,-0.5568643212,-1.9461606741,0.1166291758,-0.0909571052,0.7195361853,0.2246374786,-0.6057894230,-1.2935698032,-0.3281378448,0.0623177253,2.6832988262,-0.9763700366,-0.1302705854,-0.4990547597,0.0406178869,-0.7492877841,0.0462367646,0.1370660067,0.9836137891,-1.3005661964,-0.3430009186,-1.6979012489,-0.6472293139,-0.5646333098,0.3106422722,-0.9889423251,-1.7156691551,0.6313973069,-1.9979158640,0.3836011887,-0.3882793784,-1.7840440273,0.7430712581,0.7994744778,-1.2537626028,1.3493068218,-1.0350950956,-1.1018353701,-1.8689471483,-1.4032249451,0.1971910149,-0.4794034660,0.2162466347,1.1728647947,0.2414074838,-1.8751955032,-0.6058746576,0.3895087242,-0.3630200624,-0.7610980868,-1.3947130442,0.3746354878,-0.7805181742,-0.8008945584,-1.1045049429,-0.8625071049,0.2624949515,-0.8144610524,-1.1403104067,-1.4325137138,-1.2969572544,1.1078927517,1.2386733294,-1.0097072124,0.4666787088,-1.9054232836,-0.3379438221],[-0.9478093982,-0.1815164536,-2.1254286766,0.3806206882,0.3613242805,-0.3807277381,0.6509169936,-1.1140364408,-1.6195075512,0.1372729540,0.8177353144,-1.6663779020,1.6164512634,0.5298832655,-0.6429971457,0.4450341761,-0.3613208830,0.4252066314,-0.3191341460,-0.6303688884,-0.6494750381,0.3941224813,0.7792255282,-0.7178655267,0.5349315405,-0.3798153996,-0.3439167440,0.6115089655,0.8245086074,0.1483690888,0.8659130931,-0.4422810674,0.2366063893,-0.8263459802,-0.2544108331,0.7105622292,1.2936662436,-0.0061172731,-1.3991149664,-0.2684777379,0.5478208065,-0.5861970782,1.2108368874,1.2831014395,2.4623322487,1.7803323269,0.0805140138,-0.1960275471,0.0362248681,-0.1431263834,0.4654183090,-0.7120475769,-1.4517489672,-0.3825183213,0.3767770827,-0.2440060675,0.2647753358,-0.5264542103,-0.1851858348,-0.5366991758,1.2142837048,0.0382215828,0.2983546853,-1.3661375046,0.6581992507,-0.3689630628,-1.1523690224,0.2207235843,-0.3588210046,-0.5084909201,0.4237173200,-0.2514657378,-1.4737418890,0.3986872733,1.4346228838,-0.6114665866,-0.0471816361,0.7091511488,-1.2037036419,-1.2475736141,-0.2429477423,0.0104824426,0.9887840152,-0.5232872367,-0.6540614367,2.9717705250,-0.8098665476,-1.2220925093,0.6366750002,0.3437053561,0.2384938598,-1.5848807096,0.1682298779,0.7390740514,-1.1523597240,0.5150222182,0.2875719666,-0.3649416268,0.5651715398,-1.4896733761,0.8772756457,-1.3336732388,-0.7780230045,1.8497377634,0.2610572875,0.7758993506,0.0206704233,-1.0282015800,3.1966378689,0.2818923295,-1.2109206915,-0.8321417570,-1.3174719810,1.2380001545,-0.2314254940,-0.8635729551,-0.1830273271,-0.8175476193,-0.4133780003,-1.5508835316,0.1624538600,-1.0400195122,-0.2823365629,-0.7531560659,-2.6896286011,-2.2430329323,-0.0545572713,0.1620842814,-0.1392417997,-1.0881642103,-0.2334382683,1.0073410273,0.9778640866,0.0852946490,-0.3693783879,-1.1538172960,0.9188566208,-1.2099559307,-0.6782140732,-0.1138747409,-0.4954129457,-0.7024160624,0.1558332443,-0.1858649552,1.5633962154,0.2435840368,0.2163445055,-1.9303464890,-0.4743229151,-0.9498266578,0.3237558603,-0.2460615784,-0.4553605020,-0.7298208475,-0.3582246900,2.1175317764,-0.8765597939,0.5634529591,0.0625522062,-0.9692933559,1.1842176914,0.4230535030,0.4874381721,-0.5687189102,1.8937847614,-0.1945808977,1.4794397354,-1.3262608051,-0.8606801033,0.2191320211,0.7188069820,0.0009182103,0.6576800942,0.8063944578,0.8810168505,-0.5035043359,-0.7760851979,0.1666544825,0.1153337806,-0.8356602192,-0.5901991129,-0.3215951622,-0.9975553751,-0.6187186241,0.2700315416,1.2824653387,-0.1581376642,-2.1533644199,-0.7170999646,-0.6648142934,0.2680378854,-0.4821653366,0.0698350668,0.6788092852,1.2517094612,0.0046356483,0.0086138453,-0.6417683959,-2.4156284332,-0.4155599475,-0.6338680983,0.7500723004,-0.5130292773,0.5746928453,-0.1184019148,-0.2648290098,-2.0818395615,-0.7137652040,-0.9739657044,0.7537212372,0.2535819113,0.6169744730,-1.4554769993,0.3506465554,0.7433042526,1.3686823845,1.4085261822,0.9369397163,-1.1510248184,-0.8294801116,-3.0858607292,1.1538147926,-1.0315407515,0.4071751833],[-2.3339128494,-2.4373297691,-1.5677690506,0.5979134440,0.9495761991,-1.1302498579,1.5698679686,-0.0112411575,-0.4865964055,-0.6322322488,2.2198269367,-0.1834526360,-1.1265172958,1.3004908562,-0.8718284965,-1.0175803900,0.1230721474,1.2845140696,-0.3991899788,-1.1820335388,-1.5773611069,-1.8915426731,1.0482774973,0.4631630778,0.5453273654,-1.2216370106,-1.2610167265,0.3723307848,-0.7056763768,0.5499272943,-1.4075728655,-0.2471286952,-0.1996609718,-0.6414245367,-2.0822985172,1.0572780371,-0.2430976778,0.9677157402,0.2788408399,-0.0577941090,0.6746733189,-0.7687827945,0.1710018367,0.2308249325,-1.4008908272,-1.5655425787,-2.2899365425,1.1975847483,0.7657328844,-1.6059250832,0.6931754947,0.6015011668,-1.5637310743,-0.3585254550,1.7346768379,-0.4272238016,0.6572923064,0.7208228707,0.4170411229,1.6985064745,-0.6069838405,0.1985742748,1.3401014805,0.1548866481,1.1064743996,1.3651067019,-1.2938311100,0.1128372401,0.4152404964,1.8631957769,0.6998057961,-0.7854279280,0.7744868994,1.1030099392,-0.6815320849,-0.6766694188,0.7262734175,0.1705344319,1.5088578463,-1.4776148796,-0.0586480461,1.1006953716,0.7420361638,1.6587747335,0.1205245256,-0.0020093988,2.2212829590,1.0334048271,0.7167780995,0.7675334811,-0.4229756892,-1.8542928696,-0.1795200706,-0.3950515389,-0.1793856621,0.2677515745,-0.2145903558,0.9883928895,0.6095905304,0.8403566480,-2.7272720337,-2.5840220451,0.4287470877,-0.8445891142,0.5398139358,0.1210710034,0.9378256798,-0.3317985833,-1.0996099710,0.0415950157,-0.3239310086,0.3395366967,0.6336280704,0.7191601396,-0.7759758234,0.1708994955,3.4803307056,-0.9206820726,-0.4048551321,-1.0755497217,-0.8828337789,-0.5957428217,-2.1283802986,-0.5010408759,0.4380800128,0.1451422721,0.0548269637,0.7946581244,1.5738191605,0.4238313138,1.1378115416,-2.3402349949,0.2621157765,-0.5924669504,0.6416893601,-0.0999486968,0.0793482810,1.2093353271,0.6182104349,-0.1222885475,1.1483215094,-1.2530092001,0.8336417675,-0.5853553414,-0.8618545532,-1.0123625994,-0.2709977627,1.1808912754,1.0666790009,-0.5410199761,0.2418011427,-0.0650454462,-1.3687447309,0.6643924713,1.2235996723,-1.2306170464,-0.3710289299,-2.3630490303,-0.0823582411,-1.3592556715,-0.0260777790,0.4334647059,0.2359000891,-1.6590589285,-0.6708953977,1.0374119282,0.5894627571,0.1390295923,0.4612559378,-1.1581498384,-0.6379216313,-0.5233878493,0.2453523725,-0.6686667204,-0.2729541361,0.0582876019,-0.1641405523,2.6693544388,-0.7174984813,1.1088682413,0.5329573154,0.0444557518,-0.5908399820,1.7005236149,-0.1409078687,-1.1669863462,0.6829881668,-0.1745918244,0.2864745855,-1.8335051537,0.4798454344,0.3556045294,0.5267731547,0.4601722360,0.5871078372,-0.0579544082,-1.0574859381,0.9699077010,-0.1474561393,0.9218323827,-1.1894884109,-0.6066743135,-0.8240451217,0.6136225462,0.6406924725,-1.2699052095,-0.7971712947,0.7746338248,-0.9440367222,-0.0133051872,-0.3265692890,-1.1451457739,-2.6992063522,-1.1471670866,1.6526821852,-0.1636654586,-0.7557908297,1.0712401867,0.9058020711,0.6016139388,-1.8284126520,1.7088695765,-0.1438959986,0.9743440151],[1.1755664349,-0.6853446364,-0.9805140495,-0.7326768041,0.6781848669,-0.9718044400,-1.4187847376,1.5915788412,1.6385931969,0.6819905043,0.9449669123,-2.1971836090,0.6113591194,-1.1820850372,-0.3689990342,1.5513277054,-0.5906776786,-0.4767121673,0.8203312755,1.0888025761,1.1581448317,-1.5558147430,-1.1395241022,0.8870739341,0.1858982444,0.2697058022,-0.9096365571,0.3069518507,1.2159620523,-0.2464475781,0.3862953186,-0.3224743903,0.2444050610,-0.6824684143,0.0999120772,-0.6802471280,-1.1838104725,0.5632140040,0.6371393204,1.0253103971,0.3471388221,-1.4211109877,-0.3159162104,-0.5746237636,-0.1463756412,0.5199471116,0.4381284118,-0.9439998269,-1.1170305014,-1.6122260094,1.5769633055,1.3907139301,1.1999559402,1.3022366762,-0.6340661645,-0.0995960012,-1.6399950981,-0.6119499803,0.1380764097,-0.6237627864,1.1016682386,-1.9356443882,0.5634610653,0.4649307132,0.5689449906,-0.6906414628,1.0428044796,-0.9816251993,-0.0944080576,0.9025543928,-0.9932280779,0.7247273922,-1.6060585976,0.6052199006,-0.0746025890,0.2692794502,0.0836235955,-0.6294842362,-0.2354930639,-0.3489116728,1.4909719229,2.2565157413,1.1001173258,0.1191658676,0.2973752916,-0.6802826524,0.4829027653,0.6770628691,-0.8792845607,-0.5022274852,0.4680658877,0.0524122715,1.2018536329,1.2017225027,0.2157837749,0.6835497022,-0.5031975508,-0.8575496674,-0.7981103063,-1.3048182726,-1.0906238556,-2.1067719460,0.2701582015,-0.1355769187,0.1928640008,0.7015531063,-0.2581508756,0.0451877564,0.8290317655,2.4921574593,0.6755216122,0.6239854097,1.9080717564,-0.2598963976,2.0262806416,0.6672372222,0.1927894354,-0.5935207009,1.2906221151,1.1303480864,1.8212866783,0.1627627015,-0.7681046128,-0.2628323734,-0.1882967800,-0.4641387463,1.1165431738,0.0123880208,1.1824625731,-0.5635791421,0.4094505906,0.5499470234,0.6730532646,1.1384918690,-0.0318734385,0.5249695778,0.9564266801,-0.1829735190,-0.8484261036,0.1849524081,-0.6116765738,0.4040069580,1.1256523132,1.3207218647,2.0485391617,0.2022244185,1.0597351789,1.2315956354,0.3742661178,1.4664480686,1.0261442661,0.9445564151,-0.7210229635,0.3330769539,1.1520477533,0.0342893526,-0.1105533689,-1.1973357201,-0.3417419195,0.8220157623,0.7772755027,-1.5686694384,1.6022686958,-1.6342042685,0.9400302172,-0.4801312387,0.1337077469,-0.0188103653,0.0742905065,-0.7465792298,-1.0140510798,1.7433282137,-0.2325217575,-0.2834181190,-0.8204702735,-1.2446351051,-0.2062532604,0.8481276631,0.8852192163,-0.8912778497,0.3083090186,-1.8603940010,0.3590204418,-0.1462968290,0.8163167238,1.3781898022,-0.5198218822,-0.6847628951,-0.8724231124,-0.2871099412,0.0967746899,-0.0191583131,0.5907481313,-1.5904307365,1.3970313072,-0.2175879031,0.7887555361,0.8525711894,0.9093270302,1.1224218607,-2.1383500099,0.8649312258,0.8823870420,-1.1048812866,-0.9562670588,-0.8385741711,-1.4619178772,-0.1579514742,-0.2121369094,1.2227568626,0.5742275119,0.0359639078,-4.6599531174,-0.0978555605,0.8772867918,0.9253914952,-1.5906937122,0.3186887503,0.7967299223,-1.0925660133,-0.1356026530,-2.7123830318,0.7223868966,-1.3690061569],[-1.2025033236,0.7792565823,-1.1973105669,1.7177031040,0.0948203802,2.5072460175,-1.5663551092,-0.6132227182,0.1630306989,0.5378769636,0.5238105655,0.2860624492,-1.1264125109,0.1445035040,-0.5489215255,-0.3247438967,-1.3651691675,-0.6719406247,-0.3707260489,-0.8329691887,1.3739992380,1.4054895639,0.6142627597,0.3801291585,-1.9824923277,0.5518612266,-0.9396815300,-0.4278084040,0.0637271479,-0.8120394945,-0.5692890882,0.9053315520,-1.4505615234,0.7103596926,-1.2559800148,0.0491107069,2.4521374702,-0.9200552106,0.2403309941,0.0198677108,1.2498037815,1.2705478668,-0.3080240786,0.8807117343,-1.7154098749,-0.2565684021,0.1640652120,-1.4016714096,0.7097941637,1.9095629454,0.9995204806,-0.4326931834,-0.6567226648,0.1821094900,1.0034320354,-0.3981125355,2.0536632538,1.8618398905,1.3602993488,-0.4498952627,2.0644013882,0.9212067723,-0.2571714818,-0.3730241656,0.8073030710,1.3182489872,0.1169435158,1.3609436750,-0.1411994398,-0.1029899791,-1.9461628199,0.4719499946,-1.4018038511,1.0566523075,2.1330528259,-0.7859642506,0.0822543800,-1.3732144833,1.7473028898,-1.0639010668,-1.2659695148,0.2229897678,0.4247764647,0.4805686474,-0.8474889398,-0.3383961320,0.8022504449,-0.5559909344,-0.4041362405,-0.5135216713,-1.3163421154,-0.1205931008,1.5122067928,-0.0104675554,-0.2192304283,-0.1263844669,-1.1371041536,-0.5089555979,-0.1946117580,0.8600028753,0.0638399795,-3.5213615894,1.0045546293,0.5567478538,-0.6885315180,-1.4055252075,-0.3676231802,-2.9203844070,-1.1243324280,-0.3046654463,-0.7170249820,-0.6134189367,1.1806969643,-0.0221272781,1.1801662445,0.3245626092,-0.2887108624,-2.0269603729,-1.4936410189,0.1252851188,0.5235942602,0.7694662213,0.1733997315,1.3879876137,0.9336732030,-0.2111702561,-1.0503019094,-2.0826382637,1.4609395266,-0.0856881514,-1.7071745396,-1.1867042780,2.1376116276,0.8348627687,0.2069842666,1.7539217472,0.3535583615,-1.8743047714,-0.0553962588,0.8481658697,-0.9995617270,0.3901985884,0.1343593001,1.6648913622,1.2137875557,0.3217611909,1.4884146452,0.1555677801,1.0586683750,-0.0288448110,-1.7163136005,-0.4568204582,0.6596849561,0.3663463891,-1.0010029078,0.6994525790,-1.2128429413,-0.5362828374,1.0057533979,-1.0551908016,-0.5448315740,-0.3002343476,0.2546850741,-0.2694426477,-1.7030316591,-0.8511922359,0.4108665884,0.8940765262,0.4596008062,-2.0727031231,-1.5894145966,0.6068012118,-1.3510271311,0.8960603476,-0.6474032998,0.2486500740,-1.2714422941,-0.0038882706,-1.2690234184,-0.7396576405,-0.0264970679,-0.3070549965,0.3133223951,-0.0787381977,0.8041164279,0.2555361092,-1.9416064024,-0.9758633375,1.1594858170,0.6093669534,0.9097701311,-1.1062548161,0.8592962623,-0.2846635282,0.6063728333,-0.2664417028,0.3516781628,-0.0238830093,0.6255849004,-0.7902545333,-2.0264768600,-1.1567298174,0.1229565144,0.0750426725,0.3840484023,0.6105062366,1.4286056757,1.5390007496,0.0215927530,-0.6384230852,-0.0326580927,-0.4754877985,-0.0593326017,0.2306943089,-1.5714039803,0.2333770394,1.1175312996,-0.4276227057,-0.2300469875,-0.7984775901,0.5063858032,0.7095662355,0.0707808509,0.3261324763],[-0.5166196227,0.7755348086,1.4209582806,0.4472828507,1.5131568909,-0.5091314912,-0.1463637352,-1.2307108641,0.8167770505,-0.8237021565,1.6269707680,-1.3415946960,0.0487488285,-0.8004634380,-0.6269339323,1.8451014757,0.4949062467,-1.3215615749,0.6344664693,0.2089666575,-0.4782479703,0.3076966107,-1.6594357491,1.3730232716,-0.2208669931,-0.5210494995,0.2851623893,0.8996701241,-2.5622427464,-0.0674514249,0.0234192368,1.0144755840,-0.9546979070,0.8461613655,-0.0649054199,-0.6372078061,1.3974251747,0.3746976852,0.7681015730,0.2594341636,1.7057238817,0.1613429636,0.0854009017,-0.8134281635,0.9436557889,-0.9423817396,0.5801091790,0.1318775266,-0.2848078012,0.1335559636,0.5876556635,-0.3178980350,-0.0128314896,0.0865386352,0.3319094777,-0.2248116732,0.0240327138,0.6187692881,0.3574542105,0.9088279009,0.0697148070,0.4406653047,-0.1967163235,-0.4700434208,-2.1740925312,-0.3500843346,0.1516977102,0.2233592272,0.2852753103,0.6240082383,0.1125972867,0.3403343856,-1.4563639164,0.3733136654,2.1346266270,-0.9266167879,0.3932163715,-2.0109872818,-0.0350480564,0.5640030503,-0.3578226268,0.1307384968,-0.2989596725,-0.7109010220,1.3044710159,0.4014618695,-0.5151721239,-0.0912843123,-0.8773889542,0.3478965759,-1.0700992346,-1.2515821457,0.6234746575,0.5236243606,-1.6252604723,-1.2834419012,-1.0935277939,1.0660840273,1.4242330790,0.1721225530,-0.4695689976,0.3664133847,1.7772021294,-0.9540381432,1.0367292166,-2.3827981949,-0.3921341002,-0.5850293636,-0.2167987227,1.3427859545,0.6053371429,-0.6285265088,0.3417230546,1.3946480751,-0.3851041198,-0.5210492015,-0.8341775537,1.8781659603,0.4993408620,0.9196839929,1.4708603621,-1.3287057877,-1.8750246763,-0.6538562179,-0.4436583817,0.3147097826,-0.2363654673,-1.1563998461,-0.7322943807,-0.8094955087,0.1966855228,1.8921687603,0.1011165977,1.0714277029,-1.1122522354,-0.8491483927,1.0026543140,-1.8699041605,0.4910463989,-1.3007510900,-0.7558706999,-1.2062203884,-1.3015846014,2.2981455326,-0.3842036724,-0.4086307585,-0.5081439614,1.0238132477,-0.0232919119,-0.7040374875,-0.6754306555,-0.5368385911,0.4415324926,-0.2231244147,1.5856546164,0.3726929724,-0.8339967728,-0.3401282132,0.4355895817,0.4122020900,0.5042055845,-1.2808250189,-0.3089379668,0.2095138282,-0.9887799621,0.5872370601,-0.4417540729,-1.7355332375,1.5637019873,1.1506174803,0.0401921943,0.8149108887,1.6987164021,-1.0433235168,-0.4187589586,0.3610968888,-1.5596761703,-0.3735758960,0.2021843344,0.6289901733,0.8797625899,0.5344867110,0.1503546983,-0.7813857198,0.2976354361,0.7332540154,-0.7058255076,0.0368137881,0.3777216375,-0.5616474748,0.0333609879,0.7632266283,-1.1654425859,-0.4225851893,-0.4182069302,-1.7396740913,1.2583700418,-1.4786802530,0.2079833299,-0.6789172888,2.0099575520,0.2491786927,-1.5300163031,-0.3059594631,-1.3193536997,1.9566799402,0.6859662533,0.2039905638,0.1241332442,-0.0059582829,0.4895803034,-2.1932377815,0.4535388052,-1.3756812811,-0.7140263915,0.5834588408,-0.4265936911,-0.7711508870,-0.0634818077,-2.0736818314,0.4877722561,0.7690168023,-2.2140629292,-0.1235852242],[-0.7416536808,1.1636881828,0.7511378527,-0.1915588677,-0.6453274488,-0.4290712178,-0.7228347063,-1.0105632544,-0.5159531832,-0.4688208699,0.6070471406,0.2960442603,0.7811290622,1.2193843126,-0.7364284992,-0.5761831999,0.0020575896,-0.4753360450,-1.2325395346,2.1083066463,0.3523429036,-0.9894808531,0.4212930799,-0.3872063458,-0.7398980260,0.6265519857,1.1915148497,0.5910297632,0.9382076859,-1.6488192081,0.9358170629,-0.0788703635,-0.7320182323,0.4131195545,-0.6173188090,-0.5903555155,-0.2789388299,-0.1472140253,-1.2764873505,-0.1217904240,1.2791328430,-0.6618398428,0.6625029445,0.5320830345,-0.4551331699,-0.5017670989,1.8236455917,0.6914116740,0.3026286960,0.1815621406,0.2498879433,-1.2506207228,-0.0442353934,-0.1216291115,0.5298925638,-1.5443810225,-0.4020614624,-0.0662033781,-0.0818239599,-1.4185363054,-1.3683942556,-0.0888589248,1.7211959362,-0.3998090625,0.0440240912,0.6941738129,-0.0163219403,-0.2769297659,0.3046324849,-0.2503691316,-0.3313120604,0.2744653523,-1.2622573376,-0.2323275506,0.0178010147,-0.8282309175,1.0964192152,0.2451598942,1.5029834509,-1.4286749363,0.5833849907,0.5235205889,1.7843471766,0.3572329283,-1.2430522442,-0.5951647162,1.7783439159,-0.9175342321,1.5678970814,1.7504197359,1.7831438780,-0.7279756069,0.4373401701,-1.2401688099,0.0116553763,0.0059235734,-1.5822329521,0.4546556175,-1.1685270071,0.2259088755,-0.5036152601,-0.4659640491,0.0927771181,2.4530568123,0.6292487979,1.6537941694,0.3002971411,0.3096996248,-0.6645023823,-1.5523566008,-0.4031520486,-1.6061965227,2.7819669247,-1.0583498478,-0.1395788193,-0.1203634888,0.1022626758,0.8720859885,1.4830600023,-0.4962566197,1.0328984261,0.3373303413,2.3986284733,0.1229605749,0.6004591584,-0.7395074964,1.4079188108,-0.0688579455,-0.2888651490,0.0585243143,0.7603533268,-0.7401866913,-0.6543632150,1.1241512299,1.3855564594,0.2501965463,-0.0244971812,0.7574801445,1.2781273127,1.5284435749,0.6185496449,0.5040275455,-0.2804699838,0.1216544211,-0.6845163703,0.7941741347,-0.0366092920,-0.6983697414,-1.8522673845,0.2542896867,-3.2831485271,0.3953566849,-0.5015579462,0.6498671174,-1.1276022196,1.1505984068,0.1903680265,-1.2231880426,-0.6573792100,-0.4833384156,-0.4638567567,1.3860788345,-0.6521995068,0.1385852545,0.3805219531,0.1140052676,-2.4107685089,0.8874932528,1.6550575495,-0.3197214603,-1.3988662958,0.4590135515,-0.2101721168,0.4094855785,-0.9342734218,0.6102930903,-0.0219919235,0.8110104799,-0.1327028722,-0.5929984450,0.0896767303,-1.4792257547,-0.6357883811,-0.1910596639,-2.4750189781,-0.5322038531,0.5241442919,-0.6994981766,0.8529137373,-0.4903007746,-0.8817639351,-1.3482192755,-0.0317793489,1.2985696793,-0.3465938270,-0.5808402896,0.0075849593,-0.7275836468,0.8141966462,0.4521128833,0.2785006464,1.8277621269,-0.5190520287,-0.8914665580,0.2689761817,-0.9662404656,-0.6470789313,0.2914797664,-0.8155297637,-1.3739093542,-2.3848102093,-1.1209721565,-0.8182407618,-0.1182447374,-0.3398118019,0.4466997087,0.2464601994,1.0585348606,-0.2261178344,-1.3365659714,-2.1779890060,-2.3538610935,0.9866024256,0.8527196050],[-1.0528454781,-0.8125745654,-0.3296542466,-1.1564801931,0.7735942602,0.9443894625,-1.1332502365,0.3197281063,0.2241103351,-1.9744640589,-0.9417819977,-0.2481104732,0.3611280024,0.8934960365,-1.1568579674,-0.5745681524,0.0709185824,-0.2200739235,-1.0778328180,-0.1161477938,-0.0324398316,-0.4999149740,-1.7372184992,-0.0684553757,-0.8528540730,-0.5410062075,1.1508429050,-0.1721008271,0.1344915330,-0.5637725592,-0.3995460272,-0.6501408219,0.9465684295,1.9391757250,-0.5684672594,1.5173685551,-0.6282674074,-0.9417642951,0.0144041413,0.0889731869,-0.2330531925,0.2974446118,1.2360913754,0.1447630525,-0.7290937901,-0.7561873198,1.1870754957,0.2414626628,0.2983992696,0.7458506227,2.1857104301,0.8399736285,0.7549082041,0.7326828837,-0.0318742953,-0.6576696038,0.0198269822,-0.6515598297,-0.4805519581,0.6991463900,2.2100965977,-1.7864370346,-0.6630361676,-1.1771378517,-0.6534810662,0.4054700434,-0.6601068974,-0.7464119792,0.2744610012,1.6158492565,-1.7060116529,0.2919547558,0.1256903559,-0.7018199563,0.2004989535,0.3009130657,0.8931428790,-1.2528479099,-0.9481801987,0.7718013525,-0.8815566301,-1.0193971395,0.0920027643,-0.7858209014,0.9181225896,-0.7570737600,-0.0542134568,1.0911778212,-1.1670839787,-0.3656161427,0.6322442293,2.5866792202,-0.9024486542,-0.1006447151,0.0434250124,-0.6026551723,-0.7511234283,-2.1465878487,-1.3147332668,-1.0441503525,1.6845152378,-0.9341364503,-0.3411107957,-1.7455786467,-0.5735567212,-0.3543936908,0.8459254503,-0.0149782002,1.0783048868,-1.4044744968,-0.5981349349,0.7644030452,-0.2567512095,-0.4749291241,0.3467444479,-0.0457928218,0.3171402514,-0.3241792321,-2.8057308197,0.9024006128,-0.5445762277,0.9712939262,0.4558846056,-0.1432908773,0.1366594285,0.5268583298,0.4409561157,0.3136016130,0.3074067235,-0.5648976564,-0.8228408098,-0.9566798806,-0.9248560071,-0.2933781147,1.1406354904,0.9162472486,-0.0694442838,-1.0854730606,0.4244476855,-1.0796084404,-1.1030313969,1.4148253202,-0.4556069970,1.4371075630,0.0990088806,-0.7168421745,-1.4504446983,-0.2324653566,0.7561825514,0.1022730097,0.6021108031,-1.4685246944,-1.6902762651,-0.9539712071,-2.0895330906,0.4726255536,-0.4029881656,0.0007847449,-0.1372971684,-0.0785327554,-0.7824497223,0.3144697845,0.9834409952,-1.3111101389,0.1794102639,-0.5454942584,-1.2275632620,-1.1055543423,0.1894212812,-1.2745862007,0.3700563610,-0.0895574987,0.1387081742,-0.1900572032,-0.8278012872,-0.2102169394,-1.1714949608,1.8205448389,0.1352094561,1.0171357393,-0.7764116526,-0.9255347848,0.3677875102,-0.1336479634,1.0903134346,-0.5522167683,-0.1727647781,2.1905345917,0.0677361563,0.0585403368,-0.5572353601,-2.4835076332,-1.2631917000,-0.9267233014,-0.3717812896,-0.2926731408,0.0466216952,-0.7515469790,-0.5927543640,1.4340451956,0.3440235853,-0.4755963087,0.3029372394,0.4228192568,0.8591195941,0.9187337160,-0.9696948528,-0.8918405771,-0.7458072305,-0.3890754282,-0.7237933278,0.2729220390,0.5538294315,1.1950474977,-0.1350257993,-0.0628644973,-1.0330879688,0.8075287938,0.4006544948,-1.5620900393,-0.2522285283,-0.7478344440,-0.7600804567,-2.5066511631],[0.2466514409,0.6960947514,0.8043972850,-0.7229521275,1.8516392708,-0.5131801367,1.9044151306,0.7484899163,0.3236445189,-0.2882115245,0.7676124573,-0.3992791474,1.0807728767,-0.6194197536,0.1106903628,1.4164921045,-0.4014247060,-0.8979519010,1.2561978102,0.4640975296,1.9639859200,-1.8645833731,0.4402790070,0.2453724444,-0.8920995593,-0.2342789471,0.9694778919,-1.5542093515,0.4818401933,1.1038746834,0.0457208902,-0.3195508420,-0.1857536137,-0.5587615967,0.2741684318,1.5274512768,-3.0374977589,-0.5423014760,-0.0150344083,-0.6761445403,-1.0511331558,-0.4787085354,0.2322977632,0.5701154470,2.6077516079,1.5823068619,-0.4669645429,0.9781920910,-0.9199672341,-0.1703516096,0.4301041663,0.7154524922,0.4585685730,-0.8489269614,1.4372155666,0.8874803185,-0.1794517487,-0.0144406203,1.2646210194,-2.9158418179,0.4403441846,-1.3688344955,0.8333687186,0.5244902968,0.2989941537,0.9304910898,0.6709625721,0.9794834852,1.3098096848,-1.1596137285,-0.8816054463,-0.5656709075,0.9917655587,0.4784300923,-0.5290926695,-1.1244556904,0.3402399421,-0.8361654282,1.4321713448,0.8068723083,2.2013251781,-0.1939378530,-0.4361837804,0.3527769744,-0.8224888444,0.2225948572,-0.0207322240,0.1455113739,-1.2164512873,0.0929658487,-1.1042196751,1.4114979506,-0.6173691750,1.9363077879,1.2560794353,-2.7780065536,0.7533286214,1.3619881868,1.1684308052,-0.8395903707,1.9072697163,0.4078255296,-0.2256548852,-0.5843978524,0.0805882514,1.6368948221,1.3414212465,-0.6055875421,-1.0454299450,1.3621159792,-0.2686824799,-0.0739303008,0.5343942642,-1.3068132401,0.9384565353,-0.3994387090,0.4542995989,0.7322161198,0.7305169106,-0.5614368320,1.1140637398,0.6526744962,-0.0719099790,0.9098330736,1.7640006542,0.4237023890,-0.6691291928,0.5890297890,-1.9160044193,-1.0037022829,0.9969838262,0.8288206458,0.7137361765,0.4376760721,-0.3152429163,-0.1279374659,0.3058860302,2.4879970551,-1.6800762415,1.3998795748,0.1315685362,-1.5002568960,-0.2054777592,-0.5640303493,1.3824599981,-1.5066652298,-0.1466574669,0.0915420279,-1.5763106346,0.3803360164,0.3596437871,1.6621896029,-2.0491542816,0.5765722990,0.1055285856,-0.6198460460,0.1618170589,-0.2169940472,-1.3427215815,2.0164062977,-0.1058017090,0.1578689814,-0.2270162851,-1.2023625374,0.1935603917,-1.2298405170,0.8036319017,0.4344801307,0.8951734900,-1.2423279285,1.8490303755,-1.7592107058,-0.0522167459,-0.4019152522,-0.5006610751,-0.3094775975,-1.4385825396,-1.0389883518,0.1763581932,-2.0889842510,-1.5009762049,-1.7347135544,-0.5499082804,-0.4091817141,-0.5084925890,-1.7259931564,1.1904336214,1.4008835554,-0.7470872998,-1.3493409157,0.7245262265,-1.4676628113,0.5550032258,0.9892581105,0.4554331899,-0.9375141263,0.2234862894,1.4883899689,-1.2857881784,0.6532883048,0.4550784528,-0.4044540226,1.1459242105,-1.0726145506,1.7762796879,0.6335772276,2.1317172050,0.0332975127,0.5179934502,-0.0992914513,-0.7737841010,-1.8695286512,-0.7923610806,0.1761527956,-0.7032017708,0.5412458181,-0.9078052640,0.4937606454,1.1527210474,1.1989228725,-1.6906565428,-0.6281328201,0.2278220356,0.4933513403],[-1.0064896345,-0.0000735259,-0.8934984803,0.1968393624,0.6673569083,-0.5315707326,0.3909243643,-2.2143652439,0.9027023315,1.4943727255,0.9048801661,0.2294406146,0.9573911428,-1.2223769426,0.1271428764,-0.4152106047,1.0905336142,1.6445459127,-1.5889605284,0.1431340277,1.3259994984,1.2536746264,0.8460054398,0.7108342648,-0.1171533167,-0.7995001078,0.7147882581,1.0996558666,-0.8065167665,1.5425846577,-1.9306179285,-0.5112408996,1.7824292183,-0.3771158457,-0.5927338004,1.0465234518,0.9517975450,0.8834025264,1.7332577705,0.4638868272,-1.2697304487,-0.1591738909,-1.7999488115,-0.9415250421,-0.1269199103,-0.0580913350,-1.2965680361,2.1223971844,0.0455871597,0.2907867432,-0.3369020820,-0.6078436971,-1.5405783653,-0.1512064636,-0.6500342488,1.2631211281,1.0848257542,-1.4378325939,1.8252682686,-0.3971527517,-0.4756045938,-0.8820447326,-0.0112926476,1.2548937798,-0.6723657846,-1.2332930565,-1.6561949253,-0.4634982944,-0.6589117050,-0.8096425533,-0.0774505436,-0.9581915140,-1.0015362501,0.3436736166,-0.7924914956,0.4270743132,1.6997318268,0.1629018933,2.6756398678,-1.3755729198,1.0998866558,1.8532304764,-0.2589176893,-1.5839751959,1.2968481779,0.1454011351,1.0266642570,-0.9606741071,0.2631573379,-0.6971349716,-0.7989433408,0.6325493455,0.5290307403,-0.8914008141,-1.4983018637,0.1810698956,-0.0936394185,0.7586063147,-1.2714086771,-0.1377002895,2.1368517876,0.0857330486,-0.9389966726,1.3523417711,1.9735085964,1.8241761923,-1.9587459564,0.3082878888,-1.1944063902,-0.4493401945,1.2215254307,0.1729801297,-0.0349820703,1.2029552460,0.5404530764,-0.0075979042,-1.0764358044,-0.7699571252,1.1834006310,0.5547564626,0.9156490564,-0.9856804609,-0.2785350978,-0.5186696053,1.3056958914,1.0631380081,0.1274959594,-0.2401805520,0.1479017437,-0.0582404770,-0.0790114775,1.0718803406,0.7031837702,-1.9135128260,0.3803895712,0.5894224644,-0.2192968726,-0.6255428195,-0.6426970959,-0.9397515059,0.5840689540,1.3284479380,1.3117339611,-1.5271680355,-0.3960041106,0.6294850707,2.4824326038,-0.9239836931,-0.5315219164,-1.0664471388,-1.2447993755,0.5901226997,0.2070456147,0.9431948662,-1.0233124495,0.6501449943,-0.0389963426,-0.1377603114,-0.1678514928,-1.2874910831,-1.5522481203,0.8630752563,-0.3618498445,-0.2759974897,-0.8493277431,1.7580159903,0.9600577354,-1.8509514332,-0.9705380201,1.6187961102,2.1266665459,0.0816572905,-0.4026440084,-1.1554012299,-0.2847146988,0.2674738467,-0.8082942367,-0.8134348392,1.1221520901,0.0270938650,-2.2677624226,-0.9180805683,0.2363503575,0.2875520289,0.8363741636,-0.0643644705,-1.5559432507,1.6432741880,0.5946225524,0.4202830791,0.0109968660,0.6184870005,-0.4283002019,-0.3038954735,0.2131291479,0.0461715311,1.5333558321,1.9102970362,0.0692449957,1.0393091440,0.1182040423,-0.7626225352,0.6060833931,-0.4008977413,-0.5854520202,-1.5112763643,0.9768438339,1.2555006742,0.3712321520,0.3047838807,0.5041245818,0.1352996081,0.6537587643,-0.1001813114,-0.8632913828,-0.7738704681,-0.8903281093,0.0303989761,-1.5796298981,-1.8807632923,0.4715923071,0.7007640600,-0.4200832248,-0.6220460534],[-0.0537967421,0.6815012693,-1.2996648550,0.7641907334,0.1130914018,0.3476420939,0.8941524029,0.0757900104,1.6656370163,0.4651370049,0.7107572556,-2.1352045536,0.9271841049,1.3995282650,-1.4733744860,0.3333723545,1.2920279503,-0.0935617089,-1.4775958061,-2.2185895443,-1.0155769587,-0.0170312412,0.4430365562,-0.0669785142,2.0955364704,-1.2891235352,-0.3313333094,0.1508097947,-1.0214672089,0.6050590277,-0.3226830959,-0.6393830776,-0.7354826927,0.5948662162,0.0514817126,-0.3487473130,0.2998561263,-2.3917312622,1.9389803410,1.3662700653,-0.2757565379,-0.3776474595,0.0289556049,0.4271129966,-1.4703136683,-0.8241951466,-0.9340753555,-0.0965261236,-0.3129947484,-2.1099956036,-0.4961585999,-0.6232930422,-0.4650352299,0.9700036645,-0.1744728535,1.5180350542,0.6357852817,-0.2957075536,-0.7334786654,0.3546252549,0.1438398659,0.0114881210,2.3138237000,0.6166037321,0.5094236135,0.2026752681,-0.8440943360,-0.5575982928,0.3867180049,0.9236090779,0.2406300455,0.9912159443,0.5896103978,0.3247047365,-0.0631345809,-0.8472291231,0.4703036845,0.5695025325,0.1397066861,-0.2487258315,1.1258966923,-1.0027259588,-1.0338441133,1.1072216034,-0.7892639637,1.6951647997,0.1438161433,0.0723784938,1.4623792171,0.6937738061,0.0695512146,2.1728897095,-0.3971661031,1.0819935799,0.3043802977,1.4986712933,0.6935018301,0.5144372582,-0.7089034915,2.0252778530,2.0352370739,-0.5140570402,-0.4385772347,0.1052131802,0.7070764899,1.1298842430,0.9526279569,-0.3403368294,0.3679586947,-0.5590703487,0.5506619811,-1.2554082870,0.9212560058,0.3789348006,0.4162181020,0.8884142041,0.8954343796,1.3765288591,0.1669536531,2.1947975159,-1.0765970945,-0.4415110648,-0.8056438565,-0.5395519137,1.4391063452,-1.0550506115,-0.3433833122,-0.7339180708,-0.5391919017,-0.2920258939,2.4956951141,1.7541620731,0.3999114931,-0.5657392144,-1.4513942003,0.0673754513,1.3311697245,-0.9032511711,-1.1899926662,0.9078088403,-0.5799974799,-1.0377428532,-0.6368058324,0.3528950512,0.4219335318,-0.1826667786,-0.5943683386,2.2517185211,0.7124376297,-0.5504871011,-0.8670164347,-0.2532586455,-0.9611055851,0.5109337568,-0.9788072705,0.7710443735,0.0522208549,-0.1642433405,-0.2398169041,1.0718778372,-0.2769748271,0.8341576457,-0.9128410220,0.6022919416,-1.1872302294,0.5882807374,-0.7837983370,0.0029770739,0.9149373174,-0.0023518181,-0.3423418403,-0.2342749983,-0.2588711977,0.5861242414,0.2901591957,-1.9919506311,-0.5077320933,-1.0557484627,-1.3903524876,-0.8617141247,1.8194987774,1.4909827709,1.1573592424,-3.5810458660,-0.9443206787,-0.6563444734,-1.9958139658,-0.4275041521,-1.9974306822,1.0515087843,1.4036556482,-0.8239413500,-0.2179917097,0.8843204975,0.4367180169,-0.9152120352,0.5092750192,-0.4325858355,-0.2898980379,0.1608916968,0.0433968008,0.7225730419,-2.3706922531,-0.1152571887,-0.2443416417,-1.7064037323,-2.2502951622,0.9623617530,-0.9202928543,0.5554278493,1.4936690331,-0.8538382649,0.4118251801,0.4199622273,1.0834783316,0.7692968249,0.9061453938,-1.7944680452,-1.6934518814,1.0800925493,2.9148349762,-0.6273124814,-1.0704936981,-1.0964157581],[1.2692940235,-0.0670206621,0.4683865309,-0.2278233916,-1.6969687939,-0.1294624507,1.3910541534,-0.8832511306,-0.5370650291,-2.7819578648,1.0052194595,0.0291601941,0.1069035754,0.1541337669,0.4313090146,-0.0532100685,-0.2221809328,-1.2927292585,2.0467059612,0.6717846990,-1.4643356800,-0.5161830783,0.8706104755,-0.7400351763,1.7521550655,0.7392818332,1.1075246334,-0.9495987296,-0.2971502244,-1.7795100212,-0.3664148152,0.3774253428,-0.3637647033,1.0382292271,0.9270262718,0.9909930229,0.4617742598,-1.7854360342,1.1072719097,-0.4553121328,-0.1896470487,0.7345716357,-0.1584715992,0.1256743520,-0.4396451116,1.7899280787,0.4290977120,-1.1429880857,0.1356596798,0.2704877853,-2.9028491974,0.2635847330,-0.2302935719,1.4411734343,0.7899709344,0.7097681165,-0.1670908928,-1.1633665562,0.8314404488,-0.5223177075,-0.6025066376,-1.0426368713,-1.0971044302,0.0751154199,-0.0060958047,0.5358031988,-0.3496257663,0.1946886927,-1.0601308346,-0.3055567741,0.7915786505,-0.6497223377,0.1561425328,0.0680144206,0.1885385960,-0.0341186747,-0.0913931131,3.2813217640,0.7469103336,-0.7709512115,-0.6652572155,0.6616647243,0.8398932219,0.1677317768,0.2261482328,0.5642980337,-0.9490278363,-0.4119906127,0.8089081645,2.3489282131,0.0059719253,0.1277764440,0.4831945300,-0.1986074001,1.3758893013,0.9559911489,-0.3276863098,-1.9237053394,-0.0273960028,1.2066987753,0.5038545132,2.0547614098,-1.0013649464,-1.2404457331,-1.6175206900,-1.8106907606,-2.2587397099,1.4911020994,0.5077068210,0.1578625590,-0.2871367633,0.2982170880,0.2387153804,-0.8506680131,0.7121008039,0.2630681098,-0.4266733825,-1.2599084377,0.1100775227,-0.7961258292,0.0884582102,-0.4036722779,0.2554541230,-0.7653817534,1.2740881443,-0.9017828703,1.0660438538,0.2381526232,1.3428379297,-0.9337721467,0.5818094611,0.6203938723,0.5277485847,-1.3358016014,-1.0161755085,-0.3920041919,-0.3217634261,0.0977156609,-0.7099808455,-2.1828842163,-0.2492626160,0.5232057571,0.6404082179,-2.8672451973,0.0099505708,0.0627208427,0.5408713222,-0.8533446193,-1.0616909266,2.1865887642,0.7885946631,1.6995860338,0.8987043500,-1.2937207222,-1.3201901913,-0.3269158304,-0.5847586989,1.1706972122,-0.3119771183,-0.8133928776,2.2317814827,-0.0656682402,-1.2794408798,0.7121872902,0.6827929616,-0.9849511385,0.2409691066,1.4361633062,0.8795505762,-0.1011779606,-0.3517198861,-1.3311268091,0.0398764387,-2.5397872925,0.5240123272,-1.1318515539,0.1736470163,2.3500032425,-1.0210710764,-0.9013856053,1.2843419313,0.1951693445,0.2922241092,-0.9190829396,1.3920661211,0.3929746747,-0.2270657122,-0.0221442040,0.5061912537,-0.7442373633,-0.3657039106,-0.6952665448,0.6621089578,-0.1160282418,1.0427018404,-0.4019634724,0.3994499743,1.1279504299,0.1322755814,0.0824102312,0.1473124176,0.7525485158,1.3919463158,0.9325844646,-0.2164360136,-2.0686099529,-0.3924939334,-0.2780714035,0.0101592699,0.6358350515,-1.4313091040,-1.1910624504,-2.2517459393,0.1745254099,1.0222985744,1.6670593023,0.5249434710,-0.1815162599,1.8214089870,-1.4406580925,-0.9919193387,-0.5310760140,0.2919684052,-1.6313235760],[0.3511316478,-0.9309894443,1.1974879503,-1.6794967651,-0.8118848205,-0.8416833878,2.4623873234,0.3009655476,-1.4257361889,0.6312994361,2.2594048977,0.6614672542,0.9749627113,-0.8439388275,-0.6263849735,1.0214941502,1.8435775042,-0.7835627198,-0.5893369317,0.9996767044,-1.7351483107,0.5586682558,-0.9594988823,1.1005119085,0.2692739367,-1.0257135630,-0.7306185961,-0.3707855940,1.0002390146,1.7818371058,0.2506727278,-0.1208279952,0.7990749478,0.4406612515,-2.7838218212,0.3292020559,-1.1683198214,-0.7424931526,-1.6718312502,-2.2292375565,-1.5895670652,-2.1763262749,0.6465926170,-0.2387440056,-1.2814507484,1.5724308491,-0.0866635591,0.3556214273,-1.5556114912,2.2516131401,-0.0788190961,-0.9873566628,-0.5010582209,0.5617690086,-1.3074728251,-0.6975981593,-0.2382981330,-0.8825020194,-0.3499997556,0.8037531972,0.2698097825,1.4658676386,0.8141592145,0.6054009795,0.7439873219,-0.9993648529,0.3399610519,2.7317314148,-1.0810840130,0.1323891431,0.3207344413,-0.6511629224,0.3457941711,-0.7831311226,-0.9879703522,0.3776917756,0.8772977591,-1.7880016565,0.2965702415,-0.9034597278,1.7864934206,0.9782868624,-0.2956964970,0.6477823257,0.0506903753,0.5080415606,-1.3493453264,0.2018167675,-1.4426267147,-1.1808420420,1.0383150578,0.9860555530,1.8076187372,-2.0575242043,-0.6333572268,-0.2041875720,-0.9048840404,-1.3473826647,-0.2191231549,-0.5989145041,-0.9193871617,0.7825738788,-0.6629906893,-0.0044529312,0.4017296731,0.5302216411,-0.6302867532,0.1900966913,0.1169983447,0.6267263889,0.2086966336,-0.3434092402,-0.0818031728,0.2871176302,1.8508354425,-0.4121738076,-0.3695137501,-0.8422460556,1.0362706184,-1.5687952042,1.3191411495,-1.4238070250,-0.1425267160,0.1950446665,-0.0501813479,-0.9971384406,0.5666148663,-0.7734076381,2.1213474274,1.0192799568,-0.2502143681,-2.5300765038,-0.1859777421,-0.5562409759,-2.1014626026,-1.5545033216,-1.9805938005,-0.3614606857,1.0864875317,-0.6382172108,-0.1713171601,1.4588235617,-0.1968735009,0.3457186520,-1.2535402775,-0.4798072278,-0.5499140024,-0.6521920562,0.0745719150,-1.3990981579,1.5811384916,0.9943895936,0.8224885464,-0.9130792618,0.6656284928,0.4408087134,-1.0863834620,1.6354051828,-0.1830042750,2.0229175091,0.9130825400,-0.7718485594,0.5636826158,0.9207437038,0.6272261143,-0.0113689406,0.1329929233,-0.7101846933,0.7689437866,-1.1801084280,-0.2835783064,0.4113340974,0.5327725410,-0.7214313745,-1.5421614647,-1.9424954653,-1.6779434681,-0.0172744263,-0.5212426782,2.3069400787,0.4098668694,-0.3884662688,-1.4270358086,-0.4474042058,0.0626090840,-0.6908984780,1.6261773109,-1.3247954845,-0.9704781771,-0.8681657314,-1.9094009399,0.7300789952,0.0771100372,-1.2461156845,0.3015325367,0.3156217039,-0.0172931459,-0.4923977852,0.8767017722,-1.7975519896,-1.3432596922,-0.1768195033,0.2010150999,-0.0109114358,0.2074261457,-0.0014144401,1.3219326735,-0.1442534029,1.5599341393,0.5865324736,-1.0671136379,0.9432078600,-2.4022483826,-1.6965386868,-0.3520489037,-0.6729676127,-0.4498138726,-0.1366031319,2.2423534393,-0.6495697498,-0.6955966353,0.2690102160,0.0754883811,-0.2480758727],[0.4548644722,-1.6006692648,2.9957056046,1.1254948378,0.1499847472,0.1920881867,-0.5768033862,0.2211746275,-0.2526126206,-1.2733937502,-2.1981089115,-0.1316194385,0.6110743880,-0.4321219325,-2.6817026138,-0.1910612583,0.9476387501,-0.9832076430,-1.1690601110,0.3118586838,0.1494942456,-0.0717568323,0.1027661785,1.6578212976,-0.4804956317,0.4835452437,0.1303046048,0.8426531553,-0.4875191748,1.4141306877,1.9516247511,-1.5209347010,0.1807358265,1.0086404085,0.1693771333,-0.6385271549,0.3150402606,0.2134657651,0.1275645643,-0.2692812979,-0.4176293314,-0.5310031176,-1.2867263556,0.4024953544,-0.6629074216,-0.8900157213,0.0173238255,-1.3357878923,-1.4739190340,1.2099581957,-0.8706635833,-1.1307188272,2.5382192135,-1.3760643005,-0.8997769356,0.0810065046,-0.9796665907,-1.9800330400,-1.3330463171,-0.0266266223,0.6131624579,0.7309258580,-0.4710527658,1.8186960220,1.0128185749,-0.2675081193,0.3097981215,-0.3390848637,1.3907425404,-0.1129092425,-0.1128220260,-0.7463982105,0.3480891287,0.1308476329,-0.8871914148,-1.1553707123,-0.8099475503,0.9398341179,0.6741917133,0.5551754832,2.3883841038,0.0637566000,-0.3006283641,1.7931590080,0.6349275708,0.0974078625,1.9796898365,0.9637216330,1.2067064047,2.6625752449,-2.9752113819,-0.2145902961,0.8895272017,-2.5124297142,-1.1934756041,0.6111067533,0.5439530611,0.9195749760,-0.0145593109,-0.1723228544,0.8032369018,-0.1235672086,0.8890910745,-0.5182160139,0.6868421435,1.3251487017,2.2621772289,-0.9907764196,1.2263659239,-0.7818847299,2.5534036160,0.0250207074,-0.6922243237,0.4500946105,0.8247621059,-1.5440511703,1.0582951307,0.9684923887,0.0219670925,-0.6095378995,0.2694644332,0.6649788022,-1.0967034101,-0.7213160992,1.1593426466,1.0333893299,-1.1508731842,0.6200610995,1.0737234354,-2.6945307255,1.0907959938,-0.4236430526,0.5467333794,0.3633519411,-1.4687079191,0.2660523951,0.4187117815,-0.2689535320,-0.7421191335,0.4190295637,-0.0754861236,-0.9021491408,1.7387038469,-1.0696322918,1.2910215855,-0.9210324287,-0.5588905215,1.5117577314,-0.7016773820,1.5223786831,1.0122696161,-0.4869149625,-2.6730711460,-0.4777221382,0.3872422874,0.1491771042,1.2386612892,0.1987817436,0.7457051277,1.2785915136,0.1476992965,0.2074522674,-0.4072807729,-0.5786799788,-3.3096375465,-0.6840371490,-0.8496714234,-1.0677458048,1.0516594648,0.3639971912,0.0243675150,-0.0318917781,-0.0292874929,0.4616909027,1.9434530735,1.6624674797,0.8748117685,2.0247778893,-1.5770218372,1.6974775791,0.7939242125,-0.1475799233,-0.4257296026,-0.9291113019,-0.1765965670,-0.9489408135,-1.0270024538,0.3457305431,-0.1267700195,-0.8588191271,0.0220551956,0.4580956995,-0.8898931146,0.6698141694,1.1612095833,1.5577360392,-0.0882073417,-0.1198285371,0.5890731215,-0.3089555502,-0.9081842303,-0.1492018104,-1.7106012106,-0.7759175897,-0.4991606772,2.0636332035,-1.2211964130,0.2082914710,-1.8170255423,0.1874208301,0.8306953311,0.1591050774,0.1346683651,0.5078101158,0.3186313212,0.0747938156,1.1232910156,1.1165424585,-1.5080624819,-1.8543579578,0.4946228862,-0.3128373027,-0.8260221481,-1.1768620014],[2.7075240612,-0.8599362969,-0.1943347603,0.3728359938,1.5643889904,0.1224704236,-0.2554540038,-2.4267845154,-1.2763670683,0.1865660548,0.9339870214,2.1913509369,1.0479716063,0.5310890675,-0.4444085956,0.0102695441,-0.4931077957,-0.8591019511,0.3635742068,0.1819271296,1.4570590258,0.6254669428,-0.0766212940,0.7366033792,0.4073336720,0.3157582879,1.2674711943,1.5260124207,-0.1460522711,-0.6169248223,0.7802696228,-1.1895266771,-1.6959145069,1.0615260601,0.7290648818,-0.0039269542,-0.2192747742,-1.2065370083,-0.5926264524,0.4990629554,0.6350317001,0.2500172257,-0.7767462730,0.1204522327,-0.0855962262,-2.9598872662,-0.0594991967,0.0107230311,-0.6957206726,0.6268827319,0.8848359585,-1.0965247154,-0.3538480401,-0.2793050408,-0.6663282514,1.6578052044,0.9893534184,0.6942030787,-0.0255823508,-2.1172115803,0.8453764915,0.0800878629,-0.0446344092,-1.3436617851,-0.3629655540,0.7153369784,1.8806054592,0.5140509009,1.3209331036,-1.5869755745,0.0630695596,1.0406191349,-1.1759856939,0.7265060544,0.5853495598,-0.4912182987,-2.3420684338,0.2360036075,-0.0184030049,-0.2514174879,-0.5149710774,-0.9576873779,1.1475124359,-0.3164398074,-0.6195050478,0.9377791286,0.6614924073,-2.1293914318,-0.8671734333,1.8095625639,1.2773045301,0.7302829623,0.9080862403,0.4408243597,0.1983782947,-0.6720163822,-0.5482321978,0.1505797505,0.5328464508,-0.5543004274,-0.2349988371,0.3065464199,0.2317438871,-0.2143950164,-1.9504706860,0.3647886813,-0.9620810151,-0.5049009919,1.1725252867,-0.2122905999,-0.0769631341,-1.5674600601,0.1936446875,0.9210233688,-0.7128981352,-1.8007926941,0.3849214613,0.9164446592,-1.5944501162,0.4832324684,1.2739291191,0.3470695019,0.7192894220,-1.2342510223,-0.5707786679,0.3982577324,1.6125490665,0.8283737302,-0.4355474114,-0.5516900420,1.8830914497,1.4993001223,-0.4664620161,-0.6440277100,-0.1982753575,1.7335997820,0.7828823328,0.4520802200,-1.0521594286,0.6122381687,2.5034127235,-0.2546629012,-0.0021838548,-0.3127757013,-0.2439358383,2.8342952728,-1.1100224257,-1.8255298138,-0.2851569653,0.6020030379,-0.0797088295,2.1328616142,-1.2060884237,-1.0391825438,1.9026936293,1.5160888433,0.2516080439,-0.7265576124,0.6257002354,0.2562513053,0.3520869315,0.2826059759,0.7028957605,-1.1565545797,3.2492113113,0.6998537183,0.3435056210,0.0474957936,0.0238234904,1.0075225830,1.0553672314,-0.5581072569,1.4102920294,-0.8471719027,-0.3154569268,0.3686653078,0.3602911532,0.4066274166,-0.6914747953,-0.2434081584,0.1288081408,1.6579293013,1.3141508102,1.4821339846,-2.0403368473,0.6059473753,-1.0832898617,-1.4585893154,-0.3728159070,-1.8068834543,0.3847947121,0.5434930921,-0.3806275427,0.8868139982,1.4337956905,0.3013240099,0.3587332964,-1.2386696339,-0.8911620975,0.0859268382,1.1943483353,-0.6367681623,-0.6623798609,-0.2061398029,-0.5292823315,-1.5271534920,0.3254251480,-0.0908186063,-0.7286639810,1.3047002554,-0.0827621073,-0.1901878268,-1.0190961361,0.9524612427,1.5536764860,0.1080959290,0.7212635875,0.1993038505,-1.4570226669,-1.6779817343,0.3893408775,0.3303967416,1.0638159513,1.7132321596],[0.8575462103,-1.6333636045,-0.4927726984,0.7085943222,-0.6649975181,-0.7540698051,0.3705411851,1.5952237844,-0.9408431649,0.8396894932,-0.0682655498,0.1102770716,-0.1911044121,-0.3837050498,0.1033612564,0.4805106521,1.1790075302,-0.3850603402,-1.5772523880,0.1418904960,0.2107743323,1.0745302439,-0.1163458973,-0.2989508212,-0.1717454642,-0.1741078496,-0.5722891092,-1.1931658983,-1.0240820646,1.3909329176,-0.5920624137,-1.3037887812,-0.4021603465,0.4387072921,-0.4330362976,1.8886967897,1.1421550512,-0.6339848638,-0.4418257177,0.3721695840,0.2701981068,1.3258050680,-0.5577306747,-0.0739254504,0.3768513799,0.4234072566,-0.5511019826,1.1147866249,2.3339898586,0.4122173786,-0.5530776381,0.3115015626,0.0289805699,1.5773285627,-1.2268995047,0.4624019861,1.0431036949,-0.2830632031,-0.6748375893,-0.0909511521,-0.0818173736,-0.7964636683,1.1432673931,-0.5838388205,-0.7343836427,0.8128823042,-1.9741328955,1.0655366182,-0.1382042468,-0.4833156466,-1.5031466484,0.0463636853,-1.9227097034,-0.6942155361,-0.0444220677,-2.9480149746,-1.3248479366,-0.7946349978,0.1203350052,-1.8178205490,-0.8409462571,0.0176582448,0.7654770017,-1.1241449118,1.0637383461,1.5732291937,-0.2002674639,-0.3629679680,-1.1078330278,0.3732503355,0.0650348887,1.0154954195,0.5308608413,0.8413332701,-0.8172613978,-1.2249294519,-0.1462770700,-0.8106113076,1.2757968903,0.8820825815,0.8402888775,1.7235717773,-0.2222751230,-0.4631427824,-2.0011272430,-1.3081351519,0.3727766871,0.2681252360,0.9854740500,-2.2283413410,-1.8407127857,-0.8173459172,-0.1097566634,-0.2490268350,0.0852077156,-1.7533066273,-0.0306593943,0.0494587459,-1.7486133575,1.0223621130,0.9939898849,0.8368474841,0.4449765682,-0.9100048542,0.1528287232,1.8625997305,0.2812252343,-0.4299602211,1.2618894577,0.4911216199,-0.2961321175,-0.3500322402,0.9766417146,-0.6072682738,-0.3793948293,-0.2228083014,0.7391654849,-1.3898029327,1.9329074621,-0.9888586998,-1.6468340158,1.3287822008,-0.4032287598,-0.6147022843,0.8088668585,0.2952104509,0.4654255807,0.4560590386,-1.1324355602,0.8947141171,0.4242992103,-0.7234010100,0.1395297050,-0.5978423953,-0.0356286205,0.9698703885,0.5998618007,1.1406219006,-0.3814889491,-0.9062640071,-0.4771881998,-0.2948560417,0.7076987028,-0.9771244526,-0.5480884910,-0.3703163564,-0.2734317183,1.2818864584,0.8690806031,-0.2168426067,-1.0013710260,0.3965295255,-1.3480749130,-1.3271099329,-0.3585632443,0.1892410517,0.3836632073,-0.3862533867,-0.1512429714,0.8624945879,-0.5295600295,0.9678261876,0.5995805264,0.8620498776,-1.7274713516,0.9354030490,0.8139658570,-0.8361479044,-0.1532518119,0.0936337113,-2.4775524139,-1.9780313969,0.2005530298,0.8046504259,1.7265368700,-1.1824663877,1.0006707907,0.2549159527,-0.5395684838,-1.3906004429,1.1599717140,-1.1941506863,-0.0242273454,1.0806976557,-0.6423122287,-1.7502503395,-0.0535725392,1.0014064312,0.3842054307,-0.9795367718,0.0303774159,-0.5324507952,-0.3401973248,-1.1212528944,0.0285949763,0.4956311285,-0.2267297208,-1.3515195847,0.4184161723,0.1721993089,0.7958292365,0.1263063103,0.2711284459,-0.2135034651],[-0.7038419247,0.2187733203,-0.5362712145,0.1083598137,-0.2600317895,-1.2274924517,0.6941085458,-1.5299791098,0.4951930940,0.0788252577,1.3300241232,-0.9726471305,1.7447634935,0.7137358189,0.4610167146,-0.3599465191,-0.1522779316,1.7052100897,1.5464543104,-0.1919985265,-0.2842799425,0.6308130622,2.0006842613,0.4859058261,0.8829320073,-0.1657233387,-1.2302795649,-0.5572073460,-0.8428971171,-0.0465976670,-0.2116975784,-0.1333411336,-1.3210440874,-0.6006979942,-0.4126179218,0.0754443184,0.7098649144,-0.7600352168,-0.6254181862,-0.4199784398,-0.0825512707,-1.0666607618,1.6464910507,-1.1209568977,-0.7917913795,0.1129865870,0.0059932861,-0.4723086357,-0.2796922922,0.5758187175,0.9736551046,0.2789586484,-0.5730145574,-0.4830294251,-0.2489068210,-1.9525696039,-0.6475285292,-0.2628311813,-0.7734012008,0.8395585418,2.5376198292,-0.3365868330,0.5846264958,1.0598243475,-1.1626652479,0.0405477248,-0.9395235181,-0.2485307753,0.0545488633,-0.9857591391,2.4720458984,2.0487780571,-1.2066218853,-0.0878543928,0.9413759112,0.9323689938,-2.0413239002,0.2184711546,-1.9071009159,1.1700785160,0.5026462078,1.8074660301,-1.5341821909,-0.2498243898,-0.4873360395,-0.6551302075,0.8344846964,0.3601314127,-0.4684588313,1.2849541903,2.2285103798,-1.0161749125,-0.0508062355,0.7545269728,1.4866827726,-0.9345136285,0.7692088485,-2.3289158344,0.2548599243,0.2512003779,-0.9487016201,0.2089540064,-0.9035751224,1.4267714024,-1.0731743574,1.7076709270,1.2272638083,0.2197493315,-0.5237355232,-0.1973887086,-0.2108744830,-0.5565663576,-0.6995753050,-0.5322136283,0.3217452466,1.1632359028,-0.6271314025,0.3296809494,1.0473831892,0.4364182651,-1.3696779013,-0.2976042032,0.0399223492,0.7550673485,0.5357517600,0.4226417542,-0.0695470870,0.1780906320,-1.4290182590,-0.4919155836,-0.6635327339,-0.6278638244,1.0992287397,-1.2899185419,0.5348879695,-1.5645805597,-0.6093159318,1.9165036678,1.6862343550,1.2086102962,-0.5938796997,0.2510013580,1.5818654299,-0.0920653418,0.4793538153,0.4611625969,0.4175966382,-0.7902413011,1.3530526161,2.4345374107,-0.2522033453,-0.4852724969,-0.9632862210,-1.0252519846,-0.7647008300,2.6910192966,-0.3028256297,-0.1939556748,-0.6827206016,0.4865101576,-1.1748796701,0.9050163031,2.6230053902,0.1278017312,0.9562465549,1.0719659328,0.7716265917,-1.8127437830,-2.0730187893,0.2582882345,0.6441308260,1.4222255945,0.5335395932,0.4913542569,-1.1710500717,1.2177139521,0.6230660081,1.5083622932,2.5376482010,-1.0855648518,-1.1384391785,0.4204219878,-1.2163747549,1.5409317017,-1.7850402594,0.2881009877,-1.7824188471,-0.6656666398,0.2255385816,-0.2045584470,0.6886078715,0.4971415102,0.4256645739,-0.2858218849,-0.1442836076,0.8134331107,-1.1828783751,1.7522083521,2.3498823643,0.1647088081,1.1515992880,-0.7531849742,1.1517670155,-2.1913006306,0.9838366508,0.7988774180,1.2420282364,-0.3205684125,-0.0106539931,0.1312756538,-0.5737478733,0.6149320602,-1.3822107315,-0.3531156182,-0.5641967654,-1.1561065912,0.8050509691,-0.3803443015,-0.7108378410,-1.6383564472,-1.3067328930,0.8056710362,-0.9770047665,1.4766548872],[-0.5168541670,0.3145562112,-1.1484050751,0.7664395571,0.0075045303,0.0210218653,0.5821520686,-0.0796812326,-1.5467196703,0.1330328286,0.9529478550,0.6269364357,-0.2962416112,0.4320018291,-0.5550686717,0.9274399877,1.1973664761,1.2910717726,-1.4917968512,0.0722649843,0.5383219123,-0.2111446708,-1.4312043190,0.1909869164,-0.2979548872,-0.1053253263,0.4545270801,-0.1573643684,-2.6192853451,0.3299501240,-0.7100323439,-0.5290266275,0.6963036060,1.0016850233,0.3713839650,-0.0537927784,0.5277480483,0.9580436945,-0.3080750406,0.5621162057,1.1065112352,-0.1435552835,-1.2238268852,-0.6453787088,-0.0446425751,2.3824806213,-0.3643109500,-0.8466729522,1.1849539280,-1.9625260830,-0.2382405549,-0.3108474314,0.7180377245,-0.2890521586,-1.0908354521,0.3532983065,-0.9382154346,0.5689409971,-0.3626126945,0.1292094141,-1.3126652241,-0.4163072109,-0.8589738607,-0.0338691622,-1.0064257383,0.1432997137,-1.2719420195,0.4668884873,-0.3053086400,-0.0977246687,0.2234309167,-0.0470772311,1.0469348431,-1.3022555113,-0.8693733811,-0.1281105727,0.8558647037,0.8242882490,1.8926994801,2.1205914021,-0.3344083428,0.7537657022,-0.1441276968,-0.7482367158,0.4418944120,0.0492459051,0.3857703805,0.6495937109,-0.9396342039,-2.2726123333,0.5253503323,-1.3726311922,0.7066520452,0.8941339850,1.0972421169,-1.2170317173,1.4726543427,1.2693320513,0.7312103510,-1.4249899387,-0.4283734262,-1.2901015282,1.9668451548,0.6691100001,-1.5574185848,0.4526828229,-0.8244184256,-0.4453442395,1.3221229315,-0.2892827392,0.2289659083,-1.0478963852,0.4761168361,-0.2551328838,-0.4333049655,-1.1237722635,0.8928756714,-1.5040786266,-0.9676579237,2.4354195595,0.2679666579,0.8277609944,0.3292419612,-0.1868321896,0.6399228573,0.0246101934,-0.5365167260,-2.2954347134,0.9310479164,-0.1232519746,-0.7564478517,-0.4085958004,1.0798666477,0.4010500014,-0.0367867351,1.1169235706,-0.9824579358,-0.0519333407,1.1601675749,-0.5204966664,-1.4007287025,0.3084657490,1.1606221199,-0.2109643072,0.4174713492,0.7629770041,0.9784686565,0.6438628435,1.3034147024,0.0565330721,-0.1181432754,0.2954445481,-0.0077796308,-2.3304641247,-0.6992905736,-0.1077904403,1.6501281261,-1.1473655701,-0.4608682096,0.0156514887,1.2672146559,0.4234745204,0.1822052300,0.4332121909,0.4516955316,0.7653177977,1.6070424318,1.0253183842,-0.2590387762,-1.0479173660,-1.6088930368,0.1823649406,-0.9678751826,-0.7864817977,-0.0212973952,0.5997629762,-1.3160744905,-0.9506019354,0.5000987053,-0.1930724084,-1.2007164955,-0.9722674489,-0.9011637568,0.9087528586,-1.0918318033,-0.0354075357,-1.4514678717,0.9590007663,0.6602175236,1.3757650852,-1.0226017237,-0.8164103627,-1.0483584404,0.3173210919,-0.4989819825,1.5050772429,-1.3538606167,-0.3477463424,-1.2884012461,0.5413200259,-0.0952065215,-2.5451941490,0.8496850729,-0.8102738857,0.3639152944,1.0038706064,0.8044942617,0.2769761086,-0.8043497801,-1.0582702160,0.9640245438,-0.9525019526,0.2505484223,1.7418069839,1.1239496469,-0.2312931418,0.5900598764,0.2988237441,-0.9258888364,-0.7468770742,0.6754369736,-1.1314466000,2.2557041645,0.3076329231],[0.1141875535,-0.7910183668,0.9181339145,-0.2924492657,0.1954403967,-0.0891278759,0.9766722322,0.8747189045,-1.0259243250,0.6990704536,0.5039597154,-0.2485073507,-1.1559615135,-1.5401313305,-0.5223922729,0.7553474307,0.5209439993,2.2820031643,0.6241645813,0.1820227355,-0.8249456286,-1.0354923010,0.0518557951,0.6551367044,1.0242308378,1.2647337914,-0.3354948759,-0.8350704312,0.1489525437,0.8589512110,0.6270372272,-0.5417641401,-0.2549240887,-1.0517722368,-1.3470808268,1.7237402201,0.2793397903,1.0387977362,0.3658451438,1.7909607887,0.1156069040,-0.6626947522,-0.6802643538,-0.3461824656,-0.1924321353,1.7015250921,0.9053295851,0.1271467358,1.0843471289,-0.3697059751,-0.8226702809,0.3727995455,1.2557933331,1.3963470459,0.8964594007,-1.0065754652,1.3490592241,-0.2992320955,1.1706128120,1.8907653093,-1.7348724604,-0.1472353786,0.5626885891,0.1557893902,-1.5522532463,-0.7784453034,-1.1265046597,-0.5841084719,-0.3729529679,-1.0711065531,-0.4288546741,-1.5960400105,1.3952161074,1.3239439726,-2.5381610394,-1.0360237360,0.3437467515,-2.5755796432,0.0326888487,-0.2264865786,-0.7182295322,1.0793272257,-1.1627691984,1.3597364426,-0.2644090056,0.1385675967,-0.3974082470,1.2229468822,0.9545928240,-1.6005281210,0.4387021065,0.3070425093,-0.0559233315,1.7117067575,1.7901859283,1.9087609053,0.2534044385,-1.0119205713,-1.4722032547,0.5966674685,-1.1095695496,1.1928594112,0.0251557846,0.4326552749,-1.1571912766,-0.6874693632,-0.3447971344,0.3170865774,0.7671335340,-1.7947992086,-1.3456940651,-1.4110554457,0.1072896123,0.1770090908,1.9528205395,-0.5405386090,0.5682284236,0.6587491035,0.9563859701,1.0822306871,1.2021546364,0.7378921509,-0.5463986397,-1.7351492643,0.6233202815,-1.3622575998,-0.4995904863,1.4538668394,-0.5640485883,-0.4524624050,1.5386605263,-0.4490965605,-1.7440617085,1.1747021675,-0.8111938238,0.8217701912,-0.5931746364,1.0802215338,-0.2538038790,2.3748760223,0.1568471640,0.6048847437,0.1726618111,1.2744444609,-0.0180251598,1.1177147627,-0.4916356206,-0.3290631771,1.3595130444,1.1628024578,0.6827121973,-0.2205483317,-0.2955637276,0.6364265680,-1.0403375626,0.8137436509,-0.2903918028,0.2760030925,0.4262830317,2.0322773457,1.5663291216,0.3227753341,-0.0340636298,0.0319516100,-0.3528116345,-0.0407276042,-0.2693226039,1.4170272350,1.7600680590,0.7054511309,1.4845144749,-1.2117332220,0.8845197558,-0.4061044455,-2.0150425434,0.0858845934,-0.3021506965,0.2173276544,0.1497766674,1.0825449228,-0.5751865506,0.4710103273,-0.1027014628,-1.6194263697,-0.7232217193,-0.9368378520,-0.2567723989,0.2749138474,-0.6792963147,0.2497180104,-0.1856022477,-0.5589736700,0.4200106561,1.6372632980,-1.2062540054,0.7298827171,-2.3512728214,-0.7061026692,0.6459239721,0.9156780839,0.2385244370,0.7191008329,0.1381670535,0.2389231473,0.8946838975,0.8140711188,0.3687241673,-0.8611537218,-1.2541856766,0.6498557329,1.6523033381,-0.7426189780,-0.2608479857,1.2524713278,-0.7394489050,0.1951705813,-0.5104800463,-0.1717242897,-0.4723890722,0.0514790267,1.2266012430,0.6670045853,-0.2716105282,-1.6211389303],[-0.7709470987,-0.4063224494,1.0450941324,0.6701294780,0.4660073519,-0.3111612499,0.4569470286,-0.3755842745,0.2935816944,-0.7299827933,-0.3773456812,-0.0588846505,0.5191531777,0.6253951788,0.5395801663,-1.1540787220,1.1528965235,-0.7550507188,-0.4361100197,-0.8517587185,-0.2650027573,-0.4558273852,-0.1218817607,-0.1632884443,-1.7337768078,-0.2454500645,-0.4841493666,-0.2076745182,0.9510493875,-0.1895506978,1.7178503275,-0.0648765713,-1.3134737015,0.0220830422,1.5550876856,1.1541197300,0.9463537931,0.8542752862,-0.3284910917,-1.6964749098,0.8325127959,0.5698849559,0.1683153063,0.4132466018,-0.4727652371,0.9073839188,0.3624867201,-0.5237817168,0.5296620727,-1.5915416479,0.7892110348,1.2042807341,-1.2157537937,-0.0659838766,-1.0977131128,1.8772678375,0.0199707374,-1.0153725147,0.9905872345,0.2679038346,-0.6155877709,1.6769979000,2.2211329937,1.2426955700,-1.1432538033,-1.3055896759,0.1563118249,1.5841238499,-2.2452995777,0.4084261954,2.3925390244,-1.5944696665,-0.2753798664,0.7038434148,-2.1634190083,1.5523954630,0.0431083739,-0.6386553645,0.1563305855,0.4483917058,-2.9002535343,-0.2124763727,1.2646445036,0.9671489596,0.1235830262,-0.6239078045,0.1781397164,-1.2477340698,-0.8786101341,0.0340745784,0.9605447650,0.3556161225,-3.0921344757,0.3453056216,-2.2019863129,1.5977934599,0.0587060228,-1.2354182005,-0.5321778059,-0.8846167326,1.3429207802,0.1889484078,-0.7125145197,-0.9847447276,-0.3402684927,-1.9771710634,1.4227645397,1.8663989305,-0.6787713170,0.0083999801,1.4120321274,-0.8796091676,-0.8530539870,-0.2235978842,0.7683705688,0.3531757891,-0.8423203230,0.8098526597,0.2967545092,0.1743304431,0.6617275476,1.5916795731,0.3336613774,-0.3279126287,-1.1561247110,0.8219054937,-0.0077004205,-0.0758062452,0.9670962691,-0.9370991588,0.1727723330,-1.0711120367,1.2559354305,0.6726061702,-0.1603709906,-1.5501614809,-0.0739242285,-0.6497949958,0.3207924664,-1.1628973484,0.4206815660,0.7446953654,-0.5787026882,-0.3373136222,1.4212529659,-1.6684862375,-0.1762335151,0.3530758023,0.4051634967,-1.2940163612,0.9606325030,0.4437426329,-0.2314557731,-0.0697056428,-0.2034419626,-0.7468641400,0.9059106112,1.1647446156,-0.9227508307,-0.7264728546,-0.7138158679,-0.7802699804,0.7355730534,-0.7290205956,0.5469813347,-0.7842488885,0.4940427542,-0.9318450689,0.3016936481,-1.1100772619,0.4054757953,-0.7490972877,0.5455661416,-2.0676968098,0.1622077972,1.2108730078,-0.5942135453,-2.1084134579,-0.3815261126,0.6267691255,1.2911912203,-1.4211307764,-0.0792804286,1.2469061613,-1.2400754690,-0.6282653809,0.6214656830,-1.9671771526,0.9299145937,-0.1225127801,-1.6832170486,0.3292508423,-0.6738933921,0.3709022999,1.6576431990,1.8432923555,0.1324210614,0.0297149438,-0.7616475224,-2.1235435009,0.1847538352,-0.5866488814,-1.1501641273,-0.1941118091,0.2461440712,-0.2032909393,1.0314236879,0.0010392868,-0.8659663796,-0.7367367148,-1.2445702553,0.5127481222,1.2003036737,-0.9545887113,0.3125925362,-1.1271697283,-0.0129429530,-0.7439265251,-0.2158260792,0.9935899973,0.6832865477,1.1826785803,-0.6790092587,-0.7337641716],[-0.3919575214,1.2301563025,0.1421116590,-1.9185277224,-0.1422203928,0.0144021548,-1.2244794369,0.8910320997,0.8001833558,-0.6226376295,-1.9729691744,-0.3718367219,-0.6450943351,1.4727654457,0.4042130411,-0.2300358117,-0.1837593764,-0.3389971852,1.0521665812,-0.5878276825,0.8147978187,0.2012608349,0.5305464268,-0.0704385564,0.6780981421,-0.5457631946,-1.0154840946,0.3954752088,-0.8780980706,0.6862570643,-1.6729224920,0.8030760288,0.4177638292,0.5515066981,-0.8780927062,-0.3644495308,-0.7639865279,-0.2581588328,-0.0097340150,0.1328005195,-1.3428583145,-0.5613111258,0.0030085165,-0.2764977217,0.3225880861,1.0092039108,0.2349234968,-1.1569188833,0.3209208548,0.8061857224,0.9328452349,1.5121496916,1.4083685875,-0.7483514547,1.3963418007,-0.2169407457,-1.0577676296,0.2618527114,0.0820487142,0.2296931744,1.1636767387,-0.6497741342,-0.7683633566,0.3333502710,0.8443777561,1.1035634279,1.1778227091,0.9579150081,-0.8281220198,0.4982230663,-1.7778943777,-0.7316639423,-1.6550662518,0.1353182048,0.4598369896,-0.0059965760,0.6672404408,0.6366089582,0.3507957757,0.2280575633,0.5433624983,0.6431276202,1.0851905346,0.8459165096,1.2322151661,0.9934787750,0.1766625792,0.1074638367,-0.8066105247,1.8972502947,-0.6080517173,-0.4537470043,0.5049105287,-0.0842246264,0.8485350013,0.9383783340,-0.1987407804,2.5588440895,0.9118668437,-1.2429528236,0.4617349505,1.2518606186,0.6015286446,-0.4189657271,-0.3078119159,0.0632796735,1.3473374844,1.3317080736,0.8101310730,1.3187366724,-0.0723575279,-0.6994635463,0.8570897579,-1.5637105703,-0.2297752500,-1.2368377447,0.4984486699,-1.7071242332,-1.1083687544,0.0562235564,-0.4866240025,1.7403073311,0.0683723688,-1.6624240875,1.0369526148,0.4173009694,-0.9422541261,-0.3057643175,-0.0225711260,-2.3352799416,-1.2981127501,0.1158406734,-1.7067408562,-0.8495888710,-0.2003936023,-2.6660759449,-0.1373704821,1.8758572340,0.4224371314,-0.9995551705,0.7457840443,-0.6162922978,-0.6332775354,-0.3346899152,-1.7673785686,0.1534919441,0.5546674132,1.5832332373,0.0081346072,1.0892113447,0.9690399766,-0.3808458447,-1.3704863787,-1.2239551544,1.3384201527,-1.7577083111,-0.4571655095,-0.2167893350,0.2889954448,-1.5132929087,-0.0505164191,-1.0508286953,0.9178054333,-0.5142904520,0.6449629664,1.2783317566,0.0204112101,-0.9757503271,0.8718924522,1.2775079012,-0.8486067653,-0.7375156879,-0.8481579423,0.2910692394,-1.4055770636,-2.4370431900,-0.2421151698,-0.1659164876,0.0634354576,0.8462583423,-0.3317031562,-0.0410253853,0.8783381581,-2.0833046436,1.9415665865,-0.4809909165,0.9779080153,-0.1633502096,-1.0577044487,-1.0115772486,-0.9951361418,-0.8234838247,1.3299692869,-0.3004491925,-0.3629858494,-1.1682218313,0.0332856104,0.0584488660,0.5619188547,-1.4354987144,-0.6348283291,0.5653066039,-0.7393788099,0.4992229044,-0.8277471066,-0.8877035975,-0.4010598660,0.1966238767,1.2567228079,1.7160092592,0.6613878608,0.9422925115,0.6065798402,0.1258662939,0.1002879664,0.3995673954,0.5345766544,1.5022646189,-0.6795228720,1.2247700691,0.3355742097,-1.9241726398,-0.5274044275,0.3859989941],[-0.1902812123,2.0812780857,-0.6521640420,-0.9521286488,-0.1160523742,1.2861061096,-0.9185868502,-0.0956847370,0.7666553259,0.0705903322,-0.9173219800,-0.6386165023,-0.2875458598,0.4759561718,-0.2991779745,1.0411814451,-0.2388623655,-1.9096094370,0.9282014966,0.1911507100,1.0705833435,-1.3557881117,1.5480512381,-0.0446682349,1.5442548990,-0.6466215253,0.8872923851,-1.2207497358,1.0928955078,1.7613279819,2.2592256069,0.0676627234,-1.4281723499,0.0194905587,-0.7902153730,-0.2634108365,1.2093659639,1.7138423920,0.4568839371,-2.7929172516,2.0279846191,1.9827568531,-0.3400610685,-2.5195252895,0.1077408493,1.2092367411,1.3849642277,0.4741785228,0.6720715165,-1.3682155609,1.3626445532,3.0927917957,0.2808203399,-0.0680369139,0.9278790355,-0.3832291663,-1.0694628954,-1.3236962557,0.2294029742,-0.1003467962,0.5837204456,1.4401695728,0.6600863934,-0.1587504148,-1.0510327816,2.8563518524,0.3107630312,0.0793141946,-0.7014832497,-2.2048156261,-0.1639941931,0.8510155678,0.8897792697,-0.2509429157,0.7656675577,-1.8384664059,-0.9290833473,0.8442848921,0.0714782104,0.7431453466,0.7526600957,-0.3000283539,-0.8917927146,0.3781193793,-0.8924313188,-0.7258448005,1.2255562544,0.4353717268,-1.0612089634,-1.0652147532,0.1068682298,1.5616360903,-0.5242556334,0.9643884897,0.7915732861,-0.3687317967,-1.5341099501,-0.6866817474,-0.0766677111,-0.5075598955,0.0492559411,-1.5805724859,0.4028103650,0.5294559002,-0.2617757022,-1.5436277390,-0.4214951396,0.7148287296,-0.7212103009,0.4432360232,-1.1655206680,-1.7276140451,1.6665664911,0.1276424825,-2.0064013004,2.2659885883,-1.1007127762,-1.7148460150,-0.9983851314,-0.1229355112,-1.5004343987,-1.0642398596,0.5147129297,-0.1959828287,1.0724027157,0.5129589438,0.2611027360,1.0203869343,1.8625689745,1.0872251987,0.1776206940,-0.6679985523,-2.3908557892,1.9265812635,-0.0750824064,-0.6875794530,-1.8375434875,-0.1266568005,0.4062012732,0.9031251669,-0.1045055613,-1.1981265545,-0.3047462404,0.7788920403,-0.5198747516,-3.0495319366,0.4344826937,0.5080818534,-0.0456377529,-0.6228168607,-0.3361916542,-0.4655468166,-0.8032031059,0.0986663103,-1.4225085974,-0.7214075327,-0.1927438825,-1.2750662565,0.3828608692,-0.2010356784,-0.3411757648,1.0567926168,0.8308400512,-0.3758359253,-1.7007135153,-1.0153723955,0.7814183831,1.3333725929,-0.0588039719,-0.1536338776,1.7262624502,0.5375443697,-1.4447143078,-1.0294841528,-1.0612380505,-0.1466866732,0.0248447470,1.2826215029,-0.6459833384,-1.5521104336,-0.7346411347,0.4375969768,-0.6678929329,0.2032340467,0.7792050838,-0.1258176714,-0.7511096597,-0.6474458575,0.2450655252,-0.9021635652,-0.5974918604,-1.1855431795,-0.8081023097,-0.3418832421,-0.0899530277,1.9315698147,0.0990892202,0.6180377007,-0.9881890416,0.7278859615,0.2622669935,1.4801809788,-0.4546771049,-1.0858519077,-1.0294415951,-1.2702590227,-0.2850361466,1.2038623095,0.4503741860,0.5454785228,-0.6018747687,0.4178525209,1.3235566616,-1.9712238312,-1.0972834826,1.2963863611,0.1940562278,-0.6993190646,0.1960473210,0.3936943710,-0.6680406928,0.2631404102,-0.0992291048,1.5811232328],[0.5356354117,-0.0902978182,-0.2183861434,0.0835172236,0.2231216878,0.4585287869,-1.0632811785,0.2157344222,-0.7864517570,0.0688186809,-0.8122203946,0.9727475047,0.4813622832,0.9644367099,-0.0104756039,0.2411717325,0.3121369183,1.1740385294,-0.8667576909,0.3110567629,-0.4314704835,0.1743235737,0.5403572917,-0.6680797935,1.0002886057,1.6705429554,1.1789067984,0.5091510415,0.2972907722,-0.3430694044,-0.8662321568,1.3162040710,-1.8510755301,-0.4570961595,0.9468869567,0.5335932374,-3.9929611683,-1.1525392532,0.4048165977,-0.7148792148,-0.0899725407,1.1440914869,-0.5628069043,-1.5965940952,2.3284101486,-0.6404005289,-1.9125361443,-0.8831542730,0.5702723265,0.3274931014,-2.0143942833,0.5962705612,-0.2118565291,0.2940602303,1.0698854923,-0.7329437733,0.5447551608,-0.7397634983,0.8467305899,-0.5462845564,1.9932767153,-0.0706784353,2.3692502975,-0.4771787226,-1.3188631535,-2.1861901283,1.1592558622,-0.2879912555,1.3112417459,-0.1735279113,1.7638275623,-0.1482204646,-0.5775184035,1.6037025452,-0.2173911482,0.8606928587,-0.7163739204,0.5022127032,0.6795402765,0.7757351995,0.9296262860,2.1179580688,-0.1177830398,-0.6922940612,1.8955492973,-0.1037179306,0.8602941632,-1.3067930937,0.0278356988,-1.8676902056,0.8719504476,0.2133634984,-0.4246577621,0.2922644913,0.5142137408,0.7787869573,0.3955132365,-0.0308551546,-0.5254090428,-0.0208215415,0.5863494277,0.2410605252,0.2540073991,-0.0505717807,0.3257252574,0.0478928015,0.0124214934,1.1643067598,1.0866931677,0.1932839900,0.7468745708,-0.4936366081,-0.1967490166,-0.5898027420,-0.3114001453,1.2001738548,-1.1749259233,-3.6432943344,-0.7237185836,2.1023168564,0.3772802353,0.9598003626,0.6345593333,0.9760369062,0.5842297077,-0.4238112867,1.0510704517,1.4212644100,1.2052621841,1.1709578037,1.2481642962,0.1114866585,0.7139697671,-1.6507397890,-1.1089572906,1.0787169933,-2.2450892925,1.2657155991,1.0965223312,0.4065240622,0.4205618203,-0.1879614741,-0.6797156334,0.9142844677,0.0223195404,-1.9562933445,-2.0345225334,-0.3191227913,-0.6532308459,0.7075082064,0.8971603513,-0.2222088575,-0.7548856139,-0.8718494773,0.5969509482,0.4012758136,1.4944665432,-0.0430379659,-2.5413269997,0.2290947139,-1.7839071751,1.7898751497,0.9773983359,0.8138353229,0.3676052094,-0.4572495520,2.3534555435,1.7259155512,1.3094695807,-0.2298201919,-1.5486131907,-0.4831325114,-0.1515334994,1.1911811829,-0.1305567771,0.7306992412,-0.9365200996,-1.0807114840,-0.4267658591,-0.6438264847,0.4536756277,-0.4718679786,0.0732847527,0.9452646971,-1.5916801691,-0.0438594930,-1.6057710648,0.2780228853,0.1078691557,-0.4012733102,0.5433638096,-1.8002210855,0.0649706200,0.1245789826,0.8710531592,0.1115417033,-1.9357055426,-0.5548510551,1.2520089149,1.4839948416,-0.5209493041,-0.9624369144,0.8466321230,-0.5569323301,0.2157223821,0.0956537575,-1.0298379660,0.4497537911,0.1934795082,-1.4493902922,0.3130516708,-1.3260329962,-0.2592625022,-1.8027895689,0.5699852109,0.5679404140,-2.5769262314,0.9336313009,0.7585353851,-0.6365656853,-0.4001408815,0.9239498973,1.9906063080,1.4039293528],[-0.5344263315,-1.6005463600,-1.1294572353,-0.6989622712,-1.0434752703,-1.8072863817,-0.2911771536,-0.3740274906,-0.2669128776,-0.6695302725,-1.0134661198,-1.7485753298,-0.2880801260,-0.0940565765,0.1530825347,1.2911791801,1.3317542076,1.1100609303,0.9411215782,0.6393887997,-1.3498729467,-0.1483654976,0.1755969822,-1.0591218472,-0.4806758165,0.0374209993,-0.2632440925,-0.2745012343,0.6968136430,-1.8665977716,-0.1406579614,0.7593407631,0.4745651782,-1.2644783258,0.6004345417,1.4316047430,-1.1685087681,0.2429454327,0.4121043384,-0.2837245464,0.0339051634,-0.4033333957,0.4898167849,-1.8071894646,-0.0860238522,0.3482603729,0.8476430774,1.7993699312,-0.4857821763,-0.9498022199,-0.0087727476,-0.9771353602,-0.8784105778,1.8610028028,0.4241184890,0.4889109433,-0.5960755348,-0.1157418191,0.2188333124,-0.9271811843,1.2049478292,-0.7487514615,-2.4948208332,2.3248777390,0.1948041320,-0.9396010637,-0.0695591941,1.1261837482,-1.4390196800,0.1074865907,0.2451537251,1.4741477966,-1.3714965582,1.4597207308,-0.2240107805,0.4680463374,-0.7798717618,-0.8509687781,-0.0445612594,1.0413498878,1.0824186802,-1.3900460005,0.8383417726,-0.5728988051,-0.7608184814,-1.4918603897,-1.1130791903,0.9705112576,-0.9052938819,0.0602722466,1.7449322939,1.3911231756,-0.2824020386,-0.0249941666,-1.8760643005,-1.9072006941,-0.2799922526,-0.4541512132,-0.3491178155,1.3265537024,0.3157596588,0.7349253297,-0.3306770921,-0.0793977678,-1.4547114372,-0.9750895500,2.5718612671,-0.4372460544,-0.3933745921,-0.2879850864,1.6932580471,-0.2667824626,-0.4276626110,-1.5534998178,0.5865116119,-1.1775484085,-0.3417048752,0.2226299047,0.8676363230,0.8768593073,0.4840772152,1.3084653616,1.7750610113,0.5020828843,0.8089952469,0.6637986302,0.2111792713,0.3420891464,-0.5921617746,0.2682724297,-0.4276701808,-1.0372554064,0.0703604743,-0.8647756577,0.5419068933,-1.7433658838,-0.3307072222,-0.5725274086,0.6627629995,-0.3101287186,0.5326774120,-0.7019532919,-0.8407902122,0.7224192023,-0.1563667655,-0.5524084568,-1.0491452217,-0.7639257908,-0.2581304610,-1.5388603210,-0.6796113849,0.8086640239,-0.1712522805,0.5022003651,1.1249178648,0.2038956285,0.1236568317,-1.4507499933,-0.4801124036,0.1786764115,-0.2940721810,0.9317100644,-0.3659346402,0.7731112838,-0.8082608581,-1.8103444576,-0.4421080351,-0.9311588407,-0.6190632582,-1.6836090088,0.6225354075,1.8004719019,-0.3675890863,-1.5402535200,0.5061526895,-0.7011044621,-0.2044154704,0.7838950157,0.6873581409,0.3003750443,0.2717193067,-0.7483507395,-0.2657977045,-1.1034845114,0.9093827009,-1.7455307245,0.2808555067,-0.6657431126,0.2908801436,-0.3246472478,0.1302491426,1.6909010410,-0.1852371842,-0.3363511562,-1.5134866238,2.3311779499,-0.0808570161,-1.1184997559,-0.1656611711,-0.2683204412,0.5825760961,-1.1025604010,-0.9419777393,-0.7417781949,-1.4885994196,0.1455330104,0.0818805471,1.8152869940,-0.3648020625,-1.0479524136,1.0624965429,-0.1902331710,-0.5713142157,0.6154579520,0.3105306625,1.3375319242,-1.4385323524,0.4703989625,-0.0446280465,0.2165392637,0.2389579415,-1.2069883347,-1.5069841146,2.8056120872],[-0.4463079572,0.4630065858,0.3852918446,1.0595120192,-0.1924976408,-0.7959891558,0.5518957376,0.1073498875,2.0644276142,-1.3379882574,2.1439268589,0.0684103593,0.1015025824,-0.6148912311,0.0664112791,1.3932160139,0.1381065100,-0.4723600745,0.6620807052,-1.3079439402,-0.9490098953,1.5134844780,1.3218621016,-0.4307395518,-0.0436251275,-1.3971976042,-0.9197613597,-0.1284685731,0.9387473464,0.0908714011,-0.6960137486,0.8947666287,-0.4395277500,0.0078539103,0.0331664793,-0.4104016125,0.2590791285,2.3322181702,0.5433509350,0.3989716768,-2.2112383842,0.9127438068,1.0851579905,0.0210881140,1.1078456640,-0.0505299978,-0.5449906588,1.7212568521,-0.7205293179,0.1527352482,-0.5153023005,0.0899326950,-2.0822553635,0.7137017846,-0.6866122484,-0.3113394976,0.3766130805,-0.3086671531,-0.0307912417,0.9296104908,-0.0212098304,-0.7371267676,-0.3977877498,1.2362489700,1.0222923756,-0.5818612576,1.8817348480,1.1494332552,-0.0183267836,-1.1659201384,1.5057953596,2.7619345188,-0.7941793203,0.0287719052,-0.7107766271,-0.2424992919,1.6478064060,-0.6357446313,0.8620203137,0.1403571814,0.3091052175,0.2087818533,0.9582784176,1.2466231585,-0.0951789767,0.0370424651,0.0931980982,-0.0357219428,-0.9581607580,0.6691345572,1.4618363380,-0.9407261610,-0.1881764531,0.1779287457,0.2282896638,0.0024585791,0.9555747509,-0.0863538906,-1.6428929567,0.4155323207,-0.4702777863,1.0200219154,-1.0292131901,1.9110612869,-0.3506657481,0.3369286656,0.1099356636,-0.8530461192,-0.5134611726,1.2525533438,1.9823640585,0.8887806535,0.1952644587,-0.3287902176,0.6369973421,1.1657168865,-0.3110114336,1.9505547285,0.5987884402,-0.3918232918,0.1895684898,-0.8541156650,-0.1494368315,-0.9396588206,0.5325658917,1.4862364531,0.5386831760,1.0839402676,2.7512357235,1.0610109568,-0.3076118827,0.0839342251,-0.2132758647,-0.4443840086,1.1139613390,-0.5371167660,0.6878733635,0.6211798191,0.1295899898,-0.1400801837,0.3262644410,-1.6013503075,0.3448503315,-2.4370861053,0.0091410428,0.4318372011,2.4131960869,0.7610641718,-1.2417212725,-0.5548300147,0.2673580945,0.4718003273,1.0521353483,1.0287045240,-1.2894179821,-0.0358988978,1.2699348927,-0.1881172657,0.5786081553,0.3217543662,0.5109526515,0.6527564526,-0.1036807373,-0.8156668544,0.0486890860,-0.8705747724,0.0336206816,-0.1434892714,0.0206560418,-0.5102793574,0.0312600732,-0.0512645692,0.3963692784,-0.9462928772,1.7282290459,0.0291529056,1.6862194538,1.0479409695,-0.1493015438,-0.4607142508,0.7110564113,-0.7917346954,-0.3492127657,-0.1107221022,1.2990982533,0.0823019817,1.3518792391,1.6583864689,-0.5439693332,-2.4000995159,0.4834471345,0.2429045290,-0.3895867467,1.5748558044,-1.8866602182,0.5014733672,-1.0549618006,0.6886458993,-0.5591328740,1.7372102737,-1.0586389303,-0.7030788660,-1.7595483065,0.5768519640,1.0110862255,-1.0807354450,0.7039963007,0.2570515871,0.2202721834,0.3636488020,-0.0619608201,-0.4254122078,0.1702727824,0.2321278155,-0.2789296806,0.3148307204,-0.2219217718,-0.0343165696,-0.6693635583,-0.7482005358,0.3152418435,-0.6094266772,0.5564612746,1.0861303806],[-0.3645062149,1.1158699989,0.7505072951,1.6706467867,0.9701129198,0.9791996479,-0.4747344851,0.8000352383,-0.8672477603,0.0212012902,0.1365957260,-1.9201898575,-0.6686220169,-0.2664861679,-0.7977192998,-0.1907032579,1.1660275459,-0.8042742610,0.9925575256,-1.2276042700,-0.7777420878,0.4423106909,-1.0314748287,-1.1646858454,-0.1148602143,-0.0822476819,1.2950305939,-0.1583028287,-0.3801110685,2.1181035042,1.1377567053,1.1714947224,-1.4710574150,0.4374298453,-0.3345782161,-0.2080748528,-0.4310014546,-0.3359941840,-0.6037350297,-0.4509580731,-0.2071041912,0.4953692853,-0.9209408760,-1.6250727177,1.3350207806,0.5784474015,-0.7418002486,-1.0781373978,-1.4611221552,0.8532691598,-0.2607238293,0.5639829636,-0.5600467324,-1.1986761093,1.9563639164,0.9690504670,0.2546267211,-0.3837289810,-1.2347412109,-0.1670784652,2.9372646809,0.8678464890,1.1291265488,1.5747507811,-1.6810820103,-0.2933495343,-0.6225114465,-0.3882344663,-1.3630297184,0.0092936410,-0.5639885068,0.2759515047,-0.2828184068,1.1528820992,1.1894996166,1.1782293320,1.7409573793,-0.3719946146,2.2909574509,3.0704162121,-0.9461196661,-0.2518333197,-0.6309245825,0.5706209540,-0.4163119495,-0.0939932466,-1.1150132418,-0.4613604546,0.0398271419,-0.2534232736,-1.9071460962,-1.8280466795,-0.4866101444,-1.3184527159,-1.4179421663,0.2209537029,-1.1249928474,-1.9038407803,-2.7663104534,0.6664661169,1.0175549984,0.1773152649,0.2420164049,1.5144679546,0.0260805730,0.3917195499,0.2187479883,-1.1952811480,0.0412240624,0.1335472763,-0.3340080678,0.0762820840,1.1268568039,-3.0501861572,-0.8245661259,1.7237170935,-2.8219740391,-1.8330252171,0.7809044719,-0.3621836007,1.7247868776,-0.1848417073,0.3827749491,0.8851286173,0.1156658381,2.0285098553,0.8909919262,-0.7913900018,-0.4023923874,1.4397592545,-0.2121118456,1.3215402365,-0.1021919549,-0.2679600120,-0.6018244028,0.0751775131,-0.2411542386,-0.8728830814,-0.7452493310,-0.9146154523,-0.3126709759,-0.0148961348,0.3722932041,1.4483546019,0.3125064075,-0.4321153164,-0.3628958166,1.5365076065,-0.4864256084,-0.4952484965,-0.4730603993,-1.5365190506,-0.7496447563,1.2180451155,-1.2079240084,0.6674391627,1.0864683390,-0.2232403457,-0.1725623906,-1.0300061703,0.8511503935,1.1540714502,0.2684497833,2.0633459091,1.5850899220,0.9936402440,-1.7767057419,0.2575030923,-0.2837025225,0.9801308513,0.0955806598,-1.3171532154,0.0740581304,0.7199848890,-0.3346158564,0.2480404526,2.2816629410,-0.7901241183,-1.2123996019,-0.7006688118,-3.1003596783,-0.4071684480,-0.2404175103,0.3279474974,-0.1991623193,0.8064158559,1.3032054901,-2.3590478897,-0.0860624835,-0.0414754823,-0.7682408690,-0.6589986682,-0.3123554587,1.5079144239,0.2001926452,-1.2423874140,-0.8025832772,-0.2944001555,0.7860500813,-0.5993912816,0.0452994257,-0.1173223034,-0.3171161115,-0.6040194631,0.4698894918,-0.2821411490,-1.8677010536,-1.0488799810,1.0262153149,-0.9576758146,1.3599735498,-1.7847856283,1.2750602961,1.3729448318,-0.0263605714,-0.9299097061,-0.1553459466,-0.5619153976,-0.2708025575,-0.5707209706,-0.3943482339,0.3243137002,-1.2652244568,1.3428239822],[-0.4716863632,-0.1471398324,0.5845143795,0.7798466682,-0.2970978022,-0.2879599035,1.1741337776,-0.3029748797,-1.0286338329,-2.5272741318,1.8863145113,0.5938357711,-1.5568106174,0.1661666781,1.0948197842,-0.4422018528,0.4641939998,-0.8608870506,-1.7977205515,-0.6338832378,-1.5577857494,1.3675371408,-1.8018686771,-0.4680710137,-1.3529676199,-0.3295443952,-0.0289831124,0.8985692263,0.5144635439,-1.5340685844,0.7415658832,-0.4952560365,-1.2680888176,1.5494275093,0.9896262884,1.1337209940,0.1172252595,0.5003512502,0.3104941547,0.8642488122,-1.9873039722,0.2901904881,-0.8896881938,-0.3933593929,-0.3691793084,-2.2398619652,0.4009266198,1.4610875845,-0.2109690011,0.3212065697,1.1923828125,-0.9447377920,-0.7677613497,-0.0199948680,0.7007796764,-0.9966187477,0.1880163848,-1.5960267782,1.1941541433,1.3885799646,0.3051355183,-0.4914822578,-0.0551109239,0.8271973729,-0.0875257626,0.5913065076,-0.3327606320,-0.6633206010,1.1625896692,0.5635477304,1.5795736313,0.7323051691,0.5140512586,0.7973369956,-1.0491675138,-0.8615369797,-0.0855538324,-1.1763240099,0.6985024214,0.7214810848,0.3039073646,0.7869974971,0.7645076513,1.6823736429,-0.3224197328,-0.6913388968,-1.0375618935,2.2595710754,0.3254954815,-3.2841098309,-0.9829556942,0.6173440814,-1.9146822691,1.6384691000,0.3188497424,-0.9541326165,-1.3458615541,1.1278688908,0.3943688273,-0.2816155255,-0.6670376658,1.7449932098,1.6885739565,1.6031520367,0.1641594321,1.3355803490,-0.5794470310,-1.4504963160,0.0844430998,0.2493409365,0.1463551372,-2.4972960949,-0.4465410709,-1.2534028292,0.9957426190,-0.7179170251,-0.9940697551,-1.5139750242,-0.3549970388,-0.0613587759,1.6371300220,-0.3277179301,-0.7049137950,-0.0345475487,0.8238404989,-0.0779935420,0.1792568564,-1.1482546329,-0.6932830811,0.6941252351,0.0120476391,1.0842584372,-1.1157293320,-0.2156767249,0.1917534322,0.2484143525,1.1325517893,-0.8084028959,1.9061423540,0.2140457481,2.5162801743,0.1694203168,0.2875690758,-0.4746890962,0.2268090844,0.6107716560,0.8992943764,1.2538052797,1.0825774670,-2.5980246067,0.8615135550,-1.2262650728,-1.2255992889,-1.7337546349,-0.0070146662,0.8009208441,1.0482748747,0.1379574686,1.4142774343,-2.7599356174,-2.6668968201,0.5720087886,1.0927113295,0.5224719644,-1.5818355083,1.6267343760,0.8453273177,-1.3154888153,0.7564472556,0.8823671341,1.7488454580,-2.1088161469,-0.3071520329,-0.9865061641,-3.3408041000,0.8878111839,-0.5089591742,-1.7002439499,-1.6869919300,-0.7074144483,-1.5640826225,0.0950985998,-1.5500789881,0.6208086014,-0.3661846220,-0.8483721018,0.1937151700,-1.3323022127,-0.3852874339,-0.2245144695,0.5444493294,2.3293950558,0.1018384323,0.0900604799,-2.0430760384,0.1845244169,1.0585149527,-0.4266940057,-0.2745861709,-0.9895468950,-1.4220205545,0.1086603552,0.7137717605,-0.7071261406,-0.6727264524,-1.1171315908,1.1003943682,1.4516644478,0.2674415112,0.7746139765,-0.4443411827,-1.4839503765,-0.4744228423,0.6212148070,0.1285764873,0.4126030803,-0.8194864988,0.2797495127,0.7431498766,-0.6712838411,-0.5915322900,0.0478469022,-1.9627013206,-0.1429765671],[-1.1604765654,0.7248870730,0.5285183191,0.3912655115,0.8200900555,-0.0187557004,0.3092218041,0.8215553164,-1.2147023678,0.8691491485,0.1782266051,-0.3846947849,0.2185835987,1.0123075247,0.5845950842,0.8014039397,-1.2928036451,0.2337188721,-1.0127472878,-0.1021579877,0.4277488589,-0.4217742980,0.8380681872,0.9014368653,-0.5033906698,0.3400486410,0.0378595553,1.4128333330,0.8601081967,1.4224760532,-0.5014793873,-0.1299059242,-1.0861430168,-2.1957287788,-0.4724318981,-1.2804791927,0.8848083019,-1.0232039690,-1.9286930561,1.9506202936,-1.2104187012,-0.8170687556,1.6399490833,-0.2306459248,-1.4863326550,-0.9082179666,-0.2592566311,-0.6489036679,-1.1253803968,0.2403939068,0.0076182713,-0.6155346036,-0.6273389459,1.2138336897,0.1259267330,-0.4360826612,-0.2855346501,-0.9321278930,0.2889509797,0.0191639476,0.3197908700,0.1481243372,-0.3380688131,-1.4064606428,-0.1481387913,2.0216214657,0.1358122379,0.3295311034,-1.3799560070,1.0495952368,-0.6023465395,0.7122180462,-0.2905054092,0.0740248412,1.8817989826,0.0416676588,0.6724176407,-1.2744238377,0.3802832663,-0.3467430770,-1.1206740141,1.2709239721,-0.0360667482,-0.4205509424,0.8479788303,-0.2158568501,-1.0330330133,0.4496297836,0.2781554163,-1.9734966755,-0.7712466717,-0.1627359092,0.2592321932,-0.5584867597,0.3639693856,-1.8700847626,0.0532161370,-1.0296891928,-0.7521325350,-0.2848433852,-0.1023412347,0.3919997215,-1.0115511417,-1.2037749290,-0.5682365894,1.3685394526,-0.0325128809,0.1605796665,-1.9592183828,-0.7536043525,0.5448845029,-0.4187681377,1.5351369381,0.7247617841,-0.1284294128,-0.7700523734,-1.3904502392,0.2299453765,-0.0612412915,-0.4955629408,0.4037678242,-2.3232903481,-0.5163176656,-1.1535639763,0.1474791318,-0.3062639832,-0.3380345702,0.4434403479,0.1649090350,-0.0253193621,-0.2594115436,0.3455506265,-1.1220024824,1.1439352036,1.9330021143,-0.2664181888,0.6555473804,0.4029106796,0.6844402552,0.5201416612,-1.7882884741,-0.0639681965,0.9145079851,1.1875967979,-0.2303394228,1.4555683136,0.0759437382,0.3192932904,0.1900645643,0.1694294959,0.2299467325,1.1252853870,1.3348319530,-1.4127507210,0.8178436756,0.3104344010,0.0983516201,0.3478089571,-0.8033730388,0.4526213109,-0.2079967260,-0.3332097232,-0.7742238641,1.0865453482,0.3817871213,-0.8886569738,0.9300866723,0.2929508984,-0.1157365143,0.7728258967,-0.6975427270,-0.9684841633,1.3194321394,-1.7829133272,-0.3412713706,-1.2375971079,-1.0775034428,-0.5838529468,-0.7925577760,-0.5509713888,1.3133839369,0.3206467927,1.3325715065,0.4925050139,1.9970691204,0.4951836169,0.1311973631,1.4893194437,0.5594164133,0.4966352582,1.2948005199,-0.8946422338,0.3424288332,-0.4328266084,-0.8383091688,0.3484825790,-0.5664611459,-1.5805536509,-1.2075260878,1.0009816885,-1.1555238962,-1.1081479788,0.2666878104,-0.0843278915,-1.4013813734,-0.0587022007,-0.5409751534,-0.9161397815,-0.0049323305,-0.5609408617,-0.9947663546,1.7990250587,2.1042752266,0.3246460259,0.8142860532,-0.4318092465,0.0749454051,0.5255390406,0.8453169465,0.5342290401,1.3789731264,-0.1871801764,0.1282057762,-0.8915588856],[-0.0542284548,-0.1044168994,-1.5793139935,-1.0756293535,-0.9769575596,-0.3783388138,-0.1792947501,0.1676430255,0.9718354940,-0.3327423930,-0.9197884202,-0.6990206838,-0.7812698483,-0.7872581482,1.0362886190,1.0649689436,0.3260521293,2.1018323898,0.0040883296,-2.0931577682,0.6810662746,-0.8956177235,-0.6246948242,2.6490068436,1.4860161543,-1.1300300360,-0.0098225111,-0.3665522337,-0.1155728847,-0.2216357440,0.0329407454,-0.3452160060,-1.6651244164,1.1996959448,-1.0849312544,0.0575289764,-0.6823420525,0.4595836699,0.4365171492,-1.0625749826,0.6479692459,0.1152886003,0.6351547837,0.8655322194,-0.1900938898,0.6194599867,1.3249210119,-2.9902887344,-0.9829337001,2.3041112423,0.8291086555,-0.6690052748,-0.2973387539,0.2854040861,0.2512688935,-1.0038155317,0.0745838508,0.6835303307,-0.3019574583,0.0656837150,0.2763928175,1.0574886799,0.2528705597,-2.0649085045,-0.2525582910,0.5633801222,0.7122578025,0.6231449246,1.2130430937,-0.3417893350,-0.6274027824,-1.0826811790,1.3845734596,-1.6314278841,0.9325391650,-0.2828492224,-1.1294640303,0.3213407695,-0.1412851214,-0.5739104152,-1.8523945808,0.1375545859,0.8524460196,0.9795963764,-0.4191142619,-0.1105024815,-0.8044444323,1.7718458176,-2.1773252487,-1.9565769434,-0.0790803954,1.5214020014,-0.9446907043,0.2151204497,-0.7931734920,0.1091351062,0.0690722018,-0.2768059075,-0.1884625107,-0.0551569052,-0.8582618237,0.7444183230,-0.3289690912,-0.4206421971,0.6482777596,-1.8037947416,-0.6215838790,0.2966335118,-0.5860925913,0.2072690427,0.3849021196,-0.0163337681,0.7419171929,-0.7282328010,0.3230105639,-1.0278416872,0.2014814466,-0.1945166439,-0.0878563076,0.5807428360,0.4235515296,0.5991556644,-1.6961510181,-1.7235015631,-0.1653775871,-0.0229291823,0.3718184233,-0.5282102823,-2.0017440319,-0.5723828673,-1.0784522295,-0.1570475549,-0.4171837568,1.1093618870,0.8538563251,-0.6578562260,0.2458041012,1.4572660923,-0.4246588945,0.5097835064,0.6360350847,-0.2560229897,0.0219479911,-3.1653001308,0.7525913119,0.1623850465,-1.1981235743,-1.5419058800,-1.2821328640,-2.3250749111,-0.0390541479,1.9492053986,-1.6592499018,-0.2550086081,-0.8584333658,-0.6264243722,0.0905184820,2.3940489292,0.8812188506,-2.6074683666,0.5467622280,-0.2826860547,-1.7907680273,-1.6223297119,0.0945725664,-0.6790642738,-0.0352940187,-0.6122911572,-0.7683660984,-0.5392516851,-0.3982014656,0.8679015040,-2.1680600643,-0.5140717030,0.8265195489,1.5463507175,-0.5618708134,-0.9446499944,-0.9218199849,-0.5487257838,0.5468548536,-0.1935274601,0.2828486264,-0.7948492169,-0.0389313214,-0.7580267787,1.2519108057,2.5005273819,-0.4626162350,0.7376989126,0.7565030456,2.0083138943,0.8451092243,0.8647089005,0.0845709145,0.6045849919,-0.2940496504,-0.7108674049,-1.6182552576,-1.1536600590,-0.5232933760,0.8932416439,0.0766904950,0.2250192761,-3.3111915588,-1.2070434093,-0.5528053045,0.2814783454,-0.0727163851,0.4501400888,-0.5304975510,0.0220623203,0.5409076810,0.5520184040,1.1103562117,-0.7519988418,-2.4101579189,0.8522349000,0.3122522533,-1.3813184500,-0.9066482186,0.0503111966,-0.5682703853,0.8344324231],[0.0455108099,-0.5007439256,-0.0220513381,0.6138712168,-1.5130780935,-0.8231852651,-1.2862572670,0.0757517591,-0.3834818006,0.9912862182,-0.2152566612,-1.8530063629,-0.8569292426,0.8364073038,0.1884991527,-0.8838205338,-1.4345126152,0.4621810317,1.8554944992,1.7764538527,0.2408635467,-0.3988132477,-1.3176436424,0.3313519657,-1.5404409170,0.8066987395,0.6141310930,1.0647925138,-0.5903156996,0.4407163262,1.0937725306,-0.6846204996,-0.8255679607,0.0301491339,1.1726086140,0.5547693968,-1.9036145210,-2.1507112980,0.9439404011,0.8304248452,0.8022865057,-0.0395314060,-1.7726892233,-0.7131135464,-1.3535375595,-0.2295022607,0.6360743046,0.3305475414,0.1700846702,-0.4246495068,-1.0522756577,-1.6014963388,0.8445893526,0.4519396126,-0.7346076369,-0.9541358948,0.0660870895,-0.2796024382,1.5112785101,-0.2288440019,0.7933291793,0.9457479715,0.0437120758,1.0272008181,2.6187674999,0.2938658595,0.0002436232,-1.8856309652,-0.2801553011,-0.8098967075,0.2391258031,-1.5088934898,0.1154030189,0.1677009463,-1.0893034935,0.1279728115,-1.1742339134,0.0815478116,-0.4688488841,1.9693156481,1.1393784285,0.7566793561,0.0332948640,-1.1088514328,0.6060234308,0.3168972135,0.9489172697,0.8915824294,-0.0744909570,-2.9674692154,0.2204231173,1.3969603777,2.8875684738,1.1382696629,0.0757457167,-0.3666677177,0.0659102425,0.0668415278,-0.5749508739,-0.2340170890,-0.5543112755,0.2262243181,0.3229537606,-0.3703001440,1.2843537331,-1.3588908911,-0.4063398838,0.3984783888,0.4746867716,-0.5155292749,2.0688710213,-0.7366701961,-1.2884358168,0.0306869484,-0.6278145313,1.6479524374,1.2733566761,-1.5114120245,0.5874340534,1.4164183140,2.1109287739,0.6936261654,-0.7801271081,-0.1424248666,1.4228651524,0.7660459280,0.0757838488,-0.3184366226,1.6464478970,-1.3185890913,-0.6462491751,0.5449139476,-1.3927319050,1.1904312372,0.5407568216,-0.0330950134,-0.3919748962,0.7456977367,-0.6043021679,1.7865223885,-0.3086243570,1.0165492296,0.2271374613,-0.7343394756,-0.6063373089,-1.8273447752,-0.8988688588,0.3666749895,0.2657340765,-0.2736480534,-0.7589226961,0.2684849799,-1.1223801374,-1.1474984884,-1.2845572233,-0.3152797520,0.3679023683,-0.2113581449,0.5975563526,-1.9758757353,0.2240671962,-0.5090435147,-1.2870079279,-0.0849780515,-1.1679377556,0.4966672361,-0.6675986052,-0.4594515264,-2.6474492550,-0.3685693741,-1.2190847397,1.2057003975,-1.1603654623,1.1231783628,-0.5407581925,-0.5821552873,-1.8120853901,2.3940880299,1.2630169392,-1.4847002029,0.0002179273,-0.2718341053,-0.6148485541,0.6212561131,-0.8127297163,-0.2228314281,0.6530621052,-0.2918465734,0.8982840776,-0.3148573935,0.0192828048,0.1528849155,-1.0955338478,0.1683052927,0.7518684864,0.0263153650,-0.5481123328,-1.0166777372,-0.0326641724,0.6306336522,-1.0569366217,-1.3100726604,0.8450725079,1.1383305788,0.2161744982,1.4873583317,0.1062991545,0.0131904045,-1.0483144522,0.8597632051,-1.7767972946,1.5863535404,2.7823541164,1.6619212627,0.3272190094,0.6373428702,-0.3165480196,1.0289241076,0.0949189141,-1.5440686941,0.3050356805,0.1199226975,-1.2530963421,1.2903825045],[-0.0241483971,-1.6955019236,0.4644385278,0.2589190602,-1.3097108603,-0.5500065088,-1.0812994242,0.6089993715,1.7156662941,0.4168740809,-0.6155125499,-0.1195571795,0.1958514154,2.2196655273,0.7892631292,0.7401278615,-0.2391066402,0.9518038034,0.5104688406,0.7215762734,1.3206062317,0.2645880580,1.1824287176,0.9417513013,0.7604390979,0.1519564837,-1.5020102262,-0.4238476455,1.3813818693,-0.3240005970,-0.9087860584,-0.4296375513,-1.5056200027,1.0380312204,0.9179576635,-0.2080131173,0.4261639714,-1.2480202913,-0.1067449674,-0.1843754649,0.2243936658,1.2510346174,-0.4090640247,-0.3410273790,-0.8742214441,0.0221707560,0.0311127305,1.9829660654,0.4096661210,0.5662220716,-0.6245689988,-1.7555264235,-0.0548473075,1.2680255175,-1.4447710514,-0.7565324306,0.5842719078,-1.8277680874,0.1456336826,-1.0918613672,-0.9649658799,1.3064036369,1.1600183249,1.1379429102,2.1565191746,-0.8689957857,1.5636215210,0.4368431270,-0.6594150066,0.2378106266,-0.0386009179,-0.2020491958,-0.7171027660,0.7523133159,-3.0087399483,-1.2096756697,-1.4830690622,1.1132252216,-0.0736747012,0.2278408259,0.8607809544,0.0230039097,-0.0100709805,0.5663136840,-0.4573984742,-0.3147220016,-0.5916610360,0.3633759320,-1.7309250832,0.7415539622,0.9788953662,0.4454882145,0.9996737242,-0.7172723413,0.4500461519,0.2097088099,-0.1300976425,0.3683418930,1.6021925211,1.3492051363,0.5966399908,-0.2869676948,-2.2014067173,-0.6466356516,-0.4141579270,0.0322575942,0.2764803469,0.0523899347,0.9976919293,-0.4418748617,1.1962385178,0.8795982003,1.3394536972,-0.5127735734,-0.5311194062,0.8626922965,-0.7423558831,0.9309554100,-1.4290853739,0.0858087614,-2.6495206356,1.6978728771,-0.4349122941,-0.3831622601,-0.0684857070,0.7442887425,0.5322425961,2.0612838268,0.1241032407,-2.3649044037,0.3680788875,0.1251933873,1.7623183727,0.3210356832,-0.2840740085,0.7346179485,0.1568394750,-2.1416144371,0.0518188179,1.2533953190,0.8536785245,-0.2809889913,-0.4789661765,-0.7346751690,0.5946739316,-0.7996873260,-1.3966721296,-1.7271291018,-0.7277098298,-0.0455504879,1.6599112749,-0.2437718362,-1.4668591022,0.0005579666,-0.6929865479,0.9924173951,0.1146905273,-0.0723900497,-0.3171693087,0.4661208093,-1.2721763849,-0.9968279600,1.4310706854,-0.1644446850,0.2184965909,-2.4001052380,0.5335850120,0.8570533395,-1.2253437042,2.6228101254,-0.8978168368,-0.4737388790,1.2820608616,-0.2572124600,-2.3622217178,1.6324220896,0.2936998904,0.4136489630,0.4954855144,-0.6417990327,3.0474805832,-1.4472365379,-3.3824377060,-0.5294840932,2.2834894657,-0.2915201187,-0.5834859610,0.6736102700,-1.0685982704,-1.0622173548,-3.0860776901,-0.1529570520,-0.9341332316,0.6026499867,-1.6047893763,0.5600776672,0.2471535951,0.3888663054,-0.5745077729,1.1368119717,-0.1875921935,-0.2457357347,1.0788971186,1.3245619535,0.4084223807,-0.9082864523,1.3701889515,0.5639448762,0.6552087665,0.2393982857,0.5696803331,-1.8782240152,0.1603340507,-1.0327521563,-0.1901682466,0.5955625176,0.8033736348,-0.3817498088,0.4623665214,0.5622440577,-1.9576177597,-0.2124888599,0.0532335043,1.7872124910],[1.0935274363,-0.6834817529,-0.0209047794,-0.2557629645,-0.0322177224,-0.4546183646,-0.6292889714,-1.8742160797,-0.8476565480,-0.0208909176,0.9334092140,1.3976117373,0.7560239434,-0.6076855063,-0.4940192103,-0.2218679041,-0.0247830916,0.4010295868,-0.2099526823,1.6346033812,1.0749132633,-0.7803043723,-0.6348884106,-0.6152335405,0.9498234391,0.2001388222,-0.5263258219,-0.8058626056,1.0469522476,0.3226033449,0.1166816801,0.9418267012,-1.3817039728,-0.2754204571,0.6587382555,-1.1658596992,-0.5008330345,0.6715875268,1.0270637274,0.6949701309,1.0310188532,0.3335841298,-1.1015419960,-1.6726310253,0.0740614757,0.3277696967,-0.3727508783,0.1862053722,0.2605803013,1.8626759052,0.0707811713,-2.1108138561,-0.6218534112,0.4231096804,0.2173334807,-0.7314818501,-0.7883370519,-0.1343939155,0.4867017269,-1.6478499174,0.8052513599,1.0267293453,0.0742363185,0.0273618270,1.4169248343,1.9043926001,-0.3838944435,0.5245365500,0.2650912702,-0.3235317171,-0.8705883622,-1.5474305153,0.6036111116,0.0714829117,-1.1413797140,0.1435114741,-0.6295248866,-0.0765427724,-0.1058588624,-1.4466910362,-0.6485618353,0.0132218618,1.4305149317,0.3642526269,1.3517880440,1.3447993994,0.7358057499,0.7414743900,-1.2427915335,-0.3946131766,0.1648515463,1.1895046234,-0.2594691217,0.8554831743,-1.0946022272,1.1648385525,-0.2886697054,-0.7935397625,0.0376145579,1.3619145155,-1.3953189850,-0.1312556565,-0.4305959046,-0.7347027659,1.2504189014,-1.1138132811,1.8914943933,0.6668202877,0.5314742923,-1.1446431875,0.8470588326,-0.3559998870,1.8328109980,-1.0825778246,1.5532661676,1.4346286058,1.6508401632,0.1166369244,-1.7578734159,-0.2210106105,-0.3382823765,-0.7606144547,0.7037070394,-1.4212046862,0.7512388825,0.1273766011,-0.0193244293,1.8327016830,0.1493382752,-0.0756782517,-0.0355336964,-1.6446069479,-0.8279816508,-0.0862731785,-0.6101303697,0.6401699185,-0.1602170914,-0.9755163193,0.3217153549,0.1617767066,1.2197539806,0.1922034472,-0.4725489616,-1.2373657227,-1.7496455908,0.5991289616,-0.5498912930,0.0673314482,0.8375106454,0.8581475019,0.9310656190,-1.3271347284,-0.1280384958,1.7297261953,-0.2735048532,1.1787731647,2.2917857170,0.8384599686,0.9903156161,-0.2916140258,0.3382817805,1.0331203938,0.3903645873,-0.6004440784,-0.5967748165,0.9446532130,1.4222265482,0.0261503533,-1.4113605022,-1.8482435942,-1.0479794741,0.6841892600,-2.3524160385,-1.1195981503,-1.0631546974,0.3228687644,1.1693851948,2.5934445858,-0.4578936994,0.1776099205,0.0838143975,0.8039101362,0.5364631414,0.1926894188,0.4836985767,1.1132575274,0.1154624745,0.1372138709,-0.2826304138,-1.1644810438,-0.0809413269,-0.4451850951,-1.1100091934,-0.5838759542,0.3142054081,-0.6515339017,-1.3535983562,0.3675374985,-2.6818180084,-0.4444566667,-1.1640266180,0.0140349520,-0.3826368749,-0.7109439969,0.6577385664,0.0257320683,-0.4784716666,-0.6491413116,-1.1920108795,0.7272107005,-0.4145368934,1.3471291065,0.3028367460,1.9280700684,0.8856548667,-2.2657513618,-0.1738021672,1.1121703386,0.8565863967,0.0598084405,-1.1367391348,-0.3166633844,-0.4727935791,0.1086008698],[0.0661494732,-0.5598045588,-0.2818463743,0.7618486881,1.0420223475,-0.0124819009,-0.2460182905,-0.2402713597,0.2803384960,0.6347672939,-0.7395008802,0.3717006445,1.8899036646,0.7324008942,-0.2531517148,2.1485793591,1.1714525223,0.7662360072,-0.7396721244,-1.2587963343,1.2214463949,-1.1060516834,0.2466125786,-0.0663431361,0.5754920840,0.0819447786,1.2410321236,-0.0663276836,-0.8556157351,0.1471997797,-0.1371292323,1.4430338144,0.1633171737,-1.1141107082,-1.4421511889,0.2584901750,0.1109801009,0.5133917928,-1.6274529696,-0.3198641539,1.5091960430,-0.5411385894,0.4407715201,1.6539150476,-0.1996767372,2.6565704346,0.0422839411,-1.1861289740,0.6739498377,-0.0582477152,0.0878551304,-0.0720350966,-0.9809566140,-1.0485097170,0.1298928559,1.5011910200,0.3872312605,1.3922791481,-0.8527609706,-0.1992137879,-0.3257340789,-0.2213805318,-1.5234873295,-0.1974401027,-0.8294673562,1.2558363676,0.7330928445,0.6681995988,0.8794973493,0.6108455658,0.5544348955,-0.6794480681,-0.7796459198,-0.7026695013,-2.1434984207,0.0411658436,0.3642963171,-1.1052713394,0.2658634484,-0.4520622790,1.2095198631,1.5577833652,0.3189541698,-1.2588919401,0.8697476387,0.2926436365,1.0497710705,-1.7970030308,-1.8265334368,-0.0994349569,0.3744624853,0.2839959860,0.5508910418,0.5231243372,0.7695473433,0.2353003919,1.0889656544,-0.0385504179,-1.6064237356,0.2855109274,-0.4909099638,1.2451142073,0.3549577296,-0.1303832978,-2.4352939129,0.5502526760,1.4097541571,-0.8143534660,-0.4583834112,0.5625880361,-0.0530144200,-0.0092057148,0.0317580067,-1.3821700811,-1.1400830746,0.2341952622,2.0570058823,0.6172062755,-1.9741103649,0.9930515289,1.0489962101,1.2614012957,-2.0655992031,-0.6154462099,0.3555959165,-0.2700898647,-0.9979034662,-1.8507244587,0.9184408188,-0.3622143865,-0.1437021047,0.9383545518,0.0250540581,0.5107842088,-1.4865230322,0.6810082197,-1.9815660715,1.1557435989,0.1040510312,0.8280178308,0.8028651476,-1.1024335623,-0.7035324574,-0.0274420045,-0.6655241251,-2.0715312958,0.2726945877,-0.6792376041,-0.1231326684,1.4458146095,0.2314967811,-0.6573718786,0.6712464690,-0.7653050423,1.4972485304,-0.9735265374,1.5854916573,0.2542545795,-0.1565025300,-2.3489050865,0.0153308529,0.7762720585,-1.4138087034,-1.5466600657,-0.2691728473,1.6029889584,0.7616153359,0.9480590224,0.9128962755,-0.7814031243,0.6315351129,-1.1406873465,-0.0793438703,0.7408534884,0.3598624766,-1.6224952936,0.7605880499,-0.4141521752,1.9717255831,-2.0489830971,0.3203971088,-0.5467629433,-2.4523425102,-0.9296472073,-0.2040585130,-0.5218498707,1.4715273380,0.9998838305,-0.4258372784,-0.1646424234,-1.0052925348,1.8012080193,0.7845478654,0.8161594868,-1.8706011772,0.0538863353,-0.5203756690,0.3057786524,-0.3377862573,0.2710774839,0.1396749020,-0.0968563706,-1.6479283571,-1.2091325521,-2.3561117649,-0.0236232337,-0.4046415389,0.0204066318,-0.2489682585,1.7718049288,0.7430356741,0.3051402569,-0.3081843555,0.6530711651,0.6653307080,0.9015319347,1.2185962200,1.2392010689,1.0117390156,0.1708698869,-0.0968572199,-1.0288013220,-1.4395042658,0.4888134897],[-0.1481398195,0.2349804938,1.0387753248,-0.8918272257,1.1819578409,1.1469008923,-0.8863270879,-0.0768607333,1.6103775501,0.2564200461,-0.6540455222,-1.4819751978,0.3392748833,2.3457913399,-1.9290598631,-0.5931926966,3.8038439751,0.0150417322,0.4764592648,-0.7765587568,-0.1422771513,-0.0276196878,-1.0998263359,1.0197985172,0.7770087719,1.2607687712,0.3303723335,1.3598996401,-0.1502128541,-0.9660602212,-0.2033868581,1.8268421888,0.7102911472,-1.2924709320,1.4800399542,1.2848676443,-0.6317282915,-2.0580847263,0.7169486284,0.8882303238,0.9122008085,-0.0191191155,-0.9266655445,-0.7308059335,1.1142582893,-0.4701244533,-1.5005794764,-0.4836395681,-0.5511965156,0.8093124032,-1.2713971138,0.4431983829,0.5598557591,0.1382537782,0.5284568667,0.0855464339,1.0339580774,-0.9627118111,-0.0867765024,-0.5620479584,-0.0449695885,-0.5436484218,-2.0981886387,-0.2393644899,-1.0797698498,-1.1307771206,0.2753635347,0.7339200974,-1.4783803225,0.0846191719,0.4496826828,0.4637614489,-0.5138217807,0.8895533085,0.4025216401,0.4540904760,-0.2553262711,0.0476168953,0.3607804775,-0.9458797574,0.5130068660,1.0887874365,0.0324329808,1.7249981165,-0.5506155491,-0.2265087664,0.1730436236,-0.8672905564,-0.9517143965,-0.6819998622,1.8925633430,-0.5262590647,0.2710691392,-0.0368213281,0.2499300092,0.1145922467,0.4249081314,0.5221466422,-0.6290580034,0.5858543515,-0.7500481606,0.4885399044,-1.0888670683,-0.1751397997,-1.0800218582,-0.5044698715,-0.6824820638,0.3367467821,-0.0662877113,0.0004860878,0.3974931836,0.9878262281,-0.2203817666,-2.3339583874,-1.3713185787,0.7010454535,-0.1792138517,1.6398990154,-1.3528527021,-0.0485312343,1.6457034349,-1.5228041410,-0.9622097611,-0.7212290764,0.1708365530,-0.1943217069,-0.1791012436,-1.5973223448,1.0700832605,-0.0263415724,-0.0783090517,0.5096049309,0.2359365672,-2.0904521942,1.3343380690,0.0267197248,1.0169812441,-1.6574496031,-0.7635102868,0.0978857651,0.8636631966,0.2661770284,-0.9758553505,0.6706265211,-1.1487759352,0.7276757360,0.0401194058,-0.8232668638,0.1121659577,1.8414586782,-0.5288928747,-0.5942555070,-0.0549913421,-0.3535532951,0.4613077641,-0.8000953794,-1.1293758154,0.7078111172,0.2381000668,-0.2260020673,-1.0287990570,0.6059779525,1.0500333309,0.4062401950,0.5371391773,0.0047277003,1.6705203056,0.4563495219,0.2244141847,-1.6428673267,-1.8438435793,0.3145678639,-0.4614353478,0.4700029194,-1.7696510553,0.0000749491,-0.0976910740,0.5977476239,0.2733864784,0.3712363243,0.7553363442,0.3672829568,1.1928751469,1.4054775238,-2.3258380890,-2.9297554493,0.2743716836,-0.7235479355,1.5969778299,-1.8682086468,1.4272466898,-2.3928434849,-0.7951687574,-0.3718027174,-0.2111870050,-0.4080311656,0.4804771543,1.6419860125,0.9514871240,0.6680909395,-0.5509938002,-0.0222789478,-0.1015945151,0.1699242294,-1.2516422272,-1.5921986103,0.0323502608,1.3214668036,-0.0185927413,-0.4885272980,1.0027142763,0.0604401119,-0.5876231194,-0.2673060298,1.1417536736,-0.5216576457,-0.9080795050,0.2505319715,1.5503072739,-0.5802096725,-0.5431979895,-0.8207826614,-1.8523241282,0.7910633087],[-1.5131099224,-0.5346738100,0.8551876545,-0.8079803586,0.3514728546,-1.0119205713,0.1222014651,-2.3043191433,-0.5708674192,0.5543924570,-0.8940060139,0.2141406089,-0.4124077559,-0.0373211876,-0.4836105108,0.9895735383,1.3710926771,0.3094007373,1.0810010433,0.7413520813,0.0272918269,-2.6059725285,0.7118025422,-0.3565561175,-1.5837817192,-1.8069742918,0.0314307399,0.7519264817,-1.6187287569,-0.0656537339,-1.2725286484,-0.0628013238,-1.2660244703,-0.2694690824,-0.9322804213,-1.7178848982,0.3884723186,1.1313183308,-0.8840425014,-1.5170679092,0.8701610565,0.6604436040,-0.3095566034,1.1559892893,-0.0074164211,0.9563302994,0.5253620744,-1.1275501251,2.1611838341,0.2815635204,1.0019308329,0.3967360556,-1.3517037630,-1.6757366657,-0.4909931421,-0.2150374204,-0.4196070433,0.0863581672,-1.1695873737,-0.2097769529,2.7132956982,0.6640202999,-0.3107252419,0.1927692294,-1.1766229868,0.5481172204,-0.0203474443,1.5921030045,0.8253238797,0.2750462592,-0.3395743668,-0.1995105147,0.2876706421,-1.2705643177,-0.4173485339,-1.0307558775,-1.9259363413,0.3010179400,0.5864936709,1.1382341385,2.5246179104,-0.5178236961,0.8482388854,0.9648278952,0.5560377836,-0.2276707441,1.0964823961,-0.2981500030,-1.1091046333,0.2975334525,0.6693788767,1.2441065311,1.2527660131,0.1742370129,-0.2687437832,0.8906718493,1.1863725185,-1.3319261074,0.1384135783,0.9704707861,0.6115167141,0.1373640597,-0.4339033961,-1.5552423000,0.6139218211,2.3210573196,0.5673650503,-1.5066421032,-0.9496950507,-1.8230872154,0.2817157209,1.3412139416,-0.3125414848,-0.7926390171,-1.8839703798,-1.5029090643,-0.4812536836,-0.8076776266,0.1639763117,-0.7107151747,0.9037547708,1.1245650053,0.4608503878,1.4936001301,0.8328832984,-0.1224921197,0.8565199375,-0.8202630877,-0.1175216883,2.6427290440,0.7497962117,-0.1043440327,-0.5497339368,-0.1805608422,0.2127023935,-3.0390629768,-1.0456823111,0.2500087917,-1.0225489140,-0.7522569895,0.1320671141,0.0960535109,0.2754767239,0.7104935646,0.4646618962,0.8646464944,2.8653013706,-2.1563317776,-0.9281902909,0.3972793519,-0.9229031801,-0.2998534441,-0.3409955204,1.2158575058,-1.6193940639,-0.4401818812,0.1239653826,-0.1996781826,-0.8049672246,-0.3598818183,0.2379658520,0.7701300979,-1.6182599068,-0.8581345081,-0.2470516115,-1.7491062880,1.4807767868,-0.5614758134,1.2209767103,0.2904929221,-0.0265204366,0.5695201755,0.1617171317,0.7726086974,1.0486824512,-0.3704072833,1.1730215549,-0.5312774777,1.3397569656,-1.0443192720,-1.3226132393,-0.2041894644,-1.5650993586,-0.0995839760,0.2208478004,-0.4249693155,0.9977360368,0.5591554642,-0.0693533272,-0.9527857304,0.7313643694,-0.1154942065,0.6147226691,-0.5344278216,-0.4461407363,0.6166067719,1.1970794201,0.3441028297,0.9995790720,-0.1399023682,-0.2110286802,-2.0446467400,2.6426510811,1.7442144156,-1.2394208908,-1.1079133749,0.2935610712,0.0725700185,0.3695668578,0.5065841675,-1.0672954321,0.5645725131,0.7228435874,-0.8897636533,0.4221113324,-0.4366683066,-0.4785924852,2.0385699272,2.5463235378,-0.9433593154,-0.9427928329,-0.7519164681,-0.6663823724,-0.3502293527],[-1.4659262896,0.2184054852,0.3883730173,-0.5445824265,-0.4340218902,3.7029385567,-2.9254779816,0.0942560732,0.7164333463,-0.4786452055,-0.1175122783,-0.6979387403,0.2932125032,-2.5538980961,-0.6223775148,0.2257152498,-0.3763697743,0.8137111068,-0.5992847085,1.3243182898,-1.2376658916,-0.0529220700,0.3752489388,-0.2927281559,-0.8624119759,-1.7237237692,1.1811594963,-0.3744150698,-1.1541925669,-1.5252789259,1.3609256744,0.3352670372,-0.7940573692,-0.5847789645,0.0678094849,-0.2327625602,-2.0279872417,0.0251677223,-1.4056166410,-1.6464124918,-0.2422740161,0.8370131850,1.5657683611,-0.4537927806,0.7318181992,-1.0169379711,0.7392541170,-0.3775894642,1.7589201927,-0.9612664580,-1.5072410107,-0.9849436879,0.2215117812,-1.2333002090,-0.1126676127,0.1513326466,1.5155856609,-0.8208940029,1.8061890602,0.1750951260,0.0165317189,-0.0252383649,1.7736938000,0.5884373784,-0.9119014144,-0.6989759803,-1.0164759159,0.0299540386,-0.0303175766,0.5049239397,-0.6621921062,-0.2147170007,0.2130482644,-1.0169265270,-1.0796117783,0.9682517052,-0.2484214753,-0.0983709693,0.4231301248,0.9714114070,-0.3323786557,1.5941475630,0.0305444896,0.4641273916,1.1204073429,-1.7273485661,-1.9588909149,-0.0995916799,0.4214471579,0.2391090691,-0.8550423980,0.5174822807,0.6435247660,0.7362871766,0.2990854383,1.6299250126,0.0470980965,-1.4124903679,2.0001816750,2.4043455124,0.4756488204,0.3924701512,0.3098393083,-0.0730008855,0.0644879714,-0.5879397988,3.1723389626,-1.1309983730,0.4812060297,-1.0504877567,1.6183617115,-0.6508139372,-0.3729372919,0.8800085783,-0.1217752844,-0.7541463971,-2.0912880898,0.0173039250,-0.3026672900,1.9386875629,-0.5450688601,-0.8840935230,-1.0172849894,0.2282519937,-0.7308015823,0.0178454760,1.1036424637,1.0490378141,0.0145709217,-0.6570018530,0.6453847289,0.8068607450,-0.9951968193,1.6353204250,0.2871158421,-0.1186865047,-0.7178679705,-0.2702004313,0.7870482802,1.2354224920,-0.5233058333,-0.9856014848,-2.3766124249,-0.3460740149,0.8292262554,-0.1120812893,-0.5926383138,1.5636755228,0.5033448339,0.0718692541,0.0410754010,0.3172487020,1.0507453680,0.0159926750,0.4478392601,-1.4229664803,0.7920886874,0.1866728514,-0.8899513483,1.5150766373,-0.7372040749,2.3344931602,1.7175177336,-0.2363054156,1.2435344458,0.1209539473,2.1040270329,-1.0785930157,1.0357654095,-1.2035720348,0.6071705222,-1.2331465483,0.0831739157,0.5158026218,1.2514107227,0.5857951045,0.5511865616,-1.0304999352,-0.7450944185,0.8752056956,-0.8220462799,-0.5751981139,-0.3852460086,0.4066129625,-1.4922865629,0.5390688777,1.0325790644,-1.0658810139,0.8860808015,-1.0557252169,-0.3435255885,-0.3831379414,-1.5366101265,0.7045727968,-0.4226921499,0.2454482317,-0.9025840163,0.5418877006,-0.0653132945,-0.6542792320,-0.2471905053,0.6364162564,1.2548311949,0.8018607497,-0.0924706161,1.7474905252,-0.0242728647,0.7594467402,0.2591527998,1.2631505728,-1.0255336761,0.5002287030,-1.4737606049,-1.7188454866,1.5862935781,-0.7704221606,0.2158858180,-0.6859569550,-1.4657487869,0.1420394927,-0.7383154035,-0.6743084788,-2.7707183361,-0.1151084825],[-0.6279640198,-1.6584491730,-1.2487521172,-0.1758100241,-0.2976056635,0.6281543970,-3.0891396999,0.6630514264,-0.9883195758,1.5522327423,0.6034018993,0.2812715769,-0.2210479379,0.0146459658,-0.3543334901,-0.4833302498,0.9713264704,-0.2638811171,-0.6870847940,0.2799275219,0.2579723001,0.0368682183,0.7310587168,0.2262041271,-0.2016255260,1.0671253204,0.0625227988,0.6604328156,0.6529068351,-1.1744426489,1.3701006174,-1.0094944239,0.5174626112,-0.8501479626,-0.3785215020,0.2888900936,0.8252806067,1.7142086029,0.7759332657,-1.7127082348,0.5845321417,-0.3192206919,-0.1254154444,-0.1918734610,1.2321667671,2.6393880844,0.2824049890,-0.6123250127,-0.3582670987,0.9553580880,-0.5618331432,-0.2247440219,-0.8032472134,-0.4080768824,-0.9748763442,-0.2706478536,0.5201159716,0.0428278558,-0.6018859148,0.7460201979,0.0247440431,-0.0502817854,-0.7423041463,-0.8392975330,1.2616049051,-0.9946495891,1.9058818817,-0.1542883366,-0.1210363731,-1.2415443659,-1.2345449924,0.9470774531,0.3019158542,0.8561933041,-2.6018064022,-0.7974361181,0.2222371846,0.4300654829,2.8669002056,-0.5035877824,-1.6985487938,2.0215651989,0.1090312600,-1.6191835403,-0.0101143373,1.0134818554,0.4818102717,-1.9236172438,-0.6230416894,-0.1731863171,1.4452866316,0.1037637815,1.5183800459,-0.4374295473,-2.5311095715,0.2603306174,-0.4341614842,-0.0227584727,0.0439706743,0.5057605505,-1.7936534882,-0.5852804184,0.1672791690,0.8425058126,-0.6111262441,-0.9124225974,-0.2337473482,-0.0449558981,-1.8964269161,-2.0928378105,0.8949176669,0.9105094671,0.9112442136,-0.4834975004,3.5092229843,-0.5729820132,2.0387670994,1.0817768574,0.3173472285,0.1624213457,0.8563386798,0.4270950556,-0.3292297125,1.1796474457,-0.0568520613,-0.6135614514,1.0171507597,-1.0845253468,-0.0473044775,1.4754977226,-0.0622475371,0.7117826343,-1.0041016340,-0.9487560391,-0.0518494286,1.4331901073,0.2149605602,-1.1410475969,-0.6128460169,0.5419374704,0.3689844310,0.8537623286,-0.7307296395,0.4909996092,1.1410055161,-0.7519097328,0.8367847800,-0.4134968519,-0.3831160069,-0.1194213182,0.2132193446,0.8922790289,1.1994528770,1.8140201569,-0.5317217112,0.7226940393,0.4397925436,-0.4289795458,1.2970182896,0.5386045575,-0.3921145797,0.3799457252,-0.2168604136,-1.2815352678,-0.5558221936,-0.6706677079,1.1508843899,0.8142685294,0.0629897863,1.0782709122,-0.3097993731,1.1608967781,-0.0450324751,0.9269484878,0.1099951491,1.0994768143,0.0873215795,-0.6024252772,0.0804315135,1.1702136993,1.2390604019,0.0422957130,-0.2660755217,0.0252159797,0.3653047979,0.6382434368,-0.1962671429,0.5976749063,0.2093169689,1.3053292036,-0.1818927377,-0.8006437421,0.7526998520,-1.2449376583,1.6166766882,1.4126682281,-0.9880897999,0.1460569501,-0.3867262900,0.8620718122,-0.9725794196,-0.2764593959,1.1102123260,-0.8179458380,-0.4496610463,0.1712568700,-0.5782694221,0.7184853554,-0.7871074677,0.1874635667,-0.9851943851,0.9395116568,-1.3483319283,1.3027681112,0.5989563465,0.3927023411,-1.0175455809,0.7853148580,-0.4428822696,1.2068151236,1.2453286648,0.2258458585,1.1729338169,-0.3745219409],[-0.1013393402,0.7370291352,0.2293027043,-0.5496422052,0.4026574790,1.7350859642,0.8466253877,1.6243348122,0.2167896926,0.0440245979,-0.1143867150,-2.1155114174,-1.3466324806,0.3417681754,1.0367302895,-0.0448520556,0.1536931545,-0.1178123504,0.0942037478,-0.2840364575,-0.8906734586,1.0415467024,-0.3474560380,1.7721912861,-0.6977778077,-0.6533368826,0.2706574500,0.4579048455,0.5070598722,0.2563111782,1.0382679701,-2.1179025173,0.1377092451,0.0209780280,0.9947603941,-0.0382568911,0.2360651493,-1.6812630892,0.4709948599,1.8487871885,1.1363065243,0.6794203520,1.6432129145,0.3331112862,0.4847803116,-0.2373197377,0.8562826514,-0.8727971315,-2.7746782303,-2.3559737206,-0.2040310651,-1.7742786407,1.9584763050,0.7933133841,0.6449356079,1.4588258266,-0.0484815538,-1.0195758343,0.2607858479,0.3664327860,-1.2107329369,-1.9957568645,-0.4150539041,1.5697511435,1.9711836576,1.5477201939,0.0248154551,-0.7006254792,0.6331615448,0.5916293263,0.0568252914,0.1661783904,0.2813738585,1.0866055489,-1.0118426085,-0.0859973654,-0.7634933591,2.1033339500,0.3371011317,-1.1005963087,-1.3655861616,-0.8443711996,1.1577920914,-0.6241202950,-1.8529772758,-0.6705492735,-0.3144679070,-0.1968949288,-0.2270013392,-0.1578439176,-2.0221607685,-0.1185465753,0.4237858653,-0.0993085131,1.3173325062,0.1934987605,-2.9459500313,-0.1461422294,0.2651478350,-0.8410601020,-1.7123898268,0.0538811646,0.0466864482,0.0800267011,-1.7519488335,0.2975313067,1.4007811546,-0.9453305006,0.5971914530,-1.0898880959,-0.7380421162,-0.4164339900,-1.2008498907,-0.6115810871,-0.1443822831,1.1616984606,1.1686754227,1.2920237780,1.4396482706,1.3408201933,0.4317942858,-1.3623890877,-0.7680445910,0.5570045114,0.4734558761,1.1375391483,0.5354587436,0.4446857274,0.5467852950,-0.1238997132,-0.6851717830,2.1651611328,-0.4368666112,0.0831865296,0.0198221132,0.4435079098,-1.6825557947,-0.1355063766,-1.6379758120,-0.2781943679,-0.2904914320,0.8691003323,0.2901981473,1.5223109722,-1.3436995745,0.4895364940,-0.6511492133,-0.3522801995,-0.8010972738,-0.3451113701,-0.1639397591,0.6182901859,-0.8302804232,-0.0971979052,0.1578138769,1.5443570614,1.3465884924,-1.1086828709,1.7096700668,-0.1851585954,-0.0696520433,0.8750807643,-2.0553681850,-0.8725947142,0.6036568880,1.8323498964,-1.1572742462,-0.2411930561,0.6965112090,0.6085639000,-0.6458541751,0.1013178676,-1.0230764151,-0.8841358423,0.6133187413,1.1686683893,-0.1828366816,0.1179393306,0.6337798834,0.4901778698,-0.9213451147,1.1282690763,-1.3957413435,-0.9345256090,1.4516988993,-0.3256015778,-1.7746162415,1.1917959452,-1.2848377228,-2.0234019756,0.6253730059,0.5754631162,1.0525467396,-1.4490629435,-0.5101265907,-0.5352268815,-0.1461670250,0.4182570577,-0.8196110725,-0.2917360961,0.1357769370,-1.4317878485,0.5729535818,1.0932507515,0.9999333620,0.9448190331,-1.1056905985,1.2392117977,0.0591306388,1.4897656441,1.6061283350,-0.6775377393,-0.4090647399,0.3424934745,-0.9212122560,-1.3457553387,-1.0322591066,0.2986054420,-0.8338737488,-0.0210380573,1.5235228539,0.1132932231,-1.1214870214,0.2483399361],[0.6102077365,0.1721328497,-1.2462910414,-1.4098676443,-0.0068863430,0.6785459518,1.0933088064,0.0307702217,-0.8330343366,0.0852738246,-1.6568013430,0.0855158269,-1.0569945574,0.3314259350,-0.7124000192,-1.2863686085,-0.0895156786,-0.0622664876,0.4961340427,0.1014336422,-1.3754044771,1.3578120470,-0.0705222040,-2.0296418667,-1.0720357895,0.5115970969,1.0631728172,-1.0660487413,-1.4533889294,-0.2905147672,0.2778032422,1.8945484161,0.6868228912,0.5472155809,-1.9280380011,-0.4031965435,-0.5989333391,-0.2061021477,0.8257248998,0.9506226182,-1.1330982447,0.4963971674,0.5451398492,1.2906789780,-1.5282840729,-0.8242005706,-0.0166608077,0.0335856453,2.2408883572,0.4733282924,-0.7696737051,-0.8335739970,-1.4651881456,0.9389994740,1.8702383041,0.0202532560,-0.4591764808,0.2830611765,2.2367684841,-1.4029413462,-0.1545487493,0.2303007990,-0.1837261021,-0.8145059943,-0.0218058843,0.0932727531,1.0467610359,0.7866660357,0.4435950518,-0.0753492266,0.3163510263,2.2630624771,-1.5913091898,-1.3097940683,-0.8521066308,2.0647218227,0.2911355495,-0.9971901178,-2.6643984318,0.1316338181,-0.7985268235,0.1305931062,-0.2064609826,1.2719758749,0.5087019205,0.2554793358,0.6482380033,0.7646680474,-0.6734172106,0.2700004578,0.0559221543,-1.0332484245,-0.4205034673,-0.9777092338,1.4239164591,-1.2049342394,1.1088242531,1.6507546902,0.2748976350,-1.1399236917,-0.5846256018,-0.4150563776,0.1476821899,-0.7782986164,0.9938122034,0.3498940170,-1.1976610422,0.4588616490,-0.3367350996,-0.4915359616,-0.2937815785,0.7800746560,-1.1308561563,0.7726313472,0.3986551762,0.5029765964,1.6762688160,0.4709570110,2.0376982689,-0.7127743363,-0.1392951310,-0.9906319976,-0.2955588996,0.2931172252,1.2992885113,1.7970376015,0.4209524691,0.9865259528,-0.9265616536,0.5395507813,-0.2611883581,-0.0639136136,-0.0800991580,0.5079711080,0.2602309883,-0.4143621325,-0.9424008131,0.6740956306,-0.3937194943,-1.0629720688,0.5936015248,-0.6519501209,1.1757981777,-0.7144102454,-0.1102670729,-0.3803231716,-1.3536223173,-0.8384763002,-1.8306593895,-1.6056002378,0.2277152538,-0.5050942302,-0.4376785755,-1.3809183836,-0.9032619596,0.9237828851,0.5372219682,0.2870643735,-1.5153040886,-1.6259210110,-0.0451527424,0.2230039835,-0.7418859601,-0.4045087099,-0.1136229634,1.6879923344,-0.9585251808,0.0452248342,-1.4134316444,1.5886644125,-2.2719659805,-0.8550316095,-2.3864667416,-0.4699884653,-1.4324147701,0.2486818582,-1.2270326614,-0.6884349585,-1.8654394150,-1.2348514795,0.4219813049,-0.3581189215,-0.3238225877,0.7833189368,-0.1920600682,0.7204907537,-0.1709788293,2.3878262043,-0.2370357811,0.9252066016,0.4822584987,0.6451716423,-0.6012758017,1.0694143772,-0.1956869662,-0.8577533960,0.4864356816,-0.2107046247,0.6681656241,-0.8437408209,1.2435525656,0.6121364236,-0.5362357497,-2.3874721527,-0.7291497588,0.6766259670,0.3310092092,2.5184285641,-0.9417285919,0.3908450902,-0.9795922041,0.4405451715,1.6861338615,-0.8826841712,-0.3646148145,-2.1359474659,-0.8896039128,-0.8913089633,1.8602352142,-0.0924306363,-2.5201990604,-0.4996497631,-0.7356934547,3.0227034092],[-0.2779353559,-0.9125910401,-0.5229797959,-0.3131267428,-1.1275002956,1.0743371248,-0.5666452646,0.3104890287,0.6701961160,-1.6568621397,-1.3614599705,-0.3111041188,0.1598366499,-0.3078313470,0.1877021044,1.6093233824,1.2444223166,-0.2770603597,0.2751346529,-0.3467116654,-0.6917365193,0.5257941484,0.7689983845,0.3357297182,0.1524437368,1.6918798685,0.3152635694,-1.0607805252,0.6609638333,0.3341110945,-1.0415834188,0.8687285781,0.8544582725,0.1155287027,0.3381574154,1.2941756248,-0.2241853923,0.5553392172,0.8699635267,1.2132816315,0.1029268280,-1.5269670486,0.1696930826,-0.3773308396,-0.1566898823,-0.8203327060,-0.5906712413,-0.5726389885,1.7057299614,1.0661014318,-0.8808380961,1.9740517139,-1.5351676941,0.0559960082,-0.1033231318,0.2611945570,1.6715939045,0.6242154241,-0.9938521385,-0.0939270034,-0.6858149171,0.3675427139,-0.1992009878,0.6083980203,-1.5106667280,0.5414758325,-1.5780515671,1.3849695921,-1.3922752142,0.1662127376,1.1939231157,0.1069043130,0.0348040052,-1.3020936251,-0.7385377288,-0.9714508057,-0.1246909872,-1.2599545717,-0.5691602230,0.0913027450,-0.3601581752,-1.2885582447,-0.6923422813,0.0218632966,1.1481493711,0.3199811280,0.6208224297,0.1437385976,-0.0351644456,-0.2080980837,-0.7999038696,-0.0143128783,1.8700890541,-0.8725613356,1.9158043861,-0.5309497118,0.5074033737,-0.2050864398,1.8342417479,1.3920772076,1.2333165407,-0.0904451087,-1.6107214689,-1.2150744200,1.6213306189,1.3490490913,-1.2004847527,0.4088242948,1.6381580830,-0.0337113850,-2.1315181255,-0.6664378047,-0.3451897502,-0.5859259963,-0.7312455773,-0.8802397847,1.0734250546,-0.2296246886,-1.1476899385,0.0753600448,-0.1252810806,0.5062884688,-0.0693661049,0.0646183267,1.8327145576,0.6647765040,-0.3912880421,2.0176594257,-0.6505314112,1.8023506403,-1.4049191475,-1.0420913696,1.1070687771,-0.5325237513,-0.3083402216,-1.1807949543,0.1285699606,0.1529561281,-0.1071198657,0.6465570331,0.2372447997,0.6004552245,-0.3670341671,0.1296212673,-0.7611243129,-0.0786540359,0.7158681154,-0.4898069501,-0.0783000439,-0.2104012817,-2.9151592255,0.0618991032,-0.8712686896,-0.9809957743,-1.3866854906,0.2422163934,0.7125367522,-0.1848194599,2.3176703453,-0.4459954202,0.5021077394,-0.2677321136,-0.6174522042,0.0102121448,-0.8748270869,-0.5692952275,-0.2101370692,-3.1677110195,-0.9791932106,-0.4656033218,-0.8843027353,0.0018855287,1.6177743673,0.7415726781,1.2843861580,-1.5041314363,0.3272034526,1.0130852461,1.0509754419,0.1386350840,0.1534719616,-0.8860257864,-0.2906185389,-0.4992392659,1.0161581039,-0.3753769994,-0.8682446480,-0.9607298374,-0.9157308340,-2.2028121948,-1.2302342653,-1.0565544367,0.6787397861,0.4913366139,1.1512709856,0.2273500413,-0.3140943348,0.9650218487,1.1202888489,1.4143911600,0.1416300088,-0.6327657104,1.1599075794,0.9035273790,0.3643698692,-0.1275601387,-1.2958214283,0.4469485879,0.1940546483,1.2936275005,0.2017951459,-0.8351511955,-0.0646853894,0.1063179448,-0.7791896462,1.3193002939,-0.0662499070,-0.1504552513,0.5341873169,-0.4398691654,-0.0537598990,-0.8293182850,-0.8617116809,0.7136631012],[-0.3939204812,1.5737469196,-0.0540837012,1.4036841393,0.1378640980,0.9466567636,-0.7069780827,-0.4565291703,-0.1137186959,-0.1762137711,-1.0199981928,-0.2759044766,0.7262592316,0.2267369032,1.0947389603,0.2019783854,0.3519686162,1.2753775120,-0.0727279261,-1.5853526592,2.0832924843,0.9841842651,-0.9553976655,-0.1030969620,0.1650699079,-1.0748144388,-1.8021926880,-1.3359909058,0.3166110814,-0.2559000254,1.7907859087,-2.5822634697,-0.7786031961,-0.8376648426,0.3916146159,1.1735594273,0.1578970551,-0.0654573068,-1.0033574104,0.4908643365,-0.3393624723,0.2656250000,0.5900098085,0.1851583868,1.1900016069,-0.0636883900,-0.6513484716,-0.1623256058,-2.4267354012,-0.3015221357,0.8572072387,-0.3075442612,1.3093880415,0.1426732540,-0.0542080179,2.3609998226,-0.5969130993,-0.9994596243,-2.0351839066,1.2536141872,-0.0141155524,1.1488707066,0.3340318203,2.5600366592,0.3605966866,0.6146651506,-1.4186687469,0.2727081180,0.0324210338,0.1732143015,1.2016236782,1.1515592337,-0.8048486114,-0.4166365862,-0.8107392192,-0.2680552602,0.7969276309,-0.2536909580,-1.2414793968,0.0307318419,-0.7246635556,-0.0614210144,-0.0980322510,-0.1273739636,-0.8733004928,0.8253334761,0.3285685182,0.2195947468,-0.4676854312,-1.7725604773,-0.0706478730,-0.8255471587,-0.8020666242,0.7796689868,0.0053337635,0.5974035263,-0.5471351743,-0.5485429168,-0.0285736788,0.0145894540,-0.8462850451,-0.3286907971,0.5849106312,-0.0293410309,0.3513242304,-0.6039078236,0.6773921847,0.8920910358,-0.3698989749,0.8443539739,-0.9569555521,2.1414206028,-1.4383490086,1.8881260157,0.2543646097,0.1823499501,-1.8379988670,-2.3626091480,1.9306805134,1.1447869539,-0.7892955542,-1.3583947420,0.1193050891,-0.5391947627,-0.8748807907,-1.0574221611,0.1377330273,-0.1817522049,0.3828250766,-0.6502447128,-0.4013140500,1.5177444220,-1.1044421196,0.4088673890,-1.6362469196,-0.0767254755,-2.1334729195,1.6754659414,-1.0141512156,-0.2841005921,-0.7273792028,-0.1547815204,-0.3518456221,-1.6101245880,0.3806446791,1.5120161772,-0.3957258463,-0.3997440934,1.0012264252,-0.5169299841,0.2179621607,0.0726020262,0.8056242466,-0.3302493989,-0.5640383363,1.6068236828,-0.8795584440,-2.0162665844,-2.8408839703,0.0471107960,-2.4121069908,-0.4298157692,-0.2259212136,-0.2872554958,-2.6355597973,-0.0566891581,-0.3242926300,-2.4704289436,0.1055299938,-0.9959416389,0.3086238503,0.9671957493,-0.5396192074,0.6040809751,0.5914162993,-1.0520744324,0.2196249813,1.1837112904,0.7854421139,0.2140046954,0.0103036715,-0.4791815281,1.0802093744,2.2894952297,0.5970520377,-1.0285168886,-0.8583711982,0.5456027985,-0.4148845971,-0.3353210390,0.1510142237,-1.1208082438,0.4330037832,1.5839723349,-0.4517824650,0.7348692417,0.6062211394,1.1804046631,-0.2156865895,-1.9402424097,0.7674831152,-0.6204615831,-0.9424259663,-0.7667592168,-1.7198344469,0.1882339567,0.3915354908,-1.2951434851,-1.4637438059,0.4099708200,-0.3732828200,0.2660309076,2.2915365696,0.8822026849,1.2864061594,-1.0476447344,-0.5300865173,-0.8190235496,-0.7607998252,0.6160025001,1.4188951254,-1.2633959055,0.9278722405,1.2578333616],[2.5271196365,0.2058767378,0.4043089151,0.9949845672,-3.1527338028,1.2946237326,0.9710022211,-0.1420307904,0.5478397608,-0.8206241131,0.3207755387,-1.1367980242,-0.3007896543,1.0672885180,1.1049232483,1.0061985254,-0.6842412949,0.4629309773,-0.6022585034,-1.8163396120,2.4398572445,-0.5302162170,-0.1656367332,-0.4852881432,0.3775443435,0.5550758839,0.1274569929,-0.5471289158,-0.3199679852,1.2949386835,-0.6124832630,-0.1371923089,-0.2731872797,0.6984583139,-0.9401183724,1.2994360924,0.8181514740,0.7731116414,1.0036376715,1.5827988386,-1.2974874973,-0.1434350908,0.6647178531,1.7826393843,-0.5892428756,0.2183041722,-0.9415503144,0.6296509504,1.4211704731,1.8153684139,0.4850921035,-0.8563700914,0.6953250170,0.7074752450,-0.9068002105,-0.7238519192,0.8457453251,0.6591275930,-1.3648034334,0.9885088205,0.9899643660,-1.4283225536,-0.2611402273,0.6861331463,0.6731824875,-0.9871585965,0.3236202002,-0.2243195921,-1.3043832779,-0.9144351482,-0.0288481209,0.3471129537,1.9157763720,1.3639667034,-0.0679871738,0.0099768415,-0.7806731462,-0.3309215307,-0.5239000916,-0.7487209439,-2.0122323036,-0.7074913979,1.0903810263,-0.1148329824,0.7587329149,-0.8750514984,0.1420132518,0.7049495578,0.9195414782,0.0968873128,-0.5733492970,1.3557649851,0.6867563128,0.5227616429,1.4819657803,-0.5568839312,-0.4434660375,-0.6686267257,-0.2037937194,-0.2320554256,-1.2747259140,-0.5040209889,0.1536800563,0.3864961267,0.1177268252,0.4147295058,-0.4290784895,-2.1149220467,0.1111558527,0.5690610409,-0.5074170828,-1.3116228580,-0.3538000584,0.9582650065,-0.4444060624,0.4389349818,0.6684995890,2.0895423889,-1.5145853758,-1.0995954275,-1.9046037197,1.4437774420,-0.3302248716,-0.9572323561,0.3254705071,1.1758222580,-0.3314490020,-1.9969503880,-0.2365534455,-0.0082780914,0.2744103074,0.0309559889,0.4002328515,-0.7249093652,-0.1263285577,-1.1013489962,1.3815333843,1.8019288778,0.0788859054,-0.8572096825,-0.7831746936,-0.6566607952,-0.5514259338,1.1561759710,0.0486259237,-1.4882011414,-1.7478359938,-0.4349352121,-0.5980690718,-0.0105753476,0.6441072226,0.8070641756,1.3366454840,-2.4222583771,-1.7629585266,0.4242415428,0.5227168798,1.2409411669,-0.9768437743,0.5091602802,0.4706636965,0.8562821746,0.5442659259,0.9753083587,-1.0318638086,1.0895196199,-0.3157981634,-0.2498181760,-0.0072620874,0.3074987233,0.7405095100,0.0923283026,1.5314687490,0.4712828100,0.7196969986,-1.2597471476,-0.8728388548,-0.7292687893,0.1062755734,-0.7407863736,-1.3908658028,0.0886164233,0.6232225895,-1.4311265945,0.9013275504,-0.3596685529,0.1025697961,0.0446388386,0.6425230503,-0.2347142994,1.9980522394,-1.0065909624,0.1190448180,-0.2422155142,0.0530792251,-1.4308447838,0.2798071504,0.6393198371,0.4337556064,1.0134214163,-0.7661970258,-0.3400665224,0.6894814372,-0.6211586595,-1.1139371395,-0.4133455455,-0.8308604360,0.1453806609,-0.7555692196,0.6801390052,0.8128879666,0.5958173275,1.0729752779,0.4843373299,-0.4979102314,1.3864185810,0.9193938971,-1.5006879568,-0.4991283119,-0.2648375630,0.4287530482,0.8467736244,0.9204974174,-1.7422208786],[-1.4175399542,1.3389828205,-0.1382659376,0.5003929138,-0.8001933694,-1.1260420084,-0.7873304486,-0.4730960131,0.1702415347,-0.4686396122,-1.4204618931,1.7457896471,-1.9309419394,0.6022258401,1.0969513655,0.2393837273,0.2116515785,0.5585256219,1.0960396528,0.9194073081,0.2306050062,-1.1204822063,0.5107620358,0.7941451073,0.1340462267,-0.8106143475,0.3599536419,0.5154025555,0.0600135475,-0.7232104540,0.3722366095,0.2504935563,-0.0045491043,-0.6716938615,0.2279936522,0.1172791049,0.5441622734,0.7014442682,0.1636167318,1.9247628450,1.3642557859,-1.3011333942,1.4473772049,-0.2376226336,0.6195828319,0.2934015989,-0.5696805120,0.0819147378,-0.0124860499,1.4523462057,0.4789505899,-1.6496356726,-0.0578171164,0.6523512602,1.5945521593,-1.2960835695,-0.6454660892,0.3898889124,-0.2493240684,-0.7321791053,-1.3518134356,-0.8230043650,1.4176563025,-1.4937674999,0.6788947582,0.0714109913,-0.1779415756,-1.9323011637,0.6027720571,-0.2723703384,1.1922283173,0.9429624081,-0.1027375609,2.0723633766,-0.8419672847,-0.7306387424,-1.5093916655,0.6934100389,-0.0410907716,0.7249060273,-0.1354019344,0.7563098669,0.3674355447,0.7718367577,-2.6049280167,-1.3262532949,-1.2036824226,-0.4017079473,0.4643644989,0.3264178038,-2.6617326736,-0.7910155654,-0.3290653527,0.2054002136,0.9232746363,-0.3793618381,-0.9501160979,0.2249486148,-0.6229797602,-0.9222994447,-0.8020499349,-0.1218004450,-0.1192131341,0.1981703192,0.2027932554,-0.5831612349,-1.5477751493,-0.6242799163,-1.9537978172,-1.3120753765,-0.8792384267,-0.2546037138,1.8232898712,1.9963171482,0.4631770551,-0.2157828957,-0.0033036820,0.9925291538,-0.6906385422,0.6020773053,1.8886865377,-1.2071678638,0.0356135145,1.9553414583,0.7453075647,-0.5721513033,0.3352523446,-0.8478366137,-0.7228087187,0.8866829276,-0.9401163459,-1.9854770899,1.0203789473,-0.4592664838,-0.5595240593,0.4180965722,-0.1041441485,-1.3117289543,-1.1925373077,-1.3618315458,-0.2454625666,-1.3739359379,-0.1230784133,1.3046419621,-0.3713450730,-0.1312476695,0.7927514315,-0.8698234558,-0.1680863947,-0.6188244224,0.3212174177,1.1077984571,-0.8575962782,-1.2911204100,-0.1478271186,0.8769837022,-0.4112337232,1.2420055866,-0.2285336703,1.3366562128,-1.6899101734,0.2851883173,-0.9931355119,-0.2546955347,-0.7716149092,1.0934343338,0.0426416881,1.5614147186,-1.0928755999,-1.6933665276,-0.4665859342,1.6828256845,-2.1104826927,-0.0419604629,-2.0320513248,1.1672431231,-1.0663675070,0.4055701494,2.9667816162,0.3135769367,-0.1272612065,-0.0019367305,-1.7865858078,-0.7973675132,0.0983067825,-0.7250729799,-0.1266469806,0.0654597431,1.6557376385,0.1442626268,-1.7148323059,-0.7256135941,0.2546228766,0.9104588032,0.4652565122,1.3994613886,-0.7664901614,0.1988223642,-0.6781148314,-2.5610532761,-1.2301489115,0.5389909148,0.4202900231,-0.1574928910,0.7148252130,0.0710679144,-1.8049743176,0.0158903282,1.6363402605,0.1195532903,1.5923948288,-1.0496184826,0.0179251097,-0.7340781093,-0.0044836081,0.2723886967,-0.1416033208,0.1716030240,-0.4047521949,0.2403499484,1.2817729712,0.3288176060,-1.0603730679,2.0009100437],[0.3797760308,2.0110740662,-0.2068622559,0.7860251069,-0.5047575235,0.7063904405,-1.1786493063,0.3923666775,-1.0244634151,1.8500113487,0.5193219185,-0.9555152655,1.3364324570,1.4916113615,-0.5875380635,-0.2249176949,0.5363004804,-0.4077191949,1.2315989733,-0.0209633969,-0.7538880110,-0.4608057439,0.1677348167,0.6560636759,-1.0530261993,-1.1236027479,0.5489697456,1.1629155874,1.0196343660,-0.0328121632,-0.7612766027,-0.3375090957,-0.0431881398,-1.3949629068,1.6447660923,0.6821311712,-0.2217063755,0.7762494683,-0.1715980470,1.7823339701,-0.4120734036,0.3940093219,0.4754644036,0.0328231864,1.2554707527,-0.5431787372,1.2304182053,-0.8765879869,0.9375509024,1.1339820623,-0.3679790795,0.9494005442,0.9975679517,1.0307329893,2.1102905273,0.7695582509,0.5826106071,0.3467871845,0.0634680912,0.7141479254,0.2264734805,2.0048527718,-1.0984022617,1.2317520380,-0.8595981598,-1.2654389143,-0.6585014462,-1.2234858274,0.8852560520,-0.4570990801,-0.5080608726,0.1632690579,1.0896167755,0.3817169368,-0.0259354860,-0.4530935287,0.2448261827,-0.0249542110,-0.5475708842,0.8203950524,-0.0454053171,-0.1523360014,0.1439412832,-0.6669614315,-0.2771725059,0.3978259265,-0.4932720065,-0.9815487266,0.0112280538,0.0622864105,0.1676025093,-0.3985056281,-0.7404897213,0.8559097052,-0.8888737559,-0.3696354628,0.7236633301,-0.8088818192,0.1112069711,0.8000018001,-0.2828109860,-1.3591109514,0.0597446077,1.1427609921,0.9351323247,-0.5741488338,0.8525617719,0.3621784449,1.7871932983,-0.1630001217,-0.4958277643,0.3296086788,0.1071355641,-1.2501461506,0.2655957639,0.9955238700,0.3480997086,0.5811783075,-2.4376964569,-1.1687315702,-0.0813406780,-0.5605207086,0.8873816133,-0.6016888618,0.6948810816,0.3314882815,-3.5329918861,-1.4900625944,0.9414265752,-0.0608544908,-0.2282129824,0.2961200774,0.2064784765,0.9235963821,0.8165992498,1.0421718359,0.6450840831,0.3433330655,0.4462094009,-1.2916414738,0.2994377315,-0.6783190370,-1.4601011276,1.0403785706,-0.2990740836,-1.3489027023,0.4652444422,-1.8370354176,-0.8784723282,0.9383602738,-0.9043878317,-1.5784652233,1.0723438263,0.2476393133,-0.8018630743,-0.4048002362,-0.0732393563,0.9851552844,-0.7021204829,0.6982021928,-0.6263809204,0.7441608906,-1.2284842730,-0.0449464247,-0.6855147481,-1.4545799494,2.1253712177,0.6591351628,1.0927282572,1.2526097298,-0.6281160712,0.4327580929,1.6550632715,-2.2945992947,-0.0502533279,1.5303075314,-0.5409997702,1.1727341413,-0.5400050879,-0.4364598989,-0.0967092067,-0.4712429643,-0.6747933030,0.5726812482,1.1124210358,-0.0922250971,-1.5018668175,-1.0531485081,0.0086183911,-0.0545431152,-1.5454921722,-1.6756819487,1.5852866173,0.0307595395,1.1762250662,2.2206773758,-1.2185657024,0.5149343610,0.9203640819,0.5588316321,-0.0089860512,-1.4906351566,-1.3091335297,-1.5451923609,0.4636520147,-0.8842917085,0.2086382061,-1.3636584282,-0.5502992868,-0.3902366161,-0.5923804045,-1.5842198133,-0.0137870312,0.0795252994,1.9029623270,-1.1827250719,0.2084009647,0.1835195422,0.4632900953,0.3212157488,1.2559412718,-0.1729954481,-1.1393396854,0.4833806455],[2.6549761295,-0.0624392033,0.2982380688,-1.1094950438,-0.4198685288,0.0068994844,0.3850246668,-0.5606642962,0.3175287545,-0.3674174547,0.4738435745,-0.5514103174,-0.9766073227,0.5305664539,-1.0486661196,0.2787618637,-0.8301656246,-0.1464786083,0.4386092722,1.0080722570,0.1367931962,0.6738983989,0.3898560107,2.6003527641,0.8472453356,0.8475563526,0.4127884507,1.4216381311,0.4300921261,0.4178426266,-1.4710381031,0.2721470594,0.8424003720,-0.2040570527,-0.8711925745,0.6280260086,0.2913725376,0.5029097199,-0.7581620216,0.1996175051,-0.8232809901,1.2004920244,0.5533089042,-0.8662698269,1.6752365828,0.5223246217,-0.0690548792,-1.2152057886,-0.0013798352,-0.4171872437,1.5942960978,-1.9580565691,0.9489240646,-0.5740866065,0.2111291140,0.9112172127,-0.2032184899,-0.8871902823,-0.3309149742,-0.7680382133,-0.2359905690,-0.9237763286,-0.0109956544,-1.9898606539,2.5647330284,-0.6217227578,0.0064217350,-0.1358610094,-1.1649965048,0.6981079578,-0.5372963548,-0.1233563721,0.2467169762,1.5969997644,1.5088599920,0.8951876760,0.3103065491,-0.2586609125,-0.0645775348,1.1478109360,0.7551135421,-1.5989308357,-1.0639597178,0.8607670069,-0.9049039483,1.0084027052,-1.0110167265,1.8463071585,-0.7842986584,-1.8329027891,1.6607512236,-0.1428315938,0.1744260192,2.4861826897,-0.1852011085,-0.3658530414,-1.5189673901,-0.8197247386,-1.4171614647,-0.7043625116,-0.0423857495,1.9210053682,-0.2739804387,-1.6898438931,-0.2022707313,0.5238206387,-1.7040828466,-0.6559479237,-0.2088008374,-0.4546749890,0.6832018495,-0.1399490386,-0.0158994384,1.2031900883,-2.3218762875,-0.8002865911,0.1409174502,-0.8941115737,0.1592972279,-0.7806494236,-2.2411077023,-0.6832921505,-0.2099120617,0.2427671105,0.5994983912,1.7587484121,-0.4734762311,0.9576066732,1.0439326763,0.3215130866,1.1165105104,-1.1664500237,-0.2707373798,0.8627887964,-0.5434683561,0.4097805321,0.8309146166,-1.2992010117,0.8051446080,-0.0502262339,1.2870852947,-0.0100108348,2.0543682575,0.3049731255,1.5535672903,1.2411839962,0.9481426477,-0.8539407849,1.0286288261,0.0647694692,1.0538676977,0.1396375895,0.6957244277,0.2910648584,-0.9009956121,1.1273261309,-0.0948203057,-0.8096640706,-0.4723818004,-0.6017938852,-1.5425246954,-1.3817493916,0.0491434112,1.0174921751,0.2119679451,0.1593198776,0.2184775919,0.0306259133,-0.0538404770,-0.8706930876,-0.2231848836,2.4912929535,-0.5015062094,1.0344709158,0.2444898188,1.4917860031,-0.6608645916,-1.4219645262,-0.2831400335,-1.3320243359,1.7224861383,1.0793416500,-2.1341109276,1.2406805754,-0.7852844596,0.0782570988,0.6343486905,-1.2520850897,0.7410355806,-0.6997129321,0.1451499909,-0.1370168477,0.9041714072,-0.1804198623,0.0712918416,-1.4128985405,0.7510366440,-0.1180425957,-0.1724923551,-1.7639014721,-0.0395201556,0.7621546388,-0.5221664310,0.7228161693,-0.1106093898,0.4198686481,0.8827266097,0.1813685149,-0.6062692404,1.4166896343,-1.0538520813,-1.7832826376,0.1536631137,0.7696893811,1.4886592627,0.2027274966,-0.3299671710,-2.1470396519,0.1910541207,1.2473070621,-0.6160305738,0.9013956189,-1.1003696918,0.0428022183],[-0.2240002900,1.3721675873,0.2639842331,0.9908525348,-0.6108818054,0.6185465455,0.8091144562,-0.0958564207,-2.4742417336,-1.4627320766,1.2571899891,2.1967086792,-0.6468477249,0.4778275192,0.4487043321,0.8870905638,-0.5353265405,0.8139606714,-0.9202001095,-0.9111188054,-0.4403496683,-1.0960507393,-0.7820392251,0.3184137642,-0.8955340981,1.3307368755,2.5789258480,1.6368669271,-0.1470758021,-0.5990946293,1.8969904184,0.0959065408,0.3653283119,-0.8044139743,-0.1963423491,0.6994704604,0.9716321230,-2.4430494308,-0.0891370699,1.6713135242,-0.0061685285,1.3528705835,-0.7780772448,-1.2525608540,0.5983073115,0.7732192874,-0.6087434292,0.9746813774,-1.3182982206,-0.8068457842,-0.0047311722,1.7748740911,-0.6146159172,0.4845753610,0.6594731808,-1.1879068613,-0.3992961049,0.6266974211,0.4242372215,0.9081152678,0.0307728127,-0.8630605340,1.9725608826,0.9605245590,0.5259451270,0.1010104865,0.9442236423,-0.9346194267,-0.3141328990,1.5740138292,0.8825586438,1.2932610512,-2.2646677494,-0.8570740223,2.5405399799,-0.9077283740,-0.0705576167,1.1092141867,1.1942255497,0.1073936000,-0.7081599832,-0.1220684573,-0.1524242908,2.7508671284,1.3673774004,0.2572293878,-0.7890204787,-1.9807367325,-0.0600517057,-0.3794493377,-0.2172252089,-0.1736208946,0.4301603138,1.1614133120,1.8514386415,1.1475257874,0.4402486980,1.6443941593,0.8660095930,-1.0846067667,-0.7280923724,-0.7247751355,1.1563705206,-0.6895301938,-0.2457161993,0.6665600538,-0.1587394476,-0.9480696321,0.6335174441,-0.8753589392,-0.3239900768,-0.4539276361,-1.1788517237,-0.0075284201,-0.3648636639,0.7042046785,-0.8619546294,-0.8080089092,-1.4834641218,-0.5682203174,0.0640545860,-1.3926751614,-0.4957585931,-0.9867656231,2.2077383995,0.8689068556,-0.1288353801,-0.2259429991,-0.3784595132,-0.1942718327,-0.4684728384,1.3292363882,-0.5330204368,0.9046695828,1.6821259260,0.0649134219,0.9101820588,-0.6161113381,0.5321943164,0.1378296018,-0.7966747284,-0.3984784782,-0.7223089933,0.1889540851,-0.3044525981,-0.5599074960,0.2821009755,-1.0995750427,-1.1855823994,0.1406255960,0.1761286408,-1.1918876171,1.7644279003,0.8393975496,-0.1007546857,-1.5601830482,-0.2472936213,-0.6002095938,-0.5832691193,0.5630375743,-0.2304146737,-0.5821666121,-0.8668646812,0.1507098377,-0.8167546988,-0.7604181767,-1.3662077188,-1.5394088030,-1.5751433372,-0.4525258243,-0.3810411990,-0.2660856843,-0.7493767738,-0.8499608636,-0.3706573248,0.0562004596,-0.0263963435,-0.0762054101,0.0347596668,-1.7043470144,-0.3087723255,0.0925105289,0.3055296242,-0.6038087010,-1.3270583153,0.9827371836,0.2591477334,-0.5748916268,0.2182596624,3.1320657730,-0.1381565481,-0.7192556262,-0.0967331678,-1.2724033594,0.1888975799,0.9075344205,0.4408209026,-0.5638342500,-2.3636741638,1.3404798508,-0.4956731200,-0.1394995302,-0.5335831642,-1.9022147655,0.7844576836,-1.7396374941,-1.0470457077,0.0059125903,1.1089094877,-0.9378197789,1.2084882259,0.6415619850,1.1708856821,0.9288407564,-0.4969819784,-1.6088763475,-0.2479011863,0.3180273175,-1.4168658257,1.8286755085,0.1195928678,0.7719916105,-0.7462855577,0.1041880324],[-0.9672582150,-2.6459031105,-0.1165833101,-3.1117084026,1.9623993635,0.8309456706,0.1377648413,-2.3812682629,0.6896886230,-0.1855984628,0.3543067873,0.9753615856,-0.6475732327,-0.8925844431,-1.0086665154,0.0782525837,-0.6183305979,0.6852272153,0.1687427461,-0.8541080356,-0.2065338194,0.4088042080,1.0348843336,-1.8947924376,0.0465237722,-0.4570004940,-1.2192322016,0.8203142881,0.3576781452,-0.2713727951,-0.0683235750,0.2432604730,1.3800150156,1.1533381939,-0.9297128320,0.5261302590,0.9035075307,-1.2131354809,0.2522619963,-1.5607283115,0.7069227099,-1.2718977928,0.5051910281,-1.1626993418,-0.0957609937,1.2475578785,0.4460627437,1.6918580532,-0.3250173628,0.6188752055,0.3587437868,1.6347857714,-0.5673042536,-0.6678325534,-1.5126785040,-2.0843534470,0.9316207170,-0.7939562201,-0.1389047503,-1.0928068161,1.3225532770,-1.9624245167,-0.5996420979,-1.2197872400,-0.9119546413,0.4276641309,0.0259381942,-0.4532708824,-1.7074309587,-0.8331914544,0.7614836693,0.0791674033,-0.6992869973,-0.8943839669,-0.2679222226,0.2194266021,-0.8759810328,-0.5073649287,-1.5110708475,2.2356791496,-1.3427168131,1.5245552063,-0.3868552744,0.1256234497,-0.1761559248,1.0805300474,2.1072673798,0.9131340384,-0.5150635839,-0.5568994284,0.1934793591,0.2294926941,0.0569185466,-1.5624272823,-0.3017155826,-1.1742011309,1.6733289957,0.3321471512,-2.5649945736,-1.0484580994,1.5249539614,0.6937811971,-1.4968557358,-0.0413185954,0.0577384606,-1.4827886820,0.6641077399,-0.4808337986,-0.5935848355,1.5506532192,0.2863505781,0.3010305166,0.7863500118,1.0970948935,1.0758353472,-0.6738184094,0.3312270939,-0.4622066617,1.8852907419,0.0494275317,0.6119221449,-0.9082134962,-0.2488493621,1.5805485249,0.5710447431,-0.0100161182,1.0434415340,2.6215713024,0.6912501454,0.0497087650,0.8760806322,-0.3577581644,1.4436415434,0.0282163415,-0.4429470897,0.0530390367,0.0762220472,1.1364022493,-1.6470911503,0.6877381206,0.8901814222,0.7812303305,1.3263591528,0.0477996878,2.5010333061,-2.1394217014,-1.9078707695,1.3412349224,0.4777317047,-0.0139821116,-0.1222880036,-1.0256145000,-0.3868508935,0.4203441143,-1.0873005390,-1.2217711210,-0.5821444392,-0.8056239486,-1.0757553577,-0.1141380593,0.7272791266,0.2058192939,0.0688363239,-0.2044557482,-1.1473109722,0.4968492687,-1.4591329098,0.1536498815,0.5128298402,1.3853830099,0.3292523623,-1.7917298079,-1.6369482279,0.8037596941,-0.3507469296,-0.9165260196,0.6277545094,-1.8771381378,-1.1168591976,2.0002963543,0.9762254953,-0.2480376661,-0.5302870870,1.9402929544,0.0267861206,-1.3505047560,1.7754563093,0.7663806081,-1.2515767813,-0.4576311707,-1.6291935444,-0.0677957162,0.6024122238,-0.9290607572,-1.2449533939,0.6113210320,-0.1610737294,0.0371984690,-0.2587880194,0.6847470403,0.3342157006,1.4733811617,0.6087473631,1.1020613909,-2.3822624683,0.6188970208,-0.6561591625,0.2269476801,0.2247478813,0.1287886500,1.0781495571,-1.5613390207,0.2265433371,-0.3973863423,0.8339469433,-0.6040529013,1.7165282965,0.7048300505,1.7899045944,0.2359161973,1.1899683475,0.9303969741,-0.5528115630,0.8526928425],[0.4013674855,0.0542528853,-0.9132806659,0.2976969779,-0.4440785944,0.8972502947,0.8655430675,1.1715748310,-0.1977638453,-0.2246027142,0.0634627566,-0.1137804464,-0.7993347645,-1.0984220505,-0.6201526523,1.0808788538,-1.2932568789,1.0611051321,0.5813468099,0.4603781402,0.5116732717,-0.1236280203,-0.3366009891,2.3701324463,-0.2713805735,-0.9819604158,2.4698693752,-2.2890300751,1.6352475882,2.1706984043,1.7595663071,1.3014839888,-0.3608497679,1.2908350229,0.0412799045,1.0770276785,-0.8863621354,1.3306670189,0.6597825289,0.4063103199,0.0115959998,1.3941509724,-0.8768700957,1.2024830580,-0.2052872479,-0.3123206198,-0.1067714840,0.6414546371,0.4451971352,0.8133208752,0.4083485901,0.7939832807,-0.4251754880,0.4888560474,-1.1338469982,0.3686774671,-1.5876305103,-1.1317766905,0.3729013801,-0.0210948307,0.1209617406,-2.4760849476,1.0929088593,-0.2816660404,1.1428662539,0.4579641521,2.0584762096,-0.6132389307,0.2567722499,0.0170761682,1.2307608128,1.3749313354,0.8315657973,-0.4026392400,2.3707995415,-1.1325217485,-0.7696317434,0.9405528307,-0.4137949049,-0.1175819635,0.7805142403,1.8320804834,0.3148280978,0.3389058113,0.4954504073,-0.0342098735,1.2256599665,0.6079592109,-0.3224415183,-1.5470880270,-1.8527430296,0.7457096577,0.0043871086,-0.3397135139,0.6516956687,0.8055987954,-0.2371622622,-0.1931099892,1.1936423779,0.1004671603,1.0995388031,-0.2976596951,-0.1392439753,0.5259703994,-1.7792139053,0.7952412963,-0.2315692008,-1.1396455765,-3.1083474159,0.2638598979,0.1089628115,-0.7766565681,-1.1014711857,-0.1453552842,-1.4054925442,-0.0237163752,0.2195326090,0.5424352884,0.8379007578,-0.2166462690,0.2039635777,0.6850985885,-0.2847076654,0.7809556127,-0.3086790740,-0.3514919281,0.9503691196,-1.7364805937,-1.2029470205,-0.1090737283,2.3331315517,-0.7135651708,0.2003588378,0.1698276401,-0.8578826785,-1.7039403915,-0.1923047006,-0.6373355985,1.4842764139,0.8991172314,2.2821702957,0.2180647701,-2.0747385025,-1.3818309307,1.2608627081,-0.5096740723,-1.3451678753,1.1649746895,1.5543422699,0.6470571160,1.7453366518,-1.4800138474,-0.8186712861,-0.6925798059,0.3354967237,0.0370089076,-0.0598824024,-1.0103763342,0.2989828289,-1.2845501900,0.1014768258,-0.5032425523,0.8546702862,0.4338257015,-0.5709214211,-1.6417970657,0.2938322723,0.7571172118,-1.2316056490,0.5295363069,-0.3947353959,-0.5521669388,0.8162702918,0.0821020678,0.5902984738,-0.0606680997,-0.9304512739,-0.4575162828,-0.5528289080,-1.0609980822,0.1770114452,-0.7643995881,-1.8346927166,-0.3263114691,-0.7260974646,-0.2625757158,0.2421620190,0.8234714866,-0.3505697548,-0.1155276671,2.2433609962,-0.6450213194,0.5869686007,0.1855111122,-1.3059544563,-0.0481009148,-0.5584850907,0.9967713356,-0.1201881021,-0.0914791599,-0.6160403490,-0.5912835598,0.5793107748,0.8839061856,-0.6082174778,-0.1807097793,0.0818651319,-0.3470787704,1.5814223289,0.3732798398,0.0022905916,-1.1152783632,-0.8789777756,0.3264580667,-0.0843966827,2.0768249035,0.3223586380,-1.4704591036,-0.3605157733,-0.0795432106,1.3762940168,-0.3133580089,-0.3769643009,1.0150228739],[-1.6130084991,0.7662905455,0.1537739336,-0.5736240745,-2.0595908165,-1.2742669582,-0.4551030099,1.1831039190,1.2629544735,-0.8028033376,-0.6116998196,0.2680445313,2.1303925514,0.6933071613,0.8110494614,1.6245647669,0.0246582367,0.1792008579,-0.6756305099,-0.2165194303,-1.7439608574,-1.0947178602,-0.5515912175,1.1397629976,-0.1814203858,-1.0873012543,1.9065188169,0.4820602238,1.2316817045,-0.7144633532,0.7393344641,-1.4779632092,0.8479858041,-0.6084827185,-1.4399133921,-1.2956974506,0.2913162410,-0.9487263560,-2.4500293732,1.2459511757,-0.8626870513,1.2747393847,0.8515521288,-0.9107000828,-0.0041014827,-1.0741207600,0.1243414879,-1.6088092327,0.3258252740,-0.5649567246,0.2144881934,0.1281290799,1.3411060572,1.1804949045,0.6733310819,1.5220293999,1.1019228697,-0.6039502025,0.4356611669,-0.4059346914,-0.6738638878,-0.2382538617,-0.3904451728,-0.9828028679,1.2583773136,1.6638858318,0.0410357155,-1.3966635466,0.1359305084,0.0167859215,0.6521763206,-0.2972988784,0.1731460541,0.1411231160,1.0567188263,1.6228011847,-0.5794419050,0.4272536039,-1.5739947557,-0.8606178761,1.0731964111,0.1164292842,1.1253268719,1.0621339083,1.8113048077,-0.2671763003,0.6486790180,0.6535865068,2.1206243038,0.3569327295,-0.3855935633,2.1172261238,-1.2179837227,-0.3329633772,1.1804958582,0.4870519936,-0.0225906167,-0.1032814756,0.2909719646,1.6734399796,0.6995837688,1.6279866695,-0.9171071053,-0.4649128914,-0.7849975228,0.1026836857,1.7261559963,1.4697047472,-0.9684790373,0.0554991364,-1.7402436733,-0.2492827326,-0.0431437008,-1.3587737083,0.6815609932,-0.8693833351,1.2369399071,-1.8863832951,-0.2015711665,0.5212758183,-0.8070146441,0.6563775539,-0.6166521311,1.0231083632,1.2899951935,0.6691089869,0.5692141056,1.3590040207,-1.2761324644,0.2012262344,0.1072357595,0.0752140060,-0.5331254601,0.2735618353,1.3830242157,0.4706042409,-1.1795712709,-0.3537247181,-0.5190734863,-1.1926151514,0.1233324781,0.3509021699,-0.5091461539,-0.0536347479,0.5855531096,0.5949400067,0.2779593468,1.2884949446,-0.9481160641,-0.4234532714,-1.1077837944,1.2787302732,2.0269575119,1.4635083675,-0.9697679877,2.6235475540,1.2805155516,-0.1222272217,0.3515428901,1.0016529560,-0.0150964251,1.6880726814,-0.2948833406,-2.3703005314,1.8436559439,-1.2131029367,-0.2624935508,0.9433795810,1.8394038677,-0.8153470159,-0.9672814608,-0.2367526293,1.2682460546,-1.8260550499,-0.6101045012,-0.8524032831,2.4874143600,0.6338421106,0.7834047079,0.6262134910,0.8847147226,-1.1235461235,-0.9794437885,0.3526352048,0.4729247093,-0.2141827941,-0.7229529023,0.0829834938,1.3534119129,-0.4975259006,1.0918790102,0.5944721699,-1.4659571648,0.8361387849,-0.3421808183,-1.0087279081,-1.2426744699,0.3281867206,0.0251218807,-1.8685960770,0.9150018096,-1.9452970028,-0.8615595698,2.1733384132,-0.9754824638,1.6710414886,0.4690485597,0.8657920957,-0.4328330755,1.2893782854,-1.0481760502,1.3680763245,-1.3068273067,0.9040389657,0.9369824529,0.8598939776,-0.8227170110,-0.3654539287,-0.6739743948,0.3190602660,-0.9016777277,-0.4370117188,1.0818015337,-0.7992702127],[-0.8296168447,0.5022249818,0.3412137330,-0.3512504399,-0.3503727615,-1.8021380901,2.2804696560,-1.1720472574,0.2263271660,0.2877822518,-1.1337782145,-0.3909066916,-1.2376701832,1.0095298290,0.5217128992,-2.2821540833,0.5881370306,-0.2536352277,-2.5339996815,0.0288899709,0.1474454105,-1.5530334711,-0.1875560880,-0.2964601815,-0.1737500578,1.6984698772,0.9925410748,-0.1834912300,0.0390073396,1.2886043787,1.1899611950,1.6747417450,-0.0493365452,0.6665617228,-0.3943178952,0.4373920858,1.0100117922,-0.2434471995,0.4705701172,-0.0913033038,-0.1496414989,-0.1203377172,-0.9248952270,3.0170164108,-0.2714706659,0.4221477807,2.1623134613,0.8745349050,1.3869328499,0.7160037160,0.8574240804,-2.4605793953,1.4067883492,0.7826843858,0.0132020181,0.9728497267,-1.4302023649,-1.5503321886,0.7042499185,-0.0613900833,0.6003428102,0.6352524161,0.4287099242,1.4980370998,0.3880011737,-0.4563947618,-2.7111141682,-0.4926683903,-0.6083946824,0.9790099859,1.3553739786,1.1713775396,0.4362692833,-0.8042359352,-0.7024526000,0.3011744618,-1.4273091555,0.9567792416,0.6883066297,0.4607406855,-0.5390653014,0.6746558547,-0.8313179016,-1.0302064419,-0.0380053222,-1.8747175932,1.3076663017,-0.2346269339,-1.3506914377,-0.4520126581,-0.4520035982,1.7138769627,0.1253602654,0.2660092711,-0.0731366202,0.2919928432,0.0841064826,1.7978950739,-0.3284988999,1.1751443148,0.5153539181,0.2937392592,0.7174130082,0.4991062880,-0.1639733016,-0.3563329577,-0.4395076931,-0.2766478062,-1.0674500465,1.8819682598,0.7912315726,-0.1282933652,1.8472059965,-1.5106694698,0.7845364809,0.4449427724,-1.2240473032,0.5322827697,-0.1730926335,0.9488574266,0.4537747204,0.0691529736,3.4170293808,-0.9167636633,-0.3978354037,-0.5361738801,-0.4047685564,0.7148522139,0.2225925773,0.6639422774,-0.1292048544,-1.1110087633,-1.0102939606,0.2644718587,0.9000436068,-0.7264199257,1.3637973070,-1.2778736353,-0.1999248862,0.5343344212,-0.4080455005,0.1374131441,-0.2867443264,2.0778164864,-0.9607052207,0.9882478118,-0.3585515916,-1.4047431946,0.9531259537,0.2088538110,0.1369061917,1.3903574944,0.4575279653,-1.3510797024,-0.4881621003,0.7812165618,-0.4035978317,-1.0870518684,-1.5355210304,1.0299673080,-0.0183984078,0.1233158112,0.2659848630,-0.9376357794,-0.2414065301,-0.3026311100,0.0925273895,-0.0816137046,-0.0960379541,-0.0317799374,-1.3813295364,0.1347098649,-0.0820175707,-1.2389616966,1.6740804911,-0.2374209315,0.9658219814,-0.7939558625,1.2590600252,-1.5339493752,0.3371162415,1.4419755936,-0.4797958732,0.0273850411,0.7507920861,-0.1079000458,0.9697693586,-0.6925873160,-1.4347451925,0.3423879147,-0.6725934148,-1.7418110371,-0.0797282755,-0.3020401299,-0.3803686202,1.0125890970,-0.9325743318,-0.1049213335,0.0712086931,-0.2040427327,-0.7109842300,-0.7788132429,-0.4137320518,0.5208038688,0.1299502850,-0.1351567805,-0.6129224300,0.0145029593,0.5655912757,-0.8317987919,0.5818741918,0.6020925641,0.1391010433,0.8798360825,-0.3785371780,0.8962466121,-0.1627954245,-0.1554978937,-0.0491830073,1.0199846029,-0.5086528063,0.1819584221,-0.1886064410,-0.5281379819],[0.5540053844,1.2801107168,0.5145618320,-0.2815166414,-1.8504830599,-1.4968154430,0.5677479506,-0.8508284688,0.4678899050,-1.3412883282,-0.1465302259,0.2389729619,-0.6580345631,1.4656655788,-0.5119754076,0.3634024858,0.8378620148,0.6821438670,-0.1599557549,0.8418995738,-1.6232244968,1.2753105164,-1.5744374990,-0.3053387403,-0.7655331492,1.0510780811,0.8004363179,0.2560030520,-0.6166889668,0.0854177177,-1.9206666946,-0.5267802477,-0.7869284749,-0.4115960598,-1.1991744041,-0.5750448108,0.2946201861,-0.7443304062,1.4293489456,-0.4902796745,-0.0483880341,0.6640558243,0.8327726722,1.8407005072,0.7809294462,0.0030927989,-1.5531117916,0.9506762028,-1.3810019493,1.1338344812,1.1597759724,0.1388784498,1.6293661594,0.1739115119,0.2015552223,1.5030477047,-0.4344612062,1.8117796183,2.2257161140,1.4357246161,0.7157769799,-1.1085772514,0.3848017156,-0.8877408504,0.5003702044,-0.8792350292,0.4230170250,0.8809534311,2.0490174294,0.0204322394,-1.2171998024,0.7011641264,0.5943517089,-1.5732725859,-0.7199173570,0.7867372036,3.0451655388,0.1843346208,0.7179120183,-0.1636048704,1.5786978006,-0.7018404007,-0.4428640306,-0.0944580883,-0.4445796311,-0.9633267522,-1.4163391590,0.3170681596,-0.8742064238,0.0117640896,-0.9380337596,0.4167654812,1.0110545158,0.8198906779,1.4507848024,-0.4969290495,0.9055968523,-0.0989162475,0.2817564011,0.1054915190,-0.2824602127,-2.7734065056,-1.1041206121,-0.9647773504,1.5364704132,0.5445757508,0.7791507840,-0.0502580889,-0.9385029078,-0.9562464356,0.5013201833,-1.3731353283,0.6837669611,0.7879750133,0.4322114587,-1.0507004261,-0.4315394759,0.5607248545,-0.3958632052,0.4185329974,1.0252966881,-1.0072593689,0.8607437611,0.4936594069,0.7559320331,-1.0182080269,-0.1923506409,0.5055880547,-0.3494649231,0.4047941864,-0.3316917419,0.1759418398,0.0013057090,1.2570570707,-0.2191038728,0.9023769498,0.7408381104,0.5125377774,0.9699680209,-1.3627812862,-1.5679252148,-0.0293262303,-1.8678886890,2.1712396145,-0.1571822166,-0.3564154208,0.0553524904,1.1599270105,1.2627038956,-1.3459423780,0.6730185747,-0.0750857741,1.0460783243,-0.4429310262,-0.3001503050,0.0078839790,-0.2567608058,0.9423267841,-1.5697605610,0.6863792539,0.9789968133,-0.0081998017,1.7019345760,0.2806222439,-1.4620214701,0.5297275186,1.1651798487,0.8243795037,0.8680918813,-1.7472949028,2.0696997643,-0.2578157783,-0.0249881465,0.6565919518,0.0272306334,-0.8412454724,0.2397221625,-1.7836779356,1.2204616070,0.5548753738,-0.4932431579,0.3310505152,1.0874546766,-0.1795441955,0.9982895851,-0.2980313897,2.0983877182,-0.7227540016,-0.2248633504,-0.1296203732,-0.9494423270,1.4522721767,-0.9315572381,1.8279565573,0.1612818539,0.2126344889,0.6816353798,1.1261162758,0.9266378284,-0.3534713686,0.9973428845,0.4953095913,0.8687413335,0.6141946316,1.4996018410,0.1254296452,-0.7379559278,-1.2608875036,0.3750228584,-0.6231452227,3.0864508152,0.4766899645,-1.1117894650,0.2051137537,-1.7827329636,2.1619787216,0.6303663850,0.5379341841,2.3428905010,0.1024986655,-1.9776982069,-1.9109508991,2.0136795044,0.5614098310],[-1.5266704559,0.1959593594,0.6133632660,1.1159074306,-0.5647493005,0.4011000693,1.7993389368,-0.0339073166,-0.2392009497,1.6570544243,0.7138476968,0.3991074860,-0.8774254918,-0.4240868092,1.0801563263,-0.1920657456,0.3216613829,0.2953280807,-0.4288747013,0.4682311714,-0.9422526956,-1.1518111229,0.7867501378,-2.3118257523,-0.9087553024,1.6632435322,1.0761674643,-0.2712795138,-0.8323598504,-2.2843811512,0.2200830430,0.6697045565,-0.9509277344,-0.8576626182,-1.9374886751,0.8887458444,0.5036352873,2.0676424503,1.0283118486,-0.2918054461,-0.8564800024,1.6496522427,-0.1889360398,-0.3224303722,-0.5871344805,-0.9989820719,-0.0417234078,0.4708833098,0.8459716439,0.3378955126,-1.3688849211,1.9070968628,0.6046031117,2.0323977470,-1.0468131304,0.7598228455,0.1940415055,1.2512792349,-0.3850702941,-0.1691117436,-0.8089761138,0.8820098639,0.2849083245,1.2971107960,-1.0834826231,-0.1189776212,-1.1106203794,-1.3206923008,0.7566649318,-1.9178347588,2.7865297794,-0.5657482147,0.2699114084,0.3090089858,0.6460570097,-2.5136375427,-0.1902826726,-1.3483183384,-0.7845107317,0.8349735141,0.8349094987,-0.9854658246,0.1150932983,-0.3015092909,0.4304955900,-2.6301472187,-2.3760235310,0.2100565583,-1.6220120192,-0.6598060131,0.0390680023,0.9663178325,-0.0312575251,0.3434973955,-0.5051045418,0.2603479624,-1.6800607443,-2.0438206196,-0.7828196287,0.8378771544,-1.0073713064,0.3594104946,-1.0887169838,-0.2854229510,-0.6326141953,-0.7492433786,0.1771231443,-2.0925412178,0.7221005559,1.0629680157,0.8875825405,0.9514683485,1.1898092031,-1.3159422874,0.3789256513,0.0052759089,0.6215171218,-0.1703885645,-0.8301671743,-1.4931890965,2.1389789581,0.0210836176,0.0997546017,0.3395195603,1.5730198622,-2.4545269012,1.4472402334,1.2666022778,0.2692795396,0.7309278250,-1.1675630808,0.1287209988,-0.2159086317,1.9478616714,0.6417044997,0.4954462051,0.2918036580,1.1044400930,-0.0565379076,1.2288534641,0.9528832436,0.0863463283,0.5586382151,2.1897523403,0.1289473176,-0.8775374293,0.5111732483,0.6179109216,-1.8111522198,0.0995398909,1.3215231895,-1.1074717045,-0.8939023018,0.2247423381,-1.0083754063,0.8858939409,-0.2505567372,-0.3213457167,0.8018403649,0.2690301239,-0.5970544815,-0.9925628901,-0.5494341850,0.1522569507,0.1563308090,0.6602344513,-0.0799589828,0.8824433684,1.1563723087,-0.3943536580,-0.8907910585,1.6997411251,-1.1539968252,0.2835911512,-0.9666313529,-0.9075029492,-0.3648546338,-1.1380269527,-1.6632107496,0.0931004658,1.5548486710,0.6206492782,0.0812515914,0.5045549870,-0.0324226506,0.9576783180,0.6753861904,0.0415677503,-0.8783605695,-0.5465931296,-0.5524756312,0.4131335020,-0.6216126680,0.9888315797,-0.3318295181,3.0655612946,1.0167248249,0.0809800029,1.0825935602,2.0115580559,0.3990780115,-0.1417630613,-0.7643346786,0.9107628465,-1.2469594479,0.2185842395,-0.5608953238,0.7581259012,-0.5780990720,-0.3210789263,-0.1193744466,-0.6779866219,-0.5087441802,-0.1424155980,1.1234807968,0.1186617762,-0.0176850781,2.0666704178,1.0098127127,1.3752470016,-1.2679327726,-1.1970074177,-1.0551103354,1.1164258718],[0.3721234500,-0.9255678058,-0.9442209005,0.4268756509,0.3172981739,-0.3741634488,-1.4125268459,-2.3680481911,-0.1069736257,-0.4030350745,-0.4043666124,0.1732717007,0.4453865588,0.0835408270,-0.3626154065,0.2124658823,0.7893753648,0.8026905060,-1.3991522789,-0.3200124800,0.9662565589,-0.9130623341,0.4093767703,-2.2999207973,-0.2824724913,0.8781957626,0.0921964645,2.3364260197,-0.3582146466,-0.2903281450,-1.2173844576,-0.5441349149,-0.8577818274,0.6779240966,0.0227339007,0.9655373096,-0.2647223771,-1.9813855886,-0.5777205229,-1.6076692343,0.3734553158,-0.2309042215,-0.0047361567,-0.1747259945,1.4980055094,2.3268282413,0.0851788521,-0.3865151405,0.6857342720,-1.9992141724,-0.5850980878,-0.8566595912,0.5763618946,-0.4204704463,-1.0309052467,-0.3967194259,1.0643119812,-1.3884699345,-0.2525475919,0.0347681716,-0.4270209372,1.8029408455,-1.7515459061,0.0540627949,1.4515020847,2.2537171841,0.7252607942,-0.3768343031,-0.4213645160,0.3329466283,0.4443221986,-0.1684675515,-0.5755594373,0.6468083262,0.6131626964,1.2769823074,-0.3528927565,-0.2351228446,-1.2301682234,0.4037817121,-1.3678727150,-1.0195753574,-1.1714248657,0.4959396422,0.7116174698,0.3401472270,-0.5646832585,1.3907834291,0.2785235345,-0.8563492298,0.3405274451,1.1643620729,1.2994824648,-2.8496544361,0.0588610545,1.9530397654,0.5605524778,-2.7157382965,0.9060288072,-0.6560640931,0.4947168231,1.2521569729,0.7251318097,0.4450671673,0.1428324431,0.5182865262,-2.0029828548,0.4784449339,-0.3109569848,1.4094113111,-0.4395599067,0.0288746692,0.5327027440,0.9105619788,0.7118870020,0.0834584162,0.6378134489,0.4777068794,0.6714622378,-0.2473972887,0.9334355593,-0.0149712618,-1.2888498306,0.3178930879,0.3878506124,-0.4473615289,-0.0200192370,-1.1375455856,0.2947454154,-1.6181722879,-1.4102656841,0.6471197009,0.0660516992,2.5253288746,-1.4385683537,1.1881508827,-0.5137376189,-0.3100413084,1.8618991375,0.2196703702,0.8764735460,-1.6034924984,0.9608685970,0.7578120828,0.1340373456,-1.2120584249,-1.0811951160,-0.5332429409,-1.2666810751,1.1997977495,-0.0147280395,-0.3411561847,0.0395528637,-1.0613685846,0.8180955648,1.3017721176,0.8937417865,-0.7969278693,1.0919103622,-0.6793391705,0.5171425939,1.9261372089,-1.4655749798,-0.9694035649,-0.2225221097,-1.3643192053,-0.1500839889,0.5843650103,0.7056804895,-1.0301253796,-1.0684210062,-1.0028663874,-1.5143984556,0.0788298324,-1.7967107296,-0.3639734983,-0.7513176203,0.1762141883,1.1550230980,-0.4082490206,0.9812905192,0.4297397435,-1.0552676916,0.4100736082,0.1593143344,-1.5990869999,0.9750310779,-1.0146784782,-0.4160335660,1.6984248161,1.1609507799,0.7490220666,-0.8482598662,-0.6165418625,-0.0323930122,1.6698666811,0.1386055946,-1.1053417921,0.2234233469,-0.0349204689,0.5528994203,0.7125429511,0.0064929891,-1.0954294205,-0.4205610752,0.8501464128,1.0036913157,-1.4143588543,0.4902287722,-0.0818249434,0.1800193787,3.1680388451,-3.1549611092,-1.7138682604,-0.3571625054,1.7889220715,-1.3005938530,-1.1199634075,1.1873326302,-0.3849809766,-1.2680217028,1.4615029097,-0.6191694140,-0.2495485842],[0.2082553059,0.1791836023,0.3728811145,2.1315107346,1.3774552345,-1.4320223331,-0.3597615361,-0.1586332172,1.0218647718,-1.1757861376,0.7682100534,-0.3420245349,0.7694000602,-0.1330372542,1.1873819828,-0.3006245792,0.7143628597,-0.2423644811,-1.5537725687,0.3462584019,0.4230340719,1.0040265322,1.0997759104,0.7828722596,0.4102944136,-1.1098705530,-1.0867903233,0.0346978270,1.1880047321,-0.5926411748,-0.5738988519,-0.4091166556,-0.8370656371,1.0624153614,-0.9222751856,-0.2183016688,0.8524411917,-0.2983358800,1.0201835632,0.5510687828,-0.6143435240,0.0868408009,0.6313892007,-0.6356560588,0.9379420280,-0.9612347484,0.0437515527,0.1527118981,0.0146253034,0.1002622694,1.3769297600,1.8511936665,-0.9757603407,-0.1710399836,-0.3755131066,0.6657829881,-1.4439826012,0.8646513224,-0.5456988215,-0.3876164854,1.1869660616,-1.0903005600,0.2324468642,0.4489884079,0.2146653384,-0.3496420681,0.2460367978,-0.3882636726,-3.5028512478,-0.8601097465,-0.3408805728,1.0436613560,-0.1229604706,-1.1517763138,0.6286780238,-0.5831050873,0.2989288568,-0.8618805408,-1.0957497358,-1.6585500240,0.4104961157,0.3074675798,1.7889269590,1.4891097546,-2.1270682812,-1.0630022287,-1.0487341881,0.6423425674,0.2223631740,-0.6816553473,1.4914273024,0.8645065427,0.4738709927,1.0744345188,-0.0626458526,-0.1355794221,-0.5250419378,-0.5246269703,2.2068521976,0.8553270698,-0.0701120049,0.0589456148,-1.3844010830,0.1336048990,1.4933985472,-0.7632629871,0.0417235643,0.0002870396,-0.5796526670,0.8333060741,1.2923172712,2.1858572960,-0.3396995068,0.5367954373,0.9259405136,-0.2317183018,0.5833922625,-1.8033591509,1.9753354788,-2.3160240650,-0.5039364696,1.1172219515,-0.5671532750,-0.4029072225,1.2319533825,-1.3085190058,0.4285664260,2.4368367195,-0.8638158441,1.3917367458,-0.4627754688,-0.0249045976,-0.6280714273,-0.3255177736,0.6238153577,2.4476258755,0.3078250289,0.2715431154,-1.6325097084,-1.1116009951,1.1871554852,1.6313469410,-0.1071722433,1.0070543289,-2.4956753254,0.1061718240,0.0120313754,-0.2095249593,0.3418090045,-1.2988194227,0.9380907416,-0.1236676201,-0.4573471248,-0.7191598415,0.8948004246,0.4774148464,-1.7359133959,0.6295673251,1.4042685032,0.4560325742,0.2498383373,-0.7552278042,-0.2820581496,-0.9741064906,-0.0094816862,0.4927726090,-1.5904263258,2.0433969498,-0.2484231889,-0.3923215866,-1.2565356493,0.4341219962,-0.2114793807,0.0118124252,0.1367721707,1.8070771694,-0.6738863587,-0.7573686242,-2.5797410011,-0.0804657713,-0.4304643869,0.3704101741,0.0622019991,-0.3475724757,-2.1042618752,0.5381350517,-0.7691090703,0.1761910468,0.1809306294,-0.3777577877,-0.9449252486,0.2289055586,0.2820708156,0.4839225709,-1.6858013868,-0.8856335282,0.9421400428,-0.0791686252,-1.6161797047,-1.0410964489,0.6739453673,0.0878255740,2.3287050724,0.7083165050,-1.3314689398,-0.5426526666,-0.1750532389,-2.0199997425,-1.9016313553,-2.2880182266,-0.6802806854,-0.5845252872,-0.3345496356,-1.3252086639,-1.4987232685,-0.5198655128,-0.8556139469,0.7470584512,0.7125549316,0.0654351264,-0.6348320246,0.7843273282,-0.4739659429,0.2645100951],[0.7582881451,-1.4469571114,-0.8762317300,-0.1305171847,-1.8916240931,-0.2047363669,-1.4006288052,-1.4427716732,0.4075613022,-1.5529645681,0.4010265768,-1.2205121517,-0.5830441713,0.7983804941,1.3810188770,1.2851274014,1.6508965492,0.8096008301,-0.7591167688,0.7297912836,-0.7162643671,1.0841537714,-1.2955800295,-0.7949441671,-1.1049574614,-1.8323066235,0.4911190569,-0.0560565628,-0.7176681757,-1.8872069120,0.5780738592,-0.8776240349,1.1596308947,-1.4674416780,0.7553941607,0.1954972744,-0.5818178654,0.2693494856,-1.0041868687,0.6039539576,-0.6923959255,1.9953961372,0.1902663261,-0.0727277547,1.5628709793,-1.3869262934,-1.2224737406,-0.9052775502,1.8306998014,0.1413167864,1.7664381266,0.7349786758,-0.1183590814,0.8745171428,-0.1398411989,0.2598904371,0.0656923652,0.8621975183,0.8937485218,0.2206461132,-1.2316112518,-0.7846273184,0.7935965657,-0.7879238129,-0.7651246786,-0.1138043851,0.6265539527,-0.1829637885,-1.1814017296,0.0661041215,0.9544205070,0.3241200149,0.1085844710,-1.2307775021,0.3755576015,-0.3521512449,1.3882151842,0.5725190043,-0.0776720494,-1.3191725016,-0.4986121058,0.3813388348,1.4137277603,-1.5064461231,0.0442575030,-0.5795447826,-0.6121605039,0.7674696445,-0.3727608919,-0.1838898957,1.0338747501,1.9497634172,1.0179313421,-1.1559066772,-0.3884924650,-1.3412394524,-0.8717449903,0.7448266745,0.4458397627,1.6077286005,1.6052932739,-0.5146309733,-0.3278999627,0.7633868456,-0.8159669042,1.4002127647,0.1569492966,1.9102462530,-0.1955651641,-0.6491927505,0.2193007469,-0.4692718685,-2.2204196453,-0.3630209267,0.2141387612,1.0269374847,0.7454722524,0.3479391336,-0.2595340014,-0.0004230243,-0.5843359828,1.0068081617,0.5920389295,-1.4571993351,2.0111148357,1.6898576021,0.1999664903,-1.2243306637,-1.9395003319,0.7388504744,1.3946845531,-0.8135850430,-0.8188219666,-0.1027977988,1.2393451929,0.3642773926,-1.1666374207,-0.5038280487,0.1114103869,-1.5347028971,-0.5841313601,-0.8017553091,-0.1922202855,-0.3450426459,0.3865180016,-1.2264298201,0.5880392790,0.2358413041,0.9229251146,-2.5078446865,0.4116154313,0.6169677973,0.3932493329,0.7349784970,-0.1033508405,0.5472589135,-0.8508823514,0.4239557981,1.7755675316,0.0498641804,0.0039831121,-1.1012924910,-0.0702986047,0.0500299931,0.9911147356,-2.1317226887,0.0189382583,-1.2867163420,0.5649483800,0.5770050883,0.6580603123,1.4723309278,0.0125505729,0.7892917395,0.5260727406,0.7284713984,0.9198905230,0.4369905293,0.1269310713,-2.4369902611,0.2138117403,0.6558313370,0.1240368336,-0.6512229443,0.7850050926,0.8441942930,-2.6291081905,0.0859598890,-0.8896072507,-0.6072485447,0.5743405223,1.1331851482,-0.4991896152,-0.6477818489,-0.5778541565,0.0418310277,0.4881654680,-0.9791178107,0.1232551485,-0.8317633867,-0.4315126240,1.2505896091,-1.6251081228,0.3261093497,0.7886345387,0.2484288216,-0.5134832263,-1.5124263763,-1.6093171835,0.5770623088,0.5730774999,0.4694595337,-0.1032461897,-1.6306973696,1.0383374691,0.3179350495,1.6510140896,-1.4021530151,-1.6068631411,-0.9987185001,-2.0106415749,2.4225859642,-1.4711263180,1.0438575745],[-0.6822944880,0.5343873501,-0.2162118554,0.4941727519,-0.0919690430,-0.0362560339,1.7026308775,-0.5808956623,0.2818481922,0.4702845514,-0.7051241398,0.4703357816,0.4183109701,-1.0918869972,-1.1447758675,0.1090682745,-1.0183391571,-0.0740790442,0.8214829564,0.2784597278,-1.2427382469,-0.8619667292,0.9212391376,-0.3883512020,0.0477222577,-0.8693955541,-0.1270150840,0.4906855226,0.0592403039,-0.2530156374,-1.9048664570,-0.0264112484,-0.5457171798,-0.0354791954,1.5265716314,-0.1019395888,0.7486965656,-1.5187047720,-1.0162277222,0.6233072877,0.4643627703,-0.8993694186,0.4188348353,-0.2046624869,0.0254781116,1.2895642519,0.6310182810,0.3446836472,-0.5590118766,-0.2556653917,-0.9896861911,-0.5274890065,0.2978168428,0.0847402886,1.4692370892,-1.3065133095,-0.6991641521,0.3440778255,-0.2718342543,-0.5695889592,0.0174058340,1.0984630585,0.1190534905,0.9798473120,1.8879725933,0.8673732877,-0.9161585569,0.4036022723,0.0035582194,-2.3451409340,-1.0702661276,-1.6849539280,0.5669997334,-0.0257244613,0.0361120626,-0.0581826456,1.0888381004,0.4999228418,0.5219503641,0.4908412993,-0.4648894370,1.2614723444,0.3910034001,2.0438618660,2.3622827530,-1.1208287477,-0.5126113296,-0.9887000322,-0.5191606879,-0.2261203378,-0.1252585500,0.0878077075,0.0369837433,-0.3490452766,1.1845042706,-0.2965066135,0.8207860589,0.1675992906,-0.9627301693,0.2979668677,-1.6763862371,-0.1788655072,-1.2550932169,0.7845742106,-0.0772865936,0.3855877817,-0.7745981216,-0.1232291982,0.7760265470,1.3818191290,-1.0101274252,-0.7251560092,-0.2056710720,0.0522477031,0.1851748675,-0.5892301798,-1.8814272881,-0.8631243706,-0.6360574365,-1.1729818583,-0.9179823995,0.9177124500,0.7739092112,1.4054862261,0.8030353189,0.4492469430,-0.8796300888,1.2875020504,-0.9119425416,-1.7323316336,-1.3451622725,-1.0565350056,0.0448045060,0.3449862301,-1.5712947845,0.5012857318,0.5056712031,0.2673935294,0.2346226573,0.8127566576,0.8522439599,1.3639987707,-1.5733017921,0.4449477792,1.0757219791,-2.1031193733,0.0343645252,1.1781619787,-1.4266718626,-0.0131833591,-0.2667540014,-0.0093084434,-0.5884740949,1.7538708448,1.1161477566,-0.1020128131,-0.2309615612,-1.2694759369,-0.5607418418,-0.4304545224,-0.3253737688,1.4808640480,0.7316930890,-0.2684265673,0.4855044782,0.3031046987,-0.8914096355,-1.4147173166,0.0380460210,-0.7299775481,-0.2678869963,-0.3101365864,0.0350842513,0.2402429581,0.7029187083,2.7874162197,-0.1721138507,-0.6986640096,1.1523491144,-1.3354657888,0.0191495139,1.1237381697,-0.3511058688,-0.4177630544,-0.9081257582,-0.2498989403,-0.1846230924,-1.6133767366,-0.8762238026,1.6221534014,0.2161289006,0.4476101398,0.2794209719,0.8718365431,-0.1944708526,-0.6252462268,-0.7593362927,-1.4583418369,-1.4333637953,-2.1012365818,0.4250461757,0.9667654634,0.2156297415,1.4574354887,-1.2711378336,0.6589086056,0.5638456345,1.1474407911,0.1375724077,-0.2267200947,-0.1479162425,-0.0163359381,1.4957658052,1.5470947027,-1.1362988949,-0.0993754491,0.0944555551,-0.0559778437,0.9665927291,-0.2777977884,0.9807844758,-1.7313894033,1.2457816601,-0.7532542348],[1.5628550053,-1.8272314072,1.1460046768,0.5990837812,0.6483633518,-0.8140592575,-2.5026910305,1.5262742043,-0.5325720310,0.1642222852,0.3184698522,0.9115745425,0.9058817625,1.2242089510,-0.7914894819,-0.5712547898,-0.2330745012,0.7534739971,-1.3289783001,0.2245698124,1.4651367664,-0.6622340083,-1.4486229420,0.4370337129,-0.4643350840,0.5815520883,-2.0180966854,0.6800788641,1.9025919437,0.4159368277,0.3808253109,-0.3558321893,0.2614868283,1.4179209471,0.3718845248,0.9492228627,0.1516134590,0.3136004806,0.0753630325,-0.8924691677,-1.5429160595,0.2965213656,-0.5717129707,-0.0413963459,-0.9464138150,-1.2638531923,-1.4999424219,0.4140037298,-1.3621689081,-0.1094720140,0.1787716299,-0.3219559491,-1.4167441130,0.5873966813,1.1247311831,-1.4185732603,-0.6824306846,0.3787460029,-0.2558441162,1.1735316515,1.0506619215,0.3373606205,-1.7914761305,1.0255435705,-0.4845902026,1.3263299465,-1.1724784374,0.5220450163,0.8054077625,0.0096950140,0.8125115037,0.3299151063,-1.0037217140,0.3730991185,-0.9277126789,1.4822149277,1.5176270008,0.0745661706,-0.9604218006,-0.1578906775,0.2494293302,-0.9300476909,1.5165371895,-0.5894943476,-0.1630633473,0.0795842856,0.3101577163,-1.2102751732,2.5860579014,-0.9084997177,-0.9838130474,-0.4032730162,0.4376574755,-1.1266232729,1.2498075962,0.5042888522,0.2260986120,0.6913132668,-1.5524065495,1.0065914392,1.2285039425,-0.3059667349,0.6399753690,0.1240009218,-0.5298648477,0.7409871221,0.6777750254,1.4533698559,0.7625650167,0.1595736295,-0.1319786757,0.8643478751,1.0831538439,0.0846288800,2.1185903549,0.4222730100,-0.5730257630,0.8299391866,-0.3804766834,1.1045529842,-0.5073445439,0.1418423355,1.2586370707,0.8240965009,-0.0925577134,1.7273045778,-0.3900260031,-0.0798248723,0.8808295131,1.2586567402,0.9773184061,0.9998583198,-0.2488287389,0.2702918947,1.4333872795,0.5639662147,-1.0274194479,1.8915591240,-0.9012168646,-0.3748772740,1.9961894751,0.3488658071,0.2414206564,-1.1801184416,0.8474820256,0.2638389170,0.4284295440,-0.4961136580,0.5766240954,-0.8417623043,0.4841005206,-1.2339916229,-0.7195590138,0.6247155666,0.2961598039,0.1626374722,-0.1993692070,0.4295589626,0.1204819679,0.2459912151,2.3836979866,0.9892594814,1.5916163921,0.0971853957,-0.8566622734,-0.2674368918,0.5222064853,0.9773113728,-0.3150422573,0.8756898046,-0.4676228464,0.2016017884,-0.7167613506,-1.4885647297,1.2574212551,-0.6553381085,-0.6623219252,-1.7356320620,0.2538641989,1.0851593018,0.5627361536,1.1075919867,-0.7186845541,-1.4439043999,-0.8579331040,0.5339972377,-1.0072786808,0.6900771260,-0.8863855004,-2.2532014847,-2.5111033916,-0.4095038176,0.2482353151,-1.2830238342,-0.8127713799,-0.0843408108,0.1111695021,-1.0864380598,-1.5028867722,-0.8787872195,-1.5936022997,-0.1871830970,-1.0399664640,0.0147543494,0.7644811869,0.9782803655,0.0003450581,-0.2709909379,-0.1951631755,0.3221687376,0.3713282347,-0.4911470413,0.7329746485,0.0925365835,0.2304435521,-0.9668128490,0.4846392572,-1.1093183756,-1.3233888149,-0.0245773736,-0.0953248069,0.1685880274,-0.0482572317,0.5383268595],[1.0802992582,0.2973013222,0.9245066047,-1.3101730347,-0.4364936352,-0.4648250341,-1.1351803541,-0.8205749989,-0.8568386436,2.3754029274,-0.9961820841,0.8363527060,0.2676135302,0.8208408356,-0.9117810726,-1.0280570984,-0.9798866510,-1.0432735682,-0.6724710464,-1.4416898489,0.7651339769,0.5950027108,-0.1433894634,0.6269071102,0.8044335842,1.0749381781,2.0385212898,-0.1334894150,-1.8284147978,2.0971255302,0.5444354415,-0.7383652925,-1.5236394405,-0.5464919806,-1.5434194803,0.3095881343,-1.1199891567,2.2750618458,-0.4669438601,-1.8398299217,-0.1546743661,0.0634817109,-0.4727517366,1.8044909239,1.3376697302,1.1377027035,-0.3198766410,1.7948639393,-0.2277033031,-0.3369950950,-0.5130826831,1.2988376617,-0.7731146216,-1.0604265928,-0.2253106982,0.1074710190,-1.8205831051,0.7368350029,0.7032628059,0.9893723726,0.6483827829,1.4742196798,0.0884756595,0.4978652000,1.0682324171,0.6970139146,0.8740690351,-0.9838154316,0.2248925716,-0.5559855700,0.4227550626,1.1992962360,2.3643376827,0.8882084489,-0.5913435221,-0.7380414009,-1.0349892378,0.9173351526,-0.5621873140,-0.0949840769,-0.1317703575,0.0631912276,0.9709743857,0.3090033233,0.9268954396,-0.4655930102,0.2861961126,-0.2644225359,-0.3251528144,-0.8222166300,-0.0960599184,1.4646030664,1.4194988012,-0.4227886796,-1.8527027369,0.2715824246,-0.5044975877,-1.8221150637,0.1931934655,-1.5894869566,0.2379000485,-1.0019147396,-0.2313501835,0.0035350423,0.0740525126,-2.6869721413,1.5776410103,-0.5823276043,1.4910770655,-0.9430855513,-0.0499442630,-0.2272549123,-0.0936595276,-0.1898984909,0.5837596059,-1.1511113644,0.3232372105,-2.0862889290,-0.0450896136,0.1430051774,-0.3782043159,1.2295246124,1.5990449190,0.4077310264,1.0603466034,0.1066178828,0.9763474464,0.5174518824,-1.0154049397,0.1633393317,0.9158678651,-0.4462140799,2.7762815952,-1.7528024912,-0.8473525047,1.3816362619,-1.4776809216,-0.0661653429,-0.5095384717,0.1321265846,0.3022744358,1.0764505863,-0.8410828114,-3.0817403793,-0.0300313104,-0.0764164105,-0.2109280974,0.1390453428,0.4264347255,0.0728728324,0.2737209201,-0.4620195031,-0.0491712280,-0.3356860578,0.4319370985,-0.2530729175,-0.4461062849,-2.0072276592,0.9238764048,0.1594048589,0.0638244227,-1.3263802528,0.1089418977,-0.5390468836,-0.5481974483,-0.1197349951,1.0218429565,-0.7934098244,-2.4163577557,0.7732511759,-0.4387922585,0.6461163759,-1.3936929703,-0.6618784070,2.1668913364,1.0993745327,-0.1202072054,-0.4973158836,-0.6462906003,0.0609707199,-0.9450492859,-1.3187497854,-1.0022757053,-0.1960598081,0.7324171662,-0.4829072952,0.4753527939,-1.4519374371,0.8318330646,1.3754912615,-1.1183576584,1.5637904406,0.3519226611,-0.4841500819,0.3794345558,0.5975720286,0.6764720678,0.5199920535,-0.0438902751,2.0463252068,-0.5165079236,0.0998032242,-0.9830651283,-0.6394013762,1.1262912750,-0.8103187680,1.0153911114,-0.6205669045,0.3662223816,1.1925779581,0.8172106147,-1.0794445276,1.8628367186,0.0463979691,-1.1959922314,-0.0023014785,-0.3665894270,0.6613500118,-0.1001069471,-0.9814707041,0.3034345806,-0.7231702805,0.7263912559,0.8849028945],[-0.0591529347,-0.7766273022,-1.2518745661,-0.6941793561,0.5461867452,-1.4127789736,-1.1363685131,0.4137958586,-0.2157655805,-0.1393927634,2.9335629940,-1.3698664904,0.4863412976,1.7759577036,0.9367753863,-0.5121124983,0.5831712484,-1.5519332886,0.2913210690,-0.5899180174,0.0952703059,-1.8471230268,-0.0709084868,1.4827005863,1.4237951040,-0.5086781383,-1.6200858355,-0.1084599942,-0.3973819017,-0.7838751674,1.9469268322,0.4652900100,-0.6131108403,0.4059070349,0.2010302395,-0.1316693425,-0.0746570900,2.3412575722,-0.1811244637,-0.7504181862,-0.3903382123,-0.2243826389,0.9201111794,0.4877253771,-1.4580579996,0.8116889000,-0.5697486997,-0.4015645683,-0.4854067564,-0.6364825368,0.9034591317,0.6379844546,1.5771756172,0.5850499272,-0.6220704913,0.4983624816,0.4711814821,1.1179269552,-0.2089185417,0.5540845990,1.2248898745,-0.5977462530,-0.5361250043,-0.0362544581,0.8173483014,0.1797466129,0.6411272883,0.5963133574,-3.6270735264,0.3135236800,1.7770348787,-0.5708009601,1.2956660986,2.3631596565,0.6470772624,-0.1276730001,-0.4334301949,1.7504278421,-0.7313968539,1.3321459293,-1.1168040037,0.0644597635,0.6098915339,-0.8870542049,-1.1227594614,-0.1024430543,1.0980104208,-0.1197945774,-0.7725059986,-0.2856721580,-1.4923102856,0.2849843204,1.2012387514,-2.6904499531,0.1604321897,0.0795088485,-0.9636630416,0.4146036506,-0.2131204307,-0.5044959784,0.3896358311,0.4751556814,0.2694335878,-1.3584232330,-1.5450501442,0.2078277916,-1.4040329456,-0.9148842096,-0.5544590950,-1.9398692846,-0.7526990175,-0.2037417889,-1.1072591543,1.9583116770,0.4932885468,0.2148445249,0.7477450967,1.5735576153,-0.0146483416,-0.7617738247,0.5999017954,0.7779759765,0.5772922635,0.2835288644,-1.2251069546,0.0834239051,0.7457368970,0.5876018405,1.0709481239,0.5153665543,0.7868348956,0.9774563313,0.1690707654,0.4484441876,-1.3282804489,-1.5179308653,0.4773723185,-0.2764462531,1.5602777004,0.3877660036,-0.3111630082,0.9336119294,0.9268003702,1.2970597744,0.4492572844,-0.5420606732,2.1234147549,0.4513804018,-0.7160360217,-0.3417129517,-0.3907018900,-0.0474371351,1.1380760670,0.4578434527,-0.2161199600,1.1832199097,-0.5278459191,-0.7826381326,-0.9901517034,-0.2069672644,0.7864428163,0.3632103205,-0.3682445586,-0.4018748999,1.0661147833,1.0399434566,1.4891061783,-1.6929762363,-1.0862518549,-1.6203837395,0.1468081474,-0.3604899645,0.8857752681,0.4906992018,0.0710786209,-1.1928530931,-0.1335477829,0.3187680244,0.0093177333,-0.9354539514,0.2795247138,-0.3294781744,0.4063001871,-0.9572349191,-0.8981993794,0.3415729702,0.8799454570,0.6367980838,0.7841370106,1.1163245440,-1.9475696087,-0.4421205521,2.8911242485,-0.8780661821,-1.0232143402,1.6205269098,-0.7261584401,0.0078559443,0.6348084211,1.1820709705,-0.1163733304,0.8389843702,1.2746939659,-0.1599865109,-0.9582620859,1.6713591814,0.6692322493,-0.6660379171,1.0564323664,0.9545536041,1.2530609369,-0.2479295433,1.6680572033,-0.4401457906,0.7781750560,-0.3038768470,-0.4356323183,0.6398789883,-1.5581569672,-2.0755450726,0.6866158843,1.3195549250,-0.3794221878,0.5103654861],[1.9730210304,-0.7375587821,0.6860241890,-0.9076411724,-0.6199430823,-0.9175467491,-1.3471761942,-0.9156367779,-1.3426939249,-2.0646798611,-0.6329957247,-0.0766262189,-0.5408920050,0.9520366192,-0.8571756482,-1.1599135399,-0.5584220886,1.6461673975,0.1045306548,0.2158393413,1.7292610407,1.2974575758,-1.3764995337,-0.4452150762,-0.1526507884,-0.9902333617,1.5345755816,-0.1589398384,0.9030612707,2.0708162785,-0.9861791730,0.2067744881,-1.9495503902,-0.5985112190,0.9967989922,-0.8435174227,-0.2820928097,0.3777058125,-0.3613773882,-0.5519063473,-0.6273316145,-0.4378284812,0.2569901943,0.9104852080,0.0983464718,-0.2658705711,0.3255655766,-0.5762305856,-0.9602346420,0.2911734283,-0.5379109383,-0.9457253218,0.7973366976,-0.7568753958,-1.7424373627,-0.7140328884,0.7866174579,-0.0874564052,1.0776548386,0.0858798251,-1.2913337946,0.8008134961,-0.3137482703,0.5386188030,0.4172564447,-0.2212724090,1.0290312767,-0.6964595914,1.6124269962,-0.5174486041,-0.6736907959,-0.3899573386,0.1325595826,-1.0444660187,-0.2226642817,0.7306677103,0.5076549053,0.9035003185,-0.4636362791,-0.5478364825,-0.5060825348,-2.3333342075,-0.9416838288,-0.0387029424,-0.1578006744,-0.6142289639,0.8859317303,-0.8811742663,-0.6772906780,-1.1823346615,-1.3328503370,-0.0093972646,0.5491439700,-0.6104214191,-1.2850006819,-0.6465294361,0.1817125827,-1.4026774168,-0.1775815934,-1.2878239155,-0.4031934738,-0.8606334329,-0.7982609272,-0.6417425871,-0.6398243308,-1.3228656054,1.2034947872,0.8919556141,-1.8049753904,0.4476211667,0.2820806801,1.8818633556,-0.9953379631,-0.3909469247,1.3604015112,2.3987371922,0.3861819506,1.0936772823,-1.1391612291,0.8738053441,2.6254327297,-0.1070521250,-0.4421439469,1.9917935133,-0.3215175569,0.6542347670,-1.0363366604,0.5616360903,-0.9750161171,0.5296633244,0.5032150149,-0.8450386524,0.2101103216,-0.8270325661,0.2830041051,-0.9392392635,0.8031272888,-0.8688564897,2.0318152905,1.5412871838,0.3922415972,0.3853695095,-0.1013855189,-1.2406502962,-0.8898990750,1.6640088558,1.4869363308,0.0144009460,-0.0533166006,0.1196023002,-0.0499731563,0.0728715211,1.6635298729,0.1403089315,0.2601655424,-1.2603138685,0.5634941459,0.5055314898,0.7348325253,-0.4419586062,-0.1415433139,0.7567729950,0.9354738593,-0.0609624647,-0.3537159264,0.8630941510,0.5925130248,0.1479303986,-0.5582378507,0.8495017290,-0.1681861877,-0.8794690967,1.5341011286,-0.9373859167,0.3684532642,0.2468343675,0.5768260956,0.1956495643,1.7508788109,-0.2407533079,0.0557939187,-0.3764891624,-1.4740674496,1.0650231838,-0.6109123230,-0.8943587542,-0.2481622696,0.5509840250,0.1316975355,-1.0856409073,-0.5210722685,-0.4651280046,0.0003967007,-0.9472005963,0.3264623880,0.0406349227,0.6955839992,0.6747277975,-0.8340126276,-0.6404592395,-0.4645507634,-0.5293129683,0.1810609400,0.5685809851,1.0409775972,1.8969447613,1.4060031176,-0.9676909447,-1.1406483650,-0.8645566106,-0.9631246328,-0.4495841265,-0.3670941293,0.1041967198,0.9390909672,-1.9684011936,0.2957823277,0.0055266228,1.9068297148,0.2354781479,0.0417030305,-1.7206138372,-1.1816716194,-1.6864529848],[-0.2795744240,-0.4381642938,-1.3464711905,-0.2227810323,0.4033103883,0.2749841511,0.2874121964,-0.0273968987,-1.6804682016,-0.8865109682,-0.2472569197,-1.1258128881,0.9342336655,-0.5807120204,-0.1418465823,0.0790987611,-1.0749927759,2.2134726048,-1.2006694078,0.2691707015,-0.2489442676,-0.8762915134,1.6632809639,-1.0335812569,0.8721647859,-0.1475594044,-1.2235310078,0.5650429726,0.2195928097,0.0491285175,-1.0111253262,-0.8699862957,0.1976990700,0.3421122730,0.4036723971,-0.0111169284,0.1301707774,-1.9352202415,-0.3411969543,1.5464907885,-0.8455371261,-0.2012025714,0.5543738008,-0.3549480140,0.9601488113,0.5847229362,-0.9601101279,0.1428328454,0.7909606099,-0.2072594315,0.5749731660,-1.2849420309,1.2086113691,0.4027648866,-0.9209063649,1.3428764343,-0.0497116633,1.1252496243,-1.4034109116,0.0854413658,0.5364454985,-0.4646377563,-0.2525386512,1.5714304447,-0.3075014353,0.7793367505,1.7497104406,0.1035423651,0.4156560600,-0.7896444201,1.5372807980,-1.0925191641,-0.3885779679,0.7863638997,-3.7716197968,0.3558905423,0.2657609880,-0.0128362346,0.2688941061,-0.3216839731,-0.8197370768,-0.4061123729,0.0722145662,0.6458423734,-0.5728822947,1.5331975222,-2.2644534111,-0.7149686217,0.7342334986,0.0273310989,0.4219648838,-1.8402411938,-0.5089277029,0.5984560847,0.6699381471,0.4165192246,-0.5552244186,1.8353990316,-0.6733164191,0.3321333230,-1.0508931875,-2.3420100212,-1.1035705805,0.2500540316,1.2323869467,0.9139543176,0.7953773141,-0.2368800640,-0.8238989115,0.9296641350,0.1124773398,-0.4949807525,2.1422438622,0.3602790236,-1.2223575115,-0.5173635483,0.2391503006,-1.1317293644,-0.8869965672,-1.9696947336,-0.8611670136,-0.4780994356,0.1703852862,0.0392747857,-0.8513867259,0.5941985250,0.7758460641,-0.2394779474,0.5881170034,-0.3526118100,-0.9439957142,1.4639183283,0.8294926882,0.2655101418,0.2055927962,1.5004353523,0.5081455708,-0.7707537413,1.5194914341,0.9345728159,-1.1830033064,-0.7755748034,-2.1746835709,-0.1002500951,-0.9036408067,1.1387488842,-0.0545942672,0.7622455359,-0.3603307605,-1.1414390802,-2.1771762371,-0.3917343915,0.9779875278,0.0251153242,-2.3134996891,-1.2564883232,0.5975403786,-2.0236263275,0.4483772218,0.4159247279,-0.9448549747,1.5468273163,0.4482175410,-0.6612976193,-0.4731059968,-0.4449138641,-1.9738931656,0.7042369246,-0.0061952700,-0.4430695176,0.7511484027,-0.6052109003,-0.7535775304,0.2447089404,-1.4141345024,-0.6515182853,1.1660512686,-1.8217883110,-0.6141258478,1.2125263214,-0.0212147497,-0.6908379197,-0.2760960758,-0.3090609014,0.0992453471,0.3898756504,0.6739951372,-0.2484564334,1.1012392044,-0.4972129166,0.4972700775,0.1244544089,0.8972615004,-0.9175131917,0.8274303079,-0.4539879858,-0.2072115093,0.5159810781,0.9847689867,1.0576518774,0.4988316596,2.1206722260,-0.2258833945,-1.7573635578,-0.6614010930,-0.5449556112,-0.3869622052,-0.1445140392,-0.1908085942,-0.1185690537,1.7395980358,2.7544355392,0.3639674187,-0.6550902724,0.3023473620,-0.2460299134,0.0533388853,0.6053739190,-1.3006967306,-0.2727494538,-0.8468128443,0.9766883850,-0.6045349240,0.2835266888],[0.2119962424,2.5724422932,-0.1248733848,-1.5877096653,0.4133778512,1.3906018734,0.3611457348,-1.4941810369,0.9097679257,-0.7895452976,-0.0131430048,0.4787768126,-1.4094642401,-0.7671945095,0.4627577662,-0.2127619982,-0.5586658716,-0.0810662135,-0.6329986453,-1.5000852346,0.1569671482,0.4610330462,0.5216055512,1.0951989889,-0.6010218263,0.5522876978,0.4321692884,0.6307811141,2.0530083179,-0.5217092633,-1.4897885323,0.6250126958,-1.5613023043,-0.0225469321,0.3653716445,-0.1923164874,0.4022558033,-0.3239069879,-1.1409021616,0.2954473197,-3.0193588734,-0.4682822824,-1.4773166180,-0.9042945504,1.0454072952,-1.1082838774,-1.0444512367,2.1708672047,-0.4832859039,-0.6777984500,1.5595563650,-0.2915486395,0.0221522450,1.6878607273,0.9222207069,0.5000102520,1.1645658016,-2.7195198536,-1.5850836039,-0.3821529746,1.1708577871,0.5796393752,0.0682036281,1.1589673758,1.6575613022,-0.7662689090,-1.2816127539,0.1619419008,-1.3800725937,1.1070289612,0.9855763316,-0.9594042301,-0.8101201653,1.4170614481,1.6217786074,2.5478603840,-0.9287158251,-0.8941593170,-0.7534239888,1.6682021618,0.3101405501,-1.0944006443,-0.0103884470,-0.6301780343,3.0968067646,1.1102194786,-0.8093426228,-2.0855097771,1.2799286842,-0.7997585535,0.6396434903,-0.7613869905,0.3158714771,0.2002303302,1.6275659800,0.6539255381,0.4313740432,0.8388326764,0.3235431612,-0.3483707905,-1.0462989807,-0.6488026977,-0.1168876439,-0.2802293897,-0.3582161069,0.7303908467,0.0247646552,-1.3823878765,-0.6700788140,-1.7218041420,-1.5610544682,-1.3730332851,0.0397199988,0.0339602269,-0.0213197358,1.0941541195,-0.2352717072,-1.6322890520,0.0799121186,-0.9556933045,0.9889248013,0.2835512459,-0.4392039478,-0.9414988756,0.4072564840,-0.9518223405,1.3833876848,-1.1340763569,-1.8409861326,0.5475594997,0.5772783160,0.0075433361,0.4877021611,0.7304438949,0.3374955654,-0.1783489287,0.5135381818,0.5862373710,0.4374061227,-1.2511309385,-1.0444641113,-1.2574287653,0.8642635942,0.8503614068,-2.4733905792,1.0161190033,-1.7650945187,-1.0526522398,0.9574536085,0.2752321362,0.3666337430,-0.3609675765,0.8975061178,-0.3459243774,-1.4031714201,0.3312491775,-0.6045495868,1.2702319622,1.0439270735,-0.9935119152,0.2147451937,1.1008656025,0.4299190640,-0.1372111440,0.6169565320,-1.5516846180,2.0646662712,0.9198255539,-0.3870903254,-0.2804978192,1.3506839275,1.4641412497,-0.8443898559,0.1116885245,0.3858242631,-1.1577074528,0.0645114258,-1.4962903261,0.5474752784,-0.3548987508,1.0024415255,0.5446375012,0.8305513859,2.6467790604,0.6494945884,0.0595578328,0.3597752154,0.2516870499,-0.1181238145,0.4422335625,-0.1153549701,0.0733081177,-0.2758054435,-0.1951264143,0.1095378771,0.3252094388,0.8930382729,1.0510355234,0.9716653228,0.1504226327,0.5727462769,0.2539176345,0.0424893796,-1.0405927896,-1.4690296650,-0.0724262744,-0.1799088567,0.0264645424,-0.1598746330,-1.4990167618,-0.2105566263,0.1332618743,0.7448723316,-0.9744883180,-2.1181311607,-0.7248430848,2.7197012901,-0.5189481974,0.6716082096,0.1522336751,1.0515916348,-0.3552295566,0.0526108779,0.9851053357],[1.0241189003,-0.5028194785,-0.8301829100,-0.3506364822,2.1013507843,-0.2975785732,-1.1915485859,-1.9310274124,0.0281611960,0.7278511524,0.0044026095,-1.3758076429,-1.2809020281,0.7183985710,0.2994307876,0.9146254063,-1.8804464340,-0.5685744286,-0.1684366763,0.6454023719,-0.1555209905,0.0017470572,-0.0461361818,-0.4751440883,1.6674766541,-0.2610110044,-1.3076088428,-1.0537695885,-0.1325166225,-1.6590558290,0.1996072829,-0.4648935199,1.6472010612,-0.7532877326,-1.2189908028,-1.2708369493,1.7304326296,0.9739592075,-1.6090298891,0.9269093871,-0.5141512752,0.3343603611,1.0514758825,-0.7026612163,-2.2228353024,1.6996566057,-0.0153976502,-1.1894229650,0.1029311270,0.9860951900,-0.0424149409,0.2471846044,-0.6132379174,0.1702657193,-0.9761342406,0.1023765877,0.9823054075,-0.2936163843,0.5247262120,0.2902828455,1.3442021608,-1.2850005627,0.8672465086,-1.5464787483,1.5290105343,-1.3002258539,-0.4517161846,-0.3036861420,0.4617317021,0.4716469944,0.1666602939,0.2115351707,-0.5178134441,0.9480272532,-0.0098326812,-0.2219161838,1.0748243332,0.0413280837,-0.3751921058,1.0054109097,1.0878173113,0.2635469437,0.8403769135,0.2879878879,-0.1068011448,-0.9250847697,0.2024889141,0.1903910190,0.7410366535,0.8781462908,-0.2524499595,-0.1668965071,0.7493336797,0.8561979532,-0.0669981241,-0.2866696715,0.0031576664,-0.3875811696,0.4032322168,-0.0186206941,-1.0397504568,0.5258228183,-2.0523731709,1.4011635780,-0.3832056522,0.2031720877,0.6009286046,-1.0024389029,0.9176901579,0.1213238463,0.4889194071,0.5418915749,0.6136344075,0.1356879771,-1.2291907072,-0.7746384144,-0.1509719342,-2.0120000839,-0.1013733819,-0.3087831140,0.5470914245,-1.7141972780,-0.2830403149,1.5329728127,-1.1248511076,-1.0335241556,-0.5896264315,-0.0678327307,3.0144801140,-0.1954812109,-0.8513020277,-0.8838554025,0.8399129510,0.4587032497,0.6862203479,0.4576050341,-0.9797825813,-0.9266083837,1.2219574451,0.1443065107,0.6248232722,0.2345218509,0.0487408191,0.1535460949,0.3320425153,-0.1576162130,-0.6590068340,-0.0338018723,0.4617294669,0.1008614600,-1.2741967440,-0.3311012089,1.3876305819,0.9555911422,-0.8060911894,-0.4946635365,0.9969357848,-0.1563356966,0.2311514914,0.8098935485,0.1144014150,0.3466364443,1.6067898273,-0.4078532457,0.8250998259,1.1099538803,-0.0130652459,-0.8130413890,-1.2216724157,-1.2805823088,-0.7013052702,-1.1525663137,0.5510089397,1.4384489059,-0.0168083627,-0.2640907764,0.5070714951,-0.6443662643,1.8311312199,0.6864961386,-0.1773331016,0.2223710716,-0.0704963654,-0.9840225577,-0.3356443942,1.0691854954,0.1201403961,0.5101134777,-0.8782663941,-0.5339717269,-0.9799844623,0.7834316492,0.7693622708,0.2057720423,-0.3445561528,-1.4284933805,-0.0017824746,0.9755434990,0.7316427827,0.0141161242,-0.0398816057,-1.2709821463,-1.5649716854,-0.3071387112,-0.5473391414,0.9616869688,1.5145448446,-0.7195694447,-0.3944427967,0.4071395993,0.1094340906,1.9397739172,0.0562009476,0.5139475465,-0.0046406253,0.0536796823,-0.8677092195,-0.7403584719,0.0448231101,1.3504319191,-0.3382182717,-1.2779160738,0.2713087797,-0.4566406012],[1.0716338158,0.2072720528,0.3301466703,-0.2964983284,0.5896486044,-0.4153626859,0.5670648217,1.1750820875,1.4873989820,-2.3126528263,-0.8691516519,0.4574732780,-0.8475249410,0.8546934128,-0.6666801572,0.1129124910,1.2397251129,-0.3301310539,-1.1247928143,0.1654882878,0.2989130020,-0.2470651269,-0.3026075363,0.1921841800,-0.1730351001,0.4377845526,-0.2193670720,0.6114021540,0.6966733932,0.3748733699,-1.4609225988,1.5907051563,0.9739543796,0.8248856068,-0.3926510513,1.5374946594,-1.1393166780,-0.2162578404,-0.7730450034,-0.4251903296,0.5946242213,-1.1560473442,-1.5628254414,1.1010065079,0.2154533267,0.7111494541,-0.0475142524,-0.0388706252,-1.2525222301,-0.5174888968,-0.5561483502,-1.1154675484,0.1784954369,0.8350765109,-0.4459244013,-2.4139649868,-0.8504996896,0.5401726961,1.9785127640,-0.6393229961,-1.3613717556,2.0546019077,0.0595806055,-0.4285869002,0.9154852033,-0.3232243061,1.5594203472,-1.1150901318,2.0952014923,0.5021405220,-0.7830901742,-0.7693719864,-0.1663081795,1.4347996712,0.2960758805,-0.0027330727,0.2682686448,0.0127709676,0.2424747497,0.4638241827,1.7889491320,1.1757079363,-1.0222588778,2.6014802456,1.5808367729,-0.2146649361,0.7575861812,-0.4495947361,-0.5908117294,0.5592653751,-1.3259441853,0.5948846340,0.4069502056,0.8398963213,-0.4906890988,0.6471970677,0.6398313046,1.6471980810,1.4133006334,0.8426824808,1.0106039047,1.0245624781,0.0944321230,1.4529879093,-0.6713413000,-1.5175800323,2.1336538792,0.7717633247,0.6927742362,-0.4853357077,1.3379818201,-1.6785788536,-1.0422717333,0.5675963759,1.6423405409,0.1245919019,0.9202539921,0.0586088188,0.4284663498,0.1830497682,-0.1064387262,2.5498642921,0.1319633871,-0.8298854828,0.2510634661,-0.8929324746,0.6934131384,0.8412130475,-0.3025356829,-1.3898062706,0.7125902176,-1.8900271654,-1.2866756916,0.3507175446,1.0069307089,0.3439614475,-0.4223397672,0.4167747498,-0.6913272738,1.3721678257,0.1986006349,-0.5031130910,1.3957043886,0.4102612138,0.7169515491,-0.6730329394,-1.6226317883,2.0858526230,0.0266461149,1.1714679003,-2.1153361797,0.0740644708,-0.6488339305,0.0858231783,-1.3241528273,-1.0434464216,-0.3803936243,-0.6672549844,-0.1567256898,-2.3133258820,0.1390058398,-0.4018515348,-0.4512442946,0.3724332154,-1.1800881624,-1.3332873583,0.7908762693,-1.3407357931,1.6218533516,0.1306812912,-0.6198000908,0.0341969095,0.6919595599,-0.1018244103,0.3843306005,1.2551382780,0.1839106530,1.3701967001,-1.9215074778,1.5717345476,-1.0818972588,-0.2024257630,0.0068036322,0.7846305370,-0.4962382913,3.0493257046,2.2000374794,1.3815038204,0.5984432697,-0.6466378570,0.1070459485,0.0651153326,-0.4481697977,0.4599678218,1.3020391464,-0.7581306696,-0.2754032910,-0.0371919386,-0.5418404341,0.7452659607,-1.1534259319,0.9381926060,0.6659347415,-1.1937292814,-1.5376166105,0.1310789287,0.2398359627,0.5549139977,0.2483216375,-0.6198633909,0.8630862236,-0.1393041015,1.0174250603,0.3143958449,0.0683760419,0.0315545090,1.3078721762,0.4492118359,-0.4071181715,0.5910757184,0.3741379380,0.2453961819,-0.8749790192,1.2522603273],[-0.3844586313,0.0375177786,-0.6383576989,0.7360663414,-2.3159997463,-1.3596284389,0.4820798934,0.7575388551,0.8861390352,1.0024284124,0.5143792033,-0.6864389181,-0.7257505059,1.7803571224,0.5731166601,0.8511525989,0.8942019343,0.3404513001,-1.7471883297,0.8651315570,0.4115097225,0.9011681676,1.6878750324,-1.6129397154,0.5108485222,-0.2522997558,0.7438052893,0.5033487082,-0.4613233209,0.4820011854,-0.4439652562,-1.4911876917,-0.3602609336,-1.5907628536,0.6927799582,-0.1477584839,-1.0334455967,-1.5674887896,0.4550902545,1.5604319572,-1.4422808886,0.5788357258,-0.2870239317,-0.4004207551,-0.8785344362,-1.4270888567,0.9402923584,0.8992320299,0.4155126214,-0.8877316117,1.7329494953,-1.5584703684,0.2766417265,0.6156333685,-1.3220993280,0.7813851833,-0.6368366480,-0.6957759261,-0.5995090008,-0.3846778572,-0.0360092148,0.2149695456,-1.0402828455,-0.8459938765,-0.9112716317,1.0995310545,-0.9367135763,-1.9902956486,-0.0227747150,-0.4758028686,0.9322866201,0.9273737073,0.8070508242,-1.7838457823,-0.6699023843,-1.5915911198,-0.2805326581,-1.4233908653,-1.4749735594,0.1594439596,0.0329884961,-0.8356202841,1.3274855614,1.0824410915,1.0488785505,-0.4469273984,-0.9992522001,-0.9932340384,0.5085969567,-0.5421534181,1.9483562708,1.1397675276,-0.0622630306,0.1688272506,-0.7671681643,-1.1403740644,-0.7531850934,0.3955881298,0.6842200756,-0.5735817552,-0.5361065269,-0.7205488682,-0.0343498290,0.4714860320,-0.1568353325,0.5368303061,0.7607414722,-0.7243655324,-1.3951404095,-0.8374814987,0.4954386055,-0.0606812388,-0.6688954830,-1.6458029747,0.6280921698,0.2495674193,1.5667924881,1.3704221249,-0.1882616132,-0.3503639698,0.6975366473,0.4451564252,-0.6488027573,-2.6514444351,-0.9436893463,-1.0788868666,-1.0912772417,0.0611912012,-1.4206846952,-0.6495169997,-1.2007901669,-1.3360172510,-0.0519311912,-1.2045214176,-1.1121664047,0.6280339956,-0.5048688650,-0.4457834363,-0.4472849965,1.1495735645,-0.5288956165,1.2007362843,-0.4974692166,-0.3109487891,-0.9244396687,-0.2693705857,-1.0468840599,-0.6721541286,0.5465496778,0.0884050056,-0.4098713398,-1.0514574051,1.4148094654,-0.4750121534,0.6167279482,0.9246222377,-0.4253928065,-0.6544781923,0.7891964912,0.3140968978,0.7648479939,-0.1695642024,-0.1479424685,-0.1064250320,-2.2116746902,2.3312244415,-0.6676529050,-0.9457675815,0.9474021792,0.7973690629,0.1704794019,-0.7035692334,0.7092891932,-1.8698717356,-0.2639527023,0.2181621939,-0.4548698962,1.5478996038,-0.4820409119,0.6898891330,0.6085611582,0.0193841830,1.8152153492,-1.6339917183,-1.3337707520,-0.5393002033,0.1599944085,0.4317499101,-1.7887134552,0.7140734196,-0.8148634434,-0.4388350248,0.6136690378,-0.8982139826,2.4423866272,-0.9562060833,-1.4921243191,-1.7531054020,-1.9354444742,0.1809691191,-0.9431200027,0.1708678603,-1.9138517380,-1.3267153502,-0.8237664700,-0.1747481823,0.8606752753,0.8452942967,-0.3969329000,-0.7858321071,-1.0196638107,0.1269257069,1.0621590614,1.5306878090,-1.5189861059,0.9269375205,1.2335311174,-0.4745801091,-0.6515474916,0.7936541438,-1.3056498766,0.2814712524,1.4409999847,0.8073496222],[-1.8721452951,0.1184589267,0.8932977319,-0.5317001343,1.8536045551,-2.7512083054,-1.3216940165,-1.3283517361,-1.1938714981,-1.2970254421,1.1642961502,0.3630233407,-1.1360250711,0.4634298086,0.2409296781,-0.9782558680,1.8368214369,-0.0928911120,1.2228311300,-0.4003975093,-0.3872050047,-0.2419685572,0.1414249092,-0.3762511015,0.1452577263,-0.2219312191,0.7660420537,-0.7883638740,-1.6409637928,1.1355031729,1.3309619427,-0.9560182691,0.4649753571,1.1633868217,-1.4728108644,0.2295930833,-0.1990584433,0.3435695469,0.8488633037,-0.3607235849,-1.8055906296,0.2481548339,0.7938757539,-1.8822200298,1.2305692434,0.9074109793,-0.3646234274,2.4204812050,-0.8100681305,-0.5929164290,0.8905769587,-0.7271680236,0.3828736246,1.0301723480,0.7428522706,1.0397125483,0.5743683577,0.9277793765,0.3855857253,-0.4105027914,0.5453033447,1.5967614651,-0.2897183895,-0.9443405867,0.9612666965,-0.2591313124,0.7493833899,0.9668287635,2.2179422379,1.4288818836,-0.3036374748,0.3314071596,0.1685507894,-1.0392718315,0.5757080317,1.8846974373,-1.0078357458,0.4577882290,1.0183068514,-0.7346784472,-0.4650122225,0.0640585124,-0.7890337110,0.7123513818,-2.1669125557,-0.7675073147,-0.9449701309,0.8574027419,0.7751913667,0.1898535192,1.6693571806,-0.3673186004,-0.0210592635,0.4653729498,-0.6247548461,0.1980665624,0.7378042340,-1.0381380320,0.8246774077,-0.1759186089,0.6200713515,0.3294046819,0.2967543006,-1.0276694298,0.0227232017,1.1291382313,-0.6281598210,0.8043770790,1.0822321177,-0.1657423079,-0.0776807144,-0.1167014837,0.3684995174,1.9841905832,0.4295887053,0.6405870318,0.6379342079,0.8725138903,0.3877270222,0.8504217863,-0.0968416929,-2.1198146343,1.4159415960,-0.3126500249,1.5506500006,1.0458365679,-0.3016704917,-0.2235870957,-1.2391153574,-0.5851184130,0.5509126782,0.4651997685,-0.4118579626,1.5482922792,-0.0040421048,-0.9912667274,0.0142603684,0.8469740152,0.1916214526,-0.0247817542,-0.0518606752,-0.7906127572,0.6237937212,-0.2116433233,1.8560798168,0.6758502126,0.4323855340,-0.5427485704,-0.2471507788,0.8266699910,-1.2941006422,0.0277775917,-0.0114485594,-0.1420166045,0.4924000800,0.4590295553,2.5640699863,0.3363261521,-1.5822482109,-1.6178342104,1.3771090508,1.1515468359,-0.9595029950,1.7391210794,0.6310431361,-0.8181252480,0.3020314276,-0.8029966950,0.6402264833,1.1647168398,-0.8939806223,0.1350935549,-1.0642356873,-0.5717391372,-0.6346552372,0.5374536514,1.3573206663,-0.3348610401,-1.6466457844,-1.3988031149,0.6637595892,-1.4271485806,0.4709821343,-1.4963316917,1.3809591532,0.3187876642,1.0921761990,-2.2258255482,-0.5045356750,-0.7421919107,-0.2157391161,0.7126739621,0.6691911817,-1.1976441145,-0.8853336573,0.8246211410,-0.9868354797,-0.0254206806,0.0362228155,-0.1663735956,-0.9848557711,0.7515419722,-0.7585852742,-1.0937254429,-0.1821856797,-0.6961805224,0.3334859610,0.4389159083,0.2469871491,-2.0076756477,0.4291914105,-1.2334452868,1.3440158367,0.0627860799,-0.9219018817,-1.2558528185,-1.9622758627,0.1995464712,0.4981423914,-0.5179405808,1.9443092346,-0.4232400060,0.0491162017,-1.3236367702],[-0.6165354848,0.7170464396,0.1112871245,-0.8130300641,0.2534252405,0.5796111226,1.9850246906,0.4960811436,-0.2429900765,-0.4833355546,0.6328579187,-0.6804760098,-0.1585241705,0.3408620358,-0.4281844199,-0.8441763520,-0.8560928702,-0.5648352504,0.0108333835,0.3450529873,0.7727462053,0.5534866452,1.6443219185,0.5270298719,0.2769190669,-1.0014778376,0.5765803456,-0.8796996474,0.5687882900,-0.0461753644,-1.2852069139,0.7221946120,-2.7615418434,0.0332008004,0.2197542191,-0.2995602190,-0.0586884245,1.8317363262,-0.6810992956,1.5084025860,0.3555968106,-0.2981703579,0.9358145595,1.9291380644,1.2278101444,-0.2416273504,-1.0936526060,-0.2135961056,1.5356572866,0.9269886017,-0.4677660167,0.5069100261,-0.5780176520,-0.2702128291,-0.5411428213,-0.3239991665,1.5921620131,-0.3274900615,0.0973749086,-0.1159433350,-0.7867313027,-1.3489117622,-0.6642484665,0.1416206062,-0.3015295565,0.7219636440,-0.5237560272,0.3400763571,-0.1233913451,-0.2009309828,0.8506157398,-2.1546499729,0.3547619581,-1.6757706404,-0.1415503621,-0.8680157065,-0.0768875107,-0.1592646539,-0.5014755130,0.0249998868,-1.8573825359,-0.3155896068,0.5905711651,1.0066084862,-0.5826127529,-1.1200389862,0.1239113286,-0.4321572483,0.2175883204,-0.1679378450,0.6261950731,0.5056493878,-3.1710517406,-0.1413709968,0.6767768264,0.3793789148,0.3022807539,0.7109302282,0.2650648355,-1.0345375538,-1.6468921900,1.0483371019,1.0237673521,0.3251764774,1.2741376162,0.2375988215,-0.7581788301,-0.5037139058,0.0572597869,-0.8450796008,-1.1608557701,-1.0911349058,-0.5320959091,0.9157405496,0.4893115759,1.6712793112,-0.5092890263,-0.9690325260,1.4766815901,0.7314090729,-2.4690501690,-1.3202705383,-0.3301222920,0.8962110281,2.0390863419,1.0254950523,0.1695734710,-0.2835012674,-1.0160237551,-0.2788228989,-0.0651427954,-0.3828991652,0.2328362316,-0.7724130750,0.5774948001,-1.8650261164,0.2756900489,-1.3820643425,-1.4922863245,-0.3723741472,-2.0753431320,-1.7942260504,-0.3470088840,1.7341159582,0.8438899517,1.0441874266,1.4977054596,1.0677449703,0.1652990133,-0.1096124351,0.1542658657,-0.0562523417,-0.0727530718,-1.1196274757,1.4136055708,-0.4164610803,-2.0809350014,0.0478956066,0.9895527959,-0.8603312373,-0.6730191708,-0.5802956820,0.7802674770,-0.9864706397,-1.9121707678,0.6803776622,0.1682149619,-0.7673743367,0.7056384087,-1.0421631336,-0.3231330812,-0.9202150106,-1.7474627495,-1.6890426874,-0.0504816063,-0.7599089146,0.6442566514,-1.4180889130,0.8134864569,0.3692963719,0.3944670260,0.1946570277,1.3176888227,1.3182153702,-1.4753251076,0.2395100147,-2.1885092258,1.0353777409,0.4226379693,2.4927759171,-0.7165281773,-1.2685643435,0.7180500031,-1.0596232414,-0.1318729967,1.2955181599,1.4145418406,0.3703507483,-0.2906166613,-2.1202604771,1.5874541998,-0.4480857253,0.1233151332,1.2725905180,-0.4341225624,0.8674485683,2.0624625683,0.5839276314,1.4544333220,0.3140803874,-1.1686724424,1.4655153751,-0.1854373813,0.0675033107,-0.1820190549,1.3975430727,0.3684431016,-0.5828213096,1.6327118874,-0.5973852277,0.6849465966,-0.5043992400,3.1433739662,1.0341066122],[0.1156917736,1.5954945087,2.6153404713,0.6946935058,1.7668703794,1.1358177662,-0.5453659892,0.3216398060,-0.3820880949,-0.7366890311,-0.5614402890,-0.6733281612,0.5156762600,-1.2313117981,0.5289177299,1.3301786184,-1.7023651600,-0.9341116548,-0.8916813135,-1.1652710438,-0.1437996477,0.9670381546,0.2372026592,2.4822874069,0.5470020175,0.4206858873,2.6281704903,-1.5163463354,-0.5004493594,1.2942957878,0.6173464060,0.2298103869,0.4356625080,0.3032996058,1.1061048508,-0.3733479083,-1.9930585623,1.1398425102,1.1698259115,-1.2241064310,0.7899841070,0.2383679450,-0.7305232286,-0.4423871934,0.7964922190,-0.4245458543,0.3769742846,-1.8120678663,0.5874359608,-0.0313255563,-0.7554463744,0.6911882162,-2.2524123192,0.5145077705,-0.4655631483,-0.5376864076,-1.6770755053,-0.6433816552,-0.0429741144,1.1280231476,1.1490014791,-0.0671586543,-1.2202082872,0.7045454979,-1.0823764801,0.1428896487,-2.9295396805,0.6375863552,-1.7725259066,0.6920745373,0.0506045334,-1.8579263687,1.0407875776,-0.1360074580,-0.8596337438,0.0067301448,-0.1954611391,-0.1238392219,-0.8143445253,-0.6136426926,-0.9348247051,0.4640007019,0.7521933913,0.6640645862,-0.1378400475,1.1553994417,1.0206215382,0.3986735940,-0.4069051743,-0.0785310194,0.4717287719,0.2707085907,0.1270379424,-0.4223307371,-0.3201220334,-0.3389673233,0.4850408137,0.4809367359,0.2361204922,-1.0482326746,1.0621023178,-0.1028745249,0.1858724207,0.3908754587,-0.6750885248,0.2174529731,-1.0515180826,0.2160267383,0.2982191145,0.8590397835,0.0086025409,0.5641648173,0.4481794834,-0.6979264021,0.3443371356,0.3175536394,1.2855532169,-0.7491867542,0.3338579834,0.7577729821,0.3670987785,0.2343304902,-1.9538394213,0.7163391709,0.3700796068,-1.7885614634,-0.9458172917,3.0109028816,1.4560543299,0.6147252321,0.6920373440,0.6372101307,-1.3865582943,-1.7684077024,0.5304525495,-0.0079795672,0.8801140785,0.9465383291,0.1581633389,-0.8426349163,0.9479221702,-0.1354119480,-1.5711832047,-0.3637903631,0.4112898111,0.1602426320,-0.1830785424,0.7868187428,0.4419188499,0.0504920371,-2.2241265774,-0.6185190678,-1.1892626286,0.6705932617,1.0291326046,3.0448400974,-0.8008592725,0.9581548572,1.3260705471,1.1762444973,-0.9842439294,2.0668766499,-0.3960740864,-0.1752684265,-0.7192801237,-1.4173226357,-0.5742908716,-1.1648813486,0.9344018698,-0.1113579199,1.6854794025,0.3993677199,-1.1008958817,-0.5278835297,-0.7492056489,0.7847382426,0.4598958194,1.8640118837,0.4370495081,-0.9409788847,-0.5925995708,1.2170940638,-0.4932170510,0.7076863647,1.2122285366,1.3748151064,-1.1437553167,0.1958882958,0.1235001981,0.0688811988,-2.3859999180,-0.5745288730,1.7441747189,-0.8688986301,0.6938255429,0.2857529819,0.0937574208,-0.6805610657,0.6712647080,-0.2787623405,-1.1043686867,-2.3983387947,-0.8090463281,-1.0024791956,-1.1571393013,0.3918244541,0.3093427420,1.1425877810,0.9271572828,0.7051991820,1.2124754190,-0.4664890766,0.0037588740,1.7412911654,-0.4158358276,-1.5634813309,-1.3250594139,-0.2688057721,0.2384223789,-0.5030126572,0.8553127050,0.4374273419,1.1957253218,0.0462666750],[-1.8070065975,-0.4504835904,-1.3686780930,-0.3058474660,-0.0495945141,0.1645372808,0.1849104613,-1.7716939449,2.3738782406,-0.1378646493,2.4643244743,0.5075989962,-0.1704922765,-0.0185646489,-0.0093221944,2.1728961468,0.9254801273,-0.5384533405,0.9051353931,0.3683229089,-0.8445315957,-0.3709745407,-0.9525077939,0.6499077082,-0.0273034014,-0.4088584185,2.1410124302,0.8542435765,0.9650903940,-0.6568892002,1.6100125313,0.7433479428,-1.6218360662,1.1497720480,-2.6727676392,0.3415644169,-0.4711919725,-0.4807176888,2.3651483059,-1.6804189682,-0.7778606415,-0.5134579539,0.8294714689,0.3388239145,0.5550401807,0.3536906242,0.8353669047,-1.1603729725,-0.9736599326,1.4669877291,1.6135033369,-0.1117185876,-0.3129193783,1.0819211006,-0.7731477618,1.1307908297,-0.6786795259,0.2116177529,-0.0737237260,-0.4597691596,-1.1927083731,-1.3991508484,-0.2123755366,-1.2098524570,1.4069837332,-0.2333389074,0.0752018467,1.4358216524,-1.5128277540,0.4061498940,-0.7061683536,0.2710811794,1.2111703157,-0.4321515858,-0.3260831237,0.0225785729,0.2212020308,1.4104518890,0.2452560216,-1.5462257862,-1.2980943918,-0.1110542938,1.1513741016,-0.3489472270,-0.8876929283,-1.0408627987,0.4383756220,-1.5169585943,0.4438357651,-0.3842916191,2.2116289139,0.2712109387,-1.6035920382,1.0682024956,1.2236803770,0.5410172343,-0.5111415982,-1.3180493116,0.4625207186,-0.0136173591,-0.5168929696,2.2955324650,-1.3852297068,0.6380922794,0.0593690909,1.6699051857,-1.6007295847,-0.3379013240,1.0723598003,-0.9798635244,0.8318992853,-0.8471778035,0.3142752051,0.5284886360,1.1218874454,1.8486001492,2.7345709801,0.4374895692,-1.2230619192,-0.2786168754,1.1966751814,0.7694658041,-0.4521544278,-0.4873460531,0.0760180429,-2.1347806454,0.9052285552,-0.5944761634,-0.3380056322,-0.1664792448,0.5691460371,-0.5731565952,-2.2000503540,0.0652889013,0.9432873130,0.3104563355,0.5494231582,-0.9663535953,-0.5229769349,-0.2445623577,0.5183826685,-0.9060575366,-0.9137417674,-0.1996527612,0.1335192323,-0.4433824420,0.3377778530,-1.6413617134,-1.7930675745,-0.1941461861,-1.8694553375,-0.9999119043,-1.4513466358,0.6723552942,0.1838610321,0.7941121459,-1.1660890579,-0.8116760254,-0.9434131980,-1.1249563694,-1.1655663252,0.6973057985,-0.6014246345,1.2728025913,0.1407245845,0.8612917066,-0.2324358374,1.4373520613,0.8628004193,1.5325776339,-0.1889394820,-0.5824972987,-0.9653490782,1.3446065187,-0.7162488699,-0.1897031218,2.0714435577,-0.6121030450,2.0374407768,-1.4945025444,-0.2567083538,-0.8972054720,0.1291142851,-0.3037604690,-0.8362293243,0.7068982720,-0.4486835897,0.2427803576,1.0932581425,-0.1559365541,0.7241747379,-0.7676721215,1.5684889555,0.7923648953,0.9502861500,-0.3227502108,-1.2624378204,0.2234718055,0.4752887785,0.2963661849,0.2820369303,-1.3083859682,-1.8737363815,-0.3256417215,-0.1815035492,0.3622895777,-0.2257685065,0.1857555956,0.0977337956,-1.1289507151,-0.1029930860,0.0768001899,-1.2031559944,-0.1173039898,0.0196279101,-0.2452402562,1.1445440054,-0.7530807257,-0.6296615005,-1.0584937334,0.7693899274,0.8202177286,-0.3264771104,1.0771063566],[-0.4935064316,-0.5727998614,0.1287579536,-0.9393902421,0.9558954239,0.7553883195,0.6227222681,-0.0084682750,0.2514541149,0.8778777122,1.0188686848,-0.2856886983,-1.8559024334,0.0303529315,0.1430030465,-0.3618414998,-0.0351615176,-0.6785556078,-0.9759765863,-0.4848660827,-0.2929255962,-0.1925849020,-0.7591896653,-0.3228958249,-0.0205045063,0.3483543694,0.2820397615,-1.6995092630,-0.7122277021,-1.3917955160,0.5476497412,-0.3970116079,-0.0744653568,-1.1424483061,1.3950189352,1.2138121128,0.7932989001,0.3546606302,-0.1634015590,1.4944585562,0.7755617499,0.7647002339,1.2824169397,0.3859403133,-0.1856594533,-0.9363436103,0.4315616786,-0.2700640857,0.3939462602,1.4889338017,1.0297901630,2.0167706013,-0.5965355635,-0.8565047979,-0.9398768544,0.2297771126,-0.5574417710,-0.1267592162,-2.3832242489,1.1377506256,-1.1425361633,-0.5246202350,-1.5346717834,0.1314383298,0.0684951022,1.1131851673,0.2731709778,0.6787238121,-0.5446536541,0.0450214595,0.8453890085,1.8095526695,-0.2790301442,1.1491837502,-0.4293262064,-1.3085041046,1.0832661390,1.3500005007,1.1210151911,0.5666023493,-0.4597205520,1.0817568302,0.7697666287,0.8304558992,-0.2385064662,0.8480877876,-0.7320706248,-0.5562512279,-1.0011099577,0.5937754512,-0.8616179228,0.8863510489,1.6711868048,1.0623165369,0.6543915868,-0.1773722172,-0.0344006717,0.5973542929,1.2686825991,-1.5581994057,-1.1329184771,-1.2591944933,-0.6952503920,2.6306145191,-2.2633259296,1.0291699171,0.9703497291,0.7907989621,-0.9672769904,-0.7324648499,0.5958577991,0.2923487723,0.0744326934,1.4750037193,-0.1998525858,-0.1739941984,-1.8254380226,0.3562199175,0.7012470961,-0.2482330352,0.4486927986,0.0901318491,0.7526713014,0.2792355716,0.5251368880,0.3370932341,0.6470039487,-1.2970519066,-1.4007022381,-0.9944904447,1.5340650082,0.7310197949,-1.6445587873,0.4814043939,-0.5434415936,-0.9484261274,-1.4867604971,-0.5797979832,-0.1667503268,-1.4086076021,0.9149457216,0.4772666395,-0.6579295397,-2.5258929729,-1.9978051186,-0.2341778129,1.2802121639,1.7618249655,-0.8002349138,0.3736089170,1.1083867550,0.1761967391,0.4046993256,0.9638820887,0.8057360649,0.8382278085,-0.0438889638,-0.2375431359,-2.0383751392,-1.0941091776,1.5616539717,-2.0328569412,-0.9788680673,-0.6005727053,-0.5285338163,2.7921683788,-0.7145404220,-0.8009266257,-0.2848567963,0.6346355081,-0.4794748127,1.5668632984,-2.4055385590,0.4379455149,0.5932794213,0.6854198575,0.9216139317,-0.3407145441,-1.3855520487,-0.7462887764,-0.3033876419,0.0186376907,-0.0690334588,-0.2632014751,1.6265926361,1.4949783087,-0.6602244377,-0.7783056498,-1.3226950169,1.5882486105,-0.0956622288,0.7664166093,-0.5125342011,-0.7737212777,0.6168346405,0.7540829182,1.1455770731,0.3315310776,-0.1345494241,-1.5078363419,1.8647091389,1.8537608385,-1.0891287327,-0.5876540542,-0.3828836977,0.5380443931,1.4859291315,0.1173877418,-0.0830619484,-0.2604106069,0.9005637169,0.1149029881,-0.8461684585,0.9646666050,-1.3376936913,2.1376085281,2.3744444847,-0.1123782471,-0.6790237427,-0.2712702751,-1.2458635569,-0.2799184620,-0.0585089289,1.2172613144],[-1.3013261557,0.4938486814,0.0018817389,0.1639826447,-0.1096468046,0.1895873547,0.6653810740,1.3794633150,0.3325508237,-0.3157938123,-0.4053605795,-1.2119607925,1.6195567846,1.0380227566,0.0044104871,1.2443182468,-0.5239819884,-3.1816990376,-1.6022835970,-1.0422313213,1.6971408129,-0.1422895491,-0.3386924267,0.0337778777,1.3895584345,-0.6798459888,-0.8735042214,1.4608218670,0.9804942608,-0.0700358227,-2.2575590611,-0.5207765102,-0.5970131755,1.4351145029,0.5342422128,-0.5326063633,-1.2744994164,-1.6573332548,0.1688603461,0.8684266806,-0.4371896982,1.9104814529,-0.1397882998,0.7860920429,0.3916041851,0.1504721791,-1.0214750767,0.2536700964,0.0443325303,0.4851707220,0.0727755502,-0.1179084405,1.6902691126,0.2719678283,-1.1939750910,-0.0545632914,-1.5358672142,-0.0898130834,1.1942884922,0.9992315173,1.0113880634,-1.4582569599,0.6580271721,-0.8810248375,0.7868432999,0.6703838706,-0.4072601795,0.8141466379,0.7604040504,0.2462893724,0.0050235200,-0.7474465966,0.6245143414,-1.0206460953,-0.8153722286,1.1893461943,0.4940040112,0.4967643023,-0.2958534658,1.8670926094,1.3553891182,-1.0535255671,-0.1981259435,-0.3524850905,-0.6183097363,-0.0779046416,0.7972301245,-0.5016496778,0.4480767846,-0.2101167887,-0.0168276727,-0.2014111727,-0.5612958074,1.3728827238,-0.3534809053,1.5735669136,0.9426801205,-0.0221692715,-0.9934756756,1.4621682167,-1.4462176561,-1.5750948191,-0.2356135994,-0.3192728162,1.4097740650,0.3186092973,-0.0515878908,0.4262378216,-0.0413998328,-0.7294147015,0.5786992311,0.5278693438,-0.2732734382,-0.0219900087,-0.5813630223,1.7156662941,-0.3235935867,2.4618074894,-0.6561592817,0.0910400450,1.2659327984,0.8638816476,1.0664783716,-1.2248061895,0.6043577194,1.0140352249,0.4129702151,-0.5681805015,-0.0218119808,1.3225139380,-0.0147479223,0.0736440644,1.1261093616,1.4245935678,0.9596525431,0.5288068652,1.1710895300,-0.7957918644,-0.1596978903,-1.3176041842,-0.3115763068,-0.0672747642,-0.3569861352,0.2234518230,0.3094016016,0.9527436495,-1.2522807121,-1.2509135008,-1.6274449825,-1.2778787613,-1.0911378860,0.2284249663,-0.3604622483,1.1675195694,2.0061020851,0.7697836161,-0.9829636216,-2.2009537220,-0.1629348844,-0.7106205821,-0.4898200333,-1.4288221598,1.0543422699,0.1862355769,-1.5556902885,-0.2575588226,0.5807380080,-1.8932569027,-0.4353618026,1.5906252861,-0.9965630770,-2.0895090103,0.9025256634,0.3491262496,-1.4236516953,0.1656542122,0.2637915611,-2.3229284286,-0.7769226432,-0.4434509277,0.2784052789,-1.7719700336,1.9006904364,-0.3290591836,0.4922101796,-0.5404827595,-0.9461563230,0.6440004110,-0.3489286602,-0.9171313643,0.4032333195,0.5990763307,-1.7341868877,1.2567689419,-0.2003704309,1.5010881424,-0.1889965534,-0.6302958727,0.1775143445,0.2809249461,1.7796574831,0.4534760118,0.1546487063,-1.5418481827,-0.5078651905,-0.8182194233,1.2638050318,-0.1745502055,-0.4214827716,-1.0797538757,-0.1164133102,-0.7410816550,-1.3497550488,-0.1168411151,-1.3411626816,-2.0407903194,-1.2275632620,-1.3231233358,0.0243034586,0.4815312326,1.4609789848,-1.4686429501,0.1677938551,-1.0592018366],[-0.2852744162,-0.0706676841,-0.1581604034,0.9220636487,-0.1272540987,-0.2593966126,-1.7004094124,-0.5371990204,-0.7370152473,-0.0971123576,-0.3767374456,2.3517546654,2.6098873615,0.4989961088,0.0687675998,-0.3759937584,-0.8398221731,0.9545980692,-1.4972980022,-0.9398316145,1.1029561758,0.2521598339,-0.0855884403,-0.9977290034,0.8008356690,1.5740467310,0.4187212586,1.9298514128,0.1145944372,-1.1502786875,0.0434628800,-0.1572289616,1.0625115633,-0.8050823808,0.2032386810,-0.0399338417,0.5518578291,0.8568918109,1.7390502691,0.6615341306,-0.5819038749,-1.2263169289,0.1351081878,-0.1532971412,0.2919063568,0.6675716043,-0.0507290736,-0.6389257908,-0.1211285144,-0.1745242774,-0.2457411587,0.2195800394,0.6483494043,-2.5658056736,0.9271953106,-0.3155171871,0.2342108935,-0.4574358165,1.2141743898,2.0983936787,1.2039935589,-0.1287723780,-0.0756367967,-0.2917146683,-1.5107499361,-0.0836014524,0.7631447315,1.9176464081,-0.5091345310,0.0782249793,0.1212063432,0.6442352533,-0.7711468339,0.5957395434,0.2253505737,-0.9019545317,0.5706453919,-0.5385786891,0.9147205949,-0.7850139737,-0.1509184986,1.2764766216,-0.5555887818,0.5547939539,1.4719022512,-0.2048540562,0.1632129550,-1.7694426775,0.3638873100,0.2493520528,-0.5556089282,0.6050454974,-1.1778032780,1.2873336077,-2.7311754227,-1.8287607431,-0.3213097453,0.2433038205,-1.2711951733,0.2429094762,1.5860184431,1.2297668457,0.9442595243,-0.8065596819,-1.2923262119,0.8501077890,0.3705936670,1.4010832310,-0.4744163156,-0.2130869031,-1.9867215157,1.2959045172,-1.9881888628,0.1270659864,1.1208313704,-0.1007476598,-1.1433990002,0.1744137406,0.2917715907,0.2511937320,1.7150257826,0.9280009270,1.1453393698,-0.3360704184,0.4720384777,-0.3854337335,-0.4944903851,1.6074732542,-0.7994801998,1.4878766537,-0.0383373089,-0.4597311318,0.4766893387,0.7497329712,0.7272310853,-0.5855666399,-0.1302246153,0.3075602651,-2.5834276676,1.1175756454,1.2175463438,-0.4559751749,-1.0002273321,-0.4586575031,-1.1881129742,-1.0883032084,0.4656453729,0.1931448430,-1.4194663763,-0.1684705317,-0.0929002762,1.4055173397,0.1011286005,-0.2336747944,-0.7242429256,-0.4713487029,-1.6743985415,-0.4759764373,0.6226197481,-1.0360772610,-0.7374758720,0.3776315451,0.7032527328,0.3570717871,1.6821124554,-0.7404009104,-0.6392166018,-1.8168293238,-0.2498717159,0.9376166463,0.3249538541,1.8151599169,-0.8932920694,0.8224828243,0.2315905243,-0.4471608400,0.3912411332,0.9867906570,-0.1927725524,-0.5247645974,-1.5280073881,-0.1495466530,0.0160353705,0.0425342433,0.7152678370,0.3655344248,-0.9652132392,-0.8394863605,-2.0853965282,1.5263340473,0.8999857306,-0.9665651917,1.3215459585,1.6256463528,-0.6425731182,-0.3002862930,1.3954046965,-2.0651483536,0.3873505294,1.7838572264,0.8483130932,0.7518290877,0.5456620455,1.5536112785,-1.3005512953,-0.0011652977,1.1804986000,-0.3673217297,0.0885236189,0.8989169002,-2.2710711956,0.1494538188,-0.0522320829,-0.7923706174,0.2892282903,-0.2980457246,-1.0174229145,0.9635899067,-0.8385112286,0.6136938930,0.2540693581,-0.9374857545,0.9380664825,0.1181033030],[1.1129386425,0.9579056501,-0.9558597207,1.0601702929,0.7341737747,0.2400165498,1.3471019268,1.6240410805,0.8954927325,-0.2616872191,-1.5816322565,-0.2222897261,0.1012802795,-0.6242704988,0.4884990752,0.9954836369,1.2131110430,-0.7540560365,-1.0051815510,0.6316062212,-0.3438287973,0.2564253211,-0.3662753701,-0.4446938932,0.6212993264,0.5648611188,-0.1582486629,-0.5771440864,1.2945601940,1.4222235680,0.9416752458,0.6796181798,0.1929067373,-0.8631902337,-0.0065787770,-0.2801783681,0.1711493433,1.6927223206,1.9421662092,-0.6135012507,1.3712947369,-0.6469724178,0.0916923508,-2.0832099915,0.3947067261,-1.0328670740,1.3029398918,0.2731808722,0.1960372478,1.0402327776,-0.0111813899,-0.0176756624,1.6581752300,0.6408863068,0.2431170642,0.1895090193,0.5048427582,0.7047714591,0.9869223833,0.2082526386,-0.1569980681,1.8514502048,-0.1799520552,0.4686902165,-0.7296008468,0.9013362527,-0.1100251153,-1.0055503845,-0.9817690253,0.8043930531,0.5625268221,1.8719913960,-1.1463146210,-0.0346449055,0.1817469299,0.9270682931,-0.2574214339,-0.0812553093,-0.5912104845,-0.3345954716,-0.8301431537,1.0862048864,-0.7808485031,-0.0158352572,-0.3031056821,1.6571618319,-0.8905954361,0.0837379172,0.6826679111,0.5788830519,0.2168190330,0.5412689447,1.2712599039,0.7540125847,0.6827997565,-2.8907747269,-1.5387016535,-1.5241500139,-0.3924700022,0.0892303810,-0.5330017805,-1.7088195086,-0.1490052342,-0.3517466784,-2.0205872059,0.7012131810,0.9212424755,-1.2042565346,-0.8458803892,1.0954351425,0.1554866433,0.6053807735,-1.4512552023,-0.0093505876,0.7159829140,1.6241915226,-1.0060654879,0.5553455949,-1.4876133204,-0.9805587530,-0.6491589546,0.0377813093,-0.4374382496,-0.4266566336,-2.0373737812,-0.3526954651,-1.3506067991,0.7612988949,-1.1326609850,-0.2315673083,0.4062868357,-0.4989110529,-0.0301045813,-0.2450016141,1.1841666698,1.1055570841,-0.3115909100,1.5708198547,0.0280597452,0.0123594552,0.4701385498,-2.1805176735,-0.4992506504,-1.8012614250,0.6283216476,-0.8813948631,0.6601570845,1.7074941397,-0.0323648006,0.8825834394,1.3912378550,-1.6596704721,-1.6631295681,0.8406025767,0.0631161332,0.1085712463,2.0449030399,0.0345824733,1.1145036221,0.4704167247,-0.1383501887,-0.4539908171,-0.0641246885,-0.3558729589,-0.9145979285,2.0010707378,1.6858053207,0.3380937576,-1.3622864485,1.6141508818,-1.9733695984,0.9540565014,-2.1892340183,1.3951656818,-0.5408843756,0.6286481619,-0.7919251323,0.0105986586,0.8478817940,0.0002034874,-0.9074158669,1.5799069405,-0.2770311236,1.0688809156,0.9766724706,-0.7193752527,-0.4294345379,-0.1861296296,0.7368682027,-0.1072331071,-0.8955709338,-0.2778291702,-0.2356924266,0.0711850822,-0.6330330372,0.2691031694,-0.5513830781,-0.7187303901,0.0601756237,-1.5095027685,0.8429980874,0.4193808436,0.0188407283,1.1542465687,0.9463533759,-2.3277208805,3.1905584335,-0.8127261996,1.6442073584,1.6567442417,-0.8463044763,0.1784670353,-2.0980811119,0.8293775916,1.9536508322,-0.7747604847,1.1398042440,0.9690032005,-0.2423246056,1.3939158916,0.1454046220,1.7181164026,0.0453010388,-0.0042198766],[-1.3608663082,-0.5003177524,-0.8680332303,-0.9361079931,-0.3540412188,1.5443038940,0.1527195871,1.9295225143,0.8829023242,1.0667285919,0.7185322046,-0.1005984619,-0.5712147951,-0.2969015241,-1.0879497528,0.1453319192,1.1827607155,-0.6063740849,-0.5514721870,1.3078466654,0.8379271030,-0.6013044119,-0.9620295167,1.0878808498,-1.4823607206,2.0232403278,0.6163513064,0.5846143365,0.1666686833,0.4976269603,-0.1785562485,0.7540314794,-2.4182591438,1.3574743271,-0.5127028823,2.2171654701,0.8127683997,1.0364259481,0.0117332144,-0.7456771731,-1.8012324572,0.5935599804,1.0617488623,0.1362540573,0.5942974091,-0.2413583249,0.8704653382,0.6090841293,0.8453356624,0.2453041673,-0.7767860293,0.4689100981,-0.2965798974,-0.2062119991,0.3263522387,0.2172193527,1.1833193302,0.2821285129,2.3209342957,-0.1554056257,0.1805481464,1.7754137516,1.7161456347,1.2346943617,-0.8394340277,0.3948509991,-1.2444956303,0.7143678665,0.1295471340,-1.6973470449,-0.9740580320,0.9478079677,0.0702200681,0.6023330688,0.3383901715,-0.5129318237,-0.9346300960,-0.2646407187,0.5493919849,0.3167619407,1.0960144997,-0.1926865131,1.1150927544,1.4298577309,0.5634369850,-0.8687397242,1.0241733789,-0.7390829921,-2.4886791706,0.1626223028,-0.0862197801,2.2243971825,-0.4995155036,0.1662081778,-0.3019536138,1.2557128668,0.2725491226,-0.4110666811,-2.0384042263,0.4620322287,-0.9412257075,-0.3955143094,-0.9924059510,0.1638202667,0.3578505516,-0.4322696328,-0.7486923337,1.4323714972,0.4394994676,1.2358547449,0.6357761621,-1.2816650867,0.3573113084,-1.6242008209,-1.2238067389,0.1763255596,0.7219309211,-1.1676596403,0.3330031931,1.0878335238,-1.0375856161,-0.5326374769,1.8756786585,1.5115497112,-1.2856667042,1.1450330019,-1.0966980457,1.1074422598,-0.6321702003,1.5199065208,-0.5065426230,-1.0368436575,-0.0203417726,-1.3390274048,-1.2279171944,-1.0808128119,0.3871849775,-0.3497922719,2.3117475510,0.6136760712,-1.0078067780,-0.8818475604,0.8308844566,-0.2642922997,-0.8865725398,-0.2153000832,-0.2417999208,-2.1490194798,-0.7199160457,0.0679587796,-0.2582596838,-0.8022152185,0.9388174415,-0.9667415619,2.2158970833,0.9668938518,-0.6219958067,-2.9191241264,0.7569295764,-0.3887217045,-0.9876939654,-1.3517054319,1.3579764366,-0.4132209122,-0.3938776553,1.0867658854,1.0614277124,-0.3257678151,0.5012996197,-0.9414650202,0.8639216423,-0.1593751311,-0.8448317051,0.5117835999,-0.0846990570,-0.3059445918,-0.5071154237,-0.5131493211,0.0728861243,-0.0135237006,-1.0407599211,0.1897208542,0.8545592427,-0.5512021184,2.0016546249,2.1962902546,0.6786051393,0.6614071727,0.3245737553,0.7180817723,0.5551066399,0.0685995594,0.1573880464,2.0783345699,0.5659430623,2.3344035149,0.3795775771,-1.1248421669,-0.6414177418,2.1331074238,-1.0174721479,0.3696995974,1.3464163542,0.4752027988,1.4939831495,0.4542889595,-1.4727809429,-0.4702996016,-0.3798926771,-0.0606959425,-2.7277064323,-1.2520660162,-0.1419366747,-1.7190331221,-0.6506875753,0.0628099963,-0.7629907727,0.6080714464,-1.9897142649,-0.3891165257,1.0277502537,1.1713209152,1.4797284603,-1.3836252689],[-0.2505295575,-0.4447467029,0.1925516874,-0.6498235464,0.2901880443,1.6240186691,0.4743281007,2.1063592434,0.5284393430,-0.8047794104,0.2104256451,0.7566365600,0.9716638327,-0.2522500753,-1.2101258039,-0.9453462958,-0.8787621260,-0.0724296495,0.6382610202,-1.0404309034,-0.4261433184,1.5532246828,0.5275081992,-0.3574368060,-0.7761614323,-1.7335830927,1.0692806244,-0.5937536955,0.4269003272,-0.3536632359,0.9446827769,2.5059707165,0.0690293163,0.2944066226,-0.7414646745,-1.0082733631,1.1343686581,0.2710540593,0.7274960876,-0.2651674747,-1.4867268801,0.5070603490,-1.2814332247,-0.3046406507,0.5114221573,0.3708107173,0.1999140978,0.7607744932,0.0477416702,0.4222696722,0.2854658961,-0.9390783906,-0.7843756676,-0.8260797858,1.4266068935,0.8196808696,-1.1903074980,-0.4611110985,-0.4952437580,0.6804166436,-0.6994231939,-1.1552095413,-1.7887793779,1.3153162003,0.6558619738,0.4816769361,-0.0126672778,-1.3285335302,-1.2688424587,-0.1523873210,-0.4339653254,-0.1709483713,0.2231142521,1.4828188419,-1.4240397215,-1.4417916536,0.4451074302,-0.1589459926,1.2505066395,1.3375947475,1.1642736197,0.3594298959,0.7713550925,-0.6801846623,-0.4980467260,0.9504658580,1.6608889103,0.4158179760,-0.2972727418,1.2367286682,-1.0620138645,-1.8265507221,-0.5331158042,-2.1153428555,-0.6209367514,0.4675408602,0.6143893600,-1.7113894224,0.2891764641,-0.6705935597,0.4754212797,0.3198952973,0.7838545442,0.3976619542,-1.1661629677,0.4603621066,0.9248743653,2.1145966053,0.0825881287,0.6749655008,0.8121272326,-0.2189459950,-0.1639946699,0.3587153256,-0.7834233642,0.6059847474,0.5444562435,-1.4124641418,-0.0233196020,0.0880037472,0.1275142431,1.0809987783,0.6402114630,0.7634710670,0.9324608445,0.3846043348,-1.7244272232,0.3992971778,-0.3185180128,0.1687958091,-0.0704131722,0.6610387564,-0.6090794802,-0.4523200393,1.0825520754,1.0088615417,-0.5682323575,-1.3370556831,1.0058012009,1.6292598248,-1.2862094641,-0.6961910725,0.2314819843,0.7249957323,0.6836280823,-0.1757106632,-0.9063949585,-1.3477391005,-1.1346484423,0.3227340579,1.3607194424,-2.8635163307,0.2292383462,0.4060978889,-1.3265417814,0.2751484215,0.7121296525,-0.6921774745,0.6154708266,0.2200543880,-0.5754459500,0.3323008418,2.1421134472,0.7250728011,0.6870709658,0.5515963435,-0.5132011175,-0.1644386500,1.9557372332,0.5852532983,0.7971339822,-0.4506426752,1.4849123955,0.2836766541,-0.6032394171,0.4705572426,-1.0361155272,0.2024379373,2.0730745792,-0.1682272553,0.3044758141,-0.1541289538,-0.6910542250,1.4061708450,0.4611213803,-0.2320941687,-0.4938946962,-0.2043031901,-0.0456889868,0.9397354126,0.9698880315,0.0257637091,-0.2238553017,-0.2935015857,1.0003639460,0.8938386440,1.1174061298,-1.4390172958,-1.3158385754,-0.4139026999,0.4302385151,-2.2730138302,-0.6109037995,0.6164110303,0.3562093377,0.9595016241,-0.5901470184,1.8615894318,0.3402916193,-0.0181768406,0.8151611686,1.2026921511,-0.9299021959,0.6094628572,1.2707104683,0.4547021091,0.9683445096,-1.0731282234,0.7785558105,1.6914439201,-0.2198672742,-2.2547419071,1.2058824301,0.6307622194],[-1.8844203949,-0.3400211334,-1.4175158739,-0.1408693939,-0.2282040119,0.6934574246,1.0840233564,-0.1311188787,-0.4547898471,0.0308669638,-2.9230649471,0.1924819946,-0.0251757074,-0.1361604929,1.3143212795,1.3681744337,2.5020995140,-1.7766501904,2.9016273022,0.4275854528,-0.0154027585,-1.1051633358,0.2060504258,0.1583422422,0.3727222979,0.9876830578,0.0626214370,-1.1221803427,3.4378697872,-2.8637402058,0.9245373607,-0.2257023752,-0.5427214503,-1.8706995249,0.5202276707,1.4523717165,-1.8733434677,-1.2647411823,-0.5458576083,-0.6326601505,1.9794954062,-0.0096601248,2.2758998871,0.3492978215,0.2012365162,0.2095893174,0.6796535254,0.0502694286,-0.0435023643,0.2123025507,-0.4695489109,-0.3350050747,0.2190171927,-1.0715231895,0.3171286583,-0.2802983224,-0.6058686972,-1.5530456305,0.9496842623,0.2735980153,-0.4321633577,-1.6320595741,0.8468621969,0.6839948893,0.2943500280,1.5179771185,-0.3009389341,1.1591215134,-0.4598817229,1.6021217108,-0.2987172902,0.1620407104,0.0022509200,-1.0108187199,-0.9158037305,0.6183546185,-0.0589140914,1.9188567400,-0.0930142999,0.4509327114,1.3160575628,0.1009306461,-1.2808429003,-0.3061289489,0.2605623901,1.1934047937,1.3779098988,-0.8680241704,-1.0800518990,-0.0498448424,0.4656108022,1.0705919266,0.5382865071,-1.0938647985,0.6651583910,0.1397799551,-0.7773083448,0.6970692277,-1.1914598942,0.4689689577,-1.5771685839,-1.7258087397,0.9759751558,0.9858368635,0.9479118586,0.8612651229,-0.4446322322,-0.3175063431,1.2068784237,-0.5929441452,0.1791196018,-0.9298995137,-0.4742261767,-0.4334574640,1.3930308819,1.5312166214,0.5114341974,-1.9366881847,-0.0120311510,0.2527293563,-0.3314227462,0.5194210410,0.1215722114,-0.5180037618,-1.5782446861,-0.3423944712,1.1970065832,-0.0611416213,-1.3710052967,0.8990535736,0.5925429463,1.5193586349,-0.4414147139,-1.0274288654,0.3192766309,-0.2304987013,-0.0761491209,0.0690865219,1.3222659826,0.0257438123,-0.2477024496,0.2511856556,-1.5257942677,1.4712384939,-0.0396799296,-0.0264192075,-1.6288158894,-0.7242621779,1.9021406174,-0.5092082620,-0.0244931318,-0.5751573443,0.9152583480,1.1320323944,0.3048408628,0.7192915082,0.0029749731,1.7288789749,-2.0123455524,-2.4080128670,-1.7593473196,-2.4347224236,1.6136039495,0.4126974642,-0.7342951298,-1.0291619301,-0.2701182067,-0.2743706107,1.5435656309,-0.4356430173,-0.3694144189,0.5576394796,-0.2556772828,0.4563857317,0.2570762634,0.3890536427,-0.8774389625,-0.2383845448,-0.5045922399,1.0469880104,1.6212552786,-0.2120764405,0.6647508144,-2.0476558208,-0.8174135685,-0.8982970119,0.7912128568,-0.7010203004,-1.3018070459,-1.6871378422,0.5631584525,-2.1441884041,-1.6575580835,0.9514712691,1.6940368414,0.0827083439,-1.3294876814,-1.1565612555,-0.1200775355,0.4073521793,0.0284242388,0.0219066888,0.5942062140,-0.4101213515,0.8342248797,0.3034593165,0.2702042758,-0.8202449083,0.8972294331,0.1362715960,0.6182371974,0.3332054317,0.3803861737,-0.0642565563,1.5682134628,0.0266444050,2.2625157833,-0.8821482658,0.9058094025,-0.9818913341,-0.6558931470,2.1423466206,0.8509710431,-0.3466790020],[0.8928981423,1.8331450224,1.0823125839,-2.2121467590,-0.2042878270,-0.8251465559,-0.3494953811,0.5181334019,1.1187945604,0.8085275888,-0.9451194406,-0.1899499297,-1.5023754835,0.5746624470,0.3227597475,0.9226501584,-0.6544494629,-1.2606118917,1.4417942762,-1.2978820801,-0.0916388780,0.2793105245,1.1175570488,1.7324228287,-1.0782588720,-0.0125431186,0.6125292182,-0.2632619441,-1.0598345995,0.2111615837,-0.3366095126,0.9723577499,-0.0001053959,-0.9998166561,0.3668155372,-0.1650307626,-2.6319227219,-1.3921569586,0.5988414288,-1.7351586819,1.4667917490,-0.5279781222,-0.3885532022,0.3108111024,-0.1206910387,0.8990362287,0.1039673463,0.1421874166,0.4403844476,-0.5696710944,-0.2248260379,0.9770126939,1.2786602974,0.8683383465,2.3025348186,0.0278422982,-0.1820368022,0.8733520508,-0.3343454897,-0.5557032824,-0.5515244007,0.5276218653,-0.7114800811,1.7720230818,-0.3970922530,0.4389536083,0.0650350600,-1.1340105534,-0.7053775787,-1.8313711882,-1.6354445219,-0.4521007836,0.3524754345,0.4045782983,-0.1586718410,-1.0496007204,-0.8052915335,-1.4610325098,0.3669247329,-0.9069761038,-0.7325763106,-0.9586330652,-0.1848229915,0.1050765216,-0.1473184377,-0.1583449841,-0.3924146891,1.3609659672,0.5217448473,1.9404394627,1.0777040720,0.3909571171,0.6589066386,0.1673523486,1.0541881323,0.3378158808,-0.7065559030,0.1609089524,1.7601629496,0.2293126136,0.9480530024,-1.2335206270,-0.0344778039,0.7309923172,-1.5902619362,-0.4351943135,0.6615905762,-0.3688139915,2.3102419376,-0.9426203370,-0.5544672608,-0.6343543530,1.0669492483,-0.2379402369,-0.4261119664,1.8326580524,-0.7513182163,-1.4482220411,-0.7471646667,-0.8635528684,-1.2731425762,0.6533293128,0.6602415442,1.2218433619,0.6049968004,-0.5613385439,0.0613612570,0.9906570315,0.5033934712,-0.6237265468,-0.0041219913,0.8261147141,0.2427824140,-0.3063735664,-0.0811720788,0.1534022242,-1.1207720041,-0.6678703427,-0.2377390862,0.8940547705,0.2770272493,1.3696693182,-0.0493086502,-0.0745674670,0.4672318697,-0.3150355816,1.4334459305,-0.1679604053,-1.9706282616,-0.3613967896,-1.6547517776,1.1160451174,-1.6003566980,2.0415852070,2.0366399288,-1.0523581505,0.3376710713,1.0529056787,1.6930022240,0.9688546062,-1.4531956911,0.8070805073,-1.3145008087,-1.4333742857,0.0553366765,0.4915426075,0.3531095684,-0.7884507179,-0.1045462191,-0.5995621085,-1.3206208944,0.6596292257,-0.9884840250,1.9658976793,-0.4340378940,0.0516867973,-0.5761942267,-0.4747160673,2.0759608746,0.9117660522,-1.3595043421,-0.0750775263,1.0767716169,0.8811606765,1.1698845625,2.6263203621,1.5705279112,-1.4361846447,0.8416768312,-0.7291257381,-1.9000356197,0.1692651510,-0.5643911958,2.2524120808,0.3646518886,-0.2983020544,-0.4583005309,0.9823272824,0.0525144637,-0.7168455720,0.5617127419,-1.3276793957,0.0449519083,1.7765147686,0.3622149229,-0.7156533003,2.7877490520,1.1703428030,1.6707651615,0.9258809090,-1.0642477274,1.1153731346,0.5419448614,0.0624054596,-1.0790256262,-0.2572177947,-1.1862916946,0.9607563615,0.1119695157,0.5639559031,-1.7139394283,-1.2567867041,-0.2268253267,0.3858694136],[-0.0999460742,0.4597186148,-1.6977158785,-1.0717468262,1.4142743349,0.2119618356,1.3924894333,-0.9887067676,-1.0531584024,-1.2928825617,1.2426022291,0.5878120065,-0.9949681759,0.8283343315,-0.2635304928,0.4014886022,-0.6090260744,0.5791243315,0.5834465027,1.5511604548,-1.2230901718,-0.6187438369,-0.3775131106,0.9911983013,-1.7839671373,0.1290750504,1.6611936092,-1.9548208714,0.5270512104,0.9121604562,0.4895628095,0.0196124874,-0.4402401149,-1.8178555965,-0.2203289568,1.2370438576,-1.3454525471,2.4535121918,-0.1176287308,-1.0192205906,0.5798735619,0.0242904909,-0.0841123089,0.5548579097,-0.7952302098,-0.8613741398,0.8380604386,1.0689024925,-2.4395861626,0.3448410630,-0.5043354034,0.0641216338,1.1720117331,1.2903999090,0.4812498987,1.1072002649,1.0949929953,-2.3961193562,-2.7901766300,-0.2823035121,1.2701352835,-2.0818095207,2.6965289116,-1.3935769796,-0.2706691027,1.0988687277,-2.1704502106,0.6163380742,1.3522902727,0.7691040635,-3.0363001823,2.5481321812,-1.5075689554,-2.3067941666,0.8177285790,-2.4801552296,0.9572334290,1.1474175453,-0.5582699776,-0.2533708811,0.1953659654,0.0904167295,0.6058718562,0.3594837487,0.5418453813,-1.2951164246,0.4690193236,1.7330023050,0.1667902321,1.1768552065,-0.0910997316,-0.2359167486,-0.0121873543,0.4238246381,-0.0331050605,-0.5837310553,0.5397078991,0.3845285177,0.1106715724,0.1258557290,-0.4165729582,1.0588035583,0.4541542530,-2.6346459389,-1.2074881792,-0.1499014944,-0.3375472128,-1.5428763628,1.2837784290,-0.6475168467,0.3657090068,0.4615589976,1.6139746904,1.0400425196,0.8091872334,0.0503223762,-1.4139139652,0.1132474169,1.5251283646,1.6179705858,1.2431045771,1.7049555779,0.1189959124,-0.2425130904,0.1800673753,-1.4013088942,0.6485371590,0.8376440406,1.1608923674,1.1170085669,-0.8883395791,-0.1524236053,-0.8715451360,-0.1062527671,1.8281518221,-0.6069445014,-0.6816360950,-0.3067930639,0.9655979276,-0.2940889299,0.9251574278,1.1493262053,-0.5966495275,0.5556038022,1.9187790155,-0.2040707618,-0.4193641543,0.2341037840,-0.2179090828,0.7499462962,-0.2418686002,0.5113593340,-1.8201842308,-1.1509025097,-0.7798062563,0.1130120382,-0.3003650606,-1.9360016584,-0.9553108215,0.9728742838,-0.1972751766,-1.7404620647,-0.4687793553,1.1957448721,-0.5498569012,-0.7883052230,-0.1498728544,0.2681272030,-0.2486549169,-0.4183347225,-0.3109455109,0.1175777838,0.7868638635,0.9115644693,0.1073277220,-1.0881794691,0.8313896656,0.4506823123,-0.9034633636,-1.0354187489,0.6935775280,1.1028820276,0.6876826882,2.0347700119,0.5668647289,-0.3477827907,1.6544166803,0.3295305669,-0.1434911638,-0.1425676346,-0.4272623062,-1.2900912762,-0.2381984293,0.6456766725,-1.0504271984,-0.9134973884,-1.0105756521,1.1871837378,0.2558339238,0.9838050008,-0.5840255618,1.6918026209,-0.7227749228,0.5577063560,-0.2009231299,0.9748980999,1.0686962605,0.1406216919,1.4599475861,-1.1134356260,0.4520905316,-0.3960721493,0.9606674314,0.4791612625,-0.5085137486,-1.1975919008,1.2257055044,-0.7944650054,1.0772857666,-0.8134288192,-0.1460259855,-0.7566825151,0.5483761430,0.4096541405],[-1.1490131617,-1.2600245476,0.4720077217,-0.8848083019,-0.3584946394,-0.3885389566,-1.6692073345,1.8913199902,-0.9435986280,-1.6560325623,-0.4170581698,1.4248950481,0.7457003593,-0.2701747119,-1.4329892397,-0.9257540107,-0.5308506489,0.2141064256,-0.7413557172,-0.7860522866,-1.6483081579,-1.3483446836,-1.2292420864,-0.5480573773,-0.5060109496,-0.5933606029,-2.0655674934,1.2769864798,-0.0294818971,-0.5929264426,0.6686656475,0.5973464251,0.2934952974,-1.2685967684,1.0854260921,0.0394030698,0.6842738390,-0.7315492630,1.6883585453,1.6437082291,-0.8642318845,-0.1803095788,-0.1081145182,-1.4840042591,-1.0925523043,-0.0899927989,-0.3712283671,-0.8431726694,-0.9773939252,0.5591737628,0.4611403346,0.0734049380,0.5970427394,1.9101938009,-2.3871018887,-0.4583065808,-0.5112628341,-1.3271329403,0.7063751817,-0.6450681686,-2.1740829945,2.9771606922,-0.7224324942,-0.6503081918,1.5257606506,-0.2274675965,0.4022358656,0.4434968829,-1.0218620300,-0.1315025240,0.0179176256,-0.4155089855,0.4858019650,-0.8543961048,-0.5012856126,0.8567308187,0.2221969515,0.1999620646,0.6042551994,0.6662653685,1.0663129091,-0.4896272123,-0.2026205212,-1.1394944191,-1.2054032087,-1.1226356030,-0.4939437509,0.1943001896,0.6055281162,0.7169389725,-0.3576804101,0.6503378749,1.6525493860,0.5573248863,0.1776629537,-0.4948503077,-0.0657189265,0.1559060067,-0.1242087036,0.0716841519,-1.5930215120,0.0682995692,-1.2083141804,0.9281802773,-0.0046851207,0.9349762201,0.2173268646,1.3025127649,0.0904640332,-0.3832762539,-1.6360230446,0.4152020812,-1.4499077797,0.8201241493,0.8912186027,0.0771826804,-0.5154270530,0.9378012419,-0.4901067615,0.5525051951,1.2330327034,-0.3991430104,1.4831937551,0.7175683379,0.2221147567,0.3608321846,0.2894625068,-1.6595556736,-1.9314495325,1.8751310110,-0.4770455658,-0.5211828947,-0.6297601461,-1.1078292131,1.9312903881,0.2273961604,-0.3932147026,-1.0034306049,-2.2730729580,0.0832397044,-0.5162621140,-0.9493662715,0.7647309303,-1.4768458605,-0.7454246283,0.2828139067,-0.3133463562,0.0699866787,-0.8258736730,0.9094543457,2.2345933914,-0.2716061175,-2.0914297104,-0.8450802565,-0.1864923686,-0.2835510671,-0.1033757925,-1.5680092573,-1.2161769867,-1.6124955416,1.3700754642,0.7851352096,-0.3823657930,1.3108464479,-0.8988699317,0.5083639026,0.3493502140,-0.0577305481,1.0597113371,1.0081275702,-0.6358467937,-0.4504096806,-0.3147639036,1.0732730627,-0.8271607161,-0.6222040057,-0.7991218567,0.7039133906,0.3510959148,0.5554497242,1.4979805946,-2.1750752926,0.2891605496,1.9272480011,-0.9401828647,1.3731950521,-1.7251560688,1.3734343052,0.3356144130,0.9673383832,-1.1017390490,-1.1915374994,-1.7516787052,-1.0826570988,0.6179661155,1.1888171434,-1.0887053013,-0.2124068141,-1.1567455530,-1.0044180155,-0.9566497207,0.7797390819,-1.7934116125,-0.3947986662,-0.4438236058,0.4794886708,1.2552320957,-0.7013828158,1.3951803446,-0.0517238714,-1.4133031368,-1.8694245815,-0.5054339767,-1.2462067604,-1.2365274429,0.3690735698,1.2771483660,0.0578073971,0.3081986904,-0.2649683654,0.5571885705,-0.5460993648,0.5755186081,1.1222224236],[0.0097143548,-0.1952548772,1.3565348387,0.9633554816,0.7406532764,0.4492635727,0.3314831555,0.1872891337,0.5805309415,0.3974446356,-0.5829093456,-0.0477444232,1.0294648409,1.5680623055,1.6048808098,-1.7844184637,1.3980298042,0.5945453644,-0.5468156338,1.2185471058,-1.0988667011,-0.5286205411,-0.5052637458,-0.1807851791,0.1174189076,-0.9720669985,-1.5364803076,1.3941923380,2.1483066082,0.4284505546,-0.1856944412,-0.7900459766,0.0108980974,0.9540181160,1.2205339670,0.4898815155,-0.9313154221,-0.9521727562,-0.5750778913,-0.2174461782,2.1930093765,1.5066055059,-1.0824288130,-0.2641721070,-1.2634152174,-0.1384104043,0.4317926168,-0.6106199026,-0.4305661917,-0.2218575478,-1.1112926006,0.1515053213,-0.3358800411,-0.1186274439,-0.7633775473,-0.2121664286,1.9252479076,-0.5415530801,-0.4101755917,-1.9593467712,-0.1169006303,0.1504407674,1.2334483862,1.0831296444,-0.4920203984,-1.1058235168,-1.3844621181,0.1942979842,-1.2118817568,0.9645847082,-2.0245478153,0.2150682956,-1.8037940264,2.1545493603,0.2532272041,0.3322596550,0.8775139451,0.0413930006,-0.5169070959,0.8127090335,3.4397461414,0.8755329847,0.6949456334,-1.6751369238,-0.7122853398,0.5115014315,-1.2610419989,-0.0193399414,1.1886892319,0.3200702965,-0.4151143730,-0.4470207989,-0.7033379078,2.1508812904,0.2372479588,-0.8369963765,-0.6052927971,-0.6665679216,-1.3898322582,-0.1554756016,-0.9192994833,-0.7518845797,-0.3134998679,-0.2572743595,-1.5091480017,-0.5846824646,0.3287142515,1.6392349005,-0.0196348205,0.5276853442,1.1864744425,0.7721220851,0.6423051953,-0.7868347764,-0.1904338896,-0.0519685000,0.2901331782,-0.0316568166,0.3051229119,1.2303379774,0.2227156460,-1.8869382143,-0.0082769394,0.2376345396,0.8816763163,-0.1637671590,-0.8250055313,0.9399932623,-0.2580229640,2.0155923367,-0.1516564339,0.1265739948,2.1453938484,0.8217768669,0.9791879654,-0.2263262570,-0.4864189625,0.0454811975,1.3710892200,-0.9420208931,0.9791120291,-0.4800106883,0.8337278366,-1.3876479864,0.8446266651,0.8098564148,0.5751532912,-0.9790716171,-0.0171645191,-0.5026745200,0.6194605827,-0.8599231243,-0.3917999566,-1.4034454823,0.8840796947,-1.6543312073,0.0696845874,-0.4744838774,0.8552336097,1.8941549063,-0.4313303828,-0.2739033401,0.3928846419,-1.5880547762,0.9341297150,0.3463015854,-1.3363854885,-0.0499476194,-2.9639329910,-0.5594952106,0.7434971333,0.1999131292,-2.0764930248,-0.5285277367,-0.2470802069,-2.0624265671,0.9572924376,0.3903219700,-0.6605809927,-0.6549845338,-1.0186446905,-0.0800478607,0.8186472058,0.6616176367,0.0518838763,-0.9223122597,-0.5421885848,-0.2825040221,-0.3912544847,-0.2029047906,0.6893842220,-0.8635937572,0.5363397598,0.6943371892,0.0873916373,-0.2490534633,1.2992305756,-0.7386673093,2.0405597687,0.8024853468,-1.4069356918,1.1822263002,-1.3656258583,-0.5556980968,0.1787927747,0.1019315794,-1.3714215755,0.1987746656,-0.3629374206,1.8519080877,0.6735958457,0.4217328429,-0.3506659269,-0.1316977292,1.2174890041,0.3776035905,-0.8396139741,-1.1620602608,0.3407081068,-0.3355799913,-0.7459722757,-1.8262302876,-0.8597204685,-1.4879345894],[1.1369839907,1.1803423166,0.0483212918,-0.3535215557,1.7021528482,0.1325516403,-0.1556667686,-2.0734696388,0.3474398255,-1.1306002140,-0.5391064882,0.0367188156,-0.1031585932,0.5129575133,0.1265309304,0.0982659385,0.4955267012,1.3003282547,0.6503933668,0.3970629275,1.8765217066,-2.2937936783,-1.0489534140,-0.3891129792,2.0643260479,-0.1525699496,-0.3799881637,0.4029478729,0.8279522061,-0.6166064143,1.0913140774,0.1232835948,0.0043251989,-1.2592334747,1.1358189583,1.3957405090,-0.0802572221,0.3133191466,1.0224242210,0.5248175263,-0.6712270975,-0.6747437119,0.0586817972,-0.4509712160,-0.9520729184,-0.7986681461,0.9456746578,2.1470000744,1.4799757004,-1.8066306114,-0.0310747288,0.8656225801,0.5112625360,2.5125331879,0.2004410326,0.2644173801,-0.5642243028,-0.3665525317,-0.8501471877,2.1454489231,0.0146866925,-0.2753212452,-2.2508983612,0.4200601578,-0.5757294893,0.3302652538,0.6833993793,-0.1050571799,1.3361978531,-0.8743016124,-0.1878415346,0.5748300552,0.4990630448,-0.9741004705,0.2689557374,-0.8159407973,1.5243690014,1.1392233372,1.6356216669,-1.5474957228,-0.0423558354,0.0719309151,0.0674327686,-0.4663897157,-1.0062319040,1.1016836166,-0.3533191681,-0.9534746408,-0.5174071789,0.3306112587,1.3137228489,-0.4841048717,-1.5414144993,0.7894215584,0.7925677299,-0.2099342644,1.6130772829,-2.2307887077,0.6066238284,0.4182311296,-0.8662330508,-0.1302055269,0.6588820815,0.7003934383,-0.6792898178,2.0538380146,1.4943599701,-1.2634873390,0.2456917167,-1.6332542896,-1.1893531084,0.7613589168,-0.3110498488,0.1274812669,-1.4536637068,1.2830148935,0.3669683635,-0.6743721366,0.3679584563,-1.8977421522,0.8388814926,1.7857834101,0.3952881098,0.9302375913,0.1583839953,-0.4621100724,-1.2903282642,-1.6294634342,0.7264854312,-0.0749876797,-1.2859839201,-0.3973017633,0.2206417620,-0.6645338535,-0.2155216634,0.5685132742,0.1122510657,-1.3511683941,-0.8059957027,-1.0774103403,0.4130584002,-0.2617705166,-0.1687471569,-0.2825335562,-0.3444823325,-0.9686818719,-0.1723157912,0.2521053255,-2.1374604702,0.4649966657,0.5567176342,2.4056639671,0.2969538569,1.8013700247,0.0617722496,0.4764775336,-1.2795540094,0.6798602343,1.4741630554,0.7309804559,0.3686659932,0.0889440775,-1.4905848503,2.1611554623,0.4314791262,0.1953540742,1.5225317478,0.7390133142,-0.4725934565,1.1161459684,-1.9433223009,-0.1393656880,0.2824656963,3.1519126892,-0.5079812407,-0.8862577677,0.6268798709,-1.2120158672,0.7088871598,-1.6071879864,0.8341999054,0.6533966660,-0.1934999079,-0.1612865776,-0.1745747179,0.6138933301,-0.2608250678,0.0169208851,1.4444025755,0.0534766540,0.2234792262,-1.0242514610,-0.7009769678,0.4972992241,0.1313402355,0.2999706864,-0.9448517561,0.2542484105,1.3423204422,-0.7973331213,0.4570710957,-1.5696889162,0.8125736117,-1.5350893736,0.5826305151,0.2596044540,-0.8103986382,2.3475239277,0.2638120949,1.5259733200,0.5666651726,-0.9071298838,-0.8720138669,-0.9882591367,0.4064132273,1.3104044199,0.1794917881,-0.3529058397,0.8675439954,1.5312627554,-0.1812909395,0.4155770242,1.4033385515,0.1539124548],[-0.1244183630,-1.5379589796,-0.8867199421,-0.6548126340,1.8252388239,0.6651643515,-0.6815838814,-0.0382827222,0.4838942289,0.7248817086,0.0481229275,-0.5961760879,1.7160128355,-0.5742226243,0.4242486656,1.3640313148,-0.0682704076,-0.1149882078,-0.7724986672,0.5856184959,-0.5331230164,0.2725982368,-2.2121868134,-0.5297753811,0.3282147050,-0.2310766876,0.9220627546,-0.4692978561,0.9146265388,1.2192900181,1.1941511631,-0.1149736866,0.9485147595,-0.4857148528,0.6253182888,0.1254618019,-1.3645582199,0.4393002987,-0.9393618107,-0.3738214076,0.3530284464,0.3340075612,-1.0520275831,0.3124981225,0.2227747589,-0.3774113357,0.3376967609,-0.1335430741,-1.4239124060,0.3777493238,-0.6754847169,1.7558667660,-0.3419993520,-0.5435870886,1.5623818636,-1.0864465237,2.7157764435,-1.1701687574,1.0629879236,0.1548271179,0.2290085405,-0.5235656500,-1.2878988981,-0.5667066574,-0.0501692370,1.7568033934,-0.1276576668,-0.8953829408,1.0946452618,-0.3542604744,-1.2604600191,-0.3953779340,0.2435953319,-0.7940179110,-0.5217481852,0.4265375137,-0.3192216158,-0.8071255088,0.5322649479,-0.3526906967,-0.0918452591,-0.6988605857,-0.2911368012,-0.6488785744,-0.6978461146,0.3133989573,-0.7867271304,0.3081840575,-0.1553696841,1.6520466805,-0.8007677197,-1.9127302170,-0.8846508265,0.7461178899,1.0180773735,0.9306701422,-0.1126613319,0.1929772198,-1.0579559803,-0.2422025204,1.8559801579,0.3720010519,1.1825789213,-0.1039613783,1.0414545536,-0.3254982829,0.9571298361,-0.4382165670,1.0003467798,1.5939487219,0.5429488420,-2.0420072079,0.5728467107,-1.8618476391,-0.2918467820,-0.0487218536,-0.9199935198,-1.0266177654,1.6822305918,1.4191178083,0.6947760582,1.0251603127,0.6672874093,1.2495917082,-0.8335829377,0.2156088799,1.1924821138,-0.4395660460,0.6522639990,-1.6726173162,-1.0532317162,2.1165602207,-1.9447240829,-0.0893126354,0.2683942318,-0.1623480618,-0.2299717814,-1.2482939959,-1.4748595953,-1.1440454721,-0.1509122998,-0.5654972196,0.9534760118,2.3310527802,1.1738048792,1.2242695093,-0.1066419408,-0.4332847297,-0.8702556491,2.0648863316,-0.9727774858,-0.3140878677,-0.5490342379,0.6551995277,0.4908427596,1.0332645178,1.1219984293,1.3135952950,-0.4913543463,0.6445788741,1.7172362804,-2.7000076771,-1.9069814682,0.5195291638,-1.2169370651,0.2950018048,0.5568491220,-0.6202567220,3.0056195259,1.6280162334,-1.7385234833,1.4424477816,0.8223228455,-0.0615691058,-0.8732482791,-0.0132957678,1.3641716242,0.8280566931,-0.6633744240,-0.0640525073,0.1176194623,-1.9407721758,-1.8422350883,-1.4371339083,0.1744921356,2.3886828423,0.1898625940,-0.5868138671,0.9374192953,1.4271277189,0.1327908039,-1.2658762932,-1.1797477007,-2.2054107189,-1.2667415142,-1.1865519285,0.3430148959,0.7459679246,-1.5639922619,0.6891250014,0.6442363858,-0.1363123357,-0.3265230358,0.8321313858,-0.0314064920,1.1229662895,1.1464596987,-0.6990435719,-1.4267598391,0.8259562850,1.7598297596,1.2947094440,-0.5276837945,-0.2367414087,0.5686634779,0.6113168597,-0.1315857321,-0.0734574869,0.0436162390,0.1786444336,0.6293717623,-1.1854665279,-1.8017184734,1.1474248171],[-0.2221667469,-0.8318022490,-1.1769648790,-0.6398457885,1.1581661701,1.0694650412,-1.1927752495,-0.3782918751,-0.5945242047,1.5984972715,-0.4153506756,0.9948699474,1.1457722187,0.2799803019,0.0362116992,0.1169509590,1.1984179020,-2.1507964134,-0.2818993628,0.1193391681,1.1043921709,-0.1576315612,0.7816631794,-0.1992943287,-0.7222493291,-1.6471685171,0.5236202478,0.3679266572,-0.2783089876,0.1911206990,1.7007887363,-0.2828701138,-0.6807302237,-2.0343830585,0.0755103827,0.1925597936,1.1212855577,-0.6325791478,-1.4917219877,0.1507131606,-1.4023673534,-1.4923455715,0.2636044025,1.2325544357,0.8019176126,0.7070560455,-0.4439917505,0.3190809786,0.3905167878,1.6310776472,1.1720501184,-1.2261582613,1.4022692442,0.6078658700,0.8062526584,-2.9621202946,-0.3987344801,0.3414399922,1.7619208097,1.5134036541,0.6041021347,0.5934362411,-1.1736227274,1.3398493528,-0.5730978847,0.1356645226,0.2863060832,-0.0154768359,-1.0178600550,-2.3243145943,-0.1675287634,0.6049609184,0.2159761786,0.0666745231,-0.0865136832,1.2352000475,0.0394294672,1.0100305080,0.2578782439,0.6886412501,-2.9555711746,0.3245435357,-1.1066570282,0.0578106418,0.8995000124,-0.5578057170,0.8344925642,-0.3650110662,-1.9382914305,-0.9237997532,0.1845855862,1.0926511288,-1.4047546387,1.5311555862,-0.8047308326,-0.7788903713,0.4715638757,0.4318743646,-1.6874840260,-2.7890527248,-1.5477393866,-0.4640494883,0.4432340562,-1.0020309687,-0.5192461014,0.1724080592,1.3290741444,0.6016322374,-2.4097959995,0.5145076513,-1.1226621866,0.5319163203,-0.9135279655,-0.3886004984,0.2594231069,-0.8287024498,1.3806180954,0.2069475651,-0.8147791028,-1.8920502663,0.3105571568,-0.6040369272,-0.5843714476,-1.3048652411,-0.9555811882,1.6718477011,0.2061201781,0.2449650019,0.6344435215,1.8947550058,-1.1782568693,-1.3522021770,0.5857774019,-1.2353422642,0.5616509914,1.3375346661,1.3691686392,-1.7339766026,-0.5559787154,-0.4727867544,-0.9000478983,1.2659494877,0.4353246093,-0.0236592907,0.9436045289,-0.6716923118,1.0447865725,1.3527976274,-1.0631425381,2.4571130276,-0.1190586910,-1.3206595182,0.2101318240,-1.4100403786,0.7463139296,-1.5983033180,-0.1158415601,-0.3557868004,-0.1035592780,-2.1708722115,0.3732755780,0.8958846927,-1.5255739689,0.0184564441,0.5205706954,1.0403897762,-0.0308587700,-0.9651818871,-0.2936720550,0.0902625993,0.4703084528,-0.3178536892,0.3725000322,-0.5491930842,0.6766549349,-0.5850317478,1.9644398689,2.4331896305,1.3197941780,-0.5910408497,0.5551754832,0.7306241393,-1.0116714239,-0.7693768144,0.1477881372,0.5307281017,-0.5618506074,-0.3604781926,0.5028930902,-0.5547525883,2.3241920471,0.7301399112,-0.0238630101,-0.3112625182,-0.0513433814,0.0332490802,-0.3835499585,-0.3944619596,-1.0770056248,-0.8284639120,-2.2706835270,1.1904422045,-0.7383732200,-0.7856744528,0.2669454515,-0.4751650691,-0.3949821889,2.0143547058,0.1436878145,-1.2537568808,-1.7973768711,0.3158451319,-0.2288792729,-0.6633766294,0.3909885287,1.5668081045,-0.8202961683,-0.2624257505,0.0357421786,-2.8803546429,1.0226624012,-1.6296515465,-2.3747596741,-0.0938686728],[-0.4877255559,0.4899860322,-0.5391430855,1.3695386648,0.6441672444,0.7667090893,-1.3685532808,0.6206942797,1.2448111773,-0.8042119741,-0.5956533551,0.0014253046,-0.8887516260,1.6678035259,2.0496942997,-0.1034380123,-1.0603953600,-0.4802497625,0.1270350516,2.1686303616,-1.5617771149,0.4015067816,0.1245800033,-0.3452774882,0.8196158409,-0.1079464853,0.8407748342,2.0225064754,0.1155442074,0.5847235322,0.4388331473,0.1234409586,-0.2910670638,-0.6586142182,2.0476639271,-1.8610835075,2.2706551552,-0.1068520844,-1.3966004848,0.8132696152,-0.4033440351,-0.1480889022,-1.0043388605,0.3936049640,0.1324205846,0.5566450357,-0.1675183326,0.5736991763,-0.6849721670,0.9904925227,-1.8894196749,1.5981130600,0.7703539729,-0.1610601544,0.9238270521,-0.0515182950,-0.8868377805,1.0417042971,0.6102308035,-1.8677076101,-0.3840411901,-0.1164660156,0.0456374995,-1.8475975990,0.5355637670,-0.6964988112,0.7957055569,-0.1288603097,-0.3235868216,-1.6325315237,-0.5247679353,0.7241875529,0.7006774545,0.6510369182,0.2127076685,0.3019182682,1.7634776831,1.0604306459,-0.0160322711,-2.0777280331,1.1360679865,0.4793742299,0.4298013747,-0.9959617257,-1.0719579458,0.3770346940,1.4366446733,0.7699731588,-0.9030251503,-0.8420140147,-0.2891960144,-1.3440425396,-1.4827768803,-0.8795897365,0.8493244648,0.1213431358,2.4658744335,-0.5044754148,0.8377598524,-1.3034893274,0.4661247730,0.5391213298,-1.0591400862,-0.0326998606,-2.0926170349,-0.3239274919,-0.9163483977,0.7162846327,-0.0685323477,0.4281194508,-1.1246926785,-0.1664071679,-0.6484211087,-2.1521315575,-1.0705305338,-0.8617205024,0.2989324927,1.2873754501,-0.2156222463,-1.4065039158,-0.6083480120,0.0496303663,-0.0799441859,-0.3430354595,-0.5707333684,0.2139833272,0.1441838294,-1.6315569878,-0.4222122729,0.7287571430,0.4752489328,1.4714828730,0.7369390726,-0.5534162521,0.4637760520,0.8952754736,-1.3604298830,-2.1174213886,2.2016246319,-0.2555569708,0.5857450366,0.1036471277,-0.6595315337,0.5226187706,-1.8577984571,0.8471649885,-0.5092989802,0.8184267879,1.6758291721,0.4162685573,2.5203919411,-1.0321208239,-1.4705890417,0.5317592025,0.8014125228,0.9071531296,-1.5176868439,0.4063048065,0.7387344241,1.2877237797,0.3409604430,1.8790210485,0.9491173029,0.2067094594,-0.9716271162,0.3574623168,1.0916240215,-0.2871577740,-1.7678191662,-1.4303933382,-0.2483468205,-0.1160075366,-0.1474684626,0.4271591902,0.7722973824,-0.8035597205,0.2369090319,0.5585380793,-1.6493217945,-0.9682631493,-0.2224204242,-0.3922516108,0.8767914772,0.9763500094,-0.3122895956,0.9651550651,0.6296004653,-0.1026753411,-2.0676610470,-0.7020508647,-0.1088828817,1.3923741579,0.1279061884,-1.0830278397,1.7001254559,-0.7274262905,1.5997858047,-1.7580193281,-0.1680758595,-0.7228416204,-2.0124907494,1.3119337559,0.2128693163,1.7944684029,1.8756083250,-0.3741932511,-0.2608598173,1.0983625650,2.3297736645,0.3219519854,0.4034484327,1.2642672062,0.8221021891,0.4798425138,0.0399289317,-0.2851095498,0.5849317908,0.7732047439,1.2581137419,1.8532098532,-0.6349226832,0.4174361527,0.4140231311,-0.6864744425],[-0.1428144574,0.4959804416,0.7714887261,0.2804813683,-0.5753896832,-1.2983716726,-0.5563231707,-0.0272231661,-0.4999603927,-0.2157250941,-0.7681286335,-2.0508408546,-1.0966194868,1.6365146637,-0.5183550119,0.6196789742,-1.3453786373,0.5836728811,-0.4710648656,-0.5288444161,-0.2772250473,-2.3445336819,-0.6098615527,-0.3629501164,-0.1410931945,-0.4874231517,0.7105670571,-0.5983030796,0.5260640383,-1.0717720985,0.3782913387,0.7160261869,-0.8814377785,-0.2456881255,-0.6174123883,0.2108476460,-0.3944846392,-1.3366764784,0.0207539462,-0.5891013741,0.2604995966,1.6924297810,-1.1274348497,0.5514831543,0.6292963624,-0.7273381352,-0.1221026108,-1.3238046169,-1.8603323698,0.0118839173,-0.0494511090,-0.7719197273,1.1183060408,1.1271860600,-2.0702965260,1.1230894327,-0.5023233891,-0.5238962770,0.1952285916,-0.2513331473,-0.3816655576,0.6158700585,-0.7880089879,-1.8909306526,-0.7469483018,0.0433481671,0.1173256263,0.4212864637,-0.3761186600,-0.9839541912,2.0143651962,-0.3616572618,0.2649833560,-0.1013830006,-0.6988646388,0.6318305135,-1.3341007233,0.5088815093,0.2114373595,0.6875721812,0.2607598007,0.0316194594,0.3898535371,0.7478242517,-0.9930844903,-0.5001050234,0.8352737427,0.3108315766,-0.0316842087,-0.7823688984,0.1578320265,0.9126993418,-0.2352722585,0.9408181310,0.6109023094,-1.9303262234,0.5498456359,-0.6723092198,-1.4504003525,0.5355149508,0.3497037590,0.1889631450,0.1326940656,-0.6038624048,0.5095413923,-1.0285254717,0.2388528883,-0.2497551739,0.4331155121,-0.0697732419,0.6253356934,-0.1688880771,0.9176494479,-1.8235207796,-1.0847314596,-0.5756394863,0.3497035503,0.2342435122,-2.6334090233,0.5881343484,1.7326449156,0.4294790924,-1.3273426294,-2.0741112232,-1.2210987806,-0.1957486868,0.2234403342,0.4510703385,-1.2828766108,0.2367241681,-0.2933946550,-2.7297432423,-0.3880361617,1.3037126064,-0.3852106035,0.5652311444,-0.3321615160,-0.5173915625,1.3139929771,0.8717244864,-0.6854110956,-1.9135602713,0.6436522007,-0.8172343969,0.2767552435,-0.1575471610,-0.5616329312,0.4937283993,-0.3603928089,0.4888756573,0.4363217950,0.7434330583,1.0488585234,0.2988363504,0.8631799817,0.4911931753,0.5605632067,-1.2499432564,-1.8365777731,-1.5436005592,0.2294761688,-0.9752155542,-0.7505861521,0.8158047199,0.8112878799,-1.1278395653,0.1002079770,0.1714561433,-0.0902783796,0.1032009050,-0.4986415505,-0.1339252293,-1.1308209896,-1.2856656313,-0.6171336174,0.0464790165,0.8600177169,0.7068345547,-0.5624985099,0.4751739502,-0.3413746953,1.0102072954,0.0230765510,0.2426536977,-1.8445314169,0.7333210111,0.5530741811,0.2356170565,0.1776789725,-0.2307106107,-0.0811266378,0.7031916976,1.5869663954,-0.3827952743,2.0957953930,-0.0665417612,-0.5297813416,1.3718634844,-0.2815313339,0.2085584700,-0.3606263101,-0.4346255958,0.9304487705,0.6789392829,0.3673276603,0.1621620655,0.4276526272,-0.1493246108,0.8981700540,0.7842456102,-0.7916181684,0.1158784404,0.4618153572,0.3543474972,1.4618315697,-0.0058341846,-0.8900887370,-0.4046399891,0.9123374820,0.5789139867,-0.4968176186,0.1236161813,-0.4920249581,1.8629844189],[3.1188952923,2.1145527363,0.4688876569,-0.9595867991,-1.0293675661,0.6830950379,1.3771774769,0.1248709261,-0.4470041692,-1.2351502180,-0.0911605358,-0.3967476189,-0.0899577588,-1.3996888399,-0.5305298567,0.7499350905,0.5719929934,-1.4103066921,-0.3479000628,-0.2184364051,0.1737931967,0.0222933087,0.5647415519,0.8052543998,-1.0294834375,0.8431189060,1.5975235701,1.1170238256,0.6672995090,-0.8730934262,-0.5672993660,-1.2327417135,-1.4132713079,0.9281820655,0.7359776497,1.0769482851,-0.1153248027,0.2036904693,-0.8216454983,-0.7670912147,1.1390515566,-1.3738186359,0.3531615138,0.4086511433,-1.4268133640,1.0887863636,-0.4670736790,1.5938810110,-0.6779985428,-1.6609425545,-1.1497362852,0.0858812928,0.3081794083,2.7873344421,0.0017950319,0.7034717202,0.0402048379,-0.0539211519,-0.8005341887,1.2653822899,1.1613918543,0.0606069304,-0.9181155562,-0.2290524691,-0.1122236475,0.0549350791,-0.8877182007,-0.3766615987,-0.5949606895,-1.6478073597,0.8510466218,0.2196233720,-0.3607971072,0.8401561379,0.0543761253,1.4959983826,-0.5632653236,-0.2425057590,0.0648398325,0.7764827609,0.7312174439,-1.1121135950,-0.6776387691,1.0912896395,0.9742969275,-1.9399904013,-2.1802775860,2.1129066944,0.8291311264,-1.1868286133,0.5817332864,-2.1304891109,1.0515449047,0.4106768072,-1.2035242319,-0.7135511637,-0.4709822536,-0.5258688331,0.7505949140,-0.8277429938,-2.7819249630,-1.1523625851,-0.8028509617,-0.5026053190,-0.1736410111,-0.3603467047,-0.2416286469,0.1397867203,-0.1340719461,2.9043488503,-1.2880799770,-0.0549019389,-0.6406171918,-1.3738036156,-0.4437952936,-0.6030911207,-1.5600773096,-0.3178840280,0.7513098717,-0.0685902163,-0.5909373164,0.0339408405,0.0979840606,-1.9291602373,-1.1436108351,-0.0531030297,1.4900053740,-0.8120594025,-0.4823431373,-0.6355569363,-0.2808527946,-0.0437979996,1.9277224541,0.9682134986,-0.3746079803,-0.2915026248,-0.4581932127,1.6945850849,0.6297083497,-1.9033675194,1.3773635626,1.8451112509,1.3157194853,-0.4916698933,-0.7966602445,-0.3494751751,0.7045466304,-0.4455379248,-0.3833731711,1.3972334862,-0.5082765222,-0.1657868773,0.0295438245,-0.1786982268,1.2381014824,0.0978887677,-0.6584629416,-0.2192865759,0.9467751980,-0.0771661103,-1.0597406626,-0.1748822480,0.7894371152,-0.4872848988,2.2833466530,-0.5168839693,-1.3883464336,1.3372455835,0.2018319666,-0.2333852649,-0.3624745011,-0.5905905366,-0.7401220202,1.0021631718,1.2818202972,0.2926157117,-1.0420081615,0.4264304936,-0.0421497524,-0.5193667412,-0.8652704954,0.8540445566,1.0223616362,-1.1298987865,1.0555826426,0.0991955176,0.4002920985,1.1874296665,-1.5832803249,0.9813435078,1.2280945778,0.5722233653,-0.9180369377,1.3107286692,-0.2247706950,-1.1927497387,-0.7917345762,-1.1626080275,0.9272098541,0.5459935069,-0.6274787188,-0.0236869771,0.4932527244,1.1298884153,0.0539514609,-0.5518273115,1.1233708858,-0.0687913820,-0.1571341753,0.1179229915,-0.3325984180,0.2601344585,-0.1911605597,-0.4110339582,1.2266099453,0.2822836637,1.2508134842,-0.3510955870,-0.1616224051,-1.6587704420,0.1728632450,-1.3107187748,0.9391930103,-0.0142953722],[0.7546882629,-0.8996400237,0.9940255880,-0.2330982387,-0.4980289638,0.2528294027,2.6544928551,1.5781192780,0.4817900956,1.1860171556,0.3339211643,1.6164300442,-2.9351656437,0.2499151379,0.7760117054,0.7523182631,-0.1247479841,0.3462697268,-1.0257136822,-2.8691601753,-1.1061319113,0.7648423910,-0.9595790505,-0.0078468043,-0.2652287781,0.9949512482,0.5908301473,0.7397772074,0.2819495201,0.9627394080,0.1467755586,-0.7852812409,-0.3648975492,0.7329995036,0.9112494588,-0.6349129677,1.0441013575,1.1536734104,0.7727792263,0.1085980162,0.3048982322,0.2045264691,-0.0964837223,-1.1621612310,-0.1049908698,-1.8086529970,1.1371610165,0.3990722299,-0.2340318710,-0.8070739508,-0.5447965264,-0.9057466984,-0.7056326270,-1.3920232058,-2.2496838570,0.6314688921,-0.0317909122,0.5471078753,0.2917467952,1.1517522335,-0.2557494044,2.4834494591,0.1827259809,-1.1629668474,0.0617253222,-1.1677007675,-0.6916891932,0.1345950365,1.0560717583,0.7259826660,-0.9369471669,-1.2553867102,0.8967444301,0.0060753874,-0.1242959350,-1.3210841417,0.0899388492,0.8379094005,1.1313247681,0.3248853981,0.3055925071,-1.3190481663,-0.4589845836,-1.0276916027,-0.4143671095,-0.2306914777,-0.7529071569,-0.3986451030,-0.4679827392,1.0484802723,0.7876905203,0.3911187947,0.9792061448,-0.1854469031,-1.1137851477,-0.4035208821,-0.8948764205,-1.0624626875,0.4093474150,-2.7420592308,0.5685927272,0.1045850515,0.8899027109,-0.6605038643,-0.0338754356,0.4368726313,-0.5698662400,0.3048413694,-0.9810655117,-2.1477074623,-1.0738272667,-0.4583679438,-0.1335805207,-0.6811560988,0.6706511378,0.0815103278,-0.0946848914,-0.8044914007,-0.1711455286,-0.2423008829,0.8169430494,-0.7940458059,1.6181575060,0.2855857611,0.8950667381,-0.6162665486,0.1139321849,-0.6352536678,-0.0765659958,-0.7508867383,0.1942287982,-0.7451517582,-0.8653050661,-0.2339722216,1.0282814503,0.2243474871,0.0027308404,1.3209453821,0.2174192816,-0.2634245455,0.1675521731,1.2263836861,0.0566299632,-1.2975367308,0.3696041107,-0.9182784557,-1.3032855988,2.0671389103,-1.4618188143,0.4181285203,-0.1183236837,0.4636798799,0.7680442333,-1.1236658096,-1.3899666071,1.2378437519,-0.6141276956,0.0449728668,0.3780954778,0.5110079646,-1.5044722557,0.5645886660,-0.7422678471,0.9555368423,-0.8196920156,0.5401763320,-0.5338007212,-0.0242628828,0.1132424623,-0.6125180125,0.7859786153,-0.0349822044,0.0732879490,1.1536586285,0.1490181834,1.1301728487,0.7858697176,1.2088547945,-1.8960126638,-0.0871836841,-0.8826141357,2.8862614632,-0.8807513714,0.0494962074,-1.1702387333,-0.6589772105,-1.6818491220,0.9417175651,0.7542197108,-0.8279682398,-0.4384127259,-0.1200602949,-1.6220785379,0.1089254692,1.3359794617,2.4555397034,0.0173700582,-0.0874115005,-3.2741274834,0.1969607472,-0.2217420191,1.3908416033,-0.3902492821,-1.8578752279,-0.8916897774,-0.4912000895,0.9974804521,0.2037437856,-0.3271367252,-0.6474327445,-0.1907690018,-1.6302834749,0.6798949838,-1.6188380718,2.0891149044,0.4113075733,1.0672979355,-0.1462323815,-0.1751722842,0.3902901411,-1.0913881063,-1.5157181025,0.9157704711,0.7945489883],[0.7722069025,0.4243962467,0.9691821337,1.4120765924,-1.0809820890,-0.6280459762,-0.6338055730,-0.4000754654,-0.7342044711,2.2752947807,-1.5868879557,-0.5589603782,-1.3043125868,0.3147029877,-0.2022144794,0.8512126803,-1.4063329697,-0.6932082772,1.4493491650,0.0560668260,0.9236955643,-0.5947605968,-0.2050286680,-1.2416573763,-0.6073765755,0.4851435125,0.4411651194,-0.1564729959,0.2195530385,0.3204922378,-1.1177259684,1.0289072990,1.1188204288,-0.0411395468,0.6005723476,1.5676715374,-0.1191437095,-0.5251495838,-1.5412362814,-1.2549991608,-1.7545846701,1.3483644724,-0.9670109749,0.8642841578,0.3205362856,-1.7713440657,1.1340459585,-1.0328261852,-0.1334087253,-0.3448979259,1.2247921228,0.7575573921,0.6027758718,-0.6131908894,0.7743551135,0.5759877563,-0.1214414164,-0.0009457574,0.8447682858,-1.7850353718,-0.1575969607,1.1574027538,0.1245122924,0.6179469824,-0.9759038091,0.1979667097,-0.5714919567,1.1229913235,0.3113834560,-0.4361872375,0.0621054992,-0.8684849739,1.5532866716,0.4511200488,-0.5184523463,0.4275344014,0.4822023511,-0.5793940425,-1.2044733763,-0.0556597859,0.3327965736,0.1595038027,1.5718646049,1.1590943336,-0.7317303419,-1.0252996683,0.3034447730,0.0385221504,-1.4219295979,1.6513744593,0.5207285881,-0.5225039721,-0.5949496031,-0.2466515452,-0.7081933618,1.7684363127,0.2115738690,0.1667692959,0.9263616204,0.5199217200,-0.8082281351,2.0617651939,-1.6870878935,-0.5724027157,-0.0029290456,-0.8472099304,-0.3068249524,-0.7209590673,-0.4516503513,1.2118517160,-0.5764548779,-0.6438353658,0.3038035929,-0.4023731649,0.5642825961,0.1920437813,-1.0712314844,1.2746276855,1.6927058697,0.4486570656,1.3200535774,0.0160080902,-2.3641381264,0.5951035619,-0.1099106371,-0.1813999414,0.6511411071,-1.0492880344,-0.9243508577,-0.5737870932,-0.9042535424,1.7057296038,-0.6014401317,-0.3643075228,-0.2186646163,0.1627379209,-0.8163861036,-1.1460425854,0.9978126884,1.1337602139,-3.2709360123,0.9469668865,-2.5101165771,0.6990010738,0.8786799312,-0.5839535594,-0.6743929386,-2.4371728897,-0.0503076240,0.8705804348,-0.2608596385,0.0898400098,-0.3027840257,-1.2079818249,0.0692925304,-2.0684671402,-0.0220023058,0.3078320622,-0.7879421711,-1.0444283485,1.2165591717,0.1952220052,1.4552105665,1.0593975782,-1.0149605274,1.2875981331,-0.4663090110,1.5024882555,0.9685600400,-0.2157099992,0.2602674961,-0.9132912755,0.0151141891,0.4012723863,1.0047228336,-0.9285416603,-2.0817165375,-0.2928998768,1.5027165413,-0.1005974039,-0.3758979440,2.0350098610,-0.4853464663,2.9407486916,-0.5036264062,0.2076288760,0.2921140492,-1.1655380726,-0.9878220558,1.4906018972,-0.8561943173,1.8389580250,-0.1549417078,-0.2442326397,-0.7554346919,-0.8123953938,0.4190253019,0.6878544092,-1.6918449402,-0.0898365825,1.0332804918,1.4314122200,-2.1892902851,0.0718886927,0.7476850748,1.8579891920,-0.8819519877,0.9258650541,-0.9806548357,-2.4932901859,0.6940491796,1.6708992720,0.1564088911,-0.5294389725,0.7051659226,-0.4034671485,0.8142341971,1.4990714788,-1.3586189747,-0.5893127322,0.2642396390,-1.9446407557,-0.2521522045,-1.6591632366],[0.6953535676,-1.0761489868,0.5135737658,-0.8951653838,-1.4820936918,1.6783659458,-1.3350443840,1.3742983341,1.4251997471,-0.4624463916,0.2494481504,0.8023421764,-0.5172142386,1.5071295500,-0.1262708008,0.1829024106,-0.3753479421,-0.7461107969,-0.9378911257,0.8594884276,-1.3996262550,0.2003267109,1.0123081207,-0.7833893895,-0.3972645700,-0.6808474064,0.4937455356,-0.5152196884,-0.7318217754,0.2788423002,1.9877916574,1.0418436527,-0.6853435636,0.0596736781,1.0391395092,0.7398166060,-0.9924521446,1.1051650047,-1.2520617247,2.8686056137,-0.1694054306,1.9018117189,-1.0746030807,-1.2265579700,-0.8510357141,-2.1456563473,0.6359826922,0.5066269636,0.9674390554,-1.1224880219,-1.0891484022,-0.1846278459,-0.5367618203,1.8058052063,-1.1354886293,-1.9588290453,0.7469393015,0.1361483037,0.1992729604,-0.8409942389,0.1434929073,1.8949650526,-0.5321903825,-0.2382396460,0.0410348624,0.5797135830,1.5907917023,-0.2758657932,0.1504357457,-0.5587373972,-1.4018324614,1.4326224327,-0.6219562292,0.4503026307,0.4008595347,0.3934817910,1.2696865797,1.8898544312,-0.1165970489,0.6457780600,2.2075743675,-0.9847111702,-1.2228060961,-1.0882190466,1.0747212172,1.1284934282,-0.7757837176,-0.1252997965,-0.1738859862,-1.0792155266,1.7511061430,-0.1438600868,-0.9502803683,0.2619571090,-0.2889937162,0.0514364801,0.5962271690,-1.3032315969,-0.3775050342,1.3788658381,-0.5376601815,-0.3634388149,-0.5220135450,0.6863537431,-1.0816643238,0.8089571595,1.1289632320,-0.1704909503,0.0241376441,0.9559123516,-0.2972258627,1.2451685667,-0.7107396126,-0.4264647663,-0.2931824625,0.5185298920,-0.1059233323,0.6447357535,0.7338988185,0.5817427039,-0.6184529066,0.1341575831,-0.4200459421,0.8393083811,-0.3927664757,0.4707723856,-0.0673559830,0.3716157675,0.3322022855,0.0640629455,-0.7132993340,0.0691948086,-0.8344638944,-0.1031417176,-0.4747044742,-1.6690871716,0.6154571176,-0.6817176938,-2.0737211704,-0.3878850639,1.4523063898,-0.6992966533,-0.4581898749,0.1936287731,0.2572551072,1.4544843435,-1.6571730375,-1.7270624638,0.0417574421,-0.9110734463,-0.0972457156,1.5952712297,-0.8988962173,0.5708675981,2.3974201679,-1.5280815363,-0.1108923852,-0.3071365356,-0.9979051352,-0.1661987454,-1.0751403570,-2.5248780251,-0.2498296946,0.9437659383,0.9217237830,0.7171913981,-1.4578920603,1.2408026457,-2.4817552567,0.2420867234,-0.3377861381,0.0360229239,-0.5133728981,-0.3208512962,-1.3301918507,-0.6975618005,0.6498871446,0.8788000345,0.0332687683,1.0863420963,0.9952074885,1.0205569267,-1.5511695147,1.1222990751,-0.2344319969,-0.6389635801,0.2854325175,-0.7309339643,-1.9008569717,-1.2902566195,-1.4089726210,-0.0378140882,0.8407415748,-0.6611420512,-1.2427229881,-0.5391062498,-0.7145795226,-1.0734699965,1.3682132959,-1.0783115625,-2.2720541954,-0.7096360922,-0.2335556746,1.0443377495,-0.4696555734,-0.3875656724,-0.8236765265,0.2266024798,0.1619909555,-0.0944048539,1.6084132195,-0.3988568187,0.1957289577,1.5395137072,-0.1689223051,-1.5008987188,-1.3173402548,1.1269226074,0.3471261263,0.3052436709,0.3154096305,0.0921308771,0.1269512475,1.8258371353],[0.1363988519,-0.9507242441,0.7849888802,0.3052140176,-0.2709810436,-0.8993048072,0.2169116437,0.1867516041,0.7598631382,-0.8012745976,-1.1976956129,0.0239624623,0.3074752390,1.2831456661,-1.3402698040,-0.4516152740,-0.0013061600,0.4203725457,0.3012847304,0.2762047648,-0.9804582596,1.1610437632,0.5745239854,0.6747921705,1.4261837006,-0.5042745471,0.5857908726,0.8970323205,0.1667999625,-0.4011804461,1.7648248672,-0.8087148070,-0.2742171586,0.4606503844,-0.0885505006,0.2718375921,-0.8004333973,0.6516600847,1.3673778772,-1.5335977077,-0.0962385684,-0.8232393861,1.6774146557,0.2271115631,-1.5541958809,0.3077866137,-1.7232172489,-2.4555950165,0.8917973042,1.7451490164,-1.7838643789,0.6180729270,1.0859361887,1.1453979015,0.9061566591,-0.3431972563,-0.0705898181,1.0991281271,-0.2261981666,0.3282406032,1.0704436302,-1.1716351509,0.9848763347,0.2437052131,1.9884727001,0.4174208343,-0.1021214947,-0.3435896039,1.3310985565,-0.5973044634,-1.0328304768,-1.0166631937,-0.2418605387,-1.0737953186,-1.5078314543,-1.9658055305,-2.4816696644,0.0847596675,0.9371824861,0.5712789297,0.1847821474,-1.6016809940,-0.5153174996,0.3281519115,-0.0960697681,-0.7360475659,-1.3864686489,1.0394927263,-0.0865700766,-1.2883691788,-2.0292968750,-2.0399634838,0.6458489895,0.4366787672,-0.2919606566,-0.8011184335,-1.1781026125,1.2439732552,0.7523661256,-0.4787828326,-0.1176917329,-0.3622176647,-0.9125710726,-0.3694320023,-1.2082426548,-0.5479109883,-0.8315254450,-0.7423962951,0.1000797302,-0.2114170194,-0.6813454032,0.8390816450,0.4109891355,-1.2436894178,0.5534462929,-1.4438923597,0.8049284816,-0.8555861712,2.1477587223,-1.3949433565,-0.2616662383,-0.9542909265,0.1196367517,1.5450251102,-2.3118369579,2.7454116344,-0.5705853701,0.0806028023,0.2696255445,1.1088483334,-0.6098455787,0.2533075511,-1.6156879663,0.4766279757,-0.9498989582,1.3911211491,0.0607453063,0.3484802246,0.2540907860,-0.9823598862,-0.6200393438,0.6513434649,0.6371821165,1.2473409176,0.4007479548,-0.6293382645,-0.3995212615,-0.3992503285,2.1516108513,0.8495406508,-1.1839305162,-1.5761988163,-0.3321847916,-0.6933479905,-0.5461527705,0.2077549547,-0.0470405109,0.9233145118,-0.3973110914,0.0615227558,-0.0844099149,-1.7535184622,2.8657610416,1.7541314363,-2.3690114021,-1.1117147207,-0.9374561906,-0.7801645994,1.7342389822,0.5957742929,1.4701598883,-0.4405146837,0.6708141565,-1.2153530121,-0.1325355917,1.0050604343,-1.2139691114,-1.3798935413,2.3798618317,-1.9887228012,0.3868185282,0.8959914446,1.6038486958,1.5363543034,-0.7630540729,-1.4995023012,-1.8495545387,-0.2919710279,-1.5776841640,1.7685606480,1.1297414303,2.3360943794,-0.1433131546,0.0687443092,0.5931475163,0.3375712037,1.2554653883,0.2474237680,-0.9403129816,-0.2061767131,-0.4198792279,0.3141739070,1.7077875137,-1.4152840376,0.1187793538,-0.7630447745,-0.6395958662,0.0102028055,0.1042164192,0.1145903766,-1.4752571583,0.2472184002,-2.1607117653,-0.4515348971,1.2877726555,0.8726791143,0.5221362114,0.0558537394,-0.7062091231,-1.0879805088,0.5266829729,0.1596421599,2.6657764912,0.8143402338],[0.9257330298,2.1734650135,0.5997606516,-0.1436795443,0.3496937156,-0.9546177983,-0.1620016545,0.9732399583,-0.5467224121,-0.2262014300,0.6731067300,0.2159280777,-1.3774738312,1.1656154394,0.6550341249,1.9122684002,-0.9768811464,0.9356287122,-1.2053273916,2.4479525089,0.6917709708,-1.5790313482,-1.4568662643,1.2126783133,0.6390294433,-0.5516815782,0.8256443143,0.4482912719,-1.8841407299,-1.3875689507,-0.1513385475,0.4152318239,-0.6240538955,-0.2882059515,-0.1251111031,0.0627681017,0.2525975108,1.4493167400,-1.2855988741,-0.0614147782,1.2215783596,0.3575677872,0.3007901907,-2.1066825390,0.6250435114,-0.6612740755,-0.1341212541,0.2166793793,-0.9497941732,-1.2139436007,1.2116563320,-0.9395774007,0.3062827587,1.1016186476,0.0750297382,0.1435457021,-0.8186900616,-0.4630457759,0.5266259909,-0.0092119304,-0.5354993343,-2.7169761658,-1.1486746073,0.2088882625,-0.6807827950,-3.0617961884,-0.3313309550,-0.1758318841,1.4389998913,-1.4673558474,-0.7488585114,-0.0289274994,-0.3636911213,-1.2437512875,-1.5300729275,-1.1790323257,0.3857260942,-1.5467594862,-0.1613754779,1.4048904181,-0.5613690019,-0.7192589045,1.7300932407,-0.5238711834,1.0657387972,-0.9189323783,-0.5509738326,-0.5299234390,-1.8563530445,0.7000102401,1.3967370987,0.1085478887,0.9389107227,0.6333724856,1.0824018717,1.2884047031,-0.4879322350,1.5856468678,-1.3458198309,0.4698221385,0.9904260635,-0.7995867133,0.4108642042,0.8107822537,1.3460342884,0.7748122215,0.6732077003,-2.0124661922,-0.1145293713,-0.4353868663,1.0582432747,0.4740674198,-0.8044639230,-0.0220145155,-1.1738559008,0.1407169402,0.1516216695,0.4735494852,0.9077764750,0.9353476763,-1.6902123690,-0.0617531240,1.0522574186,1.1403518915,0.4502256215,0.3180954456,-0.8513272405,1.7308795452,-0.9569784999,-1.1495845318,0.0731476098,1.3367174864,-0.1736793220,1.0153282881,0.7569540739,-0.4068577588,-0.9060294628,-0.1295568198,1.8163871765,-1.3072237968,0.6215203404,0.6559920907,0.6238611341,0.8602724671,1.3712561131,1.7863894701,-0.1854007989,-2.0719249249,1.9330905676,-0.2852199376,-1.1075080633,-0.1679364741,1.5029848814,-0.3086349368,-1.5096340179,-1.3769066334,-1.7379231453,0.3038515449,0.5172749758,-0.6259486675,0.7043590546,0.6408320069,0.4185501933,-0.1646103710,0.3187517822,0.8196350932,-0.4609601498,0.3704506457,2.3777439594,0.5094441772,0.5521851182,-1.3909140825,1.1121610403,0.8421571851,1.1006048918,0.6569547057,-0.8560597301,-1.2483242750,-1.7567801476,-0.8713760972,0.2235589772,-0.2905384302,0.6802270412,-0.0276177991,-0.9667946696,-0.3455171287,-1.4121873379,0.2059963942,-0.6952722073,-0.7459636331,1.0301487446,-0.6903392076,-0.1190134883,0.2581081092,0.2567854524,0.7716189623,-0.2509430051,1.0092092752,-0.5644721389,-1.3445675373,1.3076673746,-0.9773423076,0.3826100528,-2.6880400181,0.4268960953,-1.1517989635,-0.8691557050,-0.6256707907,0.5513302684,-1.1145905256,0.8849994540,-0.8217748404,0.9004893899,-0.5353831649,-1.2508430481,0.7113912702,-2.6005218029,-0.7853864431,0.3203434646,2.5373580456,-2.3225741386,-0.2459121794,-1.1617994308,0.7776864767],[-0.1386852562,0.8962039948,-0.3026582301,0.0292508341,-0.2019753456,-1.3439552784,1.8885324001,-0.2357038409,1.0730520487,1.1235663891,0.1953596920,-0.2643015087,-0.0000123232,0.8602338433,-1.4039980173,-0.5216125846,-0.0435859486,0.3864676356,-0.6059318781,1.2893551588,-0.7495974302,-0.7594406009,-0.2251145393,-0.8615306020,-0.3031273782,-0.7514218092,-0.0263201371,1.0012412071,-0.4263814092,0.3582890928,0.3132919073,-1.7429720163,-1.0658556223,-0.5562067628,0.6188708544,1.2195292711,0.2364767790,-0.7100563049,0.6203096509,-0.9709291458,-0.7118019462,-1.3894826174,0.1876191944,0.7731317282,1.7870222330,1.3483870029,-0.2876165211,1.2424424887,1.0495731831,0.2199662328,-0.6506385803,1.4809036255,0.1745363027,0.5330241919,0.2584872544,1.7948721647,1.2031013966,0.2966628075,1.0648564100,0.2883406579,1.7498236895,-0.3993926644,-0.5972360969,2.3710198402,-0.1631287187,0.6456073523,0.1387225240,0.9679862261,-0.4325176179,0.3984373510,-0.1760327667,0.6858507991,0.1566565484,0.7886711955,-2.6216802597,0.6207202673,0.0285687726,0.6761151552,-1.3356418610,0.7205585241,1.2382751703,-0.6817107797,-1.1695361137,1.7380303144,0.8015923500,0.2585445940,0.3077518046,-1.6452590227,0.4001936316,-0.7441572547,-1.4529861212,0.2506214380,-0.0533802733,1.1744852066,0.7447588444,-0.3415437043,-1.0891363621,0.6034783721,-0.1482316405,-0.4128837585,-0.0329800732,-0.7001284957,-1.2899012566,-0.7876139283,1.2155237198,0.0087155225,-0.0377709866,-1.3937059641,0.7941431403,0.1870978177,0.6264303327,0.8145871758,-0.7728582025,-0.2493467182,-0.5739196539,-0.5565655828,-1.1334906816,-2.2793505192,0.4410243928,0.4123649299,0.0763852969,1.4099802971,-0.7776322365,0.9002335072,-1.7927150726,0.3387904763,-1.5426032543,0.1109750047,-0.1004541144,-0.7143050432,-0.9074044228,0.1946927160,-0.6922547221,-0.5126389265,-0.2491152287,0.8359513879,-0.5823138952,0.1778424084,-0.1264958382,-1.2308934927,0.0432014875,-1.7107870579,-1.2794522047,-1.4112915993,0.2468446046,0.2457828671,-0.3186832964,0.7732830644,0.9469830394,0.5959275961,-0.2578312159,0.3627173305,-0.3071349859,-0.1745721549,1.1378543377,0.7938080430,-0.5381634235,0.4013376534,-0.9682885408,-1.1399817467,-0.9329072237,-0.5803301930,0.1786864400,0.6346725225,0.5647393465,0.0286316723,0.0992199630,-1.0894353390,1.3608300686,-0.4605360329,-0.8283517361,-1.8477090597,-0.7479529381,0.4846981764,-0.8102111220,-2.7513906956,-0.8704450727,0.0546552204,0.9808416963,-0.4524506330,-1.1581110954,-1.8916167021,1.5128071308,-0.2710370421,-0.0432773046,1.0887061357,0.8253607750,0.9289151430,-0.1334707141,-1.1068481207,1.0271012783,-0.4083566964,-0.9534019232,0.9252843857,1.2556301355,-0.3255678713,0.6176265478,-0.1261923760,0.7121532559,1.1629872322,-0.7318741083,-0.0021026048,-0.6439540982,0.7954640985,-0.5480089784,-0.9157254100,-1.1337063313,1.0404101610,-1.9267649651,1.2752271891,-0.6368082762,0.1870581359,0.9406405687,-0.3973135352,-0.0837642848,0.1083026305,-0.1242073551,-0.2883494198,0.0421624370,0.0474995077,0.6321162581,-1.5836554766,-1.1032531261,-0.5887776017],[-0.1524914503,-1.6673083305,0.9382166862,-1.0333873034,0.4561629593,1.1972030401,-0.6820040345,1.9431983232,-0.2159686685,1.8049536943,0.5602540970,0.0748924091,0.5605863333,-1.7343138456,0.6010981798,0.4802421331,-0.8561789393,0.1013424918,-0.4451296926,-0.3943363428,-0.0749234781,1.0080494881,1.6752249002,0.3181442618,-0.2150738686,-0.0598962717,-0.6142386198,-0.4861352444,1.5881255865,-0.0372432843,0.1362068504,-0.6435383558,0.3061073124,-0.1080607101,0.9142783284,-0.6750063896,0.1442564428,-1.9035853148,-1.9769639969,-0.2643598616,-0.1542481184,0.6130289435,-0.9667575359,0.4809058309,-0.8523111939,-1.5576803684,-1.7187989950,0.3377987146,0.7529808879,-1.0509667397,-0.7730994225,-1.2046824694,0.4276761115,-0.8403949738,-0.4825850129,-1.4747754335,0.7216343880,0.4391319156,-0.2775903642,1.8374639750,0.8860115409,0.5017250776,0.1924185902,-0.3279211819,0.5628093481,0.9199988246,0.2608970404,-0.1163030937,0.9977791309,-0.1468842328,0.8259064555,0.8571233153,-1.1635246277,-1.2943441868,-0.6337320209,0.0078612473,0.2800707519,-0.4688892663,-1.5412379503,-1.2810702324,-0.9014613032,0.7189544439,0.6765769720,-1.1631652117,0.4434119165,0.9910751581,0.3494699001,-0.4065314829,-0.3706806600,-1.8773276806,0.2003043443,-1.7980109453,0.4200835824,-2.1893362999,-0.2583111823,-2.0468666553,-0.2437464893,1.0890002251,-0.0624531172,0.3706884086,0.4278266132,-0.1703085750,0.8178662658,1.0959708691,0.5633809566,0.4105978310,-0.4165204167,0.4329338372,-3.3593549728,-0.7902920246,-0.5976426601,0.2041165531,0.7518958449,0.9105246067,0.6583688259,0.8852194548,-1.0034178495,2.1018614769,-1.3160889149,-1.2050503492,1.3702383041,-0.9215651155,1.1412268877,-0.5294218063,-2.0968155861,0.9883624315,-0.4318448901,0.9150150418,0.6191473007,0.9672363997,-0.5683717728,-0.9324068427,2.0865449905,-0.2670659721,0.3249341547,0.2030726075,0.2451879531,-1.6215312481,-0.5928865671,1.1713583469,0.9839522839,-0.0872299895,-0.3621385992,-0.9829427600,0.8884603977,-0.1793822050,0.5179939866,1.3586529493,-0.9791272879,0.3234313130,0.5289196372,-1.9766300917,-0.0034390925,-0.7666292787,0.7933413386,0.8603540659,0.9405009151,0.6027413607,0.5473909974,0.6982603073,0.3897379637,0.3500108123,0.1734568626,-0.6509557366,-1.7006138563,1.1740543842,-0.2860489190,0.7598106861,-1.4153439999,1.2704614401,-0.1627092808,1.0294121504,0.4012169838,2.0456750393,-0.0367478020,0.6690829992,-0.3728141785,-0.2102022916,0.0779081285,-0.9818955064,0.1598586440,-0.7887569070,-1.6699723005,1.1914888620,1.7864699364,-0.5499423742,0.5733178854,-0.5447788239,-1.7513740063,-0.9034187198,0.5524613261,-0.1586202979,0.0848929808,1.8023984432,0.4640448987,0.3638080955,0.2646439970,0.1547241062,-1.4154069424,1.4598560333,0.8310509324,-0.4224171937,-0.0165275075,0.2615670860,-1.1916067600,0.2750308812,0.6544116735,-0.3800196648,-1.8660432100,0.2894992232,1.1039299965,0.1959735453,-0.0791411027,0.1106350198,0.1640786827,-0.1299720556,-1.6187663078,-1.1956890821,1.1165030003,0.6533244848,0.3864424229,0.2913912237,1.0975930691,1.4230506420],[-0.0867393687,-1.6922757626,-0.2748315632,1.2141071558,0.5547963381,0.0646452904,-1.1866117716,-0.0320456326,-2.1211907864,-1.9493031502,-1.0958501101,-1.0897547007,-0.1803523153,0.3207212687,1.2608158588,-0.5926650167,1.2983776331,-1.7294532061,-0.4939606190,-0.8452877998,-0.4836902916,0.6845428348,0.1369161159,1.1284254789,1.4544321299,1.5625711679,-0.1469358951,0.1927613020,-0.7056970000,2.1935093403,-1.0594838858,-0.1013154238,0.1026509628,0.8066827059,-0.2695352435,-0.5729221702,-0.3849198520,-1.0156600475,-1.5466378927,-0.1314595789,0.3548667729,1.6491620541,-1.0673861504,-0.5086348653,0.5900128484,0.5537997484,-0.9028851390,0.3827905357,-0.4903090894,0.1471827030,0.5189530253,-0.3839559853,-0.0064701196,0.0774409026,1.1993261576,-1.8423675299,0.8041459918,0.0865443349,0.9438959360,0.4559169412,-1.9510413408,-0.0187463369,-0.5971686840,-2.1307387352,-1.0078853369,-1.1159837246,0.0679549426,0.3910646141,0.6450982094,0.2970682085,-1.4807668924,1.1750893593,0.6418598890,1.7684969902,-0.5421749949,0.0230772309,-0.5444252491,0.6334091425,-0.1794523001,0.8945975900,-0.3239262700,-1.0767819881,0.3691432476,0.6942480206,-0.0642852709,-0.8203831315,-1.1774146557,-0.2505559921,0.8347317576,-0.1928883344,-0.5916996598,-2.2447366714,1.4082713127,-0.2975973487,-1.0574928522,1.0062141418,1.1215678453,0.6743121743,1.5402876139,0.5045122504,0.4825377464,2.3591935635,-0.1581576616,1.3925447464,-0.4449972510,0.6436666846,0.6480388045,0.6492596865,-0.3070269525,0.5588863492,-0.8348108530,0.7839213610,0.6336535215,-1.4643620253,-0.8206500411,-0.7092571259,1.5029654503,0.9617568851,-1.5836352110,1.0906035900,-0.5501468778,-0.9884999990,-0.1372122318,1.2027401924,-1.1109777689,-0.5006306171,-1.2436493635,0.8456214666,0.9922400117,-0.5578774214,-1.5881406069,-0.0128620062,0.7343695164,-0.6115727425,-1.6536990404,-0.0360801816,-0.2093567550,0.7303119898,0.3931238949,0.9025912881,-0.2072690576,0.2031912059,1.1792848110,-0.8231495619,1.9168200493,0.2706981301,-0.7323203087,0.8667071462,1.5296094418,0.2382735461,1.0494682789,-0.3883500099,0.1169275641,0.9171454310,-0.1054672450,0.0179931466,-0.9391468763,-1.3003677130,-2.8754465580,-0.1241761670,-0.1267907768,-1.4475177526,-0.0179561060,-1.7879973650,0.7654473186,-1.1107738018,0.1945697963,-1.0288122892,1.6509739161,0.1266491860,0.4879723191,-0.2690182924,1.3367944956,0.5753632188,-0.0968106240,-0.9281479716,-1.2006169558,0.0849150419,0.9885011911,-0.5190308094,1.0386395454,0.1456762850,0.4775098860,-0.4643878937,0.6115694642,-1.2517410517,0.8366264105,0.4481097162,-1.3137805462,0.1005480215,-1.1128195524,-0.0262514874,1.1235595942,1.6695141792,-0.1654336303,0.7355045676,-0.8682185411,0.0562831461,-1.4327114820,0.0250446890,0.0975233465,-1.1841610670,0.7637282014,1.5198569298,-0.7049927711,0.8754818439,0.3610333502,-0.6621111035,0.2180406004,-2.0024406910,-0.8863433003,-0.1631226391,-0.3874653280,0.5548259616,1.1022121906,1.0524899960,-1.2551360130,0.9903590679,0.8982601762,0.8245859742,-0.3155473173,0.0082518505,1.3746151924,-1.0334572792],[1.6923905611,0.3971545994,0.2894242704,-0.5514086485,-0.3389030099,-1.0362766981,-0.5482236147,-3.2779073715,2.1528837681,0.1756819487,0.3480467200,1.8974837065,-0.7496224046,-0.3268892765,1.3972829580,-0.3456662893,1.4522365332,-1.6095561981,-0.0210978836,1.5160992146,1.0965744257,-0.2358710319,0.6267479062,1.7261326313,-0.3205836713,-0.1390050799,-0.7758511901,2.0753962994,0.1757676750,-1.7555754185,-0.2505002618,0.2840710282,-0.5393370986,-0.4869838059,-0.1303175837,-1.1189780235,-1.0521354675,-0.0231210198,0.0181847401,0.2078745663,-0.0338686258,-0.4818850160,0.0961250514,-0.9138529897,0.9418380857,1.3300968409,1.5610092878,0.4450918734,-0.1296869665,-0.2519723773,-1.5859483480,-0.9557150602,-0.2045584917,0.0128896115,-0.2095763832,-0.4511358142,0.5498040318,1.8048099279,-0.0429422446,2.2919325829,0.0038200787,0.2721864283,0.4414996207,-1.3005160093,0.1768380255,-0.9363270402,0.4566507041,0.3796249628,0.4649937451,0.7009580731,1.0819793940,-0.5470877290,-1.1549626589,-0.2274688035,-0.4943396151,-1.7293069363,-0.1460026801,1.5145381689,0.1582663655,-1.7069764137,-1.0666083097,-2.0992667675,-0.3120955825,1.5823105574,-0.2526357174,0.6126517653,0.7511574626,1.9334533215,-0.1653770506,0.0007328019,-1.1496319771,1.0879269838,-0.4144269824,-0.1782887280,0.2705714405,-1.3183277845,-0.7594566345,2.8128893375,-1.7517324686,-0.1413636059,0.4365155399,0.8948895335,0.3178620934,-0.6415089965,-0.0946490541,-0.0807673633,-1.0004671812,-1.5345644951,-0.2357628345,-1.5820407867,1.0702680349,-1.8865656853,-0.0901094601,-0.0760608092,-0.1922454238,-0.5639578104,-0.5438117981,-0.2419601381,0.9866610169,0.3484502435,0.2144006640,-0.5236511827,0.2716521621,-1.1190679073,1.0188894272,-2.0297060013,1.5627001524,-1.2868920565,1.2256826162,0.2894935012,-0.7158158422,-1.3701560497,-1.0779329538,-0.0256836489,0.3960071802,0.4327994287,1.2228806019,-0.1757212430,-1.3786426783,-1.9703372717,-0.5863664150,-0.0080045890,0.4041854441,-1.3040728569,-2.0099992752,0.8277711868,0.7524700761,-2.8358297348,0.3762879968,-0.0772758722,-0.1378519088,0.0027876387,-1.0244839191,-0.3540144861,1.7863705158,-0.3465160429,-0.5774444342,-0.2480545044,-1.1455147266,-1.4086068869,-0.7194760442,-0.4547182322,1.7560970783,-0.0071993573,-0.6203646064,-0.1874046624,1.5937808752,0.5413727164,-1.4341344833,-1.6528888941,-0.2458195090,1.1216475964,1.8508564234,-0.0923150182,0.1672917157,-1.5417040586,1.4463324547,0.5009288788,2.8749001026,-0.7061102390,0.0816367939,-0.2914707661,-0.0285660401,0.0983645543,0.2963778079,-0.3110705614,0.8581354618,-0.7723867297,0.8028447032,0.2162812054,1.0721161366,0.0564372241,0.6005313396,-0.1831871420,-0.4489869177,-0.9456906915,-0.0371265896,0.4959995449,-1.7010400295,-0.6790872216,0.0206640214,-0.1886631548,0.1214666665,0.5371381640,-0.6261289120,-1.1847596169,-1.1002988815,1.4077138901,0.3684228063,-1.1791006327,-0.0171466414,0.1485635340,-0.1480452716,-1.2246426344,1.0717194080,0.5924861431,-1.5882186890,-1.8824286461,-0.7987720966,1.6857186556,-1.9873899221,-0.6040264964,0.9996188879,0.7584842443],[-0.5855907798,1.0974962711,-1.0957789421,2.0606269836,-0.5865968466,-0.8793839812,-0.7873229384,-2.5463385582,-0.6912784576,1.2159292698,-1.9346731901,0.3178563118,-1.4861592054,-1.6317461729,-1.2586566210,0.9521486163,-0.2169671655,0.7148294449,0.7958927155,0.9093554616,-0.1415878832,1.3334403038,0.5828847289,0.8194612265,-0.1050379872,0.4659985602,0.6288198829,-1.1417542696,-1.3998000622,-0.3798598349,-0.4254036546,-0.3694149554,0.5209847093,0.8141931891,0.2401541024,2.4843487740,-0.5280447602,-0.9274998307,-0.1653894037,-0.1363191158,-0.8200877905,2.1133174896,0.8898195028,0.8735603094,-0.0686016232,-0.9245443344,-0.7718877792,0.4842652678,-0.2915898860,0.7432041168,-2.1153192520,0.9568510652,-0.2215748578,-1.8422077894,-1.2303696871,-0.1241944283,-0.7307086587,-0.6680089831,-2.3341002464,2.2076082230,-0.2407692969,0.5288803577,0.2479257435,-0.1337349117,-0.5434597731,-1.7410125732,2.0310096741,1.1825866699,-0.5783034563,0.2525941133,1.1653379202,1.0282540321,-1.0832226276,0.1212414354,-1.0867486000,0.2782225609,1.0789091587,1.1128096581,-0.7514941692,0.4164353907,-1.0873526335,0.5424474478,-0.2048412859,-0.2030930966,0.5863599181,-0.8489893079,-0.3559145629,-0.5795920491,1.5493372679,0.4460066557,0.0156894699,0.1472632438,-1.3260676861,0.7402061224,-0.6366640329,-0.3111352026,-0.1385994852,0.0820364133,1.1148211956,-0.0950735360,0.2773782015,0.9351531863,0.1982546598,1.3031610250,-1.2071324587,0.5444293618,-0.4473699927,-0.4108420610,-0.0568518825,0.4197574854,-0.6447360516,0.3293173313,2.5226984024,2.6850180626,1.9191200733,0.0876055658,-1.8289321661,-0.3188391030,0.1750170290,0.7548211813,0.6596356034,0.3386530280,1.7292153835,-0.1219225526,0.6411107183,0.7875508666,0.2660456002,-0.1457971931,0.7125737071,1.7682244778,-1.1409661770,0.6875343919,-0.9086990356,1.6409335136,0.8345350623,-0.9138651490,1.2705948353,-0.2345217317,0.4959183931,-0.6939529777,-1.3865205050,0.0969705433,-1.4421837330,-1.9025647640,-1.1793817282,-1.9167758226,-0.3548213243,-1.7836306095,0.4976726174,0.9922037125,-0.0509538837,-0.7649970651,0.8111050725,0.1019998342,0.2750899792,-0.0343734659,0.9133414030,-1.3589154482,0.5406615138,-1.3692927361,0.7702706456,-0.8848457336,-1.3492389917,1.5141055584,-0.8034302592,1.4701484442,0.7194578648,-0.3280103207,-0.1737902015,0.3155027628,0.8393989801,-0.7880792618,1.4342643023,-0.2792739272,0.7217347622,-1.1199485064,-1.6488090754,-1.1588809490,0.2449669391,0.9734916687,-0.8612310886,0.2776378691,-0.1442465335,0.2899083197,0.1926309317,0.2951425612,0.1448704749,1.0455939770,1.5262594223,-1.8003038168,0.2336754799,-0.9080817699,1.2704061270,-2.1403088570,-0.9260414243,-0.1840798110,-0.3402787745,-1.6307851076,-0.2041860521,-0.1278241426,0.5095170140,-0.6179625392,0.0498966575,-0.4110610485,-0.5157909989,-0.6018150449,-0.4285067618,1.0633392334,-0.7848201990,0.1631879508,-0.2149524540,-0.3020791411,0.6054601073,-0.0504952818,-1.0965297222,-0.7816454172,-0.2012111992,-0.6565100551,-0.1104105487,-0.2088105083,-1.9115939140,-0.2466346174,-0.2519866526,-0.4590988457],[-0.0494616181,-0.6155298948,0.2704988718,-1.8088672161,0.7919092178,-0.1851701885,0.4661017060,-0.3695489466,-1.0320540667,-1.1022082567,-0.6764156222,0.4022864997,-0.2559004724,-1.2627768517,0.2470270246,-1.8833502531,-1.2920002937,-0.5564384460,-2.3891890049,-1.1999177933,0.7154252529,-0.1808043569,0.1423880011,-0.0719283819,1.1965063810,1.0246064663,0.2847369015,0.0961781964,-0.2521937788,0.7456894517,-0.0504549034,-2.1432087421,1.7391443253,-0.0206905380,0.9230058789,0.1440964341,0.3340483606,-0.5639219880,1.1893047094,0.8010678291,-0.0857279226,-1.6603605747,-1.8865714073,-1.5158432722,2.2416851521,1.3034378290,-1.2954757214,1.3497186899,0.6987150311,-0.5378327370,0.9773977399,-0.6292833686,-0.2258862704,-0.0231768172,0.2287255973,-0.0307552647,0.2602588534,-1.1946291924,-0.4701487720,-1.6524150372,1.5998086929,0.8542186022,-0.8980391622,-0.3424322009,-0.7620440722,0.5442741513,-2.6877915859,-0.6870270967,-0.1067433506,-0.5056813359,-1.1792836189,2.6766495705,-0.7369825244,-0.4661085606,-1.5385745764,-0.0672003403,0.3695683777,-0.7869833112,0.4003332257,0.3783661425,0.5367749333,-0.5993347168,-1.6679698229,1.3760275841,-0.1786604971,-0.0802356526,1.0228990316,-1.9961047173,-1.0266917944,-0.5655768514,0.8560996652,-1.4708665609,0.7181494236,-0.1992621869,-0.3848416209,-1.4284293652,0.0078248149,0.0301324613,1.0360671282,0.0559637584,0.7436648011,0.2172421217,-0.4548107982,-1.8581829071,-0.2697221041,-0.7858588696,0.2048793733,-0.1674392670,-0.5469172597,1.3462063074,1.0827282667,1.1228549480,1.0326079130,-0.9947468042,1.4552406073,-0.3257787228,-0.7388832569,-0.7861909270,0.2828303277,0.3056680560,0.2146909982,-0.2772153020,0.3958362639,-1.3230555058,-0.8603905439,-0.2827332020,-0.3374444246,1.5174309015,-1.3406960964,-0.7753478289,-0.6090564132,0.4148904383,0.1143402532,0.6358146071,-1.0189952850,-0.2470776290,1.8384991884,0.2994590998,-1.3782683611,-0.9034779072,1.0411815643,-0.8380994797,-0.0788478255,-1.2593122721,0.0735025629,-1.2232187986,0.0042018746,0.4956058562,1.0793445110,0.3841406405,-0.5286387205,-0.8840914369,0.1363095641,0.2901922762,0.5978170037,0.2368114442,-0.1575534791,-1.1024419069,1.1252269745,1.5449730158,0.3530938923,0.7228652835,0.1168344319,-0.0158468988,-0.1661573797,0.5381613970,0.3136244714,0.0325458795,-1.2324377298,1.0933052301,-0.1038841233,-0.2963826954,-0.9586788416,0.7200127840,1.4924416542,1.7752082348,-0.0550629981,-1.6644073725,-0.5077100992,0.2773285806,1.0041874647,0.4742742479,0.4997110963,0.2690739036,1.5814677477,1.5842721462,1.1153169870,-2.0206856728,-0.6906757355,0.8339030147,-2.4003989697,1.7535443306,1.4520895481,0.5084596872,-0.1650930792,0.1084622741,0.5147604346,-0.7298306823,-1.9664291143,0.6762105227,0.5794365406,0.7264261246,1.2572396994,-1.8034398556,-1.4639929533,-1.3501420021,0.3714019656,0.7283082008,0.5316600204,0.4573111832,-1.6541035175,1.7793091536,-1.6135524511,1.4555981159,0.8226402402,0.7780967951,0.1016966924,1.1728751659,-0.4354915619,-0.4268640280,0.8830320835,-0.7973753214,-0.8927229643,1.0299344063],[-1.4835407734,1.2919406891,0.9132064581,0.3831681311,0.3834024370,0.9610646367,-2.4891815186,-0.8557786345,0.2069403827,-1.5988606215,-0.9463294744,1.0044388771,0.5957759619,0.3084153235,0.2008418739,-1.1726783514,0.2280961126,-2.1500291824,1.9621980190,0.0185094886,0.4452189505,0.9216417670,-1.0115605593,2.1943533421,-0.1640067250,-0.4602047801,-0.2296462357,0.9627048373,-1.0101568699,-1.4202458858,0.4832316637,0.0462592840,-1.5302428007,-0.3245599866,-1.0381706953,1.2537368536,-1.6339751482,-0.1652428210,-0.3389556408,-0.0146003719,-0.7025259137,2.6607224941,-0.6276010871,-1.0365874767,0.4586429000,-1.0496587753,-0.2999071777,1.3842225075,1.0823534727,-2.3419559002,0.8761150837,-1.8273769617,-0.2215786427,-0.3491138816,-1.9336280823,-0.0474423543,-2.4403247833,-1.2015104294,-0.2873283327,-0.8806145191,-0.8555080295,0.0481235869,-0.8469879031,-1.1857453585,-0.9132013321,-0.4592289925,0.0136311175,0.6643366218,-0.0491968133,0.2498437613,-0.8947763443,-0.0383777991,0.0546499304,-0.3900771439,-0.4424458146,1.0053803921,-1.6437406540,-0.2351907790,0.0837585628,-0.8065138459,0.3854508698,1.7295747995,1.0369310379,1.0545397997,-0.1897726953,-0.6609727740,-0.6300903559,1.0252816677,1.3816955090,0.0301201232,0.4552626610,-0.0021572479,-1.2396833897,1.6422270536,-0.9795646071,0.3572402298,-1.4496890306,1.4168978930,-0.8599024415,-0.1259931624,-0.8222708106,0.3623867333,0.1708000451,-0.3181733787,-1.9245707989,0.0009699502,-1.5193839073,0.6653414369,0.2419777066,1.6192805767,0.9414151907,-0.6941750050,0.4210597575,0.2198369801,-0.5225008726,0.5529451966,-0.1377501786,-0.0109302262,-2.6844522953,1.0686687231,-1.0907949209,-1.6353297234,0.2023297548,0.6188598275,-0.2628114522,1.0546277761,-0.1554549485,-0.4642913342,-0.0471217670,-0.0422636569,0.7548336983,0.4991921484,-2.0590631962,0.3868599534,1.4013535976,-0.9825395346,-1.3489254713,-2.1613402367,-1.1077320576,-0.0115719670,-0.1067917049,0.2030382305,1.0208203793,-0.9721820354,-0.8007465005,0.7475744486,-0.1014548764,0.1717854142,-0.8900861740,-1.1787598133,0.2836291790,0.6105752587,0.0714654922,0.2032693624,-0.0388820209,0.0560920835,0.2171260417,1.4609282017,0.3397671580,0.6166418791,1.0261284113,1.1398640871,0.0585809164,0.3839966655,0.1210960820,-1.4486052990,0.9515943527,1.1694599390,-0.4625164866,-0.5766818523,1.3717787266,0.4527533352,-1.0115367174,-0.2389606684,-2.1135818958,0.9200044870,1.2121772766,-0.0131268874,-1.0547879934,-0.4625411332,-1.1228022575,-1.9511053562,0.2523438334,0.0335213430,3.1744377613,-0.3097344339,-0.0796215609,2.3843185902,0.1680177748,1.1310538054,-0.2883731723,1.0993871689,-0.0557040833,-1.3803638220,-0.1418625414,-0.0994756371,-0.3607069254,0.1630599797,-1.8716735840,0.5809935331,-0.0887572840,-2.0139195919,-0.1107501611,-0.1171881333,0.0923445001,-1.1246478558,-0.0643767715,-0.6560503244,-0.0982639194,-1.1385906935,0.3597646356,0.7687802911,1.0907343626,0.9932415485,1.5794206858,-0.4556078315,-0.0845103040,0.1953143030,-0.4470531642,-0.3148096800,-0.5831922889,0.7251600623,-0.3664899170,1.1142308712],[-0.2529630065,0.1108265817,-0.0751002133,1.6626136303,0.1881306767,1.1572419405,-1.5092159510,0.4438718259,1.0644911528,0.0690576583,-0.5484440327,-1.1120855808,0.2062901407,1.7126512527,-0.6370053887,0.4516954422,-0.6429297328,-0.4736404121,1.9491024017,-1.1057304144,-0.0583164655,0.0027293719,0.0433787964,-0.9135349393,-0.7275012732,0.1881026775,-0.6704826951,0.3486644924,0.7248296738,-0.3045236170,0.4539546967,0.5046046376,-0.4356022477,1.3943544626,0.8857178092,1.4404188395,2.6176686287,-0.5996223688,0.9162171483,-0.5749520063,0.0330234617,-1.6783542633,0.2071019560,0.2668921053,-0.2316341698,2.5724213123,0.0771679357,0.0226416700,-1.0104405880,2.0703089237,0.2559503615,0.3081150055,0.4898057878,-0.8981673717,0.1386307776,-2.3951253891,-1.0491285324,-1.4085648060,2.2214100361,-0.0909142420,-1.2632048130,-1.3839141130,-0.4299320579,1.6684657335,2.2021830082,0.5257831812,0.5739874840,1.2139933109,1.1535638571,0.1974461824,0.5228608251,-0.4663076997,-0.8205826879,1.2096979618,0.4772752225,-0.2883346677,1.1037921906,-0.4656058848,-0.0572867543,0.1929015964,-1.2480163574,-0.2613046765,0.3618875444,-1.8665893078,0.4445092082,0.3430060148,-0.3016341925,-0.6712490320,-1.9647588730,-1.4762607813,0.6007431149,-2.1796936989,1.7869907618,0.4041797519,0.7283347845,0.2149192095,0.0791092888,-0.0756125897,2.0927040577,0.6142238379,0.6972085834,-0.2808259726,-0.6297846437,-0.6537150741,-0.6563082933,0.0635813102,1.1474578381,-1.3920365572,0.8224881291,-1.2620323896,2.2960383892,-1.5349198580,-0.7157036066,1.8768742085,0.6480322480,-0.0833342448,1.9466234446,-0.9626903534,0.1901564449,0.2450418174,0.6438478827,0.5414466858,-1.6009700298,1.2528609037,-0.5713541508,-0.1698438674,-0.0962642208,0.7907935977,-0.3675736487,0.2545646727,-0.1731839627,-2.0102469921,-0.0823436901,0.4109261632,0.9124290943,-0.5108228922,-0.1702895910,0.5690754056,0.1842122078,1.1614910364,-0.7973006368,-1.7424471378,-0.1686261892,0.9480429292,1.2066537142,0.9066698551,0.6930658221,0.3705970645,0.5162458420,0.6608072519,0.4866171181,-0.4750882685,0.9606438875,-0.1774173826,-1.1164638996,0.6565501690,0.2430785298,1.3326601982,0.2276160568,-0.7638772130,-0.6724875569,-1.3945460320,0.7054424286,-0.3470198810,0.7814444304,-0.9647708535,-0.8465443850,-0.5008096695,-0.5663548112,1.0299090147,-1.2219821215,-0.9223409891,-0.0413170531,0.7327345610,-0.4150656462,-0.3697193563,1.3278313875,1.1300793886,0.8595128059,-2.1058371067,0.4117957950,1.4069085121,-0.8631376624,0.9940280318,0.0828463808,0.6927403212,-0.5555388927,-2.1278247833,-0.2419712394,0.2147070467,0.8770046234,-0.8252259493,-0.3071250319,-0.4251047075,0.4736202955,1.4847452641,0.7876216173,-2.3725543022,0.8668217063,-0.5885546207,-2.0502035618,0.1267351955,-1.3493905067,0.6182358861,0.1972312331,-0.8721260428,-1.7464439869,1.3869823217,0.1516992599,-0.0960684121,-0.5976011157,-1.6084567308,-0.1104261652,-0.3266306520,0.9532612562,2.8422458172,0.7323229313,2.3837733269,-1.4173332453,-0.2441341728,-1.0598491430,0.0659141317,0.1451799124,-1.3646274805],[-1.3189326525,1.0868996382,1.1407871246,-0.7739912271,0.2757473886,1.0186146498,0.3553026319,0.0994194523,0.4954006076,1.3101518154,0.9655687809,-1.3013892174,1.6685962677,-1.6873700619,1.2311973572,-0.7829509974,-0.5541905165,0.2968224883,0.1373548359,-0.3030631542,2.0867037773,0.2923654318,1.0142391920,-0.3932260871,-0.0352618359,0.5864498615,-0.5295163393,0.6852313280,-0.1643852890,-0.3441419303,0.2067484409,0.1120802909,-1.8413914442,1.0845415592,0.7977465987,-1.6193389893,-0.3356162012,0.5115650296,0.1433262229,0.6185114980,1.3554399014,0.7320787907,0.7957426906,1.4062149525,-1.7205719948,-0.2503417730,0.5847280025,-1.2508269548,0.0660410076,1.6999821663,-1.6192164421,-0.5963765979,-2.1174936295,-0.8630749583,0.3964715302,-0.6267700791,-2.0068068504,-1.7333778143,0.3489642739,-0.3829937875,2.0327324867,1.9875861406,0.2677262127,-1.4474372864,0.3494439423,-0.8642144203,-0.5014429688,2.0740737915,-0.3882977068,-0.4317372143,-0.4059042037,1.0080896616,-0.6950193048,0.5184195042,-0.9137551188,-1.4409933090,-0.7854880691,1.8851081133,-1.7892971039,-0.8158497810,-0.3761253655,-0.2095122784,-0.7854520679,-1.0257658958,0.3675001562,-0.7535496354,1.1216030121,1.0277554989,0.1140085235,0.3061859906,-0.8787401319,1.1658949852,0.1283266991,-0.2367148250,0.6065468788,-0.2742420435,1.1453546286,-0.0845240206,-0.0997853056,-0.4292584658,-0.9366860986,0.7969179749,-0.0418951027,-1.9022171497,1.3403178453,-0.2650252581,-0.1395718455,-0.1130125746,-0.2644657791,0.8363052011,1.4173330069,-0.0139653347,0.0140969520,-1.3867497444,-0.2896922827,1.4394404888,0.0371060893,0.4729910493,-1.6448743343,1.9283530712,-0.0540869758,-1.5597012043,1.3833075762,1.4959639311,-1.2113410234,-0.8035944700,-0.0742011741,-0.0012471536,0.5972085595,-0.5049758554,-0.8977391124,-2.4309682846,-0.7694957852,-1.2447004318,1.8496736288,-0.1600045711,0.3851931393,1.6098812819,1.3233433962,-0.5853709579,-1.4895451069,-1.6313624382,-1.0789774656,0.1750666648,-0.8689029217,-1.2971252203,-0.1602051109,-0.4810235202,0.8413378000,0.4719519913,-1.4436874390,-0.4337729514,0.1231140718,-0.1029697135,-0.2789793313,-1.8605657816,0.9095398188,-0.3967422843,2.4552280903,0.7007195950,-1.0971454382,0.1571447253,-1.6993728876,1.1674585342,-0.6445943117,-0.3044758141,0.6822560430,-0.6443675160,0.2809943557,-0.8231162429,0.3418474495,-0.9646231532,-0.9446858764,-1.1312217712,0.2950759232,-2.0998363495,-0.9623675346,0.5983527899,-0.5706901550,-2.8149156570,-0.9644336700,-2.2830367088,-1.4362345934,-2.5027389526,-1.0116815567,-1.3881716728,-1.0808703899,1.1213512421,-2.9426026344,-0.8580161929,0.8869637847,1.3504574299,1.6489382982,-2.0368595123,0.4648270607,0.4125159681,1.5089983940,0.1617003679,0.2972085178,0.3435890079,1.2503364086,1.6973853111,0.5223636031,-0.6074130535,-1.4800554514,1.0115360022,1.2797082663,-0.9638424516,0.0238593630,0.7001688480,1.0566946268,-1.2123521566,-1.5452622175,-0.7135031223,-0.2995873392,1.3495112658,1.0317986012,-2.1518487930,-0.2608613670,0.4728642404,0.1432889700,-0.5136743188,0.8816134334,0.4160916507]],[[0.9742739797,-0.5911037326,0.5096975565,0.7329760790,0.1781499535,0.4407790899,-0.7507807016,0.9301936626,-0.1136841103,0.2096375674,-0.3831914961,0.1437592804,1.9191459417,1.1668577194,-0.0390432589,0.7552515268,0.7185825706,-0.1992563307,0.2191366404,0.4416317344,-1.7806103230,-0.5829799175,-1.6506224871,-0.3800404370,0.5495235324,-2.4511246681,0.2122054845,-0.6588183641,0.2381319553,1.4138728380,0.1893429458,-0.2653420866,-1.4341834784,-0.0134915356,1.0526467562,-0.4494713843,0.3941705823,0.4576940835,-0.6274823546,-0.4867767394,-0.3145232499,0.7940014005,0.2057987899,-0.9098162651,0.3779762983,0.4768439829,0.5508027077,-0.2443206012,0.3771245480,0.9358108640,1.0886226892,0.9422910213,0.5694215894,0.1278016269,-1.6078619957,0.9920114875,2.0334231853,0.8720038533,1.2157073021,1.0835859776,0.0771418586,0.9433466792,1.1806234121,1.7133400440,0.8349636197,-0.3409295380,-1.4217534065,-0.9918186665,1.3714574575,-2.8138613701,1.5928198099,0.0514569990,-0.3350365162,0.1681042314,2.0830357075,0.1723357737,-0.7525523901,-0.6203945279,-0.4310478270,0.8727289438,-1.6561784744,-0.0651366413,-0.2049668729,-0.6367900372,-0.4473701417,-0.4513550401,1.0316622257,-0.6272398233,0.8326088786,0.6830444336,-1.3253782988,-0.5727469921,1.1235693693,0.3231909275,0.7557680011,-1.1440552473,1.2230265141,0.4768500626,0.2638700008,-0.3666841686,-0.1768282652,-1.6205668449,1.7069422007,0.8102005124,0.3011316061,1.5844978094,1.3711355925,-0.9427987337,1.6457824707,-1.0328581333,-0.2671949267,-1.4802416563,1.9849010706,-0.5626236200,0.0849812925,0.2822172940,0.7228338718,-0.7783156633,-1.2473411560,0.2193592489,1.0145031214,-0.0329789929,0.2228203714,1.1445417404,-0.7094033360,-0.7841330767,-1.0221625566,-0.5994387865,2.6038806438,-0.0352037288,-1.0069522858,0.7614061236,-0.3151522279,-0.4787478149,1.0351151228,0.9559835196,-1.6696609259,-1.5952768326,0.8844750524,-0.1393707395,-1.9339787960,0.5813475251,1.3983411789,0.7789682746,0.6822609901,1.1423463821,-2.8080708981,1.2361961603,0.1793654710,0.2740286887,1.7551431656,0.8033042550,1.5408403873,1.2425165176,-0.3679214418,-0.4071923494,1.0675301552,1.0157678127,-1.6916062832,-0.0623279475,-0.0539634116,-0.8613557220,1.2769105434,1.3883987665,-0.0332604498,-0.3398702443,-0.6552860737,1.4018083811,1.0553525686,0.1255802065,-2.8219633102,0.5146217942,1.4022740126,-0.0046099331,0.8136485219,-0.9195903540,-1.3112980127,-0.4670576155,1.4047838449,0.5764878988,0.3580507934,-1.5716396570,0.6349464655,0.7771478295,1.0435512066,-0.3213168979,-0.8127188683,-0.2205226868,-1.2179155350,-1.9275535345,1.7975726128,0.5448293686,0.7044969797,-0.3992608190,-0.5087509155,2.1094746590,-0.7843719125,-0.2291737050,-0.7270941734,-0.7784538865,0.4416177571,-0.2448096573,0.8384055495,1.4899570942,-1.1972079277,0.3306650817,-0.5491546988,0.3423768878,-0.7401416302,0.6068357825,0.3473240435,0.3175152540,-0.0742286593,0.3980647922,0.7580665946,-0.0345872045,-0.8518533111,-0.4360183477,0.6210014224,2.7143337727,0.7998730540,1.5790160894,-0.4784846306,-1.3615520000],[-1.1311392784,-0.8737077713,0.4509058297,-0.9993168712,-1.9487090111,-1.0683714151,-0.6897867918,0.1995844096,0.6471304893,-0.7248669863,0.5458530784,-0.0195752960,-0.6879757643,0.0733593181,1.4149322510,0.6252916455,-1.2030237913,-0.8833094239,0.4984725118,-0.8347005248,1.1032197475,0.3863393962,0.4053649902,-0.3788195848,0.0324828625,0.3399371207,2.4221336842,0.5677464008,1.5449701548,1.0187703371,1.1051464081,-0.0787986144,0.7276761532,-1.7961450815,-0.7372569442,-0.4961169660,-1.8179187775,0.4991966486,0.4435710609,0.6528558731,0.0615191497,0.5835493803,1.1446419954,0.1317789853,0.4888918102,0.7580172420,1.5825773478,-0.4546769559,-0.4600296915,-0.2378547490,-0.0576866977,0.9007138610,-0.5783562660,-0.9360529780,0.8583307266,-1.1411458254,-0.2024374753,-1.1240649223,0.5279912353,-0.4751847982,0.6631527543,1.6318333149,0.0969477370,-0.4408459961,-0.4316680431,0.5758231282,-0.7765583992,-0.3135903776,0.1644244492,-0.9466973543,2.4841709137,-0.1300010383,0.9978062510,0.7827248573,2.0063924789,1.5243164301,-0.3948339224,0.0537491739,0.7989837527,0.2240976840,-2.4085738659,0.7977293134,0.8477253914,1.0057256222,0.8337087631,-0.4097576737,-0.4895608127,-0.6883573532,1.0892106295,-0.0431515090,-0.6405534744,0.6923466921,2.7715198994,-0.0121520748,0.4013367295,0.2561378479,-2.0776169300,1.7508301735,1.3922433853,-1.0740218163,0.5858798027,-0.8643900752,1.1791176796,0.5617208481,-0.2729750872,0.4421900213,1.9618458748,0.1082156897,-1.0202893019,0.6061573029,-0.9337441325,-0.6937928796,-0.6611673832,0.5429068208,-0.4369900823,0.4420260191,0.2651502788,0.1907901913,-0.1864834428,-0.6720966697,-0.1947799325,-0.4772402048,-0.3070857525,0.5616519451,0.4010860026,1.2816871405,-0.1345330328,-0.3634308279,0.2095918655,0.5743795633,0.5211598873,-0.3771933019,-0.5893455148,0.4899389446,-0.5980780721,-0.2535832822,0.0282106660,-0.7644257545,-0.4702759683,1.3134050369,-0.1494729519,-0.7538666725,0.0136112459,0.0777157620,-0.1643834263,-0.2342753261,0.8124720454,-0.6758717299,0.1944087893,1.5166828632,-1.3424445391,-1.7649502754,-0.8097494245,-0.0012850497,1.4531955719,-0.4727535844,-0.8327094913,1.0155029297,0.3293326497,-0.6450911760,0.3764901757,0.1045299768,-0.1445874125,-0.2936522067,0.9235664010,-0.2783232927,1.3137370348,0.0172006525,-1.3841036558,1.5561493635,-1.8639020920,1.0495033264,-0.5255575776,-0.1408940107,-1.9543750286,-0.1556380987,0.1417855322,-0.2631154954,-0.4426996112,0.5870085359,-0.5000515580,-0.5050081611,0.5463028550,-1.0903365612,1.5628612041,-2.7882788181,-1.3344719410,0.8784140348,-1.6840302944,-1.8166384697,0.0311815683,-0.4051390886,-0.0798487365,-1.5385483503,0.9227674007,-0.5343394876,0.0507932045,-1.4572614431,0.3584845960,0.3457713425,-0.5961431861,-0.6469150782,0.1584200561,0.4568893313,0.3769147992,0.4354852736,-0.5486419201,-0.2239340246,0.6698993444,-1.0490089655,0.5744723082,1.5347912312,1.5397765636,-0.1221618131,0.5470622778,-0.6007783413,-1.2336231470,0.9368577600,-1.1393072605,0.0572789684,0.5984082222,-0.1399062723,-0.9106681943,-0.8067533970],[0.1479307711,0.9215002060,0.3112674356,1.3075610399,-1.7026318312,1.3720930815,1.2918262482,0.5612526536,-0.9967388511,1.2426233292,-2.2951490879,-0.0152674960,-0.9043204188,-0.5141695142,0.4446947277,1.1492751837,0.0521705337,0.6528357267,-1.2749031782,1.0507358313,-1.0188709497,1.7837384939,-0.9868627191,-0.2265499085,0.2017863989,-0.9687949419,-1.9731079340,0.1696194410,0.5233901739,1.4001473188,0.4637394249,0.7948895693,0.0983602554,-0.4224130213,-1.2093416452,1.1387294531,-0.9182177186,0.5414411426,-1.1186860800,-1.8909265995,-3.0632636547,0.4568120241,2.0387444496,0.4237249792,-1.9272114038,0.9853472114,1.3389278650,-0.1318121105,-0.8920201063,-0.2773597240,-0.4517774582,-0.7616629004,-1.1646945477,0.9879109263,-0.3608299196,0.4053466022,-0.4165501297,0.3574571311,-1.2699836493,-1.6300295591,0.5598648787,-1.4674304724,0.4131439328,-0.4219754934,2.0304031372,-1.4413354397,0.4799681306,-0.0420383178,1.6915184259,-0.5232487917,-0.8039526939,-1.0601407290,-1.5404548645,-1.0749971867,-0.2996744812,1.3575615883,1.0603859425,1.0530307293,0.1001334339,0.3675707281,1.1156344414,-0.7672547698,-2.4473090172,0.3544388413,0.7223312259,0.8279894590,-0.6757613420,-0.1550412029,-0.0579219274,0.1946964562,-0.6854659319,-1.0218989849,-1.1483523846,-1.4241236448,0.5684677362,-1.2633242607,-0.5404099822,-0.6181235313,0.8761564493,0.2720177472,0.6743654013,-1.3917897940,-1.3827804327,0.0717442855,0.1175106540,2.1652770042,-1.1354564428,-0.6457980275,0.0317389071,-0.7400742173,0.9619795680,-0.0593260564,-0.6544432044,-0.1337867081,-1.2988967896,-0.3555890620,-0.6045878530,0.2226153016,0.9822761416,1.1462154388,0.2895379364,-0.4104592204,-1.1960166693,-0.1224415079,1.1972869635,0.6265248656,0.3610025048,-1.7972739935,1.3773232698,-0.6522130370,-0.7375943065,-0.4990181625,-0.3867849708,2.1639332771,0.1491455138,-1.3630967140,1.8294426203,-0.8609524369,1.4427920580,-0.4151449502,0.1516723335,0.2960756719,0.3308898211,0.4983998239,0.1312209517,-1.0052082539,-0.8411090374,-0.6049742699,-0.0483746789,1.0027732849,1.0322577953,0.2257779986,2.2114481926,0.2201044112,-1.3947371244,-1.0045163631,0.5725672245,-0.4024380744,-0.1606439054,0.5383403301,-0.9939112067,0.4404547215,0.4778611660,-0.7972090840,1.2639709711,-0.0904689953,0.0119346408,-0.2128958404,0.1294909567,0.6935775876,-0.8069289327,0.0381242223,0.0640896633,-0.8519039750,-1.1961863041,-1.6041916609,0.3708937764,0.4002925456,0.6246642470,-0.3795958757,-0.2372878045,0.5787289739,-0.6773391366,0.1989562958,-0.0726481527,0.6693786979,0.5971757770,-0.1213830337,-1.7431809902,1.4139084816,-0.3425191939,0.2435627282,-0.0741600394,-0.9220610261,1.0343546867,-0.8678551316,-0.3198132217,-0.6899349689,0.4179919362,1.2124581337,0.0375941433,2.2486476898,-1.0220209360,0.6465313435,0.4278988540,0.9193848372,0.4222237170,-0.0933749005,-1.0670326948,0.7879496217,-0.9432157278,-0.9096699357,-0.7628371716,-1.6910816431,-0.1087954193,1.1553153992,-0.2835746109,0.9786539078,-0.1439840198,-0.5912532210,-0.7626441717,0.2139990479,0.8680096269,-2.1934850216],[0.7954643965,1.2878558636,0.3072288930,0.9573891759,0.5701214671,1.2529802322,-0.0815569162,0.8128978610,-0.9811226130,0.2701521218,0.9489287138,-1.9259961843,0.2753039300,-1.3883477449,-0.5270357132,-0.2019829303,-1.1113721132,0.1056974530,0.4965963960,-0.1771866232,2.4821672440,0.8626716137,-0.1475741863,-1.8463534117,-0.1793068349,-0.8907314539,-0.4722703099,-1.5586285591,0.1077410355,-1.9330302477,0.7558904290,0.2947430611,-0.6818034053,0.8157358766,-1.2121231556,0.3502351940,-2.2251851559,-1.0909916162,-0.0607460439,-0.7485980392,0.5062255263,0.2378816903,1.2893656492,-0.6041642427,-0.6925551295,-1.1607208252,-0.2876462042,1.0311911106,-0.0499792546,-1.3574689627,0.8249968290,-0.8970457315,-0.1135150492,0.2143322527,1.3763307333,0.8509597182,1.7496386766,0.6001273394,1.4015005827,0.3860958517,0.5434395671,-0.4725933969,-0.2877027690,-0.6129447818,-1.4196848869,0.3062418699,-0.1553191394,-0.9508764744,-0.0760941654,-0.8062614202,-0.8954066038,0.0484242700,-1.0906312466,-0.1732535660,-0.3397390246,0.3176231682,-1.5844293833,1.2997056246,-0.6607711315,0.1271229833,0.0425397344,-0.4639439285,-0.9103606939,0.3158217371,-0.3238605857,1.6766819954,0.9372931123,0.2473313659,-0.8097632527,0.0890621245,-0.8043243289,-0.3928592503,-0.3789885342,-0.7877340317,-0.7610335946,-0.4013801515,-0.0189523175,2.3299083710,0.8038778305,-1.9815123081,-0.7011249065,-0.9096357226,0.9332553148,-0.2004610598,1.7981244326,-0.0330081321,-0.2495200932,0.8726334572,-0.6514548063,-0.5128629804,1.6354423761,0.3905225694,-0.8390223980,0.8316834569,-0.0595287457,2.0173168182,-0.8562936187,0.6865690351,1.3743298054,0.3907682300,-0.2398041636,-0.1251194626,-0.6344032288,2.2980499268,-0.4630606771,0.2461500913,1.3341190815,-0.3485464156,1.2087879181,0.1016988084,-2.1994295120,0.1064224765,-1.8571059704,1.5093195438,0.1532788873,-0.5053347945,0.3831263781,-0.1220782623,1.1170195341,1.0353970528,-0.7229524255,-0.3387831151,1.6032934189,1.7609171867,-1.2161747217,0.5180673003,-0.6836446524,-1.6527055502,-0.8767384887,1.0462769270,0.2828881741,0.4827982187,0.2307805121,0.7823326588,0.9507107735,1.4595805407,0.6798090935,-0.8676077724,0.3908489645,1.0838125944,0.6227587461,0.0919146538,0.6267312169,0.7369835973,-0.4665488899,1.5379717350,-1.0313144922,1.0398963690,0.8687855005,0.2055855989,-1.7716643810,0.2428264916,0.1570801437,-0.8153182268,-0.2022062540,0.5724145174,0.5329394341,1.6678758860,0.5379707813,-1.2548235655,1.5401520729,-0.2270848900,-1.1637035608,0.7588675022,0.6385750175,0.2570756972,1.0582165718,0.3091140985,1.6547634602,-0.2367554605,1.2576045990,0.0599439889,0.0446613617,-1.8018302917,-0.2758919001,0.6777992249,-0.1838328093,-0.9692851901,0.9890301824,-0.2224184275,0.5610032678,0.7233167887,0.0912953764,-1.4654313326,-0.5956678987,-1.0447202921,-2.0769333839,0.9551025629,1.1798607111,-0.5978704691,1.7572724819,-0.1310936958,-0.2408575267,-1.5555117130,1.5630480051,0.4313979149,0.6554265618,0.0821268037,-2.1948366165,1.2969125509,0.8677711487,0.3639212549,-0.5787127018,1.5105884075],[1.0996694565,-0.5503646731,-0.4026169181,-0.0874640346,-0.4514757693,1.4189465046,-0.4797129035,0.3733192980,0.2657961547,-0.8370972872,0.3150520921,-0.9210009575,0.1015821472,-0.5309605002,0.0664992332,-2.0074801445,2.2297945023,0.0819370598,0.1479992270,0.1758866161,0.8668915033,-1.0901758671,-1.8159155846,0.5403104424,-0.1107598692,1.0183703899,1.9810978174,-0.4993706942,-1.5936735868,-2.0820102692,0.8839993477,1.2424896955,-0.3754470050,-0.4873703122,-1.3391138315,-2.8575394154,0.8945623040,-0.0517046452,1.1353893280,1.0172095299,0.1628982425,-0.3798680305,-0.8960040212,-0.2011121809,-0.0567418709,0.2119727731,0.6210333109,-0.1791775525,0.8862378597,0.8935353756,0.8979471326,-0.8102669120,0.0424867161,-0.6294496655,-1.0798614025,-0.2119414806,0.7185308933,0.8276394010,-1.5393027067,0.7995367050,0.6526834369,0.3812628984,1.6862120628,-0.5823438168,0.5357362032,-1.8718117476,1.8158414364,0.0115653146,1.0090109110,-0.0481402576,-0.0621399991,0.5543729663,-0.0510954261,-0.4064195454,-0.5600697398,-0.1420308799,1.2426292896,0.2519005239,-0.0725310817,-0.8719974160,-0.6856629848,0.0003807768,-0.1563270241,-0.4662569165,-0.0164176356,-0.3071641624,-0.4057198167,1.3780448437,-0.9845128059,1.2129591703,-0.2401970625,1.7031100988,1.1683377028,-0.9992445111,-0.8173805475,0.2220273167,-0.3547341526,0.4088989198,0.3500408828,-0.4819968045,0.3505795598,-0.8347722888,-0.6508687139,-1.4876539707,-0.2163732499,-0.2417284250,0.9192639589,-0.0174811445,-0.6125581861,-1.7423535585,-0.2791055739,0.3562209904,0.2949730754,1.1177265644,-0.5972783566,-1.7783836126,-1.1208477020,-0.2938118279,-0.4967626035,1.0432761908,-1.0701702833,1.0008829832,2.2375409603,-0.0449116081,-1.3642407656,1.4596179724,1.1632910967,-0.5697431564,-1.7598483562,0.0133777484,-0.4007295072,0.7930163145,0.9166504145,-0.4451917112,-0.3771822751,0.5663695931,0.6020166278,0.6256309748,0.3488479257,1.4128754139,-1.1541855335,1.3467293978,0.0763891712,-2.2180795670,-0.8790378571,0.7706360221,-1.0272171497,0.3823670149,-0.4660917819,-0.5811817646,-0.9728909731,-0.3998017013,1.3444324732,-0.2047507763,1.6922044754,0.7689399719,0.7740138173,-0.9281789660,-0.4968115091,-0.7952126861,-0.0446061157,0.2427589148,1.3154057264,-0.9638298750,-2.4491629601,1.8243825436,0.6550926566,1.3024812937,1.2062610388,1.5797829628,1.6227316856,-1.1122174263,-1.0028078556,-0.3198828399,0.6408913136,0.5411697030,-0.3132085502,0.0905704945,0.4082139432,1.0586953163,0.0793558359,0.2908940911,-0.4207540154,-1.0434504747,0.8331912756,-1.2780961990,-1.4766488075,-1.6978293657,-0.0705931783,0.2020439953,-0.0928261653,0.6194707155,0.0804834291,0.3376404047,3.4837553501,0.2669399977,-1.4626413584,0.8958411813,1.1550073624,1.0528649092,0.7435577512,-1.5690900087,-0.4255738258,0.9140344858,1.2361495495,-1.2471395731,0.7626679540,-0.1928639710,-1.2733954191,0.7811949253,-0.4019659162,-0.4094708264,0.0163625069,1.4163241386,0.2105273604,-0.4444972277,-0.5089505911,0.4650335014,-0.2184793502,-0.4025595784,1.0784583092,1.0624145269,0.0769374445,0.0645173937],[0.8455424905,0.9994160533,0.9331492186,-0.7005475163,-0.2368602306,-1.2239063978,0.2130108923,-0.5972399712,1.8470237255,-0.6240378618,0.5833202600,-0.5017367601,-3.0102710724,-1.3837611675,0.1786506176,-0.3187453449,-0.5781022906,0.0315280780,-0.4391703606,-0.2296233028,2.3488371372,0.3843484819,-0.5347214937,0.1268615276,0.8902928829,3.8520202637,0.7244234681,1.1944116354,-0.6678529978,1.0415666103,-0.8658940792,0.9077596664,-0.3378550708,0.9775423408,-0.0754776895,0.6063098907,0.6713734269,0.0194545370,-0.3463670015,2.1391818523,-0.4366911352,1.7306786776,-0.1830381155,0.2859286666,-1.4076973200,-1.0983570814,0.2282403558,-1.1998163462,0.4056214094,0.9875051379,-2.2728385925,-1.3111302853,-1.8743584156,-1.2082616091,-1.5854942799,0.4378720522,1.4147380590,0.3267107308,-0.5729110837,1.2912527323,-0.6939992309,-0.0833639130,-1.6856313944,1.0955225229,0.5547582507,-0.9104179144,0.4392534792,1.9463402033,-1.1000357866,-0.7706585526,1.2468438148,-0.4902793765,0.5345413089,-0.9500870705,0.1477532536,1.2505365610,0.0899634138,0.3677219152,0.8650166392,-0.2937146723,-0.1290635020,0.3401563466,-0.3108875155,-0.7258995771,1.3177022934,-0.9953907728,-0.0467900336,1.0885127783,0.1454632878,-1.2228176594,-0.7175816894,-0.7583139539,1.9346177578,-0.7771400809,-1.0648733377,0.1046056002,-1.6488226652,-1.0661364794,2.0642158985,-1.8892747164,0.1650791317,1.0846154690,-0.1468636394,-1.1351752281,-1.2594205141,0.4703114927,-0.3027814627,0.2559323013,-1.8508899212,-0.9306232929,0.9595783949,0.1062435508,0.8443471789,0.2098277062,-0.8347812891,-0.7564993501,1.0179579258,1.0825403929,1.3440455198,0.1753898412,0.4277119637,0.2193833888,1.4308446646,0.9249140620,-1.7949538231,0.0283557717,1.7641273737,-0.7250720263,0.6685691476,1.1260482073,-0.3413906991,-0.4857299626,-0.0957151726,-0.4632709324,-0.7183370590,-0.0102504687,1.3925209045,-1.1265658140,0.8746911287,-0.9904094934,-0.4190586209,-0.2849096358,0.6538196802,0.7931527495,-0.3543632627,-0.8418226242,0.6238386035,-0.7723533511,1.4631421566,0.1103246212,-0.8867618442,1.6526117325,1.2548710108,0.2880513072,-0.4933074713,-0.5117329955,0.6032633185,-1.3497583866,-0.5111608505,0.6675429940,1.8336050510,0.3924017251,0.5415292382,1.6077739000,-1.6158584356,0.5651822686,-2.9483938217,-0.1493530571,1.1534879208,0.2299067229,0.2548667490,0.3125708103,-0.1421936601,0.0962879658,1.4915516376,-1.4980111122,0.2542267740,-0.3468973637,0.4250290394,1.9612369537,-1.1135908365,-1.4034284353,-1.3214765787,1.4184826612,2.2413375378,0.5875048041,0.0437724106,0.7939956188,-0.6215597987,-0.5815342665,-0.0857137591,1.4495682716,-3.0346090794,-0.5904287100,-1.1807073355,-0.8728227615,0.1840069443,0.5226079226,-0.7583761215,1.5459980965,1.1350790262,1.2372657061,0.2346145958,0.9069361687,-0.1464601457,0.8803369403,1.6036959887,-0.0359401405,-0.5794182420,-0.8478991985,0.4720226228,-0.0823028833,-1.2614284754,0.4648005366,1.1294105053,1.5758064985,-1.3074091673,0.4784797132,-0.0898916945,-0.4759197831,1.2124656439,-0.2539686859,0.0967589170,0.0534103699],[0.2857500017,-1.3306634426,0.4249919951,-1.1950973272,1.0747199059,-0.2170769274,0.0450050756,0.1741701663,-0.2596285939,-0.7920688391,-0.4063050151,0.9191468358,1.7260661125,1.4962009192,-0.2412421852,0.3035558164,1.5768409967,2.0336244106,-1.6690231562,-1.3676552773,-1.2393416166,-0.5456511378,0.8899183273,-0.1883179098,1.1415015459,0.3300242722,0.6155939698,0.7084887028,1.5292983055,-0.3306114674,-0.0788572133,-0.4493339062,-0.2701256275,0.5901252031,1.5623044968,0.3643705249,-1.4534931183,0.5249486566,-1.2966890335,1.1808009148,-0.0224379860,0.3152523935,-0.2782708406,0.8091103435,-0.4187511802,-0.4770900905,-0.3365066946,0.5623541474,-2.0182147026,-2.1983170509,1.1045867205,1.2624795437,-0.0276180673,0.1718224883,0.1991921812,-0.2634349763,-0.0887101442,-0.8009012341,0.3470789790,-1.4324259758,1.4175089598,-1.6810934544,-1.0911394358,0.5939617157,-1.1652457714,0.7605130076,-0.6547212005,0.3845574260,0.7944753766,0.6379547119,-1.1523764133,0.1821886748,-0.5537084341,1.5448677540,0.8237330317,-0.5419313908,-0.7200653553,0.6935390830,1.1008996964,0.9097881913,-0.9642880559,-0.4383548498,0.8910411000,-0.3298941553,0.9730165005,1.2895667553,1.1915460825,-0.6843030453,-0.9302793145,-0.2390873879,0.1579615325,0.2212390304,-0.6818502545,-1.7127672434,1.1771396399,1.1114701033,-0.3492176831,-0.0472809970,0.7482308745,-0.0730901062,-0.5841656327,0.4504536092,-1.1025894880,0.0581932291,0.6327283978,0.1792763323,-0.8223001957,-0.2284287810,-1.2400608063,0.4701416492,-1.1249814034,-0.4419503808,0.4520464242,0.4384505749,0.4458945990,-0.4454752505,0.6874499321,-1.9018042088,-0.8928960562,1.0825363398,0.2178558558,-1.2581878901,-2.8035640717,1.6928415298,0.0167334545,-0.7488485575,-0.6693356633,-1.2117948532,0.1265929341,0.5291302800,0.3899676800,0.4554530382,0.4301682711,0.4348187447,1.0085790157,-1.7501940727,1.1373521090,1.1931743622,2.0794131756,-0.2773678005,0.8379659057,0.6542209983,0.2838761508,0.5350176692,1.3807713985,1.7053838968,0.8826751113,-0.3271561265,-0.3581827283,-1.2562394142,-0.8812279105,-1.0632889271,-0.1390643716,-0.7417252660,0.5785334706,-0.5842409730,-2.0249407291,0.0031048066,1.6786370277,-0.1028214321,1.4051040411,-1.2307188511,0.0745233521,-1.3036009073,-0.4741173089,0.7923251390,-1.6625601053,0.5009847879,-0.4260414243,0.4607705772,-1.4588941336,-0.8828140497,-0.2869668007,-2.0858445168,-0.6617110372,0.9500411749,-0.4903324246,-0.1242794171,-0.6876080036,0.9306303859,-0.2304113060,1.0719676018,-2.1756176949,0.5917387009,1.5272648335,-1.1067727804,0.3803062737,-2.1274337769,0.5029523373,-0.2445702404,-0.0006700782,0.9635633230,-0.2261878550,-1.6574515104,2.6968736649,-1.1250771284,1.0807653666,0.0584954619,-1.4129952192,0.8620643020,0.8713108301,1.6652963161,-1.6135631800,-0.1984029859,-0.2902413011,-0.6188423038,0.7531723976,0.1087239310,-1.6153539419,0.4767957926,-0.4382492602,-1.0736072063,1.2319749594,-1.2424856424,-1.4469009638,0.5112141371,-0.2780409157,-0.9318844080,0.4025871754,-0.7289595604,0.4709553123,0.1000782549,1.2074514627,-0.2669787109],[-0.3595815599,-2.1342899799,-0.4210056067,0.4573920667,0.9248502851,-1.2482802868,0.6569746733,0.4175840020,0.0368951969,-0.0607408360,-1.8906854391,-0.0682709813,-0.5059621334,0.2151026875,1.8872957230,-1.1071970463,0.1205267534,-1.0122386217,-0.6658754945,1.1404993534,-0.5657894015,-0.8201801777,0.6875778437,2.0678489208,-0.6186278462,1.0876693726,1.6158015728,-0.1567347497,-1.2611894608,1.4985532761,0.4219383597,-0.1459040344,-0.6523817182,0.7554470301,0.2956840992,1.1193044186,1.6107659340,-1.5123630762,-0.4555642009,0.7889720201,0.8956725001,-1.3317306042,0.6401042938,0.2166453004,0.8215140700,0.0503329076,-0.0965074822,-0.1113443226,-0.2042803764,-0.6873565316,0.5317636728,1.1151205301,1.5243155956,-1.2777767181,1.7654581070,0.0725174472,1.0041288137,-1.6742922068,-0.1915116161,-0.6583703160,0.0271707252,-0.7335672975,0.1892032921,0.2953191400,0.0094617000,-0.1575515270,0.3323207200,-1.2621731758,0.1978865266,1.3326668739,-0.6343193650,0.9134386778,0.0729104951,-0.1353257895,-2.0858786106,1.0808997154,-0.3004504144,-0.2755873203,-0.7367097139,0.3021540642,0.2700982988,0.4530409575,-0.7007525563,-0.8959960938,-1.1999323368,0.4935321510,-0.5960142612,-0.9639679790,-1.2851704359,0.4559888542,-2.5519254208,1.4413543940,-1.0547283888,-0.4198956788,0.2472599745,1.0853878260,-0.5223800540,1.9698237181,0.5240100026,0.7831386328,0.4234170914,-0.1564492285,0.1359184831,-0.1162117422,-0.4195275009,-1.1759042740,0.3552658260,-1.9931274652,-1.4269702435,0.6328967810,0.4862891734,0.0348587781,-0.0004265035,0.1926831156,0.8282247782,2.8950171471,0.4765744507,-1.0976687670,-0.7158638239,0.7778658867,-0.2204324454,0.0522998534,2.3419752121,-0.2337766886,1.5526149273,-0.2743647099,-1.2966957092,-0.5145607591,-0.6441264153,-0.9436242580,-0.5575792789,-0.2886804938,-0.5300575495,-0.7995063663,0.4609216154,0.0503422990,-0.1202928945,1.2408776283,-0.5033631325,-0.0448194630,0.2108783871,0.0507609658,1.4534044266,-0.9200549722,0.0032031226,0.6261848807,0.2485037446,-0.7718658447,0.2863176465,0.0988020748,-0.7075133920,2.0979690552,0.0536587685,-0.4469783306,-0.6916967630,-1.4713886976,-0.6406098008,-0.9751100540,-0.7251353860,0.7354019284,-0.0255986620,2.1644103527,0.1879548430,0.9535272121,1.1498577595,-0.9122163057,-0.8776962757,-1.5234766006,-0.9056487083,0.4926247299,-0.3897213340,0.4013664424,-0.1390452832,0.0777551010,0.8162872195,-0.3259987235,0.1420471519,0.1295469105,-0.5685203075,-0.0366480462,-0.7493227720,-0.0521788187,1.3544013500,0.9088275433,0.9768580198,0.5170670748,1.2871779203,-0.1010567024,-0.0856445506,2.6273643970,-1.6656155586,0.5624790788,-0.5161991119,-2.1735665798,-0.3417043090,1.9319416285,-0.9397265315,-1.7080259323,-0.7614494562,-1.2761473656,0.4220181406,0.9958624244,1.0256894827,-0.1235863641,1.8414309025,0.0351293795,-1.0325827599,-0.4244602025,0.1708007455,0.5898537636,0.2644017041,0.1715549082,-2.2165124416,1.0189716816,-0.5055457950,-0.5834990144,-1.2355498075,0.8014555573,0.3301530182,-0.8047369719,0.2689402997,-0.8028509617,0.8982989192,1.5976356268],[-0.2384607941,0.5498903394,-0.3153492212,0.0371826515,-0.2799673378,-0.2572603822,0.8187432289,0.8705870509,0.7939828634,1.4407110214,0.0046773427,-1.0408636332,-0.4254775345,-1.0005455017,0.1042390913,1.2375731468,-0.0690675676,-1.8301037550,-0.2057546228,-1.1653020382,-0.9878067374,-0.1361900568,1.1316952705,1.0342150927,-0.6592921615,0.2408573180,0.2436449975,-1.2514510155,-0.5721936226,-0.0520525165,-0.4139078557,1.8763142824,1.1753531694,0.5226545930,2.1683094501,0.6907484531,2.2160840034,1.1130344868,0.4593990743,0.7844545245,0.6997361779,-3.1261579990,1.5720311403,1.7266691923,0.3456651866,0.4389714897,-1.6385536194,0.4049912691,0.2712102234,0.1878780574,1.3390874863,-0.0299388077,-1.3081719875,0.0632864684,1.2408345938,-0.9132910371,0.5147520900,1.1721259356,0.0147358524,-0.3792796135,0.8482502699,0.9654959440,-1.4460035563,0.4597578347,0.1704647690,-1.1115249395,-0.7208780646,-0.3793327212,1.2741154432,-1.6364229918,1.5149157047,0.0631237701,-1.2959022522,0.4598662853,0.0281794146,-0.7976759076,0.3872869909,-0.8583239913,0.1689493805,1.5861971378,-0.8286434412,-0.9093782902,0.0804205164,0.0313321017,-1.8652266264,-0.3101404309,1.6558412313,0.4230325520,-1.1583219767,0.9033274055,-0.4117244184,-1.0604335070,-0.5190912485,-0.5632234216,0.1435869932,-1.5192936659,0.0426138528,1.4924136400,1.2707833052,1.9864619970,0.5108605623,1.2856057882,0.8407601118,-2.0035340786,0.8936482668,-0.3260267377,0.8181855083,1.0708587170,-1.8654093742,-0.4303693771,0.3521697521,0.1649127901,0.2585572302,-0.7880253196,0.2695698440,-1.3194481134,-0.6205232739,-0.0030871087,-1.2652636766,0.5406863093,0.4429259598,0.4298315346,-1.2147737741,-0.1896856129,-0.4850975573,-0.3081047237,2.3780131340,1.2015973330,0.2664550841,0.7288299203,0.7034993768,0.0048138006,-0.3591594398,-0.7174685001,1.7085003853,-1.3032776117,1.1584476233,1.0995483398,0.5414423347,0.9730444551,-1.0727057457,-0.5748537183,-1.3290205002,0.7744051814,1.4045319557,-0.2159052491,1.7948094606,1.6460920572,1.3411537409,-0.4496222734,-0.5609518290,-0.0992444083,-0.0853365064,1.2166137695,-0.0213507153,-1.4752011299,2.1615161896,1.2456383705,0.2582988143,0.2717613876,-0.1682876796,-1.4384615421,-1.0607285500,-0.1907466799,1.6710277796,1.2433642149,-0.7574949861,-0.0647511855,-2.9306302071,1.5923978090,0.0510240272,0.3966749310,1.0205726624,1.1665779352,1.6779688597,-2.3571422100,-0.5891851187,0.3185988665,1.8541604280,-0.0838108212,-0.4632383883,-0.1345472634,0.8540049195,0.7585686445,-1.0865671635,-0.8296186924,0.4694327712,0.3256563544,1.4290058613,0.7655954361,1.2069232464,-0.1459943205,0.8065757751,-0.3280249536,-0.0742716119,-0.4548668861,-0.0853952095,-1.1056786776,0.9523906708,1.7686954737,-3.2922542095,-1.5821778774,-1.3896070719,-1.2549228668,0.3908016682,0.6844573021,1.4914137125,-0.4642134607,1.1045403481,-0.4899351001,-0.9562233686,-0.2807927430,0.0053708674,1.1767382622,0.1583883911,0.2319228053,-0.8078649044,-0.1561281085,0.3615372777,0.4036455154,0.9082906246,-0.0023141834,-0.0452877991,-1.5662910938],[-0.1033781320,0.5418262482,-0.9377847910,1.0195208788,-1.1293481588,0.0335480943,-0.4405394495,-2.2315773964,-0.4775855541,2.0507366657,-1.3063079119,1.6552379131,-1.1819411516,-0.0324509963,-1.4311203957,0.8767514229,-0.1345372945,0.8536100984,1.5587157011,-0.2368998080,0.1880593002,0.2717109025,0.6395956874,-1.2362473011,-0.0319846682,-0.8295824528,-0.2055542320,1.5347025394,0.7827878594,1.2703878880,2.7300498486,-0.2844167948,1.1753647327,0.4245087504,-0.5369454026,1.2073285580,0.0930977017,-0.5196133256,1.0421657562,0.1690351069,-0.4875896275,0.6770972610,0.0044279699,0.3105448484,-1.2954131365,1.3982955217,1.7429810762,3.6478598118,-0.7394582629,-1.1202218533,-0.4658226073,-0.9041526914,1.1250499487,-0.1801972389,0.1294232458,0.7745076418,0.5126174688,1.4980577230,-1.0465172529,-0.4988512099,0.5188845992,1.1863158941,-1.1096326113,0.5208597779,1.2098367214,1.0017185211,0.6673674583,0.5668189526,-0.0521996841,-0.8716872931,0.1681198180,-1.2476809025,1.9259685278,-0.5453615189,-0.8922271729,1.9886724949,-0.3135860860,0.7334384918,0.5198380351,-0.4526397884,-0.9506946802,-0.2661958933,-2.5570335388,-0.0557328612,1.1625821590,-0.4434820712,1.1122316122,-0.0545103103,0.2586460412,-0.6618434787,1.4436861277,-1.3688731194,0.7621035576,-0.2985791862,-1.0211650133,-0.1357472390,1.0373653173,0.3776987195,-0.4136635363,-1.6953018904,-0.1204522103,-1.1567699909,-0.7138453722,-0.3276610970,-0.6435422897,-0.7366944551,-0.1425680071,1.7642153502,2.8425951004,0.0610087626,-0.4151884615,-0.1082942411,0.0857538283,-0.8597403169,-0.9152332544,0.1713453978,0.9422041178,-1.0082613230,-0.7561030388,-0.3161784708,-0.1782092303,0.0957031474,-2.5437932014,-0.6769134402,-1.4552763700,-0.1227465048,0.4310432673,-0.9728448391,0.3342837393,1.1829782724,-0.0065461015,0.5665283799,0.1695615500,-0.0044607525,0.0285913981,-0.8989881277,1.1593964100,-1.4732522964,-1.4600700140,1.7094210386,0.0207724255,2.0963575840,-1.2747777700,-0.6057670712,-0.8993343115,0.2116632611,-0.1160674915,-1.2055330276,0.7836779952,0.5415595770,1.0465161800,-1.6241914034,-0.8055034280,0.5579012632,0.8232837319,1.6447900534,-1.2311408520,0.2137642205,-0.9106723666,-1.9084692001,-0.2605872154,0.2076506913,-0.9598652124,-0.9803130627,1.2351433039,-0.9859101772,-0.3903642595,-1.0345238447,-0.3124050200,0.7462546229,-0.5244422555,-2.4146971703,-0.1631130129,0.5022974014,0.9429028630,-0.8194223046,0.0069113285,-0.2531582713,0.2928455770,-0.6136545539,-0.3634734154,0.8031544089,0.0733277798,0.5689120293,0.9162364006,-1.4241131544,0.6944862008,0.1126557738,0.2768815458,0.0754733682,0.4862255752,0.3217608333,0.3620449007,-1.7405321598,1.1516690254,0.9695023298,1.0705699921,1.1136047840,0.7925577760,-1.7781143188,0.8929021358,2.0690708160,0.5849423409,-2.1492705345,0.5832387209,-1.0383287668,-0.4568869174,3.1900093555,-0.7241430283,1.1837641001,-0.1022715420,1.8591374159,-0.3411387801,0.3150487542,0.0275884047,-1.3723717928,0.0300342757,0.4547230303,1.8566195965,0.2995341718,0.6253018379,0.4939307272,-0.0265730321,-0.3234100342],[0.3595822155,0.6894904375,-0.6438739300,-0.9093788266,1.2942886353,1.2176066637,-0.9077116847,0.1113884673,1.0686095953,0.1878102571,-0.5059827566,-1.6027270555,0.0214889795,-1.3829032183,-1.7231186628,-0.3605511785,0.5595287085,-0.4247933030,-0.3993293941,0.0908166617,0.1786594391,0.7535029650,0.6296449304,-1.5375419855,-0.2698430121,-0.7669034004,1.2986503839,-0.5635685921,0.1221668199,-0.3815145195,-0.4466152489,-1.1692444086,1.6965802908,0.3721182048,1.7709367275,-1.6176935434,0.2316016853,-0.8005692959,0.7475548983,-0.8234938383,0.7365947962,0.7701295614,-0.9460060596,-0.4202794731,0.3530687392,-1.8267436028,0.2953773141,-0.6090833545,-1.0919172764,-1.6189069748,-0.0322289020,0.4412971735,-0.4813476503,-0.6603649259,-0.8035909534,-0.8457508683,0.2187346816,-0.9387763739,0.0936134532,-0.0921785682,2.2268385887,-0.4825090468,1.5371801853,1.9999849796,-1.3969053030,-0.0445936732,-1.4784121513,-1.8323581219,0.3004155457,-0.3889199495,-1.3225893974,1.8505661488,0.0649111494,1.6666123867,1.4079970121,0.2198562622,0.4401271045,-1.1840212345,-1.4271931648,-0.7065832615,-1.2413952351,-1.3692640066,-0.0096538514,-0.6498388648,-0.2452564836,-0.9085732102,-0.1391163319,-0.8336672187,0.5473741293,0.8369570374,-1.1456743479,0.0624656156,1.8641681671,1.6749701500,-0.5735353827,-1.4077433348,-1.9448838234,0.5631722808,-0.6609920859,0.2506439686,-1.1130369902,-1.0673143864,-0.3806003332,1.4953833818,0.6200692058,-0.6979174018,-3.6335635185,-0.8773848414,1.7083442211,-0.7415100336,-1.0148513317,1.2024078369,0.7076079249,-1.2916729450,1.0737441778,1.0685819387,-0.5960540175,1.1803132296,-0.0333345570,0.0980365053,-0.4779715836,-0.8072968125,-0.0824001506,-1.9273227453,0.7178004980,-1.0592756271,-0.9261478186,0.1021981537,-0.2795476615,-0.9590332508,1.0488163233,-0.4690090716,1.6720734835,0.6879519224,-0.0157241132,0.5268683434,-0.5965974927,1.6848707199,-0.8259072304,0.7594338655,1.2602584362,-0.6720221639,-0.0584140681,1.0932010412,-0.5520174503,-0.6187421083,-0.8900236487,-0.8719530106,1.2347486019,-1.3540980816,-1.2299486399,0.6464612484,-0.0974468589,-0.3091480434,0.5283569694,1.0791188478,0.2561168075,1.4158717394,1.7008452415,-1.4289003611,1.3387390375,1.1170512438,0.5130531192,1.0400758982,-0.6153922081,-0.1668823063,0.7639955878,1.0470120907,-0.6455356479,0.7287098169,-0.5354412794,1.2531111240,-0.4258435369,0.3980273902,-1.6516693830,0.5757321715,1.1960839033,-0.4860200882,0.3503985405,-0.0763770789,0.0167850349,0.8753983378,0.5204891562,0.8198471665,-1.4274868965,-0.5994745493,1.2439334393,0.8900793195,-0.3447135091,1.6019076109,0.4453103244,0.3134413362,0.2098031342,0.8220742941,2.1609876156,0.4291853309,1.9126983881,-1.1369169950,0.1236273199,-0.0741692111,0.1741669476,0.1106243953,0.4574111104,-0.7266279459,-1.6765234470,0.8124452233,0.4615250528,0.2150113583,1.4487680197,0.6443644762,-1.3313271999,0.6483889818,0.0551732592,0.0946426168,-0.5147324800,1.3383497000,-0.6886813045,-1.3321741819,1.4576708078,0.3417120576,0.1916114688,-0.0513381623,1.6174348593,0.7993416786],[0.1274137199,-0.8448427320,1.2690343857,-0.5826838613,0.0557822697,-0.7457270026,0.0534833819,-0.5319348574,0.5449818373,0.0617729798,0.1800281107,-2.3204801083,1.0729317665,0.3631948233,-0.4377925098,0.1569657326,0.5283293128,-0.5462140441,1.2347840071,0.5054156780,1.6532857418,-0.6099229455,-0.6667820215,-0.5278279781,-0.4208771884,0.4319642186,-0.1246592775,-1.0448354483,0.0785995051,2.6429173946,0.0422035158,0.2535940409,-0.9304233193,1.0765348673,0.3263046741,-0.6096877456,0.6794693470,1.5073816776,0.3643631041,-0.7833818197,-0.8042606115,-1.1636455059,2.1159131527,0.6903854609,0.1691145897,1.3395061493,-0.3655520380,1.3998910189,-0.4119296074,0.1565577090,-0.9600688219,0.1500866711,-0.4632772505,-2.9622406960,-1.1724597216,0.6507877707,-0.5097853541,0.3659832180,0.4810813665,-0.0095656291,0.4939456284,0.1545771658,2.0188348293,-0.3500584066,1.2580102682,-0.3938673437,1.1757705212,0.5819926858,-0.1100534201,-0.4838815629,-1.1713180542,1.0384311676,-0.4346498549,-0.6087762117,-0.5977138281,1.2044967413,0.5018853545,0.3364687562,-0.2894819081,1.2561717033,-0.1431188136,-0.0513663627,0.1945189983,-0.1383751035,-1.2002265453,-1.5160598755,-0.3494048119,-1.1222406626,-0.1564642340,0.1808167845,-0.3393794000,0.5907009840,-0.9833743572,0.9756684303,-1.3281390667,0.2585115135,-0.3627780676,0.8321187496,-1.8786463737,0.4968658686,0.0689388067,-0.0079110470,0.3008804023,-0.7525809407,-0.3469925225,-1.5925863981,-0.7068521976,-1.3115445375,-0.2572110593,0.1371918172,-0.6321007609,-0.4874186814,-0.0709755793,0.0640492663,0.3276489973,0.3932814300,-0.2793305516,0.4494642913,1.7821687460,-0.9390084147,1.5413779020,0.0079562068,0.1199712157,0.1818806082,-1.7790522575,0.0833725855,0.3141438663,0.2376887649,0.3440466821,0.7322938442,-0.6423465610,-0.9430661798,-1.0507161617,-1.1356643438,-0.9131414294,-0.1980059296,-1.0524215698,2.0543701649,-0.5051992536,-0.2961077988,-0.1411919743,0.2223303318,-0.5895858407,-0.5231840014,0.6206701398,0.8080673814,-1.3365075588,0.3356038332,-1.6119444370,0.8270612359,-2.3478622437,-0.1573795974,-0.8355489969,-0.1855920851,0.2819520831,0.1627157480,-0.4957902730,-0.2017339319,0.0587787591,-0.8486970067,-2.2888104916,-0.9550905228,0.0425533839,-0.8235985041,0.2843135595,1.3483312130,1.6403613091,-0.1185661405,0.2167840004,-0.4161785841,-1.7830762863,-0.5561192036,-0.0708618909,-1.0473921299,1.0332918167,0.1671882719,1.5324164629,-0.6358706355,-0.2964316905,-1.5015150309,-2.2901973724,-0.2640973926,-1.3396550417,0.7526706457,-1.1424431801,0.2186951339,-0.4216327071,-0.4880354106,0.5038363338,1.6515885592,-1.1511272192,-0.6911903024,-0.8230931759,-0.2474271357,-0.3428781927,2.3004641533,-1.2634423971,1.2604916096,-0.4096618295,-0.2799764574,0.7918033600,-0.6636201143,-0.0015233173,1.6005517244,1.3663774729,-1.7475005388,1.0692484379,0.2425226271,0.4911504090,0.6091113687,1.1200417280,0.6362381577,0.1359951049,0.1392289251,-1.5782338381,0.9691632986,-2.3665611744,1.0742968321,0.5516464114,-0.5444959402,0.8318801522,-0.7501646280,1.2486605644,0.3589930236],[-0.3600519598,-0.7901878357,1.5135924816,-0.2313470095,-0.7407914996,0.8029611111,0.2708649933,-1.6062647104,0.5937306881,-0.3154849410,0.1361830235,-1.0001647472,-0.0763395801,0.0344249457,0.3669148684,0.5293010473,0.0763758123,-0.2572541237,0.5138447285,-0.1288889498,0.6439459920,0.5794592500,-0.7824993134,2.2121942043,-0.3496154547,0.4119117856,-0.8420560360,0.4544038773,0.0109467581,0.0010534312,-0.2950676680,0.1750898808,-1.2973902225,-0.0623139627,0.2126978934,0.6877377033,0.3352945447,1.7457261086,-1.0843807459,0.7854700685,0.0600954890,-0.0855508447,0.7329173684,0.2756085098,-1.0048882961,-0.7781894207,0.8054320216,0.0727083459,2.3196663857,0.3006044030,-0.2822029293,0.4725848138,-1.5897241831,0.0941874236,-0.9508988261,1.4989129305,-0.3068813980,0.2385282964,0.4217751622,0.7946755886,0.7492550611,0.5127679706,-2.2465128899,-0.1663811803,0.5605801344,-1.8675581217,0.4418747425,-1.0016011000,0.7059368491,-0.0510153249,-0.2325836271,1.5391352177,0.9156982899,-0.8702345490,-0.5608828664,0.8970757127,0.3895307481,0.6618192792,1.2001011372,-0.1240015626,0.4234113991,-0.7949218750,-1.7483150959,0.1998164356,-1.5106359720,-1.1773667336,-0.6218593717,0.3076383173,0.5031841993,1.5083596706,-0.7131744623,0.2623642981,-0.0309111103,-0.1339277327,-0.1106061265,-0.1045916006,1.0869770050,1.6638536453,0.6696953177,-0.2361247987,0.9860504866,0.4088005424,-1.0221886635,0.0886315182,-0.4157384634,0.1207668409,0.4294110239,0.1377238184,1.2307782173,-0.1232055947,-0.8324245811,-0.3747373223,2.4342443943,-1.1951713562,-1.1546982527,-0.9615500569,1.2484122515,-0.3975964487,0.8206714392,-0.8452029824,-0.2450099438,-1.6266416311,1.1472618580,1.0578019619,0.1282176673,1.1876020432,2.5764918327,-0.0823692530,1.1622513533,1.2577518225,0.7237423658,0.4700427651,-0.7886118889,-0.4397766888,-0.8123567104,3.0631451607,0.5537109971,-0.0900671929,-2.6399536133,-0.1701684147,-0.6035872102,-0.6521043777,1.1507827044,1.2750959396,0.2453898042,0.3872298300,-0.8278653026,0.9271902442,1.6431338787,-0.4256422222,0.5522973537,0.1817890704,0.4363834858,1.0876022577,-0.4799385071,0.9566588998,1.3544300795,-0.1174629778,-1.2165155411,0.8368718028,-0.8922112584,-1.5480458736,0.0400244184,0.3244729638,0.0817489848,-0.9257849455,-0.7675449848,-0.7365167141,1.6259465218,0.6298605800,-1.4840712547,-1.4950245619,0.1297564805,-1.0976947546,-1.1410716772,-0.9285092950,1.0353862047,2.7001140118,0.9804233909,-0.3006408811,0.1535374373,1.3037381172,-2.0147192478,-0.1158716157,-0.6375645399,-0.1695705652,0.6505923867,0.6162424684,-0.8195439577,1.2143636942,1.0763442516,-1.8476772308,0.3398576081,-0.2474824041,-0.9045435190,-1.1834901571,1.6120089293,0.4015118480,0.3753013313,-1.0195120573,0.3743047416,1.5763468742,0.2818703055,-0.0415413007,1.1435390711,1.0312833786,0.5930593014,0.4299762845,-0.7077791095,-2.0817489624,-0.8573213220,0.8970362544,-1.1242753267,2.1089203358,-0.1735206395,0.7554202676,-0.6492454410,-1.1986908913,0.3451207876,1.0742040873,0.2218725234,1.5225878954,0.1871998757,1.1086134911],[0.1162764803,-0.2187245935,0.3414756060,0.1085516512,-0.7073196769,1.2486360073,-0.9194564819,-0.8338657618,-1.0484052896,-0.1623076200,-1.9373788834,0.5603187084,0.5427429080,1.5060900450,0.0874666423,-2.4002182484,1.3652335405,0.5865104795,-0.3230989277,-1.3929996490,1.1904447079,0.3914655149,-0.6465815902,2.4218268394,-0.6179955602,0.0971411094,0.0927094892,1.1953548193,0.6744587421,-0.6237062216,-1.4494575262,0.5976842046,-0.3370549381,0.0000538884,-0.9751046300,-1.4531168938,-1.2553899288,-0.5801687837,-0.0705595762,-0.6704121232,1.3072650433,1.5074343681,-0.1624190360,-1.0658824444,-0.6713882089,-1.2154396772,-1.0532267094,1.3715112209,-1.1906760931,0.8829891682,0.1461430639,0.3025677800,-1.0018360615,0.7597610950,0.4627263546,-0.9727967381,-1.8330810070,0.6842470765,-0.0457889512,0.0743883923,0.1160207540,0.4871888459,-1.3810641766,0.7513743639,-1.6535325050,0.2805307209,-0.6730520725,-0.6538075805,-1.0937461853,0.8920948505,-0.5652847886,-0.2217692137,-0.9390385151,-0.2413975745,-0.0277992953,0.4737479687,-0.0896343440,0.4736139476,1.7478946447,-1.3817198277,-0.5763424635,-0.7620200515,-1.2223913670,0.0114631774,-1.4424446821,0.1032015532,-0.4778788388,0.6391831040,-0.3346210420,0.7208385468,0.6453597546,-0.6174742579,1.3672366142,0.4193812609,0.2484990656,-1.2752428055,-0.4317915738,1.3278837204,0.4312328100,-1.5758079290,-0.8080574274,-0.0961645246,0.2024365366,-0.2257446200,-1.4323306084,-1.1698937416,-0.1322149932,-0.2605111599,0.1435681283,-0.3786023855,0.9958216548,-0.0179624446,0.3827536106,-0.9316131473,-0.9425467849,-0.8002999425,0.9446987510,-1.1967359781,0.5227084756,0.3323342502,0.1467415690,0.3252617717,-0.2421308011,-0.3801048696,1.5932736397,0.8452807665,-0.3431094885,1.2928600311,0.0758657232,-0.8354057074,0.6789767146,-0.0734707043,0.4803312421,1.3606332541,0.8671076894,0.9994120598,-1.1754306555,1.4058384895,-0.9871997833,1.0385127068,-0.4434279203,0.8501870036,-0.6956353188,-0.6379371881,0.8698224425,1.4933327436,-0.9885481000,-0.6133674383,-2.2802202702,0.9688157439,-0.2850471735,0.0442743227,-0.3362223208,1.1406686306,0.5380051732,0.6882110834,1.2516015768,-1.3449673653,1.0782428980,1.6350847483,2.2968215942,-0.7110384703,1.6386603117,-0.9126443267,-0.3877492547,-0.1043018326,-0.0090003163,0.6059634686,-1.0972509384,1.7331362963,1.3298485279,-0.6946768165,0.8652173877,-0.8342711329,0.3025682271,-0.8910675645,0.6248769760,-1.6545723677,-0.9224897623,1.7043474913,-2.1606175900,-0.7243913412,-2.4045393467,1.6853871346,0.8936536908,1.0344368219,0.1707872897,-2.0846495628,1.3154208660,0.2850298584,-0.3303071558,0.9252412319,-0.5945811272,0.2951497138,-0.4839909971,-0.2808298469,-1.2129616737,-0.2385444343,-0.9515303373,1.8470141888,0.4530581832,-1.3355108500,-0.1361455619,0.8572332859,1.6664037704,-1.7786149979,1.1624349356,0.4030810893,1.3191045523,1.9514727592,0.1954828501,-1.1131478548,1.8760913610,-0.4106545448,0.9871078134,-0.3514513075,-1.1491596699,-1.0029109716,-0.4375024438,-1.7376359701,-0.1112331674,0.3778693378,-1.2642121315,-0.9193267226],[-0.4944682419,-0.7798122764,-0.4271022081,1.0683219433,0.1328333765,0.7467706800,1.3215091228,-1.1968958378,-1.1443666220,0.6172171831,-0.0051761493,-0.2794991434,0.3744995892,-0.3662587702,0.4500087798,1.5653235912,-0.2852126658,-2.0067307949,1.3534057140,1.2254853249,-0.3875033557,-0.4426376224,-0.9592947960,0.6190398335,-0.3124010861,-0.5994002223,1.5445845127,-0.0053672073,0.9292419553,-1.4328645468,-0.9411212206,-2.3532216549,0.0921249315,-0.3608577251,1.1292972565,0.7657831311,2.4220712185,0.1606874019,-1.4992787838,0.2326702625,1.0695062876,1.6562827826,-2.1346623898,0.8291113377,1.9936954975,-0.0670990944,0.4760950208,0.6264278293,0.3247528374,0.6127083302,1.3447989225,2.1764240265,-1.0073144436,-1.2598137856,-0.3319583833,-0.9408110976,2.1974444389,1.1426124573,-1.1883490086,-0.6941516399,-0.8429222703,-0.4689262211,0.1896196008,2.4364590645,1.4495763779,0.9550628066,-1.3749413490,1.0702964067,-0.7423433065,-1.6166442633,0.3353677094,0.8143854737,-2.1510691643,-1.4398878813,0.4295713007,-0.6825832129,1.4495866299,0.0767971873,0.8710640669,-0.9394101501,1.2120714188,-0.1885681599,-0.6676308513,0.5666903853,-0.5110077858,0.8325109482,-0.9449007511,0.3893716335,0.0370208956,0.3440933228,0.0902620628,0.1622183025,1.9678138494,1.0434346199,-1.4853209257,-2.2796955109,-1.0654275417,-0.7173523307,-0.8523059487,1.6222326756,-0.6020047069,-0.1935931742,-1.0439174175,-0.8322221637,-1.7539211512,0.6675662994,1.5207853317,-0.7924110889,0.3303536475,-1.4821587801,-0.2574607432,-0.7888179421,-0.2782670259,1.1471936703,-1.1040791273,-1.1629147530,1.2700695992,-0.4843160808,-0.5253778696,0.8253301978,1.6041414738,0.3806031942,0.7163059115,1.0897123814,-0.9337714911,-2.2876377106,-1.0739610195,-0.3832927644,-0.1494626701,2.3470876217,-0.1703584939,0.3322209418,-0.8650518060,0.5197970271,-0.8209413290,-0.6846882701,0.6192402244,2.3423302174,1.3338751793,0.1373818070,0.7219576836,0.9726226330,-0.1536917388,0.9913292527,0.2540746629,0.1626318097,-0.3465512991,0.4595924914,1.2181417942,-1.0088903904,1.2518771887,-0.1303908825,-0.9561480284,1.8820406199,1.2069588900,0.3529352248,-0.8306714892,-1.4193843603,0.1155640781,-0.5738064051,0.6434262991,0.4984490573,-0.1118573099,0.7238074541,-0.5256049037,0.6390516162,-1.1183315516,-0.0976157486,-0.1260827035,-0.5548433065,1.7662122250,-0.2686588168,1.0087834597,-1.4268404245,1.1659404039,-0.9612586498,-0.3340024650,1.5359613895,0.1235429570,2.9805519581,1.0028824806,-1.0232834816,-1.4397224188,1.3039492369,-0.7188103199,0.5913882256,-0.9829553962,0.2619152367,0.0651016235,0.1422494650,0.0322833695,0.7933985591,0.3038003445,0.6963039041,-0.4510253370,-0.0787391812,1.2138137817,2.1563596725,1.1974829435,-1.3827234507,-0.5387663841,0.3749031723,0.3817621768,0.5126075745,0.7817907333,2.8445768356,-0.7148271799,0.1042771265,0.0861898214,-0.2078901976,0.1141170561,-0.3227742314,0.0504558496,-0.5519875884,-0.2047938854,-1.8481737375,1.2803936005,1.1041901112,-0.1439001858,-0.2400241643,-1.0030885935,1.1916694641,1.3330214024,0.0078061824],[0.0833170936,-1.3876982927,-1.3872941732,0.8939967155,0.2273259908,1.3578511477,-0.1441743076,-0.7695208192,-2.5133471489,0.1119075045,-0.4572762549,1.0314948559,0.9591494799,0.3220496476,0.1193715185,-0.0206195451,0.6939818263,0.0383000039,1.0197111368,-1.3477758169,-1.8404810429,0.8510962725,0.8117278218,0.0951100439,0.6611233950,-0.7656561732,-0.9922797680,-0.9179639220,-0.3361779451,-0.5715629458,0.5896957517,0.1407307684,0.7236926556,0.7947764397,0.4836243689,-1.3869879246,-0.1118176356,0.2992152870,-0.4100322425,-0.7881225944,-1.8818029165,0.9634225965,-0.0309839323,0.8337911963,-1.9074735641,0.7431362867,0.4745285511,-1.1356483698,1.0912343264,1.2545942068,-1.4796434641,-0.4746834636,0.3405517042,1.1815891266,-0.8854237795,0.0957699418,-0.2880924642,-0.0463199876,-1.3265682459,-0.1858589202,1.2988238335,0.6762433648,-0.5991950631,0.3327338696,-0.5831449628,1.4332697392,1.9494212866,-0.7886651754,0.5240651965,0.1793854237,0.6290607452,0.4811582565,-1.4723852873,-0.0745147914,0.5363513231,-0.8404342532,-0.4846772850,-0.9714007378,-0.2141099870,-0.2834431529,0.1940501779,-0.5316337943,0.9639087915,1.6239774227,-1.3215353489,0.9414067268,0.7094277143,0.8003622890,-0.8477427363,0.9165262580,1.6716412306,1.2298988104,1.4809641838,-0.4254212081,-0.4077859819,0.3305155039,1.0415029526,-1.1496744156,0.1712993681,1.4701676369,0.4293683171,0.1795604676,0.0516814254,0.2441919297,-0.3086892366,1.1564617157,-1.2884559631,-0.7572581172,0.6286239624,0.5937031507,-1.1899099350,0.3375867009,-0.4291598201,0.5017452836,-0.8552255034,-0.8198128343,1.4330694675,-0.2992851734,-0.4924618900,-0.1553115398,-1.0980613232,-0.1505965292,-1.7112737894,0.5384337306,0.9439564347,-0.4357626140,0.5460928082,-0.8451873064,-1.3298441172,-0.5760731101,0.2821652591,0.7656444311,-0.0708712041,-0.7376880646,0.7219296694,-0.7072268724,-0.2859232426,0.6533915401,-0.8342830539,1.1105629206,1.6008365154,-0.6448624730,2.8344218731,0.6748697162,-0.3423575163,-1.4236814976,-1.1950898170,-0.5678108931,0.7195250988,0.5357619524,0.4457416534,-0.4799274206,-0.1652086526,0.1340560615,0.6014035344,-1.3409739733,-0.5987482071,0.0769548640,-0.3581912220,1.1240202188,-0.2238033116,-1.4019105434,1.6488307714,-0.2175414711,1.9418623447,-2.5849790573,-2.4083001614,0.3356060684,-2.0707116127,0.9840598702,-1.2379816771,1.0672813654,0.2889554203,-0.0074083852,0.4752945006,0.1841205209,1.2986530066,-0.0591709130,0.1742814332,-0.0644250885,-1.1764115095,-1.7949217558,0.7206540704,0.4559027553,0.3954004347,-0.7530158162,-1.8091009855,-0.3017390668,0.2449345738,0.7457346320,-1.5417525768,0.8053346276,-0.5716369748,1.6070737839,-1.2270644903,1.9155560732,1.0043300390,1.8278682232,-0.6639636159,0.6661688089,0.4428050816,0.0892480984,0.2309782803,0.8315519094,0.1459945887,2.3326561451,0.5832629204,-0.2406825274,1.9596458673,-0.6441795826,-0.8311957121,-0.9647058249,-1.5046192408,0.5703581572,-0.5187826157,1.4320012331,0.2922801673,-1.2171894312,-0.6377372742,1.4113386869,0.5109692216,-1.4242004156,-2.1111421585,-1.6401022673],[0.3583278954,-0.0144035704,0.8650972247,-0.0165269319,0.9271504879,1.5814197063,-1.2962620258,1.4669339657,-1.8920297623,-0.5688353777,0.1828519851,2.2016677856,-0.1028954163,-0.9203289151,-0.1486750543,0.3816725910,0.4249047637,-0.5387316942,-1.5233194828,-0.8753087521,-0.2341729850,-0.2465784848,0.0562043898,0.1882509887,-1.0140819550,-0.2454162836,0.8557434082,-1.3931586742,-0.4529600143,-1.4558101892,1.3878477812,2.2907056808,-0.0797444135,-0.4271253645,-0.2662831247,0.5487980843,-1.0244842768,-0.8371625543,0.8822185993,-0.9598054290,-0.0063711633,0.8817792535,-0.8519062996,0.0839718059,0.6059336662,-0.8637058735,0.5584779978,-0.0299070161,-1.4362994432,-2.0078036785,-0.2094628215,-2.6569430828,-1.1245808601,-1.3522570133,-1.3392878771,0.2719299793,-1.0510017872,-0.5297577977,0.0293570850,0.3279094696,0.9607840776,-0.0345142111,-0.0836286321,-0.4210097790,-0.3227880597,-0.4815520048,0.4784446359,0.8643121123,-0.4473959506,1.6125975847,-0.3212073445,0.2659475207,1.1676646471,0.5559692383,0.7779700160,0.6278859973,-0.2657962441,-1.6537842751,0.5440248251,-0.3830282688,0.7568864226,-1.9953464270,0.2166253179,-0.1360926181,0.3883821666,1.4031965733,0.1071888581,-1.0993431807,-1.4928320646,-0.0251265056,0.4121911228,-0.7856203318,-0.3717033565,-1.3687229156,1.2271392345,-1.2881999016,-0.6959694624,-0.8874993920,1.4558117390,0.3226454854,-1.3846375942,-1.0860627890,1.0846112967,0.4653168321,2.5614218712,2.7988910675,0.9378732443,0.0436836891,-0.2922166884,1.7467682362,-0.9949976802,-1.1876443624,1.5535912514,-1.5948337317,1.3830382824,-1.7654812336,1.8867820501,-0.0235525575,1.3374916315,2.3650345802,0.6751262546,-0.6060769558,0.0485105924,-0.1316827983,0.0151644247,0.4798516929,-0.7473517060,-0.9493153691,-0.5720299482,-0.1864403486,-0.9693927169,-0.0035792037,-1.6661548615,1.3482993841,2.0277435780,0.3658308387,-0.8208331466,0.4718213677,0.6236744523,0.4980045259,-1.3271654844,1.3967874050,-0.1287742406,-1.1586636305,-0.2044127733,0.2533079088,1.2015191317,0.1085055918,-1.5884567499,1.0797991753,-0.5118536949,1.7802431583,-0.3157459795,-0.2945337296,1.5226759911,0.0407008827,0.9528471231,0.7287182212,-0.7189680934,-0.0266107712,-2.0186710358,1.2203032970,1.1047537327,-0.8241505027,0.8184968233,-1.7929810286,0.9446066618,0.0339597128,-0.3622136414,0.4354104698,-0.4571644366,1.4052172899,-0.2578752041,-0.1565036923,1.5309950113,1.0972181559,-0.2565720379,-0.4348123670,0.0947441384,0.9954605699,-0.5722588301,0.3652120829,0.5728793740,0.1401024908,-0.1164690852,-2.0019223690,0.2620675564,0.8731961846,-0.0032438957,1.2527974844,-1.0324943066,0.9453257918,-0.8272736669,1.6740643978,-0.9105581641,-0.3780149817,-0.0894450843,2.1181685925,-0.4071816206,0.7425952554,-0.7459664345,2.4063830376,-0.4379978478,0.9473155141,1.5492707491,-0.0160185508,2.9175112247,-0.3762186766,-0.8467391729,-0.0670067072,0.0901584849,-0.4757851958,0.0956222340,-0.3810622394,-0.9159494042,-1.2364341021,-1.1209826469,0.8402428627,1.3614509106,-0.6751222014,-0.3164610267,0.5810042024,-1.0004385710,0.6155641079],[0.8530336022,-0.6594149470,-0.4928688407,0.7073107362,-0.4533206820,-2.0503511429,0.7237859964,-2.3081104755,0.7931211591,-2.2254080772,-0.5442180037,1.7981696129,-0.4281713963,0.3618839681,-0.5407407284,-0.2953153551,-0.2099083960,1.0288715363,0.8415134549,0.2290135175,-0.4205887914,0.0023723538,1.2285922766,1.1784408092,-1.0734616518,2.3597648144,1.1720961332,-0.6565869451,0.7126896977,0.1096705943,-0.0268855970,-1.7138684988,0.7106988430,-1.0446763039,-1.2062015533,-0.3407447040,1.3466372490,0.6082682014,1.3727915287,-0.0571841560,1.0773525238,0.2110953927,-1.5239913464,0.6126305461,0.6644054651,-0.9984656572,1.3315540552,-0.3504710495,-0.1185619310,1.3879070282,-0.9998387098,0.5702096224,2.1872117519,0.7356611490,-0.3437471986,-0.6454073191,0.6962169409,-0.4820179045,-2.4766716957,0.9680387378,-1.1541639566,-2.0597898960,0.1503677368,-0.7456140518,1.4302666187,0.8453559279,-0.9875950813,1.8884847164,0.7621361613,0.6842812896,-2.5535175800,0.1136048585,-1.4359040260,-0.9684360027,-0.2280776203,-1.2567373514,0.4444446266,0.6932767034,0.5163866282,1.1237536669,-1.4660167694,1.9476058483,2.8177776337,-0.6734972596,0.5416264534,-1.4821751118,1.6413025856,-0.6007871032,-1.6072694063,0.3414277434,-1.1667805910,-0.3343241811,-1.9332513809,-0.1265669763,-0.1862681210,0.1386926919,-0.4867524207,0.4340206087,0.0739651024,0.9674845934,0.8658832312,1.0888012648,-1.3989909887,-0.0956165418,-1.4477106333,0.6629640460,-0.2317875922,-1.2718285322,-0.9693523049,0.5948191881,2.1844816208,0.5021012425,0.0804309994,1.8132755756,-1.8225727081,-0.8481187820,0.2896516323,0.6112565994,1.3894894123,0.8772185445,0.5663446784,0.7064805627,0.5084221363,0.0819462836,-0.4970123470,-0.6747379303,1.2025511265,1.6175380945,-0.8847118020,-0.4290374517,1.5003818274,0.4689340889,1.6352735758,-1.1227899790,0.8453468680,-0.2696839273,0.7648316622,1.3724489212,-0.1384429038,-1.1072838306,-2.4854152203,0.1164309531,-0.0418152809,-0.8732971549,-0.3142679930,0.7281127572,-2.0499973297,-1.5354900360,-1.2159559727,1.0636960268,0.0742390528,-0.4086834490,0.6344577670,-0.0591966920,-2.1613221169,-0.1440149993,0.3146380484,-1.0171301365,-1.6388372183,-0.9077087045,-0.2367976457,-1.8067845106,0.7331464887,-1.2680211067,-0.5259135365,-0.2642040253,0.4714785516,0.4978014529,-0.4443696141,0.4366762042,0.9659940600,1.0915999413,-1.3117097616,-0.4258576334,0.2191979438,0.5577743649,-0.3677088618,-0.2587479949,-0.2951894403,-0.1285194010,-1.7410380840,0.2037728727,-0.0169256441,-1.9258970022,1.1891771555,-0.1056173444,-0.2551109493,0.0803601369,0.3078361452,1.4552494287,-1.7418527603,-0.1786758006,1.3843970299,-1.8449815512,0.2047158927,0.7096581459,-0.2328862250,-0.3915565908,-0.6430918574,1.1635156870,0.7798740864,0.3655540943,0.2442674339,0.9248095155,1.3240067959,-0.3801167905,-1.3439861536,-0.6244315505,2.1410539150,1.2676112652,-1.6609078646,-0.8470503092,1.0326119661,1.1942317486,-1.9800231457,-0.7131582499,-0.3807858229,-0.8833439946,-1.1810083389,-1.3195772171,-0.0540605076,-0.0507104397,-0.0006503842,0.2843582630],[-0.1849077493,-0.9837927222,-0.4980651736,0.5068479180,-0.0115457019,-0.1425283551,-0.1853843629,-1.3138334751,0.6789863110,-0.1799073815,0.1523042619,0.8186873794,-2.3586330414,-1.1386350393,-1.8707106113,-1.3997253180,-1.6075724363,-1.7040657997,0.5580219626,0.8402561545,-0.4480331242,-0.8665820956,1.6506772041,-0.8412654996,-0.8535141945,1.0686321259,-1.8647556305,1.6807284355,-1.4227255583,0.7940302491,1.1190499067,0.1854119748,1.0127215385,0.4834828973,-0.0771656111,-0.3799327314,-0.0321431160,0.4283321500,-0.2940807045,0.4353268147,1.0631655455,0.1714275032,0.8127216101,-0.4509384334,-0.1770134270,-0.6772844195,0.1549729407,-0.0158453472,-1.1719727516,1.0151515007,0.5892142057,0.7689551711,1.2890911102,-2.1213917732,0.7964041829,-0.7404226065,-1.7036547661,-0.2855556011,0.8033329248,-0.1386539191,0.5521915555,-1.6757645607,1.1778060198,1.4586355686,1.4529005289,-0.1698000878,-0.6933089495,-0.2682672143,-1.6710566282,-0.0512348413,0.6260530353,0.7085033655,0.7272201180,0.3297210336,0.3958292603,0.2138654739,-1.0701493025,0.9027658701,0.6664137840,-0.4717260897,-1.6510242224,-1.2329187393,0.3288979530,-0.5451109409,0.5936804414,1.3355246782,0.1301572770,-0.2022518814,-0.8950264454,1.2793928385,1.5864177942,0.4107882977,-0.0241710469,0.5844276547,0.5538517237,-1.6992001534,0.6862621903,2.2658751011,0.8761395216,0.3837254047,-0.2647534311,0.4720512033,0.1841666549,0.2001533508,0.0668039396,-0.1149130762,-0.4432745576,1.1688767672,0.3746031225,1.0524545908,-1.1692347527,-1.2809635401,1.3166548014,-0.6610518098,1.4376925230,-1.0611702204,-0.5353015661,0.1217178032,0.6271507144,0.7130964398,0.5698034167,-0.0544330068,-0.0827762783,0.4918554723,-0.5561770797,0.9431702495,-1.2986550331,-0.9600256085,-0.1253585219,0.6193585992,0.2552787066,1.2878757715,-0.5704673529,-0.3831481934,1.0959254503,2.0086066723,-2.1474649906,-0.1529575735,0.8048326969,-0.4581208527,-0.0162758399,-0.2110669166,0.1082363203,0.6949726939,-0.5438286662,-1.1747916937,1.6570202112,0.5229940414,-1.2811999321,-1.8111737967,0.4468028843,2.0589504242,0.7505765557,1.5820052624,-0.9668190479,-0.3587667346,0.3426084518,-0.4487540424,-0.5114123821,1.7574405670,-0.4817022383,0.6514472961,-0.7424803972,-0.5344078541,-0.1580218077,0.3720195889,0.5527791381,0.4424994588,-0.1315764487,-0.4980100691,-0.5082452297,-0.1763692796,-0.4118542671,-0.7443788052,1.9393750429,0.8029169440,1.1061604023,-0.0243216679,-1.8400745392,0.6343280077,0.2006740719,0.5488215685,0.2727405727,1.0560905933,-0.6984013319,0.3588303328,-0.1235076785,0.0187691264,0.5021178126,1.6440634727,-1.8855284452,0.6837254167,-0.3057632446,1.5436513424,-1.0427814722,-0.5129017830,0.6372142434,-1.2669779062,-0.0673971698,0.4885091484,-1.0358808041,-0.4340546429,-0.3351761401,-1.0594669580,0.4214607775,-0.2359634340,0.1719900966,-1.6630712748,-0.7779294848,0.7398859859,0.9763131142,0.9751464725,0.3020714223,0.7720235586,0.8978174329,-1.3763316870,-1.1374385357,0.4448396564,0.0724404156,-0.7544128299,1.1938458681,-1.3248474598,0.0271121897,0.3545929790],[0.1016891822,0.8709347248,1.2646727562,-1.1754066944,0.7666998506,-1.0119038820,-0.3860526681,-0.8156091571,-1.6459562778,-0.5340557098,0.7559775114,0.2526681721,0.3399650455,-1.1079380512,-2.3619008064,0.1570393592,1.3197020292,1.2027004957,-0.4237133265,0.7454201579,0.6623315215,0.1792276502,-1.3496153355,0.3558274209,-0.6212102175,-0.8429967761,0.0761769116,-0.9115606546,-0.1469836384,-1.2940363884,-0.3781982064,-0.3462474942,-0.8051198125,0.9975365996,-0.4378167391,-0.3652188480,-0.4983713627,-0.3541530967,0.4649488926,0.4458468854,0.1212970540,-1.2046818733,1.1245608330,-1.0488841534,1.5659842491,0.3328139484,0.4238919020,-0.0850975588,-0.2209404260,0.2532872260,-0.3511019945,-0.0293736942,-1.6951823235,0.3349150717,-1.3477668762,-1.0378328562,-0.0748831630,-0.6586401463,-1.7394788265,0.2208534479,0.3594329655,0.4248667955,-1.0971237421,-1.8624976873,-0.8188779354,0.5210671425,0.1090878546,-0.7082766294,-1.1870042086,-1.0185188055,-0.0105035771,-1.0859087706,1.0135194063,-0.2151187509,0.8128567934,1.1123318672,-0.2369445115,-0.7196567655,1.3356708288,-2.0030705929,-1.7887951136,0.5355278254,-1.9258182049,-1.0679100752,0.4666056633,0.6799566746,-0.6068207026,0.0376300998,-0.8954614997,-0.1887252182,0.0670508146,0.2367895395,-0.7261124253,0.1988874674,1.1370029449,0.2628233731,-0.1744217128,1.0385823250,1.9289591312,-0.6241374612,-1.0708029270,0.7845878005,0.2213747650,1.2395253181,-1.6699419022,-0.5792590976,-0.7590466142,0.1721247435,0.6796442270,0.2983916998,-2.5116040707,0.4435839653,-1.0932787657,0.3189210296,-1.4347689152,-0.4141550660,1.2627629042,1.2916916609,-0.7968822122,-1.6745941639,0.6268520951,-0.2823364735,1.5425739288,-0.6658401489,-0.2700735033,1.0813314915,-1.7792905569,1.3893454075,-0.2835470736,1.1381509304,-0.5500882268,-0.3804549575,0.2835711241,0.6406458020,-1.9321515560,0.4521977603,0.0565207116,0.0297215879,0.1266185045,-1.1317875385,-0.0913078189,-1.4813172817,-1.6310836077,-0.1334815770,-0.5777752995,2.1017003059,1.8821809292,-1.0327913761,-0.4687115550,0.9024651051,-0.2668906152,-0.3431535661,0.9062344432,-1.4410095215,-1.8423423767,0.1403119117,0.3281984925,1.4122068882,-1.4440245628,-0.2864913046,-1.1767127514,-0.2729947269,-0.3813755512,1.2839496136,-0.3271493614,0.1473392993,0.5373675227,-0.5618190765,-1.8962076902,-0.4639317989,1.9236954451,-1.6677190065,0.1464398950,-0.3466111720,-0.0322197936,-0.4217617810,0.6667121649,-0.6521687508,-0.1843066812,-1.4433963299,-0.0677214116,0.1818892509,0.2523166239,0.0736405179,-0.3877593279,0.8917374611,-0.3416461349,-0.9169380665,0.5122210979,-0.4665381312,0.5767468810,-1.4800413847,1.7983590364,-0.6339222789,-0.2424547374,-0.4834831655,-0.8287428617,0.3135240376,0.8550209403,-0.0855247900,1.5032401085,-1.1833488941,-0.0465497859,1.0746041536,-0.9027399421,-0.5185764432,1.8878662586,-2.1824460030,-0.5935040116,1.2031533718,-1.2403950691,-1.1820745468,0.2421696186,-0.3005659878,-0.5867491961,-0.1843579561,-0.0163780041,0.4539252222,-0.9440818429,-0.3242641687,-0.9195576310,0.1757366210,0.5645468235,0.6839561462],[1.7646182775,0.0367704555,1.0817803144,-0.3612675667,-0.8710808158,0.6322647333,-0.3519022763,0.7156081200,0.2653824687,0.0456245095,0.6150549054,-0.2387576997,-0.9838037491,1.1115764380,-0.7091395259,0.5565314293,-0.1952365786,0.7066562176,-0.1020995602,-0.7186366916,-2.5378692150,-0.2224598378,-2.9098627567,-0.6451359391,0.3340830505,0.3082834482,-1.3182439804,-0.3153700233,0.3725237250,0.6292732954,-1.1076341867,0.1932128668,-1.1732625961,-1.2407459021,0.0289576929,0.4187066257,-0.1023253500,-0.5126762390,0.3325288594,-0.9176725149,0.3762895465,0.4857420325,0.6293258667,0.4774833620,-0.6179270744,-0.7745063305,0.2900193036,0.6556027532,0.7161750197,0.0969872698,-0.2772769630,0.9734714627,0.9022426605,0.5066912174,-1.3773590326,-0.4419030547,-0.9061026573,-1.2645193338,-0.6771960258,-0.7416522503,-0.4033834934,-1.7905031443,-0.7197990417,-0.1203307286,0.1458987892,1.0860456228,-1.3431071043,0.5452275872,-0.8861137033,0.0133407973,1.8350932598,-0.7494224310,-0.9539094567,-0.1490415484,0.4838488698,0.1974841207,-0.1861835718,0.4435926378,-0.8599298000,0.8017390370,0.8421186805,0.1312466115,0.6086190343,2.2241444588,0.9180830121,-0.1491888463,0.3608322442,-1.1499377489,-0.3584724963,-0.3109835982,-0.0412371941,-0.2816070318,-0.3876631856,-0.2826207876,-0.3808020353,-0.8287383914,-1.2302982807,-1.8592271805,-1.9187139273,1.2635271549,0.0478380919,0.5384370089,1.9528542757,0.0862558335,0.0315737054,-1.4575328827,-0.6429215074,0.2418305725,-0.8390690684,0.2132374197,2.1373596191,-0.7756822109,0.7898190618,0.3226329386,-0.6459203362,-0.8996171951,-1.1986262798,0.5406119227,0.3710613251,0.0836729556,-1.3073230982,-1.7027620077,-0.7785076499,-1.2514255047,-0.7407965660,-0.6227626204,0.7248088717,-0.9243300557,-2.0425086021,-0.1531879455,-0.8591698408,0.2854239345,-0.8992351294,0.6543528438,0.2941486835,-1.4519603252,-0.4732797444,-1.6374208927,-0.0668385923,-1.9343371391,-0.7832041979,-0.9100412130,1.6664811373,-0.2630090714,0.0338229686,0.2412451953,0.6096515656,0.3015289009,-0.1897550225,-1.3486781120,1.5449596643,-1.4853315353,-1.6852138042,-0.1775777191,-0.1915055960,-0.2492433488,-0.5336878896,-2.4988777637,0.8858756423,-0.8840622902,3.1992139816,-0.1212008819,-0.2876134515,0.0497608893,0.4297440648,1.5458735228,-0.1417457461,0.2272661030,0.4704810679,-1.0789257288,-2.4185669422,-0.8468856812,0.4660403132,-0.5138173103,1.7465744019,-1.1475670338,-0.1167239025,-1.4896283150,0.5956840515,0.9185001254,-4.8521175385,1.9427913427,0.1110671237,-0.8844205141,-1.5047377348,-0.0306469593,0.4130496979,-1.3862971067,-0.1759836525,-0.0979491323,2.1978659630,-0.3581565320,0.0429174267,-0.4899995327,1.6953723431,1.6068502665,0.8885279894,-1.8128461838,0.8054482937,0.3749721348,-0.1091318503,-1.4783062935,0.4781311750,1.1914623976,0.8886550069,0.9224624634,1.4349462986,0.6760825515,-1.1191790104,0.9839367270,0.5592536926,-0.3300490081,0.9047982693,0.6609485149,-0.0711592138,-0.9454471469,0.9492801428,-0.0225257389,1.1091521978,-0.3238544762,-1.5080913305,0.8890247941,0.1120769978,-0.3998112082],[-0.0686101541,-1.0087525845,1.1175528765,0.7182284594,1.0960410833,-1.1370993853,0.5957803726,-1.2771836519,1.3723937273,-1.0925372839,-0.0115856826,0.5942286253,-0.4304260015,1.0643801689,1.0266114473,0.5688973069,-1.3226926327,-0.0613019504,0.1347569972,0.3081897497,2.7131412029,0.4377599359,0.6334095001,0.3660160601,0.0500705987,-2.2779371738,-1.4221690893,-0.6358492970,0.1561469287,-0.9243812561,-0.2980656326,2.0219075680,1.4666409492,-2.5164093971,-0.3836316168,0.8046503663,-1.5332547426,1.2595324516,0.9782047272,-0.6557393074,1.0916570425,0.5810801387,1.0126934052,1.5139583349,-0.3119343519,-1.6407750845,0.7864783406,-0.8549495339,-0.7213819623,0.3934730887,-0.7415456176,0.3942766190,1.1725586653,-0.3623433709,0.6115031838,0.3008793890,1.5449849367,0.3181261420,-0.6514180303,0.2031024098,-0.2640877068,1.7698686123,0.6486811638,0.4432561994,-1.1322586536,-0.0290353186,-0.0781910121,-0.8639219999,-0.1526577324,0.1938848794,-0.5969279408,-0.4834834635,0.1573596299,-0.0592252091,-0.2702434063,-0.1515217274,-0.6052051783,1.9567449093,-0.4848371446,1.4323050976,0.0834007561,1.0406645536,0.0405718312,0.4692253768,-1.5332688093,0.2604510486,0.1545928121,0.5367847085,-0.3339368403,-0.4093825817,2.1708376408,-0.7797004580,2.4333009720,1.3753080368,0.2541124821,-1.9115686417,-2.1534113884,0.7856022120,-0.3899005949,-2.4047720432,0.1866433322,-0.6678485274,-1.3123197556,1.2337210178,1.6666272879,-0.1272123754,-0.9279864430,0.7762946486,0.9160736203,1.7719123363,0.7667344213,-0.1590543240,0.6486815214,-0.6689631939,0.5103842020,0.4520286322,0.1566184610,0.3792171776,1.0502470732,0.4308493733,-0.3508707881,0.7412344217,-0.3274298310,0.9234266877,0.9893772602,-0.8493971825,0.0270343702,0.4415463209,0.6724033356,0.1209470704,0.5693858862,0.0050581717,-2.1161911488,-0.6223433614,-0.5580531955,0.2278435379,-2.5865576267,-1.7090963125,-0.0353317894,0.5161625147,1.2569897175,-1.4674524069,0.3912377357,1.5026664734,-0.4857432544,0.0476039536,-1.3667999506,0.2982648015,0.2895137668,-1.2085553408,0.1725535691,-0.1539426297,0.6676985621,0.1313693672,0.3164666891,-1.0863517523,0.0785836726,0.7611423135,-2.1127955914,-1.4819056988,-1.0567213297,-0.8317061067,-0.4102806449,1.3269690275,-0.4393536747,0.0698533207,-0.6893197298,2.1084895134,0.2268670052,0.8511615396,0.3854487240,0.4431305826,1.0205435753,0.4819956422,0.5102502108,1.5450580120,0.8968815207,-1.0848166943,0.7952970862,2.0239479542,-0.7240387797,0.0220920593,-1.1563844681,-0.1014404744,0.3656550944,0.5663932562,1.0644277334,-0.5995697379,-1.2106146812,-0.8576260209,0.5674060583,0.6007801890,1.5416058302,-0.6908056140,-1.0867762566,0.2649148703,1.4798949957,0.7393624187,0.2967948020,-0.2771933377,-1.1488587856,1.6003367901,1.6553497314,0.6626985073,-0.6280882955,0.0936584100,0.4981217682,0.9492636919,-0.4618748128,0.2677331269,1.2989481688,0.0265531186,0.3216917515,-0.2674317956,-0.0111542614,0.2487320751,-0.3743169010,-0.9511997104,1.6651052237,-1.6196660995,-0.5902080536,0.3614725173,-0.4152885973,-0.7183656096],[0.3520698249,1.4739658833,2.4271173477,-0.5691303015,0.9813193679,-0.5925389528,-0.5128179193,-1.1542140245,0.5358704329,1.3890676498,0.5996650457,-0.2719255090,-0.0948925391,-0.7808255553,-0.6444920897,0.9755622745,-0.2208294421,-0.3493718505,0.2699482143,-0.7280527353,1.3917051554,1.4498575926,-1.2113507986,0.3349868953,1.3251229525,-0.9171952009,0.4963794351,-1.7986804247,0.0789784789,1.2199368477,0.3269993365,-0.1948697865,1.3435184956,-1.0425250530,1.3683353662,0.2499956936,-1.4721306562,-0.2956948280,0.5207143426,0.4645217061,1.4431642294,1.2225159407,-1.3223319054,1.0886017084,0.9286890626,-0.1376378387,-0.5915322900,-1.0464158058,0.5867432356,-0.8797436953,-0.7235412002,-0.6804904342,-2.0942678452,1.0294381380,0.7672358155,1.0971862078,0.6071217656,0.6025097370,0.8277061582,1.1057502031,0.5251725912,-1.1081231833,-0.4811174273,-0.3702764511,-1.3508657217,0.2259329855,-0.0718035176,0.1823831201,-0.2373090535,1.1184384823,1.3183101416,-1.9148426056,-0.4826577306,-0.4699207842,1.2780392170,1.4454777241,-0.2684786618,-0.2248070687,-2.3880140781,0.4959293306,-0.3267105520,1.3605651855,-0.5306251645,-0.0932557359,-0.5050152540,-0.7892034054,1.0203313828,1.2451971769,-0.4140982628,-0.3513950109,-0.9037069678,-0.0241906717,-0.9075904489,-0.6557211876,-0.1932841986,2.3157627583,0.5288925171,-0.6291821003,0.9991913438,-0.4957025051,-1.1099767685,0.0842455551,0.0248599015,0.0334466957,-1.6484893560,-0.8135964274,-0.0473386273,-0.7118042111,0.2833947837,-0.0167807993,0.2662077546,0.6260454059,-1.7930265665,-1.1567467451,-0.5512157679,0.6015375853,-1.2865725756,-1.9408442974,-0.1325186938,0.4100147486,-0.8295570016,0.1938338429,0.0979138017,2.5477495193,-0.4140664637,-1.7858626842,0.6429503560,-0.3502498269,-0.2588729262,-1.0729311705,-0.3541617393,0.3570177555,-0.6643162966,-1.4513443708,-1.6973505020,-0.0602100231,0.2496821880,-0.9163889885,-1.5405261517,-0.4512024224,-0.6341095567,-0.0097503057,-1.2270138264,-1.3736895323,0.1918285489,-0.9191774130,-0.6565604210,0.6039719582,-0.1332715899,-0.8285505176,-1.0401451588,-2.4385795593,2.2394487858,-0.2572565675,-1.8150984049,0.4597521126,0.4808021188,0.5467767715,0.6092917323,1.1429659128,0.2309983224,1.5645856857,-1.2469551563,-0.2577306330,-1.1540452242,-1.3649905920,0.6702425480,0.2360917628,-0.1505737454,0.3084661663,0.9137232900,-0.8299762607,1.7046604156,0.4507624209,1.6921821833,0.0341390707,1.4222145081,-0.5597399473,-1.0490896702,0.6614012718,0.7688275576,-1.1840931177,0.3911654651,-0.4891320169,-0.7211934924,0.8481540084,-1.0097805262,-0.3961494267,-0.4591137469,0.1611833125,0.1956343353,-1.2476583719,0.4416362345,1.1361719370,0.0931860134,-0.9278345704,-0.7907865644,0.6913827062,-1.3883576393,-2.3996989727,-0.3636757135,1.1655604839,-1.4245017767,1.2208950520,-2.0990407467,-0.9538810849,-0.5822237134,0.1402315944,0.0464295745,-0.5008851290,-0.2975231707,0.8082782030,-0.3997581005,-1.6819230318,-1.0130381584,1.7739200592,-1.0784034729,-0.1895323843,-0.7823113799,0.8008701801,-1.5388237238,-1.5723116398,-0.4203703105,0.1166450381],[1.0583444834,1.5439826250,-0.5834670067,-0.9337796569,1.9155551195,0.1372870356,0.7505593300,-0.8319081664,0.6255533695,-1.4047561884,0.1378288567,-0.8832240701,0.5066967607,-1.3870570660,-0.2949680090,-0.5365743637,0.7241915464,-0.0380382910,-0.4298193157,-0.3633871377,-0.1892514825,-0.9280431271,1.6019604206,0.9871795177,0.4851090908,1.4912133217,-0.0962753594,0.2499185055,-0.5457340479,-0.4101860225,0.2294923216,2.2696387768,-0.3302621543,-0.2712452114,1.5598413944,0.2520926595,3.1268947124,-3.2329590321,-0.2350266725,-0.1970554143,1.6455147266,-0.4833549261,1.8614071608,0.1210813671,2.4485468864,0.3696423173,-1.1890643835,1.1097162962,-0.9812495112,-1.5117832422,0.4652085900,1.0399012566,0.4068359435,1.6711966991,0.4793457091,0.3833797872,-1.2494162321,0.1649812311,-0.4304262102,-0.7297959924,0.8276520371,0.7847200036,0.5139238834,-1.9722926617,1.0811411142,0.1848393828,0.0775561258,-0.2357782871,0.1912350357,0.5073704123,0.0564742349,-0.8233909607,-0.8220529556,0.1913980395,0.8005524874,-0.1441638619,0.7258716822,0.2305646241,0.9593602419,-0.1887491792,-0.6131626964,-0.2583982646,-0.7700645328,-1.6128127575,-0.6146055460,1.2405661345,0.4227060676,-1.0741027594,0.3241637647,-1.4035487175,-0.0658114403,0.6804963946,0.5681362152,-1.3921498060,-1.0300476551,-0.2371842414,1.8360530138,-0.9216187596,0.9207280278,-0.5574217439,-0.4712163210,0.2855410874,0.3498257697,0.5446513295,-0.0277582034,0.9110357165,1.0241811275,-0.4904206991,0.0496969819,0.6285252571,-2.9861686230,-2.1029310226,-0.3938883245,-0.2817675471,0.2416146547,0.9426638484,0.3875665367,0.1224803925,1.0414155722,-0.3351289928,-0.4107232392,-1.5650410652,-0.8893459439,0.0849630237,-1.7995682955,0.9912497997,-1.8885445595,-1.1762053967,-1.2450914383,2.7380778790,-0.3616687357,-0.2209551036,-1.7513599396,-0.8957287669,-2.2897720337,-0.0876673013,0.6963365674,0.3551538885,0.9297693968,0.0609674230,2.4712345600,-0.4208989441,0.5249027014,-0.2286021709,0.2329783738,0.1197842136,-0.9520625472,-1.8651957512,0.2399929166,-1.2985645533,-0.6591146588,0.2112485319,-0.6970937252,-0.6887322068,-2.0674235821,-0.0465840735,-0.9805367589,0.4654271901,0.5672482848,0.7053048015,-0.6611399651,1.5472348928,-0.9092199206,-0.0546701252,-1.0351397991,1.3140008450,-1.5395072699,0.0404680595,0.9997606874,0.6848905087,0.4523021281,0.4604639709,-0.1260150522,0.1083656922,0.5990031958,0.7269591093,-1.7155878544,0.6685690880,-0.6821148396,-1.3120325804,-0.5852013230,-0.3806896210,2.0486714840,0.6823633313,1.0913326740,-1.6782044172,1.5952193737,0.6944016814,-1.1871331930,0.9472473264,0.5641186237,-1.0589094162,-0.8352508545,0.8253346086,0.5406584144,1.0564606190,0.6108237505,1.3554942608,-0.0652598366,0.3243359029,-0.4841716290,1.2476578951,0.2305612564,0.9184203744,0.7574568391,1.0057890415,-0.4710326791,0.0801935941,0.0023979940,0.6727606058,-1.4283015728,2.0315153599,2.1044881344,0.8079706430,0.2700591683,-0.1420193762,-0.4140471816,-0.8383685350,0.3478310406,-0.1102372110,1.3906400204,-0.4889595807,-0.1187727004,-0.8679751158],[-0.0362039842,-0.0926602483,-1.0390424728,-0.5257984400,-0.0478054583,-0.9813799858,1.6373766661,-0.0322443843,-0.4711576700,-1.7374867201,-0.3807227612,-0.1615162790,-1.1044647694,0.1208630577,0.8739864826,0.4768305421,0.6930779815,0.3838321269,-1.5314662457,0.3809277415,0.1716790795,0.3956367671,0.9040478468,-1.4133884907,0.8503904343,1.1443232298,-0.0336308964,1.1579585075,-0.3186286688,-0.1953814030,1.1942832470,-0.5953484178,-0.9550271630,0.2698577642,0.1056953147,-0.0633471906,0.3749876022,0.1402860582,1.2944872379,-0.4121560156,0.4817930460,-0.0993803889,-1.5939522982,-0.4357390404,-0.4619060159,-2.3683354855,0.2805862129,-0.3046934605,-1.2935652733,-1.1554673910,0.1633427739,-0.5986722112,1.2634762526,0.9150058627,-1.0899602175,-0.5049229264,-0.0143814981,1.5127053261,-0.8898098469,0.3804249167,-1.1658823490,1.2145162821,0.1391834766,0.9084006548,-0.0797242448,1.7454173565,0.8128961921,-0.3380572498,-0.8773782849,-1.1081911325,0.7548649311,0.2150626332,1.1833186150,-0.3489762843,2.1793832779,0.3019517660,-2.0800480843,1.3498222828,-0.6469517350,-1.0101433992,-0.0867938995,-0.3999692202,0.5472961068,0.4918823838,0.9543326497,1.3563923836,-0.8218570352,0.0116919894,1.2118961811,-1.2016319036,0.4808796048,-0.4284862280,0.2089256644,0.8705192804,-0.3592517376,1.0025163889,-1.3868917227,0.2231241167,1.6292181015,-0.7550464869,0.4189785719,1.3076460361,-0.3605156541,-1.1180034876,-1.1126116514,-0.6930063963,0.3895854354,-1.1811661720,-0.7325514555,0.1279658824,-0.5388088226,-0.3408534229,-0.2591611445,-0.1481301636,1.1229610443,2.1432387829,-0.3806581497,0.8855195642,-0.5941774845,0.8604633212,1.0511958599,1.0734173059,-0.2655522227,-0.3933710754,0.3349715471,-0.8550754786,-2.2061393261,0.9727731943,0.7498283982,1.6936372519,0.5487921834,0.0691117644,2.7273132801,-1.8114776611,1.1478933096,1.0795128345,-1.0359307528,2.3957827091,-0.7863538265,0.5545166731,-0.2966851890,-0.2659127712,-1.2508859634,1.1771451235,0.4923931360,1.2278065681,-0.7286225557,-0.0930251777,-0.5998708010,-0.2299500108,-0.4579644799,0.5729542971,2.3201682568,-0.8726349473,0.3805692196,0.8630654216,1.6897252798,-0.8851308227,-0.3429644406,-0.7624091506,1.3988976479,-2.7959020138,0.4372125268,0.0636285543,1.8906742334,0.0905186236,-1.0840857029,2.0455691814,0.9350713491,0.1695361584,1.2804718018,-0.6724713445,-0.3662121296,2.0804431438,1.7761652470,2.8678522110,-0.0991531014,-0.3440111876,-0.9633069038,-1.3825284243,-1.4679977894,0.2173433304,0.0366753228,-0.7773872614,-0.0060850084,0.2425571680,-0.3767814338,0.8276780248,2.0392413139,-0.2837807238,1.2338672876,-0.3843937814,0.5756478906,1.6414058208,-0.3666916788,-0.9714407325,-0.2356366962,-1.2913995981,-0.8960181475,0.1272795498,-2.2407054901,1.1917226315,0.4333340824,-0.3930523992,1.1191014051,0.6791875958,0.4361049235,1.0977270603,1.0344529152,0.3062209487,-0.0446278788,0.0910232663,0.5822839737,-0.1521469802,-1.1190607548,-0.2476054430,-0.7367978096,-0.4856684804,0.6659203172,-1.2921715975,1.8827733994,0.5838482380,-0.2108717859,-0.0243280269],[0.1365130842,1.4073339701,0.9856215715,0.3627391756,-1.3479270935,1.1825037003,-0.2711353302,-1.2824739218,-0.6591538787,-0.5253095627,-0.4016566873,0.4823636115,-0.6241627932,0.4012881219,2.4049458504,1.8389296532,-0.8160229921,-0.8917958736,0.2054117322,-0.5678172708,0.0877340585,-1.3987337351,0.3651104569,-1.0884479284,-0.0357880071,-0.7008450627,-1.4183855057,-1.0605516434,-0.5557442904,-1.4653967619,0.3975590765,0.9693913460,-0.5058177114,0.1420748681,-0.1394046992,-1.1864151955,0.5770588517,-0.4497047067,0.6133778095,-0.2091179043,-0.8822066188,0.9266041517,1.9369000196,-0.9379522204,-0.5523123741,1.6441167593,2.0208413601,-0.5548222065,0.0124798529,0.5058183074,1.1044448614,1.3189389706,0.1909236014,-0.3080491424,0.5630736947,-0.0837698281,-0.3891959786,-0.9374899864,0.6200323701,0.7414302826,-1.2315765619,1.9131008387,0.8843234777,-3.2155885696,1.7159427404,0.2663838565,-0.0736449733,1.1531600952,-1.0004495382,1.8306009769,0.4198108017,-0.5446417332,-0.3730129302,-0.5347300768,-1.3494056463,0.5697371960,0.3700086176,-0.8028948307,0.1239058226,-1.1565580368,-0.9970475435,-1.4502310753,-0.5761058331,-1.4261182547,-1.5865499973,0.8991324902,0.6901183128,0.3546708822,-0.3055213392,-0.1828400046,0.0213740449,-0.7595441341,-0.1451109946,0.2397838533,1.0029948950,0.7565121055,1.3437296152,0.9344487786,-0.0852555111,0.8469684124,0.5529686213,-0.2060049176,0.1530812085,0.6497737765,-0.7188387513,-0.6574992537,1.2068393230,1.4107749462,0.5801166892,-1.9565601349,2.3294208050,1.6649837494,-0.1143305302,1.1704581976,0.1861006618,-0.3432528973,0.7738602161,-0.7633075118,1.6620934010,-0.0148074497,-1.5225170851,0.4448566735,-1.0804424286,-0.2595146596,-1.2039898634,1.1865116358,0.1244517788,-1.7411261797,0.6327655911,-1.4254326820,0.6072432399,2.2639195919,-1.7797319889,0.1546354294,-1.6730787754,-0.6163903475,-0.3629196286,0.1454897076,0.6794376373,1.9794061184,-0.4524740875,1.8563537598,-0.1063185409,-0.9688279629,-0.1407084316,-0.5214707851,-0.2364092022,0.1224080771,-1.4739046097,0.2961368263,-1.8272217512,-2.0639176369,-1.3346837759,0.5843850970,-0.5584329963,0.0578550063,1.0357207060,1.7963436842,1.2943576574,-0.7003925443,0.8481504321,0.6891615391,2.0482206345,-1.2533652782,0.2402896881,0.6737480164,0.4490898848,-1.6458679438,-0.9338932633,-0.3591641188,-0.0254199263,-0.5341939330,-1.2297190428,1.7889677286,-1.4703223705,-0.1667094529,-0.8333431482,-0.8809360862,-0.0065073199,-1.0043187141,-1.0234842300,0.3805767596,-0.4275160134,0.5769917965,0.9083298445,0.7803912163,-0.8485795259,-0.3394566476,0.6788250804,-0.5510795116,1.0690231323,0.6100328565,-1.1385766268,-1.2801609039,1.3604898453,-0.1692980230,-0.9945854545,0.1831673235,-1.6491280794,-1.4920870066,0.5627005696,-0.2753271461,0.1657936871,-0.5342975855,0.2055897117,1.9000588655,1.7738777399,0.0581240840,1.7368767262,2.5812046528,0.3220744729,-0.3783294261,-0.3094934523,0.8187251091,0.9548506141,0.9769791961,2.0029132366,1.7150564194,0.0704670697,-0.3864088058,-0.6378495693,-0.0315747187,0.3025371432,0.4778320789],[0.3217836022,0.6492074132,-0.9109751582,-0.7691422701,1.4026347399,-0.9658105969,-0.1002218947,1.5368416309,-1.2793962955,-0.1795984358,-0.3374651372,-0.8700996637,0.4942047596,0.1025737524,-0.9360316396,1.4573398829,-1.1856033802,0.3710526824,0.5413604379,-1.3624647856,-2.2412455082,-1.4766739607,0.7899505496,-2.7598965168,0.4047561288,-0.6828455925,1.9673783779,0.6626994610,-1.0715668201,0.2047873735,0.0702401102,0.2123592794,-1.3596889973,-0.6493543983,0.7024531960,-0.7617487907,0.2701660991,0.5384673476,0.4484993517,1.4303277731,-0.2872730196,0.2255844325,-2.0188221931,1.1860226393,-0.0449120253,1.4178494215,1.1506255865,1.9027528763,-1.0151681900,1.4701216221,-0.3491511941,1.9576206207,1.7094520330,0.8521990776,-0.6261596084,1.6764523983,0.7147725224,-0.4068088233,0.6308442950,-0.0767498091,0.6154634356,-0.5383847356,-0.3225371540,0.4823666215,-0.3547261953,-0.2532891035,0.6753273606,-0.5546728969,1.0751554966,-1.3161867857,-0.9454118013,-2.4545342922,-0.0518297330,1.0433616638,-0.5003373027,-0.3093115389,1.7492716312,0.2976246774,-0.8289340138,-1.5835789442,-0.3375042379,1.9913307428,1.0772083998,0.0367077291,-0.0472101569,0.4658238590,-2.6238150597,0.9969777465,0.0754188821,0.3507779837,0.4585555196,-0.0999287292,-0.3928496540,1.7388290167,-0.8560393453,0.0992961749,-2.1302506924,-1.3153241873,0.8314374089,0.0599960461,-0.1368577778,-0.3394154906,-0.5296584368,0.2793669999,0.7099635005,-0.0835562423,0.6021608710,-0.1196800545,-1.3378901482,-0.3641176224,1.2721914053,-1.4890229702,-0.6869053245,-0.4686065614,0.6426337957,-0.5083370805,0.2268973142,-0.1974736899,0.2527569532,1.7184084654,0.8922979236,-1.5970692635,-0.4582478404,-0.4076239169,-0.1977892369,-0.7555869222,-0.3449466527,-1.2325458527,0.1456603408,-1.2386804819,0.1323075742,-1.8673850298,-0.0677953959,-0.8390944600,-0.2440842092,-1.1514731646,0.3661558926,1.9396024942,0.5101702213,-1.1429055929,-1.1398406029,-0.5165202022,0.3982322812,0.5776135921,-0.2317762971,-1.4540951252,-1.3447998762,0.8522616029,-0.5578137040,-0.2578280866,-0.0890806839,0.0349463075,0.6758262515,1.1467132568,-1.6023062468,-1.1934000254,0.7290937901,1.1782319546,0.3880271018,0.0174299795,0.7336113453,1.0139441490,0.5473438501,1.4955612421,-1.4451948404,0.6801576614,-1.3201237917,0.6385329366,0.0782859474,-0.0863106698,0.6152905822,1.3551502228,-0.0890230834,-0.0486132428,0.8171393275,-0.1492211223,-1.7590962648,0.9003215432,0.1985468268,-0.2245959789,0.6709069610,-0.9071885347,-0.0217190348,1.0644416809,0.2005730122,2.5758602619,-0.1752953529,-0.4584687352,1.0486868620,-1.1132804155,1.2848652601,0.0694471598,1.1292618513,0.9819657207,0.8070746064,-1.1838014126,0.9127116799,-0.0829647705,-1.8536667824,-0.0619276203,0.0663916543,-0.9791166782,3.1375920773,-1.5782490969,-0.0379903316,0.9323066473,-1.2534710169,-0.5725334883,-1.3511468172,0.7215948105,-0.5781445503,3.3989739418,-0.2205531001,0.3763498962,-1.2892030478,0.6944087744,-0.0149582736,-1.7329707146,-0.1396349519,0.8098288178,0.1786913425,-2.2183401585,-0.0800998956,-0.0742241070],[0.3845080733,-1.6463791132,0.7107392550,0.9891936779,0.7867619395,0.0749255121,0.2001665980,-0.1341113001,0.0893330351,-0.9379463792,-0.9581400156,-1.2719728947,-2.7973825932,0.5311394334,0.5251041651,-0.4277549386,-0.2632635534,-0.0099409046,0.7350210547,0.4192803800,0.0018239388,-0.0443116762,-0.0689535663,0.7293576598,-0.4967725575,-0.5072520971,-0.1359933764,-1.2276622057,0.6094608307,-1.6442064047,-1.6363502741,1.2297830582,0.0999848545,-0.0986485630,-0.0178937595,-0.2287111878,-1.5501198769,0.2100454867,0.8902381063,0.1387624741,-1.6011765003,-1.2235500813,0.3127393425,0.3142060637,0.8466959000,-0.4800876677,1.5377589464,0.2422349155,-0.9983308911,-0.0291883871,-0.1622420102,1.0121430159,0.6871752739,-0.4247593284,1.0550050735,0.3513199985,-1.4561393261,2.1552989483,1.3274356127,-0.5354955792,-0.6845734119,0.4479947090,0.3242218196,-0.5768624544,-0.3779707849,-1.0639026165,-0.2439878583,0.0086092334,1.6724859476,0.0878778622,1.1755329370,0.1444046646,0.6706321836,1.2580265999,-0.4368670881,-0.3676288128,-0.1864498854,-0.5201572180,1.1724785566,0.2010924965,1.6923856735,0.2290286273,1.6569188833,-1.7776489258,0.4573445320,1.3080205917,-0.3747129440,1.2378357649,-0.5824361444,1.1357333660,-0.9598888159,1.1630618572,-1.5749995708,0.7833538651,0.1442527771,-0.2554296255,-0.9792585373,0.4596745968,0.6203821301,0.5021705031,0.9194006324,-1.0334367752,1.6282829046,0.8093135357,-0.2234846950,1.2612539530,-0.9625234604,-0.6355591416,0.4715775549,0.0068701040,-1.0172108412,0.2494076043,-0.5978446007,-0.7592504621,0.8392009139,-1.3911912441,0.4128962755,-0.7735784650,0.0836772174,-1.5262907743,2.5723426342,-0.1008686349,-0.2837621868,1.0058437586,-0.7359182835,1.1496107578,-0.8255521655,0.7245010138,-0.8101114035,2.0787091255,0.8559275866,-0.5351945758,0.6291060448,0.0856874585,-0.0063943989,0.3543403745,-0.5699222088,0.2983883321,0.7868961096,-1.0871948004,-0.2908948958,0.0930618271,-0.8280441761,1.1082425117,0.2538062036,0.2027538866,0.1541826278,0.2480404079,-1.4028227329,1.1533126831,1.6422873735,1.0832636356,0.8322021365,0.8517474532,0.7841898203,-1.7440323830,0.4765495658,1.0487160683,-0.7914233804,-0.4405639768,-0.2160148919,0.1842859089,1.1629821062,-0.0767243430,-0.7312394381,-0.6774958372,0.3769921958,-0.5758907795,-1.5134738684,1.4181641340,-0.7427386642,-1.0262612104,-1.0112211704,-0.8954152465,1.0703066587,-0.3507716954,1.0855424404,-0.3428264856,1.1746728420,0.3893014789,0.9072918296,-1.5879071951,-1.7463401556,1.5358440876,-0.2369948775,0.2931667566,0.4586500227,0.2925986946,2.3936650753,-1.6313507557,-1.3064545393,-0.2756645977,-0.1694798321,-1.8411900997,0.1854819059,-1.0566730499,-0.1226647571,0.0523388982,1.1009833813,-0.8073753119,-0.0405311659,-0.1907524914,1.6200549603,-0.8034185767,0.9933840036,0.8365669847,-1.0088229179,1.3915550709,-0.6660532951,-0.4306976795,-0.2386864126,0.0843213871,0.5830325484,0.1325431019,-0.7190958261,0.4042440355,0.1904487908,-0.3511888981,-0.0677566752,0.7692873478,-0.9150775671,-0.2478805482,-0.1723368913,-1.7062830925],[0.1571991444,-1.2567185163,-1.2486581802,-0.3304790258,-0.1429319084,-2.0155272484,0.1789071411,-0.6599000096,0.2713014483,-1.5153937340,0.5429274440,-0.5614190698,2.1147344112,0.1305354685,2.3193845749,1.0932368040,-1.4592151642,0.8998706341,-0.5619531870,0.1160965636,-0.1549596637,-1.3603447676,-0.1224236861,0.1485626996,0.2863123417,-0.7079705596,1.2752975225,-1.8051139116,-2.4880578518,0.5680893064,0.7016727328,0.1439006329,0.4267070591,0.1790274084,-1.6500613689,-0.8564890027,-0.9328461289,-0.1378492862,-0.9564430118,-0.2885970175,0.2648217380,0.2335142493,0.8945885301,-1.3390487432,0.6258945465,1.6428123713,1.6459735632,0.5991826653,0.1086787358,-0.7074826956,0.9272280931,0.4744967520,-0.4278449118,1.0480604172,-0.4526143372,1.3277521133,-0.0564636923,1.3267620802,0.2104748785,0.3823283613,2.0361435413,-0.5682298541,-0.9197977185,0.2950363457,-0.0598035455,-0.3787401319,0.7625000477,-2.9526107311,1.0240620375,-0.1243128777,0.9883049130,-0.2431208044,-0.5595453978,-2.2868940830,1.2129172087,1.8089084625,1.2684593201,0.5851132274,-0.2312829345,1.8950910568,1.1503691673,0.6162592769,1.1002618074,-1.1529587507,-1.5807942152,1.0720236301,-0.8970527649,0.7691181898,-0.3447100222,-1.0881081820,0.6037220359,0.1806760728,-1.5382072926,-0.6406788230,1.2235890627,0.6681796908,0.5669136643,-0.9254377484,0.4890148640,-0.3914958239,0.3539678752,0.9485787749,0.3038055897,-0.1886343509,0.5120302439,-0.1569281220,2.2453000546,-0.4241433740,0.7208776474,0.3618130088,0.5907598138,1.0482422113,-0.7412730455,-0.1428454220,0.0379181169,0.2372295856,-0.2962428927,0.6757543683,-0.6153737903,2.5128779411,0.1799121350,1.3834085464,0.5711925626,0.9366635680,-0.7422040701,-0.5731911659,-1.0898987055,-1.5423468351,-0.4778312743,1.5007041693,-0.7780981064,0.2048971504,-0.5669568777,0.3746558130,-0.4002239704,0.6586911678,-0.3613056540,-0.9264248013,-1.0720075369,1.2189369202,-0.3600791097,-0.7939486504,-1.1968816519,-0.6835036874,-0.2224785089,-0.8625183702,1.2658147812,2.2403526306,0.8670475483,-0.9484440684,0.6970045567,-2.1961770058,-1.6428921223,1.0440677404,2.1168622971,-0.0151213622,-2.2564005852,-0.6929907799,0.2016869783,2.4280624390,-0.0699802339,-0.5621324182,-0.7554766536,-0.6078110337,0.8083912134,-0.8453896642,2.1699969769,-2.6128845215,-0.0212052483,0.4015052915,-0.5356868505,0.9977837801,0.1122059673,0.0778343603,1.3476768732,0.2835381627,-0.0142331179,-2.0367105007,-0.7371923923,-1.2246629000,2.2353093624,-0.7961319685,-1.9554548264,-1.2512631416,-1.0218428373,1.2124526501,1.1355682611,0.6193997264,0.2487773597,-0.2672690749,-0.4545343220,0.7911229730,-0.3282171786,-0.1699972153,-0.6668633223,-0.4596285522,1.5497270823,-0.0206197053,-1.1669906378,0.6233358383,0.7453544140,-0.4479801953,1.0420253277,0.1992836148,0.5260112286,-0.7913386822,0.5561696887,0.9695681930,0.4419442713,-0.2964356244,0.1331102252,1.4653676748,-0.4023957253,-1.9205198288,-0.1914123297,0.5629822612,0.4740603566,-1.2222735882,0.2560369074,1.1243366003,0.3371331096,1.7473640442,-0.4876609147,1.2396174669],[-1.2924531698,-0.8329555988,0.8106189370,-0.3956554234,-0.5539463758,-0.2932499051,1.0425081253,-0.3843310177,-0.2666507065,-1.8152126074,-0.6074080467,-0.0640053749,0.3151269555,0.6183727980,-1.3044564724,0.8262851834,0.3810087442,1.1846222878,0.3196219802,-1.6368120909,-1.0149875879,0.5286067128,0.2212347835,-1.3540037870,-0.2165237516,-0.0980249420,-0.1607593447,0.6369425654,-0.1619250774,-0.5857174993,2.6423957348,0.9990339279,-0.3168399930,0.3381016850,-0.3765596449,-0.3019950092,-1.2485268116,0.0268364325,-0.4783943295,0.1249814257,-0.4447103739,1.1281132698,-1.7312458754,-1.7791578770,-0.5942288041,-0.5394236445,-0.3606801033,0.0106915636,-0.0157427508,-0.8622555733,-0.0226995777,-0.0577265061,-0.1762067825,0.2407553047,-1.0976681709,-1.3848628998,-1.8822962046,-0.0017791095,-0.5731220245,1.9252378941,-0.4404511452,-0.4575495124,-1.0234162807,-1.6612168550,2.4114139080,0.2933978736,1.0817257166,1.0481520891,0.3456228673,0.0791619942,1.0115976334,3.3180394173,-1.0060584545,-1.0057104826,2.2909924984,-1.4247964621,0.7731633782,2.4017169476,-2.6709401608,0.1828530282,0.2281860262,0.2734143734,2.3842585087,0.7326384783,1.1197537184,-1.3578523397,0.9588159919,-1.0450409651,0.7155377865,0.1096529886,1.2337069511,-0.2804929018,-1.8858106136,0.3575464189,-0.0844301283,-0.8718315363,-0.9173175097,-0.6598443389,1.4191524982,-1.0103415251,1.1741003990,1.8414045572,-0.3743493259,1.0813381672,0.4680810869,-1.5805219412,1.5340808630,0.7638397813,2.2288279533,0.1072080508,-0.7255489826,1.1193445921,0.1728860289,-0.6225542426,1.1737477779,0.5976502299,-1.7421793938,0.2293184102,0.8057854772,-1.6511553526,0.4416771233,0.3466319740,-0.0454043299,-0.1209124625,1.6247614622,-0.7286786437,0.5841590762,-0.1831379086,0.5785822272,-0.4785403013,0.9208111167,0.3044312298,-1.4587692022,-0.1678542942,-0.6752638817,-0.2496262640,-1.6186631918,0.7310032845,-0.4667387307,0.0839912370,0.0778162628,-0.8575924635,-0.2125936151,-0.6107373238,-1.1801326275,0.6009773612,-1.1573953629,0.7150562406,-1.4650510550,1.7268512249,-1.0357637405,-0.4485883117,-1.4579374790,0.8677504659,-1.4565110207,1.5130494833,1.5456116199,0.0075432993,0.0928202122,-0.4449352622,-1.2877678871,0.8085286021,-1.3784750700,-0.0561535470,-0.4697008133,1.3949416876,-0.4398849010,-1.9523414373,-1.8618255854,-0.4945134521,-2.0245842934,-0.7269943357,-1.2706829309,-0.8815476298,-2.0346591473,-1.1727267504,-0.7687079906,-1.7004654408,-0.2370119691,0.4915896654,-0.7950913310,2.6508188248,0.6726636887,-0.4534219801,1.8084821701,1.0631934404,-0.7404429317,0.1299059689,0.2656057477,0.7115128636,-0.0633414313,-0.6535018086,0.7916250825,0.3691915274,0.0040150597,-0.6620145440,0.2596068382,0.4594883025,-0.5084525347,-2.4214904308,-0.9198400378,0.5192216039,0.4900861979,0.8938479424,1.7531205416,0.2292024791,0.2872477174,2.1692998409,-1.0889079571,0.8446975350,0.0144895865,-0.4923832715,0.2821266651,-0.1231191456,1.4335643053,-0.2643232346,0.8737241626,0.9721201062,-1.1100924015,-1.3487612009,-0.2477868050,-1.1834433079,-1.2005623579,1.0191519260],[0.7395281792,1.1104048491,-0.0219590571,2.0232183933,0.5489047766,-0.0274477396,-0.7132084966,0.2385355234,-1.7889448404,0.7535198927,-1.0275415182,0.0347306617,1.3003791571,-0.1093231812,-1.3040199280,-0.2029657960,0.5065885186,2.0733292103,1.3696361780,0.9754089713,-2.0454306602,-0.0382960923,-0.5714712143,0.2167275846,0.6750763059,-0.3483577669,-0.7746589780,0.6165201664,1.3618383408,1.2122330666,1.3800063133,0.5365054607,-1.9177306890,-0.5251973271,-1.0988540649,0.0278137494,-0.7636909485,0.6048128009,-1.1625010967,-0.7021710277,-0.6840939522,-0.2839722633,-0.9208801985,-0.3885153234,-0.9178700447,1.5394451618,1.4378056526,-1.4322452545,1.8636082411,0.4008088708,-0.4350753725,-1.1601147652,-1.1638344526,-0.0322148390,0.4595857859,0.9118782878,-1.5388293266,0.6242665052,-1.0804662704,1.2000719309,-0.1710569859,-1.0350323915,0.4459679723,-0.1440854073,1.3036524057,-1.7022899389,-0.9515473843,-0.6042004228,-0.5069737434,-0.2445824593,-0.6166288257,-0.1103350669,0.0383932628,0.3011859059,-0.5186234117,-1.4050905704,-0.3328391314,1.2396959066,1.4162265062,-0.3398074210,-0.1771123707,-1.0423727036,0.9640633464,-0.4511846006,0.9080410600,0.5163629055,0.9133321047,0.2308627367,0.9364045858,0.4994279742,-0.9036644697,-1.0831034184,-0.4359258115,0.0862119645,-0.6307891011,-0.0972120389,1.4832203388,-0.3554448485,0.2293844819,0.7009916902,-1.2562806606,-1.1880155802,0.8979741931,1.0003473759,0.8966107965,-0.1492953151,-0.1308869272,0.2263855487,-0.0130055640,0.4222289026,0.4016860127,-0.4904920459,0.7437108755,1.9387946129,1.6307559013,-0.5204173326,0.3517319560,0.0115969433,-1.4235920906,-0.4553895593,-0.1439766139,0.7038559318,-0.2786969543,1.5749616623,-0.0669014901,-0.8004282713,-0.7990862727,-0.4329472184,0.2307552397,1.6242434978,0.5721014142,-0.7301663756,-1.3531529903,-0.8805804253,3.4693908691,-1.3049875498,-0.4333794713,-0.2324779332,-0.7562892437,-0.0847490132,0.1238018200,2.0294234753,1.1775977612,-0.7600325942,-2.2919991016,-1.6500588655,-0.9651675224,-0.0731742159,-1.8575273752,-1.3492054939,-0.1087381393,-2.6830763817,-1.9446489811,-1.1404054165,-0.3651441932,-0.5104267001,0.8189151287,0.6377196312,0.5000778437,1.4043917656,0.2186368108,0.5616023540,-0.7406401038,0.7284206152,-1.1032539606,-0.1957814097,0.6142774224,-0.1744652241,-0.8923316598,0.5519254804,0.0332063623,-0.6735925674,-1.1467906237,0.5041069388,-0.0882616341,0.5556857586,1.6077281237,-1.5892181396,3.5689439774,-0.5289149880,-1.3064787388,0.1786384881,-1.1435884237,0.4807915092,-0.4079792202,0.0394755080,-0.6162922978,-0.6694833636,0.8787062168,-0.8178895116,2.6429357529,1.2247593403,0.0740591511,0.5969020724,0.4470560551,-1.7656972408,-0.9531686306,0.6847508550,-0.7460402250,-0.0438507125,1.1078300476,1.3620145321,0.3313216567,1.3880150318,0.4393956661,0.8094944358,0.4581452608,0.4303210080,-0.1151056662,-0.8882232308,1.1301658154,1.4888341427,-0.5208579302,-0.5673535466,0.0692926571,0.2035409361,0.4037210345,-0.4218095243,1.1342378855,-0.4237312078,-0.3344818950,-0.0618350804,-1.1918430328,-1.4559003115],[-1.0172894001,-0.8198907971,-1.1291286945,-1.2913620472,1.1161898375,0.9469137788,1.3321481943,0.6899978518,0.6809046268,0.2032615840,0.0857793987,0.1699928343,-0.1400194466,0.4252460599,0.1930265427,-0.6893171668,-0.0112007195,0.5478432178,-1.6128880978,0.5252740979,1.9394230843,0.2731349170,-1.6321368217,1.1112415791,-0.9279717803,0.1051486731,-2.1612613201,0.8048269153,-2.2167458534,0.2070944011,0.3835910261,1.3284368515,0.1795494854,0.3918175399,-1.3203768730,-1.4642324448,0.3817415237,0.5377549529,-0.2622683346,0.6836646795,0.3972084820,-0.4240120351,0.0491578691,0.1181937605,-0.7205049396,1.6013940573,0.1794056743,-0.3333885372,0.8673188090,0.7273941040,-1.3895088434,0.8160765767,1.0607300997,2.6599748135,-1.0260976553,-0.4237062633,0.6025332212,-0.3183956444,-1.0793176889,1.1519999504,-0.4337845445,-0.2645676732,-1.4520622492,0.3616208136,-0.7904537320,1.5489475727,0.0706833676,-0.7320204973,0.5462500453,0.9109166861,-0.2380175889,0.8746619821,-0.5622177124,-0.7081284523,0.1509067416,-0.9439134002,-1.5680643320,2.3613348007,-0.4723900259,0.2858115435,-1.2789443731,0.4329091311,-2.0931222439,-0.1401796043,1.3160349131,-0.5561891794,2.8609473705,-0.9401915073,-1.1080731153,0.3751921356,1.6322127581,0.6731638312,0.0600589253,0.4372738302,0.4696765244,0.9339655042,-1.3887944221,-0.2863088250,0.7338685989,-0.1506843418,2.5223164558,0.8215770125,-0.2824140787,-0.1048718765,0.8483038545,-0.8062709570,-0.2444159091,1.4299122095,-2.1488876343,0.4855590165,0.7521313429,-0.0362242311,0.9838863015,-0.9353536963,0.0884423181,2.2920429707,-0.1700564772,0.2196080536,-0.7678773999,0.1562586427,-1.7027182579,0.6213787794,-0.9202602506,0.6407473683,-0.1986954063,0.6446263194,-1.6563129425,-1.0442481041,1.4251111746,0.0319977812,-1.0316035748,1.1146814823,0.9316073656,0.4386779368,0.3499840796,-0.3451722562,0.1396140605,0.6647171974,0.8718655705,-1.3025929928,-2.4178709984,-0.2907205820,0.3312182128,-1.2822395563,-0.1600407064,-0.1754976213,0.9503936768,-0.2703380287,-0.1025897637,-0.9934663773,0.0249258950,1.2744728327,-0.8118573427,1.1149960756,0.1397129893,-0.7670947313,-0.6668843031,1.3282564878,0.5447962880,0.5307455659,1.3833870888,-0.0055647413,1.0632278919,-0.5771430731,-1.2756719589,0.1890316457,2.1345887184,-0.8619757295,-1.0646456480,-0.5164789557,0.5170358419,-0.2600931823,-0.4217748642,-0.4166208506,-1.8200933933,0.1230064183,-0.0741620585,-1.4303330183,-0.4159705937,0.3553825617,-1.4155803919,1.0580601692,-0.1156407297,0.5948669910,2.3360552788,0.2601666749,-0.0094588660,-0.9592040777,0.6389791965,-2.1392457485,-0.7021299601,-0.5663596988,-0.7259340286,2.0994577408,-2.1976509094,1.4211015701,1.4037487507,-0.5544083118,0.4942214787,-1.3881099224,0.5758545995,0.5089108348,0.7930374742,-0.9582809210,0.7822327614,1.4710839987,-1.6592271328,0.6535981297,0.3041735291,1.0828449726,0.4052551985,1.3086524010,-1.1172833443,-0.6074464321,0.3001531959,-1.0475873947,-1.2174046040,0.0880611092,-0.3901859820,-1.6000955105,1.0908169746,-1.0064556599,-0.0125739658,1.0592347383],[0.4535114467,0.9690465331,-0.9640182257,-0.8309549093,-0.0977261811,0.3498162031,0.6293475628,-0.2722138464,-3.0948634148,0.9942316413,0.7735679746,-0.2930199504,0.3510787189,-0.7872864008,0.3636801541,-1.6712719202,-0.5944473147,-0.7780117393,-0.0843283460,-1.4538215399,1.6016918421,-0.7877728939,-0.3877354562,-0.1187206879,-0.0575074181,0.7608479857,-0.7298144102,-0.8270982504,1.1454995871,0.9177365899,2.6103880405,0.0275212266,0.3144996762,-0.1462692618,-0.9897634387,-0.5029512048,0.6674743891,2.3210890293,1.0447092056,-0.4356173873,0.7188963890,0.3660807908,-1.3855001926,0.8256156445,-2.0150454044,1.1064456701,-0.6010576487,1.2496664524,0.1507250816,0.6486197710,1.2184954882,-0.1510147303,2.0641293526,-1.4498168230,-2.0317718983,1.2131361961,0.8736751676,0.1112035960,0.0250869896,-0.3772203326,-1.6222440004,-0.5841495395,-0.4699107707,-1.3665386438,-0.2026459277,1.0634649992,-1.0488432646,0.2799955606,0.5228345394,0.5676972270,-0.7177141905,0.8343642950,-1.1448087692,0.4393841922,0.5103766918,1.5104135275,0.0295173712,-0.8243343830,-0.7828230262,-0.6596770883,0.6153479815,-0.2708790898,-1.9587074518,-0.3796555698,0.2415509522,0.5919256806,-1.1046775579,0.1845662445,0.2122246623,0.6119584441,-0.3259581625,-0.3889915943,-0.7639250755,0.6078098416,-0.0667440817,-0.3067077100,0.5324385166,-0.4453656077,-1.1343411207,-0.5743299723,-0.2151301205,-0.2456267476,1.0301148891,0.4816589355,0.2701259255,0.0924869999,-0.7086822987,0.3224029541,-0.8674822450,0.1477069706,-1.4585555792,-0.7812615037,0.7472718358,-0.2691109478,-0.2376997769,-1.3751879930,1.0826475620,-0.4113312662,0.2539284527,-0.1638429761,1.3389561176,-1.0764062405,0.5501658320,0.7915554047,1.5759218931,-0.6999486089,2.9310317039,0.1950336695,0.4499127269,-0.5996899605,1.4481815100,1.6616878510,1.1369335651,0.2755799592,-0.4852996469,1.3345595598,0.3228714168,-1.3306968212,-0.4727328718,-0.6471650004,-1.1132016182,-0.0713217631,0.3562543690,-0.3879558742,-0.8892503381,-1.2379481792,0.8939266205,0.3848524690,-0.0012620862,-1.1376874447,0.5002755523,-3.1173944473,-0.3279280066,-0.8613476753,-0.2599486709,-0.5934633017,-0.5903510451,0.8368667364,1.4305448532,-1.0941609144,-1.2185326815,1.2903679609,-0.1296792477,-0.4799862504,-0.5509805679,0.4149641097,-1.7689479589,-1.5645645857,0.7041118741,1.1712943316,-1.5331878662,1.7426551580,-0.9360130429,-0.0077561252,1.1543563604,-0.4171964824,0.8241519928,-0.5553875566,-0.1821218431,0.5545003414,-0.9405351281,-0.7519527078,-0.1459754258,-1.1258925200,-0.0645399243,-0.0919869691,0.4392310381,-0.3822971284,0.9538373351,2.5823388100,-0.0202496629,-1.1847659349,-1.6862163544,-0.3795819581,-0.3198311329,0.2588284314,1.2887588739,-0.1577174366,0.3362316787,-0.3093566597,2.0915386677,-0.1336469650,-0.7076271772,-0.2241379768,-0.9879319668,0.3666142523,0.5691226721,0.8202407360,-1.7439919710,-0.5116857886,-2.3288261890,0.0181529243,-0.2852157354,-0.6766232252,-0.6026852131,-0.2991698682,-1.4322966337,0.4033516049,0.4984943569,-0.6238273978,0.6512020230,-0.1294459999,-0.2563645840,-0.0850734860],[-1.6635020971,0.1540960521,-1.0692788363,1.2856413126,1.3886764050,1.2394807339,-0.3786866367,0.0405688733,0.0261330679,0.7440382838,1.1676402092,0.9192047715,0.3690302074,0.6918146610,1.0192283392,-1.1002117395,1.5993460417,0.0842537582,-0.2882339954,-1.6641160250,-0.2254856080,0.2086762488,0.1024906188,1.9029095173,-0.1601699591,-0.1618199199,0.0761273950,0.7380160093,-1.0263552666,1.2613258362,0.7727131248,-1.4276200533,0.2427285612,0.9233678579,-0.6908003688,1.1374328136,-1.5583342314,0.3230153918,0.5732840896,0.5374802351,0.0478959084,0.6080489159,1.9260646105,-0.1249577850,1.1183328629,-0.2025958151,0.9080084562,-2.0576729774,-1.0507560968,-1.0327750444,-0.2893113196,-0.5023987293,0.6811234355,-1.2514544725,0.3200329244,-1.2652056217,0.3359721005,-0.0252995491,-0.7358947992,1.0601046085,1.8516906500,0.8605123758,-0.3745788634,0.5313572884,-0.7876136899,-0.6817808747,-1.7882242203,0.7149001956,-0.8047255874,-0.2163498551,-2.7289826870,0.0985979512,0.5790400505,0.2990364432,-0.3108656704,0.4906449318,-0.0220372509,0.3153430820,-0.6366631985,0.5582022667,-0.0725143254,1.6017705202,0.3281800449,0.0548433438,-0.6032658815,-0.0313083753,-0.6846083999,0.8272629976,0.3764564991,1.2219976187,0.3424511850,-1.9520853758,0.5699878931,-0.2838295102,-1.2962739468,-0.3520423770,-0.0343406647,-1.7270351648,-1.5421413183,0.1779328436,-0.9161617756,-0.4136451185,1.1171588898,-0.4643569887,-2.5890705585,-0.0978169665,0.7960562706,-0.8719828725,-0.5912626386,0.6984806061,-0.5227940679,0.4376514256,-1.0884710550,-0.2189821005,-0.1773026586,-0.2215059102,-0.3727942407,-1.6285271645,-0.9006853700,0.0758861899,0.5086434484,0.3348540664,-1.3667118549,1.2991203070,-0.4577853084,0.2003161907,-0.0515099727,-0.8061420918,1.6344506741,0.0430850051,0.3335125148,0.4028064907,-0.2435404509,1.1255035400,-0.3624939919,1.3392149210,-1.1797457933,0.3497534692,1.7610549927,1.1116669178,-0.1403288543,0.5325508118,-0.4736567140,-1.1872699261,-0.8908538818,0.5292066336,-0.6279459596,1.1363736391,0.0741886124,-0.0612685420,1.8719406128,-0.7305373549,-0.7488399744,1.4618368149,1.0680894852,-0.5133447647,0.3789470494,0.6920509338,-0.0438605398,0.6057320237,2.3371551037,1.6273084879,0.4970181286,0.1759300977,-0.4805933237,2.0895617008,0.2881127298,0.1409717500,-0.9227795601,0.2344621420,0.1607084423,-1.3211661577,0.2385593355,-0.5965131521,-1.2087808847,0.2794382572,0.1888812184,1.9271669388,-0.6863006949,-0.7454478145,1.3094059229,0.0873842910,-1.5320618153,0.5683317780,-1.1412267685,-0.0090721669,0.4966467023,-0.9476978183,0.3827875555,-0.1609680057,0.6752560735,-0.7131679058,1.4439325333,0.8276761770,0.4239243269,0.0850653648,0.4328988194,-0.4906337261,0.0462160818,-0.0169979334,0.7315130830,-2.4731819630,-0.3385759592,0.8918678761,-0.3003723025,1.4887953997,-0.5745051503,0.1477561146,0.9416487217,-3.4703557491,-0.5359448791,0.7355026603,1.0419493914,0.2170319855,-1.0533621311,-1.3572306633,0.2210417986,-0.2556262016,0.8509920835,1.1125905514,1.2063499689,1.5522272587,-1.2215622663,-1.2061839104],[-1.2950669527,0.2025153190,0.8298991323,-1.6604008675,0.6038448215,-1.3068019152,-0.0830953941,-0.1207791269,0.1019082442,-0.4354915023,1.0499731302,2.1721775532,0.8277935386,-0.8255026937,-0.3824424446,-0.6085035801,0.4934102297,-0.3934786618,-0.3122988045,-0.6388264894,-0.4257539213,0.1983288080,0.1787884980,-0.6914410591,0.3757758439,0.3676111102,0.0018327866,-0.3925342262,0.0072206180,-0.7803044915,0.7397019863,-0.3460148573,0.6108574271,-0.1799765527,-0.5952509642,1.7402403355,1.2058306932,-1.7820718288,-0.4921284318,-0.1624327749,-0.4435231984,-0.9929452538,-0.2207475901,0.9404109120,0.5551370382,-0.2299775779,-1.3604174852,0.6192497611,-0.1024679542,1.5234620571,-1.3118313551,-0.4637701213,-0.3026833534,-0.6559758782,0.7063180208,-0.1408954412,-0.2510201633,0.9387018085,-1.2084950209,0.9204330444,1.0249712467,-1.1086518764,-0.6219455600,1.0969376564,-0.9379596114,-0.8758180141,0.6695581079,0.4155954719,0.1114221439,-1.5905003548,-0.6057395339,1.1781516075,0.5552685857,0.0532497801,-0.8565030098,-0.0965542048,0.4256413877,0.5131956935,0.7127240300,-0.9340515137,-0.2032660544,-0.7067037225,0.8651694655,0.4279882610,0.0549685322,-0.3155857027,0.9081064463,-0.7040295601,0.5488665700,-0.7164933681,0.6457732320,-1.0947531462,0.5694273710,0.2458718419,-0.9906666875,0.2122940868,-0.1173533797,-0.4974184334,-0.6052320004,-1.2996211052,1.6052107811,0.0167007670,-0.8896363974,-0.1235675514,1.2180057764,-1.2930305004,-0.0934745148,-0.8663709760,0.5964619517,0.0963961259,0.7674152851,0.8012778759,0.5961014032,1.0097689629,-0.7147139311,-0.8749305010,-0.6941998005,0.9761347175,-0.4838216305,0.8083631992,0.0955365077,-0.7880268097,-0.0426202081,1.8439741135,0.4282551110,-0.0741682723,1.0584558249,-0.6867261529,0.8829703331,0.5617801547,0.3434916139,0.8344439864,0.4797274470,-0.3557169139,-0.3400082290,-1.2465457916,-0.0580307133,0.3834264576,-0.2598493397,0.3235251904,1.4750258923,1.1747642756,0.3843635023,-0.6985956430,1.5027586222,1.4956634045,0.8067368865,0.1969479024,0.0862167776,2.6182115078,1.2178329229,0.4351801872,-0.2157924771,0.0267683528,-0.2018048316,-0.2664071023,2.3273704052,-0.2287864387,-1.2879660130,-0.2599785924,-0.9273023009,0.5641449690,0.7914847732,0.9565588236,-0.8911844492,-0.0868590698,0.8247932792,1.1331245899,-0.7190009356,-0.5144978166,-0.2154463977,0.4985496998,0.9098943472,-0.2854295969,1.0122947693,-1.0168125629,-1.7378758192,1.9295582771,0.2372324765,1.4199649096,-1.4571065903,0.9987430573,1.4982053041,-0.1646643132,-1.0409728289,0.2996204197,-0.3438556194,0.1763267219,-1.2381007671,-0.5269424915,1.0394705534,-0.5188234448,-2.2872128487,-0.8193461299,0.7512777448,0.5032290816,-0.2287727147,0.0983647630,-1.3239839077,-0.3555992246,0.3286261261,0.3280561864,-0.0815626234,-1.0128548145,-0.6927438974,0.8964392543,0.8159549236,-0.3390179276,-0.3015466630,-1.4631524086,0.4497293830,1.3367420435,0.3859525323,0.7919450402,-0.7560143471,-0.4574146271,-1.0860054493,-2.2208795547,-1.6294802427,-0.0413829871,-0.2428517640,-0.4857134819,-1.1662257910,-0.9579188824],[0.7621256113,1.3296331167,0.3138369620,-0.4716928005,0.6125260592,0.3406782746,0.0522802398,1.0202609301,-0.0340798423,-0.9259276390,-1.2390106916,-1.5818425417,1.1560853720,1.0905010700,0.7594812512,-0.5976432562,-1.1142151356,1.5844764709,-0.3622460365,-0.1226565912,-1.9871875048,1.4768576622,0.2379653603,-0.9133228064,-0.7880818844,0.9237326980,1.8276497126,0.7280939817,-0.5917063951,1.3107105494,0.7642640471,0.8204598427,0.5962021351,-0.3783559799,-0.4161889851,-1.4495459795,0.0745454207,-1.3453316689,0.9977020025,-0.4342275560,1.0422127247,1.1708983183,0.4311306477,-0.6837021112,2.1621861458,-0.5827428102,0.4477067292,-0.0211558007,0.1824309528,0.4264740944,0.1203467995,0.9909948707,-0.1859768480,-1.0240132809,-0.3639334142,-0.0936418921,1.0512423515,0.3733913898,0.6839901805,-1.9282292128,0.5622627139,-1.2091028690,-0.1106549725,-1.2581288815,0.8361244202,-1.1004523039,-0.1258943379,-1.5989625454,1.8102275133,0.4441590011,-0.5214574933,0.9797497988,1.2888114452,-0.8276115656,-0.0220992491,2.4498775005,-1.8377934694,-0.8514962792,0.9489679933,-0.0830500126,-0.0359301940,-0.1420495808,1.0556294918,-0.5497905612,1.0126923323,-0.7473862171,0.4052971900,-1.0751425028,0.1719949245,0.3107260466,0.4252511263,1.0308989286,2.1891362667,-0.7951731682,-0.9288254380,-0.8406194448,-0.5870738029,1.4166525602,-0.0508782193,-0.5263801217,1.7727472782,0.1618817300,-1.6156774759,0.9979695082,-1.5955677032,-0.1549735814,1.2778108120,1.1898374557,1.2261581421,0.1232592538,-0.4701025486,0.8268409967,1.4406825304,2.0723600388,0.6214050651,0.1583388299,1.0687154531,1.2600852251,-1.7723824978,1.7461088896,1.7796293497,0.1533010453,-0.4502541423,2.8840515614,-0.0278098118,0.2774529457,1.2356098890,0.9059458971,1.3719466925,-0.1427477598,-1.5313657522,-0.1988708526,0.0918548554,-0.7983700037,-0.7351155877,0.0807933062,-1.1822363138,0.4204095602,1.6773831844,-0.6493386626,-1.3810855150,1.8829346895,2.2381758690,-0.3106609285,0.1599945575,0.5009009242,0.2571044564,-0.2493799925,1.4916465282,-0.0642954409,0.1002008095,0.0911449566,-0.2840108871,0.0648470521,-0.5927429795,0.6111506224,0.2633831203,0.6087102294,-0.1314200610,-1.9108045101,0.3572774529,-0.2191513479,0.4376700521,1.1833018064,-0.8609423637,-1.1047463417,-1.0791573524,0.6086138487,2.6700644493,-0.7331061959,-0.4262737930,-0.6036394238,-0.7591724992,2.0492615700,-0.4318020344,-0.4356160462,0.4063673317,-0.0951606557,0.1538648903,-0.7921331525,-0.5645645261,0.6250701547,0.2017248124,1.6619211435,-0.0490566455,0.8955783844,1.0819174051,-0.8749611974,-0.5747096539,-1.3129776716,-0.8326802254,-0.5609090924,1.6952314377,1.0350174904,1.0616799593,0.9478091002,1.2279448509,-0.0191568751,-0.1780776232,-0.5881857872,-1.0479393005,0.2833366990,1.5692890882,0.0149694970,-1.1679285765,0.6675914526,0.2725379169,0.5069898963,-0.8566572666,0.2730761468,0.7228215337,-0.7045961022,-0.8416188359,0.4508380890,0.1944348961,-0.4541225731,0.7563244104,-0.1131819710,0.7128592134,1.4402623177,0.1528645456,1.0313960314,0.6298319697,1.2467131615],[-1.1085821390,-0.9003415704,-1.4343361855,1.0231429338,0.0072061568,-1.8835331202,-0.7989682555,1.0520348549,0.3447473049,-2.5748610497,1.6703809500,0.0138393026,1.1409097910,-1.1738827229,0.7495995164,0.5710754991,-1.0748022795,-0.6461317539,-0.2453123629,1.1448947191,0.8853188753,0.7727425098,-0.0282142591,-0.1457471848,2.4659855366,-0.7119729519,-0.1440470964,0.3802301586,0.0594930463,-0.7933204770,-1.5470353365,0.9903058410,-1.0986013412,1.3785964251,-0.6182013750,-0.2940633595,-1.6926203966,1.8205765486,-0.0862483159,1.7647430897,-0.7206363082,-1.1045497656,-1.6156738997,-1.0962505341,2.8444337845,0.5576008558,1.6882987022,-1.1970424652,0.4513413310,0.9049255848,0.2253855616,-0.6835092306,1.8480138779,-1.3671431541,-1.0080400705,-0.9925702810,1.2446583509,-0.5805303454,-0.1164868474,-0.5888524652,0.2782382667,-1.0922057629,0.9598587155,-1.0899872780,-0.7082154155,0.3065707684,-1.5383666754,2.1593325138,-0.5230810642,1.0688405037,0.5502248406,0.2891297638,0.3945007026,-0.7157661915,-1.4161725044,0.5869718194,0.1101675332,-0.9291577339,0.9818926454,-0.2781111300,-1.0967657566,0.1139538214,1.1031048298,1.0086518526,0.6278707981,-0.1493199915,-0.4495257139,0.2201409191,0.4300591052,1.2368929386,0.6283149719,2.0061483383,0.7451803088,-0.2425652146,-0.2525975406,1.1054331064,-0.5970991254,0.9908251166,0.4830693007,-2.2409613132,-0.8885762095,-0.0143143935,1.0882319212,0.2041956186,-0.9607824087,-1.1832842827,-0.3721539676,-0.3367851079,0.2151497006,0.0878699198,-0.6686208844,-0.3243316412,-0.5673260689,0.2042728662,-0.0433752351,-1.7372738123,-1.4927453995,0.2947275341,1.8073326349,-2.2004306316,-0.1771846712,0.5147547126,1.0263893604,-0.3667627871,-0.3732538819,-0.8672274947,-1.5185539722,0.1291330457,-2.3146927357,-0.0381119587,-0.6768706441,-1.1104094982,0.1284427047,0.5935358405,-1.7596523762,1.6137292385,1.4737104177,-0.3111882508,-1.1401060820,1.3278982639,-0.7087484002,-0.5006707311,0.9336168766,-1.2285281420,1.1852614880,-0.6464233398,0.3168686628,1.0129629374,-0.9842917323,-0.8322681189,0.3770053089,0.4055832326,-0.7535272837,-0.5763852000,1.2844207287,0.2033589929,1.0656037331,-2.3893010616,0.2218722403,0.5269311070,-0.5230823159,0.2871558368,-0.6817102432,1.0522680283,-2.2263123989,-1.4442704916,-1.7066681385,-2.1534831524,-1.1033091545,0.5186153054,0.9830374122,-1.4134610891,0.7257140875,-0.3652280867,-0.8273479342,-0.0123440940,-0.7055724263,-0.0065825772,-1.4281263351,1.0454120636,2.1973111629,2.1394906044,-0.2185653597,-0.7726922631,0.0578657836,-0.1241003796,1.3615926504,-0.4700821936,0.6276119947,0.7610633373,-0.3084032834,-1.5049169064,1.1694749594,-0.3002104461,-1.7592983246,-0.7689751983,-0.6482440233,-1.3943746090,-0.6685357690,-0.2740963697,-0.1009262577,1.3430944681,-0.9573187232,-0.4756442606,-0.1120443791,0.4514842033,-0.1808938533,0.0360276587,0.7046375275,0.5217391253,0.1769850850,-0.3516299427,-2.1484513283,1.0110119581,-2.1996293068,0.8975638151,-1.2163029909,-1.0508220196,0.6954064369,-1.2345478535,0.6365551353,-0.1546197087,0.5181404352,-1.8113756180],[-0.5626225471,-1.3982005119,-0.4505920410,-0.2327025235,-0.0388276242,1.5718796253,-1.1048280001,-0.1993524432,0.4389081597,0.7646227479,-0.5730227828,-0.5616265535,-0.3296748400,-0.7627649307,1.8219025135,1.0186827183,-0.7380231023,-0.8434823751,-0.6264718771,0.7264119387,-0.5833688974,-1.6604254246,0.0730206072,-2.4509689808,-0.5503947735,0.6281988621,0.3269348443,0.0925165340,-0.4955527484,-0.2733169198,-0.5889083147,1.7769594193,-1.1998845339,-1.7700980902,-0.8073721528,0.0494673774,-0.8524341583,-3.2187163830,0.5676971078,-2.2943770885,-0.1579331160,-0.7956486940,2.0401656628,-0.1461989731,2.1896297932,1.5324863195,-0.5835031867,-1.5549209118,-0.4209410548,-0.0279977787,-0.0782606602,-0.7426626682,0.0121981045,-1.5553230047,0.5819070339,0.0326453373,0.6056950092,-0.4712967277,0.4049454927,-2.1062376499,-1.1619187593,-1.2910219431,-0.6611953378,-0.4365321100,1.7293006182,0.4786488712,-0.2935953736,-0.4926992655,0.7536605597,-0.3894644082,-0.7081032395,1.0595657825,-1.0791609287,-2.6619422436,-1.6152977943,-1.1396865845,-1.2950086594,-1.2873415947,-0.1941757053,0.5377328396,0.8336847425,1.3057926893,0.7935741544,-1.1920603514,-1.2580881119,-0.1752467602,1.4270538092,0.3531105220,-0.2798886299,0.2382898629,0.5952281952,-1.3373407125,-1.5224821568,0.6214637756,1.1671440601,0.1557538509,-1.3565181494,1.1534074545,0.0552476197,0.8227424026,-0.2473755926,1.7100614309,-0.6010099053,0.0952295437,0.7519122362,0.6125386953,-0.1340970695,-0.2649757266,1.2478978634,1.3848875761,0.2351858020,-0.7321636677,-0.7952386737,0.4714525342,-0.7676478624,1.6104269028,1.1910309792,1.6014332771,0.8937367201,-0.5093985200,0.7270994186,0.1354524046,1.1415119171,-2.6321196556,-1.8495267630,-0.5692487955,-1.5690562725,-0.2997058034,0.9866194129,-0.0548178814,0.1174306124,-0.1186611429,-1.4897632599,-0.0651347712,1.4272356033,0.8654854298,0.3896121979,0.0904301852,0.0244684536,0.9569140673,0.5948505998,-0.6984776258,-0.2093743682,-0.1100684255,-0.4356186390,0.9230844378,-0.0802567452,1.0543168783,0.8095011711,-0.9505899549,1.5887154341,0.9821254015,-0.4611521661,-1.1893174648,1.1560828686,2.1422307491,-0.3830331862,0.2881630659,-1.0069626570,0.9505732656,-0.6370865703,-0.5005722046,-0.5218068361,0.0570890643,2.1391377449,-0.3942484856,-0.0125554344,-1.0029050112,-1.6481519938,-1.3115295172,-0.6975313425,1.4723504782,-0.5125975609,-0.7702323794,-0.0430547260,-0.0723921731,0.3706169128,0.9811360240,0.0741774514,-1.2132822275,-3.3102068901,0.2480957657,0.1500856727,-1.3044787645,-0.7662258148,-0.5189638138,-0.7032186985,2.0692028999,-0.4092837572,-1.9459711313,-1.0947782993,-0.8841983676,-0.0839635059,-0.8565853834,0.7222669721,1.4330573082,2.0029449463,0.6141778827,0.4747509956,-0.5246588588,-1.0898851156,1.2181375027,-0.7382065654,-1.0789153576,-0.4607920051,-1.2100571394,-1.8802586794,0.2697679102,-0.0484813824,-1.1286646128,-0.0754335970,-0.4993629456,-0.9535911083,-1.2571040392,-0.3249886036,-1.7524542809,0.1891974956,0.5681154728,-0.1350892484,-0.1838773489,-1.8538595438,2.1625046730,1.0804477930,0.8050185442],[0.0583708286,-0.8403818011,0.4379777312,0.2320236266,-0.5907236934,0.5305605531,-0.2355878800,0.8831157684,-0.3832046986,0.1206885651,-0.2441270649,1.4652829170,0.1251680106,1.6854983568,-0.5830694437,-0.8190350533,1.2951843739,1.1042377949,2.4435899258,0.6359903216,1.0345590115,0.0918972865,1.2627774477,0.6608592868,-2.0900311470,0.3947992027,1.1637389660,-0.5153884292,-0.8478496671,1.4432096481,-1.4082688093,-0.2457654625,0.3125251830,0.4924857020,-0.0458401740,1.0880835056,2.3190202713,-1.3728237152,0.9819077253,1.4168225527,-0.8269272447,-3.1057460308,-0.2784600556,0.0084761279,1.1402667761,-0.1192869022,-1.1153726578,0.4603817463,-0.4175572991,0.5515987873,-0.8892507553,-0.9335702062,0.1421700418,0.8622007966,-0.1144793853,0.7831965089,-0.8715204000,-0.0539018773,0.7396158576,-0.2736086249,0.1636993736,-0.3604790866,0.6992456913,-0.4008758366,-0.5472824574,-1.3291764259,-1.1045629978,1.3297357559,-0.8264792562,-0.6032384038,0.1439969242,-1.2441627979,0.7882626057,0.2286830693,0.4731123149,1.1700091362,0.3139848411,-0.3568086028,0.3434693217,-0.7872848511,0.1237977520,-0.9416929483,0.4094843268,-2.0613017082,0.8186947703,0.2799305320,1.6776723862,-0.9916943312,-0.6852881312,0.8760497570,0.3148263097,0.6045241356,-0.8686185479,-0.6712832451,0.0449492224,0.1750403196,-2.0695581436,0.2488401681,-0.4023043811,-1.9713195562,0.9366938472,0.3973978460,-0.8258979321,0.2593599260,-0.4171347618,-2.0138168335,-0.2195019126,1.3088977337,-1.0805025101,-1.2335897684,1.7145081758,-0.7687286735,-1.5577871799,-0.7377884388,-1.0902771950,1.3249225616,-0.0432096124,-1.5101686716,-1.4007518291,-0.0030049440,0.5537450314,1.1865566969,-0.5845212936,1.2676486969,0.6573771834,0.4485799372,-1.3348814249,-0.2957971096,0.7416982651,-1.2257262468,1.2528117895,-1.4885023832,0.5322642922,-0.6019110680,-1.2413516045,0.3678843975,0.5308324695,-0.1434124857,-0.0195947830,-0.3229777813,0.5714412928,1.2527539730,-0.1473772079,-0.0419942997,0.0068792035,0.9574351311,-0.2035091221,-0.0903420448,-0.3691366613,-0.5072920918,-0.5398548245,-1.3546386957,-0.6282287240,0.5655155778,-0.5688884854,-1.9287480116,1.2996890545,-1.3261818886,0.2849215567,0.6106883883,-0.1060630158,-0.4809936285,0.0340173580,-0.9125748277,0.7783385515,-0.1310227066,1.2201029062,-0.3814059794,1.3374848366,-1.2288342714,-0.1551678479,1.0072926283,-0.6144304872,0.0839847699,-0.6462374330,-0.0756285638,0.7238970995,0.2455556691,-1.4122244120,-1.8843190670,0.2838617563,1.0761822462,0.2097860724,-2.2004213333,1.2754834890,-0.4360707998,0.8155106902,-0.0288391188,-1.7672243118,0.2476938218,1.0521885157,0.7191450000,-1.6686151028,0.0634164885,-0.3962253332,-0.9131394625,-0.3257185817,0.9493992329,-0.1860646755,0.4772316515,0.7308128476,-0.3646406233,-0.1680960655,-0.6723977923,1.8579225540,0.3541227877,1.3605973721,0.8575440645,-1.5376170874,1.1852170229,-0.8388038278,0.6530769467,-0.5707948208,0.1520410031,-0.5276001096,-0.6008660197,-0.7531970143,-0.1710914224,-0.0899885967,0.5186805725,1.0073469877,1.0555337667,-0.9564317465,1.0060201883],[-0.3865070343,-0.2376815975,-0.2118640542,0.9394926429,0.6976744533,-0.2015943229,-0.9899131656,0.5405406356,0.8071326613,-2.0545885563,0.4489324391,-0.7159293890,0.7658372521,0.8533259034,-0.2951918840,0.5808369517,0.2168655097,-0.5482198596,0.3273206055,-1.5042330027,-0.4117659926,0.0382566266,1.1967242956,-0.8289934397,-0.5600231290,1.0414994955,-0.1932150424,0.0411262065,-0.4130541384,-2.0425164700,0.2735721767,-1.5550554991,-0.7226807475,0.7977443933,0.1485011876,0.9446786046,-0.4927309155,0.6173900962,-0.3243460059,1.2881433964,2.2453174591,-1.1089458466,0.6956653595,-0.9906356335,-0.0505813882,0.3978154063,-0.9886980057,-0.2622124255,0.0275835395,-0.3144522607,0.9326926470,-0.3414547145,0.4837387204,-0.2374359816,0.3685643673,-0.8963376880,0.6249161959,-0.2738130987,-1.0066910982,0.5856098533,-0.2894586325,-0.0821105018,-0.3100988269,-0.4248235822,0.2665741146,-1.7740272284,-0.1976617873,2.3445346355,-0.7961371541,0.3188365102,0.3922545314,-2.0804495811,-0.6469821334,0.7504340410,1.1765389442,0.9240966439,-0.3588912785,1.2186727524,1.1245067120,-1.7694383860,-1.4732788801,-0.2784310579,1.3830229044,0.1213222444,-1.1693680286,-0.7890093923,0.2241205424,0.0043442440,1.6878660917,-0.1504489183,-0.2639863193,-0.0475605465,0.4061010182,0.6480402946,0.9678143263,0.8842444420,-0.2457452416,0.9848177433,0.4075931609,-0.6050483584,-2.1982643604,-0.9470968246,0.6483018994,-0.9678875804,-1.0634075403,0.1625384986,1.2762326002,-0.8872547746,0.1652131081,0.5170665979,-0.0231691115,0.6141547561,0.3632443547,1.0437520742,-0.4099225700,-1.5331041813,-0.1598736197,-0.8773027062,1.1364567280,-1.7034946680,0.9508250356,0.0975447968,-0.6070606709,1.7351464033,0.4743512869,1.2093871832,1.2255852222,-1.2209358215,-2.1459023952,-0.5637799501,-1.3911374807,0.4593183398,0.5979510546,0.4527249634,-0.0445172526,-1.0204105377,0.6146193743,0.1513276845,-0.5046305060,1.2186865807,1.6874295473,1.6179510355,1.0376192331,-0.0943819210,2.2098233700,-0.5462614894,-0.8779112697,1.2222505808,1.2055162191,-1.5389888287,1.5902979374,-0.4417829216,1.7803531885,-0.5176959634,1.8613972664,0.7545554042,0.6129772067,0.4468424320,0.9953684211,-1.3956910372,2.0089464188,-0.4462093711,1.5784149170,0.3510382175,-0.0703873858,0.2534176111,-0.1998426467,-0.8111057281,0.3384461999,0.6583834887,0.8084661365,-0.9903705120,0.4196079671,0.3588507473,-0.5640988350,-0.1225297973,0.2798833847,2.6040446758,-1.2985949516,1.3740561008,-0.1327347904,-0.3478104770,0.6357547045,-0.8267017603,-0.0101069696,1.1574569941,-0.5891892910,-1.2464402914,-0.3473787606,0.3087954521,0.2835386992,-0.4206701517,0.2591592669,-1.2195488214,-0.1933093965,-0.5647271276,-0.8968281746,0.1931795776,1.3121608496,0.0971596316,0.9895777106,-0.4723748267,-0.5463014245,0.0892185643,-0.0195480473,1.8247439861,1.5372071266,0.0110428948,1.2742878199,-0.2674755454,-0.3730522692,-1.5067985058,0.4618104398,-0.1699099392,0.7135494947,0.6154643297,0.4335562885,-0.2648655772,-0.0513244420,0.5973302722,2.1014957428,0.2783280611,-0.5136555433,-0.7486355305],[-1.0634144545,-0.5323330760,-1.5201735497,0.5931404233,0.4664847553,-1.2523989677,0.6027503610,0.0915953219,-1.3840013742,-0.8534832597,-0.3615029752,-2.4294033051,0.1112238094,0.0172027890,-0.2286780924,-0.8285278082,2.2321007252,1.9687647820,0.2778130472,0.5375308394,-0.8006395698,0.2198387384,-0.3095584214,0.8869483471,-2.8386275768,-0.9168944955,1.2239718437,-1.0722330809,-0.5158202052,0.6645996571,0.8560354114,0.1393965185,0.9551382661,0.4548152685,-1.9653131962,0.8772224784,-0.2593773305,-1.1952117682,-0.3621657491,0.0389585756,1.1604454517,-1.0822688341,0.5837280154,0.5754579306,-0.5814859867,-0.2461313754,0.0065278327,0.4200200737,-1.5244244337,-0.2629826665,1.3117327690,2.0171401501,-0.8675763011,0.2327602357,0.1637782604,0.4368565083,0.9518216848,-0.7166954875,-0.9328627586,0.8369590044,0.3547633886,0.1785100549,-0.0237964932,0.8859897852,-0.6042553782,-0.3762065172,-1.6474996805,-0.3082104325,1.0396336317,-1.5162066221,-0.9067734480,-1.0850670338,1.2581267357,-0.2214823216,-0.8220285177,0.4123277068,0.7663524747,-0.8339322209,0.6646717787,-1.0077667236,-0.5644736290,-0.9657685161,-0.0941603929,0.4566901326,0.3407334983,-0.7013095617,1.0366376638,-0.1873145849,-0.3645239472,-0.5983661413,-0.0358053558,0.9808912277,-0.3805266619,2.4766175747,0.9397842288,0.6496300101,0.0903372765,0.1319412291,-0.3573524952,0.7131528258,0.7749145627,1.8421769142,-1.1503663063,1.0779078007,2.4074707031,-1.2194875479,2.3397269249,-1.4365357161,0.4038117230,0.2114246339,-0.7000735402,1.7133610249,-1.6811267138,-1.3434336185,0.3275044262,-0.9965205193,0.6525971889,0.1109063923,0.5218975544,-0.2513494194,-0.1506602168,-1.0675710440,0.7060869336,-0.2939028740,0.7074757814,-0.1937825084,1.8526563644,-0.6828480363,0.1911466569,0.1817938238,-0.8393192887,-1.2712761164,-0.2967023253,-1.5580698252,1.0618728399,-0.7327790260,-0.0453399830,0.7231896520,0.2260579467,0.0045189788,-0.0292946324,-1.5553299189,0.3317606151,0.7949090004,0.6328572035,0.4006032944,2.1425490379,-0.6258561015,-1.3994067907,-0.5250332952,-0.5599845052,1.0373660326,0.8478179574,0.0953366831,-0.6932147741,-1.3818480968,1.0562552214,-0.4130301178,-0.1007573754,-0.6099736094,0.2333106846,0.9275327921,-0.7962839007,-0.6180340648,-1.5996370316,1.8704158068,0.8759252429,1.3884956837,2.4303166866,0.4895366132,1.3197805882,-0.1183772013,-0.1369920820,1.3485703468,0.6114919782,-0.0323543325,-0.6448656321,1.2038643360,1.1820948124,-0.2048023045,0.6790464520,0.4966803193,1.5222219229,-2.9318621159,0.4659831226,-0.9096521735,-0.1747777015,-0.0647855178,-0.8331367373,-1.5714011192,-0.2310294956,0.4662609696,1.3658882380,-0.5742117167,0.4292911887,0.5996888876,-2.3400518894,-1.5466567278,0.7606943250,-0.7578786016,-0.2359795421,-1.0300453901,0.1930342764,0.0036071180,-0.0514582843,-0.4616616070,1.7474654913,-1.2599040270,0.9778512120,0.3718870580,2.0573425293,0.0001507712,-1.3301340342,1.2774391174,1.6158695221,0.9101704359,-0.3199186623,-0.4357638061,-1.6184130907,0.4685378373,0.7183168530,0.1964053959,-0.0726887211,-1.1159377098],[0.5207815170,-0.9250515699,-0.8146038651,1.5330342054,-0.6654334664,1.2355058193,1.2149068117,-0.8081433177,-0.1008972526,-2.6701030731,0.1122501418,-0.1568444520,0.6637252569,0.8165430427,0.2798295319,-0.7888636589,-1.4502432346,0.2165211886,-1.3123550415,-0.0796058998,0.9572609067,-0.3779307306,0.8503680825,0.4904717207,1.5245455503,0.1286518425,-0.3477151990,0.2938793302,-0.6671968102,-0.0250478294,-1.3159838915,-1.5637085438,0.7293962836,-0.0600265227,1.3219997883,-0.7478125095,1.6180843115,0.7206456661,1.0755368471,-0.6887121797,1.2116334438,0.1095357239,1.8574286699,0.5284230113,-1.0005639791,-1.7658504248,0.8115992546,0.1519138515,-0.8634884953,0.0946448296,-0.6255034208,-0.5461331010,1.1926347017,1.6944109201,0.7403153777,-0.4044455886,0.7798045874,0.7493108511,0.6431380510,0.1134475395,0.2037412971,0.2755376995,1.1141532660,0.9121242166,1.5169768333,-0.9620696306,-1.9707722664,0.8506955504,0.3520783484,1.6420823336,1.1664842367,0.0395719223,0.5504347086,-0.5065367222,-0.7530210018,-0.1424470097,0.3533922434,-0.5853402615,0.2046187520,-1.9142156839,0.9022826552,-2.0429213047,-1.1913815737,0.2445087582,0.6097036600,0.3708125651,-0.1089679971,-0.4501316845,1.0040342808,-0.7449307442,0.3174760342,0.1585830599,1.1094741821,0.2051526755,0.2611257434,-0.9332591891,0.5158112049,-0.0843781233,-0.5690991879,0.9467744827,1.3892228603,-0.1915261894,-1.4694588184,-0.6829240918,1.8040553331,1.4982194901,-0.6782018542,-0.2488974929,2.0247976780,0.7601909637,0.2023352534,-0.0089879557,-0.3011207283,1.2983838320,-0.6964088082,1.2567225695,1.4973249435,-0.0151675921,1.1256455183,0.5787013173,-0.7115837336,0.4392588437,0.6161403656,-0.3426590860,0.4325609505,-0.0037368375,-0.2693090141,-1.4960060120,1.1586462259,0.7610878944,-0.5264061093,-0.7910407782,-0.3750860989,-1.5029484034,0.7323586345,-0.9829524159,-0.6052945852,1.6173005104,-0.3960254788,-0.3167100251,1.4021726847,0.1614335328,-1.7691595554,0.4891624153,1.7525867224,-0.6651031375,-0.2736934125,-1.5221942663,-1.3538130522,0.9329830408,1.7201684713,-2.1362659931,1.3983808756,1.0712672472,0.8699352741,1.9718719721,2.8391821384,1.1305669546,0.1621609330,-1.0223643780,0.1640578508,0.3482623994,0.0992343724,0.9226043224,-0.5990541577,-0.5642827153,-0.3903325796,1.5199590921,1.2961598635,0.4010730088,-1.3141818047,0.6916372776,0.0341012813,0.2534821630,-1.5459098816,0.7012271285,-0.9198668599,-0.3922315240,0.5712463260,1.0828225613,-1.4264631271,1.8115274906,-1.2499611378,0.8060184717,-0.5210914016,0.2651344240,-1.2977297306,-0.3151990473,-1.4129822254,-0.3059104383,-1.2792009115,-1.3503084183,0.3274895549,-0.8965561390,0.1319619566,0.2559824288,-0.8015326262,-0.8983778358,-0.8235607147,-0.4976842105,0.9133901596,-0.9979527593,-0.0663572326,-0.1228386909,-0.3515924215,-1.6459765434,-1.1646555662,-0.5235185623,-0.3147287667,0.8435072899,0.6037076712,0.1352328509,-0.9873789549,-1.3439757824,-0.5656504035,-0.7461355925,0.1942629516,0.4915701151,0.9518358111,0.3652679920,-0.8218551874,0.0957542583,-0.4124514759,-0.3918253779],[1.0724167824,0.1508688480,-0.1930464804,-0.3708569407,-0.2351885140,0.2533005774,0.3928980827,0.1800833941,-0.1871147901,0.1421864182,0.8545504808,-0.2546977401,1.2093913555,-0.0946916267,0.4536883533,1.8203556538,-1.4522353411,1.0122228861,0.3969878256,0.8579048514,-2.4725697041,-1.1170048714,-0.3419044316,0.2210522890,-0.0589853376,2.0522193909,1.3139799833,0.0032415872,0.9737606645,-0.3109631836,0.6522809863,0.1454812586,0.3584970832,-0.6686333418,1.0057399273,-0.4018355012,0.4271542430,0.7752248645,0.7583116293,-0.0558623113,1.9829217196,-0.0791059658,-0.1452606767,0.6852729917,1.0473220348,-0.4100103974,0.8070753813,-0.5444563627,0.5194547772,-0.7474954724,0.0800675452,-0.4973312616,0.7176632881,0.3519951701,1.1283892393,1.4414153099,0.7251556516,-0.2250899971,0.7314395308,-1.5019141436,-0.7311096787,-1.1236633062,1.5846997499,0.3824317753,-0.8789307475,1.9165765047,-1.2047196627,1.9510960579,0.1713950783,0.2518723309,-1.0073115826,0.0976815745,-3.2116572857,1.0625407696,0.3055147827,-0.1818564683,2.4210405350,0.2135395557,1.0038594007,0.2944985330,-0.1435895413,-1.9485757351,-0.3004073203,0.4409326613,0.6442545056,0.5157012939,-0.5109453201,-0.3380121291,0.4624716043,2.5077555180,-1.0816066265,-1.3965083361,0.6884524226,0.2567251027,1.2027677298,0.2761328220,-1.8016265631,0.4354678392,-0.4259582162,-1.4475284815,-0.4208916426,0.9427179694,0.4259283841,0.0079847854,1.1683820486,0.6213946342,1.0260797739,-1.8386904001,0.5446011424,-2.1929266453,0.3169703484,0.1466955394,0.2231619656,-0.5830193758,-0.9312384129,-0.4481034577,-0.4660280049,0.0486355685,-0.0424672253,1.1460434198,1.0676603317,0.2260905653,0.7005288601,1.4022908211,0.0001941096,-0.3831669390,-0.0770072937,-2.1831009388,-1.5603170395,0.4208289385,-0.9125955701,-0.5532537699,1.8200123310,1.6134670973,2.5783603191,-2.3645694256,-1.2364205122,-2.1010518074,-0.3773110211,0.3891334534,-1.8309109211,0.1862813532,-0.5885137916,0.5469305515,-1.2611180544,0.2711682320,-0.2295575142,-0.8624431491,-0.2658065259,-0.5441913009,-0.6111053228,0.3815867305,2.4132184982,0.2860789001,-0.9591971636,-0.2013577223,-0.2921943665,-0.5352737308,0.2526368499,1.0676268339,-0.0170707032,-1.2185895443,-1.6883766651,-0.0279599130,-0.4728796780,0.4600687027,-2.1366546154,-0.6944203377,0.4398227036,0.8040372729,0.6955130100,-2.2377107143,-0.1955517530,-0.8615193367,0.0634703040,-0.3660758734,0.1347871870,0.2589198947,1.4126286507,0.6336156130,-0.7764823437,1.5431591272,-0.7278642654,0.6717827916,-0.5883294940,-0.9275021553,-0.5374195576,1.4585071802,0.4842028618,0.7848344445,0.1956220865,-1.4329868555,1.4823243618,0.9601696134,0.3370892107,2.0254490376,-0.1215440184,0.3890198767,0.9085683227,2.1455557346,2.4146547318,-0.2953370810,-0.1564641893,-1.0599550009,0.9626013637,0.0099062221,2.0847289562,-0.6303656697,1.3723542690,-0.7465828061,0.3475526869,-1.6720502377,0.4044115543,-0.0562933162,-1.5195279121,0.7755575776,-0.8021618128,-0.2751158774,-0.7458245754,0.1956940740,-1.0238747597,-0.9840406179,1.0258538723,2.0173861980],[0.1315372139,-0.5138539672,-0.5669877529,0.5406587720,-0.3071814775,-1.9967687130,-0.4333548844,-1.9966330528,-0.4374782145,1.7048588991,0.3344865143,0.3502452672,-1.5361242294,-0.2268669605,-0.4311481118,1.0138232708,-0.5507349372,0.5594305992,1.0659108162,1.4947098494,-0.7976589203,1.1181552410,1.3706774712,1.2994995117,-0.1372869313,-0.0259376168,-0.3461629748,0.4131499827,1.1723839045,-0.2778978348,-0.5928153992,0.0675194412,-0.8313975334,-1.1776564121,-0.4460479021,-0.6112349629,0.4268786907,0.8870689869,-0.6189897060,-0.1257660389,-0.9239557385,-1.8860009909,1.1535906792,-0.8335864544,-0.6902356744,-0.1756162494,-0.7561747432,0.5024242997,0.5363253355,0.0824064240,0.2639508545,-0.6364569664,-0.3769787550,-0.4377987981,-0.5338432789,0.5736504793,0.2441345602,-0.9038761258,-0.6077290177,-1.3279690742,-0.4976146519,-0.4994116426,1.3304718733,1.0966929197,-1.5022242069,-0.1590254903,1.6550925970,0.7573397160,1.3145182133,1.0289083719,-1.0945633650,-0.8774670362,0.1911343187,-1.0744937658,-1.0281189680,-0.4112767577,1.8617081642,0.2614100873,-0.1120274514,-1.1018739939,-0.5599132776,-0.9639128447,1.4452749491,-0.2379340380,0.0965326279,1.2240991592,-1.1137310266,-1.5898432732,-1.2747541666,-1.2285566330,-0.2418234199,-0.5687929392,0.6784836650,2.7191262245,1.0439400673,-0.5211651921,-0.1766644418,0.2213129550,-1.2737783194,-1.0919522047,3.2457404137,-0.5371184349,1.0131412745,1.3599733114,-0.1014137343,0.9044458866,-1.5049918890,0.3647313118,-0.7347620726,-0.7936519980,-0.4339566827,-2.1577720642,0.3033457994,0.6737972498,0.2448626608,0.6937257648,2.6209950447,-1.6615227461,-0.8978943825,0.3889890909,-0.4596033096,-0.7707110047,-0.1208278388,-0.5201724768,1.3842766285,-0.4282974005,0.7007796168,-0.4204046428,0.1363129318,0.4809338152,-1.1034513712,-0.4319312572,0.4005286396,-0.6485356688,0.2712681592,-0.7505175471,-1.4745311737,-1.1564862728,0.5677739978,0.6589148045,-0.8751621246,0.1891540736,-0.5418350697,1.8105150461,1.7355394363,0.2601662576,1.4141949415,-0.4384791851,-0.7842199802,0.4371353090,0.6943714023,-0.5380707979,0.4983506203,-0.5410212874,-1.1434106827,0.9545881748,1.3708180189,0.5112654567,-0.2825934291,0.8243518472,-0.3324177563,-0.9993035197,-1.6045049429,-1.7449423075,-0.8640743494,1.4113116264,0.3710772991,-0.3376179934,0.5454092026,-0.9610939026,-0.6323919296,-0.3632386029,-0.7752295732,1.8468760252,-0.0391475186,0.0493343733,-0.5033078790,-0.4727648795,-0.5759646893,-0.1179095358,0.1768748015,-0.0848587006,0.9794515967,-1.0092216730,1.5041515827,-0.2412413508,-0.3741708994,-1.3928172588,-0.1496126205,0.8959793448,-2.1168699265,-0.1839516461,-1.0776592493,-0.8422918320,-0.2578078508,0.7120755315,0.5457919836,1.2602188587,-1.3638870716,0.3252347112,0.3103461266,-0.6843565702,-0.8641714454,-1.3515841961,-1.3709787130,0.8442222476,0.6913160682,1.1721013784,-3.0055868626,0.1678446978,0.1083558276,-0.1259460449,-1.4384572506,-0.4379824996,0.6088418961,0.3323244452,0.1043943688,0.0842534676,-0.7801011801,-1.0713446140,-0.5946297646,-0.0363405943,0.6251944304,0.8738729358],[0.8417578340,-0.1302471012,-1.7174865007,0.8975894451,0.7157564163,0.7714152336,-1.2969895601,0.2700895965,0.6959099174,-0.7589926720,-0.1790614128,-1.4486862421,-0.0582850389,3.2882657051,-1.0190426111,-0.2596707344,1.2057558298,-0.9189699888,2.0075521469,-0.6953464746,0.5769160986,1.0798561573,0.3362890780,-0.9042438269,-0.6373935342,-1.2408188581,0.5856894851,-0.6166279912,-1.0748144388,-1.1937690973,0.2725262344,1.7097991705,1.2231884003,-0.1864460707,-0.3886844814,-1.1401408911,1.1001895666,0.0698222071,-0.8169481754,-0.2381762117,-1.0990309715,0.0363042206,-1.0470657349,0.5295069218,-0.8829963803,0.6988388300,0.0920695812,0.2125919312,0.2035269290,0.4504480660,0.5238382220,-0.4745627046,-2.6972100735,-1.2137219906,0.8994110823,-0.5333787799,-0.2330003232,1.0217889547,0.1832010299,-0.1263355166,0.6309781075,-0.9253399968,0.1927347630,-1.1983563900,2.1006891727,-0.6179254055,-0.7291156650,1.5111485720,-0.9217118025,-1.3076606989,1.3268530369,-0.0251848493,0.7743625641,0.3491370082,-0.2782400250,1.5292787552,-0.9830396771,0.0843514204,-0.6756237745,0.4932007790,1.2645338774,0.4777994752,0.4024190307,-0.4145161808,-1.9489889145,-1.1476491690,0.6487477422,-0.5789185166,0.5087463260,0.6426893473,1.1313562393,-0.1236293092,-1.0429756641,-0.1709813625,-0.0481598303,0.7611688972,0.6133790016,0.3980072141,-0.2242967188,-1.0123447180,-0.6344790459,0.9931277633,-0.5333514810,1.0943918228,0.0513154417,-0.0190032981,0.6046923399,0.6036578417,-0.7517727017,-0.5252866149,-1.1066845655,0.9470067024,0.0642272905,0.5968157053,-0.2527898848,-1.1888705492,0.5340593457,-0.0894497484,0.4148004949,0.1358723789,-0.1815864891,1.2741400003,-0.4312350452,-0.5020402670,0.1918521374,-0.3785336614,0.3328542411,-0.9043675065,-0.1148370057,0.2523318231,0.5270742178,-1.7881046534,0.9077923894,-0.6153175235,-0.8822738528,-0.8405439258,-0.0023670495,-0.4354154170,0.0477021970,0.8350443244,1.4262235165,0.6257942915,-1.5644793510,-1.2305992842,-1.1948616505,-0.3577411771,0.9498245716,1.3089897633,-0.1266740113,-1.2482118607,-0.7585697770,0.3164147139,-1.6025328636,1.4389153719,0.8901625276,-2.9274833202,-1.3023848534,-0.1057398170,0.1712914407,-0.9100197554,0.9962213039,-0.3809780180,-3.3738229275,0.8629325628,-1.3146237135,0.6146615744,-0.7009440660,0.7499541044,2.5462694168,0.2149894536,-1.3944144249,-0.2389547080,-1.4857633114,1.2554066181,-0.7945663929,0.1306199729,-0.4769009650,-0.2954279780,-0.4177624583,0.7645667791,0.0706775561,-0.0036673227,-0.1686885506,-1.2878538370,0.9614015222,0.9980438352,1.6333302259,0.3764960468,-1.7786319256,-0.1212472469,-0.5497505665,-1.6330521107,-1.0370134115,-1.8117271662,0.1047022566,-1.6284438372,0.1362413913,-0.6433253288,0.5122199059,0.8732233047,-1.8397277594,0.3793027401,-2.2091073990,0.1155540869,-0.2137276232,0.8290930986,0.4650845826,-0.8942251801,0.4178222120,-0.6271676421,0.0223560892,-0.9844577909,0.5359445810,-1.5864883661,-0.5337475538,-0.0250616819,-1.2855857611,0.8331471086,-1.4578803778,-1.2561945915,0.7204806209,0.5493844748,0.9813920856,0.6658095717],[0.2176280916,0.2478718907,-0.6187527180,1.7733623981,-1.0680766106,0.3115855157,0.9599409699,-0.6318791509,-1.7253450155,-0.1581967920,-0.2847787440,0.7510251999,0.6722778082,-0.5759375691,0.3976946175,-0.2699123919,1.1467082500,1.2894232273,-0.7950692773,-1.2313041687,2.5918538570,1.4108102322,-2.5290796757,-2.7064659595,2.0493443012,-0.1575528234,2.5390417576,-1.0978960991,0.6092931628,-1.7137644291,1.3925527334,-0.0829210207,0.9157499075,0.2611907423,-0.5531715155,-0.2612552643,0.1815842986,-0.2074785680,-1.3900825977,0.0094462279,-0.5616672635,0.6902035475,0.3310068846,-0.4598512352,-0.3554174304,2.5179774761,1.7027835846,0.7137155533,-2.5243444443,-0.0778169781,-0.7383882999,-0.4915468097,-0.4458041489,-0.1211165935,1.6788327694,-0.5277331471,-0.6074772477,-0.1313184202,-2.0344195366,0.6652619839,1.1473989487,-1.1869680882,-1.2858363390,0.6587207317,1.9654532671,0.7096003890,-0.7670360804,0.3632927835,0.1158852875,0.6583322883,-0.1609952003,0.1934109926,0.3165058196,-0.5613026023,0.1166221797,-1.3770916462,1.3263069391,4.2417716980,0.0803468376,-0.2049070597,1.5271331072,1.3036043644,0.5373516679,-1.9896894693,0.5142652392,0.6343153715,0.0603445619,0.5120731592,0.4221506715,-0.4696768820,0.0600340888,-0.4477773011,-0.0307193249,1.6819882393,0.2639681995,0.2140350789,1.3125169277,-0.3825906217,0.0022184977,0.1268048286,1.5481674671,-0.5468621850,-2.6106348038,-0.1282699555,-0.4177289009,-0.2894179821,0.9866001606,-1.2609921694,-0.9886783957,0.7946244478,0.8661491871,-0.4253971875,0.5601807833,-0.7023891211,-0.5714130402,-1.4841006994,-1.3401839733,0.0336159281,-0.0244027600,-0.1347808540,1.3022232056,-0.4897793531,0.6417346597,-0.8937379718,0.1637545079,0.2157087028,0.5360102654,-0.1275391430,1.7299357653,0.8968551755,-0.8804937601,-0.3912613988,0.8144260049,-0.4776334465,0.0370308645,-0.5445691347,-1.5247189999,-1.0484591722,1.0209940672,-0.0507922173,1.0969961882,0.0081091505,-1.0735074282,-0.4637985229,0.5490329266,0.7707652450,-0.9320597649,-0.5035763979,-0.3571883440,0.2904200852,-0.1153651178,-2.4413213730,0.5403788090,-0.8432013988,-0.7396709323,0.1283033639,-1.4762566090,-2.9589779377,1.3033297062,-0.2565463483,-0.3692474365,-1.1774467230,0.5953117013,-1.0849800110,-0.5788462162,-0.9446442127,-0.9873202443,0.8887941837,0.5191269517,-0.0704374313,-0.5656064749,-0.2987744510,0.4930670857,-0.4615590572,-0.2847803533,0.4016345441,-1.1605231762,-0.1913904548,-0.1202806160,0.2625747919,1.4581007957,-0.5056554079,1.7628393173,-1.4981478453,-1.3307580948,0.0612610243,1.5295269489,-0.3462225795,0.8649011254,0.4291520715,0.4492061138,1.0830732584,0.1221628636,-1.0343058109,0.3313332498,-1.0550051928,-0.3262061775,-0.2889346778,1.5725835562,-1.2517917156,0.0634046644,1.1232205629,-0.2517988682,-1.6302595139,-0.2371418178,1.7263067961,-0.7344884276,1.6452118158,0.2126683742,-1.0070602894,-1.1667169333,0.1198763698,-0.1139037609,-3.2478482723,0.1158405244,1.2873246670,-0.2967154384,-1.9113676548,-0.3062439561,0.2123287916,0.7192662954,0.2661412656,0.7092028856,1.0256335735],[0.4500994980,1.9605486393,0.0655291229,-0.6529865861,-0.1378134042,-1.7298293114,-0.1140466258,-0.0287944637,-0.3820255995,-1.4283573627,-0.2104176134,0.8270501494,1.6666053534,-0.2273907214,0.4807750881,-1.1465576887,-0.7090556622,-1.8215582371,-0.1210410222,-0.3808588386,0.0259706769,0.1462689787,0.2766871750,-1.3793306351,0.2024340034,0.4679293633,0.7783885598,0.2412308306,1.4358551502,-0.5162473917,-0.3459844589,0.3041249514,-1.2383747101,-0.7322098017,0.1910422891,1.8059240580,0.9277132154,-2.0171482563,0.1951866299,0.6639137268,1.2373590469,1.2164707184,0.1519757211,0.6530285478,0.9477738738,0.2688260674,-1.2582738400,-0.9600851536,1.0390130281,-1.0648518801,-0.8012365699,-0.2251798809,1.1142963171,0.9341740012,-0.4841320217,-0.0355923399,0.3871592879,0.5604581833,-0.4087530673,2.0429780483,-0.4825588167,-0.6649580002,0.8005616665,-0.3599886000,0.1193309203,2.4247162342,1.1964256763,-0.0806443915,1.5196145773,-0.4052409232,-1.2675876617,-0.6781725287,1.5818355083,-0.9401290417,-0.5591271520,-0.6824262142,0.4200727046,-0.0844191387,-0.2632116973,1.2377724648,-1.8268915415,-2.0282392502,-0.7599870563,0.4133173823,-0.3004525900,0.3071993589,-0.0633047372,0.4281289279,-1.2683299780,-0.4179551601,-0.3186133206,1.1701862812,0.9885575175,-0.1080873460,0.2283486128,0.3161881268,0.6529545784,0.7720674276,-0.1712777019,-0.0354263894,-0.2102745920,1.9657728672,1.0205694437,1.4390608072,-0.5366148949,-0.7892500162,1.9705640078,0.3855912387,0.0629414320,0.6265225410,1.1438483000,-0.9780292511,-0.6935688853,-0.9762239456,-1.4685491323,-1.4372947216,-1.0730566978,1.4005103111,0.5869930983,0.0406822041,-0.3796490729,-1.4310114384,-1.5541208982,-1.1360594034,0.1388325095,-2.1530423164,0.1147642508,0.0471051075,0.3919199407,0.7114022374,1.1730850935,-1.3012977839,0.3481751382,0.2530570626,1.7267895937,1.5392260551,-0.9095569253,-0.5668856502,-0.4846865833,-1.9252877235,0.0159935709,-0.2737687826,0.8173170686,-1.4598239660,0.4550220966,-0.4915318787,0.3947442770,-1.4720059633,1.1339253187,1.3018774986,0.6073305607,-0.0294520799,-0.0245533902,0.8299852610,-0.2093410045,-0.8937062025,-0.2104247361,0.4905937016,0.9270766973,-0.8489860892,3.0901689529,-0.9478366971,-0.7347039580,-0.0914564133,-0.1481568217,2.1250960827,-0.6915543675,-0.7773598433,0.7704299688,-0.8429703712,1.2658998966,0.1026027575,-0.9573163986,-1.0858606100,0.2484246790,1.5868668556,-0.5566447377,-1.2599195242,-1.3841241598,1.1235691309,1.7500667572,0.0562867485,-0.2367135286,-0.5502758026,0.8646404743,0.2884162962,0.2858077288,0.3000690043,-1.2110253572,1.4334803820,1.3211159706,0.1077606380,-0.5899074078,1.2182687521,0.9392931461,1.7326339483,0.6656506658,1.1504017115,2.5321750641,-1.3975093365,0.1026985720,0.0797936320,-1.2145488262,-1.6663728952,0.1640934795,-0.7657446265,-0.9649801254,-0.2156695276,-0.3802717328,-0.0361323729,2.2739071846,0.5935225487,-0.7992947698,0.5043234825,-0.0635420382,0.0635460094,-0.4986944795,-1.2183550596,0.8497512937,1.3682345152,0.0592218079,1.8176462650,0.0380014367,-0.1648121178],[-1.1811887026,1.6452852488,-1.1771593094,0.9260467887,-0.2866005898,-0.3300427198,1.0535910130,-0.4557752013,0.9618560672,0.3766882122,-1.8635056019,1.7812006474,-0.6835395694,0.2296070904,0.0775303841,-1.5565257072,-0.9657327533,-0.1612064391,-0.9848763943,-0.7570252419,-0.5434094667,0.0870989487,-0.3932954669,-1.4176454544,-0.6030968428,-0.5313537717,-0.7281560302,0.2537410855,0.8596052527,0.5377355814,0.0889668092,0.6040338874,-0.8646016121,0.4936359227,-1.3471070528,0.1851646155,0.5584734082,2.0683081150,-0.6238692403,2.4669466019,-1.0006500483,0.4998898208,0.5908782482,0.1898878664,0.9036785364,-2.6795573235,-0.6285718679,-2.2651124001,-0.0100735221,1.6246125698,1.2570762634,0.9649873972,0.4480108619,0.3538306952,-0.6394305229,0.6310080886,0.2673046887,1.3926157951,-0.5519245267,-0.6487637758,0.6073168516,-0.4114406705,0.3761738241,-1.4606273174,-1.4759228230,1.7613903284,1.2103283405,0.6520544291,-0.0654612854,-0.8724196553,1.2886662483,-0.9957754016,-0.2996768653,-0.5182425380,1.1075606346,-1.5807965994,0.1618001610,0.0243788250,-1.6595828533,-1.7371256351,0.1204244867,-0.2156693637,-1.6937848330,-1.0714200735,-0.2341331691,-0.3915313184,0.2276837975,0.2442232966,0.6898937821,0.8300570250,1.6709381342,-1.4706568718,0.6646685600,-0.7181670070,-0.0028348088,1.4022594690,2.1807963848,0.6828091741,-2.4582383633,1.1281207800,-0.6115419865,-1.1212936640,-1.2312363386,-0.4915806353,0.5487788320,-0.4697555602,-0.2441457063,0.4054579735,-2.1404464245,1.5693150759,-0.5173771977,-0.4552463293,1.1641325951,-0.4407202303,2.2006056309,0.2322905511,0.1107065305,0.2862088978,-0.2815102637,-0.3408988416,0.2058108151,-0.1996606290,1.8439724445,0.8755335808,-0.2496846020,1.5357745886,0.2453305572,0.6203836203,0.8212997913,1.3025162220,0.1074199975,0.8545994759,2.0262827873,1.6857354641,-0.5158991218,-0.9814931750,-2.5545742512,0.5971880555,-1.2477747202,-0.8447640538,-0.2711225450,-0.1848308742,-0.6540018916,0.2878918946,0.3621478677,0.6957350373,-1.6501054764,-1.1784461737,-0.8048467636,1.2036799192,-0.7421025038,1.2379450798,1.7582762241,-1.5390353203,0.4638553858,-1.3758928776,0.1921562254,-0.2725992501,2.1471478939,-1.2595988512,-0.3549736738,0.2839905918,-0.1939975917,-0.0150380591,0.9684531689,-0.8801771998,-0.6602508426,1.1853135824,-2.7470576763,0.5793197155,-1.2005586624,0.9979766607,0.1249912754,-0.7145936489,0.0190785658,-0.0678644404,-1.1773444414,-1.3656818867,-0.9308581352,-1.5273888111,0.6262164116,-0.9275572896,0.6307232380,-0.3932364285,0.3676517010,0.1095421910,0.4901863337,-1.3060173988,1.4643207788,1.2106331587,0.3033938110,-0.7580705881,-0.7045764923,-0.2487854362,0.7929059267,0.2234160602,-1.1094235182,0.1523469090,0.6313382387,-0.3104008734,1.2057654858,1.2911474705,0.1881976724,-0.0363846235,-0.6520819664,0.7624713182,0.9030723572,1.4012763500,0.9809569120,-0.1520080268,-1.1902999878,2.4968516827,0.5791797638,-0.5331690311,1.8849302530,1.2531266212,-0.5741626620,0.6255322099,-1.2660032511,0.0748630986,1.6345337629,0.2266604304,0.6084037423,-0.7702188492],[1.9156914949,0.4047849774,-1.2006857395,-0.1776676178,-0.1083059758,1.7594144344,-0.1029189825,-0.5665433407,0.6066942215,-0.5196262002,-0.2490082681,-0.9327793121,0.3236896098,-0.3677179217,-0.1818056256,-0.5839886069,-0.2038229257,-0.0597114004,1.1402399540,-0.0260799173,-0.6193695068,-0.3960160315,-0.9552299380,0.6448193789,1.4817017317,1.5544289351,0.9933441281,0.4653750658,0.3257007897,0.1263968647,0.1049721017,0.7257955074,-0.3700866103,-0.5341292024,2.2867617607,0.2005052119,-1.9354718924,0.8988186121,-0.0689575225,-0.6440743804,0.5772926807,-0.4547063410,1.4799640179,0.0647191256,-0.4898588955,-0.8096262217,-1.0249881744,1.6329885721,-1.0937043428,-0.6664443612,-0.2176919729,1.8012778759,-0.9251882434,-0.5116018653,-0.1121688411,-0.2361086607,1.7645183802,-0.9666455984,-0.8906248212,2.5078215599,-0.3365430832,1.4512392282,0.2457870543,-0.7407913804,-0.5314186215,-0.2175210267,1.6160784960,0.1030726433,-0.2955012023,0.3930938840,0.3691558838,-1.4468399286,1.4991459846,0.7861236930,1.4582768679,1.0606905222,1.4576917887,0.4594173431,0.9900460839,0.0367541350,-0.0919883996,-0.4150067866,1.1121563911,-1.8991169930,-0.0029766932,0.9736584425,0.5819031000,-0.3772553504,-0.3100667000,-0.2848064303,0.2640193999,-0.3516303599,-0.4092839956,-2.0935854912,-2.2584433556,0.3784409761,1.2266635895,-0.6566455960,-0.0365018025,0.3758405745,-1.1574118137,-1.6608725786,0.3360662758,-0.4174785316,1.7872010469,-0.5762202740,-0.9449684024,0.1880954653,-1.2142983675,-1.3987449408,-1.0537490845,0.0820185915,-0.5293899179,-0.5560578704,-0.5497375131,0.2450578064,0.4957640767,0.8905921578,-0.9192040563,0.5279426575,-0.0151319457,-0.3531607687,-1.6664785147,0.5860685110,-0.1222635657,-0.4696131647,-0.8306789398,-1.7213633060,0.4148206413,-0.9287552238,0.5041882396,0.6945232153,-0.7018371820,-1.7570469379,0.3231323659,1.0354024172,0.1899425387,1.2799785137,0.5467972755,1.1144099236,-0.1551059335,-0.8696814179,0.6797798276,-2.4373333454,0.6826940775,0.5466501713,0.8491961360,0.5381039381,-1.3631504774,0.6576964855,-1.6812366247,-0.1420921236,0.0644702837,0.3568619490,-0.4917500317,0.0291998181,0.5068673491,-1.1943705082,1.3364574909,0.0296131652,-1.1348272562,0.3270067573,-0.7610291839,0.1490225941,0.1008909866,-1.4996412992,-0.6174682379,-0.6078441739,0.2064402550,-0.8942359090,-2.6421942711,-1.3883210421,0.1245091781,0.6594743729,-1.2438187599,0.7344162464,-0.3856297135,1.3615497351,-0.6648976207,0.1783328503,-0.0513411611,-0.3603737056,0.3984665871,-0.4077322781,1.3554097414,0.2356388569,-0.2174219340,-0.1714425683,0.4619775414,1.7577792406,-0.9965714812,1.1256047487,-1.0318431854,0.0069990228,0.1054399237,-1.0228053331,-0.3113851845,0.8585723639,0.2263864875,-0.6888257265,1.6379286051,0.7351904511,-1.3622685671,-1.4697754383,0.6048600674,-1.0446178913,-1.6048693657,-0.0697253868,0.7481107712,-0.8034046292,0.2074136585,-1.5064373016,1.5586475134,0.0487470403,0.2746212780,1.3123952150,-0.0130932564,-0.5733627677,0.1082773730,-1.0324797630,1.4006639719,0.1419160068,-1.2351351976,1.4697282314],[-0.4961551726,0.3234536946,2.5705366135,0.6191286445,0.2001783103,1.8302137852,1.4770355225,1.2808552980,-2.0362477303,-0.7708904147,-2.2161989212,-0.6043381095,0.1801552922,-1.8412429094,0.2060479224,-0.2533358932,-0.8370944858,-0.7637611628,0.3168261349,0.5018517971,-0.2618725598,2.0702135563,2.1030116081,-0.0092163961,-0.9284299612,0.5606794357,-0.2881985605,-1.1079137325,-0.0594206415,0.2303173840,0.0974219963,-1.1072945595,0.6925656199,-0.3535038233,-0.6043095589,1.5273171663,1.1504018307,-1.5695350170,-0.6234639883,2.2262411118,0.5078104138,0.5744566321,0.9775137901,-0.2812677026,-0.6070799828,-0.6489948034,-0.2179257274,0.9815560579,-1.8020489216,-1.3939456940,0.2427239269,-1.1940877438,0.7539755106,0.2131537497,-0.9054090977,-0.9812750816,0.2375390530,-0.7263105512,-0.4075903893,0.4848812819,0.7323713899,-0.2195205986,0.7746742368,-0.5940266252,0.5959823132,1.5027459860,0.7207845449,-0.9153077602,0.7144644260,-0.4492384791,-0.1792533398,0.4524716139,-1.4900296926,0.1651769876,-2.2085511684,1.5511286259,0.7659050822,-0.3244543970,-0.4670624137,0.2195701599,0.7060580254,-1.4186925888,-0.3362497389,-0.8520848751,0.8867446780,0.3390224874,1.0442818403,-0.9417509437,-0.7377603650,0.0769798011,-0.7142660618,-0.7156736851,-0.1892230809,-0.5361557007,-1.3213405609,0.2317953408,-0.6825730801,1.0135622025,0.5309364200,0.2421779037,0.4017920494,-0.4618558586,2.3104119301,1.6827931404,-0.8894057274,-0.3898538053,0.8753790259,-0.6747983098,0.3014372587,-0.6322235465,0.0239859428,0.9638617039,0.9551012516,0.7515984774,-0.0033408087,-0.9071614146,0.3515233696,0.5865153670,-0.2021835446,0.4500505626,0.9413334727,-0.5805366039,1.7458322048,0.0380769074,-0.0990866572,-1.0987000465,0.0226462241,0.5436627865,1.1500978470,-0.4444086850,1.5100064278,-1.8368489742,-1.3223434687,1.0303223133,0.6480669379,0.4582530856,0.7627970576,0.1246709898,1.7402290106,-1.6799961329,-0.5588530898,0.7539991736,-0.6544169188,0.0672161505,-0.3272388577,0.5595880747,1.2202001810,-0.3627043962,0.8311170936,0.3439565897,-0.1743943244,0.2378960103,-0.4759786129,0.0559345298,-0.4318015277,0.3085387945,0.7614166737,-0.6752606630,1.1796420813,0.3769035339,-0.0317898244,1.9001022577,0.9514611959,-0.0556233376,1.3591566086,-1.2247810364,0.1007181108,-0.7172840834,-2.5895702839,0.4835788608,-0.9515837431,0.4092848897,-0.5652714968,1.6664245129,0.1597400755,0.3711966276,-0.6338654160,2.0866260529,0.5391490459,0.1859727055,-0.5546110868,-0.5572327971,-0.6505861282,-2.2647030354,0.4426457882,0.1652098149,-0.2662654519,-1.3499914408,-2.2734627724,0.5519658923,-0.0034194007,0.3603643179,-0.2471415550,-0.1274028271,-0.0114346435,-0.9155646563,0.1687090397,1.8955256939,0.2796881795,-0.5321885943,-0.7036546469,-3.1717319489,-0.7830032110,0.9552480578,-1.7417746782,-1.4782881737,0.2072343230,-0.3422027230,-0.0000039180,-0.7132492065,0.4970744252,0.2872682214,-0.0240418799,-0.0924070626,0.2005609572,-0.2416967005,-0.4522222579,-0.7911132574,1.1676659584,1.0429853201,0.4757418931,0.6947637796,-0.4414412677,-0.3858654201],[-0.0315578692,-0.0620186105,0.7743752599,1.6831254959,-1.5434107780,0.2417434901,-0.4039699435,1.0505112410,-0.0232733302,2.4460079670,1.8306061029,-0.2217491418,-0.8391377330,0.4859440923,-1.8701001406,1.6040929556,-0.7600015998,1.1947399378,0.8387134075,0.2391007692,-1.1038827896,1.5782461166,1.1515152454,0.8457059264,0.5228255391,-0.7674822807,-0.2895836532,-0.6218723059,-0.5329485536,1.9727894068,-0.8545120358,-0.0130121699,-0.2113794982,1.2866762877,0.4343684018,-0.2603058219,0.4808883667,-0.7769899964,-0.2809680998,0.6315764785,-0.2057825625,0.6242778301,0.3901982903,0.5251529217,-0.3042872846,1.0285104513,0.1606976837,0.1045264378,2.1321480274,-0.1765216440,-1.4004218578,-0.4953492284,-0.3176076710,1.2840528488,-0.0970956907,-1.2646052837,-2.8805596828,1.0095061064,-1.6003195047,1.3290346861,0.1850828230,-0.4360282421,0.7396296263,-0.9351475835,0.3767962754,1.6771639585,-0.7094474435,0.6432923675,-0.2963850200,-0.9941192269,-0.4628743827,0.8353748322,0.6730687618,-0.1000754759,-1.0190597773,1.6968350410,0.6228394508,-0.8952073455,-0.7015125155,1.4203540087,-0.0927628949,-0.2419510335,1.0769065619,-1.0362827778,0.0588853620,0.4759929478,0.1233649552,-0.8965017200,0.2282161117,-1.5211969614,0.6221343875,1.4138190746,-1.0571297407,0.7189363241,0.6118679047,1.3299641609,0.8587504625,0.0770226419,-1.5501102209,-0.6436623335,-0.1981956661,0.5344657898,0.0764187127,-0.2980852723,0.1402880549,0.4510383606,0.1825833172,-0.6221678257,-1.8053606749,0.3611282408,-0.8057336211,1.4318394661,0.5070827007,0.5489898324,0.9191237092,0.6925198436,-1.1693946123,-0.9724825025,1.0652650595,-1.3838489056,-0.4112027287,-1.3346097469,-0.5550889373,2.1472911835,1.1219021082,0.3011842966,1.1503249407,-1.3700573444,-0.2416723967,-0.8239528537,0.2148872912,-0.1503032893,-1.6401677132,-0.4880145788,0.1143725961,0.0351575762,-0.2061207592,-0.7204528451,-1.8366031647,-0.1839381754,-1.5259851217,-1.1155805588,0.1875498742,-0.9459524155,0.1837504357,0.2436221540,0.1203817278,-0.0044349479,-0.6723968387,1.5836912394,-0.0804802477,1.4106041193,-0.6819021106,-1.0202736855,0.9404947162,-1.2492835522,-2.0210082531,0.0438118316,-1.1920198202,0.2237372845,1.3341157436,-0.4678844810,0.5081447959,1.5526033640,-0.8240080476,-1.7554879189,0.2668016553,0.0097025707,-0.1617574394,-0.3676709831,0.5528894663,-2.0743434429,-0.5731478333,0.1601055115,0.3278245628,1.2261792421,0.5595084429,0.8565822840,-0.0552693568,-1.5954004526,-1.1731684208,0.2836863399,0.0900228396,0.3806756735,1.6710040569,0.4169368446,0.0130567746,-0.3834365010,-0.6937999725,0.7352904677,-0.4927332103,-0.1502003819,0.6302086115,1.2963200808,-0.4325444400,-0.5431685448,0.7643723488,-0.3315337896,1.4310835600,0.6894853115,1.2302139997,-2.7370042801,-2.0758631229,0.8194510341,-0.4422242343,0.4619798958,0.4433602989,-0.4337116778,-0.9711326361,-0.7581747770,0.1246169358,1.2777051926,1.1372745037,-1.4670459032,-0.4377576411,0.5218424201,-1.3545349836,0.4560755789,0.5561665297,-0.2870936692,-0.9668313265,-0.5786038041,-0.4808922112,0.6795433164],[-0.9512211084,-0.0520349778,0.6492468119,-0.4023033082,-1.4939315319,0.1896193027,-1.4484800100,-1.0535236597,0.4588146806,0.8064754605,-0.6888238788,-0.2907008529,-0.2247964591,0.5738497972,-0.3463048637,0.7758319378,-0.8346418142,0.3204473555,0.0675710440,0.6141188741,0.0013366932,-0.7410218120,1.6322492361,-1.2700757980,-1.0345075130,-1.0681058168,1.1929841042,-1.1688381433,-0.5785624981,-1.2230278254,0.5554084182,-0.4128984511,-2.2871227264,-1.1279395819,0.8433239460,0.1518727094,2.0980987549,0.4355578423,0.8607432842,1.7625917196,0.0613590404,-0.1310444772,-1.1538994312,-0.2084873766,0.6174628735,-1.7004872561,-1.0596629381,0.8424559832,0.6678346395,-0.7544751763,0.0714253187,0.6278148293,0.4992212653,1.0457350016,-0.1810946763,2.0991773605,-0.8590780497,0.2861403227,-1.4618811607,1.0282397270,-0.5337978601,-0.1446364969,-0.9851919413,-0.0448441878,-1.3567535877,-0.4009573758,-0.5194885135,1.9878838062,0.6508784294,1.0800508261,0.7166387439,0.2542805076,-0.1705901474,-0.8281711936,-0.2093469799,-0.3811714649,-2.3445951939,-0.3831149042,-2.7244744301,-0.8163167834,1.1056801081,-0.7393755913,-0.0141836274,0.7436538935,-0.7740547657,0.3735834360,-0.2184483260,0.9375358224,-1.2539305687,0.5076622963,1.3608239889,-1.1145510674,0.3206633925,-2.1811640263,0.3985752165,-1.3613519669,1.0203813314,-0.0525459312,1.2631360292,2.2825152874,-0.8123476505,-1.4406666756,2.0423271656,0.1090389937,1.4997259378,1.4937498569,-2.8995282650,-0.6637921333,-1.2237437963,-2.7321140766,-0.8622496724,1.9248254299,-0.0118800914,1.1111546755,0.4902781546,-1.1836172342,-1.7035608292,-0.6081807017,1.3975087404,-0.7155966163,0.2222000808,0.1491065621,0.5913746357,-0.5024318099,-1.7523351908,-0.8049672246,0.0088898782,0.5498868823,1.2858682871,-0.8413038254,-1.2562233210,0.8540031314,2.7723174095,-0.0660058931,-0.0704578236,1.4507570267,-1.7885080576,0.0580867492,0.0526900925,-1.2424575090,-1.1932390928,0.0536607280,-0.5483915210,-1.1195905209,1.2937419415,0.4144541919,0.5000180006,0.2320733964,-1.2249164581,1.0522677898,1.1093354225,1.0206264257,1.8482019901,1.0249714851,-0.6673058867,-0.4649356008,1.1188815832,-2.9661209583,1.1700307131,-0.8108969927,1.0020540953,0.1462125331,0.7842738628,-0.7974629402,0.2370982617,-0.2903953195,-2.3647882938,-0.4633262753,0.3321042359,0.0682333335,-1.5955102444,0.4221783876,-1.0460327864,0.0833548754,-0.0800244212,0.0916623548,0.8428997993,-0.0078278603,-0.0038369715,-1.8239731789,-0.6578385234,0.5915154815,1.1527293921,-1.2100425959,1.4766799212,0.2016633749,-0.1270681322,0.2116021514,-0.7006193995,-0.3207107484,-0.5239910483,-0.2711711824,-0.0808895975,-0.2620672882,-0.5259278417,-0.2087262869,-0.0847980306,-0.6684075594,0.1318341643,-0.3069368303,-0.3217082024,1.2094240189,0.3053298891,0.0364941768,0.4884267449,-0.2790943980,-0.9987446070,0.0164896939,-1.4116181135,-0.0203813892,2.0271334648,-0.4562243521,-0.7777702212,-0.0108297598,0.1521138549,-1.5504790545,-0.4843142927,1.1425955296,0.9327201843,-0.2901050746,0.5929782987,-0.7860559225,0.2315905243,-0.1417619437],[0.0613079369,-0.0808374882,-1.5937153101,1.0773460865,0.2746188343,-0.7450028062,-0.6539436579,0.8430441618,-0.8579425216,0.8792074919,-0.1914065927,0.8935930133,-2.3076658249,-1.2254799604,-0.9316254854,-1.3976552486,-0.3991250396,-0.6948458552,-1.5662037134,-0.0899952948,-1.2958993912,-0.2588024437,0.0242907070,-0.1605933458,0.1624411196,0.6484341621,0.0786128640,0.1670193970,-0.7773826718,-0.7471885085,0.2092164755,0.2134766579,0.5705438852,-0.4578116536,0.5587601662,-1.0892243385,1.0605726242,-0.3094975352,-1.4312770367,-0.1730458140,0.4337505400,1.0741657019,1.6160172224,0.1272350997,-0.1938657910,0.6725165844,0.5144274831,-0.5641947389,-1.0385379791,0.2151829004,-0.4242037535,-1.5490713120,-3.3838853836,0.3964085877,0.4213456511,-0.3092971146,0.5519814491,-1.0131312609,0.1652939320,-0.7183796763,-0.2181751877,0.4323822856,-0.4072811007,-1.3852882385,0.0991055146,-0.9232534170,-0.0023331596,0.5881130099,0.4912368953,0.5116588473,-0.4789597988,-1.3557976484,0.8674719930,0.8575125933,-0.1776522994,-0.1775404066,-0.1856035888,1.3404828310,-0.9878146648,1.6222991943,-0.2595064640,-0.5972997546,-0.5425067544,0.5912622809,0.3834866583,1.3222781420,-0.8192518950,-1.0907922983,-0.6246671081,0.6052445769,0.4081635475,0.2373645455,0.1690674126,0.5802407265,-0.5613754392,0.2456480712,-0.9186141491,-0.2769204080,-1.7706328630,0.4763954878,1.5354692936,-0.0598922297,-0.5450186133,-0.1883676797,-2.5338580608,0.4641127586,0.3048112392,-0.6876582503,-0.4315545857,0.3876003921,-0.8581926823,-0.6158384085,1.0436244011,-0.3341654837,-0.0467620641,0.7010356784,0.1533711851,0.9586766362,-0.0557326339,0.3174902499,-1.2664761543,0.1697869450,-0.8404021859,-0.4100980163,0.2164599746,-1.5345907211,-0.3343243003,0.1733925790,0.6227089167,-0.3400717676,0.2302381098,0.7133636475,1.0837763548,0.6489784122,-0.4140874445,-0.4357428849,1.0704756975,-0.1154135317,0.4830461740,-0.2553482950,0.6317064762,1.4388637543,-0.6082205772,0.6572952271,0.4606111348,-0.4769994318,-0.6360312104,-0.0991303176,1.0446186066,0.9986180067,1.2831765413,2.1639351845,1.1235325336,0.2174135298,0.1962851286,0.7651981115,-1.2645888329,2.4407944679,0.0277731884,0.3911207616,0.3047181964,1.5580196381,-0.7807807326,0.1302722842,0.9293032289,1.0180013180,-0.1589470506,0.9012888670,2.0110130310,0.4158630371,0.0888017491,-0.3296423256,0.2352096140,-1.3284860849,-1.0254509449,1.5939886570,0.1282520890,0.4186613858,0.3527747393,1.8500404358,-0.6223271489,-0.6476612091,-0.2049963474,-1.3311686516,-0.1923779994,0.9806227684,0.0004467586,-0.6268636584,0.0827400163,0.9971995354,-0.9771696329,-0.2499272972,-0.3006271124,-0.3148871660,1.7323715687,1.7476186752,1.2388221025,1.5113986731,-1.3835494518,-0.7499847412,0.6338880062,0.5752592087,0.4825147986,-1.0033938885,0.3850116134,-1.2665108442,-1.2748463154,-0.0880842432,1.2344012260,-1.1817024946,1.1284167767,-0.3691191077,0.0791317150,0.3606567383,-1.3921650648,-0.8959398866,1.5028594732,-0.6054318547,0.2944380939,-1.0308992863,-0.0682621300,-0.9367830157,-0.4790829122,0.3819047511],[-0.0056958185,-0.0107525261,-0.1028386652,-0.0407153480,0.8296846747,-0.1679052711,0.7480251193,-0.2704329193,-1.6717312336,-0.9499000311,0.0938736945,0.0792115331,0.9397433400,0.4047400951,-1.5396602154,-0.6735089421,-0.1467912346,-1.1075301170,-1.2755281925,-0.2567361891,-1.1061099768,-0.7344524860,-0.3856717646,-2.1764931679,0.0527051352,-1.4594818354,0.1187220141,0.0322499759,-0.8149708509,-0.0294837058,0.8411209583,0.5211537480,-0.0760316700,0.5130376816,1.3642598391,0.0968394950,1.3154854774,-1.4872413874,0.9520645142,0.3005496562,0.6140743494,0.4355062246,-0.5848545432,-2.5305237770,1.3488068581,0.3234812021,0.0880915597,-1.1336556673,0.0930403545,-0.1541861147,-0.3016662002,-1.4424922466,1.5437906981,-0.6499189734,-2.3192703724,2.1626596451,-0.9241775870,-1.1621884108,0.9141823053,-1.0259217024,1.8104969263,-0.3519425690,-0.9066295028,0.5648496747,-0.8370807171,1.4850924015,-1.0320893526,-0.5918660760,0.1176175773,0.1124671847,-0.5896363258,-0.5832690597,1.6353310347,-0.7493469119,0.0465250574,-1.0367507935,-1.4493075609,0.7984340191,0.0820988566,0.5954594016,-0.3189447820,0.5058848262,-0.7114982605,0.5402135849,0.9257064462,-1.4224647284,-0.7928759456,0.0240264982,1.6763641834,-0.3487647474,-0.1829279959,-0.3225548267,-0.3663921356,0.1824751198,1.0975593328,0.1602485627,-0.6861212850,0.7449244857,-0.1191331446,-1.4907772541,-0.0717278123,0.5842505693,0.2929826379,0.0632381737,-0.1342022121,0.4378673136,0.4098827243,0.8411206603,-0.0789814442,-0.2399690449,0.5405627489,0.6278280020,-0.0108255902,0.0505150706,0.9220871925,0.4489594996,-1.0202002525,0.0912503898,0.9841045141,0.9010125399,-1.1630496979,0.0682095513,0.8780198693,0.0305589475,0.3894029856,-0.7775303125,-0.0655512884,-1.6206985712,-1.1411087513,0.6450503469,1.2038553953,1.4712681770,1.2446136475,0.2164686620,0.2948817015,0.0351893939,-1.1672149897,1.1487790346,0.4665854871,-1.8246629238,-0.4153800309,0.1016899049,0.9141763449,-1.3728073835,-0.1546313763,0.8691310287,-0.7926211357,-0.1904437244,-0.6742334366,1.5098134279,-0.7134387493,-1.3717981577,-2.2027521133,0.1625714153,-1.5810571909,-1.4009828568,-1.3369977474,-0.9206066728,0.5045040846,1.0087817907,0.7085490823,-1.8204631805,0.3505395055,0.1601599306,-1.2087808847,0.4851200879,0.6961211562,-0.8115411401,0.6887942553,-1.4158912897,-1.6351356506,-1.5797746181,0.6322211623,0.7213320732,0.4606862962,0.8185957074,0.4297682345,-0.0461610332,-0.4848355055,0.0079860473,-0.2723667622,-1.7874740362,-0.9821856022,-0.1429887712,-0.0252348650,1.9776024818,-0.0204313695,1.2319833040,1.4091806412,2.0734572411,0.7006293535,-1.7150832415,0.2069371343,-1.6790082455,-0.2211100459,-1.1749776602,0.0365302674,-0.3868663907,-0.0528737232,-0.9732688665,-0.3719446659,1.6559091806,0.3440146744,0.3946353495,-1.2677277327,-0.1625610441,-0.9379303455,0.5799641013,-0.0261510685,-0.7596936822,-0.7694777250,-1.0063365698,0.2434116006,0.4409281015,0.7339676619,0.8679358959,1.1445062160,-0.8765382767,1.2840873003,0.0781262815,0.6481414437,-0.9263640642,-1.4005607367,-0.5750855803],[-1.0007141829,-1.0111794472,-2.0524334908,-0.6442151070,0.7671940923,-0.1145124212,-1.1316365004,-0.4282675087,-0.5749440789,0.1833458543,-0.1921777874,-1.4095654488,-0.8744406700,-1.5981025696,-0.5336658955,-0.9894477129,-0.4347616434,-0.5107789040,1.3670536280,1.3113130331,-1.3499810696,-0.3468316793,-0.0047309161,-0.7323764563,-1.0284543037,-0.0138079068,0.1132538766,-1.5646991730,0.0751108974,0.8303509951,-0.1938802451,-1.9379101992,0.0315455496,0.6615291834,-0.3474954963,0.0518144444,0.2939739525,0.0353117324,-0.0989409313,-0.3005047441,0.1867303848,-0.2230330557,0.1489258558,-0.3088251352,-0.8473169208,-2.1500792503,1.2460508347,-0.5631950498,-0.0580349751,1.0495109558,-1.5226836205,-0.3884546161,-1.3195962906,-0.4702957571,0.9313479662,-0.6823721528,0.2000008970,-0.5764540434,0.8537899852,0.9402247071,-1.2445976734,1.5359268188,2.0948321819,0.0036094575,-0.8047494888,0.6427945495,-0.2275648564,0.9242594838,0.5359516740,0.4261472821,0.1482180059,2.3247148991,-0.0195343848,0.1890586466,0.5467409492,0.1936537623,0.1615436673,0.7448188066,-0.1855718195,-0.0946201459,-0.7014371753,1.3129265308,0.2720538378,-0.8413616419,-0.4604212642,-0.4750130177,0.1501253098,0.5798549652,0.6560567021,1.0006505251,-0.0545246713,-0.8020230532,0.1682970226,-1.3182489872,-1.1121965647,0.4462005496,0.6017032266,-0.9011870623,0.0598643757,-0.4081072211,-1.7386128902,-1.3758144379,1.4552366734,-0.4543621540,-0.0994410738,2.2818775177,0.5025957227,0.1134583354,-0.7030552030,-0.4476456046,0.2547734380,1.2222807407,-0.9206188917,0.2277347744,0.0041218023,-0.5322578549,-0.6025855541,0.7921522856,0.7033787966,-0.5238165855,-0.3242940903,0.2282759100,-0.7225562334,0.2189976722,1.8640956879,-0.2014419436,-2.0383541584,0.7670707107,0.0946207866,-0.9718177915,0.5874319673,0.6024810076,-2.0453674793,0.3536788225,0.9645302296,-0.1440635771,-0.5780105591,0.4737685323,0.5964793563,-0.6031044126,1.7813662291,0.2994776368,0.0040594037,-0.4686329961,2.2406263351,-1.0148507357,-0.1007227823,0.5790359974,-1.3669410944,2.4542148113,-1.0502914190,-0.3570873737,-0.5959746838,0.1286456734,-0.2735119760,1.1831076145,-1.9064669609,-0.0173027087,-0.8793388605,0.9526354074,1.7210581303,-0.5359464884,-0.4232902229,-0.7482769489,0.3254696131,-1.7439950705,-1.3261831999,0.6918687820,1.3905178308,-0.2113317549,-0.6825257540,-0.3879581392,1.7752315998,-1.2752637863,-0.1166420951,-1.2915318012,0.1837764680,-0.1601903141,-1.2379313707,-0.0902454183,-0.2519405782,0.5891952515,-1.2630279064,0.6976277232,-1.1750415564,-0.5748518109,-0.6612970829,-0.2794839144,-0.5347990394,-1.3255223036,-1.1630715132,-0.1496529430,-0.2865560055,-1.6683949232,-0.9315761328,0.7481259704,1.2013155222,-1.5000940561,-0.6404548883,0.7500379086,-0.4412990510,0.5010219216,-0.5410768986,0.0552483760,0.4714125097,0.0664171502,-0.5142256021,1.9464106560,0.3117389381,0.1133240163,0.0764206946,-2.1993992329,0.5718014836,1.3705084324,-1.5698078871,-0.5611788034,0.9989655614,-1.3109512329,0.2550405562,0.2179651111,-0.2458942831,0.7524243593,-0.8888767958,-0.9764984250],[1.2953237295,0.4097941518,0.1875261217,-0.6974471211,0.5086981058,1.6450030804,1.9019715786,1.1486008167,0.7689781785,0.5361904502,-0.6677723527,0.6889824867,0.9869290590,1.0165002346,0.5444405675,0.5614240766,1.5270713568,-0.8393536806,-0.9333688021,-0.0409526937,-0.0998738036,-1.0882169008,-0.3863084316,0.3301858604,-2.1117844582,0.4842156470,1.3418143988,-0.5743961930,0.0791617855,-1.6336181164,-1.3499776125,-0.0084255254,-1.4727298021,0.7257235050,1.0187690258,0.8479738235,0.0947717726,-0.4799260199,0.0153171076,1.7028288841,-2.4398531914,-0.7412115335,-0.7696051002,1.3584727049,0.5375123620,-0.4170902073,-0.3068646789,-0.3780648708,-0.6893613338,-0.3763635159,-0.5504732132,0.6834804416,0.1133769155,-0.8619261384,1.0548527241,-0.3079719841,1.3956831694,0.2459279150,-1.6134513617,0.0425924845,-0.2693678439,0.4709191024,1.9377353191,0.7869468331,-0.0102772806,0.7438308001,-0.9149633050,0.0896701813,-0.4285502732,-1.3545906544,0.3714318871,-0.9662501812,0.0663880482,0.0755720139,0.8606848121,-0.3789045811,-0.5214172602,-0.0249759257,0.3855774105,0.7961698174,-1.1770154238,-1.4023742676,-0.2600174546,0.4639261067,0.4731233418,-1.4157626629,-0.8307561874,-0.4470773637,-0.3219674826,-1.9117525816,1.8478777409,0.4157984257,0.2571049929,0.3644884229,2.1297881603,-0.2326315641,0.4425879419,1.5757837296,0.0731884092,-1.9210509062,-1.3611035347,0.1254009157,-0.7626289129,1.3449469805,1.7275245190,-0.5753748417,0.7083702683,-1.1395683289,0.9009451270,-1.1356049776,0.1987769753,-0.0978872925,1.1407341957,0.9220844507,0.4430917501,0.6003719568,-1.2657593489,-0.3270651400,-0.5956538320,0.1456205398,0.6889383793,0.1033993214,-1.3690428734,0.7404351234,-0.1215081289,-0.1990689635,0.8810068369,0.3059888184,0.4480573535,0.0953786969,-0.1013018936,0.4117246568,0.3502552509,0.0194713138,-0.2473152578,0.0101099778,0.9885329008,-0.8361511827,1.9361438751,-1.5811871290,-1.1002520323,0.9397823811,-0.0545123816,0.7610992193,0.5216606259,-1.0363000631,-0.7652194500,-0.0382754430,0.9650835395,1.8669173717,0.1112473533,-1.2015788555,0.6443081498,0.6186507344,0.0816031545,-0.4066605866,-0.4410473108,-0.5908294320,1.1830356121,0.2105588764,-0.1489932090,-1.3678106070,0.5889723301,0.3990076482,1.5867253542,-1.5419884920,-0.3135329485,1.2928620577,0.5041116476,-0.9222351909,-1.7613097429,-0.4756738544,-0.7624484897,0.4504918158,-0.4556908309,-0.0724610984,0.7097553611,1.1203337908,-0.3484529257,1.1659314632,0.1397081167,-1.5629338026,-0.7076761127,1.4508572817,0.5759756565,-0.9922260046,0.2997192740,-1.5346330404,-0.9537140131,2.3760414124,-0.0492447056,-0.2149013877,-0.7276610732,0.0869092569,-1.3743280172,0.0617098734,-1.3765540123,-0.3937661052,0.5421155691,-1.4106866121,0.4895611703,1.1866769791,1.5226407051,-0.4637456834,-1.9443510771,2.0126781464,1.2011687756,0.4509884119,0.4688838422,0.1416492164,2.0351710320,0.4427682161,0.4046050906,0.5470671654,0.1857168674,-0.1370263547,-0.7957639694,1.1775459051,-0.2308127433,-0.3254511952,0.6586336493,-0.6313273311,-2.5197975636,0.6868247986],[0.9503057003,0.5775336623,-1.1326794624,-0.1656296253,1.3421090841,-0.0567202605,-0.6803632975,-0.4050382078,0.6355462670,-1.5059229136,-0.2022885233,-0.2591747940,0.3714067638,0.8712681532,-0.3979974091,-0.9245500565,-1.0143244267,-0.4404507279,0.5970969200,0.2319602817,0.5535842180,-0.4240028858,0.5902436972,-0.5997185707,0.1466287971,2.3729963303,3.0086288452,-0.5136756301,-0.4350767136,0.9363536835,-0.0597128756,-0.8972352147,-0.6416056156,1.2984263897,1.8004604578,-0.7545091510,-0.3338066936,-0.2864007652,0.4252581596,0.5166036487,-1.0704407692,0.3727474511,-0.4662492871,-0.8777085543,1.0085009336,0.4567803144,-1.4343152046,-0.0693903938,0.6386763453,-1.2341254950,-0.0431989394,-0.1082422808,1.1039363146,-0.9819817543,0.6192163229,-1.2208844423,0.5299162865,-0.1968284845,0.7186149955,-0.2716750801,-1.0598012209,-0.6523534060,-0.7307460904,-0.1305984706,0.4185395837,-0.1593158543,-1.0495679379,-0.3906728327,-0.2174431086,0.1957010031,-0.8210510612,-0.7583345175,-0.6436064839,0.6527817845,-2.2995562553,-1.4217740297,2.6572241783,-2.4107866287,0.2148666978,1.1167837381,0.0689454675,-1.3653887510,1.1535874605,-0.4121600389,0.2929527760,0.3599597514,-0.6850985289,-0.4406382740,-0.5719017982,-0.1960382611,-1.7868933678,-1.3133436441,-1.1885946989,1.0874711275,-0.5319527388,-0.6841382384,1.2883000374,0.0816733018,0.5675491691,-0.3255595267,0.3317215741,1.0300211906,0.5762638450,0.8731001019,0.8163179755,0.7308938503,-0.6628636718,0.3519839942,1.2436153889,-1.1196715832,-1.0711348057,-0.1543116719,-1.0863668919,0.4676058888,-0.6846093535,1.3797492981,0.3526509106,0.1648032069,0.0237876102,0.2023786604,-1.1174907684,2.2319457531,-1.5312778950,0.6684376597,-0.8544011116,0.6476187706,-0.7095187306,-0.5701030493,0.2349684983,0.7085313201,-0.2691114843,0.3251794875,-1.1168712378,-0.3028653264,-0.2788650990,-1.0868864059,0.4872075617,-0.5189303756,1.1145070791,0.4584412575,-0.5998378396,-1.4327603579,0.1275960505,0.2901370525,0.5808152556,-0.3115820587,0.8584854007,0.8452209234,0.3459075093,-1.5614558458,1.6772382259,0.3585601747,-0.8467258811,-0.4606466293,1.7624496222,-0.6848064661,1.2684752941,0.8637610078,0.4877776504,0.8418605924,0.1541614383,0.8735924363,0.3841915727,1.6905543804,-1.1739689112,-1.3215125799,0.1166850626,-0.5511772037,0.1038629040,1.0686504841,1.3229821920,-1.6473169327,-0.8972117901,-1.7887773514,-0.7082397938,0.0563377440,-0.7301474810,-0.6179089546,-0.7773608565,1.2667914629,-0.5679482222,0.7532033324,1.0075652599,-0.5667064786,0.8302335739,-1.4011286497,-0.0043775220,-0.7475999594,-0.7050182223,-0.8238686323,0.0595666282,-0.9293041825,0.3678091466,-0.5593569875,0.8309092522,1.4071900845,-0.5506002307,-0.8832412958,-0.4212633371,1.7079013586,-0.2129941732,-1.2215648890,0.9746375680,0.5255259275,0.1676276326,-0.7460688353,-0.3985492587,-0.2419589013,1.0022572279,0.3096496165,0.0151867429,-0.9989675879,-0.8324916363,-0.2698731124,0.7522588968,-0.7222785354,-1.3158954382,0.2006363124,-0.5149350762,0.0061105140,-1.7563093901,0.0626748428,-0.5142523050,0.9311300516],[0.2424508929,1.6043307781,-0.3289819360,-0.0494096093,0.7316811085,-3.0088012218,-0.3715510070,0.6926851869,0.0006539202,-1.5461733341,0.0995478183,0.2765178084,1.4613583088,-1.3338665962,1.1406615973,0.3171734214,-0.4835242331,-0.2373184413,-0.0232107006,0.8536342382,1.6586548090,-0.5782043338,0.4537277818,-1.6588265896,-1.2349920273,0.0675332099,-0.2628900707,0.3805647194,0.8191167712,-0.1620512605,-0.6835185885,-0.8386046886,-0.4493160546,0.1507066637,-1.9121794701,0.5574362278,0.7430413961,-0.2474150509,1.0876133442,0.4239715040,-0.3730142415,-0.0829822570,-1.4674832821,0.2024282217,-0.8522486687,2.2314176559,-0.5353853703,-0.1779713780,-0.2383777201,0.5534749031,-0.7674822211,-0.3580855429,-0.8130614161,-0.4385259449,0.4330780208,-1.6723484993,0.4503363967,0.2866947651,-0.3895815313,-0.2773104012,-0.2627112865,0.8851501942,-0.4606986940,-0.6227043271,-0.2819042802,-0.1450483352,-0.1862329692,-0.9947724938,0.1286654323,0.8603655100,0.8581410646,0.0987485275,0.5210778713,1.4589275122,0.6336409450,-0.8249817491,1.1077028513,0.0143080968,2.6831395626,-0.4208842516,0.1807337701,0.8042305112,-1.6470074654,-0.5645323992,-1.4007233381,-0.1503272057,-1.0351790190,-1.1351163387,0.3737889528,-0.4462286234,1.0661048889,0.3744573891,0.9195076823,-0.4215391576,-0.5308960676,-0.2440245748,0.2437709421,1.8744490147,0.0601186305,-0.2889816463,0.6083256006,-0.5951191187,-1.1753454208,-0.4446639717,1.6190202236,0.2731115520,-1.2823290825,-0.6856404543,1.0023931265,-1.0596644878,0.0099540399,-0.9112805724,-1.9877477884,-0.3559651077,0.6992651820,-0.5291158557,-0.2777605951,-0.6945871115,0.4263310730,-0.4606139064,0.5313073993,0.3477262855,-0.6290251017,1.0485340357,-0.4984591007,-0.7014091611,-0.1268873066,1.7174379826,0.3756468594,0.6778792143,0.6780077219,-1.9288370609,2.1379055977,-0.3503865302,2.6057658195,1.4180355072,-0.8483371139,0.4836727679,0.2986367643,-0.4045870006,-0.3841612935,0.4206042886,-0.2121454924,-0.7215571404,-0.9440875053,-1.8669323921,-1.2715229988,-1.3166813850,-0.2099368423,-0.4971982241,-0.6165689826,-0.8177767992,0.7123737931,0.0018365026,0.2189357430,0.4399895966,-1.4153739214,-0.4745371640,0.8956039548,1.8533095121,-0.6692598462,0.8224695325,-0.4222893119,0.0309825372,-0.0985280126,0.0038888101,-0.6720025539,-1.0864828825,-0.5846391916,-0.8959212899,0.7892262340,-0.5727711320,-0.2901080549,2.3455419540,0.5100870132,0.6484789848,0.2203816473,-1.2419773340,-0.2932655215,0.0602538697,-1.2372095585,-1.3545566797,0.7277938724,-0.5110531449,1.8365730047,0.1518082768,-1.2500865459,-0.8315503597,-1.6715674400,-0.6968267560,0.1409083009,-0.4922994673,-0.7246807814,0.0654167756,-0.7244126201,-0.1597559899,-0.5303936601,0.0489419326,1.0108239651,0.6513858438,0.5965271592,-0.8281033635,-0.4502198994,1.3260207176,0.1846620142,0.3190106750,2.7237520218,-0.2407226712,-2.0167696476,0.0360962264,-0.6404058337,2.0376145840,0.4976940155,-0.1475096196,-1.6457666159,-0.3195464015,-0.4240814745,-0.7984828353,-1.7798746824,-0.5305024981,-0.9451339841,-0.3695907891,-0.6193536520,-0.8740698695],[1.6382918358,0.8859256506,-0.6783981919,-0.6647996902,1.8402383327,-0.4664635360,-0.2869035602,0.8843571544,-0.6794208288,-0.0783729777,2.0344007015,-1.1569439173,0.5343900919,-1.2190382481,-2.7440197468,0.3725993037,-0.9437746406,-0.8705223799,-0.2391330153,-0.3168485463,-0.1414447427,1.3143306971,-0.0398241282,0.9941142797,0.5481469035,2.4456527233,1.1136090755,0.8380909562,0.8857203722,0.4825400114,-0.4374621511,-0.3491562307,-1.1849820614,0.9566195011,-0.6884231567,-0.6400103569,0.4962954223,-1.2663766146,-0.3680319488,-0.4868077934,-0.5154715776,-0.2976514101,-0.8406806588,-0.0521839485,-0.2806237936,0.2266739756,-0.2517160773,1.0988744497,1.2061139345,0.1976041198,-0.5565751195,-0.4206511378,-0.6106412411,0.6980303526,0.8038991690,-0.5697152615,1.1649491787,0.7628110647,1.1644734144,0.9731919169,-0.7703890800,2.5272583961,0.2993666232,0.9850234985,0.7090337873,1.2726600170,-0.9547067285,-2.2828862667,1.8040145636,-0.3618632853,-0.2424062788,-0.0360498950,0.6348895431,-1.1561373472,1.6774686575,-1.1213855743,-2.1498403549,0.4187663794,-0.8002408743,-0.7238284349,1.1115944386,1.5962899923,-0.8676059842,0.7034949064,-1.2499570847,0.1972162724,-1.4184181690,-0.1329640746,-2.0752580166,-1.3371758461,-1.7525701523,-1.4658756256,-1.5818002224,1.6401294470,-0.4202954471,0.6047040820,0.3399446011,-0.8635126352,2.1736896038,-0.6485803127,0.7760526538,-0.5920584798,-0.1532118171,0.0984346718,-1.1254554987,-0.4605285823,1.1066592932,-0.3280683458,-0.2730957568,0.0873861089,0.3761961758,0.3713440597,0.1002379730,-0.3835586011,0.5256533027,1.5230630636,1.6018843651,-1.2051194906,1.0897098780,0.7993661761,0.0984100774,0.5448901057,-1.2182017565,0.2122826129,-0.0533185825,-0.8356292844,-0.0336980745,1.2712229490,-1.2643465996,-1.6775107384,-0.9985657334,1.0351241827,-1.5880872011,0.7903474569,-0.0167097095,-1.2681533098,-1.6248126030,0.6737528443,1.4385950565,0.2779538631,1.9929624796,-0.1199937686,-0.4443750381,0.9745465517,0.4621928632,-1.5144137144,0.9511830211,0.8978304267,-1.1884974241,0.0929282978,-0.5976312757,-0.9062644839,-0.4685940742,0.0498959236,0.1588799804,1.4182624817,-0.0872410312,-0.4276543558,0.0140459212,-0.8659215569,1.4454176426,-0.3340324461,-0.0349760242,-0.5698040128,-0.9010173678,2.1993246078,-0.8341708779,0.8671241403,1.1048197746,1.0847196579,-0.2546294630,1.2404844761,0.4475292861,0.2714076340,-1.4715112448,-1.2379156351,-0.2507206798,-1.3513888121,0.2730809152,-0.0653217137,-0.1397357732,0.5785194635,-1.1722716093,-1.1036603451,-0.0131531553,1.6345367432,1.2507511377,-0.5973930359,-0.0155039188,-1.9891339540,0.3615313172,-1.4075814486,-1.5032497644,-0.1248913184,0.6577493548,0.0023436123,0.2874972820,-0.8602231741,-0.0809271559,0.1523856521,-1.0670419931,-1.0369086266,0.2998365760,-0.6676939130,0.6504074335,1.3342704773,0.5072284341,-1.4595576525,0.7439997196,-0.9475765228,0.2244775295,0.4849691689,-0.3844098151,0.1123687103,-0.6872268319,0.3756891489,0.6597490907,0.7952565551,0.4855103195,0.3044109941,0.1490117610,-0.5080056787,1.2712522745,1.2272012234],[-0.3299134672,-0.0199540183,1.8290190697,0.5677607656,-0.5968216658,-0.2418765873,-1.9487468004,-0.2772829235,-0.5381264687,1.2586739063,1.5063394308,-1.3124256134,-0.2960510552,-0.7850624323,-0.6174619794,1.1621369123,0.5564032793,-0.7371488214,-0.3468227684,2.3980824947,-0.1111919731,-0.2277705222,-0.6066004038,1.5664755106,0.8420552611,-1.3558931351,-1.0245736837,0.8999151587,-0.5651986003,-0.8465386033,-0.4856346548,-0.9645081162,0.1725240499,-0.7497085929,0.9174042344,-1.7963420153,0.0882364661,0.5786577463,-0.0313097350,1.8228253126,-0.1195402592,-0.3144756854,-0.7695646286,0.6737496853,1.7059468031,0.1110388115,-0.1290749758,-0.7515364289,1.1298259497,-1.9274809361,0.2750416994,0.9833395481,0.0931785628,0.5955570340,-1.3676586151,0.5912148952,-0.6878173351,-0.2396046817,0.4571484029,0.3076282442,-1.1353645325,0.5450010300,1.2716612816,-0.7891782522,-1.1442798376,0.5127852559,-0.3669523299,-0.8581065536,1.3917527199,-1.8237631321,0.1568117738,0.6871304512,-0.8506605029,-1.0436689854,0.7592877746,-0.4107601345,-1.4867564440,0.6242988706,0.4792506993,-1.1623834372,1.2963851690,1.8081043959,2.4934203625,-0.2422652245,-1.7136925459,-1.8513816595,-0.8949121237,1.1286860704,0.3440145850,-0.4634765983,-0.0613686703,1.7311720848,-1.2469245195,1.2263395786,0.3082263768,0.2303123176,1.1889343262,1.2646365166,0.0945713371,-0.1564330161,-0.1214945167,0.6576443911,-0.8273627162,0.1350538135,0.2720624208,-0.1212793067,0.7692606449,-0.7661889195,0.6638042927,1.2159714699,-0.6703962088,0.4307232499,-0.5342184901,1.0875924826,-0.5991690159,0.3697001636,1.0889072418,-0.2163190544,-1.2570856810,0.6826377511,1.3048549891,0.8470559716,0.3935773671,0.7606564164,-0.9260808825,-0.3522996008,0.4411866665,1.0888419151,1.8529471159,-1.0159670115,0.1586700678,-0.2941544652,-0.7677201629,0.6702014208,3.0485541821,-0.7110427618,0.0087685026,0.2127590925,-0.6027239561,1.8547546864,-0.9267640710,-0.9728785157,-0.5564861894,0.1114080474,0.6857814193,0.5599179268,1.0410881042,0.2579667568,0.9110988975,-0.9568061233,0.1526732743,-1.4196509123,-0.7998716235,1.2206411362,1.0975108147,0.8231616616,-0.0641085878,-0.6946839094,1.6023092270,-0.6245922446,-1.9093415737,1.3474690914,1.4625424147,1.6814835072,-0.1330091953,-1.6498638391,0.0761528090,-0.4358590245,0.2756622732,-0.8825276494,-0.3384377956,-1.3949745893,1.0857973099,-0.4014986753,-2.3074398041,1.1703572273,2.3323669434,-1.2561129332,-1.0790011883,-0.7676408887,0.3213644028,1.0556669235,-1.2354131937,0.6065539718,-0.6644827127,-0.8000290394,0.0954485089,0.3187395930,-0.6467474699,-0.3926055431,-0.8689112067,-0.3884165287,-0.4145347476,0.0493419617,0.2418716103,-0.3547745347,0.6621610522,0.8859947324,-0.7657930255,-1.9621956348,-0.6089704037,-1.1985352039,0.7472391725,0.6400617957,-0.4264973700,0.2084112614,-0.4279962480,-0.1868031621,-0.1573167592,-0.1913782507,-1.0503474474,0.1423900723,-0.7872502208,1.3039880991,0.2708427310,2.0053474903,0.8214874268,1.5796195269,-0.7677533031,0.6795278192,-1.0888278484,0.7510353327,2.0150184631,-1.5912657976],[0.8089207411,-0.1970883459,0.8027809858,-1.2548944950,0.5105749965,0.4831301868,-1.5909609795,0.5785495639,1.3184320927,0.0695535466,-0.2157524079,-1.7929224968,1.2852239609,0.4923533201,0.3263441026,-0.9354546666,1.4339323044,0.2739963531,-0.0995752290,0.2425071448,0.5357111096,0.4545810223,0.5341140628,0.0805591345,1.3591556549,0.1434901059,0.2359346896,-1.2574553490,0.3136060238,-0.4058042765,-0.8217685819,1.3852506876,-0.9113008380,0.7037917376,-0.7449426055,0.9320917130,0.8437697291,-1.9806908369,-0.0670251027,0.5133256316,-0.4757016003,-1.2597001791,-0.4355548322,1.3151249886,1.1659435034,1.6419327259,-0.6083964109,0.4444638789,-0.4170851111,-1.5645383596,-1.3208391666,-2.3012502193,-1.0070270300,-1.3298571110,0.1743980348,2.5718021393,-0.6891092062,-1.6248441935,-0.5891755819,1.6138558388,-0.0993815511,0.6511113048,-0.2385976166,0.6658123732,-0.7941064239,1.3413614035,-1.2250225544,1.1066839695,-0.7866426706,-0.7299267054,-1.1541323662,2.5609581470,0.5317540169,0.3519812822,-0.6689489484,-0.1380787790,-1.1013877392,1.0932749510,0.9128538966,-0.9880322218,0.7098346949,-0.3154917657,0.6754376888,-0.0766175464,-0.3399885595,-1.0069338083,-1.2817411423,0.1082873270,1.1880060434,-0.3071840405,-1.3175830841,-0.3208683729,-0.7482139468,1.1091876030,0.0395907052,-0.0068238205,-1.8176137209,-2.2272694111,-1.0444130898,1.7971816063,-1.3809732199,-0.8909279704,1.7548052073,-1.9525624514,-0.4300846159,1.0891311169,-0.3463200033,-0.1930101216,0.8065233231,0.1266611665,-0.4590313137,1.0066936016,0.9566790462,-1.7197459936,1.0318695307,-1.4141339064,-0.0097696707,-0.1190411523,2.0956239700,0.4579995573,-0.6332737207,-1.4304943085,0.9629821777,0.3824220598,-1.4023016691,-0.5606012344,1.1431181431,-1.8398458958,-0.0627954900,-0.6901521683,1.6339272261,0.2412263602,0.1537550688,-1.1139428616,0.8870545030,-0.1872967929,0.5684547424,-1.0961941481,-1.1251753569,1.1886631250,-0.4543816447,-0.7960464954,0.9073171020,0.6863700747,-0.4552413225,1.5016720295,0.0206240546,-0.5202254653,0.5245888233,0.3571237922,-0.9080306292,1.0753819942,1.7178027630,0.7182201147,-0.5630775094,-0.0887095183,0.1610158384,-0.9418390989,-0.3211406469,-0.5363221169,0.6615352035,0.7728872895,0.5593809485,1.5761929750,0.4294976890,0.4844272435,-0.8199804425,-0.6379592419,-0.8718496561,0.4785815477,-1.6744025946,0.7056298256,1.1006934643,-0.3888773322,0.6314990520,-0.3749686182,-0.5715098977,-0.9321445823,0.6626054645,-0.1253063679,-0.6241140962,-2.5891406536,0.4141782820,-0.6123164296,0.5643692017,-0.7155497074,0.5605821609,1.6867932081,0.1108570546,-0.4897870123,-0.6119366884,-0.0666864291,0.4495974481,0.0085499445,-2.1522080898,-0.7636897564,0.2133529484,0.4067107141,0.8969731927,0.7151698470,-0.4244272411,1.2850111723,0.7844523191,-0.1529647559,-0.1405523121,0.3252138495,0.2649473548,0.5022157431,0.0850721747,-1.4688378572,-1.3880082369,0.6504700184,0.6187646389,-1.3998051882,-1.2288914919,0.1766103208,-0.4094978273,-0.2433090359,2.1143550873,1.4301925898,-0.4404291213,-0.0626156554,-0.4196424484,1.0478527546],[0.9831517339,-0.4679268897,0.3133228123,-0.9002415538,-0.7844398022,-0.3946162462,-1.5743603706,0.6414580941,-0.2618705630,0.6462222338,-0.6590774655,1.0195286274,1.0634303093,0.0653074756,0.7577853799,-0.0516320206,0.6106615663,-0.4302142560,0.8421626687,-0.2350383252,-0.8129963875,0.5303506255,0.7106423974,0.0993085951,0.2506384850,0.9918402433,0.8889755011,1.3261528015,-0.2485965937,1.1315611601,-2.1970417500,1.4103869200,-0.4955347478,-1.8461620808,-0.0154520124,0.2505555451,-1.5311112404,-2.5265429020,-1.0538671017,-1.6120213270,-1.6950901747,-0.5707432032,-0.3110536635,0.0624820627,-0.1097832322,-0.5843027830,1.3145704269,0.2922441363,-0.4884521663,0.1748099029,-0.8954635262,0.8497131467,1.3345705271,-1.5859323740,0.1828643680,0.0375167504,1.4893962145,-0.8841351271,0.1239212006,0.1301903278,-0.2170444280,-1.4712740183,0.2646017671,-0.1447753906,0.9968265891,0.0993815586,-0.6090484858,-1.3570221663,0.7227269411,-0.0219510533,0.7023817301,0.0429944210,1.1319042444,-1.0029493570,-0.9856308103,-1.9643776417,-0.6212342381,-0.5674418211,-0.2722060084,0.5762308240,-1.7308312654,-0.1668733805,-0.3293681741,-0.3582380116,-1.1411564350,1.1042435169,1.6444514990,-0.4203042686,-0.7046890855,1.9541190863,-1.5065824986,0.1620152444,-1.4102603197,0.4765975475,-0.2991090417,1.2118850946,-0.3334017396,0.3071345687,-0.6667149067,1.2665424347,-1.2411324978,-0.3781921864,-0.1119469181,0.0351102464,-0.1019369289,0.0713379160,1.8285521269,1.1005022526,0.3116882145,0.7197256088,1.1275570393,-0.1129998118,0.3050722182,0.1734147966,0.2428309172,1.0162949562,-0.5538579226,1.0453534126,-0.2064165026,0.2954626679,1.1894056797,1.6967473030,-2.0918354988,-0.5620151162,-0.6162993908,0.1899436861,-0.1020427197,-1.0234582424,0.4789095521,0.0332386084,-1.4767955542,-1.9037156105,1.2354108095,-0.0018077961,-0.9863997698,-1.2346477509,1.8885161877,0.2936889231,0.8233779669,0.0891269967,0.4238564074,0.4432381392,-0.8548453450,0.3422487080,1.9250447750,-0.8454698920,-0.6730789542,0.1415164173,0.1184886545,1.3533905745,-0.2967815101,1.1134816408,1.2478499413,0.1484726369,0.7938684225,-0.0986338928,-1.1890305281,0.2958755791,0.1388797909,-0.6163766384,-1.4716683626,0.8207072020,-1.1493715048,-0.2178318650,-0.9332431555,0.0315085016,1.5580168962,-0.3341762424,-1.3798252344,1.9283100367,-0.2898777425,-0.9559110403,0.7655615211,-0.4317200780,0.4280168116,0.7353171706,1.4805815220,-0.2730192840,1.7999408245,-0.6273670793,-2.6943187714,-0.6990944147,1.1060009003,0.8578431606,-0.0840007663,-0.5585980415,-1.2817488909,-2.4378654957,1.6060805321,0.8997866511,0.6551699042,-0.3740907013,1.3194372654,-0.4168642759,-0.8414853215,-0.7929033041,0.3561357856,-0.3539623916,0.2779755890,0.4590013325,0.5880220532,-0.2773589492,-1.8300926685,1.2118127346,-1.0504593849,-0.9004604220,-0.0812280998,0.2696877420,0.0787571594,0.1149807945,1.3142724037,-0.1294342428,-1.7291231155,-0.0914452523,0.1939240694,0.6503858566,0.3664641976,0.7926014066,-0.2039321363,-0.4512749016,-0.5041785836,0.1370467991,-0.3436326385,-1.7019400597],[1.3645384312,-1.0790003538,-0.0885153338,0.7273120880,-0.0962846130,-0.3853226304,0.9186987877,1.6411539316,0.0192694329,0.9914398789,1.0116108656,-0.2402564734,-0.2873272002,-1.0821862221,-1.9422683716,-0.5636457205,-0.9914990067,0.7986121178,-1.1720125675,-0.8651517034,-1.0168634653,1.3929049969,-0.5326869488,0.2972522676,-0.7363356948,-1.0323220491,0.2945378721,1.2807190418,0.2211526334,-1.8246880770,0.2985337377,-0.1622590274,-0.5336527824,0.5530636907,0.9003993869,1.0602345467,0.5563013554,-0.4820825756,-0.3741382360,1.2548416853,0.5277336836,1.0442218781,-1.0164369345,0.9119832516,1.8069756031,-0.8453161120,-0.6892279387,-1.9548838139,0.5399494767,0.5839894414,0.6357702017,0.4068482518,-0.8281987309,0.7952207923,-0.5682151318,0.6527551413,0.4371876717,-0.8736358881,0.7888579369,-0.2305625677,0.3886904716,0.4261684418,-1.7041398287,0.1460140944,-1.3941833973,-2.3413891792,0.3794504404,-1.5065003633,-1.0899549723,0.5814636946,-1.8431659937,-1.9167656898,1.0604321957,-0.6192001104,0.1313293129,-1.9399279356,-2.5218393803,0.8841423988,1.4313941002,-0.0698332638,-1.7999757528,-0.5638673902,-0.0389924608,0.8049888015,-0.9703688622,0.3512552083,-0.2308388054,0.1044155210,-1.2669880390,0.4026793242,0.9971115589,-1.0468974113,-0.1251971573,0.1129145920,-0.6320928931,0.7838298678,0.4312362075,1.1978577375,-0.6947177052,-0.2477532178,0.4473966956,1.3458278179,-1.3238114119,1.2147200108,-0.3629972637,-0.1120925844,-1.6106210947,-0.6715474129,-0.3695737123,-2.1063737869,0.3464091718,-0.1429023743,0.0154739022,0.2679836154,-1.3591574430,-0.4414421618,-0.5401853919,1.2818124294,-0.8664167523,1.2607386112,-0.5515830517,0.0223816186,-0.9567921758,2.1468987465,0.4018232524,0.3902551234,0.2271844745,-0.7034388781,-1.1465339661,1.3632919788,0.7479792237,-1.4447779655,-1.3149640560,1.6014412642,0.8386735916,-0.9777460694,0.0764696151,0.8212373257,0.7903966904,0.2094957680,1.1794633865,0.3876317143,0.2945517600,-2.2883436680,-1.4445925951,-0.0050359364,-0.0154747693,-1.0497438908,-2.1858596802,0.3199483752,0.4688971639,1.3384841681,-0.1049553305,-3.5104098320,0.0139802890,-1.0582283735,0.0157511942,-0.1317526698,0.5070823431,0.2266791016,-1.0741680861,-3.3275530338,-0.1029340774,1.3333551884,0.1533930302,-1.3400937319,-2.1410186291,0.6519215107,0.0122133074,-0.0897119120,0.7088866830,1.3721456528,0.7374876738,-0.1000189185,1.3357746601,-1.4139446020,0.6488098502,0.5202904344,0.3015293181,-2.3389220238,0.8091678619,0.6978511810,0.9705064893,0.3247336447,0.0050228359,-0.8070387840,1.5278664827,-1.6689298153,0.7235268950,0.3391357362,-0.4543726444,0.7102225423,0.0603245422,-0.2386469692,-0.1376174837,2.2262902260,0.7978293300,-0.1623223573,1.3508191109,-1.1919469833,0.5105102658,-1.2463465929,-0.7253215909,-1.5177099705,-1.8005180359,-2.0953040123,-1.4494000673,2.0138463974,-0.9422380328,0.4424597621,-0.1397095770,-1.1371566057,-0.0423600972,0.1930373460,0.2709112763,-1.1953920126,0.5570302606,-1.5900931358,0.4856736660,-0.5310302377,0.1510527581,0.3505887687,0.6495220065,-0.3104924262],[-0.2100895643,-0.6556578875,0.0484873876,-0.4949260950,-0.5762988329,-0.0790554658,-0.8858715892,-0.7482577562,0.4565393925,0.8495110869,0.5410699844,0.2003258765,1.1066104174,2.4233675003,1.1743931770,-0.3243298531,-1.0906335115,-1.0359903574,1.1506870985,0.8038827181,0.0065249666,1.2016972303,-0.4348216951,0.4592243135,0.9285517335,-0.9519800544,0.9002231956,-0.0417470746,-0.6550382972,0.3652853668,-0.4817460179,0.4155755043,0.2305550873,0.0383341387,-1.3802094460,-1.0148710012,-0.4687513411,0.2466093600,-0.0788045675,0.3197855353,0.0394861288,-0.9513927102,-0.0540379882,-0.9681669474,-0.6324351430,0.1304977834,-0.1444966942,0.4706237316,0.3779866993,0.2013960183,0.2234681100,0.4384725988,1.8610857725,0.1288777292,-0.5215864182,-1.0086157322,0.7050257921,0.7817319036,-2.3805580139,-0.2110639066,-0.3800990283,-1.5330798626,0.8230214119,0.2022407502,1.2536604404,-0.3143420517,-0.3846403956,0.8717328906,-0.2717157006,0.2214356810,-0.7705176473,0.7046155334,0.6591244936,0.0442096666,-0.8060303926,1.8267966509,-0.2754566073,-0.8256037831,-1.5357751846,-1.0315228701,-0.8241909742,-1.0270508528,0.3138917387,-1.6600333452,-0.3888295591,0.0759017095,-0.5470116138,0.9134837389,0.9181766510,0.6794993877,1.8896580935,1.2877893448,-0.5754200220,0.1308063716,0.6196119785,0.2646083534,1.3591319323,-0.1376973838,-0.4108622372,1.0915858746,-0.4022491574,-0.2787691951,-1.4625638723,1.5055365562,1.3839673996,0.3047131300,-0.1136100814,0.0511076674,0.9380768538,1.2948635817,1.8470802307,0.0492418595,-1.2036857605,1.1663081646,0.4418370724,-0.2655751109,-0.9233930707,0.8207384348,0.5034034252,0.5235258341,0.9936625957,1.0459955931,-0.2367948443,0.4002315700,1.7345780134,-1.6753051281,0.9616335630,-0.5974237919,1.1064232588,1.8339977264,-0.2873453200,-0.2717796862,0.6485736966,-0.7970598936,1.2907695770,-0.1626946926,1.3069441319,-0.6765155196,-0.8155764937,-0.4989302754,1.5502585173,0.6216536760,0.3171115816,-0.9816764593,0.2284350544,0.1353972405,-0.7532781363,0.5987361670,0.2846145630,2.6697471142,0.8273726106,0.3656917810,0.3345196545,1.3987786770,-0.6768500805,0.9154971838,0.9169610739,0.6782941818,-2.2053909302,-0.2868286967,1.4951024055,1.3020434380,-0.1167226881,0.5561181307,1.3928959370,0.8201711178,-1.3544884920,0.7482158542,-0.9934462309,-0.7618706226,1.5368840694,-1.0926820040,-0.7544658780,-0.7657099366,-0.2162702084,-0.6240744591,-0.6869540811,-1.3244251013,-0.0615085214,-0.2740594745,0.7680806518,0.5850219131,0.7697795033,-0.8452374339,-0.2424748093,0.8591217399,-0.1297827959,-0.3272397220,0.5281672478,2.8159592152,0.0707529411,-0.0019932245,0.7365709543,-0.1870024353,-1.0456169844,-0.0245733839,0.3742526472,0.0466749147,-0.9359126687,-0.5803662539,1.2145210505,0.2210874259,0.8019478917,-1.6911070347,-0.8182746172,-0.7799111605,1.5378501415,-0.0670324489,0.5352994800,-0.9907878637,0.0810974091,0.0742879882,-2.1321122646,-0.2070179731,1.0230563879,-1.4774166346,-0.0569191352,-1.1915849447,1.2854273319,0.5736584067,-0.3161622286,0.0788122043,-2.6321368217,0.7516192794],[-0.6747380495,1.7880039215,0.0600294322,0.8084431291,0.8202488422,0.7850489616,1.7138910294,0.8183184862,1.3371344805,-1.0704429150,0.7927608490,-0.7731685638,-1.5480746031,0.9363201261,-0.8637107015,-0.5787624121,0.5742868185,1.2831314802,1.1431300640,-0.1921418905,-0.1575302482,0.7531200051,0.1107663810,-0.1392740458,0.1642324924,1.1048551798,-1.0657318830,1.2152526379,-0.0280989539,1.4245631695,0.3038326204,1.0398333073,0.5144931078,1.3959500790,-1.4316306114,-0.4561814964,-0.5418317914,-1.2976498604,1.6599001884,0.6544582844,0.4120349586,0.1889998317,-2.0116407871,0.0190515742,-1.0315045118,-0.2084806114,-0.9521855116,1.3717107773,0.8201537132,1.2848557234,1.0412549973,1.1097702980,-0.7424427867,-1.3213529587,1.4162770510,-0.2218238860,0.1572814584,0.0420207120,0.8158637285,0.2443248034,1.5342217684,1.2111642361,0.7858352661,-1.0876142979,-0.3700317442,-0.2657727301,0.4492015839,0.0319982916,-1.0240249634,-2.5414311886,0.1361544430,-0.2829069793,0.5869057775,-0.8466492295,1.4590594769,0.6824162602,1.3178961277,0.1956192702,1.9053285122,0.2270766795,0.8468378782,-1.1426655054,0.1628837287,-2.0766322613,0.3954115212,-0.7994513512,0.8160585165,0.2688303888,0.6566452384,1.4310846329,1.2181339264,-1.1336773634,0.7861024141,0.6040832400,1.0225002766,-0.0625130981,-0.9539570212,-0.5392764807,-0.9254257679,-0.8865107298,1.5495288372,0.3416455984,-1.4592988491,1.1174447536,-0.8456580639,-0.2513967156,1.6444369555,0.4698045254,-0.3425119817,0.4703255594,-1.1068733931,-0.6331372857,0.0833904520,2.0509891510,1.1461318731,-1.4606683254,-1.3410766125,0.0828015357,0.8186067343,0.3983367383,0.0698770657,0.4072398841,-0.0359241553,-0.1619289368,0.8406882882,1.8046275377,0.1601787359,0.6465020776,1.1391314268,0.0097278478,-1.3131502867,-0.8829632401,-0.5586873889,-0.8721792102,-1.7141357660,-1.6847779751,0.9566531777,0.0900609344,-2.1673758030,-1.5872809887,-0.4245087802,-0.6903084517,-0.3588181734,1.3054128885,0.7735430002,-1.5612035990,0.0641486570,1.7226607800,0.4291057289,-0.8341003656,0.6271658540,0.8506008387,-0.6141491532,1.2932060957,1.0017577410,0.9150823951,0.9628162980,0.2699362934,0.0677945018,0.6238765121,0.8336200118,-2.0658693314,-0.8160885572,0.6131067872,-0.6992746592,0.0867898762,-0.9291868210,0.3126922250,-1.0321285725,1.1907221079,-0.8579597473,0.3683812320,0.0951150134,1.4797778130,0.1138498932,-0.1657631844,2.1404304504,0.6909690499,1.0141973495,1.3627005816,0.9600053430,-0.2231254727,0.8661641479,0.6439171433,-0.3375835419,-1.4438884258,0.5471693873,0.2623979449,1.4910924435,0.1808727682,0.0957317352,1.1674259901,1.0773286819,-0.8375552893,-0.1509543806,-0.3889933527,1.2889469862,-1.6503235102,-0.6901810765,1.3719030619,0.2788748145,0.2398491651,-0.1003404036,0.2019098103,0.8351427317,-0.3547764421,0.0477411337,-0.2419873178,1.1113913059,-2.3828821182,-0.3928278089,0.2063228041,1.4378747940,0.6241122484,0.6654278040,-0.8381136656,-0.1943056434,0.8262708783,-0.8781173229,0.7365460396,-0.2867861390,1.0262185335,-1.0862541199,-2.2211563587],[-0.0351699404,-0.1053142026,1.0903381109,0.0490656979,0.6290349960,-0.1448164582,-1.0651451349,-0.2358892411,-1.5510635376,-2.0036041737,-0.8184807301,-0.7312480807,1.6621870995,0.0102839842,0.0925677046,1.7449682951,1.3801565170,-0.6567270160,0.5415236354,-0.2259770483,-0.6260111332,1.5308971405,1.0322775841,0.9775959849,0.1011903435,-0.1734205186,-2.5078294277,1.1471949816,-1.0667153597,-0.1465885788,-1.1242318153,-0.6100597382,-1.3902848959,-0.9408340454,-0.5747480989,-0.7050228119,0.2044974118,0.1679736227,-1.5893906355,1.6384850740,0.0945908651,-0.5734204650,-1.4962567091,0.8425856233,-0.7955751419,-0.7523640990,-1.3928989172,-0.6434774399,-0.0267408527,-1.3192676306,-1.1446229219,0.8909816146,0.7566283941,0.4858905971,-0.4653729498,0.2230161875,-0.3106475770,-0.9960952401,-1.9371960163,-0.0624998100,0.2280373722,-1.4546812773,0.7684928179,-1.3647917509,-2.4256286621,-0.8073800802,1.0723882914,-0.9055548310,0.8978029490,0.3257782757,0.6538295746,-0.2891353965,-0.1892136186,2.2979393005,1.4039591551,-1.9162375927,-1.5704979897,-1.2596937418,-0.5759724975,-2.3971576691,-0.5725220442,-0.8249366283,-0.3085795045,0.5204046965,-1.2902706861,2.2250847816,-1.3025299311,-0.5453331470,0.4720706940,-0.6122683883,-0.6175191402,1.1907383204,-0.6027181149,-0.1497769952,0.3475134373,0.0696816221,0.4372898340,0.6333782673,-0.1903668344,-0.5238707662,-0.2407862246,-1.7575255632,0.6175248027,0.4117534757,0.5725713968,-0.1537111849,-0.2685767412,-0.1302557737,0.6916129589,-1.7661845684,-1.2384231091,-1.5353415012,-0.8544256687,0.8948006630,-0.8968293071,0.4587621093,-0.6570677757,-0.6455622911,-0.3647587895,0.6582919955,0.0151019888,0.6143096685,-0.4849007130,-0.0224776696,0.8556644917,2.1596455574,2.0912363529,0.6864846349,1.9000794888,0.3535133600,-0.4887687564,0.2296750844,1.7741783857,0.8200288415,-2.3570518494,-0.6121400595,1.1244128942,-1.9312944412,0.2823550999,0.2266518772,2.0878908634,-1.7262382507,0.3788723648,1.0462868214,-1.2126829624,-1.5116736889,1.8529467583,-0.1404440552,-0.1386186481,-0.7274141908,-0.9291960597,-0.7300670147,0.5603982806,0.3851436377,-0.1989250630,3.1476488113,-1.4545503855,0.2226756215,0.6318311691,-0.5034347773,-2.0605225563,-0.2405045331,0.7324491143,0.2608508170,1.1732854843,0.3615217805,1.1910451651,1.1258134842,0.8236389160,-0.1661823541,2.0450708866,0.1467454135,-2.3809611797,0.4373816252,-1.4531396627,0.8588591218,-0.1570577621,1.0039044619,0.2207947969,1.0857644081,2.3349044323,-0.4538405836,-1.0556931496,-0.1135712489,-1.3338731527,2.0147054195,0.5743921399,-0.0406258553,1.0636233091,-0.7739616632,-0.4744435549,0.6629287601,0.6593925357,0.2389109284,0.3244543970,-2.4939632416,0.2376910746,0.2401821762,0.1714852750,-1.6680035591,-0.2254704535,0.1158540174,-1.5890207291,-0.0886749774,0.8084253669,0.2941642106,-0.2246614844,-0.6902972460,-0.9486091733,0.1864258349,1.3744177818,1.2856972218,0.0242699496,0.7165759802,-0.7738999128,-1.5016100407,1.3694467545,-0.5406073928,-0.5955885649,0.8310559988,0.7758463025,0.0332392007,0.0597205423,0.7009345889],[-0.1214630902,-1.2265552282,0.9551652670,-0.9825314283,-0.0588454418,1.3474606276,0.2749947906,0.7564658523,-0.9267882109,0.2939739227,-0.7862483263,0.3147085011,-1.0929533243,-0.4037205279,0.1726106852,-0.9854048491,0.8076828122,0.5334472060,-0.5711045861,0.2277940810,-0.3884009719,0.0958303660,0.4074799716,-0.1783480793,0.6015118957,-0.2043264210,0.6595416665,0.5929055214,0.2067304552,-0.4159007668,0.0456653237,-1.0317988396,-0.0439903848,0.2529656291,-0.5262223482,-0.0891434103,-0.5803508162,0.2372836471,0.1403294653,-1.1127064228,-0.7271862030,-0.3711755276,0.4906193912,-0.9802835584,-0.4931763113,-2.4230916500,-0.6911710501,0.2911610901,-0.3813099563,-0.9066234827,0.4782035351,0.0682695433,-1.2720994949,-2.0620915890,-2.0388426781,-0.8398178816,0.1875489354,-0.6239607930,0.1227937043,-0.8136839867,-0.4940828085,-0.2377815545,-1.1428968906,-1.3174897432,0.2686184347,0.2657438219,-0.0437186994,1.7247594595,-1.2954945564,-1.5489299297,-0.0903909281,-0.2665699422,0.8504773974,0.9000029564,0.1670958251,0.2365684360,-0.6508274674,0.6521868706,-1.2457232475,-0.4461270869,0.0367440879,0.8848665357,0.4277692735,0.4192968607,-1.8041940928,-0.8850283027,-1.4226913452,-0.1611105055,1.5754060745,-0.7368183732,0.8704988360,0.5925328135,0.3334204555,2.3063228130,0.5068302751,0.6421422362,0.2884857357,-0.6937939525,-0.4742089808,-0.5211635232,0.5846737623,-0.6871022582,-0.5160235167,0.4397167861,-0.1327747852,-0.3461024463,0.8904767632,-0.3413029611,-0.7286113501,-0.3106502593,-1.1513842344,-1.4070318937,1.4902381897,0.7819204926,-0.5711467862,0.9897950888,1.0421978235,0.0064553972,0.2257383913,-0.5662776232,-1.6784616709,0.1300082803,0.3907846808,-0.1842308342,-0.5939362049,0.4409166574,-0.4065026641,-0.2743496001,0.2813808620,0.3920689821,-1.1994462013,-0.4680491686,0.3003554046,0.8930982947,1.3606835604,-0.8178442717,-0.9329233766,0.8158513904,-0.2013928294,-0.7077201009,1.0480406284,1.7602568865,1.0767135620,-1.1549855471,-0.4318770170,0.2878724635,-0.1249971166,0.0026210824,-1.9303877354,0.2099825144,-1.8180750608,-0.5248698592,0.5573206544,-1.3221145868,2.1918184757,-0.1596627235,-0.7318202853,-0.3619192839,-0.2752758861,-1.1489162445,-1.0136946440,0.8739019036,-1.1305433512,-0.6433794498,2.3157281876,0.4867565036,0.2869711816,0.7111119628,-0.7375450134,-0.2039852291,-0.2768211663,1.3463346958,-0.3390847147,-0.6786736250,-0.7262747884,0.8902787566,0.9133289456,-0.2505047619,-0.0415848196,-0.3829526007,0.5145270228,0.5090532899,-0.8748589754,0.5908644795,-0.8097217083,-0.6209503412,-1.6769145727,-0.7063034773,-0.0019511768,-0.7479312420,-0.4937320054,1.2623711824,-0.0325878747,-2.1162991524,-1.3957357407,-0.4022807777,0.2463667393,-0.2883108854,-1.0714792013,1.3542736769,0.3475864828,1.1947330236,1.4123615026,0.4734445214,-0.0291386191,-0.7498828173,1.8264600039,0.8586798906,0.8935324550,-1.2798057795,-1.2434326410,1.4402679205,-0.3074516356,-0.3824082613,0.9012572169,1.3327422142,2.0704622269,-0.7559067011,-0.6273303628,-1.3138438463,-0.5460834503,-0.1313429028,0.3516915143,-1.2709006071],[0.2318837494,1.3678478003,-0.5695576668,0.9467203021,-1.0783817768,1.8729202747,-0.0971904844,0.0755688325,0.0776100382,2.0316021442,0.1603736281,0.5459352136,-0.9726063609,-1.1684632301,-0.6058312654,-0.8898193240,0.2162799835,0.2188819796,0.6266080737,-0.9264500737,-0.3930732012,1.2791380882,-1.4856559038,0.6038267612,-1.5345782042,-0.8310987353,-1.1575284004,-1.6151446104,-0.3164165914,1.6876577139,0.3008544147,-0.0813643858,2.1535699368,0.6083881855,2.0528795719,1.0854481459,-1.8040492535,-0.8503279090,-0.9169733524,-0.7187844515,-0.0188348703,0.9217076302,-1.1328121424,-1.0641452074,0.6557211876,1.6046416759,-0.8734141588,-1.1367805004,0.7530630827,-1.2990561724,-0.0882783160,0.0793868899,-0.1091432571,-1.5902625322,0.3879348934,0.0378575362,-1.3857095242,0.0273315851,2.5557720661,0.2602879107,1.6669652462,0.0515464433,0.2788645923,-0.4719827473,0.6808572412,-0.5266553164,1.5635540485,0.6531999111,0.5617042780,0.3659709394,1.2030426264,-0.3417057991,-0.3226774931,0.9697617888,1.1391057968,0.6558498740,0.2402107716,0.1402628124,0.0865132883,-1.1623040438,1.4959161282,0.4355273247,-1.5472781658,-0.1442041993,0.4357748032,0.3796449900,-0.1988835186,1.2006313801,-0.3266404867,0.4909732342,-0.1155063435,0.7805097103,-1.5917422771,0.2558749318,0.0028779260,-0.9973745346,0.3408148885,0.6907517910,1.7726098299,1.2931444645,0.0731726885,0.0033546537,-0.5160992742,0.9919461012,-2.0193617344,0.0540464595,-1.0469379425,1.8422071934,0.2026942223,-0.2223332375,-0.8785908818,-0.4768011868,-0.6508824229,1.4654546976,1.9984804392,-0.3018554151,0.5743958950,-0.9103906155,2.3126950264,1.1369340420,-1.1978269815,-0.5591164827,-1.3324406147,1.4774297476,0.4085949659,1.0875927210,1.2534742355,-0.2782524824,-1.2699588537,0.2338881344,-0.1154674813,1.3444168568,0.5542261004,0.5446771979,-0.7692145109,-0.8970652223,-0.4507069588,1.7428719997,-0.4185885191,-2.0615775585,0.5384839177,-0.1812524796,0.5966222882,-0.0884481892,-1.3890218735,-0.5953192115,-0.8984060884,0.2904509902,1.3895179033,-1.2992860079,0.3779320717,-0.0900823697,0.1684620082,-0.2983181477,0.7044838071,-0.3086249530,-0.4766749740,-1.6568388939,0.3046866953,1.1033426523,0.9387074709,0.5648691654,1.1102232933,-0.5370892882,1.6050131321,0.3298608363,0.8434912562,1.5994970798,0.3545402586,-0.9816116691,0.5793850422,1.8203120232,0.6507356763,-0.6160635948,-0.4756502509,-0.1226160377,-0.4512326717,0.9120926261,-0.0639616176,0.6028991938,-2.1917812824,-0.7067122459,-0.1262857169,1.8095263243,-0.6735818386,-0.0236892402,-1.2794020176,-0.3109898567,-0.4389147758,0.1079273149,1.0574131012,-0.3642730713,-0.5820379853,-0.0673111752,0.6866626143,-0.9802171588,-0.4072963893,1.5017372370,-1.3176370859,0.0456849113,0.6719582677,-0.6977844238,-2.4139885902,0.9166274071,1.9176933765,0.0757182539,0.2686379850,1.2910649776,-0.5725409389,0.0763535500,-2.0364468098,0.0009822916,1.2169647217,0.5885798931,-0.9567729235,0.0846289247,1.2598075867,-1.2678459883,1.2660022974,0.5127959251,-0.3129904270,-0.7242843509,-0.2958310246,0.2941564023],[0.3908843100,0.1993192434,0.8601083755,-0.0118097821,2.9144937992,0.5598602295,-0.7121825814,0.6126078963,-0.4042938352,0.2696480155,0.7432195544,-0.3382290304,-0.3175045848,-1.0074145794,-1.1684331894,0.0796425790,-0.4121067822,0.4740624726,1.3114098310,1.0700200796,0.1490948498,0.6387857795,-1.7708073854,-0.9917168021,-2.5573399067,0.6741188765,-1.3366583586,-0.5455180407,-0.2197612971,-1.4311057329,-0.6745821834,-1.5028071404,1.0440351963,0.7176719308,-2.6579515934,-1.0510629416,-1.2848021984,-1.3853023052,0.4791358113,2.5045385361,-0.4729622602,-1.2650476694,-0.7059856057,-1.0171307325,-0.7169947028,0.1002878100,-0.3277071118,-0.2671995163,0.5931706429,0.2614943087,-0.1734369695,0.0991713181,0.1256472766,-0.3000682294,0.7157314420,1.0789563656,0.2905321717,0.0915650576,-0.2459096760,0.9417273402,1.1695841551,-0.4629750252,0.6636813879,1.2860013247,0.1320593655,-0.4724427462,-2.4939298630,0.8686680198,0.7980304360,0.0212606862,0.0895598605,-0.6142240763,0.2135329247,1.5683747530,-1.3909196854,1.9122843742,-0.1695288122,1.0792137384,0.2949759364,2.9196517467,0.1250986755,-0.9956979752,0.8259578943,-0.4130439162,-1.5986818075,-1.3872983456,-0.3145367205,0.1179488599,-1.7167025805,-0.3398543596,-0.2405494601,0.1653799564,2.2529299259,0.1527817100,-0.0522611253,1.6368466616,1.5746954679,-0.6853815913,0.7834398150,-0.4030088484,-0.9495356679,1.5865054131,0.0136783160,-1.1354285479,1.2099617720,-0.7505401969,-0.3670379817,-0.9019443393,-0.0718230233,0.3455559909,-1.0303230286,1.2956593037,-1.3617696762,0.6880182028,-1.8390884399,1.1746094227,0.2042525560,1.8721073866,-0.2429490089,-0.9037572145,0.4598532319,1.4084200859,1.4145408869,1.2590812445,0.4491093755,-0.2917197943,-0.1804877669,0.3747444153,-1.2223923206,-0.0042179539,-0.0286030844,-2.1629972458,0.8808540702,-0.4634902477,-0.4097259641,0.1188818812,0.8841651082,-0.7411962748,1.3089855909,0.2433037907,0.2072167397,0.0030417852,-0.4322900772,0.0580277666,-0.8421820402,-1.1227341890,-0.4095312059,-0.6351722479,-0.9240295887,0.1515339166,-2.7080621719,-0.8268998861,0.6793233752,0.0793776214,1.5474833250,0.0869547799,0.4061647654,-0.1170388460,0.7532392740,0.2103636712,0.2250218242,-0.3768270612,0.3614977598,0.9480288625,-1.6463083029,0.4892472327,-1.7122375965,0.5919700861,-1.8297494650,-0.3791226149,-0.4474929571,-1.0941311121,-0.6403060555,0.1746183336,-1.7660150528,0.0863045081,-0.5001140833,0.6139918566,1.6994225979,0.1999011487,-1.9983192682,-0.7136126161,1.4435380697,-1.1255981922,1.4526222944,0.5327004194,0.0478152595,-0.1422421634,1.5422103405,-0.4843623638,0.5077458024,-0.4235121012,0.4101985395,0.2985644937,-0.4474204183,-1.4437913895,0.0946096405,-1.0242272615,0.3150271475,-2.1535305977,-0.0214283168,2.2918503284,1.6116689444,-2.0501370430,0.4185002446,-1.2304670811,0.5945562720,-0.4853458703,1.2488714457,0.2259249687,-0.1908799410,0.0583935194,0.4459559619,0.4773775935,1.6011041403,0.7079492211,0.4374597669,-0.5841044784,0.9688305259,0.8779850006,-0.0789694786,-0.1578722596,-1.8777650595,0.0557074770],[0.2137283534,-0.0753122121,-0.5718395114,-0.6800060272,1.3663361073,-0.3309047520,0.4557212293,0.4449327290,0.9501424432,1.0727024078,-0.9129831791,0.7791337371,0.8639063239,-0.7943357229,0.9940410256,-0.1175013930,1.6619648933,1.3453713655,0.1633823812,-0.1596832424,1.5186958313,0.9925870895,0.7669295073,-0.2227122933,0.9052174687,0.1742046773,-1.8046705723,-0.3167853653,-0.4060903490,-0.3562828302,0.9901017547,0.5220876336,1.0828599930,-0.9335242510,-0.8265603781,-0.5601838231,-0.1037390828,-0.0347839370,0.8033310771,-0.7626441717,0.1769310981,-0.8562979102,0.6401655078,0.4897463024,1.2890980244,0.3583827019,1.1658328772,-0.0718992352,-0.8952594399,-1.3356628418,-0.9950582385,-0.6626952887,1.5818070173,-1.6137483120,1.6730548143,1.0554708242,-0.7988551855,0.1265448183,1.4124367237,-0.4724154770,-0.9337536097,0.8365861177,2.3900251389,0.8229386210,-1.1517981291,0.6948511600,-0.1482112110,-0.5633614659,-0.3132559359,-0.1994884610,-0.8831216693,-0.1983086616,-1.2630894184,0.5556268096,0.9699519873,-0.1344390661,0.5338910818,-0.8431673050,0.2585937381,-0.5014234185,-0.9196692109,0.6794525981,-0.4677133858,-0.3580030501,-1.0047782660,-0.4246050119,-0.8890936971,-0.1875797808,-1.3680918217,2.4937102795,-0.2065069526,-0.3412998319,0.9275464416,-0.6399300098,-1.0326942205,-1.2631584406,-0.7057926655,0.2132282406,0.1349920183,-0.0767279118,-0.0180238225,0.9167872071,1.2965246439,-0.4996848106,-0.1523783654,1.3595687151,0.5616595745,-1.2922233343,-0.2067446113,-0.0955479667,1.0408117771,0.2260160744,1.5209286213,0.7850476503,1.2328828573,0.4577864110,-1.0894739628,-0.5421277285,1.8188209534,-0.4353050292,1.0453416109,0.9334639907,-0.8626065254,1.1428796053,0.4184126854,0.8542534113,0.2786234617,2.2667417526,-1.3422918320,0.0365474224,-0.6399766803,0.2765336931,0.6240792274,0.9419114590,1.2190315723,0.9540512562,1.8149379492,-0.5880147219,0.5229048133,-1.8234910965,0.5798783898,-0.4019498825,-2.4706156254,2.2076125145,0.0733084604,-0.8490743041,-0.1897683442,-0.0908034369,1.3684400320,0.3928494453,-1.5093868971,-0.9352346063,-1.6818646193,-0.3056127131,-0.9864490628,-0.1267329901,-3.4295580387,-0.8379977345,-0.2364562154,-0.0146869794,0.2902716100,-0.3721041679,-0.5221943855,0.1014696434,-0.0193115585,0.1473674327,-1.5494464636,-0.5399037600,-0.0246628616,-2.0717842579,1.0729697943,-1.1848657131,0.4384807050,-1.3108595610,-0.8152757883,-0.1412737817,1.5141129494,-0.1638685614,0.8585687280,-1.1072340012,-0.8543540239,1.0696147680,0.1962230355,-0.1166522354,0.4228607118,0.3876480758,-0.2862882912,-0.9716600776,-1.0824446678,-0.5052465796,-0.0500417948,1.0491191149,0.8568240404,-0.6495133638,-0.3671525717,0.0186419897,-1.5158751011,-0.0458600409,-0.4317787886,-0.0550946295,-0.8744986057,-0.1637722999,-1.1921287775,-0.5725089312,1.6396950483,-0.1935610920,-1.8141798973,0.5524504185,0.0224844627,1.7025898695,-1.1939163208,0.0611830056,0.4557733834,-1.8053938150,-0.1088439003,1.3869347572,-1.3557333946,-2.6577615738,-0.5621011853,-1.5449957848,-1.4346992970,1.6745887995,-0.7087044120,-1.1405794621],[0.8468211293,-0.6259702444,-1.0013346672,0.8264898062,-0.0107424371,-0.0377501473,-1.6358803511,0.1592251062,-1.2914931774,0.4267860949,-0.0591404252,-0.3762255311,0.7003886104,0.3074934483,2.0389971733,1.0119352341,-0.8226264715,0.1964271665,0.6776129007,-1.9366357327,0.2702591717,-0.7943894267,-0.5161536932,-0.9527539611,-0.1551645100,-0.3698361218,0.5130124688,1.6964564323,-0.4065807760,-1.2760372162,1.2567688227,0.0966834277,0.5076339245,0.3789902031,0.8715313077,1.5521655083,-2.2539381981,-1.0728610754,1.2374900579,0.6128914356,-1.1944071054,1.9067157507,1.6737689972,1.1818349361,0.3899155259,0.4014538825,-0.5103981495,0.1652175784,-1.9009075165,-0.2527178824,2.0375125408,1.6334018707,0.0990877450,-0.2214143127,-1.2417927980,1.4322711229,-0.8110809922,0.6237356067,-0.8600297570,0.9621957541,-0.1469984949,-0.7951014042,-1.2351981401,0.3188866675,-1.0440903902,-0.9646935463,-0.4229103625,0.4964722693,0.1531331837,2.7971539497,-0.3278644085,0.5300644636,-1.1382143497,1.2793962955,0.2531116009,0.8383103013,-2.0147705078,0.6147590876,-0.2501372099,-1.6798156500,-1.1679533720,1.9104994535,0.7335022688,-0.6242956519,-0.1997241825,0.6580860019,-3.5438032150,2.2302262783,0.2623988092,0.4263668358,-0.5162287951,0.9116119742,0.7537800670,0.8147564530,2.0275216103,-1.0469170809,0.0447648279,0.4066671729,1.4574295282,-0.5511431098,0.1064528897,2.2086167336,1.0268939734,-0.6577744484,-0.1552726179,-0.0653077587,0.8309588432,-0.2720114291,-0.4248123765,0.4521262944,1.8212699890,-0.2710549235,-2.6737802029,0.8627644777,0.4140576720,0.7305349112,-1.1009577513,-0.0405872762,-1.7268170118,-0.9778028727,-0.3233233988,-1.1608653069,0.0106323780,0.0213840399,0.5337206125,-0.5303680301,-1.8097007275,1.0664070845,-0.0200915802,-0.8092259169,-0.6417803168,-0.2247069329,0.2727496624,-0.0173603650,-1.0712687969,0.2251258940,0.0471394435,-0.0443964601,0.7301927209,0.3327662051,-0.1669893563,-0.0900482908,-0.1879023314,-1.1513732672,-0.3616277277,-1.6583201885,-0.1897500455,-2.0509023666,1.4391367435,-0.3599698842,-0.6990902424,-1.3491034508,-1.0897932053,-2.0706076622,0.7112004757,0.0375067219,-0.6403739452,0.4059659839,0.6470164061,1.8503149748,-1.6849238873,-1.3065844774,1.0666915178,0.9946438670,-0.6309731603,0.2481388897,-0.6018763781,-2.1274530888,-0.4725838006,1.3387097120,-0.3620197475,0.4139246047,0.5800728798,0.0942340717,-0.3209901452,0.0889385194,1.1412287951,-2.3286304474,-2.0180230141,-0.7359105945,0.4361290336,-0.3935990930,-0.9925333858,0.5121065974,-1.0553890467,0.0176953413,-2.1810755730,0.4622453153,-0.2540527582,-0.7596648932,-0.0581332184,-1.7947191000,1.0270496607,-1.2321225405,-0.3689308763,0.6238868833,0.1727206409,-2.8847465515,0.2620530725,0.9489654303,-0.3782636225,-0.2526799142,1.0940079689,1.0400370359,-0.4539102912,1.4904272556,0.3527225256,1.2626172304,-0.8670571446,-0.3333664834,-0.3730029464,0.2025904655,-0.3326246440,-1.2971646786,0.9655994773,-0.1379029453,0.7102864981,0.8918810487,-0.9970547557,-1.4230805635,0.8162118793,0.7776215672,0.1969143003,-1.3174074888],[-0.5844110250,2.4673445225,-0.5167017579,0.6806137562,-1.1483550072,-1.0401893854,1.7456251383,-1.6846861839,-1.0622183084,-0.0755550563,0.6051344872,1.4599839449,1.8236228228,0.0921196342,0.3357533216,-0.5934019685,-0.3730606437,-0.0822857097,0.6652532220,0.9473050237,0.1076403409,-0.1182667017,-1.6469464302,1.0899176598,-0.7156873941,0.5861592293,-0.8050276041,-1.7994321585,-0.5858718157,1.1953302622,0.8592127562,0.5409115553,0.5861743689,0.9212389588,-0.5880656838,-1.4463483095,-0.1815917939,-0.4911633432,0.5372625589,0.7450032830,-0.7984898686,1.0677664280,0.8281040192,-1.9189672470,1.7005119324,0.1223435998,-0.8320993185,0.2740614414,0.7001248598,-0.1688963920,0.5390168428,0.7814664841,-0.6030362248,2.4221360683,1.2155414820,0.1503422856,1.1967055798,1.4468495846,-0.4375343025,-1.6593545675,-0.2992757857,-1.3680181503,-0.2705932260,-0.8648747206,-1.6523784399,0.6376203895,-0.4645305276,-1.8024930954,1.7780036926,0.6793705225,-0.3502464890,-0.3452021778,0.7236866355,-0.0813378990,0.4103395045,1.2789365053,-0.8342955112,0.2764317095,0.1251170337,-2.4885394573,2.0236468315,0.4303716719,-1.5654964447,-0.2343400568,-0.6242378950,-0.0403054543,-1.2115743160,-0.1085954681,-1.2910075188,-1.7384176254,-2.4113729000,-0.0978536010,0.6314791441,2.3066551685,-0.3984035254,1.6925071478,-1.2291654348,0.4835025668,0.5373200178,-0.6602330804,-0.3534172177,-0.8657686710,-0.0444904864,0.1402515024,-1.3653799295,-1.6854676008,-0.6546943188,0.7114022970,0.3269849718,0.7538878918,0.8704966307,-0.2834851146,-0.3685553670,-2.0719401836,2.0501718521,1.5302041769,0.1144342571,0.0451931544,-0.3070267141,-0.4794501960,-0.6803899407,1.5648218393,0.6181089282,-0.7039632797,1.2172878981,-1.4121437073,0.5843098164,-0.1053195447,0.5416721106,0.3049403727,-0.6772118211,-0.9164791107,1.6576575041,0.3724136651,0.6646793485,0.4738456309,-0.6636233330,-1.9127037525,0.4893687069,-1.0458182096,1.0213255882,1.1951941252,-0.0156067144,-0.6898881197,-0.3947982192,-0.3666029871,-1.5148080587,-0.8758003116,0.0948660448,-2.5608656406,0.7091881037,-1.9902096987,0.2046894580,0.7916052938,-1.7560980320,0.3696931303,-0.7017374039,-2.2834138870,0.0468963534,1.3962758780,-1.0620392561,1.0181268454,-1.5523796082,0.8203259706,-0.3706235588,0.2112499028,-0.8861809373,0.9715576172,0.0313248895,0.3653339744,0.0832739323,-0.4625202119,0.3381362259,-0.2845149934,0.0662649721,-0.9863808751,0.0047100959,-0.0938413143,1.2307792902,-1.4328401089,-0.2935240567,0.2290964574,0.8445306420,-0.2453528792,-0.2585487962,1.1594229937,1.3921138048,-0.6235087514,0.7814606428,-0.7613408566,0.0034697903,-0.2389897704,-0.3329078257,-0.2609840333,-0.2696022391,-0.9208405614,0.2603013515,0.9995867610,0.0641251057,1.4203132391,0.7948572636,-0.6019089222,2.1973364353,0.9349791408,0.3494509459,0.1695573479,1.1671073437,0.6311008334,-0.5451691747,0.0396844149,-0.1348823607,-0.7637632489,0.9957833290,0.3538493514,-0.9998304844,-0.3069563210,0.0764340535,1.2341115475,-0.4459015131,-0.8853935599,-0.1158692837,0.3361707628,0.5471341014,-0.7158341408],[-0.7225044966,1.5734320879,1.5906124115,2.3091294765,0.5050844550,-1.6723904610,0.1712281853,-0.2585093677,1.0001472235,0.5437983274,-0.0820056647,-0.0909900889,0.0230511352,0.7223117948,-0.7696962953,-1.0347841978,-1.2189891338,-1.9741156101,0.9746890068,1.5432999134,-0.9736856222,0.4056926370,0.4569702744,0.8798183203,-1.0623204708,1.8637923002,0.3763478100,-0.1089622602,-2.6937398911,-0.3388677537,-0.3510338366,1.0932720900,-0.4909926355,-1.4203675985,-0.1821767986,-1.2154457569,-0.7117774487,0.4508901238,-0.7958429456,0.2701161504,-2.1759605408,-0.6888856292,0.1048809215,0.3557719886,1.4286407232,-0.9892821908,-0.9702765942,-0.3957818449,-1.2025412321,-1.6958194971,-0.4899291694,1.2200242281,0.2535707057,0.3699500263,0.1447062790,-1.5934619904,-1.1716135740,1.2759325504,-0.8569272757,-0.6799893379,0.8412258029,-0.3783779144,0.6456711292,0.7831376195,0.2905963957,-1.3606551886,-0.8964964747,-1.4888651371,0.7162798047,1.0289067030,-0.8015437126,-0.1136556342,-0.2551667094,-0.6595488787,-0.2111678421,1.4372746944,-1.0267357826,-0.5512586236,-0.8835692406,1.0588902235,-0.1691202819,0.4261890650,-0.9193620682,2.5551671982,-0.7390261292,0.8471983075,-0.4127860069,0.3208262622,-0.6716014147,0.8648295999,0.0461724885,-0.8361611366,0.4000117183,1.2253652811,-0.8352629542,-0.4399746954,0.2978938818,-0.6611793637,0.4268187284,-0.3053519130,0.8972523808,-0.3825258315,-0.2013663799,-0.3409064114,-1.4009368420,-0.0572384521,-0.8031398058,1.5172399282,0.6763442755,1.4432848692,-0.2264365107,-1.4626077414,-0.1477045119,-0.3010220528,-0.4390170574,1.0616732836,-1.6102832556,-0.2703692913,0.1356078684,0.2474282086,1.6711789370,-0.1386670470,-1.0705966949,-1.0397030115,-1.6174699068,-0.1875222772,0.3125074208,0.7058945894,1.6676181555,-0.8247151375,-2.6883125305,-0.0254399497,0.2976902723,0.7683712244,-0.2267342806,-0.0494219884,-0.6753277779,0.7129440308,0.3559135199,-0.3811956644,0.9569373131,2.0430958271,1.7159265280,-2.7088499069,0.4763430059,0.6436210275,-0.3214740455,-0.0530350506,0.8985423446,-0.5783354640,-0.6321326494,1.0727838278,-0.5652558804,1.9493138790,0.1443732232,0.7849040031,-0.9399734735,-0.0713980570,0.8070620894,1.4234772921,0.6374298930,-0.2836876512,1.9719129801,0.2325410694,0.8821958899,0.3814108372,-0.2397420704,0.1884869784,1.9887346029,0.9581700563,0.6938206553,0.8666102886,-1.2881469727,-2.5677328110,-1.0640289783,0.3348835409,-0.0858363882,0.4353652298,0.1455067396,-1.4057297707,0.2277472913,-1.1727318764,-0.6120919585,1.2771542072,-0.8003570437,0.6429291368,1.9732037783,-0.7130243778,-0.1332945228,-0.2530682981,-0.2103462666,1.7769855261,0.3102103472,0.1589248031,0.8054945469,1.4725772142,0.6171528697,-2.3603472710,0.2448849529,-0.0658212975,-0.4911062121,0.3746935725,-0.7771610618,0.5732238293,-0.2027808279,-1.3332324028,1.3494966030,-0.3272345364,1.5416948795,0.3958894610,-0.5925759077,2.2976679802,-1.9548562765,0.7169862390,2.1682713032,1.2674481869,0.0668742955,1.0733947754,0.7876006365,-0.4546663463,0.0326386094,0.0436380059,-1.1151885986,-0.2926286459],[-0.2934510708,1.1921510696,1.5980594158,-0.8130311966,1.1409503222,1.3313578367,0.8444764614,-1.2648280859,0.9731772542,-1.1700046062,1.0548166037,-0.5340769291,-1.8655465841,0.1589542329,1.6472337246,-1.1182059050,1.2130138874,1.1797075272,0.5020463467,-0.7686650753,-0.1314456314,-0.7434322238,-1.5818883181,-0.6749847531,-0.0076208692,0.7187939286,-0.5591735244,-0.6265111566,-1.3190778494,1.0906132460,-1.0671056509,0.0342655107,-0.4741897285,-1.1158161163,-0.2803910077,0.5985125899,0.5886665583,-0.1324573904,-0.8394958973,0.0530796871,1.2963345051,0.0079052718,-0.9098715186,1.1129853725,-1.6644388437,-0.6847685575,1.1993128061,2.5089540482,-0.3476056159,-1.9554597139,0.5798164010,-1.9721982479,1.0853871107,-0.3682597578,2.2866818905,1.1455234289,-0.0026167410,1.1169599295,1.3755278587,0.5090430379,-0.6836943626,-0.4428147078,-0.0736249387,1.4757091999,-0.8413479924,-1.1724884510,-0.7022691369,-0.3636963964,-1.0440857410,-1.0499132872,-0.9251657128,0.4470546544,2.1343755722,-1.0937464237,-0.3145466745,0.0791752636,1.8015995026,-1.1708307266,-0.3710770905,0.0021474429,-1.2492738962,1.2540581226,-0.3887788653,-0.0979342535,0.6381052732,0.5091582537,-1.3069186211,-3.2205343246,-1.6814134121,-0.2490135878,0.6927216053,0.3460028768,0.4161698520,1.7263311148,1.3670994043,-1.2537178993,0.8223969340,0.1540791988,0.4974704087,-1.4824185371,-1.3710247278,1.9001389742,2.0547459126,0.4271369278,-0.3725138009,-1.3944122791,0.2006481439,-0.2336889803,-0.9436457157,-0.6381720901,0.3321398199,-0.2137336731,-0.6376451850,-2.6190822124,-1.4501197338,1.0656617880,0.6423869729,0.3586390316,0.0616030321,-1.3555532694,-0.6935083270,-1.4561247826,-0.6963708401,-1.9475837946,-0.1860286593,-0.9210773110,0.7253800035,0.7981371284,0.4483519793,1.6668821573,0.1371351928,1.3965914249,1.5389333963,-0.6738855243,0.7440813780,1.6378698349,2.2377979755,-1.6630915403,0.6449002028,0.0226498768,-1.0483651161,-2.1157052517,0.7034059167,1.0860549212,0.6332697272,0.5163491964,-0.7699266076,1.0532444715,-0.2817136347,-0.1954266131,0.6466432810,-1.1061758995,0.4786778092,0.6414473653,1.2876604795,0.5732532144,0.3575452566,-2.2497246265,1.2791950703,0.5004061460,-1.9408650398,-0.5976858735,0.3575026095,0.2238463461,1.0250608921,1.5798437595,-0.7968079448,-1.5439547300,1.2245321274,-1.3009462357,-1.3025891781,1.0694528818,-0.6745738983,0.0826234743,-0.1688699126,1.3309227228,1.3732751608,0.7255052924,0.2665914893,1.3750485182,0.0715328604,-1.1448044777,1.1675745249,0.2949744761,-1.1843786240,0.1423856020,-0.6588349342,1.2551316023,-0.1594646573,0.6699225903,0.6179960966,-1.7483422756,1.3339719772,0.3029723167,-1.1223556995,-0.4683272839,-1.6160479784,-0.3782319129,-0.7951239347,1.4735752344,1.6484818459,1.9880499840,-0.1242350116,0.1590626389,-1.8769822121,-1.0277003050,0.3765939772,2.0874798298,-0.9381197691,-0.2639953792,-1.9528026581,-0.3390138745,-1.9990794659,-0.6516773105,0.9337078929,0.6412097812,0.4076411426,0.0087392740,-2.6370751858,-1.2452684641,-0.3792372048,2.0947167873,1.2018176317,0.9371799231],[0.2011674345,-1.8033559322,0.5050754547,-0.5637676120,0.0107321963,0.6286424398,0.0492702387,-1.6498469114,0.0104927039,-0.1102788448,1.2789262533,-0.2689329088,-0.9935606718,-0.8252071738,1.4261518717,-0.1880145818,-0.9265387058,0.4540956914,-1.2168729305,1.8058356047,-0.1168055758,0.5073953867,0.3392245173,0.7426418662,0.5837747455,-0.7182095647,0.3108665347,-0.0449238829,-0.6104008555,0.5157542229,-0.9972209930,1.6061866283,-0.2779401541,1.4404076338,2.3598353863,-0.0747601092,0.8856165409,-0.4305108786,2.0704896450,-1.2632654905,-0.1376118213,0.3399257362,0.4172295034,0.0091022998,-0.5716640353,1.1888803244,0.1341378689,0.7014256120,-0.5142672062,0.2883620858,-1.7981014252,-1.5133697987,0.6216871142,0.1071367636,0.4802518785,0.7183364630,-0.5698022246,0.8356449604,1.3484818935,-0.5819205046,0.2602949739,-0.1543977708,1.7979315519,0.4334428012,0.3003485203,-0.4186342061,-1.8175272942,0.1426157504,0.2628866732,0.3613359332,-0.2564287484,-1.0144606829,-0.8692834973,0.2546459138,0.5684943199,1.2995219231,-1.7172603607,-0.2453630418,-1.2629826069,0.7414638996,0.2217936367,-0.9192535281,-1.1625478268,-0.7427448630,0.6232370138,1.4456577301,-1.3260273933,-1.3074417114,0.3686677217,-1.7282638550,0.1614782363,-0.8077552915,-0.5116407275,0.9091966748,-0.8138747811,0.8174273968,-0.2035060823,0.1387581825,-1.0483995676,-1.1789760590,-0.1418992430,2.0072953701,0.2946895659,-0.1563377231,1.9827004671,-0.5443348885,-0.6208754182,-0.0153734796,-1.0330175161,1.5375193357,2.0169827938,-0.8606604338,0.7027775645,-1.1580990553,-0.8982186317,0.4647661448,0.1343112588,-0.4590297639,0.1715912819,1.5083426237,0.0586618483,0.7613887191,-0.6067987680,-1.7963507175,1.2548172474,1.1760858297,1.3762103319,-0.8332884312,1.9168746471,1.2767934799,1.7126957178,-0.8065787554,-1.2705796957,0.5567392111,0.6912044287,-0.1580438316,2.1341280937,-0.5214887261,0.4928900898,0.4675154984,-0.1240027398,-0.6425836682,-0.0069333049,-0.6486710906,-0.6996839643,1.3014633656,0.2614509761,-0.2373386174,-1.3424488306,-0.4017966986,2.2234640121,0.6983535886,-2.3241012096,-0.9949077964,-0.2743484080,0.5673701167,-0.8014136553,-1.0267834663,-0.3481470346,0.2476348877,0.0547130778,1.3874765635,0.7338072658,-0.3730239868,-0.4653834701,-0.6865920424,0.3276156187,-0.5956894159,-0.0898129418,-0.8099248409,-0.9437165260,-0.1692812443,-1.0413445234,-1.7066810131,-2.0119111538,1.3718327284,0.2448530644,-3.1781678200,1.8929358721,-0.2453606278,0.7510632873,-0.2731828988,-0.0230197310,-0.6737012267,0.8937125206,-0.3046015501,0.3742146790,-0.1818317771,-0.4224560559,0.4293473661,0.0446226597,0.2028374076,0.6543111205,2.0454571247,-1.4786034822,0.3524400592,-0.4629359841,-0.3411388695,1.7129424810,0.4731816351,1.0813529491,1.3048049212,-2.1584177017,0.0670639798,-0.5799887776,-0.0091375699,-0.3846576512,1.0352602005,1.5867995024,0.9555609226,0.3972346187,0.1691245884,0.7903192043,-0.7628286481,0.7735763192,3.3371245861,0.5317189693,-1.3681658506,2.1529917717,0.4164772034,-0.4630190432,0.3562625945,-1.3067346811,2.7096297741],[1.1794174910,1.2390013933,-0.3905018866,-1.2902466059,-0.6624063849,1.3344436884,-0.6159383059,-1.1763250828,0.3439395428,-0.6466949582,1.2733309269,0.5567304492,0.1190240979,0.4002852142,-0.1299002916,-1.4366146326,0.5557325482,-0.3819403052,-2.2948584557,0.7538915277,0.4088020027,-0.5706181526,0.9400187731,-2.2233648300,0.2392009497,-1.4521886110,1.4968353510,-1.1174561977,2.2601320744,-1.8304297924,1.3039175272,-0.0048594456,0.7367036343,-0.3838936687,0.5286387205,0.1551837325,-0.4323429465,-0.6561080813,0.5545196533,0.5781207681,-0.3996987045,2.3530459404,-1.3647236824,-0.1340306848,0.1753463745,0.0249666255,-0.3483884931,-0.0333038643,-1.9605727196,-1.5999488831,-1.3416832685,0.2788104117,-0.7476895452,0.8272824883,0.9861788750,-0.6861533523,0.1893069595,-1.6422375441,0.5166320205,-0.3142879903,2.8894968033,-0.3574573696,-0.7443041205,1.9324642420,-0.6711962223,0.8798464537,-1.0091282129,-0.5488491654,1.5701944828,0.2705782056,0.8730167150,0.9876908660,-0.0415039994,-0.6048167348,0.2426076084,0.1948920041,-2.5501427650,-1.7648794651,-0.0415776409,1.1606342793,0.3051763773,1.4745252132,-0.3646067381,0.1290424615,-0.2389613539,-2.2449805737,1.3227287531,-0.3557299078,-0.2465161532,1.0724699497,-0.0132042440,-0.6840516329,-0.4048824012,1.0555580854,0.2012488693,1.4333781004,0.5459110141,1.4253919125,0.1036763117,-0.8666305542,-2.1211733818,-1.3604879379,1.2255884409,1.0984667540,-0.2646973431,-1.2453637123,1.2485244274,-0.4639970064,-0.1485658139,0.4927731454,-1.1315159798,0.6742309332,0.1616810262,-0.2924068868,-2.2380015850,1.5050231218,-1.1828123331,-1.0715100765,0.2920324206,0.4176152349,-0.2618497610,0.3900544941,1.2369613647,2.2400150299,-0.6591055989,-0.3432904482,-0.9165864587,0.3899152577,-0.8128656149,-0.6911546588,-1.8697581291,-1.9872611761,-0.8085523844,-0.0751557872,0.2399503887,-1.3846840858,-0.1106751859,1.8108611107,0.2479185611,-2.0952146053,-0.0888442099,-0.1570831537,0.0162983797,-0.7323496342,0.9778578877,-0.4015278518,1.5470517874,-1.1965001822,0.5501039028,-0.3973948061,-0.5402927995,0.1135616377,-1.1342197657,1.5787736177,0.7312040925,1.3263958693,-0.9830656648,-1.5166749954,1.2743954659,-1.6303781271,0.5302878618,0.4884847105,-0.0564946122,1.4196146727,0.0485945493,-1.6874896288,0.0264888797,-2.2721157074,0.5597230196,0.0342372134,0.3025023639,-1.1467289925,-0.1418615282,0.4835216701,-0.4516725242,0.2854830325,-1.2665178776,1.5576566458,1.0947488546,0.2143646628,-0.2931555212,-0.6761689186,-0.5095722675,0.1104367077,-0.9266905785,0.1388486475,-1.7587862015,1.1457693577,-1.7809176445,1.7472233772,0.2431917340,0.9257518053,-0.1105109155,0.7692781687,0.8367552757,0.7457382679,-1.5059551001,-0.8665480614,-0.8388282657,-1.1409242153,-0.6999360919,-0.3496262133,0.2825672925,-0.7251719236,0.3119471371,0.6827612519,0.8235883117,-1.2512964010,0.2607354224,-1.1948734522,1.6742203236,0.1868904084,0.8913527727,-0.0905617997,-0.4602533579,-1.3090718985,0.2370790094,-0.5549578071,-0.3860284686,-0.3769266605,-0.3289939761,1.2747739553,-1.1576405764,-0.7707039118],[-1.1006671190,-0.0534144416,-1.0460067987,-0.0220385697,-0.0187210329,1.0596003532,0.4855110049,0.9563537836,0.6240862012,-0.2181169987,-0.1297433525,1.3240571022,-1.3595471382,-0.0788813084,-0.4473232031,-0.8702023029,1.2506643534,0.9834125638,-1.0927263498,-0.6073045135,0.3495458961,0.1213660464,-0.4297527075,-0.3331126571,0.7537350059,-1.9193907976,-0.8438121080,-1.9194872379,-0.2119607925,1.8239436150,-0.2754066885,-0.5554115176,-1.0301826000,0.7643379569,-0.0227805395,1.3933587074,-0.1937872171,0.9693816304,0.6298752427,0.2584379315,-2.3164453506,-1.8396610022,-0.3373851478,-0.4343889356,0.5835508108,0.5079726577,0.7551559806,0.8957632184,-0.5572155714,1.4874306917,-0.5413039923,1.7688095570,-0.1760699153,-0.4086333811,-0.0682857335,1.2715420723,1.3316938877,-0.6478120089,1.7098898888,-0.2466237396,-1.8732783794,0.1492802203,0.8813462257,-1.7876430750,-1.2078424692,-0.5233929157,-1.0958852768,0.1002101004,1.2844595909,1.1891831160,-1.2014337778,-2.2963385582,0.9300718904,0.7659554482,0.7081581950,-1.4520958662,1.1127053499,0.2289859056,-0.7110806108,0.4880263507,1.5001052618,1.5714887381,0.7933399081,-0.3028346896,-0.1017167792,0.6995674968,-0.0198632907,-0.7587066293,0.7734092474,-0.3478929996,0.0377644747,-0.6838935614,-0.7128896117,0.5029635429,1.9088362455,-1.1404583454,-1.6888059378,-1.8423115015,-0.0389658883,1.1948564053,2.4157738686,-0.9063307643,0.6310094595,0.9897720814,0.7520170212,-0.4465702176,-1.6844987869,-1.9101190567,-2.3421952724,-0.4606425762,-0.5885396004,-1.0535298586,0.6714470387,-2.0614287853,-1.5895788670,-0.9668141007,-1.8850803375,1.0163460970,-0.4743680656,0.9278936982,-0.6074044108,0.8148276806,-0.1381165832,1.0583986044,-1.5783662796,0.7843630314,0.3421233296,-0.8151394129,0.8304324150,-0.5892474651,0.0996279344,-0.1991706789,-2.5094156265,-1.5909996033,0.2353384942,0.6018269062,-0.8568947911,0.2557218075,0.6682499051,-0.1879249215,0.0074428706,0.4798038006,-1.7355819941,1.9003592730,1.5346397161,-0.6947375536,-0.6060379148,1.2424333096,-0.4308054745,0.6093084812,-1.5997244120,-1.5050390959,-0.3377222419,-1.2019016743,1.1436249018,0.9114562869,0.1259697229,0.4765327871,0.3271493912,1.5688660145,-0.0834053159,0.6043325067,0.0358087867,-2.0696275234,1.1063140631,0.7426227927,-1.2348821163,-0.2855467498,-0.1477694511,2.1614141464,-0.7326849699,0.5784502625,-0.1212657541,-0.6860406399,0.2312044650,0.7677243948,1.4630963802,-1.4831662178,0.3842955828,0.3330720067,-1.4914379120,-0.6227321029,-0.6727771759,0.4612464607,2.0365641117,-0.8002539277,-1.1648292542,0.1289562881,0.4260367751,-1.7791209221,0.3649482727,-0.7779896259,-2.4400405884,-0.0435852557,-0.3500059247,1.1347149611,0.3358536363,-1.8796350956,-0.6165884733,-0.3348190784,-0.5539978147,-0.6812824011,-0.5439455509,-1.9715577364,0.3464953303,0.8795771003,0.1512504965,0.0113260550,-0.7210302949,0.4954958558,2.0371608734,-1.3999308348,2.0279750824,1.0738673210,0.7125077844,-1.7527254820,1.9098694324,-0.4230732620,-1.3026645184,0.4003872573,0.2172527611,2.9619445801,-0.8010261655,1.4232937098],[2.3403263092,1.3790491819,0.1906539947,1.5853165388,0.1980482191,0.5038337111,1.2534674406,1.1705026627,-0.9915919900,0.7790826559,0.5936439633,-0.7537182570,-0.6383305788,1.0891114473,-0.5446549654,-0.2027605176,-1.1356508732,0.8898881078,-1.1803863049,0.0760592073,0.3940411508,0.8132078648,-1.2225940228,-0.2539710402,0.9250649810,-0.4512603581,-0.0236611646,0.2419966459,0.5349957347,-0.2259923965,-0.7846670747,0.4239006341,-0.0066497908,0.8761689067,-1.0638122559,-2.5720088482,-0.5335486531,-1.0095447302,-1.7223300934,-0.6535322070,1.1655402184,1.1832163334,-0.2269054651,1.7193808556,-0.1380582601,1.5151145458,0.2277833372,0.0959253907,-0.5170901418,-1.0309077501,-0.4735969007,-0.0485035852,-1.1286866665,-0.7307674885,-0.5463230610,-0.0409587808,2.0975720882,0.3434947133,0.9940972328,0.3789162636,1.0542325974,-1.8494417667,-0.4977275729,0.7369968295,0.5268564224,0.1865140498,0.8466647863,-0.6316319108,0.1136949435,-0.6331836581,0.3402028978,0.5535160899,-0.2889496684,-0.7305825353,0.4190462232,-1.0515617132,-0.4801098704,0.0261303224,1.5960645676,0.1610219181,1.7804647684,-0.3047503829,-0.8188756108,-0.3621890247,0.3808596432,0.2823227644,1.5666717291,0.3734123111,-0.8837423921,-0.7322314382,0.1986591965,-0.0118880589,-0.7151897550,0.0778959244,0.8643652201,0.6962469816,-0.5422630906,-0.9411893487,-0.8344305754,1.1522097588,1.3389163017,0.0177985504,0.5214101672,0.6627396941,-0.0895537585,0.3205173910,1.1086225510,-2.7512409687,0.7842624187,0.2301861048,0.4868739247,0.1914014667,1.4830135107,0.0156363230,0.6475189328,1.8308417797,0.4691707790,0.3148375750,0.7041759491,-2.6743814945,-1.5270642042,0.2455077767,-0.5976601243,0.9597811699,-0.4113830030,0.7531632185,-0.4996159077,0.6813071370,0.7660171390,-1.4191707373,0.3065551817,-0.1291158944,-0.4976983070,-1.2929699421,-0.1371014863,0.7562644482,1.0583063364,-0.4212669730,-1.9234799147,-1.6065359116,-0.2558904588,-0.4901143014,-0.0949113965,1.1031668186,-0.5381546617,0.2360364795,0.0465177372,-0.3361516595,0.1274346560,1.9163759947,0.6101396084,-1.1195522547,-0.7814033031,1.3373019695,0.2333422899,1.0861701965,-0.0382700153,0.8989006877,-0.0218960550,0.1018365696,-1.5967543125,-0.0113325082,1.1269840002,0.0552202910,1.4069075584,-0.9980571866,0.0247818157,-0.7040320039,0.3176784813,1.6605193615,1.0455915928,-0.8923811316,0.9219143987,1.1073287725,0.7103539705,-1.4154820442,1.0690660477,0.1309919506,0.0383858383,0.9466379285,1.3524271250,0.8901629448,0.4715458155,-0.1681349725,1.7018618584,-0.4245993197,1.5963537693,0.4084403515,-2.2818739414,-0.2058727890,1.1010887623,-0.7389399409,-0.2773337960,-1.1447142363,0.5750554800,1.7250024080,-0.9898491502,-0.4104329646,-1.6079535484,0.2768232226,-1.6983526945,-0.5670870543,2.1507933140,-0.2975871563,2.1437575817,1.3332302570,-0.3730902672,-0.1122499704,-0.7847885489,0.9248625040,-1.3942272663,0.6543221474,1.0042756796,-0.9440436363,-2.4518766403,1.3270406723,-0.1127001345,2.2061531544,-0.3989170194,-0.4466468096,-0.5355738997,0.0526149832,0.6064239740,0.5198518038],[-0.8466613293,-1.5090380907,-0.4535782039,0.4313685894,-0.1462003887,0.1641013771,2.1521577835,0.3630537689,0.2070374936,1.7983050346,-0.0564059503,0.5928288698,-0.1816093922,-1.5518671274,-0.3621249795,-1.0239580870,1.0249418020,-0.0258395690,-1.7892510891,0.6218369007,0.6613118649,2.2723629475,-0.4432788789,1.0698333979,-0.8403570652,-1.4873845577,1.5990829468,0.3635856509,-0.1802307367,-0.3384521902,-0.2343396097,-1.4471864700,-1.3438107967,-3.1347074509,1.3278874159,1.4307368994,-0.2085703611,1.5691628456,-0.0372373499,1.5322495699,-0.8938466907,-1.6205263138,0.9487670660,0.0607111529,-0.0718811527,0.4896041751,-1.2187587023,0.8618543148,-0.2260780036,-0.4962435067,-0.8952035308,0.1894100606,1.9990708828,0.6714808941,-0.3252743185,-0.3147354424,-1.2198890448,-1.1942043304,1.2294464111,0.9353618622,-1.7695857286,0.3260254264,0.8139547706,0.1962550431,0.1314067394,0.5124615431,0.7095163465,1.3730001450,0.7314730287,0.8850992322,0.5249949098,-0.0936656371,1.0094020367,-0.4246652126,-0.0342488736,0.5519418716,1.0054379702,0.2137652487,-1.7681372166,-0.9254562259,-0.4854847193,-0.6538339853,-0.2760174870,-0.2612164021,0.9103474021,1.3952193260,-0.3264450431,-0.0597991198,-0.0280044656,-0.7869258523,-2.5064163208,-1.2314569950,1.4557712078,1.2016288042,0.4458014369,-1.4862548113,0.9897671938,0.9338101745,0.7926711440,-0.1426515281,-0.9763342738,-1.1071892977,1.8700821400,2.0336098671,-1.5667554140,-0.5214136839,1.1199978590,0.7776375413,-0.1399716139,-0.0623685531,1.1362810135,1.0089765787,-1.3147529364,0.6011255383,2.0578691959,0.2968917489,1.1136337519,-0.6109186411,-0.6683439016,0.9962469935,-0.8060427904,0.3595867455,0.7470183372,1.0633322001,-0.9276279807,0.3000597060,0.7063956857,1.0950361490,1.4885321856,1.4787135124,-2.1218326092,0.9264213443,-0.1775084585,0.9011278152,0.2840862274,0.0977649391,-1.0456649065,0.1681640744,0.0973941088,-1.7393004894,0.2261245102,-0.0660531223,-0.3362882733,0.8595430851,-0.4416444898,0.0228420068,-0.7061185241,-0.1840548813,-1.2924098969,-1.0702615976,-0.1731543988,0.9668654799,0.7544121146,-0.5534548759,-0.1589929909,-2.7193152905,-0.5930914283,-0.4942319989,-1.1334577799,-0.1847745329,-0.8339127898,0.1843493879,-0.3052302599,0.5634783506,-0.0959699973,-0.9182578325,-1.9924126863,0.3829368651,0.2503584921,0.2333265692,-0.3078962266,0.5468981266,0.1616246402,-0.4544194043,0.2356493622,1.1045236588,-1.3996573687,0.5584018230,0.5636180043,-0.8096659184,-0.1284091473,0.4092279077,0.2285577953,-0.3103590608,-1.3424509764,-1.2082949877,-0.9283201694,2.0955545902,1.1210683584,-0.8812590837,1.2313210964,-0.3197748065,1.7898057699,0.0698545128,0.7923595309,-1.0690809488,0.6286915541,-0.1670196503,0.9845463634,1.0878648758,-0.1941708624,0.5016475320,-1.1764376163,-0.7215655446,0.3653467000,0.7173801064,0.3487474024,1.1008827686,0.6648312211,-0.7699224949,1.8959935904,-0.1441157013,-0.2147849053,-0.7026571035,-1.2497316599,-0.7573789358,-0.5733095407,-0.4854148626,-0.7615327239,-0.5698633194,2.8097834587,0.8722813725,1.2352868319,-0.9432064891],[-0.0974869877,1.7055737972,-0.1014531478,1.0176905394,1.5702327490,-1.7046865225,-0.3277420104,-0.3331027627,-2.1215126514,-1.7561151981,-0.2295200676,1.6113495827,-0.2443063259,2.5411062241,0.3537022173,0.4745986760,0.6965101361,0.1384816021,1.1178592443,-1.1784735918,-1.1290528774,0.2061873525,1.0603878498,-1.1349711418,1.2598598003,1.7623924017,0.0733724684,-1.3333655596,-1.0912177563,-0.6205294728,0.7761052251,0.1271678060,1.2515132427,-0.5780774951,0.7281842828,-0.7465434670,1.1616883278,-0.6196599603,0.3022282720,0.8347775340,-0.0490818210,1.5939271450,-0.9503580332,-0.4347720146,-0.7656218410,0.7085364461,0.9605593681,-1.0164953470,-0.2222494930,-0.8314278126,0.4303712249,-0.7721483111,-0.3019029796,0.5900394320,0.0245243497,0.1932301819,-0.0898328945,0.4563172460,0.6518187523,0.0078258719,-1.2128522396,0.5727971196,-1.3422386646,-0.8164447546,-0.0112239150,0.4942510426,-1.7097381353,0.6157289147,1.2700591087,-1.0838717222,-0.4467307925,-0.2466057837,-1.0219964981,-0.3065997958,0.3694855869,-0.0713111088,0.4026225805,0.6370279789,0.8591992259,1.0705020428,1.8353241682,-0.4750539660,0.6103578806,-0.5137580633,0.7581811547,-0.1050673276,1.5978858471,0.6333515048,0.0460980870,-0.7376776934,0.2096036524,-0.4071459174,0.7349792719,-0.4330402315,0.7938969135,-0.0671691522,-0.3470112383,1.8795491457,-0.1271699071,0.9610880613,-1.4886234999,-0.8252080083,1.3477144241,1.5563865900,0.1703135967,-0.9434504509,0.0534256622,0.1484180987,-1.2368420362,-0.1398315430,0.1595588923,1.5863084793,1.3536993265,0.5934506059,1.1358667612,-0.1774367094,0.9255916476,-0.1539941430,-0.6215142012,-0.6159935594,-0.4488615990,-1.4912188053,-0.5317133665,-0.2063709795,-0.4096370935,0.9561219215,0.5284463167,-0.8182538152,-0.5056212544,-0.1462823451,0.0752253309,-1.2333616018,-0.5615788102,-1.0774196386,-1.4043681622,1.4619164467,-0.6012640595,-0.2459688783,0.3324500620,1.0232548714,1.4352263212,0.1944244057,-0.1463443935,-0.2790006101,-0.6674974561,-1.3180520535,0.3201041818,0.3282475471,-1.4190750122,0.3450450897,-0.1171793342,0.1504306048,-0.7891467810,-1.0476018190,-0.3834323287,-0.1902684718,-0.4965841472,-1.3784673214,-0.1844371557,-0.4675067067,-0.6985540986,0.4536924958,0.3140238225,0.0564287789,0.7583971620,0.6013430953,-1.8449608088,-0.1788474470,-0.0510619581,0.5304078460,-0.4194923639,-0.7391867638,0.1276831329,-0.6782696247,0.1311978102,2.7735209465,-0.2934664488,0.5471537709,0.0393295288,1.0524917841,2.4084842205,-0.1510079354,-0.0755359530,0.2978080213,-2.5932602882,-0.1261765659,-0.0178737193,-1.1064251661,1.6046415567,-1.8980734348,-0.2849923372,0.4307873547,-0.7775550485,-0.9093212485,-2.1181476116,0.5486614108,-0.8518691063,0.1899482012,-1.2635831833,0.0362131745,-0.0570133552,-0.2774310708,0.5166860223,-0.6961334944,-1.1381798983,0.0194949433,-0.2258192897,-0.0000388383,1.0765069723,1.1303399801,1.8657202721,-0.4969576895,0.5364794731,-2.0242605209,-0.8213713765,-1.4851415157,0.8490946889,-0.0753547922,0.3361081779,0.0728952736,0.0231189914,-1.2837551832,0.9499085546,0.1681417376],[-1.4466546774,-0.2972466350,1.1298607588,-0.2603873909,-0.9126073122,-0.4502425492,-0.5704441667,-0.9064111710,-0.8553234339,-0.4430517554,0.2436235696,1.4905276299,-1.0585029125,0.4943399131,0.7875627279,-0.3641681075,1.1554255486,1.2991687059,0.1732139587,-0.7331561446,0.1793219149,-0.1800402552,0.5449168086,1.3240280151,-0.1309091151,0.4784817994,-0.4267643988,2.0781397820,-0.7498555183,1.6521127224,-0.1664374918,-0.7460601330,-0.9441272020,-0.4832925498,-1.5331821442,-0.4067618549,-0.7364491224,-0.9086607695,0.3413515687,-0.8198691010,-1.7870553732,2.4673192501,-0.2311452627,0.3308919966,-0.1159708798,-1.7726674080,-0.9126517773,0.1084937230,-0.2664954662,1.1111640930,0.1229815185,0.6907859445,-0.1474541426,0.9019960761,0.2317667753,0.8363870382,-0.3399315774,0.7041403055,-0.1264661849,1.1791597605,-0.9016402364,-0.1811943799,2.3676764965,1.1302993298,0.3110736012,1.5677846670,-0.7050484419,-0.1518790126,-1.7081288099,0.6985465884,0.7089632154,-1.0309268236,-0.1347338855,-0.0365764089,0.1824831814,0.5456715226,-0.0108612115,0.1843760610,-0.0527592488,1.1165183783,2.0685019493,0.0236146916,-0.1301397979,-0.1713315994,-0.7619674206,-1.6910823584,-0.5223678350,-1.7465475798,-0.2006603628,0.7275930643,-0.4736891985,-1.4242618084,0.0546379611,-0.3088788986,-0.1335759610,1.9706972837,-0.2325765043,0.2274934202,2.2387323380,1.4907130003,0.3462500870,1.3234370947,-1.9889664650,0.7430348992,0.6448816061,0.5218971968,-0.1231725663,1.5066103935,0.8738765717,2.7089717388,-0.8890984058,0.0795818791,-0.0192832518,-0.9809086919,-1.2750723362,0.2867516875,0.2137882411,-0.9008516073,1.5888748169,-0.4282222986,0.4386356473,-0.1085125059,0.6985945106,1.9744687080,0.4545195699,1.3060090542,-1.3972746134,-0.2608897984,0.6921256185,-0.2758881450,1.0756412745,-0.1134573221,-0.0426482968,-0.9878318906,1.5196024179,0.6308060288,-0.9928121567,-0.4773582518,-0.5678960085,1.1409707069,0.0252071396,-0.9686629176,-0.1847696155,-1.1870638132,-0.4968124926,0.7360694408,-1.6462267637,2.3052799702,-0.8024062514,-0.6898394823,1.2083338499,-1.2181578875,-0.8410550356,-1.2503207922,-1.3341989517,0.5452713370,0.0004595034,-0.1027127653,0.5961635113,-0.5245649219,-1.8084018230,1.0589570999,-0.7794679999,0.6349820495,0.3862176836,0.8346452713,-0.8713117242,0.8153094649,-0.5405928493,0.2869177163,-0.6896800995,0.0557697900,0.1182901114,-1.7857323885,-1.0742902756,-0.4063031971,1.7793726921,-1.7563652992,0.6893250942,0.8844146729,-1.0142682791,-1.7650473118,0.2160642594,-0.7887262702,-1.1181463003,-0.1376193762,0.1754952371,-0.2765376568,-1.7079218626,0.9067323804,-0.0235241614,-0.3631467521,0.8013179302,-0.1235235482,-0.1538935751,0.5224968791,0.8662751317,0.2375682592,-0.8459567428,0.3390309811,0.6798582077,-0.9715719819,-0.1630507708,-0.1906574965,-0.3275502622,0.2688953876,0.4053089321,0.1773621142,-0.9372261167,-0.1691677719,-0.3724011481,0.8112956882,0.1267861128,-0.5620681643,0.2927173972,-0.8813909888,0.0762910396,-0.0483648069,2.1920011044,1.7463549376,2.3207309246,0.2887925506,0.7549450397,-1.4306452274],[0.3228493333,-0.0288458075,-0.2087575942,-0.8381208777,-0.9550005794,0.7841243148,0.2179069519,-1.0709420443,0.3135614693,0.0914136097,0.0320585743,0.3876436651,0.3842196763,0.8502239585,1.0612168312,1.8604735136,0.0568598658,0.2732332349,0.0278589707,-0.8741342425,0.3974182904,-1.3401019573,1.6500184536,0.5345094800,0.4902154505,1.3779866695,1.4287627935,0.5794705153,-0.4708151221,-1.2481455803,-0.9796716571,0.1646438241,-0.2908467948,0.1688180864,0.2806825936,0.3941893280,-0.5539347529,-0.0067497804,-0.2968336046,-0.4971628189,1.9962546825,0.2005653828,-0.6932837367,-1.6317993402,0.3809179664,-1.5123542547,0.1992826313,0.3300321996,-0.3034650087,-0.1181315854,0.0882374570,0.1668884158,0.3019710481,1.4882793427,-1.1674114466,1.4780186415,-0.3395954370,1.6217408180,1.3903936148,0.0755978227,-1.1194069386,1.7365732193,0.1781414896,-0.9232194424,1.8039987087,-0.8838068843,-0.4083239436,0.3007299900,0.1699735075,-0.8590050936,-1.7392672300,1.5597413778,-1.3260040283,0.2854619622,-1.9159833193,-2.3440728188,0.4461363852,-0.0687278062,1.9588936567,-1.1906564236,-0.7276904583,0.2522965074,0.9506995082,-0.0913084596,0.7821321487,-0.0195057746,1.2213931084,-0.1069381684,0.7629193068,0.1845369637,0.3870324492,-1.0695841312,1.0488708019,-0.5519348383,0.5487302542,-2.5772011280,-0.2137518078,1.2179993391,0.9156683683,0.3357705176,0.5025701523,0.3491545618,0.7370117903,0.3278817534,-0.7132780552,-0.5390459299,-0.7518448234,0.5354685783,0.0184270069,0.2321900129,-2.1776525974,-0.0709908977,-0.3209287524,-0.2798088789,2.0468642712,0.9956209064,-0.2023959607,1.3509565592,-0.1920386255,-1.5777788162,1.6549317837,0.4675681591,-1.9365965128,0.9573156834,-0.7165775895,0.9248271585,2.1137826443,0.0793551803,-1.1837171316,0.7460383773,0.6538752317,1.3912616968,-0.1166813821,-1.7531785965,0.2474077493,-1.1580266953,-0.7121775150,1.8413614035,1.9593071938,-0.2458700389,-0.5498188734,1.1876565218,-0.3367114961,0.8950569630,-0.8747928143,-0.6351636052,2.0111460686,-1.3398618698,-1.7623288631,-0.3162357509,-0.3991983831,-0.1123126596,0.4193961918,-0.8965550661,-0.3267290592,-0.8449197412,-0.9209045172,-0.2399897277,0.4476865530,-0.7418269515,0.5197879672,0.0386816151,1.1803443432,0.2628131509,-0.2394232601,-0.3722534180,0.6683313251,1.3366475105,-1.4936909676,-1.4146258831,-0.2376010120,-0.2817085683,1.8941197395,0.4649803042,0.8323036432,0.4888179898,1.1489104033,0.6556180716,-0.7000198960,-0.6820146441,0.4848237932,-0.5177137852,0.0824175030,-0.8797442317,1.1639926434,1.1476025581,-0.8527011275,2.0083818436,-0.2703168988,-1.2097411156,-0.2276968956,0.2739947438,-0.5334571600,-1.5935918093,1.2445474863,-0.3134967685,-0.2985264361,-0.7968905568,0.8554915190,1.6267149448,0.6216568947,1.5030903816,-0.1151232570,0.8909834027,1.8018869162,-0.0360328667,-0.8202154040,-0.2220973074,-0.3517731130,-0.5957280397,-0.1514147520,-0.4198016822,0.2551528811,-0.1152595282,0.2171547413,-1.4629511833,1.1438225508,1.2084295750,-0.8150461316,-0.7321365476,0.3776405454,-0.9554858208,-0.4959684014,-0.2907817662],[-1.0054425001,-0.4517457485,0.1696795672,-0.6147547364,1.1529808044,0.0660743788,1.5753021240,-0.4344066083,-1.7833483219,0.0277798567,-0.5330213308,-0.9183149338,-0.6086820960,-0.4047652185,-0.6416500807,-1.0837899446,0.0532945246,1.1404436827,-0.0453740098,0.7457920313,-1.9460426569,1.7987947464,0.9324730635,-0.8482402563,0.6843633652,1.3739305735,0.3718565106,-0.1377663314,-0.9630939960,-0.3043234050,0.0718399286,-0.2703020275,-1.1004296541,-1.3805667162,-0.5695853829,0.4196306765,1.9971634150,0.8231883645,0.2833084464,-0.2101227045,-1.8163676262,-1.7911117077,0.1537543535,1.9449908733,0.1513811201,-1.1334222555,0.5895084143,-0.5986622572,0.3019756973,0.7390938997,0.9933456779,0.6807634830,0.9933760166,0.5551660061,-0.8488301039,-0.5885685682,0.4313100278,-1.3862833977,0.7359342575,0.9195241928,-0.0853480026,-0.1070248708,-1.2493082285,0.5755701661,-1.0534989834,0.0406493060,-0.5918547511,0.0460599251,0.7082794309,0.7456382513,-1.0815974474,-1.2848150730,0.7570005655,0.9823464751,-0.4332475960,1.9694807529,-0.5031172633,-0.8139947653,-1.9751011133,1.2804644108,0.5563730001,-0.4628264606,1.1694854498,-0.4353146255,-1.6423184872,-0.5444234014,1.8026432991,-1.5473425388,0.3108020425,1.2917841673,-0.3807676435,1.7654125690,-0.6993806958,1.1027238369,0.6636332273,-0.6843606830,-1.4899740219,0.2734349966,1.2600288391,-0.2286336422,0.2327901125,2.4362354279,0.1817132533,-0.7989073992,1.1250274181,-1.0452672243,-0.8149611950,-0.2601708174,-1.3295027018,2.2031581402,1.1800258160,0.2624831796,-0.0770635083,0.7740607858,-0.3567553759,-1.1034004688,0.1847368032,0.6796195507,-1.0074751377,1.1585245132,1.4195199013,-0.6195434332,0.7116128802,0.6401365995,0.5280639529,-0.9293158054,-0.0831103027,0.5697512031,1.0032013655,1.0931603909,-0.4489105642,-0.2979501188,1.0878436565,1.6233495474,1.4550083876,-1.1979384422,-0.9113218188,-1.2620170116,0.1533333361,0.3634153008,-0.3303910196,0.1722114384,0.1782008410,0.9415059686,-0.7712236643,-1.5482803583,-1.4313429594,-0.3579024374,2.3041453362,2.1099629402,1.6484801769,0.1672390848,-0.0169916637,1.7207933664,0.2674842775,-0.4923659265,-0.6619585752,0.8079488277,-1.6243458986,-1.7960036993,-0.7675559521,-0.0231633689,-1.0103166103,1.2741718292,-0.6419000030,-0.7730162740,-1.4375553131,1.1895669699,0.3485536277,-0.4232673645,1.2198886871,0.4842070043,0.0227463860,0.5305594206,-0.8054476976,-1.7734887600,0.3406913579,-2.7052669525,-0.0847135931,-0.6385948658,-0.5110730529,0.3484685719,-1.3105498552,-2.5472373962,-0.3775078654,-0.0988472104,0.4710415006,-1.2057213783,-1.7653845549,-0.0680527538,0.7711173892,-0.7321217656,1.0039491653,-0.7371895313,0.5678637624,0.3336027265,-0.5274707079,-0.9774408937,-0.3786778450,-1.4249675274,-0.7147436142,0.3129166365,-1.7783929110,-0.4209950566,-0.5415373445,-1.2222980261,1.3970868587,0.8098871112,-0.2767983675,-1.8499927521,0.1502419710,-1.5492517948,0.7092196941,1.4950456619,-1.1154440641,-0.5161579847,1.2957165241,-0.5796651840,0.6269744039,0.9531516433,0.2816024125,0.4732829630,-0.7617855668,1.6947957277],[-0.0578788295,-0.4390718937,-1.1987463236,-0.4341277480,-0.0426968820,0.7203280926,0.3357464373,0.8612458110,1.0400627851,-0.0488214567,-1.2987916470,0.8273304105,1.0139622688,-0.0649127662,0.7446480393,-0.1540967673,1.9260265827,1.2363505363,-0.4862475693,0.4868167043,1.7191879749,-0.0682080835,-0.9640083909,0.3089770079,0.9302484989,0.9873762727,-0.5898210406,0.4689652324,1.5302232504,0.7320433855,-1.3513690233,0.7897057533,0.2824765146,0.5834311843,0.1131865606,-0.8406014442,0.4050630033,-0.4101101756,0.5337298512,-1.3036297560,-0.6484665871,-1.1359941959,2.0571515560,0.6670948267,-0.4131903946,-0.5690752864,1.4522426128,-0.9800069332,-0.1403042823,-1.0196304321,0.9959092736,-0.2827495635,1.9643141031,-0.6609107256,-0.7525837421,0.5954794288,-0.3523457646,0.3810907602,-1.0135425329,1.4329707623,0.8080863357,0.8896982670,0.2208904177,0.3672073483,-0.6885126233,0.4308250546,1.1822276115,-2.0149857998,1.9600487947,-0.5171279907,-0.1703243852,0.8568869233,1.2518591881,0.2865099013,0.8115671873,-1.4276633263,0.5710371733,1.0781027079,2.2357771397,0.7279140949,-0.2229265273,0.5872340798,1.4881185293,2.3195126057,0.5637220144,0.6914354563,0.2569299936,-0.2285066694,0.4804783762,-0.2613047361,-0.4131895602,-1.4262596369,0.4228245020,1.0623100996,0.2604335845,0.6574383378,-0.0409760214,-0.1292385161,-0.6623541117,-0.6476683617,-1.6716749668,-0.1830161214,-0.3365137577,0.6030265093,-2.0935566425,1.2513388395,0.3918113708,0.7372437716,-0.0178348534,-1.1537516117,1.0530462265,-0.8401392698,1.8309344053,-0.9686001539,-0.3691553771,-1.4921547174,0.0017514684,-0.4581293762,-0.6935526133,1.0855137110,-1.0821048021,0.7962128520,-0.1281221062,-0.8133049011,-1.2505276203,-2.2208952904,-0.2130862474,1.0099068880,-1.4203333855,-0.2492176443,-1.3636775017,0.1963888705,1.9370468855,0.5962476134,-1.6595464945,0.1022451743,-0.5378738046,-1.8219336271,0.0530054718,-0.3030683696,-0.0812117681,-1.8513782024,0.3097834587,-1.5337352753,-0.2538897991,1.3182882071,1.1397893429,0.4829955697,-0.9046905637,-0.8380981088,-0.2948737741,1.9916062355,0.0728184804,-0.1014356986,-0.1235939339,-0.5081471205,1.2604811192,-0.1343659312,0.2073383778,0.3723934889,-0.4487771392,-0.4661149383,-0.3359813094,-1.3195090294,0.3540134430,0.3601185679,0.0083168242,0.3976401091,-1.6742360592,0.2504509389,-0.6635281444,0.8641865849,-1.1257250309,-0.3801716566,-0.1833992749,0.3565971553,-1.3327323198,0.6035583019,-0.5465292335,-1.5566965342,0.6932677031,1.6228110790,0.6801456809,-1.5924171209,-1.5193947554,-1.0084333420,0.4458001256,-0.8410766125,1.5464582443,0.6618500948,1.3115892410,-0.8060818911,-2.0020639896,-0.7427703142,0.9982657433,0.0429815277,-2.2929155827,0.7949628234,-2.0755562782,-1.1538249254,2.2420399189,0.3406197131,0.0048286184,0.2839885652,-0.4365164936,1.3366078138,-1.8391609192,0.3603280187,-0.0859675929,0.5570289493,0.5718832016,0.6106733680,-0.3083177805,0.0144931190,-0.2672089338,-1.2353001833,3.0229272842,0.3395426869,0.5264291167,1.2634595633,0.1770529449,-1.4849345684,-0.6991217732,-0.7283034325],[-0.6939549446,1.5446032286,-0.2754387259,-0.2167219073,-0.7424051166,-0.9166474342,1.5751912594,-1.6181910038,-0.8253030777,-1.0706801414,0.8698484898,-1.1994905472,-0.8146388531,1.2920223475,-0.4625878930,-0.3836516738,1.0654515028,1.6666485071,-0.1208937168,-1.4995495081,1.2321619987,-0.6949200034,-0.5739510059,-0.1775606424,-0.9272752404,0.5301542282,-1.1949777603,-0.4780461788,-0.5277665854,-0.9728552699,0.4626088738,0.4503042698,0.9948613644,0.2734299004,0.1685026735,-0.6112354994,-0.9500792027,1.8824837208,1.7905997038,-1.1740034819,0.0983359888,-1.1613261700,-0.3503814936,-0.8911704421,1.0788980722,0.5437412262,0.6564418674,-2.4974653721,0.4571970403,0.0749873742,0.0620500483,-0.2306726277,0.1126581952,0.6336370111,-0.4393058419,0.6253045797,-1.3444340229,0.3629311323,0.9891084433,1.3607655764,-1.6233907938,0.3538567126,1.2099943161,0.7256388068,-0.4102138281,0.4880381823,-0.6662564874,-0.3059380352,-0.1223294958,0.9289811254,-0.9578050375,0.7637707591,-2.6466319561,-0.8069033027,0.2090090364,-2.1910057068,0.8204702735,-1.6458770037,0.1837251335,-0.4417119026,-0.3663473129,0.7258780003,-1.0395754576,0.3824823201,-1.3599922657,-1.3667659760,0.2267869860,1.0660798550,0.7252836823,-0.0433388688,-0.8327400088,0.5989152789,-0.1539460868,0.8072755933,-0.1370912343,-0.3414518237,0.6859472990,-0.0732531026,0.4887581468,1.4984587431,-0.8649017811,0.1688008904,-0.1289001107,-0.9218314290,-0.3603636324,0.9271414876,0.0079463189,-0.3383709490,0.8675869703,0.1249827370,0.4651188850,1.1462744474,1.2493416071,-0.4183758795,-2.5614197254,0.5680629611,-0.6206008792,1.7465558052,0.3847604096,1.4685661793,-1.1315544844,1.1918085814,-0.6606284976,0.4041602910,-1.4739680290,1.1007773876,-0.4919222891,-0.1771818995,-1.2822349072,-1.0434724092,0.3237209916,-0.7618917823,0.3099120855,-0.5639561415,-0.0917711481,-0.5850586891,0.6370205283,1.0494756699,-1.9202592373,1.2494323254,0.5849095583,-0.8469260335,1.3565355539,-0.0963294879,1.2453507185,-0.1400534064,0.8450857997,0.4133973122,0.7155808806,1.4432688951,1.5716166496,0.6924489141,1.4065362215,-0.4133162200,1.0888381004,1.3608398438,-1.0978585482,-0.2764387131,0.5563209057,0.8842734098,-0.3509296477,-0.9768342376,-0.7402306199,0.2903321981,0.1414962411,-0.4799614251,-0.3082216084,0.0267795082,1.2097295523,0.2476162165,1.1630425453,-0.0845261142,-0.2282030135,0.2696264684,1.2454012632,-0.1312718689,1.4098299742,1.0410118103,-1.6086004972,-0.5174929500,0.8974530697,1.0401508808,2.2852160931,-1.0753257275,0.0802447274,-1.2287327051,-0.0713328272,-0.8569629192,1.1176137924,0.5114846230,0.6936994195,-0.8175045848,-0.6783365011,-0.7737123966,-0.3651148975,0.6318411231,-0.6266897321,-0.6617985964,-0.5669463277,0.2499994934,1.1892088652,-1.9240505695,-0.3544670045,-1.0515387058,-1.8288736343,-1.2691622972,-1.3089474440,-0.0297324564,1.2603071928,-0.4449788332,-1.7621899843,-0.8053673506,0.5204708576,0.2721701562,0.2346938401,-0.6873981953,0.2655944824,-0.0325214528,0.5738306642,-0.5787928104,-0.8237167597,-1.2874125242,1.1350760460,-2.1356105804],[0.4515221119,-2.1500139236,-0.4114108384,-1.1628191471,0.1297886521,-0.2495429516,-0.0260325857,0.9392570257,-1.3046522141,0.6903501153,0.6977592707,1.0701305866,-0.2887674570,-0.4104785323,0.1533241421,1.3199142218,0.3096901178,0.4101806581,-0.7352973223,-0.9126303792,-0.7222605348,-0.4227232933,-0.2477057576,0.5480054617,-1.1432491541,0.3385492861,0.2351884991,0.0385535322,0.8181906939,1.5828475952,0.7214038968,0.0897340551,-0.3189024925,-0.3974009156,0.9097927213,1.9650508165,0.1053364798,-1.2227591276,-0.0153107131,-1.2919523716,-0.7455282807,0.9781656861,0.6193071604,0.4077815115,-0.7046490908,0.1537861079,-0.0643131956,1.2573903799,0.9574343562,-0.5834718347,0.1090049073,0.5478889346,0.3504471183,-0.2250039279,0.8269471526,-0.0169237554,0.6220702529,-0.2975426614,1.2149496078,1.1151980162,-0.4507243931,2.2206854820,-1.8293071985,2.6430711746,-0.1439989805,-0.0610520355,-0.2829388976,1.0629047155,-3.1820971966,0.4401958883,0.7499151826,2.5593225956,-0.9967869520,-0.2721943557,-0.0896866843,0.6282408834,-0.0261002425,-0.9709675312,0.0230096858,-0.0140103968,-1.6569482088,-0.1382927299,0.7253227234,-0.7418099642,0.9728586674,0.4095517993,0.0483989120,0.0979183540,0.3018720746,-0.3589890599,0.5353608727,0.5360045433,0.7536387444,0.7505097389,0.5256783366,1.3380362988,-0.2981445193,1.8530151844,-1.1281915903,-0.0498322956,-0.7330253720,-0.6458021402,-0.5024451017,-2.2123041153,-0.9805191755,0.8008458018,-1.0077247620,0.6239641905,-0.8552334309,-0.2744834423,-0.1157714501,0.0847591385,-0.3774202764,-0.3264375031,-0.0360224955,-1.0819702148,0.3266488910,0.5532034039,-0.3854060173,-0.1618631631,-1.9549201727,-1.7788834572,-0.1174470335,-0.2493743002,2.1404275894,0.5593461990,0.6229158044,-1.8816431761,1.4607295990,-0.3706546724,0.0042911167,0.3046770096,-0.0710433796,-0.1237470731,0.4302512109,0.4529004395,0.6425746083,-0.1122784540,0.9161322713,-1.1342129707,-0.5450671911,0.2877930403,-0.2188682407,1.2186625004,-1.1504718065,0.5568408370,1.3841053247,-0.1620552093,0.7049739957,-1.2413967848,-0.9855633974,1.3243196011,0.3625219762,0.2983141243,-1.2990237474,-0.2681641281,0.1363256872,-0.1956966519,0.2655187845,-0.1531553864,1.2532037497,1.8852713108,-0.0882719830,1.7805033922,-0.4716087878,-0.9298340678,-0.3180843890,1.2743237019,1.0903788805,1.0813274384,-0.6996761560,0.7500818372,2.1777994633,2.2021887302,0.2121428996,1.6765725613,-1.2105097771,-1.5527480841,-1.3053249121,1.9286055565,-1.3529680967,1.0474323034,0.2311239541,-1.3959530592,-0.6062406301,-1.8592184782,1.0289849043,-0.1385550201,2.3706924915,1.1001138687,-2.1009137630,0.6809428930,-1.6257436275,-0.6237348318,0.6981055737,-1.8643096685,0.4661882818,-0.2186380774,-1.3467729092,-0.9527578950,0.2725820839,1.3400031328,-0.5603945851,2.5774965286,-0.5632506013,-0.3768528998,0.0353212729,1.4043915272,0.0230951086,-0.0061584190,0.6321358085,-1.9023759365,1.5614373684,-0.1971298605,-0.7250073552,-1.0256675482,-0.4578739703,-0.6443290710,-1.0963523388,-0.0895931944,-1.4758558273,1.6121660471,1.1630821228,-1.7796010971],[0.0145237353,-0.5974840522,1.0503715277,0.3695131540,-0.3743128181,0.0213968288,-1.0481902361,1.0413702726,0.3212760389,0.3309848011,-0.6946014166,-0.7083328366,0.9349833131,-0.1450703889,1.1152725220,-0.8652854562,-0.4565080702,0.3007369041,0.9256248474,2.0625565052,0.2747337222,-0.2963839471,-2.6546571255,1.2803245783,0.1107093021,-0.6835398674,0.4723945856,0.9344331622,-0.7295447588,-0.2803449929,0.1233135983,0.6120019555,0.5841313004,1.7533856630,-1.0543075800,-1.2622847557,0.4116769433,-0.6446053982,-0.2465761304,-0.7146756053,1.3107023239,0.6113011837,-0.1636191308,0.4882713854,-0.3482097387,0.1603152454,-0.4162207544,0.1293164194,1.7988963127,-0.8851790428,0.3992467523,-1.7737257481,-0.3428349793,1.8912366629,0.0722259954,-1.6537472010,-1.6307649612,-0.4667405784,1.1868702173,-0.7915568352,0.4578755200,0.2519705594,-0.9535153508,1.2609884739,0.4392801523,-1.3256343603,-1.0367951393,-0.2201278061,-0.6169109941,-1.0181512833,-0.4103388190,0.1084682569,-0.6541967392,0.7603313923,1.3501311541,-1.6931246519,-1.5221095085,-2.0480751991,0.4730706215,1.3266686201,-0.0894597024,0.3304823041,0.1989917606,-0.4702607393,0.2213799506,-1.5739086866,1.7835437059,0.3327397406,-0.4354090691,-0.1039255261,1.8387240171,0.3680784702,0.6319974065,-1.5963237286,0.0214680173,-1.0597387552,0.1692428887,-0.1024794206,0.3460634351,2.1404290199,-2.7088286877,-3.3357634544,2.4715435505,0.5048002601,-0.0024718831,-0.3783724308,-1.8512169123,0.0197386872,0.5260105133,0.0301570203,1.1527439356,-0.6599699259,-0.7668974996,0.9182029963,-2.0533874035,-0.5711094141,-0.8333142400,0.5297877789,0.1696676463,-1.1874402761,2.6695325375,-0.3015936315,0.0863558426,0.6479152441,0.4200901091,0.0836594850,1.3033480644,0.6622453928,-1.5946129560,1.1357042789,-1.2606321573,1.0469983816,1.1802233458,-0.1032438353,1.1054823399,-1.2712923288,0.0872885138,1.3110666275,-0.3147211671,0.7948974371,-0.6520865560,2.3930139542,-0.6381352544,-0.1903901398,0.7538375258,-0.6007530093,-0.4748378098,0.4931529164,-0.3945479691,-0.7058361173,-0.4707331657,-0.6427865028,1.1003127098,1.2145934105,0.6543659568,0.1378059983,0.2667719126,-0.4505184293,0.7668704987,0.3069391251,-1.0402356386,-1.0768088102,-0.3584763110,1.4544060230,-1.4216730595,0.3472038507,0.2086229771,0.5839641094,-1.4341008663,-0.6708384156,0.2508373260,0.8319552541,0.5872681737,-0.4998468161,-1.1823883057,-0.3054107130,-1.0352274179,-0.6697105169,-0.8784491420,-0.8758939505,1.3824001551,1.1037901640,0.5120603442,0.2342875302,0.7024837136,1.1357293129,-1.0685719252,0.1360779256,-1.2616283894,0.1958422810,-0.7124678493,0.4309418797,-0.8070568442,-3.6666617393,1.2247881889,-1.8534743786,-0.8791117072,-1.1550799608,1.3362749815,0.6916596889,0.1911562085,-2.1766350269,-0.2334984988,0.1585360169,1.7308087349,0.2101554424,-0.4219810665,0.6413149238,3.6585171223,-1.6798986197,2.1572251320,-0.9573594332,2.1295564175,1.6554572582,1.4173837900,-0.1184742749,-1.3791389465,-0.1836668104,-2.5507338047,-1.8902881145,-0.2113366872,1.7934159040,-0.4378238618,-1.6616495848],[-0.5122920871,-1.7967692614,-1.2768464088,1.4960579872,-0.4098713994,0.2792552710,-0.6689916253,-0.2977707684,-0.5372231603,0.6325138211,-0.3380914032,0.1361745149,0.5060076714,-2.7306387424,0.5758006573,0.0367980748,0.4948950410,1.3093698025,-0.6927099228,0.5162115693,-2.8285930157,1.4057217836,-0.8556907177,-0.5703662634,0.1480660737,0.2890440822,-1.7777055502,0.6198061109,0.9095965624,0.2691595852,0.0894911662,-0.0680072829,0.8367866278,1.0884227753,-0.7422149181,-0.3737275600,-1.6788115501,0.1495314837,-0.9467903972,-0.1744271517,-0.7147907019,0.5539358854,-1.1186755896,0.6251395345,0.6120824218,0.1489314586,0.7221364379,-0.6673222780,0.5348725319,-0.0781799033,-0.4313751459,-1.4768824577,0.6255025864,0.5036774278,-2.0770540237,0.8228256106,-0.2019738853,-0.0633517280,0.7215458751,2.4272737503,-0.4849633276,0.7702387571,0.7469590902,-0.1873421669,-1.3352429867,0.8034677505,0.5133347511,0.3616528809,-0.0800107121,1.6482689381,-0.2358781248,-1.1339635849,-1.1484259367,-0.2858934999,-1.7178094387,0.0422221310,0.7282291651,-0.5086132288,0.2948485017,2.0248792171,0.9847933054,-2.2134039402,0.1210705861,0.9005804658,-1.7938214540,-1.1630740166,0.6061529517,-1.3917492628,0.0659918785,-1.5254364014,-0.8436492085,-1.4030380249,-1.1273025274,0.6605858207,0.5244283080,-0.5535778999,0.5154033899,-0.0494096726,1.9339607954,1.0602059364,-0.2275834680,0.1745944023,-0.3988296390,0.2007772624,0.4210469723,1.1622320414,-0.1475005299,1.6775492430,0.3825560212,-1.4785602093,-1.0011734962,0.9199843407,-0.3728865385,2.1775677204,1.0654608011,-0.3552238345,1.2858337164,-0.0408678167,0.7288712859,-0.6996934414,-0.9515546560,1.4982056618,-0.8414525390,1.5781154633,-0.8477112055,-0.2554596066,0.7263152599,-0.8979797959,1.9918254614,-0.0547553524,0.2416841537,-0.0709533319,-0.8204307556,-2.0357275009,-0.1745917797,1.0008513927,-1.1479756832,0.6434391737,0.3278440535,-0.8116710186,-1.0752303600,0.3474791348,0.0802315548,0.1050836816,1.6445997953,-0.6685065031,0.2889428735,0.1904595792,1.9781349897,0.0672670230,-0.0199888926,-0.3196633160,1.0699506998,0.5876069069,1.4938871861,-0.2294600457,-0.6761348248,-0.9675368667,0.3901038170,0.9939051867,0.1740347892,1.2410316467,0.2331166565,-0.4013488293,0.0780516416,0.1232619882,0.4011231065,-1.1274694204,1.7268871069,-0.3178609014,1.1595098972,0.5392588377,0.5934121609,0.0710647479,-1.4837100506,-1.5382093191,2.3655643463,0.4086492658,-1.4912976027,-0.2096750140,-1.2068072557,0.1913460344,0.9109123945,0.3377050161,-0.8817027807,-1.7224570513,-0.7705970407,-0.5596790910,-1.0413635969,-0.4826701283,2.7578041553,-0.7895061970,0.2393582165,-0.1677271277,-1.5210280418,-0.9905596375,-1.2625994682,1.8897997141,-1.3356403112,-0.2018123269,1.1257551908,-0.6290640831,0.6600694656,-0.0313108787,0.3371075690,0.8491163850,0.9079491496,0.7914938331,0.5391131043,-0.4028544128,-0.3595976532,0.9391801357,-0.9340050220,-1.5900048018,1.0865169764,1.5760774612,-0.9845061302,-0.4753736556,-0.4715046287,-1.0118333101,-0.3843485117,-0.0267880801,0.0341033973,1.6630090475],[0.7196737528,-0.9001411796,-0.1763302237,-1.8384820223,-0.5711111426,-0.6256062984,-1.0332602262,-0.2247431427,-1.4039026499,-0.6790704131,-1.2205200195,0.5722601414,-0.8030456305,-0.4784704745,-0.2942892313,0.6278070807,0.1514698267,0.4336849153,-0.3661702573,1.2590053082,-0.3364170790,-0.2155324370,-0.0285432246,0.7176227570,0.2905178368,-0.4122419655,-0.0120573184,-1.0927163363,-0.4791768491,1.0728253126,-0.4578598142,-0.1820637584,-0.6342881322,-1.2003618479,-0.5084941983,-0.1252528578,-0.5457894206,0.3418527246,-0.2050891072,-0.3844342828,0.8712399602,-0.0385699309,0.0265888497,1.3547886610,-0.3125000596,1.2956840992,0.9276458025,0.3030017912,0.9541946054,-0.7833327651,-0.9277109504,-1.5816023350,1.3954637051,-0.0899096355,0.9990671277,0.7146677375,0.2361557186,-1.3013103008,-0.1263361424,-0.2064883858,-1.6868683100,-1.2527776957,-0.4654269218,-1.3014801741,-0.1094226167,-0.1444881260,-1.4770342112,-1.9824503660,-0.3436704278,-0.8657111526,-1.2764846087,0.4240189195,0.5265209079,-0.7517572045,-0.8044191599,0.8929079771,-0.3144601882,1.0459331274,0.3026503921,-0.8141273856,1.7526532412,0.5301589966,-2.1711890697,0.2059477270,0.8248731494,-1.6328703165,0.4597865939,0.8475491405,0.2477189749,0.0925393179,-0.4595969319,-0.1853267998,0.3266243339,0.6708236933,0.6769939065,-1.0828754902,0.2627280354,1.3661319017,0.1279993355,0.9695571065,0.1432794780,1.1143633127,1.1418329477,0.1405775994,-0.4752911925,-0.0177135766,1.0221664906,-0.3322848976,-0.5167743564,-0.7432743311,0.3890959322,0.1556181014,-1.8614654541,1.6840809584,-0.7621278167,0.1582335532,-0.2684752047,0.5648133755,-0.1776323318,-0.6139817834,0.1006056443,-1.0148520470,-0.0685680658,0.9740679264,-1.0096333027,-1.2929003239,-1.1497299671,-0.2576673329,0.2650620937,-0.5298980474,0.1767422408,0.3737693727,0.0116174156,-0.8848586679,-0.2841534019,0.2786668837,1.2182451487,-0.8785102963,-0.6624893546,-0.2506814599,-1.6583607197,-0.0440883674,0.7125759125,-1.9737573862,-0.4872403741,-0.1577950418,-0.8622025251,0.1490759254,-1.1009280682,-0.7130812407,2.9556460381,1.6436719894,0.6758390665,1.0099107027,0.0647838414,0.3598891795,-1.5434740782,0.2607182264,0.7817473412,-1.9403500557,1.4225147963,-2.1519820690,0.5616424084,-0.8769695759,-0.2707855403,2.4196352959,0.2859359682,0.0165289156,-1.7499842644,0.3692513406,0.5031059980,-0.1664648801,-2.0056574345,-0.7654278278,1.0434603691,-0.6375411749,0.2848365307,0.0735133290,0.8452553749,-1.4876236916,-0.2486355007,1.4121173620,0.7177804112,-0.7401977777,0.3545803130,-0.5262724757,-2.8600194454,-1.9175211191,0.4451819360,-0.0762447342,-0.1314585060,0.8150295615,0.1831329763,-0.0879756361,-2.0604195595,0.2432845235,-0.1897565573,2.0000879765,0.2861734331,0.0464978926,0.1709866971,1.3898837566,-0.3738046587,1.2233188152,1.0814232826,-2.1520938873,1.0503945351,0.7663048506,-0.7665901184,0.5697636604,0.7073093057,-0.7511551976,-0.9094853997,-0.4420854747,0.9772966504,-0.1575781852,-0.1842168272,-0.3205026686,-0.6077862382,-0.0882692859,2.2927906513,0.9862863421,-0.2564762235,1.4920821190],[-0.2900837064,2.2671697140,-0.0333347134,-1.1995899677,-0.3519096971,0.0096649854,-1.1579176188,0.1301806569,-0.3239947259,0.2254819572,3.1275162697,-2.1915900707,1.6642601490,0.3813494444,1.1953544617,-0.1923963130,-1.3483207226,0.6332259178,-1.1693799496,-0.8710323572,2.1890964508,0.5400183797,1.4145956039,0.7931981683,0.0518532507,1.3498840332,-0.3464334905,1.8504086733,0.2446140349,0.6482831240,0.9984034300,1.3537937403,1.1133658886,-0.3245791793,1.3398704529,0.4796682596,-0.4233224094,-0.6885091066,0.1215857938,0.6505106688,-0.4377563298,0.6642935276,-0.7825295925,-1.0611126423,-1.0111138821,2.4128932953,0.8924791813,-0.4122083485,1.2046082020,0.5568127632,0.7432289720,1.3462983370,-0.4105149806,-0.0584207624,0.8289735913,0.0149429040,-0.9931199551,1.9934371710,-0.4420913756,-0.3624625206,-1.1244492531,1.4691061974,-0.0499073863,0.2002401203,-1.1248832941,-1.2726184130,0.9452376962,-0.4676527977,-2.2370703220,-2.2448759079,0.9237034917,0.4242494106,1.7204461098,-0.3976499140,-0.0906097963,1.4850599766,-0.1435298324,0.3990937471,-1.3640803099,0.1675154865,0.9451347589,0.1672854722,1.1541965008,-1.3738102913,-1.6310482025,0.4398053586,0.9419565201,0.4221067429,0.8890461326,0.9389849901,-0.5344372392,-0.2121012062,0.1934045106,-1.1489433050,-0.1732483953,1.6298263073,0.7644013166,0.0068043680,-1.4946343899,-0.1486235559,1.0825355053,1.0867308378,-1.5739576817,-1.1025074720,0.3009252548,0.1753404289,0.1530693769,1.4307221174,-0.9575708508,1.3203545809,-0.6723961234,0.4719530344,-0.2558166087,-0.4151659906,0.2092033029,-0.3377778232,-1.7142255306,-0.0780475810,0.1520982683,-0.6602406502,0.8542318344,0.0446649306,-0.1880837232,-0.8102146983,0.1999365538,-0.8471016884,0.6609727740,0.0869688913,-0.8234723210,-0.0038673137,-0.7962244153,0.4736574292,-2.1708366871,0.5728909373,0.0155894784,0.1019155160,-0.9545924664,-0.0417357050,0.0830023661,0.3825495839,-1.5725542307,-0.5406516194,1.8002802134,-1.1717652082,-0.1948368400,1.2039487362,0.5995358825,2.4622888565,-1.4696532488,-1.8450510502,-0.8768398762,0.0656154454,-0.6295272112,0.0151507873,0.9739693999,-1.3466656208,1.7466659546,-1.2967122793,-1.5379266739,0.0025664088,-0.5259938240,-0.7475365400,0.1608507335,0.2720134556,0.6598511338,-0.2686775327,0.2295012027,0.1086877584,0.2343160212,0.4769563675,-1.0597660542,-1.9808396101,-0.7161743641,0.8676317930,-0.5259642601,-1.6665190458,0.2771702707,-0.5351759791,0.5666056275,-0.0272630919,-1.2385288477,0.6036389470,0.5278122425,0.0797868595,0.0228599031,-1.9098942280,1.5135689974,-0.2744904459,0.3447202146,-0.2918914258,0.0123673119,-1.0660989285,-0.7806712985,0.4060901105,-0.3038170338,-0.5685398579,0.7254636288,-1.4262027740,0.3587500751,-0.0250621587,0.4670973122,1.7571234703,1.2794920206,-0.7825925350,1.7953877449,1.4440474510,0.2549488842,-0.5291935802,-1.8028976917,0.2191513032,0.2555356026,0.0594101250,-0.2747816145,-0.8839666843,0.5966591239,-0.2426848710,-0.2062047273,0.6629035473,-1.1385248899,0.0354150161,-1.4973627329,-0.4372212291,0.1723114401,-1.4453716278],[-2.5439577103,1.0714870691,-2.3216881752,0.6471449137,-0.2535232008,-2.4768803120,1.8743288517,0.3274333775,-1.4912362099,-0.4837066829,2.1048803329,1.4479202032,-0.6346545815,0.2597027719,-0.5986030102,-0.9634928703,1.1574920416,0.4726140201,-2.7617619038,0.4451379776,1.5339012146,0.6199848056,-0.6614613533,0.4931938350,-0.7092412114,0.4458418787,-0.7853662372,-0.2501262426,0.1611269265,-0.5285350084,-1.6091034412,-0.1904814690,0.5117421150,0.3120868802,-0.4000595510,-0.7666798234,0.9289351702,-0.2531622946,1.1696226597,-0.4199027717,0.3557711840,-0.4890812635,-0.5624075532,1.2869091034,1.1910554171,-0.3527726829,1.6044161320,1.2182590961,-0.0506892353,0.2301639915,2.1127395630,-1.8038836718,-0.1752304137,0.3366157413,0.1188599542,-0.3038607836,-0.6614480019,0.6676669121,0.8247133493,1.1330426931,-0.3428809941,1.3835335970,0.7142785192,-0.2841734886,0.8137839437,-0.9763744473,-0.5805509686,0.8007320762,0.7538213730,0.6759210825,0.6667888165,0.1327999085,0.5388278365,-2.5514461994,0.2968763709,0.0856194794,-1.2785900831,1.3809236288,0.1545603722,-1.6245212555,-0.9858641624,0.0683231354,0.9755828381,-0.1573059112,-0.5441203117,1.4863932133,-0.3820349276,-1.8454742432,-0.9734540582,-1.0773816109,-1.5850419998,-0.7565042973,-0.8189355731,0.4546409547,0.1463429332,0.2750940919,0.3259109557,-1.6890685558,0.6469773650,-0.5459377170,-0.3998305500,0.8339851499,2.4585688114,0.1916992962,1.0748101473,0.9357467890,-0.0929414183,-0.6584252119,-0.3570420742,0.0665325448,0.7298585773,-0.8224898577,-0.6889728904,1.1683588028,0.0241510794,0.6844079494,1.1921299696,2.1824836731,0.4918388128,1.9142444134,0.4802091420,0.7324468493,1.3169702291,-0.0847907364,-0.4453571439,0.3699448407,-1.0258462429,-0.3116862476,-0.0516151153,0.7953120470,0.6979629397,2.1893255711,0.3988084495,-1.1665880680,0.6708530784,0.6878688931,-0.8248539567,-0.5847064257,0.5866395235,-0.3840931952,1.4220393896,-1.1018649340,-1.1234416962,0.4223946333,-1.5349240303,-0.0431141518,0.2294328213,-0.6951954961,-0.4299486279,0.9124315977,-1.1123651266,-0.0226851795,-0.6936195493,-0.0072121215,-1.1652355194,0.1803177744,0.5420455933,-0.1974164546,-0.1034566686,-1.1659656763,1.4620234966,0.0676573440,-0.1860376447,-0.0255707297,0.5868535042,-0.7124443650,1.8730525970,-1.4280098677,-1.2111260891,0.0161925871,-1.0231288671,1.1533496380,0.0843438879,1.8062423468,0.4847639501,-0.1865802705,-1.1985164881,-0.7404387593,0.4595836401,0.3563097715,1.2884920835,-0.1245478317,-0.0213925559,-0.2822938859,-0.7291942835,-0.8705769777,0.9221906662,1.9151195288,0.2899247408,0.1849166900,0.0617027730,0.9693223834,1.1434739828,1.3673071861,1.3153882027,0.0179758314,-0.6445850730,0.1999620199,-0.6685073376,2.1232419014,-0.5719731450,0.6747733951,-0.0902681053,0.5864809155,-1.6446373463,0.9344949126,0.1072397977,0.9640066624,0.2265440077,0.1202564687,1.8057010174,1.2321231365,0.1216363013,-0.8128648996,-1.1337174177,-0.4002026916,-0.3188920021,-1.5820558071,0.6039871573,-0.9864867330,0.6012089252,2.4563450813,-0.5732899904,0.9521592259],[0.3260547221,1.8853037357,-1.3204982281,2.0496821404,-2.2340407372,-0.6887130737,-2.2608993053,2.0778744221,2.7424345016,1.0892106295,2.5908606052,0.7477040887,-1.4859695435,-1.3533649445,0.5172604918,-0.2675825357,-2.2346560955,-0.7207927704,0.6210254431,0.4351610541,-1.5440757275,0.7776510715,0.8183146715,-1.9388126135,0.6442348361,0.1557279527,1.6979936361,-0.4870493710,-0.2069469690,1.2790093422,0.0868159607,1.1734422445,-2.3761756420,-0.2298842072,1.1400439739,-0.6254145503,-0.4515355527,-0.3749254644,0.8611729741,0.4854235649,-0.1822100878,-2.0930531025,0.2222319990,-1.4868695736,-0.3692798316,-1.3354876041,-1.3781586885,-0.2719875872,0.9334027171,0.5190452933,-0.8337520361,0.4389167726,-1.2404805422,-1.3586823940,0.8238016367,0.0736838952,-1.8239998817,0.6703809500,0.2388516814,-0.8363975883,-0.5493038297,1.1531976461,-0.2203984410,0.8369795084,-0.3004584610,1.3316445351,0.2204501927,-1.3941072226,0.5796142817,-1.4743403196,0.2360322624,-0.0470898226,3.3544440269,-0.0383296460,0.5620162487,0.3400013149,1.6698427200,0.3058384955,1.0403901339,0.7896620631,2.2139503956,-0.8617098927,0.0116197933,0.2294577211,-1.6858422756,0.8578336239,0.0620743819,-0.1545120627,0.3085938692,0.7593377829,0.6065549850,-0.5367869735,-0.5062915087,0.9420825839,0.3617729843,1.5440033674,0.1615864933,0.0659199730,2.2409822941,1.7344744205,0.8405517936,1.2798552513,0.2383038104,-1.1642180681,1.2421092987,-0.6393301487,-0.7457547188,1.6431002617,-1.1643340588,3.0044844151,0.3767578304,0.5810221434,-1.0409605503,-1.0302780867,1.0567331314,-0.5488164425,-1.0528279543,-1.1218669415,1.9058232307,-0.3550024033,0.3868098855,0.3766503632,2.7125041485,0.2050943375,-0.1935515255,-0.2608304918,-0.7655370235,-1.3006209135,1.7410725355,0.3444109559,0.9625317454,-1.5197042227,-2.2566416264,0.1158713400,-0.3307718933,0.0338330679,-0.2279780209,-0.6273221374,0.5892687440,0.0315760486,-2.4211452007,-0.5741143823,-2.9741172791,1.8141866922,-1.2270373106,-1.7089072466,-1.2472568750,0.2550107539,-0.2996766865,0.6542093158,1.1454129219,0.3152971268,0.0964322165,0.3907919824,-2.0905296803,-0.6558496952,-0.3131229579,-0.4145734608,-1.6045930386,-1.1430335045,-1.1225223541,0.5742155910,0.2588180602,-0.8291150331,0.3271596730,1.0199117661,0.4195581079,-1.2312644720,1.2604261637,-0.1361154765,0.9647939205,-0.8266542554,0.4887631536,1.7701907158,0.7247890234,0.4453731775,-0.8973620534,0.3199851513,0.0539685972,-0.8106490970,0.0733280629,0.0133526186,-0.6579945087,-0.3815068007,1.3540554047,-1.3347084522,0.4231979549,-0.3235702217,0.0383079089,0.2308514863,-0.9960447550,0.0932694674,0.1048167944,-0.6200649142,1.2804870605,-0.0674211308,0.6397436261,-0.1728785187,0.1322377771,0.6113134623,-1.3866189718,0.2125978023,0.0523288436,-0.3402370811,-0.2536427975,1.3289234638,-0.4033775330,0.6017901897,-0.8228362799,-0.7473247647,-1.4831603765,0.3024454117,0.1503181756,-0.6277262568,0.4740160704,-0.1196713299,1.7544775009,-0.0974104479,-0.1266327649,0.9487879872,-0.4919475019,-1.5392595530,0.6162174344,0.5011700988],[0.9101887941,0.8777204752,1.2595125437,0.4018152654,-1.7166287899,-0.8698635697,0.4974375665,-0.0144640859,-1.7648240328,0.9489676356,-0.4182487428,0.4941950142,-1.1213845015,0.2394562066,-1.8312802315,-1.4354618788,0.9216833115,-0.8544840813,1.0875998735,-0.9171810150,-1.1399039030,1.3363617659,1.1059136391,0.8108481765,0.2172601074,-0.6312165856,0.6373825073,0.4367862642,-0.8210284114,-0.7013685107,-0.4239511788,0.5677524209,-0.2321190834,0.2549104989,-0.9880585670,1.1797896624,-1.8412686586,1.3705574274,-0.4650182724,2.6413793564,-0.4931051433,0.0091245165,0.2830466032,0.3595472872,-0.4690435529,-0.6955348849,-1.0818399191,-1.2531290054,-1.5354416370,0.6554519534,1.2130562067,0.1278675348,1.0614162683,0.0805788711,0.1987828314,0.7905623913,-1.1146622896,-0.7665066123,-0.4313217998,0.2837044001,0.1385461688,-0.6566702724,-0.7525277734,-0.3352095187,-0.0488350540,0.6274835467,-1.3588238955,-0.6190199256,0.0382770114,0.0188390687,-0.3117430210,0.7832928896,-0.0662060976,0.0303651243,0.4428240955,1.1008049250,0.4989125729,0.3406849802,-1.3838136196,-2.5580713749,0.1072607487,-1.7010232210,0.5015914440,0.3309070468,-0.2240238488,1.1814460754,1.0675925016,0.2472229749,-1.1753770113,-1.4115841389,0.6599791050,-0.8903461695,-0.7513751388,0.4441344738,0.0984115377,-1.6328358650,-2.7814333439,0.0352627784,-0.1952062249,2.5094332695,0.2350155562,-0.7036231756,-0.6226969957,0.1006892025,0.2735290527,0.3248717487,-0.9859005213,0.5884060860,-1.6339645386,0.6399517655,0.0949723125,-0.8603503108,0.7330551744,-0.4018296599,0.0621890053,0.9158736467,1.1411347389,0.4787520468,1.3159184456,0.6555212736,-0.1050648540,0.5479673743,0.1422740966,-0.0826835856,-1.1075340509,0.2643381357,-0.1198114604,-1.2939515114,0.4103282094,0.9645326734,-0.5595097542,-0.3438761234,0.9068605900,0.7637982965,1.2814041376,-1.3839570284,-0.5809727311,-0.3262540698,-1.5268710852,1.2061607838,0.9503488541,0.0565067790,0.5170537233,0.8136867881,0.9898092151,-0.5544456840,-0.2870925963,-1.6100064516,0.2155759931,-2.0440261364,-2.6521677971,0.6410305500,1.6874045134,-0.6198192239,0.6183931231,-0.3552034199,0.3808440566,-1.5048646927,-0.7682568431,-0.3286943138,-1.2153249979,-0.8321967125,0.6857488155,-0.7768926024,0.4683583379,1.4393649101,-2.0580306053,0.2417555600,-0.1287895143,0.9849979281,-1.2673319578,1.4937902689,0.6139543056,-0.7011052370,1.6358752251,-0.5591640472,-1.8699896336,-1.2037640810,-0.4650449753,0.4130979478,-0.4781028628,-0.3773052096,0.4896984100,0.5291000605,0.2909781933,-0.1239582077,-0.2593092024,-0.7551556230,0.0570368096,0.7843011022,-1.3545408249,0.3017587662,0.3814600408,0.1865427494,0.3018524945,1.1108403206,1.4385936260,1.2737234831,-0.1687798649,-0.2700041831,0.0939677283,-0.5982590914,0.0442983992,-1.2631759644,2.1559460163,-1.0446652174,1.4373403788,-0.5136585832,0.5201707482,-0.1930320710,1.4882138968,0.1718825847,-0.6136702895,0.1616206765,-0.1911924034,-1.3606525660,-0.6224917173,-1.8901438713,2.1093156338,-0.5344481468,0.5858753324,-0.1767421663,0.5121246576,-0.4941667914],[0.4248593450,0.3161675930,-0.8331077695,-1.4626350403,-1.7189588547,-0.2488492578,-2.3246755600,0.8617739081,0.4979015291,0.7135391235,1.3816496134,1.7785661221,0.1185933501,-0.0692301467,1.3845703602,0.9387071133,0.1926213652,0.1305808425,-2.9852871895,-0.5626637936,-0.8844199181,-1.3015922308,-1.2187414169,-1.1000576019,-0.3812155724,0.7491910458,-0.7949401736,-0.1017555743,0.5167959929,-1.2674736977,-1.3182394505,0.3694709241,1.0132818222,2.0066077709,0.0588135272,-1.0413175821,0.4451495111,-1.5961176157,-0.5127106905,-0.0736058503,1.2943279743,0.9867642522,-0.3749952614,0.0219509210,-0.4264596999,0.6405826211,-0.2793408334,0.3446552455,0.2953101695,0.4174514413,-1.3424669504,-1.2077305317,-0.5029827356,-0.9704607725,-0.5791391730,-0.3410248160,1.2203289270,0.3139457703,-0.6450718641,1.9737294912,0.0937890559,-0.7589262724,-0.0004526210,-0.1392761469,-0.5184856653,-0.6445096731,-2.0933535099,3.6174440384,-1.4014939070,-0.3285408914,-0.0851450935,-0.4968545437,0.8777781129,1.2304735184,0.9676017761,0.2992458940,-1.0884692669,0.7705311775,1.2788481712,-0.4678036571,0.8469130397,0.5697970390,0.3013136983,0.7260673642,0.2109190971,0.9467196465,-1.5477573872,-0.7547515035,1.0165404081,-1.2013443708,-1.0009258986,-1.0381124020,-0.9545734525,1.0061957836,-1.3585246801,0.2498244196,-0.6100196242,-1.9319077730,-0.3646067679,-1.4917006493,0.7829834819,0.7965960503,-0.8203605413,0.7800762057,-0.1055379063,0.3846244514,-0.9033451080,0.9355486035,1.3010201454,1.5083156824,1.0601898432,-2.1822016239,0.0641560182,-0.1708367616,2.2009952068,-1.4505349398,0.6033412814,-0.3475028276,-0.0245241802,0.9902088046,0.7871859670,-0.2820515037,0.6837606430,1.7600520849,1.7086303234,-1.3337498903,0.5148547888,-0.1411535144,-0.5787584782,-0.1753298044,0.7662317753,2.2890543938,-0.3373456299,-0.7940471768,0.0870957598,0.2046343088,-0.3655799031,0.1403742582,0.7487806678,-0.0493908785,-1.0613024235,1.6651679277,-0.0721103400,-0.9661730528,-0.0118546663,-1.2443343401,-1.8370332718,-1.1943985224,-1.3811161518,0.0265875105,0.7113240957,0.4174372852,0.7321901321,0.3165853620,-0.8200309277,0.9104493260,-2.0742957592,-0.8789162636,-0.6044740081,1.7389211655,0.3027465940,0.8017164469,-0.1329290718,1.1411834955,-1.2318832874,-0.2818616629,0.3668227494,-0.1943957955,0.1467296183,-0.9400892258,0.6216479540,-0.3850940168,-0.2756251991,-0.0756106824,0.6708564162,1.7877844572,0.2348715514,-1.1913688183,0.1658644378,0.3144343793,-1.8043081760,0.8749025464,1.1378152370,-2.5943167210,-1.2024546862,1.2210763693,1.1203122139,0.1245990917,-0.9790798426,-1.1268764734,-0.4267218709,-1.6712464094,-0.2931473255,0.9772987962,-1.3596971035,0.0209441744,0.3558107018,1.1550899744,-0.5986868143,0.3185078800,0.6811698675,-0.3164308965,-0.1549912542,0.0994456187,-0.1038790345,-1.7121478319,0.2452230304,-1.0489326715,-0.2093195915,-0.4916971028,0.8829078674,-0.1305598617,0.0435559079,-1.4980833530,-1.2615302801,0.0852286965,0.5818434954,-0.8840133548,-0.4041324556,-1.0049831867,-0.5677416921,-1.8189404011,0.6888154745,0.4762073457],[-1.6676757336,-0.0117815798,0.9636470675,0.2832666636,-0.3563674390,0.9574931264,1.1856490374,0.1930582970,1.1372985840,1.4829488993,0.4178263247,-1.0189751387,1.3939164877,-1.5043624640,-0.2485114038,-0.4893157482,-1.0204097033,1.2906845808,1.1540821791,-0.4610202312,-0.4744220972,-0.0785018057,0.5601623058,0.2905760109,1.2008045912,-0.9055359364,-1.2546924353,-1.3627620935,-0.4191326201,-0.2417927384,-1.6857676506,-0.6772042513,-0.7309972644,1.0813498497,-2.3207206726,0.3441968262,-0.6235502958,1.6584020853,-1.7576291561,-0.8164684176,2.6233417988,-1.4983351231,0.0849152654,-0.2290281355,-0.6555573940,-0.1725483537,-0.3922250867,1.2119016647,-1.9553462267,-0.6105675101,1.1522092819,0.6248836517,0.0559881888,0.4049755335,0.0688843802,-1.0096157789,0.3500415385,-1.3688995838,-1.1148210764,-0.2586264610,0.0292662345,0.2617202699,-1.1088030338,-0.2185048759,0.1528363377,-0.1823088825,-0.7560430765,-0.8633227348,1.8211431503,1.2482187748,-1.0431731939,-0.1619359106,-0.2505675554,-1.1278392076,1.4434571266,0.1162277162,0.8356947899,0.2004559189,2.7481603622,-0.6122629046,-0.1336817294,0.9449667335,-0.2098537385,-1.0220044851,0.1911619157,-0.6354675889,-2.6473999023,-0.4538244307,-1.0034236908,-0.4797924757,0.3036291897,-0.7502538562,0.3391863406,-0.3267144859,-1.5663520098,1.5844787359,-0.1541496515,0.0080233496,-0.2457676530,0.5802427530,-0.2375820726,0.4829804897,-3.2749388218,0.6998037100,-0.8014594913,-0.9856346250,-1.3622802496,0.9920208454,-0.8667515516,-1.1467056274,0.8337497115,1.4432774782,-0.4017332196,-0.1752787232,-0.1312951446,0.5203475952,1.6061456203,-0.3765571713,1.2273452282,1.0681328773,0.5741702914,1.6537784338,-0.1392436624,1.5222073793,-0.4666440189,0.8264855742,0.2139969468,-0.7227970362,0.1318987459,0.6431787014,0.0269627497,2.9194891453,-1.3445217609,-1.3357516527,-0.3646216989,-0.3339246511,-0.9061585069,-1.7167688608,-0.2931183875,-1.2002370358,0.0027208761,-0.8410323262,-0.8513520360,-0.8169867396,1.7177858353,-1.0786051750,-0.9716501236,0.7862368226,0.4703477919,1.5872616768,-0.6943163872,0.5795159340,-0.4428988099,-0.0336347744,-0.0675015002,-1.1188058853,-0.0059059928,0.3079329729,-1.5865097046,1.1954914331,0.0107388850,0.2328452766,0.3771139979,0.1556452364,0.5254788399,0.7656033635,1.8018838167,1.3587571383,-0.1060156971,0.5899567604,1.1444541216,-0.3648647964,0.3766710162,0.0900015309,-1.0156743526,1.4167870283,-1.1703493595,-0.6448116302,-0.1051116288,0.0020963072,-0.0745361373,0.8696280718,2.8341960907,-0.3513678610,1.3280119896,0.1946055442,-0.5135858655,-0.5002872944,-0.3048385382,1.4506249428,0.0044972659,-0.0088738408,0.6910936832,0.6415634751,-0.6344572902,0.2697565854,-1.0585714579,-0.5347191095,0.4512789249,-0.7875731587,-0.1061345711,-0.3123532832,-0.6738702059,0.7054605484,1.8631844521,0.6854217649,-2.4223566055,-0.3866092861,0.4214096665,-0.6705033183,0.6293103099,-0.9606555700,0.4024822712,0.9286620617,-0.2251343727,0.0375308469,-0.6891617179,-0.4474659264,-0.4347704947,-0.2488876730,-0.6395900249,-0.6902248263,-0.8352594972,-1.1157217026],[1.1579307318,-1.0021015406,-0.9917744398,-0.5251912475,0.3097694218,0.9646864533,0.4288187623,-1.4920967817,-0.5031630397,2.0002958775,-0.8010430932,-0.1773892045,0.7078548670,0.7890493274,-0.7094715238,0.0394469835,1.5130943060,1.0473628044,0.7980496287,-0.2446120083,-0.8357499242,0.3035597503,-0.0023617276,1.8378235102,1.0127995014,1.0019253492,-2.0041329861,0.5986630321,-0.8298110962,-0.4977650940,-0.7103406191,1.1140469313,-1.0463049412,-1.1186000109,-0.6619963646,0.4847993851,1.1088175774,0.1798587739,0.6712608337,1.7970820665,-0.7250568271,0.5614169836,-0.3647833765,0.2266235948,0.4138481319,-1.9311686754,-0.9302152395,1.2558653355,-0.9853758812,0.7505119443,0.8275774121,-1.0908306837,1.4267879725,2.1317381859,-0.0587358475,-0.6239714026,0.5322213769,1.2944272757,-0.5415900350,0.3425603509,-0.0551128164,1.2260411978,-0.3073526919,-0.0158004314,0.0446104370,-0.2874431610,-1.3118320704,0.3087975979,0.4541290104,-1.3672093153,1.0977481604,-1.3857321739,-0.3321656585,-0.7112305164,1.7794216871,0.7596542239,0.9350003004,-0.6077075601,0.3117588460,-1.4983228445,1.2511718273,0.2724351883,-0.1407505125,-0.5702785254,1.9123885632,-0.0837195367,2.3620803356,0.5817832947,-0.6662903428,-1.4938615561,2.1392271519,-0.7084274292,0.3544390500,-0.1004240811,-0.7890850902,-0.6086986065,0.4826420844,0.5797073841,-0.9093092084,0.5444802046,-2.0700707436,-0.4536957741,-1.0652302504,-0.5660454035,-0.6453261375,0.5172896981,-2.2897562981,-0.8132987022,0.9573528171,0.1642129719,-1.1041778326,-0.2960029244,0.8012885451,-0.6051740646,0.2041897923,-0.2072525769,1.1589853764,-0.4625106752,-0.4647492766,-0.1737691313,1.1656908989,2.6465435028,0.7356799245,-0.6035447121,0.7761955261,0.1258596480,1.3203146458,0.0428092740,-0.5732164979,0.5334172249,-0.4383377433,-2.6443896294,-0.4882341325,-0.4131014943,-0.6506836414,-0.0589487515,-1.1589503288,0.0197763573,-0.4185853302,0.7655056715,-0.3029468656,0.7333875895,0.5726686716,-0.2836067080,0.0208763424,-1.9726077318,1.9458686113,0.8031986356,-0.5565538406,0.3064076304,-1.3078083992,1.3880399466,-2.4986119270,0.2955805659,0.4166693389,0.3991940618,-0.3132807612,1.9816641808,1.2151714563,1.7478718758,0.2781778276,-1.2538698912,0.3590858877,-1.6911876202,-0.3242392540,0.0751941875,2.0730650425,0.8241178393,1.5471822023,0.4346613884,-0.2523943782,-1.6861196756,0.3959576786,-0.3940330148,2.2799172401,-2.1057817936,-0.6796046495,-0.2385020703,-1.6790004969,0.6090376377,-0.2106702179,-0.5197761059,0.8246834278,-0.6630619764,-0.6153048873,0.3878263831,-1.6378360987,0.2738104463,-0.6956075430,-0.6798497438,-0.0288074967,-1.8039088249,-1.1410199404,2.5300855637,-1.3932182789,1.1908702850,-1.0227982998,-0.9062312245,0.8252307773,1.0381580591,-0.2450716347,-1.2224311829,-0.5532920361,0.0643740818,0.4217404425,0.9086120129,1.4833407402,1.8077830076,0.1879525930,-0.2831076384,0.8077802658,-0.0759647638,0.7265745401,1.4762508869,0.3692570925,-0.8803898692,0.5382549167,-0.4830761254,1.4964990616,-0.4836547375,0.4294926524,1.5498590469,-1.8430325985,-2.7364418507],[1.1671922207,-0.4623150527,-0.0125561962,1.8133157492,0.4415445030,1.3514368534,-0.6283529401,0.9533058405,-0.9504081607,-1.3233529329,0.0559342392,-0.3043320477,-0.6132194996,0.2827467918,0.0211385582,0.6981754899,-0.3219046593,-1.1821082830,-1.6019649506,-0.7358622551,-1.5021909475,1.3134816885,1.0149642229,-1.4239060879,-0.4501751363,1.4480149746,-1.8850461245,-1.0855251551,-0.0552835427,1.1126552820,-0.3457433283,0.3627731502,-1.2939021587,0.0297653861,-1.0057828426,1.3407930136,-0.0574732423,-0.0764587745,2.4194180965,-1.4760574102,-0.5405495167,1.3480896950,-0.9823193550,1.7727591991,-0.7762312889,-1.1459913254,-0.9690763950,0.3205409050,0.2968346179,-0.0867063925,1.4222966433,0.0621957928,1.0828131437,0.5762202144,1.8638826609,0.0221075136,-0.4786196649,-0.5847828388,-0.0353326574,1.8716874123,0.3440319300,1.1833447218,-0.8264019489,-0.4332354665,-1.6377058029,-0.2957936227,0.4296649694,0.0330034830,1.4408249855,-0.9070200324,-0.2233535200,-0.5568699241,-0.0201630127,2.8606996536,0.1218447164,-1.1993663311,-0.1101554483,-0.0428591706,0.5230171084,0.9578502774,0.3164121807,1.4629195929,0.7499135137,-0.6314778328,-0.8180010915,0.9594846964,-0.0734098107,0.3534791470,0.2100719959,-1.3744202852,0.9371819496,1.8197331429,0.0385162905,-1.4419538975,0.4320627749,-1.0061644316,0.3089143634,-0.1311210543,-0.8641072512,-0.7125442624,0.9392834306,-0.6008676887,2.2745432854,-0.0422098339,1.4438676834,0.7602827549,1.7542577982,0.3981570899,1.5356993675,0.2983169258,2.0158410072,-1.3415718079,1.8989692926,1.7546209097,1.2540712357,-1.6169677973,-0.8762540221,0.9787555337,-0.6943745017,-0.0487157442,-1.8384969234,-0.0759068280,0.4606977403,0.4347637594,-0.4993920028,0.2148143649,-2.0123169422,1.6227184534,-0.2299919575,1.4253363609,-0.3838140070,0.2162957191,1.7839958668,-0.6574246287,0.2178135365,0.4388503432,0.5152842999,2.0197911263,0.3589576781,-0.1519799829,-1.3294918537,0.3256805837,-0.3265584409,-1.4352842569,-1.0348974466,0.6977605224,-1.4681086540,-1.1425429583,0.6803500652,0.4048239589,-1.0909260511,1.1290786266,0.8217989802,-0.9025979042,1.1533937454,1.0704193115,-0.4884494543,1.4770652056,0.0373433903,0.0077836541,0.6826797724,-0.8579919338,0.2251290381,-0.9385836124,0.7698971629,-0.1225737706,0.1218759790,-0.3805133700,0.0967485979,-1.8844478130,2.3024020195,-0.6849756837,-0.3304445446,-1.2974334955,-0.6761263609,-1.4177010059,-1.3832147121,-1.3262499571,-0.3938881457,-0.6269910932,0.2355906516,0.5818787813,-0.0156290065,-0.4996424615,1.8249180317,-0.2702373266,-0.0378957987,-1.3491808176,0.8316279650,-1.3577632904,-0.7994315624,-0.3274593353,-0.8930548429,0.6743643284,0.5958985686,0.0216204133,1.6101511717,-0.9585517049,-0.3917265534,-1.8435053825,-2.2900974751,-0.6511493921,2.3449501991,0.1335494220,-0.1154850870,-0.0498552248,-0.5847663879,-0.7291586995,0.2750283182,0.7288957238,-0.2256766409,0.7525697351,0.9032928348,1.1697378159,0.3740186095,-0.6911426783,0.3261582851,-0.2271312475,0.4647946060,0.0876392424,-1.1337498426,0.8469501138,0.7095554471,1.2220104933],[1.0301719904,-0.7379205823,0.9471271634,0.3801421523,1.1546025276,0.5475896597,0.0115259765,1.3954277039,-0.7114993334,0.9818466902,0.1740345955,-1.6722321510,-0.5188888907,-1.2580188513,-0.0774400756,0.3143561482,-0.5942593813,0.4429920316,0.6551231146,1.8638199568,-0.8231512308,0.9583299756,-2.8441357613,-0.4623622894,-1.1016976833,-0.3292089701,-0.3490474820,0.7918364406,0.1010522470,-0.5955529213,0.8099021912,-0.6571288109,-0.5177548528,0.7080375552,0.4156391323,0.6487434506,1.0146337748,-1.5045164824,-0.4959926903,0.1978851706,1.0364204645,0.0211838745,1.3547075987,1.0349222422,0.2016938329,-0.7771456838,0.3141413033,0.4111408889,-1.7946816683,-0.8617743254,0.4630215764,-1.4796572924,0.2156282216,0.0648754835,-1.1287294626,0.8153375983,0.8462402225,-0.9069088697,-0.2586428821,1.5503832102,1.2572857141,-0.8906383514,-0.1220265105,-0.3822940588,-0.1107923314,0.3378605247,-0.4759807289,0.8223170638,0.5999978185,-0.5032872558,0.7468966246,-0.5116183758,-0.6416741610,-2.3637897968,0.5536193252,1.0693031549,-0.1984776855,0.5212368965,-0.1010358110,0.1642877162,2.5420272350,-0.8539747000,0.0189713798,1.4053024054,-1.0632971525,-0.9598603249,1.3750352859,-0.4135048091,1.8846297264,1.1503682137,1.0024423599,-0.0212843530,-0.7975702882,-1.5567365885,0.1432193369,-1.2462441921,0.1648118943,-1.9669402838,-0.0362720527,1.8989335299,-0.7362707853,-0.0263065547,-1.3074949980,-1.6733379364,0.4589193165,-1.5560327768,-0.2851602435,-1.1524580717,-0.7894440293,0.2129414976,-0.0265334882,-0.9173115492,-0.1233007684,-2.1116597652,-0.2548934221,-0.1000629812,-0.2341962755,-0.5075214505,-1.0097604990,1.6759552956,-1.1805329323,0.3522910178,1.1721547842,-0.8795605898,-0.2225883007,0.3419721127,-0.5531402230,0.5414720178,1.1503173113,0.8090146184,-1.1797549725,-0.4781074524,-0.4606604576,-0.6850347519,-1.0958294868,-0.8130276799,-0.4801674187,-0.5496889949,0.1198312268,0.9770823121,1.0006096363,1.5605578423,-0.2776606977,1.5436390638,1.3431284428,-1.0694115162,0.0319580436,1.0235497952,-1.6327320337,0.2094062716,-0.2779122591,-0.2549938858,0.9235299230,-1.7200658321,1.3078080416,0.3141606450,-0.9713298678,0.6507166624,0.5240988731,-0.8743537068,0.2245029956,-1.6577179432,-0.8995621800,-0.5963401794,-0.3028064370,0.6373192072,1.0106426477,-1.1714172363,-1.1551836729,-0.1290144324,-0.4418173432,-0.5398723483,0.7920086384,-0.1006233692,-1.2925938368,0.4630675614,0.5318786502,1.0925804377,-1.8792281151,1.2360200882,-1.0882061720,0.2990363836,2.0535075665,-1.3437252045,1.7961989641,1.0472359657,-0.6580789089,0.3637872040,0.8887823224,-1.0567698479,-1.4380348921,0.3630021513,1.0801453590,0.0848029777,0.3457524478,-0.5923113227,-1.1642155647,-0.6858039498,0.2838497758,0.1613612771,0.3430170417,-0.0750150234,-2.2998411655,0.1167422906,-2.1216418743,-0.7399117947,1.0929222107,-0.8083111644,-0.4524880946,-0.5343817472,-0.2154788822,0.5326138139,-0.4642713368,0.3502049446,0.2950757146,-0.3348174989,0.6952251792,-0.7663141489,-0.1191459745,1.8016871214,1.1503630877,0.0879920721,1.6557919979,-0.9713310003],[-0.0952440724,0.0566643141,-0.4121611416,0.3589834869,-1.2417886257,-0.3981787860,-0.0955326259,-1.1746367216,-0.0903539136,1.8228377104,0.1594277769,1.3675057888,-0.8630368114,1.4712395668,0.1054009870,-0.2368630320,2.3829298019,-1.6929466724,-0.7913834453,-1.4271126986,0.2466607392,0.1234057769,-0.3978918791,2.7214033604,-0.6303295493,0.0198053010,-1.1812092066,-0.1202231199,0.3901976645,-1.0169278383,-0.3118689060,-1.2093325853,1.5801656246,-0.1262614280,-0.2772842050,1.7430753708,0.7829131484,-0.9974278808,0.5059066415,0.2753480971,-0.8170706630,1.4233903885,1.3587424755,0.3708223104,0.2664794624,0.0544346385,1.0021533966,-1.4790313244,0.5714965463,0.5185027122,1.2404508591,-0.3423627019,1.0433201790,-0.5620381236,-0.7999502420,0.3974117339,1.3396379948,0.5925657749,0.5062606335,-0.9846734405,-1.2468835115,0.1999926269,-0.7213277817,0.5002495050,-0.7886040211,1.4698174000,0.5244092941,0.2752928734,0.4098138809,1.4521061182,-0.7069355249,1.2058763504,1.6311770678,0.2563337386,0.5263522267,0.1898272932,0.8347653151,1.3658587933,1.1031242609,-0.0868932381,-1.0478790998,-0.6240808964,-1.3151839972,-0.6831984520,0.8852385879,1.6934367418,-0.3677411377,-1.4583725929,-0.6255928874,0.8111643195,2.1374673843,0.7400379777,0.1656147391,1.7086645365,-0.0734900981,0.8953284621,-0.6813739538,-0.0614439286,-1.2261948586,2.0596497059,-1.5261852741,0.6271134019,-0.3758432567,-0.9482771754,0.0166640133,-1.3925417662,-0.6260951161,-0.0622643456,-0.3813502789,0.5994868875,1.6862668991,0.5603588223,-0.0449138060,-0.1375849396,-1.6965512037,0.3292586803,0.3680608571,1.5089581013,-1.2935491800,-1.4540292025,2.0181722641,0.1847225428,-1.5040644407,0.8418682218,0.3571032286,0.1277844757,0.6452940702,0.8302066922,0.3316073716,-0.4937435985,0.1727012694,0.3508908153,0.4942966700,-0.1890991032,0.1464315653,1.2672983408,-0.5420016050,0.3489682674,-0.1656922847,1.8272436857,-0.0872897953,0.6986168027,-1.1467864513,-3.0869462490,-0.6514915228,1.3085490465,-0.1464149803,0.5489206314,0.3809079528,0.1223399192,-0.2563851178,-0.3303251863,-0.8524194956,-1.0438624620,0.9942628741,-0.7164640427,0.2523762882,1.8409081697,-0.0395202041,-1.9294208288,-0.4429220259,-0.2262780219,-0.6784824133,-0.4743759632,0.1318203062,0.1481006593,-0.0424164087,-0.5459465981,-0.7982281446,-0.4270841181,0.8304693103,-0.3444064856,0.0296886452,-0.0255151410,0.2025533468,0.6004497409,-0.5635874867,-1.3864352703,-0.5710804462,1.3912957907,0.1319333613,1.6396538019,-0.8336297870,-0.6034071445,-1.9802199602,-0.0118512791,-0.1108769551,-0.0129075134,-0.2374236435,-1.5954982042,-1.3275305033,-0.4772657752,-0.2762099206,0.8561545610,-1.0112084150,-0.6464535594,1.1537920237,0.2840941846,0.1511519849,-0.1263342947,-0.3606207073,-0.9840847850,1.4167820215,0.6281455755,0.8761416674,0.0240065102,0.0428546667,0.7045249343,1.3199563026,-0.2455899715,-0.4333233237,0.0839606822,-0.5357459188,-0.5189141631,0.0773424804,-2.4033913612,0.5579027534,0.2220277339,-0.1305405945,1.1741777658,0.1842390150,-0.1543905437,0.2372447699,-0.0578634739],[1.1268147230,-0.6168632507,-0.6207220554,-0.3247307837,-0.6050617099,2.4619061947,-0.5523837209,1.7028896809,0.0241982024,-0.3090949655,-0.1183786169,0.6874657273,0.5977433920,-0.3424105048,1.0477715731,0.0600404032,-0.4835692644,0.2377821803,-1.9360288382,-1.1095243692,-0.2601641417,0.7298178673,-0.9078957438,-0.7949860692,0.5281986594,-0.0193301439,-1.4828484058,0.4641183615,-0.6966081858,0.6504861116,-0.7225518227,-0.0420571826,-2.2333300114,-0.6292539835,0.3642581701,2.1744987965,-1.4005479813,0.6594906449,0.0190161932,0.7265573144,0.4081478715,-2.0283536911,-0.2277533710,0.2533299625,-0.2692781985,0.8519235253,-0.9961074591,-1.3291726112,1.1561888456,-0.1695564687,-0.6412171721,-0.5963202119,-1.2543619871,0.3706256449,0.2367728949,-0.8329231739,0.1212827340,-0.1572631747,0.4193502367,-0.5477141738,0.4441763759,-0.1068404242,0.6890825033,1.2757037878,-1.3237390518,-0.7205080986,-0.2300838381,1.9304093122,-0.8222891092,2.3237411976,-1.5258569717,-0.2129818201,1.1545015574,0.1849959046,-0.4873098731,0.5797542930,0.4350897372,-0.4880842566,1.4818840027,0.7041978240,-0.0506795347,0.4129129052,1.2196661234,-1.2100301981,-0.6907811761,0.2639520168,0.6720523834,-0.1672959328,-1.3558725119,1.5750267506,0.5315433145,-1.3783342838,-0.5600621104,0.1594666988,-0.1252618730,0.1798083037,-0.3277032673,-0.8285724521,-2.2868967056,-1.3793725967,-0.7088655233,-2.1242513657,1.8299680948,-0.0487202220,-1.1619849205,-0.7737154961,1.1845551729,0.0657233149,0.5641449094,1.8346886635,-1.1153488159,-0.8920660019,0.1958157718,0.3952474594,-0.2617920339,-1.6110399961,-1.3485546112,-1.3979731798,-0.6749272346,-0.2331542522,-1.0221537352,-1.5719534159,0.0787544250,1.4476664066,1.0816694498,0.5359908342,0.5086901784,1.3539590836,2.1905260086,-1.3281347752,-1.5861293077,1.0052930117,0.2073535621,-0.2195786089,-0.6525210738,0.1171994656,0.2404762059,1.7054648399,-0.0166308656,-0.9980779886,-0.3845568895,0.2131299078,0.8598623276,-0.2320811450,1.2193504572,-0.7898730636,0.5503804684,-0.6646599174,-0.4484573305,-1.3752053976,-0.0077703991,0.4516531229,0.5158145428,1.5019677877,2.3400790691,-0.1689572185,-0.7131705284,0.0862102732,-1.8778115511,-0.5737338662,0.3488899767,0.0695413202,1.4736529589,0.1018898711,-0.3780619502,-0.3608337045,-0.3679491580,-0.5431326032,-1.0214086771,-0.1403140426,0.3082767725,0.1509161294,0.6119741201,0.7313207984,0.8829240799,1.8567088842,0.5742434263,0.4984557033,-0.4186875820,-0.9313029051,-1.6411021948,-0.5562921762,-1.4994580746,-1.5915665627,-0.0610120445,1.5467461348,-0.7506592274,-0.4631983638,-0.1926139444,1.7024631500,1.6659804583,0.5192429423,-0.5472186208,0.0565663762,0.2284213901,1.5348349810,0.2286630720,0.0394604020,-0.8655991554,0.8045617938,1.2748690844,0.5713320971,-0.1608036757,0.7733766437,2.1223337650,-0.9744271040,0.7814424634,-0.5671689510,0.4748123884,0.9607690573,-1.0613287687,1.2581557035,-0.7631398439,-1.7736225128,0.0356541537,-0.6820560694,-0.7282853127,-0.1053154469,-0.9815471172,0.4679682255,0.1102410853,1.7661479712,0.9696770906,-0.6157441139],[0.0832946599,0.0154042337,1.4200258255,0.1912794560,0.8820790052,-0.7564318180,-0.9813390970,2.7474064827,-0.4309709370,-0.5363450050,-0.1029144824,0.4400387704,-1.0086545944,0.8715492487,-0.4610286951,1.2774317265,-0.3406333029,1.0627976656,-0.0177198425,-1.4871954918,-0.3110130727,-0.1273606718,-0.5646942258,0.3632155955,1.1745493412,-0.5503141880,-1.6987376213,0.5262299180,-0.5710070729,-1.2713433504,0.8851079941,-0.6829869151,-1.9171721935,0.7940423489,0.5627143979,-0.9600135088,-0.2346548587,-2.2996087074,0.8724299669,0.0616741218,0.4606336057,-1.6986548901,-0.1236542985,-0.8901436925,1.6886969805,-0.6276292801,-0.7237911224,0.5590992570,-2.4663658142,-1.0565769672,-0.4022848010,1.6344102621,-0.4803332984,-0.4069189131,1.0568474531,-1.2874617577,1.5985314846,0.8697004318,1.5187076330,0.3975264132,-1.4097440243,-0.0381349325,-1.1354163885,-0.4750063717,-0.2226876616,0.7452670336,0.2666867077,0.1722257733,0.4386547804,-0.4073028862,-1.2752696276,-0.3434637189,-0.2251634151,-0.8087444901,-0.7511361837,0.2216085494,-0.5121274590,-1.8204050064,0.7472478747,0.5685710907,0.1281473041,-0.8394502401,-2.5131106377,-0.9077122808,-0.6943283677,-0.5469990373,0.7462623715,-0.3083970845,-0.8384279609,0.8198192716,0.4979269207,0.0270626321,-0.1213922799,0.7247827053,0.3348164260,-0.2508352697,0.7643464804,-0.4294463694,-0.1170479879,0.0250977706,0.7436251640,-1.5160830021,-1.5851063728,-1.5032289028,-0.4924083948,0.6285675168,-1.1842933893,1.1135993004,1.2736849785,-0.0435120836,-1.3845635653,1.7388744354,1.7109934092,0.6870909929,-2.3529503345,0.5297210217,0.0182195548,-0.8128156066,-0.1349581033,0.5240533352,-1.1022468805,-1.2241221666,-0.2381365448,-1.0415886641,-0.1002316251,-0.4005219340,-0.2848261595,0.0950556099,1.7183667421,1.2604315281,1.8246142864,-0.2740058899,1.5894554853,0.2125496417,-0.8694303632,0.3586742282,-0.6369240880,-0.3130733967,0.1877637058,-1.3034642935,-0.5413097143,-0.9775701761,0.3182064295,-1.2245349884,1.3321487904,-0.1348018348,-0.0079483762,0.5871071815,1.6216561794,-2.3066439629,-0.3211205304,0.0831940025,-1.0155971050,0.3312295079,1.2365626097,0.4100286365,0.4295566678,-0.4415833354,1.0282262564,0.7797985673,-2.6591732502,-1.3471249342,0.7747009993,0.7138849497,-0.1456113309,0.9523909688,1.2731977701,-0.9705725312,0.6225118637,-0.6567438841,0.1674103588,-0.1905986071,-0.5853870511,0.9797718525,-0.5967161059,0.4017387033,-1.3391740322,-0.7596985698,0.9018880129,1.2323490381,0.6116399169,-1.2004809380,0.7040430307,1.9253069162,-0.1669967473,-0.4774456620,-0.1453061104,2.7242598534,1.0761202574,-0.7772324085,-1.8398270607,1.1820985079,2.2291193008,0.1210172623,0.5169967413,-2.0369746685,1.3334836960,0.9594944715,1.3979049921,0.0152120637,0.3863286078,0.0464983992,-0.8423812985,-0.4377827644,0.6164746284,-0.6307107210,-0.4342782497,0.2298891991,-2.0079159737,-1.4043928385,0.1458393335,-0.6708733439,-1.6447790861,0.0703957826,-0.2523188591,0.3924876451,1.0161181688,2.0849382877,0.8516621590,0.9680776596,0.1318573952,-0.7458209395,0.1732912064,-0.0171494074],[1.5169693232,0.3570988774,0.9281348586,-1.3707672358,-0.0427510217,0.5875717402,0.0559275597,1.6824957132,0.8721467257,-0.8102717400,-1.0185278654,-0.8793839812,1.4426631927,1.5465738773,-0.8688929677,-0.1653910875,0.3350679874,0.9537213445,-0.9668954015,1.6751704216,-0.6430878043,0.3188067973,-0.3906269073,-1.2710647583,0.3388320804,-1.1807581186,0.9608909488,-2.0899810791,0.7109305263,-0.9204555750,-0.9144626856,-0.6868638396,-0.1635092348,-1.7601587772,0.1764658093,0.1667043418,-1.5872178078,-0.0366247445,1.0080679655,0.4422006309,-2.4767754078,-0.9045382738,-0.7697851062,-0.1585956812,-0.2367852628,1.1754802465,-1.1316013336,1.7630548477,0.2800961733,-0.6288664937,-1.2302342653,0.3262609243,0.2660154700,-0.0530349985,-0.4598177373,1.6693603992,-0.1160294041,0.4715109169,0.9127645493,1.6281094551,0.9660769105,1.7898586988,1.6077495813,-0.9647401571,1.1253098249,-1.2583026886,0.8288027048,-0.2743187547,-0.4070868790,1.4383150339,-0.7290006876,0.1344997436,-0.1674019545,-0.3628393710,-0.4540843666,0.6296550632,-1.8585166931,-0.1117752790,-0.1778820157,2.4086344242,0.1753031015,-0.4950847924,-2.2426016331,-1.4452265501,-0.2448479533,-0.1808102429,-0.2288896441,2.1536924839,-1.3813554049,0.2147695422,-0.0196479950,0.0244122334,-0.8968064189,-0.6828643680,-0.1684126258,-1.0011633635,-0.7751793861,-1.1394294500,-0.8886438608,-1.8494819403,0.1862669736,0.3815818429,-1.2124689817,0.4131552875,1.2002992630,1.5946217775,-0.5457906127,0.4128240943,0.2786887288,0.2461876273,-1.3450722694,-1.1776552200,-1.2061234713,0.2029906958,-1.4223995209,0.5777701735,1.6634222269,-0.2870059907,0.2412759215,0.3775524497,-0.9314845204,0.6768042445,-0.4252702296,-0.7470455766,0.2802719474,-0.5464428067,0.6813836694,0.7286756039,-0.5409637690,-0.7058429718,-0.1084262133,2.0533525944,1.2147784233,0.1359319836,1.0503759384,1.0613586903,-0.2144398689,0.1920922250,-1.0345438719,-0.5036805868,1.6150988340,1.6786446571,-0.4663047791,1.0681966543,-1.3721605539,0.1843423843,-0.5531916618,-0.8321005702,-1.0742633343,0.3251988888,-0.7832528353,0.7555800676,-0.1313978136,-0.0314237624,0.5373218656,-0.7306606770,0.6202054024,1.9043105841,-0.4875988960,0.2367156595,-0.4076660872,1.0004101992,0.3512436152,1.2557044029,-0.5128421783,-0.4611277282,0.1007170677,0.0925083235,-2.1999547482,0.1508994251,0.6993888617,0.7741886973,-1.5526758432,0.7084724307,0.1943209916,-0.4375484288,0.7226394415,0.1368113905,0.0707129464,-0.0306051858,-0.0026378226,-0.1765782833,-0.5268459916,1.9544335604,0.2153434902,-0.5700322390,3.2406566143,-0.1734665334,0.2326574624,0.2949871421,-0.8500581980,0.4187697768,-1.0791419744,0.8719388843,-1.4023494720,0.4674481452,0.4940485954,-0.4561758637,-0.2247166485,0.7698090076,1.1066019535,0.9135959744,1.8430663347,0.3033389449,-1.0240417719,-0.5697768927,-0.5450455546,-0.7129164934,-2.1384212971,-0.4906723499,1.0716369152,1.0448619127,1.1970314980,0.0851596743,-0.7071413994,0.4134632349,0.5083128214,0.4324565530,0.4515452981,-0.4726723731,-0.4617720246,-0.0498823039,0.1511984169,-0.1242134199],[1.7733763456,0.6548894644,-0.9969044924,-1.0999536514,-0.4729947448,-0.8252610564,-0.3246595860,-0.2494323254,0.8223640919,0.5732952356,1.1071861982,-0.9319651723,0.3545930386,1.3546044827,0.3564473987,0.8952826262,-0.0207289252,-1.9850325584,1.6648896933,1.0103242397,-0.8624767661,-0.5156413913,-0.5986755490,-0.5133580565,2.7740614414,-0.0843275264,0.5370041132,0.0354410969,-0.4008655846,1.6665250063,0.0965174660,-0.5183625817,-0.9195118546,0.3387037218,1.1137673855,-1.5738148689,1.6057540178,1.3714587688,-2.3321993351,0.3463236392,0.7981258631,1.7536504269,0.0942437500,-0.3396389484,-1.2540856600,0.7180249095,-1.1979964972,0.5125359297,-0.8158826828,0.1937242448,0.5850766301,-0.2552036047,-1.0627428293,-0.0171058923,-0.3047343791,-0.5120671988,0.6329017282,-1.2476128340,0.6401041746,-0.8136311769,-0.0626717582,-1.4570958614,0.2782136798,-0.8262432218,-0.5279885530,-0.2686576247,0.0438584201,0.6822136641,-1.4326016903,0.5378295779,-0.7443700433,2.0603389740,-1.1167420149,0.3034130335,-1.0690333843,-1.0751850605,0.0629330426,-1.8225570917,-1.5243411064,1.2396709919,1.9703440666,-0.4888984561,0.4765799642,-0.3094685376,-0.9791085124,-0.1147989929,-1.6355508566,0.4577118158,0.2286115885,-0.1039451063,0.7752788663,-0.3030622900,-0.0726639628,0.2677658200,-1.6213132143,-0.4331697822,0.1601847857,-0.7287859917,0.3674771488,-0.1830540895,-0.8931375146,0.4875233173,0.9322209358,-0.1905431449,3.1566889286,0.3486742377,1.0539091825,0.4353290200,0.6332775354,-0.7265618443,-0.7866077423,2.5195691586,-0.0017073880,-0.6077119708,1.2275959253,0.7180952430,-1.3357245922,-1.0510390997,-0.7974286675,-0.4727028608,1.5297054052,-0.4044028521,-0.7877702117,1.6482814550,0.8431814909,0.2029867172,-0.6682254076,1.6193115711,-0.5954051614,-0.3906408250,1.9035891294,0.6554541588,-2.8776378632,0.2603771389,0.7601162195,0.5754011869,0.7990735173,1.1752868891,0.4685513079,0.9743111730,-0.2307738513,-0.0264357254,0.4110282660,-0.3004532456,0.2818160057,-0.5710403919,1.0028342009,0.5024909973,0.4595133066,1.4817398787,0.2799629867,0.5637354851,-0.6241150498,0.8133111000,-0.0602380522,1.1828916073,-0.1272625327,-0.8460607529,0.3959697485,1.4310905933,-1.0955809355,-0.2613841593,-0.1580035388,0.3657860160,1.2321866751,-0.6503368020,0.0420608893,-0.8093909621,0.5546837449,0.2265381217,0.3556352556,0.7656990290,-0.3249478340,-0.1872305423,-0.8028757572,1.8795111179,1.1952743530,0.8303111196,-0.1781547517,0.5080417991,-0.1168814152,1.1812840700,1.0313570499,-1.7196556330,0.2446504086,0.1371421814,0.0973915532,-0.4585001469,-0.8280373812,-0.4593352675,-0.6757261753,-0.0793135092,-0.4180409610,1.2445529699,0.2011100203,0.9248659015,-0.4210970700,-0.8320764899,-0.1646859646,-2.5782234669,1.1942738295,1.5204107761,1.2211294174,0.7107045054,-0.1187882051,-0.8583485484,-1.0267019272,0.5830163360,-0.1584341824,0.6857743859,-1.6815145016,0.2011470497,0.1775249690,0.4979387820,0.8372054100,0.5836708546,-0.1584636122,0.2616901696,0.5922822356,1.0962525606,-0.2246936560,0.3583438396,0.5079059005,2.4989459515],[0.7612602115,-0.6034088135,-0.6274229288,-0.0049325931,0.4828433096,-0.1417066455,0.5503855348,0.9566833377,-0.4469841719,-1.8436908722,0.0132681746,0.6910766363,-0.9844883680,0.6257191896,-0.0121382996,0.9449425936,0.4866577685,-0.1009661183,-0.1784197092,0.7964317203,0.1096719578,-1.0981303453,0.1085784137,-0.2215659022,0.6406838894,-0.7363584042,-0.1355459243,0.1792862862,-1.5051630735,-3.0252084732,2.3134453297,-0.1067946255,-0.2164448351,0.5656723976,0.1987833977,-0.4938872755,0.3542607427,0.7917511463,2.0004119873,0.1138286367,1.0661466122,-1.2925044298,0.0914441869,0.6103685498,-0.9954180121,-0.4100271165,-0.4298735261,1.2885445356,-1.7387659550,-0.5108031631,0.1936938167,-0.2972007990,-0.5979477763,1.5313448906,0.0219627060,-0.7502336502,3.1279857159,-2.1839592457,1.0440800190,-0.6610156298,-1.2552039623,0.4473415315,-0.9450294375,-0.2153885067,1.1292688847,1.4167747498,0.0083199078,-0.7387729883,-1.5310351849,-1.2331342697,-0.0405933820,-0.4422261715,-1.0382130146,1.3626192808,1.1903202534,-0.1882884353,-1.8005816936,1.5561032295,3.1526033878,0.4462724328,-0.5650293827,0.2617058754,-1.4923347235,0.1959274411,1.6606786251,2.1215653419,0.4635251164,-1.6947836876,0.5675776005,-0.0769856125,0.8061911464,-0.7121078372,0.3398130536,0.2747018933,-0.9738521576,-1.0838885307,1.6086019278,-0.1070212498,-2.4784710407,1.2567377090,-0.9551062584,0.8460628986,-1.0296751261,-0.3630988300,0.4910036623,1.3033028841,0.6093656421,-1.5684084892,0.2002652586,1.8276238441,0.1794807613,0.7145086527,0.4715159535,-1.3110510111,0.3708418012,0.5671648383,-0.1211298183,-1.9422543049,2.7631845474,-0.0019290138,-0.9331898093,0.2469065487,0.9991958737,0.1478086561,0.5288000107,-0.2993927896,0.6493500471,-0.5947993398,0.0689935088,0.3103188276,-0.5490578413,0.5102764964,1.0045766830,0.4634752870,0.3118042052,-0.1784106791,-1.2324748039,-0.2715279460,1.3155361414,0.6285444498,-0.7697691917,0.3814707100,-0.7372010946,1.6317893267,0.0496188216,1.8298569918,0.2486533225,0.2665860057,-0.5764042139,-0.1545846164,0.5823374987,-1.2349112034,0.9288624525,0.2240541875,-0.8932236433,0.6563537121,-1.9804890156,0.2280142903,-0.9330589771,0.8488634229,-0.0275313407,0.7628030181,-0.5166394114,-0.0639637858,0.5297335982,0.0976115614,-1.2929050922,-0.2473904490,0.0017580237,-0.0445541181,-1.2251801491,-0.9520522356,0.3250408471,-0.9763738513,-0.3191157579,0.0639646500,0.3773847818,-1.7942626476,-0.3388437927,-0.7155359983,-0.9516233802,-0.8122008443,-1.0962382555,0.8137555122,-1.6893570423,-0.7392467260,-1.0180219412,0.3281373978,0.8430277109,0.6054344177,0.9184139967,-0.4063994288,0.0855568647,-0.1104435921,-0.7943481803,0.2338262349,1.0993648767,-1.3479268551,0.3901562393,-0.3676171601,-0.5882406831,-1.7047501802,0.0156326983,0.3451995254,0.1704274416,-0.5342517495,0.1036691368,1.9168593884,-1.0093951225,-0.2074879557,-0.1844485551,-1.1339180470,-0.7229844928,0.3331287503,0.3014786243,1.4839015007,-0.2690161169,1.0232335329,0.7889760733,-0.4552712739,0.3501521647,-0.2285785526,0.9389357567,0.0447394811],[0.1851441115,-1.2355756760,1.2031196356,0.0116550243,0.9264087677,0.3385564387,-0.5845701694,-0.6950528026,1.0220985413,-0.7311565280,-0.9815580845,0.8116582036,-2.7727127075,0.1836193055,-0.8430696726,-0.9484882355,0.3477781117,-1.2728915215,-0.9775258899,2.0550086498,-1.1804845333,-1.6777393818,-0.2583949864,1.9016064405,-0.2666935325,-1.0596487522,1.1103088856,-0.5217511654,-0.1671878248,-1.3836013079,0.5602372289,-0.7295765281,-0.0698853359,-0.5959703326,-0.6792480946,-1.0985150337,0.4957344830,1.3550078869,-0.3935103118,1.1229667664,0.8153097034,-0.6519557238,-0.4303318560,-2.8154788017,-0.3087665737,0.7753482461,0.6616540551,1.1300858259,0.6900553107,-1.8309692144,-1.0626448393,-0.7698283195,-0.8680559993,-0.7421313524,1.8048762083,-0.0947011113,0.4935582280,-0.1392302960,0.1740383208,0.1270922571,-1.5548484325,-0.0411530435,-0.6951826811,0.3269871473,-1.4882113934,-0.3722811639,-0.3547021151,0.1275018305,-0.4833881557,-1.8390951157,0.1966317892,0.5947086215,-0.0949502289,0.2317803800,-0.9899123907,-1.4969880581,1.9577594995,-1.1843186617,1.5552152395,0.5649688244,-1.7155886889,-0.6562070251,-0.8054563999,-1.6628246307,0.7222976089,1.0663632154,-0.1094901860,-1.5984297991,-0.5933257937,-0.0686090812,1.2584441900,-0.3986067474,-0.4922445416,0.2149075568,1.6925339699,0.4467121065,0.0721166655,-0.1689761132,-0.4769457579,0.1210901141,-0.1196763217,-0.3491147757,-0.6679611206,-1.1081523895,0.8642325997,-0.3652341068,-1.4192044735,0.5283550024,-0.4291144013,-3.9799249172,2.5193560123,-0.0001015828,-0.3741996288,1.1290762424,1.6821254492,1.0898627043,0.0934686586,0.3777738214,-0.5232481956,-0.5119860172,0.8392738700,-1.3982032537,-0.5136648417,0.2465056181,1.1420283318,-0.4828293025,1.1170063019,0.2153869420,-1.0433691740,0.7852531672,-0.3490794599,-0.5112475753,1.0232989788,-0.0248605367,0.6658914089,-0.0275805313,-0.3348940313,1.8200131655,-1.0081651211,-0.2610522807,1.3033430576,-0.4144646525,-0.0354455411,0.8998509049,0.5341755748,-0.4204404354,0.2261348367,0.2233097106,-1.1665488482,-0.5937159657,-1.4060828686,0.4686772823,0.0984703451,-0.7751576304,0.3141351938,0.8974028826,-0.1393257678,0.7376810908,-0.3796384633,0.7480918169,0.2854343355,0.4962151349,0.0067210966,1.3005772829,1.0714277029,0.6444802284,-0.5154283047,0.0870009586,1.0096105337,-1.8330725431,0.1132716388,-0.0708507001,0.1738734245,-1.0237518549,0.2748146057,-0.2124217898,-0.1312001646,-0.7777321935,-0.8523564339,-0.5532600880,-0.4056203365,2.1330664158,0.2694247961,0.0929921642,-1.1530809402,0.5591195822,0.1892250180,0.6476491690,0.0823355317,0.4193497598,-0.6377058029,0.2406051308,-0.6672044992,2.6748664379,-0.5676947236,-0.3393431604,-0.5449929833,0.2807078958,0.9032731056,-0.6467321515,0.8295060992,-1.8630459309,0.4991222322,0.4042034745,-0.0883236080,-0.0470565073,0.9142104983,1.0030266047,0.2447029352,0.6141192913,-0.8892118335,-0.7647240758,-1.1047742367,-0.8095286489,-2.0631930828,0.2100882679,0.2161284685,0.1404330134,-1.7657244205,0.2368792892,-0.1763023734,-0.4331485033,0.1648295075,1.7212885618],[-1.1766096354,0.1771905273,0.0255289283,0.0521093681,0.2013487816,0.8420627713,0.9915324450,1.5613195896,0.6682843566,-1.4083592892,-1.2373524904,-1.3657908440,0.5665727258,0.1114991754,0.5536442399,1.0835049152,-0.0504580699,0.2503991425,1.9299334288,-1.5645395517,0.4178295732,0.1182594225,-0.2803184986,-0.0412648320,0.1382355541,0.7969234586,-0.1608946025,-1.3431527615,0.2850189209,0.2608601153,1.9069812298,-0.3040925264,-1.0234804153,0.8924683332,0.9798862338,0.7105198503,-0.5560311675,-1.9237110615,-1.5857894421,-0.0867523253,0.7510196567,-1.6821100712,1.7742727995,-1.1292693615,1.2592233419,-0.5400198102,0.4569769502,-0.1657794416,0.5541830659,1.0131599903,1.1774101257,0.3166932166,1.1890761852,0.4440013170,1.7804197073,-0.8230453730,0.0510413684,-0.8502710462,0.2226673365,-0.8931104541,0.7461739779,0.5200843215,2.4808294773,-0.4306486249,0.7000415325,-0.6627852321,-0.6666034460,0.6417964697,-1.3115973473,-0.4170653224,-0.0155315036,0.2604958713,-1.9038473368,0.8179932237,-0.8048866987,0.1132161394,0.6200553179,0.8735640645,-0.8754689097,-2.9979059696,1.8664747477,0.2899555564,-0.3376359046,1.4218786955,0.0991161391,1.5541207790,-0.0542410761,1.2513942719,-1.3293449879,-1.5720999241,0.2153496891,0.4149667025,-0.3295794129,-0.2767211795,-0.0946821421,1.7693696022,0.0545876808,-0.6418469548,1.1964954138,0.4129462242,0.3027030230,2.1559007168,-0.6829319596,-0.3651222289,-0.0442247801,2.2708928585,-0.7823866010,0.4403840601,0.1520252228,-1.2476013899,0.5838144422,-0.5700789690,0.0327484533,-0.8868691325,2.3026103973,-0.3992435634,0.4614451826,-1.6609683037,-2.1689975262,-0.7229250073,2.8979327679,0.7089292407,-2.3434951305,0.1656176299,0.5335359573,0.0808340311,0.3161955476,-1.3906550407,-0.6597616076,-0.3027254045,-0.3078902364,-0.3183090985,0.2484171093,-1.8363064528,0.1134862378,0.9222687483,1.4002178907,-0.6588960290,-0.5069347620,-0.1029141247,0.0792785361,0.8484690189,0.5411299467,-0.2156495005,-1.4614349604,-0.0294835921,-0.0679289103,1.7664188147,-0.3468401432,-0.9617422819,-1.0435235500,1.4455068111,0.4401999414,0.0200419500,0.0252560489,1.3682826757,-0.7683274150,-0.8309261799,-1.9147306681,0.1773295254,-0.1637866497,0.8641271591,-1.4150470495,1.3570704460,-1.2145545483,0.1236755922,-0.0837656856,-2.5834338665,0.3076185882,0.2000334412,-0.2788176537,-1.8437790871,-0.7380565405,0.6632315516,2.2895617485,0.6810886264,-0.4265203774,1.3332300186,0.9933365583,1.0180410147,-2.1297695637,2.2985951900,-1.2535015345,-0.7859506607,0.3115945160,-1.4633288383,-0.2197153121,0.1873781085,-1.5290776491,-2.2650036812,0.4986795783,-0.4243268967,-0.4248904884,1.2769355774,-0.3564965427,1.3732653856,-1.4011700153,-1.5260056257,0.6965202093,0.1845299453,-0.7892647982,-0.1872230172,0.4882991016,0.1650603563,-1.6113953590,0.2592717409,-0.1188143864,1.2228955030,-0.8016638756,0.6876689196,0.7840186357,-1.5337510109,-1.1500921249,1.2201095819,0.5806935430,-0.8960040808,0.8627079129,-1.1348323822,-0.9670757055,0.3248357773,-0.4929454625,0.1369240880,-0.5122059584,-1.0240994692],[0.8177993298,-1.5714014769,-0.9127979279,0.3507647216,0.5462158918,0.0155025600,0.2486607134,0.4373335540,-0.0667797402,0.3301390707,0.5512520075,-0.0004508996,0.2623131573,0.8791959882,-1.3433330059,0.0341191590,-0.1302566528,-0.6102820635,-1.9780943394,1.0417385101,-0.1486586779,1.3084896803,-0.1083629355,0.2558193207,0.6630388498,-0.7649769783,0.0660102144,0.6541601419,0.1619965583,-0.2343661040,1.1309125423,0.9972749352,0.3769946396,0.9053672552,0.0170451328,-0.2916249335,-0.0637479946,-0.8857896924,1.0207086802,-1.3605722189,1.5603965521,0.4978734851,-0.0114495857,0.2822905183,-0.0966921449,0.4231345654,-1.5016077757,0.5884187222,-0.8756873012,0.5964536071,1.5745950937,1.0954363346,-0.8237411976,-0.2152891606,-0.2709789574,-0.2081480324,2.2277188301,0.3763382137,1.2043631077,1.1044359207,-1.3858469725,1.2034906149,0.0602937639,1.5043460131,-0.4531169236,-1.3994597197,0.9959958196,1.4748657942,-0.6896502376,0.5163658857,-2.1955311298,-0.7591708302,-2.2448000908,0.4303465486,-1.8543956280,1.4948334694,-0.3116846383,0.4466024935,-0.0118529536,0.7244918942,-1.6666995287,0.0236608721,-0.4630951583,-0.5484100580,-1.1173944473,0.4898535311,1.6996483803,-1.8060977459,-0.4497342110,0.4933337867,0.0872113928,1.6383810043,0.2239727825,-1.2708234787,0.3466352224,-0.4335897863,-2.5197424889,0.7378028631,-0.8447279334,-0.0559481047,0.6839315891,-1.1025488377,0.2024087608,2.9150803089,-0.3840768039,-0.0326391011,-0.0803115442,-0.8246007562,0.3065483272,0.0389659554,0.8100610971,-0.3687470257,1.0887268782,0.0036491603,-1.0394093990,-1.0970540047,-0.7277394533,0.2685888410,0.7355921268,0.9661358595,0.5190473795,0.0508830100,1.2438833714,-2.2499885559,0.8982744217,-1.2954920530,0.8556146026,-0.4744929671,-0.6942690611,-0.1644901037,-0.2620332837,-0.1451232731,0.0785102472,1.0016654730,-0.2739449739,0.8009263277,-0.1702002585,-0.8886307478,-0.1733903736,-1.1252379417,-0.4982013106,-0.5461481214,-0.7811477184,0.1073800698,-0.7631527781,-1.3092365265,-0.1617819816,-0.0008865069,0.5713884830,-0.1136347950,0.1617997438,0.1273338348,-0.7498638630,0.8821802735,-0.0337238349,1.3859173059,0.5330554247,0.1874614954,1.6891919374,2.0251679420,-0.4745604992,0.0918694958,0.1943952888,0.6713520288,-2.0351748466,2.6817276478,-0.1689636558,-1.2043520212,-0.8034988046,-0.6250877380,1.4457765818,-1.4667338133,-0.8495544195,-0.3539527953,-1.0773371458,-2.4111745358,0.5679433942,-0.2081842870,0.9879924655,1.2360608578,0.7949596047,0.4199788272,-0.6940913200,-0.8832488060,0.5228080153,-0.2757284045,0.2805615962,-0.4129777253,-1.1507169008,1.8317346573,0.1299459189,-1.3232452869,-0.2197397053,1.6015273333,-0.7579594851,0.5799200535,1.2151949406,-0.2547016442,0.1068254113,0.0516264960,1.1671109200,1.3634036779,0.0900574327,-1.0950243473,0.8401091695,0.6654623151,-0.3086810708,-0.7375400066,-1.0307489634,0.1048286334,-1.5460332632,1.8186627626,0.8098951578,0.9708928466,0.7102554440,0.9713087678,0.3875834644,-1.1385778189,0.5401791930,0.4986961782,0.2263549715,-0.2634185255,-1.5598323345,0.3776816428],[-1.7435592413,-0.6064981818,-1.0359470844,0.7043692470,-0.5080588460,-1.9649239779,1.1656636000,-0.8278691173,-0.3729766309,-1.5564032793,-0.8267266154,-1.1934009790,0.6669597030,-1.0290228128,1.0644114017,2.3462307453,-0.2566364408,1.2686136961,0.2236939818,-1.1905844212,-0.8348674774,1.0824284554,-0.8072878718,-2.0340442657,-0.1297576725,0.1229395643,0.4321765304,-2.7568418980,-0.5900364518,-0.2194618881,0.9851743579,0.5149931312,0.7409499884,0.9859756827,-0.7990174294,-0.9565790892,-0.6254098415,-0.5258603692,-1.1526063681,0.5984534025,1.7542530298,0.5825209618,-0.0108615747,1.1506799459,1.6939212084,0.6322537065,-1.0341697931,-1.4992163181,-0.9192206264,0.2501222789,-1.6166472435,-0.3118333817,-1.5592268705,1.2363440990,-0.2899449468,0.2497586608,0.8226853609,0.0091154482,0.5934258699,-1.0807846785,0.6175705194,-0.2989517748,0.0668154061,-0.3421589136,-1.2376976013,0.2341058552,-0.2281744033,0.9363278747,0.7818625569,-0.5937199593,-0.2155658305,-0.2269501239,0.0548860207,1.2610039711,-1.1864619255,0.2939851582,-0.1575176418,0.3959463537,-0.5379269719,0.0852746218,-0.1205217764,1.1457941532,-0.6422573924,0.6109707355,-2.3347873688,0.8817616105,-0.4318682849,-0.2186090648,2.3368148804,-0.8696759343,-1.3311178684,-1.4940352440,-0.1126572788,-0.1654232144,0.6756837964,1.0262820721,-1.5286720991,-1.7455735207,-0.0086124055,-1.3048062325,0.7480476499,-0.1624615192,0.8573369384,-0.2427486479,1.5718251467,-0.4005960226,0.1661429852,-0.3388045132,-0.8788685203,1.1277475357,-1.8028851748,-0.9683129191,-0.2205161005,-0.3632950187,0.9993163347,-1.1250728369,0.2273952663,-1.6806476116,-0.5850839615,0.4633552432,0.5404006243,-1.1591656208,-0.4043771923,-0.5775872469,-0.4467805326,-1.3486293554,0.1647246778,0.5355979800,0.6689383984,1.6987106800,1.0737128258,1.5600349903,0.5412341952,-0.2705770433,-0.5515835881,0.4355035424,-0.1320427060,-1.2384092808,-0.3141611516,1.3099900484,0.6195878386,-0.0317559093,0.4572177231,-1.0924277306,1.0926088095,-0.3314957321,1.8907419443,-0.4943482280,-0.0626017973,-0.0550512932,-0.0231284834,0.7892349958,0.3658465743,0.4307425022,0.1753020287,0.4385994077,0.9858914018,-0.7419704795,1.7252422571,-1.0884103775,0.9333866835,-0.4368148446,-0.6445410848,-0.7094064951,0.5206435919,-1.9676936865,-2.0886249542,-0.0491473936,1.7515029907,-1.2138712406,-1.0226154327,-0.5423930287,0.7038816810,-0.8431472182,0.7065833211,0.2752345502,0.0155479768,0.4675925374,0.9834657311,1.3818086386,-0.0449825265,-0.8379833102,0.6575056911,-0.9417086840,-0.9865073562,-0.8108641505,0.4895179570,-0.2885791063,0.6517304778,-0.0437750444,0.9180089235,1.0069895983,0.5761268735,0.4679514468,0.0979376882,-0.6210292578,0.2446375191,-0.2866552174,-0.4697014689,-0.6670892835,-0.2212103456,0.1083477437,-1.3208975792,0.2513886690,-0.6949612498,1.3961971998,1.0441313982,-0.8038179278,-0.6545550227,1.0894525051,-0.8547613025,0.4289109409,-0.9605471492,-1.3116114140,-1.4649935961,2.2020881176,-0.7328217626,0.2469987720,-1.4316263199,1.0094193220,-0.5877754092,-1.2018294334,-0.4497393668,-2.0317924023],[-1.3781358004,0.2250173539,-0.2393929213,1.0711579323,1.0380489826,-1.6511549950,-0.3789921403,0.9080212116,0.8696871400,-1.6957534552,0.3965432644,0.0837741494,0.0010892110,-2.4933798313,2.0837154388,-0.2696723044,0.7178445458,-1.4326989651,-1.0441292524,0.9024427533,-0.2075345218,-1.0056159496,0.8656446338,0.0663467273,-1.0112538338,0.4815161228,0.6747781038,1.7164748907,0.0540352389,0.0186312608,0.6076316833,0.5149070024,-0.6382617354,-0.2592832446,-0.1123041362,-2.3297955990,-1.2326625586,0.1074425578,-0.0259655807,-0.4965038002,0.4682292342,-0.7256876826,2.5770611763,-0.5546184778,-0.7165570259,-0.5988902450,1.3805335760,-0.9438036084,1.8827584982,2.2911372185,-0.3109315038,-0.4171719849,-0.9282898903,-1.2628384829,-0.8774039149,-0.1966391057,0.1648458242,-0.0454749390,-0.4523634613,-0.8597000837,-1.8105161190,-0.5551453233,-0.4640676081,0.0456892885,0.9407829046,-0.4700866640,-0.4446507394,-0.9410798550,0.1821057349,-0.8256917000,-0.5830290914,1.5628407001,-0.1577423513,0.3348858356,-1.0220993757,1.5164207220,-0.4169793427,0.0705198646,-0.9684671760,-0.7755367756,0.8577855229,0.7357011437,0.8049449325,-0.5722336173,2.4884157181,-1.0409582853,-0.5645226836,-0.8657156825,0.8756935000,-0.0816876218,1.6169995070,-0.4836786091,-0.5187353492,-0.1280231029,0.3786218464,-1.0286502838,0.9747040272,-0.4097430408,-0.0414662100,0.1704852134,1.2174123526,-0.4135402739,-0.9620710015,-0.6299701929,0.7686001062,1.6596417427,0.1165930480,1.5519549847,-0.0122310994,0.6999790072,-0.3696699142,-0.6388676167,-0.3132658899,-0.3769345582,0.4913886487,0.9003452063,-0.6527366638,-0.9828684926,0.0031222911,0.3023118973,1.1393951178,0.0686765835,0.8947140574,0.1511452794,1.2494877577,0.2830680609,1.0252991915,-2.0512757301,1.0091516972,0.8578301668,-0.4312323630,-0.0823090598,-0.6327573657,-0.6221244335,0.1637613624,-0.4898373783,-1.0175414085,-0.1421252489,0.7623457909,0.6941013932,0.8787897229,1.0183321238,-1.3048707247,0.6710067391,-0.5081533790,0.7300412655,-1.0180243254,-0.6606916189,1.6511225700,-0.3809156120,-0.8625769019,0.1733328998,-0.3080062270,-0.4351976812,0.6185950637,0.1875175387,0.2302682847,1.6791144609,0.6098973155,-1.7957258224,-1.9296709299,0.4632092416,0.9872512817,-0.2040626854,-1.6726781130,-0.7090438008,0.1458017230,-1.1318336725,-0.2968883812,-2.6211783886,0.1238219291,-0.3862033188,0.1050975546,0.4096537530,0.3024674058,-0.8510958552,0.7929092646,-0.2956811488,-0.1781456769,-0.4592222571,0.6618961692,0.7615174055,-0.3307532370,0.0029948372,1.0480338335,-0.9730817080,-0.7652286887,0.1419316083,0.9494506717,-0.6568839550,-1.5343198776,0.7911569476,-0.0277832802,-0.0588698722,-0.9085233808,0.0808649138,-0.0074584479,0.2145736217,-0.0665548891,-0.3323315680,0.0640165284,-0.9991938472,1.8833183050,-1.0895928144,0.8137967587,-0.4096502066,-0.2939639390,-1.3181260824,-0.7032576203,-0.8461838961,-0.0827098563,-1.9362289906,-0.3094807267,0.0122073861,-0.4892766178,0.8326879740,0.8380709887,-0.5376721621,-1.5581637621,0.4174664915,-1.3119213581,-1.4869244099,0.8268741965,-0.0097436709],[-0.6998175979,-0.6954088807,-0.2588140666,1.1530423164,-0.8297799826,1.8891391754,-0.7696596980,0.0943683162,-0.0774358884,0.7215057015,-0.5595492125,1.3661303520,0.7425178885,-0.0379569791,-0.5930621624,0.3185641766,1.4783425331,-1.1458002329,-0.9837580323,1.0978168249,0.8367465734,0.6999016404,0.6078990102,0.6545804143,-0.9899599552,0.6728334427,-1.1816710234,-0.8178920746,1.1061612368,0.3675162494,-0.5634465814,-0.2326633036,-0.6539015174,0.4468860626,-0.9113517404,-0.3453916311,0.1714687198,0.2547176480,-0.2261466980,-1.6652747393,1.0987795591,0.7416546345,-0.5794039369,1.2014195919,-0.4735201597,0.7171515822,-0.3532316089,0.5127348900,2.0128924847,0.9054009914,0.0433341525,-0.2319250405,1.2205764055,0.7686790824,1.4405000210,-0.3859676123,0.6124944091,-0.7336784005,-1.2352366447,0.7807996273,-2.3257813454,-0.3928811252,-0.2172492445,0.6651454568,1.7893996239,-0.3957181275,1.1372895241,0.7369244099,-0.8327875733,0.6632367373,-0.0397752263,1.4086065292,0.4437205195,1.4935532808,0.0716269463,-1.0937371254,0.4778392613,-0.4515686929,-1.5649410486,-0.0638999194,-0.4206497371,-0.5077050924,1.3147307634,0.2272905707,0.0551176965,0.3449913859,-0.6504779458,-0.0079302043,-0.6291233301,1.4332242012,1.2907489538,0.3631163239,0.5036269426,0.5870404243,-1.5838334560,1.8516979218,-0.1510378718,0.0084957005,1.7752424479,0.3407488465,-1.7290748358,-0.1309578121,0.1811762899,0.9766051173,0.8586809039,-1.4115585089,-0.5120242238,-0.9084070921,-0.6975479126,0.2765782475,0.5603165030,0.8318665028,0.1136660054,-1.2513357401,0.3720861077,0.6408976912,1.5998200178,1.0436722040,-0.3570035696,-1.7902721167,0.0918675587,-0.1112808436,0.3380652964,0.3301384747,-0.6644927263,0.6445029974,-0.6061426997,1.9779821634,0.1567000747,0.1088435352,-0.8300416470,0.1776271611,-0.3211027384,-1.1594855785,-0.4041348696,1.4324586391,0.2450418323,1.3597365618,-0.2198139727,0.7452784181,0.5204197764,-0.1552402973,-0.0877770483,0.6449214816,-2.0652017593,-0.4502398968,1.5628504753,-0.2074006945,0.5728432536,0.5312701464,-0.5582386255,0.4903011024,0.4000514448,-0.3402361572,0.1078620851,0.1054711565,0.0762601420,-0.8976479173,-2.7556171417,0.6805824637,-0.8755759597,0.5861801505,1.0099159479,-2.0829658508,0.8546541333,0.4041488171,1.4322775602,1.9776724577,2.1114580631,0.2207719088,1.9176034927,0.9244675040,1.6649698019,0.2163754702,0.0035872904,2.6416580677,-1.4437009096,0.4020408988,-1.1299657822,2.0063018799,0.3601942062,0.0552108772,-1.6590025425,-0.6537028551,0.6874204278,1.6122521162,0.2168688625,0.9429037571,0.5354552865,-0.9631315470,0.9434006214,1.5907050371,0.2731074691,0.9513245225,-0.2791555226,1.1296684742,0.7820475101,0.1685398072,-1.9420115948,-0.9730284214,1.1653062105,-1.4316834211,-1.3316079378,-0.0116488738,-0.1106063351,-0.3327346146,-0.1284098923,-0.4964047372,-0.3758531809,0.4843265712,1.5204824209,1.1139789820,-0.3578812182,1.8609610796,0.9948360920,-1.3479914665,0.8106604218,-1.0499941111,-0.6323813796,-1.4502238035,-0.7411080599,-1.1738317013,1.0985153913,-1.7316554785],[0.5024092197,0.6399983764,1.8322696686,0.2193556577,-0.2059195042,1.5266751051,0.8948894143,1.6918249130,0.5572015643,-1.3783254623,1.2419376373,-0.1381992251,-0.3036934733,-0.4471315444,1.3595137596,0.5195784569,1.0076969862,-1.0152332783,0.4372532666,-2.2233533859,0.3535255194,1.5505034924,-0.5713678002,0.0189786758,1.5610871315,0.6443794966,0.3136890829,2.5183229446,-0.1112032235,0.4580637813,0.4732072651,-0.6347094178,-0.1455913633,-0.3562349081,-0.0463523492,-0.2493555695,0.7978110313,1.3423910141,0.2120787650,-0.9567505717,1.3303672075,-0.3680039048,0.7403077483,-0.2130557746,-1.3168724775,0.9083569646,-0.4936971068,1.0476306677,0.5149489641,0.5401264429,-0.3668811321,0.5347924232,0.4814083576,-1.0775488615,0.9366270900,-0.3202053010,-0.3182254136,-1.4235715866,2.0460758209,-0.6138587594,1.4505295753,-0.0747881457,-0.8538289666,0.0534842983,-0.2949616313,0.3182134926,2.0032804012,0.2364036888,0.9947002530,-0.5805375576,1.5063704252,-0.9826948047,-1.6039320230,-0.7312178016,0.2631585598,0.0081778783,-0.2120543271,1.4288504124,0.0310508497,-0.4847147167,-0.3707052171,-0.1414912492,-0.1123282909,0.5229693055,-2.4375815392,-1.9124066830,0.1451980472,-2.0455174446,-0.4165692627,1.6499553919,0.0982848108,0.9086429477,1.1524801254,1.1450995207,-0.8715981841,0.2962348163,-0.0864251330,-0.7070257068,-0.0059363027,-0.0952185914,0.9882470965,-0.2431317419,1.0272905827,-2.2877068520,-0.0339677818,0.2500078678,-1.6668512821,0.3891637921,0.5523136258,-1.7448424101,0.6041775346,1.7510638237,-0.5160238147,0.7353807092,-1.2460041046,0.0510762073,-0.1354765445,-0.8791379929,-1.5496599674,0.1134271175,-0.2463372648,-0.0203294978,1.5303599834,-1.0988429785,0.3622785807,-0.7890076041,1.4901959896,-0.2522631288,0.6070600748,1.0430524349,-0.1125693843,-0.4349509776,0.7843998075,0.6623550057,0.0715067312,-0.2036272436,0.0255890638,1.2453645468,0.1566366106,-0.4086671472,0.8515979648,0.4264606535,0.0430518128,0.1734429896,-1.9741030931,0.6002647281,-1.0860610008,-0.4001491666,0.4027245641,-0.4328278005,-0.6476949453,-0.0458221361,0.4308312535,0.8188905120,2.1767871380,0.5373866558,-0.9187107086,1.3414472342,-0.8143109679,-0.4182454348,-0.9356334209,1.1228146553,1.3687461615,-1.2463581562,0.5486357808,0.3156320751,2.7218301296,0.4282638729,-1.6033006907,-0.2760243118,1.5372369289,0.3407447040,0.8939761519,0.2236253321,-1.2835608721,0.5933554173,-0.8735864162,-0.1540386975,1.7338123322,-0.2981918752,0.6927968860,-0.5642343163,-1.1485286951,0.7087337375,0.7157472372,0.1084082723,-1.2599521875,-0.1718678474,-0.1303246766,-1.6611258984,-0.1089953035,-0.8575206399,1.2285900116,-0.5913127661,-0.8356810808,-0.8497319818,1.6891064644,0.9818716645,-0.7118170857,-0.0595040582,1.1489208937,-0.1327075511,-0.7262676358,-0.6392194629,-2.7723453045,-1.0678972006,1.4211807251,-0.5049116611,1.0029976368,-1.7353630066,-0.0004422014,1.3126903772,2.6282687187,-0.8856875300,1.2361621857,1.1301437616,-0.7847929597,-1.4580696821,-1.9722865820,-0.3570451140,1.5762296915,0.6130509377,-0.8741570711,1.8246018887],[-0.4657349586,-1.2615884542,-0.4083859324,0.3294874430,1.1362862587,1.5787801743,-1.5691125393,1.1296247244,-0.8173444867,0.0448391438,-0.2004632354,-0.1206745356,2.1760234833,1.4095352888,1.7142325640,0.1296830177,-0.0240912717,0.1981743574,-0.5931743383,-0.6481423378,0.2112238854,0.0259035602,-0.7714331150,0.4130635262,0.4231062829,-0.1897049546,-0.2473408580,-0.5636284947,-1.4844306707,-0.5463525057,1.6333186626,-0.4062886238,0.6718820930,-0.1990920752,-0.0755890235,-2.4272682667,-1.0069460869,0.2937736213,-1.8922125101,0.0387951620,3.0649490356,-0.2021356225,-1.7997900248,0.1907552779,0.2254720032,-1.4364736080,1.7227665186,-1.0507551432,-0.3714186549,0.2494156957,0.1630332470,0.5737173557,0.3798969388,0.6762976050,2.1938946247,-1.3434695005,-1.8401567936,-0.4973259270,0.2549467981,-0.4350290895,1.7625454664,0.2511341572,2.0254280567,1.4629654884,-0.2105290741,0.1842921525,-0.8437554836,-0.1074019298,1.3886370659,0.6858979464,-0.7213442922,0.7010446191,0.2083124965,-0.5340302587,-1.8905137777,-0.2197514623,1.8318955898,0.5987322330,0.5050485134,-0.0544060431,-0.5782997012,-0.2975089848,-0.1296807826,-3.1675622463,0.4436289370,-0.8021137118,0.6727771163,-1.4421380758,1.5984972715,0.7499288321,-2.0262806416,-0.2081878334,-0.5387793183,-0.0523785017,1.5313670635,0.7524619102,-1.2980279922,-0.4864748120,-1.9687265158,-1.1095793247,-1.3467646837,0.1210370436,-0.4014209509,-0.3103523254,-0.5346525311,0.2820588350,-0.1820099205,-0.1331241280,0.2891933918,-0.2156425118,-0.4738779366,-0.4227135181,-0.3396925926,1.0411024094,0.5079514980,2.2190010548,0.1058152467,1.4273592234,-1.4032130241,-1.1333082914,0.4115984738,-1.2759627104,-1.7923833132,-0.9866417050,0.0260736253,0.2983654141,-1.0883109570,0.1540415883,0.7682629824,0.1405778527,-1.7093214989,0.8520715237,0.3354863822,1.0088881254,0.9830012321,-0.1587079316,0.2750813067,-0.5337952971,0.5333610773,0.5658074617,0.1133866310,-0.5827506781,-0.1758695394,-2.3300492764,1.3145494461,1.5094611645,-1.1486440897,-0.9897459745,-1.2299499512,0.4605937004,-0.7130784392,0.9297976494,0.1480864882,-0.6089158058,-0.8862819076,-0.0294247605,-2.0403492451,-0.4883415103,-1.0245062113,0.2432617396,-1.2315342426,-0.3076558113,-1.0241663456,0.4212306142,0.1273297518,-0.8465504050,-0.7393798232,0.0760505572,0.3653028309,0.1351646036,-0.0163296815,2.1622841358,1.6225196123,-0.9306966066,0.9292284250,0.3719254434,0.1162487194,-0.0589749292,0.2543700635,-1.6679431200,0.4688638747,1.8470935822,-2.0336110592,0.2948815525,-0.5645554066,-1.2335046530,0.6282685399,-0.3543842137,-0.7103201747,-0.4193950295,-0.7470821142,-0.7791161537,0.9769921899,1.0773099661,0.6259647608,0.8145341873,0.2484787703,-1.5553941727,0.2921783328,-0.8426584005,-1.1621177197,-0.0484674945,0.8237134218,-0.8495957255,1.6625444889,-0.5229098201,1.4123066664,1.4815165997,-0.5415441394,0.0697141588,-1.3264377117,-0.2319623828,1.2776615620,1.8105273247,0.4019567072,-0.3616358936,-1.6378117800,0.3181915879,0.9920549393,0.4044793546,-0.6421396136,-0.7581509352,1.0275832415,1.5470706224],[-0.7398016453,0.6584480405,1.1294622421,-2.4815869331,-0.8624873757,1.3350468874,-2.7070226669,-1.4999361038,0.1586216390,1.8153672218,2.8114330769,1.5783907175,-0.1268175691,0.7761794925,-0.0535948388,1.4350873232,-0.3202698529,-0.6328254342,0.6196979880,0.9736643434,-0.8059627414,-0.0634836704,-0.4779267907,0.8468277454,0.9047194123,-2.0163111687,0.8780838251,-0.4427151680,1.5682283640,-0.0647773743,1.9337058067,-0.0634923875,-0.8278262615,0.1841171682,-1.2062327862,-0.1774312556,-0.3325543106,0.5048562884,0.1616335213,-1.3787573576,0.2442163378,-0.4408593178,-0.1357885301,-0.5383255482,-0.9141966701,-0.1598681062,-0.3700854778,-2.1424248219,-0.7458830476,0.1234179586,0.9672073126,-0.1455941498,0.4572854638,0.9470093250,-0.1653638631,-1.2481099367,0.6318738461,-0.0626924559,0.5399544239,0.8403316736,0.9862011075,-0.3642502129,-0.6327438951,-1.3328721523,0.6068063378,0.8446694613,-0.6332240701,0.5425485373,0.7139165998,-0.0305935647,-1.4514397383,-1.3369790316,0.5513197184,-1.5224503279,0.5453975797,0.8725185394,1.4529868364,0.5624259710,2.0542755127,1.7089699507,0.9274854064,0.4328777492,0.7026924491,-1.3295015097,0.2746844888,0.2033832967,-0.4538957775,0.1762087941,1.7823884487,-1.4706496000,-1.2204672098,1.4729512930,0.1214189082,1.0871387720,-0.3612067699,1.7784131765,0.9309613705,0.2597780526,-1.4110622406,0.2922568917,-0.6586437821,-0.0512520969,-1.3286775351,1.6530493498,-0.7626387477,0.9156857729,1.9149209261,0.2932203114,-3.0808136463,-1.0101010799,0.6143974662,0.2382051498,-0.0467596762,-0.0474947765,0.9090418816,-0.2340825051,0.5424991846,0.4172537029,0.4624036252,0.4270164073,1.6592540741,-1.0672186613,-0.1578507125,0.0752028897,-0.9551435709,0.7953690290,-1.1591863632,0.7421058416,-0.2349259704,0.5437689424,2.1095736027,-0.0088392152,0.8944605589,-1.1591901779,1.0720796585,1.0443638563,-0.4178381264,-1.0055652857,0.2256257236,0.1818790883,0.8913904428,-0.2827850282,0.5382263660,0.5552778840,0.5970648527,-0.3059228361,-1.1920317411,1.1359806061,-0.6876258850,0.9734085798,-0.1243617684,0.7628061175,-0.8110380173,-0.7298203707,-0.0372581482,1.0960302353,0.8897184730,1.0013744831,1.2954847813,1.0943894386,1.0337970257,-0.6451089978,-1.1438543797,1.6907882690,0.6538919806,0.2325033396,0.7079269886,-0.4387639165,-0.8021144271,-1.0609641075,-0.5739922523,-0.0819813982,-1.1191276312,-1.4699013233,1.2190533876,-0.7642792463,3.6206753254,0.2141495645,0.5528623462,0.1748252213,-0.6519388556,-0.6073905826,0.3652521074,-0.9204075933,-0.7603232861,1.0630009174,-0.9325689077,-0.5900637507,-1.1881865263,1.2027990818,-0.8413612247,0.2781607509,-0.2080128193,0.1023407057,-1.2445551157,0.6239708662,1.8129197359,1.0736453533,-1.8964418173,0.0017032215,-2.1157252789,1.3135426044,0.5474115610,0.2074623406,-0.4299831986,0.8374266624,1.6306321621,-0.3326140046,-1.8029530048,0.1280977726,0.6147022247,-1.1669268608,-0.2353281975,1.2309724092,0.2109920084,0.5916821361,1.1949048042,-1.6691788435,2.2587573528,-0.8347861767,0.7055326104,-1.2176433802,-1.5963435173,-0.9394806027],[1.1430879831,-1.0527108908,-0.2372030914,0.5021895170,-0.1742438227,-0.6610577106,0.9951126575,-0.5641317368,-0.9586183429,-0.4543020725,1.5888733864,-0.5602775812,-1.4585738182,0.2766699195,-0.2458266318,0.0311310496,0.3165929019,-0.4411346018,-0.7168897390,1.6787896156,-0.5344859362,0.1416471153,0.1917462647,0.5545337200,-0.9684610367,-0.2502814233,0.8965836763,-0.3063308001,1.5524247885,-0.2281495482,0.5966132879,-0.0518924855,1.4180803299,1.1030684710,-0.5906302333,-0.4166026711,-0.1812176853,1.2438272238,0.0661688223,-0.4422826469,-0.0799380690,0.1919439882,-1.7107120752,1.0774523020,1.3768286705,-0.4153911173,0.5705907941,-0.0298591200,-0.6760396361,2.0611596107,0.6797024012,-1.9420374632,0.1937771142,-1.4224283695,2.3359291553,1.5439224243,-0.1088899821,0.3045460284,-1.9224047661,-0.8653616309,-1.4767225981,-1.0221436024,1.1010134220,1.6597355604,-0.9422587752,0.2524269819,-0.5980673432,-0.8463162184,0.1441775709,-0.0865074918,-1.4574160576,-0.6760374308,-1.1465848684,0.6518920660,0.3316673040,-2.1897077560,-1.5613869429,1.1978230476,1.8515207767,-0.1663241088,1.5600320101,0.6311469674,0.5801112652,-0.6303533316,-2.2313139439,1.1076989174,1.4743758440,0.0909270272,-2.3126523495,0.0237566549,-0.5718167424,-0.2207095027,2.1772885323,-0.4963387847,-1.0081264973,-1.5023629665,0.7746482491,0.5341107845,-0.5666756630,-1.3972266912,-0.6807674766,0.9254412055,0.5504223704,0.6605623960,0.8508396149,1.3720737696,0.1541136503,0.8757134080,0.5460981131,-0.3596907854,0.1870216429,0.0117444582,0.6488144994,-0.5489287972,-0.3713762760,-0.7516688704,1.4095368385,-0.9506681561,1.2551702261,-0.2971719801,0.7826270461,-0.1850647628,0.0467204042,-0.5125409961,-0.4186106026,0.6029446721,-0.1644520611,-0.7338615656,1.0119118690,-0.9809918404,0.1098827720,0.0214030836,-0.0427952483,1.5712121725,-1.2539938688,-1.1002814770,1.1327694654,0.7072475553,0.3967822194,-1.5087244511,-0.7145330906,-0.3722834885,0.6798279881,-1.1214778423,1.6043719053,-0.5292537808,0.6880461574,1.8885463476,-0.0577552915,0.2836357951,-0.5647696853,0.2792871892,0.8215509653,-1.3532710075,1.7211893797,-0.0139528438,1.8308835030,0.7426056862,0.7111511827,-1.6903294325,0.1072090045,-1.0851238966,1.3005352020,0.0922427997,0.3189244568,-0.2317275256,-1.6285777092,0.2906110287,0.5595755577,-1.5583685637,-0.5369408727,-0.2023313344,-0.4653785825,0.2014144212,-0.7816708684,-0.3592549860,0.8583226800,0.8177946210,0.5941096544,1.9991146326,-0.1164116636,1.6596740484,-0.1440827549,0.4968259335,0.6306084991,0.6978434920,-0.7464147806,-0.2029277682,0.0740947649,1.0179958344,0.3487094343,-0.9844735861,0.5505439043,0.2724183202,1.6677926779,2.0806522369,0.4009736776,-0.5573444963,-1.5428788662,0.3109628558,-0.1744219512,1.1310318708,-1.1780905724,0.2007820308,-0.7422007322,-0.9340967536,0.5350461006,0.0442560837,1.1695052385,-1.3321918249,0.6491150260,-0.4747289419,0.5556890965,0.7429620028,-2.0583868027,1.9008646011,0.3469739258,0.0191483814,1.1078541279,-2.2555680275,-1.9164475203,0.2035084963,-1.1060945988,0.3864742815],[1.8186519146,1.0955337286,-0.1587141752,1.2405792475,1.1290600300,0.1848345250,-1.3021864891,-0.5888038278,0.5011674762,-0.2290555686,0.1941754520,-0.0028979329,0.1620416641,-0.1024544761,1.3428152800,0.5823216438,0.5299244523,-0.4311875105,-3.9000253677,0.1455959678,0.1971248835,1.5607292652,1.1137356758,-0.4552126825,1.4894210100,-0.2594603002,1.3953379393,0.0575970933,1.1616679430,-0.5125772357,0.2737622857,0.6855733395,-2.2379720211,0.0204583369,-0.4642761946,0.5056542158,-1.1518577337,-0.0480799004,0.8714804649,-0.4403902292,0.9177758694,-0.2967298031,-0.1093246862,-0.3690237701,-0.8198360205,0.8055944443,-1.2107851505,0.3110791147,-0.1688169837,-0.3388747573,-1.0188363791,0.5383563638,0.2944939733,-0.4924473166,-1.0070977211,-1.6481902599,1.4362820387,0.6015846133,0.5694240332,1.6722046137,-1.7027004957,-1.3625906706,-0.1457322389,-0.5113507509,0.2680238783,-1.7027720213,0.7206459045,-0.5888085365,-0.6532110572,-1.0642852783,2.3046946526,-1.4179235697,0.2645181417,-0.8454362750,0.4878188074,-1.3037599325,0.7964784503,-0.1442486495,0.3195949793,1.4605239630,-0.6226558089,-1.4823346138,-0.4593512416,-0.9331763983,-2.1541390419,0.6264881492,0.0523369238,1.1512045860,-0.4332700968,-0.9674804211,-0.1992558837,1.5613337755,-0.3819993436,0.6795611978,1.0330165625,-0.6436550021,-0.0710420236,-0.8683775067,1.9464271069,1.7365958691,0.8267059922,-0.2149627805,0.3993192315,-1.1981478930,0.6476678252,1.6629068851,0.1144966111,-1.2598162889,1.8402992487,0.1836557686,-2.4232974052,1.7620264292,-1.8486828804,-0.9964275956,-1.1717942953,0.6456684470,0.2873594165,-1.3820856810,-0.7862964869,-0.2583193183,-0.8816519976,-0.5542247891,-0.6523939967,0.5725956559,0.4938937128,-0.7512942553,-0.0124218957,-0.9071369171,1.4349648952,1.1218518019,-0.1233512685,1.0522289276,-1.6539301872,-0.1291158497,-0.2237756848,-1.1045513153,0.4521828890,-0.9502961040,-0.1316948235,-0.7792828083,0.8250408173,-0.8836473823,1.2532879114,0.7888385057,-0.8419920206,2.7643673420,1.0004105568,-0.4344625175,0.2776719928,1.3631216288,-0.7419697642,-1.4716395140,-0.1492524296,0.0200016722,-0.5150440335,-0.4084215462,1.4968609810,0.4010550976,-0.7996406555,-0.3596092463,0.5699978471,-1.6187551022,-0.2355066538,0.4505786896,1.0382069349,0.1090804636,1.1643670797,-0.9958294630,0.8406545520,-0.6185593605,0.4861762524,-0.1435554177,-1.1838173866,0.6962852478,0.8449316621,-0.5324366689,0.0854075029,0.3886263371,-1.1088260412,0.0637926683,-1.5521410704,-0.8580669165,0.5527657270,-1.3862566948,-0.3908568025,0.9630132914,-0.7347410917,1.8605796099,1.0593957901,-0.3667660952,-0.6616054177,-0.5578662157,-1.0552052259,1.4580582380,1.1121358871,0.9594700933,0.2913878858,-0.1537316144,-0.5127632618,-0.9405737519,-0.6017516255,0.2945948243,-0.3719144464,-0.9353292584,-1.3446847200,0.9422808290,-0.6510933638,0.8299900293,-0.3449546695,0.6160442829,0.4052671790,-0.6935716867,1.0162554979,0.6377590299,-0.4352501035,-1.4257146120,-1.2524747849,0.7853822112,-1.5440671444,-0.1379649192,-1.7845451832,-0.3356660903,1.3356161118,-0.1294857413],[1.7964646816,0.0304738451,0.7768535614,3.1488924026,-0.2558109760,-1.0679833889,-0.7244794369,-1.2258969545,0.6932333112,0.0211398620,0.1776879132,-0.5041844249,-0.2590360343,1.1173756123,-0.4724331498,1.3155813217,0.4622018635,0.6426978111,-0.5433134437,0.4678083062,-0.3611486554,0.8685443401,0.8873092532,0.9602596760,0.2302647084,1.2853378057,0.5569596887,0.0047805822,-0.6112685800,-0.6782135963,1.8684548140,-0.0579861216,-0.5789955854,-0.5955775976,-0.0142703950,-1.9806233644,-0.3151291013,0.1911353171,-0.7444159389,0.3003838956,0.3280220032,0.3322880864,-0.4767027795,0.2453860193,-1.5312238932,0.4211049378,1.9572474957,-0.2563537657,-0.7224268913,1.6377799511,-0.7879798412,0.3240051568,0.9042937756,0.7157511115,-2.7979767323,-1.2734445333,0.9677686691,0.9257705808,1.2029485703,-1.3839800358,0.7151804566,-0.2824457288,1.1473516226,0.2747390270,-0.7925121188,-0.7553021908,1.7981562614,0.6151124835,-0.2200139612,0.2743393481,0.9763574004,-0.3318104446,0.3161179721,-1.2327262163,0.7387780547,1.0659815073,0.5092077255,0.4646466076,-0.1616929471,-0.3656149805,-0.9359837770,-1.6265611649,-1.7118582726,1.0020611286,-1.0089641809,-0.5619198084,0.0407424457,0.4716152847,-0.7087352872,0.2111394554,-0.0356240310,0.1253914833,0.3591753542,0.6934660077,0.6154317856,0.5487660170,-0.6675722003,1.3222116232,-1.3435800076,-0.3621238470,0.6056843400,-1.4640836716,-0.1939837933,1.3377373219,1.3163279295,-0.2840470374,0.1725505888,-1.0558664799,-0.5492330790,-0.2993440926,-1.0710476637,0.0941991359,0.1093792766,-2.4019165039,-0.7428352237,0.9365871549,-0.1508529186,-1.0279309750,-0.2757891715,-0.3821061254,-0.1312558502,-0.9542873502,-0.9391035438,0.7704802155,2.5137343407,-1.8576669693,0.8858221173,-1.0607217550,0.8156251907,0.7083178759,-0.2483287305,-0.3928205669,1.5658286810,-0.4305017591,0.7362316251,1.7274297476,0.6333814859,0.7388486862,-0.0623305477,0.3513028622,1.0972409248,-0.4740725160,0.1975655705,-0.0587218367,-0.9982358217,-1.9500787258,0.6160622835,0.8414084315,0.1016046256,0.6070315838,0.1194629371,-1.4341905117,-0.1593689024,-0.6355468631,-2.6104624271,-2.0266444683,1.4117767811,-0.3176827431,0.9845973253,0.0557049438,0.4061246812,-0.8875346184,0.6713454127,1.0031460524,0.3599224389,0.9649901986,0.0233377106,2.1343255043,-3.0345256329,0.0627000332,1.7538636923,1.6268380880,0.3056345880,-0.3472956717,0.4976058006,1.6890598536,-2.3212182522,0.3656421304,0.1159397736,0.8868479133,1.6109799147,-0.6043145061,0.0901816487,-0.5041403770,0.1524684727,0.4275237024,0.8015805483,0.1325070113,1.7922909260,0.0355416387,-0.0134482626,-0.1337601840,-0.4896154702,-0.4701613188,-1.1700284481,1.0802490711,-0.3751786947,1.8959558010,0.7834692597,-0.0290208533,0.2484013289,0.5301579237,1.0153846741,-0.1129957139,0.0102426130,1.1725256443,1.3199801445,-2.3764221668,-0.0671974123,0.1287298054,-0.0941253453,0.4301971197,1.1682566404,-0.8586180806,-0.0630043671,-1.1315516233,0.8523266912,0.8010767102,-0.5508226752,-2.2161388397,-1.2388525009,1.3747489452,-1.3621162176,-1.3913899660],[-0.8186293840,-0.4310851693,-0.3265868723,-0.6595625281,2.3377609253,-1.4617418051,-0.3850235641,-0.5001681447,-0.7882581949,-0.6133599877,-0.7521061897,-1.3350371122,0.8898956776,-0.1522801816,-0.1465325207,-0.3226323426,-0.4473139346,-0.5643959045,-0.9613133669,0.8519867659,0.3872238696,1.4215050936,0.3426579237,-0.9598645568,1.6452548504,1.7389665842,-0.6744043231,0.3472268283,0.2202254236,0.2865630090,-0.6780747771,2.7566978931,-0.8731807470,-0.7633216977,0.6871228814,-0.6211286187,-0.4921645522,0.0778925493,0.2215663344,-0.2430664897,-0.3674600422,-1.3664894104,-1.6891640425,-0.6190827489,0.6096336246,0.4023823142,-1.6844097376,-1.2404732704,1.2487276793,1.0760413408,0.7050176859,0.2361517102,0.0590851456,0.6469197273,-0.0474967249,0.4412597120,0.5177218914,0.2961484194,0.6438617706,-1.9988317490,1.8926148415,0.1096209288,-0.1981773376,-0.4132628739,0.1424571574,0.2664468884,-1.3339275122,-0.5177214742,-1.3208273649,0.3496943712,-1.0425800085,0.9637414813,1.1737784147,0.4498517811,-0.0644505918,0.5101744533,2.1366763115,1.4451804161,-0.4666092992,-0.4894590378,0.3287404478,0.3134740591,-1.2987185717,1.3868933916,-0.2306082249,-0.6893220544,1.3180077076,1.3273700476,0.4140501618,0.7191207409,-1.2703672647,-1.2914323807,0.7756549716,-0.5866320133,-1.0754280090,1.3346775770,-1.1625427008,-0.0874402970,0.3637840152,-0.7374979258,0.0501301140,2.0589954853,-0.1709011495,1.1750373840,-0.7916604280,0.6599710584,-0.1943487674,-0.7274425626,1.4040251970,2.2787554264,1.5396624804,-0.0670272559,0.2121014595,1.2338556051,0.1547452658,0.0879635513,-0.7265217304,-0.2062857002,-0.5102232099,-1.6904525757,0.6855853796,0.4538678527,0.6846259832,0.2891429365,-0.3029614091,-0.3062827587,0.7371539474,-0.2199067771,-0.1700534672,1.6199733019,1.2885725498,-0.7799361944,0.1090501919,0.4491660595,-0.0288286079,-0.0628983006,0.8857010007,0.2705981135,0.3398106098,1.2366462946,2.3887815475,0.2669627666,1.2540435791,-3.3561549187,1.2307398319,-0.9899719954,-0.8115420938,0.3570990562,-1.3176939487,-0.5587753654,1.2871930599,0.5928227305,-0.1087322533,-0.9933822751,1.3560775518,0.1624943316,0.0694571063,-0.0019136409,0.7378163934,1.8368467093,-0.5628502965,-2.1004903316,0.1066910550,0.9081601501,-0.0222555753,-0.3945684731,1.4343575239,-0.5552595258,1.7829042673,0.0356468856,-0.2347169667,1.4369341135,0.8930010200,0.4602114558,-1.9390178919,-0.4052565396,-1.1254340410,-0.2811224461,-0.5239998698,-1.0315814018,0.1420727074,0.3060365021,0.0850403309,0.0480993278,-1.2978804111,1.0131076574,1.3917434216,-1.8141626120,-1.2823776007,-0.1916575879,-0.3455839157,-1.2886086702,-0.8586668372,-0.6362670064,-1.4075217247,-0.9333543777,0.1762785763,-1.3649258614,1.2995932102,-1.1005318165,0.4033268690,-0.3083316386,0.0538614839,1.3773829937,-0.0019546524,-0.2184900641,0.2411379069,0.8984738588,-0.2639446259,0.2973072827,-0.1650132984,1.6599066257,-0.5840214491,0.2017857730,1.3713948727,-1.0020972490,-0.4293357432,1.4016997814,0.1269748211,0.9718328714,0.1917524785,1.1341800690,-0.8607709408,-0.6005527973],[-1.3925638199,-0.3704949021,-1.1400412321,-0.1199970022,0.3022707105,-0.3858364522,0.2210528702,0.0352988578,-0.2549173236,0.2986409068,0.4361649752,-1.0413330793,0.3294892013,1.4552787542,1.5862495899,-0.0227510352,-1.0812748671,-2.4277453423,1.2509670258,0.2331276834,0.4919901788,1.9083909988,1.1447935104,0.7911493182,-1.0623035431,1.0173264742,-0.5506180525,1.3086786270,-1.1312993765,-0.2539741099,-0.7769720554,-0.8569401503,-1.0207500458,-0.4745619595,-0.0598216392,-0.8905045986,0.7787166238,1.3677231073,0.1858465374,0.4457555711,0.7118093371,0.2962641120,3.0088768005,-0.4178437889,1.5481282473,0.4459330738,-0.9167686105,0.8193830848,0.7471025586,1.5379596949,2.2011966705,0.3746042550,-0.7496331334,-0.9353886843,0.0856559277,-0.2704099715,-0.2366278768,1.7826231718,1.6654078960,-0.1156392843,-0.3040034175,-1.1702992916,0.3079847693,0.2505033910,-2.8715040684,-0.2713453174,-0.6889963746,0.9085056782,-0.4304555953,-0.8376456499,-0.5170556307,-0.3983263373,-0.1063844338,0.5968197584,-0.8843860030,-0.5697482824,0.9696322083,0.4466669858,2.6486330032,1.0537792444,1.1315171719,-1.6960200071,1.4721640348,0.9752472043,-0.7270838022,-0.0512396209,-1.2781599760,0.0176169593,1.8218171597,-0.7481163740,0.5921722651,-0.3314355314,0.5687924027,-0.0314355269,-0.1451000571,0.0710702613,-0.9201598763,-0.7968252897,1.6790449619,-1.2463930845,-1.1037224531,0.2885740697,1.1588948965,0.6854236126,0.2255977839,0.2921647429,-0.6920856237,-1.1240699291,-0.3644988835,-1.7979272604,-1.1228899956,1.2272027731,-0.5487995744,-1.3136093616,-1.0297425985,0.7001258731,1.7890787125,-0.7551582456,-0.8016943336,0.0867572725,-0.5807051063,-1.9119019508,0.5016669035,-0.7634596825,0.4129818976,-0.6169055104,0.2314937115,0.0343069099,-0.3614818454,-0.0572039559,-0.0302359294,-0.4526371956,-1.9301646948,1.3086107969,-0.2420634627,-0.4892685413,-1.1492084265,-1.3499071598,0.6945834160,-2.6993854046,0.1622554362,1.8245487213,-2.3775310516,1.2132954597,0.4981803596,-0.4224981964,0.6930828691,0.7331420183,-0.8765794039,-0.1939884126,-1.3364939690,0.9969375730,-0.7817802429,0.1041869819,0.5689641833,-0.1894062161,1.7746828794,3.4010379314,1.7501906157,1.1160522699,0.5270024538,-0.5151191354,0.7056614161,0.3733058274,-2.1757833958,0.7578504682,1.2810604572,-1.3550686836,-0.0728973374,-0.8941698670,2.7244987488,0.6562985778,-1.3424791098,-1.4879219532,-0.4622645676,-0.8685433865,-0.8274912834,1.8957636356,-0.2304519713,0.7528501749,-0.4163318872,1.2387114763,0.4693082869,0.5144113302,-0.7112178802,-0.5271394253,0.2068586797,0.3400960863,0.4101056755,-0.7695389390,-1.8163501024,1.2588369846,-0.7120572329,-2.1530566216,-0.3808211088,0.1169305518,-1.2409464121,-0.4366751611,0.2634794712,1.7055178881,1.6841148138,1.5426491499,-0.9426856041,-0.3110214472,-0.6535518765,-0.9700681567,-1.0188583136,1.0102300644,-1.3473342657,-0.9368382692,0.2325300127,-1.4934231043,-1.2163499594,-1.3792191744,-1.7362242937,-0.4512777925,-0.5170749426,0.4252825677,0.7567505836,-0.5467865467,-1.1445674896,-1.5563454628,-0.2236038148,0.1928770691],[0.7539950013,-0.2075280994,0.5398771763,-0.9476968646,-0.9016711712,0.2438880652,-0.0737104565,1.0209364891,-0.9426563382,-0.4375885427,0.4001968503,0.2803052068,-0.2396769971,-0.5233165026,-0.0707564503,0.8131171465,-0.3948679268,1.0977296829,0.9422441721,0.0974572301,0.7161006331,-1.0236706734,0.3119589984,-1.7411181927,-0.1571304351,0.9331872463,-0.3369457722,0.4094284177,0.4208583236,0.4179211259,1.1235785484,0.7556237578,-0.3103762269,0.8064317703,0.0034907553,0.0354942977,-0.5652838349,-1.3458013535,0.3002234101,-2.1501505375,-0.0334357657,-2.6188161373,-1.2006733418,-0.0334124565,0.6445909739,-0.6677893400,1.4247175455,1.0710269213,0.4332047105,1.3313224316,-0.4722385705,0.6907301545,0.1094585806,-0.2851660252,0.3637922108,1.1852264404,0.1758918315,-0.7584307790,0.2633218169,1.0621172190,0.2851718664,-0.6434780955,-0.1335996836,0.1274035126,-0.5857658982,1.0504328012,2.0061223507,-0.4886282980,0.0543444939,-0.2162028104,-0.6167577505,-0.7822800875,0.9975462556,-0.0760741457,0.8697055578,-0.5896819234,1.4034289122,-0.6588875651,0.1298187226,0.3436029553,0.2405471802,0.8691177368,-0.2007995248,-2.4776923656,0.1163100004,-0.3875218034,-0.1351628304,0.4279449880,0.0200409684,-1.2814878225,-0.0035458973,0.5171042085,-0.7989640236,-0.5903617144,0.7532331944,-1.8532392979,-0.7018445730,1.1726995707,0.8857141137,-0.6251412630,-1.4943467379,0.8706994653,0.1993800104,0.6574698091,1.3126466274,-1.0984554291,-0.3523005247,1.7237235308,-0.4371421337,-0.2530871630,-0.3125239313,-0.1090791225,-0.2442880720,-2.3474428654,1.0265904665,-1.1621482372,0.5418043733,-0.1000503302,1.6922651529,0.4630069435,0.1329090595,0.6170014739,0.7838420868,-0.5633745193,-0.8491060734,-1.1731932163,1.7671117783,0.2489637136,0.3668170869,1.0373156071,0.4797249138,-0.3064503670,1.5058437586,-1.7131545544,0.5895314217,-0.1014877111,0.0677429512,1.6819930077,-0.5216151476,0.2966107726,-0.6731379032,-2.9088072777,-0.1597213149,-0.8009475470,-0.3718868494,-0.5039060712,-0.8859555721,-0.1667399555,0.8699729443,0.2995379269,-0.0808590502,0.1149650365,-1.2343654633,-0.8192324042,-0.2976688147,0.6526411176,0.8078739643,-2.2390789986,-1.3788408041,-0.0203030016,1.0580019951,-0.1675913930,-0.3040016294,0.4077051580,1.6256030798,-0.4806309640,1.5464959145,-0.2882610261,-0.6253515482,1.1862369776,0.2702769339,-0.0170092937,-1.1964570284,0.9761407971,-1.3807101250,-0.0267551281,-0.7193158865,0.1454325020,0.8919139504,1.2186563015,-0.5411350727,-0.0417981744,0.4048648477,-2.4425880909,1.4544825554,0.9795028567,-0.9254541397,-1.0124781132,-1.2881475687,0.0540674552,0.0661320314,0.2954331338,1.0461391211,-1.0540885925,-0.4005309641,-0.1517889798,0.4216340780,0.0530298762,0.1122975871,-0.3990632594,-0.7042989731,0.1255299598,0.3206685483,1.1385912895,-0.4752325118,0.6582928896,-0.1538144350,0.6831755042,1.7123292685,0.5671478510,0.6366145015,-0.3570763469,-0.0657979697,0.4211172163,-2.2837688923,1.0036052465,0.2398010492,1.0281908512,-0.4071367979,0.9219275713,0.0722149760,-2.6470177174,1.3578057289,0.1536241025],[0.4696274400,0.8234762549,-0.4083556831,0.3090576828,-0.6590387225,0.2341024876,0.1462763995,0.2002189904,0.2424708307,2.4530985355,0.1110666841,-1.9045937061,0.6433965564,0.4160172939,-1.7808061838,-0.1339179873,0.8255435228,-0.2238264829,0.4268227518,1.0249240398,0.8471112847,1.3706696033,1.1877846718,-0.9383885860,0.3343340456,-0.4794832766,-0.4944494367,-0.1750551313,0.3527951539,-0.8698111773,0.1937713474,-0.1925773323,0.9743717313,0.6055154800,1.2223517895,0.3614206910,-0.2067714334,-0.9952375889,0.3734723628,0.6979523897,-0.4219244421,-0.5955637097,0.9482497573,-0.7519682050,-0.7860441804,1.3740886450,0.9647676945,1.7791903019,-0.7668116689,1.3360780478,-0.6458644271,-0.1355869919,2.8971319199,-1.1330628395,-0.6012628675,0.5005002022,-0.4042484164,0.4834759235,0.3865878284,-0.3293069601,0.8250538111,0.2771557271,-0.5596065521,0.9277037382,2.3185040951,-2.0182914734,-1.8434972763,1.8513425589,-0.2051957250,-0.3890289664,0.3600519001,0.9884135127,1.2457922697,-1.3364551067,1.7537808418,0.2910317779,-0.2233881950,1.5224063396,-0.4635525346,-1.0450754166,-0.4129071236,1.3809294701,-1.6089481115,0.3078214824,0.9460044503,-0.4184794128,-1.7909796238,-1.0131839514,-0.3165536225,-2.1457698345,-0.9870025516,-0.9171952009,-0.1917024702,-0.7953843474,0.6767178178,-0.3210875988,1.3021671772,-0.4075162709,1.5203324556,0.3550294042,-1.4809036255,-1.2452964783,0.1944308579,-0.9956462979,-0.3705863357,0.5113051534,-0.5084799528,0.7373731136,-1.0020056963,-1.2280149460,0.0769010037,-1.0816389322,0.6120415926,0.5528837442,0.9699832797,-1.6074030399,-0.3329996169,0.2555321753,0.7930832505,0.5816390514,-0.6598396897,-0.2890957296,-0.7990741134,-1.4835071564,1.6605551243,0.8653522134,-1.0595493317,0.9085577726,0.3510276079,0.5514123440,0.8528275490,0.0775258243,-0.0942507908,-0.5741663575,0.4272858500,0.4936881065,-0.4890993237,0.8531798124,-0.5760458112,-1.8211534023,-0.6887232065,-1.4060014486,0.7858404517,-0.7537473440,-1.0371315479,-1.1446654797,0.8815869093,1.2654161453,-0.6383075118,-0.8257868290,-0.9015462995,0.1437274814,1.3519058228,-0.1625126898,-1.9967465401,-1.8057687283,-1.2218840122,-0.4923840463,0.6176373363,-0.7052366138,-0.7661755681,-0.2294089645,-0.0140243210,1.3474508524,0.9664565325,-0.5530868173,0.5022634864,-0.4175859094,1.2790845633,0.3473213315,-0.5481665730,0.1712707430,0.5872876048,-0.2753806710,-0.7414954305,0.4202188551,1.1896193027,0.8167036176,0.0596480407,0.9580199718,-1.3611825705,1.7055370808,0.0063230065,1.8501033783,1.4732143879,-0.0210349392,-2.0528321266,-0.0954066589,1.0599880219,0.7781971097,-0.4838570356,0.4811122417,-1.8912461996,0.3263382018,-0.2272473574,-2.0044629574,1.1873849630,1.5935777426,-2.1323070526,0.3736905158,1.1431974173,2.2025403976,0.5080515146,0.7770988941,-1.9484568834,1.8847243786,-1.0159929991,-0.3248388767,-0.1660611182,0.4894922078,0.1777799129,1.0510669947,-2.4797639847,-1.1504589319,-0.8618552685,0.9332055449,-0.4171830714,0.1696135104,0.4340079725,-0.0387988202,-0.3269042671,0.1180056855,-1.0725272894,-2.8303813934],[0.2598698735,-1.9996215105,0.7559278011,-0.9798888564,1.3117982149,0.0267896838,0.7077409029,-1.3086125851,0.4044201374,-2.3094742298,1.6848480701,0.1338439882,-0.6248208880,1.6932362318,0.0373727605,1.5558153391,-0.1714841425,-2.3042349815,-0.4278883338,-0.9955630898,-0.5958486795,-0.0447826646,-0.1579531431,1.5243920088,1.9227956533,0.0402987376,-0.5851413012,1.5017855167,2.1803050041,-3.0837733746,-1.4648975134,-0.3116869926,1.9565404654,-0.3280645013,0.4791809916,-0.5352160931,1.1737276316,0.4787496030,0.4083316028,0.3431856334,-0.6605724096,2.7108654976,0.1147972420,-0.4463745356,-0.5844126344,-1.1667927504,0.5431652665,-1.9812204838,2.9475674629,2.6613872051,-1.2210031748,-1.1133307219,1.1255795956,0.5629199743,2.0009100437,1.5682934523,0.3448376358,1.2724161148,1.6458244324,-0.6888008714,0.2147175968,0.7798329592,-0.3226068914,1.0111198425,-0.5555141568,0.0550549068,-0.4487963021,-0.0010088464,-0.2460303754,0.3926325738,-0.4658337235,-0.0550008826,0.9606356621,-1.3934432268,0.2840017378,0.4586493075,1.6391483545,0.2773736715,-0.4988467097,-1.2484349012,0.6145244241,-0.2343735695,-0.6073554158,0.1037323400,-0.4924787283,-0.8200624585,-0.8451112509,0.3545969725,-0.9189939499,1.3295829296,1.4435455799,-0.4522539079,-0.6177167892,0.4639691710,-0.1216813028,-0.2014279813,-0.3888003230,-0.7420611382,-1.0614147186,0.1071903482,-0.1916001439,-0.8809502721,1.1320753098,1.8845841885,-0.6628830433,-1.5071902275,-0.1309629083,0.3128623366,-0.8688427210,-0.6582813859,-0.2740010321,-0.7021917105,-0.0590858348,0.5003613234,0.4435341060,-0.2532876730,-0.8129278421,0.0408533216,0.9117563963,0.1363046020,0.2125895768,0.2870524526,0.4323830903,0.8272413611,-0.8387100101,-0.5534974933,-1.4483233690,0.5585784912,-1.7981764078,0.2371001542,2.4801561832,-0.3021345437,-1.9256451130,-1.3350820541,-0.2666696012,1.0199847221,0.4717119038,-1.3698065281,0.4468599856,-0.0044480967,0.1817408204,-0.4527272880,-0.1334051341,0.3836607635,-0.4943971932,-1.4046680927,0.9504975080,0.0615183525,-0.5815150142,1.3562990427,-0.6733461618,-0.1815000921,1.1351847649,-1.8547165394,0.0427258536,1.2927519083,-0.7696661353,-0.8274409771,-0.2532393634,-0.0308168735,-0.7520497441,1.3307759762,-1.3623781204,-0.4073944986,1.5803796053,2.0406692028,0.5565537810,0.2322279662,-1.0841896534,-1.7789659500,3.2291426659,1.6942933798,0.9701338410,-1.2909091711,-1.8891059160,1.4711436033,0.4616388381,3.1894185543,-1.5284318924,-0.4539346397,1.7737467289,-1.0063527822,-1.0183699131,-0.7966518998,-0.6973011494,-1.8228462934,-0.6940974593,-1.1141934395,-2.0748908520,-0.6731943488,-0.1411335617,-0.5918131471,0.3981718421,-0.4185449481,0.6575914025,1.2034171820,0.7911134362,0.4504041076,-0.1689087003,-0.6241304278,0.1604327261,1.1599665880,-0.5865268111,1.4177463055,0.7862163782,-0.9866256118,1.3610484600,0.1662872732,0.7623544335,0.4777636528,-0.4978094697,-0.9205987453,-0.6019160151,-0.5025660992,0.2650274038,1.7796533108,-0.6799931526,-1.1500374079,2.1551897526,1.2656960487,-1.8137335777,0.8788180947,-0.1766525656,1.0173312426],[-0.3976323307,-0.4874837101,1.3260022402,2.5298173428,0.3581320345,0.6151618958,-0.9057765603,0.1278074235,0.4101307094,2.0343420506,3.1579160690,-0.2835023105,-1.6285960674,0.2310625762,-0.1825703532,0.7172837853,-0.9383327365,0.7119765282,-0.1827139258,-1.0331274271,-1.5335197449,-0.0797828063,0.9077846408,1.8982933760,1.4115979671,-0.4417716563,1.4773440361,-0.0987058356,0.1569772065,0.1336328238,1.4706661701,0.3962785900,0.3535576165,-1.0396783352,-0.8096060753,-0.9630953074,-0.7505313754,-0.1620675176,-0.0327635854,-0.5966364145,-0.0887803882,0.2664997876,0.6335312724,1.0054351091,0.0889114439,-0.5581123829,1.3313112259,-0.0271414258,1.7692632675,-0.3311189711,0.2609533072,0.4870017171,0.5392374992,1.2806769609,-1.3460060358,1.5021263361,-1.1109247208,-1.3770757914,-0.5652141571,0.5334227085,-0.3018633425,-0.9164126515,-0.4580081999,0.1177384034,0.8895198107,0.8998060822,1.0108145475,0.6404081583,1.3359845877,1.2474398613,3.1175408363,-1.2394305468,1.5618791580,-0.8826349378,0.3026507795,-1.4841973782,1.9171879292,0.6733652949,0.2914884686,-0.0425878204,0.7053133845,-0.2756962776,-1.1116207838,0.2003324777,0.4464299381,-0.4472725391,-0.5560350418,0.5862920284,-1.1646692753,-0.7891066074,0.0170574617,1.8001844883,-0.3394494653,0.5136883259,-0.5891038775,0.0483943895,0.4620130360,0.4717587233,2.6874232292,0.8009564877,0.0847277343,0.5359853506,-0.5240750313,-0.2925489545,-1.8751555681,-0.0490361750,0.9783039689,1.7588734627,0.6002600789,-1.5460512638,-0.3047045469,0.1539754272,0.0915840715,-0.5992405415,-0.8042455912,1.4659844637,0.1012221351,-0.1497529149,1.3030563593,0.1541244388,1.2717978954,0.8867399096,-0.4318663478,0.0770084262,1.6258184910,-0.6968182325,-0.3166991472,-0.7483909130,0.5786557198,0.0780947804,0.2426024377,0.0229945462,0.6200746894,0.2310543954,-1.1079101563,-0.4020124376,1.2420753241,-0.6679664850,0.2542519271,1.0629830360,-0.2348063737,-0.0438308194,1.3501619101,-0.7894948721,-0.5703001022,0.3089890480,1.5118911266,2.1752941608,-0.4885034859,0.8232432008,-1.0765361786,-0.0186190307,0.3972360492,-0.6259902120,-0.7424067259,-0.0163867436,0.1629671752,-0.0893045589,0.4563429058,-0.3665952981,-0.0754317641,0.5265722871,-3.5460734367,-0.6457802653,0.7653337717,-2.1300947666,-0.6183684468,-0.1406880170,0.5203972459,-1.0640109777,-1.0158450603,-0.4575025737,0.2972842455,-0.8521395326,-0.7556797862,0.9271347523,0.7557883263,0.9758871198,-1.1998525858,0.7437992096,1.5684306622,-0.5971830487,-0.5554358959,-0.2583679557,1.1155749559,-1.0676947832,1.0032776594,-1.8513797522,-0.3517214060,0.6328516006,1.7444175482,-0.8807442784,1.4790461063,-0.1968304813,0.4180056751,0.7282510996,-1.6500500441,0.3640683889,1.1129844189,0.3351568282,-0.0096940594,-0.8879166842,-0.2117554396,-0.6859401464,0.3963009417,0.4607585669,-0.2736138701,1.7940083742,-0.1557259709,0.4512560666,0.7477252483,-0.5220136642,-0.0076781982,0.2178294808,-0.6283373833,1.9523649216,-2.1755685806,0.3284509480,0.0459350087,1.3447287083,-2.9746835232,-0.4574049115,-0.4559212029,-0.7927967906],[0.9932083488,0.5941705704,-0.0860538781,0.4814716876,-0.5481458902,-1.3757659197,-0.6130162477,-0.9668075442,-0.2767145038,1.1864753962,-0.2110439092,0.0695132017,-0.1974823922,0.2356964946,1.3622299433,1.2369188070,-1.0267732143,-1.3356039524,0.4908373058,-0.4819887877,-0.4039094150,-0.6929343939,1.2614177465,0.1905747056,1.1455303431,-2.1158614159,0.2583775818,0.4038105011,1.1752716303,0.4600075483,1.1974021196,-0.8707522750,-0.2817451358,0.0584771670,0.0356177837,-1.0699961185,-0.8630678654,-1.3466277122,-0.6549223661,1.1585097313,-0.1777383983,-0.2447415143,-0.0686794594,-0.4224056900,0.1100068390,-0.2732768059,-0.0694849864,-1.6778137684,-1.1774704456,0.2881167829,-0.3823485672,-0.2653631568,1.6647901535,-0.6484985948,-0.3476527035,0.0990425199,1.2877132893,0.2625562549,-1.7195298672,0.1339644045,0.9347825050,0.6773042679,1.3379974365,-1.1952675581,-1.4742416143,0.4088601172,0.5837417245,-0.8978670835,0.9447718263,1.2271732092,-0.1702542156,-0.7172225118,0.8332493305,-0.9608859420,0.6667991877,0.1675264537,-2.4551825523,-1.3160276413,-1.3327628374,-0.3683435619,0.2090760469,0.2522863746,-0.3697250783,-0.6468358636,-1.3560963869,-0.2139449418,-0.0944914892,1.3924498558,0.8243910074,1.6508191824,0.0590403564,-0.2889220715,-1.5590000153,-0.1294312924,-0.6941680312,1.7140065432,0.3213033080,-1.2664537430,-1.2225905657,-0.4669961035,1.2360336781,-0.0267958753,0.7786743045,0.3648271859,-0.2769484818,-1.7265735865,-2.1605618000,2.6050536633,0.5951356888,0.1564329863,-0.4526445568,0.3408089876,0.7182588577,1.1662245989,1.1812388897,-1.6080279350,-0.6149919629,0.0630393922,1.1043940783,-0.3757201433,2.8040859699,-1.2027796507,0.4753766060,-1.0247412920,0.0444119908,-0.0440445766,-0.9484578967,-0.9130100608,2.2063055038,1.8155006170,-0.4957681298,-0.0989568830,-1.0796426535,0.7822895050,-1.5829148293,-2.0142185688,-0.2487754971,-0.5554159880,-0.4700220823,1.3454676867,1.0648335218,1.0868573189,-1.7082974911,1.4550259113,-2.0609056950,0.1190925390,-0.3908568621,0.7174708843,-1.6193270683,-0.8048923016,0.3411201835,-0.4839953780,0.2841969430,1.0325852633,0.4843482375,-0.3298141658,0.8892413974,-0.3105459511,-0.9782136679,0.9867196679,0.3042748868,-1.2289379835,1.2453569174,-0.9088758230,3.6785209179,-0.8134478331,0.1131959110,0.1259584725,0.1765042692,-0.4288792610,-0.1435288638,-1.2279988527,-1.8669244051,1.1203285456,-0.7590832114,-2.0584406853,-0.3421135247,-0.6633723378,0.3519513607,-0.0436158255,0.2055463493,-0.3620091081,-0.6658552885,0.8952859640,0.4214867353,-1.3065363169,0.5173885226,-0.2587612569,-1.5498518944,-0.7420065403,-1.0491241217,-1.3573287725,0.8090244532,1.1611310244,1.3288847208,-0.1811762154,1.1714485884,-2.1874475479,-0.5001651645,2.5885710716,0.0558268018,-0.2819325328,0.3446589708,-0.4038097858,-0.1715606302,-0.1005059630,0.0135818729,1.7682633400,-1.9547574520,-0.6286581755,0.5792235732,-0.7384209037,-0.5796393156,-0.0029503813,1.5651570559,2.1308877468,0.3770836890,-0.2992276251,-0.4751704931,0.9646815658,0.8811886907,1.2541283369,-1.8058665991,-1.2195588350],[-0.2300940603,-0.2072250694,0.3269861937,-1.7808606625,0.9240894914,-0.2287798822,-0.7127582431,2.2856419086,-0.9465098381,-0.1695698649,-0.2585377395,-1.0501010418,-0.7296738625,0.5356189013,0.0110701267,-0.4649605453,-0.1267327815,0.3069141507,-0.2413745672,-1.9641823769,-0.4948186278,-0.3253749609,-1.1279431581,-0.2000069767,-0.0331251286,0.9191393852,0.2092423439,-1.7219876051,0.2334833145,0.5516022444,0.1393381953,0.8058815002,-0.6283417940,0.5322570205,-0.1071301252,1.4491105080,-0.4998187125,-0.8268104792,0.0094867535,-1.2482749224,0.4494889379,-0.0147008393,2.0177221298,-1.5565807819,-0.9726316929,-0.7886752486,-0.9195621014,-0.3232328296,-1.6523182392,-0.6833917499,1.1239200830,0.1679050326,0.3240798414,-0.7736946940,-0.5426825285,0.5655519366,-0.5882941484,-1.1837854385,1.1089788675,-0.5024980307,0.5236488581,1.0887473822,0.1232571155,-2.0052480698,0.4221430421,0.4283335209,-0.4680225849,0.8110977411,-1.0689373016,-0.5722622871,-0.9641005993,0.1786365956,0.4109344780,-0.2174949795,-1.2787374258,-0.2087196559,0.5712300539,0.4369576275,-1.6268904209,-2.0347113609,1.1750626564,0.3646789193,0.1261532754,-1.7006433010,2.5722634792,-0.4614850283,0.4970600307,0.4907185733,-0.0615386628,-0.0598403178,-1.1332823038,0.3685927987,-0.9918488264,0.2320727259,-0.3870346844,-0.2509341836,0.0421222299,-0.3261132240,0.1328040659,-1.4239165783,0.4410168231,0.8100617528,-0.0148307206,-0.5331758857,1.3579202890,-0.4871828556,-0.5692438483,-0.0303262174,-1.2603951693,0.3815560937,-0.2728674710,2.5167510509,-0.0868564248,-1.0303136110,0.7523828149,-0.0511697941,-0.8479565382,-1.6042995453,-0.7471022010,0.1003591493,0.9704436064,0.2647618651,1.9099409580,0.7866731882,0.1858402044,0.1325393170,0.9469413161,-0.4874721766,-0.0009565527,0.8496548533,-0.6209245324,0.2415722907,-1.9862576723,-1.4180949926,2.0506248474,-2.1965746880,-0.0406296737,-0.3715413809,1.2954125404,0.6550036073,1.6960650682,-0.6649290323,0.6570203900,-0.0266126283,1.8540788889,-1.2913211584,0.3372913897,-0.4722357988,-1.3040305376,0.0621721260,-0.0789120272,0.0933005512,1.3070337772,0.4582025707,0.7568929195,-0.0107256742,-0.1645700037,-1.1813340187,-0.2204653174,0.2581693232,0.8963260651,0.0670128316,0.8854981661,0.1641241163,-1.2728308439,-0.9655811191,-0.4384520650,1.6505980492,-0.5964840651,-0.2924530804,0.1418313682,-1.1281623840,-0.1900935769,-1.4205986261,-1.0110018253,-0.8043029904,1.2034075260,0.2360412180,0.7618456483,2.4332990646,-0.2489451468,-2.5425071716,-0.7711074948,-0.4524628222,-0.8661909103,-0.7690161467,1.0050204992,1.8060809374,2.0945205688,0.6686277986,0.6050437689,-1.2227077484,0.4604870081,-0.7087969184,1.8917765617,-0.0848279819,-2.1807665825,1.7026932240,0.1401837468,-0.4388398528,0.0585128926,1.5256692171,-0.2458079308,-1.3954952955,-0.8067660332,0.4578137696,-0.0693819672,-0.5427391529,0.5748937130,1.9503580332,-0.9398818016,1.0572952032,0.7284172177,-0.4724606872,0.7204971313,0.7258913517,-0.2430327237,-0.3732158244,0.8272249699,0.4962197840,-1.8620917797,-0.1028145254,1.1597520113,-0.4121288359],[-1.2027361393,0.6976327300,-2.4462964535,-1.7038713694,1.3121316433,-0.5465721488,1.1759943962,-0.0005320666,0.1727978289,0.4999870956,1.2192925215,0.0659877881,1.3122242689,-1.5925445557,-1.8933696747,-1.5688316822,0.7063481808,-0.0540814176,-0.4093583226,0.5415523648,-0.2810840607,1.2379930019,1.8455535173,-1.9942268133,1.9342218637,0.2190882415,0.2802676260,-0.6883954406,-1.4517098665,0.7597011328,1.3933690786,-0.4069502056,1.2743493319,0.7263096571,-1.1906430721,-1.7904323339,-0.7842006683,0.0496981777,1.1656367779,1.2140377760,0.5458866358,0.1088893190,-0.7287024260,-0.3762235045,-0.0407645404,0.0176126137,-0.3848236799,-0.3484652638,0.4573885798,-0.7914643884,-0.5282921195,-1.6176098585,-1.3753209114,-0.7362214327,0.2440520674,-0.5911464691,0.0714291632,1.3157017231,0.8958864212,0.7456697226,0.5918785334,-0.8514651656,1.1709183455,-0.0985445157,-0.7902092934,-0.8238563538,0.6776641011,0.6242868304,-1.7112609148,1.1113905907,0.3983590603,-0.3072124124,0.9468228817,1.0867260695,0.0427968390,-0.2378879040,1.1288268566,0.1591834277,-0.8023855686,-0.0328822881,0.9165640473,1.8892006874,-0.5545433760,0.2005936652,-0.1401738226,0.7532416582,-0.7191939354,0.1173081622,-0.9961249232,-0.0623728819,0.8635931611,-1.3111095428,1.0600906610,0.3490823209,1.0388455391,-0.1214415208,-1.2302621603,-0.3991114795,0.6279140115,-0.7386919856,-1.3221846819,-0.8256247640,-1.4266526699,-0.1054395214,0.1213134825,-1.1871228218,1.2489768267,-0.6177254319,-1.7859925032,-1.0651925802,1.6506885290,-0.0826005265,-1.7071485519,2.9751720428,0.7023817301,-0.5732915401,-1.4239773750,-0.4566698670,-1.0727081299,-1.4537141323,0.1222363412,-1.3030611277,-1.7162252665,0.1611646712,-0.0376786999,-0.5798072219,-0.8318155408,0.3107450306,0.3105241358,0.1700328141,-0.7377840281,0.1470898688,-0.0813722014,1.3564113379,0.1340189725,-0.6625532508,-0.0410240777,-1.3023015261,-1.0368205309,0.5293307304,-0.5021158457,-1.0542820692,-1.1481188536,0.1947138011,0.3171689510,-1.0173329115,0.5187175870,0.1057950556,-0.7455425262,0.4904182851,-1.0010027885,0.3528049588,-0.0746711195,1.0803209543,2.0294268131,0.4842899740,1.8676234484,-0.5828618407,0.7272090316,0.0941341445,0.9208273888,-0.4615549743,0.3433523774,1.0225626230,-1.3546812534,1.1875417233,-0.1132918969,-1.1203707457,1.2187744379,-0.4989327192,-0.9112264514,1.1047421694,1.0056560040,1.5598826408,0.6722717285,-1.3777923584,0.6470066905,0.2687519193,-0.2030589879,0.5464590192,-0.0320212431,-0.2359107584,-1.0255286694,-0.3290923834,-1.6468341351,0.0409779586,0.3232469857,0.1093000770,-1.0682225227,-1.3022935390,-0.1504627466,-1.0588222742,-0.3715017438,-1.1889908314,0.2288609296,0.2993321717,0.4552179277,-0.5519235134,0.0778525472,-0.1051733196,1.2543287277,-2.6627442837,-0.5184801221,-1.8136134148,0.1367356926,0.0721394643,-0.7105522156,0.7686612606,0.3240852356,0.8016172051,-0.2594587207,-1.3651039600,0.7902607918,-0.0750836357,-0.0406575575,-1.5975726843,-0.9080748558,-0.8948983550,0.7329773903,2.6953494549,-0.8728044629,-0.4765469730,1.2926892042,-0.8532977700],[0.2273871750,0.8500044942,0.8162901998,0.3053658307,0.1576725543,-1.2945351601,-0.2502624691,1.2744140625,-0.7250322700,-0.2637364864,-1.2683051825,0.9549968839,-1.9983717203,-0.9407818913,0.2056222409,0.9628034234,2.5071811676,0.4945580959,-0.2393418849,-0.4995599389,-1.2183736563,0.6175827384,0.3530254960,1.8969041109,-0.7900554538,1.6417257786,0.5082850456,0.7210969925,0.1719370484,-0.8770158291,1.2253770828,0.0307714008,-0.5999941826,-2.0702910423,0.1392152160,1.5867739916,-0.2774418890,-0.3968944550,-0.6462703943,-1.8079203367,-0.1435433030,0.2059432119,1.6489461660,-0.2428348064,0.8678718805,-2.1709468365,-0.0419791229,0.5046077967,-0.6470251679,-2.1490387917,1.5061701536,1.2396297455,-0.4780521691,0.0269159973,-0.7107182145,0.2932947576,-0.0467100739,-0.6241620183,1.5232892036,2.2675340176,-0.1043779999,2.0489764214,2.2696111202,1.2227553129,1.6606631279,-0.5712388754,0.0546040423,0.2590203583,1.2644896507,0.2683848739,2.7493901253,-0.9891241193,0.4144333899,0.3645968735,-1.1913316250,0.0431974605,1.7358802557,-2.4658632278,0.0826712474,0.4959769845,0.5519586802,-0.2668684125,-0.9466699958,1.6705639362,0.7358431220,0.7760394216,-1.1162991524,-1.0943129063,0.5646335483,0.5407114029,-0.5756079555,1.8289971352,0.8284856081,-1.6793274879,-0.0482726954,-0.3428310454,0.4674871862,0.4687924385,-0.7644754052,2.9350652695,1.5090997219,-1.2739346027,-0.8650171757,1.0715694427,0.7076815367,1.1324560642,-0.7596591711,0.4681403041,1.6337355375,0.3445463777,-0.3208348751,1.2135955095,1.3381587267,-0.1650899053,1.0665588379,-1.1150864363,1.0701800585,-1.3052628040,0.0422085188,1.9998065233,-1.7469418049,-0.2975669801,0.5028178692,0.8254718781,-0.4733499885,0.4095201790,-0.4338210821,-0.4989029765,-0.9439502954,0.1678249538,1.9921586514,-1.6073846817,-0.8297287822,0.0909254551,-0.6161510348,-0.5609905124,-0.2859306931,0.6985344291,-0.6712636352,-1.4290097952,0.5024514794,-0.6953155994,-0.8284509182,1.5240137577,0.3114162683,-0.5688574910,-0.5349234939,-0.4808453023,0.1530985087,0.2676455081,-0.2223212123,0.1744278967,0.2862839401,-0.6046664119,0.4904140234,0.0750555247,0.7289562225,0.6942309737,-1.0967652798,-0.7154685855,0.4513160884,2.0987625122,0.9161058068,-0.9617047906,-0.0101944050,-0.3066106141,0.0998871475,1.0319311619,0.2301389426,-1.0832765102,1.8243718147,-0.9366260171,0.1783067882,-0.1649248004,1.3017970324,0.6445472836,1.7660093307,1.3376543522,-1.1191633940,-0.1635971516,-1.6489555836,0.0993696302,0.6865174174,0.9365585446,0.2498519421,0.1184616387,0.6225427985,1.2603803873,-1.4978742599,2.1375963688,-0.1365803182,-0.1895365268,-0.2252049595,-0.1031861082,0.4588972330,-0.1754950583,-0.7891124487,-1.4106558561,-0.2436112761,1.3145904541,-0.2643491924,0.2929087579,-0.2023253739,-0.3809172213,-0.2256442457,-0.7095449567,1.6255832911,2.3382439613,0.4938969910,-1.9099097252,0.4098784924,0.9579098225,1.1237193346,-0.0052647912,0.0314161777,-0.2493994832,-0.8705860972,-0.8726430535,0.7882552743,0.7331334949,1.7048001289,-0.8281053901,-1.4328604937,-0.6085548997],[1.3891786337,-0.5192667246,0.4646701813,0.1095870212,0.7222380638,-1.0051553249,0.9021429420,-0.3742950559,-1.1925488710,0.6312025785,1.0276609659,-1.8227084875,0.1302038729,0.5725678802,0.7566816211,-0.6131682992,1.6283639669,0.5575514436,-1.5517416000,-0.3973201811,0.3933172822,-0.6209436059,2.2299447060,0.9194506407,0.1127791330,-1.3650295734,-0.3201951385,0.3234144151,0.8839156032,-1.6540412903,-0.6980065107,-0.6564571857,-0.7529318929,-1.6853379011,-0.8302558064,0.9642930627,-0.3338047266,1.1360625029,-0.1617259681,0.8039368987,-0.7213765979,0.9166084528,-0.2778778970,-0.1063275859,0.2079641819,-0.0063563455,0.2102287859,-1.0888133049,0.4468370378,1.1374158859,-0.5253376365,-0.4778427184,0.2243154347,-0.6749815345,1.0322270393,-0.9930639863,-0.4859096110,-0.3069780767,0.4539842308,0.7087320685,1.3813514709,0.0334968083,-1.5020490885,0.2795063853,0.9115028381,0.0795381293,-0.9493045807,0.0057000839,1.8318384886,0.2391988933,-0.2994287610,-0.1245497540,0.3006445765,-0.4049408138,0.0507116504,0.1041309908,-0.4472372830,0.2908937633,-1.3848526478,0.9086578488,0.3378473818,1.4619905949,-1.5845800638,-1.1525335312,-0.6863923073,-0.1426220536,-0.0538095944,-0.9843934178,0.7708935142,0.3634274900,0.7222071290,-1.4486782551,2.1479396820,-0.9167398810,-1.0138820410,-1.2438874245,1.6870024204,0.0624750219,-1.7277748585,1.0282175541,-1.0794452429,-0.9668543339,0.6164269447,1.0389000177,0.6272831559,-0.0910195485,0.3704117835,0.0858492926,-0.6236388683,0.6556786895,-1.1343281269,-0.6852283478,0.4187905490,-0.3736714423,0.4399876595,0.4863699675,-1.2893429995,1.2970787287,-0.9376946092,-0.0708927140,0.4519757032,-1.2560162544,-0.7564775348,-1.2853606939,-1.4764347076,0.0187401716,0.5181655288,0.1931071728,2.0165812969,-0.6870343089,-0.5474315882,-0.9243340492,-0.0350501649,0.4463641942,1.0694886446,-0.3940467834,0.8290531635,2.7378354073,0.6988120079,1.1682595015,1.2565833330,2.2958664894,1.0740492344,-1.1948891878,0.7294963002,-0.8258320689,-2.4308962822,-0.2404142767,0.3294350803,0.5455643535,1.1034344435,0.1110701412,0.3185352981,0.4520530403,-1.4553079605,-0.5798675418,0.3087005615,1.6675961018,-0.4612086117,-0.8659790158,-0.8224008083,-1.1954746246,2.3250622749,-0.0474405810,1.1397987604,0.4894039333,0.6380180717,0.8720409870,1.3701871634,-0.6702001691,0.1365573555,0.8938791752,-2.2082872391,-1.5051922798,0.4718648493,0.4459237456,1.0747903585,-0.3032374978,-1.9131064415,-0.9425461292,-2.1948966980,-0.5122132897,-1.1340646744,1.0650100708,-0.0310556535,-0.3919205070,1.9453985691,-0.9041585326,-1.1375190020,0.5593029857,0.3512958884,-0.2719665766,-1.2004096508,-0.9829906225,-0.5594509840,0.7834479809,-1.0296547413,-2.9886469841,-1.3362042904,1.3961710930,0.0321802497,1.6879734993,-1.9495279789,-1.3146616220,-0.6059178710,-0.1589753181,-1.5342533588,-0.0959366262,-1.0127555132,1.6585209370,-1.0970559120,-0.0442612171,1.0565162897,-0.4931052327,-2.3154871464,0.8893052340,-0.5490179062,-1.0743147135,-0.7600001097,1.4808275700,-0.6100429893,-0.0965769142,-0.1723432541,-0.3031660020],[-0.9866677523,-1.1144934893,-0.0944731832,1.2081950903,-0.5706077218,1.4837689400,1.4544682503,0.3421855569,0.5417101383,0.1148955226,-0.1729538888,-0.6006920934,-0.8390375972,-0.8688585758,1.9264659882,-0.5148521662,0.0482680723,1.4802362919,0.0934116393,0.1687042266,-0.5380268097,-0.1822315454,-0.6247577071,1.0951794386,-0.5520597100,1.1341189146,-0.9410278201,-0.6713935137,-0.7394921184,0.4502103925,-0.6198716164,-0.6839472651,0.1225380972,-0.2928015292,0.5678059459,1.3917983770,1.9239299297,-0.2305802852,-0.5392155647,1.9057199955,0.1113132760,0.2179827541,-0.3769581318,0.5038686395,0.5441030264,2.1462361813,-0.1754538417,-0.2272842228,-0.7777884603,0.3594806194,1.0193614960,-1.1204891205,0.9485693574,-1.2363845110,0.6520394087,-0.2308014482,1.6299146414,-1.1982326508,-1.4894257784,0.1250338554,1.6823540926,-0.0901871324,-1.1973161697,-1.0665248632,-1.6871834993,-0.8226858974,-0.2290683836,-0.2399158925,-1.1904006004,-1.1496270895,0.4385099411,-1.4581204653,1.4406924248,-0.3567754328,-1.2256165743,0.6519488692,-1.6003174782,1.6349955797,-0.4684672058,-0.8184659481,-1.4700213671,0.9492305517,0.9033931494,0.0192416534,-1.0961804390,-2.7277271748,-0.6291338801,-0.4750710428,-0.9310365915,-0.4051821530,0.6260236502,-1.2584766150,0.1520262361,-0.5098328590,-0.5226279497,0.5744693875,1.2434821129,0.0292647406,0.1323002875,2.9719250202,-0.2966171503,0.2232382149,-0.8948405385,1.9650298357,-0.4943556488,0.2295539081,1.1664227247,1.3640789986,1.3521430492,1.5934813023,-0.1320769340,1.0461800098,-0.6914286613,0.5779567361,-0.5560915470,-1.5874744654,0.2247067690,-0.3761298060,-0.6025452018,-1.5737394094,-0.8176524043,0.1051435396,-0.5349892974,-0.1311412752,-0.2104492337,1.1861252785,-1.3101953268,0.0564993545,-0.6865065694,-1.7655328512,0.8406751156,0.9682949781,-0.3303029239,1.2768537998,0.5624393821,-0.2575190067,0.1811937392,-1.0224636793,0.0395928435,-0.6465151906,1.3817548752,-0.8498784900,0.9605399370,-0.5322082043,-0.5752568245,-0.2549653947,1.0302226543,0.1062326059,-1.2118384838,0.1191878691,0.0014828539,-1.1837663651,1.0874757767,-2.8219051361,0.3327438831,0.3325736225,-0.9217550159,0.7638429999,1.3450644016,-1.1933127642,0.5198947787,-0.5931326151,-1.2922502756,0.1143693775,-0.0845904127,1.0581725836,0.9305245876,0.4070013165,-0.7341075540,-0.1060848758,2.1465601921,0.1787221581,1.7992449999,-1.2903478146,0.1223225221,1.4852894545,0.2455696911,-0.8079850078,-0.9959421754,1.3645406961,0.8025732636,0.2571508884,0.4182025492,0.7682970166,-0.2245483398,-0.9194589257,-0.4008037746,-1.7039490938,0.7321809530,1.0105509758,0.8314722776,-0.9105592966,-1.0720477104,-0.4215989411,0.7376125455,-2.4036571980,-0.3848825991,0.8627403975,-1.0709260702,0.9025353193,-0.2341202199,1.4916242361,1.0881841183,-0.4007686973,-0.3403339386,0.5308698416,0.7718102336,1.6546257734,2.7061583996,0.3537329733,-0.0448146500,0.1036679819,-0.7673404217,1.4605641365,-1.6149725914,-1.0541573763,0.0763381571,-0.8486981392,0.0386477970,-0.4559693038,-0.8339561224,-0.4026286006,-0.4288522899,-0.3586120307],[0.1533448994,-0.1136549860,-0.8437499404,0.6916508675,-0.4252757728,-1.9518626928,0.4714142382,0.5827259421,-1.2929222584,-0.3891405165,0.4851530194,0.5295843482,1.3490113020,-0.5542256832,0.0399686582,-0.6475527883,0.7677624822,0.1303908378,1.0854752064,1.7191438675,-1.0001112223,-0.6612618566,-2.0385735035,0.9662070274,-0.9222030640,0.2547031045,-0.1173657700,1.5544829369,-1.2023463249,0.2614167929,-1.9884563684,-0.0514406934,0.4254758358,0.2848817408,-1.1909495592,0.4748627841,-1.1672532558,-1.2991082668,0.2625207603,-0.2106069028,0.1302159131,0.6869119406,-1.2869962454,-0.4398226440,0.1023867726,1.9681735039,-0.3628941178,-0.3845193088,-1.8106294870,-0.3800882399,-0.3710056543,0.0102245891,0.5141667724,-0.9211145639,-0.1126588136,-1.1501650810,1.9226061106,1.2949194908,-0.2108839899,0.7516497374,-0.6861072183,0.4273181260,1.4809700251,-0.2807698846,-0.6138682961,0.1348924786,-0.1911712438,-1.2088664770,-1.3775349855,-1.5261063576,-0.3294872344,0.9192034602,0.6944690347,-0.2111102492,1.3722932339,1.0175527334,0.1142073944,-1.1281518936,0.5902181268,0.8330125809,0.6166064143,1.2236114740,0.4062391222,1.2697188854,-0.8363413811,0.1143634394,-0.4184566140,-0.3388504982,-0.0837198272,0.2457090765,-0.2588721216,-2.6595571041,-0.2221146077,0.2140495479,-0.8823900223,-0.4630803168,0.2623826265,0.4191049337,0.8740573525,0.9312805533,1.6630910635,-1.1229330301,-0.3988712132,0.3089698553,0.6604155898,1.0093617439,-1.6651716232,-0.7364777327,0.3452317119,-0.4719576538,0.0853512287,-0.6963783503,-0.2385855615,0.4833333492,-0.0953106210,1.8122999668,0.3582427204,-0.7948063612,0.4761179686,0.2154228389,0.6026530862,0.3933068812,0.6928840280,0.2977566421,0.6478030682,1.0729994774,0.4346579313,-0.9511593580,0.9497818351,0.8753393292,-0.3284352720,2.1367778778,1.6212879419,-0.6599621177,0.4766357243,-1.4166018963,0.7105426192,0.1458755434,-0.1393823624,0.1677254587,0.3930626810,0.0387067795,-1.0487891436,-2.1851563454,0.4409761131,-1.3168141842,-1.2756685019,0.4949780107,-0.0053673140,-0.3223702908,-0.6519203186,0.5039027929,0.2957015634,1.2806518078,-0.7631545663,-0.7418268323,1.3974416256,0.4859984517,-0.2325882018,-1.1511465311,-0.0982916951,-0.2832758725,0.0918571278,-0.1249718368,-1.4539741278,-0.2667857409,-0.5932501554,1.5480179787,0.1944077015,-1.2783476114,-0.1039233878,0.2209801972,0.5416983366,1.8491382599,0.7789201140,1.4660717249,-0.7854235172,0.7227502465,-0.0031879030,-0.3045358062,-1.3084830046,0.7555395365,-0.3509569466,0.8722602725,0.2640450895,-0.0512377284,-0.3850245178,0.2748367488,-0.8619273305,-0.5983606577,-0.2911280096,-1.1638661623,-0.1225519106,-1.1247522831,-0.7245661020,0.3172141910,1.0484007597,-0.6781160235,1.6836032867,1.7057974339,-0.7776958346,-0.7127845287,-1.7799401283,-0.9424460530,-0.2567984760,-1.7354787588,0.5987724662,0.6738944054,-0.7204313874,1.4037210941,0.9783397317,0.1899294406,0.8302225471,-0.8875858188,-0.1858076006,-0.8468185663,-1.0604618788,0.0126434006,-1.1486490965,-0.5749979019,-0.4174935520,1.3039525747,-1.1012734175,1.0699433088],[-1.4980803728,1.2565586567,0.4683552682,-0.4248012006,0.5665093660,-0.8894879222,-0.9242884517,0.5567165017,-0.2315455079,0.6107094288,0.0718754381,-0.4053229690,-0.1032371223,0.1640097946,0.0202594232,1.6416261196,-1.9592071772,0.7429304123,-0.7617667913,-0.9404531121,-1.0880265236,-0.4952817559,-0.1792133898,-1.0586580038,-0.2037551254,-2.7976267338,-0.2782055438,-0.3884856105,1.4003995657,-0.7439337969,0.1906057298,-1.2973126173,-0.5760190487,0.0010075861,-0.3164952993,0.4198083282,0.5122329593,0.6419844627,1.5902930498,-0.9253531694,0.4934574664,0.1158690378,-1.2287720442,0.8400771618,0.8152031302,-0.9727423191,-0.5200578570,-1.6350430250,0.7669951320,-0.7027513385,-0.2783911526,1.2756760120,-1.0061872005,0.5207966566,-0.1774211377,0.1268888563,1.1396467686,0.2380308509,-0.2564617991,1.5399690866,0.1576723307,0.7258236408,0.9801822901,-1.7694141865,-0.4557214975,-0.4337657094,1.1572785378,0.5312779546,-0.2003088146,0.6846209168,1.2052086592,-0.2296351045,1.1039553881,1.3138203621,2.8796958923,1.0311387777,-1.8142951727,-0.3141567707,0.2261027247,0.5151367188,3.0695443153,0.2548579574,-0.9201954603,-0.3639743030,1.4982109070,0.2461914569,-0.2713237107,-0.9528974295,0.7874175906,0.3267644346,-0.8727814555,0.1782025397,-1.1037790775,0.1650889963,-0.1240965798,-1.1618554592,0.1651325077,-0.6387265921,0.2335938811,1.3918951750,-0.9376947284,1.0478419065,-0.3835449815,1.6917550564,1.0707421303,1.5578179359,-0.4176296294,0.8883716464,-0.0825842023,1.0968365669,1.4495837688,0.3236922026,-0.0004275652,-0.0215025973,0.0394743606,0.8524720073,0.8768084645,0.8922595978,0.2185391933,0.4017852545,0.8576724529,0.8088468909,-0.3175498545,-0.4993957877,0.4370413125,0.0653270409,-0.5538098812,-0.8825448751,-0.1844616681,0.0126108462,-0.5641058683,0.2735268176,0.3900638819,-1.4210958481,1.3800935745,1.7626299858,0.2206579298,0.2094408274,0.1169288829,-0.0439985879,1.6005190611,1.9413633347,-0.9205535054,1.0154896975,0.5654467344,1.1234375238,-0.0849402025,-0.5356362462,-1.8453865051,-0.5912247300,2.3808486462,0.6380431652,-0.0779390782,-0.5372304916,-1.0714815855,-0.9567815661,-1.9812661409,-0.8885300756,0.3465723991,-1.5381587744,0.5294940472,-0.8159481883,0.1816445738,0.5993745923,-0.0688724965,0.4917052090,-2.1673610210,-1.2983940840,0.6069096327,-0.1220805272,-0.3988957107,1.2535202503,1.5654284954,0.2309513092,1.1095582247,-0.2400104105,-0.3079902530,0.2744587660,1.0943692923,0.2639151514,2.7983736992,1.5682145357,0.2655566633,1.0198340416,-0.1856144965,0.0719759017,0.6974091530,-0.3650955856,-1.0268156528,-0.0775278434,0.3819551766,-0.2707291245,0.4601927996,0.1960742921,-2.8733341694,-0.7153529525,-1.5700154305,-1.1252595186,0.8889979124,-0.1681040972,0.4190223217,-0.1275238544,-2.1446073055,0.4539523423,-1.7821577787,-0.7988014817,0.9233134389,0.0920516923,1.9260506630,1.1535536051,0.1917728931,-0.8712940812,-1.3023717403,0.2304571271,-0.2696764469,1.7969834805,-0.2386470139,0.3282249868,-0.6356702447,0.5387446880,0.8461692333,-0.2104572505,-0.8630337119,0.7488220334],[0.9744762778,-1.9335272312,0.2642577291,-0.7576102018,0.1086469367,1.0653381348,0.5454222560,-0.3583743274,-0.2401515096,0.8858023286,-1.5596271753,1.0870817900,0.4798313379,0.9968619943,1.0761747360,0.0166343134,0.7263668776,3.0349943638,0.2674559653,1.0315163136,-0.6091318130,1.3296933174,1.1060048342,0.1791629940,0.7824376822,0.3643795848,-0.0688682124,0.3397498429,0.0978621468,-0.6770811677,0.6139251590,-0.5844218731,-1.0284618139,0.8847949505,1.6860802174,-0.1588840932,-1.4511556625,0.9644990563,0.1010972485,0.7338010669,-0.1568467021,-0.9706460238,-0.6732884645,-1.0313397646,-0.8497923613,-1.4788682461,0.7079484463,0.3125767708,0.0050403099,0.4007194936,-1.1395459175,2.0679659843,-0.0851880386,-0.5869293809,-0.4736531079,0.7341102362,-0.3778316677,-0.5041046143,0.6410043836,0.0255725123,-0.6730322838,-0.4380423427,-0.6478739381,-0.0050521214,-0.3049477041,-0.8393627405,0.7373325229,0.3903099298,1.0192760229,0.7384288907,-0.3072429597,-1.5440155268,-1.2261928320,-0.6180872321,1.3727256060,-0.3824479878,-0.0834695697,0.7977703214,-0.7664447427,0.3092956245,-0.8667956591,-0.0599133223,-1.5001881123,-0.5292657018,-2.3725910187,-0.4090963602,-0.3331638277,-1.2272868156,0.8172550797,1.5381202698,-1.1170098782,-0.8876941800,0.1524083763,-1.9791057110,0.3771382570,-0.5076460838,1.2998002768,-0.8550341129,-1.0795487165,-0.1310768127,0.5569698811,-1.9104845524,0.6803012490,0.1322748959,0.3465833664,-1.1199702024,-0.3691485822,-1.7317962646,-0.8497638106,3.6870191097,2.2949135303,0.5006017685,1.6048829556,0.9236067533,0.5055042505,-0.6939823627,1.1900153160,-0.1262883246,2.3308765888,0.1819272786,-0.5119418502,1.1648005247,2.2314093113,0.6448895335,-0.6988681555,-0.0384671949,1.5574637651,-0.6268103123,-0.4058221579,0.4050198793,1.0012731552,0.7926679254,0.1292645335,-0.0604107268,0.3225554526,-0.4775817096,0.8939678073,1.1663762331,-0.5386640429,0.9164229035,-0.1918735504,1.4974851608,-0.6885712743,0.0340448506,-1.2865372896,0.7451160550,-1.6031799316,-0.4937097132,1.3747198582,-0.5014317036,0.8943790793,-1.3215848207,0.5586235523,1.5413256884,-1.4550051689,-0.9163423181,-0.5487048626,0.1736886799,-1.1643437147,-1.8016388416,0.9181058407,0.4756911993,0.0509872176,0.1600379348,0.1163054481,0.5942176580,0.2267961651,0.4649511874,-0.2198812366,1.7069232464,0.0407197364,-0.0010570940,2.6112511158,-0.8208794594,-0.8140652776,0.9350974560,-1.2511539459,1.4331966639,-0.6402844191,0.6937068105,0.5366545320,-1.0673049688,-1.0099349022,-0.7698056102,-0.1967914104,0.4551593065,0.6804695129,-0.2212719023,0.1560983211,0.2077799588,-0.2559347749,-0.6590756178,1.8647357225,1.4707669020,-1.7711633444,-0.0389776789,1.2798991203,-0.2931283712,0.4195695221,1.2389466763,1.4354935884,0.8050691485,-0.7181984186,-0.5427057147,-0.0143227829,-2.0996563435,-0.2624518871,-0.0271753389,-0.4848319292,0.1522143483,-0.3941994607,-1.3695958853,0.8463138938,0.7625041604,-2.6974196434,0.1153355688,0.9856745005,-0.1841914952,-1.1387770176,0.5851966143,0.9297739267,0.7000173926,-0.7721249461,-1.0423361063],[0.6085122228,-0.1843611747,0.2995264232,1.6914433241,0.1621672064,-0.6695007086,-1.5838277340,0.1919981539,-0.2652933002,1.2727167606,0.7557719350,1.3111314774,1.0354379416,0.0457438529,1.8368517160,0.5285716057,-0.0358069725,-0.3260233402,-0.0109968940,1.7571246624,0.8397163749,0.0457721874,-0.2342656553,-1.1719472408,0.4112628996,0.4550276399,-0.2209479064,-0.3792432845,-0.5196670890,0.6338948011,-0.4687461257,0.3789885342,-0.6974380612,0.6244715452,0.6647689939,0.1989351660,0.9490326047,0.9538723230,-0.3232532144,-1.2485311031,-1.1898362637,-1.3543343544,-0.1478466094,-1.7027926445,-0.2219823599,-1.3504548073,-1.4105155468,1.4693017006,-0.9841487408,-0.1953340322,0.8112298846,1.3687382936,0.2290214747,-0.8213371038,-0.5722841024,1.8054988384,-0.2536306977,0.0729738325,0.4649703205,0.0422217771,-0.3143117428,1.3984079361,-0.0015608553,-0.5787816644,0.5725606084,-0.3906839788,0.0330548063,-0.5729920268,-1.3764246702,1.0837403536,0.8048295379,1.1178059578,0.9453338385,0.0617002957,-0.6828507781,0.8997985721,1.0651228428,-0.1373525262,-1.5338754654,0.1621236652,0.2117942125,-0.9399450421,0.9874948263,-0.6034541726,0.1361092329,-0.3756698966,-0.0457318723,-0.6480113268,-1.4728063345,-0.3526277840,-2.5419843197,0.2226303667,-0.0023718607,-0.2957418263,0.4968234003,0.7623850107,0.4839349389,0.0289388001,-1.1439795494,1.4620765448,1.2326036692,-0.9133926034,0.4964179397,-1.0620882511,0.7359864712,-2.2167191505,-0.7982082963,2.1331815720,-0.9304955602,0.1575941294,-0.7669993639,-0.3956198692,-1.8070354462,0.3882973194,2.8525314331,0.9374451041,0.9887226820,-0.6674115062,0.2041293830,-0.6884194613,1.2304478884,-0.2918583751,0.3572739363,-0.4392817020,0.7722851038,-0.2283833772,0.2234459370,0.6006508470,-0.5135402679,1.1010036469,0.0592561252,0.9372741580,1.2490446568,0.0454107709,-1.2143180370,1.2831765413,-0.2877854705,1.1066776514,1.7132198811,-1.5615347624,1.2326842546,-0.6733837128,0.5636820793,0.2575648725,1.9115114212,-0.6896926165,-0.9419258237,-0.0117741860,0.1123029664,-0.2908022702,-0.3897866011,1.2393567562,-0.5410919189,-0.0594159551,0.5686154366,1.2410963774,-0.5654718280,-1.1199505329,-1.6332823038,-2.3909821510,0.7570685148,-0.1796463877,-1.0146815777,-0.6059055924,0.8664308786,-1.8528240919,0.6743731499,-0.6671823859,-0.8835974336,-1.0965219736,1.5275717974,0.7300167680,-0.2790175378,0.7463711500,0.1410504282,-0.8136600256,0.7124322653,0.6585643291,0.4684002995,1.6137416363,1.1167794466,-0.6949005127,-0.1380605549,-1.5743788481,0.1458098441,0.1653126776,1.1618109941,1.2810046673,-1.1934560537,0.2221141905,-1.3247392178,-0.9758756757,1.7848703861,-0.4239293635,-1.5496392250,-1.0143722296,-0.9373916388,-0.0033001278,0.4032337964,-0.0522733368,0.5052266717,-0.3572324216,-2.1708812714,-0.8520315289,1.2847210169,-0.5304069519,-0.4453616142,-1.2954775095,-0.3580422103,-1.8790763617,0.9117733240,-0.7384397388,0.3458448648,-1.1715149879,-0.5691010952,-0.3036805391,-1.4098423719,2.2985846996,-1.0172985792,0.4456582069,0.1196669713,-0.1597136408,1.3163573742,0.6440555453],[-0.4741426706,-2.1343107224,0.7861979604,1.0434623957,0.1084718406,-0.0941078067,-0.1117874235,-1.9170396328,0.3132521510,-0.3455376625,-0.5181628466,-1.0792073011,1.1498949528,-0.3038112223,0.3882457614,0.5555512309,0.8595563769,0.9541366696,1.4233583212,-2.9718353748,-0.7919180989,1.3287876844,0.6337758303,-2.0233647823,0.1994745880,-0.0833952278,-0.8845870495,-0.2299005687,0.3188984990,0.4911392033,1.5346169472,-0.2562862933,0.3037653565,0.6775609255,-0.2594868243,0.9171200991,-1.1272230148,-0.0771622583,1.9170622826,0.0905564949,-0.1810506284,0.0498206317,0.1376063973,0.8997771144,1.0737489462,0.0820906535,1.3511236906,-2.0293605328,-1.1584099531,0.2168816030,-0.0768081844,-0.5279827118,0.7440552115,2.0216920376,-0.1949700266,1.0104807615,-0.7675292492,0.1056777388,0.4347531796,0.7299080491,-0.7642510533,-0.0106105693,-0.2643010318,-1.8072508574,1.3940663338,0.3064333797,0.4452610612,0.6079790592,0.3127636611,0.5288425088,0.0853652284,0.1245446801,-1.2992873192,-1.6149802208,-1.0670560598,-0.6736964583,0.6929746270,-0.7262415886,0.9602357745,-1.0777587891,-0.2623558342,0.2148114890,0.8735753298,0.0214401037,-1.0976064205,-0.4945487976,0.7895449996,0.2681889236,0.7866907716,1.1217195988,-0.0987510309,0.4238417447,0.6273287535,0.4225327671,-0.8348941207,1.6016408205,1.5246837139,1.4992862940,1.3512203693,0.9674935341,0.0298249535,1.2098042965,0.8318428993,-0.2006926239,-1.1404662132,-1.2379252911,-0.3444960713,-0.5028432012,1.1103632450,0.7753596306,-0.6887470484,-0.8987953067,-0.0610145554,1.9189139605,-0.1576679349,0.0352119133,-1.6327770948,-0.4427270293,0.7562611103,-0.2466283292,-0.0400478803,0.9635592699,0.5934650302,-0.8815698624,-1.0712459087,-0.6394456029,2.0791168213,-1.5084906816,0.1330436021,0.5975986123,0.3697303534,-0.6510151029,-1.2109297514,-1.4117516279,0.4338284135,-0.0864954367,0.0024174221,0.9538350105,-0.4261922538,0.4513969421,1.9549031258,-1.6480233669,-1.2982463837,1.6858726740,-0.8666954041,-0.3963478506,-0.8370561600,-0.6677888036,-0.0802014470,-0.9439086914,-0.4302936494,-0.6549695730,0.2192734480,1.8352367878,1.7390035391,-0.4004605711,0.3200043440,-0.2482670993,0.8343432546,0.9010576010,-0.6557250619,-0.3174437582,-1.4815505743,0.3341366053,1.3810726404,0.3408678174,0.2780138254,0.4463123679,-1.0640423298,0.5948966742,0.9701595902,-0.3354811668,2.1228146553,-0.3853601515,0.8973402977,-1.6078014374,0.4986139536,1.5752862692,1.0740270615,0.5252494216,-0.9065878987,-0.3239499032,-0.8208144307,1.4252241850,0.7748389244,1.6811554432,-0.4814466238,-0.4224803448,0.3014148474,-0.4887431860,0.5419479012,-0.2225292623,0.3341558278,1.3281743526,-0.7061454058,1.1406314373,1.7456609011,-0.3581234515,1.8958735466,-0.9984493256,0.9563930035,-0.5737659335,0.8312965035,-1.0460712910,0.2084058076,-1.1228160858,-0.3064347208,0.0921172574,-0.1723273247,0.2616755962,1.8572127819,1.2639824152,0.0297166966,0.7248386741,-1.0261186361,0.6415510178,0.5504319668,0.0855867118,-0.2484924197,1.0947587490,0.6734647751,0.0738114491,-0.6452680826,1.9789581299],[0.3102551699,-1.3629002571,0.0274566971,-0.0190395452,0.2973560095,0.1557310671,-0.1468652487,1.4216049910,-1.3144340515,0.8182174563,1.2978197336,0.6167443395,0.2798875868,-1.1098065376,-0.4280781448,0.2754967213,-0.0831788480,-0.6103398204,0.4578257203,0.8553571105,-0.5358012319,-0.1420752406,0.3383913934,-1.3604153395,1.0594539642,0.6419425011,0.3380404115,1.0207705498,0.6028524637,-1.0487345457,0.3747584224,-0.9873114228,0.5715779662,0.8900516033,-0.3863794208,-0.3268699944,1.1698690653,0.8926932216,0.2617246211,0.1278757006,-0.8974625468,-0.4251092672,0.7840434313,0.2114642859,-0.4872782528,1.1878559589,-0.0447122194,-1.6312956810,0.9626309872,1.5028502941,-1.1069815159,0.5493133068,-0.0670908988,-0.9915824533,-0.5925250053,-0.9678844213,0.6036627889,-0.1400649548,-0.9178651571,-0.0356261842,1.5151066780,-0.1383053213,-0.3510780334,-0.4292304814,-0.5429307818,-1.5706568956,0.1042975038,0.7102490067,1.8552374840,0.3347614706,-0.7081097364,1.1634874344,2.1036181450,-0.7794679403,-0.5313645601,-2.0222191811,-0.6882897019,1.2757804394,0.9143882394,-0.7780158520,1.6167298555,1.0925157070,0.7997589111,-1.6580874920,-0.3671254814,-0.1214561462,0.5333232880,1.8171951771,-0.5902642608,-0.7296072245,1.4643578529,0.6638869047,-0.5257128477,-0.1212170571,-0.9509884119,-0.6017122865,-0.3032298088,0.3008897901,1.7487587929,-1.4716196060,-0.4065019786,0.1656065583,-1.1072632074,0.7372105122,-1.1134939194,0.6655398607,0.5232836604,-1.3998881578,-0.2136789113,-0.3169328570,-1.0540878773,0.0523890071,-0.3415353894,0.2425192446,0.2672145963,-1.0849430561,0.1197969317,1.6511855125,0.8567194939,-0.4198167622,-0.0061530750,-0.2666466236,-2.3022949696,-0.1626988351,-1.1963500977,0.0330941156,-0.2138646841,-1.1792644262,-0.8235653639,-0.5738009810,-2.5206964016,0.9328417182,1.1077494621,-0.1811641604,0.2459245920,-1.6585836411,-0.5189835429,-0.2116440684,-1.9794597626,-0.3549507856,-1.1766176224,-0.1413746923,-0.9840304852,0.2118366063,0.1956791133,0.0256031454,-1.7828342915,-0.9711146951,0.9384088516,-0.9248518944,-0.1011454016,0.3087741137,-0.8797972202,0.3986951411,-1.6176860332,1.1128709316,-2.2385592461,0.6260812283,0.1871027201,-0.0254999250,0.3770545125,0.3706595004,-1.7061337233,0.3555938900,0.3026677072,2.3950519562,-0.8477203250,-0.2803919017,-0.5236788392,1.4797307253,-0.2061602622,-1.2524269819,-0.4490454495,0.3581798375,0.4895872772,1.5629991293,0.8327241540,-0.3907397389,-0.6531038880,-1.0658258200,0.8748501539,-0.7687430978,-0.7552185655,-1.2149530649,1.1354509592,1.5423836708,-0.5928421021,-0.6696397662,-0.8266085982,0.5995821953,1.6138184071,-0.3615201712,-0.8327620029,0.4664305151,0.9396260381,-0.6726012826,-0.3961285055,1.1466346979,-1.1610549688,-0.3767644167,1.1379449368,0.1347682774,-0.0495987386,1.2985827923,-0.7667092681,-0.2375340462,-0.7819229364,0.4346813858,0.9445708394,-0.5190451145,0.9228600860,-2.0988132954,-0.2211748511,0.0915240720,0.5281891823,0.5730536580,0.3545318246,0.9084903002,0.7303026915,-0.6365323067,-0.2045491636,1.2701267004,0.0289865974,-0.0711520687],[1.9361691475,0.1886532605,1.2679781914,0.7062788010,0.2168968320,-0.2863503993,0.2347160429,-0.6281237602,0.0405537225,-0.2235246599,-0.2665000856,-0.8118832111,1.4879057407,0.3624395132,-0.2154130787,-0.6729236245,-0.9089426398,-0.1334685832,-0.1044594646,-0.6322259903,-1.7956866026,0.1443674415,1.1078892946,0.1278653294,-1.4188404083,-0.8868185282,-0.3755210340,0.6156448722,0.2973950207,-0.3261060119,0.3075840473,-1.0595344305,0.6999576688,-0.1738804132,1.1124271154,0.0801854208,0.5716896653,-1.0374674797,-0.0721909776,1.9828381538,-1.3460509777,-0.5868654251,2.1197776794,-0.8811689019,-0.5918867588,1.3096985817,0.5816102028,0.6569513679,2.4478139877,0.0861110836,0.5811345577,0.4229134023,-0.3605940342,1.5369105339,0.4504128993,0.3325541914,-2.6560053825,0.0780802891,-0.5812519193,-0.1294345409,0.2331547439,0.5030118823,0.5096856356,1.6935214996,-0.5959262252,0.2305141240,1.0273890495,0.2305488288,0.8759493828,-1.8008491993,-0.7317401171,1.7506126165,0.4269411564,-0.0429548435,-0.1476143003,-0.2250723988,1.7548139095,1.6987363100,0.4552177787,1.5571171045,0.6796240807,0.2440751046,1.0473498106,0.5143606067,1.4031444788,-1.2559754848,2.0805716515,-0.0157355964,0.3213089406,-0.2585878968,-0.9314686656,-0.2365136594,-0.5920739770,-1.4044491053,0.3690233231,-0.7666178346,-0.3709909916,0.2319030166,-0.9386507273,-1.5684202909,-0.9699392319,-0.1636213660,0.1689850539,0.2926900685,-0.6215610504,1.4952127934,0.7245684266,0.4298216999,-0.6791428924,-1.4297268391,0.2240037620,0.8946802616,-0.6121846437,-1.0005315542,-0.9501293302,-0.2909827232,-0.5344090462,2.3198983669,-2.2879838943,-0.9288844466,-0.0034081170,1.0812544823,-0.8310899734,0.8552720547,-1.0219707489,0.2177965343,-0.9733440280,-0.0174913909,0.4193081558,-0.3332124054,0.4880933762,-1.5816885233,-1.4639986753,-0.6394988298,0.2832532227,0.1952934414,-0.8858428001,-2.3023381233,0.0895171762,1.8412590027,-1.1203542948,-0.8483638763,0.2196273506,0.3113439679,0.1000621319,1.1735875607,-0.8860414624,-0.8599038124,-0.6715561748,0.1867471635,-0.2337619066,-1.1266306639,0.9036043286,-0.9000715017,-2.0205488205,-0.9581099749,0.7905055285,-1.2348706722,0.4671658576,0.4668186605,0.3785419464,-0.0996996388,-0.1534485519,0.4217807949,-0.4923402369,-1.4948923588,-0.1059649363,-1.4504761696,-1.2151101828,-1.5624502897,-1.6301554441,-1.0333613157,-0.6717498302,0.1625663489,-0.9796378016,0.2504408658,-0.4498012662,-1.3315489292,0.5227864385,-0.4967396855,-0.7746353149,-3.2565047741,-0.7625097036,-0.2435188144,-1.6264389753,0.2406946421,0.3750441670,-0.5929222703,-1.7938333750,-1.2085139751,0.0064371475,-1.7667037249,1.7038869858,0.4753684402,0.5051966906,-0.2259977311,1.6209121943,-0.4586403966,0.4619432092,0.7081202269,-0.1198732406,-1.9040656090,2.5461783409,-0.5098463893,-0.5149093866,-0.6470791101,0.0598744750,-0.0804928392,-1.1862183809,0.7540088892,0.5231956244,-0.7941627502,-0.5351686478,0.1464129686,-3.2503798008,-0.9900084138,0.0208800603,-0.0939341038,-0.7024986148,-1.4116461277,3.3354561329,-1.6807198524,0.7628878951,-1.1663935184],[-0.9231708646,0.5788217187,-1.5420737267,0.3719091415,0.4810803533,1.2950202227,1.1624336243,1.7827768326,1.2083466053,2.1456758976,0.0785913095,-0.7186422348,-2.0778264999,-0.7815722823,0.2554871440,-0.2430197746,0.3738383651,-0.1691911221,-1.5751345158,-0.6850867271,0.9226065278,-0.4163829684,0.2016043663,0.2430605590,-2.0118079185,0.6240944266,0.4783047438,-0.2266190797,-0.0495828427,1.2547620535,-0.7426257133,0.0852039978,-0.3938572705,0.3231543005,-1.2266924381,-0.7905484438,0.7264937758,-0.1436624080,0.6927309632,-0.1799502820,0.7293747663,0.2032651603,0.9094336629,-0.4490212798,-0.5067827702,0.2291303724,1.0174266100,-0.5093548894,-1.4442439079,2.3562941551,1.0144230127,0.8205124140,-0.3604024351,0.5040035844,-0.0327425338,-0.9325399995,0.7155613899,0.1095617339,-1.8171060085,0.1261477470,0.8790358305,0.9427734017,0.8065360188,0.9868785143,-0.6248038411,1.1574034691,-0.4969314933,-0.9548960328,0.2056397945,0.6338357329,0.7086056471,0.3203323781,-2.4203219414,0.5915226340,0.7968711853,-0.7548759580,-0.7817936540,1.4170876741,-0.5229552388,-0.2530987263,0.1287870258,0.1348793209,0.3662788272,0.9481594563,1.5518367290,-0.1748871803,1.5927872658,-0.5910189748,-0.2056038827,1.4960036278,1.5196142197,0.6698520780,-1.1990022659,0.4433546960,-0.9905802011,-0.8024225235,-0.3416491747,-0.1046441644,-1.0017228127,0.4857330620,0.3659887016,0.1782243699,2.0004279613,1.5760009289,1.3417978287,1.2599104643,-0.5527366996,2.0692360401,0.4941767156,2.0470237732,-1.0875812769,0.7576946616,1.7907387018,-2.9291129112,0.8575381637,-1.3554950953,-0.4839210510,-0.0655705929,-1.1057314873,-1.3319424391,1.8408344984,-1.6634352207,-0.4494746327,-0.1036276221,-0.6675081253,-0.5608288050,0.0666391999,-0.5574812889,0.2405882627,1.4979455471,-0.3060584068,0.0120285843,1.6143784523,1.1359167099,-0.9106744528,1.2418780327,-0.9291626811,-1.9791833162,-0.9713079333,0.3243151903,-1.4647428989,-1.2395029068,0.0382350758,0.8031321168,-0.8530051112,0.4882732034,0.8558873534,0.6460304260,-0.5682061315,-0.9189545512,0.8990505934,1.2603000402,-1.0781060457,-0.7336502671,0.8740951419,0.7658302784,-0.2238070220,0.6157169938,0.6769030094,1.2593274117,-1.8752324581,-0.0772253424,-0.2439778447,1.8605688810,0.3804485798,-1.8733338118,-0.5691103339,0.5537204742,-0.6699575186,0.4479410946,1.0299412012,0.7603972554,-0.4966503382,0.3634217978,0.8501811028,1.6847851276,-1.2144894600,-1.4988577366,-1.1940227747,-0.5858005285,0.6833459735,0.4837764502,0.8482929468,0.9312326312,0.5123922825,-0.9934224486,0.7389886379,1.5904425383,0.1515328288,1.8001233339,-0.5165967941,0.6810074449,1.1736761332,0.4009504914,-0.1953288466,0.5581561327,0.1997068077,2.1620819569,-0.0908827558,1.0417481661,1.6549855471,-0.7760302424,-0.3316659927,-0.2115495950,0.5966792107,-1.0887472630,0.2982098758,1.5184670687,0.4310917556,-1.4303095341,-1.6879490614,-0.5256771445,2.3723435402,-1.1201488972,-1.6246181726,-0.7182115316,2.2989356518,-0.7916751504,-0.1085834578,-0.0711057559,0.3724485934,0.3015347719,0.2816698551,-0.5369709134],[-0.7400909066,-0.4385055602,0.4664757252,0.7917868495,-0.0000355124,-1.0004467964,2.1101815701,0.9932739139,0.8110388517,1.0303093195,2.1443128586,0.2792387009,0.8437840939,1.1218912601,-1.3918541670,0.4506911635,-2.0793647766,-0.0121497940,-1.0440405607,-0.9657757878,-0.5087488890,-0.0789047405,1.4459255934,-1.5358483791,-0.4629052877,-0.3632760942,-1.0295374393,0.1215865463,-1.5990295410,0.9378693700,-1.1836384535,-1.0631419420,0.6355529428,-2.4377219677,-0.0904419720,-0.4806424975,0.2118658572,0.4217100441,0.1558341831,-0.9841300845,-2.7493028641,0.9413385987,0.3539286554,0.6233276129,-0.9138478041,-0.5024273396,-1.6548289061,1.0006732941,1.9455966949,0.0933097750,-0.1363495439,0.0907459259,0.0056755515,0.3079532683,0.9652287364,-1.1859726906,2.0238280296,0.2781577110,0.5562726259,0.7591621280,-0.1311170906,0.9709328413,-0.8000256419,-0.3964389563,0.3583988249,-1.2603985071,-0.6288074255,0.1997476220,1.5333764553,-0.9561508894,-1.1526873112,-2.0702159405,0.2626678348,2.6558797359,0.3057968020,-0.0556512699,0.4347166121,-0.2524664700,0.5330020189,2.1679821014,-0.0680899248,0.0938287526,0.3151662648,1.3177949190,0.2194497287,1.4449121952,-0.2030343860,0.3360009193,-0.9564311504,-0.8714737892,1.4866508245,0.6670100093,1.0294823647,-0.8069849610,0.7517868876,0.6363819838,-0.2490818948,-0.2208531499,-0.7862807512,-0.0412395597,1.8023437262,0.8893542290,0.8283433318,1.1427662373,1.0990648270,-0.1380199790,0.0603353344,-0.6218895912,0.6183639169,1.6191837788,-0.2276409566,-1.7644103765,-1.5434100628,0.5247124434,0.5242751837,0.7079523802,0.1106121019,-0.4096989632,-0.9592539668,1.7692834139,-0.4390759170,0.9115946293,3.0482261181,-0.0887733400,0.4471243024,0.1182788983,-0.0481228195,0.2647619545,0.1651706994,-1.0986613035,-0.2626098692,0.8710223436,0.2023158371,0.0185922198,-1.6146917343,-0.7584802508,-1.8991247416,0.1000484303,0.0444046557,-1.0100754499,2.0759148598,0.6520631909,2.0937871933,-0.1748518348,-0.0945110694,1.4562783241,-0.3171853125,-1.3235573769,-1.0152374506,-1.1026420593,0.2675947845,0.6974791884,-1.4816414118,-0.5089939237,0.8205317855,-0.6032758355,1.1610811949,0.2848510146,-0.2113544494,-0.0693584830,0.4601627290,0.2715222836,0.3543163836,1.1241178513,-1.0009111166,0.4233255386,0.8821615577,-1.2673573494,1.0599933863,-1.9858329296,1.7473354340,-0.8925181627,-1.5392049551,0.3679618537,-0.9061989188,1.6234297752,0.3026809692,-1.2521638870,-0.9746393561,-1.6885068417,0.1270830184,-1.1797159910,-0.1793961972,-1.6843693256,1.1783792973,1.2475259304,-0.2204871029,0.7822663784,-0.7288885713,0.5897322297,1.5894587040,-2.0798897743,-0.9088625312,1.0042586327,-0.4040690362,1.0435874462,1.0213680267,-0.5034691691,-2.8381204605,0.5957017541,-1.7376792431,-0.3987622857,-0.5514874458,-0.3678982854,-0.1919072121,0.9552351236,-0.2671971619,-0.7120574117,0.7922269106,0.7821462750,-0.8510670066,0.4528163671,-2.8724989891,-1.7404062748,-0.0358726680,-1.0349525213,0.9958155155,0.7042294741,1.8772702217,1.2644282579,0.5117878914,-0.7683165669,-0.3760642409,0.3992835879],[-0.6319594383,-0.1234124750,1.3657724857,-0.4243614376,0.3409079909,0.8824703693,0.3096629381,1.1415899992,-0.3601651490,-0.4110813737,1.4603716135,0.2897032797,-0.6120595336,-0.6877930760,-0.0798668489,-0.9543554783,1.1657077074,-0.7224388719,0.5840330124,-0.4707277417,0.9872319698,-2.4506506920,0.0268113595,-0.4014832675,-0.9910459518,1.8364913464,-0.1808601767,-0.6913096905,-0.4763865769,-0.8570129275,0.6800124645,0.2375830114,0.2993802130,0.6163609624,1.0929050446,-1.2402175665,-0.9390913844,-1.3111484051,1.5088058710,1.3480857611,0.5016481280,-0.2476610988,0.7036614418,-0.0966909081,1.3176031113,1.0656447411,0.2396778166,-1.9526786804,0.8595772386,-0.2595057487,0.9080550671,0.0765460432,2.1173491478,-1.0088042021,-1.7454794645,1.2033076286,0.4952604175,0.1535671651,1.0042064190,-0.2208959013,0.5841389298,-0.3922788501,-0.3614006042,-1.8522236347,0.2663365901,-0.4901424944,-0.7449171543,-0.3399548233,-0.3400003314,1.3681870699,1.0906018019,1.0649048090,1.2599071264,-0.1997389942,-1.5190547705,0.9784672260,-0.8858841658,-0.7331888676,1.7371307611,-0.9528511167,1.2391638756,-0.6803607941,-0.8854863644,0.5288819075,0.0407255515,-1.7051877975,-1.1911907196,-0.6170998812,1.8401297331,1.2869774103,0.7416225672,0.1012784541,0.9506714940,-0.6890847683,1.1691758633,0.2125397325,-0.5651474595,1.1363813877,0.3282095492,0.2325947732,0.8333733678,-0.3224644661,-0.0435054190,0.8779395223,-0.1092095077,-1.3758436441,-0.8854057789,-2.1870152950,1.4359880686,-0.5213032365,0.3130353987,-0.3004281521,1.5077716112,0.7161954641,0.4850605428,-1.3312293291,0.1999393255,0.7143920064,-0.2711220086,1.0470339060,-0.5351563096,2.1122450829,2.2901554108,0.1676862389,-0.3756659031,0.2262535393,-0.4149861634,0.7650973797,-0.7531386018,1.2070817947,1.2787936926,0.1595245451,0.1642178148,0.2609710991,-0.7997441292,-0.1890465617,0.4980888069,-0.8788653612,-1.4926164150,0.2842349410,0.1910644174,1.9113937616,1.1137232780,1.0943866968,0.9748689532,0.8634682298,-0.2590862811,-0.0192987844,-0.8789740801,-0.7913516760,0.6180747747,-0.7121517062,0.3878835738,-1.3793500662,-0.1918176562,0.0958151296,3.1094701290,0.3986200392,0.2136057466,1.1821939945,-1.1471401453,0.8588299155,-2.0046691895,-0.1928652674,-0.6466591358,0.0150989927,-1.0306770802,0.9040096998,-2.1417479515,-2.7447814941,-1.1748067141,0.5511487126,0.0278319884,0.3978701234,-0.4533764422,0.6338236332,0.2078187764,0.6372888684,-0.6748731136,-0.1286127865,-0.5645707846,0.4383159578,-0.7623828053,0.7904329300,-0.1006841809,-0.4627493024,1.0368299484,0.4300449491,-1.7973104715,0.7144021392,-0.4045287371,-0.8542004824,-1.1299022436,0.7849571109,1.1772171259,-0.8059206009,-1.1383796930,0.4343900383,-0.2633340955,-0.4062776566,0.1033182815,-0.5581795573,-0.0883774608,1.0320638418,1.4132361412,1.6514214277,0.1695428044,0.2159408629,0.8343988657,-1.1636316776,1.0585942268,-0.1515603811,0.8289833069,-0.0556437746,-1.2486417294,-0.3393330276,-0.3956955671,-0.1733885854,-0.6103331447,-1.6589229107,-1.3826439381,0.8436100483,1.1876813173,-0.5175749063],[-1.0741704702,-0.2104587704,0.8762942553,-1.0773400068,-0.2500136793,1.3766224384,-0.4211629629,1.6774389744,0.4335694313,0.1595920026,0.7415378690,0.6724870205,0.3878264725,-2.0489420891,0.8812448382,0.1120989919,-2.1312644482,-0.3068031967,1.4072669744,-1.8038721085,-1.9736406803,0.0415054634,-0.0308436342,0.9076535702,0.3400208652,-2.2132844925,2.3224930763,1.3219075203,2.3032946587,0.6108966470,-0.1145442501,-1.5732045174,1.1812853813,0.2263955772,0.4435280859,-0.4413294494,-0.2191576660,-0.4763602614,-1.1287884712,0.9238072634,-0.2126406729,-1.4508607388,-1.0319195986,0.1644870937,-1.5002006292,-1.2616881132,-1.5085011721,0.1622222662,0.4869650006,0.5460504889,0.1898836493,2.0961461067,-0.3474938273,1.8628062010,-1.1646004915,-0.9452017546,-0.5434473753,0.2812504768,-0.5410401821,1.7726508379,0.5803670883,1.0367780924,-1.6005064249,-3.0092163086,0.1392125040,2.4194037914,0.3260611594,-0.9767702222,0.5747098327,0.1420334429,0.2335471809,-0.0718801618,1.0421361923,0.5592299104,0.5113453269,0.2498618364,-0.9558623433,-0.3218068182,-0.1402815431,-1.6505886316,0.0835994780,0.9163863063,-2.8156573772,-0.7224086523,-0.2607764006,-0.7745448351,0.7839495540,-0.7016636133,-1.2114204168,-1.3935538530,-0.1395538896,-0.5087388754,0.8473874331,1.3882595301,2.1699202061,-0.2208291292,-0.3559795022,-0.7718069553,1.0199204683,-1.1986038685,1.2818330526,-0.8575738668,0.7588049769,-0.3571656942,-0.8376092315,-0.1244146153,1.0222516060,-0.2545220852,1.8373160362,-1.4680243731,1.3336762190,0.2705392838,-0.7611726522,0.6549308896,-0.0792744756,0.9494455457,1.1246320009,0.0557211973,0.8274643421,-0.0141875586,-0.4157760143,-0.2442740798,-0.1689703465,-1.4288641214,0.2172202319,0.7753660679,0.6355167031,-0.6720003486,-3.3398942947,-0.3721321821,0.7748588324,0.2912347913,-0.0677062869,0.2498321682,-2.0455522537,0.3599575460,0.8134485483,-0.7892555594,-0.9455010891,-2.1506998539,-1.1472947598,1.6073971987,0.3767109215,-1.2686046362,-0.3529419899,2.0551450253,0.9534871578,-0.3973580897,-1.4923707247,-0.5456050038,0.0431090258,-0.2758043110,-0.2455203533,2.1469547749,-0.9647199512,-2.1054956913,-1.6152031422,0.6804012656,0.7725505233,0.6639801860,-0.5187103152,-0.6598019600,-1.0491077900,-1.5657329559,0.5533331037,1.2845594883,1.1052020788,0.9238629341,-0.4194052517,0.5052201748,0.6276740432,0.4903647304,0.5449354053,0.8436488509,0.5930407643,-1.2337615490,-0.7152583599,-0.7161428332,-0.5219246745,1.1452192068,0.6020815969,-0.6623034477,-0.3540771306,0.2188376784,0.2990494668,-0.3308923244,0.3350869715,-0.0748877674,-0.0690385029,1.8809734583,-0.6586229801,0.3756047189,0.4063786864,0.9886311293,-0.9031563401,-1.2128840685,-0.5440398455,0.8093140125,0.1141676530,-0.3551281095,-1.1269905567,-1.5992106199,-1.0362365246,-0.3802249730,-0.0893609747,-1.0580019951,1.2933925390,-1.9946708679,-0.7844277620,1.2969027758,0.4970777929,0.8003904223,0.2300457060,1.2736281157,-1.0021532774,0.4828888178,-0.1670626998,-1.8273473978,0.2340641320,-0.6276967525,-1.6696863174,0.0048749070,0.9814209938,0.6604838967],[-0.5587424040,-0.5823022723,0.1978456527,0.8078650832,0.8510124087,-0.6664382815,0.8562269807,-0.6985203028,-0.3025896847,-0.7559245229,-1.0003705025,1.5326434374,-0.0137318987,-0.6858326793,-0.3156582415,0.4162408710,-0.6101112366,0.4867403209,-0.3826802969,-1.2199368477,-0.4200175107,1.0042294264,0.3964896202,-2.3662467003,0.9675166011,-0.3118633628,0.0873739272,-0.4918978512,0.6635258794,0.3169205189,-0.3862748146,-0.7754532695,0.7976115346,-0.9872649908,-0.9692402482,-0.2291807681,-0.7056533694,1.3953976631,0.3147648871,0.4875667989,0.6540688276,-0.9398791194,0.7224289179,1.1777349710,-0.9915122390,0.9375629425,-0.5861776471,0.4290914237,-0.8124514222,-0.2063853294,-0.9410168529,-1.1546858549,-0.1660162508,-0.2981323004,1.5257365704,-0.6887041330,0.4322051406,0.0187902357,0.3851979673,-1.5070592165,-0.0877329856,1.6083472967,-0.7919498086,1.7719179392,-0.5008546710,-0.7702132463,1.5552997589,0.0259561371,1.3426429033,0.0170335490,0.5972861052,-0.3192002177,0.6835110784,0.4572851360,1.8811956644,-2.5470488071,-0.4798873067,-0.4972526431,-0.4892331660,-0.7172479630,-0.0641170889,-0.0721785799,-2.2250163555,-0.7003098726,-0.9463029504,0.5387721658,0.1171965152,0.7794079781,0.9873548150,-1.2788540125,2.0102021694,0.1099134684,-1.4208669662,0.1849313974,-0.2944383323,1.1591405869,-0.2540302277,0.5338650346,2.5842995644,0.4451530874,-0.3505676389,-0.1859664619,-0.1787521392,-0.0542083494,1.6706476212,0.9630910754,0.6391386390,1.6442874670,0.0071885237,-0.5470194221,-0.0029828497,-0.0276721809,-1.6777534485,-0.8231400847,-1.2297281027,-0.7362014651,1.1312052011,-0.3959090114,0.2928639650,0.0256736502,-0.9436841011,-0.8668084145,0.1719511896,1.7600250244,-0.3097595572,-0.7261739969,-0.0894311666,2.0879607201,-1.0827295780,1.1696344614,-1.8928852081,-1.4066766500,-1.6461080313,0.0934822261,-0.0529018976,0.1754294932,-0.4250826538,-1.6005401611,0.2298563123,-0.5026178360,1.5759580135,1.0479354858,-1.3923493624,1.5411509275,0.5450043082,-1.5957812071,0.0078355968,-1.0235074759,1.4455271959,0.2516079247,-0.0059909956,0.5103890896,0.6152398586,-0.1133571044,0.0452356748,0.8750831485,-1.5508222580,-0.8830829859,0.2080126405,0.0176873859,0.0009350795,0.3643147647,-0.3425868452,-1.1389331818,0.2679989040,-0.2351417392,-0.3743028641,-0.2967391312,-0.3949703872,-1.2074548006,-1.5331141949,1.0537494421,-0.6973699331,-0.3979943097,0.9710779786,0.1486331820,0.7276759148,-0.3589974344,0.1361308545,0.5942563415,-1.1173524857,0.1162814796,0.1708787680,-0.9081650972,-1.2323310375,-0.2043872625,-0.9505392313,-1.7209446430,0.0769074261,0.3739518523,-0.9559456110,0.6215204597,0.5835911036,2.1823852062,-1.1971487999,0.4832358956,0.6317840815,-0.8399761319,-0.8457883000,-0.2453269064,-1.3662616014,0.2259954810,-0.8711085320,-1.0088886023,-0.0795122460,-0.2749667168,-1.4504717588,-1.1735156775,-0.1123355925,-1.2269114256,-0.8250812888,0.0331604481,1.8441269398,0.6902368665,0.6502194405,0.4802848697,-0.4361600578,-1.9481081963,-0.9514036179,-1.0143113136,0.4540541172,-1.0325177908,-0.1135502532,0.8384748101],[-0.1321249306,-1.8348624706,-0.9096595049,-0.0661195368,-1.1773709059,-0.2655218244,-1.0226361752,1.7931406498,-0.5638654232,-0.8700715899,0.1031277925,-0.3818658292,0.9453787804,0.2722228169,-0.4883063138,-0.7425808311,0.3954738379,3.5413982868,1.1422576904,-0.1177065670,-1.3207576275,-0.4200673103,-0.9338312745,0.4874849916,-0.7146152854,0.5772513151,-0.2517969310,-0.5349732637,0.9662621617,-0.6398757100,-0.2928552926,0.3574216366,-1.0193123817,-0.5647442937,1.3439898491,0.9467440844,-1.8994320631,-2.3763782978,-0.0155165065,0.9333648086,1.2560585737,-0.5713263154,-1.1139186621,-0.6125965118,-1.5679283142,2.0549027920,-0.7713843584,-0.5557987094,-0.5585767031,0.1265952587,0.9497575760,-1.6862744093,0.4159049392,1.6351566315,-1.0996099710,0.1305427402,0.2132267654,0.2193509191,0.1171146780,-2.9186856747,-1.6857429743,1.2380404472,0.1488410085,-0.5628337860,1.2209196091,-1.0582157373,-0.4046570361,0.1282916665,-0.8336083293,0.6628674865,-0.6343099475,0.4895289838,-1.5728354454,0.1174273640,-1.0989309549,0.7623879910,0.6364852190,-1.3394111395,-0.9577367902,1.2531023026,-0.0068191397,-1.0997195244,0.5822053552,-0.8811379671,-0.9198532104,-0.0741320997,0.0454016849,0.2210511118,-0.3389450312,0.2945863605,1.6904438734,0.2295662761,0.3174210787,0.8400560617,-0.5025610328,0.4230135083,2.0097754002,-0.6062899828,-1.3862564564,-1.6698477268,0.5712623000,0.1846959591,1.3817620277,-0.4764849842,0.0659929290,0.3075664937,2.1360414028,-0.3807438016,-0.2041121721,0.4065637887,-1.4117066860,-0.2051517218,0.6111496091,-0.4619669914,-0.3576725721,0.0207850803,0.5707749128,-0.5527526736,-0.1327189803,1.0202493668,0.6725012064,-0.1995927095,0.4116949141,1.3201146126,-0.8511425257,-0.4216827452,1.0985009670,0.5239596367,1.6738646030,1.0952597857,-0.4176313579,0.6746681929,2.0459949970,0.9361134768,0.5849192739,-0.8187872171,-0.8112853765,0.6149257421,-0.0066334433,-0.8003438115,0.0657561496,0.3939533532,0.1361585706,-1.2089350224,-0.2615604103,0.9463102221,-1.8061428070,1.6406424046,1.3682647943,0.6324747801,2.7865254879,-1.1303302050,0.3894205093,-0.0782135278,-1.0114086866,-0.6274721026,0.5245721936,-1.5063163042,-1.9922429323,0.3584777415,0.5166468620,-0.1857368648,-0.8542599678,-0.9650046229,0.4234408140,-0.3074883819,-1.7931255102,-1.5989216566,-0.3567943275,1.4546153545,0.6844121814,0.5107967257,-0.0391141847,-1.4022755623,0.3946054280,0.6230074763,0.3575500250,0.6456065774,-0.8244645596,1.6284826994,1.3106117249,0.5425267220,-0.0468281060,-0.1053166837,1.5829575062,2.1437420845,-2.6331455708,-0.8834359646,-0.0580697283,-0.2573893964,-1.0412811041,-0.9003218412,-2.2390277386,-1.4651108980,-0.1499534696,0.3560865223,-0.8711300492,0.4428383112,-0.7426754832,1.9199404716,-0.1389176399,0.8840380907,-0.0846645460,1.1519436836,0.6512896419,0.4417738318,0.5237726569,-0.0788306445,-1.0378000736,-0.0144220442,3.2748272419,-0.4563313723,0.1812780350,1.0921074152,-1.0814076662,-0.7821266651,-0.3949780762,-0.3073712885,-0.4651679397,-0.1078325585,0.8263167739,1.2388310432,-0.0984496921,-0.9567881227],[-0.2131212503,0.4681727588,-0.8755455017,-0.3477372825,-0.3609948754,0.2883796096,-0.1543822289,1.0686825514,-1.2054982185,-0.2616977692,-0.2440364957,0.3487615287,0.3591597080,0.8647454381,0.0454029441,1.3971421719,0.4091089666,-1.1374706030,0.3737558722,0.0758722574,-1.3365100622,0.3250429630,-2.3053030968,-0.0668937638,-1.0083193779,-0.7527424097,-0.0073583261,0.4523725808,1.3834358454,1.5274974108,0.5206997395,-0.1829435676,1.9268072844,1.7041747570,1.2395449877,0.8001048565,-0.6456826329,0.1719383746,-1.1980223656,0.1657909304,-0.3878783286,0.1869106740,-0.5592908263,-0.5458331704,-0.7450099587,-0.0316931196,-0.0618060678,0.3835448027,0.4217584133,1.1206994057,-0.4465995431,1.7511516809,1.3176025152,-1.5822561979,1.2087854147,-0.0655139759,-0.2714920640,-1.0172853470,0.8885841370,0.6089287400,0.7097648382,-1.0400695801,0.7560692430,-0.1527360380,-0.0720087662,2.0048029423,0.9776907563,1.8429362774,0.5951318145,0.9638172984,-1.4097329378,0.9143978953,-0.6318609118,0.3108898103,-1.0307670832,1.5938689709,0.1314638257,0.1371712685,2.2998940945,-1.1544933319,0.9436885715,-0.1268677711,2.5043098927,1.9020754099,0.0107692750,-0.6744157076,0.4011289477,0.3623802662,1.7662875652,-0.9019196033,0.2089881748,0.7885096669,0.1734537929,1.3288722038,-1.6579455137,1.5910657644,0.4592954218,1.3853743076,-0.4122804701,-0.6964764595,0.1028952003,-0.2327424735,1.3980441093,-0.2943899035,-0.1668646485,0.7435558438,0.1972083002,0.6509979367,0.2048090845,0.6409730315,0.0838581249,0.9075406194,-1.0100653172,-0.8159732819,1.1770539284,-1.1766952276,-0.4897406101,1.5740053654,-0.5039058328,-0.0056537632,1.2350809574,1.9353353977,0.7992894650,1.1764726639,0.5102623701,0.1331552863,1.6801360846,-1.4638708830,-1.4318088293,0.7316586375,-0.0776355043,0.1326100230,1.3219194412,-1.2926105261,0.5352171659,0.2413900346,-0.5880570412,-1.6252239943,-0.2039538324,-0.2277193218,-0.7866405249,0.4781764746,-0.2251786292,0.4531523883,-1.5962477922,1.4734230042,0.5247167945,1.3522722721,0.8866689205,1.1717010736,0.4088672400,2.3660075665,0.4089567363,-0.2065753043,1.2208466530,1.6604276896,-0.8562346697,0.3405393362,0.0647998601,0.8287518620,-0.6682997942,0.9128044844,0.8960058093,-0.5783953667,-1.8637040854,0.9895142913,0.1827133447,0.3518616557,0.5791466236,-0.6660627127,-0.6760500669,0.2832719982,-0.3868727982,-0.6564489603,-0.5221094489,1.2264560461,-1.3206096888,-2.1526074409,0.3476690948,0.3220793307,-0.7065263391,1.2328867912,-0.0474791229,0.6002050042,-0.6559346914,-1.1499810219,-0.2271601856,0.0775771588,1.4095139503,0.3245908320,-0.6881970167,-1.1337358952,-0.4698321223,0.0208422132,-0.2039146572,0.6458313465,-0.1945850253,-0.2234966606,-0.4036347568,-0.1006233916,-0.5893812776,-0.3851362467,0.6804109216,-0.2284515202,1.5971822739,-0.5365006924,0.3082237542,0.5073539615,0.1667853743,0.5216529369,-0.5112001896,-2.3981661797,-0.5508834124,-0.4447124004,-0.1626759470,0.1065026373,0.2490579486,-0.6605647802,0.1807012260,0.9926020503,-1.1432589293,-0.6874733567,-0.0878316462,1.0072650909],[1.5095521212,-1.6570911407,-3.3662109375,0.3644823432,0.9877869487,1.3811259270,0.1469745040,0.6534839272,-1.9952636957,-0.6510771513,-0.7774798274,1.4520498514,0.2092368752,0.4919971526,0.1677229851,0.6899916530,-0.3618898690,0.2964192331,-2.1922900677,-2.8469946384,1.2499256134,-0.3056958020,0.2330211699,1.8649996519,0.6184995770,0.0053699506,0.3032593131,1.4434729815,0.7936376929,0.1969661117,-1.3081755638,-1.2562611103,-2.0427565575,0.8932399750,-0.4132222235,1.3045953512,-0.9867917299,0.1995005459,-1.7860189676,0.7441645861,-0.3486237228,0.0734504163,-1.0310095549,2.6190226078,-0.2775891423,0.9633615017,1.5110583305,-0.8518043160,-0.3761514425,-0.1686878651,0.2582244277,-0.1115956157,-0.7016026974,-0.4886007607,0.2954337895,-0.7916867733,-1.0443811417,-1.2603008747,0.9914335608,-1.0003927946,1.2466765642,-2.2859148979,0.9926853180,-0.6228358150,0.2328247130,0.1546843797,-1.4545462132,-0.3578131497,-0.7590592504,-2.1569862366,-0.8818256259,0.6978793740,-2.1174297333,0.7130247355,-0.3263854086,1.4276014566,0.9877415895,0.5409177542,1.0951485634,0.1220243201,-1.7321188450,0.4447348714,0.6616247296,-1.3816518784,0.8129363656,-0.5154392123,1.1574823856,2.6286344528,-0.5050146580,3.4021975994,1.4216302633,-1.0183618069,-0.3884378374,0.8395509124,-0.2105975747,0.4740676880,-0.0842759758,-1.9411085844,0.4352273345,-0.1354239732,-0.9024779797,0.7749136686,-0.4862678647,-1.8972977400,-0.4917300045,-0.0748297572,0.9854032397,0.2509278655,0.8058610559,0.1157606617,0.7863579988,1.4600559473,1.1236486435,2.3348710537,-0.5767028332,-0.6702319980,-1.2023346424,0.5142666698,0.2149520069,0.6355566382,1.5429649353,-0.2103935927,0.3804515302,0.6273841858,0.3169680238,0.3265507519,-0.3104707897,1.5206472874,1.2834477425,-1.0403629541,-0.7318644524,-0.5680179596,-1.3175101280,0.0111222519,0.0148605723,-0.0837772042,-0.1081169844,-1.1673394442,0.5008950830,-0.8864564896,0.4724477530,-0.0436774604,0.1715297550,-0.3525592387,-0.9352099299,0.3690447807,1.9065877199,-1.3537904024,1.1824322939,-0.9419884086,0.1806846112,-0.2388978302,-0.4788640440,0.9938432574,-0.4293144345,-0.9021407962,-1.0721393824,-0.3390442729,-1.4578844309,1.4132852554,0.5697637796,-2.3599929810,-0.2966747582,0.9824299216,-0.3056213856,-1.2084267139,-0.4508726001,-0.6144681573,-0.2360159010,1.3597880602,-0.6465095282,-0.3063315749,-0.3036143482,-1.1422830820,-1.6548137665,0.3659083247,-0.6401305199,-1.0536401272,0.8842850327,1.0639809370,-0.7625054121,0.7067868114,-1.8095438480,1.6055014133,1.9578210115,-0.0146530466,0.1607527882,0.0634327382,0.9684318304,-0.6058360338,1.0671213865,-0.1546684355,-0.4690584838,1.0974758863,-0.3586915731,-0.5239579678,2.1667027473,-0.6392484307,-0.5902347565,-0.5282523036,1.4030483961,0.5651695132,-0.2470132709,-1.3528504372,0.7387406826,-1.3053969145,0.4459773898,0.6248256564,-0.4102109075,-1.4706453085,-0.1719833761,0.6154183149,-0.1918688715,0.7560840845,0.2232549936,-0.4626301527,1.8654347658,0.4972397983,-1.4983967543,2.1595275402,0.4665803611,0.0884945244,-1.1619651318,-0.2765117288],[-0.1715537906,-0.3518653810,-0.9461241961,-1.1457321644,1.2026308775,2.4595475197,0.4014116824,0.6902657747,-0.2663109899,-1.3605632782,-0.5054963231,-0.4462118149,-0.6951832175,0.6664215326,-0.1848514378,-1.6823832989,-0.3947536945,-1.2504912615,0.6512746811,0.1747264266,-0.0348788761,-1.1168770790,-2.1164686680,-0.6573825479,-0.4557669163,0.4483180940,-0.3280870020,1.1538000107,0.6581790447,-0.1787044406,-0.6257719398,-0.2563052475,1.0647335052,-0.6776366830,-0.0441894680,0.8303117156,-0.6585288048,-1.7951769829,-0.4860822260,-1.2639374733,0.8270537853,-1.5556081533,0.4140720665,-1.3718259335,-0.4767510295,-0.7637899518,0.2245294452,-0.2684869766,0.9843692183,-0.5093818903,-0.8165298700,-0.0653958842,1.1830229759,-0.5425726175,2.0562729836,-0.5610432625,-0.5977123380,0.9381351471,-0.2507341206,-0.3926568329,0.6979966760,-1.9392474890,-0.5521243811,-0.9458277822,-1.1004375219,-1.6574097872,-2.4388780594,0.7281301022,1.3262432814,-0.3952213824,-1.2761065960,-0.4463176131,-0.1996756494,0.0460203737,-1.8395447731,0.8358345628,0.0776033774,0.3844222426,0.2353907377,-0.5447943211,0.9198400974,-1.0590447187,-0.7494023442,0.7968676686,-0.1726324707,1.8498902321,-0.1682839394,0.1879395097,-1.7048875093,-0.4969268739,-0.1124747172,2.0928378105,0.1882959604,-0.1514537334,2.7660396099,0.4743964076,-0.1937914789,2.7596483231,-0.7834467888,1.0347872972,0.0152278300,-0.7610386014,-1.9078309536,0.3805552125,0.3727011681,-0.0694252625,-1.4736522436,0.2938594818,1.3245332241,1.7787688971,-0.0070120962,1.6617984772,0.8333523273,-1.7519874573,-0.2134312093,-0.0100032743,-1.4998146296,-1.8115137815,0.9965004921,-0.3041839004,0.4738995433,-1.0041937828,0.5569674969,0.8938269019,-0.8305382729,0.0593917184,0.3627528250,-2.4793670177,-0.0537039153,1.3435860872,-0.1680946201,-0.2323862910,0.3036299050,-0.5398108363,-1.0809445381,0.3632449210,0.6647271514,0.7428128719,-0.1369682550,-0.0496443063,-0.1413195878,0.9341860414,1.8201687336,1.4392632246,-0.4426941872,0.5988597870,-1.0540181398,1.5845466852,-1.8445963860,-0.6951264143,-0.7396306992,0.6659613252,0.5334790945,-0.1635098904,0.0039233668,-0.6267506480,-0.6743814945,-1.3609571457,2.2019507885,1.0629549026,-3.3453154564,-0.4029050469,0.1154041514,0.7916713953,-1.0441731215,-1.8813159466,-0.8272339106,-0.0320983604,-0.6287982464,0.2015765011,0.4855963588,1.6028063297,0.8580584526,1.3570153713,0.7573254108,-1.0525548458,-0.8210426569,-0.3900167942,-1.5697445869,-0.6533695459,-0.0810939893,0.5408882499,0.7513781190,-0.4506444633,0.9149255753,1.4994388819,-0.0303670596,1.0417119265,-0.2023945749,0.8270350099,-0.1792833060,-0.0137997605,-0.6589713097,-0.3070053756,0.0456016734,-0.0929416865,-0.8315680623,-0.9879750609,-1.9678112268,0.4647029638,0.3763283193,1.1709362268,0.8068950772,0.3331092000,-0.2832021415,0.1535404325,0.0047497978,-0.6976765394,0.0749599189,-0.1627242714,-0.6791628599,0.5772633553,1.5324664116,-1.1410440207,1.7173656225,-0.3408393562,0.9652553201,0.2716450393,-0.7487616539,-0.1968285292,0.1606321335,0.6696608663,-2.2959189415,-0.8271072507],[-0.0054756412,-0.2375295013,0.0163776018,-0.8759958148,0.5110145211,1.9992854595,-1.2928618193,0.5799829364,-0.7743336558,0.9561066628,0.0935266614,0.3667050898,-1.1060663462,1.1112999916,0.8064696789,-0.7583508492,1.5314108133,1.2391928434,0.0666859671,1.9098252058,1.2984734774,0.4810087383,0.2941254973,-0.0169484559,0.4986978769,-0.3291710019,1.6480085850,-0.9981449246,0.5191267133,-1.3664671183,0.2130313367,-1.1082155704,-1.0726953745,0.0305134151,-1.2059307098,0.3432687521,0.2289526612,0.8410323858,0.9339575768,0.9122983813,0.1333952397,-1.7370053530,-0.3468855321,0.2191209644,0.5838090181,-1.1512639523,-2.1934931278,-0.4012824297,-0.5996697545,-2.5864920616,-2.8078482151,-1.9428218603,0.5865498185,1.0440528393,0.2802296579,0.2726215720,-0.7898737192,-0.5721588135,0.9211881757,0.9148751497,0.1910182387,1.4056829214,0.4825052023,0.3429075181,-0.5708855987,0.3786260784,-0.9315177202,1.0208618641,-0.9725951552,1.0935645103,-1.2232342958,1.0409260988,0.6748644114,-0.8847046494,1.7760386467,-0.0125171226,0.8016899228,1.8771826029,0.2736357450,-0.6259844303,-0.9930151105,0.3236394823,-0.2595492899,-0.1605339348,-0.1315153837,0.6235751510,0.6957799792,1.6935269833,0.4405800402,0.0049002608,-0.5700479746,0.2159934789,-1.4336683750,-0.3797477186,1.4713785648,1.8703402281,0.0075660832,-1.4935919046,-0.8821055293,-0.6234281659,0.9677629471,-0.7328437567,-0.2575615942,1.0310820341,1.8457337618,0.0857489705,-1.4493061304,-1.2911778688,-0.0060188952,-1.2698920965,-0.5432638526,0.1579181850,-0.7544699907,-1.0726186037,1.2530040741,0.0434547551,0.6073220968,-0.6502922773,-0.3982699513,-0.2588134706,0.0140502499,0.7154316902,-2.1796958447,1.3421326876,0.9615692496,-0.5703505874,2.5212097168,-0.0208987631,1.0348778963,1.9658333063,0.3773489296,1.4068762064,-2.7714827061,-0.5579730272,1.2298541069,0.6545353532,0.8766176701,1.4339731932,1.1497399807,-0.5089753866,-0.5128499269,0.5389691591,1.3455309868,-0.8251724839,-0.8975332975,0.8422950506,0.8494792581,-0.1416618824,-0.6297848225,2.3947386742,0.1711565703,1.2305666208,-1.0552343130,2.2050673962,-1.1041059494,0.0757807940,0.5562387109,-0.3192164898,-0.7332569361,0.2332962155,0.8301983476,-1.3935382366,-0.1565247476,0.2479268014,0.0560801663,1.7070498466,0.7718081474,-0.7962634563,0.7645039558,-2.1454570293,0.6315026879,0.4187367857,1.1163557768,1.2457189560,1.1284046173,-0.0940276459,-2.3251667023,0.5586260557,-0.3234356046,-0.3314171135,-0.7101279497,-0.1197030023,-1.7641125917,0.6344687939,-0.3916080594,2.1762027740,-0.9665816426,0.0275986735,-1.0265176296,-0.7908039093,-0.2755824327,-0.1384730488,1.0248076916,0.8314927816,-2.0504336357,1.2907943726,-0.2341943830,-0.6094402671,0.5007750988,-0.3549800217,0.1185565591,-1.1103465557,-0.2405512780,0.4953965843,-0.0646155775,-0.4671186805,1.1927411556,-0.6523253322,-0.5857589245,-0.2545292974,-0.1239542216,-0.0203897934,-0.0034907917,-0.6640003920,-1.3702145815,-0.8654106855,0.0326672718,1.3138712645,0.4208591878,-0.9769559503,-0.8214844465,0.1297749430,-0.1297391802,1.0146135092],[1.7940646410,0.2197928280,-0.1319720149,1.0318607092,-1.8993104696,-1.7712563276,0.2810631990,0.4633574188,1.6809766293,-0.3751383126,0.7755542994,1.0474252701,0.2971350551,0.5951340795,1.6116594076,0.1424589455,1.2979151011,-0.4820315242,-0.1617800742,0.0396738835,-0.8928088546,1.1956772804,0.3390346169,-0.4060118496,2.4652819633,0.8918821812,-0.0780620202,0.7143768072,0.3408457935,-1.7165350914,-1.1632332802,0.3031347394,-0.0014043999,0.3238669932,0.6181593537,0.0723976716,1.0229631662,1.1097664833,-2.3270466328,1.3248306513,-0.2984437943,-1.4610385895,-0.0516262911,0.5079306960,-1.0296018124,0.0228735339,1.1103858948,0.0523694046,-0.0578812994,0.1035495922,-1.4611114264,0.3591971695,-0.9242418408,0.3373346329,0.5315905213,-1.1278209686,-0.1850767732,0.0478607044,1.8553361893,-0.1041360721,-1.0382961035,0.5020739436,0.0162114426,0.4946565330,-0.3373891413,-0.8022236824,1.7341907024,-0.0187101774,0.7622405291,-1.3417707682,2.0679044724,-1.0047054291,-1.5931515694,0.6079227328,0.6830322742,0.1589563638,-1.3811666965,-1.2644420862,-1.9886835814,1.9554864168,1.1086750031,1.0685912371,0.0946351141,-0.1953218877,0.3039997220,0.4832267761,1.2116614580,-0.5229575038,-1.0833784342,-1.9380421638,-0.1046403423,-1.2439750433,-0.1244251058,1.0000693798,0.9575945139,-1.6811643839,0.2811778784,0.8408188224,-0.7864719629,-2.2276020050,1.8322992325,-0.4967197478,-0.7136280537,0.0266603082,-1.4942921400,0.8555268049,-0.7208786607,-0.8286910653,0.0459153466,-2.0120952129,0.3852713704,0.0434668995,-0.9490215778,-1.5576071739,1.1132159233,1.4050958157,-0.4533327222,0.4062691331,-0.7710663080,0.4899501204,-0.3605414927,-0.7879081964,-1.4783047438,-0.6426596045,0.0291680340,-1.8412855864,0.3601514101,-0.1602088660,1.1215077639,-0.6920269132,0.4370068312,-0.6374166012,0.9822850823,0.0243866947,0.1872158647,-0.0480282828,-0.2081749886,0.2828293145,-2.1054990292,-0.4270705283,-0.7249308825,-0.9879758954,0.0718391016,-0.5815575719,-1.1283096075,0.1646392792,2.2088565826,-0.8298310637,0.1257507503,-2.0224769115,0.6178207994,-1.2828671932,0.4831278920,-2.5446667671,0.0235447213,0.6328905225,-0.2720804214,0.2828456461,-0.2629654408,0.2357223034,-0.2633309066,0.3823617697,-1.0612810850,0.8905143738,0.2787717283,-2.9553375244,0.4997931123,0.3578976393,0.5189617276,0.3165110052,0.1713823676,2.0032389164,0.5444811583,-1.5100010633,1.6583312750,0.4102133214,2.2827858925,-1.4170377254,1.0672702789,-1.0848878622,-0.9437910318,-0.7768186331,-0.7263257504,1.1376026869,0.1669677347,0.1908601671,1.6095134020,1.1411566734,-0.1959963888,1.8457901478,2.7180738449,0.0899037942,-0.4714163244,1.5490199327,-0.0940073505,-0.6156017184,-0.8397650123,-1.5853828192,0.6788233519,-2.0469789505,-1.0404422283,0.6478714943,-0.5592913032,0.7406380773,-0.6209233403,-1.7690067291,-0.3744283617,0.5275703669,0.1906655729,0.1695944369,-0.1484553218,-0.5499172807,0.6520490646,0.1473938972,-0.8528057337,-0.6439852715,-0.2095579505,-0.1239432767,0.3361501396,0.2245066762,1.9661922455,-0.0664852038,0.4860768616,-0.1386903822],[0.9536022544,1.0937627554,0.1194592714,-0.6387403607,-0.5548762083,-0.5451083779,1.4894568920,-0.7443156242,0.7375551462,0.3067426980,-2.0427732468,-0.4271575809,-1.3816052675,-0.5549044013,-1.5412914753,0.9646945596,1.0175510645,0.8287231922,-0.6588662267,-0.8034091592,-1.0398589373,-0.1990352720,-0.2321966738,-0.8626244664,0.7902417779,0.5957292914,0.0277387053,-0.0978775173,-1.7130082846,0.7651479840,0.6190039515,-1.8823497295,-2.8895819187,-0.3021363914,0.6384620667,1.1293169260,0.8372123837,1.8616006374,-1.3465864658,-0.9666275382,-0.8175053000,-2.2610628605,1.1906888485,-0.8104630709,0.0317354426,-0.2019810528,2.2502446175,-0.2259142995,-0.2013495713,-0.4183434844,-1.7510526180,2.2945125103,-0.7594260573,0.3623003066,0.2451833487,-0.5534833670,0.8765788078,-1.2913513184,-1.5042854548,0.7121409178,-1.0445988178,-0.9468733072,1.0692250729,0.2658461034,-0.5082159042,2.3868613243,0.3205399811,-1.0530476570,1.8602855206,1.8128275871,0.6259533167,-0.7656450272,0.3985169232,0.3867060840,-0.1872877032,0.4220613837,0.8917585015,-1.4155749083,-1.2021927834,2.2409737110,1.1562517881,0.3536393642,-1.7080003023,1.5424200296,0.9600786567,-0.0872785896,0.7514562011,0.9496883750,-0.6002808809,1.6835938692,-0.2155499607,0.9516291618,0.4370562732,-0.4840465486,0.1115614027,0.2884726524,1.3234615326,1.0874466896,-0.5007370710,-1.0555545092,-1.3426496983,-0.3183835149,-0.0679995045,0.3516796231,-0.7929608226,-1.4250048399,0.3407011926,-0.5564637184,-0.5598407388,-2.0848259926,-0.3684155047,-1.5428141356,0.0320821367,-0.7246804833,-0.5784972310,-0.0024182077,-1.0722943544,0.6343519688,-0.2621089816,0.2908956707,0.8434097767,0.7944604754,-0.0688873604,0.2049677670,-1.0564239025,-2.9353809357,-1.1070784330,0.8343291283,-0.2105655819,0.8942486644,0.1070115343,2.8148813248,-0.1915903836,-1.2917917967,0.6141395569,0.2729116678,-0.0089775594,-0.9163486362,-0.3805558085,0.4296191335,-0.5568725467,0.0454204902,0.3127158284,-1.2098184824,0.5218726397,-1.8089548349,-1.7078270912,-1.0075867176,0.2304183394,-0.6987965107,0.6294561028,1.4788361788,0.2974897623,0.9382640719,0.6637336016,1.1047103405,-0.1571140289,-0.9894824028,-0.5497223735,-2.0647106171,-2.5246694088,-0.3880341649,-0.2562047243,2.0650627613,0.2814728320,1.5140551329,-1.5470811129,-0.5152064562,0.2855536342,-0.7161605358,0.1456434429,-0.7985317707,-0.1925467849,0.7353378534,0.8663336039,0.0977301821,-1.6826072931,0.3238239288,1.8810269833,0.0977288559,-0.6490523815,-0.7416719794,-0.4745566249,-0.4449836314,-1.5103685856,-0.0510294065,0.3497232795,-1.3869361877,0.1665868908,1.4021207094,0.9121657014,-0.4573785961,-0.7126724124,1.1894726753,-0.7016700506,-0.1637506485,-0.1672487110,-0.2228942066,0.7889564633,-1.7819786072,-1.0300257206,1.5504140854,-1.2054677010,-0.5473297238,-0.6004752517,-1.4439833164,0.7139390111,0.3109297752,-0.3902191520,0.9313625693,-0.0723820999,0.5176621675,0.1685560048,0.6693775058,0.4388497770,0.4009196460,-0.7448923588,0.1883382797,-0.0165072586,0.1768764108,1.0786801577,-0.4834615886,1.3717924356,-3.5873508453],[1.6853481531,-1.1144967079,-0.6112350821,0.4245436490,-1.7847050428,-0.1747217029,0.1853655428,0.1908560395,-0.6552680135,1.0942722559,-1.4203670025,0.0901684761,-0.8357254863,-1.3958472013,-0.6363635659,-0.3363048434,1.3161414862,2.3062155247,0.2548306286,-0.5943790674,0.1230811551,1.3223766088,1.0881372690,-1.4070535898,1.0152657032,-0.4547869563,-2.4484944344,-0.7348918319,-0.1146792397,1.8282408714,0.4467991889,0.5553500056,1.0960659981,1.1745150089,-0.6352566481,0.9568100572,-0.1503251493,-0.6595150828,0.4656138420,-0.2699167728,0.3962543607,0.0893835947,0.9601736665,-0.2942351103,-0.0805045590,-0.1939863265,0.7092486620,0.7895736694,0.5779416561,0.3850767314,-0.4842936695,0.6331396103,1.1849069595,-1.2239178419,0.6399813890,-0.1655349433,-0.2754193842,-0.4561723173,-0.0982396007,-0.7950536609,-1.3405947685,1.4434165955,1.4510871172,-0.5008798242,2.3326554298,1.7171764374,-1.1214941740,-2.1384959221,-1.4371384382,-1.2622073889,0.3689718246,0.3668732345,-0.1287642419,0.4816688895,-0.7438533306,0.7954478860,-2.9465436935,0.0280521810,0.5243663788,0.8543690443,-0.5550677776,-0.9916635752,1.0783920288,-0.2361106575,-1.0624667406,-1.0621988773,-0.3515598178,0.0059884796,-0.2377974391,-0.2549857199,-0.5997722745,1.4417372942,-1.2261022329,-1.3927828074,0.3786606491,-0.8822529912,1.9552049637,1.3698495626,-0.7608377934,0.4118162394,0.3870486319,-0.1944354028,-0.6407830119,-0.4655601978,0.3296374083,0.3511639833,0.6457598209,-1.6749877930,0.3648128510,-0.3509193361,1.3478113413,-0.6998479962,0.1091958135,-1.2648110390,0.6919533610,0.8096422553,0.4792396128,2.1586918831,-1.0145262480,-0.4494864345,2.1004250050,-0.8175516129,-0.1288737357,0.6267175078,-0.9657980204,-0.3166680932,1.2221369743,0.6191287637,0.8959646225,0.2814862728,-0.3159213364,1.3342118263,-0.9356304407,-0.7985250354,2.5087397099,0.2187114060,-0.3500434756,-0.5621758699,-1.1118510962,2.3423166275,-0.8084353805,-0.6880099177,0.9092693329,-0.4087174237,1.1080206633,0.9253252149,-0.9031172395,-0.6476074457,-1.5008963346,0.4107389748,0.5725745559,0.0407793745,-0.8929830790,0.7959543467,0.6446728706,1.5393860340,0.2424186319,-0.7756873369,-0.3757202029,0.9156033993,1.5491391420,-0.2052761167,-0.7376536131,-1.8269327879,-0.6759383678,1.0747044086,0.3764349818,-0.0615763552,0.0539816134,1.3830044270,-0.2138948143,0.6070758104,2.2050757408,-0.5970668197,1.5818897486,-1.7589093447,0.2374013215,0.6174883246,1.5783377886,0.2767152190,-2.0720968246,-2.1631407738,1.1022502184,0.9568095207,-0.7925844789,-0.0129932240,-0.4571443796,1.1118091345,-0.0802769810,0.5072802305,1.4994150400,0.9595949650,-0.5461329818,1.2634196281,-1.2565197945,0.0333052836,0.4604368210,-0.5963924527,-0.4048521519,-0.0185353886,-0.4433542490,1.0058584213,-0.6991336942,1.2530970573,-1.8591336012,0.1965983361,-1.3790326118,0.6991924644,-1.1815651655,0.5590909123,-0.5781280994,-0.2810287178,0.9894128442,0.5948468447,-0.2116240114,1.6549823284,2.1434628963,0.1538797021,-0.2101962715,-0.2762024403,-1.0265514851,1.2561662197,-0.0961916596,1.8024312258],[-1.0695019960,2.2832481861,1.1988962889,-1.3095893860,-0.6166658401,-1.0923980474,-0.4167923927,-1.1538887024,0.6146585941,-1.0539424419,0.2147403657,-0.9397309422,1.2403391600,0.6924793720,-0.1568285823,-0.2127067894,-0.4619454741,0.4466876090,-0.8593928218,0.3885909915,0.5803668499,0.3129754663,0.5921312571,0.2810893059,-1.0458991528,-0.3443108499,-1.3744816780,-0.3332820535,-0.8077567816,0.4288160503,-1.7361116409,-0.1163452491,-1.3128036261,0.0584229752,-0.1550195813,0.5011024475,0.9355930686,0.5465435386,0.4411371946,0.1784846634,0.4730665684,0.3743475676,0.3358268142,0.3381077945,-0.0342798606,-0.8083887696,-0.3979845345,-1.1604353189,-0.0066471668,0.1081213802,-0.0444181934,0.1806796342,-0.7541006804,-0.7393528223,-0.9072701335,0.2659946382,0.8296968937,-0.2166975737,-0.0818051770,-0.1148054823,-0.7135875821,1.2623482943,0.9717949629,0.6293833852,1.6772850752,1.0530080795,-2.5207023621,0.0990564898,-1.2991740704,1.4384398460,-0.5041558743,-2.4514865875,-0.7514006495,1.0182894468,-0.4337368309,1.8021686077,-0.4107547402,0.0077835210,0.2561873496,1.2580149174,-0.2291361839,0.1945354342,-0.7990037203,-0.4128118157,0.3472887278,0.6855760217,-0.0667856112,0.5450399518,-1.0061272383,-1.0191876888,0.8211130500,-1.1527760029,-0.5438255668,-1.2694709301,0.2857372463,1.1246901751,-0.3271239996,-1.5309205055,0.9586626291,0.7188817263,1.3707944155,2.0927855968,0.9110329747,-0.4163654447,-0.2413679957,0.1141957939,-0.3054531515,-0.1258421391,0.4647783935,-1.0801485777,-0.4261832237,-0.9162673950,0.7924376130,1.0909430981,-0.6960784197,-1.6909022331,0.2807614803,-0.9859634042,1.3558187485,0.4552947879,-0.6819797754,-0.5628001094,-1.1365072727,0.0257628281,1.3203532696,-1.7315850258,-0.2278526872,1.1781377792,2.3463416100,0.1275173873,-2.5863084793,0.6135267019,-0.8700727224,-0.8105805516,1.5111283064,-0.3862929642,-0.4356092811,0.0636376143,-0.5927600861,-0.5895982385,0.1878833175,-1.9611967802,-0.3901644945,1.0398082733,1.2147284746,-1.1976518631,2.3415234089,-0.7130673528,0.2361291051,-0.3015916646,0.5939913988,-1.4054470062,-1.0503392220,0.6779701114,0.5623335242,0.0663100109,0.4308245182,1.0027855635,0.8165775537,0.4989019632,-0.0002661952,1.5691658258,-0.7713332176,0.1407069266,-1.0722569227,1.5917398930,0.6614595652,0.2007022649,-1.0523856878,0.0822489485,-0.6039589047,0.4681268930,-0.5547893047,-1.6950765848,-0.6791362166,0.1981026828,1.9102047682,-0.5541394949,-0.9890259504,0.7984296083,0.0003176213,0.0550370589,0.4107954204,-0.0231982525,-0.7725265622,0.6694859266,1.2758612633,-0.1744779348,-0.0360984839,-1.6794584990,0.4439693987,1.3904777765,0.5694403648,1.7294931412,1.5751372576,-1.9432759285,-0.3927997947,0.9067459106,0.5820047259,1.2948682308,0.1216424406,1.1153415442,1.5315512419,1.1464210749,1.7029515505,-1.0568026304,-0.0944661796,0.3944582939,-0.1618760973,0.0606435351,0.6873081326,1.0236778259,-1.4742099047,-0.4439935684,0.6182740927,-0.0559816100,0.4584692419,0.9900916815,0.5068622828,0.1548262686,1.0944187641,0.4731070399,0.9248341918,-1.4233351946],[-0.8062060475,0.1411530674,-0.7675983310,-0.8649474978,-0.0317371860,0.7700589895,0.7849109173,-1.3150564432,-0.1650596410,0.1696455181,-0.0934922174,-0.6370592713,0.5575283170,-1.4024553299,-0.2018749118,0.2528863251,0.1751198769,-1.1257034540,-0.1473133713,1.0206111670,1.2502821684,-1.1858514547,-0.4971847236,-1.9140037298,0.1850125492,0.6661189198,1.7885531187,0.4540091157,-0.5431872010,0.5952700973,0.3110172749,0.7823548913,-0.3349719644,-0.3264983892,-0.7418109775,-1.7509810925,0.1852183491,1.0977188349,1.5520267487,0.3697099686,1.7792718410,-0.1528856754,0.2824684978,-2.4584684372,-0.2338568270,-0.1353407651,-0.7570917606,2.0325019360,0.3802107573,0.0842941403,0.1059182584,-0.0346744955,-1.6290546656,0.5493621230,-0.3867263496,-1.4424030781,-0.6036384106,0.0387400873,1.5152940750,0.1151001677,-1.0722410679,-0.2107968777,-0.4670877755,-0.1375235170,-0.1439756453,-1.0298039913,1.0332676172,0.8170245290,-0.1887505949,-0.0494192839,-1.0711476803,-0.8699306846,0.4188597202,-0.3706445992,-1.1961505413,-0.7853118181,-0.4474318326,0.4565109611,-0.3028509021,0.0739333481,0.5674366951,0.1040408462,-0.0226907004,-0.2704212666,1.5229213238,1.6927081347,-0.4871771336,0.7022641897,1.0417212248,0.6593301296,0.6112542152,-0.9994509816,0.6857087016,-0.1322730035,-1.0452054739,-1.3029451370,1.2493374348,0.0080011049,-1.2995612621,-0.0949502289,-0.3533111811,-0.8815700412,-1.4849035740,0.6015857458,0.3696425855,0.1216674820,0.1807393730,0.3852241039,-0.8649283648,-0.2445823848,1.3928322792,0.8241620660,-0.1049988940,1.2019417286,-0.8042196631,-0.1875663549,2.1202955246,0.5893884301,0.6650680900,-1.4238115549,-0.4695525169,-1.4288089275,-0.3693912327,0.8288694620,-0.1480082422,0.8329437375,-1.5620113611,1.4933506250,-1.9748326540,0.3324063122,-0.8589459658,0.5188101530,1.1035842896,-0.2568058074,0.8886639476,-0.7919666171,0.4869928360,1.8516781330,-1.1125978231,-1.1663391590,-0.2702799141,-0.9070065618,-1.8252818584,-0.3983536363,0.9135220051,0.6276210546,-1.2996852398,-0.8030201793,-0.8869251013,-0.1006307080,-0.1849675477,0.4097519517,1.1918514967,-1.1746904850,-1.5114037991,-0.0913629159,-0.5989208817,0.9410465360,-0.8433012366,-1.2563877106,1.0473858118,0.9126477838,-0.5827721953,-0.5856482387,-1.7571837902,1.3324482441,0.4234672189,-0.6457720399,-1.0114756823,-0.0575404130,0.4846711755,0.5327658653,0.0827659070,0.2714155316,0.4643567502,1.9540315866,1.3832021952,-1.0131411552,-1.0373722315,0.8032507300,0.2130549699,-0.8439952135,0.9219255447,1.1542927027,-0.0212097038,1.6666793823,-0.5585919619,0.2945807576,-0.0132242423,0.5881583691,-0.6531445384,-0.2318715155,0.0456866808,0.0064122831,0.4922307730,-0.6649486423,1.2209202051,-2.3153216839,-0.8188648820,-0.7945204377,-1.2346620560,-1.2860282660,-0.8244218826,0.7727444172,-1.7617459297,0.0262414347,-0.3023742139,-0.0454284027,0.5600838661,0.3081323504,-0.6677283049,-0.0851749256,0.4230430126,1.1981501579,-0.5143482089,0.1208445281,-0.3302524090,1.4326996803,-0.4619111121,0.8303595781,-0.2305427641,0.4566539824,0.0284742471,1.1725553274],[-0.6778904200,0.7474908233,-1.4357063770,2.0686724186,-0.6995326281,0.1032049581,-0.9478116035,-0.1820008606,-0.8253668547,1.1953212023,0.3244872391,0.6627004743,1.0053045750,0.2691727877,0.2567492425,0.6107055545,-0.7879512310,0.9819718599,0.6871561408,-0.8443669677,0.0294165127,0.6981332898,-0.4156535268,-0.4417705238,-1.1484087706,0.4935772717,0.9644652009,-2.2177762985,1.2895581722,1.9430149794,-0.1687173247,0.2524688244,-0.0231861938,0.2763607204,-0.8558312058,-1.3785704374,0.9749888182,-0.0315952413,1.2336918116,0.8569123745,-0.1242811829,1.0706355572,0.0651801452,-1.0987329483,-0.0845275149,0.3220182061,1.3133853674,-0.3922113478,-0.8745101690,-1.6545143127,1.3099434376,-0.3884691894,-1.0207983255,0.9145443439,0.6784790158,-0.4679728150,-1.8396611214,-0.6501001120,1.0174416304,-1.2393320799,-0.0569090955,-1.5452183485,0.4984608293,-1.0995006561,0.4160440862,-0.2431684881,-0.0810485184,0.1673014760,-1.2867910862,1.1799737215,-0.2366669923,0.2107269764,0.1286269724,0.7097382545,-0.7196391225,1.0831000805,0.2915651202,-0.8028673530,-0.4497925341,-0.2961663902,-0.3938611150,1.3102715015,0.1274573952,-0.3415600657,-1.3402360678,0.3736258149,-0.1554016620,-2.0350890160,-1.6971189976,0.2403908372,0.2295648903,-1.4248719215,2.1251647472,-1.4979672432,-1.2219846249,0.3304457068,0.0374567807,2.1840059757,0.5777245164,0.1611160338,0.8961535096,0.3680781722,-1.9025293589,-2.9646284580,-0.6679293513,0.0742716938,1.4247859716,-1.4310384989,0.6191782951,-0.2454420179,1.8304854631,0.5033181906,1.5404642820,2.0885241032,-1.3709269762,1.6574317217,-0.5738112330,0.4851473868,-0.5364739895,1.9374666214,0.5323956609,-0.7937283516,0.7213377953,-1.9963128567,0.5997330546,-2.5905666351,0.4053169489,1.0426921844,-0.2191019654,-0.5558608174,1.5116729736,-0.8256971240,-0.2864851356,-0.5010135770,1.4569622278,-0.4448128939,1.1936591864,0.6703691483,2.1985344887,-0.6182096004,-0.7770881653,-0.7601132393,0.8608002663,1.1269603968,0.0908432677,0.1105712205,0.9951350093,1.0663802624,-0.6953923702,0.0459341668,-1.2705870867,-0.4319158792,-0.7496430278,0.5289142132,-1.6569365263,-1.0187835693,0.9269422889,-1.4888494015,-0.4321640730,-0.1888191402,-0.4988208711,-0.7500018477,0.5415527821,1.2704626322,-0.5392972231,-0.5853786469,2.0392589569,0.8061009049,0.4328102469,0.1999326795,-0.1828068048,2.0369491577,0.5376186967,-0.9456459880,0.2697933614,1.6897407770,-1.3979105949,0.2796856165,-1.2426117659,0.6467071176,1.0034134388,-0.5081450939,-0.1762165129,-0.9470931292,-0.9216216803,0.0103546353,-0.9105349183,-1.5967898369,1.6333882809,-1.1184823513,0.7191648483,-0.1556680799,-1.1775581837,0.6844239235,-0.0761843994,-0.8918274045,-0.9387372136,-1.4445216656,-0.8445951939,-0.1898192763,-1.2781102657,-1.6075053215,0.6881842613,1.5046265125,-0.6197365522,0.2504410148,-0.2626022100,-0.5783621669,0.9059876800,-1.2948360443,-0.2569444478,-0.8420674801,-1.3039224148,0.5142272115,0.4427238107,0.1815748215,0.4946224391,-1.3786232471,-0.2891066074,0.4617694318,-0.8750137687,-0.6624167562,0.2299284190,-0.3934533298],[0.1057336032,-0.5486158133,-0.7816815972,1.4666053057,1.4891735315,1.4241904020,-1.1920310259,-0.6360286474,-0.4250755310,0.0566578321,0.7702762485,0.1289665252,-1.5876783133,-0.0016417794,0.4250714779,1.0069613457,0.9110912085,0.6452820897,1.2802551985,0.8736882210,-0.1176333576,1.3896194696,-0.8711903095,-0.8319627643,-0.2103565633,-1.5452733040,-0.2017096281,0.0036522455,0.3344618678,-0.4068647623,-0.7161149979,-1.2258971930,0.5839748979,0.6048594713,-1.1431585550,-1.7302323580,-0.0968581066,0.1390904784,-0.3141165972,2.1717050076,1.9471942186,0.5632810593,-0.2581654489,-0.8188413978,-0.7301347852,-1.9451324940,0.1461771280,-1.3031450510,-1.2256352901,0.2075386643,-0.3776623309,-0.6919597387,0.7050512433,1.5114662647,0.1402460635,0.9202152491,0.4513795972,0.7388632298,1.0826296806,-0.3678231835,0.3671057522,1.1094573736,0.3479338884,-0.6016224623,0.0073265564,0.4310854077,1.7144032717,-0.3383898139,1.0131033659,0.2747268379,0.7387757301,-0.4018067122,0.6781775355,-2.1581020355,0.2278467268,1.7021756172,1.3773645163,-2.0554974079,-0.6872193217,0.1226048097,-1.9852946997,-0.3543234169,0.1991903782,-2.1077291965,0.9524010420,-0.8265848756,0.0515594371,0.7495928407,-0.5910230279,-1.9738554955,-0.4401849806,1.2871997356,0.3963395059,-1.3662760258,-1.0966602564,-0.1497679204,0.2017935365,1.5538893938,-1.4132012129,0.8584647775,0.0906719267,-1.0717674494,0.1683136374,-1.6558039188,0.5440172553,1.2324019670,0.5137600899,0.2472419590,-0.0660363659,-0.3757608831,0.5444558263,-1.7358322144,-0.0051293653,-0.2503592670,-0.4951822162,1.2275048494,0.2725945115,0.3664792776,-0.6326197982,0.0382645912,0.7879148126,-0.1890475899,0.6260702610,0.6560769677,-0.8796280622,-1.0355696678,-1.1185706854,1.8544151783,0.7461084127,1.7544984818,-0.6223727465,-0.4497933388,0.8481234908,-0.3922756314,-0.4858475626,0.0968091264,0.0066539571,1.0293912888,0.2278260440,2.0721802711,0.4783202708,0.7750809193,0.3829811513,-1.0834962130,-0.6774073839,-1.5615712404,-1.6606448889,0.3167343438,-0.4906793833,0.0483574681,2.2943816185,-0.0707739368,-0.4499971867,-0.1112928838,-0.2830072939,-2.6263105869,-2.1136035919,-1.1006478071,0.7729149461,1.4741439819,-1.2733880281,2.3177292347,-0.5778938532,0.0909742787,0.4740823805,0.5157712102,0.5300472379,-0.8119449019,1.2384713888,0.5614928603,0.2497849166,1.2373347282,0.1882596016,-0.1199569330,0.5125343800,-0.4821256101,-0.5073617697,-0.3014651537,0.6116203666,0.1299709827,0.6942937970,1.1415054798,-0.2746116817,0.1055720225,-0.5667106509,0.4699902534,0.3701259792,-0.4664997160,-0.0554599166,-0.9459332824,-0.7991710901,-0.0254140403,-0.1795408279,0.5957148671,-0.9031828642,-1.5857150555,1.0605216026,-1.5644696951,2.2349119186,0.2240771055,0.9779759645,-0.3761295676,-0.6323259473,-1.1920045614,1.5147486925,1.5310157537,-1.3137631416,0.0094543528,-0.2456318289,0.5364572406,-0.9972238541,1.2738656998,-1.3176814318,0.3259899318,1.0292425156,-1.3389029503,1.4183219671,-0.4529511333,-1.0125648975,2.5792348385,0.6706108451,-0.4555190206,-0.8167887926,-0.2441108376],[1.4933758974,2.3693647385,-0.0329419933,0.2555276155,0.1001068726,0.4355405271,0.7977789640,-0.3352335989,0.1421558112,-0.2073267847,0.0726978779,0.4240096211,-2.3663616180,-0.8102269769,1.0085755587,0.2688770592,-1.0159114599,-0.1881680489,1.8253681660,0.8918188810,0.0901473612,-0.1998090148,0.6020976305,0.9907689691,-3.2210643291,1.6036646366,-1.0785632133,-0.4379940927,-0.7611535192,-0.0714974627,-0.6375125051,-0.3485542536,0.3473993838,-1.2568857670,-1.3050128222,0.0136722717,0.7082781196,0.8472917676,0.8057292700,-0.1659359187,-0.5133125186,0.3766638637,-0.2679806054,-1.4026973248,-1.0416235924,0.7020118833,-0.1503003240,-0.5658562183,0.1204976216,-2.6717855930,-1.9358696938,-0.8094495535,-0.6051175594,-2.0002884865,0.7022117376,2.1806385517,-1.7201800346,-0.0582118407,-0.1393259019,-0.1016402170,-0.2271406353,-0.0488020889,0.1912101656,1.2248835564,-2.2081222534,0.0591728874,-1.0548201799,0.2093756348,0.7086314559,0.4830908477,2.0755898952,0.0622914173,0.9292717576,-0.4380646348,-1.6233458519,0.1461371183,-0.5482233763,0.7338901758,0.0389567427,0.6998253465,1.0116010904,-0.3618020415,1.8255698681,-0.9448072910,0.1480549872,0.9181790352,1.1242004633,-0.7497963309,0.5569706559,-0.3830719590,-2.8234889507,-1.1781505346,1.4321072102,1.2316467762,0.2388788015,0.0512132496,-0.4228218794,1.1188286543,0.3400440812,-0.4093388915,-0.3253835738,-0.5566400290,1.2506176233,-0.5119938850,-0.7112380862,1.1241469383,0.2668454647,-1.7113490105,0.2358381599,1.1011041403,0.8496887684,-1.2886971235,-1.3477289677,-2.8305227757,-1.0628187656,-0.2909052968,-0.1643280834,0.0555402003,0.5318995118,0.5488132834,-0.3408282399,2.4728338718,-0.0778830498,0.2826209366,0.3594366014,1.1223710775,-0.2526025772,-1.8722442389,1.2654098272,-2.6454839706,-0.4887316823,-0.4427509308,1.2608346939,2.1664521694,1.1900120974,-2.3902049065,-0.6655550003,-0.1544005424,0.5118381381,0.0688272491,0.3676896989,0.1625170857,1.4110800028,0.0116869053,0.6106952429,0.0455630608,-0.1926245689,-0.2303574234,0.8177344799,0.1466995478,0.0532553457,-0.6141367555,0.5387769341,0.0295939930,-0.0525368638,0.5687243342,0.6703144908,-0.6449947357,0.3593075573,0.7457771301,-2.0517790318,0.0016169018,-0.0653042272,-0.1865885556,-1.4693999290,1.1893911362,1.0076799393,-1.4579092264,1.3490563631,-1.6361253262,-0.5041018128,-2.0277647972,-1.4027677774,0.4853744805,1.1966117620,-1.9860762358,-1.1433326006,-0.4991798997,0.8099403977,0.9741308689,-1.1231752634,1.2916612625,1.2223510742,1.1004766226,0.1277393401,0.9839468002,0.0020423641,-1.3422156572,1.1989815235,1.0622923374,-1.0130317211,0.0652642846,-0.5992264152,0.4082034826,0.0806819052,-0.3798908591,1.2465872765,-1.0993893147,-0.5118710399,-0.2986849844,0.5246425867,0.6732012033,0.0628702417,-0.9138420820,-0.9044297338,0.0181843303,-1.6680605412,-1.1724907160,-0.8832670450,-0.2367167175,-0.0361848995,1.4360933304,-0.2679459751,-0.8920229673,1.0423166752,-0.0651038066,1.7097898722,-0.9543641210,-0.7677184939,0.1742484719,-0.2672248483,-0.4275380373,-0.1414251775,0.9008878469],[0.8739157319,-0.3322092593,-0.5973111987,0.2851661444,-0.1468812525,-0.2907929420,-1.0881502628,-0.6024230123,-0.5802869201,-1.2372754812,-0.4289747477,-0.5685067177,0.9802915454,0.0765238106,0.9593495131,0.6458209157,-0.6384899020,0.6581201553,0.4170886278,1.3364971876,0.8242484927,1.4422096014,-0.4826723039,0.2676834166,-0.1370688677,-0.3821313083,-0.9061530232,1.1765571833,0.4614237249,1.9140841961,0.0065699657,0.4581893384,2.2392342091,-2.1551437378,-0.6505623460,0.1773225814,0.1001090482,1.1466196775,-0.5088146329,0.9910753965,-0.6102650166,-1.6703640223,-0.5117351413,-0.6359770894,-1.0455538034,0.3068882227,-2.8139216900,-0.7243748307,2.3602602482,1.1162489653,-0.5636335611,-1.3498125076,2.0970036983,-1.6311956644,-0.3248442113,-0.7526709437,-0.6360383630,2.2367043495,0.8790027499,-0.4985416234,-0.4463484585,-0.7190703750,-0.3197208345,-0.1645631343,1.6601192951,-0.0426488407,2.3123860359,-1.5778800249,-0.0058000362,0.1096969396,-0.0175224114,0.9351284504,0.9580059052,-1.3353056908,0.1352744251,0.7962580323,-0.7297627926,0.9289662838,0.7584497333,-1.0704288483,-0.1412631720,-0.2476228029,-0.6682301760,0.0307615045,-1.4234244823,-0.7623709440,-0.9366347790,2.0120165348,-0.5410431027,-0.4628232121,-0.1133372635,-2.6215143204,-0.1878034025,-0.1376096606,0.2618097961,1.3662329912,-2.2204368114,0.8991046548,0.7960936427,0.4082904160,0.3135900199,1.0021790266,2.2778322697,-0.6833299398,0.2828281820,0.4902434647,-1.6065645218,-1.2876976728,0.4138703048,1.4634865522,-0.8037552238,-1.8263963461,-0.1062341779,-0.4714684784,-0.2142926008,0.8328797221,-0.2681073248,0.4731663764,-0.6831670403,1.0777174234,0.3799136877,-1.1204904318,-0.1482727826,-2.2189519405,1.4719551802,1.0429495573,0.2535926402,-0.9038581252,-0.2051886171,0.2231454849,-0.9873346686,-0.1028130278,-0.9490976930,0.5669447184,1.3465336561,0.5354964137,-1.4442919493,-1.3446625471,0.1368197799,0.3121242821,0.6082185507,-1.9605544806,0.7699592113,-0.9161998630,1.6719350815,0.0406786539,-0.2060671747,-0.3644065261,-0.4067469239,0.6496877670,-0.4569232166,-0.4409435093,0.1681868136,-0.8629373312,-1.1391000748,-0.5719874501,-0.6937422752,-0.3806343377,1.2496166229,0.1289080679,0.9666724801,-0.5838910937,0.1186358780,-0.7255809307,1.3304361105,0.2326592952,0.8757634163,-0.7049357295,-1.0692155361,-1.2293150425,0.2959873080,0.0319055319,0.1620924175,1.4239611626,0.7566784620,-0.3976113498,-0.4200448394,1.8963367939,-0.5100295544,-0.1136146113,-0.7116947174,-1.1979464293,0.0628819540,1.6701619625,1.3206782341,-0.3954262137,1.6946469545,-0.1842424870,2.0869755745,0.4656747282,-0.9359200001,1.0937289000,-0.2609280646,0.8597438931,-0.4747339189,-1.7674329281,-0.7370138168,1.6124833822,0.8634741306,1.2746123075,0.4331390262,-0.2857427001,-1.2503889799,-0.4103792012,1.2073097229,-0.2294325531,0.1557102054,0.1661618203,0.1306845248,-0.3510292768,-0.7382842898,-0.1517895609,-1.5608966351,0.0113451760,-2.5729622841,-0.2821547389,0.2182861120,-0.0131761897,0.4549012780,0.2538626492,0.8881043196,-0.2298544347,-0.0876468346,-0.3201176524],[0.2374811918,0.8972510099,0.4730105698,-0.6594987512,-0.0209075324,1.3408588171,1.4259743690,0.6906683445,1.4695892334,-1.7932443619,-0.3990622759,0.3883207440,-0.1653114706,0.7263434529,1.0113916397,0.7325154543,1.0358587503,0.2038902938,1.6042402983,0.0524286889,1.0023922920,1.1868354082,1.6180857420,0.8422029018,-0.3649192750,0.6122750640,0.4988044798,-0.5457030535,-0.4046074748,-0.2460051328,-2.1963894367,0.2778351903,-1.2490707636,-0.5998269320,0.4359167516,0.9332191348,-0.2432841212,0.1132514253,2.0727136135,0.7468464971,-1.4363973141,1.6554725170,-0.5935822129,-0.0301943701,-0.0096276281,0.9447058439,-1.1380391121,-0.3193050027,0.3661886156,0.7435558438,-0.1640613824,1.1098930836,-0.3936080635,-0.4477424622,-0.3403979838,-2.5856881142,0.9082188010,-0.8674103618,1.4339728355,-0.8392519355,-0.5988446474,0.1905030608,-1.3781613111,1.0868411064,1.0486859083,1.1246263981,-0.7191772461,-1.0842479467,-1.6448795795,0.8328526020,-0.9384015203,-0.4217948020,-1.3769412041,1.9037914276,-0.5599355698,0.1158174872,-0.1574285477,0.4571927786,-1.7404035330,-1.8275996447,0.7204089761,-0.2950896919,1.4025130272,0.2427419722,-0.0037648869,1.3575364351,-1.1893963814,-1.1178938150,0.3041092753,-0.8188020587,-1.1610109806,0.2950367033,0.0649628937,-0.4260863960,1.3116868734,0.5123817325,0.3020256758,0.1874590069,-1.5938930511,-1.7766227722,-0.2056816667,-0.1434836537,-0.6319875121,-1.3853851557,1.7522269487,0.1705312729,1.9834896326,0.0428273194,-0.2813680470,-0.0961206928,-0.6106506586,-1.2427818775,1.4810787439,-0.6395637393,1.4544156790,-1.0006572008,1.0430054665,0.2761348486,0.3180278540,-0.5181749463,-0.0602431931,0.0929836407,0.4769335091,0.3015520573,0.5975839496,0.6896985769,-0.1892197728,-1.0921987295,-0.4295704961,0.4336581826,0.3824291527,0.1809328645,-1.0461263657,-1.4477847815,0.6859626174,-0.7645117640,1.0940294266,1.4802038670,-0.9201808572,-1.0781209469,0.9212292433,-0.3049114645,-0.0715369731,1.0498589277,1.0016895533,1.4182119370,-0.5187898278,-0.1311899126,0.9670145512,0.5622414351,0.4447497129,1.1318860054,0.7751047015,-0.3009999692,-0.4801048636,0.0647643581,0.9612099528,0.8127611876,0.6242893338,-0.0070319353,1.0763334036,0.3677817285,-1.3627214432,-1.0657974482,0.8311042190,-1.3935128450,-0.2092558742,0.4993962944,-0.9601699710,1.0647552013,2.2043251991,0.3423570991,0.1696930826,-0.2994166017,1.0733228922,-0.3528176844,0.4736300111,-0.5256693363,-1.5099061728,-0.2726048827,1.8399487734,1.0151726007,-0.4483799636,1.0994441509,-1.8038136959,0.4645211697,0.3011919260,-0.7694646716,0.9885981083,-0.3152141571,-0.8028303981,-0.1730686873,-0.6892525554,-0.9899894595,0.6263012290,1.6653257608,0.3095535338,-0.5246341228,0.0186689477,1.6799933910,-0.5161258578,1.8456840515,1.9402329922,-1.6643905640,0.7645756602,-1.0416288376,1.8423246145,0.5390047431,-1.9061768055,0.8301377892,1.0769793987,0.6682855487,0.9632487297,0.6451629400,0.0955077112,0.6230443716,-1.1125954390,-1.1393494606,-0.2665373385,2.2958316803,-1.8586397171,0.1168465689,-0.0136828171,1.4055063725],[-1.3613090515,1.2799606323,-0.8697892427,2.4187178612,-1.5788184404,0.5349386334,0.4891916215,0.6985910535,-2.4071002007,0.6912989616,-0.1731216162,-0.3386763334,-0.6762531996,1.8666601181,-0.0154876197,1.4744048119,0.8798419833,-1.8359698057,-1.4010410309,0.1690005064,0.4600860775,-0.2688275576,-0.1073251069,-0.6607923508,0.0372763202,0.1201879010,-0.1509473771,-1.0607008934,-1.1784601212,0.0837295130,-1.1999645233,-0.7227930427,-1.1312910318,1.4536602497,-0.3281475008,-0.2820889950,-1.9610961676,-2.2980158329,1.6167727709,-0.9151739478,0.7914054990,1.4067481756,0.5194048285,0.8777590990,0.5513826609,0.2247539461,-0.0472301021,0.9914721847,2.1558043957,1.8301255703,0.1900593936,0.2965781391,-0.0984601229,1.0304844379,0.4766424894,-0.7833252549,-0.5008711219,-0.5573242903,0.4439268708,-3.2847006321,0.4752749503,0.5018771291,0.2221382260,0.7174639702,-0.7699761987,0.3723175824,0.5160827041,0.4284012020,-0.8318826556,-0.3510650694,0.6850978136,0.6224149466,0.1465418190,-0.3880501390,0.1311548799,-0.3739495575,-0.0982051417,0.0241817795,-0.2103700042,2.3744344711,-0.5839608908,-0.1087427288,-0.2498640120,-0.0379346982,0.3635368645,-1.4021166563,-0.3856891692,-0.5266512036,0.5923498869,0.5811980963,-0.1913233995,0.2475237846,-0.6341478229,-0.4015057981,-0.2930896282,-1.6198561192,-1.5492149591,-0.4098072052,-1.1009782553,0.0696988776,2.2207129002,0.3850325644,0.1232785955,-0.1360888630,0.5543779135,-0.0335183479,0.1661666334,-0.9750319123,-0.3781283796,-0.6851177812,-1.3397881985,1.8235151768,0.7576472163,0.4756061435,-0.1756661981,-0.7518137693,0.8833954930,-0.1723460406,1.4779287577,1.1817060709,0.6576637030,1.7607759237,0.3329871595,0.3841156960,-1.6105310917,-3.4629151821,1.5630116463,0.1894923598,1.1110873222,-0.4295303226,0.0688258111,-2.6791574955,-2.3411090374,-0.6443280578,-2.5961453915,1.8770301342,1.6789271832,1.1087728739,0.4900375903,1.0108305216,-0.7254547477,-0.8507792354,1.0462187529,-0.5669525266,1.7076981068,-0.2117697448,-1.2549630404,0.7878744006,-0.6559647918,2.6261720657,0.0378729068,-0.9183604121,1.5393013954,-0.1139022857,1.5634179115,-0.0301232282,0.0856970623,0.2333744764,-0.3873595893,1.9680051804,1.4474378824,1.2159805298,-0.0786215439,-0.8479925394,0.3741477430,-1.3048787117,-0.6006184816,-0.1725184321,0.3311712146,-0.2673325241,-0.3190022707,0.5827477574,-0.7233198881,-1.2103803158,-0.0765999481,-1.9542196989,0.3852671683,1.4858161211,-0.0310763177,-1.7405352592,0.8110358119,0.1677609235,-0.3663225472,0.4820788801,2.0430974960,-0.2657999992,-0.4802150428,1.6891940832,0.6045791507,-0.2234309465,0.8571979403,-0.2686817646,-1.5307800770,-1.0786039829,-0.9064611197,-1.1571456194,-0.0541361533,-0.5406714082,-0.2749684751,0.1239802092,-0.5381905437,0.6174876094,-0.9688651562,0.9094956517,0.2015793473,2.0235207081,0.2642515600,-0.7107654810,0.6661833525,0.7415869832,1.9120591879,0.9989323020,0.4370322526,0.5388150215,0.2214304805,-0.4388776720,0.0455226041,-0.9726260304,1.4213956594,0.7784917951,-0.8738764524,-0.6479167342,0.9478707910,1.3753795624],[0.0296977572,0.8786270618,-0.5038436651,0.1264273673,-1.8705068827,2.3100185394,-0.8938060999,0.3092682958,-0.2958122790,2.1298017502,0.2210101187,0.8279167414,-1.3819217682,1.0468441248,0.7928908467,0.6151984930,-2.1460978985,0.0850671232,-0.2827851772,-1.4269472361,-0.7818253636,2.2002530098,-0.9858335257,0.5671643615,1.5863406658,-0.0924895555,-1.0210804939,0.6836526394,-2.2170433998,1.1493161917,0.3413541317,-0.2154352516,0.5662803054,-0.7847980857,-0.4791492820,-0.8131125569,0.8970845938,-0.4572354853,-0.7225107551,-0.5756103992,-2.2434556484,1.0185506344,-0.5351034999,-1.3657653332,2.0675470829,-1.0386697054,1.4762983322,-0.7982075810,2.0018270016,0.2635194659,1.2353080511,0.4287237525,0.0174695142,0.9563567638,1.6330941916,1.6011214256,-1.2320027351,0.1984476298,-0.9122123718,0.7111468911,0.7488287091,-0.3852639496,-0.1148628294,-0.2698810697,-0.3712545931,0.2244575918,-0.0854918435,-0.0593928993,0.1030734554,-0.4511945546,0.5479831100,-0.6317163706,0.6489620805,-1.0175821781,-0.1811434031,1.5832920074,-0.6779805422,1.3851190805,-0.3345347941,-0.1046513170,-1.6282833815,-0.9392483830,0.2728080750,-0.0925905555,0.2160407156,0.6644548178,-0.2666618228,2.6946973801,0.0434410423,-0.6470096111,1.7980229855,-1.2989336252,0.3984160423,0.6778487563,-0.0981895402,0.9798938036,-1.2538865805,-1.4360401630,-0.5902845860,-0.8581837416,-0.2199495137,-0.3045339286,-0.5929993987,-1.2001744509,0.4193471968,0.4130449891,-0.0643541962,-0.3918884695,-1.9008547068,-0.6400154829,-0.0310537405,0.3388310373,0.7863161564,-0.7796239853,-0.6914700270,-0.8911319971,-0.1382533163,1.1118685007,-0.6569674015,-0.1794133633,-0.5717086792,-0.4314243793,0.7357690930,0.2798558176,0.8829190731,-0.3237231374,-0.6520662904,0.2529103756,-0.3372556865,0.5908755064,0.5626513362,-2.2970151901,-0.2403149754,-0.3434230983,0.6380447745,-0.5477387309,-1.4343986511,0.8501843214,0.6045268774,-0.5088319778,0.1553441733,0.7751315236,-0.5564307570,0.0922820196,1.0430363417,-0.0166171789,0.0004913778,-0.3558220565,-1.0550065041,0.5611869097,0.5342825055,-0.0304869320,0.2164323777,-0.1864604950,0.0301791150,-1.0917770863,-0.1840087622,-0.0184875894,-0.6151685119,0.3353316188,-0.1206564009,0.0764194801,-0.2989043295,-1.5547710657,-0.6537920833,1.8045568466,-0.1564288735,-1.1329820156,0.5988346338,-1.8747942448,1.9920927286,-0.2383167297,-0.0050372910,1.7662442923,0.4502291381,-1.6901876926,0.2092062235,-0.3719630539,-1.4031659365,0.8186979890,0.3760634363,-2.2954051495,0.5926311016,0.0171566214,0.2273662835,-0.4911201298,0.2143833786,0.5893830061,0.1842756271,0.9189839959,1.7500014305,-0.0224729050,-0.0380397290,0.3788199723,-2.2946538925,1.7362905741,0.5558519959,1.1632210016,-0.3035589755,0.7126848698,0.2978425324,1.0315711498,0.4382598400,1.0446311235,0.1455101073,-0.7554751635,0.1581776291,-1.1658589840,-1.2868412733,0.7790217996,1.1975550652,0.5179507136,0.4167861640,-0.1070448756,0.8847790956,-2.1269643307,-2.1746633053,0.9809335470,-1.8558607101,-0.0589242019,-0.3949842155,0.1342334598,-1.9917581081,2.0874900818],[0.2646040916,-0.3620138764,-0.4476041794,-0.0092560723,2.7727761269,-0.9875227213,-1.1261665821,0.8644894958,0.0200013202,-0.5129029155,0.4734310210,-0.0488065369,-2.4048309326,-0.1564999670,1.1135571003,1.1397062540,-1.4627355337,0.7660913467,1.1386791468,-1.7050300837,-0.8706223369,1.3747351170,-2.1854331493,-0.7849680781,0.5419611335,-1.3487257957,-1.3122342825,0.1711963266,0.6263733506,0.1525226533,1.4732654095,-0.0650783703,-2.6287243366,1.1032720804,-0.2081544995,0.2773850560,-0.8050147891,1.7604209185,-0.4776040614,0.3857266903,1.1773180962,0.5346040130,0.7014524937,-0.2645107806,-0.1771366298,-0.8814995885,0.1855238378,0.5930702686,1.3211059570,0.3621626794,0.4415043592,0.9820915461,-1.1620858908,-1.5575093031,2.3486135006,-1.4105807543,0.5415712595,-0.3308941722,-1.2917026281,-0.1719277054,0.1799248606,-0.0432348475,0.2673465908,0.1946123540,0.8462187648,-0.8374105692,0.1203589514,1.9621579647,1.5041606426,0.6325783730,0.4140176177,-1.5472197533,-0.8060904741,1.3864337206,-0.7399890423,0.3658272028,1.8032287359,-1.0524270535,0.2010727376,-0.5947147608,-0.4959990084,0.5370004177,0.4044150114,-0.5982482433,-1.7088147402,-1.0533634424,1.2031201124,-0.1458389908,0.5778471231,0.0965054780,-0.5388431549,0.8363574147,0.4032280743,-0.2996425927,0.0294992831,1.9186033010,0.9316451550,1.4993157387,-1.7577295303,1.2611371279,1.0125068426,-0.2110064179,0.6125735044,0.7863415480,0.5930656195,-0.4154962897,-0.7229909897,-0.7670542598,-0.4934742451,-1.7241714001,0.9628956914,1.3955906630,-0.4624428153,-0.7550997138,0.2275715172,0.8299591541,0.9396640062,0.9205953479,-1.9562321901,0.5347333550,-0.3697306812,-1.0418801308,0.7764693499,-0.5618635416,-1.3113641739,-1.2335938215,0.7353008389,0.1957557499,-0.7724125981,1.1139376163,1.3518786430,0.0805120543,1.5585693121,0.5469241738,-0.5058984160,0.8069629669,-0.7987787724,0.1504934281,-1.6120203733,0.0503893793,0.7792242765,-0.3512535393,0.0884182081,1.1663037539,0.6744574904,-0.3506546021,1.3014886379,-1.0340659618,-0.3779429495,-0.9123903513,-0.3398987949,1.0012029409,-0.2418974638,0.8322359920,1.2388926744,1.4504948854,-0.0846828520,-1.9022624493,2.3461015224,1.5761616230,-0.4713822901,-0.2990472019,1.4670183659,-1.4660875797,0.1758549064,0.5109443665,-1.3963633776,0.3641264141,0.4785705507,-0.7390723825,-0.1571729332,-0.0894001424,-0.8202293515,-0.0035593198,-0.3045049012,-0.6465057135,-0.5934047699,0.8569859862,-0.4925061762,-1.3001301289,0.1930694282,0.0667027608,1.0578979254,-1.4051775932,0.5545969009,-1.9266557693,-0.3564795852,1.1653716564,-0.6970770955,1.0272433758,0.1583375931,-0.3123612106,-0.9408250451,-0.0537493266,0.4240416884,-0.1350065917,0.4084877074,0.0649588257,1.2394356728,0.0960153639,1.5031138659,0.2776180506,0.7171580195,0.8699058890,-1.8727498055,0.3917263150,1.8253725767,0.2035031319,-0.3448802531,0.8481007814,-0.4494965672,0.8984521031,-0.3145972490,0.1103251651,-0.2218849212,-2.0152313709,1.2233344316,0.6462559104,-0.1851824820,0.5452057719,-0.1294742972,-1.7381348610,1.4482873678,-0.0049621151],[-1.7968306541,-0.4029459059,-0.3125780225,0.9872809052,0.7336838245,0.1305475235,-2.8211119175,1.8661111593,-1.2324781418,0.2694059908,0.9574196935,-0.4235167801,1.3931517601,-0.5280845165,-0.7973154187,-0.1673012674,-1.2300739288,-0.1831893921,-0.9138897657,0.5805856586,-0.0222635027,-0.4146025181,0.9250200391,1.1654540300,0.1428458989,2.0506296158,0.8355020285,0.6833344102,-0.6771549582,-0.1392180920,1.3074415922,0.6128761172,0.9603426456,-0.9988200068,1.3938060999,1.4483420849,2.3093905449,0.8961781263,-0.4832390547,-2.5290327072,0.6685495377,-0.9248668551,0.5174524784,1.5624305010,-1.0747776031,0.0684774816,1.2940604687,-0.2854608595,0.8098345399,1.0701078176,0.0960196704,-0.7887099981,0.3971018493,0.3612840474,0.0241373852,-0.2990406454,2.1660528183,1.1026490927,-0.9874019027,-0.0388727337,0.2446159571,-0.5563523769,0.2279099971,-1.0013686419,1.0143290758,-1.5527062416,0.8506965041,1.2558175325,-0.2148320824,0.8770825267,1.1198774576,0.3611235023,-0.2671338916,0.0430668853,-0.1653925180,-0.9529352188,0.0276208688,0.1230975538,0.8873025179,-0.1752104014,0.2513716519,-0.2348491400,2.1551887989,0.7347248793,-0.4261212051,-1.4302924871,0.9446163177,2.5085418224,0.4724510610,-0.1860275865,-0.5170155168,0.8182741404,0.5141733289,0.0327660330,-1.5501236916,-1.9620072842,-2.0062484741,0.9044618607,-0.1077708378,0.6405754089,-0.5198460817,-0.1346986145,1.0445538759,-0.0471237898,0.6623798609,-2.9060950279,-2.2693784237,-0.9272943139,-0.7652207613,0.6250887513,1.0728448629,-0.3229994476,0.6231277585,-0.8498978019,-0.7956701517,0.2162239701,0.3675494194,-1.1451079845,-0.9653976560,0.2594099939,1.9920861721,-1.0531497002,1.0164002180,-2.6073150635,0.1700922847,-0.6301968694,0.2218909413,-1.0393652916,0.1635822654,-0.8220946193,1.2132215500,-0.2270285189,-0.0932268277,-1.9150704145,0.5888198614,0.8222286105,-0.9247884750,0.5923997760,1.2341030836,0.2066896558,0.0557046607,-0.4123021960,-0.5475937128,0.0371494219,-0.4496034384,-0.3558986485,-0.0194285549,-0.2425739467,1.0339695215,-0.9851918221,-0.5976397395,-0.8841294646,1.6174160242,-1.9489552975,0.7389916182,-2.6600670815,0.4461476505,-0.8088853359,0.8462277651,-0.6602464914,1.6914891005,-0.2079225481,0.1770420671,-0.1914528757,1.8777939081,-1.0028346777,-0.3571747243,-0.5823912621,-0.3817023933,-0.7646320462,0.3977974951,2.1370313168,0.0228019860,1.0142006874,-0.5090188980,2.4213712215,0.2384048551,0.4153284132,-0.5815684795,-0.3279374242,2.0014894009,-0.2259496450,-0.7761495113,-0.7924335599,-1.6703323126,0.2091366947,1.5131661892,-0.2143556774,0.4151538014,1.4834935665,1.0899714231,0.3293718398,2.0831754208,0.1978849769,0.2735405862,-0.6502671838,1.2972633839,0.1335616857,-0.3772461116,-0.2008029372,0.0285819676,-0.5361480117,-0.2193754762,0.2928190529,0.1224274337,-0.9885019660,0.0658708662,-1.2322801352,1.8402957916,0.6772176027,-0.7635117769,-0.2598209679,-0.9752302170,-0.9368657470,0.5052095056,-0.5493775010,-0.7561277747,1.1606968641,-0.8948866129,0.7956269383,-1.0711044073,0.4195824265,0.4358394146,-0.5332286954],[-0.7573457956,-0.6774635911,-1.6076130867,1.3072478771,-0.5927714705,0.1389958411,0.7185947895,3.7669422626,-0.0290491264,1.9280542135,-0.7334471941,-0.6078340411,0.8206052184,-0.7893049717,1.1871837378,0.5642030239,-0.4392837882,-0.5427054763,1.4480323792,0.9012516737,-1.4478220940,-0.2742679417,0.3881469667,-0.7936916351,0.9068234563,-0.2690926790,0.6635046005,-0.8194814920,0.8768583536,-0.2403390408,-0.4118672311,-0.1585797518,0.7974650264,-1.6701654196,0.2306298316,1.2565150261,0.7238920927,0.2698653042,-0.1637378335,-1.1882640123,0.1818947494,-0.7447010875,-1.3373509645,2.0189678669,-0.2577935159,-1.8308736086,0.5547665954,0.5500036478,-1.5290178061,0.2087140232,-2.4184942245,0.1614990234,-0.5703580976,-0.3850896358,0.3657353520,-0.6663386226,-0.5914227366,-0.1594028920,0.5978386998,0.6462613344,-0.5100587010,0.9332790375,0.1504412591,2.2288496494,-0.0264447983,-0.3791185021,-1.0874207020,1.3430011272,0.1849463731,0.1412463486,0.4610200822,-0.4325022697,-0.1002900004,0.4237989783,-0.0037172488,-0.7439040542,1.3982913494,0.6422342658,0.2297436744,-0.3174774945,-1.8848919868,0.4019989967,-1.0464760065,-1.4771206379,0.9808022380,-1.1731636524,-0.5929113626,1.4529060125,1.8952001333,0.8750221729,-0.8161361217,-0.8773684502,1.5960000753,-0.0505642667,2.1601340771,-0.5453072190,0.0826762095,-0.7931586504,-0.4187310934,0.8851934075,-0.0452485234,0.7194694877,-1.1667468548,-0.8115680814,1.0736287832,-0.0052045393,0.2035568357,-0.2131816298,-1.4077930450,0.4754085839,-1.6127541065,0.9811051488,0.3119699657,3.0805301666,1.3608472347,-0.4485848546,-0.5215060115,2.0166110992,-0.3327984214,1.2407314777,0.6968078017,0.0539997965,0.0699510276,-0.1169878095,0.8041012883,0.1101086438,0.0337015390,0.6410145760,1.6915745735,0.0526624918,-0.2966575623,-2.1406083107,-0.8456923366,0.7050642371,-0.0481108874,0.1677255780,-0.3968006372,-0.2574980557,1.8308932781,0.7751365304,1.7864623070,0.4759546816,0.3503513336,1.3485484123,-1.1443794966,0.4006808996,-0.9962391853,-1.1875445843,-0.9584485888,0.6679587364,0.8440365791,-1.6700847149,-1.6379746199,1.4682731628,-0.1765896976,0.0353947654,0.1196492165,2.1720237732,1.5219050646,0.2371326536,-0.2789554000,-0.6909232736,-0.1486644447,-0.7018318176,-0.6372700930,0.4759524167,-0.6893990040,1.1211477518,-1.2002559900,0.2479523867,-0.1926452816,1.7607023716,-0.5664526820,-1.9813816547,1.9055066109,-1.4104237556,-0.5574155450,-1.0331728458,-1.3837974072,0.2761999965,-1.9508929253,-0.0858562961,-0.4836938679,1.6077718735,0.7905129194,0.2117312104,-0.4057725370,-0.2354757786,-0.2259673476,-0.3836088479,0.3403173387,-0.2910141647,0.2482576072,-1.5559599400,0.3220538497,0.7779824138,-0.2298941314,0.7137753367,0.0410445072,0.8990941048,-0.0591654740,-1.2160205841,0.7276846170,1.3633673191,0.2227507532,0.0442474857,-0.0720062032,-0.4402479529,-0.2421229333,-0.1403012723,-0.8662195802,0.7516864538,-0.5260761380,0.3029883206,-0.1204497144,-0.8628455997,0.6711946130,-1.5811458826,-1.0912187099,1.5749882460,0.8000087738,1.4918595552,-0.0239492729,-0.6851808429],[1.6194691658,-0.6268472075,0.7402077913,0.3828008473,0.2770670056,-1.6399253607,-0.7085242867,-0.4483987689,1.1894177198,0.1778671741,0.7329412103,0.2275713235,0.9885006547,0.3365898430,1.0390247107,-0.3508904874,-0.7135981321,-0.5375814438,-1.8914824724,-0.8740919828,-1.0973818302,-1.7287472486,1.2013589144,-2.6296367645,0.7503966689,0.1960070133,1.0780261755,0.5065403581,-0.5458502769,0.3345984817,0.1483134329,0.9152549505,0.1478965133,1.5200810432,-0.7546271682,-2.1418476105,-0.2708940506,-0.2287595272,0.6522660851,-0.2509295642,2.2269825935,0.5684635043,1.0577745438,0.7382932305,-0.0012409586,1.4664787054,-0.1449816078,-1.0437244177,0.6076418161,-0.4137753546,1.8593870401,-1.4749391079,1.8089412451,0.0769915134,0.5412765741,0.3424822688,1.3275287151,0.1800303906,0.0366178341,0.3738724887,-0.7425840497,-0.0967953280,0.6599724293,1.0047409534,0.9127472043,0.5290979147,-1.2235962152,1.8528863192,-0.3413074315,-0.3027526140,0.4114413261,0.9746417403,0.1835276186,1.0362179279,-0.6989639401,-1.2286016941,-1.2848297358,-0.9485433102,1.5757834911,0.2003630102,0.3445673585,1.0008212328,-0.0349921770,3.7082657814,0.8298263550,-0.6615275145,1.3744132519,-1.6828603745,0.5283031464,0.6071580648,2.1209676266,0.9397947192,0.5499040484,1.2169115543,-1.5446734428,0.0012747148,1.7734676600,-1.0784699917,-0.9334080815,-0.3895108104,-0.4134129286,2.5833759308,-1.1176213026,-0.5504791737,0.8675289750,-0.5212416053,0.4319742918,1.1117717028,-1.8729016781,-1.6570641994,-0.9629516006,1.2397440672,0.0037459065,1.0992046595,0.0913702697,1.1402840614,-1.0717334747,-0.9748516679,-0.2703653872,-1.2980529070,0.6257337332,0.6659516692,0.3891419768,0.8163871169,1.4753880501,-0.2891272604,-0.5772545338,-0.3888491094,-0.7375437617,0.1387289315,-0.0836863145,0.1055305451,-1.0039715767,-0.2395810485,-1.6525434256,1.6949305534,0.3663100302,-0.7106671929,0.3623050153,0.1398703009,0.0709123462,0.4497546256,-0.4463917315,-1.8335397243,0.1431326866,-0.5487794876,0.2520763278,-0.4617098868,-0.6493166685,0.8026529551,-1.4981682301,1.4404788017,0.2838287055,0.7307789326,-0.5151305199,0.7872204185,0.8811286688,-0.1221335679,-0.0872639865,-1.3045835495,-0.4357743859,2.2551152706,0.2483468205,-1.9551292658,-0.4188409150,0.0355005972,0.7761195302,0.5764486194,-0.9114215970,0.4186478853,0.0519385114,-1.3720073700,0.9566460252,1.0162568092,-1.2849221230,-0.1405265629,1.1921300888,2.2774107456,0.6626639366,1.2861585617,-0.4691481292,-0.5023907423,-2.0549042225,-0.5954032540,0.6965740919,-0.0113108791,0.9470972419,-0.4827988446,-0.7364525795,1.5049593449,-0.5338101387,-0.5999995470,0.2748598456,1.2512134314,1.3383157253,0.8594609499,0.2186160684,0.7017844915,-0.4627053738,0.0034496156,-0.1348692179,0.3913027644,-0.7634017467,1.2350702286,-0.2394351661,-0.3701135516,0.5651393533,-0.0991172194,-0.0025406210,1.0179543495,-1.3121225834,-0.4948684275,1.2599962950,0.1616281420,0.6271054745,0.3267571032,1.7896078825,-0.1520906836,0.7437182069,0.1602447033,-1.1462504864,0.7538537979,0.1994169801,-1.5422637463],[-0.1115837023,-1.1347322464,-1.5603486300,-1.3231898546,-0.5864630938,0.2438378185,1.0190917253,-1.4585080147,0.2916617692,0.1212151349,1.8103116751,0.0997496024,0.8982300758,-0.6879515052,1.4924706221,-1.4564908743,-0.7765069604,-0.9859484434,0.2000730485,1.2155988216,0.0159493014,1.4958109856,0.1811317950,0.5959467888,-1.3172075748,0.2131920159,-1.2470915318,-1.2693005800,-0.5841072202,-0.0466538183,1.4094572067,0.7706953287,1.0238623619,0.5501586795,-1.9324756861,0.8537841439,0.0228950474,-0.9649195671,-0.1006398126,3.7234966755,-1.4024811983,1.4509285688,-0.0061165541,-2.3573803902,-1.7164319754,1.4367122650,0.4027778208,2.0568888187,-1.4577573538,2.1596512794,0.0112678483,0.4823514819,-1.0151923895,-0.2748779655,-0.1498049647,-0.9562303424,-0.4943629801,0.9176456332,-1.1512298584,-0.6620446444,-0.7779641151,0.3872365654,-0.9894063473,-0.7422857881,-0.3691743314,-0.7506567836,-0.1978930682,-0.8630726337,1.1947299242,-0.6482266784,0.6255560517,1.0509376526,-0.1812707335,0.3463947773,1.2976157665,-0.3504452705,-0.4335976541,1.2804650068,-1.0233523846,-1.5230047703,-1.8323204517,1.0930193663,-0.9281481504,-0.5940221548,-0.7265738249,-1.2965354919,-1.0185232162,0.7093110681,1.1391676664,-0.7207520604,0.0884170756,0.2882946134,0.5193282962,-1.4298005104,-1.6081708670,-1.2717075348,0.4682482779,-0.0170615129,1.7580053806,-0.9173281789,-0.4693002105,-1.9680835009,-0.3299243450,1.0726827383,0.8806637526,-0.7417538762,-0.6689109802,-1.7082813978,0.4313643873,0.0347285047,-0.9227795005,-1.4209941626,0.7686933875,1.9805835485,-0.8032012582,1.6413334608,1.3727482557,-1.0299570560,-1.0851159096,1.5018138885,-1.3383134604,0.9689275622,-0.9780363441,-0.4031761587,-0.3334401846,0.6129492521,0.1054987162,1.3304780722,-0.1858203262,-0.4620821774,-1.9098011255,-1.3124620914,-0.7047039866,-0.7252709270,-0.1312124729,-0.9187059402,0.6097781658,0.2633499205,0.6997118592,0.8780248761,-0.1637497842,0.2091602236,0.5623223186,2.0462377071,-0.8705199957,-1.1179158688,-0.5697802305,-1.0332211256,-0.3013561070,-1.1498878002,-1.2747392654,-0.5573811531,-0.4904537797,-0.7212755084,0.3405832052,0.2075432539,2.3557858467,-0.3203197718,-1.0107301474,-0.4470166564,-0.8744552135,0.9583758712,-0.3872155547,-0.3068799078,0.2179597914,0.5574518442,1.4925589561,-1.0170717239,-0.7130813599,0.1082530543,0.7955113649,0.4275724590,-0.7770849466,0.3904846013,-2.1188902855,-0.8035373688,-1.8661222458,-1.3831884861,1.5261973143,0.9782083035,2.0999190807,-0.9385385513,-0.7025492191,-0.5098134279,-0.0868382528,0.6528349519,-1.4956269264,1.7219619751,0.6622504592,0.0382152312,0.4614329040,0.5472880006,0.5798900127,0.5045868754,-0.6498355269,-1.9459872246,1.3929944038,-0.6809290648,-0.2962422967,0.0108508496,-0.9638430476,-0.9484646916,0.5165789723,-0.1640670300,-1.7002995014,0.8765365481,0.1269982159,-1.2878123522,0.6062132716,-1.5158621073,1.2426468134,0.0607079677,0.4924314320,-0.1854637265,-0.1353620142,0.2781190872,0.2474597394,-0.9971805215,-1.0900882483,-1.0746543407,-1.8340034485,0.3812614679,-0.8598755598,-0.1040497944],[1.3925131559,0.3986272812,-0.0415031575,-2.3675251007,-0.0305877812,1.6660884619,-0.1070648730,-0.5445333719,-0.7710431218,0.3810178041,0.2547812760,0.0593349338,-1.8837507963,0.5730901957,-1.0793691874,-0.4145066440,0.8921239376,1.0510808229,1.2406498194,0.1838686913,0.2870498300,-1.5952960253,0.3279765248,-0.4117076099,-1.3453356028,-1.2739416361,1.5920748711,0.5261922479,-0.0942941159,-1.0936295986,-0.6000131369,1.0554364920,1.1460963488,-0.8146736026,0.8606137037,1.5689191818,-0.5961349010,0.0758933052,-0.3385274708,1.3630273342,-0.5600120425,0.9728752375,-2.6392047405,0.3673083484,0.9900626540,0.3304453790,-0.1471444517,-0.2834138274,-1.0353995562,-0.5525274277,-0.5576259494,0.1510661095,-0.2260467261,0.2252246886,-0.6449528933,0.5770357251,0.6935384870,-0.1395560056,0.2972361147,0.1321164519,0.1647407860,0.3600349426,0.2307815105,-2.8135678768,0.0207150485,1.1604722738,-0.6801362038,0.8473317623,-1.2867816687,0.9362281561,-1.0985594988,0.3639356196,0.0338978171,1.3219554424,0.1933799237,-0.4506433904,0.8257185221,-0.2247683406,-1.3679728508,1.0385216475,0.9364742041,-0.6909128428,1.5373262167,0.6860220432,0.4178827405,0.6212495565,0.4344584346,-0.8601510525,-0.3218546510,-0.8268820047,-1.5539178848,-0.0130258258,2.2126760483,1.7009296417,1.0740090609,-0.3324290514,-0.8985465169,1.4576840401,0.4782959223,-0.3509194255,1.1262203455,-1.2219204903,-0.2200453132,1.3531450033,-0.1954530925,2.3975813389,1.8100578785,-0.5661913753,0.1861822009,1.3492763042,1.3133230209,-0.6185108423,1.3017319441,0.9068949223,-1.0815957785,-0.4340805113,-0.3503829241,1.4641578197,-2.0569789410,1.0820784569,1.7439060211,1.8248600960,-0.7774598002,-0.4939185083,1.0127602816,0.7717211246,-0.3367897570,-1.2080198526,-0.6399074793,1.1590770483,-0.1210201681,1.1723109484,0.8146342635,0.1039312407,0.8836551905,-0.5924564600,0.1072652787,-0.3974935710,-0.2205309421,-1.1659206152,-0.8362118006,-0.8025512695,-0.2125316709,0.7141345739,0.6295037270,1.1987661123,0.1260221899,-0.1531082541,-1.0310268402,0.0731412917,-0.1965200156,-0.4568440616,0.8053567410,0.2883935869,-0.6044726968,-0.1260431856,-0.0165450405,0.6680634618,-0.2518533468,-1.0752733946,0.3414058983,-0.7781500220,1.3908762932,-0.4280652702,0.3502120078,1.5740196705,1.5462254286,1.3005847931,0.2207820117,1.2206280231,1.2158526182,-0.4125004411,0.3144003749,-1.4685095549,-0.0741400197,0.5391044617,1.2650680542,1.1636080742,-0.6286639571,0.3842039406,0.4505725801,1.3273755312,-0.7348456383,0.7658395767,-0.4287714362,0.3170690536,-1.6846287251,1.2574837208,2.2304196358,0.1366726011,1.1004315615,-0.1765469760,-1.1436213255,-0.4769022167,-0.0776945502,-0.5553138256,0.6878874302,0.7855275273,1.6607589722,1.0358875990,0.4534233212,-1.6446279287,-0.1976266950,2.1636257172,-0.7871826887,-0.2852320671,0.4802152514,-1.8419892788,-0.2016286850,-0.0867040604,-1.0645453930,-0.8415598869,1.5475199223,-0.3730809987,-1.4787642956,0.2056966722,-0.3578105569,-0.1109743640,-1.2149100304,1.2062696218,1.3023071289,0.9274048209,-0.7516996264,-1.3269295692],[0.3645735383,0.2758433521,1.9448007345,-1.4305678606,-0.0758651718,0.0477598943,0.7821953893,-0.5015829206,-1.6125231981,0.1059435233,0.6842874885,0.9025937915,-0.6946648359,-0.2183943540,-0.3142189384,-0.0116561214,1.1320439577,0.6315576434,-1.1208635569,-0.3343107104,0.5499612093,0.8175386786,1.5596295595,0.5662127733,1.5119391680,-0.1291352212,-0.1208290309,0.5914816260,-0.8418577313,2.0425741673,-0.2564697862,1.2515211105,0.6368747950,-1.1409413815,-1.0605794191,-0.1664215922,-0.9259193540,2.5111291409,-0.0861898288,-2.0625565052,0.6647928357,0.0328417681,0.5346663594,1.9692085981,0.4651908278,0.0526132919,-0.4012923241,1.6575239897,1.1805459261,0.5314475894,0.7792928219,-0.5494493246,0.3818164170,-0.0567653663,0.4041859508,1.4742921591,-1.0135000944,0.4748797119,0.7541258335,-1.0927159786,0.0926267952,0.9003172517,-0.5001785755,0.5012148023,-0.6418183446,-0.9861421585,0.9376791716,-0.5699564815,-0.6094684601,-1.0713261366,1.9327700138,-0.9455596209,-0.9719607830,0.6471062303,-0.0367334001,-0.5560221672,-0.3756830692,2.0556206703,-2.3344113827,0.3166075051,-0.0935946330,0.9930018783,0.5013293028,-0.1114913821,1.7273072004,-1.0956972837,0.1108219996,0.0784082487,-0.1691263020,0.2886268198,1.7784488201,0.1265564561,-0.5149015784,-0.2519254982,-1.0531100035,0.7891155481,0.4315459132,-1.5275148153,1.4729213715,-0.3302557170,0.0881882012,-0.8817841411,-0.7540366650,0.8779074550,-0.8743508458,0.2878976464,-0.5434914231,1.0279676914,0.9953140020,-0.8819229007,-1.3638242483,0.3675503135,-0.3905891180,-0.7033363581,1.2411277294,-0.3832196891,-0.5711125731,-1.4024157524,-0.3557167053,0.3662314713,-0.6938498020,0.0533289164,0.3695681691,0.1856286973,0.0036913755,0.1313543767,0.8270304203,1.3178888559,-0.3671407402,-0.2499872595,0.3193106651,0.3666144907,-0.4536692798,0.0637796372,0.6246864200,0.0064440314,0.4970657527,-1.6432136297,1.0414912701,0.5748828053,0.1528365761,1.0264930725,0.0289772060,-0.7684674859,1.8304229975,1.9216315746,-1.1279611588,-0.2746619284,-0.1850537807,0.7062392831,0.3555676937,1.0713543892,0.4431940615,0.1439880580,0.9895448685,-0.0911555588,-1.7978384495,0.5766109228,-0.0966744050,0.0376248136,-0.9085863233,1.7782652378,1.0522457361,0.6517040730,-0.8900560141,0.6581432223,0.6606887579,-0.4885232449,-1.1564961672,1.9684470892,2.1824321747,-0.3049057424,-0.8701323867,2.3631441593,0.3133211434,-1.1149035692,-1.3495191336,1.9155670404,-1.0757849216,-0.6060916781,0.3758997321,-0.9581190348,0.6761481762,-1.5759015083,0.0877688527,0.9830061197,0.5539374948,1.0762681961,0.2258782238,1.5641163588,-1.3022426367,0.4803126752,0.6107187271,-1.0131540298,-1.5784831047,-0.8989061117,0.4761588275,-0.9230661988,0.5590661168,-1.0126085281,-0.3330108821,0.4091787934,0.8562009931,1.3837891817,0.9529459476,0.0378294997,1.7116134167,0.1376933604,0.4267888963,1.4159948826,1.1380883455,-0.8711742163,1.3790419102,-1.1110154390,0.0842593387,-0.2383049726,0.3928242922,-0.0319515020,0.6350234747,-0.5745252967,-1.0377408266,-0.5247514844,-0.3461388052,0.2433131039],[1.9716480970,0.8219295740,0.6601530910,-0.9692701697,0.3216661215,1.2229069471,0.9288674593,0.3407813609,-0.0449075103,-0.6433902383,-0.3510982990,0.4039245546,1.4285668135,2.1478550434,0.1891006231,0.5928854346,-0.0630526319,1.3296774626,-0.2089577764,1.1789846420,-0.8879231811,-1.1200343370,0.5014985800,-0.6400913596,1.3707021475,0.8418042064,-0.0632963851,-1.1129459143,1.2401936054,-0.2269525230,1.8008857965,1.1790270805,0.5483096242,-1.9282461405,-0.0091208350,-0.0526960641,-0.5831608176,0.7987315059,-1.6418938637,-0.9462875128,0.3626517355,0.7144135833,0.0920099840,-0.1160614640,-2.1590559483,0.8512916565,-0.9435035586,-3.0365452766,0.3538107574,-0.9204629064,-1.2248587608,-0.4802761674,-0.0648367479,0.3905614316,0.5931641459,0.4110111594,1.9805357456,-0.5985766649,1.4174280167,0.2611472011,0.4249542952,0.8081253767,-0.4066265523,1.3087968826,0.3670448959,-0.4409090877,0.1955190897,0.8306531906,1.0105479956,-1.2028720379,1.6242797375,-1.0599883795,1.1703038216,0.3944687247,-0.0508274548,-1.7612975836,0.9173631072,0.1352318525,-0.9668301344,0.9786379337,0.9339951277,-1.3411684036,-0.8798589706,1.2675123215,-0.3759378195,1.0647661686,1.0037868023,-0.5811907649,-2.7897138596,-0.1828412116,-1.3186894655,-0.6859114170,0.3702635467,0.1879754215,-0.2166812122,-0.4224203527,-1.1869359016,1.4768501520,-0.1624441296,1.3615174294,0.0276617203,1.2067369223,1.8776559830,0.4511460364,-1.2373962402,0.7331387997,1.7680873871,0.9055304527,0.3431780040,-0.7190120220,-0.8236030340,0.3431842923,-0.2256057709,0.1887262464,0.7279495001,0.1026036218,0.0037209450,0.2864089906,0.1306706518,0.3032578826,-1.9690222740,-0.3773420453,-0.3418140411,-1.1602931023,0.6077716947,0.9177352190,-0.7473893762,-0.3076132238,-1.1850211620,0.1065358967,1.5186152458,0.7725510597,-1.0014253855,-2.1937179565,0.3250127137,-0.6440759301,-0.1811335981,-1.1045614481,-2.1818692684,-0.2243866771,-0.0173453316,0.3536208272,-0.0725474954,1.8809698820,-0.2315787226,0.8793317080,1.1665952206,0.0993929505,-1.4947397709,1.0953342915,-2.2650103569,1.1013573408,-0.3124994040,-1.2784965038,-1.7349416018,-0.7721288204,-0.9405128360,-0.0988894254,0.9771395326,0.6489016414,1.6465309858,-0.1293012649,0.0067455000,1.6675785780,-1.0477951765,0.4693665802,-1.3260034323,1.6122599840,0.8219315410,-0.2401393354,-0.4720887244,1.1021941900,0.4860531688,-2.6774544716,0.7096341848,2.5483486652,-0.2957075238,-0.6031370759,0.1926307380,-0.4034855664,0.1463181674,0.5350070596,2.0819504261,1.8139798641,-1.0643401146,-0.3128781021,-0.1442638040,-2.1851909161,-1.5748522282,-1.0711203814,-0.4753435254,0.0661187768,1.1281665564,0.1130188406,0.8959493637,-0.0683412403,-1.3801412582,-0.9871893525,-0.7403144836,-0.8445615172,1.6470441818,1.0460041761,0.6926950812,-0.7393996716,-0.9453443885,-0.2465624362,-1.2683348656,1.9672598839,0.9712939262,-0.7164001465,1.2384980917,1.3134400845,-1.2783517838,-0.9249799252,-0.9393361211,-0.6579603553,-0.2182013690,0.9198998809,1.2773807049,1.1855596304,-0.6381624937,-1.0942879915,0.4838123024,-0.4306229949],[0.1810165793,-0.1996294707,1.2885905504,-0.1678638160,-0.1881410182,1.3225277662,0.0953734741,1.8875477314,0.0770172328,1.6916360855,-1.7926950455,0.4905884266,1.0937842131,-0.0595675372,0.9946762919,2.0324313641,-1.0170233250,2.0547215939,1.6750184298,-1.0087105036,-1.6904520988,0.3167631328,0.6377083659,-0.7585461140,-0.6136844158,-1.8868082762,-0.1847042292,0.8395951390,-0.1964798272,0.0806577355,1.1836705208,0.6784030795,0.8249893785,-1.2950379848,-1.1152267456,0.6911618710,-0.4725620151,-0.6914433241,-0.2887046039,0.8318896294,0.5734977722,0.7145007849,1.7885981798,0.0377317369,0.7398540378,-0.6245130897,0.7582843900,0.8068283796,1.8076828718,0.6233335733,0.6361873746,-1.2487183809,0.2187319547,-0.6943225861,0.3531611562,0.8272674084,-1.9904072285,-0.9367148280,-1.0429241657,0.6981034875,-1.5150380135,1.4994618893,-1.4663890600,2.2427587509,-0.3023525774,-0.6387372613,-0.8940060735,-0.1322064400,-1.0280362368,-0.4068548679,0.2255519480,0.9413877130,0.0257321727,-0.1329892278,-1.6131180525,0.8192076683,0.1125036925,1.3352439404,0.6392394900,-3.1156716347,2.0869877338,1.0112729073,2.4250719547,-0.1496810317,0.7387430668,1.2361007929,0.0684093386,-0.9277183414,-0.1533320844,-0.7304266691,0.6191169024,-0.7954546213,-2.1412179470,1.2499122620,-1.3871273994,2.1028695107,-1.0435626507,-0.5314572453,0.2357278168,-0.7836205363,1.6709499359,1.4669779539,3.1746845245,1.3023787737,2.3591947556,-0.1380295604,0.9297934175,-0.5155693889,-0.1036393195,-1.1186670065,-0.1851234734,-0.2512214482,1.1714904308,-0.2232470512,-1.4790050983,1.2210992575,1.7563029528,0.4102286100,0.0601698495,0.2168567479,-0.8762237430,0.9252765179,-1.7906960249,-1.3530749083,0.0037942464,-0.6046807766,-0.4491837025,0.1465787441,-0.2187786400,2.6423318386,1.8395512104,-1.2164244652,-0.0062938496,0.7514982820,-1.7471815348,1.3979443312,0.4519581199,1.2722970247,0.0478293486,0.0334270969,0.5252046585,0.8231018782,0.4410232306,-0.2736770213,-0.6498092413,0.6456122398,-1.5126706362,-0.9126222134,-1.0825484991,0.8857665658,-0.4664177299,1.2132717371,-1.1514225006,-1.0739794970,0.1377086192,1.9164336920,-1.2203828096,-0.7694478035,1.3315281868,-0.7050071955,0.8304207921,0.6928241253,-0.4450389445,1.1498022079,-0.6999378204,-0.6708523631,2.8639185429,0.7906737924,0.6672154069,0.7954690456,0.1095909402,0.1944302619,0.0010998135,-0.4005476534,0.2124238312,0.4286309779,0.2146281302,0.6275988817,-0.0620207116,-0.4323121309,-1.7828309536,1.5191432238,-0.3685704768,0.3279248774,0.2474352568,0.7215018868,0.7299074531,0.0056346082,0.6240631938,-0.7521722317,0.7414829135,1.3525691032,0.2047773600,0.8658626080,-0.2844838798,0.5375939608,0.0502211004,-1.9202650785,0.6328778863,-1.0243452787,-0.3291671872,1.2854202986,-0.3071571589,1.1487460136,-0.6724426150,0.9937167764,0.0755469427,0.0181107987,0.3345252872,0.0915125236,-0.1587377340,0.9943240285,1.0686699152,-0.0836843699,0.8532247543,-0.1283108890,0.5830957890,-0.5074785352,-0.8843220472,0.9858810306,0.4600271285,-0.9244024158,0.2015917897,-2.2051551342],[-1.3684561253,-0.0070179529,0.9507652521,-0.8961302042,1.3040012121,-0.4097203612,0.5474057198,1.3817955256,-1.2054364681,0.5918625593,0.2912769020,0.3815670609,-1.7695758343,0.2565854192,0.6678588390,-1.9968492985,-0.4847605526,-0.4586411417,-0.2693426907,-0.1771221757,1.7575697899,0.0226126239,1.3108720779,-0.1741402000,0.5192768574,1.5016207695,-0.3459168375,0.8827842474,-1.0666747093,-0.6622051001,0.2035106122,-0.0336035080,0.4757933319,2.3770163059,0.7890660763,-0.4296334386,-0.6260621548,0.8593677878,0.9754520059,-0.5508008003,-1.9562290907,-0.4794707298,2.1985228062,0.7364495993,0.2542623580,0.2583675385,-0.0006999811,-0.0937647223,0.0307967123,2.1079499722,-0.9306992292,2.4908797741,-0.0683824047,0.8416444063,-1.4192576408,-0.2434727401,0.7902024388,-1.5382992029,0.9410758018,-0.7255437374,-2.1972353458,1.4353498220,0.6533204317,-2.7115106583,0.6271278262,0.3922728002,0.1899161339,-0.2647828162,2.3061597347,-0.3845082223,-0.2433109134,0.4348921478,-1.5491979122,0.1541192085,-0.4593312144,-0.9138321877,-0.1297478825,-0.2293144763,1.6444797516,0.9206303954,0.2121412903,-1.3027334213,0.8136668801,-0.7190756798,-0.3822887242,0.2437139153,-1.1598749161,-0.2366128266,0.9961916208,0.9275527000,1.0301316977,0.8716277480,1.1247831583,0.7137174606,-0.0485413894,0.5693393946,-1.9486206770,-0.4884814322,1.6192941666,0.8910787106,0.0180339478,-1.7390911579,0.2681433856,-0.1390723884,1.0840507746,-0.3824338019,-0.5290316939,-0.6295898557,-1.5379174948,1.1584496498,0.6609149575,-0.6455150247,-1.4774763584,1.3007858992,-0.8344992399,0.2422126979,-1.0668669939,0.7216842175,-0.2747242153,0.6934828758,-0.3771536648,0.2721296549,1.6877368689,-0.3436425626,-1.3526052237,0.6161730289,-0.3104036152,0.4038379490,-0.8696197867,-0.0881191045,0.8002542853,0.2152355313,-2.1688718796,-0.1544931531,0.5612537265,-0.7872266173,-0.6685209274,1.3880770206,-0.0438574255,0.6691686511,-1.1393299103,1.5715401173,1.7690942287,-0.1770737022,-1.2838332653,-0.1482763737,-1.1807324886,-0.1246396676,0.5556858778,-0.5587546825,1.9595792294,-1.0273351669,-0.2892897129,2.1519374847,1.4404784441,-1.6324875355,0.9220415354,-0.8804201484,0.9086288810,1.3144823313,0.1263292283,1.1237426996,0.1560661346,0.9178130031,1.0348485708,-0.0764593482,-0.1803882867,0.4778284132,-2.6791322231,-1.4770958424,0.0729591846,-1.4535052776,0.1024222225,-1.0942270756,0.3704869747,0.4322766662,0.2548621595,0.6323297024,-1.3469249010,0.9590857625,-0.7356510162,-1.4300709963,-0.8788210154,1.3715491295,1.1217643023,-0.5817310214,-0.6780789495,0.1847612560,-2.2621667385,-1.3844285011,-0.4063298106,-0.4010620713,1.0016220808,0.0181291588,-0.3874219656,-0.6327757835,-0.6283560991,-1.4798924923,0.0586887263,-0.3952905834,-1.0938864946,0.9593856335,-1.2852833271,0.5518447161,-1.1919951439,0.4875165224,0.4057821333,0.9937305450,-0.9238799810,2.6145756245,0.4748609662,-0.3027443588,0.2600146830,-0.2662793696,0.5243310928,-0.7458996773,-1.3094813824,0.2878949642,1.0456981659,-0.3525495529,-0.8172661662,0.2186447978,-0.5194003582,-0.2409159392],[1.5012190342,-0.3814423680,-0.2719659209,-1.6522637606,0.8976294994,1.6063685417,-0.3337241113,-0.7124217749,0.8000403643,0.0348635353,-1.2435899973,0.2742917240,1.2672042847,1.3071616888,-0.3089774847,-0.5630499721,-1.6949775219,-0.3910141885,-0.4686586857,-0.4318181872,-0.4553795457,-1.5290528536,0.7596347928,-1.2247896194,0.7929475904,3.2648231983,0.0765919611,-1.7044337988,0.5527252555,-1.2542990446,1.0776225328,0.7700014114,0.3957244158,1.0734199286,-0.3395042419,-2.0320532322,-0.7339397073,1.7088943720,-1.7991150618,-0.3706928492,0.2754571736,0.2273407429,-1.0307626724,1.1256923676,-0.8246914744,0.3956819177,-0.9917507172,0.7536754012,0.6936249733,1.8786382675,0.5164870024,-0.5184623599,0.1207001433,0.2281433642,-0.9246230721,-0.0446610264,-0.3871702552,-1.5042041540,-0.0066607548,-1.5743414164,-0.2049447745,-2.2182295322,0.2431137711,0.5944489241,0.5098665953,1.4379987717,-0.4044253528,1.0512136221,-0.6692307591,-0.5159274340,-0.5512489080,0.5259061456,0.2661730647,-1.5106946230,1.9273676872,0.5774011612,-0.4199075699,1.3673183918,-0.5175925493,0.0277764630,-1.0282031298,-1.2392059565,-0.3175444007,0.0651226565,0.1687745154,-0.7958122492,-1.1371675730,1.8399970531,1.0105547905,-0.5101677179,-2.0513987541,-0.2365717590,1.0404173136,0.8387218714,-1.5863597393,-0.2953415215,-0.3548167348,-0.8969500661,-0.5921047926,-0.4963432848,-0.7090011239,-1.4159350395,0.0659552068,0.6083223820,1.4280581474,-2.7973577976,0.2793337703,0.4457551241,-1.1316871643,0.7324762344,0.0465258770,1.2708383799,-0.8990916610,0.9488687515,0.9713152647,-1.1941430569,-0.3017987013,-0.9432987571,0.7103397846,0.1358730197,0.1415656805,0.5150106549,-1.7309122086,-1.9883096218,0.7167133689,0.0778244287,-2.1529717445,-0.9673052430,0.3667652607,0.1836275011,0.5833052397,-0.4578369260,-0.4256786406,-0.9554052353,1.3549624681,-0.4652006924,-0.2363625318,0.0994006470,-0.6568480134,0.0348669849,0.3077637255,0.3547363579,0.8136194944,-0.6794636250,-1.0101652145,0.7936237454,0.7654732466,-1.3967742920,2.0011563301,-0.8323894739,-0.1461954713,1.0044622421,-0.6343291402,0.2242664993,1.7496575117,-0.4828544855,2.1647851467,0.7960585952,0.0863240808,-1.8356450796,-0.1754117906,0.9070207477,-1.2061300278,0.6664801240,0.2954353392,1.9272356033,1.0407719612,0.4945245683,0.5699928999,-0.1160124764,-0.4865684211,-0.0170620270,-1.6850682497,0.6151177287,-0.6133508086,-1.4380831718,1.6350381374,-0.2955036759,-0.4080443978,0.6777896285,-0.7060639858,-0.4976985455,-0.9679428339,0.0706002116,0.0416285954,0.2848752439,-0.8778271079,-0.1663546264,-1.4794622660,-1.9278879166,1.6978425980,1.7794172764,-2.9531173706,-1.2205433846,1.5037174225,0.5455800891,0.6231148243,-0.1637870520,-0.1267165393,-0.6366458535,0.4950649738,0.3843171895,0.3920426071,-0.8354402184,-0.2788834572,1.0194483995,0.0986561626,-0.5203149319,-0.0823765323,0.0222817399,-1.2570695877,0.0349741466,0.2028528899,0.2815420926,-0.1766107976,0.2764383554,-1.3478579521,1.4862694740,-0.4602116346,0.1060000360,0.6406856775,0.6851381660,0.9093105793,0.5848791003],[0.1880259067,-0.9595815539,-0.2873590291,0.5662950873,-3.1337368488,1.4183876514,-0.3170253038,-1.9341831207,-0.7753921151,-0.5650481582,0.9571687579,-0.0570281744,-0.2114823312,-0.1237578094,-0.5994918346,1.2810485363,2.4784626961,-1.4082845449,0.6607084870,1.2495912313,-0.8925880194,-0.8004648685,-0.1284520179,-0.9761826396,-0.3523084521,1.5950284004,1.3361389637,-0.4479740262,1.9581365585,-0.1791668683,-0.7813001275,0.9748125076,1.9654608965,2.1079697609,-1.1303747892,-0.3680176139,-0.1541169137,-0.4737612605,0.2893137038,0.0942446738,-1.2750767469,-1.6875716448,0.0291447006,0.9104262590,-0.7338591218,0.1654359996,0.4833254218,0.1837635785,-0.0783622116,2.0477280617,0.4575521946,1.4642084837,1.8992501497,1.3101515770,-0.8264897466,1.1015996933,-1.8358174562,-0.5262581706,-1.3427133560,-1.8201973438,-1.3885704279,-0.7330588102,0.9829092622,-1.1346485615,-1.8879934549,-0.4256469011,-0.5990062952,2.0774192810,-0.2175528258,-1.3139890432,0.8544276953,-0.8505318165,-1.5247815847,1.0540881157,-0.7676452398,-0.0154721728,0.0645280629,1.1965137720,-0.7829078436,-0.7461743355,-0.9474745989,-0.4311519861,-0.1322558373,-1.4352858067,0.3022035360,0.8314113021,1.5980254412,-1.2497850657,-0.4789477289,-0.4042346478,0.9194238782,-1.9574488401,-1.5083035231,-1.0556405783,0.3304715455,0.4454304576,0.7204989195,-1.4876512289,2.3934466839,-0.3657112420,-0.6892605424,0.7944124937,1.1503497362,-1.1436711550,0.4530783594,0.1073921546,0.2563025057,-0.2650616765,-0.5858703256,0.3914135396,-0.0370595977,0.6986816525,-0.6059454679,0.7438039780,1.5383099318,1.5709109306,-0.0677266866,1.0556417704,-1.0941773653,-1.9463193417,-1.5928672552,0.7118583918,-0.1078235880,-0.1363062114,-1.1848173141,1.0763760805,-2.1852428913,-1.0170569420,1.0829125643,0.2023819089,0.3322030008,-0.8313293457,-1.2125116587,-0.7498506904,-0.6480713487,-1.0642518997,-0.0535658970,-1.5605756044,-0.3149810433,-0.6010142565,-0.4871916175,0.8242281079,1.8240424395,-0.0996669456,1.5988140106,-0.5990390778,0.0666736066,0.0582151599,-0.4938335717,0.1771529615,-1.6437077522,-0.5018564463,-0.3862938881,0.2370947152,0.4438247681,1.8506821394,-0.8399331570,1.2295398712,-0.0405258946,0.5014107227,0.3826455772,0.0491048954,-1.0085386038,-1.4686864614,0.6855700016,-1.0127818584,0.3318201005,1.3520195484,-1.1127836704,1.1357103586,-0.9928646684,0.8556365967,-1.0570096970,-1.6995784044,-0.9159128666,0.6399106979,-1.0972138643,0.7753602862,0.4758385718,2.3228681087,-0.0455329157,0.8702580333,2.2876014709,2.2667026520,1.0052275658,-0.5927371383,1.3670648336,-1.1437461376,0.3765378594,0.7944192290,-0.1012149379,1.0979666710,-0.3328878284,-0.5348561406,-0.0316795930,0.9100095034,0.7350540757,-0.7042039037,-1.0100349188,-2.6791682243,1.0962806940,0.8025962710,-0.0379705727,1.4441269636,-1.5085335970,0.0003205369,-0.0880573839,1.1053018570,0.1118759215,0.9201207161,3.1936144829,-0.5558418632,0.4877890944,0.0970641673,0.4611878693,0.2145508379,0.0961457565,-0.0012075664,0.4158501625,-0.0128498999,2.2061691284,0.3245992959,0.5149602294,-0.8647653461],[-0.4778534770,0.1767767519,1.1144613028,0.7067543268,0.2953458130,0.4151735902,1.3802677393,0.4514097869,1.4310191870,0.1890203953,-0.0607744493,-1.0145285130,0.4822328091,0.0662695393,0.5966459513,-0.3585332334,-0.4946506321,-0.5472767353,-0.0253793262,1.2802063227,-0.5363033414,0.4587896466,0.2496608347,-0.7562409639,0.6417131424,-0.3292468190,0.4829611182,0.0107420720,1.8334723711,-0.8969556689,-0.5865179896,1.4124860764,0.9335811734,0.6803567410,-0.7399356961,-0.9453975558,0.0589627735,0.9332634807,0.2295327932,1.2144051790,-0.9396538734,0.7404459715,-1.4058126211,0.3758881092,-0.5415832400,0.3609177470,0.4216291308,0.0672683716,0.4954783320,-0.3137744963,0.0770393088,-0.7609525323,0.6436778903,0.6758462191,-0.2501106262,-0.1796221882,-0.4029569924,0.2886023521,0.1287038326,1.7206531763,-0.2571421862,1.5516285896,0.1779292226,1.8721263409,0.4874676168,-0.3857848346,0.7131745219,-0.1883796006,-1.1051175594,1.0015405416,-0.0806301087,0.5015828013,-0.7649217844,1.2179169655,-0.6834611297,-0.3220885992,0.1348572373,0.3637213707,-0.0528043061,1.0957769156,0.2416266203,0.3221211135,-0.1729038656,0.4899542034,-1.3926836252,-0.8075534701,-0.4263211191,-0.3185393810,1.8184756041,-1.4812299013,0.4924119711,0.1793278903,-0.3062870204,0.7527695894,0.6813973188,-0.4136724770,0.1930301487,2.4636559486,1.0544172525,0.5969682932,0.3538553417,-0.7953445315,0.4444134831,-0.9988148808,0.2708710730,-0.2231728882,-1.5424858332,0.0671981946,0.7165861130,-1.7390002012,0.1175277680,0.5987126827,1.2449854612,-0.4765720367,0.9063929915,0.5871992111,-0.2316968739,-0.4770857990,-2.0407674313,-0.8889138103,-0.4778654575,1.0535147190,-0.8856458068,-0.0943486169,-1.9650886059,1.5634150505,0.9272034764,0.8704501390,-1.2399034500,-0.0045439149,0.0171208791,1.5940405130,0.5065575838,-1.0628005266,0.6307820082,-0.0456372090,-1.1207317114,0.0359663069,-0.6097961068,0.2768594623,-0.9132888913,-0.8086346388,-0.5995755196,1.0584326982,-1.3386423588,-0.0910859630,1.2630821466,-1.1991431713,-1.0497963428,0.6557671428,-2.8756728172,0.0049800058,1.1699956656,1.7030316591,0.5046100020,-2.2967424393,-0.4492062628,-0.4919267297,-0.8143878579,-0.7950159311,-0.5516642928,-0.1837606877,2.1352300644,1.1462190151,2.0455040932,-0.5048813820,0.8189655542,0.1658494473,0.3154302835,1.2556283474,0.7457754612,0.3385994136,-1.6795654297,-0.6067926884,1.2469490767,0.5089952946,-0.0530367568,0.5155069828,-0.6019174457,-0.1175310239,0.3800863326,1.4116129875,0.4822998345,0.6734839082,-0.7265940905,0.9609754682,-1.0695693493,1.2773134708,-0.0383228734,-1.1684441566,0.6060315967,1.6832801104,0.7533211708,1.5771530867,0.4684437513,-0.2532663345,1.5824003220,-2.6677989960,0.7695742846,0.4340322316,0.6323025227,-1.0616002083,0.6552160978,-2.0156300068,-0.2300008535,0.6324152350,-0.8441838026,-0.7557621598,-0.1748217940,-0.6759279966,-0.8832311034,2.3019340038,-2.3870930672,0.0982676297,0.1205674782,1.3445794582,-0.2255969346,0.6980847716,-0.4308630526,0.8111328483,0.9373266101,-1.6052649021,1.4621964693,-0.2540515065],[0.8156651855,1.0076911449,-1.7301025391,-1.6204510927,0.7440069318,0.0341309048,1.5476413965,-0.8900391459,-0.3766621947,-0.6074256301,0.3939062655,-0.5218016505,1.2918245792,-1.3143782616,1.5072596073,1.0231168270,-0.2359389961,-1.1725305319,-0.9327100515,0.6893956065,-0.2224030793,0.1603256017,-0.4459508359,-0.1136763841,-0.1959608495,0.7789053321,1.0510292053,-0.0700007603,0.5266662836,-0.9966610074,-0.9432050586,0.7437862754,1.3008739948,0.3434124291,0.0258266907,0.0869140998,0.6960690022,0.7120946646,0.0515276678,-0.0726527721,-0.3411781490,-1.3804582357,-0.2881645858,0.5035392642,-1.3463668823,-0.6638413668,-0.5775796175,-0.5389131308,0.4343502820,-0.2203941643,1.6989192963,0.5641354322,1.6838576794,1.2410887480,0.1104821563,1.7673959732,-0.8824916482,1.3487502337,0.5774065256,-2.2027637959,0.3938376009,0.2434094995,-0.4469590783,-0.1877329051,-1.0865863562,-0.6238707304,0.1701171249,1.0134552717,-2.2195823193,-0.5561084747,0.0621400401,0.2244819254,1.0336713791,-1.3963009119,-0.2985310555,0.3228592873,1.3612934351,-1.4880374670,-1.6463325024,0.4461197257,-0.3382137716,-0.4326903224,1.2684216499,1.2643234730,1.0895330906,1.2925497293,0.1350784600,-0.4523508847,-0.8258852959,-1.7149457932,0.6874905229,-0.7296146750,1.2389507294,0.9146369100,1.3814866543,-0.0028754754,2.3910377026,0.1634847671,1.5939153433,0.1600411683,0.3318611383,0.4221729338,-0.9187315702,0.3347060084,-0.4368286729,-1.2759553194,1.1613256931,0.4264028370,-0.2972605228,-0.0180641264,-0.6553592682,-0.5903983712,-0.5169649124,-0.4399635792,0.0741655752,-1.2233260870,0.2017238438,0.9079591632,1.0736171007,-0.4128850102,-1.1717312336,0.6035646200,1.5451973677,0.6227061152,1.2984673977,1.6092420816,-0.9485616088,1.8946329355,1.2957694530,1.0307075977,1.4255412817,1.7190335989,-1.3016798496,-0.9480474591,1.1068694592,-0.8514854312,0.3590765297,0.4884167910,-0.2413618565,-0.2864337265,1.9658122063,-1.2297976017,-1.3506499529,-1.0592341423,-1.2962996960,0.6311379671,2.3245151043,-0.8188385367,0.6870383620,0.0071533509,0.4189999998,0.5118589997,0.9282453060,0.1976849586,-0.2832446694,0.4292204082,-0.1534999758,1.5900549889,-0.2832624912,0.9434610009,0.7822171450,0.7615031600,-0.3012326658,-0.1083905846,-0.3890056610,-0.5637946725,0.0940228626,-0.1532761008,0.4407454431,1.1213968992,-0.7484930754,0.4968990684,-0.8124283552,0.8374453187,-0.5514011383,0.7271595001,1.1901440620,0.4425831437,0.7223262191,0.0920367613,0.1827518195,-0.9553514719,2.2778782845,-0.0951565281,0.6599346399,-0.4985764623,-1.2338274717,-1.0685179234,0.5681238174,0.4908271134,0.2003903836,-0.1043929160,-0.6512681842,-0.0636266619,0.0732113644,0.4932597578,-0.9207543135,-0.1076298058,-1.6838189363,-0.2428099662,0.7228756547,-0.4640580118,0.6245583892,-0.7864403129,0.9854980707,0.1483794004,-0.1238286272,-1.9773298502,-0.6761142612,-2.0313007832,-2.1170983315,-1.5734508038,-0.8779585361,-0.0909841582,1.4897482395,0.6401346922,-0.4880248904,-0.4623247981,-0.2730410695,0.2345179766,-0.2584047318,-1.0733059645,-0.6118106246,1.8113634586],[2.0679802895,-0.9476329684,0.7383434176,0.0674313679,-1.6945581436,0.3529974222,-1.2724621296,1.0977891684,-0.8179125190,-2.8983523846,-1.2201880217,-0.1745642573,0.0033349809,0.2662228346,0.7162650824,-0.4651221633,0.5385454893,1.1434221268,-0.4105019867,-1.6019604206,0.6570641398,0.3827970028,-0.8969772458,-0.6783589125,0.4421870112,0.3902520239,1.2652155161,0.8713499904,-0.1916929632,-0.1945413202,-0.4036102295,-0.0592381246,-0.7947420478,2.4768147469,-0.5596383810,0.0513080209,2.2183132172,0.7428569198,-1.1224207878,1.2905070782,1.6988288164,-0.0087889144,-0.3099016845,-1.0261619091,1.2815786600,-0.3887642324,-1.8203357458,0.6565842628,0.8482696414,-0.3657097220,2.5571284294,0.9646287560,0.5014576912,-0.7785434723,-1.0838786364,-1.0007174015,-0.6832125783,0.6869196296,-0.8909994364,0.7105440497,-1.5718376637,-1.0136063099,-0.4651289284,-0.9581088424,-0.4869917929,0.8937326074,2.2701437473,-0.5487756133,-1.2599574327,0.2497866601,-1.0012441874,-1.0906326771,-0.0996188372,-1.5389875174,-0.3022162914,2.1879057884,-0.7253212333,1.0624971390,-1.2916557789,-0.3524868786,-0.9313794374,-0.7848801613,1.5815815926,0.1106806770,0.4807272553,0.6900759339,1.2145121098,1.1435748339,-0.7682368159,-0.1096613705,0.6715409160,0.0713482127,0.2610453367,-0.2741608322,-0.2731364667,-0.6862300038,-0.4789772928,-1.3881918192,-0.6710447669,-0.2162043303,1.2445957661,1.1760710478,-0.5608954430,0.2300017178,-0.5237638950,-1.4979652166,-0.4840055704,-1.0160231590,0.0633943900,0.2442258596,-0.1619214714,-0.1240627095,0.2674188316,2.1347377300,0.7680336833,0.7131571770,-2.2288031578,0.3683202863,1.3970509768,0.2402867228,-0.3467687964,1.0369762182,0.9858167768,1.7241109610,0.3450718224,-0.3579460382,-1.4150452614,0.3929713666,0.4741328359,0.2615441382,1.3558253050,-0.1826655865,0.7189281583,1.1006554365,-1.2778162956,-0.6737187505,0.2032980323,-2.3750617504,0.5275279284,0.0699259490,0.0522095188,-0.0179084744,0.7750706077,0.3565326333,0.2300423533,-1.4687960148,-0.0426535085,-3.7455356121,-0.7975733876,-0.2791737318,1.7593774796,0.0835426897,0.8876459002,1.1769098043,-0.8923800588,0.6272987127,-0.8653988838,0.5507037640,-0.1309112310,-0.7270892262,-0.5536189675,0.1476335227,0.9601663947,0.4076004028,-0.8485045433,-0.1765632182,0.2317116857,0.0372135900,1.3644959927,1.3756741285,-0.2507399023,-1.8323187828,1.4180147648,0.3964681327,0.2041271180,-0.3245566487,-0.4420765638,-1.3968116045,-1.3434318304,-0.9108263850,0.6881069541,0.6643660665,0.2395091802,-1.3126059771,0.9548830390,-0.2024228126,-1.4638279676,-0.4380912483,0.1316522658,-1.5384185314,0.2050340176,0.6454075575,-0.9426968098,-0.5764552951,0.6355412602,-1.1904445887,-0.0340364538,-0.8278380632,0.5902572274,0.3412825763,-0.1369599551,-0.0623027273,1.5578794479,0.7105544806,1.0725549459,-1.9698865414,0.8213605285,-1.2326803207,-0.4292006195,-2.2102420330,0.7281572819,0.3447653949,-1.3590006828,0.3094686866,0.2325506210,0.1171084866,-0.0100068245,-2.9082498550,-0.0394074060,-0.7112797499,1.0732913017,1.0485405922,0.8922514915,0.0037649970],[0.3218860924,-0.2153628469,0.1521495283,-0.9304068685,-3.3970711231,-0.5037402511,1.1651905775,0.7021167278,-0.7433487773,0.3981168568,0.9380275011,-1.5853767395,0.0831920207,0.0841076896,-1.0592501163,-0.9714565873,-0.7586913109,-0.6948418021,-0.4219135642,-0.8064761758,0.7126872540,0.4738564491,0.3092348576,1.3096922636,2.0170652866,-0.8423181176,-0.0324107222,0.6601939201,-0.5238676667,1.1682136059,1.5572181940,-0.7529825568,1.6637852192,-0.4154839218,-1.3779836893,-0.1081061214,-1.5806005001,1.8208845854,-0.3819041550,0.5165681243,1.8766587973,0.8654612899,-0.9601491094,-0.3456733525,-0.6894811392,-0.7221453786,-0.6066665053,1.0309771299,0.8200556040,-0.7337735891,-0.1614723206,0.9861069918,1.4574701786,-2.1444618702,0.3525089025,1.7056667805,-0.9310776591,0.6707212329,-1.3782510757,1.1024730206,0.8600340486,-0.2841083705,-0.4831273854,-0.1850067079,0.1130082682,0.2305518389,0.3413489461,-1.0034881830,-1.0885648727,-0.9350720048,0.1794212759,0.4361413419,-1.8060265779,-0.1309453547,1.4649746418,-1.3094321489,-0.3479111791,-0.8606664538,0.3332029581,3.0923016071,1.3583788872,0.0134683866,-0.8860229850,0.2574064434,1.1946133375,-1.1781209707,1.3553206921,-0.6204507351,-0.6351287365,-0.0614457428,-0.5441697240,-1.1895301342,-0.1810240746,-1.1449270248,-0.1968882829,0.5040771961,-0.4403439760,-0.4055637121,-0.4553186297,-0.9085854888,0.0759990811,-0.1860614270,-0.2720640302,-0.1599345505,0.3134514391,0.4481948316,1.9346771240,-2.1949660778,0.8848531246,0.6827428937,-0.9667618275,0.1164045185,0.3335077763,-0.9087823629,1.8471758366,-0.6295967698,-0.8704705238,-1.4105505943,-0.7595064640,-1.2473758459,-0.3623605371,1.0579477549,0.0246553812,0.5469079018,1.0893458128,0.4062927067,-0.3033017218,-1.2926683426,1.0524569750,0.2503056228,-0.3500367701,0.6117320657,-0.6270340681,-0.2911286056,-1.0969469547,1.0318131447,-2.5585491657,1.3501057625,-0.2241195887,-0.4697246253,-0.5922012925,0.3770401776,-0.1585283130,0.8999029994,0.2463238984,0.3050529659,0.0677323788,-0.6213287711,0.5435842276,-0.3985887468,-0.9317046404,-1.2294517756,0.0828140751,1.2535772324,0.0256637987,2.8696305752,1.4507285357,-0.4330395758,0.8850038052,0.4619824290,0.5875666738,-0.3392009139,-2.4040269852,-1.2309052944,0.8450149894,1.1948267221,-0.4159831703,-1.7273284197,-0.4782824218,-0.0359106883,-0.4035047591,-0.1818247736,-0.6230558157,-1.1475069523,0.4307565391,-0.7822400331,-0.4539892375,-0.5645713806,0.7916404605,-1.2734173536,1.1932449341,-0.3262642026,0.1361260116,0.4524342418,1.8424292803,0.2679949403,-0.7157343030,0.9898098707,0.4705828428,-0.5157236457,-0.0149328671,-0.3967804015,0.2779393494,-0.6822986007,0.1624890715,-2.0525116920,0.0262571834,0.2762793601,0.4157950878,-1.2589560747,-1.0236248970,-0.8549474478,0.4833740294,1.2150045633,0.3890248537,-0.0069801174,0.9563995004,0.1099641994,-0.3607292473,-1.0780054331,-0.1775603443,-1.5771030188,2.8553757668,-0.6350323558,1.4035706520,-1.5484172106,0.4918170571,-1.0095106363,2.1850802898,0.9022112489,0.4329253137,-2.1208972931,0.9020392895,0.5670707226],[0.2932744026,-1.7365777493,0.3774078190,0.7109208107,-0.1204144135,-1.3939466476,-1.0072985888,-0.1054091528,0.6280057430,-2.4682083130,0.2006152719,-0.9499309659,-0.8038668036,0.5076775551,-0.9942888618,1.0489151478,0.1733068526,0.0147161372,-1.7972850800,-0.1931378543,-0.4037202895,-0.7168515921,-0.3700949550,0.2424467951,1.3536716700,0.5044452548,-1.1023502350,0.1263127476,0.1913612634,0.4485045075,-0.0773018971,2.0170025826,-1.0194779634,0.3922486901,2.9099795818,-0.1763591021,-0.5611923933,0.2001597732,0.4984797239,-0.4362840652,-1.5781536102,-0.0572126582,-1.6782734394,0.1083617732,-0.4752859175,1.9072747231,2.0408916473,-0.8214539289,-1.2110806704,-0.0955808163,-0.7157170177,-1.5102883577,1.8471789360,-0.6968463063,0.5167600513,1.8500298262,-0.6309816837,-0.0008967539,0.6174471974,0.2788871229,-0.8685209155,-0.1644606888,-0.6464869976,-2.1174771786,-1.7138003111,-0.1598047316,-0.3160598576,0.6028695107,-2.0008893013,0.5322968960,1.4469785690,0.6468976140,1.1313585043,-1.0836691856,-1.6371321678,0.1979190111,-0.4971550107,0.3789435327,1.2085425854,2.3498613834,0.2326178402,0.6639389992,-0.2816003561,-1.1688593626,-0.7957587242,2.1797263622,-1.2281219959,-1.0931417942,2.4704101086,0.3591396511,0.3633414209,1.7679173946,-1.5819662809,0.5613010526,0.8031804562,-0.9179223776,0.3878885508,-0.1153258309,0.2231075317,-0.1290087104,0.7834047079,-0.2219590694,-0.0516575351,0.7322729826,-0.6734268069,1.3610804081,0.7266793847,0.7462473512,-0.9291234016,2.1887683868,-0.9468863010,-0.3495929539,-0.0428985730,1.3444181681,-0.0642006993,-0.4990572035,0.9972393513,-0.1914417893,1.2694454193,-0.6293994188,-0.1585151702,-0.2537268102,-0.0725091919,0.7362402081,1.1210161448,-0.4297934473,-2.9094355106,-0.1323205382,-0.2460999191,-0.7158421278,2.1082756519,1.1742449999,0.9783309102,-0.2963083088,-1.1627272367,-0.0309837069,0.8325194120,0.3749943376,1.0111228228,0.1109903753,0.0558381490,0.2111046016,0.0354266241,0.9828637242,0.5850839615,0.0928330123,-0.9776461124,-0.8418869376,0.5141755939,-0.1413520724,1.1167169809,0.8080258369,0.8365648985,-0.6641654968,-1.2707079649,0.1409545690,0.9196810126,-1.2050665617,-0.6511962414,0.4858438671,0.5146605372,1.5368059874,-1.4482113123,0.7256953716,-0.4815798998,-0.4271739125,-0.7080703378,-0.7192265391,0.7303133011,-1.1464543343,-0.0006264384,1.9445780516,0.0367070660,-1.9715096951,-0.9793316126,0.9981327653,0.4227823615,-0.3801508546,0.8276200891,-0.3732515275,-0.5242275596,0.7299136519,0.9283463955,-0.9516673088,0.0363494493,-0.4172412455,0.6668561697,0.7712953687,1.2420126200,-1.0641741753,1.0731617212,-0.3424093425,0.0912520811,0.8670433164,-0.2390884310,-1.2840259075,-1.0081032515,-0.4506661594,0.0318231210,0.9985192418,-0.4479965866,1.9789948463,1.1024099588,-0.8284841776,0.9265839458,0.8247851133,1.1524419785,0.6611723304,0.8174999952,-0.0465653911,0.7161703110,-1.5024360418,-0.6508199573,0.2619676888,0.8435713053,-0.8028890491,0.7184874415,-0.9584380388,-0.5889627934,1.1051890850,1.0269550085,1.1142108440,0.8566098809,-0.1889745146],[2.1708762646,0.3211729527,-0.7073199153,0.2713794112,2.1762907505,-1.6629705429,-1.5230276585,0.2944081128,-1.8447911739,-0.4856126010,0.1250060797,-2.6290686131,-0.7232851386,0.4715391695,1.0080659389,0.0524155572,0.4955901802,0.0261317994,-0.4716481268,-0.0053549618,0.9015104771,-1.0712935925,-0.8792697787,0.2951650620,-0.7993769050,-0.7291873693,0.1309042275,0.8772374988,-0.8512756228,0.6310116649,0.2843870819,-0.6464396119,0.4193823040,-2.7665982246,-0.6511185765,-0.6155664325,-1.3428583145,-0.3334512115,-0.3145128787,-0.3880962133,-0.0138087487,-0.0732112080,0.8911177516,1.0543974638,-1.6987104416,-0.9329395294,0.6918064356,0.4133537114,0.9851148129,0.5021405220,-0.4187388122,-0.3045681715,1.2685724497,1.5336577892,0.4741467834,-0.4501512349,0.4564183354,0.3882172406,0.5919806957,-0.4965659082,0.3365334868,1.6907614470,-0.7521701455,1.6877909899,0.6436438560,1.6518549919,1.0253766775,0.9109950662,0.7273064256,0.5286173820,1.1155961752,1.8448877335,-1.5188112259,-1.7974079847,-0.9361957312,-0.5099250078,-0.1010021567,0.8071030974,0.1876153201,-0.0064344634,-0.0497765839,-0.5475576520,-1.0387189388,-0.3532833457,-0.5067877173,-0.9035395980,0.7844749689,0.9171721339,0.8255385160,1.1045073271,0.9061017632,0.8320184350,-0.3664151728,-0.6954748034,-0.3559719622,0.1738903522,0.9105868340,0.2641392946,0.5053906441,0.4349961579,0.8912056684,-0.1841396093,-0.7150786519,-1.3447219133,-0.4978642464,-0.0933635756,-1.1518702507,-0.6728532314,-1.6219307184,0.3163990974,-0.7288520932,-0.5077275038,0.3367995322,-0.7175356150,0.4415037036,0.7889863849,0.6716121435,0.9814752936,1.2690737247,-0.6693392992,-1.1873300076,1.4108334780,0.3188565969,-0.2024255991,0.7412149310,-0.4959668815,0.0802198648,-0.7489406466,-0.1091844365,0.9267287254,0.0638568923,1.1230969429,-0.2486508340,-0.2947012782,0.2912669778,-0.3035437167,1.0478081703,0.9640623331,0.7009201646,-0.8394269943,-0.4298658669,-0.0208310578,-0.9664135575,-0.4727172256,0.3689211309,0.8614180684,0.8856139779,-1.0648728609,-0.4798837602,1.6359280348,-1.5494107008,-0.2393066734,1.6723638773,1.0869294405,-0.1503750980,-1.3417633772,0.3851701319,0.8203778267,0.2636116445,1.2380123138,-0.2437708676,-1.6408116817,0.9459151030,-2.6160981655,0.1274162382,-0.8868399262,1.2920694351,1.2345162630,-0.1554797590,-1.2826472521,0.3236345053,-1.3748104572,0.1620595008,-0.5028721690,0.7731192112,0.0074709491,1.0263241529,0.3440847397,0.4407630861,0.2580718100,0.4818874002,-0.9904689193,0.3456379175,-0.7643799186,-0.5512456894,0.6247019768,-0.0195069723,-2.3036928177,0.4286721349,0.3994229734,-0.3317076266,0.9082452059,1.3464412689,0.8724412322,-0.2232536077,-0.9121289849,-1.0593949556,-0.9844374061,0.4946773946,-1.2179546356,0.8431268334,-1.0629291534,0.9424785376,0.6598713994,-0.5156948566,1.2651425600,0.6092521548,-0.9670048356,0.5924138427,0.8206498027,-0.1055323258,-1.3936834335,-0.2509214580,0.4976290762,0.7790489793,-0.6068552732,0.7373483777,-1.0231674910,0.7975035310,-1.8134320974,-1.6039186716,0.0184440892,-1.5174918175,0.1511792094],[2.4891557693,-0.7333879471,0.5969586968,1.1847419739,1.4520896673,-0.1143778041,-0.4433355927,1.7384566069,-0.3925912380,-0.0808646306,-0.4252757728,0.8303156495,-0.0310105458,-0.4736564159,0.0218356308,-0.5272080898,-0.3808960319,1.7675691843,1.5582160950,-1.0983140469,0.5440816879,1.2187278271,-0.4388285279,-0.2428838164,-0.4820106030,1.0899262428,-1.1954586506,0.8806062937,-0.0880520642,0.1150913313,-1.0539691448,0.6986237168,1.8885309696,0.9215537906,1.8435418606,-2.1448302269,0.3851986229,-0.7131193280,-0.3901919425,-1.3624395132,1.5063632727,-0.4614166915,-0.3000406027,-0.1013559550,0.0810948461,1.5957373381,-0.5460469723,-1.3816664219,0.5379842520,-0.6066759229,2.2901933193,-0.7612150311,0.5551570654,-0.9203477502,0.7038357258,-0.6596065164,-1.1327278614,-0.7435246110,1.0183490515,0.6444005966,-0.6632887721,0.1449518502,-0.9932124615,-0.7061646581,0.6948489547,1.6899902821,-0.3036556840,-0.7680572867,0.4792656898,0.4494149387,1.9874323606,0.1310714483,0.5772669911,0.4271465540,-0.3796473444,-0.5137271881,1.3119901419,-0.6844062209,-0.5254566669,0.0645294040,-0.2315711081,-0.9539589286,-1.5046663284,-0.3089866936,1.3342427015,-0.7204396725,0.3384650350,0.0651963726,1.0387514830,-2.1739177704,-0.7222473621,-1.8249527216,0.5022072792,0.0201506056,-2.2221145630,-1.7429068089,1.2421858311,-0.2116023302,-0.7000771761,-1.1120636463,0.2420840859,0.5119414926,-0.9397857189,-1.3358485699,2.8444836140,-0.9619747400,-0.8597752452,-0.3271184266,-0.5138223767,0.0424317084,-0.0836657956,1.0398344994,0.1050056592,0.7023622990,0.0821461901,0.9920848012,0.6409959793,0.3868029118,0.1679462940,0.4380161464,-0.8118830323,-1.0857101679,-0.8440061808,0.8092541099,0.1587680727,-0.5558717847,-0.2662746608,-0.7828058600,-1.8595489264,0.6069732308,-1.0864323378,0.0739274174,-3.2514317036,1.3766409159,-0.4158675373,-0.6441162229,0.5918098688,-0.3352716565,-0.6861219406,0.6173312068,0.8340231776,2.1576929092,0.7559766769,-1.9395177364,-1.5068037510,-1.6640449762,-1.0494740009,0.3865607977,-0.8776586056,-0.3444928527,2.2688899040,0.1278932095,1.4168598652,0.5408000350,-0.4373477995,0.7047700286,1.6746137142,0.0112737603,-0.4115543067,0.4049486816,0.3209570348,-0.0363627002,0.0535618663,0.9652701616,-1.0335603952,-0.8917180896,-1.0212895870,-1.2253913879,1.3610954285,0.5986154079,1.2577533722,-1.1704236269,-2.7986218929,-0.2227247208,0.6540288329,-0.5734651685,-1.3834633827,-1.4270046949,-0.7254546285,-0.6367143393,1.8409622908,0.0058467253,1.4262659550,0.1178915799,0.5660122037,0.9328692555,-0.3919641376,-1.2900167704,0.5978199840,1.0307321548,-0.6817595959,1.5728329420,-0.9726041555,-0.3166731596,1.4570447206,-2.8043234348,-0.1258659363,0.1362825036,-0.2868614495,-0.8499239087,0.7297567129,2.0664722919,-1.0674616098,-0.0646936595,-0.4705346525,0.8866371512,0.0606195629,-0.2033787519,-1.0553119183,0.2734993100,-0.1094169468,0.2509588599,-1.0711268187,0.2432459742,0.7302077413,2.0591175556,0.9453964829,0.7122648954,0.1843675375,-0.8156602979,-0.9698758125,-0.3812840283,0.5082962513,-0.6116515398],[-1.4949227571,-0.8798989654,0.1354723424,-0.3616631031,0.4147650898,-0.3228835464,-0.5915602446,1.3671255112,-0.5830543637,1.1036515236,0.0076271435,-0.8378448486,-0.9426112771,1.0647289753,1.1454114914,0.6047657728,-0.5769218802,-0.3423543870,-0.0147391697,-1.2193109989,0.8734396100,-0.5175681114,-0.8919221759,0.6767447591,-0.3358008564,-0.5971326232,1.1393744946,0.1059352309,-1.0928663015,-0.0023749452,-0.8992219567,0.5955408216,-0.1915006489,0.9519176483,-1.6528651714,0.4768061042,0.2691729665,1.0863753557,1.7779698372,1.6851940155,-0.5937855244,-1.2471388578,-1.4843024015,-1.6690483093,0.8079186678,0.6009854674,-0.1532800347,-0.4540135264,0.8403998017,0.0214106571,-0.6578392386,1.5833458900,0.3572075069,0.0490770936,0.3193014264,-0.0974152535,-0.2502352893,0.7973176241,-0.7291021347,0.0530070215,1.3097430468,0.3382050693,-0.4680342078,0.5104855895,0.3980289996,2.7617027760,2.8023548126,-0.9081757069,0.0257167872,-0.6720604300,-1.0278261900,0.5908465981,0.1360614896,0.0536954999,-0.6864715219,0.2259481102,-1.7827547789,-0.4555638433,0.4290686250,1.1622430086,-0.4761041105,-0.1950169206,0.2584655583,0.9476106167,-1.2855743170,-0.1756140739,-0.1077017039,0.9727046490,2.2783825397,-0.7615939975,-0.0914252028,-0.4492025077,-0.1590934247,0.3942250311,-1.8457733393,2.9138922691,-0.6026506424,-0.6460301280,-1.1496130228,0.2545246184,-0.1576082557,0.4011459947,-1.0190671682,-1.5612306595,-0.8178557158,0.1179975867,-0.1237922013,-0.1505875736,-1.5492738485,0.5640113950,0.4910785556,-1.4756758213,-0.0605845712,-0.7968961596,-0.3090079427,-1.4476984739,-0.0446362980,-0.7001911998,0.9888365865,0.0728237554,0.5284053087,0.1921956390,-0.0171768777,0.6343825459,1.2951797247,0.6299999356,0.5879244208,0.2705804408,-1.6909981966,2.4678122997,-2.3150990009,0.7861059308,1.1524168253,0.0536516793,-0.7031561136,2.6576049328,-1.3356398344,0.4453574717,-0.0873836875,0.8940993547,1.7681417465,-0.9471057057,0.7181583047,-0.9346099496,-0.8268125057,0.8860337734,-1.6342958212,-0.9823036790,0.2693607807,1.3396952152,0.0485697836,-0.3400638998,0.7500931025,0.3225327730,2.6488277912,0.3102334738,0.8130268455,-1.1668528318,0.9298003316,0.3419047296,0.3981050849,-0.7024795413,-0.0838392451,2.1503913403,0.3166868687,-0.4655977488,-1.0422612429,0.5906660557,-0.3481139243,-0.0162188802,-0.1142532825,-1.6149265766,0.0337558053,-0.8138551116,-1.9581266642,1.8291319609,-1.5578610897,2.1736629009,-0.7472221255,-1.7286211252,0.7932048440,-0.5200570226,0.1771980226,0.7093726993,1.9725366831,1.6122074127,1.6713001728,-0.2878654003,0.9476270676,-1.9848715067,0.6367865801,-0.8831424713,-1.0610102415,0.6461561322,-0.9139950275,0.2092022151,1.7103219032,-0.3820849359,-1.5692948103,-1.8255572319,0.3962948620,-0.1347055435,-1.3692386150,-1.3420454264,-0.0727353171,0.6803857684,0.2496832758,-0.7072216868,1.5534228086,0.5460235476,-0.1979304999,0.4633287787,-0.3761905730,-1.1127810478,-0.2559715807,0.0235608853,-0.6353579164,-0.6384484768,0.6815594435,-0.4563060701,1.2554326057,-0.4534022510,-1.7506196499,-0.5014964938],[-0.2295739204,-0.1924448907,-0.4733704925,-0.2241870314,-0.8210021853,0.5572533607,0.2077593505,1.3226039410,0.8503068686,0.3388375938,-0.5830029249,2.0334277153,1.0397149324,1.2391461134,1.0362479687,-0.5698149204,-0.6259616017,-0.2362915277,-0.3661133647,-0.8876191974,-1.6699805260,1.5485368967,0.0311318263,-0.7217914462,-1.2424839735,-0.0392079912,0.3139655590,-1.0111051798,0.7933912873,0.3894520998,0.2327877879,0.6048811674,0.8615930080,1.2502133846,0.3906288147,-0.5203686357,-0.1070114747,0.8577687740,-0.7629805207,-1.5210009813,1.3381309509,-1.6148548126,-0.2387400120,-0.1157045662,0.1873877794,-0.9930270314,-1.7484232187,-0.9100748301,-0.2736967206,-0.8384456635,0.0644456372,-0.6747201681,1.2409303188,-1.3965411186,0.5551806688,-0.2347596139,-1.5017775297,-0.4084243178,-0.6948349476,0.6206647754,1.0374354124,0.8068956733,-0.9258612990,-0.3243401945,0.3644148707,-1.1015172005,0.7313768268,0.8892873526,0.1480387300,-1.0648075342,-1.4160317183,-0.9014066458,-0.4222130179,0.2445816100,-0.5626324415,-0.4851743877,-0.3403742909,2.9591856003,-0.0881588683,0.5897176266,-0.2818697095,-0.4818490148,-0.0522636548,-0.7201578021,0.8085952997,0.7062283158,1.3574514389,3.1008806229,2.4156870842,1.3525607586,-2.0091259480,-0.0851096436,0.9077156186,-1.2800422907,-0.8717706203,-0.0972527489,-0.5018403530,-0.3444746137,0.7191011310,0.1668957919,0.5495115519,0.8817923069,-0.9869719744,-0.4999747574,0.4192822874,2.0949213505,1.0182723999,-0.9415771961,2.1363947392,-0.1266851574,-0.1686273217,0.1555329263,0.2692514956,0.0107193282,0.6864767671,-0.3113112748,-0.6079178452,1.0384211540,0.3583306968,-0.0989136025,-0.1000434235,0.1119473800,0.9623701572,-1.6869838238,-1.7702426910,-0.5314478874,-0.1993282437,-0.2009946853,1.0972038507,0.1889088154,0.7660380602,0.6959185600,-0.0903240070,0.4758541286,0.4996165335,0.3103198409,-0.2523146272,0.0269732401,0.7140724659,-1.4831380844,1.7946801186,-0.2277995795,1.5964488983,1.7150933743,-0.1319871843,-0.3674093485,0.7914146185,-0.3072839379,-0.8771335483,1.3631716967,0.6096215248,0.0368815847,1.0796775818,-1.5741424561,1.8392524719,-0.1673394144,-0.0709104463,1.3131312132,0.1592441201,0.7896984220,-1.4784001112,-0.2822154462,1.2148447037,-0.0967611820,-2.2488405704,-0.1258734167,0.1380659342,-0.7507057786,-1.7866828442,-0.1211231351,0.4181524813,0.3321387172,-0.1422798783,0.5991910100,1.8145097494,-0.2987695634,0.1185233369,1.7562841177,-0.0889143497,-0.8222903013,-1.4978560209,1.5606752634,-0.7209838033,0.9876718521,0.7995440364,0.3725219369,-0.1506541669,-1.4730818272,2.0495085716,-0.4954752922,1.1434779167,-0.5692482591,-1.4061460495,0.6421436071,-1.8218369484,-1.2275563478,-1.0374466181,1.6718900204,0.3865244985,-0.7263987660,0.5036387444,-0.1251687109,0.0561733358,-0.0534702614,1.3122061491,0.3336322904,0.4826311469,1.7704149485,0.1591441482,1.5266855955,1.2783801556,-0.6986181140,1.0752340555,0.4369335771,-0.4910381734,-1.2765337229,-1.5475559235,0.1101098582,0.4548444748,0.4179526865,3.5148334503,1.3793034554,1.2272132635,-0.7775411010],[0.1388985366,-0.9853470922,-2.1588702202,1.4759497643,1.0479900837,-0.2329456359,0.4264881909,1.6505819559,-0.6207741499,-1.3017786741,0.8918532729,-1.8384761810,1.6479315758,1.4943436384,-0.2758189738,-0.4083052278,-1.7543193102,-0.8631698489,-0.3464847505,-1.0863679647,-1.0439672470,0.7048197985,-0.6013585925,-0.5195314288,-0.4424979389,-1.3421670198,-0.1133228242,-0.1815241724,-0.4386858344,1.2019025087,-0.5518383980,1.4825443029,-0.1924960464,-1.3046330214,0.8492895961,0.3339430988,-0.4778641462,-0.7398019433,-0.1873675436,0.9931180477,0.1758742630,1.1829441786,0.5680868030,-1.5838301182,0.1504982114,0.6607076526,-0.0231101718,-0.1185555682,-0.1340669245,-1.0390249491,1.7121312618,-0.0431025848,-0.3924097121,0.5211667418,-0.8721234202,1.0714064837,-0.2642340660,-0.4705439508,1.5070914030,0.0275213439,-0.8787099719,0.2398306876,-0.9827113152,1.2846596241,1.2042113543,-0.2682869136,-0.0602881312,-0.1187253222,0.1935069561,0.6068314314,-0.9962551594,0.1362517178,-0.5295902491,-1.4795718193,-0.3208876252,-0.0433504656,1.3398160934,-0.5696412325,-0.1615510434,2.2393598557,-0.4070459902,1.1034712791,0.8850687742,0.4269449711,-0.0309651364,-0.3605507016,0.7600423694,-0.7894233465,-0.1974647790,1.5508267879,-0.8302157521,-0.3430190384,0.0971933901,0.7561179996,0.1921995878,0.6613284945,-1.3063372374,-1.8925228119,-0.5819588304,0.0686352998,0.7780308723,-0.1821022481,-0.2784351110,0.9828819633,-0.3580800891,-1.4925870895,1.1827394962,-1.2048211098,0.0015660966,-0.7056044340,0.3516597152,-1.3311002254,0.1076234132,-0.9575064182,-0.5319263339,2.2485411167,-0.8890992403,-1.5095508099,-1.4435843229,1.2062375546,-1.0373508930,-0.9600437880,-0.2674483955,-0.8744071722,-0.8395624161,-0.2602224052,-0.0716149807,0.1754660159,-0.0509066172,-1.3418159485,0.9362290502,2.0501604080,-0.8092800975,-0.2566420734,-0.9628132582,0.4812192321,-1.7650332451,1.6551828384,0.2969235182,1.9384123087,-0.7422243357,-0.8657963872,1.9764961004,0.2587822676,-1.2811017036,0.8559010625,-0.2178717405,0.5513018370,0.6288032532,0.3456377685,0.0518872105,-0.2260817885,1.3474808931,0.4960836172,1.1231657267,-0.2785956562,0.1864937544,-0.9686026573,-0.4685764611,0.0343641862,0.8239734769,-0.0382086560,0.8648043871,-1.5192743540,-0.3801605999,-0.7757578492,0.6177380085,1.1645232439,-0.3902976811,-0.4834443033,-0.5099754930,1.1100440025,1.9989259243,0.0120947240,0.7738932967,-0.1626637578,-0.2748828530,0.1500775963,-0.1423937976,0.7558955550,-0.2406939566,-0.8073249459,-0.8291538358,0.5597535968,0.6038845181,-0.4026309848,0.6417809129,-1.0559430122,1.4719893932,0.5502031446,0.4619366527,-0.3297341466,0.9770031571,1.1077107191,0.6487148404,1.0592761040,2.2452018261,-1.4517990351,0.3241731226,0.5708991289,-0.2178623378,-1.4555898905,0.8619402647,-0.7125315666,-0.4648037255,-1.1987832785,-0.3875809610,-0.8089615107,-1.6652204990,-0.6737309098,0.4696156383,-0.9500765204,0.5246605277,-0.3600152433,1.0752583742,0.3630267680,-2.2392160892,-0.4884197116,0.0897057503,0.5078607798,0.2434695959,0.6669661999,0.1939474046,1.1956789494],[1.2731165886,-0.2553974092,0.5463133454,0.7415204644,0.8253844976,-2.2207424641,-0.5939807892,-0.5405517220,0.5992439389,0.5527491570,0.2647654712,0.3965257108,0.7488173246,-0.6501255035,0.7335247397,2.3412547112,-0.2428223044,1.3753275871,-1.7682363987,-0.0811214820,-0.1482257694,0.0008704733,-1.1559543610,0.4070131183,1.3749901056,-0.3521507978,-0.5838592052,0.7910311222,-0.8108868599,-1.7694545984,0.6860965490,0.6280745268,-0.0189835038,0.7401046753,-0.0957614034,1.5742759705,0.6285890341,0.6735109091,-2.2882628441,0.9412603974,0.8516026139,-0.2982982099,0.7832838893,1.8826241493,-0.6292356253,0.5742823482,1.9699435234,-0.9881187081,-0.6871194839,-0.7571073174,1.7136073112,0.0445418991,-1.3317672014,-0.0278986376,0.1168358251,-1.9115582705,-0.3436582088,-1.2878416777,-0.4543692172,0.4965259731,-0.3469325006,-0.2261100411,0.7186712027,0.4477775395,-1.1620095968,0.6727824807,-0.0883801803,-1.8939939737,-1.2754259109,1.0608910322,-0.8201976418,0.2681608796,-1.5161629915,2.1384537220,-0.0212910455,0.6318595409,0.0462469086,0.3144905567,0.3767325878,-1.1331812143,-0.2977827787,1.0825668573,0.5229936242,-0.8243591189,0.2687449157,0.4557905197,-1.6796916723,0.0562063754,0.5043262839,1.0077110529,0.6924237013,0.9910323620,-1.1296902895,-0.8210905194,-1.4609053135,0.9427023530,-0.6383852959,0.1021938547,0.5141855478,0.0296744183,1.2136639357,0.6528182030,-0.4292894602,1.3012942076,0.8445175290,-1.8041859865,-0.0520955026,0.6637779474,0.8166362047,0.4969393611,0.1612919420,-1.2245982885,0.8383113146,0.1584201306,-0.6001465321,-0.6403214931,-0.2489199936,-0.8059918284,-1.7376863956,0.2539567053,0.9096975327,0.9964336753,1.1921913624,1.0994590521,0.0866026059,0.1853996962,0.3626529872,-0.6009641290,-0.8141295910,-0.1610737592,0.0605895333,-0.6976833940,-0.6364158392,-1.6901087761,-1.4714292288,-0.8151713610,-1.0487115383,0.6838766336,-0.5414502025,-0.5800019503,0.4200898111,0.8287611604,0.2556479871,1.0243107080,0.6832750440,0.9061375260,-1.0559997559,-0.5638802648,-0.0439743996,0.1173364073,1.6787053347,0.3498084545,1.0181390047,-0.5923074484,0.3546623290,-2.6732180119,0.7222507596,-0.9528815150,0.6282278299,0.6948962212,0.7244352698,-0.4419042766,0.6062676311,0.4268949032,1.2982889414,-0.0492773838,0.1728408486,-1.8239735365,0.0842023790,-0.1465672702,-0.5029066205,-2.5519418716,-0.8701648712,-0.3346950412,0.8195809126,0.9252887964,-0.4902196527,0.7730194330,-1.9046716690,-1.5051252842,-2.4048972130,-0.2875409722,0.8639270067,0.7805999517,1.4741975069,-1.3624743223,-0.8319619894,1.3560540676,0.3230776787,-0.8606194854,-1.7348588705,1.6464912891,0.9878881574,0.6745097041,-0.7136139870,1.6001394987,0.1300480217,-1.5410406590,-0.5384830236,-1.4236744642,0.5284870267,0.1341856122,0.2507516742,1.0759317875,1.0543617010,-1.2375645638,0.1014309525,-1.2856109142,1.5152353048,1.0129420757,-0.5068862438,-0.0818387121,0.8557261825,0.6897322536,-0.6936151981,0.8198645711,0.5045341253,1.4971163273,0.7517387867,-1.0433621407,-2.2742910385,-0.5024763942,-1.5331497192,-0.6632850170],[0.1812437326,2.9718608856,1.1071363688,0.1692442596,2.0369729996,0.4972907901,1.4878046513,-0.1821295917,1.4015305042,-0.8876495957,1.7378798723,-0.8951967955,1.0968136787,0.7477356195,0.5420276523,0.8818691969,0.7651853561,-2.1793744564,0.1379532367,-1.5338999033,-1.6719213724,0.8710910678,1.4140412807,-0.8285297155,-0.4181405008,1.0015592575,1.0259017944,-1.0662993193,-1.5625612736,1.5425119400,-1.5811508894,0.0358812883,-0.4583835900,-0.2951638699,0.6391587853,1.2410137653,-1.0080453157,0.0030940096,0.8546661735,0.5233162642,0.1683357954,-1.0702836514,0.6986927986,-2.6348586082,-0.2744528651,1.0474078655,0.6944044828,0.4282088280,0.5804508924,-0.2621594965,-0.1507897526,-0.6044706702,-0.5499491096,-0.8979150057,0.6255866289,1.9500479698,-0.3619381189,-2.0538542271,-2.2297744751,1.2849069834,-1.6552209854,2.3148140907,0.7873542905,-1.1823868752,-1.3611557484,1.6840388775,0.8037516475,0.1811046898,0.1025839970,-0.0465708077,0.9594902396,-0.5949816108,-0.4117023051,-1.0228437185,-0.0251171757,0.8824501038,0.7591711283,0.6231270432,0.5917374492,0.6219696403,-0.0495552123,0.4270074368,-1.1682341099,-0.8261880279,-0.5186452866,0.9526984692,0.2365787178,0.7776110768,0.7848405242,-1.3479496241,-0.6416763067,-0.3390326500,-0.5502781868,-0.9254025221,-1.2017199993,0.1044468060,0.5422565341,1.1859241724,0.0673230439,0.3402942121,1.9644356966,-1.2449916601,2.0009057522,-1.7514717579,-0.1355766803,-0.0298841372,0.0156432688,-1.6379914284,0.0300169382,-0.1188428104,0.4865311384,-0.5542267561,0.2956261337,0.8642213941,-0.3620271087,0.5115072727,2.4552910328,0.1400547177,1.0980756283,-0.9341176748,-0.0969975144,1.0263452530,-1.5575366020,-0.6205061078,1.1078485250,0.9087997079,0.0788982362,-0.2932177484,-2.6274516582,-0.5674450994,0.8807153702,1.8894174099,0.5257295370,1.7268207073,-0.0833458751,1.6114909649,0.5052893758,0.8463229537,0.4828009903,-0.0895236433,-0.2982712388,0.4179650843,-0.0710343793,-0.5111294389,0.3934566081,1.1269727945,0.5989762545,-0.1793426871,-0.5627002120,-0.3595992625,0.5753319860,0.3222674429,-1.3311716318,-0.5409787297,0.1170743182,0.1916641593,2.1233403683,-0.3985538185,1.1539492607,2.1156904697,-0.2566215396,-1.5207226276,2.0685455799,-1.1869761944,-0.8740059137,-0.4242081940,1.5217876434,-1.5788558722,0.5758740306,0.3150611818,-0.7949121594,0.5349891186,0.4776623845,0.4408864379,-0.5021479726,-0.2214682847,-0.4992967248,-0.0067575639,-1.8457427025,0.9280284047,-0.6053181887,0.7470316887,-0.0616565011,-1.2198575735,0.0267692748,-0.6104433537,0.0101288538,-0.3280124068,1.1101564169,0.3608153462,0.4453727007,0.9636499882,-0.8304093480,-0.4685767293,0.9103437662,-0.6982381940,1.1686404943,1.1610959768,-0.1441449672,1.0166361332,1.8670653105,-0.1394449472,-0.4043854177,0.5049604177,-0.6021443605,0.2187743187,0.3758377731,-2.1905653477,-0.4398531020,0.1216888949,1.1135267019,-1.3173891306,0.5598527193,-0.0590468831,0.6931667328,2.4740109444,-2.0966489315,0.1619170755,-1.5034503937,-0.6038945317,0.5554865599,-1.4479967356,-0.8917023540,-0.6455559731],[-0.4626631737,0.4900054634,-0.3706803024,0.5256991386,0.1678807884,-0.5502416492,-0.3375319541,-0.3431487679,2.1183767319,-0.7996915579,-0.3146005273,0.3703980744,-1.0030088425,-1.9712535143,-0.2197760195,-0.2830720842,-0.1667875350,0.5648902655,-1.6655410528,0.7426096201,-0.1408229321,-0.3646937609,-0.3127844632,-3.0347666740,0.1353932172,0.6488391161,0.5635000467,0.6422799826,-0.5255937576,1.3670353889,-0.1994901747,0.2347858846,-0.5399846435,-0.2052275538,0.7761011720,0.3517878354,-2.4709758759,-0.5613272190,-0.0516717657,-1.9915071726,-0.8145096898,-0.1598192304,-0.4733167291,0.3067771792,-0.4550936222,0.4928416610,-1.5474539995,-0.5095470548,1.3454902172,0.6438094378,2.3404154778,1.0623267889,0.2608698010,1.3165721893,0.3916930854,1.2871226072,-1.1001724005,0.4859510958,-1.0528469086,0.0048558200,0.5584079623,0.8512579799,-0.9370487928,-1.2440121174,1.4365291595,0.0317177959,0.2058206797,0.0645339563,0.2769898772,0.0672572851,1.3564913273,-0.8004226089,-1.0108019114,0.2683846056,-0.3449343443,0.5715727806,1.8105565310,-0.4251840413,0.0933929682,-0.0318874568,-0.0917476565,0.2792342305,-1.3499195576,-0.3130746782,0.1089106649,0.7911434770,0.1674464196,-0.5621147156,0.0358672328,0.3739733100,0.1566600949,1.6247915030,-1.3720308542,0.0518179424,0.2691105306,0.1931755841,0.0992060304,-0.0557342619,0.6415981650,0.2318139821,0.1195221543,-0.7894995809,0.7086682320,-0.2938629985,0.4510083795,-0.2968471646,0.9600773454,0.8050220609,0.9683947563,0.5477582812,2.1819415092,0.2233613133,1.3260169029,-0.3495138288,0.0223538298,0.1567016244,-0.7688352466,-1.0201549530,-0.0924670175,0.0193462633,0.3662928641,0.5027229786,0.4168736339,-0.6279435754,-0.4090660214,-0.2738506794,-0.7319854498,-0.7716166973,-0.4655268788,0.0913858637,0.6756829619,1.0749363899,-2.9845871925,-2.3411133289,0.7401157618,-0.5512769222,1.1327184439,-0.3372319043,-1.0306859016,-2.1232624054,0.1218878105,0.2254609168,-1.8603311777,0.1255722195,0.1211966723,-1.5937081575,1.2448704243,-0.4653194249,-1.1432752609,-0.8448824286,0.4323055148,-1.5212147236,0.4646244943,1.2543433905,0.6006751657,-0.5522754788,0.6215143204,-1.3684450388,-1.2913494110,-0.7055885792,-0.7935898900,-0.2950033844,0.8652331233,-0.3805498779,-1.6659548283,-1.2524651289,0.0942175016,0.4701784253,0.4993986189,0.3036863804,0.3379638195,-1.0666376352,0.9350460768,0.0076516639,0.8255095482,2.0182163715,-0.2377916873,0.4250042140,-1.2792454958,1.1763788462,-2.4107418060,0.0559295639,-0.8411914706,0.9756729007,-0.2996968329,1.1991971731,-0.0036011224,1.5130709410,-0.7111702561,-1.3614077568,2.2163901329,1.9284011126,0.6578730345,-0.5633271933,-0.5350072980,-0.6862948537,-1.6068059206,-1.5318435431,0.6679149866,1.0064121485,1.1902654171,0.2330749482,1.5748063326,1.1538707018,-0.9454938173,0.4218579233,-0.6603856087,0.5557044148,-0.3310923576,1.2670267820,0.6889605522,0.5879198909,0.8793924451,0.5530157685,-1.3612712622,0.6004394293,1.5862592459,1.7088806629,-0.3228707016,-1.5339814425,1.3532996178,0.1337690353,0.4033794701,0.9362673759],[2.5070328712,-1.4719178677,1.1299579144,-1.2589129210,0.8528534174,0.8711306453,1.6040962934,1.7402243614,-0.6684724689,-0.8084316254,-1.1939450502,-0.2090555429,-1.5653285980,-1.6939487457,-0.7268587351,-1.8237613440,0.1104834676,-0.5062472224,1.1678825617,-1.4700238705,-0.8294458985,-0.4625340104,-0.2558137476,-0.5204338431,0.5611996055,-1.4619174004,-0.6257733107,-0.7569713593,0.7612380385,0.6510775685,0.4265863895,0.8792460561,0.0824565366,-0.6086297035,-1.2662295103,-0.5894017816,1.4452910423,0.6569434404,-0.3121954501,-1.8380763531,2.0977277756,1.1606415510,-0.8928475976,-0.2812048793,-0.7891488075,-0.5135432482,0.1404644102,1.7727484703,-0.7247069478,-0.4047034383,1.1784008741,-0.0882060081,0.4330866337,-0.2398931980,0.6297299266,-0.7113015652,-0.2888081670,-0.6917343736,-1.0160704851,-0.0501448251,-1.7557926178,0.8874514103,0.6896318197,-0.9837778211,1.9327976704,-1.7024279833,1.1284931898,-1.7392871380,-0.1856941730,1.3723155260,1.3488906622,-0.2660662234,0.9858436584,0.3364506662,0.4291782975,1.2164518833,1.5936523676,0.5462658405,-1.1891343594,-0.3331855237,0.0216330234,-0.3720126450,0.6676066518,-0.1610603034,-0.6453488469,-1.2660588026,0.1885916442,-0.1913194507,-0.2533226907,-0.4697803855,0.0147942333,1.9812374115,-0.0728941634,0.4952958822,0.0687560663,-0.6622215509,0.0422153100,0.0999927744,1.8953807354,-1.4996274710,-0.5814818740,0.7663285136,-0.6812976599,-0.1066205800,0.2546959221,1.2007323503,1.3327698708,0.2031353116,0.7176383138,1.2447898388,0.4891205430,-0.9856312275,0.8673879504,-0.6054337621,0.4199489355,0.1581125259,-1.4761838913,1.1224590540,2.9868655205,2.0953581333,-0.6189078093,-0.7180119157,1.2323864698,-0.4477436841,-0.4142808020,-0.1765896827,-0.9405736923,-0.3397848606,0.6380889416,1.3421542645,-2.2330129147,0.2280873209,0.7671604753,1.4022625685,-1.3546260595,-1.7892752886,-0.9778614044,0.2651600242,-0.7413028479,0.2085521221,0.6731303930,-1.5684994459,-1.0101099014,-0.5023919940,-0.6700144410,0.7095361352,0.8025005460,-0.2038030177,0.4760249555,0.4454511702,-0.5939602852,-0.7164628506,-1.5171319246,1.4800959826,-0.0717306584,0.9497123361,1.4563376904,1.5433458090,2.4010305405,3.3334319592,0.4822435677,-0.5614119172,-0.9017120600,0.9639977217,-0.2467245311,0.6018739343,0.6744873524,-0.7295719981,0.2263079584,-0.0595782287,0.4756217003,-0.1380271614,1.4944998026,-1.1411798000,0.4533945322,-0.5606918335,0.6212491989,-1.1574071646,-0.2401100546,-0.5401071310,-0.8303443193,0.3478609622,-0.8683995605,-1.3129228354,-0.1908754110,-1.8695154190,-0.3507457078,-1.6960946321,-1.4109058380,-0.0006375873,-1.9163777828,2.3475019932,-0.9438844919,0.9024425149,1.2787234783,0.2669394016,1.0342036486,-1.1763765812,0.2994127572,-1.5218186378,-0.6456666589,-0.0896530226,-0.3566700816,-0.5822973251,0.2027257532,1.2007070780,-0.9537147284,-1.7942665815,-1.6502112150,0.3794765770,-2.1909353733,0.3905528784,1.6661679745,-1.2986001968,-1.2059516907,-0.9303827286,-1.6575475931,0.2639370561,0.2804372013,-0.1337700784,0.4575373232,1.4409017563,-1.3529820442,1.1944533587],[0.1142227128,0.1391180605,1.3320208788,-0.5805708766,-0.4466572404,-0.8611194491,-0.1826741546,0.3247959912,-0.4794469774,-1.5512284040,-0.6987625360,0.7923681736,0.3309248388,-0.5274670720,1.7341567278,-0.5651237965,-1.1047180891,-0.8757085800,1.9002363682,-0.5654552579,-0.1503885388,2.3177163601,-1.0674678087,0.4179449975,-0.8482495546,0.1508035660,-0.2906613052,1.7630039454,-0.4032920003,-0.8586524725,-0.8267387152,-1.0485448837,0.4945226312,0.8998538852,-1.2160298824,-0.4785618782,-0.3782664239,-0.1571599543,-0.7503957748,-1.5480009317,0.2291377932,0.6442772746,1.8228875399,-0.0811340883,-0.9858629107,0.1007367000,-1.0231248140,0.8747323155,0.7531514168,-1.6565316916,0.9526207447,-0.0294028688,0.2464197874,0.1333850771,0.5864880085,1.0717082024,1.4090615511,0.8718538284,-0.0508552119,-0.0712610632,-0.1940155625,-0.7806551456,0.5808362961,-0.0453044698,0.9831092954,-0.0407605805,1.6183751822,-0.4711570144,0.9464051723,-0.1902614683,-0.5595943928,0.7917163372,0.8945364356,2.0407261848,-0.0378948264,-0.5753856301,0.7319956422,0.7065110803,-0.1146564707,-0.3133381903,-0.6908829212,1.3543372154,-1.2197914124,1.0682621002,-0.4357069135,0.4142681360,-0.8415632844,0.0666707754,-0.1261732876,0.4309399724,1.7831349373,0.2475240082,0.5416277051,-2.3900408745,-0.0873001814,0.4318037629,-0.0842064768,0.0386220776,0.0086299293,0.4828562737,-0.3126920760,0.6790862679,-0.0472784191,1.0052809715,-0.7735071182,0.4777276814,-0.1993416548,-0.7341573834,2.2931449413,-0.8017128110,-1.2624527216,0.4948807955,1.9594209194,-0.2333582640,-1.1403502226,0.7428901792,0.4308326542,-0.0369284041,-0.8011110425,0.2558170557,-0.8303340673,0.4260979295,-0.7176937461,0.3255190253,0.2934720814,0.0140975080,-1.6804726124,-0.3438348174,-1.4502503872,-0.9531344771,-0.1847796291,0.1688913107,0.6571382880,1.5159125328,-0.1185081005,-0.9134144187,2.3134839535,0.9210193753,-0.1931522787,1.1115853786,-1.5775060654,-0.4333670139,0.9147577286,-0.8540613651,0.4533645511,-1.5374147892,0.3218281269,-2.0726091862,-0.2654472589,0.4145475924,-2.7371721268,0.0493369773,-0.8245742321,-0.7906406522,-0.4056213796,-1.2431070805,0.1490028054,-0.5710224509,-1.1668577194,0.3953489661,0.7068749070,-0.3533940911,-1.9965847731,0.8758977056,0.4603376687,1.2675856352,-0.7088031769,-0.6654269099,-0.9213801622,0.4947441518,-0.0715525448,-1.0028457642,0.3830953240,0.4520997703,-1.2615735531,0.9056553841,0.3861218393,0.1425308436,-0.1260088682,0.9447017312,0.6559101939,-1.8171663284,1.4004011154,-2.0061945915,-1.8255103827,0.1024764478,0.1558491588,1.2420189381,-0.4764030576,0.3463437855,-1.7062416077,-1.5446808338,-0.3215472400,-0.4196771085,1.8047077656,-1.8244951963,-1.2479240894,0.8842428923,-0.7375934720,-0.4316256344,-1.1759667397,0.3775776029,0.3401539326,-0.7903994322,0.4455604553,0.5148267150,1.0482620001,-2.2607221603,0.0069156969,-0.9996435642,0.4997765124,-0.8113646507,-0.2739197314,0.7623291612,1.9041544199,0.0038511909,-2.2380661964,1.2144907713,-1.3699985743,0.5216916203,-0.9953954220,-0.0678855330,-0.9681578875,-1.0197651386],[0.4510965943,-0.9384776354,0.4080997109,0.2205782682,1.4572985172,-0.0716064721,-1.3010287285,1.0668503046,0.5560036302,0.0169553459,1.3414565325,0.3216943741,0.4991309047,1.4584230185,0.2209969759,-0.6442297697,1.2691410780,-0.0391359441,1.1090129614,0.3669699728,-1.2093535662,-0.2478190958,1.0072946548,1.3478624821,1.4052371979,0.7991831899,-0.5470215678,0.1076327264,-1.2209036350,-0.2053703666,0.6416805983,0.8527611494,0.8009208441,-1.1441555023,0.5289140344,-1.3059980869,0.8233359456,-0.9994716048,0.4629640877,-0.9534198642,1.9949338436,1.0057520866,0.4678486586,-0.4734914899,0.0677830651,0.3521122932,-0.1107401550,1.0441912413,-0.4773075581,1.3873674870,0.5615459681,-0.7976940870,-1.3771500587,0.8169613481,1.3486270905,0.5091228485,-0.3491946459,-0.4665772915,0.2233534604,0.6446563601,0.9125355482,0.9257678390,0.8465470672,0.0578327812,1.3858836889,-0.2434680015,1.6562070847,0.9416037202,0.5632153749,-0.5659677982,0.5274534225,1.1877981424,1.4489836693,-0.7451183200,-1.5661666393,-1.1835601330,-0.9559128284,0.1651993841,1.1368753910,-1.0665529966,-0.4836068749,-0.8145627975,0.3149281442,-0.1883910000,0.4216847420,-1.1827241182,-0.0994306877,-0.3438635170,-1.9663894176,0.2089749277,-1.4949587584,0.3206232488,-0.6770358682,1.1477482319,-0.2624453902,1.4652047157,-0.0285696778,-0.0104226936,-1.3233613968,-0.5543357134,-0.1505497992,0.2329614758,-0.5039017200,0.2137409151,-1.2017041445,-0.6092761159,0.5041880012,-2.6073977947,-0.5180581212,-1.2378791571,-0.7136105895,-0.7491711378,0.8616729975,-0.6471394300,-2.4227328300,-0.1132138893,-0.6853970885,0.3431085646,0.8768229485,0.2715342343,-0.3802629411,-0.8611323833,0.3513724804,0.5549112558,-1.2833085060,0.7726469636,-0.4660710394,2.5595307350,-1.7120479345,-0.4161309004,-0.2351716012,0.3630282879,0.9382425547,0.6695708036,-1.7179670334,-0.5959632993,3.7025876045,-0.2213580161,-0.1287264526,0.6192150712,1.0531377792,-0.2467512041,1.4417458773,-0.3152952492,-0.7391575575,0.1590342820,0.8058029413,-0.1784851551,2.3034143448,-0.4181099534,1.3932461739,-1.1737822294,0.7538549900,0.6269468069,-0.7819752693,-0.7092577219,-0.5887012482,-1.0844168663,0.9721927047,1.2547427416,1.7547675371,-0.7578601837,-0.3038023710,0.6221553683,0.2105609030,0.6844446659,-0.0726488233,-0.8142358065,1.5024870634,-0.9599415660,0.8041617870,1.2129323483,0.2786012888,-0.4806325436,-1.5507179499,-0.6431815624,-0.9598134756,-0.0414928049,0.9750754833,0.2134624720,-0.7340024114,-0.9867333174,-0.9676715136,-0.3153736889,1.7369050980,0.1728345603,-1.6357651949,0.5001330972,0.0361833759,-0.3400045931,0.6917695403,-0.6486070156,0.5865759850,0.6650483012,2.3782908916,-0.7281708121,-0.1784517318,-0.6568906307,-0.0668870881,0.4001824856,0.5983645320,1.4374731779,0.8282849193,-0.2202301174,1.4063117504,-0.2658022344,-1.0646981001,0.2660436034,0.2665685117,0.8515069485,0.0041442462,0.3150144517,-2.1227982044,0.7639856339,-1.4190670252,0.7797390819,0.9185681343,2.0322315693,-1.4458156824,0.1613995284,-0.4561886489,-0.8995773196,-0.2119886428,0.1786645055],[0.1776530445,-0.3201810718,0.1161821634,-0.5768741369,0.6252367496,-0.1033955812,-1.0653221607,-2.6090035439,-1.9755235910,-0.0643296540,0.3567645550,0.1918041110,-0.2986488640,1.0968306065,-0.4158258736,1.2320274115,0.6485459208,1.6077394485,-3.1362752914,0.3075769842,0.4591005147,0.3467891812,-0.3982659280,1.1793015003,-1.7780370712,-1.3126175404,0.4909677207,1.6737562418,0.0124732330,-0.3899201453,-0.1232452691,-0.6100804210,-0.9292196035,0.8099145889,0.6836316586,1.4735952616,-2.2071619034,1.9934275150,0.6117762327,-1.5051028728,-2.4555799961,-1.0627012253,0.9527903199,1.5368833542,0.0395082273,-1.3321162462,0.5243310332,-0.1798415333,0.5129242539,-0.4493110180,0.0150246527,-0.9672916532,0.9956399202,0.2072731555,0.5924943686,-0.5087363720,-0.1341209561,2.0133500099,1.2745864391,-1.7311307192,0.8783261776,0.2382151335,-0.5218320489,-0.7136888504,0.9078969955,0.1662100703,0.6880350113,0.6426402926,-0.1699856371,0.8449928761,-0.2059611827,1.2312691212,-0.1832074076,0.7024871111,0.0767141655,-1.0377204418,-1.2589073181,1.2890216112,-1.6496952772,1.3731424809,-2.5840425491,0.8842554688,0.7858142257,1.2424460649,-1.1142414808,-1.6087180376,0.5588724613,-0.3181532919,0.1282300502,0.3226558864,1.7192137241,1.8144487143,0.0832170099,1.0794529915,1.0780323744,0.1856908500,-0.6103316545,-2.1893501282,1.1166290045,0.7507947087,2.0451416969,-0.0583368354,-0.7521981597,0.1606955081,0.9020569921,0.2915190160,2.0094237328,-0.5653720498,-2.0505139828,0.3951239884,-0.1658412069,0.1319901049,-0.8073076010,1.1072100401,-1.6461715698,-1.6490888596,-0.7021602392,0.1291684657,1.0130834579,0.4326122403,0.0710866079,0.6522153020,0.9110949039,1.2549139261,0.3565925658,0.4046155214,0.4020137191,0.3690601587,-0.4672718048,1.3912384510,-0.4798047543,1.5850034952,0.2715241313,-0.0765904635,-0.1810461581,-1.1558505297,-0.3667913377,-0.3131943941,1.6469752789,0.1343065500,-0.6560097933,0.5086302757,0.4044632912,0.6018323302,1.7113457918,-0.1906548142,0.0425776057,0.0373008773,0.1381624639,-0.4156305790,-0.7981631160,0.0582099520,2.6061866283,-0.7722873688,-0.0470007211,1.5131361485,-0.0309810881,-1.0579044819,0.8854081631,0.7483155727,-0.2261427641,-0.7398789525,-1.0611298084,-1.4868919849,-0.4077556431,-0.5551923513,-0.3160439432,0.7666338086,1.4771444798,-0.1199424937,1.2570329905,-0.6283643842,-0.6694485545,1.3534792662,0.1671885401,-0.9664984941,0.1351937652,-2.2209308147,-1.7459772825,0.8721058369,0.4668622911,-0.9920146465,-0.3378461897,-1.5689595938,0.0369126610,-1.8374615908,-0.9583182931,-0.0569251254,0.0614638180,0.3806518018,-0.9281975031,-0.3982549608,0.6343744397,0.5360945463,0.4420194328,-0.1513541192,-0.1408912539,-1.2138690948,0.9535938501,-0.1115452573,-1.2844252586,0.2494014949,1.4498769045,0.8719550371,0.0237522461,-1.6092516184,0.1648471653,0.5427899361,-0.4420486987,-1.4820035696,-0.0689525232,-0.7968429923,-1.6705019474,0.7698445320,-0.1265096515,0.7726453543,0.1753320694,-0.5076130033,-0.1544819772,0.2653084695,-0.0847224519,0.6755714417,1.0591664314,-0.0757832155],[0.0805232376,0.0671821758,1.6652826071,0.2529174089,-1.0575348139,0.0438446961,0.6698242426,0.5310724378,-0.4435330033,-0.3427950442,1.0558329821,-0.9846804738,0.5860679746,1.3339457512,-0.8269437551,0.9077727199,0.1672893316,-0.8939241171,0.3120681345,-0.6364495158,-0.7267069221,2.6667218208,1.1729916334,1.2637227774,-0.3764693141,0.1535074413,-0.2704104781,0.0951040611,-0.7366018891,-1.2720896006,-0.6093081832,-0.2956297398,0.1220130399,0.9173392057,-0.2534488142,0.4520286620,-1.8082159758,0.7244623899,0.7396792769,0.6426346898,0.0797319785,-0.5158020258,0.5891546011,-1.4976643324,-1.2477073669,-1.7936444283,0.8376747966,-0.7089396119,-0.1811517924,-0.7688514590,-1.4071519375,0.2784728110,-2.5875277519,0.9116972685,-0.5199452043,0.1479708701,-0.8809148073,1.8356801271,-0.8060521483,0.4518039227,-1.6225919724,-2.0098841190,0.1720021516,-0.4275765419,0.1004268378,0.1585029215,0.8063095808,-0.1378083378,-0.0826101378,0.6346687078,-0.3701512814,0.4816154242,0.3261772096,0.7613685131,0.6079658866,-0.6945112944,-0.2238053679,0.0034116469,0.0198742934,-0.6288275123,0.3040928245,1.0772180557,1.1524056196,1.1629301310,-0.3494142294,0.5766470432,0.3408490717,0.4503265023,-1.0144721270,-0.0498800389,-0.5482375026,0.3489536345,0.7263273001,-0.9238932133,-0.3699231446,-0.8713849783,-0.7335543633,0.8255431652,-1.5280839205,0.5197145939,1.0876455307,-0.2757599950,1.5692808628,0.7538465858,0.9047079682,1.1547479630,1.0163842440,-1.1070152521,-0.2055583447,-0.9297230840,-1.8402305841,-0.0856739655,-1.4860048294,-0.5312367082,-0.2028853595,-2.2556784153,0.3083379269,-0.6643678546,-1.4581902027,0.1362811774,-0.2612127960,-0.3958458602,0.2158193737,2.3531103134,0.5730254650,1.5000845194,-1.6561217308,-0.9346296787,-0.0294601489,-2.1529641151,0.3335566819,1.1823263168,-0.5181627870,0.6104934812,-0.7329621315,-1.1848534346,0.4069354832,0.0345559642,1.9944534302,2.3979656696,-1.1336600780,1.2679555416,-0.4079175293,0.0792932883,-0.4174874723,-0.2649638653,0.9532611966,-0.3875336945,0.6469477415,-1.6494807005,-1.5241768360,-0.9117911458,0.9734886289,0.2089319974,0.5428323150,-1.9180088043,1.2095471621,-0.2803136408,-0.8650236130,0.3140363991,0.2163758725,-1.3617240191,-0.9943023324,0.6911141872,0.7621330023,-1.2232488394,1.6510376930,0.2612263560,0.1864361614,-0.0706841946,-1.0403263569,-2.0321514606,-0.7808988094,-0.8977251649,1.6592231989,-1.3123985529,-0.5749298930,1.1344422102,1.6615151167,0.1504340172,-0.0827697366,1.0395177603,-0.7313262820,1.0351142883,0.2846813202,0.3578853905,-0.8776550889,-0.0276330616,-0.2406794727,0.2422078401,0.2753272355,-0.9000979066,-2.1857159138,0.2421487421,1.9852120876,0.6831824780,-1.3812738657,0.2342664003,0.7960075736,-1.0990927219,-0.6038036942,-1.5455021858,-0.2055137306,-1.2040677071,0.1401914656,-1.1692546606,-0.7095637321,-0.2126831561,-1.3855490685,-0.2365769148,-0.3908159137,-0.0181708690,-0.4164932966,1.7714226246,0.8974324465,0.5611874461,0.8519725204,0.9009156227,-0.1347933710,-0.3339865208,-1.3960329294,1.1774704456,0.2161447108,-0.6004638672],[-1.0433120728,1.2550470829,-0.3633813262,-0.5906378031,-0.0185135938,-0.1028014421,-1.7445312738,1.6993048191,-0.6432486773,-0.9254992604,0.5182227492,-1.2138725519,0.8419089317,-0.6030687094,-0.1839088202,0.4044023156,0.1647076309,2.3706243038,-0.8371855021,-0.3153517246,-1.0361464024,-0.1055759564,-0.8192484975,-1.3087396622,0.8839029074,0.8903806806,-0.1596081257,-0.3812036216,-0.8509666324,-2.1042964458,-0.1953709871,0.7087575197,-2.1301450729,-0.0047883722,1.4961576462,-0.6381576061,2.3768637180,-0.9427425861,-0.1516778916,-2.2839558125,0.5090414286,-0.2221545130,0.4502406120,1.1437622309,-0.1963302344,0.3094167709,1.3509380817,-0.5044822097,-1.6174205542,1.5378000736,1.3241184950,-0.2176084369,1.0739580393,-1.3880198002,2.6526143551,1.3435379267,0.9719251990,0.1656426191,-0.4877803326,-0.9836961031,-0.7674241066,-2.3378081322,0.3354743123,-1.1810867786,1.2881627083,-0.2078816146,-1.2664439678,0.5565501451,4.0197734833,0.8854044676,-0.5009924173,1.0933628082,-0.4822470546,1.1983449459,-0.7168338299,1.0617597103,1.4151138067,0.2771531641,0.7030990720,-1.2996754646,0.5242429376,-0.5445105433,0.7130293250,-0.3225224316,-0.6599298716,1.4526163340,-1.0813671350,-1.3591839075,0.8215834498,-1.0169330835,0.1585911661,1.6827707291,-0.9877544641,-0.8410540223,-1.2568205595,0.6768353581,0.3739871085,0.2208058387,-0.3934979141,-0.4662604630,0.5557742119,-1.4550930262,0.6333280802,0.2107170522,-0.0745086297,0.4422843456,1.3056447506,0.2425055355,0.0369345658,-0.1842762232,0.6566666365,-0.2707042992,1.6456437111,0.3681226671,0.6295041442,0.9925812483,-0.3826599419,0.1413362175,0.4062003493,2.3438417912,-0.3850591481,1.5470700264,-0.8475514054,-0.6330932975,0.4426223338,1.1135199070,0.0156516992,0.9530420303,-0.2288440615,-0.5985863209,0.8398433328,0.3938765526,1.7500195503,-0.6720721722,0.2872419059,0.6539419293,0.1856593937,0.1378008425,-1.3960168362,-0.1134697646,-0.2837387621,-0.8654352427,0.3288089037,0.0301165637,1.1624650955,0.0011483978,-0.6361009479,0.5182092190,-0.0406220965,1.7414946556,-1.6548664570,-0.6029675603,-0.5882458687,1.4716166258,1.1737931967,-1.1955555677,-0.1590917110,0.2381560504,0.9164401889,-0.1090425402,0.5899898410,-1.2650150061,0.5999732614,-1.0951660872,-0.0711921379,-0.7832251191,-0.6987342834,-0.0219315849,1.1395345926,0.2375022620,-1.7151470184,-1.5024656057,1.7313393354,-0.5030255914,-1.2392387390,-0.1868620217,0.9655178785,-0.5649629831,-0.6893165112,-0.7033599615,-2.2348511219,-0.0525545701,0.3070724905,0.6070462465,1.1996613741,-0.5176995397,1.0299232006,0.8695846200,-2.1147446632,0.4078527093,-0.8786563873,0.4362991750,1.1177811623,0.5423116088,0.9149170518,2.4281659126,0.2401718497,-0.5961187482,0.0141318729,0.3701690435,0.1808289289,1.2599830627,0.2183745056,-0.3465143144,0.8355005383,0.2451741993,-0.1894374192,0.6959276199,1.5179269314,0.2976191342,0.5701390505,1.4372801781,2.0795273781,-0.3756849170,-3.5909032822,0.0716620237,0.6304141879,0.0261821225,0.8042483330,0.9069849849,0.3464861512,-0.5794659853,-0.5438749790,0.0289033912],[0.9105071425,-1.4992305040,-0.4793045819,0.8542497158,-1.0811475515,0.2171867788,-0.5041297674,1.7892602682,-0.0283070989,-0.0441253670,1.2115254402,-0.7234268785,-1.1590812206,-0.1522644460,-0.9474188685,0.1717792600,-0.2749402225,-1.0617312193,0.6321954131,-1.3946050406,-0.5197392106,0.8903101683,1.0188506842,0.0586597063,0.3688014150,0.7033883929,0.5548697114,-0.9274749160,-0.6818982363,0.4796386361,0.2311522514,1.9238237143,-0.6357836723,-0.1479309797,2.6409428120,0.5992001891,0.0839401186,-0.3585545421,1.6161761284,-1.1302142143,1.3481178284,0.5973325372,0.5636794567,-0.0097401869,-0.3516861498,0.8041421175,2.3142130375,1.7594890594,1.5309214592,1.8990132809,-1.8272774220,0.8267484307,-2.2523925304,-0.1968108118,-0.2863500416,0.4057592750,-0.7191252112,0.2487123907,-0.6566113830,-1.3493041992,0.1164432839,0.1086965054,-1.4072995186,-1.7230648994,-1.9000509977,-2.4684510231,1.2062046528,0.4911671579,-0.0894651636,-0.3995863199,0.0991533101,0.0014761668,-0.7353255153,1.0571197271,0.3556884527,1.1477404833,1.5356293917,2.0558340549,0.2317831367,-1.1191799641,1.3284108639,-1.4650167227,1.5791805983,0.2670737803,2.0296282768,0.7951829433,0.4688805938,0.0782255381,-0.3811810315,-2.0412206650,0.9424459338,1.4058378935,2.0227613449,-0.4164577723,0.1712241471,-1.2878608704,0.2662688792,0.4602661729,-1.9420094490,-1.0917539597,1.0351030827,-1.3396145105,0.4247620106,0.4522379935,-0.1753645241,0.3741957843,0.6832326651,0.2857375145,-0.7751410007,0.2919989228,0.3030637205,-0.7733544111,0.2354594618,-1.6997842789,0.8201661706,-0.2567533255,-0.4506866932,-0.3111549914,-1.0207248926,-0.5771256089,-0.9786616564,0.1340002865,-0.8608180285,0.5694761872,-0.4418152273,-0.1610226035,0.2886717916,1.9575083256,0.7234182358,-1.0948028564,-0.2901492417,-1.3798342943,-0.6869062781,-0.5215283036,0.2655810118,-0.2533085644,0.2296121866,-0.6031663418,-0.8315091133,-0.5224909186,-0.4015553594,0.2074885517,1.6709539890,0.8634210825,1.5834872723,-1.7893948555,-1.3206269741,-0.6768729091,2.1363854408,1.2582120895,1.0345571041,-0.1696926504,-0.7326226830,1.0371384621,-0.9781098366,-0.2509648204,1.0754765272,-0.5080704689,-1.7204457521,1.3550037146,-0.7376111746,0.2516528964,0.0143696088,1.4959465265,0.9013447165,0.0666519925,1.2546199560,-0.6876458526,0.1289302558,-1.2125207186,-1.3062785864,0.1477067918,-0.9687612653,-0.6614953876,0.4936519265,-1.0251137018,1.5563279390,-0.0668517277,-0.6016780138,1.2578229904,-1.6711659431,-1.7495858669,-0.6962295771,1.7163276672,-0.6159476042,0.8581643105,-0.0230642557,-0.1786852777,-0.5286377668,0.2368431985,-1.6006636620,1.2182400227,-0.9850192070,-0.7063335776,-1.0004578829,0.2919587493,-0.1049265787,-0.7563028336,0.4225782156,0.0824065655,0.2042005062,1.2740459442,0.1095527783,1.1274046898,-0.6521725059,0.0609520078,-1.1165735722,0.9280680418,0.1987430453,0.4761828184,0.1669905335,1.0124512911,-1.2776417732,-0.3814474642,0.7289512157,-0.3052847087,0.7282732129,-0.5393836498,1.2419661283,0.5911603570,0.3793136477,0.1260632575,-0.1371300519,1.8880878687],[2.8134269714,-0.2924009860,0.3732911646,-2.3748250008,0.3623911738,-1.5661566257,1.3808929920,0.9725285769,0.2446972132,0.7711442113,0.3922756314,-2.1892728806,-0.0823605731,-0.2208417803,0.4101850986,0.2744546831,-2.2105195522,-1.3529791832,-2.0533165932,0.7840827703,-1.2388772964,-0.0091828620,0.2710341811,0.7293034196,-1.9300572872,0.2674436867,-0.6061401963,1.2984786034,1.4081494808,-0.7296541929,0.4568593502,-1.4688404799,0.3038083017,-2.5448539257,-0.9092229009,-0.3816877902,0.7162517309,0.7011740804,-1.4708878994,1.0043964386,-0.0533178635,0.1780886203,-0.0656738058,-1.8832274675,-0.7870368361,0.2258797288,0.8136019707,-1.6132806540,-1.6500467062,-0.7262764573,-1.8060714006,0.1261256337,0.0837224349,0.0827596188,0.7586573362,0.6176240444,-0.3255368769,1.4318530560,0.0577022061,-0.2958157361,-0.6132124662,-0.8152903914,0.8220098615,0.2450569719,1.2573981285,1.5494660139,-1.2309507132,-0.9299182892,-0.6454734206,1.0468801260,1.1484909058,0.5869446993,-2.1177897453,0.4626471102,1.0198746920,-0.1138288900,0.4247196913,1.1570379734,-1.4135748148,0.0725923181,-0.2679024637,2.7661833763,0.2679988146,-0.5027480125,-0.0181943942,0.6711435914,-0.0941974670,0.1855315417,0.2543438077,-1.4764968157,-0.9377606511,0.3014186025,0.2370459586,0.7084241509,0.1626368910,0.3276515603,0.1585559845,0.2503229976,-1.1399179697,-1.0920814276,-0.6667899489,-0.8548904657,0.7033684850,-0.7761536837,2.1346483231,-0.2393279821,1.3535968065,-0.9506493211,-0.4076920450,-0.5581548810,0.1968198419,-2.0824365616,-0.6504297853,-0.7817162871,1.5999169350,-0.4707145393,0.2664442956,-0.0279191937,-2.3622436523,-1.2645890713,-0.0324613638,-0.4157408774,-1.9002436399,0.8337079883,-0.5889626145,-0.5031498671,-0.0618600398,-2.4662024975,0.0627399012,0.8152358532,2.1090805531,-0.0630077720,-0.1326286197,0.1434296221,0.8538603187,-0.1592878252,1.6691845655,-0.4810115099,0.4189854264,0.2810520232,1.4511069059,0.1108039841,-1.6083573103,-0.2608066201,0.6873603463,1.0323524475,0.3345609605,1.4037721157,-1.1845630407,-0.2300915718,-0.1948073506,0.5498065948,-0.2714291513,-0.7518230081,-1.1877788305,0.3030099869,1.2471328974,-2.2817478180,0.5460087061,0.8085818887,-1.4102207422,-0.7649542093,1.1998691559,-0.3109331429,-0.5489453673,-1.1739296913,-0.7573813200,0.3622814119,-0.1803682745,-0.5011548400,-1.0009831190,0.1868225932,-0.6029005051,-0.8547886014,-0.1348072588,-1.1864544153,0.7664079070,-1.4393889904,-0.9475734234,-0.5573213100,0.8925701380,0.5009971261,0.7198488712,-0.5231234431,-0.1421491206,-0.6183432341,0.0882850811,-1.6847449541,-0.5839746594,0.2101413608,-0.6959885955,-0.4180793464,1.7377842665,0.5398337245,2.4762804508,1.5915863514,-0.7944422960,-0.3500894308,0.1223903745,1.8400007486,-0.0912151709,-1.1437829733,-0.3499307334,0.8078536987,-1.0026249886,0.2237447202,-1.0386419296,0.2516576052,1.2548787594,0.0778929368,-0.4539818168,-0.5510498285,-0.0578782186,0.4725125134,1.8870012760,2.4323177338,0.2010755539,1.1056728363,-1.8366775513,0.9536247849,0.7939686775,1.1003919840,2.3458998203,1.4692678452],[0.6490464211,1.9670273066,0.4042147398,0.6566967964,-0.7697871923,1.5224796534,-1.0281040668,-1.0950347185,0.8894008994,-0.7297667861,0.6717054844,-0.2670283914,-0.5510465503,0.0973181725,1.1938539743,-0.0754729062,-0.4562895000,-0.8172528744,-0.0423791744,-0.5283162594,-0.3101071119,0.1037842855,0.3862933218,-2.5298702717,0.0148019399,2.0022518635,0.4043906629,-1.8893733025,2.0781323910,-0.2799007893,1.5677926540,1.0496910810,0.2109147310,0.7038741112,-1.3184523582,0.7989603281,0.2703145444,-2.6894779205,0.5198162794,-2.2797818184,1.5013480186,0.9218204618,0.8572191596,0.8756232262,-1.7740607262,-0.4072049558,-0.0811734796,1.3906067610,0.8266698122,-1.7439453602,-0.0620697737,0.1208140701,0.1889812201,1.7714536190,2.0588123798,-2.7417387962,1.1500811577,-0.4305594563,-0.8185398579,2.0702614784,0.5266123414,1.0601838827,-0.5946053863,0.8639861941,-0.1706417352,0.6222539544,1.3736292124,-0.7181694508,0.0831693858,-1.1598395109,-0.4952351451,1.2962372303,0.0690178201,0.1662487537,0.3988175988,0.1684028208,1.8821505308,-0.5882483125,-1.0507441759,1.7254124880,0.1957102418,-0.3709122241,1.4042637348,-0.2510891557,-0.4316080213,0.0685362518,0.6092994213,0.7451376319,-0.4354406893,1.3262227774,1.4801108837,1.3739769459,0.1878964752,0.4108370543,0.9605338573,-1.5134578943,-0.1964982003,0.6435061097,0.0736133829,0.8728601336,-1.3197245598,-0.1270552576,0.0426341854,0.8820625544,1.1046253443,-0.2755198777,0.2538744807,-0.9900706410,1.3647599220,-0.8029752970,-0.9458059072,1.7950054407,-0.3628331125,-1.4039831161,-1.1162095070,0.4925628901,-2.0292842388,1.5619550943,-1.8280488253,-0.0055643721,-0.5366774201,-0.7328761816,-0.8478962183,0.0121869650,-1.6183778048,-0.4681866765,0.2380838990,0.3942469358,-0.0017471110,0.6457191110,0.1618220210,0.3286625147,0.1949937940,-1.8389343023,0.0252901614,-1.1302721500,0.5937021971,-0.1768487543,1.4522219896,-0.4664727151,-1.4856642485,-0.9684375525,-0.6833172441,-0.4522916675,-1.3983139992,0.2946139574,0.7114067674,-0.4360235631,1.3628817797,0.3709173501,-0.1819907874,-0.1708724201,0.6615654826,0.5053963065,-1.2913050652,-1.1293932199,-0.3762286007,1.4005522728,-0.3681354821,0.2439703345,-1.9376685619,0.0061770533,-0.3084800243,0.0341180898,0.6517161727,-1.0235981941,0.3485942483,-0.8717853427,0.1752543598,-0.8681605458,0.5755802989,-0.4126141667,-0.7222152352,0.9932783842,-0.8108548522,-0.8651838303,-0.4406715333,1.2634201050,0.3228797913,1.7299368382,-0.2031331807,0.7388085127,-0.3652395010,-1.0800300837,-0.4905096591,-0.3866543770,2.2770588398,-0.6903892756,0.9328139424,0.8298351765,0.0020154689,-0.4209246635,-0.6973096728,0.0860553086,-0.5382195115,-0.3424751163,-1.3047096729,3.0107023716,-0.2132201046,-1.7896158695,-0.2584707439,0.7357210517,-1.6454079151,-0.1919890344,0.8854669333,0.5473588109,2.1477603912,-0.3015838265,-0.3280054033,-0.5345777273,-0.0564358309,-1.0776684284,-0.0625013635,-0.8091907501,-0.9522696733,-0.0291139688,0.8151841760,0.4640530646,-0.4346605241,0.7263224721,-0.9717104435,0.0229827929,1.1674512625,-1.3929512501],[1.2918885946,-0.2988780737,0.2283933908,0.3405737579,0.3639356792,-2.1020929813,0.8405295014,1.0143632889,-0.1635631919,-0.2963094711,0.1203462780,-0.5817037225,1.0243686438,-1.1446744204,-0.1163422912,1.0779199600,0.7442581654,-0.5289862752,0.3235702515,0.8104526401,-0.6944915652,0.7105034590,-0.4998841286,1.3602881432,0.9865704179,0.9607505798,-0.5986943245,0.3859920204,0.8108100891,1.3603370190,0.1453486830,0.4959866405,1.0660791397,-0.3927115500,1.5472337008,-0.9704090357,0.6275779605,-0.1618680060,0.5084855556,-1.0714122057,-0.8444173336,2.5354545116,-0.4143082500,1.4401415586,-1.8746173382,0.0717201754,0.9315521717,-0.1924583465,0.2826135755,-0.4956521988,-0.5171129704,2.2835865021,1.2014240026,0.4965412319,0.6352142096,0.2207820266,-1.5204813480,-0.2159054577,1.4612492323,0.4836098850,0.7791002393,-0.7924147248,-1.9963259697,-0.9082238078,-0.0739156827,-0.1735779345,-0.6885651946,-1.6898833513,0.1898370534,1.3742725849,0.4149084389,0.9179326892,0.1505246013,-2.1365816593,0.6585453749,0.0833003744,0.1664917320,0.2179910988,0.1975180507,-1.2237678766,-1.4798632860,-1.5096775293,-1.6420594454,-1.0052191019,-1.4127585888,0.2579631805,-0.4421169162,-0.0934734941,0.4567604363,2.2722504139,-0.3473166525,0.1571355462,-1.1206731796,0.4965228736,0.5498935580,0.1304169148,-0.7870458961,0.8281716704,0.4544727802,-0.0937828198,-0.8460122347,-0.0507329628,-1.3765958548,2.1486096382,-1.7733092308,-1.1574308872,-1.3433600664,-0.3764662147,0.2813237309,0.2202699929,0.6045076847,-0.1821339726,1.2025723457,-1.3828216791,-0.9747838378,0.2388301194,-0.8707327247,-0.1252780706,-0.6416910291,1.3086551428,-0.5332537889,0.0063522253,-0.6723710895,-1.6051152945,0.3130868971,-0.1302162409,1.4862173796,-1.0393095016,-1.2276927233,1.7093112469,-1.7208932638,-0.3173751235,-1.1881891489,-2.3398308754,0.2897851169,-0.5540105700,0.3708947003,-0.6902725697,0.4827075303,0.5152086020,-0.8001226783,0.5289755464,0.6751053929,0.1977580190,-1.8228195906,-0.5684699416,1.7509375811,-0.7015925646,0.6359348297,1.3688038588,1.0106269121,-0.6654335260,1.0200300217,0.0526707098,0.4489641190,2.0376715660,-1.5846176147,1.3363882303,0.4211618304,-2.3855931759,0.4183519781,0.7159678340,0.1074562818,-0.2730627060,0.3309553862,1.5496168137,1.1344338655,0.9619284868,-0.8347564340,1.0374566317,1.4408031702,0.1390461475,1.7395902872,2.9890756607,-0.3393157423,-1.5076727867,1.4666700363,0.2921347022,0.3553029597,0.9788808227,1.2384502888,0.3073142171,-0.6225633025,1.0165023804,-0.7128052115,-0.0668653473,-0.5550617576,0.4804181457,2.0454192162,0.0330334306,-0.2663131058,-0.7029079199,-0.2169049233,-1.0697338581,-0.8404394388,-1.1840552092,-2.3508896828,0.1685397476,-0.1027170047,0.5997434855,-1.9493203163,-0.8024225235,-0.4092237949,-0.4173745215,0.2873328328,0.4117684960,0.2399783283,0.2981419265,-1.6334201097,-0.9804607034,-0.3757415116,0.6015896201,-0.7534538507,-1.1112320423,1.6509275436,-0.1514259428,-0.4161066115,-0.3126336038,1.3812828064,0.8161883950,-0.0387242921,0.7398338914,0.0209227726,-0.4717726409],[0.1601802260,0.7710543275,0.1447940767,-0.0283841789,-1.6753015518,-0.8868216276,-1.0859651566,-0.6444486380,-2.2279274464,1.3281667233,0.0351902060,0.0617455207,-1.5774900913,-0.5398665667,0.7255911231,-0.4908719063,2.2670347691,-1.9021373987,1.2350738049,-0.9386199713,0.7089691162,0.7957819104,-0.9514808655,1.2497242689,-0.3672064841,-1.3906115294,-0.5919629335,1.7621365786,-1.4293707609,0.3684018254,-0.7765325308,-0.1114501879,-0.9075161219,1.2576494217,-0.8669949770,0.4935116470,0.1877672523,-1.3370648623,-0.7650950551,1.1577278376,-0.6422299743,-0.6276326776,0.1233862042,0.2219813317,0.0170367789,-0.7312810421,0.6946656704,-0.7900857925,0.5711570978,-0.6337569356,-1.2564939260,-1.9684580564,0.7597616911,-0.6197546721,1.0595633984,-0.1620984077,-1.4307351112,-1.2425879240,0.3334033489,-0.8551697135,0.7142262459,0.2552497387,-1.4737071991,-1.2400416136,1.2253839970,-0.4654192924,-2.1136255264,0.6339798570,1.6607668400,0.7691323161,-0.0107232919,-0.3207734525,-0.8382658362,0.3205773532,0.0976580307,0.2763164639,-0.4415012300,-0.7790068388,-0.6566889286,1.4151773453,0.4216744602,-0.8975771666,-0.4491711557,1.3207405806,0.9556534886,-0.2062499523,-0.4255151749,-0.5410502553,0.7635242343,1.0545524359,-0.4757571518,-0.3823243380,0.3522265255,0.6711670756,-1.7673711777,1.4733440876,-0.5324310660,-1.6187958717,0.6993540525,-0.9426357150,-1.1725157499,-0.3552233279,0.5078729391,1.4298133850,0.2571879029,1.4719634056,0.5778535008,-0.2065822333,2.3589992523,0.2131077051,-0.3065221906,0.4680456221,0.1980422735,-0.8443328142,-0.2465468198,2.7550952435,-0.0163592007,0.0708196610,-0.3586863875,-0.1086756513,1.2919684649,0.9084424376,0.7995363474,0.3683059812,1.1267640591,-0.0028436235,0.1808570176,-0.1837261468,-1.2601875067,-0.2753442824,-0.0100960284,-0.1514759511,1.3774771690,0.2780973613,0.1607485414,-0.3234707117,-0.6531004310,-0.4414724708,1.2662265301,1.3459706306,-1.0425080061,-0.2603740394,0.0437238626,0.2833698094,-0.0748640820,-0.3282703161,0.6336637735,0.0157164261,-1.1069000959,0.0525508635,0.2700309157,-0.3761922717,-0.4791140258,0.4749079943,-0.2400727868,-1.6324510574,0.1481562406,-0.2036515921,0.5730488300,0.3368732631,0.6147609949,0.8890558481,-0.7400008440,2.0510735512,0.0945650861,-0.3494800329,-0.6844366193,0.1977184117,-1.5378758907,1.7580767870,0.4568970799,0.7572392821,1.4373345375,0.2000212222,0.0556829832,-0.2715509832,1.1920673847,0.8942832351,-1.2035263777,0.4194567502,0.4005192220,0.5703665614,0.5575336814,-1.3979532719,1.0963864326,0.0250807907,-0.0674761832,-1.3193664551,-0.3257948756,-0.1157574356,0.1749069393,1.5000028610,1.1391853094,-0.9547190070,1.0541182756,-0.2260276377,0.7414301634,0.9308434725,0.1472192556,0.2028086632,1.0589454174,-0.3633924425,-0.3617959023,-1.1625291109,2.2101442814,0.2819685638,0.1859583557,-0.2687607408,0.0440233089,-1.0015631914,0.6896772385,-1.3712488413,-1.0608513355,-1.1808810234,-0.4728468060,1.0679525137,0.9100573063,1.6535276175,-0.5239212513,0.5605090261,0.9432093501,-1.9324355125,0.8190538883,-2.1978337765],[-0.2977302969,0.6257901192,-0.1263576597,0.9088796973,1.5055203438,0.1521550417,-2.2133748531,-0.1194746345,-0.5715026259,1.5380450487,-1.0065170527,-0.5844327807,2.0164713860,0.0378994644,-0.1642175913,0.5951735377,-0.7040801644,-0.2119091898,-1.3055820465,-0.0513718873,-0.3308599591,0.0339758396,0.8075446486,0.1187833175,0.6131482124,-0.4814960957,0.0764794052,0.9820422530,-1.7076627016,1.3723251820,-0.7145006061,-0.5938451886,-0.0183429793,0.7288543582,0.8106418252,-1.5039621592,0.3948372304,1.0601956844,1.0729640722,-0.2177887261,-0.3614816070,0.6474339366,0.4636586905,0.1736662537,0.7257980108,-1.3463358879,-0.1328872144,-0.4435177147,0.2674212754,0.8134050369,-0.8270956278,-0.7177349329,0.6141757965,0.6915240884,-0.4994766414,0.5097733736,-0.1994809657,-0.9650039077,-1.3885537386,1.5378166437,0.5882808566,-1.0985815525,-0.8555923700,-1.3664350510,-0.5528749228,0.2774377763,0.6846002936,-0.6543120742,-0.9933399558,-2.6504812241,0.3394016325,-0.2626033425,0.4674544036,-0.2633951604,-0.2068358809,-1.0403506756,0.4053127766,1.1273319721,0.1939062625,1.3327549696,0.7872596979,-1.1306591034,-0.3493178189,0.1557210684,1.1165831089,1.3770676851,-0.7949782610,-0.6838003397,0.6526792049,0.6445679665,-1.1254947186,-0.2173337489,-0.8494017124,1.1666642427,1.2532933950,0.9628742337,1.0837029219,-0.6331955194,0.8385151029,0.3361526430,0.3915138543,-0.0933211073,1.1968898773,-0.5742031932,-0.0500956103,0.9402880669,0.7142255902,0.5690819621,0.7714705467,-1.5002479553,0.3130998313,1.1639127731,0.6727718711,0.0101976758,0.9451437593,0.5655166507,1.6275627613,0.8402985334,-0.3411326110,-0.3251051605,-0.7312197685,-0.2371633053,-0.1245433688,-1.7427054644,-0.6266471148,1.6334964037,-0.0915187076,-0.3417382836,-0.0914409012,0.6394451261,-0.9623674154,-0.2364961803,-1.0329637527,-0.1231606677,-0.2556661069,1.1051532030,-0.3228335083,-0.9258052707,0.6563531756,-0.4999623001,-1.6209917068,-0.1153591797,-0.8811206222,1.0089468956,1.2612160444,0.5947718620,-0.2692162991,0.2056351900,0.5102504492,1.7642725706,-0.5403218269,1.0996212959,-1.2504529953,1.0675100088,0.5051488876,-0.8187961578,-0.8085387945,0.0946609080,-0.1569521129,0.7736035585,-2.0229930878,-0.5482857823,1.0867170095,-0.1797525287,-1.0244921446,-0.3236004710,1.0927515030,-0.2026444674,0.4056845307,-1.0475788116,1.4953615665,2.0108199120,-0.3379966319,-0.3342337012,-0.4520601332,0.9962898493,-1.3105881214,1.2578663826,0.2881660163,0.5212141275,2.0398442745,0.8010252714,-0.7030718923,-1.5192036629,-1.2763825655,0.7861399651,-0.5948548317,1.0182152987,-2.0379059315,-1.1396664381,-0.4048861265,0.0318119638,0.8137828708,-0.8330386877,-2.0578346252,1.3120894432,-0.2637521923,-0.4078529477,0.2702302933,0.7227398157,-0.8606963754,-0.5009006858,0.4711194634,1.6186574697,1.4121378660,-0.0002848467,0.6598795652,1.2394162416,-0.5023958683,0.4566825926,-0.8102485538,-0.0185096450,1.5662649870,-1.0101697445,0.5655524731,-0.2134593427,0.0219880417,1.3343462944,-0.9169719815,-0.1619948745,1.0586124659,-0.3841233253,0.1626750678,2.1752021313],[1.5525988340,-0.2626132071,-0.6253407598,-0.3021035790,0.5044687390,-0.1912412196,-0.0795418695,-0.6908488274,-0.4781500995,1.0784960985,-0.4422915280,-0.4586786628,-0.1436938047,-1.0484225750,0.3351688981,0.9214617610,-0.1563158184,0.3626534641,0.6382313967,1.2679969072,-0.0452971682,0.7486544847,0.5606036186,0.9569472671,-0.0700276718,-0.3959464133,0.1904693246,-0.7078367472,-0.0979526788,1.9592298269,-1.1198773384,0.4000999331,-0.3875292540,1.4898318052,0.6270580292,0.5690717697,-0.3728976846,-0.5502703786,-1.1284275055,-0.0041970341,-1.6097604036,1.1527045965,1.6746832132,1.5408614874,2.0273838043,-0.5349593759,0.7125822902,0.7540348172,-2.8705582619,-1.8638910055,0.5038121939,0.6373669505,-0.5574759841,-0.5050622821,-0.8507771492,-0.5713256001,0.8568009734,0.5190439820,0.0653787851,-0.9294126034,-0.6523821950,0.4704572558,-0.8607774973,-0.1529620588,0.0826679990,-0.1757801771,-0.5495941639,0.7834180593,0.0671116933,0.7145394087,-1.3842253685,-0.7460412979,-0.6277463436,1.0182669163,0.0907923281,0.9149090052,1.0882586241,1.7943143845,0.4608672261,-0.0809834972,-0.8602779508,1.1938617229,0.6517693996,0.0092181843,-0.7635804415,1.2017304897,-0.8502430916,-0.1846573204,-1.6849205494,-0.2419226319,0.4188693166,1.2817186117,-0.4473271072,-0.1569877714,0.7320977449,-0.0141397016,0.4355955422,0.4715900123,0.6773759723,-0.4215749204,0.3264049292,2.4344141483,-0.0604711324,-1.0446228981,1.5301111937,1.3959794044,-1.4553323984,1.7583258152,1.5257213116,-3.0210578442,-0.9142537117,1.7957469225,1.1200355291,0.5840203762,0.5999845266,0.0407632738,0.6290143728,0.2058549970,-1.1277829409,-0.8097259998,1.6846119165,-1.2507945299,0.1569381058,0.7785435915,0.7929396629,-1.8754302263,-1.1775150299,-0.9298060536,1.7393478155,0.4945442080,0.4163727760,1.5628179312,0.2502279580,1.3238953352,0.4312912524,0.0220150761,-0.2147385180,-1.8523584604,-2.0134761333,1.4432740211,-0.5076486468,1.3906644583,0.5807424188,1.2673971653,0.2372799963,-1.5070856810,1.0928071737,-1.5873508453,-0.1693418473,-0.4200076759,0.2688405514,-0.3601242900,-1.2954751253,-0.7431051731,-1.2575507164,-0.2012025267,2.2574408054,0.1104647666,-0.1512969583,0.0174025111,0.4458542466,-0.7248215079,0.9082215428,-1.3589265347,-0.9368333817,-0.7312052250,1.0721185207,-0.7326382995,-0.5639464259,0.6095701456,-2.0346000195,1.0954475403,-0.2311936021,1.4384429455,0.5641620755,-0.2077282071,2.3088765144,0.9453276396,-0.1790513992,-0.6986765265,0.0849507675,1.8025000095,0.4350953996,0.7734685540,-0.3476094306,-0.3814012110,0.4308768213,-0.9413963556,0.5490882397,-1.2022879124,-1.0516980886,-0.9482901096,-1.6367952824,1.8386996984,-0.7118865252,0.1055972502,-0.7026752830,-1.0756582022,-0.1422663033,0.3937110305,-1.8437414169,-0.2732995152,-0.2043423504,0.5992166400,0.6439359784,1.7993170023,-0.7580192089,-1.3359285593,1.4962481260,0.1730103940,-0.6356427073,0.0779678747,-0.3810184300,0.4025068283,-1.0337539911,-2.1191465855,0.2678066492,-0.5662587285,-0.5005620718,-1.0199508667,0.2415617257,-1.9484695196,-0.5845148563,2.9574551582],[-0.1329732090,-0.2781083882,2.0026571751,-0.6654509902,-0.9315814972,-0.5124284029,-1.0220663548,1.5914081335,0.7247942090,-0.8515052199,0.6342797875,-0.4960220754,0.5928196907,-0.9952784181,0.8246176839,1.0320022106,-1.9398835897,2.0664064884,-0.9641517997,-1.8633273840,-0.0337739438,-1.3035702705,0.1331416517,0.1698399037,-0.1870108247,0.3671292365,-0.4189546108,-0.4695348442,-2.6624681950,0.5423961878,1.5010020733,0.3459192812,0.0675365776,-0.8721294999,-0.4595796466,-0.5068891644,0.5873795748,0.3420045674,-0.0998604968,0.4836119413,-1.3472015858,1.6239326000,0.6348768473,0.0179399494,-0.4565315545,-0.2640585005,0.8355404139,1.1033011675,1.5110306740,1.1274770498,-0.7093459368,-0.0744580925,0.2659905851,-0.1485776901,-1.6719353199,0.7489042878,0.0443355255,-0.9490557909,-0.8815689087,-0.6441028118,1.2353124619,-1.6454527378,0.3230223954,1.0776292086,-0.8387464285,-0.5173830986,1.4620894194,-0.0286578313,-0.7814294100,1.4929542542,-0.7920092344,0.3892004192,-0.4025924206,-0.9289677143,0.8831150532,-2.4916441441,-0.7269167304,-0.3389181793,0.0518149436,2.3087661266,-1.0090583563,0.9877404571,0.1650268286,0.9553805590,-0.7569466829,0.7209989429,-1.1299874783,-1.3067044020,1.1002798080,1.8736176491,-0.1802768111,0.9645875096,0.7066676021,-0.1683974713,-0.9512957931,-0.8372158408,-0.3797715008,-0.2867600620,1.0953788757,-0.4406148195,-1.3737680912,-0.8403751254,0.7122806311,0.6732238531,-1.2772501707,1.4470598698,-1.6477565765,0.8578764796,0.9182669520,1.3492974043,0.6436387897,-0.1199597120,-1.4940854311,0.1026622728,0.7803411484,-0.5787874460,0.9917155504,0.4909535050,-0.5574346185,-0.6866763234,0.2321932614,-2.1183209419,0.4316413403,-0.4042158127,0.8837202191,-0.1353183091,-0.7172310352,2.7005834579,1.1152646542,0.8556065559,0.3109346032,-1.0019065142,-0.1889768243,0.6665068865,1.8737750053,0.5609683990,-0.4682036340,-0.3340584040,1.0639505386,0.1632687896,-0.2742658556,1.0798113346,-0.6057754159,-0.2749799490,-1.1890770197,-0.2930025756,1.8735110760,2.2746007442,0.8219443560,1.9643771648,1.3195849657,0.1458032131,0.5102447271,-1.6404271126,0.6829751134,-0.6760859489,-0.7591422200,1.1757338047,-1.2378255129,0.8542348742,-1.6906770468,-0.5590405464,-2.0516254902,-0.7075557709,-0.6136775017,0.7814898491,0.3277451098,0.7363686562,-0.7694044113,2.5541281700,-0.9461388588,-1.0866278410,-0.6960321069,0.3335900009,1.2417874336,1.3287516832,0.9328956604,-1.0503815413,0.2631334960,0.5054531693,-1.5257029533,-0.2071426213,1.1814788580,0.2966313064,-0.9725993872,0.3309081495,-0.3167668879,0.6853064895,-0.2060380578,2.7066907883,1.5249216557,1.2280702591,0.5085874200,-0.3147315979,-0.1998735368,-0.7020621896,-0.7529008389,-0.6573159099,-0.4857104123,0.2419129610,0.4221928120,-0.8887619972,0.4519138634,-0.5847700834,1.5164148808,-0.3560203314,-0.9716092944,-1.1448581219,0.1373418719,-1.6014651060,1.3189338446,0.2547961473,-1.4622094631,-0.1789774597,0.8794451356,-0.3040826917,1.5860936642,-0.7125411630,-1.5092651844,-0.4747304022,0.4026572704,-1.7841160297,1.4312634468,0.6950429678],[0.4651306272,0.8870432973,0.0887216479,0.4096937478,-0.5062332749,-0.2962773740,1.1971416473,1.3719564676,1.9563847780,0.8283421993,1.7993576527,-1.2152526379,-0.0858739465,0.2326095849,-0.2911576033,1.0595628023,-0.5787106156,-0.4047865868,-0.5017714500,0.0693776011,-0.4276338816,-1.3622661829,0.4445276856,-0.7927712202,0.9814260602,0.5316746831,1.5762796402,0.8111576438,1.9288363457,-2.2223255634,-1.3883488178,-0.3999559581,0.1310270727,-0.3544170260,0.9905593395,0.2306950390,1.9439210892,-0.5027143359,-0.0143833691,0.3165681660,-0.1115248650,-0.0917987972,0.2415666580,-0.0787839368,0.4116333425,0.5505738258,1.1692513227,0.7651976943,1.2653208971,-1.2472923994,1.1245467663,0.4015773833,-0.2108438611,1.5653553009,-0.0090202373,0.3653865457,-1.0638052225,0.3883264959,1.2481312752,-0.1546858996,1.3234074116,-0.8675174117,2.0376651287,-0.7730429769,0.4148543179,0.2054794282,-0.8716959357,0.5587248802,-1.0899597406,0.5063377619,0.2100979537,-1.6693204641,-0.4869732261,0.8901972771,0.7649075389,1.4790863991,0.1984126270,0.2106683105,1.2196710110,0.4870115221,-0.8584403992,-0.6259924769,-0.4319318235,-0.2345139086,1.1433793306,1.2349792719,1.1137609482,0.3417917192,2.7702353001,0.5551794171,-0.7779662013,-1.3389432430,0.3691887856,2.0352487564,0.1049689651,-1.3625838757,-1.2054003477,-0.1510334313,1.4791287184,1.5493704081,-1.2234296799,1.1098680496,-0.3641369641,0.2689851820,1.0093431473,-1.0931015015,0.8034202456,-0.1910215169,-1.2416673899,-1.4777083397,1.0865781307,-0.2192025781,-1.4991483688,-0.3896990120,0.7066301703,0.6640823483,-0.2975024283,-0.1652295887,0.2889887094,-2.8535153866,0.6054579616,-0.0322866999,-0.8152453303,-1.0316278934,-0.4888164401,-1.3081847429,-0.1466890872,2.9084668159,2.2889723778,-0.4269299209,1.8235203028,0.6112687588,0.1520540267,0.2729745209,0.4491570592,2.5501518250,1.4125373363,-0.3937563598,0.6757388115,0.1855499148,0.8891111016,0.4965530932,0.6705762148,-1.0051658154,0.4118530452,-0.3306933641,0.7756954432,-0.1614267975,0.3098185360,1.0678449869,0.8791055083,1.5784460306,0.1212876365,-2.1881835461,0.2398423702,-0.6166628599,-1.0495116711,-0.2046049982,-1.5786684752,-0.4201811254,1.0646731853,0.6574370265,-0.4980138242,-0.9568085670,-0.3948968053,0.5613503456,-0.2417133749,0.0204319879,0.4374487698,0.9105584025,-0.1001281738,-0.0798568353,0.0035209819,1.7699304819,-0.8864417672,1.1483435631,0.9128133655,2.3566586971,0.0609659180,-1.8903415203,-0.9600875974,-0.0770855024,-0.2616542280,-1.2499434948,0.5119954944,-1.7796143293,-1.5820428133,0.8977500200,-0.7375071049,-0.7488090396,-2.2484743595,-0.8623650074,-0.7743161917,1.3640680313,0.4618594646,-0.4605868459,0.1531334072,0.2061138004,-0.7647488117,-0.0337435305,-0.7385146022,-0.5625520945,-1.2569856644,0.2129893452,-0.9553509355,0.0948104262,2.5250504017,-1.6878075600,0.6934902072,0.6625792384,-0.5822758675,-0.7410813570,0.0621082224,-0.0299436729,1.1133687496,0.4249279499,0.5673895478,0.1886189878,0.0354875848,0.1282968819,-0.6992625594,1.1561349630,1.9655958414,0.4005120397],[-2.1143636703,-0.5863888860,0.7493094206,0.5219535232,0.8459887505,0.5666259527,0.1399505585,-1.2476874590,-0.4998947680,0.3239963055,2.3878009319,0.4304938018,-1.2388671637,-1.0990287066,-0.3986670673,-2.1690292358,0.5610336661,-0.2620897889,-0.7111989856,0.2920810878,-1.6883717775,-0.1379259974,-0.9189206958,0.4376925528,1.5696545839,0.6147102118,0.5508036017,-2.5583667755,0.3781401813,-0.5038151741,-1.7127869129,-1.4350970984,-0.1412201822,-0.0816503242,0.4081589580,0.7809235454,0.6255474687,-0.2427212447,-0.2043065876,-0.6814991832,-1.8603205681,-0.5025548339,-1.9000107050,-0.1304477006,-0.6851273179,1.0265516043,-0.6661650538,-2.3975861073,-1.7486999035,0.0605677813,0.6212993264,0.4197052121,-0.3831737936,1.3025627136,1.1073590517,-0.8643981814,1.3570622206,-0.4238911569,-1.2117905617,0.1408114433,-0.1960694492,0.6899011731,-1.3563886881,1.4247977734,-2.2074992657,-0.5906651020,0.6465181708,-2.1144294739,0.8956786990,-0.9129745960,2.0979588032,-1.1935856342,-1.2795675993,-0.8881188035,1.0661122799,1.6177762747,1.1079305410,-2.0232996941,-1.2102463245,1.5625468493,1.1707061529,0.2248326093,-2.2631869316,0.3136663437,2.0882434845,-1.1118528843,0.8269920945,2.2001664639,-1.0525869131,0.1732597351,-0.6301019788,-0.2861230671,-0.7035486102,-0.6158868670,0.6836638451,1.2540742159,1.1350761652,1.1168063879,-0.0671314672,-0.5278354883,-1.6708488464,-1.8426793814,-1.4898799658,1.5074266195,0.8699860573,0.4585454762,-1.2405214310,-0.3520327508,0.2623713315,1.2189782858,1.9564009905,-1.8678971529,0.2386119515,-0.3934918940,0.0209807158,0.0772104561,-1.7392042875,-3.5995438099,0.9020640850,-0.1195738018,-0.5754822493,-1.1097879410,-0.1072186753,-0.9343097210,1.1105796099,-1.1826517582,-0.1520720869,-1.3386286497,-0.6448604465,-0.0941165313,-1.3919219971,-0.0296481308,0.1762068868,-0.2783095539,-1.6376761198,-1.0517594814,-0.6948207021,0.0998614058,-0.6569946408,-1.8207826614,-1.1987988949,0.8665890098,-1.1092697382,-1.3847301006,-0.3812083602,-0.3967207372,0.7238845229,0.1558206677,-0.3647925258,-0.5851247311,0.4368338585,-0.7570932508,0.0958884582,0.7646985650,0.6894189119,-1.3329033852,0.0163381044,-0.3504721224,-0.7617780566,-0.8003528714,0.9703143835,0.2425616533,1.4180577993,0.8548147082,-0.4770228565,-2.0487129688,-0.4198290110,-0.5687003136,-1.3870117664,-0.5220409632,1.4553519487,0.5901443362,-2.1908631325,0.0048135603,-0.9869437814,-1.1687397957,-0.6182921529,0.3242558539,0.1612989306,-0.5659643412,-0.6204778552,-0.6357564330,-2.4414963722,1.7249587774,-0.4532583356,0.7349098921,0.6877690554,1.6072475910,1.3938087225,-0.1504776627,0.1785371900,1.7748429775,-0.2517042160,1.1803649664,-0.4024221003,0.2015507370,0.2357416600,-0.1340533942,-0.1057958901,1.6359223127,0.3880910873,-0.3377050757,-0.4242796600,0.1702330261,1.4504141808,-0.2115017474,1.4328291416,-0.5159255266,0.0500065312,-1.7297106981,1.3677421808,-1.0256661177,0.7512863278,-0.5840461850,-0.9218277931,-1.3074322939,-0.7596263885,-0.1303110570,0.6998056173,1.6276044846,0.9300687313,-0.6382806897,0.6924938560,-1.4637494087],[-1.0807045698,-0.7944297791,-0.2523753047,-0.9769495130,0.0696765333,0.0790520534,0.4146587253,-0.3838759363,-0.8621741533,0.5710561872,0.5936986208,0.4419238269,-0.2544811368,-0.0549358837,0.2354987264,-0.5314922929,-0.5649043918,-2.2214460373,0.5590788722,1.1864672899,-0.7585781217,-0.6145302057,-1.5558300018,-1.4099539518,0.8280416727,0.9231147766,0.7842741013,0.9364667535,0.1571408212,-0.0711527541,-0.3171508014,0.5082898140,-0.4674690962,0.2489854246,-0.6435582042,0.1705356985,0.0941540748,2.0618970394,-1.3521838188,-0.6742211580,1.9083428383,0.3393023908,-1.2060261965,-0.3309186101,-0.1274856180,1.2129625082,0.0350924097,0.8869325519,0.4675163925,0.3396925926,-0.6079598665,0.8370874524,-0.9059494138,1.4852802753,-0.2368526459,0.5075811744,1.3175470829,0.9353399873,-0.5524255037,0.1175686345,-2.5573158264,-1.6868667603,0.2753569782,1.0403324366,-0.6371220946,-0.2227719426,-0.2769407034,-0.5384197831,-1.2360104322,1.6299774647,0.3417989910,-0.1112552434,0.3643215597,1.7209881544,0.5956207514,0.1271493584,-0.3662891388,0.8358281851,-0.4693106115,-1.0826193094,1.5552965403,-0.3990328312,0.3532243371,1.3296083212,-0.1214048937,1.4217294455,0.2872732282,0.0609251931,-1.2650548220,-0.2146121860,-0.5631961823,0.9530636668,-0.5803635716,1.0396372080,0.0411144495,0.4864621758,-0.6175361276,-0.0901167020,-0.4780751169,1.1069477797,-0.1862807423,-0.8247175217,-1.6179413795,0.7025146484,0.3208766282,-0.1970423460,0.7278860211,1.6373358965,0.6480888724,-0.8358708024,0.6518102288,-0.1171416640,-1.0953273773,0.3416704834,0.2952704728,0.0544999093,0.1129286736,-0.2682801485,-0.7968363166,-1.3915562630,-1.0005905628,0.0861226767,-0.2342195213,-0.2970535755,0.3015836179,0.6606718898,-0.3637237549,0.2571111917,0.4363758266,-0.2371700704,1.7235711813,-0.2948478460,0.4730113745,-0.1083832458,0.9850977063,-0.3806095421,-1.6842314005,-0.3037002683,-1.1704878807,1.0020384789,0.3830957711,-0.6013025641,1.1467078924,0.0981238931,1.7418649197,0.6734358072,0.3837615550,0.2948663235,1.7496265173,0.6189327836,-1.7904872894,-0.4210635424,0.9242000580,0.6071117520,0.6344060898,-0.7756280303,0.8779548407,-1.7009711266,1.8138630390,0.3526122868,0.1569912732,-0.2284603566,0.4965924919,0.1646198183,0.5903906226,0.6860850453,0.9792392254,0.8350661397,0.2694632113,0.5770038366,0.6414442062,-0.0007064881,0.1891670227,3.3849787712,0.3436114490,1.0508141518,0.8994019032,-1.4829807281,0.5896140933,-0.7613607645,-0.2228245735,-2.2727417946,-0.7928166986,0.8713169694,0.7702066302,-0.2003716528,-1.1520656347,1.3296504021,0.1453818381,0.1850291640,2.1702175140,-0.0070129517,-0.9936382174,0.7370295525,0.6775777340,-0.3520190716,0.2269107550,0.3641117215,0.2620734572,-0.3037872612,0.7699797153,-0.5900830030,-0.0836289078,1.4655891657,-0.4953507483,0.9964833856,0.6668241024,1.6373746395,-1.0739023685,1.5138012171,-0.9158896208,0.3178542852,0.1864966750,0.7493636012,-1.6645890474,-2.7454783916,-1.2306253910,-1.6678169966,0.2192802429,-1.4617450237,-0.8441195488,-0.5457106829,-0.5780082941,1.8181878328],[0.0347079597,-0.3133533299,-1.1121459007,0.7268883586,0.3151122928,-0.5868066549,-1.7393980026,1.4476518631,0.4207965732,0.6836621165,1.8285927773,2.0717456341,1.6458361149,-0.2653712630,1.2877157927,0.4153696597,-0.8352056146,-2.2649061680,0.0756139308,-1.1362289190,2.3471145630,0.6214581132,2.1959691048,-1.1375454664,-2.3548367023,0.2680402696,0.2878903747,-2.3156638145,0.7746370435,0.5787066221,-0.5267652869,0.7834333181,-0.6082605720,-0.2690319419,0.3098383546,-2.3032941818,-0.5287114978,-0.4524430037,0.5710779428,0.2465500385,-0.0645666569,0.3519504070,-0.5015529990,0.3956600130,1.3372106552,0.8897716999,-0.4648024738,-0.2677031755,0.2189188153,0.4962541163,-0.4791700840,2.3462817669,-0.5984350443,1.2644433975,0.1036138311,-0.6658110619,0.0362412110,-0.2366671264,0.1581058204,1.3862762451,0.6495568156,-0.2329453528,0.0977031738,-0.5559794307,-0.8441994190,-0.0818201452,-0.1046767831,0.1273758411,0.4763332605,-0.1459098011,-0.1045340598,0.7774121165,-1.0574012995,-0.3140885234,-0.1191266701,0.8644908071,-0.0673007965,-0.6035963893,-0.5750159621,0.1821621656,0.6175658703,-0.7133141160,-1.4616454840,-1.5156626701,-0.5073814392,-2.4346702099,-0.7047346234,0.9782006741,0.8120400906,-0.4308604598,-0.4996556640,-0.5441598892,-0.2324918956,-0.7524700165,0.4344911575,0.1049314365,-1.2540309429,0.8012621999,1.1741023064,0.1364507526,0.0647116527,1.0092266798,-0.9059830308,-2.1721153259,1.1897959709,-0.1393132359,-0.8553556204,1.7093845606,-1.1029998064,-0.1866924912,1.3107669353,1.1628562212,1.0969941616,-1.2107185125,0.5536623001,1.1172728539,-0.2946329415,1.9040652514,0.0030044951,0.0876778588,-1.6686580181,0.0381268524,-0.3123620749,-0.5600023866,-1.0734754801,-1.3627321720,-1.2353109121,-0.0829710588,-0.1355157346,-0.9162235856,0.3678059578,-0.2945487499,-0.4046669006,0.8659502268,-0.5342887640,0.8775099516,0.2651894093,-0.0377184190,0.7347764373,1.7639116049,0.0837946832,1.2767306566,0.5433607101,-1.1277854443,-0.4092319906,0.3871065974,-1.3732262850,0.0484902039,0.3826854825,-0.8100257516,0.5566810369,-0.0652687475,-0.5655643344,-0.7411604524,-0.2269089818,-0.8638867736,0.9950011969,0.2840652168,0.5898693800,0.1182757393,-0.6138235927,-0.4195107520,-0.6143563986,0.2658936679,-2.7899060249,2.2320573330,1.3054738045,0.5085213184,-2.8490052223,0.5092891455,-2.4805524349,-0.7394009829,0.6821433306,1.6671719551,0.4611730278,-1.3810774088,0.7877162099,1.0247899294,-0.4438346326,1.6174777746,-0.5054686666,0.1760485619,-1.3114635944,0.4309389293,0.4800304770,-0.1087075472,-0.5780231953,-0.7047863603,-2.1476862431,-0.2139531970,0.8352521062,0.8809152246,0.2713364661,0.3603095114,0.3651247323,-0.8552098274,-1.1180651188,-0.6216140389,1.9746416807,-1.6354407072,0.6205993295,-0.3181858361,-0.5416350961,-0.8335084915,-1.0118873119,0.4987774491,-0.3509887457,-1.1358166933,-1.7522422075,0.6429727077,-0.1995020509,-1.5381036997,0.1544527858,-1.2883707285,0.5439504981,-0.0020331419,-0.2399383932,2.7470574379,0.5867281556,-1.3848453760,-1.5909763575,-1.3414326906,1.2320175171,0.9906088114],[1.5391710997,1.7031075954,0.1598866582,0.5140388608,0.7190224528,0.2349463552,0.8574712276,-1.1896023750,-1.2586706877,0.2605868578,-1.1289823055,0.5695583820,0.5411376357,0.3698288500,0.3675069809,-2.2430713177,-0.6659237146,0.8024030924,-1.3820905685,1.0470803976,-1.4281877279,-0.3887173235,0.3055648804,-1.5657507181,-2.1697707176,1.4391497374,-1.5002782345,0.8511645794,0.5123485923,-0.0351729281,-0.1442015767,-0.3420092463,1.5379468203,0.9169666171,0.0578503907,0.2137975246,0.1961234212,-0.5389728546,-1.6089800596,0.4754639268,0.2233437747,0.1733196378,1.2684612274,-0.2865017951,-0.1468806267,2.4246311188,-0.0136099840,-0.1019738093,-0.6110169291,-0.8038156033,-0.0695337504,0.4967861176,1.9961833954,1.2340877056,-0.0171101298,0.0730238631,1.2438621521,-0.5428292155,0.5201815963,-2.0732119083,1.0970755816,1.7306132317,-1.7191520929,-1.0230888128,-1.6075240374,0.6967043877,1.0618695021,-0.3732878268,0.6697552204,-0.9198107123,0.7653980255,0.6345857382,0.3547610641,0.7504929900,-0.3338160217,-0.5197306275,0.9472274780,0.2227191478,-0.1130388081,-0.0185331739,0.5474200845,-2.2555432320,0.4503259957,0.9500397444,-1.8120361567,-0.9349394441,-0.0216028914,0.5007613301,-1.4258812666,0.9653455615,-0.2123300284,-0.9972632527,0.4599822760,1.4621566534,-0.9623032212,0.0821985975,0.7254318595,-0.8844993114,-1.1279693842,-0.0905174986,2.1163721085,0.5569891334,0.4357909858,0.1275379509,-0.3727503419,-1.5048179626,-0.9216700792,2.6693191528,1.8621063232,0.0340112634,-0.2996329963,0.1685769856,-0.0407568179,0.2233879268,-1.7338620424,0.4193114340,-1.7407432795,-0.3416656554,1.9146230221,0.7406899929,-1.3047709465,0.4453069568,-1.8995198011,0.1776600182,-0.1196291074,0.6559990048,0.4963692725,-0.4981673658,-0.2931072414,-1.5256278515,-0.3368673921,-0.0629800633,-0.3711330593,-0.4460727274,-0.1216611713,-0.7857735753,1.7829257250,0.5151084661,-1.0354783535,0.3958749175,-0.7858343124,0.3499429822,-0.4504361749,1.8571742773,-0.5264532566,0.8362266421,-0.6549919248,-2.2250118256,-0.3075691462,-0.7183296084,1.3063586950,0.2724380791,-0.7761501074,-0.3567362130,0.0337915644,-0.6230852008,1.2878644466,0.8142136335,0.7467980981,0.0545031875,-0.7823157907,-2.0375680923,-1.5598759651,0.7258884311,0.2529396415,-0.1873801351,-0.2569098175,0.9535322785,0.3202666640,0.9372274280,-0.7806515098,0.0057575130,2.1948187351,2.1471142769,-0.3790979385,1.2801339626,0.6664164662,0.8390047550,0.2793636620,0.3506715596,-0.8946903348,-0.1892145276,-0.2091954798,0.8307814598,0.7596340775,-0.6366750598,-0.2397034466,0.0246799253,-0.7540863156,0.6899034381,0.7105519772,1.9139291048,-0.9092925787,1.0814040899,-1.6866900921,1.2264014482,-0.0857531503,1.1177709103,0.0718116313,0.6214321852,0.8597166538,2.5825061798,-0.0952331051,0.2181185037,-0.2722103000,2.0228359699,-0.4731457233,-0.4897209406,-0.8466237187,-0.7927527428,0.2535848320,-1.7146792412,1.4753947258,0.1289210320,1.0652494431,0.8515291214,0.4173741341,-2.0765564442,1.2504419088,-0.0443437658,1.5116868019,1.8558517694,0.3491135240,1.6937165260],[-1.0338196754,-0.4067380130,0.8262225389,1.4143344164,0.8692837954,-0.2377124280,-0.3093731403,2.4539537430,1.8264842033,-1.1981890202,-0.0252850149,0.4000261128,1.0661538839,0.8577992916,1.7236268520,-0.3746747077,-0.6475515962,1.0553491116,1.0886489153,1.0331463814,1.4672757387,-0.2489519864,-1.0970305204,0.9412765503,1.0543107986,-0.0536026321,-1.0659681559,-1.0156000853,-1.5400930643,0.4246522188,-0.0187172014,0.8591113091,0.5056961775,1.0604348183,0.4011115432,0.6088192463,-0.4373769760,-0.4630948007,-0.1633478701,-0.0268928893,-0.9721285701,-0.0090062944,-1.8252615929,-0.8835651875,0.8288798928,0.9291929603,-0.6319452524,-0.7508799434,-1.2358655930,-0.2250781059,0.8632010818,-1.3284317255,-0.1848070771,0.9850524068,2.7465870380,0.3367196023,-0.1066792235,0.6596503854,0.5575050712,1.2563635111,1.6511420012,-2.0832810402,-1.7445597649,1.0956785679,-0.3525366783,0.7681734562,-0.8345808983,-0.4927855432,-0.7883468866,-0.7326194048,-1.2440841198,-0.0345288664,-2.7205607891,0.6942610145,0.7805992961,1.9402210712,-0.3773797154,0.6861529350,1.4874285460,-1.1206152439,0.4860053062,-2.2346808910,1.6082966328,-1.4920960665,-0.0897716880,-0.9688981175,0.2630314231,-0.2303248346,0.7283701301,-0.2108328491,0.8876338601,-0.0404112227,0.3710825443,-1.7251782417,1.6026358604,-0.5407242775,-1.5847551823,-0.2540806532,1.0504814386,-0.6451629400,-0.2521845698,-0.7335251570,1.4679886103,0.9870139360,-0.2953719795,-0.6325708032,-0.2677554488,-0.6401776671,-0.9570604563,0.7377889156,-1.1807512045,0.2154497951,2.0238044262,-0.0856862068,-0.9026969671,-0.7318025231,-1.8274935484,-1.2555172443,2.3201255798,-0.0519613996,0.1016197801,-1.6232725382,-0.1265503317,0.3801241219,1.2250181437,-0.8491208553,-0.0472059064,0.8237924576,0.0123016173,2.5513939857,0.4294205010,0.9571111202,-1.0746427774,0.7808172703,-0.0097871711,-1.0527443886,1.9884264469,1.4375798702,0.1371785998,1.1927306652,0.5296163559,0.7076142430,0.7082402110,-0.0101034651,-0.9877721667,-0.7654364109,0.8536295891,2.0100181103,1.7068982124,-1.0156911612,-0.3960444927,1.1031227112,-0.5324784517,-1.4763936996,-1.0825276375,1.8315578699,0.2433653921,-0.8337994218,0.4364992380,-0.0291025154,-1.1196365356,-0.6079728007,0.3150257468,0.9560776353,-0.4641246200,0.0915727168,1.4087785482,0.5882574320,-1.0789182186,1.1352727413,0.3621936738,0.7451221347,0.9481421113,1.1590278149,0.3194058836,0.2874425352,2.1013104916,0.5960393548,-0.9192075133,2.1953134537,-0.2614995241,-1.1423128843,-0.6494238973,1.2167494297,-0.4384835362,-0.4017018974,0.9502454400,1.7171781063,-0.3878672123,0.1367998719,-0.2946698964,-0.3433934152,-2.0157806873,0.3329694569,0.1684727818,-0.6111791730,-1.2262228727,-0.3766339123,-0.0963002965,2.6844952106,-1.4412274361,-0.3843735456,-0.6183712482,0.2814792395,0.4891471267,-1.6365194321,-0.5198187828,-0.4364172220,-0.1316254586,0.7158564329,0.9850757122,2.1309149265,1.7007933855,0.4094660878,-0.4367023706,0.1017921567,0.9821092486,-1.8584427834,-1.8049651384,0.3871385157,-0.7662376165,0.8493132591,0.8043100834,-0.7121513486],[-1.0511052608,1.5637362003,-1.4827083349,-0.5826970339,-0.1382329762,-1.4593697786,0.8752971292,1.5086021423,0.0051624221,-0.6862916946,-0.0043434026,-0.1981390715,1.6246167421,-0.7251117826,1.4283695221,0.3859604895,1.3972984552,1.5540999174,2.5077965260,0.6106916070,-0.2925674021,-0.6402983069,1.1182700396,1.3853303194,-1.2084366083,-0.0332357697,1.1627514362,0.3403298259,-1.4742140770,0.1268104017,0.0190874953,-0.2759317756,1.9947268963,0.3381502628,-1.2353051901,0.1884480715,-0.0715656877,-0.4184769392,1.6307207346,0.8701319098,1.6656786203,-0.2166256756,-0.8495945930,-0.9305421114,-0.2624989152,-1.0245912075,-0.3905347288,-0.8387540579,-0.7649152875,0.7450212240,-0.1490360945,-1.2048982382,0.0052565383,0.2848755121,0.2007502466,1.8125782013,0.6278348565,0.3501205146,-0.7780155540,0.2294713855,0.2744221985,-0.7625269294,1.2621465921,1.7222411633,2.0579175949,-2.1386463642,0.1349654347,1.3889241219,1.0150574446,-0.1360161752,1.1493418217,0.9397243857,-0.2867267430,0.1947177649,-0.4196768999,-1.1011104584,-0.4494042397,-2.1123704910,0.8957493901,-1.3408659697,-1.5817927122,0.5287882090,-0.4619643688,1.8015409708,-0.3952269852,-1.9611263275,0.7645410895,0.0038085454,-0.5192066431,0.6467928290,-1.2985756397,-1.0088950396,0.0489339791,0.7028077841,-1.9100688696,-0.2068070322,1.0011336803,1.4111698866,-1.7828242779,1.2758779526,1.0975503922,0.1588366926,1.1677552462,-0.1960469037,-2.4964635372,-0.5020163655,0.4433894455,1.3550033569,-1.8140020370,1.3322262764,-0.0272594821,0.5605358481,0.6819627285,-0.3883218765,-0.4051531553,1.9179229736,-0.1898309290,-0.4401783049,0.3061546683,0.5527454019,-0.0615562722,1.1755992174,-0.3579424620,0.6841399074,0.8924797177,1.0648961067,0.3632277846,0.5677139759,1.2489463091,-0.1964088529,0.6993002892,-0.3229619265,0.2260445952,0.9134680629,-1.1335862875,0.8838149905,-0.2615242898,-0.5182797909,-0.7718955278,0.6097289920,-1.0060567856,-2.0341730118,2.3055205345,-0.0929920897,0.8716686368,-0.9269929528,-0.2685537636,1.5324709415,-1.0759367943,0.1769119054,0.6477789879,0.6450597644,-0.0311753843,-0.3823924959,0.9851641655,-0.0099959550,0.3076856434,0.8313928843,-0.9556804299,0.9590794444,-1.6914818287,0.1404424012,0.1872563809,0.6698793769,-0.6741681695,-0.6608606577,-0.5262241960,1.7734227180,0.5527629256,1.2757896185,0.1524516791,-1.0622953176,-1.8591362238,0.6258513927,-1.0650826693,0.1954589635,0.0974331871,-0.5126291513,0.7299199700,0.3523078263,-0.3958586156,0.3189874291,-1.6703007221,0.1114622653,2.7657816410,0.3965610564,-0.3496138453,-0.9666339159,-2.0601592064,-0.5482038856,0.7389851809,0.7662830353,0.6029362679,-0.3516857326,-0.2998183370,-1.4955811501,-0.3838070035,1.3036487103,0.5547218919,0.3298881352,1.1064683199,0.9430986643,0.3839015961,1.4636781216,0.5495438576,-1.0143105984,-0.1891726404,-2.5100672245,-1.1935496330,0.2183128446,0.9599092007,1.5681704283,0.8648939133,-0.5642094612,-0.3421497941,1.4679930210,0.1009292603,-0.1126660332,-0.0972139761,-0.9727141261,1.3819562197,-0.2575739920,0.2227812111,1.6426393986],[0.5266095400,0.3665766418,1.4545422792,0.8404076695,-0.8013294339,0.9210820794,-0.6172357798,-0.4117927849,-0.7112870812,-0.2017239034,-0.0910126418,-0.8601288199,0.0719281286,0.7666918039,-0.0444614887,0.2221361250,-1.0537574291,0.6026690006,0.1948057115,-0.5820829868,0.9025380611,-0.1448483169,-0.1274862140,0.0873810202,2.6046392918,1.4802066088,2.0170738697,-2.2271463871,2.0235662460,0.1746399105,1.1856992245,0.3476963341,-0.1823394597,-0.9577738643,-0.6233409643,-1.0143755674,0.8863685727,1.2911611795,-0.7955857515,1.9125989676,-0.7551600337,-2.2882008553,0.6161273718,0.1451403648,-0.7093462944,1.2617323399,-0.6054198146,0.2747418582,0.4786858857,0.6184700727,0.0912145898,-1.2048335075,0.3290942907,-0.2360955030,0.5835716724,-0.9597758055,1.7403790951,1.5781999826,-0.7483271956,0.5618293881,1.2583650351,1.1227834225,-0.4328562021,0.1996002942,-2.6929018497,-0.8850705624,0.3370223641,1.0766296387,0.2751036286,-0.0249971207,0.3484513462,-1.4661811590,-0.5508713126,0.7615688443,0.2132720649,-0.5305895209,0.5651596785,-2.1194589138,1.5398962498,-0.1179601401,-0.0704612285,0.8395457268,3.8448247910,-1.6787865162,1.2805011272,0.5926063657,1.0369732380,-0.9114980102,0.7235304117,-1.1771003008,-0.8924003839,1.5978136063,-0.9991945624,1.4228422642,-0.2267849296,-0.8862344027,0.7951184511,-1.0446032286,1.5943152905,1.0219110250,-0.5544071198,0.0632716194,1.3749527931,-0.3679391444,0.8503480554,-0.5020872355,0.6588804126,-1.8511382341,1.3213031292,1.8603336811,0.5004802942,0.3071632683,-0.2847483456,0.2404857427,-1.5341200829,-0.3211204112,2.2744309902,-0.1365518570,0.8923602700,-0.3667230308,-0.5734806657,-0.3486274481,-0.1186143011,0.8307850957,0.6011481285,-0.7103462219,-0.6508090496,-0.8464833498,0.6043699980,0.2763639688,-0.1737786531,-0.0291063003,1.4368228912,-1.5170634985,-0.9887233973,1.1650506258,-0.8643849492,1.2665343285,-1.3787747622,0.3148807883,-0.8181608319,1.1779116392,-0.7715457678,1.0275965929,-0.1303185225,0.1004789919,-1.2106908560,-0.0878942758,0.1088947505,0.6320188642,-0.9480621219,-1.3948607445,0.0503652915,-0.3959770799,-0.1595087647,-1.2462719679,-0.8731278181,-0.3450042605,1.7942147255,0.4963313341,1.0791658163,-0.0937815085,-1.0531029701,-1.2803995609,-0.3876163363,0.3790248632,1.1142001152,-0.4398929477,-0.8452531099,-0.3301915526,-0.3507989347,-0.9891392589,-1.3544497490,0.9459623694,0.2598392963,0.5682955980,-0.9399594665,-0.2605684400,-0.8619480729,-0.5104118586,0.5194774270,-0.9169836044,-2.6900908947,2.7366669178,-1.4323606491,1.6291747093,-1.4959498644,-1.1548160315,2.5480496883,0.2900446653,1.4242911339,-1.2634080648,0.1963342428,1.4480313063,0.3778089583,-0.6415191293,-0.2120681107,-1.3008939028,0.3545155227,-0.2053026557,-1.9529192448,-0.0525757596,-1.0794268847,0.6703519821,-0.4758709371,1.1867605448,-0.3449138403,0.3547198176,1.6804623604,-0.9776315093,-1.9091552496,0.6955775023,0.9033918381,-0.1566496491,1.4611063004,-0.8640043139,-0.9245055914,-1.2249439955,0.7369412780,1.0058851242,0.5710602403,0.1333852112,-1.2250558138,-1.7569497824],[-0.5837014914,1.0665876865,-0.8548253179,1.4985412359,-1.5736359358,1.4790285826,-0.5080267191,-0.4287172854,-1.9432892799,-0.6099470258,-0.2015367150,-0.8354988098,0.1913418621,1.0483424664,-0.2596043646,-1.7170075178,1.5210067034,-0.6716116071,-0.0163926631,-0.4215934277,-0.0219555702,-0.7757620215,-1.5904581547,-0.2381323576,-0.9930461645,-0.7130764127,-0.0591275357,1.0727508068,-0.3543343246,3.0782167912,1.2767871618,0.7470397949,-0.9672718644,1.4354689121,-1.2770084143,-1.4952831268,0.6237402558,-1.3215762377,-1.4172451496,-1.2935512066,0.5550708175,1.3703895807,-0.5524062514,0.4399808049,-0.4097305834,-0.3165017962,0.3495762050,0.7616158724,-1.1521909237,-1.0199534893,0.7851632833,-0.7123095393,-1.3383508921,-0.6731940508,1.0400739908,-1.7057328224,-1.6736978292,1.1462876797,0.6484333873,-0.7210823298,-1.1814935207,-0.0193868037,1.2531992197,0.1176470742,0.1281784475,1.0190414190,0.3068197668,0.2843888700,0.6067525744,-0.4399292171,-1.8810433149,-1.7169803381,0.1544450819,0.3690387607,0.1635134518,1.8912612200,-1.0243716240,0.8578745127,-1.1951819658,1.0820078850,-0.0740952268,0.4703108072,-0.4310616255,-1.6714330912,-0.6286044717,1.5786759853,-0.2143421322,1.0809478760,0.4545911551,-0.6744666100,0.0244885739,0.5634146333,0.1647529304,-1.3875241280,0.2178342938,-0.0701511055,1.8721164465,0.2166976780,0.5953648686,-0.3440511525,-0.3522693515,1.8573738337,-1.2786791325,-0.6153733134,0.7561426759,-0.5053442717,-0.0918319449,0.5894739628,0.1524143517,-1.6383651495,-0.4843915999,-0.2537766397,1.3841521740,-0.7449327111,0.0421048775,1.2323815823,-0.7230997086,-2.1666941643,0.0070414278,-0.3546690643,-0.6129224896,0.5168722272,0.0161491167,-1.3249355555,0.2833559513,0.2064211667,0.6386342645,-0.2798440158,1.6235107183,-1.9121247530,-1.0733910799,0.7745764256,-0.2634502351,0.1581316441,-0.2726663649,0.2395518273,0.0382109508,-0.9823060632,0.2630215287,1.1353650093,-0.9041807652,0.3829956353,-0.0649039596,-1.3555053473,0.7701690197,-0.8610923290,0.8052645326,-0.8781912923,-0.7311577201,-2.9039270878,0.0947883651,0.4915755689,0.6707758307,-0.3843032420,-0.0357921906,-0.5924118757,0.9842528701,-1.4029403925,1.2588713169,2.0025391579,-0.3116514683,-1.9610450268,0.6682580709,-1.6717112064,-0.5060207844,-0.3277585804,1.4305614233,-1.0951304436,0.2470769286,-0.0456573442,-0.3003408611,0.5671326518,0.0664090812,3.3978607655,0.1929901093,-0.3330067396,0.0479260944,0.6180526018,1.0526547432,-2.1311256886,0.1027629226,0.8785018921,0.0608107410,0.4887917042,0.6694629788,0.3460426033,-0.4522186220,-1.4803252220,0.0475699566,-1.0768768787,0.2929372489,0.2556186914,-0.4252198935,-1.8915692568,-0.2100862265,-0.0845040902,-0.7917732000,-0.4677768350,-0.4222314358,1.2808054686,-0.3017964065,-0.1799859554,-2.1371099949,-1.6400431395,0.5384729505,-1.4935162067,1.4454184771,0.4328987002,-0.4091602862,1.2936794758,-0.7945672274,1.1871507168,1.5372850895,-0.8579943776,-0.7523997426,-1.3768255711,-0.5170516372,0.5266361237,0.5500448346,0.2686732411,1.1339052916,0.1505801380,1.0301640034,-0.5608243942],[0.5116033554,-0.1938695461,-0.2318379730,-0.6758121252,0.6146999598,0.7851496339,-1.3886620998,-0.3124918342,-0.1945108175,1.3500403166,0.5355877876,-0.6446334124,-0.3353732824,0.1246494949,1.4483478069,-3.0476114750,1.8138883114,-1.7988743782,1.6526031494,-0.1701273024,1.5951917171,-0.4962559342,0.5138649940,-0.1693724394,0.2989767194,0.9507718682,0.1399330646,-0.2367779166,-0.5848656893,0.0408673994,1.1533704996,-0.1025816724,-1.8772822618,0.7537982464,-1.0198754072,0.0566727556,-0.8193967938,0.6501716375,-0.6657895446,-0.7310741544,-0.4085451663,-0.9486625791,-0.0811254904,-0.2931586504,-0.2771906257,1.2176634073,-0.7611948252,0.5342000127,0.2206405699,-0.3270626366,1.6594908237,-0.9000232220,-1.8313491344,1.1096508503,-0.1459466666,-0.6582320333,0.3887246549,-1.8797705173,-1.1226110458,1.8616946936,-2.1927931309,0.0301647671,0.0473821871,0.6955246329,-0.5873502493,1.2350827456,-0.7400141358,1.6582258940,0.8974621296,-0.8383839130,-0.1603887081,0.4101619422,2.2152121067,-0.6797272563,-0.2635640800,-0.3556258082,-0.7574476004,-0.1034703329,0.2876872122,-1.0428086519,-0.5722566247,1.1083568335,0.5434480309,1.1623055935,-0.1919558942,-1.3977537155,0.3964772224,-0.3976658583,1.0213907957,-0.6081345081,-0.1984753758,0.7719884515,1.6116856337,-0.5647926331,-2.0718548298,1.2286254168,-1.5091139078,-1.1186282635,-0.2798033059,0.0749905184,0.3844977021,-0.0438706018,-0.2827286422,1.2734726667,1.2332676649,0.5621038079,-0.1019901857,-0.4522658288,-1.5339000225,-0.7295792103,-1.2359548807,-0.5382317901,1.6926423311,0.0574849769,0.5758835077,1.9828463793,0.5544055104,0.1886905134,2.5439355373,-0.3392686546,-0.6245530248,-0.4217739999,-0.1226378530,-1.1134990454,0.3403982222,-0.2209866643,-0.0216140505,0.8030695319,-0.5919985175,-0.0856432617,2.3269383907,0.2625982463,1.1156468391,0.2020006925,1.2754621506,-0.8897251487,0.9669903517,-0.5493559241,1.4233219624,1.3478831053,0.4568857253,-0.0930040702,-1.2832940817,0.2640421391,0.6426716447,1.6632509232,-1.4980686903,-0.5740880966,1.1556472778,1.3272485733,-2.1616599560,-1.6268901825,-0.5095205903,-0.1517010182,0.0697079748,0.6218604445,-1.7932162285,-1.5523602962,-0.2364832610,-0.2116670161,-0.9998782873,-2.7679753304,1.1789580584,1.2540689707,0.2540247738,-0.4325055182,-0.3744247854,0.5653214455,-0.2487467080,0.9951139092,-0.3737806082,1.4783589840,-0.4989001155,0.1346275359,0.4608412683,1.2888094187,-0.4664494693,0.9327733517,0.6842293739,0.4536570907,0.7156196237,-0.8881968260,2.7164895535,0.3652488589,0.6087380052,-2.6499264240,0.4615913928,0.5398000479,0.6077411175,0.1409384161,-0.6123158336,-1.3663132191,-0.8288936615,1.7153819799,-0.5907678604,0.9421477914,1.1129484177,-1.4384629726,0.5338668227,-0.7236166000,-0.3594409227,0.3317614794,-0.0182189792,-0.9622032046,1.2848628759,0.0738611221,1.8387027979,0.9678576589,-0.5062652230,-1.4060206413,0.7187169194,-0.0001753581,-1.2740622759,0.3961002231,-0.4440979064,0.2215769291,1.1455968618,-1.5288711786,-0.1165431365,0.1862287372,0.4621602297,-1.2539870739,-1.9266433716,-0.3532809019],[-1.2628905773,0.6617218256,-0.4527367353,-0.2476387024,-0.3344658315,-2.1655943394,-0.0272593759,1.5402753353,-0.8876109123,-0.5740729570,1.4609684944,0.8156918883,-0.6739629507,-2.1460916996,0.1425817907,1.2147607803,-1.1786000729,-0.6290073991,-1.2190272808,1.9812610149,-1.8409180641,-0.5344301462,0.0046728416,1.4701166153,0.0316930152,0.6217797399,0.3374963999,-0.0923807099,-1.0196141005,0.2970777452,0.5648963451,0.0846193135,-0.5526602864,-1.0114645958,0.0666290522,0.7835562229,0.2549487650,-0.5999336243,-0.9358597994,-1.2035146952,0.0867796391,0.2015919834,-0.6562302709,0.5852459669,1.0319597721,1.6525110006,1.0681165457,-0.3666232228,0.4162625074,-1.4362779856,0.9617007971,0.2182102948,0.4531165659,0.0359582119,0.2627207041,-0.6436700225,-2.5046739578,-0.1782875061,0.6615344882,-0.8519901037,0.2686338723,-0.6474667192,-0.1683191508,0.6466437578,1.4409619570,-0.2617304027,1.9130183458,-1.5077996254,-0.9462426305,-0.2838377357,0.0526938327,-0.6236193180,-0.4251126945,-0.0550731048,-0.8132697940,-1.2440909147,-1.0892269611,0.3136571050,-0.0490135439,-0.6237413883,0.6349942684,-1.0433175564,-2.0065948963,-1.9424735308,-0.6845101714,-0.6202213764,-0.3914147615,2.2720336914,0.0739588216,2.2880032063,0.5641351938,0.4740304947,-1.8853971958,0.4114308357,-1.1214060783,0.4880087078,0.4406731129,-1.2576910257,-0.6754882336,2.2553937435,-1.3919758797,-0.6938131452,-0.0537123717,-1.1831597090,-1.1901445389,0.6069281697,0.8246663213,0.3340826333,-0.2523243427,0.0757556856,-0.2096677721,0.7798483372,-0.8629242182,1.6895529032,0.1870408505,0.3471640348,1.1528530121,-0.5211679339,-0.3400958478,-1.8149099350,0.1189164966,0.4202474952,0.2755539119,0.3063024580,-0.1489593983,-1.3223544359,-0.1619068384,0.9510185719,-0.0225095376,-0.8107722998,-0.3870677650,1.5546485186,0.0615551658,1.6332503557,-0.7618871331,0.9830802679,1.2946121693,-1.3857856989,-1.5067552328,0.2777404487,-0.4606186450,-0.4907822013,1.3923071623,0.6343035698,0.9364335537,-1.7004997730,-0.4155659676,-0.1267781258,0.3403526843,-0.5315839648,-0.7173085213,-0.0580628328,-0.4464083612,1.3359794617,0.0476373993,-0.2086803317,-0.1372452974,-0.0591464713,1.3616489172,0.0903304443,-0.3211252391,-0.9177269936,-0.9550664425,0.3909903765,-0.2268156558,-0.0082257940,1.2424693108,-1.3258881569,1.4352700710,0.7880265713,0.1414163709,-0.2054441571,-1.1014525890,0.6595254540,-2.3181002140,-0.0876651481,-0.6974890828,-1.4716603756,0.7876362205,0.1344156265,-1.3554036617,0.6036042571,0.1190322861,0.9254586697,1.2508783340,0.2331293225,0.4055940509,0.7998869419,-0.5757550597,-0.9135111570,0.7675039172,-0.7781230807,2.4180750847,2.7852742672,0.1045434698,0.6787592173,0.4007455409,-0.3564361036,1.5347712040,-1.4705452919,-0.7479464412,-0.9566254020,1.2649875879,-0.1646696180,-0.5155603290,-0.7206394076,2.1404452324,-0.7790234089,0.7116395235,0.9158282876,-1.4436525106,-0.4211129844,0.7956809998,-0.9565706253,-1.3517844677,-0.3119147122,0.2453809679,1.4581965208,0.7543629408,1.6049604416,-0.4158651829,-1.2043303251,-1.0034831762,-1.1723743677],[1.0647040606,0.2321242392,1.0505148172,0.8940162659,-0.1205676943,0.3933456540,-0.4981866777,0.8221628666,-1.1024289131,0.9230841994,0.1395951062,-0.2751727700,-0.4984712899,-1.3625754118,-0.4306615889,-0.5384787917,-0.9555833936,2.1937851906,-1.1307214499,0.4357461333,-0.8771927953,0.8578191400,0.3055385947,0.2204068154,0.6302809715,-0.0163004622,-0.8319802284,-1.3126729727,-1.3170480728,0.5569218993,-0.3531013131,0.3641794920,0.3810200691,-1.4238847494,0.1009590551,-0.3800279498,0.7649206519,-0.6368266344,1.7794678211,-1.1880887747,-1.4316164255,-1.6969466209,0.5561639667,1.1471374035,0.9899902344,-0.1633966267,0.6074290276,0.5961022377,-2.8764624596,-1.0727797747,-0.8768104911,-1.0192153454,2.8190350533,-0.2256239206,-0.3943177760,0.8255993724,-1.1556965113,-2.1723108292,0.7178436518,0.6824865341,0.5922962427,-0.3120145798,1.0898365974,-0.3011100590,-0.1291559339,0.4131253660,-0.3478083014,0.3667017221,0.3330966830,0.3752444685,-1.4574064016,-0.2556218207,-0.3968566358,0.5036401153,-0.9107782841,1.3879652023,1.2289830446,0.7992539406,0.6849098802,0.0707288608,-0.0696446821,-0.1699579954,0.4269837737,-0.5689053535,1.0672354698,-0.7523844838,-0.6921916008,0.7129670978,0.0622992143,-2.1855492592,-1.0172541142,0.6609465480,-0.2331690043,0.2233446985,1.7745434046,0.0969029590,1.6294189692,-0.6170974374,-1.2387092113,-2.4741020203,-0.0289155487,0.2518738508,-0.1491267532,-0.3006697595,-0.9428971410,0.3010059893,2.3177731037,0.1611628830,0.6096910834,-0.5344636440,-1.1325211525,0.5708059072,-0.2217664272,-0.1881082803,-0.2953444421,-2.7062401772,-0.0275921281,0.9700416327,-0.5384776592,1.4731795788,-0.6910365820,-0.8083460927,-0.2849730253,-0.9540877938,-1.0074419975,-0.3939363658,0.8493119478,1.3113639355,0.1255604029,-0.5570324659,-1.1712133884,-1.6394242048,1.0471559763,-0.6554031372,0.3733699918,0.8540666103,0.1184398606,-0.1343704611,1.3181189299,-0.0528781712,1.9076501131,-0.1207094342,1.0920281410,0.5713807344,1.3109537363,0.7840664983,-0.7930327654,0.7855635285,1.5472874641,0.6432893276,-0.0242837500,-0.3870694935,-0.5971293449,-0.2899690568,-0.7334938049,-0.0300292429,-0.9743447304,1.1392588615,-0.9915491343,-0.8986487985,-0.8678231239,-0.6919215918,0.4460268319,0.7036303878,-0.3525555134,0.2887427211,-0.7344346642,-1.1392246485,-1.1486465931,-1.1256231070,0.2172904909,2.2096951008,-0.2768126130,-0.4194982052,0.0895349309,-0.4994725883,-0.4294939935,1.5925871134,0.9207655787,0.1646861136,-0.3742833734,1.3894538879,-0.4878883362,-1.2071138620,-0.1191699728,-1.4390913248,0.2861081958,-0.1579031646,-0.3974367380,0.1752463430,0.3748835325,-0.3617827296,0.8169750571,1.3362919092,-0.1484031230,0.8671530485,-2.8536005020,-0.0906554833,0.0088043632,1.6732629538,2.1423466206,-0.7370496988,-1.2664498091,0.6641715765,1.2869904041,0.6079743505,1.3022139072,1.3500844240,-1.5029324293,0.6907622218,-0.4061397612,-0.8483211398,-0.3681233823,-0.8578148484,-0.1548754722,-0.5499112606,0.2328155786,-0.6580201983,-0.0703464821,0.5699104667,0.1111837327,0.7781289220,-1.5467842817,0.5690723658],[0.4452523887,0.0673167929,-0.0463277884,-1.8743733168,-2.8230288029,0.5497969389,1.9723832607,0.5564820766,-0.3615648150,0.0729349479,-0.0699678808,-0.7118794918,1.3303546906,0.0751053691,0.5226886272,0.9767999053,-1.7849285603,1.5602053404,1.0742962360,-2.6806015968,-0.4691926837,-0.5361781120,-0.2826193571,-2.2090110779,0.9901263714,-0.0261297319,0.8651404977,-0.2917624712,0.4948405921,-0.7538119555,1.5201109648,-0.7561060786,1.1786170006,-1.1826119423,0.7336524725,0.9188173413,-1.9997373819,0.4963880479,-0.8404771686,0.4128607512,0.4337054193,0.2035663426,-0.7117573619,-1.7409338951,-0.5598160028,-0.9008379579,-0.4741972387,0.0416170508,-0.7776945233,-0.2547941804,1.1234530210,-0.3588818014,-1.5244874954,1.1329869032,1.7894061804,0.7349005342,0.1345884353,-0.5785191655,-0.1267298460,-1.1764208078,1.3696135283,0.4106918871,-0.0976169482,-1.2190376520,-0.9109613299,-0.0955943689,-1.0170726776,1.0108891726,0.1099565029,-0.7064110637,0.2942879200,-0.5495336652,-0.4723607302,0.5777610540,-0.7016943693,0.7423120141,0.0810565427,0.8770937324,-0.1352943182,0.6857259870,-0.5274185538,-0.3148475289,0.4312712550,0.8180384040,-0.1410973966,0.5621446967,-0.8120179772,-0.0752384365,-0.0236309376,0.9926939011,-0.5143352747,0.1516641676,0.7576454878,0.1885048598,0.8092653155,-1.4317138195,0.7512933612,-1.7853584290,-0.3403401673,0.5114456415,-1.0479772091,-1.7917723656,-0.7522316575,0.9491505623,-1.5872008801,-0.1898578852,0.7266498208,-0.9619154930,-0.1413860023,0.9052723646,-0.3882459998,0.1237693802,-0.2326326221,0.3206429183,-1.3629474640,1.1452333927,0.7874519825,0.4181374013,-1.5147526264,0.0730068237,1.3311444521,0.7093572021,-0.9712299705,2.1352322102,1.0231037140,-1.1636049747,1.0286316872,-0.5036383271,-1.6746673584,0.4073672295,-0.0182898492,0.1493997425,-0.7960625291,-0.4407243431,2.3877747059,0.4815349579,0.5813381672,2.2279050350,0.9356135130,-0.4409787059,-0.8286496401,0.7311822176,0.2700185478,0.5507662892,-2.5871624947,0.6772342324,0.2074013352,0.4615665972,-0.4190964699,0.4770605862,2.4217619896,-1.9182502031,-0.8544636965,0.8381118178,-0.4485734701,-1.0919756889,-0.2648122609,1.0098994970,-0.5472998023,1.2348033190,-1.4701932669,-1.9155840874,-0.1090629399,-0.1855992675,0.1095966473,-0.6782081723,0.1180862114,-1.2295197248,1.5274536610,2.3235061169,0.2913398147,-0.3180084825,-0.9371327162,-0.5686488748,1.8743900061,0.0845806673,1.3588453531,0.1544833183,-0.4382197857,-0.6617422104,-0.2704090774,-0.0473123975,-0.1375723928,0.5586237907,-1.4751049280,-0.3098645806,-0.8870976567,1.3249599934,-0.2478749007,-0.4527670145,1.0433957577,-0.4988560379,2.0353519917,0.1222634986,0.8268195987,-0.2070628256,0.4064068496,0.7379081845,-0.1530139744,1.1486705542,0.7987783551,-0.0146313990,-0.8424440026,0.2351379693,0.5839104056,1.9307159185,-0.1338217556,-1.2729601860,-0.9533520937,1.4415456057,-0.0951500610,0.3972956836,0.2098439634,0.5354406834,0.8776354790,0.5473576784,-0.2294035405,-0.8871148229,1.8933590651,0.1261401772,-0.5996055007,-0.0468349196,-0.4060138166,0.0416313224],[0.0498172343,0.5637643337,-2.2129881382,-1.1236852407,-0.1304418892,-0.9155464172,-2.0547239780,0.6958734989,0.0104737310,-0.0415819399,0.4621022642,0.0875720531,-0.5839158893,-0.1784689277,-0.4025821686,0.9536439776,-0.1893357337,0.4697566330,-0.8019378185,0.4339478314,-0.4863450527,-0.7643893361,-0.4413174093,0.7450090647,0.4705619812,-0.5967475176,0.2144325823,-0.0487319790,0.4239455760,-1.8733527660,-0.6309217811,-1.3656258583,0.5353654027,0.3123120070,1.3560433388,0.0231874250,-0.7732568383,-1.4160184860,1.1718209982,-0.6379296184,0.9241999984,1.1896660328,1.4317412376,-0.1303557307,0.5576506853,0.7538163662,0.4692847729,-1.0503007174,2.2192955017,0.7662305236,1.4199377298,-1.1396961212,0.6984863877,0.8586908579,-1.4875825644,0.1292233914,3.3051817417,0.7467623949,0.8142894506,0.5744169950,1.0687381029,-0.8679308295,0.2648360729,-1.8214925528,-1.5498822927,1.0336124897,-0.3167379498,1.6213614941,0.0680482686,-0.0182913449,0.0870022327,-0.7723815441,1.0164847374,-1.1489291191,0.3398660421,0.4741336107,-0.1662758589,0.0326907784,-0.3206206262,-2.1174454689,0.3675934374,0.3425858915,-0.5936652422,-0.7317870259,-0.8706053495,0.3625514805,-0.7185558677,-0.3813236952,1.6017394066,-0.4086554348,1.1893210411,-0.3677315414,-0.8534657359,0.2419654280,-0.2601073980,-1.7017725706,-2.0002150536,-1.2726695538,-2.8272268772,-1.3180217743,0.2695805430,0.8193210363,-0.0601939671,0.2057047337,2.1258807182,-1.0387762785,-1.0656303167,0.8474498391,-2.0752680302,-0.7518011332,-1.0824229717,-0.0520494208,0.8673002720,0.7077292800,-0.6896764636,0.5508205891,-0.5087099671,-0.7953282595,-0.7552286386,0.3783166111,0.1480294019,-0.4080043733,1.3215683699,-0.2240578383,-0.9652453065,1.6936491728,-0.5004338622,-0.4313458204,1.8143146038,-0.8686926961,1.2914714813,-0.2357162833,1.4363687038,-1.2961885929,0.1975327134,-0.8971417546,-1.4519535303,-0.4755281806,1.1832439899,0.5823754072,-0.7990812063,-0.1475500166,-1.5841269493,-1.0030679703,1.5403479338,0.8109509945,-2.1969048977,-1.9337809086,0.9102554917,-0.2021962404,-0.8042155504,-0.3982497156,2.4267811775,0.6462002993,0.5270777345,0.2776626945,-0.3879787624,-1.2539374828,-1.0608892441,0.8798408508,-1.0080276728,-1.2297723293,-1.1807438135,0.7539607286,-0.3988789618,-1.2075831890,1.9517722130,-0.0741142184,1.6240398884,0.2602418065,1.8968186378,0.9252575040,-0.6450889707,1.4229569435,-0.1587112993,0.8568664789,-1.0669229031,0.6148740053,0.7432900667,-2.3107280731,-0.7096374631,0.1439763308,-1.7630770206,0.8053269386,1.0544917583,-0.2164501399,-1.1995971203,-1.1014965773,-0.6774827838,-2.0162796974,-1.3040640354,1.7306530476,0.4679165184,1.5393271446,-1.1890480518,0.1019532531,-0.9641910195,-0.6676917076,0.3440614343,-0.2543410659,0.5562074184,1.3031641245,0.3274115324,-2.5846605301,1.2806965113,1.9282523394,-0.1295706928,0.4334105253,0.8793066144,-0.5240471959,-0.1192059442,0.2685222328,0.6581997275,0.3720701039,1.0289461613,0.6471891999,-1.3104859591,-0.7705357075,-0.6850705743,0.2172274739,1.4807068110,-0.2180891037,-0.6771414280,1.5424208641],[1.0833897591,-2.8045880795,0.4607556164,0.7872321010,1.3725421429,-1.5756440163,-0.3378772438,1.1134512424,-0.0857403502,-0.7604584098,2.4517266750,1.1246706247,-1.8834089041,-0.3396808505,0.4468193650,-1.3754568100,0.2401239425,0.2437317073,-1.6011885405,-0.6616942286,0.3145923913,-0.5798115730,1.6361236572,-0.8283223510,-0.0387554541,-1.4568865299,-0.9266756773,-0.6654282808,-1.0409642458,0.1891269684,-1.0281019211,-0.6576402783,-0.2294373065,0.3463232517,-1.1547359228,-1.8444094658,0.9560483694,-0.6993877292,0.3474283516,0.2573563457,-0.5045565963,0.6014789939,0.0792550221,2.0729916096,0.5574637055,2.2566828728,-0.7126176953,-1.8690760136,1.1867282391,-1.2382060289,0.5142056346,0.9849852324,-0.9336848855,0.1257813275,0.0927541032,1.3971587420,-0.3119037747,-1.1926451921,-1.2881635427,-1.3171620369,0.4146163762,-0.4774722159,-0.3868423402,-1.0103545189,-1.1745402813,-0.2091901600,1.3277802467,1.1256504059,0.2741110325,-1.3244829178,1.5284399986,-1.0842255354,-0.8799241185,0.4729141891,-1.9505088329,-0.4351518452,-0.8081485629,0.3269541264,-0.7062594295,1.6481834650,0.4895448983,-0.9180828333,-1.4240669012,-0.1890579164,-0.9585830569,-1.0736232996,-1.2009921074,0.0086217457,2.2900185585,0.9107310772,0.3459389806,-0.2933109701,1.7081165314,0.7132967114,0.1095507592,1.3954622746,-0.0916409269,-0.4428087771,0.3978556991,-0.9287421107,0.6775330901,-2.0309498310,-0.2861202955,-1.3187100887,0.6868452430,-0.1280173510,0.3954797983,-1.4868066311,2.6867311001,-0.2837397456,-0.4692958593,0.8649511933,-0.4000707567,-1.3636385202,-0.4336688221,0.4807041883,-1.1696223021,0.4139298797,-0.3658573031,-0.8346003294,-0.6913728714,0.6472249031,-0.9495275617,1.0565907955,-0.1551950276,0.0402307771,0.5411900282,1.9618538618,0.3323952258,-0.1880543530,0.3495495617,-0.2323200107,-0.0774672851,1.1608976126,-0.6110397577,-0.0833180025,0.5343641639,-0.1402324885,0.7121111155,1.1572567225,0.6025365591,-0.0661304966,0.9814050198,-2.0910024643,0.6004716754,-0.8897766471,0.5802534223,1.2903648615,1.0397731066,-0.8031101823,-0.8996082544,-0.2524068058,-1.2267825603,-0.9092355967,-1.0889654160,0.8441944718,0.8110027909,-0.0405239128,-1.0715160370,-0.1124812588,-1.5997368097,0.1420539320,0.6181820631,-1.4807455540,0.2736857831,-0.0874448717,1.4611581564,-1.0566525459,0.4870162010,-0.0450987816,-1.0364639759,0.7334445715,1.5605502129,0.7317325473,-0.8760612011,-0.3242733181,0.5894088149,0.5113537312,-0.2457536161,-1.0911314487,1.1978468895,1.4237911701,0.2477284819,-0.2938754857,0.5031723976,0.0124401199,-0.4126176238,-0.7320761085,-0.4052883387,-0.2108996511,-0.7781730294,2.1468038559,2.6817145348,0.2516969740,-1.0822778940,-0.0529402457,0.8459635377,-1.2501021624,-1.1299482584,-0.7857146263,0.1984169632,-1.6308257580,0.9140481949,-0.2152125388,-0.2536766827,1.7120695114,-1.6373603344,-1.5643184185,-0.4590578377,0.4787595272,0.0577515550,-0.8291638494,-1.4593600035,-1.9532253742,-0.7408310175,2.6602454185,1.3658504486,-0.4614217579,2.1605172157,-1.6152997017,0.0586317293,-1.2505980730,1.0217580795,0.0332866982],[-0.1231489927,0.8719854951,0.3502269387,-0.3011886775,-0.3683407009,-1.3110753298,-0.5376300216,0.3520507216,-1.2554615736,-0.3561529517,-1.4860554934,2.0891554356,0.4023406506,-1.2871041298,0.4452855885,-0.3306802511,-1.0909045935,1.2342441082,-1.3968569040,1.2671681643,0.3808578551,0.1622464359,-0.8135055304,-0.2238749415,0.1271271706,0.3600368202,0.1486675888,-0.3422999382,-0.9236380458,-0.1554073095,0.7903410792,0.1860257089,0.1308351606,-0.2578380704,1.5718585253,-0.7012122273,1.0408076048,-1.0936492682,0.8577693105,0.4340395927,-0.2430948913,0.2685880661,-1.8785663843,0.9641314149,0.3043458462,-0.6693259478,0.6128401756,-0.5462137461,2.3161671162,0.0049466984,0.0762785673,-0.8627022505,-0.1909276098,-1.7148029804,0.8741716743,-1.0369155407,-0.9876949787,0.0601231158,-0.9681298733,-0.8266518116,-0.0769524872,-0.5266578794,0.6762766838,1.3591698408,0.6121618748,-0.3233613670,0.0347821042,0.8331773877,1.1339416504,-1.0403983593,0.1039372236,0.3434944749,0.7209177017,-0.6197436452,-0.6891433597,-0.7516368628,0.9467796683,-0.2635918260,0.6381617785,-0.5782182217,-0.3497321010,0.9256712794,-0.6284171343,-1.8411649466,0.2277844101,-0.4046567082,-0.2888169885,0.7736723423,0.7876480222,0.9130166173,2.5497612953,1.3560099602,0.5737367272,0.2998895347,-0.7253481150,1.6977071762,0.1428591907,1.6922923326,-0.7899493575,-1.6153711081,0.8779649734,1.6200410128,-0.7291763425,-1.0820019245,0.7858555317,0.2406725436,-0.2910236716,-0.0595621802,-0.4121816158,0.3560032547,0.9713022709,-0.3304999173,-1.7688531876,1.1638083458,-0.9774751067,0.4061269462,1.4668464661,0.2371457219,-0.1761687845,-0.2912491560,-1.8847373724,-0.6073658466,-0.5818338990,-0.3286408484,-0.6674704552,0.3853674531,-1.2600563765,-0.0117033292,-1.2790175676,-0.0959654301,2.4882016182,0.3786404729,-0.7029803395,0.7567839026,0.1165871844,-0.1981320381,-0.4176677465,0.7999218106,-0.3878305852,2.3828504086,0.4363039136,-0.3107819259,-0.3244489431,1.2963690758,-0.5949802399,-0.3192541301,0.1184017360,-1.0748354197,-0.2322329283,0.0255772974,-0.1782385856,-0.4525050223,-0.9569429159,1.7335696220,-0.5961167216,-0.9838139415,-1.5097740889,-0.5643052459,-1.0459479094,2.0024902821,1.1414196491,-0.7664759755,-1.2552784681,-0.1251813024,-0.2468750924,-0.2969430983,0.2811274230,1.1297291517,-0.4285236895,1.6837493181,0.1570269614,-1.3796472549,-0.0221588574,-0.9108480811,-1.4940923452,0.3765714467,0.1872464269,1.0197986364,-0.4883226156,-0.3771733940,0.0635235235,-0.3010345995,0.2305924147,0.3498499691,0.6820290089,0.7376207113,0.2440648973,0.2061410099,1.6231516600,-0.4384661913,-1.0267047882,-1.4744080305,-1.1771376133,0.1827095300,-1.0081615448,1.2591556311,-0.1388425082,0.4758206606,-0.2766729593,0.5053372383,-0.0261183511,-0.1096292585,0.3588334322,-0.1530779451,-0.1198340133,-0.2816464007,0.1439685822,-0.2829401791,1.0409681797,-0.0788520947,-1.1657164097,2.2557945251,2.0186963081,0.2609864473,0.6588216424,-0.0292505622,-3.0369510651,1.2542692423,0.5731019974,-0.0044440515,-0.4187056422,0.6035118103,1.5395250320,0.9933514595],[-0.5339363813,0.7204960585,-0.2520633638,0.1129418686,0.3106788397,-0.1949516982,-0.7873948216,-0.9607906938,0.3240345418,1.3103933334,-1.2638764381,1.3863197565,1.8211445808,-0.3672892451,0.4053455591,1.7521601915,1.6447889805,0.7403008342,-0.8374760747,1.8835134506,1.4278148413,3.0550069809,-1.2181924582,-0.8227337599,-0.1327370256,0.3542967737,0.6749712229,-0.3170649409,0.9784115553,0.3086686730,-0.1767462045,0.0843835175,-0.6492677331,0.3701886237,-2.1043498516,-0.0279286541,-0.8700038195,0.6007041335,-1.9832814932,0.2308917642,-1.3054196835,-0.3247613013,0.2073209584,-1.3216863871,-0.1985991448,0.9323601723,1.6048812866,-0.0951557308,-0.2890453935,0.6215823293,-0.0441857241,0.9499992728,1.8083887100,0.1037293598,0.8567683101,-1.0534691811,1.9651103020,-1.9569315910,-0.4348264337,0.6502805948,-0.9168347120,1.1094492674,-0.9395704865,0.8174906373,0.9895029664,-0.1846548319,0.4872570932,0.3263651729,-0.6341340542,-0.1032819226,-0.7346142530,0.1689246595,-0.1534329206,-1.5958054066,0.2307239473,-2.1755774021,0.4595609605,1.0350043774,1.1936333179,1.2886185646,0.5671181679,0.0456784628,0.4965856373,-0.8134568334,0.9582699537,-0.0257120635,1.0541176796,0.1182258353,1.0523356199,-1.8147698641,0.7733538747,-1.3099697828,-0.9093947411,0.5400250554,-0.3680446744,-3.2436428070,0.1594742090,-0.6183111072,-0.5293322206,2.0307416916,-0.1469859183,1.3975157738,1.4203033447,0.5249443054,0.7566241026,-1.0997810364,1.1431112289,0.8553681970,1.1881549358,-0.3714130521,0.1984697133,-0.5287451744,-0.3881629407,0.8084456325,-1.3102331161,0.2904533148,-0.6494663358,0.3399093449,2.3053667545,-1.9359093904,1.7290390730,1.5100733042,-0.2926071882,0.3544541597,1.9625288248,-0.6443831325,-0.4173543453,1.1030114889,0.3282997310,-0.0072064842,0.4895175695,-0.5674248338,1.6213970184,-0.4362982810,0.3757034838,-0.8909569979,0.5226764083,-0.6865759492,0.8143765926,1.2320840359,-0.7808359861,0.0008738514,-0.8514294028,0.4705692232,-0.3333962560,-0.3492262959,0.2567463517,-1.1781015396,-0.4792070985,-0.3375956416,0.8531564474,0.8517883420,-0.1833593547,-0.3668282926,0.4279274046,-1.1330124140,0.6293390393,-0.1875369698,0.1160475165,2.1682090759,0.8720598817,1.4349334240,1.2770864964,-0.1004557535,-0.5276843905,-0.1601087302,1.3145697117,-0.0181776304,0.5276513696,1.3746075630,-0.8591616750,0.0153728090,-0.1448028535,0.6654556394,-0.8472108841,1.1001250744,0.4171614349,-0.0177354254,-1.1216690540,0.7256075144,0.1507796347,1.0267204046,0.6718088388,0.2002222389,-0.8297312260,-0.8101356030,2.3435692787,-0.4786757231,1.0909048319,-1.7758495808,0.2627611756,0.2653958797,-0.5927811265,1.5539902449,-0.9272117019,-0.2532606721,0.0267162919,0.5292546749,0.3089748025,-0.3464745879,1.7746504545,2.9259448051,-0.6218938828,-0.3926011324,-0.3428994119,-1.1926394701,1.5049992800,-0.8424344659,0.7986744046,-0.7885806561,-1.1607004404,-0.0135042230,0.9922061563,1.9919799566,-1.2437075377,-0.4395283461,-0.5311597586,-0.0297754165,-0.5898804665,-0.7635480762,-1.5769889355,1.0587168932,-0.0034756528,0.1553152949],[-1.9455130100,0.2943831086,1.3622395992,0.4102196097,-0.4908825755,0.4014503062,0.0278171897,-0.0536309853,-1.2789026499,-1.2796289921,0.4953480065,-0.4660226703,-0.9548496604,-0.9615236521,0.9491387606,0.3494040668,-0.9034793377,-0.8746790290,0.3864723444,-0.7408620119,0.8857078552,-0.9054695368,0.3242028058,-0.9857577682,-1.7794227600,0.7796273828,-0.2548395693,-0.9350469112,-0.0115598422,-0.0557284057,0.4348230958,1.1245038509,2.6359810829,-0.2186294347,-0.0949227884,-0.2960619032,-0.7614319921,0.6499288082,0.0611613914,0.4410744905,0.7678567767,0.0930860490,-1.0300735235,0.3483096361,0.7597416639,0.0011209270,-1.1556539536,1.1331659555,0.9468092322,0.2521147132,0.1224794090,-0.5496848226,1.5198215246,-2.1303267479,-2.2218475342,-2.5806076527,0.0837756246,0.4271501005,0.8224028349,1.1306557655,0.3919700682,-0.0115966676,0.0475504175,-0.6004253030,-1.2305142879,0.5945131183,1.0072363615,0.3247313201,-1.2197523117,-1.1868721247,-0.9118242264,0.1778739989,0.3262349665,-0.8597450256,0.0205650963,1.7530291080,0.7721672654,-0.1331358552,1.0669914484,0.8614687920,0.9448447227,0.4930521548,-1.0212017298,0.7986354828,-0.7440475821,-0.1944002360,0.3420980573,0.5632216334,-2.4997861385,-0.4994763136,0.6333222389,-2.0620503426,0.8145817518,1.4459090233,0.9552916884,-0.7975873947,-2.1032149792,0.0418324843,0.5111634135,1.7133549452,-0.3090857565,-0.6058284640,-0.8708721995,1.2175831795,-0.2389851511,0.4239602387,-1.5448274612,-0.3783124983,-0.0029062431,-0.6120293736,0.2799707651,-0.0570160225,2.0717158318,-0.3598082066,1.4251867533,-0.0134970779,-0.9720110893,-0.2252669930,0.5460293293,1.5419213772,-1.5147491693,-0.3293933272,-0.0275527146,-0.9521688223,-0.3834776282,1.4057289362,-0.7203474641,-0.9101272821,-0.1940222532,1.3746294975,0.4223842025,1.1517481804,-0.3333800137,0.0154191507,2.1487598419,-0.9099595547,1.2981448174,-0.4607750475,0.7508655787,-1.1511445045,0.6852880120,0.3033760786,1.1724207401,0.0635844842,1.9570583105,0.8574123979,0.5666555762,0.7369867563,-0.3596095443,0.0712693930,-0.2050676346,0.7476635575,-1.5979381800,-1.6696869135,-0.0033412154,0.2548270524,-0.4833313227,1.0923733711,-0.2952604592,-0.8811125159,-0.4041282833,1.2807453871,0.1407522559,-0.2607857287,0.8139761686,0.2080699205,-1.0974228382,-1.0547447205,-0.5768285394,1.7892030478,0.4703820050,-0.7589623332,0.4739449620,0.3060959876,1.4044001102,-0.4405074418,1.5767858028,0.2276372015,-0.0085633667,0.1122219488,-0.1186968386,-0.7556450367,-0.2551757693,0.2367250472,-1.3682280779,-0.0868984833,-0.9292290211,0.4767979980,1.2660095692,0.7017489076,-0.0994677246,0.4604631066,0.5536651611,-0.6214803457,-0.1921214759,-0.9230517745,0.2861795425,1.9730838537,1.5194038153,-0.2218993008,0.0676458031,0.8111193776,0.5127606988,-0.0850184634,-0.3213681579,-0.2438399494,-0.1359906793,2.0709638596,0.3468191326,-2.5300788879,-0.7631182075,0.2191569060,0.0090567889,-0.7353634834,-1.5984612703,-2.1198539734,-0.1774644852,0.7098138332,0.8213640451,-0.0107929576,1.2051004171,0.0632010624,-0.3080267012,0.9183343053],[0.0939481556,0.8973917365,1.3931572437,-0.2151860446,-1.0463820696,1.2186774015,0.4525365233,1.2896789312,0.3542840779,-0.4334890842,0.0643699095,0.1865993589,0.5796870589,-0.7730751634,0.4719973505,1.7717124224,-1.5757242441,0.2662694454,-0.4041916132,0.4621180892,0.6618397832,0.1943725944,-0.9175354242,-0.0798297524,-0.7954104543,0.4863788784,-1.4996377230,0.8618410826,-2.9106974602,0.3565086722,1.5209159851,0.5370289087,-0.1094295308,2.1429982185,-2.2595870495,-0.4742313921,0.4032390714,-1.3632237911,2.1153512001,-0.8269336224,0.4764483869,-0.3866333365,0.4835948348,-1.0510087013,0.9150114655,-1.4388750792,-0.3364607096,0.0429121219,-0.1839305013,0.3848934770,1.5128203630,-1.1400394440,-1.4442991018,-0.4257532358,0.8354291320,-0.2160586119,0.9489045739,0.3786438107,-0.6682091355,1.0235065222,-0.8382804990,-0.1399486065,-0.9926248789,-1.2264190912,-1.3117069006,-0.1271556914,0.0079912404,-0.3760568500,0.5664597154,-0.8595212698,-0.1183861569,0.8529757261,-1.9854843616,0.2800190151,-1.7601602077,0.6664977670,0.6719502211,-0.7664304376,1.0983656645,0.2911262214,-1.1661839485,-0.8158940673,-0.2933756411,1.4010317326,1.6039435863,-0.5165402889,-0.1408138573,1.1347715855,-0.4194507301,-0.5617449284,0.5707189441,-0.0715572983,1.7359468937,-0.6072747111,0.2172731906,-1.6416498423,1.1322650909,-1.7049622536,0.2442215234,0.7345991135,0.9624508023,-0.7012826800,-0.4934635758,0.2794271708,1.1199611425,0.0138948634,1.3469048738,-0.5083768368,-0.3632907867,-0.4739925861,-0.8815130591,2.1984884739,2.9767618179,1.0400345325,0.5973387957,0.6966767311,0.3020858765,0.7583796382,-0.7667793036,-0.0830394998,-0.6338041425,0.4115710855,-0.0886307508,1.4750047922,0.6065263748,-0.6084166765,0.4203138649,-1.1035499573,-0.1466142833,0.0930199549,-0.4346646070,-1.3677084446,1.7228507996,0.7114614844,-0.7142797112,0.3464543819,-0.6122346520,0.7374718189,-1.2237185240,-0.4646267891,-1.4108309746,-0.4445342124,-0.4988172054,1.8301521540,-0.7865855694,-1.0325416327,0.3002009392,0.1759860516,-0.3930571973,-0.0841103196,0.6128829122,1.1748423576,2.0594334602,-0.1751677841,1.9157706499,-2.0087039471,0.0548449084,0.9300742149,0.7866141200,1.1222592592,0.1036242396,0.6618089676,0.0268117934,-1.9981175661,0.9294196367,2.2584624290,1.0756754875,1.1363080740,-0.2930243909,-0.0277448576,0.8949297667,-1.3616064787,0.3977112472,-0.7987060547,1.2334747314,0.2763086557,-2.1072409153,-0.2869181931,-1.5621356964,0.7045946717,0.8275618553,-0.4897731841,-1.0612022877,-0.2257146686,-0.4813244045,0.8550677896,0.4653412998,-1.9319355488,0.0123473499,0.6307618022,0.1360304952,0.3419711292,0.2482180744,1.0112982988,-2.8517372608,1.0242309570,1.3946468830,-1.4900366068,1.3303142786,1.1665626764,-0.0065014083,-1.8222254515,1.1139158010,-0.4737008810,1.4344037771,-1.1761858463,-1.6211301088,1.0432368517,1.0384088755,1.0714992285,1.5375100374,-0.7071874142,0.0438755527,-1.1680675745,-0.1133877560,-0.8412772417,-0.7477636337,0.6549735665,-1.6748837233,0.1416903734,1.8518723249,0.5723341703,-0.2291723639,0.4399379194],[0.8588612676,1.9773676395,-0.4936513007,0.2287907153,-1.0384471416,-0.1597029120,0.1248741448,0.7425230145,0.5512924790,0.7303834558,-0.8165476322,-0.7640091777,-0.1144194901,0.8935913444,1.6052113771,-1.2538806200,0.2855737209,-0.4613464773,1.2086780071,0.4985507429,-0.3405464590,-0.8937238455,1.2616927624,-0.9317969084,-1.0411241055,0.8100412488,0.7059498429,1.3697428703,0.9094738960,-1.9717726707,0.2420305610,-0.9077606201,1.6671775579,2.3022015095,-0.1790511310,0.8325935602,1.3162128925,-0.2899599671,-0.4036357999,-1.3132460117,0.1316954643,0.9602703452,-0.1307659447,-1.0357890129,0.5832430720,-1.4192433357,1.2996268272,-0.1998086125,-0.6707410812,1.8675396442,-0.2962323725,0.9742482901,0.2611752450,-0.1437780410,-0.6985815167,0.6297942400,-0.9077713490,-0.3890576065,0.4322726727,0.1401295662,-0.0165128782,-1.4022303820,0.3342009485,-0.0561159886,-1.2076241970,0.0269585140,0.0797272325,0.1862457544,-2.2476606369,0.9444752932,-0.2633162439,0.5973744392,-1.3259795904,-0.4369872510,-0.6111346483,0.4619466960,0.1574052423,0.4900255203,0.1863169968,0.2730141282,0.0945252627,0.6730496883,1.2070502043,-0.7237103581,-1.5864273310,-0.5616668463,0.8356405497,-0.0633497909,0.1185418442,-2.5133163929,-0.2127508223,0.9225611091,-0.4511755705,-1.3210129738,-1.5038857460,0.3074418604,-1.4114911556,-0.9896835685,0.4621183872,-0.1546157897,-1.0632112026,-0.6646524668,-0.2710254192,-0.0559235252,0.2106004804,0.3330169618,0.2115703970,-0.1708970219,-0.2765752971,-0.3595123887,-2.1213698387,-0.0695509166,0.9423075318,-0.6175131798,-0.5573751330,0.0207770932,0.8922007084,0.4503358305,0.0453439653,-0.9838991761,-0.1429029554,-0.0101154400,-0.3242063522,0.8762276173,-0.1310543418,-1.3132834435,0.1048069224,0.9953035712,0.0019832081,-0.5763996243,-1.5993032455,0.2357865274,-0.5236660242,1.2498191595,-0.3972760141,0.5428637266,-0.3759145141,-1.0959819555,0.4462662935,-0.4387346506,-1.1529084444,-0.0285182241,-0.1020001173,0.5932555795,-0.5795496702,-1.0888371468,-1.7439676523,-0.2361356318,-0.0777337477,0.7394383550,0.0782520548,1.6987942457,1.7381364107,1.1641181707,1.4951943159,0.1576152593,-0.6602454782,-1.6606831551,-1.8109960556,-1.5184327364,-0.6278802752,1.0903507471,1.0761795044,1.2518355846,-1.6058274508,-0.3573199511,-0.9369678497,1.4489934444,-0.0281910524,1.0515197515,1.3035051823,-0.7908535600,1.5493855476,0.1785422266,-0.2503399253,0.5442001820,1.4513047934,0.2108101994,-0.7557499409,-0.8626450300,-1.3776224852,1.0434657335,1.1152778864,-0.5381469727,1.1608798504,-1.1071029902,0.7692725062,1.7361100912,0.6548727751,-0.2329286039,-0.2109164447,0.8669708967,-0.5419610739,0.3878645599,1.0662782192,-1.9696036577,0.8957763910,0.0708186775,0.8453901410,0.0108403713,1.4044237137,0.6272301674,-0.7871339321,0.7302926779,0.5642443895,-2.1129529476,-1.1371386051,-0.0070197177,0.9867774844,0.0979930758,0.2202437371,-0.3924337029,0.5245616436,0.9684751630,-0.3208032548,-0.6954401731,0.2925103903,0.6001781821,-0.2031298876,2.5664210320,-0.7845811844,-0.0679450184,0.0334720835,1.3929741383],[1.5113071203,0.2075864077,0.6620129347,-0.6452699900,0.7408705354,-1.7304400206,-0.1776096076,1.5553163290,-0.1056940630,-0.6364824176,1.5327748060,-0.9357856512,-0.2761799097,0.4746173322,-0.8172100186,-0.6547473669,0.5243036151,-0.0618794486,-0.2851223946,-1.0006905794,0.7876641750,-0.8730727434,-0.6232350469,1.6723668575,0.5514597893,-1.0533039570,-0.3889478743,0.4371635914,0.9450132847,-2.1121516228,0.7922255397,-0.3742645383,-0.1040292904,-0.1706566215,0.8582338691,-1.1603547335,0.6076740623,0.9498484135,-0.8680899143,-0.8084941506,-0.5008729100,-1.4534268379,1.1655235291,-0.9484149814,-0.0762682483,0.0358931646,-0.2384106368,0.9499883652,-0.9772548676,-1.0445420742,0.1405598968,0.5456544757,-0.5860677958,2.0359721184,-0.7959108949,1.8235174417,1.8872886896,1.1418535709,0.3126354218,0.3064375222,-0.2798117399,-0.4689850807,-1.6476761103,-1.5992029905,-1.2629358768,-1.7541141510,1.1695880890,-0.0457241274,0.4733829498,0.0854842886,2.0468397141,0.2154959887,0.0350889787,0.2971332371,0.9285229445,0.9239901304,0.4604280591,-0.5467797518,0.4883889854,-0.9885988235,-0.5728197694,0.9264045358,0.7810805440,1.0949702263,0.1750663072,1.4096968174,0.2180642933,0.7436999083,-0.1266501397,1.3566181660,0.3130574524,1.2390726805,-0.1934143156,-0.7612330914,1.0759313107,-0.3289625347,0.1660896689,0.0740403831,-1.1717393398,1.1129521132,-1.1576689482,-1.7789555788,0.9600282907,-0.7468833923,0.5374038219,0.9328088164,1.5861493349,0.8423115611,-0.4052751660,1.0446491241,0.0374817997,-1.1891011000,-2.3277878761,0.3875864446,0.2966091335,-2.2180492878,1.0663291216,0.2598075867,1.9161059856,-0.7282618880,-0.6266215444,-0.6739975810,-0.6415785551,-0.6179529428,0.6047268510,-0.2844314575,-0.4305409491,1.9450337887,-0.1466508955,0.4638362825,2.0536992550,-1.2035881281,0.2433279604,0.3226449490,0.2997124195,-2.0240993500,1.5049318075,2.5287919044,-1.6473007202,-1.8275324106,0.7434623837,0.3066866100,-1.1217963696,1.3457329273,0.6340570450,1.1217298508,-0.0488672554,-1.2668669224,-0.4204682112,1.2064194679,-0.1509321928,2.8461978436,-0.0451924764,0.4221574366,1.7345610857,-0.5981693268,0.7031428814,-0.0062330775,0.8340855241,1.6620643139,-2.1220414639,0.2365728617,-0.1516558230,0.5030402541,-1.7541236877,1.3960247040,-1.2516578436,0.9715057611,-0.1300617158,0.2552434802,1.9452192783,-1.0372388363,0.5538521409,-0.3709153831,-1.3116137981,0.6361788511,0.9690408707,-0.9008204341,-1.6484543085,-0.0956472978,0.3216061890,-0.4470887482,-0.7181816697,-0.4564741850,0.0436974354,-0.8783827424,-0.2153731734,-0.7284212708,0.0316979624,0.5592319369,0.3584336936,0.3528334498,0.7266294956,1.2041256428,0.6806427836,0.3226889074,-1.6938356161,1.0152925253,-1.2883669138,1.5473442078,-1.6940101385,0.7440575957,0.5449872017,-1.5002492666,-1.1375263929,0.4404759109,-0.9481218457,1.0618398190,0.3651114106,-0.1956363469,-0.0600685142,0.2220172286,-0.2049867213,0.7872170806,0.5148497224,-0.4739772379,1.0607751608,-2.2600011826,-0.2448202819,-0.5458866358,-1.6628972292,0.6307308674,0.6686230302,-0.8363482356],[-0.5782997608,-0.6807925105,-0.1028094068,-0.3188337684,-0.2476856112,-0.8180071115,0.1356047243,1.5968712568,-0.8921045065,0.7458104491,-1.2947020531,-0.0416521616,-1.5273494720,-0.4211159647,0.4188219309,0.0392890759,0.6555977464,0.9114678502,0.3619571626,-0.5588787198,0.2664752603,-0.3256011307,2.4297068119,-0.5743183494,0.2944850326,0.9484425783,0.9825353622,0.3974099755,-1.0577732325,0.6464989185,-0.2026707530,0.3146737516,-1.5660982132,-1.0093059540,0.9294809699,0.9666360021,0.3079728484,-0.0658802390,0.5567436218,-0.4141434431,-0.9001114368,-0.2555243075,0.4303688407,0.6368939877,-0.3189966381,-0.1767177880,2.1670389175,-1.1297926903,0.0641836300,0.1511009932,-0.8907510042,1.9690883160,0.8412009478,-0.9752152562,-1.5275386572,1.0577732325,1.3252483606,0.8089088202,1.4796154499,-0.2566390634,-1.3153479099,-0.8288609982,-0.2103146762,0.1020352095,0.9253606200,1.2352576256,0.5293609500,0.4743773043,-0.8591988087,-0.1014773473,0.5960752368,-1.3404738903,1.1582435369,1.2524724007,-0.6086274385,-0.7040278912,0.8515852094,2.5486783981,-0.6445094943,0.4089204371,0.3661203384,0.1784386188,-1.1837905645,-1.6509048939,1.1145668030,0.2557139695,-0.0395323709,0.0242085606,-0.4647791684,-0.2464168668,-0.5249590874,-0.0612839907,-0.1260320395,-1.0659192801,0.7829201221,1.1975088120,-1.8215229511,1.2382860184,-0.5807151198,-0.2608066201,-0.0539864413,-1.5416053534,0.7714299560,0.5916702747,0.5156317949,1.1404858828,-1.3494912386,2.1378667355,0.3047274649,1.2195792198,-0.3578601480,0.0984548479,0.7212519050,2.6335790157,-0.4315360487,0.5843237042,-0.0755212381,-0.2289883345,0.9112005234,-1.0294852257,2.1142601967,1.0679942369,0.5550442338,-0.1199776754,0.5793484449,1.8686089516,-0.6061916947,-0.0292256158,-0.1874776036,0.4629638493,0.4734807909,-1.0361982584,-2.4138829708,-1.1330380440,-0.5306935310,-0.6419487596,-0.5470802188,0.1643695980,-1.6450990438,-0.2671853602,-0.1303537935,0.0227936525,0.9756251574,-0.0457911752,-0.4323091805,-1.2599148750,-0.1694476306,0.4077905118,1.0774275064,-1.1228400469,0.2199149877,0.4889201522,-1.2091072798,0.4924613833,-1.7255488634,0.9194186926,1.4995775223,-1.0543442965,0.4016973972,0.9137138724,1.4287455082,-0.7213507891,0.5033758283,-1.1633116007,-0.7225621343,-1.4303677082,-0.2655269504,-0.8245450258,-0.6388269663,1.1980402470,-1.7719768286,-0.5820679665,1.0126674175,1.2914206982,-1.1844006777,0.3093956411,-1.2403331995,-0.4625792503,1.7663393021,0.6885182858,-0.1327944398,-0.8755931854,-1.5265610218,0.2222946286,-0.3382628858,0.6167198420,-1.0755565166,0.0484146737,-0.9836882949,-0.3923490345,0.5466002822,-0.3768848777,-0.7443243265,0.3661516309,-0.1037210673,-0.6928908229,0.7701465487,-0.2680769861,-0.1214702353,-0.6213663220,0.2482643723,1.5134902000,-0.0739834234,-1.4493509531,1.1660684347,0.1881067455,-1.2092013359,1.2742975950,0.3987053037,0.3768714368,1.4138619900,-0.8284464478,-0.0801821947,-1.1233460903,-0.6918120980,0.5208147764,-0.0682908595,-0.9105952978,0.1875498742,-0.5142341852,-0.5274975300,0.7494561672,-0.7152031660,0.4310196340],[-0.4823740721,-0.2510137260,0.1944553405,-1.3107248545,-0.1923373193,-0.1888710707,0.2006835639,0.8063291907,0.7279618979,0.2301612198,0.5022042990,-1.6600764990,1.0828719139,1.8728606701,0.6034314036,-0.4063622355,0.0974950045,-0.8997538090,-1.5935285091,-1.0190300941,1.0629904270,0.0831695795,-0.4738741815,-0.4485514760,0.3416296840,-0.1248249412,1.1692199707,2.2329859734,-0.3595784605,2.1218981743,-0.0728280321,0.4060373604,0.5449692011,0.8206561208,-0.4518414736,0.2907862067,-2.1724636555,1.1614646912,2.5593364239,0.7427705526,0.9906321168,2.0261092186,-0.0148052406,-0.2360571027,0.0363243632,1.2760270834,2.2971293926,-1.1610804796,-1.7893850803,-0.3227211535,-1.8779487610,-1.3441138268,-0.8956089616,-0.0985141471,0.4440960586,-0.9870734215,-1.1733098030,-0.4680004716,0.0355096310,-0.9436638355,-0.1876551807,-0.2832317352,1.1657112837,1.5924739838,1.0328596830,-0.4097533226,-0.4622452855,-0.5567474961,1.1893956661,-0.3589572310,-0.8171840310,-0.3599012196,0.5284134746,0.6601926684,-0.2853067815,-1.5502932072,0.6783145070,-1.3337225914,0.9995742440,-0.2768847942,0.7775377631,0.6332675219,-0.1205610335,-0.9084852934,-0.1494361758,0.3467653990,-0.8398606181,0.4216800332,-2.1474878788,1.0741838217,0.5535035133,1.4500633478,0.5962111950,-0.3862569332,1.0310724974,-1.6946784258,-0.4748837650,0.9095218778,1.2876939774,-1.8530343771,-0.6701546907,1.8218663931,0.6484426260,-2.1453423500,0.1260983646,0.6405251026,-0.1364991963,-0.5790331960,-1.1921503544,-0.0568375923,0.2709504068,0.9835256934,0.8560069799,-1.0371228456,0.1756413430,0.4604872167,0.1207154393,0.0869964808,0.1099092290,0.7736787200,2.4697232246,-0.8884851933,-1.8836847544,0.4033220410,-0.9149255157,-1.0660659075,-0.1941040903,-0.9507152438,-0.6525584459,-0.2845203578,0.5010703206,-1.1385391951,-2.5650165081,-1.0482982397,0.7601513863,1.0343254805,0.1622663587,0.4804539382,0.4948939681,-1.1310155392,-0.4456132054,-0.0148028927,0.6583849192,0.8705469370,-1.4775637388,1.5122822523,-2.5054395199,0.6433568597,1.1670595407,-2.4150571823,-0.7000072002,0.4520107210,0.2669504285,0.0745046884,-0.0961903110,0.8950932026,-0.0153444046,2.3216211796,-0.0087271752,-0.7285081148,0.8591826558,-0.0213805623,-0.9274692535,0.8245937228,-0.0500126071,0.9498620033,0.0939764380,0.2396673709,0.3366479874,-0.0664266646,1.4263436794,0.2282274365,-0.6703722477,1.6196011305,1.5359443426,0.8145934343,0.0678382739,0.0837390274,-0.2199727595,-0.4387906492,0.3200174570,1.2475085258,-0.4494763911,1.4635165930,0.7197579145,-0.1485113055,0.0235467963,-0.0737998858,-0.1217009351,1.1287385225,0.3565534651,0.0661123544,-1.4903062582,-0.1748026311,-0.3600355685,0.4882857800,0.7067445517,1.7316313982,-1.5993480682,1.1871122122,0.0806279257,0.1233804226,1.4097801447,0.0073844674,0.4032472372,0.1401428878,1.4815948009,-0.3940936625,0.9161916375,-0.1888489723,-0.2546358109,0.1415531337,0.2554948926,0.5829980373,0.5339160562,0.1090676412,1.0032021999,-0.1615457684,-0.6464024782,-0.5283038616,0.3620505929,2.9733176231,-1.0681328773,0.7487841249],[0.8224366307,1.5359381437,2.7004611492,0.6946715117,0.9688691497,-0.2185768485,-1.7118986845,0.6170027852,-1.4078933001,0.2380145192,0.9218633771,0.8282817602,-0.2167852372,1.2319794893,1.1953899860,-0.7662591338,-1.2329901457,-0.6564483643,-0.7983369231,0.4234818816,0.4227693379,0.3662743270,0.3555461764,0.3225057721,0.0377100483,0.7185549140,0.1276012510,-0.8430910110,-1.1099169254,-0.7077657580,0.9883331060,-1.2329790592,-1.6483401060,0.0518403128,0.4276541770,-0.2361366451,1.1640164852,1.8784108162,1.5808711052,-0.7807156444,-0.8628751636,-1.4693769217,1.1178039312,-0.8612729311,0.3775274754,0.0553550571,-0.2481898218,1.1510961056,1.4155236483,0.4141672552,-0.1486062407,0.1500177085,0.7754682302,1.1698340178,1.0434730053,-1.3811140060,0.4370983243,0.5352761745,-1.8544534445,1.2646948099,0.2929232419,0.0995730236,1.7987678051,1.3470214605,1.2089182138,0.1961900741,0.1137968972,-1.6796602011,-0.0636957809,-1.4094570875,0.2016211897,-0.8527985215,-0.3630916178,0.8872061968,-1.0532958508,0.3649423420,0.8411453366,-0.3478016555,-0.1131411865,0.4069535732,2.3565943241,-0.6662643552,-0.9734390974,-0.6908809543,0.9773233533,-0.1573923081,-0.7164798379,-0.4443368018,-0.1431108266,1.3523162603,1.8951483965,0.3178819716,-0.5887733102,0.8732778430,-0.6213280559,-1.3105956316,-1.8258597851,0.3260253668,-0.9911643267,-0.7561984658,-0.2424144000,-1.4406697750,-1.8866000175,-0.4525777102,0.4981562197,2.9362752438,-0.4877498746,-1.0544553995,-0.3383483291,1.2915300131,-1.2027258873,0.3907198012,-0.0700768232,0.8171388507,2.8624243736,1.7433649302,0.0142839653,-0.7587402463,-1.0983080864,-0.1429131776,0.9946195483,0.2118909806,0.2142890394,0.7555453181,-0.7680301666,-0.7556154132,0.1945812553,0.9759526253,1.1216096878,-0.0747907311,0.7729748487,1.2828271389,0.2414376140,1.2615263462,0.6379812360,-0.4018948674,-1.7069851160,-0.4766686261,1.3859781027,0.9483999014,-2.4435849190,0.9283960462,0.1634889543,-0.3825234175,1.8444286585,0.2194792330,-0.9868534803,0.5217725635,1.0495344400,0.8497440219,0.6917409897,1.0312966108,1.2406227589,-0.3547858298,1.9061973095,-0.7241984010,0.4865470529,0.6772283316,-0.7669066787,0.3942653239,-0.8978323936,0.8897718787,0.2239701748,2.4911932945,0.3996796012,-0.5503692627,-0.8688344955,-0.4812298417,0.0351235606,1.9813669920,1.5126305819,-2.0804917812,1.2661659718,0.0236697476,-0.6080989838,-0.9334201813,-1.1945544481,2.2148840427,-0.0787384063,-0.4709452689,0.3463014662,0.2351722717,-0.6441224217,0.7173498273,-1.0471272469,-0.9710624814,0.3333386481,-0.0502374433,0.0522217862,0.7412327528,-0.9452767968,0.4947839677,1.9146962166,0.4911620319,0.6288027763,-0.2933082283,0.0901089683,0.7404099107,-0.1309597939,-1.9170587063,1.6293636560,-1.7506906986,0.4758150280,-0.6292086244,-0.1218387485,0.2569647133,-1.7762563229,1.1726000309,-1.1609822512,-0.4973379970,1.3849647045,0.2108934671,1.7501076460,0.1977767199,-1.5061140060,0.2052410543,0.8849083185,1.0420837402,-1.5378690958,-0.9945856929,0.3950080276,-1.1667097807,0.7336266637,1.0541245937],[-0.7604739070,1.0298671722,0.4831345081,-0.9305076599,2.3098583221,0.8978148699,-0.7780020237,-0.0763000399,0.1405724138,-0.9309204221,-1.2948073149,-0.9992104173,-0.7626069784,-1.1946477890,-0.0887558684,-1.1694561243,1.4181926250,-0.5647112131,0.1377933472,-0.8132561445,1.9780479670,-0.2693868876,2.5489280224,-0.4812024236,2.1438763142,-0.6436809897,-0.2257929593,-0.1043324471,-0.0241426136,0.3608437479,-1.3132336140,-0.2473047823,1.9679007530,-0.0772148073,-2.0944638252,-1.2689850330,-0.3390084207,0.0193167645,1.2736999989,1.4182841778,0.7875558734,-0.5130640268,0.0768273100,-0.9786463380,0.2218512148,-0.9420463443,-0.2778419554,0.2273272276,-0.4655575454,-1.0925114155,-0.1088153049,-0.8575080037,-0.7718771696,-2.1138706207,0.1061449572,-1.2214846611,-0.4387984276,0.6525202990,0.7284477353,-1.1624752283,2.0951714516,1.4666159153,0.5122377276,1.6254560947,0.5767518282,-1.4740482569,-0.3197341561,0.0622204207,2.2914700508,0.5785361528,0.5546657443,0.7368097901,-0.2011972219,-0.3459620476,1.4479689598,0.7183705568,-0.7114289403,-0.2445436120,0.9475785494,0.1670360416,-0.1054515019,1.5192409754,-0.2389630973,-0.3069087863,0.3849918544,-1.9849205017,-0.8939456344,0.1685190648,0.6565925479,-1.5131690502,0.3487373590,-0.4754433334,-0.5820451379,0.9503807425,-0.6067024469,0.4588730037,1.6100572348,0.5035930276,0.4674057364,-0.3812543452,0.4779359996,0.5542572141,0.8604867458,-0.0800580233,-0.5947277546,0.3877373040,0.9289138317,-2.0369527340,0.8429705501,-0.8200830221,-0.3597954512,0.1188161224,-0.8078942895,-0.6227641106,-2.1208841801,-1.0918895006,-1.2143417597,-0.9062018991,0.2425887883,1.0418492556,-0.8915598392,1.4448651075,0.5658872128,-0.6691966057,0.0779437646,0.3703321815,1.0202715397,0.5159808397,-0.3667911887,0.1917651892,-0.0155972950,0.6114939451,-0.5516857505,2.0143985748,-1.4417407513,0.1094853356,1.6910899878,-0.5058087707,-0.2841334641,-0.4427628219,-0.7798770070,0.3184933960,1.5184088945,-0.3982077539,-0.3256989717,-1.9115608931,-0.4265502393,1.9067032337,-0.5795803070,-0.9464166760,0.1618944108,0.2046693861,2.7043745518,-1.1141778231,-0.2270487696,0.0677588880,-0.8169810176,-0.0516205356,0.2156272531,-0.0458132848,1.1395378113,-1.9288947582,1.0327042341,-0.6658371091,-1.0002037287,0.7938078046,-0.5091620684,1.1180939674,-0.8911152482,-0.6446077824,0.1085245833,0.1545051485,0.7162056565,0.0998768061,1.5463054180,0.3392734230,-0.1677765995,0.0178067535,2.2100381851,0.1243686378,-0.4336081743,-1.1434708834,-0.2915322185,0.2938709855,0.1741888076,-0.9570773244,0.2976694703,-0.8821976185,-0.0567142293,1.8015016317,-0.0966258571,-0.8744275570,-0.3556698859,-0.2422062755,-0.0088157170,1.0579258204,0.6251289845,2.1922950745,0.8293181062,1.1037898064,0.1161758080,1.3735295534,1.8517149687,1.2776745558,1.1805377007,-1.1309832335,-1.4316225052,-0.7249827385,0.4579662681,-1.0276892185,-0.9429395199,0.7407530546,-1.1371870041,0.8070212007,0.8429144025,1.3206818104,0.6324065924,1.9122589827,1.9089622498,0.9503044486,1.8724390268,-0.3448031545,-0.0643757060,0.5141147375],[-1.0741199255,-0.6546777487,-0.2585322559,-0.4660071731,0.2123210430,-0.6828444600,-0.3794018924,1.5062401295,0.1076194942,1.1968497038,1.3637975454,-1.5723030567,-0.2526699901,-0.8103145957,-0.7729591131,-0.3791863918,-0.5915085673,-0.0657836497,0.5478895903,1.4897552729,1.2893457413,0.3316682577,-1.2735449076,0.3352213204,-0.1605404168,-0.6274160743,0.0444982313,0.4427059889,-0.1887391508,0.0424868092,0.6873580813,-0.5684332252,-1.8209024668,-1.5618481636,1.1418203115,-0.1679338664,-0.1575482935,0.8625968695,1.7374920845,-0.6851282716,-0.7020391822,-0.5403820276,0.0701741204,0.6840214729,-1.0883448124,-1.0662660599,0.3783966303,-1.2807009220,-0.6299644113,1.1646709442,0.6178892255,0.2659975588,0.5481295586,0.8629608750,0.3323490322,-1.7330682278,0.7170284390,1.3835469484,-0.1560150534,0.6267575026,0.0135885328,0.5174806714,-1.3381071091,0.2403987944,-0.6325420737,-0.3140203655,-0.5688653588,0.0899808258,0.2951270342,-0.3967520297,0.8154203892,0.6676385403,-1.0550323725,-1.7451119423,1.2019145489,0.8572792411,0.5403226018,-0.3976074755,-0.5785681009,-1.1575276852,-0.7872358561,-0.7130684257,-0.6214036942,-1.0228501558,0.0767670125,-0.4886718988,1.1801239252,-0.2055412382,-0.6231933236,1.3826036453,-0.8254976273,-0.2961417437,1.4713494778,0.2379731238,0.8579406738,0.3441528976,-0.5790320039,-1.0394935608,-0.3038409948,-0.4149534404,-1.0315365791,-1.7151025534,-0.5805560946,1.2495222092,-1.4802988768,0.2671444416,-0.6707246304,-1.2410545349,0.1354297549,0.9997917414,0.4972026944,-1.0840744972,0.6837875843,-0.3927797079,0.3305899799,0.6962628365,-1.0900052786,-0.6791628003,0.3703728318,-0.3813745379,-1.2939769030,-0.4187581241,0.2346390933,-0.2559554577,-1.0008970499,-1.3414121866,-0.4729372263,1.4051200151,-1.5527420044,-1.2766270638,-1.0660550594,-0.5740276575,1.2553514242,1.3542720079,-1.4046258926,0.3287573159,-3.0902717113,-0.8251926899,1.3747756481,-0.5134324431,-0.0683978125,-0.5844672918,2.3122715950,-0.1144952849,-0.1771259159,0.6621459126,-0.9766175747,0.9550002217,-0.2011980265,1.0007474422,0.2201173604,0.5645438433,0.1237144545,-0.7314903736,-0.0032809498,-0.6201736331,-0.3656804562,0.4783496261,-0.1402065754,-0.8107504249,0.4245607853,-1.9665911198,0.0242772475,-0.2220647484,-1.0378764868,0.8202825189,1.1987079382,-0.3498147130,-0.1208228320,-0.7644951940,0.0111101391,-0.5653514862,0.5157155395,1.1558949947,0.1688937247,1.6910996437,-0.8339630961,-0.7835124731,-0.0265033338,0.7289951444,-0.0396559015,-0.6672050953,-0.9940689206,-1.0564441681,0.9359007478,-0.7483726740,0.7249618769,0.7593082190,0.7683255076,-1.2546075583,-0.1201548800,-0.1855690032,-1.3772150278,-1.6465450525,-0.4068576992,1.6283794641,0.1573139578,-0.4508792460,0.4044491053,-0.5097197294,1.2035160065,0.4231516421,1.4437040091,-0.1109154448,0.2013794035,-1.0002640486,-0.3121077120,-0.6141272187,0.1143021211,-0.3780872822,1.0046535730,1.8508744240,0.5487187505,-0.7260112166,2.1805567741,0.3423650563,-0.8168572783,-0.4555746317,-0.3628090918,-2.2633161545,-0.6448135972,-0.7452431321,0.2334146351,0.8234783411],[0.5006471276,0.4734275937,0.6055223942,0.5627831221,-0.4720727503,-0.1004450172,-0.2453504503,-0.3004291356,0.2103607357,0.8166112304,-0.9881518483,0.6973307133,1.4465472698,0.7168634534,0.5701670647,0.9973092675,0.7807003856,0.0557702519,0.8964763880,-2.2280788422,-1.1141041517,0.7094801664,1.4749150276,-0.1043331474,1.5968731642,0.6388558745,-1.8664846420,-0.9645528793,-1.4716819525,0.1554246396,-0.7584255934,-0.6129170060,-0.1678290069,-0.2931369245,0.8345580697,0.4046899974,-0.2836818397,-0.1639537364,-1.3903989792,-0.4709609151,-0.1503601223,-0.4822956324,1.0982751846,0.7710070610,-0.7754664421,-0.2240831107,-0.0304530412,1.0826807022,-0.5305797458,0.6023666263,-2.1382906437,-0.0566323511,1.1664443016,-0.7720962763,-0.9089879990,-0.4142166376,2.5208303928,-0.1248839349,0.0180679932,-0.9986228347,-1.2627335787,-0.5651069880,-0.4552381933,0.5557269454,0.9943681955,-0.2865622938,-0.5457049608,1.6967656612,-0.5081860423,0.0036205626,-1.0185145140,0.2515618503,-0.3381818235,1.0141428709,-0.5825280547,0.6387898326,-0.3677871227,-0.5683578253,-0.7212477922,0.1982797086,-0.2174220383,0.9147565365,0.4437566400,-0.3062036335,-0.9490559101,-0.4480834901,0.4285705388,-0.1649990529,0.8649626374,-0.6476624608,0.7200577259,-0.3377147615,-2.0285484791,0.7261819839,-1.1678304672,-1.2852076292,-0.4837974906,1.2880570889,-0.1298787147,-0.1980783939,-0.3344875872,-0.3914431930,-0.6124061942,-0.6765239835,1.3272296190,-0.4486954510,-0.3164072931,0.0308305565,-0.3133567274,-0.1732591838,-0.3273687065,0.9443682432,1.1220173836,0.1123387292,1.3723402023,2.0625617504,-2.4485590458,-0.5619884133,0.3550988138,0.6343781948,-0.3924500942,-0.8131564260,-0.2328732610,0.2468012124,-0.5624132156,-0.8416020274,-0.9421200156,0.9453729987,0.1409107000,1.7535657883,-0.3472236991,-1.4894933701,-0.5080354214,1.9926500320,0.5554619431,-1.6960769892,1.7523661852,-0.5975508094,-0.4664914012,-1.0947922468,-0.6454249024,-0.2021548301,0.2050734758,-0.2703858316,-0.3102994859,0.3622126877,-1.8181655407,-0.1004303396,-1.0705139637,-2.0765106678,1.7234400511,-0.1522772014,-0.3638286889,0.3420642316,1.4070423841,-1.3161656857,-2.1506977081,-1.5808782578,-0.3616223037,1.1524333954,-0.4334156513,0.7738724351,-0.8342115879,-0.7282395959,0.6749752760,-0.4777715802,1.4923013449,-0.6583909392,0.5604837537,-0.5135775208,-0.4712557197,1.6082854271,-1.3844133615,2.2784225941,1.3843681812,-0.9408109784,0.0214956999,0.1947668046,-1.2912441492,-0.3801298738,-0.8062320352,-0.7280040383,-1.0826486349,1.0985850096,0.1326422393,1.8456447124,2.1873707771,-0.7083841562,0.6465144753,-0.5518972278,0.2373841107,-0.0164395105,-0.0291097742,1.6453253031,-0.5479516983,0.1853130013,0.6488700509,-0.3811258376,0.0817243606,1.0087109804,-1.1935225725,0.2637767792,-0.8994972110,0.3363832235,0.2713788450,0.5522467494,-0.6908838749,3.2648844719,1.3637685776,-0.5434085727,-0.8160778284,2.5939521790,0.5945166349,0.0170803964,-0.8449776769,-0.8151623607,-0.6342274547,0.9066578150,-0.6744567156,0.2777738869,0.2134030014,-0.6038596034,0.5620812774,0.4132402539],[-0.9826997519,1.0715109110,0.4687927365,-0.1042525694,-0.4841112792,-0.2296179980,0.9663137794,-0.5415360332,-0.3242196739,-0.0827071369,1.3674101830,-1.2061275244,-1.1793091297,2.2461023331,-1.5597070456,-0.7396000624,-0.1240326911,-0.0979401693,-0.8028639555,-0.9611197710,-1.2690706253,-1.4909585714,1.5873177052,0.1216608360,-1.4530825615,1.1822804213,0.6036565900,0.6341340542,0.6647869349,1.0432860851,0.1756239235,-0.1246411651,-0.0421862118,0.7873358130,0.0727108419,-1.8087786436,1.8381971121,1.0782800913,-0.3556542397,-0.9404379129,1.1988965273,-0.3806185126,-1.1939523220,0.2033593506,1.0147236586,-2.7281723022,0.7938317657,0.1125748605,0.5684316754,-0.6089876294,0.2789086998,0.9851705432,1.7537573576,0.2681056559,0.8519150019,2.4660148621,-0.2989487350,0.0854466408,-0.4715458453,-0.5302187204,-2.3346014023,-0.4460463226,-0.5775428414,-0.6922178268,-0.3119456172,0.3294471204,-1.3128342628,0.3397966027,-0.2910468876,0.9310884476,0.5599357486,-0.0645182207,-0.0022406222,-0.5151447654,0.5345206857,0.3004088998,0.2320223451,2.1810531616,-0.1001604870,-0.6051329970,-0.5047010779,0.0176231600,-0.5135729313,1.3429766893,0.4130504131,1.0463596582,0.8617561460,0.0478619747,0.0463153683,0.3848271072,1.6161820889,0.8716033101,-0.7510064244,1.6390513182,1.7839026451,0.5109132528,-0.4672897458,1.1222280264,0.9888499975,-0.9561136365,0.5663571358,1.4470038414,-0.7865455747,-0.4490454197,0.1197444052,-0.0093231238,0.5896588564,-0.2719801962,0.1982195675,-0.2045348287,0.4862102866,-0.9825719595,0.2153809816,-1.1695139408,1.4600381851,-0.4786804616,-0.2161083519,1.3162412643,-1.0952185392,-0.2379431874,0.0441102535,-0.4496622980,-0.5129234195,-1.9090203047,0.5068237185,0.2302060127,-0.7661902905,0.0725760013,0.9885923266,1.0863655806,-2.9028239250,-0.5360904932,-1.9112136364,1.8043782711,-1.2130129337,1.3801456690,-0.3277495205,0.2450582385,-0.7642333508,-0.3404273093,0.1624981314,1.7061364651,0.8927001953,0.8685998917,-0.5195849538,-0.0039161691,-1.3448412418,-0.3250283897,-0.7400012612,0.1771302372,0.6164767146,1.7528487444,-0.3680059314,-1.0127396584,0.7688501477,-0.2467727959,0.7563650608,0.9357182980,1.0622066259,2.3305165768,-0.5720127225,0.5467406511,0.5635195374,-0.9249114990,2.7588467598,0.8314345479,0.1939035803,0.7628485560,-0.4104726315,-1.9406030178,-0.3032291830,0.4363195598,1.2151136398,2.1058390141,1.1828629971,0.3339031935,-0.8795049191,1.2889955044,-0.8493562937,0.2111147940,-1.2545552254,-0.6699841619,0.1545667648,0.9607257843,0.9349159002,0.8397400975,0.6369614005,1.0919691324,1.8135930300,0.9556264877,-0.1990593225,-0.8042902946,0.6528923512,-0.0553436130,-0.9927421808,-0.5951382518,-0.0551041886,0.6262081265,2.3066689968,-0.2077073157,0.7109154463,-1.1168620586,0.5868679881,-1.0205169916,-0.4096455276,0.5808160305,1.0296586752,0.6482652426,-0.2194568217,-1.4834964275,-0.1184953302,0.4194943607,2.1360015869,-0.4918103814,-0.7194541097,-1.3561576605,0.2670113444,0.6546127200,-0.2104466856,-0.1670428365,0.5703315735,-0.4559410810,1.0440343618,1.2773948908]],[[-0.5565972328,1.6024007797,0.2661893368,-0.4304800034,0.2327338457,-0.6957453489,0.8529608250,-0.1704961210,1.0020779371,-0.4330396354,-0.1417545974,-0.1366312355,1.1569111347,1.1959494352,-0.6120578051,0.3761216402,1.3039016724,-0.6344389319,0.5409040451,-0.2106793523,1.7126660347,-0.1599847823,1.6979123354,-1.2285583019,-1.1358175278,2.6169531345,0.8238010406,-1.1758143902,1.1397869587,-2.3761627674,-0.2448349744,0.1050523967,1.4465601444,0.3278667629,-0.9939356446,-0.9700256586,-1.2412242889,1.1203885078,1.4587260485,-1.9321591854,-0.2354511768,-0.0673358142,0.4001037776,0.2343670577,0.7390849590,-0.2944712937,-0.3466485143,-0.1584270895,0.3662830889,-0.6116271615,-0.2264357954,1.5210326910,-0.6150326133,-1.5202091932,-0.5913832784,-0.4052457213,-1.4865701199,-1.9483112097,-0.5207968354,0.4274829328,0.3046709895,-2.0096416473,-1.5396296978,-0.4011812806,1.5112646818,-0.7303193808,-0.5129240751,-0.8286438584,0.0695798472,1.7603245974,-0.3305148482,-0.8455023170,-1.4774661064,1.2175356150,0.3046442270,1.5573654175,0.2028430849,0.1601100266,0.9338047504,-0.1322722286,0.5623129010,-1.1053166389,0.3339771032,0.7388482690,-0.5675597191,1.4587401152,0.9002025127,0.7754671574,1.2447451353,-0.9453841448,-0.0203836951,-0.7514674067,0.2120767683,0.2850379646,0.1254609972,0.2035339177,-0.3764954805,-0.9387801886,-0.1428789049,0.5332627892,-2.6065831184,-0.8216148615,-0.7328824997,-1.4360533953,0.2980169356,1.7235522270,-0.1267851740,1.2048579454,-1.2153682709,-0.4437190592,0.7440560460,0.9141807556,1.5864833593,0.6928018332,-0.9534311891,0.6793599129,0.5651528239,0.2193024904,-1.1105041504,-1.0860613585,1.1356219053,-1.1252977848,-0.3664375544,-0.0169660207,-0.5677758455,0.5198579431,-0.4156208634,-1.2413494587,-0.4517956972,1.2629374266,-1.9606316090,-2.0649142265,1.2586480379,-1.0318564177,0.6451462507,-0.0639000982,0.3058444858,0.3714894652,3.2189693451,0.8671776056,-0.6356307268,-1.2256245613,-0.6236977577,2.0042676926,0.3193460405,-0.2986065447,-0.0811628103,1.1276055574,0.4359809458,-0.8637416363,-0.8411214352,0.0813471526,-2.5876824856,-0.4164364338,-1.0778590441,-0.4280856252,-0.1837351769,-0.4342537224,-2.1249547005,-0.7090559006,0.4856309593,-0.2251652330,-0.4225297570,0.7283768058,-0.4008677304,1.7381820679,-2.5891318321,-0.0358014069,0.7630400062,1.2215707302,0.5623279810,-0.2767444849,-0.7788773179,0.6425501108,-0.2790244818,-0.0601394959,0.9226582050,-1.2856285572,0.5124117136,0.6543196440,0.6985861063,-1.1863965988,-0.7743391395,-0.8036898375,0.1511544138,-0.2061739713,0.2215616107,1.4245580435,0.6433502436,0.2171405405,1.0231777430,0.8845224380,1.2237645388,0.7285240889,-0.2870639265,-1.1345304251,0.4285645187,-0.9715670943,0.5583679676,-1.2529093027,-0.0554292351,-1.1577489376,-0.8337481618,-0.9095815420,0.5952699184,0.4276467860,0.4741272628,-0.4778809249,-0.5791444778,2.0132668018,0.0643128380,-1.7063057423,0.2727234066,0.1687286496,0.6589840651,0.2412974089,0.3876089454,0.1535113156,-0.7328495979,0.6013470292,-0.8155531287,-1.5449550152,0.8408113718,0.0877092257],[-0.5942927003,1.5020433664,0.5605551004,1.1409633160,1.9613232613,1.1202725172,-0.3546351194,2.1160829067,0.5874300599,-0.6321915388,-1.2108942270,0.1716439277,-0.2549261749,-1.2034045458,1.0263698101,0.9422594905,0.2088890970,-0.1047871783,-1.2705695629,-1.1836208105,-0.7867333889,1.7524260283,-0.7368371487,0.3624839783,-0.5364046097,-1.8370509148,0.1137978658,0.6024752259,-0.2508539855,-0.2528652251,1.6051051617,0.0342720747,1.5153194666,-1.0420209169,-0.4630776942,0.1474878788,-0.9713253379,0.0588521026,0.3362514973,0.2530673146,0.3107353151,0.1848080754,0.2764378190,-2.1524662971,2.7178449631,1.1053645611,0.8582649827,-0.1122928783,1.1051639318,-0.7339845896,0.0696830601,-0.2674444020,-0.2647155225,1.1227550507,0.2607275844,1.8020198345,-0.6892679930,-0.0933775827,-0.2251587808,-1.3664307594,0.3179258406,0.0487223379,0.4735718071,0.0241525117,0.8647046685,0.6103027463,-0.2139733285,-0.4484854937,1.4012920856,0.3160155714,-0.3197103739,1.7200762033,3.0469586849,-0.2641649544,-1.4224961996,-0.7580975294,-0.9873674512,0.1957168132,-0.4102197289,0.2144638300,-0.7272182703,-1.2818526030,-0.3538502753,0.5934340358,0.0904515535,-0.5685857534,-0.7609325647,0.4194073081,0.8674047589,-1.3904449940,0.0177442655,0.8740746379,-0.5924335122,-1.7502161264,-0.0792060792,-1.3580681086,1.1974959373,1.0274500847,-0.8471487761,1.3548214436,-1.0140715837,0.9863840938,-0.9353408813,-0.8694415092,0.3170388937,1.4484037161,0.6906737685,0.8037082553,-1.4583121538,-0.3531333804,-0.3408255875,1.5303416252,0.5340528488,0.6250340343,1.3767137527,-0.5115512609,2.1108081341,1.5340073109,-0.0016632290,0.0813266411,-1.1583846807,-0.7179369330,-0.7177065015,1.1551008224,1.6540576220,-0.4208811224,0.4496422112,-0.9338514805,-0.5616552830,0.6697068214,0.3913882077,-0.3269231319,0.2208180726,0.5489981771,-0.8624801040,0.0121480618,1.1665678024,0.1535249054,-0.1479054093,-1.0810278654,0.5306854844,-0.4219224155,-0.4571480155,-1.6084355116,0.9218780398,-0.2359026372,0.2739748061,0.2933753133,2.9100708961,-0.9172291756,1.7333257198,1.4234604836,-0.3695767522,1.8046455383,0.5196769238,-0.0223775655,-0.0609745122,0.4093607068,-1.4313142300,0.4745879769,-0.2658665776,0.1218895316,0.0092011830,0.6915896535,-1.2806612253,0.4025485814,1.1575285196,0.4067355394,0.9415536523,0.4428307414,-0.3000774086,-0.9426550865,-1.0320085287,0.7782731056,0.6642094254,0.8447062373,-0.7672371268,-1.0206696987,-0.1935468763,0.6972218752,-0.8549840450,0.0458985455,-0.0316515081,-0.9947295785,0.0599444099,-1.6356289387,1.4886174202,-1.1804332733,-0.5649894476,-0.3141291738,-0.5540065765,-0.0268782862,0.1907479763,0.3300601244,-0.0094888257,1.2847651243,0.7848905921,-1.8625837564,-1.5045292377,0.7757042050,-0.5151198506,3.4884924889,0.6410551071,-0.8507729769,0.2530885935,0.4635319412,1.5692179203,1.8518171310,-1.3878117800,0.6403602958,-1.1527994871,0.5967409611,0.1180173531,0.5744929910,0.2153960019,-0.4859421551,0.2957971692,0.4451091886,1.2154741287,0.3185328245,0.1403361112,0.0406874493,0.5657766461,-2.6195344925],[-0.7967351079,-0.9214567542,-0.7220990062,1.1061993837,-1.4456450939,-0.2186345011,-2.0822770596,-1.3654150963,1.8305156231,-0.0043937741,-1.7268742323,-0.4598450661,2.7442691326,-0.7711686492,0.2119555622,0.6657399535,1.8725705147,0.8713087440,-0.0707813352,-0.2834530771,-0.1813057214,0.8837855458,1.0758835077,-0.0943579450,-0.2998608947,-1.2143330574,0.1564082950,-0.6589041352,-1.5153027773,-0.4579784274,1.2980852127,0.2589216232,-0.6551944017,0.5270182490,0.7990094423,0.5013885498,-0.5969684124,0.9251852036,0.1497516632,0.3835472763,0.2245106548,0.3399053216,0.9988560081,-0.2961629331,-1.2441223860,-1.0278291702,0.6651023030,-1.1447552443,-0.9833480716,-1.8347622156,0.7757753730,0.4770247340,-1.5743138790,-1.1904482841,-1.2004079819,0.4256134331,1.4869117737,-0.0860315859,-0.2575958669,-0.0120158978,0.4862571061,0.0525990017,0.4884842336,0.2983732820,-1.5491154194,-0.9905139804,0.6269329786,0.1086431965,-0.2456014603,-0.1656596661,-0.9914613366,-0.2911224961,0.3206650615,-2.0059871674,-1.4480019808,0.7958505154,-0.9722334743,0.0591765344,-0.6312087774,0.2814213037,-0.4718524516,-0.4777416587,-0.8816665411,0.5701669455,-0.8234921098,-1.6473771334,-0.2114145905,0.7337220311,1.7068860531,0.1092660576,0.5084820390,-1.0156943798,-0.6823235750,0.4121778905,-0.0907771587,0.0038814421,2.6090002060,1.1189103127,-1.9125928879,-0.3776756823,0.4361971617,0.0557256676,-0.0821820721,-0.7631011009,0.4519874156,-2.4382345676,0.1086765230,0.7393144965,-0.0481885150,-0.3635574579,-0.3686617911,-1.4666473866,1.1164562702,-0.9669422507,0.7040798068,1.2670004368,-1.7580957413,0.0831064582,1.8570041656,-0.5323807597,0.7726399899,-0.5263054371,-1.6371046305,1.4653873444,1.3102437258,-1.8714300394,0.7401259542,1.4640793800,-0.3641062081,0.5393807888,-0.4796035886,-1.0403650999,-0.4575759172,1.8171423674,1.2859230042,0.8897191882,1.0549505949,1.3600300550,-0.3504465818,-0.6137373447,1.4029437304,0.7413427234,1.4288338423,1.3940635920,1.4501439333,0.7185511589,0.4196183681,1.1800745726,-1.0972715616,1.0779728889,-0.3046537638,0.1432365626,-0.7692264318,0.4684534371,-0.4527581930,-0.0037421312,0.9633156061,-1.6476392746,-0.6606883407,0.7935672402,1.4174605608,1.2771332264,0.1160060540,0.8817456961,1.2700384855,0.5530409813,-1.2621611357,-0.3488104641,-0.0174375270,0.5478835106,-1.4185255766,0.0217602048,-1.1966099739,-0.6348711252,1.1889677048,0.0976005048,2.5740497112,-0.3773222566,-0.4613899291,-0.0986436456,0.0541701391,0.2876920104,-1.7548836470,0.9253419638,0.2831991613,1.9411002398,1.4710776806,-1.4093031883,-0.5841726661,0.6404128671,-0.8920534253,-1.5517531633,0.4828912020,-0.2645669878,1.1087429523,-0.2828297615,-0.0158070270,1.5413781404,-0.8813383579,-0.4438351989,1.0622408390,0.9529253244,0.6253203750,0.9075292945,0.9945212007,-1.6742419004,-1.6183062792,-0.2126778513,0.6402559280,-1.2418143749,1.5323899984,1.8869425058,-2.1036224365,-0.2886890471,-0.2596866786,-0.2835617661,-1.5854977369,0.6081105471,-0.1809121221,1.2577123642,0.2052631229,-0.8163881898,-1.1063590050,-0.0912579298],[0.3450472057,-1.5712219477,0.8036522269,1.0187789202,0.4720780253,-1.2311170101,0.5175920725,-0.4290986657,-0.3666216731,0.6965180635,-0.1685263962,-1.5898982286,0.0220934488,-0.1997891963,-0.0530815497,0.8660960197,1.2025402784,-0.2280056924,-0.4191038013,0.9244895577,0.3037866652,-1.0225591660,-0.3245607018,-2.1816282272,-1.6213059425,-0.0621134415,0.5913257599,-0.1323050410,1.2850565910,-0.9554300308,0.2916534841,-0.1600199640,-0.0251585990,-0.8426494002,-0.2910325229,-0.0537761226,0.9424139857,0.6624855995,-0.5286464691,0.7076581120,-0.6090992689,1.5335679054,0.0755734891,-0.1748557091,-0.2534921765,-1.8603761196,1.1692599058,0.6042902470,0.9103702903,-0.1232990026,-0.4136359692,0.4350080490,-1.5793688297,0.6769706011,-0.4317162931,-1.4103405476,2.2864291668,0.0951946005,-1.4713804722,1.1892334223,1.1890237331,0.4429376125,-0.8387430310,0.0935311168,-0.7773849964,-1.8710905313,1.7024439573,-0.3445017934,-0.4553095698,0.7642885447,0.9821681976,-0.4583004117,0.0066860346,-2.2333476543,-0.1283345819,-0.5246235728,0.9105899334,-0.8629112840,-1.2106477022,-0.4193559587,0.0832006633,0.8583486080,-0.7869291306,1.7270901203,0.6822558641,-0.7230071425,-0.9777082205,-0.3344391882,-0.3701584637,-1.7645907402,-0.4649545252,1.0170435905,-0.0217694566,-0.9039642215,-1.6144140959,-0.7381874919,-0.6702867746,-1.3525016308,-0.4430875778,0.7287974358,-0.2385735363,-1.9752480984,2.0346944332,0.7689108253,-1.3528065681,0.6410573125,1.3192390203,0.3270509541,1.3804658651,0.2953875363,0.2034599483,-0.0645830110,-1.0361405611,0.5138510466,0.4093931317,-0.1102928594,0.0347731411,1.5495173931,0.4664959610,-0.3283931017,0.8254403472,-0.5107836723,-1.5615645647,-1.3665698767,0.3007652760,0.2336972654,-2.3706405163,-1.4505566359,0.3662475348,-2.1614456177,0.1082526296,0.0599631332,-0.4569309950,-0.0464944579,-0.8626391292,-0.5210534930,-1.3991318941,-1.0450747013,1.2357361317,0.7472088337,0.0294626709,-0.0323381051,0.8234859109,-0.6799733639,0.4621934593,-2.5489547253,-0.9245632291,-0.7865313292,0.6090202332,-0.3597594500,-1.5277130604,-0.7501274347,0.0724710599,0.3506417572,-0.0656960309,-0.4374740124,-0.4982353151,-0.1413731575,-0.6359506249,0.5705936551,-1.5379847288,2.0364022255,-0.4719463289,1.2827460766,-0.0661825612,0.2151302248,2.1430366039,-1.1678416729,-0.7821418643,-1.0774587393,0.0208406113,2.9643104076,-0.7014997602,-0.6581448317,0.8276968598,-1.2026946545,0.2845969498,-0.2574639916,-2.1206636429,0.2223912179,1.0973002911,0.5591518283,0.4466454387,0.2330162525,-0.7495750189,-0.0029619916,-0.4267501831,-1.6162149906,-2.1914863586,-0.6577326059,0.4989123344,-0.6556199193,-0.0895094723,-0.0112790493,0.0701447576,0.2679291070,1.5603930950,-0.9185328484,1.2158505917,0.6496582031,0.9756454825,1.3745002747,-0.8222441673,0.2244423032,0.1343432516,-0.3055539429,-0.9232123494,-0.1694074422,-1.1765798330,-1.1712293625,0.3721295893,-0.6939271688,-1.7612162828,-1.1052387953,-0.8033535480,-0.2181822509,-0.8658139706,0.2206589878,0.6181307435,0.5405413508,0.5935147405,-0.4217613637,0.4479676485,1.5241919756],[-1.2515456676,0.3748424947,2.0765490532,0.3203200698,0.8813945651,0.8924462795,-0.1924320161,-1.5929996967,-2.3032474518,0.1078558564,0.6173242927,-0.8837097287,-1.4029815197,-0.5150201917,0.1820349246,0.8237689137,-1.0956190825,-0.6264654398,2.0228900909,0.5752149820,0.0658197179,1.3183259964,-0.6357634664,0.8268523812,-0.3430708945,-0.0185604282,-0.7815250754,-1.1980940104,-2.0872871876,0.4336906970,-0.4385131896,-0.5323140025,-0.3113310635,-0.3358350396,-0.8802808523,-0.1359132677,-1.0820661783,0.6475699544,-0.2714052200,0.3617736399,-0.2199012190,-0.2596733272,-0.9010024071,0.7449892759,0.4877879322,-0.9734915495,-1.0764521360,0.6607186198,1.9671131372,-1.5323868990,-1.6164841652,-1.3796384335,0.9423570633,-0.2951205373,1.3519340754,-0.2381277531,-1.0184181929,-0.8733766675,0.4789345264,-0.8836178780,1.3279944658,-0.9801152349,-0.1271719635,0.2661816180,-0.9542296529,1.6332507133,-2.5235753059,1.0538122654,-0.7360922098,-0.3780718446,1.5566005707,0.2872518003,0.8437467813,0.3021098971,-0.5063041449,-0.5947359204,-1.4161554575,0.5075585842,1.0832319260,-1.0159112215,-0.7424957156,0.4478953183,0.6075967550,1.0550105572,-0.1584615558,-0.7829976082,-0.7657365203,-0.1861712188,0.2837609649,-0.3579142988,-0.1085585579,-0.2172659487,-0.5554970503,1.2288670540,1.9843388796,-0.7815829515,1.4802480936,-0.8235428929,0.8025032878,2.2790677547,0.9906148911,-0.6867852211,-0.7537479401,0.1955792606,0.0749062449,1.9827543497,0.9090567827,0.9319398403,-0.7689310312,0.2998859584,0.0846697241,-0.3599868119,-1.5779273510,-0.6001316905,0.7877557874,0.0443977341,-0.5797268748,0.2533348501,-0.2036195248,-1.3632600307,0.8793420196,-0.4388233125,-0.7233103514,0.5692442060,0.4922772646,-0.1630162150,0.3571025729,0.1961718649,0.2645778954,-0.7818415165,0.7580390573,-0.0369912013,-0.7612245083,-1.0428425074,-1.6256395578,1.5037935972,-0.1167479455,-2.5956823826,-0.7661041617,0.3693991005,0.4504629374,0.3117121756,-0.5613887310,-1.1832345724,1.2855526209,-0.6030053496,-0.6560151577,0.1852611452,-0.5990884304,2.4433631897,0.0511516444,0.4812693000,-0.0832996145,0.2482223809,-1.9432893991,-1.6592720747,-0.4858410358,0.5000698566,0.1994387358,-0.1384883970,-0.9286952019,1.1757085323,1.1086024046,-0.8868203759,-1.0367414951,-0.1277745217,-0.8232452273,0.4371286631,-0.1243734136,-0.3757143617,0.1735518128,1.7695791721,0.0277947448,-1.3792670965,0.3176753223,1.3203235865,-0.3832557797,-1.4734238386,-0.2980696857,-1.0584447384,-0.5828962922,0.5312491655,0.3283676207,0.9943018556,-0.7106354237,-0.9073405862,-0.5333749056,-1.0341758728,0.1330567151,-1.0525498390,-0.2770160139,-0.2599232197,1.0627063513,-0.0400909074,0.4328006506,1.7536995411,1.3945586681,1.4096744061,0.2681570947,-1.3833025694,0.3761285543,1.2858004570,0.6580190659,-0.1391668916,-2.1075122356,-0.6969650984,-0.4540668726,0.2670806050,-0.2743560374,-0.0209443346,-0.7997515202,-1.1989635229,0.0803686678,0.7327975631,-1.4023789167,0.1931434423,0.9743856788,0.8532840014,1.1824666262,-0.7256034613,1.2055327892,0.5179441571,0.4593677819,-1.0894452333],[0.9806094766,-0.2745089829,-1.4509284496,-0.4190232158,0.7034731507,0.5885317326,-0.8590492606,-0.1408797503,-0.1429534405,-0.1686502695,1.5083961487,0.0664177239,1.6594139338,1.6427066326,-0.3360191584,1.2271697521,-0.6477968693,-1.8092172146,-0.9437155128,0.2422796041,0.9709712267,1.1938385963,-0.2580324411,0.2235572189,-0.0080338093,-1.0137414932,-1.0294257402,-1.2949119806,0.4745123088,2.0134837627,-0.3586051464,-0.7135395408,-1.3953684568,0.3262714148,-0.9641054869,1.4204113483,-1.4024853706,2.1189253330,-0.5846406817,-1.1655416489,-0.8957322836,-1.1574605703,-0.1748732775,-0.4009687603,1.1054528952,1.9881278276,1.3893102407,-0.7504013181,0.7380410433,0.2013456821,-0.4352020621,-0.1771009415,1.5475465059,-1.0222883224,0.6397832632,0.8793185949,-1.1897361279,2.3307409286,-0.0984229222,-1.2100880146,-0.3881158829,-0.5199695230,0.4241195917,-0.0116548957,-1.4528745413,0.6298332810,-0.6663973927,-0.4127940536,1.3213293552,2.1988401413,-0.1545394957,0.6720430255,-1.5908136368,1.1947685480,0.6126347780,0.2747550309,-0.6444818377,-0.3922859728,-0.2779766321,-0.5158533454,1.0888648033,-0.4051799178,0.0381805673,1.0520000458,0.4421851337,0.7295815349,0.2826419771,0.3008019626,1.0624518394,0.0210728142,1.0455101728,0.0507967658,1.4170062542,0.6779723763,0.1280576140,1.0865745544,1.4637603760,0.8787689209,-0.3339741528,-0.9834817648,1.2178926468,0.2774706781,-0.3207702339,2.3072702885,-2.0267996788,-1.0228863955,0.2627939582,-0.6198943853,0.2880648971,0.1017346680,0.0353729799,-0.5069180727,-1.6419990063,-0.7368588448,-1.0442868471,1.7887995243,-0.0079993038,0.1718894988,-0.6887407899,-0.3678089082,-1.5573518276,-0.4891292155,-1.2914336920,-0.5052575469,-0.6315882206,-0.9011530280,0.1310378164,1.2318689823,-0.7219327688,0.8740233779,2.0904352665,0.2487895340,-1.0377489328,-0.4964048266,0.8279573917,-0.3556987345,-0.4572505951,1.0447623730,-0.7082029581,-0.1460758001,-1.6420407295,-0.5102956891,-0.6610686183,0.6488505006,-1.1323249340,-0.3774205446,0.6173532009,-0.2405151129,-0.0985287949,-1.2055841684,-0.2576752603,-0.3729512990,-0.9217440486,-1.7550470829,0.4352207780,-0.4867249429,0.5257884264,0.6555952430,-0.1585087776,-0.3430123627,-1.8562735319,-0.3113770783,0.5841275454,0.5621509552,-0.3464486003,-0.5415142775,0.0917498544,0.2995837927,0.3927473724,-2.8245084286,-0.3212784827,-0.3785587549,-0.7113556266,-0.6022580266,-2.6119797230,-0.0254186876,1.0269163847,-0.5338106155,-1.4207351208,0.8609966040,0.9684557319,-1.1536953449,-0.1923975646,0.7043721676,0.8859874606,0.4263644814,0.8193140626,-0.1051984951,0.1981133819,0.6787722111,0.4743790627,0.4614576697,0.4642948210,-0.9053800106,0.2947457731,-1.3450095654,0.4394025803,0.4799166918,0.4150917530,-2.1966297626,-0.0429796278,1.6986823082,-0.5377891064,-0.6587689519,-0.8812151551,-0.5194984078,0.0733721107,0.5582324266,0.3964279592,0.2120293975,0.3054826260,-0.2871125340,0.7834923267,0.3903766870,-0.0846089721,-0.1705707610,1.2015510798,-0.6100164652,0.5426404476,0.0102491574,-1.3187043667,-0.3852408826,0.4882010818,-0.6264355183],[0.6432433724,2.6491410732,-0.7734444737,-2.0508220196,0.3473735750,0.4667849839,-1.8796936274,-2.5656268597,0.2532318532,1.7867356539,1.5840818882,-0.9908537865,-0.1217880398,0.1486516148,0.5512774587,-0.2852463126,-0.7899572849,-0.3199961782,0.0614964701,0.0901733115,-0.3096054196,-0.1612850279,1.2000919580,1.2057310343,0.1887879372,0.4619420469,2.2673599720,-0.0099240970,-1.8448913097,-0.8179546595,-0.8862370849,0.2955748141,-0.1304718554,0.9787134528,0.8472347856,-0.1288524419,0.5706324577,1.2143545151,-0.5389542580,0.1138983741,0.7284115553,2.3312344551,1.2056900263,-0.9562098980,2.7902858257,0.1274458468,-0.5790623426,0.5577117205,0.6300501823,-0.1559722424,0.5293494463,1.9820027351,-1.3815310001,1.7998952866,0.7246081233,-0.1543004215,0.6131431460,-0.6942011118,-0.3525010645,-0.5144149065,-1.2278832197,1.4018414021,1.7004867792,0.6758140326,-1.8631590605,-0.0994367823,-1.4539935589,-0.8443304300,0.0253531449,0.1577637196,-0.8598079681,-0.1473255455,0.8919650316,-0.7243660092,-0.1229981408,-1.6209475994,-1.6732126474,-0.8255846500,1.2784086466,-0.3470903039,-0.1857288927,0.4499431551,0.5625912547,1.1732017994,0.1230131313,-1.8694161177,1.3007078171,-0.3756277859,-0.2890734673,0.0999187604,0.0808265805,1.2099618912,1.0885607004,2.2550585270,0.8889443874,0.7035228610,0.8246964216,-0.6405169368,-0.3325651288,0.0771268159,-0.6547657251,-0.8956646323,1.3768603802,-1.4912687540,-0.2551057935,-0.9616905451,0.4821599722,-0.1796675920,0.2831281424,-0.1340553463,1.0645695925,-0.6067219973,0.8581216931,-0.8533557057,0.1947769821,-1.3405125141,-0.0545026809,-0.0804695264,1.3081027269,-0.3672732115,-0.9950604439,-0.1673544943,-0.6271149516,1.0561723709,-0.8029696941,0.1047154367,-0.6536308527,0.4128457010,-0.3468890786,0.2208393216,-0.3895644844,-1.8310519457,-0.2431140691,0.1881937236,-0.7013317347,0.0772262961,0.3569529653,-0.5789977908,0.8872517347,-0.5732261539,0.4532896876,1.1999888420,1.3070069551,0.5669031739,0.6928222179,0.1783494949,-0.8095983267,-1.6969149113,-0.2449482530,0.8840457797,-0.4317087829,0.2946268618,-0.7384603024,0.8370640278,0.2964679301,0.6661210060,-0.0482210517,0.2334891409,0.2837832868,0.5236741304,-0.4918816090,-1.4834574461,1.5049879551,0.3132415712,-0.0655054599,0.2337341458,0.5793153048,0.4204308987,0.1527949274,-0.5522327423,-0.3710898757,-1.4416307211,0.3492567837,1.9169337749,-1.1547886133,0.1601953954,1.6751732826,-0.0111167338,0.3223155439,0.7473372221,-0.8906780481,-0.5188031793,-0.2556838393,0.7918527722,-0.1100981310,0.1180559546,0.8671013117,0.9141765833,-0.2264145017,0.9463275671,1.2277970314,-0.2910000980,0.3737040758,0.0451757126,0.0928840712,0.1828407347,-0.7114292979,-0.7026841640,-1.1814341545,-0.4881572127,0.0155485887,-2.7573442459,-0.6148686409,0.0310324896,-0.7967752814,1.1301997900,0.8300131559,0.2524907291,-1.0185047388,0.6041662097,-0.6465168595,-1.6469916105,-1.1552380323,-0.2510072887,1.3071713448,0.2094026506,-0.8110154867,1.0590174198,2.1924555302,-1.1202188730,-0.8820242882,-0.2305018008,0.4774096608,0.4056183100],[0.3316685855,0.7246740460,-0.2441137731,-1.1167860031,-1.3302159309,2.1725409031,0.8256790638,0.8306998014,0.0433921330,-2.6837403774,0.1183398664,-1.5742039680,1.3130691051,0.5735969543,1.3894512653,-0.2349755168,-1.4868464470,-1.5704554319,1.2836638689,-1.4032113552,0.0997329354,-0.3284315169,-0.7569980025,0.4101175666,0.2967536747,0.0999957025,-1.7917844057,0.1166789830,0.9223045111,-0.5208253860,0.3779269159,0.3759181201,-0.9334786534,-0.4578769207,0.3737753034,-1.1865646839,1.4694766998,2.1069328785,1.4723044634,0.8738681078,0.8008302450,-0.6091771126,0.6025853157,-1.7750355005,2.5449066162,1.9469372034,-0.0892248526,-0.7289734483,1.7631760836,0.3075451553,-1.0003085136,-0.4178240597,0.8397405148,-1.3949640989,1.2439329624,-1.2816982269,0.5767304897,1.1978013515,-0.0095124440,1.4640440941,0.9479848742,0.4466987848,-0.4483757317,-0.1509431601,1.0327948332,-0.4387851655,-0.7053198814,-1.7758510113,2.3635933399,-1.2871538401,1.2330980301,-0.0103951599,-2.0432476997,1.1372212172,-0.2039745152,1.1895712614,2.1988720894,0.4431562126,-0.6325745583,0.7957141995,1.5974450111,0.2394031286,-0.7119681835,-0.5758239031,0.2845700085,1.1215269566,0.3454354703,-1.1157178879,0.7630261183,-1.0396157503,-1.2194751501,0.4127437174,0.8683571219,-1.4072741270,-1.0504204035,1.4434605837,1.7263585329,1.0688557625,-0.6204966307,0.2786242962,0.0499957725,1.7350540161,1.0227462053,0.0887895748,-0.1811278611,0.0582000278,-0.4657486677,-1.7719552517,0.8368805051,-1.1324976683,0.3575865328,0.5382347107,-0.8788229227,0.5577881932,-1.4245994091,-1.3215843439,1.1583495140,0.6064372659,1.8785479069,1.2669287920,-1.6431622505,1.4687012434,0.1747233868,-0.7408152819,-1.7918393612,0.0706328824,2.3624007702,1.3255273104,-0.8967608809,-1.8747004271,0.0701705366,-1.8117138147,-1.1072317362,0.4218684733,0.9634417892,0.2764050663,-1.0511800051,1.2283619642,0.5277022719,0.9080645442,0.8044661283,0.7120621204,1.1845030785,0.6737482548,0.2570166290,1.1729450226,0.9283907413,-0.2583913505,0.2896552980,0.4090211391,-1.5187941790,-0.1357347518,-0.4136520922,-0.9496188164,-0.7845684290,-0.6877620220,1.4111977816,0.7903782129,0.2329047620,2.4450430870,-1.2297238111,-0.3623290360,-0.1056897566,1.4517501593,0.0444726944,-1.6526907682,0.1572376639,0.8969747424,-0.4539208114,0.3395579457,-0.7769137025,-1.1884061098,1.7207614183,-0.2107622027,-0.3655122817,0.7336954474,3.9021322727,-0.1486244053,0.0116885044,-1.4245262146,-0.8629705906,-1.4355378151,1.3567553759,-1.2594320774,1.1831315756,-1.1354767084,2.2162899971,1.4533467293,-0.6889243126,0.6834006310,0.1057725549,-0.8852538466,0.1678477079,-1.3591594696,0.5192753077,0.3029523492,0.7172673941,0.9541074038,-1.1572390795,-0.2347368300,-1.7002626657,0.2449831665,-1.1469874382,0.8641168475,1.4774310589,-0.0520692803,-1.3869181871,-0.2344624996,0.1029674485,0.8140225410,-1.2963079214,1.6603183746,0.0933022127,-1.0267788172,0.7393246293,1.4649448395,0.3346224725,-0.4709796906,0.3763844967,-0.2554529607,1.2848366499,-0.1008833647,-1.4221745729,1.3263037205],[-1.5512310266,-0.0674574524,0.1974201351,1.9573367834,-0.5765719414,2.5311250687,0.0921669155,0.5961879492,-0.5765373111,1.1144328117,1.0982080698,-2.1409747601,1.4250926971,0.9062033296,-0.4229671955,-1.7703527212,-1.4376366138,-1.2039889097,0.4187853932,-0.9995331168,0.0991715044,0.5590418577,-0.3176335394,-0.7645334005,0.1076866239,-0.2528138459,-0.5841336250,-0.4563258290,0.6959413290,-0.3995080888,-0.2161872089,-2.4993913174,0.1716826558,1.9722093344,-0.1077965498,-0.2359120846,-0.6924694180,0.3745718896,-0.6471085548,1.1404736042,1.0013725758,0.0168315694,1.0300970078,1.0599291325,0.0489125997,-0.7580710649,-1.4751648903,0.6038507819,0.0657226518,-0.1146567091,-1.2612990141,-0.1839995682,1.0401122570,-0.4304883778,0.1605222672,0.3470155001,-0.2415146828,0.6425074339,-1.8182214499,0.5159596205,0.1558146328,0.0451287888,-0.0311791990,1.4505674839,-0.9712655544,0.7230480909,-0.5097474456,0.1868300736,-0.7318282127,-0.4006540775,0.0385716334,-1.0778014660,0.9501018524,0.2879245579,1.0167944431,-0.4306978285,0.3810772598,2.1066005230,-1.9030117989,1.2764573097,1.3993682861,-1.0606230497,-1.3270933628,0.8174833655,-0.8962209225,2.0776956081,-0.2050100565,0.9393951297,1.3354527950,0.9743983150,-0.3406037092,-0.5477521420,2.6748812199,0.4349660575,2.3099956512,0.7869509459,-1.4177607298,0.9141976237,0.6136193871,0.6456041932,-0.6823461056,-0.4874903858,-0.1170524731,1.9651896954,0.0810671002,-2.3023612499,1.1835066080,0.8766115904,-0.9631479383,-2.1853756905,-1.2428277731,0.0720189661,-0.4796108902,-0.1266144216,-0.9395406246,0.8748876452,0.4869423807,1.9948581457,-0.6947699189,0.2270700186,-0.3317213953,0.6422330141,-0.4053525925,0.6357408166,0.1552224904,0.1389403045,0.9798236489,1.6588976383,0.6248997450,1.3167870045,0.1932007372,-0.8761605620,-0.5672311187,-0.2326073647,0.3890257478,0.2928788364,-1.3203250170,-0.9401980042,0.3664203882,-0.5633349419,1.0236297846,-0.0916399956,-1.0432430506,-0.7348905802,-1.4612178802,0.0253731403,-0.0812676325,-2.3844251633,2.1396183968,-0.3534555733,-0.5941804051,-1.4541046619,0.3458630145,-2.0107643604,-0.4030187726,-0.5923594832,0.2931079268,-1.4421180487,-0.9520923495,-0.9846225381,-1.5098954439,-0.7086515427,-0.0347315706,0.2972258031,1.1616653204,0.1510649323,-0.5060365200,0.0093106842,-0.0073053623,1.0132656097,-0.3046438992,-1.3726711273,0.0134564815,0.8839237094,0.9101039767,-1.4731736183,0.2599923015,-2.1118168831,0.9287536740,-0.9762788415,0.2646948993,-0.3983414769,0.8249574900,1.6295404434,1.0062322617,0.1468868256,1.0601128340,-0.1219236851,1.4774035215,-0.1063908115,0.3392222524,0.2582385242,-0.8551683426,1.0945085287,-1.0423394442,-0.4286342561,-0.9640535116,2.5036330223,-0.0431590155,0.9850620627,-1.0826802254,-0.0051798890,-0.8156575561,-1.6702399254,0.6556189060,-0.6202479601,0.8388648629,1.3374948502,1.1133385897,-0.6639132500,0.8211407065,0.7570949197,0.6676589847,-0.1204034463,0.2454028726,0.6149377823,1.1963888407,-0.5744004846,-0.7452492714,0.1945770830,-0.9737981558,-1.4163233042,-0.4329152703,0.0736613274],[1.3573538065,-0.1728477329,0.2341295928,-0.1851958036,0.8775706291,0.1613099128,1.1886209249,0.9928414226,0.5726636648,-1.0495097637,-0.7697876692,-0.0529748313,-1.2455561161,0.8108440042,-0.8752728701,0.4357014596,-1.5372251272,-0.7729937434,0.2567974627,0.4485482275,1.8845136166,0.2564440370,0.1844030470,1.7274739742,-2.3057599068,0.7775979042,0.5623003840,0.2061211616,-0.5265920162,-0.3165593147,-0.7587300539,0.5017391443,-0.0206113029,-0.5766850710,0.9890679717,-0.6054541469,-0.4035939872,-0.5519093871,0.3898775876,-0.8180458546,0.8589459658,-0.4786759317,1.0392462015,0.2402326465,-0.3096106946,0.2151809037,-0.6961288452,-0.4223067760,1.2795596123,-0.2458327711,0.1790923178,-0.7446967363,1.5123780966,1.8332709074,-0.1309396625,-0.5887723565,-0.1028404683,-0.8091694713,1.5737030506,0.0129924612,1.7106083632,0.3016880155,1.5860959291,-0.7288613915,-1.2153372765,-0.2323260754,-1.5699255466,0.4336345196,-1.5333116055,0.0587876514,0.6924380064,0.9327794909,-0.1513533443,1.1799548864,-2.2671871185,0.1525774598,0.8967125416,0.6222859025,-0.2084840834,-0.1403931379,0.1346957684,-2.0565283298,0.4950318336,-0.4319967330,0.9162702560,2.7992014885,1.5372692347,0.8877406716,1.0195660591,-0.4253476262,0.5052951574,-0.8647077680,0.2749290466,0.1537522674,0.5428519249,-0.6285081506,-0.4747620225,1.7780978680,0.5352087021,-1.5661109686,-0.5283264518,0.0906709209,2.0831224918,0.7579495907,0.5779919028,-0.2779310644,0.1220835596,0.6336193681,-1.0817463398,-1.1549593210,-1.4704773426,-1.1104469299,0.3906410336,2.1418421268,-0.4694980979,1.9122390747,1.2186927795,0.2590028644,-0.0949972644,0.5684119463,-0.9523119330,-0.1190907806,-0.6136029363,-0.0960134193,0.6986835003,-0.9704755545,-0.6268044114,-0.5610452294,0.6267525554,-0.4279605448,-0.3012984395,0.0593986809,0.9925241470,1.3790812492,1.0318965912,1.0415772200,0.2689839602,0.3476128876,1.5841513872,-1.3008955717,-0.6530005336,0.2141337395,0.9915444255,-0.1424658149,1.3862416744,-0.9926421642,1.7042853832,-0.8251213431,-1.5188190937,0.4262528419,-0.4792524576,-0.9867950678,1.0061618090,0.5120648146,0.5273358226,-1.1984391212,0.7163698673,0.6981152892,0.1147046685,-1.1462678909,-0.7785617113,0.9478487372,1.5295920372,1.4200506210,0.3724110425,-0.5420010090,-0.4563983381,0.6563023329,-0.4331812859,0.0931108072,-0.1951667070,-0.3696748018,-0.5538032055,-0.6306383014,0.8567004204,-0.2205395103,-0.8111694455,-0.8887060285,1.2530621290,-0.3134068251,2.2097487450,-0.7303216457,-0.8344842196,-0.4096687436,-2.4196662903,-0.0106813936,0.4485253394,1.8782069683,-1.8911976814,-0.1464397758,-1.3518520594,0.7725373507,0.0136093041,-0.6495004296,-0.5794745684,-1.0514913797,-0.2449943721,0.5634900331,2.0310289860,-0.7452318072,-1.1391593218,0.1187069863,0.6543679833,-0.0734154284,-1.2560356855,0.7408869863,-0.4925400317,-1.0945181847,0.0686069354,0.3952773809,-0.1935277283,0.4949653149,2.5847566128,-0.6973088980,-0.1285281926,1.4140560627,-1.1363401413,-1.0524957180,-0.5918847322,-1.4523485899,-1.9308159351,0.0701744705,-1.3957195282,1.1885744333],[1.2093832493,0.2166377157,-0.3030340970,0.0597176440,-0.5784593821,-0.1520032585,0.0584271438,-0.8972605467,0.6736978889,0.6967254877,0.4667727053,0.0286635868,0.2837365568,0.4581332207,0.3056941926,-0.0116381422,0.9084653854,-1.3365603685,2.8502945900,1.0096521378,-0.8975444436,-0.3106841445,-0.1088178307,-0.8972753286,0.9920865297,-0.0723804459,0.5263554454,-1.0233750343,1.2341985703,-0.3395036757,-0.0130303865,-1.0015542507,-0.3587833345,-0.5115778446,-0.9865487814,-0.1158994958,0.8093836904,1.4579401016,0.3082989156,0.2266820669,0.9964211583,0.0073694857,-2.0784134865,-1.7294149399,-0.4265480638,0.4939056933,0.7890198231,-0.4054355621,-0.5412698984,2.1344156265,0.7287756205,-1.2158207893,-1.0318105221,0.6429752111,2.7805309296,1.1857364178,-1.5435954332,-0.1381753832,-0.2538266480,1.1751824617,0.0941778049,-0.4272075593,-0.4489406049,-1.2893304825,0.3536476791,-2.1066591740,-2.2367551327,-0.5366755128,-0.5199564695,-1.3183445930,0.8450385332,1.8122462034,-0.1849884093,-0.5415422916,2.2770223618,1.5768443346,-1.4913719893,0.6609851718,0.9181861281,-1.9391117096,-1.4061818123,0.2801476419,-0.3844480813,-1.1504819393,0.3682684898,-0.6189881563,1.1590030193,2.6743111610,0.4546943903,0.2690356374,-0.1853425950,-0.2681950033,1.4984539747,0.0044779256,-0.3983008564,0.1030340418,-0.0268113017,0.1630320698,-0.4327528477,-0.7730445266,-1.0588603020,1.5333759785,1.5113227367,-0.3076021969,0.1370654255,0.0941180363,0.8745601773,1.1456735134,0.4186946154,0.3277677894,1.2020785809,-1.2507486343,-0.4188847244,0.6161471605,0.3914790452,-0.9654144645,0.2758603990,-0.0457552262,0.6747549772,0.9733185768,-0.6183296442,0.0650520474,0.7692757845,0.5387182832,-0.8589666486,-1.7312017679,0.0673047602,-0.0327091403,-0.1677000821,0.6749193072,-0.8480613828,-0.4110155404,0.1744018346,1.9193584919,0.2909437716,0.2042945921,1.2015007734,-0.1746187657,2.0067448616,2.2541844845,-0.9878281951,-1.7081015110,1.0036889315,0.8530163169,-0.0468556173,0.9961929917,-0.9784662127,0.4863100350,-1.1680382490,-2.0089857578,1.6244140863,0.0916079283,-0.3821515739,2.1537449360,0.3477901518,0.4862693548,-1.6556663513,-0.9747734666,-0.6179747581,-1.8434002399,0.2441639751,1.2477802038,1.0027505159,0.7601565719,0.4941864014,-0.8031545877,-0.1345568299,1.3695721626,-0.6639223099,0.3247958720,0.0043942472,0.1913578659,0.0745675713,-0.6932394505,0.0181727502,-2.3302776814,-0.7034760118,-0.3380787969,-0.1769035012,0.7856540680,0.1328079104,-0.3098344803,-0.9356276393,-0.0556659177,0.2006022036,3.4013428688,-0.1138542965,-0.7904957533,-2.3656320572,-1.6644995213,-0.0752599388,1.2415909767,0.4848107994,-0.2712499797,1.0699157715,0.6268231869,0.6100659966,1.6827350855,0.2057249993,0.4664883018,-0.8904610872,-0.6790813208,0.4590415955,-0.0912852958,1.3914935589,-0.7131012678,0.2295179665,-1.3685091734,0.4473334253,-1.7683360577,-0.5036785007,1.3914653063,0.5441012383,-0.5553440452,-1.2935613394,0.8311233521,-0.6089043617,-0.1083713174,-1.0062488317,1.0542385578,-1.4220438004,1.4223126173,-0.5420668721,0.7962802052],[-0.4968725145,0.7224124670,-0.9014311433,-0.0173226800,-1.8692378998,0.7442633510,1.1637054682,-0.9639398456,-0.1304800659,-1.3041782379,0.1481969059,0.1446544379,-0.3365060687,-0.5480625033,-0.5852605700,-0.5312329531,0.2836891413,0.4132503867,0.4652761221,-0.3472861648,0.1682231873,2.2558338642,-1.2319662571,0.5227214694,-0.4734502733,1.7072815895,-0.3913713694,-1.9416004419,0.1609739214,1.0534284115,0.6118378043,-0.8480007648,-0.6930434704,2.4445428848,0.3759610951,0.0456087664,0.3976513147,0.1482326239,0.0915998146,-0.1041516364,-1.3221958876,-0.8823789954,-0.9341890812,1.6293078661,-0.2838116288,0.0902046338,-1.2891747952,0.9338698983,0.0727711618,-1.2760754824,0.3651596606,-0.1811099648,1.4663398266,-1.3555800915,0.5154141188,-0.6398217082,-0.9735441804,0.9723172188,-1.4711934328,1.0343658924,1.1213243008,-0.2257323861,-1.5045515299,1.3349263668,-0.2323711216,-0.1518178135,-1.8108577728,1.8254790306,1.2762485743,-1.0044380426,-0.8060824275,-1.1536781788,-0.0594115667,-1.5260562897,0.6320920587,0.5781828165,1.8738344908,-1.7082090378,-1.2055032253,-1.6750184298,0.9773060679,1.7373665571,2.0059866905,2.3705921173,-0.9223784804,-0.6740148664,-1.4445827007,0.5108569860,0.4014914334,1.7079445124,-0.9667310119,-0.0251678526,1.2655253410,0.5475633144,-1.3689777851,0.9426730871,-1.2328010798,0.9257682562,-0.2959577143,-0.6005290151,0.5494224429,2.0729205608,-0.6353592277,0.3192138970,0.6131855845,0.1523712426,0.0587622598,0.2394811511,1.7130912542,1.2879834175,1.1557379961,-0.3637089729,-0.2145810127,0.9560777545,-0.0508564450,-0.1313996017,0.7307595611,0.5367805362,1.8643679619,-0.9024128914,-0.2627148032,2.0112290382,-0.5909074545,1.4932234287,-1.0802291632,0.6225048304,1.3895535469,-1.4506711960,-1.3074595928,0.2020994872,0.3432753086,0.2278840840,0.6360409260,-1.0779962540,-1.0535291433,1.5352840424,-0.2123942673,1.2253196239,-0.8935346603,-0.9941996932,0.3262534142,-1.1904532909,-1.7328804731,-0.2980934978,0.1107948944,0.5599442720,-0.4482952654,1.2198443413,1.1387603283,-0.1491803825,0.1041010097,0.4254328609,0.3654618561,0.9308492541,-0.3119447827,-1.3580926657,1.0101203918,-1.5451142788,0.2026536763,0.3477017283,2.3403952122,-0.0724377260,-0.1430427432,0.3848078549,-0.1609339267,-1.2239019871,0.0277106408,1.5609391928,-1.3043521643,-1.1645221710,-0.2278364003,0.7276721001,-0.1473096013,0.2433789521,-0.9658976197,-0.2630626559,-1.3427623510,-0.3987060785,0.6780408621,0.3158074319,-1.4230879545,-0.2691633701,-1.5128043890,-1.4413479567,0.0664281994,0.2924026251,-0.0307732169,-0.0908246115,-0.6818583608,-0.3328570724,-0.8593428731,-0.4004930854,1.3459855318,1.1941367388,-0.5967947245,-0.0030466982,0.7044922113,0.5484733582,0.1524624825,0.1162964776,-1.1390522718,1.6581176519,0.0612685196,-1.6157741547,-1.9226813316,0.7732533216,-1.3893855810,-0.9835121036,-0.4904064536,-0.7418014407,2.0079159737,-0.5394204259,-0.4050091803,-0.8270653486,-0.3081693947,0.2111128122,-0.2667162418,0.2949900031,0.1134409904,-1.0153115988,0.0493623950,0.2900373936,0.7097696662,0.7082123160],[0.3717222214,0.2401635796,0.0780070871,-0.9557723403,-0.1404657364,0.0148379868,-1.7255346775,-0.7337841392,-0.8126275539,-0.2047343999,-0.6537588239,-0.1437726617,-0.3824236989,-1.8198882341,-0.3698104620,0.7416635752,0.7381506562,0.3432342708,-0.6773697734,-0.5082986951,0.0642926022,-0.0629437044,1.2554669380,-0.1163146794,0.5489455462,0.6377999783,0.4149386287,-0.7792364359,2.1353731155,-0.5781716108,-0.8798395991,-0.9598072171,0.9588785768,-2.1271765232,0.6209558249,0.1527183652,0.3872463405,-0.5138054490,-0.0146184405,-0.2630323172,-0.9618073702,-0.3712114990,-0.8843356967,1.9723078012,-0.4195107520,-1.6190768480,-2.0020143986,1.8401244879,0.2102288306,0.0879932866,0.0495191440,-1.1159087420,-1.1406750679,0.5224584341,-0.2365093976,0.3528335094,0.5825575590,-0.0848055333,-0.1709183455,0.3926368952,0.7267042994,0.9824547172,2.0305740833,-0.3654267192,-1.7048155069,1.1908239126,0.5972874165,1.1509351730,2.3433780670,0.5069172382,-0.3373184502,-1.2413910627,-2.4824178219,-0.7841681838,-0.2749174833,1.4181344509,-1.1041259766,-1.2695769072,0.4413174987,1.4447129965,3.0843203068,1.1354111433,0.9596952200,0.0692582726,-1.8266723156,-0.1291946471,1.1994426250,0.1244117692,0.5631824732,2.2121365070,0.4624514878,-0.7957055569,-0.0160569772,0.1778953522,-1.3429569006,-0.0165645760,-2.1498680115,0.4199761152,-0.3032145500,-1.1267306805,1.4630862474,-0.0667405128,-0.1823576540,0.8276292682,0.2431415319,-0.0560194775,1.2737932205,-0.1540903896,0.5906959772,0.8791603446,-0.3834092021,0.0394203626,-0.5252980590,-0.2408296615,0.7998421788,-0.6245598793,0.8748078346,0.6978314519,1.8217324018,1.1725424528,-0.5058450103,-1.1825522184,-0.1523513198,-0.9517591596,0.3418970406,0.8446737528,-1.1665372849,0.3165706992,0.9696199298,-1.2162469625,-0.5531826019,-0.3029631674,-0.1280330271,-0.2350587249,-0.9241914153,-0.0322184227,0.0353361554,-0.5270760655,-0.2926283181,-1.0359057188,-0.5217298865,-0.0545432195,0.9752314091,0.4386629462,-0.1634233445,0.3504447937,-1.6003768444,1.5135135651,0.6002835631,0.8065099120,-2.4900567532,0.8633902073,-0.6167137623,-0.3883438408,-1.3907675743,1.3066443205,-0.7658559680,-0.5670421124,1.6960184574,1.9653801918,0.6198975444,-0.7111523747,1.6282720566,0.4226017296,0.2790413797,-0.9668534398,1.1160160303,0.6634340882,-0.2094600797,-0.4581094086,-1.0761604309,1.4729379416,-0.6079205275,0.6071959138,0.3631769419,0.0164318662,-1.3065373898,0.6455099583,0.1708582193,0.0014818548,-1.8358229399,0.8673004508,1.3817694187,0.7489470243,0.1228798330,0.7249348164,-0.2352134287,0.4495169818,-0.9030604362,1.0610578060,0.8425629735,0.6626394391,0.7027125955,-0.8264446855,0.4003860652,0.1967862993,-0.5250782371,-0.0714196265,-0.6816290617,-2.0005903244,-0.1332174838,-0.9352809191,-0.0467789285,-0.3418357372,-0.5217453837,1.4358524084,-2.8482091427,-0.1477034539,-1.8174880743,-1.4565387964,0.5249419212,-1.3412015438,1.0536248684,0.6991011500,0.9847779870,0.0895238072,-0.2042881399,-0.1511867195,-0.1911108941,-0.8483758569,-1.1394416094,-0.6263279319,0.3262504339,-0.4187115729],[-0.9000268579,-0.5546446443,-0.8508963585,-0.4535124004,0.0920558199,0.0759566426,0.2565580606,-1.8011915684,2.1070511341,-0.3016174436,0.5993249416,-0.6456158757,0.2734361589,0.2095994651,-0.9430122972,0.0983353779,-0.7922813892,0.6698214412,1.5299458504,1.0799561739,-0.4676948488,0.8438532948,1.4016919136,0.5662866235,0.6506659389,1.3225227594,-1.1737947464,0.0492078066,1.6108115911,1.2239793539,1.2101303339,-0.0170063470,0.1157550141,-0.5027875900,-0.5176725388,0.8887032866,0.5435528159,1.5120570660,-0.1064564437,0.6360615492,0.6020301580,0.8799878359,-0.0621630587,2.2621920109,2.6340875626,-0.5016446114,-1.1367765665,1.3416843414,0.4375230372,0.7510082126,1.6931953430,0.9512465596,-0.8695816398,0.9354771376,0.5091691613,-0.4463708401,-0.0330573730,-1.3820322752,1.2793287039,1.1265002489,0.3270427585,1.1429853439,-0.3402656913,0.3560847044,-1.0059175491,0.2028916031,-1.1150841713,-1.3619811535,1.0493425131,1.5576870441,-0.5290608406,1.1438901424,-1.5659375191,0.2392479181,-0.3553093076,0.0694101900,0.8125998974,1.6412402391,0.0621431805,-0.2580104470,-1.6815711260,1.6350235939,1.7907772064,-0.3013070822,-1.3498184681,0.9766227007,0.8020770550,0.1571407765,-0.3929232657,-0.3565121293,0.8001793027,0.7584725022,-0.9664206505,-2.0399837494,-0.7562393546,-0.0191347543,-0.1659532487,0.2461894006,0.0534501150,-1.4511636496,0.8290471435,1.4202630520,-0.6461420059,-1.1758359671,-0.9557312131,1.4096255302,-1.0442581177,1.8160980940,-0.3281961679,0.1080035195,-0.4689996839,-0.4098793268,-1.8007936478,-1.4797694683,1.2166144848,1.3121565580,-0.6322200298,-0.3365573585,0.7352458835,-0.2050330043,-0.4896292984,-2.1675162315,-0.0871381313,-0.6976063848,1.2434237003,-0.2033266723,-0.0167122856,0.1015309244,2.7080247402,0.2532948852,-0.8676658869,0.0866448730,0.9485441446,0.1438010484,0.1264382154,-1.1578149796,0.5896145105,1.3902870417,1.6312388182,0.3860552907,-1.3228954077,0.6822083592,-0.8747211099,-0.1168340445,-0.0816032961,0.0949813724,-0.2675711513,-0.2850045860,0.6082111001,-0.3288976252,-0.7892969251,0.8910753727,-0.4391539991,0.4801815152,1.5932966471,-0.7188422680,-1.0578689575,0.4190469682,-0.3070425391,0.4496214390,0.3089589775,-0.6861286163,1.1116139889,0.5191278458,0.7216537595,1.0761941671,-0.1724572033,-0.2763061225,0.1145950407,-0.0036275154,0.1950309277,-0.4134352207,0.2700786889,-0.4687091112,0.3374997079,-1.2691653967,-0.9487433434,-2.3327836990,0.4507500529,0.9267107248,-0.3292925954,1.1686035395,0.2126315683,0.1942957640,0.5142114162,0.8292499781,-0.2244424075,0.4714294672,1.2915810347,-1.1542066336,-1.3901450634,-1.0315078497,-0.5843418241,-0.3625900745,-0.6811479330,0.0109514073,0.4034639895,-0.7629212141,-0.6994753480,-0.5534796715,2.0600101948,0.9628931880,0.0250039492,0.1217478886,0.6854715347,-0.3456495404,-0.6771116853,3.5718197823,-1.3524976969,-0.7330499291,-1.1353269815,-0.6868650913,-0.1242099255,-1.8488539457,-0.1784221828,-0.5338147879,-1.4749981165,-0.9773634672,0.1918828785,-1.6867041588,-0.9012171626,0.9544080496,0.3894421458,0.8154364228],[-0.5928393602,0.7729503512,-1.2539441586,0.9535008669,-0.4108581543,1.1859742403,-0.7260681987,0.0907370001,1.5868970156,0.1683713347,-0.4438912570,-0.3875832558,-1.8504856825,-0.8048030138,1.2810553312,-1.0632429123,-1.1179351807,-0.6259436011,0.2457038760,0.7051655650,0.5252198577,-0.7441893220,0.3223232329,-0.8422292471,-1.6582146883,0.2706842124,0.5295950770,-0.6153351665,0.4503405988,-0.6227647662,0.5138043165,-0.2224375010,-0.1313886046,-0.9083277583,1.6377744675,-1.3036310673,0.9719820619,0.6772412062,0.5541941524,2.0696039200,2.1902463436,-0.9359971881,1.5101017952,-0.8271532059,1.1484556198,-0.7858132720,-0.8876923919,0.9454967976,0.4654867053,0.8108379841,0.9280185103,-2.0502116680,-0.2926620543,0.4862718284,0.8723092675,0.6239894629,1.6616897583,0.4626489580,-0.6597118378,0.7611908317,0.4519271851,0.1392660737,0.3012660742,-0.1104904786,0.5151509047,-0.4535919130,-0.5744910240,0.1589653343,0.6228471994,0.6753728986,1.4053241014,-0.3785912097,-1.3496842384,0.0238295868,1.2929308414,-0.7779426575,-0.0595474653,-0.0276483335,0.2683479786,0.3438584805,0.5323349833,0.0969401300,0.7111643553,0.6105911136,1.3095695972,0.3804032207,-1.2960375547,0.0383879021,0.4270890057,-0.9619238973,-0.7499616742,0.1058324948,0.6600143909,-0.1229051799,0.8982565999,0.6313751936,-1.2400015593,-1.3406934738,-0.7260510325,0.1453833431,-0.2229319066,-0.7966413498,0.0715794265,0.8668941259,-0.2490485162,0.9793452621,-0.4966464639,-1.3573478460,0.2178915888,0.0893864110,0.7565530539,-0.3809821308,0.3454945385,1.0413463116,-1.5530769825,0.6615539193,-0.6907436252,-1.0928965807,-0.3175019026,2.4032316208,0.1700476259,-0.2063445449,0.4634450078,0.9587469697,0.1602792293,0.4382282197,-0.6603542566,-0.1854121238,-0.7110162377,-1.0502065420,0.3033640385,-0.0090368362,0.9971413612,0.0437622704,-1.6310248375,-0.8269742727,1.7787942886,-0.7838104367,-0.0933950394,0.7300506234,0.8639712930,-0.7480256557,1.2835432291,-0.9399489164,0.7385509610,-1.0714541674,0.4192541242,1.2823365927,0.4535315037,1.6193950176,-0.8308219314,0.6912407279,-0.2242636681,1.3361150026,0.6657164097,1.0476800203,-0.3936673701,0.3589158058,-1.9339230061,0.4487364888,2.0723960400,-1.0328774452,0.7694115043,0.2298396081,-1.9235595465,0.8481762409,-1.0481160879,-1.7839934826,1.8505026102,-0.1471647471,1.4117530584,-0.4248147607,1.0637511015,-1.6170943975,-0.1204279289,-0.8135151863,-0.9916515350,-1.3610657454,0.2157028317,-0.4446689188,0.7377965450,-1.4382053614,0.8609056473,0.4337145686,0.4578846097,-1.0002191067,-0.2169005722,-0.3418893814,1.4400826693,-0.6663337350,-0.9091744423,0.3776062727,1.3034206629,-0.6550186872,0.0032101073,-0.8028377891,-1.3056483269,-0.1703900844,0.6938927770,-2.5968184471,0.6109979749,-0.1903804541,-1.7227438688,0.0623724312,-1.3457913399,0.3999651670,0.4954300821,1.3090932369,-0.4598393142,-0.6244497299,1.5473086834,-0.1307564676,1.5282249451,0.0975896195,1.3858814240,-0.4262283444,-0.0787541643,0.0616688319,0.5138702989,-0.3498030901,-1.3233053684,0.4028116465,0.3263849616,0.5358191133],[-1.7932841778,0.9994078875,-1.3617011309,1.1623231173,-1.7890124321,0.6985490918,-1.1428382397,0.2147274613,0.1396360397,-1.1729005575,-0.6266290545,1.2859871387,-1.0260407925,-0.5826871395,-0.1028865278,-1.7940053940,2.0454301834,1.0318255424,-1.1083452702,-0.4782355428,3.4777047634,2.4325971603,1.7189693451,-1.4776905775,0.5881302357,0.5027113557,2.8661079407,-0.1098669693,-0.1802531779,1.1672160625,-0.0681959763,-0.5658228397,0.3528211415,1.2640615702,-0.4119260609,0.4474448264,-0.6144564152,-0.3673212230,-0.0331863575,-0.2325524390,-0.5525498986,0.4451538920,0.3916987777,0.0216711983,0.8088118434,-0.2224570215,0.5602177382,-0.8138303757,0.0004522767,1.0792688131,-2.1361529827,0.9336476922,1.6088411808,1.8209517002,-0.8839716911,-0.2459857762,-1.2426134348,-0.1604559571,0.8543289900,-0.8487345576,0.7471719384,0.0556920283,0.4832492471,0.7741348147,1.4826248884,-0.9840801358,0.7220378518,1.0715200901,-0.4242345691,0.0419250503,0.8400822878,0.0718306378,-0.2302375734,-1.6227520704,1.2532012463,-0.6595457196,1.4351598024,0.4085188508,-1.9150269032,1.5044716597,-0.0597980022,1.1048128605,-0.4878143370,0.3595606387,-0.8408216834,-0.6972226501,-1.1642152071,0.9077614546,3.1851241589,-1.2781811953,-0.9502065778,1.4101806879,1.1578328609,0.5755327940,-0.3855011463,0.1394493580,1.5561326742,0.3201376498,-0.3746150136,-3.3304371834,1.5891109705,-0.4015725553,1.0671237707,0.4332732856,0.5515481830,0.8586447835,0.1360096633,-0.0430612415,0.8356793523,-1.8970340490,0.5594943166,0.8892461061,1.5231126547,-1.9689277411,-0.4791465700,-0.1135723814,-1.3317320347,1.2973388433,-0.7804918885,1.8386099339,-0.4639483690,-0.1304014027,0.4642805159,0.4224203527,1.5378631353,-0.6809688807,-0.7566910386,-0.8838561773,-1.6089427471,-1.5269782543,-0.5458895564,-0.0781098157,2.1578493118,-0.1167186201,-0.7636818290,-0.5750817060,-1.4051170349,0.8595709205,0.2047751695,0.5402824283,0.0225952975,0.5965875983,0.0429086573,-0.8807626963,0.2503642142,-0.2607371807,0.7006023526,0.1024034098,0.2675665021,0.8997215629,-1.3800114393,0.6278210282,-0.6009278893,0.0229070876,-0.8209497929,0.6459999681,-1.3651671410,-0.9422188997,0.8445447087,-0.5556010604,-0.2866741419,-0.6988923550,-0.2834370732,-1.5584450960,-0.3284163177,-0.2235734910,-1.2259905338,0.5285528898,-1.0578813553,0.3370731175,-0.1701500416,0.2106822878,0.1954178661,-1.2928462029,-1.0101048946,-1.0162965059,0.7101376653,1.1929751635,0.2930506766,0.3283641040,1.8373104334,0.1537463218,1.6340095997,1.5657008886,-0.1655842960,1.2367951870,2.0852239132,-0.3394646943,-1.6677970886,-0.1699476838,0.1307020038,1.3757307529,-0.1364614964,0.5837905407,-0.6506878138,1.0382732153,-1.1101716757,2.2023341656,-0.5978506207,1.2223081589,0.0622529648,1.0693994761,-0.1176077351,-0.5627841353,1.4076569080,1.7500611544,0.6664909124,0.2402775288,0.5856772661,0.1869566590,1.1902452707,-0.3557678163,-0.2289677262,-0.5311463475,-0.7184414268,0.8175365925,0.2107467204,-0.5429624319,-0.1168661341,2.7603781223,0.7776145935,0.1007762924,0.1165065393,1.5572880507],[0.3648286164,2.0247426033,-0.3560695946,-0.0204400718,-0.7988411188,-0.9985968471,-0.5758203864,0.2325834781,-1.2333204746,1.3030353785,-0.7864224315,-0.5838130116,-1.3188313246,0.2502593100,1.2273973227,-0.2414512187,-0.8900685310,0.4520162046,0.5013954639,-1.2339198589,0.5234656334,1.8722342253,0.2422109544,-1.3943357468,1.5739291906,0.3023069799,-0.1267141402,-0.4534452260,0.7923907638,-2.1231665611,-0.2892504334,-1.2547494173,1.1192709208,0.1841019541,-0.5269345045,-0.3538885415,-0.3898421228,-1.2952687740,-0.7927324176,1.1888612509,1.3829047680,1.0012235641,0.1509179920,-0.5995969176,1.0717393160,-0.6755110621,1.8635855913,1.7817964554,-1.5924193859,0.7871559858,0.1822643280,0.6299715042,-0.4798375964,-0.2566851377,-0.8125913739,0.0916184112,0.3510672748,0.7465915084,-1.2982705832,0.4566431642,-1.4501985312,-1.0551677942,1.1857924461,0.2242699265,-0.2082041055,0.1747808009,1.4196221828,-0.8689292073,-1.5527167320,0.0466211066,0.3829758465,1.4496682882,-0.6000432968,-0.2342979163,0.9764523506,-0.1101170778,-0.2743099630,0.5450106263,-0.6299778223,-1.0088422298,0.5787731409,0.6363965869,-0.9875950813,0.4465334415,1.6913824081,0.8224819303,0.6353470683,0.6847267747,-0.8218975067,-1.7893120050,-0.3205640018,1.1568148136,-0.6012837887,-0.0253048092,-0.8224580884,0.4368734360,0.5922030210,0.2948924601,0.4711373150,0.2339244038,-0.8675713539,-0.1623468995,0.2907239795,0.0648862347,-0.7679846883,0.4781250954,-0.9584442973,1.2950092554,0.4279106259,0.1744094789,-0.2890577018,0.4377071559,-1.0612857342,0.6737443805,1.4462653399,-0.0192514602,0.4609615207,-1.7821609974,-0.3013353348,-0.3335744143,-1.1731162071,1.3955233097,-1.2123579979,0.2251206934,1.1979632378,0.0889767930,0.5139861107,0.9769540429,0.3754872680,0.7002015114,-0.4718477726,-0.9561746120,1.3625870943,0.5210314989,-1.7651660442,-0.5209172368,1.1049113274,0.0181649495,0.4430385828,0.0258633532,-1.1045073271,-1.0388607979,-2.2845268250,0.8429036736,-1.1021239758,-0.2099017054,-0.6958201528,1.9615095854,-0.3146707714,0.7117929459,-0.9054128528,0.3786839545,1.2535066605,-0.5864236355,-0.6019744277,0.3552685678,-0.7152664661,-1.2186499834,-0.8128777146,-0.1254124641,0.5544507504,0.8033244610,-0.0788029656,-0.3765355647,-0.1530294567,-0.0445062034,-1.0340093374,2.1138308048,-1.4381253719,-0.3641769290,-0.7461310029,0.5013814569,-0.4317185283,-1.5411124229,-0.0315037817,0.1206132025,0.3322246075,-0.3178224564,-2.4873731136,0.9532303214,-0.2243309021,0.8475925922,-0.1884335577,2.1482484341,0.1499400139,0.5643538237,0.4252497256,-1.0068868399,-0.1776017845,-0.8614460230,-0.3621585071,0.3836019039,0.9026619196,0.9432954192,-0.3222702444,-0.6358183622,-0.6205145121,-1.2305152416,-0.1459206790,0.9727852941,-0.6101299524,-1.1514422894,-0.5444167256,0.2890828252,-0.1478310674,-1.2646784782,-0.7198884487,0.7411797047,0.3236621618,0.6102467179,1.9273406267,-0.7630124092,-0.8950296044,0.1038967371,1.3054035902,1.1023359299,2.0166347027,1.2310870886,0.3453706503,0.4032297432,-0.3820646107,1.4327986240,1.2221440077,-1.3450983763],[-0.6113471985,-0.0736481622,-0.4696511924,0.0348328836,0.2735553682,-1.7579873800,-0.2284794301,-0.2847909033,-1.1847519875,0.9780002832,-0.0279450770,1.1050629616,0.5406913161,-0.4539557099,0.3432274759,-0.5386041999,-2.3486204147,1.0494381189,1.0514642000,0.1160023734,0.4832822382,-0.0234396216,0.0183158182,1.3490015268,-0.5867910385,-0.5934441090,1.6418461800,0.8020620346,0.3180578947,-0.0109635685,-0.8504745364,-0.3389580846,-0.4799004793,0.2578036189,0.4269354939,-1.3231627941,-0.5754043460,0.1770341992,1.9017140865,0.4532558918,-0.7606176138,0.7406585813,-1.0882092714,1.8031427860,-1.3217358589,0.4403271973,0.3200185895,-0.0844190046,-1.0422834158,1.3751612902,-1.3435511589,-2.2676715851,0.1229116991,0.4220400751,0.8152249455,-0.5771619678,-1.5192883015,-0.1184898242,0.0107239718,-1.6067227125,1.5956927538,0.0263506193,0.9669717550,-0.8889227509,-0.4053593278,1.3851294518,-0.4103517532,-1.2005337477,0.8040002584,0.0697512850,0.6278099418,1.3422721624,-0.1635767967,-0.1250185519,-0.3778596222,-1.0309101343,0.5122585893,0.3585168719,-0.3668381572,0.7736611962,-0.5901617408,1.0048699379,1.6593602896,0.0736119524,-0.5228585601,-1.1776105165,-1.2693563700,0.7466336489,-0.4147734344,-0.6222800016,-0.4369660020,1.1900970936,-0.0045385598,-0.3916423321,-0.4234701991,0.4515590668,1.6646592617,-0.7858273387,-1.0369464159,-0.8603183627,1.8155955076,-0.9686090350,0.2055028230,-0.0671775788,-0.1541100144,0.1235477552,0.6986100078,1.5323183537,0.3527896404,0.5008907914,2.1498534679,0.3520805240,0.8896867037,2.5267851353,1.0206768513,-0.8351369500,0.4967375398,0.8105723262,-0.7931277752,0.2910664976,-0.7967676520,0.2498087585,0.3448672295,-0.1164075732,0.0368722863,1.3828399181,0.4213814735,-0.9003941417,0.5536766052,-0.4180638790,-0.4147278368,-0.9282360077,-0.7860546708,1.1377804279,0.4159342051,-0.7434610724,-0.7378995419,-0.0775714889,-1.1062474251,-0.1626705676,-0.0320903845,-0.6710093617,-1.9599610567,-1.0496373177,0.4051431119,-2.0050468445,0.3016331196,1.1520593166,1.2890785933,0.6122150421,0.4631689787,0.2665107846,-2.0319108963,1.7829045057,-0.5972828865,-0.2010582238,1.9431426525,-0.1263195872,-2.0065937042,0.9118897915,-0.4932822585,-1.6785792112,0.7633743882,-1.4086085558,-0.7630506754,2.4220027924,-1.4050699472,0.3170969188,0.3332512081,0.5363234282,1.2329015732,-1.1386669874,0.7243200541,-0.5739594102,1.1940883398,-1.3103320599,0.0189716071,-1.0934224129,-0.6630554199,0.6524068117,0.4000760615,-0.4352589548,1.8864926100,0.1805183440,-0.5843188763,2.0612359047,0.2389136702,-1.0740573406,-0.3438945413,0.5787284374,0.6672785282,-0.3425982594,-0.2763700783,-0.0430908315,0.3072268665,0.4272223711,0.7463321090,2.3031678200,1.1491247416,0.7463536263,1.6505769491,-0.7130047083,0.4484150410,2.5032739639,1.5361375809,-0.0852901042,1.7603957653,-0.7834419012,-1.5477027893,-1.2571730614,1.3132101297,1.6227086782,-0.1173110008,0.7305071354,0.3547957242,1.1275627613,-1.3428343534,0.1347589195,-0.2234453708,-1.6528010368,1.1485866308,1.3285199404,-2.2826619148,0.2097360492],[0.7691443563,-0.3663109541,-0.9105324745,-2.1862375736,0.4142344594,-1.2459869385,-0.5919961333,-1.1263803244,-0.7219035625,-1.2844762802,-0.6319314837,0.4255679846,-0.9710791111,-0.1546536833,-0.9671045542,0.8408493400,0.4944689572,-0.9499318004,-0.9606438875,-1.6236588955,-0.0765073076,0.1454095095,-0.6725363731,1.5028852224,0.3382230401,0.9947757721,-1.5230675936,0.3305849135,1.0095059872,-1.2323187590,-0.9129067063,-0.6490970850,0.7627770901,-1.5042232275,1.9205620289,0.6990779638,0.3981493115,1.7094851732,-0.8547058105,0.6071660519,0.3208642602,1.3405076265,-0.8251224756,0.7542603016,1.5279054642,-0.0644014403,0.0308947098,-0.4144746959,-1.2328224182,1.3450195789,0.5506920218,0.7623227835,0.4588803649,2.4532387257,0.4284676611,-0.6673672199,-0.7856223583,0.6798230410,1.3754965067,1.2486560345,0.8557427526,-0.6578801870,2.8154249191,-2.0665171146,-0.8970668316,0.8705096245,-0.5161067843,-0.8774737120,-1.3747942448,-0.0044820942,-0.0629592091,0.8383932114,0.1018836051,0.2315552235,0.8842407465,1.9258217812,-0.3826536238,-0.7059099674,-0.9088483453,-0.0775755346,-0.8568040133,0.6119330525,-1.1394847631,-1.0729565620,2.5123028755,-1.1320472956,-0.0720144957,-1.3061287403,0.0605295226,0.5201559067,-0.7216553092,-0.0817440078,0.4655059576,-1.0059340000,1.6584295034,1.3138953447,-0.1685477644,-1.7573693991,-0.5177462697,0.2565937340,-0.7753594518,-0.3755358458,0.9658132195,-1.6226822138,2.0549082756,1.1139340401,-0.4460828304,0.3770333529,-1.0984185934,-0.7914294600,-0.7182283401,0.8725944757,1.4336870909,0.1247830093,-2.4568533897,0.8807242513,1.4794603586,0.5188699961,0.1264361292,-0.9099682570,-0.3541872203,1.0382727385,-1.1467587948,0.0937129781,-3.0879652500,-0.7070456743,-0.8758177161,0.7819408178,-0.7893208265,0.3356752098,-0.3563073277,0.4893903732,-1.0363898277,-0.9055857062,-0.7337564230,0.1480658501,0.9461278319,0.3862171173,-1.0495040417,0.9934561253,-0.2928678691,2.2781779766,0.4375535846,1.4886761904,-0.2329892665,1.1606227160,-1.1704735756,-0.8371050358,0.4440965652,0.0336679667,1.2123416662,-0.3476849496,0.5047299862,1.0939006805,-0.1639223248,-1.1171123981,-0.8182387948,0.5606721640,-0.4868011773,-0.1622991860,-1.1784236431,-0.0316150524,-0.6546902657,1.9915369749,-0.1884418726,-0.1821149737,-0.5000926256,1.5627294779,0.5795538425,1.7219066620,0.3575980067,0.7396416664,-0.7974137664,0.1111887917,-1.3111822605,0.1688636690,-1.2101415396,2.0626072884,-0.1517615914,-0.1261129677,0.5988250971,0.3151803315,0.2292727828,1.5950623751,0.7607192397,-0.6130860448,1.2753077745,-0.0473849252,1.0907490253,-0.5991789103,1.1112835407,-0.7966700792,-0.9286405444,1.1475811005,0.7448164821,-1.0517472029,1.8815093040,0.4007554352,0.5530241728,-0.3934260905,-0.0581542328,-1.2024121284,-1.1866624355,0.4203956425,0.9341752529,1.2243565321,0.8899989724,0.3466911018,-0.7389758825,-0.9940132499,-1.0827097893,-0.6211391687,0.0189784765,0.6920895576,-1.3577948809,-0.1852061152,-0.3256109953,-0.3312273026,2.7818863392,-0.6032305956,-0.5881502032,-0.2108903974,0.9609686136,-1.7761542797],[0.1638919264,-1.8034422398,0.3438547850,-2.0366148949,1.2626676559,-1.0085458755,0.8398509622,1.1671663523,-1.6237409115,1.2003210783,-0.6494108438,-0.3809467852,-0.8366717696,1.0897408724,0.7005752921,1.8595799208,0.7068119645,0.2377082705,-0.4191710055,-0.0960142314,-0.4157134593,-1.4508297443,0.6521006823,2.3216221333,0.6430325508,1.4736698866,-0.4491530657,1.0629155636,-0.9583804607,-0.6451090574,-0.6868199706,-0.0433918200,0.9207156301,1.4857054949,0.3920950890,-0.4053091705,0.8437995315,-1.5179128647,0.1888910085,0.0703740716,-0.7147838473,0.2936773896,-1.1163834333,0.4440954924,-1.5620125532,-1.4867972136,0.9728970528,0.0287208781,0.2518751621,-0.8563625813,-0.0962809995,0.9173402786,0.1593800485,0.3545618653,-0.5275490284,-0.1848464012,1.1037555933,-1.7516676188,1.1288124323,0.4776148796,-0.0878350437,0.8084073067,-0.5648819804,1.1433855295,-2.9104449749,0.8359450698,0.0287205018,-0.3140828609,0.2962246537,2.6138598919,1.9440122843,0.2640841007,-0.9142881632,-0.5235984325,-1.0183387995,0.2797614336,-0.6935437322,-0.2669579685,0.8215245605,-0.2807157934,-0.4905641377,0.4833571315,0.2829817235,0.5185962915,0.6601308584,-0.3351995051,1.0649538040,0.1437657326,-0.1996699721,0.1168802828,1.0141957998,0.1364440620,0.6416521668,-0.0714805797,-0.1246618554,0.6482216120,0.7308468819,0.2386373132,1.0028398037,0.5547579527,-0.5872223973,-0.7867373228,-1.1379771233,0.0934178531,-1.1745666265,-0.7475016117,-0.5521036983,0.1610332578,-1.0222641230,-1.0741324425,-0.0681060180,-0.5505965948,-0.0340255834,-2.0789461136,1.2974512577,-0.2434522808,-0.6320980787,1.6777137518,0.2020441890,0.7013676763,-0.9742774963,-0.4643732905,-1.7642954588,0.2217412889,-0.8830285072,-1.1129447222,-0.6823812127,1.3527050018,-1.2479234934,-1.7996045351,-0.8925056458,-0.0426734425,0.5319902301,0.2391276062,1.0110605955,-1.4177103043,0.4677368701,-0.4861982763,1.6502017975,-0.2858231962,0.4730424285,0.4991550446,1.8836057186,-0.4640003443,-0.0351408608,-0.4028859138,-0.2978712618,0.0881469771,1.7128058672,0.0741602257,-2.0762367249,0.6053909063,-1.3824832439,0.5659181476,-2.0131797791,-0.3630668521,0.6377583742,1.1255798340,0.1482716650,0.0555110313,-1.0066103935,-0.0993747562,-0.5288520455,1.3735204935,0.4501582086,0.9103261232,0.5915299058,0.8719922304,-0.1844073534,-0.4059509635,-0.7417193651,-1.1776884794,-0.2048167139,-0.7765574455,-0.3557467163,-0.4653972089,0.1155521870,-0.0342244320,-1.6729363203,1.5951782465,0.3936387300,2.1581971645,0.0029275578,-0.2868529856,-0.9359773397,1.2794549465,1.4388250113,-0.2805157900,0.1168797389,1.3458018303,1.0483114719,-0.6029849052,0.7616730928,0.5633711815,-1.0731381178,1.0358542204,-0.7741640210,1.0627392530,-0.4310510159,0.1969968379,-1.5273967981,0.9693744779,0.4679141045,-2.0622494221,-1.4941822290,-0.2936666906,-1.3504440784,-0.4829608202,-0.4429127276,0.8962433338,0.4110181332,-0.7450675964,0.3003093004,-0.1053996682,0.5656231642,0.0190361906,-0.5192342401,0.3490352929,-0.2058995962,1.3725743294,-0.8968298435,0.5423145294,-0.8962261677,-0.1491219550],[-2.0275385380,-0.6608038545,-0.9174212813,-0.3269410431,-0.6984471083,1.6944645643,0.4648699462,-2.1589312553,0.0426462181,-0.4002126157,-0.9844839573,-0.9373098612,0.3330574632,0.8121154308,0.1654017717,0.0265124217,0.8958481550,1.0384033918,1.3312896490,0.2691825032,-0.5273260474,1.1580491066,0.9136486650,-1.8261531591,-1.6294533014,-0.4599237144,1.3166241646,1.2717188597,0.3420768678,0.1444726288,-0.9680743217,0.4697493017,0.8182046413,-1.4236667156,0.7308368087,1.0863107443,-1.4559316635,-0.0180493183,0.0013391395,-1.5703587532,-2.0040662289,-0.1724464595,-0.7469660044,-1.5486024618,0.9595509768,-0.9743630290,-0.0639142990,-0.3483835161,-0.5184774995,-0.1823514253,0.2897956967,-0.2570497990,-0.0183025580,0.1666794717,-0.4107077122,0.3243245482,-0.6374425292,-1.3727964163,-0.7992866039,1.1616351604,-0.0549143553,0.0692056417,0.4817972779,0.3330696821,-0.3786138296,-1.9377900362,-0.8723055124,-1.9915663004,-0.7240679860,-0.5480913520,-0.0218152720,-0.1139964089,-1.6564424038,-0.9848123193,-0.5869332552,1.3057488203,1.0492138863,-0.3129112720,-0.9047678113,-0.7177250981,-1.2049781084,1.3314702511,-0.6231637597,0.7844091058,-0.0176781621,0.4116369486,-1.3127673864,-0.2670100331,-0.0723978579,1.0006428957,-1.9221795797,-1.2662967443,0.5329766870,0.0479580536,1.1082743406,-0.0548312664,-0.8269295096,1.1963032484,-0.2248819619,-1.2792636156,0.4042378664,0.0060892166,0.2407629043,-0.2863627970,-1.7510541677,-0.8826270103,-0.1805585325,-0.8399214149,-0.3694585264,0.6691579819,-0.6126208305,0.5864830017,0.3455734253,1.0982719660,-1.2218780518,-0.7412814498,0.6595743299,0.6852849722,-0.4938407838,-0.8290623426,1.8494502306,0.7113271952,-1.3318347931,-1.2668334246,0.6168457866,0.0138242068,2.3361804485,0.0133694801,-0.9318174124,1.0154223442,-1.8149859905,0.1063761935,-0.4912840724,0.0053320178,0.1622054279,-0.3720568120,-1.7061535120,-0.3411099017,1.8075840473,-1.4172304869,-1.3801350594,-0.1177887693,-0.4706535637,-0.5100216866,0.0099745048,1.0853533745,0.3263893127,0.0064063426,1.8728448153,-0.5153175592,-0.0248616636,1.0729645491,-0.9651572704,1.2770721912,-1.1947435141,-0.4683704376,-1.5828979015,-0.2075870931,0.7596347928,1.1891981363,-0.2871383727,0.7696259022,1.1756484509,-0.2555998862,-1.1806615591,-1.5777649879,0.7371015549,-0.8200165629,-0.3479668200,1.5302962065,-0.8369179964,1.9417058229,0.3632695973,-0.5283840299,-1.4799474478,-1.0299297571,-0.3006336093,0.7646048069,-0.7382662892,-0.0858026668,-2.0297603607,0.7612405419,0.4500703812,1.1386395693,1.0415931940,-0.6529315710,-0.1752320975,-0.0934556127,-0.3674602807,1.1684161425,1.7406518459,0.2140564770,0.1849590987,-0.4445253611,0.9117878675,0.6770923138,0.8259384036,0.0250431057,-0.6785590649,1.5031665564,0.4678463638,-0.2961201370,-0.8332222700,-1.7620338202,-0.5066124797,-1.4213819504,0.0325794555,0.4598464966,0.8975044489,0.7793223262,0.2708227634,0.3689299226,-0.2866435945,-0.4612148404,0.2356104404,0.4840772152,0.0035614318,0.5616604090,-1.1055561304,2.8946905136,1.2669999599,0.4352328479,-0.6449033618,1.0168118477],[0.1424993426,-0.6114996076,0.1845914721,-1.2095587254,0.1187560558,0.1327622831,-0.9325071573,-0.0071629351,0.8303387761,-0.0023879316,-0.4078947604,0.5116908550,0.5126677752,-1.8852918148,-0.6545630097,-0.8021213412,1.7759143114,0.0790987164,-0.0372745730,-0.5091186762,0.3631534874,0.1563995630,-1.1762232780,-0.2444291711,-0.9540668130,0.8484722972,-0.2094481140,0.2584297061,0.4090232253,2.4077796936,-0.6442091465,-0.3512670100,-1.6794028282,0.6157962680,0.9134595394,-3.0511646271,-0.1610290557,0.0838241577,-0.2765303552,0.5775899887,0.5077181458,0.6141081452,-0.2548968792,-2.2971007824,-0.4871230423,-0.7298045158,-0.2915547490,-0.8250328302,-2.1933531761,0.5092428923,-0.9196913838,-0.2856418192,0.4852777123,-0.5238398314,-0.8249314427,-0.2213008106,0.4922359288,0.0046398826,0.0410495177,2.0978188515,-0.6671367288,-0.6185826659,0.9899863005,-1.0268276930,0.9838751554,0.1415936202,0.9602013230,-0.4019086659,0.0917325616,-1.7731301785,-0.6001939178,1.4872521162,-2.1950342655,1.4718630314,-0.4139128923,-0.9561069608,-0.8113186955,0.3326387703,0.8214855194,-0.2038237303,0.1721137315,-1.4140814543,0.8006810546,-1.1539144516,-0.4913604856,0.0820094123,-1.4642955065,-0.1441533267,0.3502728939,-0.5427587628,-0.1549838632,0.9814139605,1.6057529449,-1.7895412445,-0.5796220899,0.1910269558,0.1117242649,-0.8288728595,1.0306375027,-1.0849578381,-0.6961807609,-0.8261684179,-0.1974864304,-0.8071389794,0.9624972939,1.9254288673,0.3915759921,-1.8992302418,-0.6777492166,-0.3872167468,0.3622063696,-0.8133463264,0.4327911437,0.5757126212,1.9948111773,-0.8023193479,1.4258333445,-0.3936565518,-0.7623519897,-0.1646928489,-2.3818755150,-0.3824919164,1.1486035585,0.4143595994,0.5045929551,0.2677341700,1.3893061876,-1.6509420872,-0.1595229954,0.5341352224,0.5419927239,0.5675623417,0.0678644925,-0.4693374336,-0.2899969816,-0.2941041589,-0.3687418997,1.0642687082,0.3990949392,0.2112073898,2.1691763401,2.5314116478,1.1707247496,0.7853136063,0.8281221390,-2.0882747173,0.2841241062,0.3198890388,0.1984391659,-2.4289112091,1.0632315874,1.9254181385,-0.2166331410,-0.5206612349,0.3861836493,2.1343843937,-0.9316899180,-0.4525576830,0.0293869525,1.5127466917,-0.5325924158,0.2071255594,0.6559854150,-0.4750492275,0.6517692208,-2.0260581970,0.5454990268,0.9007616639,-2.1234264374,-0.6924746037,-0.2750160992,-0.7721719146,1.6402645111,0.1960601807,1.0715852976,-0.8867476583,-1.2654846907,0.8506244421,-1.0026184320,0.4239800870,0.0143493805,-1.0331511497,-0.2458608299,-0.2627170682,2.2830805779,-0.4507263899,0.4234290123,1.9715867043,1.0045151711,-0.8947780132,1.0483188629,-0.2448585927,0.9080960751,0.6717668176,0.7670707107,2.6760594845,0.2887377441,-0.5765333176,-1.3535066843,-0.1563234329,-1.6242492199,0.8874552846,0.4395448565,0.2620094121,0.8149400353,-0.7656787634,-0.1232830957,-0.6337431669,0.0293763019,1.4243355989,1.0572086573,-1.8027933836,-0.5571980476,-0.3124030530,1.2649337053,2.0032932758,0.9319395423,0.1501328796,0.1547182649,0.0764016360,2.4216063023,0.7080580592,0.2585445046,-0.2951288521],[1.1590728760,-1.4281768799,-0.6427973509,1.0778914690,1.3738418818,-1.0305179358,1.2834491730,-0.1111884490,0.0416006520,1.2782590389,-1.3953489065,-0.2880483568,0.2030022740,-1.4335784912,-0.8353443146,-0.7202455997,0.5462498069,1.4243865013,-0.1032523811,-0.0441531800,0.1812570095,0.6535425186,0.7359706759,-0.3278496861,1.6425333023,2.3137261868,-0.1360592395,0.5812934637,-0.6448676586,0.0685226098,0.8798252344,-0.7275282145,-0.6837694049,1.2179878950,0.5052300096,-0.1817120165,2.7077167034,0.5897852778,0.0560361706,-0.2262458354,-0.1569620222,-0.8156437278,-2.8309299946,-0.0347006284,2.3700158596,-1.4517055750,-0.7827415466,0.2200433016,0.6763939261,0.5266736746,-0.3205535114,0.4175000787,-1.4401388168,-1.2638221979,-0.0063566575,-0.4596130848,2.0484826565,2.6739189625,0.1921385825,0.8656622171,-0.7989099026,1.2951737642,0.5948476791,-1.1622223854,-1.0827726126,-0.6607543826,1.0801876783,1.7353651524,0.2039402127,0.4578662515,1.4513297081,0.6782011390,-1.0781478882,0.9227527380,0.8389589190,-0.3721253276,1.6220294237,-1.5711258650,0.5092188120,1.6772679090,0.2103672624,0.3513631523,-0.6458117366,-0.1160704643,2.5399868488,0.2986964285,0.7530531883,2.7717247009,1.3016248941,-0.5547922254,0.8124800920,-0.5440471768,2.9482123852,-1.9910891056,-1.2753846645,-0.8022624850,-1.4422289133,-0.5684272051,-0.1325278580,0.6433197260,-0.0040892162,-2.0867819786,0.3624523878,1.7563529015,0.1436444223,-0.7461550236,-1.4902784824,0.2751983702,0.2716933787,-0.5269842744,1.3409408331,-0.5586332083,0.5571196079,-0.4478425086,1.0341912508,0.5109900236,-0.9873884916,0.0708701387,0.3049991429,0.5304082632,0.1405647397,-1.0756629705,1.1812801361,1.6754041910,1.5391957760,0.1541727930,-0.3437381983,0.8482227325,-0.1725202948,0.8681633472,-1.2652460337,0.9696620107,1.3187575340,0.1618376225,-1.4064910412,-0.4977366924,-2.9630827904,0.2724413574,0.5373820066,-1.1703877449,0.2719387710,-1.0549979210,1.4473180771,1.9692100286,-0.1396738887,1.2972766161,1.3983153105,0.7203134298,-0.4064352512,-0.5069192052,-0.7329676747,0.4564698935,1.4903695583,-1.5688288212,-0.2548439503,1.0869340897,-2.3693585396,-0.0668379068,-0.2766051590,-0.5685678720,0.1007213444,0.4818153977,0.3051174581,-0.6083016396,-0.4800639451,0.2737148702,1.9641357660,0.9281412363,0.3207195699,0.2103534937,-1.6923233271,1.1626750231,-2.2308335304,1.4662967920,-0.0578541830,1.0219566822,0.8956917524,-0.4066852629,0.6047781706,-0.2009098828,-0.0700096786,1.5064666271,-0.0695334747,-0.8002229333,2.0655736923,-1.8748968840,1.4135584831,-0.1391232759,1.5362677574,0.2097577006,0.2475161254,0.9883912206,-1.5831539631,0.4373769760,0.3146182895,0.6461459994,-1.4481978416,0.4126372039,0.2698713243,0.2971876562,0.6936231256,-1.7687585354,-0.1024047658,1.8644307852,0.1501193047,1.4666852951,-0.0993625149,0.8320145011,0.2680338025,0.9386171699,-0.4379406571,0.0308498964,-0.5973451138,0.1201213524,1.8185228109,1.2896347046,-0.7865766883,-1.1671062708,1.0264364481,-0.4385924637,1.1909672022,-0.2365868241,-1.2961139679,0.7524974346],[0.2822886109,-0.8649634123,0.0341063626,-2.8012323380,1.8497134447,-1.4406055212,-0.8967682123,0.4737245440,1.4397727251,-1.7417879105,1.3006533384,-0.5665988922,-0.8679270744,1.1672435999,0.8426421881,0.5777239799,-0.0019734991,0.2628566623,-0.6765850186,-0.9454251528,0.6963109970,-2.0113916397,1.2220158577,-0.3644271493,0.3940259516,-0.4764729738,0.6744356155,0.4133459032,0.7129043341,-0.1390728951,2.4798042774,0.4449064732,1.0364000797,0.3874957561,-0.2127955854,-2.7277810574,0.8964817524,-1.1030762196,-0.9547590613,-2.6085407734,0.8240185976,-1.0024412870,0.5572627783,1.2201204300,0.6980441809,-0.3182870150,-1.0811100006,-0.8984940052,-0.9392712712,2.1252901554,-1.0693789721,0.6708747745,-1.2933703661,-0.4726088941,1.1888222694,0.2114872038,-0.4543708563,-1.3946108818,-1.3129746914,1.8129559755,-0.6399911642,-0.3761229515,1.0799931288,-0.8454584479,-1.1660637856,1.2091180086,-0.0841020271,-0.6888543367,-0.2568129599,0.1405620426,0.3394407630,-1.4461472034,-1.1847535372,1.1745417118,-0.5321114063,-0.0360228755,-1.2600207329,0.2216902971,0.3386104107,-0.4858584404,-0.2366298586,-1.5974326134,1.6561534405,-0.0659240410,-0.1518712789,-0.2975880206,-1.6202889681,-0.5111511350,0.4328710437,2.2999274731,0.1036824062,1.0497152805,-0.6470173001,1.2935286760,-0.8573095202,0.2456389517,-1.6096886396,-1.5166975260,2.2229051590,1.7827144861,0.0722996220,1.2876883745,3.4461617470,0.7335449457,1.2204228640,0.0536431260,0.0680795535,0.1912591606,1.3810964823,1.6709645987,-0.5143932104,-1.0899485350,-0.7233867645,-0.6821279526,1.3069764376,-0.2939797640,-0.5712470412,-1.4585288763,0.6952814460,-1.2076721191,1.0499768257,-0.0916740969,-0.0482324064,-0.5674934983,0.5605602264,0.7756689787,-1.1728376150,-1.5867017508,-1.2057219744,-0.2544875145,0.2107366025,0.5100553036,-0.3370763659,-0.9028173089,0.3074392378,0.0235144868,0.4365579486,0.2447894067,3.3895692825,0.7273297906,-1.8519378901,-0.7392191887,0.6342939734,-1.1659489870,0.2778316140,3.2687973976,0.8361719251,1.5480146408,-0.3752118051,-1.6580158472,-0.1478787810,-1.5146595240,0.6179021001,0.7550742626,-0.0943882689,0.7755956650,-1.5603001118,1.3169560432,-2.0728368759,-1.0825216770,0.6124401689,-0.2606959045,-1.1539610624,-0.7913170457,1.8059349060,0.4528495371,0.1427119225,-1.5344995260,0.1410109401,1.0175800323,-0.6247666478,0.9975888729,0.1294607967,0.1843679994,-1.1771456003,0.9625459909,0.9107032418,-0.7806379199,0.3773173392,-0.6000570655,0.5929875374,0.6998752356,-1.2417523861,-1.2792830467,-0.0448341966,-0.0431805402,-0.4964704514,1.9926270247,0.7569426894,-0.7253473401,0.1321403235,-0.0756466463,1.7732292414,-0.3561931551,-0.3286891282,0.7018263936,-0.8119922280,-1.1326178312,1.7631685734,0.4812170267,-0.1956695765,0.4439469278,-0.2367948741,1.3929479122,-1.0372287035,-0.0327260606,-1.0332548618,-0.6647860408,0.3720833957,0.1972508132,0.4465264380,1.3022334576,-0.3980585933,-1.0992186069,-1.1671767235,1.2631903887,0.3971188068,-1.5323983431,-0.4032920003,-0.6760450006,-0.2094847560,0.3652860224,-0.3081654310,-1.8190023899],[-0.1701465398,2.2619700432,0.2899451852,-0.7703804374,-1.1064257622,1.2357764244,-0.2967175245,-0.2518014610,-1.5689984560,-0.0665395111,-0.0862559229,-0.9216248393,1.2766802311,1.5393574238,1.0558137894,0.0327471532,-1.6642035246,0.8495980501,-1.7295979261,1.3477027416,0.1795098633,-0.2721120119,0.7435341477,-0.4873876870,0.5400074124,0.4466554821,-0.1522121280,-1.0432454348,-0.0845364034,1.7754360437,0.3426231146,0.4469730258,2.3664221764,0.6919433475,-1.1116325855,0.8461518288,-0.2383031100,0.8409350514,-0.7595874071,0.3608250618,0.8970346451,0.2595388591,1.0047931671,0.2540408075,-0.3400612473,-0.0989332497,-0.0843240842,-0.3543157279,-1.7633622885,-2.1017308235,1.2253223658,-1.1555352211,-1.7978022099,-0.3463512659,-0.0117409742,0.5840808153,-1.0323102474,-0.4460191131,0.0180029366,0.5165215135,1.0082906485,0.4918971956,-1.4555004835,0.4997964203,-1.1232254505,0.9987378120,-0.8648818731,0.1568584144,-0.7785245776,-0.3282842934,1.8889665604,0.1762571484,0.1230012551,-0.1905523986,-1.9035918713,0.6908513308,-0.4261215627,0.1221275926,1.0988082886,0.0977977887,-1.7845711708,0.7103321552,-1.1468501091,-1.2720302343,1.1274603605,-0.8638828993,-0.8295898438,-1.3732126951,0.1212136671,-0.7930681109,1.4917933941,-0.4714035690,0.2895286381,1.6895601749,-0.8141436577,-0.8679010272,0.1823181361,1.4328541756,-0.4204801917,0.2739147842,1.2020156384,-1.8207857609,0.0096109519,-0.8583530188,-0.9887015224,-0.1826705188,-0.6696085334,-1.5090953112,-0.1405836046,1.4303534031,0.3717813790,-0.6468277574,0.6109654307,-1.4839514494,-1.3144893646,-0.7689852715,0.6979426742,-1.9050546885,0.5673009753,1.0644997358,1.4237437248,0.8367416859,1.4612578154,0.9356698990,-0.5868975520,0.8214159012,-0.3272928298,0.4603173733,0.0749733821,0.1287813038,-0.2579364479,-0.1763682663,0.3552583456,-0.6590710282,0.8675601482,0.1804513335,0.0760865286,-0.2730111480,-0.7868251801,-0.1163439304,0.0811429247,0.1231359839,0.5234337449,0.1796350628,0.9489487410,1.6153148413,0.9410284758,1.0284470320,-0.9127117395,0.2013400793,-0.4045197666,0.7818467021,-0.3640500903,-1.8174712658,0.4952382147,-0.3121580482,-0.2593118548,0.1520194411,-0.0602648407,1.1699638367,-0.4944905341,2.1621308327,-1.3320986032,-1.6290991306,-0.7247669697,1.2197005749,-0.4413643777,-0.3647455573,-1.7348531485,-0.3638899028,0.0509795621,-0.5163988471,1.0410149097,1.0232046843,-2.2675664425,1.1889684200,0.4370892942,0.6450393200,0.7288362384,0.1710212827,1.3312356472,1.3694489002,-2.7367355824,-0.8831370473,0.4819488823,-2.1779723167,0.5980419517,-0.5120624900,-0.3950772583,-0.9564168453,0.3127147853,-0.3291445673,-0.9316582084,-0.6530855894,-0.5801360607,-0.8719137311,0.6725584865,0.3189466000,-0.9949298501,2.0379827023,-0.3290540576,-1.4721562862,-0.5032038689,-0.9637928605,-1.2094340324,0.7158033252,-0.0927066430,-1.2442517281,-1.0542521477,-1.1061631441,-1.2389072180,0.5120587349,0.2246490270,-2.1872861385,-2.1402411461,1.5952260494,-0.0812475979,0.6093641520,-0.5080390573,-0.2121757418,0.2089391798,-1.5400040150,0.3724039495,-1.1847088337],[1.5130584240,0.7076172829,-1.5021262169,-0.6401950717,0.4719422758,0.0465151928,0.5896236300,0.1516773552,-0.7396040559,1.5477974415,0.4146561623,-0.0829181895,0.0063156439,1.1727257967,1.2963408232,-0.4674797058,-0.1541069448,-1.4523546696,-2.4150819778,0.3667570949,2.6291785240,-0.4988396764,2.0458302498,-0.0747351050,-1.2224282026,0.2986982167,-2.0429217815,-0.9583296180,0.8893783092,0.2481311560,-0.5463197231,-0.1864299476,0.8800899386,-2.0384750366,-0.5730707049,2.2102062702,-1.3429448605,0.9100778103,2.2369208336,2.9532630444,0.2449691147,-0.0423857160,-1.2589607239,1.9544759989,-2.0986361504,1.1010017395,-0.6761249304,-0.5158253312,2.1491205692,0.6918260455,0.4865469933,-0.0197637565,0.3024237752,1.2770613432,1.5790829659,-1.3644719124,0.1178247705,-0.2447064966,2.2056901455,0.5842333436,-0.8112859130,-0.8845480680,0.5018767715,-0.9628272653,-0.6917787194,0.5183368325,0.2223260701,-2.4080696106,-0.6258693337,-0.2220265567,-1.1970661879,-0.4511078000,0.1818592101,1.0391007662,0.1656878889,-0.1195623577,-0.4278938174,-0.3380023241,0.7096031904,-1.2523366213,-1.0001627207,2.1837749481,1.1153163910,-0.3457722962,0.2794967890,-0.3424712420,-0.4181531668,0.5340260863,-0.9294694066,-2.6755039692,-0.5858883858,1.1506539583,1.0913171768,-1.0706495047,0.0423627459,-2.0051174164,-1.3628343344,-1.1141849756,-0.0660549477,-0.9681855440,-0.6248205304,0.6539865136,-0.7431252003,1.1930276155,0.0813564435,0.4269303679,-2.9126818180,0.6502090693,-0.5792543888,-0.0636761934,-0.2226995528,0.2611485124,-0.1963765770,-0.7510466576,-0.4036752284,-0.7355467677,0.8295586705,0.2841912508,-0.5753408670,-0.4433884919,-1.4987837076,0.7639639974,1.1248340607,-0.3037438393,3.0878508091,-0.6452888250,0.1875440478,0.0079111867,0.2569694221,-1.0926113129,0.2290089428,0.3807959557,-0.0390078202,-0.0512206107,-0.0093297828,1.1520316601,0.3930533230,-0.6344077587,0.0940024257,0.8543282151,0.3782372475,0.9214416742,-0.3992048502,0.2794377804,-0.0629065409,-1.1470851898,1.0025026798,-1.7029709816,-0.3982113898,-0.1038763076,1.7992355824,-0.7881760001,-1.4537860155,-0.4431024492,-0.0188008342,0.8453229666,0.3609706759,-0.1380304694,1.3005646467,-0.7697758675,2.0646855831,1.5479987860,-0.6123995185,-0.7303442955,1.0699342489,1.2121047974,-0.2541789114,0.4415258467,-0.2636577189,-0.3476258814,2.0897309780,-0.0466064326,-0.9225341678,1.2898052931,1.8792136908,1.5108665228,-0.1160245165,-1.3695697784,-0.2568242252,-0.8984883428,-1.3459892273,0.3001153171,0.2522737384,0.4287333488,-1.4466867447,0.4010054767,1.7777253389,-0.0489532314,-0.3533559740,0.1734737754,-1.0124557018,0.0805245712,-0.1957098097,0.1044771150,-1.0432385206,-0.3111307025,-0.6315842271,0.2103316486,0.9604236484,-1.4323203564,0.0213550087,0.2500723600,1.5048394203,0.8332036138,0.5828694701,-1.1305952072,-0.5224618316,2.7289142609,0.5754151940,-0.1905027777,-0.7241643667,0.1696766764,0.3804264367,1.3739337921,-2.0588717461,-1.2422560453,-0.2515717745,0.0229164604,0.0536120906,-1.9283407927,1.7888227701,-0.1094532013,1.1283302307,0.0836060122],[-0.1131427437,-0.1455607265,-1.7971167564,0.4484315515,-2.7939643860,-1.3108272552,-0.3902270198,2.8373355865,-0.2132748961,0.9357783794,-1.0359768867,-0.1403545290,0.7936987877,0.7273046374,-1.9549911022,0.0507963784,-0.6482225657,-0.0779463053,1.0098884106,-0.4915691316,-1.7953245640,1.0999864340,0.3593125641,-0.2098834813,-0.6835131645,-0.0330367573,0.1991713047,-0.6592006683,-1.2774186134,-0.9763864279,-0.4536683857,-1.0885910988,1.1693351269,-1.0799335241,-0.7590987682,-0.8095877171,0.2647846639,-0.3310623765,1.2105258703,1.8539288044,-2.3340659142,0.1285315156,-0.3244792521,0.8191900849,0.9074296355,1.1251484156,-0.6740199924,-1.7290600538,-0.2522689700,-1.2518934011,-0.3544958234,0.3565763831,0.5419793725,0.2216321081,0.0744594485,0.4009789228,-0.9030396938,1.1985112429,1.3050527573,0.2644869983,1.2111719847,1.4938938618,-0.3135474324,1.6932300329,-2.1511843204,0.8760904670,0.3790178001,0.1553075314,-0.7556515932,0.4605605006,0.1281558573,-0.5230821967,1.4032703638,-0.2235340029,0.7719467282,-0.8938195705,1.8719172478,1.7318835258,-1.2508921623,1.4505107403,-0.7890445590,-0.1284002364,0.5220596790,0.3965850472,0.3776007891,-0.8109610081,2.5998513699,-1.2116770744,0.6401030421,0.0995526016,0.2845474482,-0.3137201369,0.5138025880,0.5688924789,0.0490821227,-0.5854231119,-0.1241295859,-1.2401731014,1.7278945446,1.3531194925,0.8374717832,0.7225297093,0.7827263474,0.6421630383,1.3519514799,1.1704369783,0.1243656427,-0.4696682394,-0.5816171169,-0.0667436421,-1.6302013397,0.2320342809,1.2981059551,-0.3229315579,2.4780550003,1.4481487274,-0.8240926862,0.6625467539,0.4279925823,1.1662126780,-1.0683534145,0.6251668334,1.0923655033,-0.3942586184,0.9659493566,-0.9605253339,0.9597508907,0.2220281959,0.0806372091,2.1207196712,-0.9710009098,0.1539418548,1.8333917856,0.5665449500,0.3243786395,1.5739785433,-1.0496633053,-0.3769021332,0.4410339892,-0.3513799310,-0.2858914435,-1.0758993626,-0.2779846191,-0.9377691150,-1.5190898180,0.1464475989,0.5293513536,-1.0593185425,-0.7463225126,0.6748778224,-0.9203764200,-0.8092591763,-0.6285470128,0.6317136884,0.5938884020,-0.2016290873,-1.0690140724,0.8877389431,1.0266020298,1.7298130989,2.1594829559,-0.0914283618,1.1662026644,1.1089171171,-0.5123221874,-0.3319963813,-0.3509967923,-0.8236360550,1.2866132259,-1.0179052353,-2.3028659821,0.2056182623,-0.5770489573,0.6231693029,-1.2684029341,-1.0089328289,-0.6858816147,0.2580859959,0.5197789073,-1.2355266809,0.8098257780,-1.1747789383,1.2804471254,0.4993601441,1.1285796165,-0.1324673593,-0.3163600266,-0.1993279010,1.1616035700,-0.4632444382,-0.3264450729,0.3072538078,0.1358029395,1.4254649878,0.1142488122,-0.0371584930,0.4293601513,-0.9944052696,0.6163930893,-0.1263526529,0.6469143629,1.0459427834,0.6449648738,0.3584982455,-0.3914584517,-0.8582537770,1.0382779837,0.4811164737,-0.7321038842,0.8452780843,-0.8298352361,-1.4297674894,0.8056114912,-1.3645051718,0.2932583988,-0.4535759389,0.8559157848,-1.0738148689,0.2676266730,0.0838424116,0.1785915345,-0.0274679642,-1.0839809179,0.1531840116],[-1.6444801092,-0.5802404881,0.6161803603,0.4401788712,-0.0861002505,-0.1942696422,-1.3705750704,-0.3469303250,0.4158354700,-1.9293687344,-1.3931224346,2.0027859211,0.7087925673,0.5379871130,1.9546526670,0.7514656186,-1.3095040321,-0.0490565225,0.5774003863,0.0596319139,0.1235771477,-0.8998639584,1.1102211475,0.3734438121,-0.1030125245,-0.0789468586,-1.3038878441,0.0609854460,0.3799828291,-1.2476456165,0.6118445396,-0.3204786479,-0.4230053425,0.5840607882,-0.6879864931,-1.7932746410,0.9541391134,-2.3621644974,0.4547835588,-0.2293474674,0.9833710194,0.1935296506,0.1290696412,-1.5336457491,-0.7804637551,0.8936599493,-1.0555058718,-0.8772373796,1.6407520771,-0.3166005313,1.5166699886,1.1298456192,-0.3969652653,0.7104807496,-0.9392107129,-0.1705227792,-1.2460235357,1.4000566006,0.2025852352,0.2420491725,0.6935530305,-0.4681382179,-0.7721756697,0.9676588178,0.1198359430,0.1922744066,1.8341497183,-2.7488372326,-1.0611772537,0.1251877993,-0.3508950472,0.8396126032,1.1544494629,-2.1873486042,0.1149946302,-0.6635305285,-0.2694257200,0.6697966456,0.4052609503,-0.4259499311,-0.0662061721,0.0750824437,-2.0473349094,0.5177432895,0.4888772666,0.9255126715,-1.5766785145,1.2748899460,-0.7883622646,0.2249861956,2.6988117695,0.3158188462,1.5834808350,0.1978325099,-0.2180462778,0.5772840381,-1.1148155928,-1.2709002495,1.0019147396,0.1085478067,0.0644170269,-1.2529100180,-0.0105301021,1.6947497129,-0.2921498418,0.6520886421,0.7268299460,-0.5517195463,1.1416976452,-0.8316757083,-0.1236225069,0.7180331945,-1.7250076532,-0.5391926169,0.7702272534,-1.8290551901,-1.0975006819,0.1231060103,0.1836741120,-0.2079072148,-0.3733049035,-0.2890796661,0.1109581962,0.0841258988,0.4721401334,-0.5287404060,-1.0826221704,-0.0720474869,-0.5139278173,0.6535246372,-0.1900931895,0.5698981881,-0.2145587355,-0.1457729191,1.0641998053,1.2511655092,0.8574934602,0.8291519880,0.4578750432,-0.7188019753,0.0051001688,0.1767217517,-0.0890806839,-1.6345053911,0.6368270516,1.6084293127,-1.2352643013,-0.0038276801,-0.9700303078,1.0549601316,-1.4505219460,-0.4465314746,0.8876798153,1.0627869368,0.4874746203,-0.5734726787,-0.5883601308,0.5869722366,0.2755430341,0.0495398827,0.6863536239,-0.8936468363,0.9656808972,-0.0415965989,-0.8703505993,-1.5649480820,0.7539380193,-0.8508267999,-0.7657111287,0.9372233152,-0.5270434022,0.3940381706,0.9115097523,-0.6412190795,-0.7600647807,0.4493492544,1.0575815439,1.2218286991,0.2423966229,0.5654261112,1.1719374657,-1.3715431690,-0.1721050143,0.2998908460,-0.6642464995,-3.1117630005,0.3729720712,-0.3114005923,0.2432861477,-0.3534573615,-0.1880740076,0.5819046497,-0.3220902979,-0.3700904846,1.7209761143,-1.0574181080,-1.2687592506,-0.5830300450,-0.6538046002,0.6858478785,1.8021428585,-0.0014441964,-1.8535352945,1.3842341900,0.5036410093,0.8910794258,-0.2037441581,-0.8835229874,0.2358111292,0.8903995156,-0.6490919590,0.1825971007,-0.4593852758,-1.3794941902,-0.5826588869,-0.2057380974,-0.1274713874,-0.0318536498,0.5356215835,1.3765367270,0.6221526265,-0.5182808042,-0.0778893232,-0.1184563860],[-1.5069004297,1.8594160080,-0.6981487274,0.9367927909,0.4855525494,-0.1547571570,-0.5993320346,1.1967276335,-0.2093621343,-0.0519363619,-0.8732603192,0.1621766239,-0.7586136460,1.8687325716,-1.0110545158,1.7136559486,0.2294831127,-2.1291050911,-0.0054081036,1.2772713900,0.4153493345,1.1322646141,1.2953122854,-0.4021786153,-0.8659353256,1.3614841700,0.4887880385,-2.2888154984,0.7517309785,0.7804059386,1.4343571663,1.1280318499,0.4125819206,1.6644282341,1.0138138533,1.1136579514,0.0347738303,-0.0656888932,1.3539829254,-0.8064793944,1.7992246151,-2.0517346859,1.1155378819,0.3399031460,0.1162784547,0.0452033058,-1.8890718222,1.8050925732,0.7577112317,1.9084178209,-0.2905920446,-0.8593422771,0.5828835964,-0.0455377772,0.5747262239,-0.8365653753,-1.5725414753,0.6745752692,0.4211437404,-0.1733942330,-1.3247904778,-0.3337387145,-1.0581274033,-0.1005994231,1.0474176407,-2.9184198380,0.3160714209,0.6893534660,-0.7181006074,-1.0671374798,-1.9399932623,-0.8578689098,-1.2643039227,-1.0543541908,-1.3231947422,2.0581784248,-1.4635936022,-1.8555091619,0.5931416750,-0.0641021878,-2.0411744118,1.4487619400,-0.9831895232,1.1797959805,-0.9788149595,0.5968145132,1.6935912371,-0.7220646143,-0.5078238249,0.8034408092,1.7207052708,0.6739747524,-0.0758223757,0.5789842606,0.9166283011,-0.2725763619,-0.3547967076,0.9579622746,-0.4331351519,-1.4609791040,-1.0095167160,0.1861312240,0.0965674669,-0.5023012161,0.7978777289,1.1768301725,-0.2873516083,-0.3553848565,0.7831780910,-0.3031035066,1.0905922651,-0.3384682238,-1.5104107857,-1.4142198563,0.5335105062,-0.9249476790,0.3325738907,-0.6441063881,0.3561105132,0.1303441375,1.3840107918,-0.7485288382,-0.9498573542,0.4881732464,-0.0074557588,-1.2329013348,0.8200548291,0.0354016982,-1.2749636173,-0.7796729207,-0.9187532067,-0.7172759771,0.4123795033,-1.2273064852,1.5586960316,1.1174397469,-0.2219770998,1.9217811823,0.6616954803,-1.4222782850,-2.0599975586,0.0867804214,-2.2925868034,1.3697549105,0.8076735735,-1.4938260317,0.0309710633,-0.4543955624,0.9160174131,-1.3730944395,-0.7710475326,-1.1737235785,-1.4549480677,0.3577015102,-1.7589148283,0.0929171890,-0.4369859695,-0.3473596573,-1.5033946037,-0.0985980779,0.0101886811,-1.2029874325,1.4330537319,1.2072029114,-0.0131212957,0.7697699666,0.1444311142,-0.0711760968,1.4660404921,1.2070227861,0.2769919932,0.7466009855,0.4890920818,-0.8605375886,-2.1168172359,1.1051851511,-3.1067254543,0.4607944191,0.5312582850,0.7263253331,-0.1775484681,-0.4622008502,0.3566424549,0.8464894891,-0.6093367934,0.3593332469,0.2175244391,0.7817227244,-1.7723919153,-1.4126907587,-0.3926410079,-0.5199172497,-0.5307080746,1.0303177834,0.2988476455,-2.0974349976,0.5404635072,-0.0937601328,-0.1606501788,-0.1147610173,-0.8684297800,-1.1027407646,0.5567104220,0.6773251295,-1.6910611391,-1.4039698839,0.0599403195,1.2236547470,-0.1871442646,0.0593416244,0.8148764968,0.3711216450,0.6319756508,-0.0623319261,1.5143686533,0.9789410830,-0.7867607474,-1.0358027220,-1.3657990694,-1.8243933916,0.4192204773,-0.2765286565,0.3474358320,0.4425532222],[-0.2964217961,1.0173045397,-0.0976274237,0.6084778309,-0.5599411130,0.2848835588,-0.2352688760,-0.1809513718,0.1363043934,0.1136598513,0.4099833667,0.8103279471,-0.4263535440,0.4786857963,-0.4269832969,-1.9123014212,-0.8065711260,0.5510292053,-0.0150344782,-0.3349220753,0.0880308002,-0.8002303243,0.3176805675,0.8337778449,-1.5033525229,0.5691682696,0.0770459697,1.5434830189,-0.0361299738,-2.0033481121,-0.7982869744,-1.1101838350,-2.1667561531,-0.8929808140,0.2909214199,-0.0600078702,1.8678102493,0.6645435691,-0.5375238657,0.3015767038,0.8310984373,0.8583566546,-1.2841429710,0.2107521594,0.9149329662,-1.4058991671,-0.7324476242,-1.6634695530,-1.3212348223,-0.8275483251,-0.1830658913,-0.9442102909,0.6262663603,1.0551890135,0.8133642077,-0.4543934464,0.9712342620,-0.3185590208,-0.1727422923,1.0257860422,-0.2464818507,0.5092759728,-0.3984328508,-0.7305040359,-0.5991800427,-0.7956777811,-0.1845896840,-0.0305629540,2.0064487457,0.9583771229,0.0363172144,-0.0191859677,-0.3577366471,-1.2744368315,-0.0630775541,-0.0684572831,1.8726197481,-0.1023565009,1.0510714054,0.4325604439,-0.8658845425,-2.0781784058,-0.0383642949,-1.1078237295,0.5350592732,1.4228900671,-1.1049482822,-0.0801641643,-1.0005115271,-0.4297798574,-0.0559677742,0.2867806852,0.3633767962,-0.2082037479,1.1467409134,0.7982702851,0.0084525729,0.1348223388,-0.0987749547,0.4950430095,0.2327910811,-1.5314847231,-0.8182778955,0.2929429114,0.5952238441,1.5427302122,0.8737509251,0.7998639345,-1.2590091228,0.4758330584,0.5333705544,-0.8768574595,0.3324282169,0.0279628281,0.3362767398,-1.4143075943,-0.2139455825,-0.8838997483,-0.4397753775,-1.3462122679,-0.0181162562,0.5175273418,0.3846479952,1.2185842991,0.6779496670,0.0439935140,0.0856824592,-2.0083913803,0.2829786539,-1.2908842564,0.9698345065,-0.1842066050,-1.7637593746,0.8919836879,0.4382879734,1.2267940044,-0.1170026511,-0.2494598031,-1.7412210703,0.6522830129,0.0244456157,-2.2340483665,-0.4830648899,0.1403207183,-1.3167569637,0.1011837199,-0.6489614844,1.0936598778,0.1544923484,-0.9260298014,-1.3525762558,0.7894437909,0.1807664037,-0.6345968843,1.0378496647,-1.1073594093,0.5320475698,0.1034940556,0.7239211798,-1.7146366835,-0.9290454984,0.5460405350,0.6197391748,-0.6808706522,0.5027135015,-1.1057575941,-0.2300989777,-1.2203605175,-1.9491988420,1.2034761906,-0.4060276747,0.4978751242,0.1822032332,-0.6505709291,-0.3541910648,0.2075365186,-0.0094287647,0.2557637393,2.1727738380,-0.8296252489,-0.3163543642,-1.2971352339,-2.0529043674,0.3074778914,-0.2195957899,1.0099247694,-0.0117843701,-0.1268891692,-0.9868423343,0.3460584581,-2.3466484547,1.3982958794,-1.0281090736,0.6707727313,1.4497638941,-1.6454523802,0.3650892377,0.3026635051,-0.4742977619,-0.4233018160,-0.6639545560,2.2567322254,1.9358763695,-0.4846771657,0.6917907596,-0.0628971905,-1.6291610003,1.3567776680,-1.1375604868,-1.7353007793,0.1414480656,2.2786235809,-0.5899149776,0.1783048213,0.7491911650,1.6066801548,1.2810688019,1.7628934383,-0.9193839431,-0.1078255475,-1.4992743731,0.1968777180,-1.1903437376,1.4254845381],[1.0927671194,-3.0272002220,1.9271652699,0.5826958418,0.9965778589,-0.4075433016,0.4611845911,1.0407460928,-0.8860122561,0.8657599688,-0.1595807970,0.4464957416,-0.4966609478,2.9157238007,-0.6837469935,-0.2011281997,0.0069960882,-1.0792711973,-2.0461034775,-0.2763215601,-0.1097814068,-0.2376969159,0.8770941496,-0.9377326369,0.6465483904,1.5600159168,0.4526254237,-0.8150980473,0.0279371645,-0.6981883049,-0.8752954602,-3.2129752636,-0.6134070158,-1.5156295300,0.4352482259,0.1551049948,-1.2635306120,1.9354977608,-0.6287520528,0.5079177618,-2.1001939774,0.3900247812,0.5513725877,-0.5999241471,-0.7899345160,-0.7181108594,0.1940894574,0.6138250828,-0.4738227427,0.8976388574,-0.8623569608,0.7657828927,-0.0611073673,1.1355913877,1.4761710167,-1.0551246405,-0.7247058153,-1.9813483953,-3.0640888214,0.1663405299,0.0124569135,-0.2628451884,0.8943400979,-0.2469936311,-1.9407814741,-2.2188706398,0.4381474257,0.1332459599,-1.7151442766,-1.0381941795,1.9939316511,0.8085494637,-1.1738445759,1.0872621536,-0.1853503287,-1.1145530939,0.4168367386,1.0869874954,-1.6734377146,-1.0069420338,1.4226447344,1.3044387102,-1.2753057480,0.2289368361,-0.3836416304,0.0978016034,0.2819712460,-0.1764324158,0.8378456831,0.2236027420,-0.2701498568,0.6950197816,-0.0077380328,-1.1970260143,-0.3734492660,2.0470554829,-0.4651390612,0.9548599124,-0.2230303735,-0.8019583821,0.3957392275,-1.2221180201,0.4072573483,0.3880814612,-0.6441312432,0.6174584627,1.0846157074,0.6076990962,1.3721759319,-0.0749501362,1.0494583845,0.1591409296,1.2146483660,1.2344856262,0.1549460143,0.9441838861,0.8796211481,-0.5252056122,1.1343315840,-1.0764510632,0.2324069738,-0.1777675897,-1.1545977592,0.7995833755,1.0012146235,0.0061407532,-0.3563856184,-0.2577395439,0.7515842319,1.1220419407,1.0445507765,-0.9168112874,0.6538673639,1.4195749760,1.0493332148,-1.3196792603,-1.7301003933,-0.7242970467,0.6112730503,-1.0061122179,0.7732378244,0.5763732195,0.1579078883,-1.6517345905,0.7781819701,-1.1136115789,0.0968965143,1.1728216410,-1.3469579220,-0.0480953678,0.8859339356,0.1404125839,0.4784304798,1.3342672586,-1.2530455589,-0.5268494487,0.7873428464,0.4130172133,0.1780058444,-0.3267858326,-0.0826260373,-0.3123799562,-0.9598277211,-0.0987945795,0.7427921295,-0.7078848481,0.4165566862,1.3543208838,0.4051081240,-0.8848313689,-0.2286397219,0.2070284635,-1.6439064741,1.2955049276,-0.2987741232,0.9125178456,2.1810588837,0.2293576598,0.8749178648,-1.3403472900,-0.5969886184,-0.4366630316,0.0945704505,1.6732033491,0.4051296711,-0.4160036445,0.9374626279,-1.2810299397,1.7304445505,0.0030098860,0.4869622588,0.8563162684,1.0231707096,0.3627135456,0.2392707467,-1.9643951654,-0.7161080241,-0.2974090278,0.2324644774,-0.3435841501,-0.6919144392,-2.9642279148,0.4640024602,0.0311744493,-0.1404165179,-0.4386716485,0.4086935222,0.1824422330,0.7120854259,-0.6734026074,-0.8858430386,0.3498681784,1.5669513941,0.9241248369,-0.8324046135,1.1146484613,0.9041340947,0.2956512868,-1.0480246544,2.8630208969,1.4399596453,0.5258901715,0.5015463829,-0.3171377778],[-0.4555572867,-1.0919431448,0.5997264385,-1.7279002666,-1.0438963175,-0.0601046048,-1.2033896446,-0.1048328951,1.7255532742,-1.2785211802,0.8037407398,-1.1056637764,-0.8321886659,-0.6981978416,0.6783164740,-0.6690093279,-0.4299170375,-1.9805632830,0.3072091937,-1.4346629381,0.2722908854,0.3559779823,0.5817649961,0.9715235829,0.9819499850,-0.3409855068,-1.3291027546,-0.8595953584,-0.2314220369,0.5113080740,0.1259804070,-0.0081106750,1.8190455437,0.8666212559,-0.1718907505,1.0014348030,-1.0544644594,0.0663620755,0.8409745693,0.1343867183,-0.4915670455,-1.1927989721,0.4487790167,-0.3024437428,-0.9536862969,0.0255348217,-0.2934833765,1.4988203049,0.1637242138,0.1977759004,1.1715903282,1.3605438471,-0.7619611025,-0.5757738948,-0.2096434981,-0.7292847037,-1.3548737764,-1.3679894209,-0.4481071532,-0.6871653199,-0.1607779413,0.3172212541,0.0824580118,-0.0072252350,-0.8245114088,0.8536661267,-0.6266992092,-0.1128967926,1.2639377117,-1.9712094069,0.5339668989,-0.2932064533,1.5378162861,1.1618890762,-0.8614155650,-1.0626561642,1.5011287928,0.0631416887,-0.6804838181,-0.1782898456,2.5135641098,0.4825144708,-0.4274825752,-1.5566039085,-0.1402496248,2.8049387932,-0.9801715016,0.5661875606,0.7939589024,-1.0236939192,1.9633581638,-0.1391828954,-0.2312912196,-1.1576309204,0.5281303525,0.9900379181,0.2896926105,0.0827321708,2.2535164356,0.1212359220,0.5381548405,1.5913234949,2.0978007317,-0.7286148071,-0.5753691792,0.0828324556,0.0549454726,-3.2569141388,-1.8817790747,2.0503520966,-0.2591432333,0.7470239401,0.3248349428,-0.4882176816,-1.3225115538,0.4865359664,-1.0414841175,-0.7895792127,0.8525590897,0.7897837162,-0.6213835478,0.0934152529,-1.9919574261,-1.2485752106,-0.9487129450,-0.5130205154,1.1634507179,0.3678902984,-0.7489489317,0.8876307607,-0.1656009853,-0.4229520857,0.7181720138,-1.0354518890,-0.3245650828,-0.0019222775,1.8520420790,-1.1361961365,-0.7627169490,-1.5301635265,-1.6102468967,0.3663013875,-0.2587484717,-0.3138886690,1.6451128721,-0.4497563839,-0.1718848050,-0.4313556850,0.2476296723,-0.4489573836,2.0903165340,0.7821831107,-0.0666693375,0.7762039304,-1.5657635927,-0.3966723382,0.6482068300,1.2220047712,0.2189141363,-0.3889785111,0.1967448145,0.0520267896,0.4590506554,0.5041958094,-0.1697325110,-1.2416745424,0.1428915411,1.6692961454,-0.0450778902,-1.6467303038,-0.8072274923,-1.3155603409,-0.9456545711,-1.0992310047,0.6658183336,-0.5641582012,-0.3236243725,-0.9900243282,1.3859374523,1.8923567533,0.1954327673,-0.5008239746,-0.8863476515,0.0526065789,2.1536448002,-0.2155486196,-0.8135938644,1.8556420803,1.4562864304,-0.0517918803,0.6744711995,-1.7987359762,0.9923582673,0.3135000169,-1.9504567385,-0.1242371649,1.5248242617,-0.1120917201,0.2942447364,0.4463695288,1.0370858908,-0.2582800686,-0.5030901432,1.0489833355,-1.9220529795,-1.3229802847,-1.1880118847,-0.6967065334,-0.1832891852,0.6462819576,-0.9041520357,0.8098919988,0.8147661090,-1.4799150229,-0.3029147983,0.3950447738,-0.3467047513,0.9894205332,0.9808540940,-1.3005290031,-0.9212888479,0.8779946566,-0.5321686268,0.4928537011],[-0.1064901054,2.0072026253,0.2842397392,0.0219647326,0.2706302404,0.4453656971,-1.0214610100,-1.1261841059,0.5503368974,1.0498505831,-1.1867488623,-0.3131522238,0.2150351256,-0.7758720517,-1.2624981403,1.5144072771,0.5820170045,0.1279897690,-0.4706283510,-1.0356974602,-0.2843975723,-2.1004853249,-0.1418736279,0.3538064361,1.2380549908,-0.0495617092,-0.2308800519,-0.6746603251,-0.5816107988,0.1941962689,-0.7019876242,1.4929260015,-0.8704730272,-0.3817136288,-1.3759821653,1.1188722849,-0.6615345478,1.7322149277,-0.6132860184,0.0772217736,0.7857915759,0.5138965845,0.8683391213,1.0741430521,1.1444586515,0.2174246758,-1.2567451000,1.2492321730,-1.2271503210,0.2029304504,-0.4345974922,-0.5613096952,0.9899421334,-0.7794497013,0.9226804972,-1.4347459078,1.8226745129,0.2517725229,1.3048853874,0.2598344386,0.8284549117,-0.1853842437,0.9909932017,0.8433710337,0.2945882678,0.1580699831,1.1083993912,-1.2196359634,1.0195169449,-0.7478786707,-0.7492168546,-0.7852820158,0.8240640759,-0.1690352857,0.3865447342,3.0318000317,-1.3104873896,-0.2106412500,-0.0911137834,0.1675297022,-0.4070737362,-0.5210168362,-0.6016403437,1.5758588314,-0.4493618309,0.5540955663,1.4807208776,-0.4585899711,1.5710061789,0.2264640331,0.7540287375,0.2100552619,0.1138534173,1.2335205078,-0.1995334029,1.2598119974,1.9921194315,-0.1144331172,0.1608610004,-1.0974568129,1.5505696535,-0.4277614951,-0.7436270714,-0.5684842467,1.5652070045,-1.9541970491,0.1121145114,-2.2796773911,-0.4204390347,0.3072674870,-1.1443328857,-2.1288378239,-1.4380398989,-1.1068999767,0.7632105350,0.8052682877,0.0307863411,0.5012815595,-0.7604537010,-0.1332142353,1.9456522465,0.9724579453,-1.0146543980,0.4536070824,1.9216108322,-0.9443646073,-0.4025148451,0.5432344079,0.2296927720,0.8836902976,-0.4197762311,0.2263710350,-0.4735622704,0.2265483439,-1.0051529408,1.0260937214,0.0039349413,-0.6438735723,-1.5596687794,2.2677609921,-0.0115529187,-1.9361091852,-1.2214096785,2.3677110672,-0.2835217118,-0.4986249208,0.0987454504,-0.2859016657,0.5878900886,-0.2068942785,-0.4712097347,-0.0293408167,0.1004185751,0.5754767060,1.2288657427,0.7598702908,-0.3827791214,0.2701179087,1.9307528734,0.3766086996,-1.2595345974,-0.1427586228,0.2123726457,-1.0897004604,0.9504085183,0.0233728476,0.0613599457,0.1773449779,-2.6466653347,0.4382828176,1.0714106560,-1.0919705629,0.2308921069,-0.5418452024,-0.6811359525,-0.0733657703,-0.1108241528,-0.6205454469,1.3386415243,0.3169844449,-0.0132978726,-1.6194648743,-0.1214035675,-1.2176237106,2.3920612335,0.2677967548,1.9048371315,-0.1090479195,-0.2812086046,-0.8757746816,-0.1215230674,-0.8371799588,1.3561232090,-0.5343367457,-1.3576158285,-0.0527018160,-0.8965148926,-0.1940020174,-0.1157381758,1.3223857880,-0.0518980287,-0.5180981755,0.7235001922,-0.5034645200,-0.9930452704,0.5033494830,0.5044299364,-0.1784189790,2.0653004646,-2.3807713985,0.2423002571,-1.1575675011,-0.8651419282,-1.2484507561,-1.4254765511,2.6023020744,0.5604815483,1.2961738110,0.7792427540,-1.0478221178,-0.4755157530,-0.2933375537,-1.2906724215,-0.5171805024],[1.1661533117,-3.0772411823,0.4945068955,0.8361061811,-0.4539458454,0.6520614028,-0.0636517256,0.1846643686,0.4559874535,-0.7601983547,0.7996939421,-1.3556103706,1.4006017447,-1.2842981815,0.0201086011,-2.4067430496,1.0017011166,-1.2208684683,1.1259891987,-0.1002016664,0.4338685572,0.5633941889,-2.0205998421,0.7079035640,-1.3134320974,-0.7481291294,-2.0741927624,-1.3709870577,0.2138299495,-1.3161886930,-0.5571706891,0.5849643946,0.8160891533,-1.3734140396,0.1693484336,0.4666670263,-0.1848581731,0.6021742225,0.7658632994,-0.2800158262,-0.8746697903,0.6097396612,-0.6713853478,-1.1039898396,1.4990520477,0.1935194135,0.6815602779,0.9476210475,0.0892999396,-0.3394016922,1.2948087454,0.2439201474,2.0538434982,-0.0148545131,-2.3645238876,-1.2313162088,1.4056997299,0.3840621412,1.3106213808,-0.3852375150,-1.1932288408,-0.1657445431,0.0863959864,-0.2528146207,-0.2925288379,-1.4965672493,1.0718343258,1.5679070950,2.0106396675,-0.0910225138,-0.4950281084,0.9399472475,1.1115689278,0.4600949883,1.2392388582,-0.0673111305,-0.8725827932,-1.6198756695,0.1760318428,-0.0825519711,0.2308563888,-1.0177457333,-0.1170732826,-0.8819388747,0.2377493382,1.0289160013,-0.8309114575,-0.8098101616,1.5898245573,1.0980596542,0.3096333444,-0.4698989689,0.5669177771,-0.0059816218,0.4326222241,2.6160898209,-0.5428299308,-0.5957408547,1.2545629740,-0.7029317617,-0.0110392943,-0.3283358514,0.4800703526,-0.4944664538,-0.3524605036,-1.0188369751,0.1142610535,0.4262864292,-0.7915928364,0.4737672210,0.6954224110,1.0050495863,-1.1403791904,0.0269745216,0.8302710652,-0.7582543492,0.0169747900,1.6924520731,1.5741794109,0.1987948418,1.1307752132,0.6800326705,0.3710531890,-0.7827582359,-0.8127122521,1.0480321646,0.3776837587,-0.8211373091,-0.1559403688,1.6256326437,1.0607466698,-0.3728995621,-0.5707845092,-0.3217938840,0.4106550217,-0.9551732540,0.0835519657,0.2593034208,0.7858951092,-0.0355189852,0.4143232405,0.3124325573,-0.6817540526,-1.0727124214,-0.8389334679,1.4706026316,-0.7480475903,0.1054771543,-1.3369356394,-0.0947246999,-2.0778944492,-1.3957155943,0.2285256088,-1.2759611607,-0.3343304098,0.0893022716,0.7450934052,-1.3093935251,2.0228989124,1.3606815338,-1.8446570635,-0.5683281422,0.2735865414,-0.4449757934,-1.9793847799,-0.0671787336,-0.4893623888,-0.6754632592,-0.7179415822,0.2092989981,-1.4558817148,-0.0806349814,1.6811379194,-1.4438048601,-0.7252404094,-2.2249646187,-0.8503048420,-0.1294687390,-0.4447180629,-1.4720023870,0.9454625249,0.5489059687,-0.0052853036,0.4739248753,-0.6366956234,-1.6935415268,-0.7258015275,0.1571865827,-0.6548541784,1.8914365768,1.0652371645,0.4324242473,0.8018820286,-0.5653391480,2.6766371727,0.1969307512,-1.0498875380,2.2293496132,-0.6575648189,-1.5168567896,-0.3831220865,0.0663847551,-0.0127926245,-0.0289653186,0.8270576000,-1.2334251404,-0.1683523655,1.0262917280,2.0759630203,2.6905012131,-0.3199998736,-1.9919358492,1.3505409956,0.4858950078,0.4985644221,-0.0941948369,-0.1956441253,-1.8708121777,0.5297023654,-0.2530247569,-0.7849372029,1.8887374401,-0.1904240698,-1.0549265146],[-0.9172002077,2.4479074478,0.0858606398,-1.1978263855,-0.4034654200,-0.3708377481,1.3119311333,-1.9337878227,-0.2400770187,0.7398082018,-0.7001578808,0.2794561386,0.1471390873,1.7028954029,0.4503670931,0.6144703627,1.8867530823,0.1782859117,0.5037370920,-0.8585119843,-0.9124016762,0.9520156980,-0.4813594818,1.0137593746,0.5279722214,-0.4412348866,1.2565177679,-1.2011139393,-1.2171599865,0.3179094493,2.3057527542,0.0162226334,-1.0044425726,0.8488297462,0.8104820848,-0.3958480060,2.2574179173,-0.8400303125,-0.0290416647,-0.8049724102,0.1110918969,0.1712533683,-0.3029280901,0.9060761929,0.4949313998,1.5811786652,1.1496472359,1.7891368866,1.6989605427,-1.2844060659,-1.4256250858,1.9526252747,1.4928870201,-1.0814081430,0.9162069559,2.1888005733,0.4808399081,-1.2371330261,-2.3750650883,-0.5375353694,-1.4709631205,-0.9133584499,0.0837792382,-1.8881543875,0.6336411834,-1.2488851547,1.1549003124,0.9698441625,-0.1542241722,0.0810905918,0.1434110105,-1.2068035603,-0.2149506211,2.2364194393,-0.7714393735,0.1282031387,-1.6601262093,0.2144560665,0.2410408109,1.5626209974,-0.1163768768,0.9992430210,-0.8798793554,-0.5838215947,0.3413982987,0.1006489098,-0.0714010820,0.0203747340,0.4074044526,1.3556555510,-0.2700893581,-0.9345957637,-1.3730533123,1.3425372839,0.5145732760,-0.7645043731,-0.7455899715,-3.1266086102,-0.0076880553,-0.8621249795,-0.9780056477,-0.4648793340,0.1431073397,0.8786411285,0.0829805061,-0.6876224875,0.9151754379,-0.4527014196,-0.3110263348,0.1074373797,-1.1986666918,-0.6184980273,-0.9270381927,-0.2079146504,-0.4372631609,1.1997834444,0.1383371651,0.2398205400,0.4831244349,0.6147862077,-0.6656706929,0.8235280514,-0.5021594763,0.9050056338,1.0267746449,-0.0839773044,0.2934941649,1.4526919127,-0.5227711797,-0.6307641864,0.7450272441,-0.6373867989,-0.2456774414,1.1337466240,-0.9645944834,-0.5025370717,0.2286235988,-0.4299673736,-1.4565142393,-1.0908288956,-0.6438586116,0.9676718712,-0.7037091255,0.0187808182,-0.8907798529,0.9865032434,-0.2691493332,0.2577497959,-1.9395964146,-0.6774106026,-0.9088381529,-0.0304178633,0.2201902270,0.8568982482,0.7380009890,-0.5199701786,-0.7361801267,0.6977995634,1.9890062809,-0.2889457047,-1.9138393402,-0.9784683585,0.7392961979,0.3418764472,-0.1733308434,0.4325277507,0.1451773793,-1.2198106050,-1.4039171934,1.1004782915,0.1187280118,0.6469854116,1.0802356005,-0.1612346619,0.4949934781,-0.0387254246,-1.1470397711,0.9689227343,-0.6782378554,1.5090565681,-0.1389405578,-0.2824213803,-0.1344258338,-0.1288654208,0.1623164266,-0.1821363419,1.1691294909,0.8031123281,-0.5740057826,2.0071382523,-0.1318964064,-0.4972498715,-0.4756318033,0.9943248630,-0.6209115982,-0.6978612542,-1.5944736004,0.3212108910,-2.1554422379,1.2142683268,0.9055288434,-0.3738659322,-1.6954593658,0.8965313435,0.9389824271,0.9087389112,-0.5691789389,-0.5289321542,1.2088074684,-0.8341738582,0.5690646172,-0.8160740733,-0.2697964609,-0.8304342031,-0.7724876404,-1.3361012936,-2.0136146545,-0.7321896553,-0.5210469365,0.4472853243,1.3969060183,-0.0323589183,-1.2332663536,-1.2633476257],[-0.5402527452,-0.8164409399,0.8692822456,1.5394263268,-0.6449768543,2.7324981689,2.2829384804,-0.0243462957,-0.4708231091,0.7802814245,-0.8517262340,-0.5438752174,1.3492876291,-2.1100623608,-1.0599167347,-1.5305445194,0.9629749656,0.3840721250,1.6027978659,0.6158267260,1.7967692614,0.6978132725,-0.3588051498,0.5557240844,2.0012843609,0.4117252231,-1.2055360079,-0.8306830525,0.7206227779,0.1442007422,-1.0898617506,-0.5330889225,1.3166472912,0.7317251563,0.7280780673,1.6997103691,0.1930397451,-0.1861350387,-1.4679410458,-0.6894759536,0.5577561855,-2.7302639484,1.7028381824,0.4612136781,-0.4558180273,-0.3445228040,0.0529292449,-0.3675781786,0.8226752877,0.4586767554,1.3503264189,0.8677229285,-1.7889962196,-0.9069355130,-0.1365360618,-0.8261981606,-0.0284881555,-0.7588346601,-1.0901699066,-0.9001386762,0.9422703385,0.5553904176,-0.2856693268,0.4512157142,0.1980567575,-0.6102285981,1.1556385756,0.1393267065,-0.5263919830,-1.1312519312,-0.3653927445,0.8627732992,1.0686572790,0.3649345636,0.0013756203,0.1762937754,0.1261652112,0.0962752104,-0.7591469288,0.0422343500,0.1832052916,-0.1315644085,-2.0348117352,-0.5923904181,-0.5324992537,1.1435881853,-0.9107784629,-1.7445204258,-0.4858583212,1.5137287378,0.4766238034,-0.1806454360,-0.9680721760,-0.3028369248,-0.3065006733,-1.1280874014,-0.7446612716,0.8452233076,0.3190322518,0.8402855992,-1.6233057976,-1.4034744501,0.4287578166,1.1428534985,-1.5632708073,-3.8884437084,-1.2155498266,1.5331667662,0.2305153459,0.6118981242,-0.5459505320,-2.5060958862,0.9057552218,-0.0377277546,-0.2225462347,0.5662169456,0.7287843227,0.8154967427,-0.8809786439,0.6194652319,0.2452322841,-0.1912601143,-1.1317753792,0.6059587002,1.3791689873,1.5434559584,-0.3077296615,-0.0549331531,0.6082119346,-0.4465995431,-1.6164356470,0.7707487345,0.3566562831,-1.0651414394,1.2344359159,0.2838335931,-1.4992698431,0.9605651498,1.3660390377,0.4354725480,0.1872466207,0.6108676195,1.2492525578,0.1859040558,1.0766217709,0.2355428487,1.1754379272,0.0057427166,-1.5887515545,0.3769720495,-0.2890666425,0.1181629524,-1.0365852118,-0.9767994285,-1.2941398621,0.4882920086,-0.9798254371,1.8120832443,-1.2130056620,0.2267473191,-0.1387804300,-0.3132337034,0.4717891514,0.2309700698,0.0566052124,0.1241219193,2.1627066135,-0.6082924604,1.2542783022,-0.3301045597,-1.0780427456,2.4032220840,-0.2280689329,-0.2645161450,0.1353555322,2.1393721104,1.4293290377,2.6570754051,-0.7824953794,0.6957845688,-0.2106664479,1.4867365360,-0.6972081661,-0.0462230630,-0.6337217689,-2.0922307968,-1.1119211912,-0.0705752820,0.3467822969,2.1771509647,1.1771267653,-0.3236551881,-1.8883998394,0.4537965953,0.2125834823,-0.3116625547,-0.0193837136,-0.7874299884,0.0644968003,0.4301182330,2.2707767487,1.0474052429,-0.5718730092,-0.6267511845,1.4229493141,-0.5729410052,0.8310288191,0.5821508765,-1.0076065063,-0.1922570169,0.0469058715,-0.1675707102,1.0838055611,-1.3790389299,1.3507223129,0.6134766936,0.0176201425,0.8428995609,-1.4992263317,0.0356088988,2.2975597382,-1.0299751759,0.2906056643,-0.5603120923],[-0.5430343151,-0.5503050685,-2.0349340439,0.3507160246,0.0798382014,1.8635390997,1.4128968716,-0.3009924293,-0.6323676705,1.2433440685,-2.6133000851,1.0574719906,-2.4669671059,-1.0075842142,0.7337694168,0.4244298339,-0.5422949791,-0.5591009259,0.1210958511,1.4105414152,-0.2632221580,0.7265729308,-1.6804473400,-0.7905883789,0.5711280704,0.6042720675,-1.0270519257,-0.5709877014,0.4242979884,1.0850281715,0.5821233392,0.2940220535,1.1921521425,-0.3279542625,-1.0906670094,0.3363183141,-0.4115214050,0.2423718274,0.9718721509,1.3256907463,1.3731383085,-0.3152098656,0.2616109848,0.7625992298,-1.1046898365,0.7405341864,0.3569501936,-0.0304454751,0.4198069572,-1.1654562950,0.3820678294,1.9267227650,-0.4919791222,0.2134685665,-0.5913387537,0.3960608840,-0.3911811709,1.5581468344,-1.7947453260,-2.1093726158,1.1553223133,-0.4462725818,0.3671998382,-0.2049529552,-0.3518499136,1.1037057638,-0.8849042058,0.6334500313,0.8141261935,-0.3845141530,-0.9080251455,2.6200330257,-1.3253928423,0.2427710891,-2.1761622429,0.3425274491,0.4839399457,-0.1631598324,0.8371133208,-1.6910206079,-0.5188969374,-0.5083132982,-1.0096994638,-0.0340005644,1.6234545708,-2.2919981480,-0.8237686753,-1.6870236397,0.2045819759,2.6661920547,-0.3744850755,-1.3754044771,-0.5262570381,-0.1487215459,0.3899958730,-0.3349808455,-0.2147201151,0.8382616639,0.0128131611,0.2007857412,0.4351505935,1.2793378830,0.4645488858,-1.1116894484,-0.2591662705,-0.4701961279,1.0315747261,-0.1856772453,0.9001066685,-1.3104590178,1.3114802837,-1.0081175566,0.3077223003,-0.6241681576,1.4459326267,-2.0826654434,-1.4616817236,0.5706226230,-0.0821249560,-0.4760001004,0.5327296853,1.5234986544,0.1988669932,0.6964995861,0.0093918424,0.7568479180,0.5697199702,-1.8994301558,-0.6445646286,-1.0114517212,-0.3073485792,-1.0358322859,1.1701234579,-0.7295694947,0.1950688511,-0.0414369553,0.7513110638,-0.3562017083,-0.3402531445,1.0202345848,-0.1407631487,-0.7549223304,1.6024886370,-1.5254733562,2.2856884003,1.0988514423,-0.2321708053,0.5067417622,-0.8190221190,-0.9867876768,0.9519258738,-0.6780868769,0.1301680803,0.8194435835,-0.4938240647,1.2405018806,0.6402520537,0.1182226539,0.7945907116,0.0175350085,1.5766327381,1.1100355387,0.5462289453,-0.2211159617,0.5913339257,0.5664343834,0.5663946867,-0.3709756434,0.3571896255,0.7610222697,0.1248397306,0.0199591443,-0.4644696116,0.4498883486,-0.7748786807,-0.2893411219,-1.7340050936,1.2877976894,2.4911108017,0.5305491090,0.7266719341,0.1545901895,0.9030400515,0.3245202005,-0.1829281300,-0.3200109899,0.3289081156,-2.9458432198,0.8093186021,0.3413748443,0.5557262897,-0.1546023935,1.4225945473,1.1587522030,0.2325889766,0.5621083975,-0.9386121631,0.0797256827,0.5854457617,-0.8971057534,0.2960605621,0.4373312593,0.2856488526,-0.2587785721,0.8635243773,0.1729522645,0.4400544167,0.6298682094,-1.7541642189,0.0078843795,-0.4985109866,0.1519829929,-0.4746786654,0.6637639403,1.1471449137,-0.2405385375,0.5008134246,-0.4119012356,-2.0247213840,0.6055507064,-1.0243568420,0.3559515476,-0.8437631726,3.1102230549],[0.7065474987,1.5697489977,0.2100787759,0.3337875605,0.3581123650,0.4796786904,-0.1394378543,1.7159233093,0.9836455584,-0.4811730683,1.2921022177,-1.4980825186,0.3234030604,-0.9263525009,-1.9753719568,-1.2800205946,-0.6501467228,-0.4534287751,0.6986515522,-0.6066743135,-0.3495893776,-0.4151754379,-0.1613209397,-0.3423712254,-0.4407419860,0.3743268549,0.0228571836,-0.2950489223,0.4956834912,-1.0063159466,0.8071094751,-1.4763947725,1.2184668779,-1.2583113909,0.4009124339,-0.9315493703,0.3108949065,-1.0812360048,1.2314478159,-0.0053389100,-0.2175952047,-0.3266625106,-0.6629793048,-0.6949992776,0.1832964271,-1.3664089441,-0.7330182791,0.0111999176,0.3048385084,0.3065909445,-0.3205359280,0.3547713757,0.2387059480,0.0288662184,0.3407653272,0.0688019916,-0.6346371770,-2.9236137867,-0.0657542571,0.2381646484,-1.4751336575,-1.0005645752,0.3903593123,-1.5890965462,-0.5685404539,0.7253620028,-0.4060972035,-0.7793660164,-0.9706671238,-2.1348261833,1.4878499508,1.6540946960,0.0158814453,-1.3564432859,-1.2082302570,0.5394089818,0.4960101247,1.3246908188,-1.0439869165,-1.4069757462,-1.2993481159,2.4255983829,0.9048818350,-2.4917924404,1.1100304127,-1.5330778360,-1.1706732512,0.1931117922,0.2511897683,-1.3754106760,0.3725407124,2.4211819172,-0.4166229665,1.7748960257,-0.3293336928,-0.3670873642,0.2008082420,-0.7868642211,-0.7763853073,-1.8592094183,-0.5754011869,-0.4025988281,-1.4596866369,0.6588805914,-1.7689900398,0.5031039715,1.2791875601,0.1997129023,0.0951175317,0.7077681422,-0.3682105243,-0.6120201945,0.2989490330,0.4668666124,-0.0516404063,-0.0072434973,-1.4731035233,-0.6742974520,-0.1071094051,-0.0481653474,0.3348087370,0.7697380781,1.0444390774,0.6389073133,0.7965302467,-0.5480006933,2.1861627102,-1.7186702490,-1.0547710657,-0.7335004807,-0.4895069599,-2.5456478596,0.4805270433,-0.2115930915,0.6833215356,-1.2567358017,0.5541188121,1.4430524111,-1.2658931017,1.3923492432,-0.5163353086,-0.8217576742,0.5181081295,-0.0065804077,-0.1548570544,-1.4562957287,-0.6649558544,-0.2081828266,0.7567382455,-1.9596194029,1.1570088863,2.8178348541,-0.9480741024,0.3081744909,1.0048877001,-1.0370769501,-0.9271890521,-0.5718141198,0.5977467895,-1.5901451111,-0.5971243978,0.7271687388,-0.3104576766,-0.7801139951,-1.4054712057,0.5915648341,0.6452729702,-0.5297361016,0.0908126980,-0.5574408174,-0.8353986740,-1.0422720909,-0.3353568017,0.7226265073,2.0564503670,-0.5431280732,0.6647246480,0.1695094258,0.0573868677,-0.9498914480,-0.3513489366,-3.0019948483,-0.0066395034,-0.4072309434,-0.9640353918,-1.3792288303,1.2849546671,0.4020608366,0.5306547284,-0.1897467226,-0.9025427103,0.2592071891,-0.2822673023,-1.5999060869,0.8814782500,2.0872652531,-1.8682709932,1.2595149279,-0.7809629440,-0.6400290132,-1.4980249405,0.2437685728,1.3451523781,-0.0861069784,0.0169560835,0.8107209802,0.0504279770,1.0673788786,-0.3041729927,0.7378834486,-0.2011504918,0.2117267698,-1.3680343628,-0.7541540265,-0.8309316635,0.0250439066,1.0769543648,0.1214284599,-0.0662044212,0.4549642503,-1.0856857300,-1.6837859154,-0.1463906467,-0.4956187010],[0.0208592359,1.2435624599,0.5489192605,0.2411355078,0.3825907111,-1.6864269972,0.6340034008,1.2933397293,-1.2414149046,1.3236659765,-0.4560811222,0.0856558979,-1.0383340120,-0.0595628954,1.1350011826,0.8011539578,0.3942179084,1.4639997482,-0.4066986144,0.4662700593,0.9133059978,0.9119114280,0.2083911151,-0.2895689011,-0.6915959716,0.5094326138,1.2007409334,-2.1352896690,0.2560206056,0.2516901791,-1.3380346298,-1.4319194555,-1.0343267918,0.6933841705,0.0523918718,-1.6278096437,-1.7473222017,-0.5706251264,0.3746893108,-1.4037268162,0.6328114271,1.1285928488,1.0005843639,-0.0091215968,0.2692708373,0.5430057049,0.6359573007,0.6018438339,-0.9794576764,0.4686716497,0.1191736385,0.6245647073,-1.6013067961,-0.3516278565,-2.0193760395,0.8631431460,0.6426948309,-1.2152938843,-0.7574124932,-0.1954472661,-1.4739801884,-1.7282948494,0.7948954105,-0.3243267536,0.4970909953,-0.9655341506,1.0447359085,-1.6254166365,-2.4415161610,-0.0110456059,0.6514555812,-1.2094271183,0.8311422467,0.7294400334,-0.7013913989,-0.9180595279,-1.5018708706,-0.2198044360,0.3838777542,-1.1265043020,1.1347600222,0.8210456967,-1.1263074875,1.6014512777,0.1510402262,2.6066834927,-0.8219428062,-1.2476803064,0.6389467120,-0.4005217850,-0.7135859728,-0.6143370867,0.6910892725,0.2512392700,1.4253145456,1.7152156830,0.3454139829,-0.9362812042,1.2939212322,0.5791168213,-1.5516303778,1.2336236238,1.2780562639,-0.2307296097,0.5151617527,2.3914289474,-0.8676834702,0.1678168923,-1.8745352030,0.0887532830,-0.0148388622,-0.5805263519,1.8545913696,-1.0860618353,0.3407716751,-0.0286480375,0.8063637614,0.1843805462,-0.2964666188,0.2472796142,-0.4235060513,0.4835164249,1.4205632210,0.7961882353,0.1411911696,-1.9163612127,-0.6891729236,-0.6566789150,-1.4901683331,-0.6937078238,1.4161351919,-2.0617113113,-1.0494184494,-0.8323073983,0.8514946699,-0.3380451202,-0.8639111519,-0.8786121607,-1.7560200691,-0.7598963380,-2.0147390366,2.4679574966,-0.2002087235,0.3709254861,-0.1325344741,-0.7265365720,-0.1155025512,0.0221462883,-0.5978046656,1.7831364870,0.3680877686,1.3986713886,0.2212256342,0.8435522914,1.4509996176,0.2756426930,-0.0882249475,1.1780829430,-1.2781015635,0.9661692381,0.0849523470,1.4569551945,0.1979640424,0.0714182109,1.2154024839,0.6585696340,-0.3421055973,-1.0114430189,1.7284212112,0.1652764082,0.0440218449,1.9963196516,-0.2177901864,-0.3685661256,0.3754687011,0.7273710370,-0.0082708513,-0.8066374063,1.3471446037,0.9464960098,1.4551038742,1.7335010767,-0.7727061510,1.2335720062,-1.0976076126,-1.2013162374,0.8994884491,0.8080055714,0.0058319694,-0.7987020612,-0.6682382822,0.3353258371,0.2434836775,1.0560492277,-0.6121442318,0.4928761423,-0.3694842756,-1.8778828382,-0.0021931289,0.4397992492,0.0179698374,0.6603020430,-1.2249333858,0.3797037899,0.0017936496,-1.1764934063,0.1773265749,0.2379674315,1.8248631954,2.0317924023,0.5488117337,0.6360759735,-0.0691221654,1.2608212233,-0.0978059471,-0.4273525476,1.1747417450,1.1928589344,-0.7382788062,-0.1795980632,0.3162508309,1.9766232967,-1.1666280031,-0.3134149909],[0.0035257055,0.4764416814,-0.4459216595,-0.0344706252,0.9892446995,-0.7527502179,0.1697129011,1.0399791002,0.8508217335,-0.1022319868,-1.0322203636,-0.8765253425,1.6454482079,0.2713077366,0.0806060657,0.6563324928,-0.3451429307,0.1096281931,-0.4809369147,1.7522681952,0.2002703995,0.1564730555,-0.9563501477,-2.6347720623,1.6463611126,0.2392531037,-0.2259557843,0.3380222619,1.8624340296,-0.1859715879,0.0106931049,0.3789506555,0.1891666949,0.2392414361,1.6173669100,-1.2887517214,0.4338289499,-0.0806743056,-0.4152863920,-0.4792827368,-0.3852518499,1.1886054277,0.2293532044,-0.0757466853,-0.1865629554,-0.6457681656,-0.1063916758,0.2182568908,1.9584716558,-1.3472338915,0.7995253205,0.1375773102,-1.4171931744,-0.0779861510,1.7121382952,0.3812084496,0.1072792262,-0.4451794624,-0.4806461930,-1.0805552006,-0.7667006850,-0.2458909303,-0.5287251472,-0.1539849043,0.4048659801,0.4522774518,0.9267548919,0.0533838384,-0.5965754986,-0.7000019550,0.6710319519,0.8343346715,-0.7737452388,-0.6778567433,0.8769618273,1.4523413181,0.3435254991,-0.9673869610,0.1777562052,-1.7617540359,1.0451806784,-1.2181231976,1.2019933462,1.7208796740,2.4286599159,-1.6599638462,0.6040220857,0.6516717672,-0.2622128725,-0.7239272594,-0.8162904978,1.4671630859,0.2137639672,0.3574137986,-1.0039931536,-1.0461215973,1.1023905277,1.2569910288,1.0668817759,-0.0154753542,-0.9374274015,0.4624779820,-0.1082559749,0.6684598923,-0.5393890142,-2.1180541515,-0.9959486723,0.2782751918,0.2944386601,-1.7939523458,0.4444533885,-0.1882642359,0.5147325397,0.2929969430,-1.8092688322,0.1214284822,-1.0645792484,-1.6098444462,-0.5044631362,-2.3047981262,1.7936631441,0.5146069527,0.9426302314,-0.6366305947,-0.2010298967,0.6305668950,-1.2798579931,0.1300441027,0.1669493467,0.7836005092,0.3279234469,1.2725434303,-0.0735878423,0.5170953274,-0.0156838894,0.2086202949,-0.3578739464,0.5611712337,-0.1845562160,-1.3503365517,-0.2838805020,-1.0690042973,-0.1829974055,0.7866693735,-0.2664945722,0.8257446289,0.4535391033,-2.3860180378,2.7660934925,-1.6022106409,-1.6800652742,0.1000329405,0.2255189270,0.4775266945,0.9800982475,0.0695611909,-0.1636302322,0.7325497270,0.9747005105,0.7435159087,1.4804545641,1.2320071459,-0.8016436100,0.4638025165,-1.3180469275,0.9796977043,-1.6520098448,2.0815844536,1.7063485384,-1.1672502756,-0.2736607194,1.4327979088,-0.1547749192,0.7636130452,0.3382462263,-1.3285471201,0.9728714824,1.3283112049,0.5871783495,-0.3835515380,1.2146409750,1.5588822365,-0.4628805518,0.1570315212,0.7060463428,1.1901321411,-0.6649191380,-0.7949004173,-0.2459549904,-1.0208480358,0.6603729129,-1.2617355585,0.2647471428,-0.2705723345,-0.1136778742,-0.2293915749,-0.2497108281,1.0774041414,0.9895689487,0.7714352608,-2.4308023453,1.0879162550,-0.0287893172,-1.7348122597,0.8923826814,0.2280374914,0.5680179000,0.0501384847,1.2335151434,1.3422961235,-0.3221537471,0.1614855081,-0.6970582008,0.8141419291,-1.5891063213,-0.2522450089,-0.0983667821,-0.1677764803,0.3889777958,-2.3217234612,2.2559692860,0.9853873253,0.1528448313,0.1335994452],[0.0960316658,-1.1012336016,2.1527287960,-0.7543258667,1.4281525612,-0.5952357054,-0.6522353888,0.0763411149,-2.4124116898,0.1143976673,3.1385512352,-0.6567717791,0.5438843966,0.2478468269,-0.8372309208,-1.0154943466,-0.0634565204,-1.0248450041,-1.1556870937,-0.7422597408,-1.4936764240,-1.8463222980,-0.9808491468,-0.7588642836,-0.1710610092,0.0920800194,-0.7544130683,0.6935185194,-0.3152893484,-1.2180192471,1.5641496181,0.2795628011,0.5534697771,-0.3952704966,0.7293699384,-0.4047926664,-0.0643341914,-1.7703485489,0.1241352558,-0.9134722948,0.3001177013,0.7077474594,0.8252443075,-1.1510851383,-0.5846912265,0.2272734344,-0.1831062436,1.2051370144,-0.1876637787,-0.5765454769,-1.0280808210,0.3914548457,0.9839670062,1.0020608902,-1.4458919764,1.3140779734,0.0528018363,0.2362228930,-0.4548761249,1.6524966955,1.7901469469,-0.2248424739,0.1754400581,0.9779208302,0.5776523948,0.9024821520,1.1635148525,-0.3024394512,0.3784768581,-1.7918648720,0.3735060096,1.0666931868,-0.4633731842,-0.5668267608,0.9597026706,-0.7974790335,2.2758290768,0.9152048826,0.9018004537,1.3983670473,-0.9318888187,0.3431453705,-0.2198560387,-0.6310589910,-0.5225726962,1.9764684439,0.3742882907,-0.4727636278,0.2065766305,-0.2785574496,0.2065834999,0.3348022997,-1.0158301592,0.0557430349,1.2103167772,-0.8823601604,0.5185846686,0.2634567320,2.1054568291,0.3655582368,1.0914030075,1.0806419849,-0.7831611037,-1.0495216846,0.4184381962,-0.0781125873,1.3796223402,-2.1329607964,-0.6260906458,0.1810334921,0.5751695037,0.2468598932,-0.1141770110,1.3287128210,-1.1890742779,-0.4207553864,0.7122940421,-0.9073532224,-0.3651553094,-0.4226305783,-0.0830838680,-1.0728796721,-0.1376402676,-0.7822508216,1.2550245523,1.8266563416,0.7291188836,0.8544880748,-0.2885992229,-0.6249886155,-0.1918189228,1.0057471991,1.3464815617,-0.0350044779,-0.7850279212,1.2537226677,-0.7549847364,-0.7763412595,-0.5457606316,0.8749659657,-0.1705278009,-0.1407030970,0.1812559962,-0.2123323083,1.0182181597,-1.2406775951,-1.7901406288,-1.7229549885,-1.9905031919,-0.5597758889,-0.6072100401,-0.2092664540,-1.9350706339,1.2083387375,-0.7741381526,0.7506406307,0.9133481979,-0.7715125680,0.6030091047,0.5493309498,0.4792211056,-0.2988275588,0.9531223774,-0.0966653600,-1.4549390078,-0.5645372868,-0.3757342100,1.3526890278,0.7144587040,0.2461370528,0.0577381253,1.6676069498,-0.5236766338,-1.4429916143,-0.7393116355,-1.2088468075,0.0605625138,-1.5278388262,-0.9210611582,-0.2517165542,-1.9440078735,0.6988514066,0.1982673407,0.3086035848,0.5408528447,-0.9238742590,1.0745069981,1.2209699154,-0.5641684532,-1.5028772354,1.5196889639,0.2270999849,0.5806286335,-0.1388664693,0.2372200936,0.6519583464,-0.1114310697,-1.2452012300,0.8265328407,-1.1701641083,-0.9429509640,-0.0014479960,-0.0517590232,0.7875321507,0.2763226330,-0.3950987458,1.4887293577,-0.0903558433,-0.6510680914,-1.0548905134,-0.1655217260,1.2616093159,-0.0127523160,-0.1354828477,-0.2113614231,-0.6180521846,-0.0597930215,-0.2407428026,-1.5077199936,-0.1971070170,-0.6861489415,-0.0931828097,-0.8914435506,0.1905655414],[-0.8744983077,0.0098695075,-0.3797313571,0.7740601301,0.0845667273,0.2132222652,2.2988576889,-0.1039733142,0.7457541823,-0.8987375498,-1.3461180925,-0.8154997826,-0.5025764108,-0.3514509797,-1.0883080959,0.8634200692,1.5210120678,-0.8854886889,-0.7128968239,1.0747821331,0.7041067481,0.8238183260,0.4915956557,-2.5970296860,-0.5477147698,-0.9649059772,2.5200729370,-0.3640636206,0.8815293908,-0.9766589403,0.4918274581,1.3674652576,1.0179756880,-1.2977014780,-0.8691310287,0.3466084301,-0.6623682976,-0.3450996876,0.6613887548,1.5178869963,0.4877279997,0.9343072176,-1.0097370148,0.6456452012,0.3174564540,0.7257652283,-1.3117456436,2.4636328220,-0.6551507115,-0.2571562529,-0.4730632901,1.5332181454,-0.7421475649,-1.1132484674,-0.5764532685,1.0589469671,-1.1869277954,-0.5670490861,0.6283428073,-1.1403493881,-0.9345607162,-0.4807460606,0.3763595819,0.4279716611,1.1986782551,-0.2374680340,-1.2819801569,0.1377163678,0.4830263853,-0.5013511777,-0.6719954610,1.5218192339,-0.1730282307,3.6219646931,-0.1248321682,-0.1220605448,0.2830874920,1.6604256630,0.5359668732,1.3583760262,1.6301083565,0.6225463152,-0.9911639094,-0.8537741303,-0.6471802592,1.8175919056,-0.1935535520,0.0851218626,-1.0140290260,-0.1194371432,-0.7439462543,0.3609744310,0.1996553689,0.4417337179,0.0744811743,0.6526217461,0.5125229955,-0.2765215933,1.2034013271,-0.4930281639,0.6316803694,-0.8473200202,1.1323636770,-0.9325317740,-0.1085536703,0.2190643996,0.4257893264,-0.1137780547,-0.8615435958,-0.6558609605,1.7273952961,-1.1156474352,0.4715724885,-1.2323323488,-0.9245020151,0.2340586185,0.4171338379,0.5103055835,-0.8030940890,1.3934131861,-1.7207546234,0.8516317606,-0.7090355754,1.3007886410,-0.2409402430,1.5606890917,1.2331166267,1.1565121412,1.1419471502,0.4737171829,-0.0646478087,0.7538887858,1.1985429525,-0.8085051179,-1.7302092314,-0.7214023471,1.0235490799,1.9192208052,-0.0632185340,0.5950103402,-0.0955924690,0.0445052236,-0.2363787144,-0.6596962214,0.9489222169,-1.8176273108,0.3249565661,0.5321244597,-0.2519031167,-1.0410438776,0.3578720987,-0.2318406850,-0.4318566322,-1.0400800705,0.0432879440,0.2405186594,0.6484411359,-1.4198623896,0.5731082559,1.6903151274,-1.2536605597,0.8934561014,-1.7532277107,0.3545506299,0.3714213669,-0.4971423745,1.3633073568,-0.8850327134,0.5577733517,-0.0002178800,-0.7989585400,1.0268219709,-2.0480480194,1.0952467918,0.4634559453,-0.0991200507,-0.2166755944,-0.1800895333,0.2274161875,-0.1490253955,-0.1709231734,-1.1723672152,-0.7563461661,-0.4348413944,0.8389985561,0.5278283358,-0.0855753422,0.6674067378,0.1275337189,0.1393093020,0.0813582242,0.1333002299,-0.5479602218,-0.8135802746,-0.7446947694,1.0566511154,0.8022787571,-0.8659976721,0.3931536376,1.5148967505,1.2328475714,-0.4744767845,-0.0396147296,-0.3586524427,0.0859779865,0.8946546912,-0.2486164570,-0.3365946710,-0.6286838055,2.3325791359,-0.0751589090,-0.1462642252,-0.6692479849,0.1297315210,-1.5226391554,-0.3211878538,-0.5852881074,-1.2526824474,-1.5365287066,0.4731831849,0.8354712129,0.1848156005,1.1968663931,-1.0546797514],[-2.1675684452,-0.7382207513,2.6727535725,0.5967551470,2.2287185192,1.3105413914,1.5683557987,0.3387235701,-0.5780242085,-0.9694200754,-0.0542619228,-0.7023670673,-0.3473659456,0.1283979267,0.9860328436,1.8086926937,-0.2518500388,-0.7601784468,-0.6687451005,0.5814293027,1.2638016939,-0.0553046763,-1.8108512163,0.1957877129,1.5420058966,-0.6303429008,-0.0445778482,-2.4625091553,-0.2039029151,-1.0149819851,-0.3121074140,0.7373507619,0.0069484706,1.5634709597,-0.4883809388,1.6265809536,0.4967319071,-0.6871525049,0.5922659039,0.8526945114,0.6075148582,0.2839967012,-1.9847686291,1.2767001390,1.7508671284,0.2804496586,-0.0903932750,0.7302295566,-1.0770375729,-2.1788797379,0.0533087961,0.5184302330,-0.9022838473,0.7812830806,0.7872949243,-0.8752070665,-0.7053655386,1.6697065830,1.4777274132,0.0331411175,0.4019483328,0.0795291141,-1.7472738028,-1.6027665138,-0.8463315964,-1.3971626759,2.2727987766,-0.7152912021,1.2079150677,-2.5324273109,-0.2201086730,-0.8586494327,-0.6715086699,1.0920449495,-1.4367811680,-1.1334613562,0.8020817041,-1.4378912449,1.9518941641,1.0430494547,-1.9164535999,-0.7914465070,0.4349558949,-0.6983420849,0.8788402677,-0.6938580871,-1.6382540464,-1.3454945087,0.3314038217,0.7301472425,-1.6229919195,2.4571003914,0.9816461205,-0.1934306026,-0.6380050182,-0.6916678548,-0.0730158985,-0.6435752511,0.2499887049,0.9054161906,-1.9165015221,-1.7656427622,-0.9856516719,1.3381596804,-0.0236687176,-1.7433477640,-0.4669364989,-0.3746806979,-0.6021181941,-0.3435127437,0.5429405570,1.6806927919,0.3754731417,-1.7427126169,1.1184897423,-0.0914018527,-0.2547079027,0.0444798470,1.2944244146,0.9571075439,-0.9411146641,0.0178249963,-0.2201198488,0.2704386711,0.1415091902,0.7426109314,-0.1449626833,-1.3681911230,-0.9975286722,-0.7867771983,-0.1971043646,1.1083836555,-0.2619640827,-2.1241817474,0.5822571516,-0.5158762932,1.4686164856,0.4476768076,0.1179724336,0.8681359887,-1.6180802584,1.3157556057,1.3680702448,1.2255792618,-1.2426224947,-0.5846403837,1.3134173155,0.8510391116,0.0566169582,0.5327958465,0.5105850697,-0.0755836368,-0.1719992310,0.5381946564,0.0138440710,1.1242705584,0.3068951666,1.7257516384,0.9957590103,0.8762136102,0.1220595390,-0.5659295917,0.2277036905,-0.5829985142,0.2336221337,0.5245922804,0.6979594827,1.2982851267,0.6850818396,-0.5728030205,0.5248136520,-0.3277309239,0.4528121650,0.4383815825,-0.9423355460,1.1154732704,0.1451929361,-0.7689108849,0.2972296774,-0.5694436431,-0.1806611866,-0.9186413288,0.7959067225,0.4119114578,0.5434252620,-0.0524524115,0.5843896866,-1.5658318996,0.2693421543,-0.2651713192,-0.6462969780,0.9390318394,-2.3189029694,-1.9477659464,0.9899591804,-0.3546177149,0.1199787855,-0.4854714870,-1.6318863630,-0.1745455414,-0.4783552885,0.1253313422,-1.0133267641,-0.1568865627,-0.9031783342,0.0745390579,-0.1635649502,1.6315912008,0.3370809257,-0.5317426324,0.1358670890,-0.7163832188,-1.0768035650,-0.9230828881,0.2219968587,-0.5246748924,0.6829332709,-0.2877226770,0.2317267507,0.4880829155,0.6605032682,0.7699363232,1.2062032223,-1.3281532526],[-2.0703158379,-0.3716563880,-0.3761352897,-1.0077327490,0.0220914204,-1.0736358166,-0.5218760371,-0.6671406627,-0.3167509735,0.2668158710,0.7138624191,1.6762847900,0.5364630222,0.7131706476,-0.4521187544,2.2354354858,1.0070247650,-1.4814945459,-0.9544055462,0.5616211295,-0.4388044178,-0.5384815931,2.5188238621,0.3578646779,1.0072035789,-2.3006868362,-3.0283837318,0.1582179517,0.0786268860,0.2975986898,0.3020519316,0.2252492458,-0.1169260144,1.1698973179,-0.2377756834,-0.9876517653,0.9444678426,0.7366877794,-0.2140249312,0.6662899256,-0.1698795259,0.1597381830,-1.1117084026,1.5774035454,0.7038061619,-0.8269827366,-1.2444053888,0.6046368480,-1.2741203308,-0.0075483327,0.1214978695,0.1185389608,0.1672204137,1.4469048977,0.9831426144,-1.2359778881,-0.0607702211,1.0580685139,2.1470263004,-0.6063710451,0.3545796573,-2.0619568825,-0.6264045238,-0.6515289545,-1.4760322571,0.6560636163,0.1312911958,-0.6359328628,1.0950239897,0.8734268546,-0.0087732999,0.2270132303,0.2920324504,-0.5871611834,-0.6498407722,-0.4679347277,0.7037794590,-0.3263746202,-0.6026700735,1.5626196861,-0.3909978867,1.2148580551,-0.1074242890,0.2417563349,0.8551151156,-1.1614420414,0.4012592733,-2.6826231480,0.2988246977,1.0284411907,0.9653552771,0.0241382923,-0.6744859815,0.0252455268,-0.5691433549,0.9945526719,-0.7929928899,-0.0188295841,1.7620083094,1.1326615810,-2.1382300854,-0.4516732097,-1.7331445217,0.9578317404,-1.9982173443,2.0854375362,-1.4002338648,-1.2631605864,-0.2547115088,-1.1759921312,-0.1266184151,-0.1415224522,-0.2777317166,1.8104367256,-0.8818674088,-0.1876362264,-0.6606785655,-0.8318985701,-1.3102446795,-0.1048089117,0.2003894299,-0.4948415160,-0.6170853972,0.3395398855,1.1125634909,0.8719718456,-1.1671817303,0.1546787918,0.1532687992,-0.7465642095,-0.3725242317,-0.5178562403,-1.1987986565,0.6836966276,-1.4780657291,1.3997263908,-0.7279686332,-1.4971139431,-0.7506303191,-0.8950587511,0.3868414760,0.0534542501,-0.2117811590,-1.5080431700,-0.7876279354,1.1501566172,-0.3116948903,-0.3763196170,-0.9797936082,0.5246469975,-1.2769556046,-0.6016128659,0.0543154590,-1.8809392452,-0.8625527620,-0.4563853145,0.5321665406,-0.2617306411,-0.6068289876,1.8533170223,-1.9562035799,-0.5140955448,0.4745437205,1.1698094606,-0.5805649757,-0.3693213165,-0.9228043556,-0.7345765829,0.5629547834,0.3122575879,-0.3497559130,0.9135907888,-0.7911636233,0.1847207397,0.3809929788,-0.8879823089,-0.1338323057,-0.3490095735,2.1165955067,-3.1589138508,-0.5436285734,-1.7455108166,-2.1049289703,-0.1302325428,0.4640696049,-0.7451955676,0.5188206434,-0.4138991535,-1.8001177311,1.1008683443,0.4474474490,-0.1786603630,-1.0551677942,0.5523467660,1.5565564632,-1.0508276224,-1.2913016081,0.4513285160,-0.7465084791,-0.2419379652,2.0543141365,-1.3258612156,0.8564865589,-0.7004728317,0.9789661169,0.2529447079,0.3756442666,-1.1552598476,0.2251007259,-0.6217958927,-1.2075531483,-2.1160759926,-0.5377485156,-0.4078000784,1.2833286524,1.2621468306,1.9884413481,-1.9587829113,-0.3365916312,-0.3414989412,-0.1485837251,-0.5208674073,0.9343284965,-1.1504461765],[-0.7001681328,0.5539194942,-1.4810436964,-0.6036912203,0.9199162126,-0.3080572486,0.4597660601,0.0606138296,-1.4523596764,0.0182523839,0.3977748454,0.2504182160,-0.0457268581,0.1204058975,0.8092479110,0.7734143734,0.7709316611,0.4113011956,-1.3937188387,0.4055718184,-0.4176865220,-0.3609451056,0.2942406535,1.7711349726,-1.3721432686,1.8080185652,-0.6679579020,-0.6259421110,-0.4016775787,1.0835027695,0.9990584254,-0.1049466804,0.0027226934,-0.8598833680,0.5965404510,0.4464733899,-0.9998130798,0.3802746236,0.1903662384,1.0069195032,1.7607945204,-0.8770285845,0.0015625909,0.1690426022,0.4705689251,0.2795690894,-0.3947535753,0.2917557359,0.4942769408,1.7180832624,1.8849114180,1.0528432131,0.0112085836,1.9515383244,-0.8529167175,2.5300376415,0.9883424044,1.5475788116,2.0330502987,-0.4147629738,-1.7989500761,1.3826758862,-1.0269834995,0.5999916196,0.9867404103,-0.2702056170,0.3600501716,-0.7583336234,-0.5758518577,-1.0197910070,0.6279180646,0.5060213208,-0.1114388481,-0.7055656314,-0.1577177644,1.3694399595,-0.1905371100,-0.6570346951,0.3182444572,0.6559875011,0.1073881984,-0.8325852752,0.7134782672,-0.0001436342,1.1665621996,-0.9761604667,0.4922949076,0.1762838662,0.8394795656,-0.9334070086,-0.6946863532,1.1474127769,0.0837756693,1.0693693161,1.5215953588,-1.8261275291,-0.5862078667,0.8109973669,-0.0215907376,-0.3971729875,0.1443639547,0.7105217576,-1.5630483627,0.8906173706,-0.0771133453,-0.7070862055,-0.6657230258,-0.2032502890,0.4337565005,0.6398509741,-0.0961007103,-2.0606822968,-0.3801790178,-0.7287710309,-1.1679856777,0.8636924028,2.0345959663,0.8056660891,0.0790649727,0.1292844862,-0.2441989928,0.6843965054,-0.0466990806,0.2574798167,1.7390024662,-1.0807328224,0.3902039528,-0.0904696211,-0.6639072299,-0.9807626605,-0.5844775438,0.7836155295,0.0463408977,-0.5736125708,-0.2846724689,-0.8746067882,0.1992656440,-1.1177204847,1.5649669170,0.1965961903,0.8586657643,-1.4847395420,1.4379744530,-0.3827043474,1.0768358707,0.5697392225,-2.5409808159,-1.1086584330,-1.4590468407,1.9061291218,-0.4039606154,-0.3141342402,-0.6788002849,0.4221759439,-0.6185811758,1.2265267372,1.3419264555,0.9365577102,-0.8626218438,0.0714093968,0.3579553068,-0.7517531514,-0.1326746047,0.4824095666,-0.1637495905,-0.2193266600,-0.7112823725,1.6637583971,0.6090463996,-0.3620583117,0.7486864328,1.4172860384,-1.3887069225,-0.4702116251,-1.8121628761,-0.4067336619,-0.4508442581,1.6809742451,-0.6856459379,-1.0960397720,0.6169106960,-0.5126711130,0.7367286682,-0.1227271408,-0.9381112456,0.3682388067,0.5237234831,-0.1901168376,-0.7895736098,0.0390522219,0.1888903528,-1.9014731646,1.7833452225,0.3512957990,0.0422377065,1.1961449385,-0.4127797782,-1.7096159458,0.4396434426,1.6618039608,-0.2889024615,-0.4047232866,-0.3430258334,0.4502983093,-0.8993715644,-0.2819147706,-0.6409164667,0.3729336262,-0.7371287346,0.4244229794,-0.9481826425,1.7409205437,-1.9509413242,-1.0926223993,-0.7474913001,-0.3169195354,0.2738073468,1.0230799913,0.5049025416,-0.1530090719,-2.4460172653,0.3451049626,-0.7874396443,0.4234744608],[0.8010879755,-0.7266035676,-0.0287357550,0.5261215568,0.4974059165,-2.2859344482,-0.5524745584,-1.6442023516,0.2132236212,0.4708622992,1.5447111130,-0.4303492308,2.1815609932,0.3391433060,-1.6993885040,0.5555101037,0.1248709261,-0.5245776772,-0.8237133026,0.0852307156,-0.6021905541,-1.9891923666,-0.2042742521,0.0665240660,-0.5630695820,1.3211286068,2.6150777340,-0.8884028792,-2.7250652313,-0.7953874469,0.4453403652,-0.0466313288,-0.8660322428,0.9148148894,1.3114209175,0.0853873789,-0.0865378082,-0.8240836263,1.5242363214,-0.1447027177,1.0685709715,0.6887947917,0.3693505526,1.2384396791,-0.1528254151,0.4717794359,0.2557708025,-0.2342842966,0.2985180914,1.4603908062,1.1556718349,-1.0860780478,0.8485520482,0.8186067343,-0.4889854193,-1.1824482679,-0.0552532971,-2.0472691059,0.3136096597,0.0956232697,-1.2475279570,-1.0659813881,-1.4372975826,1.3939661980,-1.5543537140,-0.2287389338,0.8398764133,-1.0875582695,0.3782944977,0.5431341529,0.8844070435,0.0712101758,-0.7438293099,1.4641083479,-0.2514543831,-0.7803747654,-0.7884621620,-0.5040911436,0.1236531809,1.3709342480,-1.2878456116,-0.0993014053,-0.5816023946,-0.1617263407,1.4658393860,-1.1843436956,0.1996217370,0.7093799114,0.6478058100,2.4174399376,-0.9946103692,0.9256981015,0.2354338616,-1.5637323856,-0.4097500741,0.9532274008,-1.7446402311,0.3983721733,-0.9129959345,-1.8641510010,-0.0650398284,1.8328038454,-0.3282188177,-1.4489845037,-1.2304841280,0.3586666584,-0.2042628974,-0.3145161271,0.9496209621,-0.3764860332,0.7279493213,-0.8917263150,0.3433779478,1.1824033260,0.0408701040,0.4175057113,-0.6440492272,0.9012810588,-1.0648696423,0.9149550796,-0.5418872237,-0.3558420837,0.8348129988,0.1753689051,0.0140529331,1.4487305880,-0.0789629817,-0.0814047679,-1.5064017773,-1.0320729017,0.5171748996,-2.2602293491,-1.7476472855,-1.5101404190,0.3992056251,-1.5515506268,0.7638925314,0.6154313684,-0.2482495755,0.0194012262,0.0098772850,0.4108774662,-0.0947879553,0.4412504435,0.1092329770,-0.7233854532,-0.5602306724,0.9036995769,1.2431905270,0.0285293981,-1.6290500164,1.2356460094,-1.9054615498,-1.5224015713,0.4625911415,0.1904840916,1.5625308752,0.4856654108,-0.0246778559,-0.1548504829,1.2992460728,0.1943925023,1.2544627190,0.6112693548,1.4020404816,0.8133472800,-0.7494078279,1.3856695890,-1.6867324114,0.1892388910,1.2696735859,-1.3791120052,2.2082250118,-0.9735414386,-0.2286798209,1.3652757406,-0.8393858075,-0.6100654006,-1.1962810755,-1.3717755079,-0.7170159221,-0.8828419447,-0.3372710645,-3.3247058392,-0.9084119201,1.6302831173,0.8176118731,-0.0111802975,-0.2251095921,0.5649040937,1.4012663364,0.7089836597,1.2665133476,-1.1005299091,2.4844765663,0.1761873364,-0.0088386908,-1.3534541130,0.5137736797,-0.5706036687,0.8757398129,0.0449838303,2.4435112476,0.2079927921,-1.9996918440,-0.6087223887,-0.6385773420,1.6442676783,-1.4658410549,-0.2207080126,1.0008113384,-0.7584369779,1.4786957502,-0.4342084229,-0.0399412848,0.0586465672,-0.5101655722,0.0631559715,1.0645006895,0.9889351726,0.7960072160,0.9740015864,1.7200036049,1.4311993122],[0.3660484254,-0.0995860323,0.2606077790,1.2410806417,-0.9831237793,-1.6199501753,-0.1432815939,1.0276182890,1.9477783442,-0.1715546250,0.9028071165,0.9455989003,1.7721807957,-2.3770511150,0.0422005840,-0.2649149299,0.1626689434,-1.7165741920,2.0142836571,1.2888416052,1.3182411194,-0.2988808751,-0.9702196121,-1.3273938894,-0.1208585128,0.1018750072,-0.3974203765,-0.9312421679,1.5495072603,-0.2257322222,-1.5345994234,-1.2101567984,0.5442845821,2.3244996071,0.6885232925,-0.3893218637,0.5857572556,0.8583837152,1.1273128986,1.5923551321,-0.6475256681,-0.4328983426,0.3193134367,0.3736357391,-0.0583607033,0.9926229119,1.1097198725,0.0237506758,-0.3030325472,-0.4242801964,0.0150921475,0.8205795884,-0.1523793340,-0.1743751317,0.1002234742,-0.5741091371,0.4680297673,1.3265185356,0.7591851354,0.7968487740,2.4510006905,1.0455299616,0.1616481096,0.4591381252,-0.5118848085,1.3974906206,0.0444282293,-0.2367054075,-0.5850853920,-1.7926813364,1.0850917101,-0.0153463064,0.8085225224,-0.0650879070,-0.8191810250,-0.3907743394,-0.8836235404,0.8422560096,-0.9372207522,0.1334853172,-1.4340825081,-1.0172991753,-2.9318859577,1.7403756380,0.6695041060,-0.5234505534,-0.4980879426,-2.2248792648,-0.3270122707,1.4429141283,0.0257454775,0.9544253945,-0.1030614227,-0.0222805012,0.8595652580,0.7901018858,0.0139517374,-1.5680670738,-0.7101589441,-0.9396982193,0.5569794774,1.6559907198,-1.0252135992,-0.5993455648,0.7724023461,-0.0052897935,-0.7790496349,-0.5721173286,-0.1611474603,0.1147837266,-0.4912993908,0.2682353854,-0.1408393532,1.1909285784,0.1201085225,-1.1754007339,1.0866947174,-1.3127019405,0.5852900147,0.4494962990,-0.0917720422,1.5772259235,-0.0028221165,-0.9338590503,0.4315549433,-0.7911572456,-0.7792272568,0.4389078021,0.2131284922,-0.4663459361,0.0148027567,0.6259170771,-0.8518497944,0.0935833678,0.2924852371,0.3590088189,-0.5718174577,-1.1496263742,1.2068468332,-0.1130583733,-0.2927685678,-0.6665418744,1.2084698677,-0.5074796677,0.6037199497,0.5330540538,0.0671800524,-3.1931815147,1.3412168026,-0.0472922698,1.2909542322,-0.0951048583,-0.3293198049,-0.3368616104,-0.6205972433,0.1633470356,1.0400021076,-2.4417200089,-0.1430457532,-0.3949324489,1.5079751015,-0.0933669582,0.7111688852,0.4588423967,1.0893750191,0.6711891890,-0.1723787934,0.3056735396,-1.0339529514,-0.5147407651,-0.0625401884,-0.4944783449,-0.2281164825,1.8994734287,-1.1122488976,0.8064043522,-0.9070198536,-0.7515420914,0.4880173504,0.5601146817,-1.6990242004,-0.5601556897,0.0330446623,-1.1386178732,-0.0413848385,-0.4364368320,-0.0484491810,-0.8381305933,-1.9084656239,0.3641952574,0.4897067845,0.2050378919,-0.5196628571,1.1518422365,0.2143942863,0.6736453772,1.3390365839,-0.9382547736,-0.5802168250,-0.2404362857,0.0388144962,1.2864723206,-0.7813917398,-1.4558441639,-0.5873885751,0.9965207577,0.0476341769,-1.6072156429,-0.6474640965,0.7038937807,-1.2242337465,-0.3836533427,-0.3961207867,-1.0149700642,1.0830006599,-0.2466066629,1.1158556938,0.6452320218,-0.1450219601,-1.9907687902,0.2957869768,0.7098045945,-0.0348816589,0.7518092990],[-0.5935727358,-1.2528318167,-0.5177393556,1.5163943768,1.2004730701,-0.9606300592,0.4015153646,0.4090261459,-0.0634377599,1.2732558250,-0.3282780945,0.9231778979,0.2382800877,0.8408981562,-0.7702533007,0.4584795237,0.1061263978,0.0769923702,0.4656309485,1.1552935839,1.3841668367,-1.3653978109,1.5498673916,-0.3918735981,-0.0810078084,0.6211736798,0.7627735138,-0.0829695016,-0.9319149256,-0.5280153751,0.2751726806,0.3330031931,-0.7518762946,-0.1446366012,0.1988864541,1.2466605902,0.4027922451,-1.2030483484,0.4533600509,0.7313265204,1.0936702490,0.0037894470,0.0050270087,0.0251088440,-0.0352713205,-0.5701832175,-1.9719990492,0.4366341829,-0.1345392913,-0.1814393252,-1.8553251028,0.0617607608,0.6062008142,-0.4244881868,-1.0817292929,-0.6807661653,-0.6261473894,-0.0540766194,0.2786253989,0.8372100592,0.0017704318,0.5310544372,0.6352730989,0.9221379161,0.4457738400,1.5647474527,-1.6837626696,1.2410209179,-1.0796965361,2.2747447491,0.5723548532,-0.5651129484,-0.1718373448,-1.6550757885,-0.3474877179,0.6547319889,-0.7872471213,-1.5143712759,-0.3837069571,1.4854574203,1.3285131454,-0.2988612354,1.3719160557,-0.3424156010,0.4441322386,0.3561630547,0.3292013407,-0.4663069844,-0.3964741230,1.4079635143,-0.0168015026,0.2487754226,-0.6136851311,-1.5734347105,-0.8163513541,-0.2254180610,-0.0962739587,0.2816110551,-0.9428220987,0.1577043533,-0.2643011510,-1.3468902111,-1.6785992384,0.6566977501,-0.1757939309,-0.4243683219,-0.8836905956,0.1485837400,-0.7256431580,0.0940819830,-0.7429986596,1.3145816326,1.5680030584,-1.3079791069,1.0795658827,1.6322503090,-1.3270734549,0.2825237513,-1.8363772631,1.4280408621,0.8779270649,0.1488070041,-1.1796215773,-0.7179731131,0.7257335186,-0.0448335893,-0.4180825651,-0.9488744736,1.3562891483,0.6680294275,-0.4531681240,-1.1939545870,-3.9058244228,1.6860084534,0.0763802007,0.8831697106,-0.0188103337,-1.6418547630,-1.3217287064,-1.7891827822,0.3362942636,0.3228811324,-0.1052464694,0.3700760007,-0.2983788848,-1.7491821051,0.0411379486,-1.1203415394,0.6413006186,0.0217642933,-0.6283631921,0.5370006561,0.0651604682,-0.6565572619,-0.4752445519,-1.2263764143,-0.6557531357,-0.1826541275,2.3952448368,0.1607136875,1.0272328854,-0.8275399208,1.0438835621,0.7619448900,-1.1307396889,-1.2205406427,1.3907771111,0.6221396923,-1.0409410000,-0.7856572270,-1.1721094847,-1.8064625263,-0.3921511769,-0.2310555279,1.5131512880,0.4917284548,-0.0432878397,1.6977798939,-1.3908723593,1.4940390587,1.8423306942,-0.1109176725,0.2542709708,0.2416193187,0.1212384477,0.2414720356,-0.4855381846,-0.6720389128,-1.0101215839,1.1480742693,-0.0514740795,0.2556264997,1.1470924616,0.9148079753,-1.1060256958,0.4811087251,1.2506557703,1.3353089094,1.7559952736,-0.1265173256,-0.8010993600,0.4566311240,0.3428300023,-1.7524900436,-0.9317060113,-0.8634976745,0.3061068654,-0.1965166926,1.0003229380,-0.1569621265,0.1014276296,0.8939095736,0.9408544898,0.2834113538,-1.3292179108,0.9167075753,0.2990117371,-0.6239182353,0.9428245425,0.2198539376,-0.4146504402,0.7951261997,0.2012115568,0.6641840935],[1.9132390022,0.0612025335,0.3415098488,-1.0602945089,0.8366987109,-1.6226406097,1.1113557816,-1.3292145729,-0.3143319786,0.3570109010,-0.1764928848,-0.1717350632,1.6343352795,2.5383000374,-1.0527589321,-0.7758113742,-0.3192665279,-1.0465342999,1.9890724421,0.3994129896,-0.5866478086,-0.8957120180,-1.6844896078,-1.3510472775,0.3692949712,-0.2429165691,-2.3788287640,0.5294952989,-0.9295908213,-2.0615482330,-0.6898310184,0.0278328750,1.6711131334,-1.6555784941,1.4271929264,0.0273575801,0.2971141934,-1.1789822578,0.9930096865,1.0365034342,-0.1519527435,0.4682734311,-2.3577785492,1.3565287590,0.9537298679,-0.2706686854,0.2951039374,0.3883643150,-0.0566962734,-0.3984859884,0.2041887492,-0.9976084828,0.7897940278,1.4099681377,-0.3404171765,0.3241531551,-0.4855938256,-1.2421112061,1.5360037088,-0.9463829994,-0.3573916852,-0.0232634433,-1.2513146400,-2.2355549335,-0.7908390164,-0.4839990735,-1.6056691408,0.0734652206,-0.2260978222,-0.5341376662,1.0534939766,1.5730623007,1.0264832973,-0.9042997956,0.7410283089,1.2227886915,-0.9809672832,0.8328786492,2.7288522720,-1.6969711781,1.2512172461,-0.3696545064,-0.2950987518,0.0518392660,0.8288123608,1.2777222395,-1.2569802999,-0.3112655282,-0.5696928501,-0.2377070040,1.1983821392,-0.2934773564,0.8378570080,1.7270666361,-0.8205666542,0.3335210681,-0.8726528287,0.6561540365,1.0646369457,-1.4809577465,-0.3941823542,-1.0625725985,-1.1441496611,1.1643037796,-1.7063095570,0.1826176941,-0.3895122707,-0.5045616031,0.5522103906,0.0773333758,-0.0574592091,-1.5021427870,-1.7225257158,1.7434196472,-1.8855978251,-0.1019827873,0.4856334627,-1.9518294334,-0.1191142574,1.0445772409,-1.1645052433,1.6964392662,0.5589596629,0.6847245097,0.6614169478,0.4820757210,0.3499038517,0.6179351211,-1.9481719732,0.2576283216,-2.2164435387,0.2630864084,0.0669203550,-1.1707545519,0.6772699356,0.1684508771,-1.2875515223,1.0062065125,-1.1552302837,-0.8029338717,-2.2737193108,-0.7485864162,0.6612769365,-1.1619497538,2.1173989773,-0.3294250369,0.3562559187,0.2107373476,0.8565757871,0.6712851524,0.5338254571,-0.8010561466,-1.6697434187,0.3606756032,1.2867676020,-0.2094346732,0.5576734543,1.0525827408,0.9210794568,-0.2470230609,-0.0034603099,1.6976779699,0.6332430840,0.0199560001,0.0755996332,-1.0569541454,-0.6732931733,-0.4159931839,-1.5330858231,0.5702850819,-0.1691996306,-1.8038196564,-0.9590678215,-1.5392922163,0.3994826972,-1.0821446180,0.3027870357,1.0220326185,-0.3363763094,0.0301486459,1.5134326220,0.5621951222,-1.2312422991,0.6054644585,1.1600813866,1.3731253147,0.5823098421,0.1793181896,-0.9241508842,-1.3068791628,1.0078873634,-0.0847553387,0.6938104630,0.1029771790,0.1699392349,-0.5331149697,-0.5968712568,0.8790861964,-0.2934698761,1.2495779991,0.1272739172,-0.7125212550,-0.4451971948,0.4075957537,-1.0298527479,-0.2791715860,0.3461883962,1.6816877127,-1.6031706333,0.0056235124,1.0078634024,1.3490788937,-1.5436534882,0.1696927845,-0.5270277858,1.6223733425,-0.5162442327,0.3828381300,0.4506391287,-0.9998950362,-0.6521401405,-0.3246061206,0.2585060298,0.9276508689],[-2.0880565643,0.8682402372,0.0620451719,0.2503739893,-1.0289133787,0.1270450950,-0.8162886500,1.4642271996,-0.3115217388,3.0074081421,-1.2791824341,-0.2708213627,1.7376896143,-0.5052874088,0.6811109781,-0.7712689042,-1.5408807993,1.1143568754,-0.4985656142,0.7032669187,-1.3779296875,0.0579764359,-0.3669286370,0.5524052382,-1.1774955988,-0.9593952298,-2.5553278923,0.3986499012,-1.8606998920,0.5338377357,0.6182241440,-0.3920603395,-0.9350994825,-0.6044998169,-0.3034831882,-0.4526685476,-0.5963338017,-0.7564249039,-0.2662839592,-1.2210955620,-0.6124317050,-0.5627401471,-0.4255528152,2.0560801029,-0.8246173859,-0.9876614213,-0.6199842691,-0.4691637754,-0.1785079837,0.5794753432,-0.6497477293,-0.3748328388,1.7727497816,-0.4439648986,-0.2879000008,1.4938969612,-1.4431997538,-0.1966993660,-1.5506589413,-0.0824496299,-0.3735535145,1.8474872112,0.4143075943,-1.3707956076,0.4110006988,0.1122452915,-0.8783571124,0.1543612927,0.0001337494,1.3457558155,0.1956868619,0.0120426994,0.4659405947,1.0559220314,0.4118642807,-0.2827820480,-1.0537084341,-1.7089247704,-1.2605371475,-0.6627028584,0.0284964610,-1.5919424295,-0.8879739046,1.3372726440,-0.3084809184,0.2604015470,1.5567058325,-0.9774645567,0.9206609130,-0.3910360932,-2.0741565228,0.6043616533,0.8186147213,-0.2919317484,-1.8189321756,-1.3367571831,0.0049841022,1.1277798414,-0.1117683128,-0.7106814384,-0.0921612084,0.4363644421,0.5508530736,0.2112184912,-0.4217892885,-0.3625360429,-0.5210886002,-0.7911268473,0.6036680937,-0.9205818176,0.9471231103,1.3281573057,0.5098133087,-0.2765147686,-0.0400175266,0.9152051806,-0.0791764036,-0.4207233191,-1.4149441719,0.1290478557,-0.0053360481,-1.1818076372,-1.2097790241,-0.6358364820,1.0997394323,-0.3663605452,-1.3995124102,0.5268882513,-0.0040545431,0.8065931797,0.5818359852,-0.7355961204,0.2951637805,-1.8553963900,0.7826838493,0.9054294229,1.3948525190,0.4250205755,0.3838944435,-0.6590161324,-0.5383474231,1.9159451723,-0.3733713031,2.2129595280,1.8961585760,1.6835546494,-0.5816136003,-2.1392905712,-0.9826152921,-1.5338691473,-0.4559504688,-0.1861871928,-0.4478240609,-0.9198418856,0.6889845133,-0.7898256183,-1.5680773258,0.6147295237,-0.5621627569,-0.3072946072,1.9393631220,0.9291343093,-0.6737373471,1.3822253942,0.2642171383,0.2372154593,0.3773427904,-0.4426511824,-1.9745779037,0.8025727272,-1.0189781189,-0.6997631192,0.2891789675,-1.1383354664,-2.0471360683,-0.1752144843,-0.0870466754,0.2797400653,0.2387488484,-0.1481952071,-0.2302711010,0.8003568649,-0.6089862585,0.0534715429,-0.5716961026,-0.9841057658,1.2828774452,-0.1688161492,1.2798538208,-1.0940895081,-0.2482675761,-0.7160670161,-0.1450303942,0.3953606784,0.6148098707,-1.0022808313,0.3334307969,-0.9138579369,-0.8279646039,-0.2285257876,1.1132116318,0.6443887353,-0.1507997662,0.8884394169,-1.0409892797,-1.2017220259,0.4876228273,0.6594662666,1.0187479258,-0.8132072091,0.4418626726,0.6325352788,1.2463976145,-0.1135608032,-0.9164980054,0.7859613299,0.4463684261,0.6968076229,0.6227259040,-0.6469994187,0.9220806956,0.7630829811,1.0957705975,-0.4319012761],[-0.4152270555,-0.2044944763,0.9506226182,0.5847282410,0.2539539337,-1.7795450687,-0.6887845397,0.6866399646,-0.0558012277,1.2429003716,-0.5696885586,-0.1877678037,-1.1906847954,-0.4526516199,1.0057507753,0.7095575929,-1.6089472771,0.6746912599,-0.0422934592,-0.1314605325,1.1723577976,1.3062764406,0.0529669598,-0.3620713353,-1.5355778933,1.1157461405,0.1266220212,-0.3168752789,-0.2654770017,-1.4859975576,-1.1428658962,-1.8808702230,0.7916238904,-1.0978664160,-0.3868590593,-0.0748722702,-0.5582896471,-0.3408991992,0.0357785411,-2.8540141582,0.1198951602,-0.3569850326,0.6131075025,-1.6270140409,-0.2847393453,0.0791950300,0.7125725150,0.9216774702,-0.5362963676,1.0582467318,1.0025253296,-0.4564027488,-0.9334833026,1.2776427269,1.3159724474,0.9056265354,-0.0394703001,0.2864100337,-1.8664575815,0.1179925725,0.2843076587,0.6929876804,-0.8222893476,-0.7008691430,-1.1756200790,0.0781022459,-0.9581169486,1.2042080164,1.2288545370,-1.2023644447,-2.1400723457,0.8589876294,0.9031086564,-0.6311286092,0.2257191837,-0.2904500663,2.5082805157,-0.7749772072,0.4404409826,1.5937997103,-0.6869628429,-0.8768021464,1.1542279720,0.3378003240,0.7085487843,-0.0843881667,1.3874114752,-0.8350510597,-1.8387274742,0.0471214205,-0.6551354527,-0.8090748191,0.6171238422,-0.1551373601,-1.3722908497,-0.7655444741,1.6287691593,1.5615321398,0.3192647994,-0.9870176911,-1.4553993940,-0.6677874923,-0.0301759038,1.0221860409,0.2797620595,1.9955043793,0.2484874576,0.0276262611,0.2522889674,-0.8351356983,-1.0892126560,-0.1633870602,-0.4688637555,0.2814452946,0.6965830326,-0.8071414232,0.4933938980,1.1570245028,0.5350809693,-1.7146855593,0.3662984073,0.6917946935,0.4543409348,1.3346126080,0.8517884612,-0.8048650026,1.4301743507,0.5187414289,2.5650734901,0.5571698546,-1.4397175312,-0.0305997543,-0.5573413372,-0.3947639763,-1.7622674704,-0.5569675565,-0.9407226443,-0.4493946731,-1.6613689661,-0.8826978207,0.3443049788,-0.1960539669,1.6036450863,0.2909748852,-0.1233137473,-0.2173997611,0.3150077462,1.0498563051,-0.6585019231,0.0957669914,-0.2687816024,-0.6586936712,-0.3349936903,0.9559844136,0.2193197906,1.3311969042,1.7547625303,-0.4384146035,-0.5910891891,-0.0369080007,-1.7037577629,1.1412252188,-0.1880388260,-0.6701797843,-0.2139253467,-0.4461108446,0.4609785080,-0.6667152643,-1.0494136810,-0.9170075059,0.4246536493,0.4500367045,-0.4323231876,-0.7817713022,-0.8252133131,0.9703499675,0.5069776773,-0.5588415265,-0.5356715918,-0.2954487205,-0.6099143028,0.0746136606,1.6815915108,-0.3075112998,-0.8263124824,-0.7620713711,-0.8745070696,-1.0211056471,-1.0041357279,-0.7308058143,-0.2631337643,0.5446914434,0.6488231421,-0.5848956704,-0.2323754430,0.6658927202,-2.3170225620,1.2412644625,0.7278534770,-0.1591682732,-0.0863246992,0.0435153991,1.1766989231,-1.3432213068,0.7669236064,-0.8524904251,-3.0434315205,-1.0095849037,-0.0935433209,0.4370863438,-0.2978182733,-0.8723636866,-1.6289815903,-0.4529634416,-0.2159748822,2.2257893085,0.5159680843,-0.0077975448,0.6277788877,1.8291573524,0.7833284140,-0.6949352622,-2.3123188019,0.8569595814],[-0.3096266389,-1.2400895357,-0.3410450518,0.5242138505,1.9794309139,-0.1311748475,-0.6779469252,-1.0015225410,-0.7074307203,-0.6888180971,1.0624884367,0.6066510081,0.8800684214,-1.3701462746,1.2867304087,0.5365315676,1.2999631166,0.6271314621,0.3918847144,1.9875098467,-1.5191190243,0.4580776691,-0.0840576738,-1.1077204943,-0.3676539361,-0.5172212720,0.4895897508,0.6502180099,1.4189645052,0.4343042374,-0.3245664239,-0.3949351311,0.3454012871,-0.0033489121,1.3653379679,0.7686623335,2.2122874260,-0.2360703945,0.5532864928,-0.1661921591,1.6751831770,0.0132257845,1.4176884890,1.7192947865,1.6324951649,0.3114824593,0.7976205945,0.4467043281,1.4926853180,-0.2878831625,-1.6661862135,-0.0044908966,-0.2455088496,-3.2745137215,-0.2838715017,-0.1518843770,-0.8731604815,0.1849374026,1.0501680374,0.2308520377,-0.1487602592,1.1539299488,1.0388782024,-0.7927692533,1.0399427414,-0.3865675628,-0.3344063163,-0.7397027016,1.5190846920,0.0405279025,0.9089628458,0.3263829648,-2.2702953815,0.2929112017,1.2355471849,0.7129567266,0.5659303069,1.5859659910,0.3280636072,-1.5915364027,1.1417033672,-0.4248448312,0.7930880785,0.0883681923,1.8440197706,0.1610373408,0.5732743144,0.0075075594,0.4905359447,0.6674293280,-0.0319283381,1.1117783785,-0.8937884569,-0.5062664151,-1.0149229765,-2.0016167164,-1.0205111504,2.2229752541,0.0113997580,0.8003988266,0.2961736619,-1.6971895695,0.8078061342,0.2169072777,0.0451913550,0.5088980198,0.2208980471,0.6954385042,0.8863599300,-0.3556869626,0.5780002475,1.1718003750,1.1406605244,1.0717451572,0.4423497319,-0.5674478412,0.2898722291,-0.9614549875,2.5144340992,2.3160614967,0.8769077659,-1.0370091200,-2.4022161961,0.6049721241,-0.4953610301,-2.3281629086,-1.4473972321,-0.9300103784,-0.9152269363,0.0122897141,-1.7381287813,-0.9521546960,1.2463902235,1.2603173256,1.0538812876,-0.4877656996,1.4871590137,0.7009145617,0.6607995033,-0.5052556396,-0.3830589950,0.2948637903,0.6955531240,1.4680861235,-0.8497928381,0.3967521787,0.3792177737,-1.8099541664,0.8800607324,-0.2358582616,-0.0895350128,0.4097339213,0.1439082474,-0.8039250970,-0.5788547993,0.6057286263,-0.0667697638,0.5830227733,0.9660890102,-0.2354748845,0.4906316698,-0.2285811156,0.6608774662,1.3144094944,-2.1506252289,1.1890547276,-0.0107625481,0.5143793821,0.6493283510,-1.3720718622,1.7808263302,-0.5661188364,-0.1417411417,0.3248441517,0.5483885407,-0.3008512259,-1.2304807901,-0.3662090898,0.1400451660,-0.0959511250,-2.2191183567,1.1093850136,-0.0595243648,0.9957280159,-1.1562790871,0.7081853747,1.2150986195,-0.1032100692,0.6972517371,0.7193431854,-1.0294797421,-0.1768095046,0.9343592525,0.2372649163,0.7474091649,-1.5894510746,0.7022696137,0.8446638584,0.7430752516,-0.6214725375,-0.4367303252,-0.4456193745,0.8655875325,-0.5149858594,0.3280645013,-0.1024664417,0.7322508693,-0.6337894201,-0.4929722548,2.4898169041,-0.8463423848,-0.5593562722,-0.5450578332,0.8441916108,1.4418475628,-1.0300234556,-0.6466807723,0.2288228869,0.0954682603,-0.5915262103,-3.0876836777,-2.0283315182,0.5751449466,-0.6132372022],[-1.1095269918,-0.0689608827,-0.0798729211,-1.4768925905,2.1811816692,-0.2500698566,-0.5732578039,1.0652742386,-0.8546937108,-1.1292870045,1.2695087194,-1.2510520220,-0.8316658735,-1.6696878672,0.4545242190,0.1797860861,0.0970116705,1.2030805349,-0.7623143792,0.8427641392,0.2079904526,0.7502828240,0.2300081998,2.2469732761,0.9968439341,0.6118012071,0.4577815235,1.2006202936,0.4174729586,-0.2497812212,-0.2987058163,1.5730762482,-0.7681244016,-0.4798147976,-1.6172803640,-1.7546819448,0.2139769793,0.3619492650,0.0847462118,-1.7089012861,-0.1561836153,0.3462845981,0.6182725430,-1.8326427937,1.2170871496,-1.0436891317,-1.3648991585,-2.2823894024,-0.5620194077,-0.5828886628,-1.7087174654,0.6809772849,-1.0947186947,-0.2100659460,-1.0447046757,0.5272279978,-1.5686168671,-0.1761078984,1.0267621279,-0.9264883399,-0.5448895097,-1.1664751768,0.9067573547,-0.1943168789,-0.9029732347,0.3409390152,-0.3423842192,-0.4933809340,-0.1203116477,0.6469014883,1.4632275105,-2.2619640827,-0.4403368235,-0.2874864936,0.3995994627,0.0293659437,0.4649489224,0.5351887345,0.5258300900,0.2087713480,1.2079454660,-0.0458886884,0.5520437956,1.7916995287,0.3296727836,-0.2645019591,0.1053925529,-0.3000160456,0.5894111991,-1.7004368305,-1.1702919006,0.7144211531,-0.3652415574,-0.4975321591,-1.2789793015,-1.5159935951,0.9037019014,0.9386519790,-0.2034149766,0.0920588151,-1.9900647402,-1.4680039883,-0.8426797390,0.0067214174,0.9794570804,0.5074473619,-0.4286257327,-0.6218476295,1.3840569258,2.5538566113,0.6628007293,-0.2189400643,0.4221946299,-0.7699494958,-1.4606316090,1.2698231936,-1.3451348543,-0.1210730821,-0.9649879336,-1.0809884071,0.5695738792,1.4865843058,-1.7563409805,-1.7095904350,-0.8119915128,-0.5803133249,0.2327409685,-0.6860216260,1.4256445169,0.5897830725,-1.0782427788,-1.7356255054,0.8439821005,-1.0922048092,0.3640676439,0.1694461256,-0.2845337391,-0.0015914980,-0.6929503083,-0.3809933960,0.7967649698,0.3464201391,-0.8488762379,0.0735314563,0.3796829581,-0.3305215538,0.1394914091,0.2664026618,0.5161311626,-1.1287683249,-0.2626350820,0.5476616025,0.3476051390,-0.6108629704,-1.8545712233,-0.7726867795,-0.1765599549,-0.6290928125,-0.2056999356,0.6018232107,-1.4811753035,0.9063700438,0.5083891749,-1.2830897570,-0.4046699703,0.7701845765,0.0314547308,0.4233820736,0.9221630096,1.7657654285,2.3693754673,-0.1023481190,0.2427469939,0.3453752100,1.0699369907,0.3138153553,2.0743460655,-1.7169541121,-0.4115565419,-0.1496352702,1.1596267223,-0.7840048075,-0.7558175325,0.8825569153,0.6057924628,-0.0026302291,0.6777153611,0.7162336111,0.8861199617,0.9713836312,1.0570205450,0.1008922532,0.2519199550,-1.7796982527,1.0196323395,0.1301587075,-0.3126945794,-0.5350653529,0.2415903062,-1.2447651625,0.8951871991,0.1416310817,-0.5771242976,0.3480791152,-0.5791205168,0.5143607259,0.6251235008,0.4787263274,-0.5682345629,0.5818405747,0.5476638079,-0.2344136238,0.4926259518,0.0020974108,0.4648458064,-1.2469049692,0.9685260057,0.2488428503,0.8673420548,1.3650695086,-2.1453146935,0.0148590002,2.2118170261,-0.4790977240],[1.8335345984,-0.4013042450,1.8113731146,0.8131791949,-0.4838180840,-0.5870887637,-0.3348998725,-1.5233494043,-1.4847657681,0.4049981236,-0.3059670329,0.1298778802,0.7187888026,-2.7429368496,0.4560085237,0.3684889376,0.2669047713,0.5020672083,-0.0811798126,-0.0078832060,-0.2715966403,0.2410754412,0.0067627095,-0.8049398661,-0.3467500806,-1.1980543137,0.3195302486,-1.2405413389,-1.0926430225,1.2750275135,-0.2405105084,-0.1525734663,0.5744650960,-0.3584517539,1.9834860563,-0.0039220667,-0.2727905512,0.8813328147,-0.8601807952,-0.0736739039,0.3197454810,-1.5756418705,0.5200622082,-0.4795474708,2.6535255909,-0.9987680316,1.2702453136,0.3841578662,-0.4805614948,1.2172632217,0.7606619596,-1.5414643288,0.1976353973,0.7160565257,0.4035159051,0.6984516382,0.1601044387,1.0915095806,0.3534328341,-0.1184051558,-0.4150804579,-1.8912463188,0.4045003653,-1.8975316286,-2.2630052567,-1.4463518858,0.3579699993,2.1300139427,1.2352734804,-0.2789600790,1.3526080847,0.3938015997,-1.3408533335,1.7260075808,-0.0077996096,1.5074975491,1.8468036652,0.6475793123,-0.3252647221,-0.8967847228,0.9662185311,0.1558922529,-0.1998231411,0.6230989695,-1.4783391953,0.6823682189,0.8574786782,-0.8982612491,-0.0487452298,2.1404359341,-0.7900241613,0.2257759124,-0.4074427187,-1.0557363033,-0.9133280516,0.3864759505,-0.2905539274,-0.7860685587,0.7287256122,-1.2111824751,1.0915997028,-0.5741703510,-0.8632349372,1.8330698013,-0.2431361377,-1.3883429766,0.5145610571,0.4060795903,-0.3183524907,-0.0028131553,0.1949463636,0.1193532422,-0.5316559672,0.8666998148,-1.0010826588,-0.5808494687,0.8329043984,2.0336697102,-1.1984018087,0.7143980265,0.1848467290,0.6511341333,0.5259750485,-0.3475879431,-0.9058694839,-0.5467703342,-1.4293168783,-1.7306935787,0.6718448997,0.5225819349,1.1162972450,-1.5372996330,-0.7658146024,-0.6932194233,-0.5045186281,0.7642450333,-0.5672447085,-0.3762857020,1.3930920362,-1.0889377594,-0.9572563767,-0.0917122737,0.6868191957,-0.4004735351,-1.3633692265,0.6427147388,-0.7196304798,-0.8512632251,0.5361620188,-1.1725454330,0.1028381735,0.5060002208,-1.3521600962,-0.8992948532,-0.2606385052,1.1963212490,0.4467527270,0.6490583420,0.6809411645,1.4820773602,-0.7598612905,0.7339666486,1.4499570131,-0.1634774804,-1.6448104382,0.0809721798,-1.1142492294,-0.2901205420,-0.1146303713,0.2195672244,0.4080210030,-1.0791637897,-1.8717880249,1.6003428698,-0.1694111526,-0.3783018291,-1.6030508280,0.2283682078,1.0475113392,0.7307556272,-0.4210613668,-0.3517955542,-0.4050840735,-0.5490471125,-0.3479705453,1.9079108238,0.3872929811,-1.1613664627,-0.1516494900,0.3707110584,1.4472540617,-0.2746278346,0.4029833674,-1.2051960230,0.2465004772,-0.7065382600,0.0371126421,0.8646813035,-0.6913815737,-0.3071915507,-0.1538460851,-1.7917917967,-1.2598940134,-1.8016839027,0.9368255734,-1.3379613161,0.5263180733,-1.5782839060,-0.4033615589,1.0468375683,0.8125714064,-0.4790661335,1.0822108984,0.5306308270,3.2135152817,0.6620602608,-1.6667072773,0.0143323876,2.9585022926,1.2215119600,0.4183119833,-1.1557128429,-1.1458501816,1.0365495682],[-0.2553441823,-0.4749603271,-0.4489164948,-0.1246030629,0.9618656039,0.0579839200,-0.3051787019,0.1202649400,0.0543751307,-0.6086128950,0.0081361765,0.4306674302,-0.0972862765,0.3353385031,-0.2733195722,-0.1604502201,0.9772319198,-0.1361977905,-0.5129996538,0.6338104606,0.1387738585,0.1811870933,-0.4650717676,-0.2434235364,1.3702030182,-1.0426175594,-1.5735248327,-0.5793191195,-0.3952281177,0.6459346414,-1.9756667614,0.3489671350,2.3008513451,0.0771680027,0.6087843776,-1.4710805416,0.3890483677,-0.5295361280,0.9832375646,0.1260338128,0.8573789001,0.2447784096,0.3204292953,-2.5175051689,-0.7122991681,-0.8745144606,0.0141174290,0.9317763448,-0.9803977013,-1.5740404129,-0.3517892361,-0.8231508136,0.0262164883,0.6211560369,-2.0268793106,1.0612952709,-0.3552500010,0.9536997080,-1.2097529173,0.4185555279,0.2573874295,0.6497887969,0.8740681410,0.4048483372,-0.4238315225,-1.9540976286,0.4655889571,-0.0313888192,-0.2161873132,-0.2317292094,-0.0759339109,0.0675848275,0.0878292397,-0.9489600062,-0.1432806849,1.1445298195,-2.7369377613,-0.5840117931,-0.5899749398,1.2879940271,1.3466374874,1.0962083340,0.2304671705,-1.7148902416,1.3411163092,0.1041158214,-0.0788779110,-2.2844712734,-1.0280727148,-1.2645610571,1.1588771343,-0.4571908414,1.2539838552,-0.2162689269,-1.7553088665,0.2741356492,0.7430927157,-1.1741378307,0.3358246982,0.0238060430,0.0031688462,-1.3877177238,0.5490756631,1.1530783176,1.6128815413,-0.3410646319,-1.2937667370,-1.6469665766,-0.7334685326,0.1037245989,0.8387489319,0.0148075577,0.5645905137,-1.3039965630,-0.5846779943,-1.5839500427,-0.2638729811,-0.0072477343,0.8843932748,-0.3676111102,-1.5938299894,-0.8872205019,-0.4249498248,-0.1557797194,-1.3146603107,-0.1697295159,-0.2765368521,0.4457674026,0.9021023512,0.1397904456,1.7429642677,-0.4599343836,-0.6597484946,0.4328436255,1.4983532429,1.2432944775,-0.4208336174,0.4489599764,0.1689072102,-0.4926060140,-0.0422246605,-2.3662722111,-1.8588395119,0.3812257349,-0.7578253746,1.2569688559,-0.3351842761,-0.7692585588,0.4607789814,0.8273998499,-1.1147665977,0.8162701130,-0.2671040297,1.4073350430,-0.7888681293,0.1561743319,-1.2401427031,-1.2775328159,-0.7646373510,-0.6475517750,2.0057315826,1.0876049995,-1.1954290867,-0.1807238609,-0.8135599494,-1.2610627413,-0.9346448779,0.3905242085,-1.4709957838,0.6816416979,-0.4965876043,-0.1572820246,-0.4188430607,1.1873232126,0.9308753610,-1.9551130533,1.0496579409,-1.0345253944,0.8236737847,0.5495346189,0.5497707725,2.3413565159,1.6484090090,-0.7264957428,0.3208186626,0.2824824750,-0.6003243327,0.4457696080,1.7169154882,0.0159275644,-1.6579325199,-1.0831270218,-0.5027672648,-0.6477096677,1.4063690901,0.3528015018,1.1226716042,-1.5352687836,0.3197790682,-0.2060278803,1.3496638536,0.6235291958,0.0421331450,-0.3542109728,-0.6579025388,-0.6250056624,0.6840335131,1.3491804600,-0.6798034310,1.0311497450,0.3708248138,2.3034310341,-0.8823706508,0.2779397368,0.2376018465,-1.0999842882,-2.3625836372,2.5004153252,-0.0031369352,-0.5761529207,-0.9552867413,-1.1486686468,0.2252877206,1.3544378281],[1.1923971176,-1.0452986956,0.4250714779,-0.6534120440,-0.9170339704,-1.8241276741,0.6940741539,1.0891760588,1.1354733706,0.8429929614,-0.5788779855,-0.3875228167,-0.5509646535,-1.9952298403,0.8218613863,0.6798110604,1.1460310221,-1.0449521542,-0.5703985691,0.5750790834,-0.7848528624,-0.5177422166,0.2719036937,0.5763064623,1.7560209036,0.7867377400,-0.8005876541,-1.3305873871,-1.3257182837,0.7724911571,-1.8970799446,0.9184693694,-0.5014103055,0.5121781826,-0.2338661253,-1.9732248783,-1.2685248852,0.5421708822,0.3566212654,-2.7719295025,0.0120019102,0.2266089767,1.5233207941,-0.5277122855,0.8992790580,0.9162186384,1.3245271444,-0.3380609453,-0.2191164792,0.1802497059,0.7791525126,0.3603124619,0.6356837749,-0.0949871168,-0.1510977745,1.0524718761,-0.8924908042,0.1134970784,0.7762354612,-0.1041747034,-1.2717193365,0.1911706775,0.2469402403,-0.4907810986,-0.0449020043,-1.1582369804,-1.8383409977,-1.4759110212,1.8879642487,1.2204946280,0.7254795432,-2.1488111019,-1.9087783098,0.8363744020,0.1254971474,1.1508957148,0.1561222225,-0.1453067958,-1.0493863821,-0.3168115318,0.5137270093,-0.6062394381,-1.0988081694,1.2591933012,-0.1684797853,-1.2144109011,1.3161044121,-0.2794598639,0.6305717230,-1.1718168259,-0.5043300986,1.3296680450,0.6659186482,0.0582517385,0.0609587096,-1.5843622684,0.9506537914,1.3065859079,-2.2422685623,0.5747953057,-0.1019831151,0.1581490189,-1.2338908911,-0.7650310993,-0.4184915423,-0.9221194386,0.6965978742,-0.8460552692,0.1583629102,0.8735681772,-0.3400534987,-0.1723876595,2.3087937832,0.6242477298,-0.3329307139,0.1324429065,-0.5053774714,-0.9988173246,-0.9100036621,0.0229380503,-1.2823932171,-0.3242506385,-0.6225237250,0.7587878704,-1.6966271400,-0.3292137086,-0.9888989329,1.8447217941,0.6552241445,0.5138090253,-0.7122284174,-0.6896860600,0.2713268697,0.2952925265,0.1329986453,-0.2784538269,-1.2005258799,-0.4423667192,-1.3582398891,2.2271566391,1.7319980860,0.1650353372,0.6235536337,0.9041349292,-0.7492755055,-0.6408665776,0.5311431289,1.1350789070,-0.0610747412,1.5564525127,-1.1107958555,-0.9086764455,0.4045865536,-1.2979991436,-0.3478908539,-0.6960718632,-1.7867015600,-0.3869774938,0.3975443840,0.3818458319,-0.9753320217,-0.2342271954,-1.3683166504,-0.2547301948,-2.3559172153,0.3542802930,0.6389033198,-0.2198933065,-1.5690653324,-0.4407686889,0.8176122904,-0.0587791465,1.2063684464,1.1797443628,-0.3437459171,-0.4739488363,-1.5506465435,-0.3452513516,0.8360907435,-0.8022053242,1.5924566984,-1.5707868338,-1.0464003086,-1.2346476316,-1.5263248682,0.8003885746,1.1033037901,-0.1054809988,1.4715224504,-1.7682429552,-1.1850502491,-0.8126041293,0.8842704296,0.9425088763,0.0330735892,-0.2409315258,1.5489830971,0.6468773484,-0.1781851649,-2.7890059948,-0.0014940457,0.4603638053,0.1230962947,1.5464887619,0.3794255853,-0.7880982161,0.7934423089,-1.5169560909,0.0380557850,-1.4559270144,0.5339881182,0.3679506779,-1.1586297750,0.0873472542,-0.5471687913,0.0661044717,2.1989641190,0.2341248393,1.7468538284,3.7643828392,-0.9423438311,-0.0853690207,1.9483058453,-0.2876728475],[-0.1539317667,1.6529641151,1.3316450119,0.9574147463,-1.1216984987,-1.0181498528,-0.7338421345,-1.3882849216,1.0699387789,-0.6837269664,-1.2928775549,-0.1253340095,0.4996267855,0.9240604639,0.5986737013,1.7402137518,0.3486900628,-0.5245723724,-1.1849862337,0.2726819515,-1.6281638145,0.2654599845,-0.4116245806,0.4574624300,-0.1537874192,-1.9013284445,1.4959928989,-0.0532272644,-1.2966313362,-1.3777812719,-0.1085461006,-1.1366568804,2.4738366604,-0.6625595093,-0.3700547814,-0.6848813891,-0.9847663045,-0.3399543166,-0.7798151970,1.1458569765,1.2416856289,-1.1501663923,-0.6315066814,1.1676069498,-1.4100657701,-0.5687700510,-0.5040619969,1.4950553179,-0.7586460710,0.3278019726,1.1096271276,-0.3833582699,0.2235788554,-0.5815556645,0.0966813415,-0.6273540854,0.6732571125,0.0662461072,-0.1287237704,-0.8700714111,0.8890676498,-0.3600372672,1.6472837925,-1.6016446352,0.8628495336,0.4772661924,0.0748748258,-0.3812872469,-0.7134984136,0.6940374374,-1.6486783028,1.4736716747,-0.0254137758,0.0967500806,-0.1661897451,1.1823667288,-0.6112408042,-1.5404152870,1.2952648401,0.7924426794,-0.4812881052,0.8054927588,-0.2136539519,-1.7206912041,-0.3016053438,0.2895011902,2.0912322998,-1.9579486847,-0.8064804673,-0.3919655383,1.6497608423,1.0967392921,-0.0155962491,-0.0089084962,0.5606082082,-0.7090340853,0.7310581207,0.0730704814,0.6230039597,0.4864089191,0.4549812078,-0.2649352551,-1.1511001587,-0.9840397239,-0.2999227345,2.2628390789,1.1271271706,0.3714485168,0.2622233927,1.4365916252,2.0530672073,-0.0107169645,0.6136042476,0.0226049926,-0.0324297585,-0.7721080780,-0.5181171894,-0.3055936694,1.3205901384,-1.5773636103,-0.3167185187,-0.1485596001,-1.1861242056,0.1441131532,0.7637791038,-1.9861276150,-0.2936843038,0.8477237821,0.0517439954,0.0406466722,-0.9705188870,-0.7261933684,-0.0312301386,0.3390150666,-1.0390940905,0.2602939308,2.0953760147,0.0473578013,1.2825541496,-0.3004450202,0.5410876870,-0.1447229385,0.4664852023,0.3632920980,0.5882064700,-1.5549762249,0.0268624183,-1.0717278719,-0.1177723333,-0.4627850652,0.3465367556,2.2943594456,1.7421190739,0.0722834393,0.5992326736,-0.7836269140,0.5283228755,0.0322488360,0.5551918149,0.2175752223,-0.4546332657,1.4639328718,-0.6801419854,-0.0971610919,0.6066857576,1.3741084337,0.2518433928,0.6852304339,0.2889463305,-0.1472174823,0.2218355238,-0.0487602502,-1.0502138138,-2.2987830639,1.5475156307,0.3123504221,1.0007822514,-1.7533344030,0.3010531962,-0.1631305069,0.4981911182,-1.0523788929,1.5803315639,-0.5132887363,0.2444366068,-0.7612324953,-1.0226480961,-1.5267606974,1.2850830555,1.5239545107,-1.4156295061,-0.8883552551,1.0721236467,-0.7571452856,-1.2760622501,-0.0873958692,1.1551344395,-0.2928124964,0.1826197952,-0.1817886829,0.0830894485,-1.1084549427,2.3446435928,-0.1304149628,0.8465356231,-0.0870315060,-0.2174553275,0.1470153332,0.6346374154,-0.2693883181,1.3204683065,-0.3679193556,1.1488084793,-1.1435592175,-0.4943830669,-0.2291122079,0.2982085347,-0.9264073372,0.9086833000,0.8751946092,0.3901867867,1.0084059238,0.2700186968,0.4779162705],[0.2833504975,0.0971349180,-1.3063620329,-0.7672454119,0.1625791639,0.1736915410,-0.8138611317,-0.3182066679,-0.5231506228,-1.3166358471,-0.2597752213,-2.8546957970,0.2950502336,-2.2095010281,-1.1995346546,0.1285841912,-0.0192146059,-2.2865560055,0.8165867925,-1.3509417772,0.7002499104,0.3727229536,-0.4820721447,0.8639489412,0.8133208156,0.0887968391,-0.3125910759,3.0393307209,1.4080055952,0.2269381732,-0.0300363377,0.7396939397,0.0493804477,-0.4246927202,0.1304106563,0.9637285471,-2.9354915619,-0.3544369638,1.2836763859,-0.8559589386,1.8721460104,0.9731093049,0.5492524505,-1.0334913731,1.3251916170,1.3108726740,0.5195982456,-0.4179671109,0.6543501616,0.4693245590,0.8555210829,0.0848020539,-0.1515882611,-1.7304179668,-1.0883326530,1.0102947950,-0.2386721522,0.8745925426,1.3899519444,0.7876927257,0.6158488393,0.5215525627,1.4409991503,-0.0144396611,2.2355680466,0.4180262387,1.0790933371,-1.4335969687,0.7017084956,-0.0270159505,-0.0620286353,1.0418047905,1.2607593536,-0.8740250468,0.3189888597,0.2929864824,-0.8016648293,1.1566418409,-0.3863493800,-0.6131212711,0.4015072286,-0.3994033933,0.5563696027,-0.3870293200,-0.7484621406,1.9729208946,0.3187820911,-1.0970444679,-0.8086558580,-2.5966191292,0.6420378089,0.6515240073,-1.5672519207,-1.1887092590,0.1352421641,-0.1588215977,-1.1035181284,-0.3870321214,-0.1285940260,-0.0611574203,0.5199727416,-0.6207996607,-0.8858322501,0.2921783328,1.7415742874,1.1214604378,0.4722931385,-0.0375859216,0.9042145610,1.5266809464,0.9645444751,1.1944296360,0.3133407533,0.0609702654,-0.7147462368,-0.9949486852,0.0263745487,-0.9406294823,0.6160652637,2.3014707565,-1.1834588051,-0.8900075555,-0.2471520454,0.4672086537,1.9268618822,-0.6749277711,0.6205279827,-0.2686580122,-0.7099538445,-0.9639384151,0.7790831327,-0.5402551293,-0.0003102511,-0.8768327236,-1.0687489510,-0.4675288796,-0.2242846638,0.8382201195,-0.3525581658,-0.7518466711,0.4557478726,0.2760375738,0.6195541620,0.3318205774,0.8327250481,0.0537931845,-0.3553763926,-2.0381026268,1.6230796576,-1.5148580074,-0.1860478520,-2.0974383354,-0.2627943754,-2.4067049026,-0.4767214060,1.3863887787,-0.7196408510,-0.1204650104,0.7024926543,-0.8471753597,-0.4811016917,-0.2101222873,-1.5753133297,0.0518422872,-0.6725054979,-0.0771305710,0.2476974279,1.1442660093,0.8213924170,-2.0979964733,0.0083712144,-2.8003196716,0.5200743675,1.1690288782,1.6796238422,1.7359300852,-0.3625329137,0.8729770780,0.0395113789,0.4805897474,0.8971570730,1.0508792400,-1.1386889219,0.2386064231,-2.1639914513,0.1895284206,0.7108013630,-0.1431214809,1.1821506023,0.4952346385,0.0679816455,1.9600552320,-2.0621733665,-2.3717496395,-0.0206102040,-1.3767124414,1.6856360435,-0.5215978622,-0.1548906118,1.1633713245,0.1333243102,-0.9365465641,-1.1232398748,-0.1756432503,-1.5008443594,2.0137960911,0.5293657184,0.4739171267,-0.0677911416,2.7306401730,-0.8007552624,0.8647549748,0.1554780453,0.2040397525,-0.1762325615,0.3810936511,-0.2150106728,0.1929219216,1.3094671965,-1.0703794956,-0.6601840854,0.1119203046,-1.0493140221,0.6418649554],[1.0578992367,0.3493487239,0.0387381762,-0.7578765154,1.7252588272,-1.1950592995,-1.0095664263,0.8863735199,-0.9332340360,1.0328786373,-0.3667743504,-0.6750990748,-1.0239139795,0.6232453585,-0.4233544767,-0.6697765589,0.3974470198,0.1741041839,-0.2741079032,-0.0697648525,-0.2788359821,0.0967028365,0.4162475765,0.0730348527,1.6830052137,-1.7050038576,-0.9182111025,-0.0687336549,-0.4993008077,-2.6333050728,1.2867982388,-0.8157199621,1.8138675690,1.3882590532,-0.6029289961,0.0287784822,1.1772009134,-0.2755854428,-1.2099580765,-0.2019699514,1.2501586676,-1.0892593861,2.4252128601,0.8154894710,-1.0352143049,-0.2517317533,-1.3882302046,0.9432877302,0.7587483525,1.0898754597,1.0566821098,-0.8384352922,-0.3188551664,0.1994140446,-0.7795777917,2.1310243607,-1.7256475687,-0.3948467374,-0.7480053306,0.7033433914,0.5914207101,0.0639219061,-0.3472526371,-0.2467501760,0.1587737948,1.3566329479,-0.1416103244,1.3385957479,0.9539729953,0.0049228636,0.9412178993,0.6962132454,0.8853695393,0.0664721727,-2.7985961437,-0.9155045152,-0.0428128801,-0.6666034460,0.9059969783,-0.3675763011,-0.5666050315,-0.6845122576,1.0391764641,-1.1581093073,-0.0269874949,-1.3750796318,0.9732193351,1.2611368895,0.0356923081,0.2246537358,-3.0932531357,2.3984165192,2.2979116440,-1.7785514593,1.3218277693,0.1280211210,2.0275268555,-1.0245759487,0.8668836355,-0.8607371449,-0.7002310753,0.0155158956,-1.4728592634,1.7080414295,-0.7406947017,0.8791487813,-0.4585764706,0.3052116632,0.0468687899,0.5164999366,2.0355801582,0.6521666646,1.8466851711,1.7116943598,0.6319134235,1.3719582558,-1.6670839787,0.4300492704,-0.6846798062,0.9375258684,0.3836639822,-0.0728900358,0.0684805885,0.3087892830,0.6202475429,-0.7178317904,-0.2522747517,1.1459041834,2.0370550156,-0.2767766118,-1.3960233927,-0.9338543415,1.1877619028,-2.6348557472,1.9516296387,-0.1824237257,-2.0092451572,-0.9741954803,0.4679149389,0.9008089304,-0.5722278953,0.6044968963,0.6506953239,-1.6755926609,1.5076848269,-0.5969001055,-1.0222656727,1.9562164545,-1.2689777613,-1.7846150398,0.1661764532,-1.5170683861,-0.7610163093,-0.8992091417,-1.6415470839,1.3204326630,-0.2228368372,0.7749367356,0.4024796188,-1.3226431608,0.2422006428,-0.4162628353,-0.5555692911,0.0719668418,-0.1871663779,0.5951982737,-0.5845536590,-2.1090695858,-0.1772703975,-0.0548733100,0.4509508312,-0.9313350320,-1.7773886919,-0.3563153446,0.2669277191,0.2997245789,-0.4658081532,0.5417457819,-0.2356391251,0.4109375477,1.5283123255,-0.3351726830,-0.3406950831,1.2914118767,1.3758453131,0.8519734144,0.4109028578,-0.3435279429,0.8422434330,-0.0266405381,-1.5175741911,-0.1276775450,-1.5115371943,0.3101183772,-0.6066270471,1.0376282930,-0.0093769059,-0.0544994585,-0.5122949481,-1.2871826887,-1.0043041706,-0.8428524733,0.1637625247,0.1111728325,0.0281228572,1.3473829031,0.0723849237,-0.1692962050,-0.8452322483,0.5937932730,-1.5416975021,0.5983489752,-0.2016807795,-0.4967584610,0.1199091077,0.4734321535,-0.0088558868,1.2080721855,-0.3648279011,-0.6955690980,-0.4260819256,-0.7084145546,0.0491476320,-1.5102821589],[1.0569622517,0.5366889834,-0.0563135706,1.5393899679,1.7595765591,-0.3918105364,0.9197299480,0.3016016781,-0.1586421728,0.0058817188,0.4183733761,1.5566524267,-0.6086767316,-0.5249670744,-0.8945267797,1.9990261793,1.0346226692,0.9244108796,1.8088890314,0.4348047376,-0.7111964226,-1.3284702301,-1.1754543781,0.3776286244,0.4044982493,1.7677334547,-1.0616512299,2.2402262688,-0.1550719440,0.1358435005,-0.1329102516,-0.7924153805,-1.3541342020,-0.0541118197,0.7407044172,0.1474341452,1.3359956741,0.7409295440,-1.4758354425,0.0812779739,1.5620260239,0.1671449095,0.3341617882,1.3199590445,-1.4782437086,-0.5552400947,-0.2017293274,-0.4659951627,0.2432397157,-0.6974768043,-1.2883409262,-0.7465677857,-1.1168094873,0.4712827206,1.6376899481,0.6912307143,-1.3686044216,0.1189413965,-1.4354121685,-0.1401813477,-1.1164814234,0.3037294745,-0.3992435634,0.2656225264,0.3130260110,-0.8659200072,-0.4726854563,-0.0152176367,-0.0930923000,0.3841016293,0.9208854437,-1.3471815586,-0.2411450744,1.6320923567,-0.6548956037,-0.4167873561,-1.5803978443,-0.1135653034,-1.2573654652,-2.0530848503,0.9897403121,-0.4235695899,1.5027476549,-0.6127468348,0.2498405725,-0.1621945947,0.5481315255,-1.0397114754,1.0767316818,0.6571477652,0.8535944223,0.3218203485,1.0353807211,0.2963234782,-1.5258238316,1.6403651237,0.4474198818,0.5159975290,0.0112749394,-0.4295137823,-0.0053783916,0.8010345101,0.0232382063,-0.5535251498,-0.7191053629,-1.7811300755,-0.3628551364,0.5310444832,2.4867811203,1.9475063086,0.4156304598,0.6378083229,-1.8176249266,-0.4333888888,1.0649061203,-0.7765557766,-1.1090966463,0.0572323352,0.2367252856,-0.1627569199,-0.0342329107,1.1516398191,0.0576944612,2.5441424847,0.4243032932,0.6389544606,-0.7369576693,-0.4054327011,-0.1950183809,1.2195286751,-0.7707237601,0.3494344354,-0.8499855399,-0.4434051812,-0.8354090452,0.6398029327,0.0468047112,-0.0552285872,0.5982164145,1.0310800076,1.2464448214,1.5467491150,1.9671247005,-0.7465594411,-1.2460572720,-2.2472889423,0.7259353995,0.6860581040,-0.9589980245,0.7132995129,-0.4153469801,-0.3921484649,1.2021461725,-0.0087545915,0.3043210506,0.5953631401,0.1601187736,-0.7451269031,0.2441005409,0.3973066509,0.2535099089,-1.4309951067,-1.6279504299,-1.1323072910,-1.2013375759,1.3001255989,1.8920334578,-1.2963359356,-0.4052232206,-1.9754989147,0.7988344431,0.4170355797,-0.4807196558,-2.0655698776,-0.7232215405,-0.8220478296,-0.4829567671,0.0755685493,0.6312243342,0.5088500381,-2.4266726971,-0.6307898164,0.1054759845,-0.3693656921,1.7723187208,0.6556220055,-1.3453546762,-0.6720908880,1.1442575455,-1.4109114408,1.2649291754,-0.8008374572,0.0384092443,-0.2519807816,-0.2426828742,1.2038187981,0.4667410553,-1.0764840841,1.2552109957,1.3426625729,1.5059294701,-0.9154779315,-0.0498299412,2.0281126499,0.6835604310,-1.8105126619,0.4737175107,-0.8432004452,2.0750002861,0.2878215015,-1.1522181034,2.0514914989,-0.7687619328,-0.6637822986,0.6724092364,0.6839038134,0.0562897585,-0.3016141951,0.4957450032,1.0637838840,-0.6770959496,0.2365283668,-0.2895208299,-0.8617882729],[-0.9594106674,-0.0157391503,0.8399378061,0.8110836148,0.4611687660,-0.7466703057,-0.0806623921,-0.9150440097,-0.9413416386,0.3335480094,-0.3684301674,0.4436233044,1.0678399801,0.5630221963,1.3437930346,-1.7708089352,0.3103902936,0.6368618011,-0.4932948351,-0.2312989533,-0.3020412326,0.0923235640,1.3628299236,0.7628677487,0.7790932655,1.5329159498,0.2183266133,-1.2446256876,1.0062347651,-0.0709650144,-0.7959281206,1.2688463926,-0.8175793290,-1.0273636580,-0.6656317711,1.1612483263,-0.5840453506,1.4325695038,-0.8769946098,-0.5628446341,-0.2621915340,0.6548326015,0.0389738902,1.7369393110,2.3276939392,0.2912586033,-0.1409829259,0.8625109196,-1.4595911503,0.2316573858,0.9324889183,-0.6206426024,0.5359648466,-0.3008208573,0.0612839088,-1.7155638933,0.9352427721,0.1170468032,1.0926513672,-1.5903171301,1.6055085659,-0.1270006150,2.5916931629,0.3588993847,-0.4123798013,-1.4389734268,-0.4076504111,-0.3596978486,1.8169152737,0.3336016238,2.4945390224,-0.6098358631,-0.3644248843,0.8882802129,0.4999442995,0.5015850663,0.6149253845,1.1175642014,1.8079290390,0.2978225946,0.6117570400,1.7117923498,0.4619747698,-0.2899212837,0.7065873146,0.1721730530,1.0118405819,0.7275610566,-0.6780998707,-0.1265953183,0.9778655767,-1.3090860844,-0.4225935042,-0.1998132616,0.0992397964,0.9909891486,-0.8389635086,-0.2723429799,1.9301704168,0.9522368312,-0.8642147779,0.6867908239,1.2381422520,0.8961113095,1.7952609062,-0.2819908857,0.7153847218,0.0273373686,-1.3802032471,-2.0942184925,1.0688688755,0.2658931017,0.9404011965,-0.0069303596,0.5577315092,0.2194401324,0.5258272290,0.5876083970,-1.1635379791,-0.5140573978,-1.4443894625,-2.1708698273,-0.1201778799,-0.6402384043,0.2971675098,0.1304511130,0.1133562103,0.6318150163,0.1640680581,1.5028864145,-1.7749533653,-0.7292850018,0.1292218417,0.1633696705,0.5901848078,0.9318404198,0.0090533029,0.5410732627,0.6497133374,0.0382000655,0.2489194572,-2.0864832401,-0.4515062571,-1.2362923622,-0.2087910324,-1.2355883121,0.5569347739,1.3323907852,0.5438510180,-0.1150219962,0.0459278598,0.4252981246,-1.4181237221,-1.7148958445,0.6399292350,1.3488681316,-0.5249696970,-0.7036726475,0.3729901612,1.3503198624,-0.6562246680,-1.8335790634,-0.7283580303,0.4336759746,-0.0378222503,-0.2591528594,0.6934357285,0.2126318961,-0.2649644017,-0.5409034491,-0.1988546699,-0.0906299800,0.1306391060,-0.6612063646,0.3144090772,-2.1315474510,0.5624923706,0.6593207717,-0.6201909781,-0.5569365025,0.3504077792,-1.3898166418,-0.3267312050,-1.5478287935,-1.0569232702,0.4089502990,1.0574078560,0.3555086255,1.3506741524,0.7100051045,-2.4160699844,0.2878858447,0.1611437201,-0.5060824156,-0.5658532381,0.0795270875,-0.5061843991,0.2112802863,0.1915567666,-0.1671071202,-0.4966573715,-0.3014395535,1.4349435568,-0.0081969770,0.8687512875,1.7044713497,0.7172669172,0.1723873615,0.7683315873,0.8068082333,0.7374470830,-2.3381924629,1.4708920717,-0.6122639179,1.1907120943,1.3497942686,-1.3654749393,-0.3542251587,0.6783241034,0.7457907200,-0.1441514641,1.7073394060,0.1043201610,0.4953304827],[0.4898969233,-0.7908129692,0.3112324178,1.2854849100,-1.1699900627,0.0307697803,-0.0978933945,0.2470248342,1.5309244394,0.0981095359,1.4879208803,0.6778614521,0.1257022619,-0.6889806986,-1.7376300097,1.6067085266,0.2240911871,0.3167164326,-1.0415329933,0.6432635784,-0.1780188829,0.2714662552,0.5061727762,-0.0400048196,0.6168119907,1.1573162079,-0.6214919686,-0.0559455715,0.2033772618,-0.0524638593,0.4651843607,0.7974411249,0.9521196485,-0.8156989813,-0.7175847292,0.0430429354,-1.1026031971,0.7501711249,-0.0570429079,0.2406162471,-0.1664299071,-0.4878765345,-0.8518081903,1.0820053816,0.2024823874,-0.6305008531,0.5867401361,-0.1111482680,0.9543295503,-1.5515456200,-1.3526494503,-1.2814865112,-1.2083516121,0.6300700307,1.2377527952,0.1157994419,-0.1324049979,0.8532496691,-0.2272783518,-1.9050981998,-1.4512780905,0.5527977347,1.9561060667,-0.5117322803,0.2994587719,-0.8818569779,0.7253333926,-0.2888867855,-0.0488190539,0.6111413240,0.8727059960,1.0785720348,0.6621670127,1.1721650362,-0.5383167863,0.7041527033,0.8776189685,-0.1341997683,0.5267609954,-2.2195818424,-0.0641778633,-1.4765670300,-1.6097735167,-0.2215120941,1.0128235817,0.3994425237,-0.6306587458,0.8556500673,0.5664199591,-2.7598052025,-0.5542398691,0.9546697140,0.2972911000,-0.5923198462,0.8346678019,-0.5135295391,-0.2254577875,0.3647433221,-1.1742988825,-0.0221651979,0.2011860758,-1.1618353128,0.3587828279,-0.3569490016,-0.6757637858,0.1543741524,0.5523005724,1.1413083076,0.9759478569,1.3029022217,1.5852334499,0.5111664534,0.9931082726,0.9090556502,-0.8942973018,-0.8136970401,-0.3231808841,-2.9619100094,-1.7260572910,0.6305112243,0.9046978951,-0.6565795541,1.1913491488,-0.6926817298,0.8507476449,-1.4126199484,1.5484261513,-0.3532052040,-0.1554211229,-0.5210480094,0.5052769184,-0.5785877705,-1.8058321476,0.4119521677,1.2402284145,0.5937663317,0.9732915163,-0.3283392489,0.1560894847,0.2356980294,0.0013753889,0.4547128677,0.2108037323,2.2417321205,-1.5689145327,1.9793784618,0.6024902463,0.1141058430,-1.3041629791,0.5927038789,-1.4569309950,2.2926297188,-0.7402815223,-0.0437542982,2.4251124859,1.0239573717,-0.9977756739,-0.7223476768,-0.6301113963,1.4681988955,1.1038920879,0.2033595741,-0.6396667361,-0.1190782934,-0.0005420816,-0.1301973015,1.5639034510,0.0902540758,-0.1481139511,0.6896650195,-0.5098571777,-0.3564096689,1.5234655142,0.8508958220,0.8980773091,0.5878921747,0.0211714022,0.1310488582,-1.7261478901,0.4319587052,3.3127100468,1.2555725574,-2.4129700661,0.1548143625,-1.3937963247,-0.0669993684,-0.4945868850,0.6425649524,0.4099414349,-1.2230517864,1.1344254017,0.2476155311,-0.0826025009,-1.7094839811,-1.3960864544,0.0779310912,1.0671143532,-0.7367090583,-0.8694255948,-2.1651918888,0.2056596577,-0.6978781819,-1.3931280375,1.1689006090,-1.0735945702,-0.1320720315,0.3157189786,-1.1467095613,-1.8069293499,0.3965733051,0.6804736853,1.1677420139,0.1507781297,1.5749038458,-1.6790566444,-0.4240026474,-0.0486205667,-0.4190273285,0.0764597952,0.2051582783,-0.1262133867,-0.0686233491,0.9195166826,0.2387665063],[-0.1942841411,-0.9263069034,2.2984178066,1.1526581049,0.8616171479,0.3174169362,-1.7027870417,-1.7472238541,1.0208576918,-0.3556275070,-0.1951083988,0.7903609872,1.3327525854,0.8751086593,0.2799882591,0.1954333037,-0.3937276900,-1.7281918526,0.0852415413,0.9325007200,-0.1659661531,0.7091153860,-0.4390468597,0.0979182422,-1.1979483366,0.4096597433,0.1257333606,-0.6742489338,-1.5028485060,-0.7201532125,1.8562434912,0.0316390693,-0.5786090493,0.8689778447,-1.4990413189,1.0552029610,-1.6764425039,-1.5236849785,-1.5653573275,0.4090256691,0.1788931787,-0.1072124392,-0.7379006743,-0.0220090747,0.2177771926,0.6614961028,0.3497148156,0.2820779681,-2.2396645546,2.3719971180,-0.4914728701,0.3381384015,-0.9512355924,-0.4113211334,-0.1319223195,-0.7636014223,-0.0705542788,0.0945843682,0.5141342282,-0.2185685933,-0.5152382851,0.0872560367,1.3440091610,0.6572620273,0.4007388651,-0.5317484140,0.7063996792,0.9795297980,0.9454191923,0.4927286804,0.2224742323,0.0633541867,0.1593330353,-0.7465167642,1.2386281490,0.2575991154,-0.4987689257,0.2045920640,0.6867196560,1.4968909025,-0.3125460148,-0.9242553115,-1.4892401695,2.4101431370,-0.0921221748,-0.9982684851,-0.1168525666,2.1869966984,-1.4745687246,0.5221762657,0.1889460534,-0.8488870859,-0.9581833482,0.9912944436,-0.4461146593,-1.0186274052,0.8116633892,-0.7028707862,-0.2395506054,-1.3628484011,0.6342271566,0.4812361300,-0.8224328756,0.1313161552,1.5504002571,-0.9111961126,0.2188706696,1.0305371284,-0.0790315494,-0.9508096576,1.2417500019,0.4015571475,0.2897460163,0.1677240729,-1.4348064661,-0.5332908034,1.5793312788,-0.1691692472,0.1776634902,-0.2642582059,-0.5794829726,0.8309162855,-1.6052863598,-0.9368858337,0.9558969140,-0.0408717506,0.3745765686,0.6062545776,0.6897696257,0.6617829800,0.6259799600,-2.4486322403,0.3745160699,1.1456580162,1.0062994957,0.3313815892,1.0699118376,-0.6684012413,0.5727875233,0.6733343601,-0.0626238659,-0.4325711727,-1.7898069620,-0.7044024467,0.2529786825,-0.1920649707,0.3323691189,-1.1825512648,0.5753129721,-1.4658842087,-2.1032445431,1.1631325483,-0.5351141095,1.3923389912,0.8280168176,-0.7771325707,-1.1401008368,-1.7066495419,-0.6760823131,0.4116258919,-0.6975342631,0.6350879073,1.9184994698,-1.0803020000,0.3992839456,1.0008097887,0.1050618738,0.6947386265,0.9541548491,-0.4823937416,0.0798426941,2.0793166161,-1.0948320627,-0.7919138074,-0.3036155403,0.2675095499,-0.3363015354,1.2337770462,-0.0565242618,1.1298168898,0.2538796067,-1.4344059229,0.7543268800,-0.4848037958,0.1588084549,-1.9065899849,0.7605556846,-0.9614201188,-1.6162747145,-1.3088313341,0.7706814408,-1.2203983068,1.0250405073,0.1936548799,0.6911200881,-0.9923204184,-0.7455840111,0.4099641144,0.4822587371,1.1336737871,-0.3551127613,0.3467658460,-0.2646628320,2.0391478539,0.3230980039,0.8732517958,0.3943471909,-0.7295206785,0.6893681884,0.6827322841,2.2047653198,0.2522270083,1.0462615490,0.3333738148,0.9523355961,-0.5945228338,0.4556433558,0.6373681426,0.7313504815,1.3079185486,2.0483045578,0.9635858536,-0.7311703563,0.8531662822],[0.1186441779,0.2476326227,0.7424806952,-0.1205264479,2.3297095299,0.2212003767,0.0787303224,0.4282938540,-0.7846913934,0.7370166183,0.1164571047,0.4009823501,-1.5018119812,-0.7576122284,0.2048009932,0.4468692541,-0.7943142056,0.9658300877,-0.0924994797,-0.5092731118,0.1271789968,-0.5681103468,1.2777352333,1.3446389437,-0.4176205099,1.1254996061,-0.1987000555,0.3697093427,0.6449429393,2.1140766144,0.0287610460,-0.2087999135,-2.2224323750,-0.1076841950,-0.5155770779,0.3130055368,-1.2204751968,-1.6488747597,-0.7013992667,0.9725614786,1.8311501741,-0.0420313030,-0.4865702391,-1.2558275461,0.0240110978,0.0409290753,0.5420319438,-1.2074099779,-1.4793547392,0.0677488744,2.0287189484,0.1211663187,-1.0394985676,-0.2092629373,-0.5419185162,-0.2526287735,0.6307049990,0.7881709933,-1.3417301178,0.5147671700,-0.1236193851,0.3639915884,0.0800566152,-1.4012484550,-1.3367093801,-0.9948487282,-0.2640236318,0.0116868606,-0.1084337905,-0.8910443783,-0.2605855167,0.7310494781,0.2275884598,-0.4247497022,-0.3506610692,0.5176494718,0.8392418623,-0.1413768381,0.4017984271,-0.8027892709,0.0070667868,-0.1382342279,1.1400476694,-1.2793195248,0.9133566022,-0.4161378741,-1.3781870604,0.1653951555,0.4276550412,0.1956777126,2.8762161732,-1.1102243662,0.1171223372,-1.6822745800,-0.9032545090,1.2693028450,-2.2613315582,-0.8250658512,-0.2805194855,-1.4849926233,-1.2322430611,2.0261244774,0.0272019934,-1.8280910254,-1.6233180761,-0.1065211222,-0.2862212360,-0.4451537132,-0.2557170689,0.4339169562,-0.9887716174,1.2868506908,1.1453354359,0.8280850053,0.1992067993,0.2173833102,-1.7708069086,-0.2707109451,0.7886084914,0.0197867677,1.5219205618,0.4495322108,-1.8151134253,0.1503618360,-0.8707200885,0.1044267043,-0.1727676690,-0.3222731054,2.0104050636,-0.2804686129,-0.0797644407,2.4352912903,-0.3499047756,-1.8723024130,-1.0576988459,1.5366865396,0.2987216115,-0.5741385818,0.7206164002,-0.5140460730,1.1396903992,0.4905533195,-0.7538064122,-0.0299488585,0.6763259768,1.1116752625,0.0127552878,1.0710752010,-0.6004653573,0.2281350344,-0.9140645266,-0.3691907525,-0.3375107050,-0.8282238841,0.7861650586,0.3188356161,-1.5241420269,-0.7093237042,0.6100569367,-0.1316596866,0.3417309523,-0.4438876510,-0.8978509307,-1.0393338203,-0.8868865967,1.9541177750,1.0689802170,0.0547712296,0.5502784848,0.0277117081,0.6421114206,-0.6505494118,0.6176762581,-0.9987822175,-0.1520087272,0.1977556646,1.4567295313,0.9476689100,0.3120063245,0.1578296274,1.3490449190,0.9076947570,-1.5802649260,0.0284059606,0.3208372891,0.6578550339,1.2572970390,-1.0926563740,1.2521040440,-0.6177709699,-1.0928300619,1.1918451786,-1.0383467674,0.2914623618,0.5544426441,-1.9991108179,-0.6491318941,-0.1269466877,-1.3599926233,0.2060403973,1.1349284649,0.5194821358,0.8900290132,-0.7164104581,0.5118864179,0.2188488990,-0.6501572728,-1.0021990538,0.9629164338,-0.6489469409,0.1203286722,-0.3681460321,0.0381681658,-1.0675135851,0.2147252262,-1.4677051306,0.9678803086,0.5567008257,-0.4400278926,0.5860223770,0.7764663100,0.7028333545,-0.2341640592,-0.5149731636],[0.7935917974,-0.5220533013,0.6459440589,-0.2909165323,0.6664276123,0.5232625008,0.9840770960,0.6887192130,-1.1589708328,-0.1337791681,2.1475710869,-1.1999230385,-0.0397767313,0.2972322702,-1.1360603571,-1.0197287798,0.0860073194,-1.0629270077,0.7163521051,1.6431040764,0.8835981488,-0.8887983561,-0.4056135714,0.5577527881,-0.5359670520,-0.6085321307,-1.0848847628,2.3708302975,-0.3000078201,0.9129256010,-0.8528723717,0.7104384899,-1.4673618078,-0.0468426906,1.5498622656,-0.6497247219,0.1055054367,-0.6367157102,-0.7790260911,0.1303004473,-0.8807298541,1.2931112051,-0.0242201276,-1.3028495312,1.0931696892,0.8489127755,0.5620557070,0.3824717402,-0.8211005926,-0.2894999683,-0.3295023143,1.9161242247,-0.3875193894,-1.5950083733,-1.2387349606,0.9860461354,1.1215429306,-1.4169552326,0.7774617076,1.3781566620,0.4257713854,0.1041262969,0.6187925935,-0.1452303976,-0.1544398069,-0.3350002766,-0.7903438210,2.2526700497,0.5864151716,-0.0685864314,-0.6246374249,-1.3993204832,-0.4303893149,-1.2822613716,0.1559318602,1.5798255205,-1.0765495300,0.4107350409,1.3614157438,-0.7937434912,-0.1587989628,-0.0826069266,0.1638277918,0.3035634458,-1.0530942678,0.3941660225,-0.3778450787,0.5913537741,1.7228548527,-0.2308357060,-1.0750149488,1.7766871452,-0.2589304149,-0.6578678489,-1.3033595085,1.0156477690,-2.2127144337,-0.7063292265,-1.1299027205,-0.8751794100,0.0943318456,-0.8357192278,-0.6747906208,-0.1819349229,1.2726923227,1.1324168444,-0.0653668642,1.7812215090,1.4365423918,1.2911921740,0.6556486487,-1.8097792864,-0.4941692948,1.0009518862,0.9566744566,0.1753100008,0.7363275886,1.0797368288,1.5626415014,1.5590084791,1.1770696640,1.8266727924,-0.8828266859,-0.5780462623,1.1614030600,0.0819185227,1.1126433611,0.2359033823,1.5360554457,-0.7368761301,1.1381391287,-1.1523531675,0.4186245203,0.4560465217,-1.5258060694,-1.1486253738,0.0944953263,1.1487392187,0.4432187080,0.4212090075,0.1783284396,-1.0721273422,-0.6282792091,0.1184149534,0.1077746376,1.6965463161,-0.0277585164,-0.1918672770,0.5346242189,1.8567492962,0.9038385153,-1.7754197121,0.0008006179,1.3203227520,-0.9212475419,-0.1841087192,-0.1914251149,-0.2672573626,-0.7262297869,0.5493069887,-0.0501594692,0.4068301320,-0.4298945665,-2.2442214489,-0.7497539520,2.2230682373,0.1048255190,-0.7576667070,-0.2962541580,-1.4607772827,0.6312759519,0.2765136063,-1.7077609301,-0.3536328375,2.0978047848,-1.0209709406,-1.9278744459,-1.0835400820,-0.6693260074,-1.7807040215,-0.0103057222,-1.2683321238,-1.4392585754,0.5706706643,0.2299688011,-0.8338006735,0.0692932904,-0.0159100499,-1.2371246815,1.5750194788,-0.4039532244,0.0204617586,-0.3619193733,0.6404605508,0.3293938637,1.2330603600,1.0708794594,-0.2778260112,-1.0820939541,-1.1345940828,-0.9574249983,0.1622495651,1.4215382338,0.4582139850,0.4866745770,-1.1338460445,2.6203932762,0.3832954764,1.0140392780,-0.9759644270,-0.1634153575,-0.7954094410,-0.3030864894,-0.3183051348,0.4918797612,0.8558522463,0.3166932762,0.5860859156,-0.6056419611,0.1423945576,1.3445808887,3.1942925453,0.3613812327,0.7262311578],[-0.0039374535,-2.8172578812,1.2778655291,-0.5762477517,0.8407689333,-0.7480339408,-0.7672458291,-0.1195229515,0.2324158102,-1.8714499474,0.5519493222,-0.5039342046,-1.7590987682,1.6869199276,-1.9298254251,-1.3470934629,-0.6425828338,2.0217587948,0.6767479181,0.3098316789,-1.4022588730,0.6783840060,0.8065295219,-1.1054807901,0.5464166999,-0.3851571679,0.3955411017,0.3208754361,1.5504038334,-0.2144103050,-1.5548576117,-0.2614115775,-1.4425181150,-0.2540749609,0.3945147991,-0.7667369843,1.4624339342,-0.0706093833,0.2780528069,0.8718714118,-1.7170612812,2.0644669533,1.3981959820,-0.0241681747,0.0153257139,0.1551327109,0.8673528433,0.4098148942,-0.4804344475,0.4392388463,-0.6017239690,0.0909047872,-0.6866038442,-0.8969714642,0.9651571512,-0.6932944655,0.6336821318,0.8397692442,2.6203660965,-0.6895518303,-1.7960164547,-1.6501295567,1.8551411629,0.1128157750,-1.7513470650,0.8404824138,1.3223236799,-0.5824505091,-0.0671811923,-1.2150722742,0.9815346003,-0.3406918347,-2.1729745865,1.5029435158,1.4895166159,-1.0849428177,0.0340151675,0.3322142959,-0.5848049521,0.1907823086,-0.8129360080,-0.0226928703,-0.0279446561,-0.2016942650,-1.1095571518,0.8078178167,-1.1353489161,0.2967233658,0.5234735012,-0.7270941138,1.1792708635,0.6389978528,0.3985453844,-1.2015575171,0.7863968611,1.1112177372,-0.5595420599,-0.5727396607,-0.8466938138,-1.9493989944,-1.0869103670,-1.0681582689,0.3008961976,-0.5245221853,-1.5388499498,0.6930798888,0.5540941954,0.9507279992,0.8663878441,-0.4860620499,-0.1471722573,-0.1502821445,-0.4803615212,0.6171793938,0.5383598208,1.2860487700,-0.1998111606,2.9812784195,2.1534991264,0.6020771861,-0.2183421254,0.5264843106,0.2851968110,0.4971400797,-0.9605002403,0.9666528702,-0.0700143129,-1.3566243649,0.3706783354,0.4595982730,-2.5560221672,0.3511317670,-0.6403661966,-1.2884974480,-0.6789479256,-0.6717274189,-1.2106477022,-0.0163370986,0.3457172811,1.6352626085,-0.4889360368,0.5581252575,0.3349420130,-0.1122701392,-0.9621224999,0.8743726611,0.5053303838,-1.8802504539,1.1275217533,-1.1256831884,1.3232104778,0.3116884828,0.9008846283,1.4898868799,0.0042057778,0.1580962092,-0.9577461481,0.0220026318,-0.4892366827,0.1239973605,1.1565297842,-0.4127237499,-0.1431506574,0.0075374050,-0.2322238684,-0.3758442998,0.4405148923,-0.2959225178,-1.9549883604,0.0191233903,-1.1686453819,0.3244661689,-0.7563298345,-0.9059374928,0.2940572798,1.7263318300,-1.4125913382,0.8717287779,0.8929125071,1.0991194248,0.7424063683,-1.1658024788,0.1678114682,-1.3147382736,1.0513582230,0.8278679848,0.9144086838,-0.2248248011,-0.6977533102,0.7616482973,0.1096616462,1.9541888237,0.2071506679,-0.0072634853,1.6672128439,-0.2180464268,0.2142009735,0.6916992664,0.8198504448,0.2656022012,0.4191500843,-0.1563543826,-0.0766619965,-2.0684545040,-0.5974961519,-0.3178389370,0.9397915006,-0.2236629725,-0.6539345980,-0.1499699354,1.3334772587,-0.2018540204,-0.8706596494,1.8530994654,-1.2432664633,0.0694535226,-0.6399093866,0.5563447475,1.1774454117,0.3720839918,0.0705601349,-1.5265114307,-1.3808004856,-0.7226364613],[-0.0671254769,-0.7904212475,0.1652627289,1.6826713085,0.9410175085,-0.1707097888,-1.1935794353,0.5031608939,0.7144062519,0.3102995157,0.0893526301,0.4119528830,-0.6044661403,0.7267200351,-0.5432944894,-0.4802464545,-1.2587251663,-1.6022361517,-1.0864841938,-1.4701102972,-1.3144232035,0.9932833910,-0.4609676600,0.4559967816,0.9519225955,1.9400188923,-0.9149392843,-0.5548046827,-0.0177294016,-0.7559466958,0.4510883987,-0.0255159680,-1.1708710194,-0.7565687299,1.3700623512,-0.8828909993,1.1024614573,-0.4919537604,-0.8594636917,-0.1812487990,-0.5608417988,0.7042586207,1.0489093065,-0.0738557726,-0.3878282309,0.0897080898,0.6622738838,0.3745981753,-0.2529806495,-0.2844143212,-0.0747609586,0.3734287322,-0.3683740795,-0.9679385424,-1.5396726131,0.1518522203,1.1448954344,-1.1830432415,2.3214013577,-0.2190132141,-1.0381678343,0.4893311262,1.2217268944,0.4482461214,-0.7320929766,-0.0737974867,-0.4775159955,-0.2962387800,0.9918540716,-0.2520774901,0.7377212644,-1.1158239841,-0.3469292819,0.7313711047,0.6559993625,-0.9429005980,-0.9391338825,0.8579667807,0.0583344996,-0.0099074114,1.0966665745,0.1336360723,-0.1326661706,-0.9921869636,0.5340189338,-1.0867534876,-0.4550673962,1.3559694290,-0.1116294712,1.0091249943,0.1412471086,0.5255767107,-0.3881579638,1.3969186544,-0.1676418036,-0.1452837437,0.9945383072,-0.2506922185,1.1986724138,-1.7370616198,-1.3467944860,0.8549793363,0.2784049809,-0.6807608008,-0.6913851500,-2.2432849407,-2.3831932545,0.7839045525,0.0188699197,1.0645942688,0.1286784858,0.4382385910,-0.6687325239,0.5978592634,1.1087765694,-0.1641141921,-0.3620709479,-1.7388533354,1.3200438023,-0.2533108592,-0.4184384346,-1.5299417973,0.5822222233,-0.5063092709,-0.1981868595,-1.3021932840,1.2743741274,0.0432033651,0.4569822550,-0.5047770739,0.1699194163,-1.3279764652,0.3524358571,0.1321837306,0.5692359209,0.6053616405,0.2480343133,1.6470834017,0.3169894814,-0.5630028248,-2.4546029568,-0.3362979591,-1.3944190741,0.7804767489,-1.0859674215,0.4189290106,0.8998404145,-0.6749925017,-0.7698157430,-2.2536473274,-2.3058500290,1.2095606327,0.1959027052,0.3622468412,0.0256938580,-0.9933300614,0.4945476353,0.9161879420,1.4648804665,-1.8006176949,0.7049255967,0.8880855441,0.4263364673,0.2365619391,-0.3710543811,-0.7134770155,-1.3234868050,-0.0384293981,-0.1907331049,0.2600144148,-0.8479497433,0.5255256295,-0.5529755354,0.1325170100,0.2094378769,0.3922344744,-0.2805377841,1.1552339792,0.0220841095,0.7056201100,-0.8617077470,0.5466896892,0.8280608654,1.7718749046,0.5791893601,0.3954809308,0.6250145435,1.8468208313,-0.5212256312,-0.7269707918,-0.0188041087,0.2731238902,-1.7006938457,1.2754198313,0.4935974479,-1.1783540249,0.6881445646,-1.0047477484,0.8587782979,-0.4211801887,-1.1423093081,1.8459129333,0.4986789227,0.3107667565,2.1952350140,-0.2329679281,2.2344813347,-2.3799045086,-0.6172450185,0.9939242005,1.6428675652,-3.8169021606,0.8213685155,-0.2383841127,0.8085193038,0.8943603039,-0.7287612557,-0.8341719508,-1.6358424425,-0.0163045768,-0.3030619025,0.0600189455,1.0085133314,-0.7476360202],[-0.5483891964,-0.9263272285,2.6085176468,-1.2780238390,-0.9766176939,-0.5389821529,1.3002591133,0.9012833238,1.6520648003,1.8740491867,2.7003669739,1.3751206398,0.7383854389,0.4513852000,-0.0785458460,0.9784024954,-0.9420097470,1.0711314678,0.1228365153,-0.4089529514,-0.0938136727,-0.4688596725,-0.0395304710,-1.4214097261,1.9610992670,-0.8136855960,1.5312091112,0.7010276914,-1.3808984756,-0.2077739984,-1.0954723358,-1.5649642944,1.6708961725,0.0262108147,0.4416314065,-1.0797450542,-0.4252988994,-1.1980069876,-1.4782294035,-0.9501484632,0.0933405459,0.8175611496,0.4934800565,-0.0887110531,-0.2513252497,0.2430290729,0.0722308606,1.1764454842,-0.5189903378,-0.4726536572,2.1911048889,-0.4614456296,-0.1552190930,0.6430007815,2.3364167213,0.8803315759,1.2821724415,-0.1783988476,0.6398929954,-0.6916404963,0.0685556680,0.4741430879,1.7757326365,1.3293614388,-1.6313576698,-1.1642336845,0.3845895231,1.0874531269,0.8030692339,0.9298844934,1.1026318073,-0.4465782642,-0.9066677690,1.6973699331,0.3255869448,0.4437576234,0.6298837066,-0.1388432831,0.4877704084,-0.5768527985,-0.6939834356,-0.6422843933,3.4686851501,-0.8099491000,1.0799697638,-1.1004024744,0.4231131375,0.1130894870,-1.2025954723,-0.4120817780,-0.9697688222,-0.8534077406,-0.5020948052,1.3400472403,1.7938925028,0.6689025164,0.9888962507,0.2849170566,-0.0868029594,-0.2804565132,0.2670521736,1.1490031481,0.2609162629,1.0914986134,-0.1469503343,-0.1726162434,-1.0124996901,1.6172940731,-1.4287543297,0.0207366310,0.4133826494,-2.1272759438,-1.0434956551,0.3534581065,1.8912388086,-2.0706202984,0.7521951199,0.1733450145,0.3660833538,0.4582214057,-0.0752727687,1.0993863344,0.7004218102,-0.2836722732,-0.3275241554,-0.9340184927,1.6282463074,0.6356199384,0.0225061178,-0.3679436743,-0.3817060590,0.6974017620,0.0895482525,0.4078374803,-1.0503004789,-1.8791811466,-1.1724802256,1.0532171726,-0.0058436240,0.2517630458,-0.7201429009,1.2702769041,-0.0458424352,0.4193159938,1.5528337955,-1.1905040741,-1.3259557486,-0.5397362113,0.7404883504,-0.9480809569,0.7349120975,0.2989366651,1.1667926311,-0.0275125522,-1.9530737400,-1.0072118044,0.5982735753,-0.7819896340,0.2409633547,1.0072343349,0.5993404388,0.1452574730,0.4981725514,0.4348387420,1.0504794121,0.9790147543,0.6895599365,0.1251991391,-0.8385702968,0.8198758364,0.3724953830,0.5099990368,-0.4112640321,0.1107642874,0.3995991051,-0.3792087734,-0.1735995412,0.0200130008,-1.1897200346,-0.0422304757,-1.0703550577,-0.0569381416,-0.0801360905,-0.8206369281,0.7945466042,1.3026648760,1.1943631172,0.4166095257,-2.3133356571,-2.0431711674,-0.0859138668,-0.7850488424,0.4934567809,-0.0740535557,0.5806821585,1.1419591904,-0.5928174853,0.3389694393,0.9335060120,1.5325789452,0.2682217956,0.6445912719,-1.1811283827,-1.3357715607,-0.2605241239,-0.1114650816,-1.0182495117,-0.1397437006,0.4789614975,-1.3894425631,-0.8259648085,-0.9689841270,0.7458997965,0.2991378307,0.8106826544,-0.3790820837,0.7313722372,0.6382530332,2.0013265610,0.2483182549,-2.0251369476,0.1511827558,-1.8691866398,0.3484821618],[-1.3629926443,0.0344463214,0.2344449162,0.4313440025,0.2729780376,-1.5452675819,-0.0550522581,-0.6612935066,0.5417645574,2.0425043106,0.2200262994,-0.1124477014,-0.3067903817,2.6385672092,0.0758075565,0.9185811877,-1.1661971807,0.0555665419,-2.0132887363,-0.0612099841,-0.0498628020,1.1586894989,-0.4992930293,-1.0344942808,-2.0566396713,-1.7438385487,0.4182908833,1.9617835283,0.3961114585,1.9309381247,-0.4613923132,0.1689138561,-0.1322667748,-1.2334487438,0.9377317429,0.7998334765,-0.6520880461,-0.1707964838,-0.5642157793,1.2804228067,0.0208851863,-0.5527569056,-0.4434795380,0.0876937509,1.7762702703,0.8643834591,0.7251709104,-0.2246964127,-1.1765624285,-0.8831675053,-1.3195488453,-1.7548000813,-0.6290677786,0.4152307808,0.5666602850,-0.2163264304,-1.6369053125,1.3941433430,-1.7514255047,-1.9723445177,-0.7031470537,-0.9993395805,-0.8704144359,-1.2276570797,2.4672963619,1.1786454916,0.2572571933,0.2190767229,0.6950595975,0.7149975300,-0.0045571011,0.5289281011,-0.6817349195,0.7423099875,0.0222218968,-0.3022816181,-2.5626525879,-0.8660374880,0.5463693738,0.1478862166,0.4148804545,-0.2576435804,-1.3388051987,-0.9884957075,-0.1858710945,1.3034764528,1.2491785288,-0.2166267633,0.3619436622,-0.5870447755,0.4298285246,0.4556368589,1.4775171280,-1.2154983282,0.4434569776,1.2889420986,-0.1644225568,1.5875742435,-1.2383381128,0.1369165182,0.8227247000,-0.2720991671,1.3772487640,-1.0866830349,0.5779238343,-1.3674609661,0.3796960711,-1.2340633869,1.5236074924,0.8480984569,0.8815364242,-0.2750021517,1.0931401253,0.3164792061,-0.6241439581,1.0555865765,-0.3296152651,-0.9498324394,-0.6139930487,0.0078044580,1.9462395906,0.1260036528,-0.0267669503,1.5000147820,-2.2058892250,0.2527365386,1.4516202211,0.0328859240,1.4210357666,-1.8086668253,-1.2372293472,1.3431107998,-0.3939828873,0.3295665383,0.0458708517,-0.8294584155,2.3301115036,-0.1783146113,-0.3502511084,-0.8924494982,1.0734086037,-1.8520869017,-0.9093677998,-1.1956225634,-0.4636309147,-0.2347074598,-1.0941042900,0.7002073526,-0.0075926357,-1.6863496304,1.6743515730,-1.3652117252,-2.9964680672,-0.0707102716,0.1781398654,0.4942721725,-1.4362123013,-0.7788179517,1.6206996441,0.2547658384,-1.1822535992,0.9864816666,0.3597769141,-1.1920862198,-0.5226871967,-0.6126774549,-0.7237797976,-0.1754818857,0.2489106804,0.8328000903,-1.1851280928,-0.5307146311,-0.6191227436,0.7989680171,0.1682512611,-1.1042748690,1.3994650841,1.0414515734,0.0627358332,0.6830793619,-0.0892262608,1.5641775131,-0.1822139621,1.6051907539,0.8997560143,-0.0619025528,-1.2406721115,1.5553014278,0.6757091284,0.2351159900,-3.2029783726,-0.2392832935,0.0999327451,-1.0192216635,-0.5507404804,-0.4247370958,-0.1092737615,-0.4355481267,2.0317144394,-0.2517910004,1.1875435114,1.0763266087,-0.6341527700,1.2765887976,-0.6087099910,1.2905139923,0.2440712750,-0.2550278902,0.6374529004,1.1602060795,-0.5558677912,-0.7458577752,-0.1815477461,0.1130491644,-0.1535087526,0.8310692906,0.2297729254,1.5552196503,0.3676000535,0.1164899766,-0.2793305516,0.0842381567,1.1744824648,-0.5380910039],[1.1350588799,1.6307651997,0.5054000616,0.3415246606,0.1714084893,1.2019101381,0.0502630584,0.5542773604,0.9386487603,-1.0577836037,-0.2838217020,-0.6229845881,0.0756353810,-2.2039809227,1.0031237602,0.8466241360,-0.4926395118,-0.3171692491,-2.5975430012,-0.9221938848,-0.4696512222,-0.1379987448,1.6596388817,-0.7340003252,-1.3171730042,0.3707151115,-1.3485159874,-1.1177152395,0.8394469023,1.7678155899,-0.5140664577,0.3884877563,1.1650243998,-0.7375556231,0.0720298588,-1.3064266443,0.8903331757,-0.4982529581,-0.2980082631,-1.1973793507,0.4307799637,1.4663467407,-0.4940899312,-0.1184043512,0.6036401391,-0.6061331630,0.1731641889,-0.4107045531,-0.2922571301,-1.0013037920,0.1425794065,-0.4769991040,-0.8599395156,-1.5917048454,-1.4707101583,-0.5162295699,0.0312709026,0.9078719616,1.0566560030,-0.0541705601,0.4534251690,-0.3871499896,-0.1667667329,-1.3632845879,0.8195942044,-0.7354663014,-0.6091674566,0.8308277726,1.1752266884,0.7835314274,1.3266550303,-0.0654770806,-0.3728080094,-0.8336992264,1.1622248888,-0.3846301138,1.0055280924,-1.2250599861,1.1725132465,-0.1270800084,1.9019786119,1.5142729282,-0.1225099191,-0.1151846126,-0.1686504632,-0.0648666248,0.1656596959,-1.8448135853,-0.4019766450,-1.2433512211,-1.1153597832,0.9793652296,1.3056718111,-1.3503453732,0.5518975854,-0.2453743219,0.0945341364,0.9384822249,-0.4939763248,0.1662771553,-0.0385320112,0.4536897242,1.6621969938,-0.8693246245,0.5161611438,0.3339156210,1.2932506800,0.8945618272,-1.8559041023,-1.0705528259,-0.6615146399,0.0996204913,0.4900225997,-0.2379509211,-0.0018551836,-0.0410420187,-0.6998842955,-0.6224127412,0.7980476022,1.6843181849,-1.3955712318,-0.0199635271,0.6951984763,0.5448355675,-1.4337213039,-0.6472737789,0.5337479115,-3.0123813152,0.1207514852,0.3387083411,1.0219434500,0.3195720017,0.4922348559,0.4318786860,-1.5374253988,0.2995702624,0.6266925335,-0.1475751251,-0.9628657699,1.3692935705,1.7332062721,1.0504539013,1.6112185717,0.3021770716,-1.5211652517,-0.0542259924,0.7082381248,0.2385789454,-0.5415235758,0.2251457572,-0.7766765952,-1.1232200861,1.4787577391,0.0738270581,0.9700493217,1.4055027962,0.2390944362,2.5265150070,1.1471470594,-1.9478795528,0.3010020554,-1.1164835691,0.5054375529,-1.1840339899,0.8414674997,-0.2105533332,1.0772610903,0.1378324628,-0.2625436783,-1.2023355961,-1.2959524393,-1.9612358809,0.4218974710,-1.6659665108,-0.5268239975,0.9753533006,1.0594240427,-0.2273750156,-0.7501193285,1.5347290039,0.6678853631,-0.0581064932,-0.1379352659,-0.2763449848,-0.2625544667,0.9622181654,-0.5553058982,1.4604641199,-1.2602964640,0.0350733362,1.3494240046,0.0647262409,-0.2970011532,-0.1286823303,-0.4147179127,0.3933965564,1.0783635378,-1.1027158499,0.9179580808,-1.1704337597,-0.1262967438,-0.6603304148,0.2892201245,0.3097912669,-0.4064918160,0.6283999085,-1.0411736965,-0.0379688144,0.0183309894,0.4731063843,0.9111999869,-1.3404321671,-1.8525221348,0.4234216213,-0.0911732167,-0.1249522865,0.3678356707,0.5566195846,-0.0299841147,-0.0175340120,1.2085926533,0.6181845069,0.7668458223,0.2914898992],[-0.3310023546,0.5573409200,1.2957259417,-1.1349855661,-1.5247069597,-0.1056646109,0.8937266469,-2.4366216660,0.4809564352,-1.0557091236,-2.0977416039,2.2568325996,0.8655489683,1.5234045982,0.7148271203,0.9997466803,-0.3668726683,1.5002795458,0.5295903683,-0.0688808635,-0.3074559867,-0.5956582427,-0.7585545778,-1.7481904030,-0.7318330407,-1.0182477236,0.2035604119,1.8202784061,-1.6629431248,1.1729696989,-0.8748952746,0.4623175859,1.7992956638,-0.2273533642,0.8438876867,-0.5174407363,-0.3450641334,-0.2517184317,-0.4905411601,0.6074772477,0.8887094259,-1.1760468483,0.2600905597,-1.2200479507,0.8933705688,2.1258146763,-0.5572912097,-1.8600933552,0.7779752016,2.5773346424,0.1733140200,0.1253862232,0.5314847827,0.2223214656,-2.9037373066,-0.9753293991,-0.8351879120,0.0942138582,-0.6895788908,0.7909314632,-0.9552357197,1.1238473654,0.5297656655,0.7592256665,0.2255097628,-0.8313544989,0.9092111588,-0.3085229695,1.8506548405,-0.2688187361,-0.0903450549,1.1477758884,-0.8157768250,0.5765594840,-0.6213289499,-1.0924533606,0.3988735378,-2.0458335876,-0.5939063430,0.9805639982,-1.0290400982,0.0781027302,0.2894194126,-0.0519753769,0.8758471012,-0.3554590642,-1.8353921175,0.0100867106,-0.4113383293,-0.2478044927,1.4757231474,-0.1460000873,-0.1348475516,-1.9121975899,1.1605008841,0.4398345947,-0.8364595175,-1.1910685301,-1.2645142078,0.6124418378,1.1392834187,-1.2183645964,0.0038925342,-1.0627678633,-0.7069594264,0.4486484826,1.2478134632,-1.8462580442,-0.6957844496,-1.5319015980,0.6585696936,0.7199642658,-0.2957348526,0.6642016768,1.3873531818,0.9918613434,0.2797018588,-0.4639515877,-1.5886434317,-0.4204191566,0.8649352789,0.9510036111,-0.9391480088,2.0292148590,-0.1282894462,-0.7238674164,-0.1788964719,0.0195331033,1.0771375895,-0.4234253764,1.4801914692,-1.7666741610,-0.5446242094,0.6746259928,0.1982609779,-1.1595536470,0.9801321030,1.8839025497,0.2379632145,0.5130231977,-0.6771757603,-0.3835451603,0.6720554829,-0.5601616502,-0.2693561912,-0.7150922418,0.9604054689,-0.1766158491,-2.1141669750,-0.1035743877,-0.1573709249,-0.6674705744,-1.6187994480,-0.3529086113,0.0866112337,2.2035353184,1.0520107746,-0.4452825189,-0.3899340332,0.1289973855,-1.2968221903,0.3952779174,-0.2544807494,-0.4893265963,-2.1965639591,0.2055465132,0.7529613376,-0.3973491192,1.2315872908,1.6232351065,-0.0800956413,0.0522243381,-0.6569432616,-0.7473552227,0.1069898829,-0.9588980675,0.2506351173,-0.0965848640,-0.0504180305,0.5609652400,2.0323703289,0.0770998970,-2.0426630974,-1.7376391888,0.7450774312,1.9501069784,0.6154040098,0.9178984165,1.4370058775,-1.7092440128,0.5272009969,0.5203163028,-1.1656514406,-0.5942999721,0.1818185747,1.5111366510,-0.0468265265,-0.5543540120,-0.1215622202,-0.8664098978,-0.2492751628,0.0694557428,0.1134820506,0.3137679100,-1.2232038975,0.8229790330,-0.7367380857,-1.8020118475,0.0884521306,-0.8212783933,0.2604509294,2.0971336365,-1.0609384775,0.7672310472,0.5581717491,0.0284287650,-0.7300484776,1.6223838329,1.8920705318,0.7585506439,1.2013385296,-0.6682579517,0.5687101483,-0.0458690971],[0.7914537191,0.6461397409,0.5958306789,0.3313758969,0.5736564398,0.1297451556,0.6420159936,-0.3512704968,-1.6068793535,1.0904586315,-0.1694664806,0.8543040752,0.4681234062,0.6300991774,-0.5073465705,0.0981798470,0.5250322223,0.2505944371,-0.7383431792,1.5344400406,0.8768033981,0.9915237427,-0.1485308558,2.7769432068,-0.7796960473,-1.1073904037,0.6106311679,-0.1776532382,-0.7403364182,-0.0547751002,0.1981050968,0.4532022774,0.9392779469,-0.8190399408,-0.7111040950,-2.2401287556,0.0680859536,-0.5838615298,0.2028556317,-0.4420544803,-0.5240891576,0.1586313397,-1.1596050262,2.0908944607,-0.6640832424,-0.0376775339,-1.3674635887,-0.0834356248,-1.1543424129,-0.2867710888,-0.2558217049,-1.2492008209,1.9734691381,1.3248716593,1.6726819277,0.1237697229,-0.5251456499,-0.0927522108,0.9765563011,0.4406595230,-2.0358545780,-0.0322042890,0.2834545076,-0.2984555364,2.5113687515,0.6123604178,-1.7609668970,0.1541956365,-0.1179339066,-1.6794335842,-0.0388058573,-0.3712675571,-1.3224407434,0.8247251511,0.1571902931,-0.4441169202,-0.0230798498,-0.0669803545,0.8565628529,0.4665855765,-0.7799400687,-0.8101128936,-0.5259451866,-1.4127696753,-0.9107655883,-0.0380207002,-1.1302808523,0.2475105673,-1.0421621799,-0.2518622875,-0.6521123648,0.9967294931,0.3215339780,0.1610585898,1.0579377413,1.0964981318,1.0885875225,1.5951294899,-1.4319165945,0.5318467021,0.3024968803,-2.2370159626,0.3017950356,-0.3017114103,-0.6175892949,0.8086286783,-1.0254920721,-0.7773222923,0.3564308882,0.8522893190,-1.5144516230,-1.2158509493,-0.0528383479,-0.2945184410,-1.1897754669,0.1364751011,-0.3321232796,-2.0284500122,-0.6192065477,-0.9456180334,-0.1601357013,0.4249179065,0.1432779431,-1.2779705524,-0.1355749816,-0.5009277463,0.6132161021,-0.2418944091,-0.2416060418,-0.2200713307,1.8523660898,-0.1587850600,-1.9892187119,0.9731972814,1.1787183285,-2.0948641300,-0.8218100667,0.9060323238,-1.3545004129,0.5676369667,0.1284798980,-1.0084096193,1.4587696791,-0.5944750905,-0.5187753439,-0.6267364621,0.0450868420,0.2594417930,0.8670076728,0.4846119881,-1.4180843830,1.0617189407,-0.2597389221,0.9757429957,-0.9346953630,-0.4718877971,-0.1700360179,-0.2180638164,1.1518630981,-0.5961496234,0.1541005969,1.2585749626,0.1049211174,0.1370392442,-1.1218445301,-1.2547951937,-1.3814834356,0.9695735574,-0.1086363643,-0.8288313150,0.5897889733,0.1763927042,-1.6123449802,-0.3767241836,1.0482236147,-0.6818988323,-0.8554486036,-1.0561922789,-0.9120386839,-1.0435223579,0.7464059591,1.5055519342,0.5351999402,0.1048421785,-1.5796937943,-0.1231038421,-0.1131892130,0.0296576433,-0.1823898405,-1.0289748907,-0.8570144176,0.3856546283,1.2521356344,0.4814043343,-0.6368389130,-0.4543634653,0.7679942846,1.6410802603,1.9045054913,0.1016824543,-0.9051506519,-1.0915925503,0.5618121028,-0.0764838457,-0.1261715442,-0.2757646739,-1.6904895306,0.1201585308,0.4685803950,-0.7667537332,0.5887155533,1.0020669699,-0.4119359553,0.5783369541,2.0256936550,1.3819185495,0.9228142500,-0.8997434378,-2.1333208084,0.5421010852,0.5343675017,-0.1246316880,-1.4611852169,-0.3485037684],[-0.1807983965,-2.3376026154,-1.5892693996,0.0738651678,1.8555306196,-0.3728599250,-0.7936410904,-0.7694765329,0.1537291110,-1.2192952633,1.0588021278,1.7063795328,-1.4852479696,0.7988227606,-1.5708323717,-0.0804250613,-2.3823585510,-0.7975266576,-0.6703687906,-0.1327716857,2.0303711891,0.4827000797,-1.0179685354,-0.1614174098,0.3180173337,0.4515491724,1.3316066265,0.6386704445,-0.2509791851,-1.0985279083,-0.5632628798,0.0922431201,-0.0308225118,-0.3874062300,1.0752867460,2.0628256798,-0.0924772248,0.5879930258,0.2235952467,1.9383288622,0.2087660283,1.0126709938,-0.9500865936,0.8185034990,-0.6236126423,0.2088777721,0.1101473123,0.3424575925,-0.1206818819,-1.3793892860,-0.4113827944,0.9614917040,-0.9937870502,1.7580798864,0.0080232378,1.3879599571,-1.9699418545,-0.0177614167,0.3617301583,-0.0824940875,0.4315568209,0.5943727493,0.3909710050,-0.2284035534,-0.0388397276,0.7088179588,-0.8342286348,1.0389921665,-0.4749735594,-1.2656178474,1.6008704901,0.0157479066,-0.8812649250,-0.2172022909,0.5726627707,1.1687834263,0.7123550177,-0.2866460979,0.9056574106,-0.1671238095,0.7700747252,-1.4749193192,-1.9875162840,-1.9177916050,-1.0036017895,2.3900575638,0.1375515759,-0.6110166311,1.5173581839,-0.5640364289,1.3913329840,-1.1090196371,-0.4102989435,-0.4230274558,-0.8586001992,-0.0699972957,0.5191432238,-1.7878763676,-2.2792520523,0.4116008580,-0.5971567035,0.6625709534,-0.1563580632,0.0519339070,1.3340390921,0.3207374215,0.7841516137,-0.1863603294,0.8826067448,0.8950135708,2.3930306435,-0.1827965528,-1.4979075193,0.6857637763,0.4625678360,0.2142915875,-0.6569813490,-0.3911370039,-0.0136631001,-0.4137517214,1.3745381832,0.9061549902,-0.6315728426,0.3884846270,0.7096084952,0.2195208967,-0.9828910828,-0.6491459012,-2.3690044880,-0.1559623629,-1.0859808922,-1.3553333282,0.2614978254,0.3400335610,0.7255932689,0.0967543870,0.3278695047,-1.0589563847,-1.6091902256,0.5406217575,2.1439728737,0.6192141771,0.9494862556,-1.4554423094,-0.0448695049,-1.4860472679,0.5595998764,-0.9465503693,-2.0627651215,0.4471147358,-0.6683236361,-1.2262574434,-0.2228455693,-1.3789155483,-0.5255289674,0.1673466712,0.5638135076,-1.4176610708,0.5824736357,-1.4399206638,-0.2605627477,-0.5131645203,1.5817571878,0.9342992306,2.0273759365,-0.9074780941,-1.4963614941,0.2098528743,-1.3330277205,-1.0364831686,-1.1096246243,1.2138468027,-0.6777035594,-0.9634899497,-0.2676663101,1.7159404755,-0.5196566582,0.5119627118,-0.3616420925,-0.3848065436,-0.2256528139,0.3118752837,-0.3490059674,0.1993868798,-0.9058728218,0.4433694780,-0.3112775981,-0.0912733003,-0.6117324829,-0.5883981586,1.7798914909,0.0368696302,0.5935785770,-1.6113942862,-1.8226302862,0.7137543559,0.3825588524,-0.0519888066,-0.5621618032,1.2645081282,-1.4425507784,-1.0361926556,0.5223254561,-0.3062395751,-0.6406183243,-1.5627833605,0.7695292830,-0.3185529113,0.1646706760,-0.3141778708,0.6940912008,-1.3859407902,0.2595100701,0.9433202147,-0.9889213443,-0.7876140475,1.5091377497,-0.0022732059,-0.9895153642,-0.7657221556,-0.3768515289,-0.7495852113,0.9069278836,0.8732688427],[-0.6578397751,-1.8318632841,-0.5395345688,0.3903403282,1.3636548519,-0.5079500675,-0.4028911889,-0.9917718172,0.4559382498,-0.0439116731,0.6223576069,-1.9363251925,0.2596221566,-0.4668768048,-0.1926385313,-1.1734788418,-1.9986387491,1.9325946569,-1.3068764210,1.1438997984,2.0557627678,1.9167425632,-1.3739305735,1.8942967653,-1.3069814444,2.5766277313,-1.2845826149,2.6390814781,0.0117756752,-0.8140695095,-0.0189126283,-1.7700136900,0.1316619217,0.6881023049,-0.0800242648,-0.3123969436,-0.1349624991,1.1459685564,-0.3061838448,0.8619158864,-1.3219367266,1.3408268690,-0.2174276561,0.7989725471,0.1819919050,0.3068866432,1.5338616371,2.0315186977,0.7892776132,-0.7427718639,0.5892996192,-0.0023908424,0.2674568594,0.0645150989,-0.4013667107,0.8512451053,0.6213958859,0.8364474177,1.9893764257,-0.1589120328,0.4406788349,-1.8999271393,-0.0583184175,2.8087897301,-0.6425076127,0.5418109298,0.3948427141,1.2454727888,0.4545042217,-2.7257065773,-0.8574002385,2.5853521824,-0.3502472937,-0.5604780912,-0.6599280238,-1.1559455395,1.3983881474,-0.8947708011,-0.3445873260,0.0215723384,-0.5061492920,-0.1462244987,2.1528968811,-0.8219048381,-0.6240154505,0.9768301845,0.6534996629,1.4312934875,0.5451650620,-0.1355044693,-0.4385142028,1.1829334497,0.8356802464,-0.5545064807,-1.4946864843,-0.9651259780,0.8002617955,1.6885603666,0.4508632421,-0.5405079722,-0.7677236199,2.1276350021,-1.5085190535,1.0470719337,-0.3742038906,0.4816255867,1.2239198685,2.2002627850,-1.1459487677,-1.9393211603,0.8348519802,-0.8973862529,-0.0270348825,-0.4584574997,0.2868529260,-0.0345946364,-0.9449191689,1.6783432961,-2.5124733448,0.2714243531,-0.7285017371,-1.1114827394,0.4374779165,1.2229865789,0.1188821420,0.6417844892,-0.0339858830,0.1788398623,-1.2683531046,-0.8834310770,0.0883188769,-0.7413329482,0.4079362750,-1.5605533123,0.6710539460,-0.3997884691,-0.8656931520,0.5296225548,-0.1787042618,-0.6306082010,0.1262859553,-0.0460716784,0.9800592065,0.6499250531,0.2291618437,1.3460974693,-0.4842807353,-1.3417708874,-0.3950604796,-1.0232306719,2.1392402649,-0.4437207878,0.3694180548,-0.8835060000,-0.7907400131,-0.6032630801,-0.4702125788,0.0179922599,-0.4642401636,-1.3101488352,0.8742898703,-0.7117217183,0.1661468446,-0.5677089095,-0.8415204287,0.5891016126,1.4322475195,-1.4599040747,0.4364635944,0.7957346439,0.4348514676,0.2555656135,0.7927491665,0.1485411376,0.5828484297,0.8439933658,-0.5871391296,0.0046702735,0.2771545649,-1.2712996006,0.2197576612,0.0016449381,0.9348805547,-0.1721846610,-0.2281569988,0.4430972934,-0.7958605289,0.9351830482,0.1677804291,-1.0998831987,-1.4856913090,-0.7987915874,-0.0081814257,0.3317958713,0.0209362600,0.9280778766,1.3373113871,-1.8254497051,-0.4517202079,-0.4237102866,0.4339829683,-0.0882978663,-0.8270207047,-0.5583283305,1.0447891951,0.3826203048,0.6272954345,-0.4586470723,0.5723845959,0.7396498322,1.7548961639,1.2826986313,-0.7351671457,-0.8461158276,0.0586707927,0.4825576544,1.0927946568,-0.8203950524,-1.0296913385,-0.1650319546,1.3045974970,-0.0735537857,-0.5020622015,-0.2131796926],[-1.9353233576,-0.0282856803,-0.0519692637,1.2257838249,-0.3289196491,0.0804954693,0.6277905703,-0.4614386857,0.8901169896,0.6736842394,0.2361200452,-1.7104322910,1.4444857836,0.6761213541,1.8072758913,0.5737406015,0.1480455250,-0.5947783589,-0.0989431441,1.4684959650,1.1539797783,0.2363432348,-0.1957866400,-1.4308358431,-1.0022342205,0.6599704027,1.3189892769,-0.0529710427,0.7598709464,1.4087154865,-0.1161145121,1.4070295095,0.5922804475,-2.0745668411,1.0892683268,0.6900909543,0.2436967641,0.5517532825,1.2490776777,0.1645147800,0.5463775396,-1.2008447647,-0.6278283596,0.5528708696,1.0359449387,0.0826989859,-0.2087807506,-1.0988688469,0.8048434258,-0.8141828775,0.3546221852,-0.2129131258,0.2873154879,1.7163056135,0.3986051679,-2.2500936985,1.6220793724,0.1039571762,-0.6963388920,1.6507164240,-1.1810199022,1.0891958475,0.4307694137,1.2198220491,-0.9328013062,-0.7104882002,0.2273416668,1.2003076077,2.2550117970,-0.9103827477,-0.5597643256,-0.4322056770,-0.4192131460,0.7286592126,0.0349851511,0.8051463962,-1.2571941614,0.2547564805,0.2583035529,-0.4347037077,-0.0791249797,-1.6814187765,0.5205547214,2.3092405796,0.7493299246,0.9001592994,-0.1037041694,0.7711718082,0.3438462913,0.4822458625,-2.0766065121,0.0221872889,-0.0421145484,-0.7678053975,0.3982978165,1.2170741558,-0.4540759325,-0.7831273675,-1.0605846643,-1.6356533766,0.2026299387,-0.1641400456,1.0439503193,-1.0182771683,0.1616994590,-0.2499357611,0.7871867418,0.1861835867,0.5747888088,-1.7889155149,-1.9064621925,-1.0385804176,0.7762565613,0.3348379731,0.2335910946,-1.0033197403,0.7750243545,0.6567839384,1.4885655642,-1.0188703537,0.4090184867,-0.0310165752,0.5363311768,1.5438086987,0.6926738024,0.9071796536,-0.1791453212,-0.2743999660,0.0375481956,-0.0360734574,-0.0225227624,0.7798106074,-0.3104155660,-0.5394257307,-0.3660118282,-1.2562537193,0.1709382087,-1.0761003494,-0.0932355300,-0.1811507940,0.5199954510,-0.5508889556,-1.0994771719,-0.4694336057,0.7088356018,-0.0675911307,1.3022845984,-1.4211859703,1.2567162514,-2.3688504696,0.7742468715,-0.4273701310,0.1674273014,-1.3203834295,-1.8911298513,0.5281402469,1.3740572929,0.3885111809,1.0885382891,-1.7151808739,-0.3717588782,0.0876440331,-0.5725683570,0.0155486111,-1.0659922361,0.4667593837,-0.0940362737,-0.4041534066,-0.1656819880,2.3223657608,-0.7614206076,0.9490546584,-1.4844082594,0.7458498478,-0.0060643018,0.7887188792,-1.1180588007,-1.0001761913,0.7314289808,0.2357216030,0.3824612498,-1.7703744173,-0.2727081478,-0.1264617443,0.6515104175,0.5096127987,0.6367074847,-1.0184148550,1.2223472595,1.3987852335,0.7798677087,0.6547874808,-0.3030628264,0.0454207473,-0.3884822726,0.3867644668,-0.9542946815,-1.8427627087,0.3048293889,1.9156039953,1.2040926218,-0.2316040397,-0.5864921212,-0.2658803463,-0.4716253281,-1.5087419748,-0.0129999518,-1.6092796326,0.7645639181,0.7615116835,1.2913290262,-0.4876889288,-0.4697682858,0.3820580542,0.4084323943,-0.6136229038,0.6022604704,-0.3676231503,-0.7209435105,0.5586885810,-0.5583328605,0.4996970892,-2.0316460133,0.0585249402],[-1.0543115139,-1.0559836626,0.3332637548,0.0095748026,1.8420743942,1.0140199661,0.0083231404,-1.5318380594,0.6986145973,0.6373190880,-1.8509548903,0.2487269789,-1.2360033989,-0.3390091062,0.4469988048,-0.1896353811,-0.3229845464,0.4780968726,-1.3126137257,-0.2899254262,-0.1851799935,-0.3376224339,0.4259453416,-0.4082024992,-0.8978323936,-0.6690765023,0.2631308734,0.5530337095,0.1995165348,0.2926838398,0.8894163966,-0.0055672377,0.7018143535,-0.8826447129,0.0749236047,1.2906317711,1.0046035051,1.0409386158,3.0276360512,-0.0133288782,-0.0506551601,-1.8908251524,-0.2714906931,-1.3030624390,-0.2462271601,-1.8104127645,0.1279558837,0.1793313920,-0.7023834586,-1.0055930614,-1.4191739559,-0.4562571943,0.4048222899,1.3166447878,1.1341559887,0.7166735530,0.5539758801,0.4354004264,0.3141281605,0.6505777836,0.2128535211,1.6090195179,0.1628310382,-0.6526899934,1.2286890745,-1.2203204632,0.4666250944,0.7341588736,0.1583673805,0.1684402227,0.1241835728,0.7640827894,1.2957168818,0.7631736398,-0.6601703763,-0.0008463254,0.0685535297,1.1067372561,-0.4385348558,1.7345260382,1.9808120728,-0.6242682338,-0.0815576091,-0.1650884449,0.4348444045,0.5211656094,-1.6433618069,-1.7573856115,-1.4892438650,-0.4256701171,0.2377105355,1.5365910530,0.3241511881,-0.5333830714,0.5189927220,-0.3648990393,-1.9341444969,-0.4795877635,-0.9685363770,0.3621700704,-0.4226962328,1.9258793592,0.5076354742,0.2610789239,-1.2897127867,1.4333382845,-1.6838537455,-0.7709188461,0.8705440760,0.1586957872,0.2956838310,-2.0230813026,1.1002036333,-1.1002752781,-0.5147218108,0.1019927114,-1.0212808847,-0.5737935305,-0.7874208689,0.4350974560,0.1936040074,-1.3571660519,-0.4171037376,-1.6063512564,-0.2483906001,-0.2693999410,-1.5144617558,-0.1226147190,1.5742284060,-0.2891094089,1.2006629705,-0.3703657985,-1.6033035517,-0.4120010436,0.2618357539,0.0800257102,-0.0225607231,1.0052714348,0.2401950359,-0.9681993723,1.6243599653,1.7711733580,-0.4288421571,0.8745658994,-0.7261585593,-0.7740636468,-0.3783999681,1.0468926430,-0.1909528077,2.1233911514,1.6014511585,-1.4648680687,-0.2578746378,0.5449219942,-0.8318146467,0.2432440072,-0.5521076918,0.4936685264,-0.0429933108,-0.7929572463,3.4898121357,0.6794958115,-0.6419728398,-1.2206847668,1.3815455437,-1.5034804344,-1.4729547501,-0.7631402612,1.4460397959,1.1735513210,1.1392821074,-0.3697892427,0.1053928807,0.1785945147,0.1319828779,-1.2402095795,-0.7929094434,0.0239162408,1.2175018787,1.2581635714,-1.1096943617,-0.7467578650,-0.1821890175,1.7982856035,-0.9656202793,0.0464049689,-0.4019823372,0.8537561893,-0.9632617831,-1.3833892345,-0.3336651325,-0.6310629249,-0.6288160682,-0.6705541015,1.1295614243,1.2133406401,-0.1866526157,1.8917940855,-0.2755779922,-0.6542409658,-0.3929320574,-1.2494572401,0.3466140926,-0.5797623396,0.8853974938,-0.2189182490,0.6220045686,-0.3418531418,-0.9100940228,0.8505608439,-0.5887028575,-0.7855650783,-0.3946730494,-0.2303701043,1.3030629158,1.7425899506,0.1629963517,-0.1453389823,-1.2816724777,-0.6563048363,0.0003808588,-0.6638936400,0.8562603593,-0.1051234752],[3.0289349556,-1.4466205835,0.1836807728,-0.5522636771,0.5423675179,2.7219474316,0.8910558820,0.6465635896,-0.0759895593,0.1817708462,0.3457686603,0.7239876390,0.7132585645,-0.4586695135,1.0333595276,-0.2189355940,-1.1411225796,-0.8677021861,-0.9688470960,-0.6339297295,-0.0074916431,0.0354148932,2.4297785759,-0.8822140694,-0.9148740768,0.8378556967,-0.1476698518,0.6026245952,-0.0974400416,-0.6325020790,-0.8441191316,0.5103759170,-0.7266747355,-0.7533026934,1.4277828932,-0.7310283780,-0.0555546284,-0.9097919464,0.9007401466,1.1238573790,0.0252667442,-0.7156252265,-0.2277295440,-2.1477272511,1.5932493210,-1.8628616333,-0.3776150644,-0.1090053990,-1.5743252039,0.9123138785,-1.3972257376,-0.0600600950,0.3769355714,-0.4396723211,2.1093904972,0.3233568668,1.2451854944,0.7502902150,0.8297964334,-0.0762333572,-0.0797017440,-1.1705890894,-1.7427301407,-0.5095266700,0.3880380094,-1.5055037737,1.0071586370,0.5120072365,-1.0015641451,0.6942522526,-1.1639013290,-1.5982449055,1.4353106022,1.3250426054,-0.7664409280,-0.2600101531,-0.8542203307,-0.8446595669,-1.4417269230,-1.6894496679,-0.7739345431,-0.4729284644,0.6903986931,0.5380149484,-0.6231607795,0.1010175794,0.6848499775,-0.6755223274,0.3829651773,1.0227255821,0.0714008585,-0.0293932445,1.8250042200,0.2873611152,1.8083481789,0.5629999042,-0.8379108906,0.4814782143,0.9358680248,0.3034250140,0.4835818708,0.6362583637,-1.9723367691,0.1129239202,-0.9730765224,-1.2520319223,-0.6632204652,-0.5693321228,0.0858491659,-1.6128265858,2.4252493382,-1.2824521065,-1.2813147306,2.0825462341,0.9443973303,0.3961180151,-0.6054111719,-2.1568701267,0.8435489535,1.5585157871,0.8996023536,-0.1726988107,-0.2633880973,-0.3633803427,1.7543785572,-0.5090419054,-0.1032537743,0.2715066075,-0.4920640588,-1.5715212822,-0.9911488891,-0.4591449201,0.1021347195,-0.1885639131,-0.1514699608,1.2429906130,-1.2038488388,0.1201531067,0.6239452362,0.8239920735,-0.8424552679,-2.4820933342,0.0825293586,0.7349550724,1.4615254402,-1.8211261034,0.7355304956,0.0824137405,-0.0821021348,2.4546854496,-0.6104809046,-1.4369820356,0.3347872794,0.0953120217,-0.1711449176,-0.2775689960,0.2559177577,-0.6192325950,-0.0518681258,0.7144017816,0.1674056649,-1.9963171482,-1.2033586502,0.1521248519,2.5265574455,-0.6133159995,-0.3060063124,2.1903588772,-0.0703226253,-0.3838198483,-0.9298690557,2.1403698921,-1.1335151196,1.8761792183,-1.1630308628,0.7195526361,-0.9941395521,0.2390076518,-0.5244604349,0.2172025740,-0.3227881193,0.4246652126,-0.7177217007,-0.1237884983,1.0893511772,0.3389956653,0.6737839580,-0.4500505924,1.9005181789,0.3046309054,1.0528272390,0.6181700826,1.2530003786,0.1073710024,0.6634507179,0.6261904240,0.9524511695,-0.3014968634,1.7174031734,-0.7578988671,-0.4663391113,-1.4782775640,-0.2628045380,0.5235355496,2.8543355465,-0.7934467793,0.2137494832,1.1492207050,0.7139776349,0.2760613859,0.4369187057,-1.4526598454,1.1321744919,0.4283018112,0.6453626752,-0.3679089546,0.8216310740,0.2988089919,-0.6367230415,1.3324908018,-0.2185408473,0.4428556859,-0.9793181419,-2.3876283169],[0.7414361238,-0.5878328085,-0.1686553657,-1.0306122303,-0.5140874386,-0.6933035254,0.8230751157,-1.2295557261,2.2675883770,-0.0020896329,-1.5306205750,-0.2049885243,-0.6176739335,-1.0552675724,-1.0267729759,0.7805772424,1.4680932760,-0.6374821067,-0.0440820456,0.6838778257,0.1650949717,-0.6097522378,-0.9951433539,-1.1233109236,0.2587343454,-0.6517683268,0.6634348631,-0.1877324134,0.0308681466,0.0288206376,0.5415495634,-0.5201122165,0.4827626944,-0.1858599484,-1.3083807230,0.2003160715,1.1300369501,2.4712626934,-0.2974512577,0.7705679536,0.0965008363,-0.8537307978,1.1313229799,-0.4410588145,-0.3641335070,-0.4061614573,0.2904089391,-0.2795744836,-0.6002675891,0.2109877020,-0.0930366442,-0.0394295789,0.2261722982,0.8024736047,-1.8428796530,-0.6576650143,-0.3111714125,0.8684728742,-0.5444998145,-1.9280499220,-0.1636284292,2.2250468731,-0.1079393625,0.6490951180,-0.6822468638,-0.1159646735,0.9660808444,-1.0916345119,1.2806700468,-0.6005136371,0.4279638827,1.6419439316,-0.7441155314,-0.5071685314,0.2477568686,-0.6498341560,0.8419561386,-0.7465654016,0.8345745802,0.7719214559,0.0275003444,-0.8203216791,0.7621380091,1.8413269520,1.9317811728,0.7190065980,1.2122613192,-0.5506513715,1.2079052925,-0.5110875964,-0.9887308478,-1.4687793255,0.7604703307,0.8583361506,-1.1693745852,0.9667693973,-1.0014497042,0.9113854766,-0.8501291275,0.1033337414,-0.2851611078,0.4838745594,0.0306431390,-0.9261587858,0.5031406879,-0.7204946280,0.0854555964,0.5342746973,-0.2392912656,-0.3483181894,-0.7283043265,-0.1952078938,-0.2268493474,-0.5073602796,0.4071733952,1.7896782160,-0.7856964469,0.1932238191,-0.8698413372,-0.2894476950,-0.4371841550,0.6789008379,-0.5095900297,-1.1314054728,-0.9796065688,-0.9307839274,-0.2411198616,-1.8260837793,-2.1603901386,0.7138137221,0.4682048559,1.8073990345,0.3030933738,-1.5624271631,-0.1223475486,-1.4457436800,-0.1992979646,-0.4670667946,0.5985659361,0.0050599673,1.0435233116,-0.3494057655,-0.1419804990,1.0799071789,-1.1984112263,-1.6463519335,0.9736518264,0.2760215700,0.4218560159,0.2312800139,0.0733595192,1.0696476698,-1.1329557896,-0.2230795324,1.1043237448,0.3311566710,-0.1949632615,0.4872407019,-0.5673235059,0.3815425336,-0.5976793766,0.3133822381,1.1188403368,0.1023709923,-0.7640777826,0.5128422379,-0.9548774958,-0.7417376637,0.8232577443,0.6041470766,-1.5107276440,-0.6352846622,-1.6938943863,-1.4230499268,0.8208574057,-0.1627612412,-0.3808040321,2.1364395618,-0.9052449465,-1.3772284985,-1.6807270050,-0.5424402356,0.9703767300,2.0392134190,0.0691738948,-0.1845367402,1.0785111189,2.0295569897,-1.2068397999,-0.4048538506,0.3063526452,0.0183172151,-0.7925714850,-2.4151265621,-0.5461390615,0.5046216846,-1.1571860313,-1.0568937063,0.0951267779,-0.2945894301,0.6409581900,-0.5636038184,-1.2628661394,-0.5963248610,1.0266063213,-0.1695082039,-0.8970633745,-1.5442864895,-0.6414470077,-0.6634951234,0.0513509884,0.4815643728,1.3058356047,0.4562318921,-1.1240118742,0.8064363599,-0.2292824686,-0.9644187689,-0.8825575113,0.4957913756,0.4160384238,0.7359398603,-1.0344359875,0.6110323668],[0.0616292097,-0.8699955344,0.9175645113,0.5920697451,0.2377380133,0.1208677292,0.1229188591,-1.1193079948,-0.5608552098,-1.0286526680,1.4120683670,0.2869732082,-0.7669487000,-0.4248317182,-1.6202235222,1.1397325993,-1.1340955496,1.7128037214,-0.0321023352,2.4242587090,-0.3608433008,-0.3228870332,1.5381108522,-0.1577602327,1.7750029564,-0.7555779815,-0.2008749545,-0.7190867066,-2.0994808674,0.6139115691,-1.2123483419,0.8159266114,-1.8295583725,-3.5421888828,-0.3941495419,-1.0236091614,0.2525290549,0.5810935497,-0.1076203361,0.4633731544,-0.4760022163,-0.0666669309,0.9731261134,-1.4544069767,0.0888629779,1.7643368244,-0.1727118343,0.5579123497,-1.1497603655,-0.3366352916,0.8751723170,-1.4169456959,-0.8309580684,0.3659218550,-0.3941937983,-0.3234885931,-2.1415123940,-0.2322276533,1.7834504843,-0.4405650198,-0.1744079888,-0.4390559494,-0.8015988469,-1.1638343334,-0.5664111972,0.2518135309,-0.6597012281,0.2770966589,-1.0669745207,0.2963372171,-0.0308781285,-0.3477644324,-0.8671655655,-1.2455052137,1.2770495415,0.1706280708,0.1064133495,-0.7048476338,0.6347349286,-0.7335917354,-0.8754142523,1.1479344368,1.5040304661,1.3156964779,-0.5855301023,-0.1134957299,-1.0665155649,0.2300466001,0.9959614873,-1.3386521339,0.4310475886,0.4835477769,-0.5265149474,1.6178389788,-0.0525867380,-0.9931297302,-0.2980194092,-0.8178262711,-0.7970831394,-0.5230959058,-0.2952790558,0.1519235820,-0.5994967222,0.8724045157,1.0875824690,-1.2039092779,1.0752569437,-1.4313342571,-0.0718548223,0.5625308156,-1.0406832695,-0.5584868789,0.6927548647,1.2421534061,0.2440869510,-0.3585351706,0.6971446276,-0.0839696378,0.1699494272,1.3358668089,1.2711428404,-0.5545876026,-1.3880506754,2.2152044773,-0.8038774133,0.5055804849,1.7750318050,-0.0108200340,-0.4221525490,-0.0152974008,-0.7703688145,0.0354482979,0.3172935545,0.6298582554,-0.8587480187,-1.3135603666,-0.5014687181,1.8155835867,-0.6328458190,-0.3373099566,-1.5931397676,-0.2987871766,2.0653538704,-0.3638643920,-0.9729212523,0.7749390006,-0.7497231364,-1.8904943466,0.8607858419,0.1471386701,-1.5295069218,0.5777686834,0.8261218667,-1.0119875669,0.0897994936,-0.4430561960,0.0323836058,0.2219475061,-1.2818908691,-0.2708592713,1.4573109150,0.8804024458,-1.5916829109,1.0492521524,-1.1633678675,0.8123183846,0.4321608245,-0.8351872563,-1.6270632744,0.5444349051,1.7567660809,-1.4698419571,-1.3057603836,0.6106351018,-0.6856634617,-0.0027157622,0.7870089412,-0.1850146800,2.8850343227,1.5643398762,-1.0507323742,0.6868153811,1.1048454046,0.1835930198,0.5201048851,0.7623929977,-1.4696066380,0.0583677441,0.4692377448,-0.2277192771,0.9734441638,-0.0795424357,-0.2654288709,-2.2028191090,-0.2876582742,-0.0065556336,-0.0743332133,0.5298947096,-1.4907051325,-1.0873161554,-2.1639416218,-0.4339273572,0.3658332825,-0.2206286639,-2.2076971531,-0.4790238142,-1.8818588257,0.4069420695,0.2041137069,-0.2685712576,2.1940827370,0.3173443079,-0.5103394389,-0.3893362880,-0.4261973202,1.2880473137,1.3300783634,1.0021003485,-1.7409325838,-0.1299262196,1.4271765947,0.1755771786,-0.1361971200,2.6091077328],[-2.1213953495,-1.4876091480,0.4027919471,-0.7992261648,0.8391885757,0.6907860637,0.0751595125,1.9657648802,0.5034283400,-0.5381843448,2.5232529640,-2.2388384342,0.5314210057,2.4042863846,-0.1817212552,0.4993635118,0.2077349424,0.8697637320,0.4360201657,2.3369359970,0.2676490843,1.1903718710,0.8799911141,-1.1188682318,-0.9714317918,-0.9767854810,-1.3006596565,-0.2873510718,1.4437073469,0.9303485155,-0.3495375216,1.2017552853,-0.9662505388,-0.7631691694,1.6332331896,0.8243235946,-0.4884332120,-0.0201347098,-0.7768230438,-0.7631254792,-1.2737504244,0.4627576768,0.1729757488,-0.3228688538,-2.4766850471,-1.1090526581,-0.6864037514,-0.8046090007,-0.7138791084,1.6377116442,0.6567335725,-0.0047580036,-2.2416980267,0.1491056532,-0.0847144574,0.2720263004,-1.6646059752,-1.1872817278,-1.5886445045,-0.2330412865,0.2284714878,-0.9844371080,-0.1336304843,-0.5829446912,-0.2540107667,2.2281854153,-0.5904042721,-0.5926827192,-1.3128408194,0.1390305161,-0.1672947854,-0.7939913869,-0.6221416593,-1.5166251659,-0.1779445559,0.0959212556,-1.6828132868,-0.7401673198,0.5231422782,-0.3322941959,-0.1783479750,0.4485481381,-0.3100477755,0.0970409885,1.1743543148,-1.8261427879,0.4978103042,-0.0108789131,0.5395866036,3.3376805782,0.5630101562,-0.4743687510,-1.9228357077,-0.0009492107,1.3383150101,1.5428452492,-0.2962970436,-0.5699378848,-0.4364693761,0.2970700264,-0.5584040284,-1.9995604753,-0.7018766999,0.7856997252,1.6371961832,-1.1782302856,0.5775887966,1.2874151468,0.6118171215,-0.5430070162,1.0252311230,1.9605115652,1.1515276432,-1.1050601006,-0.3522197604,0.3429833055,1.9247678518,0.1838604361,-0.4415161908,-0.7715782523,0.1724594831,0.7692704201,-0.5029193759,-0.2041465640,0.6117989421,0.2968843579,-0.9996942282,1.7067486048,0.6819940209,-0.6585046053,-0.1444742084,-1.0080267191,1.4498666525,0.3252133429,-0.8631876707,-0.0172087178,-0.9234526753,-0.1951841712,0.8916925788,-0.4279014468,-0.4757214785,1.1231888533,0.5568086505,0.5508191586,-0.1973502934,-1.0614956617,-0.9130429029,-1.2311202288,-0.1490184516,0.6185660958,-1.0488156080,0.4432644546,0.8579077125,0.6526910067,1.9192267656,0.6657885313,-2.7699515820,-0.8622987866,-1.5329411030,0.5477091670,-0.3008966446,0.4089683890,-0.2756463885,0.2297792137,0.7346068621,-1.2895530462,-0.6784411073,-1.2107373476,-0.4332395196,-0.0472933054,-2.2021653652,-0.2635502517,-0.7713882327,0.9504997134,0.5086578131,0.8237007260,0.6284089088,2.4425268173,0.6307875514,2.4813482761,0.1044523641,0.6090664268,1.5078846216,0.6996105909,0.5948370099,0.2660599947,-0.2112916857,1.5669039488,0.9084113836,-0.6771945953,-0.1638096422,-0.4631693959,1.1034935713,-0.1055989787,0.0793656483,0.6271271110,0.7019786835,0.3838609755,-1.0738798380,-0.6967238188,-1.4907166958,1.2653003931,2.0242471695,-0.3331431746,1.3018358946,0.2629842162,0.0544529743,0.5886738896,0.7199140191,-0.2402157187,-0.3608579338,0.7160065770,1.2522007227,-0.8780410290,1.2056716681,-1.2108603716,-0.2572793365,-1.8514751196,-0.1546487361,1.5449539423,-0.2368666679,0.0098899864,-1.0413780212,0.2774117887],[-0.2326158434,1.7425477505,0.9690331817,1.6429095268,-0.5832452178,-0.1323042661,-1.0757516623,0.5345213413,-0.3577066958,-1.2440371513,-1.1071386337,1.0463796854,1.0088303089,-0.0133500062,-0.3407829702,-1.4332808256,1.2677539587,-0.6842421889,-1.5895107985,0.1031334102,0.3260625005,-0.8146274686,-0.5703064799,1.5360972881,1.1497735977,-1.0545229912,1.1621849537,-0.0205927342,0.0773499012,1.5707740784,2.3440666199,-0.0833226591,1.2771021128,0.6326636076,0.3524564505,1.0547939539,1.2561269999,0.9254574180,-0.4101853073,0.7814167142,0.5550388098,-1.8132902384,0.5165917873,-1.6028478146,0.1231929883,-0.8797899485,-2.0823435783,0.3851499259,-1.3730741739,0.3431421518,-1.2870882750,-1.0696678162,-1.9551162720,2.1060843468,0.5244371295,-1.8513286114,-1.8891903162,1.6993538141,0.4485278726,0.8354133368,-0.3720458746,-0.5648186207,0.2110529244,-0.4747396111,-0.7716717124,-1.2227087021,-0.3932309449,-1.0957214832,-0.3843332529,-0.4385857284,-0.2289460748,0.1591261774,1.8326412439,0.3320563436,-0.3171687424,1.8090025187,1.0536510944,1.5990201235,0.6480306983,0.9056814909,1.0043029785,-0.2869877517,0.1809553057,0.9514772296,-0.0422179550,0.7730164528,0.5366948843,-1.3359624147,-1.2184035778,0.5566676259,-1.3658822775,-0.4281991422,-1.7386028767,0.4616151452,-1.1900107861,-2.4713108540,-0.4954342842,0.2345984131,-1.8488981724,-2.1743729115,-0.6402022839,0.0303491205,-0.7284300923,1.9178283215,1.6713473797,0.4037494957,0.2346402109,2.0459208488,0.3015230596,0.7130191326,-0.6341357231,1.4106793404,0.2356214225,0.1327618361,-0.2230495960,-0.4091452658,0.2005991936,-0.0833660811,-0.8700286746,2.7954144478,1.7317944765,-0.6571676135,0.6018900871,-0.0520828702,0.3507285714,-0.5154905915,-0.1296486408,-0.5092809200,1.2773561478,1.0116991997,-0.6150847673,-1.4773579836,0.7563268542,0.8471134305,-0.4720822871,0.4313528836,0.0375151336,0.8350439072,0.4294562638,-0.3305866718,0.4861770868,0.6976563931,0.6272200346,-1.6637167931,-0.2475887835,1.0828968287,1.9375406504,0.0171681941,-0.2749762535,-0.7067273855,-1.8384919167,-0.6727235913,0.4818950891,0.8734824061,1.1389626265,-1.3718824387,1.0223542452,0.4262120128,0.2025973946,0.1423050314,1.4298902750,-1.3199853897,0.4568197131,1.5345735550,0.5935459733,-0.4387113154,-0.1879886091,-0.3233093619,0.4590752423,-1.5666143894,-2.3096513748,-0.8251853585,-1.0219471455,-0.0014929610,0.4029170573,1.0825065374,1.4356482029,0.6444023252,-0.1995886266,-0.2893678248,-0.0991581753,-0.1651691198,1.0189162493,-0.1947890669,-1.2835777998,-1.7958240509,1.9413176775,1.4828227758,-0.8889564276,-0.5395320058,-0.5694774985,0.0775188878,-0.7718951106,0.8657597899,-1.5139957666,-1.2930766344,-1.7993720770,-0.2752868831,0.9181541800,-0.7155758739,1.0465465784,1.2638782263,2.2021529675,-0.9961844683,0.0974123031,0.0123674851,-0.2089405060,0.9067828655,1.2267822027,0.3396924734,0.1184439957,2.0369150639,0.3253271878,1.8651583195,1.4540983438,0.5370510817,-0.7455057502,0.2601398528,1.7201685905,-1.7035471201,-0.2939456701,-0.3423474729,0.9926267862,-0.7185393572],[1.6899878979,0.0963688120,1.5225541592,1.2421973944,1.1041691303,1.5041767359,0.4050821960,0.5243073106,-0.1115907133,-0.1680535376,-0.0969657302,0.0803446621,1.2012168169,-0.4391567707,1.2089624405,-0.7686284184,0.2084608078,0.5243440270,-1.6587768793,0.4990506470,-0.6774152517,-0.4325406551,0.6804471016,-1.4003276825,-0.1406010389,-0.6582599282,1.6774594784,-0.6680455804,0.2592495978,0.8286221027,-0.5804581642,0.2537096143,0.9645189643,-0.3157314360,0.5982891917,0.8521072268,0.5036994219,0.0840232447,-0.1938900799,0.0138512980,-0.2024308294,-1.0314016342,0.0266645644,-0.6234315038,2.1404976845,-0.3796528280,0.8915473223,-0.7924473882,-0.2630101442,1.2968854904,1.0652110577,0.4945122898,0.4680919051,0.4252710938,0.0305738803,-0.7643531561,1.3036431074,0.1892702878,1.0127239227,-0.4154120982,0.1690457165,0.2398145199,-0.2753032148,-0.0394092537,-0.1768219471,-0.6261203885,-0.4798114896,2.0902595520,2.4536385536,0.2569797635,-1.3700937033,0.2429075092,-0.0302436706,0.0957037061,-0.5650098920,0.4821854830,1.5858329535,-0.3087330759,0.0644669831,-0.0620031543,-0.0495426171,0.6991023421,-0.1768588275,0.2576231658,1.0498266220,1.1218752861,2.0640258789,0.2962799668,-0.5372537971,0.0778273121,-0.6351861358,1.9494534731,-0.7365251184,0.2154163718,-0.8799818158,1.1496148109,0.3462239504,-0.7713207006,2.7176063061,0.6472253799,1.5203940868,0.0201782063,1.1566685438,-0.6539087892,0.2394919395,-1.7418583632,-1.1859192848,1.3754141331,0.5204149485,-0.1954313666,0.4072748423,1.2499068975,-0.2092721015,1.2599943876,0.4698984027,-0.1057309136,0.6285521388,-0.1283890307,0.2182361186,0.5527539253,-0.5691096783,1.1039172411,-0.0255232845,1.2810142040,-0.3833185732,0.1088434681,-0.0079321684,-0.6392134428,-0.3211532533,-0.6840104461,2.2359540462,-0.7377978563,-2.1295697689,-0.8262901306,-0.2741989493,-0.7584269047,0.2667566240,-0.1170935258,1.2125899792,1.3901118040,-0.1025477350,-0.5680760145,0.5732792020,-0.6226810813,1.0726773739,-0.7555589080,1.3011289835,-0.7514100075,-0.3121113777,0.1953123808,-0.7970280051,-0.0469507203,-1.1711391211,-0.5039377809,-0.1742249131,1.7650339603,1.0166648626,-2.3796470165,2.1885800362,0.6374771595,-1.4910366535,0.5055052042,0.2328806818,0.8630269766,-2.2999992371,-1.5238486528,0.5972530246,0.1055067182,-0.5324211717,-0.5370553136,-1.2598272562,1.6097830534,-0.7878819704,-1.7461416721,-2.0719652176,-1.1854764223,0.3427107036,0.5295251608,1.7611126900,0.9704126120,0.5100552440,-0.6306912303,-0.4613394737,0.9809895158,-1.7130533457,-0.7521519661,0.1366459578,0.5631435513,-0.1118881255,-2.4540958405,-0.0765665099,0.7642238140,0.6514412165,-0.2037922889,0.9903054237,0.3785459697,-1.6247358322,1.2173683643,0.2166759223,-0.8612121940,0.4859002531,-1.0364907980,-0.4721792936,-1.6349135637,-1.4085305929,-0.8389165998,1.2321178913,1.3330185413,0.3066735268,0.9116269946,0.6791136265,2.8482429981,-0.4379735291,-2.2370553017,-1.5435577631,-0.1815133095,1.2109670639,0.5453848839,0.4780768752,1.5647675991,2.2001993656,2.1024439335,0.8078509569,-0.3416246474],[-0.6584426761,-0.1349043101,0.3547728360,-2.2675919533,-0.2931696475,1.0089668036,-0.6153686643,0.1034570783,0.6448723078,-0.8765313029,0.4534460008,-0.3699859381,1.2004622221,-0.0722925514,-0.0325831361,0.9020727873,-0.7152183652,-0.8456809521,1.1576007605,0.6171180010,-0.0472845919,-0.9105609059,0.1960259825,-0.2576677501,-1.9657860994,0.9828581214,-0.9305165410,0.8664217591,1.2800661325,-1.1278365850,1.8148369789,0.2521477938,0.8906397820,1.5773588419,0.4439918995,0.0060209022,0.8361096978,0.4923763573,-1.1642525196,0.7681359649,0.2648225427,-1.1226508617,-0.2650801241,-1.6352653503,-1.3773202896,0.6504764557,-0.9845471978,0.5976469517,-0.0603680871,0.3551152945,0.9604079723,1.4429877996,-0.9509329796,0.4106124043,-0.8905282617,1.4084507227,-1.2862893343,1.1871342659,-0.2408435643,0.7524929643,0.9669978023,-0.0317020342,0.1478836387,0.4667394161,-1.1262389421,0.4174720943,-0.4278752208,0.0893254876,0.3123355210,-0.6510843039,3.6935560703,0.3481149971,0.9524300694,1.2629102468,0.9691338539,0.8425455093,0.4645085037,0.8258065581,-0.2780044377,1.0713196993,-0.7075175047,-0.1032134444,0.5142312646,0.5319851041,-0.0098116705,-0.1779374182,-0.2449193746,0.8509523869,-0.6144673228,-0.4817785025,-1.2756514549,-0.6126600504,-1.1293950081,-0.1908688098,1.0651584864,0.3274653852,-0.8781397343,-1.2023971081,0.5885425806,0.7167418599,0.0696014911,1.5573089123,-0.5623143911,0.3726352453,-1.0666670799,0.7341268659,-0.7441030145,-0.5995749831,0.4551900327,-0.1493398994,-0.0821552277,0.1749963611,-0.5736764073,0.6820808053,0.1590649784,1.7094728947,-1.2093794346,0.1417449266,0.0874756873,0.0445520170,0.1454997957,0.6226649880,-0.7579197288,-0.8653601408,-0.8565465808,0.0884546340,-1.1490837336,1.2844370604,1.7234747410,1.4080694914,-0.3957767785,-2.0336396694,4.2858557701,0.1105877906,-1.0418227911,1.2223359346,-1.1886203289,0.1480172127,-2.7951111794,-0.5097351670,-1.1575583220,0.9877367616,0.2623551488,1.2971171141,-0.9021095634,-0.7333549261,0.3958832324,0.0376789942,-1.4897017479,0.7928319573,0.3467888832,-0.2384968996,0.5339655876,1.0606970787,0.6961131096,0.5275928378,-1.0800560713,-0.6828938127,0.7291389704,0.4540881515,-0.9229554534,-0.2239529192,-0.8285161257,-0.7912678123,-0.0725147203,-0.1320558935,1.2266693115,-0.1467476189,-0.5098367929,0.1437175274,0.0615985021,1.1454741955,-1.6718358994,0.2883616388,-1.0447431803,1.8523039818,0.0715519041,1.2537438869,-2.2294402122,-0.9726954103,-0.8541699648,-1.2510904074,-0.2680608928,-0.0982486233,-2.7065048218,0.4315496087,0.4952709377,2.3173794746,1.7902907133,1.0368921757,-0.4934299886,-0.6587244868,0.2156995833,-0.7816968560,0.1607544422,-1.4151940346,-0.4447843432,1.6773577929,0.3867525756,1.1128669977,1.8861371279,0.9591953158,2.4224495888,0.8388727307,-0.8159924150,1.5163911581,0.6435745358,0.4092240930,1.7183226347,-0.0800361559,-1.0487719774,-0.2141771913,1.0178449154,0.7592806816,0.5728220344,0.6055765152,-0.8523482084,-0.8808542490,-0.4607036412,0.7181840539,0.5790495276,-0.3860639036,-0.6235859394,-0.3401636779],[-1.3322459459,-0.3735280037,-0.9619418383,0.9170179963,-0.0155261103,-0.7936813235,0.2904969156,-0.2934159935,1.5998722315,0.2245161682,-0.6810392141,-0.7174949050,-0.0229964331,-0.6433348060,0.8996737599,-0.1399249732,-0.9068041444,1.2186594009,-0.4671718776,-0.0194043443,-0.5684084296,-0.8678084612,0.5519516468,-0.9348483086,0.4797087014,-1.7733447552,0.5873536468,-2.0052974224,-2.0336976051,-0.2158455700,-0.2398948669,-1.9766912460,-1.7725622654,0.7912243605,0.6274369359,0.0156383421,-0.4388458133,-0.2509407997,-0.7350332141,0.0592636503,-0.2649568617,-1.1821020842,0.1051000580,-0.4178300500,-3.0862278938,2.1433157921,1.8003720045,-1.3339478970,-0.4993157387,-0.0595764108,-0.0815746859,-1.8172090054,0.9544544816,0.0266386792,-0.7247435451,0.4710936844,-0.0355378725,0.5382333994,-0.2017002255,-0.2736326456,-1.0850501060,0.4361391664,0.7275456190,-1.5133862495,-1.0869604349,-0.4892189503,-0.2241794765,-0.7801714540,-1.8050248623,0.4219087958,0.0985516831,0.7518414855,0.0099756178,-1.3204857111,-0.7362210155,-1.3625191450,1.4659570456,1.5852022171,0.9222944379,-0.6225005388,-0.4121412635,-0.1012708023,1.0839953423,-0.1868243515,1.3171703815,0.7383409142,0.7114495635,-1.3378200531,0.4559955299,-1.0297137499,-1.0663639307,1.8599836826,0.2518027425,0.3776589930,1.6198328733,-0.1385766864,0.0779930502,-0.9513822794,1.6096773148,-1.1503460407,0.2158631533,0.9789539576,-1.6471439600,0.3016359508,1.4386115074,0.7306500077,-1.8357973099,0.6295026541,-0.4132591188,-1.5983790159,-0.0675861537,-1.8248832226,0.5449636579,0.6511113644,1.1925294399,1.4464730024,-0.6434930563,2.7133228779,0.9117081761,-0.4531750381,-1.2206915617,-0.3705402911,1.2997782230,-0.8172878623,1.0327092409,-1.2872046232,0.3879989088,-0.3668969572,1.4443743229,0.0700865909,0.1676746309,1.4242259264,-0.8452426195,-0.4409632683,-1.3721363544,0.9033271670,1.2147731781,-0.7221063972,0.3519815505,-1.0255091190,0.2319916338,1.2535445690,0.1925023496,-1.2753477097,-1.1186707020,0.8844894767,0.0683436990,-1.2587802410,0.2794378698,0.8379051685,0.2669829130,0.3449877799,2.3400497437,1.0804201365,1.4391826391,0.7339784503,-0.8790752292,0.0026909518,-0.8914278746,0.4804829061,1.4641373158,0.6813607216,0.8553986549,1.7090407610,-0.5028632283,1.4325996637,0.2645716369,1.0826915503,-0.0516118817,-0.3066533506,1.0699689388,0.1768194139,-1.2282488346,0.2069570571,1.3318647146,-0.9424896240,-1.0964196920,-0.4572708905,0.4500499368,0.1980328262,-1.9181954861,-0.0174330324,0.0999595150,0.0267650168,-0.5508563519,1.4106292725,-0.2244167775,0.6556907296,-0.0054974109,-0.5443715453,-0.1919281334,0.3812180161,-0.4736113846,0.6195918322,0.7323521972,0.9373790622,-2.6295375824,0.3766013682,0.5224608183,-0.8840683699,-0.7412922978,-0.5194513202,-1.3004968166,-0.0051153982,-0.7945897579,-0.1301139295,-0.0011560072,-1.0379506350,-0.6288818121,-0.0096841361,-0.2014763355,0.6976454854,0.1354101002,1.5714323521,1.0308767557,-1.3190690279,0.8940302134,-0.5858420730,2.8178141117,-2.2886848450,-0.1681453437,0.5785136223,-0.8217912912,0.3408507407],[-0.6147227287,-1.1310694218,0.9431957006,-0.6122029424,-0.6093533635,-0.2190532982,0.2843426764,0.6691458821,-0.8678124547,1.5971509218,-0.1540083289,-2.2375392914,-0.5721747875,-0.3298413455,-1.0728195906,0.5367963910,2.1423006058,1.4925087690,-0.1332240850,0.7596751451,-0.9900237322,1.6975897551,-1.1039967537,-0.2365738451,-0.8477472663,0.5158218741,0.9538124800,-0.1393971145,0.2098560929,1.4214780331,-0.6133620739,-0.3443203270,0.7987253070,0.2965416312,-0.6252503395,0.0912849978,0.3837498128,-1.9019155502,-0.9714212418,0.8765338659,0.2010715902,-0.3521747291,1.3726044893,-0.2925174534,-0.6309738755,-0.1950662732,-0.5796111822,0.2951392829,0.3845759332,-0.1556660980,-0.2773683071,0.2328125387,0.7338165045,-0.1784263700,0.1823998541,0.7299521565,0.1716358215,-0.0308698844,0.2138171941,0.3089756966,0.7322928309,-1.0055240393,-0.4655968249,1.4510535002,0.8053041101,0.8307817578,0.5806190372,1.4012625217,0.4137353301,2.2663216591,0.9053751230,0.2676714361,-0.2638470531,-1.1524258852,-0.2055503577,2.0920567513,-0.0389869474,-0.5045543909,-0.1704713106,1.7036591768,2.3972051144,-0.3624917269,-0.0865611285,1.2785141468,0.6021286249,0.2753924131,-1.7304373980,-0.9938117862,0.2796024084,-0.3657841682,-0.0245826654,1.2244658470,0.2047881931,-0.3773700595,0.8032777905,-1.9749366045,-0.1891875565,-0.0975503027,-0.0583613701,0.5487793684,0.3212392628,0.7469420433,0.4802857339,-0.0953975916,-0.0016161953,0.4400027096,-0.3889231980,-0.2829461396,-1.2563219070,0.6948705316,-0.8324922323,-0.4046234488,-1.0711333752,1.3630745411,-0.4418192506,-1.3101440668,0.5938716531,1.8128287792,-1.2612177134,1.0351736546,-1.6114523411,1.9638142586,-1.2056689262,0.3683494329,-0.4289750457,0.5871503949,-0.4308027625,-0.4693752527,0.4237685800,1.0735077858,-0.4528256953,-0.4611294568,-0.4493314326,0.2348384559,1.4725769758,0.8286721110,-0.8574910164,1.1002789736,0.2178244442,-0.0682177767,-1.1096816063,-0.8833054900,1.1900689602,0.8638876677,-0.7919912934,-1.1155849695,-0.0368701331,-0.8000605702,0.0575532131,-0.6001101732,1.7475066185,-0.1024022102,0.7054311037,0.8884329200,1.4535988569,0.8446703553,0.6319045424,2.1480684280,-0.3688235581,0.6120865345,0.6029073000,-0.9015443325,1.8531484604,2.4086322784,0.5047215819,-0.4523896277,0.0560955927,0.0015819527,0.7949462533,-0.4127202928,-1.2027662992,-0.4916600585,0.9999223351,-0.6061544418,-1.1319777966,1.1700640917,0.6804245114,1.5015945435,2.0653812885,0.0969319269,0.2641748488,-0.0362505354,0.6237525344,-0.8102051616,-1.5553128719,0.2032443285,0.2953858972,0.5768323541,0.1966675222,0.9187663794,-0.9852730632,-1.7754678726,0.5656357408,1.5518661737,0.1246112362,0.7218389511,1.1248434782,0.7300990820,-1.7761151791,0.4202294946,-0.6030195355,0.4817877710,-0.3983127773,-1.2921808958,-0.7133482099,-0.0695136040,0.0268135220,-1.8523842096,0.3018036783,0.5316495299,-0.1636061519,0.3650557697,-0.2680281103,0.0313414261,-0.6150335670,-2.5063366890,0.8658521771,-0.8896884918,-1.4359755516,0.7071300149,-0.8117466569,-0.8193208575,-1.6796098948,0.2288485318],[1.1684442759,1.3867622614,0.3720430732,-2.2209444046,1.0519523621,-0.3739921451,1.1328148842,0.4190898538,0.5840306878,0.3622587621,1.3552360535,-0.6779456139,-0.4645721912,0.7942563891,2.9545433521,0.1648689657,0.2273791283,0.8398232460,0.1645840257,1.2705302238,1.9689812660,0.5604508519,0.2973692119,-0.3421580791,-0.7549017072,-0.9641358852,-0.0671323538,-0.2883839011,0.9323253036,-0.4751592577,0.2069311291,-0.9538527727,-1.8852221966,-0.1730320305,0.0738568977,0.7191287875,-0.1032047048,-0.5027757883,-0.6713217497,1.1065231562,-0.4910871089,0.3194410801,-0.3272990882,-0.5864269733,-1.6019154787,-1.2150155306,-0.4830319285,-2.2106018066,0.5272894502,1.1339933872,-1.6582157612,0.1045286432,0.9769923687,0.6356340647,-0.0837953761,0.8529375196,-1.1835395098,-0.0311263781,0.1229125187,0.1845661104,1.4570873976,-1.0240747929,0.2515321076,-1.5033500195,0.1378750950,-1.1685484648,-0.4524605870,1.0256450176,-1.7534796000,-0.7314636707,-0.9282721281,-0.0200786255,0.3444993198,1.2372258902,0.2347012162,0.0017177498,0.7588968277,-0.8208673596,-2.9165871143,-0.5337390900,1.1897857189,-0.4590893388,1.1943334341,0.6033994555,-0.9690102935,-0.4756906927,-0.5146775246,-0.6181944609,-0.4739496410,-0.8306716084,0.7850691080,1.0141893625,-0.1255545616,3.0548119545,-0.5503615737,0.7137859464,-1.8925294876,-1.6286408901,0.1578745246,0.2897020578,1.1416177750,1.6445769072,0.0266005080,-0.6002950668,1.8851222992,0.5006518960,0.8041281700,-0.4171819687,0.9126561284,-0.9665496349,-3.0324594975,0.1772412509,-0.7743625641,-0.7814670205,0.5761278868,0.1479105800,-0.3336394429,0.4467318654,-1.1148625612,-0.7398648262,-0.1982903183,0.2771024704,0.0037587737,-0.7214294672,-0.7772885561,0.3695692420,-0.8413255215,0.4420565069,-0.3661336303,0.2388688028,0.7574204803,1.8069114685,-0.3563021123,-0.6996699572,0.4083276093,0.9007179141,0.9037804604,-0.1237754449,1.1282645464,-0.3496119678,0.7314071059,1.5345368385,-0.2547729909,-0.8683311343,1.1891893148,1.1465867758,1.3951969147,-0.3881577253,0.5991099477,1.2029726505,1.2379013300,0.1110376492,-0.5402413607,-0.4763735831,-0.6523308754,-0.4530446529,-1.2403632402,-0.4820930958,1.9094609022,-0.8305430412,-0.5602543354,-0.3380995393,1.2139499187,-0.6730381250,1.8736811876,0.5619761348,1.2849764824,-0.7612153888,0.6084617972,0.5260947943,-0.3612978756,-0.0199031364,-0.0179262515,2.7262132168,0.1001558080,0.0873316899,0.4253403842,-0.2572331727,0.3998314738,0.5780014396,0.5289971232,1.1858935356,1.5703359842,0.6258378029,-1.3216575384,-0.3723444939,-0.7417381406,1.0053710938,0.2518371046,-2.2449259758,0.7364364266,-0.7436603308,0.2557405531,-2.1114578247,1.4893753529,-0.1315749884,0.0937679037,0.3575634956,1.3543114662,-0.5280987620,0.0921488330,1.1167320013,-0.1133646667,1.3830109835,-0.3453722000,-0.0628339872,-0.6401051283,-0.6774563193,0.0201832801,-1.3557044268,-1.7727380991,-1.3186904192,-0.8905112147,0.6380245686,-1.9156514406,-0.6751238108,-2.3304250240,-0.0731968582,2.6341967583,-0.1446866840,0.9468255043,0.1266835779,0.5929878354,0.2527090013],[-0.7906475663,-0.1088170782,-0.0180691257,1.7883660793,-0.7638135552,-0.3626271486,0.7353001237,-1.9343894720,0.9877597690,-1.2362815142,0.3096195161,-1.3171862364,-1.2322047949,1.4253273010,0.0599365123,0.4724906981,2.3247697353,0.3512053192,0.4358356893,-0.2270360738,-2.0736320019,2.1042101383,0.6262249351,2.0178065300,-0.4194451571,0.1140522435,1.6493346691,0.5421503186,0.4119198918,1.8837475777,-0.1114307493,-0.4739866257,-0.1118430570,0.3266671002,1.0846471786,-0.7667149305,-0.3911579847,0.7709604502,0.9599793553,-0.6258073449,0.4235780239,0.2291588485,-0.5419658422,-1.3261935711,1.4792988300,-0.5172199011,-1.8417431116,-0.9530127645,-0.5386762023,0.5477579832,-1.3143244982,1.2823202610,-0.8449356556,1.7781059742,0.2892118096,-0.6741756797,-0.2984076440,0.3891500831,0.0456696413,-0.7156158090,0.1342495084,-1.8112068176,0.9137851596,0.0664710850,-1.2650182247,0.2943314910,0.2052179128,-0.0621892735,1.0672193766,0.6473845840,0.4740798473,-2.3851220608,0.1769553572,-0.8922551274,0.3362334967,0.9756082892,1.1115776300,-1.6051492691,0.3320700526,-0.5360628963,-0.4085548818,-0.1155323014,-0.9104030132,-1.3911604881,-0.6406959891,-0.7174149752,0.2314108014,0.9844489694,-0.7310201526,-0.7337293625,0.1070518643,0.1595049948,-0.2646666467,-0.6786507368,-0.1914107949,1.4713796377,-0.9440070391,1.6280828714,-0.9151198268,-0.3460160792,0.0264972281,1.2194769382,0.9777076244,-0.9459745288,0.2136550546,-1.1340179443,0.6967772245,-1.0132279396,-1.6084231138,-1.2107805014,-0.7951170802,-0.9152131081,-1.5939664841,0.2302491367,-1.0035600662,0.5383255482,0.3087547123,0.0042029740,0.5027353168,0.4750975668,0.0975661203,1.3311247826,-0.4950006306,-2.6560649872,-0.9025616050,0.8317630887,-2.5742125511,-1.1284319162,-0.0401342697,-1.3941005468,-1.0090293884,-1.0960578918,1.2098025084,1.3029168844,0.0013557881,-0.2298772782,-0.7874842286,1.1899797916,-1.7895762920,0.6294350624,0.2206836939,0.1826732308,-1.4839683771,-0.6505553722,0.6233601570,-0.8338543177,0.2824832499,0.3885993659,1.0900565386,1.8766107559,0.8075032830,0.1507251561,0.8691424131,0.3986098170,0.5120561123,-0.0635230392,1.1407741308,0.9157882333,0.4682558179,-0.5680037141,-1.6270081997,-0.6732350588,0.9897184372,-0.1337077469,-0.4015781581,-0.5095635653,0.4383413196,0.1368908882,2.3518426418,0.2311750799,0.4614313841,-0.6706026793,-0.9506882429,0.6988849044,1.1486055851,-1.3775639534,-0.1598063409,0.2637604177,-0.1926568598,0.8263894916,-1.2977201939,-0.8600575328,-0.6402958632,1.6776019335,-1.9129065275,1.3457714319,-1.0432997942,0.3238283992,-0.1716978103,1.4572438002,-1.0432966948,-1.5072041750,0.0903891772,-1.2257448435,-0.6195468903,-0.8568475842,0.2043592334,0.2641375661,-0.8464022279,0.2488178313,1.0052314997,0.4492959678,-0.0295740031,0.2290060669,-0.1275434643,0.3373597860,0.1626713276,-0.6524628401,0.7707580924,0.3476366401,-0.0531140119,-0.0087814219,1.6692032814,1.6984108686,-0.1209266335,-0.8074813485,-0.1444534808,0.6372165084,-1.3039739132,-1.8125053644,-0.1793181598,0.4690181315,-2.4916374683,-1.8491213322],[-0.5576793551,1.3404135704,1.0666805506,-1.5593063831,-0.9231505394,0.9490910769,0.4438292682,0.2828915715,-0.1954534054,0.0018107980,-0.1907095164,-0.1145968288,1.9409518242,1.0856090784,-0.9538422227,-0.8621790409,-0.4659977257,-1.7791017294,-0.5010645986,-0.6775425076,0.7356759906,-1.7972897291,1.5602645874,1.2374145985,-0.0871757418,-0.6793118119,0.3375435770,0.8931931853,-0.2421072423,0.8488683701,-1.2290204763,-0.2693973780,-0.1366267353,0.0585083216,-1.7161861658,-0.5805538893,-0.9939773679,-0.2650007010,0.4637797177,0.7559719682,-0.7087855339,0.1012896299,1.1064854860,0.6333464980,0.2938173711,0.2699801624,1.7075775862,0.9169011712,1.4294006824,0.3349779844,-0.6877442598,1.5435626507,0.8220308423,-0.4070742130,0.7013475299,0.1943705380,0.0186969098,-2.3783740997,-0.3056056201,0.2207050323,0.9657750726,1.0608078241,-1.1249374151,-1.8694682121,0.7967307568,-0.1668088734,0.8929074407,0.5497352481,-2.2809424400,-0.0579407215,-1.5476003885,0.8956011534,2.7000272274,-0.2422973365,-0.0058167744,-2.7817559242,-0.8345428705,0.5110921264,0.2318991423,-0.2348987013,-1.1566245556,-0.9918349981,-1.8085412979,-0.5614330173,-0.5285215378,1.1941301823,1.4853051901,0.3416150510,1.3896061182,0.8218294978,1.3636242151,0.2253000736,-0.5104549527,0.3647124767,-1.9561990499,-0.8729413152,0.0701007172,-0.2375694811,-1.7378988266,-1.0403276682,-0.8218318820,0.3926897645,0.1339692175,0.1656624079,-0.2436903417,-0.5954623818,0.1061979011,-0.4252826273,-2.0241460800,0.2748028040,-0.0702078119,0.3288494349,1.0060845613,0.4409485757,-0.6922742724,0.2370910496,-0.5651920438,1.8939553499,1.2779444456,0.7899533510,-1.6924171448,0.9689711332,1.3714234829,-0.1442247182,0.5031644702,-0.2282197922,0.1537615359,1.8720436096,2.1535537243,0.0067972033,0.1998646408,0.2613457739,-0.1279868037,-0.2985032201,-0.3642401695,-1.3701349497,0.7696627378,-0.5171828866,0.0796941370,1.0161266327,-0.1516731530,-1.4251996279,1.0709221363,0.9388007522,1.3731763363,-0.5839059949,-0.2922695279,-1.2067660093,-1.0474648476,0.0075881016,-0.0970899835,0.7804442644,0.2210815251,0.0727487653,-1.4848616123,0.4621598125,1.5490251780,2.5317053795,-0.0357220359,0.1732001901,1.8565897942,-1.5049402714,0.9150194526,-1.2461922169,-0.6568403840,0.8576791883,-0.1446761936,-1.4876954556,0.0227297749,0.6311181784,1.8708921671,0.5190147161,0.9861489534,1.2462815046,0.1173469424,0.9257962704,-0.9750080109,-1.2085790634,-0.4388735890,-0.3076315522,-0.0882748291,-0.7719042897,1.8123188019,1.5279533863,-0.8639286757,0.7459006906,0.2987340391,-1.1948617697,-0.3614636660,0.7074557543,1.8297299147,0.5339940786,0.8260959983,1.7657806873,1.2677180767,0.1828723848,0.0288342815,0.8607107401,-1.5287158489,-0.1524372548,-0.6711853147,-0.3347103596,0.0846325532,2.3551928997,1.0067535639,0.3421706259,-0.5376572013,-0.1644151658,-1.3603549004,0.2899155319,0.6658204794,0.7675340176,0.6614913344,-0.0999178067,-0.6015158296,-0.2123781145,-1.5590256453,-0.3314684033,-0.3496010900,0.0747257844,-0.3252461553,-0.8702195883,1.9684408903,0.8418162465],[-0.3862409592,-0.8120633960,0.7752045393,-0.3333698511,0.5466673374,-0.1422412992,0.7192386985,0.4905242622,-0.1880952865,0.2143013477,1.3246078491,-1.2478592396,2.0077860355,0.5948082209,-0.2080378234,-1.5965000391,-0.2735635638,1.3294361830,-0.7973580956,1.4204609394,-1.1398384571,-0.0142109981,-1.5199676752,1.1904004812,0.3158183992,-3.0604200363,0.1238618642,0.9671947360,-0.2805726528,-2.0813367367,-0.3729003072,-0.1396138072,-0.5271163583,0.1263424456,-0.4353207052,-0.2924160659,-1.1753132343,-0.1308605820,1.1438410282,0.1119249091,0.3604819775,-1.0717647076,-0.2294389606,0.1563594043,-0.6368703842,-1.2792086601,0.0777032673,0.0775017738,1.3998067379,-0.6097219586,0.4177627861,0.4011989236,-0.5329536796,2.3956975937,0.1811924279,-0.8800264001,-0.4670289159,-0.3687879145,-2.8602409363,-0.6590387821,-0.2655385137,-0.3368494809,-1.9276156425,-1.5855481625,1.0563932657,-1.3409841061,0.3984279335,-0.2940606773,-0.5712441206,-0.0067037856,-0.6415054798,-0.4785518944,-0.1218238324,0.2206111699,-0.4235476553,-1.3250722885,1.7152470350,-1.3964438438,0.2865219414,-0.4178703427,-0.6859941483,1.0773335695,-0.4156297147,-2.6461470127,0.4012067020,1.0373991728,-1.4278997183,-0.2466971874,0.0849356949,-0.8817490339,0.5450678468,0.1807463765,-1.0566569567,0.1781830043,-0.0064033647,0.3077405691,0.6102537513,0.0652264729,0.2542940974,-0.2305352241,-1.6249768734,1.2014248371,-0.2054571807,1.1181812286,1.1214034557,1.1425615549,-1.2870421410,-0.1241030917,0.5297012925,-0.0382554494,0.3882488012,-0.6190707684,-1.5623250008,0.7568140626,-1.6866751909,0.0804848298,-1.4858338833,0.2297888249,1.4524667263,-0.5968704820,-0.0833020732,-0.2826476097,1.5136855841,1.4851976633,-1.1327924728,-0.8760012984,-0.6094191670,-1.4564394951,-0.7331520319,-0.9318554997,-0.8895848989,0.8252238035,0.1218120083,0.0662026778,0.6619882584,2.0303890705,1.0554903746,0.7169943452,0.1783896387,-1.1470445395,-0.3398191333,1.3697793484,-0.0253568981,1.1636499166,-1.2545666695,-0.1280414611,0.6935864687,0.1563157439,-0.1740721315,1.1867508888,-0.0169510152,-0.0078435130,0.7718135118,0.1622573286,0.2028463036,1.2996282578,1.9638586044,-0.8999834657,1.1722446680,0.4890742600,1.1593204737,1.3701916933,-0.2957761586,0.7029923201,-0.2342174649,0.3841736317,1.0627040863,1.0547840595,-0.3150112927,1.1945139170,0.4034687877,-0.5125347376,1.0919033289,0.4950831234,-0.5558901429,0.3558287621,-1.0803325176,1.0500856638,-0.6788198948,0.5517796278,0.7566818595,0.5830651522,0.6025416851,1.1403605938,1.9247280359,0.5235610604,0.4997765124,0.7582163215,-1.1293796301,-0.3010092080,-0.9889057875,0.6745214462,-0.8895672560,1.0809191465,0.8366023302,0.4471575320,0.6851827502,2.1633660793,1.8103870153,-0.3494542837,1.1950857639,-0.3678962290,1.7989248037,-0.7417379022,0.5473921895,1.0161283016,-0.3572591841,-1.0351872444,2.6292521954,-0.4950847328,1.0285687447,0.0339635722,-1.6448830366,-0.5516107082,0.0174649954,0.4552769065,-0.1754626930,0.1529568881,-1.2097570896,-0.8168189526,2.3861827850,0.6213911772,-0.8158392310,0.5483052135],[0.0878597200,-1.0948483944,0.2889161110,0.3388940394,-1.2642379999,-1.5323306322,-1.0071874857,1.0103991032,-2.4863183498,-1.1070255041,-1.3683352470,-0.4224180877,0.7639471889,-0.5177192688,-1.0074887276,0.1551092863,1.3369476795,0.5466856360,0.3232005239,-1.6532018185,-0.7173513174,-1.0652648211,-1.3265746832,-0.5660467744,-1.3132909536,-0.8130620122,-0.1815343946,1.2690964937,1.1019312143,0.9516880512,1.7226719856,0.4446351230,0.5170425773,1.0413635969,0.4556803405,0.1972223371,1.8935484886,-0.2192615122,-1.4354544878,1.5602838993,-0.6460107565,0.8395658135,0.8136681914,-1.7072492838,-0.7649660110,-0.2882774472,0.8429179788,-0.2447516322,0.9172566533,-0.2275722325,0.2997744977,-1.0302278996,-0.1052929237,0.3555447757,-0.0440536961,0.2784536779,-0.5446269512,-0.7629913092,0.0162314177,0.5546040535,0.9613997936,0.0616658404,-0.2609883249,1.0197192430,-0.6264053583,-0.5688632131,1.5122493505,-0.3373487294,-1.6997095346,0.5993637443,-0.9272513390,0.0842745379,0.8615155816,0.4361516237,0.0753251612,0.2608761787,-0.9517005086,0.9069390893,-1.0981204510,-0.8780823946,0.7374699712,-2.0681431293,1.6813424826,-1.0838361979,-1.1617941856,-0.2509556115,-0.5042505264,-1.1139525175,-0.2048496157,0.1186023057,-0.3202336729,-1.1404109001,0.2558881640,-1.7212871313,-1.0247824192,-1.7450982332,0.6630205512,-3.1837379932,0.4456369877,-1.1326383352,-2.2103080750,0.1183590889,0.6692360640,1.2288067341,-1.9078927040,-1.5564306974,-2.0014181137,1.4565919638,-0.3936092556,0.1973627359,-0.0117214527,0.2743053138,2.0025804043,0.1375686824,0.4163160026,-1.0364779234,1.3307567835,-1.9111024141,-1.5484838486,-0.1650904715,0.8083508611,1.1169708967,-0.3402090073,0.9838241339,-0.1346430779,1.8263845444,-1.7422721386,1.0641328096,-0.4452352822,-0.0513705388,0.4752159119,1.6146486998,0.5102171898,-0.8978606462,-0.8856185079,-1.4149117470,-1.1215506792,-0.7647787929,0.2960679531,-1.1323281527,0.6728130579,0.2450481653,-1.2277650833,0.2975145876,-2.0824947357,1.9949676991,0.1659021229,1.0105559826,0.2631520331,-1.9276918173,-2.7682528496,0.4685620964,0.2694377601,0.2202383578,-0.0813253671,-0.9782060385,-0.6891325712,0.8377320766,0.0159451328,1.3094940186,0.4035644233,1.0255818367,0.5153917670,-0.1808453053,0.1277346462,-0.0179174189,-0.9238844514,-0.5562965870,-0.3335781991,-0.1112420335,1.0880632401,-0.9739040136,-0.6805738807,-1.2655941248,-1.1422700882,0.3249793351,-0.0527833514,-1.7724030018,-2.8175151348,0.8032355309,2.1276588440,2.1617305279,-0.0527869426,-1.0185065269,-0.4677870572,0.2683751881,0.0304780379,-0.1207695752,0.0642614439,-1.6406875849,1.1186835766,1.4654172659,0.7769879103,-0.5351511240,0.3237947226,0.2958904505,-0.1936709285,1.3598756790,1.1916985512,2.2793455124,-0.0956685990,-1.3434432745,-0.1309389323,0.2118412554,-0.6874347329,-0.8510482907,0.0485115610,2.1660628319,1.8733015060,-0.4796108902,0.7988507748,-2.6731781960,-0.4153742194,-0.1474187672,0.9681109190,-0.4934926331,-2.7047929764,-0.3549762070,-1.0457595587,-0.5838010311,0.0829716697,0.0265663285,-0.9133974910,-0.0864409208],[0.2548206151,-1.3221129179,-0.9357287288,1.3260126114,-1.9737194777,1.4048113823,0.4354310632,2.7296981812,0.4620052874,0.1094514206,-0.8226955533,-0.8031890988,0.0278037265,-0.7649024725,-1.1145186424,-0.0622494183,-0.4565858245,1.1405748129,-1.0133868456,-0.5143136382,0.6725979447,-0.0492859147,1.0113819838,1.1048659086,0.6603216529,-0.3727340400,0.7112011313,0.7985103130,-0.2034807801,-0.3387661576,-2.1463022232,-1.1241475344,0.7797137499,-0.2611505687,0.9198236465,0.8049122691,0.3930318952,1.0530589819,-0.3481652737,1.6241984367,-0.2310409844,0.8785673976,-0.5285487175,-1.6051872969,-2.2492547035,-0.2764416635,-0.5564856529,-0.1424106061,-1.6674304008,-1.1508232355,-1.1337341070,0.2225996703,0.9500737190,-0.8307194710,0.2994698584,0.6526410580,0.3251613379,0.0689228550,1.1470413208,-1.4659957886,-0.0308823474,-1.1581293344,1.3566701412,0.0564573184,-1.8906033039,-0.9888930917,0.8690288067,-0.5142025948,-0.6776256561,1.2080899477,0.0295389090,0.5598149300,-0.4604727030,-1.0156501532,2.0301139355,-0.8149390817,0.4358777702,-2.2680025101,0.7438141108,1.9459930658,-0.1377315372,-0.0557070971,0.6126846671,0.4535076916,1.3437209129,-0.4962809682,0.7099696994,0.7285888195,0.5127623081,0.1487104893,-0.4102285206,0.1795057654,1.2648516893,0.8477721214,-1.3445240259,1.1553347111,1.1411762238,-0.1662096381,0.9143199921,-0.0289703812,1.4069156647,0.3577559888,-2.8750307560,0.5885147452,2.0400743484,0.7269452214,-2.0255157948,-0.7129239440,-1.8075281382,-0.0634173900,-1.1268689632,-2.0150179863,0.8664655685,-0.5846761465,-0.7784549594,1.7345633507,-2.3513545990,-1.0365632772,0.8950498104,-0.2481182218,-1.4468442202,0.0471748933,0.3714194298,-1.3437989950,0.1846590042,0.1699056774,0.0043113870,-0.2608861625,-0.6190173030,-0.6328819990,1.0290852785,-0.0337157957,0.4794288874,2.3265194893,0.6430379152,-2.3729488850,0.9642911553,0.1792990863,-0.7555418611,-1.6815297604,-0.7509630919,0.5920519233,0.5271238089,-0.3613370359,-0.8814773560,-0.4798953831,0.4807034135,1.0450673103,0.4689154923,-0.8064729571,-0.1262860000,0.5163205266,-0.0193388909,1.3125473261,-0.0543386117,-1.2151954174,0.2638210654,-0.0364308245,-0.5281606317,1.2426241636,-0.3205830455,-0.1426075846,-0.4687280953,0.1643962413,-0.3802402318,0.3136390150,-1.4041841030,0.5760033131,0.1114661992,1.5098890066,0.6955231428,0.1370026469,-0.7484128475,0.8863014579,0.6602393389,-1.9834080935,-0.5476236939,-0.1973284334,2.7404897213,0.3918314576,-1.0367541313,3.3678669930,1.2490388155,0.4868883491,-0.1266106814,-0.3812051713,-0.0928058252,-0.6301029325,-0.2929019630,0.8924237490,-1.1051132679,-0.5074141622,0.8979958296,0.6536089182,-1.9230524302,-0.1654659212,1.8929178715,1.2809594870,-0.0103674419,0.3907116354,0.7662822604,1.8371510506,-0.3340227604,2.5882060528,0.2204035223,0.5281216502,-1.7063302994,0.9104158282,-1.6062641144,-0.7288155556,0.2892820239,-0.9528056979,-1.0483649969,-1.4120820761,-0.4685563743,-0.9451056123,1.8808521032,-0.7579481006,-0.1606787741,0.5887008309,0.4289551675,0.3829198778,-0.3382299244,-1.3812276125],[0.4090642035,1.3374681473,1.2288401127,1.5761895180,-0.8516318798,-0.2723531723,-1.2398227453,0.2986592054,-0.1322129071,0.0139572183,-1.0466662645,-0.2781076133,-2.0794694424,-0.0169874635,-0.2712719142,0.0039244746,0.4328824282,-1.1713631153,-0.7448069453,-2.3252754211,0.8434809446,1.0628112555,1.0627806187,1.3218361139,-0.1161545292,1.3281513453,0.2959490418,-1.1790206432,1.4481866360,1.1699938774,-1.1192673445,0.6433444619,0.2693232000,-0.1480689347,-0.1910275519,-0.8702706695,-1.4614882469,1.8815187216,-1.4435530901,1.4433736801,0.3612532616,-1.4280672073,2.8515341282,0.6932016015,-0.9333375096,-0.3010189831,-0.5288223028,-2.0452833176,0.0003303100,0.0397834890,-0.2597131431,0.8163148165,1.0304787159,1.0927882195,-0.3876484334,0.6974347234,-0.2137241662,0.0889874473,2.0182070732,-0.6308559179,-0.4557030797,1.8793838024,1.0392211676,0.3013942242,1.0411463976,-0.7474292517,0.4135686755,-1.2054290771,-0.4202263951,0.7777327895,-0.4293861985,-1.3665663004,-0.6443052292,0.9349673390,-1.2924108505,1.2160620689,0.3794503212,-0.5429808497,1.1152054071,-0.0982226357,-0.2433270663,-0.5958116055,0.1453744769,1.3489847183,-1.0879634619,-0.7124319077,-1.0658148527,0.6417328715,-0.9150858521,1.7110761404,-0.1847843677,-0.5252919793,-0.1499496102,-0.6333470345,0.5339637995,-0.0101217143,1.5284255743,1.4889923334,2.0363731384,0.2519434094,1.1972477436,0.1169527769,1.0913250446,-0.7577394247,-2.2312619686,2.0685327053,0.9227512479,-0.4438139200,-0.7689087391,0.5014011860,-0.0705506951,0.8355823159,1.4453328848,-1.8012226820,1.9324140549,-0.3285877109,0.3485644460,-1.0819559097,-1.6279023886,0.3250808120,-2.0706045628,0.3532761335,0.1403892487,-0.1772062927,-1.8101980686,0.4101774693,-0.0899073407,1.7163796425,-1.1482729912,-0.9280102253,-0.9309043288,0.7067950368,-1.5563870668,0.7764381766,0.4183218181,-0.5949311256,-0.5479576588,-0.3660928607,0.1221372858,-0.8077557683,-0.7920037508,2.3722608089,-1.5027210712,0.0144450190,0.1528333724,0.7655932903,0.4014534056,-2.3351640701,0.4352296889,0.4105211794,0.6907932162,-0.9530596733,-0.6473321915,-1.4276818037,-0.2839665115,-0.6459219456,1.0340801477,1.1909698248,0.4201194346,2.0286815166,1.7382160425,1.2285472155,0.3203367889,-0.1061791256,-0.0371501744,0.1596977413,2.9247212410,0.4545726478,0.1455145329,1.1376500130,-0.6597639322,-0.7237402797,1.7943882942,-1.0624425411,1.0409113169,0.7168018818,-0.7416293025,1.6516425610,-1.2652876377,1.5150113106,-0.4677824974,0.3317876756,-0.1702616215,1.0125669241,0.6174751520,0.0796523988,-0.3447691798,-0.8938841224,-0.2084598839,-1.6965242624,-0.3470969200,-0.5429576635,-1.2105997801,-0.8608617783,-0.9803560376,-1.0367319584,-0.2582563162,-0.0054417914,-0.9185775518,0.0305522084,-1.5969980955,1.0509661436,0.4743795395,0.3633412123,2.1253325939,0.2701029480,0.7550885081,-1.0644488335,-0.6703055501,0.5664753318,0.2437788546,0.9583575726,-0.7638131380,-0.0029742755,-1.6142610312,-2.2065219879,-0.6328999400,-1.1739649773,-0.1321678013,0.4885062575,1.3792656660,0.9682490230,0.0666629449,-1.4227808714],[0.9389460683,0.6142101288,0.4952200651,-0.3560309410,-0.4492895603,0.8873738050,0.4932395518,-1.5655148029,-1.1107832193,1.0344661474,-0.6064270735,2.0916547775,0.8920361996,-0.9328433275,-0.5210302472,-1.3629417419,0.6946575642,1.2764936686,-1.5056940317,0.1522615552,0.5383676291,-1.5878702402,-0.4938810766,0.8755227327,0.9683454633,-1.1522424221,-0.0225333683,0.9137845039,-0.5549679995,-0.7432829738,0.5882457495,1.3948264122,-0.5563979149,-1.2109256983,0.6127566695,0.7170585394,-0.9698428512,-0.0640512407,0.9629530907,-0.6542029381,-0.4348601699,1.3914189339,0.0691725537,-0.7639746070,-1.0185455084,-0.6848862767,-0.8079086542,1.0332250595,2.0122599602,0.2306910008,-0.1354916990,0.6538707018,-0.5724883676,-1.4610589743,-1.1935203075,0.2267056406,-0.8769122958,-1.8110815287,-0.0701249689,-2.1590735912,-1.7739510536,0.7410432100,1.1597032547,1.0969405174,1.9822311401,1.7125688791,0.4174836278,0.7853913903,0.5818963051,-0.2479056716,-0.7231642604,-1.0702124834,-0.3364195526,-0.1131938845,1.1898450851,0.3929497898,-0.9904890060,1.3256819248,-1.4747922421,0.8332212567,-0.3751766980,-0.5573330522,1.6865506172,0.4411950111,-0.1255817413,2.5350151062,-0.0580064729,-1.1477249861,-0.9363329411,1.0586061478,1.6392625570,-0.5135805011,0.5801268816,-0.0548072122,-0.9464817643,0.0946878791,-0.7109422088,-0.3775828481,0.6923879981,-0.1164232716,-0.0274552815,0.9514126182,-0.4382908940,-1.1965507269,1.2529064417,0.2980563641,1.3691887856,-0.0335965566,-0.5210404396,-0.0635088459,-0.6337655187,0.4338080585,0.0755116045,1.0962269306,-2.0093507767,-2.4804511070,-0.4190257192,-1.1947860718,0.7652868032,0.1273288131,0.8648121953,0.1008700207,-2.7048275471,-0.9510456920,-1.4726403952,0.0437071584,0.0826288909,2.0108227730,-0.6876509190,1.0703777075,0.3906943798,0.5436729193,0.9461514354,0.4150171578,-1.2524062395,0.3780267239,0.4788390398,0.1191774979,-0.2176552713,0.6323692799,1.2053620815,-2.3339111805,-0.6712306142,-0.2902439833,0.9778840542,-1.2306065559,0.6382020712,1.4408342838,-1.4152365923,0.7717177272,1.0857385397,-0.0721675083,1.7905182838,0.2215190679,0.3466854990,0.4562369287,0.5969560742,2.9985487461,-1.3164579868,-0.3309495449,-0.1202616245,-1.3430130482,-0.3943373263,-0.2237624377,-1.0122376680,1.0934064388,2.1965367794,-0.9623855948,0.4112857878,-0.7151125669,-0.2869275510,0.2724553049,-0.4762646854,-0.4325575829,0.6262726784,-0.8784438968,-1.5952491760,-1.9096953869,0.3377545774,-0.7852398157,-1.2339268923,-0.6910353899,-0.6821838617,-0.4460454285,2.0578155518,-0.8237828016,-1.1603840590,0.4606530964,0.7554635406,-0.4197842777,-1.7064237595,-0.4686983824,-0.8817868829,0.2867316008,1.2538535595,-0.1737776101,-0.2247789651,-1.5630481243,0.4511615634,0.6216961145,0.0750527158,0.3924145699,-0.3754163980,2.8608353138,1.5530757904,0.0425318517,-0.1855948865,1.3114998341,-1.9979593754,-0.1392807513,-0.5736815333,-1.1196737289,-1.2200103998,-1.2405149937,1.6866083145,-0.1392551512,1.4387949705,-1.4744572639,-0.1115004197,-0.1053058803,1.5830054283,0.5361398458,-1.1185491085,0.1301326603],[-0.1323783994,-0.6407990456,-0.4196328819,-1.3846974373,1.3309514523,-2.7046353817,0.0045477180,0.8607456088,-1.1245253086,1.2274123430,1.0389893055,-1.9995344877,0.8867872953,-0.7012874484,0.4385780990,-2.4129841328,-1.8402497768,-1.1964808702,-1.0597640276,-0.4711083174,0.9958802462,-0.7689711452,0.4363094270,-0.3747068644,-0.8480607271,1.2542796135,0.2656555772,2.2228662968,-1.8119665384,-0.9371265769,-1.4262866974,0.1876880974,-1.0126883984,-2.0700709820,-0.0803233460,-0.7962924242,-0.0414651409,1.7242867947,1.1771477461,-0.8121748567,-0.1645865291,0.9184431434,-1.0123202801,-0.3763069212,-0.1460562795,0.2211588472,1.1201910973,-0.0191150960,0.7085775733,1.4897265434,-0.0689364448,2.1542754173,0.4996797740,-0.4819819331,-1.2270764112,0.9048069715,1.1497319937,0.1581166685,0.9835773110,0.8247444630,0.0351414941,-1.9824454784,0.2739833295,1.2348875999,0.7725579739,0.2573614717,0.5969483852,0.2761405706,-1.0226633549,1.7934087515,0.1439145803,0.9556096196,1.9798411131,1.5616306067,0.4073262513,0.8567109704,-0.5908481479,-0.4551822841,-0.6076927781,3.1823654175,-0.2167487592,0.2382182330,-0.1999024898,1.5419658422,0.5953208208,-0.5438700318,-1.2184040546,1.0318024158,-0.3316418827,-0.8802697659,-0.8947135210,-0.0105047869,-3.3704369068,0.4810324013,-1.9140502214,0.3950212300,1.6429020166,0.8465449214,0.4476325512,0.4394526482,0.6456232071,-0.1141211912,0.3965323269,0.4823193252,1.1300865412,-0.1235338151,-0.5488389730,0.5645562410,0.2612779737,0.3265740275,0.6936011910,-0.2548986077,1.5862183571,-0.9947738051,0.7125069499,-0.2931319177,1.1054304838,2.4689922333,0.8357470036,0.0130731547,1.4690824747,1.7760980129,1.7123028040,0.7116003633,0.0806625262,-1.2783840895,-1.2734508514,-0.5681645870,-0.6348869205,0.4688507617,0.6399840117,-0.1546345800,0.0401923917,-0.4833675027,0.7989139557,-0.5271605849,2.2883720398,0.9557285309,-2.1976850033,-0.9270659089,-0.7950120568,-0.0018317529,0.1464039981,-1.2294088602,1.3516594172,0.9153459668,-1.5380679369,-0.8945394158,0.4787234664,-2.2154150009,0.4985513687,-0.3055807352,0.3027708232,0.7394517064,-1.1887978315,-0.8729275465,0.0136809060,0.2144620121,0.1222232655,2.0651092529,-1.3659356833,-0.6718088388,-0.3926681578,1.7661414146,1.9525667429,0.5477364659,0.4586538374,-1.6999478340,-0.9211118221,-0.7009931207,-1.2132214308,0.1789101660,0.1699265093,-0.5791007280,-1.1458439827,1.6576943398,-0.0798066929,-0.5319992304,1.1932622194,1.2408850193,-0.9731871486,-1.6201738119,-0.4493438900,-0.4226911962,-0.0897471532,0.7771114111,1.4575105906,-1.3590695858,-0.2422187775,-0.0717459098,-1.5813609362,0.3929780126,-0.6107853055,1.0224369764,0.9272652268,0.2391461581,-0.7454996109,2.5418419838,-0.0836168155,0.0664021298,-2.0217483044,0.0662804097,0.8211984634,-0.5589304566,0.2068057358,0.0640166700,0.9859380722,-0.9006294012,0.8219537735,0.1385826170,0.9545242786,-0.0526152104,0.2759426534,1.2005162239,-1.2941392660,-1.3588281870,0.0705319345,0.7191971540,0.9778149724,-0.8361923099,-0.0056291926,2.0522983074,1.5833069086,-2.3987114429],[1.3274357319,0.0064704483,-0.6484159827,0.9853000641,-1.3528258801,0.9159484506,0.2959030271,1.7370500565,1.2756021023,-0.0047396058,-0.2977020144,-1.0068558455,0.3262749016,-0.2303540409,0.2995219529,-0.3161655068,0.0587377883,2.1757032871,0.7780051231,-0.6155528426,-0.6651592255,-1.9044895172,1.0170710087,-0.2350578159,0.8182260394,-0.8092999458,1.7549093962,1.1859247684,1.2325001955,-0.4970070422,-1.8003635406,0.0549518168,-0.8563482165,1.9319586754,1.1410189867,1.4234589338,1.0574718714,-0.3678317964,-0.6011454463,0.9331198335,0.4304300845,0.9622161388,0.7820424438,0.5289376378,-0.1422957182,-0.0972463861,-1.1529253721,-0.3259295225,-0.7833554745,1.3779989481,-0.8609403372,-0.1602535546,-0.0599776134,-1.4946795702,0.2707850337,-0.5179989338,0.8004550338,-1.2054712772,0.5004020333,-1.8651032448,0.5174418092,-0.9989871979,0.3034828305,0.3723188043,0.3597621620,1.8270949125,0.4796812832,-1.3419852257,-1.7499210835,-0.1308304220,-0.8859434128,-0.4169880450,0.1174671724,-2.3178379536,-1.3677680492,0.3057938218,0.4547121227,-0.3513031304,-0.2731293142,0.0115191638,-1.1899770498,-0.7389093637,1.1789582968,-0.8494232297,1.5048207045,-1.4718470573,0.0784921274,-0.8367189169,-0.7885698676,-1.6579880714,-0.3294337988,-2.6300072670,-0.4856718183,-0.2715861797,0.2706677616,0.3428797126,-1.0286426544,-0.4359491169,-0.8634329438,1.7400062084,-0.6620771289,-0.8842269182,-2.4243388176,-1.9477770329,-0.2599925101,1.3951880932,1.7454518080,-0.8353650570,0.9496917725,-1.1362177134,2.7656846046,-0.3285786211,0.1080850959,-0.3550520241,0.5351243019,0.0626995116,-0.9387552142,-0.3435013890,0.7270622849,0.6199610233,-1.6607036591,-0.6297575235,0.7958660126,0.2775168121,0.1991933733,1.5361841917,0.0672869310,0.3522060812,-0.6162402034,-1.7315293550,-0.0968044847,1.3629524708,0.0375886112,-1.2578284740,0.4546712935,-2.2027161121,-0.7068659067,0.1386501193,-0.5204744935,0.8022690415,0.1464944929,-1.0731513500,-0.0275365952,0.3999854028,0.6108346581,0.7355435491,1.8661667109,0.4773959816,-1.0730679035,0.8396475315,1.1034216881,-0.5024297237,0.7762069702,-0.0361183025,0.6873787045,-1.6265043020,0.0699692369,0.3768470585,-0.9322768450,0.0477588251,0.9751135707,-1.9907674789,0.1195630580,0.3530760705,0.3995818198,1.1568886042,-1.2285298109,-0.2479773462,-0.0089573311,0.7797752023,0.3299199641,0.9332191348,-0.4034706950,-0.3111452460,-0.4305924177,1.3256598711,1.1262835264,1.1973778009,0.1951432228,0.2563470602,-0.2257311791,-1.0581756830,0.5406070352,1.2327010632,0.3645146191,-0.9438908100,-0.5273874998,-0.0965472907,-0.7064593434,1.3101346493,-0.6243762970,0.5321087837,0.8991381526,-0.0843996927,-0.9281385541,-0.0536656491,1.8132817745,-0.6429740191,0.2113274634,-0.2402074486,1.0136674643,0.3831409812,-1.0056535006,0.6262342334,-0.0874363706,1.3416323662,-0.4109298885,-0.0141236670,-0.7411656380,2.0815262794,-0.7702627182,-0.3985530734,-0.6752582788,-0.9599013925,-0.3416412175,-1.3532416821,1.8661503792,0.1309303790,0.1339515895,-0.5823944807,1.7389822006,1.0031000376,-1.0844048262,-0.4892614186],[0.5706951022,0.7267456055,-1.5972158909,-1.3844085932,0.1251437813,-0.1611333787,1.2971414328,-0.4890547395,-0.2959007025,-0.2359022498,-1.5221960545,1.6418451071,1.8629655838,-0.6008669734,-0.6448215842,-0.6749278307,-0.2803279161,2.2114477158,-0.2965689003,-0.3168765903,1.3606915474,-0.1464030147,-1.2688233852,-0.8596234918,-0.2300802022,-1.5350259542,-1.1681234837,0.0532827750,-0.6676311493,1.1589723825,-0.2325885743,-0.1550059468,1.2478905916,0.2482963949,1.3774282932,-0.8201007247,0.7307916880,0.3846693635,-1.3935933113,-0.5091059208,0.1876038760,1.2699453831,0.0560138673,0.8535129428,0.5293108225,-1.3195542097,-0.3675530255,-0.5306439996,-1.3628910780,0.4709477127,-0.4559128284,-0.2349215001,-0.9671003222,-1.4965330362,0.3901738822,1.0340222120,-0.1132831424,-0.8353191614,-0.2977720499,0.4513333738,0.8387823701,1.0269012451,0.7130354643,-0.3210241795,0.6128639579,1.6848592758,1.6609615088,-0.0710362568,-0.4100494087,-1.2127778530,0.8047491312,-1.7605817318,2.9120903015,0.9917885065,-0.5696196556,1.1640443802,-0.3759628832,0.5348631740,0.8605149984,-0.4322288930,-0.8272349834,0.8973447680,-0.9950327277,-0.0557409078,1.0292906761,1.1893057823,-0.2426627278,-0.4762936532,-0.3401097953,1.9560794830,-0.9409772754,-0.1492427140,-1.4113221169,-1.7686339617,1.7728070021,1.4844536781,-0.6800952554,0.8788170815,0.8911567926,1.2230223417,-0.8783909678,-2.2849061489,-0.5757948756,0.9280388951,-0.7595967650,0.0587394610,-0.1624921113,-0.2155980915,0.2433040589,0.0557866134,-1.1757665873,-0.1818657964,-0.2644596994,-0.9477189779,-0.9844954610,-1.3387968540,-0.0450486951,-0.6944232583,-1.0707694292,-0.7842970490,0.6350764632,-0.7319315672,-1.6097750664,-0.4874561727,-1.2056317329,-0.3934192061,-0.9384541512,-1.2435036898,0.7331660986,-0.1061901152,-0.2536563277,-0.4899016917,0.2329620421,-0.8986451030,0.2935437560,-0.6980230212,-1.5478031635,-0.6342258453,1.7035830021,0.1028883606,-0.7894119024,-0.1351793706,0.5689413548,-1.0633289814,-0.0086180344,1.7042182684,-0.6117020249,1.2448474169,-0.6777845025,-0.2346623689,-1.9456766844,-0.0306932554,2.1096534729,-2.2844188213,0.0728366897,1.2431104183,-0.5718863606,-1.0966272354,-0.9830226898,0.4332626760,-0.6825378537,0.8575298190,1.3153325319,1.9670081139,-1.3972771168,-0.1857836246,0.1219161525,-0.5779262781,0.6101424098,0.2377535701,0.2397469133,0.2591325343,-2.0442194939,-0.6908274889,0.2541438639,1.8209025860,-0.7531303763,0.8614359498,1.5990792513,0.8102792501,-0.6263534427,0.0995728076,0.2731429636,0.7282792330,0.0114688184,-0.4514726102,0.4547981322,-1.7314876318,0.9617160559,0.2156383395,2.5990610123,-0.8828431368,-0.1147982627,0.1282157302,0.4696044922,-0.5134382844,-1.5148773193,1.6586340666,-1.0218036175,-0.0250433777,-0.2382687032,0.2340601683,-0.7567276955,-0.3305819333,-0.3038704395,0.8338783383,0.1608898491,0.4396271110,-1.8737742901,1.3429384232,-1.3435566425,1.1009212732,2.0582981110,-0.9400506020,0.7694303989,1.4895060062,1.1229710579,-0.4763310552,-1.1265016794,0.4342498481,-1.0764023066,0.1756174266,0.2279533893,0.4279670715],[0.2260316908,-0.9843562841,0.6740505695,-0.8490674496,-0.3237790465,-0.4662958682,0.1734868139,-1.4428119659,0.6804466844,-1.0618598461,-0.7360239625,-1.2143082619,0.4160400033,-1.0800274611,0.3315010965,-2.1402599812,1.2378108501,-0.7899955511,-0.1041101590,2.0045559406,-0.4333100617,-0.4725388587,0.1273653358,1.4571205378,1.3495620489,0.5959379077,0.2545638978,0.3057740331,-1.0019536018,0.5784690976,0.2518575788,0.6918970942,1.5384705067,-0.2609039545,-0.2850797176,1.3235375881,1.1674975157,-1.4240127802,0.6997848153,-0.8154026270,-0.3378392756,-1.8234149218,0.6829866767,-1.2062163353,1.6120146513,-0.7018817663,-2.2123842239,0.8546330333,-0.4547747076,-0.3642478585,-0.3706068397,-0.2988089323,0.6899989843,-0.6115986109,1.2595679760,0.2240811884,2.6741890907,-0.5876898766,1.5658192635,0.6845703125,1.0701469183,-0.3652437329,-0.2096356004,0.3791528642,1.5315600634,-0.1469570398,-1.5674178600,0.7937241197,-1.5094797611,-0.4463437796,-0.4219004214,-0.3260750473,0.3198117614,-0.9380064011,-1.2634960413,1.2424186468,1.4660786390,-0.4180395603,0.1620767266,-0.2839062512,1.7759104967,-0.8857923746,-0.5912535787,-0.5251594782,0.3382563591,-1.5086387396,-0.3405946791,-1.8012282848,-0.0873163119,-0.5842204690,1.2199001312,-1.4205188751,-0.2968263030,-1.3448718786,0.6785737872,0.7722478509,-0.2272520810,0.0527814403,0.9790414572,-1.3844666481,-0.7844890356,-0.7075823545,-0.3223336637,1.5800389051,-1.0154743195,0.3925264180,0.5471366644,0.2460810691,0.1381963342,-1.3894932270,1.9471534491,-1.5014412403,-1.0413231850,0.3324744999,-1.6873404980,-0.2129056156,1.4749987125,-1.4523178339,-3.1190896034,-1.2718390226,2.4424221516,0.9950337410,-0.0496420674,0.9837993979,1.9826014042,0.7137126327,0.3330027759,0.0999203622,-1.2657548189,-0.4869438410,0.9282532334,1.1198828220,-0.3685782850,-0.3691154420,0.2586824596,0.4915171266,-1.1458977461,0.8304197788,-0.2539549768,-0.1996763498,-0.5544936657,-0.0025407872,0.1407305300,0.4582334757,-0.4711025953,-0.2920936346,0.6538417339,1.5560307503,1.7619452477,1.2360332012,-0.1315362304,0.0920973346,-1.4905031919,1.2301188707,-0.4421279728,-0.6929170489,-0.7973194718,0.0634542629,-1.2857584953,0.0631699786,-0.0861530378,0.2839331925,1.2970875502,0.2933334410,-2.3788189888,-0.6523681879,-0.0305581670,-1.1570004225,-0.7451606989,-1.3816602230,-1.4926953316,1.2165389061,0.4167850018,-1.4080522060,-1.4262783527,1.2082084417,0.6638391018,-0.7060305476,1.1353538036,1.3180701733,0.8578649163,-0.8731014729,-0.9829578400,-0.5651094913,0.6818859577,-0.6653128862,-1.0673677921,0.8933325410,1.2560358047,-1.1704576015,1.4296464920,0.6040555239,-1.8464752436,1.3270595074,0.1488941312,1.6056982279,0.5017446280,-0.8903104663,0.1706050634,0.8819519281,-0.0030436872,1.6358895302,-1.2797946930,1.2676116228,0.1627141535,0.7670778036,1.4692165852,0.0669638589,-0.3648871481,0.4743653536,1.0513372421,-1.1020389795,0.7753785253,1.0108804703,1.0408326387,-0.5209835172,0.6818500757,0.0904824138,1.0239518881,0.6365932822,0.4699539542,-0.7060739398,-1.0360990763,-0.4540962279],[-1.3326988220,-0.5124089122,1.0929511786,0.5940018296,-0.4353291094,1.6788744926,0.7090277076,0.6970493793,0.6935103536,-1.5174489021,-0.6674986482,0.3959916234,0.0314268805,1.1867270470,0.1292989403,0.2488492429,-0.2043378651,-1.3287620544,-0.7390286326,0.8793639541,0.5443399549,0.4859343767,1.2453608513,-0.3818763494,-1.6809335947,-0.3598356843,-1.3391313553,0.6595156789,0.0735542253,-0.0190615188,0.3803745508,1.8210678101,-0.0958271772,1.8411321640,-1.3670797348,0.5427723527,-0.7703274488,-0.6727721691,2.0768125057,-0.8670269847,-1.2214968204,1.0898650885,-0.0725031719,-0.0864036158,-0.4280133247,0.4899945259,-1.1253395081,2.0763638020,0.1353816539,1.4642211199,-1.5274518728,0.9599250555,-0.7167305946,-0.1460455060,-1.6847262383,0.4852546155,-0.2923988700,-0.4610815942,1.1864080429,-0.3434349298,-1.2183084488,-0.4544598758,1.3759692907,-1.1702294350,-0.8205097318,0.0596311018,0.2823673487,-2.4523892403,0.1126904637,0.4746390879,-1.6707005501,-0.9480802417,-0.1634900272,-0.4575511217,0.6539748311,1.3743329048,-0.2066034228,1.9918875694,0.0467024818,-0.5654628277,0.5026557446,0.7354540229,-0.2884488404,-1.1007701159,0.3083924353,1.0963255167,0.1424771547,-0.1030726954,0.8625275493,-0.7215863466,-0.9913249612,0.6632069349,0.4783222377,0.3580340743,-0.1725240052,-0.6392794847,0.2011111081,-1.6484777927,-1.0174485445,-0.0872349590,1.1921215057,0.5025959015,0.2836691439,0.8681715727,0.3389258087,0.5296366215,0.6545069814,-1.9461959600,0.5430054665,0.5104394555,-0.7187666893,0.4541369677,0.9566402435,-0.0594963208,1.6902428865,0.8972370028,0.2773114443,-0.4134947062,-1.2408342361,0.4204489291,-0.4516286850,-0.2110308409,1.0829126835,-0.9779353142,1.0323063135,-1.2429711819,0.6037898064,-0.3205806017,-1.2381771803,-0.3923279941,-0.0805774033,0.4635975063,-0.2898182571,-0.2014379799,-1.4034485817,1.1136484146,0.0691273436,0.2618164718,-1.8693686724,-1.2310501337,-2.6746325493,-0.6870961785,0.4724680781,-1.7013285160,0.7355017066,-0.2741308510,2.2835941315,0.7250497937,0.7654329538,-0.5647279024,0.1644261926,0.5307115316,0.5746467710,1.9586399794,-0.5265694857,-0.4879961312,-1.4865173101,-1.1455962658,-0.4128835797,0.6693022847,-1.5804158449,1.2603547573,0.7857226729,1.2067354918,0.4648757875,0.9003841877,0.5806150436,-1.7880207300,0.5372216702,0.7575263977,-0.5589818954,-1.0746775866,-0.0021043734,-1.6893839836,1.7304996252,-1.1568057537,-0.0617405735,1.0687190294,0.5711197853,-0.4428283572,0.1552816480,0.2527465224,-1.1368750334,-0.5381591916,1.5881439447,0.2860608101,0.3805446029,-1.6672601700,0.7602344155,0.4308459461,-0.5576328635,1.4369031191,0.5004381537,0.6119653583,0.7021585107,0.3857577443,-0.7897416353,-0.6196731329,1.0033321381,0.5967955589,0.6828916669,-1.1994363070,-0.9904943109,-0.3934526742,-0.4732136130,0.2426480353,-2.0425484180,0.8007996678,0.5071797967,1.5479482412,-1.9100352526,-0.9601728916,0.9510821104,-0.0823726431,1.3400896788,0.0834527016,0.6438699961,-0.9652031064,-0.4854627550,0.1482844353,-0.9956216216,-0.3950155675,-1.4705321789,-0.6609185338],[-1.8514585495,0.1311995983,1.6940715313,-0.1337316930,0.5798290968,-0.2847125828,0.3875262141,-0.0368419625,-0.2958053052,-0.2373793572,0.3187682629,-1.4628130198,1.7015430927,0.3156693578,-0.0703716800,-1.4071842432,-0.5739857554,-0.6005715132,0.0037154576,0.3062636256,1.8498947620,1.4188458920,1.4881933928,-0.9052850604,-0.1264310330,0.3862965405,-0.8140952587,-0.3810549974,0.6935589314,0.2257216722,1.0393418074,-0.1589675099,-0.1574358940,-0.8825321794,-0.1856806725,-1.1158484221,0.0572987758,-1.7708715200,-0.1011177674,0.7603165507,-0.8440169096,-0.2493463904,1.7614215612,-1.5872883797,-0.1093700677,-1.3668683767,0.0597839355,-0.4041580260,2.0201151371,-0.8874104619,1.0089551210,1.1857368946,0.8527296185,-1.5749006271,0.4625138044,0.0153149329,1.2475622892,-0.1801164150,-0.1654497087,0.2987264693,-0.0666087642,-0.5388320088,0.5844370723,-1.0518535376,1.1758712530,2.0751616955,0.0651127324,1.0435625315,1.2183630466,-1.1029225588,-0.3894551992,0.4537330270,2.0570921898,-0.6587897539,0.7516878843,-0.3802327514,0.6569559574,0.3319684267,-0.7577400804,-0.6714701653,0.6479204297,-0.0799216107,0.8066828251,0.1515915394,0.4205271602,-1.3345781565,1.2583372593,-1.6809546947,2.8598580360,0.6408307552,1.4548523426,-2.0656576157,0.9247038364,0.1503020823,1.2612864971,0.1509641409,0.4554774463,-0.3452061713,0.7462652922,-0.1073673740,0.9654961824,-0.1182610542,0.4436274767,-0.9892361760,-0.7423120737,0.0164196324,1.1263713837,-0.9633116126,2.3793332577,0.0003982415,1.7510664463,1.4233546257,-1.7395235300,0.7623349428,0.3986560702,-0.3507902622,-0.9979709983,-0.2359415144,0.1348109394,0.1125656813,-0.6708725691,0.3356072605,-1.2268385887,-0.1155574620,1.5565766096,0.4939498603,-1.1511844397,-0.2640740573,1.2304129601,-0.6129906774,1.4380507469,0.5469411612,-0.2832261324,-1.2993297577,0.9013664722,0.2450891584,-1.5840128660,-0.9167362452,-0.8300950527,0.1757986993,-1.6784344912,0.0636152625,-0.4752668142,1.0027318001,-0.4386060238,0.4709543586,0.2046563327,0.7120774984,-1.0557335615,-0.1954088658,-0.3462091386,-1.4930646420,-0.1731721908,-0.0131932432,0.1934194416,0.6658257246,0.6255763173,0.7684777379,-0.2945936620,-0.6855698228,1.6016274691,0.4373116791,0.2726550102,0.8263636231,0.8291152120,-0.3335766494,-0.8821152449,-0.2419444174,0.3788249791,0.7710736990,0.1886449307,-0.7076955438,1.0469366312,-0.1463537216,2.6089265347,-0.6633446217,-0.2353220284,-0.2212995142,-0.7482500672,-0.2117432356,-0.6482862830,1.7328232527,0.2244552821,-0.6652676463,0.3964279890,0.5592606664,-1.9637831450,-1.8922384977,-2.4780626297,-0.1493539512,-1.4736962318,-0.5816019773,0.9401268959,-1.4714045525,-1.1942760944,0.3668976128,0.0830259994,-0.8098945618,-0.2031483948,0.7353839874,0.0058557466,0.1347817332,-1.1094521284,0.2028308660,0.2542048991,-1.1559451818,0.2734256089,0.2041602880,0.7460332513,-0.0899389461,-0.2154787630,-0.2122800201,0.9198871851,1.8418318033,-1.5593268871,0.2610054612,-0.7024561763,1.7462798357,1.4986221790,-0.8376168609,1.0504339933,1.7258865833,-1.9487813711,1.1911542416],[0.2755780518,-0.2877275944,-0.9894776344,-0.3523977399,0.3716616035,0.7340951562,0.0465161428,2.0946755409,-0.4287497699,1.2823368311,-0.1708106399,-0.3339754343,-0.3745219409,0.0234510638,-0.5941810608,-0.6525736451,1.0548883677,0.2085448205,2.1524362564,-0.2876541615,-0.8876055479,1.3150926828,0.5178136826,1.6735327244,0.2665548027,-1.4265080690,-0.4362727702,-0.3537387848,-0.4326723218,-0.2196337879,0.1769788712,-0.5016754270,-1.9468058348,-0.8031057119,0.8652901649,-0.1611579061,0.2561255395,1.7606273890,-2.1001949310,-1.3642786741,-0.2298175246,0.2183056921,-1.8855527639,-0.4925869107,-0.2614548802,-0.8557839394,-0.1850143373,1.1688899994,-0.6215562820,-0.7275066376,-0.3990510404,-0.0416359417,-1.7970087528,2.5328645706,0.2876232266,-0.9678805470,-1.0853585005,2.2810258865,0.5573899746,0.7743658423,0.6814321280,0.8439531326,0.8247879148,-1.6703401804,-0.6495918632,-0.5389128923,-1.3334633112,0.5207111835,0.6436123848,-0.6462132931,0.3580046296,1.6225868464,-0.4871959984,0.6803805232,-0.2100579590,0.1405427754,-0.5697072148,2.1073331833,-0.9879789948,-0.5540517569,1.3131885529,-0.5549680591,2.8242397308,-0.5965105891,1.5914705992,1.4811695814,0.3065227866,-1.2098796368,0.0168109611,-0.1309269071,-1.2207504511,0.6678852439,-2.1338050365,-0.6641706824,-0.3057371974,-0.0899235457,0.1791422814,0.8580896258,-0.8787598014,0.4738532007,0.6654763222,-0.8824122548,-1.6115596294,0.5279802680,0.0964465886,0.9635578394,-0.3902786672,-0.3802396059,-2.2409791946,-1.0808991194,-0.7525261045,0.2250808775,-0.3743494749,-0.1169842705,0.0614006370,-0.3293505013,0.3097124100,-0.7995985150,1.8154002428,-1.5777002573,-0.5507161617,0.6225386262,-0.3113619685,-0.9763633013,1.4898904562,0.9300459027,-0.5811542869,1.2458498478,-0.2211971879,-1.5934373140,0.0132478783,0.9656499028,-0.1373882741,-0.0728489086,-0.5899522305,0.1944191307,0.4169383943,0.2829756737,0.1644549668,-2.1832423210,0.6266455054,-0.5795871019,-0.9039760232,-1.7385983467,0.2067382336,-0.2576915324,-0.4712277055,-0.5019993186,0.1468517184,-0.9671674371,0.0666483492,0.9021647573,-0.8958581090,0.8980016708,-0.1871165037,-1.4828082323,-0.5323514938,0.0343699008,-0.0326057225,-0.0458269492,-2.2640006542,-0.2263824195,-0.0426686592,-1.0875192881,-0.0749526247,-0.1680147648,-1.7522611618,-1.0544925928,0.5329548120,-0.4137636125,1.4957363605,-1.1262655258,-0.9431248903,1.1091076136,0.3187142015,-0.4425956309,1.9129506350,0.1657711565,-0.1740225405,1.1547033787,-0.4135216177,0.0847586095,0.6157559752,-0.2677897513,1.0562356710,-0.4857402742,-1.3100956678,1.5738106966,0.3299025297,-1.1636645794,-0.5389083624,0.3641978502,1.1415319443,0.3955402672,0.1479743123,-1.5591546297,-0.2825756967,0.9462650418,-0.6458616853,-1.2211798429,-0.6016332507,0.7985215187,1.0275365114,-0.0160687435,0.4915259778,0.1202829033,1.1910266876,1.3510931730,0.9124286175,-1.5193203688,-2.6604340076,1.6008164883,0.1068808660,-1.2210118771,0.8822169900,1.0649386644,-1.4589542150,-0.7992488742,-0.1713563204,0.4389387369,-0.2636434436,-1.6290132999,1.1303752661,0.1080426127],[0.8657753468,-0.9457086325,-0.5738193393,1.2187113762,-0.5768360496,-0.8167025447,1.2510757446,0.3992554247,0.2217383087,-0.6826551557,1.1338157654,-1.2801879644,0.9553549290,0.2114500254,-1.3725106716,2.0755689144,0.3925169110,-0.4047083855,0.5372302532,-1.0917904377,0.1228743196,0.1735812575,1.3295997381,-0.9710007906,-0.0175675768,0.6346111894,0.5424829125,0.4861537218,-0.5748277903,-0.4538476467,-0.6988639235,0.3595512807,-0.3014974296,0.4904040098,-0.4762947261,-0.2976291776,0.2722819448,1.8400909901,1.1112102270,0.3468321562,-0.7620072961,0.4823642671,-0.0431677178,-1.1301577091,0.7625041604,1.0072216988,-0.7397775650,1.2135487795,0.4348278940,0.2891020775,-0.2582736611,-0.4398835003,-1.1159187555,-0.4120377302,-0.3862161338,-1.0398606062,0.5330690145,0.0991028398,-0.3123979270,1.4060606956,-0.2284294963,-0.2856896520,-0.3457508683,-1.8428200483,0.7488943934,0.1384963244,-0.2578194737,-0.1127146631,1.1032286882,0.5590403080,-0.1274744719,1.1027128696,-0.3745149970,1.4169546366,1.0180193186,0.3566385806,-0.2235454619,-0.9186764956,-1.8063689470,0.6170316935,0.8247284293,0.1426244229,0.1843836755,-1.2132909298,-0.3551035225,-1.0385468006,-1.0946146250,-0.6682890058,0.0346868075,0.7390833497,0.6645213962,-0.6030436158,-0.5423620343,0.4705986083,-1.2859250307,-0.0684260353,-0.3965751231,0.0891710445,1.4725625515,-0.5956141353,0.2861987054,0.1781372875,-0.1194024906,-0.2217310071,0.9313843846,-0.6242418885,-0.0836989135,-1.6565767527,0.7462492585,-0.2159588784,0.2116937339,-0.2267422825,-2.0483932495,-0.0378425568,0.7382618189,-0.8901466131,-0.1636219770,0.1239845902,0.3381330073,-1.2129957676,1.0842293501,0.9588112831,-0.4454301894,-0.7255096436,0.5139003992,1.0043842793,0.4047205746,2.6048500538,-0.7641742229,-0.1398930699,-1.2959104776,-1.8625189066,0.8397760987,0.0908519179,1.7395669222,-0.4368506372,-0.4642670155,0.8948456049,0.0972507000,-0.5477635264,1.8051584959,-0.6752122045,1.8902174234,-0.2260762304,0.0208178144,1.9998450279,-0.6578727365,0.1301973611,-1.1438729763,-1.0635344982,0.3119478226,-0.4038535357,1.0320700407,0.0470034257,-0.3322484791,0.8958112597,-1.0423126221,-0.2093100846,-1.0931046009,-0.1595915258,-1.0094487667,-0.2817739248,1.2365338802,0.1999276727,-1.5332267284,-0.7521827817,-0.0498312488,-0.1254887134,0.2784412205,-0.4777686596,0.1808462292,1.7320671082,0.6485695839,-1.1794214249,-0.0287689120,2.1534762383,2.1387667656,-0.1862127781,1.3411992788,1.2107347250,0.9686575532,-1.3244681358,-0.3789440989,-0.5815922618,0.3594802916,0.5171622634,-0.6009837389,0.3175195158,-2.2224929333,-0.6074861288,0.2733509541,0.0802273899,-1.0692006350,-0.9597536922,0.0113089839,-1.3741334677,0.2265929431,-0.6278473735,1.2340617180,0.0882001072,-0.3142201900,0.2227886170,0.8545964360,-0.0908002779,-0.2843574286,2.4294245243,-1.6052583456,-0.2354816645,-0.7199215293,-0.6534951329,-0.1538714468,-2.6277620792,0.9282026291,0.4786596000,0.7063364983,-0.9769585729,0.5844084620,-1.4787604809,1.5954923630,-1.5470777750,-0.0792327374,0.7591784596,-1.0835412741,-2.8589627743],[-0.0519933216,1.1929847002,0.1627516150,0.9921510220,-0.1297453046,-0.5478489995,-1.8640022278,0.2769920826,1.2735897303,0.7470535040,-0.0304872170,-0.0428298526,1.0499415398,0.4667961597,1.2053909302,-0.0714367554,-0.5911626816,3.0160570145,-1.3844703436,-0.1397884190,1.4430978298,-0.6048903465,-0.7319226265,0.0735818520,0.1856525242,-0.3724898100,2.3749258518,0.4259150028,-1.3816146851,-1.0676062107,0.0282882024,-1.3178880215,-1.5067243576,1.7033506632,-0.3256493509,-0.3136684895,1.1504480839,0.3979339004,-0.8979489803,0.6665918827,-0.2177965790,1.7094330788,-0.0203922130,0.5821586847,1.4792309999,-0.0451435968,1.9510873556,-1.4385012388,0.6818209887,-0.6088625789,1.1092604399,0.2422867864,-0.4770748615,1.6948552132,1.1704604626,0.4564086497,-1.0953822136,-0.7527635098,-1.5625915527,0.1791021079,1.8587647676,0.0916229114,0.4258229136,-0.9329550266,-0.0671339333,0.1869338453,-0.1333879828,0.6261107326,-1.5891088247,-0.2642643452,1.1548100710,-1.3183220625,0.6512417793,0.9680756330,0.9950332046,-1.0700780153,-0.2943134904,1.7982032299,0.4225844443,-1.5542390347,0.2829584479,-1.2698465586,-0.8019006252,2.0648713112,-1.2976447344,-0.2495082319,-2.9228045940,0.5553191900,-0.6955707669,-0.0912849903,0.0168256983,-0.3094053566,1.1338710785,1.4277110100,0.6253098845,1.1908096075,0.2107233107,-0.7185921669,-0.2519544363,1.7649414539,0.6261631846,-0.2532690167,-0.4259355664,-0.3002924323,0.7058870196,-1.5119395256,0.6891105771,-2.2670624256,-0.3775601983,-0.7435219288,0.6637009382,-0.6702621579,0.2118441910,2.0264430046,0.2950162590,-1.9733177423,-0.6014866829,0.7231934071,-0.5663225651,0.2073399425,-2.2557966709,0.0920352787,-0.1102471575,-1.1182092428,0.8314582109,-0.6517904997,0.5727631450,0.6002388597,1.0145868063,1.2000041008,-0.3893697262,-0.2220023125,0.0439428948,-0.0033232714,2.1105318069,-0.7621703148,-0.1955248266,-1.0612522364,0.5935337543,-0.7442343831,-0.4376539886,-1.2502030134,-1.1209059954,0.6724943519,-0.6831957102,-1.3959691525,-0.3748416901,-1.3967071772,-0.6097630262,1.0784497261,-0.0811343193,-1.4515217543,1.7069116831,-1.4116988182,0.0189672094,0.0360026099,1.5870809555,-0.8664822578,-1.3608570099,0.5752806664,0.0327789560,-0.4369576275,-0.1481619924,0.5401315689,-0.5292395949,-3.2350654602,1.1074713469,0.9541392326,-0.6092355847,-0.6445279121,-0.1951347739,0.6513268352,-0.7532553673,0.3051186800,0.9178270102,-2.5410294533,-1.6669428349,0.4259666800,0.6748510599,-0.4968488812,1.1697164774,-0.1687025577,-0.6268527508,-1.2338035107,-1.3931992054,-0.3564191163,1.4220254421,1.0918205976,1.2700427771,1.5146982670,-0.3999033272,0.0118509382,-0.3351794183,-0.4617187977,-0.2531777620,1.2234048843,0.2188955843,-1.2126897573,1.0774984360,1.5508300066,1.3986558914,0.4946941733,2.1584038734,2.4771494865,0.5729361773,-1.1950562000,0.5807843208,2.3439888954,2.1803681850,-1.5197203159,-1.3456399441,0.0533661768,-0.7593796849,-0.4989078045,0.0440334193,-1.8984165192,-0.4864933789,0.6840640903,0.2822242975,0.7824342847,-1.0771510601,-0.5577027798,1.4288663864,1.0138487816],[0.4544593394,1.0341511965,-1.6534334421,-0.5851719975,-2.5957257748,0.6972800493,-0.2410648167,-1.1962827444,0.6448838711,-0.4857121408,-0.3649063706,1.0979299545,-1.5546585321,0.1901791841,-0.4854544401,0.0275086965,0.9509277344,-0.7999033928,-0.1402400732,0.0964038074,-0.1028219312,0.5689355135,0.3108150065,-0.3040729761,-0.4496122599,0.4359531999,-0.1692104042,0.0689164475,-0.7930421829,0.8997336626,0.7421265841,0.2194436193,1.0569325686,0.0017485305,1.0466918945,0.3023291528,0.3153903782,0.8032365441,-1.1121325493,2.6082203388,-1.3207836151,-0.3867725432,-0.3457061648,1.4187725782,-0.6497259736,0.7847685814,0.2821341455,-0.1602710783,0.0889612883,-1.4676092863,0.6674157977,-1.0673069954,0.2699068785,0.1724976599,-0.3715167940,1.0822068453,-0.7078635693,0.4691339135,0.3721350431,1.1494282484,-0.0420525447,-0.4953653812,-0.4252645075,-1.8390085697,1.0680310726,-0.0759820119,1.0406930447,-0.6761109233,-0.2665938139,0.3006088734,1.1225448847,-0.9760149717,-0.0417896174,0.0025571589,0.4015204310,0.7486844063,-1.3603100777,-0.8001003861,-0.4906891882,-1.0416312218,-1.0722851753,2.4610190392,0.2289647162,0.3938124776,-0.3345311284,0.5383934975,-0.2748134732,-0.6480213404,2.1349983215,0.1050079092,0.8258822560,-0.4955825508,1.7874985933,-0.3892707229,-0.7922250032,-0.3951547444,-0.0935009569,0.1839891970,-0.8543516397,-1.0464440584,-0.7042970657,-1.1806395054,-1.2209229469,-0.2441308498,0.6465824842,-1.5547673702,-1.7660431862,-0.4821627736,0.1407624334,-0.1678617150,0.9397349954,0.2312805504,-0.0280464403,1.6767039299,-0.9470214248,-0.9351007342,-0.9674376249,0.5599937439,-0.7846594453,-0.4185580611,0.1562634110,-0.5272597671,-1.4458227158,1.7874987125,-0.8945755363,-1.2059798241,0.2898784280,-1.5602114201,1.3168905973,0.4170904458,-0.5218951106,0.0991009474,1.1913528442,-1.0922662020,-0.8239323497,-1.3484790325,0.5477755070,0.6710631847,-0.4247452617,-1.6761363745,0.4697081149,0.0555586666,1.6238911152,-0.6668378711,-0.7418172359,-0.7404046655,-0.2535289526,0.7424023151,-1.4185131788,0.1802333444,0.1009450033,0.3009157479,0.1991927475,-2.3257699013,0.6526278257,-1.6105198860,0.7812666297,-1.1860648394,0.3073795140,-1.5439826250,1.0786151886,1.2680554390,1.1521686316,1.1935445070,-0.1329169720,-0.3085395694,0.0761978850,1.8443210125,-0.2638729513,-1.5977312326,0.8382647038,0.0859807804,0.1356368512,1.1917499304,-1.0192261934,-0.3402422965,-1.1250737906,0.2373281419,-0.5993966460,0.4661963880,-0.5152217746,-0.5071100593,0.5282868743,-0.1788586527,-0.1281480938,-1.3527747393,1.0704989433,0.5577857494,0.0059412615,-0.3464069664,-1.0306385756,-0.1926615536,1.6605831385,1.3970597982,1.7178465128,0.5896280408,-2.2436287403,1.4201089144,-0.6560614705,-2.3737287521,0.0326512717,0.2078212202,-0.0501976386,-0.3021792769,1.1992337704,-0.4503636658,-0.2515912652,0.1691752970,-0.0690513998,-1.7739503384,-0.0254569277,2.1342687607,-2.2958354950,0.4292617738,1.1680651903,-0.0823156610,0.6136156321,-0.1594337076,0.3683827817,-0.0694643259,-0.4721207917,0.3415882885,-0.6719304919,0.0745203495],[0.0926815197,0.7809058428,0.8021600842,1.2536708117,1.2047463655,0.8457165956,-0.2689283490,-0.0406811796,0.7122931480,-0.0020184009,-0.7400744557,1.5476503372,0.7627727985,0.6692178249,-0.5025582910,-0.3137260675,-0.2285013795,0.3727241158,0.3642551601,0.6032511592,0.8335736990,0.1891970634,-0.3147652745,1.5904812813,1.0627577305,-0.6074346304,-1.1208091974,0.5797903538,-0.5553388000,-0.7626169920,-1.4770214558,2.1922209263,-2.2505557537,1.1951607466,-0.6032582521,1.1439125538,0.7489135265,0.8095982671,-0.3108332455,-0.3310345113,-0.8971514106,-1.5992891788,-1.4045076370,-1.4514647722,-0.5385984778,-0.9099396467,0.7821102142,0.3216437697,-0.7002050877,-1.6994129419,-1.0463931561,1.0386457443,0.1906262785,0.0693022907,0.3480342925,0.6630804539,-0.9648827910,0.1472559869,-0.0458867997,-0.9605195522,0.2654193640,-0.4124167562,1.2029577494,0.1682241857,0.1582312435,0.6462152004,0.7105493546,-0.5076053143,1.2874636650,0.7914259434,-0.3501081765,-1.3159191608,0.9087712765,0.3375329971,-0.0894550458,-3.3134696484,-1.0573728085,-0.4799038172,0.1022162735,0.0140804453,-0.4237525165,-0.0029144140,-0.4657649696,-1.2013605833,-1.8574804068,-0.7658002973,0.2583065629,-0.7350707054,0.5241965652,0.5456067324,0.5849019289,-1.0737682581,-0.3010232449,-0.4734374583,-0.3524091244,0.4543142021,-0.4802360535,1.5676404238,-0.0989632905,-1.2484837770,-0.8684524894,-2.6856575012,-1.2519989014,-0.3295105100,1.1998020411,1.5729202032,0.9265779257,1.4141265154,-0.2236066461,0.9494342804,0.8590515852,0.1530281156,1.1916043758,-0.1836208552,-1.3414688110,0.8105289936,0.2061579078,-0.5158743858,-1.1998797655,0.8480063677,-0.9648107290,1.0120599270,0.1300519258,0.1588593423,-0.1458848864,0.4267154038,1.5206574202,-0.2062469721,-1.0819133520,-0.9290098548,-1.7997145653,0.7159577608,-0.5012516975,-0.9157263041,0.1787528396,0.0459793061,0.5049911737,-0.8993741870,-3.0976221561,0.7384113669,1.5854134560,1.1512668133,-1.4262179136,-1.2774196863,1.7157492638,0.8839167953,-0.6359609365,0.8234870434,0.8796422482,-2.3434367180,0.5187589526,1.3172487020,0.2944667935,0.2227121741,0.5968134999,1.2236213684,-0.4995519519,-0.5660246611,-0.6994220614,-0.3106907010,-1.2275322676,0.7521572113,0.3444543183,1.5920861959,0.2251924723,2.2333335876,-0.0396176800,-0.1746587604,-0.1322690248,-0.4289151430,-0.7689406872,1.2304511070,0.2261408120,0.6992877126,0.8450301290,-0.6699578166,-1.3206387758,-1.4513704777,-1.8766469955,1.6361585855,-0.3785747290,0.8501096368,-0.0773385093,-0.1470464617,1.6808183193,0.3025913537,0.2117206156,0.0503673851,0.2844220996,0.3684289753,0.9366687536,0.6129627228,-0.6902558804,-0.3202367425,-1.7368016243,0.1083192900,0.2266173512,-0.2204572260,-0.8055815101,-0.5609047413,-0.5874257684,-1.4272302389,-0.4451175928,0.4001646340,-0.5392788649,0.0617882907,1.2349941730,-1.5404955149,0.6322777867,0.8958539367,-1.2067916393,-1.0665080547,-0.6116001606,-0.6186541319,-0.4733894765,-1.3370724916,-0.4977751374,0.6712226272,0.0883457363,2.0432891846,0.0867749825,0.2891819477,1.6255686283,0.5093495846],[0.8634660244,0.7550491095,-1.4651743174,1.2748829126,1.3423838615,0.0865078941,-0.7244066596,0.8706378937,-1.7124751806,0.5211428404,-0.5917902589,0.5395089984,0.0224716794,0.4386703372,-0.8138235211,-0.2059317082,-0.9192074537,-0.0604479648,-0.4525226951,-1.9594230652,-1.5452584028,-0.2895898819,-0.0253855549,0.6934217811,0.3090851307,-0.2218995094,-0.0529302284,1.2924019098,0.9603215456,0.3587602675,-0.9552350640,1.1490943432,-0.0335796215,1.6779471636,-0.3259856403,-0.1739709228,1.5214538574,-0.6064954996,0.1615271866,0.6834179759,0.3536682725,1.0816570520,0.3024944663,-0.6590800285,-0.8109408617,-0.1566258967,0.4882825315,-1.2851366997,0.6194458604,-1.6078065634,0.0334874280,0.9258344769,0.3397276998,-0.0351016968,-0.8456002474,-0.4561108053,-0.8913973570,-0.1025945768,-0.2888323367,-0.2083857954,-0.1048913971,1.9896476269,-0.4569800794,0.0361020789,1.0713363886,-0.7877628803,-1.0399837494,1.3942734003,-0.4270233214,0.3786351383,0.9357964993,0.9731655717,-0.7636753917,-1.6540989876,0.8779069185,-1.4263557196,-0.0950654224,-1.5323438644,0.0343191847,-1.1123816967,-0.2708704770,0.1493589282,0.5749483705,1.4909499884,0.0755190328,-0.9530516267,0.6965177059,-0.6865991950,-0.9001799822,-1.1961323023,0.1245036572,-0.3526382148,-0.4453476369,-1.8010739088,0.5374770164,0.2574934065,-0.3085855544,-0.0640777200,-0.8079072833,-0.5141606927,-0.5505889654,-0.1982488185,-0.3675305247,0.4208337367,0.0965360552,-2.0656948090,-0.1829688996,0.2506989241,0.2846823037,0.3001767695,0.2364291400,-0.7418822646,0.5268235207,-1.3270870447,-0.8376921415,-2.0307159424,-0.4796929359,0.7163648605,-0.4497404099,0.0741514638,1.1273583174,-1.0137497187,1.3068083525,0.0346848592,-0.8894021511,-0.9632061720,-1.5303732157,-1.1514923573,0.8990085125,0.9828225970,1.2495281696,0.5501637459,-0.7472378612,0.8517258167,0.4913127124,1.1090178490,-1.0322618484,-1.5959858894,-0.3413833380,-0.1173982024,-1.6070528030,-0.2624657154,-0.6415647864,-2.6126191616,-0.5836005211,-1.0906150341,-0.8369953036,0.2202195823,1.8754501343,1.1174076796,1.0884023905,-0.2120194435,-3.0440869331,1.4379463196,0.1025658697,-1.5558474064,0.6945379972,-0.3602610528,0.0620819256,-2.5303983688,-1.4525141716,-0.0484901033,1.0190861225,-1.1029773951,-0.1772495806,-0.0186824985,-0.4372339845,-0.7663955092,-0.2264468819,-1.8660597801,-0.0174309351,1.2895250320,2.8766248226,0.5093661547,0.6372798085,0.3782185614,-1.2775412798,-0.9037016034,0.7714995146,-0.7808824182,-1.0733382702,1.5326451063,1.6875671148,1.1831212044,-0.3538039029,0.4219298065,1.1515035629,0.1067457125,-0.6593474746,-2.3907163143,-1.6215455532,0.0232353490,-0.4400199056,-0.0634604543,0.2586629093,-2.2663216591,0.6431171298,-0.6066202521,1.8293699026,-0.6815040708,0.5783654451,-0.8462299705,-1.7036626339,0.2688787878,-0.1152593791,0.0679218322,-0.4225247502,-1.4610382318,0.5065841079,-1.9560542107,0.4685810506,-0.6444819570,-1.4162796736,0.9209262133,-0.9962162375,-0.8262240887,0.9954686165,-0.6650307178,0.9970404506,-0.1991078705,-0.6442340612,2.0496962070,0.0618126243,-1.0454064608],[-0.6637465358,1.9695299864,0.1250195652,-0.7327752709,-0.1359323561,-0.4412856400,-0.3675518930,-0.2591361403,-2.5769243240,-0.5419714451,0.2259461731,-0.1193021312,0.7141500711,-0.6071059704,0.3218783438,1.9557578564,-0.2104028761,-0.6749036908,-0.4633462429,-0.5990371704,0.9030663967,0.6569451094,-1.5796002150,1.3999508619,-1.0496129990,0.3546217382,-0.0530159920,-1.1967575550,-1.5856877565,0.0531225428,0.8695508242,-0.3861188293,-0.7244033217,-0.1855974942,1.0887128115,2.3390982151,-1.7254819870,1.5232543945,0.3753961921,0.9166776538,0.2828798890,-0.3334253728,-1.2294572592,0.3655229807,2.0233829021,0.2879462540,-1.4131339788,-0.7401593328,0.6440915465,-2.2614407539,-1.4203243256,2.6897394657,1.0188958645,0.8922799230,1.5838291645,-0.3026787937,-0.4696189463,0.7466695905,-0.6594412327,1.2971545458,1.1126279831,0.0633644760,-0.5827593803,-1.2563552856,0.1899198294,0.8947163820,0.4806729853,0.0808907971,0.8153989911,-0.5255755782,1.4518349171,-1.5716581345,-0.5564303398,-0.1252691299,0.8958023190,0.7547018528,0.6371154785,-0.1164987236,-1.7575292587,-0.8542814851,-0.7686762214,-1.0782096386,0.6460056305,-0.7257015109,-0.3034015298,0.6748173237,-3.3844981194,0.1262123883,-0.1035634950,1.4074908495,0.5247564912,-0.3472731113,0.8271560073,1.0331898928,-0.6009578109,1.1177165508,0.5174444914,-0.2727499604,1.3153707981,0.4109996557,-0.1952133924,0.3007640541,-0.5758360624,-0.5602527857,0.3082463443,-1.3120938540,1.7105407715,-1.5808700323,-0.8947734833,1.3084439039,0.5117423534,-1.4647477865,0.0225376338,-1.7247672081,0.4059568346,-0.5297921300,-0.4586297572,0.5186428428,1.1221458912,1.1941851377,0.7090333700,-0.1947846115,1.9539064169,0.2526306808,-0.7416341305,-0.3133064806,-0.3473016918,-0.0026028380,-1.0973331928,-1.1440614462,0.5836762190,-0.4917782247,2.3554165363,1.5738666058,0.5428704023,0.0142914858,-1.2510061264,-2.2476670742,2.6870701313,0.8242061138,1.8147623539,-0.3681277633,0.8724951744,0.4893138111,0.4355538487,1.4844454527,-1.4029507637,0.2526011765,0.1869145930,-2.3647630215,-0.4800653756,0.9991058707,-0.5212966800,0.1787936091,0.2968055010,-0.9514818192,0.7149151564,0.4679512978,-0.2365171462,-0.0139078815,-0.1421744227,1.4185181856,-0.8405714035,0.1105919480,-0.5550268292,1.3210300207,0.3122994602,2.4679713249,-0.6052124500,1.2478696108,0.0954830274,0.9173564315,-1.9533030987,-0.9786008000,-0.3256475627,1.0131014585,0.1920856088,0.8135458231,0.5758315325,2.8805861473,-0.5539932847,-0.1677597016,-1.1043391228,1.4870136976,-0.3989987969,-0.7816363573,0.9050074220,-0.6455032229,-0.4766314626,1.0585227013,0.3848082721,2.5556023121,-0.7868890166,-1.9005753994,-0.7796630263,1.6994067430,-0.8025813699,0.5371515751,0.4411693215,-0.1469948888,0.5937262177,-1.2914032936,-0.0362044945,0.7089365125,2.9356086254,0.0261324681,-0.0169340111,-0.0850357786,-1.1811912060,0.5618770123,0.1779046506,1.4448437691,0.1256714612,-0.9184864759,-1.1742694378,1.0258781910,-1.4049621820,-1.8019059896,1.6122846603,-0.9469008446,-0.9792156816,-1.0730874538,-1.9244215488,0.1242880449],[-1.3025438786,0.9926157594,0.0525438972,0.1790150851,-0.8310004473,-0.1794541776,-1.1104866266,-0.2170728147,-0.4860383868,-0.0960857049,-0.4243250787,-0.3056462705,-0.0738886967,-0.2791997492,-0.1564156264,-0.8399229050,1.1458208561,0.6837478876,1.6160286665,-0.1099802181,0.1030385345,-0.4512418509,0.8318041563,0.9506053329,-1.7757173777,-1.9854574203,0.3565828502,-0.3700554073,-0.6152406931,-0.9687829018,1.3810563087,0.2320157290,-0.0397417285,-0.4070185125,1.0224226713,0.6736516356,-0.1206513643,0.3238360882,-0.7017626762,-0.0897023901,1.4467226267,-0.7196639776,-2.5757844448,0.3970977664,-0.2733588219,-0.3398293257,-0.2725647688,-0.4187522829,0.3255429864,1.5826820135,-0.5754086375,0.0943736285,-0.4439761937,-0.1710792184,-0.4510376751,-0.0273558944,0.2152557671,1.4247077703,0.0805678144,-0.6716182232,-0.9144644737,1.0538321733,1.6822096109,0.2016815394,1.2589432001,0.4971260726,-0.4490357339,1.7315534353,-0.4510636628,0.1474699676,0.9106471539,0.1625524759,-1.4341237545,-0.7749769688,-0.3517271876,0.8197718859,-1.1186920404,0.6920174956,-0.4592382312,-1.0409634113,1.7210446596,0.1002967730,0.0334235467,0.9438540936,0.5847578645,0.4156909585,-0.7136794329,0.5987677574,-1.1312581301,0.4730856419,-0.9133135080,-0.4913589358,0.1282602251,0.2038312554,1.4765167236,0.5084848404,-0.2917088866,0.3517220020,0.6517812610,0.6369057298,-1.2051205635,-0.6937068105,0.0933035389,-1.1256998777,-0.8713313937,0.6497979164,1.6654264927,-0.2665706575,0.9316011667,1.0475007296,-0.4163180590,-0.2547314763,2.2949197292,-0.7823335528,0.4881402254,-0.5845407248,-0.2045453191,0.1332875192,-1.0624183416,-0.7866970897,-1.6789759398,-1.4453302622,1.1875790358,-0.9976070523,0.7905008197,2.6770126820,-1.5829631090,-1.4054129124,0.2432076484,0.5539929271,0.9461371303,-1.2753396034,-0.4961832166,-0.1548965722,-0.4539283216,0.2310051918,0.1122577861,-0.3159879148,-0.1546914428,0.9573659897,-0.9810982943,-0.0313256383,2.7361254692,1.1906349659,-0.1153252274,1.1036010981,0.1202579215,-1.8402842283,-0.9810041785,1.6234278679,1.7131366730,0.9054279327,-0.0411528088,-1.1291643381,0.9142575860,-0.1347265393,0.7485822439,-1.1155381203,0.8338834047,0.0044410489,-0.1199790686,-0.2514708042,0.2934603393,0.0102223698,-0.9591860771,-0.4407611191,2.4567036629,-1.4015727043,0.3956102133,-1.2079521418,-0.3189548254,-1.0416499376,0.3006887138,-0.2559400499,-1.5648876429,-0.5479289889,0.9294615984,-0.7143161297,-1.3360306025,1.2542421818,1.6548587084,-0.7020130157,-0.0768703371,0.0540100858,0.7866579890,1.0969753265,1.6252093315,-0.5440499187,-0.7300460935,0.5121057630,-0.1335268468,2.5226573944,-2.0427842140,0.1997319609,-1.1326947212,-2.2240474224,1.7012556791,1.6482611895,-0.0981576741,0.6183930039,0.7943008542,-1.0404632092,-1.2540209293,1.0974723101,0.3400676847,-1.7999849319,-0.5149138570,-0.1294233352,0.2476343811,0.8727270365,0.0723300055,0.2251117826,1.4213521481,0.1532366574,2.6486074924,0.1167246997,-0.4805921614,0.3937732577,-0.9335401058,-1.4705386162,1.1323963404,0.0321465358,0.9102234840,1.3990534544],[0.4905488491,1.0244057178,-1.6630067825,-0.3994784951,-0.3401413262,-0.3454761505,-0.0475850701,0.4195607305,0.1365256459,0.0498067103,0.4276661575,-0.5091152787,1.7417982817,-1.5653539896,-0.1948224455,1.4137815237,-0.3796446621,-0.6055162549,-0.4563442171,0.9318701029,2.0537283421,0.7551569939,0.1563235372,0.4450795949,0.2379143834,-0.8415094614,0.1656335741,2.3153066635,0.6370079517,0.6397714615,1.8445942402,-1.9889305830,-0.8778341413,-0.2747113407,1.4435391426,0.3957173228,-0.1096090600,-0.8891395926,-1.2789076567,0.0116699347,1.0117725134,0.3835023642,-0.5092669725,-0.0680650771,0.9934516549,-1.1740164757,0.1203645915,-0.2318346053,1.2257432938,0.1388643533,-1.1649215221,0.5175055265,0.2016887665,-0.3390520215,0.6372504830,-0.8617182970,0.3922643661,-1.1405359507,0.2569530010,-0.4474111497,0.0448078550,-0.2087467760,1.6296164989,-0.0296883136,-1.2102688551,0.5677373409,0.0009645627,1.3742536306,1.1661435366,-1.1338286400,0.7868994474,0.4527561665,0.2778838873,0.6215717793,-0.9352455139,-0.3557741642,-0.2322310805,-0.9921100140,-1.4324650764,0.6023430228,-1.1864603758,-1.0976997614,0.8242604136,0.1729867607,-1.1119036674,0.2921111584,0.0507399924,0.6631184220,1.2487870455,1.4756360054,-0.7475041747,0.5366319418,0.1717128307,0.4651950300,-0.0540710352,-0.2115193903,-0.2900735438,0.7111625075,-0.1079788059,-1.6012848616,0.3743604422,-0.0169442687,-0.9762680531,0.3801927269,0.1165735647,0.4455650449,0.1271342188,-0.5844639540,-0.3766897917,0.5482994914,0.2216329426,-0.5788456202,1.3263802528,0.6550196409,-0.7106993198,-1.1114065647,-1.2377024889,-0.9838712811,-1.2531896830,0.4703316987,0.0795856193,0.1470217109,-2.2242867947,-0.6280842423,0.3862711787,-0.2358767688,-1.0647182465,-0.2574432194,-1.4735280275,1.1477354765,0.8785506487,0.6193646193,0.0978459045,-1.0593994856,-1.1262053251,1.2516567707,-1.4200671911,-0.3345801234,0.2187118828,-0.1080835685,-1.2882615328,0.7958784103,-0.6218714714,-0.2906329036,-1.4599126577,-1.1279275417,-0.9053555131,-1.0980588198,0.1504440308,1.1394290924,0.3642742634,-0.1579071581,-0.9825789332,-2.1468319893,0.0083792591,0.7393897176,1.4622263908,0.1394752264,-1.5417640209,-0.2361054122,0.7392612696,0.8448073268,0.5637169480,0.0669474825,-0.0490082353,-0.1221956983,-0.0868485868,0.0036756690,0.0283785351,0.9145377278,1.3506923914,0.2616867423,2.4223361015,0.3500064015,0.5549456477,-1.2616983652,0.2947146297,0.8523635864,1.2621779442,1.0800333023,1.5876103640,-1.9408265352,0.2570088506,1.7748100758,-1.6168836355,-0.9866812229,-0.0237126984,1.4818791151,-0.2154493928,0.0660840422,-0.6549353004,0.0116032315,0.4643039107,1.3120938540,1.0078448057,0.4175274968,1.7760876417,-0.2337876260,-1.1625869274,-0.5908479095,1.1094672680,-0.5619422197,0.2336943001,0.5299299359,0.7005144358,0.9736207724,-0.4927281737,1.4668943882,-1.0239930153,-0.0770049617,0.7348200083,0.2523647845,-0.2476083785,0.9233145118,-0.0127188815,0.7690323591,1.3147650957,-0.0070816535,-1.9878875017,-1.5712317228,0.7105522752,0.5278979540,0.2327578664,-1.2572900057],[-0.1001996845,-0.1817302555,-0.3798861802,2.3356816769,-1.6914775372,-1.1774146557,1.2655371428,-1.5351748466,0.7914337516,-0.6201122403,-0.3057745695,0.1126926094,0.6709082723,1.1037712097,0.1790883690,0.6736957431,-0.4431216717,0.8462934494,-1.1090414524,1.5360138416,-0.6972581744,-0.4059306085,-1.8871546984,2.0013945103,0.0334238783,-0.2949000001,-1.3512518406,0.6522342563,-0.7653886080,2.2844948769,0.8337475061,0.3221344352,-0.5305902362,-2.4431941509,0.8988071084,-1.0610204935,0.8465533853,-0.1923000813,-1.3131548166,-1.5066918135,0.6025505066,-0.8746331334,-0.3928633332,1.1593921185,0.1034096554,-0.3026155829,0.4849913418,0.1135137603,1.6720339060,-0.3694227338,-1.4470843077,0.7193400264,1.0495063066,0.0959530175,-0.6709577441,-0.3056570888,-0.6415513754,-0.5251059532,-0.6818318367,0.3147960603,-0.4887236953,-0.9638143778,-1.1264886856,-0.4230669439,-1.3753056526,1.4559468031,1.4765875340,1.2504413128,-0.4936591685,0.3515748084,0.2022776455,0.9292532802,0.8853971958,-0.6428562403,0.2419474721,0.6410183311,-1.7333700657,0.9614998698,-1.3203852177,-1.3492267132,0.1798286587,-0.5843139887,-0.3930367231,0.0276744086,1.1229655743,1.2574152946,0.5836015344,-1.8359272480,0.0817378908,1.3239192963,-0.4715490639,-1.7266557217,0.3238314688,-0.2207290232,1.0489692688,0.9688036442,0.4010276198,-0.3819698393,0.1248324960,-0.0166277848,-0.9547690153,-0.4959850311,-0.5902844071,1.8655183315,0.0865403786,-0.6640671492,0.5868545175,-0.2740457356,0.6969856024,1.4407171011,-0.1463461518,0.8920519948,0.3390126824,-0.0442162864,0.4553397000,-0.6127408147,0.3977233469,-1.7088475227,-0.1218002737,0.1388552785,0.4274210334,-2.0180900097,-0.1601694822,1.6350325346,-0.4815888107,1.4442528486,0.1157166064,0.2844043970,0.1153378934,1.5692323446,-0.2419142425,-0.5717042685,-0.1694380194,0.8772971630,-0.6982527971,-1.3658602238,-0.7143605351,-1.2969588041,0.3269412220,0.8335607648,0.4132811427,-0.0035941750,0.5502625108,-1.2089297771,-0.4677336812,0.4932954907,-0.2773734927,0.3793223500,-0.2142858654,1.4164570570,-0.0279256701,-1.4179240465,0.9715839028,0.3082867861,0.3752784431,0.9423430562,1.3675347567,-0.6815214157,0.5373745561,-0.4753558040,-2.2577373981,-0.1867559999,-0.4563869536,1.1202806234,1.1302267313,-0.9389406443,-0.7574245334,0.5267103910,-0.8715178967,-0.1430505812,-1.9246406555,1.8750369549,-2.5997474194,-0.8381543756,1.5143324137,-0.5030655861,-0.5764772892,1.0302103758,-0.3347219825,-0.0474841893,0.0351716802,-0.8788820505,-0.7732199430,0.1133688316,-2.5959587097,0.0701987743,-0.9390360713,-2.1080806255,1.2983478308,-1.7246512175,-0.6754827499,0.1538365036,-1.6606847048,-0.6705852151,0.4778603315,-0.4059040546,-0.4880833328,0.2577357292,-0.1147083938,-1.2455481291,-0.5508422256,0.3719415665,-0.5658921003,0.5245736241,-1.1117680073,0.0928273723,0.0218124297,-0.3010913432,0.8775908351,-0.2169012874,-0.8590472341,-0.3291174471,-0.5996320248,0.1607422233,0.4372586012,1.5891768932,-0.0624184571,0.7525455356,2.5343019962,0.7479329705,0.8544104695,0.4639654458,-2.2253143787,-1.0417768955],[-0.0010615842,0.2537246644,-1.0748190880,-1.3404649496,-0.2640745640,0.3361747563,0.2933667600,0.7682360411,0.1233829632,-0.7152925134,0.7858465314,0.2019590288,0.5854577422,0.9132457376,-0.3124257922,-0.6049257517,-1.7231265306,0.7158805132,0.7621665597,0.4707882106,-2.9688909054,1.0587520599,-0.1631650478,0.0793522522,0.1336528510,1.0867087841,-0.5280027986,0.2855147421,-0.0667707622,-0.0035638765,0.4216847718,0.5443183780,0.2461861372,-0.1809778661,-0.8805108070,-0.1438997090,0.3478353322,0.8228884339,-0.5667278171,-0.4600711167,0.6176699400,2.3540959358,-0.8628973961,0.9088322520,0.6932885051,1.0956834555,1.0052500963,-0.4408296347,2.0047576427,-0.9447867274,0.9418581724,-2.8230834007,-0.3410270214,1.3397841454,2.1751554012,-0.0430340357,0.4309962392,-0.0155594079,-0.6336702704,-1.5577777624,0.1310043186,-0.4537871182,2.4866731167,1.1449908018,-1.5187022686,2.0298993587,-1.5842804909,0.2514736354,0.9716297984,1.7878285646,-1.2653026581,-0.1265586317,-0.1470164657,-0.7129474878,0.4800057709,2.8610293865,-0.1240404844,1.0638287067,-1.5669780970,0.8731978536,0.3589456379,0.2653459311,-0.5578699112,-0.6540279984,-0.2132600099,0.0867705345,1.2787603140,1.3769998550,1.2984076738,-0.0174389817,-0.0729143023,2.0917215347,1.5554282665,-0.5141366720,-1.5810707808,1.3129007816,-0.3244799376,0.7690106034,-1.3313903809,-0.1335199028,-0.1562579721,-1.2194261551,-0.0974660292,-1.2154504061,-1.0153028965,-0.9581129551,1.4123982191,-0.1402374953,-1.2557382584,0.6547852159,-0.9383400083,1.0114247799,-0.7177949548,0.8543280959,-1.2188569307,-0.4752506912,-0.6735640764,0.0805893838,2.9027316570,0.0258577615,-0.2556987703,0.0209448729,0.2968816459,1.4511637688,-1.0195590258,0.3860342205,-0.2076390535,-0.7492733598,-0.2612061799,0.2322777659,-0.0773505419,-0.8092628717,-0.3922980130,-3.0591294765,-0.7700003386,-1.5647056103,-1.1685365438,-1.1834771633,-0.7745651007,0.6517621279,-0.6666411161,-1.2065341473,0.8831382990,-1.3153542280,-0.0880349576,0.5406996608,-0.2709668577,1.2457705736,-0.4655426145,0.5110310316,1.6702328920,0.0507460684,-0.6514158845,0.4647938311,0.5530887246,0.1307623982,0.0550314337,0.3614056408,1.2339098454,-0.5118358135,-0.1313847899,-2.1826581955,-0.1817007065,0.0118881213,1.1555757523,-0.5332305431,-0.7610717416,-1.1782858372,-0.1647985727,0.6561669707,-0.7847946286,0.1190716103,1.7935775518,-2.1962480545,-1.1013448238,-0.8153190017,2.0948555470,-1.3238399029,1.0909674168,0.7659215927,-1.5969687700,-0.5738921762,-1.4300869703,0.7761298418,0.1169549376,-0.2567743063,-1.3555506468,-0.0286894329,0.3896198571,0.1592593044,-0.1276924163,0.7255066633,0.8067373037,0.9591949582,-1.6959720850,-2.4361855984,0.6476137042,-0.7000209093,-1.7535645962,0.0522507057,-0.9426590204,-0.1567277461,-0.2495714128,0.3145912290,0.7321646214,-0.2300930619,-1.1882681847,0.7567175031,-0.7333874702,1.4053243399,-0.3211281300,-1.7157675028,-0.7250712514,-1.2069982290,1.3195250034,0.2129811645,-1.8306376934,0.0801075250,-1.1997556686,-0.3797031641,-0.1439069211,-1.3755490780,-1.9295804501,1.0497505665],[2.1053328514,1.3196328878,-0.0395816118,1.2614026070,-0.0005954857,-1.2713522911,-0.1604739726,0.4342354238,-1.0333087444,2.0633223057,1.0521577597,0.3827154636,0.7366225719,0.4496108294,0.0940372646,-0.7754995227,0.1078162566,-0.7633399963,0.4329693019,-0.7188794613,0.6669287682,1.4806238413,-1.2925173044,0.3922462165,-0.1607157439,1.3540416956,-0.2693899870,-1.8579820395,0.0982898772,0.9795159101,-0.0613298379,-1.5851001740,2.1930634975,0.8165221810,-1.4409997463,-0.8420747519,0.8619980812,0.3620576262,0.0261950083,0.1866548657,1.8038645983,-0.2240944952,0.7091770172,0.7871082425,-0.6826594472,0.8542404771,-1.8168696165,-0.7128713131,-0.0538153574,0.1904992610,1.2892400026,-0.9406535625,-1.7142122984,-0.4913613498,0.6450148821,-1.1126114130,-0.2288775891,-0.4744396210,-1.3762102127,0.5254834294,-0.5287557244,-0.5262985229,-0.1062486768,-0.6348058581,-0.9365763664,-0.1497449428,-0.1949767023,0.2172982693,-0.3022725284,0.8047858477,1.3903118372,1.3883538246,-0.7042567134,-0.1121719480,-1.3455646038,0.3870194852,-0.0965950713,0.9518412352,-0.1091744900,-1.1043176651,0.6279343963,1.4129952192,0.5325927734,-1.6628987789,1.5105077028,-1.2438029051,0.8583120704,0.2078500837,0.5437254906,1.4735174179,0.2169694006,-0.1485404968,-0.7130903602,1.4223324060,-1.0756270885,-0.9242228270,-0.7942515016,-0.6728418469,-0.6729523540,2.3121747971,-0.0744164959,-0.2179881036,-0.4866858423,-0.8224667907,1.1551148891,0.7958337069,-0.3938231468,0.2431697249,-1.9529821873,-0.2154482901,-0.2813555598,0.0592441037,0.2069272697,1.7651711702,-0.3149431348,1.1901835203,0.9350205064,2.1286346912,-0.1006811112,0.0827367827,-0.6280285716,-1.2208617926,-0.6480045915,-0.6513798237,1.3888345957,-0.0509894826,-1.5703502893,0.5715800524,0.9913051724,-0.1888523996,-0.3661287725,-1.8164740801,1.6539181471,0.2706396580,-2.3285901546,-0.6339191198,1.0078198910,-0.2167435288,2.8713746071,-1.0747338533,-1.0448617935,0.4725149870,-0.4177502692,1.1482754946,0.7925708890,1.6601495743,0.0192887355,0.2908635139,0.2582688928,0.7782739997,1.9146323204,0.7430780530,0.0700267404,0.0750506967,-0.8305863142,0.3035894930,0.2465319186,-0.2787216604,0.5338457227,-0.6506549716,0.9350888729,-0.1797624230,0.7599012256,-0.3284780979,-0.8421573043,0.1916376352,1.1291347742,1.5996108055,0.8662851453,-0.1602092385,1.8883359432,0.3574674428,0.2051007152,1.5985630751,0.8055735826,0.7127355933,-0.3533496857,-1.4823664427,-2.0089395046,-0.9939237833,-0.6550972462,-0.0262048952,1.2165206671,0.1078934073,-1.3689708710,0.1513076723,-0.4456702173,-0.0554245599,0.7030171156,-1.2767373323,-0.7908762097,0.2892021835,-1.2196539640,2.0823202133,1.0406847000,-0.2916181982,-0.3322509229,-1.2407140732,-0.1473717541,0.5597736239,-0.3266303241,0.8758299947,-1.4328693151,1.1302080154,0.9464007616,1.0138847828,1.8907960653,-0.7582595944,0.1726988256,0.1735392958,-0.5599799156,1.8208374977,-0.7746500969,-0.6911838651,0.0445401333,1.2483352423,-0.7943223119,0.0519079305,-1.5848653316,0.8621872663,1.2930183411,2.7784893513,0.3236986399,1.4388288260],[0.7258377075,-0.8048277497,0.2800495028,-0.0499605834,1.3089578152,-2.4130134583,-0.2753242254,-0.7478988767,-0.6888538003,-0.8414610624,-0.8301863074,-1.1255892515,-1.0282196999,0.5207557082,0.2261744589,0.3474826515,-1.0355030298,2.4439811707,-0.8515596390,-0.1384771913,-1.0451205969,0.6859315038,-0.9893190861,1.7343578339,0.1348031312,1.1066228151,-1.0953452587,-0.1847550869,0.2814736664,-0.6099718213,1.6427118778,-0.1525585204,-1.4401333332,0.5616949797,0.7036054730,-0.3839696050,0.0347902104,1.3739222288,-1.3430659771,-1.5973757505,0.3135956228,-2.0644445419,-0.5595622063,1.2138389349,-0.0590524115,-0.2654310167,-0.0230365172,2.3398439884,-0.4614900947,1.2380152941,-0.2797059715,1.4746888876,0.2444090098,1.7653659582,-0.4687747955,-0.3881003559,-0.3427613378,-1.1154838800,-0.9173331857,-1.2664217949,0.8875035048,0.6352165937,-1.1627904177,-0.6139476895,0.5397555828,-1.8344066143,-0.2994301021,0.2945496738,-0.2418930382,-1.9963649511,0.3640803993,-0.1844701171,-0.6950638294,-1.4857170582,1.1242765188,0.7085320354,0.2723022401,-1.5546014309,0.1953181177,-1.5684660673,-0.6086485386,-1.3542300463,-0.4276607335,1.4777511358,0.0993386284,-1.8139836788,-0.7365110517,0.6629828811,0.2414772660,0.5044490695,1.0643922091,-0.1175867841,0.0788819045,1.0972431898,0.2484007776,-0.0112973209,0.7241719365,-1.8626170158,0.0347400159,0.1793256998,-0.8870456219,-0.0319259167,0.8181232810,0.6993884444,1.2714951038,1.4938253164,1.2755190134,-0.7280711532,-0.0053603342,-0.6611225605,1.2958151102,1.1125426292,0.0500253364,1.4127457142,1.2092069387,-0.6113958955,0.5983562469,-0.7948383093,-0.0255965069,0.5676263571,-0.4503745735,0.8968400955,-0.5737510324,-0.8267991543,1.5909577608,-1.5717008114,1.0053443909,-0.1096132845,0.4106061757,-1.1032598019,-0.0910399705,-1.2157471180,0.9933787584,0.3237740099,0.1134621352,-0.1503658146,1.2782334089,-0.7552730441,-1.2234134674,-2.0203626156,1.0978006124,0.7394206524,1.2319457531,0.5734107494,0.4472402632,-1.9491572380,1.7882524729,0.7341278791,-0.3616535962,0.6778038144,-0.5595866442,-1.1381254196,1.4615677595,1.6242483854,-0.5140312314,-0.2640652955,-0.0112548228,-0.3889789283,0.6068689227,-2.8274755478,-0.0085445140,0.4904544950,0.1616977453,1.0789258480,-0.0618563630,1.7721537352,0.3887664080,0.9643759727,1.4623794556,0.1064803451,-0.8805568218,0.2923339605,0.9559921622,0.3187932968,1.0508491993,-0.2954924107,-1.0405685902,-0.0598033965,0.9603899717,2.1053576469,1.3079080582,1.2512524128,-0.6764339805,0.4047291279,-0.8339336514,1.8518990278,0.9332556129,0.7688630223,1.9514070749,-0.3171782494,-0.4678969979,-1.0559155941,2.0887012482,-1.8049836159,0.0430028215,0.3988151848,0.4500122964,-0.2574366927,-0.3913790882,-0.6121475101,1.0972211361,-0.0593496226,0.3291324973,-1.7988839149,-1.0083388090,-1.6112517118,0.1174412668,-0.3404005766,-0.4726842642,-0.4491213262,-1.0615438223,-0.2107420564,-0.3817735910,0.2995887995,-0.9482329488,-0.2399285883,-1.4830079079,1.4051352739,0.8220074773,1.0181708336,0.7705841660,-1.4086661339,0.1062365994,-0.3218088448],[-0.0925237611,-1.0620304346,1.6652784348,0.0058519463,1.4551757574,-1.4438506365,0.2192535847,0.3335919678,-1.2820657492,1.2886893749,0.3952491581,1.1653424501,-1.0537530184,-0.0074237022,-1.2312995195,0.4738241136,0.2731259465,0.7465436459,-0.6233760118,-0.2805426717,1.5110511780,1.1256532669,-1.1806602478,-1.3495339155,0.3354279399,-0.1391374320,-0.8745224476,-1.7112647295,-1.4185328484,0.1687638760,1.9717078209,1.1529743671,0.6822891235,-0.4956736565,0.0578047819,-0.7183652520,0.5551437140,0.3473408520,0.4653676450,-1.8341126442,2.2764511108,0.4477431774,-0.6344168186,0.5948296189,1.5689496994,-0.1255524755,0.9398848414,-0.4700069427,-0.0290891398,0.9742378592,-0.8186925054,0.4190974534,-1.6216185093,-0.3389663696,0.5089208484,1.5303941965,0.3098924458,-0.7619076371,0.3587762415,-1.3091235161,0.8170292974,-0.6985152364,-0.1834008396,-0.1097707674,1.0684025288,1.3673287630,0.0024511134,-1.0490896702,-3.0076134205,1.4514784813,1.2511434555,-0.0860626996,0.5120401382,0.6248494387,-1.5006062984,1.8949669600,-0.7988373041,0.0659881309,-0.2443456352,-0.5105406046,-0.2946806252,0.8995938897,-0.6719914079,-0.9645525217,0.4793780148,-0.3133210838,-1.4314814806,-0.9944772720,-1.4454166889,0.0671993122,-0.5245435238,-0.3248140812,-0.0862735212,-1.2868779898,-0.2069968730,-0.8418411613,-0.8098770380,-0.0245797615,0.2288850546,0.4888761342,-0.9548777342,1.2134466171,1.6297017336,1.0540798903,-3.5775067806,-0.5092924237,1.5152270794,1.3439284563,-1.3064225912,0.5239573121,0.6371377110,-0.5815712214,1.4705703259,0.2761515379,-2.2635154724,0.8119996786,0.2082707137,-0.3286717832,-0.2771964967,2.2118780613,-2.1735358238,0.1249354333,1.4037474394,-1.4499368668,0.3003337979,-0.5304105282,-1.7573840618,-0.6856645346,-0.5914506912,0.7638018727,-0.2675991952,-1.0980226994,-0.4089368284,1.1221373081,-1.0884951353,-0.6319181919,-1.2549594641,0.7170437574,0.0045071896,-0.6356463432,2.3410551548,0.8706154227,0.5790865421,-0.8792712092,-1.1271067858,-1.8982055187,-0.6447094083,-1.5107764006,-1.7840856314,-0.5934742689,0.6970037818,0.8130237460,1.6075373888,-0.1049850583,-0.2260528356,-0.8203536868,-0.2570720613,-3.0439462662,-0.8317464590,-0.0221328903,-0.7338784933,-1.1955124140,2.2275626659,-0.6152293086,1.0770606995,-0.8655061722,1.4497618675,-0.2080348730,0.0465875715,-0.6258221865,1.1146214008,-0.4552742243,-0.5624456406,0.8417857885,-0.5142645240,-0.2130188048,0.2848901749,-0.4078902900,1.4543926716,0.9406331182,-0.4867027402,0.0767036080,-0.2662543356,1.5841546059,-0.0371828750,-0.3927924335,0.5836333632,0.7966336608,-1.4907039404,-0.7028891444,-0.7504969239,-0.7478713393,-0.9128293395,1.1214281321,-0.9350854754,-0.4669199884,2.3918759823,-0.2142266184,1.8552978039,-0.9463956356,0.4829190373,1.2582088709,0.4328311682,1.1292302608,0.3805397451,1.2985144854,0.6846257448,1.6851937771,1.7153955698,0.8266667128,0.3266970515,-0.9034817815,-1.9895290136,-0.1561795175,0.7916913629,1.2313185930,-0.0866586864,0.8667936325,-0.9921289086,0.2599111497,-0.5031987429,0.2345968187,1.1069636345,-0.6858910918],[0.6509312987,0.8513160944,-2.5334670544,-0.0781184584,0.6531443000,0.2202208340,1.1924302578,-0.0056578512,0.9925175905,0.4677110612,0.4470631778,-0.7054750323,-0.3973045945,-0.4325635731,-0.6860013008,1.0001733303,-0.9480412006,-0.8789937496,0.6653664112,-0.6843322515,0.8633398414,-0.5817058086,-0.6112054586,0.8448106050,0.6149102449,-1.7258707285,-0.2807382941,-0.1708089411,1.5577105284,-1.2645984888,0.9760804176,0.5516930819,-0.5967000127,-0.3854245543,-0.1739615947,-0.0207591373,-1.2616332769,-0.4005171359,0.3221673667,-0.2869417071,1.0295534134,0.1294983327,0.6947427392,1.2161408663,0.9053342938,1.4678857327,0.9029391408,0.5970921516,0.7944867611,0.4798160493,0.5198258758,1.2812861204,-1.1480890512,-0.0881863534,-0.2476950139,0.4717128873,-0.6225486398,0.2505447567,-0.6700253487,1.4420573711,-1.6963062286,-0.7611740232,0.6617693901,-0.0086765150,0.5258134604,-0.1621825248,-0.6918660998,0.7783997059,0.0281068888,-0.1641781330,-2.5166752338,-1.4063049555,-0.4924475551,-0.6959780455,0.9172568917,-0.5700587034,-1.9835411310,1.5791540146,-0.4925122857,-0.0458450317,-2.0372576714,-0.9087883830,-0.6633548141,-0.7705920935,-0.3299379647,-1.0611827374,-0.1742917597,1.0259675980,1.1829209328,-0.9411280751,-1.0046323538,-0.7954505086,-0.4666687250,-0.1476753652,2.2875802517,-0.2079681754,-0.2876192033,1.2012648582,0.6918829083,0.0862621292,-0.7218672037,-0.0940898657,1.8701169491,-0.0919047073,0.4451364577,0.5785190463,1.0394281149,0.4340084791,0.6246455312,-1.7443118095,0.0707430169,-0.2417886257,0.6761582494,0.2193481177,0.0652367175,0.2281389534,-0.1589985937,-0.0161593799,-1.1481174231,-0.8527640104,-1.9987218380,-2.2161722183,-1.1947005987,1.8645793200,-0.5195894241,-0.7349023223,0.1043193266,-0.7930381894,-0.5610723495,0.9559897780,-0.1279041469,-0.0052697291,-0.2292125374,-0.8965917826,0.1681797057,-0.5898171663,-0.3136174977,0.3532792330,-0.8734504580,-1.3299645185,-0.7033492327,-2.1503171921,-1.2717120647,-1.0888218880,-1.3648161888,-0.1491083056,0.5447972417,-1.7824438810,0.6282010078,0.1593699604,-2.5262157917,-0.7317644358,0.6677094698,-0.2354516834,-0.7564174533,-0.0117295058,1.3500658274,0.6012318134,0.6721284389,0.4980325699,-0.7003022432,0.5756027699,-0.5126936436,-0.0571699105,-0.7423155308,-0.8981233835,0.4272961020,-0.0257868152,0.8800768852,-0.8634132743,0.4586406350,-1.4525709152,1.0027289391,-0.2363725603,0.7175219059,-0.8529829979,-0.4932558239,0.1452093124,0.3464656770,-0.5518008471,1.5215642452,-0.3498942256,-0.8275839686,-1.7511603832,-0.5457261801,0.8822903633,0.0703879967,-0.9963659644,1.2804361582,2.6976993084,0.4033238292,0.8038191795,1.2802057266,-0.2588358521,-1.1650398970,1.1707892418,-0.5047108531,-1.0936511755,0.5469737649,0.0263632089,1.0759764910,-1.0194599628,-0.3734528720,-1.9364116192,0.0224650707,1.0462511778,0.4847756624,0.1901340932,1.2200636864,-0.5608323812,0.0520195700,0.0540600084,-1.3133441210,0.3604013920,2.1903557777,1.2696301937,-1.7450065613,1.4612462521,1.1277128458,-0.9614953995,0.1591152400,-1.0061297417,-0.3439558744,0.5855348110],[-1.4501737356,1.5458877087,-0.6284275055,1.8623217344,-0.2068486512,0.0354135484,0.3395684361,0.2642773390,-0.7321042418,-0.5713635683,1.0891259909,0.3595553041,-1.1960923672,-0.3504222929,0.4999573231,-0.5098363161,-0.8443207741,-0.3192293346,-0.4329998493,-1.4102517366,-0.2385727018,1.1790127754,0.2475124747,-1.0066397190,0.9068338275,-0.5423150063,-0.3165667355,1.6968829632,-1.1119974852,0.4216455221,-0.8087496161,0.1220505908,0.1811276525,0.2846711278,-0.9815238118,0.0810643435,1.3072894812,-1.5053919554,0.8824597597,-0.9424833059,0.5472854972,0.1866397262,-0.3782856166,-0.6489279866,-0.1960052550,-0.7824270725,1.1514835358,0.9470567107,0.6862918735,0.4898491502,-0.2240593582,-0.5082946420,1.5246375799,-0.7205083966,-0.7252399325,-1.2068340778,0.6562124491,0.8725742698,-1.3504531384,0.2655869424,-1.1543271542,-0.3576900065,-0.3631928563,0.0644443333,1.0960409641,-1.2485544682,-1.3851460218,0.4644928277,1.5118932724,-1.6827667952,-0.1037741601,-2.8227734566,-0.4538195133,1.8951827288,0.2947148085,-0.4407174885,-0.6552804112,0.8836204410,-0.7145662904,0.2673783898,-1.0053929090,0.0218670834,-1.3796296120,0.3507533967,-0.2442570925,-1.0691617727,-0.7232266665,1.7802512646,1.4755240679,0.2832114100,-0.1803559959,-2.4272570610,0.9166278839,-2.0436441898,-1.2705802917,0.5730054975,1.5823352337,1.0499491692,-0.0399254858,1.3576145172,0.5052609444,-0.0344939381,0.7515062690,0.9314097762,-1.3302785158,-0.2806506157,1.6494643688,-0.6945965886,1.4947701693,0.5611131191,0.1360976994,0.1329657882,0.6428858638,-0.8688668013,0.5326808691,0.1221128181,-2.0173878670,0.2001792789,-0.4099375308,-0.6947799921,-0.2808746397,-0.1188127324,-0.5621849895,0.1748513132,-0.6217819452,-2.3000860214,1.0840605497,-0.7496613264,-1.5219525099,-1.7810281515,-0.8950442672,-0.0703199431,0.3249710202,0.4448671937,0.0597899482,-0.3028272986,-0.8436756730,-0.9046571255,-0.3266875148,0.4347041845,0.3949435353,1.1973904371,-1.1560224295,-1.0398107767,-0.0699188486,-1.4301743507,-0.0146584222,1.3570151329,-1.6681015491,1.2011176348,-1.0629158020,-0.2667770386,1.9444168806,0.8705314398,0.1332209855,-2.6169438362,1.2772220373,0.4033893347,-0.6000118852,-0.9993430972,-1.0177826881,1.3402198553,-0.4007156491,-0.1191185564,0.4988934398,0.0814124867,1.3495470285,0.2988345027,0.5763264298,-0.2183794379,0.4989469051,0.3067570627,-0.2668756247,0.0877958536,0.5085925460,1.4055000544,0.6140383482,0.0896825865,1.3617206812,-0.3267118037,0.6306826472,0.6890478134,1.1796201468,1.0196118355,-0.5822272897,-1.0949366093,0.0220815465,1.0912129879,0.8769557476,0.9900035262,0.9397202134,-0.0994466692,1.6669824123,1.4517176151,1.3211481571,0.2491372228,0.0155243585,-0.6893531084,-0.1140175387,-1.8416621685,-0.2381248474,0.7401183844,0.3654841483,0.2882166505,-1.0167877674,0.7473244667,-0.1891875267,0.9808660150,0.9236414433,0.6881241798,0.0554025508,-0.0573162176,0.5692203045,0.7792938352,0.6142871976,-0.6288341880,0.2108489275,-0.3727882206,-0.4873711467,-0.3820485771,-0.3673192561,1.5540332794,0.2947099507,-1.5285689831],[0.3804260194,-2.0671718121,0.0550171919,0.8531175256,-0.6018936634,-0.8865386844,-0.5568744540,-1.4733337164,-0.8281208873,0.3987645507,-1.1531097889,-0.1092760414,-0.9467664361,-0.7997373939,0.3220396340,-0.3594762981,-1.1803499460,-1.3892723322,1.6811511517,-0.2709219158,-1.3034292459,-1.0962494612,0.1192781776,0.3181969225,0.6427437663,-0.4708162844,-1.8047537804,-0.1646451503,-0.2703527808,0.6318225265,-0.4536555111,0.1989347637,0.8032560945,-1.3178848028,0.6166307330,0.2845878303,-0.9893379807,1.2858288288,0.7459064722,0.9172755480,-0.4152849317,0.6127344370,-0.1464545727,0.3386917412,-1.1799244881,0.5358649492,1.8083715439,-0.9430469275,0.4336068034,0.8175193667,0.6452999711,0.8691123128,0.2442887574,1.8094760180,-1.6751033068,0.0677766576,-0.9925397635,-1.0871496201,-0.2437263578,-0.4215831161,-0.1992474645,-1.3104773760,-1.4547755718,-1.1687017679,0.3781206310,0.5475878716,0.9851617813,0.4662987888,0.3638120294,0.2394635975,0.9616629481,-0.0387977324,0.9407829642,0.0011507085,1.8726496696,0.1954480559,0.4769034386,-1.0875762701,-0.7599243522,-0.8566319942,-1.9922207594,-2.2812983990,-0.2973401546,-0.5694825053,0.0664883927,0.4117467999,1.1683961153,-3.0562281609,1.7612978220,1.7013229132,0.2869657576,0.0255660973,0.3154593110,-0.2584732175,-0.1614930183,2.3610413074,0.4873589873,-0.5125331879,-0.3015183806,0.3679594100,-0.3626063764,1.0938476324,-2.5488677025,-1.3180434704,1.4499435425,0.7009496689,-0.1576300114,0.4049410820,0.1778909564,1.3396008015,-0.2206843495,-0.2286064029,2.0505385399,-0.4772749841,-0.0627959520,0.0254990458,-1.3595477343,-1.3190525770,-1.0695114136,-0.6161350608,1.0919641256,-0.3321318030,0.7260578275,1.1332423687,-0.2503430545,1.1677489281,0.7886906266,0.2164149135,0.7647698522,-1.3076148033,1.1607283354,1.8137441874,2.8209869862,-1.1829620600,1.2793636322,-0.4161130786,1.4560747147,-0.8382224441,-0.2064589411,-0.3601056635,-1.9282368422,0.2948355377,0.6405059099,-0.9806084633,0.0274974220,-0.4893453717,2.7329032421,1.4204442501,0.6902386546,-0.4438398480,0.8475421071,-0.8242413998,-0.2056256086,1.2381081581,1.8283888102,-1.7836816311,0.7437072396,-0.2981782556,0.4307036698,1.2446408272,-1.5674799681,1.7146155834,-1.9383628368,0.3435726762,-1.5765607357,0.0504746065,0.8115888238,-0.3472369909,-1.1612019539,0.3186337650,-0.6062232852,-0.7585208416,-0.5607734323,0.5222164392,0.7790774703,2.6089880466,0.6626429558,-0.2798112035,1.1184885502,-0.1381485909,0.0895912275,0.7735747695,-1.2175766230,-0.2237391025,-0.1026925892,2.0417473316,0.7975260019,1.5041879416,1.0822825432,0.7253676057,-0.8952884674,-1.8488601446,0.1530598551,-1.9314616919,-1.6300439835,-0.8163452148,-0.5146376491,0.5783777237,0.0899859294,-0.2165891677,-1.5324651003,-1.3432455063,-0.5275440812,-0.6659939885,0.0604406931,-1.3241438866,-0.9218361974,-1.9925515652,0.2451740056,-0.3967302442,0.1276450008,1.2230695486,0.0030670676,-0.2247872502,-0.0633816347,1.4691959620,0.5458295345,-0.5807732344,0.9379773140,0.3422805071,1.5076049566,0.6746079326,-0.1290075630,0.5062861443],[0.1298588663,1.8845261335,0.9324017167,-1.0008316040,0.1478935778,0.5123214722,-1.5427743196,-2.1174151897,0.0422072709,-0.3248087168,-0.4121863544,-1.5678561926,-0.7964280248,0.7105121613,0.2873294055,0.6519631743,0.5355852842,-0.6085993052,1.1440550089,0.9712348580,1.3347989321,0.2167384177,-1.2789026499,1.2023334503,-0.4959734082,-1.8249573708,0.3743883073,0.0709712729,0.2931367457,1.6866817474,0.1300894916,-0.1589677930,0.9902415276,0.1049473360,0.6208986640,-1.7520277500,-1.6000163555,-0.9802316427,0.0425251871,2.1325788498,-0.9563222528,-0.4039032757,-0.6859712005,-1.1440685987,2.8847551346,-1.4577374458,-1.6533602476,1.5437217951,-1.6310477257,-0.3676165342,-0.5928305984,1.3807523251,-0.5501603484,-0.5219319463,-0.0825198963,-0.3111062944,-0.7352650762,1.1713025570,-0.0221288241,0.6046815515,0.2135297805,0.1448385417,0.9258138537,0.6199473143,0.8385660052,2.1954982281,1.4447660446,-0.2872459292,0.2114256471,-1.0960211754,-0.8197528720,-0.2152145207,-0.1540306658,-1.6386111975,-2.5233354568,1.2656054497,0.9129872918,-0.6085659862,-0.6617404222,0.6074315906,-1.9679963589,0.7266260386,0.2714606822,0.1947142482,0.5195682049,0.8111485839,0.3197706044,-2.0950548649,0.5762258172,-0.8431512713,-1.2846280336,0.9035053849,-0.3686189353,0.8176126480,0.6898447871,1.2300739288,3.2548313141,0.9898021817,-0.7532128096,-0.9045040011,0.5490559936,0.8705734015,0.7328756452,-0.8265963197,-1.5101081133,0.3340276182,0.5381116867,0.6491934061,-0.1450466812,1.7645809650,-0.7833189964,-0.5206372142,-0.3905070424,1.1084893942,0.5302432775,-0.7692154050,1.2692584991,0.9881228805,-0.5279560685,-1.0720012188,-1.6803950071,-1.5877646208,1.9102419615,0.3213319182,-1.6464880705,-0.1613440216,0.3271096349,0.5641773343,-0.2901338339,-0.6286158562,1.0368025303,1.4191545248,0.3205574453,0.8432882428,-0.4522081912,1.1206735373,0.9856461883,-0.1103538498,0.3788451552,0.8254184127,-0.9615396261,-1.7476845980,0.3499236703,-0.8517538309,0.7227854729,0.9791315794,0.7726531029,0.6913672686,-2.0398874283,-1.3073374033,1.6918557882,-0.8667309284,-1.0342205763,-0.3692653477,0.5241845846,-0.4410466254,2.1805980206,0.5975066423,-0.6366894841,-1.4811692238,-0.3947469294,0.1086023152,0.1037365198,-0.5710659623,1.2462812662,1.2071619034,-0.6543464661,0.3778489232,1.4369262457,0.8684048057,0.9227760434,1.9248412848,-1.0600451231,0.5242900252,2.4939887524,-1.4329419136,0.5927869678,-0.0053195218,1.4319832325,-0.6679624915,0.1154298410,1.2070343494,0.1147793829,-0.6715605855,0.0334689096,1.6597380638,-0.9170382023,-0.5053157806,0.1027670279,1.1581686735,-0.5237666965,-0.4403850436,-1.6496279240,0.7673044205,1.1748074293,-1.4838258028,-1.6586328745,-0.6399556398,-0.5451463461,0.2890629768,-0.3449325264,0.5661980510,-0.2247092426,-0.3704339266,0.4762049317,-0.5505967736,-0.3622225821,0.8294441104,-1.0188047886,-1.3211250305,-0.5238136649,0.0719858110,0.5811215043,1.3551915884,0.9022732973,0.8348205090,1.7601321936,-1.2035567760,-0.1610284895,-0.6985741258,-0.8050824404,1.6160351038,-0.2635310590,-1.2230753899],[0.0442702919,1.4457842112,-0.2831886113,-0.5522205830,1.8225749731,0.5256469846,-0.0092465924,-1.1489993334,0.7133934498,0.7125946283,0.4428358972,-0.2494825572,-0.8214784861,-0.3348659575,1.7764191628,0.3546428382,1.4700632095,1.0542569160,2.1549079418,-1.7109314203,-0.4482811987,-2.1268780231,2.0661754608,0.1601428241,-1.0449494123,-1.0156296492,0.6733638048,0.9290160537,-1.0412443876,0.3149002790,0.7868452668,0.8227619529,0.1526217610,-0.1421793401,0.6412929296,1.3321976662,1.2902623415,0.0031562971,-1.4625291824,0.2808877230,0.5652723312,1.1053085327,-1.2109456062,-0.3344887495,0.9515791535,-0.9712296724,-0.4897898436,0.5153864026,-0.5969720483,-0.2086329162,1.0944644213,0.1678696126,1.6910209656,-1.6885894537,0.8107523918,-0.1487727761,0.8241059780,2.5300707817,-1.1119675636,-0.0557166263,-0.9830359817,-0.0218584090,1.2924665213,-1.5760532618,0.2903743386,-0.0360350795,0.1165624261,0.5244975090,-0.1452632844,-0.7145482302,0.9750678539,0.4187014699,-0.1647199988,0.0827688649,2.8011438847,1.1906450987,0.5398116112,-0.6014398932,0.4039287567,0.2727628350,1.3142924309,-0.5986911058,0.3755066097,0.3850517273,0.5240681767,0.1509334147,-0.2982423306,-0.9690389037,0.1482082903,-0.5794170499,1.1058530807,-0.6659970880,0.5248563886,0.6544551253,-0.1016000733,1.7019340992,0.1335662603,-0.3516343236,0.7148193121,1.2314994335,-0.9258267283,0.0363452137,-1.1703430414,1.7012137175,0.2636000216,2.5404133797,-0.0304493718,0.4889936447,0.7376376390,-0.3831209540,1.7555421591,0.2273544669,-0.8490986824,-0.1827587038,-0.7989795804,-0.8379164338,-2.0694205761,-0.5316456556,-0.2463656515,-1.3631684780,-0.7051585317,-1.6138539314,1.3198417425,0.3702345788,0.0753858536,-0.0132740363,-0.9950203896,0.6775140762,-1.5192737579,-1.1931970119,1.0480457544,1.0526509285,-0.2511567771,-0.2708320022,-0.9000599384,-1.0730860233,-0.8332417607,0.6814241409,-1.1012221575,-0.5161744356,-1.9944025278,0.5473926663,0.2162017673,0.6118625402,1.6970700026,0.9290080667,-0.6630474329,1.9621838331,0.6916124225,-0.5143244863,1.3465787172,-0.1014270186,1.6098065376,-0.4065241218,0.2456360906,-1.0644011497,-0.6824391484,0.9376380444,-0.4972450137,-1.3954156637,-0.7931784987,0.3863891661,-0.5077642798,0.1169658527,-2.2643694878,1.2981832027,-0.0722546279,1.3718563318,-0.9511761069,1.1634547710,-2.4079461098,-0.2811202109,-1.4747558832,-0.7960328460,1.6813354492,0.1580020636,0.4906089008,-0.6557968259,0.1914050281,0.8196141720,-0.5575736165,-0.1627945155,0.2597014308,-0.4878998101,-1.5364331007,0.8920765519,0.1526391953,0.2662674487,-1.8847845793,0.3603782356,-1.0658118725,3.2456190586,0.2545815408,-0.0740487799,1.0587692261,0.4594543278,-1.2144246101,0.8521980643,0.1920867115,-0.6966613531,0.9356654286,1.4123430252,1.9032716751,-1.1575750113,1.2239912748,-1.3523010015,1.3675932884,0.0028783996,-0.1164716333,0.0636274517,-0.1263964623,0.6503781676,0.4069949985,1.0436509848,-1.4743063450,0.1385995448,0.1789027303,0.0112371799,0.2460647076,-1.1011744738,-0.5300943255,-1.6115192175,-1.9061876535,-2.5654923916],[0.0496643037,1.0157072544,-0.2320692241,-0.2537081838,-1.9836074114,-1.2122704983,-0.8500339389,0.1666402519,-0.5580587983,-0.7058231831,-0.6347995400,0.2047363073,0.2284586877,-1.1299531460,0.7173625231,1.1171007156,1.0772426128,0.0177433770,-1.2954846621,-0.5299506783,-0.5749281049,0.5966389179,2.7219719887,0.6849224567,0.8479590416,2.0062556267,-0.0862625539,0.9452161789,0.8045234680,-0.3388015628,-0.2429846227,0.6233448982,1.2148296833,0.1562897563,-0.3656148314,0.1104548946,0.9932194352,0.5684428215,-0.9437820315,1.0924483538,-1.5989911556,-2.0403063297,0.8947772384,0.3354680538,-0.8832319379,-0.9793969989,1.6826566458,-1.0103543997,-1.2948708534,0.0603040457,-0.3768114746,0.6062066555,1.3397293091,0.4506835639,0.4326622188,-0.4533344805,-0.3971926570,-0.6595103741,-2.4956014156,1.7504210472,1.3513900042,-0.6080017090,-1.0249046087,-0.1988066733,2.1393291950,-1.8256448507,1.2898792028,-0.9707172513,-0.6307196617,-0.3292931020,-1.3354488611,0.3118260503,1.2624188662,-0.3525214195,0.0506781153,1.5740100145,0.9165746570,-0.3342539668,-0.2621529996,-1.2329554558,-1.5453443527,2.1995866299,0.6019462943,-0.3987810612,0.5707879066,-0.4209889472,-0.0048142010,-0.0332821496,-1.5761977434,-2.1236410141,0.4128189087,-1.1925446987,0.4046672285,-0.3922690153,0.9129861593,-1.2042829990,-2.5270962715,1.1464778185,-1.0409102440,-0.2294565439,-0.1564095914,0.2098131180,0.4423882961,0.6708213687,0.0011970691,1.3835297823,-2.1146876812,0.4195026755,-0.5974859595,-0.7789434195,-1.1835699081,-0.4481227994,-0.5926176906,-0.5601764917,-0.2906090021,-0.0740258768,-0.4800468087,-1.0527486801,-0.7800763845,0.3000288010,-0.3290872574,0.9112530947,0.1389426142,1.1165764332,1.5068144798,0.5325995684,0.4274400175,1.1449474096,0.7173746228,1.2336890697,0.6168799400,-0.5105078816,-0.4883999825,0.2610085905,-1.3890395164,-0.2097653002,0.1856197417,-0.8605422974,-0.3273102641,0.2982771695,-0.0288848169,-0.9086425304,0.3567273021,-0.5856960416,0.7619949579,-1.1993529797,-0.4374361038,-0.0871637315,-0.8370314240,0.2818605304,1.3145585060,-1.1522951126,-0.1624574959,2.4466850758,-0.2131539136,-0.3643190265,1.1231597662,-1.4967659712,-1.3696490526,2.5906090736,-0.8502928019,-0.0154352458,0.0708632097,-0.0892392695,0.9884635806,-1.4502840042,1.9888542891,0.0451550037,0.2025863230,-0.3004295826,-0.0864960775,0.2483033687,-0.1879534423,-0.3177562356,-0.5728502870,-1.3048775196,-0.5676960945,0.9221876264,1.3602030277,0.2550102472,0.5108075738,0.6118447185,-1.2540539503,0.7811601758,0.1337841600,-0.5227097869,0.6001027226,-0.5171794891,-0.4444061518,-0.9055517316,0.3879831135,-0.3776228130,-1.8549275398,-0.4994918406,-0.5876417756,-1.6503987312,-0.2694072723,0.9541515112,-1.4059816599,0.6579794884,0.8008347750,0.8226274252,-1.1246813536,-0.0056366222,0.5829159617,-1.5171155930,-0.2439792901,0.2839211524,-0.7109612823,-0.7272608280,1.2907973528,-2.4604752064,0.2687241733,-1.1931895018,0.1179150194,-0.2157852799,-0.2656730711,-0.8884975314,0.7275415659,0.2283545136,1.6116662025,0.7428886294,-0.0892703235,-0.0904256403],[-0.1552596092,0.4787852466,1.3613634109,0.1748903543,-1.6370717287,1.0790226460,-0.1739098728,-1.0114746094,1.4560087919,-1.9859833717,1.3460286856,0.1526861787,-0.4367708862,-0.4582619667,0.9152957201,2.7462913990,0.1391252726,0.7504043579,0.7596225739,0.5596187115,-0.0583412908,1.6963244677,-0.3965150118,0.1869936138,1.8145555258,-1.3269416094,-2.2988479137,-0.5298420191,-0.8341919184,-0.8106333017,-2.5397062302,-0.0860494003,0.4506426752,-1.2943652868,1.0642453432,0.9367136955,-0.1269537657,0.5206781626,-1.1788055897,-0.0925211310,1.0476361513,-1.2335727215,-0.2922436893,0.0496872887,-0.2639459372,0.3845275939,-0.1538703591,-0.1169964671,0.1776241362,0.6594055295,-0.5636582971,-0.9561285377,0.4923869669,0.2409788370,1.6332099438,1.9169098139,1.1394318342,-2.5501418114,-0.3268522024,0.9455291033,-2.0710549355,-0.1209253818,-0.9449245334,1.5542948246,-0.2254855931,0.5607367754,-0.4980247319,1.2718555927,-0.9843304157,-1.6917517185,0.3631607592,0.2028281391,0.0599766299,-0.0586545616,1.1578779221,-0.1312680691,-1.6256289482,-1.0176999569,-0.1003823131,-1.0121793747,-0.4727144837,-0.5200366974,-0.2728419602,-0.4269649386,0.7875617146,0.9885934591,-0.4017859101,-0.0173298735,0.6428719163,0.4834312797,-0.2587702572,-1.1848987341,1.4192327261,0.1049401015,-0.3738996387,0.7806690931,-0.6665983796,0.0297318734,1.0643459558,-0.3537979722,-1.3078233004,0.7726968527,2.1866788864,-1.1002861261,0.1728644818,-0.3756933510,-0.9906257391,-0.8670617938,0.2038979679,-0.4590037465,-0.9317319393,0.1539378166,1.7603360415,-0.7288731337,-1.2079353333,-1.7485420704,-0.2674083114,-0.7923513055,0.8678576946,-0.8161421418,-0.5595346093,0.5180754662,-0.5601761937,1.9338699579,-0.2529518902,-1.1536580324,0.8240326643,-0.2938970327,-1.0393499136,0.0384243019,-0.1298382133,0.6317511797,0.9699698091,-1.2614362240,-0.3775847554,-0.3486490548,0.2697852552,0.7138586640,0.1024048105,-2.2221372128,0.0392785184,0.5322803855,-0.3420674801,-0.3462099135,1.7591598034,1.8691898584,-0.1903523058,-0.5461303592,-1.8728995323,-0.3152549863,-0.1878576875,-0.8418833017,0.4303056002,-0.0654961094,0.3263684511,0.5223084092,-1.4092383385,-2.7141454220,-0.6595085263,-0.6816119552,-0.3690533638,-0.3258565962,-0.6590272784,-1.3374688625,0.2712288499,-0.5335494280,0.3359538317,-0.5624163747,-1.7754486799,-1.3497921228,1.7255154848,2.0891745090,-0.1147161946,-0.4825686216,0.1295297593,0.3011428714,0.9685909748,-2.6775097847,2.0935235023,0.5985561609,-0.5186195374,1.1494662762,0.0178137068,-1.7103466988,0.0695706159,-0.4838339686,-0.6910316944,-0.3891272545,0.4077689052,0.1103762239,0.3082531989,1.6875544786,-0.8012143373,-1.2962436676,-1.1299370527,0.8236520886,-0.2105285227,-0.0350589901,-0.1153961793,-1.2768372297,-0.4285729825,1.0847262144,0.7416942716,-0.3530087173,-3.1282429695,-0.9469532371,-0.6899254918,0.8435842395,-0.2861720920,-0.3562988639,0.2356502563,-0.5722347498,0.5325630903,0.2162115574,-0.2787752151,1.9393185377,0.2387497127,-1.0231500864,-0.1833423823,-1.3151319027,-1.1783853769,-0.8859948516,1.2054790258,-0.5636700988],[0.4386736155,-0.6196897626,0.8047269583,0.1394855082,1.4760224819,-0.7151355743,-0.3518398702,-0.6956925392,0.4250043929,0.1587561816,-0.8085920811,0.9429353476,-0.4326693714,0.8953675032,0.9709054232,-0.9297704697,-0.1016962975,-1.1935923100,-0.9966936111,0.2610221803,0.0391289629,0.8596969247,-1.1022953987,-1.3328871727,-0.3936585784,0.3379068673,0.3390775621,1.8913849592,0.1280214936,-0.1877435148,0.7314977646,0.8108587861,0.2118465602,0.5429373384,0.2095481604,0.0367294289,1.3247373104,1.1130683422,-1.1776549816,-0.1899321526,0.9921732545,1.9340195656,-0.7474932075,0.5622502565,0.2565377355,-0.5058898926,-1.1801614761,1.1046266556,-0.2420715690,1.0296734571,-0.4392384887,-1.2390613556,-0.6452093124,-0.3723985553,-0.0273250490,-0.8667244315,1.2947559357,0.6697645187,0.8879160285,0.1957534999,-0.1988423914,1.2720395327,0.2759205401,1.7492319345,-0.2943438888,1.1912611723,-0.0809823349,0.0359159783,-0.4873678982,2.9271998405,-1.1786690950,1.0902714729,-1.0070717335,-0.0559710450,0.5503617525,-0.2708517611,-1.4462407827,-0.0244226474,-0.9263724089,0.3695653975,-1.0758725405,-1.0543339252,1.0808296204,-1.3966062069,2.2895793915,1.8400338888,-0.5700233579,-1.0836747885,0.7417201400,-0.2145137787,0.5379716158,0.3843903542,0.0347256139,0.3680998087,0.3415450454,1.0920809507,1.0413075686,0.0939790681,0.7268617153,-1.1950058937,0.4225138724,0.7281823754,-2.2234709263,1.2858834267,-0.9244327545,1.6425834894,-0.8119661808,3.3916356564,-0.2394975424,-1.2916408777,-1.2434978485,0.1182641983,-1.0540636778,1.0878865719,1.0936306715,2.7451908588,-2.4534094334,0.0842225254,-0.7882358432,-0.2610311210,-0.1405747086,-1.2278044224,0.4670206010,-1.5480242968,-1.0367826223,0.1100453958,-1.6257070303,-0.6632278562,1.6480108500,-0.3857504129,1.3263442516,-0.7813788652,-0.2630614638,2.2347497940,0.0668686181,1.6011503935,-0.2215524614,2.5348920822,0.6628156900,0.6786739826,1.7699764967,-1.7675346136,2.8429641724,1.7460889816,0.1622756273,-1.2240581512,0.9790400863,-0.4723766446,-1.2521955967,0.1068915650,0.2467359900,0.4016210437,-0.4117667079,-0.4852364361,-0.0951911286,0.6827812195,-0.2690612674,0.6709222794,-0.4353961647,0.8948027492,-1.9490350485,-1.0914882421,-0.3209503293,-0.2963206768,0.4339114130,-2.2697405815,1.2201591730,-1.8413844109,-0.3661860824,0.7219944000,-0.9503106475,0.6586489081,0.0980350003,1.0423717499,-0.4973531961,-0.5744463801,-1.3544663191,2.0338106155,0.0617796816,0.5912122130,-0.7075412273,0.0460111462,1.2361793518,1.7257440090,0.7506306171,0.5277210474,0.6417139173,0.3805473745,-0.9311766624,0.9438769221,-0.8593149185,-1.7188237906,-1.4166681767,0.7779759169,1.1453680992,0.2659092546,-0.7271529436,0.1307234317,0.7076218128,-0.6337662339,0.8103998899,-0.7279103398,0.2717936635,0.5227202773,-0.8961473107,-0.1444521397,-0.7144158483,-1.4684110880,-1.8797703981,1.4193205833,0.2078864872,0.7992144823,0.0320600830,0.8656871319,1.2021867037,-1.2167718410,0.8329434395,0.1805313081,0.3732955158,-0.6465490460,-0.9415496588,1.2734313011,-0.8172774911,-0.9249973297],[0.1996483356,0.4317770898,-1.0799595118,1.4785776138,-0.0096156178,-0.8270172477,-1.0379046202,0.2136165649,1.1392287016,1.2750461102,-0.8241134882,-0.2134141624,1.7874548435,1.2697911263,-0.7215440869,-1.2200564146,0.7094538212,0.2863227427,-0.0210470576,0.3863923252,-0.3042067885,1.0375249386,0.6831886172,-0.8644843698,-0.3198086619,0.4823707640,-0.2265983522,-0.4702896476,-0.5963839889,0.9628387094,-0.3546466827,-1.3467926979,1.0188089609,-0.8363088369,-0.0953657553,-1.3766548634,0.5656586289,-0.4472826719,-0.6763988733,0.9349191189,-0.8283550143,0.5968904495,-0.5544840693,-0.1959493458,0.0332162529,0.4148295224,0.0375486575,-1.0577303171,-0.9384614229,-2.0667753220,0.4751504958,1.0487740040,1.3003197908,0.3228363395,0.2529530525,-1.1515924931,-1.2190105915,-0.9456254244,-1.1880462170,2.6874439716,-0.3045670986,-0.7014102340,1.5996931791,0.3379813731,0.9407749772,1.1743144989,0.9814895988,-0.3301385045,0.7717459202,-0.2328396291,0.9203398228,-1.0166190863,0.0462331660,0.0152617535,-0.0070052808,-0.6647570729,0.5429480076,-0.5906264186,-0.0308912378,1.0884922743,-0.0040217643,-1.1533929110,-0.1063377857,-1.0902687311,-0.0371520258,-1.8188394308,-0.7905772328,0.1596994698,1.1226239204,-0.6813367605,-0.1751530617,-0.0979226381,-0.8810763955,1.1648347378,1.3413946629,1.1284626722,0.5128263235,0.0187418163,-2.0666255951,1.0853357315,1.0145881176,0.6455498338,0.3209685683,0.5687319636,1.0641225576,1.4271905422,-1.3905673027,-0.9891104698,0.0020802424,2.3130781651,0.6045076251,-0.8978734016,-0.3523695469,-0.9741426110,0.3125937879,-0.1361470819,0.7558745146,-0.6038464308,2.2138009071,0.2676672935,0.4894275367,-0.1646461040,0.2514430285,-1.4151527882,-0.1414795518,0.3838063478,-0.8808335662,1.9905422926,-0.3512460887,-1.5725287199,0.7633091211,0.0000131410,1.6761398315,-2.1059546471,-0.3404865861,-0.5632696152,-1.5203379393,-0.5280867219,-0.7296812534,0.6876379251,1.4581366777,-1.1583836079,0.6405873895,-1.0695174932,-0.1391310841,0.7101902962,-1.3572121859,0.1433014721,0.6498459578,-0.2396032214,0.4949466884,-0.0583442226,1.3597508669,-1.1603624821,0.9705311656,2.1088654995,-1.7841472626,-1.5925489664,0.4765196443,0.7140438557,-0.2969127893,-1.0004702806,-1.4487510920,2.0203588009,-0.4833310246,-2.3372316360,-0.5886305571,1.6995788813,0.0083491514,-1.0568113327,-0.1681756973,0.7782428265,0.6581347585,0.0151447775,-1.6557891369,0.8034425974,-1.0629384518,0.7985153794,2.1845128536,0.5347851515,1.6842222214,-0.6536903381,0.9090065956,-0.6756687164,1.9171681404,-0.0657464713,1.3801714182,-0.5541518927,0.2358659059,-0.1883283705,1.0601769686,-0.2588506639,0.4196306467,-1.2001607418,0.5803898573,-1.4160250425,1.7342479229,-1.1105753183,0.0719698370,-0.0096054645,-0.6200602651,0.1690012068,1.2434685230,1.4427556992,-0.1093287393,0.1923421919,1.8308950663,-0.6960420012,0.1550203562,1.0692973137,0.2098853588,-0.3324133456,-1.2730194330,0.6015979052,0.6712191701,0.2805779576,-0.5934511423,0.9138393998,0.0384552255,-0.0258487891,-0.8879203200,1.0867121220,-0.2558456957,1.3400003910],[0.8616962433,0.9091723561,0.3765018284,-0.0544774011,-1.1591389179,-0.1942774951,0.9726710916,-0.9714103937,1.3434149027,-1.1338416338,-0.7891272902,-1.9806178808,1.4280047417,-0.3443213403,1.2054544687,0.9668977261,-0.4083234668,-1.2918647528,-0.6399744153,-1.3426039219,-0.5441907644,-0.0543023236,0.1115766466,-0.4973062575,0.0533691123,-1.0189346075,0.0490081087,2.0632190704,0.9324282408,1.4523895979,0.5729588866,-0.4189784825,-0.3286801577,-1.1514205933,-0.8807796240,0.2938400507,0.7655924559,2.1366088390,0.0733786449,-1.6295683384,0.1775692850,0.8521859646,0.4896481931,0.5674389005,0.2375758439,0.0813688114,-0.6042775512,-0.5440589786,-1.2309296131,-1.4950492382,1.4114691019,0.4880595207,-1.9063181877,0.3281845450,-0.3974549770,0.2618822157,0.7912273407,-1.7842062712,1.4989305735,-0.2391272038,3.0528898239,0.3499550223,-1.4347679615,1.0079482794,0.4206511974,0.6000564694,0.1201554537,-0.5846073031,0.3514090776,-0.8588541150,-1.0708264112,1.9690668583,-0.0061912374,-0.7901587486,0.9585415125,1.9639558792,-0.4290848672,0.5445072651,1.2167263031,1.4984652996,-0.3440700471,0.3850102127,1.1216851473,0.6268048882,-2.3510096073,-1.3539459705,-0.0425043851,1.0418293476,-0.2806175649,0.4425379932,-1.3477407694,-0.9427426457,-0.0101525290,0.9552149773,0.2844652534,-0.0253380742,0.4472802281,-1.0420781374,1.8255304098,-0.1858208627,-0.9919012189,0.4774014056,-0.1065087095,0.5237557292,0.1012536585,0.4370443225,0.4235878587,0.6120776534,0.6691703200,1.0026834011,-0.3116049469,-0.0213360675,1.8255027533,-0.4455459118,-0.0459221825,-0.7012569904,-0.1852261573,0.5536851883,2.1449239254,-0.1896124780,-0.3305495679,-1.0523796082,0.5392750502,0.7831043005,0.3730578721,-0.1366892159,-1.0765298605,-0.4404587150,0.9915053844,-0.0562079400,-2.5473208427,1.1598278284,-0.7649890184,1.2011524439,-0.0606509931,-0.0063281334,-0.7450198531,0.6501030326,0.2470915914,0.3507081568,-0.8862549663,-0.6400193572,0.8826782107,1.1692306995,1.7553559542,-1.0607961416,0.6163920164,0.2505285740,1.2890933752,-0.5036508441,-0.5445509553,1.3842186928,0.3668037355,0.2400822192,0.4661004543,0.4137141705,-1.2596691847,-0.8630129099,-0.2596816719,-1.3117531538,2.5076625347,0.8216830492,-0.8308066130,-0.2402574420,-0.3923582733,0.2755875885,-0.8734244704,0.0743628964,0.1546310931,-0.0208904427,1.4379118681,1.2219953537,0.2602660060,1.0976809263,-0.0790601820,1.4060403109,-1.7074382305,0.1989279836,0.7320793867,1.0455595255,0.8985303640,0.5794932842,0.7482882142,1.0181425810,-2.1934535503,-0.1195093766,-0.5527552962,0.9271039963,0.0382056795,1.3007956743,0.2055685967,1.7423490286,-0.0785754547,-0.0049815332,-0.0163416192,-0.4962151647,0.1003125310,-0.8750685453,1.3657735586,-0.3143217564,0.1486933380,-0.6111947298,1.1490167379,-0.5325989127,0.0778353959,-0.8201375008,0.9185136557,0.3281737566,-2.0297646523,1.1189615726,1.0363020897,-0.2897744775,0.3120375574,0.7430668473,-0.1351819932,0.2756820321,0.2833786309,0.1408464015,-0.7921109796,1.6592693329,0.8912046552,-0.2330278158,-0.7310743332,-0.6375401616],[0.7888815999,-0.5637577176,1.5878105164,0.5654777884,-0.7989003062,0.6526470780,-0.5024884939,-1.4605481625,0.6715288162,1.0051063299,-2.2796263695,-0.0421458483,0.5515695214,-1.6218701601,1.2987296581,0.1798663735,-0.5212946534,0.5497487783,-0.9817274809,-0.7634006143,-0.4252837300,-1.6081527472,-0.5749193430,-0.0845824331,0.3179959953,-0.8117060661,-1.2607413530,-1.7419383526,-0.8477881551,-0.7736322880,0.0881378502,2.0652372837,-0.6772291064,-0.2555656135,-0.1456658989,1.2135969400,0.6333103180,0.6250420213,-0.5774396062,0.2879560590,0.8352848291,0.4350744784,-0.3484890163,0.0795898214,0.8474707603,-0.1157193184,0.4633275568,-0.3415514231,-0.6989926100,0.6122351885,-0.8388199210,-0.7292807698,-1.0647550821,0.4956947267,0.1355170608,1.2068425417,0.2432746291,-0.2817509770,0.2291960120,-2.0235092640,1.2210216522,1.1453696489,-1.3078383207,0.5784826875,-0.1166832522,0.4518258870,1.1240378618,-1.8123457432,0.6662940979,-0.9611188173,0.7314282060,-0.2969223261,-0.3550669253,1.4163671732,-0.9020425677,0.0199772660,-1.1304079294,0.1079952568,0.4287727475,-1.6183984280,0.6845467091,2.8033049107,-0.2084693313,0.1491068602,0.2246538848,-1.1328402758,1.1970536709,-1.1747390032,0.6738021374,2.5621573925,-0.6892337203,-1.5325818062,-0.8725388050,-1.8731801510,-0.8512567282,1.2584059238,0.6210688353,1.8926613331,0.5126205683,0.7967803478,-0.4531759024,0.6966472268,-0.9069048762,0.0096370932,-0.6075945497,-0.7283154726,0.5826425552,0.2152446657,-1.2975306511,-0.5846914649,-0.4268600941,-1.0764298439,-0.7263673544,0.9428517222,-2.0351703167,1.5888681412,1.1721462011,-0.7234143615,-0.6291493773,2.0337893963,0.2800778449,0.0618867502,-1.0652469397,-0.1269562542,0.0394340493,-0.2710518539,1.1494917870,0.7313186526,1.4539731741,0.4317398369,0.1555168629,-0.6679779291,-0.6185320616,-0.4576709569,0.8619299531,-0.3402082324,-0.0162530318,-0.9234033823,1.1426278353,-0.4658351541,0.4760331213,0.1220721081,-0.0449118540,-0.7347796559,0.4099939167,-0.4844644368,0.8656507134,-0.0493310839,-1.6068956852,0.6101850271,0.3961558640,-0.5334519148,-1.2353732586,-2.1528379917,-0.6461086869,-0.7687655091,1.0920033455,-0.0443054624,0.9873304367,0.1205482185,-0.0901331529,0.2483174056,-0.0232608952,-0.9315232038,-0.4337712228,-0.7576855421,-1.4757883549,0.4191368818,1.0992763042,-0.5225518942,-0.3033358455,1.4686160088,0.2693915963,-0.5548349023,-0.0744934976,-0.9019102454,-0.4306880832,-0.4611977935,-0.8646348715,-0.9281977415,-0.2186366916,-0.7222694755,-0.5996643305,-0.7126638293,0.9143796563,-0.1718460768,-0.7571513057,-0.8897672892,-0.2077621669,0.3679377139,-0.6376763582,0.6803770065,0.1529729366,1.6237425804,-0.7013158798,-0.7598114014,2.4127118587,-0.0613814630,0.5617071986,-0.4261292517,1.3765552044,-0.1034885272,-1.5822913647,0.2727092505,-0.4394279420,0.0855139270,0.5925198197,0.9543700218,-1.3967151642,1.0356595516,1.2689288855,-0.0545611903,0.2998319864,0.2138415426,-1.5542737246,-0.5578077435,-0.0780339167,-0.7181679010,0.4998109639,-1.1115146875,-1.6112943888,1.5544286966,0.7367937565,-1.1129119396],[0.7780324817,0.4822565317,-0.8536413908,0.4397369325,-1.3432642221,-0.9365406632,-1.2382225990,-1.2970274687,-0.6214320660,-0.4893566370,0.1583636999,-0.2710636258,-0.1458918750,0.6112800837,0.6005087495,-0.7187950611,0.8169072270,1.6199951172,-0.7268152833,0.2404350340,0.2810488343,-0.4852738678,0.9925027490,0.0956963748,-2.8951740265,-0.3318829834,1.3719614744,0.5818605423,-2.0010683537,-0.4226765037,1.0245097876,0.7852441072,-0.4349224865,1.4291239977,-0.4860503674,0.3763881922,-0.4843531251,-0.3523619175,0.7469093204,0.6595010161,-0.3882180452,0.9919152856,-0.2797157764,2.4180109501,0.0411199592,-1.8055504560,0.5286077261,0.1346055269,0.3227880299,-0.3051802516,-0.3027456105,-0.0010736089,0.4762355387,2.2697131634,-0.3243034482,0.1008412465,-1.4537488222,-2.0363309383,2.1591031551,1.5303931236,-0.7778717875,0.2341108471,1.3119387627,1.3180574179,-0.8847883344,0.5251903534,0.2385541797,-0.3019807041,1.0521885157,0.0388435014,-0.1639668941,1.7076059580,-0.6597477198,1.3579747677,-0.7429273129,-0.0546745248,-0.1375604272,-0.9347351789,-1.1826816797,2.0394957066,-0.4085650742,0.4151297510,-2.3462462425,-0.3660867512,0.2620778680,0.4003766775,-0.0485555828,0.8496025205,-0.8509175181,-1.4193309546,0.8738920689,-1.3105102777,-0.1310589164,-0.0740571916,0.1930714250,-0.9857059717,-1.6849098206,0.4512572885,0.1277438253,0.3671044409,-0.1350663900,-1.0645258427,-0.0029378785,-0.0521377474,-0.0649472773,-0.6643292904,0.5142106414,-1.7635880709,1.6859691143,-1.0984498262,-0.7941613793,1.1221563816,1.3669830561,-0.0905384943,0.2765818536,0.6256956458,-0.3617914617,-0.2852146924,0.7979028225,0.2126920968,0.8998239636,2.2797007561,0.2099189162,-0.3656977713,1.4928067923,-0.4475935102,-0.4616768956,0.2791537642,0.2556216717,-0.6943846345,0.4273623228,0.1741427183,0.1316052079,1.1936930418,1.5984715223,1.0562391281,1.0298056602,-1.2818797827,-0.8192679286,-1.9437376261,0.6600036025,0.0188401472,0.4665989578,1.4290597439,0.7851303220,0.2261232734,0.7731981277,0.5813441277,0.5581420064,1.3034487963,1.0300573111,2.0724573135,-0.7095886469,-0.8896782398,0.1834590137,0.6919991374,-1.0479919910,-0.9896458387,-0.2847566903,0.7601591349,0.5769826174,1.3861632347,-1.7981691360,0.5600209832,0.1504398137,0.6467001438,-0.3333204985,0.2885289490,-1.5847134590,-0.6807080507,-0.4037251472,-2.4674272537,0.1292898357,-0.1086133048,-0.3344806433,-0.4761407077,-0.4093746841,0.8121746182,0.9868445396,-0.4382044971,-0.7254872918,-2.4136452675,-0.3176845610,1.9473218918,0.2170838863,0.2626359165,0.1446496695,2.9867980480,-1.6840299368,1.3493431807,-0.6842327714,0.2324342579,0.8585680127,-0.6588760614,-0.3588640988,-0.3767255545,-0.5748527646,0.2612748146,0.1919667572,0.9821797609,0.4179340303,-2.2200431824,0.2395582497,-1.1190239191,0.7308257818,-1.2480312586,0.7303984165,-0.6898256540,0.3308303952,-0.6568310857,-1.0560706854,0.3887234926,0.5087570548,0.7185772061,-0.2959088087,-0.2250892967,0.4436039627,0.7100724578,-0.3117805421,-0.7210696340,1.4668381214,-0.6216652989,0.4426654875,0.4567816257],[-0.8505325913,0.8974496722,0.8174543977,0.4282577932,0.8738188148,-0.3808510900,-0.5091495514,2.3714761734,0.1522615403,-1.2650282383,1.3257935047,-1.4895132780,2.3313984871,-1.2792991400,0.7764997482,-0.7210823894,-0.1681635827,0.1529988945,0.4695449471,0.1967510283,0.2921490669,1.3127886057,-1.2193900347,0.2852847874,-1.2616995573,-0.7584261894,-0.1306209564,0.9959502816,-0.8483362198,-0.6962991357,0.3551653624,-0.2280855030,-0.7159309983,-0.3786600828,-0.4753340483,-0.1977416724,-0.9485037327,0.3293333650,1.2257366180,-0.1455384195,-1.4984012842,-0.1919256151,2.1786687374,-1.0019141436,1.0610024929,0.6823122501,0.0800496340,0.9403626323,0.5023782253,0.3214206100,1.6189458370,-0.1983540952,1.7617611885,1.1387735605,-0.8849478364,-0.8603631854,0.5942631960,1.4637345076,-1.2320071459,1.5357743502,-2.1328830719,1.7089049816,1.1534118652,-0.0069381776,1.1776837111,-1.5097141266,0.2115106434,-0.1186552644,0.1261516213,-0.3330809772,-0.1185161993,1.1600736380,0.3577325046,0.7345383763,0.7769987583,0.7655192018,1.6940709352,0.6339361072,1.0973848104,0.4087703526,-1.7375053167,-1.0483746529,-0.4302817583,1.5430914164,0.0006288328,-1.2928329706,0.3590690494,-0.3027823269,-0.9697774649,-1.5566012859,0.5141104460,1.1575758457,-1.1170324087,0.0305965990,-0.1459143758,-0.7981154919,0.4145491421,1.4571155310,-1.3967839479,-0.0655855462,-1.0104351044,-1.1136292219,-0.1907861084,-1.8058137894,-0.2570574582,-0.6587927938,-1.3274798393,-0.1549696177,0.2305874527,-2.0742232800,-0.0295557063,1.3538379669,0.3438618183,-0.2380728573,-0.2246685326,1.5747584105,0.6863652468,1.3693779707,-1.2951009274,1.3435353041,2.0449006557,-1.6192568541,1.7248616219,-0.1750580519,0.3598082364,-0.8229542971,-0.9378738403,0.1098855510,-1.0314471722,0.0641856417,0.6701291800,1.5685896873,-1.6793966293,0.4545374811,-0.0591463894,-0.0386027545,0.1191953644,-1.2499835491,0.9295192957,-0.5419933200,0.1619770080,0.1354142129,1.5632513762,-0.3869238198,1.2132548094,0.0554801188,-0.6512631774,-2.6216135025,-0.2254151106,0.4330559969,-0.5367842317,-0.8901445866,-0.8257043958,0.5760731697,-1.5437245369,-0.7159849405,-0.9570081830,-1.3737111092,1.6673197746,1.3580203056,0.2791488171,1.0237982273,0.9078584313,-0.8839287162,0.0231411308,-0.3945089281,-0.2564010918,-0.5195775628,2.2790925503,-0.9839748740,-0.1140683740,0.7237284184,0.1794247627,0.9464429617,-0.4277381003,2.1282601357,0.0217097141,-1.7380595207,1.3669571877,-0.2642624676,1.5720680952,-1.8166810274,0.5242646933,-0.8335030079,-2.0098788738,-0.6536083221,0.2945873141,-1.4782997370,-0.1516771913,1.1408663988,-1.4657688141,-0.3113844097,0.7662166953,-0.4563453794,-2.4438807964,-1.4533135891,-0.4236075282,0.5318500400,-1.8602722883,-0.6261865497,2.0996797085,-0.8635161519,-1.2204080820,0.4388450384,-2.0069413185,0.2024406493,-0.8838145733,0.0309155099,0.7476049662,0.8999215364,-0.8639245033,0.0988463908,-0.6415423751,-0.6678134799,-0.0940411165,-1.7194350958,-0.4066540897,-0.4778663814,-2.6078004837,-0.9008126259,-0.2668288946,0.7076120973,1.3418669701,0.4082632065],[-0.4423490763,2.5291156769,-0.0829270333,-0.9171121716,-0.7369276285,-1.6547844410,0.3231929243,-1.0614206791,0.3161898851,-0.3398891389,-0.0626130924,-0.9866225123,1.2956130505,-1.7446339130,1.8742061853,-0.0142730968,-0.3598657548,-0.6989034414,1.3945087194,-0.0364240184,-0.6663540006,-1.7180625200,-1.2895328999,-1.8650425673,-0.2848217785,-0.6872664690,0.6471204162,1.3397916555,0.3441358507,-0.6507552266,-0.0044669984,-0.5204245448,0.3537707329,0.6218783259,0.8653511405,-1.9395599365,-0.3520051837,-0.3968527317,-0.1872099787,2.5866026878,1.3923414946,0.4628205299,0.0473618843,1.5894398689,1.1269013882,0.3289514780,0.0482734181,-0.6642089486,1.3771688938,-0.0996530503,0.3009798527,1.1523993015,-1.0771466494,0.8700125813,-1.5626024008,-0.1853292584,-0.4131501615,0.3208904266,0.7904941440,0.1825299263,0.0197186228,-1.1914066076,-0.9240799546,0.8452393413,-0.1513461173,-1.2526690960,0.1262753606,-1.6897244453,-0.4013103247,1.9904233217,-1.3785617352,-0.0342551842,1.2655111551,0.2004050314,-0.4384508431,-0.9500871301,-1.6665233374,-0.7320570350,1.6763832569,1.4295755625,-0.2407578230,0.8036060333,-1.8491408825,2.9870183468,-0.6388540268,-1.8830124140,-1.6932355165,2.1526920795,0.1216804460,-1.4847382307,0.1124377400,-0.0760384947,1.3658516407,1.2392852306,0.0693709552,0.2876286805,0.0789157599,-0.8022515178,-0.2305098027,1.9006695747,0.9412038922,0.3962628841,0.6482299566,-0.8974066377,-0.4834775031,1.2487920523,1.1562736034,0.6536567807,-0.3170304596,0.2463564575,2.4179408550,-0.4878221750,0.8989579082,0.4149280190,-0.3923274279,1.2863267660,0.7034213543,-0.7199124694,-0.4644466341,1.4654096365,0.3432749510,-0.9306016564,-0.7491942048,-0.1750261039,-0.7223092318,0.1536332071,-0.3865046799,0.5169742107,-0.4115245044,1.6728409529,1.1052545309,0.2504644990,0.6183803678,0.1082001925,-0.6866397858,-0.1700413972,0.4843541086,1.3254795074,0.1514006704,-0.2798740566,0.4530131221,-0.1875061393,-1.5138655901,0.2696782053,0.3310975432,-1.5183287859,-0.4288371205,-1.0433771610,0.3195703924,0.2241828144,-1.7004652023,0.8005896211,-1.3455075026,-0.7087361217,-1.0131795406,0.4448340237,0.8383929729,0.7657484412,-1.0465499163,1.9463306665,1.6850688457,1.6361572742,-0.4113813639,0.8208792210,-1.3112231493,-0.4735149145,-0.9699966908,0.8628758788,0.9517716169,-0.9734086990,-0.6572603583,0.5387002826,-0.2033254206,0.6137370467,1.5639606714,0.3512731194,0.2677857578,0.1491088122,-0.6859575510,-0.1748508811,0.0057693808,1.2078301907,-0.2483086884,-1.5793319941,-0.1355962008,0.5014082789,-1.3090702295,-0.9902474880,0.1836945713,0.3580158055,0.1922743022,0.0095682470,-0.2284177840,-0.5888280869,-0.9356576204,0.5763784647,-1.0042114258,-0.3705872595,-0.2505876124,-0.7080980539,0.1050026119,1.1257755756,-0.4953170419,0.9546896219,-0.6923274398,0.3183096647,-0.5688853264,-0.6310497522,0.2750060260,0.0254878420,0.2571436465,0.5700058937,-1.7250375748,-1.4157315493,-1.4600043297,-0.8703076243,0.0653536320,1.1396144629,1.4283490181,-0.6810770035,0.5877169967,0.1974487454,0.0439355373,-0.5478091240],[-0.8501861095,-0.1452676207,-2.0167658329,-0.1970911473,0.0701859370,0.7894083858,-1.7675129175,1.2736612558,1.4884271622,0.5635340214,0.6796386242,1.2423328161,0.0540213957,1.3500727415,-0.2239092141,1.4953960180,0.2478047758,0.5561262965,0.2338157147,-0.1017286703,-0.3675415218,-1.7073664665,1.6565083265,1.6814539433,-0.0360563397,0.4497372806,0.0980281234,1.4531240463,-0.6414846778,0.1412290037,-1.0449951887,-0.4438725412,0.9779627919,0.7688708305,-0.5568965077,-0.9761307836,0.7911739945,1.4035484791,-0.5847625732,-1.4764374495,-1.2948927879,0.7119658589,-0.8156015277,-0.2008070648,0.8081845641,1.5236290693,-0.3609860241,-1.0343962908,1.7678860426,-1.1862468719,-1.7334928513,-0.8210963607,0.3756120205,0.2281047553,-0.8125753999,1.4891103506,-1.8475048542,-0.1294121295,-0.6484277248,-1.0389952660,-0.2641805112,-1.4201703072,0.9795006514,-1.3405352831,1.4375221729,2.0169596672,1.1405482292,0.7233772874,0.5449270606,-0.7012047172,-0.2516741455,0.1970013082,-0.7681846023,0.5858729482,-0.3477722704,0.2033023685,-2.0873692036,0.9893494844,-0.5800985098,0.6182278395,-1.9546772242,-0.6086624265,-0.7773972154,-1.2600343227,1.8167859316,-0.0987552106,-0.1810840815,1.2722183466,-0.3335888088,1.5315827131,-1.3085680008,-1.2660052776,0.9042910337,0.3279466331,0.3045103848,-0.3153513670,-1.5934838057,0.4933577776,-0.3654850125,-0.5737106800,0.2879352868,-0.4627600908,-1.0108826160,-0.6777721047,-0.8984144926,0.2504751086,0.8091649413,2.6401121616,1.7911562920,-0.8305351138,0.8344239593,-1.4438226223,0.3508011997,0.2592615485,-1.5862669945,-1.4240231514,1.5163314342,-0.8924900293,-0.0843262747,-0.7113832831,-0.2796256840,-1.0860145092,-0.3189810216,1.4416177273,0.5681290627,-0.3779550195,0.4941016138,0.2960478067,-1.2442852259,0.0931274667,-0.0518829748,0.1845656782,-0.7338954210,-0.5237256289,0.8684477210,0.6828365326,0.5501427054,0.8220844269,0.7082092762,1.0605940819,-0.2782573998,1.0126662254,1.1477307081,-0.0826153383,-1.4017388821,-0.7767349482,1.1191364527,-0.4544364214,2.3977463245,0.5580090880,0.8164888024,-0.7969041467,-1.2697274685,1.5049589872,0.1810080558,-0.2684362829,-0.3502424061,-0.4023967385,1.2367614508,2.0550661087,-1.4493792057,0.1685050875,0.4869134128,0.8313385844,0.6722729802,-0.5500596166,0.2919599712,-0.0776337832,0.3386179805,-0.5368257761,1.8082138300,-1.5536627769,-0.2598360777,0.4972334504,-0.5991903543,0.1945166141,0.2947003841,1.3485603333,-1.8906611204,-2.3424110413,-1.2688889503,0.0344363041,0.2485752255,0.9743855000,0.5960764289,0.3309855461,-0.5189491510,-0.8919455409,1.1610465050,-0.5045477152,1.3423473835,1.5131756067,-0.6779407263,-1.7057740688,1.3286938667,-0.2861714661,0.3773959279,-0.0487306230,0.9769710898,0.0155381076,0.2035691142,0.2230139375,-1.5775175095,0.3229602873,-1.2306278944,0.0244359765,-1.4809737206,1.2955615520,-0.0120471921,-1.2050281763,-1.8280923367,-0.1050783247,0.9775385261,0.7549197078,0.0924711451,-0.1297639459,0.2104711384,0.2063230872,0.0318034813,0.6998687387,-0.0475344919,0.0653456599,-0.4324573576,-0.2389912307],[-0.0385269783,-0.3596199453,0.9366604090,0.2594126165,0.8580753803,2.4031043053,0.6737225652,0.6189863682,0.3409185708,-1.7746075392,-2.3424885273,-1.2195376158,0.0443152338,-0.3048034012,0.4648899436,1.8452289104,0.3149746358,0.0522343889,0.3195095360,-0.7438256145,0.2410807163,0.5562276244,0.2206500024,0.3171322346,-2.2169415951,0.5470215678,0.9518302679,1.1646939516,-0.4791328311,1.3888785839,-0.3748681843,0.7778351903,0.8615957499,-0.8165302873,0.5390430093,-0.6619489193,-0.1029545069,0.0851025358,0.9509700537,-0.5260720849,0.8613053560,-0.9844242930,1.3661401272,0.5948407054,0.1009211168,0.0307289679,0.1981360763,0.8672862649,1.1369253397,-0.1251385957,0.1661457717,-0.9388157725,-0.2449741960,-2.4094924927,0.0623265356,-1.2366584539,1.8895184994,1.3102403879,0.9462187886,0.3420774639,0.3925039768,1.2095966339,-1.4361178875,0.5183697343,0.3537858129,1.2890650034,-0.2607053518,0.2743812501,-0.6387026906,0.6534357667,0.1286210418,0.5044686198,0.1814749390,-0.4254333973,1.5863105059,1.7321563959,1.5264887810,0.1210952625,-0.5436092615,-0.9529995918,-1.6551265717,-0.1553141028,0.2441254854,-0.7731710076,-0.2470936179,0.2712610662,-0.6617076993,1.0395247936,0.6841502190,-0.9874541759,1.3303834200,-0.6565662622,1.1300296783,-0.1293801665,2.3106272221,1.4594422579,-1.0526741743,-1.0220749378,0.0577553660,-0.5097522736,0.3840633333,-1.2838438749,0.7513307929,-1.6153922081,-2.3277418613,-0.6914343238,-0.4368054569,0.3361124098,0.8448717594,0.4833911061,-0.4330910444,0.6864856482,-0.7480033040,0.5588150024,-1.5406918526,0.8605827093,0.0865264311,0.6239672303,-1.1139910221,-1.5116748810,-0.7817136049,1.5633437634,1.6206901073,-0.1340485662,0.5526981354,-0.0745343417,-0.5309984684,1.1839113235,0.3718740642,-2.9650082588,0.6733871102,0.3198423088,0.6197563410,-0.0403066762,-2.3952093124,1.2700175047,-0.8881990314,-0.7429559231,-0.8290517330,-1.1331794262,-1.8599679470,0.7857927084,-0.2451763600,0.3910550177,0.1410217583,0.7675020099,0.7578459978,-0.3758762479,1.2606920004,-2.8472449780,1.4555310011,-0.2381768674,-2.2845680714,0.1599343717,0.7724515796,-0.1817385405,0.8262177706,-0.7928229570,-0.8502887487,0.1392326504,0.4767868519,0.7469817400,0.4203766584,1.9672571421,1.5491867065,0.0936574414,0.1747819185,0.1045727581,2.1363508701,-1.2314023972,-0.6329893470,-0.4213211536,0.8636973500,-0.3888166547,-0.4820105731,0.4317566156,-1.1235409975,1.3373652697,0.7710515857,-2.7142024040,-0.2182231843,1.6929919720,-0.0195358247,1.3989555836,-0.3875274360,-1.0358554125,-1.1316094398,-1.5382269621,0.9663111567,1.0785235167,0.3704659343,0.2607283294,-0.2777065337,1.8606232405,-0.6065781116,0.3053328991,0.6207325459,0.0930356830,1.2091175318,0.1401986331,0.8139212132,0.0959253460,-0.8038667440,-0.9163048267,-1.9060095549,0.1097709686,-0.9959290624,-0.3285111785,1.2623658180,0.2582879066,1.3890106678,0.1220844612,-0.4873121381,0.5140516758,0.3178030849,0.5869648457,-0.3526543379,-0.3959585726,2.3274693489,0.1709618419,-0.9874722362,0.3988903761,0.3557094634,0.9122059941],[-1.7059065104,1.5565053225,1.3327462673,-0.3147378266,0.1606792957,-0.5181043744,-0.0744991302,-0.1525735259,0.8446281552,-1.9320089817,1.4879050255,0.1854588836,-0.1515202969,0.2979521453,0.5038195252,0.8791267872,-2.6462893486,0.1011591628,0.5374528766,-0.0020966744,-0.0970841497,1.9185301065,0.5449617505,-0.4810386002,-1.5802356005,0.3312494159,-0.8791120052,0.3050711453,1.0962580442,0.5063979626,-0.0172869842,0.9996516109,-1.5001869202,-1.8311299086,-1.1142653227,1.2140845060,-2.3504610062,0.3383857608,-0.2453690469,1.0279306173,0.0342444330,1.4575858116,0.7556669116,-0.2027938962,-0.3228623867,-0.1250157803,-0.2761982381,-0.0912716240,0.6139395237,0.1320572197,-1.7598555088,3.8252158165,2.3801681995,-0.5304226279,0.4853073955,0.4158081114,1.9410279989,1.3093279600,0.4837481976,1.0330768824,0.1908234954,0.8460777402,-0.4605898261,0.4409380257,-0.0441601798,1.8047442436,0.9391747117,-0.9743002653,-0.6022531986,-0.9103568196,0.7067342997,1.0348250866,-0.1873424798,-0.4120945930,-1.6666873693,0.1700548679,0.8848538995,-0.5189235806,0.6518719196,-1.4336264133,2.1102340221,0.6158944368,2.0755994320,0.9486870170,1.4196146727,-0.6370040178,1.4725530148,-0.2676182985,-0.1825240552,2.1773166656,1.9086070061,-0.1137179285,1.5164897442,-0.2023674399,0.2179028988,1.7951279879,0.3171834350,0.4730903208,-0.9259622097,-0.5089650750,0.7982944250,1.1375516653,0.3526133299,1.3780834675,-0.0489090681,-0.1768110543,0.0780580640,0.0144599462,-0.0084795738,-1.2281516790,0.9156061411,0.2464201152,-0.0167766493,-0.1420723051,-1.5855610371,1.6599810123,-0.0597732589,-0.0273799822,0.8296191692,0.8847360611,-0.5626728535,-1.7483476400,1.1403344870,-0.0690325201,1.3960663080,0.6275839210,0.5832805037,0.0118617360,1.2410900593,-0.6006539464,0.2910424471,-0.5646578074,0.4662118554,-0.5337926745,0.3868106008,-0.4784782529,0.5543790460,-0.4056519866,0.7447139025,-0.0469487794,2.4892342091,1.3938800097,-0.1264778972,0.4362581074,-0.8975135088,-2.5608577728,-0.7694765329,0.6344984770,-0.1369070113,0.4036124349,-0.2340947241,-0.9804391861,0.4205627739,1.2156971693,0.1035736427,-1.7539309263,1.0600017309,1.2703372240,-1.2950500250,-1.5065121651,-1.3031721115,-0.1997547597,-0.5540348291,1.3706039190,0.3994727433,-0.2424000353,-0.5245046020,-0.0897789225,-1.1353756189,-2.1022062302,1.2222563028,-0.2435424030,-0.0676702186,-0.8102147579,0.3558183014,2.0535726547,1.0213717222,-0.2912064791,-0.7373991609,0.3800884783,-1.0866839886,-0.2113780975,-1.0243973732,-1.0861248970,-0.1851570606,0.4320114851,0.5590879917,-0.3961796761,0.9062599540,1.5587188005,0.7504378557,-0.4613761306,-1.9730011225,-2.1190068722,0.4325800836,0.0537056699,0.8598451614,0.6354776621,-1.6893638372,-0.2476013154,0.9518808722,1.5663367510,1.6191905737,-1.6549032927,-0.3134933412,1.4668923616,-1.8313214779,-0.3367227614,-0.5707110763,1.6618161201,-0.3145177364,0.0156628042,0.1667782664,0.8142173886,-0.3663666248,0.2926310003,1.5102934837,-0.8687940836,-0.5283280611,-0.5114175677,0.6322653294,-0.1110791191,0.0009631776,2.9895756245],[1.4097995758,-0.5420150161,0.9210902452,0.5834612250,0.7994394302,-1.3424725533,-1.0264796019,-1.3861973286,0.1247647181,0.1029771194,-0.4731315672,-0.5921694040,0.7910331488,0.6274262667,-0.1472722888,2.5348424911,1.7897282839,0.6568221450,1.7179471254,0.5924511552,0.5724430680,1.4984492064,-0.5010377765,0.0174681451,-0.5767454505,-0.4235750437,-0.3686167598,0.1869814843,0.8584616184,0.0341167487,-1.2190752029,-1.8360806704,-1.7777047157,0.3351447582,0.2758966386,0.2172588408,-0.1049501747,0.4783955216,-0.3389906883,-1.5148092508,-0.2531677485,1.5903692245,-0.2204401195,0.6823341250,0.6670246124,0.3315284848,1.3474885225,-0.6062452793,0.0387766436,0.3242726922,1.6093249321,1.7843703032,0.5993479490,0.9222139716,-0.7880330086,2.2105967999,-0.7617409229,1.2312707901,0.6148130298,-1.4690411091,-0.4746180475,-0.6721571684,1.0992940664,-0.0940861106,0.6958556771,-1.2895159721,0.2084652185,-0.7831521630,2.0253167152,-2.4375717640,-0.0784258023,-0.2919144630,0.9683870673,-0.2354955524,0.5750846267,-0.3641513586,0.8403220177,0.5809874535,0.6007576585,0.0460261889,-0.3048984408,-0.7436544895,-0.1375458986,-0.0768285319,0.6623673439,0.4524120688,0.3483710289,0.6208192706,1.7647602558,0.4740043879,-0.2359790057,-1.0643504858,0.4502330124,0.4655013680,-0.9948694706,-0.2585768700,1.3167483807,0.1989564151,0.2323360294,0.7958525419,0.3262020648,-1.3898649216,0.1671944559,-0.9119162560,0.2252446115,-0.9267496467,1.1519112587,-0.5130313039,0.1882673055,0.6804435849,-0.4746681154,-0.4878197908,0.1800938845,0.1051743627,-1.0580087900,1.4008100033,0.5489974618,0.5481874347,-0.5832469463,-1.2305303812,-0.3625142276,0.0120345866,0.2264911383,1.2822241783,-1.2907737494,-0.8618292212,-0.9438347220,0.4134423137,-1.0703722239,1.4725133181,1.4156024456,-0.4514758289,-0.5120148659,-0.6924622655,0.3693538606,0.8347567320,0.6099922657,0.8409538865,-0.2589316368,0.1839996278,0.3036150634,-0.0842262655,-0.2866791785,1.1813234091,-2.3766260147,-0.9739966393,-0.9959408641,-0.7160070539,0.3816682100,-1.7369004488,-0.7823039889,-1.6728433371,0.5592993498,0.6593230367,-0.6877341270,1.5265905857,-0.0742725432,0.8444102407,-0.7459825873,-0.0516187772,0.2302361876,2.4707098007,-1.7039589882,0.1352397501,0.4142930210,0.1444460303,-1.1740654707,-1.2802608013,0.1036694646,-0.3136852682,0.0832345337,1.2571145296,1.8054370880,0.7833987474,0.2159276009,1.3131119013,1.0496573448,-0.2494902462,-0.9532986283,0.8174754977,1.8209816217,-0.4751147926,1.5540887117,-1.6372287273,-0.5182297230,-1.2041864395,-0.7944150567,-0.2081774175,-1.7688667774,0.9372014403,2.3383221626,1.2419533730,-0.1160321012,0.8310201764,0.3418564796,0.0724793375,1.4042508602,-0.0997533277,-1.5711517334,-0.1969444603,1.4441918135,-0.7084956765,-0.4584809840,-0.4085344374,-1.7429348230,-1.4977319241,-0.2693070471,0.9721183181,-0.4137994051,-0.6265276074,0.5437030792,-1.2866847515,-0.7732895613,-0.8717803359,-0.0661167651,0.0868368670,-1.4115611315,-1.1569627523,-1.9649760723,-0.1715185195,-0.9807068706,0.1386085302,0.4682126045,-0.7425556779],[-0.6112797260,-0.9979614019,-0.3386137486,-0.1766004115,-1.7047775984,0.1936862469,0.1481569558,-1.5413073301,0.3674076796,0.0811279714,-0.9216552377,2.4965655804,0.8719742894,-2.0750610828,-0.8497855663,0.5540226102,0.2907184958,0.6172452569,-0.4571862221,0.4977155328,-0.9748178720,0.5232276917,-0.0748741329,-1.2881364822,1.3667689562,0.1484711468,0.9896100163,0.3713888526,-0.4138089418,-1.0227590799,0.3866163194,-0.9294429421,-1.0626591444,-0.0199987087,0.0961680189,0.2672078311,1.2036789656,0.7388323545,1.2408499718,0.6849725246,-0.2830341756,1.7859346867,-0.6817632914,0.6170616746,0.7021603584,-0.0052990373,0.0670303106,-0.3759177029,-0.5266339183,0.5763165951,-1.1446790695,1.0514925718,-1.2716163397,-0.3700241148,-1.7133955956,0.4637143910,-0.0871691108,0.4271546900,0.7751936316,-0.4274554849,-0.4503265619,-0.5333029032,0.5149218440,-0.2053171545,-0.6556538939,1.2024029493,-1.4963507652,-1.0015004873,0.2347760946,0.5991263986,-0.5835256577,0.0645779371,0.4332801104,-0.8684640527,0.6871785522,0.2869982421,0.0550472997,-0.6587293148,-0.5711126924,1.2947942019,-0.7356818914,-0.3393787742,-0.3914661109,-1.1938683987,-0.0813956112,-1.5593749285,-0.3439086378,1.2804453373,-0.9656952024,1.6998059750,0.3437785804,1.2053562403,-1.3430669308,-0.2202706486,-0.2032942921,-0.4607440829,0.6888646483,0.9494631290,-0.5992566347,0.6588060856,-0.8213128448,0.3451172411,-0.1872069687,0.5875008702,0.6419531703,0.8461623788,1.4980093241,1.5428068638,-1.0534813404,0.1087473035,-1.9668846130,-0.4938659072,1.0318287611,1.4068181515,-0.1060440391,0.1013416573,-0.3797146380,-0.0020411925,-0.4893866777,1.2483414412,-0.9816740751,1.0710183382,0.3810484111,-0.2293544710,-0.9001345038,2.1937174797,-1.2616010904,-1.2372906208,-1.6419368982,0.2051976770,-0.3173463643,1.1138305664,0.0690412819,-1.4165972471,-0.9509050846,-0.4597766995,0.2946996391,-0.7381731272,0.9412861466,-0.6347444654,-0.2386984080,-0.5287588239,-0.2509096265,1.6997805834,1.9507800341,-1.3863464594,0.0064332704,-0.7473371029,-0.5901803970,0.7893643975,-0.0761919171,0.6964703202,-0.6725970507,0.2126092315,0.7568253279,-0.9808959961,0.1224446669,1.5959967375,-0.4214015007,0.7471168041,0.4177250266,2.1514711380,-0.2517547309,-0.5861119628,-0.7115761042,-0.2423496544,0.4907200933,1.3639013767,-0.0233906917,-1.2201986313,0.5158403516,0.2809210420,0.1244746000,0.2069127262,1.4315224886,0.3899920881,-0.2881478667,-0.4212123156,1.2691807747,0.1441033632,-1.9708248377,-0.3076345325,0.6847977638,-0.2083214968,-0.9662112594,0.5662410259,-0.4710945487,0.6587586999,0.4371346533,1.2881650925,1.8142797947,-0.2653055191,-0.8061235547,-0.2660584152,1.1144868135,0.4511666894,-1.0467418432,-1.2955060005,0.4813936055,0.0717912763,-0.2718241215,-1.4206292629,0.5015422106,-1.5696884394,-0.7510741353,0.5357149839,-0.4761382341,0.3027769625,1.0439693928,-0.2968443632,0.4115947187,0.1417464614,-1.7686996460,0.5488359928,-0.4652283788,-0.2023715526,2.0392110348,0.9476948977,-0.5790069103,1.3782595396,0.0463112034,-1.4538191557,1.0799579620,-0.4495364726],[0.6995849013,0.9958032370,0.1410770565,-0.7974423170,0.7335600853,-0.8051303029,-0.5576012731,0.6688166261,-0.6395130157,2.0262405872,1.8946053982,0.4527872503,0.3898622692,-0.6865835190,-0.8698310256,-0.1018036604,-1.2498087883,-0.9337173700,-0.6132207513,0.1670824289,0.5964810252,-1.0693768263,1.1424970627,-0.4055011570,-0.0484616831,1.4438123703,-0.3742063642,1.9146231413,-0.8712691665,-1.6244757175,-0.9451323748,-1.8532702923,-0.4922710955,-0.7652184367,-1.5269926786,1.0519231558,-0.7186142802,-1.1369289160,0.1738671213,-0.8412277102,-1.2510764599,-1.5560952425,-1.2814587355,0.2802022398,-0.1892165393,-0.3990669250,-0.4418022037,-0.1641026884,-0.0208841823,1.4467082024,-0.1773938984,-0.2656075656,0.9284580946,0.8337346911,1.3918434381,-1.5541487932,-2.3029310703,1.2056473494,-0.7153151631,-0.6681523919,-1.8057876825,3.0629129410,1.0158607960,-2.2026958466,1.0273900032,0.9297968745,0.7835995555,1.2467330694,1.9681586027,1.1947727203,-0.2920994461,-0.2260138839,-0.4726005197,0.7220215201,1.1053067446,0.6730644703,0.2965455055,0.5883324146,0.9228509068,0.1861357391,-1.1552476883,-0.1277422905,1.7361689806,0.5216132402,-2.2289402485,-0.5762023330,-0.3333890736,1.0773743391,1.5460308790,-0.4793303311,-1.2360799313,0.8915560842,0.5807973146,-0.9052379727,1.8003457785,1.5324009657,-1.0009678602,-1.3704814911,-2.0461034775,-1.6933704615,-0.3919988871,1.0435934067,-0.1997463107,-0.2654405534,-0.3796259463,1.1171686649,-1.1622449160,0.2430635393,1.2648487091,0.2460895479,0.7015656233,-0.2465724051,1.4588391781,-0.3708950579,0.1133627743,-1.8181365728,-1.2764829397,0.1797037721,-0.9096319079,0.1833432168,0.6413455606,0.2378626913,1.0471014977,2.1745960712,-0.2824437916,0.6198886037,1.4454858303,-0.6241021156,-0.0874009505,-1.2409427166,0.6960406303,-0.4450871348,0.4783905447,-0.7847806215,0.3396017849,0.8402874470,-0.3396469653,-0.5243570805,-0.4028583765,-0.4860184491,1.6486966610,0.9750620127,-0.4632399082,-0.0831132308,0.4595403671,-0.3360282183,1.2276799679,2.9337465763,-2.7567064762,-1.8911068439,-0.5368624926,-1.2443982363,-1.9356986284,-0.4737740457,1.1805994511,0.4548642337,-0.3293735385,2.1251163483,-0.1460131258,0.5125373006,-0.6775860786,-0.3787424862,-0.4837478399,0.1269359440,-0.5968005657,0.0434482470,0.0882226899,-0.4975458682,-0.8245134950,0.5286591649,0.0518661812,-0.1529803872,0.2797895372,-0.2412377894,-0.3942654729,-1.4258618355,-0.1979169846,-0.4607566595,1.9837756157,1.3194955587,0.4016983807,-0.1167668849,0.0360824801,0.7133179903,-0.0223064348,1.4471542835,-0.4008077383,0.9587165117,1.2700641155,0.2602819502,-0.1262758076,-0.7956354618,0.9213441014,-0.8105702996,1.2343528271,0.1792832315,1.0454692841,1.8160818815,-0.2565951049,0.6567875147,-0.9121094942,0.0882756636,-2.3321690559,-0.2221315503,0.4362458885,-0.4040724635,-0.9728463888,1.8061887026,2.8223960400,1.9238047600,0.4281252325,-1.0113406181,-1.0569467545,0.7463124394,0.7058548331,-0.2373242229,-1.2442312241,1.4772787094,-0.1161244661,-1.1747477055,0.0377737507,0.3676355183,1.0512756109,1.0939421654],[0.3265855610,0.5304086208,-0.4535779953,-0.1297783256,0.9609361291,-0.6423221827,0.1481865197,-0.1527392268,0.2322229594,0.6704090834,-1.1654951572,-0.3747846484,0.7840104699,1.1033495665,0.5778945088,0.5689970851,-0.0741880238,0.8559090495,-0.6340079904,-0.2442087978,-1.2459036112,-1.0313720703,-1.2667640448,1.8217892647,0.8577761650,2.7689638138,-1.1735712290,-0.5874087214,-0.7750111222,-0.9776890874,-0.9262546897,-1.0075428486,1.1539785862,-0.2632883787,1.9082475901,0.7450637221,-0.4940591156,0.6841157079,-0.5536890030,0.6943057775,0.6760548353,0.9306396246,1.3879038095,-2.4359529018,1.0196198225,-2.3955872059,1.2872189283,0.4549034834,-0.0617699176,-0.7158759832,-0.4316509962,-0.6489620209,0.5559383035,1.0439940691,-1.3500230312,0.6517568827,2.4761118889,-1.4052060843,1.4435579777,-0.4848107994,0.0389968641,0.7329373956,0.0906467363,1.3932663202,1.6153687239,-1.0152901411,0.3226601481,0.3640216887,-1.2530380487,-0.3811959624,2.3525457382,-0.3899695873,-0.3235520124,-0.5818331838,1.0788103342,1.2848529816,-0.5897274613,1.8995616436,-0.0095527787,0.4356729984,-1.2555909157,-0.7084031105,-0.5457687378,0.3907106519,0.4804974198,1.0105402470,-1.1896303892,-0.9705888629,0.7118433118,-0.6612381339,-0.3134835660,-0.6854033470,-0.1347203255,1.0188876390,1.8438681364,-0.5356816053,0.8939056396,-0.2387982011,0.3773968518,-0.5067309737,0.3475973010,0.1116873249,-0.3291538656,-0.4861176014,-0.1610072404,-2.0568583012,-1.5355416536,-0.7158253193,-0.9671429992,0.3337282538,-0.5167418122,1.8286303282,-2.3692855835,1.1425521374,-0.4168982506,1.1942421198,-0.5849003196,-0.4810088575,-0.2370169908,-3.2313024998,0.1890334487,0.2700953782,1.3857370615,1.1544367075,-1.5452423096,-0.7408530712,0.3481022120,0.4148260951,-0.3747694790,-0.6615316868,1.5172027349,-2.1836273670,0.0456692502,-0.0833318532,-0.9288641214,1.1208633184,-1.5847899914,0.8110761046,0.2517983913,0.3271596134,-1.8162002563,0.4868193269,0.9116107225,-1.9339040518,-1.3720557690,-0.6756509542,-0.1909210086,-1.2321888208,-0.0097519904,-0.6435344219,1.5714240074,1.3172254562,0.7528502941,0.3804392219,0.7566663027,0.3085334003,0.4007671475,0.3258664608,-0.9499855042,1.4211901426,1.2636379004,-2.1053454876,1.0966620445,-0.2560794652,-1.1866722107,-0.1082164645,-0.9384649992,-0.4065252841,-0.3620557785,-0.6806376576,0.6351390481,-1.2025653124,-1.6817848682,0.4998220205,1.2873026133,0.6916964054,-0.1651842296,1.1613186598,-0.5508473516,-1.3952718973,-0.7212381959,-0.2760586739,-0.9742767215,0.9609985948,1.8242496252,2.2517738342,-0.8537518978,-1.2370250225,1.6758298874,-1.2859879732,0.8290636539,-0.0007398854,-0.6669198871,0.7239009738,0.4930982590,2.3166418076,0.2593319416,-0.4904332459,0.9100781083,-1.5406177044,1.0337108374,-0.0417323932,0.1815911382,0.8316265345,1.1683583260,0.2908100188,-0.8651617765,0.1349878907,0.5809453130,0.6313925385,0.9107890129,-2.0079622269,2.2667987347,0.1362239718,-0.4542431533,0.8893927336,-1.3563278913,-1.4928052425,-2.5350847244,-2.5609161854,-0.5255851150,0.0438330211,-1.6740049124,-0.3690219522],[2.2857151031,0.9468349218,-0.3893817365,-0.0589064620,0.4195207953,-1.5684384108,-0.9302867651,-0.9671948552,0.6613282561,-0.8047224283,-1.7897052765,-0.2297337502,0.0934528559,0.2821525931,-2.0221195221,0.5018086433,-0.1606549770,-0.0830269158,1.6817605495,-1.3330610991,0.2334875464,0.9362577200,-0.8867135048,0.6424555779,0.6285135150,0.8862875104,-0.7600226998,-0.1048611850,-0.1568768024,-0.1049152017,-1.1293969154,0.4765573442,-0.4045842588,1.0629390478,-1.0136929750,0.1548542231,-0.0260496736,0.7368959785,0.6437106729,0.8629876375,0.9513613582,0.5345712900,-0.3428974450,-0.8951515555,-0.7072141767,-0.8290418386,-1.9441916943,0.3352486789,-1.6981418133,-0.6064324975,0.9298272133,-0.0455442220,0.4545024931,-0.6905018687,1.6557446718,-1.8656061888,-0.0764164850,0.1352482140,-0.9649400115,-1.1107592583,0.8871802688,0.3857004642,-1.3479410410,0.2313199341,1.1432842016,0.9774471521,-0.8249517083,0.9975424409,-0.4786781669,0.9934655428,-1.5249751806,1.4200520515,-1.3656040430,0.1201347858,-1.4304561615,-0.5459815264,0.3317955434,2.7635042667,0.5319283605,-0.4035831988,0.1656674147,-0.0348275118,0.9128665924,-1.0659881830,1.5218554735,0.4108059406,-0.8818579912,-1.8898864985,-0.8056340218,-0.2248883545,-0.3115158677,-1.9761034250,0.2347872108,0.0435139611,0.8959118724,-1.4635934830,-1.4139857292,0.5881743431,0.4636629820,0.4652281404,0.1577140689,-0.4330151975,-0.6331516504,2.2929763794,0.1627715528,0.4049665332,-1.0525262356,1.3194918633,-0.2121516019,-0.8326429129,-0.1556195468,-0.2294962704,1.7370721102,0.0576815419,-0.3365804553,-1.9751281738,0.3499570787,0.8850831985,0.6897427440,0.3805027306,0.8950417042,0.5748343468,0.9603128433,-1.5605746508,-0.9406257868,0.1844527274,0.4658717513,0.5011162758,-0.2088306248,1.1758521795,1.1635521650,0.4772557020,-0.7171283960,-1.7251302004,-0.7932379842,-0.7980470061,0.2869731784,-0.9468921423,-0.0588519983,-0.9246168733,-0.4363086224,-0.7276189923,-0.0040393011,0.1989732385,0.8241624832,0.4181136787,1.7991360426,1.6254104376,-1.0304411650,-0.8917178512,1.1133543253,-0.8911256790,0.3151792586,0.9218831658,0.9283659458,-0.7275673151,-1.8838251829,0.8151659966,-0.3029778004,0.3363099098,0.7451679111,0.6459904313,1.5648649931,0.4582104683,0.6197059751,-1.5433871746,-1.1824321747,1.0177524090,-0.7003726959,0.1140275300,0.5453913212,1.4336909056,0.2446464747,0.3940807581,1.9681952000,0.6926275492,0.2997940183,-0.5876314640,1.6486163139,1.3615334034,0.3137921095,0.7396493554,1.2286649942,0.6699882746,0.6917827129,1.3034600019,0.4483648837,-0.1363343149,0.9878246188,-1.0063430071,-0.3920277953,-0.2899178565,0.5260047913,0.6267307997,1.3453166485,0.2274762243,-0.0629633963,-1.1903443336,-0.6259784102,-1.4462623596,-1.6484795809,-0.9891631007,-0.6025660634,0.4475584924,0.2266521007,-0.0169600137,0.3846054673,0.3575282097,0.6688492298,-1.5007214546,1.0728921890,-0.9921048880,-0.8433125615,-0.2272939533,-0.5822410583,-1.4138011932,1.1975219250,0.6921180487,-0.2646894753,-0.7491828203,-0.4853253067,-2.2885367870,0.1974256933,1.4541417360],[-2.2559049129,-0.7094967365,0.4081711471,0.0805931389,-2.1951055527,-1.2974768877,-1.3876715899,2.2653400898,-0.9290544391,-0.0399216227,-0.6982507110,-0.5406550765,0.2805226445,0.2198031545,-0.5062025785,0.1667801738,-0.1336404383,-0.2559153736,-1.5829130411,1.7256107330,0.3249048591,0.7238846421,-0.0049705473,0.1070211902,-0.0440496467,-0.5861712694,-1.2517702579,0.9920070171,0.0184310395,-0.2521992922,1.0478001833,-0.8511126041,-1.0877283812,0.8057253957,0.5307689905,-0.2180033922,0.8828327656,-0.4397656024,-1.9520184994,0.7932648063,0.5082966685,0.3833715320,-0.9113702178,-0.3640641570,0.7479833961,0.6907197833,-1.7358108759,-1.0114843845,1.0597826242,1.4337526560,-1.9482045174,0.1001625508,-0.9070497751,0.6013016701,-0.3366676569,1.5244055986,-0.4733226001,-0.4377292693,-0.6595449448,0.9301330447,-0.4823285341,-0.4746858478,1.0638920069,1.8176459074,-0.0334640481,1.5302972794,-1.4440655708,0.3651750982,-1.0116682053,-0.1984334737,-0.1631214619,1.5133635998,-0.7044069171,2.1420927048,-0.1501505226,0.0840540752,0.5596846938,-1.3633971214,0.6780669689,-0.2560397387,-0.3821427524,-0.0927638486,-0.6855667830,2.3247995377,0.0162809864,-0.8881989717,0.5334579349,0.1402356327,-0.9644435048,0.2259918451,0.4266486466,0.7260495424,0.3118670881,-0.6226944923,2.6717684269,-0.5756723881,-0.4234935641,0.3955070078,0.6205416918,-1.3463940620,0.3289612532,1.3118835688,-1.3313224316,1.2080878019,-0.4787815809,0.1577800959,0.0663677827,-1.1360330582,-2.5606052876,-0.6011534929,-0.3796326518,-1.1565338373,0.1681053191,-1.2738326788,0.8146806955,1.1786562204,0.3972067237,1.4141077995,-0.8077833652,0.2641517222,-0.6645264626,-1.1914525032,-1.9402074814,-0.1699234396,-0.1685755104,1.9913597107,-0.5442168713,0.7998579741,-0.2674789727,0.2076923251,0.8118554950,3.0703585148,0.6901968718,-0.7633002996,-1.9414342642,-1.8525712490,-0.3263784945,-0.7097280622,-0.3387582898,0.5192115903,-0.0137907621,1.2764714956,-1.2416212559,-1.1341321468,-0.2834493518,0.2832931876,-0.8440576792,-0.3899221122,-1.2861567736,-0.0039009314,-0.3991547823,-0.1507866830,-1.0784094334,0.6873516440,0.2398162633,-0.8915411830,0.0160620846,0.1190035343,1.3595669270,2.2851662636,1.2656277418,0.5647453666,0.4386366606,-0.2786923051,-0.0450435877,-0.9073960781,0.8157382011,-0.2528479397,0.1099881604,-0.5259025693,0.9864002466,-0.3098907769,0.1118538901,-0.1846686453,-0.4830997586,1.3937410116,0.6692713499,-0.2855373621,-0.3015303910,-1.2393440008,1.0436235666,-1.3757392168,0.3743044734,-1.0209194422,-0.9742291570,0.4092015922,0.2553415895,-0.6028506756,-1.3848016262,-0.2793629766,-1.3943332434,-0.2918968201,0.9001582265,-0.4785775244,-0.6464923620,-0.0132998796,-1.1716102362,0.2067309022,1.0480587482,-1.4701331854,-0.2972986698,0.7241115570,-0.4221785963,0.3793757260,0.5570320487,0.8278382421,-0.5132340789,0.4097019732,0.1758598834,2.0416235924,2.2838354111,-0.4602448940,0.4201773107,-1.0172317028,-0.3338675797,-1.9052309990,0.0349817537,-0.3715404868,1.1575589180,0.4315404296,-0.6823685765,-0.2479079217,-1.3237612247,2.5582401752],[0.6202364564,-1.0272048712,-0.7202998400,1.8100709915,-0.2270940989,0.4384859204,-0.4506998360,0.4351607263,-0.1438863873,1.5774437189,-0.4889984727,-0.7328954935,-0.9708195329,0.1281945407,-0.1182762161,-2.0853116512,-0.9564751387,-1.6145592928,0.5195460916,1.4585510492,-1.0158007145,0.0924392417,1.1844977140,1.8504533768,-0.3173408806,1.0457521677,-0.8087899089,1.1489765644,-1.3961595297,0.2893914282,-0.2376532108,-1.2573392391,-0.8585844040,-0.4192205966,0.6898103356,-0.1702734679,0.7137411833,-0.1038140580,-0.5801587105,0.3293689787,-0.1780804843,2.1423149109,0.5859926939,0.8150727153,0.0960851610,-0.0801161528,-1.5072997808,0.3067455888,0.9733224511,-0.6691715717,-0.0896074921,0.7155662775,-0.8350681067,-0.8200449944,-0.2977332771,0.4310656786,-1.8563512564,1.2392419577,-1.5641089678,-0.0520131513,-0.3065806627,-0.1996008605,-0.1993302405,1.4793595076,-0.6917536855,1.1229088306,0.0617509261,0.2330419570,0.9543156028,0.3231183290,-0.3006335795,-0.5459107161,-0.3147141337,-0.0181828812,-0.2925636470,-0.1001829058,-1.5876607895,0.1932607740,-0.6446776390,-1.1246975660,0.7368519902,2.3475573063,-0.1616914719,0.3552694023,0.0810829923,-0.2951989174,0.3914378285,0.3118802309,0.6587822437,1.0558242798,1.5984497070,-0.2858626544,-0.9805120826,-0.7844312191,0.2835466862,-0.6526885033,-0.2016610503,0.8101894259,-1.3438426256,0.1658969969,-0.5662457943,1.3452924490,-0.2444649488,1.6518733501,-0.9791072011,-0.7222418189,-1.0671290159,0.4526766241,0.4453455806,1.2471383810,1.1432536840,-1.8908467293,-0.4724683166,0.4400090277,0.4122885764,0.7332125306,-1.7321538925,1.9177784920,-0.2359334975,0.7171579599,-1.1484898329,0.2975558341,0.1324217170,0.8498703241,2.0220580101,0.0642231554,2.0187501907,0.6166564226,-0.3392534256,0.8539775610,0.3858638108,2.0248098373,-0.1607054621,0.4017896354,-1.4190493822,0.3904750347,0.2559507191,-1.5319329500,-0.1452561766,-0.4501852691,0.3744405806,0.7547407150,-0.3966326714,-0.1949160248,-0.9319941401,0.5192106366,0.1668777913,0.7877882123,-0.5676600933,-0.5376181602,1.7631980181,-0.4717208445,-0.2970747948,-1.6887958050,0.8346431851,-0.2908143401,0.4495654404,1.1724085808,0.3258660734,-0.3408461511,0.9547800422,0.7426418662,0.7121489048,0.0964861736,0.3108291626,1.7955456972,0.6886371970,0.6476287842,-0.9715218544,1.6482530832,-1.3654054403,-1.4465010166,1.8486927748,-1.2850440741,0.1249407083,-1.0885308981,-0.2485388815,0.2764838040,0.5420431495,-0.4004425406,1.9209617376,-0.6851661801,1.1391372681,-0.1613515317,0.4473843873,0.4043840468,-0.2895264924,0.5565383434,0.7464129925,0.6821542978,-0.6891804934,1.7582477331,-0.0945984796,1.8338278532,-1.2337061167,1.6532959938,0.5494328737,-1.1798790693,1.4739180803,-1.6404334307,-0.5586791635,-1.6484837532,2.4162697792,-0.3091666996,0.0423371978,2.1097922325,1.0761109591,-1.7526555061,0.1723993570,-0.3163443506,0.3997488320,1.1593134403,0.2288526297,0.9422805905,0.4081701636,-0.6158750653,0.4793061912,-0.7150249481,1.4441509247,-0.4498966634,-2.6479845047,1.2745366096,1.1473503113,1.8102370501],[0.1164595559,-0.6753442883,-0.6330440640,-0.3043009639,0.6173820496,0.4520477951,-0.9501532912,-0.4605540037,0.0309729315,0.2326854914,0.4442227781,0.2579292059,-0.1524424553,-0.2695141137,0.6597102284,0.3571333289,-0.9604148865,-0.1528694034,0.4026290774,0.8940368891,-2.1126663685,0.6282519698,-0.9926656485,1.4733834267,0.6488913298,-0.9213027358,-0.5731467009,-0.6374110579,0.6401329041,1.3533540964,-0.3756834865,2.6135437489,0.1940688789,0.0630342886,-0.1559038311,-0.8732242584,1.2136697769,-0.4508092403,1.0960631371,0.3103138506,1.1365141869,-0.8697562218,1.2769855261,-1.9234557152,-1.1386402845,1.4014794827,-0.9552711248,-0.0660765618,1.6641439199,-0.8903948665,0.0384727232,-0.9314847589,0.7644599080,1.3544265032,-0.8408039808,1.2758435011,1.6310828924,0.8584744334,0.7314692736,0.2873908281,0.0925639942,-0.1756385863,0.7188848853,0.1778239012,0.0075021056,-0.3179973066,-0.8962370157,0.5907183886,1.3393832445,0.1291782856,0.2163106054,-0.1918319762,-0.4468945563,0.1792847812,-0.2239666730,1.4377971888,1.4812185764,0.4640325606,1.1958353519,-1.6090112925,0.7192595005,-1.8846025467,-0.4219196141,0.8524013162,-0.3068114221,0.7166014910,1.0810377598,0.0509062372,-0.8165169954,2.6991014481,-0.4963667691,1.1587369442,0.6737996340,-1.4490827322,0.6106519103,0.3198771179,0.0267605837,1.4674906731,0.4409443736,1.1011567116,-0.4959523380,1.0790488720,1.3844223022,0.1370456964,0.4587925076,1.1820701361,1.2755533457,1.1850805283,0.5455582142,0.2740237713,0.3107175529,-1.3007590771,1.2341132164,0.5541449189,-1.4907549620,-0.3145397305,-0.8253541589,1.0124372244,0.4967445433,-0.1998725533,2.3858799934,-0.2239804566,0.6783373952,0.6622564197,0.5010342598,-0.8637469411,-1.2166178226,-0.7888168693,-0.4862669706,-2.1496996880,0.2434635609,2.2176091671,0.6138218641,-0.1121539772,-0.0708785951,0.2982955277,-0.8135273457,-0.1176461875,-0.3864695132,-0.3397121727,1.9965733290,-0.4525742233,-0.7801883817,0.1617140174,2.0445573330,-0.6421329379,1.0683028698,1.4681972265,0.4830242693,-0.7713213563,0.9028714299,-1.1294329166,1.1345814466,-0.5910274386,0.6970822215,1.0826804638,-0.8912727237,2.1045033932,-0.1427501440,-0.0198568925,0.4012412727,0.5061278939,0.6106132865,0.8168576956,0.8838734031,1.8924901485,0.1710574180,-0.7019839287,0.3490320146,1.5296567678,-1.1244053841,-1.1642284393,-0.4193575084,0.5746083260,-1.1689153910,0.2877218127,-0.1881029308,-0.3931901157,0.4175453186,0.7864643931,-1.7730191946,1.7171610594,-0.5239853263,-1.6591190100,0.2596905231,2.7576310635,0.3869003057,-0.9029765129,-1.6944363117,-0.6992952824,-0.1057624966,-0.2710611224,-1.0353348255,-0.3324691951,-1.1543858051,-0.8658341169,0.1334683746,-0.1229031235,0.5339189768,0.6564360261,-0.4034191966,0.4229221046,-1.0462310314,0.4336838722,-0.2808940113,-0.6553801298,-0.0031763159,0.1415689141,-1.3357542753,0.0714949444,-0.4073644578,0.4600543976,0.3203541636,-0.1955058873,-1.4824770689,-0.4383046925,0.8585121036,0.0071368990,1.4751278162,0.6233286858,0.6207359433,-0.5246831179,-0.0414664410,1.7709965706],[-0.0709905550,-0.2821904719,0.1701576114,0.8789879084,0.8838764429,-0.3726766109,0.7503628135,1.8942296505,-0.5893586278,-0.9376010895,-0.4995022118,2.0414099693,-1.1939601898,2.3895537853,-0.5800492167,-1.4941869974,0.3914678097,0.7654280066,-0.3711814880,-0.1482408494,-0.8229711652,1.0125423670,1.9014531374,2.1342532635,0.8377076387,-0.5943709016,1.0816322565,-1.4450104237,-0.9806501865,-0.0139226625,-0.0261511095,-0.1476468891,-0.1782859713,0.0654901490,-0.9970036745,1.8520280123,0.2998270392,0.4222496748,0.3607931137,-1.9088587761,-1.3702299595,0.1112247109,-0.3712612391,-0.8905547857,1.5990093946,1.0519772768,-1.6142657995,0.9828546643,1.1408561468,1.0751584768,-2.1395509243,0.8739467859,0.8484569192,0.4239789844,0.7949110866,-2.2604801655,0.2834377289,0.1236173958,0.6627572775,-0.1966073513,-0.8987166882,0.1549092829,-1.7537448406,1.3848297596,-0.0972593948,0.2573923469,-0.3925730288,-1.3261764050,0.6730273962,-0.0823751241,-0.2015870810,1.3150098324,-0.4746868014,-0.9555039406,0.0567918830,-0.2349265516,-1.2320506573,1.6329518557,-0.3194123209,-1.8545962572,-1.5797518492,-0.7913144827,-0.5306677818,-0.7435374856,0.0374403261,-1.5283485651,1.6451562643,-1.7834663391,-0.1449975967,0.9374970794,-1.1378582716,-0.4348377883,-0.3451203704,1.6907564402,-0.0922324732,-0.2340613306,0.3029639721,0.6703021526,-1.2160876989,-0.1070491970,-0.8925184608,-0.9153434634,1.5998293161,-0.2169963121,-0.0921376273,0.7837740779,0.3923157454,-0.4793142974,-0.6740366817,1.2255915403,2.2740068436,-0.3493728042,0.7215506434,0.4427873790,-1.2372415066,0.3400833905,0.3696044385,-1.1532834768,-1.3326276541,-1.3978134394,-0.7902693152,-0.0793885812,-0.0871508643,-0.8185797930,0.3032013774,-0.9155107737,-0.4498315752,0.0133975549,1.2002675533,-0.0543997772,-0.5718390942,-1.1672619581,0.2706866860,-0.6121038198,1.5340633392,1.3675135374,-0.3314547539,-0.6407935023,0.1904756725,-0.0176693890,-0.1542854309,-1.0666533709,-0.8065037131,-0.5172939897,0.1125126705,-0.8618041277,-1.0027049780,0.1088037044,0.8530612588,2.4807577133,-0.4043349624,-0.6118181944,0.4480062425,0.1232022196,0.2338018715,0.9920058250,-0.8674075007,0.3546648920,-0.7531710863,-1.1711133718,0.5628138781,-0.4150498211,0.3105407059,-2.4759883881,1.6793575287,1.7537196875,0.1912416220,0.2514575124,-0.5988570452,0.0741735473,-0.2699765563,-1.2148730755,-1.7194465399,0.2601687908,0.6379894614,-0.1272419095,-0.5413345098,-1.0075840950,0.4337921739,0.4659135342,-0.3689009249,0.0981860682,-0.5749191642,-0.5014486313,0.3198375106,-0.7704556584,0.9145529866,-0.4085680246,1.4911134243,1.4047791958,1.1867433786,-1.6885709763,-0.3014551997,-0.1118771732,0.7182075381,1.0829957724,-0.8519905210,0.1300453693,-0.4334434271,0.4877637625,0.8054375648,-1.4181320667,-0.4921872914,0.4618836343,-0.6419098377,0.4662364423,-0.5979802012,0.3456205726,-0.3643333912,-0.2508301735,-0.2602106929,-0.9041038156,-0.7807335854,1.4818336964,-0.0963200405,0.9439044595,-0.0088408943,-1.5938279629,-1.6104105711,0.1701638550,0.1085248068,-1.1758224964,0.3988420963,-0.4401808977],[1.3763401508,-0.4395669699,0.4183611572,-0.8105944991,-0.6222549677,0.2567108572,-1.8000807762,-0.3589302897,-1.5436463356,-0.1948490441,-1.3861801624,-1.2604641914,-0.4003143311,2.4831156731,0.9226174355,-0.3849322796,0.2759652436,1.2357858419,0.5366231799,0.2259910405,-0.2262105793,0.9908719063,0.6953120828,-1.4158452749,-0.0114128450,1.1648558378,0.0026331618,-0.6417639852,1.7007479668,-1.0409736633,1.2814551592,-0.6953024864,-0.6436872482,0.5956673622,-0.8900166154,0.8791947365,0.4307781756,-0.5199248195,1.3481355906,-0.3190667331,0.5465198159,-0.1034876481,0.2821509540,-0.2298322320,-1.2568703890,-0.5725135803,1.8433989286,-0.1977019906,1.4076530933,-0.4182282388,-0.0286837257,0.4214601517,0.2952394187,-0.1422851980,-2.3230524063,0.5700261593,0.8659446836,0.4792743027,-0.0860511214,-2.4152247906,-1.1893260479,0.6089113355,-0.3519750535,-1.0636354685,-0.9764991999,1.4291633368,0.8169809580,0.7607301474,-0.8915486336,0.2099952251,0.7656538486,-0.9530802369,0.9028717279,-0.6837580204,-1.1406611204,-0.0980555266,-0.8678302169,0.0618282855,-0.9474887848,-0.4634743035,-0.8415288329,-0.2886526287,0.3071372211,0.2244463861,0.3965576887,1.1935093403,0.7316077948,-0.6506578326,0.2081241310,0.6839416623,-0.9692512155,2.0871205330,0.5729706883,3.4624316692,0.5383459926,0.3396901488,-0.5575518608,-0.5723373890,-1.7078907490,-1.6338896751,2.9699966908,2.5188298225,0.0883686095,2.0530881882,-0.6892683506,1.7217940092,0.8135957122,1.0825539827,-1.3073396683,0.2213107646,-0.8297741413,0.8229870200,-0.2925997078,0.2938055098,-0.2028401792,-1.0501542091,0.3662391901,0.0436582863,2.8220024109,1.6359175444,-0.2000506669,-0.6892738938,0.6414157748,0.1814173460,-1.2759850025,0.7353927493,0.9251579046,1.1306307316,-0.8013715744,-0.3105247319,-0.2614181936,1.4937690496,-1.1773779392,-0.9231160283,-1.9470348358,-1.2480545044,0.1362063885,0.6162065864,-0.4116608500,0.4412190020,-1.4277477264,0.7063032985,0.8237750530,0.4653929472,0.2365056872,0.0285083521,0.9960690141,-0.8011170626,-0.5131413341,-0.4017004371,-0.2410633862,-0.6117011905,0.5667151213,-0.4457055628,-0.7197325230,-2.7344152927,-0.2064120770,1.0151226521,-1.3595682383,1.3549883366,-0.1983778775,1.3015027046,-0.1049090922,0.7218281031,-0.1797719151,0.7467459440,-1.6058382988,0.8894509077,0.2281947583,1.1969484091,-0.6350207329,0.7831915617,0.5507146716,-1.3282835484,-1.1195918322,1.0832135677,0.4004996419,2.0190610886,-2.1272804737,0.5968118906,-0.6430553794,-0.9771640897,0.4279308915,-0.8297137618,-1.1755281687,1.0167371035,-1.0099674463,-0.3996248245,0.8061579466,0.5100678205,0.6430982351,-0.0116527425,-0.0569755547,-0.8373223543,-0.1155486926,0.8459715247,0.3501706421,-0.7146452665,-1.3728239536,-0.6906177402,-0.8787280321,0.3087691367,-2.0644254684,-0.8531473875,-0.7981085181,0.9267256856,-0.2735356688,1.6844605207,-0.3576886952,0.4597080350,0.5911514163,0.9546337128,0.7858938575,-1.0806646347,0.0153782917,0.8166173100,1.3586717844,0.6759258509,-1.1599477530,-0.9371449351,1.9049314260,0.4763910472,-0.0252852160,0.5366802216],[0.1893805712,0.0235877261,-1.2631338835,0.1262231022,-0.4659309089,-2.0677807331,-0.2172206342,0.2418677211,0.2863640487,-0.5541240573,-0.4875922203,0.9178158641,0.8355688453,-0.3356583416,-0.4909035861,1.3364617825,-0.5274421573,0.0552913696,1.2746754885,-0.4226909876,0.6258471608,0.8982531428,0.7511479259,-0.2033134550,-1.7190890312,1.2277430296,0.3163192570,0.2681086063,-0.3645800948,0.3647791445,0.5008106828,-1.0483267307,0.0229089204,-1.1120997667,0.4731096923,1.0203477144,0.0132737607,0.3177182376,0.1482971162,-0.9006447196,0.7418464422,0.8549311161,-0.4601553679,0.4142823219,-0.5291186571,-0.2832478881,-0.9400891066,2.0045392513,0.9907512665,1.2803305387,-0.8706706762,-0.0438755862,-0.1437354237,0.1352631003,-0.2147992402,-0.4306062162,0.7384632826,0.1026570722,-0.9193673730,-2.8592047691,0.5031278133,-1.6948381662,0.0623850375,0.4504397213,-0.2354303449,2.3214147091,-1.8391581774,-0.0035846911,-2.3443999290,-0.3101183176,1.7663384676,0.3807913065,1.7143090963,0.2600724101,-2.4273955822,1.1260516644,-0.1763039678,1.6863145828,1.0932096243,-0.2558988035,-1.1969647408,-0.1792815775,-1.8289457560,1.5591448545,0.8391063809,1.2969179153,1.6576949358,-0.4130849540,-0.7088075280,-1.2217174768,0.1126335263,0.0402188487,-1.5738127232,-2.1769027710,-0.2445770651,-0.3699555695,0.3325831890,-1.2605797052,-0.0567351133,-0.0740334541,-1.6811589003,1.3763895035,-0.4706969857,-0.9843922257,-1.1051831245,-0.4698985219,0.8479211926,-1.0044732094,0.2730768919,0.8285003901,-1.1652021408,0.7067195177,0.5655376911,0.4849031866,-1.0408544540,0.2165114880,-1.6860473156,2.1083023548,0.6807785630,-0.3152302206,0.8114103079,0.0277315732,0.8226593137,-0.7260422707,0.0740356222,1.3696192503,0.4227825999,1.0678523779,-0.6593800187,2.4530041218,-0.5130468011,1.3998401165,-1.1113494635,2.8427562714,-1.2515159845,1.7911391258,1.4213436842,0.1319545954,0.9271252751,-0.5539256930,-2.1196947098,0.5384703279,1.7564214468,0.2037496120,-0.4946107566,-1.3231574297,0.6391705871,-0.2147297114,-0.9286428690,0.3146469295,0.3201300204,1.6130293608,0.9120296240,0.2281010896,-0.9520408511,-1.7568795681,-0.0860790312,-0.3216405213,1.5971487761,0.2164636552,-0.8063322306,1.4201835394,-0.9114590883,2.0482690334,0.4544996917,-0.9965992570,1.7234151363,0.4029419422,-0.0605051927,1.1774067879,-0.1526352167,-1.3627843857,1.0119982958,0.5999497771,0.1030077115,0.0237347633,-0.9818653464,1.2113623619,-1.0080310106,0.6341429949,0.3184710443,-0.6536894441,-0.3419989645,0.8524641991,-0.0375635810,-0.4750849605,-1.1283820868,0.0493629947,0.1907393783,1.8657075167,1.6624958515,0.4755136073,0.3853852451,-1.7855322361,1.5675200224,-0.3439013660,1.4749809504,0.4404431283,-2.3438589573,-0.6509875059,1.7350389957,0.5399621725,-0.7919213772,-0.7674368024,1.3210825920,1.8883692026,0.7461881042,-0.1613261849,0.9186052084,0.9145761132,0.4075089395,1.4331464767,0.7037459612,0.5462380648,-0.7761021852,1.0948152542,0.9146957994,-0.7808047533,-0.9392088056,-1.2161570787,-0.4258204401,-0.0622376651,0.6151955724,-0.1300780773],[-1.2082298994,0.1191710830,-0.4793582261,0.3698245883,0.5303626657,-1.4388060570,0.3178140521,0.2438377589,0.5394156575,0.0692542270,0.7180391550,0.0523633435,0.5539424419,0.2073472142,-0.2396894544,0.8435296416,-1.1016947031,1.1617139578,-0.2411237210,-0.2124344260,0.0503696166,1.1276808977,-1.8850216866,0.9905751944,-0.1987105459,-0.0699739307,-1.5369192362,-0.4018937349,1.0958739519,-0.5575975776,1.7045712471,0.7433077097,-0.5954273343,0.6866076589,0.9778364897,0.2530172765,-0.1848416775,-0.4636569917,0.2392144501,-0.3474380970,0.8023971319,-1.1096651554,0.8329088688,-1.2316823006,-1.0217108727,-0.3689428568,-0.6006485224,-0.6255954504,-0.6185607910,0.2374453992,0.7998460531,-0.4682604969,0.4123714864,1.7706925869,1.1507540941,0.9583311677,-0.4108321667,-0.1481068283,0.0342794694,-1.1916863918,0.8081879616,-0.9666482210,1.0906995535,-0.1639183015,-1.1320599318,-1.5155814886,0.3599909842,-0.9653348923,-0.7701853514,0.7542246580,0.0639560819,0.2171070725,0.0094398977,0.4814985693,0.6337531805,1.2952977419,0.0742636099,0.6645027399,0.1214422286,0.6358388066,0.5883584023,-0.6672433019,-0.5090835094,-1.8338547945,-1.2418490648,-1.8587424755,-1.2738869190,-0.4197821021,-0.1913201660,0.8862715364,0.2564136982,0.2612314522,2.2699978352,0.9013975859,-0.6366124749,0.4804432690,-1.0621896982,-2.1655814648,-0.9202276468,-0.6696566343,-0.5409637690,0.7462758422,-0.3009483218,1.7442085743,0.2338610142,0.0837169215,0.3850113153,1.1419742107,0.5510025620,0.8352980614,0.4338721633,0.4724246562,0.2723473907,1.7705891132,-0.2735621929,-0.6433638334,0.6840257645,-0.2410134673,1.8150671721,-1.8267018795,2.3056502342,0.3518390059,-0.3171675801,1.3055222034,0.8078785539,1.3020491600,-0.9885943532,0.8765869141,-1.5162200928,0.2587037385,-0.6520469189,1.3673703671,-1.5630866289,0.9796712399,0.2590078712,0.4991450906,-1.1541076899,-1.1035073996,-0.3751842976,-0.9048180580,0.3524917662,-0.8524733782,-0.6322894692,-0.4022717774,0.6911383867,-1.1806863546,0.0684107095,0.5824618340,0.4333196282,-1.2544018030,-2.2081205845,-0.9804613590,0.1319895834,0.9541259408,0.3229891062,0.0549656115,1.7258254290,1.1441347599,0.7627381086,-0.3379273117,-0.0155780120,0.1145182028,0.0303633697,0.4696642160,-0.2231311798,-0.1135704070,-1.4808541536,-1.3070743084,-0.8091095090,0.2676424682,1.4319150448,-0.1194170862,0.3829805553,-0.4280880392,-1.2592542171,0.5967583060,0.0245815534,-0.5553333163,-0.0784272552,-0.9085083008,-0.2988561690,0.0019547611,1.6142442226,0.0618199073,0.1645226181,0.6071572304,-0.9973375797,1.0976777077,1.4667928219,-2.1746215820,0.3314406872,0.0927470177,-0.2048122436,1.4143673182,-1.1663620472,-0.2159975767,1.9280686378,0.7685463428,-0.1528172344,0.0260222536,-0.7631306052,-0.7862910032,-0.2226518989,0.5308263302,0.8197928071,-1.6154203415,0.3681326807,-0.7986581922,-1.6969778538,-0.8310033083,-1.1390073299,0.4657533169,-0.0444163941,-0.0632248893,-1.8191641569,1.0638192892,-1.3606250286,0.6245934367,-1.3430202007,-1.1037573814,0.2160263807,0.8564948440,0.1884693950,0.1738087684],[0.0949733108,-0.9783391953,2.1088376045,0.8860807419,-0.1600560844,-0.9818916917,-1.6848745346,-0.4343046248,0.6651635766,1.9885319471,-0.2870945036,0.9635787606,0.0447185487,-0.4152676761,-1.3084661961,0.5204074383,0.5992215872,1.9145693779,-0.8939692974,1.0495287180,0.2583296001,-1.0402622223,0.6333838105,-2.7544460297,-1.5477565527,-2.1013078690,0.0633282438,-1.1671156883,0.2825079858,0.6212230921,0.2638114393,0.5210747719,0.2484542876,-0.3608361185,1.8238667250,-0.1757961065,0.4721578658,-0.6213430166,0.2073867470,0.0125586027,-0.5172837973,1.1994928122,0.4062629342,-1.2676839828,-0.5387573242,1.6995825768,0.4816310704,0.8849418759,0.2497922778,-2.0610547066,-0.8003373742,-0.2149155587,-0.2607664168,0.0678154752,-0.1214217842,-1.7614785433,0.4538424313,1.5986852646,-0.5152428746,-0.9982887506,0.2431342751,1.0021549463,0.7107254863,1.2005827427,-1.3013552427,-1.7587054968,-0.8021962047,0.9416404366,-1.0701202154,1.7020624876,0.8600663543,-0.0366093032,1.1781638861,1.1968842745,-0.5840277076,-0.0184764396,-1.8357027769,0.4158545732,0.3273654580,-0.3873716891,1.6004836559,-0.2800341547,0.7892519236,-0.1054666266,0.5070289969,0.3303568065,-0.4675278366,-1.6173273325,1.1189441681,0.3584862351,-1.4084955454,1.0983003378,-0.1029772460,-0.9113822579,-0.6339533329,0.0714032203,0.6224611402,-0.5115113854,1.3892023563,0.5634469986,-0.6348069906,0.2799148262,0.2578309178,-0.7327276468,-0.6066296697,0.5482274890,-0.9846584797,-1.4991323948,-2.6056971550,0.5932595730,-1.4395385981,-0.7592237592,-0.9646526575,0.6389320493,-1.6647413969,0.7618963122,0.5511775613,0.5638883710,0.0919290781,1.2376972437,-0.4143681824,1.6608420610,1.1134301424,-0.3549555540,1.1900693178,-0.2941959202,2.0036094189,0.1889229864,0.2768132389,0.3631108105,-0.4527046084,-0.0876503065,0.4518642426,1.2216837406,-0.5853601098,-1.2460826635,1.4365702868,0.3918838799,0.4344147444,-0.3103542626,-1.4794760942,-1.1437619925,1.4064294100,0.8123748302,-0.8287097812,0.0644502342,0.5039708614,0.0464049242,0.6796227098,0.2334593832,-0.0366540141,-1.4885165691,-0.5560668707,0.0822257921,-0.2212876827,1.6971707344,0.1416028738,-0.3375744522,-0.2809337080,-1.0013574362,-1.1856667995,-1.1524541378,-0.0177265145,-0.1572738290,-0.4203666151,2.2288005352,1.6894711256,2.2608036995,0.0578856580,-1.6283227205,0.5768634677,-0.1903793067,-2.0627646446,-0.4917488992,0.8252843618,-0.9127709270,-0.4938484430,0.9456370473,-0.2809567451,-0.3177897632,1.0033947229,0.2217080295,0.8518112302,1.9643433094,1.8874276876,-1.5707550049,-0.8626135588,-0.2519744337,-0.1130561605,1.0470582247,-0.1848319024,-0.7244028449,1.4268809557,1.9367127419,-0.5947025418,1.2139691114,-1.1336416006,0.4106018245,-1.2320458889,-0.0817621946,-0.6174464822,0.8128951788,1.5221180916,-0.2531063259,-1.6988849640,-0.2455919683,-1.8654962778,-0.8597924113,-1.6687711477,0.0930599421,0.9343118668,-0.5308892727,0.9207680821,1.6391845942,-0.0630735978,-0.5165376663,-2.3341619968,0.9599221349,1.2102273703,0.7616304159,0.0274917819,-0.0581777953,-2.3147866726,2.0967600346],[1.4149196148,-1.0977212191,-1.6241617203,0.0720105991,-1.2375843525,0.3066580296,1.5010496378,0.2146603018,-0.1769921929,-0.2609245181,0.0112806298,1.3085396290,0.1515700370,-0.1343057156,0.5356020331,-0.3263483346,-0.8350747228,0.9119567275,-0.5826714635,0.1920842826,-0.7925053239,-0.3949542046,1.8459690809,0.3976787925,1.0924711227,-0.2221397012,-1.7229070663,0.4710488021,-0.1177469268,0.7858834863,-0.5207809806,-1.7956728935,0.2063074410,0.1493661255,-1.9783360958,-0.6310718060,-0.5640435219,-0.7965235710,-1.4358403683,-0.9247434735,0.4182759821,-0.7810117006,-0.1456603557,-1.8540166616,0.9421648383,-0.9929693937,-2.0774114132,-0.7155947685,2.6200604439,0.1311487406,0.1741613895,0.1489846259,1.5386333466,0.7290978432,0.3322045505,-0.8202380538,0.5158880353,-0.6750556827,1.3344002962,-0.4489007294,-0.1453179717,-1.0834563971,1.0381031036,0.2068814933,-1.0295704603,-0.3696275651,-0.6469484568,0.3838300407,-0.4384109974,-1.5211050510,0.6330561638,0.1468900144,-0.2644926608,-0.0833066702,-0.1666891873,0.1699847132,1.7822586298,1.4239218235,-0.4095877409,-1.2577289343,-0.2261657417,0.1536862552,-2.4841010571,-0.1690602601,1.1236718893,1.3120075464,0.4703026712,0.7917539477,-0.0024559440,-1.0582803488,-0.8924607635,1.9422979355,0.1470964402,1.5199949741,-1.4546091557,-0.2491374910,0.1628889889,-0.0223796498,0.0273962896,-0.8624451160,-0.3553950787,-1.1562964916,-0.5350723267,-1.1832722425,0.0762151852,0.8057109118,-2.6641361713,0.9860156775,-0.3996573389,1.4996862411,1.6512048244,0.9693275690,0.9770048261,0.5826814175,0.7136456370,0.0839540139,0.2687855661,0.7190995812,0.3238495588,-0.0822469965,0.5498287082,0.8601994514,0.4141937792,0.1921910942,-0.5373055935,-0.0399440303,-2.9793040752,1.1761871576,-0.5453553796,-0.2182899714,-0.3996349275,1.3795903921,0.6229868531,-0.1509124041,-0.4743267298,0.3084315658,0.6045871377,0.2694589198,0.8859691024,1.3523759842,-0.3754374087,-0.0136016058,-0.5773574710,-0.1826155484,1.0299216509,0.9875227809,0.3665340245,-1.8713101149,0.4778993130,1.3742740154,0.8267615438,-0.4324137568,2.3989701271,1.5465426445,-0.7949463129,0.8532446027,0.9856741428,1.3061232567,0.8775233626,-1.6787419319,-0.3526584208,1.4827687740,0.3022454977,-0.1343125403,0.8234794140,0.0794778913,1.1625801325,1.5019149780,-1.1354607344,-0.6904141903,2.9148166180,-0.5319117308,0.8360420465,-0.8598858714,-0.3846859634,-0.8635641336,0.7971241474,-1.5385718346,-0.9597117305,1.4342877865,0.0260773115,-0.6131992936,-0.0396655016,0.0818362907,1.3144561052,1.8665846586,-1.1760950089,1.0667315722,-0.2050069720,-0.5495911837,-0.5504364967,0.3499258161,1.7730232477,1.0658493042,0.0046089194,0.5828529596,2.8937945366,0.7601087093,0.4959368408,-0.3398263752,-1.1179248095,-0.0759203956,-0.7791621685,0.5058390498,0.3832107782,0.6314857602,-1.2179418802,-1.2486593723,0.2801901996,1.0210988522,0.3753940463,-1.4467526674,-0.6495608687,0.0395036824,0.2830541134,-1.3890956640,2.1286034584,1.4011929035,-0.0469790399,-0.0941228271,-0.0834299549,-1.4725753069,0.7305154800,-0.2272632569],[0.2950935662,-0.7175719738,-1.7210181952,-0.2352459580,1.8049403429,0.2901862264,0.2384487092,-0.4335539639,2.1697747707,0.5837168097,-2.0065824986,-1.5619883537,0.5572479367,0.6204081774,-0.6503604054,0.3235428333,-0.6237254143,0.0254766438,-0.9556758404,0.9256320000,0.1110662892,-1.3950949907,1.6628248692,-0.5550198555,0.2506259084,-1.2280722857,-0.8849002719,0.2853524089,0.7186892629,0.3485122323,0.4830940366,-0.9126710296,0.0648288727,0.5861564875,-1.1012845039,0.8903179765,-0.5021155477,0.2146847993,1.5970772505,-1.6596252918,1.7038996220,0.5897356868,-1.9231088161,0.1841837913,-0.8422321081,-0.2690632641,0.3078442216,-0.6378785968,-0.2733365893,-0.4018662572,1.1151080132,0.1203839406,0.5301672816,1.9052218199,0.5232292414,-0.5509648323,0.1532472223,-0.6852358580,-0.0168571491,0.2829575539,0.9693676829,-1.3901803493,-0.8579546213,1.6857061386,-1.8683937788,1.1424872875,-0.1706787050,-0.9109064341,-1.0370491743,-0.5145337582,-1.4502618313,-0.8660738468,1.1110023260,0.4196021557,0.2588478923,0.2073096335,-0.7804108262,-0.7706721425,0.2005842775,-1.2953560352,-2.0662872791,-0.1776489168,-1.5487222672,-0.4867807925,0.9630966783,0.0749432296,-0.4267623425,-0.8447555900,0.5929803252,-2.1236004829,-0.0424731523,2.4609448910,-0.9759138227,-1.7832497358,-1.6214101315,0.5515912771,-0.7159795761,0.4605422914,-1.0591150522,-1.1180940866,-0.1050870642,-0.4247130156,0.8251236677,-0.6458255053,0.0270585772,-0.7671965957,-0.6085761786,1.5552158356,-0.6036732793,-0.9812249541,-0.8282076120,-0.2138131559,-1.4566339254,0.5174087286,2.0507590771,-0.1476765126,1.0797405243,1.3560801744,1.0424228907,0.3342956603,1.2495017052,0.7392379045,1.1074708700,0.1809802502,1.7764987946,0.2816728055,-1.9343605042,-0.0202827193,0.3671101928,0.0815860778,0.0512436032,-0.5376491547,0.4952427447,1.7752697468,-1.6656461954,0.1051459163,-0.6446504593,0.8385035396,0.6940054297,-0.5207248926,-0.1721844971,1.7069374323,-0.5153077841,-0.5635568500,1.5965918303,0.1318876296,0.3199602365,0.8840111494,-0.2084402442,-2.4241857529,0.9587558508,0.4362331629,-0.9273656011,-1.2563002110,0.1305574328,1.1341838837,-0.4569153786,0.0596981235,-1.3988307714,-1.5971320868,-0.4953704476,0.7124653459,0.3637164533,1.4194965363,0.6654574871,0.6962931752,-0.0628936514,0.4574303627,-0.8212164044,-1.2626589537,0.7945659757,-0.1253741384,0.2529629171,-0.5118837357,-0.5959960222,-1.4390065670,-0.1195924282,-0.1827707291,0.3372287452,2.5050127506,0.5503866673,-1.8325817585,0.2706216276,-0.2371074110,-0.1965096891,-0.6216363907,0.4986001253,-0.0241942480,1.1589165926,-0.5790826678,1.8832604885,0.2265321314,-0.4650239050,0.8145852685,1.2787189484,1.2368496656,0.2810806036,0.5590633154,0.0165573135,-0.1671055108,0.2981031537,-1.7313406467,0.0702708587,0.9237394333,0.7955741882,-1.0611020327,0.1046951190,-0.6292627454,-1.1767753363,1.6472417116,0.9162951112,2.4193027020,-0.4519712627,0.1546483189,1.0768247843,-0.2482834458,-1.0319919586,0.9817290902,-1.2656478882,0.3102760911,-0.6407822371,0.2745261490,1.4232051373,0.0962205753],[-0.5079040527,-0.2026175261,0.4734267294,0.3398221433,0.1162773520,-0.3965824842,-0.0524844863,-1.3785188198,-0.5009530783,1.3419600725,0.4458302557,1.6235131025,0.5704522729,-0.5884652734,0.1655925512,-0.5934273005,0.8961495757,0.1395948231,1.8811582327,-0.4704824090,0.6056877375,-1.8371515274,-0.3883416951,-0.2261543870,0.1075822636,0.8845273256,-0.2323669195,1.0182594061,-1.2435951233,0.1850990653,0.6556530595,0.7260800004,-0.0777541026,0.6657271981,-1.5444598198,-0.4093579352,-0.1104109064,-1.2403335571,-0.7037749887,-0.0475286730,-0.0110759111,-0.3843451142,0.9995355010,0.2956174314,-0.1288992912,0.8820962906,-0.7885088921,-1.3096528053,0.1360455155,0.6605646610,0.4773285687,0.1000278816,-0.2009449899,0.9680691957,0.9661629796,0.3475872874,-1.0521339178,-0.2399311066,-0.6726133823,0.6403880715,-0.2364929765,-1.4091043472,-0.5644134879,0.7580716014,-0.1551982909,0.0443324931,-0.6542751789,0.1354361475,0.1713111103,0.3366525471,-1.1363209486,-0.8197965026,0.9096105099,-0.3897396922,-1.1618636847,0.2323047370,-0.1626732200,-0.3283954561,2.0063948631,-1.6640892029,-0.2251854539,-0.5884788632,-0.7459271550,-0.3304202259,0.5145342350,0.0916930512,1.8832346201,1.8666164875,1.2778718472,-0.6346173882,-0.6968966126,-1.2571614981,-0.0928414315,-0.8907172680,0.4955960512,2.0003273487,0.3299041986,0.5431902409,0.9838431478,-0.6571618915,1.1528811455,-1.0470914841,1.3171378374,0.2243078053,0.3119038641,-0.3337868452,2.0032920837,-1.0856931210,-1.9538714886,-0.1341106296,1.0241699219,1.2208647728,-1.6466290951,0.2385760546,-0.1349076927,-0.9472653270,0.3611437976,1.8478543758,-2.3713274002,-0.9970322847,-0.0145900678,0.6273401976,-0.2074701786,0.3642231822,0.8874557018,0.6856731772,1.0776827335,0.4891940057,-0.6078050733,-0.3043553829,1.4372410774,-0.9145619869,-0.1745103747,0.8110514283,0.5476842523,0.6497424841,0.5543809533,-1.0524549484,-0.6980483532,0.3797794580,-0.8720604777,-0.0382295512,0.1213534698,1.2860654593,0.6915104985,0.9888854623,-0.4740723670,0.3520619273,0.6744676232,-0.4462754428,-1.4237420559,0.2925428450,-0.4372058511,0.7580588460,-0.1709490865,0.2753054500,1.5881859064,-0.8896048069,-0.9051228166,-0.0149402972,-0.0148151759,-1.5531495810,0.0139444396,0.5084567070,-0.8614171743,0.5642536879,-0.5657557845,-0.1187821403,-0.4563186765,0.3088122904,2.5398862362,-0.0289198011,1.4139448404,-0.8513980508,-0.1655536145,1.7412998676,-0.0283007547,-0.6911320090,0.6281622052,0.6107546091,0.6955740452,0.9466020465,-0.4255324900,1.3051526546,-0.6912388802,0.7697520256,-1.6392782927,-0.4025473893,-0.1954599470,-0.0032584365,-1.1283636093,0.8464527726,0.1064395979,-0.3436134756,-0.9426211715,0.5683631301,0.4941053391,-1.6244103909,0.4565205276,0.7516614795,0.6100686193,0.2671549320,-0.4684554636,0.4916934669,2.2799537182,-1.0597896576,-2.0844984055,-0.0247393716,-0.1110131443,-0.0192019120,-0.8450926542,0.2091219872,1.3058104515,1.6750828028,0.6700471044,-0.2928491533,1.7783085108,1.0262509584,-3.0318641663,1.6394308805,0.4991608560,1.4980392456,-0.1323428154,0.6211748719],[2.0379743576,-0.5224591494,0.3273642361,0.1952998042,-0.4680666924,0.4209490120,0.5322787166,-0.7942463160,-0.5132656097,2.5295221806,-0.4084399343,-0.2231356502,0.1975568682,1.2763007879,-0.6899693608,1.0642337799,-0.5653979182,-1.2869952917,1.2466315031,-0.1241233274,1.0194280148,0.2496183366,-0.1709581017,1.7310829163,-0.1135782599,-0.1945802718,-0.4631730318,-0.1747877598,0.2316061854,-0.2133623809,-0.7444867492,1.7741721869,-0.5213237405,-0.0052383416,0.6835482717,-0.2477262169,-1.1038888693,0.3005446196,-0.9216147065,0.0160131194,-1.3955249786,1.8256142139,-0.2967949808,-0.6049112678,1.0554323196,-0.0910923630,-1.7532095909,0.6685379148,-0.2274804860,-0.7936704755,-0.6907336116,-1.1049113274,-0.4015493691,1.2235264778,0.0144734150,0.1218120903,-0.3425313532,-0.0198845416,0.4693568945,-0.1412683278,-0.1441329271,-1.2095980644,0.3068394065,1.4111222029,0.1158419624,-0.9842948318,0.3015630543,1.1471153498,0.7848933339,-2.0293517113,0.5302294493,0.2043271512,-1.1321178675,0.6940064430,0.8048830032,-0.0286177434,0.1219166294,0.6416437030,-0.4542204440,-0.4865248501,-0.0323937610,0.2326822728,0.7127761841,-0.7575730085,0.7263642550,0.0730553716,-0.2330107689,0.6749507189,-0.0529052541,0.2778891921,0.6335431337,-0.9314393997,1.5075997114,0.0337050445,-0.5642749071,-0.7681024075,-1.2881439924,1.7528229952,1.3125530481,0.1685075015,-0.4345751405,0.6745186448,0.4473234117,3.4154362679,0.0939195305,1.4169496298,-1.7073221207,0.1352761835,0.6206020117,0.1212979928,-1.6183773279,0.2478966564,-2.0915911198,1.3072786331,0.0073723211,0.1456651837,-0.5119955540,0.6050021648,1.0283262730,-0.7437700629,0.6922110915,1.8326694965,0.3287743926,0.9887950420,-1.2720338106,-0.8566733599,-0.4071277976,0.6455544233,-1.4670102596,0.5637975335,0.0863739476,1.1576341391,-0.5866075754,1.1902070045,0.4292430878,-0.6513916254,-0.2522836328,-2.7996482849,-0.0108595164,-0.3560985923,-0.0908001214,0.9481230378,-0.9476252198,-1.5085779428,1.0062683821,-0.8160352707,-1.3991876841,1.4791408777,-0.1391293257,-0.2057078928,-0.8553714752,0.5186762214,0.5040826797,-0.8638767600,-0.9191851616,-1.3979185820,-0.6784217358,1.2306538820,-0.9332259297,0.7083811760,-0.4577125907,-0.0620121956,-0.0351362042,1.0652267933,-1.1760458946,0.0446300060,-1.5669469833,0.4134811461,1.5374922752,0.0544624180,1.0646126270,1.5449422598,-0.4001969397,0.2695350945,0.8955142498,-0.1994645745,-0.8738275170,1.7183253765,1.0541538000,-0.6865471601,0.8660798669,-0.9972230196,-0.0987603590,-1.1042673588,-0.1311311722,1.1802910566,-0.4166555107,0.1959019303,0.1618142575,1.2000273466,1.7318041325,-0.7448762655,-0.6215144396,0.8590826988,-0.9501357675,0.2647697330,-1.0188965797,0.1645448953,-0.5019816756,-0.1122976840,-0.7125408649,-1.6597757339,0.0543932766,0.6145295501,-0.9701308012,-1.8752157688,0.5507056713,-0.3657512069,-1.3984634876,-2.3039720058,0.5814130902,0.9320523739,-0.6677701473,0.4140245020,-0.2230456620,1.1848533154,-0.2327275574,-2.3117311001,1.0083969831,2.0880084038,0.7807578444,-0.2806711793,0.3659831285,0.3532699645],[-0.1186335459,-0.2383485287,-0.2226555645,-0.1092954203,-0.7526459098,0.0635053217,-0.4191754162,-0.3093976378,-0.3990128934,-0.5901996493,-0.6269081831,0.5943827629,0.9215744734,0.6849617362,-1.1751044989,-0.1513048857,-0.4341301620,0.8947159648,0.0362598374,-0.6948597431,0.9275332093,0.5545841455,-0.4738935828,-0.1504694670,2.4883825779,1.1122415066,0.1489479244,-2.0799446106,-0.3141900599,0.5396690369,-1.3634753227,-0.9709356427,-0.3622013927,-0.3671687841,-0.1727541834,-0.7378624082,0.5973001122,-1.1517913342,1.4300465584,-0.7156851292,0.8373429775,0.2585847974,0.1721901596,-0.0018438506,-0.8097422719,2.1313259602,0.7968180180,-0.2775542736,0.8631947637,-0.8420573473,2.7444453239,-0.6136727929,0.3800564408,-0.2119310200,-0.4017619491,-0.4494703114,0.4593496323,0.5182594061,1.0070230961,0.3051910102,-1.7294261456,-0.3718368113,0.8044009209,-0.5573958755,1.2480124235,1.8836368322,-0.1714086086,0.0244216826,0.2147856355,0.6537339091,-0.6575426459,-1.7292504311,0.3809339404,-0.5554701686,0.8253012896,1.0695204735,-0.8808111548,-0.4070812464,-0.5109980106,-1.3897756338,-0.0042871847,0.4558518827,0.4185920954,-0.2674489021,-0.3960486948,0.8053077459,-0.1883954406,2.6233503819,-0.1203388870,-1.6325961351,-0.5712729096,0.0376099385,-1.0410552025,-1.2952389717,1.7780630589,0.2537894547,1.2101197243,-0.5471873879,-2.1763610840,1.1458199024,0.3720822334,1.6003818512,-0.4095530510,-1.9625519514,-1.5254100561,1.0795557499,0.6412642002,-0.2453266233,-2.3478925228,0.6769649386,0.3459262252,1.0892885923,0.0059136492,-0.4414263368,0.0774223357,0.4073279500,0.9646092057,1.3409656286,-0.5068500638,-1.1570055485,0.4112512171,-0.8605152369,-1.1234155893,-0.9586995840,-0.6890590787,-1.5872890949,-0.4864371419,0.0497915000,2.1994481087,-1.7377170324,0.6416237950,-0.7923098803,0.1304780692,0.3397642672,1.4351961613,0.2319086790,-1.0006706715,-0.3280332685,-1.0791562796,0.9721171260,0.4853863418,-0.9484644532,0.3009895980,0.3293378353,-1.5128226280,0.9338934422,1.1475496292,0.0169203840,0.3695321083,0.0343017913,0.5545833707,-0.5545576215,-0.3831988275,0.8395510316,1.2106777430,-0.3786276579,-1.2317663431,-0.2997151911,0.9177377224,0.9696442485,0.3414999843,0.8652888536,2.6955928802,-0.3250256777,-0.9790326357,-1.0462377071,1.8175184727,-2.4941484928,-1.0403255224,-0.5982604027,0.2539756596,-0.7462789416,-0.9838350415,1.3585087061,0.1825647354,-2.0929007530,0.4379278719,-1.1159633398,0.8405852318,0.5710203648,0.6748986840,-0.3078625500,0.2797708809,0.1595890224,0.1293483973,0.2212487906,-1.0180786848,-0.6851283312,0.5324478745,1.7520087957,0.7664175630,0.6118838191,1.4230185747,-0.0700611845,-0.3144167364,2.0584418774,-1.1153713465,-2.3753750324,0.0852801129,1.0693234205,0.3637231290,0.5887388587,-0.4191713929,0.1261838675,-2.0334804058,-0.4840212762,0.2973466516,-0.5006974339,0.9281368852,-0.1265035123,0.4268484414,-0.1641852856,-1.4899976254,0.7738882303,-0.1064092964,0.9350059032,0.7224861979,0.6837131977,-0.2813137174,-0.2018849254,-1.4496183395,0.2822894156,0.6741409302,0.5833388567],[-0.3834654987,0.9216980338,-0.5402904153,0.0449905135,1.6210678816,0.4038809836,0.8260396719,0.9907310605,-0.1848691553,-1.2419217825,0.9926421046,0.6334283352,0.8985478878,-0.5619766712,-0.8210536242,0.1006678417,-0.0309645627,-0.0446182489,0.7577447295,-0.6273584366,2.8296339512,1.2464458942,-1.1661547422,-1.5976299047,-1.6750463247,-0.6959130764,0.4801782668,-0.3721280694,0.2400489748,0.3327723145,0.3250359595,0.5707746148,-1.3501437902,0.2438921332,0.8684121370,0.0373390540,-0.4162268639,-1.5516031981,0.4893818200,-0.9510130882,0.4297042787,0.3235373199,0.1469300836,0.7171718478,-0.2045047581,-0.2003860474,-0.9752749801,-0.8899661303,-1.1855645180,0.3116814792,-1.2083834410,0.3561777174,0.3181398809,-1.3351159096,0.2889765501,-0.3798525333,-1.2264633179,0.1558524966,-1.3657807112,0.0333562158,1.3740893602,0.2170120925,1.4955347776,0.0376551785,1.0529077053,0.9597856402,0.1183744967,-0.0969234258,0.6593038440,2.1102674007,0.8496427536,-0.9985044003,0.6058217883,-1.0085011721,-1.0398656130,-1.0336683989,-0.0988367200,-0.6505918503,0.9194741249,-0.7106096745,-0.8037624955,-0.6252217293,-0.7874242067,0.9015535116,0.0489723794,-0.3165658712,0.4750823081,-0.8362310529,0.5985621810,0.0831746012,-1.9769351482,0.4109248817,0.9616567492,-0.1925200224,-0.6872479916,-0.0291667711,0.4817531705,-1.2618397474,0.8041153550,-1.0452064276,0.9234446287,2.3404836655,-0.5770075917,0.1780190021,-0.6452456117,0.1780572832,-1.1206645966,-0.7152308822,-0.9273412824,1.5543338060,-0.9651429653,-1.2783868313,0.3413947523,-0.3924708366,-0.6756599545,-0.1602455676,-0.1887249947,-1.0889669657,-0.3058025539,-0.9298772216,0.9752748013,-0.1857045293,0.3902679384,2.4783458710,0.6216580868,0.8140361309,0.5323762894,-0.1220407784,-0.1911884248,0.9695015550,1.7107357979,-0.8332494497,1.0158033371,-1.3568849564,0.3202181160,-0.4249622524,1.0535852909,-0.0648548529,-1.1206247807,-1.3313488960,0.2760379314,-0.8660236001,-0.2355240434,1.1059592962,-1.0518497229,-0.8911786079,0.4755552411,-0.4749928117,0.8618794680,0.8533287644,0.0164538156,-0.1435441524,-0.2964879870,-1.2023267746,-0.0720503479,-0.9484385848,-0.2501189113,0.6416484714,0.7757827044,0.7773790359,-0.2722290456,-1.0522669554,0.5333345532,-1.8526387215,0.3252865672,-1.0955603123,1.5631155968,0.6736435890,0.1962209195,-0.0650429949,-1.5892469883,2.2871513367,0.5928719640,0.5669403672,0.2304450274,-0.4885673523,2.0260953903,-0.5616394281,1.2725027800,-1.2598658800,-0.7209414840,-0.8498359323,-1.6869456768,-1.4535824060,0.6325818300,0.8368428349,-0.0728568211,-0.8834574223,1.3237099648,1.5845621824,-1.4369770288,0.1884049624,-0.3132542372,-1.3461782932,-0.1914197803,-1.0135910511,0.4060253203,0.8252695799,0.5558263063,1.1091421843,-1.1206122637,2.4767508507,0.5643259287,0.9157356024,0.5002053976,0.7908154130,0.8274473548,-1.7692761421,0.7141329646,0.4376039803,1.3221139908,0.1192799956,0.1986379474,-0.6428962946,-0.8024330139,-0.6532709002,-0.6049788594,-1.2296186686,0.6375156045,0.9582261443,1.9821717739,0.9358143210,0.8946135044,0.0913714394],[-0.6747932434,1.0523104668,0.0653479770,-0.7234938145,0.1506555378,0.2563973963,0.6422882080,-2.1984939575,0.1423068494,1.3988722563,0.9099609852,0.4305366874,-0.0558515005,-1.2119708061,-0.4905354083,0.4168024957,-1.1127213240,0.2744130790,-0.9005704522,-0.2425050139,0.9652739763,-0.1550932974,-2.7414751053,2.7117772102,-1.4189043045,-1.0313910246,1.4430273771,0.3495107293,-0.1600436270,-0.4971172214,-0.6566462517,1.7511360645,2.6167964935,-1.6863998175,-0.8562343717,-0.9014871120,-0.5291117430,0.0483883694,-1.5449298620,2.5941178799,1.1498541832,-0.8130968809,-0.9091265798,1.7938588858,-1.9932663441,0.1483443230,0.5673434734,0.3636609614,-1.4151256084,0.9914378524,0.3327944577,-0.7365987301,-1.1241701841,-0.3251023293,-0.1126096621,-1.0407176018,0.2644003928,-1.2129006386,-0.6472233534,2.0263800621,-0.2055637240,1.1558588743,-0.5569451451,1.0327348709,2.9583289623,-0.0690718144,-0.1696743220,-0.2801229656,-0.1217102259,1.6591004133,1.0041955709,0.7745190263,0.4571551681,0.2016106397,0.6540086269,0.6006768942,-0.6057759523,-2.4989545345,-0.4389500022,-1.1566992998,0.2245195806,-0.1928787529,1.0130279064,2.8678760529,3.4200766087,-1.1019076109,0.7968271971,0.3168827593,0.9232071638,1.5975202322,-0.4766547978,0.1227640808,1.5033007860,-0.1045565084,-0.5908822417,-1.4120235443,0.3343747258,-0.6939716339,1.3138922453,1.7582516670,-2.0948252678,-0.3867600262,1.4881784916,-0.2614794075,-1.0096803904,-0.8288538456,1.1242891550,0.8932532072,-0.7381454706,-1.5702298880,-0.6367726326,1.0390914679,-0.6599599719,0.1851871461,0.2441084385,0.1874911785,-0.2299124449,-0.0557615831,1.2884424925,-0.4906115234,1.0648677349,-0.7498172522,1.3501983881,1.2616695166,0.9143901467,0.1215201989,-0.1740020961,-0.8299691081,-1.3438180685,0.6896972060,0.2989670038,-0.9304523468,2.1257295609,-0.3191212714,-0.9800383449,0.9961398840,-0.3433336616,-0.9051725268,-0.5238730311,-0.1880698651,2.2426862717,0.3711197674,0.4215188324,1.9264063835,-0.8985811472,0.9724038243,0.9724006653,0.5963121653,-0.1313816905,-0.9208067656,0.0731222332,-1.0595364571,1.7986855507,-1.4451892376,-1.6628885269,0.1031904817,1.8886716366,2.1079497337,-0.0436575674,1.6163161993,0.7489252687,1.0819302797,-0.6431999803,2.6041235924,2.1054654121,0.1228456721,-1.3445225954,0.2740955949,0.8061972260,-0.7437826395,-1.1212561131,1.0593879223,-0.7465335727,0.6351316571,-0.1259510219,0.7468909025,0.9277933836,-0.6623082161,0.9495425820,-0.3312621713,0.3156095445,-0.0621979199,0.2871080637,0.5507833958,0.7471680045,-0.1918139458,-1.0566934347,-0.3854509890,-0.9094166160,0.3269947469,1.6029323339,-1.3439395428,-1.0700266361,0.8704138994,1.7585107088,0.8698053956,0.1614292413,-2.1836338043,0.4575574696,-0.3837276399,-0.7936035395,-0.7575538754,0.7196499109,0.3993514776,-1.1120111942,-2.4084584713,1.1744660139,0.6642578840,-0.1801867336,-1.1095536947,1.8082687855,-0.5784759521,0.7034670711,0.0047463123,1.9643903971,-0.6102766991,-2.0074377060,-0.0763465017,0.2575736046,1.8768074512,-1.2388337851,-0.7563742399,0.5345876813,-0.8474777937],[0.0104724141,-0.0560626648,-0.2254574150,0.7342847586,-0.1823126227,0.9410361052,0.1615654975,1.0111799240,-0.9020431638,-1.2960655689,0.5210303068,-0.0611365028,0.1001681834,0.7305082083,0.5266997218,-0.6311574578,-0.1671253741,-1.4037280083,-0.2443351150,-1.2517610788,-0.6421838999,1.4461832047,0.2268501073,0.0399844833,0.7946876884,-1.2721235752,1.2676312923,0.3134599328,-0.3629707396,0.5874877572,1.3795229197,-2.0228421688,-1.0742843151,0.7796605825,1.8933137655,-1.9481340647,-0.7232157588,0.6457300782,1.0992538929,1.2602574825,1.2173357010,0.3225525320,-0.8072761297,1.9048274755,0.2093349546,0.0960786119,-1.5297433138,-2.1362874508,1.6078487635,-0.0083657373,-0.4866380692,0.3900962174,-1.9635704756,0.3925728798,-0.9276992679,0.9480867982,-0.4484598935,-0.2534022331,1.1133691072,-0.4018307030,1.1105887890,0.0697086081,-0.0130785871,-1.2711316347,-0.4720092714,-0.6093285680,-0.3815086186,-0.2593854666,0.6868306398,0.2982982695,-1.0888745785,-0.0356637649,-0.7768434882,-0.8902729750,-0.7600454092,-0.4218118489,0.3945529759,-1.7036703825,-0.6038737297,-0.6489652395,0.7995288372,-0.2697657645,-0.2542925477,0.1636188477,1.6742730141,1.2430789471,-1.2469968796,-0.3124547601,-0.1815218925,1.0640125275,1.8174276352,0.1209504902,-0.2399097234,-2.0230894089,-1.1016820669,-1.4689011574,0.1792734712,-0.1051847711,-0.5941409469,0.1108767465,0.5730710626,-1.2479622364,-1.0237392187,1.3488510847,1.1845982075,0.3580740094,0.9250638485,-0.2230274081,0.1302635074,0.6828508377,0.7219886780,0.4433879852,-0.5854877234,0.7401592731,0.8713111281,0.2033105642,-0.5434444547,0.3103612363,-0.2429424971,0.3660286367,-0.9809601307,-1.3397420645,0.1668384820,-0.2828427851,0.4434334338,0.5371432900,1.9583348036,1.2049523592,0.2372088730,-0.3331832886,1.1592470407,0.2292878628,-0.7659143209,-0.3226299882,1.0478326082,-0.0462466627,-0.0130897462,-1.5684120655,-0.3709748387,0.9952441454,0.5843552351,-0.5555575490,-0.9959812164,-1.1858104467,0.3064933121,-0.3683856726,0.9699641466,0.9785125852,0.0002325818,-0.4245741367,0.3247098625,-0.1118231490,-1.9157896042,0.2442909628,-0.5570378900,0.1214372441,1.2457281351,2.3516080379,-0.0346703567,0.2227616310,1.0316162109,-0.3360175192,-0.1249095425,-1.2381279469,0.5380402207,0.2509429753,1.4545207024,0.3763480484,1.1317685843,0.1574275047,-0.4038485885,1.0350737572,1.5813468695,-1.2058871984,1.8397190571,-0.5369747281,-0.1675504595,-1.6808416843,-0.3973738253,0.8955420256,-0.9474033117,0.8863657713,-1.3639428616,-1.1407256126,0.0768821016,0.1626100093,1.1859166622,0.8617249727,0.0092263343,0.5938783288,-1.6519448757,0.4657230377,2.0321743488,-0.6253198981,0.4228032529,0.1740853935,-1.1310549974,-1.1406570673,-0.3442859054,1.0565874577,-0.2801092267,1.8965030909,-2.4183242321,-0.1742828339,-0.7835971117,0.2747937143,-0.6171855330,-1.6374839544,-0.0998052806,-1.5593715906,-1.2759281397,-1.4176751375,0.7536286712,0.8720196486,-1.2819939852,0.7087141275,1.9042751789,2.7620675564,1.1157127619,-1.0292673111,-1.2326740026,0.4892507195,2.1081411839,0.4370924532],[-0.5593727827,-1.4354031086,1.0570830107,0.4167571962,-1.1469852924,0.4386380911,0.4627600610,0.0152175874,-0.5675275326,-1.2706890106,0.3094842136,0.3353318870,-0.5180909038,1.4484337568,-1.8907704353,-0.0004836629,-0.3969363868,1.0633287430,0.0843856484,-0.1612736881,-0.0583505481,-0.0020569484,0.0918041766,-0.5388286114,0.4628135860,-1.0131388903,-0.4627156258,1.8314408064,-0.5217072964,-0.1163138002,-1.9294489622,-1.0067454576,0.5082025528,-0.3337702453,-0.2269964814,-2.1366028786,0.1338530332,-0.3698782027,-1.0692406893,0.3903188705,-0.0497018024,1.1616665125,0.9621710777,1.4915152788,0.1182670668,1.5708578825,0.7640832067,-2.0470445156,0.2740463912,-1.6569459438,-0.2085611820,-0.9933959246,0.7703897953,0.1919436753,-0.1551560611,0.5901817679,-0.7839456797,0.5055692792,-0.6600452065,-2.1255404949,2.5530233383,-1.4889702797,0.6624996066,-0.8144878149,1.2584605217,-0.6314144731,-0.9057261944,0.0311285201,-0.8325083852,0.3457152247,1.0724612474,-0.1779902279,0.6224679351,1.1843028069,0.3990680873,-0.3444859684,2.2503702641,0.5720024705,-0.6524844170,0.8113818169,0.6207444072,-0.2953625619,-0.2325160801,0.5621783733,-1.1866376400,-0.2479910105,1.3240972757,0.4694396853,-0.8371084332,-1.1386457682,-1.5861579180,-0.1940881610,0.5150237679,-0.7020881772,-0.1579501629,-0.3900207281,-0.9069029689,0.2974373698,1.0060987473,0.3582396507,1.3689446449,0.8325985670,-0.6259139180,-1.0219625235,0.8717699647,1.5553936958,1.9892309904,1.7129807472,-1.7524341345,-2.4156734943,0.3734627664,-0.9791836739,-1.1838378906,-1.6001518965,-0.2521800399,-0.9588496089,-0.0467710644,1.4703460932,-0.1281604916,1.9166052341,-0.7853434682,-0.8153104782,-1.2059153318,-0.0041961134,0.8652752042,-0.1252185553,-0.3175365925,0.0003913178,-0.8116727471,-0.1722526997,0.2703862786,1.1430435181,-0.0179268271,0.5940791368,-0.0597756766,-0.7731306553,1.7648396492,0.6532765627,-0.0256540347,-0.2082199454,-1.4125479460,-0.2929659188,-0.0797623321,0.4935064018,0.6900241375,-0.8711327910,0.2377964407,-1.3421390057,-1.0594630241,0.8106474876,-0.9261490703,0.0168073885,-0.1170370802,0.4168663323,1.3247480392,1.9524668455,-0.7113932371,0.8084810376,-1.1566987038,-0.2625491023,-0.1377955228,1.8415939808,0.2191815525,-0.3335691094,1.3335675001,0.6392911077,0.2304753959,0.1254295856,0.4711110294,0.5567567945,-0.1583541334,-0.1846426278,-0.3851806819,1.0271519423,-0.0131389173,1.4698441029,-0.5619645119,-0.0572414175,0.5521336198,-0.3570716977,0.4461381435,1.5787917376,-0.2964460850,-0.6109013557,0.5612183213,2.4743282795,-2.0427081585,0.7436702251,0.5880488157,0.1136561483,0.4198854864,-1.5047591925,0.5283017159,0.4352711737,-0.8595504761,-0.1644246876,-0.3773303032,-0.6257966161,-0.6975237131,0.4315404594,0.1575031728,0.5167097449,-1.4270300865,0.4468098283,-1.6486895084,0.4399907589,-0.2159845680,1.4771642685,-0.0137156723,-0.0051958985,-0.0996746719,1.8157832623,1.2126486301,-0.8141968250,0.0797947198,-0.3740171492,-0.0235440154,-0.4075264931,-1.5856205225,-0.8143793941,1.2097389698,-0.6563789248,0.0917219445,-0.2388767302],[-0.5819990039,-1.9098675251,2.0859701633,0.4343566298,-0.4059936106,0.4935959578,1.1469419003,-0.8147927523,-0.0642666221,1.0216310024,0.2484840751,-0.2188244909,-0.4161131978,0.9300234914,0.8221677542,1.3278201818,0.7848719358,0.7350556254,0.3275857270,-0.9404194951,0.8318222165,1.3112833500,-1.7765289545,-1.7815247774,-0.3423713744,0.4871589839,-1.4034453630,-0.3804571629,0.4861087799,-1.2812838554,-0.0835635960,1.6544206142,-1.2440884113,0.8436793685,0.7396528721,1.1865768433,0.0844733939,0.1610046029,-1.5059672594,-2.0161046982,0.4744549394,-0.1027829498,-0.8932421803,2.6681511402,0.8733780980,-0.6771495938,-2.1171841621,0.4179076850,-1.5294778347,0.6122385859,-0.5188607574,0.0253527220,-0.5863035917,-0.6137494445,-1.3239506483,-0.1866202801,1.4357578754,-2.6238973141,0.9697104692,1.3150304556,-0.1740820110,-1.2808016539,-0.7015659213,-0.3883333206,-0.0155479601,-0.5180083513,-2.6574621201,1.2562646866,1.7780845165,-0.8762903214,0.4460975826,-0.5522505045,-0.1025809869,1.6547645330,0.4384832084,0.8500418663,-1.0074121952,0.8443838954,0.3932457268,-0.2968220711,-0.8347958326,1.3171423674,0.9378548265,-1.2623959780,0.7586356401,0.2216345519,-1.1635854244,-0.2694393396,0.7349928617,1.1965305805,-0.4751051366,0.2625339031,-0.5044584274,0.3648485243,0.6711106896,0.8469839096,-0.0794413686,1.4676808119,-0.5880222321,1.2756909132,2.1141371727,1.2257579565,-3.1443977356,-0.8633837104,-0.4782708287,0.7528486848,0.0728427917,-0.1478959173,-1.8716983795,-1.2341648340,1.9306218624,0.4773486257,-0.7664555907,1.0502686501,-0.6320495605,0.7806579471,0.5659831166,-0.7911544442,-1.2014272213,-0.4031826258,0.3939342797,2.0412855148,-0.5606901050,-0.3510407507,0.4227169454,-1.3822602034,-0.2495191246,-0.4121963382,-0.4797803164,-1.6132888794,-0.1541571766,0.5172279477,1.8714196682,-0.8915430307,-1.0249711275,-0.4603399038,-0.8396726847,-0.2851386666,0.4890405834,2.7800784111,0.2114791274,0.5173261762,-0.4926160276,-0.5128554702,0.3635650575,-1.2499142885,0.1577661932,-1.1878240108,-0.1372671872,0.2076225579,-0.9449370503,-0.6277039647,0.0855376199,0.9168802500,-1.4615525007,-0.8221495152,-0.0306883696,-0.3794527352,0.6988317966,-0.3404355347,2.0155558586,-0.9869990945,2.3109531403,0.8517220616,-2.0357089043,-1.3717461824,0.4539968669,-0.3938590884,-1.1531527042,-0.6132876873,1.6227045059,1.5988278389,-0.4305523634,-1.5203722715,-0.2126695216,-0.7949711084,1.0062519312,0.1088095903,-0.1143707111,-1.5254942179,-0.5971631408,0.0266929027,0.1798896790,0.6165832877,0.1774144769,-1.1522259712,0.3164618611,1.3300391436,1.5971671343,1.4609972239,-0.1620344818,-0.0415391847,1.7246509790,-0.3874904513,2.1607668400,0.3433647454,0.3206478953,-0.6261118650,0.5304078460,0.0695448145,-0.0012609762,-0.5348551273,-0.4886779487,-0.1492735595,-1.1154803038,0.2743631601,-0.8677986264,-0.8241101503,1.0104525089,0.6254281998,-0.9113668203,-1.6015820503,1.7713805437,-1.2572556734,0.1901635826,2.1025061607,-0.0653650835,1.8718448877,-1.3963491917,2.3231360912,1.5577259064,-1.4847933054,0.1091064811,0.1189436316],[-1.8201619387,0.9907189012,0.7037143707,-0.1040920019,-0.3882758319,0.6091170907,-0.3306565285,0.2832741141,-0.2206534594,0.8937234282,-0.1409606785,0.9197987914,0.9479706883,0.4320091307,0.8930882215,-0.2672516704,-0.2465402484,0.1139625460,0.5704907179,0.3211796284,-0.7757544518,0.2262502760,-0.6387628913,-0.5648298860,-0.1335153878,-1.9806940556,0.8351049423,-0.6341540217,0.2234416455,0.7767619491,1.0302518606,1.2044047117,-0.6560946107,-1.1046437025,-2.5660791397,0.7642706633,1.7174856663,-0.2373072505,-1.4493148327,-0.0723819956,1.2432292700,1.9301974773,-0.0527655259,1.5347336531,0.7458348274,-0.3054525852,0.2830949426,0.7241360545,0.4799636900,-0.4515165687,-0.9522143006,-0.2386518270,-1.5671073198,1.1002331972,0.9565708041,-0.0753502250,0.3522382975,1.5541101694,1.0659878254,0.2211657315,0.7039881349,1.3312822580,1.1425888538,-0.8043620586,0.1118339002,0.7051146626,0.2405101657,-1.2077591419,-0.7707319856,-0.4111624956,0.8228642344,1.3478077650,1.3317564726,1.2580941916,-2.1362700462,-0.6044573784,1.3983079195,-0.2267834991,1.1038742065,1.8309376240,2.5118257999,-0.7461085320,0.4706372619,0.3283693790,0.5749509931,0.0096055260,1.5103399754,0.4314630032,0.8736536503,1.0169622898,-1.6128311157,-1.1360261440,0.5721042752,-0.6031692624,0.1187613904,1.1165670156,0.1368981600,-0.9850955009,0.6313478351,0.7843387127,1.2135647535,0.3128623366,0.5757420659,-0.1311307251,-1.4725977182,0.2902924418,-1.6542005539,1.3504993916,-0.3761866093,-0.0340541266,1.3819690943,-1.8941106796,-0.2743933499,-0.2576075792,-0.4365912676,0.4990610778,0.9072874784,1.0279914141,-0.2289631665,0.5705673099,-1.8134893179,-1.4707256556,-0.1991883665,0.1905540079,-0.1706116498,-0.4919151664,0.8360471129,-0.4966766536,1.3094147444,-1.3342524767,-0.8660213947,0.4778424203,-0.3906137049,-0.2144182622,0.3572425842,0.2118220329,1.1288861036,-0.0217239484,1.0320818424,-0.3704183996,0.1348206401,0.0780876055,-1.1623704433,0.0471805632,1.8262777328,-0.1326367855,1.2541949749,0.4579695463,0.6132331491,-0.6418583393,0.8945944309,1.3853417635,0.6127995253,1.7553474903,-0.8952273130,0.5639309883,1.1167125702,1.1573514938,0.6090118885,0.4605760276,1.3694016933,0.3081550598,0.4055664539,-0.4725796282,1.7730118036,-0.0457734875,0.0639425218,-0.6374250054,0.5051111579,0.5209814906,-1.3633556366,0.6776292324,0.5936003327,-0.9643524885,1.3238570690,-0.9794591069,-1.8232733011,-0.9560294151,-0.9466639757,0.4216081202,0.0111554582,-1.2830356359,0.0037587516,0.2466487288,-0.7067456245,-0.1197778061,1.3785123825,-2.2226142883,-0.1396190971,1.3109309673,0.0336780511,-0.5899109840,1.8874821663,0.4395863414,-1.2137041092,0.9638198614,0.5589596629,0.1619139463,-2.0408895016,0.0560152270,0.7049338222,-1.8372228146,0.0895555690,-0.8652178645,-0.5274806023,-1.4498169422,1.8824826479,-0.7552757263,0.8409298658,-1.4074972868,0.2751502097,-0.0361059606,1.3069361448,-0.4460712075,0.7948856354,0.2241593003,-0.0092800902,0.1511366814,-0.5700770020,0.9738789201,1.4050635099,0.7358398438,0.3554930687,-0.5356953740],[0.6154323220,0.9979453087,-1.3390628099,-0.8583290577,-1.7452572584,0.1104434729,0.3890730739,0.0756772608,-1.6073110104,-0.3842329979,-0.5743327737,-0.5342398882,-0.5943794250,-0.6329166293,0.1348872036,0.0209345836,-0.6921874881,1.2580894232,-0.1487711817,1.4409718513,-1.9829041958,-0.3945930004,-2.0652801991,1.9432833195,0.8998360634,-0.2134810537,1.7895114422,1.2492611408,-1.2575070858,0.2093546093,-1.7946289778,-2.2766561508,1.5815219879,-0.6217867732,-1.5840293169,1.0552384853,-1.4108628035,0.0078826984,0.0647610202,-1.2973845005,-1.4553879499,-0.6940916181,-0.0104205608,0.2180850804,-1.1110692024,-1.4055019617,0.6910184026,0.1690434515,-0.5238924026,-0.9038918614,0.1617193222,-1.5668437481,0.5746982098,0.7966463566,0.3421330154,-0.1778702289,0.0627256557,1.2577672005,0.7419780493,-0.4220174253,-0.5558896661,0.5636611581,-0.5715605617,1.2305231094,-0.6542611718,0.5123797059,-0.5756513476,-1.2421853542,0.9539251924,-0.7216297388,0.2888312638,0.0398992859,1.9420181513,-0.9858289957,0.1439062804,-0.1981014162,-1.3955296278,0.1860184371,0.5092500448,-2.2733435631,0.4448447824,0.7383634448,0.9620494843,0.9712896943,-0.5859873295,-0.4633949995,-1.1635951996,-1.0255144835,0.9860011339,0.3003800213,-0.6828405857,-1.2880755663,-0.0181694888,1.3963208199,0.6787196398,-1.3546409607,1.0398122072,-0.0002420847,0.4149239361,-1.3920470476,0.4387226701,1.7786114216,0.0279869679,-0.5754590034,-2.8383243084,0.3669659793,0.3989785910,-0.3432056606,-1.4067237377,-0.5664580464,-0.7988797426,1.9535807371,0.9568747282,-0.2729994059,2.0905108452,-0.9002789259,1.7589235306,-0.2031082958,0.2045028061,-0.3391548991,-0.0187342484,0.6179451942,-0.1164130643,-1.0554273129,1.1348347664,-0.5240642428,-0.0533059984,1.1834615469,1.4722721577,-0.3002305627,-1.1545814276,-0.0434754863,0.3521986306,1.1709691286,0.2513211071,-0.4330072105,0.3237198889,0.2494965345,0.2712747157,0.1401934177,0.0521037690,-1.1359419823,-0.8886265755,-0.4292696416,-0.6217500567,-0.2999503016,0.2239636630,-1.4546821117,-0.4629549384,0.2481509894,1.1152031422,1.2186678648,-0.8037263155,-1.8307890892,-0.9193554521,-1.9347736835,-1.7906540632,1.4603732824,0.6034803391,-0.7582035661,-1.4562839270,-0.4283702075,0.2467783093,0.8750617504,-0.5399234891,0.0219904687,0.1347843856,-0.6984317899,1.2807254791,0.6972069740,-1.1246480942,1.2445069551,0.0746976510,-0.3634368479,0.9292349815,-2.1193280220,1.9793330431,0.9654034376,-0.8357360363,2.3896911144,0.5750907063,-0.6683084369,0.4982488751,-0.2650178075,0.3913143575,-1.9028433561,1.2230871916,-1.6067589521,-2.1641135216,-0.6277225614,0.8032400608,0.6450325251,-1.3358204365,-1.5891611576,-0.3048976958,-1.1766319275,0.9003731608,-0.2710793614,0.7207944989,0.2262839079,-0.2502329051,-0.2332939208,-0.3703612685,0.4698289037,-0.7305675745,-0.2357319444,1.5106195211,1.0950709581,-0.6133319139,0.5051881671,1.0278711319,-1.7371171713,-0.2370272130,-1.0186152458,-0.5381708741,0.6036685109,0.0080919098,1.3370512724,0.3532626629,-0.7991682291,-0.2605423033,0.9962968826,0.3493235409,0.4128884375],[-0.7413421273,1.0753172636,-0.1606328785,-2.4201703072,-0.7577661276,0.9301439524,0.3775624931,0.3262410164,0.8824862838,0.4710078835,-0.8032360673,-0.5725351572,-0.0582918525,0.7470893264,-0.4320694208,-0.4195062816,0.7359803915,0.9830491543,0.1097064689,0.5909559727,-1.5239113569,-0.5554866791,0.1916795075,-0.2884576321,0.4674206376,2.1317734718,0.8106547594,-1.1282305717,1.1774668694,0.5297613740,-1.5435117483,1.0490826368,-1.4259819984,-1.2291139364,-0.0065258709,-0.2216066718,0.3152341843,0.3517228663,1.0046916008,1.1837831736,1.6787571907,0.5730426908,1.1133731604,-0.7634158134,-0.4686076939,0.7512238622,2.1902413368,-1.1080244780,-0.0866195410,0.7111128569,-0.0059136329,1.6001064777,0.2407293469,0.7014157176,2.2532317638,-1.1692081690,1.0892179012,-0.1247122213,0.3142854273,-0.0842473879,-0.1432422101,1.1002390385,0.5898386240,-2.9799461365,-0.5483270288,-0.3715531826,-0.3583878279,-0.0210139882,-1.1149870157,2.4004268646,0.1063649058,0.7702438831,-0.4968264401,0.0172923505,0.2284315526,-1.0430088043,-0.9970272183,-1.6039516926,0.2978511751,0.3246447444,-1.1764000654,-1.0792928934,-0.7115822434,0.9049133658,-0.1356133074,0.7902173400,-1.8608924150,-0.0342043675,-0.2184225768,0.2755118310,-0.5773473382,1.9079804420,-0.9805600047,-1.7185441256,-0.2441821545,-1.5625159740,-0.0510449111,-0.6830695271,1.4691959620,0.3331987858,-0.1861740500,-1.2373799086,-0.2372961491,-0.1537933350,-0.1583372951,0.9213557839,0.1104177907,0.7021344304,-2.2697911263,-0.9931976199,1.3266222477,-2.0886709690,0.9214906096,-0.4929367900,0.7904263139,1.5942453146,-0.5685650706,1.1563068628,-1.7574599981,0.3507694304,0.9195406437,2.1849930286,0.4669177234,0.0332165174,-0.9302831292,-1.1052159071,0.5699044466,2.7418849468,-0.8644914031,-1.1204868555,1.0906944275,-1.8325343132,1.2423360348,1.0562793016,-1.8252551556,-0.1053487584,0.6263771057,1.5337488651,0.3819944263,-0.3788675964,0.4347829521,1.0719298124,-1.9757854939,0.4538616240,0.9466922283,2.0393373966,0.8154541850,-2.2556941509,0.2045707256,1.3789530993,0.4618313015,0.6645997763,-0.1469683647,0.9429551363,0.3144655824,0.5231790543,-0.3446117938,-0.9457993507,0.7486024499,1.3672702312,-0.7518332005,-1.3094704151,1.2875912189,-0.6445900202,-0.4857350588,2.7472012043,-1.5506581068,-1.7447063923,-1.6849077940,-1.1065803766,-0.0241784956,-0.0330930389,0.8838341832,0.7734587193,0.5042662621,-0.6122062802,0.7282840014,0.9780203700,0.7290042639,1.1951694489,-0.0285034478,-1.7318187952,2.8072602749,-0.8139628768,-0.6801139712,0.3368071020,0.6303592920,0.0890115127,-1.2492383718,0.6210021377,-1.3981107473,-0.0684296265,-0.8358044028,0.7615279555,1.1252840757,-0.2928672731,-0.1489151269,-0.2545097768,1.7402290106,1.0581798553,0.1409235895,1.3134722710,-1.2648737431,0.3441012800,-0.9336190820,-1.2345448732,-0.1902813017,0.8853290677,-0.5469426513,-1.2303242683,-0.2285664976,-0.2023745328,-0.4210486412,-0.1622483581,-0.6331458688,0.6673703790,-0.8797009587,0.1470505744,-1.6115312576,-1.7434673309,-1.1627935171,0.2992941439,-0.7151190042,-1.2173782587],[0.2558501363,-0.5031922460,-0.1038606390,-0.1561047882,-0.8796012998,1.0886113644,0.3119497001,-0.4022977650,-0.1012258232,1.7479351759,1.3647730350,-0.0543092415,0.2226750851,1.6659430265,-0.5892763734,-0.3167293370,0.7162985206,2.7947816849,-0.2856855094,0.3759886622,0.6820854545,-1.9602570534,2.3843588829,-0.8123612404,0.6058959961,0.7391664982,-0.4000688791,-0.9828444123,0.7060558200,-0.7857539058,1.1159673929,0.2286228538,-0.4661549330,1.4146889448,-1.5722010136,0.8475800157,-2.1850621700,0.4052006304,0.9268170595,-0.5286844373,0.3858979344,-0.9194214344,-0.8946732283,0.0659045056,-1.2474843264,0.4395863116,-1.1277976036,0.7343308330,2.2652714252,0.2736219764,0.8193798661,0.4121380746,0.9677261710,-1.4630614519,-1.0397695303,0.3285118639,1.4500451088,-0.1098707393,0.8462182283,-0.6662430167,0.5424329638,1.0663647652,0.3155307770,-0.6716760993,-0.8097543716,1.6464439631,0.5302342176,-0.3151896596,0.2689172924,-1.2359784842,-0.1069266498,-1.5323109627,1.3193717003,0.6514467597,1.5726830959,-0.5646753907,0.8270943761,0.0647617579,-0.5997740030,-1.9733934402,1.2835294008,-0.7041236758,0.2832516134,-1.2470406294,0.0342308544,0.5632337928,1.7738374472,1.0023999214,0.5243485570,0.4966398180,-1.9411910772,-0.5846640468,-0.1156490222,-1.8383711576,-0.6315868497,0.2895682454,-2.7668647766,0.1243848652,0.5403708220,2.5971651077,0.9537038207,-0.5311449766,-1.9729572535,0.1686208397,-2.2282452583,0.0519388616,-0.8657557368,2.1935203075,0.9435275793,0.3583879769,-0.7221446037,0.8322559595,-0.1130532399,-1.4182372093,1.7974859476,-0.6891320348,-0.2459898293,0.3112476468,0.2605895996,0.4272195399,0.5657240152,0.1126472205,0.5242422223,-0.5227910876,0.8922997117,-0.4843709171,0.3712804914,-1.6282891035,0.1348460317,0.3993672132,1.3468558788,0.7090018392,0.4272810817,0.3625778258,-1.5106115341,1.8112719059,-1.1968251467,-0.1324630231,-0.7746610045,-0.1489947736,-0.2863193154,-0.7271369696,0.4484030902,0.4039295614,-0.6630720496,0.2465493828,-0.1913393289,-0.5363966227,0.0706059188,-1.9758604765,1.2545558214,-1.2935758829,0.1870326698,0.5147288442,0.2047286779,-0.9113526344,-0.3674494028,-1.3195163012,-0.5636649728,0.4023266137,0.3050526977,-1.3710098267,0.7285389900,-0.2600562572,0.9642859101,0.7061238289,0.0804846361,0.3684956729,-1.3706005812,1.7705264091,-0.0568827242,-1.1363366842,-0.5154197812,0.4832886755,0.1776915193,-2.1355118752,-0.2538057566,1.1987390518,-0.9549040198,1.4837807417,-0.3976415396,0.2713989019,-1.3256140947,0.0872604623,-1.8873546124,0.4834865928,0.7455752492,0.0950498506,0.2772811353,1.2658462524,1.5551054478,1.7185579538,-0.2580177486,-0.5866049528,0.3638573885,0.4724708498,-1.0699458122,0.3302752972,-0.5717364550,-1.9355258942,-0.4169055820,-0.3732923865,-0.4488600194,0.1205200925,-0.6611413360,0.3152336776,0.5914426446,0.5891466737,-0.4318289161,0.5248982906,-3.0162057877,-0.7288278341,0.5338007808,-0.5691620111,0.0026917364,1.3950619698,0.9638216496,1.2086063623,0.5523138046,-0.1347372085,0.4186432958,-0.0125459330,-2.3909592628,1.2674869299],[0.8219062686,-0.9020726085,0.8937808871,0.5497428775,-1.8376889229,-0.3232720494,0.9673829675,-0.1577348262,-0.8051065803,-1.4380515814,0.5605085492,-1.6228570938,0.8229484558,0.6320642829,0.8324034214,-0.7958431840,-1.2507016659,0.3585999310,-0.2646588385,1.5732150078,0.3994229436,-0.6318780780,0.2118939310,0.4288383722,0.1017487049,0.3347709477,1.3281470537,0.5289729238,2.2454297543,-1.1805934906,-0.1550690830,-0.1861247271,-0.6157492399,-0.2086265832,1.8791502714,-0.5473974943,1.2587167025,0.0400549732,-1.2131153345,0.4959487915,-0.6701539159,0.0377192609,2.0914919376,-1.2017105818,-0.5418961048,0.0715248883,1.1804654598,0.0372791365,0.5071865916,-1.1190364361,0.1947530657,-2.2451906204,1.8115687370,-0.3882434666,0.4428933263,-1.3330305815,0.0781882405,-0.1043641046,1.0043901205,-1.0324101448,-0.4917244315,-0.5101444721,0.7722641230,-0.5372172594,0.3941535652,-1.0564483404,-0.1142743900,0.2367779464,-0.4811623394,0.8114396334,0.7052668333,1.3426921368,0.4971502721,-0.4514482915,0.1606788486,-1.0752604008,-1.5647202730,0.3447371721,0.2896904051,0.6959680319,-0.0190260354,-0.1225974485,0.6586026549,-0.3244411349,0.7685539126,-0.1682316661,-0.9780043960,1.9468239546,-0.1223541796,1.4135782719,-0.3331348896,1.5832772255,-1.2763670683,0.3524271250,-1.0963369608,-1.3600704670,1.3364082575,0.8700780272,-0.3765659034,-2.6367402077,-1.0929170847,-0.0520115234,-0.9117823839,0.5583176613,-0.8233562708,-0.1356477439,-0.5791643262,0.2063644528,0.0742090866,-0.9129800200,-0.5781509876,-1.3166632652,0.5495023131,0.6567738056,0.5245886445,-0.5641840100,0.1216162369,-2.2587895393,-1.1103811264,-0.2331566811,0.4222166240,1.4316504002,0.8986779451,0.2228398025,1.3977748156,1.0694563389,-0.8107503653,0.7228331566,0.2350391448,-0.1462830901,1.1465845108,3.0110960007,0.1884022951,-0.2710390687,-2.4918198586,-1.3116731644,0.6081784368,-1.8274118900,0.3237744570,-1.0996891260,1.6525988579,1.7144389153,-0.7473198771,0.1106017306,0.0607738420,-0.4209593534,0.5615531802,1.1473829746,1.3023934364,-1.0771951675,-0.0043456485,1.1957966089,-0.6975986362,0.6175321341,0.0798140764,0.3990066051,-0.1930454522,-0.2801606953,-0.6360430717,1.5740067959,-0.6150626540,-0.6657387018,0.6328179836,1.5874658823,-0.8234479427,-0.9082185626,-0.9049493670,-0.1048155203,1.4692788124,-1.4920828342,-0.2135273218,0.1324931532,0.3250708878,-0.5595640540,-1.6266405582,0.6432067156,1.0292233229,0.4580583870,0.1775811911,-0.0012351129,-0.4039856195,-0.8623089790,-0.4011097252,1.3185437918,2.4721014500,1.5110800266,1.1565639973,0.1544111222,-1.4040644169,1.1386544704,0.4006298184,0.1019146889,1.1850543022,0.5945206881,-0.3560276926,0.8948068619,-0.3113974035,-0.5629203916,0.8216427565,-0.8694606423,2.0849277973,0.4553703964,0.2011788487,1.6399581432,1.1543406248,-0.6563993692,-0.1601136774,0.3659283817,-1.5479089022,-1.6454467773,-0.5885052681,0.4701368809,0.8731890917,-1.0473310947,-1.6058140993,0.5154587626,0.0192641634,-1.1761554480,0.3215034902,-1.0969194174,-1.1431628466,0.9230197072,-0.8684829473,1.2271327972],[1.0552119017,-0.4195388556,0.7144517303,-1.5553835630,-0.5773956776,0.9839149117,1.9320902824,0.4686033130,-0.5418919921,-0.2432720214,-2.3585970402,0.6843538284,-0.5589225292,1.4080122709,0.5970636606,-0.6819794774,-0.7745859027,-0.5762728453,3.3780813217,0.9202184677,-0.5481423736,1.3764128685,-0.3434719145,0.3977651894,0.3125531971,-0.3766749501,-0.9650712609,-0.6414336562,0.2432779819,-0.0810078681,-1.4610075951,-0.5868371129,0.6019322276,-1.1608246565,0.4770873189,-1.1839425564,-0.9647042155,-1.2911399603,-0.7851390839,0.0474583209,0.5524968505,-0.4648094177,0.6342483759,0.2583498955,-1.6864966154,0.0093507320,0.6306871176,0.3638943732,-0.5551549792,1.9378409386,0.4792116284,-1.3169698715,0.3789154291,0.5505794883,0.3445309401,-0.2305428237,-0.5491927266,1.4039272070,-0.1311127692,1.2018909454,1.4131186008,0.6897209883,2.8037290573,-0.9207084179,0.3258124292,0.5043047667,0.0766622126,0.6969939470,-0.2453171611,-0.0269711744,0.1035700291,0.1819505543,-0.5836421847,-0.6208786964,-0.8684943914,-1.0072883368,2.5586364269,1.1484230757,1.4538383484,1.0172985792,-1.1306617260,-1.1712599993,1.4082340002,0.1663869172,0.8057622313,0.0210015643,1.0863547325,-0.4501363635,1.1340610981,0.3947748840,-0.9205060005,-0.1627232879,-1.3064237833,0.7684119940,-0.2114374936,-0.4287933707,-0.4915237129,-1.6247475147,-1.9873917103,-0.0473227836,0.3810624182,2.1965646744,-0.4863922298,-0.2136739194,0.2542577088,0.6342818141,-0.6232274175,1.3386460543,-2.8707752228,-0.8340262175,1.6175956726,-1.5389409065,1.0650939941,1.5738481283,-0.2559369802,1.8801785707,-0.6821094751,0.5025829077,2.0232660770,0.7300717235,-0.7082805634,-1.3114689589,0.5112272501,1.6427576542,1.4437679052,-0.4480295777,1.2616261244,-1.0043283701,0.0544764996,-1.6036715508,-0.1986960471,0.7933214307,0.6842283607,-0.9457077384,0.3122385442,-0.3216295838,-1.5933513641,-0.9579170942,0.3627172709,-0.8289911747,0.6612448692,-0.1123323664,-0.1781046540,1.4285470247,-0.7790344954,2.9845044613,1.1620213985,0.8558660150,-0.2367271185,0.3755499125,1.6477419138,0.1121707782,-0.9373643398,-1.3632234335,-1.5604901314,-1.6682239771,-0.2411317378,0.1953369826,-2.2244782448,0.4661431313,-0.6613999605,1.1189903021,0.4893060029,0.8133631945,0.1535202712,1.6659163237,-0.6708444357,1.8384982347,-2.3238289356,-0.0551512279,-0.9679136872,0.0428756699,-0.7018769383,-0.1918913126,0.7778669596,0.8716066480,0.6106519103,-1.2859187126,-0.4537215829,-1.1330381632,1.0173296928,0.6480510235,1.4370362759,0.1523553282,-1.6252946854,0.6880706549,-0.4669993222,-0.1189476922,-0.0044371127,2.1690118313,0.2642358840,0.2700255215,1.5732755661,-0.3509848118,-1.8715351820,-0.0526743382,0.3962967694,-1.5842945576,1.2396576405,-0.8333008885,0.1840726137,0.7715519667,-0.3682126701,0.3607615530,2.0035834312,0.1541242748,-1.3954974413,0.0325314477,-1.6350804567,-0.9458541870,1.2335181236,0.3805272579,1.0706945658,1.7222070694,-0.6014351249,-0.0275845937,2.3488023281,-0.9489246607,1.0424493551,0.5509977937,0.5912781954,-0.3194181025,-1.8344695568,-1.5199538469],[-0.2534407377,-0.6822112203,-1.5001311302,-0.5672664046,-0.0814578012,-0.7873686552,-1.0422301292,-0.3025351763,0.4241429865,-1.5904347897,0.1101631746,1.0006101131,-0.6859989166,0.9753202200,1.0051900148,0.5923214555,-0.4300889373,0.7324078083,1.2622574568,0.0162067059,1.2897980213,-1.6987493038,-0.7873823047,-1.1084727049,2.4867002964,-1.0069047213,-1.1833405495,-2.6878020763,-0.4799250960,1.2554645538,-0.5688960552,0.0881453305,-0.4376583397,-0.6516036987,-1.0665112734,0.1584681869,3.1608436108,0.5720539689,-1.5262326002,0.4792236686,0.2597243190,-0.0070801945,0.4739401639,0.1779567897,0.4280982018,0.3195990920,-0.8183332682,1.4376466274,-0.3180052340,1.3227945566,0.3346593678,1.6663300991,2.1549162865,0.5529408455,2.3215246201,1.6306368113,-0.2287237942,0.3626566231,0.6241252422,0.2874783874,-0.5562575459,-1.0645240545,0.3074939847,0.9751630425,0.4938282371,0.4887374043,0.0105593298,0.6010999084,0.1875414848,-1.4502644539,-0.5887443423,-1.1079670191,-0.7399981022,-0.5830377936,1.9610450268,1.9490827322,0.1687097102,0.6483928561,-0.3631643653,1.2609117031,0.5470111370,0.3083211482,0.9779278040,-0.2285307497,0.6608920097,0.4434262514,2.3462135792,1.2344197035,-2.0416243076,-0.5078871250,0.6721323133,1.3655580282,0.6412608624,-0.1009380966,0.8257616758,-0.8150367737,-1.0085000992,-0.2928051651,-0.6150723100,-0.4936757684,-0.2183852941,0.9689996243,1.4067459106,0.2134040892,-0.4075092673,-0.4920285940,-1.7855901718,-1.0941070318,-0.2059654444,0.6229280829,-0.7334624529,1.2113645077,-0.4594071805,-1.4258073568,0.5498878956,-1.5167644024,-1.0164055824,-0.9841329455,0.4166430235,-2.2984161377,-0.7978627086,-1.9428042173,-0.4716677964,-2.6861188412,0.2464052886,-0.7541234493,0.0767890364,2.4924387932,1.0294005871,0.4078120291,0.0231720693,-0.2605399787,1.0949954987,-0.4825623930,-0.8047713041,-0.9314247966,0.1557550281,-0.2464975119,-1.7501392365,-2.1452300549,2.0917215347,-0.5078848600,-1.1502003670,0.8368268609,-2.2316017151,-0.6839256287,0.8083926439,1.5101549625,1.7092049122,0.3911790252,2.3782274723,-1.8589886427,-0.3594858348,1.6014767885,-1.1541914940,-1.0710375309,-1.0111941099,-0.3062404394,-0.2918458283,0.4250952005,-1.4448601007,-0.6035043001,1.5781127214,-1.5689852238,-0.8176691532,1.5150188208,0.6178444624,1.6630421877,0.4960637987,0.2892667055,-0.4297368526,0.3063223064,-0.3640555441,-1.3150182962,-1.5254424810,-0.1548462957,0.1717245430,-2.3199658394,-0.6687420011,-0.8737539053,-2.1310255527,0.1657736599,1.0154526234,-0.8788712025,0.4433715641,-0.6407283545,-0.1064218804,0.9291561246,-0.4667377174,1.0356763601,-0.2875490189,1.7318445444,-0.1297231019,0.2333392501,0.1582544744,2.0523178577,-0.7429432273,-2.1222071648,-1.4895672798,0.3120341897,-1.4317234755,-0.6542963386,1.3853491545,-0.8467838168,-0.7864772677,-0.7140951753,0.7617847919,2.6022748947,0.4526357949,-0.1344853640,-0.7801381946,-0.3832502067,-0.0866161734,0.4906598032,-1.0615791082,0.8940999508,-0.3497473001,-1.2624950409,0.1075066999,-0.0659220442,-0.6305398345,0.2686221302,-0.1823195964,-1.4534528255],[1.9937129021,0.1483629793,-0.2734782398,-1.5328689814,1.9309226274,0.0419135392,-1.9509328604,0.3718533218,-1.2233750820,-1.3658379316,-0.2958777845,-0.4734799266,0.7543007135,0.2856521606,-0.4374417663,0.8792541623,1.3735964298,-1.4071791172,0.1599424481,0.2732090056,-0.1884759814,1.5184983015,-0.3780666292,0.1182340458,-0.3417127430,-0.6154448986,0.9049147964,-1.8434865475,-0.6616224647,-0.2895358503,-0.4325897992,-0.6810391545,-0.1942910403,1.0399843454,0.7656468153,0.8919506073,0.8561698794,0.5976582170,0.8780407310,0.2451337427,-1.2113918066,-1.2457233667,0.5290142298,0.8976787925,-1.4235173464,-1.2115622759,-0.9235364795,-0.5710926652,-0.6926459074,-0.8372098804,0.1629962623,0.7053339481,-0.0904416740,-3.0984549522,0.6686881185,-0.7333278656,-0.6111493707,0.7304154634,-0.8588292599,0.8808065653,-0.6583173275,0.7451996803,-0.2612974644,-0.7302530408,0.4351638556,0.9296533465,0.6713768244,0.3829293251,1.4064166546,-1.2013900280,0.1457719803,-0.9313827753,2.6163449287,0.5473232865,1.4893498421,1.4373891354,-0.7347108722,-0.8683974743,1.4311701059,0.2228661329,0.4464642406,1.1559196711,0.0410200059,-0.1286565661,0.2647444606,0.6956012249,1.3178719282,1.0041838884,2.1021845341,1.0517970324,1.4445554018,-1.1228381395,1.5592223406,1.2020138502,-0.0028192063,0.5974342823,-0.8048190475,-0.9230668545,0.3318743408,-0.8511945605,0.0609234236,2.2532401085,0.2028496712,1.1774448156,-0.5526928902,-1.9647001028,-1.8609727621,1.0291814804,1.0270837545,-0.1857776791,-0.8119634390,1.0355705023,0.2473882139,1.0378277302,0.5087451935,-0.9540367126,0.8271316886,-0.2665016651,-1.7770938873,-0.6815657616,2.0232789516,-1.2229034901,0.2241365314,-0.0978290513,1.2142428160,-0.3724579513,-0.3604663312,-0.4075214565,-0.1006050706,0.7029295564,0.9857544303,-1.3855587244,-0.7235353589,0.1705339402,0.2272443473,1.5756491423,-1.0199034214,-1.4712222815,-0.1937682778,-1.4571503401,-0.4014287293,1.7913830280,1.4787840843,-0.8528799415,0.8736353517,-0.1563538909,0.1690182835,0.4658602774,0.1371072829,0.2078346610,-0.4466711581,-0.9114081264,-1.0166107416,-2.6908288002,-1.5399973392,1.4427289963,2.1239728928,-1.7548342943,-0.4824391305,-0.5262094140,-0.5427834392,-0.6966464520,0.4204410613,0.4263037443,-0.4591600001,0.2637559772,-1.9718030691,1.6417837143,-1.1051946878,-0.3639093637,-0.9486865997,0.4432385862,0.3481712639,0.4060375094,-0.5423926711,-0.3527583480,-0.4264259338,-0.9308899641,1.5440005064,1.5066274405,-0.1226757243,-0.3284087479,0.1419685334,-1.1529750824,0.2755541205,0.7012867928,0.0936654210,0.8950644732,-2.6700437069,0.2992852926,0.6254771352,0.1787260175,0.9335781932,-0.5372442603,-0.9831199050,0.8023634553,0.9252654910,0.0591549464,0.6226165891,0.2587251365,0.8355274200,-0.7327592969,0.6793955564,-1.4561997652,0.7465295792,-0.7011393905,1.5439178944,1.1826778650,0.2308337688,0.3512136638,0.1431451440,1.1339086294,-0.2589430809,-0.2183725685,-0.1566379517,-1.6944813728,0.3345756829,1.6899503469,-0.0316444114,-1.1810456514,-0.3684019446,0.3207881749,0.7920659781,1.6422004700],[0.2716714144,-0.5603635907,-0.3599757254,0.1677870452,0.0078198100,-0.4939026535,-1.7546492815,1.0226032734,-0.0200631488,0.5507588387,0.4915273488,-1.8573807478,0.8740385771,0.0427967496,-1.6065392494,0.7721800208,-1.1217762232,0.6055000424,-0.9973308444,0.6799186468,0.1740625948,0.0296238065,0.5217496157,0.1923450679,-0.5879488587,0.5928028226,-0.5710805058,-0.7600550652,1.1986136436,0.8760007024,-1.8550120592,0.7245780230,0.0701237619,-1.0581716299,-0.1062726602,-1.1632649899,2.3888280392,0.0001293403,-2.5444760323,1.6522936821,-0.4627757967,1.1748318672,-0.1822272688,-0.7353000641,-0.9973748922,-1.2355535030,-0.4939834774,-0.9968354106,-1.7728964090,-0.0487909168,-1.1636132002,0.7202252746,1.9825910330,-1.6155759096,0.5454311371,-0.7507312894,0.6600294113,0.0698041469,-0.8771593571,0.4602817297,-0.5655164719,0.0821633413,-1.6148333549,-0.2856899202,-0.9010733962,-1.5962631702,0.0608109497,1.7148712873,-0.1972455084,0.7661412358,1.7797145844,-0.7024210095,1.1253442764,1.9075862169,1.8911184072,0.2929170728,-0.4096447527,-0.0711998194,-0.7124832869,0.2756503224,0.8862338066,1.5982501507,-0.8595206738,-0.8688374758,-0.6461665630,0.7948951125,0.2192122340,-0.0730743557,0.1788961291,0.0994319618,2.5646259785,2.1651239395,0.6786749363,0.8574656248,0.1200354621,2.7303805351,-0.4879829586,0.8700460196,-1.2592363358,0.5971306562,1.3760768175,1.4629197121,1.4793893099,-2.2686212063,-0.3273313642,0.4973333180,-2.2781610489,1.1125961542,-0.3504355848,2.1755020618,1.2387313843,-2.1263351440,-1.7304564714,0.6072626114,0.8712618351,0.1535709053,-0.8973566294,1.4609305859,0.0118550872,1.2186610699,-0.3652283251,-0.0985303745,-0.4372721910,-1.1246514320,-1.9076437950,0.0822057426,2.1654286385,-0.5866769552,-2.5962011814,0.7465396523,1.0702034235,0.3408767581,0.8843899369,0.9880162477,0.3733038008,0.7469571829,-0.3654667139,-1.1835292578,0.0639124513,0.9607874751,-0.6897494197,-0.1239338443,-2.1009445190,-0.1710033417,-0.0617053173,-0.0188370664,-0.2777236104,-1.4326847792,0.8153862357,-0.0844529122,-0.1488251239,-2.0507738590,0.6880062222,-0.5953730345,0.8839527965,-1.1588287354,-0.8564046025,1.0334230661,-0.8303106427,1.0943323374,-0.1897011697,-0.7996029854,-0.3756458759,-1.0631002188,0.6210219860,-0.8794413209,-1.2669420242,-1.6641253233,-0.9473111033,-0.1794932634,-0.7986416817,0.4215012491,-0.3709406555,0.1757474840,0.9827396274,0.0277207959,-0.1375899166,0.9105586410,-0.6598195434,0.6502033472,-0.9171069860,-0.0821832120,-1.2704133987,-1.4377149343,0.8652183414,-1.9529612064,1.0661842823,0.9509752989,0.7923091650,-1.2883765697,-0.1132074520,0.0863654539,2.7263689041,-0.2265016735,0.2427878529,-0.7156814337,2.1491582394,1.4603886604,-1.8682442904,-1.1941982508,-1.4207702875,0.2143935561,0.6873126030,-0.8676314354,0.8319645524,1.4122692347,0.3956587911,-0.3590189219,1.4693763256,0.0357302763,-1.7043617964,-0.5215309858,1.2631664276,-0.5264201760,-0.1750185192,1.4218360186,0.2109541893,-0.8578353524,0.0341349170,-0.8854100108,0.6071174741,0.2590273321,0.6369572878,0.2007258683],[-1.5683698654,0.0528304949,0.2185943723,-2.0792095661,0.3473087549,1.5566731691,-0.9429470301,0.3542098105,0.1842521578,0.0781935975,0.9226818085,-0.1423786730,-0.4596142173,0.4551231563,1.8416955471,1.6079021692,-0.8521170616,0.5756788850,-0.5148565769,-1.2641240358,0.1978517026,-1.4766743183,1.1515684128,0.1763038188,0.2640692890,-0.8323398232,0.0230683852,0.6586131454,1.5933395624,-1.4179666042,-0.7774670720,-0.4964284897,1.4293264151,0.2270773351,-0.0167941060,-2.0283677578,0.0885260925,0.3764227331,-0.7332847118,0.9534940124,-2.1248283386,1.4294171333,0.7428011894,0.3662443757,-0.0806344151,-0.0672872812,-0.1971713156,0.2399710268,-0.5236250758,1.1410593987,-0.0949220136,-0.1947147548,1.1396582127,-0.1080093160,0.3194012046,0.2399757802,1.7890506983,-0.1936757118,0.8872783184,-0.1072830781,1.3524184227,-0.4339081645,-2.6261727810,-0.5521342754,-0.3567785025,-1.3351668119,-0.5639197826,-1.2869068384,0.4983268678,-0.3498931229,0.0223497171,-0.6007211804,-0.0052249120,-0.0424862429,-0.4661367536,0.1067087874,-0.0690322369,-0.8722267747,-1.4753071070,1.1081073284,-0.3371511996,0.5770972967,0.4407486618,-1.3237779140,-0.4519773126,-1.7085222006,-0.8972691298,0.9348768592,-1.0095119476,0.8008344769,0.5961164832,-0.3867681026,0.0516395830,-0.5375924706,0.7138876319,-0.9433989525,1.0816320181,2.1809961796,-1.1811590195,0.3935625851,0.3030322790,-0.6758750677,-0.9937831759,-1.3757630587,-1.3634648323,-0.8317195773,1.4114764929,-0.3476401865,0.2501884401,1.3920596838,-1.5477883816,1.7170302868,-0.8295890689,0.6617835760,-0.5374920368,0.1471432447,0.3597033620,-0.1635111123,0.8404725790,0.1525148302,0.7160716057,-0.8264759183,-0.5981168151,1.0916924477,0.9427590370,2.3003776073,0.5416697264,0.4278137088,-0.3079363406,0.1193876937,-0.2164104432,-0.0375090204,2.0504729748,0.2739741802,1.0871514082,1.1895177364,0.0692935809,-0.2275115550,1.4736508131,0.6942522526,0.1104047671,-0.8769634366,0.7775915861,-0.3444183767,0.0806557834,0.7060059905,0.7481377125,-0.9507731795,-1.2001460791,-0.2362423986,-1.4188394547,1.1630134583,-1.5362975597,-0.5948074460,1.5993782282,-1.2341548204,-1.5226598978,-1.5064285994,-0.7967886329,-1.6548575163,-0.7267010212,-0.2619487047,0.8821213245,0.9663928747,1.4349234104,2.9031307697,0.6623058319,1.7527112961,1.4296424389,-0.1722200811,-0.0848347172,0.3168185651,1.6467040777,0.2583158612,0.6035906076,0.8671565652,-0.5262527466,0.3319886625,-0.5497463942,0.4200908840,1.6204705238,2.8567569256,0.6891232729,0.3979914188,0.5920624137,-0.7559434772,-1.0975944996,1.1700562239,-0.4009858072,-0.7570546269,-0.8614937663,-0.8159902692,-1.5159968138,-2.3159945011,0.0993413925,-2.6141450405,-1.0592906475,-0.3929703832,0.5702621341,0.2737638056,3.2217895985,-0.4519669116,1.0404402018,-2.1363830566,-0.0241962317,-0.0399352834,0.8125180006,0.9310963154,-0.2669395208,2.5663955212,-0.6801775694,-1.4735100269,-0.8673360348,-1.0219819546,1.7570893764,-0.4413214624,1.6531904936,0.0096711023,-1.3588843346,-1.0291464329,-0.2288335562,-0.2092395574,-1.4058778286,-1.1912089586],[0.3284628987,-1.0475353003,0.3501040041,-0.0520703569,-0.4176031649,-0.1001003236,0.3600634038,0.1698016822,-0.5551374555,-0.0693989471,1.8276351690,-0.4938232303,0.9474299550,-0.3534296453,0.8308686018,1.2829548120,-0.9371561408,0.2828264236,0.8204187751,0.6815150380,-0.6391969919,-0.9756089449,-0.5878043175,1.2785863876,-0.4788477421,-0.9972364902,0.5297797322,-0.4876992702,-0.9401187301,0.1255615801,0.2447793335,-0.2591195703,0.2090267986,0.7089790702,-1.8272619247,-0.1440818161,0.8749162555,-0.2225515842,0.4995204806,0.0576416738,1.8956968784,-1.2364346981,-0.7038460970,1.0908890963,-0.4718849063,1.5693057775,0.4428057373,-1.1780397892,-0.7212728858,-1.7846194506,-0.1182599962,-0.7501379848,0.2489561588,1.8320907354,-0.6475961804,0.3235092163,0.1731548309,0.0735771284,-0.5053666234,0.7075278163,0.6965507269,0.5890418887,1.1716029644,-0.0515752882,0.3394494951,0.7700596452,2.5715339184,0.6072596908,-1.3007305861,0.0350546315,1.1603693962,-0.4538965821,-0.9783593416,0.0265502688,-1.2858632803,0.8167668581,0.1688446403,1.0662164688,-0.1026733741,-0.2595639229,1.0778741837,-0.4531909525,0.7623999119,1.2975021601,-0.0175253283,0.5181298256,0.2790040970,0.2507131994,0.0025475521,-1.6279962063,-0.1111964807,-1.8634821177,0.6435146928,1.0264328718,-2.0913350582,-0.5856692791,-0.1057725400,0.9844291210,-0.5320448875,-0.3761405051,-1.4245996475,0.6602067947,0.6311514378,-1.3559193611,0.3137751818,0.0596246235,1.0330663919,0.2284503877,-2.2971618176,-0.2615343928,1.0323364735,0.8715969324,1.4434144497,0.5397428274,0.6451210976,0.1844959557,-0.5831304789,-2.3598949909,-1.2856405973,-0.8049598336,0.3824418187,0.3135951459,0.5857704282,-1.8370773792,0.0837134793,-0.7865746617,-0.0764674395,-0.9077297449,-0.0669379085,-1.1441036463,0.5880070925,0.8309264779,0.4098588824,0.6063267589,0.1993841529,0.5601068735,0.0649905801,-1.5127788782,-0.6029203534,-2.0260241032,-0.0003807083,-0.1481437981,-0.8094789982,0.0633503646,1.7339324951,1.2790162563,-0.9811885953,0.9675010443,0.9853165150,0.5567837358,-0.3229420185,-2.2369308472,0.8412714005,2.2719078064,0.6248325109,0.0663437992,-0.4816880822,-1.0125662088,0.3245003521,0.8556165695,0.9044156671,-0.4809673429,1.5824408531,-0.6986125112,0.1831489205,-0.3048543334,-1.6329776049,-1.0484548807,-0.6673472524,-0.8022212386,-1.0551776886,0.3699105084,-0.9840784669,0.1571353376,1.7181296349,-0.1393129677,-2.6459677219,1.3768317699,-1.6159496307,-0.0081165750,0.4292753935,-0.2024205625,-0.9072603583,-0.2654633522,0.9579659104,0.1673599780,-0.8797431588,-0.5267537236,1.4190117121,0.6785064340,-1.1449259520,1.6081007719,-0.0469748788,-0.1846182495,0.2468699813,-0.2540178895,-0.7599125504,-1.4848744869,0.3800463378,-0.3932013512,0.1033055782,-0.8647897243,1.3579629660,-0.6633557081,-0.2434872836,0.5606272817,0.6669448614,-0.8341302872,0.3966738284,-0.3269149959,1.6633266211,-0.1126521826,1.4816939831,1.4156904221,-1.1166630983,-0.0606553070,0.0546188205,0.5445508361,-1.1936712265,-1.6309795380,0.8516145349,0.5265679359,0.1179899126,-0.5334897637],[0.5507861972,0.7433545589,-0.7290908694,0.0266585760,0.2498948872,-0.4962490797,0.9441946745,-0.5713911653,-0.0213359054,-0.7809194326,-0.3588345647,1.6766144037,-0.6612223983,-1.1081395149,0.7486165762,-1.3002353907,-0.0150593566,0.5272737145,0.7198087573,-1.7407058477,-0.8895874619,-0.9602290392,-1.2390866280,-1.4357074499,1.9074325562,-0.7550415397,-0.6781569719,0.4313832223,-0.8729441166,-0.1306226403,-1.0444041491,-0.3175023794,-0.0994322449,-1.2217093706,-3.0405607224,-2.0784544945,-0.3157816529,-0.2450707406,0.1222063676,0.6923543811,0.4367699325,2.0453593731,-0.1379508525,-0.2453946471,1.9804742336,0.8290540576,1.7626791000,1.0024492741,0.0583503172,0.4870532155,-0.5370153189,-0.3089361191,1.2510309219,0.3513412178,-0.9523450732,1.0798521042,-1.5914729834,-1.3701014519,-1.4990744591,0.9432504773,-0.8200966716,-0.7967817187,-0.1315291226,0.1992424875,-1.4237283468,-1.0715271235,-0.3152555823,0.6245658398,-0.5692147017,0.6681869626,1.0679280758,-0.0746023059,-1.1953831911,0.1121772081,0.9854134321,0.1716614068,0.7160137892,0.8524336815,1.9281487465,-1.0551660061,-0.0363917984,0.8865175247,-1.9527336359,-1.4990662336,1.0684392452,-0.6240852475,-0.3335091770,0.0973984450,-0.1852093339,0.5368331671,-1.4069367647,2.1364684105,-0.7955299020,0.8053387403,-0.2109974325,-0.2769947350,0.9727430940,-0.8858503103,0.2244490236,1.5067592859,0.9171571136,0.0369015187,0.2488748580,-0.4096579254,0.1632209867,-0.1903329939,-0.0624251887,-1.4613902569,0.3192446530,1.0520952940,0.1580966562,0.8568762541,0.0363764167,0.0138445962,1.7657972574,0.0549255759,0.9261724949,-0.2794926465,0.2829901874,3.4744529724,0.6240840554,0.0514252074,0.3288902938,-0.9306465983,-1.0744475126,-0.7351902723,-2.2935986519,0.7545247674,-1.1019783020,-1.3874216080,0.7956593633,-0.2586257756,0.8067504764,0.6845870614,1.1044504642,-1.3624535799,-0.0332132876,2.5060169697,1.7505385876,0.6042174101,2.1284039021,-1.6996500492,1.5683763027,-0.3298414052,0.6837784052,-1.2028763294,-1.4863746166,-0.4961459935,1.9162024260,-0.7335478663,-1.2341134548,-1.7722808123,0.8240976930,-0.0894157216,-1.5428103209,0.0126349842,-1.2780302763,1.0979908705,0.0222587101,0.4325967729,0.9945772886,-0.5362792611,0.1228516474,0.2050954252,1.4675050974,-0.5703064203,2.1974587440,-0.3568199277,0.2641554475,-1.1423335075,1.1377191544,0.0303159729,-0.2573347986,1.0057265759,-1.4090272188,2.4242238998,-0.6121561527,-0.0515848696,-0.1749645323,0.3523877859,-0.1276333034,-0.1103901342,-0.2866850197,-0.0464149900,-0.3077701330,1.4960175753,-1.3631552458,-1.7209351063,0.6491242051,1.4050065279,0.2676798403,0.2793314755,2.0455369949,-0.2602713704,0.6774649620,0.2130657732,-2.2492074966,-1.3942852020,-0.8942496181,-0.4105454981,0.8247628212,-1.2053675652,-1.0555409193,-0.4163510501,0.9191569686,0.0591878742,0.8970375657,0.0131746996,0.3099085987,-1.1245883703,-1.4458359480,-1.0214380026,-0.4532108903,-1.0915752649,-0.2660917342,1.9693787098,0.0519596413,0.3412489295,1.4262259007,0.1354945898,1.2808616161,-2.3592946529,0.2309911996,0.6223339438],[0.2537646294,0.3454696834,-2.0824322701,-0.1931213439,-0.3224272430,1.1653181314,-0.2260784060,-0.4215541780,0.4131685793,-0.3051755428,-1.3343261480,0.1708700806,-1.3212068081,-0.4973501563,2.0076658726,0.4784404337,0.0108408937,0.6487086415,-2.2793097496,-1.5167176723,-1.1210008860,2.0984489918,-0.9964933395,-0.2537392676,-0.6362712383,-0.6566887498,-0.4824739695,0.9052526355,-0.9712961316,-0.1638876498,-0.0615659095,-1.1938039064,-0.2663016915,1.2792323828,0.7501752377,0.3072949052,-0.3662904799,-0.8046880364,-0.0044827433,0.4288261235,-1.3418343067,-0.8763881922,2.3424198627,0.4666526914,0.4213204086,-0.4127517939,-0.7668270469,-0.8369839191,1.0337042809,1.5320605040,0.9207615256,-0.4779024720,1.1084336042,-0.6088128090,2.3399221897,-0.8646865487,0.2438831180,-0.5149421692,-0.5303933620,-0.5218394399,-1.6868261099,-0.0578323007,0.0224060658,-2.2675683498,-0.0580441840,0.7456520796,-1.2059559822,-1.2292213440,-0.2225447744,-1.3295269012,0.0634491891,-1.0590066910,-0.3777595162,1.3155578375,1.4310772419,-0.5424057841,0.1443112940,-2.4531741142,-0.8215950131,-0.1773539037,0.8877364993,0.1353154629,-0.6411932111,-0.5869292021,-1.4347361326,0.0225070901,1.1605197191,0.6720536351,-1.4405599833,-0.5473454595,0.7145345211,-1.1814160347,1.8402242661,0.3987374604,1.1327170134,-0.0116247721,0.3855871856,2.2576406002,0.3361343741,-0.6971358061,0.0903513953,1.9358855486,-0.0837212652,0.7940992713,0.6294739246,-0.0230218563,-0.4086762071,-0.7781276107,-1.5784299374,1.9371912479,-2.1188192368,0.3531058133,1.1220483780,-0.5264663100,-0.1083615795,-0.0701917931,0.1257195175,-0.3426410854,1.4279152155,0.8725541234,0.2656742334,-1.4155806303,-0.5311722755,0.5770264864,-1.9181660414,0.2317260057,0.6019353867,0.9077202678,-0.4718322754,-0.5740461946,-0.1232953146,0.9156028628,0.4621352255,0.4633850753,-0.7176672220,-0.9824862480,-0.1898862273,-0.2449781001,-1.5985085964,0.9512001276,0.8413665891,0.0192009527,-1.0423125029,-0.9620074034,-1.1568783522,0.0290435944,1.1404398680,-2.0211308002,1.6410949230,1.1472212076,-1.1088819504,0.8519145250,-0.5757724047,0.0184833016,0.5957829952,0.5147176981,-0.7896971703,-0.9568346143,0.3608250320,-2.3002521992,0.9077754617,-0.6996142268,-1.3030370474,-0.4180361331,-0.2474314123,-0.0635405704,-0.4888866842,1.1069529057,0.4685696661,0.1609025151,-2.3913598061,1.0130462646,-0.2157368809,-1.0683450699,-1.5078005791,-0.5091286302,-0.1590702087,0.3359768391,0.5903284550,-0.0485109314,-1.3737658262,0.5323999524,0.1814595312,-0.9770289063,-0.0009856647,-0.5759761333,-0.8987975717,0.5400237441,0.9631063938,0.7029232979,0.1237453818,0.6921179295,-0.9819133878,-1.1649929285,1.9168785810,0.0260932688,0.5961437225,-1.0539348125,-0.1854109466,-0.4541092813,-0.8012022376,-0.8627537489,-0.3639629781,-1.5960626602,1.1273057461,2.2912368774,-1.5819373131,0.0768916383,-0.7541574240,0.9644643068,0.4963940382,-0.3556044698,-0.1880870759,0.2791007459,0.9201265574,-1.6708345413,0.0320946388,-1.5955154896,1.6061308384,0.0217349976,0.3895410895,1.7398505211,-0.1265877634,0.4094446003],[-0.6966269612,-0.6312574744,0.4016741514,0.8904861808,0.6256996393,-0.4016552866,0.4941623807,0.4165548086,-0.1249488592,0.7846920490,-0.4188575745,-0.1406167001,0.5847015977,-0.3898477852,-0.9432166815,0.0343675576,1.8963041306,1.1666419506,1.9057749510,1.6193721294,-0.5440173149,-0.2137131691,0.7101011276,0.6619938612,-0.5311143398,0.6764826179,-0.9523943663,1.3086831570,-0.3232148290,0.3837288022,1.1706738472,-0.1583350450,1.3312753439,0.2241350561,-1.6838308573,0.5691562891,-0.4549115300,0.3088451326,-0.1045141220,-0.0612401441,-0.7073890567,-1.0161758661,0.5968431234,0.8028610945,-0.2996942997,0.2950571775,0.4948324561,0.4771435261,0.2741064727,-0.2335633188,0.2663442791,-0.0397798195,0.5287874341,0.3386964500,0.0797383338,-0.0556877255,-0.8294295669,2.1340575218,-1.6558796167,0.3286934495,1.6617243290,1.0909644365,-1.7356817722,-0.3916744590,0.6007047892,0.0267706569,-0.6286586523,-0.8454390168,-1.2340822220,-0.5262479782,0.2183321267,-0.7545596361,-0.9005653262,-0.5328521729,-0.8601483703,-0.7806350589,-1.0522665977,1.8136574030,0.9027171135,-0.7261156440,-0.0490528308,0.5775359869,0.5649013519,-0.7593803406,0.3464302421,0.6238087416,0.6746863723,1.7811609507,0.0354143009,-0.7106010914,-1.6929748058,1.2977504730,-1.0984780788,1.1117619276,0.1678503603,-0.5709893703,-1.7195818424,-0.9556397796,-0.9034484625,-0.0908293873,-1.0700818300,1.3671873808,-1.8064297438,0.7554394603,-0.5026817918,1.7806242704,1.0917088985,0.8868745565,-0.0075888634,-0.4598862231,-0.7556013465,0.0549330562,0.2221430540,1.3729677200,-0.9540419579,-0.3789446354,-0.1505113691,0.7726684809,1.7614964247,1.0252877474,-0.3415622711,-1.3802000284,0.0690797642,-1.1985107660,-0.5829547644,-0.0029087467,0.6864536405,-0.7441490889,0.9273821115,1.1335115433,-0.0438062884,-1.3134324551,0.2154727727,0.3803556263,-0.3672275543,-1.3546211720,-1.5402970314,0.0953702331,-0.4527512193,0.2291198820,-0.7780883312,0.3277203739,-0.0930765718,-1.0328152180,-0.8990030289,0.9657142162,-1.7825680971,1.2948658466,0.8651069999,0.5233416557,0.6785563827,0.1730827838,0.6709386110,-0.2403714061,-0.3969297111,-0.0562244244,0.9086267948,-0.0806159154,-1.3319112062,0.4982792437,0.0500809923,0.6967548132,1.0448199511,-1.1299753189,-2.7560486794,0.1868395656,1.3268688917,0.5976457596,-1.8544654846,-0.6114086509,2.1996905804,-0.0069061555,0.9068157077,0.0332768932,0.5828394294,-0.3923009932,-1.1152610779,-1.0476948023,-0.0980769023,-1.9471046925,0.3602553010,0.6499018669,-0.3639253378,-0.6350816488,0.1268273890,0.4119163752,-1.2266123295,1.4781122208,0.3314739168,-1.3378620148,0.2472490221,-0.8766486049,0.0906429887,-0.6079481244,-0.6427965760,-0.8671147227,0.8152963519,-0.0899893939,-0.0517604873,0.2418456525,0.3122246563,-0.7654765248,1.3197525740,-0.1157104000,1.7435361147,-0.6417414546,0.6864396334,-0.8152928948,-1.3649516106,-1.5759655237,-0.1911571473,0.9851586223,-0.1836736351,-0.1282128096,1.4109607935,-0.3864745796,-1.0339030027,-1.3751579523,-0.5158736110,-1.0271582603,1.2922033072,-0.2954572439,0.4869286120,-0.2285408676],[-0.0162027627,-1.9017342329,1.6025589705,-0.4845978022,-1.2267167568,-1.0501669645,-0.6654804945,-0.9559248090,-0.6065716743,0.0243174918,0.7494941950,-1.8077092171,-0.1077916324,0.8796631098,0.0062837857,0.5151043534,-0.6206137538,0.1735114008,-0.9676117897,-1.3759051561,0.0525254011,-0.5183722973,-0.1062775850,-0.8050181866,1.2755728960,0.0143800322,1.2767688036,0.9166401029,-0.3082312644,1.8080588579,-0.6768823862,0.3165327907,0.8092871904,-0.4501982927,1.8188048601,-0.9475402236,-0.0415148437,-0.8145132661,0.2790819407,-1.1835186481,0.7343748808,-0.5032110214,1.4122532606,0.2738788724,0.8091881275,-0.0099356575,-0.6308447719,1.1316365004,-0.3096952140,0.2567524314,-0.3351007700,0.8001866937,1.6676236391,-0.8846474290,-0.2603313923,0.1370266527,0.0493109673,0.9733078480,-0.3513320982,-0.8878127337,-0.7262001634,0.2246757895,3.0239856243,0.2840051949,-0.7080864906,0.4759110510,-0.0502727516,-1.0398198366,0.4720818400,0.3196711242,1.2657147646,-1.1127299070,1.2408133745,-0.3051033914,-0.1297694147,0.6046590209,-0.4969549775,-0.7131097913,-0.9040707946,0.6418151259,-1.3217216730,0.0984766409,-0.9235069156,1.1407945156,0.2827794254,0.2023810148,-1.7084463835,-0.1919159442,-1.6301255226,-0.4870570004,-0.0667773858,0.4956967235,1.3667733669,1.2660340071,-0.0160720777,-1.4300978184,1.5479718447,-0.5026729703,1.6225996017,-0.4625676870,-0.9789949059,-0.6550973058,0.4892952740,-0.0918960720,0.4369480312,-2.7726590633,-0.3816050291,1.4429813623,0.7978885770,1.0307193995,-0.2216360867,-0.0587479360,-0.1162854955,-0.3884234428,0.0990402028,0.2731604576,0.1190148816,0.0518279746,-0.1054238603,0.4520575404,-2.0491440296,-0.3809584081,0.3005086780,0.3459797204,0.3234552145,-0.1599316448,-1.6980955601,1.7123942375,1.4055458307,-0.8293477893,0.1827742159,1.4559041262,0.0637789965,0.0475166366,0.6155928373,-0.9956529737,-1.4131263494,-0.1182635501,0.8190299273,0.5125033259,0.0770660043,-0.7871782780,0.4206069112,-0.0446137898,-1.1966786385,-0.2150852829,-0.2339724153,-0.7401009202,0.5566579700,0.6771113873,1.4938442707,1.3028098345,-1.3230277300,1.6066470146,-1.3189405203,-1.7869796753,-0.8453440666,0.1779462099,-1.0992565155,0.8223231435,-0.3773572743,-1.4823824167,1.8274016380,-0.9582762718,0.8690104485,-0.1457026154,0.4907839894,-0.4807015657,-0.1869495213,-0.9083141088,-1.5402960777,0.0317795537,0.0290756747,0.2935406268,-0.0085515557,0.8803539872,0.3078594208,1.2030415535,-0.8128335476,1.6358748674,1.1200956106,0.4681635797,-0.2939386666,0.9326472878,-0.6292216778,-0.7046082616,0.2003094703,-0.8784189820,0.2474409193,0.5440397263,0.1008563042,0.5313048363,-1.4959849119,0.2413581014,-1.0873416662,-0.7164362073,1.2811007500,-0.7253606319,-0.3197246790,-0.9055584073,1.0824757814,-1.0967745781,1.4039075375,-0.8226977587,-1.5650757551,-0.1779900938,0.3357891142,1.6376110315,0.2842205167,0.0611847453,1.0054154396,1.5567678213,-0.4624824226,1.7156288624,-1.6495491266,-0.9886603355,-0.5040734410,2.1461417675,0.4368059337,0.2772457898,-1.4140040874,0.0257757753,0.3154803514,-0.2752482891],[1.2735975981,0.9660745859,0.3129498661,-0.9183510542,-0.0528000295,-1.4535831213,-0.4218615294,0.2285152674,0.4834912121,-1.0394859314,1.5353862047,-0.6263337731,-0.3788434863,-1.7689143419,-0.9368940592,-0.4472614527,0.7502661347,1.1232037544,-0.2082219273,-0.3345964253,1.1510394812,-1.1813848019,0.5503615737,0.5231403112,-0.8894044757,-1.1481416225,0.3089981377,-0.0089807743,-0.3435496390,-1.4222484827,-0.6001983285,-0.2960148454,-0.6301397681,-0.2517532110,0.5226249695,1.6253954172,0.6440001726,0.6713175774,-0.9343277812,-0.4876067936,0.2829792798,1.1756912470,-0.6696252823,-0.0702424943,-1.6962444782,0.5822570324,-0.3008073270,-0.7424628735,0.4427735806,-1.6259888411,-0.2963338494,-0.9052641392,2.0158989429,-0.0409712046,0.0850246251,1.2611733675,-0.5877819061,1.5001118183,0.3374356329,-0.8833405972,-0.8129408956,1.0231391191,0.2039081007,0.5219868422,2.1710143089,-1.5364967585,-0.9825080633,-0.4576882720,0.1181099117,-1.3131301403,-1.3747745752,0.6685074568,-1.1598657370,0.1115264669,1.2732436657,0.9837774634,1.0079425573,1.2042628527,1.4757161140,1.9824581146,-0.0836161450,0.6380442381,0.6169739962,-0.6086562872,-0.5009913445,-0.5547283292,0.9836614132,-0.1071657091,1.2167388201,-0.7316679955,-0.1268628389,-0.5843551159,-0.3292862475,1.3337472677,1.3200297356,0.7653281689,0.2057382762,0.2780999839,1.9284055233,-0.4090837836,0.1546208113,-0.2178405672,1.8453789949,-0.0609560572,-0.8356684446,-2.0992302895,-1.6502887011,1.8192211390,0.5958074331,0.6264472008,-0.6886128783,0.0464336015,2.2116112709,-1.5227183104,0.2625297606,1.5642783642,-0.5167988539,-0.1535461694,1.0193160772,-0.4149324000,-1.3377883434,0.2785515785,-1.4212867022,0.5633701086,-0.5416283011,-0.5272933245,0.0087975692,-0.5657374859,-0.5035797358,0.5643779635,1.5762873888,-1.0485948324,-2.0079891682,-1.6455342770,0.7365643382,-0.0382613279,1.7036528587,1.3061082363,-0.6127634645,-0.4571321309,0.9667922258,-0.2397709042,0.7455832362,-1.3719718456,0.1176320612,-0.9625248313,-0.3331040144,-2.3206830025,-1.1050237417,0.0505683310,-1.6521037817,-1.1555784941,-1.8600842953,-0.1436365545,-1.3095002174,0.5672355294,2.2935447693,0.2653402388,-0.4710046947,0.0420597941,0.6235096455,-0.3959354758,0.6133328676,0.8685811758,0.3623816073,0.8595988750,-1.2475121021,-0.8317956924,1.3300708532,-0.6564030051,-0.3196218610,-0.9565773010,-0.9911345840,-0.7629100680,-0.9317533970,0.4946089685,-0.0070229843,0.5263068676,0.1096296310,-0.7030386925,1.3341176510,0.0588235408,-1.3557839394,1.3701558113,1.3979157209,-0.0241335556,-1.5421690941,1.3926768303,-0.0572788082,0.0924422741,0.1306796372,-0.4506987333,1.0594346523,0.2627052665,-0.7358296514,-1.7386627197,-0.8884651661,-1.5768609047,-1.2932181358,-0.3603904545,-0.5076446533,-1.8667639494,1.0743008852,0.7913588881,1.0181564093,0.6479498148,-0.0823470876,-0.0806593820,-0.4157681167,2.2641456127,0.4007044137,-0.3426760733,0.3771518767,0.5061627030,1.5559914112,-0.4237650633,-1.1088336706,0.7923083305,0.6938353777,-1.3250257969,1.5535944700,0.0049045268,1.1246401072,0.0097699733],[1.5038679838,0.5279566050,0.6128170490,0.3623874485,-0.0392698757,0.3612433374,0.6001952887,-0.7416902184,-0.3430719972,0.1613178700,-0.6219967604,0.2432402670,1.3867510557,1.4010163546,-0.6260728836,-0.2785092890,-0.8145068884,-2.1860978603,0.4015526474,-0.5687218904,0.0735837519,2.3719646931,-1.2226122618,-1.7341774702,0.5211261511,-0.7689599395,-0.4643093646,-0.1880650520,1.8296451569,0.1179625243,-1.2873387337,0.2452962250,0.7529872656,-0.4656670690,-0.7779188156,-0.9317938089,0.2511917353,-0.8010789752,0.3875940442,-0.2854766250,0.2135651559,0.9496786594,0.4434437156,-0.2921417952,-0.2893070579,1.8729990721,-1.3912776709,0.8823206425,-0.7693408728,0.8297691941,0.6176412106,-1.6273237467,1.3451734781,0.7532227635,-0.9126815200,1.1919921637,-0.1777725518,-1.3362666368,-1.0282952785,0.7495453358,0.7746673226,0.5302920341,-1.1779735088,0.4359086156,0.9370511770,0.6625077128,-0.2525390983,-0.1961964667,0.6796283722,0.1560912132,2.0079288483,-1.3467699289,-0.9276216626,-0.0741150156,-1.6958822012,-0.2605181336,-1.2530063391,0.2627429664,0.7165470719,-1.8141559362,-0.9000178576,0.7370605469,1.4289104939,-0.6930576563,1.7424666882,1.6449006796,0.5382955670,-0.7510215640,-0.2363715321,1.0369923115,0.9657538533,-0.7431786060,-0.8046639562,1.4332410097,0.1665688902,-0.6746644378,-0.3744138181,-0.2142071873,0.3782551587,-0.0284325425,-0.5066626072,2.2650198936,1.4231873751,-0.0743084699,0.4235864282,1.5384422541,1.3585460186,-0.0352351889,-0.2189443856,-0.8031013012,2.0303540230,0.5452772379,0.2255565971,0.0925863236,-0.4921898842,0.2845351994,0.2062941492,0.3464781344,-0.0111541692,-1.7251620293,-0.7196274996,1.0001733303,0.1533291489,-0.7373467088,-0.4225189388,0.1528119892,-1.0180543661,-0.2779996693,-1.5780826807,-0.4613038599,0.5342951417,-0.1558797210,0.0374767967,-0.3804990351,-0.4957786202,-0.2577657402,0.5751443505,-1.0979690552,0.0594886243,-1.0828264952,-0.1919242889,-0.3676927984,0.5205117464,0.1584236622,0.1191426143,-0.8184567690,2.6164002419,1.8043608665,-1.4470647573,-0.7504413128,-0.0128210383,1.7690523863,-1.0356879234,-0.8855569959,-0.8161084652,0.3014318943,0.0012616556,-0.5786834955,-0.5376681089,1.7274968624,0.7023185492,0.0531656817,0.0197559502,0.0192194246,-0.4238943756,1.0916885138,0.3940524459,0.6978895664,-0.5300963521,-0.1750378609,-0.7282319665,0.3309630752,-0.7242958546,0.8023917079,1.2333136797,0.0600263327,1.8488427401,-0.7148504257,0.4385460615,0.9688469768,0.0299088582,-0.9750109315,-1.0483665466,0.0277658887,0.2368310541,-1.2071895599,1.2570800781,2.0445125103,1.6918454170,-0.6865190268,0.2112120986,1.5974457264,-1.1287384033,-1.5620183945,-0.9195592403,-1.1469166279,-0.2893614471,0.6627973914,1.0217088461,-0.3918666542,2.0864372253,-1.0320661068,0.3124392033,1.3697936535,-0.3738217652,-0.1886551231,-1.0122405291,0.8189871907,-0.3247868717,0.3251466453,-0.2431050241,-0.1908031404,0.0159445219,-1.1716156006,0.4313590527,-0.4181154966,-0.5454003215,0.3107697368,-0.4312063754,1.6988054514,0.3427794576,0.5752515793,0.3136224449,0.0746937171],[-0.2343776822,1.3644443750,1.7880711555,-0.8330552578,-0.6701244116,0.0977286845,0.6908764243,-1.6960651875,2.1555716991,-1.0845538378,-1.0772690773,0.9238383770,0.1551842541,-1.7129218578,1.0820883512,-0.6496840119,-2.1078553200,-0.8854192495,-0.0315521695,0.1816882342,-1.6160395145,-2.0274572372,-2.2314357758,0.3817986846,1.3415185213,-0.7032774687,-0.6273907423,-0.5772133470,0.8161108494,-0.3537312150,0.8900743127,0.7412501574,-0.1986007243,-0.0569934621,-2.0071425438,0.8271296024,-0.2330507785,0.3007969260,-0.4647093117,0.4949223399,-0.4368942380,-0.0580027997,-0.2060611546,0.5465889573,-0.7176531553,0.2781758308,-0.0540548712,-0.8538475633,0.7752673030,-1.1576044559,-1.6881654263,0.2807030082,-0.8262535930,-0.0240261238,-0.9551573396,-0.2277644128,-0.2805421650,-0.2276842594,0.8241704702,1.1559234858,-0.7621541619,0.1560282558,-1.3470283747,2.0022385120,0.4317237735,1.2540224791,-0.1634758413,0.8219774365,1.1389209032,1.5196316242,-0.1628244519,-1.0992746353,-0.9544483423,-1.6510956287,-0.6569509506,1.0500578880,0.6824148297,1.5513504744,0.4548636675,2.7451868057,1.0425418615,0.1934823841,1.9569149017,-2.2074351311,0.9075617790,2.0576922894,0.8540446162,-0.9392482042,0.0225526020,-1.3599926233,0.0982640311,-0.4961391985,-2.2542142868,0.7189049721,0.5635581613,0.7037670016,0.3800044656,1.0090733767,0.7960253954,1.5524606705,-1.0578310490,0.0954674557,0.6344092488,0.2032889277,-0.2713959813,0.8102875352,-1.6120643616,-1.3914624453,1.3278867006,0.7482947111,0.8191539645,-0.2149544358,0.5363849998,-0.9918892384,0.8326933384,-0.4563343227,0.9957206845,-0.3911642730,-0.5761083364,-1.6257008314,0.3507175446,-0.6471960545,-0.6633731127,-1.1012862921,-0.1357158273,-0.4140460193,1.1256126165,-0.2547148466,1.1853294373,-0.6736295223,-1.2852391005,-0.6020596027,1.0396854877,-1.1734688282,1.0483592749,-0.1930020899,1.4001460075,0.6863822937,0.1874726713,0.1960242987,0.3801824749,1.2780096531,0.0715012625,-0.5726922750,-1.1324484348,0.3092543483,-0.1041374058,-0.7648612261,-0.6558532715,-1.1425851583,0.3871780038,0.9160779119,-0.2528611720,0.3229587674,-1.3621455431,1.0407729149,-1.5615122318,-1.7674455643,-0.1657711864,0.3561912477,-1.0535864830,-1.3808097839,0.0066822111,1.7632035017,-0.5058322549,0.4460215271,1.8662033081,-1.4253808260,-0.0295527857,1.3246070147,-0.4824059010,-1.3144742250,1.6353757381,-0.7058399916,0.6552345157,-2.3853127956,2.0654399395,-0.8080892563,-0.7019706368,0.4695746005,-0.5398542285,1.9267045259,-1.6482932568,-0.7667036057,-0.0920951143,-0.3394153118,0.4104304612,0.0820508972,-1.0529474020,-0.1035258546,-1.2655054331,-0.8173121810,-1.5338441133,0.7748808265,-1.3071223497,0.7862274051,1.2642416954,-1.6465983391,0.7156240940,0.0004608648,1.3903490305,0.7563246489,0.7296988368,3.1193788052,-1.5182796717,-0.4620943666,0.6188848615,-0.6592494845,0.1667702049,0.1148974299,-0.4236586988,-0.6178045869,0.5739185214,-0.7387426496,-1.9529211521,1.4813373089,0.9259881973,-0.3275189400,-0.5661215186,1.0159183741,0.3718490899,1.7064806223,1.0632951260,0.2158806324],[-0.0558133088,0.9810810089,1.0780202150,1.5175112486,-1.0642248392,0.8798881769,1.0489073992,0.7985528708,-0.2425360382,-0.5692839622,-0.4421700239,1.1829074621,0.1835972518,0.4506121576,0.9762408137,1.1348419189,0.5359632969,-1.2192471027,-2.3011717796,-0.2138852626,-0.4985000491,-0.1059588790,-0.0988596678,1.0034368038,-0.5149103403,-1.8779270649,-0.6865703464,0.1736295819,0.8265163302,-0.5225960612,-1.3647763729,-1.0260533094,0.2010704130,0.6330992579,-0.8404617310,0.3592998385,0.3706113994,0.5354285836,1.0670536757,-0.1831074059,-1.4682694674,0.0098087247,1.5302503109,0.1058414206,-0.1238831207,0.4766884148,-0.4761192799,1.2386989594,0.8832945824,-0.0387233086,1.3760423660,1.6366331577,-1.9266943932,0.2121179402,-1.6006190777,0.1265319735,-0.9488639832,0.3272991180,1.2126785517,0.6757702231,-0.9969081283,-0.8930254579,-0.6828251481,2.1810488701,0.1021359116,-1.7115709782,0.7546977401,1.5872236490,0.2561224401,-0.6409754753,-1.3000752926,-1.5575027466,-0.3797564209,-0.2285472006,1.1859147549,0.0516077168,0.9234830737,-0.8794897795,-0.2575468421,-1.9377614260,-1.5830955505,-1.5876456499,1.1509450674,-0.4737126529,-1.1898517609,1.3928337097,1.6262747049,-0.4686792493,1.0509574413,1.5402649641,-0.7110273242,0.7389134169,-0.4548214078,-0.8832143545,-0.7396560311,-0.8138209581,0.4599758983,0.0499489568,1.7926896811,1.0037494898,-0.0817498639,-0.9807576537,0.1917232126,-0.7828310132,-1.9145452976,0.4286828935,0.9644827247,0.0961133540,-0.3806517720,-0.5880751610,0.0334974229,1.4758988619,0.0888777971,-0.6958780289,0.1805043221,-1.7607322931,0.2534747124,0.0275088325,2.2588088512,0.7110636234,-0.1102534533,-0.0299374927,-0.5157200098,-0.4786728323,-0.7900257111,0.4293768108,0.3290570080,0.0805050284,0.6053853631,-1.1858336926,0.1329994649,-0.8362782598,-0.4463221431,0.2802517414,1.5189036131,-0.9790352583,-1.0094639063,0.5211237669,-1.6937114000,-0.1594283283,0.6029412746,-0.0587336123,0.0071533583,0.0523125641,-0.2256063074,0.1295899749,0.1687475741,0.6759440899,-1.6508245468,-0.6614670157,1.0911030769,-0.4605453908,0.4364682734,1.3217781782,1.1893281937,0.1233901903,-0.9958592653,-0.3711001277,0.5379791260,-0.0070412341,-0.3989662826,-1.0785380602,0.1701700389,-0.3741233051,-1.2407640219,0.4376882315,-1.1232664585,0.4728543460,1.7027870417,0.3569022715,1.6514447927,0.0602583177,0.3123974800,0.1193226576,-0.5213288069,-1.4381340742,1.4283020496,0.5926224589,0.1629691124,0.5686670542,-1.0246697664,-1.1791366339,-0.9171492457,-1.4742364883,0.7465749383,0.4492428005,0.9631430507,1.0591726303,0.1239316314,-0.3596034646,-1.0907340050,-0.6030093431,-1.1057633162,-0.8140255809,-2.1808836460,0.8547969460,-1.1567357779,1.3341215849,1.3263473511,-0.6043748856,-0.6644125581,0.8303882480,-0.7894672751,-0.5509001613,-1.1912413836,-0.4110831618,0.0052479375,-0.5508015752,1.2821806669,0.8930993080,2.6927654743,-0.0384279191,-3.2195563316,-1.0180802345,-0.8228209019,1.4848512411,0.1804568619,-0.0416801386,-0.4823452234,-0.3673158586,-1.7843573093,-0.6924807429,0.5369516015,-0.6816094518],[-0.7692908645,-0.4790070057,-0.4438000023,0.0430159196,-0.1231807619,-2.3414075375,0.4058639705,-1.1069461107,0.9379891157,0.5567139387,1.6205422878,-0.0333523005,0.4900620282,-1.5986202955,0.5159187317,-0.8220391870,0.5268859863,-0.1108082831,0.9905723333,0.6257894039,1.1551108360,0.2970907986,-0.0550662689,-0.3417685330,0.4716485739,-1.9665927887,0.6737215519,-0.3018049300,0.1630557626,0.4377413690,1.6650433540,-1.2340313196,1.0803861618,-0.9714391828,1.0315043926,2.7991528511,-0.9023184180,0.3342669308,0.8662059903,0.4187904596,1.6631202698,0.9461517334,0.0383105762,0.3874709606,-0.9331662059,-1.5427381992,-1.2080690861,-0.0268997028,-0.3367968798,0.8820804358,0.8696154952,-0.9525085092,-0.1397367120,-1.0950832367,0.9954438806,0.1454400718,-0.1136238873,2.0428493023,1.1829013824,1.1000951529,-1.1066437960,0.1360276341,0.7086342573,-1.2057681084,0.1230445802,-0.0300690066,-1.3471719027,-1.1712793112,0.4815954268,-0.0332318284,0.0187704470,0.8783824444,-0.2106922418,2.4618752003,1.7784832716,0.2102992982,-0.2804616690,1.8727797270,-1.0322837830,-0.6775320172,0.5041838288,-1.9068819284,-0.4035941362,-0.4970909655,0.4764173329,0.7262157202,-1.3696603775,-0.2180223465,0.9146516323,-0.6380504966,-0.6836457253,-0.9392985106,2.5385532379,0.7848135829,-0.4739396274,-0.1609451324,-0.3473201990,0.4547399580,0.0022027995,0.3719330728,-0.2290373445,-0.8378288150,-0.5939671993,-0.4501929879,-0.9564500451,-0.7512276769,0.3331740499,1.7069742680,-1.8661465645,1.6704733372,-0.7440801859,-0.5513499975,-1.3664151430,0.4946258068,0.6614918113,1.0545921326,0.7613997459,-0.1494040340,0.1473492235,1.0284640789,0.1794055253,1.3030784130,-1.0664352179,0.8017191291,1.0707521439,0.4280396998,0.7229521275,0.6674688458,-1.4753994942,-0.1161597595,-1.5815646648,-1.1041642427,0.2921075225,-0.6162883043,-1.0617330074,0.0474488698,-0.2354591340,-0.3827132881,0.0709788352,0.9242456555,-0.7282121181,0.5422099233,-0.7606021762,-0.6083860993,-0.0204535741,2.1217327118,-1.1327162981,0.5107544661,0.0726600289,-0.0524952970,-0.1610554010,0.5236122012,0.4384548962,-0.2347787917,-1.5538160801,-0.2386575490,-0.0072724433,0.5456941724,-0.1086401194,0.7346530557,-0.5908902287,-0.1918531358,-1.8580100536,-0.5037835240,2.0335335732,-0.9577164650,0.0651107430,0.9060782790,-0.5345256925,0.1751286089,0.4575206637,0.3239696324,0.0477281995,0.8175007701,0.9586806893,0.2785039842,0.5604625344,1.0212132931,0.4966633320,-0.0258158334,0.9687435031,-1.3536084890,0.8649412394,-0.8730837107,-0.8804031014,-1.6783584356,-0.5945875645,1.2743442059,-0.6184026599,0.1897069663,0.0141431382,0.2350640744,-0.7193506956,0.1511185616,-1.5639774799,-1.2819161415,0.0610125959,-1.1097881794,-0.8521714807,0.5019956231,0.0409483574,0.1502437443,-0.7988834977,-1.8650946617,0.5831124187,-0.7960220575,0.2591686845,-0.0523419008,0.6130117774,-0.4703146815,0.8446735144,0.9293216467,-0.7193233967,-0.3462554812,0.0266267434,-0.2921901047,-2.0190968513,0.8755489588,-0.1333952248,2.0068006516,1.4492278099,0.1540557742,1.7440308332,-1.0899720192],[-0.3867556453,0.1545641422,-0.7254714370,0.7313339710,2.5498569012,-2.4311244488,2.2926156521,1.3991469145,1.1989866495,1.3550738096,-0.9609283209,0.3282889426,0.8142353892,1.5970354080,-0.2387523353,0.0792709738,-0.5485795140,0.4264426529,0.2009727061,0.3822948635,-0.3160178065,0.9924373031,-1.0522011518,0.6819784641,0.1449483186,0.7345592380,-0.4123094082,-0.8291899562,-0.1473677456,-1.9918057919,1.2308152914,-0.4426414371,-0.3787471354,-3.0473160744,-1.6859333515,-1.3871006966,-0.0958412439,2.0981168747,0.5395963788,0.5177062154,-0.6460278630,2.3688600063,-0.7596668005,0.8309730291,-1.3052463531,-1.1465224028,-0.2534221113,-0.0777573884,1.5662372112,-0.8439161181,0.5423812270,-0.4233368933,-0.0301173329,-0.1885342002,-0.7636304498,0.6386103034,-0.3293936551,-0.5986554623,1.6507289410,0.6110025048,-1.1714577675,0.2430572808,1.4613201618,1.2600826025,-0.5669208765,-2.4710955620,0.7282733917,0.0715143606,2.3573234081,0.2791848481,0.3507069945,-1.4286094904,-1.3957622051,-0.6828049421,-2.1207742691,-0.5035426021,-0.0339629166,0.7257949114,0.6201557517,1.2286385298,0.9077768326,-0.3393974602,-0.8418522477,1.1805078983,0.0826563761,1.4109766483,-0.1218153611,-1.1622179747,0.6720657349,0.8871431947,-0.6016659737,-0.2201759368,0.5747567415,-1.4447933435,0.4896268249,0.7228952050,-2.4493446350,-0.7032957673,-1.5452859402,-1.4874507189,1.7770699263,-0.5654845834,0.9598588943,-0.2666018903,-0.1979894638,-1.8093239069,0.5844339132,0.0779933855,-0.7883568406,2.2556648254,0.2092601806,-0.0397630557,1.2062901258,1.1393060684,-0.5246899128,-0.6015848517,-0.1014802530,1.8655210733,-1.2383139133,0.0829883963,0.3803620636,-0.4085579813,3.0920526981,-0.8149982691,0.2523723245,-0.3126744330,0.2925846577,1.2427134514,-0.3900679052,-2.2280981541,-2.0701429844,-0.5498985648,-0.3402158022,-2.2202494144,1.0159556866,0.8057820797,1.4112379551,0.4200439751,-1.3517037630,-0.4663037062,0.8230061531,1.4041137695,0.0515968874,-0.3207317591,-0.9438809156,1.0704278946,0.8321710229,-0.0838056281,0.1407217234,-0.4895201027,-0.1014306247,1.3761303425,1.7014065981,-0.2475039661,1.0797243118,0.2581546605,-0.0555630252,-0.7183734179,-0.4093923867,-0.4314723313,1.6992679834,-0.5810607672,-0.4299448729,-0.1757195890,0.4817943871,0.8706088662,1.1988463402,-2.6215486526,-2.0343215466,0.4191960394,1.6816884279,0.2876927257,0.9413163066,0.7395244837,0.4954246581,-1.6228507757,-0.6659519076,0.6042791605,-0.3217880428,0.1207242757,-0.2647857964,0.2765117884,-0.1759828329,1.1069335938,0.4527967274,1.9177540541,-1.4181673527,-0.5517150760,1.4401162863,1.2062914371,0.1064064354,1.1001257896,0.5693667531,-0.0823822916,-0.8558875322,-0.3975574374,0.8107486963,-0.7338398695,-1.6810228825,0.2814816236,2.0815069675,-0.8396160007,0.7340915203,-0.1046710759,2.3806743622,1.1243662834,-0.7335973978,1.3209965229,-0.0835166797,-0.4221691787,-0.2683244646,-0.2047377080,-0.8506658077,0.3369838893,-1.3512299061,0.5600090027,0.4778011739,-0.9195017815,1.3022620678,0.0175112654,0.5426177382,0.1755894125,1.0299211740,0.7771252394],[-1.3532475233,-0.3722300828,-0.1029285714,0.7554801106,0.5151497126,0.3328151405,0.2304541320,0.1953039616,-0.5153769851,0.4043092728,0.1988413036,-1.2689567804,0.4159680605,0.3145648837,1.0199477673,-1.7042839527,0.8625801802,-2.2885460854,0.4905623198,-0.5468719602,-1.4397684336,-0.2191286832,-0.3316198587,-0.0544967875,2.1939680576,-1.3659399748,-0.3305815756,-0.7713046670,0.9388977885,-0.7135739326,0.1881475300,0.7682553530,-1.1076250076,2.2674283981,-1.1733065844,-0.0992131382,-0.0065833130,-0.7474772334,-2.5292699337,1.3511906862,-1.0686225891,2.1275341511,-0.7644865513,0.7966610789,-0.0352354646,1.6254175901,-0.4221188426,1.3619796038,0.6780332923,0.3214618266,1.1740883589,0.0146083022,-0.9775192738,-0.8737570047,-0.3637194037,-2.3226566315,1.0262193680,0.3523386419,-0.8140775561,-1.0497467518,0.1893426478,-1.9934339523,-0.4920479059,0.0709401369,-0.2049642354,1.6265887022,1.4623658657,-0.3380526900,0.6974494457,-0.6214071512,0.6323260069,-0.1815361679,-0.2832997441,0.4412190616,1.0376160145,-0.0181217846,0.2928813994,-1.3679971695,0.6803232431,1.3326506615,0.5246801376,-0.4922191799,-1.0667096376,0.5750662684,-1.2777228355,0.4103414118,0.0826276764,-0.0194279980,1.1380794048,-0.7974613309,-0.1227394193,-0.4622454047,-0.9462473392,-1.0382105112,3.0540573597,0.3594521284,0.5559291244,0.7019115686,-0.1078845412,-0.8671279550,0.3546288610,0.7513562441,0.1686596423,-0.2308986932,-0.6836241484,-0.9934930205,0.0928095505,-0.4998281896,1.3151831627,0.7407268286,0.5519161224,-1.9649449587,1.5386253595,1.6443663836,1.4250090122,-0.4822165370,-1.0676637888,0.3187181652,0.4153170884,-0.8883006573,-0.2742860317,0.2505654395,0.3850674331,1.8112043142,-0.6003881693,-1.2677028179,-0.0657587349,0.5034014583,-0.1584416926,1.0296255350,0.7896546721,-0.1844998002,-0.8324058652,0.7251090407,0.3456537127,-0.1612031162,0.5519640446,-1.2285329103,0.4051709771,1.7382738590,-0.7669152617,1.3024536371,-0.2334417999,-0.8495087028,0.0963180736,-1.4320399761,-0.4009811580,-0.0825446695,-1.6872545481,0.3303807080,1.4178494215,-0.7337759733,0.9069579244,-0.9080467820,-1.1837409735,-1.2833039761,-0.7491318583,2.5821473598,-0.5403444767,-0.4830243587,-0.2678615451,0.8788146377,-0.0934342444,-0.6283229589,-0.1921992749,-1.9643006325,0.2488131225,-1.1531807184,0.3248987794,-0.2471186668,-0.7058179975,0.5422291160,1.1079134941,1.4611216784,0.8116730452,-0.0140767116,0.4220570624,-2.0194842815,-1.1250903606,-0.9013854265,-0.1820059419,0.3428933024,-1.2460854053,1.4754422903,0.9102709889,-0.7721157074,-0.0860439911,1.6187556982,0.5236514211,0.6717063189,0.8059553504,-2.1456305981,1.8775573969,-0.8045200706,-0.7242151499,-1.2274599075,-0.3879765868,0.0832767859,-0.9325447679,-0.5355188251,0.7970010042,-1.0122157335,-0.2169611752,-0.4314942956,-0.7536115646,0.9737334847,0.5086837411,-1.4131776094,0.0424655750,-0.5832741857,0.7751629353,0.6793202758,0.4981907010,0.6098150611,-1.0555416346,-0.4501825869,1.2009423971,-1.1430513859,1.2606869936,-0.2908736765,-0.3532292247,0.2760093510,0.0486681461,-0.9852678180],[-0.5179210901,2.9281203747,1.8839042187,-1.2333828211,-0.5978958011,1.6062750816,-0.3856277764,0.3353347480,0.9008058310,-0.4921723604,0.4860668480,0.1493294090,-0.3788942695,0.3776074946,-0.1087439433,0.8221364617,-0.0039486336,0.0350735076,-0.7033908367,-0.3409045637,1.2027546167,0.0638573021,-0.4985602200,-0.3220447898,0.8035058379,0.0312587582,-1.0049045086,0.0859072581,0.4213287532,1.3841396570,-1.9018831253,0.2462813258,0.3927054703,0.8714232445,-0.5415591598,0.9047995806,1.5702162981,0.5778482556,-0.3848070502,-1.4516043663,-0.1781129241,1.0578336716,0.1071197763,0.4987775087,0.4444977641,0.1904703826,-0.0500574149,0.8652390838,-1.3767344952,0.6599958539,-0.1546868533,-0.4128006697,0.4206206501,0.7529597878,1.6279267073,1.6883649826,0.3809396625,1.4642118216,0.0701802224,0.9007957578,-1.0695656538,0.0756013691,-0.2558471262,0.6178277135,-0.1640721709,0.5888280869,-0.0450142436,-1.9780269861,1.5108190775,0.0201296490,0.3490985334,-0.8458254933,-2.2395009995,-0.1078830585,-1.2979288101,2.2681913376,1.1489452124,0.5595183969,0.6483691335,-0.8494003415,-1.2000247240,-1.7947679758,-0.3354310393,-0.8372511864,0.4137484133,0.4951325357,-1.3353157043,-0.4037382603,0.1360496730,1.7940540314,0.5145080090,-1.1026799679,0.3898757398,-1.3311768770,0.6445837617,3.1182668209,0.7394161224,0.5409551263,1.4241337776,-0.4103944302,-0.9392827153,-1.0108066797,-0.9068341255,0.0709434673,0.9242113829,-0.2061729133,-0.5818979144,1.0988814831,-1.0460087061,-1.2680871487,0.4011656642,0.7063562274,0.5195142031,0.1841870248,0.4354915619,-1.1544827223,0.7519261837,-0.8682871461,1.2139902115,1.8845286369,-0.3824562132,-2.9315984249,1.0975605249,-0.6656762958,-0.3710460961,-0.4581338763,-0.1719742119,0.6240089536,1.4918496609,-1.3402218819,0.5351183414,0.3458021283,0.4096876383,0.6907798052,1.3452779055,-1.4073811769,-0.7307488918,0.4452004731,0.9288562536,-1.3342907429,-0.9971263409,-0.2311260700,-1.9997789860,-1.1380411386,0.5919696689,-1.6074502468,-0.5450325012,0.1557483375,0.4483186901,0.7535129189,0.3043299913,0.7550746799,-0.7720427513,-1.0777661800,1.1778817177,-1.8641024828,0.5344553590,-0.0597600453,-0.5732216239,0.9112659693,0.1287456751,0.8489707112,0.4421401024,0.6027107835,-0.1285692751,-2.4745180607,0.1951732039,0.7277007699,-1.4732520580,-0.4286610782,-0.1154345125,-0.0769550353,-1.2591086626,-0.1017499194,-0.0210780762,-1.1737729311,-1.1453151703,0.3725708425,0.8576273322,-0.4034656584,0.1232902855,0.4509639442,0.3320712149,0.2331693470,-0.6623501778,1.1165158749,0.3908354342,-0.2755193710,0.7766165137,2.2021074295,0.7839115262,0.1298831701,-0.8191106915,-1.3449116945,-0.2769561410,0.0589544252,1.4789159298,0.6346148849,-1.5675116777,-1.2349410057,0.0871513784,0.4487423897,-0.2801265121,0.8739984035,-0.6375115514,0.2110447139,-0.3336372674,0.9330590367,-0.5646930933,0.5073488355,-0.0384628698,-0.3513286114,-0.8750590682,-0.9063674808,-0.3104023635,0.4549559355,0.0239758920,-0.5857896805,0.0329510607,0.6853071451,-0.9402374029,-1.0898227692,1.1488786936,0.3411648273],[0.7200110555,0.1597515345,-0.3885217905,1.3693848848,-0.3649668992,1.1565055847,0.0767926425,-0.2566693127,0.4097830355,-0.1426362097,0.5309285522,-1.0245457888,-0.0216459669,0.5516811013,0.1327580810,-0.2019371092,2.0907456875,-0.6933072209,-2.1148211956,-2.0299284458,0.3427873254,-0.0116709834,0.4638992548,0.6261547208,0.5721890330,2.9009780884,-1.1405460835,0.1877879500,-0.1944549531,-0.1844120920,-0.5303893089,-0.4590159655,-0.1941795200,-0.1404060721,-0.7878784537,-1.3694969416,-0.0244819149,-2.6898519993,-1.0086953640,1.1471830606,0.5464369059,0.9971829057,-0.0929342508,0.7358595729,-0.2085779309,-0.9564427137,0.2623963654,-0.7120863795,-1.0331848860,0.2814248204,0.2566384077,2.1200895309,-0.7847632170,-0.9394590259,2.1373596191,0.4847322106,-1.0768883228,0.0257185530,-0.3627569079,1.1377524137,-2.3272347450,-0.7009713054,-0.3798924983,0.5566964746,0.2665447593,0.2588733137,-0.4382838905,-1.5288709402,0.7454527020,-2.0549416542,-0.7336265445,0.7902216911,1.0842144489,0.8565483689,-0.5160650611,-0.2388527393,0.1325882673,1.2349766493,-0.5192912817,0.6799807549,0.3111981750,-0.1828193814,-0.6931616664,1.0630458593,-0.3297859728,0.3573651314,0.1918357015,1.0749788284,0.8603224158,-0.5975485444,-1.3843562603,0.9666813612,0.2949757576,-1.0964585543,0.9764108658,0.0149889840,1.0665789843,-0.8470121622,0.8645345569,1.3421913385,0.7372649908,-1.0292098522,-0.7991414666,2.0994555950,-0.0386573002,-0.1209427640,-0.1018707603,-1.8904802799,0.2138135731,0.5829991698,0.1209255829,0.2682153881,0.1597340852,2.0694198608,-0.8160514235,0.9405336976,-0.2612581849,-0.7488277555,0.2938060462,-0.4454466403,2.0709934235,1.0594415665,-2.0218315125,0.1861803085,-0.3184541464,1.0500028133,-0.7971460223,-0.4653572142,-0.1952562183,-0.3340694904,-1.5124847889,0.8677157164,0.1723863482,1.2384344339,-0.1841048002,-1.2000898123,0.4072071910,1.3923242092,-0.7195712328,-0.9771297574,0.7300099134,0.0627533272,1.9138554335,-0.0584848188,0.8246739507,-0.4379732609,1.9184089899,-1.5187605619,-0.8122234941,0.8544716239,-0.3368932903,0.4989197552,-0.6576798558,-0.5772010684,1.0729271173,0.8953892589,0.4057393372,1.2137151957,-0.8393722177,0.0525206178,1.3459855318,2.1250915527,0.1242673770,-2.0543036461,-0.7189696431,-1.6973289251,-1.5774748325,-0.0314979739,-0.2412691116,0.2239166349,-0.5194889903,1.0742644072,0.1609522700,1.0746810436,0.1200929657,-0.2726342082,1.4211553335,0.0857824907,-0.2053765953,0.3187888861,0.5143779516,-0.5992152095,-0.6816874743,2.9358263016,2.0151743889,0.5622567534,-0.0935387388,0.1492368132,0.5007673502,0.5098925233,1.1149957180,2.1785109043,0.6689604521,-0.9561447501,-0.7765575051,-1.2729759216,-2.5599219799,1.2151373625,0.5340461135,-0.6273633838,-1.3690152168,-0.7996125221,-0.2076777518,0.5959313512,0.0312181860,-2.0051684380,1.3749988079,0.5573381186,-1.8676750660,-1.1792148352,-0.4203329086,0.0985771790,0.6573711634,1.1733105183,0.3345836997,-1.2246304750,-0.5836922526,0.4716092944,0.4792221189,-1.2958402634,1.8630772829,0.2784361243,1.0937455893,-0.2099616975],[0.4214741886,-0.5144574046,-0.2383110076,1.2241157293,1.1667163372,-3.0629582405,0.2755071521,0.9796141982,1.1982212067,-0.7435212135,0.0888084620,-1.1646779776,-0.8349477053,0.1836696565,1.0464693308,1.4091336727,0.3512982130,-1.1093584299,-0.0148777813,-0.7249640226,0.5148887038,-1.1883825064,0.7954053283,0.0648263469,-0.3492616117,0.6866639853,0.1444717646,1.5906834602,1.1906172037,1.1974231005,-0.2618243992,-0.7974213362,-0.7960732579,-0.3851014674,-0.9355871081,-0.6335410476,0.1873655766,1.1265091896,-0.0838515311,0.1801059991,1.3423806429,0.8518787622,-0.6987745166,-1.1183056831,0.0339035653,1.2098129988,1.0811673403,-0.1526868492,0.5863786340,-0.6288794875,-0.2684370577,-0.1966408342,-1.8438454866,-0.7123699784,-1.7283849716,-0.9004950523,0.4246796668,-0.3308639824,-0.0020428838,1.0246882439,-0.8577266932,0.6733812094,-0.9730071425,0.5584130287,0.3536276817,0.5739370584,0.5155097246,-0.0672519654,0.1801947504,0.1836582124,0.5291321278,0.5086742640,0.0655369684,-0.8849287629,0.6907548904,-0.0488187782,3.3209657669,-0.0501876213,0.8470934629,0.1091856286,0.5762174726,0.1049321070,0.4455679953,-0.8758905530,1.8857425451,0.7774903774,-0.0076641762,0.8670355678,-0.3381590247,-0.8557353020,-2.0405881405,-0.2047748268,0.5438323617,-0.9618906379,0.7381688356,0.3258624077,1.4617965221,-0.0028879009,-0.2676487267,-0.8923782706,-1.8217835426,-0.0186384395,1.7530089617,-1.1791179180,-0.4352858365,0.9670229554,-0.3449574411,-0.3940675259,0.4946045578,0.2536029518,-1.0679191351,-1.2482924461,-0.1159172207,0.8719373941,-0.3312026560,-0.3494094908,0.6839279532,-0.5084854960,0.7218002677,-0.2618828714,0.3507729471,0.2381303906,0.7938268185,0.7570793033,1.0622991323,0.6175446510,-0.9916316271,1.6769894361,-0.3215295076,-1.0892727375,-1.4173982143,-0.5432204604,-0.2167730033,1.6159602404,-2.2631447315,0.9030470252,0.6519251466,-2.4697923660,-0.8317915797,0.0972767100,-1.2725318670,0.6019590497,0.0795885250,-1.1497843266,-2.4003005028,-0.9797340035,-1.4273698330,0.6937153935,-1.8866250515,-1.2132887840,0.2422034889,0.9020537734,-0.6387479305,-1.2040439844,-2.5717265606,-0.9823888540,-0.8603173494,-0.1113521829,-1.5185772181,0.3711701632,0.7167190313,-0.2066891938,-0.6412582994,-0.9086228609,1.2789639235,-0.5552526116,-0.2739999890,0.6681884527,0.2850576937,1.1664259434,-1.3593558073,0.3325866759,-0.2983123660,-0.1339031905,1.3430116177,-0.1880369335,1.9697154760,-0.1896455884,1.6410228014,-1.5998033285,0.4156921506,0.5480934978,-2.6319608688,0.3182291687,-0.4545027018,-0.0404811949,0.5020309687,0.4129009247,-0.9264763594,-0.8234125376,0.6390948296,-0.0019334590,-0.5494630337,-1.3261779547,-0.7078437209,-1.0352882147,0.8757877350,0.3657563031,1.5296053886,0.6850796938,1.3457660675,-1.2615861893,0.0619928725,1.7164348364,1.1516953707,1.2520174980,-0.0763857737,-0.9964673519,-0.6538204551,-0.7642871737,-2.7673952579,0.0269780625,-3.5376372337,0.2615314126,-1.8532675505,-0.3508562446,0.3244189620,0.4992468953,0.2832391262,-0.8766116500,0.3057894111,1.6057144403,2.1141822338,-1.0781127214],[-0.9287056923,-0.8534093499,-0.2685569525,1.3060340881,0.1535290927,-1.2112205029,-0.4509466588,1.3873354197,-2.5044455528,0.5372287631,-0.6981587410,-0.4939994812,-0.2896173000,-0.2453738302,0.3176817298,1.4238258600,0.7012366652,0.2226518840,0.8875376582,-0.2895725965,-0.5405652523,-0.5801152587,-0.6099621058,-0.1794458330,0.3522992730,-0.3782028258,0.5201066732,0.7829122543,0.1191232353,0.1265148669,-1.2816922665,0.5576660633,-1.7459415197,0.2186397910,-0.5908021331,0.9046570659,-0.6815707088,0.2523507774,-0.1003115773,-0.6361647248,-0.2780383527,0.8642255068,0.9515980482,0.9820371270,-1.4303667545,-0.0983356610,1.4673168659,1.1516301632,1.9530824423,1.2597833872,-1.4902018309,0.3255969882,0.7496146560,-0.0849756375,-1.0007236004,-0.5960429311,-1.0154364109,1.2511830330,1.2626070976,-0.5875334144,1.4028255939,0.4695758522,-0.0255566500,1.8417292833,-0.0399172194,1.1069836617,-1.2826505899,-0.1472184211,0.9856489897,-0.1729861200,-0.3876519799,0.6959972382,0.2856332362,-1.0207430124,-0.9289630055,0.0632748380,0.4820849895,1.5037152767,0.4867504835,0.0797959343,-0.1955450326,0.7896358371,-1.1532626152,-1.1208647490,-1.6553353071,1.8035339117,-0.0838049203,1.3239436150,-0.4971853793,0.8107408881,-1.1551069021,-0.1953871846,-0.7771847248,-0.7813466191,0.5065687299,-0.1820823550,0.2684529126,0.9292242527,0.7593740225,-0.9009737968,0.4765752554,0.1643548906,-0.3488487899,-0.3658882976,-0.2314924598,0.4069265723,-0.9488435984,0.7248171568,1.4139008522,1.5861266851,-0.1279027164,-0.0553276762,0.7255465984,0.1429184526,0.1820705831,0.2178043425,-0.0077814292,1.4773465395,-1.1627372503,-0.6053957343,0.4770019352,-1.0118191242,-1.3295882940,0.7362958193,0.7915869355,0.3331216276,0.5359749794,-3.0860996246,0.3936527073,-0.7873288989,0.1485450566,-1.2330771685,-1.2370146513,-0.5632748008,-0.4914303720,1.0046528578,-0.7574470639,1.2790346146,-1.9569309950,-0.6932571530,0.3972877562,-0.1010534316,1.6112462282,-1.3199983835,-0.5487880111,-0.5936339498,-2.0641529560,0.5989998579,0.8124501705,0.3341778815,-0.7117627859,-1.7005486488,-0.7905029655,-0.7358307242,-0.0822679624,-0.2129524499,-0.0102335811,0.2852599919,-0.2062803209,-0.5369126797,0.9724613428,-0.3041396439,-1.0042915344,1.0672525167,-0.2289439440,1.3452187777,0.5357630253,-0.4374930859,0.6523374319,0.0650269166,1.2034343481,2.5041522980,-2.2119021416,-0.0325773247,-0.2486386299,-1.2621959448,0.9550855756,-1.0251768827,1.3576012850,-1.0082174540,0.4790438414,0.7539923787,-0.2292823493,-2.0603711605,1.2167100906,-0.0632605180,0.6282228827,-1.2875719070,1.2758817673,-0.0361953229,-1.1444544792,0.0995380729,2.0643885136,0.0476284251,0.3599074483,1.3913724422,-0.8276139498,0.4267857373,-0.9940422773,-1.6722874641,-0.4820932746,0.9585770369,-0.3583551347,2.2475891113,0.6795741320,0.0427838154,-0.0337097496,0.2812221050,2.2352130413,1.5787955523,-0.0525405966,0.7612043023,-0.2395873815,-1.1629893780,0.9450313449,-1.5587869883,-1.0885256529,1.2096309662,-0.6176126003,0.2794592083,0.4000840187,-0.2106546909,2.1949429512,-1.7044574022],[-0.2265886366,0.5426079631,1.1125903130,-0.5519072413,-1.1224460602,0.8363450170,-1.5743186474,-0.5271378160,0.6508792639,0.5543196201,-1.1395976543,0.1506068110,-1.3732875586,-0.5603883266,-0.3320862949,0.0500928350,1.4821004868,1.3670789003,1.0397155285,0.9093261957,0.2789103091,-0.4556762576,0.9363845587,0.5859700441,-0.8330316544,0.9558687806,-0.2014526874,-0.0478864685,-0.8698826432,0.2509627938,0.1928938031,-0.5245307088,-0.7604442835,-0.9027975798,-0.8322229981,-0.2535308897,0.2865951061,-0.8358671069,-0.0175675061,-0.4595412612,0.2229304463,0.2540488839,0.1446684897,-0.5048643947,1.2162379026,1.4006818533,-1.8685104847,-2.1495907307,0.3018081784,-0.9322087169,-0.6951150894,-0.3476719856,-0.8317773938,0.5383685827,-1.4129815102,-0.1823303849,1.0443432331,1.1775488853,0.0203446262,0.6534597278,0.1316889226,0.9037830830,-0.8564657569,-0.2451224774,0.5288985372,-0.3417075574,-0.4940801263,1.2461600304,0.1253131926,-0.1442823708,-0.2593114078,-0.7173941135,1.5848474503,1.3283705711,-0.4759116769,0.5617510080,-0.2803081572,-0.1529893428,0.7441819906,1.0778692961,-1.6746370792,-0.0191578642,-1.6233841181,1.3962148428,2.4005887508,-0.7849439979,1.6532175541,0.8943389058,-0.7218570709,0.0025848437,0.1503669173,-1.1533824205,-0.6082597375,0.5895243287,0.6123408675,1.0880017281,-0.8497795463,0.2664059699,0.1507609338,-1.1018321514,-0.7293382287,-1.2999854088,-0.5079394579,1.9566550255,-0.6045329571,-1.1847981215,1.3479480743,-0.4842537940,0.4533431828,0.2714575231,-0.0489397943,-0.0387133919,-1.2142137289,-0.5738778710,0.4517196119,-0.0616835095,0.7554846406,-1.2030466795,0.2224065214,0.8758687377,-1.5098052025,-2.5789873600,-1.3044232130,-0.9159380198,0.3439801037,1.0308520794,-0.5655797124,0.6416946054,-1.5301198959,-0.2333142459,1.3705869913,0.5391677022,2.1740953922,0.6560552716,0.4826991856,-1.5273411274,0.6695680618,0.4246776700,-0.1707382351,-1.0786836147,0.3128767908,0.5183787942,0.5231448412,-0.7981930971,0.0068109701,-0.9226770401,-0.7949106693,-0.1485601366,0.5388772488,1.1333990097,0.2188962549,-1.0067234039,-0.3539472222,0.7378255129,-0.2585050464,-1.1470578909,1.2719910145,0.6277831793,-0.6327692270,-1.3171602488,-0.6405596733,-0.4403110743,0.5331921577,-0.0618506931,0.1133509055,-0.2367716134,0.0172743201,-0.9363154769,0.0419243276,1.2715209723,-0.6324470639,0.4981867671,-1.4744064808,0.9432847500,-1.3852499723,-0.0248926673,-0.1987091005,0.9218626022,0.3695135117,0.6386186481,0.9051753283,-0.6882691979,0.1940152645,1.0115746260,-0.1039159670,0.0934223235,0.6569828391,1.5856434107,1.3781589270,1.1807434559,-1.2077684402,-0.0713830218,-0.6892064810,0.7983953357,-0.1320799440,0.9095534086,1.0258624554,-1.0350824594,-0.1084372476,-0.7068145275,1.0966942310,0.2121248394,1.6011934280,-0.1354372799,-1.6426924467,1.9219787121,1.9771164656,-1.2754851580,0.4517096877,1.3728480339,-0.5387133956,-0.5070499182,1.0105915070,0.7020894289,-1.4419972897,0.0110328682,-0.1098614261,-1.4754793644,0.3179679811,0.3669794202,-0.9281835556,1.2898329496,-0.9199350476,-0.9801771045],[-0.9031108022,-0.3989333808,-0.3263574541,0.1047135815,-2.0553271770,-0.9773147702,-1.4125535488,-2.2337026596,-0.9101406932,-0.2648207545,0.5565056205,-0.6932720542,-0.8804928660,-1.4365410805,0.0302592870,-0.1010962129,-0.3346745968,0.3059233427,-1.3067972660,-0.0088376841,-0.3915492594,-0.6771771908,-0.5156219602,0.1084053442,-0.7851771712,2.0248520374,0.0768793225,1.6671743393,1.1379973888,-0.0037817969,-0.1691045761,0.1110712513,0.0237912405,-0.2556541264,-1.9002453089,-0.7983082533,-1.8317855597,-1.2898895741,0.0141506037,0.2845778465,1.1239186525,0.4390246570,-1.0800429583,0.9424997568,0.0881775469,-1.6444958448,-0.8998572230,-0.9650136828,0.2967081070,-1.0165929794,-0.5798018575,1.3800915480,0.6454607844,-0.0140456948,-0.5817818642,-0.1011662409,1.0112067461,-1.8833494186,1.3017792702,-0.7716539502,0.5666996241,0.2956871092,-0.2049203664,1.9108538628,-1.3607987165,-0.0717590377,0.0956634954,-0.5079314113,0.3618172109,-0.3595963717,-1.4459455013,0.2162457108,-0.7750922441,-0.2253048420,-0.6305879951,-0.6282056570,0.1342151314,0.9079183340,-0.3357201517,-0.9684320092,-2.0011854172,1.0298330784,-1.1672452688,1.2127306461,-0.5632342100,-1.1428827047,2.3399078846,-0.8843747973,0.5320113897,-1.1435959339,-0.8313342929,1.1637715101,0.8253864646,-0.9990708232,0.3957966864,0.4645176828,-0.6763690710,-0.2837693691,-0.4347972870,-0.7891721129,0.8290168047,-0.3206014335,-0.3041936457,0.7272228599,1.2169837952,0.7897850275,1.2175595760,0.9864512682,0.3498557806,0.9575622678,0.1763862520,1.5062904358,-1.4601914883,0.8869418502,0.4095101357,-0.0545322858,-1.0357341766,0.0918669775,0.7407202721,0.4786416292,0.6654282212,0.2292959541,0.2076932490,0.2183427811,-0.7750036716,-0.8375609517,-0.8351103663,-0.9253761768,0.0818053335,0.2907602191,1.0790300369,3.0961189270,-0.4322378933,-0.5297884345,-0.7185241580,-0.0790990219,-0.5318148136,0.3011080623,-2.3876340389,-0.6010844707,-0.6294558644,0.9789693952,-2.1652061939,-0.9091853499,-1.5246109962,1.6593085527,-1.1628981829,1.6147589684,2.2718012333,-1.5532001257,-2.0527155399,-0.4091469049,-1.0820819139,1.5153158903,-0.4930484593,0.8512840271,-1.1784952879,1.0569536686,0.0160085466,-0.0003073825,-1.4744670391,-1.2892800570,-1.2932577133,0.6164194345,-1.7043364048,0.2677577436,0.6191200018,-0.2715763450,-1.5342735052,-1.0618538857,-1.2137271166,-1.9104957581,-0.5562978387,2.0095219612,0.4219472408,0.2807143331,0.2929550111,0.0925148204,-0.7408627868,-1.2786219120,0.5953165293,1.1873878241,0.7985800505,1.1434313059,0.1420213878,0.3704443574,2.0830335617,0.8558987975,-1.0863219500,-0.6328001618,-1.0930259228,-0.7155646086,0.2989422679,-0.1714941859,-0.2596023083,1.0251544714,0.4840430021,0.5426319242,-0.0317259394,-0.5928928256,2.3365657330,0.8119526505,-0.3559478223,1.2486777306,0.3161210120,0.3295256793,-0.2986513674,-0.8083889484,-0.1058205962,-0.2012898922,0.1898895204,1.2744652033,-0.5763427615,-1.7786310911,-0.4619722664,-0.0343275331,-0.0804154128,0.0896555558,-0.2375993878,-0.6510353088,0.5634438992,-0.2756451070,-1.1919237375,0.0106554357],[1.0845719576,-1.4242180586,-0.7915678620,-0.4528565705,-0.5867254734,-0.3834092915,1.3105416298,-1.1719135046,-1.8663834333,-0.4044742584,-1.0571340322,2.1624724865,-0.3283438981,-1.6711210012,-0.2128783464,-1.1012598276,1.2401621342,0.1001905724,-1.6413034201,-0.4412343502,-1.6417485476,0.5818608403,1.0299129486,2.2170233727,-0.7258693576,0.3411789238,-0.0758841783,-0.1068310887,-1.5048621893,-0.7476377487,-1.0252985954,1.2971277237,0.9907476306,-1.0029420853,-0.8703005910,1.2154511213,0.1695734859,-0.6100532413,-0.2701125443,-1.8159172535,0.0744715631,1.4320803881,0.8319642544,-0.3548873365,-0.4736134112,1.2513190508,0.7541353106,-0.7968253493,-1.1330760717,-0.4850624204,1.3151828051,-1.8339599371,1.2641755342,0.6874196529,-0.1858444214,0.2456031442,1.2378497124,0.3065797091,0.1393100023,0.8186387420,0.5621598959,-0.2971632183,-0.9280245304,0.5608353615,-0.5151640177,1.3092679977,-0.1564584374,0.0101362988,-0.1147716045,-0.3308030069,0.5426612496,1.0478041172,-0.7588689923,0.9058098793,-0.6638476849,0.0271008201,-0.5996529460,0.9415428638,0.3311151862,-0.6850024462,-1.6973967552,0.8252288699,0.5447505116,-0.5340859294,-1.0320228338,-1.9213482141,-0.3236763477,-1.1210227013,-2.0754590034,0.9175598621,-0.1582947224,0.0355038270,-1.8584797382,0.8211516738,0.5859157443,0.9854813814,1.0377193689,0.8087520003,0.0553114042,-1.1089491844,0.2617270350,0.7748783827,-1.3112766743,-0.2853017449,-0.8751548529,0.4537158906,-0.4724691808,1.9114776850,1.1201875210,-0.6998970509,-2.2755661011,0.6124466658,-0.4858125746,0.1044009998,0.6781061888,1.0112067461,-1.0053843260,0.5832854509,1.3902379274,-0.3954279721,1.4138845205,-1.1117755175,0.0675625056,-0.7510002851,0.1992310286,0.1564755589,1.4238003492,1.1698158979,-2.0644772053,-0.7029905319,0.2314308137,0.2684699595,0.7005752325,-0.1916425675,0.1933514178,-0.3600777090,-0.5549749136,-0.6029216051,-0.7787041664,-2.5662279129,-0.7273243666,0.6202687621,-0.7350562215,0.2137556970,1.7911796570,0.5588005781,0.0419100970,-0.6966994405,1.0583099127,-2.8149473667,-0.2643944621,-2.2827417850,-0.7935805917,-1.1352584362,-0.2278170884,-0.8056265712,0.7685412169,0.3496689200,-0.3449663222,0.3916354775,0.0579748861,-1.4494156837,-0.1893656999,-1.6724202633,1.0861943960,0.5076280832,-0.0133326286,-1.2809752226,1.3494324684,-1.2507998943,0.7153228521,0.2078970075,-0.6573593020,0.7824165821,-0.5148826241,1.3713448048,-0.0101848487,-1.4278615713,0.6900991797,-0.7380038500,1.6913758516,1.6196644306,0.9960871935,-0.6944152117,1.6037415266,1.3314589262,0.2126102298,0.8190729618,-0.3324639797,-0.4693499804,-1.2738293409,-1.9276175499,1.8065828085,0.6967000365,0.1457846165,-0.8485788107,-0.0902344882,-0.4437417090,-0.8114491701,0.6733375788,-1.0148701668,1.2667295933,-0.2555393577,1.0527654886,0.0487716459,-1.0939552784,1.0991251469,-0.9288706183,0.6616157889,-0.1064211130,0.5533397794,0.4668231606,-1.0746521950,0.5423960686,1.0720642805,0.8632643223,0.1105926335,-0.4526108205,-1.0977783203,-0.8176969886,-0.2230648398,-1.1865390539,-1.0723834038,-1.0440925360],[0.9188759327,-0.1689279526,0.3107523918,-0.1185576096,0.3864873350,0.5850405693,-1.3451581001,1.1591099501,-1.7213473320,-1.1616033316,-0.3630917370,0.9260309935,1.9034808874,0.7039475441,0.8226540685,-0.2596986592,1.2070049047,-0.9120343924,-0.1701831520,-2.9533367157,1.2373387814,0.7368846536,-1.1026964188,0.7084195614,-0.5575756431,0.6083027720,-1.9660063982,0.0480993725,1.7596119642,-0.2430192381,0.5924292803,-2.4542522430,-0.1702857912,-0.8102533817,0.3578571081,-0.3168414533,-0.7957355976,0.8105921149,1.2186963558,0.2567883730,0.9143864512,-0.0124621931,2.1438231468,-0.7441107631,-1.0311625004,-1.0583885908,-0.3230136037,-1.2296295166,-0.5679237843,-2.8217861652,1.2539693117,-0.6278217435,0.2693663836,0.5483108163,-0.1207809523,-1.0742874146,-0.6069313288,1.5203852654,-2.7174751759,0.3162423968,-0.9960474372,1.1332710981,1.0375409126,-0.0346815698,-0.2378142923,-1.8274402618,-0.7565240860,0.3035017252,0.6736583114,-0.3378650248,1.4695996046,0.8090537786,-1.8708773851,0.9925187230,0.3028483391,1.4664566517,-0.6074188352,1.0674394369,0.1149998233,-0.0621099845,-0.1732854992,1.2566143274,0.4596324861,0.1921480447,0.6472542882,-0.0070908745,-1.1495875120,-1.5454713106,-0.8810761571,-0.9932910800,0.4973982275,-0.0830497742,1.3150237799,-0.8044264317,0.3923761845,-0.8494360447,-0.5066404343,0.4646987021,0.1593044102,-0.0949085504,0.2508442402,0.8306478858,1.5014774799,0.6009095907,-0.2514753044,0.5792323351,3.3733355999,-1.7200666666,1.0272295475,-0.0590384156,-0.4829144776,-0.7770543694,1.1213577986,0.2282679379,0.3859090507,-1.0854517221,1.3076217175,-0.0168237090,-0.7895203233,-1.0785923004,0.0915632099,0.9873991013,0.6320911050,-0.6186110973,0.6905904412,0.6880133152,1.5512328148,0.9121033549,0.5840374827,-0.4260689318,1.5989313126,1.5102443695,2.8731038570,0.1307470798,-0.0351151600,0.4154373407,1.0803951025,-0.7689779401,0.2349200249,0.2127694488,0.1637901515,0.2540091574,-0.5789597631,-0.2085085064,-1.4617660046,-1.2019745111,1.3285419941,-0.5355551839,-1.0052437782,0.8944669366,0.6750456095,-0.3523706496,0.5630317926,-0.2512718141,1.2605104446,-0.1540245712,-0.7847375274,-0.5049604177,0.5393610001,-0.1454569995,-1.9208295345,0.1988556087,-0.5276432037,-1.9898313284,1.3303182125,1.1966954470,0.8466907740,-1.3549520969,0.1534716338,-1.6951408386,-1.0888959169,-0.7846040130,0.0208831783,-0.5135362148,0.1377027631,-1.1420041323,-1.0348410606,1.0945382118,-1.6735677719,0.8930306435,1.2586365938,-0.4186193049,-0.5548886657,0.7520841360,-1.2721277475,3.2311539650,0.4641966224,-2.0481486320,1.9270210266,0.4723309278,0.3861116469,-0.5980855823,2.4221987724,1.1139601469,0.7150088549,-0.5016779900,-0.2652854025,-0.5569392443,0.5937147141,0.7529249787,0.9008342028,0.4063133895,0.1512439102,1.2978001833,0.2491936237,0.2895989120,0.8108627796,-0.2134086490,1.6224154234,-1.1592159271,0.2234223038,0.1212291792,-0.9138811827,-0.4205074310,1.5794224739,0.1617780775,-1.6059589386,-0.4434491992,0.6146141291,-1.1083737612,-0.3466961682,-0.4387931228,-0.2230413556,-0.1678518355],[-2.6264977455,-0.1727611125,1.0355259180,0.3443305790,0.5022798181,-0.5143413544,-0.4986781478,-0.8272845745,-0.0147204055,-0.5727688074,0.0031921251,0.3674898148,2.0267584324,0.5630730391,1.0627936125,0.2569692135,0.3916804790,-1.0073970556,1.2933391333,0.0424646772,0.5398423076,-0.5044786334,-0.5983367562,0.7175628543,0.4539347291,-0.8410576582,0.2607088089,-0.2576574683,0.1949142963,-0.2623168230,0.6356629133,-0.1043076217,0.1324877888,-1.0602743626,-0.2008478791,1.2160608768,-0.0292315781,-0.1890411377,-0.8686054349,-0.9382474422,-1.0121945143,-1.4328993559,1.1785588264,1.5638674498,0.1427608132,2.0439651012,0.9626225233,0.2164034098,-0.3151763380,-0.4807561934,1.9710719585,-0.4390293062,1.7081115246,-0.3730249107,-0.8752344847,1.0402947664,1.0650632381,2.2234606743,-0.2990089655,0.1338202506,-0.0058257026,0.8717163801,0.4131239653,1.7113682032,-0.3095157444,0.4589198530,-0.6647123694,-1.9829771519,0.2398172468,0.7303733230,0.0065799267,-0.2386463732,0.7785472274,0.1026007161,-0.9697332978,0.9232811332,-0.3040901721,-0.3353936076,0.4633235931,-0.3067607284,-0.1129981503,0.0151031548,1.4305889606,0.2822506726,-0.3901175261,0.2373402417,-0.3301363885,-0.4907036722,0.3101215363,1.6566424370,-0.1352761388,2.0479047298,-0.8596891165,1.3939336538,0.5532745719,-0.1697778255,1.0746655464,-0.6602277756,0.4712713957,0.8705766201,0.2959489226,-0.1473409534,-1.7874796391,0.4816951752,0.0301115476,-1.0984210968,0.3432916701,-0.6739798784,-0.8314334154,-0.0407379083,0.4170181155,-1.1367658377,0.7947146297,1.3205595016,0.4236109555,1.4152296782,-1.2873961926,0.5734673738,-0.1166800484,-0.4258343875,0.7109622359,0.9978282452,-0.7153664827,-0.1323070973,-0.1627074182,1.2122015953,0.7887623906,-3.2201621532,-0.1679748148,1.3945446014,0.7399389148,-0.2768034637,0.4165839553,0.8265034556,0.9802414775,0.4776431918,2.0166103840,-0.0249875244,0.4829532504,-0.7448346615,-2.0533394814,0.9954663515,0.5559939742,0.1557032168,1.2560248375,-0.0910743177,0.7448199987,1.5151149035,1.0682363510,-0.3936732411,-0.4404679239,-1.2789002657,-0.7345156074,-0.8073781729,-0.2055337727,0.8966993690,-0.0118007520,-0.7772536278,-1.2498084307,-2.0847239494,0.4443413317,-0.9075754285,-0.0707240850,-0.3365614414,1.1858187914,-0.7117362022,0.9870203733,-0.6684255600,1.4725475311,0.1366336048,0.7424795628,1.4361493587,-0.2917276919,-0.3563250899,0.6698901057,0.8398872614,-0.8428919911,0.7355796695,-0.0806103125,-0.4388633668,0.4438080788,-0.6025252342,0.1011375487,0.4766345918,-2.1126976013,-1.3606692553,-1.0401051044,-0.0643365011,1.7286696434,0.7811232209,1.2519133091,0.1582637727,-0.1840040833,-1.0580223799,2.1860694885,0.7420462370,-0.6358695626,0.3461574614,0.0260392353,0.2544353902,2.4137811661,0.5811787248,1.4372696877,-1.1470407248,0.0356767289,-1.9682257175,-1.6825907230,0.7045783997,0.9582905173,0.4860132933,-0.4299276471,0.4619325399,-0.3726094365,-0.4181045592,-0.6713925600,-0.4985078275,-0.6091927290,-0.6080089808,0.3473231494,-0.0035682917,1.2791190147,-0.2127529532,1.0742793083,0.7116464376],[-0.6488238573,0.2859148681,-2.0417239666,0.4539825618,2.0291399956,-0.5095365644,-0.9993411303,-1.1150147915,-0.6259767413,0.1875912845,-0.2886156142,1.0134998560,-1.1804151535,0.1758203655,-1.2308334112,2.5587472916,-0.9636149406,-0.0713531971,1.5979670286,-1.5108559132,0.1449113637,1.1441174746,0.2312457860,0.2274304181,0.1459915936,-0.6081528664,0.0709826797,-0.0125633776,-1.3180460930,-0.3260259330,0.4805007279,-3.2030875683,0.4306159019,-0.5212286115,0.8575561643,-0.8960441351,-0.1932385713,-0.7063098550,-0.9964183569,0.7934882045,-0.4802501500,0.3568688333,-1.0943177938,-1.6923885345,0.7779947519,2.6947827339,-1.0811645985,-1.4731924534,0.0109302318,0.3299140036,-0.1448013633,2.4050495625,0.1636990309,-0.1965565681,1.5348142385,0.2748448253,-0.9842527509,-0.4361257255,-0.5287380815,0.7196028233,0.2204200625,-0.5418440700,1.3916079998,0.7451648712,0.2643483579,0.6969248652,0.2191476822,-0.3153337836,-1.1294671297,0.2236620039,0.8282073736,-1.0530409813,-0.4204666913,-0.2162797153,-0.7607799172,1.0307723284,0.7421389818,0.4817281663,0.3772906065,-0.2255851179,0.2419869155,-0.2462873459,1.2878042459,1.2064702511,1.9481141567,-0.3638069034,-0.2458624840,-0.8163990974,-1.3361914158,0.1759304553,0.8073284626,0.2768845856,-0.1631110460,0.7782697082,0.2213267684,0.6580312848,-0.8218443394,-0.5532718897,-0.5647550225,-1.2183798552,0.5782849789,0.5329989791,-0.6715250611,0.8244873285,1.0194652081,-1.1014153957,1.6967984438,-0.0429516323,-1.0793919563,-1.1583365202,1.1959785223,0.8431320190,-0.4192778170,-0.3327940702,1.9657697678,0.3050765395,0.8305168152,0.9101333618,0.5969888568,-0.2304041535,1.8251065016,-1.1460005045,-0.2217972875,-0.5881155133,-0.4704960585,0.4888032973,1.3412656784,-1.0570495129,0.4420121312,1.2955129147,0.8408915997,0.7732490897,-0.6764742732,-0.0356562771,-0.6429589391,1.8658261299,0.5035269260,1.2822653055,0.9182057381,0.4520080388,-0.5673279762,-0.4225480556,-1.2345860004,0.9104699492,-0.6104401350,0.5325096846,0.2594264150,2.1227545738,0.6246936321,0.3436859250,0.4797151685,0.5576152802,0.4796886742,0.6413477659,0.1015166044,-0.5291261673,0.0093640964,1.1927584410,1.5470068455,0.9478160739,-0.8584091067,1.4589502811,1.0034278631,-2.0495164394,1.0458933115,-0.6270156503,-1.6971756220,-0.4960333407,1.4561544657,1.0882210732,-0.0976122990,-0.7173095345,0.1641712338,-0.6962606907,-1.3123685122,0.7208009362,1.4251564741,-0.9525543451,-1.6627132893,-1.3450037241,0.9273759127,0.3668547571,0.0847202390,-0.8547070026,-0.1450030655,-1.0790885687,0.6394917965,-0.7484422326,-0.0812485516,-2.3238573074,-0.8937191367,-0.2410787642,0.7658015490,-0.1973870993,0.1931478530,-0.5485792756,-0.3950333297,0.3372679055,0.7965800166,-0.1148310974,1.0055104494,-1.7035870552,-1.2542206049,-0.0020835998,-1.0969371796,-0.6635608077,-0.6708840728,-0.1306800544,2.0890042782,0.2768505216,2.3234868050,-0.1149376929,1.7064332962,-0.0357143879,0.0917023197,0.2306758016,-0.8789747953,0.6359471083,1.0145800114,-1.2079186440,0.7884379625,0.5274050832,0.1402386427,0.3174589276],[-0.5563088655,-1.8163865805,1.6633933783,-0.1732443124,0.1822638363,-0.8089761734,0.6185746193,1.4769704342,-0.8918659687,1.0805387497,-2.2777967453,0.0982830897,-0.9784666896,0.1409724802,-0.6910139918,-0.2581627965,-0.3227139115,1.2393940687,1.4970542192,0.3486480415,0.6801899076,-1.2906173468,-0.8694254160,0.6916615367,-0.3949315548,0.4967114031,-0.9639406204,1.0947629213,-0.5677868724,0.1172745824,0.1036742851,0.8766137958,-0.5469971895,0.0912009105,-0.8483621478,-0.1651878506,-0.8982597589,1.8080214262,-0.1094860435,-0.6009634733,-0.3665615022,-1.1674404144,-0.3560737967,0.4168348908,1.2016643286,-0.5615233183,0.6447062492,-0.2535068691,-1.1192104816,1.1256830692,-1.6240452528,0.9639360905,0.4438114166,1.8228126764,1.9016685486,1.3340772390,-0.9280285835,1.7382543087,0.5836077929,2.3370118141,0.5108041763,1.4469654560,0.4597733915,1.0623366833,0.0035629983,0.0866807178,-1.3463780880,1.1956572533,0.3716119528,0.0912752375,0.5642582178,0.7687278390,-0.0072249584,-0.4405217469,1.2104653120,0.0289006699,-0.6262733936,0.2481508404,1.7004103661,-1.0955580473,-0.9726043940,-0.9737624526,0.2870694399,0.5214788914,1.4835518599,-0.2087257504,0.0239505731,0.2233554721,0.2403004915,-0.8426833749,-1.2004982233,0.7131180763,-0.6364238858,2.0007157326,-0.3677787781,0.5677682161,-0.5169160366,0.7849178910,-0.1691215634,0.3505437374,1.2626199722,-0.3145025969,0.0967818573,-1.1018123627,2.0346674919,-0.5044198632,2.0918335915,0.5418912172,0.1253499985,-0.4790270329,0.9885153770,-0.2244774550,-0.7532069683,-0.2590924203,-0.3724360764,-0.5268012881,0.0099415304,0.9852399230,-0.9677956104,0.9610522985,-0.4307636321,0.9771789312,0.4913146794,-0.1979523748,-1.2591580153,0.1725802869,0.3992780149,0.2902160883,0.6774837375,0.1915705055,-0.5545125008,-1.6164081097,-0.4995488822,0.1495799124,0.7770783901,-0.6658815742,-1.3216475248,0.7363036275,0.4968822896,0.5971414447,-0.6678465605,0.1934313029,0.4205310643,0.5923535228,0.0337040164,0.4831328690,-0.3680884242,0.3568235636,-0.3181792498,-0.8712489605,-0.8680101037,-0.5461862683,0.6563332677,-0.3947722018,-0.2092790902,0.0315701514,-0.4167560935,0.0570355020,1.4990587234,-0.8132793903,-0.4818907976,0.4890195429,0.6042798758,1.5244840384,0.4387151301,0.3640931845,1.5731611252,0.5005360842,1.0017102957,-0.4325468838,-0.7100720406,-1.3056461811,0.3504725397,0.2902453542,0.3730077446,-1.8038679361,0.3144704103,1.1459712982,-0.9655853510,0.4122045934,1.5242187977,-0.2994287908,0.0495324992,-0.8841370940,-0.9477353096,-0.5417408943,-0.0676480904,-0.4663005173,0.9185313582,0.0686773732,-0.6691791415,0.3122829497,-1.1253046989,-0.1556600630,0.5729102492,-1.0837713480,0.0669136345,0.4971903265,0.0897533596,-0.1047038808,-0.6622824669,-0.7894172668,0.8324003816,0.7106827497,0.3483260870,-0.7259815335,0.6386246681,-0.5196985602,0.5204224586,0.0865338221,-0.7810475230,-1.2443351746,0.3589504063,0.4397775829,-1.1927919388,0.9172086120,-1.2508444786,1.7453820705,-0.3246877491,-0.4532547295,-0.0731760189,-1.3975434303,-0.7634076476,1.5434905291],[-0.4080433547,0.4650013745,-1.4238005877,0.2563230693,1.7736479044,-0.7137615681,2.2522010803,-1.3089007139,1.1525106430,-0.1405597478,0.5320132971,-0.6810653210,-0.4485176206,-0.3747822344,0.4281465709,0.8007715940,0.0123932073,0.1402721852,1.9548217058,0.3309944570,0.3134448826,-0.8624955416,0.1731562018,-1.1307023764,-1.2180870771,0.3798313737,0.1041512787,-1.0410223007,-0.2304405719,0.0916122720,-1.1846817732,-1.1531682014,-0.4354380965,0.4104143679,-0.4816984534,0.4900153279,0.7041178942,-0.7546210289,0.5832744837,1.2078104019,0.4386845529,-1.5632371902,-0.8126469851,0.1477559507,-0.0461850464,-0.3673082590,-0.4501196742,-0.1551027894,-0.7583582997,-0.0730031282,-1.9887934923,-0.8846876621,0.6732758880,-0.0811193660,1.0592772961,-0.5077518821,-0.3238532543,1.3735461235,-1.4465440512,0.1821355820,0.0676640049,-0.4560262859,-0.3496088684,-0.6075907946,0.6537126303,-1.7022186518,0.0755707473,-0.1904866546,-2.1203536987,1.7095605135,0.7199447155,1.7510561943,0.8169274330,0.8673193455,-1.2237163782,2.6919562817,0.3033113480,0.2355719060,0.4901683927,-0.7814581394,0.8745526671,0.2152803242,1.0555243492,0.3528210223,0.3482930064,-0.0836526528,-0.8183871508,-1.3136165142,0.5067337751,0.9809406996,-0.3727931380,0.1372788996,1.2173398733,1.0570574999,-0.1092743352,0.4346898496,-0.7528158426,0.8488703370,-0.8521622419,-0.8124353290,-2.1803562641,-0.7766513228,1.1283991337,0.3751853406,-1.3530814648,1.3838658333,0.4173292816,0.7445374727,-0.0002430229,-0.9045234919,-2.6463234425,-0.3570845723,0.2908712029,0.3473463953,0.5702397823,0.7549381852,1.4791334867,0.9874882102,-0.6921150684,2.3449747562,0.6544520259,0.3682841957,1.9040485620,0.4765006304,1.1654456854,0.7307538390,-0.4611986578,0.6390597820,0.3741174340,-0.5228999257,1.6194049120,-1.0346127748,-0.5952832699,-0.8493008614,0.8470287919,-0.5518957973,0.2897983789,-0.0187721644,-0.3662812412,-0.3504948616,-0.3711920381,1.7615973949,-0.4802156985,-1.8253751993,-0.1795722097,-0.3333802223,0.4528505504,0.7250844240,-0.1168686971,-0.2354764342,-1.0015610456,0.5932791233,-0.7151992321,1.1872414351,-0.6099489331,-0.8342801929,1.0395519733,0.4641788900,-0.1812096685,-0.2494318336,0.1936044991,-0.3829216361,-1.2802710533,-0.0509898625,-1.1535187960,2.3395931721,0.9372795224,-1.0627675056,-2.2381188869,-0.0003268473,0.1890586317,1.9689356089,0.4381758869,0.6209080815,-0.0069314591,0.6565827131,0.9101958871,-0.8087453246,-1.0782167912,-0.4138877690,-1.1793274879,0.3645130098,2.4339489937,1.2218075991,-0.5169935226,0.3686336875,0.3838019669,1.0012456179,0.0827877894,0.6399840117,-1.1393259764,1.1500494480,0.1755813956,-0.4248540998,-0.9612060785,1.2496811152,0.6430417895,-0.8739003539,-0.6526424289,1.1801552773,-1.3901848793,0.6684985757,0.7047442794,-0.0454604439,0.9043201208,-0.0832923055,-0.0246616453,-0.7082564831,-0.6657384038,1.1625177860,0.7331817746,-0.8065531850,3.0549848080,-0.6254408956,-0.9780409336,0.3719343543,-0.7863800526,0.2642794549,2.1103971004,-0.2257129401,-1.3434938192,-0.6517789364,0.9215201735,-0.5927384496],[0.9416487217,0.0437185653,-0.3107615113,-1.8002775908,-1.5352293253,-1.2877420187,0.2497640550,0.0889306590,-0.3507105410,-0.5227136612,-0.5778329372,-0.7980781794,0.2110481560,-1.3815633059,-0.6894747019,-0.2972003222,0.8221981525,-0.2503884733,0.7942054272,0.9488335848,-0.1884921640,1.0855621099,-0.6987811923,-0.2730012834,1.4528568983,-0.6969278455,0.5462369323,0.4251119494,1.1552058458,0.3428002000,0.8982730508,-1.4678502083,-0.5434156060,-0.0516024604,1.3498048782,1.7462068796,1.0836268663,0.8198674917,-0.4759975374,-1.0495749712,-0.9996607900,0.3143595755,-0.8451169133,0.9353263378,0.9249797463,0.2017759383,-1.9286822081,-1.1570881605,0.5033632517,1.1476683617,0.2429254949,-0.6879550219,0.9943078160,-0.2067762166,0.6849880219,1.4044768810,-1.4275686741,-0.8624293804,0.7384086847,-0.5561550856,-0.8264702559,0.7136894464,-0.3004029989,0.8249571323,1.3415811062,-0.1680783927,1.0953153372,-0.8835517168,-1.0230761766,-0.2327672988,1.2700972557,-1.1392649412,-0.4342262745,-0.7223634720,0.5530021787,0.5501202941,0.3454295397,-1.0601592064,0.3098625839,0.3677841425,0.9607761502,0.6529376507,1.4435209036,1.0172544718,1.3574310541,-0.0099963387,0.0024282169,-0.4879335463,-0.4324880540,1.4777532816,0.2670559883,-0.4485886097,0.6798025966,0.6807366610,-1.4864747524,0.2098133266,0.4501485527,1.2317266464,-0.2362726778,-0.4584268034,2.8149886131,0.1683895439,0.2099463046,1.2645759583,1.2813298702,-1.3021427393,-0.0147156315,-0.4178023934,1.9107762575,-0.6563686728,-1.0691428185,-0.4230622947,-0.0132462140,1.8421353102,1.7196012735,0.5972613692,0.3515547812,-0.5387497544,-0.0692220926,-0.3597345054,-1.0044300556,0.5427931547,-0.8332656026,-0.5242043734,0.8488559127,0.1747637987,-0.3172430992,-1.2048152685,0.6520953178,0.4357360303,-0.5587982535,-0.4179782867,0.4264180362,-0.0548314825,-1.1908137798,0.4734176993,-0.6122033000,0.4030275941,-0.2191144228,-0.3238929510,0.6194945574,-1.2550044060,1.5398229361,-0.4604188502,0.7527544498,-0.3760625124,-0.6907941699,0.6619018316,0.3901435733,-0.9779542089,0.2812705040,1.3486820459,0.8936386704,-1.0297094584,-0.7766075730,0.5017185807,-0.9354196787,-0.4815663397,-0.0947765708,-1.3963850737,0.2590472400,2.3648326397,-1.9255220890,1.3736010790,-0.2367615402,1.2566763163,1.9916495085,-0.5050504208,-0.6587370634,0.1339435875,-1.5667083263,0.4359713495,0.0420700498,0.0357943885,-0.0433804691,2.2872498035,1.2421827316,-1.6219403744,0.2979706526,0.5869822502,-0.6416541934,0.3410756886,-2.2526934147,-0.8089647293,0.2043059468,-0.1011173427,-0.4876793921,-1.3751064539,-0.2896009982,-0.3614101410,-0.4906738997,2.8373856544,0.2476367205,0.0999422297,-0.9083200097,1.2840486765,-0.9753172398,0.0642992556,-0.8871639371,0.3426522911,-2.1190073490,-0.7735089660,-1.3035955429,-1.9201433659,1.1583014727,-1.3426201344,1.3124009371,0.4313886762,0.0344963595,-1.2075973749,1.2700092793,0.6499128342,3.3665959835,-1.6459656954,-1.3581641912,-1.8216447830,1.4461368322,-0.1574293673,0.1464605033,-0.7976961136,0.1109806001,0.1583217680,1.0877591372,-0.0723408088],[-1.0505943298,0.2393178642,-1.4363778830,0.3752524555,1.2342381477,-0.8808445334,-0.2301429659,0.3931305408,-0.4250062704,-0.5529979467,0.6862195730,2.4019544125,-1.3151100874,-0.2301423550,0.4676499367,0.5954039693,0.4348103404,-0.0355928615,1.3828746080,-0.2989363372,-0.8919566870,-1.3971213102,1.7783776522,-0.2302737534,-1.7624164820,-0.6463377476,1.1267231703,0.4442473948,0.4791038632,-1.4811384678,-2.1068751812,0.5931427479,-0.1246527433,2.1817100048,1.6543698311,-0.1731125265,-0.1042543724,-0.0780078098,-0.7192997336,-0.4809856117,-0.1577761769,0.4041523635,-0.0076569053,1.1993273497,-0.3402960598,0.7366626859,1.3511939049,0.4218282104,0.1720520854,1.8980066776,0.2961442173,1.1082334518,-0.7020488977,1.1318085194,-0.6802171469,-0.6433147788,-1.0897530317,-0.1884802282,-0.8276434541,-0.7988906503,0.3866085112,-0.0302876048,1.0736109018,0.0648761988,0.8674221635,0.4764135778,-1.1911543608,0.7603228092,1.4816516638,-0.8416389227,0.1508948207,-0.4492126107,-0.5611734986,1.6207737923,0.8324872255,-0.5176488757,0.5176472068,3.0010211468,-0.5873510242,-0.7627546191,0.7660671473,0.6905959249,-0.2636062503,-0.4258366227,-1.2875888348,1.4470908642,0.0458003469,-0.5081302524,1.5856559277,-0.5258768201,0.1839931756,-0.4267107844,-0.7687863708,-1.9748845100,-0.7904258966,-0.3710695207,0.2210629135,0.1301920861,-0.8979573250,0.2273373753,0.7875413895,0.7730968595,1.2694029808,0.2548620105,-0.8165200353,2.0619447231,0.0528843738,-1.4856121540,0.5538668633,0.4425628781,-0.2612531483,0.2037329376,1.2159631252,-0.1642043442,0.4371479452,-0.2363016307,-1.6688332558,-0.8167054057,0.2186635584,-0.2214595377,-0.2448555678,-0.3115261793,0.9378111362,0.1662903726,-0.6850562692,-0.8621481061,-0.8770141602,0.2879326642,-0.7475222945,-0.7961471677,1.0106245279,-0.2903393209,0.1513967961,-1.1147676706,-1.2387049198,2.6652097702,-0.1084619164,1.3810008764,0.6159277558,-0.5572487116,-0.4658169150,-0.2820259333,-1.1114195585,-0.3663528562,-0.4245306551,-0.1668689102,1.3447589874,0.0826546401,0.5931445360,-2.0726764202,-0.6349374056,0.1409984976,-0.8965637088,-0.2465610951,-0.7543249726,1.3068647385,0.5193663836,1.2673950195,0.0530828945,-1.3074204922,-0.9408559203,-0.3132842779,-0.3520522416,0.2639456987,-1.6094528437,0.2965346277,1.1979731321,-0.6135300398,-1.2268162966,0.1642137766,0.9970766306,-1.3188539743,1.0603978634,-1.1148290634,-0.0663175508,-0.7761843204,-0.8369712830,1.0939637423,-0.1550624073,-1.6109610796,-0.0460047275,0.6645396352,-0.2567320764,0.0598233193,0.7579881549,1.9713249207,0.3284783065,-0.5257778764,0.9011698961,1.1173251867,-1.0251053572,-0.0538991652,0.2657756507,0.6858976483,-0.0486010723,0.1332474202,-0.5991258025,-1.6124672890,0.0241628140,0.0723342896,0.6667144895,0.3458332419,-1.0246504545,0.0992837548,-0.8964792490,0.0723521560,0.2398944497,-0.7628162503,0.4543907344,2.3749954700,-0.1462941468,0.0003235369,1.0902603865,0.5219897628,0.3533296883,-0.7274948955,0.3543509543,0.3556120694,0.8606618643,-0.6099217534,0.6097067595,-1.0419270992,0.1097846627,-0.1417976171],[-1.1922610998,0.0365081467,1.3653956652,-0.5518307686,0.3183691800,0.1170945540,-0.1883369386,0.9821451306,-0.0275580082,0.6468264461,-1.5413923264,1.6071134806,-0.9581078291,0.7696560621,-0.1862657368,0.4136942625,-2.6068975925,0.8061500788,-0.7720109224,0.1059482396,-3.1292214394,0.0776122808,-1.3263397217,0.3734543025,0.9427181482,0.1115776300,1.9515994787,0.7113924623,-0.2507297397,1.5339950323,-1.1204895973,0.3802853823,0.5562388301,0.4393317401,0.3805435896,-0.1796236187,-0.0068756798,0.3063318431,-0.8758431673,0.0523967259,-0.9203234911,0.4118686318,-1.1167979240,-0.8827121854,1.2052417994,-2.1289260387,0.5620676279,0.9299391508,1.7372515202,0.9391069412,0.1322578937,1.6648646593,0.6191270351,-0.4869742095,0.0202155150,-0.0032952912,-0.7536495328,-0.0577502102,1.4988473654,1.1853531599,-1.1569428444,0.4523760378,-0.4761290252,0.4584850967,-0.0814426169,1.1522524357,0.5754417777,0.9752656817,-0.6391966939,0.8959468603,-0.5191081166,0.4095037282,1.2617907524,1.3778895140,0.9799590111,1.6725335121,-3.0943217278,0.9498860240,0.8754180074,-0.3936840296,-0.4376997054,-0.2135491520,-1.3722800016,0.3389026225,0.1849378496,0.1413071156,0.1889852732,-0.9622039795,-0.6044231057,0.8672102094,0.0209429972,0.6589286327,-0.1708002537,0.1002857089,-0.8388501406,-0.1158685908,-1.9335631132,-0.2489332408,-1.6315416098,0.5039172173,1.1015135050,0.5260147452,0.3572916090,-1.0680633783,-0.7171474695,-1.0039732456,-0.6688591242,-1.7563149929,0.3410589695,-0.0830375180,-1.8177490234,-0.1606295109,1.2918634415,-0.8302479982,-0.1817802340,-0.2532959580,1.9223433733,-2.0590584278,-0.7776770592,1.4901905060,1.2545901537,0.8289508224,0.1364070326,0.0048171692,0.4144740701,0.1384837925,-1.2213534117,0.4205737710,-0.1566273570,-1.3302257061,-1.0808893442,-0.3223316371,2.3164737225,-0.6410893798,0.7436528206,-1.2338428497,-1.8407684565,-0.9513223767,-0.5103200674,-0.5803955793,-0.3061780930,0.4369329214,-0.9844542146,-0.3322239220,-0.5980868936,2.0401651859,0.7489195466,1.0855104923,-1.0209662914,0.9438039660,1.3219096661,-0.7857168913,-0.7631080747,0.3792274594,1.9473079443,-0.4503182769,0.3687690496,0.5508050919,-0.0530326739,0.3569258451,-0.2823581696,0.1155722141,0.0433711931,0.0135188764,-1.3273210526,0.2165891230,-0.6997963190,-0.0507710278,-1.7662363052,1.1606320143,-0.0638204664,-0.1239236444,-1.5254353285,0.8884639740,-0.0218050219,-0.4060950279,1.0596505404,1.6093878746,0.6677417755,-0.8113313913,0.2368712276,-0.7918953896,1.3205071688,-1.1475688219,-1.0701115131,1.3642249107,-1.0599621534,0.0186037924,0.5424012542,-0.7976353168,0.5737085938,-1.7811143398,-0.8941566944,0.6157245636,-1.4740045071,-0.0339163728,-0.2008125037,0.4697420597,0.7060800195,0.3085568547,1.4709632397,-1.4332449436,-1.3343772888,-1.9902968407,-0.3875651062,1.7211320400,1.4731040001,-1.1057852507,0.8125382066,1.5489228964,0.2021583468,2.3115398884,-0.4626514018,-0.0988822505,-0.3713912368,-0.1346644759,-1.1158742905,-0.1639193296,0.0890217721,-0.4499882460,-1.1167421341,0.4916646779,0.2613768578,0.2538814545],[0.2545026839,1.0541639328,3.0064599514,-0.7094870806,0.7198755145,0.1062187478,-0.7458360791,1.3711556196,0.9849750996,1.1028059721,0.2148303241,0.2942796648,0.3881600797,-2.2538924217,-0.0313082747,0.0384150855,1.4320955276,-0.9805296063,-0.8168244958,-1.0708725452,-1.3232696056,-0.0492793471,1.1589303017,0.9985959530,0.1406734884,-0.2315083295,-1.3020762205,1.6821084023,0.0845298097,-0.0012636767,1.3425643444,-1.1528156996,-0.4846612811,-1.1822725534,-0.6761192083,-0.0896490365,0.0943385810,0.3186304867,0.7720057964,-0.2023819238,0.2094048411,-1.2419801950,-0.3391025066,3.1322660446,0.1629785895,1.3776082993,-0.4834138453,-1.4403148890,-0.9649205208,0.6832093000,0.3958668113,0.5880941153,0.0307263527,-0.8174967766,-0.0751677454,0.4808998108,0.1181297526,0.2792820930,-2.1766908169,0.3593552709,1.2126170397,-0.7078419924,-0.1993245333,1.2566301823,0.3003346920,-0.4112148881,0.4869786203,-0.0476831608,0.6511651874,-1.5540522337,-0.4369316995,1.0390468836,0.9457051158,0.2111051232,-0.3961375952,0.9423094988,-1.1002054214,-0.7350199819,0.0146141937,0.5745733380,-2.6855244637,0.5434151292,-0.1790276021,0.7951074839,-1.7326436043,0.0923662931,-0.5095089674,0.5917858481,0.8422880769,-0.1820607781,-1.6822385788,0.9818657041,0.5777400136,-0.7624858618,-1.0306129456,-0.2130795270,0.9831990600,0.8381908536,0.4186193347,0.2878623307,3.3586564064,0.6768281460,0.7891095877,-1.5845159292,0.6934824586,0.1463712752,0.1077918932,-0.0776537731,-0.9601588845,1.5597449541,0.2852105498,1.1821978092,1.4600056410,-1.0947434902,-0.5802459717,-0.3190021217,0.2932513952,-1.2169111967,-0.1552423239,0.2548938096,-0.6420126557,0.9541916847,-0.4548354447,0.2853848338,1.0490132570,-0.1054982617,-0.8711935282,0.0640761554,2.0955986977,2.2802312374,-0.7062234282,-0.9325771928,-2.7064745426,0.5550458431,-0.8726333976,0.8819828033,0.4131059051,-0.4404965639,0.5863825679,0.9154740572,0.2949877381,-2.6654002666,1.2274175882,0.1260483563,1.0107161999,0.4425277412,1.2145001888,2.3537797928,0.3502727151,0.4897087514,-1.4731489420,-0.6829451919,0.7784183621,0.0769444928,0.8899565339,0.6568589211,-1.4877281189,0.8486710787,0.2527044415,-1.2365397215,1.8013707399,2.1016540527,-0.0500311032,-1.3022519350,-1.9501366615,-0.5360963345,0.3994601369,0.5160616636,-0.8235284090,-1.7060825825,-0.7978700995,1.7503622770,-0.1441345215,-0.9500775933,0.5486234426,0.1444375366,-0.7123869658,-0.1756013632,0.1871870011,1.1465924978,-0.3897903562,1.8086135387,1.1280059814,-0.0083743073,-0.3285886049,0.2197051197,0.7689400911,1.0436525345,0.2513706982,2.6073219776,-0.3112938404,0.2724854648,-2.2912137508,0.2034903765,1.1328512430,-0.0851072371,-0.1136390269,-0.2366341352,1.2237306833,-0.2443148643,-0.1149372980,0.8067960143,0.7126696706,-0.3885196447,0.8958705068,0.6093592644,0.7420414686,-1.1293281317,0.0535399579,0.1800847501,0.9242040515,1.0947157145,-0.4837334454,-0.8320499063,-0.0283501279,-0.7321954966,-0.4648154080,0.5936284661,-0.1790808439,0.6123202443,-0.3115934134,-0.6220819950,0.3708849251,1.4454218149],[-0.1824720055,-1.4103887081,0.2352046967,1.5774737597,0.8297368884,1.1899679899,1.4147301912,-0.1582700610,-1.1319595575,-0.5135700703,0.7510833144,-1.5830452442,-0.6762850881,-0.6799480915,0.4081239104,0.8614497185,-1.4354953766,-1.4303246737,-0.3100005388,0.6109189987,0.3791231215,-0.7022027969,1.0591596365,-1.4798945189,-1.0763273239,-0.6885050535,-0.5565457940,-1.9920260906,-0.5430017114,-0.2988832593,-0.4576859176,0.6534484625,0.0158297122,0.0252145696,-0.5811895132,1.6511220932,0.0293052960,0.8481232524,1.3957945108,0.9147311449,1.9018837214,0.0225104298,-1.0985888243,-1.5755480528,-1.9768010378,-0.4817121029,-1.4537837505,0.7854406238,-0.3301056623,1.2437915802,-0.0110594444,1.3535820246,-1.1919013262,0.6878582835,0.4309915602,0.0344051160,-0.4072600603,0.1183931530,-1.1715111732,1.1703085899,1.4365798235,-1.1344217062,-0.5182979107,0.8712841272,0.8400304317,1.4912987947,0.3017890751,0.2879858613,1.5178281069,-0.3468378782,-0.1891960204,0.3158127964,0.2165258229,-0.2097465992,0.6558660865,-1.6493350267,0.0418719389,0.2596367896,-1.1072080135,0.1650775224,-0.6635113955,-0.7450125813,2.1797268391,0.7675815821,1.6386095285,-0.7342829704,-0.8826609850,0.8525941968,0.0779680610,-0.2251982987,0.5225286484,1.7051614523,-1.0837510824,-1.2669957876,0.1919908673,0.8322687149,-0.7541927695,0.6820940971,-0.0351917483,-1.4053186178,0.2227966338,1.3239190578,1.3002049923,1.4524662495,0.3714876473,0.2805170417,-0.0919749141,-0.5110696554,1.1964526176,-0.0531565435,-1.1680847406,-0.6295111775,1.6872906685,-2.5313727856,0.2121602297,-0.8588676453,0.5005645156,-1.4779620171,-0.4762757719,-0.2537456751,0.5073809624,0.6816993356,0.5533781648,0.9636012316,1.4837093353,1.5566785336,1.2752995491,-1.1811940670,1.4324865341,0.2819110155,-0.4524187148,0.7230597734,-2.5340185165,0.0218055099,-1.2718653679,-2.7177481651,0.5364590287,-0.0088465754,-1.0927875042,-0.2092958838,0.3976587653,0.1919134855,-0.5732107162,-0.0421538688,-1.5942142010,-0.3043152690,-1.3545215130,-1.0652045012,2.5165457726,0.8235344291,-0.8325254917,0.7410484552,-0.8188034892,-0.3710550368,-0.1469221711,1.4208518267,-0.6055439115,-2.3149800301,0.0533712320,0.9889836907,0.5201842785,1.0583690405,0.0227681696,1.2871819735,0.8295145035,0.1761369258,-1.2930803299,0.5198744535,0.3537299931,0.9064239860,0.3577017486,-1.2018020153,1.5393267870,1.1606206894,0.3462671638,1.0494418144,2.0337548256,-0.6492514014,-0.2536161840,1.1055673361,0.7026463747,0.8310782909,1.0652977228,-0.2652146518,0.6976580620,-0.1085451543,-1.5246456861,0.3379697502,0.6745842099,0.6699867249,0.7807105184,0.3559247553,-1.4706202745,0.2427008599,0.1194189042,-0.9471550584,0.5363782048,0.7715194225,-0.7956759334,1.3846803904,-0.0059837541,1.6803729534,0.4307575822,1.3302416801,-1.9971408844,0.9966135025,0.6510790586,0.3689449728,0.3054286242,-0.0868136510,-0.4496328235,-0.4203562438,0.0446175151,0.0464101844,-0.6030803919,0.9469773769,0.8830232620,-0.6825973988,0.0063548535,0.9545045495,1.7176597118,-0.4867136478,0.5350939631,0.2192623615],[1.4326761961,-1.8599370718,-1.3125212193,0.1645046771,-1.1676248312,1.4429554939,-0.5579243302,0.4837940633,0.9947431087,-1.2013341188,-0.4283203483,-1.7186342478,0.0546005592,1.0362206697,-0.9954963326,0.0477595143,-0.0268921070,0.6498132348,0.1365755051,0.1145898923,1.2277548313,-0.7165322304,-0.3180812597,-0.9940687418,-1.9625616074,0.7006590962,-0.0359061249,-0.3734052181,-0.2956706583,1.2970714569,-2.0354444981,1.5242056847,1.4134936333,0.6256355047,0.6216192245,0.5148414969,1.6640155315,0.0376209542,-0.2404932976,-1.3875989914,1.0557792187,0.9804635644,0.2911759019,1.6869058609,2.4566636086,1.6708139181,-0.6846850514,-0.0785990059,-1.9915647507,-2.2253570557,1.7390940189,-0.0221805908,-0.5199558735,-0.4286879599,-0.8465876579,0.1239822507,-0.3726306856,-0.8203553557,-0.0182525702,0.5630733371,-1.8752394915,0.4155042470,0.5997116566,-1.9111266136,-0.1225134805,-0.9004780054,0.5674170256,0.9721233249,-0.5763071179,1.2280766964,0.0869190246,1.5497611761,0.0690125600,-0.3725866973,1.1458171606,0.7421871424,-0.2898759842,0.4488764405,-1.1340010166,-0.6257569790,0.8099257350,-0.5755894184,-0.3711819947,-1.2420692444,0.5750556588,1.2347095013,-1.3155068159,0.5820550323,-0.1800024062,-0.7291352749,0.9901236296,1.6943480968,0.3219739199,0.4525323808,-0.8705807328,1.2545053959,0.0177927706,-0.9325863123,0.5151397586,-0.0098660281,0.4695507288,-1.9551370144,0.1708181202,-1.2006139755,-0.6353746653,-0.3414449990,0.6378657818,1.6013050079,1.3553612232,0.3463516533,0.4996110201,0.5760862827,-0.3864963055,0.6062389612,-0.9762124419,3.0961830616,-0.3332355917,-0.6250987053,-0.3858212531,-0.0016092911,0.1147701219,0.1219455898,3.0642769337,0.9969142675,-0.3090046346,-0.6291542649,1.7868961096,0.3111509979,2.1643211842,0.2989636660,-1.6422638893,0.0548259765,0.0236156844,0.6794649363,0.6361483335,-0.2971331179,0.1430229992,0.3408798873,0.1371335685,-1.1627069712,-1.6395305395,-0.7312899828,0.8405165672,-1.4277316332,-0.6171786189,0.9497544765,0.1589775980,1.0119231939,1.3794178963,-0.4360499978,-0.0422094576,1.5773836374,2.0762181282,0.7507163882,-0.3115755916,0.7139906287,-1.1984947920,1.6259472370,-1.8023258448,-0.2536619902,-0.3082579374,0.0895618051,-0.2826513350,-0.7755914330,0.1211613193,-0.5160586238,0.3996881247,2.6127579212,0.7454584837,-0.5768119097,-1.1571289301,0.2560498416,-0.8254200220,-2.6075785160,0.8396331072,1.7745233774,-1.1578730345,-0.2286801040,0.1026872993,-0.4894656837,-1.0386230946,0.6681470871,1.1531586647,-0.2183658928,0.3109624088,-0.6623362303,-1.8053112030,0.3087073267,1.0570780039,1.0746499300,0.5073927045,-0.8723320365,0.6104628444,0.4474971294,-0.7325591445,0.2552275360,0.4838557839,-1.0039037466,-0.1963803470,-0.6780635118,-0.4196233749,1.5498566628,-1.6033440828,0.3494834006,1.0629042387,1.0952367783,-0.9827461243,1.2384337187,-0.7188549638,0.0698258132,-0.5090520978,-1.3466193676,0.2113907784,-1.9054464102,0.7425320148,0.0713237897,-0.3361792564,-0.1544833332,1.2751290798,0.9343932867,-1.1927865744,0.1609488577,-0.0817530602,-1.6265294552],[0.0792161152,1.0164269209,0.0274453927,0.5830265284,0.4806797206,1.7057460546,0.3446226120,0.1727063209,-0.5576907992,0.5787950158,0.2685793638,-1.1247400045,-1.1956520081,0.7687556148,-0.8483432531,-0.1325481385,0.6503481269,1.2317855358,2.5165569782,0.9462372661,0.7442957163,-0.1037486866,0.8058280349,1.2004961967,-0.2012088895,1.2878060341,-1.6287704706,1.1572757959,0.2970125377,-0.8433011770,0.6941140890,-0.3813734949,-0.7616585493,0.2744237781,-0.0081879422,-2.0205621719,-0.4257437587,0.0422513634,-0.0920020416,-1.5582008362,-0.0928666592,-1.2283902168,-0.1833214313,-0.3190955520,-0.0863488317,0.9307079315,-1.9677612782,-1.5400369167,1.0738964081,-1.0055810213,1.1223951578,0.5734156966,0.4714262486,-0.0594240949,-0.8080806136,0.2278677970,1.0362339020,-0.2495359480,-0.7958166599,-1.1433826685,-1.1669473648,0.2910191119,-0.2219422013,0.1698516011,-0.5612007976,1.1821777821,-0.9412812591,0.3577224612,-1.4375506639,0.6173115969,-0.0753971860,0.4404014051,0.3479704261,1.7728350163,-2.4903116226,1.9318493605,-0.1871684641,-0.0486194454,0.5669991970,-0.1999413967,0.3919151723,-1.0137641430,0.4496327043,1.2215998173,-1.3674079180,-0.3145882785,-0.6682460308,1.7609888315,0.4312697351,0.4117654860,-0.8673826456,-2.2342579365,0.8778496981,1.7627274990,0.1801376790,-0.9213985801,-0.7761957645,1.5885344744,0.3222749829,-0.7482187152,-0.0693738833,-2.5485017300,-0.0396765657,0.9220715165,-1.1876281500,-0.2048502117,-0.8791155815,-0.2799369693,-0.7904237509,0.7771903872,0.0180385746,0.2614985704,0.1531894207,0.4526953697,-0.7578824162,-1.0335227251,1.0775671005,0.2108330131,0.2002221793,0.4302573204,-0.9542171359,-0.1206719950,-2.2216730118,-1.1611789465,-0.8942284584,0.3513225317,-0.4492657483,1.7326287031,0.8167504072,-1.3099080324,1.0597997904,-0.0260978416,-0.0922782496,-0.2445535958,-0.8588480353,0.0233809687,0.8380134702,-0.8886740804,0.0055783619,0.8041345477,0.5155995488,1.3099330664,0.0607562996,-0.4853078723,-1.2040776014,-1.8143754005,-0.9156990051,1.1903975010,0.3376074731,0.4168293178,0.2208614945,0.0823195726,0.3248101473,1.0169280767,0.3660759330,0.7446341515,-0.6754641533,-0.9308695197,-0.9214447737,-0.9902174473,0.0467686094,-0.4429434836,-0.2878521383,0.4659559727,-1.5754804611,0.7648607492,0.6077049971,-0.2009276748,0.1410205662,-0.7378753424,1.5970349312,2.4544296265,-1.7936341763,-0.6051549911,1.5142178535,-1.1078259945,0.3795293868,1.7709084749,0.4984016716,-0.2436304092,-2.2577059269,-1.2458858490,-0.7088252306,1.5962058306,-1.9355672598,0.6987320781,0.3166677952,-0.8615584970,-0.1896579862,-0.6173701882,-1.0844488144,0.5718342662,-0.5237622857,-0.4043986797,1.4371533394,-0.5186876655,-0.8964678645,0.9899360538,-1.3119916916,-0.7182676196,1.0389785767,-1.1774294376,0.6187053919,-0.4993627965,-1.1766394377,0.0940822363,-1.0231488943,-0.4538815022,-0.6294190288,0.5030683875,-0.2208803594,0.9774012566,-1.0498013496,1.1215320826,-0.2495442778,0.3619700968,-1.0616446733,1.0567066669,1.3423643112,0.7119333148,2.0674512386,-1.6267881393,0.6471201181,0.2160703689],[1.3485805988,-0.1709989458,-1.2881512642,-0.3260785937,-1.8688436747,-0.1713963598,-0.2822999358,-0.7701655626,0.1642793864,-0.4175589979,-0.6222335696,0.2136606127,0.4244953096,1.0451035500,-1.6823191643,0.0403030105,-0.4710847735,-0.2437952608,1.3693777323,-1.1758570671,-0.0542855076,-0.9699545503,-0.5364821553,-1.7010054588,0.2163471431,-0.2669135332,-0.3398927152,-0.0248095077,-0.4684832692,-2.0178093910,0.4001567960,-0.6108651161,0.5783423781,0.1649526060,0.8978267312,0.7421654463,0.4261821806,-1.0519768000,-1.4087666273,-0.8967926502,-0.6909884214,-1.0733422041,1.3887276649,-1.5292494297,0.6003373861,-0.3892106116,-0.0517769493,0.0179459751,0.1571146250,-0.5726046562,0.0190555137,0.5697455406,-0.8141471148,-0.3097047508,0.4373070598,-0.3019420505,0.1362971067,-1.0670320988,-0.0000817540,0.8114110827,0.4083184600,1.8282098770,0.1381276697,-1.9155849218,0.0719738081,0.2208820730,1.2493988276,0.3405856788,2.0780653954,-1.2551661730,1.0017586946,-0.5354287028,0.8841439486,-0.5767771602,1.1559445858,0.3145794272,0.1051806211,0.6909182072,0.7290110588,0.1197143495,-0.6287478209,-0.6085677147,-0.8197476268,-1.5091049671,1.0208386183,-0.0439724736,1.7148467302,-0.0321364142,1.3864095211,-1.0215556622,-3.0513429642,-1.1673578024,2.1856009960,0.2588728666,0.3434796929,-0.8460996151,-1.9793503284,-1.1928198338,0.9098986983,0.4554537833,-1.0023721457,-1.8009947538,-0.6367038488,0.9891266227,0.0203659870,0.3414032459,-0.4097161591,0.2278744727,1.2209628820,1.1159952879,-0.9623272419,-0.4903887212,0.4621538222,0.2723306119,-0.9540486336,-0.2404303402,0.3517660499,-1.4416646957,0.4990543127,-0.7126743197,0.3536555171,-0.9577149749,-0.3402226269,-0.8734661937,1.0238445997,-1.2684786320,0.2645486295,0.2472764254,1.1231633425,0.8844839931,-0.5912511945,-0.3987582028,-0.6692000628,-1.7035512924,1.1779968739,1.1480795145,-1.9619866610,-0.6322578192,0.4063148499,0.8685815930,0.1516608745,-1.1480426788,0.0343636200,-0.2617740929,0.8340689540,0.8000271916,-0.4284797311,-0.0311497673,-1.1331477165,0.1529792249,0.7315949202,-0.4204881787,-1.7638682127,0.6233843565,0.1415092200,0.2131320089,1.0914654732,-1.6148589849,0.4392548501,0.3568493426,-1.3790297508,-0.1812242717,-0.4438568354,-0.5541661978,-0.4909126163,-0.9593079686,1.0866227150,0.0711556524,-0.6503744125,0.3807986677,-0.0300570186,-1.6283313036,0.6702497005,0.2368342578,0.6066207290,-0.6736672521,-1.0897010565,-1.1997205019,0.2919110060,0.5207223892,0.6099629998,0.6813975573,0.4636356533,0.0687451810,-0.4502599537,0.1851959527,-1.4338147640,-0.4437556267,1.4200047255,-0.6315543652,-0.7744462490,0.6045300961,-1.2394708395,-0.2095934153,-0.3702944219,-0.6200354695,-1.3909845352,-0.3433581889,-0.9693596363,-0.3587476015,0.3847161233,-0.2518064678,0.4357206225,-0.9091473222,0.6839234829,-0.0295625720,1.4972803593,-0.1460575759,-0.0939826742,-0.2217037976,0.0946994647,-0.2410782129,0.9098507762,-1.3256295919,-0.0817464665,1.3583462238,0.7781915665,0.0069436296,0.5164428353,-0.0651496351,0.7075184584,-1.4202232361,0.2231310159,-0.3650387228],[0.1650255471,-1.3219217062,1.0231046677,0.7444481254,0.5359328985,0.5666531920,0.3261220157,-1.3667412996,0.1605643630,0.2437143475,0.4470121264,0.3345755637,1.5117784739,-1.4013890028,-0.3585534990,-0.2405987382,-1.2795827389,0.3206909299,-0.4760054648,1.4264730215,0.2798275650,-0.0359120257,0.4687083960,-0.4645172954,-0.1489082575,1.6419786215,0.8950193524,-0.2498975992,0.0254452918,0.7679093480,0.3977237940,-1.2860594988,-0.3435108960,1.3776507378,-0.2499962747,0.5837259293,-0.3566249013,-1.5012432337,-0.0366155952,0.6415373683,1.7652125359,-1.3682287931,-0.0861081928,0.0383691862,1.2417683601,-0.1283382177,-0.1103384346,-0.7885918617,0.0254839975,-0.7123177052,-1.1390565634,-0.0729266852,0.2223683745,-2.5684566498,0.4270484746,-0.0046181614,0.0415315256,0.6546118855,-0.9823287725,-1.3519179821,1.2908511162,-0.5305488706,1.7785286903,0.8555805683,1.3566535711,-0.4564712942,-1.4938685894,-0.9814547896,-1.3854124546,-1.5733133554,-0.8129420280,0.3343071938,-0.2484793961,0.7299104333,0.6596966982,-0.7074251175,1.6392796040,-0.7114089131,0.3561543822,0.1788314283,-0.0102422200,-1.5114012957,-0.0173943099,-1.0720241070,-2.3937253952,-1.3889380693,0.0471616276,-0.4784292877,-0.8774627447,-0.7773531675,0.5064339042,1.4271920919,0.6994634271,0.3179759979,2.0188155174,-0.6257854700,0.4875732064,0.9742701650,-0.6120517254,-1.6027874947,-0.5442904234,-0.2421242595,0.5398359299,1.3619903326,0.7030728459,-0.4203302264,-0.0356382467,0.4759574831,-0.3176399469,0.9230871201,-0.7705651522,-0.8988927603,1.8517060280,0.6137269139,-0.1375556439,0.7560847998,-0.7871061563,-0.8101333976,0.9718251228,-0.2960273027,-0.8531348705,-0.7449046373,-2.3861715794,0.5155227184,-1.4067013264,0.3101865947,-2.1656732559,-0.4105100334,0.4387335181,0.6727346778,0.2653877437,0.9309439659,-1.1482447386,1.7589145899,0.4293440580,0.6618949175,0.6083560586,-0.7772499323,-0.2988936305,1.9642839432,-1.3446459770,0.3042408824,-1.8033968210,1.1037830114,-1.7535353899,-0.5489560366,0.9455407262,-0.8433424830,-0.1801532805,-0.4749929905,0.2693964243,0.4919607341,1.2626482248,-1.0708043575,0.1030553579,0.1843638122,-0.7110225558,-0.0900483355,2.1595232487,0.1592033505,-0.9272395968,0.9635785222,-3.7438845634,-0.7730277777,-0.9164265394,-0.2171987295,0.0299481545,0.4585601985,-2.6037380695,0.2609812319,1.3040894270,0.9559362531,-0.0413816720,-0.6852220297,0.4935119152,0.7921469212,0.1276420653,-0.1001019925,-0.0484789126,-0.5497506261,-1.6832387447,-1.5431174040,-1.1059116125,-0.1819771081,0.1616688073,-0.5764904022,-0.9880220890,0.0783481523,1.4437611103,1.9365442991,-0.3549702466,-2.2470059395,-0.4686661363,-0.9983262420,1.7535051107,0.6303505898,-0.7734314799,2.6364636421,0.0392718092,1.5254333019,-0.6479625106,1.6099628210,0.3199017942,0.3757773638,-0.4623837471,-0.1848932803,0.3996022344,-0.0442292541,1.3251973391,-0.4424172640,0.0063154404,-1.4659434557,0.5882980824,0.6704141498,0.9195390344,0.5987154841,-0.6095600128,2.0387501717,1.5419700146,-1.3648542166,-0.9416725636,-0.5326945186,-0.6290355921,0.2064234316],[-1.1643296480,1.3079528809,0.4787613750,-1.1614365578,-1.2261186838,0.6149030924,1.1901096106,-1.7869648933,-1.7015125751,-0.0369055569,-0.1572385281,-1.8780126572,-0.1200147271,-0.4242606461,-0.3192552626,0.0104762856,-0.6464591026,0.8706829548,0.5600137115,2.6534612179,1.3147951365,-2.0153663158,-1.6875379086,-0.6459116340,-0.4426926076,0.1987432837,0.4108413756,-0.4009189308,0.9362496138,-0.7191836238,0.6938658953,-1.7758921385,-0.1403557509,-0.0228065960,1.0006980896,0.8325053453,0.6643543839,1.0108941793,-2.3972070217,-1.1385048628,0.4741702378,-0.6687298417,0.4963333309,-0.7473987341,-0.3865411580,0.2450307906,-0.8685840368,-1.5923013687,0.0233582482,0.8655768037,0.9436566830,-0.7575323582,1.0158644915,0.6127064824,-1.7185145617,-0.4814927578,0.0290120076,-1.8096933365,-0.8135496378,0.5268061161,-1.3745579720,0.8449429870,0.7756015658,0.2198592126,0.1512816399,0.7091155648,1.1741852760,-0.5143500566,-1.6707895994,1.2580791712,1.5318725109,-0.5002031326,-0.2448760867,-0.2390441149,-1.4903237820,0.6865671277,-0.3201767504,-0.0794070512,1.9951273203,-1.4163293839,0.9918632507,1.7666332722,0.1555749178,-0.3725374937,-2.1025719643,-1.2156616449,-0.8836985230,-0.3397898972,0.4215038717,0.4887259007,-0.4905745983,-1.3774695396,-0.6583486795,0.0738252997,0.7343323231,0.4211414754,0.1247600242,1.7488490343,0.2251959741,-1.0532218218,0.4716623127,0.5744453073,-0.0024379981,1.1637272835,-1.9481000900,1.4603103399,1.5550012589,-1.2255028486,0.6156579256,0.6069850922,0.8603722453,-0.5207680464,0.8478476405,-0.3461050093,-0.6320928931,-0.7256397605,-0.9042200446,0.7228765488,-0.0928905234,-0.7883547544,1.6062039137,0.0710300654,0.5437248349,-0.4915961325,-0.1041649207,-0.3031794131,-0.4425407946,1.1392168999,-2.0505964756,0.4079806805,-0.0800484493,-1.3515720367,-0.9124820828,-0.8583441377,1.2894556522,0.5908492804,-0.5798185468,0.0520669743,-0.1212026998,0.4420488775,0.6347669959,2.2531759739,0.4614149332,2.1107082367,-0.6364678741,0.6796067357,-1.2275124788,-2.1137130260,-1.4560955763,-0.1521469504,-1.1474013329,-0.0936479792,-0.4342494607,-0.3847503066,0.9745326042,-0.8003863692,-0.1759050786,0.4681669176,-0.2825888991,0.5702939630,-2.0764098167,0.0134140672,-0.7991058230,1.5231801271,0.3928492367,0.8087913990,0.1406501085,0.2782938778,-1.2058972120,0.7704486251,-0.6398083568,1.8859362602,0.8982129097,1.1188243628,0.0342440680,0.1872243583,1.8391888142,1.3006812334,0.3444184959,0.1678676009,-0.2385491282,0.1145463064,-1.5685496330,-0.4964090586,-0.3101575971,0.3440920413,0.1293883473,0.5067656636,0.3827549219,-1.0888370275,-1.4858487844,-0.9661257863,-0.7745360136,-1.4245500565,0.1393938810,-1.4260451794,-0.5082685947,0.8763096929,-0.6662014127,-1.2093539238,0.7315164804,-1.1380810738,0.0281808823,-0.9322289228,1.2290621996,0.7308822870,0.1425308883,1.4224876165,0.7261432409,1.1260491610,-0.8474053741,-1.1105213165,-0.8556407094,-0.7260117531,1.1177033186,1.1724927425,-0.1795573980,1.0290118456,-1.1191790104,0.0396585315,-0.0047235782,0.2664375901,-0.5832667947,-0.8921521902],[-0.8755141497,-0.9598096013,0.4069607854,2.1447017193,0.3723194003,0.3736964762,-0.6830319166,1.3127006292,-0.6640972495,0.7430377603,1.0540186167,2.0303764343,0.8735508919,-0.6355731487,0.5069352984,-0.9557527900,0.9533885121,-0.4849050343,-0.1885131001,-0.6945106983,-0.6633606553,0.2577122450,0.6968512535,0.0700863674,-0.8694693446,-0.9572583437,-0.1153896451,0.2002226561,-0.3849981725,0.2202363312,2.0618686676,0.3926514387,-1.5631883144,-1.2940770388,-0.5507266521,0.7313650250,-1.7164524794,-0.7593462467,0.3257178962,1.1041924953,-0.6681020856,-1.2724117041,-2.1357870102,0.3565237820,-0.0748446658,2.0107169151,-0.8546269536,-1.1847459078,1.8804349899,-0.1341653168,-0.5399327278,0.6568149924,-0.6100254059,-0.0584002621,-0.3279354274,0.0255116895,-0.4765267670,-1.3591419458,-0.1435545087,0.5789028406,-1.7496489286,1.4535236359,-0.6286903024,0.2937216759,0.1816752106,0.3662569225,-0.8155812621,-0.7177008390,-0.6351452470,-0.5987464190,-0.7643153667,-0.0924506560,0.0002173648,0.4418594539,0.2845681310,0.1451430768,-0.9784609079,-1.6784154177,-1.2964258194,1.5535236597,-0.7936209440,-0.5138964057,-0.9194343686,-1.3460687399,2.6226949692,0.8296312690,0.6559735537,0.5372175574,-1.2835997343,-1.2348910570,-0.9327340722,1.3215962648,0.4506095648,-0.1519411802,-0.1253918707,0.1870906502,1.6497941017,0.0869149938,-1.0887746811,0.7199364305,0.4501723051,0.2700583041,-0.4041815102,1.0006670952,1.3125792742,0.6810870171,-0.1800761074,2.4339106083,0.1763603538,0.8710671067,-0.3742243946,0.4617210627,0.3632403314,-1.1402145624,0.8772246838,-0.2340421081,-0.8595511913,0.0905576646,-1.4307808876,0.5439644456,-0.0927907899,-0.6408239603,0.6136070490,0.5124572515,-0.6934475303,-0.6896564960,0.5295175314,-0.1667579859,0.2426082194,0.0189859830,2.2494912148,-0.2460813224,-0.2038732916,-0.6019621491,-0.0026605215,0.1960952580,-1.1902012825,0.4211482704,-0.4473219514,0.4534853399,-0.3401862979,0.1513924450,0.3664656281,0.2547777295,-1.0560954809,-1.9132366180,1.4109679461,0.6244448423,0.4116883576,0.4919942617,-0.6390546560,-0.0430185869,1.6317751408,0.2780894935,1.6450288296,-0.3439806104,-1.5669082403,-0.0530644134,1.5441375971,-0.4733426571,0.6746113300,1.0411593914,0.9855762720,-1.7978531122,-0.9804763794,0.6059101820,1.1089842319,-0.8183146119,-0.9320350289,-1.1980115175,0.1660242826,-0.7525303960,1.5922327042,-0.8294901252,0.9423011541,-1.8002551794,0.0447457209,1.4288845062,0.5406267047,-0.1151995510,0.8838722706,-1.7344129086,0.5595968366,0.4175852537,0.6582928896,0.7738341093,0.7438519001,1.0156049728,-1.3063168526,-0.9007453322,-0.5195229053,-0.0660142750,1.7721335888,-0.5381574631,0.0451898575,-0.8444939852,-0.0029482909,0.9423898458,-0.8007932305,2.1239731312,-0.2745014429,-1.9707047939,-1.2178057432,0.7433405519,0.2328127027,1.4669008255,0.1985478550,0.8833305240,0.4210582674,-0.6766483188,0.7286726236,-1.7391421795,0.4351043105,0.6838805676,0.1998222470,1.5584337711,-0.7463527322,1.0878604650,-0.5838196278,0.3922228813,-1.2806158066,0.0529088899,0.3663826287,-0.1530306935],[0.0078097042,0.3014991581,-1.7583640814,0.5166698098,-1.0459570885,0.3360862434,-0.3021540940,1.6351559162,0.2339851409,1.3475879431,-0.2810580730,1.2364726067,0.8393220305,-1.1669842005,-0.9311234355,0.3072465360,-0.7316344976,0.6209425330,-0.4304316342,1.0347040892,0.3869737089,-0.8412460685,0.9965993166,0.2931939960,-1.5351545811,-0.2541728318,-0.5098071098,-1.9362185001,-1.0679433346,1.2295447588,-0.6092607379,-0.2998788953,-0.2101449221,1.5696338415,-0.9006688595,1.9490916729,-1.1761932373,1.4566138983,0.7829972506,-0.9729502797,-0.6038942933,-0.3258980215,0.3648591340,0.6088670492,0.5756407976,0.1295090169,-2.0584166050,0.7405277491,-0.0565741882,1.6712781191,-0.0073989523,-0.7615351081,-1.7625714540,-1.6687066555,0.2233912647,1.7904453278,-0.2203775346,0.3719655573,-1.2596836090,-0.1352416873,1.6784518957,0.9244667888,0.6470136046,0.1950820833,0.2686503530,-0.3164552450,-1.4811400175,-0.4500595033,0.9400957823,-0.8417010903,1.2031250000,0.1384577602,0.7147232890,0.8561578989,0.8689596653,-0.7192590237,-0.4665651917,-0.1493928432,-1.8951672316,0.9723851681,-1.4815393686,-0.3151030242,-2.1755998135,0.3345840275,0.4477573931,-0.8480839133,1.4286578894,-0.5938410759,-0.3036641181,-0.1643898785,1.4175655842,-1.2077329159,0.5321143866,-0.3946001530,1.0930041075,0.8004240394,0.0015130466,-1.2983140945,-0.2006176710,-1.0674712658,0.6152316332,1.3683578968,0.5303028822,-0.3842371404,-1.3599123955,1.2266104221,0.7833281159,1.6608939171,-1.1905647516,-0.5298700333,0.9390087724,-0.7273864150,-3.0896890163,-0.9112982154,-0.2719693184,0.4283858538,2.2000403404,1.2904229164,-2.8206224442,1.5973047018,-0.3084849417,-1.2416521311,0.3893934488,0.9960338473,-1.6180120707,-0.7857273817,-1.6723463535,-0.1164723262,-1.4589574337,-0.2019911110,0.2670248449,0.9341351390,-0.5450060368,-0.9017308354,-1.1602119207,-0.9146684408,-0.4528957009,-0.7189208269,-0.8239266276,-1.4149652719,1.0071654320,-0.7333939075,-1.2482929230,0.3368687928,1.5294657946,2.1932830811,-0.0322217532,1.3643707037,-0.7223550677,-1.0060669184,0.4713765979,0.0374758020,-0.2954056263,-0.5196225643,-0.3418472409,0.8694543839,-0.0366826206,-0.3693021238,-0.0720364004,-0.3977731168,-0.2981844246,0.3481273651,-2.2333576679,0.1124164909,-0.5860416293,1.2649736404,0.4698347449,-1.6995165348,-0.6768918633,0.5547665358,0.1519433111,0.4794569612,-2.0995452404,0.1126223803,-0.5789268613,0.7632908821,0.6681728959,-0.1237390935,-0.2411789000,-0.8700891137,-0.0535563789,0.1604736447,-0.2040840238,-0.3879729211,0.3112916052,0.7895922065,-0.0806431398,-1.9122922421,-1.2963095903,0.9608740211,-0.5996281505,-0.2180650234,-0.1556250751,0.5601133108,0.5118433237,-0.6297109723,-0.2477449030,0.4377675653,0.1655773968,-2.1302938461,-1.1546865702,1.3413959742,-0.7347362041,0.8989260197,0.4398216605,-0.8333153129,-0.5954267979,-0.4634758532,-0.0472172499,-0.1780685633,-0.6798447371,-1.1439967155,-0.0540322363,-0.7338622808,-0.3657207787,-1.3311915398,-0.0009648965,0.8482714295,-0.3821905851,0.5203379989,-0.4978805780,-0.6867644191,-0.3023690581,-1.7219276428],[-0.0706413612,-0.9097322226,-0.6544209719,-0.6343010068,0.1573838145,0.8664366603,0.7233697176,-2.0078625679,0.2135981321,0.4005902708,0.3631705046,-1.0551964045,0.1154454425,-0.6361332536,-0.7936405540,-0.8201953769,1.5267411470,-0.3820387125,0.8344710469,-0.4640203118,-0.1984673291,1.1495521069,-0.2665902674,-0.3317551613,-2.1129162312,-0.9538705945,0.7641342282,-1.5903077126,0.8508225679,1.7642945051,-1.1222983599,0.5385093093,0.4542241693,-1.7904034853,-1.0321934223,3.2340333462,0.3957667649,1.8712517023,0.1348981857,0.3125168383,0.2443830669,0.4996775389,0.2636229098,1.9391344786,-0.4150807559,-0.5804721117,-1.0143501759,0.1284436435,1.1321600676,0.3555642068,-0.1672841161,0.6568336487,1.0217926502,1.1322041750,-0.7075734138,-1.3878320456,0.4098901749,0.2341521829,-0.1374025643,-0.0444334671,0.4156866968,0.9274237156,-0.5546074510,0.6505785584,-0.1995093673,-0.0293728597,-0.3378144503,0.5294931531,-0.8031910062,2.0809404850,-1.1738935709,0.7713395357,-0.7774733901,-0.5353500247,0.3451361656,-1.3199455738,-0.3418870568,-1.0163743496,-1.1710753441,-0.2852978408,1.0949852467,1.3117394447,-0.8261899948,-0.6163251400,0.0279291794,-0.4021877348,0.5516725779,0.5848386884,1.0316517353,0.3473063111,0.6851702929,0.8027648926,0.5897445679,-1.0397065878,0.8500480652,-1.5022568703,0.5201315880,1.5309634209,-0.0968860835,1.6017364264,0.3336514235,-0.0345757678,1.5534172058,1.7362976074,0.6443283558,0.8172535896,-2.2950198650,-1.5091173649,-0.1997069120,1.0716969967,-0.1073218584,-0.7956750989,-0.6962655783,0.4508700073,-0.3004487455,-1.1373023987,1.7564595938,-0.1807855815,-0.3760594130,-1.2967089415,-0.6098486781,0.4165080786,-0.8964832425,1.7748460770,-0.7647678256,-1.5128406286,-0.3202617466,-0.2043563724,1.0446738005,-0.7297920585,-0.0262035001,-0.1444185227,0.3840039670,0.2060625702,-0.2181037962,-1.1294969320,-1.0868163109,0.3352339864,1.9930715561,0.5076326132,1.3730456829,-0.0023539420,-1.5736697912,-0.3311335146,0.9903514385,0.0521080866,-2.4414179325,-2.5822451115,0.1565915942,1.4188001156,0.7368789911,0.2345260978,1.0152734518,2.2886154652,-1.0148390532,-0.3122152388,-0.2702304125,-0.5222117305,1.4889755249,0.5128302574,-1.7104084492,0.1142061651,-0.2926707566,-0.8836719394,1.8603035212,0.1356332749,0.8755128384,-0.9803749323,-0.2293416560,0.2006159127,1.7615724802,0.3264065087,0.6724275351,-0.0469790995,3.1880333424,0.4425591826,-2.3789470196,-2.4320871830,-1.5881433487,-0.4209427834,0.4819173217,-0.5247082114,-1.5172423124,0.8938916922,-1.6846462488,0.2446191460,-1.0680624247,-0.3203495741,0.0754074827,0.9786596894,-1.0031156540,-1.4559154510,0.3935267925,-0.3842234910,0.4690336883,-1.0834065676,1.4030132294,-1.4369273186,1.1485811472,1.4488285780,-0.9468003511,-0.5519796014,-0.4509304166,-0.4519639015,0.0976786837,-1.9000874758,0.7958002090,0.1177812293,-0.3470512629,0.0172776580,0.8916116953,0.1383883506,0.1076137498,-0.1316533536,-1.9620102644,-2.0274169445,1.5324951410,0.7383908629,-1.1564779282,1.5768051147,-1.4502425194,0.9668387771,0.3540835083,3.4085419178],[1.9100345373,-0.8650856614,1.4155201912,0.5690464377,0.5973732471,-0.8678830266,0.8286939859,-0.2351627350,0.9233389497,-0.5177605748,0.4496781528,0.0800872520,0.2445594221,-0.2652956247,2.1359424591,0.1638048589,-0.7652742267,-0.8763234019,0.3231536448,-0.6016790271,0.2801153660,0.8367733955,0.5688657761,0.6027054191,1.1826387644,0.0957681388,0.1674305946,-0.1956560016,0.5108280778,1.0816714764,0.6208047867,-0.1561427861,0.8358264565,0.9209582806,-0.8191552162,-0.1509196013,0.8101909161,1.2557537556,0.2925243676,-0.0419176333,1.1210883856,-1.0228914022,-0.1098661944,-0.8591143489,0.2410670966,0.9973797798,-1.4664530754,-0.0980027616,1.8604106903,-0.6354730725,0.2107730508,0.5374374986,-1.1003706455,0.8711256385,-0.6260376573,0.2263965756,-0.0102861263,-0.1871914864,0.8020673394,0.5475625992,0.2951956689,1.2020192146,-0.4511943161,0.6142093539,0.0216287971,1.3507753611,-0.3723137379,-0.1512116939,-1.7927660942,-0.4930469692,0.6896463633,-1.8515053988,-1.6282465458,-1.2689808607,0.3326217532,2.1312882900,1.7744779587,-0.8050757051,-1.5977858305,-0.3082677424,0.1215020418,1.4367951155,1.2268460989,-0.0081329355,-0.5573827624,0.1035985723,-1.2760721445,-1.1780765057,-1.2280064821,1.1160129309,1.7673761845,1.7334949970,-1.7602155209,0.5039381385,-0.3720277846,0.3963406086,1.0447177887,-0.8420517445,-0.0861799866,-0.1530893594,0.2599826753,1.1811122894,0.5532831550,-1.3378735781,-0.8013755083,-0.8412433267,1.0781079531,-0.0028998200,-0.6660804749,-0.7412942052,2.3283331394,-0.4305628836,2.7005894184,-0.6134419441,-1.0312664509,1.0619452000,-1.1738936901,1.2846373320,1.7262413502,1.3725773096,-0.7013794184,-1.3772867918,0.4172764719,0.4020915329,-1.1206417084,-1.2978309393,-0.0802910849,0.9490975738,0.0574220717,0.1611342430,0.6615610719,-1.6403830051,0.2258825600,-0.2138030678,-1.7920988798,-1.5084474087,1.6496242285,-1.0713797808,-1.8907377720,0.0772534534,-0.2337310910,0.0140136406,0.5051288605,0.4664832950,0.8642095923,-1.8167469501,1.0395417213,-1.1637365818,1.1356519461,-1.7042986155,-0.0973814875,1.1059724092,0.6865385175,-0.0097261248,-0.0064713955,0.2397290915,0.4610339105,2.3903193474,0.5248602629,-0.8353090286,1.6538916826,-0.5744239688,0.4300434589,-1.4513535500,-1.2397629023,-0.9852696657,0.1993034035,0.0661344454,-0.5709816217,-0.0929837227,0.2432375997,-0.0583449788,0.0009275326,0.8209046721,0.0518882200,0.8052539229,-1.6794208288,0.2178734690,0.7934340835,-0.8915417790,-0.9696401358,0.0556026287,0.7279754281,0.4486949742,1.3063867092,0.0655736998,-0.5748510361,0.0804484934,-1.2538518906,1.0344552994,0.0385405272,0.1673927009,-0.4514338970,-1.3391981125,-0.3614654243,0.9321334958,0.4562244713,0.2335039377,0.1003533304,-1.6953806877,0.1109338477,0.4388502240,-0.1420196295,-0.5670718551,-0.4239734113,0.6679923534,0.1954314113,-0.7743871212,0.7897585630,0.2001348287,-1.4090903997,-0.6529703736,-0.5033426285,0.8819975853,-0.8331340551,0.4697431326,0.1163358986,0.7791193128,-0.4779369533,0.3048064709,-0.5940655470,-0.4552982152,-0.4650419652,0.0361061431],[1.5899168253,0.0063533471,1.6877865791,0.0515951440,-0.9018155932,1.2319109440,-0.6424393058,-0.2568293810,0.6929336190,-0.3502320945,1.4825685024,1.3803684711,1.1104739904,0.6726384163,1.6660864353,-1.1913731098,0.4051263034,-0.3095752001,-1.2381680012,-1.6911195517,1.1668710709,-0.6490058303,0.4584015310,-0.9608908892,0.0266461428,-1.2123868465,-1.4942896366,0.1524978429,-0.6395063400,-0.3380450904,-0.9216715693,-0.0510670282,1.1863936186,-0.6247690320,0.3869996667,-0.5245535374,0.3170322776,-1.0178136826,-0.1742559820,-0.1370685846,0.8142974377,0.7342265844,-1.9699764252,2.2751204967,-0.3226063848,-0.5424374342,0.6009609699,-1.7587728500,-1.5937917233,0.9980217814,0.4808895290,0.5623027086,0.3096584976,-1.2413680553,0.0726389959,-0.2213159949,2.1800575256,-1.4602934122,0.0018683164,-0.1173354015,-1.0309307575,-1.3381360769,-0.7076413631,-2.1738369465,0.6837378740,-0.7091113329,-2.7662103176,0.7608048320,1.0504329205,-1.4428687096,-0.7895605564,0.7236565351,-1.1905863285,-0.6919526458,0.8573622108,1.5371603966,-0.9198681116,0.7844922543,0.6338860393,-0.4500072896,-2.0559315681,-2.0172803402,-1.2834616899,-0.3556898534,0.6204849482,2.3478055000,-1.3467277288,0.1222354546,-0.2214947045,-0.2462989539,-0.0867773294,0.6118643284,0.6414287686,0.4616425931,-0.1879353225,-0.3813198209,1.3266900778,0.4950915277,-0.8527886271,-1.9915175438,-0.0268161800,0.2655802965,0.3149074912,1.0536828041,0.8504412174,-0.7472555041,-1.5911126137,1.3834769726,-1.4433643818,0.6882103086,-1.4758917093,0.8962612748,-0.6082796454,-0.8149115443,1.3695408106,-1.9525241852,-0.5935515165,2.1671218872,-0.9892462492,-0.2221836448,-0.2219780684,0.9403944612,0.1978086531,-0.7817671895,-0.4124997258,0.3687230051,1.0968439579,-1.8594348431,0.0029549592,0.1101333052,1.5394915342,-0.0038964562,1.0981504917,-0.0400299132,0.2016086131,-1.3140232563,0.2941615582,-0.7528982162,-0.0452784672,0.7580948472,-1.3945977688,1.3409752846,-0.3101814091,-1.2464029789,-1.1343907118,1.7912290096,0.7197954059,-0.5015527010,-0.1126659065,1.1124942303,1.3593585491,-1.0211582184,-0.7661721110,-0.6488007903,-0.9884676933,0.7920768261,0.5087880492,1.3648190498,0.6742984653,-1.6158971786,-0.8027673960,0.2757093608,0.2329666018,-0.6251592040,0.0634803176,-0.1018172055,1.5068968534,0.6900904775,-1.7025721073,0.3553458452,0.3871901035,0.9579328299,0.7569516301,0.7862011194,1.3912926912,-0.1798933148,-0.3273005486,-0.3807692528,-0.6068280339,-1.0164443254,0.7183098793,-0.3550116718,-0.0208350029,-1.5521923304,-1.1644581556,1.0302523375,1.1733825207,0.2134277225,0.0292529687,-0.5484084487,-0.3583379388,-0.8931321502,-0.1975669116,0.2763205469,1.1366759539,-2.0308594704,0.3366666138,0.3184001446,0.4373573661,-0.8740926385,1.0525960922,-1.2406895161,-0.5679717660,-0.5852907300,1.1207735538,0.4359603524,-0.8621008396,0.2411166579,-0.1937544495,0.6315796971,-0.4637977481,-0.8728907704,-0.3243234754,0.3536755145,-0.2431924790,-1.2344973087,-1.1073101759,1.7516149282,-0.4201321900,-0.6599223614,-0.6240724325,0.0641648844,0.8778529763,-0.7815441489],[0.8508049250,-0.8690682650,0.9813565016,-2.0868430138,-1.1037631035,-0.1131165251,0.0414229818,-0.8837306499,-1.4935542345,1.1044009924,-3.1893181801,0.5092691183,0.4799568355,1.3159337044,-0.0970403627,-0.2791163325,1.7901148796,1.5886393785,-0.3469516039,-0.9854775667,1.3777000904,-0.4683577418,1.6312767267,-0.5955067277,0.6797708273,0.6795912981,0.5310679078,-0.8304234743,0.7193893194,-0.0236661416,2.5271956921,-0.7473905683,-1.0720716715,-0.1273948997,-1.3011174202,-1.6902400255,-0.4170007110,1.1451983452,0.8875586390,-1.9904561043,-0.6132237911,0.0021884013,-1.0157800913,-1.2629107237,-1.6704556942,-0.9015039206,-0.7487605810,-0.0734295398,-1.4195762873,0.9139781594,1.6136165857,-0.3155820668,-0.5592501163,1.7205772400,0.1843333095,-0.2459741086,0.7138553858,0.9178287387,-0.3423841894,0.6098057628,1.5959429741,0.6128612757,-0.6838819981,0.6335126162,-1.2277721167,0.0185308941,-0.8606790900,0.1154111698,1.4965243340,-1.1527279615,-0.6795991063,-0.9856798053,-1.2678070068,-0.4434808195,0.0805942714,0.2353551388,0.0121626621,0.5561687350,-0.0758090466,0.1448151320,0.2649245858,0.7526773214,-0.2883499265,3.2922325134,1.2451647520,-0.6571530104,0.7045717239,1.0751260519,0.2896658480,-0.0684312060,-0.7031256557,-0.7291713953,-0.4636211991,0.5477852225,0.4574085772,-2.8637101650,-0.3515471518,-0.6543455124,1.0587671995,-1.0377566814,-0.1081912965,-0.7057687044,-1.2898498774,-1.6401636600,-0.0102215903,-1.6310093403,-0.6036027074,-0.3954981267,-1.0030182600,-0.6082266569,2.5081093311,1.6789019108,0.2176964432,1.2520422935,-0.4151733816,-1.2843372822,-1.0400305986,0.3440242708,-0.4483019412,0.6690663099,-0.7175396681,-0.1458407640,2.2244884968,0.1947859079,0.1055293828,-0.6040461063,0.8576923013,-1.5997809172,1.8376449347,2.8909277916,-0.5177230239,0.2298939377,1.8100821972,-0.5709523559,-0.5115388036,1.2967069149,-0.5621591806,-1.1462398767,-0.9591106772,-1.6882694960,-0.7014160156,0.6921666265,-1.1444196701,-0.1640264839,0.4573389292,0.5338413119,0.8252105117,0.1567942351,1.4680320024,-0.2286765873,-2.0002055168,-0.6564577818,1.9003719091,0.4424801469,0.4283055365,0.6784015894,-0.9360645413,0.5108249784,-0.0562780872,0.0913101286,-0.8051065207,-1.8382200003,1.4490640163,0.5964669585,1.4912917614,-0.3538664877,1.1519131660,-1.2095607519,0.6885273457,-0.0599056147,0.8238558769,-0.0441347621,-0.9308155775,-0.7122085690,-2.2147786617,1.1632386446,0.6279171109,-1.3401852846,-0.9253047109,1.6279819012,0.5150529146,-0.6503204703,0.1271840185,-0.1690019965,-0.5984864831,0.5934988856,1.3468483686,-2.3228442669,0.3811374903,0.0807556435,0.4608016312,1.1064083576,-0.9534149766,-0.5410642624,1.6721212864,0.3387215137,-0.5848330259,0.1684350818,1.1708821058,-0.0112401946,-0.0421408676,-0.0110325404,-0.0399069786,1.8626705408,-0.3295717239,0.2178692967,0.3128709197,-0.6914327145,-0.0489474349,-0.4623219073,0.6858608127,0.4037072062,-0.4837623537,-1.6117572784,0.0941191539,-0.6782262921,1.2615525723,-0.2805505693,2.6711010933,1.3798089027,-1.1973799467,-0.4020443559,-0.5639002919,1.5455411673],[0.6903855801,0.9709954858,0.0853693932,1.2079653740,-0.1136911586,1.1985101700,-0.8023996949,0.8236210346,-0.7027028799,-0.5922062993,-2.2783963680,-0.3009352088,-0.7893054485,-0.2613692284,1.8687489033,0.1451711506,1.1809588671,1.4143635035,0.2772610784,-0.3749769330,0.6385892034,0.7431727648,-1.3622537851,1.3739572763,-0.9690158963,1.0907326937,-1.2023899555,1.2161438465,1.5248703957,-0.1389713138,-0.3804358244,-0.1093302220,0.5503053069,-1.3575862646,0.4582281709,-1.1639021635,-0.3647545874,-0.7115260959,1.4336994886,0.2221960723,2.0658452511,-0.2943881452,-0.7061772346,0.6280207634,0.5662272573,0.3857337832,-0.4284617901,-1.3776395321,-0.6398698092,0.4830209613,0.9864068627,-0.7793473601,-0.4055784643,-0.5901540518,-1.5036782026,2.4848642349,-0.3595902622,0.4562184513,-1.2775880098,-0.4926495552,-1.4293605089,-1.3942863941,0.7871758938,-1.9546544552,-0.9582464099,-0.8009232283,-0.3317953348,0.5323982239,0.4935196638,0.8553711176,0.1533751488,0.9288468957,0.1826938540,0.8980613947,0.4672872722,-0.0300932415,0.3488621712,2.1993985176,-0.3082002103,0.6237960458,-0.5904917717,-0.5733503699,-1.4770421982,-0.3424594402,1.8895410299,-1.0194392204,-0.7236206532,-1.3778822422,1.0129799843,1.1487203836,-1.0671883821,0.8054333925,-1.6088992357,-0.9679784179,0.4632321298,1.2936443090,0.6811715961,0.0065504359,-1.3251696825,-0.5823521614,0.3334719539,-1.1553093195,0.6754829288,-0.6283892393,0.0070697991,-0.1301544905,0.0000159070,1.6803572178,1.0456736088,-1.2322541475,-1.7359474897,0.8063043952,-0.3912761509,1.4771649837,-0.3736453354,1.5715459585,1.1237367392,-1.5193575621,0.5520567894,0.5154913664,-0.0857104436,-0.9226989746,-0.1511355489,-0.1532225013,-0.4101878107,2.1041259766,-1.2773672342,-0.4122236669,-1.2621785402,-0.6445494890,0.2983115911,-0.4472069144,0.0983226672,0.2284848690,0.4287489355,-0.2973191440,-0.8167265654,1.2091214657,-0.4534170032,0.9312779903,-0.3093512654,-0.3528625667,-0.0741263479,1.0887343884,-1.2669620514,0.3354581296,0.0044241310,0.3022244275,1.1900383234,0.2975670993,1.4324852228,0.6217397451,-0.5294061899,0.4970742464,-0.5087625384,0.1304465979,-0.1666449457,0.5193110704,0.8491689563,0.5580003262,-0.5960376263,2.3947246075,-0.5828321576,-1.2493512630,0.7954925895,-0.5065029263,-0.5937213898,0.9579040408,-0.0251192842,-0.5465407968,1.1391458511,-0.4674501121,-0.1967873126,0.2859996259,0.9837669730,-0.9668199420,-0.1133453920,-0.2571675479,2.5318741798,-0.5155376792,0.8728454709,0.0778044462,-3.1305611134,-0.4914603829,1.1896528006,-1.4559942484,0.3755713403,-0.5113667846,-0.4030328989,-0.8139418364,1.3834196329,-0.1454866529,0.2846169770,1.1209895611,0.8712860346,-0.0718860626,-0.6956607103,-0.4459330440,-0.4794711769,1.5473222733,0.0038444141,-0.9808133841,0.2412267625,0.9256925583,-0.5301594734,-0.4780997336,-0.6812683940,0.5242022872,0.1786995083,0.8638377786,-0.3517528772,0.5365935564,0.5407190323,1.1510950327,-1.0237739086,0.6385636926,2.2565796375,0.7124504447,-1.9051090479,0.5709059238,1.9324462414,-0.8584792018,0.8580051064,0.2227552831],[-2.1613814831,-0.2940069735,-0.1814101785,0.7617862225,0.6633594036,0.1448772103,0.4526188672,1.6508759260,-1.7932395935,-1.7927554846,-1.2987227440,-3.7009484768,1.4530611038,0.2657386363,-1.1115087271,0.3476705849,0.9187952876,-1.2118660212,0.9229725599,1.2597514391,-1.9153442383,0.4390523732,-1.1919585466,0.2263162732,0.8045522571,-0.8975812793,0.1102162153,-1.7435343266,-1.5475306511,0.1274931878,2.3257565498,0.2937721014,-1.1089141369,0.5591292381,-0.3039319515,0.7630262375,0.2723177671,-1.6435691118,-1.1236854792,0.1609627306,0.6847237945,-2.4690279961,0.0209366865,-0.5310238600,0.4430907667,-0.3956150711,2.5748353004,0.9656668305,0.5004914999,0.4974248707,0.1869066209,-0.3793111145,-0.5843949914,1.0160140991,1.0422772169,0.0353021547,-0.2838065624,2.4264302254,0.1781947762,-1.4493917227,-1.2831250429,-1.2559127808,0.8513959646,-0.4708440900,-0.9816243052,-1.4453076124,1.0640504360,-2.4597752094,2.4951465130,-1.3210877180,-0.3973611891,0.6279271245,1.2438503504,-0.8696066141,-1.6361762285,-0.0915040895,0.5303424001,0.1272949874,0.6879992485,0.6507039070,1.6750425100,-1.7682852745,0.4391796291,0.0041834023,-0.2630576193,-1.8533036709,-0.8588609695,0.1978735924,-0.7241343856,-0.4095613658,-0.6204192638,0.2809041440,0.0266254898,0.5631239414,-0.0666484684,-1.3585243225,0.1720818728,-0.9857338667,0.6432245374,0.4694710672,0.3492645621,-0.5567824244,-0.4697691798,-0.4117129147,2.1153721809,0.5442541838,-1.6541137695,2.1774716377,-0.1963181496,-0.8331007361,0.7331581712,-0.0479035601,-0.4869109094,0.6976183653,-0.1689549834,0.6171053648,1.2108881474,0.5635103583,-0.8354082704,1.2755782604,0.9359452128,1.2179498672,0.2265103161,1.6606619358,0.5011734366,0.1251170188,0.2605842352,-1.5394775867,0.4511664212,-0.2313937992,1.1197661161,-0.1140006483,0.3820356131,1.3873690367,0.6571901441,0.5457989573,-0.6308066845,-1.1354978085,-2.4026119709,-0.3027979434,-0.5020709634,-0.2321306318,-0.0201238710,0.9409797788,0.7256790996,1.3103921413,-1.0225174427,0.8154883385,-1.1430218220,0.5008414984,0.3574302495,0.0495680012,1.5883718729,-0.1069783717,0.1745323092,-0.0496963039,1.0220882893,0.4006874859,-1.0913971663,-2.3291058540,-0.6900204420,-0.9169560671,-1.1179387569,-1.0036914349,0.2173535079,1.0790767670,0.0816364065,0.2541635036,-0.8011046052,0.4594263136,-0.6509816051,1.1677899361,-0.3477513790,0.5618755221,1.1921788454,1.5922267437,-0.2455349416,-1.0317528248,0.2817627192,-0.8351853490,2.2162134647,0.3737241030,-0.9866889119,-1.3697648048,-0.1978318542,-1.4279685020,0.1537464261,1.1027641296,-0.4549758434,1.3040695190,0.9517257810,1.3341605663,-0.5766361356,0.4542465210,0.6612707376,-1.2149206400,0.7250176668,1.0353204012,-0.3413012028,-1.3613491058,0.4005934596,-0.1110523567,-0.0788928047,1.1674675941,-0.3902478814,0.8044789433,0.7681032419,0.0321332254,-0.2218496799,-0.8551516533,-0.4030755162,-0.0721132234,-1.2544465065,1.4844565392,-0.5042630434,1.1852766275,-1.2150322199,2.2204818726,-1.1828657389,0.7846149802,-1.9904559851,0.1393995136,0.6561467052,0.6043061018],[-1.0907354355,-0.7484697104,-0.7126812339,0.8230614066,-1.3216540813,0.1326915920,1.2171856165,-0.4447000027,0.8073532581,0.6256611347,-0.0607976690,0.7808696032,0.3649677634,0.5778753161,0.4065142274,-0.5495318174,0.4771209359,2.5231893063,0.6165393591,-0.5275948644,0.5816348195,0.5120906830,-0.7348868251,-0.4592056274,0.4749583006,1.1358423233,0.6339197755,1.2536251545,0.0191743113,1.3371372223,0.6908062100,-1.5238791704,0.7582221627,-2.0854206085,1.0246943235,0.3916818202,-0.3441234529,-0.3393303752,0.5980219245,1.7809047699,-0.0322760791,-0.5638660192,0.7236633301,0.1639522463,1.5012469292,-0.3719091415,1.1622062922,2.8788545132,0.0829108357,-0.0003996070,1.1071796417,-0.9313130379,0.1179228798,-0.6248297095,-0.6864803433,0.3658074439,-0.1976805627,-0.7697430253,1.1433596611,0.4378886223,1.0920422077,-0.1473139822,-0.4550720155,0.8509523273,-0.4368683994,0.1370265186,0.9994106889,0.0470677242,-1.2024868727,0.7732259631,0.7193604708,-0.4950317442,-0.6972939372,-1.1340928078,1.4360136986,-1.0777835846,0.6327699423,-0.7165985703,-1.7260651588,0.7511957884,-0.8365247846,1.4378730059,-0.2593916655,-0.7531266212,-0.0018816765,0.2780501246,0.3728078902,0.5840899348,1.1903779507,-0.7974692583,0.6395269036,-0.1412059069,0.5012361407,-0.1481624842,-0.0530277118,0.2532734871,0.7361589074,-0.5505169630,1.3500739336,-0.7229986191,0.5972332358,1.5588985682,-0.9929527044,0.6563365459,2.0401813984,0.2912432551,-1.0593951941,-0.7136363983,-0.4101233482,-0.8217135072,-0.9647257328,1.5899295807,1.4830676317,-2.0082218647,0.6201471686,0.0921110585,1.0209305286,-2.2872138023,0.2827239335,-2.0732119083,1.1393668652,-0.8216959834,0.2826004326,0.0577638298,1.0376316309,-1.2894252539,-0.5568271875,0.3712979555,1.0854558945,1.6734513044,-0.2837608159,0.7808902860,1.2088090181,-0.3496015966,-1.8581924438,-0.3988831639,-0.4860283136,0.5964819193,-0.1913231909,-1.8144276142,-0.9861682057,-0.9875866175,-0.3508193493,-1.0703758001,0.9739978313,1.7451748848,1.1622909307,1.2399636507,0.1838206500,-2.1057317257,-0.3868809640,-1.4580152035,-1.9583094120,-0.5214228630,-0.8149047494,-0.2598052323,-0.0679654256,1.5236089230,-1.8181490898,0.7602464557,0.2036284059,-0.6710840464,-0.4183948934,1.6409194469,0.0934851244,-0.9035621285,-0.1599463671,-0.2006375492,-0.1909182221,-0.9695960283,0.5366060734,0.0499979891,-0.5023054481,1.6040503979,-1.2374203205,-0.2361531258,-1.3638973236,1.9122620821,-0.8687421679,0.5375559926,0.4242394269,-0.5340645313,0.7021860480,0.2912759781,-0.4078489244,-0.0786173418,-1.5945433378,-0.3502337337,2.0544710159,-0.1928384900,0.9104574323,-0.0862179101,0.6111246347,-0.3534440100,0.5634724498,0.4857985377,0.2631002367,1.4814050198,1.1656149626,1.1763496399,-1.2619029284,1.0174994469,0.8297733068,-0.5920416117,1.5649130344,-0.7932690978,-1.9636081457,0.0073007345,1.7777563334,0.4939030111,-0.8862183094,0.5388250947,2.3175821304,-0.6583781838,0.0406383574,-0.6774467230,-0.0744541883,-0.6628022194,-0.7723457813,0.8113358021,-0.0109207612,-1.4630174637,0.8977974653,0.0017468558],[0.8730773926,1.8256112337,0.9446329474,1.9473477602,0.5063759089,-0.2418232411,1.6875723600,0.5409396887,-1.7663731575,0.7723557353,0.7904483080,-1.1151059866,0.6893996000,0.0964727402,0.3555314541,2.8903863430,-0.2023495734,1.1651166677,-0.1074468717,-0.4425264895,0.3537662923,-0.5301949382,-0.1266164333,0.1925617903,1.4034749269,-0.4170812666,0.2439171225,0.6235732436,0.0569755919,0.4600847065,1.2430694103,-0.2150766850,0.5950499773,0.1923431158,-0.1020159572,-0.9160646200,0.8229887486,1.0482641459,0.5558660626,-0.7286831141,1.7109791040,-1.3896939754,0.7022241354,0.7677382231,0.8328258991,-0.7697517872,0.6147429943,2.0839247704,0.7161874175,-0.3919250667,0.9053443670,0.0932821408,-0.7115653157,-0.2126543075,-0.8772642612,-0.1721208841,1.3694014549,0.2511317730,0.1415636986,0.0112932771,-0.6726903915,-1.6037570238,-0.5378059745,0.0845245272,0.7646923065,-0.2101514488,0.9166517854,0.4953337014,-1.5049014091,1.1143501997,1.6012240648,0.6021217704,-0.9676260948,0.5672973990,-0.0398906879,-0.2725612223,0.6513987184,0.5207162499,-1.9729640484,0.8367348313,1.0528236628,-0.4983050823,0.5411375165,-0.6223208308,-0.5388869643,0.9614911675,-2.0136291981,2.0114109516,0.2237331420,0.7709437013,0.1642037779,0.9776116610,-0.2966648638,-1.1799397469,0.7032052279,0.0894018561,0.7504652143,0.3706242144,0.5047916770,0.6028758287,0.4940436482,0.8593719602,0.5868232846,0.1310377866,-0.3590817153,-0.8620396852,0.6448566914,0.6952871084,0.9641913176,0.1289675683,0.9945142865,-0.3077834547,-2.1486518383,1.8851884604,2.6745195389,-0.3878851533,0.3051368296,-1.3988252878,-0.6323857307,-0.8161635399,-1.3922702074,-0.6827468276,-0.2562257946,0.5322270989,-0.7437602878,-0.2868638039,0.7098515034,-1.4383655787,-0.6667670012,-1.3716266155,-0.1098348424,0.7378181219,-0.0879292637,1.1370954514,1.7925657034,-0.2012682259,-0.2694388330,-0.4670928419,-0.5821979642,-0.7644075751,0.6695444584,0.0306869764,-1.6800422668,-1.6340225935,-0.4223179817,0.7865099311,0.1305941641,-0.7552067041,-0.0254859049,1.9027297497,-1.3717647791,0.7811303735,-0.2215402573,2.1291007996,1.1312514544,-0.2409127802,0.5062365532,-0.5534172058,-0.3004655242,-0.5346511602,-0.8108330965,0.3282672167,-1.0185667276,-0.3533863127,-0.8215651512,0.2005594671,1.8420407772,0.3635246158,0.7861365080,-1.0597175360,-0.1873623729,0.7724383473,1.2314850092,-0.7266256213,-0.2569041550,2.0182263851,0.4881033897,0.3779078424,-1.1142752171,-0.5767669082,0.1944286525,2.3854238987,0.7869626880,0.8664147854,-1.8784008026,-0.5955214500,0.6883810163,-0.6199914813,-1.1991696358,1.2152301073,-2.4070537090,-1.0307068825,-0.6643091440,-0.5183378458,-0.4899657965,-1.8663284779,1.7655661106,-0.7547045350,-1.4823474884,0.0760180503,-0.1749891788,-0.3796023130,0.0475712158,1.7384783030,-0.2914048135,-0.1437018216,0.1325060278,-2.3676879406,-0.2258290648,1.6418195963,-0.7734972239,0.7461740971,0.6074157953,0.4117653370,-0.6067041159,-1.6472831964,1.7200328112,0.0232646484,0.4374954998,1.2455363274,0.5894487500,-1.7584183216,1.0120782852,1.0967348814],[0.0198814012,0.3985802829,0.6743020415,-0.8145176768,-0.7799610496,0.3777892590,1.0087720156,2.0344851017,0.0204913002,-0.0241909269,-1.8100771904,0.7795275450,0.2824576795,-0.6737608910,1.0618784428,-0.3735115230,-0.1141227409,0.0508635230,0.1573044062,-0.7637298703,0.1603048593,0.1740360856,-0.0722323060,1.0070261955,1.3884991407,0.5985660553,-0.5918496847,-2.4490859509,0.3502865136,-0.7734159231,-0.7966606617,1.0198645592,-3.6166062355,1.3282010555,-0.9895777106,-0.0881013423,-0.5323204994,-1.1653339863,-0.9234332442,0.1528675258,0.6556922197,0.2687607110,0.2779617310,-0.4949930906,-1.0904607773,0.9958393574,0.6449023485,-0.0139402952,-0.6758316755,0.0389272645,0.2130638659,-1.5142279863,0.5219237804,0.6738876104,0.0340359621,1.2885986567,0.3472016752,-1.3768179417,-1.1709387302,-0.1141757742,0.3047106266,0.3039871454,-0.5379764438,-0.0836184621,-0.8551713824,-1.5300897360,-0.6361967325,-1.9284130335,1.1020956039,-0.2246899605,0.5269551873,-0.9494376183,-0.9701120257,0.7764113545,-0.2850833535,0.5099344254,-0.1375637800,1.1417900324,-0.2920076251,-2.0219731331,-0.7772913575,0.0708815753,-0.3063801825,-0.4740124345,-0.2298611403,0.7288610935,0.2877394259,0.1773329675,0.4523156583,0.6529164314,-0.7750585675,1.6021527052,0.8939141631,-0.0089290179,0.2659767866,0.1753713489,-0.5379931331,0.2873095870,2.2902197838,0.9296517968,0.3582972586,-1.6173806190,1.1959173679,0.3798225224,0.3370334506,-0.6950842142,-0.1877526492,-1.0495693684,-1.9517418146,0.9486764669,1.3959928751,-0.1565124840,-0.8272699118,-0.6829131842,-2.2610342503,0.2792412639,-0.8317098618,-0.0329254083,-0.0633183643,0.0907527655,1.6180012226,0.1789626926,-0.5308724046,-0.1154055297,2.2669715881,1.0755400658,-1.2605806589,-1.3714374304,0.6071957946,0.6227938533,-0.2147837281,-0.3323607743,1.2683321238,0.0792714432,-0.7269679904,-1.4837813377,0.1482709199,-0.8914306164,-0.3136545420,0.8342787623,0.5124506950,0.3868113160,0.6767075062,-1.5070546865,-1.1736109257,-0.4428335726,-1.2228515148,1.6584920883,0.8860940933,-0.3657595515,0.2537226081,-1.5250725746,0.7131353021,1.0954420567,-0.4071093798,0.6125251055,0.2761376500,-0.2350142002,1.3670639992,-0.3232049346,-0.7421479821,1.3723272085,0.1204612628,-1.4851864576,0.3227433860,-0.8716908097,1.0561125278,1.1365944147,0.4882908463,-2.3186235428,1.2475165129,-1.0494215488,-1.1936898232,-0.8012831211,-0.1370985806,0.7666711211,0.1403384358,0.3451739848,0.7921397090,-0.0043549207,0.4684241712,1.0302612782,-0.1617009193,0.5881261826,-2.0054824352,0.3759513497,1.6423543692,0.8723756671,1.5837312937,1.2748030424,-1.9394344091,-1.3062525988,0.8021035790,1.2067923546,-0.6591643095,1.4958468676,1.4278705120,0.2870118320,0.2882831693,-0.9594781399,0.1612415165,1.7277601957,0.8202464581,1.2654410601,1.6296563148,-0.4528054595,1.2124234438,-0.9555593729,0.7217754126,1.3135437965,-0.9282189608,1.1860563755,1.3845413923,-0.0372342989,-1.0035890341,0.7181358337,2.2787775993,-1.8576666117,0.1417283565,1.1989032030,-1.0669670105,-0.5698820949,-1.3290554285,0.0011848081],[-0.6872947216,-0.0984507203,0.0053924564,1.8844401836,0.6896256804,-1.1454101801,0.2944171727,0.5787634254,-1.6376229525,1.6078158617,-0.6052874327,1.0842406750,-0.6343843341,1.1234267950,0.6614363194,-0.5274475217,-1.6983145475,0.1424814761,1.5018779039,1.6019635201,0.2406056076,-1.3378336430,0.3889307678,0.7279734015,-1.4437752962,0.2451101989,1.3260084391,0.4043070674,-0.6334703565,-0.0905047655,2.2889361382,-1.6737362146,0.4014829993,0.9645870328,-0.3940742314,2.2201044559,-1.1043834686,1.0660539865,0.6407681108,-2.9392998219,-0.1941942126,0.0725432858,-1.6567014456,-0.9918910265,0.0521232933,0.2113792449,1.2878824472,1.0472283363,-0.0297677517,0.6040073633,0.8790017366,0.5193853974,1.0043952465,0.1468016207,0.2064592689,-0.1191236898,-0.7948093414,-2.2682187557,0.3695141971,0.0789367929,-0.2442376614,2.2216601372,2.7800517082,-0.0808964223,-0.9864557981,0.1934024394,-0.1443898380,-0.5223660469,2.2616927624,-0.0752847716,0.6587827206,0.4554153383,0.4489373267,-0.7989477515,-1.1259423494,-0.9596457481,1.1747367382,0.0612658411,-0.9934889078,-1.4369478226,0.4548711479,1.2188794613,-0.8638844490,-1.6974016428,0.3228982687,0.4121471941,-1.0056366920,-0.4581864774,-1.6678930521,-1.3405390978,0.4931705296,-0.0594159402,-0.9209959507,-0.5359746814,-1.2050573826,-0.2416513860,0.0205140915,0.8051583171,-0.3039194643,0.4501983523,1.6569852829,0.8124082088,-0.2882144153,-0.0287091434,-0.8580284119,-0.5605448484,1.5844684839,1.6319321394,0.2433615923,1.8722686768,1.3541325331,0.2546465099,0.7951636314,0.4162734747,0.7153671980,-0.5670320988,-0.4803200364,-0.4373582602,-0.8799190521,0.5002053976,-0.4195810854,1.4329137802,0.7451282740,-0.2605960965,-0.1553190053,-0.1571091563,0.5476624370,-0.5343143940,0.3311653733,0.8613076806,0.4169470966,0.1481061727,-0.9566766620,0.3370350003,-1.6706675291,0.6576215029,1.1566696167,-0.2248429358,-1.4964891672,-1.4325475693,-0.4104234874,-0.0051225480,0.2184714079,0.8567622304,0.0752849728,-0.5590589046,0.0686668381,1.1503990889,1.2555845976,0.8356750011,-1.2136638165,-0.6048707366,-0.0624855757,0.0132966116,-0.0085763428,-0.8367882371,-1.0246938467,2.9996769428,-0.2250829041,-1.3752338886,0.4163417816,1.5430538654,0.8278558254,1.0882321596,0.6639254093,0.0291860960,-0.6185283661,-1.1871991158,0.5030259490,0.8486614823,-0.4215537012,0.5474107265,-0.1873566657,-0.5774893165,0.9267744422,-0.6131139994,0.6064761877,2.2037541866,1.8484804630,0.2465688884,0.4237998426,2.8766086102,-0.6650875807,-0.2597856522,0.4639374614,1.1446344852,-0.6116654873,-0.0218625776,-0.4358089566,-0.5646321774,0.5649581552,0.7810892463,0.7661207318,-0.0211674664,1.7043101788,2.2350108624,-0.1355992109,-0.4077370465,1.7714235783,-0.8152344227,-0.3246816397,0.2365811616,-0.8649201989,-0.8507444263,-1.2081509829,0.7433677316,-1.2326159477,0.6326366663,-0.6106397510,0.0574978702,-0.7415626049,0.9871259332,2.2205648422,0.6921710372,-0.8000081778,0.9471917748,-2.2180683613,-1.0951820612,-0.4547932744,0.7424654365,1.8997622728,-0.5095831156,1.2237663269,1.2456455231],[-1.2396967411,-0.3688427806,1.2640622854,-1.0342392921,-1.4049681425,-1.5210585594,-1.0076230764,1.3294607401,0.6158753037,0.7308965325,0.0848912075,-0.1969574690,-0.0819591358,-0.3404512405,-0.6221250296,-0.7510188818,-0.4768586755,1.2083463669,0.4217658043,1.4242808819,0.0852572173,-0.3053416908,0.9350500703,-0.4598187804,1.8786497116,1.3874312639,0.5443605185,-1.4051083326,0.1745427251,1.4360253811,-0.9045711160,0.0467361361,0.5182443857,1.4112025499,1.2951264381,-0.7757521868,-0.2411681116,-0.4241921604,-0.5597399473,-0.6066641808,-1.1611486673,0.2278152853,-0.0413084850,1.1212860346,0.6273789406,-1.0726985931,0.7191551328,1.1449415684,-0.5668084621,-0.8096838593,0.2828166783,-0.6697654724,0.0814880952,0.4383083880,-0.3998520970,0.6673691273,-2.2718055248,0.4157564938,-1.9803010225,-0.4021820128,-0.9504793286,0.5314682722,1.3823496103,-0.1732390821,-0.3971373439,1.3028131723,-0.8584709167,0.7786294222,1.4142960310,-1.2077190876,0.3213968873,1.8558900356,0.2112760097,-0.3709301651,1.1207209826,1.1596553326,-0.5884460211,-1.1001182795,0.3257922828,-1.1892958879,-0.7330425382,-1.5030636787,0.2551620305,0.2792249322,-0.3615036905,-0.6149381399,-0.5369812846,-1.1995630264,0.5322521925,-1.3133509159,-0.6877193451,-0.0793563500,0.7657913566,1.0005102158,-0.1015864313,-0.6221787930,-0.6660726666,-1.0681372881,1.1551531553,-0.1612300426,0.4519759119,-0.8223438859,0.7364921570,2.1589651108,-0.6829474568,-1.0340731144,-1.4623531103,-0.8314996362,0.9340252876,-0.1944855750,0.0987237468,0.0290169753,-1.8448588848,-0.7846073508,-0.0732088313,-1.7333260775,-4.0291380882,-0.5306329727,0.7409014106,0.2832567990,-0.9075359702,0.1050713211,-1.2936125994,0.9663115740,-0.5140835047,-0.5231055021,0.4298277795,0.8415327668,0.3716501296,-0.0323132835,-0.6347941160,1.3319286108,-0.8133419156,-0.9136698842,1.3833984137,0.8047218323,-0.8619208336,-0.9462947249,0.1797300130,2.6600546837,-1.1597884893,0.4171127379,-1.4330574274,-0.2685092986,-0.5563176870,-0.1373858452,0.2979919016,-1.6659470797,0.9449476004,-1.2770717144,-0.1901160926,-1.1533769369,-1.0791780949,0.9957335591,1.4861640930,1.0744752884,1.2719250917,-0.4285410643,-0.7043931484,1.0934284925,0.3522428274,0.3764531016,-0.3339338601,-1.4555534124,-0.4577522576,0.2179024220,1.1649961472,0.9591401219,0.6655773520,0.3939776421,0.5124578476,0.2843569219,1.3237973452,-2.1637573242,1.0497915745,-0.0111749861,0.5166006684,-1.2268714905,-0.1555034965,-0.0298048109,-0.8875045776,0.1314747632,-0.3267731369,-0.5512117743,0.6083897948,0.6339176297,-0.8001993895,-1.3872141838,0.4129168987,0.6903015971,-0.8204827309,-1.4886994362,-0.3166840076,-0.0989368260,-0.5029759407,-0.2094892561,-0.2207775265,0.6260372400,2.3226847649,-0.8254155517,-1.2173273563,0.4000590742,-0.1477404535,-0.2987808287,0.7859148383,0.5849666595,-0.2389931232,-0.3345387280,-0.4545707107,0.0136067895,2.3920662403,-1.5474188328,-0.0663272217,-1.2421488762,0.7367187738,-1.2968086004,0.4109822512,1.8854326010,-0.3313354552,-0.7640479803,-0.1868978590,-0.2272865027,0.1732350737,0.0523185283],[0.1169780940,-0.2806419432,2.2475368977,0.2511340380,-0.5130397081,-0.5074169040,1.0990157127,1.2466640472,0.3869113028,0.3809750080,1.1946322918,-0.3047500253,-0.5252722502,0.0020444905,-2.1285502911,0.2623390555,0.5451027155,1.8167289495,-1.1404950619,0.5926309824,0.5969594121,-0.0148727661,0.0002652447,0.4641184211,-0.4436666667,-0.6841523051,0.5644008517,0.1762598157,-0.3095212281,0.9205504656,-0.0046450230,-0.4804340601,0.5649686456,-0.3629473150,0.1853141338,0.6242626309,-0.6242429614,0.3195052743,-0.2832978666,-0.7747485638,0.6012946367,-0.1605652124,-0.4696919918,0.7488157749,1.0230870247,0.5574153662,-0.6297346950,-1.2690787315,-0.6727031469,-1.1326258183,-0.9907282591,-0.4507670999,0.4559905231,0.9178841710,-0.6517288089,-0.4528707564,2.0510644913,-0.4466638565,0.3832630217,-2.3713276386,-0.7958715558,1.5951139927,-0.6162417531,-0.7024154663,-0.7358942628,-0.0514691435,-0.2618040740,0.3440458477,-0.2452831417,-1.1451121569,-0.2014667243,0.4369078875,0.3151161969,0.4121712446,0.2650465369,0.9768393636,0.9651262760,-0.1851613075,0.7949992418,1.2735664845,0.9264928102,-0.3236273825,0.4478474557,-0.3741316497,-0.1214300245,0.0958605856,-0.4238029718,-0.4743156135,-0.8259438872,0.7757176757,0.3365464211,-1.4995009899,0.7254462242,0.8834500313,-0.6244534850,-1.6591092348,-1.0582569838,-0.4764253795,-1.7733248472,0.9393494129,0.1969945282,0.1639864594,0.1929222494,1.4174526930,-1.1465125084,0.8462514281,0.5178537369,-1.0700050592,1.4103268385,0.2136932462,1.4934782982,1.8879579306,-0.7683445811,-0.8442191482,1.9063091278,-0.6955130696,1.6429194212,-1.1053818464,0.1641834229,0.1306985021,-0.1009309143,-0.1554664522,0.6429009438,-0.3547316790,-0.3141651452,-0.7517613769,-1.2651112080,0.4944086671,-1.7528989315,-0.0695738867,0.3000741899,-1.3996940851,-1.2547707558,-1.1402056217,-0.5491457582,-0.1278792620,0.6167907715,0.4023316503,-0.2770686746,0.8350918889,-1.2336196899,-0.8996614218,0.8900222778,-0.1889329702,1.2801160812,0.6478344202,-0.6443380117,0.3230400383,0.5593442917,1.1015638113,-2.2827229500,-0.5276812911,-0.3871492147,0.8352118134,-0.2477561980,-1.7110137939,2.6469244957,0.3301985562,-1.3280184269,-0.8027424216,-0.7288699746,-0.8320157528,0.1406433880,0.0071848044,0.6772457361,0.7906083465,1.5656526089,1.8632999659,1.4107118845,-0.1728944927,1.5014334917,0.5151723027,-0.5827595592,-0.6851282716,-1.1576546431,-0.7370390296,1.1400220394,0.1113723516,0.8906426430,-0.1768481135,1.7654592991,-0.0840447322,0.6247481108,-0.9882045984,1.0037338734,0.0724427253,0.1596299112,-0.5648909807,-1.0839179754,-0.5491756201,0.8615249991,-2.6529829502,-0.5975295305,-0.2969869673,0.8291987777,0.9888102412,-0.0241858158,0.0042725853,-0.6767337322,0.4008062184,-0.1150729880,0.6527836919,-1.3201049566,0.5537902713,-1.0040954351,-0.8293229342,0.4181952775,0.2711547315,0.7081342340,-2.2100968361,0.4100460112,-1.2368681431,0.0858222991,-0.6589938402,-0.0142544145,1.3100295067,0.4607637525,-0.3612421453,2.7282576561,-0.4968651235,-0.4072162211,0.4305685759,-0.9169256687,1.4734205008],[-0.6685152650,-0.8737441897,-0.5534303188,0.5200399160,-1.1848164797,0.3177312613,0.0415630788,0.5504212379,1.3617442846,0.5164669752,-0.0615786836,-0.6427606344,0.1296616346,0.3920331001,0.2797629535,-0.7505016327,0.0564294457,-0.1437963694,0.2603136599,-0.9009698629,-0.2277601510,-0.7939273715,0.3479827642,1.3618816137,-0.5092176795,0.3851687908,-1.3853656054,0.7346545458,0.5953921676,0.6145576239,0.5108073354,-0.7996370196,0.2773365974,0.2169533074,1.1366590261,-1.0889068842,-1.0152673721,0.7809972167,0.7224917412,-1.3904248476,0.2733822465,0.6747696400,-0.0467090495,-1.1458916664,0.7344310880,1.2709969282,1.0786265135,1.3192250729,-1.7286123037,1.0670614243,-0.0010136778,-0.5207394958,-0.3519857526,-0.0657513589,1.2044017315,-0.4093344510,-1.0834403038,0.0430439971,-0.1649459302,-1.7983412743,0.6118800044,-0.7838023305,1.0087105036,-1.5372934341,-1.0053912401,0.7615340948,1.9778778553,0.9270530343,1.1736884117,0.0775798485,-0.4577569664,-1.3965622187,1.8189158440,-1.9773858786,-0.7674823403,1.9165551662,1.1599441767,1.7834349871,-0.1174152419,0.9704736471,-1.1975587606,-1.5968848467,-1.7459049225,0.5906978250,1.1235213280,2.0732831955,-1.0014953613,1.6119662523,-0.4356044829,-0.1917356253,0.7411944270,0.5390760899,0.4717051387,-0.6492152214,0.8157361150,-1.8203171492,0.2394956350,-1.0516985655,-0.1302240044,0.4464549720,-0.8567866087,-1.8950551748,0.0301094223,-2.3413836956,-0.9489056468,-0.9709588885,0.1027731523,0.6144888401,1.4732381105,1.7237547636,-1.2133656740,0.1944560856,-0.0028214678,0.4646196365,-0.4514550865,-1.0959115028,1.0776283741,2.0255107880,1.2845118046,-0.0877527371,-0.5224267244,1.7399580479,0.7644951344,-0.8440381885,0.5369064808,-0.7958087921,-0.0592793599,-0.6017829776,1.6896573305,-0.6590547562,-1.0762586594,0.4362809658,0.0668928027,2.0457201004,0.5018579364,0.5019522309,-1.1521987915,-0.2816174030,-1.7676477432,0.0686004385,0.5971288681,1.5740783215,-1.5005308390,0.2959667742,0.0992885381,-0.4789198041,-0.4450925291,1.6048885584,1.9666018486,-1.5215646029,0.6102808118,-1.0968976021,1.1952879429,1.0654774904,0.0481246337,1.0292104483,0.6108282804,-1.9799226522,-0.1822997779,-1.0944604874,0.3795200884,0.6275728941,-1.0811973810,0.5579342842,1.4074038267,0.2341215163,-1.9398125410,0.1486764103,-1.7583855391,-0.8627243042,0.8016073704,0.2244357616,-2.3210380077,-0.1822131872,0.2372455597,-0.0160580892,-0.6138968468,0.2037782669,-1.0922092199,1.6620970964,-0.1207506806,1.4912840128,1.9458825588,-1.0110816956,-0.3563345075,-0.7647183537,0.3050349653,-0.0155192744,0.8703370094,-0.2007468790,-0.0845041946,-0.2028828859,-0.7978484631,0.1054977104,0.3118167818,-1.6269580126,1.1409548521,0.9135065079,0.1632136852,-1.0845395327,-0.8973467946,-0.1036828682,-0.3324064910,1.1789542437,-0.3770294487,1.5620265007,-1.3538585901,0.1679062247,0.1029169038,-0.4538665116,0.0385230370,0.7132785320,1.3319801092,0.2151621729,-0.6791537404,0.0503540374,-1.7433155775,-0.2129102945,2.3486294746,0.6435081959,0.0417948477,0.4045048654,0.5760819316,0.2136069536],[1.5347999334,-1.6873236895,0.6115571260,-0.0327798501,-0.5411595106,-0.4453768730,-2.0417563915,1.9538553953,1.5702822208,0.9667896032,0.0068834368,-1.4329059124,1.7121027708,-1.2107728720,0.8673722148,-0.0087148799,-0.2151390463,0.1782071441,-1.7905542850,1.1115261316,1.0436437130,-0.8217792511,-0.8795315027,-1.5809032917,0.3619697690,1.7345861197,0.5589722395,1.2194062471,1.2847113609,0.7623335123,-0.4450575113,-0.5541297793,2.3819768429,1.0045148134,-0.2210227102,0.0310857967,-0.3054305315,-0.2346151024,-0.2350860834,-1.0135045052,-0.4001455605,-0.4735128582,-2.3997213840,-1.6969120502,0.6540305614,0.7910987139,2.6275141239,-0.5053266883,-0.6437256932,0.4331637621,-0.1179857254,1.3676888943,-0.3311014175,0.0152680846,-0.0236807577,0.9302479625,0.6600639820,-2.4037752151,-0.9464588761,0.0438113771,-0.0725328401,1.0720617771,1.0761135817,-0.2737096846,-0.5588893890,0.2598156333,0.9769091606,-0.6441979408,0.7905815244,0.8603966832,0.2026889175,1.3673669100,-1.3858262300,0.2233661562,-0.1094158739,1.4453797340,0.0151959099,-1.6613028049,0.7575907707,-0.1962304562,-0.1996283978,-0.7416828871,-0.7916600108,1.4287346601,-0.4484005272,0.6876532435,-0.6248690486,0.3298631310,0.9127324820,-0.9684745073,1.3250454664,2.2452054024,-0.9814113975,0.0123218549,-0.0707478821,-0.0363855138,1.0943500996,-2.1510896683,0.5944954753,-1.0275869370,-0.7499034405,-0.9592984319,0.5362375975,-2.1820952892,0.6188629866,0.7973918915,-1.7052952051,-0.7358623147,1.4108181000,0.3214335740,1.3040961027,0.6007806063,0.6384257674,0.5477932692,-0.0138832955,-0.8589347005,0.8096865416,0.3102856278,-0.2026515752,0.9718768001,0.6564921737,0.1603172570,1.0024086237,-0.9002149105,-0.8006245494,-0.9411728382,-0.1576966792,1.1415449381,-0.0577506274,1.1866910458,0.3541076481,0.6679896116,1.3162442446,-0.2557232678,-0.3763893545,0.2416353524,0.7076978683,-1.8903992176,1.1509604454,-0.5034546256,0.2765678763,-1.8116631508,0.0736504048,-0.5345338583,-0.2187395692,0.8730805516,0.2851711512,0.4014697671,0.5744179487,-0.9496762156,-0.9646509886,-0.2687695622,1.8421496153,0.8335016370,-0.6742351651,-0.3898480535,-1.3010454178,1.2313911915,2.5872275829,1.0136504173,-0.1823874265,-1.9427940845,0.0121092172,1.0977327824,-0.9339600801,0.0772805214,-0.7465289831,-3.4953188896,-1.5749645233,0.0567488112,0.4932927489,0.4036336541,1.5102025270,-0.3151389062,0.4847653508,2.2489926815,1.2595604658,0.4640718997,-0.2695495784,-0.7470452785,0.7966135144,-1.2423455715,1.0008935928,0.1510142535,-1.3154373169,-0.5294527411,-1.4460440874,0.0734137148,-0.9246321321,0.0041351891,0.3118188381,-0.4110965133,-0.4202423096,0.1939986348,-1.1355917454,1.0640664101,0.1575757712,0.2295729369,-0.5353895426,0.2008222491,-0.0894518197,-1.2742060423,-1.2324142456,-0.0447078608,-0.1281779855,-1.2925449610,-2.0690896511,1.6433898211,-0.6176621914,-1.8369152546,0.8185908198,0.3368025124,0.2383316606,-0.3955019116,1.0942413807,1.0905435085,-0.6878349185,-0.8977550864,0.5481200814,-0.8589039445,0.2464670092,2.2294669151,0.3704622686,-1.6462700367],[-1.0096625090,-0.7288820148,0.0720273480,-1.4444570541,-1.4314428568,-1.0962486267,-0.6008301377,0.6626305580,0.3884952664,0.1886538416,-0.0186148025,0.9334484339,0.5448017716,0.1605713814,-0.5728644133,2.3174045086,-1.1572582722,0.3675760925,1.2822412252,0.6096727848,1.0833481550,-1.4809343815,0.6521091461,0.4051893651,-0.0447794273,-0.4249199927,0.9600380063,-0.1428283155,0.1274694651,1.0010346174,0.9665576220,0.7157662511,0.1009588018,0.9976038933,-0.3180441856,-0.7530906796,0.5247619748,-0.1527070850,1.2734057903,0.4071713686,0.9690848589,-0.7642352581,1.0456936359,0.6039264202,-0.6427201629,0.2151586413,-0.3214791715,-0.4597530067,0.5271275640,-0.6031562090,-0.3741852939,1.9624742270,1.3004586697,-0.9262704253,-0.5459587574,1.9289519787,-0.1279011965,-0.8018730879,-0.3704701662,0.1392263919,2.2456700802,-0.2131865770,0.2803228796,-0.2014343739,0.4005430639,-0.7303941250,-0.6890636683,-0.1513775736,0.8517774343,-1.3185310364,-0.8762566447,-0.1303947270,0.9660004377,-0.4019751847,0.4808919728,-1.3781931400,1.8092302084,1.2631082535,1.0598397255,1.1649284363,-0.5299757719,-0.2302153260,0.4181659818,1.0113337040,-0.0762433708,-0.1666911095,-0.0848165229,-2.1126856804,-1.9476577044,-0.3496899605,-0.4967019260,0.5956168175,-1.2740421295,1.6090040207,1.1050789356,-0.3681654334,-0.4480681121,-0.2363000661,-0.2702892125,0.8397852182,0.4432892799,-0.6930931807,0.8435317278,1.0990096331,-1.4728732109,0.3252948225,0.1570708901,-0.2961833179,-0.0988946781,-0.6090566516,1.7864310741,0.3255374432,-0.7772111297,0.8540177941,-0.3736964464,-0.1458571851,-0.4800638556,0.6023507118,1.3466943502,-1.3057725430,-0.6299296618,-0.1963314563,-0.6180570722,-0.6634348035,-0.9157820344,-2.1389577389,-0.5336833596,1.3248667717,-0.6429237723,-1.1559734344,0.6903609633,1.2517040968,0.9739649892,1.2991584539,0.2655103207,0.3484174013,-1.2541536093,0.1894829720,0.8233504891,-0.0764275566,-1.2177714109,-0.8417980671,0.8581524491,0.0035927813,0.6365355849,-0.8042198420,-0.1388953626,-1.6326507330,-0.6042947769,-0.9669717550,0.2938333750,0.9714415073,-0.1223727614,-0.8723503947,-1.1460812092,-0.2133648992,-1.1128445864,0.9363123178,0.1991896331,-0.8822627068,0.1223424897,1.3291049004,0.0668822005,0.1119093895,1.6684954166,-0.9703634977,0.8244425654,-0.3810819983,0.6499918699,-0.4449295998,0.5241637826,1.8430259228,-0.1138890088,-0.7317150235,1.5404244661,-0.6431032419,1.0746787786,0.9530599117,0.2035676390,1.7534618378,0.4639586806,0.0214881431,0.3588767946,-0.4447282851,1.6261456013,1.7030267715,-1.4624485970,0.6691627502,-0.4120528400,-1.2016181946,-1.0315819979,-1.8414787054,0.7353565097,0.0985109657,-0.5989940166,0.6289370060,-0.7668067217,-1.0768667459,1.1415388584,0.2427966297,-1.5039368868,0.2121663541,1.8831692934,1.7689586878,-0.0031755727,0.1881536543,0.0798226669,0.4581492841,0.2642267048,0.9306536317,-0.4983684719,-0.1663818955,-0.9144891500,0.9846673012,0.2117036432,-0.6294923425,2.2421128750,-0.1303592473,0.4997661114,0.9499601126,0.7253001332,0.2831318378,-0.4090300798,1.1107546091],[2.2380344868,1.3806735277,-0.3407057524,0.4970670044,1.3422850370,0.1040709913,-0.1962797344,-0.6731407642,0.3965022266,1.6650975943,-0.6868150234,-0.9100295305,1.5729589462,-0.1079028100,-0.8686639071,0.2137991190,-0.8141530752,-1.2586628199,-1.0130791664,-0.1064461023,1.3744591475,-0.5557163358,-1.4183386564,0.4970445335,-0.1264280081,-0.6952810287,0.7875682116,0.9634685516,-0.0873802453,0.4830078781,-2.0697469711,-0.2951569259,1.9275074005,-0.2655444741,-0.5246491432,0.6961337328,-1.1402201653,-0.8014924526,0.4705885053,1.3276981115,0.9487622380,-0.1226340607,2.2388117313,0.9282962084,0.8358730078,0.5729625225,1.3172289133,-0.7303490639,0.1824570298,-0.6966341138,-0.8805764914,0.7411084175,0.2305240929,0.0980034173,0.8006091118,0.2612165213,1.0207033157,-0.0486149713,-0.1296018958,-0.2046490759,0.3058603108,0.1007243693,-1.8249213696,0.5788246989,0.5505513549,0.8283489943,2.3347523212,-0.3820807934,-0.6665652990,-0.6986245513,0.5523877144,0.5897316337,-1.7852385044,1.5092515945,0.5711547732,0.3464311659,0.2958600223,2.0925042629,-0.2321517169,-0.1434054524,0.5728091598,-0.2187698931,-0.4759446084,0.1177005395,0.2013361752,0.5981849432,-1.9799973965,0.2529041767,-0.5476834178,-0.1730593890,-1.7854814529,-1.1705741882,-0.3581634760,0.3817416131,0.7512064576,-0.1027551740,0.6353870034,1.3960314989,-1.2177100182,-1.2762954235,0.2829504609,-0.3222350478,-1.9788143635,1.1852393150,-0.8120482564,0.2017181963,-0.7778249979,0.1324888170,-1.7530630827,-0.3840102851,0.5514950156,-0.9082639813,0.2709457278,-0.2724392414,1.7407710552,1.0251507759,0.5038827658,0.0599922501,0.2778967619,1.1140643358,0.5126416087,0.9094967842,-0.2608771026,1.0770685673,0.3795000911,-0.1550950557,0.4098328948,0.0182807632,1.7432143688,0.0442269817,-0.0733183324,-1.7420239449,-1.0372033119,-1.4505765438,-1.0695979595,0.3520697057,-0.1199650243,0.1623314768,-0.4059309363,0.2590332925,0.3168719411,-0.2985204160,-1.1141160727,0.4796123803,0.5619376302,-0.0879992098,-0.2896755636,0.0345673114,-0.6017387509,2.2942187786,-0.4575514793,0.4433886707,-0.6393599510,-0.7018532753,-2.0840408802,0.2744977772,-1.7085973024,2.0885903835,-1.0704900026,0.6227512956,-2.3643672466,0.4737828970,1.8228685856,1.2083232403,0.5655582547,1.1943204403,-1.6110794544,-0.8104713559,0.6233947873,1.4821898937,-0.3512924016,-0.9832256436,-0.3109658360,0.2102089971,0.0661011934,-0.7598971128,0.2737687528,-0.3565664291,0.0649320632,-0.8309890628,0.8115738034,0.5818820596,-1.2470250130,1.5066984892,0.0355780944,0.5166119933,2.2526850700,-0.8431701064,-0.0671550483,-1.1754525900,0.5172681212,-1.4026856422,1.2922587395,1.1752586365,0.4308859110,-0.1489560753,1.2196234465,0.2266667634,0.6856338382,-0.0718977973,-0.7315303683,-0.7587987781,-0.6444988251,1.0108252764,1.7422361374,-1.0566121340,-0.1381981969,-2.4956552982,-0.6139257550,-1.2556135654,3.1750850677,0.2591003776,0.2450738251,0.3005379438,-0.9752496481,1.7764663696,-0.1214622557,0.5261299014,0.2342363894,-0.0667102411,-0.9946687222,-0.7998950481,-0.1393250823,0.5439608097],[-0.3280489445,-1.6486464739,0.0773164555,-0.6094115973,0.7776465416,-0.7698317170,-0.2485616952,0.7960718274,0.2313920408,0.0460020714,-1.7142246962,0.6232043505,0.2541713715,-0.4534405470,1.7899509668,0.1362471282,-1.9164524078,1.3635959625,-0.8852525353,-0.3090792894,0.3931639791,-0.9311147928,-0.3980593979,1.9160156250,-2.4146127701,0.9591246843,0.0804828331,-1.1393139362,0.5533338785,-0.0600930825,0.2490953207,0.3519489169,0.7660022974,-1.6150187254,0.6105470061,-1.2359455824,-0.8698206544,0.3892658055,-0.7539950013,0.6950904131,-0.9604725242,0.4308103025,-2.3256311417,-0.7111156583,-0.4337917566,0.8061956763,-1.3608552217,0.4315817952,-1.3107632399,0.5574386120,-2.2791669369,-0.9615024328,-0.0104346648,1.8257387877,1.4231183529,0.2945812345,-0.1324300915,0.2946366966,1.6782826185,0.6541477442,0.4029373229,0.4747828245,1.1353145838,0.3953845799,0.1936890781,0.4062623680,0.4179663360,-0.1990364641,0.2323853821,-0.1627434790,1.2631576061,1.0686953068,-0.2658118308,0.6809504032,1.5407712460,-0.6695981622,1.8600367308,0.4357393682,-2.5560443401,-0.5719974041,-0.8085638285,-0.2786338627,-0.7596287727,-0.7966396809,-0.6347700357,0.6346846819,-0.1229087114,-0.4061551392,0.4961775839,-0.9298028350,1.7950794697,-0.4195333719,-0.6384252906,0.4334599376,1.2472872734,-0.2829909325,0.1914219707,-2.3444113731,0.1594360918,1.2696460485,0.4959804118,0.2380602211,-0.1448186785,-2.1736934185,0.0322666392,-1.1135591269,0.1910399944,0.5964226723,0.6934137344,-0.3309519887,0.5232272744,0.6514256001,0.5427439809,-1.2049249411,1.2035021782,-2.1379439831,-0.0123552503,1.6907293797,1.3622356653,0.0854423568,-0.1239003465,-1.1248239279,0.3118157685,1.4227129221,1.3931421041,-0.5201041102,-0.6520749331,-0.1951945126,-0.0238409471,-1.0981628895,0.1196731627,-1.0028203726,-0.0433323942,0.1475336701,-0.2034588307,0.0879188478,-0.1838414073,0.8159422278,-0.6409673691,0.0307073314,-0.8225424290,0.1864877790,-1.2164173126,-1.0154621601,-1.4576314688,-0.7777158022,-1.0057930946,1.5576810837,1.0352222919,1.8638725281,-0.0097064106,0.8054191470,0.1223727986,-0.0801445544,-1.4936922789,-0.4619750381,0.0596807078,0.3278731108,-1.0393821001,-0.4951561689,-0.3413927257,0.1904700100,-0.1210603565,-0.1594241560,-0.5338973403,-2.1179163456,0.9923138022,1.6756576300,-0.5539448261,-0.4309539795,1.2551391125,-0.9311563969,0.6251273155,-0.0564246289,0.8008189201,-1.9777007103,-0.3098180890,0.7816771269,0.6166989803,-0.6998443007,0.4654302001,0.0343916118,2.4334328175,0.5027511716,-0.5978142023,-1.1934173107,1.1687659025,-1.3383669853,1.0300269127,-0.0677551478,0.3900418580,-0.4028481245,1.5647825003,0.4914836287,-1.0139048100,0.7788290381,-0.9364992976,-0.4769464433,-2.2719848156,-1.2316024303,1.4936300516,1.1475230455,-1.6251858473,-0.1341409087,-0.6324136257,0.9583027363,-1.8046439886,-0.6657356620,-0.2849379182,1.1019287109,0.3203172982,0.8305735588,0.8801663518,0.2541929483,-0.7876218557,-0.4848353863,-0.6038257480,-0.3463531435,-0.4620301723,0.3399290442,0.4784194231,-1.4630919695,0.5666418672,0.6698990464],[-0.1886400282,-0.0589710884,-0.8409474492,-0.5538551211,1.1771206856,0.0073921932,-0.0275823288,0.4827136993,2.1394298077,1.0037149191,1.3728575706,0.5932582617,0.6228179932,2.2393188477,0.6417520642,-0.6442270875,0.3314913809,0.4365530312,0.9575701952,0.1253282130,-2.6363835335,-1.1195566654,0.1741666943,-1.3815630674,-1.3405553102,-0.9259150624,0.7431330681,-0.5993356109,-0.9352449775,-0.2250264287,-0.3261758387,-0.9840246439,0.8795168996,-0.0585330613,0.4813308418,0.6932357550,-0.6614339352,-0.6948084831,-0.5029956698,1.0634011030,1.9179027081,-0.1568887830,1.2873591185,0.7845811844,-0.9608737826,-1.5719852448,-0.3008854091,-1.5339412689,-0.2134107649,0.4248637855,-0.3359931409,-0.1320641041,0.8435388803,-2.7100458145,-1.7172553539,0.8296487927,-0.9835475087,1.5917860270,0.1721152812,0.4304965138,0.7664323449,-1.2311582565,-2.3880450726,-0.4940323532,-1.0391459465,-0.7237001657,1.2727987766,1.6193475723,-1.0287649632,0.7299798131,-0.6349011660,-0.0274767578,-1.5900055170,-0.4706409872,0.7383552194,-1.4446251392,2.9851384163,-0.5311147571,0.0925131291,1.0023488998,0.9239675403,-0.3110629916,-0.3885660768,-0.5063413978,0.2894129455,0.8276406527,-0.4842105806,-1.5078349113,1.2256025076,0.8286691904,-0.3395346701,-0.0520976894,-0.6069356203,1.4886635542,0.8925319314,0.5489603281,0.4804161787,1.3660881519,-0.4378204644,-0.7837976217,0.3279939890,-0.4669124186,0.5884266496,-0.1041898355,-0.7743563056,1.9065687656,-0.1401365250,0.0404757708,-0.4654856622,0.1470849812,-1.2212413549,-0.2502074838,-0.5511439443,0.1273302287,-0.8361667991,0.5060011148,0.4187401831,-1.0274453163,0.1833582371,0.1491656005,0.8520369530,-1.0309195518,-0.2227566093,0.0463354513,-1.7734726667,0.2360962629,1.2328624725,1.3344942331,-0.6834138036,-0.9213144183,-0.2375682145,-1.0676547289,-0.7755971551,-0.1519525945,-1.1810057163,0.6070850492,-1.2324193716,0.7946123481,0.4432314336,-1.2385253906,0.7860576510,-1.2088580132,-0.6392730474,-0.8253847957,2.0636448860,-0.3773825765,0.2946148515,1.2927073240,0.3535074592,-0.4379634857,-0.0179980081,-0.0072805942,2.2701530457,-2.1468765736,-0.3751948774,-1.4295772314,2.2897188663,0.7464865446,0.2147173584,-1.1608482599,1.5358866453,0.5187512636,1.8804030418,1.5796387196,1.0487182140,-0.8676830530,-0.0556595251,1.3028850555,-0.5795240998,-0.5162953138,-0.0655466840,0.9422415495,0.7499783039,-0.9180640578,0.4498133957,1.1590911150,0.4048582017,-1.7504346371,2.2532100677,0.0281716362,-0.6220969558,-1.5183184147,0.9161355495,-0.8907647133,-1.3098375797,-0.3318792582,0.2209775299,-1.2912184000,-1.5370779037,0.2036247104,2.0972721577,-1.0611443520,0.9589093328,1.1591913700,1.0168168545,1.2855957747,-0.0241216458,-0.8847688437,-0.2365839183,0.0415988378,-0.1515546739,-1.0070446730,-1.1214296818,-0.3417044580,-1.3171265125,-1.5975716114,0.8810922503,0.4943160117,0.0729970559,-0.0358922035,1.4424666166,-1.5872232914,-1.3994543552,-0.4739158154,-0.4085120559,1.6491034031,0.3564356565,-0.5753759146,2.0592122078,-1.0333622694,2.1544709206,1.3682296276,-0.7064908743,1.7868995667],[-0.9723177552,-0.7266247869,0.9715178609,0.4088996947,-1.7122020721,-0.4308840632,0.3252696097,0.2743642926,-0.8869826794,-1.0270365477,-0.5657429695,0.0832255706,0.0655450523,0.4747250974,-1.3726384640,0.4130788743,0.6846900582,-0.5520215034,-2.0530512333,-2.1558058262,0.1264140904,0.3746607900,1.8946000338,0.1379344761,-1.2841641903,-0.3011165857,-1.3064757586,0.6817041039,0.9905909896,-0.3322680295,-1.0966534615,-0.5721865296,1.1613034010,0.4173618853,0.2521701753,0.0170840435,-1.1218461990,1.3891819715,2.0246393681,-0.3689926863,1.2998167276,-0.0202535987,0.4367121756,1.0193365812,0.9042968750,0.6680484414,-0.6065965295,-0.5022114515,0.5810588598,-0.3901543915,-1.1834454536,0.8150207996,-1.1420608759,-0.3598107100,-1.6044889688,-2.4384107590,0.0117071439,1.9975863695,-1.3817124367,-0.3072476387,0.0048268959,-1.7112979889,0.9031231999,-0.2966916263,-1.0396625996,-0.1289494932,-0.7083650827,-1.4692304134,-0.5094682574,1.9392372370,-1.6139283180,-1.1871839762,-1.1809390783,-1.9714508057,-0.9087123275,-0.2333478034,-0.7533107996,-0.9897646308,1.4758378267,0.1605499536,-0.0507054329,-0.2238054425,0.8934520483,-0.0074833194,-0.3475907147,0.4808534384,-1.3428298235,0.0425792374,-1.4493671656,-1.1105960608,-0.3286893964,-1.1615190506,0.7743390799,0.5156641006,1.4899449348,0.7932764888,-0.3827163875,0.9504209161,-0.1228832677,-0.5156506896,-0.2268660665,-0.6296691298,1.0760799646,-0.4033052623,-0.7914856672,-0.2877828479,1.5812536478,-1.1596530676,0.0779661462,0.6545679569,-0.2202242166,-0.7564415932,-0.0417143740,-0.3171341121,-0.4582436979,-0.9056224823,-0.9524829984,1.6070226431,-1.2132416964,2.2914252281,-1.1252080202,-0.6691984534,-0.2136556059,1.5424201488,0.3239894807,-0.1936365366,-0.3805269301,0.3153375685,-0.0132743232,-0.7174763680,-0.6745864749,0.5147384405,0.2804582715,0.8614385724,1.0986897945,1.7333666086,-0.7799022198,-0.9911209941,0.9604320526,-0.8721246123,-1.6883252859,-0.8845354915,0.7923492789,-0.5049350262,2.0076103210,-0.0851265863,0.7332333326,-0.1772662252,1.5707787275,-1.1044410467,-0.1525815576,-0.9408708811,0.4330949187,-1.2818670273,-0.3055057526,-0.1675800681,0.0284603387,-0.8147291541,-0.4210108519,0.7458789945,-1.1510759592,-0.5196728706,1.2945172787,0.3134264946,-0.5981710553,-1.7199571133,-0.9573861957,0.3585983515,0.1577864438,0.0976585224,0.9235061407,-0.4882372022,-1.2233790159,-0.3873362243,0.1936537027,0.8459808230,-2.3967981339,-1.7979650497,-1.3401989937,-0.6894354224,0.0410711132,-0.7186352611,0.0600788631,1.3476153612,0.3278820813,-1.3045172691,-1.3771083355,-1.7876477242,0.2890119255,1.3028764725,-0.3252322376,-1.6646826267,1.0996216536,0.8495755792,1.1889733076,-0.6801342368,-0.9674876332,-0.5559958220,-0.8111504912,1.2361888885,0.2234009504,1.1607664824,-0.7407107949,-0.2627224922,-0.6727229357,0.6591702700,-0.1328006536,1.9366884232,-0.0906984061,0.7591729164,0.1739566326,0.7143741846,0.5020965338,1.7826042175,0.6204472184,-0.5581421852,0.2132278383,0.3122832775,-0.9909564257,1.2599604130,-0.1934482753,-0.9713167548,1.1571723223,1.5756896734],[-0.5407086611,-0.6157051921,-1.6384927034,0.9595841765,1.1959745884,-0.2396019995,0.5296090841,0.0760802999,-1.4119911194,0.2633403540,0.8130000234,-0.0615768097,-2.6958682537,-0.0774267390,0.2242079079,-1.1902222633,-2.2081933022,1.3072851896,-0.6697205305,0.3599147201,-0.5324732065,1.8647314310,-0.6539912224,-0.0811203197,-1.6695041656,-0.1677595228,0.1529489309,0.0326982960,0.3603679836,0.2729178965,2.6728754044,0.0517382957,-1.1903650761,1.9886676073,0.3085786998,1.2711433172,0.3996826708,-0.9335381389,-0.3373581767,0.5985429883,-0.7188912630,-0.5423603654,0.8109322190,2.1182680130,-0.2123235017,-0.2050291598,-0.3468085825,-0.0957891718,-0.2557451129,0.1651563942,-0.9804444909,1.7935678959,-0.5420064330,-1.2521762848,-0.4070931375,-2.2685718536,0.2797066867,-0.1671731770,0.7808704972,-0.2584797740,-1.3019461632,-0.8495772481,1.1621996164,0.7994099259,1.4648393393,0.0249336269,-0.5417373776,-0.1430224478,0.3782896399,0.4983956516,-0.3447601497,-0.0361413844,-1.3818486929,-0.1909462810,-0.6342868805,-1.5501580238,0.4438783824,1.4791367054,-0.2694898248,-0.5962350965,-0.4852928221,-0.9123769999,1.4600878954,0.0140110198,-0.0810736865,0.4739871919,-1.6347879171,0.6561161280,0.7903645635,-0.0381236821,0.7106814384,-0.4605905414,0.0920624137,-0.7547672391,0.0979229510,0.3171488345,1.1117020845,1.4691916704,2.1095216274,0.0467688963,-0.2855694592,-0.9627265930,-1.6467630863,0.4463484585,0.1725912839,-0.4279518425,-0.1798031181,0.3101399839,0.0048362794,1.4784452915,1.5026966333,1.0328475237,-0.4919162393,-0.1434764564,0.4667615294,0.9142318368,-0.9285969734,-0.2069311589,1.4059398174,-0.5289928913,-0.4967575073,-1.3956668377,1.4277200699,0.8334128261,-0.0185762569,-0.1923116297,0.0418641306,-0.4321692884,0.8290914893,0.8341650367,-0.2333217710,0.7934345007,0.3587965965,0.5614849925,-2.0616495609,-0.1924656183,-0.0459085889,0.2538861036,-0.4057802260,-0.0318090059,1.3345776796,1.9516458511,-0.4307067394,0.8678566217,-0.1097779796,-1.6316813231,0.1627230048,0.6683475971,-1.0535581112,0.3491037786,0.4147782922,-1.2055585384,0.4886077940,-0.1130808294,0.7219112515,0.0661294386,-0.1400840580,-0.0657013953,-0.6607044935,-0.6015189886,0.7106990814,1.7140800953,0.9367416501,-1.2773195505,-0.6420869827,-0.6175547242,-1.8307830095,-0.4264650047,0.5967303514,1.3804618120,0.8144572973,1.5835609436,-0.3536988199,0.7703401446,1.4535437822,0.7620157599,-0.2588397861,1.8452669382,0.2829564810,-0.8681214452,-1.8005297184,0.4997319281,-0.4752902091,0.8748758435,-1.3430117369,0.5314849615,-0.5759868622,1.2641209364,0.8554402590,1.5168594122,0.2480181605,-0.2194981575,0.5211862922,-0.6440048814,0.7792780399,1.4497418404,0.7278845906,-0.5463015437,-0.0357734151,2.2245075703,1.7500543594,-0.1910633743,0.6573238969,-1.3362874985,-0.7534814477,1.2886375189,0.8610209227,-1.0121858120,0.9795084000,-0.1356090903,0.5075626969,-0.8574798107,0.5879177451,-1.7946065664,-0.0130303819,2.6269860268,-0.2716670632,-0.0657275766,0.1854525954,0.6998822093,0.2578449249,-1.4977544546,-0.6380822659,0.3094310462],[0.9322613478,1.0821434259,0.0982655212,1.2655304670,0.8077735901,0.7658033371,0.0170805342,-0.6523284912,0.4967969060,0.2543117404,-0.1472297460,-1.0754979849,-0.3788460195,-0.6500695348,-1.2020435333,-0.7062480450,-0.1719639748,0.1413584501,1.7674720287,1.4245822430,-0.6928767562,0.4070508480,-0.8351898193,0.6124445200,-1.4726996422,1.2751882076,-0.0078987619,0.7250511050,0.4638235867,0.2765693367,1.0345190763,0.6211076975,-0.2134134769,1.0013273954,-0.3205123544,-1.9580650330,-0.1975866854,0.5886839032,0.2872526348,1.8216292858,-0.8485202789,-1.1175162792,0.0909620300,-0.1545620412,0.6551841497,0.7736989856,-1.3568792343,0.7647265792,-0.2998824716,-1.3783611059,-2.0769608021,-2.3308551311,-0.1665841937,1.6644699574,-1.1846499443,1.8231101036,0.4575906098,-0.5072680116,-0.3290798664,-1.7656143904,-1.0600577593,-0.2223261595,0.4002893269,0.7339606881,-1.7777433395,0.1461729258,0.7029541135,0.7349690199,-0.4175417423,-0.6678780913,-1.4948253632,-1.2591323853,0.7002172470,0.4057020247,-1.8846118450,-0.4096996486,0.3572451174,-0.5729187727,0.3632394969,0.8388002515,0.7888129950,2.0745422840,0.1977431327,0.1684732288,-0.5357504487,-0.7693734169,0.4129380882,-0.3638237119,0.6388210654,0.0842956081,-0.1233994961,0.0458561741,0.0440238081,-0.8346993327,1.1818025112,0.3015581667,0.2385077327,-1.2280052900,-0.0870673805,0.3921283185,0.4252742529,-0.0002660285,0.4988144040,-0.3004822135,-0.2015036196,-0.5291435719,1.3097598553,1.4225721359,0.6904790998,-0.7867959142,1.5844599009,-0.2980011702,1.2988511324,0.2470097095,-0.4910517037,-0.4043954611,1.0191758871,0.1660731584,1.6221412420,0.0662123859,0.1093840599,0.3859095871,-0.1514603198,0.7722839713,-1.8401905298,-0.4582791924,-1.5480029583,0.8383934498,1.5307412148,-1.0848832130,-1.4832700491,1.3966538906,0.7267266512,0.5375882387,-0.7886016369,1.0743261576,-1.1768319607,0.1318906993,-0.3701928556,0.5601051450,0.3761105239,-0.0058422196,0.8287389874,-2.5389571190,2.1728942394,0.3999422789,-0.0519637764,-0.0581969433,0.1575311124,-0.7968834043,1.0298998356,0.3352723718,1.8463332653,-1.3961603642,-0.1399134099,0.8294527531,0.8428064585,0.1062459201,-1.6589927673,-0.5416683555,-0.3484183848,-1.3727669716,-2.1644611359,-1.1050168276,-0.6006038189,-0.6543050408,0.4155762792,-0.4915748537,-2.1399559975,1.0393067598,-0.1879214644,0.8151748776,-0.5987551212,-1.0111064911,-0.1526869535,-1.9934095144,0.0915879831,0.0305229817,0.0214052256,1.4779964685,0.4579023123,-0.1609235853,0.8058716059,-0.1193252355,-1.4209824800,-1.4432802200,0.3448107541,0.8484117389,0.3917312622,-1.3221226931,0.2110161185,-0.1322266310,1.2812737226,0.2105053812,0.5784762502,1.2395321131,-0.1125992611,0.6148221493,0.1073464751,3.1926834583,-1.7169188261,-0.4264818430,0.5589302778,0.0098558068,0.0445020609,-1.0925434828,0.5685714483,1.8520839214,0.2249426991,-0.1640461534,0.4741087854,-1.8028807640,2.5509712696,-0.3230691254,0.1120029092,1.4169816971,-1.1021708250,-1.3753912449,-0.6458590627,0.1685565859,-0.2544992864,1.2977367640,-0.6434763670,-0.4687041342],[-1.5133686066,0.5543805361,-1.6351692677,-0.5480977893,-1.8672090769,1.8669744730,0.1007032543,-0.7519266605,0.9288895130,1.0849411488,0.8286650181,0.2248411477,-1.4469972849,-0.8432237506,0.4922606349,1.0674357414,1.0139129162,1.8805615902,-1.0198467970,-1.4214346409,0.0867412463,0.7238012552,1.2070678473,-0.2821899652,-2.5294377804,-0.1533146352,1.5075769424,0.1515903473,-0.7290561795,-0.3938293755,-0.3148902059,-0.0655002147,1.0328027010,0.9855728745,0.0142235737,0.1424133629,2.3653128147,-0.0937726349,0.6720957160,0.3193032146,-0.0347761326,0.8808904290,0.6379359365,0.5533177257,1.1247152090,-0.0711953193,0.0823010355,1.5953356028,-1.1492849588,-0.2697221935,2.5629601479,-0.4678767622,0.3748208582,-0.3017410338,-1.1595700979,0.9848414660,-1.5886949301,2.1473009586,-0.4320541620,0.3351209760,1.1572041512,-1.6510055065,1.4763158560,-0.8780969977,0.7521370649,0.7664933801,0.6427647471,0.6748605967,-2.1987223625,-0.7755331993,-0.6144747734,0.1226312146,-0.0433201976,-0.9534757137,-0.7053830028,-0.2736400366,-1.2137296200,0.6272516251,-0.1973339170,0.1871281117,-2.7975583076,-0.1932645291,-2.1384193897,0.0670402572,-1.4602125883,-0.1883114576,-1.9900075197,-0.1766163856,1.3107856512,0.8267203569,1.0474450588,-0.4156937301,1.4552435875,-0.9403783679,0.6154562235,0.9041082263,-1.4211113453,0.1542359740,0.2054781765,1.2778793573,-0.1201702952,-1.3128867149,0.3029603362,-0.7522927523,-0.9131040573,0.9282863140,0.5733796358,-0.9316766858,0.2135121375,-0.7382605076,-0.5023554564,-0.9460375905,-0.3469051719,0.3816500604,-1.3926203251,-0.7923706174,-0.8669332266,-0.5555337071,0.2142671943,0.5651071072,-0.6646097302,-1.0670479536,0.0496083200,-2.2859947681,-0.5420932174,-0.8602784276,1.1403759718,0.5368918180,0.4115375876,1.4746530056,-0.2967051268,-0.4382048249,-0.7692515254,-2.1632175446,0.2171597481,-0.4031183124,0.8962494135,0.9963235855,-0.8190985322,-0.3271569610,-1.8990769386,0.3499003649,-2.3797082901,0.6820656061,-1.0557612181,-0.1580118388,1.3777353764,-0.9384363294,0.2778090537,-0.7599855661,0.0350417122,0.3391051590,1.0693094730,-0.4610579610,-0.7098724842,0.2182071954,0.8414065838,0.8761809468,-1.2011101246,-0.0909584537,1.6314455271,-0.3418768048,-0.6599820852,0.0627719760,1.9397410154,-2.6317727566,-0.9709402919,0.2281691581,-0.1328415275,0.3824462593,0.9747131467,2.1028826237,-1.8011776209,-0.7046427727,0.1276893169,1.2597452402,-0.2805773318,1.7098058462,-0.5087597370,1.8857795000,-1.8405008316,0.1865903288,-0.4646588862,-0.8164331317,-0.6990193129,-1.1148481369,-1.4104843140,-0.5799752474,-0.4449983835,-2.0007512569,-1.5721967220,-1.9587359428,-0.6990004778,0.0577918403,-1.6206052303,-0.1327113062,-2.6066644192,0.3853771985,0.7883592248,-0.4743609726,0.3637579083,1.8269271851,-0.3550509512,-0.3871345222,1.4265280962,1.3550875187,-0.0144073600,0.0320234708,0.2594887316,-1.8753159046,1.2990872860,1.1652233601,-0.4094679356,0.6926257014,0.4886541367,-0.3665470183,0.9978932142,-0.4778836966,-2.4469411373,0.8460370302,0.2013505697,-1.1429011822,2.0269160271,-0.3998430967],[0.2654949725,-1.0957469940,-0.7791578174,0.7616731524,1.3665670156,-0.8782943487,-0.8286035061,-1.1778832674,-0.8636802435,-0.0061646109,2.2168614864,1.0119706392,-0.2279826999,-0.6041381359,-0.6085149050,-1.2976219654,-0.2709690034,-0.0114500197,-0.4823000729,1.2648714781,-1.3784844875,0.6856546402,0.9222300053,0.6966942549,-1.8633707762,-0.1454697251,-1.3210371733,-0.1145494431,1.1239894629,1.9604538679,-0.4812983572,-0.5667022467,-1.0941706896,-0.7513188124,-0.4788049757,-0.9990747571,-0.7967398167,-0.9379817247,-0.9230493903,1.3206555843,-2.1616094112,0.9471166134,1.4199587107,0.6445659995,0.4961482286,0.4532108903,-1.3316901922,0.6221421361,1.1318142414,0.5100553036,-0.1593082994,-0.7789161801,-2.7416522503,1.1332864761,-1.0270265341,-0.2043064237,-0.8484025598,-0.1263597310,-0.1946774721,0.6200547814,-0.6168873906,-1.3963674307,-0.6660103798,0.0967090577,-1.6318593025,0.2810838521,0.6856861115,-1.1412400007,0.3920578957,0.3876029253,-2.0110924244,-0.6093295813,0.8308763504,-0.6013477445,1.4756437540,0.3031979799,-0.5353441238,-0.5915081501,0.9075561166,1.1504007578,0.6115757227,-2.3925230503,-1.2824686766,-1.3351628780,1.1867614985,0.2219233513,-0.8080751300,0.5201536417,-0.1467103064,1.5671451092,1.9779373407,-0.2780325115,0.3241499662,1.5429707766,0.3520541489,-0.0514822975,-0.1410155594,-1.3119984865,-0.3331850767,-1.3207861185,-0.0688070208,-0.1959996521,-1.2634282112,0.4501601756,-0.6468436122,0.6385756135,0.4816366732,0.8377054334,-1.1855635643,0.5892620087,-0.6773741245,-1.5393686295,1.6592665911,1.1991732121,-0.8384900093,-0.0617964156,-0.3647063375,0.0483825915,0.6977452040,0.8587312102,-0.5782601237,0.6752809882,1.8073829412,-0.4694879651,2.6854341030,-0.7661315799,-0.1617361754,-1.6490541697,-0.3861537874,1.1943726540,0.4325670004,0.9196245074,1.1767194271,-1.1586842537,2.2081480026,-0.8941178322,-0.1457774490,-1.3551640511,-2.0796351433,-1.5357860327,1.3167635202,-0.4685901403,-1.3264200687,-1.1433360577,0.5723722577,-1.3153508902,-0.0003914612,0.4272823632,-0.6007682085,-0.9464733005,0.9466275573,-0.3957338631,0.3242906034,0.3518322706,-0.9805750847,-1.4119974375,0.9316169024,-0.0752965063,-0.3946371078,1.0344431400,0.5991471410,0.7365334034,-0.8374543190,-0.0171916950,0.5245986581,-0.2836663425,-0.7435886264,-0.5999352336,0.5393315554,-0.9004894495,0.9330707788,-1.0388084650,0.5326125026,-0.2157468051,0.3797125518,-1.6414234638,-1.5240204334,-0.1943216473,2.4012343884,-0.1102185398,1.0404075384,-0.5839586258,-0.6843239665,-1.0446710587,1.3564828634,-1.1882144213,0.4350872934,0.2158848494,-0.3997232020,1.3127151728,0.7766410112,0.0745480806,0.8701055050,0.5700027943,-1.9929957390,1.7805351019,0.0368345529,-0.2137983143,0.9791098237,0.3465301096,-0.1790597886,0.7861873507,0.2161410749,1.9109491110,0.0643576086,1.4099440575,0.5143269300,-1.2256814241,0.3933446407,0.9667174220,-0.0542557761,-0.6940347552,-0.1299740523,-0.6604279876,0.0753633454,-0.4645026326,-0.9675666094,0.6419441700,1.0465860367,0.4986249506,-0.3357897699,-1.8439725637,-1.3316335678,-0.4952035844],[1.5557930470,0.6939328909,-0.9675522447,0.1987546235,0.0348333046,-1.2145023346,-1.0559561253,-1.6981366873,-0.6445525289,-0.4339305460,1.0280379057,1.7661401033,-1.4261125326,-1.3287851810,-0.8939813375,1.1151138544,0.4468123913,0.5887181163,-0.0540989488,0.8927987218,2.1387660503,1.7823724747,1.7096697092,1.8147362471,-0.1693580598,-1.0753818750,1.2679872513,-2.0095827579,0.6978594661,-1.2827425003,0.3344699740,-0.0405597240,0.1785971820,-1.7131185532,0.8015590310,-0.7889513373,0.4382426143,-0.2685180604,-0.4353355765,0.8980778456,1.4425247908,-0.6797144413,-0.5390772223,-1.2332694530,-0.8271765709,1.2962095737,-0.6857874990,0.4101184309,-1.0391228199,0.4739780426,0.2037101090,1.2740825415,-1.7347189188,1.1883305311,-0.2816534936,-0.0767895505,-0.1394811720,-0.5756545663,-0.4448460937,-0.9648231268,2.6027748585,-1.0214937925,-0.8026785254,0.1019904837,0.0143244155,-0.4809729457,-1.2737265825,-0.5467684269,1.2570265532,0.7116396427,1.6483726501,0.4744225144,-1.3267292976,-0.0733655021,-1.5249861479,-0.0464469939,2.2251887321,0.2355747223,1.2292549610,-0.8253579140,-0.1976063699,-2.2327098846,-0.2294290513,-0.8787239790,-0.4248799682,-0.9848347306,1.1690719128,-1.8133636713,-1.8084857464,0.0479786545,2.0665719509,-2.6742475033,0.5344931483,0.0683459938,-0.4919559062,-0.9514514208,-1.2987574339,-2.8886623383,-0.0731047243,-1.9328001738,0.9243254066,0.4220359325,-0.7849935889,-1.3458760977,0.3226920068,-0.6062769294,-1.6259837151,0.6885620952,1.7813590765,0.0121325897,1.6976926327,-0.3455361128,1.0504717827,0.1753101647,-0.0284505151,-0.2338887602,0.7708004713,0.4418765306,1.2308294773,-0.5319422483,-0.4612456560,0.3082421124,-0.5902484655,-0.2054267824,1.5642417669,0.1995749623,-1.3234007359,-0.6998391747,-0.6464892626,-0.3926881850,0.1428338438,0.2082134038,-0.3770585358,0.2527660429,2.2772750854,0.6177937388,-1.6407494545,0.0468776226,-0.9865619540,-0.4993889332,0.8691691160,-0.2714325190,0.9959207177,-0.4606694579,-1.1768029928,1.0289868116,0.7226113081,-0.7540940642,-0.7165128589,-0.5451242924,-0.4123264253,-0.8378821611,1.1969192028,0.2376316488,0.6620379090,-1.2662197351,-0.2605038881,-0.4416389763,1.6299383640,0.0120546557,0.1751725078,2.0253489017,-2.0747103691,0.7142991424,-0.8726304173,0.1750395596,0.4198315442,0.8568505049,-0.7676183581,1.2397425175,0.6619054079,-0.8535833955,0.9861800671,-1.0405623913,-0.2410328388,1.1349664927,0.2311729193,0.1398157328,0.7055028081,-1.5441834927,-0.2105818391,0.8658529520,-1.5761580467,-1.0042510033,-0.6100757122,-0.1319507957,1.9316471815,-0.3888913393,1.4629999399,0.3790009618,-0.7812678814,0.2428106815,1.7966394424,0.2591165304,-1.0846961737,0.4639006257,-0.3702415228,-1.3542098999,-2.5225086212,2.1170349121,0.2919540107,1.5617704391,-0.5720227361,0.1448655128,0.8777965903,-0.7311284542,0.6961936951,-0.2438761741,0.2898792624,-1.0157407522,0.1055957079,1.5368494987,-0.2627255023,0.5305185318,-0.9444933534,0.9583199024,-0.4513328671,-1.7957193851,-1.9104524851,1.5604406595,1.0587686300,-0.2916118801,-0.8341334462,0.6081914306],[0.5385888815,0.2649672329,1.9931534529,1.7101278305,-0.3339462578,-1.6201473475,0.4781301022,1.6592991352,1.8929585218,-0.5796220303,0.1620042324,-0.9191796780,0.2226093709,-0.3510306776,2.5146825314,1.3014285564,-0.5516109467,-1.7102047205,-0.8960490823,-0.9496671557,-0.4185546935,-0.6126506329,2.3513891697,-0.5045138597,0.2073156387,-0.0403192304,0.2158572525,1.6789017916,-0.0491207652,-0.0926134512,-0.9717449546,0.3141212761,-0.1822118014,1.5598946810,0.1269149184,-1.7584841251,-0.2879258394,-0.5144426823,0.8904726505,0.0560838953,1.0901510715,-0.3769854307,-0.6356492043,-0.2561465204,-0.2323470414,0.8959264159,1.5302934647,0.9458421469,-0.7589567900,1.3199381828,0.2553999126,0.0001370502,-0.9755280614,0.1034968719,-0.5387631655,1.1641650200,1.5335197449,-0.0891886353,1.0386070013,0.3652443886,0.4225503206,-0.9608186483,0.2354464382,0.8191389441,-0.0019620592,-2.2598648071,0.2033568025,0.9071549177,1.6220998764,0.0620047227,1.1191365719,-0.6640503407,-0.2967987657,-0.1196853444,0.4721096754,-1.5424871445,2.5747241974,-0.9410291314,0.2411335409,-0.5882598758,0.5838999152,1.2239956856,0.0473095626,0.0269213933,-1.7474669218,-0.1254788041,0.6390517950,-0.1682768017,0.1148226485,-0.8987408876,-1.1397955418,1.4998438358,1.4500621557,0.1748585701,-0.1256774962,0.6514495611,-1.3195337057,-1.3019288778,0.1573456079,1.1976250410,-0.6434811950,-0.0719533190,-0.0295303278,-0.9087253809,0.6040388942,-0.3942040503,0.8656547666,1.3265830278,0.1610490084,-0.5010558963,-1.4810063839,-0.2897340357,-0.6645922661,0.2873976231,0.0032383415,-2.2635951042,0.7036095262,0.2958571017,1.3461945057,0.4321117997,-1.0315165520,0.9034367800,-1.0506396294,0.3633961380,1.2920157909,-0.1767129153,0.9583020210,1.9570393562,-0.0153314872,0.2814647257,-0.3974483609,-0.2976866663,-1.3961068392,-1.6619836092,-1.1750111580,-0.3427028060,0.8588997126,-1.0289971828,-0.1670207977,-0.2226491570,0.5565141439,0.3041157424,0.1601631343,-0.6869891882,0.0558025204,-0.1517806500,0.2210749239,-1.2610491514,0.4965585470,-0.8622374535,-0.2092900723,1.9283560514,2.0117387772,1.2364124060,-1.2820222378,-0.5927175283,-1.2122610807,-0.2559721172,0.3717762232,-0.4502603412,1.7123154402,0.3875474632,0.6138361096,0.4190999269,0.7165051699,-0.9974407554,0.6195259690,-1.8539083004,-0.8086301088,-1.7899594307,0.5038084984,-0.9629930854,1.0737767220,-0.0268682484,-1.5845243931,1.5086864233,0.6216324568,1.9804166555,-1.0086367130,2.2385258675,-0.5203742385,-0.8196234703,1.1165432930,-1.2808620930,1.2763589621,-0.9164367914,0.1268753260,2.0863809586,-0.7245088816,-1.2106389999,1.2202898264,-0.8158519864,0.8383903503,1.5530946255,-0.1538568288,0.2419621795,-1.1090013981,2.5024588108,0.1697666496,1.0527145863,0.6301341653,-1.3145409822,-0.5830605030,-0.2038480788,-0.1603022367,-0.6039440632,0.6921498179,0.5524765253,0.8396931291,0.4275787771,0.5381883383,-0.0318626240,2.2095665932,1.4112273455,0.0706845075,0.3475209475,0.3752060533,-0.5024744272,0.4471912980,0.3937501609,-0.4151968062,1.5611482859,1.7722326517,-0.4225720763]]]] \ No newline at end of file diff --git a/csharp/testdata/squeezenet.onnx b/csharp/testdata/squeezenet.onnx new file mode 100644 index 0000000000000..810cf8f88bf73 Binary files /dev/null and b/csharp/testdata/squeezenet.onnx differ diff --git a/docs/ABI.md b/docs/ABI.md new file mode 100644 index 0000000000000..d57e7b94d16c9 --- /dev/null +++ b/docs/ABI.md @@ -0,0 +1,46 @@ +# ONNXRuntime ABI + +We release ONNXRuntime as both static library and shared library on Windows, Linux and Mac OS X. [ABI (Application Binary Interface)](https://en.wikipedia.org/wiki/Application_binary_interface) is only for the shared library. It allows you upgrade ONNXRuntime to a newer version without recompiling. + +The ABI contains: + +1. A set of C functions for creating an inference session and load/run a model. No global variables would be directly exported. All these functions can be directly used in .net and .net core through PInvoke. All these public symbols are C symbols, no C++. +2. (TODO) A C++ API for authoring custom ops and put them into a separated dll. + +# Functionality +[C API](C_API.md) + +# Integration +Q: Should I statically link to ONNXRuntime or dynamically? +A: On Windows, Any custom op DLL must dynamically link to ONNXRuntime. +Dynamical linking also helps on solving diamond dependency problem. For example, if part of your program depends on ONNX 1.2 but ONNXRuntime depends on ONNX 1.3, then dynamically linking to them would be better. + +Q: Any requirement on CUDA version? My program depends on CUDA 9.0, but the ONNXRuntime binary was built with CUDA 9.1. Is it ok to put them together? +A: Yes. Because ONNXRuntime statically linked to CUDA. + +# Dev Notes + +## Global Variables +Global variables may get constructed or destructed inside "DllMain". There are significant limits on what you can safely do in a DLL entry point. See ['DLL General Best Practices'](https://docs.microsoft.com/en-us/windows/desktop/dlls/dynamic-link-library-best-practices). For example, you can't put a ONNXRuntime InferenceSession into a global variable. + +## Component Object Model (COM) +ONNXRuntime doesn't contain a COM interface, whether it's on Windows or Linux. Because .Net Core doesn't support COM on Linux and we need to make ONNXRuntime available to .Net Core. + +## Undefined symbols in a shared library +On Windows, you can't build a DLL with undefined symbols. Every symbol must be get resolved at link time. On Linux, you can. + +In this project, we setup a rule: when building a shared library, every symbol must get resolved at link time, unless it's a custom op. + +For custom op, on Linux, don't pass any libraries(except libc, pthreads) to linker. So that, even the application is statically linked to ONNXRuntime, they can still use the same custom op binary. + + +## Default visibility +On POSIX systems, please always specify "-fvisibility=hidden" and "-fPIC" when compiling any code in ONNXRuntime shared library. + + See [pybind11 FAQ](https://github.com/pybind/pybind11/blob/master/docs/faq.rst#someclass-declared-with-greater-visibility-than-the-type-of-its-field-someclassmember--wattributes) + + +## RTLD_LOCAL vs RTLD_GLOBAL +RTLD_LOCAL and RTLD_GLOBAL are two flags of [dlopen(3)](http://pubs.opengroup.org/onlinepubs/9699919799/functions/dlopen.html) function on Linux. By default, it's RTLD_LOCAL. And basically you can say, there no corresponding things like RTLD_GLOBAL on Windows. + +If your application is a shared library, which statically linked to ONNXRuntime, and your application needs to dynamically load a custom op, then your application must be loaded with RTLD_GLOBAL. In all other cases, you should use RTLD_LOCAL. ONNXRuntime python binding is a good example of why sometimes RTLD_GLOBAL is needed. diff --git a/docs/AddingCustomOp.md b/docs/AddingCustomOp.md new file mode 100644 index 0000000000000..6ad742617b5a1 --- /dev/null +++ b/docs/AddingCustomOp.md @@ -0,0 +1,27 @@ +Adding a new op +=============== + +## A new op can be written and registered with ONNXRuntime in the following 3 ways +### 1. Using a dynamic shared library +* First write the implementation of the op and schema (if required) and assemble them in a shared library. +See [this](../onnxruntime/test/custom_op_shared_lib) for an example. Currently +this is supported for Linux only. + +Example of creating a shared lib using g++ on Linux: +```g++ -std=c++14 -shared test_custom_op.cc -o test_custom_op.so -fPIC -I. -Iinclude/onnxruntime -L. -lonnxruntime -DONNX_ML -DONNX_NAMESPACE=onnx``` + +* Register the shared lib with ONNXRuntime. +See [this](../onnxruntime/test/shared_lib/test_inference.cc) for an example. + +### 2. Using RegisterCustomRegistry API +* Implement your kernel and schema (if required) using the OpKernel and OpSchema APIs (headers are in the include folder). +* Create a CustomRegistry object and register your kernel and schema with this registry. +* Register the custom registry with ONNXRuntime using RegisterCustomRegistry API. + +See +[this](../onnxruntime/test/framework/local_kernel_registry_test.cc) for an example. + +### 3. Contributing the op to ONNXRuntime +This is mostly meant for ops that are in the process of being proposed to ONNX. This way you don't have to wait for an approval from the ONNX team +if the op is required in production today. +See [this](../onnxruntime/contrib_ops) for an example. diff --git a/docs/AddingExecutionProvider.md b/docs/AddingExecutionProvider.md new file mode 100644 index 0000000000000..e077737ce6679 --- /dev/null +++ b/docs/AddingExecutionProvider.md @@ -0,0 +1,12 @@ +# Adding a new execution provider + +* All execution providers inherit from + [IExecutionProvider](../include/onnxruntime/core/framework/execution_provider.h) +* The best way to start adding a provider is to start with examples already + added in ONNXRuntime + * [CPU Execution + Provider](../onnxruntime/core/providers/cpu/cpu_execution_provider.h) + * [CUDA Execution + Provider](../onnxruntime/core/providers/cuda/cuda_execution_provider.h) + * [MKL-DNN Execution + Provider](../onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h) diff --git a/docs/CSharp_API.md b/docs/CSharp_API.md new file mode 100644 index 0000000000000..3d78797f9ca3d --- /dev/null +++ b/docs/CSharp_API.md @@ -0,0 +1 @@ +# C# API diff --git a/docs/C_API.md b/docs/C_API.md new file mode 100644 index 0000000000000..7cf5737bccfbb --- /dev/null +++ b/docs/C_API.md @@ -0,0 +1,14 @@ +# C API + +## Headers +[onnxruntime_c_api.h](include/onnxruntime/core/session/onnxruntime_c_api.h) + +## Functionality +* Creating an InferenceSession from an on-disk model file and a set of SessionOptions. +* Registering customized loggers. +* Registering customized allocators. +* Registering predefined providers and set the priority order. ONNXRuntime has a set of predefined execution providers,like CUDA, MKLDNN. User can register providers to their InferenceSession. The order of registration indicates the preference order as well. +* Running a model with inputs. These inputs must be in CPU memory, not GPU. If the model has multiple outputs, user can specify which outputs they want. +* Converting an in-memory ONNX Tensor encoded in protobuf format, to a pointer that can be used as model input. +* Setting the thread pool size for each session. +* Dynamically loading custom ops. diff --git a/docs/Coding_Conventions_and_Standards.md b/docs/Coding_Conventions_and_Standards.md new file mode 100644 index 0000000000000..d1524f406323e --- /dev/null +++ b/docs/Coding_Conventions_and_Standards.md @@ -0,0 +1,44 @@ +# ONNX Runtime coding conventions and standards + + +## Code Style + +Google style from https://google.github.io/styleguide/cppguide.html with a few minor alterations: + +* Max line length 120 + * Aim for 80, but up to 120 is fine. +* Exceptions + * Allowed to throw fatal errors that are expected to result in a top level handler catching them, logging them and terminating the program. +* Non-const references + * Allowed + * However const correctness and usage of smart pointers (shared_ptr and unique_ptr) is expected, so a non-const reference equates to “this is a non-null object that you can change but are not being given ownership of”. +* 'using namespace' permitted with limited scope + * Not allowing 'using namespace' at all is overly restrictive. Follow the C++ Core Guidelines: + * [SF.6: Use using namespace directives for transition, for foundation libraries (such as std), or within a local scope (only)](https://github.com/isocpp/CppCoreGuidelines/blob/master/CppCoreGuidelines.md#Rs-using) + * [SF.7: Don't write using namespace at global scope in a header file](https://github.com/isocpp/CppCoreGuidelines/blob/master/CppCoreGuidelines.md#Rs-using-directive) + + +#### Clang-format + +Clang-format will handle automatically formatting code to these rules. There’s a Visual Studio plugin that can format on save at https://marketplace.visualstudio.com/items?itemName=LLVMExtensions.ClangFormat, or alternatively the latest versions of Visual Studio 2017 include [clang-format support](https://blogs.msdn.microsoft.com/vcblog/2018/03/13/clangformat-support-in-visual-studio-2017-15-7-preview-1/). + +There is a .clang-format file in the root directory that has the max line length override and defaults to the google rules. This should be automatically discovered by the clang-format tools. + +## Code analysis + +Visual Studio Code Analysis with [C++ Core guidelines](https://github.com/isocpp/CppCoreGuidelines/blob/master/CppCoreGuidelines.md) rules enabled is configured to run on build for the onnxruntime_common, onnxruntime_graph and onnxruntime_util libraries. Updating the onnxruntime_framework and onnxruntime_provider libraries to enable Code Analysis and build warning free is pending. + +Code changes should build with no Code Analysis warnings, however this is somewhat difficult to achieve consistently as the Code Analysis implementation is in fairly constant flux. Different minor releases may have less false positives (a build with the latest version may be warning free, and a build with an earlier version may not), or detect additional problems (an earlier version builds warning free and a later version doesn't). + +## Unit Testing and Code Coverage + +There should be unit tests that cover the core functionality of the product, expected edge cases, and expected errors. +Code coverage from these tests should aim at maintaining over 80% coverage. + +All changes should be covered by new or existing unit tests. + +In order to check that all the code you expect to be covered by testing is covered, run code coverage in Visual Studio using 'Analyze Code Coverage' under the Test menu. + +There is a configuration file in onnxruntime\VSCodeCoverage.runsettings that can be used to configure code coverage so that it reports numbers for just the onnxruntime code. Select that file in Visual Studio via the Test menu: 'Test' -> 'Test Settings' -> 'Select Test Settings File'. + +Using 'Show Code Coverage Coloring' will allow you to visually inspect which lines were hit by the tests. See . diff --git a/docs/HighLevelDesign.md b/docs/HighLevelDesign.md new file mode 100644 index 0000000000000..84e15e5fdc50b --- /dev/null +++ b/docs/HighLevelDesign.md @@ -0,0 +1,87 @@ +# ONNX Runtime High Level Design + +This document outlines the high level design of +ONNXRuntime - a high performance, cross platform engine. + +## Key objectives +* Maximally and automatically leverage the custom accelerators and runtimes +available on disparate platforms. +* Provide the right abstraction and runtime support for custom accelerators and +runtimes. We call this abstraction an [execution +provider](../include/onnxruntime/core/framework/execution_provider.h). It defines and exposes a set of +its capabilities to ONNXRuntime: a set of single or fused nodes it can +execute, its memory allocator and more. Custom accelerators and runtimes are +instances of execution provider. +* We don't expect that an execution provider can always run an ONNX model fully +on its device. This means that ONNXRuntime must be able to execute a single +model in a heterogeneous environment involving multiple execution providers. +* Provide support for high-level optimizations that can be expressed as +model-to-model transformations via a [graph-transformation +API](../include/onnxruntime/core/graph/graph_transformer.h). Such +transformations fall into two categories: global transformations, those that +require analysis and transformation of the entire graph, and local +transformations, which can be captured as simple (algebraic) [rewriting +rules](../include/onnxruntime/core/graph/rewrite_rule.h). + +## High-level system architecture +The flow is quite simple. Starting from an ONNX model, ONNXRuntime first +converts the model graph into its in-memory graph representation. It then +applies a number of graph transformations that a) perform a set of provider +independent optimizations such cast transformations between float16 and float32, and b) partition the +graph into a set of subgraphs based on the available execution providers. Each +subgraph is assigned to an execution provider. We ensure that a subgraph can be +executed by an execution provider by querying the capability of the execution +provider using the GetCapability() API. + +![ONNXRuntime high level system architecture](https://azurecomcdn.azureedge.net/mediahandler/acomblog/media/Default/blog/228d22d3-6e3e-48b1-811c-1d48353f031c.png) + +*Note: TensorRT and nGraph support in the works.* + +### More about partitioning +ONNXRuntime partitions a model graph based on the available execution providers +into subgraphs, each for a distinct provider respectively. ONNXRuntime provides +a default execution provider that is used for fallback execution for the +operators that cannot be pushed onto the more specialized but more efficient +execution providers. Intuitively we probably want to push computation to the +specialized execution providers as much as possible. + +We use a simple graph partitioning technique. The available execution providers +will be considered in a specific order, and each will be assigned the maximal +subgraphs (possibly more than one) that it is able to handle. The +ONNXRuntime-provided default execution provider will be the last one to be +considered, and it ensures completeness. More sophisticated optimizations can be +considered in the future (or can even be implemented as a composite execution +provider). + +Conceptually, each partition is reduced to a single fused operator. It is +created by invoking the execution provider's Compile() method and wrap it as a +custom operator. We support both sync and async modes of execution for custom +operators. We also support both strict and non-strict invocations. An execution +provider exposes its memory allocator, which is used to allocate the input +tensors for the execution provider. The rewriting and partitioning transform the +initial model graph into a new graph composed with operators assigned to either +the default execution provider or other registered execution +providers. ONNXRuntime execution engine is responsible for running this graph. + +## Key design decisions +* Multiple threads should be able to inovke the Run() method on the same +inference session object. See [API doc](C_API.md) for more details. +* To facilitate the above the Compute() function of all kernels is const +implying the kernels are stateless. +* We call implementations of the operators by execution providers as +kernels. Each execution provider supports a subset of the (ONNX) +operators/kernels. +* ONNXRuntime runtime guarantees that all operators are supported by the default +execution provider. +* Tensor representation: ONNXRuntime will utilize a standard representation for +the tensor runtime values. The execution providers can internally use a +different representation, if they choose to, but it is their responsibility to +convert the values from/to the standard representation at the boundaries of +their subgraph. + +## Extensibility points +* [Add a custom operator/kernel](AddingCustomOp.md) +* [Add an execution provider](AddingExecutionProvider.md) +* [Add a new graph +transform](../include/onnxruntime/core/graph/graph_transformer.h) +* [Add a new rewrite rule](../include/onnxruntime/core/graph/rewrite_rule.h) diff --git a/docs/Model_Test.md b/docs/Model_Test.md new file mode 100644 index 0000000000000..9c12351ac34f9 --- /dev/null +++ b/docs/Model_Test.md @@ -0,0 +1,47 @@ +ONNX has a collection of standard tests. This document describes how to run these tests through a C++ program named 'onnx_test_runner' in this repo. You could also run these test through onnxruntime python binding, which would be much easier to setup, but, a bit harder to debug issues. + +# Get the test data +You should have: +1. onnx single node test data +2. onnx model zoo models + +## Install onnx python package +You can get onnx python package from [pypi](https://pypi.org/). However, if you are a onnxruntime developer, you may need to work on a cutting edge ONNX version. In this case, you need to build and install ONNX from source code. + +### Install ONNX from source code +1. (windows) set ONNX_ML=1 + (linux) export ONNX_ML=1 +2. Install protobuf and put protoc into your PATH environment. When you compile protobuf, it's better to only enable the static libraries. +3. run "python setup.py bdist_wheel" and "pip install dist/*.whl" + +## Generate node test data +$ python3 -m onnx.backend.test.cmd_tools generate-data -o +e.g. + python3 -m onnx.backend.test.cmd_tools generate-data -o C:\testdata + + +## Get the onnx model zoo models +Download the files from: https://github.com/onnx/models. Unzip them. +(TODO: put a full copy on Azure blob, instead of downloading these files from different sources individually) + +# Compile onnxruntime_test_runner and run the tests +onnxruntime_test_runner is a C++ program. Its source code is in onnxruntime/test/onnx directory. + +Usage: onnx_test_runner [options...] +Options: + -j [models]: Specifies the number of models to run simultaneously. + -A : Disable memory arena + -c [runs]: Specifies the number of Session::Run() to invoke simultaneously for each model. + -r [repeat]: Specifies the number of times to repeat + -v: verbose + -n [test_case_name]: Specifies a single test case to run. + -e [EXECUTION_PROVIDER]: EXECUTION_PROVIDER could be 'cpu', 'cuda' or 'mkldnn'. Default: 'cpu'. + -x: Use parallel executor, default (without -x): sequential executor. + -h: help + +e.g. +//run the tests under C:\testdata dir and enable CUDA provider +$ onnx_test_runner -e cuda C:\testdata + +//run the tests sequentially. It would be easier to debug +$ onnx_test_runner -c 1 -j 1 C:\testdata diff --git a/docs/Python_Dev_Notes.md b/docs/Python_Dev_Notes.md new file mode 100644 index 0000000000000..13c35a8669ca2 --- /dev/null +++ b/docs/Python_Dev_Notes.md @@ -0,0 +1,16 @@ +# Python Dev Notes + +Each Python version uses a specific compiler version. In most cases, you should use the same compiler version for building python extensions. + +## Which Microsoft Visual C++ compiler to use with a specific Python version ? + +| Visual C++ | CPython | +|-------------|:-----------------------:| +|2015, 2017 | 3.7 | +|2015 | 3.5,3.6 | +|2010 | 3.3,3.4 | +|2008 | 2.6, 2.7, 3.0, 3.1, 3.2 | + +Currently, ONNXRuntime only supports Visual C++ 2017. Therefore, Python 3.7 seems to be the best choice. + +CPython 3.7 is distributed with a VC++ 2017 runtime. Unlike the earlier VC++ version, VC++ 2017 Runtime is binary backward compatible with VC++ 2015. Which means you could build your application with VC++ 2015 then run it with VC++ 2017 runtime. diff --git a/docs/ReleaseManagement.md b/docs/ReleaseManagement.md new file mode 100644 index 0000000000000..27c5413ced175 --- /dev/null +++ b/docs/ReleaseManagement.md @@ -0,0 +1,11 @@ +# Release Management + +This describes the process by which versions of ONNX Runtime are officially +released to the public. + +## Releases +Releases are versioned according to +[docs/Versioning.md](Versioning/md). We plan to release ONNX Runtime packages +every 6 months. + +(TBD: Add more here later) diff --git a/docs/Roadmap.md b/docs/Roadmap.md new file mode 100644 index 0000000000000..11abe52ce7121 --- /dev/null +++ b/docs/Roadmap.md @@ -0,0 +1,3 @@ +# Roadmap + +TBD diff --git a/docs/Versioning.md b/docs/Versioning.md new file mode 100644 index 0000000000000..292538ab005f5 --- /dev/null +++ b/docs/Versioning.md @@ -0,0 +1,36 @@ +# Versioning + +## API +ONNX Runtime follows [Semantic Versioning 2.0](https://semver.org/) for its public API. +Each release has the form MAJOR.MINOR.PATCH. The meanings of MAJOR, MINOR and PATCH are +same as what is described in the semantic versioning doc linked above. + +### ABI +Please refer to the [ABI doc](ABI.md) for more details about binary compatibility. + +## Current stable release version +The version number of the current stable release can be found +[here](../VERSION_NUMBER) + +## Release cadence +See [docs/ReleaseManagement.md](ReleaseManagement.md) + +## Compatibility with ONNX opsets +ONNX Runtime supports both backwards and forward compatibility. + +### Backwards compatibility +All versions of ONNX Runtime will support ONNX opsets all the way back to (and including) opset version 7. +In other words if an ONNX Runtime release implements ONNX opset ver 9, it'll be able to run all +models that are stamped with ONNX opset verions in the range [7-9]. + +### Forward compatibility +A release version that supports opset ver 8 will be able to run all models that are stamped with opset ver 9 provided +the model doesn't use ops that were newly introduced in opset ver 9. + +### Version matrix +Following table summarizes the relationship between the ONNX Runtime version and the ONNX +opset version implemented in that release. + +| ONNX Runtime release version | ONNX opset version
implemented in this release | ONNX ML opset version
implemented in this release | Supported ONNX IR version | +|------------------------------|--------------------|----------------------|------------------| +| 0.1.4 | 8 | 1 | 3 | diff --git a/docs/python/README.rst b/docs/python/README.rst new file mode 100644 index 0000000000000..97b1a3b34f7a1 --- /dev/null +++ b/docs/python/README.rst @@ -0,0 +1,72 @@ +ONNX Runtime +============ + +ONNX Runtime +enables high-performance evaluation of trained machine learning (ML) +models while keeping resource usage low. +Building on Microsoft's dedication to the +`Open Neural Network Exchange (ONNX) `_ +community, it supports traditional ML models as well +as Deep Learning algorithms in the +`ONNX-ML format `_. +Documentation is available at +`Python Bindings for ONNX Runtime `_. + +Example +------- + +The following example demonstrates an end-to-end example +in a very common scenario. A model is trained with *scikit-learn* +but it has to run very fast in a optimized environment. +The model is then converted into ONNX format and ONNX Runtime +replaces *scikit-learn* to compute the predictions. + +:: + + # Train a model. + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + from sklearn.ensemble import RandomForest + iris = load_iris() + X, y = iris.data, iris.target + X_train, X_test, y_train, y_test = train_test_split(X, y) + clr = RandomForest() + clr.fit(X_train, y_train) + + # Convert into ONNX format with onnxmltools + from onnxmltools import convert_sklearn + from onnxmltools.utils import save_model + from onnxmltools.convert.common.data_types import FloatTensorType + initial_type = [('float_input', FloatTensorType([1, 4]))] + onx = convert_sklearn(clr, initial_types=initial_type) + save_model(onx, "rf_iris.onnx") + + # Compute the prediction with ONNX Runtime + import onnxruntime as rt + import numpy + sess = rt.InferenceSession("rf_iris.onnx") + input_name = sess.get_inputs()[0].name + label_name = sess.get_outputs()[0].name + pred_onx = sess.run([label_name], {input_name: X_test.astype(numpy.float32)})[0] + +Changes +------- + +0.1.4 +^^^^^ + +GA release as part of open sourcing onnxruntime. + +0.1.3 +^^^^^ + +Fixes a crash on machines which do not support AVX instructions. + +0.1.2 +^^^^^ + +First release on Ubuntu 16.04 for CPU and GPU with Cuda 9.1 and Cudnn 7.0, +supports runtime for deep learning models architecture such as AlexNet, ResNet, +XCeption, VGG, Inception, DenseNet, standard linear learner, +standard ensemble learners, +and transform scaler, imputer. diff --git a/docs/python/api_summary.rst b/docs/python/api_summary.rst new file mode 100644 index 0000000000000..aad5c7042a872 --- /dev/null +++ b/docs/python/api_summary.rst @@ -0,0 +1,68 @@ + +=========== +API Summary +=========== + +Summary of public functions and classes exposed +in *ONNX Runtime*. + +.. contents:: + :local: + +Device +====== + +The package is compiled for a specific device, GPU or CPU. +The CPU implementation includes optimizations +such as MKL (Math Kernel Libary). The following function +indicates the chosen option: + +.. autofunction:: onnxruntime.get_device + +Examples and datasets +===================== + +The package contains a few models stored in ONNX format +used in the documentation. These don't need to be downloaded +as they are installed with the package. + +.. autofunction:: onnxruntime.datasets.get_example + +Load and run a model +==================== + +*ONNX Runtime* reads a model saved in ONNX format. +The main class *InferenceSession* wraps these functionalities +in a single place. + +.. autoclass:: onnxruntime.ModelMetadata + :members: + +.. autoclass:: onnxruntime.InferenceSession + :members: + +.. autoclass:: onnxruntime.NodeArg + :members: + +.. autoclass:: onnxruntime.RunOptions + :members: + +.. autoclass:: onnxruntime.SessionOptions + :members: + +Backend +======= + +In addition to the regular API which is optimized for performance and usability,  +*ONNX Runtime* also implements the +`ONNX backend API `_ +for verification of *ONNX* specification conformance. +The following functions are supported: + +.. autofunction:: onnxruntime.backend.is_compatible + +.. autofunction:: onnxruntime.backend.prepare + +.. autofunction:: onnxruntime.backend.run + +.. autofunction:: onnxruntime.backend.supports_device diff --git a/docs/python/conf.py b/docs/python/conf.py new file mode 100644 index 0000000000000..330d468cda75f --- /dev/null +++ b/docs/python/conf.py @@ -0,0 +1,93 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. + +import os +import sys +import shutil +# Check these extensions were installed. +import sphinx_gallery.gen_gallery +# The package should be installed in a virtual environment. +import onnxruntime +# The documentation requires two extensions available at: +# https://github.com/xadupre/sphinx-docfx-yaml +# https://github.com/xadupre/sphinx-docfx-markdown + + +# -- Project information ----------------------------------------------------- + +project = 'ONNX Runtime' +copyright = '2018, Microsoft' +author = 'Microsoft' +version = onnxruntime.__version__ +release = version + +# -- General configuration --------------------------------------------------- + +extensions = [ + 'sphinx.ext.intersphinx', + 'sphinx.ext.imgmath', + 'sphinx.ext.ifconfig', + 'sphinx.ext.viewcode', + "sphinx.ext.autodoc", + "sphinx_gallery.gen_gallery", + 'sphinx.ext.autodoc', + "docfx_yaml.extension", + "docfx_markdown", +] + +templates_path = ['_templates'] + +source_parsers = { + '.md': 'recommonmark.parser.CommonMarkParser', +} + +source_suffix = ['.rst', '.md'] + +master_doc = 'intro' +language = "en" +exclude_patterns = [] +pygments_style = 'sphinx' + +# -- Options for HTML output ------------------------------------------------- + +html_theme = "sphinx_rtd_theme" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +html_static_path = ['_static'] +# html_sidebars = {} + +# -- Options for intersphinx extension --------------------------------------- + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'https://docs.python.org/': None} + +# -- Options for Sphinx Gallery ---------------------------------------------- + +sphinx_gallery_conf = { + 'examples_dirs': 'examples', + 'gallery_dirs': 'auto_examples', +} + +# -- markdown options ----------------------------------------------------------- + +md_image_dest = "media" +md_link_replace = { + '#onnxruntimesessionoptionsenable-profiling)': '#class-onnxruntimesessionoptions)', +} + +# -- Setup actions ----------------------------------------------------------- + +def setup(app): + # Placeholder to initialize the folder before + # generating the documentation. + return app + diff --git a/docs/python/examples/README.txt b/docs/python/examples/README.txt new file mode 100644 index 0000000000000..c975baa3c7260 --- /dev/null +++ b/docs/python/examples/README.txt @@ -0,0 +1,5 @@ +Gallery of examples +=================== + +.. toctree:: + :maxdepth: 1 diff --git a/docs/python/examples/Sannosawa1.jpg b/docs/python/examples/Sannosawa1.jpg new file mode 100644 index 0000000000000..9684561727be5 Binary files /dev/null and b/docs/python/examples/Sannosawa1.jpg differ diff --git a/docs/python/examples/plot_backend.py b/docs/python/examples/plot_backend.py new file mode 100644 index 0000000000000..f791488340a3a --- /dev/null +++ b/docs/python/examples/plot_backend.py @@ -0,0 +1,50 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +""" + +.. _l-example-backend-api: + +ONNX Runtime Backend for ONNX +============================= + +*ONNX Runtime* extends the +`onnx backend API `_ +to run predictions using this runtime. +Let's use the API to compute the prediction +of a simple logistic regression model. +""" +import numpy as np +from onnxruntime import datasets +import onnxruntime.backend as backend +from onnx import load + +name = datasets.get_example("logreg_iris.onnx") +model = load(name) + +rep = backend.prepare(model, 'CPU') +x = np.array([[-1.0, -2.0]], dtype=np.float32) +label, proba = rep.run(x) +print("label={}".format(label)) +print("probabilities={}".format(proba)) + +######################################## +# The device depends on how the package was compiled, +# GPU or CPU. +from onnxruntime import get_device +print(get_device()) + +######################################## +# The backend can also directly load the model +# without using *onnx*. + +rep = backend.prepare(name, 'CPU') +x = np.array([[-1.0, -2.0]], dtype=np.float32) +label, proba = rep.run(x) +print("label={}".format(label)) +print("probabilities={}".format(proba)) + +####################################### +# The backend API is implemented by other frameworks +# and makes it easier to switch between multiple runtimes +# with the same API. diff --git a/docs/python/examples/plot_common_errors.py b/docs/python/examples/plot_common_errors.py new file mode 100644 index 0000000000000..7ba5c66180b88 --- /dev/null +++ b/docs/python/examples/plot_common_errors.py @@ -0,0 +1,104 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +""" +.. _l-example-simple-usage: + +Common errors with onnxruntime +============================== + +This example looks into several common situations +in which *onnxruntime* does not return the model +prediction but raises an exception instead. +It starts by loading the model trained in example +:ref:`l-logreg-example` which produced a logistic regression +trained on *Iris* datasets. The model takes +a vector of dimension 2 and returns a class among three. +""" +import onnxruntime as rt +import numpy +from onnxruntime.datasets import get_example + +example2 = get_example("logreg_iris.onnx") +sess = rt.InferenceSession(example2) + +input_name = sess.get_inputs()[0].name +output_name = sess.get_outputs()[0].name + +############################# +# The first example fails due to *bad types*. +# *onnxruntime* only expects single floats (4 bytes) +# and cannot handle any other kind of floats. + +try: + x = numpy.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=numpy.float64) + sess.run([output_name], {input_name: x}) +except Exception as e: + print("Unexpected type") + print("{0}: {1}".format(type(e), e)) + +######################### +# The model fails to return an output if the name +# is misspelled. + +try: + x = numpy.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=numpy.float32) + sess.run(["misspelled"], {input_name: x}) +except Exception as e: + print("Misspelled output name") + print("{0}: {1}".format(type(e), e)) + +########################### +# The output name is optional, it can be replaced by *None* +# and *onnxruntime* will then return all the outputs. + +x = numpy.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=numpy.float32) +res = sess.run(None, {input_name: x}) +print("All outputs") +print(res) + +######################### +# The same goes if the input name is misspelled. + +try: + x = numpy.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=numpy.float32) + sess.run([output_name], {"misspelled": x}) +except Exception as e: + print("Misspelled input name") + print("{0}: {1}".format(type(e), e)) + +######################### +# *onnxruntime* does not necessarily fail if the input +# dimension is a multiple of the expected input dimension. + +for x in [ + numpy.array([1.0, 2.0, 3.0, 4.0], dtype=numpy.float32), + numpy.array([[1.0, 2.0, 3.0, 4.0]], dtype=numpy.float32), + numpy.array([[1.0, 2.0], [3.0, 4.0]], dtype=numpy.float32), + numpy.array([1.0, 2.0, 3.0], dtype=numpy.float32), + numpy.array([[1.0, 2.0, 3.0]], dtype=numpy.float32), + ]: + r = sess.run([output_name], {input_name: x}) + print("Shape={0} and predicted labels={1}".format(x.shape, r)) + +for x in [ + numpy.array([1.0, 2.0, 3.0, 4.0], dtype=numpy.float32), + numpy.array([[1.0, 2.0, 3.0, 4.0]], dtype=numpy.float32), + numpy.array([[1.0, 2.0], [3.0, 4.0]], dtype=numpy.float32), + numpy.array([1.0, 2.0, 3.0], dtype=numpy.float32), + numpy.array([[1.0, 2.0, 3.0]], dtype=numpy.float32), + ]: + r = sess.run(None, {input_name: x}) + print("Shape={0} and predicted probabilities={1}".format(x.shape, r[1])) + +######################### +# It does not fail either if the number of dimension +# is higher than expects but produces a warning. + +for x in [ + numpy.array([[[1.0, 2.0], [3.0, 4.0]]], dtype=numpy.float32), + numpy.array([[[1.0, 2.0, 3.0]]], dtype=numpy.float32), + numpy.array([[[1.0, 2.0]], [[3.0, 4.0]]], dtype=numpy.float32), + ]: + r = sess.run([output_name], {input_name: x}) + print("Shape={0} and predicted labels={1}".format(x.shape, r)) diff --git a/docs/python/examples/plot_convert_pipeline_vectorizer.py b/docs/python/examples/plot_convert_pipeline_vectorizer.py new file mode 100644 index 0000000000000..c24919d17a568 --- /dev/null +++ b/docs/python/examples/plot_convert_pipeline_vectorizer.py @@ -0,0 +1,101 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +""" +Train, convert and predict with ONNX Runtime +============================================ + +This example demonstrates an end to end scenario +starting with the training of a scikit-learn pipeline +which takes as inputs not a regular vector but a +dictionary ``{ int: float }`` as its first step is a +`DictVectorizer `_. + +.. contents:: + :local: + +Train a pipeline +++++++++++++++++ + +The first step consists in retrieving the boston datset. +""" +import pandas +from sklearn.datasets import load_boston +boston = load_boston() +X, y = boston.data, boston.target + +from sklearn.model_selection import train_test_split +X_train, X_test, y_train, y_test = train_test_split(X, y) +X_train_dict = pandas.DataFrame(X_train[:,1:]).T.to_dict().values() +X_test_dict = pandas.DataFrame(X_test[:,1:]).T.to_dict().values() + +#################################### +# We create a pipeline. + +from sklearn.pipeline import make_pipeline +from sklearn.ensemble import GradientBoostingRegressor +from sklearn.feature_extraction import DictVectorizer +pipe = make_pipeline( + DictVectorizer(sparse=False), + GradientBoostingRegressor()) + +pipe.fit(X_train_dict, y_train) + +#################################### +# We compute the prediction on the test set +# and we show the confusion matrix. +from sklearn.metrics import r2_score + +pred = pipe.predict(X_test_dict) +print(r2_score(y_test, pred)) + +#################################### +# Conversion to ONNX format +# +++++++++++++++++++++++++ +# +# We use module +# `onnxmltools `_ +# to convert the model into ONNX format. + +from onnxmltools import convert_sklearn +from onnxmltools.utils import save_model +from onnxmltools.convert.common.data_types import FloatTensorType, Int64TensorType, DictionaryType, SequenceType + +# initial_type = [('float_input', DictionaryType(Int64TensorType([1]), FloatTensorType([])))] +initial_type = [('float_input', DictionaryType(Int64TensorType([1]), FloatTensorType([])))] +onx = convert_sklearn(pipe, initial_types=initial_type) +save_model(onx, "pipeline_vectorize.onnx") + +################################## +# We load the model with ONNX Runtime and look at +# its input and output. +import onnxruntime as rt +sess = rt.InferenceSession("pipeline_vectorize.onnx") + +import numpy +inp, out = sess.get_inputs()[0], sess.get_outputs()[0] +print("input name='{}' and shape={} and type={}".format(inp.name, inp.shape, inp.type)) +print("output name='{}' and shape={} and type={}".format(out.name, out.shape, out.type)) + +################################## +# We compute the predictions. +# We could do that in one call: + +try: + pred_onx = sess.run([out.name], {inp.name: X_test_dict})[0] +except RuntimeError as e: + print(e) + +############################# +# But it fails because, in case of a DictVectorizer, +# ONNX Runtime expects one observation at a time. +pred_onx = [sess.run([out.name], {inp.name: row})[0][0, 0] for row in X_test_dict] + +############################### +# We compare them to the model's ones. +print(r2_score(pred, pred_onx)) + +######################### +# Very similar. *ONNX Runtime* uses floats instead of doubles, +# that explains the small discrepencies. + diff --git a/docs/python/examples/plot_dl_keras.py b/docs/python/examples/plot_dl_keras.py new file mode 100644 index 0000000000000..8c0e7d2732768 --- /dev/null +++ b/docs/python/examples/plot_dl_keras.py @@ -0,0 +1,82 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +""" + +.. _l-example-backend-api-tensorflow: + +ONNX Runtime for Keras +====================== + +The following demonstrates how to compute the predictions +of a pretrained deep learning model obtained from +`keras `_ +with *onnxruntime*. The conversion requires +`keras `_, +`tensorflow `_, +`onnxmltools `_ +but then only *onnxruntime* is required +to compute the predictions. +""" +import os +if not os.path.exists('dense121.onnx'): + from keras.applications.densenet import DenseNet121 + model = DenseNet121(include_top=True, weights='imagenet') + + from onnxmltools import convert_keras + onx = convert_keras(model, 'dense121.onnx') + + from onnxmltools.utils import save_model + save_model(onx, "dense121.onnx") + +################################## +# Let's load an image (source: wikipedia). + +from keras.preprocessing.image import array_to_img, img_to_array, load_img +img = load_img('Sannosawa1.jpg') +ximg = img_to_array(img) + +import matplotlib.pyplot as plt +plt.imshow(ximg / 255) +plt.axis('off') + +############################################# +# Let's load the model with onnxruntime. +import onnxruntime as rt + +sess = rt.InferenceSession('dense121.onnx') + +print("The model expects input shape:", sess.get_inputs()[0].shape) +print("image shape:", ximg.shape) + +####################################### +# Let's resize the image. +from skimage.transform import resize +import numpy + +ximg224 = resize(ximg / 255, (224, 224, 3), anti_aliasing=True) +ximg = ximg224[numpy.newaxis, :, :, :] +ximg = ximg.astype(numpy.float32) + +print("new shape:", ximg.shape) + +################################## +# Let's compute the output. + +input_name = sess.get_inputs()[0].name +res = sess.run(None, {input_name: ximg}) +prob = res[0] +print(prob.ravel()[:10]) # Too big to be displayed. + + +################################## +# Let's get more comprehensive results. + +from keras.applications.densenet import decode_predictions +decoded = decode_predictions(prob) + +import pandas +df = pandas.DataFrame(decoded[0], columns=["class_id", "name", "P"]) +print(df) + + diff --git a/docs/python/examples/plot_load_and_predict.py b/docs/python/examples/plot_load_and_predict.py new file mode 100644 index 0000000000000..feb369feb2e27 --- /dev/null +++ b/docs/python/examples/plot_load_and_predict.py @@ -0,0 +1,53 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +""" +.. _l-example-simple-usage: + +Load and predict with ONNX Runtime and a very simple model +========================================================== + +This example demonstrates how to load a model and compute +the output for an input vector. It also shows how to +retrieve the definition of its inputs and outputs. +""" + +import onnxruntime as rt +import numpy +from onnxruntime.datasets import get_example + +######################### +# Let's load a very simple model. +# The model is available on github `onnx...test_sigmoid `_. + +example1 = get_example("sigmoid.onnx") +sess = rt.InferenceSession(example1) + +######################### +# Let's see the input name and shape. + +input_name = sess.get_inputs()[0].name +print("input name", input_name) +input_shape = sess.get_inputs()[0].shape +print("input shape", input_shape) +input_type = sess.get_inputs()[0].type +print("input type", input_type) + +######################### +# Let's see the output name and shape. + +output_name = sess.get_outputs()[0].name +print("output name", output_name) +output_shape = sess.get_outputs()[0].shape +print("output shape", output_shape) +output_type = sess.get_outputs()[0].type +print("output type", output_type) + +######################### +# Let's compute its outputs (or predictions if it is a machine learned model). + +import numpy.random +x = numpy.random.random((3,4,5)) +x = x.astype(numpy.float32) +res = sess.run([output_name], {input_name: x}) +print(res) diff --git a/docs/python/examples/plot_metadata.py b/docs/python/examples/plot_metadata.py new file mode 100644 index 0000000000000..eb091bdde7ba9 --- /dev/null +++ b/docs/python/examples/plot_metadata.py @@ -0,0 +1,43 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +""" +Metadata +======== + +ONNX format contains metadata related to how the +model was produced. It is useful when the model +is deployed to production to keep track of which +instance was used at a specific time. +Let's see how to do that with a simple +logistic regression model trained with +*scikit-learn* and converted with *onnxmltools*. +""" + +from onnxruntime.datasets import get_example +example = get_example("logreg_iris.onnx") + +import onnx +model = onnx.load(example) + +print("doc_string={}".format(model.doc_string)) +print("domain={}".format(model.domain)) +print("ir_version={}".format(model.ir_version)) +print("metadata_props={}".format(model.metadata_props)) +print("model_version={}".format(model.model_version)) +print("producer_name={}".format(model.producer_name)) +print("producer_version={}".format(model.producer_version)) + +############################# +# With *ONNX Runtime*: + +from onnxruntime import InferenceSession +sess = InferenceSession(example) +meta = sess.get_modelmeta() + +print("custom_metadata_map={}".format(meta.custom_metadata_map)) +print("description={}".format(meta.description)) +print("domain={}".format(meta.domain, meta.domain)) +print("graph_name={}".format(meta.graph_name)) +print("producer_name={}".format(meta.producer_name)) +print("version={}".format(meta.version)) diff --git a/docs/python/examples/plot_pipeline.py b/docs/python/examples/plot_pipeline.py new file mode 100644 index 0000000000000..5063479492429 --- /dev/null +++ b/docs/python/examples/plot_pipeline.py @@ -0,0 +1,69 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +""" +Draw a pipeline +=============== + +There is no other way to look into one model stored +in ONNX format than looking into its node with +*onnx*. This example demonstrates +how to draw a model and to retrieve it in *json* +format. + +.. contents:: + :local: + +Retrieve a model in JSON format ++++++++++++++++++++++++++++++++ + +That's the most simple way. +""" + +from onnxruntime.datasets import get_example +example1 = get_example("mul_1.pb") + +import onnx +model = onnx.load(example1) # model is a ModelProto protobuf message + +print(model) + + +################################# +# Draw a model with ONNX +# ++++++++++++++++++++++ +# We use `net_drawer.py `_ +# included in *onnx* package. +# We use *onnx* to load the model +# in a different way than before. + + +from onnx import ModelProto +model = ModelProto() +with open(example1, 'rb') as fid: + content = fid.read() + model.ParseFromString(content) + +################################### +# We convert it into a graph. +from onnx.tools.net_drawer import GetPydotGraph, GetOpNodeProducer +pydot_graph = GetPydotGraph(model.graph, name=model.graph.name, rankdir="LR", + node_producer=GetOpNodeProducer("docstring")) +pydot_graph.write_dot("graph.dot") + +####################################### +# Then into an image +import os +os.system('dot -O -Tpng graph.dot') + +################################ +# Which we display... +import matplotlib.pyplot as plt +image = plt.imread("graph.dot.png") +plt.imshow(image) + + + + + + diff --git a/docs/python/examples/plot_profiling.py b/docs/python/examples/plot_profiling.py new file mode 100644 index 0000000000000..3844962033f9d --- /dev/null +++ b/docs/python/examples/plot_profiling.py @@ -0,0 +1,55 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +""" + +.. _l-example-profiling: + +Profile the execution of a simple model +======================================= + +*ONNX Runtime* can profile the execution of the model. +This example shows how to interpret the results. +""" + +import onnxruntime as rt +import numpy +from onnxruntime.datasets import get_example + +######################### +# Let's load a very simple model and compute some prediction. + +example1 = get_example("mul_1.pb") +sess = rt.InferenceSession(example1) +input_name = sess.get_inputs()[0].name + +x = numpy.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=numpy.float32) +res = sess.run(None, {input_name: x}) +print(res) + +######################### +# We need to enable to profiling +# before running the predictions. + +options = rt.SessionOptions() +options.enable_profiling = True +sess_profile = rt.InferenceSession(example1, options) +input_name = sess.get_inputs()[0].name + +x = numpy.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=numpy.float32) + +sess.run(None, {input_name: x}) +prof_file = sess_profile.end_profiling() +print(prof_file) + +########################### +# The results are stored un a file in JSON format. +# Let's see what it contains. +import json +with open(prof_file, "r") as f: + sess_time = json.load(f) +import pprint +pprint.pprint(sess_time) + + + diff --git a/docs/python/examples/plot_train_convert_predict.py b/docs/python/examples/plot_train_convert_predict.py new file mode 100644 index 0000000000000..6c8f85967646f --- /dev/null +++ b/docs/python/examples/plot_train_convert_predict.py @@ -0,0 +1,218 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +""" + +.. _l-logreg-example: + +Train, convert and predict with ONNX Runtime +============================================ + +This example demonstrates an end to end scenario +starting with the training of a machine learned model +to its use in its converted from. + +.. contents:: + :local: + +Train a logistic regression ++++++++++++++++++++++++++++ + +The first step consists in retrieving the iris datset. +""" + +from sklearn.datasets import load_iris +iris = load_iris() +X, y = iris.data, iris.target + +from sklearn.model_selection import train_test_split +X_train, X_test, y_train, y_test = train_test_split(X, y) + +#################################### +# Then we fit a model. + +from sklearn.linear_model import LogisticRegression +clr = LogisticRegression() +clr.fit(X_train, y_train) + +#################################### +# We compute the prediction on the test set +# and we show the confusion matrix. +from sklearn.metrics import confusion_matrix + +pred = clr.predict(X_test) +print(confusion_matrix(y_test, pred)) + +#################################### +# Conversion to ONNX format +# +++++++++++++++++++++++++ +# +# We use module +# `onnxmltools `_ +# to convert the model into ONNX format. + +from onnxmltools import convert_sklearn +from onnxmltools.utils import save_model +from onnxmltools.convert.common.data_types import FloatTensorType + +initial_type = [('float_input', FloatTensorType([1, 4]))] +onx = convert_sklearn(clr, initial_types=initial_type) +save_model(onx, "logreg_iris.onnx") + +################################## +# We load the model with ONNX Runtime and look at +# its input and output. + +import onnxruntime as rt +sess = rt.InferenceSession("logreg_iris.onnx") + +print("input name='{}' and shape={}".format(sess.get_inputs()[0].name, sess.get_inputs()[0].shape)) +print("output name='{}' and shape={}".format(sess.get_outputs()[0].name, sess.get_outputs()[0].shape)) + +################################## +# We compute the predictions. + +input_name = sess.get_inputs()[0].name +label_name = sess.get_outputs()[0].name + +import numpy +pred_onx = sess.run([label_name], {input_name: X_test.astype(numpy.float32)})[0] +print(confusion_matrix(pred, pred_onx)) + +################################### +# The prediction are perfectly identical. +# +# Probabilities +# +++++++++++++ +# +# Probabilities are needed to compute other +# relevant metrics such as the ROC Curve. +# Let's see how to get them first with +# scikit-learn. + +prob_sklearn = clr.predict_proba(X_test) +print(prob_sklearn[:3]) + +############################# +# And then with ONNX Runtime. +# The probabilies appear to be + +prob_name = sess.get_outputs()[1].name +prob_rt = sess.run([prob_name], {input_name: X_test.astype(numpy.float32)})[0] + +import pprint +pprint.pprint(prob_rt[0:3]) + +############################### +# Let's benchmark. +from timeit import Timer + +def speed(inst, number=10, repeat=20): + timer = Timer(inst, globals=globals()) + raw = numpy.array(timer.repeat(repeat, number=number)) + ave = raw.sum() / len(raw) / number + mi, ma = raw.min() / number, raw.max() / number + print("Average %1.3g min=%1.3g max=%1.3g" % (ave, mi, ma)) + return ave + +print("Execution time for clr.predict") +speed("clr.predict(X_test)") + +print("Execution time for ONNX Runtime") +speed("sess.run([label_name], {input_name: X_test.astype(numpy.float32)})[0]") + +############################### +# Let's benchmark a scenario similar to what a webservice +# experiences: the model has to do one prediction at a time +# as opposed to a batch of prediction. + +def loop(X_test, fct, n=None): + nrow = X_test.shape[0] + if n is None: + n = nrow + for i in range(0, n): + im = i % nrow + fct(X_test[im: im+1]) + +print("Execution time for clr.predict") +speed("loop(X_test, clr.predict, 100)") + +def sess_predict(x): + return sess.run([label_name], {input_name: x.astype(numpy.float32)})[0] + +print("Execution time for sess_predict") +speed("loop(X_test, sess_predict, 100)") + +##################################### +# Let's do the same for the probabilities. + +print("Execution time for predict_proba") +speed("loop(X_test, clr.predict_proba, 100)") + +def sess_predict_proba(x): + return sess.run([prob_name], {input_name: x.astype(numpy.float32)})[0] + +print("Execution time for sess_predict_proba") +speed("loop(X_test, sess_predict_proba, 100)") + +##################################### +# This second comparison is better as +# ONNX Runtime, in this experience, +# computes the label and the probabilities +# in every case. + +########################################## +# Benchmark with RandomForest +# +++++++++++++++++++++++++++ +# +# We first train and save a model in ONNX format. +from sklearn.ensemble import RandomForestClassifier +rf = RandomForestClassifier() +rf.fit(X_train, y_train) + +initial_type = [('float_input', FloatTensorType([1, 4]))] +onx = convert_sklearn(rf, initial_types=initial_type) +save_model(onx, "rf_iris.onnx") + +################################### +# We compare. + +sess = rt.InferenceSession("rf_iris.onnx") + +def sess_predict_proba_rf(x): + return sess.run([prob_name], {input_name: x.astype(numpy.float32)})[0] + +print("Execution time for predict_proba") +speed("loop(X_test, rf.predict_proba, 100)") + +print("Execution time for sess_predict_proba") +speed("loop(X_test, sess_predict_proba_rf, 100)") + +################################## +# Let's see with different number of trees. + +measures = [] + +for n_trees in range(5, 51, 5): + print(n_trees) + rf = RandomForestClassifier(n_estimators=n_trees) + rf.fit(X_train, y_train) + initial_type = [('float_input', FloatTensorType([1, 4]))] + onx = convert_sklearn(rf, initial_types=initial_type) + save_model(onx, "rf_iris_%d.onnx" % n_trees) + sess = rt.InferenceSession("rf_iris_%d.onnx" % n_trees) + def sess_predict_proba_loop(x): + return sess.run([prob_name], {input_name: x.astype(numpy.float32)})[0] + tsk = speed("loop(X_test, rf.predict_proba, 100)", number=5, repeat=5) + trt = speed("loop(X_test, sess_predict_proba_loop, 100)", number=5, repeat=5) + measures.append({'n_trees': n_trees, 'sklearn': tsk, 'rt': trt}) + +from pandas import DataFrame +df = DataFrame(measures) +ax = df.plot(x="n_trees", y="sklearn", label="scikit-learn", c="blue", logy=True) +df.plot(x="n_trees", y="rt", label="onnxruntime", + ax=ax, c="green", logy=True) +ax.set_xlabel("Number of trees") +ax.set_ylabel("Prediction time (s)") +ax.set_title("Speed comparison between scikit-learn and ONNX Runtime\nFor a random forest on Iris dataset") +ax.legend() diff --git a/docs/python/examples_md.rst b/docs/python/examples_md.rst new file mode 100644 index 0000000000000..2c0ca6b4e019f --- /dev/null +++ b/docs/python/examples_md.rst @@ -0,0 +1,35 @@ + + +.. only:: md + + Gallery of examples + =================== + + The first series of examples briefly goes into the main + feature *ONNX Runtime* implements. Each of them run in a + few seconds and relies on machine learned models + trained with `scikit-learn `_. + + .. toctree:: + :maxdepth: 1 + :caption: Contents: + + auto_examples/plot_load_and_predict + auto_examples/plot_common_errors + auto_examples/plot_train_convert_predict + auto_examples/plot_pipeline + auto_examples/plot_backend + auto_examples/plot_convert_pipeline_vectorizer + auto_examples/plot_metadata + auto_examples/plot_profiling + + The second series is about deep learning. + Once converted to *ONNX*, the predictions can be + computed with *onnxruntime* without having any + dependencies on the framework used to train the model. + + .. toctree:: + :maxdepth: 1 + :caption: Contents: + + auto_examples/plot_dl_keras diff --git a/docs/python/intro.rst b/docs/python/intro.rst new file mode 100644 index 0000000000000..d2d6b62f2d2a3 --- /dev/null +++ b/docs/python/intro.rst @@ -0,0 +1,51 @@ + +Python Bindings for ONNX Runtime +================================ + +ONNX Runtime enables high-performance evaluation of trained machine learning (ML) +models while keeping resource usage low. +Building on Microsoft's dedication to the +`Open Neural Network Exchange (ONNX) `_ +community, it supports traditional ML models as well +as Deep Learning algorithms in the +`ONNX-ML format `_. + +.. only:: html + + .. toctree:: + :maxdepth: 1 + + tutorial + api_summary + auto_examples/index + + :ref:`genindex` + +.. only:: md + + .. toctree:: + :maxdepth: 1 + :caption: Contents: + + tutorial + api_summary + examples_md + +The core library is implemented in C++. +*ONNX Runtime* is available on +PyPi for Linux Ubuntu 16.04, Python 3.5+ for both +`CPU `_ and +`GPU `_. +This example demonstrates a simple prediction for an +`ONNX-ML format `_ +model. The following file ``model.onnx`` is taken from +github `onnx...test_sigmoid `_. + +:: + + import onnxruntime as rt + sess = rt.InferenceSession("model.onnx") + input_name = sess.get_inputs()[0].name + X = numpy.random.random((3,4,5)).astype(numpy.float32) + res = sess.run([output_name], {input_name: x}) + pred_onnx = sess.run(None, {input_name: X}) diff --git a/docs/python/tutorial.rst b/docs/python/tutorial.rst new file mode 100644 index 0000000000000..50d65196b3f4f --- /dev/null +++ b/docs/python/tutorial.rst @@ -0,0 +1,73 @@ + +======== +Tutorial +======== + +*ONNX Runtime* provides an easy way to run +machine learned models with high performance on CPU or GPU +without dependencies on the training framework. +Machine learning frameworks are usually optimized for +batch training rather than for prediction, which is a +more common scenario in applications, sites, and services. +At a high level, you can: + +1. Train a model using your favorite framework. +2. Convert or export the model into ONNX format. + See `ONNX Tutorials `_ + for more details. +3. Load and run the model using *ONNX Runtime*. + +In this tutorial, we will briefly create a +pipeline with *scikit-learn*, convert it into +ONNX format and run the first predictions. + +Step 1: Train a model using your favorite framework ++++++++++++++++++++++++++++++++++++++++++++++++++++ + +We'll use the famous iris datasets. + +:: + + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + iris = load_iris() + X, y = iris.data, iris.target + X_train, X_test, y_train, y_test = train_test_split(X, y) + + from sklearn.linear_model import LogisticRegression + clr = LogisticRegression() + clr.fit(X_train, y_train) + +Step 2: Convert or export the model into ONNX format +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +`ONNX `_ is a format to describe +the machine learned model. +It defines a set of commonly used operators to compose models. +There are `tools `_ +to convert other model formats into ONNX. Here we will use +`ONNXMLTools `_. + +:: + + from onnxmltools import convert_sklearn + from onnxmltools.utils import save_model + from onnxmltools.convert.common.data_types import FloatTensorType + + initial_type = [('float_input', FloatTensorType([1, 4]))] + onx = convert_sklearn(clr, initial_types=initial_type) + save_model(onx, "logreg_iris.onnx") + +Step 3: Load and run the model using ONNX Runtime ++++++++++++++++++++++++++++++++++++++++++++++++++ + +We will use *ONNX Runtime* to compute the predictions +for this machine learning model. + +:: + + import onnxruntime as rt + sess = rt.InferenceSession("logreg_iris.onnx") + input_name = sess.get_inputs()[0].name + + pred_onx = sess.run([label_name], {input_name: X_test.astype(numpy.float32)})[0] diff --git a/include/onnxruntime/core/common/code_location.h b/include/onnxruntime/core/common/code_location.h new file mode 100644 index 0000000000000..ff6506c9a7abb --- /dev/null +++ b/include/onnxruntime/core/common/code_location.h @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include + +namespace onnxruntime { +/** + CodeLocation captures information on where in the source code a message came from. +*/ +struct CodeLocation { + /** + @param file_path Usually the value of __FILE__ + @param line Usually the value of __LINE__ + @param func Usually the value of __PRETTY_FUNCTION__ or __FUNCTION__ + */ + CodeLocation(const char* file_path, const int line, const char* func) + : file_and_path{file_path}, line_num{line}, function{func} { + } + + /** + @param file_path Usually the value of __FILE__ + @param line Usually the value of __LINE__ + @param func Usually the value of __PRETTY_FUNCTION__ or __FUNCTION__ + @param stacktrace Stacktrace from source of message. + */ + CodeLocation(const char* file_path, const int line, const char* func, const std::vector& stacktrace) + : file_and_path{file_path}, line_num{line}, function{func}, stacktrace(stacktrace) { + } + + std::string FileNoPath() const { + // assuming we always have work to do, so not trying to avoid creating a new string if + // no path was removed. + return file_and_path.substr(file_and_path.find_last_of("/\\") + 1); + } + + enum Format { + kFilename, + kFilenameAndPath + }; + + std::string ToString(Format format = Format::kFilename) const { + std::ostringstream out; + out << (format == Format::kFilename ? FileNoPath() : file_and_path) << ":" << line_num << " " << function; + return out.str(); + } + + const std::string file_and_path; + const int line_num; + const std::string function; + const std::vector stacktrace; +}; + +} // namespace onnxruntime diff --git a/include/onnxruntime/core/common/common.h b/include/onnxruntime/core/common/common.h new file mode 100644 index 0000000000000..6ae0cd6ac07c8 --- /dev/null +++ b/include/onnxruntime/core/common/common.h @@ -0,0 +1,215 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// Portions Copyright (c) Microsoft Corporation + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/common/code_location.h" +#include "core/common/exceptions.h" +#include "core/common/status.h" + +namespace onnxruntime { + +using TimePoint = std::chrono::high_resolution_clock::time_point; + +// Using statements for common classes that we refer to in ONNXRuntime very often. +// TODO(Task:137) Remove 'using' statements from header files +using common::Status; + +#ifdef _WIN32 +#define ONNXRUNTIME_UNUSED_PARAMETER(x) (x) +#else +#define ONNXRUNTIME_UNUSED_PARAMETER(x) (void)(x) +#endif + +#ifndef ONNXRUNTIME_HAVE_ATTRIBUTE +#ifdef __has_attribute +#define ONNXRUNTIME_HAVE_ATTRIBUTE(x) __has_attribute(x) +#else +#define ONNXRUNTIME_HAVE_ATTRIBUTE(x) 0 +#endif +#endif + +// ONNXRUNTIME_ATTRIBUTE_UNUSED +// +// Prevents the compiler from complaining about or optimizing away variables +// that appear unused on Linux +#if ONNXRUNTIME_HAVE_ATTRIBUTE(unused) || (defined(__GNUC__) && !defined(__clang__)) +#undef ONNXRUNTIME_ATTRIBUTE_UNUSED +#define ONNXRUNTIME_ATTRIBUTE_UNUSED __attribute__((__unused__)) +#else +#define ONNXRUNTIME_ATTRIBUTE_UNUSED +#endif + +// macro to explicitly ignore the return value from a function call so Code Analysis doesn't complain +#define ONNXRUNTIME_IGNORE_RETURN_VALUE(fn) \ + static_cast(fn) + +std::vector GetStackTrace(); + +// __PRETTY_FUNCTION__ isn't a macro on gcc, so use a check for _MSC_VER +// so we only define it as one for MSVC +#if (_MSC_VER && !defined(__PRETTY_FUNCTION__)) +#define __PRETTY_FUNCTION__ __FUNCTION__ +#endif + +// Capture where a message is coming from. Use __FUNCTION__ rather than the much longer __PRETTY_FUNCTION__ +#define ONNXRUNTIME_WHERE \ + ::onnxruntime::CodeLocation(__FILE__, __LINE__, __FUNCTION__) + +#define ONNXRUNTIME_WHERE_WITH_STACK \ + ::onnxruntime::CodeLocation(__FILE__, __LINE__, __PRETTY_FUNCTION__, ::onnxruntime::GetStackTrace()) + +// Throw an exception with optional message. +// NOTE: The arguments get streamed into a string via ostringstream::operator<< +// DO NOT use a printf format string, as that will not work as you expect. +#define ONNXRUNTIME_THROW(...) \ + throw ::onnxruntime::OnnxRuntimeException(ONNXRUNTIME_WHERE_WITH_STACK, ::onnxruntime::MakeString(__VA_ARGS__)) + +// Just in order to mark things as not implemented. Do not use in final code. +#define ONNXRUNTIME_NOT_IMPLEMENTED(...) \ + throw ::onnxruntime::NotImplementedException(::onnxruntime::MakeString(__VA_ARGS__)) + +// Check condition. +// NOTE: The arguments get streamed into a string via ostringstream::operator<< +// DO NOT use a printf format string, as that will not work as you expect. +#define ONNXRUNTIME_ENFORCE(condition, ...) \ + if (!(condition)) \ + throw ::onnxruntime::OnnxRuntimeException(ONNXRUNTIME_WHERE_WITH_STACK, #condition, \ + ::onnxruntime::MakeString(__VA_ARGS__)) + +#define ONNXRUNTIME_MAKE_STATUS(category, code, ...) \ + ::onnxruntime::common::Status(::onnxruntime::common::category, \ + ::onnxruntime::common::code, \ + ::onnxruntime::MakeString(__VA_ARGS__)) + +// Check condition. if not met, return status. +#define ONNXRUNTIME_RETURN_IF_NOT(condition, ...) \ + if (!(condition)) { \ + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Not satsified: " #condition "\n", \ + ONNXRUNTIME_WHERE.ToString(), ::onnxruntime::MakeString(__VA_ARGS__)); \ + } + +// Macros to disable the copy and/or move ctor and assignment methods +// These are usually placed in the private: declarations for a class. + +#define ONNXRUNTIME_DISALLOW_COPY(TypeName) TypeName(const TypeName&) = delete + +#define ONNXRUNTIME_DISALLOW_ASSIGNMENT(TypeName) TypeName& operator=(const TypeName&) = delete + +#define ONNXRUNTIME_DISALLOW_COPY_AND_ASSIGNMENT(TypeName) \ + ONNXRUNTIME_DISALLOW_COPY(TypeName); \ + ONNXRUNTIME_DISALLOW_ASSIGNMENT(TypeName) + +#define ONNXRUNTIME_DISALLOW_MOVE(TypeName) \ + TypeName(TypeName&&) = delete; \ + TypeName& operator=(TypeName&&) = delete + +#define ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(TypeName) \ + ONNXRUNTIME_DISALLOW_COPY_AND_ASSIGNMENT(TypeName); \ + ONNXRUNTIME_DISALLOW_MOVE(TypeName) + +#define ONNXRUNTIME_RETURN_IF_ERROR(expr) \ + do { \ + auto _status = (expr); \ + if ((!_status.IsOK())) return _status; \ + } while (0) + +// use this macro when cannot early return +#define ONNXRUNTIME_CHECK_AND_SET_RETVAL(expr) \ + do { \ + if (retval.IsOK()) { \ + retval = (expr); \ + } \ + } while (0) + +// C++ Core Guideline check suppression +#ifdef _MSC_VER +#define GSL_SUPPRESS(tag) [[gsl::suppress(tag)]] +#else +#define GSL_SUPPRESS(tag) +#endif + +inline void MakeStringInternal(std::ostringstream& /*ss*/) noexcept { +} + +template +inline void MakeStringInternal(std::ostringstream& ss, const T& t) noexcept { + ss << t; +} + +template +inline void MakeStringInternal(std::ostringstream& ss, const T& t, const Args&... args) noexcept { + ::onnxruntime::MakeStringInternal(ss, t); + ::onnxruntime::MakeStringInternal(ss, args...); +} + +template +std::string MakeString(const Args&... args) { + std::ostringstream ss; + ::onnxruntime::MakeStringInternal(ss, args...); + return std::string(ss.str()); +} + +// Specializations for already-a-string types. +template <> +inline std::string MakeString(const std::string& str) { + return str; +} +inline std::string MakeString(const char* p_str) { + return p_str; +} + +inline long long TimeDiffMicroSeconds(TimePoint start_time) { + auto end_time = std::chrono::high_resolution_clock::now(); + return std::chrono::duration_cast(end_time - start_time).count(); +} + +inline long long TimeDiffMicroSeconds(TimePoint start_time, TimePoint end_time) { + return std::chrono::duration_cast(end_time - start_time).count(); +} + +inline std::string GetCurrentTimeString() { + auto now = std::chrono::system_clock::now(); + auto in_time_t = std::chrono::system_clock::to_time_t(now); + std::tm local_tm; //NOLINT + +#ifdef _WIN32 + localtime_s(&local_tm, &in_time_t); +#else + localtime_r(&in_time_t, &local_tm); +#endif + + char time_str[32]; + strftime(time_str, sizeof(time_str), "%Y-%m-%d_%H-%M-%S", &local_tm); + return std::string(time_str); +} + +struct null_type {}; + +} // namespace onnxruntime diff --git a/include/onnxruntime/core/common/const_pointer_container.h b/include/onnxruntime/core/common/const_pointer_container.h new file mode 100644 index 0000000000000..9edba9e1cc551 --- /dev/null +++ b/include/onnxruntime/core/common/const_pointer_container.h @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +namespace onnxruntime { +/** + Container has T* entries. e.g. std::vector, and this class provides const access to those + via iterators and direct access, as the standard behavior only makes the pointer constant, + and not what is pointed too. i.e. you get a const pointer to T not a pointer to const T without this wrapper. + See https://stackoverflow.com/questions/8017036/understanding-const-iterator-with-pointers +*/ +template +class ConstPointerContainer { + public: + using T = typename std::remove_pointer::type; + + class ConstIterator { + public: + using const_iterator = typename Container::const_iterator; + + /** Construct iterator for container that will return const T* entries.*/ + explicit ConstIterator(const_iterator position) noexcept : current_(position) {} + + bool operator==(const ConstIterator& other) const noexcept { return current_ == other.current_; } + bool operator!=(const ConstIterator& other) const noexcept { return current_ != other.current_; } + void operator++() { ++current_; } + const T* operator*() { return *current_; } + + private: + const_iterator current_; + }; + + /** + Construct wrapper class that will provide const access to the pointers in a container of non-const pointers. + @param data Container with non-const pointers. e.g. std::vector + */ + explicit ConstPointerContainer(const Container& data) noexcept : data_(data) {} + + size_t size() const noexcept { return data_.size(); } + + ConstIterator begin() const noexcept { return ConstIterator(data_.cbegin()); } + ConstIterator end() const noexcept { return ConstIterator(data_.cend()); } + + const T* operator[](size_t index) const { return data_[index]; } + + const T* at(size_t index) const { + ONNXRUNTIME_ENFORCE(index < data_.size()); + return data_[index]; + } + + private: + const Container& data_; +}; +} // namespace onnxruntime diff --git a/include/onnxruntime/core/common/exceptions.h b/include/onnxruntime/core/common/exceptions.h new file mode 100644 index 0000000000000..31e7a9f1d58ec --- /dev/null +++ b/include/onnxruntime/core/common/exceptions.h @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "core/common/common.h" +#include "core/common/code_location.h" + +namespace onnxruntime { + +class NotImplementedException : public std::logic_error { + public: + explicit NotImplementedException(const char* _Message = "Function not yet implemented") noexcept : std::logic_error(_Message){}; + explicit NotImplementedException(const std::string& _Message = "Function not yet implemented") noexcept : std::logic_error(_Message){}; +}; + +class TypeMismatchException : public std::logic_error { + public: + TypeMismatchException() noexcept : logic_error("Type mismatch"){}; +}; + +class OnnxRuntimeException : public std::exception { + public: + OnnxRuntimeException(const CodeLocation& location, const std::string& msg) noexcept + : OnnxRuntimeException(location, nullptr, msg) { + } + + /** + Create a new exception that captures the location it was thrown from. + @param location Location in the source code the exception is being thrown from + @param failed_condition Optional string containing the condition that failed. + e.g. "tensor.Size() == input.Size()". May be nullptr. + @param msg Message containing additional information about the exception cause. + */ + OnnxRuntimeException(const CodeLocation& location, const char* failed_condition, const std::string& msg) + : location_{location} { + std::ostringstream ss; + + ss << location.ToString(CodeLocation::kFilenameAndPath); // output full path in case just the filename is ambiguous + if (failed_condition != nullptr) { + ss << " " << failed_condition << " was false."; + } + + ss << " " << msg << "\n"; + if (!location.stacktrace.empty()) { + ss << "Stacktrace:\n"; + // skip the first entry in the stacktrace as we have that information from location.ToString() + std::copy(++location.stacktrace.begin(), location.stacktrace.end(), std::ostream_iterator(ss, "\n")); + } + + what_ = ss.str(); + } + + const char* what() const noexcept override { + return what_.c_str(); + } + + private: + const CodeLocation location_; + const std::vector stacktrace_; + std::string what_; +}; + +} // namespace onnxruntime diff --git a/include/onnxruntime/core/common/logging/capture.h b/include/onnxruntime/core/common/logging/capture.h new file mode 100644 index 0000000000000..dddb36bc00dec --- /dev/null +++ b/include/onnxruntime/core/common/logging/capture.h @@ -0,0 +1,115 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include "core/common/common.h" +#include "core/common/code_location.h" +#include "core/common/logging/severity.h" + +namespace onnxruntime { +namespace logging { + +class Logger; +enum class DataType; + +/** + Class to capture the details of a log message. +*/ +class Capture { + public: + /** + Initializes a new instance of the Capture class. + @param logger The logger. + @param severity The severity. + @param category The category. + @param dataType Type of the data. + @param location The file location the log message is coming from. + */ + Capture(const Logger& logger, logging::Severity severity, const char* category, + logging::DataType dataType, const CodeLocation& location) + : logger_{&logger}, severity_{severity}, category_{category}, data_type_{dataType}, location_{location} { + } + + /** + The stream that can capture the message via operator<<. + @returns Output stream. + */ + std::ostream& Stream() noexcept { + return stream_; + } + +#ifdef _MSC_VER + // add SAL annotation for printf format string. requires Code Analysis to run to validate usage. +#define msvc_printf_check _Printf_format_string_ +#define __attribute__(x) // Disable for MSVC. Supported by GCC and CLang. +#else +#define msvc_printf_check +#endif + + /** + Captures a printf style log message. + @param name="format">The printf format. + @param name="">Arguments to the printf format if needed. + @remarks + A maximum of 2K of output will be captured currently. + Non-static method, so 'this' is implicit first arg, and we use format(printf(2,3) + */ + void CapturePrintf(msvc_printf_check const char* format, ...) __attribute__((format(printf, 2, 3))); + + /** + Process a printf style log message. + @param format The printf format. + @param ... Arguments to the printf format if needed. + @remarks + A maximum of 2K of output will be captured currently. + Note: As va_list is 'char *', we have to disambiguate this from CapturePrintf + so that something like "One string: %s", "the string" does not consider "the string" + to be the va_list. + */ + void ProcessPrintf(msvc_printf_check const char* format, va_list args); + + logging::Severity Severity() const noexcept { + return severity_; + } + + char SeverityPrefix() const noexcept { + // Carefully setup so severity_ is a valid index + GSL_SUPPRESS(bounds .2) { + return logging::SEVERITY_PREFIX[static_cast(severity_)]; + } + } + + const char* Category() const noexcept { + return category_; + } + + logging::DataType DataType() const noexcept { + return data_type_; + } + + const CodeLocation& Location() const noexcept { + return location_; + } + + std::string Message() const noexcept { + return stream_.str(); + } + + ~Capture(); + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Capture); + + const Logger* logger_; + const logging::Severity severity_; + const char* category_; + const logging::DataType data_type_; + const CodeLocation location_; + + std::ostringstream stream_; +}; +} // namespace logging +} // namespace onnxruntime diff --git a/include/onnxruntime/core/common/logging/isink.h b/include/onnxruntime/core/common/logging/isink.h new file mode 100644 index 0000000000000..4951f97fb9081 --- /dev/null +++ b/include/onnxruntime/core/common/logging/isink.h @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include "core/common/logging/logging.h" + +namespace onnxruntime { +namespace logging { +class ISink { + public: + ISink() = default; + + /** + Sends the message to the sink. + @param timestamp The timestamp. + @param logger_id The logger identifier. + @param message The captured message. + */ + void Send(const Timestamp& timestamp, const std::string& logger_id, const Capture& message) { + SendImpl(timestamp, logger_id, message); + } + + /** + Sends a Profiling Event Record to the sink. + @param Profiling Event Record + */ + virtual void SendProfileEvent(profiling::EventRecord&) const {}; + + virtual ~ISink() = default; + + private: + // Make Code Analysis happy by disabling all for now. Enable as needed. + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(ISink); + + virtual void SendImpl(const Timestamp& timestamp, const std::string& logger_id, const Capture& message) = 0; +}; +} // namespace logging +} // namespace onnxruntime diff --git a/include/onnxruntime/core/common/logging/logging.h b/include/onnxruntime/core/common/logging/logging.h new file mode 100644 index 0000000000000..a8cb3f7c24e98 --- /dev/null +++ b/include/onnxruntime/core/common/logging/logging.h @@ -0,0 +1,318 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "core/common/common.h" +#include "core/common/logging/capture.h" +#include "core/common/logging/severity.h" + +#include "core/common/logging/macros.h" + +/* + + Logging overview and expected usage: + + At program startup: + * Create one or more ISink instances. If multiple, combine using composite_sink. + * Create a LoggingManager instance with the sink/s with is_default_instance set to true + * Only one instance should be created in this way, and it should remain valid for + until the program no longer needs to produce log output. + + You can either use the static default Logger which LoggingManager will create when constructed + via LoggingManager::DefaultLogger(), or separate Logger instances each with different log ids + via LoggingManager::CreateLogger. + + The log id is passed to the ISink instance with the sink determining how the log id is used + in the output. + + LoggingManager + * creates the Logger instances used by the application + * provides a static default logger instance + * owns the log sink instance + * applies checks on severity and output of user data + + The log macros create a Capture instance to capture the information to log. + If the severity and/or user filtering settings would prevent logging, no evaluation + of the log arguments will occur, so no performance cost beyond the severity and user + filtering check. + + A sink can do further filter as needed. + +*/ + +namespace onnxruntime { +namespace profiling { + +enum EventCategory { + SESSION_EVENT = 0, + NODE_EVENT, + EVENT_CATEGORY_MAX +}; + +/* +Event descriptions for the above session events. +*/ +static constexpr const char* event_categor_names_[EVENT_CATEGORY_MAX] = { + "Session", + "Node"}; + +/* +Timing record for all events. +*/ +struct EventRecord { + EventRecord(EventCategory category, + int process_id, + int thread_id, + std::string event_name, + long long time_stamp, + long long duration, + std::unordered_map&& event_args) : cat(category), + pid(process_id), + tid(thread_id), + name(std::move(event_name)), + ts(time_stamp), + dur(duration), + args(event_args) {} + EventCategory cat; + int pid; + int tid; + std::string name; + long long ts; + long long dur; + std::unordered_map args; +}; +} // namespace profiling +namespace logging { + +using Timestamp = std::chrono::time_point; + +#ifndef NDEBUG +ONNXRUNTIME_ATTRIBUTE_UNUSED static bool vlog_enabled = true; // Set directly based on your needs. +#else +constexpr bool vlog_enabled = false; // no VLOG output +#endif + +enum class DataType { + SYSTEM = 0, ///< System data. + USER = 1 ///< Contains potentially sensitive user data. +}; + +// Internal log categories. +// Logging interface takes const char* so arbitrary values can also be used. +struct Category { + static const char* onnxruntime; ///< General output + static const char* System; ///< Log output regarding interactions with the host system + // TODO: What other high level categories are meaningful? Model? Optimizer? Execution? +}; + +class ISink; +class Logger; +class Capture; + +/// +/// The logging manager. +/// Owns the log sink and potentially provides a default Logger instance. +/// Provides filtering based on a minimum LogSeverity level, and of messages with DataType::User if enabled. +/// +class LoggingManager final { + public: + enum InstanceType { + Default, ///< Default instance of LoggingManager that should exist for the lifetime of the program + Temporal ///< Temporal instance. CreateLogger(...) should be used, however DefaultLogger() will NOT be provided via this instance. + }; + + /** + Initializes a new instance of the LoggingManager class. + @param sink The sink to write to. Use CompositeSink if you need to write to multiple places. + @param default_min_severity The default minimum severity. Messages with lower severity will be ignored unless + overridden in CreateLogger. + @param default_filter_user_data If set to true ignore messages with DataType::USER unless overridden in CreateLogger. + @param instance_type If InstanceType::Default, this is the default instance of the LoggingManager + and is expected to exist for the lifetime of the program. + It creates and owns the default logger that calls to the static DefaultLogger method return. + @param default_logger_id Logger Id to use for the default logger. nullptr/ignored if instance_type == Temporal. + @param default_max_vlog_level Default maximum level for VLOG messages to be created unless overridden in CreateLogger. + Requires a severity of kVERBOSE for VLOG messages to be logged. + */ + LoggingManager(std::unique_ptr sink, Severity default_min_severity, bool default_filter_user_data, + InstanceType instance_type, + const std::string* default_logger_id = nullptr, + int default_max_vlog_level = -1); + + /** + Creates a new logger instance which will use the provided logger_id and default severity and vlog levels. + @param logger_id The log identifier. + @returns A new Logger instance that the caller owns. + */ + std::unique_ptr CreateLogger(const std::string& logger_id); + + /** + Creates a new logger instance which will use the provided logger_id, severity and vlog levels. + @param logger_id The log identifier. + @param min_severity The minimum severity. Requests to create messages with lower severity will be ignored. + @param filter_user_data If set to true ignore messages with DataType::USER. + @param max_vlog_level Maximum level for VLOG messages to be created. + @returns A new Logger instance that the caller owns. + */ + std::unique_ptr CreateLogger(const std::string& logger_id, + Severity min_severity, bool filter_user_data, int max_vlog_level = -1); + + /** + Gets the default logger instance if set. Throws if no default logger is currently registered. + @remarks + Creating a LoggingManager instance with is_default_instance == true registers a default logger. + Note that the default logger is only valid until the LoggerManager that registered it is destroyed. + @returns The default logger if available. + */ + static const Logger& DefaultLogger(); + + /** + Logs a FATAL level message and creates an exception that can be thrown with error information. + @param category The log category. + @param location The location the log message was generated. + @param format_str The printf format string. + @param ... The printf arguments. + @returns A new Logger instance that the caller owns. + */ + static std::exception LogFatalAndCreateException(const char* category, + const CodeLocation& location, + const char* format_str, ...); + + /** + Logs the message using the provided logger id. + @param logger_id The log identifier. + @param message The log message. + */ + void Log(const std::string& logger_id, const Capture& message) const; + + /** + Sends a Profiling Event Record to the sink. + @param Profiling Event Record + */ + void SendProfileEvent(profiling::EventRecord& eventRecord) const; + ~LoggingManager(); + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(LoggingManager); + + Timestamp GetTimestamp() const noexcept; + void CreateDefaultLogger(const std::string& logger_id); + + std::unique_ptr sink_; + const Severity default_min_severity_; + const bool default_filter_user_data_; + const int default_max_vlog_level_; + bool owns_default_logger_; + static Logger* s_default_logger_; + + struct Epochs { + const std::chrono::time_point high_res; + const std::chrono::time_point system; + const std::chrono::minutes localtime_offset_from_utc; + }; + + static const Epochs& GetEpochs() noexcept; +}; + +/** + Logger provides a per-instance log id. Everything else is passed back up to the LoggingManager +*/ +class Logger { + public: + /** + Initializes a new instance of the Logger class. + @param loggingManager The logging manager. + @param id The identifier for messages coming from this Logger. + @param severity Minimum severity for messages to be created and logged. + @param filter_user_data Should USER data be filtered from output. + @param vlog_level Minimum level for VLOG messages to be created. Note that a severity of kVERBOSE must be provided + for VLOG messages to be logged. + */ + Logger(const LoggingManager& loggingManager, std::string id, + Severity severity, bool filter_user_data, int vlog_level) + : logging_manager_{&loggingManager}, + id_{id}, + min_severity_{severity}, + filter_user_data_{filter_user_data}, + max_vlog_level_{severity > Severity::kVERBOSE ? -1 : vlog_level} { // disable unless logging VLOG messages + } + + /** + Check if output is enabled for the provided LogSeverity and DataType values. + @param severity The severity. + @param data_type Type of the data. + @returns True if a message with these values will be logged. + */ + bool OutputIsEnabled(Severity severity, DataType data_type) const noexcept { + return (severity >= min_severity_ && (data_type != DataType::USER || !filter_user_data_)); + } + + /** + Return the maximum VLOG level allowed. + */ + int VLOGMaxLevel() const noexcept { + return max_vlog_level_; + } + + /** + Logs the captured message. + @param message The log message. + */ + void Log(const Capture& message) const { + logging_manager_->Log(id_, message); + } + + /** + Sends a Profiling Event Record to the sink. + @param Profiling Event Record + */ + void SendProfileEvent(profiling::EventRecord& eventRecord) const { + logging_manager_->SendProfileEvent(eventRecord); + } + + private: + const LoggingManager* logging_manager_; + const std::string id_; + const Severity min_severity_; + const bool filter_user_data_; + const int max_vlog_level_; +}; + +inline const Logger& LoggingManager::DefaultLogger() { + if (s_default_logger_ == nullptr) { + // fail early for attempted misuse. don't use logging macros as we have no logger. + throw std::logic_error("Attempt to use DefaultLogger but none has been registered."); + } + + return *s_default_logger_; +} + +inline Timestamp LoggingManager::GetTimestamp() const noexcept { + static const Epochs& epochs = GetEpochs(); + + const auto high_res_now = std::chrono::high_resolution_clock::now(); + return std::chrono::time_point_cast( + epochs.system + (high_res_now - epochs.high_res) + epochs.localtime_offset_from_utc); +} + +/** + Return the current thread id. +*/ +unsigned int GetThreadId(); + +/** + Return the current process id. +*/ +unsigned int GetProcessId(); + +} // namespace logging +} // namespace onnxruntime diff --git a/include/onnxruntime/core/common/logging/macros.h b/include/onnxruntime/core/common/logging/macros.h new file mode 100644 index 0000000000000..577a3a97d1d71 --- /dev/null +++ b/include/onnxruntime/core/common/logging/macros.h @@ -0,0 +1,209 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +// NOTE: Don't include this file directly. Include logging.h + +#define CREATE_MESSAGE(logger, severity, category, datatype) \ + ::onnxruntime::logging::Capture(logger, ::onnxruntime::logging::Severity::k##severity, category, datatype, ONNXRUNTIME_WHERE) + +/* + Both printf and stream style logging are supported. + Not that printf currently has a 2K limit to the message size. + + LOGS_* macros are for stream style + LOGF_* macros are for printf style + + The Message class captures the log input, and pushes it through the logger in its destructor. + + Use the *FATAL* macros if you want a Severity::kFatal message to also throw. + + There are a few variants to minimize the length of the macro name required in the calling code. + They are optimized so the shortest names are for the (expected) most common usage. This can be + tweaked if needed. + + Explicit logger vs LoggingManager::DefaulLogger() + Default is for a logger instance to be explicitly passed in. + The logger instance provides an identifier so that log messages from different runs can be separated. + + Variants with DEFAULT in the macro name use the default logger provided by logging manager. This is + static so accessible from any code, provided a LoggingManager instance created with InstanceType::Default + exists somewhere. See logging.h for further explanation of the expected setup. + + DataType + Default uses DataType::SYSTEM. + + Variants with USER in the macro name use DataType::USER. This is data that could be PII, and may need to + be filtered from output. LoggingManager applies this filtering. + + Category + Default category is ::onnxruntime::Logging::Category::onnxruntime. + + If you wish to provide a different category, use variants with CATEGORY in the macro name + +*/ + +// Logging with explicit category + +// iostream style logging. Capture log info in Message, and push to the logger in ~Message. +#define LOGS_CATEGORY(logger, severity, category) \ + if ((logger).OutputIsEnabled(::onnxruntime::logging::Severity::k##severity, ::onnxruntime::logging::DataType::SYSTEM)) \ + CREATE_MESSAGE(logger, severity, category, ::onnxruntime::logging::DataType::SYSTEM).Stream() + +#define LOGS_USER_CATEGORY(logger, severity, category) \ + if ((logger).OutputIsEnabled(::onnxruntime::logging::Severity::k##severity, ::onnxruntime::logging::DataType::USER)) \ + CREATE_MESSAGE(logger, severity, category, ::onnxruntime::logging::DataType::USER).Stream() + + // printf style logging. Capture log info in Message, and push to the logger in ~Message. +#define LOGF_CATEGORY(logger, severity, category, format_str, ...) \ + if ((logger).OutputIsEnabled(::onnxruntime::logging::Severity::k##severity, ::onnxruntime::logging::DataType::SYSTEM)) \ + CREATE_MESSAGE(logger, severity, category, ::onnxruntime::logging::DataType::SYSTEM).CapturePrintf(format_str, ##__VA_ARGS__) + +#define LOGF_USER_CATEGORY(logger, severity, category, format_str, ...) \ + if ((logger).OutputIsEnabled(::onnxruntime::logging::Severity::k##severity, ::onnxruntime::logging::DataType::USER)) \ + CREATE_MESSAGE(logger, severity, category, ::onnxruntime::logging::DataType::USER).CapturePrintf(format_str, ##__VA_ARGS__) + + // Logging with category of "onnxruntime" + +#define LOGS(logger, severity) \ + LOGS_CATEGORY(logger, severity, ::onnxruntime::logging::Category::onnxruntime) + +#define LOGS_USER(logger, severity) \ + LOGS_USER_CATEGORY(logger, severity, ::onnxruntime::logging::Category::onnxruntime) + + // printf style logging. Capture log info in Message, and push to the logger in ~Message. +#define LOGF(logger, severity, format_str, ...) \ + LOGF_CATEGORY(logger, severity, ::onnxruntime::logging::Category::onnxruntime, format_str, ##__VA_ARGS__) + +#define LOGF_USER(logger, severity, format_str, ...) \ + LOGF_USER_CATEGORY(logger, severity, ::onnxruntime::logging::Category::onnxruntime, format_str, ##__VA_ARGS__) + + /* + + Macros that use the default logger. + A LoggingManager instance must be currently valid for the default logger to be available. + + */ + + // Logging with explicit category + +#define LOGS_DEFAULT_CATEGORY(severity, category) \ + LOGS_CATEGORY(::onnxruntime::logging::LoggingManager::DefaultLogger(), severity, category) + +#define LOGS_USER_DEFAULT_CATEGORY(severity, category) \ + LOGS_USER_CATEGORY(::onnxruntime::logging::LoggingManager::DefaultLogger(), severity, category) + +#define LOGF_DEFAULT_CATEGORY(severity, category, format_str, ...) \ + LOGF_CATEGORY(::onnxruntime::logging::LoggingManager::DefaultLogger(), severity, category, format_str, ##__VA_ARGS__) + +#define LOGF_USER_DEFAULT_CATEGORY(severity, category, format_str, ...) \ + LOGF_USER_CATEGORY(::onnxruntime::logging::LoggingManager::DefaultLogger(), severity, category, format_str, ##__VA_ARGS__) + +// Logging with category of "onnxruntime" + +#define LOGS_DEFAULT(severity) \ + LOGS_DEFAULT_CATEGORY(severity, ::onnxruntime::logging::Category::onnxruntime) + +#define LOGS_USER_DEFAULT(severity) \ + LOGS_USER_DEFAULT_CATEGORY(severity, ::onnxruntime::logging::Category::onnxruntime) + +#define LOGF_DEFAULT(severity, format_str, ...) \ + LOGF_DEFAULT_CATEGORY(severity, ::onnxruntime::logging::Category::onnxruntime, format_str, ##__VA_ARGS__) + +#define LOGF_USER_DEFAULT(severity, format_str, ...) \ + LOGF_USER_DEFAULT_CATEGORY(severity, ::onnxruntime::logging::Category::onnxruntime, format_str, ##__VA_ARGS__) + + /* + + Conditional logging + + */ + + // Logging with explicit category + +#define LOGS_CATEGORY_IF(boolean_expression, logger, severity, category) \ + if ((boolean_expression) == true) LOGS_CATEGORY(logger, severity, category) + +#define LOGS_DEFAULT_CATEGORY_IF(boolean_expression, severity, category) \ + if ((boolean_expression) == true) LOGS_DEFAULT_CATEGORY(severity, category) + +#define LOGS_USER_CATEGORY_IF(boolean_expression, logger, severity, category) \ + if ((boolean_expression) == true) LOGS_USER_CATEGORY(logger, severity, category) + +#define LOGS_USER_DEFAULT_CATEGORY_IF(boolean_expression, severity, category) \ + if ((boolean_expression) == true) LOGS_USER_DEFAULT_CATEGORY(severity, category) + +#define LOGF_CATEGORY_IF(boolean_expression, logger, severity, category, format_str, ...) \ + if ((boolean_expression) == true) LOGF_CATEGORY(logger, severity, category, format_str, ##__VA_ARGS__) + +#define LOGF_DEFAULT_CATEGORY_IF(boolean_expression, severity, category, format_str, ...) \ + if ((boolean_expression) == true) LOGF_DEFAULT_CATEGORY(severity, category, format_str, ##__VA_ARGS__) + +#define LOGF_USER_CATEGORY_IF(boolean_expression, logger, severity, category, format_str, ...) \ + if ((boolean_expression) == true) LOGF_USER_CATEGORY(logger, severity, category, format_str, ##__VA_ARGS__) + +#define LOGF_USER_DEFAULT_CATEGORY_IF(boolean_expression, severity, category, format_str, ...) \ + if ((boolean_expression) == true) LOGF_USER_DEFAULT_CATEGORY(severity, category, format_str, ##__VA_ARGS__) + + // Logging with category of "onnxruntime" + +#define LOGS_IF(boolean_expression, logger, severity) \ + LOGS_CATEGORY_IF(boolean_expression, logger, severity, ::onnxruntime::logging::Category::onnxruntime) + +#define LOGS_DEFAULT_IF(boolean_expression, severity) \ + LOGS_DEFAULT_CATEGORY_IF(boolean_expression, severity, ::onnxruntime::logging::Category::onnxruntime) + +#define LOGS_USER_IF(boolean_expression, logger, severity) \ + LOGS_USER_CATEGORY_IF(boolean_expression, logger, severity, ::onnxruntime::logging::Category::onnxruntime) + +#define LOGS_USER_DEFAULT_IF(boolean_expression, severity) \ + LOGS_USER_DEFAULT_CATEGORY_IF(boolean_expression, severity, ::onnxruntime::logging::Category::onnxruntime) + +#define LOGF_IF(boolean_expression, logger, severity, format_str, ...) \ + LOGF_CATEGORY_IF(boolean_expression, logger, severity, ::onnxruntime::logging::Category::onnxruntime, format_str, ##__VA_ARGS__) + +#define LOGF_DEFAULT_IF(boolean_expression, severity, format_str, ...) \ + LOGF_DEFAULT_CATEGORY_IF(boolean_expression, severity, ::onnxruntime::logging::Category::onnxruntime, format_str, ##__VA_ARGS__) + +#define LOGF_USER_IF(boolean_expression, logger, severity, format_str, ...) \ + LOGF_USER_CATEGORY_IF(boolean_expression, logger, severity, ::onnxruntime::logging::Category::onnxruntime, \ + format_str, ##__VA_ARGS__) + +#define LOGF_USER_DEFAULT_IF(boolean_expression, severity, format_str, ...) \ + LOGF_USER_DEFAULT_CATEGORY_IF(boolean_expression, severity, ::onnxruntime::logging::Category::onnxruntime, \ + format_str, ##__VA_ARGS__) + +/* + + Debug verbose logging of caller provided level. + Disabled in Release builds. + Use the _USER variants for VLOG statements involving user data that may need to be filtered. +*/ +#define VLOGS(logger, level) \ + if (::onnxruntime::logging::vlog_enabled && level <= (logger).VLOGMaxLevel()) \ + LOGS_CATEGORY(logger, VERBOSE, "VLOG" #level) + +#define VLOGS_USER(logger, level) \ + if (::onnxruntime::logging::vlog_enabled && level <= (logger).VLOGMaxLevel()) \ + LOGS_USER_CATEGORY(logger, VERBOSE, "VLOG" #level) + +#define VLOGF(logger, level, format_str, ...) \ + if (::onnxruntime::logging::vlog_enabled && level <= (logger).VLOGMaxLevel()) \ + LOGF_CATEGORY(logger, VERBOSE, "VLOG" #level, format_str, ##__VA_ARGS__) + +#define VLOGF_USER(logger, level, format_str, ...) \ + if (::onnxruntime::logging::vlog_enabled && level <= (logger).VLOGMaxLevel()) \ + LOGF_USER_CATEGORY(logger, VERBOSE, "VLOG" #level, format_str, ##__VA_ARGS__) + + // Default logger variants +#define VLOGS_DEFAULT(level) \ + VLOGS(::onnxruntime::logging::LoggingManager::DefaultLogger(), level) + +#define VLOGS_USER_DEFAULT(level) \ + VLOGS_USER(::onnxruntime::logging::LoggingManager::DefaultLogger(), level) + +#define VLOGF_DEFAULT(level, format_str, ...) \ + VLOGF(::onnxruntime::logging::LoggingManager::DefaultLogger(), level, format_str, ##__VA_ARGS__) + +#define VLOGF_USER_DEFAULT(level, format_str, ...) \ + VLOGF_USER(::onnxruntime::logging::LoggingManager::DefaultLogger(), level, format_str, ##__VA_ARGS__) diff --git a/include/onnxruntime/core/common/logging/severity.h b/include/onnxruntime/core/common/logging/severity.h new file mode 100644 index 0000000000000..e43f192eb1807 --- /dev/null +++ b/include/onnxruntime/core/common/logging/severity.h @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +namespace onnxruntime { +namespace logging { +// mild violation of naming convention. the 'k' lets us use token concatenation in the macro +// ::onnxruntime::Logging::Severity::k##severity. It's not legal to have ::onnxruntime::Logging::Severity::##severity +// the uppercase makes the LOG macro usage look as expected for passing an enum value as it will be LOGS(logger, ERROR) +enum class Severity { + kVERBOSE = 0, + kINFO = 1, + kWARNING = 2, + kERROR = 3, + kFATAL = 4 +}; + +constexpr const char* SEVERITY_PREFIX = "VIWEF"; + +} // namespace logging +} // namespace onnxruntime diff --git a/include/onnxruntime/core/common/ml_status.h b/include/onnxruntime/core/common/ml_status.h new file mode 100644 index 0000000000000..9f597c815bdfe --- /dev/null +++ b/include/onnxruntime/core/common/ml_status.h @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +namespace onnxruntime { + +enum class MLStatus : uint32_t { + OK = 0, + FAIL = 1, + INVALID_ARGUMENT = 2, + NO_SUCHFILE = 3, + NO_MODEL = 4, + ENGINE_ERROR = 5, + RUNTIME_EXCEPTION = 6, + INVALID_PROTOBUF = 7, + MODEL_LOADED = 8, + NOT_IMPLEMENTED = 9, + INVALID_GRAPH = 10, + SHAPE_INFERENCE_NOT_REGISTERED = 11, + REQUIREMENT_NOT_REGISTERED = 12 +}; + +inline const char* MLStatusToString(MLStatus status) noexcept { + switch (status) { + case MLStatus::OK: + return "SUCCESS"; + case MLStatus::INVALID_ARGUMENT: + return "INVALID_ARGUMENT"; + case MLStatus::NO_SUCHFILE: + return "NO_SUCHFILE"; + case MLStatus::NO_MODEL: + return "NO_MODEL"; + case MLStatus::ENGINE_ERROR: + return "ENGINE_ERROR"; + case MLStatus::RUNTIME_EXCEPTION: + return "RUNTIME_EXCEPTION"; + case MLStatus::INVALID_PROTOBUF: + return "INVALID_PROTOBUF"; + case MLStatus::MODEL_LOADED: + return "MODEL_LOADED"; + case MLStatus::NOT_IMPLEMENTED: + return "NOT_IMPLEMENTED"; + case MLStatus::INVALID_GRAPH: + return "INVALID_GRAPH"; + case MLStatus::SHAPE_INFERENCE_NOT_REGISTERED: + return "SHAPE_INFERENCE_NOT_REGISTERED"; + case MLStatus::REQUIREMENT_NOT_REGISTERED: + return "REQUIREMENT_NOT_REGISTERED"; + default: + return "GENERAL ERROR"; + } +} + +} // namespace onnxruntime diff --git a/include/onnxruntime/core/common/status.h b/include/onnxruntime/core/common/status.h new file mode 100644 index 0000000000000..ad0d3ef6a67c7 --- /dev/null +++ b/include/onnxruntime/core/common/status.h @@ -0,0 +1,115 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +// Modifications Copyright (c) Microsoft. + +#pragma once + +#include +#include +#include "core/common/ml_status.h" + +namespace onnxruntime { +namespace common { + +enum StatusCategory { + NONE = 0, + SYSTEM = 1, + ONNXRUNTIME = 2, +}; + +/** + Error code for ONNXRuntime. +*/ +enum StatusCode { + OK = static_cast(MLStatus::OK), + FAIL = static_cast(MLStatus::FAIL), + INVALID_ARGUMENT = static_cast(MLStatus::INVALID_ARGUMENT), + NO_SUCHFILE = static_cast(MLStatus::NO_SUCHFILE), + NO_MODEL = static_cast(MLStatus::NO_MODEL), + ENGINE_ERROR = static_cast(MLStatus::ENGINE_ERROR), + RUNTIME_EXCEPTION = static_cast(MLStatus::RUNTIME_EXCEPTION), + INVALID_PROTOBUF = static_cast(MLStatus::INVALID_PROTOBUF), + MODEL_LOADED = static_cast(MLStatus::MODEL_LOADED), + NOT_IMPLEMENTED = static_cast(MLStatus::NOT_IMPLEMENTED), + INVALID_GRAPH = static_cast(MLStatus::INVALID_GRAPH), + SHAPE_INFERENCE_NOT_REGISTERED = static_cast(MLStatus::SHAPE_INFERENCE_NOT_REGISTERED), + REQUIREMENT_NOT_REGISTERED = static_cast(MLStatus::REQUIREMENT_NOT_REGISTERED), +}; + +class Status { + public: + Status() noexcept = default; + + Status(StatusCategory category, int code, const std::string& msg); + + Status(StatusCategory category, int code); + + Status(const Status& other) + : state_((other.state_ == nullptr) ? nullptr : std::make_unique(*other.state_)) {} + + Status& operator=(const Status& other) { + if (state_ != other.state_) { + if (other.state_ == nullptr) { + state_.reset(); + } else { + state_ = std::make_unique(*other.state_); + } + } + return *this; + } + + Status(Status&& other) = default; + Status& operator=(Status&& other) = default; + ~Status() = default; + + bool IsOK() const noexcept; + + int Code() const noexcept; + + StatusCategory Category() const noexcept; + + const std::string& ErrorMessage() const noexcept; + + std::string ToString() const; + + bool operator==(const Status& other) const { + return (this->state_ == other.state_) || (ToString() == other.ToString()); + } + + bool operator!=(const Status& other) const { + return !(*this == other); + } + + static const Status& OK() noexcept; + + private: + static const std::string& EmptyString() noexcept; + + struct State { + State(StatusCategory cat0, int code0, const std::string& msg0) + : category(cat0), code(code0), msg(msg0) {} + + const StatusCategory category; + const int code; + const std::string msg; + }; + + // As long as Code() is OK, state_ == nullptr. + std::unique_ptr state_; +}; + +inline std::ostream& operator<<(std::ostream& out, const Status& status) { + return out << status.ToString(); +} + +} // namespace common +} // namespace onnxruntime diff --git a/include/onnxruntime/core/common/visibility_macros.h b/include/onnxruntime/core/common/visibility_macros.h new file mode 100644 index 0000000000000..ec37bb2cc0263 --- /dev/null +++ b/include/onnxruntime/core/common/visibility_macros.h @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +//define ONNX_RUNTIME_DLL_IMPORT if your program is dynamically linked to onnxruntime +//No dllexport here. Because we are using a def file +#ifdef _WIN32 +#ifdef ONNX_RUNTIME_DLL_IMPORT +#define ONNX_RUNTIME_EXPORT __declspec(dllimport) +#else +#define ONNX_RUNTIME_EXPORT +#endif +#else +#define ONNX_RUNTIME_EXPORT +#endif + +//SAL2 staffs +#ifndef _WIN32 +#define _In_ +#define _In_opt_ +#define _Out_ +#define _Out_opt_ +#define _Inout_ +#define _Inout_opt_ +#define _Frees_ptr_opt_ +#define ONNXRUNTIME_ALL_ARGS_NONNULL __attribute__((nonnull)) +#else +#include +#define ONNXRUNTIME_ALL_ARGS_NONNULL +#endif + +#ifdef _WIN32 +#define ONNX_RUNTIME_MUST_USE_RESULT +#else +#define ONNX_RUNTIME_MUST_USE_RESULT __attribute__((warn_unused_result)) +#endif \ No newline at end of file diff --git a/include/onnxruntime/core/framework/alloc_kind.h b/include/onnxruntime/core/framework/alloc_kind.h new file mode 100644 index 0000000000000..7ccb01226ac19 --- /dev/null +++ b/include/onnxruntime/core/framework/alloc_kind.h @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include + +namespace onnxruntime { +// The ml-Values fall into the following categories with respect to their +// memory management: +// - inference inputs: owned (allocated and freed) by caller, and is by +// default read-only by the runtime. +// - inference outputs: allocated by runtime, ownership transferred to +// caller. TODO: Make sure this semantics is clear in InferenceSession API. +// - weights (constant tensors): can be allocated once (statically), and +// reused by all inference calls within an InferenceSession. +// - tensor values: The lifetimes of these tensor-values are statically +// determined, which is used for memory reuse/sharing optimizations. The +// runtime allocates/frees these values at the right time (as determined +// by the static allocation plan). Note that this is simplified since we +// do not try to optimize for "slice" like ops, where we may be able to +// conditionally reuse memory/data in some cases but not others. +// Generalizing this is future work. + +enum class AllocKind { + kAllocate = 0, + kReuse = 1, + kPreExisting = 2, + kAllocateStatically = 3, + kAllocateOutput = 4 +}; + +std::ostream& operator<<(std::ostream& out, AllocKind alloc_kind); +} // namespace onnxruntime diff --git a/include/onnxruntime/core/framework/allocator.h b/include/onnxruntime/core/framework/allocator.h new file mode 100644 index 0000000000000..00f740e12931c --- /dev/null +++ b/include/onnxruntime/core/framework/allocator.h @@ -0,0 +1,191 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "core/common/common.h" +#include "core/common/exceptions.h" +#include "core/common/status.h" +#include "core/framework/fence.h" +#include "core/framework/allocator_info.h" + +struct ONNXRuntimeAllocatorInfo { + // use string for name, so we could have customized allocator in execution provider. + const char* name; + int id; + ONNXRuntimeMemType mem_type; + ONNXRuntimeAllocatorType type; + + constexpr ONNXRuntimeAllocatorInfo(const char* name1, ONNXRuntimeAllocatorType type, int id1 = 0, ONNXRuntimeMemType mem_type1 = ONNXRuntimeMemTypeDefault) +#if (defined(__GNUC__) || defined(__clang__)) + __attribute__((nonnull)) +#endif + : name(name1), + id(id1), + mem_type(mem_type1), + type(type) { + } + + inline bool operator==(const ONNXRuntimeAllocatorInfo& other) const { + return mem_type == other.mem_type && type == other.type && id == other.id && strcmp(name, other.name) == 0; + } + + // To make ONNXRuntimeAllocatorInfo become a valid key in std map + inline bool operator<(const ONNXRuntimeAllocatorInfo& other) const { + if (type != other.type) + return type < other.type; + if (mem_type != other.mem_type) + return mem_type < other.mem_type; + if (id != other.id) + return id < other.id; + + return strcmp(name, other.name) < 0; + } + + inline std::string ToString() const { + std::ostringstream ostr; + ostr << "ONNXRuntimeAllocatorInfo: [" + << " name:" << name + << " id:" << id + << " mem_type:" << mem_type + << " type:" << type + << "]"; + return ostr.str(); + } +}; + +std::ostream& operator<<(std::ostream& out, const ONNXRuntimeAllocatorInfo& info); + +namespace onnxruntime { +constexpr const char* CPU = "Cpu"; + +// forward declaration +class SessionState; + +template +using IAllocatorUniquePtr = std::unique_ptr>; + +class IAllocator { + public: + virtual ~IAllocator() = default; + virtual void* Alloc(size_t size) = 0; + virtual void Free(void* p) = 0; + virtual const ONNXRuntimeAllocatorInfo& Info() const = 0; + + /** + optional CreateFence interface, as provider like DML has its own fence + */ + virtual FencePtr CreateFence(const SessionState* /*unused*/) { return nullptr; } + + static bool CalcMemSizeForArray(size_t nmemb, size_t size, size_t* out) noexcept { + return CalcMemSizeForArrayWithAlignment<0>(nmemb, size, out); + } + + /** + * https://cwe.mitre.org/data/definitions/190.html + * \tparam alignment must be power of 2 + * \param nmemb + * \param size + * \param out + * \return true, successful. false, overflow + */ + template + static bool CalcMemSizeForArrayWithAlignment(size_t nmemb, size_t size, size_t* out) noexcept ONNX_RUNTIME_MUST_USE_RESULT; + /** + * allocate memory for an array which has nmemb items of data, each size bytes long + */ + void* AllocArray(size_t nmemb, size_t size) { + size_t len; + if (!CalcMemSizeForArray(nmemb, size, &len)) + return nullptr; + return Alloc(len); + } + + /** + * allocate memory for an array which has nmemb items of data, each size bytes long + */ + template + void* AllocArrayWithAlignment(size_t nmemb, size_t size) { + size_t len; + if (!CalcMemSizeForArrayWithAlignment(nmemb, size, &len)) + return nullptr; + return Alloc(len); + } + + /** + Create a std::unique_ptr that is allocated and freed by the provided IAllocator. + @param allocator The allocator. + @param count_or_bytes The exact bytes to allocate if T is void, otherwise the number of elements to allocate. + @returns std::unique_ptr with allocated memory and deleter. + */ + template + static IAllocatorUniquePtr MakeUniquePtr(std::shared_ptr allocator, size_t count_or_bytes) { + if (allocator == nullptr) return nullptr; + // for now limit to fundamental types. we could support others, but to do so either we or the caller + // needs to call the dtor for the objects, for buffers allocated on device we don't have destructor + //static_assert(std::is_fundamental::value, "Fundamental type required as no destructors are called."); + + size_t alloc_size = count_or_bytes; + + // if T is not void, 'count_or_bytes' == number of items so allow for that + if (!std::is_void::value) { + // sizeof(void) isn't valid, but the compiler isn't smart enough to ignore that this line isn't + // reachable if T is void. use std::conditional to 'use' void* in the sizeof call + if (!CalcMemSizeForArray(count_or_bytes, sizeof(typename std::conditional::value, void*, T>::type), + &alloc_size)) return nullptr; + } + return IAllocatorUniquePtr{ + static_cast(allocator->Alloc(alloc_size)), // allocate + [=](T* ptr) { allocator->Free(ptr); }}; // capture IAllocator so it's always valid, and use as deleter + } +}; + +template +bool IAllocator::CalcMemSizeForArrayWithAlignment(size_t nmemb, size_t size, size_t* out) noexcept { + static constexpr size_t max_allowed = (static_cast(1) << (static_cast(std::numeric_limits::digits >> 1))) - alignment; + static constexpr size_t max_size = std::numeric_limits::max() - alignment; + static constexpr size_t alignment_mask = alignment - 1; + //Indeed, we only need to check if max_size / nmemb < size + //max_allowed is for avoiding unnecessary DIV. + if (nmemb >= max_allowed && max_size / nmemb < size) { + return false; + } else if (size >= max_allowed && + nmemb > 0 && max_size / nmemb < size) { + return false; + } + if (alignment == 0) + *out = size * nmemb; + else + *out = (size * nmemb + alignment_mask) & ~static_cast(alignment_mask); + return true; +} + +/** + The resource allocator on a physical device. + This allocator will directly allocate resource from system call +*/ +class IDeviceAllocator : public IAllocator { + public: + ~IDeviceAllocator() override = default; + void* Alloc(size_t size) override = 0; + void Free(void* p) override = 0; + const ONNXRuntimeAllocatorInfo& Info() const override = 0; + virtual bool AllowsArena() const { return true; } +}; + +class CPUAllocator : public IDeviceAllocator { + public: + void* Alloc(size_t size) override; + void Free(void* p) override; + const ONNXRuntimeAllocatorInfo& Info() const override; +}; + +using AllocatorPtr = std::shared_ptr; + +} // namespace onnxruntime diff --git a/include/onnxruntime/core/framework/allocator_info.h b/include/onnxruntime/core/framework/allocator_info.h new file mode 100644 index 0000000000000..06fdc2c31ef3c --- /dev/null +++ b/include/onnxruntime/core/framework/allocator_info.h @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once +#include "core/framework/error_code.h" +//This file is part of the public C API +#ifdef __cplusplus +extern "C" { +#endif +typedef enum ONNXRuntimeAllocatorType { + ONNXRuntimeDeviceAllocator = 0, + ONNXRuntimeArenaAllocator = 1 +} ONNXRuntimeAllocatorType; + +/** + memory types for allocator, exec provider specific types should be extended in each provider +*/ +typedef enum ONNXRuntimeMemType { + ONNXRuntimeMemTypeCPUInput = -2, // Any CPU memory used by non-CPU execution provider + ONNXRuntimeMemTypeCPUOutput = -1, // CPU accessible memory outputted by non-CPU execution provider, i.e. CUDA_PINNED + ONNXRuntimeMemTypeCPU = ONNXRuntimeMemTypeCPUOutput, // temporary CPU accessible memory allocated by non-CPU execution provider, i.e. CUDA_PINNED + ONNXRuntimeMemTypeDefault = 0, // the default allocator for execution provider +} ONNXRuntimeMemType; + +DEFINE_RUNTIME_CLASS(ONNXRuntimeAllocatorInfo); + +ONNXRUNTIME_API_STATUS(ONNXRuntimeCreateAllocatorInfo, _In_ const char* name1, enum ONNXRuntimeAllocatorType type, int id1, enum ONNXRuntimeMemType mem_type1, _Out_ ONNXRuntimeAllocatorInfo** out); + +/** + * Test if two allocation info are equal + * \return 0, equal. zero, not equal + */ +ONNXRUNTIME_API(int, ONNXRuntimeCompareAllocatorInfo, _In_ const ONNXRuntimeAllocatorInfo* info1, _In_ const ONNXRuntimeAllocatorInfo* info2) +ONNXRUNTIME_ALL_ARGS_NONNULL; +/** + * Do not free the returned value + */ +ONNXRUNTIME_API(const char*, ONNXRuntimeAllocatorInfoGetName, _In_ ONNXRuntimeAllocatorInfo* ptr); +ONNXRUNTIME_API(int, ONNXRuntimeAllocatorInfoGetId, _In_ ONNXRuntimeAllocatorInfo* ptr); +ONNXRUNTIME_API(ONNXRuntimeMemType, ONNXRuntimeAllocatorInfoGetMemType, _In_ ONNXRuntimeAllocatorInfo* ptr); +ONNXRUNTIME_API(ONNXRuntimeAllocatorType, ONNXRuntimeAllocatorInfoGetType, _In_ ONNXRuntimeAllocatorInfo* ptr); +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/include/onnxruntime/core/framework/custom_ops_author.h b/include/onnxruntime/core/framework/custom_ops_author.h new file mode 100644 index 0000000000000..f5a03a3e0ac4e --- /dev/null +++ b/include/onnxruntime/core/framework/custom_ops_author.h @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +/** + this header should include all the headers that are required to build a custom op so that + custom op developers don't have to worry about which headers to include, etc. +*/ +#include "core/framework/op_kernel.h" + +struct KernelsContainer { + std::vector<::onnxruntime::KernelCreateInfo> kernels_list; +}; + +struct SchemasContainer { + std::vector schemas_list; + std::string domain; + int baseline_opset_version; + int opset_version; +}; + +extern "C" { + KernelsContainer* GetAllKernels(); + SchemasContainer* GetAllSchemas(); + void FreeKernelsContainer(KernelsContainer*); + void FreeSchemasContainer(SchemasContainer*); +} diff --git a/include/onnxruntime/core/framework/customregistry.h b/include/onnxruntime/core/framework/customregistry.h new file mode 100644 index 0000000000000..48ee0a1316dab --- /dev/null +++ b/include/onnxruntime/core/framework/customregistry.h @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/status.h" +#include "core/common/logging/logging.h" +#include "core/graph/schema_registry.h" +#include "core/framework/op_kernel.h" +#include "core/framework/kernel_def_builder.h" +#include "core/framework/kernel_registry.h" + +namespace onnxruntime { + +/** + Represents a registry that contains both custom kernels and custom schemas. +*/ +class CustomRegistry : public KernelRegistry, public onnxruntime::OnnxRuntimeOpSchemaRegistry { + public: + CustomRegistry() = default; + ~CustomRegistry() override = default; + + /** + * Register a kernel definition together with kernel factory method to this session. + * If any conflict happened between registered kernel def and built-in kernel def, + * registered kernel will have higher priority. + * Call this before invoking Initialize(). + * @return OK if success. + */ + common::Status RegisterCustomKernel(KernelDefBuilder& kernel_def_builder, const KernelCreateFn& kernel_creator); + + common::Status RegisterCustomKernel(KernelCreateInfo&); + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(CustomRegistry); +}; + +} // namespace onnxruntime diff --git a/include/onnxruntime/core/framework/data_types.h b/include/onnxruntime/core/framework/data_types.h new file mode 100644 index 0000000000000..487fec6d8a8ed --- /dev/null +++ b/include/onnxruntime/core/framework/data_types.h @@ -0,0 +1,533 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include +#include + +#include "core/common/common.h" +#include "core/common/exceptions.h" + +namespace ONNX_NAMESPACE { +class TypeProto; +} // namespace ONNX_NAMESPACE +namespace onnxruntime { +/// Predefined registered types +//maps +using MapStringToString = std::map; +using MapStringToInt64 = std::map; +using MapStringToFloat = std::map; +using MapStringToDouble = std::map; +using MapInt64ToString = std::map; +using MapInt64ToInt64 = std::map; +using MapInt64ToFloat = std::map; +using MapInt64ToDouble = std::map; + +//vectors/sequences +using VectorString = std::vector; +using VectorInt64 = std::vector; +using VectorFloat = std::vector; +using VectorDouble = std::vector; +using VectorMapStringToFloat = std::vector; +using VectorMapInt64ToFloat = std::vector; + +class DataTypeImpl; +class TensorTypeBase; + +// MLFloat16 +union MLFloat16 { + uint16_t val; + + explicit MLFloat16(uint16_t x) : val(x) {} + MLFloat16() : val(0) {} +}; + +inline bool operator==(const MLFloat16& left, const MLFloat16& right) +{ + return left.val == right.val; +} + +inline bool operator!=(const MLFloat16& left, const MLFloat16& right) +{ + return left.val != right.val; +} + +// DataTypeImpl pointer as unique DataTypeImpl identifier. +using MLDataType = const DataTypeImpl*; +// be used with class MLValue +using DeleteFunc = void (*)(void*); +using CreateFunc = std::function; + +/** + * \brief Base class for MLDataType + * + */ +class DataTypeImpl { + public: + virtual ~DataTypeImpl() = default; + + /** + * \brief this API will be used to check type compatibility at runtime + * + * \param type_proto a TypeProto instance that is constructed for a specific type + * will be checked against a TypeProto instance contained within a corresponding + * MLDataType instance. + */ + virtual bool IsCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const = 0; + + virtual size_t Size() const = 0; + + virtual DeleteFunc GetDeleteFunc() const = 0; + + /** + * \brief Retrieves an instance of TypeProto for + * a given MLDataType + * \returns optional TypeProto. Only ONNX types + has type proto, non-ONNX types will return nullptr. + */ + virtual const ONNX_NAMESPACE::TypeProto* GetTypeProto() const = 0; + + virtual bool IsTensorType() const { + return false; + } + + // Returns this if this is of tensor-type and null otherwise + virtual const TensorTypeBase* AsTensorType() const { + return nullptr; + } + + // Return the type meta that we are using in the runtime. + template + static MLDataType GetType(); + + // Return the types for a concrete tensor type, like Tensor_Float + template + static MLDataType GetTensorType(); + + static MLDataType TypeFromProto(const ONNX_NAMESPACE::TypeProto& proto); + + // Registers ONNX_NAMESPACE::DataType (internalized string) with + // MLDataType. DataType is produced by internalizing an instance of + // TypeProto contained within MLDataType + static void RegisterDataType(MLDataType); + + static const std::vector& AllTensorTypes(); + static const std::vector& AllFixedSizeTensorTypes(); +}; + +std::ostream& operator<<(std::ostream& out, MLDataType data_type); + +/* + * Type registration helpers + */ +namespace data_types_internal { +/// TensorType helpers +/// + +// There is a specialization only for one +// type argument. +template +struct TensorContainedTypeSetter { + static void SetTensorElementType(ONNX_NAMESPACE::TypeProto&); + static void SetMapKeyType(ONNX_NAMESPACE::TypeProto&); +}; + +/// Is a given type on the list of types? +/// Accepts a list of types and the first argument is the type +/// We are checking if it is listed among those that follow +template +struct IsAnyOf; + +/// Two types remaining, end of the list +template +struct IsAnyOf : public std::is_same { +}; + +template +struct IsAnyOf { + static constexpr bool value = (std::is_same::value || + IsAnyOf::value); +}; + +/// Tells if the specified type is one of fundamental types +/// that can be contained within a tensor. +/// We do not have raw fundamental types, rather a subset +/// of fundamental types is contained within tensors. +template +struct IsTensorContainedType : public IsAnyOf { +}; + +/// This template's Get() returns a corresponding MLDataType +/// It dispatches the call to either GetTensorType<>() or +/// GetType<>() +template +struct GetMLDataType; + +template +struct GetMLDataType { + static MLDataType Get() { + return DataTypeImpl::GetTensorType(); + } +}; + +template +struct GetMLDataType { + static MLDataType Get() { + return DataTypeImpl::GetType(); + } +}; + +/// MapTypes helper API +/// K should always be one of the primitive data types +/// V can be either a primitive type (in which case it is a tensor) +/// or other preregistered types + +void CopyMutableMapValue(const ONNX_NAMESPACE::TypeProto&, + ONNX_NAMESPACE::TypeProto&); + +template +struct SetMapTypes { + static void Set(ONNX_NAMESPACE::TypeProto& proto) { + TensorContainedTypeSetter::SetMapKeyType(proto); + MLDataType dt = GetMLDataType::value>::Get(); + const auto* value_proto = dt->GetTypeProto(); + ONNXRUNTIME_ENFORCE(value_proto != nullptr, typeid(V).name(), + " expected to be a registered ONNX type"); + CopyMutableMapValue(*value_proto, proto); + } +}; + +/// Sequence helpers +/// +// Element type is a primitive type so we set it to a tensor +void CopyMutableSeqElement(const ONNX_NAMESPACE::TypeProto&, + ONNX_NAMESPACE::TypeProto&); + +template +struct SetSequenceType { + static void Set(ONNX_NAMESPACE::TypeProto& proto) { + MLDataType dt = GetMLDataType::value>::Get(); + const auto* elem_proto = dt->GetTypeProto(); + ONNXRUNTIME_ENFORCE(elem_proto != nullptr, typeid(T).name(), + " expected to be a registered ONNX type"); + CopyMutableSeqElement(*elem_proto, proto); + } +}; + +/// OpaqueTypes helpers +/// +void AssignOpaqueDomainName(const char* domain, const char* name, + ONNX_NAMESPACE::TypeProto& proto); + +} // namespace data_types_internal + +/// All tensors base +class TensorTypeBase : public DataTypeImpl { + public: + static MLDataType Type(); + + /// We first compare type_proto pointers and then + /// if they do not match try to account for the case + /// where TypeProto was created ad-hoc and not queried from MLDataType + bool IsCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const override; + + bool IsTensorType() const override { + return true; + } + + const TensorTypeBase* AsTensorType() const override { + return this; + } + + size_t Size() const override; + + DeleteFunc GetDeleteFunc() const override; + + const ONNX_NAMESPACE::TypeProto* GetTypeProto() const override; + + virtual MLDataType GetElementType() const { + // should never reach here. + ONNXRUNTIME_NOT_IMPLEMENTED(__FUNCTION__, " is not implemented"); + } + + TensorTypeBase(const TensorTypeBase&) = delete; + TensorTypeBase& operator=(const TensorTypeBase&) = delete; + + protected: + ONNX_NAMESPACE::TypeProto& mutable_type_proto(); + + TensorTypeBase(); + ~TensorTypeBase(); + + private: + struct Impl; + Impl* impl_; +}; + +/** + * \brief Tensor type. This type does not have a C++ type associated with + * it at registration time except the element type. One of the types mentioned + * above at IsTensorContainedType<> list is acceptable. + * + * \details + * Usage: + * ONNXRUNTIME_REGISTER_TENSOR(ELEMENT_TYPE) + * Currently all of the Tensors irrespective of the dimensions are mapped to Tensor + * type. IsCompatible() currently ignores shape. + */ + +template +class TensorType : public TensorTypeBase { + public: + static_assert(data_types_internal::IsTensorContainedType::value, + "Requires one of the tensor fundamental types"); + + static MLDataType Type(); + + /// Tensors only can contain basic data types + /// that have been previously registered with ONNXRuntime + MLDataType GetElementType() const override { + return DataTypeImpl::GetType(); + } + + private: + TensorType() { + using namespace data_types_internal; + TensorContainedTypeSetter::SetTensorElementType(this->mutable_type_proto()); + } +}; + +/** + * \brief Base type for all non-tensors, maps, sequences and opaques + */ +class NonTensorTypeBase : public DataTypeImpl { + public: + size_t Size() const override = 0; + + DeleteFunc GetDeleteFunc() const override = 0; + + virtual CreateFunc GetCreateFunc() const = 0; + + const ONNX_NAMESPACE::TypeProto* GetTypeProto() const override; + + NonTensorTypeBase(const NonTensorTypeBase&) = delete; + NonTensorTypeBase& operator=(const NonTensorTypeBase&) = delete; + + protected: + NonTensorTypeBase(); + ~NonTensorTypeBase(); + + ONNX_NAMESPACE::TypeProto& mutable_type_proto(); + + bool IsMapCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const; + + bool IsSequenceCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const; + + bool IsOpaqueCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const; + + private: + struct Impl; + Impl* impl_; +}; + +// This is where T is the actual CPPRuntimeType +template +class NonTensorType : public NonTensorTypeBase { + private: + static void Delete(void* p) { + delete static_cast(p); + } + + public: + size_t Size() const override { + return sizeof(T); + } + + DeleteFunc GetDeleteFunc() const override { + return &Delete; + } + + CreateFunc GetCreateFunc() const override { + return []() { return new T(); }; + } + + protected: + NonTensorType() = default; +}; + +/** + * \brief MapType. Use this type to register + * mapping types. + * + * \param T - cpp type that you wish to register as runtime MapType + * + * \details Usage: ONNXRUNTIME_REGISTER_MAP(C++Type) + * The type is required to have mapped_type and + * key_type defined + */ +template +class MapType : public NonTensorType { + public: + static_assert(data_types_internal::IsTensorContainedType::value, + "Requires one of the tensor fundamental types as key"); + + static MLDataType Type(); + + bool IsCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const override { + return this->IsMapCompatible(type_proto); + } + + private: + MapType() { + using namespace data_types_internal; + SetMapTypes::Set(this->mutable_type_proto()); + } +}; + +/** + * \brief SequenceType. Use to register sequences. + * + * \param T - CPP type that you wish to register as Sequence + * runtime type. + * + * \details Usage: ONNXRUNTIME_REGISTER_SEQ(C++Type) + * The type is required to have value_type defined + */ +template +class SequenceType : public NonTensorType { + public: + static MLDataType Type(); + + bool IsCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const override { + return this->IsSequenceCompatible(type_proto); + } + + private: + SequenceType() { + data_types_internal::SetSequenceType::Set(this->mutable_type_proto()); + } +}; + +/** + * \brief OpaqueType + * + * \param T - cpp runtume that implements the Opaque type + * + * \param const char D[] - domain must be extern to be unique + * + * \param const char N[] - name must be extern to be unique + * + * \details Only one CPP type can be associated with a particular + * OpaqueType registration + * + */ +template +class OpaqueType : public NonTensorType { + public: + static MLDataType Type(); + + bool IsCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const override { + return this->IsOpaqueCompatible(type_proto); + } + + private: + OpaqueType() { + data_types_internal::AssignOpaqueDomainName(D, N, this->mutable_type_proto()); + } +}; + +template +class NonOnnxType : public DataTypeImpl { + private: + static void Delete(void* p) { + delete static_cast(p); + } + + public: + bool IsCompatible(const ONNX_NAMESPACE::TypeProto&) const override { + return false; + } + + static MLDataType Type(); + + size_t Size() const override { + return sizeof(T); + } + + DeleteFunc GetDeleteFunc() const override { + return &Delete; + } + + const ONNX_NAMESPACE::TypeProto* GetTypeProto() const override final { + return nullptr; + } + + private: + NonOnnxType() = default; +}; + +// Explicit specialization of base class template function +// is only possible within the enclosing namespace scope, +// thus a simple way to pre-instantiate a given template +// at a registration time does not currently work and the macro +// is needed. +#define ONNXRUNTIME_REGISTER_TENSOR_TYPE(ELEM_TYPE) \ + template <> \ + MLDataType TensorType::Type() { \ + static TensorType tensor_type; \ + return &tensor_type; \ + } \ + template <> \ + MLDataType DataTypeImpl::GetTensorType() { \ + return TensorType::Type(); \ + } + +#define ONNXRUNTIME_REGISTER_MAP(TYPE) \ + template <> \ + MLDataType MapType::Type() { \ + static MapType map_type; \ + return &map_type; \ + } \ + template <> \ + MLDataType DataTypeImpl::GetType() { \ + return MapType::Type(); \ + } + +#define ONNXRUNTIME_REGISTER_SEQ(TYPE) \ + template <> \ + MLDataType SequenceType::Type() { \ + static SequenceType sequence_type; \ + return &sequence_type; \ + } \ + template <> \ + MLDataType DataTypeImpl::GetType() { \ + return SequenceType::Type(); \ + } + +#define ONNXRUNTIME_REGISTER_NON_ONNX_TYPE(TYPE) \ + template <> \ + MLDataType NonOnnxType::Type() { \ + static NonOnnxType non_onnx_type; \ + return &non_onnx_type; \ + } \ + template <> \ + MLDataType DataTypeImpl::GetType() { \ + return NonOnnxType::Type(); \ + } + +#define ONNXRUNTIME_REGISTER_OPAQUE_TYPE(CPPType, Domain, Name) \ + template <> \ + MLDataType OpaqueType::Type() { \ + static OpaqueType opaque_type; \ + return &opaque_type; \ + } \ + template <> \ + MLDataType DataTypeImpl::GetType() { \ + return OpaqueType::Type(); \ + } +} // namespace onnxruntime diff --git a/include/onnxruntime/core/framework/environment.h b/include/onnxruntime/core/framework/environment.h new file mode 100644 index 0000000000000..e0830d884ec7a --- /dev/null +++ b/include/onnxruntime/core/framework/environment.h @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include "core/common/common.h" +#include "core/common/status.h" + +namespace onnxruntime { +/** + Provides the runtime environment for onnxruntime. + Create one instance for the duration of execution. +*/ +class Environment { + public: + /** + Create and initialize the runtime environment. + */ + static Status Create(std::unique_ptr& environment); + + /** + * This function will call ::google::protobuf::ShutdownProtobufLibrary + */ + ~Environment(); + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Environment); + + Environment() = default; + Status Initialize(); +}; +} // namespace onnxruntime diff --git a/include/onnxruntime/core/framework/error_code.h b/include/onnxruntime/core/framework/error_code.h new file mode 100644 index 0000000000000..12ce1d71ee082 --- /dev/null +++ b/include/onnxruntime/core/framework/error_code.h @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include +#include + +#include "core/common/visibility_macros.h" + +#ifdef __cplusplus +//Windows user should use unicode path whenever possible, to bypass the MAX_PATH limitation +//Evevy type name started with 'P' is a pointer type, an opaque handler +//Every pointer marked with _In_ or _Out_, cannot be NULL. Caller should ensure that. +//for ReleaseXXX(...) functions, they can accept NULL pointer. +#define NO_EXCEPTION noexcept +#else +#define NO_EXCEPTION +#endif + +#ifdef __cplusplus +extern "C" { +#endif +typedef enum ONNXRuntimeErrorCode { + ONNXRUNTIME_OK = 0, + ONNXRUNTIME_FAIL = 1, + ONNXRUNTIME_INVALID_ARGUMENT = 2, + ONNXRUNTIME_NO_SUCHFILE = 3, + ONNXRUNTIME_NO_MODEL = 4, + ONNXRUNTIME_ENGINE_ERROR = 5, + ONNXRUNTIME_RUNTIME_EXCEPTION = 6, + ONNXRUNTIME_INVALID_PROTOBUF = 7, + ONNXRUNTIME_MODEL_LOADED = 8, + ONNXRUNTIME_NOT_IMPLEMENTED = 9, + ONNXRUNTIME_INVALID_GRAPH = 10, + ONNXRUNTIME_SHAPE_INFERENCE_NOT_REGISTERED = 11, + ONNXRUNTIME_REQUIREMENT_NOT_REGISTERED = 12 +} ONNXRuntimeErrorCode; + +//nullptr indicates success. Otherwise, this pointer must be freed by +typedef void* ONNXStatusPtr; + +#ifdef _WIN32 +#define ONNXRUNTIME_API_STATUSCALL _stdcall +#else +#define ONNXRUNTIME_API_STATUSCALL +#endif + +//__VA_ARGS__ on Windows and Linux are different +#define ONNXRUNTIME_API(RETURN_TYPE, NAME, ...) \ + ONNX_RUNTIME_EXPORT RETURN_TYPE ONNXRUNTIME_API_STATUSCALL NAME(__VA_ARGS__) NO_EXCEPTION + +#define ONNXRUNTIME_API_STATUS(NAME, ...) \ + ONNX_RUNTIME_EXPORT ONNXStatusPtr ONNXRUNTIME_API_STATUSCALL NAME(__VA_ARGS__) NO_EXCEPTION ONNX_RUNTIME_MUST_USE_RESULT + +//Used in *.cc files. Almost as same as ONNXRUNTIME_API_STATUS, expect without ONNX_RUNTIME_MUST_USE_RESULT +#define ONNXRUNTIME_API_STATUS_IMPL(NAME, ...) \ + ONNX_RUNTIME_EXPORT ONNXStatusPtr ONNXRUNTIME_API_STATUSCALL NAME(__VA_ARGS__) NO_EXCEPTION + +#define DEFINE_RUNTIME_CLASS2(NAME, TYPE) \ + typedef TYPE* NAME##Ptr; \ + ONNXRUNTIME_API(void, Release##NAME, _Frees_ptr_opt_ TYPE* input); + +#define DEFINE_RUNTIME_CLASS(X) \ + struct X; \ + typedef struct X X; \ + DEFINE_RUNTIME_CLASS2(X, X) + +//ONNXStatusPtr is pointer to something like this: +//struct ONNXStatus{ +// ONNXRuntimeErrorCode code; +// char msg[];//a null-terminated string, var length +//} +DEFINE_RUNTIME_CLASS2(ONNXStatus, void); + +/** + * \param msg A null-terminated string. Its content will be copied into the newly created ONNXStatus + */ +ONNXRUNTIME_API(ONNXStatusPtr, CreateONNXStatus, ONNXRuntimeErrorCode code, _In_ const char* msg) +ONNXRUNTIME_ALL_ARGS_NONNULL; + +ONNXRUNTIME_API(ONNXRuntimeErrorCode, ONNXRuntimeGetErrorCode, _In_ const ONNXStatusPtr status) +ONNXRUNTIME_ALL_ARGS_NONNULL; +/** + * \param status must not be NULL + * \return The error message inside the `status`. Don't free the returned value. + */ +ONNXRUNTIME_API(const char*, ONNXRuntimeGetErrorMessage, _In_ const ONNXStatusPtr status) +ONNXRUNTIME_ALL_ARGS_NONNULL; +#ifdef __cplusplus +} +#endif diff --git a/include/onnxruntime/core/framework/execution_provider.h b/include/onnxruntime/core/framework/execution_provider.h new file mode 100644 index 0000000000000..04e30f150bb1b --- /dev/null +++ b/include/onnxruntime/core/framework/execution_provider.h @@ -0,0 +1,127 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include "core/common/status.h" +#include "core/framework/tensor.h" + +namespace onnxruntime { +class GraphViewer; +} // namespace onnxruntime +namespace onnxruntime { + +struct ComputationCapacity; +class KernelRegistry; +class KernelRegistryManager; + +/** + Logical device representation. +*/ +typedef std::map AllocatorMap; + +class IExecutionProvider { + public: + virtual ~IExecutionProvider() = default; + + /** + Get all IAllocators for <*this> execution provider. + */ + std::vector GetAllocatorMap() const { + std::vector values; + for (auto& kv : allocators_) { + values.push_back(kv.second); + } + return values; + } + + /** + Get allocator with specified MemType + */ + virtual AllocatorPtr GetAllocator(int id, ONNXRuntimeMemType mem_type) const; + + /** + Get execution provider's capability for the specified . + Return a bunch of IndexedSubGraphs <*this> execution provider can run if + the sub-graph contains only one node or can fuse to run if the sub-graph + contains more than one node. The node indexes contained in sub-graphs may + have overlap, and it's ONNXRuntime's responsibility to do the partition + and decide whether a node will be assigned to <*this> execution provider. + */ + virtual std::vector> + GetCapability(const onnxruntime::GraphViewer& graph_viewer, + const std::vector& kernel_registries) const; + + /** + Get kernel registry per execution provider type. + The KernelRegistry share pointer returned is shared across sessions. + + NOTE: this is a tricky but final solution to achieve following goals, + 1. The execution provider type based kernel registry should be shared + across sessions. + Only one copy of this kind of kernel registry exists in ONNXRuntime + with multiple sessions/models. + 2. Adding an execution provider into ONNXRuntime does not need to touch ONNXRuntime + frameowrk/session code. + 3. onnxruntime runtime (framework/session) does not depend on any specific + execution provider lib. + */ + virtual std::shared_ptr GetKernelRegistry() const = 0; + + /** + Copy tensor between execution providers + */ + virtual common::Status CopyTensor(const Tensor& src, Tensor& dst) const = 0; + + /** + Copy tensor between execution providers on specified exec queue + */ + virtual common::Status CopyTensor(const Tensor& src, Tensor& dst, + int exec_queue_id) const; + + /** + Returns an opaque handle whose exact type varies based on the provider + and is interpreted accordingly by the corresponding kernel implementation. + For Direct3D operator kernels, this may return an IUnknown supporting + QueryInterface to ID3D12GraphicsCommandList1. + */ + virtual const void* GetExecutionHandle() const noexcept = 0; + + /** + @return type of the execution provider; should match that set in the node + through the SetExecutionProvider API. Example valid return values are: + kCpuExecutionProvider, kCudaExecutionProvider + */ + virtual std::string Type() const = 0; + + /** + Blocks until the device has completed all preceding requested tasks. + Currently this is primarily used by the IOBinding object to ensure that all + inputs have been copied to the device before execution begins. + */ + virtual common::Status Sync() const; + + /** + Called when InferenceSession::Run started + NOTE that due to async execution in provider, the actual work of previous + Run may not be finished on device This function should be regarded as the + point after which a new Run would start to submit commands from CPU + */ + virtual common::Status OnRunStart(); + + /** + Called when InferenceSession::Run ended + NOTE that due to async execution in provider, the actual work of this Run + may not be finished on device This function should be regarded as the point + that all commands of current Run has been submmited by CPU + */ + virtual common::Status OnRunEnd(); + + void InsertAllocator(AllocatorPtr allocator); + + private: + AllocatorMap allocators_; +}; +} // namespace onnxruntime diff --git a/include/onnxruntime/core/framework/fence.h b/include/onnxruntime/core/framework/fence.h new file mode 100644 index 0000000000000..2f103fcd68b76 --- /dev/null +++ b/include/onnxruntime/core/framework/fence.h @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/graph/basic_types.h" + +namespace onnxruntime { + +/* + We use a simple fence mechanism for async compute. Assumptions in this fence mechanism: + * Execution provider command queues, which execute in the same order of submit + * No fence needed for kernels within one execution provider command queue + * Fence is used to synchronize between command queues, and execution providers + + Fence usage: + 1. Fence object would be created by allocation planer for input/output when KernelDef::ExecQueueId() is not zero + 2. If fence object exists, executor would call BeforeUsingAs* prior to kernel::Compute(), and AfterUsedAs* afterwards +*/ +class IFence { + public: + virtual ~IFence() = default; + + /** + Called by executor before MLValue is used as input in a compute kernel in provider_type and exec queue_id + This should wait in the specified provider's exec queue for previous write to MLValue to finish + */ + virtual void BeforeUsingAsInput(onnxruntime::ProviderType provider_type, int queue_id) = 0; + + /** + Called by executor before MLValue is used as output in a compute kernel in provider_type and exec queue_id + This should wait in the specified provider's exec queue for previous read to MLValue to finish + */ + virtual void BeforeUsingAsOutput(onnxruntime::ProviderType provider_type, int queue_id) = 0; + + /** + Called by executor after MLValue is used as input in a compute kernel in provider_type and exec queue_id + This should update the read fence of the MLValue + */ + virtual void AfterUsedAsInput(int queue_id) = 0; + + /** + Called by executor after MLValue is used as output in a compute kernel in provider_type and exec queue_id + This should update the write fence of the MLValue + */ + virtual void AfterUsedAsOutput(int queue_id) = 0; +}; +using Fence_t = IFence*; +using FencePtr = std::shared_ptr; + +} // namespace onnxruntime diff --git a/include/onnxruntime/core/framework/framework_common.h b/include/onnxruntime/core/framework/framework_common.h new file mode 100644 index 0000000000000..c3336f2e32c1c --- /dev/null +++ b/include/onnxruntime/core/framework/framework_common.h @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include +#include "run_options.h" + +namespace onnxruntime { // forward declarations +class Model; +class GraphTransformer; +class NodeArg; +} // namespace onnxruntime + +namespace onnxruntime { +class MLValue; +using InputDefList = std::vector; +using OutputDefList = std::vector; + +using NameMLValMap = std::unordered_map; +} // namespace onnxruntime diff --git a/include/onnxruntime/core/framework/kernel_def_builder.h b/include/onnxruntime/core/framework/kernel_def_builder.h new file mode 100644 index 0000000000000..a8d34f10376f9 --- /dev/null +++ b/include/onnxruntime/core/framework/kernel_def_builder.h @@ -0,0 +1,227 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "core/common/common.h" +#include "core/graph/basic_types.h" +#include "core/framework/data_types.h" +#include "core/framework/allocator.h" + +namespace onnxruntime { +class KernelDefBuilder; + +typedef std::map MemTypeMap; + +// note that input/output might be on CPU implicitly when the node is from CPU execution provider +inline bool MemTypeOnCpuExplicitly(const MemTypeMap& mem_type_map, size_t index) { + auto iter = mem_type_map.find(index); + return iter != mem_type_map.end() && (iter->second == ONNXRuntimeMemTypeCPUInput || iter->second == ONNXRuntimeMemTypeCPUOutput); +} + +class KernelDef { + public: + const std::string& OpName() const { + return op_name_; + } + + const std::string& Domain() const { + return op_domain_; + } + + void SinceVersion(/*out*/ int* start, /*out*/ int* end) const { + *start = op_since_version_start_; + *end = op_since_version_end_; + } + + onnxruntime::ProviderType Provider() const { + return provider_type_; + } + + const std::unordered_map>& TypeConstraints() const { + return type_constraints_; + } + + const std::vector>& MayInplace() const { + return inplace_map_; + } + + const std::vector>& Alias() const { + return alias_map_; + } + + const MemTypeMap& InputMemoryType() const { + return input_memory_type_args_; + } + + const MemTypeMap& OutputMemoryType() const { + return output_memory_type_args_; + } + + // legacy interface for winml, should not be used in onnxruntime + const MemTypeMap& MemoryType() const { + return output_memory_type_args_; + } + + int ExecQueueId() const { + return exec_queue_id_; + } + + bool IsConflict(const KernelDef& other) const; + + private: + friend class KernelDefBuilder; + + // The operator name supported by <*this> kernel.. + std::string op_name_; + + // The operator since_version range supported by <*this> kernel. + // A kernel could support an operator definition between + // and (inclusive). + int op_since_version_start_ = 1; + int op_since_version_end_ = INT_MAX; + + // The operator domain supported by <*this> kernel. + // Default to 'onnxruntime::kOnnxDomain'. + // Please note the behavior of std::string("") and std::string() are different + std::string op_domain_; + + // The type of the execution provider. + std::string provider_type_; + + // The supported data types for inputs/outputs. + // Key is input/output name defined in op schema, Value are supported types. + std::unordered_map> type_constraints_; + + // An element means that output j reuses the memory of input i. + std::vector> inplace_map_; + + // An element means that output j is an alias of input i. + std::vector> alias_map_; + + // The memory types of inputs/outputs of this kernel + MemTypeMap input_memory_type_args_; + MemTypeMap output_memory_type_args_; + + // execution command queue id, 0 for default queue in execution provider + int exec_queue_id_ = 0; +}; + +class KernelDefBuilder { + public: + explicit KernelDefBuilder() + : kernel_def_(new KernelDef()) {} + + KernelDefBuilder& SetName(const std::string& op_name); + KernelDefBuilder& SetName(const char* op_name); + + KernelDefBuilder& SetDomain(const std::string& domain); + KernelDefBuilder& SetDomain(const char* domain); + + /** + This kernel supports operator definition since (to latest). + */ + KernelDefBuilder& SinceVersion(int since_version) { + kernel_def_->op_since_version_start_ = since_version; + return *this; + } + + /** + The start and end version should be set accordingly per version range for + each domain registered in OpSchemaRegistry::DomainToVersionRange in + \onnxruntime\onnxruntime\core\graph\op.h as below. + Key: domain. Value: pair. + std::unordered_map> map_; + */ + KernelDefBuilder& SinceVersion(int since_version_start, int since_version_end) { + kernel_def_->op_since_version_start_ = since_version_start; + kernel_def_->op_since_version_end_ = since_version_end; + return *this; + } + + /** + The execution provider type of the kernel. + */ + KernelDefBuilder& Provider(onnxruntime::ProviderType provider_type); + KernelDefBuilder& Provider(const char* provider_type); + + /** + Specify the set of types that this kernel supports. A further restriction + of the set of types specified in the op schema. + The arg name could be either op formal parameter name, say "X", or type + argument name specified in op schema, say "T". + */ + KernelDefBuilder& TypeConstraint(const std::string& arg_name, + const std::vector& supported_types); + KernelDefBuilder& TypeConstraint(const char* arg_name, + const std::vector& supported_types); + + /** + Like TypeConstraint but supports just a single type. + */ + KernelDefBuilder& TypeConstraint(const std::string& arg_name, MLDataType supported_type); + KernelDefBuilder& TypeConstraint(const char* arg_name, MLDataType supported_type); + + /** + Inplace mapping from inputs to outputs allowed. + It means that uplayer runtime could do memory in-place optimization + as it will not impact the correctness of this kernel. + */ + KernelDefBuilder& MayInplace(const std::vector>& inplaces); + KernelDefBuilder& MayInplace(int input_index, int output_index); + + /** + Alias mapping from inputs to outputs. Different from Inplace that the + content of the tensor is not changed. This is to take care of operators + such as Identity and Reshape. + */ + KernelDefBuilder& Alias(const std::vector>& aliases); + KernelDefBuilder& Alias(int input_index, int output_index); + + /** + Specify that this kernel requires an input arg + in certain memory type (instead of the default, device memory). + */ + template + KernelDefBuilder& InputMemoryType(int input_index) { + kernel_def_->input_memory_type_args_.insert(std::make_pair(input_index, T)); + return *this; + } + + /** + Specify that this kernel provides an output arg + in certain memory type (instead of the default, device memory). + */ + template + KernelDefBuilder& OutputMemoryType(int output_index) { + kernel_def_->output_memory_type_args_.insert(std::make_pair(output_index, T)); + return *this; + } + + /** + Specify that this kernel runs on which execution queue in the provider + */ + KernelDefBuilder& ExecQueueId(int queue_id) { + kernel_def_->exec_queue_id_ = queue_id; + return *this; + } + + /** + Return the kernel definition, passing ownership of the KernelDef to the caller + */ + std::unique_ptr Build() { + return std::move(kernel_def_); + } + + private: + // we own the KernelDef until Build() is called. + std::unique_ptr kernel_def_; +}; + +} // namespace onnxruntime diff --git a/include/onnxruntime/core/framework/kernel_registry.h b/include/onnxruntime/core/framework/kernel_registry.h new file mode 100644 index 0000000000000..ff70ef5dd38e5 --- /dev/null +++ b/include/onnxruntime/core/framework/kernel_registry.h @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/framework/op_kernel.h" + +namespace onnxruntime { +class KernelRegistry { + public: + KernelRegistry() = default; + + KernelRegistry(std::function)> kernel_reg_fn) { + kernel_reg_fn([&](KernelCreateInfo&& info) { + std::string name = info.kernel_def->OpName(); + std::string provider = info.kernel_def->Provider(); + auto st = Register(info); + if (!st.IsOK()) + ONNXRUNTIME_THROW("Register Kernel ", name, " in ", provider, " failed:", st.ErrorMessage()); + }); + } + + // Register a kernel with kernel definition and function to create the kernel. + Status Register(KernelDefBuilder& kernel_def_builder, + const KernelCreateFn& kernel_creator); + + Status Register(KernelCreateInfo& create_info); + + // Mainly for provide debug info + std::vector GetAllRegisteredOpNames() const; + + // factory functions should always return a unique_ptr for maximum flexibility + // for its clients unless the factory is managing the lifecycle of the pointer + // itself. + // TODO(Task:132) Make usage of unique_ptr/shared_ptr as out param consistent + Status CreateKernel(const onnxruntime::Node& node, + const IExecutionProvider& execution_provider, + const SessionState& session_state, + std::unique_ptr& op_kernel) const; + + // Check if an execution provider can create kernel for a node and return + // the kernel if so + const KernelCreateInfo* TryFindKernel(const onnxruntime::Node& node, + onnxruntime::ProviderType exec_provider) const; + + private: + // Check if the node's input/outpuData/attributes are compatible with this + // kernel_def, If so, the kernel defined by the kernel_def is used to + // execute this node. exec_provider is used to match kernel when node has no provider + static bool VerifyKernelDef(const onnxruntime::Node& node, + const KernelDef& kernel_def, + std::string& error_str, + onnxruntime::ProviderType exec_provider = ""); + + // Kernel create function map from op name to kernel creation info. + KernelCreateMap kernel_creator_fn_map_; +}; +} // namespace onnxruntime diff --git a/include/onnxruntime/core/framework/ml_value.h b/include/onnxruntime/core/framework/ml_value.h new file mode 100644 index 0000000000000..051dc1d297398 --- /dev/null +++ b/include/onnxruntime/core/framework/ml_value.h @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include "core/common/common.h" +#include "core/common/exceptions.h" +#include "core/framework/allocator.h" +#include "core/framework/data_types.h" +#include "core/framework/tensor.h" + +namespace onnxruntime { +/** + Represents both tensors and non-tensors. +*/ +class MLValue { + public: + MLValue() : data_(nullptr) {} + virtual ~MLValue() = default; + + MLValue(void* pData, MLDataType type, DeleteFunc deleter) { + Init(pData, type, deleter); + } + + void Init(void* pData, MLDataType type, DeleteFunc deleter) { + data_.reset(pData, deleter); + type_ = type; + } + + bool IsAllocated() const { + return data_ && type_; + } + + template + const T& Get() const { + ONNXRUNTIME_ENFORCE(DataTypeImpl::GetType() == type_, DataTypeImpl::GetType(), " != ", type_); + return *static_cast(data_.get()); + } + + template + T* GetMutable() { + ONNXRUNTIME_ENFORCE(DataTypeImpl::GetType() == type_, DataTypeImpl::GetType(), " != ", type_); + return static_cast(data_.get()); + } + + bool IsTensor() const noexcept { + return DataTypeImpl::GetType() == type_; + } + + MLDataType Type() const { + return type_; + } + + Fence_t Fence() const { + return fence_.get(); + } + + void SetFence(FencePtr fence) { + fence_ = fence; + } + + void ShareFenceWith(MLValue& v) { + fence_ = v.fence_; + } + + private: + std::shared_ptr data_; + MLDataType type_{nullptr}; + FencePtr fence_; +}; +} // namespace onnxruntime diff --git a/include/onnxruntime/core/framework/onnx_object.h b/include/onnxruntime/core/framework/onnx_object.h new file mode 100644 index 0000000000000..e4989eda8176b --- /dev/null +++ b/include/onnxruntime/core/framework/onnx_object.h @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "error_code.h" +#ifdef __cplusplus +extern "C" { +#endif +/** + * Just like the IUnknown interface in COM + * Every type inherented from ONNXObject should be deleted by ONNXRuntimeReleaseObject(...). + */ +typedef struct ONNXObject { + ///returns the new reference count. + uint32_t(ONNXRUNTIME_API_STATUSCALL* AddRef)(void* this_); + ///returns the new reference count. + uint32_t(ONNXRUNTIME_API_STATUSCALL* Release)(void* this_); + //TODO: implement QueryInterface? +} ONNXObject; + +/** + * This function is a wrapper to "(*(ONNXObject**)ptr)->AddRef(ptr)" + * WARNING: There is NO type checking in this function. + * Before calling this function, caller should make sure current ref count > 0 + * \return the new reference count + */ +ONNXRUNTIME_API(uint32_t, ONNXRuntimeAddRefToObject, _In_ void* ptr); + +/** + * + * A wrapper to "(*(ONNXObject**)ptr)->Release(ptr)" + * WARNING: There is NO type checking in this function. + * \param ptr Can be NULL. If it's NULL, this function will return zero. + * \return the new reference count. + */ +ONNXRUNTIME_API(uint32_t, ONNXRuntimeReleaseObject, _Inout_opt_ void* ptr); +#ifdef __cplusplus +} +#endif diff --git a/include/onnxruntime/core/framework/op_kernel.h b/include/onnxruntime/core/framework/op_kernel.h new file mode 100644 index 0000000000000..5654c3ad42059 --- /dev/null +++ b/include/onnxruntime/core/framework/op_kernel.h @@ -0,0 +1,313 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include "core/common/exceptions.h" +#include "core/common/logging/logging.h" +#include "core/common/status.h" +#include "core/framework/execution_provider.h" +#include "core/framework/kernel_def_builder.h" +#include "core/framework/ml_value.h" +#include "core/framework/op_kernel_info.h" +#include "core/framework/op_node_proto_helper.h" +#include "core/framework/tensor.h" +#include "core/graph/constants.h" +#include "core/graph/graph.h" +#include "gsl/span" +#include "onnx/defs/schema.h" + +namespace onnxruntime { +class ExecutionFrame; +class OpKernelContext; +class OpKernelWrapper; + +class OpKernel { + public: + using DoneCallback = std::function; + + explicit OpKernel(const OpKernelInfo& info) : op_kernel_info_(info) {} + virtual ~OpKernel() = default; + + const onnxruntime::Node& Node() const { + return op_kernel_info_.node(); + } + + const ::onnxruntime::KernelDef& KernelDef() const { + return op_kernel_info_.GetKernelDef(); + } + + virtual Status Compute(OpKernelContext* context) const = 0; + + virtual Status ComputeAsync(OpKernelContext*, + DoneCallback) const { + ONNXRUNTIME_NOT_IMPLEMENTED(__FUNCTION__, " is not implemented"); + } + + const ONNXRuntimeAllocatorInfo& Allocator(int id, ONNXRuntimeMemType mem_type) const { + return op_kernel_info_.GetAllocatorInfo(id, mem_type); + } + + const OpKernelInfo& Info() const { return op_kernel_info_; } + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(OpKernel); + OpKernelInfo op_kernel_info_; +}; + +class OpKernelContext { + public: + using ArgMap = std::unordered_map; + + explicit OpKernelContext(ExecutionFrame* frame, + const OpKernel* kernel, + const logging::Logger& logger); + + virtual ~OpKernelContext() = default; + + /** + Return the number of inputs for a variadic argument. + @param arg_num The operator argument number. + @returns Number of inputs the argument has. + */ + int NumVariadicInputs(size_t arg_num) const; + + MLDataType InputType(int index) const; + MLDataType OutputType(int index) const; + + template + const T* Input(int index) const { + const MLValue* p_ml_value = GetInputMLValue(index); + return p_ml_value ? &(p_ml_value->Get()) : nullptr; + } + + // Fetch output (non-tensor) with specified index. + template + T* Output(int index) { + if (index < 0 || index >= OutputCount()) + return nullptr; + + MLValue* p_ml_value = nullptr; + ONNXRUNTIME_ENFORCE(GetOrCreateOutputMLValue(index, p_ml_value).IsOK()); + return p_ml_value ? p_ml_value->GetMutable() : nullptr; + } + + // In the case that memory allocation has not been done for an output tensor, + // The memory allocation will be done on-the-fly with given tensor shape. + // Return nullptr if the output is an unused optional output. + Tensor* Output(int index, const TensorShape& shape); + + const logging::Logger& Logger() const { + return *logger_; + } + + int InputCount() const { + return static_cast(kernel_->Node().InputDefs().size()); + } + + int ImplicitInputCount() const { + return static_cast(kernel_->Node().ImplicitInputDefs().size()); + } + + int OutputCount() const { + return static_cast(kernel_->Node().OutputDefs().size()); + } + + /** + * return an allocator on device 0, with memtype of ONNXRuntimeMemTypeDefault + * + */ + Status GetTempSpaceAllocator(AllocatorPtr* output) const; + + /** + Return the fence of current node's input. + @param index The index of the input. + @returns Point to the Fence of the input MLValue. + It is null if the input MLValue doesn't have fence or the input is optional. + */ + Fence_t InputFence(int index) const; + + /** + Return the fence of current node's implicit input. + @param index The index of the implicit input. + @returns Point to the Fence of the implicit input MLValue. + It is null if the input MLValue doesn't have fence or the input is optional. + */ + Fence_t ImplicitInputFence(int index) const; + + /** + Return the fence of current node's output identifed by index. + @param index The index of the output. + @returns Point to the Fence of the output MLValue. + It is null if the output MLValue doesn't have fence or the output is optional. + */ + Fence_t OutputFence(int index) const; + + protected: + onnxruntime::NodeIndex GetNodeIndex() const; + const SessionState& GetSessionState() const; + + const MLValue* GetInputMLValue(int index) const; + const MLValue* GetImplicitInputMLValue(int index) const; + MLValue* GetOutputMLValue(int index); + + private: + Status GetOrCreateOutputMLValue(int index, MLValue*& value); + + int GetInputArgIndex(int index) const; + int GetImplicitInputArgIndex(int index) const; + int GetOutputArgIndex(int index) const; + + ExecutionFrame* execution_frame_{nullptr}; + const OpKernel* kernel_{nullptr}; + const logging::Logger* logger_{nullptr}; + + // The argument starting index in ExecutionFrame. + int node_input_start_index_{-1}; + int node_implicit_input_start_index_{-1}; + int node_output_start_index_{-1}; +}; + +// Fetching output tensor without shape is not allowed except when it already exists +template <> +inline Tensor* OpKernelContext::Output(int index) { + MLValue* p_ml_value = GetOutputMLValue(index); + ONNXRUNTIME_ENFORCE(p_ml_value, "Please fetch output tensor with specified shape."); + return p_ml_value->GetMutable(); +} + +using KernelCreateFn = std::function; + +struct KernelCreateInfo { + std::unique_ptr kernel_def; // Owned and stored in the global kernel registry. + KernelCreateFn kernel_create_func; + Status status; + + KernelCreateInfo(std::unique_ptr definition, + KernelCreateFn create_func) + : kernel_def(std::move(definition)), + kernel_create_func(create_func) {} + + KernelCreateInfo(KernelCreateInfo&& other) + : kernel_def(std::move(other.kernel_def)), + kernel_create_func(other.kernel_create_func) {} +}; + +using KernelCreateMap = std::multimap; + +// Forward declarations for the non-specialized BuildKernel method. +template +KernelCreateInfo BuildKernel(); + +namespace ml { +template +KernelCreateInfo BuildKernel(); +} // namespace ml + +namespace contrib { +template +KernelCreateInfo BuildKernel(); +} // namespace contrib + +// Naming convention for operator kernel classes +#define ONNX_OPERATOR_KERNEL_CLASS_NAME(provider, domain, ver, name) \ + provider##_##name##_##domain##_ver##ver + +#define ONNX_CPU_OPERATOR_KERNEL(name, ver, builder, ...) \ + ONNX_OPERATOR_KERNEL_EX(name, kOnnxDomain, ver, kCpuExecutionProvider, builder, __VA_ARGS__) + +#define ONNX_CPU_OPERATOR_ML_KERNEL(name, ver, builder, ...) \ + ONNX_OPERATOR_KERNEL_EX(name, kMLDomain, ver, kCpuExecutionProvider, builder, __VA_ARGS__) + +#define ONNX_OPERATOR_KERNEL_EX(name, domain, ver, provider, builder, ...) \ + class ONNX_OPERATOR_KERNEL_CLASS_NAME(provider, domain, ver, name); \ + template <> \ + KernelCreateInfo \ + BuildKernel() { \ + return KernelCreateInfo( \ + builder.SetName(#name) \ + .SetDomain(domain) \ + .SinceVersion(ver) \ + .Provider(provider) \ + .Build(), \ + [](const OpKernelInfo& info) -> OpKernel* { return new __VA_ARGS__(info); }); \ + } + +#define ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(provider, domain, startver, endver, name) \ + provider##_##name##_##domain##_ver##startver##_##endver + +#define ONNX_CPU_OPERATOR_VERSIONED_KERNEL(name, startver, endver, builder, ...) \ + ONNX_OPERATOR_VERSIONED_KERNEL_EX(name, kOnnxDomain, startver, endver, kCpuExecutionProvider, builder, __VA_ARGS__) + +#define ONNX_CPU_OPERATOR_VERSIONED_ML_KERNEL(name, startver, endver, builder, ...) \ + ONNX_OPERATOR_VERSIONED_KERNEL_EX(name, kMLDomain, startver, endver, kCpuExecutionProvider, builder, __VA_ARGS__) + +#define ONNX_OPERATOR_VERSIONED_KERNEL_EX(name, domain, startver, endver, provider, builder, ...) \ + class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(provider, domain, startver, endver, name); \ + template <> \ + KernelCreateInfo \ + BuildKernel() { \ + return KernelCreateInfo( \ + builder.SetName(#name) \ + .SetDomain(domain) \ + .SinceVersion(startver, endver) \ + .Provider(provider) \ + .Build(), \ + [](const OpKernelInfo& info) -> OpKernel* { return new __VA_ARGS__(info); }); \ + } + +#define ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(provider, domain, ver, type, name) \ + provider##_##name##_##domain##_ver##ver##_##type + +#define ONNX_CPU_OPERATOR_TYPED_KERNEL(name, ver, type, builder, ...) \ + ONNX_OPERATOR_TYPED_KERNEL_EX(name, kOnnxDomain, ver, type, kCpuExecutionProvider, builder, __VA_ARGS__) + +#define ONNX_CPU_OPERATOR_TYPED_ML_KERNEL(name, ver, type, builder, ...) \ + ONNX_OPERATOR_TYPED_KERNEL_EX(name, kMLDomain, ver, type, kCpuExecutionProvider, builder, __VA_ARGS__) + +#define ONNX_CPU_OPERATOR_TYPED_MS_KERNEL(name, ver, type, builder, ...) \ + ONNX_OPERATOR_TYPED_KERNEL_EX(name, kMSDomain, ver, type, kCpuExecutionProvider, builder, __VA_ARGS__) + +#define ONNX_OPERATOR_TYPED_KERNEL_EX(name, domain, ver, type, provider, builder, ...) \ + class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(provider, domain, ver, type, name); \ + template <> \ + KernelCreateInfo \ + BuildKernel() { \ + return KernelCreateInfo( \ + builder.SetName(#name) \ + .SetDomain(domain) \ + .SinceVersion(ver) \ + .Provider(provider) \ + .Build(), \ + [](const OpKernelInfo& info) -> OpKernel* { return new __VA_ARGS__(info); }); \ + } + +#define ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(provider, domain, startver, endver, type, name) \ + provider##_##name##_##domain##_ver##startver##_##endver##_##type + +#define ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL(name, startver, endver, type, builder, ...) \ + ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX(name, kOnnxDomain, startver, endver, type, kCpuExecutionProvider, builder, __VA_ARGS__) + +#define ONNX_CPU_OPERATOR_VERSIONED_TYPED_ML_KERNEL(name, startver, endver, type, builder, ...) \ + ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX(name, kMLDomain, startver, endver, type, kCpuExecutionProvider, builder, __VA_ARGS__) + +#define ONNX_CPU_OPERATOR_VERSIONED_TYPED_MS_KERNEL(name, startver, endver, type, builder, ...) \ + ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX(name, kMSDomain, startver, endver, type, kCpuExecutionProvider, builder, __VA_ARGS__) + +#define ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX(name, domain, startver, endver, type, provider, builder, ...) \ + class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(provider, domain, startver, endver, type, name); \ + template <> \ + KernelCreateInfo \ + BuildKernel() { \ + return KernelCreateInfo( \ + builder.SetName(#name) \ + .SetDomain(domain) \ + .SinceVersion(startver, endver) \ + .Provider(provider) \ + .Build(), \ + [](const OpKernelInfo& info) -> OpKernel* { return new __VA_ARGS__(info); }); \ + } + +} // namespace onnxruntime diff --git a/include/onnxruntime/core/framework/op_kernel_info.h b/include/onnxruntime/core/framework/op_kernel_info.h new file mode 100644 index 0000000000000..31e80ce69974d --- /dev/null +++ b/include/onnxruntime/core/framework/op_kernel_info.h @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/framework/execution_provider.h" +#include "core/framework/kernel_def_builder.h" +#include "core/framework/ml_value.h" +#include "core/framework/op_node_proto_helper.h" +#include "core/graph/graph.h" +#include "gsl/span" +#include "gsl/gsl_util" + +namespace onnxruntime { + +class SessionState; + +/** + A very light-weight class, which works as an aggregated + view of all data needed for constructing a Kernel instance. + NOTE: it does not own/hold any objects. +*/ +class OpKernelInfo : public OpNodeProtoHelper { + public: + explicit OpKernelInfo(const onnxruntime::Node& node, + const KernelDef& kernel_def, + const IExecutionProvider& execution_provider, + const SessionState& session_state); + + OpKernelInfo(const OpKernelInfo& other); + + const ONNXRuntimeAllocatorInfo& GetAllocatorInfo(int device_id, ONNXRuntimeMemType mem_type) const; + + const KernelDef& GetKernelDef() const; + + const IExecutionProvider* GetExecutionProvider() const noexcept; + + const onnxruntime::Node& node() const noexcept; + + bool TryGetConstantInput(int input_index, const Tensor** constant_input_value) const; + + private: + ONNXRUNTIME_DISALLOW_MOVE(OpKernelInfo); + ONNXRUNTIME_DISALLOW_ASSIGNMENT(OpKernelInfo); + + const onnxruntime::Node& node_; + const KernelDef& kernel_def_; + // For non cpu/cuda case, this pointer should be set so that function kernel + // will delegate kernel compute call to compute call. + gsl::not_null execution_provider_; + ProtoHelperNodeContext proto_helper_context_; + const SessionState& session_state_; +}; + +} // namespace onnxruntime diff --git a/include/onnxruntime/core/framework/op_node_proto_helper.h b/include/onnxruntime/core/framework/op_node_proto_helper.h new file mode 100644 index 0000000000000..ce79412df800d --- /dev/null +++ b/include/onnxruntime/core/framework/op_node_proto_helper.h @@ -0,0 +1,139 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/status.h" +#include "core/graph/graph.h" +#include "gsl/span" + +#ifdef __has_attribute +#define ONNXRUNTIME_HAVE_ATTRIBUTE(x) __has_attribute(x) +#else +#define ONNXRUNTIME_HAVE_ATTRIBUTE(x) 0 +#endif + +#if ONNXRUNTIME_HAVE_ATTRIBUTE(nodiscard) +#define MUST_USE_RESULT [[nodiscard]] +#elif defined(__clang__) && ONNXRUNTIME_HAVE_ATTRIBUTE(warn_unused_result) +#define MUST_USE_RESULT __attribute__((warn_unused_result)) +#else +#define MUST_USE_RESULT +#endif + +class IMLOpKernel; + +namespace onnxruntime { + +/** + A set of wrappers with common signatures for use with both OpKernelInfo + (as its base class) and InferenceContext. Used by ABI kernels for both + shape / type inference and kernel construction +*/ +template +class OpNodeProtoHelper { + public: + explicit OpNodeProtoHelper(const Impl_t* impl) : impl_(impl) {} + + /** + Get a single attribute + */ + template + MUST_USE_RESULT Status GetAttr(const std::string& name, T* value) const; + + /** + Get a single attribute + Call this function only when onnx doesn't have default value + */ + template + T GetAttrOrDefault(const std::string& name, const T& default_value) const { + T tmp; + return GetAttr(name, &tmp).IsOK() ? tmp : default_value; + } + + /** + Get a single attribute + Call this function only when onnx doesn't have default value + */ + template + void GetAttrOrDefault(const std::string& name, T* value, const T& default_value) const { + if (!GetAttr(name, value).IsOK()) + *value = default_value; + } + + /** + Get repeated attributes + Call this function only when onnx doesn't have default value + */ + template + MUST_USE_RESULT std::vector GetAttrsOrDefault(const std::string& name, const std::vector& default_value = std::vector{}) const { + std::vector tmp; + return GetAttrs(name, tmp).IsOK() ? tmp : default_value; + } + + /** + Get repeated attributes + */ + template + MUST_USE_RESULT Status GetAttrs(const std::string& name, std::vector& values) const; + + template + MUST_USE_RESULT Status GetAttrs(const std::string& name, gsl::span values) const; + + uint32_t GetPrimitiveAttrElementCount(ONNX_NAMESPACE::AttributeProto_AttributeType type, + const std::string& name) const noexcept; + + bool HasPrimitiveAttribute(ONNX_NAMESPACE::AttributeProto_AttributeType type, + const std::string& name) const noexcept; + + uint32_t GetInputCount() const { + return gsl::narrow_cast(impl_->getNumInputs()); + } + + uint32_t GetOutputCount() const { + return gsl::narrow_cast(impl_->getNumOutputs()); + } + + const ONNX_NAMESPACE::TypeProto* GetInputType(size_t index) const { + return impl_->getInputType(index); + } + + const ONNX_NAMESPACE::TypeProto* GetOutputType(size_t index) const { + // Work around lack of a const method from the onnx InferenceContext interface + return const_cast(impl_)->getOutputType(index); + } + + // Try to query an attribute, returning nullptr if it doesn't exist + const ONNX_NAMESPACE::AttributeProto* TryGetAttribute(const std::string& name) const { + return impl_->getAttribute(name); + } + + const ONNX_NAMESPACE::AttributeProto* GetAttribute(const std::string& name) const { + const ONNX_NAMESPACE::AttributeProto* attr = TryGetAttribute(name); + ONNXRUNTIME_ENFORCE(attr != nullptr); + return attr; + } + + private: + OpNodeProtoHelper() = delete; + const Impl_t* impl_ = nullptr; +}; + +// The methods on the following class are called by OpNodeProtoHelper, implementing +// the same signatures as InferenceContext other than const-ness. +class ProtoHelperNodeContext { + public: + ProtoHelperNodeContext(const onnxruntime::Node& node) : node_(node) {} + ProtoHelperNodeContext() = delete; + + const ONNX_NAMESPACE::AttributeProto* getAttribute(const std::string& name) const; + size_t getNumInputs() const; + const ONNX_NAMESPACE::TypeProto* getInputType(size_t index) const; + size_t getNumOutputs() const; + const ONNX_NAMESPACE::TypeProto* getOutputType(size_t index) const; + + private: + const onnxruntime::Node& node_; +}; + +} // namespace onnxruntime diff --git a/include/onnxruntime/core/framework/run_options.h b/include/onnxruntime/core/framework/run_options.h new file mode 100644 index 0000000000000..25268d56356d3 --- /dev/null +++ b/include/onnxruntime/core/framework/run_options.h @@ -0,0 +1,36 @@ + +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include "core/framework/onnx_object.h" + +/** + * Configuration information for a single Run. + */ +struct ONNXRuntimeRunOptions { + const ONNXObject* const cls; + std::atomic_int ref_count; + unsigned run_log_verbosity_level = 0; ///< applies to a particular Run() invocation + std::string run_tag; ///< to identify logs generated by a particular Run() invocation + + /// set to 'true' to terminate any currently executing Run() calls that are using this + /// ONNXRuntimeRunOptions instance. the individual calls will exit gracefully and return an error status. + bool terminate = false; + ONNXRuntimeRunOptions(); + ~ONNXRuntimeRunOptions() = default; + + // disable copy, move and assignment. we don't want accidental copies, to ensure that the instance provided to + // the Run() call never changes and the terminate mechanism will work. + ONNXRuntimeRunOptions(const ONNXRuntimeRunOptions&) = delete; + ONNXRuntimeRunOptions(ONNXRuntimeRunOptions&&) = delete; + ONNXRuntimeRunOptions& operator=(const ONNXRuntimeRunOptions&) = delete; + ONNXRuntimeRunOptions& operator=(ONNXRuntimeRunOptions&&) = delete; +}; + +namespace onnxruntime { +using RunOptions = ONNXRuntimeRunOptions; +} diff --git a/include/onnxruntime/core/framework/run_options_c_api.h b/include/onnxruntime/core/framework/run_options_c_api.h new file mode 100644 index 0000000000000..7589ad1236289 --- /dev/null +++ b/include/onnxruntime/core/framework/run_options_c_api.h @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/framework/error_code.h" +#ifdef __cplusplus +extern "C" { +#endif +struct ONNXRuntimeRunOptions; +typedef struct ONNXRuntimeRunOptions ONNXRuntimeRunOptions; +typedef ONNXRuntimeRunOptions* ONNXRuntimeRunOptionsPtr; +/** + * \return A pointer of the newly created object. The pointer should be freed by ONNXRuntimeReleaseObject after use + */ +ONNXRUNTIME_API(ONNXRuntimeRunOptions*, ONNXRuntimeCreateRunOptions); + +ONNXRUNTIME_API_STATUS(ONNXRuntimeRunOptionsSetRunLogVerbosityLevel, _In_ ONNXRuntimeRunOptions*, unsigned int); +ONNXRUNTIME_API_STATUS(ONNXRuntimeRunOptionsSetRunTag, _In_ ONNXRuntimeRunOptions*, _In_ const char* run_tag); + +ONNXRUNTIME_API(unsigned int, ONNXRuntimeRunOptionsGetRunLogVerbosityLevel, _In_ ONNXRuntimeRunOptions*); +ONNXRUNTIME_API(const char*, ONNXRuntimeRunOptionsGetRunTag, _In_ ONNXRuntimeRunOptions*); + +// set a flag so that any running ONNXRuntimeRunInference* calls that are using this instance of ONNXRuntimeRunOptions +// will exit as soon as possible if the flag is true. +ONNXRUNTIME_API(void, ONNXRuntimeRunOptionsSetTerminate, _In_ ONNXRuntimeRunOptions*, _In_ bool value); + +#ifdef __cplusplus +} +#endif diff --git a/include/onnxruntime/core/framework/tensor.h b/include/onnxruntime/core/framework/tensor.h new file mode 100644 index 0000000000000..a548b2b540c15 --- /dev/null +++ b/include/onnxruntime/core/framework/tensor.h @@ -0,0 +1,201 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include + +#include "gsl/span" + +#include "core/framework/allocator.h" +#include "core/framework/data_types.h" +#include "core/framework/tensor_shape.h" +#include "onnxruntime_config.h" + +namespace onnxruntime { + +// TODO: Do we need this class or is IAllocator::MakeUniquePtr sufficient/better +class BufferDeleter { + public: + BufferDeleter() : alloc_(nullptr) {} + BufferDeleter(AllocatorPtr alloc) + : alloc_(alloc) {} + + void operator()(void* p) const { + if (alloc_) + alloc_->Free(p); + } + + private: + // TODO: we may need consider the lifetime of alloc carefully + // The alloc_ here is the allocator that used to allocate the buffer + // And need go with the unique_ptr together. If it is using our internal + // allocator, it is ok as our allocators are global managed. But if it + // is provide by user, user need to be very careful about it. + // A weak_ptr may be a choice to reduce the impact, but that require to + // change our current allocator mgr to use shared_ptr. Will revisit it + // later. + AllocatorPtr alloc_; +}; + +typedef std::unique_ptr BufferUniquePtr; +using BufferNakedPtr = void*; +//TODO:ensure dtype_!=nullptr +#ifdef __GNUC__ +#pragma GCC diagnostic push +#ifdef HAS_NULL_DEREFERENCE +#pragma GCC diagnostic ignored "-Wnull-dereference" +#endif +#endif +/* + We want to keep tensor as simple as possible, it is just a placeholder + for a piece of memory, with additional shape information. + Memory is owned and managed by Executor / Workspace, so Tensor just uses + it, and won't do any allocation / release. +*/ +class Tensor final { + public: + /** + Create tensor with given type, shape, pre-allocate memory and allocator info. + */ + Tensor(MLDataType p_type, + const TensorShape& shape, + BufferNakedPtr p_data, + const ONNXRuntimeAllocatorInfo& alloc, + AllocatorPtr deleter = nullptr, + int64_t offset = 0); + + ~Tensor(); + + /** + Copy constructor and assign op will just pass the shape and memory + reference to another tensor. Not deep clone/copy. + */ + Tensor(const Tensor& src); + + ///requires other.buffer_deleter_ == nullptr + Tensor& ShallowCopy(const Tensor& other); + + Tensor(Tensor&& other); + + Tensor& operator=(Tensor&& other); + + /** + Returns the data type. + */ + MLDataType DataType() const { return dtype_; } + + /** + Returns the shape of the tensor. + */ + const TensorShape& Shape() const noexcept { return shape_; } + + /** + Returns the location of the tensor's memory + */ + const ONNXRuntimeAllocatorInfo& Location() const { return alloc_info_; } + + /** + May return nullptr if tensor size is zero + */ + template + T* MutableData() { + // Type check + ONNXRUNTIME_ENFORCE(DataTypeImpl::GetType() == dtype_, "Tensor type mismatch. ", + DataTypeImpl::GetType(), "!=", dtype_); + return reinterpret_cast(static_cast(p_data_) + byte_offset_); + } + + /** + May return nullptr if tensor size is zero + */ + template + gsl::span MutableDataAsSpan() { + // Type check + ONNXRUNTIME_ENFORCE(DataTypeImpl::GetType() == dtype_, "Tensor type mismatch. ", + DataTypeImpl::GetType(), "!=", dtype_); + T* data = reinterpret_cast(static_cast(p_data_) + byte_offset_); + return gsl::make_span(data, shape_.Size()); + } + + template + const T* Data() const { + // Type check + ONNXRUNTIME_ENFORCE(DataTypeImpl::GetType() == dtype_, "Tensor type mismatch. ", + DataTypeImpl::GetType(), "!=", dtype_); + return reinterpret_cast(static_cast(p_data_) + byte_offset_); + } + + template + gsl::span DataAsSpan() const { + // Type check + ONNXRUNTIME_ENFORCE(DataTypeImpl::GetType() == dtype_, "Tensor type mismatch. ", + DataTypeImpl::GetType(), "!=", dtype_); + const T* data = reinterpret_cast(static_cast(p_data_) + byte_offset_); + return gsl::make_span(data, shape_.Size()); + } + + void* MutableDataRaw(MLDataType type) { + ONNXRUNTIME_ENFORCE(type == dtype_, "Tensor type mismatch.", type, "!=", dtype_); + return p_data_; + } + + const void* DataRaw(MLDataType type) const { + ONNXRUNTIME_ENFORCE(type == dtype_, "Tensor type mismatch.", type, "!=", dtype_); + return p_data_; + } + + void* MutableDataRaw() noexcept { + return p_data_; + } + + const void* DataRaw() const noexcept { + return p_data_; + } + + /** + * Resizes the tensor without touching underlying storage. + * This requires the total size of the tensor to remains constant. + * @warning this function is NOT thread-safe. + */ + inline void Reshape(const TensorShape& new_shape) { + ONNXRUNTIME_ENFORCE(shape_.Size() == new_shape.Size(), + "Tensor size (" + std::to_string(shape_.Size()) + + ") != new size (" + std::to_string(new_shape.Size()) + ")"); + shape_ = new_shape; + } + + size_t Size() const noexcept { + return shape_.Size() * dtype_->Size(); + } + + // More API methods. + private: + void Init(MLDataType p_type, + const TensorShape& shape, + void* p_raw_data, + const ONNXRuntimeAllocatorInfo& alloc, + AllocatorPtr deleter, + int64_t offset = 0); + + void ReleaseBuffer(); + + void* p_data_; + /** + if buffer_deleter_ is null, it means tensor does not own the buffer. + otherwise tensor will use the deleter to release the buffer when + tensor is released. + */ + AllocatorPtr buffer_deleter_; + + TensorShape shape_; + MLDataType dtype_; + ONNXRuntimeAllocatorInfo alloc_info_; + int64_t byte_offset_; +}; +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif +} // namespace onnxruntime diff --git a/include/onnxruntime/core/framework/tensor_shape.h b/include/onnxruntime/core/framework/tensor_shape.h new file mode 100644 index 0000000000000..a012a30d364f8 --- /dev/null +++ b/include/onnxruntime/core/framework/tensor_shape.h @@ -0,0 +1,139 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include +#include +#include +#include +#include "onnxruntime_config.h" + +namespace ONNX_NAMESPACE { +class TensorShapeProto; +} + +namespace onnxruntime { +#ifdef __GNUC__ +#pragma GCC diagnostic push +#ifdef HAS_NULL_DEREFERENCE +#pragma GCC diagnostic ignored "-Wnull-dereference" +#endif +#endif +class TensorShape : private std::vector { + // TODO - Use a custom STL allocator to avoid heap allocations in the common case. + // We use negative numbers for unknown symbolic dimension. Each negative + // number represents a unique symbolic dimension. + // Private inheritance is used to prevent ambiguity of element versus dimension size + public: + TensorShape() = default; + + TensorShape(const int64_t* dimension_sizes, size_t dimension_count); + + TensorShape(const std::vector& dims); + + TensorShape(const std::initializer_list& dims); + + TensorShape(const TensorShape& /*other*/) = default; + + TensorShape(const std::vector& dims, size_t start, size_t end); + + /** + Return the dimension specified by . + */ + const int64_t& operator[](size_t idx) const { + return std::vector::operator[](static_cast(idx)); + } + + int64_t& operator[](size_t idx) { + return std::vector::operator[](static_cast(idx)); + } + + bool operator==(const TensorShape& other) const noexcept { + auto thisVector = static_cast*>(this); + auto otherVector = static_cast*>(&other); + return *thisVector == *otherVector; + } + + bool operator!=(const TensorShape& other) const noexcept { + return !(*this == other); + } + + size_t NumDimensions() const noexcept { + return size(); + } + + /** + Copy dims into an array with given size + */ + void CopyDims(int64_t* dims, size_t num_dims) const { + memcpy(dims, data(), sizeof(value_type) * std::min(num_dims, NumDimensions())); + } + + /** + Return underlying vector representation. + */ + const std::vector& GetDims() const { return *this; } + + /** + * Return the total number of elements. Returns 1 for an empty (rank 0) TensorShape. + * + * May return -1 + */ + int64_t Size() const; + + /** + Return the total number of elements up to the specified dimension. + @param dimension Return size up to this dimension. Value must be >= 0 and < this.Size(). + */ + int64_t SizeToDimension(size_t dimension) const; + + /** + Return the total number of elements from the specified dimension to the end of the tensor shape. + @param dimension Return size up to this dimension. 0 <= dimension < this.Size(). + */ + int64_t SizeFromDimension(size_t dimension) const; + + /** + Return a new TensorShape of the dimensions from dimstart to dimend. + */ + TensorShape Slice(size_t dimstart, size_t dimend) const; + + /** + Return a new TensorShape of the dimensions from dimstart to end. + */ + TensorShape Slice(size_t dimstart) const; + + /** + output dimensions nicely formatted + */ + std::string ToString() const; + + /** + Calculate size between start and end. + Assumes start and end are between 0 and dimensions.size(), inclusive, and that + start < end. + */ + int64_t SizeHelper(size_t start, size_t end) const; + + /** + empty shape or 1D shape (1) is regarded as scalar tensor + */ + bool IsScalar() const { + return size() == 0 || (size() == 1 && at(0) == 1); + } + + static const TensorShape& ReinterpretBaseType(const std::vector& dimensions) { + static_assert(sizeof(TensorShape) == sizeof(std::vector), "Size of TensorShape prevents safe casting from vector"); + return *static_cast(&dimensions); + } +}; +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif +// operator<< to nicely output to a stream +std::ostream& operator<<(std::ostream& out, const ::onnxruntime::TensorShape& shape); + +std::ostream& operator<<(std::ostream& out, const ONNX_NAMESPACE::TensorShapeProto& shape_proto); + +} // namespace onnxruntime diff --git a/include/onnxruntime/core/graph/basic_types.h b/include/onnxruntime/core/graph/basic_types.h new file mode 100644 index 0000000000000..24702b4df7c4d --- /dev/null +++ b/include/onnxruntime/core/graph/basic_types.h @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include +#include +#include +#include + +namespace ONNX_NAMESPACE { +class ValueInfoProto; +class TensorProto; +class TypeProto; +class AttributeProto; +} // namespace ONNX_NAMESPACE + +namespace onnxruntime { +using NodeIndex = size_t; +using Version = int64_t; +using NodeArgInfo = ONNX_NAMESPACE::ValueInfoProto; +using InitializedTensorSet = std::unordered_map; +using ArgNameToTypeMap = std::unordered_map; +using ProviderType = const std::string&; +// TODO - Evaluate switching the types below to support transparent comparators and enable +// lookups based on gsl::cstring_span<> and std::string_view. This would reduces allocations +// converting to std::string, but requires conversion to std::map> +// instead of std::unordered_map]>. + +using NodeAttributes = std::unordered_map; +class IOnnxRuntimeOpSchemaCollection; +using IOnnxRuntimeOpSchemaCollectionPtr = std::shared_ptr; +} // namespace onnxruntime + +namespace onnxruntime { +class OpKernel; +class OpKernelInfo; + +using KernelCreateFn = std::function; +} // namespace onnxruntime diff --git a/include/onnxruntime/core/graph/constants.h b/include/onnxruntime/core/graph/constants.h new file mode 100644 index 0000000000000..b20848cd41a9b --- /dev/null +++ b/include/onnxruntime/core/graph/constants.h @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include + +#include "core/common/common.h" + +namespace onnxruntime { +constexpr const char* kNoOp = "NoOp"; +constexpr const char* kConstant = "Constant"; +constexpr const char* kFunctionOp = "_kFunctionOp"; +constexpr const char* kConstantValue = "value"; +constexpr const char* kOnnxDomain = ""; +constexpr const char* kOnnxDomainAlias = "ai.onnx"; +constexpr const char* kMLDomain = "ai.onnx.ml"; +constexpr const char* kMSDomain = "com.microsoft"; +constexpr const char* kCpuExecutionProvider = "CPUExecutionProvider"; +constexpr const char* kCudaExecutionProvider = "CUDAExecutionProvider"; +constexpr const char* kMklDnnExecutionProvider = "MKLDNNExecutionProvider"; +constexpr const char* kNupharExecutionProvider = "NupharExecutionProvider"; +constexpr const char* kBrainSliceExecutionProvider = "BrainSliceExecutionProvider"; +} // namespace onnxruntime + diff --git a/include/onnxruntime/core/graph/function.h b/include/onnxruntime/core/graph/function.h new file mode 100644 index 0000000000000..b3161c2d3387e --- /dev/null +++ b/include/onnxruntime/core/graph/function.h @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/graph/indexed_sub_graph.h" + +namespace onnxruntime { +class Graph; +class Node; +} // namespace onnxruntime + +namespace onnxruntime { + +// Function representation class. +class Function { + public: + virtual ~Function() {} + virtual const ONNX_NAMESPACE::OpSchema& OpSchema() const = 0; + + virtual const onnxruntime::Graph& Body() const = 0; + + virtual const IndexedSubGraph& GetIndexedSubGraph() const = 0; +}; + +std::unique_ptr MakeFunction(const onnxruntime::Graph& graph, + std::unique_ptr customized_func); +} // namespace onnxruntime diff --git a/include/onnxruntime/core/graph/graph.h b/include/onnxruntime/core/graph/graph.h new file mode 100644 index 0000000000000..ed732f70df805 --- /dev/null +++ b/include/onnxruntime/core/graph/graph.h @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/graph/graph_base.h" + +namespace onnxruntime { +class Function; +struct IndexedSubGraph; +} // namespace onnxruntime + +namespace onnxruntime { +// A graph viewer representation class. +class GraphViewer { + public: + GraphViewer(const Graph& graph); + + // Graph name. + const std::string& Name() const noexcept; + + const std::string& Description() const noexcept; + + bool GetInitializedTensor(const std::string& tensor_name, const ONNX_NAMESPACE::TensorProto*& value) const; + + // Graph inputs excluding initializers. + const std::vector& GetInputs() const noexcept; + // Graph inputs including initializers. Contains no nullptr values. + // This will match the number and order of inputs from the GraphProto. + const std::vector& GetInputsIncludingInitializers() const noexcept; + + // Graph outputs. Should have no nullptr values. + const std::vector& GetOutputs() const noexcept; + + // Get graph value infos. + const std::vector& GetValueInfo() const noexcept; + + // Get const Node given specific node index. May return nullptr if node as been freed. + const Node* GetNode(NodeIndex node_index) const; + + const GraphNodes& Nodes() const noexcept; + + int NumberOfNodes() const noexcept; + + int MaxNodeIndex() const noexcept; + + const std::vector& GetNodesInTopologicalOrder() const; + + const std::vector& GetRootNodes() const; + + const InitializedTensorSet& GetAllInitializedTensors() const noexcept; + + const NodeArg* GetNodeArg(const std::string& name) const; + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(GraphViewer); + + const Graph* graph_; + + // The topological order of node index. + std::vector nodes_in_topological_order_; + // Graph root nodes. + std::vector root_nodes_; +}; +} // namespace onnxruntime diff --git a/include/onnxruntime/core/graph/graph_base.h b/include/onnxruntime/core/graph/graph_base.h new file mode 100644 index 0000000000000..00d9985f5d998 --- /dev/null +++ b/include/onnxruntime/core/graph/graph_base.h @@ -0,0 +1,809 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "core/common/common.h" +#include "core/common/const_pointer_container.h" +#include "core/common/status.h" +#include "core/graph/basic_types.h" +#include "core/graph/constants.h" +#include "core/graph/graph_nodes.h" +#include "core/graph/node_arg.h" +#include "core/graph/onnx_protobuf.h" +#include "core/graph/function.h" +#include "gsl/gsl_util" +#include "gsl/pointers" + +namespace onnxruntime { +class Graph; +struct IndexedSubGraph; +class Node; +class OpSignature; + +// A node representation class. +class Node { + public: + // Node types. + enum class Type { + // A node refers to a primitive operator. + Primitive = 0, + // A node refers to a function. + Fused = 1, + }; + + ~Node() = default; + + // An edge end. It could be input or output edge end of a node. + // For node's input edge end, it's the source end, as the destination + // end is the node itself. + // For node's output edge end, it's the destination end, as the source + // end is the node itself. + class EdgeEnd { + public: + // Constructor. + // An EdgeEnd contains a Node and NodeArg. + EdgeEnd(const Node& node, const NodeArg& node_arg) noexcept; + // A control edge, which does not have NodeArg. + EdgeEnd(const Node& node) noexcept; + + // Get the that this edge end refers to. + const Node& GetNode() const noexcept; + + // Get the that this edge end refers to. + const NodeArg* GetNodeArg() const noexcept; + + private: + const Node* node_; + const NodeArg* node_arg_; + }; + + // Get node index. + NodeIndex Index() const noexcept; + + // Get node name. + const std::string& Name() const noexcept; + + // Get node operator type. + const std::string& OpType() const noexcept; + + // Get the domain of the OperatorSet that specifies the operator named by . + const std::string& Domain() const noexcept; + + // Get the OperatorSchema this node refers to. ValidateOpType() must have been called previously. + // May be null in the future. + const ONNX_NAMESPACE::OpSchema* Op() const noexcept; + Node::Type NodeType() const noexcept; + // Get function body if the node type is fused. + // The function body is owned by <*this> node's parent graph. + const Function* GetFunctionBody() const noexcept; + + // Get node description. + const std::string& Description() const noexcept; + + // Iterate through Input/OutputDefs() with index, note the loop early terminates with error. + static common::Status ForEachWithIndex( + const ConstPointerContainer>& nodeArgVec, + std::function func) { + for (size_t index = 0; index < nodeArgVec.size(); ++index) { + auto arg = nodeArgVec[index]; + if (!arg->Exists()) + continue; + ONNXRUNTIME_RETURN_IF_ERROR(func(*arg, index)); + } + return common::Status::OK(); + } + + // read only access. requires special wrapper to apply const to the NodeArg + const ConstPointerContainer> InputDefs() const noexcept { + return ConstPointerContainer>(definitions_.input_defs); + } + + const std::vector& InputArgCount() const noexcept { return definitions_.input_arg_count; } + + std::vector& MutableInputArgsCount() { return MutableDefinitions().input_arg_count; } + + // If this Node contains a subgraph, the NodeArg's that are implicitly consumed by Nodes within that subgraph. + const std::vector& ImplicitInputDefs() const noexcept { + return definitions_.implicit_input_defs; + } + + // read only access. requires special wrapper to apply const to the NodeArg + const ConstPointerContainer> OutputDefs() const noexcept { + return ConstPointerContainer>(definitions_.output_defs); + } + + std::vector& MutableInputDefs() noexcept { + return MutableDefinitions().input_defs; + } + + struct EdgeEndCompare { + bool operator()(const EdgeEnd& lhs, const EdgeEnd& rhs) const { + if (lhs.GetNode().Index() == rhs.GetNode().Index()) { + auto lhs_arg = lhs.GetNodeArg(); + auto rhs_arg = rhs.GetNodeArg(); + std::string lhs_arg_name = lhs_arg == nullptr ? "" : lhs_arg->Name(); + std::string rhs_arg_name = rhs_arg == nullptr ? "" : rhs_arg->Name(); + return lhs_arg_name.compare(rhs_arg_name) < 0; + } + return lhs.GetNode().Index() < rhs.GetNode().Index(); + } + }; + + using EdgeSet = std::set; + using EdgeConstIterator = EdgeSet::const_iterator; + class NodeConstIterator { + public: + NodeConstIterator(EdgeConstIterator p_iter); + + bool operator==(const NodeConstIterator& p_other) const; + + bool operator!=(const NodeConstIterator& p_other) const; + + void operator++(); + void operator--(); + + const Node* operator*(); + + private: + EdgeConstIterator m_iter; + }; + + // Functions defined to traverse a Graph as below. + // Read all input nodes of <*this>. + // Beginning of input nodes. Iterator should have no nullptr values. + NodeConstIterator InputNodesBegin() const noexcept { return NodeConstIterator(relationships_.input_edges.cbegin()); }; + // End of input nodes. + NodeConstIterator InputNodesEnd() const noexcept { return NodeConstIterator(relationships_.input_edges.cend()); } + + // Beginning of output nodes. Iterator should have no nullptr values. + NodeConstIterator OutputNodesBegin() const noexcept { return NodeConstIterator(relationships_.output_edges.cbegin()); } + // End of output nodes. + NodeConstIterator OutputNodesEnd() const noexcept { return NodeConstIterator(relationships_.output_edges.cend()); } + + // Beginning of input edge. Iterator should have no nullptr values. + EdgeConstIterator InputEdgesBegin() const noexcept { return relationships_.input_edges.cbegin(); } + + // End of input nodes. + EdgeConstIterator InputEdgesEnd() const noexcept { return relationships_.input_edges.cend(); } + + // Beginning of output edge. Iterator should have no nullptr values. + EdgeConstIterator OutputEdgesBegin() const noexcept { return relationships_.output_edges.cbegin(); } + + // End of output nodes. + EdgeConstIterator OutputEdgesEnd() const noexcept { return relationships_.output_edges.cend(); } + + const std::set& ControlInputs() const noexcept { return relationships_.control_inputs; } + + size_t GetInputEdgesCount() const noexcept { return relationships_.input_edges.size(); } + + size_t GetOutputEdgesCount() const noexcept { return relationships_.output_edges.size(); } + + // Add a node attribute with specified attribute name and value. + void AddAttribute(const std::string& attr_name, const ONNX_NAMESPACE::AttributeProto& value); + +#define ADD_ATTR_INTERFACES(TypeName) \ + void AddAttribute(const std::string& attr_name, const TypeName& value); \ + void AddAttribute(const std::string& attr_name, \ + const std::vector& values); + + ADD_ATTR_INTERFACES(int64_t) + ADD_ATTR_INTERFACES(float) + ADD_ATTR_INTERFACES(std::string) + ADD_ATTR_INTERFACES(ONNX_NAMESPACE::TensorProto) + ADD_ATTR_INTERFACES(ONNX_NAMESPACE::GraphProto) + + // Clear specified node attribute. + bool ClearAttribute(const std::string& attr_name); + + // Get node attributes. + const NodeAttributes& GetAttributes() const noexcept; + + // Indicates on which we will run this node in runtime. + // Executor will decide which device that this node will run against + // and set it properly. + // TODO: may change the return value type to be an ENUM. + ProviderType GetExecutionProviderType() const noexcept; + void SetExecutionProviderType(ProviderType execution_provider_type); + + // Get the corresponding . + void ToProto(ONNX_NAMESPACE::NodeProto& proto) const; + + // iterate through all input/output defs + void ForEachDef(std::function func) const; + + // iterate through all input defs + void ForEachInputDef(std::function func) const; + + // iterate through all output defs + void ForEachOutputDef(std::function func) const; + + // Replaces defs + void ReplaceDefs(const std::map& replacements); + + // Node definitions. Really a struct but we want to prevent accidental copies. + class Definitions { + public: + Definitions() noexcept = default; + + // Node inputs' definition. + std::vector input_defs; + + // The number of inputs for each argument of the operator or function which + // this node refers. + // For example, has 10 elements (inputs), and + // is {4, 6}. This means that 4 elements (inputs) of + // map to the first argument of the operator or function, and + // the other 6 map to the second argument. + std::vector input_arg_count; + + // Node outputs' definition. + std::vector output_defs; + + // For a Node that contains a subgraph, NodeArg instances that are consumed by Nodes in a subgraph. + // e.g. the subgraph in an 'If' node gets all its input values via this mechanism + // rather than explicit inputs. + // They are pseudo-inputs to this Node as it has an implicit dependency on them. + std::vector implicit_input_defs; + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Definitions); + }; + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 26439) +#endif + class Relationships { + public: + Relationships() = default; + + void Clear() noexcept { + input_edges.clear(); + output_edges.clear(); + control_inputs.clear(); + } + + // Node input edges. + EdgeSet input_edges; + // Node output edges. + EdgeSet output_edges; + + // Control input nodes' names. + std::set control_inputs; + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Relationships); + }; + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Node); + + // NOTE: These friendship relationships should ONLY be used for calling the + // following methods so that the Node can maintain its internal invariants as + // well as possible. Node::Node Node::Init Node::MutableDefinitions + // Node::MutableRelationships + // Node::ValdiateVersion + // All other calls should be made through the public Node interface. + // Friend classes should NOT be directly accessing any member variables. + friend class Graph; + + Node(NodeIndex index, Graph& graph) : index_(index), graph_(&graph) {} + + void Init(const std::string& name, + const std::string& op_type, + const std::string& description, + const std::vector& input_args, + const std::vector& output_args, + const NodeAttributes* attributes, + const std::string& domain); + + // internal only method to allow selected classes to directly alter + // the input/output definitions and arg counts + Definitions& MutableDefinitions() noexcept; + + // internal only method to allow selected classes to directly alter + // the links between nodes. + Relationships& MutableRelationships() noexcept; + + const Definitions& GetDefinitions() const noexcept { return definitions_; } + const Relationships& GetRelationships() const noexcept { return relationships_; } + + void SetNodeType(Node::Type node_type) noexcept; + void SetFunctionBody(const Function& func); + + // validate and update the input arg count + common::Status UpdateInputArgCount(); + + // Node index. Default to impossible value rather than 0. + NodeIndex index_ = std::numeric_limits::max(); + + // Node name. + std::string name_; + + // Node operator type. + std::string op_type_; + + // OperatorSet domain of node refers to. + const ONNX_NAMESPACE::OpSchema* op_ = nullptr; + Node::Type node_type_ = Node::Type::Primitive; + const Function* func_body_ = nullptr; + + // Node doc string. + std::string description_; + + // input/output defs and arg count + Definitions definitions_; + + // Relationships between this node and others in the graph + Relationships relationships_; + + // Device. + std::string execution_provider_type_; + + // Map from attribute name to attribute. + // This allows attribute adding and removing. + NodeAttributes attributes_; + + Graph* graph_; +}; + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +class Graph { + public: + // Resolve <*this> graph to ensure it's in a good shape with full + // functionality. + // 1. Run through all validation rules. + // a. Node name and node output's names should be unique. + // b. Attribute match between node and op definition. + // c. Input/Output match between node and op definition. + // d. Graph is acyclic and sort nodes in topological order. + // 2. Check & Setup inner nodes' dependency. + // 3. Cleanup function definition lists. + // Returns resolving status. + common::Status Resolve(); + + // Getter and Setter for graph name. + const std::string& Name() const noexcept; + void SetName(const std::string& name); + + const std::string& Description() const noexcept; + void SetDescription(const std::string& description); + + // Add/Remove/Get initial tensors for some graph inputs. + void AddInitializedTensor(const ONNX_NAMESPACE::TensorProto& tensor_proto); + void RemoveInitializedTensor(const std::string& tensor_name); + bool GetInitializedTensor(const std::string& tensor_name, const ONNX_NAMESPACE::TensorProto*& value) const; + const InitializedTensorSet& GetAllInitializedTensors() const noexcept; + void CleanAllInitializedTensors() noexcept; + + // Graph inputs excluding initializers. Contains no nullptr values. + const std::vector& GetInputs() const noexcept { return graph_inputs_excluding_initializers_; } + + // Graph inputs including initializers. Contains no nullptr values. + // This will match the number and order of inputs from the GraphProto. + const std::vector& GetInputsIncludingInitializers() const noexcept { + return graph_inputs_including_initializers_; + } + + // Graph outputs. Should have no nullptr values. + const std::vector& GetOutputs() const noexcept { return graph_outputs_; } + + bool IsNodeOutputsInGraphOutputs(const Node& node) { + for (auto output_def : node.OutputDefs()) { + if (std::find(GetOutputs().cbegin(), GetOutputs().cend(), output_def) != GetOutputs().cend()) { + return true; + } + } + return false; + } + // Get graph value infos. + const std::vector& GetValueInfo() const noexcept; + + // Get const Node given specific node index. May return nullptr if node as been freed. + const Node* GetNode(NodeIndex node_index) const { return NodeAtIndexImpl(node_index); } + + // Mutable node at index. May return nullptr if node has been freed. + Node* GetNode(NodeIndex node_index) { return NodeAtIndexImpl(node_index); } + + GraphNodes& Nodes() noexcept { return iterable_nodes_; } + + const GraphNodes& Nodes() const noexcept { return iterable_nodes_; } + + // Max NodeIndex in the Graph + int MaxNodeIndex() const noexcept { return gsl::narrow_cast(nodes_.size()); } + + // Number of nodes in the . + // This is smaller than MaxNodeIndex(), since there may be nodes + // removed during optimization. + int NumberOfNodes() const noexcept { return num_of_nodes_; } + + NodeArg* GetNodeArg(const std::string& name) { + auto iter = node_args_.find(name); + if (iter != node_args_.end()) { + return iter->second.get(); + } + return nullptr; + } + + const NodeArg* GetNodeArg(const std::string& name) const { + auto iter = node_args_.find(name); + if (iter != node_args_.end()) { + return iter->second.get(); + } + return nullptr; + } + + // Get NodeArg by name, or create NodeArg owned by the graph if not found + NodeArg& GetOrCreateNodeArg(const std::string& name, const ONNX_NAMESPACE::TypeProto* p_arg_type) { + auto iter = node_args_.find(name); + if (iter != node_args_.end()) { + return *(iter->second); + } + + auto result = node_args_.insert(std::make_pair(name, std::make_unique(name, p_arg_type))); + return *(result.first->second); + } + + // create a unique name for NodeArg + std::string GenerateNodeArgName(const std::string& base_name); + + // create a unique name for Node + std::string GenerateNodeName(const std::string& base_name); + + // Add node to <*this> graph. + Node* AddNode(const std::string& name, + const std::string& op_type, + const std::string& description, + const std::vector& input_args, + const std::vector& output_args, + const NodeAttributes* attributes = nullptr, + const std::string& domain = ""); + + // Copy node and add to graph. + // @param other Node to copy + // @param returns Pointer to node that was created and inserted. + Node* AddNode(const Node& other); + + // Remove node and free it. + bool RemoveNode(NodeIndex node_index); + + // Add|Remove an edge. + void AddEdge(NodeIndex src_node_index, NodeIndex dst_node_index, const NodeArg& node_arg); + void RemoveEdge(NodeIndex src_node_index, NodeIndex dst_node_index, const NodeArg& node_arg); + + // Add control edge into <*this> graph. + // The node does not consume any data output by + // , but it's designed to be executed behind. + bool AddControlEdge(NodeIndex src_node_index, NodeIndex dst_node_index); + + // Mark Graph as needing Resolve() to be called + Graph& SetGraphResolveNeeded() noexcept { + graph_resolve_needed_ = true; + return *this; + } + + bool GraphResolveNeeded() const noexcept { + return graph_resolve_needed_; + } + + Graph& SetGraphProtoSyncNeeded() noexcept { + graph_proto_sync_needed_ = true; + return *this; + } + + bool GraphProtoSyncNeeded() const noexcept { + return graph_proto_sync_needed_; + } + + // Performs reverse DFS traversal from a set of nodes in 'from' up to + // the SOURCE node. 'enter' is a visit function that will be invoked + // on a node when it is visited but its parents haven't been. 'leave' + // is the visit function invoked on the node after its parents have + // all been visited. 'comp' is used to stable the traversal order. + void ReverseDFSFrom(const std::vector& from, + const std::function& enter, + const std::function& leave, + const std::function& comp = {}) const; + + void ReverseDFSFrom(const std::vector& from, + const std::function& enter, + const std::function& leave, + const std::function& comp = {}) const; + + const std::unordered_map& DomainToVersionMap() const noexcept { + return domain_to_version_; + } + + // Serialize the into . + const ONNX_NAMESPACE::GraphProto& ToGraphProto(); + + IOnnxRuntimeOpSchemaCollectionPtr GetSchemaRegistry() const; + + Node* FuseSubGraph(std::unique_ptr<::onnxruntime::IndexedSubGraph> sub_graph, const std::string& fused_node_name); + + Status InlineFunction(Node& node); + + // Get the Graph instance for a node that contains a GraphProto attribute in attribute_name. + // Non-const as the Graph instance returned for the subgraph is mutable and owned by this Graph instance. + Graph* GetMutableSubgraph(const NodeIndex index, const std::string& attribute_name); + + // Const version for the above + const Graph* GetSubgraph(const NodeIndex index, const std::string& attribute_name) const; + + // when creating a subgraph, record that a NodeArg will come from the outer scope. + // This prevents it from being added to the graph inputs. + void AddOuterScopeNodeArg(const std::string& name) { + ONNXRUNTIME_IGNORE_RETURN_VALUE(outer_scope_node_arg_names_.insert(name)); + } + + // when constructing a Graph, explicitly set the input order to be used. + // If the Graph is loaded from a GraphProto this has no effect. + void SetInputOrder(const std::vector inputs) { + graph_input_order_ = inputs; + } + + // when constructing a Graph, explicitly set the input order to be used. + // If the Graph is loaded from a GraphProto this has no effect. + void SetOutputOrder(const std::vector outputs) { + graph_output_order_ = outputs; + } + + virtual ~Graph(); + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Graph); + + // This friendship relationship should only be used to call Graph::Graph and + // Graph::LoadGraph All other access should be via the public API. + friend class Model; + + Graph() = delete; + + // Constructor: Given a loaded from model file, construct + // a object. + Graph(ONNX_NAMESPACE::GraphProto* graph_proto, + const std::unordered_map& domain_to_version, + Version ir_version, + IOnnxRuntimeOpSchemaCollectionPtr schema_registry, + const std::unordered_map& model_functions = {}); + + // Construct a Graph instance for a subgraph. Inherits some properties from the parent graph. + Graph(Graph& parent_graph, ONNX_NAMESPACE::GraphProto& subgraph_proto); + + // internal use only + Graph(ONNX_NAMESPACE::GraphProto* graph_proto, + const std::unordered_map& domain_to_version, + Version ir_version, + IOnnxRuntimeOpSchemaCollectionPtr schema_registry, + Graph* parent_graph, + const std::unordered_map& model_functions = {}); + + // Add node with specified . + Node* AddNode(const ONNX_NAMESPACE::NodeProto& node_proto, + const ArgNameToTypeMap& name_to_type); + + Version IrVersion() const noexcept { + return ir_version_; + } + + Graph& GraphResolveNeeded(bool needed) noexcept { + graph_resolve_needed_ = needed; + return *this; + } + + Graph& GraphProtoSyncNeeded(bool needed) noexcept { + graph_proto_sync_needed_ = needed; + return *this; + } + + // During the Resolve of a Graph it is necessary to recursively descend into subgraphs if present. + // The ResolveContext holds the collection of values for the current Graph instance, be it the main graph + // or a subgraph, so that the various operations that are part of the Resolve can work iteratively or + // recursively as needed. + struct ResolveContext { + ResolveContext() = default; + + std::unordered_map output_args; + std::unordered_set inputs_and_initializers; + std::unordered_set outer_scope_node_args; + std::unordered_map node_name_to_index; + std::unordered_map> node_to_subgraphs_map; + + void Clear() { + output_args.clear(); + inputs_and_initializers.clear(); + outer_scope_node_args.clear(); + node_name_to_index.clear(); + node_to_subgraphs_map.clear(); + } + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(ResolveContext); + }; + + // search this and up through any parent_graph_ instance for a NodeArg + const NodeArg* GetNodeArgIncludingParentGraphs(const std::string& node_arg_name) const; + + // Initialize all the graph inputs, initializers and outputs + common::Status InitInputsInitializersOutputs(); + + // recursively accumulate and set the outer scope node args in the resolve context for all subgraphs + // so they can be used to resolve outer scope dependencies when running BuildConnections for the subgraphs. + common::Status SetOuterScopeNodeArgs(const std::unordered_set& outer_scope_node_args); + + // Build and verify node connection (edges). + // Verify NodeArg name/type/shape matching correctly. + common::Status BuildConnections(std::vector& outer_scope_node_args_consumed); + + common::Status VerifyNoDuplicateName(); + + // Check whether <*this> graph is acyclic while performing a topological sort. + // Depth-first going from bottom up through the graph and checking whether there are any back edges. + // NodesInTopologicalOrder is updated with the nodes' indexes in topological + // order if returned is "OK", otherwise it's undefined. + common::Status PerformTopologicalSortAndCheckIsAcyclic(); + + common::Status PerformTypeAndShapeInferencing(); + + enum class Type { + // A main graph. + Main = 1, + // A sub graph (function). + Sub = 2, + }; + + common::Status Resolve(bool no_proto_sync_required); + + common::Status CreateSubgraphs(); + + // Iterate this Graph instance and all subgraphs, calling the provided function for each. + common::Status ForThisAndAllSubgraphs(std::function func); + + common::Status InferAndVerifyTypeMatch(Node& node, const ONNX_NAMESPACE::OpSchema& op); + + // perform type and shape inferencing on the subgraph and Resolve to validate + static common::Status InferAndVerifySubgraphTypes(const Node& node, Graph& subgraph, + const std::vector& input_types, + std::vector& output_types); + + // Apply type-inference and type-checking to all inputs and initializers: + common::Status TypeCheckInputsAndInitializers(); + + // Compute set of input and initializer names and checking for duplicate names + common::Status VerifyInputAndInitializerNames(); + + // Infer and set type information across <*this> graph if needed, and verify type/attribute + // information matches between node and op. + common::Status VerifyNodeAndOpMatch(); + + // Set graph inputs/outputs when resolving a graph.. + common::Status SetGraphInputsOutputs(); + + // Sync graph inputs/outputs when serializing to proto. + void SyncGraphInputsOutputs(); + + // Clear all unused initializers + void CleanUnusedInitializers(); + + gsl::not_null AllocateNode(); + + // Release the node. + // @returns false if node_index was invalid. + bool ReleaseNode(NodeIndex node_index); + + Node* NodeAtIndexImpl(NodeIndex node_index) const { + // if we are trying to access a node that doesn't exist there's (most + // likely) either a logic issue or a graph consistency/correctness issue. + // use ONNXRUNTIME_ENFORCE to prove that or uncover scenarios where we actually + // expect attempts to retrieve a non-existent node. + ONNXRUNTIME_ENFORCE(node_index < nodes_.size(), "Validating no unexpected access using an invalid node_index."); + return nodes_[node_index].get(); + } + + std::vector CreateNodeArgs(const google::protobuf::RepeatedPtrField& names, + const ArgNameToTypeMap& name_to_type_map); + + bool IsSubgraph() const { return parent_graph_ != nullptr; } + + void AddFunction(const ONNX_NAMESPACE::FunctionProto* func_proto); + + // GraphProto to store name, version, initializer. + // When serializing <*this> Graph to a GraphProto, the nodes and + // functions in will also be fed into so that + // it's consistent with <*this> graph. + // This pointer is owned by parent model. + ONNX_NAMESPACE::GraphProto* graph_proto_; + + InitializedTensorSet name_to_initial_tensor_; + std::vector removed_initializer_indexes_; + + Type graph_type_ = Type::Main; + + IOnnxRuntimeOpSchemaCollectionPtr schema_registry_; + + std::vector> function_container_; + + // Graph nodes. + // Element in may be nullptr due to graph optimization. + std::vector> nodes_; + + // Wrapper of Graph nodes to provide iteration services that hide nullptr entries + GraphNodes iterable_nodes_{nodes_}; + + // Number of nodes. + // Normally this is smaller than the size of , as some + // elements in may be removed when doing graph optimization, + // or some elements may be merged, etc. + int num_of_nodes_ = 0; + + // A flag indicates whether <*this> graph needs to be resolved. + bool graph_resolve_needed_ = false; + + bool graph_proto_sync_needed_ = false; + + // The topological order of node index used to do node and op match verification temporarily. + std::vector nodes_in_topological_order_; + + // Full list of graph inputs. Matches number and order of inputs in the GraphProto. + std::vector graph_inputs_including_initializers_; + + // Graph inputs excluding initializers. + std::vector graph_inputs_excluding_initializers_; + + // Graph outputs. + std::vector graph_outputs_; + + // Graph value_info. + std::vector value_info_; + + // All node args owned by <*this> graph. Key is node arg name. + std::unordered_map> node_args_; + + const std::unordered_map domain_to_version_; + + std::unordered_map model_functions_; + + // Model IR version. + Version ir_version_{}; + + int name_generator_ = 0; + + ResolveContext resolve_context_; + + // the parent graph if this is a subgraph. + Graph* parent_graph_; + + // entry for node containing subgraph, with value containing attribute_name:Graph pair + // as a node may contain multiple subgraphs (e.g. 'If' has one for both the 'then' and 'else' branches). + using AttributeGraphMap = std::unordered_map; + using SubgraphMap = std::unordered_map; + + SubgraphMap subgraph_map_; + std::vector> subgraphs_; + + // NodeArgs that come from outer scope. Used when building a graph so that + // these don't get recorded as graph inputs in the GraphProto. + std::unordered_set outer_scope_node_arg_names_; + + // Explicit graph input order to be used when constructing a Graph manually. + std::vector graph_input_order_; + + // Explicit graph output order to be used when constructing a Graph manually. + std::vector graph_output_order_; +}; + +} // namespace onnxruntime diff --git a/include/onnxruntime/core/graph/graph_nodes.h b/include/onnxruntime/core/graph/graph_nodes.h new file mode 100644 index 0000000000000..406510bc2a373 --- /dev/null +++ b/include/onnxruntime/core/graph/graph_nodes.h @@ -0,0 +1,123 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include + +namespace onnxruntime { + +class Node; + +/** +Class that provides iteration services for nodes in the Graph. +It's primary function is to hide holes in the nodes vector due to removed nodes. +*/ +class GraphNodes { + using TNodesContainer = std::vector>; + + public: + template + class NodeIterator; + + // construct a wrapper of the nodes that provides iteration services + explicit GraphNodes(TNodesContainer& nodes) noexcept : nodes_(nodes) {} + + using ConstNodeIterator = NodeIterator; + using MutableNodeIterator = NodeIterator; + + ConstNodeIterator cbegin() const noexcept { + return {nodes_.cbegin(), nodes_.cend()}; + } + + ConstNodeIterator cend() const noexcept { + return {nodes_.cend(), nodes_.cend()}; + } + + ConstNodeIterator begin() const noexcept { + return cbegin(); + } + + ConstNodeIterator end() const noexcept { + return cend(); + } + + MutableNodeIterator begin() noexcept { + return {nodes_.begin(), nodes_.end()}; + } + + MutableNodeIterator end() noexcept { + return {nodes_.end(), nodes_.end()}; + } + + // Iterator to provide const and non-const access to nodes, skipping invalid nodes. + template + class NodeIterator { + // get the type being returned by the iterator. can't use TIterator::value_type as that is always non-const + using IterType = typename std::remove_reference::reference>::type; + // and determine what we will return based on its constness + using T = typename std::conditional::value, + const Node, // return const Node if this is a const iterator + Node>::type; // else return Node + + public: + using iterator_category = std::input_iterator_tag; + using value_type = T; + using difference_type = typename TIterator::difference_type; // ptrdiff_t; + using pointer = T*; + using reference = T&; + using const_reference = std::add_const_t; + + // Constructor. Will move to a valid node or end. + NodeIterator(TIterator current, const TIterator end) noexcept : current_{current}, end_{end} { + // skip to valid node or end - whatever comes first + while (current_ < end && *current_ == nullptr) { + ++current_; + } + } + + bool operator==(const NodeIterator& other) const noexcept { + return (current_ == other.current_); + } + + bool operator!=(const NodeIterator& other) const noexcept { + return (current_ != other.current_); + } + + void operator++() { + if (current_ < end_) { + while (++current_ != end_) { + if (*current_ != nullptr) break; + } + } + } + + NodeIterator operator++(int) { + NodeIterator tmp{*this}; + ++(*this); + + return tmp; + } + + reference operator*() { + // if iterator is valid we always have a non-nullptr node + // if this is a nullptr we're at end_ and this shouldn't be being called + return **current_; + } + + pointer operator->() { + return current_->get(); + } + + private: + TIterator current_; + const TIterator end_; + }; + + private: + TNodesContainer& nodes_; +}; + +} // namespace onnxruntime diff --git a/include/onnxruntime/core/graph/graph_transformer.h b/include/onnxruntime/core/graph/graph_transformer.h new file mode 100644 index 0000000000000..c9afa18020da5 --- /dev/null +++ b/include/onnxruntime/core/graph/graph_transformer.h @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/graph/graph.h" +#include "core/graph/rewrite_rule.h" + +namespace onnxruntime { + +// A graph transformer interface. A graph transformer transforms a graph in-place. +class GraphTransformer { + public: + GraphTransformer(const std::string& name, const std::string& desc) + : name_(name), desc_(desc) { + } + + virtual ~GraphTransformer() = default; + + // The name of this graph transformer. + const std::string& Name() const noexcept { + return name_; + } + + // An description of this graph transformer. + const std::string& Description() const noexcept { + return desc_; + } + + // Apply <*this> transformation to a specific graph. + // Transformation happens in place. + // The return value of "modified" indicates if the graph was modified or not. + virtual ::onnxruntime::common::Status Apply(Graph& graph, bool& modified) const = 0; + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(GraphTransformer); + + const std::string name_; + const std::string desc_; +}; + +// Rule based graph transformer. +// It provides API to register rewrite rules, and API to apply for +// all applicable rules against one graph. + +// Represents a IGraphTransformer determined by a set of rewrite-rules. +// The transformer will apply all the rewrite-rules iteratively as +// determined by the underlying rewriting-strategy. +// Several rewriting-strategies are possible when traversing the graph and applying +// rewrite rules, each with different tradeoffs. At the moment, we define one +// that performs top-down traversal of nodes. +// TODO: Is a bottom-up traversal more efficient? +// TODO: Is it worth adding the max number of passes a rule should be applied for? +// TODO: We need to define a contract about whether a rewrite rule is allowed to leave +// the graph in an inconsistent state (this will determine when and where we will be +// calling resolve(). +class RuleBasedGraphTransformer : public GraphTransformer { + public: + RuleBasedGraphTransformer(const std::string& name, const std::string& desc) : GraphTransformer(name, desc) {} + + // Register a rewriting rule. + // TODO (revisit needed): Using OpSignature* here will ask that OpSignature + // should be stored globally. Otherwise, there will be multiple addresses/pointers + // for the same operator or function. To avoid this, we may use OpSignature ID + // as the key, which should be name_domain_version. + // We will use the string type instead of the OpSchema for now. We should probably + // add a version as well. + Status Register(const std::string& op_type, std::unique_ptr rule); + + // Returns true if there are rules registered for this op_type. + bool HasRules(const std::string& op_type) const { + return op_to_rules_.count(op_type) > 0; + } + + // Returns a reference to the vector that contains all rewrite rules registered + // for this operator. It assumes that there are registered rules, therefore HasRules + // should be called before. + const std::vector>& GetRewriteRules(const std::string& op_type) const { + return op_to_rules_.at(op_type); + } + + private: + using RewriteRuleSet = std::unordered_map>>; + + RewriteRuleSet op_to_rules_; +}; + +// This is a rule-based graph transformer that applies rules by performing top-down passes of the graph. +class TopDownRuleBasedTransformer : public RuleBasedGraphTransformer { + public: + TopDownRuleBasedTransformer(const std::string& name, const std::string& desc) : RuleBasedGraphTransformer(name, desc) {} + + // Performs a single top-down traversal of the graph and applies all registered rules. + ::onnxruntime::common::Status Apply(Graph&, bool&) const override; +}; + +} // namespace onnxruntime diff --git a/include/onnxruntime/core/graph/indexed_sub_graph.h b/include/onnxruntime/core/graph/indexed_sub_graph.h new file mode 100644 index 0000000000000..fae88f813cb58 --- /dev/null +++ b/include/onnxruntime/core/graph/indexed_sub_graph.h @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include "core/graph/basic_types.h" +#include "core/graph/onnx_protobuf.h" + +namespace onnxruntime { + +class OpKernel; +class OpKernelInfo; +// Sub-graph data structure. +// It contains a node index array covered by <*this> sub-graph, +// and contains meta definition needed for customizing <*this> +// sub-graph as a FunctionProto, which could be serialized/saved +// to a model file. +struct IndexedSubGraph { + struct MetaDef { + // Name of customized Sub-Graph/FunctionProto + std::string name; + // Domain of customized Sub-Graph/FunctionProto + std::string domain; + // Since version of customized Sub-Graph/FunctionProto. + int since_version; + // Status of customized Sub-Graph/FunctionProto. + ONNX_NAMESPACE::OperatorStatus status; + // Inputs of customized Sub-Graph/FunctionProto. + std::vector inputs; + // Outputs of customized Sub-Graph/FunctionProto. + std::vector outputs; + // Attributes of customized Sub-Graph/FunctionProto. + NodeAttributes attributes; + // Doc string of customized Sub-Graph/FunctionProto. + std::string doc_string; + }; + + // Nodes covered by <*this> sub-graph. + // The indexes are from parent graph. + std::vector nodes; + + // Meta definition needed for customizing <*this> + // sub-graph as a FunctionProto, which could be serialized/saved + // to a model file. It's needed IF AND ONLY IF there're multiple + // indexes contained in above. + + void SetMetaDef(std::unique_ptr& meta_def_) { + meta_def = std::move(meta_def_); + } + + const MetaDef* GetMetaDef() const { + return meta_def.get(); + } + + private: + // Sub-graph meta definition. + std::unique_ptr meta_def; +}; + +} // namespace onnxruntime diff --git a/include/onnxruntime/core/graph/node_arg.h b/include/onnxruntime/core/graph/node_arg.h new file mode 100644 index 0000000000000..a47d5b7796379 --- /dev/null +++ b/include/onnxruntime/core/graph/node_arg.h @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/graph/onnx_protobuf.h" + +namespace onnxruntime { + +// Node argument definition, for both input and output, +// including arg name, arg type (contains both type and shape). +// +// Design Question: in my opinion, shape should not be part of type. +// We may align the protobuf design with our operator registry interface, +// which has type specified for each operator, but no shape. Well, shape +// should be inferred with a separate shape inference function given +// input shapes, or input tensor data sometimes. +// With shape as part of type (current protobuf design), +// 1) we'll have to split the "TypeProto" into type and shape in this internal +// representation interface so that it could be easily used when doing type +// inference and matching with operator registry. +// 2) SetType should be always called before SetShape, otherwise, SetShape() +// will fail. Because shape is located in a TypeProto. +// Thoughts? +// +class NodeArg { + public: + // Constructor by specifying node arg name and type&shape which is + // optional. This is called when loading a from + // normally. + NodeArg(const std::string& name, + const ONNX_NAMESPACE::TypeProto* p_arg_type); + + NodeArg(NodeArg&& other) = default; + + // Get node arg name. + const std::string& Name() const noexcept; + + // Get node arg type. + ONNX_NAMESPACE::DataType Type() const noexcept; + const ONNX_NAMESPACE::TypeProto* TypeAsProto() const noexcept; + + // Get node arg shape. + // Return null pointer if there's no shape specified. + const ONNX_NAMESPACE::TensorShapeProto* Shape() const; + + // Set node arg shape. + // Shape could only be set after setting type since shape information + // now is part of TypeProto. + void SetShape(const ONNX_NAMESPACE::TensorShapeProto& shape); + + // validate and merge type [and shape] info from input_type. + // if there is existing type info that can't be cleanly updated return an error. + common::Status UpdateTypeAndShape(const ONNX_NAMESPACE::TypeProto& input_type); + + // validate and merge type [and shape] info from input_type. + // if there is existing type info that can't be cleanly updated return an error. + common::Status UpdateTypeAndShape(const NodeArg& node_arg); + + // Get node arg info proto. + const NodeArgInfo& ToProto() const noexcept { return node_arg_info_; } + + // Indicates whether <*this> node arg exists or not. + // Optional inputs are allowed in ONNX. Empty arg name represents + // a non-existing input argument. + bool Exists() const noexcept; + + private: + ONNXRUNTIME_DISALLOW_COPY_AND_ASSIGNMENT(NodeArg); + friend class Graph; + + void SetType(ONNX_NAMESPACE::DataType p_type); + void SetType(const ONNX_NAMESPACE::TypeProto& type_proto); + + NodeArg& operator=(NodeArg&& other) = delete; + + // Node arg PType. + ONNX_NAMESPACE::DataType type_; + + // Node arg name, type and shape. + NodeArgInfo node_arg_info_; + + // Flag indicates whether <*this> node arg exists or not. + bool exists_; +}; +} // namespace onnxruntime diff --git a/include/onnxruntime/core/graph/onnx_protobuf.h b/include/onnxruntime/core/graph/onnx_protobuf.h new file mode 100644 index 0000000000000..ff68b4127f4e7 --- /dev/null +++ b/include/onnxruntime/core/graph/onnx_protobuf.h @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +//TODO(): delete this file from public interface +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wignored-qualifiers" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#else +#pragma warning(push) +#pragma warning(disable : 4018) /*'expression' : signed/unsigned mismatch */ +#pragma warning(disable : 4065) /*switch statement contains 'default' but no 'case' labels*/ +#pragma warning(disable : 4100) +#pragma warning(disable : 4146) /*unary minus operator applied to unsigned type, result still unsigned*/ +#pragma warning(disable : 4244) /*'conversion' conversion from 'type1' to 'type2', possible loss of data*/ +#pragma warning(disable : 4251) /*'identifier' : class 'type' needs to have dll-interface to be used by clients of class 'type2'*/ +#pragma warning(disable : 4267) /*'var' : conversion from 'size_t' to 'type', possible loss of data*/ +#pragma warning(disable : 4305) /*'identifier' : truncation from 'type1' to 'type2'*/ +#pragma warning(disable : 4307) /*'operator' : integral constant overflow*/ +#pragma warning(disable : 4309) /*'conversion' : truncation of constant value*/ +#pragma warning(disable : 4334) /*'operator' : result of 32-bit shift implicitly converted to 64 bits (was 64-bit shift intended?)*/ +#pragma warning(disable : 4355) /*'this' : used in base member initializer list*/ +#pragma warning(disable : 4506) /*no definition for inline function 'function'*/ +#pragma warning(disable : 4800) /*'type' : forcing value to bool 'true' or 'false' (performance warning)*/ +#pragma warning(disable : 4996) /*The compiler encountered a deprecated declaration.*/ +#endif +#include "onnx/defs/schema.h" +#include "onnx/onnx_pb.h" +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#else +#pragma warning(pop) +#endif diff --git a/include/onnxruntime/core/graph/rewrite_rule.h b/include/onnxruntime/core/graph/rewrite_rule.h new file mode 100644 index 0000000000000..34cee5e5a5a65 --- /dev/null +++ b/include/onnxruntime/core/graph/rewrite_rule.h @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/graph/graph.h" + +namespace onnxruntime { + +// The graph rewrite API for rewrite rules. +class GraphEditor { + public: + explicit GraphEditor(Graph& graph) noexcept : graph_{graph} {} + + // Add a node in . + Node* AddNode(const std::string& name, + const std::string& op_type, + const std::string& description, + const std::vector& input_args, + const std::vector& output_args, + const std::string& domain = "") { + return graph_.AddNode(name, op_type, description, + input_args, output_args, nullptr, domain); + } + + // Copy an existing node into this graph. + Node* AddNode(const Node& other) { + return graph_.AddNode(other); + } + + // Remove a node from . + bool RemoveNode(NodeIndex node_index) { + return graph_.RemoveNode(node_index); + } + + // Add control edge into . + // The node does not consume any data output by + // , but it's designed to be executed behind. + bool AddControlEdge(NodeIndex src, NodeIndex dst) { + return graph_.AddControlEdge(src, dst); + } + + // Resolve after each editing. + ::onnxruntime::common::Status Resolve() { + return graph_.Resolve(); + } + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(GraphEditor); + + Graph& graph_; +}; + +// The base class for rewrite rule. A rewrite rule represents a semantics-preserving +// transformation of a computation-graph. It can be used to represent, for example, +// the elimination of operators that serve as no-ops (for example, dropout during +// inference), as well as inlining of "function" definitions or the dual (replacing +// a complex expression by an equivalent function-call). Unlike the more general +// IGraphTransformer, a rewrite-rule is applied at a single node, representing the +// root of an expression that is rewritten. +class RewriteRule { + public: + RewriteRule(const std::string& name, const std::string& desc) + : name_(name), desc_(desc) { + } + + virtual ~RewriteRule() = default; + + // The name of this rewrite rule. + const std::string& Name() const noexcept { + return name_; + } + + // An description of this rewrite rule. + const std::string& Description() const noexcept { + return desc_; + } + + // If the condition of the rule is satisfied, apply the rule. + ::onnxruntime::common::Status CheckConditionAndApply(GraphEditor* graph_editor, Node* node, bool* modified) { + return SatisfyCondition(*node) ? Apply(graph_editor, node, modified) : Status::OK(); + } + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(RewriteRule); + + const std::string name_; + const std::string desc_; + + // The rewrite rule is applied if the condition function returns true. This can include + // a more complex pattern matching (conditions on the ascending or descending nodes of the + // node for which this rule was triggered) or some other properties of the nodes. + virtual bool SatisfyCondition(const Node& node) = 0; + + // Apply the rewrite rule to a specific node. + // The transformation happens in-place. The return-value of node may be different + // from the input-value due to rewriting. + // The return value of "modified" indicates if the graph was modified or not. + virtual ::onnxruntime::common::Status Apply(GraphEditor* graph_editor, Node* node, bool* modified) = 0; +}; +} // namespace onnxruntime diff --git a/include/onnxruntime/core/graph/schema_registry.h b/include/onnxruntime/core/graph/schema_registry.h new file mode 100644 index 0000000000000..b92b902b61604 --- /dev/null +++ b/include/onnxruntime/core/graph/schema_registry.h @@ -0,0 +1,144 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/graph/constants.h" +#include "core/common/common.h" +#include "core/common/status.h" +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wignored-qualifiers" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif +#include "onnx/defs/schema.h" +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif +#include +#include +#include "sstream" + +namespace onnxruntime { +using OpName_Domain_Version_Schema_Map = std::unordered_map< + std::string, + std::unordered_map>>; + +// onnxruntime schema registry is a supplement to built-in schema, +// Every schema registry represent a collection of schema deltas from baseline_opset_version to opset_version +struct SchemaRegistryVersion { + int baseline_opset_version; + int opset_version; +}; + +using Domain_To_Version_Map = std::unordered_map; +using Domain_To_Version_Range_Map = std::unordered_map; + +class IOnnxRuntimeOpSchemaCollection : public ONNX_NAMESPACE::ISchemaRegistry { + public: + virtual Domain_To_Version_Map GetLatestOpsetVersions(bool is_onnx_only) const = 0; + + using ISchemaRegistry::GetSchema; + + virtual const ONNX_NAMESPACE::OpSchema* GetSchema( + const std::string& key, + const int maxInclusiveVersion, + const std::string& domain) const final { + const ONNX_NAMESPACE::OpSchema* latest_schema = nullptr; + int earliest_opset_where_unchanged = std::numeric_limits::max(); + GetSchemaAndHistory(key, maxInclusiveVersion, domain, &latest_schema, &earliest_opset_where_unchanged); + + assert(latest_schema == nullptr || (latest_schema->SinceVersion() <= maxInclusiveVersion && + earliest_opset_where_unchanged == latest_schema->SinceVersion())); + + return latest_schema; + } + + virtual void GetSchemaAndHistory( + const std::string& key, + int maxInclusiveVersion, + const std::string& domain, + const ONNX_NAMESPACE::OpSchema** latest_schema, + int* earliest_opset_where_unchanged) const = 0; +}; + +// OnnxRuntimeOpSchemaRegistry is used to provide supplement for built-in ONNX schemas. +// Each OnnxRuntimeOpSchemaRegistry must register complete opsets delta from a baseline version to max opset version. +// (Please notice that baseline opsets are not include in the delta) +// For example, ONNXRuntime is build with ONNX 1.2 which is at opset7, to use onnx opset8 and opset9, +// user could create a OnnxRuntimeOpSchemaRegistry and config it as {baseline_opset_version = 7, opset_version = 9} +// it means this OnnxRuntimeOpSchemaRegistry contains the complete delta from opset7 to opset9. +class OnnxRuntimeOpSchemaRegistry : public IOnnxRuntimeOpSchemaCollection { + public: + OnnxRuntimeOpSchemaRegistry() = default; + + ::onnxruntime::common::Status SetBaselineAndOpsetVersionForDomain( + const std::string& domain, + int baseline_opset_version, + int opset_version); + + Domain_To_Version_Map GetLatestOpsetVersions(bool is_onnx_only) const override; + + // OnnxRuntimeOpSchemaRegistry must register complete delta for a opset. + ::onnxruntime::common::Status RegisterOpSet( + std::vector& schemas, + const std::string& domain, + int baseline_opset_version, + int opset_version); + +// conversion of kOnnxDomain to std::string creates unnamed temporary. Suppress C26444 (es.84) the hard way. +// GSL_SUPPRESS(es.84) doesn't work as the default arg temporary isn't in a scope the suppress attribute handles. +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 26444) +#endif + + using IOnnxRuntimeOpSchemaCollection::GetSchema; + + void GetSchemaAndHistory( + const std::string& key, + const int maxInclusiveVersion, + const std::string& domain, + const ONNX_NAMESPACE::OpSchema** latest_schema, + int* earliest_opset_where_unchanged) const override; + +#ifdef _MSC_VER +#pragma warning(pop) // C26444 +#endif + + bool empty() const { + return map_.empty(); + } + + private: + ::onnxruntime::common::Status RegisterOpSchema(ONNX_NAMESPACE::OpSchema&& op_schema); + + ::onnxruntime::common::Status RegisterOpSchemaInternal(ONNX_NAMESPACE::OpSchema&& op_schema); + + std::mutex mutex_; + + OpName_Domain_Version_Schema_Map map_; + Domain_To_Version_Range_Map domain_version_range_map_; +}; + +// SchemaRegistryManager provides a view based on built-in ONNX schema and a list of OnnxRuntimeOpSchemaRegistry as supplement. +// User need to make sure the customized schema registry is valid, otherwise the behavior is undefined. +// We may add more consistent check later. +class SchemaRegistryManager : public onnxruntime::IOnnxRuntimeOpSchemaCollection { + public: + // The schema registry priority is the reverse of register order. + void RegisterRegistry(std::shared_ptr registry); + + Domain_To_Version_Map GetLatestOpsetVersions(bool is_onnx_only) const override; + + void GetSchemaAndHistory( + const std::string& key, + const int maxInclusiveVersion, + const std::string& domain, + const ONNX_NAMESPACE::OpSchema** latest_schema, + int* earliest_opset_where_unchanged) const override; + + private: + std::deque> registries; +}; + +} // namespace onnxruntime diff --git a/include/onnxruntime/core/providers/cpu/cpu_provider_factory.h b/include/onnxruntime/core/providers/cpu/cpu_provider_factory.h new file mode 100644 index 0000000000000..26b21bd5541ea --- /dev/null +++ b/include/onnxruntime/core/providers/cpu/cpu_provider_factory.h @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/onnxruntime_c_api.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * \param use_arena zero: false. non-zero: true. + * \param out Call ONNXRuntimeReleaseObject() method when you no longer need to use it. + */ +ONNXRUNTIME_API_STATUS(ONNXRuntimeCreateCpuExecutionProviderFactory, int use_arena, _Out_ ONNXRuntimeProviderFactoryPtr** out) +ONNXRUNTIME_ALL_ARGS_NONNULL; + +ONNXRUNTIME_API_STATUS(ONNXRuntimeCreateCpuAllocatorInfo, enum ONNXRuntimeAllocatorType type, enum ONNXRuntimeMemType mem_type1, _Out_ ONNXRuntimeAllocatorInfo** out) +ONNXRUNTIME_ALL_ARGS_NONNULL; + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/include/onnxruntime/core/providers/cuda/cuda_provider_factory.h b/include/onnxruntime/core/providers/cuda/cuda_provider_factory.h new file mode 100644 index 0000000000000..fc687651d595b --- /dev/null +++ b/include/onnxruntime/core/providers/cuda/cuda_provider_factory.h @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/onnxruntime_c_api.h" + +#ifdef __cplusplus +extern "C" { +#endif +/** + * \param device_id cuda device id, starts from zero. + * \param out Call ONNXRuntimeReleaseObject() method when you no longer need to use it. + */ +ONNXRUNTIME_API_STATUS(ONNXRuntimeCreateCUDAExecutionProviderFactory, int device_id, _Out_ ONNXRuntimeProviderFactoryPtr** out); + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/include/onnxruntime/core/providers/mkldnn/mkldnn_provider_factory.h b/include/onnxruntime/core/providers/mkldnn/mkldnn_provider_factory.h new file mode 100644 index 0000000000000..5df921f401fa4 --- /dev/null +++ b/include/onnxruntime/core/providers/mkldnn/mkldnn_provider_factory.h @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/onnxruntime_c_api.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * \param use_arena zero: false. non-zero: true. + * \param out Call ONNXRuntimeReleaseObject() method when you no longer need to use it. + */ +ONNXRUNTIME_API_STATUS(ONNXRuntimeCreateMkldnnExecutionProviderFactory, int use_arena, _Out_ ONNXRuntimeProviderFactoryPtr** out); + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/include/onnxruntime/core/session/allocator.h b/include/onnxruntime/core/session/allocator.h new file mode 100644 index 0000000000000..a2a7a70c28529 --- /dev/null +++ b/include/onnxruntime/core/session/allocator.h @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/framework/error_code.h" +#include "core/framework/allocator_info.h" +#include "core/framework/onnx_object.h" + +#ifdef __cplusplus +extern "C" { +#endif +//inherented from ONNXObject +typedef struct ONNXRuntimeAllocatorInteface { + struct ONNXObject parent; + void*(ONNXRUNTIME_API_STATUSCALL* Alloc)(void* this_, size_t size); + void(ONNXRUNTIME_API_STATUSCALL* Free)(void* this_, void* p); + const struct ONNXRuntimeAllocatorInfo*(ONNXRUNTIME_API_STATUSCALL* Info)(const void* this_); +} ONNXRuntimeAllocatorInteface; + +typedef ONNXRuntimeAllocatorInteface* ONNXRuntimeAllocator; + +ONNXRUNTIME_API(void*, ONNXRuntimeAllocatorAlloc, _Inout_ ONNXRuntimeAllocator* ptr, size_t size); +ONNXRUNTIME_API(void, ONNXRuntimeAllocatorFree, _Inout_ ONNXRuntimeAllocator* ptr, void* p); +ONNXRUNTIME_API(const struct ONNXRuntimeAllocatorInfo*, ONNXRuntimeAllocatorGetInfo, _In_ const ONNXRuntimeAllocator* ptr); + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h new file mode 100644 index 0000000000000..a6aa9391a59ae --- /dev/null +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -0,0 +1,203 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include +#include +#include + +#include "core/common/visibility_macros.h" +#include "core/framework/error_code.h" +#include "core/framework/onnx_object.h" +#include "core/framework/run_options_c_api.h" +#include "core/session/tensor_type_and_shape_c_api.h" +#include "allocator.h" +#include "session_options_c_api.h" + +#ifdef __cplusplus +extern "C" { +#endif + +//Any pointer marked with _In_ or _Out_, cannot be NULL. Caller should ensure that. + +typedef enum ONNXRuntimeType { + ONNXRUNTIME_TYPE_TENSOR, + ONNXRUNTIME_TYPE_SEQUENCE, + ONNXRUNTIME_TYPE_MAP, + ONNXRUNTIME_TYPE_OPAQUE, + ONNXRUNTIME_TYPE_ELEMENT, //basic types like float/int32 +} ONNXRuntimeType; + +typedef struct ONNXOpaqueTypeInfo { + char* domain; + char* name; +} ONNXOpaqueTypeInfo; + +//Each ONNX value is a n-ary tree. +//Data is only stored in leaf nodes. +//Every non-leaf node contains a field of ONNXRuntimeType +//Each leaf node is either a tensor, or an ONNXArray. + +/** + * ReleaseONNXEnv function calls ::google::protobuf::ShutdownProtobufLibrary(). + * Therefore, you should only call ReleaseONNXEnv at the end of your program. + * Once you did that, don't call any onnxruntime, onnx or protobuf functions again. + */ +DEFINE_RUNTIME_CLASS(ONNXEnv); + +typedef enum ONNXRuntimeLoggingLevel { + ONNXRUNTIME_LOGGING_LEVEL_kVERBOSE = 0, + ONNXRUNTIME_LOGGING_LEVEL_kINFO = 1, + ONNXRUNTIME_LOGGING_LEVEL_kWARNING = 2, + ONNXRUNTIME_LOGGING_LEVEL_kERROR = 3, + ONNXRUNTIME_LOGGING_LEVEL_kFATAL = 4 +} ONNXRuntimeLoggingLevel; + +typedef void(ONNXRUNTIME_API_STATUSCALL* ONNXRuntimeLoggingFunction)( + void* param, ONNXRuntimeLoggingLevel severity, const char* category, const char* logid, const char* code_location, + const char* message); +/** + * ONNXEnv is process-wise. For each process, only one ONNXEnv can be created. Don't do it multiple times + */ +ONNXRUNTIME_API_STATUS(ONNXRuntimeInitialize, ONNXRuntimeLoggingLevel default_warning_level, _In_ const char* logid, + _Out_ ONNXEnv** out) +ONNXRUNTIME_ALL_ARGS_NONNULL; +/** + * ONNXEnv is process-wise. For each process, only one ONNXEnv can be created. Don't do it multiple times + */ +ONNXRUNTIME_API_STATUS(ONNXRuntimeInitializeWithCustomLogger, ONNXRuntimeLoggingFunction logging_function, + void* logger_param, ONNXRuntimeLoggingLevel default_warning_level, + _In_ const char* logid, + _Out_ ONNXEnv** out); + +DEFINE_RUNTIME_CLASS(ONNXSession); + +//TODO: document the path separator convention? '/' vs '\' +//TODO: should specify the access characteristics of model_path. Is this read only during the +//execution of ONNXRuntimeCreateInferenceSession, or does the ONNXSession retain a handle to the file/directory +//and continue to access throughout the ONNXSession lifetime? +// What sort of access is needed to model_path : read or read/write? +//TODO: allow loading from an in-memory byte-array +#ifdef _WIN32 +ONNXRUNTIME_API_STATUS(ONNXRuntimeCreateInferenceSession, _In_ ONNXEnv* env, _In_ const wchar_t* model_path, + _In_ const ONNXRuntimeSessionOptions* options, _Out_ ONNXSessionPtr* out); +#else +ONNXRUNTIME_API_STATUS(ONNXRuntimeCreateInferenceSession, _In_ ONNXEnv* env, _In_ const char* model_path, + _In_ const ONNXRuntimeSessionOptions* options, _Out_ ONNXSessionPtr* out); +#endif + +DEFINE_RUNTIME_CLASS(ONNXValue); + +///Call ONNXRuntimeReleaseObject to release the returned value +ONNXRUNTIME_API_STATUS(ONNXRuntimeCreateDefaultAllocator, _Out_ ONNXRuntimeAllocator** out); + +/** + * This function is only for advanced users. In most cases, please use ONNXRuntimeCreateTensorWithDataAsONNXValue + * The returned ONNXValuePtr will keep a reference to allocator, without reference counting + * \param type must be one of TENSOR_ELEMENT_DATA_TYPE_xxxx + */ +ONNXRUNTIME_API_STATUS(ONNXRuntimeCreateTensorAsONNXValue, _Inout_ ONNXRuntimeAllocator* allocator, + _In_ const size_t* shape, size_t shape_len, OnnxRuntimeTensorElementDataType type, + _Out_ ONNXValuePtr* out); + +/** + * p_data is owned by caller. ReleaseTensor won't release p_data. + */ +ONNXRUNTIME_API_STATUS(ONNXRuntimeCreateTensorWithDataAsONNXValue, _In_ const ONNXRuntimeAllocatorInfo* info, + _In_ void* p_data, size_t p_data_len, _In_ const size_t* shape, size_t shape_len, + OnnxRuntimeTensorElementDataType type, _Out_ ONNXValuePtr* out); + +/// This function doesn't work with string tensor +/// this is a no-copy method whose pointer is only valid until the backing ONNXValuePtr is free'd. +ONNXRUNTIME_API_STATUS(ONNXRuntimeGetTensorMutableData, _In_ ONNXValuePtr value, _Out_ void** out); + +/** + * \return zero, false. non-zero true + */ +ONNXRUNTIME_API(int, ONNXRuntimeIsTensor, _In_ ONNXValuePtr value); + +/** + * \param value A tensor created from ONNXRuntimeCreateTensor*** function. + * \param s each A string array. Each string in this array must be null terminated. + * \param s_len length of s + */ +ONNXRUNTIME_API_STATUS(ONNXRuntimeFillStringTensor, _In_ ONNXValuePtr value, _In_ const char* s[], size_t s_len); +/** + * \param value A tensor created from ONNXRuntimeCreateTensor*** function. + * \param len total data length, not including the trailing '\0' chars. + */ +ONNXRUNTIME_API_STATUS(ONNXRuntimeGetStringTensorDataLength, _In_ ONNXValuePtr value, _Out_ size_t* len); + +/** + * \param s string contents. Each string is NOT null-terminated. + * \param value A tensor created from ONNXRuntimeCreateTensor*** function. + * \param s_len total data length, get it from ONNXRuntimeGetStringTensorDataLength + */ +ONNXRUNTIME_API_STATUS(ONNXRuntimeGetStringTensorContent, _In_ ONNXValuePtr value, _Out_ void* s, size_t s_len, + _Out_ size_t* offsets, size_t offsets_len); + +/** + * \param out Should be freed by ONNXRuntimeReleaseObject after use + */ +ONNXRUNTIME_API_STATUS(ONNXRuntimeGetTensorShapeAndType, _In_ const ONNXValuePtr, + _Out_ struct ONNXRuntimeTensorTypeAndShapeInfo** out); + +//not implemented +//ONNX_RUNTIME_EXPORT int GetPONNXValueDataType(_In_ ONNXValuePtr) NO_EXCEPTION; + +DEFINE_RUNTIME_CLASS(ONNXValueList); + +//For InferenceSession run calls, all the input values shouldn't created by allocator +//User should manage the buffer by himself, not allocator + +/** + * \param sess created by ONNXRuntimeCreateInferenceSession function + * \param output must be freed by ReleaseONNXValueListPtr function + */ +ONNXRUNTIME_API_STATUS(ONNXRuntimeRunInferenceAndFetchAll, _In_ ONNXSessionPtr sess, + _In_ const char* input_names[], _In_ ONNXValuePtr* input, size_t input_len, + _Out_ ONNXValueListPtr* output, _Out_ size_t* output_len); +ONNXRUNTIME_API_STATUS(ONNXRuntimeRunInferenceAndFetchAllWithRunOptions, _In_ ONNXSessionPtr sess, + _In_ ONNXRuntimeRunOptionsPtr run_options, + _In_ const char* input_names[], _In_ ONNXValuePtr* input, size_t input_len, + _Out_ ONNXValueListPtr* output, _Out_ size_t* output_len); +ONNXRUNTIME_API_STATUS(ONNXRuntimeRunInference, _In_ ONNXSessionPtr sess, + _In_ const char* input_names[], _In_ ONNXValuePtr* input, size_t input_len, + _In_ const char* output_names[], size_t output_names_len, _Out_ ONNXValuePtr* output); +ONNXRUNTIME_API_STATUS(ONNXRuntimeRunInferenceWithRunOptions, _In_ ONNXSessionPtr sess, + _In_ ONNXRuntimeRunOptionsPtr run_options, + _In_ const char* input_names[], _In_ ONNXValuePtr* input, size_t input_len, + _In_ const char* output_names[], size_t output_names_len, _Out_ ONNXValuePtr* output); + +ONNXRUNTIME_API_STATUS(ONNXRuntimeInferenceSessionGetInputCount, _In_ ONNXSessionPtr sess, _Out_ size_t* out); +ONNXRUNTIME_API_STATUS(ONNXRuntimeInferenceSessionGetOutputCount, _In_ ONNXSessionPtr sess, _Out_ size_t* out); + +ONNXRUNTIME_API_STATUS(ONNXRuntimeInferenceSessionGetInputName, _In_ ONNXSessionPtr sess, size_t index, + _Inout_ ONNXRuntimeAllocator* allocator, _Out_ char** value); +ONNXRUNTIME_API_STATUS(ONNXRuntimeInferenceSessionGetOutputName, _In_ ONNXSessionPtr sess, size_t index, + _Inout_ ONNXRuntimeAllocator* allocator, _Out_ char** value); + +//Tree for PONNXType: +//ONNXRUNTIME_TYPE_TENSOR -> ONNXTensorTypeInfo +//ONNXRUNTIME_TYPE_SEQUENCE -> nullptr +//ONNXRUNTIME_TYPE_MAP -> nullptr +//ONNXRUNTIME_TYPE_OPAQUE-> ONNXOpaqueTypeInfo +//ONNXRUNTIME_TYPE_ELEMENT -> nullptr + +//The output value must be freed by ONNXRuntimeNodeDestoryTree +//ONNXRUNTIME_API_STATUS(ONNXRuntimeInferenceSessionGetInputType, _In_ ONNXSessionPtr sess, _Out_ PONNXType* out); +//ONNXRUNTIME_API_STATUS(ONNXRuntimeInferenceSessionGetOutputType, _In_ ONNXSessionPtr sess, _Out_ PONNXType* out); + +/** + * Get the n-th value from the List + * \param index starts from zero + */ +ONNXRUNTIME_API(ONNXValuePtr, ONNXRuntimeONNXValueListGetNthValue, _In_ ONNXValueListPtr list, size_t index); + +ONNXRUNTIME_API_STATUS(ONNXRuntimeTensorProtoToONNXValue, _Inout_ ONNXRuntimeAllocator* allocator, + _In_ const void* input, int input_len, _Out_ ONNXValuePtr* out); + +#ifdef __cplusplus +} +#endif diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_api.h b/include/onnxruntime/core/session/onnxruntime_cxx_api.h new file mode 100644 index 0000000000000..e740d300a8d45 --- /dev/null +++ b/include/onnxruntime/core/session/onnxruntime_cxx_api.h @@ -0,0 +1,118 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "onnxruntime_c_api.h" +#include +#include +#include +#include + +//TODO: encode error code in the message? +#define ONNXRUNTIME_THROW_ON_ERROR(expr) \ + do { \ + ONNXStatusPtr onnx_status = (expr); \ + if (onnx_status != nullptr) { \ + std::string onnx_runtime_error_message = ONNXRuntimeGetErrorMessage(onnx_status); \ + ReleaseONNXStatus(onnx_status); \ + throw std::runtime_error(onnx_runtime_error_message); \ + } \ + } while (0); + +#define ONNXRUNTIME_REDIRECT_SIMPLE_FUNCTION_CALL(NAME) \ + decltype(ONNXRuntime##NAME(value.get())) NAME() { \ + return ONNXRuntime##NAME(value.get()); \ + } + +namespace onnxruntime { +class SessionOptionsWrapper { + private: + std::unique_ptr value; + ONNXEnvPtr env_; + SessionOptionsWrapper(_In_ ONNXEnvPtr env, ONNXRuntimeSessionOptions* p) : value(p, ONNXRuntimeReleaseObject), env_(env){}; + + public: + SessionOptionsWrapper(_In_ ONNXEnvPtr env) : value(ONNXRuntimeCreateSessionOptions(), ONNXRuntimeReleaseObject), env_(env){}; + ONNXRUNTIME_REDIRECT_SIMPLE_FUNCTION_CALL(EnableSequentialExecution) + ONNXRUNTIME_REDIRECT_SIMPLE_FUNCTION_CALL(DisableSequentialExecution) + ONNXRUNTIME_REDIRECT_SIMPLE_FUNCTION_CALL(DisableProfiling) + ONNXRUNTIME_REDIRECT_SIMPLE_FUNCTION_CALL(EnableMemPattern) + ONNXRUNTIME_REDIRECT_SIMPLE_FUNCTION_CALL(DisableMemPattern) + ONNXRUNTIME_REDIRECT_SIMPLE_FUNCTION_CALL(EnableCpuMemArena) + ONNXRUNTIME_REDIRECT_SIMPLE_FUNCTION_CALL(DisableCpuMemArena) + void EnableProfiling(_In_ const char* profile_file_prefix) { + ONNXRuntimeEnableProfiling(value.get(), profile_file_prefix); + } + + void SetSessionLogId(const char* logid) { + ONNXRuntimeSetSessionLogId(value.get(), logid); + } + void SetSessionLogVerbosityLevel(uint32_t session_log_verbosity_level) { + ONNXRuntimeSetSessionLogVerbosityLevel(value.get(), session_log_verbosity_level); + } + void SetSessionThreadPoolSize(int session_thread_pool_size) { + ONNXRuntimeSetSessionThreadPoolSize(value.get(), session_thread_pool_size); + } + + /** + * The order of invocation indicates the preference order as well. In other words call this method + * on your most preferred execution provider first followed by the less preferred ones. + * Calling this API is optional in which case onnxruntime will use its internal CPU execution provider. + */ + void AppendExecutionProvider(_In_ ONNXRuntimeProviderFactoryPtr* f) { + ONNXRuntimeSessionOptionsAppendExecutionProvider(value.get(), f); + } + + SessionOptionsWrapper clone() const { + ONNXRuntimeSessionOptions* p = ONNXRuntimeCloneSessionOptions(value.get()); + return SessionOptionsWrapper(env_, p); + } +#ifdef _WIN32 + ONNXSessionPtr ONNXRuntimeCreateInferenceSession(_In_ const wchar_t* model_path) { + ONNXSessionPtr ret; + ONNXRUNTIME_THROW_ON_ERROR(::ONNXRuntimeCreateInferenceSession(env_, model_path, value.get(), &ret)); + return ret; + } +#else + ONNXSessionPtr ONNXRuntimeCreateInferenceSession(_In_ const char* model_path) { + ONNXSessionPtr ret; + ONNXRUNTIME_THROW_ON_ERROR(::ONNXRuntimeCreateInferenceSession(env_, model_path, value.get(), &ret)); + return ret; + } +#endif + void AddCustomOp(_In_ const char* custom_op_path) { + ONNXRuntimeAddCustomOp(value.get(), custom_op_path); + } +}; +inline ONNXValuePtr ONNXRuntimeCreateTensorAsONNXValue(_Inout_ ONNXRuntimeAllocator* env, const std::vector& shape, OnnxRuntimeTensorElementDataType type) { + ONNXValuePtr ret; + ONNXRUNTIME_THROW_ON_ERROR(::ONNXRuntimeCreateTensorAsONNXValue(env, shape.data(), shape.size(), type, &ret)); + return ret; +} + +inline ONNXValuePtr ONNXRuntimeCreateTensorWithDataAsONNXValue(_In_ const ONNXRuntimeAllocatorInfo* info, _In_ void* p_data, size_t p_data_len, const std::vector& shape, OnnxRuntimeTensorElementDataType type) { + ONNXValuePtr ret; + ONNXRUNTIME_THROW_ON_ERROR(::ONNXRuntimeCreateTensorWithDataAsONNXValue(info, p_data, p_data_len, shape.data(), shape.size(), type, &ret)); + return ret; +} + +} // namespace onnxruntime + +#define DECLEAR_DEFAULT_DELETER_FOR_ONNX_OBJECT(TYPE_NAME) \ + namespace std { \ + template <> \ + struct default_delete { \ + void operator()(ONNXRuntime##TYPE_NAME* ptr) { \ + (*reinterpret_cast(ptr))->Release(ptr); \ + } \ + }; \ + } + +DECLEAR_DEFAULT_DELETER_FOR_ONNX_OBJECT(Allocator); +DECLEAR_DEFAULT_DELETER_FOR_ONNX_OBJECT(TensorTypeAndShapeInfo); +DECLEAR_DEFAULT_DELETER_FOR_ONNX_OBJECT(RunOptions); +DECLEAR_DEFAULT_DELETER_FOR_ONNX_OBJECT(SessionOptions); +DECLEAR_DEFAULT_DELETER_FOR_ONNX_OBJECT(ProviderFactoryPtr); + +#undef DECLEAR_DEFAULT_DELETER_FOR_ONNX_OBJECT +#undef ONNXRUNTIME_REDIRECT_SIMPLE_FUNCTION_CALL diff --git a/include/onnxruntime/core/session/session_options_c_api.h b/include/onnxruntime/core/session/session_options_c_api.h new file mode 100644 index 0000000000000..5afbfed209b5e --- /dev/null +++ b/include/onnxruntime/core/session/session_options_c_api.h @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/visibility_macros.h" +#include "core/framework/error_code.h" +#include "core/framework/onnx_object.h" + +#ifdef __cplusplus +extern "C" { +#endif + +DEFINE_RUNTIME_CLASS(ONNXRuntimeProvider); + +//Inherented from ONNXObject +typedef struct ONNXRuntimeProviderFactoryInterface { + ONNXObject parent; + ONNXStatusPtr(ONNXRUNTIME_API_STATUSCALL* CreateProvider)(void* this_, ONNXRuntimeProviderPtr* out); +} ONNXRuntimeProviderFactoryInterface; + +typedef ONNXRuntimeProviderFactoryInterface* ONNXRuntimeProviderFactoryPtr; + +struct ONNXRuntimeSessionOptions; +typedef struct ONNXRuntimeSessionOptions ONNXRuntimeSessionOptions; +typedef ONNXRuntimeSessionOptions* ONNXRuntimeSessionOptionsPtr; + +/** + * \return A pointer of the newly created object. The pointer should be freed by ONNXRuntimeReleaseObject after use + */ +ONNXRUNTIME_API(ONNXRuntimeSessionOptions*, ONNXRuntimeCreateSessionOptions, void); + +/// create a copy of an existing ONNXRuntimeSessionOptions +ONNXRUNTIME_API(ONNXRuntimeSessionOptions*, ONNXRuntimeCloneSessionOptions, ONNXRuntimeSessionOptions*); +ONNXRUNTIME_API(void, ONNXRuntimeEnableSequentialExecution, _In_ ONNXRuntimeSessionOptions* options); +ONNXRUNTIME_API(void, ONNXRuntimeDisableSequentialExecution, _In_ ONNXRuntimeSessionOptions* options); + +// enable profiling for this session. +ONNXRUNTIME_API(void, ONNXRuntimeEnableProfiling, _In_ ONNXRuntimeSessionOptions* options, _In_ const char* profile_file_prefix); +ONNXRUNTIME_API(void, ONNXRuntimeDisableProfiling, _In_ ONNXRuntimeSessionOptions* options); + +// enable the memory pattern optimization. +// The idea is if the input shapes are the same, we could trace the internal memory allocation +// and generate a memory pattern for future request. So next time we could just do one allocation +// with a big chunk for all the internal memory allocation. +ONNXRUNTIME_API(void, ONNXRuntimeEnableMemPattern, _In_ ONNXRuntimeSessionOptions* options); +ONNXRUNTIME_API(void, ONNXRuntimeDisableMemPattern, _In_ ONNXRuntimeSessionOptions* options); + +// enable the memory arena on CPU +// Arena may pre-allocate memory for future usage. +// set this option to false if you don't want it. +ONNXRUNTIME_API(void, ONNXRuntimeEnableCpuMemArena, _In_ ONNXRuntimeSessionOptions* options); +ONNXRUNTIME_API(void, ONNXRuntimeDisableCpuMemArena, _In_ ONNXRuntimeSessionOptions* options); + +///< logger id to use for session output +ONNXRUNTIME_API(void, ONNXRuntimeSetSessionLogId, _In_ ONNXRuntimeSessionOptions* options, const char* logid); + +///< applies to session load, initialization, etc +ONNXRUNTIME_API(void, ONNXRuntimeSetSessionLogVerbosityLevel, _In_ ONNXRuntimeSessionOptions* options, uint32_t session_log_verbosity_level); + +///How many threads in the session thread pool. +ONNXRUNTIME_API(int, ONNXRuntimeSetSessionThreadPoolSize, _In_ ONNXRuntimeSessionOptions* options, int session_thread_pool_size); + +/** + * The order of invocation indicates the preference order as well. In other words call this method + * on your most preferred execution provider first followed by the less preferred ones. + * Calling this API is optional in which case onnxruntime will use its internal CPU execution provider. + */ +ONNXRUNTIME_API(void, ONNXRuntimeSessionOptionsAppendExecutionProvider, _In_ ONNXRuntimeSessionOptions* options, _In_ ONNXRuntimeProviderFactoryPtr* f); + +ONNXRUNTIME_API(void, ONNXRuntimeAddCustomOp, _In_ ONNXRuntimeSessionOptions* options, const char* custom_op_path); + +#ifdef __cplusplus +} +#endif diff --git a/include/onnxruntime/core/session/tensor_type_and_shape_c_api.h b/include/onnxruntime/core/session/tensor_type_and_shape_c_api.h new file mode 100644 index 0000000000000..1e010f458e7cd --- /dev/null +++ b/include/onnxruntime/core/session/tensor_type_and_shape_c_api.h @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/framework/error_code.h" +#ifdef __cplusplus +extern "C" { +#endif +struct ONNXRuntimeTensorTypeAndShapeInfo; + +//copied from TensorProto::DataType +typedef enum OnnxRuntimeTensorElementDataType { + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT = 1, // float + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8 = 2, // uint8_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8 = 3, // int8_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16 = 4, // uint16_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16 = 5, // int16_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32 = 6, // int32_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64 = 7, // int64_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING = 8, // string + ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL = 9, // bool + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16 = 10, + ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE = 11, + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32 = 12, + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64 = 13, + ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64 = 14, // complex with float32 real and imaginary components + ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128 = 15, // complex with float64 real and imaginary components + ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16 = 16, // Non-IEEE floating-point format based on IEEE754 single-precision + ONNX_TENSOR_ELEMENT_DATA_TYPE_MAX = 17 +} OnnxRuntimeTensorElementDataType; + +/** + * The retured value should be released by calling ONNXRuntimeReleaseObject + */ +ONNXRUNTIME_API(struct ONNXRuntimeTensorTypeAndShapeInfo*, ONNXRuntimeCreateTensorTypeAndShapeInfo); + +ONNXRUNTIME_API_STATUS(ONNXRuntimeSetTensorElementType, _In_ struct ONNXRuntimeTensorTypeAndShapeInfo*, enum OnnxRuntimeTensorElementDataType type); + +/** + * \param info Created from ONNXRuntimeCreateTensorTypeAndShapeInfo() function + * \param dim_values An array with length of `dim_count`. Its elements can contain negative values. + * \param dim_count length of dim_values + */ +ONNXRUNTIME_API_STATUS(ONNXRuntimeSetDims, struct ONNXRuntimeTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count); + +ONNXRUNTIME_API(enum OnnxRuntimeTensorElementDataType, ONNXRuntimeGetTensorElementType, _In_ const struct ONNXRuntimeTensorTypeAndShapeInfo*); +ONNXRUNTIME_API(size_t, ONNXRuntimeGetNumOfDimensions, _In_ const struct ONNXRuntimeTensorTypeAndShapeInfo* info); +ONNXRUNTIME_API(void, ONNXRuntimeGetDimensions, _In_ const struct ONNXRuntimeTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, size_t dim_values_length); + +/** + * How many elements does this tensor have. + * May return a negative value + * e.g. + * [] -> 1 + * [1,3,4] -> 12 + * [2,0,4] -> 0 + * [-1,3,4] -> -1 + */ +ONNXRUNTIME_API(int64_t, ONNXRuntimeGetTensorShapeElementCount, _In_ const struct ONNXRuntimeTensorTypeAndShapeInfo* info); +#ifdef __cplusplus +} +#endif diff --git a/onnxruntime/ReformatSource.ps1 b/onnxruntime/ReformatSource.ps1 new file mode 100644 index 0000000000000..f1680d423058d --- /dev/null +++ b/onnxruntime/ReformatSource.ps1 @@ -0,0 +1,12 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# Simply finds all .h and .cc files from the current directory and below, and runs clang-format in-place on them. +# Assumes clang-format is in the path, which means you will need to install clang using a Windows snapshot build from https://llvm.org/builds/ +# Requires a .clang-format config file to be in the current directory or a parent directory from where the script is run. +# Expected usage is to run it from its current location so that source in 'core' and 'test' is updated. + +gci -Recurse -Include *.h, *.cc | foreach { + Write-Host "Updating " $_.FullName + clang-format -i $_ +} \ No newline at end of file diff --git a/onnxruntime/ReformatSourcePython.bat b/onnxruntime/ReformatSourcePython.bat new file mode 100644 index 0000000000000..97bc68a14a07e --- /dev/null +++ b/onnxruntime/ReformatSourcePython.bat @@ -0,0 +1,4 @@ +:: Copyright (c) Microsoft Corporation. All rights reserved. +:: Licensed under the MIT License. + +autopep8 --in-place -r python \ No newline at end of file diff --git a/onnxruntime/VSCodeCoverage.runsettings b/onnxruntime/VSCodeCoverage.runsettings new file mode 100644 index 0000000000000..d26c04d90ee4c --- /dev/null +++ b/onnxruntime/VSCodeCoverage.runsettings @@ -0,0 +1,61 @@ + + + + + + .\TestResults + + + x64 + + + Framework45 + + + + + + + 60 + + + + + + + + + + + + + + + ^onnxruntime.* + + + ^onnxruntime\:\:test.* + ^onnxruntime\:\:MakeString<.* + ^onnxruntime\:\:MakeStringInternal<.* + + + + + + + + + + \ No newline at end of file diff --git a/onnxruntime/__init__.py b/onnxruntime/__init__.py new file mode 100644 index 0000000000000..e7f265f086a10 --- /dev/null +++ b/onnxruntime/__init__.py @@ -0,0 +1,21 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +#-------------------------------------------------------------------------- +""" +ONNX Runtime +enables high-performance evaluation of trained machine learning (ML) +models while keeping resource usage low. +Building on Microsoft's dedication to the +`Open Neural Network Exchange (ONNX) `_ +community, it supports traditional ML models as well +as Deep Learning algorithms in the +`ONNX-ML format `_. +""" +__version__ = "0.1.4" +__author__ = "Microsoft" + +from onnxruntime.capi import onnxruntime_validation +onnxruntime_validation.check_distro_info() +from onnxruntime.capi.session import InferenceSession +from onnxruntime.capi._pybind_state import RunOptions, SessionOptions, get_device, NodeArg, ModelMetadata diff --git a/onnxruntime/contrib_ops/contrib_ops.cc b/onnxruntime/contrib_ops/contrib_ops.cc new file mode 100644 index 0000000000000..7d0cb9cba3229 --- /dev/null +++ b/onnxruntime/contrib_ops/contrib_ops.cc @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "contrib_ops/contrib_ops.h" + +#include "core/graph/constants.h" +#include "core/graph/op.h" +#include "onnx/defs/schema.h" + +#include "./cpu/attnlstm/attn_lstm_schema_defs.h" + +namespace onnxruntime { +namespace contrib { +using ::ONNX_NAMESPACE::AttributeProto; +using ::ONNX_NAMESPACE::OpSchema; +using ::ONNX_NAMESPACE::OPTIONAL; + +void RegisterContribSchemas() { + ONNX_CONTRIB_OPERATOR_SCHEMA(SampleOp) + .SetDomain(kMSDomain) + .SinceVersion(1) + .Input(0, "X", "input", "T") + .Output(0, "Y", "output", "T") + .TypeConstraint( + "T", + ONNX_NAMESPACE::OpSchema::numeric_types_for_math_reduction(), + "Constrain to any tensor type. If the dtype attribute is not provided this must be a valid output type.") + .TypeAndShapeInferenceFunction(ONNX_NAMESPACE::propagateShapeAndTypeFromFirstInput) + .SetDoc(R"DOC( +Sample echo operator.)DOC"); + + // register schemas for more operators here + + ONNX_CONTRIB_OPERATOR_SCHEMA(ExpandDims) + .SetDomain(kMSDomain) + .SinceVersion(1) + .Input(0, "X", "input", "T") + .Input(1, "axis", "Specified axis to insert a dimension", "tensor(int32)") + .Output(0, "Y", "output", "T") + .TypeConstraint( + "T", + ONNX_NAMESPACE::OpSchema::all_tensor_types(), + "Constrain to any tensor type. If the dtype attribute is not provided this must be a valid output type.") + .TypeAndShapeInferenceFunction(ONNX_NAMESPACE::propagateShapeAndTypeFromFirstInput) + .SetDoc(R"DOC(ExpandDims echo operator.)DOC"); + + ONNX_CONTRIB_OPERATOR_SCHEMA_ELSEWHERE(AttnLSTM, RegisterAttnLSTMContribOpSchema); + + ONNX_CONTRIB_OPERATOR_SCHEMA(IsNaN) + .SetDomain(kMSDomain) + .SinceVersion(1) + .Input(0, "X", "input", "T1") + .Output(0, "Y", "output", "T2") + .TypeConstraint( + "T1", + ONNX_NAMESPACE::OpSchema::numeric_types_for_math_reduction(), + "Constrain to any numeric tensor type. If the dtype attribute is not provided this must be a valid output type.") + .TypeConstraint( + "T2", + {"tensor(bool)"}, + "Constrain outputs to boolean tensor") + .TypeAndShapeInferenceFunction(ONNX_NAMESPACE::propagateShapeAndTypeFromFirstInput) + .SetDoc(R"DOC(Returns which elements of the input are NaN.)DOC"); +} + +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, SampleOp); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, ExpandDims); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, AttnLSTM); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, IsNaN); + +void RegisterContribKernels(std::function fn) { + fn(BuildKernel()); + + // add more kernels here + + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); +} +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/contrib_ops.h b/onnxruntime/contrib_ops/contrib_ops.h new file mode 100644 index 0000000000000..2acd2ecfaf817 --- /dev/null +++ b/onnxruntime/contrib_ops/contrib_ops.h @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/framework/op_kernel.h" +#include "core/framework/kernel_registry.h" + +namespace onnxruntime { +namespace contrib { +#define ONNX_CONTRIB_OPERATOR_SCHEMA(name) \ + ONNX_CONTRIB_OPERATOR_SCHEMA_UNIQ_HELPER(__COUNTER__, name) +#define ONNX_CONTRIB_OPERATOR_SCHEMA_UNIQ_HELPER(Counter, name) \ + ONNX_CONTRIB_OPERATOR_SCHEMA_UNIQ(Counter, name) +#define ONNX_CONTRIB_OPERATOR_SCHEMA_UNIQ(Counter, name) \ + static ONNX_NAMESPACE::OpSchemaRegistry::OpSchemaRegisterOnce( \ + op_schema_register_once##name##Counter) ONNX_UNUSED = \ + ONNX_NAMESPACE::OpSchema(#name, __FILE__, __LINE__) + +#define ONNX_CONTRIB_OPERATOR_SCHEMA_ELSEWHERE(name, schema_func) \ + ONNX_CONTRIB_OPERATOR_SCHEMA_UNIQ_HELPER_ELSEWHERE(__COUNTER__, name, schema_func) +#define ONNX_CONTRIB_OPERATOR_SCHEMA_UNIQ_HELPER_ELSEWHERE(Counter, name, schema_func) \ + ONNX_CONTRIB_OPERATOR_SCHEMA_UNIQ_ELSEWHERE(Counter, name, schema_func) +#define ONNX_CONTRIB_OPERATOR_SCHEMA_UNIQ_ELSEWHERE(Counter, name, schema_func) \ + static ONNX_NAMESPACE::OpSchemaRegistry::OpSchemaRegisterOnce( \ + op_schema_register_once##name##Counter) ONNX_UNUSED = \ + schema_func(ONNX_NAMESPACE::OpSchema(#name, __FILE__, __LINE__)) + +void RegisterContribSchemas(); +void RegisterContribKernels(std::function create_fn); +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/attnlstm/activation_info.h b/onnxruntime/contrib_ops/cpu/attnlstm/activation_info.h new file mode 100644 index 0000000000000..b3ae1e445dbd7 --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/attnlstm/activation_info.h @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +namespace onnxruntime { +namespace contrib { +namespace detail { + +// Helper struct for an activation function call information +template +struct ActivationInfo { + TFunc func; + float alpha; + float beta; +}; + +} // namespace detail +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/attnlstm/attention_mechanism.h b/onnxruntime/contrib_ops/cpu/attnlstm/attention_mechanism.h new file mode 100644 index 0000000000000..c648536f997c5 --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/attnlstm/attention_mechanism.h @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +namespace onnxruntime { +namespace contrib { + +template +class IAttentionMechanism { + public: + virtual ~IAttentionMechanism() = default; + + virtual void PrepareMemory( + const gsl::span& memory, + const gsl::span& memory_sequence_lengths) = 0; + + virtual void Compute( + const gsl::span& query, + const gsl::span& prev_alignment, + const gsl::span& output, + const gsl::span& alignment) const = 0; + + virtual const gsl::span Values() const = 0; + + virtual const gsl::span Keys() const = 0; + + virtual int GetMaxMemorySteps() const = 0; + + virtual bool NeedPrevAlignment() const = 0; +}; + +} +} diff --git a/onnxruntime/contrib_ops/cpu/attnlstm/attention_wrapper.cc b/onnxruntime/contrib_ops/cpu/attnlstm/attention_wrapper.cc new file mode 100644 index 0000000000000..4555713a59fe1 --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/attnlstm/attention_wrapper.cc @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "attention_wrapper.h" +#include "core/providers/cpu/rnn/rnn_helpers.h" + +#include +#include + +using onnxruntime::rnn::detail::Allocate; + +namespace onnxruntime { +namespace contrib { + +template +AttentionWrapper::AttentionWrapper(AllocatorPtr alloc, const logging::Logger& logger, + int batch_size, int attn_context_depth, int attn_layer_depth, + int inner_cell_hidden_size, bool has_attn_layer, + const IAttentionMechanism& attention_mechanism) + : allocator_(alloc), + logger_(logger), + batch_size_(batch_size), + attn_context_depth_(attn_context_depth), + attn_layer_depth_(attn_layer_depth), + inner_cell_hidden_size_(inner_cell_hidden_size), + has_attn_layer_(has_attn_layer), + attention_mechanism_(attention_mechanism) { + auto mem_max_steps = attention_mechanism_.GetMaxMemorySteps(); + prev_alignments_ = Allocate(allocator_, batch_size_ * mem_max_steps, prev_alignments_ptr_, true); + alignments_ = Allocate(allocator_, batch_size_ * mem_max_steps, alignments_ptr_, true); + attn_context_ = Allocate(allocator_, batch_size_ * attn_context_depth, attn_context_ptr_, true); + attn_states_ = (has_attn_layer_) ? Allocate(allocator_, batch_size_ * attn_layer_depth_, attn_states_ptr_, true) : attn_context_; +} + +// rnn_cell_output is of [batch_size, rnn_cell_hidden_size] +template +void AttentionWrapper::ProcessOutput(const gsl::span& rnn_cell_output) { + if (has_attn_layer_) { + // rnn_cell_output * cell_weights, (part of the attention layer above the attention mechanism). + math::GemmEx(CblasNoTrans, CblasNoTrans, + batch_size_, attn_layer_depth_, inner_cell_hidden_size_, T{1.0}, + rnn_cell_output.data(), inner_cell_hidden_size_, + attn_layer_cell_weights_.data(), attn_layer_depth_, T{0.0}, + attn_states_.data(), attn_layer_depth_, &CPUMathUtil::Instance()); + } + + // Get the context which is calculated within attention mechanism. + attention_mechanism_.Compute(rnn_cell_output, prev_alignments_, attn_context_, alignments_); + if (attention_mechanism_.NeedPrevAlignment()) { + std::copy(alignments_.cbegin(), alignments_.cend(), prev_alignments_.begin()); + } + + if (has_attn_layer_) { + //concat([p_cell_output, context]) * stack([attn_layer_cell_weights_, attn_layer_attn_weights_]) = + // p_cell_output * attn_layer_cell_weights_ + context * attn_layer_attn_weights_ + // The first part is calulated above. Here just add the later. + math::GemmEx(CblasNoTrans, CblasNoTrans, + batch_size_, attn_layer_depth_, attn_context_depth_, T{1.0}, + attn_context_.data(), attn_context_depth_, + attn_layer_attn_weights_.data(), attn_layer_depth_, T{1.0}, + attn_states_.data(), attn_layer_depth_, &CPUMathUtil::Instance()); + } +} + +template +gsl::span AttentionWrapper::GetAttnStates() const { + return attn_states_; +} + +template +void AttentionWrapper::SetWeights(const gsl::span& wrapper_weights) { + has_attn_layer_ = !wrapper_weights.empty(); + + if (has_attn_layer_) { + //cell weight size and attn weight size in the attn layer + size_t cws = inner_cell_hidden_size_ * attn_layer_depth_; + size_t aws = attn_context_depth_ * attn_layer_depth_; + attn_layer_cell_weights_ = wrapper_weights.subspan(0, cws); + attn_layer_attn_weights_ = wrapper_weights.subspan(cws, aws); + } +} + +template class AttentionWrapper; + +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/attnlstm/attention_wrapper.h b/onnxruntime/contrib_ops/cpu/attnlstm/attention_wrapper.h new file mode 100644 index 0000000000000..2469a7b99a3fb --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/attnlstm/attention_wrapper.h @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "attention_mechanism.h" + +#include "core/common/common.h" +#include "core/common/logging/logging.h" +#include "core/framework/allocator.h" + +namespace onnxruntime { +namespace contrib { + +template +class AttentionWrapper { + public: + AttentionWrapper(AllocatorPtr allocator, + const logging::Logger& logger, + int batch_size, + int attn_context_depth, // it is also the depth of the memory + int attn_layer_depth, + int inner_cell_hidden_size, + bool has_attn_layer, + const IAttentionMechanism& attention_mechanism); + + virtual ~AttentionWrapper() = default; + + // Calculation based on output of the inner wrapped rnn_cell. + void ProcessOutput(const gsl::span& rnn_cell_state); + + gsl::span GetAttnStates() const; + + void SetWeights(const gsl::span& wrapper_weights); + + // the size after attention layer or direct from attention context. + int GetAttentionSize() const { + return has_attn_layer_ ? attn_layer_depth_ : attn_context_depth_; + } + + int GetAttentionContextSize() const { + return attn_context_depth_; + } + + private: + AllocatorPtr allocator_; + const logging::Logger& logger_; + + gsl::span attn_layer_cell_weights_; + gsl::span attn_layer_attn_weights_; + + IAllocatorUniquePtr attn_context_ptr_; + gsl::span attn_context_; + + IAllocatorUniquePtr attn_states_ptr_; + gsl::span attn_states_; + + IAllocatorUniquePtr prev_alignments_ptr_; + gsl::span prev_alignments_; + + IAllocatorUniquePtr alignments_ptr_; + gsl::span alignments_; + + int batch_size_; + int attn_context_depth_; + int attn_layer_depth_; + int inner_cell_hidden_size_; + + bool has_attn_layer_; + + const IAttentionMechanism& attention_mechanism_; +}; + +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/attnlstm/attn_lstm_schema_defs.cc b/onnxruntime/contrib_ops/cpu/attnlstm/attn_lstm_schema_defs.cc new file mode 100644 index 0000000000000..246b4da822b1e --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/attnlstm/attn_lstm_schema_defs.cc @@ -0,0 +1,335 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "attn_lstm_schema_defs.h" + +#include "core/graph/constants.h" +#include "core/graph/op.h" + +namespace onnxruntime { +namespace contrib { + +using ::ONNX_NAMESPACE::AttributeProto; +using ::ONNX_NAMESPACE::OPTIONAL; +using ::ONNX_NAMESPACE::OpSchema; + +// This Doc based on LSTM_ver7, and modification +static const char* AttnLSTM_ver1_doc = R"DOC( +Computes an one-layer RNN where its RNN Cell is an AttentionWrapper wrapped a LSTM Cell. The RNN layer +contains following basic component: LSTM Cell, Bahdanau Attention Mechanism, AttentionWrapp. + +Activation functions: + + Relu(x) - max(0, x) + + Tanh(x) - (1 - e^{-2x})/(1 + e^{-2x}) + + Sigmoid(x) - 1/(1 + e^{-x}) + + (NOTE: Below are optional) + + Affine(x) - alpha*x + beta + + LeakyRelu(x) - x if x >= 0 else alpha * x + + ThresholdedRelu(x) - x if x >= alpha else 0 + + ScaledTanh(x) - alpha*Tanh(beta*x) + + HardSigmoid(x) - min(max(alpha*x + beta, 0), 1) + + Elu(x) - x if x >= 0 else alpha*(e^x - 1) + + Softsign(x) - x/(1 + |x|) + + Softplus(x) - log(1 + e^x) + + Softmax(x) - exp(x) / sum(exp(x)) + +Bahdanau Attention Mechanism: + `M` - Memory tensor. + + `VALUES` - masked Memory by its real sequence length. + + `MW` - Memory layer weight. + + `KEYS` - Processed memory tensor by the memory layer. + KEYS = M * MW + + `Query` - Query tensor, normally at specific time step in sequence. + + `QW` - Query layer weight in the attention mechanism + + `PQ` - processed query, = `Query` * `QW` + + `V' - attention vector + + `ALIGN` - calculated alignment based on Query and KEYS + ALIGN = softmax(reduce_sum(`V` * Tanh(`KEYS` + `PQ`))) + + `CONTEXT` - context based on `ALIGN` and `VALUES` + CONTEXT = `ALIGN` * `VALUES` + + +LSTM Cell: + `X` - input tensor concat with attention state in the attention wrapper + + `i` - input gate + + `o` - output gate + + `f` - forget gate + + `c` - cell gate + + `t` - time step (t-1 means previous time step) + + `W[iofc]` - W parameter weight matrix for input, output, forget, and cell gates + + `R[iofc]` - R recurrence weight matrix for input, output, forget, and cell gates + + `Wb[iofc]` - W bias vectors for input, output, forget, and cell gates + + `Rb[iofc]` - R bias vectors for input, output, forget, and cell gates + + `P[iof]` - P peephole weight vector for input, output, and forget gates + + `WB[iofc]` - W parameter weight matrix for backward input, output, forget, and cell gates + + `RB[iofc]` - R recurrence weight matrix for backward input, output, forget, and cell gates + + `WBb[iofc]` - W bias vectors for backward input, output, forget, and cell gates + + `RBb[iofc]` - R bias vectors for backward input, output, forget, and cell gates + + `PB[iof]` - P peephole weight vector for backward input, output, and forget gates + + `H` - Hidden state + + `num_directions` - 2 if direction == bidirectional else 1 + + Equations (Default: f=Sigmoid, g=Tanh, h=Tanh): + + - it = f(Xt*(Wi^T) + Ht-1*(Ri^T) + Pi (.) Ct-1 + Wbi + Rbi) + + - ft = f(Xt*(Wf^T) + Ht-1*(Rf^T) + Pf (.) Ct-1 + Wbf + Rbf) + + - ct = g(Xt*(Wc^T) + Ht-1*(Rc^T) + Wbc + Rbc) + + - Ct = ft (.) Ct-1 + it (.) ct + + - ot = f(Xt*(Wo^T) + Ht-1*(Ro^T) + Po (.) Ct + Wbo + Rbo) + + - Ht = ot (.) h(Ct) + + +AttentionWrapp Notations: + `lstm()' - wrapped inner cell. + Ht, Ct = lstm(concat(Xt, ATTNt-1), Ct-1) + + `am()` - attention mechanism the wrapper used. + CONTEXTt, ALIGNt = am(Ht, ALIGNt-1) + + `AW` - attention layer weights, optional. + + `ATTN` - attention state, initial is zero. If `AW` provided, it is the output of the attention layer, + ATTNt = concat(Ht, CONTEXTt) * AW + otherwise, + ATTNt = CONTEXTt + +RNN layer output: + `Y` - if needed is the sequence of Ht from lstm cell. + + `Y_h` - is the last valid H from lstm cell. + + `Y_c` - is the last valid C from lstm cell. + +)DOC"; + + +OpSchema& RegisterAttnLSTMContribOpSchema(OpSchema&& op_schema){ + return op_schema + .SetDomain(kMSDomain) + .Attr( + "activations", + "A list of 3 (or 6 if bidirectional) activation functions " + "for input, output, forget, cell, and hidden. The activation functions must " + "be one of the activation functions specified above. Optional: See the equations " + "for default if not specified.", + AttributeProto::STRINGS, + OPTIONAL) + .Attr( + "activation_alpha", + "Optional scaling values used by some activation functions. The values are consumed " + "in the order of activation functions, for example (f, g, h) in LSTM. Default values " + "are the same as of corresponding ONNX operators.For example with LeakyRelu, the " + "default alpha is 0.01.", + AttributeProto::FLOATS, + OPTIONAL) + .Attr( + "activation_beta", + "Optional scaling values used by some activation functions. The values are consumed in " + "the order of activation functions, for example (f, g, h) in LSTM. Default values are " + "the same as of corresponding ONNX operators.", + AttributeProto::FLOATS, + OPTIONAL) + .Attr( + "clip", + "Cell clip threshold. Clipping bounds the elements of a tensor in the range of " + "[-threshold, +threshold] and is applied to the input of activations. No clip if not " + "specified.", + AttributeProto::FLOAT, + OPTIONAL) + .Attr( + "input_forget", + "Couple the input and forget gates if 1, default 0.", + AttributeProto::INT, + static_cast(0)) + .Attr( + "hidden_size", + "Number of neurons in the hidden layer.", + AttributeProto::INT, + OPTIONAL) + .Attr( + "direction", + "Specify if the RNN is forward, reverse, or bidirectional. Must be one of " + "forward (default), reverse, or bidirectional.", + AttributeProto::STRING, + std::string("forward")) + .TypeConstraint( + "T", + {"tensor(float)", "tensor(double)"}, + "Constrain input and output types to float tensors.") + .TypeConstraint( + "T1", + {"tensor(int32)"}, + "Constrain seq_lens to integral tensors.") + .Input( + 0, + "X", + "The input sequences packed (and potentially padded) into one 3-D tensor " + "with the shape of `[seq_length, batch_size, input_size]`", + "T") + .Input( + 1, + "W", + "The weight tensor for the gates. Concatenation of `W[iofc]` and " + "`WB[iofc]` (if bidirectional) along dimension 0. The tensor has shape " + "`[num_directions, 4*hidden_size, input_size]`.", + "T") + .Input( + 2, + "R", + "The recurrence weight tensor. Concatenation of `R[iofc]` and " + "`RB[iofc]` (if bidirectional) along dimension 0. This tensor has shape " + "`[num_directions, 4*hidden_size, hidden_size]`.", + "T") + .Input( + 3, + "B", + "The bias tensor for input gate. Concatenation of `[Wb[iofc], Rb[iofc]]`, " + "and `[WBb[iofc], RBb[iofc]]` (if bidirectional) along dimension 0. This " + "tensor has shape `[num_directions, 8*hidden_size]`. Optional: If not " + "specified - assumed to be 0.", + "T", + OpSchema::Optional) + .Input( + 4, + "sequence_lens", + "Optional tensor specifying lengths of the sequences in a batch. If not " + "specified - assumed all sequences in the batch to have length `seq_length`. " + "It has shape `[batch_size]` ", + "T1", + OpSchema::Optional) + .Input( + 5, + "initial_h", + "Optional initial value of the hidden. If not specified - assumed to be 0. " + "It has shape `[num_directions, batch_size, hidden_size]`.", + "T", + OpSchema::Optional) + .Input( + 6, + "initial_c", + "Optional initial value of the cell. If not specified - assumed " + "to be 0. It has shape `[num_directions, batch_size, hidden_size]`.", + "T", + OpSchema::Optional) + .Input( + 7, + "P", + "The weight tensor for peepholes. Concatenation of `P[iof]` and " + "`PB[iof]` (if bidirectional) along dimension 0. It has shape " + "`[num_directions, 3*hidde_size]`. Optional: If not specified - " + "assumed to be 0.", + "T", + OpSchema::Optional) + .Input( + 8, + "QW", + "The weight tensor of the query layer in the attention mechanism. Should be of " + "shape `[num_directions, am_query_depth(hidden_size of lstm), am_attn_size]` ", + "T", + OpSchema::Optional) + .Input( + 9, + "MW", + "The weight tensor of the memory layer in the attention mechanism. Should be of " + "shape `[num_directions, memory_depth, am_attn_size]` ", + "T", + OpSchema::Optional) + .Input( + 10, + "V", + "The attention_v tensor in the attention mechanism. Should be of shape " + "`[num_directions, am_attn_size]` ", + "T", + OpSchema::Optional) + .Input( + 11, + "M", + "The sequence of the memory (input) for attention mechanism. Should be of " + "`[batch_size, max_memory_step, memory_depth]` ", + "T", + OpSchema::Optional) + .Input( + 12, + "memory_seq_lens", + "The sequence length of the input memory for the attention mechanism. Should be of " + "`[batch_size]` ", + "T1", + OpSchema::Optional) + .Input( + 13, + "AW", + "The weights of attention layer in the attention wrapper. If exists, should be of " + "shape `[num_directions, memory_depth+hidden_size, aw_attn_size]. Please note that " + "attention mechanism context depth is also memory_depth in the attention mechanism.` ", + "T", + OpSchema::Optional) + .Output( + 0, + "Y", + "A tensor that concats all the intermediate output values of the hidden. " + "It has shape `[seq_length, num_directions, batch_size, hidden_size]`", + "T", + OpSchema::Optional) + .Output( + 1, + "Y_h", + "The last output value of the hidden. It has shape `[num_directions, " + "batch_size, hidden_size]`. ", + "T", + OpSchema::Optional) + .Output( + 2, + "Y_c", + "The last output value of the cell. It has shape " + "`[num_directions, batch_size, hidden_size]`.", + "T", + OpSchema::Optional) + .SetDoc(AttnLSTM_ver1_doc); +} + +} +} diff --git a/onnxruntime/contrib_ops/cpu/attnlstm/attn_lstm_schema_defs.h b/onnxruntime/contrib_ops/cpu/attnlstm/attn_lstm_schema_defs.h new file mode 100644 index 0000000000000..3b1cc3d4fb554 --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/attnlstm/attn_lstm_schema_defs.h @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wignored-qualifiers" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif +#include "onnx/defs/schema.h" +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + +namespace onnxruntime { +namespace contrib { + +::ONNX_NAMESPACE::OpSchema& RegisterAttnLSTMContribOpSchema(::ONNX_NAMESPACE::OpSchema&& op_schema); + +} +} diff --git a/onnxruntime/contrib_ops/cpu/attnlstm/bahdanau_attention.cc b/onnxruntime/contrib_ops/cpu/attnlstm/bahdanau_attention.cc new file mode 100644 index 0000000000000..c0ec00c6b88a7 --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/attnlstm/bahdanau_attention.cc @@ -0,0 +1,160 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "bahdanau_attention.h" +#include "core/providers/cpu/rnn/rnn_helpers.h" + +#include +#include + +using onnxruntime::rnn::detail::Allocate; + +namespace onnxruntime { +namespace contrib { + +template +BahdanauAttention::BahdanauAttention(AllocatorPtr allocator, const logging::Logger& logger, + int batch_size, int max_memory_step, int memory_depth, + int query_depth, int attn_depth, bool normalize) + : allocator_(allocator), logger_(logger), batch_size_(batch_size), max_memory_steps_(max_memory_step), memory_depth_(memory_depth), query_depth_(query_depth), attn_depth_(attn_depth), normalize_(normalize) { + values_ = Allocate(allocator_, batch_size_ * max_memory_steps_ * memory_depth_, values_ptr_, true); + keys_ = Allocate(allocator_, batch_size_ * max_memory_steps_ * attn_depth_, keys_ptr_, true); + processed_query_ = Allocate(allocator_, batch_size_ * attn_depth_, processed_query_ptr_, true); + mem_seq_lengths_ = Allocate(allocator_, batch_size_, mem_seq_lengths_ptr_, true); + + ONNXRUNTIME_ENFORCE(!normalize_, "not support normalize yet."); +} + +template +void BahdanauAttention::SetWeights( + const gsl::span& attn_weights, + const gsl::span& query_layer_weights, + const gsl::span& memory_layer_weights) { + attention_v_ = attn_weights; //[attn_depth_] + query_layer_weights_ = query_layer_weights; //[query_depth_, attn_depth_] + memory_layer_weights_ = memory_layer_weights; //[memory_depth_, attn_depth_] +} + +template +const gsl::span BahdanauAttention::Values() const { + return values_; +} + +template +const gsl::span BahdanauAttention::Keys() const { + return keys_; +} + +template +int BahdanauAttention::GetMaxMemorySteps() const { + return max_memory_steps_; +} + +template +bool BahdanauAttention::NeedPrevAlignment() const { + return false; +} + +template +void BahdanauAttention::PrepareMemory( + const gsl::span& memory, + const gsl::span& memory_sequence_lengths) { + std::copy(memory.cbegin(), memory.cend(), values_.begin()); + if (memory_sequence_lengths.empty()) { + std::fill(mem_seq_lengths_.begin(), mem_seq_lengths_.end(), max_memory_steps_); + } else { + std::copy(memory_sequence_lengths.cbegin(), memory_sequence_lengths.cend(), mem_seq_lengths_.begin()); + } + + for (int b = 0; b < batch_size_; b++) { + int mem_steps = mem_seq_lengths_[b]; + ONNXRUNTIME_ENFORCE(mem_steps <= max_memory_steps_ && mem_steps > 0, + "Real memory steps ", mem_steps, " is not in (0, ", max_memory_steps_, "]"); + } + + math::GemmEx(CblasNoTrans, CblasNoTrans, + batch_size_ * max_memory_steps_, attn_depth_, memory_depth_, T{1.0}, + memory.data(), memory_depth_, + memory_layer_weights_.data(), attn_depth_, T{0.0}, + keys_.data(), attn_depth_, &CPUMathUtil::Instance()); +} + +template +static void SoftmaxInplace(const gsl::span& alignments) { + T* x = alignments.data(); + size_t len = alignments.size(); + + double sum = 0.0; + + for (size_t i = 0; i < len; i++) { + T e = exp(x[i]); + sum += e; + x[i] = e; + } + + if (sum == 0.0) { + for (size_t i = 0; i < len; i++) { + x[i] = static_cast(1.0 / len); + } + } else { + for (size_t i = 0; i < len; i++) { + x[i] = static_cast(x[i] / sum); + } + } +} + +/** + * Args: + * queries: Tensor, shape `[batch_size_, query_depth_]` to compare to keys. + * keys_: Processed memory, shape `[batch_size_, max_memory_step_, attn_depth_]`. + */ +template +void BahdanauAttention::Compute( + const gsl::span& queries, + const gsl::span&, // Not used by bahdanau attention + const gsl::span& output, + const gsl::span& aligns) const { + //process query in dense query layer without bias + math::GemmEx(CblasNoTrans, CblasNoTrans, + batch_size_, attn_depth_, query_depth_, T{1.0}, + queries.data(), query_depth_, + query_layer_weights_.data(), attn_depth_, T{0.0}, + processed_query_.data(), attn_depth_, &CPUMathUtil::Instance()); + + std::fill(aligns.begin(), aligns.end(), T{}); + + // return math_ops.reduce_sum(v * math_ops.tanh(keys + processed_query), [2]) + for (int b = 0; b < batch_size_; b++) { + T* alignments = aligns.data() + b * max_memory_steps_; + const T* keys = keys_.data() + b * max_memory_steps_ * attn_depth_; + const T* query = processed_query_.data() + b * attn_depth_; + + int mem_steps = mem_seq_lengths_[b]; + for (int step = 0; step < mem_steps; step++) { + const T* keys_on_step = keys + step * attn_depth_; + T* dest = alignments + step; + + // reduce_sum(v * tanh(keys[step] + query)) on last dimension + *dest = T(0.0); + for (int i = 0; i < attn_depth_; i++) { + *dest += (attention_v_[i] * tanh(keys_on_step[i] + query[i])); + } + } + + SoftmaxInplace(gsl::span{alignments, mem_steps}); + + // Calculate the context + auto outspan = output.subspan(b * memory_depth_); + auto values = values_.subspan(b * max_memory_steps_ * memory_depth_); + math::GemmEx(CblasNoTrans, CblasNoTrans, + 1, memory_depth_, max_memory_steps_, T{1.0}, + alignments, max_memory_steps_, + values.data(), memory_depth_, T{0.0}, + outspan.data(), memory_depth_, &CPUMathUtil::Instance()); + } +} + +template class BahdanauAttention; + +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/attnlstm/bahdanau_attention.h b/onnxruntime/contrib_ops/cpu/attnlstm/bahdanau_attention.h new file mode 100644 index 0000000000000..755af6ba6d5c3 --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/attnlstm/bahdanau_attention.h @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/framework/allocator.h" +#include "core/providers/cpu/rnn/rnn_helpers.h" + +#include "attention_mechanism.h" + +namespace onnxruntime { +namespace contrib { + +// Please refer to: https://arxiv.org/pdf/1409.0473.pdf +template +class BahdanauAttention : public IAttentionMechanism { + public: + BahdanauAttention( + AllocatorPtr allocator, + const logging::Logger& logger, + int batch_size, + int max_memory_step, + int memory_depth, + int query_depth, + int attn_depth, + bool normalize); + + void SetWeights( + const gsl::span& attn_weights, + const gsl::span& query_layer_weights, + const gsl::span& memory_layer_weights); + + ~BahdanauAttention() override = default; + + void PrepareMemory( + const gsl::span& memory, + const gsl::span& memory_sequence_lengths) override; + + void Compute( + const gsl::span& queries, + const gsl::span& prev_alignment, + const gsl::span& output, + const gsl::span& alignment) const override; + + const gsl::span Values() const override; + + const gsl::span Keys() const override; + + int GetMaxMemorySteps() const override; + + bool NeedPrevAlignment() const override; + + private: + AllocatorPtr allocator_; + const logging::Logger& logger_; + + int batch_size_; + int max_memory_steps_; + int memory_depth_; + int query_depth_; + int attn_depth_; + + gsl::span attention_v_; + gsl::span query_layer_weights_; + gsl::span memory_layer_weights_; + + IAllocatorUniquePtr keys_ptr_; + gsl::span keys_; + + IAllocatorUniquePtr values_ptr_; + gsl::span values_; + + IAllocatorUniquePtr processed_query_ptr_; + gsl::span processed_query_; + + IAllocatorUniquePtr mem_seq_lengths_ptr_; + gsl::span mem_seq_lengths_; + + bool normalize_; +}; + +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/attnlstm/deep_cpu_attn_lstm.cc b/onnxruntime/contrib_ops/cpu/attnlstm/deep_cpu_attn_lstm.cc new file mode 100644 index 0000000000000..fea43a0db2c35 --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/attnlstm/deep_cpu_attn_lstm.cc @@ -0,0 +1,507 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "deep_cpu_attn_lstm.h" +#include "activation_info.h" +#include "bahdanau_attention.h" +#include "uni_dir_attn_lstm.h" + +#include "core/common/common.h" +#include "core/common/logging/logging.h" +#include "core/common/task_thread_pool.h" +#include "core/framework/allocator.h" + +namespace onnxruntime { +namespace contrib { + +using ::onnxruntime::contrib::rnn::detail::UniDirectionalAttnLstm; +using ::onnxruntime::rnn::detail::Allocate; + +extern template class BahdanauAttention; + +/* AttnLSTM operator */ +ONNX_OPERATOR_KERNEL_EX( + AttnLSTM, //name + kMSDomain, + 1, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T", {DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}) + .TypeConstraint("T1", DataTypeImpl::GetTensorType()), + DeepCpuAttnLstmOp); + +Status +DeepCpuAttnLstmOp::Compute(OpKernelContext* context) const { + const Tensor& X = *context->Input(0); // inputs. [seq_length, batch_size, input_size] + + Status status; + // auto& logger = context->Logger(); + + auto data_type = X.DataType(); + if (data_type == DataTypeImpl::GetType()) + status = ComputeImpl(*context); + else if (data_type == DataTypeImpl::GetType()) { + /* Need to update all the helpers to support double... + status = ComputeImpl(*context); */ + ONNXRUNTIME_NOT_IMPLEMENTED("LSTM operator does not support double yet"); + } else + ONNXRUNTIME_THROW("Invalid data type for LSTM operator of ", data_type); + + return status; +} + +// #define DUMP_MATRIXES to provide lots of diagnostic output +#if defined(DUMP_MATRIXES) +#define DumpMatrix(...) ::onnxruntime::rnn::detail::DumpMatrixImpl(__VA_ARGS__) +#else +#define DumpMatrix(...) ((void)0) +#endif + +template +static gsl::span FirstHalfSpan(const gsl::span& dspan) { + auto sz = dspan.size() / 2; + return dspan.subspan(0, sz); +} + +template +static gsl::span SecondHalfSpan(const gsl::span& dspan) { + auto sz = dspan.size() / 2; + return dspan.subspan(sz); +} + +template +Status DeepCpuAttnLstmOp::ComputeImpl(OpKernelContext& context) const { + auto& logger = context.Logger(); + + // original lstm processing + const Tensor& X = *context.Input(0); // inputs. [seq_length, batch_size, input_size], input will concat with attention of previous state + const Tensor& W = *context.Input(1); // weights. [num_directions, 4*hidden_size, input_size + attention_size], + const Tensor& R = *context.Input(2); // recurrence weights. [num_directions, 4*hidden_size, hidden_size] + + // optional + const Tensor* B = context.Input(3); // bias. [num_directions, 8*hidden_size] + const Tensor* sequence_lens = context.Input(4); // [batch_size] + const Tensor* initial_h = context.Input(5); // initial hidden. [num_directions, batch_size, hidden_size] + const Tensor* initial_c = context.Input(6); // initial cell. [num_directions, batch_size, hidden_size] + const Tensor* P = context.Input(7); // peephole weights. [num_directions, 3*hidden_size] + + auto& X_shape = X.Shape(); + + int seq_length = gsl::narrow(X_shape[0]); + int batch_size = gsl::narrow(X_shape[1]); + int input_size = gsl::narrow(X_shape[2]); + + // Processing attention wrapper + const int first_attn_input = 8; + const Tensor& am_query_layer_weights = *context.Input(first_attn_input + 0); // [num_directions, query_depth(hidden_size of lstm), am_attn_size] + const Tensor& am_memory_layer_weights = *context.Input(first_attn_input + 1); // [num_directions, memory_depth, am_attn_size] + const Tensor& am_v_weights = *context.Input(first_attn_input + 2); // [num_directions, am_attn_size] + const Tensor& attn_memory = *context.Input(first_attn_input + 3); // [batch_size, max_memory_step, memory_depth_] + const Tensor* attn_memory_seq_lens = context.Input(first_attn_input + 4); // [batch_size], int value + const Tensor* attn_layer_weights = context.Input(first_attn_input + 5); // [num_directions, memory_depth+cell_hidden_size, aw_attn_size] + + Status status = ValidateInputs( + X, W, R, B, sequence_lens, initial_h, initial_c, P, batch_size, + am_query_layer_weights, am_memory_layer_weights, am_v_weights, attn_memory, attn_memory_seq_lens, attn_layer_weights); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + const int max_memory_step = gsl::narrow(attn_memory.Shape()[1]); + const int memory_depth = gsl::narrow(am_memory_layer_weights.Shape()[1]); + const int am_attn_size = gsl::narrow(am_memory_layer_weights.Shape()[2]); + const int query_depth = gsl::narrow(am_query_layer_weights.Shape()[1]); // it is equal to hidden_size + const bool has_attention_layer = attn_layer_weights != nullptr; + const int attn_layer_depth = has_attention_layer ? gsl::narrow(attn_layer_weights->Shape()[2]) : 0; + const int attention_size = has_attention_layer ? attn_layer_depth : memory_depth; + + const gsl::span attn_layer_weights_span = (has_attention_layer) ? attn_layer_weights->DataAsSpan() : gsl::span(); + const gsl::span memory_seq_lens_span = (attn_memory_seq_lens != nullptr) ? attn_memory_seq_lens->DataAsSpan() : gsl::span(); + + // LSTM outputs are optional but must be in the same order + std::vector Y_dims{seq_length, num_directions_, batch_size, hidden_size_}; + Tensor* Y = context.Output(/*index*/ 0, Y_dims); + + std::vector Y_h_dims{num_directions_, batch_size, hidden_size_}; + Tensor* Y_h = context.Output(/*index*/ 1, Y_h_dims); + + std::vector Y_c_dims{num_directions_, batch_size, hidden_size_}; + Tensor* Y_c = context.Output(/*index*/ 2, Y_c_dims); + + AllocatorPtr alloc; + status = context.GetTempSpaceAllocator(&alloc); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + gsl::span input_weights = W.DataAsSpan(); + gsl::span recurrent_weights = R.DataAsSpan(); + gsl::span bias = B != nullptr ? B->DataAsSpan() : gsl::span(); + gsl::span peephole_weights = P != nullptr ? P->DataAsSpan() : gsl::span(); + + // spans for first direction + const size_t input_weights_size_per_direction = 4 * hidden_size_ * (input_size + attention_size); + const size_t hidden_weights_size_per_direction = 4 * hidden_size_ * hidden_size_; + const size_t bias_size_per_direction = 8 * hidden_size_; + const size_t peephole_weights_size_per_direction = 3 * hidden_size_; + + gsl::span input_weights_1 = input_weights.subspan(0, input_weights_size_per_direction); + gsl::span recurrent_weights_1 = recurrent_weights.subspan(0, hidden_weights_size_per_direction); + gsl::span bias_1 = bias.empty() ? bias : bias.subspan(0, bias_size_per_direction); + gsl::span peephole_weights_1 = + peephole_weights.empty() ? peephole_weights + : peephole_weights.subspan(0, peephole_weights_size_per_direction); + + gsl::span input = X.DataAsSpan(); + gsl::span sequence_lens_span = sequence_lens != nullptr ? sequence_lens->DataAsSpan() + : gsl::span(); + + const size_t initial_hidden_size_per_direction = batch_size * hidden_size_; + gsl::span initial_hidden = initial_h != nullptr ? initial_h->DataAsSpan() : gsl::span(); + gsl::span initial_hidden_1 = + initial_hidden.empty() ? initial_hidden + : initial_hidden.subspan(0, initial_hidden_size_per_direction); + + const size_t initial_cell_size_per_direction = batch_size * hidden_size_; + gsl::span initial_cell = initial_c != nullptr ? initial_c->DataAsSpan() : gsl::span(); + gsl::span initial_cell_1 = + initial_cell.empty() ? initial_cell + : initial_cell.subspan(0, initial_cell_size_per_direction); + + // output shape is [seq_length, num_directions, batch_size, hidden_size] + // so it's not a case of all the output for one direction being first. + // due to that we can only easily check that the end of the output for each direction is valid. + const size_t output_size = Y != nullptr ? static_cast(Y->Shape().Size()) : 0; + const size_t per_direction_offset = batch_size * hidden_size_; + gsl::span output = Y != nullptr ? Y->MutableDataAsSpan() : gsl::span(); + gsl::span output_1 = + output.empty() ? output + : output.subspan(0, output_size - (num_directions_ - 1) * per_direction_offset); + + // UniDirectionalAttnLstm needs somewhere to write output, so even if we aren't returning Y_h and Y_c + // we provide an appropriately sized buffer for that purpose. + const size_t hidden_output_size_per_direction = batch_size * hidden_size_; + IAllocatorUniquePtr local_hidden_output; + gsl::span hidden_output = + Y_h ? Y_h->MutableDataAsSpan() + : Allocate(alloc, hidden_output_size_per_direction * num_directions_, local_hidden_output); + + gsl::span hidden_output_1 = hidden_output.subspan(0, hidden_output_size_per_direction); + + const size_t last_cell_size_per_direction = batch_size * hidden_size_; + IAllocatorUniquePtr local_last_cell; + gsl::span last_cell = + Y_c ? Y_c->MutableDataAsSpan() + : Allocate(alloc, last_cell_size_per_direction * num_directions_, local_last_cell); + + gsl::span last_cell_1 = last_cell.subspan(0, last_cell_size_per_direction); + + if (!output.empty() && !sequence_lens_span.empty()) { + // clear tailing outputs + int32_t max_seq_this_batch = *std::max_element(sequence_lens_span.cbegin(), sequence_lens_span.cend()); + if (max_seq_this_batch >= 0 && max_seq_this_batch < seq_length) { + auto start = max_seq_this_batch * hidden_output_size_per_direction * num_directions_; + std::fill(output.begin() + start, output.end(), T{}); + } + } + + if (direction_ == Direction::kBidirectional) { + // spans for second direction + gsl::span input_weights_2 = input_weights.subspan(input_weights_size_per_direction, + input_weights_size_per_direction); + gsl::span hidden_weights_2 = recurrent_weights.subspan(hidden_weights_size_per_direction, + hidden_weights_size_per_direction); + gsl::span bias_2 = bias.empty() ? bias : bias.subspan(bias_size_per_direction, bias_size_per_direction); + gsl::span peephole_weights_2 = + peephole_weights.empty() ? peephole_weights + : peephole_weights.subspan(peephole_weights_size_per_direction, + peephole_weights_size_per_direction); + + gsl::span initial_hidden_2 = + initial_hidden.empty() ? initial_hidden + : initial_hidden.subspan(initial_hidden_size_per_direction, + initial_hidden_size_per_direction); + gsl::span initial_cell_2 = + initial_cell.empty() ? initial_cell + : initial_cell.subspan(initial_cell_size_per_direction, + initial_cell_size_per_direction); + gsl::span output_2 = + output.empty() ? output : output.subspan(per_direction_offset, output_size - per_direction_offset); + + gsl::span hidden_output_2 = hidden_output.subspan(hidden_output_size_per_direction, + hidden_output_size_per_direction); + gsl::span last_cell_2 = last_cell.subspan(last_cell_size_per_direction, + last_cell_size_per_direction); + + auto fam = std::make_unique>( + alloc, logger, batch_size, max_memory_step, memory_depth, query_depth, am_attn_size, false); + fam->SetWeights( + FirstHalfSpan(am_v_weights.DataAsSpan()), + FirstHalfSpan(am_query_layer_weights.DataAsSpan()), + FirstHalfSpan(am_memory_layer_weights.DataAsSpan())); + fam->PrepareMemory(attn_memory.DataAsSpan(), memory_seq_lens_span); + + auto faw = std::make_unique>( + alloc, logger, batch_size, memory_depth, attn_layer_depth, hidden_size_, has_attention_layer, *fam); + faw->SetWeights(FirstHalfSpan(attn_layer_weights_span)); + + auto fw = std::make_unique>( + alloc, logger, + seq_length, batch_size, input_size, + hidden_size_, Direction::kForward, input_forget_, *faw, + bias_1, peephole_weights_1, initial_hidden_1, initial_cell_1, + activation_funcs_.Entries()[0], + activation_funcs_.Entries()[1], + activation_funcs_.Entries()[2], + clip_, ttp_); + + auto bam = std::make_unique>( + alloc, logger, batch_size, max_memory_step, memory_depth, query_depth, am_attn_size, false); + bam->SetWeights( + SecondHalfSpan(am_v_weights.DataAsSpan()), + SecondHalfSpan(am_query_layer_weights.DataAsSpan()), + SecondHalfSpan(am_memory_layer_weights.DataAsSpan())); + bam->PrepareMemory(attn_memory.DataAsSpan(), memory_seq_lens_span); + + auto baw = std::make_unique>( + alloc, logger, batch_size, memory_depth, attn_layer_depth, hidden_size_, has_attention_layer, *bam); + baw->SetWeights(SecondHalfSpan(attn_layer_weights_span)); + + auto bw = std::make_unique>( + alloc, logger, + seq_length, batch_size, input_size, + hidden_size_, Direction::kReverse, input_forget_, *baw, + bias_2, peephole_weights_2, initial_hidden_2, initial_cell_2, + activation_funcs_.Entries()[3], + activation_funcs_.Entries()[4], + activation_funcs_.Entries()[5], + clip_, ttp_); + + fw->Compute(input, sequence_lens_span, num_directions_, input_weights_1, recurrent_weights_1, output_1, hidden_output_1, last_cell_1); + bw->Compute(input, sequence_lens_span, num_directions_, input_weights_2, hidden_weights_2, output_2, hidden_output_2, last_cell_2); + + } else { + auto fam = std::make_unique>( + alloc, logger, batch_size, max_memory_step, memory_depth, query_depth, am_attn_size, false); + fam->SetWeights( + am_v_weights.DataAsSpan(), + am_query_layer_weights.DataAsSpan(), + am_memory_layer_weights.DataAsSpan()); + fam->PrepareMemory(attn_memory.DataAsSpan(), memory_seq_lens_span); + + auto faw = std::make_unique>( + alloc, logger, batch_size, memory_depth, attn_layer_depth, hidden_size_, has_attention_layer, *fam); + faw->SetWeights(attn_layer_weights_span); + + auto fw = std::make_unique>( + alloc, logger, + seq_length, batch_size, input_size, + hidden_size_, direction_, input_forget_, *faw, + bias_1, peephole_weights_1, initial_hidden_1, initial_cell_1, + activation_funcs_.Entries()[0], + activation_funcs_.Entries()[1], + activation_funcs_.Entries()[2], + clip_, ttp_); + + fw->Compute(input, sequence_lens_span, num_directions_, input_weights_1, recurrent_weights_1, output_1, hidden_output_1, last_cell_1); + } + + if (!output.empty()) { + DumpMatrix("Y", output.data(), seq_length * num_directions_ * batch_size, hidden_size_); + } + + // these always get written to regardless of whether we're returning them as optional output or not + DumpMatrix("Y_h", hidden_output.data(), num_directions_ * batch_size, hidden_size_); + DumpMatrix("Y_c", last_cell.data(), num_directions_ * batch_size, hidden_size_); + + return Status::OK(); +} + +static Status ValidateRnnInputsWithExtraInputFromState( + const Tensor& X, + const Tensor& W, + const Tensor& R, + const Tensor* B, + int WRB_dim_1_multipler, + const Tensor* sequence_lens, + const Tensor* initial_h, + int64_t num_directions, + int64_t hidden_size, + int64_t extra_input_size) { + auto& X_shape = X.Shape(); + auto& W_shape = W.Shape(); + auto& R_shape = R.Shape(); + + int64_t seq_length = X_shape[0]; + int64_t batch_size = X_shape[1]; + int64_t input_size = X_shape[2] + extra_input_size; + + if (X_shape.NumDimensions() != 3) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input X must have 3 dimensions only. Actual:", X_shape); + + if (W_shape.NumDimensions() != 3 || + W_shape[0] != num_directions || + W_shape[1] != hidden_size * WRB_dim_1_multipler || + W_shape[2] != input_size) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input W must have shape {", + num_directions, ",", WRB_dim_1_multipler, "*", hidden_size, ",", + input_size, "}. Actual:", W_shape); + + if (R_shape.NumDimensions() != 3 || + R_shape[0] != num_directions || + R_shape[1] != hidden_size * WRB_dim_1_multipler || + R_shape[2] != hidden_size) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input R must have shape {", + num_directions, ",", WRB_dim_1_multipler, "*", hidden_size, ",", + hidden_size, "}. Actual:", R_shape); + + if (B != nullptr) { + auto& B_shape = B->Shape(); + if (B_shape.NumDimensions() != 2 || + B_shape[0] != num_directions || + B_shape[1] != 2 * WRB_dim_1_multipler * hidden_size) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input B must have shape {", + num_directions, ",", 2 * WRB_dim_1_multipler, "*", hidden_size, "}. Actual:", B_shape); + } + + if (sequence_lens != nullptr) { + auto& sequence_lens_shape = sequence_lens->Shape(); + if (sequence_lens_shape.NumDimensions() != 1 || + sequence_lens_shape[0] != batch_size) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input sequence_lens must have shape {", + batch_size, "}. Actual:", sequence_lens_shape); + } + + auto sequence_len_entries = sequence_lens->DataAsSpan(); + if (std::any_of(sequence_len_entries.cbegin(), + sequence_len_entries.cend(), + [seq_length](int len) { return len <= 0 || len > seq_length; })) { + return ONNXRUNTIME_MAKE_STATUS( + ONNXRUNTIME, INVALID_ARGUMENT, + "Invalid value/s in sequence_lens. All values must be > 0 and < seq_length. seq_length=", seq_length); + } + } + + if (initial_h != nullptr) { + auto& initial_h_shape = initial_h->Shape(); + + if (initial_h_shape.NumDimensions() != 3 || + initial_h_shape[0] != num_directions || + initial_h_shape[1] != batch_size || + initial_h_shape[2] != hidden_size) + + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Input initial_h must have shape {", + num_directions, ",", batch_size, ",", hidden_size, "}. Actual:", initial_h_shape); + } + + return Status::OK(); +} // namespace detail + +Status DeepCpuAttnLstmOp::ValidateInputs( + const Tensor& X, const Tensor& W, const Tensor& R, const Tensor* B, + const Tensor* sequence_lens, const Tensor* initial_h, const Tensor* initial_c, + const Tensor* P, int batch_size, + const Tensor& am_query_layer_weights, const Tensor& am_memory_layer_weights, const Tensor& am_v_weights, + const Tensor& attn_memory, const Tensor* attn_memory_seq_lens, const Tensor* attn_layer_weights) const { + // Check memory of [batch_size, max_memory_step, memory_depth_], its sequence length of [batch_size] + auto memory_shape = attn_memory.Shape(); + if (memory_shape.NumDimensions() != 3 || memory_shape[0] != batch_size) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Attention mechanism memory shape error! Expected: {", batch_size, + "}, actural: ", memory_shape); + } + const int max_memory_step = gsl::narrow(memory_shape[1]); + const int memory_depth = gsl::narrow(memory_shape[2]); + if (attn_memory_seq_lens != nullptr) { + auto memory_seq_lens_shape = attn_memory_seq_lens->Shape(); + if (memory_seq_lens_shape.NumDimensions() != 1 || memory_seq_lens_shape[0] != batch_size) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Attention mechanism memory sequence lengths must have shape {", batch_size, + "}, actural: ", memory_seq_lens_shape); + } + const gsl::span mem_seq_lens_span = attn_memory_seq_lens->DataAsSpan(); + auto item_not_in_range = std::find_if( + mem_seq_lens_span.cbegin(), mem_seq_lens_span.cend(), + [max_memory_step](int len) { return len <= 0 || len > max_memory_step; }); + if (item_not_in_range != mem_seq_lens_span.cend()) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Attention mechanism memory sequence lengths value must in (0, ", + max_memory_step, "], while ", *item_not_in_range, " found!"); + } + } + + // Check memory layer weights of [num_directions, memory_depth, am_attn_size] + auto memory_layer_shape = am_memory_layer_weights.Shape(); + if (memory_layer_shape.NumDimensions() != 3 || + memory_layer_shape[0] != num_directions_ || + memory_layer_shape[1] != memory_depth) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Attention memory layer weight shape error! Expected:{", + num_directions_, ",", memory_depth, ", am_attn_size}, Got:", memory_layer_shape); + } + const int am_attn_size = gsl::narrow(memory_layer_shape[2]); + + // check query layer weights of [num_directions, query_depth(hidden_size of lstm), am_attn_size] + auto query_layer_shape = am_query_layer_weights.Shape(); + if (query_layer_shape.NumDimensions() != 3 || + query_layer_shape[0] != num_directions_ || + query_layer_shape[1] != hidden_size_ || + query_layer_shape[2] != am_attn_size) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Attention query layer weight shape error! Expected:{", + num_directions_, ", ", hidden_size_, ", ", am_attn_size, "}, Got: ", query_layer_shape); + } + + // check attention v for [num_directions, am_attn_size] + auto v_shape = am_v_weights.Shape(); + if (v_shape.NumDimensions() != 2 || + v_shape[0] != num_directions_ || + v_shape[1] != am_attn_size) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Attention v weight shape error! Expected:{", num_directions_, ", ", am_attn_size, + "}. Got: ", v_shape); + } + + // Check attention layer weights for [num_directions, memory_depth+cell_hidden_size, aw_attn_size] + const bool has_attention_layer = attn_layer_weights != nullptr; + int aw_attn_size = memory_depth; + if (has_attention_layer) { + auto attn_layer_shape = attn_layer_weights->Shape(); + if (attn_layer_shape.NumDimensions() != 3 || + attn_layer_shape[0] != num_directions_ || + attn_layer_shape[1] != memory_depth + hidden_size_) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Attention layer weight shape error! Expected: {", num_directions_, ", ", + memory_depth + hidden_size_, ", aw_attn_size}. Got:", attn_layer_shape); + } + aw_attn_size = gsl::narrow(attn_layer_shape[2]); + } + + auto status = ValidateRnnInputsWithExtraInputFromState( + X, W, R, B, 4, sequence_lens, initial_h, num_directions_, hidden_size_, aw_attn_size); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + if (initial_c != nullptr) { + auto& initial_c_shape = initial_c->Shape(); + + if (initial_c_shape.NumDimensions() != 3 || + initial_c_shape[0] != num_directions_ || + initial_c_shape[1] != batch_size || + initial_c_shape[2] != hidden_size_) + + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Input initial_c must have shape {", + num_directions_, ",", batch_size, ",", hidden_size_, "}. Actual:", initial_c_shape); + } + + if (P != nullptr) { + auto& p_shape = P->Shape(); + + if (p_shape.NumDimensions() != 2 || + p_shape[0] != num_directions_ || + p_shape[1] != 3 * hidden_size_) + + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Input P must have shape {", + num_directions_, ",", 3 * hidden_size_, "}. Actual:", p_shape); + } + + return Status::OK(); +} + +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/attnlstm/deep_cpu_attn_lstm.h b/onnxruntime/contrib_ops/cpu/attnlstm/deep_cpu_attn_lstm.h new file mode 100644 index 0000000000000..6705ce744ce8b --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/attnlstm/deep_cpu_attn_lstm.h @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include "attention_wrapper.h" + +#include "core/framework/op_kernel.h" +#include "core/common/task_thread_pool.h" +#include "core/providers/cpu/rnn/rnn_helpers.h" + +namespace onnxruntime { +namespace contrib { + +using onnxruntime::rnn::detail::ActivationFuncs; +using onnxruntime::rnn::detail::Direction; +using onnxruntime::rnn::detail::MakeDirection; + +// The class represents DeepCPU implementation of a long short term memory (LSTM) plus a Bahdanau Attention wraper. +// The equivilent python usage could be checked int the corresponding op test directory, attention_lstm_data_gen.py. +// Also please note that detail implementation re-used lot of code from current ONNXRuntime LSTM operator, refactor +// is needed in future if this is become part of ONNX. +class DeepCpuAttnLstmOp final : public OpKernel { + public: + DeepCpuAttnLstmOp(const OpKernelInfo& info) + : OpKernel(info), clip_(info.GetAttrOrDefault("clip", std::numeric_limits::max())) { + std::string direction; + ONNXRUNTIME_ENFORCE(info.GetAttr("direction", &direction).IsOK()); + + int64_t int64_value; + ONNXRUNTIME_ENFORCE(info.GetAttr("hidden_size", &int64_value).IsOK() && int64_value > 0); + hidden_size_ = gsl::narrow(int64_value); + + // optional attributes + std::vector activation_func_names = info.GetAttrsOrDefault("activations"); + std::vector activation_func_alphas = info.GetAttrsOrDefault("activation_alpha"); + std::vector activation_func_betas = info.GetAttrsOrDefault("activation_beta"); + ONNXRUNTIME_ENFORCE(clip_ > 0.f); + + if (info.GetAttr("input_forget", &int64_value).IsOK()) + input_forget_ = int64_value != 0; + + direction_ = MakeDirection(direction); + num_directions_ = direction_ == Direction::kBidirectional ? 2 : 1; + + if (activation_func_names.empty()) { + for (int i = 0; i < num_directions_; ++i) { + activation_func_names.emplace_back("sigmoid"); + activation_func_names.emplace_back("tanh"); + activation_func_names.emplace_back("tanh"); + } + } + + ONNXRUNTIME_ENFORCE(activation_func_names.size() == num_directions_ * 3); + + activation_funcs_ = ActivationFuncs(activation_func_names, + activation_func_alphas, + activation_func_betas); + } + + Status Compute(OpKernelContext* context) const override; + + ~DeepCpuAttnLstmOp() override = default; + + private: + template + Status ComputeImpl(OpKernelContext& context) const; + + Status ValidateInputs(const Tensor& X, + const Tensor& W, + const Tensor& R, + const Tensor* B, + const Tensor* sequence_lens, + const Tensor* initial_h, + const Tensor* initial_c, + const Tensor* P, + int batch_size, + const Tensor& am_query_layer_weights, + const Tensor& am_memory_layer_weights, + const Tensor& am_v_weights, + const Tensor& attn_memory, + const Tensor* attn_memory_seq_lens, + const Tensor* attn_layer_weights) const; + + Direction direction_; + int num_directions_; + + int hidden_size_ = 0; + float clip_; + bool input_forget_ = false; + + ActivationFuncs activation_funcs_; + + // Threadpool for operator. If concurrent Compute calls are possible, it will be shared + // across them. mutable due to this. + // The alternative would be to create a threadpool in each call to Compute but that would incur thread creation + // cost on every call. + mutable TaskThreadPool ttp_{std::thread::hardware_concurrency()}; +}; + +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/attnlstm/uni_dir_attn_lstm.cc b/onnxruntime/contrib_ops/cpu/attnlstm/uni_dir_attn_lstm.cc new file mode 100644 index 0000000000000..c9201e891305a --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/attnlstm/uni_dir_attn_lstm.cc @@ -0,0 +1,510 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// copied from gsl_algorithm, gsl disable 4996 for gsl::copy() +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4996) +#endif + +#include "uni_dir_attn_lstm.h" + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +using namespace onnxruntime::rnn::detail; + +namespace onnxruntime { +namespace contrib { +namespace rnn { +namespace detail { + +// #define DUMP_MATRIXES to provide lots of diagnostic output +#if defined(DUMP_MATRIXES) +#define DumpMatrix(...) ::onnxruntime::rnn::detail::DumpMatrixImpl(__VA_ARGS__) +#else +#define DumpMatrix(...) ((void)0) +#endif + +template +UniDirectionalAttnLstm::UniDirectionalAttnLstm(AllocatorPtr allocator, + const logging::Logger& logger, + const int seq_length, + const int batch_size, + const int input_size, + const int hidden_size, + Direction direction, + const bool input_forget, + AttentionWrapper& attention_wrapper, + const gsl::span& bias, + const gsl::span& peephole_weights, + const gsl::span& initial_hidden_state, + const gsl::span& initial_cell_state, + const ActivationFuncs::Entry& activation_func_f, + const ActivationFuncs::Entry& activation_func_g, + const ActivationFuncs::Entry& activation_func_h, + const float clip, + TaskThreadPool& ttp) + : allocator_(allocator), + logger_(logger), + seq_length_(seq_length), + batch_size_(batch_size), + input_size_(input_size), + hidden_size_(hidden_size), + direction_(direction), + input_forget_(input_forget), + clip_(clip), + use_bias_(!bias.empty()), + use_peepholes_(!peephole_weights.empty()), + attention_wrapper_(attention_wrapper), + ttp_(ttp) { + activation_f_ = {deepcpu::ActivationFuncByName(activation_func_f.name), + activation_func_f.alpha, + activation_func_f.beta}; + + activation_g_ = {deepcpu::ActivationFuncByName(activation_func_g.name), + activation_func_g.alpha, + activation_func_g.beta}; + + activation_h_ = {deepcpu::LstmMergeGatesFuncByName(activation_func_h.name), + activation_func_h.alpha, + activation_func_h.beta}; + + clip_with_bias_ptr_ = use_bias_ ? deepcpu::clip_add_bias : deepcpu::clip_ignore_bias; + + attention_size_ = attention_wrapper_.GetAttentionSize(); + attention_context_size_ = attention_wrapper_.GetAttentionContextSize(); + + SetNumThreads(); + AllocateBuffers(); + InitializeBuffers(initial_hidden_state, initial_cell_state); + + if (!peephole_weights.empty()) + LoadPeepholeWeights(peephole_weights); + if (!bias.empty()) + LoadBias(bias); +} + +template +void UniDirectionalAttnLstm::AllocateBuffers() { + // allocate and fill with 0's. + const bool fill = true; + hidden0_ = Allocate(allocator_, hidden_size_, hidden0_ptr_, fill); + internal_memory_prev_ = Allocate(allocator_, hidden_size_, internal_memory_prev_ptr_, fill); + internal_memory_cur_ = Allocate(allocator_, hidden_size_, internal_memory_cur_ptr_, fill); + batched_hidden0_ = Allocate(allocator_, batch_size_ * hidden_size_, batched_hidden0_ptr_, fill); + + batched_internal_memory_prev_ = Allocate(allocator_, batch_size_ * hidden_size_, + batched_internal_memory_prev_ptr_, fill); + batched_internal_memory_cur_ = Allocate(allocator_, batch_size_ * hidden_size_, + batched_internal_memory_cur_ptr_, fill); + batched_internal_memory_clipped_ = Allocate(allocator_, batch_size_ * hidden_size_, + batched_internal_memory_clipped_ptr_, fill); + + output_iofc_ = Allocate(allocator_, hidden_size_ * 4 * batch_size_ * seq_length_, output_iofc_ptr_, fill); + + if (use_bias_) { + bias_WRi_ = Allocate(allocator_, hidden_size_, bias_WRi_ptr_); + bias_WRf_ = Allocate(allocator_, hidden_size_, bias_WRf_ptr_); + bias_WRo_ = Allocate(allocator_, hidden_size_, bias_WRo_ptr_); + bias_WRc_ = Allocate(allocator_, hidden_size_, bias_WRc_ptr_); + } + + if (direction_ == kReverse) { + inputs_reverse_ = Allocate(allocator_, seq_length_ * batch_size_ * input_size_, inputs_reverse_ptr_); + outputs_reverse_ = Allocate(allocator_, seq_length_ * batch_size_ * hidden_size_, outputs_reverse_ptr_); + } + +#if !defined(LSTM_NO_PEEPHOLE_COPY) + if (use_peepholes_) { + peephole_i_ = Allocate(allocator_, hidden_size_, peephole_i_ptr_); + peephole_f_ = Allocate(allocator_, hidden_size_, peephole_f_ptr_); + peephole_o_ = Allocate(allocator_, hidden_size_, peephole_o_ptr_); + } +#endif +} + +template +void UniDirectionalAttnLstm::InitializeBuffers(const gsl::span& initial_hidden_state, + const gsl::span& initial_cell_state) { + if (!initial_hidden_state.empty()) { + gsl::copy(initial_hidden_state, batched_hidden0_); + } else { + std::fill_n(batched_hidden0_.data(), batched_hidden0_.size(), T{}); + } + + if (!initial_cell_state.empty()) { + gsl::copy(initial_cell_state, batched_internal_memory_prev_); + } else { + std::fill_n(batched_internal_memory_prev_.data(), batched_internal_memory_prev_.size(), T{}); + } +} + +template +void UniDirectionalAttnLstm::LoadPeepholeWeights(const gsl::span& peephole_weights) { + int i = 0; +#if defined(LSTM_NO_PEEPHOLE_COPY) + // just use spans. we don't change these values so there's no point copying to them + peephole_i_ = peephole_weights.subspan((i++ * hidden_size_), hidden_size_); + peephole_o_ = peephole_weights.subspan((i++ * hidden_size_), hidden_size_); + peephole_f_ = peephole_weights.subspan((i++ * hidden_size_), hidden_size_); + +#else + DumpMatrix("P[i]", peephole_weights.data() + (i++ * hidden_size_), 1, hidden_size_); + DumpMatrix("P[o]", peephole_weights.data() + (i++ * hidden_size_), 1, hidden_size_); + DumpMatrix("P[f]", peephole_weights.data() + (i++ * hidden_size_), 1, hidden_size_); + + auto copy_weight = [this, &peephole_weights](int offset, gsl::span& out) { + typename gsl::span::const_iterator in_iter = peephole_weights.cbegin() + offset; + std::copy(in_iter, in_iter + hidden_size_, out.begin()); + }; + + i = 0; + copy_weight((i++ * hidden_size_), peephole_i_); + copy_weight((i++ * hidden_size_), peephole_o_); + copy_weight((i++ * hidden_size_), peephole_f_); +#endif + + /* + DumpMatrix("peephole_i_", peephole_i_.data(), 1, hidden_size_); + DumpMatrix("peephole_o_", peephole_o_.data(), 1, hidden_size_); + DumpMatrix("peephole_f_", peephole_f_.data(), 1, hidden_size_); + */ +} + +template +void UniDirectionalAttnLstm::LoadBias(const gsl::span& WbRb_values) { + // add Wb and Rb + auto copy_fused_bias = [this, &WbRb_values](int offset, gsl::span& out) { + // gap between Wb and Wb value for an entry + const int Wb_to_Rb_offset = 4 * hidden_size_; + for (int j = 0; j < hidden_size_; ++j) { + out[j] = WbRb_values[j + offset] + WbRb_values[j + offset + Wb_to_Rb_offset]; + } + }; + + int i = 0; + copy_fused_bias((i++) * hidden_size_, bias_WRi_); + copy_fused_bias((i++) * hidden_size_, bias_WRo_); + copy_fused_bias((i++) * hidden_size_, bias_WRf_); + copy_fused_bias((i++) * hidden_size_, bias_WRc_); +} + +template +void UniDirectionalAttnLstm::Compute(const gsl::span& inputs_arg, + const gsl::span& sequence_lengths_arg, + const int num_directions, + const gsl::span& input_weights, + const gsl::span& recurrent_weights, + gsl::span& outputs, + gsl::span& final_hidden_state, + gsl::span& final_cell_state) { + // copy spans (just T* and size, not data in span) as we may change them + gsl::span inputs = inputs_arg; + gsl::span sequence_lengths = sequence_lengths_arg; + + // if sequence lengths weren't provided, use internal array and init all to seq_length + if (sequence_lengths.empty()) { + sequence_lengths_ = Allocate(allocator_, batch_size_, sequence_lengths_ptr_, true, seq_length_); + sequence_lengths = sequence_lengths_; + } + + // LSTM Layer + gsl::span batched_hidden_state_one_step = batched_hidden0_; + gsl::span batched_internal_state_prev_one_step = batched_internal_memory_prev_; + gsl::span batched_internal_state_clipped_one_step = batched_internal_memory_clipped_; + + int output_step_length = batch_size_ * hidden_size_; + + // The bidirectional LSTM wrapper wraps this LSTM class and produces bi-directional output + // the output has layout [seq,num_direction,batch,neurons]. + // When num_direction is 2, then this class will compute forward or backward LSTM. + // The outputs corresponds to either [seq,0,batch,neurons] or [seq,1,batch,neurons] + // Setting output_step_length this way allows writing the output directly without requiring + // additional memcpy. Note that if direction is kReverse, we write to output_reverse buffer + // which is then copied to output buffer, and ReverseSequence method handles the step length. + if (direction_ == Direction::kForward && num_directions == 2) + output_step_length = 2 * batch_size_ * hidden_size_; + + gsl::span original_outputs = outputs; + const bool output_sequence = !outputs.empty(); + + if (direction_ == Direction::kReverse) { + ReverseSequence(inputs, inputs_reverse_, sequence_lengths, seq_length_, batch_size_, input_size_, 1); + inputs = inputs_reverse_; + + if (output_sequence) + outputs = outputs_reverse_; + } + + // Calculate the max and min length + int32_t max_sequence_length = *std::max_element(sequence_lengths.cbegin(), sequence_lengths.cend()); + int32_t min_sequence_length = std::min(seq_length_, *std::min_element(sequence_lengths.cbegin(), + sequence_lengths.cend())); + + ///**************************LSTM Calculations****************************/ + const int hidden_size_x4 = 4 * hidden_size_; + const int total_rows = max_sequence_length * batch_size_; + + // apply the weights to all the inputs and save to output_IOFC + ComputeGemm(total_rows, hidden_size_x4, input_size_, T{1.0}, + inputs.cbegin(), inputs.cend(), + input_size_, + input_weights.cbegin(), input_weights.cend(), // W[iofc]^T + input_size_ + attention_size_, T{0.0}, + output_iofc_.begin(), output_iofc_.end(), + hidden_size_x4); + + DumpMatrix("Xt*(W[iofc]^T)", output_iofc_.data(), total_rows, hidden_size_x4); + + int fused_hidden_rows = batch_size_ / hidden_num_threads_; + if (batch_size_ % hidden_num_threads_ != 0) + fused_hidden_rows++; + + // NOTE: we could refine the bounds checking in the calls below that use these values to instead + // explicitly check just the range for each iteration, however if it's going to run over + // it should also run over on the last iteration, so this should be good enough to catch any + // logic errors causing bounds violations. + span_T_iter C_prev_end = batched_internal_state_prev_one_step.end(); + span_T_iter C_prev_clipped_end = batched_internal_state_clipped_one_step.end(); + span_T_const_iter previous_state_end = batched_hidden_state_one_step.end(); + + { + span_T_iter c_prev = batched_internal_state_prev_one_step.begin(); + span_T_iter c_prev_clipped = batched_internal_state_clipped_one_step.begin(); + + // hidden state can be provided as input for first step, so need to special case that. + // after the first step this will switch to the output from the previous step + span_T_const_iter previous_state = batched_hidden_state_one_step.cbegin(); + + //run through steps sequentially + for (int step = 0; step < max_sequence_length; step++) { + const std::string seqno_str = " [seqno=" + std::to_string(step) + "]"; + + DumpMatrix("previous_state" + seqno_str, &*previous_state, batch_size_, hidden_size_); + + span_T_iter step_out_IOFC = output_iofc_.begin() + (step * batch_size_) * hidden_size_x4; + + // shape is [ attention_size_ ] + const gsl::span attention = attention_wrapper_.GetAttnStates(); + + // Xt*(W[iofc]^T) = INPUTt * W[iofc]^T + At-1 * WA[iofc] + ComputeGemm(batch_size_, hidden_size_x4, attention_size_, T{1.0}, + attention.cbegin(), attention.cend(), // At-1 + attention_size_, + input_weights.cbegin() + input_size_, input_weights.cend(), // WA[iofc] + input_size_ + attention_size_, T{1.0}, + step_out_IOFC, output_iofc_.end(), // input contains Xt*(W[iofc]^T) + hidden_size_x4); + + // calculate Xt*(W[iofc]^T) + Ht-1*R[iofc] + ComputeGemm(batch_size_, hidden_size_x4, hidden_size_, T{1.0}, + previous_state, previous_state_end, // Ht-1 + hidden_size_, + recurrent_weights.cbegin(), recurrent_weights.cend(), // R[iofc] + hidden_size_, T{1.0}, + step_out_IOFC, output_iofc_.end(), // input contains Xt*(W[iofc]^T) + hidden_size_x4); + + span_T_iter batched_output, batched_output_end; + if (output_sequence) { + batched_output = outputs.begin() + step * output_step_length; + batched_output_end = outputs.end(); + } else { + batched_output = final_hidden_state.begin(); + batched_output_end = final_hidden_state.end(); + } + + span_T_iter step_out_IOFC_end = step_out_IOFC + batch_size_ * hidden_size_x4; + GateComputations(step_out_IOFC, step_out_IOFC_end, + c_prev, C_prev_end, + c_prev_clipped, C_prev_clipped_end, + batched_output, batched_output_end, + sequence_lengths, min_sequence_length, step, 0, batch_size_, output_sequence); + + // copy last row to final_cell_state + for (int lrow = 0; lrow < batch_size_; lrow++) { + if ((step + 1) == sequence_lengths[lrow]) { + auto src = batched_internal_memory_prev_.subspan(lrow * hidden_size_, hidden_size_); + auto dst = final_cell_state.subspan(lrow * hidden_size_, hidden_size_); + gsl::copy(src, dst); + } + } + + if (output_sequence) { + //set to 0 if step >= sequence_length + for (int lrow = 0; lrow < batch_size_; lrow++) { + if (step >= min_sequence_length && step >= sequence_lengths[lrow]) { + auto dst = outputs.data() + step * output_step_length + lrow * hidden_size_; + std::fill_n(dst, hidden_size_, T{}); + } + } + } + + previous_state = batched_output; + previous_state_end = batched_output_end; + + attention_wrapper_.ProcessOutput(outputs.subspan(step * output_step_length, batch_size_ * hidden_size_)); + } + } + + if (output_sequence) { + // copy last output to final_hidden_state + for (int i = 0; i < batch_size_; i++) { + const int seq_len = sequence_lengths[i]; + auto src = outputs.subspan((seq_len - 1) * output_step_length + i * hidden_size_, hidden_size_); + auto dest = final_hidden_state.subspan(i * hidden_size_, hidden_size_); + gsl::copy(src, dest); + } + + if (direction_ == Direction::kReverse) + ReverseSequence(outputs, original_outputs, sequence_lengths, max_sequence_length, + batch_size_, hidden_size_, num_directions); + } +} + +template +void UniDirectionalAttnLstm::GateComputations(span_T_iter& out, span_T_iter& out_end, + span_T_iter& C_prev, span_T_iter& C_prev_end, // Ct-1 value not 'ct'. using 'C' for clarity + span_T_iter& C_prev_clipped, span_T_iter& C_prev_clipped_end, + span_T_iter& batched_output, span_T_iter& batched_output_end, + const gsl::span& seq_lengths, + const int min_sequence_length, + const int step, + const int row, + const int local_fused_hidden_rows, + bool output_sequence) { + int hidden_size_x4 = 4 * hidden_size_; + + // Activation gates. + for (int b = 0; b < local_fused_hidden_rows; b++) { + if (step >= min_sequence_length && step >= seq_lengths[row + b]) { + if (output_sequence) { + auto fill_output = batched_output + (row + b) * hidden_size_; + std::fill(fill_output, fill_output + hidden_size_, T{}); + } + + continue; + } + + std::string row_str = " row[" + std::to_string(row + b) + "]"; + + // check that we have hidden_size_x4 left starting at cur_out + b * hidden_size_x4, and get a raw pointer to that + float* pi = SafeRawPointer(out + b * hidden_size_x4, out_end, hidden_size_x4); + float* po = pi + hidden_size_; + float* pf = po + hidden_size_; + float* pc = pf + hidden_size_; + + float* pCprev_hidden_size = SafeRawPointer(C_prev + b * hidden_size_, C_prev_end, hidden_size_); + + // Input Gate + if (use_peepholes_) { + deepcpu::elementwise_product(pCprev_hidden_size, SafeRawConstPointer(peephole_i_, 0, hidden_size_), + pi, hidden_size_); + } + + const float* pBi = use_bias_ ? SafeRawConstPointer(bias_WRi_, 0, hidden_size_) : nullptr; + clip_with_bias_ptr_(clip_, pBi, pi, hidden_size_); // post: pi has input to f() to calculate i + activation_f_.func(pi, hidden_size_, activation_f_.alpha, activation_f_.beta); + //DumpMatrix("i" + row_str, pi, 1, hidden_size_); + + // Forget Gate + if (input_forget_) { + for (int i = 0; i < hidden_size_; i++) { + pf[i] = 1.0f - pi[i]; + } + } else { + if (use_peepholes_) { + deepcpu::elementwise_product( + pCprev_hidden_size, SafeRawConstPointer(peephole_f_, 0, hidden_size_), pf, hidden_size_); + } + + const float* pBf = use_bias_ ? SafeRawConstPointer(bias_WRf_, 0, hidden_size_) : nullptr; + clip_with_bias_ptr_(clip_, pBf, pf, hidden_size_); + activation_f_.func(pf, hidden_size_, activation_f_.alpha, activation_f_.beta); + } + + // Block Gate + const float* pBc = use_bias_ ? SafeRawConstPointer(bias_WRc_, 0, hidden_size_) : nullptr; + clip_with_bias_ptr_(clip_, pBc, pc, hidden_size_); + activation_g_.func(pc, hidden_size_, activation_g_.alpha, activation_g_.beta); + + // C_current. use previous C value as input, and update in-place + float* pC_cur = pCprev_hidden_size; + deepcpu::merge_lstm_gates_to_memory(pCprev_hidden_size, pi, pf, pc, pC_cur, hidden_size_); + + // Output Gate + if (use_peepholes_) { + deepcpu::elementwise_product( + pCprev_hidden_size, SafeRawConstPointer(peephole_o_, 0, hidden_size_), po, hidden_size_); + } + + // calculate 'ot' + const float* pBo = use_bias_ ? SafeRawConstPointer(bias_WRo_, 0, hidden_size_) : nullptr; + clip_with_bias_ptr_(clip_, pBo, po, hidden_size_); + activation_f_.func(po, hidden_size_, activation_f_.alpha, activation_f_.beta); + // DumpMatrix("o" + row_str, po, 1, hidden_size_); + + // calculate 'Ht' + float* pH = SafeRawPointer(batched_output + row * hidden_size_ + b * hidden_size_, + batched_output_end, hidden_size_); + + // the C_prev_clipped location is not actually used as input - it's temporary storage for writing + // the clipped Ct value to, before calling h(). As such a) it could just be a local variable + // of std::vector with size of hidden_size_, b) the previous version wasn't 'broken' by never + // incrementing what C_prev_clipped pointed to. + float* pC_prev_clipped = SafeRawPointer(C_prev_clipped + b * hidden_size_, C_prev_clipped_end, hidden_size_); + + activation_h_.func(pC_cur, pC_prev_clipped, po, pH, hidden_size_, activation_h_.alpha, activation_h_.beta); + } + + auto num_rows = local_fused_hidden_rows - row; + std::string rows_str = " rows[" + std::to_string(row) + ".." + std::to_string(num_rows) + "]"; + + DumpMatrix("i" + rows_str, &*out, num_rows, hidden_size_, 0, hidden_size_x4); + DumpMatrix("o" + rows_str, &*out, num_rows, hidden_size_, 1 * hidden_size_, hidden_size_x4); + DumpMatrix("f" + rows_str, &*out, num_rows, hidden_size_, 2 * hidden_size_, hidden_size_x4); + DumpMatrix("c" + rows_str, &*out, num_rows, hidden_size_, 3 * hidden_size_, hidden_size_x4); + DumpMatrix("C" + rows_str, &*C_prev, num_rows, hidden_size_); // Ct overwrites the input C_prev value + DumpMatrix("H" + rows_str, &*batched_output, num_rows, hidden_size_); +} + +template +void UniDirectionalAttnLstm::SetNumThreads() { + int threads = std::thread::hardware_concurrency() - 1; + + if (threads < 1) + threads = 1; + + int hmt = threads; + batch_parallel_ = false; + + // For readability of the below logic + + // TODO: Temperately removed path: parallelize by partitioning the batch rows, + // and its logic in the Compute() method. Will evaluate and finialize it + // in later performance tuning stage. + const auto num_columns = hidden_size_; + { + if (hmt > 2 && num_columns <= 128) + hmt = 2; + if (hmt > 5 && num_columns <= 256) + hmt = 5; + if (hmt > 7 && num_columns <= 512) + hmt = 7; + if (hmt > 11 && num_columns <= 1024) + hmt = 11; + + hidden_num_threads_ = hmt; + } + + VLOGS(logger_, 1) << "Hidden Threads : " << hidden_num_threads_; +} + +template class UniDirectionalAttnLstm; + +} // namespace detail +} // namespace rnn +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/attnlstm/uni_dir_attn_lstm.h b/onnxruntime/contrib_ops/cpu/attnlstm/uni_dir_attn_lstm.h new file mode 100644 index 0000000000000..be82163bba3ac --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/attnlstm/uni_dir_attn_lstm.h @@ -0,0 +1,162 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "activation_info.h" +#include "attention_wrapper.h" +#include "core/providers/cpu/rnn/rnn_helpers.h" + +#include "core/common/common.h" +#include "core/common/logging/logging.h" +#include "core/common/task_thread_pool.h" +#include "core/framework/allocator.h" + +#include + +namespace onnxruntime { +namespace contrib { + +namespace deepcpu = ::onnxruntime::rnn::detail::deepcpu; +namespace logging = ::onnxruntime::logging; + +using ::onnxruntime::AllocatorPtr; +using ::onnxruntime::IAllocatorUniquePtr; +using ::onnxruntime::TaskThreadPool; +using ::onnxruntime::contrib::detail::ActivationInfo; +using ::onnxruntime::rnn::detail::ActivationFuncs; +using ::onnxruntime::rnn::detail::Direction; + +namespace rnn { +namespace detail { + +// copying the peephole values into UniDirectionalAttnLstm seems unnecessary. don't do that until proven necessary +#define LSTM_NO_PEEPHOLE_COPY + +template +class UniDirectionalAttnLstm { + public: + UniDirectionalAttnLstm(AllocatorPtr allocator, + const logging::Logger& logger, + const int seq_length, + const int batch_size, + const int input_size, + const int hidden_size, + Direction direction, + const bool input_forget, + AttentionWrapper& attention_wrapper, + const gsl::span& bias, + const gsl::span& peephole_weights, + const gsl::span& initial_hidden_state, + const gsl::span& initial_cell_state, + const ActivationFuncs::Entry& activation_func_f, + const ActivationFuncs::Entry& activation_func_g, + const ActivationFuncs::Entry& activation_func_h, + const float clip, + TaskThreadPool& ttp); + + void Compute(const gsl::span& inputs, + const gsl::span& sequence_lengths, + const int num_directions, + const gsl::span& input_weights, + const gsl::span& recurrent_weights, + gsl::span& outputs, + gsl::span& final_hidden_state, + gsl::span& final_cell_state); + + ~UniDirectionalAttnLstm() = default; + + AttentionWrapper& GetAttentionWrapper() { + return attention_wrapper_; + } + + private: + using span_T_const_iter = typename gsl::span::const_iterator; + using span_T_iter = typename gsl::span::iterator; + + void SetNumThreads(); + + void GateComputations(span_T_iter& out, span_T_iter& out_end, + span_T_iter& C_prev, span_T_iter& C_prev_end, // Ct-1 value not 'ct'. using 'C' for clarity + span_T_iter& C_prev_clipped, span_T_iter& C_prev_clipped_end, + span_T_iter& batched_output, span_T_iter& batched_output_end, + const gsl::span& seq_lengths, + const int min_sequence_length, + const int step, + const int row, + const int local_fused_hidden_rows, + bool output_sequence); + + void AllocateBuffers(); + + void InitializeBuffers(const gsl::span& initial_hidden_state, + const gsl::span& initial_cell_state); + + void LoadPeepholeWeights(const gsl::span& peephole_weights); + void LoadBias(const gsl::span& WbRb_values); + + AllocatorPtr allocator_; + const logging::Logger& logger_; + + int seq_length_; + int batch_size_; + int input_size_; + int hidden_size_; + int attention_size_; + int attention_context_size_; + + Direction direction_; + bool input_forget_; + float clip_; + + bool batch_parallel_; + + bool use_bias_; + bool use_peepholes_; + + int hidden_num_threads_ = -1; + + IAllocatorUniquePtr output_iofc_ptr_; + IAllocatorUniquePtr hidden0_ptr_, batched_hidden0_ptr_; + gsl::span output_iofc_; + gsl::span hidden0_, batched_hidden0_; + + IAllocatorUniquePtr internal_memory_prev_ptr_, batched_internal_memory_prev_ptr_; + IAllocatorUniquePtr internal_memory_cur_ptr_, batched_internal_memory_cur_ptr_; + IAllocatorUniquePtr batched_internal_memory_clipped_ptr_; + gsl::span internal_memory_prev_, batched_internal_memory_prev_; + gsl::span internal_memory_cur_, batched_internal_memory_cur_; + gsl::span batched_internal_memory_clipped_; + + IAllocatorUniquePtr bias_WRi_ptr_, bias_WRf_ptr_, bias_WRo_ptr_, bias_WRc_ptr_; + IAllocatorUniquePtr batched_bias_WRi_ptr_, batched_bias_WRf_ptr_, batched_bias_WRo_ptr_, batched_bias_WRc_ptr_; + IAllocatorUniquePtr peephole_i_ptr_, peephole_f_ptr_, peephole_o_ptr_; + IAllocatorUniquePtr inputs_reverse_ptr_, outputs_reverse_ptr_; + gsl::span bias_WRi_, bias_WRf_, bias_WRo_, bias_WRc_; + gsl::span batched_bias_WRi_, batched_bias_WRf_, batched_bias_WRo_, *batched_bias_WRc_; + gsl::span inputs_reverse_, outputs_reverse_; + +#if defined(LSTM_NO_PEEPHOLE_COPY) + gsl::span peephole_i_, peephole_f_, peephole_o_; +#else + gsl::span peephole_i_, peephole_f_, peephole_o_; +#endif + + IAllocatorUniquePtr sequence_lengths_ptr_; + gsl::span sequence_lengths_; + + deepcpu::ClipWithBiasFuncPtr clip_with_bias_ptr_; + + ActivationInfo activation_f_; + ActivationInfo activation_g_; + ActivationInfo activation_h_; + + AttentionWrapper& attention_wrapper_; + + TaskThreadPool& ttp_; +}; + +} // namespace detail +} // namespace rnn +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/expand_dims.cc b/onnxruntime/contrib_ops/cpu/expand_dims.cc new file mode 100644 index 0000000000000..03b9d9418e7d7 --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/expand_dims.cc @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "expand_dims.h" +#include "onnx/defs/schema.h" +#include "sample.h" + +namespace onnxruntime { +namespace contrib { +ONNX_CPU_OPERATOR_TYPED_MS_KERNEL( + ExpandDims, + 1, + float, + KernelDefBuilder() + .Alias(0, 0) + .TypeConstraint("T", DataTypeImpl::AllTensorTypes()) + .TypeConstraint("axis", DataTypeImpl::GetTensorType()), + contrib::ExpandDims); +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/expand_dims.h b/onnxruntime/contrib_ops/cpu/expand_dims.h new file mode 100644 index 0000000000000..f7b320499315d --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/expand_dims.h @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/framework/tensor.h" +#include "core/providers/cpu/tensor/utils.h" + +namespace onnxruntime { +namespace contrib { +/* +Given a tensor input, this operation inserts a dimension of 1 at the dimension index axis +of X's shape. The dimension index axis starts at zero; if you specify a negative number +of axis, it starts backward from the end. +*/ + +class ExpandDims final : public OpKernel { + public: + explicit ExpandDims(const OpKernelInfo& info) : OpKernel(info) {} + + Status Compute(OpKernelContext* context) const override { + const Tensor* axis_tensor = context->Input(1); + if (axis_tensor == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + + ONNXRUNTIME_ENFORCE(axis_tensor->Shape().IsScalar(), "An axis tensor must be a scalar tensor."); + const int64_t axis = static_cast(axis_tensor->template Data()[0]); + const Tensor* X = context->Input(0); + if (X == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + const TensorShape& X_shape = X->Shape(); + + std::vector expanded_shape(X_shape.GetDims()); + int64_t X_NumDims = X_shape.Size(); + ONNXRUNTIME_ENFORCE(axis <= X_NumDims && axis >= -X_NumDims, + "Axis must be within range [", -X_NumDims, ", ", X_NumDims, "].", " Axis is ", axis); + if (axis >= 0) { + expanded_shape.insert(expanded_shape.begin() + axis, 1); + } else { + expanded_shape.insert(expanded_shape.end() + axis + 1, 1); + } + + Tensor* Y = context->Output(0, TensorShape(expanded_shape)); + CopyCpuTensor(X, Y); + + return Status::OK(); + } +}; + +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/isnan.cc b/onnxruntime/contrib_ops/cpu/isnan.cc new file mode 100644 index 0000000000000..9599c40b91224 --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/isnan.cc @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "isnan.h" +#include "onnx/defs/schema.h" +#include "core/util/math_cpuonly.h" +#include "core/common/common.h" +#include "core/framework/tensor.h" + +namespace onnxruntime { +namespace contrib { +ONNX_CPU_OPERATOR_TYPED_MS_KERNEL( + IsNaN, + 1, + float, + KernelDefBuilder() + .TypeConstraint("T1", DataTypeImpl::GetTensorType()) + .TypeConstraint("T2", DataTypeImpl::GetTensorType()), + contrib::IsNaN); + +template <> +Status IsNaN::Compute(OpKernelContext* context) const { + const Tensor* X_ptr = context->Input(0); + if (!X_ptr) { + return Status(common::ONNXRUNTIME, common::FAIL, "Null input ptr"); + } + auto& X = *X_ptr; + auto& dims = X.Shape(); + auto& Y = *context->Output(0, dims); + + EigenMap(Y) = EigenMap(X).array().isNaN(); + + return Status::OK(); +} + +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/isnan.h b/onnxruntime/contrib_ops/cpu/isnan.h new file mode 100644 index 0000000000000..6c670e40498cc --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/isnan.h @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/framework/op_kernel.h" + +namespace onnxruntime { +namespace contrib { +template +class IsNaN : public OpKernel { + public: + explicit IsNaN(const OpKernelInfo& info) : OpKernel(info) {} + Status Compute(OpKernelContext* context) const override; +}; +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/sample.cc b/onnxruntime/contrib_ops/cpu/sample.cc new file mode 100644 index 0000000000000..cd359d2e8627b --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/sample.cc @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "sample.h" +#include "onnx/defs/schema.h" + +namespace onnxruntime { +namespace contrib { +// These ops are internal-only, so register outside of onnx +ONNX_CPU_OPERATOR_TYPED_MS_KERNEL( + SampleOp, + 1, + float, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()).MayInplace(0, 0), + contrib::SampleOp); +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/sample.h b/onnxruntime/contrib_ops/cpu/sample.h new file mode 100644 index 0000000000000..5a3a3ddaad83a --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/sample.h @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/framework/tensor.h" + +namespace onnxruntime { +namespace contrib { +template +class SampleOp : public OpKernel { + public: + explicit SampleOp(const OpKernelInfo& info) : OpKernel(info) {} + + Status Compute(OpKernelContext* context) const override { + auto X = context->Input(0); + auto& dims = X->Shape(); + auto Y = context->Output(0, dims); + auto X_Data = (X->template Data()); + auto Y_Data = (Y->template MutableData()); + + for (int64_t i = 0, sz = dims.Size(); i < sz; ++i) { + *Y_Data++ = *X_Data++; + } + + return Status::OK(); + } +}; +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/core/codegen/tvm/tvm_compiler.cc b/onnxruntime/core/codegen/tvm/tvm_compiler.cc new file mode 100644 index 0000000000000..964b9fdf78838 --- /dev/null +++ b/onnxruntime/core/codegen/tvm/tvm_compiler.cc @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include +#include "core/codegen/tvm/tvm_compiler.h" +namespace onnxruntime { + +TVMGraph::TensorDescriptor::TensorDescriptor(MLDataType type, onnxruntime::ProviderType execution_provider_type, tvm::Tensor tvm_tensor) : tvm_tensor_(tvm_tensor) { + if (execution_provider_type == onnxruntime::kCpuExecutionProvider) { + ctx_.device_type = DLDeviceType::kDLCPU; + ctx_.device_id = 0; + } else { + ONNXRUNTIME_NOT_IMPLEMENTED("Non-cpu execution provider not supported on TVM now."); + } + + if (DataTypeImpl::GetTensorType() == type) { + dtype_.code = kDLFloat; + dtype_.bits = 64; + dtype_.lanes = 1; + } else { + ONNXRUNTIME_NOT_IMPLEMENTED("Non-double type not supported on TVM now."); + } +} + +class IdGenerator { + public: + IdGenerator() : cur_(0) {} + int GetNext() { + return cur_++; + } + + private: + int cur_; +}; + +// This is a special compiler step for the test case that sum two 1-D tensors +static void Compile1DAddToTVM(const onnxruntime::Node& node, std::unordered_map& tvm_tensors, onnxruntime::ProviderType execution_provider_type, IdGenerator& generator) { + ONNXRUNTIME_ENFORCE(node.OpType() == "Add"); + tvm::Array shape; + shape.push_back(tvm::var("n1")); + + tvm::Tensor t1, t2; + auto it = tvm_tensors.find(node.InputDefs()[0]); + if (it == tvm_tensors.end()) { + tvm_tensors[node.InputDefs()[0]] = TVMGraph::TensorDescriptor( + DataTypeImpl::TypeFromProto(*node.InputDefs()[0]->TypeAsProto()), + execution_provider_type, + tvm::placeholder(shape, tvm::Float(64), "T" + std::to_string(generator.GetNext()))); + } + t1 = tvm_tensors[node.InputDefs()[0]].tvm_tensor_; + it = tvm_tensors.find(node.InputDefs()[1]); + if (it == tvm_tensors.end()) { + tvm_tensors[node.InputDefs()[1]] = TVMGraph::TensorDescriptor( + DataTypeImpl::TypeFromProto(*node.InputDefs()[1]->TypeAsProto()), + execution_provider_type, + tvm::placeholder(shape, tvm::Float(64), "T" + std::to_string(generator.GetNext()))); + } + t2 = tvm_tensors[node.InputDefs()[1]].tvm_tensor_; + + tvm_tensors[node.OutputDefs()[0]] = TVMGraph::TensorDescriptor( + DataTypeImpl::TypeFromProto(*node.InputDefs()[1]->TypeAsProto()), + execution_provider_type, + tvm::compute(t1->shape, [&t1, &t2](tvm::Expr i) { + return t1[i] + t2[i]; + }, + "T" + std::to_string(generator.GetNext()))); +} + +TVMGraph CompileToTVM(const onnxruntime::Graph& graph, onnxruntime::ProviderType execution_provider_type) { + TVMGraph result; + std::unordered_map tvm_tensors; + IdGenerator generator; + for (auto& node : graph.Nodes()) { + Compile1DAddToTVM(node, tvm_tensors, execution_provider_type, generator); + } + + for (auto& input : graph.GetInputs()) { + result.inputs_.push_back(tvm_tensors[input]); + } + + auto& output = graph.GetOutputs()[0]; + result.outputs_.push_back(tvm_tensors[output]); + return result; +} +} // namespace onnxruntime diff --git a/onnxruntime/core/codegen/tvm/tvm_compiler.h b/onnxruntime/core/codegen/tvm/tvm_compiler.h new file mode 100644 index 0000000000000..5574652cc3ad4 --- /dev/null +++ b/onnxruntime/core/codegen/tvm/tvm_compiler.h @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include +#include "core/common/common.h" +#include "core/framework/data_types.h" +#include "core/graph/function.h" +#include "core/graph/constants.h" +#include "core/graph/graph.h" +namespace onnxruntime { + +//TODO: this is just initial design to represent TVM Graph, to make the basic test work. +//We may need to revisit it later to finialize it. +struct TVMGraph { + struct TensorDescriptor { + tvm::Tensor tvm_tensor_; + DLContext ctx_; + DLDataType dtype_; + + public: + TensorDescriptor(MLDataType type, onnxruntime::ProviderType execution_provider_type, tvm::Tensor tvm_tensor); + + TensorDescriptor() {} + }; + std::vector inputs_; + std::vector outputs_; +}; + +//TODO: compile a onnxruntime graph to tvm's tensor expression is a common logic for all hardwares +//onnxruntime framework should provide this functionality to executionp providers. +//We will need to register how to compiler it for each node. A detail design is needed. +//Here for testing we just provide the functionality that compile add 1D tensors. +TVMGraph CompileToTVM(const onnxruntime::Graph& graph, onnxruntime::ProviderType execution_provider_type); +} // namespace onnxruntime diff --git a/onnxruntime/core/codegen/tvm/tvm_kernel.h b/onnxruntime/core/codegen/tvm/tvm_kernel.h new file mode 100644 index 0000000000000..4ce4d3755c8a2 --- /dev/null +++ b/onnxruntime/core/codegen/tvm/tvm_kernel.h @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/codegen/tvm/tvm_compiler.h" +#include "core/graph/function.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { + +// TVMScheduleCreator is the function that create a tvm schedule based on given TVM graph. +// Different hardware may have different schedule strategy. +typedef tvm::Schedule (*TVMScheduleCreator)(const TVMGraph&); +// TVMModuleBuilder is the function that build a tvm module, given a schedule and args. +// Different tvm kernel may chose different way to build the module, like target to LLVM or other backend. +typedef tvm::runtime::Module (*TVMModuleBuilder)(tvm::Schedule schedule, tvm::BuildConfig config, tvm::Array args, std::vector& target_func_names); + +template +class TVMKernel : public OpKernel { + public: + explicit TVMKernel(const OpKernelInfo& info) : OpKernel(info), tvm_values_(nullptr), dl_tensors_(nullptr), tvm_type_codes_(nullptr) { + auto& node = info.node(); + ONNXRUNTIME_ENFORCE(node.NodeType() == Node::Type::Fused); + auto func = node.GetFunctionBody(); + const onnxruntime::Graph& func_body = func->Body(); + //1. compile the onnxruntime Graph to tvm graph. This step is common for all hardware, and provided by onnxruntime framework. + tvm_graph_ = CompileToTVM(func_body, node.GetExecutionProviderType()); + //2. create schedule for tvm graph, this step is depends on the execution provider/hardware. + auto s = S(tvm_graph_); + //3. Build module + std::vector tvm_args; + for (auto& t : tvm_graph_.inputs_) { + tvm_args.push_back(t.tvm_tensor_); + } + for (auto& t : tvm_graph_.outputs_) { + tvm_args.push_back(t.tvm_tensor_); + } + + std::vector func_names; + tvm_module_ = M(s, tvm::build_config(), tvm_args, func_names); + //TODO: do we have case that need more than 1 evaluation function? + evaluate_func_ = tvm_module_.GetFunction(func_names[0]); + //4. prepare args according to the type + n_args_ = tvm_args.size(); + tvm_values_ = new TVMValue[n_args_]; + tvm_type_codes_ = new int[n_args_]; + dl_tensors_ = new DLTensor[n_args_]; + int i = 0; + for (auto& tensor : tvm_graph_.inputs_) { + tvm_type_codes_[i] = kNDArrayContainer; + dl_tensors_[i].ctx = tensor.ctx_; + dl_tensors_[i].dtype = tensor.dtype_; + dl_tensors_[i].strides = nullptr; + dl_tensors_[i].byte_offset = 0; + tvm_values_[i].v_handle = &dl_tensors_[i]; + i++; + } + + for (auto& tensor : tvm_graph_.outputs_) { + tvm_type_codes_[i] = kNDArrayContainer; + dl_tensors_[i].ctx = tensor.ctx_; + dl_tensors_[i].dtype = tensor.dtype_; + dl_tensors_[i].strides = nullptr; + dl_tensors_[i].byte_offset = 0; + tvm_values_[i].v_handle = &dl_tensors_[i]; + i++; + } + ONNXRUNTIME_ENFORCE(i == n_args_); + } + + virtual ~TVMKernel() { + if (!tvm_values_) + delete[] tvm_values_; + if (!tvm_type_codes_) + delete[] tvm_type_codes_; + if (!dl_tensors_) + delete[] dl_tensors_; + } + + virtual Status Compute(OpKernelContext* context) const override { + for (int i = 0; i < tvm_graph_.inputs_.size(); ++i) { + auto t = context->Input(i); + dl_tensors_[i].data = const_cast(t)->MutableDataRaw(); + dl_tensors_[i].ndim = static_cast(t->Shape().NumDimensions()); + dl_tensors_[i].shape = dl_tensors_[i].ndim > 0 ? const_cast(&(t->Shape().GetDims()[0])) : nullptr; + } + + int num_inputs = static_cast(tvm_graph_.inputs_.size()); + + for (int i = 0; i < tvm_graph_.outputs_.size(); ++i) { + //TODO: we need to have a shape inference function that could calculate output shape based on the symbolic formular in tvm + //We could build that function as part of tvm module, or reuse the shape inference in onnx funciton. + //Here for testing purpose, assume the output shape is same to input shape. + auto t = context->Output(i, GetOutputShape(context, i)); + dl_tensors_[num_inputs + i].data = t->MutableDataRaw(); + dl_tensors_[num_inputs + i].ndim = static_cast(t->Shape().NumDimensions()); + dl_tensors_[num_inputs + i].shape = dl_tensors_[i].ndim > 0 ? const_cast(&(t->Shape().GetDims()[0])) : nullptr; + } + + tvm::TVMArgs tvm_args(&tvm_values_[0], &tvm_type_codes_[0], static_cast(n_args_)); + tvm::TVMRetValue rvalue; + try { + evaluate_func_.CallPacked(tvm_args, &rvalue); + } catch (std::exception ex) { + return Status(onnxruntime::common::ONNXRUNTIME, onnxruntime::common::FAIL, "TVM run failed."); + } + if (rvalue.type_code() != kNull) { + return Status(onnxruntime::common::ONNXRUNTIME, onnxruntime::common::FAIL, "TVM return not null"); // TODO: get error code. + } else { + return Status::OK(); + } + } + + protected: + virtual const TensorShape& GetOutputShape(OpKernelContext* context, int i) const = 0; + + TVMGraph tvm_graph_; + tvm::runtime::Module tvm_module_; + tvm::PackedFunc evaluate_func_; + + size_t n_args_; + TVMValue* tvm_values_; + DLTensor* dl_tensors_; + int* tvm_type_codes_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/codegen/tvm/tvm_utils.cc b/onnxruntime/core/codegen/tvm/tvm_utils.cc new file mode 100644 index 0000000000000..6d474581aad89 --- /dev/null +++ b/onnxruntime/core/codegen/tvm/tvm_utils.cc @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "tvm_utils.h" + +namespace onnxruntime { +namespace tvm_codegen { + +#define RETURN_DLDATATYPE_IF_MATCH(type, type_code) \ + if (ml_type == DataTypeImpl::GetType()) { \ + return {type_code, sizeof(type) * 8, 1}; \ + } + +// DLDataType: {DLDataTypeCode, bits, lanes} +DLDataType ToTvmDLDataType(MLDataType ml_type) { + RETURN_DLDATATYPE_IF_MATCH(int8_t, kDLInt); + RETURN_DLDATATYPE_IF_MATCH(uint8_t, kDLInt); + RETURN_DLDATATYPE_IF_MATCH(int16_t, kDLInt); + RETURN_DLDATATYPE_IF_MATCH(uint16_t, kDLInt); + RETURN_DLDATATYPE_IF_MATCH(int32_t, kDLInt); + RETURN_DLDATATYPE_IF_MATCH(uint32_t, kDLInt); + RETURN_DLDATATYPE_IF_MATCH(int64_t, kDLInt); + RETURN_DLDATATYPE_IF_MATCH(uint64_t, kDLInt); + + RETURN_DLDATATYPE_IF_MATCH(float, kDLFloat); + RETURN_DLDATATYPE_IF_MATCH(double, kDLFloat); + + ONNXRUNTIME_NOT_IMPLEMENTED("converting MLDataType ", ml_type, " to tvm DLDataType is not implemented"); +} + +} // namespace tvm_codegen +} // namespace onnxruntime diff --git a/onnxruntime/core/codegen/tvm/tvm_utils.h b/onnxruntime/core/codegen/tvm/tvm_utils.h new file mode 100644 index 0000000000000..abe35753a07e0 --- /dev/null +++ b/onnxruntime/core/codegen/tvm/tvm_utils.h @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include + +#include "core/framework/data_types.h" + +namespace onnxruntime { + +constexpr const char* TVM_STACKVM = "TvmStackVm"; + +namespace tvm_codegen { + // Helper function that converts a onnxruntime MLDataType to TVM DLDataType + DLDataType ToTvmDLDataType(MLDataType ml_type); +} // namespace tvm +} // namespace onnxruntime diff --git a/onnxruntime/core/common/cpuid_info.cc b/onnxruntime/core/common/cpuid_info.cc new file mode 100644 index 0000000000000..dab771b9bc903 --- /dev/null +++ b/onnxruntime/core/common/cpuid_info.cc @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__) +#define PLATFORM_X86 +#endif + +#if defined(PLATFORM_X86) +#include +#include + +#if defined(_MSC_VER) +#include +#elif defined(__GNUC__) +#include +#endif +#endif + +#include "core/common/cpuid_info.h" + +namespace onnxruntime { + +#if defined(PLATFORM_X86) +static inline void GetCPUID(int function_id, int data[4]) { // NOLINT +#if defined(_MSC_VER) + __cpuid(reinterpret_cast(data), function_id); +#elif defined(__GNUC__) + __cpuid(function_id, data[0], data[1], data[2], data[3]); +#endif +} + +static inline int XGETBV() { +#if defined(_MSC_VER) + return static_cast(_xgetbv(0)); +#elif defined(__GNUC__) + int eax, edx; + __asm__ volatile("xgetbv" + : "=a"(eax), "=d"(edx) + : "c"(0)); + return eax; +#endif +} +#endif // PLATFORM_X86 + +CPUIDInfo::CPUIDInfo() noexcept { +#if defined(PLATFORM_X86) + int data[4] = {-1}; + GetCPUID(0, data); + + int num_IDs = data[0]; + if (num_IDs >= 1) { + GetCPUID(1, data); + if (data[2] & (1 << 27)) { + const int AVX_MASK = 0x6; + const int AVX512_MASK = 0xE6; + int value = XGETBV(); + bool has_avx = (data[2] & (1 << 28)) && ((value & AVX_MASK) == AVX_MASK); + bool has_avx512 = (value & AVX512_MASK) == AVX512_MASK; + + if (num_IDs >= 7) { + GetCPUID(7, data); + has_avx2_ = has_avx && (data[1] & (1 << 5)); + has_avx512f_ = has_avx512 && (data[1] & (1 << 16)); + } + } + } +#endif +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/common/cpuid_info.h b/onnxruntime/core/common/cpuid_info.h new file mode 100644 index 0000000000000..2a1b89d9e70fd --- /dev/null +++ b/onnxruntime/core/common/cpuid_info.h @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +namespace onnxruntime { + +class CPUIDInfo { +public: + static const CPUIDInfo& GetCPUIDInfo() { + static CPUIDInfo cpuid_info; + return cpuid_info; + } + + bool HasAVX2() const { return has_avx2_; } + bool HasAVX512f() const { return has_avx512f_; } + +private: + CPUIDInfo() noexcept; + bool has_avx2_{false}; + bool has_avx512f_{false}; +}; + +} diff --git a/onnxruntime/core/common/logging/capture.cc b/onnxruntime/core/common/logging/capture.cc new file mode 100644 index 0000000000000..016ddb9fc06be --- /dev/null +++ b/onnxruntime/core/common/logging/capture.cc @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/logging/capture.h" +#include "core/common/logging/logging.h" +#include "gsl/span" +#include "gsl/gsl_util" + +namespace onnxruntime { +namespace logging { + +void Capture::CapturePrintf(msvc_printf_check const char* format, ...) { + va_list arglist; + va_start(arglist, format); + + ProcessPrintf(format, arglist); + + va_end(arglist); +} + +// from https://github.com/KjellKod/g3log/blob/master/src/logcapture.cpp LogCapture::capturef +// License: https://github.com/KjellKod/g3log/blob/master/LICENSE +// Modifications Copyright (c) Microsoft. +void Capture::ProcessPrintf(msvc_printf_check const char* format, va_list args) { + static constexpr auto kTruncatedWarningText = "[...truncated...]"; + static const int kMaxMessageSize = 2048; + char message_buffer[kMaxMessageSize]; + const auto message = gsl::make_span(message_buffer); + +#if (defined(WIN32) || defined(_WIN32) || defined(__WIN32__) && !defined(__GNUC__)) + const int nbrcharacters = vsnprintf_s(message.data(), message.size(), _TRUNCATE, format, args); +#else + const int nbrcharacters = vsnprintf(message.data(), message.size(), format, args); +#endif + + if (nbrcharacters <= 0) { + stream_ << "\n\tERROR LOG MSG NOTIFICATION: Failure to successfully parse the message"; + stream_ << '"' << format << '"' << std::endl; + } else if (nbrcharacters > message.size()) { + stream_ << message.data() << kTruncatedWarningText; + } else { + stream_ << message.data(); + } +} + +Capture::~Capture() { + if (logger_ != nullptr) { + logger_->Log(*this); + } +} +} // namespace logging +} // namespace onnxruntime diff --git a/onnxruntime/core/common/logging/logging.cc b/onnxruntime/core/common/logging/logging.cc new file mode 100644 index 0000000000000..6f1d424371d92 --- /dev/null +++ b/onnxruntime/core/common/logging/logging.cc @@ -0,0 +1,227 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include +#include + +#include "core/common/exceptions.h" +#include "core/common/logging/isink.h" +#include "core/common/logging/logging.h" + +#ifdef _WIN32 +#include +#else +#include +#if defined(__MACH__) +#include +#else +#include +#endif +#endif + +namespace onnxruntime { +namespace logging { +const char* Category::onnxruntime = "onnxruntime"; +const char* Category::System = "System"; + +using namespace std::chrono; + +/* +As LoggingManager can be a static, we need to wrap the default instance and mutex in functions +to ensure they're initialized before use in LoggingManager::LoggingManager. If we don't, and +a static LoggingManager is created at startup, the file scope statics here may not have been +initialized. +*/ + +static std::atomic& DefaultLoggerManagerInstance() noexcept { + // this atomic is to protect against attempts to log being made after the default LoggingManager is destroyed. + // Theoretically this can happen if a Logger instance is still alive and calls Log via its internal + // pointer to the LoggingManager. + // As the first thing LoggingManager::Log does is check the static DefaultLoggerManagerInstance() is not null, + // any further damage should be prevented (in theory). + static std::atomic default_instance; + return default_instance; +} + +// GSL_SUPRESS(i.22) is broken. Ignore the warnings for the static local variables that are trivial +// and should not have any destruction order issues via pragmas instead. +// https://developercommunity.visualstudio.com/content/problem/249706/gslsuppress-does-not-work-for-i22-c-core-guideline.html +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 26426) +#endif + +static std::mutex& DefaultLoggerMutex() noexcept { + static std::mutex mutex; + return mutex; +} + +Logger* LoggingManager::s_default_logger_ = nullptr; + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +static minutes InitLocaltimeOffset(const time_point& epoch) noexcept; + +const LoggingManager::Epochs& LoggingManager::GetEpochs() noexcept { + // we save the value from system clock (which we can convert to a timestamp) as well as the high_resolution_clock. + // from then on, we use the delta from the high_resolution_clock and apply that to the + // system clock value. + static Epochs epochs{high_resolution_clock::now(), + system_clock::now(), + InitLocaltimeOffset(system_clock::now())}; + return epochs; +} + +LoggingManager::LoggingManager(std::unique_ptr sink, Severity default_min_severity, bool filter_user_data, + const InstanceType instance_type, const std::string* default_logger_id, + int default_max_vlog_level) + : sink_{std::move(sink)}, + default_min_severity_{default_min_severity}, + default_filter_user_data_{filter_user_data}, + default_max_vlog_level_{default_max_vlog_level}, + owns_default_logger_{false} { + if (!sink_) { + throw std::logic_error("ISink must be provided."); + } + + if (instance_type == InstanceType::Default) { + if (default_logger_id == nullptr) { + throw std::logic_error("default_logger_id must be provided if instance_type is InstanceType::Default"); + } + + // lock mutex to create instance, and enable logging + // this matches the mutex usage in Shutdown + std::lock_guard guard(DefaultLoggerMutex()); + + if (DefaultLoggerManagerInstance().load() != nullptr) { + throw std::logic_error("Only one instance of LoggingManager created with InstanceType::Default can exist at any point in time."); + } + + // This assertion passes, so using the atomic to validate calls to Log should + // be reasonably economical. + // assert(DefaultLoggerManagerInstance().is_lock_free()); + DefaultLoggerManagerInstance().store(this); + + CreateDefaultLogger(*default_logger_id); + + owns_default_logger_ = true; + } +} + +LoggingManager::~LoggingManager() { + if (owns_default_logger_) { + // lock mutex to reset DefaultLoggerManagerInstance() and free default logger from this instance. + std::lock_guard guard(DefaultLoggerMutex()); + + DefaultLoggerManagerInstance().store(nullptr, std::memory_order::memory_order_release); + + delete s_default_logger_; + s_default_logger_ = nullptr; + } +} + +void LoggingManager::CreateDefaultLogger(const std::string& logger_id) { + // this method is only called from ctor in scope where DefaultLoggerMutex() is already locked + + if (s_default_logger_ != nullptr) { + throw std::logic_error("Default logger already set. "); + } + + s_default_logger_ = CreateLogger(logger_id).release(); +} + +std::unique_ptr LoggingManager::CreateLogger(const std::string& logger_id) { + return CreateLogger(logger_id, default_min_severity_, default_filter_user_data_, default_max_vlog_level_); +} + +std::unique_ptr LoggingManager::CreateLogger(const std::string& logger_id, + const Severity severity, + bool filter_user_data, + int vlog_level) { + auto logger = std::make_unique(*this, logger_id, severity, filter_user_data, vlog_level); + return logger; +} + +void LoggingManager::Log(const std::string& logger_id, const Capture& message) const { + sink_->Send(GetTimestamp(), logger_id, message); +} + +void LoggingManager::SendProfileEvent(profiling::EventRecord& eventRecord) const { + sink_->SendProfileEvent(eventRecord); +} + +static minutes InitLocaltimeOffset(const time_point& epoch) noexcept { + // convert the system_clock time_point (UTC) to localtime and gmtime to calculate the difference. + // we do this once, and apply that difference in GetTimestamp(). + // NOTE: If we happened to be running over a period where the time changed (e.g. daylight saving started) + // we won't pickup the change. Not worth the extra cost to be 100% accurate 100% of the time. + + const time_t system_time_t = system_clock::to_time_t(epoch); + tm local_tm; + tm utc_tm; + +#ifdef _WIN32 + localtime_s(&local_tm, &system_time_t); + gmtime_s(&utc_tm, &system_time_t); +#else + localtime_r(&system_time_t, &local_tm); + gmtime_r(&system_time_t, &utc_tm); +#endif + + const double seconds = difftime(mktime(&local_tm), mktime(&utc_tm)); + + // minutes should be accurate enough for timezone conversion + return minutes{static_cast(seconds / 60)}; +} + +std::exception LoggingManager::LogFatalAndCreateException(const char* category, + const CodeLocation& location, + const char* format_str, ...) { + std::string exception_msg; + + // create Capture in separate scope so it gets destructed (leading to log output) before we throw. + { + ::onnxruntime::logging::Capture c{::onnxruntime::logging::LoggingManager::DefaultLogger(), + ::onnxruntime::logging::Severity::kFATAL, category, ::onnxruntime::logging::DataType::SYSTEM, location}; + va_list args; + va_start(args, format_str); + + c.ProcessPrintf(format_str, args); + va_end(args); + + exception_msg = c.Message(); + } + + return OnnxRuntimeException(location, exception_msg); +} + +unsigned int GetThreadId() { +#ifdef _WIN32 + return static_cast(GetCurrentThreadId()); +#elif defined(__MACH__) + uint64_t tid64; + pthread_threadid_np(NULL, &tid64); + return static_cast(tid64); +#else + return static_cast(syscall(SYS_gettid)); +#endif +} + +// +// Get current process id +// +unsigned int GetProcessId() { +#ifdef _WIN32 + return static_cast(GetCurrentProcessId()); +#elif defined(__MACH__) + return static_cast(getpid()); +#else + return static_cast(syscall(SYS_getpid)); +#endif +} + +} // namespace logging +} // namespace onnxruntime diff --git a/onnxruntime/core/common/logging/sinks/cerr_sink.h b/onnxruntime/core/common/logging/sinks/cerr_sink.h new file mode 100644 index 0000000000000..42577ba26dc99 --- /dev/null +++ b/onnxruntime/core/common/logging/sinks/cerr_sink.h @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include "core/common/logging/sinks/ostream_sink.h" + +namespace onnxruntime { +namespace logging { +/// +/// A std::cerr based ISink +/// +/// +class CErrSink : public OStreamSink { + public: + CErrSink() : OStreamSink(std::cerr, /*flush*/ false) { // std::cerr isn't buffered so no flush required + } +}; +} // namespace logging +} // namespace onnxruntime diff --git a/onnxruntime/core/common/logging/sinks/clog_sink.h b/onnxruntime/core/common/logging/sinks/clog_sink.h new file mode 100644 index 0000000000000..9b0adf92f81b1 --- /dev/null +++ b/onnxruntime/core/common/logging/sinks/clog_sink.h @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include "core/common/logging/sinks/ostream_sink.h" + +namespace onnxruntime { +namespace logging { +/// +/// A std::clog based ISink +/// +/// +class CLogSink : public OStreamSink { + public: + CLogSink() : OStreamSink(std::clog, /*flush*/ true) { + } +}; +} // namespace logging +} // namespace onnxruntime diff --git a/onnxruntime/core/common/logging/sinks/composite_sink.h b/onnxruntime/core/common/logging/sinks/composite_sink.h new file mode 100644 index 0000000000000..f27abb9e6aad5 --- /dev/null +++ b/onnxruntime/core/common/logging/sinks/composite_sink.h @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include + +#include "core/common/logging/isink.h" +#include "core/common/logging/logging.h" + +namespace onnxruntime { +namespace logging { +/// +/// Class that abstracts multiple ISink instances being written to. +/// +/// +class CompositeSink : public ISink { + public: + /// + /// Initializes a new instance of the class. + /// Use AddSink to add sinks. + /// + CompositeSink() {} + + /// + /// Adds a sink. Takes ownership of the sink (so pass unique_ptr by value). + /// + /// The sink. + /// This instance to allow chaining. + CompositeSink& AddSink(std::unique_ptr sink) { + sinks_.push_back(std::move(sink)); + return *this; + } + + private: + void SendImpl(const Timestamp& timestamp, const std::string& logger_id, const Capture& message) override { + for (auto& sink : sinks_) { + sink->Send(timestamp, logger_id, message); + } + } + + std::vector> sinks_; +}; +} // namespace logging +} // namespace onnxruntime diff --git a/onnxruntime/core/common/logging/sinks/file_sink.h b/onnxruntime/core/common/logging/sinks/file_sink.h new file mode 100644 index 0000000000000..ba3ff3e0b3372 --- /dev/null +++ b/onnxruntime/core/common/logging/sinks/file_sink.h @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include "core/common/logging/sinks/ostream_sink.h" + +namespace onnxruntime { +namespace logging { +/// +/// ISink that writes to a file. +/// +/// +class FileSink : public OStreamSink { + public: + /// + /// Initializes a new instance of the class. + /// + /// The filename to write to. + /// If set to true [append to file]. Otherwise truncate. + /// If set to true [removes user data]. + /// Filtering of user data can alternatively be done at the level. + FileSink(std::unique_ptr file, bool filter_user_data) + : OStreamSink(*file, /*flush*/ true), file_(std::move(file)), filter_user_data_{filter_user_data} { + } + + /// + /// Initializes a new instance of the class. + /// + /// The filename to write to. + /// If set to true [append to file]. Otherwise truncate. + /// If set to true [removes user data]. + /// Filtering of user data can alternatively be done at the level. + FileSink(const std::string& filename, bool append, bool filter_user_data) + : FileSink{std::make_unique(filename, std::ios::out | (append ? std::ios::app : std::ios::trunc)), + filter_user_data} { + } + + private: + void SendImpl(const Timestamp& timestamp, const std::string& logger_id, const Capture& message) override { + if (!filter_user_data_ || message.DataType() != DataType::USER) { + OStreamSink::SendImpl(timestamp, logger_id, message); + } + } + + std::unique_ptr file_; + bool filter_user_data_; +}; +} // namespace logging +} // namespace onnxruntime diff --git a/onnxruntime/core/common/logging/sinks/ostream_sink.cc b/onnxruntime/core/common/logging/sinks/ostream_sink.cc new file mode 100644 index 0000000000000..1d689b25d8466 --- /dev/null +++ b/onnxruntime/core/common/logging/sinks/ostream_sink.cc @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/logging/sinks/ostream_sink.h" +#include "date/date.h" + +namespace onnxruntime { +namespace logging { + +void OStreamSink::SendImpl(const Timestamp& timestamp, const std::string& logger_id, const Capture& message) { + // operator for formatting of timestamp in ISO8601 format including microseconds + using date::operator<<; + + // Two options as there may be multiple calls attempting to write to the same sink at once: + // 1) Use mutex to synchronize access to the stream. + // 2) Create the message in an ostringstream and output in one call. + // + // Going with #2 as it should scale better at the cost of creating the message in memory first + // before sending to the stream. + + std::ostringstream msg; + + msg << timestamp << " [" << message.SeverityPrefix() << ":" << message.Category() << ":" << logger_id << ", " + << message.Location().ToString() << "] " << message.Message(); + + (*stream_) << msg.str() << "\n"; + + if (flush_) { + stream_->flush(); + } +} +} // namespace logging +} // namespace onnxruntime diff --git a/onnxruntime/core/common/logging/sinks/ostream_sink.h b/onnxruntime/core/common/logging/sinks/ostream_sink.h new file mode 100644 index 0000000000000..bf5cec174a87d --- /dev/null +++ b/onnxruntime/core/common/logging/sinks/ostream_sink.h @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include + +#include "core/common/logging/capture.h" +#include "core/common/logging/isink.h" + +namespace onnxruntime { +namespace logging { +/// +/// A std::ostream based ISink +/// +/// +class OStreamSink : public ISink { + protected: + OStreamSink(std::ostream& stream, bool flush) + : stream_{&stream}, flush_{flush} { + } + + public: + void SendImpl(const Timestamp& timestamp, const std::string& logger_id, const Capture& message) override; + + private: + std::ostream* stream_; + const bool flush_; +}; +} // namespace logging +} // namespace onnxruntime diff --git a/onnxruntime/core/common/profiler.cc b/onnxruntime/core/common/profiler.cc new file mode 100644 index 0000000000000..cc3c87d0c7701 --- /dev/null +++ b/onnxruntime/core/common/profiler.cc @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "profiler.h" + +namespace onnxruntime { +namespace profiling { +using namespace std::chrono; + +::onnxruntime::TimePoint profiling::Profiler::StartTime() const { + return std::chrono::high_resolution_clock::now(); +} + +void Profiler::Initialize(const logging::Logger* session_logger) { + ONNXRUNTIME_ENFORCE(session_logger != nullptr); + session_logger_ = session_logger; +} + +void Profiler::StartProfiling(const logging::Logger* custom_logger) { + ONNXRUNTIME_ENFORCE(custom_logger != nullptr); + profile_with_logger_ = true; + custom_logger_ = custom_logger; + profiling_start_time_ = StartTime(); +} + +void Profiler::StartProfiling(const std::string& file_name) { + enabled_ = true; + profile_stream_ = std::ofstream(file_name, std::ios::out | std::ios::trunc); + profile_stream_file_ = file_name; + profiling_start_time_ = StartTime(); +} + +void Profiler::EndTimeAndRecordEvent(EventCategory category, + const std::string& event_name, + TimePoint& start_time, + std::unordered_map&& event_args, + bool /*sync_gpu*/) { + if (!enabled_ && !profile_with_logger_) + return; + long long dur = TimeDiffMicroSeconds(start_time); + long long ts = TimeDiffMicroSeconds(profiling_start_time_, start_time); + EventRecord event(category, logging::GetProcessId(), + logging::GetThreadId(), event_name, ts, dur, std::move(event_args)); + if (profile_with_logger_) { + custom_logger_->SendProfileEvent(event); + } else { + //TODO: sync_gpu if needed. + std::lock_guard lock(mutex_); + if (events_.size() < max_num_events_) { + events_.emplace_back(event); + } else { + if (session_logger_ && !max_events_reached) { + LOGS(*session_logger_, ERROR) + << "Maximum number of events reached, could not record profile event."; + max_events_reached = true; + } + } + } +} + +std::string Profiler::EndProfiling() { + if (!enabled_) { + return std::string(); + } + if (profile_with_logger_) { + profile_with_logger_ = false; + return std::string(); + } + std::lock_guard lock(mutex_); + profile_stream_ << "[\n"; + + for (size_t i = 0; i < events_.size(); ++i) { + auto& rec = events_[i]; + profile_stream_ << R"({"cat" : ")" << event_categor_names_[rec.cat] << "\","; + profile_stream_ << "\"pid\" :" << rec.pid << ","; + profile_stream_ << "\"tid\" :" << rec.tid << ","; + profile_stream_ << "\"dur\" :" << rec.dur << ","; + profile_stream_ << "\"ts\" :" << rec.ts << ","; + profile_stream_ << R"("ph" : "X",)"; + profile_stream_ << R"("name" :")" << rec.name << "\","; + profile_stream_ << "\"args\" : {"; + bool is_first_arg = true; + for (std::pair event_arg : rec.args) { + if (!is_first_arg) profile_stream_ << ","; + profile_stream_ << "\"" << event_arg.first << "\" : \"" << event_arg.second << "\""; + is_first_arg = false; + } + profile_stream_ << "}"; + if (i == events_.size() - 1) { + profile_stream_ << "}\n"; + } else { + profile_stream_ << "},\n"; + } + } + profile_stream_ << "]\n"; + profile_stream_.close(); + enabled_ = false; // will not collect profile after writing. + return profile_stream_file_; +} + +// +// Conditionally sync the GPU if the syncGPU flag is set. +// +void ProfilerSyncGpu() { + ONNXRUNTIME_NOT_IMPLEMENTED("Needs to implement only for gpus"); +} + +} // namespace profiling +} // namespace onnxruntime diff --git a/onnxruntime/core/common/profiler.h b/onnxruntime/core/common/profiler.h new file mode 100644 index 0000000000000..8a1e7daa5ef23 --- /dev/null +++ b/onnxruntime/core/common/profiler.h @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include +#include "core/common/logging/logging.h" + +namespace onnxruntime { + +namespace profiling { + +/* +Main class for profiling. It continues to accumulate events and produce +a corresponding "complete event (X)" in "chrome tracing" format. +*/ +class Profiler { + public: + Profiler() noexcept {}; // turned off by default. + + /* + Initializes Profiler with the session logger to log framework specific messages + */ + void Initialize(const logging::Logger* session_logger); + + /* + Send profiling data to custom logger + */ + void StartProfiling(const logging::Logger* custom_logger); + + /* + Start profiler and record beginning time. + */ + void StartProfiling(const std::string& file_name); + + /* + Produce current time point for any profiling action. + */ + TimePoint StartTime() const; + + /* + Record a single event. Time is measured till the call of this function from + the start_time. + */ + void EndTimeAndRecordEvent(EventCategory category, + const std::string& event_name, + TimePoint& start_time, + std::unordered_map&& event_args = std::unordered_map(), + bool sync_gpu = false); + + /* + Write profile data to the given stream in chrome format defined below. + https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview# + */ + std::string EndProfiling(); + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Profiler); + + // Mutex controlling access to profiler data + std::mutex mutex_; + bool enabled_{false}; + std::ofstream profile_stream_; + std::string profile_stream_file_; + const logging::Logger* session_logger_{nullptr}; + const logging::Logger* custom_logger_{nullptr}; + TimePoint profiling_start_time_; + std::vector events_; + bool max_events_reached{false}; + static constexpr size_t max_num_events_ = 1000000; + bool profile_with_logger_{false}; +}; + +} // namespace profiling +} // namespace onnxruntime diff --git a/onnxruntime/core/common/status.cc b/onnxruntime/core/common/status.cc new file mode 100644 index 0000000000000..a5bc2d6f5ee8d --- /dev/null +++ b/onnxruntime/core/common/status.cc @@ -0,0 +1,94 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +// Modifications Copyright (c) Microsoft. + +#include "core/common/status.h" +#include "core/common/common.h" + +namespace onnxruntime { +namespace common { +Status::Status(StatusCategory category, int code, const std::string& msg) { + // state_ will be allocated here causing the status to be treated as a failure + ONNXRUNTIME_ENFORCE(code != static_cast(MLStatus::OK)); + + state_ = std::make_unique(category, code, msg); +} + +Status::Status(StatusCategory category, int code) + : Status(category, code, EmptyString()) { +} + +bool Status::IsOK() const noexcept { + return (state_ == nullptr); +} + +StatusCategory Status::Category() const noexcept { + return IsOK() ? common::NONE : state_->category; +} + +int Status::Code() const noexcept { + return IsOK() ? static_cast(common::OK) : state_->code; +} + +const std::string& Status::ErrorMessage() const noexcept { + return IsOK() ? EmptyString() : state_->msg; +} + +std::string Status::ToString() const { + if (state_ == nullptr) { + return std::string("OK"); + } + + std::string result; + + if (common::SYSTEM == state_->category) { + result += "SystemError"; + result += " : "; + result += std::to_string(errno); + } else if (common::ONNXRUNTIME == state_->category) { + result += "[ONNXRuntimeError]"; + result += " : "; + result += std::to_string(Code()); + std::string msg; + + result += " : "; + result += MLStatusToString(static_cast(Code())); + result += " : "; + result += state_->msg; + } + + return result; +} + +// GSL_SUPRESS(i.22) is broken. Ignore the warnings for the static local variables that are trivial +// and should not have any destruction order issues via pragmas instead. +// https://developercommunity.visualstudio.com/content/problem/249706/gslsuppress-does-not-work-for-i22-c-core-guideline.html +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 26426) +#endif +const Status& Status::OK() noexcept { + static Status s_ok; + return s_ok; +} + +const std::string& Status::EmptyString() noexcept { + static std::string s_empty; + return s_empty; +} + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +} // namespace common +} // namespace onnxruntime diff --git a/onnxruntime/core/common/task_thread_pool.h b/onnxruntime/core/common/task_thread_pool.h new file mode 100644 index 0000000000000..217c651896bf3 --- /dev/null +++ b/onnxruntime/core/common/task_thread_pool.h @@ -0,0 +1,203 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* +Changed to use std::packaged_task instead of std::function so exceptions can be propagated. + +This also allows the task threadpool to be shared across multiple operators as the caller +can keep a container of the packaged_task futures to check when they have completed. Calling +WaitWorkComplete in that use case is invalid as there may be other concurrent usage of the +threadpool. + +Example of that usage: + + std::vector> task_results{}; + + for (...) { + std::packaged_task task{std::bind(lambda, i)}; + task_results.push_back(task.get_future()); + task_thread_pool.RunTask(std::move(task)); + } + + try { + // wait for all and propagate any exceptions + for (auto& future : task_results) + future.get(); + } catch (const std::exception& ex) { + ... + throw; + } + +*/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "core/common/common.h" +#include "core/common/logging/logging.h" + +namespace onnxruntime { + +class TaskThreadPool { + private: + struct task_element_t { + bool run_with_id; + std::packaged_task no_id; + std::packaged_task with_id; + + task_element_t(task_element_t&& other) { + run_with_id = other.run_with_id; + no_id = std::move(other.no_id); + with_id = std::move(other.with_id); + } + + explicit task_element_t(std::packaged_task&& f) + : run_with_id(false), no_id(std::move(f)) {} + + explicit task_element_t(std::packaged_task&& f) + : run_with_id(true), with_id(std::move(f)) {} + }; + + std::queue tasks_; + std::vector threads_; + std::mutex mutex_; + std::condition_variable condition_; + std::condition_variable completed_; + bool running_; + bool complete_; + std::size_t available_; + std::size_t total_; + + public: + /// @brief Constructor. + explicit TaskThreadPool(std::size_t pool_size) + : threads_(pool_size), running_(true), complete_(true), available_(pool_size), total_(pool_size) { + for (std::size_t i = 0; i < pool_size; ++i) { + threads_[i] = std::thread(std::bind(&TaskThreadPool::MainLoop, this, i)); + } + } + + /// @brief Destructor. + ~TaskThreadPool() { + // Set running flag to false then notify all threads. + { + std::unique_lock lock(mutex_); + running_ = false; + condition_.notify_all(); + } + + try { + for (auto& t : threads_) { + t.join(); + } + } + // Suppress all exceptions. + catch (const std::exception& ex) { + LOGS_DEFAULT(ERROR) << "Exception joining threads in TaskThreadPool: " << ex.what(); + } + } + + void RunTask(std::packaged_task&& task) { + std::unique_lock lock(mutex_); + + // Set task and signal condition variable so that a worker thread will + // wake up and use the task. + tasks_.push(task_element_t(std::move(task))); + complete_ = false; + condition_.notify_one(); + } + + void RunTaskWithID(std::packaged_task&& task) { + std::unique_lock lock(mutex_); + + // Set task and signal condition variable so that a worker thread will + // wake up and use the task. + tasks_.push(task_element_t(std::move(task))); + complete_ = false; + condition_.notify_one(); + } + + /// @brief Wait for queue to be empty + void WaitWorkComplete() { + std::unique_lock lock(mutex_); + while (!complete_) + completed_.wait(lock); + } + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(TaskThreadPool); + + /// @brief Entry point for pool threads. + void MainLoop(std::size_t index) { + while (running_) { + // Wait on condition variable while the task is empty and + // the pool is still running. + std::unique_lock lock(mutex_); + while (tasks_.empty() && running_) { + condition_.wait(lock); + } + + // If pool is no longer running, break out of loop. + if (!running_) break; + + // Copy task locally and remove from the queue. This is + // done within its own scope so that the task object is + // destructed immediately after running the task. This is + // useful in the event that the function contains + // shared_ptr arguments bound via bind. + { + auto task = std::move(tasks_.front()); + tasks_.pop(); + // Decrement count, indicating thread is no longer available. + --available_; + + lock.unlock(); + + // Run the task. + try { + if (task.run_with_id) { + task.with_id(index); + } else { + task.no_id(); + } + } catch (const std::exception& /*ex*/) { + // LOGS_DEFAULT(ERROR) << "Exception running TaskThreadPool task: " << ex.what(); + throw; + } + + // Update status of empty, maybe + // Need to recover the lock first + lock.lock(); + + // Increment count, indicating thread is available. + ++available_; + if (tasks_.empty() && available_ == total_) { + complete_ = true; + completed_.notify_one(); + } + } + } // while running_ + } +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/allocation_planner.cc b/onnxruntime/core/framework/allocation_planner.cc new file mode 100644 index 0000000000000..d033e27e84dc0 --- /dev/null +++ b/onnxruntime/core/framework/allocation_planner.cc @@ -0,0 +1,597 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/allocation_planner.h" +#include +#include +#include +#include +#include "core/common/exceptions.h" +#include "core/platform/env.h" +#include "core/framework/data_types.h" +#include "core/framework/kernel_def_builder.h" +#include "core/framework/mldata_type_utils.h" +#include "core/framework/op_kernel.h" +#include "core/framework/session_state.h" +#include "core/framework/utils.h" + +using namespace onnxruntime::common; +using namespace ONNX_NAMESPACE; +namespace onnxruntime { + +std::ostream& operator<<(std::ostream& out, AllocKind alloc_kind) { + switch (alloc_kind) { + case AllocKind::kAllocate: + out << "Allocate"; + break; + case AllocKind::kAllocateStatically: + out << "AllocateStatically"; + break; + case AllocKind::kPreExisting: + out << "PreExisting"; + break; + case AllocKind::kReuse: + out << "Reuse"; + break; + case AllocKind::kAllocateOutput: + out << "AllocateOutput"; + break; + } + return out; +} + +// Output details of an execution plan: +std::ostream& operator<<(std::ostream& out, std::pair planinfo) { + const SequentialExecutionPlan& plan = *planinfo.first; + const SessionState& session_state = *planinfo.second; + auto& graph = *session_state.GetGraphViewer(); + std::unordered_map index_to_name; + + out << "Allocation Plan:\n"; + auto plan_size = plan.allocation_plan.size(); + + for (auto& name_index : session_state.GetMLValueNameIdxMap()) { + auto index = name_index.second; + index_to_name[index] = name_index.first; + out << "(" << index << ") " << name_index.first << " : "; + if (0 <= index && static_cast(index) < plan_size) { + auto& elt_plan = plan.allocation_plan[index]; + out << elt_plan.alloc_kind; + if (elt_plan.alloc_kind == AllocKind::kReuse) out << " " << elt_plan.reused_buffer; + + auto& loc = elt_plan.location; + out << ", " << loc.ToString(); + + if (elt_plan.create_fence_if_async) out << ", use fence when async"; + + } else { + out << "Index out-of-range!"; + } + + out << std::endl; + } + + out << "\nExecution Plan:\n"; + for (size_t i = 0; i < plan.execution_plan.size(); ++i) { + auto& step = plan.execution_plan[i]; + auto node = graph.GetNode(step.node_index); + ONNXRUNTIME_ENFORCE(nullptr != node); + out << "[" << i << "] "; + out << node->OpType() << " (" << node->Name() << ")" << std::endl; + if (step.free_from_index <= step.free_to_index) { + out << "Free ml-values: "; + std::string sep; + for (int j = step.free_from_index; j <= step.free_to_index; ++j) { + auto freed_value_index = plan.to_be_freed[j]; + auto name_iter = index_to_name.find(freed_value_index); + auto name = (name_iter == index_to_name.end()) ? "INVALID INDEX" : name_iter->second; + out << sep << "(" << freed_value_index << ") " << name; + sep = ", "; + } + out << std::endl; + } + } + + return out; +} + +class PlannerImpl { + public: + PlannerImpl(const onnxruntime::GraphViewer& graph_viewer, + const ExecutionProviders& providers, + const KernelRegistryManager& kernel_registry, + const MLValueNameIdxMap& mlvalue_name_idx_map, + const ISequentialPlannerContext& context, + SequentialExecutionPlan& plan) + : context_{context}, + plan_{plan}, + graph_viewer_(graph_viewer), + execution_providers_{providers}, + kernel_registry_{kernel_registry}, + mlvalue_name_idx_map_{mlvalue_name_idx_map} { + } + + Status CreatePlan(); + + private: + const ISequentialPlannerContext& context_; + SequentialExecutionPlan& plan_; + + const onnxruntime::GraphViewer& graph_viewer_; + const ExecutionProviders& execution_providers_; + + const KernelRegistryManager& kernel_registry_; + const MLValueNameIdxMap& mlvalue_name_idx_map_; + + // MLValueInfo: Auxiliary information about an MLValue used only during plan-generation: + struct MLValueInfo { + const onnxruntime::NodeArg* p_def_site; // the (unique) NodeArg corresponding to the MLValue + int usecount = 0; // static reference-count + MLValueIndex reused_buffer_index; // index of original buffer to reuse + }; + + // ml_value_info_ is indexed by an MLValueIndex + std::vector ml_value_info_; + + // FreeBufferInfo is used to track information about ml-values whose buffers are + // free to be reused. + struct FreeBufferInfo { + MLValueIndex ml_value; + // deallocate_point is an index into the execution-plan; thus, ml_value becomes free after + // this step in the execution-plan is completed. + size_t deallocate_point; + FreeBufferInfo(MLValueIndex mlvalue, size_t dealloc_point) : ml_value(mlvalue), deallocate_point(dealloc_point) {} + }; + // freelist_ : a list of ml-values whose buffers are free to be reused, sorted by when + // they became free (more recently freed earlier in the list). + std::list freelist_; + + MLValueIndex Index(const MLValueName& name) { + MLValueIndex result; + auto status = mlvalue_name_idx_map_.GetIdx(name, result); + ONNXRUNTIME_ENFORCE(status.IsOK(), status.ErrorMessage()); + return result; + } + + int& UseCount(MLValueIndex n) { return ml_value_info_.at(n).usecount; } + int& UseCount(const MLValueName& name) { return UseCount(Index(name)); } + + MLValueIndex& Buffer(MLValueIndex n) { return ml_value_info_.at(n).reused_buffer_index; } + + SequentialExecutionPlan::AllocPlanPerValue& AllocPlan(MLValueIndex n) { + return plan_.allocation_plan.at(n); + } + + SequentialExecutionPlan::AllocPlanPerValue& AllocPlan(const MLValueName& name) { + return AllocPlan(Index(name)); + } + + // Initialize state for a given ml-value at its definition site: + void ProcessDef(MLValueIndex id, const onnxruntime::NodeArg* p_def_site) { + MLValueInfo& info = ml_value_info_.at(id); + info.usecount = 0; + info.reused_buffer_index = id; // initially, no reuse; the ml-value uses its own buffer + info.p_def_site = p_def_site; + } + + void Reuse(MLValueIndex reused, MLValueIndex reused_for) { + ONNXRUNTIME_ENFORCE(reused != reused_for); + // find original buffer underlying ml-value we want to reuse: + MLValueIndex original = Buffer(reused); + // record that the new buffer will reuse that original buffer + Buffer(reused_for) = original; + // adjust original buffer's usecount + UseCount(original) += UseCount(reused_for); + + // update allocation plan (for use at execution-time) + auto& symplan = AllocPlan(reused_for); + symplan.alloc_kind = AllocKind::kReuse; + symplan.reused_buffer = original; + } + + // Find if there exists some input tensor that we can use in-place for output_arg + bool FindReusableInput(const onnxruntime::Node& node, int output_arg_num, MLValueIndex* reusable_input) { + auto p_output_arg = node.OutputDefs()[output_arg_num]; + auto p_opkernel_def = utils::GetKernelDef(kernel_registry_, node); + + // Note: We expect a KernelDef to be available at this point. If it is not available, the + // planner would have returned an error status earlier on. + ONNXRUNTIME_ENFORCE(nullptr != p_opkernel_def); + + const std::vector>& alias_map = p_opkernel_def->Alias(); + auto& input_args = node.InputDefs(); + for (auto pair : alias_map) { + if (pair.second == output_arg_num) { + // we _must_ reuse this input to satisfy aliasing requirement: (e.g., for reshape) + if ((0 <= pair.first) && (static_cast(pair.first) < input_args.size())) { + auto p_input_arg = input_args[pair.first]; + if (p_input_arg->Exists()) { + *reusable_input = Index(p_input_arg->Name()); + return true; + } + } + } + } + + const std::vector>& inplace_map = p_opkernel_def->MayInplace(); + for (auto pair : inplace_map) { + if (pair.second == output_arg_num) { + if ((0 <= pair.first) && (static_cast(pair.first) < input_args.size())) { + auto p_input_arg = input_args[pair.first]; + if (p_input_arg->Exists()) { + auto input_arg_index = Index(p_input_arg->Name()); + auto original = Buffer(input_arg_index); + if (1 == UseCount(original)) { + if (SameSize(*p_input_arg, *p_output_arg)) { + // we can reuse this input since it is its last use and permitted for in-place update + *reusable_input = input_arg_index; // or original; both should be okay + return true; + } + } + } + } + } + } + return false; + } + + bool SameShape(const TensorShapeProto& shape1, const TensorShapeProto& shape2) { + // TODO: This should probably be defined to be the equality operator on TensorShapeProto. + int rank1 = shape1.dim_size(); + if (shape2.dim_size() != rank1) return false; + for (int i = 0; i < rank1; i++) { + const auto& val1 = shape1.dim(i); + const auto& val2 = shape2.dim(i); + if (val1.has_dim_value() && val2.has_dim_value() && (val1.dim_value() == val2.dim_value())) + continue; // same known dimension + if (val1.has_dim_param() && val2.has_dim_param() && (val1.dim_param() == val2.dim_param())) + continue; // same unknown dimension + return false; + } + return true; + } + + /*! \brief Given a tensor-type, return the size of an element of the tensor. + */ + size_t GetElementSize(const DataType& tensor_type) { + const TypeProto& type_proto = ONNX_NAMESPACE::Utils::DataTypeUtils::ToTypeProto(tensor_type); + MLDataType ml_data_type = DataTypeImpl::TypeFromProto(type_proto); + const TensorTypeBase* tensor_type_base = ml_data_type->AsTensorType(); + ONNXRUNTIME_ENFORCE(nullptr != tensor_type_base); + MLDataType elt_type = tensor_type_base->GetElementType(); + return elt_type->Size(); + } + + bool SameSize(const TensorShapeProto& shape1, const DataType& ptype1, + const TensorShapeProto& shape2, const DataType& ptype2) { + return (GetElementSize(ptype1) == GetElementSize(ptype2)) && SameShape(shape1, shape2); + + /* TODO: we can improve this if the concrete shapes are known for both as below. + Unclear whether this is worthwhile though. + if (KnownSize(p_shape1) && KnownSize(p_shape2)) { + // Comparison of statically-known size + auto size1 = NumElements(p_shape1) * EltSize(ptype1); + auto size2 = NumElements(p_shape2) * EltSize(ptype2); + return size1 == size2; + } else { + // Comparison of statically-unknown size buffers + return SameElementSize(ptype1, ptype2) && SameShape(shape1, shape2); + } + */ + } + + bool SameSize(const onnxruntime::NodeArg& arg1, const onnxruntime::NodeArg& arg2) { + if ((!arg1.Exists()) || (!arg2.Exists())) return false; + auto p_shape1 = context_.GetShape(arg1); + auto p_shape2 = context_.GetShape(arg2); + // If the shapes are unknown, we conservatively assume they may be of different size. + if ((nullptr == p_shape1) || (nullptr == p_shape2)) return false; + return SameSize(*p_shape1, arg1.Type(), *p_shape2, arg2.Type()); + } + + // Find if freelist contains a buffer of the same size as output_arg + bool FindReusableTensor(const onnxruntime::NodeArg& output_arg, MLValueIndex* reusable_tensor) { + auto p_required_buffer_shape = context_.GetShape(output_arg); + if (nullptr == p_required_buffer_shape) return false; + auto required_buffer_type = output_arg.Type(); + auto& required_allocator_info = AllocPlan(output_arg.Name()).location; + + for (auto it = freelist_.begin(); it != freelist_.end(); ++it) { + auto reusable = it->ml_value; + auto p_node_arg = ml_value_info_.at(reusable).p_def_site; + auto& available_allocator_info = AllocPlan(p_node_arg->Name()).location; + if (!(available_allocator_info == required_allocator_info)) continue; + auto p_available_buffer_shape = context_.GetShape(*p_node_arg); + if (nullptr != p_available_buffer_shape) { + auto available_buffer_type = p_node_arg->Type(); + if (SameSize(*p_available_buffer_shape, available_buffer_type, + *p_required_buffer_shape, required_buffer_type)) { + *reusable_tensor = reusable; + freelist_.erase(it); + return true; + } + } + } + return false; + } + + void Initialize(size_t num_graph_nodes, size_t num_ml_values) { + // All ml-value indices must be in range 0 .. num_ml_values-1 + ml_value_info_.resize(num_ml_values); + + // Initialize execution plan: + plan_.execution_plan.reserve(num_graph_nodes); + + // Initialize allocation plan: + plan_.allocation_plan.resize(num_ml_values); + } + + Status ComputeUseCounts() { + // Note: for every ml-value, its definition must appear before all its uses in a topological sort of a valid model + + for (auto graph_input : graph_viewer_.GetInputs()) { + MLValueIndex index = Index(graph_input->Name()); + ProcessDef(index, graph_input); + UseCount(index)++; // Models caller's usage post-inference; ensures it will not be reused. + } + + // All initializers should be treated as input + for (const auto& pair : graph_viewer_.GetAllInitializedTensors()) { + const auto& initializer_name = pair.first; + MLValueIndex index = Index(initializer_name); + ProcessDef(index, graph_viewer_.GetNodeArg(pair.first)); + UseCount(initializer_name)++; + } + + for (SequentialExecutionPlan::NodeExecutionPlan& step : plan_.execution_plan) { + auto pnode = graph_viewer_.GetNode(step.node_index); + for (auto node_input : pnode->InputDefs()) { + if (node_input->Exists()) + UseCount(node_input->Name())++; + } + + for (auto node_input : pnode->ImplicitInputDefs()) { + if (node_input->Exists()) + UseCount(node_input->Name())++; + } + + // Identify where each output of this node should be allocated. + // This is determined by the opkernel bound to the node. + auto node = graph_viewer_.GetNode(step.node_index); + ONNXRUNTIME_ENFORCE(nullptr != node); + auto p_kernelDef = utils::GetKernelDef(kernel_registry_, *node); + if (nullptr == p_kernelDef) { + std::ostringstream errormsg; + errormsg << "No suitable kernel definition found for op " << pnode->OpType() << "(" << pnode->Op()->since_version() << ")"; + if (!pnode->Name().empty()) errormsg << " (node " << pnode->Name() << ")"; + return Status(ONNXRUNTIME, FAIL, errormsg.str()); + } + + auto exec_provider = execution_providers_.Get(*node); + ONNXRUNTIME_ENFORCE(exec_provider); + + auto& default_allocator_info = exec_provider->GetAllocator(0, ONNXRuntimeMemTypeDefault)->Info(); + auto& mem_type_allocated_args = p_kernelDef->OutputMemoryType(); + auto& outputs = pnode->OutputDefs(); + auto num_outputs = outputs.size(); + + for (size_t i = 0; i < num_outputs; ++i) { + auto* node_output = outputs[i]; + if (node_output->Exists()) { + MLValueIndex index = Index(node_output->Name()); + ProcessDef(index, node_output); + if (strcmp(default_allocator_info.name, CPU) != 0) { + // By default, outputs of this node are allocated on the default device allocator, + // except for outputs marked for allocation in MemoryType: + auto memory_type_iter = mem_type_allocated_args.find(i); + if (memory_type_iter == mem_type_allocated_args.end()) { + AllocPlan(index).location = default_allocator_info; + } else { + AllocPlan(index).location = exec_provider->GetAllocator(0, memory_type_iter->second)->Info(); + } + } + } + } + // if sync is needed, mark allocation plan as create_fence_if_async=true + // note that the input arg may come from an execution provider (i.e. CPU) that does not support async, + // in which case create_fence_if_async would be ignored when creating MLValue + if (p_kernelDef->ExecQueueId() != 0) { + pnode->ForEachDef([this](const onnxruntime::NodeArg* arg, bool /*is_input*/) { + MLValueIndex index = Index(arg->Name()); + AllocPlan(index).create_fence_if_async = true; + }); + } + } + + for (auto graph_output : graph_viewer_.GetOutputs()) { + UseCount(graph_output->Name())++; // Models caller's usage post-inference; ensures it will not be reused. + } + + return Status::OK(); + } + + void GeneratePlanForWeights() { + auto& weights = graph_viewer_.GetAllInitializedTensors(); + + for (auto& node : graph_viewer_.Nodes()) { + onnxruntime::Node::ForEachWithIndex( + node.InputDefs(), + [this, &node, &weights](const onnxruntime::NodeArg& def, size_t index) { + auto& def_name = def.Name(); + if (!weights.count(def_name)) + return Status::OK(); + + auto wt_index = Index(def_name); + SequentialExecutionPlan::AllocPlanPerValue& thisplan = AllocPlan(wt_index); + auto* p_provider = execution_providers_.Get(node); + ONNXRUNTIME_ENFORCE(p_provider); + + thisplan.alloc_kind = AllocKind::kAllocateStatically; + auto p_opkernelDef = utils::GetKernelDef(kernel_registry_, node); + if (MemTypeOnCpuExplicitly(p_opkernelDef->InputMemoryType(), index)) + // weights are not output from any node, so it's OK to put its location on CPU provider + thisplan.location = execution_providers_.Get(onnxruntime::kCpuExecutionProvider)->GetAllocator(0, ONNXRuntimeMemTypeDefault)->Info(); + else + thisplan.location = p_provider->GetAllocator(0, ONNXRuntimeMemTypeDefault)->Info(); + + return Status::OK(); + }); + } + } + + void ComputeReusePlan() { + std::vector& execution_plan{plan_.execution_plan}; + + // Identify allocation/deallocation plan for every ml-value + + // inputs of the graph: + // An input ml-value's data is owned by the caller (of InferenceSession::Run()) + // It must be allocated by the caller, and will not be reused during inference. + for (auto graph_input : graph_viewer_.GetInputs()) { + auto input_index = Index(graph_input->Name()); + SequentialExecutionPlan::AllocPlanPerValue& thisplan = AllocPlan(input_index); + thisplan.alloc_kind = AllocKind::kPreExisting; + thisplan.value_type = utils::GetMLDataType(*graph_input); + } + + GeneratePlanForWeights(); + + for (size_t program_counter = 0; program_counter < execution_plan.size(); ++program_counter) { + SequentialExecutionPlan::NodeExecutionPlan step = execution_plan[program_counter]; + auto pnode = graph_viewer_.GetNode(step.node_index); + // graph outputs + auto& graph_outputs = graph_viewer_.GetOutputs(); + // determine allocation for outputs of pnode + int output_arg_num = 0; + for (auto node_output : pnode->OutputDefs()) { + if (!node_output->Exists()) continue; + auto current = Index(node_output->Name()); + AllocPlan(current).value_type = utils::GetMLDataType(*node_output); + MLValueIndex reused; + if (std::find(graph_outputs.begin(), graph_outputs.end(), node_output) != graph_outputs.end()) { + // node_output is graph's output, so we can't reuse intermedia buffer + AllocPlan(current).alloc_kind = AllocKind::kAllocateOutput; + } else if (IsNonTensor(*node_output)) { + // we do not try sharing-optimization for non-tensors + AllocPlan(current).alloc_kind = AllocKind::kAllocate; + } else if (FindReusableInput(*pnode, output_arg_num, &reused)) { + // Reuse one of this node's input buffers as the output buffer (for in-place update) + Reuse(reused, current); + } else if (!context_.EnableParallelExecution() && FindReusableTensor(*node_output, &reused)) { + // Reuse an available (dead) buffer for this output, this is only for sequential execution. + Reuse(reused, current); + } else { + // otherwise: allocate a new buffer for this output + AllocPlan(current).alloc_kind = AllocKind::kAllocate; + } + output_arg_num++; + } + // determine if inputs of *pnode can be freed: + for (auto node_input : pnode->InputDefs()) { + if (node_input->Exists()) { + auto& sym = node_input->Name(); + auto original = Buffer(Index(sym)); + if (0 == --UseCount(original)) + freelist_.push_front(FreeBufferInfo(original, program_counter)); + } + } + + for (auto node_input : pnode->ImplicitInputDefs()) { + if (node_input->Exists()) { + auto& sym = node_input->Name(); + auto original = Buffer(Index(sym)); + if (0 == --UseCount(original)) + freelist_.push_front(FreeBufferInfo(original, program_counter)); + } + } + + // determine if any outputs of *pnode are unused and can be freed: + for (auto node_output : pnode->OutputDefs()) { + if (node_output->Exists()) { + auto& sym = node_output->Name(); + auto original = Buffer(Index(sym)); + if (0 == UseCount(original)) + freelist_.push_front(FreeBufferInfo(original, program_counter)); + } + } + } + } + + // Convert information in a freelist (about which ml-value becomes free when) into + // a deallocation plan in the format required in an ExecutionPlan + void GenerateDeallocationPlan() { + // Store (indices of) ml-values to be freed in plan->to_be_freed + // Set plan->execution_plan[n].free_from_index/free_to_index for every n that must free some ml-value. + + plan_.to_be_freed.reserve(freelist_.size()); + bool has_prev_dealloc_point = false; + size_t prev_dealloc_point = 0; + //TODO: should be size_t + int current = 0; // current index into the to_be_freed vector + + // Copy all items from freelist to to_be_freed in reverse order + for (auto it = freelist_.rbegin(), end = freelist_.rend(); it != end; ++it) { + plan_.to_be_freed.push_back(it->ml_value); + // + if (it->deallocate_point != prev_dealloc_point) { + if (has_prev_dealloc_point) + plan_.execution_plan[prev_dealloc_point].free_to_index = current - 1; + prev_dealloc_point = it->deallocate_point; + has_prev_dealloc_point = true; + plan_.execution_plan[prev_dealloc_point].free_from_index = current; + } + current++; + } + + if (has_prev_dealloc_point) + plan_.execution_plan[prev_dealloc_point].free_to_index = current - 1; + } + + bool IsNonTensor(const onnxruntime::NodeArg& nodearg) { + // TODO: unclear why we should go through a string-representation of type + auto ptype = nodearg.Type(); + auto& type_proto = ONNX_NAMESPACE::Utils::DataTypeUtils::ToTypeProto(ptype); + return !type_proto.has_tensor_type(); + } +}; // namespace onnxruntime + +Status PlannerImpl::CreatePlan() { + auto& p_graph_nodes = graph_viewer_.GetNodesInTopologicalOrder(); + + auto num_ml_values = mlvalue_name_idx_map_.MaxIdx() + 1; + + Initialize(p_graph_nodes.size(), num_ml_values); + + // Determine execution order: we use the default topological sort order for now. We can later + // explore more efficient orderings (from a memory usage perspective). + for (auto n : p_graph_nodes) { + plan_.execution_plan.emplace_back(n); + } + + // compute use counts for all ml-values + ONNXRUNTIME_RETURN_IF_ERROR(ComputeUseCounts()); + + // determine sharing/reuse among ml-values + ComputeReusePlan(); + + // convert information in the freelist_ into a deallocation plan in required format + GenerateDeallocationPlan(); + + return Status::OK(); +} + +Status SequentialPlanner::CreatePlan(const onnxruntime::GraphViewer& graph_viewer, + const ExecutionProviders& providers, + const KernelRegistryManager& kernel_registry, + const MLValueNameIdxMap& mlvalue_name_idx_map, + const ISequentialPlannerContext& context, + std::unique_ptr& plan) { + // allocate/reset here so we know it's clean + plan = std::make_unique(); + + PlannerImpl planner(graph_viewer, providers, kernel_registry, mlvalue_name_idx_map, context, *plan); + + return planner.CreatePlan(); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/allocation_planner.h b/onnxruntime/core/framework/allocation_planner.h new file mode 100644 index 0000000000000..d9eab0f5d660a --- /dev/null +++ b/onnxruntime/core/framework/allocation_planner.h @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/status.h" +#include "core/framework/alloc_kind.h" +#include "core/framework/allocator.h" +#include "core/framework/sequential_execution_plan.h" +#include "core/graph/graph.h" +namespace ONNX_NAMESPACE { +class TensorShapeProto; +} +namespace onnxruntime { + +class ExecutionProviders; +class KernelRegistryManager; +class MLValueNameIdxMap; + +// ISequentialPlannerContext abstracts how the planner accesses information (such as inferred shape) +// to do the planning. +class ISequentialPlannerContext { + public: + virtual const ONNX_NAMESPACE::TensorShapeProto* GetShape(const onnxruntime::NodeArg& arg) const = 0; + virtual bool EnableParallelExecution() const { return false; } +}; + +class SequentialPlannerContext : public ISequentialPlannerContext { + public: + SequentialPlannerContext() + : m_enable_parallel_execution(false) { + } + + SequentialPlannerContext(bool p_enable_parallel_execution) + : m_enable_parallel_execution(p_enable_parallel_execution) { + } + + const ONNX_NAMESPACE::TensorShapeProto* GetShape(const onnxruntime::NodeArg& arg) const override { + return arg.Shape(); + } + + bool EnableParallelExecution() const override { + return m_enable_parallel_execution; + } + + private: + bool m_enable_parallel_execution; +}; + +class SequentialPlanner { + public: + // This API allows user to provide a custom planner context. + static Status CreatePlan(const onnxruntime::GraphViewer& graph, + const ExecutionProviders& providers, + const KernelRegistryManager& kernel_registry, + const MLValueNameIdxMap& mlvalue_name_idx_map, + const ISequentialPlannerContext& context, + std::unique_ptr& plan); + + // This uses a standard planner context and is meant to be the primary API for creating a plan + // as the context is primarily used in test scenarios. + static Status CreatePlan(const onnxruntime::GraphViewer& graph, + const ExecutionProviders& providers, + const KernelRegistryManager& kernel_registry, + const MLValueNameIdxMap& mlvalue_name_idx_map, + std::unique_ptr& plan) { + SequentialPlannerContext context; + return CreatePlan(graph, providers, kernel_registry, mlvalue_name_idx_map, context, plan); + } +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/allocator.cc b/onnxruntime/core/framework/allocator.cc new file mode 100644 index 0000000000000..634d625aa97d1 --- /dev/null +++ b/onnxruntime/core/framework/allocator.cc @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/allocator.h" +#include "core/framework/allocatormgr.h" +#include +#include + +namespace onnxruntime { + +void* CPUAllocator::Alloc(size_t size) { + if (size <= 0) + return nullptr; + //todo: we should pin the memory in some case + void* p = malloc(size); + return p; +} + +void CPUAllocator::Free(void* p) { + //todo: unpin the memory + free(p); +} + +const ONNXRuntimeAllocatorInfo& CPUAllocator::Info() const { + static constexpr ONNXRuntimeAllocatorInfo cpuAllocatorInfo(CPU, ONNXRuntimeAllocatorType::ONNXRuntimeDeviceAllocator); + return cpuAllocatorInfo; +} +} // namespace onnxruntime + +std::ostream& operator<<(std::ostream& out, const ONNXRuntimeAllocatorInfo& info) { + return (out << info.ToString()); +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeCreateAllocatorInfo, const char* name1, ONNXRuntimeAllocatorType type, int id1, ONNXRuntimeMemType mem_type1, ONNXRuntimeAllocatorInfo** out) { + *out = new ONNXRuntimeAllocatorInfo(name1, type, id1, mem_type1); + return nullptr; +} + +ONNXRUNTIME_API(void, ReleaseONNXRuntimeAllocatorInfo, ONNXRuntimeAllocatorInfo* p) { + delete p; +} + +ONNXRUNTIME_API(const char*, ONNXRuntimeAllocatorInfoGetName, _In_ ONNXRuntimeAllocatorInfo* ptr) { + return ptr->name; +} + +ONNXRUNTIME_API(int, ONNXRuntimeAllocatorInfoGetId, _In_ ONNXRuntimeAllocatorInfo* ptr) { + return ptr->id; +} + +ONNXRUNTIME_API(ONNXRuntimeMemType, ONNXRuntimeAllocatorInfoGetMemType, _In_ ONNXRuntimeAllocatorInfo* ptr) { + return ptr->mem_type; +} + +ONNXRUNTIME_API(ONNXRuntimeAllocatorType, ONNXRuntimeAllocatorInfoGetType, _In_ ONNXRuntimeAllocatorInfo* ptr) { + return ptr->type; +} + +ONNXRUNTIME_API(int, ONNXRuntimeCompareAllocatorInfo, _In_ const ONNXRuntimeAllocatorInfo* info1, _In_ const ONNXRuntimeAllocatorInfo* info2) { + if (*info1 == *info2) { + return 0; + } + return -1; +} \ No newline at end of file diff --git a/onnxruntime/core/framework/allocatormgr.cc b/onnxruntime/core/framework/allocatormgr.cc new file mode 100644 index 0000000000000..27c0b19e8b0d1 --- /dev/null +++ b/onnxruntime/core/framework/allocatormgr.cc @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/allocatormgr.h" +#include "core/framework/bfc_arena.h" +#include +#include +#include +#include + +namespace onnxruntime { + +using namespace ::onnxruntime::common; + +AllocatorPtr CreateAllocator(DeviceAllocatorRegistrationInfo info, int device_id) { + auto device_allocator = std::unique_ptr(info.factory(device_id)); + if (device_allocator->AllowsArena()) + return std::shared_ptr( + std::make_unique(std::move(device_allocator), info.max_mem)); + + return device_allocator; +} + +DeviceAllocatorRegistry& DeviceAllocatorRegistry::Instance() { + static DeviceAllocatorRegistry s_instance; + return s_instance; +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/allocatormgr.h b/onnxruntime/core/framework/allocatormgr.h new file mode 100644 index 0000000000000..6ca8f53cb5f2c --- /dev/null +++ b/onnxruntime/core/framework/allocatormgr.h @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/arena.h" + +namespace onnxruntime { + +using DeviceAllocatorFactory = std::function(int)>; + +struct DeviceAllocatorRegistrationInfo { + ONNXRuntimeMemType mem_type; + DeviceAllocatorFactory factory; + size_t max_mem; +}; + +AllocatorPtr CreateAllocator(DeviceAllocatorRegistrationInfo info, int device_id = 0); + +class DeviceAllocatorRegistry { + public: + void RegisterDeviceAllocator(std::string&& name, DeviceAllocatorFactory factory, size_t max_mem, + ONNXRuntimeMemType mem_type = ONNXRuntimeMemTypeDefault) { + DeviceAllocatorRegistrationInfo info({mem_type, factory, max_mem}); + device_allocator_registrations_.emplace(std::move(name), std::move(info)); + } + + const std::map& AllRegistrations() const { + return device_allocator_registrations_; + } + + static DeviceAllocatorRegistry& Instance(); + + private: + DeviceAllocatorRegistry() = default; + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(DeviceAllocatorRegistry); + + std::map device_allocator_registrations_; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/arena.h b/onnxruntime/core/framework/arena.h new file mode 100644 index 0000000000000..c348f41c400ed --- /dev/null +++ b/onnxruntime/core/framework/arena.h @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include "core/common/common.h" +#include "core/framework/allocator.h" + +namespace onnxruntime { +// The interface for arena which manage memory allocations +// Arena will hold a pool of pre-allocate memories and manage their lifecycle. +// Need an underline IResourceAllocator to allocate memories. +// The setting like max_chunk_size is init by IDeviceDescriptor from resource allocator +class IArenaAllocator : public IAllocator { + public: + ~IArenaAllocator() override = default; + // Alloc call need to be thread safe. + void* Alloc(size_t size) override = 0; + // The chunck allocated by Reserve call won't be reused with other request. + // It will be return to the devices when it is freed. + // Reserve call need to be thread safe. + virtual void* Reserve(size_t size) = 0; + // Free call need to be thread safe. + void Free(void* p) override = 0; + virtual size_t Used() const = 0; + virtual size_t Max() const = 0; + const ONNXRuntimeAllocatorInfo& Info() const override = 0; + // allocate host pinned memory? +}; + +using ArenaPtr = std::shared_ptr; + +// Dummy Arena which just call underline device allocator directly. +class DummyArena : public IArenaAllocator { + public: + explicit DummyArena(std::unique_ptr resource_allocator) + : allocator_(std::move(resource_allocator)), + info_(allocator_->Info().name, ONNXRuntimeAllocatorType::ONNXRuntimeArenaAllocator, allocator_->Info().id) { + } + + ~DummyArena() override = default; + + void* Alloc(size_t size) override { + if (size == 0) + return nullptr; + return allocator_->Alloc(size); + } + + void Free(void* p) override { + allocator_->Free(p); + } + + void* Reserve(size_t size) override { + return Alloc(size); + } + + size_t Used() const override { + ONNXRUNTIME_NOT_IMPLEMENTED(__FUNCTION__, " is not implemented"); + } + + size_t Max() const override { + ONNXRUNTIME_NOT_IMPLEMENTED(__FUNCTION__, " is not implemented"); + } + + const ONNXRuntimeAllocatorInfo& Info() const override { + return info_; + } + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(DummyArena); + + std::unique_ptr allocator_; + ONNXRuntimeAllocatorInfo info_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/bfc_arena.cc b/onnxruntime/core/framework/bfc_arena.cc new file mode 100644 index 0000000000000..51da47f645085 --- /dev/null +++ b/onnxruntime/core/framework/bfc_arena.cc @@ -0,0 +1,552 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/bfc_arena.h" + +namespace onnxruntime { +BFCArena::BFCArena(std::unique_ptr resource_allocator, + size_t total_memory) + : device_allocator_(std::move(resource_allocator)), + free_chunks_list_(kInvalidChunkHandle), + next_allocation_id_(1), + info_(device_allocator_->Info().name, ONNXRuntimeAllocatorType::ONNXRuntimeArenaAllocator, device_allocator_->Info().id, device_allocator_->Info().mem_type) { + curr_region_allocation_bytes_ = RoundedBytes(std::min(total_memory, size_t{1048576})); + + // Allocate the requested amount of memory. + memory_limit_ = total_memory; + stats_.bytes_limit = static_cast(total_memory); + // Create a bunch of bins of various good sizes. + + // We create bins to fit all possible ranges that cover the + // memory_limit_ starting from allocations up to 256 bytes to + // allocations up to (and including) the memory limit. + for (BinNum b = 0; b < kNumBins; b++) { + size_t bin_size = BinNumToSize(b); + LOGS_DEFAULT(INFO) << "Creating bin of max chunk size " + << bin_size; + new (BinFromIndex(b)) Bin(this, bin_size); + ONNXRUNTIME_ENFORCE(BinForSize(bin_size) == BinFromIndex(b)); + ONNXRUNTIME_ENFORCE(BinForSize(bin_size + 255) == BinFromIndex(b)); + ONNXRUNTIME_ENFORCE(BinForSize(bin_size * 2 - 1) == BinFromIndex(b)); + if (b + 1 < kNumBins) { + ONNXRUNTIME_ENFORCE(BinForSize(bin_size * 2) != BinFromIndex(b)); + } + } +} + +BFCArena::~BFCArena() { + for (const auto& region : region_manager_.regions()) { + device_allocator_->Free(region.ptr()); + } + + for (const auto& reserve_chunk : reserved_chunks_) { + device_allocator_->Free(reserve_chunk.first); + } + + for (BinNum b = 0; b < kNumBins; b++) { + BinFromIndex(b)->~Bin(); + } +} + +BFCArena::Chunk* BFCArena::ChunkFromHandle(ChunkHandle h) { + ONNXRUNTIME_ENFORCE(h < chunks_.size()); + return &(chunks_[h]); +} + +bool BFCArena::Extend(size_t rounded_bytes) { + size_t available_bytes = memory_limit_ - stats_.total_allocated_bytes; + // Rounds available_bytes down to the nearest multiple of kMinAllocationSize. + available_bytes = (available_bytes / kMinAllocationSize) * kMinAllocationSize; + + // Do we have enough space to handle the client's request? + // If not, fail immediately. + if (rounded_bytes > available_bytes) { + return false; + } + + // If curr_region_allocation_bytes_ is not enough to satisfy the + // allocation, keep multiplying by a power of two until that is + // sufficient. + bool increased_allocation = false; + while (rounded_bytes > curr_region_allocation_bytes_) { + curr_region_allocation_bytes_ *= 2; + increased_allocation = true; + } + + // Try allocating. + size_t bytes = std::min(curr_region_allocation_bytes_, available_bytes); + void* mem_addr = device_allocator_->Alloc(bytes); + if (mem_addr == nullptr && !started_backpedal_) { + // Only backpedal once. + started_backpedal_ = true; + + static constexpr float kBackpedalFactor = 0.9f; + + // Try allocating less memory. + while (mem_addr == nullptr) { + bytes = RoundedBytes(static_cast(bytes * kBackpedalFactor)); + if (bytes < rounded_bytes) break; + mem_addr = device_allocator_->Alloc(bytes); + } + } + + if (mem_addr == nullptr) { + return false; + } + + if (!increased_allocation) { + // Increase the region size of the next required allocation. + curr_region_allocation_bytes_ *= 2; + } + + LOGS_DEFAULT(INFO) << "Extending allocation by " << bytes + << " bytes."; + + stats_.total_allocated_bytes += bytes; + LOGS_DEFAULT(INFO) << "Total allocated bytes: " + << stats_.total_allocated_bytes; + + LOGS_DEFAULT(INFO) << "Allocated memory at " << mem_addr << " to " + << static_cast(static_cast(mem_addr) + bytes); + region_manager_.AddAllocationRegion(mem_addr, bytes); + + // Create one large chunk for the whole memory space that will + // be chunked later. + ChunkHandle h = AllocateChunk(); + BFCArena::Chunk* c = ChunkFromHandle(h); + c->ptr = mem_addr; + c->size = bytes; + c->allocation_id = -1; + c->prev = kInvalidChunkHandle; + c->next = kInvalidChunkHandle; + + region_manager_.set_handle(c->ptr, h); + + // TODO(vrv): Try to merge this new region with an existing region, + // if the address space is contiguous, to avoid fragmentation + // across regions. + + // Insert the chunk into the right bin. + InsertFreeChunkIntoBin(h); + + return true; +} + +BFCArena::ChunkHandle BFCArena::AllocateChunk() { + if (free_chunks_list_ != kInvalidChunkHandle) { + ChunkHandle h = free_chunks_list_; + Chunk* c = ChunkFromHandle(h); + free_chunks_list_ = c->next; + return h; + } + ChunkHandle h = chunks_.size(); + chunks_.resize(h + 1); + return h; +} + +void BFCArena::DeallocateChunk(ChunkHandle h) { + Chunk* c = ChunkFromHandle(h); + c->next = free_chunks_list_; + free_chunks_list_ = h; +} + +// static +size_t BFCArena::RoundedBytes(size_t bytes) { + size_t rounded_bytes = + (kMinAllocationSize * + ((bytes + kMinAllocationSize - 1) / kMinAllocationSize)); + ONNXRUNTIME_ENFORCE(size_t{0} == rounded_bytes % kMinAllocationSize); + return rounded_bytes; +} + +void* BFCArena::Alloc(size_t size) { + return AllocateRawInternal(size, false); +} + +void* BFCArena::Reserve(size_t size) { + if (size == 0) + return nullptr; + + std::lock_guard lock(lock_); + void* ptr = device_allocator_->Alloc(size); + ONNXRUNTIME_ENFORCE(reserved_chunks_.find(ptr) == reserved_chunks_.end()); + reserved_chunks_.insert(std::pair(ptr, size)); + stats_.bytes_in_use += size; + stats_.num_allocs += 1; + stats_.max_alloc_size = std::max(stats_.max_alloc_size, size); + stats_.max_bytes_in_use = std::max(stats_.max_bytes_in_use, stats_.bytes_in_use); + stats_.total_allocated_bytes += size; + return ptr; +} + +size_t BFCArena::RequestedSize(const void* ptr) { + std::lock_guard lock(lock_); + BFCArena::ChunkHandle h = region_manager_.get_handle(ptr); + ONNXRUNTIME_ENFORCE(h != kInvalidChunkHandle); + BFCArena::Chunk* c = ChunkFromHandle(h); + return c->requested_size; +} + +size_t BFCArena::AllocatedSize(const void* ptr) { + std::lock_guard lock(lock_); + BFCArena::ChunkHandle h = region_manager_.get_handle(ptr); + ONNXRUNTIME_ENFORCE(h != kInvalidChunkHandle); + BFCArena::Chunk* c = ChunkFromHandle(h); + return c->size; +} + +void* BFCArena::AllocateRawInternal(size_t num_bytes, + bool dump_log_on_failure) { + if (num_bytes == 0) { + LOGS_DEFAULT(WARNING) << "tried to allocate 0 bytes"; + return nullptr; + } + // First, always allocate memory of at least kMinAllocationSize + // bytes, and always allocate multiples of kMinAllocationSize bytes + // so all memory addresses are nicely byte aligned. + size_t rounded_bytes = RoundedBytes(num_bytes); + + // The BFC allocator tries to find the best fit first. + BinNum bin_num = BinNumForSize(rounded_bytes); + + std::lock_guard lock(lock_); + void* ptr = FindChunkPtr(bin_num, rounded_bytes, num_bytes); + if (ptr != nullptr) { + return ptr; + } + + // Try to extend + if (Extend(rounded_bytes)) { + ptr = FindChunkPtr(bin_num, rounded_bytes, num_bytes); + if (ptr != nullptr) { + return ptr; + } + } + + // We searched all bins for an existing free chunk to use and + // couldn't find one. This means we must have run out of memory, + // Dump the memory log for analysis. + if (dump_log_on_failure) { + LOGS_DEFAULT(WARNING) << "BFC Arena ran out of memory trying " + << "to allocate " << num_bytes + << ". Current allocation summary follows."; + DumpMemoryLog(rounded_bytes); + } + return nullptr; +} + +void BFCArena::GetStats(AllocatorStats* stats) { + std::lock_guard lock(lock_); + *stats = stats_; +} + +void* BFCArena::FindChunkPtr(BinNum bin_num, size_t rounded_bytes, + size_t num_bytes) { + // First identify the first bin that could satisfy rounded_bytes. + for (; bin_num < kNumBins; bin_num++) { + // Start searching from the first bin for the smallest chunk that fits + // rounded_bytes. + Bin* b = BinFromIndex(bin_num); + for (auto citer = b->free_chunks.begin(); citer != b->free_chunks.end(); + ++citer) { + const BFCArena::ChunkHandle h = (*citer); + BFCArena::Chunk* chunk = ChunkFromHandle(h); + ONNXRUNTIME_ENFORCE(!chunk->in_use()); + if (chunk->size >= rounded_bytes) { + // We found an existing chunk that fits us that wasn't in use, so remove + // it from the free bin structure prior to using. + RemoveFreeChunkIterFromBin(&b->free_chunks, citer); + + // If we can break the size of the chunk into two reasonably large + // pieces, do so. In any case don't waste more than + // kMaxInternalFragmentation bytes on padding this alloc. + const int64_t kMaxInternalFragmentation = 128 << 20; // 128mb + if (chunk->size >= rounded_bytes * 2 || + static_cast(chunk->size) - rounded_bytes >= + kMaxInternalFragmentation) { + SplitChunk(h, rounded_bytes); + chunk = ChunkFromHandle(h); // Update chunk pointer in case it moved + } + + // The requested size of the returned chunk is what the user + // has allocated. + chunk->requested_size = num_bytes; + // Assign a unique id and increment the id counter, marking the + // chunk as being in use. + chunk->allocation_id = next_allocation_id_++; + // Update stats. + ++stats_.num_allocs; + stats_.bytes_in_use += chunk->size; + stats_.max_bytes_in_use = + std::max(stats_.max_bytes_in_use, stats_.bytes_in_use); + stats_.max_alloc_size = + std::max(stats_.max_alloc_size, chunk->size); + + return chunk->ptr; + } + } + } + return nullptr; +} + +void BFCArena::SplitChunk(BFCArena::ChunkHandle h, size_t num_bytes) { + // Allocate the new chunk before we do any ChunkFromHandle + ChunkHandle h_new_chunk = AllocateChunk(); + + Chunk* c = ChunkFromHandle(h); + ONNXRUNTIME_ENFORCE(!c->in_use() && (c->bin_num == kInvalidBinNum)); + + // Create a new chunk starting num_bytes after c + BFCArena::Chunk* new_chunk = ChunkFromHandle(h_new_chunk); + new_chunk->ptr = static_cast(static_cast(c->ptr) + num_bytes); + region_manager_.set_handle(new_chunk->ptr, h_new_chunk); + + // Set the new sizes of the chunks. + new_chunk->size = c->size - num_bytes; + c->size = num_bytes; + + // The new chunk is not in use. + new_chunk->allocation_id = -1; + + // Maintain the pointers. + // c <-> c_neighbor becomes + // c <-> new_chunk <-> c_neighbor + BFCArena::ChunkHandle h_neighbor = c->next; + new_chunk->prev = h; + new_chunk->next = h_neighbor; + c->next = h_new_chunk; + if (h_neighbor != kInvalidChunkHandle) { + Chunk* c_neighbor = ChunkFromHandle(h_neighbor); + c_neighbor->prev = h_new_chunk; + } + + // Add the newly free chunk to the free bin. + InsertFreeChunkIntoBin(h_new_chunk); +} + +void BFCArena::Free(void* p) { + if (p == nullptr) { + return; + } + std::lock_guard lock(lock_); + auto it = reserved_chunks_.find(p); + if (it != reserved_chunks_.end()) { + device_allocator_->Free(it->first); + stats_.bytes_in_use -= it->second; + stats_.total_allocated_bytes -= it->second; + reserved_chunks_.erase(it); + } else { + DeallocateRawInternal(p); + } +} + +void BFCArena::DeallocateRawInternal(void* ptr) { + // Find the chunk from the ptr. + BFCArena::ChunkHandle h = region_manager_.get_handle(ptr); + ONNXRUNTIME_ENFORCE(h != kInvalidChunkHandle); + + // Consider coalescing it. + FreeAndMaybeCoalesce(h); +} + +// Merges h1 and h2 when Chunk(h1)->next is h2 and Chunk(h2)->prev is c1. +// We merge Chunk(h2) into Chunk(h1). +void BFCArena::Merge(BFCArena::ChunkHandle h1, + BFCArena::ChunkHandle h2) { + Chunk* c1 = ChunkFromHandle(h1); + Chunk* c2 = ChunkFromHandle(h2); + // We can only merge chunks that are not in use. + ONNXRUNTIME_ENFORCE(!c1->in_use() && !c2->in_use()); + + // c1's prev doesn't change, still points to the same ptr, and is + // still not in use. + + // Fix up neighbor pointers + // + // c1 <-> c2 <-> c3 should become + // c1 <-> c3 + + BFCArena::ChunkHandle h3 = c2->next; + c1->next = h3; + ONNXRUNTIME_ENFORCE(c2->prev == h1); + if (h3 != kInvalidChunkHandle) { + BFCArena::Chunk* c3 = ChunkFromHandle(h3); + c3->prev = h1; + } + + // Set the new size + c1->size += c2->size; + + DeleteChunk(h2); +} + +void BFCArena::DeleteChunk(ChunkHandle h) { + // Delete h and cleanup all state + Chunk* c = ChunkFromHandle(h); + // VLOG(4) << "Removing: " << c->ptr; + region_manager_.erase(c->ptr); + DeallocateChunk(h); +} + +void BFCArena::InsertFreeChunkIntoBin(BFCArena::ChunkHandle h) { + Chunk* c = ChunkFromHandle(h); + ONNXRUNTIME_ENFORCE(!c->in_use() && (c->bin_num == kInvalidBinNum)); + BinNum bin_num = BinNumForSize(c->size); + Bin* new_bin = BinFromIndex(bin_num); + c->bin_num = bin_num; + new_bin->free_chunks.insert(h); +} + +void BFCArena::RemoveFreeChunkIterFromBin( + BFCArena::Bin::FreeChunkSet* free_chunks, + const BFCArena::Bin::FreeChunkSet::iterator& citer) { + ChunkHandle h = *citer; + Chunk* c = ChunkFromHandle(h); + ONNXRUNTIME_ENFORCE(!c->in_use() && (c->bin_num != kInvalidBinNum)); + free_chunks->erase(citer); + c->bin_num = kInvalidBinNum; +} + +void BFCArena::RemoveFreeChunkFromBin(BFCArena::ChunkHandle h) { + Chunk* c = ChunkFromHandle(h); + ONNXRUNTIME_ENFORCE(!c->in_use() && (c->bin_num != kInvalidBinNum)); + ONNXRUNTIME_ENFORCE(BinFromIndex(c->bin_num)->free_chunks.erase(h) > 0, + "Could not find chunk in bin"); + c->bin_num = kInvalidBinNum; +} + +void BFCArena::FreeAndMaybeCoalesce(BFCArena::ChunkHandle h) { + Chunk* c = ChunkFromHandle(h); + ONNXRUNTIME_ENFORCE(c->in_use() && (c->bin_num == kInvalidBinNum)); + + // Mark the chunk as no longer in use + c->allocation_id = -1; + + // Updates the stats. + stats_.bytes_in_use -= c->size; + + // This chunk is no longer in-use, consider coalescing the chunk + // with adjacent chunks. + ChunkHandle chunk_to_reassign = h; + + // If the next chunk is free, coalesce the two + if (c->next != kInvalidChunkHandle) { + Chunk* cnext = ChunkFromHandle(c->next); + if (!cnext->in_use()) { + // VLOG(8) << "Chunk at " << cnext->ptr << " merging with c " << + // c->ptr; + + chunk_to_reassign = h; + + // Deletes c->next + RemoveFreeChunkFromBin(c->next); + Merge(h, ChunkFromHandle(h)->next); + } + } + + // If the previous chunk is free, coalesce the two + c = ChunkFromHandle(h); + if (c->prev != kInvalidChunkHandle) { + Chunk* cprev = ChunkFromHandle(c->prev); + if (!cprev->in_use()) { + // VLOG(8) << "Chunk at " << c->ptr << " merging into c->prev " + // << cprev->ptr; + + chunk_to_reassign = c->prev; + + // Deletes c + RemoveFreeChunkFromBin(c->prev); + Merge(ChunkFromHandle(h)->prev, h); + } + } + + InsertFreeChunkIntoBin(chunk_to_reassign); +} + +std::array +BFCArena::get_bin_debug_info() { + std::array bin_infos; + for (const auto& region : region_manager_.regions()) { + ChunkHandle h = region_manager_.get_handle(region.ptr()); + while (h != kInvalidChunkHandle) { + const Chunk* c = ChunkFromHandle(h); + BinNum bin_num = BinNumForSize(c->size); + BinDebugInfo& bin_info = bin_infos[bin_num]; + bin_info.total_bytes_in_bin += c->size; + bin_info.total_chunks_in_bin++; + if (c->in_use()) { + bin_info.total_bytes_in_use += c->size; + bin_info.total_requested_bytes_in_use += c->requested_size; + bin_info.total_chunks_in_use++; + } else { + Bin* bin = BinFromIndex(bin_num); + ONNXRUNTIME_ENFORCE(bin->free_chunks.count(h) == 1); + ONNXRUNTIME_ENFORCE(c->bin_num == bin_num); + } + h = c->next; + } + } + return bin_infos; +} + +void BFCArena::DumpMemoryLog(size_t num_bytes) { + const std::array bin_infos = get_bin_debug_info(); + for (BinNum bin_num = 0; bin_num < kNumBins; bin_num++) { + Bin* b = BinFromIndex(bin_num); + const BinDebugInfo& bin_info = bin_infos[bin_num]; + ONNXRUNTIME_ENFORCE(b->free_chunks.size() == + bin_info.total_chunks_in_bin - bin_info.total_chunks_in_use); + + LOGS_DEFAULT(INFO) << "Bin (" << b->bin_size + << "): \tTotal Chunks: " << bin_info.total_chunks_in_bin + << ", Chunks in use: " << bin_info.total_chunks_in_use << ". " + << bin_info.total_bytes_in_bin + << " allocated for chunks. " + << bin_info.total_bytes_in_use + << " in use in bin. " + << bin_info.total_requested_bytes_in_use + << " client-requested in use in bin."; + } + + // Find the bin that we would have liked to allocate in, so we + // can get some further analysis about fragmentation. + Bin* b = BinForSize(num_bytes); + + LOGS_DEFAULT(INFO) << "Bin for " << num_bytes + << " was " << b->bin_size + << ", Chunk State: "; + + for (ChunkHandle h : b->free_chunks) { + Chunk* c = ChunkFromHandle(h); + LOGS_DEFAULT(INFO) << c->DebugString(this, true); + } + + // Next show the chunks that are in use, and also summarize their + // number by size. + std::map in_use_by_size; + for (const auto& region : region_manager_.regions()) { + ChunkHandle h = region_manager_.get_handle(region.ptr()); + while (h != kInvalidChunkHandle) { + const Chunk* c = ChunkFromHandle(h); + if (c->in_use()) { + in_use_by_size[c->size]++; + } + LOGS_DEFAULT(INFO) << (c->in_use() ? "Chunk" : "Free ") << " at " << c->ptr + << " of size " << c->size; + h = c->next; + } + } + + LOGS_DEFAULT(INFO) << " Summary of in-use Chunks by size: "; + size_t total_bytes = 0; + for (auto& it : in_use_by_size) { + LOGS_DEFAULT(INFO) << it.second << " Chunks of size " << it.first << " totalling " + << it.first * it.second; + total_bytes += (it.first * it.second); + } + LOGS_DEFAULT(INFO) << "Sum Total of in-use chunks: " + << total_bytes; + LOGS_DEFAULT(INFO) << "Stats: \n" + << stats_.DebugString(); +} +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/bfc_arena.h b/onnxruntime/core/framework/bfc_arena.h new file mode 100644 index 0000000000000..6dec27785827a --- /dev/null +++ b/onnxruntime/core/framework/bfc_arena.h @@ -0,0 +1,486 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +// Portions Copyright (c) Microsoft Corporation + +#pragma once +#include +#include +#include +#include + +#include "core/common/common.h" +#include "core/common/logging/logging.h" +#include "core/common/logging/severity.h" +#include "core/framework/arena.h" +#include "onnxruntime_config.h" + +#if defined(PLATFORM_WINDOWS) +#include +#endif +namespace onnxruntime { +#ifdef __GNUC__ +#pragma GCC diagnostic push +#ifdef HAS_NULL_DEREFERENCE +#pragma GCC diagnostic ignored "-Wnull-dereference" +#endif +#endif +// Runtime statistics collected by an allocator. +struct AllocatorStats { + int64_t num_allocs; // Number of allocations. + int64_t bytes_in_use; // Number of bytes in use. + int64_t total_allocated_bytes; // The total number of allocated bytes by the allocator. + int64_t max_bytes_in_use; // The maximum bytes in use. + int64_t max_alloc_size; // The max single allocation seen. + // The upper limit what the allocator can allocate, if such a limit + // is known. Certain allocator may return 0 to indicate the limit is + // unknown. + int64_t bytes_limit; + + AllocatorStats() { Clear(); } + + void Clear() { + this->num_allocs = 0; + this->bytes_in_use = 0; + this->max_bytes_in_use = 0; + this->max_alloc_size = 0; + this->bytes_limit = 0; + this->total_allocated_bytes = 0; + } + + std::string DebugString() const { + std::ostringstream ss; + ss << "Limit: " << this->bytes_limit << "\n" + << "InUse: " << this->bytes_in_use << "\n" + << "TotalAllocated: " << this->total_allocated_bytes << "\n" + << "MaxInUse: " << this->max_bytes_in_use << "\n" + << "NumAllocs: " << this->num_allocs << "\n" + << "MaxAllocSize: " << this->max_alloc_size << "\n"; + return ss.str(); + } +}; + +// A memory allocator that implements a 'best-fit with coalescing' +// algorithm. This is essentially a very simple version of Doug Lea's +// malloc (dlmalloc). +// +// The goal of this allocator is to support defragmentation via +// coalescing. One assumption we make is that the process using this +// allocator owns pretty much all of the memory, and that nearly +// all requests to allocate memory go through this interface. +class BFCArena : public IArenaAllocator { + public: + BFCArena(std::unique_ptr resource_allocator, size_t total_memory); + + ~BFCArena() override; + + //If size is 0, then this function returns either NULL, + //or a unique pointer value that can later be successfully + //passed to free(). Whatever, do not dereference that pointer + void* Alloc(size_t size) override; + + //If p is NULL, no operation is performed. + void Free(void* p) override; + + void* Reserve(size_t size) override; + + size_t Used() const override { + return stats_.bytes_in_use; + } + + size_t Max() const override { + return memory_limit_; + } + + const ONNXRuntimeAllocatorInfo& Info() const override { + return info_; + } + + FencePtr CreateFence(const SessionState* session_state) override { + // arena always rely on its device allocator to create fence + return device_allocator_->CreateFence(session_state); + } + + void GetStats(AllocatorStats* stats); + + size_t RequestedSize(const void* ptr); + + size_t AllocatedSize(const void* ptr); + + private: + void* AllocateRawInternal(size_t num_bytes, bool dump_log_on_failure); + void DeallocateRawInternal(void* ptr); + + // A ChunkHandle is an index into the chunks_ vector in BFCAllocator + // kInvalidChunkHandle means an invalid chunk + using ChunkHandle = size_t; + static const size_t kInvalidChunkHandle = static_cast(-1); + + using BinNum = int; + static const int kInvalidBinNum = -1; + static const int kNumBins = 21; + + // Chunks point to memory. Their prev/next pointers form a + // doubly-linked list of addresses sorted by base address that + // must be contiguous. Chunks contain information about whether + // they are in use or whether they are free, and contain a pointer + // to the bin they are in. + struct Chunk { + size_t size = 0; // Full size of buffer. + + // We sometimes give chunks that are larger than needed to reduce + // fragmentation. requested_size keeps track of what the client + // actually wanted so we can understand whether our splitting + // strategy is efficient. + size_t requested_size = 0; + + // allocation_id is set to -1 when the chunk is not in use. It is assigned a + // value greater than zero before the chunk is returned from + // AllocateRaw, and this value is unique among values assigned by + // the parent allocator. + int64_t allocation_id = -1; + void* ptr = nullptr; // pointer to granted subbuffer. + + // If not kInvalidChunkHandle, the memory referred to by 'prev' is directly + // preceding the memory used by this chunk. E.g., It should start + // at 'ptr - prev->size' + ChunkHandle prev = kInvalidChunkHandle; + + // If not kInvalidChunkHandle, the memory referred to by 'next' is directly + // following the memory used by this chunk. E.g., It should be at + // 'ptr + size' + ChunkHandle next = kInvalidChunkHandle; + + // What bin are we in? + BinNum bin_num = kInvalidBinNum; + + bool in_use() const { return allocation_id != -1; } + + std::string DebugString(BFCArena* a, bool recurse) { + std::ostringstream ss; + ss << " Size: " << size << " | Requested Size: " << requested_size << " | in_use: " << in_use(); + if (recurse && prev != BFCArena::kInvalidChunkHandle) { + Chunk* p = a->ChunkFromHandle(prev); + ss << ", prev: " << p->DebugString(a, false); + } + + if (recurse && next != BFCArena::kInvalidChunkHandle) { + Chunk* n = a->ChunkFromHandle(next); + ss << ", next: " << n->DebugString(a, false); + } + return ss.str(); + } + }; + + // A Bin is a collection of similar-sized free chunks. + struct Bin { + // All chunks in this bin have >= bin_size memory. + size_t bin_size = 0; + + struct ChunkComparator { + explicit ChunkComparator(BFCArena* allocator) + : allocator_(allocator) {} + + // Sort first by size and then use pointer address as a tie breaker. + bool operator()(const ChunkHandle ha, + const ChunkHandle hb) const { + const Chunk* a = allocator_->ChunkFromHandle(ha); + const Chunk* b = allocator_->ChunkFromHandle(hb); + if (a->size != b->size) { + return a->size < b->size; + } + return a->ptr < b->ptr; + } + + private: + BFCArena* allocator_; // The parent allocator + }; + + typedef std::set FreeChunkSet; + // List of free chunks within the bin, sorted by chunk size. + // Chunk * not owned. + FreeChunkSet free_chunks; + Bin(BFCArena* allocator, size_t bs) + : bin_size(bs), free_chunks(ChunkComparator(allocator)) {} + }; + + static const size_t kMinAllocationBits = 8; + static const size_t kMinAllocationSize = 1 << kMinAllocationBits; + + // AllocationRegion maps pointers to ChunkHandles for a single + // contiguous memory region. + // + // This class is thread-compatible. + class AllocationRegion { + public: + AllocationRegion(void* ptr, size_t memory_size) + : ptr_(ptr), + memory_size_(memory_size), + end_ptr_( + static_cast(static_cast(ptr_) + memory_size_)) { + ONNXRUNTIME_ENFORCE(0 == memory_size % kMinAllocationSize); + const size_t n_handles = + (memory_size + kMinAllocationSize - 1) / kMinAllocationSize; + handles_ = new ChunkHandle[n_handles]; + for (size_t i = 0; i < n_handles; i++) { + handles_[i] = kInvalidChunkHandle; + } + } + + AllocationRegion() = default; + + ~AllocationRegion() { delete[] handles_; } + + AllocationRegion(AllocationRegion&& other) { Swap(other); } + + AllocationRegion& operator=(AllocationRegion&& other) { + Swap(other); + return *this; + } + + void* ptr() const { return ptr_; } + void* end_ptr() const { return end_ptr_; } + size_t memory_size() const { return memory_size_; } + ChunkHandle get_handle(const void* p) const { + return handles_[IndexFor(p)]; + } + void set_handle(const void* p, ChunkHandle h) { handles_[IndexFor(p)] = h; } + void erase(const void* p) { set_handle(p, kInvalidChunkHandle); } + + private: + void Swap(AllocationRegion& other) { + std::swap(ptr_, other.ptr_); + std::swap(memory_size_, other.memory_size_); + std::swap(end_ptr_, other.end_ptr_); + std::swap(handles_, other.handles_); + } + + int IndexFor(const void* p) const { + std::uintptr_t p_int = reinterpret_cast(p); + std::uintptr_t base_int = reinterpret_cast(ptr_); + ONNXRUNTIME_ENFORCE(p_int >= base_int); + ONNXRUNTIME_ENFORCE(p_int < base_int + memory_size_); + return static_cast(((p_int - base_int) >> kMinAllocationBits)); + } + + // Metadata about the allocation region. + void* ptr_ = nullptr; + size_t memory_size_ = 0; + void* end_ptr_ = nullptr; + + // Array of size "memory_size / kMinAllocationSize". It is + // indexed by (p-base) / kMinAllocationSize, contains ChunkHandle + // for the memory allocation represented by "p" + ChunkHandle* handles_ = nullptr; + + ONNXRUNTIME_DISALLOW_ASSIGNMENT(AllocationRegion); + }; + + // RegionManager aggregates one or more "AllocationRegions" and provides + // a layer of indirection from pointers to the underlying ChunkHandle, + // allowing allocation across multiple discontiguous memory regions. + // + // This class is thread-compatible. + class RegionManager { + public: + RegionManager() = default; + ~RegionManager() = default; + + void AddAllocationRegion(void* ptr, size_t memory_size) { + // Insert sorted by end_ptr + auto entry = + std::upper_bound(regions_.begin(), regions_.end(), ptr, &Comparator); + regions_.insert(entry, AllocationRegion(ptr, memory_size)); + } + + ChunkHandle get_handle(const void* p) const { + return RegionFor(p)->get_handle(p); + } + + void set_handle(const void* p, ChunkHandle h) { + return MutableRegionFor(p)->set_handle(p, h); + } + void erase(const void* p) { return MutableRegionFor(p)->erase(p); } + + const std::vector& regions() const { return regions_; } + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(RegionManager); + + static bool Comparator(const void* ptr, const AllocationRegion& other) { + return ptr < other.end_ptr(); + } + + AllocationRegion* MutableRegionFor(const void* p) { + return const_cast(RegionFor(p)); + } + + const AllocationRegion* RegionFor(const void* p) const { + auto entry = + std::upper_bound(regions_.begin(), regions_.end(), p, &Comparator); + + if (entry != regions_.end()) { + return &(*entry); + } + + LOGS_DEFAULT(FATAL) << "Could not find Region for " << p; + return nullptr; + } + + private: + std::vector regions_; + }; + + // Returns 'bytes' rounded up to the next highest kMinAllocationSize. + size_t RoundedBytes(size_t bytes); + + // Try to add a new memory region that can satisfy an allocation of + // 'rounded_bytes' bytes. Returns true on success and false on + // failure. + bool Extend(size_t rounded_bytes); + + // Returns a pointer to an underlying allocated chunk of size + // 'rounded_bytes'. + void* FindChunkPtr(BinNum bin_num, size_t rounded_bytes, size_t num_bytes); + + // Splits the chunk specified by 'h' into two chunks, one at least + // of size 'num_bytes'. + void SplitChunk(ChunkHandle h, size_t num_bytes); + + // Merges the two chunk handles. Requires that the chunks are + // contiguous in their allocation. + void Merge(ChunkHandle h, ChunkHandle h2); + + // Frees the memory represented by 'h', coalescing the chunk if + // possible. + void FreeAndMaybeCoalesce(ChunkHandle h); + + // Adds the chunk 'h' to the proper free bin. + void InsertFreeChunkIntoBin(ChunkHandle h); + + // Removes the free chunk pointed to by 'c' from the set free_chunks. + void RemoveFreeChunkIterFromBin(Bin::FreeChunkSet* free_chunks, + const Bin::FreeChunkSet::iterator& c); + + // Removes a free chunk from the bin. + void RemoveFreeChunkFromBin(ChunkHandle h); + + // Removes the chunk metadata represented by 'h'. + void DeleteChunk(ChunkHandle h); + + void DumpMemoryLog(size_t num_bytes); + + ChunkHandle AllocateChunk(); + void DeallocateChunk(ChunkHandle h); + + Chunk* ChunkFromHandle(ChunkHandle h); + + // Information about a Bin that is useful for debugging. + struct BinDebugInfo { + size_t total_bytes_in_use = 0; + size_t total_bytes_in_bin = 0; + size_t total_requested_bytes_in_use = 0; + size_t total_chunks_in_use = 0; + size_t total_chunks_in_bin = 0; + }; + + // Computes and returns a BinDebugInfo for each Bin. + std::array get_bin_debug_info(); + + // Structures immutable after construction + size_t memory_limit_ = 0; + + int Log2FloorNonZeroSlow(uint64_t n) { + int r = 0; + while (n > 0) { + r++; + n >>= 1; + } + return r - 1; + } + + // Returns floor(log2(n)). + int Log2FloorNonZero(uint64_t n) { +#if defined(__GNUC__) + return 63 ^ __builtin_clzll(n); +#elif defined(PLATFORM_WINDOWS) + unsigned long index; +#if defined(_WIN64) + _BitScanReverse64(&index, n); +#else + auto high = static_cast(n >> 32); + if (_BitScanReverse(&index, high) > 0) { + index += 32; + } else { + auto low = static_cast((n << 32) >> 32); + _BitScanReverse(&index, low); + } +#endif + return index; +#else + return Log2FloorNonZeroSlow(n); +#endif + } + + // Map from bin size to Bin + Bin* BinFromIndex(BinNum index) { + return reinterpret_cast(&(bins_space_[index * sizeof(Bin)])); + } + + size_t BinNumToSize(BinNum index) { + return static_cast(256) << index; + } + + BinNum BinNumForSize(size_t bytes) { + uint64_t v = std::max(bytes, 256) >> kMinAllocationBits; + int b = std::min(kNumBins - 1, Log2FloorNonZero(v)); + return b; + } + + Bin* BinForSize(size_t bytes) { return BinFromIndex(BinNumForSize(bytes)); } + + char bins_space_[sizeof(Bin) * kNumBins]; + + // The size of the current region allocation. + size_t curr_region_allocation_bytes_; + + // An indicator that expansion of a region has hit the limits + // of the available memory. + bool started_backpedal_ = false; + + std::unique_ptr device_allocator_; + + mutable std::mutex lock_; + + RegionManager region_manager_; + std::vector chunks_; + // Pointer to head of linked list of free Chunks + ChunkHandle free_chunks_list_; + + // Counter containing the next unique identifier to assign to a + // newly-created chunk. + int64_t next_allocation_id_; + + AllocatorStats stats_; + + ONNXRuntimeAllocatorInfo info_; + + std::unordered_map reserved_chunks_; + + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(BFCArena); +}; +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/cblas.h b/onnxruntime/core/framework/cblas.h new file mode 100644 index 0000000000000..78b5e2255cdd6 --- /dev/null +++ b/onnxruntime/core/framework/cblas.h @@ -0,0 +1,606 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// This is the exact cblas.h header file, placed here purely in order to get +// the enums. + +//#include "caffe2/core/macros.h" + +#ifndef CBLAS_H +#ifdef CAFFE2_USE_MKL +#include +#else // CAFFE2_USE_MKL + +#ifndef CBLAS_ENUM_DEFINED_H +#define CBLAS_ENUM_DEFINED_H +enum CBLAS_ORDER { CblasRowMajor = 101, + CblasColMajor = 102 }; +enum CBLAS_TRANSPOSE { + CblasNoTrans = 111, + CblasTrans = 112, + CblasConjTrans = 113, + AtlasConj = 114 +}; +enum CBLAS_UPLO { CblasUpper = 121, + CblasLower = 122 }; +enum CBLAS_DIAG { CblasNonUnit = 131, + CblasUnit = 132 }; +enum CBLAS_SIDE { CblasLeft = 141, + CblasRight = 142 }; +#endif + +#ifndef CBLAS_ENUM_ONLY +#define CBLAS_H +#define CBLAS_INDEX int + +int cblas_errprn(int ierr, int info, char* form, ...); +void cblas_xerbla(int p, const char* rout, const char* form, ...); + +/* +* =========================================================================== +* Prototypes for level 1 BLAS functions (complex are recast as routines) +* =========================================================================== +*/ +float cblas_sdsdot(const int N, const float alpha, const float* X, + const int incX, const float* Y, const int incY); +double cblas_dsdot(const int N, const float* X, const int incX, const float* Y, + const int incY); +float cblas_sdot(const int N, const float* X, const int incX, + const float* Y, const int incY); +double cblas_ddot(const int N, const double* X, const int incX, + const double* Y, const int incY); +/* +* Functions having prefixes Z and C only +*/ +void cblas_cdotu_sub(const int N, const void* X, const int incX, + const void* Y, const int incY, void* dotu); +void cblas_cdotc_sub(const int N, const void* X, const int incX, + const void* Y, const int incY, void* dotc); + +void cblas_zdotu_sub(const int N, const void* X, const int incX, + const void* Y, const int incY, void* dotu); +void cblas_zdotc_sub(const int N, const void* X, const int incX, + const void* Y, const int incY, void* dotc); + +/* +* Functions having prefixes S D SC DZ +*/ +float cblas_snrm2(const int N, const float* X, const int incX); +float cblas_sasum(const int N, const float* X, const int incX); + +double cblas_dnrm2(const int N, const double* X, const int incX); +double cblas_dasum(const int N, const double* X, const int incX); + +float cblas_scnrm2(const int N, const void* X, const int incX); +float cblas_scasum(const int N, const void* X, const int incX); + +double cblas_dznrm2(const int N, const void* X, const int incX); +double cblas_dzasum(const int N, const void* X, const int incX); + +/* +* Functions having standard 4 prefixes (S D C Z) +*/ +CBLAS_INDEX cblas_isamax(const int N, const float* X, const int incX); +CBLAS_INDEX cblas_idamax(const int N, const double* X, const int incX); +CBLAS_INDEX cblas_icamax(const int N, const void* X, const int incX); +CBLAS_INDEX cblas_izamax(const int N, const void* X, const int incX); + +/* +* =========================================================================== +* Prototypes for level 1 BLAS routines +* =========================================================================== +*/ + +/* +* Routines with standard 4 prefixes (s, d, c, z) +*/ +void cblas_sswap(const int N, float* X, const int incX, + float* Y, const int incY); +void cblas_scopy(const int N, const float* X, const int incX, + float* Y, const int incY); +void cblas_saxpy(const int N, const float alpha, const float* X, + const int incX, float* Y, const int incY); +void catlas_saxpby(const int N, const float alpha, const float* X, + const int incX, const float beta, float* Y, const int incY); +void catlas_sset(const int N, const float alpha, float* X, const int incX); + +void cblas_dswap(const int N, double* X, const int incX, + double* Y, const int incY); +void cblas_dcopy(const int N, const double* X, const int incX, + double* Y, const int incY); +void cblas_daxpy(const int N, const double alpha, const double* X, + const int incX, double* Y, const int incY); +void catlas_daxpby(const int N, const double alpha, const double* X, + const int incX, const double beta, double* Y, const int incY); +void catlas_dset(const int N, const double alpha, double* X, const int incX); + +void cblas_cswap(const int N, void* X, const int incX, + void* Y, const int incY); +void cblas_ccopy(const int N, const void* X, const int incX, + void* Y, const int incY); +void cblas_caxpy(const int N, const void* alpha, const void* X, + const int incX, void* Y, const int incY); +void catlas_caxpby(const int N, const void* alpha, const void* X, + const int incX, const void* beta, void* Y, const int incY); +void catlas_cset(const int N, const void* alpha, void* X, const int incX); + +void cblas_zswap(const int N, void* X, const int incX, + void* Y, const int incY); +void cblas_zcopy(const int N, const void* X, const int incX, + void* Y, const int incY); +void cblas_zaxpy(const int N, const void* alpha, const void* X, + const int incX, void* Y, const int incY); +void catlas_zaxpby(const int N, const void* alpha, const void* X, + const int incX, const void* beta, void* Y, const int incY); +void catlas_zset(const int N, const void* alpha, void* X, const int incX); + +/* +* Routines with S and D prefix only +*/ +void cblas_srotg(float* a, float* b, float* c, float* s); +void cblas_srotmg(float* d1, float* d2, float* b1, const float b2, float* P); +void cblas_srot(const int N, float* X, const int incX, + float* Y, const int incY, const float c, const float s); +void cblas_srotm(const int N, float* X, const int incX, + float* Y, const int incY, const float* P); + +void cblas_drotg(double* a, double* b, double* c, double* s); +void cblas_drotmg(double* d1, double* d2, double* b1, const double b2, double* P); +void cblas_drot(const int N, double* X, const int incX, + double* Y, const int incY, const double c, const double s); +void cblas_drotm(const int N, double* X, const int incX, + double* Y, const int incY, const double* P); + +/* +* Routines with S D C Z CS and ZD prefixes +*/ +void cblas_sscal(const int N, const float alpha, float* X, const int incX); +void cblas_dscal(const int N, const double alpha, double* X, const int incX); +void cblas_cscal(const int N, const void* alpha, void* X, const int incX); +void cblas_zscal(const int N, const void* alpha, void* X, const int incX); +void cblas_csscal(const int N, const float alpha, void* X, const int incX); +void cblas_zdscal(const int N, const double alpha, void* X, const int incX); + +/* +* Extra reference routines provided by ATLAS, but not mandated by the standard +*/ +void cblas_crotg(void* a, void* b, void* c, void* s); +void cblas_zrotg(void* a, void* b, void* c, void* s); +void cblas_csrot(const int N, void* X, const int incX, void* Y, const int incY, + const float c, const float s); +void cblas_zdrot(const int N, void* X, const int incX, void* Y, const int incY, + const double c, const double s); + +/* +* =========================================================================== +* Prototypes for level 2 BLAS +* =========================================================================== +*/ + +/* +* Routines with standard 4 prefixes (S, D, C, Z) +*/ +void cblas_sgemv(const enum CBLAS_ORDER Order, + const enum CBLAS_TRANSPOSE TransA, const int M, const int N, + const float alpha, const float* A, const int lda, + const float* X, const int incX, const float beta, + float* Y, const int incY); +void cblas_sgbmv(const enum CBLAS_ORDER Order, + const enum CBLAS_TRANSPOSE TransA, const int M, const int N, + const int KL, const int KU, const float alpha, + const float* A, const int lda, const float* X, + const int incX, const float beta, float* Y, const int incY); +void cblas_strmv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const int N, const float* A, const int lda, + float* X, const int incX); +void cblas_stbmv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const int N, const int K, const float* A, const int lda, + float* X, const int incX); +void cblas_stpmv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const int N, const float* Ap, float* X, const int incX); +void cblas_strsv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const int N, const float* A, const int lda, float* X, + const int incX); +void cblas_stbsv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const int N, const int K, const float* A, const int lda, + float* X, const int incX); +void cblas_stpsv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const int N, const float* Ap, float* X, const int incX); + +void cblas_dgemv(const enum CBLAS_ORDER Order, + const enum CBLAS_TRANSPOSE TransA, const int M, const int N, + const double alpha, const double* A, const int lda, + const double* X, const int incX, const double beta, + double* Y, const int incY); +void cblas_dgbmv(const enum CBLAS_ORDER Order, + const enum CBLAS_TRANSPOSE TransA, const int M, const int N, + const int KL, const int KU, const double alpha, + const double* A, const int lda, const double* X, + const int incX, const double beta, double* Y, const int incY); +void cblas_dtrmv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const int N, const double* A, const int lda, + double* X, const int incX); +void cblas_dtbmv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const int N, const int K, const double* A, const int lda, + double* X, const int incX); +void cblas_dtpmv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const int N, const double* Ap, double* X, const int incX); +void cblas_dtrsv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const int N, const double* A, const int lda, double* X, + const int incX); +void cblas_dtbsv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const int N, const int K, const double* A, const int lda, + double* X, const int incX); +void cblas_dtpsv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const int N, const double* Ap, double* X, const int incX); + +void cblas_cgemv(const enum CBLAS_ORDER Order, + const enum CBLAS_TRANSPOSE TransA, const int M, const int N, + const void* alpha, const void* A, const int lda, + const void* X, const int incX, const void* beta, + void* Y, const int incY); +void cblas_cgbmv(const enum CBLAS_ORDER Order, + const enum CBLAS_TRANSPOSE TransA, const int M, const int N, + const int KL, const int KU, const void* alpha, + const void* A, const int lda, const void* X, + const int incX, const void* beta, void* Y, const int incY); +void cblas_ctrmv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const int N, const void* A, const int lda, + void* X, const int incX); +void cblas_ctbmv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const int N, const int K, const void* A, const int lda, + void* X, const int incX); +void cblas_ctpmv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const int N, const void* Ap, void* X, const int incX); +void cblas_ctrsv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const int N, const void* A, const int lda, void* X, + const int incX); +void cblas_ctbsv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const int N, const int K, const void* A, const int lda, + void* X, const int incX); +void cblas_ctpsv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const int N, const void* Ap, void* X, const int incX); + +void cblas_zgemv(const enum CBLAS_ORDER Order, + const enum CBLAS_TRANSPOSE TransA, const int M, const int N, + const void* alpha, const void* A, const int lda, + const void* X, const int incX, const void* beta, + void* Y, const int incY); +void cblas_zgbmv(const enum CBLAS_ORDER Order, + const enum CBLAS_TRANSPOSE TransA, const int M, const int N, + const int KL, const int KU, const void* alpha, + const void* A, const int lda, const void* X, + const int incX, const void* beta, void* Y, const int incY); +void cblas_ztrmv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const int N, const void* A, const int lda, + void* X, const int incX); +void cblas_ztbmv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const int N, const int K, const void* A, const int lda, + void* X, const int incX); +void cblas_ztpmv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const int N, const void* Ap, void* X, const int incX); +void cblas_ztrsv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const int N, const void* A, const int lda, void* X, + const int incX); +void cblas_ztbsv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const int N, const int K, const void* A, const int lda, + void* X, const int incX); +void cblas_ztpsv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const int N, const void* Ap, void* X, const int incX); + +/* +* Routines with S and D prefixes only +*/ +void cblas_ssymv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const float alpha, const float* A, + const int lda, const float* X, const int incX, + const float beta, float* Y, const int incY); +void cblas_ssbmv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const int K, const float alpha, const float* A, + const int lda, const float* X, const int incX, + const float beta, float* Y, const int incY); +void cblas_sspmv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const float alpha, const float* Ap, + const float* X, const int incX, + const float beta, float* Y, const int incY); +void cblas_sger(const enum CBLAS_ORDER Order, const int M, const int N, + const float alpha, const float* X, const int incX, + const float* Y, const int incY, float* A, const int lda); +void cblas_ssyr(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const float alpha, const float* X, + const int incX, float* A, const int lda); +void cblas_sspr(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const float alpha, const float* X, + const int incX, float* Ap); +void cblas_ssyr2(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const float alpha, const float* X, + const int incX, const float* Y, const int incY, float* A, + const int lda); +void cblas_sspr2(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const float alpha, const float* X, + const int incX, const float* Y, const int incY, float* A); + +void cblas_dsymv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const double alpha, const double* A, + const int lda, const double* X, const int incX, + const double beta, double* Y, const int incY); +void cblas_dsbmv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const int K, const double alpha, const double* A, + const int lda, const double* X, const int incX, + const double beta, double* Y, const int incY); +void cblas_dspmv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const double alpha, const double* Ap, + const double* X, const int incX, + const double beta, double* Y, const int incY); +void cblas_dger(const enum CBLAS_ORDER Order, const int M, const int N, + const double alpha, const double* X, const int incX, + const double* Y, const int incY, double* A, const int lda); +void cblas_dsyr(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const double alpha, const double* X, + const int incX, double* A, const int lda); +void cblas_dspr(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const double alpha, const double* X, + const int incX, double* Ap); +void cblas_dsyr2(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const double alpha, const double* X, + const int incX, const double* Y, const int incY, double* A, + const int lda); +void cblas_dspr2(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const double alpha, const double* X, + const int incX, const double* Y, const int incY, double* A); + +/* +* Routines with C and Z prefixes only +*/ +void cblas_chemv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const void* alpha, const void* A, + const int lda, const void* X, const int incX, + const void* beta, void* Y, const int incY); +void cblas_chbmv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const int K, const void* alpha, const void* A, + const int lda, const void* X, const int incX, + const void* beta, void* Y, const int incY); +void cblas_chpmv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const void* alpha, const void* Ap, + const void* X, const int incX, + const void* beta, void* Y, const int incY); +void cblas_cgeru(const enum CBLAS_ORDER Order, const int M, const int N, + const void* alpha, const void* X, const int incX, + const void* Y, const int incY, void* A, const int lda); +void cblas_cgerc(const enum CBLAS_ORDER Order, const int M, const int N, + const void* alpha, const void* X, const int incX, + const void* Y, const int incY, void* A, const int lda); +void cblas_cher(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const float alpha, const void* X, const int incX, + void* A, const int lda); +void cblas_chpr(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const float alpha, const void* X, + const int incX, void* A); +void cblas_cher2(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, const int N, + const void* alpha, const void* X, const int incX, + const void* Y, const int incY, void* A, const int lda); +void cblas_chpr2(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, const int N, + const void* alpha, const void* X, const int incX, + const void* Y, const int incY, void* Ap); + +void cblas_zhemv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const void* alpha, const void* A, + const int lda, const void* X, const int incX, + const void* beta, void* Y, const int incY); +void cblas_zhbmv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const int K, const void* alpha, const void* A, + const int lda, const void* X, const int incX, + const void* beta, void* Y, const int incY); +void cblas_zhpmv(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const void* alpha, const void* Ap, + const void* X, const int incX, + const void* beta, void* Y, const int incY); +void cblas_zgeru(const enum CBLAS_ORDER Order, const int M, const int N, + const void* alpha, const void* X, const int incX, + const void* Y, const int incY, void* A, const int lda); +void cblas_zgerc(const enum CBLAS_ORDER Order, const int M, const int N, + const void* alpha, const void* X, const int incX, + const void* Y, const int incY, void* A, const int lda); +void cblas_zher(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const double alpha, const void* X, const int incX, + void* A, const int lda); +void cblas_zhpr(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const double alpha, const void* X, + const int incX, void* A); +void cblas_zher2(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, const int N, + const void* alpha, const void* X, const int incX, + const void* Y, const int incY, void* A, const int lda); +void cblas_zhpr2(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, const int N, + const void* alpha, const void* X, const int incX, + const void* Y, const int incY, void* Ap); + +/* +* =========================================================================== +* Prototypes for level 3 BLAS +* =========================================================================== +*/ + +/* +* Routines with standard 4 prefixes (S, D, C, Z) +*/ +void cblas_sgemm(const enum CBLAS_ORDER Order, const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_TRANSPOSE TransB, const int M, const int N, + const int K, const float alpha, const float* A, + const int lda, const float* B, const int ldb, + const float beta, float* C, const int ldc); +void cblas_ssymm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const int M, const int N, + const float alpha, const float* A, const int lda, + const float* B, const int ldb, const float beta, + float* C, const int ldc); +void cblas_ssyrk(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE Trans, const int N, const int K, + const float alpha, const float* A, const int lda, + const float beta, float* C, const int ldc); +void cblas_ssyr2k(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE Trans, const int N, const int K, + const float alpha, const float* A, const int lda, + const float* B, const int ldb, const float beta, + float* C, const int ldc); +void cblas_strmm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_DIAG Diag, const int M, const int N, + const float alpha, const float* A, const int lda, + float* B, const int ldb); +void cblas_strsm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_DIAG Diag, const int M, const int N, + const float alpha, const float* A, const int lda, + float* B, const int ldb); + +void cblas_dgemm(const enum CBLAS_ORDER Order, const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_TRANSPOSE TransB, const int M, const int N, + const int K, const double alpha, const double* A, + const int lda, const double* B, const int ldb, + const double beta, double* C, const int ldc); +void cblas_dsymm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const int M, const int N, + const double alpha, const double* A, const int lda, + const double* B, const int ldb, const double beta, + double* C, const int ldc); +void cblas_dsyrk(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE Trans, const int N, const int K, + const double alpha, const double* A, const int lda, + const double beta, double* C, const int ldc); +void cblas_dsyr2k(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE Trans, const int N, const int K, + const double alpha, const double* A, const int lda, + const double* B, const int ldb, const double beta, + double* C, const int ldc); +void cblas_dtrmm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_DIAG Diag, const int M, const int N, + const double alpha, const double* A, const int lda, + double* B, const int ldb); +void cblas_dtrsm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_DIAG Diag, const int M, const int N, + const double alpha, const double* A, const int lda, + double* B, const int ldb); + +void cblas_cgemm(const enum CBLAS_ORDER Order, const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_TRANSPOSE TransB, const int M, const int N, + const int K, const void* alpha, const void* A, + const int lda, const void* B, const int ldb, + const void* beta, void* C, const int ldc); +void cblas_csymm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const int M, const int N, + const void* alpha, const void* A, const int lda, + const void* B, const int ldb, const void* beta, + void* C, const int ldc); +void cblas_csyrk(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE Trans, const int N, const int K, + const void* alpha, const void* A, const int lda, + const void* beta, void* C, const int ldc); +void cblas_csyr2k(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE Trans, const int N, const int K, + const void* alpha, const void* A, const int lda, + const void* B, const int ldb, const void* beta, + void* C, const int ldc); +void cblas_ctrmm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_DIAG Diag, const int M, const int N, + const void* alpha, const void* A, const int lda, + void* B, const int ldb); +void cblas_ctrsm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_DIAG Diag, const int M, const int N, + const void* alpha, const void* A, const int lda, + void* B, const int ldb); + +void cblas_zgemm(const enum CBLAS_ORDER Order, const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_TRANSPOSE TransB, const int M, const int N, + const int K, const void* alpha, const void* A, + const int lda, const void* B, const int ldb, + const void* beta, void* C, const int ldc); +void cblas_zsymm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const int M, const int N, + const void* alpha, const void* A, const int lda, + const void* B, const int ldb, const void* beta, + void* C, const int ldc); +void cblas_zsyrk(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE Trans, const int N, const int K, + const void* alpha, const void* A, const int lda, + const void* beta, void* C, const int ldc); +void cblas_zsyr2k(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE Trans, const int N, const int K, + const void* alpha, const void* A, const int lda, + const void* B, const int ldb, const void* beta, + void* C, const int ldc); +void cblas_ztrmm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_DIAG Diag, const int M, const int N, + const void* alpha, const void* A, const int lda, + void* B, const int ldb); +void cblas_ztrsm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_DIAG Diag, const int M, const int N, + const void* alpha, const void* A, const int lda, + void* B, const int ldb); + +/* +* Routines with prefixes C and Z only +*/ +void cblas_chemm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const int M, const int N, + const void* alpha, const void* A, const int lda, + const void* B, const int ldb, const void* beta, + void* C, const int ldc); +void cblas_cherk(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE Trans, const int N, const int K, + const float alpha, const void* A, const int lda, + const float beta, void* C, const int ldc); +void cblas_cher2k(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE Trans, const int N, const int K, + const void* alpha, const void* A, const int lda, + const void* B, const int ldb, const float beta, + void* C, const int ldc); +void cblas_zhemm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const int M, const int N, + const void* alpha, const void* A, const int lda, + const void* B, const int ldb, const void* beta, + void* C, const int ldc); +void cblas_zherk(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE Trans, const int N, const int K, + const double alpha, const void* A, const int lda, + const double beta, void* C, const int ldc); +void cblas_zher2k(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE Trans, const int N, const int K, + const void* alpha, const void* A, const int lda, + const void* B, const int ldb, const double beta, + void* C, const int ldc); + +int cblas_errprn(int ierr, int info, char* form, ...); + +#endif /* end #ifdef CBLAS_ENUM_ONLY */ +#endif // CAFFE2_USE_MKL +#endif diff --git a/onnxruntime/core/framework/computation_capacity.h b/onnxruntime/core/framework/computation_capacity.h new file mode 100644 index 0000000000000..937c06dc9e6d7 --- /dev/null +++ b/onnxruntime/core/framework/computation_capacity.h @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/common/common.h" +#include "core/graph/indexed_sub_graph.h" + +namespace onnxruntime { +//A structure to descripe which part of the graph could be run, and how to run it. +struct ComputationCapacity { + // IndexdSubGraph descript the subgraph that current XP could run, it could be a single node or multiple nodes. + std::unique_ptr sub_graph_; + // If execution provider want to fuse a sub-graph into a kernel generated on the fly (means can't be pre-defined.) + // It can set the kernel create function in this field, so onnxruntime will create the kernel for it. + // Otherwise onnxruntime will search kernels in pre-defined kernel registry provided by XP. + KernelCreateFn fuse_kernel_function_; + + //TODO: if there is a FusedKernelFn attached, onnxruntime will generate the default KernelDefinition for it, according to the OpSchema generated on the fly + //A default kernel definition will cover the basic fields, like the input/output constrains. + //If execution provider want to set some advanced fields on kernel definition, like memory placement / in-place annotation. + //A KernelDefinitionDecrator function is needed. + + ComputationCapacity() : sub_graph_(nullptr), fuse_kernel_function_(nullptr) {} + + ComputationCapacity(std::unique_ptr sub_graph, KernelCreateFn kernel_func) + : sub_graph_(std::move(sub_graph)), + fuse_kernel_function_(kernel_func) {} +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/customregistry.cc b/onnxruntime/core/framework/customregistry.cc new file mode 100644 index 0000000000000..0252f261bd49a --- /dev/null +++ b/onnxruntime/core/framework/customregistry.cc @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/customregistry.h" +namespace onnxruntime { + +common::Status CustomRegistry::RegisterCustomKernel(KernelDefBuilder& kernel_def_builder, const KernelCreateFn& kernel_creator) { + return Register(kernel_def_builder, kernel_creator); +} + +common::Status CustomRegistry::RegisterCustomKernel(KernelCreateInfo& create_info) { + return Register(create_info); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/data_types.cc b/onnxruntime/core/framework/data_types.cc new file mode 100644 index 0000000000000..7255d8eebc605 --- /dev/null +++ b/onnxruntime/core/framework/data_types.cc @@ -0,0 +1,736 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/data_types.h" +#include "core/framework/tensor.h" +#include "core/graph/onnx_protobuf.h" + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wignored-qualifiers" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif +#include "onnx/defs/data_type_utils.h" +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + +using namespace ONNX_NAMESPACE; +namespace onnxruntime { + +template <> +MLDataType DataTypeImpl::GetType() { + return TensorTypeBase::Type(); +} + +static bool IsTensorTypeScalar(const ONNX_NAMESPACE::TypeProto_Tensor& tensor_type_proto) { + int sz = tensor_type_proto.shape().dim_size(); + return sz == 0 || sz == 1; +} + +namespace data_types_internal { + +template +constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType(); + +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_FLOAT; +} +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_UINT8; +}; +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_INT8; +}; +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_UINT16; +}; +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_INT16; +}; +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_INT32; +}; +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_INT64; +}; +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_STRING; +}; +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_BOOL; +}; +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_FLOAT16; +}; +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_DOUBLE; +}; +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_UINT32; +}; +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_UINT64; +}; + +template +struct TensorContainedTypeSetter { + static void SetTensorElementType(ONNX_NAMESPACE::TypeProto& proto) { + proto.mutable_tensor_type()->set_elem_type(ToTensorDataType()); + } + static void SetMapKeyType(ONNX_NAMESPACE::TypeProto& proto) { + proto.mutable_map_type()->set_key_type(ToTensorDataType()); + } +}; + +// Pre-instantiate +template struct + TensorContainedTypeSetter; +template struct + TensorContainedTypeSetter; +template struct + TensorContainedTypeSetter; +template struct + TensorContainedTypeSetter; +template struct + TensorContainedTypeSetter; +template struct + TensorContainedTypeSetter; +template struct + TensorContainedTypeSetter; +template struct + TensorContainedTypeSetter; +template struct + TensorContainedTypeSetter; +template struct + TensorContainedTypeSetter; +template struct + TensorContainedTypeSetter; +template struct + TensorContainedTypeSetter; +template struct + TensorContainedTypeSetter; + +void CopyMutableMapValue(const ONNX_NAMESPACE::TypeProto& value_proto, + ONNX_NAMESPACE::TypeProto& map_proto) { + map_proto.mutable_map_type()->mutable_value_type()->CopyFrom(value_proto); +} + +void CopyMutableSeqElement(const ONNX_NAMESPACE::TypeProto& elem_proto, + ONNX_NAMESPACE::TypeProto& proto) { + proto.mutable_sequence_type()->mutable_elem_type()->CopyFrom(elem_proto); +} + +void AssignOpaqueDomainName(const char* domain, const char* name, + ONNX_NAMESPACE::TypeProto& proto) { + auto* mutable_opaque = proto.mutable_opaque_type(); + mutable_opaque->mutable_domain()->assign(domain); + mutable_opaque->mutable_name()->assign(name); +} + +bool IsCompatible(const ONNX_NAMESPACE::TypeProto_Tensor& tensor_proto, + const ONNX_NAMESPACE::TypeProto_Tensor& type_proto); + +bool IsCompatible(const ONNX_NAMESPACE::TypeProto_SparseTensor& tensor_proto, + const ONNX_NAMESPACE::TypeProto_SparseTensor& type_proto); + +bool IsCompatible(const ONNX_NAMESPACE::TypeProto_Map& map_proto, + const ONNX_NAMESPACE::TypeProto_Map& type_proto); + +bool IsCompatible(const ONNX_NAMESPACE::TypeProto_Sequence& sequence_proto, + const ONNX_NAMESPACE::TypeProto_Sequence& type_proto); + +bool IsCompatible(const ONNX_NAMESPACE::TypeProto_Opaque& opaque_proto, + const ONNX_NAMESPACE::TypeProto_Opaque& type_proto); + +bool IsCompatible(const ONNX_NAMESPACE::TypeProto_Tensor& tensor_proto, + const ONNX_NAMESPACE::TypeProto_Tensor& type_proto) { + return type_proto.has_elem_type() && + type_proto.elem_type() == tensor_proto.elem_type(); + /* Currently all Tensors with all kinds of shapes + are mapped into the same MLDataType (same element type) + so we omit shape from IsCompatible consideration + */ +} + +bool IsCompatible(const ONNX_NAMESPACE::TypeProto_Map& map_proto, + const ONNX_NAMESPACE::TypeProto_Map& type_proto) { + if (!(type_proto.has_key_type() && + type_proto.key_type() == map_proto.key_type())) { + return false; + } + const auto& lhs = map_proto; + const auto& rhs = type_proto; + bool result = true; + if (lhs.key_type() == rhs.key_type() && + lhs.value_type().value_case() == rhs.value_type().value_case()) { + switch (lhs.value_type().value_case()) { + case TypeProto::ValueCase::kTensorType: + result = IsCompatible(lhs.value_type().tensor_type(), rhs.value_type().tensor_type()); + break; + case TypeProto::ValueCase::kSequenceType: + result = IsCompatible(lhs.value_type().sequence_type(), rhs.value_type().sequence_type()); + break; + case TypeProto::ValueCase::kMapType: + result = IsCompatible(lhs.value_type().map_type(), rhs.value_type().map_type()); + break; + case TypeProto::ValueCase::kOpaqueType: + result = IsCompatible(lhs.value_type().opaque_type(), rhs.value_type().opaque_type()); + break; + case TypeProto::ValueCase::kSparseTensorType: + result = IsCompatible(lhs.value_type().sparse_tensor_type(), rhs.value_type().sparse_tensor_type()); + break; + default: + ONNXRUNTIME_ENFORCE(false); + break; + } + } else { + result = false; + } + return result; +} + +bool IsCompatible(const ONNX_NAMESPACE::TypeProto_Sequence& sequence_proto, + const ONNX_NAMESPACE::TypeProto_Sequence& type_proto) { + bool result = true; + const auto& lhs = sequence_proto; + const auto& rhs = type_proto; + if (rhs.has_elem_type() && + lhs.elem_type().value_case() == rhs.elem_type().value_case()) { + switch (lhs.elem_type().value_case()) { + case TypeProto::ValueCase::kTensorType: + result = IsCompatible(lhs.elem_type().tensor_type(), rhs.elem_type().tensor_type()); + break; + case TypeProto::ValueCase::kSequenceType: + result = IsCompatible(lhs.elem_type().sequence_type(), rhs.elem_type().sequence_type()); + break; + case TypeProto::ValueCase::kMapType: + result = IsCompatible(lhs.elem_type().map_type(), rhs.elem_type().map_type()); + break; + case TypeProto::ValueCase::kOpaqueType: + result = IsCompatible(lhs.elem_type().opaque_type(), rhs.elem_type().opaque_type()); + break; + case TypeProto::ValueCase::kSparseTensorType: + result = IsCompatible(lhs.elem_type().sparse_tensor_type(), rhs.elem_type().sparse_tensor_type()); + break; + default: + ONNXRUNTIME_ENFORCE(false); + break; + } + } else { + result = false; + } + return result; +} +bool IsCompatible(const ONNX_NAMESPACE::TypeProto_Opaque& opaque_proto, const ONNX_NAMESPACE::TypeProto_Opaque& type_proto) { + const auto& lhs = opaque_proto; + const auto& rhs = type_proto; + bool lhs_domain = lhs.has_domain() && !lhs.domain().empty(); + bool rhs_domain = rhs.has_domain() && !rhs.domain().empty(); + + if ((lhs_domain != rhs_domain) || + (lhs_domain && rhs_domain && lhs.domain() != lhs.domain())) { + return false; + } + + bool lhs_name = lhs.has_name() && !lhs.name().empty(); + bool rhs_name = rhs.has_name() && !rhs.name().empty(); + + return !((lhs_name != rhs_name) || + (lhs_name && rhs_name && lhs.name() != rhs.name())); +} + +bool IsCompatible(const ONNX_NAMESPACE::TypeProto_SparseTensor& tensor_proto, + const ONNX_NAMESPACE::TypeProto_SparseTensor& type_proto) { + return type_proto.has_elem_type() && + type_proto.elem_type() == tensor_proto.elem_type(); + // XXX: Ignoring shape for now +} + +void RegisterAllProtos(const std::function& /*reg_fn*/); + +class DataTypeRegistry { + std::unordered_map mapping_; + + DataTypeRegistry() { + RegisterAllProtos([this](MLDataType mltype) { RegisterDataType(mltype); }); + } + + ~DataTypeRegistry() = default; + + public: + DataTypeRegistry(const DataTypeRegistry&) = delete; + DataTypeRegistry& operator=(const DataTypeRegistry&) = delete; + + static DataTypeRegistry& instance() { + static DataTypeRegistry inst; + return inst; + } + + void RegisterDataType(MLDataType mltype) { + using namespace ONNX_NAMESPACE; + const auto* proto = mltype->GetTypeProto(); + ONNXRUNTIME_ENFORCE(proto != nullptr, "Only ONNX MLDataType can be registered"); + DataType type = Utils::DataTypeUtils::ToType(*proto); + auto p = mapping_.insert(std::make_pair(type, mltype)); + ONNXRUNTIME_ENFORCE(p.second, "We do not expect duplicate registration of types for: ", type); + } + + MLDataType GetMLDataType(const ONNX_NAMESPACE::TypeProto& proto) const { + using namespace ONNX_NAMESPACE; + DataType type = Utils::DataTypeUtils::ToType(proto); + auto p = mapping_.find(type); + if (p != mapping_.end()) { + return p->second; + } + return nullptr; + } +}; + +struct TypeProtoImpl { + const TypeProto* GetProto() const { + return &proto_; + } + TypeProto& mutable_type_proto() { + return proto_; + } + + TypeProto proto_; +}; + +} // namespace data_types_internal + +/// TensorTypeBase +struct TensorTypeBase::Impl : public data_types_internal::TypeProtoImpl { +}; + +const ONNX_NAMESPACE::TypeProto* TensorTypeBase::GetTypeProto() const { + return impl_->GetProto(); +} + +TensorTypeBase::TensorTypeBase() : impl_(new Impl()) {} +TensorTypeBase::~TensorTypeBase() { + delete impl_; +} + +size_t TensorTypeBase::Size() const { + return sizeof(Tensor); +} + +template +static void Delete(void* p) { + delete static_cast(p); +} + +DeleteFunc TensorTypeBase::GetDeleteFunc() const { + return &Delete; +} + +ONNX_NAMESPACE::TypeProto& TensorTypeBase::mutable_type_proto() { + return impl_->mutable_type_proto(); +} + +bool TensorTypeBase::IsCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const { + const auto* thisProto = GetTypeProto(); + if (&type_proto == thisProto) { + return true; + } + if (type_proto.value_case() != TypeProto::ValueCase::kTensorType) { + return false; + } + + ONNXRUNTIME_ENFORCE(thisProto->value_case() == TypeProto::ValueCase::kTensorType); + ONNXRUNTIME_ENFORCE(thisProto->tensor_type().has_elem_type()); + + return data_types_internal::IsCompatible(thisProto->tensor_type(), type_proto.tensor_type()); +} + +MLDataType TensorTypeBase::Type() { + static TensorTypeBase tensor_base; + return &tensor_base; +} + +/// NoTensorTypeBase +struct NonTensorTypeBase::Impl : public data_types_internal::TypeProtoImpl {}; + +NonTensorTypeBase::NonTensorTypeBase() : impl_(new Impl()) { +} + +NonTensorTypeBase::~NonTensorTypeBase() { + delete impl_; +} + +ONNX_NAMESPACE::TypeProto& NonTensorTypeBase::mutable_type_proto() { + return impl_->mutable_type_proto(); +} + +const ONNX_NAMESPACE::TypeProto* NonTensorTypeBase::GetTypeProto() const { + return impl_->GetProto(); +} + +bool NonTensorTypeBase::IsMapCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const { + const auto* thisProto = impl_->GetProto(); + if (&type_proto == thisProto) { + return true; + } + if (type_proto.value_case() != TypeProto::ValueCase::kMapType) { + return false; + } + ONNXRUNTIME_ENFORCE(thisProto->value_case() == TypeProto::ValueCase::kMapType); + ONNXRUNTIME_ENFORCE(thisProto->map_type().has_key_type()); + ONNXRUNTIME_ENFORCE(thisProto->map_type().has_value_type()); + return data_types_internal::IsCompatible(thisProto->map_type(), type_proto.map_type()); +} + +bool NonTensorTypeBase::IsSequenceCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const { + const auto* thisProto = impl_->GetProto(); + if (&type_proto == thisProto) { + return true; + } + if (type_proto.value_case() != TypeProto::ValueCase::kSequenceType) { + return false; + } + ONNXRUNTIME_ENFORCE(thisProto->value_case() == TypeProto::ValueCase::kSequenceType); + ONNXRUNTIME_ENFORCE(thisProto->sequence_type().has_elem_type()); + return data_types_internal::IsCompatible(thisProto->sequence_type(), type_proto.sequence_type()); +} + +bool NonTensorTypeBase::IsOpaqueCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const { + const auto* thisProto = impl_->GetProto(); + if (&type_proto == thisProto) { + return true; + } + if (type_proto.value_case() != TypeProto::ValueCase::kOpaqueType) { + return false; + } + ONNXRUNTIME_ENFORCE(thisProto->value_case() == TypeProto::ValueCase::kOpaqueType); + ONNXRUNTIME_ENFORCE(thisProto->opaque_type().has_domain()); + ONNXRUNTIME_ENFORCE(thisProto->opaque_type().has_name()); + return data_types_internal::IsCompatible(thisProto->opaque_type(), type_proto.opaque_type()); +} + +ONNXRUNTIME_REGISTER_TENSOR_TYPE(int32_t); +ONNXRUNTIME_REGISTER_TENSOR_TYPE(float); +ONNXRUNTIME_REGISTER_TENSOR_TYPE(bool); +ONNXRUNTIME_REGISTER_TENSOR_TYPE(std::string); +ONNXRUNTIME_REGISTER_TENSOR_TYPE(int8_t); +ONNXRUNTIME_REGISTER_TENSOR_TYPE(uint8_t); +ONNXRUNTIME_REGISTER_TENSOR_TYPE(uint16_t); +ONNXRUNTIME_REGISTER_TENSOR_TYPE(int16_t); +ONNXRUNTIME_REGISTER_TENSOR_TYPE(int64_t); +ONNXRUNTIME_REGISTER_TENSOR_TYPE(double); +ONNXRUNTIME_REGISTER_TENSOR_TYPE(uint32_t); +ONNXRUNTIME_REGISTER_TENSOR_TYPE(uint64_t); +ONNXRUNTIME_REGISTER_TENSOR_TYPE(MLFloat16); + +ONNXRUNTIME_REGISTER_MAP(MapStringToString); +ONNXRUNTIME_REGISTER_MAP(MapStringToInt64); +ONNXRUNTIME_REGISTER_MAP(MapStringToFloat); +ONNXRUNTIME_REGISTER_MAP(MapStringToDouble); +ONNXRUNTIME_REGISTER_MAP(MapInt64ToString); +ONNXRUNTIME_REGISTER_MAP(MapInt64ToInt64); +ONNXRUNTIME_REGISTER_MAP(MapInt64ToFloat); +ONNXRUNTIME_REGISTER_MAP(MapInt64ToDouble); + +ONNXRUNTIME_REGISTER_SEQ(VectorString); +ONNXRUNTIME_REGISTER_SEQ(VectorFloat); +ONNXRUNTIME_REGISTER_SEQ(VectorInt64); +ONNXRUNTIME_REGISTER_SEQ(VectorDouble); + +ONNXRUNTIME_REGISTER_SEQ(VectorMapStringToFloat); +ONNXRUNTIME_REGISTER_SEQ(VectorMapInt64ToFloat); + +// Used for Tensor Proto registrations +#define REGISTER_TENSOR_PROTO(TYPE, reg_fn) \ + { \ + MLDataType mltype = DataTypeImpl::GetTensorType(); \ + reg_fn(mltype); \ + } + +#define REGISTER_ONNX_PROTO(TYPE, reg_fn) \ + { \ + MLDataType mltype = DataTypeImpl::GetType(); \ + reg_fn(mltype); \ + } + +namespace data_types_internal { + +void RegisterAllProtos(const std::function& reg_fn) { + REGISTER_TENSOR_PROTO(int32_t, reg_fn); + REGISTER_TENSOR_PROTO(float, reg_fn); + REGISTER_TENSOR_PROTO(bool, reg_fn); + REGISTER_TENSOR_PROTO(std::string, reg_fn); + REGISTER_TENSOR_PROTO(int8_t, reg_fn); + REGISTER_TENSOR_PROTO(uint8_t, reg_fn); + REGISTER_TENSOR_PROTO(uint16_t, reg_fn); + REGISTER_TENSOR_PROTO(int16_t, reg_fn); + REGISTER_TENSOR_PROTO(int64_t, reg_fn); + REGISTER_TENSOR_PROTO(double, reg_fn); + REGISTER_TENSOR_PROTO(uint32_t, reg_fn); + REGISTER_TENSOR_PROTO(uint64_t, reg_fn); + REGISTER_TENSOR_PROTO(MLFloat16, reg_fn); + + REGISTER_ONNX_PROTO(MapStringToString, reg_fn); + REGISTER_ONNX_PROTO(MapStringToInt64, reg_fn); + REGISTER_ONNX_PROTO(MapStringToFloat, reg_fn); + REGISTER_ONNX_PROTO(MapStringToDouble, reg_fn); + REGISTER_ONNX_PROTO(MapInt64ToString, reg_fn); + REGISTER_ONNX_PROTO(MapInt64ToInt64, reg_fn); + REGISTER_ONNX_PROTO(MapInt64ToFloat, reg_fn); + REGISTER_ONNX_PROTO(MapInt64ToDouble, reg_fn); + + REGISTER_ONNX_PROTO(VectorString, reg_fn); + REGISTER_ONNX_PROTO(VectorFloat, reg_fn); + REGISTER_ONNX_PROTO(VectorInt64, reg_fn); + REGISTER_ONNX_PROTO(VectorDouble, reg_fn); + + REGISTER_ONNX_PROTO(VectorMapStringToFloat, reg_fn); + REGISTER_ONNX_PROTO(VectorMapInt64ToFloat, reg_fn); +} +} // namespace data_types_internal + +void DataTypeImpl::RegisterDataType(MLDataType mltype) { + data_types_internal::DataTypeRegistry::instance().RegisterDataType(mltype); +} + +MLDataType DataTypeImpl::TypeFromProto(const ONNX_NAMESPACE::TypeProto& proto) { + const auto& registry = data_types_internal::DataTypeRegistry::instance(); + + switch (proto.value_case()) { + case TypeProto::ValueCase::kTensorType: { + const auto& tensor_type = proto.tensor_type(); + ONNXRUNTIME_ENFORCE(tensor_type.has_elem_type()); + switch (tensor_type.elem_type()) { + case TensorProto_DataType_FLOAT: + return DataTypeImpl::GetTensorType(); + case TensorProto_DataType_BOOL: + return DataTypeImpl::GetTensorType(); + case TensorProto_DataType_INT32: + return DataTypeImpl::GetTensorType(); + case TensorProto_DataType_DOUBLE: + return DataTypeImpl::GetTensorType(); + case TensorProto_DataType_STRING: + return DataTypeImpl::GetTensorType(); + case TensorProto_DataType_UINT8: + return DataTypeImpl::GetTensorType(); + case TensorProto_DataType_UINT16: + return DataTypeImpl::GetTensorType(); + case TensorProto_DataType_INT8: + return DataTypeImpl::GetTensorType(); + case TensorProto_DataType_INT16: + return DataTypeImpl::GetTensorType(); + case TensorProto_DataType_INT64: + return DataTypeImpl::GetTensorType(); + case TensorProto_DataType_UINT32: + return DataTypeImpl::GetTensorType(); + case TensorProto_DataType_UINT64: + return DataTypeImpl::GetTensorType(); + case TensorProto_DataType_FLOAT16: + return DataTypeImpl::GetTensorType(); + default: + ONNXRUNTIME_NOT_IMPLEMENTED("tensor type ", tensor_type.elem_type(), " is not supported"); + } + } break; // kTensorType + case TypeProto::ValueCase::kMapType: { + const auto& maptype = proto.map_type(); + auto keytype = maptype.key_type(); + const auto& value_type = maptype.value_type(); + + if (value_type.value_case() == TypeProto::ValueCase::kTensorType && + IsTensorTypeScalar(value_type.tensor_type())) { + auto value_elem_type = value_type.tensor_type().elem_type(); + switch (value_elem_type) { + case TensorProto_DataType_STRING: { + switch (keytype) { + case TensorProto_DataType_STRING: + return DataTypeImpl::GetType(); + case TensorProto_DataType_INT64: + return DataTypeImpl::GetType(); + default: + break; + } + } break; + case TensorProto_DataType_INT64: + switch (keytype) { + case TensorProto_DataType_STRING: + return DataTypeImpl::GetType(); + case TensorProto_DataType_INT64: + return DataTypeImpl::GetType(); + default: + break; + } + break; + case TensorProto_DataType_FLOAT: + switch (keytype) { + case TensorProto_DataType_STRING: + return DataTypeImpl::GetType(); + case TensorProto_DataType_INT64: + return DataTypeImpl::GetType(); + default: + break; + } + break; + case TensorProto_DataType_DOUBLE: + switch (keytype) { + case TensorProto_DataType_STRING: + return DataTypeImpl::GetType(); + case TensorProto_DataType_INT64: + return DataTypeImpl::GetType(); + default: + break; + } + break; + default: + break; + } + MLDataType type = registry.GetMLDataType(proto); + ONNXRUNTIME_ENFORCE(type != nullptr, "Map with key type: ", keytype, " value type: ", value_elem_type, " is not registered"); + return type; + } // not if(scalar tensor) pre-reg types + MLDataType type = registry.GetMLDataType(proto); + if (type == nullptr) { + DataType str_type = ONNX_NAMESPACE::Utils::DataTypeUtils::ToType(proto); + ONNXRUNTIME_NOT_IMPLEMENTED("type: ", *str_type, " is not registered"); + } + return type; + + } break; // kMapType + case TypeProto::ValueCase::kSequenceType: { + auto& seq_type = proto.sequence_type(); + auto& val_type = seq_type.elem_type(); + + switch (val_type.value_case()) { + case TypeProto::ValueCase::kMapType: { + auto& maptype = val_type.map_type(); + auto keytype = maptype.key_type(); + auto& value_type = maptype.value_type(); + + if (value_type.value_case() == TypeProto::ValueCase::kTensorType && + IsTensorTypeScalar(value_type.tensor_type())) { + auto value_elem_type = value_type.tensor_type().elem_type(); + switch (value_elem_type) { + case TensorProto_DataType_FLOAT: { + switch (keytype) { + case TensorProto_DataType_STRING: + return DataTypeImpl::GetType(); + case TensorProto_DataType_INT64: + return DataTypeImpl::GetType(); + default: + break; + } + } + default: + break; + } + } + } // MapType + break; + case TypeProto::ValueCase::kTensorType: { + auto val_elem_type = val_type.tensor_type().elem_type(); + switch (val_elem_type) { + case TensorProto_DataType_STRING: + return DataTypeImpl::GetType(); + case TensorProto_DataType_INT64: + return DataTypeImpl::GetType(); + case TensorProto_DataType_FLOAT: + return DataTypeImpl::GetType(); + case TensorProto_DataType_DOUBLE: + return DataTypeImpl::GetType(); + default: + break; + } + } // kTensorType + break; + default: + break; + } // Sequence value case + } // kSequenceType + break; + default: + break; + } // proto.value_case() + MLDataType type = registry.GetMLDataType(proto); + if (type == nullptr) { + DataType str_type = ONNX_NAMESPACE::Utils::DataTypeUtils::ToType(proto); + ONNXRUNTIME_NOT_IMPLEMENTED("type: ", *str_type, " is not currently registered or supported"); + } + return type; +} + +//Below are the types the we need to execute the runtime +//They are not compatible with TypeProto in ONNX. +ONNXRUNTIME_REGISTER_NON_ONNX_TYPE(int32_t); +ONNXRUNTIME_REGISTER_NON_ONNX_TYPE(float); +ONNXRUNTIME_REGISTER_NON_ONNX_TYPE(bool); +ONNXRUNTIME_REGISTER_NON_ONNX_TYPE(std::string); +ONNXRUNTIME_REGISTER_NON_ONNX_TYPE(int8_t); +ONNXRUNTIME_REGISTER_NON_ONNX_TYPE(uint8_t); +ONNXRUNTIME_REGISTER_NON_ONNX_TYPE(uint16_t); +ONNXRUNTIME_REGISTER_NON_ONNX_TYPE(int16_t); +ONNXRUNTIME_REGISTER_NON_ONNX_TYPE(int64_t); +ONNXRUNTIME_REGISTER_NON_ONNX_TYPE(double); +ONNXRUNTIME_REGISTER_NON_ONNX_TYPE(uint32_t); +ONNXRUNTIME_REGISTER_NON_ONNX_TYPE(uint64_t); +ONNXRUNTIME_REGISTER_NON_ONNX_TYPE(MLFloat16); + +const std::vector& DataTypeImpl::AllFixedSizeTensorTypes() { + static std::vector all_fixed_size_tensor_types = + {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}; + + return all_fixed_size_tensor_types; +} + +const std::vector& DataTypeImpl::AllTensorTypes() { + static std::vector all_tensor_types = + {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}; + + return all_tensor_types; +} + +// helper to stream. expected to only be used for error output, so any typeid lookup +// cost should be fine. alternative would be to add a static string field to DataTypeImpl +// that we set in the register macro to the type name, and output that instead. +std::ostream& operator<<(std::ostream& out, const MLDataType data_type) { + if (data_type == nullptr) + return out << "(null)"; + + return out << typeid(*data_type).name(); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/empty.cc b/onnxruntime/core/framework/empty.cc new file mode 100644 index 0000000000000..2a6d2d24bcfb9 --- /dev/null +++ b/onnxruntime/core/framework/empty.cc @@ -0,0 +1,5 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// empty cc file added to satisfy cmake when building a shared +// lib from archives only. diff --git a/onnxruntime/core/framework/environment.cc b/onnxruntime/core/framework/environment.cc new file mode 100644 index 0000000000000..48bc2fcff174b --- /dev/null +++ b/onnxruntime/core/framework/environment.cc @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/environment.h" +#include "core/framework/allocatormgr.h" +#include "core/graph/constants.h" +#include "core/graph/op.h" +#include "onnx/defs/schema.h" +#include "contrib_ops/contrib_ops.h" + +namespace onnxruntime { +using namespace ::onnxruntime::common; +using namespace ONNX_NAMESPACE; + +std::once_flag schemaRegistrationOnceFlag; + +Status Environment::Create(std::unique_ptr& environment) { + environment = std::unique_ptr(new Environment()); + auto status = environment->Initialize(); + return status; +} + +Status Environment::Initialize() { + auto status = Status::OK(); + + try { + // Register Microsoft domain with min/max op_set version as 1/1. + std::call_once(schemaRegistrationOnceFlag, []() { + ONNX_NAMESPACE::OpSchemaRegistry::DomainToVersionRange::Instance().AddDomainToVersion(onnxruntime::kMSDomain, 1, 1); + }); + + // Register MVN operator for backward compatibility. + // Experimental operator does not have history kept in ONNX. Unfortunately, RS5 takes bunch of experimental operators + // in onnx as production ops. MVN is one of them. Now (9/26/2018) MVN is a production function in ONNX. The experimental + // MVN op was removed. The history has to be kept locally as below. + ONNXRUNTIME_ATTRIBUTE_UNUSED ONNX_OPERATOR_SCHEMA(MeanVarianceNormalization) + .SetDoc(R"DOC(Perform mean variance normalization.)DOC") + .Attr("across_channels", "If 1, mean and variance are computed across channels. Default is 0.", AttributeProto::INT, static_cast(0)) + .Attr("normalize_variance", "If 0, normalize the mean only. Default is 1.", AttributeProto::INT, static_cast(1)) + .Input(0, "input", "Input tensor of shape [N,C,H,W]", "T") + .Output(0, "output", "Result, has same shape and type as input", "T") + .TypeConstraint( + "T", + {"tensor(float16)", "tensor(float)", "tensor(double)"}, + "Constrain input and output types to float tensors.") + .TypeAndShapeInferenceFunction(propagateShapeAndTypeFromFirstInput); + // MVN operator is deprecated since operator set 9 (replaced with MVN function). + ONNXRUNTIME_ATTRIBUTE_UNUSED ONNX_OPERATOR_SCHEMA(MeanVarianceNormalization) + .SetDoc(R"DOC(Perform mean variance normalization.)DOC") + .SinceVersion(9) + .Deprecate() + .Attr("across_channels", "If 1, mean and variance are computed across channels. Default is 0.", AttributeProto::INT, static_cast(0)) + .Attr("normalize_variance", "If 0, normalize the mean only. Default is 1.", AttributeProto::INT, static_cast(1)) + .Input(0, "input", "Input tensor of shape [N,C,H,W]", "T") + .Output(0, "output", "Result, has same shape and type as input", "T") + .TypeConstraint( + "T", + {"tensor(float16)", "tensor(float)", "tensor(double)"}, + "Constrain input and output types to float tensors.") + .TypeAndShapeInferenceFunction(propagateShapeAndTypeFromFirstInput); + + // Register MemCpy schema; + + // These ops are internal-only, so register outside of onnx + ONNXRUNTIME_ATTRIBUTE_UNUSED ONNX_OPERATOR_SCHEMA(MemcpyFromHost) + .Input(0, "X", "input", "T") + .Output(0, "Y", "output", "T") + .TypeConstraint( + "T", + OpSchema::all_tensor_types(), + "Constrain to any tensor type. If the dtype attribute is not provided this must be a valid output type.") + .TypeAndShapeInferenceFunction(propagateShapeAndTypeFromFirstInput) + .SetDoc(R"DOC( +Internal copy node +)DOC"); + + ONNXRUNTIME_ATTRIBUTE_UNUSED ONNX_OPERATOR_SCHEMA(MemcpyToHost) + .Input(0, "X", "input", "T") + .Output(0, "Y", "output", "T") + .TypeConstraint( + "T", + OpSchema::all_tensor_types(), + "Constrain to any tensor type. If the dtype attribute is not provided this must be a valid output type.") + .TypeAndShapeInferenceFunction(propagateShapeAndTypeFromFirstInput) + .SetDoc(R"DOC( +Internal copy node +)DOC"); + + // Register contributed schemas. + // The corresponding kernels are registered inside the appropriate execution provider. + contrib::RegisterContribSchemas(); + } catch (std::exception& ex) { + status = Status{ONNXRUNTIME, common::RUNTIME_EXCEPTION, std::string{"Exception caught: "} + ex.what()}; + } catch (...) { + status = Status{ONNXRUNTIME, common::RUNTIME_EXCEPTION}; + } + + return status; +} + +Environment::~Environment() { + ::google::protobuf::ShutdownProtobufLibrary(); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/error_code.cc b/onnxruntime/core/framework/error_code.cc new file mode 100644 index 0000000000000..f2ea3ae366f2c --- /dev/null +++ b/onnxruntime/core/framework/error_code.cc @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/error_code.h" +#include "core/common/status.h" +#include "core/framework/error_code_helper.h" +#include +using onnxruntime::common::Status; + +ONNXRUNTIME_API(ONNXStatusPtr, CreateONNXStatus, ONNXRuntimeErrorCode code, const char* msg) { + assert(!(code == 0 && msg != nullptr)); + size_t clen = strlen(msg); + size_t len = clen + 1 + sizeof(int); + char* p = new char[len]; + char* ret = p; + *reinterpret_cast(p) = static_cast(code); + p += sizeof(int); + memcpy(p, msg, clen); + p += clen; + *p = '\0'; + return ret; +} +namespace onnxruntime { +ONNXStatusPtr ToONNXStatus(const Status& st) { + if (st.IsOK()) + return nullptr; + size_t clen = st.ErrorMessage().length(); + size_t len = clen + 1 + sizeof(int); + char* p = new char[len]; + char* ret = p; + *reinterpret_cast(p) = static_cast(st.Code()); + p += sizeof(int); + memcpy(p, st.ErrorMessage().c_str(), clen); + p += clen; + *p = '\0'; + return ret; +} +} // namespace onnxruntime +ONNXRUNTIME_API(ONNXRuntimeErrorCode, ONNXRuntimeGetErrorCode, _In_ const ONNXStatusPtr status) { + return *reinterpret_cast(status); +} + +ONNXRUNTIME_API(const char*, ONNXRuntimeGetErrorMessage, _In_ const ONNXStatusPtr status) { + return reinterpret_cast(status) + sizeof(int); +} diff --git a/onnxruntime/core/framework/error_code_helper.h b/onnxruntime/core/framework/error_code_helper.h new file mode 100644 index 0000000000000..b3d0564b9ad8b --- /dev/null +++ b/onnxruntime/core/framework/error_code_helper.h @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/status.h" +#include "core/framework/error_code.h" + +namespace onnxruntime { +ONNXStatusPtr ToONNXStatus(const onnxruntime::common::Status& st); +}; diff --git a/onnxruntime/core/framework/execution_frame.cc b/onnxruntime/core/framework/execution_frame.cc new file mode 100644 index 0000000000000..b6993932ef8ee --- /dev/null +++ b/onnxruntime/core/framework/execution_frame.cc @@ -0,0 +1,505 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/execution_frame.h" + +#include + +#include "core/framework/mem_pattern_planner.h" +#include "core/framework/ml_value_patterns_planner.h" +#include "core/framework/op_kernel.h" +#include "core/framework/session_state.h" +#include "core/framework/utils.h" + +using namespace ::onnxruntime::common; +namespace onnxruntime { + +ExecutionFrame::ExecutionFrame(const std::unordered_map& feeds, + const std::vector& output_names, + const std::vector& fetches, + const ::onnxruntime::SessionState& session_state) + : session_state_(session_state), mem_patterns_(nullptr), planner_(nullptr) { + auto* graph = session_state.GetGraphViewer(); + ONNXRUNTIME_ENFORCE(graph); + Init(*graph, feeds, output_names, fetches); + + // If the session enable memory pattern optimization + // and we have execution plan generated, try to setup + // memory pattern optimization. + if (session_state.GetEnableMemoryPattern() && + session_state.GetExecutionPlan()) { + std::vector input_shapes; + bool all_tensors = true; + for (const auto& feed : feeds) { + if (!(feed.second.IsTensor())) { + all_tensors = false; + break; + } + auto& tensor = feed.second.Get(); + input_shapes.push_back(tensor.Shape()); + } + // if there is some traditional ml value type in inputs + // disable the memory pattern optimization. + if (all_tensors) { + mem_patterns_ = session_state.GetMemoryPatternGroup(input_shapes); + // if no existing patterns, generate one in this executionframe + if (!mem_patterns_) { + planner_ = std::make_unique(*session_state.GetExecutionPlan()); + } else { + // pre-allocate the big chunk requested in memory pattern. + // all the internal kernel's input/output tensors will be allocated on these buffer. + for (size_t i = 0; i < mem_patterns_->locations.size(); i++) { + ONNXRUNTIME_ENFORCE(buffers_.find(mem_patterns_->locations[i]) == buffers_.end()); + AllocatorPtr alloc = GetAllocator(mem_patterns_->locations[i]); + void* buffer = mem_patterns_->patterns[i].PeakSize() > 0 ? alloc->Alloc(mem_patterns_->patterns[i].PeakSize()) : nullptr; + buffers_[mem_patterns_->locations[i]] = BufferUniquePtr(buffer, alloc); + } + } + } + } +} + +ExecutionFrame::~ExecutionFrame() = default; + +Status ExecutionFrame::AllocateMLValueTensorSelfOwnBuffer(int mlvalue_index, + const MLDataType element_type, + const ONNXRuntimeAllocatorInfo& location, + const TensorShape& shape, + bool create_fence) { + ONNXRUNTIME_ENFORCE(mlvalue_index >= 0 && static_cast(mlvalue_index) < all_values_.size()); + return AllocateMLValueTensorSelfOwnBufferHelper(mlvalue_index, element_type, location, shape, create_fence); +} + +Status ExecutionFrame::AllocateMLValueTensorSelfOwnBufferHelper(int mlvalue_index, + const MLDataType element_type, + const ONNXRuntimeAllocatorInfo& location, + const TensorShape& shape, + bool create_fence) { + if (mlvalue_index < 0) + return Status(ONNXRUNTIME, FAIL, "Trying to allocate memory for unused optional inputs/outputs"); + + auto p_mlvalue = &all_values_[mlvalue_index]; + if (p_mlvalue->IsAllocated()) { + return Status::OK(); + } + auto alloc = GetAllocator(location); + size_t size; + { + int64_t len = shape.Size(); + if (len < 0) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Tensor shape cannot contain any negative value"); + } + if (!IAllocator::CalcMemSizeForArray(len, element_type->Size(), &size)) { + return Status(ONNXRUNTIME, FAIL, "size overflow"); + } + } + // create fence if needed + if (create_fence) { + ONNXRUNTIME_ENFORCE(p_mlvalue->Fence() == nullptr); + FencePtr f = alloc->CreateFence(&SessionState()); + // it is OK to have fence been nullptr if the execution provider has no async execution, + // and allocator::CreateFence returns nullptr + p_mlvalue->SetFence(f); + } + + // if we have pre-calculated memory pattern, and the mlvalue is not output mlvalue + // try to allocated on pre-allocated big chunk. + const auto& per_alloc_plan = GetAllocationPlan(mlvalue_index); + if (mem_patterns_ && per_alloc_plan.alloc_kind != AllocKind::kAllocateOutput) { + auto pattern = mem_patterns_->GetPatterns(location); + if (pattern) { + auto block = pattern->GetBlock(mlvalue_index); + // if block not found, fall back to default behavior + if (block) { + auto it = buffers_.find(location); + // if the block is not correct, log message then fall back to default behavior + if (it != buffers_.end() && block->size_ == size) { + void* buffer = it->second.get(); + auto status = AllocateTensorWithPreAllocateBufferHelper( + p_mlvalue, static_cast(static_cast(buffer) + block->offset_), + element_type, location, shape); + return status; + } + if (block->size_ != size) { + LOGS_DEFAULT(WARNING) << "For mlvalue with index: " << mlvalue_index << ", block in memory pattern size is: " + << block->size_ << " but the actually size is: " << size << ", fall back to default allocation behavior"; + } else if (it == buffers_.end()) { + LOGS_DEFAULT(WARNING) << "For mlvalue with index: " << mlvalue_index << ", block not found in target loation. " + " fall back to default allocation behavior"; + } + } + } + } + //no memory pattern, or the pattern is not correct. + void* buffer = size == 0 ? nullptr : alloc->Alloc(size); + std::unique_ptr p_tensor = std::make_unique(element_type, + shape, + buffer, + location, + alloc); + + p_mlvalue->Init(p_tensor.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); + + // trace the memory allocation. + // don't trace the memory allocation on string tensors, as it need + // placement new, we don't support it in memory pattern optimization. + if (element_type != DataTypeImpl::GetType()) + TraceAllocate(mlvalue_index, size); + + return Status::OK(); +} + +void ExecutionFrame::TraceAllocate(int mlvalue_idx, size_t size) { + // don't trace the output tensors. + auto& allocation_plan = GetAllocationPlan(mlvalue_idx); + if (planner_ && allocation_plan.alloc_kind != AllocKind::kAllocateOutput) { + auto status = planner_->TraceAllocation(mlvalue_idx, size); + if (!status.IsOK()) + LOGS(session_state_.Logger(), WARNING) << "TraceAllocation for mlvalue_idx=" << mlvalue_idx << " size=" << size + << " failed: " << status.ErrorMessage(); + } +} + +Status ExecutionFrame::AllocateTensorWithSelfOwnBuffer(const int index, + const MLDataType element_type, + const ONNXRuntimeAllocatorInfo& location, + const TensorShape& shape, + bool create_fence) { + ONNXRUNTIME_ENFORCE(index >= 0 && static_cast(index) < node_values_.size()); + return AllocateMLValueTensorSelfOwnBufferHelper(node_values_[index], element_type, location, shape, create_fence); +} + +Status ExecutionFrame::AllocateMLValueTensorPreAllocateBuffer(int mlvalue_index_to_allocate, + int mlvalue_index_reuse, + const MLDataType element_type, + const ONNXRuntimeAllocatorInfo& location, + const TensorShape& shape, + bool create_fence) { + ONNXRUNTIME_ENFORCE(mlvalue_index_to_allocate >= 0 && mlvalue_index_to_allocate < all_values_.size()); + MLValue* p_mlvalue = &all_values_[mlvalue_index_to_allocate]; + + ONNXRUNTIME_ENFORCE(mlvalue_index_reuse >= 0 && mlvalue_index_reuse < all_values_.size()); + MLValue* p_mlvalue_reuse = &all_values_[mlvalue_index_reuse]; + + auto* reuse_tensor = p_mlvalue_reuse->GetMutable(); + void* reuse_buffer = reuse_tensor->MutableDataRaw(); + + // create fence on reused mlvalue if needed + // TODO: differentiate reuse and alias, by add AllocKind::kAlias? + if (create_fence && p_mlvalue_reuse->Fence() == nullptr) { + FencePtr f = GetAllocator(location)->CreateFence(&SessionState()); + p_mlvalue_reuse->SetFence(f); + } + + // reused MLValue share the same fence + p_mlvalue->ShareFenceWith(*p_mlvalue_reuse); + return AllocateTensorWithPreAllocateBufferHelper(p_mlvalue, reuse_buffer, element_type, location, shape); +} + +Status ExecutionFrame::AllocateTensorWithPreAllocateBufferHelper(MLValue* p_mlvalue, + void* pBuffer, + const MLDataType element_type, + const ONNXRuntimeAllocatorInfo& location, + const TensorShape& shape) { + if (p_mlvalue->IsAllocated()) { + return Status::OK(); + } + std::unique_ptr p_tensor = std::make_unique(element_type, + shape, + pBuffer, + location); + p_mlvalue->Init(p_tensor.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); + + return Status::OK(); +} + +Status ExecutionFrame::AllocateTensorWithPreAllocateBuffer(const int offset, + void* pBuffer, + const MLDataType element_type, + const ONNXRuntimeAllocatorInfo& location, + const TensorShape& shape) { + ONNXRUNTIME_ENFORCE(offset >= 0 && offset < node_values_.size()); + if (node_values_[offset] < 0) + return Status(ONNXRUNTIME, FAIL, "Trying to allocate memory for unused optional inputs/outputs"); + auto value = &all_values_[node_values_[offset]]; + return AllocateTensorWithPreAllocateBufferHelper(value, pBuffer, element_type, location, shape); +} + +void ExecutionFrame::Release(const int offset) { + ONNXRUNTIME_ENFORCE(offset >= 0 && offset < node_offsets_.size()); + if (node_values_[offset] >= 0 && node_values_[offset] < all_values_.size()) { + all_values_[node_values_[offset]] = MLValue(); + TraceFree(node_values_[offset]); + } +} + +Status AllocateTraditionalMLValue(MLValue* p_mlvalue, + const NonTensorTypeBase* type, + const MLValueAllocationParameters& parameters) { + // right now we don't need any parameter for ml value creation, + // keep it in api for extensibility + ONNXRUNTIME_UNUSED_PARAMETER(parameters); + auto creator = type->GetCreateFunc(); + p_mlvalue->Init(creator(), + type, + type->GetDeleteFunc()); + return Status::OK(); +} + +// This method is not thread safe! +Status ExecutionFrame::AllocateAsPerAllocationPlan(int mlvalue_index, + const MLValueAllocationParameters& parameters) { + if (mlvalue_index < 0 || mlvalue_index >= all_values_.size()) + return Status(ONNXRUNTIME, INVALID_ARGUMENT, + "Tried to allocated with invalid mlvalue index: " + std::to_string(mlvalue_index)); + const SequentialExecutionPlan* p_seq_exec_plan = session_state_.GetExecutionPlan(); + const auto& alloc_plan = p_seq_exec_plan->allocation_plan; + ONNXRUNTIME_ENFORCE(mlvalue_index >= 0 && mlvalue_index < alloc_plan.size()); + const auto& per_alloc_plan = alloc_plan[mlvalue_index]; + + auto alloc_info = per_alloc_plan.location; + auto ml_type = per_alloc_plan.value_type; + if (ml_type == nullptr) + return Status(ONNXRUNTIME, INVALID_ARGUMENT, + "Tried to allocate without valid type information, mlvalue index=" + std::to_string(mlvalue_index)); + if (!ml_type->IsTensorType()) { + return AllocateTraditionalMLValue(&all_values_[mlvalue_index], + static_cast(ml_type), + parameters); + } + + // tensors + auto ml_data_type = static_cast(ml_type)->GetElementType(); + + AllocKind alloc_kind = per_alloc_plan.alloc_kind; + switch (alloc_kind) { + // Right now for kAllocate and kAllocateOutput we are using same approach. + // In the future we may want to have different way to handle it. + case AllocKind::kAllocateOutput: + case AllocKind::kAllocate: { + ONNXRUNTIME_RETURN_IF_ERROR(AllocateMLValueTensorSelfOwnBuffer(mlvalue_index, + ml_data_type, + alloc_info, + parameters.tensor_shape, + per_alloc_plan.create_fence_if_async)); + break; + } + case AllocKind::kReuse: { + int reuse_mlvalue_index = per_alloc_plan.reused_buffer; + ONNXRUNTIME_RETURN_IF_ERROR(AllocateMLValueTensorPreAllocateBuffer(mlvalue_index, + reuse_mlvalue_index, + ml_data_type, + alloc_info, + parameters.tensor_shape, + per_alloc_plan.create_fence_if_async)); + break; + } + default: { + std::ostringstream ostr; + ostr << "Invalid allocation kind: " << static_cast::type>(alloc_kind); + return Status(ONNXRUNTIME, FAIL, ostr.str()); + } + } + + return Status::OK(); +} + +void ExecutionFrame::Init(const onnxruntime::GraphViewer& graph, + const std::unordered_map& feeds, + const std::vector& output_names, + const std::vector& fetches) { + // 1. resize the node_offsets and all_value_ vector + // We need to use the max index rather than number of nodes as we use Node.Index() + // when inserting into node_offsets_ + auto max_node_index = graph.MaxNodeIndex(); + node_offsets_.resize(max_node_index); + + auto& mlvalue_idx_map = session_state_.GetMLValueNameIdxMap(); + + all_values_.resize(mlvalue_idx_map.MaxIdx() + 1); + + // 2. handle the weights. + for (const auto& entry : session_state_.GetInitializedTensors()) { + auto mlvalue_index = entry.first; + all_values_[mlvalue_index] = entry.second; // this copy should be cheap + } + + // 3. handle feed in values + for (const auto& feed : feeds) { + int mlvalue_idx; + Status status = mlvalue_idx_map.GetIdx(feed.first, mlvalue_idx); + ONNXRUNTIME_ENFORCE(status.IsOK(), status.ErrorMessage()); + // we are sharing the underline tensor/object for MLValue + all_values_[mlvalue_idx] = feed.second; + } + + // 4. Handle non-empty output vector + // setup output_indices_, we dont' want to generate mem plan on output tensors. + for (const auto& oname : output_names) { + int mlvalue_idx; + Status status = mlvalue_idx_map.GetIdx(oname, mlvalue_idx); + ONNXRUNTIME_ENFORCE(status.IsOK(), status.ErrorMessage()); + output_indices_.push_back(mlvalue_idx); + } + + if (!fetches.empty()) { + // should've already verified this much before when Run() starts + ONNXRUNTIME_ENFORCE(output_names.size() == fetches.size(), + "output_names vector size: " + std::to_string(output_names.size()) + + " does not match that of fetches vector: " + std::to_string(fetches.size())); + + auto idx = 0; + for (const auto& oname : output_names) { + int mlvalue_idx; + Status status = mlvalue_idx_map.GetIdx(oname, mlvalue_idx); + ONNXRUNTIME_ENFORCE(status.IsOK(), status.ErrorMessage()); + all_values_[mlvalue_idx] = fetches.at(idx++); + output_indices_.push_back(mlvalue_idx); + } + } + + // 5. set node args + for (auto& node : graph.Nodes()) { + ONNXRUNTIME_ENFORCE(node.Index() < node_offsets_.size()); + node_offsets_[node.Index()] = static_cast(node_values_.size()); + + for (auto input_def : node.InputDefs()) { + SetupNodeArg(input_def); + } + + for (auto input_def : node.ImplicitInputDefs()) { + SetupNodeArg(input_def); + } + + for (auto output_def : node.OutputDefs()) { + SetupNodeArg(output_def); + } + } +} + +void ExecutionFrame::SetupNodeArg(const onnxruntime::NodeArg* arg) { + ONNXRUNTIME_ENFORCE(arg); + auto& name = arg->Name(); + //if the arg's name is empty, it is an not needed optional input/output + //set index to -1 + if (name.empty()) { + node_values_.push_back(-1); + } else { + int index; + Status status = session_state_.GetMLValueNameIdxMap().GetIdx(name, index); + ONNXRUNTIME_ENFORCE(status.IsOK(), status.ErrorMessage()); + node_values_.push_back(index); + } +} + +void ExecutionFrame::TraceFree(int mlvalue_idx) { + // don't trace free on output tensors. + if (planner_ && + std::find(output_indices_.begin(), output_indices_.end(), mlvalue_idx) == output_indices_.end()) { + const SequentialExecutionPlan* p_seq_exec_plan = session_state_.GetExecutionPlan(); + const auto& alloc_plan = p_seq_exec_plan->allocation_plan; + const auto& per_alloc_plan = alloc_plan.at(mlvalue_idx); + + // only trace tensors + auto ml_type = per_alloc_plan.value_type; + if (ml_type->IsTensorType()) { + // tensors + auto ml_data_type = static_cast(ml_type)->GetElementType(); + // don't trace string tensors + if (ml_data_type != DataTypeImpl::GetType()) { + auto status = planner_->TraceFree(mlvalue_idx); + if (!status.IsOK()) { + LOGS(session_state_.Logger(), WARNING) << "TraceFree for mlvalue_idx=" << mlvalue_idx + << " failed: " << status.ErrorMessage(); + } + } + } + } +} + +// generate memory pattern based on the tracing of memory allocation/free in current execution +// return error if the planner is not setup. +Status ExecutionFrame::GeneratePatterns(MemoryPatternGroup* out) const { + if (!planner_) { + return Status(ONNXRUNTIME, FAIL, "Memory pattern planner is not enabled on this execution framework."); + } + + return planner_->GeneratePatterns(out); +} + +// Return nullptr if index map to an value that is an unused optional input/output +const MLValue* ExecutionFrame::GetNodeInputOrOutputMLValue(int index) const { + ONNXRUNTIME_ENFORCE(index >= 0 && static_cast(index) < node_values_.size()); + return node_values_[index] >= 0 ? &all_values_[node_values_[index]] : nullptr; +} + +// Return nullptr if index map to an value that is an unused optional input/output +MLValue* ExecutionFrame::GetMutableNodeInputOrOutputMLValue(int index) { + return const_cast(GetNodeInputOrOutputMLValue(index)); +} + +AllocatorPtr ExecutionFrame::GetAllocator(const ONNXRuntimeAllocatorInfo& info) { + return utils::GetAllocator(session_state_, info); +} + +static inline void VerifyShape(const MLValue* p_mlvalue, + const MLValueAllocationParameters& parameters) { + if (p_mlvalue->IsTensor()) { + const Tensor* tensor = &p_mlvalue->Get(); + + ONNXRUNTIME_ENFORCE(tensor->Shape() == parameters.tensor_shape, + "MLValue shape verification failed. Current shape:", tensor->Shape(), + " Requested shape:", parameters.tensor_shape); + } +} + +// This method is not thread safe! +// Return S_OK and nullptr if index map to an value that is an unused optional input/output +Status ExecutionFrame::GetOrCreateNodeOutputMLValue(int index, + const MLValueAllocationParameters& parameters, + MLValue*& p_mlvalue) { + if (index < 0 || static_cast(index) >= node_values_.size()) { + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, + "Try to access with invalid node value index: " + std::to_string(index)); + } + + // return nullptr if it is optional + if (node_values_[index] < 0) { + p_mlvalue = nullptr; + return Status::OK(); + } + + p_mlvalue = &all_values_.at(node_values_[index]); + + if (p_mlvalue->IsAllocated()) { + // The ml has already been allocated. + // Now only tensor need to be check. + VerifyShape(p_mlvalue, parameters); // TODO find a better way to do this + return Status::OK(); + } + // It's not allocated, then allocate it with given shape and return. + // Perform allocation based on the allocation plan + ONNXRUNTIME_RETURN_IF_ERROR(AllocateAsPerAllocationPlan(node_values_[index], parameters)); + return Status::OK(); +} + +Status ExecutionFrame::ReleaseMLValue(int mlvalue_idx) { + if (mlvalue_idx < 0 || static_cast(mlvalue_idx) >= all_values_.size()) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "invalid index ", mlvalue_idx); + } + all_values_[mlvalue_idx] = MLValue(); + TraceFree(mlvalue_idx); + return Status::OK(); +} + +const SequentialExecutionPlan::AllocPlanPerValue& ExecutionFrame::GetAllocationPlan(int mlvalue_idx) { + const SequentialExecutionPlan* p_seq_exec_plan = session_state_.GetExecutionPlan(); + const auto& alloc_plan = p_seq_exec_plan->allocation_plan; + ONNXRUNTIME_ENFORCE(mlvalue_idx >= 0 && mlvalue_idx < alloc_plan.size()); + return alloc_plan[mlvalue_idx]; +} +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/execution_frame.h b/onnxruntime/core/framework/execution_frame.h new file mode 100644 index 0000000000000..3f946a26ad74c --- /dev/null +++ b/onnxruntime/core/framework/execution_frame.h @@ -0,0 +1,183 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include "core/common/common.h" +#include "core/common/logging/logging.h" +#include "core/common/status.h" +#include "core/framework/ml_value.h" +#include "core/framework/sequential_execution_plan.h" +#include "core/framework/tensor.h" +#include "core/graph/graph.h" + +namespace onnxruntime { + +class SessionState; +class MLValuePatternPlanner; +struct MemoryPatternGroup; + +struct MLValueAllocationParameters { + TensorShape tensor_shape; + // todo: is there any parameter needed for ml types? +}; + +class ExecutionFrame { + public: + ExecutionFrame(const std::unordered_map& feeds, + const std::vector& output_names, + const std::vector& fetches, + const SessionState& session_state); + + ~ExecutionFrame(); + + Status AllocateMLValueTensorSelfOwnBuffer(int mlvalue_index, + MLDataType element_type, + const ONNXRuntimeAllocatorInfo& location, + const TensorShape& shape, + bool create_fence = false); + + Status AllocateMLValueTensorPreAllocateBuffer(int mlvalue_index_to_allocate, + int mlvalue_index_reuse, + MLDataType element_type, + const ONNXRuntimeAllocatorInfo& location, + const TensorShape& shape, + bool create_fence = false); + + // ?? Cheng: What about non-tensor values?? + // ?? Cheng: There are cases we may not want to use ONNXRUNTIME_ENFORCE?? + // ?? Cheng: Graph must be immutable for GetNodesInTopologicalOrder?? + // Create tensor at index mlvalue, and allocate buffer for it. + // This tensor will own this buffer. + // This method is not thread safe! + Status AllocateTensorWithSelfOwnBuffer(int index, + MLDataType element_type, + const ONNXRuntimeAllocatorInfo& location, + const TensorShape& shape, + bool create_fence = false); + + // Create tensor at index mlvalue, with pre-allocate buffer + // This tensor does not own the buffer. + // The executor / planner need to be careful about the + // lifetime of the buffer. Tensor itself won't manage it. + // This method is not thread safe! + Status AllocateTensorWithPreAllocateBuffer(int offset, + void* pBuffer, + MLDataType element_type, + const ONNXRuntimeAllocatorInfo& location, + const TensorShape& shape); + + const MLValue& GetMLValue(int mlvalue_index) const { + ONNXRUNTIME_ENFORCE(mlvalue_index >= 0 && static_cast(mlvalue_index) < all_values_.size()); + return all_values_[mlvalue_index]; + } + + MLValue& GetMutableMLValue(int mlvalue_index) { + ONNXRUNTIME_ENFORCE(mlvalue_index >= 0 && static_cast(mlvalue_index) < all_values_.size()); + return all_values_[mlvalue_index]; + } + + // Index to the first argument of the given node. + int GetFirstArgIndex(onnxruntime::NodeIndex index) const { + ONNXRUNTIME_ENFORCE(index < node_offsets_.size()); + return node_offsets_[index]; + } + + // Return nullptr if index map to an value that is an unused optional input/output + const MLValue* GetNodeInputOrOutputMLValue(int index) const; + MLValue* GetMutableNodeInputOrOutputMLValue(int index); + + // TO DO: make it thread safe + // This method is not thread safe! + // Return S_OK and nullptr if index map to an value that is an unused optional input/output + Status GetOrCreateNodeOutputMLValue(int index, + const MLValueAllocationParameters& parameters, + MLValue*& p_mlvalue); + + AllocatorPtr GetAllocator(const ONNXRuntimeAllocatorInfo& info); + + Status ReleaseMLValue(int mlvalue_idx); + + const ::onnxruntime::SessionState& SessionState() const { + return session_state_; + } + + Status GeneratePatterns(MemoryPatternGroup* out) const; + + bool HasPlan() const { + return planner_ != nullptr; + } + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(ExecutionFrame); + + // This method is not thread safe! + void Release(int offset); + + common::Status AllocateAsPerAllocationPlan(int mlvalue_index, + const MLValueAllocationParameters& parameters); + + Status AllocateMLValueTensorSelfOwnBufferHelper(int mlvalue_index, + MLDataType element_type, + const ONNXRuntimeAllocatorInfo& location, + const TensorShape& shape, + bool create_fence); + + void Init(const onnxruntime::GraphViewer& graph, + const std::unordered_map& feeds, + const std::vector& output_names, + const std::vector& fetches); + + void SetupNodeArg(const onnxruntime::NodeArg* arg); + + Status AllocateTensorWithPreAllocateBufferHelper(MLValue* p_mlvalue, + void* pBuffer, + MLDataType element_type, + const ONNXRuntimeAllocatorInfo& location, + const TensorShape& shape); + + void TraceAllocate(int mlvalue_idx, size_t size); + + void TraceFree(int mlvalue_idx); + + const SequentialExecutionPlan::AllocPlanPerValue& GetAllocationPlan(int mlvalue_idx); + + Status status_; + + // The values for the inputs and outputs of the nodes. + // This vector contains the indices into the all_values_ vector. + std::vector node_values_; + + // All the intermediate values for the entire graph. + // Input and Output values are passed in by executors + std::vector all_values_; + + // The start index into node_values_ for all the nodes. + std::vector node_offsets_; + + // i-th kernel is still waiting for pending_counts_[i] inputs. + std::vector pending_counts_; // not used currently + + std::unordered_map value_name_to_index_; + + const ::onnxruntime::SessionState& session_state_; + + // If we already have cached memory pattern on these input shapes + // Use this mem pattern that create a big chunk for all the internal + // kernel's input/output tensors. + const MemoryPatternGroup* mem_patterns_; + + // If no cached memory pattern, and we enable the memory pattern optimization + // use this planner_ to trace the memory allocation in current executor. + std::unique_ptr planner_; + + // Record the ml value indices for output values. we won't include those + // values' allocation in memory pattern, as they can't be shared. + std::vector output_indices_; + + // Big chunks on different locations that will be used by mem_pattern. + std::map buffers_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/execution_provider.cc b/onnxruntime/core/framework/execution_provider.cc new file mode 100644 index 0000000000000..09a302d338cf7 --- /dev/null +++ b/onnxruntime/core/framework/execution_provider.cc @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "core/framework/execution_provider.h" + +#include "core/graph/graph.h" +#include "core/framework/computation_capacity.h" +#include "core/framework/kernel_registry_manager.h" +#include "core/framework/op_kernel.h" +#include "core/framework/kernel_registry.h" + +namespace onnxruntime { + +namespace { +inline int MakeKey(int id, ONNXRuntimeMemType mem_type) { + return id << 2 | mem_type; +} +} // namespace + +AllocatorPtr IExecutionProvider::GetAllocator(int id, ONNXRuntimeMemType mem_type) const { + auto iter = allocators_.find(MakeKey(id, mem_type)); + if (iter != allocators_.end()) { + return iter->second; + } + return nullptr; +} + +std::vector> +IExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph, + const std::vector& kernel_registries) const { + std::vector> result; + for (auto& node : graph.Nodes()) { + for (auto registry : kernel_registries) { + if (registry->TryFindKernel(node, Type()) != nullptr) { + std::unique_ptr sub_graph = std::make_unique(); + sub_graph->nodes.push_back(node.Index()); + result.push_back(std::make_unique(std::move(sub_graph), nullptr)); + } + } + } + + return result; +} + +common::Status IExecutionProvider::CopyTensor(const Tensor& src, + Tensor& dst, + int exec_queue_id) const { + // execution provider may override this to support different exec queues + ONNXRUNTIME_ENFORCE(exec_queue_id == 0); + return CopyTensor(src, dst); +} + +common::Status IExecutionProvider::Sync() const { return Status::OK(); }; + +common::Status IExecutionProvider::OnRunStart() { return Status::OK(); } + +common::Status IExecutionProvider::OnRunEnd() { return Status::OK(); } + +void IExecutionProvider::InsertAllocator(AllocatorPtr allocator) { + const ONNXRuntimeAllocatorInfo& info = allocator->Info(); + const int key = MakeKey(info.id, info.mem_type); + auto iter = allocators_.find(key); + if (iter != allocators_.end()) { + ONNXRUNTIME_THROW("duplicated allocator"); + } + allocators_.insert(iter, {key, allocator}); +} +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/execution_providers.h b/onnxruntime/core/framework/execution_providers.h new file mode 100644 index 0000000000000..298e5373b3e9b --- /dev/null +++ b/onnxruntime/core/framework/execution_providers.h @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +// #include +#include +#include +#include +#include + +#include "core/framework/execution_provider.h" +#include "core/graph/graph.h" +#include "core/common/logging/logging.h" + +namespace onnxruntime { + +/** +Class for managing lookup of the execution providers in a session. +*/ +class ExecutionProviders { + public: + ExecutionProviders() = default; + + common::Status Add(const std::string& provider_id, std::unique_ptr p_exec_provider) { + // make sure there are no issues before we change any internal data structures + if (provider_idx_map_.find(provider_id) != provider_idx_map_.end()) { + auto status = ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Provider ", provider_id, " has already been registered."); + LOGS_DEFAULT(ERROR) << status.ErrorMessage(); + return status; + } + + for (const auto& allocator : p_exec_provider->GetAllocatorMap()) { + if (allocator_idx_map_.find(allocator->Info()) != allocator_idx_map_.end()) { + auto status = ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, allocator->Info(), " allocator already registered."); + LOGS_DEFAULT(ERROR) << status.ErrorMessage(); + return status; + } + } + + // index that provider will have after insertion + auto new_provider_idx = exec_providers_.size(); + + ONNXRUNTIME_IGNORE_RETURN_VALUE(provider_idx_map_.insert({provider_id, new_provider_idx})); + + for (const auto& allocator : p_exec_provider->GetAllocatorMap()) { + ONNXRUNTIME_IGNORE_RETURN_VALUE(allocator_idx_map_.insert({allocator->Info(), new_provider_idx})); + } + + exec_providers_.push_back(std::move(p_exec_provider)); + + return Status::OK(); + } + + const IExecutionProvider* Get(const onnxruntime::Node& node) const { + return Get(node.GetExecutionProviderType()); + } + + const IExecutionProvider* Get(onnxruntime::ProviderType provider_id) const { + auto it = provider_idx_map_.find(provider_id); + if (it == provider_idx_map_.end()) { + return nullptr; + } + + return exec_providers_[it->second].get(); + } + + const IExecutionProvider* Get(const ONNXRuntimeAllocatorInfo& allocator_info) const { + auto it = allocator_idx_map_.find(allocator_info); + if (it == allocator_idx_map_.end()) { + return nullptr; + } + + return exec_providers_[it->second].get(); + } + + bool Empty() const { return exec_providers_.empty(); } + + using const_iterator = typename std::vector>::const_iterator; + const_iterator begin() const noexcept { return exec_providers_.cbegin(); } + const_iterator end() const noexcept { return exec_providers_.cend(); } + + private: + std::vector> exec_providers_; + + // maps for fast lookup of an index into exec_providers_ + std::unordered_map provider_idx_map_; + // using std::map as ONNXRuntimeAllocatorInfo would need a custom hash function to be used with unordered_map, + // and as this isn't performance critical it's not worth the maintenance overhead of adding one. + std::map allocator_idx_map_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/graph_partitioner.cc b/onnxruntime/core/framework/graph_partitioner.cc new file mode 100644 index 0000000000000..66365a094115b --- /dev/null +++ b/onnxruntime/core/framework/graph_partitioner.cc @@ -0,0 +1,146 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/graph_partitioner.h" + +#include "core/framework/kernel_registry_manager.h" +#include "core/graph/function.h" +#include "core/graph/graph.h" +#include "core/framework/computation_capacity.h" +#include "core/framework/kernel_registry_manager.h" +#include "core/framework/execution_providers.h" +#include "core/framework/kernel_registry.h" + +// uncomment this line to count non-CUDA ops in ONNX domain +//#define COUNT_NON_CUDA_OPS + +#ifdef COUNT_NON_CUDA_OPS +class NonCudaOps { + public: + ~NonCudaOps() { + printf("Non-CUDA ops:\n"); + for (auto i : map_) { + printf("%s: %d\n", i.first.c_str(), i.second); + } + } + + void AddOp(const std::string& name) { + if (map_.count(name)) + map_.at(name)++; + else + map_.insert({name, 1}); + } + + private: + std::map map_; +}; + +NonCudaOps non_cuda; +#endif + +using namespace ::onnxruntime::common; +namespace onnxruntime { + +KernelDefBuilder& BuildFusedKernelDef(KernelDefBuilder& builder, const onnxruntime::Node& node) { + auto schema = node.Op(); + builder.SetName(schema->Name()) + .SetDomain(schema->domain()) + .SinceVersion(schema->SinceVersion()) + .Provider(node.GetExecutionProviderType()); + auto& inputs = node.InputDefs(); + for (auto input : inputs) { + builder.TypeConstraint(input->Name(), DataTypeImpl::TypeFromProto(*input->TypeAsProto())); + } + return builder; +} + +Status GraphPartitioner::Partition(onnxruntime::Graph& graph) const { + if (providers_.Empty()) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "No provider specified."); + } + //fused_kernel_registry is prepareing the kernels created on the fly for fused sub graph. + //It is only visiable for current session. + std::shared_ptr fused_kernel_registry = std::make_shared(); + // Partitioning based on provider preference and their capabilities. + auto kernel_registries = kernel_registry_mgr_.GetAllKernelRegistries(); + for (auto& provider : providers_) { + auto capability_results = provider->GetCapability(GraphViewer(graph), kernel_registries); + int count = 0; + for (auto& capacity : capability_results) { + if (nullptr == capacity || nullptr == capacity->sub_graph_) { + continue; + } + if (nullptr == capacity->sub_graph_->GetMetaDef()) { + // The can run a single node in the if not using meta-defs. + // A fused kernel is not supported in this case. + ONNXRUNTIME_ENFORCE(1 == capacity->sub_graph_->nodes.size()); + ONNXRUNTIME_ENFORCE(capacity->fuse_kernel_function_ == nullptr); + + auto node = graph.GetNode(capacity->sub_graph_->nodes[0]); + if (nullptr != node && node->GetExecutionProviderType().empty()) { + node->SetExecutionProviderType(provider->Type()); + } + } else { + // The can run a fused in the . + // + // Add fused node into + ONNXRUNTIME_ENFORCE(nullptr != capacity->sub_graph_->GetMetaDef()); + std::string node_name = provider->Type() + "_" + capacity->sub_graph_->GetMetaDef()->name + "_" + std::to_string(count++); + auto fused_node = graph.FuseSubGraph(std::move(capacity->sub_graph_), node_name); + fused_node->SetExecutionProviderType(provider->Type()); + auto fused_kernel_func = capacity->fuse_kernel_function_; + if (fused_kernel_func != nullptr) { + // build the kernel definition on the fly, and register it to the fused_kernel_regisitry. + KernelDefBuilder builder; + BuildFusedKernelDef(builder, *fused_node); + fused_kernel_registry->Register(builder, fused_kernel_func); + } + } + } + // all done with this provider, resolve the graph before we move on to the next provider. + // This is needed since we create a new GraphViewer() that we pass into the next provider's GetCapability(). + ONNXRUNTIME_ENFORCE(graph.Resolve().IsOK()); + } + + // To see if the node with no provider can be inlined. If one such nodes can be + // successfully inlined, we re-run the partitioner on the modified graph. + bool inline_flag = false; + for (auto& node : graph.Nodes()) { + if (node.GetExecutionProviderType().empty()) { + auto node_func = node.GetFunctionBody(); + if (nullptr == node_func) { + continue; + } + Status inliner_status = graph.InlineFunction(node); + // If the node has a functionbody with no kernel and cannot be inlined + // it is a invalid function + if(!inliner_status.IsOK()) return inliner_status; + // Set the flag for re-run graph partition after successful inlining + inline_flag = true; + break; + } + } + // Resolve and rerun graph partition + if (inline_flag) { + ONNXRUNTIME_RETURN_IF_ERROR(graph.Resolve()); + this->Partition(graph); + } + + //For some cases, like fp16 on cpu, right now we don't have any kernel support that. + //But we will insert cast op to run the model, so skip the error checking here. + //If after graph transform phase, the node still not assigned, we will report error + //during kernel creation phase. +#ifdef COUNT_NON_CUDA_OPS + for (auto& node : graph.Nodes()) { + if (node.GetExecutionProviderType() != kCudaExecutionProvider && + node.Domain() != kMLDomain && + node.Domain() != kMSDomain) + non_cuda.AddOp(node.OpType()); + } +#endif + + kernel_registry_mgr_.RegisterKernelRegistry(fused_kernel_registry, KernelRegistryPriority::HighPriority); + + return Status::OK(); +} +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/graph_partitioner.h b/onnxruntime/core/framework/graph_partitioner.h new file mode 100644 index 0000000000000..a22589f1a7a4d --- /dev/null +++ b/onnxruntime/core/framework/graph_partitioner.h @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/graph/graph.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { + +class ExecutionProviders; +class KernelRegistryManager; + +class GraphPartitioner { + public: + //The order of providers represents the user preference. + GraphPartitioner(KernelRegistryManager& kernel_registry_mgr, + const ExecutionProviders& providers) + : kernel_registry_mgr_(kernel_registry_mgr), + providers_(providers) {} + + Status Partition(onnxruntime::Graph& graph) const; + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(GraphPartitioner); + + KernelRegistryManager& kernel_registry_mgr_; + const ExecutionProviders& providers_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/iexecutor.h b/onnxruntime/core/framework/iexecutor.h new file mode 100644 index 0000000000000..f5294c657dae6 --- /dev/null +++ b/onnxruntime/core/framework/iexecutor.h @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include + +#include "core/common/status.h" +#include "core/framework/framework_common.h" +#include "core/framework/ml_value.h" + +namespace onnxruntime { + +class SessionState; +namespace logging { +class Logger; +} + +class IExecutor { + public: + virtual ~IExecutor() = default; + + virtual common::Status Execute(const SessionState& session_state, + const NameMLValMap& feeds, + const std::vector& output_names, + std::vector& fetches, + const logging::Logger& logger) = 0; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/insert_cast_transformer.cc b/onnxruntime/core/framework/insert_cast_transformer.cc new file mode 100644 index 0000000000000..f9582738c5f11 --- /dev/null +++ b/onnxruntime/core/framework/insert_cast_transformer.cc @@ -0,0 +1,208 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/insert_cast_transformer.h" +#include "core/framework/data_types.h" + +using namespace ONNX_NAMESPACE; +using namespace ::onnxruntime::common; +namespace onnxruntime { +class IdGenerator { + public: + int Next() { + return id++; + } + + private: + int id = 0; +}; + +bool InsertCastTransformer::NeedInsertCast(const onnxruntime::Node* node, const onnxruntime::NodeArg* input) const { + //If the node's input is float16 and currently the node is not assigned to any XP. + //we need insert a cast to float, and put the node on CPU for default behavior. + //TODO: a better check is to check does the CPU kernel with float exist or not. + return input->Type() != nullptr && + DataTypeImpl::TypeFromProto(*input->TypeAsProto()) == DataTypeImpl::GetTensorType() && + node->GetExecutionProviderType().empty(); +} + +onnxruntime::NodeArg* AddCastNode(onnxruntime::Graph& graph, + IdGenerator& id_generator, + onnxruntime::NodeArg* old_arg, + TypeProto* new_type, + bool new_on_input, + int64_t to_type, + onnxruntime::ProviderType providerType) { + //insert cast op to cast input + int id = id_generator.Next(); + + char str[32]; + snprintf(str, 32, "CastDef_%d", id); + + auto* new_arg = &graph.GetOrCreateNodeArg(str, new_type); + + std::vector input_defs = {new_on_input ? new_arg : old_arg}; + std::vector output_defs = {new_on_input ? old_arg : new_arg}; + + auto cast_node = graph.AddNode(str, "Cast", "cast node to cast from float16 to float32 on cpu", input_defs, output_defs); + cast_node->AddAttribute("to", to_type); + cast_node->SetExecutionProviderType(providerType); + return new_arg; +} + +static bool IsInputFloat16(const onnxruntime::Node* node) { + for (auto input : node->InputDefs()) { + if (input->Type() != nullptr && + DataTypeImpl::TypeFromProto(*input->TypeAsProto()) == DataTypeImpl::GetTensorType() && + !node->GetExecutionProviderType().empty()) { + return true; + } + } + return false; +} + +static bool IsSingleInputNodeFloat16Node(const onnxruntime::Node* node) { + if (IsInputFloat16(node) && node->GetExecutionProviderType() == kCpuExecutionProvider) { + for (auto it = node->InputNodesBegin(); it != node->InputNodesEnd(); ++it) { + if (IsInputFloat16(*it)) + return false; + } + for (auto it = node->OutputNodesBegin(); it != node->OutputNodesEnd(); ++it) { + if (IsInputFloat16(*it)) + return false; + } + return true; + } + return false; +} + +Status ForceSingleNodeCPUFloat16ToFloat32(onnxruntime::Graph& graph) { + // if graph only contain 1 compute node, don't force to float32 + if (graph.NumberOfNodes() <= 1) { + return Status::OK(); + } + for (auto& node : graph.Nodes()) { + if (IsSingleInputNodeFloat16Node(&node)) { + node.SetExecutionProviderType(""); + } + } + return graph.Resolve(); +} + +Status InsertCastTransformer::Apply(onnxruntime::Graph& graph, bool& modified) const { + ONNXRUNTIME_RETURN_IF_ERROR(graph.Resolve()); + if (force_cpu_fp32_) + ONNXRUNTIME_RETURN_IF_ERROR(ForceSingleNodeCPUFloat16ToFloat32(graph)); + + GraphViewer graph_viewer(graph); + auto& order = graph_viewer.GetNodesInTopologicalOrder(); + TypeProto float_16_tensor_proto, float_tensor_proto; + float_16_tensor_proto.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT16); + float_tensor_proto.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + IdGenerator id_generator; + std::map input_def_updates; + for (onnxruntime::NodeIndex i : order) { + auto node = graph.GetNode(i); + if (!node) + return Status(ONNXRUNTIME, INVALID_ARGUMENT); + + auto& inputs = node->InputDefs(); + std::map replacement_defs; + bool casted = false; + for (auto input : inputs) { + if (NeedInsertCast(node, input)) { + auto src_arg = const_cast(input); + if (input_def_updates.count(src_arg)) { + replacement_defs[src_arg] = input_def_updates[src_arg]; + } else { + //insert cast op to cast input + auto dst_arg = AddCastNode(graph, + id_generator, + src_arg, + &float_tensor_proto, + false, + static_cast(TensorProto_DataType_FLOAT), + //right now we only cast for cpu cases. + onnxruntime::kCpuExecutionProvider); + replacement_defs[src_arg] = dst_arg; + input_def_updates[src_arg] = dst_arg; + } + casted = true; + } + } + + if (casted && node->GetExecutionProviderType().empty()) { + //set current node to CPU execution provider + node->SetExecutionProviderType(kCpuExecutionProvider); + } + + auto& outputs = node->OutputDefs(); + for (auto output : outputs) { + // todo: check is the kernel available + // here is based on the assumption that if we cast a cpu op's input from float16 to float + // then this cpu op's output will become float. + // not sure is it always correct... + if (output->Type() && + DataTypeImpl::TypeFromProto(*output->TypeAsProto()) == DataTypeImpl::GetTensorType() && + casted) { + //insert cast op to cast output back to float16 + auto dst_arg = const_cast(output); + auto src_arg = AddCastNode(graph, + id_generator, + dst_arg, + &float_tensor_proto, + true, + static_cast(TensorProto_DataType_FLOAT16), + onnxruntime::kCpuExecutionProvider); + replacement_defs[dst_arg] = src_arg; + } + } + + node->ReplaceDefs(replacement_defs); + modified = modified || casted; + } + + //Resolve it to build the edges. + ONNXRUNTIME_RETURN_IF_ERROR(graph.Resolve()); + std::map replacement_defs; + std::vector removed_nodes; + for (auto& node : graph.Nodes()) { + if (node.OpType() == "Cast") { + // if cast's next node is also cast and next cast's output type equal to cast's input type + // remove those two cast. + auto src_type = node.InputDefs()[0]->Type(); + auto dst_type = node.OutputDefs()[0]->Type(); + auto input = node.InputDefs()[0]; + int child_removed = 0; + int num_child = 0; + for (auto it = node.OutputNodesBegin(); it != node.OutputNodesEnd(); ++it) { + if ((*it)->OpType() == "Cast") { + auto src_type1 = (*it)->InputDefs()[0]->Type(); + auto dst_type1 = (*it)->OutputDefs()[0]->Type(); + if (src_type == dst_type1 && src_type1 == dst_type) { + //node *it's output's follower could be linked with node's input. + replacement_defs.clear(); + replacement_defs[const_cast((*it)->OutputDefs()[0])] = const_cast(input); + for (auto next_it = (*it)->OutputNodesBegin(); next_it != (*it)->OutputNodesEnd(); ++next_it) { + const_cast((*next_it))->ReplaceDefs(replacement_defs); + } + removed_nodes.push_back((*it)->Index()); + child_removed++; + } + } + num_child++; + } + if (child_removed == num_child && child_removed > 0) { + removed_nodes.push_back(node.Index()); + } + } + } + + for (auto i : removed_nodes) { + graph.RemoveNode(i); + } + + modified = modified || !removed_nodes.empty(); + return Status::OK(); +} +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/insert_cast_transformer.h b/onnxruntime/core/framework/insert_cast_transformer.h new file mode 100644 index 0000000000000..ec35424e147ce --- /dev/null +++ b/onnxruntime/core/framework/insert_cast_transformer.h @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/graph/graph.h" +#include "core/graph/graph_transformer.h" +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { +class InsertCastTransformer : public onnxruntime::GraphTransformer { + public: + InsertCastTransformer(const std::string& name) + : onnxruntime::GraphTransformer(name, "Transformer to insert cast node that casts float16 to float for cpu nodes"), + force_cpu_fp32_(true) { + } + + void AddKernelRegistries(const std::vector& kernels) { + for (auto* kernel : kernels) { + if (kernel) + kernels_registries_.push_back(kernel); + } + } + + void AddKernelRegistry(const KernelRegistry& kernel) { + kernels_registries_.push_back(&kernel); + } + + Status Apply(onnxruntime::Graph& graph, bool& modified) const override; + + private: + bool NeedInsertCast(const onnxruntime::Node* node, const onnxruntime::NodeArg* input) const; + + std::vector kernels_registries_; + // Currently because we only have very few cpu kernels support float16, place those nodes on float16 + // will introduce many cast between fp32 and fp16, which will slow the execution. + // A better solution is to have a cost model to evaluate does it works to place the node on float16. + // Here for simplify, we only force the single-node-float16 sub-graph to float32 + bool force_cpu_fp32_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/kernel_def_builder.cc b/onnxruntime/core/framework/kernel_def_builder.cc new file mode 100644 index 0000000000000..8555adc7563de --- /dev/null +++ b/onnxruntime/core/framework/kernel_def_builder.cc @@ -0,0 +1,158 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/kernel_def_builder.h" +#include +#include + +namespace onnxruntime { +namespace { + +//assume start1 <= end1, start2 <= end2 +inline bool AreIntervalsOverlap(int start1, int end1, int start2, int end2) { + return start1 <= end2 && start2 <= end1; +} + +template +inline bool AreVectorsOverlap(const std::vector& v1, const std::vector& v2) { + for (T type : v1) { + if (std::find(v2.begin(), v2.end(), type) != v2.end()) { + return true; + } + } + return false; +} +} // namespace + +bool KernelDef::IsConflict(const KernelDef& other) const { + if (op_name_ != other.OpName() || provider_type_ != other.Provider()) + return false; + int start = 0, end = 0; + other.SinceVersion(&start, &end); + if (!AreIntervalsOverlap(op_since_version_start_, op_since_version_end_, start, end)) + return false; + //only one case they don't conflict: + //There is a type_constraint, it exists in both hands, but they don't overlap + //check types + auto other_types = other.TypeConstraints(); + bool type_has_conflict = true; + for (const auto& it : type_constraints_) { + auto iter = other_types.find(it.first); + if (iter != other_types.end()) { + if (!AreVectorsOverlap(it.second, iter->second)) { + type_has_conflict = false; + break; + } + } + } + if (!type_has_conflict) + return false; + //if has type conflict, check if any other field has different + //for example, we register two kernel with float type, but one is inplace, another is not. + //check in-place + if (inplace_map_.empty() && !other.MayInplace().empty()) + return false; + for (auto& it : inplace_map_) { + if (std::find(other.MayInplace().begin(), other.MayInplace().end(), it) == other.MayInplace().end()) + return false; + } + + //check alias + for (auto& it : alias_map_) { + if (std::find(other.Alias().begin(), other.Alias().end(), it) == other.Alias().end()) + return false; + } + if (alias_map_.empty() && !other.Alias().empty()) + return false; + + //check memory type + auto other_input_mem_types = other.InputMemoryType(); + for (auto it : input_memory_type_args_) { + if (other_input_mem_types.count(it.first) && other_input_mem_types[it.first] == it.second) + return false; + } + if (input_memory_type_args_.empty() && !other.InputMemoryType().empty()) + return false; + + auto other_output_mem_types = other.OutputMemoryType(); + for (auto it : output_memory_type_args_) { + if (other_output_mem_types.count(it.first) && other_output_mem_types[it.first] == it.second) + return false; + } + return !(output_memory_type_args_.empty() && !other.OutputMemoryType().empty()); +} + +KernelDefBuilder& KernelDefBuilder::SetName(const std::string& op_name) { + kernel_def_->op_name_ = op_name; + return *this; +} + +KernelDefBuilder& KernelDefBuilder::SetName(const char* op_name) { + kernel_def_->op_name_ = std::string(op_name); + return *this; +} + +KernelDefBuilder& KernelDefBuilder::SetDomain(const std::string& domain) { + kernel_def_->op_domain_ = domain; + return *this; +} + +KernelDefBuilder& KernelDefBuilder::SetDomain(const char* domain) { + kernel_def_->op_domain_ = std::string(domain); + return *this; +} + +KernelDefBuilder& KernelDefBuilder::Provider(onnxruntime::ProviderType provider_type) { + kernel_def_->provider_type_ = provider_type; + return *this; +} + +KernelDefBuilder& KernelDefBuilder::Provider(const char* provider_type) { + kernel_def_->provider_type_ = std::string(provider_type); + return *this; +} + +KernelDefBuilder& KernelDefBuilder::TypeConstraint(const std::string& arg_name, + const std::vector& supported_types) { + kernel_def_->type_constraints_[arg_name] = supported_types; + return *this; +} + +KernelDefBuilder& KernelDefBuilder::TypeConstraint(const char* arg_name, + const std::vector& supported_types) { + return TypeConstraint(std::string(arg_name), supported_types); +} + +KernelDefBuilder& KernelDefBuilder::TypeConstraint(const std::string& arg_name, + MLDataType supported_type) { + kernel_def_->type_constraints_[arg_name] = std::vector{supported_type}; + return *this; +} + +KernelDefBuilder& KernelDefBuilder::TypeConstraint(const char* arg_name, + MLDataType supported_type) { + return TypeConstraint(std::string(arg_name), supported_type); +} + +KernelDefBuilder& KernelDefBuilder::MayInplace(const std::vector>& inplaces) { + kernel_def_->inplace_map_ = inplaces; + return *this; +} + +KernelDefBuilder& KernelDefBuilder::MayInplace(int input_index, int output_index) { + // TODO: validate inputs. + kernel_def_->inplace_map_.emplace_back(input_index, output_index); + return *this; +} + +KernelDefBuilder& KernelDefBuilder::Alias(const std::vector>& aliases) { + kernel_def_->alias_map_ = aliases; + return *this; +} + +KernelDefBuilder& KernelDefBuilder::Alias(int input_index, int output_index) { + kernel_def_->alias_map_.emplace_back(input_index, output_index); + return *this; +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/kernel_registry.cc b/onnxruntime/core/framework/kernel_registry.cc new file mode 100644 index 0000000000000..30a0af85cc371 --- /dev/null +++ b/onnxruntime/core/framework/kernel_registry.cc @@ -0,0 +1,225 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/kernel_registry.h" + +using namespace ::onnxruntime::common; +namespace onnxruntime { + +// Find the type that name is bound to in the given node. +// "name" can represent either a type parameter or an input/output parameter. +// Returns null if a match is not found. +const ::ONNX_NAMESPACE::TypeProto* FindTypeBinding(const onnxruntime::Node& node, const std::string& name) { + const ONNX_NAMESPACE::OpSchema& op_schema = *node.Op(); + // search inputs: + const size_t len = node.InputArgCount().size(); + ONNXRUNTIME_ENFORCE(len <= op_schema.inputs().size()); + int actual_index = 0; + for (size_t formal_index = 0; formal_index != len; ++formal_index) { + auto& param = op_schema.inputs()[formal_index]; + if ((param.GetTypeStr() == name) || (param.GetName() == name)) { + // return type of any corresponding actual parameter, if present + for (int i = 0, end = node.InputArgCount()[formal_index]; i < end; ++i) { + const onnxruntime::NodeArg* arg = node.InputDefs()[actual_index + i]; + if (!arg->Exists()) continue; // a missing optional argument + return arg->TypeAsProto(); + } + } + actual_index += node.InputArgCount()[formal_index]; + } + // search outputs: + auto& actual_outputs = node.OutputDefs(); + auto num_actual_outputs = actual_outputs.size(); + auto last_formal = op_schema.outputs().size() - 1; + for (size_t i = 0; i != num_actual_outputs; ++i) { + const onnxruntime::NodeArg* arg = actual_outputs[i]; + if (!arg->Exists()) continue; + auto& formal = op_schema.outputs()[std::min(i, last_formal)]; + const auto& formal_typestr = formal.GetTypeStr(); // for easier debugging + const auto& formal_name = formal.GetName(); // for easier debugging + if ((formal_typestr == name) || (formal_name == name)) { + return arg->TypeAsProto(); + } + } + return nullptr; +} + +std::vector KernelRegistry::GetAllRegisteredOpNames() const { + std::vector ret(kernel_creator_fn_map_.size()); + size_t i = 0; + for (const auto& kvp : kernel_creator_fn_map_) { + ret[i++] = kvp.first; + } + return ret; +} + +// Check whether the types of inputs/outputs of the given node match the extra +// type-constraints of the given kernel. This serves two purposes: first, to +// select the right kernel implementation based on the types of the arguments +// when we have multiple kernels, e.g., Clip and Clip; second, to +// accommodate (and check) mapping of ONNX (specification) type to the onnxruntime +// implementation type (e.g., if we want to implement ONNX's float16 as a regular +// float in onnxruntime). (The second, however, requires a globally uniform mapping.) +// +// Note that this is not intended for type-checking the node against the ONNX +// type specification of the corresponding op, which is done before this check. +bool KernelRegistry::VerifyKernelDef(const onnxruntime::Node& node, + const KernelDef& kernel_def, + std::string& error_str, + onnxruntime::ProviderType exec_provider) { + // check if domain matches + if (node.Domain() != kernel_def.Domain()) { + std::ostringstream ostr; + ostr << "Op: " << node.OpType() + << " Domain mismatch: " + << " Expected: " << kernel_def.Domain() + << " Actual: " << node.Domain(); + error_str = ostr.str(); + return false; + } + + // check if execution provider matches + const auto& node_provider = node.GetExecutionProviderType(); + const auto& expected_provider = (node_provider.empty() ? exec_provider : node_provider); + if (expected_provider != kernel_def.Provider()) { + std::ostringstream ostr; + ostr << "Op: " << node.OpType() + << " Execution provider mismatch." + << " Expected: " << expected_provider + << " Acutal: " << kernel_def.Provider(); + error_str = ostr.str(); + return false; + } + + // check if version matches + int kernel_start_version, kernel_end_version; + kernel_def.SinceVersion(&kernel_start_version, &kernel_end_version); + + int node_since_version = node.Op()->since_version(); + // Ideal case is, if schema is Since(5), current opset version is opset 7, + // kernel_def Since(8) Invalid + // kernel_def Since(6) Valid + // kernel_def Since(5) Valid + // kernel_def Since(4) Invalid + // kernel_def Since(4, 6) Valid + + // Right now there is no "until version" on schema, it is difficult to get opset version here.(require a lot of interface change.) + // As a trade off, we will temporary require kernel definition to have the same since version as schema definition. + // so kernel_def Since(6) will become invalid now. + // After ONNX add "until version" on the schema object, we will update this place + bool valid_version = kernel_start_version == node_since_version // the idea case this branch should be kernel_start_version >= node_version && kernel_start_version <= until_version + || (kernel_start_version < node_since_version && kernel_end_version != INT_MAX && kernel_end_version >= node_since_version); + if (!valid_version) { + std::ostringstream ostr; + ostr << "Op: " << node.OpType() + << " Version mismatch." + << " node_version: " << node_since_version + << " kernel start version: " << kernel_start_version + << " kernel_end_version: " << kernel_end_version; + error_str = ostr.str(); + return false; + } + + // check if type matches + auto& kernel_type_constraints = kernel_def.TypeConstraints(); + for (auto& constraint : kernel_type_constraints) { + const std::string& name = constraint.first; + const std::vector& allowed_types = constraint.second; + const ::ONNX_NAMESPACE::TypeProto* actual_type = FindTypeBinding(node, name); + + // If actual_type is null, this represents a type-constraint on a + // missing optional parameter, which can be skipped. + // TODO: We should check that names specified in kernel_type_constraints are + // valid names (of types or parameters) at the time that kernels are registered. + if ((nullptr != actual_type) && + !std::any_of(allowed_types.begin(), allowed_types.end(), + [actual_type, &node, &error_str](const MLDataType& expected_type) { + bool rc = expected_type->IsCompatible(*actual_type); // for easier debugging + if (!rc) { + // TODO print type information as well + error_str = "Op: " + node.OpType() + " Incompatible types."; + } + return rc; + })) { + return false; + } + } + return true; +} + +Status KernelRegistry::Register(KernelDefBuilder& kernel_builder, + const KernelCreateFn& kernel_creator) { + KernelCreateInfo create_info(kernel_builder.Build(), kernel_creator); + return Register(create_info); +} + +Status KernelRegistry::Register(KernelCreateInfo& create_info) { + auto& op_name = create_info.kernel_def->OpName(); + + // Check op version conflicts. + auto range = kernel_creator_fn_map_.equal_range(op_name); + for (auto i = range.first; i != range.second; ++i) { + if (i->second.kernel_def && + i->second.status.IsOK() && + i->second.kernel_def->IsConflict(*create_info.kernel_def)) { + create_info.status = + Status(ONNXRUNTIME, FAIL, + "Failed to add kernel for " + op_name + + ": Conflicting with a registered kernel with op versions."); + // For invalid entries, we keep them in the map now. Must check for status + // when using the entries from the map. + kernel_creator_fn_map_.emplace(op_name, std::move(create_info)); + return create_info.status; + } + } + + // Register the kernel. + // Ownership of the KernelDef is transferred to the map. + kernel_creator_fn_map_.emplace(op_name, std::move(create_info)); + return Status::OK(); +} + +Status KernelRegistry::CreateKernel(const onnxruntime::Node& node, + const IExecutionProvider& execution_provider, + const SessionState& session_state, + /*out*/ std::unique_ptr& op_kernel) const { + const KernelCreateInfo* kernel_create_info = TryFindKernel(node, execution_provider.Type()); + + if (!kernel_create_info) { + return Status(ONNXRUNTIME, FAIL, "Failed to find kernel for " + node.OpType()); + } + + OpKernelInfo kernel_info(node, *kernel_create_info->kernel_def, execution_provider, session_state); + op_kernel.reset(kernel_create_info->kernel_create_func(kernel_info)); + return Status::OK(); +} + +static std::string ToString(const std::vector& error_strs) { + std::ostringstream ostr; + std::for_each(std::begin(error_strs), std::end(error_strs), + [&ostr](const std::string& str) { ostr << str << " "; }); + return ostr.str(); +} + +const KernelCreateInfo* KernelRegistry::TryFindKernel(const onnxruntime::Node& node, + onnxruntime::ProviderType exec_provider) const { + auto range = kernel_creator_fn_map_.equal_range(node.OpType()); + std::vector error_strs; + for (auto i = range.first; i != range.second; ++i) { + if (!i->second.status.IsOK()) { + LOGS_DEFAULT(ERROR) << "Failed to create kernel for op: " << node.OpType() + << " since it was ill-formed during registration"; + continue; + } + std::string error_str; + if (VerifyKernelDef(node, *i->second.kernel_def, error_str, exec_provider)) { + return &i->second; + } + error_strs.push_back(error_str); + } + LOGS_DEFAULT(INFO) << node.OpType() << " kernel is not supported in " << exec_provider + << " Encountered following errors: " << ToString(error_strs); + return nullptr; + } + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/kernel_registry_manager.cc b/onnxruntime/core/framework/kernel_registry_manager.cc new file mode 100644 index 0000000000000..2d1ededcf5d95 --- /dev/null +++ b/onnxruntime/core/framework/kernel_registry_manager.cc @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/kernel_registry_manager.h" +#include "core/framework/kernel_registry.h" +#include "core/framework/customregistry.h" +#include "core/framework/execution_providers.h" + +using namespace ONNX_NAMESPACE; +using namespace ::onnxruntime::common; +namespace onnxruntime { +Status KernelRegistryManager::CreateKernel(const onnxruntime::Node& node, + const IExecutionProvider& execution_provider, + const SessionState& session_state, + /*out*/ std::unique_ptr& op_kernel) const { + std::lock_guard lock(lock_); + if (kernel_registries_.empty()) { + return Status(ONNXRUNTIME, FAIL, "Kernel not found."); + } + + Status status; + for (auto& registry : kernel_registries_) { + status = registry->CreateKernel(node, execution_provider, session_state, op_kernel); + if (status.IsOK()) { + return status; + } + } + + return status; +} + +void KernelRegistryManager::RegisterKernels(const ExecutionProviders& execution_providers, + KernelRegistryPriority priority) { + for (auto& provider : execution_providers) + RegisterKernelRegistry(provider->GetKernelRegistry(), priority); +} + +void KernelRegistryManager::RegisterKernelRegistry(std::shared_ptr kernel_registry, + KernelRegistryPriority priority) { + std::lock_guard lock(lock_); + if (nullptr == kernel_registry) { + return; + } + + if (priority == KernelRegistryPriority::HighPriority) { + kernel_registries_.push_front(kernel_registry); + } else { + kernel_registries_.push_back(kernel_registry); + } +} + +Status KernelRegistryManager::SearchKernelRegistry(const onnxruntime::Node& node, + /*out*/ const KernelCreateInfo** kernel_create_info) const { + std::lock_guard lock(lock_); + if (kernel_registries_.empty()) { + return Status(ONNXRUNTIME, FAIL, "Kernel def not found."); + } + + Status status; + for (auto& registry : kernel_registries_) { + *kernel_create_info = registry->TryFindKernel(node, ""); + if (*kernel_create_info != nullptr) { + return Status::OK(); + } + } + + return Status(ONNXRUNTIME, FAIL, "Failed to find kernel for " + node.OpType()); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/kernel_registry_manager.h b/onnxruntime/core/framework/kernel_registry_manager.h new file mode 100644 index 0000000000000..6cd6eb349960b --- /dev/null +++ b/onnxruntime/core/framework/kernel_registry_manager.h @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include +#include +#include +#include "core/common/status.h" +#include "core/graph/graph.h" + +namespace onnxruntime { +struct KernelCreateInfo; +class ExecutionProviders; +class IExecutionProvider; +class KernelRegistry; +class OpKernel; +class SessionState; + +enum class KernelRegistryPriority { + HighPriority, + LowPriority +}; + +// Kernel registries' manager. +// There're 2 kinds of kernel registries with priority from high to low as below, +// 1. Custom execution provider type specific kernel registries. +// 2. common execution provider type specific kernel registries. +// The 1st and 2nd ones are shared across sessions. +class KernelRegistryManager { + public: + KernelRegistryManager() = default; + + void RegisterKernels(const ExecutionProviders& execution_providers, + KernelRegistryPriority priority = KernelRegistryPriority::LowPriority); + + void RegisterKernelRegistry(std::shared_ptr kernel_registry, KernelRegistryPriority priority); + + Status CreateKernel(const onnxruntime::Node& node, + const IExecutionProvider& execution_provider, + const SessionState& session_state, + /*out*/ std::unique_ptr& op_kernel) const; + + Status SearchKernelRegistry(const onnxruntime::Node& node, + /*out*/ const KernelCreateInfo** kernel_create_info) const; + + // Get all kernel registries. There are no nullptr entries. + std::vector GetAllKernelRegistries() const { + std::vector result; + for (auto& registry : kernel_registries_) { + result.push_back(registry.get()); + } + return result; + } + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(KernelRegistryManager); + + // This list stores all kernel registries shared across sessions, including common ones and customized ones. + std::list> kernel_registries_; + mutable std::mutex lock_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/mem_pattern.h b/onnxruntime/core/framework/mem_pattern.h new file mode 100644 index 0000000000000..916d37ec0815a --- /dev/null +++ b/onnxruntime/core/framework/mem_pattern.h @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/common/common.h" +#include "core/framework/allocation_planner.h" + +namespace onnxruntime { +struct MemoryBlock { + size_t offset_{0}; + size_t size_{0}; + + MemoryBlock() = default; + MemoryBlock(size_t offset, size_t size) : offset_(offset), size_(size) {} +}; + +class MemoryPattern { + friend class MemPatternPlanner; + + public: + MemoryPattern() = default; + + MemoryPattern(MemoryPattern&& rhs) + : patterns_{std::move(rhs.patterns_)}, + peak_size_{std::move(rhs.peak_size_)} {} + + MemoryPattern& operator=(MemoryPattern&& rhs) { + patterns_ = std::move(rhs.patterns_); + peak_size_ = std::move(rhs.peak_size_); + return *this; + } + + size_t PeakSize() const { + return peak_size_; + } + + const MemoryBlock* GetBlock(int ml_value_idx) const { + auto it = patterns_.find(ml_value_idx); + if (it == patterns_.end()) + return nullptr; + + return &it->second; + } + + private: + // allow move + ONNXRUNTIME_DISALLOW_COPY_AND_ASSIGNMENT(MemoryPattern); + + std::unordered_map patterns_; + size_t peak_size_{0}; +}; + +struct MemoryPatternGroup { + std::vector locations; + std::vector patterns; + + const MemoryPattern* GetPatterns(const ONNXRuntimeAllocatorInfo& location) const { + for (size_t i = 0; i < locations.size(); i++) + if (locations[i] == location) { + return &patterns[i]; + } + return nullptr; + } +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/mem_pattern_planner.h b/onnxruntime/core/framework/mem_pattern_planner.h new file mode 100644 index 0000000000000..97a35eeee6a0e --- /dev/null +++ b/onnxruntime/core/framework/mem_pattern_planner.h @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/framework/mem_pattern.h" +#include "core/framework/allocation_planner.h" +#include + +namespace onnxruntime { +// MemPatternPlanner is used to trace allocation/free steps +// in a single iteration, record the pattern and cached for +// future request if they have the same input shape. +class MemPatternPlanner { + public: + MemPatternPlanner() = default; + + void TraceAllocation(int ml_value_idx, size_t size) { + if (size == 0) { + allocs_.emplace_back(ml_value_idx, MemoryBlock(0, 0)); + return; + } + + size_t current = 0; + size_t waste_bytes = std::numeric_limits::max(); + size_t best_offset = 0; + if (!blocks_.empty()) { + auto last_block = allocs_[*blocks_.rbegin()]; + best_offset = last_block.block_.offset_ + last_block.block_.size_; + } + + std::list::iterator best_fit_it = blocks_.end(); + for (auto it = blocks_.begin(); it != blocks_.end(); it++) { + if (allocs_[*it].block_.offset_ >= current) { + auto gap = allocs_[*it].block_.offset_ - current; + if (gap >= size && (gap - size) < waste_bytes) { + best_fit_it = it; + waste_bytes = gap - size; + best_offset = current; + } + } + current = allocs_[*it].block_.offset_ + allocs_[*it].block_.size_; + } + + allocs_.emplace_back(ml_value_idx, MemoryBlock(best_offset, size)); + buffer_size = std::max(buffer_size, best_offset + size); + blocks_.insert(best_fit_it, (static_cast(allocs_.size()) - 1)); + } + + void TraceFree(int ml_value_index) { + for (auto it = blocks_.begin(); it != blocks_.end(); it++) { + if (allocs_[*it].index_ == ml_value_index) { + blocks_.erase(it); + break; + } + } + } + + MemoryPattern GenerateMemPattern() { + MemoryPattern pattern; + pattern.peak_size_ = buffer_size; + for (auto& alloc : allocs_) { + pattern.patterns_[alloc.index_] = alloc.block_; + } + + return pattern; + } + + protected: + struct MLValueAllocationBlock { + int index_{-1}; + MemoryBlock block_; + + MLValueAllocationBlock() = default; + MLValueAllocationBlock(int index, MemoryBlock block) : index_(index), block_(block) {} + }; + + std::vector allocs_; + // blocks_ the list of currently allocated memory blocks, sorted in order of their offset + std::list blocks_; + size_t buffer_size{0}; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/memcpy.cc b/onnxruntime/core/framework/memcpy.cc new file mode 100644 index 0000000000000..a7847c2f0746f --- /dev/null +++ b/onnxruntime/core/framework/memcpy.cc @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "memcpy.h" +using namespace ONNX_NAMESPACE; +namespace onnxruntime { + +Memcpy::Memcpy(const OpKernelInfo& info) + : OpKernel(info) { + provider_ = info.GetExecutionProvider(); +} + +Status Memcpy::Compute(OpKernelContext* ctx) const { + const auto* X = ctx->Input(0); + Tensor* Y = ctx->Output(0, X->Shape()); + Status retval = provider_->CopyTensor(*X, *Y, Info().GetKernelDef().ExecQueueId()); + return retval; +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/memcpy.h b/onnxruntime/core/framework/memcpy.h new file mode 100644 index 0000000000000..a933ce40f81f1 --- /dev/null +++ b/onnxruntime/core/framework/memcpy.h @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { + +class Memcpy final : public OpKernel { + public: + Memcpy(const OpKernelInfo& info); + + Status Compute(OpKernelContext* context) const override; + + private: + const IExecutionProvider* provider_; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/ml_value_pattern_planner.cc b/onnxruntime/core/framework/ml_value_pattern_planner.cc new file mode 100644 index 0000000000000..5f765c1ede71b --- /dev/null +++ b/onnxruntime/core/framework/ml_value_pattern_planner.cc @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include "core/framework/ml_value_patterns_planner.h" +#include "core/framework/sequential_execution_plan.h" + +namespace onnxruntime { +MLValuePatternPlanner::MLValuePatternPlanner(const SequentialExecutionPlan& execution_plan) + : execution_planner_{execution_plan} { + std::set locations; + for (auto& alloc_plan : execution_planner_.allocation_plan) { + if (locations.find(alloc_plan.location) == locations.end()) + locations.insert(alloc_plan.location); + } + for (auto& location : locations) { + pattern_planners_.push_back(std::make_unique()); + planner_map_[location] = pattern_planners_.back().get(); + } +} +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/ml_value_patterns_planner.h b/onnxruntime/core/framework/ml_value_patterns_planner.h new file mode 100644 index 0000000000000..250f68c45d7e8 --- /dev/null +++ b/onnxruntime/core/framework/ml_value_patterns_planner.h @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/common/common.h" +#include "core/framework/mem_pattern_planner.h" +#include "core/framework/allocation_planner.h" +#include +#include +#include + +namespace onnxruntime { +struct SequentialExecutionPlan; + +class MLValuePatternPlanner { + public: + explicit MLValuePatternPlanner(const SequentialExecutionPlan& execution_plan); + + common::Status TraceAllocation(int ml_value_idx, size_t size) { + auto location = execution_planner_.allocation_plan[ml_value_idx].location; + auto it = planner_map_.find(location); + if (it == planner_map_.end()) { + return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT); + } + + std::lock_guard lock(lock_); + it->second->TraceAllocation(ml_value_idx, size); + return common::Status::OK(); + } + + common::Status TraceFree(int ml_value_index) { + auto location = execution_planner_.allocation_plan[ml_value_index].location; + auto it = planner_map_.find(location); + if (it == planner_map_.end()) { + return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT); + } + + std::lock_guard lock(lock_); + it->second->TraceFree(ml_value_index); + return common::Status::OK(); + } + + common::Status GeneratePatterns(MemoryPatternGroup* out) { + if (!out) + return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT); + + std::lock_guard lock(lock_); + for (auto& it : planner_map_) { + out->locations.push_back(it.first); + out->patterns.push_back(it.second->GenerateMemPattern()); + } + + return common::Status::OK(); + } + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(MLValuePatternPlanner); + + mutable std::mutex lock_; + std::map planner_map_; + std::vector > pattern_planners_; + const SequentialExecutionPlan& execution_planner_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/mldata_type_utils.cc b/onnxruntime/core/framework/mldata_type_utils.cc new file mode 100644 index 0000000000000..8eb1a6612de0c --- /dev/null +++ b/onnxruntime/core/framework/mldata_type_utils.cc @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "mldata_type_utils.h" + +namespace onnxruntime { +namespace utils { +MLDataType GetMLDataType(const onnxruntime::NodeArg& arg) { + const ONNX_NAMESPACE::DataType ptype = arg.Type(); + const ONNX_NAMESPACE::TypeProto& type_proto = ONNX_NAMESPACE::Utils::DataTypeUtils::ToTypeProto(ptype); + return DataTypeImpl::TypeFromProto(type_proto); +} +} // namespace utils +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/mldata_type_utils.h b/onnxruntime/core/framework/mldata_type_utils.h new file mode 100644 index 0000000000000..1e3321bc6e868 --- /dev/null +++ b/onnxruntime/core/framework/mldata_type_utils.h @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/framework/data_types.h" +#include "core/graph/graph.h" +#include "onnx/defs/data_type_utils.h" + +namespace onnxruntime { +namespace utils { +MLDataType GetMLDataType(const onnxruntime::NodeArg& arg); +} // namespace utils +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/mlvalue_name_idx_map.h b/onnxruntime/core/framework/mlvalue_name_idx_map.h new file mode 100644 index 0000000000000..9dbc720e2be62 --- /dev/null +++ b/onnxruntime/core/framework/mlvalue_name_idx_map.h @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include + +#include "core/common/common.h" + +//This class is not thread-safe +//TODO: this is a static hash lookup, it's easy to do it better +namespace onnxruntime { +class MLValueNameIdxMap { + public: + using const_iterator = typename std::unordered_map::const_iterator; + + MLValueNameIdxMap() = default; + + // Add MLValue name to map and return index associated with it. + // If entry already existed the existing index value is returned. + int Add(const std::string& name) { + auto it = map_.find(name); + if (it == map_.end()) { + int idx; + idx = mlvalue_max_idx_++; + map_.insert(it, {name, idx}); + return idx; + } + return it->second; + } + + common::Status GetIdx(const std::string& name, int& idx) const { + idx = -1; + + auto it = map_.find(name); + if (it == map_.end()) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Could not find MLValue with name: ", name); + } + + idx = it->second; + return common::Status::OK(); + } + + size_t Size() const { return map_.size(); }; + int MaxIdx() const { return mlvalue_max_idx_; } + + const_iterator begin() const noexcept { return map_.cbegin(); } + const_iterator end() const noexcept { return map_.cend(); } + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(MLValueNameIdxMap); + + int mlvalue_max_idx_ = 0; + std::unordered_map map_; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/mlvalue_tensor_slicer.cc b/onnxruntime/core/framework/mlvalue_tensor_slicer.cc new file mode 100644 index 0000000000000..11d2fde87bd9b --- /dev/null +++ b/onnxruntime/core/framework/mlvalue_tensor_slicer.cc @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/mlvalue_tensor_slicer.h" +#include + +namespace onnxruntime { + +template +MLValueTensorSlicer MLValueTensorSlicer::Create(T& mlvalue, int64_t slice_dimension, int64_t dim0_offset) { + static_assert(std::is_same, MLValue>::value, + "MLValueTensorSlicer can only be used with 'MLValue' or 'const MLValue'"); + + ONNXRUNTIME_ENFORCE(mlvalue.IsTensor(), "Can't slice a non-tensor MLValue. Type was ", mlvalue.Type()); + ONNXRUNTIME_ENFORCE(mlvalue.IsAllocated(), "MLValue has not been allocated so can't be sliced."); + + auto& tensor_shape{mlvalue.template Get().Shape()}; + ONNXRUNTIME_ENFORCE(gsl::narrow_cast(tensor_shape.NumDimensions()) > slice_dimension, + "Insufficient dimensions to slice on ", slice_dimension, ". Shape:", tensor_shape); + + auto dim0_size = tensor_shape[0]; + ONNXRUNTIME_ENFORCE(dim0_offset < dim0_size, "Invalid dim0_offset of ", dim0_offset, ". Dimension 0 is ", dim0_size); + + return MLValueTensorSlicer{mlvalue, slice_dimension, dim0_offset}; +}; + +template +MLValueTensorSlicer::Iterator::Iterator(T& mlvalue, size_t slice_dimension, size_t dim0_offset, + int64_t position, Direction direction) + : mlvalue_{&mlvalue}, + position_{position}, + increment_by_{direction == Direction::kForward ? 1 : -1}, + position_materialized_{-1} { + const auto& tensor = mlvalue.template Get(); + tensor_data_type_ = tensor.DataType(); + tensor_location_ = &tensor.Location(); + + const TensorShape& shape = tensor.Shape(); + sequence_length_ = shape[slice_dimension]; + per_iteration_shape_ = shape.Slice(slice_dimension + 1); + const int64_t per_iteration_shape_size = per_iteration_shape_.Size(); + assert(per_iteration_shape_size >= 0); + if (!IAllocator::CalcMemSizeForArray(static_cast(per_iteration_shape_size), tensor.DataType()->Size(), &per_iteration_offset_)) + throw std::runtime_error("size overflow"); + const int64_t slice_dimension_size = shape.Slice(slice_dimension).Size(); + assert(slice_dimension_size >= 0); + + size_t total_len; + if (!IAllocator::CalcMemSizeForArray(static_cast(slice_dimension_size), tensor.DataType()->Size(), &total_len)) + throw std::runtime_error("size overflow"); + if (!IAllocator::CalcMemSizeForArray(dim0_offset, total_len, &total_len)) + throw std::runtime_error("size overflow"); + // move tensor_data_raw_ to the start of the section to slice + tensor_data_raw_ = static_cast(tensor.DataRaw()) + total_len; + + // constrain position_ to valid bounds of 0 to sequence_length_ if forward, or -1 to sequence_length_ - 1 if reverse + if (direction == Direction::kForward) { + if (position_ > sequence_length_) + position_ = sequence_length_; // end() + } else { + if (position_ >= sequence_length_) + position_ = sequence_length_ - 1; // begin() at first valid position_ + + if (position_ < -1) + position_ = -1; // end() + } +} + +template +void MLValueTensorSlicer::Iterator::MaterializeMLValue() const { + position_materialized_ = position_; + const void* tensor_slice_data_raw = static_cast(tensor_data_raw_) + (position_ * per_iteration_offset_); + + // create a sub Tensor for the current position, and put it in an MLValue. + // + // We need the non-const data pointer from the Tensor in order to create the sub-Tensors as we iterate, + // so a const_cast is required. + // However we will only return a non-const MLValue from operator* if MLValueTensorSlicer was created with + // a non-const MLValue, so externally we maintain constness as expected. + // + // TODO: Ideally we could avoid the overhead of creating a new Tensor but that would require + // a lot more complexity (re-consider how ExecutionFrame and OpKernelContext work and whether + // they need to be MLValue based, or whether they could be Tensor based). + // Potential future performance enhancement. + auto sub_tensor = std::make_unique(tensor_data_type_, + per_iteration_shape_, + const_cast(tensor_slice_data_raw), + *tensor_location_); + + current_ = MLValue{sub_tensor.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()}; +} + +template class MLValueTensorSlicer; +template class MLValueTensorSlicer; + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/mlvalue_tensor_slicer.h b/onnxruntime/core/framework/mlvalue_tensor_slicer.h new file mode 100644 index 0000000000000..bf26ba95c7440 --- /dev/null +++ b/onnxruntime/core/framework/mlvalue_tensor_slicer.h @@ -0,0 +1,148 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include + +#include "core/common/common.h" +#include "core/framework/ml_value.h" +#include "core/framework/tensor.h" + +namespace onnxruntime { + +/** +Class to provide a slicing service over a Tensor stored within an MLValue with shape +{batch size, sequence length, }. Access to the slices is via an iterator interface. + +For each iteration an MLValue will be returned containing a sub-Tensor of the original Tensor. +The sub-Tensor applies the relevant offset to the data address from the original Tensor in order +to avoid any memory allocations/copies for the tensor data. +*/ +template +class MLValueTensorSlicer { + public: + /** + Create a new instance to slice the Tensor contained in an MLValue + into sub-Tensors contained within new MLValue instances that are accessed via the Iterator. + T must be 'MLValue' or 'const MLValue' + @param slice_dimension Dimension to slice on. + @param dim0_offset Offset to start at. Only meaningful if slice_dimension != 0. + e.g. if input is [batch, seq_len, data] and you want to slice the seq_len dimension, you need to + create an Iterator instance for each batch item, incrementing dim0_offset for each one. + */ + static MLValueTensorSlicer Create(T& mlvalue, int64_t slice_dimension = 0, int64_t dim0_offset = 0); + + class Iterator { + public: + using iterator_category = std::input_iterator_tag; + using value_type = T; + using difference_type = ptrdiff_t; + using pointer = T*; + using reference = T&; + using const_reference = std::add_const_t; + + enum class Direction { kForward, + kReverse }; + + explicit Iterator(T& mlvalue, size_t slice_dimension, size_t dim0_offset, + int64_t position, Direction direction = Direction::kForward); + + bool operator==(const Iterator& other) const noexcept { + return mlvalue_ == other.mlvalue_ && position_ == other.position_; + } + + bool operator!=(const Iterator& other) const noexcept { + return !(*this == other); + } + + Iterator& operator++() { + position_ += increment_by_; + return *this; + } + + Iterator operator++(int) { + Iterator tmp{*this}; + ++(*this); + + return tmp; + } + + Iterator& operator+=(difference_type n) { + position_ += increment_by_ * n; + return *this; + } + + // const accessor is always enabled + const_reference operator*() const { + ONNXRUNTIME_ENFORCE(position_ >= 0 && position_ < sequence_length_); + if (position_ != position_materialized_) { + MaterializeMLValue(); + } + + return current_; + } + + // non-const is only enabled if T is not const (i.e. is 'MLValue' not 'const MLValue') + std::enable_if_t::value, reference> operator*() { + ONNXRUNTIME_ENFORCE(position_ >= 0 && position_ < sequence_length_); + if (position_ != position_materialized_) { + MaterializeMLValue(); + } + + return current_; + } + + private: + void MaterializeMLValue() const; + + T* mlvalue_; + int64_t position_; + + // 1 for forward, -1 for reverse + // Alternatively we could apply a std::reverse_iterator adapter to Iterator, however the primary use case + // for this class involves passing a mix of forward/reverse iterator instances in a single collection so + // we need to handle the direction internally so only one type is involved in that collection. + const int64_t increment_by_; + + const void* tensor_data_raw_; + MLDataType tensor_data_type_; + const ONNXRuntimeAllocatorInfo* tensor_location_; + + int64_t sequence_length_; + TensorShape per_iteration_shape_; + size_t per_iteration_offset_; + + mutable int64_t position_materialized_; // position_ when current_ was created + mutable MLValue current_; + }; + + Iterator begin() const noexcept { return Iterator(*mlvalue_, slice_dimension_, dim0_offset_, 0); } + Iterator end() const noexcept { return Iterator(*mlvalue_, slice_dimension_, dim0_offset_, + std::numeric_limits::max()); } + + Iterator rbegin() const noexcept { + return Iterator(*mlvalue_, slice_dimension_, dim0_offset_, std::numeric_limits::max(), + Iterator::Direction::kReverse); + } + + Iterator rend() const noexcept { + return Iterator(*mlvalue_, slice_dimension_, dim0_offset_, -1, + Iterator::Direction::kReverse); + } + + private: + MLValueTensorSlicer(T& mlvalue, int64_t slice_dimension, int64_t dim0_offset) noexcept + : mlvalue_{&mlvalue}, + slice_dimension_{slice_dimension}, + dim0_offset_{dim0_offset} { + } + + T* mlvalue_; + int64_t slice_dimension_; + int64_t dim0_offset_; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/onnx_object.cc b/onnxruntime/core/framework/onnx_object.cc new file mode 100644 index 0000000000000..c2ae87b97b0a7 --- /dev/null +++ b/onnxruntime/core/framework/onnx_object.cc @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/onnx_object.h" + +ONNXRUNTIME_API(uint32_t, ONNXRuntimeAddRefToObject, void* ptr) { + return (*static_cast(ptr))->AddRef(ptr); +} +ONNXRUNTIME_API(uint32_t, ONNXRuntimeReleaseObject, void* ptr) { + if (ptr == nullptr) return 0; + return (*static_cast(ptr))->Release(ptr); +} diff --git a/onnxruntime/core/framework/op_kernel.cc b/onnxruntime/core/framework/op_kernel.cc new file mode 100644 index 0000000000000..3813688cc65d0 --- /dev/null +++ b/onnxruntime/core/framework/op_kernel.cc @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/op_kernel.h" +#include "core/framework/execution_frame.h" +#include "core/framework/session_state.h" +#include "core/graph/op.h" +#include "core/common/logging/logging.h" +using namespace ::onnxruntime::common; +namespace onnxruntime { + +OpKernelContext::OpKernelContext(ExecutionFrame* frame, + const OpKernel* kernel, + const logging::Logger& logger) + : execution_frame_(frame), + kernel_(kernel), + logger_(&logger) { + ONNXRUNTIME_ENFORCE(frame != nullptr, "Execution frame was null"); + ONNXRUNTIME_ENFORCE(kernel != nullptr, "OpKernel was null"); + + node_input_start_index_ = frame->GetFirstArgIndex(kernel->Node().Index()); + node_implicit_input_start_index_ = node_input_start_index_ + InputCount(); + node_output_start_index_ = node_implicit_input_start_index_ + ImplicitInputCount(); +} + +Tensor* OpKernelContext::Output(int index, const TensorShape& shape) { + if (index < 0 || index >= OutputCount()) + return nullptr; + + // In this case, it's assumed that the tensor hasn't been allocated yet, + // so that it's calling ExecutionFrame to create a tensor in the given position with given shape. + MLValueAllocationParameters parameters; + parameters.tensor_shape = shape; + //: Though we don't need to give 'ret' an initial value, GCC would generate a warning if we don't do that + //"error: 'ret' may be used uninitialized in this function" + //This warning only exists in Release build. + //I believe it's a false alarm. + MLValue* p_ml_value = nullptr; + Status status = execution_frame_->GetOrCreateNodeOutputMLValue(GetOutputArgIndex(index), parameters, p_ml_value); + ONNXRUNTIME_ENFORCE(status.IsOK(), status.ErrorMessage()); + return p_ml_value ? p_ml_value->GetMutable() : nullptr; +} + +int OpKernelContext::NumVariadicInputs(size_t arg_num) const { + auto& arg_counts = kernel_->Node().InputArgCount(); + + ONNXRUNTIME_ENFORCE(arg_num < arg_counts.size(), "Invalid arg_num of ", arg_num, ". Num args is ", arg_counts.size()); + + return arg_counts[arg_num]; +} + +Status OpKernelContext::GetTempSpaceAllocator(AllocatorPtr* output) const { + *output = execution_frame_->GetAllocator(kernel_->Allocator(0, ONNXRuntimeMemTypeDefault)); + if (!*output) + return Status(common::ONNXRUNTIME, common::FAIL, "TempSpace allocator not found"); + return Status::OK(); +} + +MLDataType OpKernelContext::InputType(int index) const { + int input_arg_index = GetInputArgIndex(index); + const MLValue* p_ml_value = execution_frame_->GetNodeInputOrOutputMLValue(input_arg_index); + return p_ml_value ? p_ml_value->Type() : nullptr; +} + +MLDataType OpKernelContext::OutputType(int index) const { + auto output_arg_index = GetOutputArgIndex(index); + const MLValue* p_ml_value = execution_frame_->GetNodeInputOrOutputMLValue(output_arg_index); + return p_ml_value ? p_ml_value->Type() : nullptr; +} + +Fence_t OpKernelContext::InputFence(int index) const { + if (index >= InputCount()) + return nullptr; + + int input_index = GetInputArgIndex(index); + const MLValue* p_ml_value = execution_frame_->GetNodeInputOrOutputMLValue(input_index); + return p_ml_value ? p_ml_value->Fence() : nullptr; +} + +Fence_t OpKernelContext::ImplicitInputFence(int index) const { + if (index >= ImplicitInputCount()) + return nullptr; + + int input_index = GetImplicitInputArgIndex(index); + const MLValue* p_ml_value = execution_frame_->GetNodeInputOrOutputMLValue(input_index); + return p_ml_value ? p_ml_value->Fence() : nullptr; +} + +Fence_t OpKernelContext::OutputFence(int index) const { + if (index >= OutputCount()) + return nullptr; + + auto output_arg_index = GetOutputArgIndex(index); + const MLValue* p_ml_value = execution_frame_->GetNodeInputOrOutputMLValue(output_arg_index); + return p_ml_value ? p_ml_value->Fence() : nullptr; +} + +Status OpKernelContext::GetOrCreateOutputMLValue(int index, MLValue*& p_value) { + auto output_arg_index = GetOutputArgIndex(index); + MLValueAllocationParameters parameters; + ONNXRUNTIME_ENFORCE(execution_frame_->GetOrCreateNodeOutputMLValue(output_arg_index, parameters, p_value).IsOK()); + return Status::OK(); +} + +int OpKernelContext::GetInputArgIndex(int index) const { + return node_input_start_index_ + index; +} + +int OpKernelContext::GetImplicitInputArgIndex(int index) const { + return node_implicit_input_start_index_ + index; +} + +int OpKernelContext::GetOutputArgIndex(int index) const { + return node_output_start_index_ + index; +} + +onnxruntime::NodeIndex OpKernelContext::GetNodeIndex() const { + return kernel_->Node().Index(); +} + +const SessionState& OpKernelContext::GetSessionState() const { + return execution_frame_->SessionState(); +} + +const MLValue* OpKernelContext::GetInputMLValue(int index) const { + if (index < 0 || index >= InputCount()) + return nullptr; + + int input_arg_index = GetInputArgIndex(index); + return execution_frame_->GetNodeInputOrOutputMLValue(input_arg_index); +} + +const MLValue* OpKernelContext::GetImplicitInputMLValue(int index) const { + if (index < 0 || index >= ImplicitInputCount()) + return nullptr; + + int input_arg_index = GetImplicitInputArgIndex(index); + return execution_frame_->GetNodeInputOrOutputMLValue(input_arg_index); +} + +MLValue* OpKernelContext::GetOutputMLValue(int index) { + if (index < 0 || index >= OutputCount()) + return nullptr; + + auto output_arg_index = GetOutputArgIndex(index); + return execution_frame_->GetMutableNodeInputOrOutputMLValue(output_arg_index); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/op_kernel_context_internal.h b/onnxruntime/core/framework/op_kernel_context_internal.h new file mode 100644 index 0000000000000..afce7fc3c1ef8 --- /dev/null +++ b/onnxruntime/core/framework/op_kernel_context_internal.h @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/framework/op_kernel.h" +#include "core/framework/session_state.h" + +// onnxruntime internal OpKernelContext derived class to provide additional +// APIs that aren't desirable to add to the public OpKernelContext API + +namespace onnxruntime { +class SessionState; + +class OpKernelContextInternal : public OpKernelContext { + public: + explicit OpKernelContextInternal(ExecutionFrame& frame, + const OpKernel& kernel, + const logging::Logger& logger, + const std::vector& implicit_inputs, + const bool& terminate_flag) + : OpKernelContext(&frame, &kernel, logger), + implicit_inputs_{implicit_inputs}, + terminate_flag_{terminate_flag} { + } + + const SessionState* SubgraphSessionState(const std::string& attribute_name) { + return GetSessionState().GetSubgraphSessionState(GetNodeIndex(), attribute_name); + } + + const MLValue* GetInputMLValue(int index) const { + return OpKernelContext::GetInputMLValue(index); + } + + MLValue* GetOutputMLValue(int index) { + return OpKernelContext::GetOutputMLValue(index); + } + + std::unordered_map GetImplicitInputs() const { + // we need to convert implicit_inputs_ to a name to MLValue map so it can be used in the ExecutionFrame + // for a subgraph (the index numbers will be different there). + std::unordered_map implicit_inputs_map; + + for (int i = 0, end = gsl::narrow_cast(implicit_inputs_.size()); i < end; ++i) { + implicit_inputs_map[implicit_inputs_[i]->Name()] = GetImplicitInputMLValue(i); + } + + return implicit_inputs_map; + } + + const bool& GetTerminateFlag() const noexcept { return terminate_flag_; } + + private: + const std::vector& implicit_inputs_; + const bool& terminate_flag_; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/op_kernel_info.cc b/onnxruntime/core/framework/op_kernel_info.cc new file mode 100644 index 0000000000000..a63057869372d --- /dev/null +++ b/onnxruntime/core/framework/op_kernel_info.cc @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/op_kernel.h" +#include "core/framework/op_kernel_info.h" +#include "core/framework/session_state.h" + +namespace onnxruntime { + +OpKernelInfo::OpKernelInfo(const onnxruntime::Node& node, + const KernelDef& kernel_def, + const IExecutionProvider& execution_provider, + const SessionState& session_state) + : OpNodeProtoHelper(&proto_helper_context_), + node_(node), + kernel_def_(kernel_def), + execution_provider_(&execution_provider), + proto_helper_context_(node), + session_state_(session_state) {} + +OpKernelInfo::OpKernelInfo(const OpKernelInfo& other) + : OpKernelInfo(other.node_, other.kernel_def_, *other.execution_provider_, other.session_state_) {} + +const ONNXRuntimeAllocatorInfo& OpKernelInfo::GetAllocatorInfo(int device_id, ONNXRuntimeMemType mem_type) const { + AllocatorPtr alloc = execution_provider_->GetAllocator(device_id, mem_type); + if (alloc == nullptr) ONNXRUNTIME_THROW("cannot find allocator"); + return alloc->Info(); +} + +const KernelDef& OpKernelInfo::GetKernelDef() const { + return kernel_def_; +} + +const IExecutionProvider* OpKernelInfo::GetExecutionProvider() const noexcept { + return execution_provider_; +} + +const onnxruntime::Node& OpKernelInfo::node() const noexcept { + return node_; +} + +bool OpKernelInfo::TryGetConstantInput(int input_index, const Tensor** constant_input_value) const { + if (input_index < 0 || input_index >= gsl::narrow_cast(node_.InputDefs().size())) { + return false; + } + auto& input_arg_name = node_.InputDefs()[input_index]->Name(); + int input_arg_index = -1; + if (!session_state_.GetMLValueNameIdxMap().GetIdx(input_arg_name, input_arg_index).IsOK()) { + return false; + } + + auto& initializers = session_state_.GetInitializedTensors(); + auto iter = initializers.find(input_arg_index); + if (initializers.end() == iter) { + return false; + } + if (!iter->second.IsTensor()) { + // Only constant Tensor input is support right now, since we're using initializers to store the data. + return false; + } + *constant_input_value = &iter->second.Get(); + return true; +} +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/op_node_proto_helper.cc b/onnxruntime/core/framework/op_node_proto_helper.cc new file mode 100644 index 0000000000000..3e1cf306c7325 --- /dev/null +++ b/onnxruntime/core/framework/op_node_proto_helper.cc @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/exceptions.h" +#include "core/common/status.h" +#include "core/common/logging/logging.h" +#include "core/graph/graph.h" +#include "core/graph/op.h" +#include "onnx/defs/schema.h" +#include "core/framework/op_kernel.h" +#include "onnx/defs/schema.h" +#include "gsl/span" +using namespace ONNX_NAMESPACE; +using namespace ::onnxruntime::common; +namespace onnxruntime { + +#define ONNXRUNTIME_DEFINE_GET_ATTR(IMPL_T, T, type) \ + template <> \ + template <> \ + Status OpNodeProtoHelper::GetAttr( \ + const std::string& name, T* value) const { \ + const AttributeProto* attr = TryGetAttribute(name); \ + if (!attr) { \ + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "No attribute with name:'", name, "'is defined."); \ + } \ + if (!attr->has_##type()) { \ + return Status(ONNXRUNTIME, FAIL, "Attibute name and type don't match"); \ + } else { \ + *value = static_cast(attr->type()); \ + return Status::OK(); \ + } \ + } + +#define ONNXRUNTIME_DEFINE_GET_ATTRS(IMPL_T, T, list) \ + template <> \ + template <> \ + Status OpNodeProtoHelper::GetAttrs( \ + const std::string& name, std::vector& values) const { \ + const AttributeProto* attr = TryGetAttribute(name); \ + if (!attr) { \ + return Status(ONNXRUNTIME, FAIL, "No attribute with this name is defined."); \ + } \ + values.reserve(attr->list##_size()); \ + for (int i = 0; i < attr->list##_size(); ++i) { \ + values.push_back(static_cast(attr->list(i))); \ + } \ + return Status::OK(); \ + } \ + template <> \ + template <> \ + Status OpNodeProtoHelper::GetAttrs( \ + const std::string& name, gsl::span values) const { \ + const AttributeProto* attr = TryGetAttribute(name); \ + if (!attr) { \ + return Status(ONNXRUNTIME, FAIL, "No attribute with this name is defined."); \ + } \ + ONNXRUNTIME_ENFORCE(values.size() == attr->list##_size()); \ + for (int i = 0; i < attr->list##_size(); ++i) { \ + values[i] = static_cast(attr->list(i)); \ + } \ + return Status::OK(); \ + } + +#define ONNXRUNTIME_DEFINE_GET_ATTR_SPECIALIZATIONS(type, list) \ + ONNXRUNTIME_DEFINE_GET_ATTR(ProtoHelperNodeContext, type, list) \ + ONNXRUNTIME_DEFINE_GET_ATTR(InferenceContext, type, list) + +#define ONNXRUNTIME_DEFINE_GET_ATTRS_SPECIALIZATIONS(type, list) \ + ONNXRUNTIME_DEFINE_GET_ATTRS(ProtoHelperNodeContext, type, list) \ + ONNXRUNTIME_DEFINE_GET_ATTRS(InferenceContext, type, list) + +ONNXRUNTIME_DEFINE_GET_ATTR_SPECIALIZATIONS(float, f) +ONNXRUNTIME_DEFINE_GET_ATTR_SPECIALIZATIONS(int64_t, i) +ONNXRUNTIME_DEFINE_GET_ATTR_SPECIALIZATIONS(std::string, s) +ONNXRUNTIME_DEFINE_GET_ATTR_SPECIALIZATIONS(TensorProto, t) +ONNXRUNTIME_DEFINE_GET_ATTR_SPECIALIZATIONS(GraphProto, g) +ONNXRUNTIME_DEFINE_GET_ATTRS_SPECIALIZATIONS(float, floats) +ONNXRUNTIME_DEFINE_GET_ATTRS_SPECIALIZATIONS(int64_t, ints) +ONNXRUNTIME_DEFINE_GET_ATTRS_SPECIALIZATIONS(std::string, strings) +ONNXRUNTIME_DEFINE_GET_ATTRS_SPECIALIZATIONS(TensorProto, tensors) +ONNXRUNTIME_DEFINE_GET_ATTRS_SPECIALIZATIONS(GraphProto, graphs) + +size_t ProtoHelperNodeContext::getNumInputs() const { + return node_.InputDefs().size(); +} + +size_t ProtoHelperNodeContext::getNumOutputs() const { + return node_.OutputDefs().size(); +} + +const AttributeProto* ProtoHelperNodeContext::getAttribute(const std::string& name) const { + const onnxruntime::NodeAttributes& attributes = node_.GetAttributes(); + auto it = attributes.find(name); + if (it != attributes.end()) { + return &it->second; + } + return nullptr; +} + +const TypeProto* ProtoHelperNodeContext::getInputType(size_t index) const { + return node_.InputDefs()[index]->TypeAsProto(); +} + +const TypeProto* ProtoHelperNodeContext::getOutputType(size_t index) const { + return node_.OutputDefs()[index]->TypeAsProto(); +} + +template +uint32_t OpNodeProtoHelper::GetPrimitiveAttrElementCount(AttributeProto_AttributeType type, + const std::string& name) const noexcept { + const AttributeProto* attr = impl_->getAttribute(name); + if (attr) { + switch (type) { + case AttributeProto_AttributeType_FLOAT: + case AttributeProto_AttributeType_INT: + case AttributeProto_AttributeType_STRING: + return 1; + + case AttributeProto_AttributeType_FLOATS: + return attr->floats_size(); + case AttributeProto_AttributeType_INTS: + return attr->ints_size(); + case AttributeProto_AttributeType_STRINGS: + return attr->strings_size(); + + // The following are unsupported through this method + case AttributeProto_AttributeType_UNDEFINED: + case AttributeProto_AttributeType_TENSOR: + case AttributeProto_AttributeType_GRAPH: + case AttributeProto_AttributeType_TENSORS: + case AttributeProto_AttributeType_GRAPHS: + default: + return 0; + } + } + + return 0; +} + +template +bool OpNodeProtoHelper::HasPrimitiveAttribute(AttributeProto_AttributeType type, + const std::string& name) const noexcept { + return GetPrimitiveAttrElementCount(type, name) > 0; +} + +template class OpNodeProtoHelper; +template class OpNodeProtoHelper; + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/parallel_executor.cc b/onnxruntime/core/framework/parallel_executor.cc new file mode 100644 index 0000000000000..99eb06a15966b --- /dev/null +++ b/onnxruntime/core/framework/parallel_executor.cc @@ -0,0 +1,261 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/parallel_executor.h" + +#include +#include +#include +#include +#include "core/common/common.h" +#include "core/common/logging/logging.h" +#include "core/common/task_thread_pool.h" +#include "core/framework/allocation_planner.h" +#include "core/framework/execution_frame.h" +#include "core/framework/session_state.h" +#include "core/framework/op_kernel_context_internal.h" + +namespace onnxruntime { + +ParallelExecutor::ParallelExecutor(const SessionState& session_state, const bool& terminate_flag) + : out_standings_(0), terminate_flag_{terminate_flag} { + auto graph_viewer = session_state.GetGraphViewer(); + node_refs_.resize(graph_viewer->MaxNodeIndex()); + for (auto& node : graph_viewer->Nodes()) { + node_refs_[node.Index()] = node.GetInputEdgesCount(); + } +} + +Status ParallelExecutor::Execute(const SessionState& session_state, + const NameMLValMap& feeds, + const std::vector& output_names, + std::vector& fetches, + const logging::Logger& logger) { + auto tp = session_state.Profiler().StartTime(); + + root_frame_ = std::make_unique(feeds, output_names, fetches, session_state); + //std::cout << "start nodes:" << std::endl; + for (auto node_index : session_state.GetGraphViewer()->GetRootNodes()) { + auto p_op_kernel = session_state.GetKernel(node_index); + if (!p_op_kernel) + continue; + + //std::cout << "\t" << p_op_kernel->Node().Name() << std::endl; + EnqueueNode(node_index, session_state, logger); + } + + // Wait for finish. + { + std::unique_lock lock(complete_mutex_); + while (out_standings_.load() > 0) complete_cv_.wait(lock); + } + + VLOGS(logger, 1) << "Fetching output."; + ONNXRUNTIME_RETURN_IF_ERROR(FetchOutput(session_state.GetMLValueNameIdxMap(), *root_frame_, output_names, fetches, logger)); + + if (root_frame_->HasPlan()) { + std::vector input_shapes; + bool all_tensors = true; + for (const auto& feed : feeds) { + if (!(feed.second.IsTensor())) { + all_tensors = false; + break; + } + auto& tensor = feed.second.Get(); + input_shapes.push_back(tensor.Shape()); + } + + if (all_tensors) { + auto mem_patterns = std::make_unique(); + ONNXRUNTIME_RETURN_IF_ERROR(root_frame_->GeneratePatterns(mem_patterns.get())); + ONNXRUNTIME_RETURN_IF_ERROR(session_state.UpdateMemoryPatternGroupCache(input_shapes, std::move(mem_patterns))); + } + } + + session_state.Profiler().EndTimeAndRecordEvent(profiling::SESSION_EVENT, "ParallelExecutor::Execute", tp); + return Status::OK(); +} + +void ParallelExecutor::RunNodeAsync(size_t p_node_index, + const SessionState& session_state, + const logging::Logger& logger) { + try { + RunNodeAsyncInternal(p_node_index, session_state, logger); + } catch (...) { + FinishNodeRun(); + throw; + } +} + +void ParallelExecutor::RunNodeAsyncInternal(size_t p_node_index, + const SessionState& session_state, + const logging::Logger& logger) { + LOGS(logger, INFO) << "Begin execution"; + + size_t node_index = p_node_index; + bool keep_running = true; + auto graph_viewer = session_state.GetGraphViewer(); + // Avoid context switching if possible. + while (keep_running) { + // TODO: Convert RunNodeAsync return Status. + // to also handle exception propagation + if (terminate_flag_) { + LOGS(logger, WARNING) << "Exiting due to terminate flag being set to true."; + ONNXRUNTIME_THROW("Exiting due to terminate flag being set to true."); + } + + auto p_op_kernel = session_state.GetKernel(node_index); + + // if a kernel has been added in the session state, it better be NON-null. + if (p_op_kernel == nullptr) { + ONNXRUNTIME_THROW("Got nullptr from GetKernel for node: ", + graph_viewer->GetNode(node_index)->Name()); + } + + OpKernelContextInternal op_kernel_context(*root_frame_, *p_op_kernel, logger, + p_op_kernel->Node().ImplicitInputDefs(), + terminate_flag_); + + auto sync_time_begin = session_state.Profiler().StartTime(); + // sync before compute + int queue_id = p_op_kernel->KernelDef().ExecQueueId(); + + for (int input_index = 0; input_index < op_kernel_context.InputCount(); ++input_index) { + Fence_t fence = op_kernel_context.InputFence(input_index); + if (fence) { + fence->BeforeUsingAsInput(p_op_kernel->Node().GetExecutionProviderType(), queue_id); + } + } + + for (int input_index = 0; input_index < op_kernel_context.ImplicitInputCount(); ++input_index) { + Fence_t fence = op_kernel_context.ImplicitInputFence(input_index); + if (fence) { + fence->BeforeUsingAsInput(p_op_kernel->Node().GetExecutionProviderType(), queue_id); + } + } + + for (int output_index = 0; output_index < op_kernel_context.OutputCount(); ++output_index) { + Fence_t fence = op_kernel_context.OutputFence(output_index); + if (fence) { + fence->BeforeUsingAsOutput(p_op_kernel->Node().GetExecutionProviderType(), queue_id); + } + } + + const std::string& node_name = p_op_kernel->Node().Name(); + const std::string& op_name = p_op_kernel->KernelDef().OpName(); + + session_state.Profiler().EndTimeAndRecordEvent(profiling::NODE_EVENT, + node_name + "_fence_before", + sync_time_begin, + std::unordered_map{{"op_name", op_name}}); + + // call compute on the kernel + VLOGS(logger, 1) << "Computing kernel: " << p_op_kernel->Node().Name(); + + auto kernel_begin_time = session_state.Profiler().StartTime(); + + // Execute the kernel. + auto status = p_op_kernel->Compute(&op_kernel_context); + if (!status.IsOK()) { + ONNXRUNTIME_THROW("Compute failed for node: ", graph_viewer->GetNode(node_index)->Name()); + } + + session_state.Profiler().EndTimeAndRecordEvent(profiling::NODE_EVENT, + node_name + "_kernel_time", + kernel_begin_time, + std::unordered_map{{"op_name", op_name}}); + + sync_time_begin = session_state.Profiler().StartTime(); + // sync after compute for outputs + for (int input_index = 0; input_index < op_kernel_context.InputCount(); ++input_index) { + Fence_t fence = op_kernel_context.InputFence(input_index); + if (fence) { + fence->AfterUsedAsInput(queue_id); + } + } + + for (int input_index = 0; input_index < op_kernel_context.ImplicitInputCount(); ++input_index) { + Fence_t fence = op_kernel_context.ImplicitInputFence(input_index); + if (fence) { + fence->AfterUsedAsInput(queue_id); + } + } + + for (int output_index = 0; output_index < op_kernel_context.OutputCount(); ++output_index) { + Fence_t fence = op_kernel_context.OutputFence(output_index); + if (fence) { + fence->AfterUsedAsOutput(queue_id); + } + } + session_state.Profiler().EndTimeAndRecordEvent(profiling::NODE_EVENT, + node_name + "_fence_after", + sync_time_begin, + std::unordered_map{{"op_name", op_name}}); + + //std::cout << "Run async node finish: " << p_node_index << std::endl; + + keep_running = false; + + // Checking which output nodes ready for running. + { + auto begin = p_op_kernel->Node().OutputEdgesBegin(); + auto end = p_op_kernel->Node().OutputEdgesEnd(); + + std::lock_guard lock(ref_mutex_); + for (auto it = begin; it != end; it++) { + auto idx = (*it).GetNode().Index(); + if ((--node_refs_[idx]) == 0) { + if (!keep_running) { + node_index = idx; + keep_running = true; + } else { + EnqueueNode(idx, session_state, logger); + } + } + + //std::cout << "handle output, current name: " << p_op_kernel->Node().Name() << ", current index: " << p_node_index << ", output name: " << (*it)->GetNode().Name() << ", output index: " << (*it)->GetNode().Index() << ", after -- output ref: " << node_refs_[idx] << std::endl; + } + } + } + + FinishNodeRun(); +} + +void ParallelExecutor::EnqueueNode(size_t p_node_index, const SessionState& session_state, const logging::Logger& logger) { + out_standings_++; + //std::cout << "Enqueue async node: " << p_node_index << ", out_standings: " << out_standings_ << std::endl; + std::packaged_task task{std::bind(&ParallelExecutor::RunNodeAsync, this, p_node_index, std::cref(session_state), std::cref(logger))}; + session_state.GetThreadPool()->RunTask(std::move(task)); +} + +Status ParallelExecutor::FetchOutput(const MLValueNameIdxMap& name_idx_map, + ExecutionFrame& frame, + const std::vector& output_names, + std::vector& fetches, + const logging::Logger& logger) { + if (fetches.empty()) { + fetches.resize(output_names.size()); + } else { + // this should've been checked before already + ONNXRUNTIME_ENFORCE(output_names.size() == fetches.size(), + "output_names vector size: " + std::to_string(output_names.size()) + + " does not match that of fetches vector: " + std::to_string(fetches.size())); + } + + auto idx = 0; + + for (const auto& oname : output_names) { + VLOGS(logger, 1) << "Attempting to fetch output with name: " << oname; + int mlvalue_index; + ONNXRUNTIME_RETURN_IF_ERROR(name_idx_map.GetIdx(oname, mlvalue_index)); + const MLValue& output_mlvalue = frame.GetMLValue(mlvalue_index); + VLOGS(logger, 1) << "Copying fetched MLValue to output vector"; + fetches[idx++] = output_mlvalue; + } + + VLOGS(logger, 1) << "Done with execution."; + return Status::OK(); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/parallel_executor.h b/onnxruntime/core/framework/parallel_executor.h new file mode 100644 index 0000000000000..a8a465c7c64e6 --- /dev/null +++ b/onnxruntime/core/framework/parallel_executor.h @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include "core/common/common.h" +#include "core/common/status.h" +#include "core/common/logging/logging.h" +#include "core/framework/iexecutor.h" +#include "core/framework/framework_common.h" +#include "core/framework/ml_value.h" +#include "core/framework/session_state.h" +#include "core/graph/graph.h" + +namespace onnxruntime { + +class ExecutionFrame; + +class ParallelExecutor : public IExecutor { + public: + ParallelExecutor(const bool& terminate_flag = false) : terminate_flag_{terminate_flag} {} + ParallelExecutor(const SessionState& session_state, const bool& terminate_flag = false); + + common::Status Execute(const SessionState& session_state, + const NameMLValMap& feeds, + const std::vector& output_names, + std::vector& fetches, + const logging::Logger& logger) override; + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(ParallelExecutor); + + void RunNodeAsync(size_t p_node_index, const SessionState& session_state, const logging::Logger& logger); + void RunNodeAsyncInternal(size_t p_node_index, const SessionState& session_state, const logging::Logger& logger); + + void EnqueueNode(size_t p_node_index, const SessionState& session_state, const logging::Logger& logger); + + Status FetchOutput(const MLValueNameIdxMap& name_idx_map, + ExecutionFrame& frame, + const std::vector& output_names, + std::vector& fetches, + const logging::Logger& logger); + + void FinishNodeRun() { + if (--out_standings_ == 0) { + //std::cout << "all out standing nodes are completed." << std::endl; + complete_cv_.notify_all(); + } + } + + std::unique_ptr root_frame_; + std::vector node_refs_; + std::mutex ref_mutex_; + std::atomic out_standings_; + std::mutex complete_mutex_; + std::condition_variable complete_cv_; + + const bool& terminate_flag_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/run_options.cc b/onnxruntime/core/framework/run_options.cc new file mode 100644 index 0000000000000..812570396bab8 --- /dev/null +++ b/onnxruntime/core/framework/run_options.cc @@ -0,0 +1,55 @@ + +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "core/framework/run_options.h" +#include "core/framework/run_options_c_api.h" +#include +#include + +uint32_t ONNXRUNTIME_API_STATUSCALL ReleaseRunOptions(void* this_) { + ONNXRuntimeRunOptions* this_ptr = static_cast(this_); + if (--this_ptr->ref_count == 0) + delete this_ptr; + return 0; +} + +uint32_t ONNXRUNTIME_API_STATUSCALL AddRefRunOptions(void* this_) { + ONNXRuntimeRunOptions* this_ptr = static_cast(this_); + ++this_ptr->ref_count; + return 0; +} + +constexpr ONNXObject mkl_cls = { + AddRefRunOptions, + ReleaseRunOptions, +}; + +ONNXRuntimeRunOptions::ONNXRuntimeRunOptions() : cls(&mkl_cls), ref_count(1), terminate(false) { +} + +ONNXRUNTIME_API(ONNXRuntimeRunOptions*, ONNXRuntimeCreateRunOptions) { + std::unique_ptr options = std::make_unique(); + return options.release(); +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeRunOptionsSetRunLogVerbosityLevel, _In_ ONNXRuntimeRunOptions* options, unsigned int value) { + options->run_log_verbosity_level = value; + return nullptr; +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeRunOptionsSetRunTag, _In_ ONNXRuntimeRunOptions* options, _In_ const char* run_tag) { + if (run_tag) + options->run_tag = run_tag; + return nullptr; +} + +ONNXRUNTIME_API(unsigned int, ONNXRuntimeRunOptionsGetRunLogVerbosityLevel, _In_ ONNXRuntimeRunOptions* options) { + return options->run_log_verbosity_level; +} +ONNXRUNTIME_API(const char*, ONNXRuntimeRunOptionsGetRunTag, _In_ ONNXRuntimeRunOptions* options) { + return options->run_tag.c_str(); +} + +ONNXRUNTIME_API(void, ONNXRuntimeRunOptionsSetTerminate, _In_ ONNXRuntimeRunOptions* options, bool value) { + options->terminate = value; +} diff --git a/onnxruntime/core/framework/sequential_execution_plan.h b/onnxruntime/core/framework/sequential_execution_plan.h new file mode 100644 index 0000000000000..a3c65f33ac8a8 --- /dev/null +++ b/onnxruntime/core/framework/sequential_execution_plan.h @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/graph/basic_types.h" +#include "core/framework/alloc_kind.h" +#include "core/framework/data_types.h" + +namespace onnxruntime { +// Every ml-value has a unique name and is assigned a unique integral number. +// While we use names at static-planning time, the goal is that at runtime +// (that is, at inference time), there is no need to refer to names, and only +// the integer index is used (e.g., to index into appropriate vectors in +// the ExecutionFrame). +using MLValueIndex = int; +using MLValueName = std::string; + +class SessionState; +// SequentialExecutionPlan: This is the data that is produced by a static +// planner for a sequential execution, to be used by a SequentialExecutor. +struct SequentialExecutionPlan { + // Allocation plan: + // ExecutionFrame::GetOrCreateTensor() should use the following information + // to decide whether to allocate a new buffer or reuse an existing buffer + + // AllocPlanPerValue: (a simplified form of AllocationPlanPerValue above) + // Captures information required to allocate/reuse buffer for a ml-value + struct AllocPlanPerValue { + AllocKind alloc_kind{AllocKind::kAllocate}; + MLDataType value_type{nullptr}; + ONNXRuntimeAllocatorInfo location; + // reused_buffer is valid only if alloc_kind == kReuse. It indicates + // which MLValue's buffer must be reused for this MLValue. + MLValueIndex reused_buffer{0}; + // if the value is used in async kernel, a fence object would be created + // note the fence object would be shared between MLValues reusing the same buffer + bool create_fence_if_async{false}; + + public: + AllocPlanPerValue() : location(CPU, ONNXRuntimeArenaAllocator) {} + }; + + // The following vector is indexed by MLValueIndex + std::vector allocation_plan; + + // The following indicates the order in which nodes should be executed and the + // ml-values to be free after each node's execution: + + // NodeExecutionPlan: represents execution data for a single node + struct NodeExecutionPlan { + // node to be executed; + onnxruntime::NodeIndex node_index; + + // ml-values to be freed after node execution: + // for (auto i = free_from_index; i <= free_to_index; i++) + // free ml-value corresponding to ml-value-index to_be_freed[i] + int free_from_index; + int free_to_index; + + explicit NodeExecutionPlan(onnxruntime::NodeIndex index) : node_index(index), free_from_index(1), free_to_index(0) {} + }; + + // Execution_plan: represents the nodes in the sequential order to be executed + std::vector execution_plan; + + // to_be_freed: vector elements represent indices of ml-values to be freed (as described above) + std::vector to_be_freed; +}; + +// Output details of an execution plan: +std::ostream& operator<<(std::ostream& out, std::pair planinfo); +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/sequential_executor.cc b/onnxruntime/core/framework/sequential_executor.cc new file mode 100644 index 0000000000000..366fde05e843c --- /dev/null +++ b/onnxruntime/core/framework/sequential_executor.cc @@ -0,0 +1,207 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/sequential_executor.h" + +#include +#include +#include +#include "core/common/common.h" +#include "core/common/logging/logging.h" +#include "core/framework/allocation_planner.h" +#include "core/framework/execution_frame.h" +#include "core/framework/session_state.h" +#include "core/framework/op_kernel_context_internal.h" + +namespace onnxruntime { + +static Status FetchOutput(const MLValueNameIdxMap& name_idx_map, + ExecutionFrame& frame, + const std::vector& output_names, + std::vector& fetches, + const logging::Logger& logger); + +static Status ReleaseNodeMLValues(ExecutionFrame& frame, + const SequentialExecutionPlan& seq_exec_plan, + const SequentialExecutionPlan::NodeExecutionPlan& node_exec_plan, + const logging::Logger& logger); + +Status SequentialExecutor::Execute(const SessionState& session_state, + const NameMLValMap& feeds, + const std::vector& output_names, + std::vector& fetches, + const logging::Logger& logger) { + auto tp = session_state.Profiler().StartTime(); + + ExecutionFrame frame{feeds, output_names, fetches, session_state}; + + LOGS(logger, INFO) << "Begin execution"; + const SequentialExecutionPlan& seq_exec_plan = *session_state.GetExecutionPlan(); + const auto& exec_plan_vec = seq_exec_plan.execution_plan; + VLOGS(logger, 1) << "Size of execution plan vector: " << exec_plan_vec.size(); + + // uncomment the line below to dump execution plan + //std::cout << std::make_pair(p_seq_exec_plan, &session_state) << "\n"; + + for (const auto& node_exec_plan : exec_plan_vec) { + if (terminate_flag_) { + LOGS(logger, WARNING) << "Exiting due to terminate flag being set to true."; + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Exiting due to terminate flag being set to true."); + } + + auto node_index = node_exec_plan.node_index; + auto p_op_kernel = session_state.GetKernel(node_index); + + // if a kernel has been added in the session state, it better be NON-null. + if (p_op_kernel == nullptr) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Got nullptr from GetKernel for node: ", + session_state.GetGraphViewer()->GetNode(node_index)->Name()); + + const std::string& node_name = p_op_kernel->Node().Name(); + const std::string& op_name = p_op_kernel->KernelDef().OpName(); + // construct OpKernelContext + // TODO: log kernel inputs? + OpKernelContextInternal op_kernel_context(frame, *p_op_kernel, logger, p_op_kernel->Node().ImplicitInputDefs(), + terminate_flag_); + // TODO: log kernel outputs? + + auto sync_time_begin = session_state.Profiler().StartTime(); + // sync before compute + int queue_id = p_op_kernel->KernelDef().ExecQueueId(); + for (int input_index = 0; input_index < op_kernel_context.InputCount(); ++input_index) { + Fence_t fence = op_kernel_context.InputFence(input_index); + if (fence) { + fence->BeforeUsingAsInput(p_op_kernel->Node().GetExecutionProviderType(), queue_id); + } + } + + for (int input_index = 0; input_index < op_kernel_context.ImplicitInputCount(); ++input_index) { + Fence_t fence = op_kernel_context.ImplicitInputFence(input_index); + if (fence) { + fence->BeforeUsingAsInput(p_op_kernel->Node().GetExecutionProviderType(), queue_id); + } + } + + for (int output_index = 0; output_index < op_kernel_context.OutputCount(); ++output_index) { + Fence_t fence = op_kernel_context.OutputFence(output_index); + if (fence) { + fence->BeforeUsingAsOutput(p_op_kernel->Node().GetExecutionProviderType(), queue_id); + } + } + + session_state.Profiler().EndTimeAndRecordEvent(profiling::NODE_EVENT, + node_name + "_fence_before", + sync_time_begin, + std::unordered_map{{"op_name", op_name}}); + + // call compute on the kernel + VLOGS(logger, 1) << "Computing kernel: " << p_op_kernel->Node().Name(); + + auto kernel_begin_time = session_state.Profiler().StartTime(); + ONNXRUNTIME_RETURN_IF_ERROR(p_op_kernel->Compute(&op_kernel_context)); + session_state.Profiler().EndTimeAndRecordEvent(profiling::NODE_EVENT, + node_name + "_kernel_time", + kernel_begin_time, + std::unordered_map{{"op_name", op_name}}); + + sync_time_begin = session_state.Profiler().StartTime(); + // sync after compute for outputs + for (int input_index = 0; input_index < op_kernel_context.InputCount(); ++input_index) { + Fence_t fence = op_kernel_context.InputFence(input_index); + if (fence) { + fence->AfterUsedAsInput(queue_id); + } + } + + for (int input_index = 0; input_index < op_kernel_context.ImplicitInputCount(); ++input_index) { + Fence_t fence = op_kernel_context.ImplicitInputFence(input_index); + if (fence) { + fence->AfterUsedAsInput(queue_id); + } + } + + for (int output_index = 0; output_index < op_kernel_context.OutputCount(); ++output_index) { + Fence_t fence = op_kernel_context.OutputFence(output_index); + if (fence) { + fence->AfterUsedAsOutput(queue_id); + } + } + session_state.Profiler().EndTimeAndRecordEvent(profiling::NODE_EVENT, + node_name + "_fence_after", + sync_time_begin, + std::unordered_map{{"op_name", op_name}}); + + // free ml-values corresponding to this node + VLOGS(logger, 1) << "Releasing node ML values after computing kernel: " << p_op_kernel->Node().Name(); + ONNXRUNTIME_RETURN_IF_ERROR(ReleaseNodeMLValues(frame, seq_exec_plan, node_exec_plan, logger)); + } + + VLOGS(logger, 1) << "Fetching output."; + ONNXRUNTIME_RETURN_IF_ERROR(FetchOutput(session_state.GetMLValueNameIdxMap(), frame, output_names, fetches, logger)); + + if (frame.HasPlan()) { + std::vector input_shapes; + bool all_tensors = true; + for (const auto& feed : feeds) { + if (!(feed.second.IsTensor())) { + all_tensors = false; + break; + } + auto& tensor = feed.second.Get(); + input_shapes.push_back(tensor.Shape()); + } + + if (all_tensors) { + auto mem_patterns = std::make_unique(); + ONNXRUNTIME_RETURN_IF_ERROR(frame.GeneratePatterns(mem_patterns.get())); + ONNXRUNTIME_RETURN_IF_ERROR(session_state.UpdateMemoryPatternGroupCache(input_shapes, std::move(mem_patterns))); + } + } + + session_state.Profiler().EndTimeAndRecordEvent(profiling::SESSION_EVENT, "SequentialExecutor::Execute", tp); + return Status::OK(); +} + +static Status FetchOutput(const MLValueNameIdxMap& name_idx_map, + ExecutionFrame& frame, + const std::vector& output_names, + std::vector& fetches, + const logging::Logger& logger) { + if (fetches.empty()) { + fetches.resize(output_names.size()); + } else { + // this should've been checked before already + ONNXRUNTIME_ENFORCE(output_names.size() == fetches.size(), + "output_names vector size: " + std::to_string(output_names.size()) + + " does not match that of fetches vector: " + std::to_string(fetches.size())); + } + + auto idx = 0; + + for (const auto& oname : output_names) { + VLOGS(logger, 1) << "Attempting to fetch output with name: " << oname; + int mlvalue_index; + ONNXRUNTIME_RETURN_IF_ERROR(name_idx_map.GetIdx(oname, mlvalue_index)); + const MLValue& output_mlvalue = frame.GetMLValue(mlvalue_index); + VLOGS(logger, 1) << "Copying fetched MLValue to output vector"; + fetches[idx++] = output_mlvalue; + } + + VLOGS(logger, 1) << "Done with execution."; + return Status::OK(); +} + +static Status ReleaseNodeMLValues(ExecutionFrame& frame, + const SequentialExecutionPlan& seq_exec_plan, + const SequentialExecutionPlan::NodeExecutionPlan& node_exec_plan, + const logging::Logger& logger) { + for (auto i = node_exec_plan.free_from_index; i <= node_exec_plan.free_to_index; ++i) { + auto mlvalue_idx = seq_exec_plan.to_be_freed[i]; + VLOGS(logger, 1) << "Releasing mlvalue with index: " << mlvalue_idx; + ONNXRUNTIME_RETURN_IF_ERROR(frame.ReleaseMLValue(mlvalue_idx)); + } + + return Status::OK(); +} +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/sequential_executor.h b/onnxruntime/core/framework/sequential_executor.h new file mode 100644 index 0000000000000..1f18b36eee20d --- /dev/null +++ b/onnxruntime/core/framework/sequential_executor.h @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include "core/common/common.h" +#include "core/common/status.h" +#include "core/common/logging/logging.h" +#include "core/framework/iexecutor.h" +#include "core/framework/framework_common.h" +#include "core/framework/ml_value.h" +#include "core/framework/session_state.h" +#include "core/graph/graph.h" + +namespace onnxruntime { +class SequentialExecutor : public IExecutor { + public: + SequentialExecutor(const bool& terminate_flag = false) : terminate_flag_{terminate_flag} {} + + common::Status Execute(const SessionState& session_state, + const NameMLValMap& feeds, + const std::vector& output_names, + std::vector& fetches, + const logging::Logger& logger) override; + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(SequentialExecutor); + const bool& terminate_flag_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/session_state.cc b/onnxruntime/core/framework/session_state.cc new file mode 100644 index 0000000000000..b3621cb1e1680 --- /dev/null +++ b/onnxruntime/core/framework/session_state.cc @@ -0,0 +1,168 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/session_state.h" + +#include + +#include "core/common/logging/logging.h" +#include "core/framework/op_kernel.h" + +using namespace ::onnxruntime::common; +namespace onnxruntime { + +void SessionState::SetGraphViewer(std::unique_ptr graph_viewer) { + ONNXRUNTIME_ENFORCE(nullptr != graph_viewer); + graph_viewer_ = std::move(graph_viewer); +} + +const onnxruntime::GraphViewer* SessionState::GetGraphViewer() const { + return graph_viewer_.get(); +} + +const OpKernel* SessionState::GetKernel(onnxruntime::NodeIndex node_id) const { + if (session_kernels_.count(node_id) == 0) { + return nullptr; + } + + return session_kernels_.find(node_id)->second.get(); +} + +void SessionState::AddKernel(onnxruntime::NodeIndex node_id, std::unique_ptr p_kernel) { + // assumes vector is already resize()'ed to the number of nodes in the graph + session_kernels_[node_id] = std::move(p_kernel); +} + +void SessionState::SetExecutionPlan(std::unique_ptr p_seq_exec_plan) { + p_seq_exec_plan_ = std::move(p_seq_exec_plan); +} + +const SequentialExecutionPlan* SessionState::GetExecutionPlan() const { + return p_seq_exec_plan_.get(); +} + +void SessionState::AddInitializedTensor(int mlvalue_index, const MLValue& mlvalue) { + ONNXRUNTIME_ENFORCE(mlvalue_index >= 0 && mlvalue_index <= mlvalue_name_idx_map_.MaxIdx()); + initialized_tensors_.insert({mlvalue_index, mlvalue}); +} + +const std::unordered_map& SessionState::GetInitializedTensors() const { + return initialized_tensors_; +} + +SessionState& SessionState::SetLogger(const logging::Logger& logger) { + logger_ = &logger; + return *this; +} + +const logging::Logger& SessionState::Logger() const { + // DefaultLogger either throws or returns a valid logger. + const logging::Logger* logger = logger_ != nullptr ? logger_ : &logging::LoggingManager::DefaultLogger(); + return *logger; +} + +void SessionState::SetProfiler(profiling::Profiler& profiler) { + profiler_ = &profiler; +} + +::onnxruntime::profiling::Profiler& SessionState::Profiler() const { + return *profiler_; +} + +static int64_t CalculateMemoryPatternsKey(const std::vector& shapes) { + int64_t key = 0; + for (auto& shape : shapes) { + for (auto dim : shape.GetDims()) + key ^= dim; + } + return key; +} + +const MemoryPatternGroup* SessionState::GetMemoryPatternGroup(const std::vector& input_shapes) const { + std::lock_guard lock(mem_patterns_lock_); + int64_t key = CalculateMemoryPatternsKey(input_shapes); + auto it = mem_patterns_.find(key); + if (it == mem_patterns_.end()) + return nullptr; + + return it->second.get(); +} + +Status SessionState::UpdateMemoryPatternGroupCache(const std::vector& input_shape, + std::unique_ptr mem_patterns) const { + int64_t key = CalculateMemoryPatternsKey(input_shape); + + std::lock_guard lock(mem_patterns_lock_); + auto it = mem_patterns_.find(key); + if (it == mem_patterns_.end()) { + mem_patterns_[key] = std::move(mem_patterns); + } + + return Status::OK(); +} + +void SessionState::SetEnableMemoryPattern(bool flag) { + enable_mem_pattern_ = flag; +} + +bool SessionState::GetEnableMemoryPattern() const { + return enable_mem_pattern_; +} + +void SessionState::AddInputNameToNodeInfoMapping(const std::string& input_name, const NodeInfo& node_info) { + input_names_to_nodeinfo_mapping_[input_name].push_back(node_info); +} + +common::Status SessionState::GetInputNodeInfo(const std::string& input_name, std::vector& node_info_vec) const { + if (!input_names_to_nodeinfo_mapping_.count(input_name)) { + return Status(ONNXRUNTIME, FAIL, "Failed to find input name in the mapping"); + } + node_info_vec = input_names_to_nodeinfo_mapping_.at(input_name); + return Status::OK(); +} + +const SessionState::NameNodeInfoMapType& SessionState::GetInputNodeInfoMap() const { + return input_names_to_nodeinfo_mapping_; +} + +void SessionState::AddOutputNameToNodeInfoMapping(const std::string& output_name, const NodeInfo& node_info) { + output_names_to_nodeinfo_mapping_[output_name].push_back(node_info); +} + +const SessionState::NameNodeInfoMapType& SessionState::GetOutputNodeInfoMap() const { + return output_names_to_nodeinfo_mapping_; +} + +void SessionState::AddSubgraphSessionState(onnxruntime::NodeIndex index, + const std::string& attribute_name, + const SessionState& session_state) { + auto entry = subgraph_session_states_.find(index); + + // make sure this is new. internal logic error if it is not so using ONNXRUNTIME_ENFORCE. + if (entry != subgraph_session_states_.cend()) { + const auto& existing_entries = entry->second; + ONNXRUNTIME_ENFORCE(existing_entries.find(attribute_name) == existing_entries.cend(), + "Entry exists in node ", index, " for attribute ", attribute_name); + } + + subgraph_session_states_[index].insert({attribute_name, gsl::not_null(&session_state)}); +} + +const SessionState* SessionState::GetSubgraphSessionState(onnxruntime::NodeIndex index, + const std::string& attribute_name) const { + const SessionState* session_state = nullptr; + + auto node_entry = subgraph_session_states_.find(index); + if (node_entry != subgraph_session_states_.cend()) { + const auto& attribute_state_map{node_entry->second}; + + const auto& subgraph_entry = attribute_state_map.find(attribute_name); + if (subgraph_entry != attribute_state_map.cend()) { + session_state = subgraph_entry->second; + } + } + + return session_state; +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/session_state.h b/onnxruntime/core/framework/session_state.h new file mode 100644 index 0000000000000..0f09424fc6f18 --- /dev/null +++ b/onnxruntime/core/framework/session_state.h @@ -0,0 +1,187 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include +#include +#include "gsl/gsl_util" + +#include "core/common/common.h" +#include "core/common/logging/logging.h" +#include "core/common/profiler.h" +#include "core/framework/allocation_planner.h" +#include "core/framework/execution_providers.h" +#include "core/framework/kernel_registry_manager.h" +#include "core/framework/mem_pattern.h" +#include "core/framework/ml_value.h" +#include "core/framework/mlvalue_name_idx_map.h" +#include "core/graph/graph.h" + +namespace onnxruntime { + +class ExecutionProviders; +class KernelDef; +class OpKernel; +class TaskThreadPool; +struct SequentialExecutionPlan; +struct MemoryPatternGroup; + +// SessionState should be modified by the inference session class only. +// It is supposed to be passed by const-ref only to all the executors. +class SessionState { + public: + SessionState(const ExecutionProviders& execution_providers) + : execution_providers_{execution_providers} { + } + + // Graph viewer. + void SetGraphViewer(std::unique_ptr graph_viewer); + const onnxruntime::GraphViewer* GetGraphViewer() const; + + // kernels + // Get kernel for specified node. + // It should called right before graph execution only. + const OpKernel* GetKernel(onnxruntime::NodeIndex node_id) const; + + void AddKernel(onnxruntime::NodeIndex node_id, std::unique_ptr p_kernel); + + const ExecutionProviders& GetExecutionProviders() const noexcept { return execution_providers_; } + + const MLValueNameIdxMap& GetMLValueNameIdxMap() const noexcept { return mlvalue_name_idx_map_; } + MLValueNameIdxMap& GetMLValueNameIdxMap() noexcept { return mlvalue_name_idx_map_; } + + // initialized tensors + /** + * Adds an initialized tensor (weight) so that it can be used by the + * execution frame to setup the appropriate MLValue vectors. + */ + void AddInitializedTensor(int mlvalue_index, const MLValue& mlvalue); + + /** + * Gets the list of all initialized tensors (weights) so that it can be used by the + * execution frame to setup the appropriate MLValue vectors. + */ + const std::unordered_map& GetInitializedTensors() const; + + // execution plan + void SetExecutionPlan(std::unique_ptr p_seq_exec_plan); + const SequentialExecutionPlan* GetExecutionPlan() const; + + /** + Set the logger to use for this session. + */ + SessionState& SetLogger(const logging::Logger& logger); + + /** + Get the logger for this session. + Falls back to returning Logging::LoggingManager::DefaultLogger if SetLogger has not been called. + */ + const logging::Logger& Logger() const; + + /** + Set the profiler for this session. + */ + void SetProfiler(profiling::Profiler& profiler); + + /** + Get the profiler for this session. It needs to be enabled via the InferenceSession to perform + profiling actions. + */ + profiling::Profiler& Profiler() const; + + /** + Get cached memory pattern based on input shapes + */ + const MemoryPatternGroup* GetMemoryPatternGroup(const std::vector& input_shapes) const; + + /** + Set generated memory pattern with a given input shapes. + Const as it's an internal cache update only. + */ + Status UpdateMemoryPatternGroupCache(const std::vector& input_shape, + std::unique_ptr mem_patterns) const; + + /** + Set enable memory pattern flag + */ + void SetEnableMemoryPattern(bool flag); + + /** + Get enable memory pattern flag + */ + bool GetEnableMemoryPattern() const; + + struct NodeInfo { + NodeInfo(size_t index0, const onnxruntime::Node* p_node0, const KernelCreateInfo* kci0) + : index(index0), + p_node(p_node0), + kci(kci0) { + } + NodeInfo() = default; + + size_t index; + const onnxruntime::Node* p_node = nullptr; + const KernelCreateInfo* kci = nullptr; + }; + + using NameNodeInfoMapType = std::unordered_map>; + void AddInputNameToNodeInfoMapping(const std::string& input_name, const NodeInfo& node_info); + common::Status GetInputNodeInfo(const std::string& input_name, std::vector& node_info_vec) const; + const NameNodeInfoMapType& GetInputNodeInfoMap() const; + + void AddOutputNameToNodeInfoMapping(const std::string& output_name, const NodeInfo& node_info); + const NameNodeInfoMapType& GetOutputNodeInfoMap() const; + + /// Add a SessionState instance for executing a subgraph in a Node + /// @param index Index of Node containing subgraph + /// @param attribute_name Name of attribute containing the subgraph GraphProto + /// @param session_state SessionState for subgraph execution + void AddSubgraphSessionState(onnxruntime::NodeIndex index, const std::string& attribute_name, + const SessionState& session_state); + + /// Return SessionState for the given Node index and attribute name if found. + const SessionState* GetSubgraphSessionState(onnxruntime::NodeIndex index, const std::string& attribute_name) const; + + TaskThreadPool* GetThreadPool() const { return thread_pool_; } + void SetThreadPool(TaskThreadPool* p_pool) { thread_pool_ = p_pool; } + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(SessionState); + + // cache of the constructed kernels to avoid spending construction + // time per executor + std::unordered_map> session_kernels_; + std::unique_ptr graph_viewer_; + + const ExecutionProviders& execution_providers_; // owned by InferenceSession + MLValueNameIdxMap mlvalue_name_idx_map_; + + // initialized tensorset + std::unordered_map initialized_tensors_; // key is mlvalue_index + std::unique_ptr p_seq_exec_plan_ = nullptr; + + const logging::Logger* logger_; + profiling::Profiler* profiler_; + + // switch for enable memory pattern optimization or not. + bool enable_mem_pattern_ = true; + // lock for the mem_patterns_ + mutable std::mutex mem_patterns_lock_; + // cache for the generated mem_patterns. key is calculated based on input shapes. + mutable std::map> mem_patterns_; + + NameNodeInfoMapType input_names_to_nodeinfo_mapping_; + NameNodeInfoMapType output_names_to_nodeinfo_mapping_; + + // subgraph SessionState. entry for node containing subgraph, with value containing attribute:SessionState pair + // as a node may contain multiple subgraphs (e.g. 'If' has one for both the 'then' and 'else' branches). + using SubgraphSessionStateMap = + std::unordered_map>>; + SubgraphSessionStateMap subgraph_session_states_; + TaskThreadPool* thread_pool_ = nullptr; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/session_state_initializer.cc b/onnxruntime/core/framework/session_state_initializer.cc new file mode 100644 index 0000000000000..cbba7164c30dd --- /dev/null +++ b/onnxruntime/core/framework/session_state_initializer.cc @@ -0,0 +1,509 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/session_state_initializer.h" + +#include + +#include "core/common/common.h" +#include "core/common/logging/logging.h" + +#include "core/graph/graph.h" +#include "core/graph/graph_transformer.h" +#include "core/graph/graph_transformer_mgr.h" + +#include "core/framework/graph_partitioner.h" +#include "core/framework/insert_cast_transformer.h" +#include "core/framework/ml_value.h" +#include "core/framework/ml_value_patterns_planner.h" +#include "core/framework/mlvalue_name_idx_map.h" +#include "core/framework/sequential_execution_plan.h" +#include "core/framework/session_state.h" +#include "core/framework/tensorutils.h" +#include "core/framework/tensorprotoutils.h" +#include "core/framework/transformer_memcpy.h" +#include "core/framework/utils.h" + +namespace onnxruntime { + +static common::Status TransformGraph(onnxruntime::Graph& graph, + const onnxruntime::GraphTransformerManager& graph_transformer_mgr, + const ExecutionProviders& exec_providers, + KernelRegistryManager& kernel_registry_manager, + const InsertCastTransformer& insert_cast_transformer); + +static common::Status SaveMLValueNameIndexMapping(const onnxruntime::Graph& graph, + MLValueNameIdxMap& mlvalue_name_idx_map, + const logging::Logger& logger); + +using SaveTensorFunc = std::function; + +static common::Status SaveInitializedTensors(const onnxruntime::Graph& graph, + bool enable_memory_pattern, + const SequentialExecutionPlan& execution_plan, + const ExecutionProviders& exec_providers, + const MLValueNameIdxMap& mlvalue_name_idx_map, + std::map& weights_buffers, + const SaveTensorFunc& save_tensor_func, + const logging::Logger& logger); + +static common::Status SaveKernels(const ExecutionProviders& execution_providers, + SessionState& session_state, + const KernelRegistryManager& custom_registry_manager, + const logging::Logger& logger); + +static common::Status SaveInputOutputNamesToNodeMapping(const onnxruntime::Graph& graph, + const KernelRegistryManager& custom_registry_manager, + SessionState& session_state); + +SessionStateInitializer::SessionStateInitializer(onnxruntime::Graph& graph, + SessionState& session_state, + const ExecutionProviders& providers, + KernelRegistryManager& kernel_registry_manager, + const logging::Logger& logger) + : graph_{graph}, + session_state_{session_state}, + execution_providers_{providers}, + kernel_registry_manager_{kernel_registry_manager}, + logger_{logger} { +} + +common::Status SessionStateInitializer::CreatePlan(const onnxruntime::GraphTransformerManager& graph_transformation_manager, + const InsertCastTransformer& insert_cast_transformer, + bool enable_sequential_execution) { + ONNXRUNTIME_RETURN_IF_ERROR(TransformGraph(graph_, graph_transformation_manager, + execution_providers_, kernel_registry_manager_, + insert_cast_transformer)); + // After transformation/partitioning, the graph now is fixed and graph viewer is created and set for execution. + session_state_.SetGraphViewer(std::make_unique(graph_)); + // populate the SessionState MLValueNameIdxMap + ONNXRUNTIME_RETURN_IF_ERROR(SaveMLValueNameIndexMapping(graph_, + session_state_.GetMLValueNameIdxMap(), + logger_)); + + std::unique_ptr exec_plan; + + if (enable_sequential_execution) { + // CreatePlan will create a new SequentialExecutionPlan instance that we will + // save into the session state. + ONNXRUNTIME_RETURN_IF_ERROR(SequentialPlanner::CreatePlan(graph_, execution_providers_, kernel_registry_manager_, + session_state_.GetMLValueNameIdxMap(), exec_plan)); + + session_state_.SetExecutionPlan(std::move(exec_plan)); + } else { + // Parallel execution still uses same allocation plan, but has limitation of memory buffer reuse. + SequentialPlannerContext context(true /* enable parallel execution */); + ONNXRUNTIME_RETURN_IF_ERROR(SequentialPlanner::CreatePlan(graph_, execution_providers_, kernel_registry_manager_, + session_state_.GetMLValueNameIdxMap(), context, exec_plan)); + + session_state_.SetExecutionPlan(std::move(exec_plan)); + } + + return Status::OK(); +} + +common::Status SessionStateInitializer::InitializeAndSave(bool enable_memory_pattern, + std::map& weights_buffers) { + const auto* exec_plan_ptr = session_state_.GetExecutionPlan(); + ONNXRUNTIME_ENFORCE(exec_plan_ptr, "Execution plan was not found in SessionState. CreatePlan must be called first."); + + const auto& exec_plan{*exec_plan_ptr}; + const auto& mlvalue_name_idx_map{session_state_.GetMLValueNameIdxMap()}; + + // lambda to save initialized tensors into SessionState directly + auto add_initialized_tensor = [this](int idx, const onnxruntime::MLValue& value) { + session_state_.AddInitializedTensor(idx, value); + }; + + ONNXRUNTIME_RETURN_IF_ERROR(SaveInitializedTensors(graph_, enable_memory_pattern, exec_plan, + execution_providers_, mlvalue_name_idx_map, weights_buffers, + add_initialized_tensor, logger_)); + + graph_.CleanAllInitializedTensors(); // remove weights from the graph now to save memory + + ONNXRUNTIME_RETURN_IF_ERROR(SaveKernels(execution_providers_, session_state_, kernel_registry_manager_, logger_)); + ONNXRUNTIME_RETURN_IF_ERROR(SaveInputOutputNamesToNodeMapping(graph_, kernel_registry_manager_, session_state_)); + + return Status::OK(); +} + +common::Status TransformGraph(onnxruntime::Graph& graph, + const onnxruntime::GraphTransformerManager& graph_transformer_mgr, + const ExecutionProviders& providers, + KernelRegistryManager& kernel_registry_manager, + const InsertCastTransformer& insert_cast_transformer) { + // The transformer order: + // 1. built-in graph rewriter + // 2. each execution provider's transformer + // 3. do node placement according to kernel definition + // 4. insert copy nodes + // 5. insert cast nodes. + + // first apply the default/system/basic graph to graph optimizations. + ONNXRUNTIME_RETURN_IF_ERROR(graph_transformer_mgr.ApplyAll(graph)); + + auto kernels{kernel_registry_manager.GetAllKernelRegistries()}; + + // Do partitioning based on execution providers' capability. + GraphPartitioner partitioner(kernel_registry_manager, providers); + ONNXRUNTIME_RETURN_IF_ERROR(partitioner.Partition(graph)); + + // Insert copy nodes. + for (auto& provider : providers) { + if (provider->Type() != onnxruntime::kCpuExecutionProvider && + provider->Type() != onnxruntime::kMklDnnExecutionProvider && + provider->Type() != onnxruntime::kNupharExecutionProvider) { + TransformerMemcpyImpl copy_impl(graph, provider->Type()); + copy_impl.ModifyGraph(kernel_registry_manager); + } + } + + // Insert cast node/s. + bool modified = false; + ONNXRUNTIME_RETURN_IF_ERROR(insert_cast_transformer.Apply(graph, modified)); + + ONNXRUNTIME_RETURN_IF_ERROR(graph.Resolve()); + + return common::Status::OK(); +} + +// Build the MLValue name->idx mapping +common::Status SaveMLValueNameIndexMapping(const onnxruntime::Graph& graph, + MLValueNameIdxMap& mlvalue_name_idx_map, + const logging::Logger& logger) { + LOGS(logger, INFO) << "SaveMLValueNameIndexMapping"; + int idx = 0; + + // we keep all graph inputs (including initializers), even if they are unused, so make sure they all have an entry + for (const auto* input_def : graph.GetInputsIncludingInitializers()) { + idx = mlvalue_name_idx_map.Add(input_def->Name()); + VLOGS(logger, 1) + << "Added graph input with name: " << input_def->Name() << " to MLValueIndex with index: " << idx; + } + + for (auto& node : graph.Nodes()) { + // build the MLValue->index map + for (const auto* input_def : node.InputDefs()) { + if (input_def->Exists()) { + idx = mlvalue_name_idx_map.Add(input_def->Name()); + VLOGS(logger, 1) + << "Added input argument with name: " << input_def->Name() << " to MLValueIndex with index: " << idx; + } + } + + for (const auto* input_def : node.ImplicitInputDefs()) { + if (input_def->Exists()) { + idx = mlvalue_name_idx_map.Add(input_def->Name()); + VLOGS(logger, 1) + << "Added implicit input argument with name: " << input_def->Name() << " to MLValueIndex with index: " << idx; + } + } + + for (const auto* output_def : node.OutputDefs()) { + if (output_def->Exists()) { + mlvalue_name_idx_map.Add(output_def->Name()); + VLOGS(logger, 1) + << "Added output argument with name: " << output_def->Name() << " to MLValueIndex with index: " << idx; + } + } + } + + // allocate MLValue for graph outputs when coming from initializers + for (const auto& output : graph.GetOutputs()) { + if (output->Exists()) { + idx = mlvalue_name_idx_map.Add(output->Name()); + VLOGS(logger, 1) + << "Added graph output with name: " << output->Name() << " to MLValueIndex with index: " << idx; + } + } + + LOGS(logger, INFO) << "Done saving MLValue mappings."; + return Status::OK(); +} + +common::Status DeserializeTensorProto(const ONNX_NAMESPACE::TensorProto& tensor_proto, + const ONNXRuntimeAllocatorInfo& alloc_info, + const ExecutionProviders& exec_providers, + MLValue& mlvalue, void* preallocated, size_t preallocated_size) { + auto alloc_ptr = utils::GetAllocator(exec_providers, alloc_info); + if (!alloc_ptr) { + return Status(common::ONNXRUNTIME, common::FAIL, "Failed to get allocator for alloc_info: " + alloc_info.ToString()); + } + + if (strcmp(alloc_info.name, CPU) == 0 || alloc_info.mem_type == ONNXRuntimeMemTypeCPUOutput) { + // deserialize directly to CPU tensor + return utils::TensorProtoToMLValue(tensor_proto, alloc_ptr, preallocated, preallocated_size, mlvalue); + } + + std::unique_ptr p_tensor; + // deserialize to CPU first for non-CPU allocator, then alloc and copy + AllocatorPtr deserialize_alloc_ptr; + std::unique_ptr p_deserialize_tensor; + deserialize_alloc_ptr = exec_providers.Get(kCpuExecutionProvider)->GetAllocator(0, ONNXRuntimeMemTypeDefault); + ONNXRUNTIME_RETURN_IF_ERROR(utils::GetTensorFromTensorProto(tensor_proto, &p_deserialize_tensor, + deserialize_alloc_ptr)); + const IExecutionProvider* provider = exec_providers.Get(alloc_info); + ONNXRUNTIME_ENFORCE(provider != nullptr); + p_tensor = std::make_unique( + p_deserialize_tensor->DataType(), + p_deserialize_tensor->Shape(), + preallocated ? preallocated : static_cast(alloc_ptr->Alloc(p_deserialize_tensor->Size())), + alloc_info, + preallocated ? nullptr : alloc_ptr); // no deleter for preallocated + + Status copy_status = provider->CopyTensor(*p_deserialize_tensor, *p_tensor); + if (!copy_status.IsOK()) { + if (copy_status.ErrorMessage().empty()) { + // The windows execution provider does not return any error message today for CopyTensor since it is + // not implemented yet. That's the reason we're adding our own error message so that we can debug better. + return Status(copy_status.Category(), + copy_status.Code(), + "Failed to copy tensor to execution provider: " + provider->Type()); + } + return copy_status; + } + mlvalue.Init(p_tensor.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); + + return common::Status::OK(); +} + +static common::Status PlanTensor(MLValuePatternPlanner& planner, const MLValueNameIdxMap& mlvalue_name_idx_map, const std::string& name, const ONNX_NAMESPACE::TensorProto& tensor_proto) { + int mlvalue_index; + ONNXRUNTIME_RETURN_IF_ERROR(mlvalue_name_idx_map.GetIdx(name, mlvalue_index)); + size_t len; + Status st = utils::GetSizeInBytesFromTensorProto<256>(tensor_proto, &len); + if (st.Code() == common::NOT_IMPLEMENTED) return Status::OK(); + if (!st.IsOK()) return st; + return planner.TraceAllocation(mlvalue_index, len); +} + +common::Status SaveInitializedTensorsWithMemPattern(const Graph& graph, + const SequentialExecutionPlan& execution_plan, + const ExecutionProviders& exec_providers, + const MLValueNameIdxMap& mlvalue_name_idx_map, + std::map& weights_buffers, + const SaveTensorFunc& save_tensor_func, + const logging::Logger& logger) { + LOGS(logger, INFO) << "Saving initialized tensors."; + + ONNXRUNTIME_ENFORCE(mlvalue_name_idx_map.MaxIdx() > 0, "MLValue indexes should have been populated."); + + MLValuePatternPlanner planner(execution_plan); + + //1. first plan the memory + const onnxruntime::InitializedTensorSet& initialized_tensor_set = graph.GetAllInitializedTensors(); + for (const auto& entry : initialized_tensor_set) { + //string/complex64/complex128 tensors will be skipped + ONNXRUNTIME_RETURN_IF_ERROR(PlanTensor(planner, mlvalue_name_idx_map, entry.first, *entry.second)); + } + + //2. allocate weight buffer on different locations + MemoryPatternGroup mem_patterns; + ONNXRUNTIME_RETURN_IF_ERROR(planner.GeneratePatterns(&mem_patterns)); + for (size_t i = 0; i < mem_patterns.locations.size(); i++) { + auto& location = mem_patterns.locations[i]; + ONNXRUNTIME_ENFORCE(weights_buffers.find(location) == weights_buffers.end(), + "Existing entry in weights buffer for ", location.name); + + auto alloc = utils::GetAllocator(exec_providers, location); + if (!alloc) + return Status(common::ONNXRUNTIME, common::FAIL, "Failed to get allocator for location: " + location.ToString()); + + void* buffer = mem_patterns.patterns[i].PeakSize() > 0 ? alloc->Alloc(mem_patterns.patterns[i].PeakSize()) + : nullptr; + weights_buffers[location] = BufferUniquePtr(buffer, alloc); + } + + //3. create weight tensors based on weights buffer + for (const auto& entry : initialized_tensor_set) { + const std::string& name = entry.first; + int mlvalue_index; + ONNXRUNTIME_RETURN_IF_ERROR(mlvalue_name_idx_map.GetIdx(name, mlvalue_index)); + const ONNX_NAMESPACE::TensorProto& tensor_proto = *(entry.second); + + auto& location = execution_plan.allocation_plan[mlvalue_index].location; + auto it = weights_buffers.find(location); + if (it == weights_buffers.end()) + return Status(common::ONNXRUNTIME, common::FAIL, "Weight buffer not found"); + + auto pattern = mem_patterns.GetPatterns(location); + if (pattern == nullptr) + return Status(common::ONNXRUNTIME, common::FAIL, "mem pattern not found"); + auto block = pattern->GetBlock(mlvalue_index); + MLValue mlvalue; + // if block is not found, means this mlvalue is not traced + // fall back to allocate separate buffer. + + // if it->second.get() is null, then fall back to the block not found case + if (it->second == nullptr) { + block = nullptr; + } + Status st; + if (!block) { + st = DeserializeTensorProto(tensor_proto, location, exec_providers, mlvalue, nullptr, 0); + } else { + st = DeserializeTensorProto(tensor_proto, location, exec_providers, mlvalue, + (uint8_t*)it->second.get() + block->offset_, block->size_); + } + if (!st.IsOK()) { + std::ostringstream oss; + oss << "Deserialize tensor " << name << " failed." << st.ErrorMessage(); + return Status(st.Category(), st.Code(), oss.str()); + } + + save_tensor_func(mlvalue_index, mlvalue); + + VLOGS(logger, 1) << "Added weight with name : " << name << " with index: " << mlvalue_index; + } + + LOGS(logger, INFO) << "Done saving initialized tensors"; + return common::Status::OK(); +} + +common::Status SaveInitializedTensorsWithSeperateBuffer(const onnxruntime::Graph& graph, + const SequentialExecutionPlan& execution_plan, + const ExecutionProviders& exec_providers, + const MLValueNameIdxMap& mlvalue_name_idx_map, + const SaveTensorFunc& save_tensor_func, + const logging::Logger& logger) { + LOGS(logger, INFO) << "Saving initialized tensors."; + + ONNXRUNTIME_ENFORCE(mlvalue_name_idx_map.MaxIdx() > 0, "MLValue indexes should have been populated."); + + const onnxruntime::InitializedTensorSet& initialized_tensor_set = graph.GetAllInitializedTensors(); + for (const auto& entry : initialized_tensor_set) { + const std::string& name = entry.first; + int mlvalue_index; + ONNXRUNTIME_RETURN_IF_ERROR(mlvalue_name_idx_map.GetIdx(name, mlvalue_index)); + VLOGS(logger, 1) << "About to add weight with name: " << name << " and index: " << mlvalue_index; + auto& location = execution_plan.allocation_plan[mlvalue_index].location; + MLValue mlvalue; + ONNXRUNTIME_RETURN_IF_ERROR(DeserializeTensorProto(*(entry.second), location, exec_providers, mlvalue, nullptr, 0)); + save_tensor_func(mlvalue_index, mlvalue); + VLOGS(logger, 1) << "Added weight with name : " << name << " with index: " << mlvalue_index; + } + + LOGS(logger, INFO) << "Done saving initialized tensors"; + return common::Status::OK(); +} + +common::Status SaveInitializedTensors(const onnxruntime::Graph& graph, + bool enable_memory_pattern, + const SequentialExecutionPlan& execution_plan, + const ExecutionProviders& exec_providers, + const MLValueNameIdxMap& mlvalue_name_idx_map, + std::map& weights_buffers, + const SaveTensorFunc& save_tensor_func, + const logging::Logger& logger) { + // if we enable the memory pattern and already have the execution plan + // go with mem pattern approach, which will allocate a big chunk for all + // the weights. + if (enable_memory_pattern) { + return SaveInitializedTensorsWithMemPattern(graph, execution_plan, exec_providers, + mlvalue_name_idx_map, weights_buffers, save_tensor_func, logger); + } + return SaveInitializedTensorsWithSeperateBuffer(graph, execution_plan, exec_providers, + mlvalue_name_idx_map, save_tensor_func, logger); +} + +static common::Status CreateOpKernelInternal(const onnxruntime::Node& node, + const IExecutionProvider& exec_provider, + const SessionState& session_state, + const KernelRegistryManager& custom_registry_manager, + std::unique_ptr& op_kernel) { + return custom_registry_manager.CreateKernel(node, exec_provider, session_state, op_kernel); +} + +static common::Status CreateOpKernel(const onnxruntime::Node& node, + const ExecutionProviders& execution_providers, + const SessionState& session_state, + const KernelRegistryManager& custom_registry_manager, + std::unique_ptr& op_kernel, + const logging::Logger& logger) { + onnxruntime::ProviderType exec_provider_name = node.GetExecutionProviderType(); + + const IExecutionProvider* exec_provider = nullptr; + if (exec_provider_name.empty() || (exec_provider = execution_providers.Get(exec_provider_name)) == nullptr) { + auto status = ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Could not create kernel for node: ", node.Name(), + " as there's no execution provider allocated."); + LOGS(logger, ERROR) << status.ErrorMessage(); + } + + common::Status status = CreateOpKernelInternal(node, *exec_provider, session_state, custom_registry_manager, + op_kernel); + + if (!status.IsOK()) { + LOGS(logger, ERROR) << "Kernel creation failed for node: " + << node.Name() << " with error: " << status.ErrorMessage(); + } + + return status; +} + +common::Status SaveKernels(const ExecutionProviders& execution_providers, + SessionState& session_state, + const KernelRegistryManager& custom_registry_manager, + const logging::Logger& logger) { + LOGS(logger, INFO) << "Saving kernels."; + + for (auto& node : session_state.GetGraphViewer()->Nodes()) { + // construct and save the kernels + std::unique_ptr op_kernel; + ONNXRUNTIME_RETURN_IF_ERROR(CreateOpKernel(node, execution_providers, session_state, custom_registry_manager, op_kernel, logger)); + session_state.AddKernel(node.Index(), std::move(op_kernel)); + } + + LOGS(logger, INFO) << "Done saving kernels."; + + return Status::OK(); +} + +static bool IsArgNameInInputsOutputs(const std::string& name, + const std::vector& graph_args) { + auto it = std::find_if(std::begin(graph_args), std::end(graph_args), [&name](const onnxruntime::NodeArg* arg) { + return arg->Name() == name; + }); + return it != graph_args.end(); +} + +common::Status SaveInputOutputNamesToNodeMapping(const onnxruntime::Graph& graph, + const KernelRegistryManager& custom_registry_manager, + SessionState& session_state) { + auto& weights_map = graph.GetAllInitializedTensors(); + auto& graph_inputs = graph.GetInputs(); + auto& graph_outputs = graph.GetOutputs(); + + for (auto& node : graph.Nodes()) { + ONNXRUNTIME_RETURN_IF_ERROR( + onnxruntime::Node::ForEachWithIndex( + node.InputDefs(), + [&](const onnxruntime::NodeArg& arg, size_t index) { + if (arg.Name().empty() || weights_map.count(arg.Name())) { + return Status::OK(); + } + + // note that KernelCreateInfo may not exist for custom kernel + const KernelCreateInfo* kci = nullptr; + custom_registry_manager.SearchKernelRegistry(node, &kci); + + SessionState::NodeInfo node_info(index, &node, kci); + + if (IsArgNameInInputsOutputs(arg.Name(), graph_inputs)) { + session_state.AddInputNameToNodeInfoMapping(arg.Name(), node_info); + return Status::OK(); + } + + if (IsArgNameInInputsOutputs(arg.Name(), graph_outputs)) { + session_state.AddOutputNameToNodeInfoMapping(arg.Name(), node_info); + return Status::OK(); + } + + return Status::OK(); + })); + } + + return Status::OK(); +} +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/session_state_initializer.h b/onnxruntime/core/framework/session_state_initializer.h new file mode 100644 index 0000000000000..6643ab5812255 --- /dev/null +++ b/onnxruntime/core/framework/session_state_initializer.h @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include + +#include "core/framework/allocator.h" +#include "core/framework/tensor.h" + +namespace onnxruntime { +class Graph; +class GraphTransformerManager; +} // namespace onnxruntime + +namespace onnxruntime { +class SessionState; +class ExecutionProviders; +class KernelRegistryManager; +class InsertCastTransformer; + +namespace logging { +class Logger; +} + +class SessionStateInitializer { + public: + SessionStateInitializer(onnxruntime::Graph& graph, + SessionState& session_state, + const ExecutionProviders& providers, + KernelRegistryManager& kernel_registry_manager, + const logging::Logger& logger); + + // First perform any transformations and create the execution plan + common::Status CreatePlan(const onnxruntime::GraphTransformerManager& graph_transformation_manager, + const InsertCastTransformer& insert_cast_transformer, + bool enable_sequential_execution); + + // initialize tensors, and save. save kernels and input/output node mappings + // @param enable_memory_pattern + common::Status InitializeAndSave(bool enable_memory_pattern, + std::map& weights_buffers); + + private: + onnxruntime::Graph& graph_; + SessionState& session_state_; + + const ExecutionProviders& execution_providers_; + KernelRegistryManager& kernel_registry_manager_; + const logging::Logger& logger_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/tensor.cc b/onnxruntime/core/framework/tensor.cc new file mode 100644 index 0000000000000..7334303e407df --- /dev/null +++ b/onnxruntime/core/framework/tensor.cc @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/tensor.h" + +#include +#include "core/framework/allocatormgr.h" +using namespace std; +namespace onnxruntime { + +Tensor::Tensor(MLDataType p_type, + const TensorShape& shape, + BufferNakedPtr p_data, + const ONNXRuntimeAllocatorInfo& alloc, + AllocatorPtr deleter, + const int64_t offset) + : alloc_info_(alloc) { + ONNXRUNTIME_ENFORCE(p_type != nullptr); + Init(p_type, shape, p_data, alloc, std::move(deleter), offset); +} + +void Tensor::Init(MLDataType p_type, + const TensorShape& shape, + void* p_raw_data, + const ONNXRuntimeAllocatorInfo& alloc, + AllocatorPtr deleter, + const int64_t offset) { + if (shape.Size() < 0) + throw std::runtime_error("shape.Size() must >=0"); + dtype_ = p_type; + shape_ = shape; + p_data_ = p_raw_data; + // if caller passed in a deleter, that means this tensor own this buffer + // we will release the buffer when this tensor is deconstructed. + buffer_deleter_ = std::move(deleter); + // for string tensors, if this tensor own the buffer (caller passed in the deleter) + // do the placement new for strings on pre-allocated buffer. + if (buffer_deleter_ && dtype_ == DataTypeImpl::GetType()) { + auto* ptr = static_cast(p_data_); + for (int64_t i = 0, n = shape.Size(); i < n; ++i) { + new (ptr + i) string(); + } + } + alloc_info_ = alloc; + byte_offset_ = offset; +} + +Tensor::Tensor(Tensor&& other) + : p_data_(other.p_data_), + buffer_deleter_(other.buffer_deleter_), + shape_(other.shape_), + dtype_(other.dtype_), + alloc_info_(other.alloc_info_), + byte_offset_(other.byte_offset_) { + other.dtype_ = DataTypeImpl::GetType(); + other.shape_ = TensorShape(vector(1, 0)); + other.p_data_ = nullptr; + other.buffer_deleter_ = nullptr; + other.byte_offset_ = 0; +} + +Tensor& Tensor::operator=(Tensor&& other) { + if (this != &other) { + ReleaseBuffer(); + + dtype_ = other.dtype_; + shape_ = other.shape_; + alloc_info_ = other.alloc_info_; + byte_offset_ = other.byte_offset_; + p_data_ = other.p_data_; + buffer_deleter_ = other.buffer_deleter_; + + other.dtype_ = DataTypeImpl::GetType(); + other.shape_ = TensorShape(vector(1, 0)); + other.p_data_ = nullptr; + other.byte_offset_ = 0; + other.buffer_deleter_ = nullptr; + } + return *this; +} + +Tensor::Tensor(const Tensor& src) + : shape_(src.shape_), dtype_(src.dtype_), alloc_info_(src.alloc_info_), byte_offset_(src.byte_offset_) { + // it may be better to refactor it a little bit to make it a compile error + // but right now just keep it simple first. + ONNXRUNTIME_ENFORCE(src.buffer_deleter_ == nullptr, + "Can't copy tensor with its owned buffer. Please transfer ownership by move."); + + p_data_ = src.p_data_; + buffer_deleter_ = nullptr; +} + +Tensor::~Tensor() { + ReleaseBuffer(); +} + +Tensor& Tensor::ShallowCopy(const Tensor& other) { + // similar as above + ONNXRUNTIME_ENFORCE(other.buffer_deleter_ == nullptr, + "Can't copy tensor with its owned buffer. Please transfer ownership by move."); + + if (this != &other) { + dtype_ = other.dtype_; + alloc_info_ = other.alloc_info_; + shape_ = other.shape_; + byte_offset_ = other.byte_offset_; + p_data_ = other.p_data_; + buffer_deleter_ = nullptr; + } + return *this; +} + +void Tensor::ReleaseBuffer() { + if (buffer_deleter_) { + // if current tensor is responsible for delete the buffer + // and it is a string tensor, need to explict call string's + // deconstructor. + if (dtype_ == DataTypeImpl::GetType()) { + auto* ptr = static_cast(p_data_); + int64_t len = shape_.Size(); + for (int64_t i = 0; i < len; i++) + ptr[i].~string(); + } + buffer_deleter_->Free(p_data_); + } +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/tensor_shape.cc b/onnxruntime/core/framework/tensor_shape.cc new file mode 100644 index 0000000000000..8db0005d7a208 --- /dev/null +++ b/onnxruntime/core/framework/tensor_shape.cc @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/tensor_shape.h" +#include +#include "core/common/common.h" +#include "core/graph/onnx_protobuf.h" + +namespace onnxruntime { + +TensorShape::TensorShape(const std::vector& dims) : std::vector(dims) { +} + +TensorShape::TensorShape(const std::initializer_list& dims) : std::vector(dims) { +} + +TensorShape::TensorShape(const int64_t* dimension_sizes, size_t dimension_count) : std::vector(dimension_count) { + for (size_t i = 0; i < dimension_count; ++i) { + (*this)[i] = dimension_sizes[i]; + } +} + + +TensorShape::TensorShape(const std::vector& dims, size_t start, size_t end) { + assign(dims.begin() + start, dims.begin() + end); +} + +/** + * Return the total number of elements. Returns 1 for an empty (rank 0) TensorShape. + */ +int64_t TensorShape::Size() const { + size_t arraySize = size(); + int64_t size = SizeHelper(0, arraySize); + //should we cache the size? as multiple operation may be expensive. + return size; +} + +int64_t TensorShape::SizeToDimension(size_t dimension) const { + const size_t num_dims = size(); + ONNXRUNTIME_ENFORCE(dimension <= num_dims, + "Invalid dimension of ", dimension, " for SizeFromDimension. Tensor has ", + num_dims, " dimensions."); + + int64_t size = SizeHelper(0, dimension); + return size; +} + +int64_t TensorShape::SizeFromDimension(size_t dimension) const { + const size_t num_dims = size(); + ONNXRUNTIME_ENFORCE(dimension <= num_dims, + "Invalid dimension of ", dimension, " for SizeFromDimension. Tensor has ", + num_dims, " dimensions."); + + int64_t size = SizeHelper(dimension, num_dims); + return size; +} + +TensorShape TensorShape::Slice(size_t dimstart, size_t dimend) const { + ONNXRUNTIME_ENFORCE(dimstart <= dimend && dimend <= size(), + "Invalid tensor shape slice argument."); + return TensorShape(*this, dimstart, dimend); +} + +TensorShape TensorShape::Slice(size_t dimstart) const { + return Slice(dimstart, size()); +} + +// output dimensions +std::string TensorShape::ToString() const { + std::string result; + + result.append("{"); + bool first = true; + for (auto dim : (*this)) { + if (!first) { + result.append(","); + } + + result.append(std::to_string(dim)); + first = false; + } + result.append("}"); + + return result; +} + +int64_t TensorShape::SizeHelper(size_t start, size_t end) const { + // Must return 1 for an empty sequence + int64_t size = 1; + for (size_t i = start; i < end; i++) { + if ((*this)[i] < 0) return -1; + size *= (*this)[i]; + } + return size; +} + +// operator<< to nicely output to a stream +std::ostream& operator<<(std::ostream& out, const ::onnxruntime::TensorShape& shape) { + return (out << shape.ToString()); +} + +std::ostream& operator<<(std::ostream& out, const ONNX_NAMESPACE::TensorShapeProto& shape_proto) { + std::string result; + result.reserve(128); + + result.append("{"); + bool first = true; + for (auto& dim : shape_proto.dim()) { + if (!first) { + result.append(","); + } + + if (dim.has_dim_value()) + result.append(std::to_string(dim.dim_value())); + else if (dim.has_dim_param()) + result.append(dim.dim_param()); + + first = false; + } + result.append("}"); + + return (out << result); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/tensorprotoutils.cc b/onnxruntime/core/framework/tensorprotoutils.cc new file mode 100644 index 0000000000000..e425c590383e3 --- /dev/null +++ b/onnxruntime/core/framework/tensorprotoutils.cc @@ -0,0 +1,216 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/tensorprotoutils.h" + +#include +#include "core/graph/onnx_protobuf.h" +#include "core/common/logging/logging.h" +#include "core/framework/op_kernel.h" +#include "core/framework/tensorutils.h" +#include "core/framework/tensor.h" +#include "core/framework/ml_value_patterns_planner.h" + +using namespace ONNX_NAMESPACE; +using namespace ::onnxruntime::common; + +namespace onnxruntime { +namespace utils { +std::vector GetTensorShapeFromTensorProto(const ONNX_NAMESPACE::TensorProto& tensor_proto) { + const auto& dims = tensor_proto.dims(); + std::vector tensor_shape_vec(dims.size()); + for (int i = 0; i < dims.size(); ++i) { + tensor_shape_vec[i] = dims[i]; + } + + return tensor_shape_vec; +} + +std::vector GetTensorShapeFromTensorShapeProto(const ONNX_NAMESPACE::TensorShapeProto& tensor_shape_proto) { + const auto& dims = tensor_shape_proto.dim(); + std::vector tensor_shape_vec(dims.size()); + for (int i = 0; i < dims.size(); ++i) { + tensor_shape_vec[i] = dims[i].has_dim_param() + ? -1 /* symbolic dimensions are represented as -1 in onnxruntime*/ + : dims[i].dim_value(); + } + return tensor_shape_vec; +} + +template +common::Status GetTensorByTypeFromTensorProto(const TensorProto& tensor_proto, + const TensorShape& tensor_shape, + std::unique_ptr* p_tensor, + AllocatorPtr alloc, + void* preallocated, + size_t preallocated_size) { + int64_t tensor_size = tensor_shape.Size(); + //tensor_size could be zero. see test_slice_start_out_of_bounds\test_data_set_0\output_0.pb + if (tensor_size < 0) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid shape ", tensor_shape); + } + size_t size_to_allocate; + if (!IAllocator::CalcMemSizeForArrayWithAlignment<256>(static_cast(tensor_size), sizeof(T), &size_to_allocate)) { + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "size overflow"); + } + + if (preallocated && preallocated_size != size_to_allocate) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "The buffer planner is not consistent with tensor buffer size, expected ", size_to_allocate, ", got ", preallocated_size); + //TODO(): size_to_allocate could be zero. We shouldn't pass zero to alloc->Alloc() + T* p_data = static_cast(preallocated ? preallocated : alloc->Alloc(size_to_allocate)); + ONNXRUNTIME_RETURN_IF_ERROR(::onnxruntime::utils::TensorUtils::UnpackTensor(tensor_proto, p_data, tensor_size)); + *p_tensor = std::make_unique(DataTypeImpl::GetType(), + tensor_shape, + static_cast(p_data), + alloc->Info(), + preallocated ? nullptr : alloc); // no deleter for preallocated + + return common::Status::OK(); +} + +template <> +common::Status GetTensorByTypeFromTensorProto(const TensorProto& tensor_proto, + const TensorShape& tensor_shape, + std::unique_ptr* p_tensor, + AllocatorPtr alloc, + void* preallocated, + size_t preallocated_size) { + int64_t tensor_size = tensor_shape.Size(); + if (tensor_size < 0) { + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Tensor shape cannot contain any negative value"); + } + size_t size_to_allocate; + if (!IAllocator::CalcMemSizeForArrayWithAlignment<256>(static_cast(tensor_size), sizeof(std::string), &size_to_allocate)) { + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "size overflow"); + } + + if (preallocated && preallocated_size != size_to_allocate) + return Status(ONNXRUNTIME, FAIL, "The buffer planner is not consistent with tensor buffer size"); + + std::string* p_data = static_cast(preallocated ? preallocated : alloc->Alloc(size_to_allocate)); + *p_tensor = std::make_unique(DataTypeImpl::GetType(), + tensor_shape, + static_cast(p_data), + alloc->Info(), + preallocated ? nullptr : alloc); // no deleter for preallocated + + /* + In the case of string tensors, the strings need to be constructed in the pre-allocated memory (placement + new) before calling Unpack (which copies the strings from the proto). Placement new happens inside the + Tensor's constructor. Hence the order of invocation of Tensor construction and Unpack needs to be reversed + in comparison to other types. This has the disadvantage of alloc/deallocing a Tensor if Unpack fails; + however restricting it to string types only alleviates this concern for other types at least. Hence the template + specialization for string. + */ + ONNXRUNTIME_RETURN_IF_ERROR(::onnxruntime::utils::TensorUtils::UnpackTensor(tensor_proto, p_data, tensor_size)); + + return common::Status::OK(); +} + +template <> +common::Status GetTensorByTypeFromTensorProto(const TensorProto& tensor_proto, + const TensorShape& tensor_shape, + std::unique_ptr* p_tensor, + AllocatorPtr alloc, + void* preallocated, + size_t preallocated_size) { + int64_t tensor_size = tensor_shape.Size(); + if (tensor_size < 0) { + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Tensor shape cannot contain any negative value"); + } + static_assert(sizeof(MLFloat16) == sizeof(uint16_t), "MLFloat16 must has 16 bit size"); + size_t size_to_allocate; + if (!IAllocator::CalcMemSizeForArrayWithAlignment<256>(static_cast(tensor_size), sizeof(MLFloat16), &size_to_allocate)) { + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "size overflow"); + } + + if (preallocated && preallocated_size != size_to_allocate) + return Status(ONNXRUNTIME, FAIL, "The buffer planner is not consistent with tensor buffer size"); + + MLFloat16* p_data = static_cast(preallocated ? preallocated : alloc->Alloc(size_to_allocate)); + ONNXRUNTIME_RETURN_IF_ERROR(::onnxruntime::utils::TensorUtils::UnpackTensor(tensor_proto, p_data, tensor_size)); + *p_tensor = std::make_unique(DataTypeImpl::GetType(), + tensor_shape, + static_cast(p_data), + alloc->Info(), + preallocated ? nullptr : alloc); // no deleter for preallocated + + return common::Status::OK(); +} + +Status TensorProtoToMLValue(const ONNX_NAMESPACE::TensorProto& input, AllocatorPtr allocator, void* preallocated, + size_t preallocated_size, MLValue& value) { + std::unique_ptr p_tensor; + ONNXRUNTIME_RETURN_IF_ERROR(GetTensorFromTensorProto(input, &p_tensor, allocator, preallocated, preallocated_size)); + value.Init(p_tensor.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); + return Status::OK(); +} + +#define CASE_PROTO(X, Y) \ + case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_##X: \ + return GetTensorByTypeFromTensorProto(tensor_proto, tensor_shape, p_tensor, allocator, preallocated, preallocated_size); + +common::Status GetTensorFromTensorProto(const TensorProto& tensor_proto, + std::unique_ptr* p_tensor, + AllocatorPtr allocator, + void* preallocated, + size_t preallocated_size) { + std::vector tensor_shape_vec = GetTensorShapeFromTensorProto(tensor_proto); + // Note: We permit an empty tensor_shape_vec, and treat it as a scalar (a tensor of size 1). + TensorShape tensor_shape{tensor_shape_vec}; + switch (tensor_proto.data_type()) { + CASE_PROTO(FLOAT, float); + CASE_PROTO(DOUBLE, double); + CASE_PROTO(BOOL, bool); + CASE_PROTO(INT8, int8_t); + CASE_PROTO(INT16, int16_t); + CASE_PROTO(INT32, int32_t); + CASE_PROTO(INT64, int64_t); + CASE_PROTO(UINT8, uint8_t); + CASE_PROTO(UINT16, uint16_t); + CASE_PROTO(UINT32, uint32_t); + CASE_PROTO(UINT64, uint64_t); + CASE_PROTO(STRING, std::string); + CASE_PROTO(FLOAT16, MLFloat16); + default: { + std::ostringstream ostr; + ostr << "Initialized tensor with unexpected type: " << tensor_proto.data_type(); + return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, ostr.str()); + } + } +} + +TensorProto::DataType GetTensorProtoType(const Tensor& tensor) { + auto tensor_type = tensor.DataType(); + TensorProto::DataType dtype = TensorProto_DataType_UNDEFINED; + + if (tensor_type == DataTypeImpl::GetType()) + dtype = TensorProto_DataType_FLOAT; + else if (tensor_type == DataTypeImpl::GetType()) + dtype = TensorProto_DataType_DOUBLE; + else if (tensor_type == DataTypeImpl::GetType()) + dtype = TensorProto_DataType_INT8; + else if (tensor_type == DataTypeImpl::GetType()) + dtype = TensorProto_DataType_INT16; + else if (tensor_type == DataTypeImpl::GetType()) + dtype = TensorProto_DataType_INT32; + else if (tensor_type == DataTypeImpl::GetType()) + dtype = TensorProto_DataType_INT64; + else if (tensor_type == DataTypeImpl::GetType()) + dtype = TensorProto_DataType_UINT8; + else if (tensor_type == DataTypeImpl::GetType()) + dtype = TensorProto_DataType_UINT16; + else if (tensor_type == DataTypeImpl::GetType()) + dtype = TensorProto_DataType_UINT32; + else if (tensor_type == DataTypeImpl::GetType()) + dtype = TensorProto_DataType_UINT64; + else if (tensor_type == DataTypeImpl::GetType()) + dtype = TensorProto_DataType_BOOL; + + return dtype; +} + +} // namespace utils +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/tensorprotoutils.h b/onnxruntime/core/framework/tensorprotoutils.h new file mode 100644 index 0000000000000..703d4cfaf5d61 --- /dev/null +++ b/onnxruntime/core/framework/tensorprotoutils.h @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include "core/common/common.h" +#include "core/common/status.h" +#include "core/framework/allocator.h" +#include "core/framework/ml_value.h" +#include "core/graph/onnx_protobuf.h" + +namespace ONNX_NAMESPACE { +class TensorProto; +class TensorShapeProto; +} // namespace ONNX_NAMESPACE + +namespace onnxruntime { +class Tensor; +namespace utils { +common::Status GetTensorFromTensorProto(const ONNX_NAMESPACE::TensorProto& tensor_proto, std::unique_ptr* p_tensor, AllocatorPtr allocator, void* preallocated = nullptr, size_t preallocated_size = 0); +std::vector GetTensorShapeFromTensorProto(const ONNX_NAMESPACE::TensorProto& tensor_proto); +std::vector GetTensorShapeFromTensorShapeProto(const ONNX_NAMESPACE::TensorShapeProto& tensor_shape_proto); +common::Status TensorProtoToMLValue(const ONNX_NAMESPACE::TensorProto& input, AllocatorPtr allocator, void* preallocated, + size_t preallocated_size, MLValue& value); +ONNX_NAMESPACE::TensorProto::DataType GetTensorProtoType(const Tensor& tensor); +} // namespace utils +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/tensorutils.cc b/onnxruntime/core/framework/tensorutils.cc new file mode 100644 index 0000000000000..d104a5d6f9e2b --- /dev/null +++ b/onnxruntime/core/framework/tensorutils.cc @@ -0,0 +1,234 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/tensorutils.h" +#include "core/framework/allocator.h" +#include "core/framework/data_types.h" + +#include +#include +#include "core/graph/onnx_protobuf.h" + +#include "gsl/pointers" + +GSL_SUPPRESS(type .1) // allow use of reinterpret_cast for this special case +inline bool IsLittleEndianOrder() noexcept { + static int n = 1; + return (*reinterpret_cast(&n) == 1); +} + +template +static void UnpackTensorWithRawData(const ONNX_NAMESPACE::TensorProto& tensor, /*out*/ T* p_data) { + // allow this low level routine to be somewhat unsafe. assuming it's thoroughly tested and valid + GSL_SUPPRESS(type) // type.1 reinterpret-cast; type.4 C-style casts; type.5 'T result;' is uninitialized; + GSL_SUPPRESS(bounds .1) // pointer arithmetic + GSL_SUPPRESS(f .23) // buff and temp_bytes never tested for nullness and could be gsl::not_null + { + auto& raw_data = tensor.raw_data(); + auto buff = raw_data.c_str(); + const size_t type_size = sizeof(T); + + if (IsLittleEndianOrder()) { + memcpy((void*)p_data, (void*)buff, raw_data.size() * sizeof(char)); + } else { + for (size_t i = 0; i < raw_data.size(); i += type_size, buff += type_size) { + T result; + const char* temp_bytes = reinterpret_cast(&result); + for (size_t j = 0; j < type_size; ++j) { + memcpy((void*)&temp_bytes[j], (void*)&buff[type_size - 1 - i], sizeof(char)); + } + p_data[i] = result; + } + } + } +} + +namespace onnxruntime { +namespace utils { +#define DEFINE_UNPACK_TENSOR(T, Type, field_name, field_size) \ + template <> \ + Status TensorUtils::UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, /*out*/ T* p_data, int64_t expected_size) { \ + if (nullptr == p_data) { \ + const size_t size = tensor.has_raw_data() ? tensor.raw_data().size() : tensor.field_size(); \ + if (size == 0) \ + return Status::OK(); \ + else \ + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT); \ + } \ + if (nullptr == p_data || Type != tensor.data_type()) { \ + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT); \ + } \ + if (tensor.has_raw_data()) { \ + size_t expected_size_in_bytes; \ + if (!IAllocator::CalcMemSizeForArray(expected_size, sizeof(T), &expected_size_in_bytes)) { \ + return Status(common::ONNXRUNTIME, common::FAIL, "size overflow"); \ + } \ + if (tensor.raw_data().size() != expected_size_in_bytes) \ + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, \ + "UnpackTensor: the pre-allocated size does not match the raw data size, expected ", \ + expected_size_in_bytes, ", got ", tensor.raw_data().size()); \ + UnpackTensorWithRawData(tensor, p_data); \ + return Status::OK(); \ + } \ + if (tensor.field_size() != expected_size) \ + return Status(common::ONNXRUNTIME, common::FAIL, \ + "UnpackTensor: the pre-allocated size does not match the size in proto"); \ + auto& data = tensor.field_name(); \ + for (auto data_iter = data.cbegin(); data_iter != data.cend(); ++data_iter) \ + *p_data++ = *reinterpret_cast(data_iter); \ + return Status::OK(); \ + } + +//TODO: uint32 uint64 complex64 complex128 +//TODO: int16_t/uint16_t/float16 is confusing right now +DEFINE_UNPACK_TENSOR(float, ONNX_NAMESPACE::TensorProto_DataType_FLOAT, float_data, float_data_size) +DEFINE_UNPACK_TENSOR(double, ONNX_NAMESPACE::TensorProto_DataType_DOUBLE, double_data, double_data_size); +DEFINE_UNPACK_TENSOR(uint8_t, ONNX_NAMESPACE::TensorProto_DataType_UINT8, int32_data, int32_data_size) +DEFINE_UNPACK_TENSOR(int8_t, ONNX_NAMESPACE::TensorProto_DataType_INT8, int32_data, int32_data_size) +DEFINE_UNPACK_TENSOR(int16_t, ONNX_NAMESPACE::TensorProto_DataType_INT16, int32_data, int32_data_size) +DEFINE_UNPACK_TENSOR(uint16_t, ONNX_NAMESPACE::TensorProto_DataType_UINT16, int32_data, int32_data_size) +DEFINE_UNPACK_TENSOR(int32_t, ONNX_NAMESPACE::TensorProto_DataType_INT32, int32_data, int32_data_size) +DEFINE_UNPACK_TENSOR(int64_t, ONNX_NAMESPACE::TensorProto_DataType_INT64, int64_data, int64_data_size) +DEFINE_UNPACK_TENSOR(uint64_t, ONNX_NAMESPACE::TensorProto_DataType_UINT64, uint64_data, uint64_data_size) +DEFINE_UNPACK_TENSOR(uint32_t, ONNX_NAMESPACE::TensorProto_DataType_UINT32, uint64_data, uint64_data_size) + +template <> +Status TensorUtils::UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, + /*out*/ std::string* p_data, + int64_t expected_size) { + if (nullptr == p_data) { + if (tensor.string_data_size() == 0) + return Status::OK(); + else + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT); + } + if (ONNX_NAMESPACE::TensorProto_DataType_STRING != tensor.data_type()) { + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT); + } + + if (tensor.string_data_size() != expected_size) + return Status(common::ONNXRUNTIME, common::FAIL, + "UnpackTensor: the pre-allocate size does not match the size in proto"); + + auto& string_data = tensor.string_data(); + for (auto iter = string_data.cbegin(); iter != string_data.cend(); ++iter) { + *p_data++ = *iter; + } + + return Status::OK(); +} +template <> +Status TensorUtils::UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, + /*out*/ bool* p_data, + int64_t expected_size) { + if (nullptr == p_data) { + const size_t size = tensor.has_raw_data() ? tensor.raw_data().size() : tensor.int32_data_size(); + if (size == 0) + return Status::OK(); + else + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT); + } + if (ONNX_NAMESPACE::TensorProto_DataType_BOOL != tensor.data_type()) { + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT); + } + + if (tensor.has_raw_data()) { + if (tensor.raw_data().size() != (expected_size) * sizeof(bool)) + return Status(common::ONNXRUNTIME, common::FAIL, + "UnpackTensor: the pre-allocate size does not match the raw data size"); + + UnpackTensorWithRawData(tensor, p_data); + return Status::OK(); + } + + if (tensor.int32_data_size() != expected_size) + return Status(common::ONNXRUNTIME, common::FAIL, + "UnpackTensor: the pre-allocate size does not match the size in proto"); + for (auto iter = tensor.int32_data().cbegin(); iter != tensor.int32_data().cend(); ++iter) { + *p_data++ = *iter; + } + + return Status::OK(); +} +template <> +Status TensorUtils::UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, + /*out*/ MLFloat16* p_data, + int64_t expected_size) { + if (nullptr == p_data) { + const size_t size = tensor.has_raw_data() ? tensor.raw_data().size() : tensor.int32_data_size(); + if (size == 0) + return Status::OK(); + else + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT); + } + if (ONNX_NAMESPACE::TensorProto_DataType_FLOAT16 != tensor.data_type()) { + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT); + } + + if (tensor.has_raw_data()) { + if (tensor.raw_data().size() != (expected_size) * sizeof(uint16_t)) + return Status(common::ONNXRUNTIME, common::FAIL, + "UnpackTensor: the pre-allocate size does not match the raw data size"); + + UnpackTensorWithRawData(tensor, p_data); + return Status::OK(); + } + + if (tensor.int32_data_size() != expected_size) + return Status(common::ONNXRUNTIME, common::FAIL, + "UnpackTensor: the pre-allocate size does not match the size in proto"); + + const int max_value = std::numeric_limits::max(); + for (int i = 0; i < static_cast(expected_size); i++) { + int v = tensor.int32_data()[i]; + if (v < 0 || v > max_value) { + return Status(common::ONNXRUNTIME, common::FAIL, "data overflow"); + } + p_data[i] = MLFloat16(static_cast(v)); + } + + return Status::OK(); +} + +#define CASE_PROTO_TRACE(X, Y) \ + case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_##X: \ + if (!IAllocator::CalcMemSizeForArrayWithAlignment(size, sizeof(Y), out)) { \ + return common::Status(common::ONNXRUNTIME, common::FAIL, "Invalid TensorProto"); \ + } \ + break; + +template +common::Status GetSizeInBytesFromTensorProto(const ONNX_NAMESPACE::TensorProto& tensor_proto, size_t* out) { + const auto& dims = tensor_proto.dims(); + size_t size = 1; + for (int i = 0; i < dims.size(); ++i) { + if (dims[i] < 0) { + return common::Status(common::ONNXRUNTIME, common::FAIL, "Invalid TensorProto"); + } + if (!IAllocator::CalcMemSizeForArray(size, static_cast(dims[i]), &size)) { + return common::Status(common::ONNXRUNTIME, common::FAIL, "Invalid TensorProto"); + } + } + switch (tensor_proto.data_type()) { + CASE_PROTO_TRACE(FLOAT, float); + CASE_PROTO_TRACE(DOUBLE, double); + CASE_PROTO_TRACE(BOOL, bool); + CASE_PROTO_TRACE(INT8, int8_t); + CASE_PROTO_TRACE(INT16, int16_t); + CASE_PROTO_TRACE(INT32, int32_t); + CASE_PROTO_TRACE(INT64, int64_t); + CASE_PROTO_TRACE(UINT8, uint8_t); + CASE_PROTO_TRACE(UINT16, uint16_t); + CASE_PROTO_TRACE(UINT32, uint32_t); + CASE_PROTO_TRACE(UINT64, uint64_t); + CASE_PROTO_TRACE(FLOAT16, MLFloat16); + case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_STRING: + default: + return common::Status(common::ONNXRUNTIME, common::NOT_IMPLEMENTED); + } + return Status::OK(); +} + +template common::Status GetSizeInBytesFromTensorProto<256>(const ONNX_NAMESPACE::TensorProto& tensor_proto, size_t* out); +} // namespace utils +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/tensorutils.h b/onnxruntime/core/framework/tensorutils.h new file mode 100644 index 0000000000000..a38e3c28256f8 --- /dev/null +++ b/onnxruntime/core/framework/tensorutils.h @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include + +#include "core/common/common.h" +#include "core/common/status.h" + +namespace ONNX_NAMESPACE { +class TensorProto; +} +namespace onnxruntime { +namespace utils { +//How much memory it will need for putting the content of this tensor into a plain array +//string/complex64/complex128 tensors are not supported. +//The output value could be zero or -1. +template +common::Status GetSizeInBytesFromTensorProto(const ONNX_NAMESPACE::TensorProto& tensor_proto, size_t* out); +class TensorUtils { + public: + template + static Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, + /*out*/ T* p_data, + int64_t expected_size); + +}; // namespace Utils +} // namespace utils +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/transformer_memcpy.cc b/onnxruntime/core/framework/transformer_memcpy.cc new file mode 100644 index 0000000000000..6f50082c794f7 --- /dev/null +++ b/onnxruntime/core/framework/transformer_memcpy.cc @@ -0,0 +1,172 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "transformer_memcpy.h" +#include "core/framework/kernel_registry_manager.h" + +using namespace ONNX_NAMESPACE; +namespace onnxruntime { + +/* + +Overview: The transformer transforms the input graph as follows: + +(1) For every initializer W that is referenced by both provider and non-provider nodes, +we create a duplicate initializer W2 and change all provider nodes to reference this +duplicate copy. + +(2) For every ml-value X that is computed by a provider node and referenced by a +non-provider node, we introduce a new ml-value X2. We replace all references to X +in provider nodes by X2, and introduce a copy from X2 to X. (All graph outputs +are considered as non-provider references here.) + +(3 For every ml-value X that is computed by a non-provider node and referenced by +a provider node, we introduce a new ml-value X2. We replace all references to X in +provider nodes by X2, and introduce a copy from X to X2. (All graph inputs are +considered to be non-provider here.) + +Note that every ml-value is computed at a unique point (either provider or non-provider), +but it may be referenced and used at multiple points (by both provider and non-provider). + +This transformer does not currently optimize copies between, e.g., two different GPU devices, etc. + +*/ + +bool TransformerMemcpyImpl::ModifyGraph(const KernelRegistryManager& kernel_registries) { + bool modified = false; + // find defs that require copy + for (auto& node : graph_.Nodes()){ + //don't need to do node placement here now, onnxruntime will do it according to registered kernels. + ProcessDefs(node, kernel_registries); + } + + // for initializers shared by different providers, create dups + ProcessInitializers(); + + for (auto arg : non_provider_output_defs_) + if (provider_input_defs_.count(arg)) { + AddCopyNode(arg, true); + modified = true; + } + + for (auto arg : provider_output_defs_) + if (non_provider_input_defs_.count(arg)) { + AddCopyNode(arg, false); + modified = true; + } + + for (auto p_node : provider_nodes_) { + p_node->ReplaceDefs(replacements_); + } + + return modified; +} + +void TransformerMemcpyImpl::ProcessDefs(onnxruntime::Node& node, const KernelRegistryManager& kernel_registries) { + if (node.GetExecutionProviderType() == provider_) { + provider_nodes_.insert(&node); + // note KernelCreateInfo might be nullptr for custom kernel + const KernelCreateInfo* kci = nullptr; + kernel_registries.SearchKernelRegistry(node, &kci); + const auto* input_mem_types = kci ? &kci->kernel_def->InputMemoryType() : nullptr; + const auto* output_mem_types = kci ? &kci->kernel_def->InputMemoryType() : nullptr; + ONNXRUNTIME_ENFORCE(onnxruntime::Node::ForEachWithIndex( + node.InputDefs(), + [this, &input_mem_types](const onnxruntime::NodeArg& arg, size_t index) { + if (input_mem_types && MemTypeOnCpuExplicitly(*input_mem_types, index)) + non_provider_input_defs_.insert(&arg); + else + provider_input_defs_.insert(&arg); + return Status::OK(); + }) + .IsOK()); + ONNXRUNTIME_ENFORCE(onnxruntime::Node::ForEachWithIndex( + node.OutputDefs(), + [this, &output_mem_types](const onnxruntime::NodeArg& arg, size_t index) { + if (output_mem_types && MemTypeOnCpuExplicitly(*output_mem_types, index)) + non_provider_output_defs_.insert(&arg); + else + provider_output_defs_.insert(&arg); + return Status::OK(); + }) + .IsOK()); + } else { + // TODO: copy between devices? i.e. multiple GPUs + if (node.GetExecutionProviderType() != onnxruntime::kCpuExecutionProvider && !node.GetExecutionProviderType().empty()) { + ONNXRUNTIME_THROW("Execution type '", node.GetExecutionProviderType(), "' doesn't support memcpy "); + } + node.ForEachDef([this](const onnxruntime::NodeArg* arg, bool is_input) { + if (is_input) + non_provider_input_defs_.insert(arg); + else + non_provider_output_defs_.insert(arg); + }); + } +} + +void TransformerMemcpyImpl::AddCopyNode(const onnxruntime::NodeArg* arg, bool is_input) { + // TODO: eliminate the const-cast. + // The current graph API seems to only allow us to get a "const NodeArg*", but + // then we need to pass in a "NodeArg*" when we create a new node. + auto original_arg = const_cast(arg); + + // create unique name for new def + std::string new_def_name = graph_.GenerateNodeArgName(original_arg->Name() + "_" + provider_); + + auto* new_arg = &graph_.GetOrCreateNodeArg(new_def_name, original_arg->TypeAsProto()); + auto* src_arg = is_input ? original_arg : new_arg; + auto* dst_arg = is_input ? new_arg : original_arg; + + // create unique name for copy node + std::string new_node_name = graph_.GenerateNodeName("Memcpy"); + + const auto op_name = is_input ? "MemcpyFromHost" : "MemcpyToHost"; + auto new_node = graph_.AddNode(new_node_name, op_name, "Copy from/to host memory", + std::vector{src_arg}, + std::vector{dst_arg}); + new_node->SetExecutionProviderType(provider_); + + // only add copy-node here; renaming references happens later + replacements_.insert(std::make_pair(original_arg, new_arg)); +} + +template +static const onnxruntime::NodeArg* FindNodeArg(const NodeArgSetType& def_set, const std::string& name) { + NodeArg def(name, nullptr); + auto it = def_set.find(&def); // this works since we use name to compare NodeArg + if (it != def_set.end()) + return *it; + return nullptr; +} + +// We duplicate any initializer that is used by both provider nodes and non-provider nodes +// to ensure that provider nodes and non-provider nodes don't share initializers, as they +// need to stay in different memory locations. +void TransformerMemcpyImpl::ProcessInitializers() { + std::map replacements; + for (const auto& pair : graph_.GetAllInitializedTensors()) { + const auto& name = pair.first; + const onnxruntime::NodeArg* provider_def = FindNodeArg(provider_input_defs_, name); + const onnxruntime::NodeArg* non_provider_def = FindNodeArg(non_provider_input_defs_, name); + if (provider_def != nullptr && non_provider_def != nullptr) { + std::string new_def_name = graph_.GenerateNodeArgName(name); + auto& new_def = graph_.GetOrCreateNodeArg(new_def_name, provider_def->TypeAsProto()); + + const TensorProto* tensor_proto = nullptr; + bool found = graph_.GetInitializedTensor(name, tensor_proto); + ONNXRUNTIME_ENFORCE(found, "Failed to get initialized tensor ", name); + + TensorProto new_tensor_proto = *tensor_proto; + *(new_tensor_proto.mutable_name()) = new_def_name; + graph_.AddInitializedTensor(new_tensor_proto); + + replacements.insert(std::make_pair(provider_def, &new_def)); + } + } + + for (auto p_node : provider_nodes_) { + p_node->ReplaceDefs(replacements); + } +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/transformer_memcpy.h b/onnxruntime/core/framework/transformer_memcpy.h new file mode 100644 index 0000000000000..c2fbb7ba26e39 --- /dev/null +++ b/onnxruntime/core/framework/transformer_memcpy.h @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/framework/kernel_registry_manager.h" + +namespace onnxruntime { + +// implements MemCpy node insertion in graph transform +// note that GraphTransformer::Apply() is supposed to be stateless, so this cannot derive from GraphTranformer +class TransformerMemcpyImpl { + public: + TransformerMemcpyImpl(onnxruntime::Graph& graph, const std::string& provider) + : graph_(graph), provider_(provider) {} + + bool ModifyGraph(const KernelRegistryManager& schema_registries); + + private: + void ProcessDefs(onnxruntime::Node& node, const KernelRegistryManager& kernel_registries); + void AddCopyNode(const onnxruntime::NodeArg* arg, bool is_input); + void ProcessInitializers(); + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(TransformerMemcpyImpl); + + // use value-based compare to make sure transformer output order is consistent + struct NodeCompare { + bool operator()(const onnxruntime::Node* lhs, const onnxruntime::Node* rhs) const { + return lhs->Index() < rhs->Index(); + } + }; + + // use value-based compare to make sure transformer output order is consistent + struct NodeArgCompare { + bool operator()(const onnxruntime::NodeArg* lhs, const onnxruntime::NodeArg* rhs) const { + return lhs->Name() < rhs->Name(); + } + }; + + std::set provider_nodes_; + std::set non_provider_input_defs_; // all input defs of non-provider nodes + std::set non_provider_output_defs_; // all output defs of non-provider nodes + std::set provider_input_defs_; // all input defs of provider nodes that should be in provider allocator + std::set provider_output_defs_; // all output defs of provider nodes that should be in provider allocator + std::map replacements_; + onnxruntime::Graph& graph_; + std::string provider_; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/utils.cc b/onnxruntime/core/framework/utils.cc new file mode 100644 index 0000000000000..914f44c9f8026 --- /dev/null +++ b/onnxruntime/core/framework/utils.cc @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/utils.h" + +#include "core/graph/graph.h" + +#include "core/framework/execution_providers.h" +#include "core/framework/kernel_def_builder.h" +#include "core/framework/kernel_registry_manager.h" +#include "core/framework/op_kernel.h" +#include "core/framework/session_state.h" + +namespace onnxruntime { +namespace utils { + +const KernelDef* GetKernelDef(const KernelRegistryManager& kernel_registry, + const onnxruntime::Node& node) { + const KernelCreateInfo* kernel_create_info = nullptr; + const KernelDef* kernel_def = nullptr; + + if (kernel_registry.SearchKernelRegistry(node, &kernel_create_info).IsOK()) { + kernel_def = kernel_create_info->kernel_def.get(); + } + + return kernel_def; +} + +AllocatorPtr GetAllocator(const ExecutionProviders& exec_providers, const ONNXRuntimeAllocatorInfo& allocator_info) { + auto exec_provider = exec_providers.Get(allocator_info); + if (exec_provider == nullptr) { + return nullptr; + } + + return exec_provider->GetAllocator(allocator_info.id, allocator_info.mem_type); +} + +AllocatorPtr GetAllocator(const SessionState& session_state, const ONNXRuntimeAllocatorInfo& allocator_info) { + return GetAllocator(session_state.GetExecutionProviders(), allocator_info); +} + +} // namespace utils +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/utils.h b/onnxruntime/core/framework/utils.h new file mode 100644 index 0000000000000..62c462943240c --- /dev/null +++ b/onnxruntime/core/framework/utils.h @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/graph/basic_types.h" +#include "core/framework/allocator.h" + +namespace onnxruntime { +class Node; +class Graph; +} // namespace onnxruntime + +namespace onnxruntime { +class ExecutionProviders; +class KernelDef; +class KernelRegistryManager; +class SessionState; + +namespace logging { +class Logger; +} + +namespace utils { +const KernelDef* GetKernelDef(const KernelRegistryManager& kernel_registry, + const onnxruntime::Node& node); + +const KernelDef* GetKernelDef(const onnxruntime::Graph& graph, + const KernelRegistryManager& kernel_registry, + const onnxruntime::NodeIndex node_id); + +AllocatorPtr GetAllocator(const ExecutionProviders& exec_providers, const ONNXRuntimeAllocatorInfo& allocator_info); + +AllocatorPtr GetAllocator(const SessionState& session_state, + const ONNXRuntimeAllocatorInfo& allocator_info); +} // namespace utils +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/conv_add_fusion.cc b/onnxruntime/core/graph/conv_add_fusion.cc new file mode 100644 index 0000000000000..5885b77ed9806 --- /dev/null +++ b/onnxruntime/core/graph/conv_add_fusion.cc @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/graph/initializer.h" +#include "core/graph/conv_add_fusion.h" + +using namespace onnx; +using namespace ::onnxruntime::common; +namespace onnxruntime { + +Status ConvAddFusion::Apply(onnxruntime::Graph& graph, bool& modified) const { + std::vector removed_nodes; + for (auto& node : graph.Nodes()) { + if (node.OpType() != "Conv" || node.GetOutputEdgesCount() != 1) { + continue; + } + + const Node* next_node = *node.OutputNodesBegin(); + if (next_node->OpType() != "Add" || + next_node->GetInputEdgesCount() != 1 || + graph.IsNodeOutputsInGraphOutputs(*next_node)) { + continue; + } + + auto& conv_node = node; + const Node* add_node = next_node; + + const auto& conv_inputs = conv_node.InputDefs(); + const auto& add_inputs = add_node->InputDefs(); + + const ONNX_NAMESPACE::TensorProto* conv_W_tensor_proto = nullptr; + graph.GetInitializedTensor(conv_inputs[1]->Name(), conv_W_tensor_proto); + + const ONNX_NAMESPACE::TensorProto* add_B_tensor_proto = nullptr; + graph.GetInitializedTensor(add_inputs[1]->Name(), add_B_tensor_proto); + + // Currently, fusion is only supported for float or double data type. + if (!Initializer::IsSupportedDataType(add_B_tensor_proto) || + conv_W_tensor_proto->dims_size() != 4 || + add_B_tensor_proto->dims_size() != 3 || + conv_W_tensor_proto->dims(0) != add_B_tensor_proto->dims(0)) { + continue; + } + + const ONNX_NAMESPACE::TensorProto* conv_B_tensor_proto = nullptr; + if (conv_inputs.size() == 3) { + graph.GetInitializedTensor(conv_inputs[2]->Name(), conv_B_tensor_proto); + + if (!Initializer::IsSupportedDataType(conv_B_tensor_proto) || + conv_B_tensor_proto->data_type() != add_B_tensor_proto->data_type() || + conv_B_tensor_proto->dims_size() != 1 || + add_B_tensor_proto->dims_size() != 3 || + conv_B_tensor_proto->dims(0) != add_B_tensor_proto->dims(0)) { + continue; + } + + auto conv_B = std::make_unique(conv_B_tensor_proto); + auto add_B = std::make_unique(add_B_tensor_proto); + + // Caculate new value of initializers of conv node + conv_B->add(*add_B); + + // Create new initializers of conv + ONNX_NAMESPACE::TensorProto new_conv_B_tensor_proto; + conv_B->ToProto(&new_conv_B_tensor_proto); + + // Replace initializers of conv node + graph.RemoveInitializedTensor(conv_inputs[2]->Name()); + graph.AddInitializedTensor(new_conv_B_tensor_proto); + } else { + NodeArg* add_B_node_arg = graph.GetNodeArg(add_B_tensor_proto->name()); + if (add_B_node_arg == nullptr) { + continue; + } + + // Update shape of tensor proto + ONNX_NAMESPACE::TensorProto new_conv_B_tensor_proto(*add_B_tensor_proto); + int64_t dim = conv_W_tensor_proto->dims(0); + new_conv_B_tensor_proto.clear_dims(); + new_conv_B_tensor_proto.add_dims(dim); + + graph.RemoveInitializedTensor(add_B_tensor_proto->name()); + graph.AddInitializedTensor(new_conv_B_tensor_proto); + + // Update shape of NodeArg + TensorShapeProto shape; + shape.add_dim()->set_dim_value(dim); + add_B_node_arg->SetShape(shape); + + conv_node.MutableInputDefs().push_back(add_B_node_arg); + conv_node.MutableInputArgsCount()[2] = 1; + } + + // Replace the input of the node following add node + const NodeArg* add_output_def = add_node->OutputDefs()[0]; + const NodeArg* conv_output_def = conv_node.OutputDefs()[0]; + for (auto it = add_node->OutputNodesBegin(); it != add_node->OutputNodesEnd(); ++it) { + auto output_node = graph.GetNode((*it)->Index()); + if (!output_node) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT); + } + + auto& input_defs = output_node->MutableInputDefs(); + for (auto& def : input_defs) { + if (def == add_output_def) { + def = const_cast(conv_output_def); + } + } + } + + removed_nodes.push_back(add_node->Index()); + } + + for (auto i : removed_nodes) { + graph.RemoveNode(i); + } + + if (!removed_nodes.empty()) { + modified = true; + ONNXRUNTIME_RETURN_IF_ERROR(graph.Resolve()); + } + + return Status::OK(); +} // namespace onnxruntime +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/core/graph/conv_add_fusion.h b/onnxruntime/core/graph/conv_add_fusion.h new file mode 100644 index 0000000000000..94a1c3889dd72 --- /dev/null +++ b/onnxruntime/core/graph/conv_add_fusion.h @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/graph/graph_transformer.h" + +namespace onnxruntime { + +class ConvAddFusion : public onnxruntime::GraphTransformer { +public: + ConvAddFusion() noexcept : onnxruntime::GraphTransformer("ConvAddFusion", "Fusing Add into Conv") {} + Status Apply(onnxruntime::Graph& graph, bool& modified) const override; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/conv_bn_fusion.cc b/onnxruntime/core/graph/conv_bn_fusion.cc new file mode 100644 index 0000000000000..69b73feffb4fa --- /dev/null +++ b/onnxruntime/core/graph/conv_bn_fusion.cc @@ -0,0 +1,177 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/graph/initializer.h" +#include "core/graph/conv_bn_fusion.h" + +using namespace onnx; +using namespace ::onnxruntime::common; +namespace onnxruntime { + +Status ConvBNFusion::Apply(onnxruntime::Graph& graph, bool& modified) const { + std::vector removed_nodes; + for (auto& node : graph.Nodes()) { + if (node.OpType() != "Conv" || node.GetOutputEdgesCount() != 1) { + continue; + } + + const Node* next_node = *node.OutputNodesBegin(); + if (next_node->OpType() != "BatchNormalization" || + next_node->GetInputEdgesCount() != 1 || + graph.IsNodeOutputsInGraphOutputs(*next_node)) { + continue; + } + + auto& conv_node = node; + const Node* bn_node = next_node; + + // Get value of attribute group + const onnxruntime::NodeAttributes& conv_attributes = conv_node.GetAttributes(); + const onnx::AttributeProto* group_attr = &(conv_attributes.find("group")->second); + if (group_attr != nullptr && + group_attr->type() == AttributeProto_AttributeType_INT && + group_attr->has_i() && group_attr->i() != 1) { + continue; + } + + // Get value of attribute epsilon + const onnxruntime::NodeAttributes& attributes = bn_node->GetAttributes(); + const onnx::AttributeProto* attr = &(attributes.find("epsilon")->second); + if (attr == nullptr || attr->type() != AttributeProto_AttributeType_FLOAT) { + continue; + } + float epsilon = static_cast(attr->f()); + + // Get initializers of BatchNormalization + const auto& bn_inputs = bn_node->InputDefs(); + const ONNX_NAMESPACE::TensorProto* bn_scale_tensor_proto = nullptr; + graph.GetInitializedTensor(bn_inputs[1]->Name(), bn_scale_tensor_proto); + + const ONNX_NAMESPACE::TensorProto* bn_B_tensor_proto = nullptr; + graph.GetInitializedTensor(bn_inputs[2]->Name(), bn_B_tensor_proto); + + const ONNX_NAMESPACE::TensorProto* bn_mean_tensor_proto = nullptr; + graph.GetInitializedTensor(bn_inputs[3]->Name(), bn_mean_tensor_proto); + + const ONNX_NAMESPACE::TensorProto* bn_var_tensor_proto = nullptr; + graph.GetInitializedTensor(bn_inputs[4]->Name(), bn_var_tensor_proto); + + const auto& conv_inputs = conv_node.InputDefs(); + const ONNX_NAMESPACE::TensorProto* conv_W_tensor_proto = nullptr; + graph.GetInitializedTensor(conv_inputs[1]->Name(), conv_W_tensor_proto); + + // Currently, fusion is only supported for float or double data type. + if (!Initializer::IsSupportedDataType(bn_scale_tensor_proto) || + !Initializer::IsSupportedDataType(bn_B_tensor_proto) || + !Initializer::IsSupportedDataType(bn_mean_tensor_proto) || + !Initializer::IsSupportedDataType(bn_var_tensor_proto) || + !Initializer::IsSupportedDataType(conv_W_tensor_proto) || + bn_scale_tensor_proto->dims_size() != 1 || + bn_B_tensor_proto->dims_size() != 1 || + bn_mean_tensor_proto->dims_size() != 1 || + bn_var_tensor_proto->dims_size() != 1 || + bn_scale_tensor_proto->dims(0) != bn_B_tensor_proto->dims(0) || + bn_B_tensor_proto->dims(0) != bn_mean_tensor_proto->dims(0) || + bn_mean_tensor_proto->dims(0) != bn_var_tensor_proto->dims(0) || + bn_scale_tensor_proto->data_type() != bn_B_tensor_proto->data_type() || + bn_B_tensor_proto->data_type() != bn_mean_tensor_proto->data_type() || + bn_mean_tensor_proto->data_type() != bn_var_tensor_proto->data_type() || + conv_W_tensor_proto->data_type() != bn_scale_tensor_proto->data_type() || + !(conv_W_tensor_proto->dims_size() > 2 && conv_W_tensor_proto->dims(0) == bn_scale_tensor_proto->dims(0))) { + continue; + } + + auto bn_scale = std::make_unique(bn_scale_tensor_proto); + auto bn_B = std::make_unique(bn_B_tensor_proto); + auto bn_mean = std::make_unique(bn_mean_tensor_proto); + auto bn_var = std::make_unique(bn_var_tensor_proto); + auto conv_W = std::make_unique(conv_W_tensor_proto); + + const ONNX_NAMESPACE::TensorProto* conv_B_tensor_proto = nullptr; + std::unique_ptr conv_B = nullptr; + if (conv_inputs.size() == 3) { + graph.GetInitializedTensor(conv_inputs[2]->Name(), conv_B_tensor_proto); + if (!Initializer::IsSupportedDataType(conv_B_tensor_proto) || + conv_B_tensor_proto->dims_size() != 1 || + conv_B_tensor_proto->dims(0) != bn_B_tensor_proto->dims(0) || + conv_B_tensor_proto->data_type() != bn_B_tensor_proto->data_type()) { + continue; + } + conv_B = std::make_unique(conv_B_tensor_proto); + } + + // Caculate new value of initializers of conv node + bn_var->add(epsilon); + bn_var->sqrt(); + bn_scale->div(*bn_var); + conv_W->scale_by_axis(*bn_scale, 1); + + if (conv_inputs.size() == 3) { + conv_B->sub(*bn_mean); + conv_B->mul(*bn_scale); + conv_B->add(*bn_B); + } else { + bn_mean->mul(*bn_scale); + bn_B->sub(*bn_mean); + } + + // Create new initializers of conv + ONNX_NAMESPACE::TensorProto new_conv_W_tensor_proto(*conv_W_tensor_proto); + conv_W->ToProto(&new_conv_W_tensor_proto); + + ONNX_NAMESPACE::TensorProto new_conv_B_tensor_proto; + NodeArg* bn_B_node_arg = nullptr; + if (conv_inputs.size() == 3) { + conv_B->ToProto(&new_conv_B_tensor_proto); + } else { + bn_B->ToProto(&new_conv_B_tensor_proto); + bn_B_node_arg = graph.GetNodeArg(bn_B_tensor_proto->name()); + if (bn_B_node_arg == nullptr) { + continue; + } + } + + // Replace initializers of conv node + graph.RemoveInitializedTensor(conv_W_tensor_proto->name()); + if (conv_inputs.size() == 3) { + graph.RemoveInitializedTensor(conv_B_tensor_proto->name()); + } else { + graph.RemoveInitializedTensor(bn_B_tensor_proto->name()); + conv_node.MutableInputDefs().push_back(bn_B_node_arg); + conv_node.MutableInputArgsCount()[2] = 1; + } + graph.AddInitializedTensor(new_conv_W_tensor_proto); + graph.AddInitializedTensor(new_conv_B_tensor_proto); + + // Replace the input of the nodes following batch normalization node + const NodeArg* bn_output_def = bn_node->OutputDefs()[0]; + const NodeArg* conv_output_def = conv_node.OutputDefs()[0]; + for (auto it = bn_node->OutputNodesBegin(); it != bn_node->OutputNodesEnd(); ++it) { + auto output_node = graph.GetNode((*it)->Index()); + if (!output_node) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT); + } + + auto& input_defs = output_node->MutableInputDefs(); + for (auto& def : input_defs) { + if (def == bn_output_def) { + def = const_cast(conv_output_def); + } + } + } + + removed_nodes.push_back(bn_node->Index()); + } + + for (auto i : removed_nodes) { + graph.RemoveNode(i); + } + + if (!removed_nodes.empty()) { + modified = true; + ONNXRUNTIME_RETURN_IF_ERROR(graph.Resolve()); + } + return Status::OK(); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/conv_bn_fusion.h b/onnxruntime/core/graph/conv_bn_fusion.h new file mode 100644 index 0000000000000..f4e145e8f01f1 --- /dev/null +++ b/onnxruntime/core/graph/conv_bn_fusion.h @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/graph/graph_transformer.h" + +namespace onnxruntime { + +class ConvBNFusion : public onnxruntime::GraphTransformer { + public: + ConvBNFusion() noexcept : onnxruntime::GraphTransformer("ConvBNFusion", "Fusing BN into Conv") {} + Status Apply(onnxruntime::Graph& graph, bool& modified) const override; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/conv_mul_fusion.cc b/onnxruntime/core/graph/conv_mul_fusion.cc new file mode 100644 index 0000000000000..65f7cccca0029 --- /dev/null +++ b/onnxruntime/core/graph/conv_mul_fusion.cc @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/graph/initializer.h" +#include "core/graph/conv_mul_fusion.h" + +using namespace onnx; +using namespace ::onnxruntime::common; +namespace onnxruntime { + +Status ConvMulFusion::Apply(onnxruntime::Graph& graph, bool& modified) const { + std::vector removed_nodes; + for (auto& node : graph.Nodes()) { + if (node.OpType() != "Conv" || node.GetOutputEdgesCount() != 1) { + continue; + } + + const Node* next_node = *node.OutputNodesBegin(); + if (next_node->OpType() != "Mul" || + next_node->GetInputEdgesCount() != 1 || + graph.IsNodeOutputsInGraphOutputs(*next_node)) { + continue; + } + + auto& conv_node = node; + const Node* mul_node = next_node; + + const auto& conv_inputs = conv_node.InputDefs(); + const auto& mul_inputs = mul_node->InputDefs(); + + const ONNX_NAMESPACE::TensorProto* conv_W_tensor_proto = nullptr; + graph.GetInitializedTensor(conv_inputs[1]->Name(), conv_W_tensor_proto); + + const ONNX_NAMESPACE::TensorProto* mul_B_tensor_proto = nullptr; + graph.GetInitializedTensor(mul_inputs[1]->Name(), mul_B_tensor_proto); + + if (!Initializer::IsSupportedDataType(conv_W_tensor_proto) || + !Initializer::IsSupportedDataType(mul_B_tensor_proto) || + conv_W_tensor_proto->data_type() != mul_B_tensor_proto->data_type() || + !(conv_W_tensor_proto->dims_size() > 2 && conv_W_tensor_proto->dims(0) == mul_B_tensor_proto->dims(0))) { + continue; + } + + auto conv_W = std::make_unique(conv_W_tensor_proto); + auto mul_B = std::make_unique(mul_B_tensor_proto); + + if (conv_W->data_type() != mul_B->data_type() || + !(conv_W->dims().size() > 2 && conv_W->dims()[0] == mul_B->dims()[0])) { + continue; + } + + const ONNX_NAMESPACE::TensorProto* conv_B_tensor_proto = nullptr; + std::unique_ptr conv_B = nullptr; + if (conv_inputs.size() == 3) { + graph.GetInitializedTensor(conv_inputs[2]->Name(), conv_B_tensor_proto); + if (!Initializer::IsSupportedDataType(conv_B_tensor_proto) || + conv_B_tensor_proto->data_type() != mul_B_tensor_proto->data_type() || + conv_B_tensor_proto->dims_size() != 1 || mul_B_tensor_proto->dims_size() != 3 || + conv_B_tensor_proto->dims(0) != mul_B_tensor_proto->dims(0)) { + continue; + } + conv_B = std::make_unique(conv_B_tensor_proto); + } + + // Caculate new value of initializers of conv node + conv_W->scale_by_axis(*mul_B, 1); + if (conv_inputs.size() == 3) { + conv_B->mul(*mul_B); + } + + // Create new initializers of conv + ONNX_NAMESPACE::TensorProto new_conv_W_tensor_proto(*conv_W_tensor_proto); + conv_W->ToProto(&new_conv_W_tensor_proto); + + // Replace initializers of conv node + graph.RemoveInitializedTensor(conv_inputs[1]->Name()); + graph.AddInitializedTensor(new_conv_W_tensor_proto); + + if (conv_inputs.size() == 3) { + ONNX_NAMESPACE::TensorProto new_conv_B_tensor_proto(*conv_B_tensor_proto); + conv_B->ToProto(&new_conv_B_tensor_proto); + graph.RemoveInitializedTensor(conv_inputs[2]->Name()); + graph.AddInitializedTensor(new_conv_B_tensor_proto); + } + + // Replace the input of the node following mul node + const NodeArg* mul_output_def = mul_node->OutputDefs()[0]; + const NodeArg* conv_output_def = conv_node.OutputDefs()[0]; + for (auto it = mul_node->OutputNodesBegin(); it != mul_node->OutputNodesEnd(); ++it) { + auto output_node = graph.GetNode((*it)->Index()); + if (!output_node) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT); + } + + auto& input_defs = output_node->MutableInputDefs(); + for (auto& def : input_defs) { + if (def == mul_output_def) { + def = const_cast(conv_output_def); + } + } + } + + removed_nodes.push_back(mul_node->Index()); + } + + for (auto i : removed_nodes) { + graph.RemoveNode(i); + } + + if (!removed_nodes.empty()) { + modified = true; + ONNXRUNTIME_RETURN_IF_ERROR(graph.Resolve()); + } + return Status::OK(); +} + +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/core/graph/conv_mul_fusion.h b/onnxruntime/core/graph/conv_mul_fusion.h new file mode 100644 index 0000000000000..6e568a8951ab0 --- /dev/null +++ b/onnxruntime/core/graph/conv_mul_fusion.h @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/graph/graph_transformer.h" + +namespace onnxruntime { + +class ConvMulFusion : public onnxruntime::GraphTransformer { + public: + ConvMulFusion() noexcept : onnxruntime::GraphTransformer("ConvMulFusion", "Fusing Mul into Conv") {} + Status Apply(onnxruntime::Graph& graph, bool& modified) const override; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/function.cc b/onnxruntime/core/graph/function.cc new file mode 100644 index 0000000000000..1acfd869c99cb --- /dev/null +++ b/onnxruntime/core/graph/function.cc @@ -0,0 +1,234 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/graph/function_impl.h" +#include "core/graph/graph.h" +#include "core/graph/model.h" +#include "onnx/shape_inference/implementation.h" + +namespace onnxruntime { +void TypeConstraintHelper(const ONNX_NAMESPACE::FunctionProto* onnx_func_proto_, + std::unique_ptr& op_schema_, + /*out*/ + std::unordered_map& input_name_idx_map, + std::unordered_map& output_name_idx_map) { + std::vector> input_types_list(onnx_func_proto_->input_size()); + std::vector> output_types_list(onnx_func_proto_->output_size()); + std::unordered_map> type_constraint_map; + for (int i = 0; i < onnx_func_proto_->input_size(); ++i) { + input_name_idx_map[onnx_func_proto_->input().Get(i)] = i; + } + for (int i = 0; i < onnx_func_proto_->output_size(); ++i) { + output_name_idx_map[onnx_func_proto_->output().Get(i)] = i; + } + auto schema_registry = ONNX_NAMESPACE::OpSchemaRegistry::Instance(); + for (auto& node : onnx_func_proto_->node()) { + const auto node_op_schema = schema_registry->GetSchema(node.op_type(), (int)onnx_func_proto_->since_version(), node.domain()); + for (int i = 0; i < node.input_size(); ++i) { + auto& in_name = node.input().Get(i); + auto iter = input_name_idx_map.find(in_name); + if (iter != input_name_idx_map.end()) { + int idx = iter->second; + const auto& p = node_op_schema->inputs().at(i); + std::string type_str = p.GetTypeStr() + "in" + std::to_string(i); + input_types_list[idx] = std::make_pair(in_name, type_str); + if (!type_constraint_map.count(type_str)) { + for (auto s : p.GetTypes()) { + type_constraint_map[type_str].emplace_back(*s); + } + } + } + } + for (int i = 0; i < node.output_size(); ++i) { + auto& out_name = node.output().Get(i); + auto iter = output_name_idx_map.find(out_name); + if (iter != output_name_idx_map.end()) { + int idx = iter->second; + const auto& p = node_op_schema->outputs().at(i); + std::string type_str = p.GetTypeStr() + "out" + std::to_string(i); + output_types_list[idx] = std::make_pair(out_name, type_str); + if (!type_constraint_map.count(type_str)) { + for (auto s : p.GetTypes()) { + type_constraint_map[type_str].emplace_back(*s); + } + } + } + } + } + + int i = 0; + for (auto& input : input_types_list) { + op_schema_->Input(i, input.first, "", input.second); + ++i; + } + i = 0; + for (auto& output : output_types_list) { + op_schema_->Output(i, output.first, "", output.second); + ++i; + } + + for (auto& tc : type_constraint_map) { + op_schema_->TypeConstraint(tc.first, tc.second, ""); + } +} + +FunctionImpl::FunctionImpl(const onnxruntime::Graph& graph, + std::unique_ptr customized_func) + : parent_graph_(&graph) { + customized_func_body_ = std::move(customized_func); + auto meta_def = customized_func_body_->GetMetaDef(); + op_schema_ = std::make_unique(); + op_schema_->SetName(meta_def->name); + op_schema_->SetDomain(meta_def->domain); + op_schema_->SetDoc(meta_def->doc_string); + op_schema_->SinceVersion(meta_def->since_version); + int i = 0; + for (auto& input : meta_def->inputs) { + auto input_type = parent_graph_->GetNodeArg(input)->Type(); + op_schema_->Input(i, input, "", *input_type); + ++i; + } + i = 0; + for (auto& output : meta_def->outputs) { + auto output_type = parent_graph_->GetNodeArg(output)->Type(); + op_schema_->Output(i, output, "", *output_type); + ++i; + } + op_schema_->Finalize(); + //construct body + body_ = std::make_unique("fused_function_subgraph", false, onnxruntime::ModelMetaData(), + IOnnxRuntimeOpSchemaRegistryList({graph.GetSchemaRegistry()}), graph.DomainToVersionMap()); + + auto& sub_graph = body_->MainGraph(); + //Add node and node args + //TODO: for better performance, we could try to transfer the nodes in parent graph to sub-graph directly, + //instead of create new nodes. + for (auto& node_index : customized_func_body_->nodes) { + auto node = parent_graph_->GetNode(node_index); + std::vector inputs, outputs; + for (auto input : node->InputDefs()) { + auto& n_input = sub_graph.GetOrCreateNodeArg(input->Name(), input->TypeAsProto()); + inputs.push_back(&n_input); + } + for (auto output : node->OutputDefs()) { + auto& n_output = sub_graph.GetOrCreateNodeArg(output->Name(), output->TypeAsProto()); + outputs.push_back(&n_output); + } + sub_graph.AddNode(node->Name(), node->OpType(), node->Description(), inputs, outputs, &node->GetAttributes(), node->Domain()); + } + //TODO: if we reuse the nodes in parent graph, maybe we don't need to resolve it. + ONNXRUNTIME_ENFORCE(sub_graph.Resolve().IsOK()); +} + +FunctionImpl::FunctionImpl(const onnxruntime::Graph& graph, + const onnxruntime::NodeIndex& node_index, + const ONNX_NAMESPACE::FunctionProto* onnx_func_proto) + : parent_graph_(&graph) { + onnx_func_proto_ = onnx_func_proto; + auto node_in_parent_graph = parent_graph_->GetNode(node_index); + op_schema_ = std::make_unique(); + op_schema_->SetName(onnx_func_proto_->name()); + op_schema_->SetDomain(onnx_func_proto_->node().Get(0).domain()); + op_schema_->SetDoc(onnx_func_proto_->doc_string()); + op_schema_->SinceVersion((ONNX_NAMESPACE::OperatorSetVersion)onnx_func_proto_->since_version()); + std::unordered_map input_name_idx_map; + std::unordered_map output_name_idx_map; + TypeConstraintHelper(onnx_func_proto_, this->op_schema_, input_name_idx_map, output_name_idx_map); + + op_schema_->TypeAndShapeInferenceFunction( + [this](ONNX_NAMESPACE::InferenceContext& ctx) { + auto schema_registry = ONNX_NAMESPACE::OpSchemaRegistry::Instance(); + const ONNX_NAMESPACE::FunctionProto* func_ptr = this->GetFuncProto(); + if (nullptr != func_ptr) { + ONNX_NAMESPACE::shape_inference::InferShapeForFunctionNode(*func_ptr, schema_registry, ctx); + } + }); + + op_schema_->Finalize(); + //construct body + std::unordered_map domain_to_version; + //TODO: set correct domain and version + domain_to_version[onnxruntime::kOnnxDomain] = (int)onnx_func_proto_->since_version(); + body_ = std::make_unique(onnx_func_proto_->name(), false, onnxruntime::ModelMetaData(), + IOnnxRuntimeOpSchemaRegistryList(), domain_to_version); + auto& sub_graph = body_->MainGraph(); + // Add node and node args into subgraph + // The subgraph preserved the input/output tensor names + // in the parent graph for later inlining purpose + auto attr_map = node_in_parent_graph->GetAttributes(); + for (auto& node : onnx_func_proto_->node()) { + std::vector inputs, outputs; + for (int idx = 0; idx < node.input_size(); ++idx) { + std::string tensor_name = node.input().Get(idx); + auto iter = input_name_idx_map.find(tensor_name); + if (iter != input_name_idx_map.end()) { + // Preserving NodeArg and input/output names + ONNX_NAMESPACE::NodeProto temp_node_proto; + node_in_parent_graph->ToProto(temp_node_proto); + const onnxruntime::NodeArg* node_arg = parent_graph_->GetNodeArg(temp_node_proto.input().Get(input_name_idx_map[tensor_name])); + auto& n_input = sub_graph.GetOrCreateNodeArg( + temp_node_proto.input().Get(iter->second), node_arg->TypeAsProto()); + inputs.push_back(&n_input); + } else { + auto& n_input = sub_graph.GetOrCreateNodeArg( + tensor_name + "_" + std::to_string(node_index), nullptr); + inputs.push_back(&n_input); + } + } + for (int idx = 0; idx < node.output_size(); ++idx) { + std::string tensor_name = node.output().Get(idx); + auto iter = output_name_idx_map.find(tensor_name); + if (iter != output_name_idx_map.end()) { + // Preserving NodeArg and input/output names + ONNX_NAMESPACE::NodeProto temp_node_proto; + node_in_parent_graph->ToProto(temp_node_proto); + const onnxruntime::NodeArg* node_arg = parent_graph_->GetNodeArg(temp_node_proto.output().Get(output_name_idx_map[tensor_name])); + auto& n_output = sub_graph.GetOrCreateNodeArg( + temp_node_proto.output().Get(iter->second), node_arg->TypeAsProto()); + outputs.push_back(&n_output); + } else { + auto& n_output = sub_graph.GetOrCreateNodeArg( + tensor_name + "_" + std::to_string(node_index), nullptr); + outputs.push_back(&n_output); + } + } + + onnxruntime::NodeAttributes new_attr_map; + for (auto& attr : node.attribute()) { + if (attr.has_ref_attr_name()) { + if (attr_map.count(attr.ref_attr_name())) { + new_attr_map[attr.name()] = attr_map[attr.ref_attr_name()]; + } + } else { + new_attr_map[attr.name()] = attr; + } + } + sub_graph.AddNode(node.name(), node.op_type(), node.doc_string(), inputs, outputs, &new_attr_map, node.domain()); + } + auto status = sub_graph.Resolve(); + ONNXRUNTIME_ENFORCE(status.IsOK()); +} + +FunctionImpl::~FunctionImpl() = default; + +const ONNX_NAMESPACE::OpSchema& FunctionImpl::OpSchema() const { + return *op_schema_; +} + +const onnxruntime::Graph& FunctionImpl::Body() const { + return body_->MainGraph(); +} + +const IndexedSubGraph& FunctionImpl::GetIndexedSubGraph() const { + return *customized_func_body_; +} + +const ONNX_NAMESPACE::FunctionProto* FunctionImpl::GetFuncProto() const { + return onnx_func_proto_; +} + +std::unique_ptr MakeFunction(const onnxruntime::Graph& graph, + std::unique_ptr customized_func) { + return std::make_unique(graph, std::move(customized_func)); +} +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/function.h b/onnxruntime/core/graph/function.h new file mode 100644 index 0000000000000..b3161c2d3387e --- /dev/null +++ b/onnxruntime/core/graph/function.h @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/graph/indexed_sub_graph.h" + +namespace onnxruntime { +class Graph; +class Node; +} // namespace onnxruntime + +namespace onnxruntime { + +// Function representation class. +class Function { + public: + virtual ~Function() {} + virtual const ONNX_NAMESPACE::OpSchema& OpSchema() const = 0; + + virtual const onnxruntime::Graph& Body() const = 0; + + virtual const IndexedSubGraph& GetIndexedSubGraph() const = 0; +}; + +std::unique_ptr MakeFunction(const onnxruntime::Graph& graph, + std::unique_ptr customized_func); +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/function_container.h b/onnxruntime/core/graph/function_container.h new file mode 100644 index 0000000000000..f29b67a30c57a --- /dev/null +++ b/onnxruntime/core/graph/function_container.h @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include +#include "core/graph/function.h" +//TODO: we need to make it a stand-alone header because both graph.cc and model.cc need to implement create instance of the graph object. +//Right now only functions_ has issue because it use vector of unique-ptr, maybe we should extend this to GraphImpl later. +namespace onnxruntime { +struct FunctionContainer { + std::vector> functions_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/function_impl.h b/onnxruntime/core/graph/function_impl.h new file mode 100644 index 0000000000000..5df32a05a6f25 --- /dev/null +++ b/onnxruntime/core/graph/function_impl.h @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/graph/function.h" + +namespace onnxruntime { +class Graph; +class Node; +class Model; +} // namespace onnxruntime + +namespace onnxruntime { + +// Function representation class. +class FunctionImpl final : public Function { + public: + FunctionImpl(const onnxruntime::Graph& graph, + std::unique_ptr customized_func); + + FunctionImpl(const onnxruntime::Graph& graph, + const onnxruntime::NodeIndex& node_index, + const ONNX_NAMESPACE::FunctionProto* onnx_func); + + ~FunctionImpl(); + + const ONNX_NAMESPACE::OpSchema& OpSchema() const override; + + const onnxruntime::Graph& Body() const override; + + const IndexedSubGraph& GetIndexedSubGraph() const override; + + const ONNX_NAMESPACE::FunctionProto* GetFuncProto() const; + + private: + const onnxruntime::Graph* const parent_graph_; + std::unique_ptr customized_func_body_; + std::unique_ptr op_schema_; + std::unique_ptr body_; + const ONNX_NAMESPACE::FunctionProto* onnx_func_proto_; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/function_inliner.h b/onnxruntime/core/graph/function_inliner.h new file mode 100644 index 0000000000000..1d277836c7665 --- /dev/null +++ b/onnxruntime/core/graph/function_inliner.h @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/common/common.h" +#include "core/graph/function.h" +#include "core/graph/rewrite_rule.h" + +namespace onnxruntime { +class Node; +} // namespace onnxruntime + +namespace onnxruntime { + +// A function-inlining rewrite-rule. +class FunctionInliner : public onnxruntime::RewriteRule { + public: + FunctionInliner(const std::string& name, const std::string& desc) + : RewriteRule(name, desc) {} + + Status Apply(onnxruntime::GraphEditor /*graph_editor*/, onnxruntime::Node* /*node*/, bool* /*modified*/) override { + return Status::OK(); + } +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/graph.cc b/onnxruntime/core/graph/graph.cc new file mode 100644 index 0000000000000..622f1ee9112fb --- /dev/null +++ b/onnxruntime/core/graph/graph.cc @@ -0,0 +1,2494 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#ifdef _WIN32 +// disable some warnings from protobuf to pass Windows build +#pragma warning(disable : 4244) +#endif + +#include +#include +#include +#include + +#include "gsl/pointers" +#include "core/graph/function.h" +#include "core/graph/function_impl.h" +#include "core/graph/graph.h" +#include "core/graph/indexed_sub_graph.h" +#include "core/graph/op.h" +#include "core/common/logging/logging.h" +#include "onnx/checker.h" +#include "core/graph/schema_registry.h" +using namespace ONNX_NAMESPACE; +using namespace ONNX_NAMESPACE::Utils; +using namespace ONNX_NAMESPACE::checker; +using namespace ::onnxruntime::common; + +namespace onnxruntime { + +#define NO_CHANGE_ON_SYNC_FLAG(...) \ + do { \ + const bool sync_needed = GraphProtoSyncNeeded(); \ + { __VA_ARGS__; } \ + GraphProtoSyncNeeded(sync_needed); \ + } while (0) + +static Status MergeShapeInfo(const std::string& output_name, + const TypeProto_Tensor& source, TypeProto_Tensor& target) { + try { + ONNX_NAMESPACE::mergeInShapeInfo(source, target); + } catch (const ONNX_NAMESPACE::InferenceError& ex) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Output:", output_name, " ", ex.what()); + } + + return Status::OK(); +} + +static bool GraphLoadedFromModelFile(const GraphProto* graph_proto) { + return graph_proto && (graph_proto->input_size() != 0 || + graph_proto->output_size() != 0 || + graph_proto->value_info_size() != 0); +} + +NodeArg::NodeArg(const std::string& name, + const TypeProto* p_node_arg_type) { + node_arg_info_.set_name(name); + // If the name is empty, it means the arg does not exist. + exists_ = !(name.empty()); + if (nullptr != p_node_arg_type) { + (*node_arg_info_.mutable_type()) = *p_node_arg_type; + type_ = DataTypeUtils::ToType(node_arg_info_.type()); + } else { + type_ = nullptr; + } +} + +const std::string& NodeArg::Name() const noexcept { + return node_arg_info_.name(); +} + +DataType NodeArg::Type() const noexcept { + return type_; +} + +const TypeProto* NodeArg::TypeAsProto() const noexcept { + if (node_arg_info_.has_type()) + return &node_arg_info_.type(); + else + return nullptr; +} + +const TensorShapeProto* NodeArg::Shape() const { + if (!node_arg_info_.has_type()) { + return nullptr; + } + + const auto typeCase = node_arg_info_.type().value_case(); + switch (typeCase) { + case TypeProto::kTensorType: { + if (node_arg_info_.type().tensor_type().has_shape()) { + return &(node_arg_info_.type().tensor_type().shape()); + } else { + return nullptr; + } + } + case TypeProto::kSparseTensorType: { + if (node_arg_info_.type().sparse_tensor_type().has_shape()) { + return &(node_arg_info_.type().sparse_tensor_type().shape()); + } else { + return nullptr; + } + } + case TypeProto::kSequenceType: + case TypeProto::kMapType: + case TypeProto::kOpaqueType: + case TypeProto::VALUE_NOT_SET: + default: + return nullptr; + } +} + +void NodeArg::SetShape(const TensorShapeProto& shape) { + if (!node_arg_info_.has_type()) { + return; + } + + const auto type_case = node_arg_info_.type().value_case(); + switch (type_case) { + case TypeProto::kTensorType: + *(node_arg_info_.mutable_type()->mutable_tensor_type()->mutable_shape()) = shape; + break; + case TypeProto::kSparseTensorType: + *(node_arg_info_.mutable_type()->mutable_sparse_tensor_type()->mutable_shape()) = shape; + break; + case TypeProto::kSequenceType: + case TypeProto::kMapType: + case TypeProto::kOpaqueType: + case TypeProto::VALUE_NOT_SET: + default: + return; + } +} + +common::Status NodeArg::UpdateTypeAndShape(const ONNX_NAMESPACE::TypeProto& input_type) { + if (!node_arg_info_.has_type()) { + *node_arg_info_.mutable_type() = input_type; + type_ = DataTypeUtils::ToType(node_arg_info_.type()); + return Status::OK(); + } + + auto& current_type = *node_arg_info_.mutable_type(); + const auto current_type_case = current_type.value_case(); + const auto input_type_case = input_type.value_case(); + + if (current_type_case != input_type_case) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Type mismatch. Current=", + current_type_case, " Input=", input_type_case); + + switch (input_type_case) { + case TypeProto::kTensorType: { + const auto& input_tensor_type = input_type.tensor_type(); + const auto& input_tensor_elem_type = input_tensor_type.elem_type(); + const auto& current_tensor_elem_type = current_type.tensor_type().elem_type(); + + if (input_tensor_elem_type != current_tensor_elem_type) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Tensor element type mismatch. ", + TensorProto_DataType_Name(input_tensor_elem_type), " != ", + TensorProto_DataType_Name(current_tensor_elem_type)); + + if (input_tensor_type.has_shape()) { + auto& current_tensor_type = *current_type.mutable_tensor_type(); + if (current_tensor_type.has_shape()) { + ONNXRUNTIME_RETURN_IF_ERROR(MergeShapeInfo(Name(), input_tensor_type, current_tensor_type)); + } else { + current_tensor_type = input_tensor_type; + } + } + + break; + } + case TypeProto::kSparseTensorType: { + const auto& input_tensor_type = input_type.sparse_tensor_type(); + const auto input_tensor_elem_type = input_tensor_type.elem_type(); + const auto current_tensor_elem_type = current_type.sparse_tensor_type().elem_type(); + if (input_tensor_elem_type != current_tensor_elem_type) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "SparseTensor element type mismatch. ", + TensorProto_DataType_Name(input_tensor_elem_type), " != ", + TensorProto_DataType_Name(current_tensor_elem_type)); + } + if (input_tensor_type.has_shape()) { + auto& current_tensor_type = *current_type.mutable_sparse_tensor_type(); + if (current_tensor_type.has_shape()) { + // TODO: Check if we need to merge shape here + // if so we'd need to provide merging routine ONNX + // mergeInShapeInfo(input_tensor_type, current_tensor_type); + } else { + current_tensor_type = input_tensor_type; + } + } + } break; + case TypeProto::kSequenceType: + case TypeProto::kMapType: + case TypeProto::kOpaqueType: + case TypeProto::VALUE_NOT_SET: + break; + } + + return Status::OK(); +} + +common::Status NodeArg::UpdateTypeAndShape(const NodeArg& node_arg) { + auto status = Status::OK(); + + if (node_arg.node_arg_info_.has_type()) + status = UpdateTypeAndShape(node_arg.node_arg_info_.type()); + + return status; +} + +void NodeArg::SetType(DataType p_type) { + if (nullptr == p_type) { + return; + } + + type_ = p_type; + *(node_arg_info_.mutable_type()) = DataTypeUtils::ToTypeProto(p_type); +} + +void NodeArg::SetType(const TypeProto& type_proto) { + type_ = DataTypeUtils::ToType(type_proto); + *(node_arg_info_.mutable_type()) = type_proto; +} + +bool NodeArg::Exists() const noexcept { + return exists_; +} + +Node::EdgeEnd::EdgeEnd(const Node& node, const NodeArg& node_arg) noexcept + : node_(&node), node_arg_(&node_arg) { +} + +Node::EdgeEnd::EdgeEnd(const Node& node) noexcept + : node_(&node), node_arg_(nullptr) { +} + +const Node& Node::EdgeEnd::GetNode() const noexcept { + return *node_; +} + +const NodeArg* Node::EdgeEnd::GetNodeArg() const noexcept { + return node_arg_; +} + +Node::NodeConstIterator::NodeConstIterator(EdgeConstIterator p_iter) { + m_iter = p_iter; +} + +bool Node::NodeConstIterator::operator==(const NodeConstIterator& p_other) const { + return m_iter == p_other.m_iter; +} + +bool Node::NodeConstIterator::operator!=(const NodeConstIterator& p_other) const { + return m_iter != p_other.m_iter; +} + +void Node::NodeConstIterator::operator++() { + ++m_iter; +} + +void Node::NodeConstIterator::operator--() { + --m_iter; +} + +const Node* Node::NodeConstIterator::operator*() { + return &((*m_iter).GetNode()); +} + +NodeIndex Node::Index() const noexcept { + return index_; +} + +const std::string& Node::Name() const noexcept { + return name_; +} + +const std::string& Node::OpType() const noexcept { + return op_type_; +} + +const std::string& Node::Description() const noexcept { + return description_; +} + +const std::string& Node::Domain() const noexcept { + return domain_; +} + +const OpSchema* Node::Op() const noexcept { + return op_; +} + +Node::Type Node::NodeType() const noexcept { + return node_type_; +} + +void Node::SetNodeType(Node::Type node_type) noexcept { + node_type_ = node_type; +} + +const ::onnxruntime::Function* Node::GetFunctionBody() const noexcept { + return func_body_; +} + +void Node::SetFunctionBody(const ::onnxruntime::Function& func) { + func_body_ = &func; + op_ = &func.OpSchema(); +} + +const std::string& Node::GetExecutionProviderType() const noexcept { + return execution_provider_type_; +} + +void Node::SetExecutionProviderType(onnxruntime::ProviderType execution_provider_type) { + execution_provider_type_ = execution_provider_type; +} + +void Node::ToProto(NodeProto& proto) const { + // Set name. + proto.set_name(name_); + // Set op type. + proto.set_op_type(op_type_); + // Set op domain; + proto.set_domain(domain_); + // Set doc string. + proto.set_doc_string(description_); + + // Set attributes. + proto.clear_attribute(); + for (auto attribute : attributes_) { + const gsl::not_null attr{proto.add_attribute()}; + *attr = attribute.second; + } + + // Set inputs' definitions. + proto.clear_input(); + for (auto& input_def : definitions_.input_defs) { + proto.add_input(input_def->Name()); + } + + // Set outputs' definitions. + proto.clear_output(); + for (auto& output_def : definitions_.output_defs) { + proto.add_output(output_def->Name()); + } +} + +void Node::Init(const std::string& name, + const std::string& op_type, + const std::string& description, + const std::vector& input_args, + const std::vector& output_args, + const NodeAttributes* attributes, + const std::string& domain) { + name_ = name; + op_type_ = op_type; + description_ = description; + definitions_.input_defs = input_args; + definitions_.output_defs = output_args; + domain_ = domain; + if (kOnnxDomainAlias == domain_) { + domain_ = kOnnxDomain; + } + + // Set each arg count as 1 by default. + // It could be adjusted when resolving the node with its operator + // information. + definitions_.input_arg_count.assign(input_args.size(), 1); + + if (attributes) { + attributes_ = *attributes; + } +} + +Node::Definitions& Node::MutableDefinitions() noexcept { + // someone fetching these is going to change something + graph_->SetGraphResolveNeeded(); + graph_->SetGraphProtoSyncNeeded(); + return definitions_; +} + +Node::Relationships& Node::MutableRelationships() noexcept { + // someone fetching these is going to change something + graph_->SetGraphResolveNeeded(); + graph_->SetGraphProtoSyncNeeded(); + return relationships_; +} + +void Node::AddAttribute(const std::string& attr_name, const AttributeProto& value) { + graph_->SetGraphResolveNeeded(); + graph_->SetGraphProtoSyncNeeded(); + attributes_[attr_name] = value; +} + +#define ADD_BASIC_ATTR_IMPL(type, enumType, field) \ + void Node::AddAttribute(const std::string& attr_name, const type& value) { \ + graph_->SetGraphResolveNeeded(); \ + graph_->SetGraphProtoSyncNeeded(); \ + AttributeProto a; \ + a.set_name(attr_name); \ + a.set_type(enumType); \ + a.set_##field(value); \ + attributes_[attr_name] = a; \ + }; + +#define ADD_ATTR_IMPL(type, enumType, field) \ + void Node::AddAttribute(const std::string& attr_name, const type& value) { \ + graph_->SetGraphResolveNeeded(); \ + graph_->SetGraphProtoSyncNeeded(); \ + AttributeProto a; \ + a.set_name(attr_name); \ + a.set_type(enumType); \ + *(a.mutable_##field()) = value; \ + attributes_[attr_name] = a; \ + }; + +#define ADD_LIST_ATTR_IMPL(type, enumType, field) \ + void Node::AddAttribute(const std::string& attr_name, \ + const std::vector& values) { \ + graph_->SetGraphResolveNeeded(); \ + graph_->SetGraphProtoSyncNeeded(); \ + AttributeProto a; \ + a.set_name(attr_name); \ + a.set_type(enumType); \ + for (const auto& val : values) { \ + *(a.mutable_##field()->Add()) = val; \ + } \ + attributes_[attr_name] = a; \ + }; + +ADD_BASIC_ATTR_IMPL(float, AttributeProto_AttributeType::AttributeProto_AttributeType_FLOAT, f) +ADD_BASIC_ATTR_IMPL(int64_t, AttributeProto_AttributeType::AttributeProto_AttributeType_INT, i) +ADD_BASIC_ATTR_IMPL(std::string, AttributeProto_AttributeType::AttributeProto_AttributeType_STRING, s) +ADD_ATTR_IMPL(TensorProto, AttributeProto_AttributeType::AttributeProto_AttributeType_TENSOR, t) +ADD_ATTR_IMPL(GraphProto, AttributeProto_AttributeType::AttributeProto_AttributeType_GRAPH, g) +ADD_LIST_ATTR_IMPL(float, AttributeProto_AttributeType::AttributeProto_AttributeType_FLOATS, floats) +ADD_LIST_ATTR_IMPL(int64_t, AttributeProto_AttributeType::AttributeProto_AttributeType_INTS, ints) +ADD_LIST_ATTR_IMPL(std::string, AttributeProto_AttributeType::AttributeProto_AttributeType_STRINGS, strings) +ADD_LIST_ATTR_IMPL(TensorProto, AttributeProto_AttributeType::AttributeProto_AttributeType_TENSORS, tensors) +ADD_LIST_ATTR_IMPL(GraphProto, AttributeProto_AttributeType::AttributeProto_AttributeType_GRAPHS, graphs) + +bool Node::ClearAttribute(const std::string& attr_name) { + graph_->SetGraphResolveNeeded(); + graph_->SetGraphProtoSyncNeeded(); + return attributes_.erase(attr_name) > 0; +} + +Status Node::UpdateInputArgCount() { + // The node refers to a primitive operator. + // Infer and verify node input arg type information. + int total_arg_count = std::accumulate(definitions_.input_arg_count.cbegin(), + definitions_.input_arg_count.cend(), 0); + + if (total_arg_count < 0 || static_cast(total_arg_count) != definitions_.input_defs.size()) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, + "The sum of input arg count is not equal to size of input defs in node (", + name_, ")"); + } + + // op_ is always valid when this is called + const ONNX_NAMESPACE::OpSchema& op = *Op(); + + // Verify size of node arg count is same as input number in + // operator definition. + if (op.inputs().size() != definitions_.input_arg_count.size()) { + // Adjust input arg count array with op definition + // The adjustment will work as below, + // In total, there're inputs, which + // will be split as <1, 1, 1, 1, ... 1, x> or + // <1, 1, 1, 1, ...1, 0, 0, ...0>. The final input + // arg count array's element number will be the same + // as op definition, and the sum of all elements will + // be equal to . + auto& input_arg_count = definitions_.input_arg_count; + input_arg_count.clear(); + size_t m = 0; + auto arg_count_left = total_arg_count; + + if (!op.inputs().empty()) { + for (; m < op.inputs().size() - 1; ++m) { + if (arg_count_left > 0) { + input_arg_count.push_back(1); + arg_count_left--; + } else { + input_arg_count.push_back(0); + } + } + } + + // Set the arg count for the last input formal parameter. + // NOTE: in the case that there's no .input(...) defined + // in op schema, all input args will be fed as one input + // of the operator. + input_arg_count.push_back(arg_count_left); + + graph_->SetGraphResolveNeeded(); + graph_->SetGraphProtoSyncNeeded(); + } + + return Status::OK(); +} + +const NodeAttributes& Node::GetAttributes() const noexcept { + return attributes_; +} + +void Node::ForEachDef(std::function func) const { + for (const auto* arg : InputDefs()) { + if (arg->Exists()) + func(&*arg, true); + } + + for (const auto* arg : ImplicitInputDefs()) { + if (arg->Exists()) + func(&*arg, true); + } + + for (const auto* arg : OutputDefs()) { + if (arg->Exists()) + func(&*arg, false); + } +}; + +void Node::ForEachInputDef(std::function func) const { + for (const auto* arg : InputDefs()) { + if (!arg->Exists()) + continue; + func(&*arg); + } +}; + +void Node::ForEachOutputDef(std::function func) const { + for (const auto* arg : OutputDefs()) { + if (!arg->Exists()) + continue; + func(&*arg); + } +}; + +void Node::ReplaceDefs(const std::map& replacements) { + std::vector*> all_defs = {&definitions_.input_defs, &definitions_.output_defs}; + + for (auto pair : replacements) + for (auto* defs : all_defs) + for (auto& def : *defs) + if (def == pair.first) + def = pair.second; +} + +// Constructor: Given a loaded from model file, construct +// a object and Resolve() it. +//Status Graph::LoadGraph(const GraphProto& graph_proto, +// const std::unordered_map& domain_to_version, +// Version ir_version, +// std::unique_ptr& new_graph) { +// // create instance. need to call private ctor so can't use make_unique +// GSL_SUPPRESS(r .11) +// new_graph.reset(new Graph(nullptr, &graph_proto, domain_to_version, ir_version)); +// +// // as we just loaded from file we want to fully initialize/Resolve, but not let that change +// // the proto sync flag +// auto status = new_graph->Resolve(/* no_proto_sync_required */ true); +// return status; +//} +using google::protobuf::RepeatedPtrField; + +Graph::Graph(GraphProto* graph_proto, + const std::unordered_map& domain_to_version, + Version ir_version, + IOnnxRuntimeOpSchemaCollectionPtr schema_registry, + const std::unordered_map& model_functions) : Graph(graph_proto, domain_to_version, ir_version, schema_registry, nullptr, model_functions) {} + +Graph::Graph(GraphProto* graph_proto, + const std::unordered_map& domain_to_version, + Version ir_version, + IOnnxRuntimeOpSchemaCollectionPtr schema_registry, + Graph* parent_graph, + const std::unordered_map& model_functions) + : graph_proto_{graph_proto}, + graph_type_{Type::Main}, + schema_registry_(schema_registry), + graph_resolve_needed_(true), + graph_proto_sync_needed_(false), + domain_to_version_(domain_to_version), + model_functions_(model_functions), + ir_version_(ir_version), + parent_graph_{parent_graph} { + ONNXRUNTIME_ENFORCE(graph_proto != nullptr, "graph_proto cannot be null"); + ArgNameToTypeMap name_to_type_map; + + // these are all empty unless we received a graph_proto as input + if (graph_proto != nullptr) { + // Copy constant nodes _value to name_to_initial_tensor_ + for (auto& node : graph_proto_->node()) { + if (node.op_type() == kConstant) { + const gsl::not_null tensor{graph_proto_->add_initializer()}; + *tensor = node.attribute(0).t(); + *(tensor->mutable_name()) = node.output(0); + + // we remove the node and add it as an initializer, but still need it to appear in the + // graph inputs to make the ONNX checker happy. add a new input due to that. + auto graph_inputs = graph_proto_->mutable_input(); + + ValueInfoProto* value_info = graph_inputs->Add(); + value_info->set_name(node.output(0)); + value_info->set_doc_string("Input to represent replaced Constant node"); + + TypeProto t; + t.mutable_tensor_type()->set_elem_type(tensor->data_type()); + auto shape = t.mutable_tensor_type()->mutable_shape(); + for (auto dim : tensor->dims()) + shape->add_dim()->set_dim_value(dim); + + (*value_info->mutable_type()) = t; + } + } + + // remove constant nodes + const gsl::not_null*> graph_mutable_nodes{graph_proto_->mutable_node()}; + graph_mutable_nodes->erase( + std::remove_if(graph_mutable_nodes->begin(), graph_mutable_nodes->end(), + [](NodeProto& p) { + return (p.op_type() == kConstant); + }), + graph_mutable_nodes->end()); + + // Copy initial tensors to a map. + for (auto& tensor : graph_proto_->initializer()) { + name_to_initial_tensor_[tensor.name()] = &tensor; + } + + // Collect all node arg name, type, shape information in the graph. + // type/shape information will be assigned to each node arg when going + // thru all nodes later. + for (auto& graph_input : graph_proto_->input()) { + if (graph_input.has_name() && graph_input.has_type()) { + name_to_type_map[graph_input.name()] = graph_input.type(); + // always create a NodeArg for graph input in case its from an initializer + GetOrCreateNodeArg(graph_input.name(), &graph_input.type()); + } + } + + for (auto& graph_output : graph_proto_->output()) { + if (graph_output.has_name() && graph_output.has_type()) { + auto& name = graph_output.name(); + name_to_type_map[name] = graph_output.type(); + // always create NodeArg for graph output, in case it's from initializer + GetOrCreateNodeArg(name, &graph_output.type()); + } + } + + for (auto& node_arg : graph_proto_->value_info()) { + if (node_arg.has_name() && node_arg.has_type()) { + name_to_type_map[node_arg.name()] = node_arg.type(); + } + } + + for (auto node_proto : graph_proto_->node()) { + AddNode(node_proto, name_to_type_map); + } + } +} + +Graph::Graph(Graph& parent_graph, ONNX_NAMESPACE::GraphProto& subgraph_proto) + : Graph(&subgraph_proto, + parent_graph.DomainToVersionMap(), parent_graph.IrVersion(), parent_graph.schema_registry_, + &parent_graph) { +} + +Status Graph::VerifyNoDuplicateName() { + const std::unordered_set& inputs_and_initializers = resolve_context_.inputs_and_initializers; + std::unordered_map& output_args = resolve_context_.output_args; + std::unordered_map& node_name_to_index = resolve_context_.node_name_to_index; + + output_args.clear(); + node_name_to_index.clear(); + // inputs_and_initializers: this is passed in as a parameter, since functions don't have initializers + // but graphs have them. + + for (auto& node : Nodes()) { + // Verify node name should be unique. + auto& node_name = node.Name(); + + if (!node_name.empty() && node_name_to_index.end() != node_name_to_index.find(node_name)) { + // The node has name and its name was used by another node. + Status status(ONNXRUNTIME, FAIL, + "Error: two nodes with same node name (" + node_name + ")."); + return status; + } + + node_name_to_index[node_name] = node.Index(); + + // Verify node outputs' name should be unique. + for (const auto* output_def : node.OutputDefs()) { + if (output_def->Exists()) { + auto& output_arg_name = output_def->Name(); + if (inputs_and_initializers.count(output_arg_name)) { + Status status(ONNXRUNTIME, FAIL, + "Error: Duplicate definition of name (" + output_arg_name + ")."); + return status; + } + auto result = output_args.insert({output_arg_name, &node}); + if (!result.second) { + // Two outputs with same name, so that insertion fails. + Status status(ONNXRUNTIME, FAIL, + "Error: Duplicate definition of name (" + output_arg_name + ")."); + return status; + } + } + } + } + return Status::OK(); +} + +// Recurse into any subgraphs to update the list of NodeArg values in outer scope. +// This information is needed to resolve any dependencies on outer scope values. +common::Status Graph::SetOuterScopeNodeArgs(const std::unordered_set& outer_scope_node_args) { + resolve_context_.outer_scope_node_args = outer_scope_node_args; + + if (!resolve_context_.node_to_subgraphs_map.empty()) { + // Build the list of NodeArg's that are valid for a subgraph of this GraphBase instance: + // - outer scope for this graph + // - any inputs/initializers from this graph + // - any outputs from nodes in this graph + // + // NOTE: We must add the most outer most NodeArgs first, and then local NodeArgs, as the local should override + // an outer scope value if they have the same name. + // + // We provide outputs from all nodes in this graph at this stage. + // BuildConnections will link the node with the subgraph to any outer scope Node/NodeArgs it consumes. + // PerformTopologicalSortAndCheckIsAcyclic will validate these links. + std::unordered_set node_args_in_scope_for_subgraph = outer_scope_node_args; + + node_args_in_scope_for_subgraph.insert(resolve_context_.inputs_and_initializers.cbegin(), + resolve_context_.inputs_and_initializers.cend()); + + std::transform(resolve_context_.output_args.cbegin(), resolve_context_.output_args.cend(), + std::inserter(node_args_in_scope_for_subgraph, node_args_in_scope_for_subgraph.end()), + [](const std::pair& entry) { return entry.first; }); + + for (auto node_subgraphs : resolve_context_.node_to_subgraphs_map) { + for (auto* subgraph : node_subgraphs.second) { + auto status = subgraph->SetOuterScopeNodeArgs(node_args_in_scope_for_subgraph); + ONNXRUNTIME_RETURN_IF_ERROR(status); + } + } + } + + return Status::OK(); +} + +const NodeArg* Graph::GetNodeArgIncludingParentGraphs(const std::string& node_arg_name) const { + const NodeArg* node_arg = GetNodeArg(node_arg_name); + + if (!node_arg && parent_graph_) { + node_arg = parent_graph_->GetNodeArgIncludingParentGraphs(node_arg_name); + } + + return node_arg; +} + +void Graph::AddEdge(NodeIndex src_node_index, NodeIndex dst_node_index, const NodeArg& node_arg) { + if (nodes_.size() <= src_node_index || + nodes_.size() <= dst_node_index || + nullptr == nodes_[src_node_index] || + nullptr == nodes_[dst_node_index]) { + // Invalid node indexes specified. + ONNXRUNTIME_THROW("Invalid node indexes specified when adding edge."); + } + // Verify whether the node_arg is input of dst and output of src firstly. + bool valid = false; + for (auto arg : nodes_[src_node_index]->OutputDefs()) { + if (arg == &node_arg) { + valid = true; + break; + } + } + ONNXRUNTIME_ENFORCE(valid); + valid = false; + for (auto arg : nodes_[dst_node_index]->InputDefs()) { + if (arg == &node_arg) { + valid = true; + break; + } + } + for (auto arg : nodes_[dst_node_index]->ImplicitInputDefs()) { + if (arg == &node_arg) { + valid = true; + break; + } + } + ONNXRUNTIME_ENFORCE(valid); + nodes_[dst_node_index]->MutableRelationships().input_edges.insert(Node::EdgeEnd(*nodes_[src_node_index], node_arg)); + nodes_[src_node_index]->MutableRelationships().output_edges.insert(Node::EdgeEnd(*nodes_[dst_node_index], node_arg)); +} + +void Graph::RemoveEdge(NodeIndex src_node_index, NodeIndex dst_node_index, const NodeArg& node_arg) { + if (nodes_.size() <= src_node_index || + nodes_.size() <= dst_node_index || + nullptr == nodes_[src_node_index] || + nullptr == nodes_[dst_node_index]) { + // Invalid node indexes specified. + ONNXRUNTIME_THROW("Invalid node indexes specified when removing edge."); + } + // Verify whether the node_arg is input of dst and output of src firstly. + bool valid = false; + for (auto arg : nodes_[src_node_index]->OutputDefs()) { + if (arg == &node_arg) { + valid = true; + break; + } + } + ONNXRUNTIME_ENFORCE(valid); + valid = false; + for (auto arg : nodes_[dst_node_index]->InputDefs()) { + if (arg == &node_arg) { + valid = true; + break; + } + } + for (auto arg : nodes_[dst_node_index]->ImplicitInputDefs()) { + if (arg == &node_arg) { + valid = true; + break; + } + } + ONNXRUNTIME_ENFORCE(valid); + nodes_[dst_node_index]->MutableRelationships().input_edges.erase(Node::EdgeEnd(*nodes_[src_node_index], node_arg)); + nodes_[src_node_index]->MutableRelationships().output_edges.erase(Node::EdgeEnd(*nodes_[dst_node_index], node_arg)); +} + +GSL_SUPPRESS(es .84) // ignoring return value from unordered_map::insert causes noisy complaint +Status Graph::BuildConnections(std::vector& outer_scope_node_args_consumed) { + const std::unordered_set& outer_scope_node_args = resolve_context_.outer_scope_node_args; + std::unordered_set inner_nodes; + + // recurse into subgraphs first so we can update any nodes in this graph that are used by those subgraphs + if (!resolve_context_.node_to_subgraphs_map.empty()) { + for (auto nodeid_to_subgraphs : resolve_context_.node_to_subgraphs_map) { + for (auto* subgraph : nodeid_to_subgraphs.second) { + std::vector node_args_consumed; + subgraph->BuildConnections(node_args_consumed); + + for (auto& node_arg_name : node_args_consumed) { + const auto* node_arg = GetNodeArg(node_arg_name); + + if (node_arg == nullptr) { + // it's a node arg from outside this graph's scope, so add that to the list we return + // so that we can add the dependency at the next level up + outer_scope_node_args_consumed.push_back(node_arg_name); + + if (!parent_graph_) { + return ONNXRUNTIME_MAKE_STATUS( + ONNXRUNTIME, INVALID_GRAPH, + "At top level graph without matching NodeArg that subgraph consumes. Name=", + node_arg_name, + " Graph may not conform to the ONNX spec and contain initializers that are not graph inputs."); + } + + node_arg = parent_graph_->GetNodeArgIncludingParentGraphs(node_arg_name); + + if (!node_arg) { + return ONNXRUNTIME_MAKE_STATUS( + ONNXRUNTIME, INVALID_GRAPH, + "Failed to find NodeArg in all parent graphs. Name=", node_arg_name, + " Graph may not conform to the ONNX spec and contain initializers that are not graph inputs."); + } + } + + // add it to the Node's list of implicit inputs + auto& node = *GetNode(nodeid_to_subgraphs.first); + + node.MutableDefinitions().implicit_input_defs.push_back(node_arg); + + if (resolve_context_.inputs_and_initializers.find(node_arg_name) != + resolve_context_.inputs_and_initializers.cend()) { + // no connection required + } else { + // if it's an output nodearg in this graph we need to create a link to the node the output is coming from + auto entry = resolve_context_.output_args.find(node_arg_name); + ONNXRUNTIME_ENFORCE(entry != resolve_context_.output_args.end()); + + // Create relationship between this node (node), and the node providing the output (output_node). + Node& output_node = *entry->second; + AddEdge(output_node.Index(), node.Index(), *node_arg); + + inner_nodes.insert(&output_node); + } + } + } + } + } + + // now build connections within this Graph instance + for (auto& node : Nodes()) { + // Need mutable input defs to be able to set any outer scope NodeArg implicit inputs + auto& input_args = node.MutableInputDefs(); + + if (input_args.size() > 0) { + // This node needs inputs. + + for (const auto* input_arg : input_args) { + if (!input_arg->Exists()) { + // This input could be optional and it does not exist in this case. + continue; + } + + auto output_arg_iter = resolve_context_.output_args.find(input_arg->Name()); + if (resolve_context_.output_args.end() == output_arg_iter) { + // No such output_arg matching this input_arg. + // This input arg should be fed when running evaluation. + // See if it's present in the outer scope. If so it will be 'fed' by the execution frame + // providing access to the MLValue from the outer scope. Pass the name back up so nodes can + // be linked correctly at that level. + if (outer_scope_node_args.find(input_arg->Name()) != outer_scope_node_args.cend()) { + outer_scope_node_args_consumed.push_back(input_arg->Name()); + } + + continue; + } + + // Create relationship between this node (node), and the node providing the output (output_node). + Node& output_node = *output_arg_iter->second; + AddEdge(output_node.Index(), node.Index(), *input_arg); + + inner_nodes.insert(&output_node); + } + } else if (node.OutputDefs().size() <= 0) { + // This is a useless node. + // It has no input/output. + RemoveNode(node.Index()); + } + } + + return Status::OK(); +} + +void Graph::ReverseDFSFrom(const std::vector& from, + const std::function& enter, + const std::function& leave, + const std::function& comp) const { + std::vector node_vec; + for (auto i : from) { + node_vec.push_back(GetNode(i)); + } + + ReverseDFSFrom(node_vec, enter, leave, comp); +} + +void Graph::ReverseDFSFrom(const std::vector& from, + const std::function& enter, + const std::function& leave, + const std::function& comp) const { + using WorkEntry = std::pair; // bool represents leave or not + std::vector stack(from.size()); + for (size_t i = 0; i < from.size(); i++) { + stack[i] = WorkEntry(from[i], false); + } + + std::vector visited(MaxNodeIndex(), false); + while (!stack.empty()) { + const WorkEntry last_entry = stack.back(); + stack.pop_back(); + const Node& n = *last_entry.first; + if (last_entry.second) { + // leave node + leave(&n); + continue; + } + + if (visited[n.Index()]) continue; + + visited[n.Index()] = true; + + if (enter) enter(&n); + + if (leave) stack.emplace_back(&n, true); + + if (comp) { + std::vector sorted_nodes; + for (auto iter = n.InputNodesBegin(); iter != n.InputNodesEnd(); ++iter) { + sorted_nodes.push_back((*iter)); + } + std::sort(sorted_nodes.begin(), sorted_nodes.end(), comp); + for (const auto* in : sorted_nodes) { + const NodeIndex idx = in->Index(); + if (!visited[idx]) { + stack.emplace_back(in, false); + } + } + } else { + for (auto iter = n.InputNodesBegin(); iter != n.InputNodesEnd(); ++iter) { + const NodeIndex idx = (*iter)->Index(); + if (!visited[idx]) { + stack.emplace_back(GetNode(idx), false); + } + } + } + } +} + +GSL_SUPPRESS(es .84) // noisy warning about ignoring return value from insert(...) +Status Graph::PerformTopologicalSortAndCheckIsAcyclic() { + nodes_in_topological_order_.clear(); + // nodes that have been processed and added to nodes_in_topological_order. + std::unordered_set processed_nodes; + std::unordered_set output_nodes; + std::unordered_set nodes_added_for_processing; + std::stack stack; + + // push the top level nodes into nodes_in_topological_order in the order they were added + // to ensure that is consistent. + auto& nodes_in_original_order = Nodes(); + std::for_each(nodes_in_original_order.cbegin(), nodes_in_original_order.cend(), + [&](const Node& node) { + auto index = node.Index(); + + // find the top level nodes in the graph. + // need to also consider nodes that only have Constants as inputs as top level nodes, + // as the constant will get replaced by an initializer. + auto input_edges = node.GetRelationships().input_edges; + auto has_inputs = std::any_of(input_edges.cbegin(), input_edges.cend(), [](const Node::EdgeEnd& edge) { + return edge.GetNode().OpType() != kConstant; + }); + + if (!has_inputs) { + // add to the topological list, and ensure we skip these nodes when walking the graph + nodes_in_topological_order_.push_back(index); + processed_nodes.insert(index); + + // mark this as added as we've fully processed it and don't need to do it again later + nodes_added_for_processing.insert(index); + } + }); + + // start at the bottom and work our way up the graph + for (auto iter = Nodes().begin(); iter != Nodes().end(); ++iter) { + if (0 == iter->relationships_.output_edges.size()) { + // This is a leaf node. + stack.push(iter->Index()); + } + } + + while (!stack.empty()) { + const NodeIndex current = stack.top(); + stack.pop(); + + if (processed_nodes.find(current) != processed_nodes.end()) { + continue; + } + + if (nodes_added_for_processing.find(current) != nodes_added_for_processing.end()) { + // we popped the stack and are back to a node that was added previously, + // so we know all the upstream nodes from it have been fully processed, + nodes_in_topological_order_.push_back(current); + processed_nodes.insert(current); + output_nodes.erase(current); + continue; + } + + const Node* node = GetNode(current); + if (!node) { + continue; + } + + stack.push(current); + output_nodes.insert(current); + + for (auto iter = node->InputNodesBegin(); iter != node->InputNodesEnd(); ++iter) { + const NodeIndex idx = (*iter)->Index(); + if (output_nodes.find(idx) != output_nodes.end()) { + Status status(ONNXRUNTIME, FAIL, "Error: the graph is not acyclic."); + return status; + } + + // avoid re-processing nodes + if (nodes_added_for_processing.find(idx) == nodes_added_for_processing.end()) { + stack.push(idx); + } + } + + nodes_added_for_processing.insert(current); + } + + if (num_of_nodes_ >= 0 && static_cast(num_of_nodes_) == nodes_in_topological_order_.size()) { + return Status::OK(); + } else { + return Status(ONNXRUNTIME, FAIL, "Error: the graph is not acyclic."); + } +} + +bool FullyDefinedType(const TypeProto& type_proto) { + switch (type_proto.value_case()) { + case TypeProto::kTensorType: { + auto& tensor_type = type_proto.tensor_type(); + return tensor_type.has_elem_type() && (tensor_type.elem_type() != TensorProto::UNDEFINED); + } + case TypeProto::kSparseTensorType: { + auto& tensor_type = type_proto.sparse_tensor_type(); + return tensor_type.has_elem_type() && (tensor_type.elem_type() != TensorProto::UNDEFINED); + } + case TypeProto::kSequenceType: { + auto& seq_type = type_proto.sequence_type(); + return seq_type.has_elem_type() && FullyDefinedType(seq_type.elem_type()); + } + case TypeProto::kMapType: { + auto& map_type = type_proto.map_type(); + return map_type.has_key_type() && + (map_type.key_type() != TensorProto::UNDEFINED) && + map_type.has_value_type() && + FullyDefinedType(map_type.value_type()); + } + case TypeProto::kOpaqueType: + return true; + case TypeProto::VALUE_NOT_SET: + default: + return false; + } +} + +// function to handle type/shape inferencing of a subgraph. +// parameters are the Graph instance for the subgraph, the input types from the control flow node that contains +// the subgraph, and the vector to write the output from the inferencing. +using SubgraphInferencingFunc = + std::function&, std::vector&)>; + +class GraphInferencerImpl : public ONNX_NAMESPACE::GraphInferencer { + public: + GraphInferencerImpl(const Node& node, Graph& graph, SubgraphInferencingFunc& inferencing_func) + : node_{node}, graph_{graph}, inferencing_func_{inferencing_func} { + } + + // Perform inferencing on the graph contained in GraphInferencer. + // Returns the graph output types post-inferencing. + // We ignore input_data currently. Re-consider if InferenceContextImpl::getInputData gets implemented + std::vector doInferencing(const std::vector& input_types, + const std::vector& /*input_data*/) override { + std::vector output_types; + + auto status = inferencing_func_(node_, graph_, input_types, output_types); + + if (status != Status::OK()) { + fail_type_inference("Graph attribute inferencing failed: ", status.ErrorMessage()); + } + + return output_types; + } + + private: + const Node& node_; + Graph& graph_; + SubgraphInferencingFunc& inferencing_func_; +}; + +// An implementation of the InferenceContext interface required by operator-specific +// shape inference for onnxruntime graphs. +class InferenceContextImpl : public ONNX_NAMESPACE::InferenceContext { + using AttributeGraphMap = std::unordered_map; + + public: + InferenceContextImpl(Node& node, + const AttributeGraphMap* subgraphs = nullptr, + SubgraphInferencingFunc* subgraph_inferencing_func = nullptr, + const InitializedTensorSet& initialized_tensor_set = {}) noexcept + : node_(node), + attr_to_subgraph_map_{subgraphs}, + subgraph_inferencing_func_{subgraph_inferencing_func}, + initialized_tensor_set_(initialized_tensor_set) { + node_output_types_.resize(node.OutputDefs().size()); + } + + void RunInferencing() { + auto schema = node_.Op(); + if (nullptr != schema) { + schema->GetTypeAndShapeInferenceFunction()(*this); + } + } + + const std::vector InferredOutputTypes() const { return node_output_types_; } + + const AttributeProto* getAttribute(const std::string& name) const override { + auto& attribute_value_map = node_.GetAttributes(); + auto iter = attribute_value_map.find(name); + if (iter == attribute_value_map.end()) { + return nullptr; + } else { + return &iter->second; + } + } + + size_t getNumInputs() const noexcept override { + return node_.InputDefs().size(); + } + + const TypeProto* getInputType(size_t index) const override { + auto p_node_arg = node_.InputDefs().at(index); + if ((nullptr != p_node_arg) && p_node_arg->Exists()) { + return p_node_arg->TypeAsProto(); + // auto p_type_proto = p_node_arg->TypeAsProto(); + //if ((p_type_proto != nullptr) && p_type_proto->has_tensor_type()) { + // return &p_type_proto->tensor_type(); + //} + } + return nullptr; + } + + size_t getNumOutputs() const noexcept override { + return node_output_types_.size(); + } + + TypeProto* getOutputType(size_t index) override { + return &node_output_types_[index]; + } + + const TensorProto* getInputData(size_t index) const override { + auto def = node_.InputDefs()[index]; + if (!def) + return nullptr; + if (initialized_tensor_set_.count(def->Name()) == 0) + return nullptr; + return initialized_tensor_set_.at(def->Name()); + } + + GraphInferencer* getGraphAttributeInferencer(const std::string& attribute_name) override { + GraphInferencer* graph_inferencer = nullptr; + + if (attr_to_subgraph_map_ && subgraph_inferencing_func_) { + auto attr_to_subgraph = attr_to_subgraph_map_->find(attribute_name); + if (attr_to_subgraph != attr_to_subgraph_map_->cend()) { + auto inferencer = std::make_unique(node_, *attr_to_subgraph->second, + *subgraph_inferencing_func_); + graph_inferencer = inferencer.get(); + graph_inferencers_.push_back(std::move(inferencer)); + } else { + fail_type_inference("No Graph instance was found for attribute ", + attribute_name, " in node ", node_.Name()); + } + } + + return graph_inferencer; + } + + private: + Node& node_; + // node_output_types_ will be populated by the operator-specific shape inference. + std::vector node_output_types_; + const AttributeGraphMap* attr_to_subgraph_map_; + SubgraphInferencingFunc* subgraph_inferencing_func_; + std::vector> graph_inferencers_; + const InitializedTensorSet& initialized_tensor_set_; +}; + +Status Graph::InferAndVerifySubgraphTypes(const Node& node, Graph& subgraph, + const std::vector& input_types, + std::vector& output_types) { + auto status = Status::OK(); + + output_types.clear(); + + auto& subgraph_inputs = subgraph.GetInputs(); + auto num_subgraph_inputs = subgraph_inputs.size(); + + if (num_subgraph_inputs != input_types.size()) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Size mismatch validating subgraph inputs. Got ", + input_types.size(), " inputs but subgraph requires ", subgraph_inputs.size()); + } + + // apply type/shape info to the subgraph's inputs + for (size_t i = 0; i < num_subgraph_inputs; ++i) { + const auto& input_type = *input_types[i]; + const auto& subgraph_input = *subgraph_inputs[i]; + + NodeArg* mutable_nodearg = subgraph.GetNodeArg(subgraph_input.Name()); + status = mutable_nodearg->UpdateTypeAndShape(input_type); + if (!status.IsOK()) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Node:", node.Name(), " ", status.ErrorMessage()); + } + } + + // Apply any current input type/shape information to the Nodes in the subgraph that are implicitly + // consuming NodeArg's from this scope or higher. + // The NodeArg's that implicit_input_defs point to would have any type/shape inferencing applied to them + // by now. As the subgraph is referring to the outer scope NodeArg, we simply replace any information in + // the subgraph with the details from the outer scope NodeArg. + auto implicit_input_defs = node.GetDefinitions().implicit_input_defs; + for (const auto* implicit_node_arg : implicit_input_defs) { + auto subgraph_nodearg = subgraph.GetNodeArg(implicit_node_arg->Name()); + + // the implicit input defs may be for a nested subgraph, so it won't necessarily match here. + // if that is the case, we will update the type/shape information when we descend into the + // nested subgraph later. + if (!subgraph_nodearg) + continue; + + status = subgraph_nodearg->UpdateTypeAndShape(*implicit_node_arg); + if (!status.IsOK()) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Node:", node.Name(), " ", status.ErrorMessage()); + } + + // all values above us should have a type by now due to ONNX requirements. + if (subgraph_nodearg->Type() == nullptr) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Subgraph input missing type."); + } + + // now that we have handled the input types, do the type/shape inferencing for the subgraph + // to flow the type/shape info through it + status = subgraph.PerformTypeAndShapeInferencing(); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + auto& subgraph_outputs = subgraph.GetOutputs(); + for (const auto* output : subgraph_outputs) { + output_types.push_back(output->TypeAsProto()); + } + + return Status::OK(); +} + +// Implementation of type-inference and type-checking for a single node +GSL_SUPPRESS(f .23) // spurious warning about inferred_type never being checked for null +Status Graph::InferAndVerifyTypeMatch(Node& node, const OpSchema& op) { + auto& node_name = node.Name(); + + // if we're building a graph we permit outer scope node args to have no type + // as the 'real' Resolve at runtime will have type inferencing + auto is_outer_scope_nodearg = [this](const std::string& name) { + return outer_scope_node_arg_names_.find(name) != outer_scope_node_arg_names_.cend(); + }; + + // index used to navigate node->InputDefs(). + int k = 0; + std::unordered_map type_parameter_to_type_map; + + for (size_t i = 0; i < node.InputArgCount().size(); ++i) { + // Number of inputs corresponding to the i-th argument. + const int arg_count = node.InputArgCount()[i]; + // The i-th formal parameter definition. + auto op_formal_parameter = op.inputs()[i]; + + // Check all actual parameters (corresponding to the k-th input) + // match the formal parameter definition (i-th argument). + for (int j = 0; j < arg_count; ++j, ++k) { + auto& input_def = node.MutableDefinitions().input_defs[k]; + if (!input_def->Exists()) + continue; + + if (input_def->Type() == nullptr) { + // if we are building a subgraph that uses outer scope values, + // allow an empty type as it will be copied from the outer scope graph at runtime + if (is_outer_scope_nodearg(input_def->Name())) + continue; + + // Logic error: This should not happen if we properly checked that every use has + // a corresponding def, for which type-inference already produced a valid type + Status status(ONNXRUNTIME, FAIL, + "Node (" + node_name + ") input arg (" + + input_def->Name() + ") does not have type information set by parent node."); + return status; + } + + // Verify that the actual parameter's type is one of permitted types of the formal parameter + DataType input_type = input_def->Type(); + auto& permitted_types = op_formal_parameter.GetTypes(); + if (0 == permitted_types.count(input_type)) { + std::string null_pointer("(null)"); + if (input_type == nullptr) input_type = &null_pointer; + // Type error in input model/graph. + + Status status(ONNXRUNTIME, INVALID_GRAPH, + "Type Error: Type '" + *input_type + "' of input parameter (" + input_def->Name() + + ") of operator (" + op.Name() + ") in node (" + node_name + ") is invalid."); + return status; + } + + // When multiple parameters have the same type-variable, they are all required + // to have the same type. E.g., when adding tensors A and B, it is an error if + // input A is of type "tensor(int32)" and B is of type "tensor(float)". + // For variadic arguments, this verification rule is normally applicable: + // e.g., Concat/Max/Mean/Min/Sum all require all input tensors to be of same type. + // However, some ops, like the control-flow constructs (Scan, If, Loop) have variadic + // inputs and outputs of different types. The check is not applicable to such ops. + if (op_formal_parameter.GetIsHomogeneous()) { + auto param_to_type_iter = type_parameter_to_type_map.find(op_formal_parameter.GetTypeStr()); + if (type_parameter_to_type_map.end() == param_to_type_iter) { + // Bind the corresponding type-parameter's value to the actual type: + type_parameter_to_type_map[op_formal_parameter.GetTypeStr()] = input_type; + } else if (param_to_type_iter->second != input_type) { + // Type error in input model/graph: + // The type-parameter T is bound to different values for different inputs. + Status status(ONNXRUNTIME, FAIL, + "Type Error: Type parameter (" + op_formal_parameter.GetTypeStr() + + ") bound to different types (" + *(param_to_type_iter->second) + + " and " + *(input_def->Type()) + + " in node (" + node_name + ")."); + return status; + } + } + } + } + + // Apply ONNX's type/shape inference to this node. + // This will call InferAndVerifySubgraphTypes if the ONNX level type/shape inferencing for the Node attempts + // to do subgraph type/shape inferencing (Scan/If/Loop nodes). + // InferAndVerifySubgraphTypes will call PerformTypeAndShapeInferencing for the subgraph, which will recursively + // handle type/shape inferencing for it. + // Once that completes, the outputs from the node containing the subgraph will be updated, and the final values + // returned here. + SubgraphInferencingFunc func(Graph::InferAndVerifySubgraphTypes); + auto node_subgraphs = subgraph_map_.find(node.Index()); + auto* subgraphs = node_subgraphs != subgraph_map_.cend() ? &node_subgraphs->second : nullptr; + InferenceContextImpl context(node, subgraphs, &func, name_to_initial_tensor_); + + try { + context.RunInferencing(); + } catch (const std::exception& ex) { + return Status(ONNXRUNTIME, FAIL, ex.what()); + } + + const auto& onnx_inferred_types{context.InferredOutputTypes()}; + + // Infer and verify node output arg type information. + int i = -1; + for (auto& output_def : node.MutableDefinitions().output_defs) { + ++i; + if (!output_def->Exists()) continue; + + // if the number of actual parameters exceeds the number of formal parameters, + // then the op has variadic outputs and the trailing extra actual parameters + // correspond to the last formal parameter. (The ONNX schema verification check + // would have checked that the corresponding formal parameter is variadic.) + + const int num_formal_params = gsl::narrow_cast(op.outputs().size()); + auto operand_index = std::min(i, num_formal_params - 1); + auto op_formal_parameter = op.outputs().at(operand_index); + + const TypeProto& onnx_inferred_type = onnx_inferred_types[i]; + DataType existing_type = output_def->Type(); + DataType inferred_type = nullptr; + + // Infer output arg type if it is constrained to be of the same type as some input: + // For example, the output of "Abs" is of the same type as its input. + bool homogeneous = op_formal_parameter.GetIsHomogeneous(); + auto input_types_iter = type_parameter_to_type_map.find(op_formal_parameter.GetTypeStr()); + if (homogeneous && (type_parameter_to_type_map.end() != input_types_iter)) { + inferred_type = input_types_iter->second; + } else if (1 == op_formal_parameter.GetTypes().size()) { + // Infer output arg type if operator definition specifies unique output type: + inferred_type = *(op_formal_parameter.GetTypes().begin()); + } else if (FullyDefinedType(onnx_inferred_type)) { + // Use output type inferred by ONNX inference + inferred_type = DataTypeUtils::ToType(onnx_inferred_type); + } else if (existing_type != nullptr) { + inferred_type = existing_type; + } else { + // This should not happen: indicates incompleteness in ONNX inference. + Status status(ONNXRUNTIME, FAIL, + "Node (" + node_name + ") output arg (" + output_def->Name() + ") type inference failed"); + return status; + } + + if ((existing_type != inferred_type) && (existing_type != nullptr)) { + // A type exists for this output but does not match the inferred type. + return Status(ONNXRUNTIME, FAIL, + "Type Error: Type (" + *existing_type + ") of output arg (" + + output_def->Name() + ") of node (" + node_name + + ") does not match expected type (" + *inferred_type + ")."); + } + + if (existing_type == nullptr) + output_def->SetType(inferred_type); + + // Update output-shape if it was inferred: + if (onnx_inferred_type.has_tensor_type()) { + auto& tensor_type = onnx_inferred_type.tensor_type(); + if (tensor_type.has_shape()) { + if (output_def->Shape() == nullptr) { + output_def->SetShape(tensor_type.shape()); + } else { + // we need to merge the shapes as a subgraph may have placeholder dimensions to represent the rank + // that have no values. + TypeProto_Tensor merge_target; + (*merge_target.mutable_shape()) = *output_def->Shape(); + auto status = MergeShapeInfo(output_def->Name(), tensor_type, merge_target); + if (!status.IsOK()) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Node:", node_name, " ", status.ErrorMessage()); + } + + output_def->SetShape(merge_target.shape()); + } + } + } + } + + return Status::OK(); +} + +// Apply type-inference and type-checking to all inputs and initializers: +common::Status Graph::TypeCheckInputsAndInitializers() { + // Check that the type of every input is specified: + for (auto* graph_input : GetInputs()) { + if (nullptr == graph_input->Type()) { + Status status(ONNXRUNTIME, FAIL, "Model input (" + graph_input->Name() + ") does not have type information."); + return status; + } + } + + // Note: The ONNX spec requires every initializer to be included in the graph input, + // but onnxruntime relaxes this requirement for various reasons. + + // Infer/check type and shape for all initializers from their values + for (auto& initializer_pair : name_to_initial_tensor_) { + const std::string& name = initializer_pair.first; + auto* node_arg = GetNodeArg(name); + // If node_arg is null, we ignore this as a potentially unused initializer here + if (nullptr != node_arg) { + const TensorProto* tensor_proto = initializer_pair.second; + TypeProto tensor_type; + tensor_type.mutable_tensor_type()->set_elem_type(tensor_proto->data_type()); + auto inferred_type = DataTypeUtils::ToType(tensor_type); + auto existing_type = node_arg->Type(); + if (nullptr == existing_type) + node_arg->SetType(inferred_type); + else if (inferred_type != existing_type) { + return Status(ONNXRUNTIME, FAIL, + "Type Error: Value of initializer " + name + " does not match its type."); + } + + // Set shape accordingly. + TensorShapeProto inferred_shape; + for (auto dim : tensor_proto->dims()) { + inferred_shape.add_dim()->set_dim_value(dim); + } + const TensorShapeProto* p_existing_shape = node_arg->Shape(); + if (nullptr == p_existing_shape) + node_arg->SetShape(inferred_shape); + else { + if (p_existing_shape->dim_size() != tensor_proto->dims_size()) + return Status(ONNXRUNTIME, FAIL, + "Type Error: Shape of initializer " + name + " does not match its type."); + for (int i = 0; i < p_existing_shape->dim_size(); ++i) { + auto& d = p_existing_shape->dim(i); + if (d.has_dim_value() && (d.dim_value() != tensor_proto->dims(i))) + return Status(ONNXRUNTIME, FAIL, + "Type Error: Shape of initializer " + initializer_pair.first + " does not match its type."); + } + } + } + } + return Status::OK(); +} + +Status Graph::VerifyNodeAndOpMatch() { + CheckerContext ctx; + ctx.set_ir_version(gsl::narrow_cast(IrVersion())); + ctx.set_opset_imports(DomainToVersionMap()); + ctx.set_schema_registry(schema_registry_.get()); + + LexicalScopeContext lsc{resolve_context_.inputs_and_initializers}; + + // technically we could add values from Node.GetDefinitions().implicit_input_defs on a per-node basis inside + // the below loop so that we only check against the specific outer dependencies of the node. + // doing that requires lots of copies of LexicalScopeContext.output_names to clear out the per-Node values + // after each loop. instead add all the outer scope values upfront so we can just accumulate new inner scope values + // during each loop iteration. + lsc.output_names.insert(resolve_context_.outer_scope_node_args.cbegin(), + resolve_context_.outer_scope_node_args.cend()); + + for (auto node_index : nodes_in_topological_order_) { + // Node verification. + auto& node = *GetNode(node_index); + + NodeProto node_proto; + node.ToProto(node_proto); + auto& node_name = node.Name(); + auto& domain = node.Domain(); + + auto iter = model_functions_.find(node.OpType()); + if (iter != model_functions_.end()) { + const ONNX_NAMESPACE::FunctionProto* model_function_proto = iter->second; + auto model_func_ptr = std::make_unique(*this, node.Index(), model_function_proto); + function_container_.emplace_back(std::move(model_func_ptr)); + node.SetFunctionBody(*function_container_.back()); + } + + if (!node.Op()) { + try { + checker::check_node(node_proto, ctx, lsc); + } catch (const std::exception& ex) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_GRAPH, "Node:", node_name, " ", ex.what()); + } + + auto maxInclusiveVersion = DomainToVersionMap().find(domain)->second; + node.op_ = schema_registry_->GetSchema(node.OpType(), maxInclusiveVersion, node.Domain()); + + if (node.op_->Deprecated()) { + node.op_ = nullptr; + } + + if (!node.op_) { + ONNX_NAMESPACE::FunctionBuilderRegistry& function_registry = + FunctionBuilderRegistry::OnnxInstance(); + auto onnx_function_proto = function_registry.GetFunction(node.OpType(), maxInclusiveVersion, ONNX_DOMAIN); + if (!onnx_function_proto) { + return Status(ONNXRUNTIME, FAIL, "Fatal error: " + node.OpType() + " is not a registered function/op"); + } + auto func_ptr = std::make_unique(*this, node.Index(), onnx_function_proto); + function_container_.emplace_back(std::move(func_ptr)); + node.SetFunctionBody(*function_container_.back()); + } + } + + ONNXRUNTIME_RETURN_IF_ERROR(node.UpdateInputArgCount()); + + // currently an Op is required by ValidateVersion, so we use gsl::not_null to validate that. + // This may change in the future to allow a null Op + const gsl::not_null p_op{node.Op()}; + + // Attribute verification and fill node attribute with + // default value defined in operator definition if needed. + // Fill node attribute with default value specified in operator definition if any. + auto node_attributes = node.GetAttributes(); + for (auto attr_def : p_op->attributes()) { + auto node_attr_iter = node_attributes.find(attr_def.first); + if (node_attributes.end() == node_attr_iter) { + // The attribute was not specified in the node. + if (!attr_def.second.required) { + if (attr_def.second.default_value.has_name()) { + // Set default value to the node attributes. + node.AddAttribute(attr_def.first, attr_def.second.default_value); + } + // TODO: Handle optional attribute but no default value specified in op definition. + } else { + Status status(ONNXRUNTIME, FAIL, + "Node (" + node_name + ") attribute (" + attr_def.first + + ") is required but not specified."); + return status; + } + } + } + + NO_CHANGE_ON_SYNC_FLAG(ONNXRUNTIME_RETURN_IF_ERROR(InferAndVerifyTypeMatch(node, *p_op))); + + // Accumulate output names of the iterated Node + for (auto& output_name : node_proto.output()) { + lsc.output_names.insert(output_name); + } + } + + return Status::OK(); +} + +Graph* Graph::GetMutableSubgraph(const NodeIndex node_index, const std::string& attribute_name) { + const Graph* subgraph = GetSubgraph(node_index, attribute_name); + return const_cast(subgraph); +} + +const Graph* Graph::GetSubgraph(const NodeIndex node_index, const std::string& attribute_name) const { + Graph* subgraph = nullptr; + + auto entry = subgraph_map_.find(node_index); + + if (entry != subgraph_map_.cend()) { + auto& name_to_subgraph_map = entry->second; + auto subgraph_iter = name_to_subgraph_map.find(attribute_name); + if (subgraph_iter != name_to_subgraph_map.cend()) { + subgraph = subgraph_iter->second; + } + } + + return subgraph; +} + +Status Graph::CreateSubgraphs() { + Status status = Status::OK(); + + // don't use NodesInTopologicalOrder as we want CreateSubgraphs to recursively create subgraphs with no + // dependency on PerformTopologicalSortAndCheckIsAcyclic having been called previously + // to populate NodesInTopologicalOrder + for (auto& node : Nodes()) { + auto node_index = node.Index(); + if (subgraph_map_.find(node_index) != subgraph_map_.cend()) { + // if we have an existing entry we have processed this node previously. + // as the subgraph is loaded from a static GraphProto we assume nothing in + // it could have changed and there's no point re-creating it. + continue; + } + + // check attributes of all nodes looking for GraphProto attributes, and create + // the Graph instance for the subgraph contained in the GraphProto. + for (auto& attr : node.attributes_) { + bool has_subgraph = attr.second.has_g(); + if (has_subgraph) { + auto& attr_name = attr.first; + auto entry = subgraph_map_.find(node_index); + + // make sure this is new. internal logic error if it is not so using ONNXRUNTIME_ENFORCE. + if (entry != subgraph_map_.cend()) { + const auto& existing_entries = entry->second; + ONNXRUNTIME_ENFORCE(existing_entries.find(attr_name) == existing_entries.cend(), + "Entry exists in node ", node_index, " for attribute ", attr_name); + } + + auto& graph_proto = *attr.second.mutable_g(); + + // create instance. need to call private ctor so can't use make_unique + GSL_SUPPRESS(r .11) + std::unique_ptr subgraph{new Graph(*this, graph_proto)}; + + // Recursively create any further subgraphs + status = subgraph->CreateSubgraphs(); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + subgraph_map_[node_index][attr_name] = subgraph.get(); + subgraphs_.push_back(std::move(subgraph)); + } + } + } + + return Status::OK(); +} + +Status Graph::VerifyInputAndInitializerNames() { + std::unordered_set& inputs_and_initializers = resolve_context_.inputs_and_initializers; + + for (auto* input : GetInputs()) { + auto result = inputs_and_initializers.insert(input->Name()); + if (!result.second) { + Status status(ONNXRUNTIME, FAIL, + "Error: Duplicate definition-site for (" + input->Name() + ")."); + return status; + } + } + + for (auto& initializer_pair : name_to_initial_tensor_) { + GSL_SUPPRESS(es .84) + inputs_and_initializers.insert(initializer_pair.first); + // Initializers are expected to be included in inputs (according to ONNX spec). + // onnxruntime relaxes this constraint. No duplicate-name check here. + } + + return Status::OK(); +} + +Status Graph::InitInputsInitializersOutputs() { + resolve_context_.Clear(); + + // clear the previous relationships, as we re-create them when resolving. + // same applies to the implicit input defs as they are built from any subgraphs within this graph. + for (auto& node : Nodes()) { + node.MutableRelationships().Clear(); + node.MutableDefinitions().implicit_input_defs.clear(); + } + + // add the subgraph pointers to the resolve context. + for (auto& nodeid_to_subgraphs : subgraph_map_) { + resolve_context_.node_to_subgraphs_map[nodeid_to_subgraphs.first] = {}; + + for (auto& attr_name_to_subgraph : nodeid_to_subgraphs.second) { + resolve_context_.node_to_subgraphs_map[nodeid_to_subgraphs.first].push_back(attr_name_to_subgraph.second); + } + } + + ONNXRUNTIME_RETURN_IF_ERROR(SetGraphInputsOutputs()); + ONNXRUNTIME_RETURN_IF_ERROR(VerifyInputAndInitializerNames()); + ONNXRUNTIME_RETURN_IF_ERROR(VerifyNoDuplicateName()); + + return Status::OK(); +} + +Status Graph::PerformTypeAndShapeInferencing() { + ONNXRUNTIME_RETURN_IF_ERROR(TypeCheckInputsAndInitializers()); + + // type/shape inferencing on the nodes is done recursively as we need subgraph outputs + // to be applied to Node outputs for the node containing the subgraph. + // Call path is + // VerifyNodeAndOpMatch + // Iterates Nodes + // Runs ONNX type/shape inferencing for each Node + // - If it hits a node with a subgraph, InferenceContext::getGraphAttributeInferencer is called + // by the ONNX level type/shape inferencing, which updates the subgraph inputs using GraphInferencerImpl + // - GraphInferencerImpl::doInferencing calls PerformTypeShapeInferencing to execute type/shape inferencing + // for all nodes in the subgraph. This leads to recursively handling all subgraphs contained in the node. + // - once we finish processing the subgraph/s we apply resultant type/shape information to the outputs + // of the node that contains the subgraph. + ONNXRUNTIME_RETURN_IF_ERROR(VerifyNodeAndOpMatch()); + + return Status::OK(); +} + +Status Graph::ForThisAndAllSubgraphs(std::function func) { + auto status = func(*this); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + for (auto& subgraph : subgraphs_) { + status = func(*subgraph); + ONNXRUNTIME_RETURN_IF_ERROR(status); + } + return status; +} + +Status Graph::Resolve() { + return Resolve(false); +} + +Status Graph::Resolve(bool no_proto_sync_required) { + if (parent_graph_) { + // Resolve must start at the top level graph in-order to handle outer scope + // connections correctly, so recurse up to that level to start + auto status = parent_graph_->Resolve(no_proto_sync_required); + return status; + } + + bool subgraphs_need_resolve = std::any_of(subgraphs_.cbegin(), subgraphs_.cend(), + [](const std::unique_ptr& graph) { + return graph->GraphResolveNeeded(); + }); + + if (!GraphResolveNeeded() && !subgraphs_need_resolve) { + return Status::OK(); + } + + // Create the Graph instances for the subgraph/s in any nodes containing GraphProto attributes (Scan/If/Loop). + // Do this upfront so we can recurse into them when building connections and doing type/shape inferencing. + // Recursively creates any nested subgraphs. + ONNXRUNTIME_RETURN_IF_ERROR(CreateSubgraphs()); + + // init all graph/subgraphs. non-recursive. + auto init_func = [](Graph& graph) { return graph.InitInputsInitializersOutputs(); }; + ONNXRUNTIME_RETURN_IF_ERROR(ForThisAndAllSubgraphs(init_func)); + + // recursively set the outer scope node args. + ONNXRUNTIME_RETURN_IF_ERROR(SetOuterScopeNodeArgs(resolve_context_.outer_scope_node_args)); + + std::vector outer_scope_node_args_consumed; + + // recursively build connections between nodes in this graph and all subgraphs + ONNXRUNTIME_RETURN_IF_ERROR(BuildConnections(outer_scope_node_args_consumed)); + ONNXRUNTIME_ENFORCE(outer_scope_node_args_consumed.empty(), + "Shouldn't be possible to have NodeArgs that haven't been handled already."); + + // topological sort of this and any subgraphs is non-recursive + auto topo_sort_func = [](Graph& graph) { return graph.PerformTopologicalSortAndCheckIsAcyclic(); }; + ONNXRUNTIME_RETURN_IF_ERROR(ForThisAndAllSubgraphs(topo_sort_func)); + + // type/shape validation and inferencing on this and any subgraphs + // recurses into subgraphs via the ONNX checker, which descends into the GraphProto in node attributes + // which define a subgraph. + ONNXRUNTIME_RETURN_IF_ERROR(PerformTypeAndShapeInferencing()); + + // perform the final steps for this graph and all subgraphs + auto finalize_func = [&no_proto_sync_required](Graph& graph) { + graph.CleanUnusedInitializers(); + graph.GraphResolveNeeded(false); + + // if we are resolving immediately after loading from a GraphProto, we don't need to + // do a proto sync + if (no_proto_sync_required) { + graph.GraphProtoSyncNeeded(false); + } + + return Status::OK(); }; + + ONNXRUNTIME_RETURN_IF_ERROR(ForThisAndAllSubgraphs(finalize_func)); + + return Status::OK(); +} + +const std::string& Graph::Name() const noexcept { + return graph_proto_->name(); +} + +void Graph::SetName(const std::string& name) { + graph_proto_->set_name(name); +} + +const std::string& Graph::Description() const noexcept { + return graph_proto_->doc_string(); +} + +void Graph::SetDescription(const std::string& description) { + graph_proto_->set_doc_string(description); +} + +void Graph::AddInitializedTensor(const TensorProto& tensor) { + if (name_to_initial_tensor_.end() != name_to_initial_tensor_.find(tensor.name())) { + return; + } + + const gsl::not_null tensor_added{graph_proto_->add_initializer()}; + *(tensor_added) = tensor; + name_to_initial_tensor_[tensor.name()] = tensor_added; + + if (!GraphLoadedFromModelFile(graph_proto_)) { + // make sure there is a NodeArg for the initializer as SetGraphInputsOutputs will add it to the graph inputs + TypeProto t; + t.mutable_tensor_type()->set_elem_type(tensor.data_type()); + auto shape = t.mutable_tensor_type()->mutable_shape(); + for (auto dim : tensor.dims()) + shape->add_dim()->set_dim_value(dim); + + ONNXRUNTIME_IGNORE_RETURN_VALUE(GetOrCreateNodeArg(tensor.name(), &t)); + } + + SetGraphProtoSyncNeeded(); + SetGraphResolveNeeded(); +} + +void Graph::RemoveInitializedTensor(const std::string& tensor_name) { + auto iter = name_to_initial_tensor_.find(tensor_name); + if (name_to_initial_tensor_.end() != iter) { + name_to_initial_tensor_.erase(tensor_name); + SetGraphProtoSyncNeeded(); + SetGraphResolveNeeded(); + } +} + +bool Graph::GetInitializedTensor(const std::string& tensor_name, const TensorProto*& value) const { + auto iter = name_to_initial_tensor_.find(tensor_name); + if (name_to_initial_tensor_.end() == iter) { + value = nullptr; + return false; + } + value = iter->second; + return true; +} + +void Graph::CleanAllInitializedTensors() noexcept { + name_to_initial_tensor_.clear(); + removed_initializer_indexes_.clear(); + + // Clearing RepeatedPtrFields does not free objects' memory. The memory is retained + // and can be reused. Need to explicitly release the cleared objects and free the + // memory. + graph_proto_->mutable_initializer()->Clear(); + const int num_cleared = graph_proto_->initializer().ClearedCount(); + for (int i = 0; i < num_cleared; i++) { + delete graph_proto_->mutable_initializer()->ReleaseCleared(); + } +} + +const InitializedTensorSet& Graph::GetAllInitializedTensors() const noexcept { + return name_to_initial_tensor_; +} + +const std::vector& Graph::GetValueInfo() const noexcept { + return value_info_; +} + +std::vector Graph::CreateNodeArgs(const google::protobuf::RepeatedPtrField& names, + const ArgNameToTypeMap& name_to_type_map) { + const auto name_to_type_map_end = name_to_type_map.end(); + std::vector results; + results.reserve(names.size()); + + for (auto& name : names) { + const TypeProto* type = nullptr; + + auto name_to_type_iter = name_to_type_map.find(name); + if (name_to_type_iter != name_to_type_map_end) { + // This node input arg type/shape does exist in graph proto. + // Assign type/shape information to node input arg. + type = &(name_to_type_iter->second); + } + + auto node_arg = &GetOrCreateNodeArg(name, type); + results.push_back(node_arg); + } + + return results; +} + +Node* Graph::AddNode(const Node& other) { + const auto& definitions = other.GetDefinitions(); + + auto new_node = AddNode(other.Name(), other.OpType(), other.Description(), + definitions.input_defs, + definitions.output_defs, + &other.GetAttributes(), + other.Domain()); + + return new_node; +} + +Node* Graph::AddNode(const NodeProto& node_proto, + const ArgNameToTypeMap& name_to_type_map) { + auto input_defs = CreateNodeArgs(node_proto.input(), name_to_type_map); + auto output_defs = CreateNodeArgs(node_proto.output(), name_to_type_map); + + const int num_attributes = node_proto.attribute_size(); + NodeAttributes attributes; + attributes.reserve(num_attributes); + + for (int i = 0; i < num_attributes; ++i) { + auto& attr = node_proto.attribute(i); + attributes[attr.name()] = attr; + } + + return AddNode(node_proto.name(), + node_proto.op_type(), + node_proto.doc_string(), + input_defs, + output_defs, + &attributes, + node_proto.domain()); +} + +std::string Graph::GenerateNodeArgName(const std::string& base_name) { + std::string new_name; + do { + std::ostringstream str; + str << base_name << "_" << name_generator_++; + new_name = str.str(); + } while (node_args_.find(new_name) != node_args_.end()); + return new_name; +} + +std::string Graph::GenerateNodeName(const std::string& base_name) { + std::string new_name; + bool keep_going = true; + + do { + std::ostringstream str; + str << base_name << "_" << name_generator_++; + new_name = str.str(); + + keep_going = std::find_if(nodes_.cbegin(), nodes_.cend(), [&new_name](const std::unique_ptr& n) { + return (n != nullptr) && (n->Name() == new_name); + }) != nodes_.end(); + } while (keep_going); + + return new_name; +} + +Node* Graph::AddNode(const std::string& name, + const std::string& op_type, + const std::string& description, + const std::vector& input_args, + const std::vector& output_args, + const NodeAttributes* attributes, + const std::string& domain) { + std::vector inputs, outputs; + inputs.resize(input_args.size()); + outputs.resize(output_args.size()); + int i = 0; + for (auto input_arg : input_args) { + inputs[i++] = &GetOrCreateNodeArg(input_arg->Name(), input_arg->TypeAsProto()); + } + i = 0; + for (auto output_arg : output_args) { + outputs[i++] = &GetOrCreateNodeArg(output_arg->Name(), output_arg->TypeAsProto()); + } + + const gsl::not_null node = AllocateNode(); + node->Init(name, op_type, description, inputs, outputs, attributes, domain); + if (0 != op_type.compare(kNoOp)) { + graph_proto_sync_needed_ = true; + } + + return node; +} + +bool Graph::RemoveNode(NodeIndex p_index) { + return ReleaseNode(p_index); +} + +bool Graph::AddControlEdge(NodeIndex src_node_index, NodeIndex dst_node_index) { + if (nodes_.size() <= src_node_index || + nodes_.size() <= dst_node_index || + nullptr == nodes_[src_node_index] || + nullptr == nodes_[dst_node_index]) { + // Invalid node indexes specified. + return false; + } + + GSL_SUPPRESS(es .84) { // ignoring return from insert() + nodes_[src_node_index]->MutableRelationships().output_edges.insert(Node::EdgeEnd(*nodes_[dst_node_index])); + nodes_[dst_node_index]->MutableRelationships().input_edges.insert(Node::EdgeEnd(*nodes_[src_node_index])); + nodes_[dst_node_index]->MutableRelationships().control_inputs.insert(nodes_[src_node_index]->Name()); + } + + return true; +} + +const GraphProto& Graph::ToGraphProto() { + if (!GraphProtoSyncNeeded()) { + return *graph_proto_; + } + + // Nodes. + graph_proto_->clear_node(); + GraphViewer graph_viewer(*this); + // Nodes must be sorted in Topological Order in the GraphProto per ONNX spec. + for (auto& node_idx : graph_viewer.GetNodesInTopologicalOrder()) { + const gsl::not_null node_proto{graph_proto_->add_node()}; + const gsl::not_null p_node{GetNode(node_idx)}; + p_node->ToProto(*node_proto); + } + + if (!removed_initializer_indexes_.empty()) { + // Move initializers. + std::sort(removed_initializer_indexes_.begin(), removed_initializer_indexes_.end()); + int lastInUseInitializerIndex = graph_proto_->initializer_size() - 1; + int start = 0, end = gsl::narrow_cast(removed_initializer_indexes_.size()) - 1; + int lastRemovedInitializerIndex = removed_initializer_indexes_[end]; + + for (; start <= end; start++) { + // Find a lastInUseInitializer. + while (start <= end && lastInUseInitializerIndex == lastRemovedInitializerIndex) { + graph_proto_->mutable_initializer()->RemoveLast(); + lastInUseInitializerIndex--; + end--; + if (start <= end) { + lastRemovedInitializerIndex = removed_initializer_indexes_[end]; + } + } + + if (start <= end) { + // Copy the initializer in use to the slot which is removed. + *graph_proto_->mutable_initializer(removed_initializer_indexes_[start]) = graph_proto_->initializer(lastInUseInitializerIndex); + graph_proto_->mutable_initializer()->RemoveLast(); + lastInUseInitializerIndex--; + } + } + removed_initializer_indexes_.clear(); + } + + // Sync graph inputs/outputs/valueInfo. + SyncGraphInputsOutputs(); + + GraphProtoSyncNeeded(false); + + return *graph_proto_; +} + +void Graph::SyncGraphInputsOutputs() { + graph_proto_->clear_input(); + graph_proto_->clear_output(); + graph_proto_->clear_value_info(); + + for (const auto* input_arg : GetInputsIncludingInitializers()) { + *(graph_proto_->mutable_input()->Add()) = input_arg->ToProto(); + } + + for (const auto* output_arg : GetOutputs()) { + *(graph_proto_->mutable_output()->Add()) = output_arg->ToProto(); + } + + for (const auto* value_info : value_info_) { + *(graph_proto_->mutable_value_info()->Add()) = value_info->ToProto(); + } +} + +void Graph::CleanUnusedInitializers() { + std::unordered_set used_args; + + const auto& inputs = GetInputs(); + const auto& outputs = GetOutputs(); + + std::for_each(inputs.cbegin(), inputs.cend(), [&used_args](const NodeArg* input) { + ONNXRUNTIME_IGNORE_RETURN_VALUE(used_args.insert(input->Name())); + }); + + std::for_each(outputs.cbegin(), outputs.cend(), [&used_args](const NodeArg* output) { + ONNXRUNTIME_IGNORE_RETURN_VALUE(used_args.insert(output->Name())); + }); + + for (const auto& node : Nodes()) { + node.ForEachInputDef([&used_args](const onnxruntime::NodeArg* def) { + ONNXRUNTIME_IGNORE_RETURN_VALUE(used_args.insert(def->Name())); + }); + } + + std::vector erase_list; + auto end = used_args.end(); + for (const auto& pv : name_to_initial_tensor_) { + const std::string& name = pv.first; + if (used_args.find(name) == end) { + LOGS_DEFAULT(WARNING) << name << " exists in this graph's initializers but it is not used by any node"; + erase_list.push_back(name); + } + } + + std::for_each(erase_list.cbegin(), erase_list.cend(), + [this](const std::string& name) { name_to_initial_tensor_.erase(name); }); +} + +GSL_SUPPRESS(es .84) // warning about ignoring return value from insert(...) +Status Graph::SetGraphInputsOutputs() { + // Reset graph inputs/outputs/value info state. + graph_inputs_excluding_initializers_.clear(); + graph_inputs_including_initializers_.clear(); + graph_outputs_.clear(); + value_info_.clear(); + + // Flag indicates that this graph is loaded from model file. + // If it's true, then graph inputs and outputs will keep the same + // as what are specified in the model, otherwise, graph inputs + // and outputs will be inferred. + const bool loaded_from_model_file = GraphLoadedFromModelFile(graph_proto_); + + // if something is coming from outer scope, consider it already added + std::unordered_set added_input_names{outer_scope_node_arg_names_}; + + if (loaded_from_model_file) { + // Collect all graph inputs/outputs specified in original graph proto + std::unordered_set specified_graph_inputs; + std::unordered_set specified_graph_outputs; + std::unordered_set specified_graph_value_info; + std::unordered_set specified_initializers; + std::unordered_map input_name_to_node_arg; + std::unordered_map output_name_to_node_arg; + + for (auto& graph_output : graph_proto_->output()) { + specified_graph_outputs.insert(graph_output.name()); + } + + for (auto& graph_value_info : graph_proto_->value_info()) { + specified_graph_value_info.insert(graph_value_info.name()); + } + + for (auto& initializer : graph_proto_->initializer()) { + specified_initializers.insert(initializer.name()); + } + + for (auto& graph_input : graph_proto_->input()) { + // add all graph inputs to input_name_to_node_arg + auto& name = graph_input.name(); + const auto* node_arg = GetNodeArg(name); + ONNXRUNTIME_ENFORCE(node_arg, "Graph ctor should have created NodeArg for initializer."); + input_name_to_node_arg.insert({name, node_arg}); + + // only add non-initializer to specified_graph_inputs + if (specified_initializers.find(name) == specified_initializers.end()) + specified_graph_inputs.insert(name); + } + + // add non-initializer outputs + for (const auto& node : Nodes()) { + for (const auto* output_def : node.OutputDefs()) { + ONNXRUNTIME_IGNORE_RETURN_VALUE(specified_graph_outputs.erase(output_def->Name())); + output_name_to_node_arg.insert({output_def->Name(), output_def}); + } + } + + // add any outputs using initializer + if (specified_graph_outputs.size() > 0) { + for (const auto& name : specified_initializers) { + ONNXRUNTIME_IGNORE_RETURN_VALUE(specified_graph_outputs.erase(name)); + output_name_to_node_arg.insert({name, GetNodeArg(name)}); + } + } + + if (!specified_graph_outputs.empty()) { + std::string missing_list; + for (auto& name : specified_graph_outputs) + missing_list += name + " "; + return Status(ONNXRUNTIME, FAIL, "Some graph outputs do not exist in the graph. (" + missing_list + ")"); + } + + for (const auto& node : Nodes()) { + // Go thru all node's inputs. + for (const auto* input_arg : node.InputDefs()) { + if (!input_arg->Exists()) { + // It's an optional input and does not exist in this case. + continue; + } + + if (specified_graph_inputs.end() != specified_graph_inputs.find(input_arg->Name())) { + if (added_input_names.insert(input_arg->Name()).second) { + // The node input is specified as graph input. + input_name_to_node_arg.insert({input_arg->Name(), input_arg}); + } + continue; + } + + auto output_arg_iter = output_name_to_node_arg.find(input_arg->Name()); + if (output_name_to_node_arg.end() == output_arg_iter && + specified_initializers.end() == specified_initializers.find(input_arg->Name())) { + // The node input is not specified as graph input, + // and it's not fed by another node neither. + if (!IsSubgraph()) { + return Status(ONNXRUNTIME, FAIL, "Node input (" + input_arg->Name() + ") should be a graph input or initializer."); + } + + // TODO: Do we need to do a comprehensive check that the input is coming from the outer scope or is it + // fine to catch this issue later? + } + + if (specified_graph_value_info.erase(input_arg->Name()) >= 1) { + value_info_.push_back(input_arg); + } + } + } + + // preserve input order + for (auto& graph_input : graph_proto_->input()) { + auto& name = graph_input.name(); + auto node_arg_iter = input_name_to_node_arg.find(name); + ONNXRUNTIME_ENFORCE(node_arg_iter != input_name_to_node_arg.cend(), + "All inputs and initializers should have entries. Missing ", name); + + graph_inputs_including_initializers_.push_back(node_arg_iter->second); + + if (specified_initializers.find(name) == specified_initializers.end()) { + graph_inputs_excluding_initializers_.push_back(node_arg_iter->second); + } + } + + // preserve output order + for (auto& graph_output : graph_proto_->output()) { + graph_outputs_.push_back(output_name_to_node_arg.at(graph_output.name())); + } + } else { + std::unordered_map output_name_to_node_arg; + std::vector ordered_output_names; + + // add any explicitly ordered inputs + for (auto* node_arg : graph_input_order_) { + if (!node_arg || !node_arg->Exists()) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Invalid entry in explicitly ordered inputs"); + } + + added_input_names.insert(node_arg->Name()); + graph_inputs_including_initializers_.push_back(node_arg); + if (name_to_initial_tensor_.find(node_arg->Name()) == name_to_initial_tensor_.end()) { + graph_inputs_excluding_initializers_.push_back(node_arg); + } + } + + // add any explicitly ordered outputs + for (auto* node_arg : graph_output_order_) { + if (!node_arg || !node_arg->Exists()) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Invalid entry in explicitly ordered outputs"); + } + output_name_to_node_arg.insert({node_arg->Name(), node_arg}); + ordered_output_names.push_back(node_arg->Name()); + } + + // add all other outputs + for (const auto& node : Nodes()) { + for (const auto* output_def : node.OutputDefs()) { + if (output_def->Exists()) { + auto& name = output_def->Name(); + // check it wasn't in the explicitly ordered outputs + if (output_name_to_node_arg.find(name) == output_name_to_node_arg.cend()) { + output_name_to_node_arg.insert({name, output_def}); + ordered_output_names.push_back(name); + } + } + } + } + + // Init graph output args with copy of all node output args. + auto graph_output_args = output_name_to_node_arg; + std::unordered_set inner_nodes; + + for (const auto& node : Nodes()) { + // Go thru all node's inputs. + for (const auto* input_arg : node.InputDefs()) { + if (!input_arg->Exists()) { + // It's an optional input and does not exist in this case. + continue; + } + + auto output_arg_iter = output_name_to_node_arg.find(input_arg->Name()); + if (output_name_to_node_arg.end() == output_arg_iter) { + // This input arg should be fed when running evaluation. + // it should be a graph input. + const std::string& name = input_arg->Name(); + if (added_input_names.end() == added_input_names.find(name)) { + // This graph input has not been added into . + graph_inputs_including_initializers_.push_back(input_arg); + + if (name_to_initial_tensor_.find(name) == name_to_initial_tensor_.end()) { + graph_inputs_excluding_initializers_.push_back(input_arg); + } + + added_input_names.insert(input_arg->Name()); + } + } else if (graph_output_args.erase(output_arg_iter->first) >= 1) { + // Remove the output arg name from graph outputs since it's + // the input of this node, which we call it intermediate result + // and store it in . + value_info_.push_back(input_arg); + } + } + } + + // Make sure all initializers appear as graph inputs as per ONNX requirements + for (auto i : name_to_initial_tensor_) { + if (added_input_names.find(i.first) == added_input_names.cend()) { + auto* na = GetNodeArg(i.first); + graph_inputs_including_initializers_.push_back(na); + } + } + + // Set graph outputs + auto end = graph_output_args.end(); + for (auto& name : ordered_output_names) { + auto graph_output = graph_output_args.find(name); + if (graph_output != end) { + graph_outputs_.push_back(graph_output->second); + } + } + } + + return Status::OK(); +} + +// calling private ctor +GSL_SUPPRESS(r .11) +gsl::not_null Graph::AllocateNode() { + std::unique_ptr new_node(new Node(nodes_.size(), *this)); + Node* node{new_node.get()}; + + nodes_.push_back(std::move(new_node)); + ++num_of_nodes_; + graph_resolve_needed_ = true; + + return gsl::not_null{node}; +} + +// TODO: Does this need (and maybe AllocateNode) to be threadsafe so nodes_ and num_of_nodes_ managed more carefully? +bool Graph::ReleaseNode(NodeIndex index) { + if (index >= nodes_.size()) { + return false; + } + + // index is valid, but the entry may already be empty + if (nodes_[index] != nullptr) { + nodes_[index] = nullptr; + --num_of_nodes_; + graph_proto_sync_needed_ = true; + graph_resolve_needed_ = true; + } + + return true; +} + +IOnnxRuntimeOpSchemaCollectionPtr Graph::GetSchemaRegistry() const { + return schema_registry_; +} + +Node* Graph::FuseSubGraph(std::unique_ptr<::onnxruntime::IndexedSubGraph> sub_graph, const std::string& fused_node_name) { + ONNXRUNTIME_ENFORCE(nullptr != sub_graph && nullptr != sub_graph->GetMetaDef()); + + auto func_meta_def = sub_graph->GetMetaDef(); + ONNXRUNTIME_ENFORCE(nullptr != func_meta_def); + std::vector input_args, output_args; + for (auto& arg_name : func_meta_def->inputs) { + input_args.push_back(GetNodeArg(arg_name)); + } + for (auto& arg_name : func_meta_def->outputs) { + output_args.push_back(GetNodeArg(arg_name)); + } + auto fused_node = AddNode(fused_node_name, + func_meta_def->name, + func_meta_def->doc_string, + input_args, + output_args, + &func_meta_def->attributes, + func_meta_def->domain); + + fused_node->SetNodeType(Node::Type::Fused); + function_container_.emplace_back(MakeFunction(*this, std::move(sub_graph))); + fused_node->SetFunctionBody(*(function_container_.back().get())); + + // Remove nodes fused above. + auto& sub_graph_ref = function_container_.back()->GetIndexedSubGraph(); + for (auto node_index : sub_graph_ref.nodes) { + RemoveNode(node_index); + } + return fused_node; +} + +Status Graph::InlineFunction(Node& node) { + // Remove the function node, add the nodes in function's subgraph into the + // maingraph. + const Graph& subgraph = node.GetFunctionBody()->Body(); + RemoveNode(node.Index()); + for (const auto& subgraph_node : subgraph.Nodes()) { + this->AddNode(subgraph_node); + } + ONNXRUNTIME_RETURN_IF_ERROR(this->Resolve()); + return Status::OK(); +} + +void Graph::AddFunction(const ONNX_NAMESPACE::FunctionProto* func_proto) { + this->model_functions_[func_proto->name()] = func_proto; +} + +Graph::~Graph() { + // nothing to do, but we put it here so we don't need to fully define types in Graph that are held in unique_ptr + // such as std::unique_ptr function_container_; +} +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/graph_transformer.cc b/onnxruntime/core/graph/graph_transformer.cc new file mode 100644 index 0000000000000..6d15f625770f9 --- /dev/null +++ b/onnxruntime/core/graph/graph_transformer.cc @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/graph/graph_transformer.h" + +using namespace ::onnxruntime::common; + +namespace onnxruntime { + +Status RuleBasedGraphTransformer::Register(const std::string& op_type, std::unique_ptr rule) { + if (HasRules(op_type)) { + op_to_rules_[op_type] = std::vector>(); + } + + op_to_rules_[op_type].push_back(std::move(rule)); + return Status::OK(); +} + +Status TopDownRuleBasedTransformer::Apply(Graph& graph, bool& modified) const { + ONNXRUNTIME_RETURN_IF_ERROR(graph.Resolve()); + GraphViewer graph_viewer(graph); + auto& order = graph_viewer.GetNodesInTopologicalOrder(); + GraphEditor graph_editor(graph); + + for (NodeIndex i : order) { + auto node = graph.GetNode(i); + if (!node) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT); + } + + // Get the rules that should be fired for this node. + if (!HasRules(node->OpType())) { + continue; + } + const std::vector>& rules = GetRewriteRules(node->OpType()); + for (const auto& rule : rules) { + ONNXRUNTIME_RETURN_IF_ERROR(rule->CheckConditionAndApply(&graph_editor, node, &modified)); + } + } + + // Resolve the graph at the end of all passes. + if (modified) { + ONNXRUNTIME_RETURN_IF_ERROR(graph.Resolve()); + } + + return Status::OK(); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/graph_transformer_mgr.cc b/onnxruntime/core/graph/graph_transformer_mgr.cc new file mode 100644 index 0000000000000..3b74c47c1c8db --- /dev/null +++ b/onnxruntime/core/graph/graph_transformer_mgr.cc @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/graph/graph_transformer_mgr.h" +using namespace onnxruntime; +using namespace ::onnxruntime::common; + +namespace onnxruntime { + +Status GraphTransformerManager::ApplyAll(Graph& graph) const { + for (unsigned step = 0; step < steps_; ++step) { + bool changed = false; + for (auto& transformer : transformers_) { + bool t_changed = false; + Status s = transformer->Apply(graph, t_changed); + if (!s.IsOK()) return s; + changed = changed || t_changed; + } + if (!changed) break; + } + return Status::OK(); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/graph_transformer_mgr.h b/onnxruntime/core/graph/graph_transformer_mgr.h new file mode 100644 index 0000000000000..fed2d1a70c6b6 --- /dev/null +++ b/onnxruntime/core/graph/graph_transformer_mgr.h @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/graph/graph_transformer.h" + +namespace onnxruntime { +// Manages a list of graph transformers. It is initialized with a list of graph +// transformers. Each inference session can further register additional ones. +class GraphTransformerManager { + public: + explicit GraphTransformerManager(unsigned steps) noexcept : steps_(steps) { + // TODO: Register default transformers. + } + + // Register a graph transformer. + ::onnxruntime::common::Status Register(std::unique_ptr transformer) { + transformers_.push_back(std::move(transformer)); + return ::onnxruntime::common::Status::OK(); + } + + // Apply the list of graph transformers registered on the specified graph + // up to the given number of steps. + ::onnxruntime::common::Status ApplyAll(Graph& graph) const; + + private: + GraphTransformerManager() = default; + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(GraphTransformerManager); + + std::vector> transformers_; + const unsigned steps_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/graph_viewer.cc b/onnxruntime/core/graph/graph_viewer.cc new file mode 100644 index 0000000000000..27473bd857a38 --- /dev/null +++ b/onnxruntime/core/graph/graph_viewer.cc @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#ifdef _WIN32 +// disable some warnings from protobuf to pass Windows build +#pragma warning(disable : 4244) +#endif + +#include "core/graph/graph.h" + +namespace onnxruntime { + +struct NodeCompare { + bool operator()(const Node* n1, const Node* n2) const { + return n1->Index() < n2->Index(); + } +}; + +GraphViewer::GraphViewer(const Graph& graph) { + graph_ = &graph; + std::vector leaf_nodes; + for (auto& node : graph_->Nodes()) { + if (node.OutputNodesBegin() == node.OutputNodesEnd()) { + // This is a leaf node (without any output node). + leaf_nodes.push_back(&node); + } + } + graph.ReverseDFSFrom(leaf_nodes, + nullptr, + [this](const Node* n) { + nodes_in_topological_order_.push_back(n->Index()); + }, + NodeCompare()); + + for (auto& node : graph_->Nodes()) { + if (node.InputEdgesBegin() == node.InputEdgesEnd()) { + root_nodes_.push_back(node.Index()); + } + } +} + +// Graph name. +const std::string& GraphViewer::Name() const noexcept { + return graph_->Name(); +} + +const std::string& GraphViewer::Description() const noexcept { + return graph_->Description(); +} + +bool GraphViewer::GetInitializedTensor(const std::string& tensor_name, const ONNX_NAMESPACE::TensorProto*& value) const { + return graph_->GetInitializedTensor(tensor_name, value); +} + +// Graph inputs excluding initializers. +const std::vector& GraphViewer::GetInputs() const noexcept { + return graph_->GetInputs(); +} +// Graph inputs including initializers. Contains no nullptr values. +// This will match the number and order of inputs from the GraphProto. +const std::vector& GraphViewer::GetInputsIncludingInitializers() const noexcept { + return graph_->GetInputsIncludingInitializers(); +} + +// Graph outputs. Should have no nullptr values. +const std::vector& GraphViewer::GetOutputs() const noexcept { + return graph_->GetOutputs(); +} + +// Get graph value infos. +const std::vector& GraphViewer::GetValueInfo() const noexcept { + return graph_->GetValueInfo(); +} + +// Get const Node given specific node index. May return nullptr if node as been freed. +const Node* GraphViewer::GetNode(NodeIndex node_index) const { + return graph_->GetNode(node_index); +} + +const GraphNodes& GraphViewer::Nodes() const noexcept { + return graph_->Nodes(); +} + +int GraphViewer::NumberOfNodes() const noexcept { + return graph_->NumberOfNodes(); +} + +int GraphViewer::MaxNodeIndex() const noexcept { + return graph_->MaxNodeIndex(); +} + +const std::vector& GraphViewer::GetNodesInTopologicalOrder() const { + return nodes_in_topological_order_; +} + +const std::vector& GraphViewer::GetRootNodes() const { + return root_nodes_; +} + +const InitializedTensorSet& GraphViewer::GetAllInitializedTensors() const noexcept { + return graph_->GetAllInitializedTensors(); +} + +const NodeArg* GraphViewer::GetNodeArg(const std::string& name) const { + return graph_->GetNodeArg(name); +} +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/identity_elimination.cc b/onnxruntime/core/graph/identity_elimination.cc new file mode 100644 index 0000000000000..be072cf9fa2ba --- /dev/null +++ b/onnxruntime/core/graph/identity_elimination.cc @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/graph/rewrite_rule.h" +#include "core/graph/identity_elimination.h" +#include "core/graph/graph.h" +#include "core/graph/op.h" +#include "core/common/logging/logging.h" + +namespace onnxruntime { + +Status EliminateIdentity::Apply(GraphEditor* graph_editor, Node* node, bool* modified) { + std::map replacement_defs; + auto id_input = node->InputDefs()[0]; + auto id_output = node->OutputDefs()[0]; + replacement_defs[id_output] = const_cast(id_input); + + // Replace (input) defs of the nodes following the Identity with the input to the Identity. + for (auto it = node->OutputNodesBegin(), end = node->OutputNodesEnd(); it != end; ++it) { + const_cast(*it)->ReplaceDefs(replacement_defs); + *modified = true; + } + + // Remove the Identity node. + graph_editor->RemoveNode(node->Index()); + + // TODO: Make sure resolve is not required here. + //ONNXRUNTIME_RETURN_IF_ERROR(graph_editor->Resolve()); + + return Status::OK(); +} + +bool EliminateIdentity::SatisfyCondition(const Node& node) { + (void)node; + return true; // No additional condition required. +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/identity_elimination.h b/onnxruntime/core/graph/identity_elimination.h new file mode 100644 index 0000000000000..e716474e45d36 --- /dev/null +++ b/onnxruntime/core/graph/identity_elimination.h @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/graph/rewrite_rule.h" + +namespace onnxruntime { + +// Rewrite rule that eliminates the identity node. +class EliminateIdentity : public RewriteRule { + public: + EliminateIdentity() noexcept : RewriteRule("EliminateIdentity", "Eliminate identity node") {} + + private: + bool SatisfyCondition(const Node& node) override; + + Status Apply(GraphEditor* graph_editor, Node* node, bool* modified) override; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/initializer.h b/onnxruntime/core/graph/initializer.h new file mode 100644 index 0000000000000..3fda9532d3653 --- /dev/null +++ b/onnxruntime/core/graph/initializer.h @@ -0,0 +1,352 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include + +#include "core/common/common.h" +#include "core/graph/onnx_protobuf.h" + +namespace onnxruntime { + +class Initializer final { + public: + static bool IsSupportedDataType(const ONNX_NAMESPACE::TensorProto* tensor_proto) { + if (tensor_proto == nullptr || + (tensor_proto->data_type() != ONNX_NAMESPACE::TensorProto_DataType_FLOAT && + tensor_proto->data_type() != ONNX_NAMESPACE::TensorProto_DataType_DOUBLE)) { + return false; + } + return true; + } + + Initializer(ONNX_NAMESPACE::TensorProto_DataType data_type, + const std::string& name, + const std::vector& dims) : size_(0) { + data_type_ = data_type; + name_ = std::move(name); + dims_.assign(dims.cbegin(), dims.cend()); + + size_ = std::accumulate(dims_.begin(), dims_.end(), (int64_t)1, std::multiplies{}); + + switch (data_type_) { + case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: { + float_data_.assign(size_, 0.0f); + break; + } + case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: { + double_data_.assign(size_, 0.0); + break; + } + default: + ONNXRUNTIME_NOT_IMPLEMENTED(__FUNCTION__, "data type is not supported"); + break; + } + } + + Initializer(const ONNX_NAMESPACE::TensorProto* tensor_proto) : size_(0) { + data_type_ = tensor_proto->data_type(); + if (tensor_proto->has_name()) { + name_ = std::move(tensor_proto->name()); + } + dims_.reserve(tensor_proto->dims_size()); + for (int i = 0; i < tensor_proto->dims_size(); i++) { + dims_.push_back(tensor_proto->dims(i)); + } + + size_ = std::accumulate(dims_.begin(), dims_.end(), (int64_t)1, std::multiplies{}); + + if (tensor_proto->has_raw_data()) { + raw_data_ = std::move(tensor_proto->raw_data()); + } else { + switch (data_type_) { + case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: { + int64_t size = tensor_proto->float_data_size(); + ONNXRUNTIME_ENFORCE(size_ == size, "size is different"); + for (int i = 0; i < size_; i++) { + float_data_.push_back(tensor_proto->float_data(i)); + } + break; + } + case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: { + int64_t size = tensor_proto->double_data_size(); + ONNXRUNTIME_ENFORCE(size_ == size, "size is different"); + for (int i = 0; i < size_; i++) { + double_data_.push_back(tensor_proto->double_data(i)); + } + break; + } + default: + ONNXRUNTIME_NOT_IMPLEMENTED(__FUNCTION__, "data type is not supported"); + break; + } + } + } + + ~Initializer() {} + + void ToProto(ONNX_NAMESPACE::TensorProto* tensor_proto) { + tensor_proto->clear_name(); + if (!name_.empty()) { + tensor_proto->set_name(name_); + } + + tensor_proto->clear_dims(); + for (auto d : dims_) { + tensor_proto->add_dims(d); + } + + tensor_proto->clear_data_type(); + tensor_proto->set_data_type(data_type_); + + if (!raw_data_.empty()) { + tensor_proto->clear_raw_data(); + tensor_proto->set_raw_data(raw_data_); + } else { + switch (data_type_) { + case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: { + tensor_proto->clear_float_data(); + for (int i = 0; i < size_; i++) { + tensor_proto->add_float_data(float_data_[i]); + } + break; + } + case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: { + tensor_proto->clear_double_data(); + for (int i = 0; i < size_; i++) { + tensor_proto->add_double_data(double_data_[i]); + } + break; + } + default: + ONNXRUNTIME_NOT_IMPLEMENTED(__FUNCTION__, "data type is not supported"); + break; + } + } + } + + ONNX_NAMESPACE::TensorProto_DataType data_type() const { + return data_type_; + } + + ONNX_NAMESPACE::TensorProto_DataType& data_type() { + return data_type_; + } + + const std::string& name() { + return name_; + } + + template + T* data() { + if (!raw_data_.empty()) { + return (T*)&raw_data_.data()[0]; + } else { + switch (data_type_) { + case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: { + return (T*)float_data_.data(); + break; + } + case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: { + return (T*)double_data_.data(); + break; + } + default: + break; + } + } + return nullptr; + } + + template + const T* data() const { + if (!raw_data_.empty()) { + return (T*)&raw_data_.data()[0]; + } else { + switch (data_type_) { + case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: { + return (T*)float_data_.data(); + break; + } + case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: { + return (T*)double_data_.data(); + break; + } + default: + break; + } + } + return nullptr; + } + + const std::vector& dims() const { + return dims_; + } + + const std::vector& dims() { + return dims_; + } + + size_t size() const { return size_; } + + Initializer& add(float value) { + switch (data_type_) { + case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: { + for (int i = 0; i < size_; i++) { + data()[i] += value; + } + break; + } + case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: { + for (int i = 0; i < size_; i++) { + data()[i] += value; + } + break; + } + default: + break; + } + return *this; + } + + Initializer& add(const Initializer& other) { + switch (data_type_) { + case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: { + for (int i = 0; i < size_; i++) { + data()[i] += other.data()[i]; + } + break; + } + case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: { + for (int i = 0; i < size_; i++) { + data()[i] += other.data()[i]; + } + break; + } + default: + break; + } + return *this; + } + Initializer& sub(const Initializer& other) { + switch (data_type_) { + case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: { + for (int i = 0; i < size_; i++) { + data()[i] -= other.data()[i]; + } + break; + } + case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: { + for (int i = 0; i < size_; i++) { + data()[i] -= other.data()[i]; + } + break; + } + default: + break; + } + return *this; + } + + Initializer& mul(const Initializer& other) { + switch (data_type_) { + case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: { + for (int i = 0; i < size_; i++) { + data()[i] *= other.data()[i]; + } + break; + } + case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: { + for (int i = 0; i < size_; i++) { + data()[i] *= other.data()[i]; + } + break; + } + default: + break; + } + return *this; + } + Initializer& div(const Initializer& other) { + switch (data_type_) { + case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: { + for (int i = 0; i < size_; i++) { + data()[i] /= other.data()[i]; + } + break; + } + case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: { + for (int i = 0; i < size_; i++) { + data()[i] /= other.data()[i]; + } + break; + } + default: + break; + } + return *this; + } + + Initializer& sqrt() { + switch (data_type_) { + case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: { + for (int i = 0; i < size_; i++) { + data()[i] = std::sqrt(data()[i]); + } + break; + } + case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: { + for (int i = 0; i < size_; i++) { + data()[i] = std::sqrt(data()[i]); + } + break; + } + default: + break; + } + return *this; + } + + inline void scale_by_axis(const Initializer& other, int axis) { + int64_t num = 1; + for (size_t k = axis; k < dims_.size(); k++) { + num *= dims_[k]; + } + + switch (data_type_) { + case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: { + for (int64_t i = 0; i < dims_[0]; i++) { + for (int64_t j = 0; j < num; j++) { + data()[i * num + j] *= other.data()[i]; + } + } + break; + } + case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: { + for (int64_t i = 0; i < dims_[0]; i++) { + for (int64_t j = 0; j < num; j++) { + data()[i * num + j] *= other.data()[i]; + } + } + break; + } + default: + break; + } + } + + private: + ONNX_NAMESPACE::TensorProto_DataType data_type_; + std::string name_; + std::vector dims_; + int64_t size_; + + std::string raw_data_; + std::vector float_data_; + std::vector double_data_; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/model.cc b/onnxruntime/core/graph/model.cc new file mode 100644 index 0000000000000..2d2f65f41ed21 --- /dev/null +++ b/onnxruntime/core/graph/model.cc @@ -0,0 +1,389 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/graph/model.h" +#include + +#ifdef _MSC_VER +#pragma warning(push) +// 'type' : forcing value to bool 'true' or 'false' (performance warning) +#pragma warning(disable : 4800) +#endif +#include +#ifdef _MSC_VER +#pragma warning(pop) +#endif +#include + +#include "gsl/pointers" +#include "gsl/gsl_util" + +#include "core/platform/env.h" +#include "core/graph/schema_registry.h" +using namespace ONNX_NAMESPACE; +using namespace onnxruntime; +using namespace ::onnxruntime::common; + +namespace onnxruntime { +Model::Model(const std::string& graph_name, + bool is_onnx_domain_only, + const ModelMetaData& model_metadata, + const IOnnxRuntimeOpSchemaRegistryList local_registries, + const std::unordered_map& domain_to_version, + const std::vector& model_functions) { + model_proto_ = std::make_unique(); + model_proto_->set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); + model_proto_->mutable_graph()->set_name(graph_name); + model_metadata_ = model_metadata; + for (auto& metadata : model_metadata_) { + const gsl::not_null prop{model_proto_->add_metadata_props()}; + prop->set_key(metadata.first); + prop->set_value(metadata.second); + } + + auto schema_registry = std::make_shared(); + for (auto schema_collection : local_registries) { + schema_registry->RegisterRegistry(schema_collection); + } + + auto* p_domain_to_version = &domain_to_version; + std::unordered_map domain_to_version_static; + if (p_domain_to_version->empty()) { + domain_to_version_static = schema_registry->GetLatestOpsetVersions(is_onnx_domain_only); + p_domain_to_version = &domain_to_version_static; + } + + for (auto domain : *p_domain_to_version) { + const gsl::not_null opset_id_proto{model_proto_->add_opset_import()}; + opset_id_proto->set_domain(domain.first); + opset_id_proto->set_version(domain.second); + } + + std::unordered_map model_functions_map; + for (auto& func : model_functions) { + auto func_ptr = model_proto_->add_functions(); + func_ptr->CopyFrom(func); + model_functions_map[func_ptr->name()] = func_ptr; + } + + // need to call private ctor so can't use make_shared + GSL_SUPPRESS(r .11) + graph_.reset(new Graph(model_proto_->mutable_graph(), *p_domain_to_version, IrVersion(), schema_registry, model_functions_map)); +} + +Model::Model(const ModelProto& model_proto, const IOnnxRuntimeOpSchemaRegistryList* local_registries) + : Model(std::make_unique(model_proto), local_registries) { +} + +Model::Model(std::unique_ptr model_proto, const IOnnxRuntimeOpSchemaRegistryList* local_registries) { + if (!model_proto) { + throw std::invalid_argument("ModelProto was null."); + } + + if (!model_proto->has_graph()) { + throw std::invalid_argument("ModelProto does not have a graph."); + } + + if (model_proto->opset_import_size() == 0) { + throw std::invalid_argument( + "Missing opset in the model. All ModelProtos MUST have at least one entry that" + " specifies which version of the ONNX OperatorSet is being imported."); + } + + model_proto_.reset(std::move(model_proto.release())); + for (auto& prop : model_proto_->metadata_props()) { + model_metadata_[prop.key()] = prop.value(); + } + + auto schema_registry = std::make_shared(); + if (local_registries != nullptr) { + for (auto schema_collection : *local_registries) { + schema_registry->RegisterRegistry(schema_collection); + } + } + + std::unordered_map domain_to_version; + for (auto& opSet : model_proto_->opset_import()) { + domain_to_version[opSet.domain()] = gsl::narrow_cast(opSet.version()); + } + + auto domain_map = schema_registry->GetLatestOpsetVersions(false); + for (auto domain : domain_map) { + if (domain_to_version.find(domain.first) == domain_to_version.end()) { + domain_to_version[domain.first] = domain.second; + const gsl::not_null opset_id_proto{model_proto_->add_opset_import()}; + opset_id_proto->set_domain(domain.first); + opset_id_proto->set_version(domain.second); + } + } + + std::unordered_map model_functions_map; + for (auto& func : model_proto_->functions()) { + model_functions_map[func.name()] = &func; + } + + // create instance. need to call private ctor so can't use make_unique + GSL_SUPPRESS(r .11) + graph_.reset(new Graph(model_proto_->mutable_graph(), domain_to_version, IrVersion(), schema_registry, model_functions_map)); +} + +Version Model::IrVersion() const { + if (model_proto_->has_ir_version()) { + return model_proto_->ir_version(); + } + return kNoVersion; +} + +const std::string& Model::ProducerName() const { + return model_proto_->producer_name(); +} + +void Model::SetProducerName(const std::string& producer_name) { + model_proto_->set_producer_name(producer_name); +} + +const std::string& Model::ProducerVersion() const { + return model_proto_->producer_version(); +} + +void Model::SetProducerVersion(const std::string& producer_version) { + model_proto_->set_producer_version(producer_version); +} + +const std::string& Model::Domain() const { + return model_proto_->domain(); +} + +void Model::SetDomain(const std::string& domain) { + model_proto_->set_domain(domain); +} + +Version Model::ModelVersion() const { + if (model_proto_->has_model_version()) { + return model_proto_->model_version(); + } + return kNoVersion; +} + +void Model::SetModelversion(onnxruntime::Version version) { + model_proto_->set_model_version(version); +} + +const std::string& Model::DocString() const { + return model_proto_->doc_string(); +} + +void Model::SetDocString(const std::string& doc_string) { + model_proto_->set_doc_string(doc_string); +} + +const ModelMetaData& Model::MetaData() const noexcept { + return model_metadata_; +} + +Graph& Model::MainGraph() noexcept { + return *graph_; +} + +const Graph& Model::MainGraph() const noexcept { + return *graph_; +} + +void Model::AddFunction(const ONNX_NAMESPACE::FunctionProto& func_proto) { + auto func_ptr = model_proto_->add_functions(); + func_ptr->CopyFrom(func_proto); + graph_->AddFunction(func_ptr); +} + +ModelProto Model::ToProto() { + *(model_proto_->mutable_graph()) = graph_->ToGraphProto(); + return *model_proto_; +} + +Status Model::Load(std::istream& model_istream, ModelProto* p_model_proto) { + if (!model_istream.good()) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid istream object."); + } + if (!p_model_proto) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Null model_proto ptr."); + } + const bool result = p_model_proto->ParseFromIstream(&model_istream); + if (!result) { + return Status(ONNXRUNTIME, INVALID_PROTOBUF, "Failed to load model because protobuf parsing failed."); + } + return Status::OK(); +} + +Status Model::Load(const ModelProto& model_proto, std::shared_ptr& model, const IOnnxRuntimeOpSchemaRegistryList* local_registries) { + // we expect a graph to be present + if (!model_proto.has_graph()) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "No graph was found in the protobuf."); + } + + // need to call private ctor so can't use make_shared + GSL_SUPPRESS(r .11) + try { + model.reset(new Model(model_proto, local_registries)); + } catch (const std::exception& ex) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Failed to load model with error: " + std::string(ex.what())); + } + + ONNXRUNTIME_RETURN_IF_ERROR(model->MainGraph().Resolve(true)); + + return Status::OK(); +} + +Status Model::Load(std::unique_ptr p_model_proto, std::shared_ptr& model, const IOnnxRuntimeOpSchemaRegistryList* local_registries) { + // we expect a graph to be present + if (!p_model_proto->has_graph()) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "No graph was found in the protobuf."); + } + + // need to call private ctor so can't use make_shared + GSL_SUPPRESS(r .11) + try { + model.reset(new Model(std::move(p_model_proto), local_registries)); + } catch (const std::exception& ex) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Failed to load model with error: " + std::string(ex.what())); + } + + ONNXRUNTIME_RETURN_IF_ERROR(model->MainGraph().Resolve(true)); + + return Status::OK(); +} + +template +static Status LoadModel(const T& file_path, std::shared_ptr& p_model, const IOnnxRuntimeOpSchemaRegistryList* local_registries) { + int fd; + Status status = Env::Default().FileOpenRd(file_path, fd); + if (!status.IsOK()) { + if (status.Category() == common::SYSTEM) { + switch (status.Code()) { + case ENOENT: + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, NO_SUCHFILE, "Load model failed. File doesn't exist"); + case EINVAL: + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT); + default: + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "system error number ", status.Code()); + } + } + } + try { + status = Model::Load(fd, p_model, local_registries); + } catch (std::exception& ex) { + GSL_SUPPRESS(es .84) + ONNXRUNTIME_IGNORE_RETURN_VALUE(Env::Default().FileClose(fd)); + return Status(ONNXRUNTIME, FAIL, ex.what()); + } + if (!status.IsOK()) { + GSL_SUPPRESS(es .84) + ONNXRUNTIME_IGNORE_RETURN_VALUE(Env::Default().FileClose(fd)); + return status; + } + return Env::Default().FileClose(fd); +} + +template +static Status SaveModel(Model& model, const T& file_path) { + int fd; + Status status = Env::Default().FileOpenWr(file_path, fd); + ONNXRUNTIME_RETURN_IF_ERROR(status); + try { + status = Model::Save(model, fd); + } catch (std::exception& ex) { + GSL_SUPPRESS(es .84) + ONNXRUNTIME_IGNORE_RETURN_VALUE(Env::Default().FileClose(fd)); + return Status(ONNXRUNTIME, FAIL, ex.what()); + } + if (!status.IsOK()) { + GSL_SUPPRESS(es .84) + ONNXRUNTIME_IGNORE_RETURN_VALUE(Env::Default().FileClose(fd)); + return status; + } + return Env::Default().FileClose(fd); +} + +#ifdef _WIN32 +GSL_SUPPRESS(r .30) // spurious warnings. p_model is potentially reset in the internal call to Load +GSL_SUPPRESS(r .35) +Status Model::Load(const std::wstring& file_path, std::shared_ptr& p_model, const IOnnxRuntimeOpSchemaRegistryList* local_registries) { + return LoadModel(file_path, p_model, local_registries); +} + +Status Model::Save(Model& model, const std::wstring& file_path) { + return SaveModel(model, file_path); +} + +#endif + +GSL_SUPPRESS(r .30) // spurious warnings. p_model is potentially reset in the internal call to Load +GSL_SUPPRESS(r .35) +Status Model::Load(const std::string& file_path, std::shared_ptr& p_model, const IOnnxRuntimeOpSchemaRegistryList* local_registries) { + return LoadModel(file_path, p_model, local_registries); +} + +Status Model::Save(Model& model, const std::string& file_path) { + return SaveModel(model, file_path); +} + +Status Model::LoadFromBytes(int count, void* p_bytes, /*out*/ std::shared_ptr& p_model, const IOnnxRuntimeOpSchemaRegistryList* local_registries) { + std::unique_ptr modelProto = std::make_unique(); + const bool result = modelProto->ParseFromArray(p_bytes, count); + if (!result) { + return Status(ONNXRUNTIME, INVALID_PROTOBUF, "Protobuf parsing failed."); + } + + p_model = std::make_shared(std::move(modelProto), local_registries); + + ONNXRUNTIME_RETURN_IF_ERROR(p_model->MainGraph().Resolve(true)); + + return Status::OK(); +} + +using ::google::protobuf::io::CodedInputStream; +using ::google::protobuf::io::FileInputStream; +using ::google::protobuf::io::ZeroCopyInputStream; + +Status Model::Load(int fd, std::shared_ptr& p_model, const IOnnxRuntimeOpSchemaRegistryList* local_registries) { + if (fd < 0) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, " less than 0."); + } + + auto raw_input = std::unique_ptr(std::make_unique(fd)); + auto coded_input = std::make_unique(raw_input.get()); + + // Allows protobuf library versions < 3.2.0 to parse messages greater than 64MB. + coded_input->SetTotalBytesLimit(INT_MAX, INT_MAX); + + std::unique_ptr model_proto = std::make_unique(); + const bool result = model_proto->ParseFromCodedStream(coded_input.get()); + coded_input.reset(); + raw_input.reset(); + + if (!result) { + return Status(ONNXRUNTIME, INVALID_PROTOBUF, "Protobuf parsing failed."); + } + + p_model = std::make_shared(std::move(model_proto), local_registries); + + ONNXRUNTIME_RETURN_IF_ERROR(p_model->MainGraph().Resolve(true)); + + return Status::OK(); +} + +Status Model::Save(Model& model, int p_fd) { + if (p_fd < 0) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, " is less than 0."); + } + + ONNXRUNTIME_RETURN_IF_ERROR(model.MainGraph().Resolve()); + + auto model_proto = model.ToProto(); + const bool result = model_proto.SerializeToFileDescriptor(p_fd); + if (result) { + return Status::OK(); + } else { + return Status(ONNXRUNTIME, INVALID_PROTOBUF, "Protobuf serialization failed."); + } +} +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/model.h b/onnxruntime/core/graph/model.h new file mode 100644 index 0000000000000..b81fd1b2be249 --- /dev/null +++ b/onnxruntime/core/graph/model.h @@ -0,0 +1,130 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include +#include +#include +#include +#include "core/graph/graph.h" + +#include "gsl/pointers" + +namespace onnxruntime { +typedef std::unordered_map ModelMetaData; +using IOnnxRuntimeOpSchemaRegistryList = std::list>; + +// A machine learning model representation class. +// Besides a main , it also holds basic information, say, +// model version, model domain, model author, license etc. +class Model { + public: + static constexpr Version kNoVersion = INT64_MAX; + + // Construct model from scratch. + explicit Model(const std::string& graph_name, + bool is_onnx_domain_only = false, + const ModelMetaData& model_metadata = ModelMetaData(), + const IOnnxRuntimeOpSchemaRegistryList local_registries = {}, + const std::unordered_map& domain_to_version = {}, + const std::vector& model_specific_functions = {}); + + // NOTE: after calling this constructor, <*this> model will + // hold a copy of . + explicit Model(const ONNX_NAMESPACE::ModelProto& model_proto, + const IOnnxRuntimeOpSchemaRegistryList* local_registries = nullptr); + + // NOTE: after calling this constructor, <*this> model will + // own the . + explicit Model(std::unique_ptr model_proto, + const IOnnxRuntimeOpSchemaRegistryList* local_registries = nullptr); + + // Get model's IR version. + // Return if not specified. + Version IrVersion() const; + + // Get model's producer name. + // Return null pointer if not specified. + const std::string& ProducerName() const; + // Set model's producer name. + void SetProducerName(const std::string& producer_name); + + // Get model's producer version. + // Return null pointer if not specified. + const std::string& ProducerVersion() const; + // Set model's producer version. + void SetProducerVersion(const std::string& producer_version); + + // Get model's domain. + // Return null pointer if not specified. + const std::string& Domain() const; + // Set models' domain. + void SetDomain(const std::string& domain); + + // Get model's version. + // Return null pointer if not specified. + Version ModelVersion() const; + // Set models' version. + void SetModelversion(onnxruntime::Version model_version); + + // Get model's doc string. + // Return null pointer if not specified. + const std::string& DocString() const; + // Set models' doc string. + void SetDocString(const std::string& doc_string); + + const ModelMetaData& MetaData() const noexcept; + + // Get model's main graph. + Graph& MainGraph() noexcept; + const Graph& MainGraph() const noexcept; + + // Add function proto to Model + void AddFunction(const ONNX_NAMESPACE::FunctionProto& func_proto); + + // Get model's serialization proto data. + ONNX_NAMESPACE::ModelProto ToProto(); + +#ifdef _WIN32 + static ::onnxruntime::common::Status Save(Model& model, const std::wstring& file_path); + + // TODO(Task:132) Use of shared_ptr* in Load/Save methods is confusing. + static ::onnxruntime::common::Status Load(const std::wstring& file_path, /*out*/ std::shared_ptr& p_model, + const IOnnxRuntimeOpSchemaRegistryList* local_registry = nullptr); +#endif + static ::onnxruntime::common::Status Save(Model& model, const std::string& file_path); + + static ::onnxruntime::common::Status Save(Model& model, int fd); + + static ::onnxruntime::common::Status Load(std::istream& model_istream, ONNX_NAMESPACE::ModelProto* p_model_proto); + + static ::onnxruntime::common::Status Load(const std::string& file_path, + /*out*/ std::shared_ptr& p_model, + const IOnnxRuntimeOpSchemaRegistryList* local_registries = nullptr); + + static ::onnxruntime::common::Status Load(int fd, /*out*/ std::shared_ptr& p_model, + const IOnnxRuntimeOpSchemaRegistryList* local_registries = nullptr); + + // 'int' rather than 'size_t' because of a protobuf design choice; let callers handle type checks + static ::onnxruntime::common::Status LoadFromBytes(int count, void* pBytes, /*out*/ std::shared_ptr& p_model, + const IOnnxRuntimeOpSchemaRegistryList* local_registries = nullptr); + + static ::onnxruntime::common::Status Load(const ONNX_NAMESPACE::ModelProto& model_proto, /*out*/ std::shared_ptr& p_model, + const IOnnxRuntimeOpSchemaRegistryList* local_registries = nullptr); + + static ::onnxruntime::common::Status Load(std::unique_ptr p_model_proto, /*out*/ std::shared_ptr& p_model, + const IOnnxRuntimeOpSchemaRegistryList* local_registries = nullptr); + + private: + // Model data. + std::unique_ptr model_proto_; + + // This is a duplication of . + // It gives better accessibility. + ModelMetaData model_metadata_; + + // Main graph of the model. + std::unique_ptr graph_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/op.cc b/onnxruntime/core/graph/op.cc new file mode 100644 index 0000000000000..f38e839ea3e64 --- /dev/null +++ b/onnxruntime/core/graph/op.cc @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include "core/graph/constants.h" +#include "core/graph/op.h" + +using namespace ONNX_NAMESPACE; +using namespace ::onnxruntime::common; +namespace onnxruntime { + +bool TypeUtils::IsValidAttribute(const AttributeProto& attr) { + if (attr.name().empty()) { + return false; + } + + if (attr.type() == AttributeProto_AttributeType_UNDEFINED) { + const int num_fields = + attr.has_f() + + attr.has_i() + + attr.has_s() + + attr.has_t() + + attr.has_g() + + (attr.floats_size() > 0) + + (attr.ints_size() > 0) + + (attr.strings_size() > 0) + + (attr.tensors_size() > 0) + + (attr.graphs_size() > 0); + + if (num_fields != 1) { + return false; + } + } + return true; +} + +Status TypeUtils::GetType(const AttributeProto& attr, AttrType& type) { + if (!TypeUtils::IsValidAttribute(attr)) { + return Status(ONNXRUNTIME, FAIL, "Invalid AttributeProto."); + } + + type = attr.type(); + if (AttrType::AttributeProto_AttributeType_UNDEFINED == type) { + if (attr.has_f()) { + type = AttrType::AttributeProto_AttributeType_FLOAT; + } else if (attr.has_i()) { + type = AttrType::AttributeProto_AttributeType_INT; + } else if (attr.has_s()) { + type = AttrType::AttributeProto_AttributeType_STRING; + } else if (attr.has_t()) { + type = AttrType::AttributeProto_AttributeType_TENSOR; + } else if (attr.has_g()) { + type = AttrType::AttributeProto_AttributeType_GRAPH; + } else if (attr.floats_size()) { + type = AttrType::AttributeProto_AttributeType_FLOATS; + } else if (attr.ints_size()) { + type = AttrType::AttributeProto_AttributeType_INTS; + } else if (attr.strings_size()) { + type = AttrType::AttributeProto_AttributeType_STRINGS; + } else if (attr.tensors_size()) { + type = AttrType::AttributeProto_AttributeType_TENSORS; + } else if (attr.graphs_size()) { + type = AttrType::AttributeProto_AttributeType_GRAPHS; + } else { + return Status(ONNXRUNTIME, FAIL, "Invalid AttributeProto."); + } + } + return Status::OK(); +} +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/op.h b/onnxruntime/core/graph/op.h new file mode 100644 index 0000000000000..505a90053ccd2 --- /dev/null +++ b/onnxruntime/core/graph/op.h @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wignored-qualifiers" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif +#include "onnx/defs/schema.h" +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif +#include "core/common/status.h" +#include "core/graph/constants.h" + +namespace onnxruntime { +using AttrType = ONNX_NAMESPACE::AttributeProto_AttributeType; +using NodeAttributes = std::unordered_map; + +// This string array should exactly match the AttrType defined above. +/* +AttributeProto_AttributeType_UNDEFINED = 0, +AttributeProto_AttributeType_FLOAT = 1, +AttributeProto_AttributeType_INT = 2, +AttributeProto_AttributeType_STRING = 3, +AttributeProto_AttributeType_TENSOR = 4, +AttributeProto_AttributeType_GRAPH = 5, +AttributeProto_AttributeType_FLOATS = 6, +AttributeProto_AttributeType_INTS = 7, +AttributeProto_AttributeType_STRINGS = 8, +AttributeProto_AttributeType_TENSORS = 9, +AttributeProto_AttributeType_GRAPHS = 10 +*/ +static constexpr const char* kAttrTypeStrings[] = + { + "UNDEFINED", + "FLOAT", + "INT", + "STRING", + "TENSOR", + "GRAPH", + "FLOATS", + "INTS", + "STRINGS", + "TENSORS", + "GRAPHS"}; + +class TypeUtils { + public: + // Get attribute type given attribute proto data. + static ::onnxruntime::common::Status GetType(const ONNX_NAMESPACE::AttributeProto& attr, AttrType& type); + static bool IsValidAttribute(const ONNX_NAMESPACE::AttributeProto& attribute); +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/record.h b/onnxruntime/core/graph/record.h new file mode 100644 index 0000000000000..27e9e142d8536 --- /dev/null +++ b/onnxruntime/core/graph/record.h @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include +#include + +#include "core/common/common.h" +#include "core/common/status.h" + +namespace onnxruntime { +namespace common { +template +class Record { + public: + typedef std::tuple Values; + + Record() = default; + + Record(const std::vector& names, const Values& values) { + ONNXRUNTIME_ENFORCE(std::tuple_size::value == names.size(), + "Parameter sizes do not match. %d != %d", std::tuple_size::value, names.size()); + names_ = names; + values_ = values; + } + + Record(const Record& other) { + names_ = other.names_; + values_ = other.values_; + } + + Status GetName(int index, const std::string** pp_name) const { + if (nullptr == pp_name || index >= names_.size()) { + return Status(ONNXRUNTIME, common::INVALID_ARGUMENT); + } + + *pp_name = &(names_[index]); + return Status::OK(); + } + + const Values& GetValues() const { + return values_; + } + + private: + std::vector names_; + + Values values_; +}; +} // namespace common +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/schema_registry.cc b/onnxruntime/core/graph/schema_registry.cc new file mode 100644 index 0000000000000..136d4931ebaff --- /dev/null +++ b/onnxruntime/core/graph/schema_registry.cc @@ -0,0 +1,248 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/graph/schema_registry.h" + +namespace onnxruntime { +// Add customized domain to min/max version. +::onnxruntime::common::Status OnnxRuntimeOpSchemaRegistry::SetBaselineAndOpsetVersionForDomain( + const std::string& domain, + int baseline_opset_version, + int opset_version) { + std::lock_guard lock(mutex_); + + auto it = domain_version_range_map_.find(domain); + if (domain_version_range_map_.end() != it) { + return ::onnxruntime::common::Status(::onnxruntime::common::ONNXRUNTIME, ::onnxruntime::common::FAIL, "Domain already set in registry"); + } + + domain_version_range_map_[domain].baseline_opset_version = baseline_opset_version; + domain_version_range_map_[domain].opset_version = opset_version; + + return ::onnxruntime::common::Status::OK(); +} + +Domain_To_Version_Map OnnxRuntimeOpSchemaRegistry::GetLatestOpsetVersions(bool is_onnx_only) const { + Domain_To_Version_Map domain_version_map; + + for (auto& domain : domain_version_range_map_) { + if (is_onnx_only && domain.first.compare(kOnnxDomain) != 0) + continue; + domain_version_map[domain.first] = domain.second.opset_version; + } + + return domain_version_map; +} + +::onnxruntime::common::Status OnnxRuntimeOpSchemaRegistry::RegisterOpSet( + std::vector& schemas, + const std::string& domain, + int baseline_opset_version, + int opset_version) { + ONNXRUNTIME_RETURN_IF_ERROR(SetBaselineAndOpsetVersionForDomain(domain, baseline_opset_version, opset_version)); + for (auto& schema : schemas) + ONNXRUNTIME_RETURN_IF_ERROR(RegisterOpSchema(std::move(schema))); + return ::onnxruntime::common::Status::OK(); +} + +::onnxruntime::common::Status OnnxRuntimeOpSchemaRegistry::RegisterOpSchema(ONNX_NAMESPACE::OpSchema&& op_schema) { + return RegisterOpSchemaInternal(std::move(op_schema)); +} + +::onnxruntime::common::Status OnnxRuntimeOpSchemaRegistry::RegisterOpSchemaInternal(ONNX_NAMESPACE::OpSchema&& op_schema) { + try { + op_schema.Finalize(); + } catch (const std::exception& e) { + return ::onnxruntime::common::Status(::onnxruntime::common::ONNXRUNTIME, ::onnxruntime::common::INVALID_ARGUMENT, "Schema error: " + std::string(e.what())); + } + + auto& op_name = op_schema.Name(); + auto& op_domain = op_schema.domain(); + auto ver = op_schema.SinceVersion(); + + if (map_[op_name][op_domain].count(ver)) { + const auto& schema = map_[op_name][op_domain][ver]; + std::ostringstream ostream; + ostream << "Trying to register schema with name " << op_name + << " (domain: " << op_domain << " version: " << ver + << ") from file " << op_schema.file() << " line " + << op_schema.line() + << ", but it is already registered from file " + << schema.file() << " line " << schema.line() << std::endl; + return ::onnxruntime::common::Status(::onnxruntime::common::ONNXRUNTIME, ::onnxruntime::common::INVALID_ARGUMENT, ostream.str()); + } + + auto ver_range_it = domain_version_range_map_.find(op_domain); + if (ver_range_it == domain_version_range_map_.end()) { + std::ostringstream ostream; + ostream << "Trying to register schema with name " << op_name + << " (domain: " << op_domain << " version: " << ver + << ") from file " << op_schema.file() << " line " + << op_schema.line() << ", but it its domain is not" + << "known by the checker." << std::endl; + return ::onnxruntime::common::Status(::onnxruntime::common::ONNXRUNTIME, ::onnxruntime::common::INVALID_ARGUMENT, ostream.str()); + } + if (ver > ver_range_it->second.opset_version) { + std::ostringstream ostream; + ostream + << "Trying to register schema with name " << op_name + << " (domain: " << op_domain << " version: " << ver + << ") from file " << op_schema.file() << " line " + << op_schema.line() << ", but it its version is higher" + << "than the operator set version " << ver_range_it->second.opset_version << std::endl; + return ::onnxruntime::common::Status(::onnxruntime::common::ONNXRUNTIME, ::onnxruntime::common::INVALID_ARGUMENT, ostream.str()); + } + GSL_SUPPRESS(es .84) + map_[op_name][op_domain].emplace(std::make_pair(ver, op_schema)); + return ::onnxruntime::common::Status::OK(); +} + +// Return the schema with biggest version, which is not greater than specified +// in specified domain. The value of earliest_opset_where_unchanged +// is also set to the earliest version preceding op_set_version where the operator +// is known to be unchanged. +void OnnxRuntimeOpSchemaRegistry::GetSchemaAndHistory( + const std::string& key, + const int op_set_version, + const std::string& domain, + const ONNX_NAMESPACE::OpSchema** latest_schema, + int* earliest_opset_where_unchanged) const { + *latest_schema = nullptr; + *earliest_opset_where_unchanged = std::numeric_limits::max(); + + // Determine if this registry contains the requested domain at the same or later + // version + auto domain_map_it = domain_version_range_map_.find(domain); + if (domain_map_it != domain_version_range_map_.end() && + domain_map_it->second.opset_version >= op_set_version) { + // If the baseline version is not larger than the requested version, initialize + // the version at which the operator is unchanged to the baseline. This will + // be updated below if a schema is found. + if (domain_map_it->second.baseline_opset_version <= op_set_version) { + assert(domain_map_it->second.baseline_opset_version < domain_map_it->second.opset_version); + *earliest_opset_where_unchanged = std::max(1, domain_map_it->second.baseline_opset_version); + } + + auto it = map_.find(key); + if (it == map_.end()) + return; + auto s_it = it->second.find(domain); + if (s_it != it->second.end()) { + auto pos = s_it->second.lower_bound(op_set_version); + if (s_it->second.begin() == pos && pos->first > op_set_version) { + // All versions are greater than specified version. + return; + } + + if (s_it->second.end() == pos || pos->first > op_set_version) { + // All versions are less than specified version, or, + // The version is greater than specified version. + --pos; + } + + assert(pos->first <= op_set_version); + + if (pos->second.SinceVersion() <= op_set_version) { + *latest_schema = &(pos->second); + *earliest_opset_where_unchanged = (*latest_schema)->SinceVersion(); + } + } + } +} + +void SchemaRegistryManager::RegisterRegistry(std::shared_ptr registry) { + registries.push_front(registry); +} + +Domain_To_Version_Map SchemaRegistryManager::GetLatestOpsetVersions(bool is_onnx_only) const { + Domain_To_Version_Map domain_version_map; + + // Build the map using each of the registries + for (auto& registry : registries) { + Domain_To_Version_Map latest_opset_versions_in_reg = registry->GetLatestOpsetVersions(is_onnx_only); + + for (auto& local_domain : latest_opset_versions_in_reg) { + auto iter = domain_version_map.find(local_domain.first); + + // If the map doesn't yet contain this domain, insert it with this registry's value. + // Otherwise, merge the existing range in the map. + if (iter == domain_version_map.end()) { + GSL_SUPPRESS(es .84) + domain_version_map.insert(local_domain); + } else { + iter->second = std::max(iter->second, local_domain.second); + } + } + } + + // check the ONNX schema registry + auto& onnx_domain_version_map = + ONNX_NAMESPACE::OpSchemaRegistry::DomainToVersionRange::Instance().Map(); + + for (auto domain : onnx_domain_version_map) { + if (is_onnx_only && domain.first.compare(kOnnxDomain) != 0) + continue; + auto it = domain_version_map.find(domain.first); + if (it == domain_version_map.end()) { + GSL_SUPPRESS(es .84) + domain_version_map.insert(std::make_pair(domain.first, domain.second.second)); + } else { + it->second = std::max(it->second, domain.second.second); + } + } + + return domain_version_map; +} + +// Return the schema with biggest version, which is not greater than specified +// in specified domain. The value of earliest_opset_where_unchanged +// is also set to the earliest version preceding op_set_version where the operator +// is known to be unchanged. +void SchemaRegistryManager::GetSchemaAndHistory( + const std::string& key, + const int op_set_version, + const std::string& domain, + const ONNX_NAMESPACE::OpSchema** latest_schema, + int* earliest_opset_where_unchanged) const { + // A greedy algorithm is used to search for a schema registration in some registry, + // while potentially inferring from other registries the allowed schema version + // given the op-set version. Each time a registry fails to locate the schema + // but indicates that this schema was unchanged across its version span, the search + // is restarted with a reduced op-set version. + std::vector unchecked_registry_indices(registries.size()); + std::iota(unchecked_registry_indices.begin(), unchecked_registry_indices.end(), 0); + + std::vector checked_registry_indices; + int version = op_set_version; + while (!unchecked_registry_indices.empty()) { + int index = unchecked_registry_indices.back(); + unchecked_registry_indices.pop_back(); + + int new_version = std::numeric_limits::max(); + registries[index]->GetSchemaAndHistory(key, version, domain, latest_schema, &new_version); + if (*latest_schema != nullptr) { + assert(new_version <= version && new_version <= op_set_version); + *earliest_opset_where_unchanged = new_version; + return; + } + + if (new_version < version) { + GSL_SUPPRESS(es .84) + unchecked_registry_indices.insert(unchecked_registry_indices.end(), + checked_registry_indices.begin(), + checked_registry_indices.end()); + checked_registry_indices.clear(); + version = new_version; + } + + checked_registry_indices.push_back(index); + } + + // if not found in registered custom schema registry, search in ONNX schema registry + *latest_schema = ONNX_NAMESPACE::OpSchemaRegistry::Schema(key, version, domain); + if (*latest_schema != nullptr) { + *earliest_opset_where_unchanged = (*latest_schema)->SinceVersion(); + } +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/unsqueeze_elimination.cc b/onnxruntime/core/graph/unsqueeze_elimination.cc new file mode 100644 index 0000000000000..3d4c7ee5022aa --- /dev/null +++ b/onnxruntime/core/graph/unsqueeze_elimination.cc @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/graph/unsqueeze_elimination.h" + +using namespace onnx; +using namespace ::onnxruntime::common; + +namespace onnxruntime { + +Status UnsqueezeElimination::Apply(onnxruntime::Graph& graph, bool& modified) const { + std::vector removed_nodes; + + for (auto& node : graph.Nodes()) { + if (node.OpType() != "Unsqueeze" || node.GetInputEdgesCount() != 0 || graph.IsNodeOutputsInGraphOutputs(node)) { + continue; + } + + const onnxruntime::NodeAttributes& attributes = node.GetAttributes(); + const onnx::AttributeProto* attr = &attributes.find("axes")->second; + if (attr == nullptr || attr->type() != AttributeProto_AttributeType_INTS) { + continue; + } + + // Get attribute of "axes" + std::vector axes; + for (int i = 0; i < attr->ints_size(); i++) { + axes.push_back(static_cast(attr->ints(i))); + } + + // Generate new dims + NodeArg* input_def = node.MutableInputDefs()[0]; + const ONNX_NAMESPACE::TensorProto* tensor_proto = nullptr; + graph.GetInitializedTensor(input_def->Name(), tensor_proto); + if (tensor_proto == nullptr) { + continue; + } + std::vector new_dims(axes.size() + tensor_proto->dims().size(), 0); + + for (int64_t axis : axes) { + new_dims[axis] = 1; + } + + auto begin = tensor_proto->dims().cbegin(); + for (auto& axis : new_dims) { + if (axis == 0) { + axis = *begin++; + } + } + + // Update shape of tensor proto + ONNX_NAMESPACE::TensorProto new_tensor_proto(*tensor_proto); + for (int i = 0; i < new_dims.size(); i++) { + if (i < tensor_proto->dims().size()) { + new_tensor_proto.set_dims(i, new_dims[i]); + } else { + new_tensor_proto.add_dims(new_dims[i]); + } + } + graph.RemoveInitializedTensor(input_def->Name()); + graph.AddInitializedTensor(new_tensor_proto); + + // Update shape of NodeArg + TensorShapeProto shape; + for (auto dim : new_dims) { + shape.add_dim()->set_dim_value(dim); + } + input_def->SetShape(shape); + + // Replace the input of the nodes following unsqueeze node + const NodeArg* output_def = node.OutputDefs()[0]; + for (auto it = node.OutputNodesBegin(); it != node.OutputNodesEnd(); ++it) { + auto output_node = graph.GetNode((*it)->Index()); + if (!output_node) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT); + } + + auto& input_defs = output_node->MutableInputDefs(); + for (auto& def : input_defs) { + if (def == output_def) { + def = const_cast(input_def); + } + } + } + removed_nodes.push_back(node.Index()); + } + + for (auto i : removed_nodes) { + graph.RemoveNode(i); + } + + if (!removed_nodes.empty()) { + modified = true; + ONNXRUNTIME_RETURN_IF_ERROR(graph.Resolve()); + } + return Status::OK(); +} +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/core/graph/unsqueeze_elimination.h b/onnxruntime/core/graph/unsqueeze_elimination.h new file mode 100644 index 0000000000000..823ea0877f032 --- /dev/null +++ b/onnxruntime/core/graph/unsqueeze_elimination.h @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/graph/graph_transformer.h" + +namespace onnxruntime { + +class UnsqueezeElimination : public onnxruntime::GraphTransformer { + public: + UnsqueezeElimination() noexcept : onnxruntime::GraphTransformer("EliminateUnsqueeze", "Eliminate unsequeeze node") {} + Status Apply(onnxruntime::Graph& graph, bool& modified) const override; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/mlas/README.md b/onnxruntime/core/mlas/README.md new file mode 100644 index 0000000000000..072795c0fd61c --- /dev/null +++ b/onnxruntime/core/mlas/README.md @@ -0,0 +1,6 @@ +# About MLAS +MLAS is a compute library containing processor optimized GEMM kernels and platform specific threading code. + +# Unit tests for MLAS +Unit tests for the SGEMM kernels are available under onnxruntime\test\mlas. These tests run over a range of inputs that then execute the various special cases for aligned and unaligned outputs. The tests have failed if any "mismatch" strings are printed. + diff --git a/onnxruntime/core/mlas/inc/mlas.h b/onnxruntime/core/mlas/inc/mlas.h new file mode 100644 index 0000000000000..975620efb5516 --- /dev/null +++ b/onnxruntime/core/mlas/inc/mlas.h @@ -0,0 +1,182 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + mlas.h + +Abstract: + + This module contains the public data structures and procedure prototypes + for the Microsoft Machine Learning algebra subprogram library. + +--*/ + +#pragma once +// clang-format off + +#include +#include + +// +// Define the calling convention for Windows targets. +// + +#if (_MSC_VER >= 800) || defined(_STDCALL_SUPPORTED) +#define MLASCALL __stdcall +#else +#define MLASCALL +#endif + +// +// Basic Linear Algebra Subprograms (BLAS) types. +// + +#ifndef CBLAS_ENUM_DEFINED_H +#define CBLAS_ENUM_DEFINED_H +typedef enum { CblasNoTrans=111, CblasTrans=112, CblasConjTrans=113 } CBLAS_TRANSPOSE; +typedef enum { CblasUpper=121, CblasLower=122 } CBLAS_UPLO; +typedef enum { CblasNonUnit=131, CblasUnit=132 } CBLAS_DIAG; +typedef enum { CblasLeft=141, CblasRight=142} CBLAS_SIDE; +#endif + +// +// Single precision matrix/matrix multiply routine. +// + +void +MLASCALL +MlasSgemm( + CBLAS_TRANSPOSE TransA, + CBLAS_TRANSPOSE TransB, + size_t M, + size_t N, + size_t K, + float alpha, + const float* A, + size_t lda, + const float* B, + size_t ldb, + float beta, + float* C, + size_t ldc + ); + +// +// Convolution routines. +// + +enum MLAS_CONV_ALGORITHM { + MlasConvAlgorithmGemmDirect, + MlasConvAlgorithmExpandThenGemm, + MlasConvAlgorithmExpandThenGemmSegmented, +}; + +struct MLAS_CONV_PARAMETERS { + size_t Dimensions; + size_t BatchCount; + size_t GroupCount; + size_t InputChannels; + size_t InputShape[3]; + size_t KernelShape[3]; + size_t DilationShape[3]; + size_t Padding[6]; + size_t StrideShape[3]; + size_t FilterCount; + size_t OutputShape[3]; + size_t InputSize; + size_t OutputSize; + size_t K; + MLAS_CONV_ALGORITHM Algorithm; + union { + struct { + CBLAS_TRANSPOSE TransB; + size_t ldb; + } GemmDirect; + struct { + size_t ThreadStrideN; + } ExpandThenGemmSegmented; + } u; +}; + +bool +MLASCALL +MlasConvPrepare( + MLAS_CONV_PARAMETERS* Parameters, + size_t Dimensions, + size_t BatchCount, + size_t GroupCount, + size_t InputChannels, + const int64_t* InputShape, + const int64_t* KernelShape, + const int64_t* DilationShape, + const int64_t* Padding, + const int64_t* StrideShape, + size_t FilterCount, + size_t* WorkingBufferSize + ); + +void +MLASCALL +MlasConv( + const MLAS_CONV_PARAMETERS* Parameters, + const float* Input, + const float* Filter, + const float* Bias, + float* WorkingBuffer, + float* Output + ); + +// +// Pooling routines. +// + +enum MLAS_POOLING_KIND { + MlasMaximumPooling, + MlasAveragePoolingExcludePad, + MlasAveragePoolingIncludePad, +}; + +void +MLASCALL +MlasPool( + MLAS_POOLING_KIND PoolingKind, + size_t Dimensions, + const int64_t* InputShape, + const int64_t* KernelShape, + const int64_t* Padding, + const int64_t* StrideShape, + const int64_t* OutputShape, + const float* Input, + float* Output + ); + +// +// Bias addition routine. +// + +void +MLASCALL +MlasBiasAdd( + const float* Bias, + size_t M, + float* Output, + size_t N, + size_t ldc + ); + +// +// Half-precision floating-point routines. +// + +extern "C" +void +MLASCALL +MlasConvertHalfToFloatBuffer( + const unsigned short* Source, + float* Destination, + size_t Count + ); diff --git a/onnxruntime/core/mlas/lib/aarch64/sgemma.s b/onnxruntime/core/mlas/lib/aarch64/sgemma.s new file mode 100644 index 0000000000000..d640929505edd --- /dev/null +++ b/onnxruntime/core/mlas/lib/aarch64/sgemma.s @@ -0,0 +1,482 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved.\ + +Licensed under the MIT License. + +Module Name: + + sgemma.s + +Abstract: + + This module implements the kernels for the single precision matrix/matrix + multiply operation (SGEMM). + +--*/ + + .text + +// +// ClearRowAccumulators +// +// Generates the code to clear the accumulators for a single row of the output +// block. +// + + .macro ClearRowAccumulators Columns, Vec1Reg, Vec2Reg, Vec3Reg, Vec4Reg + + movi v\Vec1Reg\().16b,#0 + movi v\Vec2Reg\().16b,#0 +.if \Columns\() > 8 + movi v\Vec3Reg\().16b,#0 + movi v\Vec4Reg\().16b,#0 +.endif + + .endm + +// +// ClearBlockAccumulators +// +// Generates the code to clear the accumulators for a single row of the output +// block. +// + + .macro ClearBlockAccumulators Columns, Rows + + ClearRowAccumulators \Columns\(),16,17,18,19 +.if \Rows\() >= 2 + ClearRowAccumulators \Columns\(),20,21,22,23 +.endif +.if \Rows\() >= 4 + ClearRowAccumulators \Columns\(),24,25,26,27 + ClearRowAccumulators \Columns\(),28,29,30,31 +.endif + + .endm + +// +// LoadMatrixAElementsBy4 +// LoadMatrixAElementsBy1 +// +// Generates the code to load 1 or 4 elements from matrix A. +// + + .macro LoadMatrixAElementsBy4 Rows + + ldr q8,[x0],#16 +.if \Rows\() >= 2 + ldr q9,[x10],#16 +.endif +.if \Rows\() >= 4 + ldr q10,[x11],#16 + ldr q11,[x12],#16 +.endif + + .endm + + .macro LoadMatrixAElementsBy1 Rows + + ldr s8,[x0],#4 +.if \Rows\() >= 2 + ldr s9,[x10],#4 +.endif +.if \Rows\() >= 4 + ldr s10,[x11],#4 + ldr s11,[x12],#4 +.endif + + .endm + + +// MultiplyAccumulateRow +// +// Generates the code to multiply and accumulate a single row of the output +// block. +// + + .macro MultiplyAccumulateRow Columns, MatrixAReg, Broadcast, Vec1Reg, Vec2Reg, Vec3Reg, Vec4Reg + + fmla v\Vec1Reg\().4s,v4.4s,\MatrixAReg\().s[\Broadcast\()] + fmla v\Vec2Reg\().4s,v5.4s,\MatrixAReg\().s[\Broadcast\()] +.if \Columns\() > 8 + fmla v\Vec3Reg\().4s,v6.4s,\MatrixAReg\().s[\Broadcast\()] + fmla v\Vec4Reg\().4s,v7.4s,\MatrixAReg\().s[\Broadcast\()] +.endif + + .endm + +// +// MultiplyAccumulateBlock +// +// Generates the code to multiply and accumulate into the output block. +// + + .macro MultiplyAccumulateBlock Columns, Rows, Broadcast + + MultiplyAccumulateRow \Columns\(),v8,\Broadcast\(),16,17,18,19 +.if \Rows\() >= 2 + MultiplyAccumulateRow \Columns\(),v9,\Broadcast\(),20,21,22,23 +.endif +.if \Rows\() >= 4 + MultiplyAccumulateRow \Columns\(),v10,\Broadcast\(),24,25,26,27 + MultiplyAccumulateRow \Columns\(),v11,\Broadcast\(),28,29,30,31 +.endif + + .endm + +// +// ComputeBlockLoop +// +// Generates the code to loop over K entries of the input matrices to produce +// the output block. +// + + .macro ComputeBlockLoop Mode, Columns, Rows + + ClearBlockAccumulators \Columns\(),\Rows\() + +.if \Rows\() >= 2 + add x10,x0,x6,uxtw 2 // compute matrix A plus 1 row +.endif +.if \Rows\() >= 4 + add x11,x10,x6,uxtw 2 // compute matrix A plus 2 rows + add x12,x11,x6,uxtw 2 // compute matrix A plus 3 rows +.endif + + sub x9,x3,#4 // decrement block count to process + tbnz x9,#63,.L\Mode\().ProcessRemaining\Columns\().x\Rows\().Blocks + +.L\Mode\().Compute\Columns\().x\Rows\().BlockBy4Loop: + LoadMatrixAElementsBy4 \Rows\() + ldp q4,q5,[x1],#64*4 +.if \Columns\() > 8 + ldp q6,q7,[x1,#-56*4] +.endif + MultiplyAccumulateBlock \Columns\(),\Rows\(),0 + ldp q4,q5,[x1,#-48*4] +.if \Columns\() > 8 + ldp q6,q7,[x1,#-40*4] +.endif + MultiplyAccumulateBlock \Columns\(),\Rows\(),1 + ldp q4,q5,[x1,#-32*4] +.if \Columns\() > 8 + ldp q6,q7,[x1,#-24*4] +.endif + MultiplyAccumulateBlock \Columns\(),\Rows\(),2 + ldp q4,q5,[x1,#-16*4] +.if \Columns\() > 8 + ldp q6,q7,[x1,#-8*4] +.endif + MultiplyAccumulateBlock \Columns\(),\Rows\(),3 + sub x9,x9,#4 + tbz x9,#63,.L\Mode\().Compute\Columns\().x\Rows\().BlockBy4Loop + +.L\Mode\().ProcessRemaining\Columns\().x\Rows\().Blocks: + add x9,x9,#4 // correct for over-subtract above + cbz x9,.L\Mode\().Output\Columns\().x\Rows\().Block + +.L\Mode\().Compute\Columns\().x\Rows\().BlockBy1Loop: + LoadMatrixAElementsBy1 \Rows\() + ldp q4,q5,[x1],#16*4 +.if \Columns\() > 8 + ldp q6,q7,[x1,#-8*4] +.endif + MultiplyAccumulateBlock \Columns\(),\Rows\(),0 + sub x9,x9,1 + cbnz x9,.L\Mode\().Compute\Columns\().x\Rows\().BlockBy1Loop + +.L\Mode\().Output\Columns\().x\Rows\().Block: + + .endm + +// +// MultiplyAlphaRow +// +// Generates the code to multiply a single row of the output block by the alpha +// value. +// + + .macro MultiplyAlphaRow Columns, Vec1Reg, Vec2Reg, Vec3Reg, Vec4Reg + +.if \Columns\() <= 4 + fmul v\Vec1Reg\().4s,v\Vec1Reg\().4s,v0.s[0] +.elif \Columns\() <= 8 + fmul v\Vec1Reg\().4s,v\Vec1Reg\().4s,v0.s[0] + fmul v\Vec2Reg\().4s,v\Vec2Reg\().4s,v0.s[0] +.elif \Columns\() <= 12 + fmul v\Vec1Reg\().4s,v\Vec1Reg\().4s,v0.s[0] + fmul v\Vec2Reg\().4s,v\Vec2Reg\().4s,v0.s[0] + fmul v\Vec3Reg\().4s,v\Vec3Reg\().4s,v0.s[0] +.else + fmul v\Vec1Reg\().4s,v\Vec1Reg\().4s,v0.s[0] + fmul v\Vec2Reg\().4s,v\Vec2Reg\().4s,v0.s[0] + fmul v\Vec3Reg\().4s,v\Vec3Reg\().4s,v0.s[0] + fmul v\Vec4Reg\().4s,v\Vec4Reg\().4s,v0.s[0] +.endif + + .endm + +// +// MultiplyAlphaBlock +// +// Generates the code to multiply the output block by the alpha value. +// + + .macro MultiplyAlphaBlock Columns, Rows + + MultiplyAlphaRow \Columns\(),16,17,18,19 +.if \Rows\() >= 2 + MultiplyAlphaRow \Columns\(),20,21,22,23 +.endif +.if \Rows\() >= 4 + MultiplyAlphaRow \Columns\(),24,25,26,27 + MultiplyAlphaRow \Columns\(),28,29,30,31 +.endif + + .endm + +// +// OutputRow1Element +// OutputRow2Element +// OutputRow4Element +// OutputRow8Element +// OutputRow16Element +// +// Generates the code to store elements to the output block. +// + + .macro OutputRow1Element Mode, AddrReg, Vec1Reg, Vec2Reg, Vec3Reg, Vec4Reg + +.ifeqs "\Mode\()","Add" + ld1 {v4.s}[0],[\AddrReg\()] + fmla v4.2s,v\Vec1Reg\().2s,v0.s[0] + st1 {v4.s}[0],[\AddrReg\()] // post-increment not needed for last element +.else + st1 {v\Vec1Reg\().s}[0],[\AddrReg\()]// post-increment not needed for last element +.endif + + .endm + + .macro OutputRow2Element Mode, AddrReg, Vec1Reg, Vec2Reg, Vec3Reg, Vec4Reg + +.ifeqs "\Mode\()","Add" + ld1 {v4.2s},[\AddrReg\()] + fmla v4.2s,v\Vec1Reg\().2s,v0.s[0] + st1 {v4.2s},[\AddrReg\()],#2*4 +.else + st1 {v\Vec1Reg\().2s},[\AddrReg\()],#2*4 +.endif + dup v\Vec1Reg\().4s,v\Vec1Reg\().s[2] // shift remaining elements down + + .endm + + .macro OutputRow4Element Mode, AddrReg, Vec1Reg, Vec2Reg, Vec3Reg, Vec4Reg + +.ifeqs "\Mode\()","Add" + ld1 {v4.4s},[\AddrReg\()] + fmla v4.4s,v\Vec1Reg\().4s,v0.s[0] + st1 {v4.4s},[\AddrReg\()],#4*4 +.else + st1 {v\Vec1Reg\().4s},[\AddrReg\()],#4*4 +.endif + mov v\Vec1Reg\().16b,v\Vec2Reg\().16b // shift remaining elements down + + .endm + + .macro OutputRow8Element Mode, AddrReg, Vec1Reg, Vec2Reg, Vec3Reg, Vec4Reg + +.ifeqs "\Mode\()","Add" + ldp q4,q5,[\AddrReg\()] + fmla v4.4s,v\Vec1Reg\().4s,v0.s[0] + fmla v5.4s,v\Vec2Reg\().4s,v0.s[0] + stp q4,q5,[\AddrReg\()],#8*4 +.else + stp q\Vec1Reg\(),q\Vec2Reg\(),[\AddrReg\()],#8*4 +.endif + mov v\Vec1Reg\().16b,v\Vec3Reg\().16b // shift remaining elements down + mov v\Vec2Reg\().16b,v\Vec4Reg\().16b + + .endm + + .macro OutputRow16Element Mode, AddrReg, Vec1Reg, Vec2Reg, Vec3Reg, Vec4Reg + +.ifeqs "\Mode\()","Add" + ldp q4,q5,[\AddrReg\()] + ldp q6,q7,[\AddrReg\(),#8*4] + fmla v4.4s,v\Vec1Reg\().4s,v0.s[0] + fmla v5.4s,v\Vec2Reg\().4s,v0.s[0] + fmla v6.4s,v\Vec3Reg\().4s,v0.s[0] + fmla v7.4s,v\Vec4Reg\().4s,v0.s[0] + stp q4,q5,[\AddrReg\()],#16*4 + stp q6,q7,[\AddrReg\(),#-8*4] +.else + stp q\Vec1Reg\(),q\Vec2Reg\(),[\AddrReg\()],#16*4 + stp q\Vec3Reg\(),q\Vec4Reg\(),[\AddrReg\(),#-8*4] +.endif + + .endm + +// +// OutputBlock +// +// Generates the code to store the output block. +// + + .macro OutputBlock Mode, Columns, Rows + + OutputRow\Columns\()Element \Mode\(),x2,16,17,18,19 +.if \Rows\() >= 2 + OutputRow\Columns\()Element \Mode\(),x13,20,21,22,23 +.endif +.if \Rows\() >= 4 + OutputRow\Columns\()Element \Mode\(),x14,24,25,26,27 + OutputRow\Columns\()Element \Mode\(),x15,28,29,30,31 +.endif + + .endm + +// +// ProcessRows +// +// Generates the code to process a compute and store the output block for a +// fixed number of rows. +// + + .macro ProcessRows Mode, Rows + + mov x4,#\Rows\() // return number of rows handled + cmp x5,#8 + ble .L\Mode\().ProcessRemainingCountN\Rows\() + +.L\Mode\().ProcessNextColumnLoop16x\Rows\(): + ComputeBlockLoop \Mode\(),16,\Rows\() +.ifeqs "\Mode\()","Zero" + MultiplyAlphaBlock 16,\Rows\() +.endif + sub x5,x5,#16 + tbnz x5,#63,.L\Mode\().OutputMasked16x\Rows\().Block + OutputBlock \Mode\(),16,\Rows\() + mov x0,x8 // reload matrix A + cmp x5,#8 + bgt .L\Mode\().ProcessNextColumnLoop16x\Rows\() + cbz x5,.L\Mode\().ExitKernel + +.L\Mode\().ProcessRemainingCountN\Rows\(): + ComputeBlockLoop \Mode\(),8,\Rows\() +.ifeqs "\Mode\()","Zero" + MultiplyAlphaBlock 8,\Rows\() +.endif + +.L\Mode\().OutputMasked16x\Rows\().Block: + tbz x5,#3,.L\Mode\().OutputRemaining7x\Rows\().Block + OutputBlock \Mode\(),8,\Rows\() + +.L\Mode\().OutputRemaining7x\Rows\().Block: + tbz x5,#2,.L\Mode\().OutputRemaining3x\Rows\().Block + OutputBlock \Mode\(),4,\Rows\() + +.L\Mode\().OutputRemaining3x\Rows\().Block: + tbz x5,#1,.L\Mode\().OutputRemaining1x\Rows\().Block + OutputBlock \Mode\(),2,\Rows\() + +.L\Mode\().OutputRemaining1x\Rows\().Block: + tbz x5,#0,.L\Mode\().ExitKernel + OutputBlock \Mode\(),1,\Rows\() + + .endm + +/*++ + +Routine Description: + + This routine is an inner kernel to compute matrix multiplication for a + set of rows. + +Arguments: + + A (x0) - Supplies the address of matrix A. + + B (x1) - Supplies the address of matrix B. The matrix data has been packed + using MlasSgemmCopyPackB or MlasSgemmTransposePackB. + + C (x2) - Supplies the address of matrix C. + + CountK (x3) - Supplies the number of columns from matrix A and the number + of rows from matrix B to iterate over. + + CountM (x4) - Supplies the maximum number of rows that can be processed for + matrix A and matrix C. The actual number of rows handled for this + invocation depends on the kernel implementation. + + CountN (x5) - Supplies the number of columns from matrix B and matrix C to + iterate over. + + lda (x6) - Supplies the first dimension of matrix A. + + ldc (x7) - Supplies the first dimension of matrix C. + + Alpha (s0) - Supplies the scaler multiplier (see SGEMM definition). + +Return Value: + + Returns the number of rows handled. + +--*/ + + .macro SgemmKernelNeonFunction Mode + + .globl MlasSgemmKernel\Mode\() + .type MlasSgemmKernel\Mode\(),%function +MlasSgemmKernel\Mode\(): + + stp d8,d9,[sp,-32]! + stp d10,d11,[sp,16] + + add x13,x2,x7,uxtw 2 // compute matrix C plus 1 row + add x14,x13,x7,uxtw 2 // compute matrix C plus 2 rows + add x15,x14,x7,uxtw 2 // compute matrix C plus 3 rows + mov x8,x0 // save matrix A + +// +// Process 4 rows of the matrices. +// + + cmp x4,#4 + blt .L\Mode\().ProcessCountMLessThan4 + ProcessRows \Mode\(),4 + +// +// Restore non-volatile registers and return. +// + +.L\Mode\().ExitKernel: + mov x0,x4 + ldp d10,d11,[sp,16] + ldp d8,d9,[sp,32]! + ret + +// +// Process 2 rows of the matrices. +// + +.L\Mode\().ProcessCountMLessThan4: + cmp x4,#2 + blt .L\Mode\().ProcessCountMLessThan2 + ProcessRows \Mode\(),2 + b .L\Mode\().ExitKernel + +// +// Process 1 row of the matrices. +// + +.L\Mode\().ProcessCountMLessThan2: + ProcessRows \Mode\(),1 + b .L\Mode\().ExitKernel + + .endm + + SgemmKernelNeonFunction Zero + SgemmKernelNeonFunction Add + + .end diff --git a/onnxruntime/core/mlas/lib/amd64/SgemmKernelAvx.asm b/onnxruntime/core/mlas/lib/amd64/SgemmKernelAvx.asm new file mode 100644 index 0000000000000..4c31cc8c73ca7 --- /dev/null +++ b/onnxruntime/core/mlas/lib/amd64/SgemmKernelAvx.asm @@ -0,0 +1,1009 @@ +;++ +; +; Copyright (c) Microsoft Corporation. All rights reserved. +; +; Licensed under the MIT License. +; +; Module Name: +; +; SgemmKernelAvx.asm +; +; Abstract: +; +; This module implements the kernels for the single precision matrix/matrix +; multiply operation (SGEMM). +; +; This implementation uses AVX instructions. +; +;-- + + .xlist +INCLUDE macamd64.inc +INCLUDE SgemmKernelCommon.inc + .list + + EXTERN MlasMaskMoveAvx:NEAR + +; +; ComputeBlockAvxBy16 +; +; Macro Description: +; +; This macro multiplies and accumulates for a 16xN block (where N is 1,2,4) +; of the output matrix. +; +; Arguments: +; +; Count - Supplies the number of rows to access from matrix A. +; +; VectorOffset - Supplies the byte offset from matrix B to fetch elements. +; +; BroadcastOffset - Supplies the byte offset from matrix A to fetch elements. +; +; Implicit Arguments: +; +; rbx - Supplies the address into the matrix A data plus 2 rows. +; +; rcx - Supplies the address into the matrix A data. +; +; rdx - Supplies the address into the matrix B data. +; +; r10 - Supplies the length in bytes of a row from matrix A. +; +; ymm8-ymm15 - Supplies the block accumulators. +; + +ComputeBlockAvxBy16 MACRO Count, VectorOffset, BroadcastOffset + +IF Count EQ 1 + vbroadcastss ymm3,DWORD PTR [rcx+BroadcastOffset] + vmulps ymm4,ymm3,YMMWORD PTR [rdx+VectorOffset] + vaddps ymm8,ymm8,ymm4 + vmulps ymm5,ymm3,YMMWORD PTR [rdx+VectorOffset+32] + vaddps ymm9,ymm9,ymm5 +ELSE + vmovaps ymm0,YMMWORD PTR [rdx+VectorOffset] + vmovaps ymm1,YMMWORD PTR [rdx+VectorOffset+32] + vbroadcastss ymm3,DWORD PTR [rcx+BroadcastOffset] + vmulps ymm4,ymm3,ymm0 + vaddps ymm8,ymm8,ymm4 + vmulps ymm5,ymm3,ymm1 + vaddps ymm9,ymm9,ymm5 +IF Count GE 2 + vbroadcastss ymm3,DWORD PTR [rcx+r10+BroadcastOffset] + vmulps ymm6,ymm3,ymm0 + vaddps ymm10,ymm10,ymm6 + vmulps ymm7,ymm3,ymm1 + vaddps ymm11,ymm11,ymm7 +ENDIF +IF Count GE 4 + vbroadcastss ymm3,DWORD PTR [rbx+BroadcastOffset] + vmulps ymm4,ymm3,ymm0 + vaddps ymm12,ymm12,ymm4 + vmulps ymm5,ymm3,ymm1 + vaddps ymm13,ymm13,ymm5 + vbroadcastss ymm3,DWORD PTR [rbx+r10+BroadcastOffset] + vmulps ymm6,ymm3,ymm0 + vaddps ymm14,ymm14,ymm6 + vmulps ymm7,ymm3,ymm1 + vaddps ymm15,ymm15,ymm7 +ENDIF +ENDIF + + ENDM + +; +; ComputeBlockAvxBy8 +; +; Macro Description: +; +; This macro multiplies and accumulates for a 8xN block (where N is 1,2,4) +; of the output matrix. +; +; Arguments: +; +; Count - Supplies the number of rows to access from matrix A. +; +; VectorOffset - Supplies the byte offset from matrix B to fetch elements. +; +; BroadcastOffset - Supplies the byte offset from matrix A to fetch elements. +; +; Implicit Arguments: +; +; rbx - Supplies the address into the matrix A data plus 2 rows. +; +; rcx - Supplies the address into the matrix A data. +; +; rdx - Supplies the address into the matrix B data. +; +; r10 - Supplies the length in bytes of a row from matrix A. +; +; ymm8-ymm15 - Supplies the block accumulators. +; + +ComputeBlockAvxBy8 MACRO Count, VectorOffset, BroadcastOffset + +IF Count EQ 1 + vbroadcastss ymm3,DWORD PTR [rcx+BroadcastOffset] + vmulps ymm5,ymm3,YMMWORD PTR [rdx+VectorOffset] + vaddps ymm9,ymm9,ymm5 +ELSE + vmovaps ymm0,YMMWORD PTR [rdx+VectorOffset] + vbroadcastss ymm3,DWORD PTR [rcx+BroadcastOffset] + vmulps ymm5,ymm3,ymm0 + vaddps ymm9,ymm9,ymm5 +IF Count GE 2 + vbroadcastss ymm3,DWORD PTR [rcx+r10+BroadcastOffset] + vmulps ymm7,ymm3,ymm0 + vaddps ymm11,ymm11,ymm7 +ENDIF +IF Count GE 4 + vbroadcastss ymm3,DWORD PTR [rbx+BroadcastOffset] + vmulps ymm5,ymm3,ymm0 + vaddps ymm13,ymm13,ymm5 + vbroadcastss ymm3,DWORD PTR [rbx+r10+BroadcastOffset] + vmulps ymm7,ymm3,ymm0 + vaddps ymm15,ymm15,ymm7 +ENDIF +ENDIF + + ENDM + +;++ +; +; Routine Description: +; +; This routine is an inner kernel to compute matrix multiplication for a +; set of rows. +; +; Arguments: +; +; A (rcx) - Supplies the address of matrix A. +; +; B (rdx) - Supplies the address of matrix B. The matrix data has been packed +; using MlasSgemmCopyPackB or MlasSgemmTransposePackB. +; +; C (r8) - Supplies the address of matrix C. +; +; CountK (r9) - Supplies the number of columns from matrix A and the number +; of rows from matrix B to iterate over. +; +; CountM - Supplies the maximum number of rows that can be processed for +; matrix A and matrix C. The actual number of rows handled for this +; invocation depends on the kernel implementation. +; +; CountN - Supplies the number of columns from matrix B and matrix C to iterate +; over. +; +; lda - Supplies the first dimension of matrix A. +; +; ldc - Supplies the first dimension of matrix C. +; +; Alpha - Supplies the scaler multiplier (see SGEMM definition). +; +; Return Value: +; +; Returns the number of rows handled. +; +;-- + +SgemmKernelAvxFunction MACRO Mode + + NESTED_ENTRY MlasSgemmKernel&Mode&Avx, _TEXT + + SgemmKernelAvxEntry + + vbroadcastss ymm2,DWORD PTR SgemmKernelFrame.Alpha[rsp] + +; +; Process 4 rows of the matrices. +; + + cmp r11,4 + jb ProcessCountMLessThan4 + mov r11d,4 ; return 4 rows handled + cmp rbp,8 + jbe ProcessRemainingCountN4 + +ProcessNextColumnLoop16x4: + vxorps xmm8,xmm8,xmm8 ; clear block accumulators + vxorps xmm9,xmm9,xmm9 + vxorps xmm10,xmm10,xmm10 + vxorps xmm11,xmm11,xmm11 + vxorps xmm12,xmm12,xmm12 + vxorps xmm13,xmm13,xmm13 + vxorps xmm14,xmm14,xmm14 + vxorps xmm15,xmm15,xmm15 + lea rbx,[rcx+r10*2] ; compute matrix A plus 2 rows + ComputeBlockAvxLoop ComputeBlockAvxBy16, 4 + vmulps ymm8,ymm8,ymm2 ; multiply by alpha + vmulps ymm9,ymm9,ymm2 + vmulps ymm10,ymm10,ymm2 + vmulps ymm11,ymm11,ymm2 + vmulps ymm12,ymm12,ymm2 + vmulps ymm13,ymm13,ymm2 + vmulps ymm14,ymm14,ymm2 + vmulps ymm15,ymm15,ymm2 + lea rcx,[r8+rax*2] ; compute matrix C plus 2 rows + sub rbp,16 + jb OutputMasked16x4Block +IFIDNI , + vaddps ymm8,ymm8,YMMWORD PTR [r8] + vaddps ymm9,ymm9,YMMWORD PTR [r8+32] + vaddps ymm10,ymm10,YMMWORD PTR [r8+rax] + vaddps ymm11,ymm11,YMMWORD PTR [r8+rax+32] + vaddps ymm12,ymm12,YMMWORD PTR [rcx] + vaddps ymm13,ymm13,YMMWORD PTR [rcx+32] + vaddps ymm14,ymm14,YMMWORD PTR [rcx+rax] + vaddps ymm15,ymm15,YMMWORD PTR [rcx+rax+32] +ENDIF + vmovups YMMWORD PTR [r8],ymm8 + vmovups YMMWORD PTR [r8+32],ymm9 + vmovups YMMWORD PTR [r8+rax],ymm10 + vmovups YMMWORD PTR [r8+rax+32],ymm11 + vmovups YMMWORD PTR [rcx],ymm12 + vmovups YMMWORD PTR [rcx+32],ymm13 + vmovups YMMWORD PTR [rcx+rax],ymm14 + vmovups YMMWORD PTR [rcx+rax+32],ymm15 + add r8,16*4 ; advance matrix C by 16 columns + mov rcx,rsi ; reload matrix A + cmp rbp,8 + ja ProcessNextColumnLoop16x4 + test rbp,rbp + jz ExitKernel + +ProcessRemainingCountN4: + vxorps xmm9,xmm9,xmm9 ; clear block accumulators + vxorps xmm11,xmm11,xmm11 + vxorps xmm13,xmm13,xmm13 + vxorps xmm15,xmm15,xmm15 + lea rbx,[rcx+r10*2] ; compute matrix A plus 2 rows + ComputeBlockAvxLoop ComputeBlockAvxBy8, 4 + vmulps ymm9,ymm9,ymm2 ; multiply by alpha + vmulps ymm11,ymm11,ymm2 + vmulps ymm13,ymm13,ymm2 + vmulps ymm15,ymm15,ymm2 + lea rcx,[r8+rax*2] ; compute matrix C plus 2 rows + cmp rbp,8 + jb OutputMasked8x4Block +IFIDNI , + vaddps ymm9,ymm9,YMMWORD PTR [r8] + vaddps ymm11,ymm11,YMMWORD PTR [r8+rax] + vaddps ymm13,ymm13,YMMWORD PTR [rcx] + vaddps ymm15,ymm15,YMMWORD PTR [rcx+rax] +ENDIF + vmovups YMMWORD PTR [r8],ymm9 + vmovups YMMWORD PTR [r8+rax],ymm11 + vmovups YMMWORD PTR [rcx],ymm13 + vmovups YMMWORD PTR [rcx+rax],ymm15 + jmp ExitKernel + +OutputMasked16x4Block: +IFIDNI , + vaddps ymm8,ymm8,YMMWORD PTR [r8] + vaddps ymm10,ymm10,YMMWORD PTR [r8+rax] + vaddps ymm12,ymm12,YMMWORD PTR [rcx] + vaddps ymm14,ymm14,YMMWORD PTR [rcx+rax] +ENDIF + vmovups YMMWORD PTR [r8],ymm8 + vmovups YMMWORD PTR [r8+rax],ymm10 + vmovups YMMWORD PTR [rcx],ymm12 + vmovups YMMWORD PTR [rcx+rax],ymm14 + add r8,8*4 ; advance matrix C by 8 columns + add rcx,8*4 ; advance matrix C plus 2 rows by 8 columns + add rbp,8 ; correct for over-subtract above + +OutputMasked8x4Block: + mov DWORD PTR SgemmKernelFrame.CountN[rsp],ebp + vbroadcastss xmm0,DWORD PTR SgemmKernelFrame.CountN[rsp] + vpcmpgtd xmm1,xmm0,XMMWORD PTR [MlasMaskMoveAvx+16] + vpcmpgtd xmm0,xmm0,XMMWORD PTR [MlasMaskMoveAvx] + vinsertf128 ymm0,ymm0,xmm1,1 +IFIDNI , + vmaskmovps ymm8,ymm0,YMMWORD PTR [r8] + vmaskmovps ymm10,ymm0,YMMWORD PTR [r8+rax] + vmaskmovps ymm12,ymm0,YMMWORD PTR [rcx] + vmaskmovps ymm14,ymm0,YMMWORD PTR [rcx+rax] + vaddps ymm9,ymm9,ymm8 + vaddps ymm11,ymm11,ymm10 + vaddps ymm13,ymm13,ymm12 + vaddps ymm15,ymm15,ymm14 +ENDIF + vmaskmovps YMMWORD PTR [r8],ymm0,ymm9 + vmaskmovps YMMWORD PTR [r8+rax],ymm0,ymm11 + vmaskmovps YMMWORD PTR [rcx],ymm0,ymm13 + vmaskmovps YMMWORD PTR [rcx+rax],ymm0,ymm15 + +; +; Restore non-volatile registers and return. +; + +ExitKernel: + vzeroupper + SgemmKernelAvxExit + +; +; Process 2 rows of the matrices. +; + +ProcessCountMLessThan4: + cmp r11,2 + jb ProcessCountMLessThan2 + mov r11d,2 ; return 2 rows handled + cmp rbp,8 + jbe ProcessRemainingCountN2 + +ProcessNextColumnLoop16x2: + vxorps xmm8,xmm8,xmm8 ; clear block accumulators + vxorps xmm9,xmm9,xmm9 + vxorps xmm10,xmm10,xmm10 + vxorps xmm11,xmm11,xmm11 + ComputeBlockAvxLoop ComputeBlockAvxBy16, 2 + vmulps ymm8,ymm8,ymm2 ; multiply by alpha + vmulps ymm9,ymm9,ymm2 + vmulps ymm10,ymm10,ymm2 + vmulps ymm11,ymm11,ymm2 + sub rbp,16 + jb OutputMasked16x2Block +IFIDNI , + vaddps ymm8,ymm8,YMMWORD PTR [r8] + vaddps ymm9,ymm9,YMMWORD PTR [r8+32] + vaddps ymm10,ymm10,YMMWORD PTR [r8+rax] + vaddps ymm11,ymm11,YMMWORD PTR [r8+rax+32] +ENDIF + vmovups YMMWORD PTR [r8],ymm8 + vmovups YMMWORD PTR [r8+32],ymm9 + vmovups YMMWORD PTR [r8+rax],ymm10 + vmovups YMMWORD PTR [r8+rax+32],ymm11 + add r8,16*4 ; advance matrix C by 16 columns + mov rcx,rsi ; reload matrix A + cmp rbp,8 + ja ProcessNextColumnLoop16x2 + test rbp,rbp + jz ExitKernel + +ProcessRemainingCountN2: + vxorps xmm9,xmm9,xmm9 ; clear block accumulators + vxorps xmm11,xmm11,xmm11 + ComputeBlockAvxLoop ComputeBlockAvxBy8, 2 + vmulps ymm9,ymm9,ymm2 ; multiply by alpha + vmulps ymm11,ymm11,ymm2 + cmp rbp,8 + jb OutputMasked8x2Block +IFIDNI , + vaddps ymm9,ymm9,YMMWORD PTR [r8] + vaddps ymm11,ymm11,YMMWORD PTR [r8+rax] +ENDIF + vmovups YMMWORD PTR [r8],ymm9 + vmovups YMMWORD PTR [r8+rax],ymm11 + jmp ExitKernel + +OutputMasked16x2Block: +IFIDNI , + vaddps ymm8,ymm8,YMMWORD PTR [r8] + vaddps ymm10,ymm10,YMMWORD PTR [r8+rax] +ENDIF + vmovups YMMWORD PTR [r8],ymm8 + vmovups YMMWORD PTR [r8+rax],ymm10 + add r8,8*4 ; advance matrix C by 8 columns + add rbp,8 ; correct for over-subtract above + +OutputMasked8x2Block: + mov DWORD PTR SgemmKernelFrame.CountN[rsp],ebp + vbroadcastss xmm0,DWORD PTR SgemmKernelFrame.CountN[rsp] + vpcmpgtd xmm1,xmm0,XMMWORD PTR [MlasMaskMoveAvx+16] + vpcmpgtd xmm0,xmm0,XMMWORD PTR [MlasMaskMoveAvx] + vinsertf128 ymm0,ymm0,xmm1,1 +IFIDNI , + vmaskmovps ymm8,ymm0,YMMWORD PTR [r8] + vmaskmovps ymm10,ymm0,YMMWORD PTR [r8+rax] + vaddps ymm9,ymm9,ymm8 + vaddps ymm11,ymm11,ymm10 +ENDIF + vmaskmovps YMMWORD PTR [r8],ymm0,ymm9 + vmaskmovps YMMWORD PTR [r8+rax],ymm0,ymm11 + jmp ExitKernel + +; +; Process 1 row of the matrices. +; + +ProcessCountMLessThan2: + mov r11d,1 ; return 1 row handled + cmp rbp,8 + jbe ProcessRemainingCountN1 + +ProcessNextColumnLoop16x1: + vxorps xmm8,xmm8,xmm8 ; clear block accumulators + vxorps xmm9,xmm9,xmm9 + ComputeBlockAvxLoop ComputeBlockAvxBy16, 1 + vmulps ymm8,ymm8,ymm2 ; multiply by alpha + vmulps ymm9,ymm9,ymm2 + sub rbp,16 + jb OutputMasked16x1Block +IFIDNI , + vaddps ymm8,ymm8,YMMWORD PTR [r8] + vaddps ymm9,ymm9,YMMWORD PTR [r8+32] +ENDIF + vmovups YMMWORD PTR [r8],ymm8 + vmovups YMMWORD PTR [r8+32],ymm9 + add r8,16*4 ; advance matrix C by 16 columns + mov rcx,rsi ; reload matrix A + cmp rbp,8 + ja ProcessNextColumnLoop16x1 + test rbp,rbp + jz ExitKernel + +ProcessRemainingCountN1: + vxorps xmm9,xmm9,xmm9 ; clear block accumulators + ComputeBlockAvxLoop ComputeBlockAvxBy8, 1 + vmulps ymm9,ymm9,ymm2 ; multiply by alpha + cmp rbp,8 + jb OutputMasked8x1Block +IFIDNI , + vaddps ymm9,ymm9,YMMWORD PTR [r8] +ENDIF + vmovups YMMWORD PTR [r8],ymm9 + jmp ExitKernel + +OutputMasked16x1Block: +IFIDNI , + vaddps ymm8,ymm8,YMMWORD PTR [r8] +ENDIF + vmovups YMMWORD PTR [r8],ymm8 + add r8,8*4 ; advance matrix C by 8 columns + add rbp,8 ; correct for over-subtract above + +OutputMasked8x1Block: + mov DWORD PTR SgemmKernelFrame.CountN[rsp],ebp + vbroadcastss xmm0,DWORD PTR SgemmKernelFrame.CountN[rsp] + vpcmpgtd xmm1,xmm0,XMMWORD PTR [MlasMaskMoveAvx+16] + vpcmpgtd xmm0,xmm0,XMMWORD PTR [MlasMaskMoveAvx] + vinsertf128 ymm0,ymm0,xmm1,1 +IFIDNI , + vmaskmovps ymm8,ymm0,YMMWORD PTR [r8] + vaddps ymm9,ymm9,ymm8 +ENDIF + vmaskmovps YMMWORD PTR [r8],ymm0,ymm9 + jmp ExitKernel + + NESTED_END MlasSgemmKernel&Mode&Avx, _TEXT + + ENDM + +SgemmKernelAvxFunction Zero +SgemmKernelAvxFunction Add + +;++ +; +; Routine Description: +; +; This routine is an inner kernel to compute matrix multiplication for a +; set of rows. This handles the special case of M=1. +; +; The elements in matrix B are not transposed. +; +; Arguments: +; +; A (rcx) - Supplies the address of matrix A. +; +; B (rdx) - Supplies the address of matrix B. +; +; C (r8) - Supplies the address of matrix C. +; +; CountK (r9) - Supplies the number of columns from matrix A and the number +; of rows from matrix B to iterate over. +; +; CountN - Supplies the number of columns from matrix B and matrix C to iterate +; over. +; +; ldb - Supplies the first dimension of matrix B. +; +; Beta - Supplies the scaler multiplier (see SGEMM definition). +; +; Return Value: +; +; None. +; +;-- + + NESTED_ENTRY MlasSgemmKernelM1Avx, _TEXT + + rex_push_reg rbp + push_reg rbx + push_reg rsi + alloc_stack (SgemmKernelM1Frame.SavedRsi) + save_xmm128_avx xmm6,SgemmKernelM1Frame.SavedXmm6 + save_xmm128_avx xmm7,SgemmKernelM1Frame.SavedXmm7 + save_xmm128_avx xmm8,SgemmKernelM1Frame.SavedXmm8 + + END_PROLOGUE + + mov rbx,SgemmKernelM1Frame.ldb[rsp] + shl rbx,2 ; convert ldb to bytes + mov r10,r8 + mov r11,rdx + mov rbp,SgemmKernelM1Frame.CountN[rsp] + +; +; Compute the initial results mask for zeroing or accumulate mode. +; + + vxorps xmm0,xmm0,xmm0 + vcmpeqss xmm0,xmm0,DWORD PTR SgemmKernelM1Frame.Beta[rsp] + vshufps xmm0,xmm0,xmm0,0 + vinsertf128 ymm0,ymm0,xmm0,1 + +; +; Compute the conditional load/store mask for an unaligned CountN. +; + + mov eax,ebp + and eax,7 + vmovd xmm7,eax + vshufps xmm7,xmm7,xmm7,0 + vpcmpgtd xmm6,xmm7,XMMWORD PTR [MlasMaskMoveAvx+16] + vpcmpgtd xmm7,xmm7,XMMWORD PTR [MlasMaskMoveAvx] + vinsertf128 ymm7,ymm7,xmm6,1 + +; +; Process 4 rows of the matrices in a loop. +; + + sub r9,4 + jb ProcessRemainingCountK + +ProcessRowLoop4: + vbroadcastss ymm2,DWORD PTR [rcx] + mov rax,rbp ; reload CountN + vbroadcastss ymm3,DWORD PTR [rcx+4] + mov rdx,r11 ; reload matrix B + vbroadcastss ymm4,DWORD PTR [rcx+8] + mov r8,r10 ; reload matrix C + vbroadcastss ymm5,DWORD PTR [rcx+12] + add rcx,4*4 ; advance matrix A by 4 columns + lea r11,[rdx+rbx*4] ; advance matrix B by 4 rows + sub rax,16 + jb ProcessRemainingCountN4 + +ProcessColumnLoop4: + lea rsi,[rdx+rbx*2] ; compute matrix B plus 2 rows + vmulps ymm1,ymm2,YMMWORD PTR [rdx] + vmulps ymm6,ymm2,YMMWORD PTR [rdx+32] + vmulps ymm8,ymm3,YMMWORD PTR [rdx+rbx] + vaddps ymm1,ymm1,ymm8 + vmulps ymm8,ymm3,YMMWORD PTR [rdx+rbx+32] + vaddps ymm6,ymm6,ymm8 + vmulps ymm8,ymm4,YMMWORD PTR [rsi] + vaddps ymm1,ymm1,ymm8 + vmulps ymm8,ymm4,YMMWORD PTR [rsi+32] + vaddps ymm6,ymm6,ymm8 + vmulps ymm8,ymm5,YMMWORD PTR [rsi+rbx] + vaddps ymm1,ymm1,ymm8 + vmulps ymm8,ymm5,YMMWORD PTR [rsi+rbx+32] + vaddps ymm6,ymm6,ymm8 + vandnps ymm8,ymm0,YMMWORD PTR [r8] + vaddps ymm1,ymm1,ymm8 + vandnps ymm8,ymm0,YMMWORD PTR [r8+32] + vaddps ymm6,ymm6,ymm8 + vmovups YMMWORD PTR [r8],ymm1 + vmovups YMMWORD PTR [r8+32],ymm6 + add rdx,16*4 ; advance matrix B by 16 columns + add r8,16*4 ; advance matrix C by 16 columns + sub rax,16 + jae ProcessColumnLoop4 + +ProcessRemainingCountN4: + test al,15 ; test for unaligned columns + jz ProcessedRemainingCountN4 + test al,8 ; CountN >= 8? + jz ProcessRemainingCountNSmall4 + lea rsi,[rdx+rbx*2] ; compute matrix B plus 2 rows + vmulps ymm1,ymm2,YMMWORD PTR [rdx] + vmulps ymm8,ymm3,YMMWORD PTR [rdx+rbx] + vaddps ymm1,ymm1,ymm8 + vmulps ymm8,ymm4,YMMWORD PTR [rsi] + vaddps ymm1,ymm1,ymm8 + vmulps ymm8,ymm5,YMMWORD PTR [rsi+rbx] + vaddps ymm1,ymm1,ymm8 + vandnps ymm8,ymm0,YMMWORD PTR [r8] + vaddps ymm1,ymm1,ymm8 + vmovups YMMWORD PTR [r8],ymm1 + add rdx,8*4 ; advance matrix B by 8 columns + add r8,8*4 ; advance matrix C by 8 columns + test al,7 + jz ProcessedRemainingCountN4 + +ProcessRemainingCountNSmall4: + lea rsi,[rdx+rbx*2] ; compute matrix B plus 2 rows + vmaskmovps ymm6,ymm7,YMMWORD PTR [rdx] + vmulps ymm1,ymm2,ymm6 + vmaskmovps ymm6,ymm7,YMMWORD PTR [rdx+rbx] + vmulps ymm8,ymm3,ymm6 + vaddps ymm1,ymm1,ymm8 + vmaskmovps ymm6,ymm7,YMMWORD PTR [rsi] + vmulps ymm8,ymm4,ymm6 + vaddps ymm1,ymm1,ymm8 + vmaskmovps ymm6,ymm7,YMMWORD PTR [rsi+rbx] + vmulps ymm8,ymm5,ymm6 + vaddps ymm1,ymm1,ymm8 + vmaskmovps ymm6,ymm7,YMMWORD PTR [r8] + vandnps ymm6,ymm0,ymm6 + vaddps ymm1,ymm1,ymm6 + vmaskmovps YMMWORD PTR [r8],ymm7,ymm1 + +ProcessedRemainingCountN4: + vxorps xmm0,xmm0,xmm0 ; switch to accumulate mode + sub r9,4 + jae ProcessRowLoop4 + +ProcessRemainingCountK: + test r9d,2 + jnz ProcessRowLoop2 + test r9d,1 + jnz ProcessRowLoop1 + +ExitKernel: + vzeroupper + vmovaps xmm6,SgemmKernelM1Frame.SavedXmm6[rsp] + vmovaps xmm7,SgemmKernelM1Frame.SavedXmm7[rsp] + vmovaps xmm8,SgemmKernelM1Frame.SavedXmm8[rsp] + add rsp,(SgemmKernelM1Frame.SavedRsi) + + BEGIN_EPILOGUE + + pop rsi + pop rbx + pop rbp + ret + +; +; Process 2 rows of the matrices. +; + +ProcessRowLoop2: + vbroadcastss ymm2,DWORD PTR [rcx] + mov rax,rbp ; reload CountN + vbroadcastss ymm3,DWORD PTR [rcx+4] + mov rdx,r11 ; reload matrix B + mov r8,r10 ; reload matrix C + add rcx,2*4 ; advance matrix A by 2 columns + lea r11,[rdx+rbx*2] ; advance matrix B by 2 rows + sub rax,8 + jb ProcessRemainingCountN2 + +ProcessColumnLoop2: + vmulps ymm1,ymm2,YMMWORD PTR [rdx] + vmulps ymm8,ymm3,YMMWORD PTR [rdx+rbx] + vaddps ymm1,ymm1,ymm8 + vandnps ymm6,ymm0,YMMWORD PTR [r8] + vaddps ymm1,ymm1,ymm6 + vmovups YMMWORD PTR [r8],ymm1 + add rdx,8*4 ; advance matrix B by 8 columns + add r8,8*4 ; advance matrix C by 8 columns + sub rax,8 + jae ProcessColumnLoop2 + +ProcessRemainingCountN2: + test al,7 ; test for unaligned columns + jz ProcessedRemainingCountN2 + vmaskmovps ymm6,ymm7,YMMWORD PTR [rdx] + vmulps ymm1,ymm2,ymm6 + vmaskmovps ymm6,ymm7,YMMWORD PTR [rdx+rbx] + vmulps ymm8,ymm3,ymm6 + vaddps ymm1,ymm1,ymm8 + vmaskmovps ymm6,ymm7,YMMWORD PTR [r8] + vandnps ymm6,ymm0,ymm6 + vaddps ymm1,ymm1,ymm6 + vmaskmovps YMMWORD PTR [r8],ymm7,ymm1 + +ProcessedRemainingCountN2: + test r9d,1 + jz ExitKernel + vxorps xmm0,xmm0,xmm0 ; switch to accumulate mode + +; +; Process 1 row of the matrices. +; + +ProcessRowLoop1: + vbroadcastss ymm2,DWORD PTR [rcx] + mov rax,rbp ; reload CountN + mov rdx,r11 ; reload matrix B + mov r8,r10 ; reload matrix C + sub rax,8 + jb ProcessRemainingCountN1 + +ProcessColumnLoop1: + vmulps ymm1,ymm2,YMMWORD PTR [rdx] + vandnps ymm6,ymm0,YMMWORD PTR [r8] + vaddps ymm1,ymm1,ymm6 + vmovups YMMWORD PTR [r8],ymm1 + add rdx,8*4 ; advance matrix B by 8 columns + add r8,8*4 ; advance matrix C by 8 columns + sub rax,8 + jae ProcessColumnLoop1 + +ProcessRemainingCountN1: + test al,7 ; test for unaligned columns + jz ExitKernel + vmaskmovps ymm6,ymm7,YMMWORD PTR [rdx] + vmulps ymm1,ymm2,ymm6 + vmaskmovps ymm6,ymm7,YMMWORD PTR [r8] + vandnps ymm6,ymm0,ymm6 + vaddps ymm1,ymm1,ymm6 + vmaskmovps YMMWORD PTR [r8],ymm7,ymm1 + jmp ExitKernel + + NESTED_END MlasSgemmKernelM1Avx, _TEXT + +;++ +; +; Routine Description: +; +; This routine is an inner kernel to compute matrix multiplication for a +; set of rows. This handles the special case of M=1. +; +; The elements in matrix B are transposed. +; +; Arguments: +; +; A (rcx) - Supplies the address of matrix A. +; +; B (rdx) - Supplies the address of matrix B. The elements are transposed. +; +; C (r8) - Supplies the address of matrix C. +; +; CountK (r9) - Supplies the number of columns from matrix A and the number +; of columns from matrix B to iterate over. +; +; CountN - Supplies the number of rows from matrix B and the number of columns +; from matrix C to iterate over. +; +; ldb - Supplies the first dimension of matrix B. +; +; Beta - Supplies the scaler multiplier (see SGEMM definition). +; +; Return Value: +; +; None. +; +;-- + + NESTED_ENTRY MlasSgemmKernelM1TransposeBAvx, _TEXT + + rex_push_reg rbp + push_reg rbx + push_reg rsi + alloc_stack (SgemmKernelM1Frame.SavedRsi) + save_xmm128_avx xmm6,SgemmKernelM1Frame.SavedXmm6 + save_xmm128_avx xmm7,SgemmKernelM1Frame.SavedXmm7 + + END_PROLOGUE + + mov rbx,SgemmKernelM1Frame.ldb[rsp] + shl rbx,2 ; convert ldb to bytes + mov r10,rcx + mov r11,rdx + mov rbp,SgemmKernelM1Frame.CountN[rsp] + +; +; Compute the results mask for zeroing or accumulate mode. +; + + vxorps xmm0,xmm0,xmm0 + vcmpeqss xmm0,xmm0,DWORD PTR SgemmKernelM1Frame.Beta[rsp] + vshufps xmm0,xmm0,xmm0,0 + +; +; Compute the conditional load/store mask for an unaligned CountK. +; + + mov eax,r9d + and eax,7 + vmovd xmm7,eax + vshufps xmm7,xmm7,xmm7,0 + vpcmpgtd xmm6,xmm7,XMMWORD PTR [MlasMaskMoveAvx+16] + vpcmpgtd xmm7,xmm7,XMMWORD PTR [MlasMaskMoveAvx] + vinsertf128 ymm7,ymm7,xmm6,1 + +; +; Process 4 rows of the matrices in a loop. +; + + sub rbp,4 + jb ProcessRemainingCountN + +ProcessRowLoop4: + vxorps xmm2,xmm2,xmm2 ; clear row accumulators + vxorps xmm3,xmm3,xmm3 + vxorps xmm4,xmm4,xmm4 + vxorps xmm5,xmm5,xmm5 + mov rcx,r10 ; reload matrix A + mov rdx,r11 ; reload matrix B + mov rax,r9 ; reload CountK + lea r11,[rdx+rbx*4] ; advance matrix B by 4 rows + sub rax,8 + jb ProcessRemainingCountK4 + +ProcessColumnLoop4: + lea rsi,[rdx+rbx*2] ; compute matrix B plus 2 rows + vmovups ymm1,YMMWORD PTR [rcx] + vmulps ymm6,ymm1,YMMWORD PTR [rdx] + vaddps ymm2,ymm2,ymm6 + vmulps ymm6,ymm1,YMMWORD PTR [rdx+rbx] + vaddps ymm3,ymm3,ymm6 + vmulps ymm6,ymm1,YMMWORD PTR [rsi] + vaddps ymm4,ymm4,ymm6 + vmulps ymm6,ymm1,YMMWORD PTR [rsi+rbx] + vaddps ymm5,ymm5,ymm6 + add rcx,8*4 ; advance matrix A by 8 columns + add rdx,8*4 ; advance matrix B by 8 columns + sub rax,8 + jae ProcessColumnLoop4 + +ProcessRemainingCountK4: + test al,7 ; test for unaligned columns + jz Output4x1Block + lea rsi,[rdx+rbx*2] ; compute matrix B plus 2 rows + vmaskmovps ymm1,ymm7,YMMWORD PTR [rcx] + vmaskmovps ymm6,ymm7,YMMWORD PTR [rdx] + vmulps ymm6,ymm1,ymm6 + vaddps ymm2,ymm2,ymm6 + vmaskmovps ymm6,ymm7,YMMWORD PTR [rdx+rbx] + vmulps ymm6,ymm1,ymm6 + vaddps ymm3,ymm3,ymm6 + vmaskmovps ymm6,ymm7,YMMWORD PTR [rsi] + vmulps ymm6,ymm1,ymm6 + vaddps ymm4,ymm4,ymm6 + vmaskmovps ymm6,ymm7,YMMWORD PTR [rsi+rbx] + vmulps ymm6,ymm1,ymm6 + vaddps ymm5,ymm5,ymm6 + +; +; Reduce and output the row accumulators. +; + +Output4x1Block: + vunpcklps ymm6,ymm2,ymm3 ; transpose row accumulators + vunpckhps ymm1,ymm2,ymm3 + vunpcklps ymm2,ymm4,ymm5 + vunpckhps ymm3,ymm4,ymm5 + vunpcklpd ymm4,ymm6,ymm2 + vunpckhpd ymm5,ymm6,ymm2 + vaddps ymm4,ymm4,ymm5 + vunpcklpd ymm6,ymm1,ymm3 + vunpckhpd ymm2,ymm1,ymm3 + vaddps ymm4,ymm4,ymm6 + vaddps ymm4,ymm4,ymm2 + vextractf128 xmm5,ymm4,1 + vaddps xmm4,xmm4,xmm5 + vandnps xmm6,xmm0,XMMWORD PTR [r8] + vaddps xmm4,xmm4,xmm6 + vmovups XMMWORD PTR [r8],xmm4 + add r8,4*4 ; advance matrix C by 4 columns + sub rbp,4 + jae ProcessRowLoop4 + +ProcessRemainingCountN: + test ebp,2 + jnz ProcessRowLoop2 + test ebp,1 + jnz ProcessRowLoop1 + +ExitKernel: + vzeroupper + vmovaps xmm6,SgemmKernelM1Frame.SavedXmm6[rsp] + vmovaps xmm7,SgemmKernelM1Frame.SavedXmm7[rsp] + add rsp,(SgemmKernelM1Frame.SavedRsi) + + BEGIN_EPILOGUE + + pop rsi + pop rbx + pop rbp + ret + +; +; Process 2 rows of the matrices. +; + +ProcessRowLoop2: + vxorps xmm2,xmm2,xmm2 ; clear row accumulators + vxorps xmm3,xmm3,xmm3 + mov rcx,r10 ; reload matrix A + mov rdx,r11 ; reload matrix B + mov rax,r9 ; reload CountK + lea r11,[rdx+rbx*2] ; advance matrix B by 2 rows + sub rax,8 + jb ProcessRemainingCountK2 + +ProcessColumnLoop2: + vmovups ymm1,YMMWORD PTR [rcx] + vmulps ymm6,ymm1,YMMWORD PTR [rdx] + vaddps ymm2,ymm2,ymm6 + vmulps ymm6,ymm1,YMMWORD PTR [rdx+rbx] + vaddps ymm3,ymm3,ymm6 + add rcx,8*4 ; advance matrix A by 8 columns + add rdx,8*4 ; advance matrix B by 8 columns + sub rax,8 + jae ProcessColumnLoop2 + +ProcessRemainingCountK2: + test al,7 ; test for unaligned columns + jz Output2x1Block + vmaskmovps ymm1,ymm7,YMMWORD PTR [rcx] + vmaskmovps ymm6,ymm7,YMMWORD PTR [rdx] + vmulps ymm6,ymm1,ymm6 + vaddps ymm2,ymm2,ymm6 + vmaskmovps ymm6,ymm7,YMMWORD PTR [rdx+rbx] + vmulps ymm6,ymm1,ymm6 + vaddps ymm3,ymm3,ymm6 + +; +; Reduce and output the row accumulators. +; + +Output2x1Block: + vunpcklps ymm4,ymm2,ymm3 ; reduce row accumulators + vunpckhps ymm2,ymm2,ymm3 + vaddps ymm2,ymm2,ymm4 + vextractf128 xmm4,ymm2,1 + vaddps xmm2,xmm2,xmm4 + vmovhlps xmm4,xmm2,xmm2 + vaddps xmm2,xmm2,xmm4 + vmovsd xmm3,QWORD PTR [r8] + vandnps xmm3,xmm0,xmm3 + vaddps xmm2,xmm2,xmm3 + vmovsd QWORD PTR [r8],xmm2 + add r8,2*4 ; advance matrix C by 2 columns + test ebp,1 + jz ExitKernel + +; +; Process 1 row of the matrices. +; + +ProcessRowLoop1: + vxorps xmm2,xmm2,xmm2 ; clear row accumulators + mov rcx,r10 ; reload matrix A + mov rdx,r11 ; reload matrix B + mov rax,r9 ; reload CountK + sub rax,8 + jb ProcessRemainingCountK1 + +ProcessColumnLoop1: + vmovups ymm1,YMMWORD PTR [rcx] + vmulps ymm6,ymm1,YMMWORD PTR [rdx] + vaddps ymm2,ymm2,ymm6 + add rcx,8*4 ; advance matrix A by 8 columns + add rdx,8*4 ; advance matrix B by 8 columns + sub rax,8 + jae ProcessColumnLoop1 + +ProcessRemainingCountK1: + test al,7 ; test for unaligned columns + jz Output1x1Block + vmaskmovps ymm1,ymm7,YMMWORD PTR [rcx] + vmaskmovps ymm6,ymm7,YMMWORD PTR [rdx] + vmulps ymm6,ymm1,ymm6 + vaddps ymm2,ymm2,ymm6 + +; +; Reduce and output the row accumulators. +; + +Output1x1Block: + vhaddps ymm2,ymm2,ymm2 ; reduce row accumulators + vhaddps ymm2,ymm2,ymm2 + vextractf128 xmm4,ymm2,1 + vaddss xmm2,xmm2,xmm4 + vmovss xmm3,DWORD PTR [r8] + vandnps xmm3,xmm0,xmm3 + vaddss xmm2,xmm2,xmm3 + vmovss DWORD PTR [r8],xmm2 + jmp ExitKernel + + NESTED_END MlasSgemmKernelM1TransposeBAvx, _TEXT + + END diff --git a/onnxruntime/core/mlas/lib/amd64/SgemmKernelAvx512F.asm b/onnxruntime/core/mlas/lib/amd64/SgemmKernelAvx512F.asm new file mode 100644 index 0000000000000..85ef906a4a143 --- /dev/null +++ b/onnxruntime/core/mlas/lib/amd64/SgemmKernelAvx512F.asm @@ -0,0 +1,585 @@ +;++ +; +; Copyright (c) Microsoft Corporation. All rights reserved. +; +; Licensed under the MIT License. +; +; Module Name: +; +; SgemmKernelAvx512F.asm +; +; Abstract: +; +; This module implements the kernels for the single precision matrix/matrix +; multiply operation (SGEMM). +; +; This implementation uses AVX512F instructions. +; +;-- + + .xlist +INCLUDE macamd64.inc +INCLUDE SgemmKernelCommon.inc + .list + +; +; ComputeBlockAvx512FBy32 +; +; This macro multiplies and accumulates for a 32xN block (where N is 1,3,6,12) +; of the output matrix. +; +; Arguments: +; +; Count - Supplies the number of rows to access from matrix A. +; +; VectorOffset - Supplies the byte offset from matrix B to fetch elements. +; +; BroadcastOffset - Supplies the byte offset from matrix A to fetch elements. +; +; Implicit Arguments: +; +; rbx - Supplies the address into the matrix A data plus 3 rows. +; +; rcx - Supplies the address into the matrix A data. +; +; rdx - Supplies the address into the matrix B data. +; +; r10 - Supplies the length in bytes of a row from matrix A. +; +; r13 - Supplies the address into the matrix A data plus 6 rows. +; +; r14 - Supplies the address into the matrix A data plus 9 rows. +; +; zmm4-zmm27 - Supplies the block accumulators. +; + +ComputeBlockAvx512FBy32 MACRO Count, VectorOffset, BroadcastOffset + +IF Count EQ 1 + vbroadcastss zmm3,DWORD PTR [rcx+BroadcastOffset] + vfmadd231ps zmm4,zmm3,ZMMWORD PTR [rdx+VectorOffset] + vfmadd231ps zmm5,zmm3,ZMMWORD PTR [rdx+r12+VectorOffset] +ELSE + vmovaps zmm0,ZMMWORD PTR [rdx+VectorOffset] + vmovaps zmm1,ZMMWORD PTR [rdx+r12+VectorOffset] + vbroadcastss zmm3,DWORD PTR [rcx+BroadcastOffset] + vfmadd231ps zmm4,zmm3,zmm0 + vfmadd231ps zmm5,zmm3,zmm1 +IF Count GE 3 + vbroadcastss zmm3,DWORD PTR [rcx+r10+BroadcastOffset] + vfmadd231ps zmm6,zmm3,zmm0 + vfmadd231ps zmm7,zmm3,zmm1 + vbroadcastss zmm3,DWORD PTR [rcx+r10*2+BroadcastOffset] + vfmadd231ps zmm8,zmm3,zmm0 + vfmadd231ps zmm9,zmm3,zmm1 +ENDIF +IF Count GE 6 + vbroadcastss zmm3,DWORD PTR [rbx+BroadcastOffset] + vfmadd231ps zmm10,zmm3,zmm0 + vfmadd231ps zmm11,zmm3,zmm1 + vbroadcastss zmm3,DWORD PTR [rbx+r10+BroadcastOffset] + vfmadd231ps zmm12,zmm3,zmm0 + vfmadd231ps zmm13,zmm3,zmm1 + vbroadcastss zmm3,DWORD PTR [rbx+r10*2+BroadcastOffset] + vfmadd231ps zmm14,zmm3,zmm0 + vfmadd231ps zmm15,zmm3,zmm1 +ENDIF +IF Count GE 12 + vbroadcastss zmm3,DWORD PTR [r13+BroadcastOffset] + vfmadd231ps zmm16,zmm3,zmm0 + vfmadd231ps zmm17,zmm3,zmm1 + vbroadcastss zmm3,DWORD PTR [r13+r10+BroadcastOffset] + vfmadd231ps zmm18,zmm3,zmm0 + vfmadd231ps zmm19,zmm3,zmm1 + vbroadcastss zmm3,DWORD PTR [r13+r10*2+BroadcastOffset] + vfmadd231ps zmm20,zmm3,zmm0 + vfmadd231ps zmm21,zmm3,zmm1 + vbroadcastss zmm3,DWORD PTR [r14+BroadcastOffset] + vfmadd231ps zmm22,zmm3,zmm0 + vfmadd231ps zmm23,zmm3,zmm1 + vbroadcastss zmm3,DWORD PTR [r14+r10+BroadcastOffset] + vfmadd231ps zmm24,zmm3,zmm0 + vfmadd231ps zmm25,zmm3,zmm1 + vbroadcastss zmm3,DWORD PTR [r14+r10*2+BroadcastOffset] + vfmadd231ps zmm26,zmm3,zmm0 + vfmadd231ps zmm27,zmm3,zmm1 +ENDIF +ENDIF + + ENDM + +; +; ComputeBlockAvx512FBy16 +; +; This macro multiplies and accumulates for a 16xN block (where N is 1,3,6,12) +; of the output matrix. +; +; Arguments: +; +; Count - Supplies the number of rows to access from matrix A. +; +; VectorOffset - Supplies the byte offset from matrix B to fetch elements. +; +; BroadcastOffset - Supplies the byte offset from matrix A to fetch elements. +; +; Implicit Arguments: +; +; rbx - Supplies the address into the matrix A data plus 3 rows. +; +; rcx - Supplies the address into the matrix A data. +; +; rdx - Supplies the address into the matrix B data. +; +; r10 - Supplies the length in bytes of a row from matrix A. +; +; r13 - Supplies the address into the matrix A data plus 6 rows. +; +; r14 - Supplies the address into the matrix A data plus 9 rows. +; +; zmm4-zmm27 - Supplies the block accumulators. +; + +ComputeBlockAvx512FBy16 MACRO Count, VectorOffset, BroadcastOffset + + vmovaps zmm0,ZMMWORD PTR [rdx+VectorOffset] + vfmadd231ps zmm5,zmm0,DWORD BCST [rcx+BroadcastOffset] +IF Count GE 3 + vfmadd231ps zmm7,zmm0,DWORD BCST [rcx+r10+BroadcastOffset] + vfmadd231ps zmm9,zmm0,DWORD BCST [rcx+r10*2+BroadcastOffset] +ENDIF +IF Count GE 6 + vfmadd231ps zmm11,zmm0,DWORD BCST [rbx+BroadcastOffset] + vfmadd231ps zmm13,zmm0,DWORD BCST [rbx+r10+BroadcastOffset] + vfmadd231ps zmm15,zmm0,DWORD BCST [rbx+r10*2+BroadcastOffset] +ENDIF +IF Count GE 12 + vfmadd231ps zmm17,zmm0,DWORD BCST [r13+BroadcastOffset] + vfmadd231ps zmm19,zmm0,DWORD BCST [r13+r10+BroadcastOffset] + vfmadd231ps zmm21,zmm0,DWORD BCST [r13+r10*2+BroadcastOffset] + vfmadd231ps zmm23,zmm0,DWORD BCST [r14+BroadcastOffset] + vfmadd231ps zmm25,zmm0,DWORD BCST [r14+r10+BroadcastOffset] + vfmadd231ps zmm27,zmm0,DWORD BCST [r14+r10*2+BroadcastOffset] +ENDIF + + ENDM + +;++ +; +; Routine Description: +; +; This routine is an inner kernel to compute matrix multiplication for a +; set of rows. +; +; Arguments: +; +; A (rcx) - Supplies the address of matrix A. +; +; B (rdx) - Supplies the address of matrix B. The matrix data has been packed +; using MlasSgemmCopyPackB or MlasSgemmTransposePackB. +; +; C (r8) - Supplies the address of matrix C. +; +; CountK (r9) - Supplies the number of columns from matrix A and the number +; of rows from matrix B to iterate over. +; +; CountM - Supplies the maximum number of rows that can be processed for +; matrix A and matrix C. The actual number of rows handled for this +; invocation depends on the kernel implementation. +; +; CountN - Supplies the number of columns from matrix B and matrix C to iterate +; over. +; +; lda - Supplies the first dimension of matrix A. +; +; ldc - Supplies the first dimension of matrix C. +; +; Alpha - Supplies the scaler multiplier (see SGEMM definition). +; +; Return Value: +; +; Returns the number of rows handled. +; +;-- + +SgemmKernelAvx512FFunction MACRO Mode + + NESTED_ENTRY MlasSgemmKernel&Mode&Avx512F, _TEXT + + SgemmKernelAvxEntry SaveExtra + + mov r12,r9 + shl r12,6 ; compute 16*CountK*sizeof(float) + mov edi,-1 + kmovw k1,edi ; update mask to write all columns + vbroadcastss zmm31,DWORD PTR SgemmKernelFrame.Alpha[rsp] + +; +; Process 12 rows of the matrices. +; + + cmp r11,12 + jb ProcessCountMLessThan12 + mov r11d,12 ; return 12 rows handled + cmp rbp,16 + jbe ProcessRemainingCountN12 + +ProcessNextColumnLoop32x12: + vmovaps zmm16,zmm4 ; clear upper block accumulators + vmovaps zmm17,zmm5 + vmovaps zmm18,zmm4 + vmovaps zmm19,zmm5 + vmovaps zmm20,zmm4 + vmovaps zmm21,zmm5 + vmovaps zmm22,zmm4 + vmovaps zmm23,zmm5 + vmovaps zmm24,zmm4 + vmovaps zmm25,zmm5 + vmovaps zmm26,zmm4 + vmovaps zmm27,zmm5 + lea rbx,[r10*2+r10] + lea r13,[rcx+rbx*2] ; compute matrix A plus 6 rows + lea r14,[r13+rbx] ; compute matrix A plus 9 rows + add rbx,rcx ; compute matrix A plus 3 rows + ComputeBlockAvx512FLoop ComputeBlockAvx512FBy32, 12 + add rdx,r12 ; advance matrix B by 16*CountK floats + lea rbx,[rax*2+rax] + lea r13,[r8+rbx*2] ; compute matrix C plus 6 rows + lea r14,[r13+rbx] ; compute matrix C plus 9 rows + add rbx,r8 ; compute matrix C plus 3 rows +IFDIFI , + vmulps zmm4,zmm4,zmm31 ; multiply by alpha + vmulps zmm6,zmm6,zmm31 + vmulps zmm8,zmm8,zmm31 + vmulps zmm10,zmm10,zmm31 + vmulps zmm12,zmm12,zmm31 + vmulps zmm14,zmm14,zmm31 + vmulps zmm16,zmm16,zmm31 + vmulps zmm18,zmm18,zmm31 + vmulps zmm20,zmm20,zmm31 + vmulps zmm22,zmm22,zmm31 + vmulps zmm24,zmm24,zmm31 + vmulps zmm26,zmm26,zmm31 +ELSE + vfmadd213ps zmm4,zmm31,ZMMWORD PTR [r8] + vfmadd213ps zmm6,zmm31,ZMMWORD PTR [r8+rax] + vfmadd213ps zmm8,zmm31,ZMMWORD PTR [r8+rax*2] + vfmadd213ps zmm10,zmm31,ZMMWORD PTR [rbx] + vfmadd213ps zmm12,zmm31,ZMMWORD PTR [rbx+rax] + vfmadd213ps zmm14,zmm31,ZMMWORD PTR [rbx+rax*2] + vfmadd213ps zmm16,zmm31,ZMMWORD PTR [r13] + vfmadd213ps zmm18,zmm31,ZMMWORD PTR [r13+rax] + vfmadd213ps zmm20,zmm31,ZMMWORD PTR [r13+rax*2] + vfmadd213ps zmm22,zmm31,ZMMWORD PTR [r14] + vfmadd213ps zmm24,zmm31,ZMMWORD PTR [r14+rax] + vfmadd213ps zmm26,zmm31,ZMMWORD PTR [r14+rax*2] +ENDIF + vmovups ZMMWORD PTR [r8],zmm4 + vmovups ZMMWORD PTR [r8+rax],zmm6 + vmovups ZMMWORD PTR [r8+rax*2],zmm8 + vmovups ZMMWORD PTR [rbx],zmm10 + vmovups ZMMWORD PTR [rbx+rax],zmm12 + vmovups ZMMWORD PTR [rbx+rax*2],zmm14 + vmovups ZMMWORD PTR [r13],zmm16 + vmovups ZMMWORD PTR [r13+rax],zmm18 + vmovups ZMMWORD PTR [r13+rax*2],zmm20 + vmovups ZMMWORD PTR [r14],zmm22 + vmovups ZMMWORD PTR [r14+rax],zmm24 + vmovups ZMMWORD PTR [r14+rax*2],zmm26 + add r8,16*4 ; advance matrix C by 16 columns + sub rbp,16 + +Output16x12Block: + sub rbp,16 + jae Output16x12BlockWithMask + lea ecx,[ebp+16] ; correct for over-subtract above + mov edi,1 + shl edi,cl + dec edi + kmovw k1,edi ; update mask for remaining columns + xor ebp,ebp ; no more columns remaining + +Output16x12BlockWithMask: + lea rbx,[rax*2+rax] + lea r13,[r8+rbx*2] ; compute matrix C plus 6 rows + lea r14,[r13+rbx] ; compute matrix C plus 9 rows + add rbx,r8 ; compute matrix C plus 3 rows +IFDIFI , + vmulps zmm5,zmm5,zmm31 ; multiply by alpha + vmulps zmm7,zmm7,zmm31 + vmulps zmm9,zmm9,zmm31 + vmulps zmm11,zmm11,zmm31 + vmulps zmm13,zmm13,zmm31 + vmulps zmm15,zmm15,zmm31 + vmulps zmm17,zmm17,zmm31 + vmulps zmm19,zmm19,zmm31 + vmulps zmm21,zmm21,zmm31 + vmulps zmm23,zmm23,zmm31 + vmulps zmm25,zmm25,zmm31 + vmulps zmm27,zmm27,zmm31 +ELSE + vfmadd213ps zmm5{k1},zmm31,ZMMWORD PTR [r8] + vfmadd213ps zmm7{k1},zmm31,ZMMWORD PTR [r8+rax] + vfmadd213ps zmm9{k1},zmm31,ZMMWORD PTR [r8+rax*2] + vfmadd213ps zmm11{k1},zmm31,ZMMWORD PTR [rbx] + vfmadd213ps zmm13{k1},zmm31,ZMMWORD PTR [rbx+rax] + vfmadd213ps zmm15{k1},zmm31,ZMMWORD PTR [rbx+rax*2] + vfmadd213ps zmm17{k1},zmm31,ZMMWORD PTR [r13] + vfmadd213ps zmm19{k1},zmm31,ZMMWORD PTR [r13+rax] + vfmadd213ps zmm21{k1},zmm31,ZMMWORD PTR [r13+rax*2] + vfmadd213ps zmm23{k1},zmm31,ZMMWORD PTR [r14] + vfmadd213ps zmm25{k1},zmm31,ZMMWORD PTR [r14+rax] + vfmadd213ps zmm27{k1},zmm31,ZMMWORD PTR [r14+rax*2] +ENDIF + vmovups ZMMWORD PTR [r8]{k1},zmm5 + vmovups ZMMWORD PTR [r8+rax]{k1},zmm7 + vmovups ZMMWORD PTR [r8+rax*2]{k1},zmm9 + vmovups ZMMWORD PTR [rbx]{k1},zmm11 + vmovups ZMMWORD PTR [rbx+rax]{k1},zmm13 + vmovups ZMMWORD PTR [rbx+rax*2]{k1},zmm15 + vmovups ZMMWORD PTR [r13]{k1},zmm17 + vmovups ZMMWORD PTR [r13+rax]{k1},zmm19 + vmovups ZMMWORD PTR [r13+rax*2]{k1},zmm21 + vmovups ZMMWORD PTR [r14]{k1},zmm23 + vmovups ZMMWORD PTR [r14+rax]{k1},zmm25 + vmovups ZMMWORD PTR [r14+rax*2]{k1},zmm27 + add r8,16*4 ; advance matrix C by 16 columns + mov rcx,rsi ; reload matrix A + vzeroall + cmp rbp,16 + ja ProcessNextColumnLoop32x12 + test rbp,rbp + jz ExitKernel + +ProcessRemainingCountN12: + vmovaps zmm17,zmm5 ; clear upper block accumulators + vmovaps zmm19,zmm5 + vmovaps zmm21,zmm5 + vmovaps zmm23,zmm5 + vmovaps zmm25,zmm5 + vmovaps zmm27,zmm5 + lea rbx,[r10*2+r10] + lea r13,[rcx+rbx*2] ; compute matrix A plus 6 rows + lea r14,[r13+rbx] ; compute matrix A plus 9 rows + add rbx,rcx ; compute matrix A plus 3 rows + ComputeBlockAvx512FLoop ComputeBlockAvx512FBy16, 12 + jmp Output16x12Block + +; +; Restore non-volatile registers and return. +; + +ExitKernel: + SgemmKernelAvxExit RestoreExtra + +; +; Process 6 rows of the matrices. +; + +ProcessCountMLessThan12: + cmp r11,6 + jb ProcessCountMLessThan6 + mov r11d,6 ; return 6 rows handled + cmp rbp,16 + jbe ProcessRemainingCountN6 + +ProcessNextColumnLoop32x6: + lea rbx,[r10*2+r10] + add rbx,rcx ; compute matrix A plus 3 rows + ComputeBlockAvx512FLoop ComputeBlockAvx512FBy32, 6 + add rdx,r12 ; advance matrix B by 16*CountK floats + lea rdi,[r8+rax*2] ; compute matrix C plus 2 rows + lea rbx,[r8+rax*4] ; compute matrix C plus 4 rows +IFDIFI , + vmulps zmm4,zmm4,zmm31 ; multiply by alpha + vmulps zmm6,zmm6,zmm31 + vmulps zmm8,zmm8,zmm31 + vmulps zmm10,zmm10,zmm31 + vmulps zmm12,zmm12,zmm31 + vmulps zmm14,zmm14,zmm31 +ELSE + vfmadd213ps zmm4,zmm31,ZMMWORD PTR [r8] + vfmadd213ps zmm6,zmm31,ZMMWORD PTR [r8+rax] + vfmadd213ps zmm8,zmm31,ZMMWORD PTR [rdi] + vfmadd213ps zmm10,zmm31,ZMMWORD PTR [rdi+rax] + vfmadd213ps zmm12,zmm31,ZMMWORD PTR [rbx] + vfmadd213ps zmm14,zmm31,ZMMWORD PTR [rbx+rax] +ENDIF + vmovups ZMMWORD PTR [r8],zmm4 + vmovups ZMMWORD PTR [r8+rax],zmm6 + vmovups ZMMWORD PTR [rdi],zmm8 + vmovups ZMMWORD PTR [rdi+rax],zmm10 + vmovups ZMMWORD PTR [rbx],zmm12 + vmovups ZMMWORD PTR [rbx+rax],zmm14 + add r8,16*4 ; advance matrix C by 16 columns + sub rbp,16 + +Output16x6Block: + sub rbp,16 + jae Output16x6BlockWithMask + lea ecx,[ebp+16] ; correct for over-subtract above + mov edi,1 + shl edi,cl + dec edi + kmovw k1,edi ; update mask for remaining columns + xor ebp,ebp ; no more columns remaining + +Output16x6BlockWithMask: + lea rdi,[r8+rax*2] ; compute matrix C plus 2 rows + lea rbx,[r8+rax*4] ; compute matrix C plus 4 rows +IFDIFI , + vmulps zmm5,zmm5,zmm31 ; multiply by alpha + vmulps zmm7,zmm7,zmm31 + vmulps zmm9,zmm9,zmm31 + vmulps zmm11,zmm11,zmm31 + vmulps zmm13,zmm13,zmm31 + vmulps zmm15,zmm15,zmm31 +ELSE + vfmadd213ps zmm5{k1},zmm31,ZMMWORD PTR [r8] + vfmadd213ps zmm7{k1},zmm31,ZMMWORD PTR [r8+rax] + vfmadd213ps zmm9{k1},zmm31,ZMMWORD PTR [rdi] + vfmadd213ps zmm11{k1},zmm31,ZMMWORD PTR [rdi+rax] + vfmadd213ps zmm13{k1},zmm31,ZMMWORD PTR [rbx] + vfmadd213ps zmm15{k1},zmm31,ZMMWORD PTR [rbx+rax] +ENDIF + vmovups ZMMWORD PTR [r8]{k1},zmm5 + vmovups ZMMWORD PTR [r8+rax]{k1},zmm7 + vmovups ZMMWORD PTR [rdi]{k1},zmm9 + vmovups ZMMWORD PTR [rdi+rax]{k1},zmm11 + vmovups ZMMWORD PTR [rbx]{k1},zmm13 + vmovups ZMMWORD PTR [rbx+rax]{k1},zmm15 + add r8,16*4 ; advance matrix C by 16 columns + mov rcx,rsi ; reload matrix A + vzeroall + cmp rbp,16 + ja ProcessNextColumnLoop32x6 + test rbp,rbp + jz ExitKernel + +ProcessRemainingCountN6: + lea rbx,[r10*2+r10] + add rbx,rcx ; compute matrix A plus 3 rows + ComputeBlockAvx512FLoop ComputeBlockAvx512FBy16, 6 + jmp Output16x6Block + +; +; Process 3 rows of the matrices. +; + +ProcessCountMLessThan6: + cmp r11,3 + jb ProcessCountMLessThan3 + mov r11d,3 ; return 3 rows handled + cmp rbp,16 + jbe ProcessRemainingCountN3 + +ProcessNextColumnLoop32x3: + ComputeBlockAvx512FLoop ComputeBlockAvx512FBy32, 3 + add rdx,r12 ; advance matrix B by 16*CountK floats +IFDIFI , + vmulps zmm4,zmm4,zmm31 ; multiply by alpha + vmulps zmm6,zmm6,zmm31 + vmulps zmm8,zmm8,zmm31 +ELSE + vfmadd213ps zmm4,zmm31,ZMMWORD PTR [r8] + vfmadd213ps zmm6,zmm31,ZMMWORD PTR [r8+rax] + vfmadd213ps zmm8,zmm31,ZMMWORD PTR [r8+rax*2] +ENDIF + vmovups ZMMWORD PTR [r8],zmm4 + vmovups ZMMWORD PTR [r8+rax],zmm6 + vmovups ZMMWORD PTR [r8+rax*2],zmm8 + add r8,16*4 ; advance matrix C by 16 columns + sub rbp,16 + +Output16x3Block: + sub rbp,16 + jae Output16x3BlockWithMask + lea ecx,[ebp+16] ; correct for over-subtract above + mov edi,1 + shl edi,cl + dec edi + kmovw k1,edi ; update mask for remaining columns + xor ebp,ebp ; no more columns remaining + +Output16x3BlockWithMask: +IFDIFI , + vmulps zmm5,zmm5,zmm31 ; multiply by alpha + vmulps zmm7,zmm7,zmm31 + vmulps zmm9,zmm9,zmm31 +ELSE + vfmadd213ps zmm5{k1},zmm31,ZMMWORD PTR [r8] + vfmadd213ps zmm7{k1},zmm31,ZMMWORD PTR [r8+rax] + vfmadd213ps zmm9{k1},zmm31,ZMMWORD PTR [r8+rax*2] +ENDIF + vmovups ZMMWORD PTR [r8]{k1},zmm5 + vmovups ZMMWORD PTR [r8+rax]{k1},zmm7 + vmovups ZMMWORD PTR [r8+rax*2]{k1},zmm9 + add r8,16*4 ; advance matrix C by 16 columns + mov rcx,rsi ; reload matrix A + vzeroall + cmp rbp,16 + ja ProcessNextColumnLoop32x3 + test rbp,rbp + jz ExitKernel + +ProcessRemainingCountN3: + ComputeBlockAvx512FLoop ComputeBlockAvx512FBy16, 3 + jmp Output16x3Block + +; +; Process 1 row of the matrices. +; + +ProcessCountMLessThan3: + mov r11d,1 ; return 1 row handled + cmp rbp,16 + jbe ProcessRemainingCountN1 + +ProcessNextColumnLoop32x1: + ComputeBlockAvx512FLoop ComputeBlockAvx512FBy32, 1 + add rdx,r12 ; advance matrix B by 16*CountK floats +IFDIFI , + vmulps zmm4,zmm4,zmm31 ; multiply by alpha +ELSE + vfmadd213ps zmm4,zmm31,ZMMWORD PTR [r8] +ENDIF + vmovups ZMMWORD PTR [r8],zmm4 + add r8,16*4 ; advance matrix C by 16 columns + sub rbp,16 + +Output16x1Block: + sub rbp,16 + jae Output16x1BlockWithMask + lea ecx,[ebp+16] ; correct for over-subtract above + mov edi,1 + shl edi,cl + dec edi + kmovw k1,edi ; update mask for remaining columns + xor ebp,ebp ; no more columns remaining + +Output16x1BlockWithMask: +IFDIFI , + vmulps zmm5,zmm5,zmm31 ; multiply by alpha +ELSE + vfmadd213ps zmm5{k1},zmm31,ZMMWORD PTR [r8] +ENDIF + vmovups ZMMWORD PTR [r8]{k1},zmm5 + add r8,16*4 ; advance matrix C by 16 columns + mov rcx,rsi ; reload matrix A + vzeroall + cmp rbp,16 + ja ProcessNextColumnLoop32x1 + test rbp,rbp + jz ExitKernel + +ProcessRemainingCountN1: + ComputeBlockAvx512FLoop ComputeBlockAvx512FBy16, 1 + jmp Output16x1Block + + NESTED_END MlasSgemmKernel&Mode&Avx512F, _TEXT + + ENDM + +SgemmKernelAvx512FFunction Zero +SgemmKernelAvx512FFunction Add + + END diff --git a/onnxruntime/core/mlas/lib/amd64/SgemmKernelCommon.inc b/onnxruntime/core/mlas/lib/amd64/SgemmKernelCommon.inc new file mode 100644 index 0000000000000..86a7123b922f8 --- /dev/null +++ b/onnxruntime/core/mlas/lib/amd64/SgemmKernelCommon.inc @@ -0,0 +1,316 @@ +;++ +; +; Copyright (c) Microsoft Corporation. All rights reserved. +; +; Licensed under the MIT License. +; +; Module Name: +; +; SgemmKernelCommon.inc +; +; Abstract: +; +; This module contains common kernel macros and structures for the single +; precision matrix/matrix multiply operation (SGEMM). +; +;-- + +; +; Stack frame layout for the SGEMM kernels. +; + +SgemmKernelFrame STRUCT + + SavedXmm6 OWORD ? + SavedXmm7 OWORD ? + SavedXmm8 OWORD ? + SavedXmm9 OWORD ? + SavedXmm10 OWORD ? + SavedXmm11 OWORD ? + SavedXmm12 OWORD ? + SavedXmm13 OWORD ? + SavedXmm14 OWORD ? + SavedXmm15 OWORD ? + SavedR12 QWORD ? + SavedR13 QWORD ? + SavedR14 QWORD ? + SavedRdi QWORD ? + SavedRsi QWORD ? + SavedRbx QWORD ? + SavedRbp QWORD ? + ReturnAddress QWORD ? + PreviousP1Home QWORD ? + PreviousP2Home QWORD ? + PreviousP3Home QWORD ? + PreviousP4Home QWORD ? + CountM QWORD ? + CountN QWORD ? + lda QWORD ? + ldc QWORD ? + Alpha QWORD ? + +SgemmKernelFrame ENDS + +; +; Stack frame layout for the SGEMM M=1 kernels. +; + +SgemmKernelM1Frame STRUCT + + SavedXmm6 OWORD ? + SavedXmm7 OWORD ? + SavedXmm8 OWORD ? + SavedRsi QWORD ? + SavedRbx QWORD ? + SavedRbp QWORD ? + ReturnAddress QWORD ? + PreviousP1Home QWORD ? + PreviousP2Home QWORD ? + PreviousP3Home QWORD ? + PreviousP4Home QWORD ? + CountN QWORD ? + ldb QWORD ? + Beta QWORD ? + +SgemmKernelM1Frame ENDS + +; +; save_xmm128_avx +; +; Macro Description: +; +; This macro uses AVX instructions to save a vector register as part of a +; function prologue as an alternative to save_xmm128. +; +; Arguments: +; +; Register - Supplies the vector register to be saved. +; +; Offset - Supplies the stack frame offset to store the contents of the +; vector register. +; + +save_xmm128_avx MACRO Register, Offset + + vmovaps Offset[rsp], Register + .savexmm128 Register, Offset + + ENDM + +; +; SgemmKernelAvxEntry +; +; Macro Description: +; +; This macro implements the common prologue code for the AVX and FMA3 +; SGEMM kernels. +; +; Arguments: +; +; SaveExtra - Supplies a non-blank value if registers r12-r14 should also be +; saved to make available as temporaries. +; +; Return Registers: +; +; rax - Stores the length in bytes of a row from matrix C. +; +; rsi - Stores the address of the matrix A data. +; +; rbp - Stores the CountN argument from the stack frame. +; +; r10 - Stores the length in bytes of a row from matrix A. +; +; r11 - Stores the CountM argument from the stack frame. +; +; rbx, rsi, rdi - Previous values stored on the stack and the registers +; are available as temporaries. +; + +SgemmKernelAvxEntry MACRO SaveExtra + + rex_push_reg rbp + push_reg rbx + push_reg rsi + push_reg rdi + alloc_stack (SgemmKernelFrame.SavedRdi) +IFNB + save_reg r12, SgemmKernelFrame.SavedR12 + save_reg r13, SgemmKernelFrame.SavedR13 + save_reg r14, SgemmKernelFrame.SavedR14 +ENDIF + save_xmm128_avx xmm6,SgemmKernelFrame.SavedXmm6 + save_xmm128_avx xmm7,SgemmKernelFrame.SavedXmm7 + save_xmm128_avx xmm8,SgemmKernelFrame.SavedXmm8 + save_xmm128_avx xmm9,SgemmKernelFrame.SavedXmm9 + save_xmm128_avx xmm10,SgemmKernelFrame.SavedXmm10 + save_xmm128_avx xmm11,SgemmKernelFrame.SavedXmm11 + save_xmm128_avx xmm12,SgemmKernelFrame.SavedXmm12 + save_xmm128_avx xmm13,SgemmKernelFrame.SavedXmm13 + save_xmm128_avx xmm14,SgemmKernelFrame.SavedXmm14 + save_xmm128_avx xmm15,SgemmKernelFrame.SavedXmm15 + + END_PROLOGUE + + vzeroall + + mov rsi,rcx + mov rbp,SgemmKernelFrame.CountN[rsp] + mov rax,SgemmKernelFrame.ldc[rsp] + shl rax,2 + mov r10,SgemmKernelFrame.lda[rsp] + shl r10,2 + mov r11,SgemmKernelFrame.CountM[rsp] + + ENDM + +; +; SgemmKernelAvxExit +; +; Macro Description: +; +; This macro implements the common epilogue code for the AVX and FMA3 +; SGEMM kernels. +; +; Arguments: +; +; RestoreExtra - Supplies a non-blank value if registers r12-r14 should also +; be restored. +; +; Implicit Arguments: +; +; r11d - Stores the number of rows handled. +; + +SgemmKernelAvxExit MACRO RestoreExtra + + mov eax,r11d + vmovaps xmm6,SgemmKernelFrame.SavedXmm6[rsp] + vmovaps xmm7,SgemmKernelFrame.SavedXmm7[rsp] + vmovaps xmm8,SgemmKernelFrame.SavedXmm8[rsp] + vmovaps xmm9,SgemmKernelFrame.SavedXmm9[rsp] + vmovaps xmm10,SgemmKernelFrame.SavedXmm10[rsp] + vmovaps xmm11,SgemmKernelFrame.SavedXmm11[rsp] + vmovaps xmm12,SgemmKernelFrame.SavedXmm12[rsp] + vmovaps xmm13,SgemmKernelFrame.SavedXmm13[rsp] + vmovaps xmm14,SgemmKernelFrame.SavedXmm14[rsp] + vmovaps xmm15,SgemmKernelFrame.SavedXmm15[rsp] +IFNB + mov r12,SgemmKernelFrame.SavedR12[rsp] + mov r13,SgemmKernelFrame.SavedR13[rsp] + mov r14,SgemmKernelFrame.SavedR14[rsp] +ENDIF + add rsp,(SgemmKernelFrame.SavedRdi) + + BEGIN_EPILOGUE + + pop rdi + pop rsi + pop rbx + pop rbp + ret + + ENDM + +; +; ComputeBlockCommonLoop +; +; This macro generates code to execute the block compute macro multiple +; times and advancing the matrix A and matrix B data pointers. +; +; Arguments: +; +; ComputeBlock - Supplies the macro to compute a single block. +; +; Count - Supplies the number of rows to access from matrix A. +; +; AdvanceMatrixAPlusRows - Supplies a non-zero value if the data pointer +; in rbx should also be advanced as part of the loop. +; +; Implicit Arguments: +; +; rbx - Supplies the address into the matrix A data plus N rows. +; +; rcx - Supplies the address into the matrix A data. +; +; rdx - Supplies the address into the matrix B data. +; +; r9 - Supplies the number of columns from matrix A and the number of rows +; from matrix B to iterate over. +; +; ymm4-ymm15 - Supplies the block accumulators. +; + +ComputeBlockCommonLoop MACRO ComputeBlock, Count, AdvanceMatrixAPlusRows + + LOCAL ComputeBlockBy4Loop + LOCAL ProcessRemainingBlocks + LOCAL ComputeBlockBy1Loop + LOCAL OutputBlock + + mov rdi,r9 ; reload CountK + sub rdi,4 + jb ProcessRemainingBlocks + +ComputeBlockBy4Loop: + ComputeBlock Count, 0, 0 + ComputeBlock Count, 16*4, 4 + sub rdx,-32*4 ; advance matrix B by 32 columns + ComputeBlock Count, 0, 8 + ComputeBlock Count, 16*4, 12 + sub rdx,-32*4 ; advance matrix B by 32 columns + add rcx,4*4 ; advance matrix A by 4 columns +IF AdvanceMatrixAPlusRows + add rbx,4*4 ; advance matrix A plus rows by 4 columns +IF Count GE 12 + add r13,4*4 + add r14,4*4 +ENDIF +ENDIF + sub rdi,4 + jae ComputeBlockBy4Loop + +ProcessRemainingBlocks: + add rdi,4 ; correct for over-subtract above + jz OutputBlock + +ComputeBlockBy1Loop: + ComputeBlock Count, 0, 0 + add rdx,16*4 ; advance matrix B by 16 columns + add rcx,4 ; advance matrix A by 1 column +IF AdvanceMatrixAPlusRows + add rbx,4 ; advance matrix A plus rows by 1 column +IF Count GE 12 + add r13,4 + add r14,4 +ENDIF +ENDIF + dec rdi + jne ComputeBlockBy1Loop + +OutputBlock: + + ENDM + +ComputeBlockAvxLoop MACRO ComputeBlock, Count + + ComputeBlockCommonLoop ComputeBlock, Count, + + ENDM + +ComputeBlockFma3Loop MACRO ComputeBlock, Count + +; +; Reload the alpha value which is lost after each vzeroall instruction. +; + + vbroadcastss ymm2,DWORD PTR SgemmKernelFrame.Alpha[rsp] + + ComputeBlockCommonLoop ComputeBlock, Count, + + ENDM + +ComputeBlockAvx512FLoop MACRO ComputeBlock, Count + + ComputeBlockCommonLoop ComputeBlock, Count, + + ENDM diff --git a/onnxruntime/core/mlas/lib/amd64/SgemmKernelFma3.asm b/onnxruntime/core/mlas/lib/amd64/SgemmKernelFma3.asm new file mode 100644 index 0000000000000..895709ed3134b --- /dev/null +++ b/onnxruntime/core/mlas/lib/amd64/SgemmKernelFma3.asm @@ -0,0 +1,575 @@ +;++ +; +; Copyright (c) Microsoft Corporation. All rights reserved. +; +; Licensed under the MIT License. +; +; Module Name: +; +; SgemmKernelFma3.asm +; +; Abstract: +; +; This module implements the kernels for the single precision matrix/matrix +; multiply operation (SGEMM). +; +; This implementation uses AVX fused multiply/add instructions. +; +;-- + + .xlist +INCLUDE macamd64.inc +INCLUDE SgemmKernelCommon.inc + .list + + EXTERN MlasMaskMoveAvx:NEAR + +; +; ComputeBlockFma3By32 +; +; This macro multiplies and accumulates for a 32xN block (where N is 1,3) +; of the output matrix. +; +; Arguments: +; +; Count - Supplies the number of rows to access from matrix A. +; +; VectorOffset - Supplies the byte offset from matrix B to fetch elements. +; +; BroadcastOffset - Supplies the byte offset from matrix A to fetch elements. +; +; Implicit Arguments: +; +; rcx - Supplies the address into the matrix A data. +; +; rdx - Supplies the address into the matrix B data. +; +; r10 - Supplies the length in bytes of a row from matrix A. +; +; ymm4-ymm15 - Supplies the block accumulators. +; + +ComputeBlockFma3By32 MACRO Count, VectorOffset, BroadcastOffset + +IF Count EQ 1 + vbroadcastss ymm3,DWORD PTR [rcx+BroadcastOffset] + vfmadd231ps ymm4,ymm3,YMMWORD PTR [rdx+VectorOffset] + vfmadd231ps ymm5,ymm3,YMMWORD PTR [rdx+VectorOffset+32] + vfmadd231ps ymm6,ymm3,YMMWORD PTR [rdx+rbx+VectorOffset] + vfmadd231ps ymm7,ymm3,YMMWORD PTR [rdx+rbx+VectorOffset+32] +ENDIF + + ENDM + +; +; ComputeBlockFma3By16 +; +; This macro multiplies and accumulates for a 16xN block (where N is 1,3,6) +; of the output matrix. +; +; Arguments: +; +; Count - Supplies the number of rows to access from matrix A. +; +; VectorOffset - Supplies the byte offset from matrix B to fetch elements. +; +; BroadcastOffset - Supplies the byte offset from matrix A to fetch elements. +; +; Implicit Arguments: +; +; rbx - Supplies the address into the matrix A data plus 3 rows. +; +; rcx - Supplies the address into the matrix A data. +; +; rdx - Supplies the address into the matrix B data. +; +; r10 - Supplies the length in bytes of a row from matrix A. +; +; ymm4-ymm15 - Supplies the block accumulators. +; + +ComputeBlockFma3By16 MACRO Count, VectorOffset, BroadcastOffset + +IF Count EQ 1 + vbroadcastss ymm3,DWORD PTR [rcx+BroadcastOffset] + vfmadd231ps ymm4,ymm3,YMMWORD PTR [rdx+VectorOffset] + vfmadd231ps ymm5,ymm3,YMMWORD PTR [rdx+VectorOffset+32] +ELSE + vmovaps ymm0,YMMWORD PTR [rdx+VectorOffset] + vmovaps ymm1,YMMWORD PTR [rdx+VectorOffset+32] + vbroadcastss ymm3,DWORD PTR [rcx+BroadcastOffset] + vfmadd231ps ymm4,ymm3,ymm0 + vfmadd231ps ymm5,ymm3,ymm1 +IF Count GE 3 + vbroadcastss ymm3,DWORD PTR [rcx+r10+BroadcastOffset] + vfmadd231ps ymm6,ymm3,ymm0 + vfmadd231ps ymm7,ymm3,ymm1 + vbroadcastss ymm3,DWORD PTR [rcx+r10*2+BroadcastOffset] + vfmadd231ps ymm8,ymm3,ymm0 + vfmadd231ps ymm9,ymm3,ymm1 +ENDIF +IF Count GE 6 + vbroadcastss ymm3,DWORD PTR [rbx+BroadcastOffset] + vfmadd231ps ymm10,ymm3,ymm0 + vfmadd231ps ymm11,ymm3,ymm1 + vbroadcastss ymm3,DWORD PTR [rbx+r10+BroadcastOffset] + vfmadd231ps ymm12,ymm3,ymm0 + vfmadd231ps ymm13,ymm3,ymm1 + vbroadcastss ymm3,DWORD PTR [rbx+r10*2+BroadcastOffset] + vfmadd231ps ymm14,ymm3,ymm0 + vfmadd231ps ymm15,ymm3,ymm1 +ENDIF +ENDIF + + ENDM + +; +; ComputeBlockFma3By8 +; +; Macro Description: +; +; This macro multiplies and accumulates for a 8xN block (where N is 1,3,6) +; of the output matrix. +; +; Arguments: +; +; Count - Supplies the number of rows to access from matrix A. +; +; VectorOffset - Supplies the byte offset from matrix B to fetch elements. +; +; BroadcastOffset - Supplies the byte offset from matrix A to fetch elements. +; +; Implicit Arguments: +; +; rbx - Supplies the address into the matrix A data plus 3 rows. +; +; rcx - Supplies the address into the matrix A data. +; +; rdx - Supplies the address into the matrix B data. +; +; r10 - Supplies the length in bytes of a row from matrix A. +; +; ymm4-ymm15 - Supplies the block accumulators. +; + +ComputeBlockFma3By8 MACRO Count, VectorOffset, BroadcastOffset + +IF Count EQ 1 + vbroadcastss ymm3,DWORD PTR [rcx+BroadcastOffset] + vfmadd231ps ymm5,ymm3,YMMWORD PTR [rdx+VectorOffset] +ELSE + vmovaps ymm0,YMMWORD PTR [rdx+VectorOffset] + vbroadcastss ymm3,DWORD PTR [rcx+BroadcastOffset] + vfmadd231ps ymm5,ymm3,ymm0 +IF Count GE 3 + vbroadcastss ymm3,DWORD PTR [rcx+r10+BroadcastOffset] + vfmadd231ps ymm7,ymm3,ymm0 + vbroadcastss ymm3,DWORD PTR [rcx+r10*2+BroadcastOffset] + vfmadd231ps ymm9,ymm3,ymm0 +ENDIF +IF Count GE 6 + vbroadcastss ymm3,DWORD PTR [rbx+BroadcastOffset] + vfmadd231ps ymm11,ymm3,ymm0 + vbroadcastss ymm3,DWORD PTR [rbx+r10+BroadcastOffset] + vfmadd231ps ymm13,ymm3,ymm0 + vbroadcastss ymm3,DWORD PTR [rbx+r10*2+BroadcastOffset] + vfmadd231ps ymm15,ymm3,ymm0 +ENDIF +ENDIF + + ENDM + +;++ +; +; Routine Description: +; +; This routine is an inner kernel to compute matrix multiplication for a +; set of rows. +; +; Arguments: +; +; A (rcx) - Supplies the address of matrix A. +; +; B (rdx) - Supplies the address of matrix B. The matrix data has been packed +; using MlasSgemmCopyPackB or MlasSgemmTransposePackB. +; +; C (r8) - Supplies the address of matrix C. +; +; CountK (r9) - Supplies the number of columns from matrix A and the number +; of rows from matrix B to iterate over. +; +; CountM - Supplies the maximum number of rows that can be processed for +; matrix A and matrix C. The actual number of rows handled for this +; invocation depends on the kernel implementation. +; +; CountN - Supplies the number of columns from matrix B and matrix C to iterate +; over. +; +; lda - Supplies the first dimension of matrix A. +; +; ldc - Supplies the first dimension of matrix C. +; +; Alpha - Supplies the scaler multiplier (see SGEMM definition). +; +; Return Value: +; +; Returns the number of rows handled. +; +;-- + +SgemmKernelFma3Function MACRO Mode + + NESTED_ENTRY MlasSgemmKernel&Mode&Fma3, _TEXT + + SgemmKernelAvxEntry + +; +; Process 6 rows of the matrices. +; + + cmp r11,6 + jb ProcessCountMLessThan6 + mov r11d,6 ; return 6 rows handled + cmp rbp,8 + jbe ProcessRemainingCountN6 + +ProcessNextColumnLoop16x6: + lea rbx,[r10*2+r10] + add rbx,rcx ; compute matrix A plus 3 rows + ComputeBlockFma3Loop ComputeBlockFma3By16, 6 + lea rcx,[r8+rax*2] ; compute matrix C plus 2 rows + lea rbx,[r8+rax*4] ; compute matrix C plus 4 rows +IFDIFI , + vmulps ymm4,ymm4,ymm2 ; multiply by alpha + vmulps ymm5,ymm5,ymm2 + vmulps ymm6,ymm6,ymm2 + vmulps ymm7,ymm7,ymm2 + vmulps ymm8,ymm8,ymm2 + vmulps ymm9,ymm9,ymm2 + vmulps ymm10,ymm10,ymm2 + vmulps ymm11,ymm11,ymm2 + vmulps ymm12,ymm12,ymm2 + vmulps ymm13,ymm13,ymm2 + vmulps ymm14,ymm14,ymm2 + vmulps ymm15,ymm15,ymm2 +ENDIF + sub rbp,16 + jb OutputMasked16x6Block +IFIDNI , + vfmadd213ps ymm4,ymm2,YMMWORD PTR [r8] + vfmadd213ps ymm5,ymm2,YMMWORD PTR [r8+32] + vfmadd213ps ymm6,ymm2,YMMWORD PTR [r8+rax] + vfmadd213ps ymm7,ymm2,YMMWORD PTR [r8+rax+32] + vfmadd213ps ymm8,ymm2,YMMWORD PTR [rcx] + vfmadd213ps ymm9,ymm2,YMMWORD PTR [rcx+32] + vfmadd213ps ymm10,ymm2,YMMWORD PTR [rcx+rax] + vfmadd213ps ymm11,ymm2,YMMWORD PTR [rcx+rax+32] + vfmadd213ps ymm12,ymm2,YMMWORD PTR [rbx] + vfmadd213ps ymm13,ymm2,YMMWORD PTR [rbx+32] + vfmadd213ps ymm14,ymm2,YMMWORD PTR [rbx+rax] + vfmadd213ps ymm15,ymm2,YMMWORD PTR [rbx+rax+32] +ENDIF + vmovups YMMWORD PTR [r8],ymm4 + vmovups YMMWORD PTR [r8+32],ymm5 + vmovups YMMWORD PTR [r8+rax],ymm6 + vmovups YMMWORD PTR [r8+rax+32],ymm7 + vmovups YMMWORD PTR [rcx],ymm8 + vmovups YMMWORD PTR [rcx+32],ymm9 + vmovups YMMWORD PTR [rcx+rax],ymm10 + vmovups YMMWORD PTR [rcx+rax+32],ymm11 + vmovups YMMWORD PTR [rbx],ymm12 + vmovups YMMWORD PTR [rbx+32],ymm13 + vmovups YMMWORD PTR [rbx+rax],ymm14 + vmovups YMMWORD PTR [rbx+rax+32],ymm15 + add r8,16*4 ; advance matrix C by 16 columns + mov rcx,rsi ; reload matrix A + vzeroall + cmp rbp,8 + ja ProcessNextColumnLoop16x6 + test rbp,rbp + jz ExitKernel + +ProcessRemainingCountN6: + lea rbx,[r10*2+r10] + add rbx,rcx ; compute matrix A plus 3 rows + ComputeBlockFma3Loop ComputeBlockFma3By8, 6 + lea rcx,[r8+rax*2] ; compute matrix C plus 2 rows + lea rbx,[r8+rax*4] ; compute matrix C plus 4 rows +IFDIFI , + vmulps ymm5,ymm5,ymm2 ; multiply by alpha + vmulps ymm7,ymm7,ymm2 + vmulps ymm9,ymm9,ymm2 + vmulps ymm11,ymm11,ymm2 + vmulps ymm13,ymm13,ymm2 + vmulps ymm15,ymm15,ymm2 +ENDIF + cmp rbp,8 + jb OutputMasked8x6Block +IFIDNI , + vfmadd213ps ymm5,ymm2,YMMWORD PTR [r8] + vfmadd213ps ymm7,ymm2,YMMWORD PTR [r8+rax] + vfmadd213ps ymm9,ymm2,YMMWORD PTR [rcx] + vfmadd213ps ymm11,ymm2,YMMWORD PTR [rcx+rax] + vfmadd213ps ymm13,ymm2,YMMWORD PTR [rbx] + vfmadd213ps ymm15,ymm2,YMMWORD PTR [rbx+rax] +ENDIF + vmovups YMMWORD PTR [r8],ymm5 + vmovups YMMWORD PTR [r8+rax],ymm7 + vmovups YMMWORD PTR [rcx],ymm9 + vmovups YMMWORD PTR [rcx+rax],ymm11 + vmovups YMMWORD PTR [rbx],ymm13 + vmovups YMMWORD PTR [rbx+rax],ymm15 + jmp ExitKernelAndZeroUpper + +OutputMasked16x6Block: +IFIDNI , + vfmadd213ps ymm4,ymm2,YMMWORD PTR [r8] + vfmadd213ps ymm6,ymm2,YMMWORD PTR [r8+rax] + vfmadd213ps ymm8,ymm2,YMMWORD PTR [rcx] + vfmadd213ps ymm10,ymm2,YMMWORD PTR [rcx+rax] + vfmadd213ps ymm12,ymm2,YMMWORD PTR [rbx] + vfmadd213ps ymm14,ymm2,YMMWORD PTR [rbx+rax] +ENDIF + vmovups YMMWORD PTR [r8],ymm4 + vmovups YMMWORD PTR [r8+rax],ymm6 + vmovups YMMWORD PTR [rcx],ymm8 + vmovups YMMWORD PTR [rcx+rax],ymm10 + vmovups YMMWORD PTR [rbx],ymm12 + vmovups YMMWORD PTR [rbx+rax],ymm14 + add r8,8*4 ; advance matrix C by 8 columns + add rcx,8*4 ; advance matrix C plus 2 rows by 8 columns + add rbx,8*4 ; advance matrix C plus 4 rows by 8 columns + add rbp,8 ; correct for over-subtract above + +OutputMasked8x6Block: + mov DWORD PTR SgemmKernelFrame.CountN[rsp],ebp + vbroadcastss ymm0,DWORD PTR SgemmKernelFrame.CountN[rsp] + vpcmpgtd ymm0,ymm0,YMMWORD PTR [MlasMaskMoveAvx] +IFIDNI , + vmaskmovps ymm4,ymm0,YMMWORD PTR [r8] + vmaskmovps ymm6,ymm0,YMMWORD PTR [r8+rax] + vmaskmovps ymm8,ymm0,YMMWORD PTR [rcx] + vmaskmovps ymm10,ymm0,YMMWORD PTR [rcx+rax] + vmaskmovps ymm12,ymm0,YMMWORD PTR [rbx] + vmaskmovps ymm14,ymm0,YMMWORD PTR [rbx+rax] + vfmadd213ps ymm5,ymm2,ymm4 + vfmadd213ps ymm7,ymm2,ymm6 + vfmadd213ps ymm9,ymm2,ymm8 + vfmadd213ps ymm11,ymm2,ymm10 + vfmadd213ps ymm13,ymm2,ymm12 + vfmadd213ps ymm15,ymm2,ymm14 +ENDIF + vmaskmovps YMMWORD PTR [r8],ymm0,ymm5 + vmaskmovps YMMWORD PTR [r8+rax],ymm0,ymm7 + vmaskmovps YMMWORD PTR [rcx],ymm0,ymm9 + vmaskmovps YMMWORD PTR [rcx+rax],ymm0,ymm11 + vmaskmovps YMMWORD PTR [rbx],ymm0,ymm13 + vmaskmovps YMMWORD PTR [rbx+rax],ymm0,ymm15 + +; +; Restore non-volatile registers and return. +; + +ExitKernelAndZeroUpper: + vzeroupper + +ExitKernel: + SgemmKernelAvxExit + +; +; Process 3 rows of the matrices. +; + +ProcessCountMLessThan6: + cmp r11,3 + jb ProcessCountMLessThan3 + mov r11d,3 ; return 3 rows handled + cmp rbp,8 + jbe ProcessRemainingCountN3 + +ProcessNextColumnLoop16x3: + ComputeBlockFma3Loop ComputeBlockFma3By16, 3 +IFDIFI , + vmulps ymm4,ymm4,ymm2 ; multiply by alpha + vmulps ymm5,ymm5,ymm2 + vmulps ymm6,ymm6,ymm2 + vmulps ymm7,ymm7,ymm2 + vmulps ymm8,ymm8,ymm2 + vmulps ymm9,ymm9,ymm2 +ENDIF + sub rbp,16 + jb OutputMasked16x3Block +IFIDNI , + vfmadd213ps ymm4,ymm2,YMMWORD PTR [r8] + vfmadd213ps ymm5,ymm2,YMMWORD PTR [r8+32] + vfmadd213ps ymm6,ymm2,YMMWORD PTR [r8+rax] + vfmadd213ps ymm7,ymm2,YMMWORD PTR [r8+rax+32] + vfmadd213ps ymm8,ymm2,YMMWORD PTR [r8+rax*2] + vfmadd213ps ymm9,ymm2,YMMWORD PTR [r8+rax*2+32] +ENDIF + vmovups YMMWORD PTR [r8],ymm4 + vmovups YMMWORD PTR [r8+32],ymm5 + vmovups YMMWORD PTR [r8+rax],ymm6 + vmovups YMMWORD PTR [r8+rax+32],ymm7 + vmovups YMMWORD PTR [r8+rax*2],ymm8 + vmovups YMMWORD PTR [r8+rax*2+32],ymm9 + add r8,16*4 ; advance matrix C by 16 columns + mov rcx,rsi ; reload matrix A + vzeroall + cmp rbp,8 + ja ProcessNextColumnLoop16x3 + test rbp,rbp + jz ExitKernel + +ProcessRemainingCountN3: + ComputeBlockFma3Loop ComputeBlockFma3By8, 3 +IFDIFI , + vmulps ymm5,ymm5,ymm2 ; multiply by alpha + vmulps ymm7,ymm7,ymm2 + vmulps ymm9,ymm9,ymm2 +ENDIF + cmp rbp,8 + jb OutputMasked8x3Block +IFIDNI , + vfmadd213ps ymm5,ymm2,YMMWORD PTR [r8] + vfmadd213ps ymm7,ymm2,YMMWORD PTR [r8+rax] + vfmadd213ps ymm9,ymm2,YMMWORD PTR [r8+rax*2] +ENDIF + vmovups YMMWORD PTR [r8],ymm5 + vmovups YMMWORD PTR [r8+rax],ymm7 + vmovups YMMWORD PTR [r8+rax*2],ymm9 + jmp ExitKernelAndZeroUpper + +OutputMasked16x3Block: +IFIDNI , + vfmadd213ps ymm4,ymm2,YMMWORD PTR [r8] + vfmadd213ps ymm6,ymm2,YMMWORD PTR [r8+rax] + vfmadd213ps ymm8,ymm2,YMMWORD PTR [r8+rax*2] +ENDIF + vmovups YMMWORD PTR [r8],ymm4 + vmovups YMMWORD PTR [r8+rax],ymm6 + vmovups YMMWORD PTR [r8+rax*2],ymm8 + add r8,8*4 ; advance matrix C by 8 columns + add rbp,8 ; correct for over-subtract above + +OutputMasked8x3Block: + mov DWORD PTR SgemmKernelFrame.CountN[rsp],ebp + vbroadcastss ymm0,DWORD PTR SgemmKernelFrame.CountN[rsp] + vpcmpgtd ymm0,ymm0,YMMWORD PTR [MlasMaskMoveAvx] +IFIDNI , + vmaskmovps ymm4,ymm0,YMMWORD PTR [r8] + vmaskmovps ymm6,ymm0,YMMWORD PTR [r8+rax] + vmaskmovps ymm8,ymm0,YMMWORD PTR [r8+rax*2] + vfmadd213ps ymm5,ymm2,ymm4 + vfmadd213ps ymm7,ymm2,ymm6 + vfmadd213ps ymm9,ymm2,ymm8 +ENDIF + vmaskmovps YMMWORD PTR [r8],ymm0,ymm5 + vmaskmovps YMMWORD PTR [r8+rax],ymm0,ymm7 + vmaskmovps YMMWORD PTR [r8+rax*2],ymm0,ymm9 + jmp ExitKernelAndZeroUpper + +; +; Process 1 row of the matrices. +; + +ProcessCountMLessThan3: + mov r11d,1 ; return 1 row handled + cmp rbp,32 + jb ProcessRemainingCountN1LessThan32 + mov rbx,r9 + shl rbx,6 ; compute 16*CountK*sizeof(float) + +ProcessNextColumnLoop32x1: + ComputeBlockFma3Loop ComputeBlockFma3By32, 1 + add rdx,rbx ; advance matrix B by 16*CountK floats +IFDIFI , + vmulps ymm4,ymm4,ymm2 ; multiply by alpha + vmulps ymm5,ymm5,ymm2 + vmulps ymm6,ymm6,ymm2 + vmulps ymm7,ymm7,ymm2 +ELSE + vfmadd213ps ymm4,ymm2,YMMWORD PTR [r8] + vfmadd213ps ymm5,ymm2,YMMWORD PTR [r8+32] + vfmadd213ps ymm6,ymm2,YMMWORD PTR [r8+64] + vfmadd213ps ymm7,ymm2,YMMWORD PTR [r8+96] +ENDIF + sub rbp,32 + vmovups YMMWORD PTR [r8],ymm4 + vmovups YMMWORD PTR [r8+32],ymm5 + vmovups YMMWORD PTR [r8+64],ymm6 + vmovups YMMWORD PTR [r8+96],ymm7 + add r8,32*4 ; advance matrix C by 32 columns + mov rcx,rsi ; reload matrix A + vzeroall + cmp rbp,32 + jae ProcessNextColumnLoop32x1 + test rbp,rbp + jz ExitKernel + +ProcessRemainingCountN1LessThan32: + cmp rbp,8 + jbe ProcessRemainingCountN1 + +ProcessNextColumnLoop16x1: + ComputeBlockFma3Loop ComputeBlockFma3By16, 1 +IFDIFI , + vmulps ymm4,ymm4,ymm2 ; multiply by alpha + vmulps ymm5,ymm5,ymm2 +ENDIF + sub rbp,16 + jb OutputMasked16x1Block +IFIDNI , + vfmadd213ps ymm4,ymm2,YMMWORD PTR [r8] + vfmadd213ps ymm5,ymm2,YMMWORD PTR [r8+32] +ENDIF + vmovups YMMWORD PTR [r8],ymm4 + vmovups YMMWORD PTR [r8+32],ymm5 + add r8,16*4 ; advance matrix C by 16 columns + mov rcx,rsi ; reload matrix A + vzeroall + cmp rbp,8 + ja ProcessNextColumnLoop16x1 + test rbp,rbp + jz ExitKernel + +ProcessRemainingCountN1: + ComputeBlockFma3Loop ComputeBlockFma3By8, 1 +IFDIFI , + vmulps ymm5,ymm5,ymm2 ; multiply by alpha +ENDIF + cmp rbp,8 + jb OutputMasked8x1Block +IFIDNI , + vfmadd213ps ymm5,ymm2,YMMWORD PTR [r8] +ENDIF + vmovups YMMWORD PTR [r8],ymm5 + jmp ExitKernelAndZeroUpper + +OutputMasked16x1Block: +IFIDNI , + vfmadd213ps ymm4,ymm2,YMMWORD PTR [r8] +ENDIF + vmovups YMMWORD PTR [r8],ymm4 + add r8,8*4 ; advance matrix C by 8 columns + add rbp,8 ; correct for over-subtract above + +OutputMasked8x1Block: + mov DWORD PTR SgemmKernelFrame.CountN[rsp],ebp + vbroadcastss ymm0,DWORD PTR SgemmKernelFrame.CountN[rsp] + vpcmpgtd ymm0,ymm0,YMMWORD PTR [MlasMaskMoveAvx] +IFIDNI , + vmaskmovps ymm4,ymm0,YMMWORD PTR [r8] + vfmadd213ps ymm5,ymm2,ymm4 +ENDIF + vmaskmovps YMMWORD PTR [r8],ymm0,ymm5 + jmp ExitKernelAndZeroUpper + + NESTED_END MlasSgemmKernel&Mode&Fma3, _TEXT + + ENDM + +SgemmKernelFma3Function Zero +SgemmKernelFma3Function Add + + END diff --git a/onnxruntime/core/mlas/lib/amd64/SgemmKernelSse2.asm b/onnxruntime/core/mlas/lib/amd64/SgemmKernelSse2.asm new file mode 100644 index 0000000000000..c9e53fce3996a --- /dev/null +++ b/onnxruntime/core/mlas/lib/amd64/SgemmKernelSse2.asm @@ -0,0 +1,534 @@ +;++ +; +; Copyright (c) Microsoft Corporation. All rights reserved. +; +; Licensed under the MIT License. +; +; Module Name: +; +; SgemmKernelSse2.asm +; +; Abstract: +; +; This module implements the kernels for the single precision matrix/matrix +; multiply operation (SGEMM). +; +; This implementation uses SSE2 instructions. +; +;-- + + .xlist +INCLUDE macamd64.inc +INCLUDE SgemmKernelCommon.inc + .list + +; +; ComputeBlocksSseBy16 +; +; Macro Description: +; +; This macro multiplies and accumulates for a 16xN block (where N is 1,2) +; of the output matrix. +; +; Arguments: +; +; Count - Supplies the number of rows to access from matrix A. +; +; VectorOffset - Supplies the byte offset from matrix B to fetch elements. +; +; Shuffle - Supplies the shuffle mask to extract the element from matrix A. +; +; Implicit Arguments: +; +; rdx - Supplies the address into the matrix B data. +; +; xmm0-xmm1 - Supplies up to four elements loaded from matrix A and matrix A +; plus one row. +; +; xmm8-xmm15 - Supplies the block accumulators. +; + +ComputeBlockSseBy16 MACRO Count, VectorOffset, Shuffle + + movaps xmm4,XMMWORD PTR [rdx+VectorOffset] + movaps xmm5,XMMWORD PTR [rdx+VectorOffset+16] + pshufd xmm2,xmm0,Shuffle +IF Count EQ 2 + pshufd xmm3,xmm1,Shuffle + movaps xmm6,xmm4 + movaps xmm7,xmm5 +ENDIF + mulps xmm4,xmm2 + mulps xmm5,xmm2 + addps xmm8,xmm4 + addps xmm9,xmm5 +IF Count EQ 2 + mulps xmm6,xmm3 + mulps xmm7,xmm3 + addps xmm12,xmm6 + addps xmm13,xmm7 +ENDIF + movaps xmm4,XMMWORD PTR [rdx+VectorOffset+32] + movaps xmm5,XMMWORD PTR [rdx+VectorOffset+48] +IF Count EQ 2 + movaps xmm6,xmm4 + movaps xmm7,xmm5 +ENDIF + mulps xmm4,xmm2 + mulps xmm5,xmm2 + addps xmm10,xmm4 + addps xmm11,xmm5 +IF Count EQ 2 + mulps xmm6,xmm3 + mulps xmm7,xmm3 + addps xmm14,xmm6 + addps xmm15,xmm7 +ENDIF + + ENDM + +;++ +; +; Routine Description: +; +; This routine is an inner kernel to compute matrix multiplication for a +; set of rows. +; +; Arguments: +; +; A (rcx) - Supplies the address of matrix A. +; +; B (rdx) - Supplies the address of matrix B. The matrix data has been packed +; using MlasSgemmCopyPackB or MlasSgemmTransposePackB. +; +; C (r8) - Supplies the address of matrix C. +; +; CountK (r9d) - Supplies the number of columns from matrix A and the number +; of rows from matrix B to iterate over. +; +; CountM - Supplies the maximum number of rows that can be processed for +; matrix A and matrix C. The actual number of rows handled for this +; invocation depends on the kernel implementation. +; +; CountN - Supplies the number of columns from matrix B and matrix C to iterate +; over. +; +; lda - Supplies the first dimension of matrix A. +; +; ldc - Supplies the first dimension of matrix C. +; +; Alpha - Supplies the scaler multiplier (see SGEMM definition). +; +; Return Value: +; +; Returns the number of rows handled. +; +;-- + +SgemmKernelSseFunction MACRO Mode + + NESTED_ENTRY MlasSgemmKernel&Mode&Sse, _TEXT + + rex_push_reg rbp + push_reg rbx + push_reg rsi + push_reg rdi + alloc_stack (SgemmKernelFrame.SavedRdi) + save_xmm128 xmm6,SgemmKernelFrame.SavedXmm6 + save_xmm128 xmm7,SgemmKernelFrame.SavedXmm7 + save_xmm128 xmm8,SgemmKernelFrame.SavedXmm8 + save_xmm128 xmm9,SgemmKernelFrame.SavedXmm9 + save_xmm128 xmm10,SgemmKernelFrame.SavedXmm10 + save_xmm128 xmm11,SgemmKernelFrame.SavedXmm11 + save_xmm128 xmm12,SgemmKernelFrame.SavedXmm12 + save_xmm128 xmm13,SgemmKernelFrame.SavedXmm13 + save_xmm128 xmm14,SgemmKernelFrame.SavedXmm14 + save_xmm128 xmm15,SgemmKernelFrame.SavedXmm15 + + END_PROLOGUE + + mov rsi,rcx + mov rbp,SgemmKernelFrame.CountN[rsp] + mov rax,SgemmKernelFrame.ldc[rsp] + shl rax,2 + +; +; Process 2 rows of the matrices. +; + + cmp QWORD PTR SgemmKernelFrame.CountM[rsp],2 + jb ProcessCountMLessThan2 + mov r10,SgemmKernelFrame.lda[rsp] + shl r10,2 + mov r11d,2 ; return 2 rows handled + +ProcessNextColumnLoop16x2: + xorps xmm8,xmm8 ; clear block accumulators + xorps xmm9,xmm9 + xorps xmm10,xmm10 + xorps xmm11,xmm11 + xorps xmm12,xmm12 + xorps xmm13,xmm13 + xorps xmm14,xmm14 + xorps xmm15,xmm15 + mov rdi,r9 ; reload CountK + sub rdi,4 + jb ProcessRemaining16x2Blocks + +Compute16x2BlockBy4Loop: + movups xmm0,XMMWORD PTR [rcx] + movups xmm1,XMMWORD PTR [rcx+r10] + ComputeBlockSseBy16 2, 0, 000h + ComputeBlockSseBy16 2, 16*4, 055h + sub rdx,-32*4 ; advance matrix B by 32 columns + ComputeBlockSseBy16 2, 0, 0AAh + ComputeBlockSseBy16 2, 16*4, 0FFh + sub rdx,-32*4 ; advance matrix B by 32 columns + add rcx,4*4 ; advance matrix A by 4 columns + sub rdi,4 + jae Compute16x2BlockBy4Loop + +ProcessRemaining16x2Blocks: + add rdi,4 ; correct for over-subtract above + jz Output16x2Block + +Compute16x2BlockBy1Loop: + movss xmm0,DWORD PTR [rcx] + movss xmm1,DWORD PTR [rcx+r10] + ComputeBlockSseBy16 2, 0, 000h + add rdx,16*4 ; advance matrix B by 16 columns + add rcx,4 ; advance matrix A by 1 column + dec rdi + jne Compute16x2BlockBy1Loop + +Output16x2Block: + movss xmm2,DWORD PTR SgemmKernelFrame.Alpha[rsp] + lea rcx,[r8+rax] ; compute matrix C plus 1 row + shufps xmm2,xmm2,0 + mulps xmm8,xmm2 ; multiply by alpha + mulps xmm9,xmm2 + mulps xmm10,xmm2 + mulps xmm11,xmm2 + mulps xmm12,xmm2 + mulps xmm13,xmm2 + mulps xmm14,xmm2 + mulps xmm15,xmm2 + sub rbp,16 + jb OutputPartial16x2Block +IFIDNI , + movups xmm0,XMMWORD PTR [r8] + movups xmm1,XMMWORD PTR [r8+16] + movups xmm2,XMMWORD PTR [r8+32] + movups xmm3,XMMWORD PTR [r8+48] + movups xmm4,XMMWORD PTR [rcx] + movups xmm5,XMMWORD PTR [rcx+16] + movups xmm6,XMMWORD PTR [rcx+32] + movups xmm7,XMMWORD PTR [rcx+48] + addps xmm8,xmm0 + addps xmm9,xmm1 + addps xmm10,xmm2 + addps xmm11,xmm3 + addps xmm12,xmm4 + addps xmm13,xmm5 + addps xmm14,xmm6 + addps xmm15,xmm7 +ENDIF + movups XMMWORD PTR [r8],xmm8 + movups XMMWORD PTR [r8+16],xmm9 + movups XMMWORD PTR [r8+32],xmm10 + movups XMMWORD PTR [r8+48],xmm11 + movups XMMWORD PTR [rcx],xmm12 + movups XMMWORD PTR [rcx+16],xmm13 + movups XMMWORD PTR [rcx+32],xmm14 + movups XMMWORD PTR [rcx+48],xmm15 + add r8,16*4 ; advance matrix C by 16 columns + mov rcx,rsi ; reload matrix A + test rbp,rbp + jnz ProcessNextColumnLoop16x2 + +; +; Restore non-volatile registers and return. +; + +ExitKernel: + mov eax,r11d + movaps xmm6,SgemmKernelFrame.SavedXmm6[rsp] + movaps xmm7,SgemmKernelFrame.SavedXmm7[rsp] + movaps xmm8,SgemmKernelFrame.SavedXmm8[rsp] + movaps xmm9,SgemmKernelFrame.SavedXmm9[rsp] + movaps xmm10,SgemmKernelFrame.SavedXmm10[rsp] + movaps xmm11,SgemmKernelFrame.SavedXmm11[rsp] + movaps xmm12,SgemmKernelFrame.SavedXmm12[rsp] + movaps xmm13,SgemmKernelFrame.SavedXmm13[rsp] + movaps xmm14,SgemmKernelFrame.SavedXmm14[rsp] + movaps xmm15,SgemmKernelFrame.SavedXmm15[rsp] + add rsp,(SgemmKernelFrame.SavedRdi) + + BEGIN_EPILOGUE + + pop rdi + pop rsi + pop rbx + pop rbp + ret + +; +; Output a partial 16x2 block to the matrix. +; + +OutputPartial16x2Block: + add rbp,16 ; correct for over-subtract above + cmp rbp,4 + jb OutputPartialLessThan4x2Block + cmp rbp,8 + jb OutputPartialLessThan8x2Block + cmp rbp,12 + jb OutputPartialLessThan12x2Block + +IFIDNI , + movups xmm0,XMMWORD PTR [r8] + movups xmm1,XMMWORD PTR [r8+16] + movups xmm2,XMMWORD PTR [r8+32] + movups xmm3,XMMWORD PTR [rcx] + movups xmm4,XMMWORD PTR [rcx+16] + movups xmm5,XMMWORD PTR [rcx+32] + addps xmm8,xmm0 + addps xmm9,xmm1 + addps xmm10,xmm2 + addps xmm12,xmm3 + addps xmm13,xmm4 + addps xmm14,xmm5 +ENDIF + movups XMMWORD PTR [r8],xmm8 + movups XMMWORD PTR [r8+16],xmm9 + movups XMMWORD PTR [r8+32],xmm10 + movups XMMWORD PTR [rcx],xmm12 + movups XMMWORD PTR [rcx+16],xmm13 + movups XMMWORD PTR [rcx+32],xmm14 + and ebp,3 ; check if remaining count is small + jz ExitKernel + movaps xmm8,xmm11 ; shift remaining elements down + movaps xmm12,xmm15 + add r8,12*4 ; advance matrix C by 12 columns + add rcx,12*4 ; advance matrix C plus 1 row by 12 columns + jmp OutputPartialLessThan4x2Block + +OutputPartialLessThan12x2Block: +IFIDNI , + movups xmm0,XMMWORD PTR [r8] + movups xmm1,XMMWORD PTR [r8+16] + movups xmm2,XMMWORD PTR [rcx] + movups xmm3,XMMWORD PTR [rcx+16] + addps xmm8,xmm0 + addps xmm9,xmm1 + addps xmm12,xmm2 + addps xmm13,xmm3 +ENDIF + movups XMMWORD PTR [r8],xmm8 + movups XMMWORD PTR [r8+16],xmm9 + movups XMMWORD PTR [rcx],xmm12 + movups XMMWORD PTR [rcx+16],xmm13 + and ebp,3 ; check if remaining count is small + jz ExitKernel + movaps xmm8,xmm10 ; shift remaining elements down + movaps xmm12,xmm14 + add r8,8*4 ; advance matrix C by 8 columns + add rcx,8*4 ; advance matrix C plus 1 row by 8 columns + jmp OutputPartialLessThan4x2Block + +OutputPartialLessThan8x2Block: +IFIDNI , + movups xmm0,XMMWORD PTR [r8] + movups xmm1,XMMWORD PTR [rcx] + addps xmm8,xmm0 + addps xmm12,xmm1 +ENDIF + movups XMMWORD PTR [r8],xmm8 + movups XMMWORD PTR [rcx],xmm12 + and ebp,3 ; check if remaining count is small + jz ExitKernel + movaps xmm8,xmm9 ; shift remaining elements down + movaps xmm12,xmm13 + add r8,4*4 ; advance matrix C by 4 columns + add rcx,4*4 ; advance matrix C plus 1 row by 4 columns + +OutputPartialLessThan4x2Block: + cmp rbp,2 + jb OutputPartial1x2Block +IFIDNI , + movsd xmm0,QWORD PTR [r8] + movsd xmm1,QWORD PTR [rcx] + addps xmm8,xmm0 + addps xmm12,xmm1 +ENDIF + movsd QWORD PTR [r8],xmm8 + movsd QWORD PTR [rcx],xmm12 + and ebp,1 ; check if remaining count is odd + jz ExitKernel + movhlps xmm8,xmm8 ; shift third element down + movhlps xmm12,xmm12 + add r8,2*4 ; advance matrix C by 2 columns + add rcx,2*4 ; advance matrix C plus 1 row by 2 columns + +OutputPartial1x2Block: +IFIDNI , + addss xmm8,DWORD PTR [r8] + addss xmm12,DWORD PTR [rcx] +ENDIF + movss DWORD PTR [r8],xmm8 + movss DWORD PTR [rcx],xmm12 + jmp ExitKernel + +; +; Process 1 row of the matrices. +; + +ProcessCountMLessThan2: + mov r11d,1 ; return 1 row handled + +ProcessNextColumnLoop16x1: + xorps xmm8,xmm8 ; clear block accumulators + xorps xmm9,xmm9 + xorps xmm10,xmm10 + xorps xmm11,xmm11 + mov rdi,r9 ; reload CountK + sub rdi,4 + jb ProcessRemaining16x1Blocks + +Compute16x1BlockBy4Loop: + movups xmm0,XMMWORD PTR [rcx] + ComputeBlockSseBy16 1, 0, 000h + ComputeBlockSseBy16 1, 16*4, 055h + sub rdx,-32*4 ; advance matrix B by 32 columns + ComputeBlockSseBy16 1, 0, 0AAh + ComputeBlockSseBy16 1, 16*4, 0FFh + sub rdx,-32*4 ; advance matrix B by 32 columns + add rcx,4*4 ; advance matrix A by 4 columns + sub rdi,4 + jae Compute16x1BlockBy4Loop + +ProcessRemaining16x1Blocks: + add rdi,4 ; correct for over-subtract above + jz Output16x1Block + +Compute16x1BlockBy1Loop: + movss xmm0,DWORD PTR [rcx] + ComputeBlockSseBy16 1, 0, 000h + add rdx,16*4 ; advance matrix B by 16 columns + add rcx,4 ; advance matrix A by 1 column + dec rdi + jne Compute16x1BlockBy1Loop + +Output16x1Block: + movss xmm2,DWORD PTR SgemmKernelFrame.Alpha[rsp] + shufps xmm2,xmm2,0 + mulps xmm8,xmm2 ; multiply by alpha + mulps xmm9,xmm2 + mulps xmm10,xmm2 + mulps xmm11,xmm2 + sub rbp,16 + jb OutputPartial16x1Block +IFIDNI , + movups xmm0,XMMWORD PTR [r8] + movups xmm1,XMMWORD PTR [r8+16] + movups xmm2,XMMWORD PTR [r8+32] + movups xmm3,XMMWORD PTR [r8+48] + addps xmm8,xmm0 + addps xmm9,xmm1 + addps xmm10,xmm2 + addps xmm11,xmm3 +ENDIF + movups XMMWORD PTR [r8],xmm8 + movups XMMWORD PTR [r8+16],xmm9 + movups XMMWORD PTR [r8+32],xmm10 + movups XMMWORD PTR [r8+48],xmm11 + add r8,16*4 ; advance matrix C by 16 columns + mov rcx,rsi ; reload matrix A + test rbp,rbp + jnz ProcessNextColumnLoop16x1 + jmp ExitKernel + +; +; Output a partial 16x1 block to the matrix. +; + +OutputPartial16x1Block: + add rbp,16 ; correct for over-subtract above + cmp rbp,4 + jb OutputPartialLessThan4x1Block + cmp rbp,8 + jb OutputPartialLessThan8x1Block + cmp rbp,12 + jb OutputPartialLessThan12x1Block + +IFIDNI , + movups xmm0,XMMWORD PTR [r8] + movups xmm1,XMMWORD PTR [r8+16] + movups xmm2,XMMWORD PTR [r8+32] + addps xmm8,xmm0 + addps xmm9,xmm1 + addps xmm10,xmm2 +ENDIF + movups XMMWORD PTR [r8],xmm8 + movups XMMWORD PTR [r8+16],xmm9 + movups XMMWORD PTR [r8+32],xmm10 + and ebp,3 ; check if remaining count is small + jz ExitKernel + movaps xmm8,xmm11 ; shift remaining elements down + add r8,12*4 ; advance matrix C by 12 columns + jmp OutputPartialLessThan4x1Block + +OutputPartialLessThan12x1Block: +IFIDNI , + movups xmm0,XMMWORD PTR [r8] + movups xmm1,XMMWORD PTR [r8+16] + addps xmm8,xmm0 + addps xmm9,xmm1 +ENDIF + movups XMMWORD PTR [r8],xmm8 + movups XMMWORD PTR [r8+16],xmm9 + and ebp,3 ; check if remaining count is small + jz ExitKernel + movaps xmm8,xmm10 ; shift remaining elements down + add r8,8*4 ; advance matrix C by 8 columns + jmp OutputPartialLessThan4x1Block + +OutputPartialLessThan8x1Block: +IFIDNI , + movups xmm0,XMMWORD PTR [r8] + addps xmm8,xmm0 +ENDIF + movups XMMWORD PTR [r8],xmm8 + and ebp,3 ; check if remaining count is small + jz ExitKernel + movaps xmm8,xmm9 ; shift remaining elements down + add r8,4*4 ; advance matrix C by 4 columns + +OutputPartialLessThan4x1Block: + cmp rbp,2 + jb OutputPartial1x1Block +IFIDNI , + movsd xmm0,QWORD PTR [r8] + addps xmm8,xmm0 +ENDIF + movsd QWORD PTR [r8],xmm8 + and ebp,1 ; check if remaining count is odd + jz ExitKernel + movhlps xmm8,xmm8 ; shift third element down + add r8,2*4 ; advance matrix C by 2 columns + +OutputPartial1x1Block: +IFIDNI , + addss xmm8,DWORD PTR [r8] +ENDIF + movss DWORD PTR [r8],xmm8 + jmp ExitKernel + + NESTED_END MlasSgemmKernel&Mode&Sse, _TEXT + + ENDM + +SgemmKernelSseFunction Zero +SgemmKernelSseFunction Add + + END diff --git a/onnxruntime/core/mlas/lib/amd64/cvtfp16a.asm b/onnxruntime/core/mlas/lib/amd64/cvtfp16a.asm new file mode 100644 index 0000000000000..03351ced05884 --- /dev/null +++ b/onnxruntime/core/mlas/lib/amd64/cvtfp16a.asm @@ -0,0 +1,124 @@ +;++ +; +; Copyright (c) Microsoft Corporation. All rights reserved. +; +; Licensed under the MIT License. +; +; Module Name: +; +; cvtfp16a.asm +; +; Abstract: +; +; This module implements routines to convert between FP16 and FP32 formats. +; +;-- + + .xlist +INCLUDE macamd64.inc + .list + + .const + + ALIGN 16 +MlasFp16MaskSign DD 4 DUP (00007FFFh) +MlasFp16CompareInfinity DD 4 DUP (00007C00h) +MlasFp16CompareSmallest DD 4 DUP (00000400h) +MlasFp16AdjustExponent DD 4 DUP (38000000h) +MlasFp16MagicDenormal DD 4 DUP (38800000h) + + SUBTTL "Convert buffer of half-precision floats to single-precision floats" +;++ +; +; Routine Description: +; +; This routine converts the source buffer of half-precision floats to the +; destination buffer of single-precision floats. +; +; This implementation uses SSE2 instructions. +; +; Arguments: +; +; Source (rcx) - Supplies the address of the source buffer of half-precision +; floats. +; +; Destination (edx) - Supplies the address of the destination buffer of +; single-precision floats. +; +; Count (r8) - Supplies the number of elements to convert. +; +; Return Value: +; +; None. +; +;-- + + LEAF_ENTRY MlasConvertHalfToFloatBuffer, _TEXT + + test r8,r8 + jz ExitRoutine + cmp r8,4 + jb LoadPartialVector + +LoadFullVector: + movq xmm0,QWORD PTR [rcx] + add rcx,4*2 ; advance S by 4 elements + +ConvertHalfToFloat: + punpcklwd xmm0,xmm0 ; duplicate 4 WORDs to 4 DWORDs + movaps xmm1,xmm0 ; isolate exponent/mantissa + pand xmm1,XMMWORD PTR [MlasFp16MaskSign] + pxor xmm0,xmm1 ; isolate sign bit + movaps xmm2,XMMWORD PTR [MlasFp16CompareInfinity] + pcmpgtd xmm2,xmm1 ; test for infinity/NaNs + movaps xmm3,XMMWORD PTR [MlasFp16CompareSmallest] + pcmpgtd xmm3,xmm1 ; test for denormals + pandn xmm2,XMMWORD PTR [MlasFp16AdjustExponent] + pslld xmm1,13 ; shift exponent/mask into place + movaps xmm4,xmm1 + paddd xmm1,XMMWORD PTR [MlasFp16AdjustExponent] + paddd xmm1,xmm2 ; adjust exponent again for infinity/NaNs + paddd xmm4,XMMWORD PTR [MlasFp16MagicDenormal] + pslld xmm0,16 ; shift sign into place + subps xmm4,XMMWORD PTR [MlasFp16MagicDenormal] + pand xmm4,xmm3 ; select elements that are denormals + pandn xmm3,xmm1 ; select elements that are not denormals + por xmm3,xmm4 ; blend the selected values together + por xmm0,xmm3 ; merge sign into exponent/mantissa + + cmp r8,4 ; storing full vector? + jb StorePartialVector + movups XMMWORD PTR [rdx],xmm0 + add rdx,4*4 ; advance D by 4 elements + sub r8,4 + jz ExitRoutine + cmp r8,4 + jae LoadFullVector + +LoadPartialVector: + pxor xmm0,xmm0 + pinsrw xmm0,WORD PTR [rcx],0 + cmp r8,2 + jb ConvertHalfToFloat + pinsrw xmm0,WORD PTR [rcx+2],1 + je ConvertHalfToFloat + pinsrw xmm0,WORD PTR [rcx+4],2 + jmp ConvertHalfToFloat + +StorePartialVector: + cmp r8,2 + jb StoreLastElement + movsd QWORD PTR [rdx],xmm0 + je ExitRoutine + movhlps xmm0,xmm0 ; shift third element down + add rdx,4*2 ; advance D by 2 elements + +StoreLastElement: + movss DWORD PTR [rdx],xmm0 + +ExitRoutine: + ret + + LEAF_END MlasConvertHalfToFloatBuffer, _TEXT + + END diff --git a/onnxruntime/core/mlas/lib/amd64/sgemma.asm b/onnxruntime/core/mlas/lib/amd64/sgemma.asm new file mode 100644 index 0000000000000..8522c8bdde301 --- /dev/null +++ b/onnxruntime/core/mlas/lib/amd64/sgemma.asm @@ -0,0 +1,181 @@ +;++ +; +; Copyright (c) Microsoft Corporation. All rights reserved. +; +; Licensed under the MIT License. +; +; Module Name: +; +; sgemma.asm +; +; Abstract: +; +; This module implements the kernels for the single precision matrix/matrix +; multiply operation (SGEMM). +; +;-- + + .xlist +INCLUDE macamd64.inc + .list + +;++ +; +; Routine Description: +; +; This routine transposes elements from the source matrix to the destination +; packed buffer. +; +; 4 columns of 16 rows from the source matrix are transposed to 16 columns of 4 +; rows in the destination packed buffer. +; +; This implementation uses SSE2 instructions. +; +; Arguments: +; +; D (rcx) - Supplies the address of the destination packed buffer. +; +; B (rdx) - Supplies the address of the source matrix. +; +; ldb (r8d) - Supplies the number of elements per row of the source matrix. +; +; Return Value: +; +; None. +; +;-- + + LEAF_ENTRY MlasSgemmTransposePackB16x4Sse, _TEXT + + shl r8,2 ; convert ldb to bytes + mov r9d,4 ; transpose four 4x4 blocks + +TransposeBlockLoop: + lea rax,[rdx+r8*2] + movups xmm0,XMMWORD PTR [rdx] + movups xmm1,XMMWORD PTR [rdx+r8] + movups xmm2,XMMWORD PTR [rax] + movups xmm3,XMMWORD PTR [rax+r8] + movaps xmm4,xmm0 + unpcklps xmm4,xmm1 + unpckhps xmm0,xmm1 + movaps xmm5,xmm2 + unpcklps xmm5,xmm3 + unpckhps xmm2,xmm3 + movaps xmm1,xmm4 + unpcklpd xmm1,xmm5 + unpckhpd xmm4,xmm5 + movaps xmm3,xmm0 + unpcklpd xmm3,xmm2 + unpckhpd xmm0,xmm2 + movaps XMMWORD PTR [rcx+16*4*0],xmm1 + movaps XMMWORD PTR [rcx+16*4*1],xmm4 + movaps XMMWORD PTR [rcx+16*4*2],xmm3 + movaps XMMWORD PTR [rcx+16*4*3],xmm0 + add rcx,4*4 + lea rdx,[rax+r8*2] + dec r9d + jnz TransposeBlockLoop + ret + + LEAF_END MlasSgemmTransposePackB16x4Sse, _TEXT + +; +; Transpose8x4BlockAvx +; +; 4 columns of 8 rows from the source matrix are transposed to 8 columns of 4 +; rows in the destination packed buffer. +; +; This implementation uses AVX instructions. +; +; Arguments: +; +; StoreOffset - Supplies the relative byte offset into the destination packed +; buffer. +; +; Implicit Arguments: +; +; rcx - Supplies the address of the destination packed buffer. +; +; rdx - Supplies the address of the source matrix. +; +; r8 - Supplies the number of elements per row of the source matrix. +; + +TransposePackB8x4BlockAvx MACRO StoreOffset + +; +; Load 4 columns from 8 rows of the source matrix into the lower and upper +; halves of 4 YMM registers. +; + + lea rax,[rdx+r8*2] + vmovups xmm0,XMMWORD PTR [rdx] + vmovups xmm1,XMMWORD PTR [rdx+r8] + lea rdx,[rax+r8*2] + vmovups xmm2,XMMWORD PTR [rax] + vmovups xmm3,XMMWORD PTR [rax+r8] + lea rax,[rdx+r8*2] + vinsertf128 ymm0,ymm0,XMMWORD PTR [rdx],1 + vinsertf128 ymm1,ymm1,XMMWORD PTR [rdx+r8],1 + vinsertf128 ymm2,ymm2,XMMWORD PTR [rax],1 + vinsertf128 ymm3,ymm3,XMMWORD PTR [rax+r8],1 + +; +; Transpose the lower and upper halves of the 4 YMM registers as two 4x4 +; matrices and store the output to the destination packed buffer. +; + + vunpcklps ymm4,ymm0,ymm1 + vunpckhps ymm5,ymm0,ymm1 + vunpcklps ymm0,ymm2,ymm3 + vunpckhps ymm1,ymm2,ymm3 + vunpcklpd ymm2,ymm4,ymm0 + vunpckhpd ymm3,ymm4,ymm0 + vmovaps YMMWORD PTR [rcx+16*4*0+StoreOffset],ymm2 + vmovaps YMMWORD PTR [rcx+16*4*1+StoreOffset],ymm3 + vunpcklpd ymm0,ymm5,ymm1 + vunpckhpd ymm4,ymm5,ymm1 + vmovaps YMMWORD PTR [rcx+16*4*2+StoreOffset],ymm0 + vmovaps YMMWORD PTR [rcx+16*4*3+StoreOffset],ymm4 + + ENDM + +;++ +; +; Routine Description: +; +; This routine transposes elements from the source matrix to the destination +; packed buffer. +; +; 4 columns of 16 rows from the source matrix are transposed to 16 columns of 4 +; rows in the destination packed buffer. +; +; This implementation uses AVX instructions. +; +; Arguments: +; +; D (rcx) - Supplies the address of the destination packed buffer. +; +; B (rdx) - Supplies the address of the source matrix. +; +; ldb (r8d) - Supplies the number of elements per row of the source matrix. +; +; Return Value: +; +; None. +; +;-- + + LEAF_ENTRY MlasSgemmTransposePackB16x4Avx, _TEXT + + shl r8,2 ; convert ldb to bytes + TransposePackB8x4BlockAvx 0*4 + lea rdx,[rax+r8*2] + TransposePackB8x4BlockAvx 8*4 + vzeroupper + ret + + LEAF_END MlasSgemmTransposePackB16x4Avx, _TEXT + + END diff --git a/onnxruntime/core/mlas/lib/arm/sgemmc.cpp b/onnxruntime/core/mlas/lib/arm/sgemmc.cpp new file mode 100644 index 0000000000000..d0697adf0015e --- /dev/null +++ b/onnxruntime/core/mlas/lib/arm/sgemmc.cpp @@ -0,0 +1,531 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + sgemmc.cpp + +Abstract: + + This module implements the kernels for the single precision matrix/matrix + multiply operation (SGEMM). + +--*/ + +#include "mlasi.h" + +template +size_t +MlasSgemmKernel( + const float* A, + const float* B, + float* C, + size_t CountK, + size_t CountN, + size_t lda, + size_t ldc, + float alpha + ) +/*++ + +Routine Description: + + This routine is an inner kernel to compute matrix multiplication for a + set of rows. + +Arguments: + + A - Supplies the address of matrix A. + + B - Supplies the address of matrix B. The matrix data has been packed using + MlasSgemmCopyPackB or MlasSgemmTransposePackB. + + C - Supplies the address of matrix C. + + CountK - Supplies the number of columns from matrix A and the number of rows + from matrix B to iterate over. + + CountN - Supplies the number of columns from matrix B and matrix C to + iterate over. + + lda - Supplies the first dimension of matrix A. + + ldc - Supplies the first dimension of matrix C. + + alpha - Supplies the scaler multiplier (see SGEMM definition). + +Return Value: + + Returns the number of rows handled. + +--*/ +{ + float32x4_t Row0Block0; + float32x4_t Row0Block1; + float32x4_t Row0Block2; + float32x4_t Row0Block3; + + float32x4_t Row1Block0; + float32x4_t Row1Block1; + float32x4_t Row1Block2; + float32x4_t Row1Block3; + +#if defined(_WIN32) + + if (!ProcessTwoRows) { + UNREFERENCED_PARAMETER(lda); + UNREFERENCED_PARAMETER(ldc); + } + +#endif + + do { + + float32x4_t BElements0; + float32x4_t BElements1; + float32x4_t BElements2; + float32x4_t BElements3; + + float32x2_t Row0AElements; + float32x2_t Row1AElements; + + // + // Clear the block accumulators. + // + + Row0Block0 = vdupq_n_f32(0.0f); + Row0Block1 = vdupq_n_f32(0.0f); + Row0Block2 = vdupq_n_f32(0.0f); + Row0Block3 = vdupq_n_f32(0.0f); + + if (ProcessTwoRows) { + Row1Block0 = vdupq_n_f32(0.0f); + Row1Block1 = vdupq_n_f32(0.0f); + Row1Block2 = vdupq_n_f32(0.0f); + Row1Block3 = vdupq_n_f32(0.0f); + } + + // + // Compute the 16x1 or 16x2 output block. + // + + const float* a = A; + size_t k = CountK; + + while (k >= 2) { + + Row0AElements = vld1_f32(a); + + if (ProcessTwoRows) { + Row1AElements = vld1_f32(a + lda); + } + + BElements0 = vld1q_f32(B + 0); + BElements1 = vld1q_f32(B + 4); + BElements2 = vld1q_f32(B + 8); + BElements3 = vld1q_f32(B + 12); + + Row0Block0 = vmlaq_lane_f32(Row0Block0, BElements0, Row0AElements, 0); + Row0Block1 = vmlaq_lane_f32(Row0Block1, BElements1, Row0AElements, 0); + Row0Block2 = vmlaq_lane_f32(Row0Block2, BElements2, Row0AElements, 0); + Row0Block3 = vmlaq_lane_f32(Row0Block3, BElements3, Row0AElements, 0); + + if (ProcessTwoRows) { + Row1Block0 = vmlaq_lane_f32(Row1Block0, BElements0, Row1AElements, 0); + Row1Block1 = vmlaq_lane_f32(Row1Block1, BElements1, Row1AElements, 0); + Row1Block2 = vmlaq_lane_f32(Row1Block2, BElements2, Row1AElements, 0); + Row1Block3 = vmlaq_lane_f32(Row1Block3, BElements3, Row1AElements, 0); + } + + BElements0 = vld1q_f32(B + 16); + BElements1 = vld1q_f32(B + 20); + BElements2 = vld1q_f32(B + 24); + BElements3 = vld1q_f32(B + 28); + + Row0Block0 = vmlaq_lane_f32(Row0Block0, BElements0, Row0AElements, 1); + Row0Block1 = vmlaq_lane_f32(Row0Block1, BElements1, Row0AElements, 1); + Row0Block2 = vmlaq_lane_f32(Row0Block2, BElements2, Row0AElements, 1); + Row0Block3 = vmlaq_lane_f32(Row0Block3, BElements3, Row0AElements, 1); + + if (ProcessTwoRows) { + Row1Block0 = vmlaq_lane_f32(Row1Block0, BElements0, Row1AElements, 1); + Row1Block1 = vmlaq_lane_f32(Row1Block1, BElements1, Row1AElements, 1); + Row1Block2 = vmlaq_lane_f32(Row1Block2, BElements2, Row1AElements, 1); + Row1Block3 = vmlaq_lane_f32(Row1Block3, BElements3, Row1AElements, 1); + } + + a += 2; + B += 32; + k -= 2; + } + + if (k > 0) { + + Row0AElements = vld1_dup_f32(a); + + if (ProcessTwoRows) { + Row1AElements = vld1_dup_f32(a + lda); + } + + BElements0 = vld1q_f32(B + 0); + BElements1 = vld1q_f32(B + 4); + BElements2 = vld1q_f32(B + 8); + BElements3 = vld1q_f32(B + 12); + + Row0Block0 = vmlaq_lane_f32(Row0Block0, BElements0, Row0AElements, 0); + Row0Block1 = vmlaq_lane_f32(Row0Block1, BElements1, Row0AElements, 0); + Row0Block2 = vmlaq_lane_f32(Row0Block2, BElements2, Row0AElements, 0); + Row0Block3 = vmlaq_lane_f32(Row0Block3, BElements3, Row0AElements, 0); + + if (ProcessTwoRows) { + Row1Block0 = vmlaq_lane_f32(Row1Block0, BElements0, Row1AElements, 0); + Row1Block1 = vmlaq_lane_f32(Row1Block1, BElements1, Row1AElements, 0); + Row1Block2 = vmlaq_lane_f32(Row1Block2, BElements2, Row1AElements, 0); + Row1Block3 = vmlaq_lane_f32(Row1Block3, BElements3, Row1AElements, 0); + } + + B += 16; + } + + // + // Multiply by the alpha value. + // + + Row0Block0 = vmulq_n_f32(Row0Block0, alpha); + Row0Block1 = vmulq_n_f32(Row0Block1, alpha); + Row0Block2 = vmulq_n_f32(Row0Block2, alpha); + Row0Block3 = vmulq_n_f32(Row0Block3, alpha); + + if (ProcessTwoRows) { + Row1Block0 = vmulq_n_f32(Row1Block0, alpha); + Row1Block1 = vmulq_n_f32(Row1Block1, alpha); + Row1Block2 = vmulq_n_f32(Row1Block2, alpha); + Row1Block3 = vmulq_n_f32(Row1Block3, alpha); + } + + if (CountN >= 16) { + + // + // Store the entire output block. + // + + if (!ZeroMode) { + Row0Block0 = vaddq_f32(Row0Block0, vld1q_f32(C)); + Row0Block1 = vaddq_f32(Row0Block1, vld1q_f32(C + 4)); + Row0Block2 = vaddq_f32(Row0Block2, vld1q_f32(C + 8)); + Row0Block3 = vaddq_f32(Row0Block3, vld1q_f32(C + 12)); + } + + vst1q_f32(C, Row0Block0); + vst1q_f32(C + 4, Row0Block1); + vst1q_f32(C + 8, Row0Block2); + vst1q_f32(C + 12, Row0Block3); + + if (ProcessTwoRows) { + + if (!ZeroMode) { + Row1Block0 = vaddq_f32(Row1Block0, vld1q_f32(C + ldc)); + Row1Block1 = vaddq_f32(Row1Block1, vld1q_f32(C + ldc + 4)); + Row1Block2 = vaddq_f32(Row1Block2, vld1q_f32(C + ldc + 8)); + Row1Block3 = vaddq_f32(Row1Block3, vld1q_f32(C + ldc + 12)); + } + + vst1q_f32(C + ldc, Row1Block0); + vst1q_f32(C + ldc + 4, Row1Block1); + vst1q_f32(C + ldc + 8, Row1Block2); + vst1q_f32(C + ldc + 12, Row1Block3); + } + + } else { + + // + // Store the partial output block. + // + + if ((CountN & 8) != 0) { + + if (!ZeroMode) { + Row0Block0 = vaddq_f32(Row0Block0, vld1q_f32(C)); + Row0Block1 = vaddq_f32(Row0Block1, vld1q_f32(C + 4)); + } + + vst1q_f32(C, Row0Block0); + vst1q_f32(C + 4, Row0Block1); + Row0Block0 = Row0Block2; + Row0Block1 = Row0Block3; + + if (ProcessTwoRows) { + + if (!ZeroMode) { + Row1Block0 = vaddq_f32(Row1Block0, vld1q_f32(C + ldc)); + Row1Block1 = vaddq_f32(Row1Block1, vld1q_f32(C + ldc + 4)); + } + + vst1q_f32(C + ldc, Row1Block0); + vst1q_f32(C + ldc + 4, Row1Block1); + Row1Block0 = Row1Block2; + Row1Block1 = Row1Block3; + } + + C += 8; + } + + if ((CountN & 4) != 0) { + + if (!ZeroMode) { + Row0Block0 = vaddq_f32(Row0Block0, vld1q_f32(C)); + } + + vst1q_f32(C, Row0Block0); + Row0Block0 = Row0Block1; + + if (ProcessTwoRows) { + + if (!ZeroMode) { + Row1Block0 = vaddq_f32(Row1Block0, vld1q_f32(C + ldc)); + } + + vst1q_f32(C + ldc, Row1Block0); + Row1Block0 = Row1Block1; + } + + C += 4; + } + + float32x2_t Row0Block0High; + float32x2_t Row0Block0Low; + + float32x2_t Row1Block0High; + float32x2_t Row1Block0Low; + + Row0Block0High = vget_high_f32(Row0Block0); + Row0Block0Low = vget_low_f32(Row0Block0); + + if (ProcessTwoRows) { + Row1Block0High = vget_high_f32(Row1Block0); + Row1Block0Low = vget_low_f32(Row1Block0); + } + + if ((CountN & 2) != 0) { + + if (!ZeroMode) { + Row0Block0Low = vadd_f32(Row0Block0Low, vld1_f32(C)); + } + + vst1_f32(C, Row0Block0Low); + Row0Block0Low = Row0Block0High; + + if (ProcessTwoRows) { + + if (!ZeroMode) { + Row1Block0Low = vadd_f32(Row1Block0Low, vld1_f32(C + ldc)); + } + + vst1_f32(C + ldc, Row1Block0Low); + Row1Block0Low = Row1Block0High; + } + + C += 2; + } + + if ((CountN & 1) != 0) { + + if (!ZeroMode) { + Row0Block0Low = vadd_f32(Row0Block0Low, vld1_dup_f32(C)); + } + + vst1_lane_f32(C, Row0Block0Low, 0); + + if (ProcessTwoRows) { + + if (!ZeroMode) { + Row1Block0Low = vadd_f32(Row1Block0Low, vld1_dup_f32(C + ldc)); + } + + vst1_lane_f32(C + ldc, Row1Block0Low, 0); + } + } + + break; + } + + C += 16; + CountN -= 16; + + } while (CountN > 0); + + return ProcessTwoRows ? 2 : 1; +} + +template +size_t +MlasSgemmKernel( + const float* A, + const float* B, + float* C, + size_t CountK, + size_t CountM, + size_t CountN, + size_t lda, + size_t ldc, + float alpha + ) +/*++ + +Routine Description: + + This routine is an inner kernel to compute matrix multiplication for a + set of rows. + +Arguments: + + A - Supplies the address of matrix A. + + B - Supplies the address of matrix B. The matrix data has been packed using + MlasSgemmCopyPackB or MlasSgemmTransposePackB. + + C - Supplies the address of matrix C. + + CountK - Supplies the number of columns from matrix A and the number of rows + from matrix B to iterate over. + + CountM - Supplies the maximum number of rows that can be processed for + matrix A and matrix C. The actual number of rows handled for this + invocation depends on the kernel implementation. + + CountN - Supplies the number of columns from matrix B and matrix C to + iterate over. + + lda - Supplies the first dimension of matrix A. + + ldc - Supplies the first dimension of matrix C. + + alpha - Supplies the scaler multiplier (see SGEMM definition). + +Return Value: + + Returns the number of rows handled. + +--*/ +{ + size_t RowsHandled; + + if (CountM >= 2) { + RowsHandled = MlasSgemmKernel(A, B, C, CountK, CountN, lda, ldc, alpha); + } else { + RowsHandled = MlasSgemmKernel(A, B, C, CountK, CountN, lda, ldc, alpha); + } + + return RowsHandled; +} + +size_t +MLASCALL +MlasSgemmKernelZero( + const float* A, + const float* B, + float* C, + size_t CountK, + size_t CountM, + size_t CountN, + size_t lda, + size_t ldc, + float alpha + ) +/*++ + +Routine Description: + + This routine is an inner kernel to compute matrix multiplication for a + set of rows. + +Arguments: + + A - Supplies the address of matrix A. + + B - Supplies the address of matrix B. The matrix data has been packed using + MlasSgemmCopyPackB or MlasSgemmTransposePackB. + + C - Supplies the address of matrix C. + + CountK - Supplies the number of columns from matrix A and the number of rows + from matrix B to iterate over. + + CountM - Supplies the maximum number of rows that can be processed for + matrix A and matrix C. The actual number of rows handled for this + invocation depends on the kernel implementation. + + CountN - Supplies the number of columns from matrix B and matrix C to + iterate over. + + lda - Supplies the first dimension of matrix A. + + ldc - Supplies the first dimension of matrix C. + + alpha - Supplies the scaler multiplier (see SGEMM definition). + +Return Value: + + Returns the number of rows handled. + +--*/ +{ + return MlasSgemmKernel(A, B, C, CountK, CountM, CountN, lda, ldc, alpha); +} + +size_t +MLASCALL +MlasSgemmKernelAdd( + const float* A, + const float* B, + float* C, + size_t CountK, + size_t CountM, + size_t CountN, + size_t lda, + size_t ldc, + float alpha + ) +/*++ + +Routine Description: + + This routine is an inner kernel to compute matrix multiplication for a + set of rows. + +Arguments: + + A - Supplies the address of matrix A. + + B - Supplies the address of matrix B. The matrix data has been packed using + MlasSgemmCopyPackB or MlasSgemmTransposePackB. + + C - Supplies the address of matrix C. + + CountK - Supplies the number of columns from matrix A and the number of rows + from matrix B to iterate over. + + CountM - Supplies the maximum number of rows that can be processed for + matrix A and matrix C. The actual number of rows handled for this + invocation depends on the kernel implementation. + + CountN - Supplies the number of columns from matrix B and matrix C to + iterate over. + + lda - Supplies the first dimension of matrix A. + + ldc - Supplies the first dimension of matrix C. + + alpha - Supplies the scaler multiplier (see SGEMM definition). + +Return Value: + + Returns the number of rows handled. + +--*/ +{ + return MlasSgemmKernel(A, B, C, CountK, CountM, CountN, lda, ldc, alpha); +} diff --git a/onnxruntime/core/mlas/lib/arm64/sgemma.asm b/onnxruntime/core/mlas/lib/arm64/sgemma.asm new file mode 100644 index 0000000000000..0b6eb11fa2d78 --- /dev/null +++ b/onnxruntime/core/mlas/lib/arm64/sgemma.asm @@ -0,0 +1,527 @@ +;++ +; +; Copyright (c) Microsoft Corporation. All rights reserved. +; +; Licensed under the MIT License. +; +; Module Name: +; +; sgemma.asm +; +; Abstract: +; +; This module implements the kernels for the single precision matrix/matrix +; multiply operation (SGEMM). +; +;-- + +#include "ksarm64.h" + + TEXTAREA + +; +; ComputeEffectiveAddress +; +; Generates the code to compute the effective address of a matrix element using +; the instruction template: +; +; add $DestReg,$BaseReg,$IndexReg lsl #2 +; +; For native ARM64, the macro generates a 64-bit address calculation. For CHPE +; targets, the macro generates a 32-bit address calculation to stay within the +; WOW64 sandbox. +; + + + MACRO + ComputeEffectiveAddress $DestReg, $BaseReg, $IndexReg + +#if defined(_CHPE_X86_ARM64_) + DCD 0x0B000800:OR:(:RCONST:$DestReg):OR:((:RCONST:$BaseReg):SHL:5):OR:((:RCONST:$IndexReg):SHL:16) +#else + DCD 0x8B000800:OR:(:RCONST:$DestReg):OR:((:RCONST:$BaseReg):SHL:5):OR:((:RCONST:$IndexReg):SHL:16) +#endif + + MEND + +; +; ClearRowAccumulators +; +; Generates the code to clear the accumulators for a single row of the output +; block. +; + + MACRO + ClearRowAccumulators $Columns, $Vec1Reg, $Vec2Reg, $Vec3Reg, $Vec4Reg + + movi $Vec1Reg..16b,#0 + movi $Vec2Reg..16b,#0 + IF $Columns > 8 + movi $Vec3Reg..16b,#0 + movi $Vec4Reg..16b,#0 + ENDIF + + MEND + +; +; ClearBlockAccumulators +; +; Generates the code to clear the accumulators for a single row of the output +; block. +; + + MACRO + ClearBlockAccumulators $Columns, $Rows + + ClearRowAccumulators $Columns, v16, v17, v18, v19 + IF $Rows >= 2 + ClearRowAccumulators $Columns, v20, v21, v22, v23 + ENDIF + IF $Rows >= 4 + ClearRowAccumulators $Columns, v24, v25, v26, v27 + ClearRowAccumulators $Columns, v28, v29, v30, v31 + ENDIF + + MEND + +; +; LoadMatrixAElementsBy4 +; LoadMatrixAElementsBy1 +; +; Generates the code to load 1 or 4 elements from matrix A. +; + + MACRO + LoadMatrixAElementsBy4 $Rows + + ldr v8,[x0],#16 + IF $Rows >= 2 + ldr v9,[x10],#16 + ENDIF + IF $Rows >= 4 + ldr v10,[x11],#16 + ldr v11,[x12],#16 + ENDIF + + MEND + + MACRO + LoadMatrixAElementsBy1 $Rows + + ldr s8,[x0],#4 + IF $Rows >= 2 + ldr s9,[x10],#4 + ENDIF + IF $Rows >= 4 + ldr s10,[x11],#4 + ldr s11,[x12],#4 + ENDIF + + MEND + +; +; MultiplyAccumulateRow +; +; Generates the code to multiply and accumulate a single row of the output +; block. +; + + MACRO + MultiplyAccumulateRow $Columns, $MatrixAReg, $Broadcast, $Vec1Reg, $Vec2Reg, $Vec3Reg, $Vec4Reg + + fmla $Vec1Reg..4s,v4.4s,$MatrixAReg..s[$Broadcast] + fmla $Vec2Reg..4s,v5.4s,$MatrixAReg..s[$Broadcast] + IF $Columns > 8 + fmla $Vec3Reg..4s,v6.4s,$MatrixAReg..s[$Broadcast] + fmla $Vec4Reg..4s,v7.4s,$MatrixAReg..s[$Broadcast] + ENDIF + + MEND + +; +; MultiplyAccumulateBlock +; +; Generates the code to multiply and accumulate into the output block. +; + + MACRO + MultiplyAccumulateBlock $Columns, $Rows, $Broadcast + + MultiplyAccumulateRow $Columns, v8, $Broadcast, v16, v17, v18, v19 + IF $Rows >= 2 + MultiplyAccumulateRow $Columns, v9, $Broadcast, v20, v21, v22, v23 + ENDIF + IF $Rows >= 4 + MultiplyAccumulateRow $Columns, v10, $Broadcast, v24, v25, v26, v27 + MultiplyAccumulateRow $Columns, v11, $Broadcast, v28, v29, v30, v31 + ENDIF + + MEND + +; +; ComputeBlockLoop +; +; Generates the code to loop over K entries of the input matrices to produce +; the output block. +; + + MACRO + ComputeBlockLoop $Mode, $Columns, $Rows + + ClearBlockAccumulators $Columns, $Rows + + IF $Rows >= 2 + ComputeEffectiveAddress x10,x0,x6 ; compute matrix A plus 1 row + ENDIF + IF $Rows >= 4 + ComputeEffectiveAddress x11,x10,x6 ; compute matrix A plus 2 rows + ComputeEffectiveAddress x12,x11,x6 ; compute matrix A plus 3 rows + ENDIF + + sub x9,x3,#4 ; decrement block count to process + tbnz x9,#63,$Mode.ProcessRemaining$Columns.x$Rows.Blocks + +$Mode.Compute$Columns.x$Rows.BlockBy4Loop + LoadMatrixAElementsBy4 $Rows + ldp v4,v5,[x1],#64*4 + IF $Columns > 8 + ldp v6,v7,[x1,#-56*4] + ENDIF + MultiplyAccumulateBlock $Columns,$Rows,0 + ldp v4,v5,[x1,#-48*4] + IF $Columns > 8 + ldp v6,v7,[x1,#-40*4] + ENDIF + MultiplyAccumulateBlock $Columns,$Rows,1 + ldp v4,v5,[x1,#-32*4] + IF $Columns > 8 + ldp v6,v7,[x1,#-24*4] + ENDIF + MultiplyAccumulateBlock $Columns,$Rows,2 + ldp v4,v5,[x1,#-16*4] + IF $Columns > 8 + ldp v6,v7,[x1,#-8*4] + ENDIF + MultiplyAccumulateBlock $Columns,$Rows,3 + sub x9,x9,#4 + tbz x9,#63,$Mode.Compute$Columns.x$Rows.BlockBy4Loop + +$Mode.ProcessRemaining$Columns.x$Rows.Blocks + add x9,x9,#4 ; correct for over-subtract above + cbz x9,$Mode.Output$Columns.x$Rows.Block + +$Mode.Compute$Columns.x$Rows.BlockBy1Loop + LoadMatrixAElementsBy1 $Rows + ldp v4,v5,[x1],#16*4 + IF $Columns > 8 + ldp v6,v7,[x1,#-8*4] + ENDIF + MultiplyAccumulateBlock $Columns,$Rows,0 + sub x9,x9,1 + cbnz x9,$Mode.Compute$Columns.x$Rows.BlockBy1Loop + +$Mode.Output$Columns.x$Rows.Block + + MEND + +; +; MultiplyAlphaRow +; +; Generates the code to multiply a single row of the output block by the alpha +; value. +; + + MACRO + MultiplyAlphaRow $Columns, $Vec1Reg, $Vec2Reg, $Vec3Reg, $Vec4Reg + + IF $Columns <= 4 + fmul $Vec1Reg..4s,$Vec1Reg..4s,v0.s[0] + ELIF $Columns <= 8 + fmul $Vec1Reg..4s,$Vec1Reg..4s,v0.s[0] + fmul $Vec2Reg..4s,$Vec2Reg..4s,v0.s[0] + ELIF $Columns <= 12 + fmul $Vec1Reg..4s,$Vec1Reg..4s,v0.s[0] + fmul $Vec2Reg..4s,$Vec2Reg..4s,v0.s[0] + fmul $Vec3Reg..4s,$Vec3Reg..4s,v0.s[0] + ELSE + fmul $Vec1Reg..4s,$Vec1Reg..4s,v0.s[0] + fmul $Vec2Reg..4s,$Vec2Reg..4s,v0.s[0] + fmul $Vec3Reg..4s,$Vec3Reg..4s,v0.s[0] + fmul $Vec4Reg..4s,$Vec4Reg..4s,v0.s[0] + ENDIF + + MEND + +; +; MultiplyAlphaBlock +; +; Generates the code to multiply the output block by the alpha value. +; + + MACRO + MultiplyAlphaBlock $Columns, $Rows + + MultiplyAlphaRow $Columns, v16, v17, v18, v19 + IF $Rows >= 2 + MultiplyAlphaRow $Columns, v20, v21, v22, v23 + ENDIF + IF $Rows >= 4 + MultiplyAlphaRow $Columns, v24, v25, v26, v27 + MultiplyAlphaRow $Columns, v28, v29, v30, v31 + ENDIF + + MEND + +; +; OutputRow1Element +; OutputRow2Element +; OutputRow4Element +; OutputRow8Element +; OutputRow16Element +; +; Generates the code to store elements to the output block. +; + + MACRO + OutputRow1Element $Mode, $AddrReg, $Vec1Reg, $Vec2Reg, $Vec3Reg, $Vec4Reg + + IF "$Mode"=="Add" + ld1 {v4.s}[0],[$AddrReg] + fmla v4.2s,$Vec1Reg..2s,v0.s[0] + st1 {v4.s}[0],[$AddrReg] ; post-increment not needed for last element + ELSE + st1 {$Vec1Reg..s}[0],[$AddrReg] ; post-increment not needed for last element + ENDIF + + MEND + + MACRO + OutputRow2Element $Mode, $AddrReg, $Vec1Reg, $Vec2Reg, $Vec3Reg, $Vec4Reg + + IF "$Mode"=="Add" + ld1 {v4.2s},[$AddrReg] + fmla v4.2s,$Vec1Reg..2s,v0.s[0] + st1 {v4.2s},[$AddrReg],#2*4 + ELSE + st1 {$Vec1Reg..2s},[$AddrReg],#2*4 + ENDIF + dup $Vec1Reg..4s,$Vec1Reg..s[2] ; shift remaining elements down + + MEND + + MACRO + OutputRow4Element $Mode, $AddrReg, $Vec1Reg, $Vec2Reg, $Vec3Reg, $Vec4Reg + + IF "$Mode"=="Add" + ld1 {v4.4s},[$AddrReg] + fmla v4.4s,$Vec1Reg..4s,v0.s[0] + st1 {v4.4s},[$AddrReg],#4*4 + ELSE + st1 {$Vec1Reg..4s},[$AddrReg],#4*4 + ENDIF + mov $Vec1Reg..16b,$Vec2Reg..16b ; shift remaining elements down + + MEND + + MACRO + OutputRow8Element $Mode, $AddrReg, $Vec1Reg, $Vec2Reg, $Vec3Reg, $Vec4Reg + + IF "$Mode"=="Add" + ldp v4,v5,[$AddrReg] + fmla v4.4s,$Vec1Reg..4s,v0.s[0] + fmla v5.4s,$Vec2Reg..4s,v0.s[0] + stp v4,v5,[$AddrReg],#8*4 + ELSE + stp $Vec1Reg.,$Vec2Reg.,[$AddrReg],#8*4 + ENDIF + mov $Vec1Reg..16b,$Vec3Reg..16b ; shift remaining elements down + mov $Vec2Reg..16b,$Vec4Reg..16b + + MEND + + MACRO + OutputRow16Element $Mode, $AddrReg, $Vec1Reg, $Vec2Reg, $Vec3Reg, $Vec4Reg + + IF "$Mode"=="Add" + ldp v4,v5,[$AddrReg] + ldp v6,v7,[$AddrReg,#8*4] + fmla v4.4s,$Vec1Reg..4s,v0.s[0] + fmla v5.4s,$Vec2Reg..4s,v0.s[0] + fmla v6.4s,$Vec3Reg..4s,v0.s[0] + fmla v7.4s,$Vec4Reg..4s,v0.s[0] + stp v4,v5,[$AddrReg],#16*4 + stp v6,v7,[$AddrReg,#-8*4] + ELSE + stp $Vec1Reg.,$Vec2Reg.,[$AddrReg],#16*4 + stp $Vec3Reg.,$Vec4Reg.,[$AddrReg,#-8*4] + ENDIF + + MEND + +; +; OutputBlock +; +; Generates the code to store the output block. +; + + MACRO + OutputBlock $Mode, $Columns, $Rows + + OutputRow$Columns.Element $Mode, x2, v16, v17, v18, v19 + IF $Rows >= 2 + OutputRow$Columns.Element $Mode, x13, v20, v21, v22, v23 + ENDIF + IF $Rows >= 4 + OutputRow$Columns.Element $Mode, x14, v24, v25, v26, v27 + OutputRow$Columns.Element $Mode, x15, v28, v29, v30, v31 + ENDIF + + MEND + +; +; ProcessRows +; +; Generates the code to process a compute and store the output block for a +; fixed number of rows. +; + + MACRO + ProcessRows $Mode, $Rows + + mov x4,#$Rows ; return number of rows handled + cmp x5,#8 + ble $Mode.ProcessRemainingCountN$Rows + +$Mode.ProcessNextColumnLoop16x$Rows + ComputeBlockLoop $Mode,16,$Rows + IF "$Mode"=="Zero" + MultiplyAlphaBlock 16,$Rows + ENDIF + sub x5,x5,#16 + tbnz x5,#63,$Mode.OutputMasked16x$Rows.Block + OutputBlock $Mode,16,$Rows + mov x0,x8 ; reload matrix A + cmp x5,#8 + bgt $Mode.ProcessNextColumnLoop16x$Rows + cbz x5,$Mode.ExitKernel + +$Mode.ProcessRemainingCountN$Rows + ComputeBlockLoop $Mode,8,$Rows + IF "$Mode"=="Zero" + MultiplyAlphaBlock 8,$Rows + ENDIF + +$Mode.OutputMasked16x$Rows.Block + tbz x5,#3,$Mode.OutputRemaining7x$Rows.Block + OutputBlock $Mode,8,$Rows + +$Mode.OutputRemaining7x$Rows.Block + tbz x5,#2,$Mode.OutputRemaining3x$Rows.Block + OutputBlock $Mode,4,$Rows + +$Mode.OutputRemaining3x$Rows.Block + tbz x5,#1,$Mode.OutputRemaining1x$Rows.Block + OutputBlock $Mode,2,$Rows + +$Mode.OutputRemaining1x$Rows.Block + tbz x5,#0,$Mode.ExitKernel + OutputBlock $Mode,1,$Rows + + MEND + + SUBT "SGEMM kernel" +;++ +; +; Routine Description: +; +; This routine is an inner kernel to compute matrix multiplication for a +; set of rows. +; +; Arguments: +; +; A (x0) - Supplies the address of matrix A. +; +; B (x1) - Supplies the address of matrix B. The matrix data has been packed +; using MlasSgemmCopyPackB or MlasSgemmTransposePackB. +; +; C (x2) - Supplies the address of matrix C. +; +; CountK (x3) - Supplies the number of columns from matrix A and the number +; of rows from matrix B to iterate over. +; +; CountM (x4) - Supplies the maximum number of rows that can be processed for +; matrix A and matrix C. The actual number of rows handled for this +; invocation depends on the kernel implementation. +; +; CountN (x5) - Supplies the number of columns from matrix B and matrix C to +; iterate over. +; +; lda (x6) - Supplies the first dimension of matrix A. +; +; ldc (x7) - Supplies the first dimension of matrix C. +; +; Alpha (s0) - Supplies the scaler multiplier (see SGEMM definition). +; +; Return Value: +; +; Returns the number of rows handled. +; +;-- + + MACRO + SgemmKernelNeonFunction $Mode + + NESTED_ENTRY MlasSgemmKernel$Mode + + PROLOG_SAVE_REG_PAIR d8,d9,#-32! + PROLOG_SAVE_REG_PAIR d10,d11,#16 + + ComputeEffectiveAddress x13,x2,x7 ; compute matrix C plus 1 row + ComputeEffectiveAddress x14,x13,x7 ; compute matrix C plus 2 rows + ComputeEffectiveAddress x15,x14,x7 ; compute matrix C plus 3 rows + mov x8,x0 ; save matrix A + +; +; Process 4 rows of the matrices. +; + + cmp x4,#4 + blt $Mode.ProcessCountMLessThan4 + ProcessRows $Mode,4 + +; +; Restore non-volatile registers and return. +; + +$Mode.ExitKernel + mov x0,x4 + EPILOG_RESTORE_REG_PAIR d10,d11,#16 + EPILOG_RESTORE_REG_PAIR d8,d9,#32! + EPILOG_RETURN + +; +; Process 2 rows of the matrices. +; + +$Mode.ProcessCountMLessThan4 + cmp x4,#2 + blt $Mode.ProcessCountMLessThan2 + ProcessRows $Mode,2 + b $Mode.ExitKernel + +; +; Process 1 row of the matrices. +; + +$Mode.ProcessCountMLessThan2 + ProcessRows $Mode,1 + b $Mode.ExitKernel + + NESTED_END + + MEND + + SgemmKernelNeonFunction Zero + SgemmKernelNeonFunction Add + + END diff --git a/onnxruntime/core/mlas/lib/bias.cpp b/onnxruntime/core/mlas/lib/bias.cpp new file mode 100644 index 0000000000000..63024d8ed7d1f --- /dev/null +++ b/onnxruntime/core/mlas/lib/bias.cpp @@ -0,0 +1,82 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + bias.cpp + +Abstract: + + This module implements the bias add operation. + +--*/ + +#include "mlasi.h" + +void +MLASCALL +MlasBiasAdd( + const float* Bias, + size_t M, + float* Output, + size_t N, + size_t ldc + ) +/*++ + +Routine Description: + + This routine adds a bias vector to the output matrix. + +Arguments: + + Bias - Supplies the bias vector. + + M - Supplies the number of elements of the bias vector and the number of + rows in the output matrix. + + Output - Supplies the output matrix. + + N - Supplies the number of columns of the output matrix. + + ldc - Supplies the number of elements per row of the output matrix. + +Return Value: + + None. + +--*/ +{ + // + // Step through each row of the output matrix. + // + + while (M-- > 0) { + + float* buffer = Output; + size_t n = N; + + float BiasValue = *Bias++; + + if (n >= 4) { + + MLAS_FLOAT32X4 BiasBroadcast = MlasBroadcastFloat32x4(BiasValue); + + do { + MlasStoreFloat32x4(buffer, MlasAddFloat32x4(BiasBroadcast, MlasLoadFloat32x4(buffer))); + buffer += 4; + n -= 4; + } while (n >= 4); + } + + while (n > 0) { + *buffer++ += BiasValue; + n -= 1; + } + + Output += ldc; + } +} diff --git a/onnxruntime/core/mlas/lib/convolve.cpp b/onnxruntime/core/mlas/lib/convolve.cpp new file mode 100644 index 0000000000000..f56ca76f2f2f2 --- /dev/null +++ b/onnxruntime/core/mlas/lib/convolve.cpp @@ -0,0 +1,1389 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + convolve.cpp + +Abstract: + + This module implements the convolution operation. + +--*/ + +#include "mlasi.h" + +// +// Define the number of working buffer elements required per thread. +// + +#define MLAS_CONV_WORKING_BUFFER_SIZE_PER_THREAD \ + (MLAS_SGEMM_STRIDEN * MLAS_SGEMM_STRIDEK) + +// +// Define the parameters to execute segments of a convolution operation on +// worker threads. +// + +struct MLAS_CONV_WORK_BLOCK { +#if defined(MLAS_USE_WIN32_THREADPOOL) + volatile LONG Counter; + const MLAS_CONV_PARAMETERS* Parameters; + const float* Input; + const float* Filter; + const float* Bias; + float* WorkingBuffer; + float* Output; +#endif + struct SEGMENT { + size_t StartN; + size_t CountN; + } Segments[MLAS_MAXIMUM_THREAD_COUNT]; + uint32_t TargetThreadCount; +}; + +void +MlasConvIm2Col( + const MLAS_CONV_PARAMETERS* Parameters, + const float* Input, + float* ColumnBuffer, + size_t k, + size_t CountK, + size_t n, + size_t CountN + ) +/*++ + +Routine Description: + + This routine converts the input image to a set of convolution patches + appropriate for use with a GEMM operation. + + This implementation supports sampling a portion of the convolution + patches. This avoids the need to allocate very large buffers to store + all of the convolution patches at once, when the underyling GEMM + implementation will already break up the operation into panels. Multiple + threads can also be used to process different portions of the image. + +Arguments: + + Parameters - Supplies the structure that contains the convolution + parameters. + + Input - Supplies the input tensor. + + ColumnBuffer - Supplies the buffer to receive the convolution patches. + + k - Supplies the K to begin sampling the convolution patches. + + CountK - Supplies the count of K to sample for the convolution patches. + + n - Supplies the N to begin sampling the convolution patches. + + CountN - Supplies the count of N to sample for the convolution patches. + +Return Value: + + None. + +--*/ +{ + constexpr size_t HeightShapeIndex = 0; + constexpr size_t WidthShapeIndex = 1; + + const size_t OutputWidth = Parameters->OutputShape[WidthShapeIndex]; + + const size_t StrideHeight = Parameters->StrideShape[HeightShapeIndex]; + const size_t StrideWidth = Parameters->StrideShape[WidthShapeIndex]; + + const size_t nx = (n % OutputWidth); + const size_t ny = (n / OutputWidth); + + const size_t OriginInputX = nx * StrideWidth; + const size_t OriginInputY = ny * StrideHeight; + + size_t OutputCountX = OutputWidth - nx; + + const size_t InputHeight = Parameters->InputShape[HeightShapeIndex]; + const size_t InputWidth = Parameters->InputShape[WidthShapeIndex]; + const size_t InputSize = Parameters->InputSize; + + const size_t KernelHeight = Parameters->KernelShape[HeightShapeIndex]; + const size_t KernelWidth = Parameters->KernelShape[WidthShapeIndex]; + + size_t kx = (k % KernelWidth); + size_t ky = (k / KernelWidth) % KernelHeight; + + Input = Input + (k / (KernelHeight * KernelWidth)) * InputSize; + + const size_t DilationHeight = Parameters->DilationShape[HeightShapeIndex]; + const size_t DilationWidth = Parameters->DilationShape[WidthShapeIndex]; + + const size_t PaddingLeftY = Parameters->Padding[HeightShapeIndex]; + const size_t PaddingLeftX = Parameters->Padding[WidthShapeIndex]; + + for (size_t EndingK = k + CountK; k < EndingK; k++) { + + size_t CountX = OutputCountX; + size_t InputY = (ky * DilationHeight) + OriginInputY - PaddingLeftY; + const size_t RowInitialInputX = (kx * DilationWidth) - PaddingLeftX; + size_t InitialInputX = RowInitialInputX + OriginInputX; + size_t RemainingN = CountN; + + do { + + if (CountX > RemainingN) { + CountX = RemainingN; + } + + RemainingN -= CountX; + + // + // Check if the input is in the top/bottom padding region. + // + + if (InputY < InputHeight) { + + size_t InputX = InitialInputX; + const float* InputRow = &Input[InputY * InputWidth]; + + do { + + // + // Check if the input is in the left/right padding region. + // + + if (InputX >= InputWidth) { + + *ColumnBuffer++ = 0; + InputX += StrideWidth; + CountX--; + + } else if (StrideWidth == 1) { + + // + // Copy input elements to the column buffer. + // + + size_t CountCopyX = InputWidth - InputX; + + if (CountCopyX > CountX) { + CountCopyX = CountX; + } + + CountX -= CountCopyX; + + while (CountCopyX >= 4) { + MlasStoreFloat32x4(ColumnBuffer, MlasLoadFloat32x4(&InputRow[InputX])); + ColumnBuffer += 4; + InputX += 4; + CountCopyX -= 4; + } + + while (CountCopyX > 0) { + *ColumnBuffer++ = InputRow[InputX++]; + CountCopyX--; + } + + } else if (InputX + CountX * StrideWidth <= InputWidth) { + + do { + *ColumnBuffer++ = InputRow[InputX]; + InputX += StrideWidth; + } while (--CountX > 0); + + } else { + + do { + *ColumnBuffer++ = (InputX < InputWidth) ? InputRow[InputX] : 0; + InputX += StrideWidth; + } while (--CountX > 0); + } + + } while (CountX > 0); + + } else { + + // + // The entire input row is in the padding region. + // + + MLAS_FLOAT32X4 ZeroFloat32x4 = MlasZeroFloat32x4(); + + while (CountX >= 4) { + MlasStoreFloat32x4(ColumnBuffer, ZeroFloat32x4); + ColumnBuffer += 4; + CountX -= 4; + } + + while (CountX > 0) { + MlasStoreLaneFloat32x4<0>(ColumnBuffer, ZeroFloat32x4); + ColumnBuffer++; + CountX--; + } + } + + CountX = OutputWidth; + InputY += StrideHeight; + InitialInputX = RowInitialInputX; + + } while (RemainingN > 0); + + // + // Advance the kernel indices and advance to the next channel if the + // entire kernel is complete. + // + + if (++kx == KernelWidth) { + + if (++ky == KernelHeight) { + + Input += InputSize; + + ky = 0; + } + + kx = 0; + } + } +} + +void +MlasConvVol2Col( + const MLAS_CONV_PARAMETERS* Parameters, + const float* Input, + float* ColumnBuffer, + size_t k, + size_t CountK, + size_t n, + size_t CountN + ) +/*++ + +Routine Description: + + This routine converts the input volume to a set of convolution patches + appropriate for use with a GEMM operation. + + This implementation supports sampling a portion of the convolution + patches. This avoids the need to allocate very large buffers to store + all of the convolution patches at once, when the underyling GEMM + implementation will already break up the operation into panels. Multiple + threads can also be used to process different portions of the image. + +Arguments: + + Parameters - Supplies the structure that contains the convolution + parameters. + + Input - Supplies the input tensor. + + ColumnBuffer - Supplies the buffer to receive the convolution patches. + + k - Supplies the K to begin sampling the convolution patches. + + CountK - Supplies the count of K to sample for the convolution patches. + + n - Supplies the N to begin sampling the convolution patches. + + CountN - Supplies the count of N to sample for the convolution patches. + +Return Value: + + None. + +--*/ +{ + constexpr size_t DepthShapeIndex = 0; + constexpr size_t HeightShapeIndex = 1; + constexpr size_t WidthShapeIndex = 2; + + const size_t OutputHeight = Parameters->OutputShape[HeightShapeIndex]; + const size_t OutputWidth = Parameters->OutputShape[WidthShapeIndex]; + + const size_t StrideDepth = Parameters->StrideShape[DepthShapeIndex]; + const size_t StrideHeight = Parameters->StrideShape[HeightShapeIndex]; + const size_t StrideWidth = Parameters->StrideShape[WidthShapeIndex]; + + const size_t nx = (n % OutputWidth); + const size_t ny = ((n / OutputWidth) % OutputHeight); + const size_t nz = ((n / OutputWidth) / OutputHeight); + + size_t OutputCountX = OutputWidth - nx; + size_t OutputCountY = OutputHeight - ny; + + const size_t OriginInputX = nx * StrideWidth; + const size_t OriginInputY = ny * StrideHeight; + const size_t OriginInputZ = nz * StrideDepth; + + const size_t InputDepth = Parameters->InputShape[DepthShapeIndex]; + const size_t InputHeight = Parameters->InputShape[HeightShapeIndex]; + const size_t InputWidth = Parameters->InputShape[WidthShapeIndex]; + const size_t InputSize = Parameters->InputSize; + + const size_t KernelDepth = Parameters->KernelShape[DepthShapeIndex]; + const size_t KernelHeight = Parameters->KernelShape[HeightShapeIndex]; + const size_t KernelWidth = Parameters->KernelShape[WidthShapeIndex]; + + size_t kx = (k % KernelWidth); + size_t ky = (k / KernelWidth) % KernelHeight; + size_t kz = ((k / KernelWidth) / KernelHeight) % KernelDepth; + + Input = Input + (k / (KernelDepth * KernelHeight * KernelWidth)) * InputSize; + + const size_t DilationDepth = Parameters->DilationShape[DepthShapeIndex]; + const size_t DilationHeight = Parameters->DilationShape[HeightShapeIndex]; + const size_t DilationWidth = Parameters->DilationShape[WidthShapeIndex]; + + const size_t PaddingLeftZ = Parameters->Padding[DepthShapeIndex]; + const size_t PaddingLeftY = Parameters->Padding[HeightShapeIndex]; + const size_t PaddingLeftX = Parameters->Padding[WidthShapeIndex]; + + for (size_t EndingK = k + CountK; k < EndingK; k++) { + + size_t CountY = OutputCountY; + size_t CountX = OutputCountX; + size_t InputZ = (kz * DilationDepth) + OriginInputZ - PaddingLeftZ; + const size_t RowInitialInputY = (ky * DilationHeight) - PaddingLeftY; + size_t InputY = RowInitialInputY + OriginInputY; + const size_t RowInitialInputX = (kx * DilationWidth) - PaddingLeftX; + size_t InitialInputX = RowInitialInputX + OriginInputX; + size_t RemainingN = CountN; + + do { + + if (CountX > RemainingN) { + CountX = RemainingN; + } + + RemainingN -= CountX; + + // + // Check if the input is in the top/bottom or front/back padding region. + // + + if (InputY < InputHeight && InputZ < InputDepth) { + + size_t InputX = InitialInputX; + const float* InputRow = &Input[InputZ * (InputHeight * InputWidth) + InputY * InputWidth]; + + do { + + // + // Check if the input is in the left/right padding region. + // + + if (InputX >= InputWidth) { + + *ColumnBuffer++ = 0; + InputX += StrideWidth; + CountX--; + + } else if (StrideWidth == 1) { + + // + // Copy input elements to the column buffer. + // + + size_t CountCopyX = InputWidth - InputX; + + if (CountCopyX > CountX) { + CountCopyX = CountX; + } + + CountX -= CountCopyX; + + while (CountCopyX >= 4) { + MlasStoreFloat32x4(ColumnBuffer, MlasLoadFloat32x4(&InputRow[InputX])); + ColumnBuffer += 4; + InputX += 4; + CountCopyX -= 4; + } + + while (CountCopyX > 0) { + *ColumnBuffer++ = InputRow[InputX++]; + CountCopyX--; + } + + } else if (InputX + CountX * StrideWidth <= InputWidth) { + + do { + *ColumnBuffer++ = InputRow[InputX]; + InputX += StrideWidth; + } while (--CountX > 0); + + } else { + + do { + *ColumnBuffer++ = (InputX < InputWidth) ? InputRow[InputX] : 0; + InputX += StrideWidth; + } while (--CountX > 0); + } + + } while (CountX > 0); + + } else { + + // + // The entire input row is in the padding region. + // + + MLAS_FLOAT32X4 ZeroFloat32x4 = MlasZeroFloat32x4(); + + while (CountX >= 4) { + MlasStoreFloat32x4(ColumnBuffer, ZeroFloat32x4); + ColumnBuffer += 4; + CountX -= 4; + } + + while (CountX > 0) { + MlasStoreLaneFloat32x4<0>(ColumnBuffer, ZeroFloat32x4); + ColumnBuffer++; + CountX--; + } + } + + CountX = OutputWidth; + InputY += StrideHeight; + InitialInputX = RowInitialInputX; + + if (--CountY == 0) { + + InputY = RowInitialInputY; + InputZ += StrideDepth; + + CountY = OutputHeight; + } + + } while (RemainingN > 0); + + // + // Advance the kernel indices and advance to the next channel if the + // entire kernel is complete. + // + + if (++kx == KernelWidth) { + + if (++ky == KernelHeight) { + + if (++kz == KernelDepth) { + + Input += InputSize; + + kz = 0; + } + + ky = 0; + } + + kx = 0; + } + } +} + +void +MlasConvOperation( + const MLAS_CONV_PARAMETERS* Parameters, + const float* Input, + const float* Filter, + const float* Bias, + float* ColumnBuffer, + float* Output, + size_t SegmentStartN, + size_t SegmentCountN + ) +/*++ + +Routine Description: + + This routine implements the convolution operation. + +Arguments: + + Parameters - Supplies the structure that contains the convolution + parameters. + + Input - Supplies the input tensor. + + Filter - Supplies the filter tensor. + + Bias - Optionally supplies the bias vector. + + ColumnBuffer - Supplies the thread local slice of the working buffer. + + Output - Supplies the output tensor. + + SegmentStartN - Supplies the N to begin sampling the convolution patches. + + SegmentCountN - Supplies the count of N to sample for the convolution + patches. + +Return Value: + + None. + +--*/ +{ + const size_t FilterCount = Parameters->FilterCount; + const size_t OutputSize = Parameters->OutputSize; + const size_t K = Parameters->K; + + // + // Compute the strides to step through slices of the local segment. + // + // See MlasSgemmOperation. + // + + uint32_t StrideN = MLAS_SGEMM_STRIDEN; + uint32_t StrideK = MLAS_SGEMM_STRIDEK; + + if (SegmentCountN >= K) { + + while (StrideK / 2 >= K) { + StrideN *= 2; + StrideK /= 2; + } + + } else { + + while (StrideN > 16 && StrideN / 2 >= SegmentCountN) { + StrideK *= 2; + StrideN /= 2; + } + } + + // + // Step through each slice of the input tensor along the N dimension. + // + + size_t CountN; + + for (size_t n = 0; n < SegmentCountN; n += CountN) { + + CountN = SegmentCountN - n; + + if (CountN > StrideN) { + CountN = StrideN; + } + + // + // Step through each slice of the input tensor along the K dimension. + // + + size_t CountK; + float beta = 0.0f; + float* SegmentOutput = Output + SegmentStartN + n; + + for (size_t k = 0; k < K; k += CountK) { + + CountK = K - k; + + if (CountK > StrideK) { + CountK = StrideK; + } + + if (Parameters->Dimensions == 2) { + MlasConvIm2Col(Parameters, Input, ColumnBuffer, k, CountK, + SegmentStartN + n, CountN); + } else { + MlasConvVol2Col(Parameters, Input, ColumnBuffer, k, CountK, + SegmentStartN + n, CountN); + } + + MlasSgemmOperation(CblasNoTrans, CblasNoTrans, FilterCount, CountN, + CountK, 1.0f, Filter + k, K, ColumnBuffer, CountN, beta, + SegmentOutput, OutputSize); + + beta = 1.0f; + } + + // + // Add the optional bias vector. + // + + if (Bias != nullptr) { + MlasBiasAdd(Bias, FilterCount, SegmentOutput, CountN, OutputSize); + } + } +} + +#if defined(MLAS_USE_WIN32_THREADPOOL) + +void +CALLBACK +MlasConvWorkCallback( + PTP_CALLBACK_INSTANCE Instance, + void* Context, + PTP_WORK WorkObject + ) +/*++ + +Routine Description: + + This routine is invoked from a worker thread to execute a segment of a + convolution operation. + +Arguments: + + Instance - Supplies the callback instance object. + + Context - Supplies the pointer to the parameters for the SGEMM operation. + + WorkObject - Supplies the threadpool work object. + +Return Value: + + None. + +--*/ +{ + MLAS_UNREFERENCED_PARAMETER(Instance); + MLAS_UNREFERENCED_PARAMETER(WorkObject); + + MLAS_CONV_WORK_BLOCK* WorkBlock = (MLAS_CONV_WORK_BLOCK*)Context; + + LONG Index = InterlockedIncrement(&WorkBlock->Counter) - 1; + + MLAS_CONV_WORK_BLOCK::SEGMENT* Segment = &WorkBlock->Segments[Index]; + + float* ColumnBuffer = + WorkBlock->WorkingBuffer + Index * MLAS_CONV_WORKING_BUFFER_SIZE_PER_THREAD; + + MlasConvOperation(WorkBlock->Parameters, WorkBlock->Input, WorkBlock->Filter, + WorkBlock->Bias, ColumnBuffer, WorkBlock->Output, Segment->StartN, + Segment->CountN); +} + +void +CALLBACK +MlasConvGemmDirectWorkCallback( + PTP_CALLBACK_INSTANCE Instance, + void* Context, + PTP_WORK WorkObject + ) +/*++ + +Routine Description: + + This routine is invoked from a worker thread to execute a segment of a + convolution operation. + +Arguments: + + Instance - Supplies the callback instance object. + + Context - Supplies the pointer to the parameters for the SGEMM operation. + + WorkObject - Supplies the threadpool work object. + +Return Value: + + None. + +--*/ +{ + MLAS_UNREFERENCED_PARAMETER(Instance); + MLAS_UNREFERENCED_PARAMETER(WorkObject); + + MLAS_CONV_WORK_BLOCK* WorkBlock = (MLAS_CONV_WORK_BLOCK*)Context; + + LONG Index = InterlockedIncrement(&WorkBlock->Counter) - 1; + + const MLAS_CONV_PARAMETERS* Parameters = WorkBlock->Parameters; + + // + // Compute the range of indices to use for this thread. + // + + const size_t GroupCount = Parameters->GroupCount; + const size_t BatchGroupCount = Parameters->BatchCount * GroupCount; + + const size_t TargetThreadCount = WorkBlock->TargetThreadCount; + + const size_t BatchGroupCountPerThread = BatchGroupCount / TargetThreadCount; + const size_t BatchGroupCountExtra = BatchGroupCount % TargetThreadCount; + + size_t BatchGroupStart; + size_t BatchGroupEnd; + + if (uint32_t(Index) < BatchGroupCountExtra) { + BatchGroupStart = (BatchGroupCountPerThread + 1) * Index; + BatchGroupEnd = BatchGroupStart + BatchGroupCountPerThread + 1; + } else { + BatchGroupStart = BatchGroupCountPerThread * Index + BatchGroupCountExtra; + BatchGroupEnd = BatchGroupStart + BatchGroupCountPerThread; + } + + // + // Iterate over the batch and groups allocated to this thread. + // + + const size_t FilterCount = Parameters->FilterCount; + const size_t OutputSize = Parameters->OutputSize; + const size_t K = Parameters->K; + + const size_t InputGroupSize = Parameters->InputChannels * Parameters->InputSize; + const size_t OutputGroupSize = FilterCount * OutputSize; + const size_t FilterGroupSize = FilterCount * K; + + for (size_t bg = BatchGroupStart; bg < BatchGroupEnd; bg++) { + + size_t group = bg % GroupCount; + + const float* input = WorkBlock->Input + bg * InputGroupSize; + const float* filter = WorkBlock->Filter + group * FilterGroupSize; + float* output = WorkBlock->Output + bg * OutputGroupSize; + + // + // Invoke the non-threaded GEMM directly with the input tensor. + // + + MlasSgemmOperation(CblasNoTrans, Parameters->u.GemmDirect.TransB, FilterCount, + OutputSize, K, 1.0f, filter, K, input, Parameters->u.GemmDirect.ldb, 0.0f, + output, OutputSize); + + // + // Add the optional bias vector. + // + + if (WorkBlock->Bias != nullptr) { + MlasBiasAdd(WorkBlock->Bias + group * FilterCount, FilterCount, output, OutputSize, OutputSize); + } + } +} + +#endif + +inline +bool +MlasConvTryMultithread( + const MLAS_CONV_PARAMETERS* Parameters, + const float* Input, + const float* Filter, + const float* Bias, + float* WorkingBuffer, + float* Output + ) +/*++ + +Routine Description: + + This routine attempts to launch a convolution operation across multiple + threads. + +Arguments: + + Parameters - Supplies the structure that contains the convolution + parameters. + + Input - Supplies the input tensor. + + Filter - Supplies the filter tensor. + + Bias - Optionally supplies the bias vector. + + WorkingBuffer - Supplies a working buffer sized to the number of elements + returned by MlasConvPrepare. + + Output - Supplies the output tensor. + +Return Value: + + Returns true if the operation was completed across multiple threads, else + false if the operation should fall back to a single thread. + +--*/ +{ + +#if defined(MLAS_USE_WIN32_THREADPOOL) || defined(MLAS_USE_OPENMP) + + MLAS_CONV_WORK_BLOCK WorkBlock; + + const size_t OutputSize = Parameters->OutputSize; + const size_t ThreadStrideN = Parameters->u.ExpandThenGemmSegmented.ThreadStrideN; + + if (ThreadStrideN >= OutputSize) { + return false; + } + +#if defined(MLAS_USE_WIN32_THREADPOOL) + + // + // Create an object to submit work to the threadpool. + // + + PTP_WORK WorkObject = CreateThreadpoolWork(MlasConvWorkCallback, &WorkBlock, nullptr); + + if (WorkObject == nullptr) { + return false; + } + + // + // Initialize the common fields of the work block. + // + + WorkBlock.Counter = 0; + WorkBlock.Parameters = Parameters; + WorkBlock.Input = Input; + WorkBlock.Filter = Filter; + WorkBlock.Bias = Bias; + WorkBlock.WorkingBuffer = WorkingBuffer; + WorkBlock.Output = Output; + +#endif + + // + // Segment the operation across multiple threads. + // + + uint32_t Index = 0; + size_t SegmentCountN; + + for (size_t SegmentStartN = 0; SegmentStartN < OutputSize; SegmentStartN += SegmentCountN) { + + SegmentCountN = OutputSize - SegmentStartN; + + if (SegmentCountN > ThreadStrideN) { + SegmentCountN = ThreadStrideN; + } + + WorkBlock.Segments[Index].StartN = SegmentStartN; + WorkBlock.Segments[Index].CountN = SegmentCountN; + +#if defined(MLAS_USE_WIN32_THREADPOOL) + + // + // Execute one of the segments on a worker thread. + // + + if (Index > 0) { + SubmitThreadpoolWork(WorkObject); + } + +#endif + + Index++; + } + +#if defined(MLAS_USE_OPENMP) + + #pragma omp parallel for + for (int32_t tid = 0; tid < int32_t(Index); tid++) { + + MLAS_CONV_WORK_BLOCK::SEGMENT* Segment = &WorkBlock.Segments[tid]; + + float* ColumnBuffer = + WorkingBuffer + tid * MLAS_CONV_WORKING_BUFFER_SIZE_PER_THREAD; + + MlasConvOperation(Parameters, Input, Filter, Bias, ColumnBuffer, Output, + Segment->StartN, Segment->CountN); + } + +#elif defined(MLAS_USE_WIN32_THREADPOOL) + + // + // Execute the remaining segment on this thread. + // + + MlasConvWorkCallback(nullptr, &WorkBlock, WorkObject); + + // + // Wait for the worker threads to complete. + // + + WaitForThreadpoolWorkCallbacks(WorkObject, FALSE); + CloseThreadpoolWork(WorkObject); + +#endif + + return true; + +#else + + // + // No threading implementation is available. + // + + MLAS_UNREFERENCED_PARAMETER(Parameters); + MLAS_UNREFERENCED_PARAMETER(Input); + MLAS_UNREFERENCED_PARAMETER(Filter); + MLAS_UNREFERENCED_PARAMETER(Bias); + MLAS_UNREFERENCED_PARAMETER(WorkingBuffer); + MLAS_UNREFERENCED_PARAMETER(Output); + + return false; + +#endif + +} + +void +MLASCALL +MlasConv( + const MLAS_CONV_PARAMETERS* Parameters, + const float* Input, + const float* Filter, + const float* Bias, + float* WorkingBuffer, + float* Output + ) +/*++ + +Routine Description: + + This routine implements the convolution operation. + +Arguments: + + Parameters - Supplies the structure that contains the convolution + parameters. + + Input - Supplies the input tensor. + + Filter - Supplies the filter tensor. + + Bias - Optionally supplies the bias vector. + + WorkingBuffer - Supplies a working buffer sized to the number of elements + returned by MlasConvPrepare. + + Output - Supplies the output tensor. + +Return Value: + + None. + +--*/ +{ + const size_t FilterCount = Parameters->FilterCount; + const size_t OutputSize = Parameters->OutputSize; + const size_t K = Parameters->K; + + const size_t InputGroupSize = Parameters->InputChannels * Parameters->InputSize; + const size_t OutputGroupSize = FilterCount * OutputSize; + const size_t FilterGroupSize = FilterCount * K; + + const size_t BatchCount = Parameters->BatchCount; + const size_t GroupCount = Parameters->GroupCount; + + const MLAS_CONV_ALGORITHM Algorithm = Parameters->Algorithm; + +#if defined(MLAS_USE_WIN32_THREADPOOL) || defined(MLAS_USE_OPENMP) + + // + // Schedule batches of GEMMs across multiple threads. + // + + if (Algorithm == MlasConvAlgorithmGemmDirect && ((BatchCount > 1) || (GroupCount > 1))) { + + const size_t BatchGroupCount = BatchCount * GroupCount; + +#if defined(MLAS_USE_WIN32_THREADPOOL) + + uint32_t TargetThreadCount = MlasPlatform.GetMaximumThreadCount(); + + if (TargetThreadCount >= BatchGroupCount) { + TargetThreadCount = uint32_t(BatchGroupCount); + } + + if (TargetThreadCount > 1) { + + MLAS_CONV_WORK_BLOCK WorkBlock; + + PTP_WORK WorkObject = CreateThreadpoolWork(MlasConvGemmDirectWorkCallback, &WorkBlock, nullptr); + + if (WorkObject != nullptr) { + + WorkBlock.Counter = 0; + WorkBlock.Parameters = Parameters; + WorkBlock.Input = Input; + WorkBlock.Filter = Filter; + WorkBlock.Bias = Bias; + WorkBlock.WorkingBuffer = nullptr; + WorkBlock.Output = Output; + WorkBlock.TargetThreadCount = TargetThreadCount; + + for (uint32_t Index = 1; Index < TargetThreadCount; Index++) { + SubmitThreadpoolWork(WorkObject); + } + + MlasConvGemmDirectWorkCallback(nullptr, &WorkBlock, WorkObject); + + WaitForThreadpoolWorkCallbacks(WorkObject, FALSE); + CloseThreadpoolWork(WorkObject); + + return; + } + } + +#else + + #pragma omp parallel for + for (int64_t bg = 0; bg < int64_t(BatchGroupCount); bg++) { + + size_t group = size_t(bg % GroupCount); + + const float* input = Input + bg * InputGroupSize; + const float* filter = Filter + group * FilterGroupSize; + float* output = Output + bg * OutputGroupSize; + + // + // Invoke the non-threaded GEMM directly with the input tensor. + // + + MlasSgemmOperation(CblasNoTrans, Parameters->u.GemmDirect.TransB, FilterCount, + OutputSize, K, 1.0f, filter, K, input, Parameters->u.GemmDirect.ldb, 0.0f, + output, OutputSize); + + // + // Add the optional bias vector. + // + + if (Bias != nullptr) { + MlasBiasAdd(Bias + group * FilterCount, FilterCount, output, OutputSize, OutputSize); + } + } + + return; + +#endif + + } + +#endif + + // + // Iterate over each batch and group. + // + + for (size_t batch = 0; batch < BatchCount; batch++) { + + const float* filter = Filter; + const float* bias = Bias; + + for (size_t group = 0; group < GroupCount; group++) { + + // + // Dispatch the convolution. + // + + switch (Algorithm) { + + case MlasConvAlgorithmGemmDirect: + { + // + // Invoke the threaded GEMM directly with the input tensor. + // + + MlasSgemm(CblasNoTrans, Parameters->u.GemmDirect.TransB, FilterCount, + OutputSize, K, 1.0f, filter, K, Input, Parameters->u.GemmDirect.ldb, 0.0f, + Output, OutputSize); + + // + // Add the optional bias vector. + // + + if (bias != nullptr) { + MlasBiasAdd(bias, FilterCount, Output, OutputSize, OutputSize); + } + + break; + } + + case MlasConvAlgorithmExpandThenGemm: + { + // + // Expand the input tensor to the working buffer and then invoke the + // threaded GEMM. + // + + if (Parameters->Dimensions == 2) { + MlasConvIm2Col(Parameters, Input, WorkingBuffer, 0, K, 0, OutputSize); + } else { + MlasConvVol2Col(Parameters, Input, WorkingBuffer, 0, K, 0, OutputSize); + } + + MlasSgemm(CblasNoTrans, CblasNoTrans, FilterCount, OutputSize, K, 1.0f, filter, + K, WorkingBuffer, OutputSize, 0.0f, Output, OutputSize); + + // + // Add the optional bias vector. + // + + if (bias != nullptr) { + MlasBiasAdd(bias, FilterCount, Output, OutputSize, OutputSize); + } + + break; + } + + case MlasConvAlgorithmExpandThenGemmSegmented: + { + // + // Attempt to launch the convolution across multiple threads or fall + // back to a single thread. + // + + if (!MlasConvTryMultithread(Parameters, Input, filter, bias, WorkingBuffer, + Output)) { + MlasConvOperation(Parameters, Input, filter, bias, WorkingBuffer, + Output, 0, OutputSize); + } + + break; + } + } + + // + // Advance the buffer pointers. + // + + if (bias != nullptr) { + bias += FilterCount; + } + + filter += FilterGroupSize; + Input += InputGroupSize; + Output += OutputGroupSize; + } + } +} + +bool +MLASCALL +MlasConvPrepare( + MLAS_CONV_PARAMETERS* Parameters, + size_t Dimensions, + size_t BatchCount, + size_t GroupCount, + size_t InputChannels, + const int64_t* InputShape, + const int64_t* KernelShape, + const int64_t* DilationShape, + const int64_t* Padding, + const int64_t* StrideShape, + size_t FilterCount, + size_t* WorkingBufferSize + ) +/*++ + +Routine Description: + + This routine prepares for a convolution operation by computing required + parameters including the required working buffer size for intermediate + results. + +Arguments: + + Parameters - Supplies the structure that stores the provided and computed + parameters for the convolution operation. + + Dimensions - Supplies the number of dimensions (must be 2 or 3). + + BatchCount - Supplies the number of batches to the processed. + + GroupCount - Supplies the number of channel groups. + + InputChannels - Supplies the number of input channels per group. + + InputShape - Supplies the shape of the input tensor. + + KernelShape - Supplies the shape of the kernel transform. + + DilationShape - Supplies the shape of the dilation. + + Padding - Supplies the number of zero padding elements at the edge of the + input tensor. + + StrideShape - Supplies the shape of the stride. + + FilterCount - Supplies the number of rows of the filter matrix per group. + + WorkingBufferSize - Receives the number of elements to allocate for the + working buffer for intermediate results. + +Return Value: + + Returns true if implementation can support this operation. + +--*/ +{ + // + // Support only 2D or 3D convolutions. + // + + if (Dimensions != 2 && Dimensions != 3) { + return false; + } + + // + // Build the convolution parameters. + // + + Parameters->Dimensions = Dimensions; + Parameters->BatchCount = BatchCount; + Parameters->GroupCount = GroupCount; + Parameters->InputChannels = InputChannels; + Parameters->FilterCount = FilterCount; + + size_t InputSize = 1; + size_t OutputSize = 1; + size_t K = InputChannels; + + bool AllStridesAreOne = true; + bool AllDilationsAreOne = true; + bool AllPaddingIsZero = true; + + for (size_t dim = 0; dim < Dimensions; dim++) { + + Parameters->InputShape[dim] = size_t(InputShape[dim]); + Parameters->KernelShape[dim] = size_t(KernelShape[dim]); + Parameters->DilationShape[dim] = size_t(DilationShape[dim]); + Parameters->Padding[dim] = size_t(Padding[dim]); + Parameters->Padding[dim + Dimensions] = size_t(Padding[dim + Dimensions]); + Parameters->StrideShape[dim] = size_t(StrideShape[dim]); + + // + // Compute the output shape. + // + + int64_t OutputShape = (InputShape[dim] + Padding[dim] + Padding[dim + Dimensions] - + (DilationShape[dim] * (KernelShape[dim] - 1) + 1)) / StrideShape[dim] + 1; + + if (OutputShape <= 0) { + return false; + } + + Parameters->OutputShape[dim] = size_t(OutputShape); + + InputSize *= Parameters->InputShape[dim]; + OutputSize *= Parameters->OutputShape[dim]; + K *= Parameters->KernelShape[dim]; + + AllStridesAreOne &= (Parameters->StrideShape[dim] == 1); + AllDilationsAreOne &= (Parameters->DilationShape[dim] == 1); + AllPaddingIsZero &= (Parameters->Padding[dim] == 0 && Parameters->Padding[dim + Dimensions] == 0); + } + + Parameters->InputSize = InputSize; + Parameters->OutputSize = OutputSize; + Parameters->K = K; + + // + // Evaluate how the convolution will be performed. + // + + *WorkingBufferSize = 0; + + if (AllStridesAreOne && AllPaddingIsZero) { + + // + // Detect a pointwise convolution. + // + + if (K == InputChannels) { + + Parameters->Algorithm = MlasConvAlgorithmGemmDirect; + Parameters->u.GemmDirect.TransB = CblasNoTrans; + Parameters->u.GemmDirect.ldb = OutputSize; + + return true; + } + + if (Dimensions == 2 && AllDilationsAreOne && InputChannels == 1) { + + // + // Detect convolutions where the kernel is using the entire input + // width or height. + // + + if (Parameters->KernelShape[1] == Parameters->InputShape[1]) { + + Parameters->Algorithm = MlasConvAlgorithmGemmDirect; + Parameters->u.GemmDirect.TransB = CblasTrans; + Parameters->u.GemmDirect.ldb = Parameters->InputShape[1]; + + return true; + } + + if (Parameters->KernelShape[0] == Parameters->InputShape[0] && + Parameters->KernelShape[1] == 1) { + + Parameters->Algorithm = MlasConvAlgorithmGemmDirect; + Parameters->u.GemmDirect.TransB = CblasNoTrans; + Parameters->u.GemmDirect.ldb = Parameters->InputShape[1]; + + return true; + } + } + } + + if (FilterCount > OutputSize) { + + // + // The filter count is larger than the output dimensions, so perform the + // full matrix expansion and then invoke the threaded GEMM. + // + + Parameters->Algorithm = MlasConvAlgorithmExpandThenGemm; + + *WorkingBufferSize = OutputSize * K; + + } else { + + // + // Segment the operation across multiple threads by slicing the N + // dimension (see MlasSgemmTryMultithread). + // + // Compute the number of target threads given the complexity of the + // convolution operation. Small requests should run using the single + // threaded path. + // + + uint32_t TargetThreadCount; + double Complexity = double(FilterCount) * double(OutputSize) * double(K); + + if (Complexity < double(MLAS_SGEMM_THREAD_COMPLEXITY * MLAS_MAXIMUM_THREAD_COUNT)) { + TargetThreadCount = uint32_t(Complexity / double(MLAS_SGEMM_THREAD_COMPLEXITY)) + 1; + } else { + TargetThreadCount = MLAS_MAXIMUM_THREAD_COUNT; + } + + uint32_t MaximumThreadCount = MlasPlatform.GetMaximumThreadCount(); + + if (TargetThreadCount >= MaximumThreadCount) { + TargetThreadCount = MaximumThreadCount; + } + + // + // Compute the thread stride for slicing the N dimension. + // + + size_t StrideN = OutputSize / TargetThreadCount; + + if ((StrideN * TargetThreadCount) != OutputSize) { + StrideN++; + } + + if (TargetThreadCount > 1) { + + StrideN = (StrideN + MLAS_SGEMM_STRIDEN_THREAD_ALIGN - 1) & ~(MLAS_SGEMM_STRIDEN_THREAD_ALIGN - 1); + + if (StrideN >= OutputSize) { + TargetThreadCount = 1; + } else if (StrideN * (TargetThreadCount - 1) >= OutputSize) { + TargetThreadCount--; + } + } + + Parameters->Algorithm = MlasConvAlgorithmExpandThenGemmSegmented; + Parameters->u.ExpandThenGemmSegmented.ThreadStrideN = StrideN; + + *WorkingBufferSize = TargetThreadCount * MLAS_CONV_WORKING_BUFFER_SIZE_PER_THREAD; + } + + return true; +} diff --git a/onnxruntime/core/mlas/lib/i386/sgemma.asm b/onnxruntime/core/mlas/lib/i386/sgemma.asm new file mode 100644 index 0000000000000..b03fdbfcc9970 --- /dev/null +++ b/onnxruntime/core/mlas/lib/i386/sgemma.asm @@ -0,0 +1,845 @@ +;++ +; +; Copyright (c) Microsoft Corporation. All rights reserved. +; +; Licensed under the MIT License. +; +; Module Name: +; +; sgemma.asm +; +; Abstract: +; +; This module implements the kernels for the single precision matrix/matrix +; multiply operation (SGEMM). +; +;-- + + .686 + .xmm + + .xlist +INCLUDE callconv.inc + .list + + ASSUME DS:FLAT,ES:FLAT,SS:NOTHING,FS:NOTHING,GS:NOTHING + + EXTERN _MlasMaskMoveAvx:NEAR + +; +; Stack frame layout for the SGEMM kernels. +; + +SgemmKernelFrame STRUCT + + SavedEdi DWORD ? + SavedEsi DWORD ? + SavedEbx DWORD ? + SavedEbp DWORD ? + ReturnAddress DWORD ? + MatrixA DWORD ? + MatrixB DWORD ? + MatrixC DWORD ? + CountK DWORD ? + CountM DWORD ? + CountN DWORD ? + lda DWORD ? + ldc DWORD ? + Alpha DWORD ? + +SgemmKernelFrame ENDS + +_TEXT SEGMENT DWORD PUBLIC 'CODE' + +; +; SgemmKernelEntry +; +; Macro Description: +; +; This macro implements the common prologue code for the SGEMM kernels. +; +; Arguments: +; +; None. +; +; Return Registers: +; +; ecx - Stores the address of the matrix A data from the stack frame. +; +; edx - Stores the address of the matrix B data from the stack frame. +; +; ebp - Stores the CountN argument from the stack frame. +; +; ebx, esi, edi - Previous values stored on the stack and the registers are +; available as temporaries. +; + +SgemmKernelEntry MACRO + + push ebp + push ebx + push esi + push edi + mov edx,SgemmKernelFrame.MatrixB[esp] + mov esi,SgemmKernelFrame.MatrixC[esp] + mov ebp,SgemmKernelFrame.CountN[esp] + +cPublicFpo ((SgemmKernelFrame.ReturnAddress)/4),9 + + ENDM + +; +; SgemmKernelExit +; +; Macro Description: +; +; This macro implements the common epilogue code for the SGEMM kernels. +; +; Arguments: +; +; None. +; + +SgemmKernelExit MACRO + + pop edi + pop esi + pop ebx + pop ebp + + ENDM + +; +; ComputeBlockSseBy4 +; ComputeBlockSseBy3 +; ComputeBlockSseBy2 +; ComputeBlockSseBy1 +; +; Macro Description: +; +; This macro multiplies and accumulates for a Nx1 block (where N is 1,2,3,4) +; of the output matrix. +; +; This implementation uses SSE2 instructions. +; +; Arguments: +; +; VectorOffset - Supplies the byte offset from matrix B to fetch elements. +; +; Shuffle - Supplies the shuffle mask to extract the element from matrix A. +; +; Implicit Arguments: +; +; ebx - Supplies the length in bytes of a row from matrix A. +; +; ecx - Supplies the address into the matrix A data. +; +; edx - Supplies the address into the matrix B data. +; +; xmm2 - Supplies up to four elements loaded from matrix A. +; +; xmm4-xmm7 - Supplies the block accumulators. +; + +ComputeBlockSseBy4 MACRO VectorOffset, Shuffle + + pshufd xmm3,xmm1,Shuffle + movaps xmm0,XMMWORD PTR [edx+VectorOffset] + mulps xmm0,xmm3 + addps xmm4,xmm0 + movaps xmm0,XMMWORD PTR [edx+VectorOffset+16] + mulps xmm0,xmm3 + addps xmm5,xmm0 + movaps xmm0,XMMWORD PTR [edx+VectorOffset+32] + mulps xmm0,xmm3 + addps xmm6,xmm0 + movaps xmm0,XMMWORD PTR [edx+VectorOffset+48] + mulps xmm0,xmm3 + addps xmm7,xmm0 + + ENDM + +ComputeBlockSseBy3 MACRO VectorOffset, Shuffle + + pshufd xmm3,xmm1,Shuffle + movaps xmm0,XMMWORD PTR [edx+VectorOffset] + mulps xmm0,xmm3 + addps xmm5,xmm0 + movaps xmm0,XMMWORD PTR [edx+VectorOffset+16] + mulps xmm0,xmm3 + addps xmm6,xmm0 + movaps xmm0,XMMWORD PTR [edx+VectorOffset+32] + mulps xmm0,xmm3 + addps xmm7,xmm0 + + ENDM + +ComputeBlockSseBy2 MACRO VectorOffset, Shuffle + + pshufd xmm3,xmm1,Shuffle + movaps xmm0,XMMWORD PTR [edx+VectorOffset] + mulps xmm0,xmm3 + addps xmm6,xmm0 + movaps xmm0,XMMWORD PTR [edx+VectorOffset+16] + mulps xmm0,xmm3 + addps xmm7,xmm0 + + ENDM + +ComputeBlockSseBy1 MACRO VectorOffset, Shuffle + + pshufd xmm3,xmm1,Shuffle + movaps xmm0,XMMWORD PTR [edx+VectorOffset] + mulps xmm0,xmm3 + addps xmm7,xmm0 + + ENDM + +; +; ComputeBlockSseLoop +; +; This macro generates code to execute the block compute macro multiple +; times and advancing the matrix A and matrix B data pointers. +; +; Arguments: +; +; ComputeBlock - Supplies the macro to compute a single block. +; +; Count - Supplies the number of rows to access from matrix A. +; +; Implicit Arguments: +; +; ebx - Supplies the number of bytes to the next row of matrix A. +; +; ecx - Supplies the address into the matrix A data. +; +; edx - Supplies the address into the matrix B data. +; +; edi - Supplies the number of columns from matrix A and the number of rows +; from matrix B to iterate over. +; +; xmm4-xmm7 - Supplies the block accumulators. +; + +ComputeBlockSseLoop MACRO Count + + LOCAL ComputeBlockBy4Loop + LOCAL ProcessRemainingBlocks + LOCAL ComputeBlockBy1Loop + LOCAL OutputBlock + + sub edi,4 + jl ProcessRemainingBlocks + +ComputeBlockBy4Loop: + movups xmm1,XMMWORD PTR [ecx] + ComputeBlockSseBy&Count 0,000h + ComputeBlockSseBy&Count 16*4,055h + sub edx,-32*4 ; advance matrix B by 32 columns + ComputeBlockSseBy&Count 0,0AAh + ComputeBlockSseBy&Count 16*4,0FFh + sub edx,-32*4 ; advance matrix B by 32 columns + add ecx,4*4 ; advance matrix A by 4 columns + sub edi,4 + jge ComputeBlockBy4Loop + +ProcessRemainingBlocks: + add edi,4 ; correct for over-subtract above + jz OutputBlock + +ComputeBlockBy1Loop: + movss xmm1,DWORD PTR [ecx] + ComputeBlockSseBy&Count 0,000h + add edx,16*4 ; advance matrix B by 16 columns + add ecx,4 ; advance matrix A by 1 column + dec edi + jne ComputeBlockBy1Loop + +OutputBlock: + + ENDM + + SUBTTL "SGEMM kernel for processors supporting SSE2 instructions" +;++ +; +; Routine Description: +; +; This routine is an inner kernel to compute matrix multiplication for a +; set of rows. +; +; This implementation uses SSE2 instructions. +; +; Arguments: +; +; A - Supplies the address of matrix A. +; +; B - Supplies the address of matrix B. The matrix data has been packed using +; MlasSgemmCopyPackB or MlasSgemmTransposePackB. +; +; C - Supplies the address of matrix C. +; +; CountK - Supplies the number of columns from matrix A and the number of rows +; from matrix B to iterate over. +; +; CountM - Supplies the maximum number of rows that can be processed for +; matrix A and matrix C. The actual number of rows handled for this +; invocation depends on the kernel implementation. +; +; CountN - Supplies the number of columns from matrix B and matrix C to iterate +; over. +; +; lda - Supplies the first dimension of matrix A. +; +; ldc - Supplies the first dimension of matrix C. +; +; Alpha - Supplies the scaler multiplier (see SGEMM definition). +; +; Return Value: +; +; Returns the number of rows handled. +; +;-- + +SgemmKernelSseFunction MACRO Mode + +cPublicProc _MlasSgemmKernel&Mode&Sse,9 + + SgemmKernelEntry + +; +; Process 1 row of the matrices. +; + + mov eax,SgemmKernelFrame.CountK[esp] + mov ebx,SgemmKernelFrame.MatrixA[esp] + cmp ebp,12 + jle ProcessRemainingCountN + +ProcessNextColumnLoop16x1: + mov edi,eax ; reload CountK + mov ecx,ebx ; reload matrix A + xorps xmm4,xmm4 ; clear block accumulators + xorps xmm5,xmm5 + xorps xmm6,xmm6 + xorps xmm7,xmm7 + ComputeBlockSseLoop 4 + movss xmm2,DWORD PTR SgemmKernelFrame.Alpha[esp] + shufps xmm2,xmm2,0 + mulps xmm4,xmm2 ; multiply by alpha + mulps xmm5,xmm2 + mulps xmm6,xmm2 + mulps xmm7,xmm2 +IFIDNI , + movups xmm0,XMMWORD PTR [esi] + movups xmm1,XMMWORD PTR [esi+16] + movups xmm2,XMMWORD PTR [esi+32] + addps xmm4,xmm0 + addps xmm5,xmm1 + addps xmm6,xmm2 +ENDIF + movups XMMWORD PTR [esi],xmm4 + movups XMMWORD PTR [esi+16],xmm5 + movups XMMWORD PTR [esi+32],xmm6 + sub ebp,16 + jl OutputMasked16x1Block +IFIDNI , + movups xmm3,XMMWORD PTR [esi+48] + addps xmm7,xmm3 +ENDIF + movups XMMWORD PTR [esi+48],xmm7 + add esi,16*4 ; advance matrix C by 16 columns + cmp ebp,12 + jg ProcessNextColumnLoop16x1 + test ebp,ebp + jnz ProcessRemainingCountN + +; +; Restore non-volatile registers and return. +; + +ExitKernel: + mov eax,1 ; return 1 row handled + SgemmKernelExit + stdRET _MlasSgemmKernel&Mode&Sse + +; +; Process the remaining 1 to 12 columns of the matrices. +; + +ProcessRemainingCountN: + mov edi,eax ; reload CountK + mov ecx,ebx ; reload matrix A + movss xmm4,DWORD PTR SgemmKernelFrame.Alpha[esp] + shufps xmm4,xmm4,0 + xorps xmm5,xmm5 ; clear block accumulators + xorps xmm6,xmm6 + xorps xmm7,xmm7 + cmp ebp,4 + jle ProcessRemainingCountN4OrLess + cmp ebp,8 + jle ProcessRemainingCountN8OrLess + +ProcessRemainingCountN12OrLess: + ComputeBlockSseLoop 3 + mulps xmm5,xmm4 ; multiply by alpha + mulps xmm6,xmm4 + mulps xmm7,xmm4 +IFIDNI , + movups xmm0,XMMWORD PTR [esi] + movups xmm1,XMMWORD PTR [esi+16] + addps xmm5,xmm0 + addps xmm6,xmm1 +ENDIF + movups XMMWORD PTR [esi],xmm5 + movups XMMWORD PTR [esi+16],xmm6 + add esi,8*4 ; advance matrix C by 8 columns + jmp OutputTrailingBlock + +ProcessRemainingCountN8OrLess: + ComputeBlockSseLoop 2 + mulps xmm6,xmm4 ; multiply by alpha + mulps xmm7,xmm4 +IFIDNI , + movups xmm0,XMMWORD PTR [esi] + addps xmm6,xmm0 +ENDIF + movups XMMWORD PTR [esi],xmm6 + add esi,4*4 ; advance matrix C by 4 columns + jmp OutputTrailingBlock + +ProcessRemainingCountN4OrLess: + ComputeBlockSseLoop 1 + mulps xmm7,xmm4 ; multiply by alpha + jmp OutputTrailingBlock + +OutputMasked16x1Block: + add esi,12*4 ; advance matrix C by 12 columns + +OutputTrailingBlock: + test ebp,3 + jz OutputTrailingBlock4Elements + test ebp,2 + jz OutputTrailingBlock1Element + +OutputTrailingBlock2Elements: +IFIDNI , + movsd xmm0,MMWORD PTR [esi] + addps xmm7,xmm0 +ENDIF + movsd MMWORD PTR [esi],xmm7 + test ebp,1 + jz ExitKernel + shufps xmm7,xmm7,0AAh ; shuffle third float down + add esi,2*4 ; advance matrix C by 2 columns + +OutputTrailingBlock1Element: +IFIDNI , + movss xmm0,DWORD PTR [esi] + addss xmm7,xmm0 +ENDIF + movss DWORD PTR [esi],xmm7 + jmp ExitKernel + +OutputTrailingBlock4Elements: +IFIDNI , + movups xmm0,XMMWORD PTR [esi] + addps xmm7,xmm0 +ENDIF + movups XMMWORD PTR [esi],xmm7 + jmp ExitKernel + +stdENDP _MlasSgemmKernel&Mode&Sse + + ENDM + +SgemmKernelSseFunction Zero +SgemmKernelSseFunction Add + +; +; ComputeBlockAvxBy16 +; +; Macro Description: +; +; This macro multiplies and accumulates for a 16xN block (where N is 1,2) +; of the output matrix. +; +; This implementation uses AVX instructions. +; +; Arguments: +; +; Count - Supplies the number of rows to access from matrix A. +; +; VectorOffset - Supplies the byte offset from matrix B to fetch elements. +; +; BroadcastOffset - Supplies the byte offset from matrix A to fetch elements. +; +; Implicit Arguments: +; +; ebx - Supplies the length in bytes of a row from matrix A. +; +; ecx - Supplies the address into the matrix A data. +; +; edx - Supplies the address into the matrix B data. +; +; ymm4-ymm7 - Supplies the block accumulators. +; + +ComputeBlockAvxBy16 MACRO Count, VectorOffset, BroadcastOffset + +IF Count EQ 1 + vbroadcastss ymm3,DWORD PTR [ecx+BroadcastOffset] + vmulps ymm1,ymm3,YMMWORD PTR [edx+VectorOffset] + vaddps ymm4,ymm1,ymm4 + vmulps ymm3,ymm3,YMMWORD PTR [edx+VectorOffset+32] + vaddps ymm5,ymm3,ymm5 +ELSE + vmovaps ymm0,YMMWORD PTR [edx+VectorOffset] + vmovaps ymm1,YMMWORD PTR [edx+VectorOffset+32] + vbroadcastss ymm3,DWORD PTR [ecx+BroadcastOffset] + vmulps ymm2,ymm3,ymm0 + vaddps ymm4,ymm2,ymm4 + vmulps ymm2,ymm3,ymm1 + vaddps ymm5,ymm2,ymm5 + vbroadcastss ymm3,DWORD PTR [ecx+ebx+BroadcastOffset] + vmulps ymm2,ymm3,ymm0 + vaddps ymm6,ymm2,ymm6 + vmulps ymm2,ymm3,ymm1 + vaddps ymm7,ymm2,ymm7 +ENDIF + + ENDM + +; +; ComputeBlockAvxBy8 +; +; Macro Description: +; +; This macro multiplies and accumulates for a 8xN block (where N is 1,2) +; of the output matrix. +; +; This implementation uses AVX instructions. +; +; Arguments: +; +; Count - Supplies the number of rows to access from matrix A. +; +; VectorOffset - Supplies the byte offset from matrix B to fetch elements. +; +; BroadcastOffset - Supplies the byte offset from matrix A to fetch elements. +; +; Implicit Arguments: +; +; ebx - Supplies the length in bytes of a row from matrix A. +; +; ecx - Supplies the address into the matrix A data. +; +; edx - Supplies the address into the matrix B data. +; +; ymm4-ymm7 - Supplies the block accumulators. +; + +ComputeBlockAvxBy8 MACRO Count, VectorOffset, BroadcastOffset + +IF Count EQ 1 + vbroadcastss ymm3,DWORD PTR [ecx+BroadcastOffset] + vmulps ymm3,ymm3,YMMWORD PTR [edx+VectorOffset] + vaddps ymm5,ymm3,ymm5 +ELSE + vmovaps ymm0,YMMWORD PTR [edx+VectorOffset] + vbroadcastss ymm3,DWORD PTR [ecx+BroadcastOffset] + vmulps ymm3,ymm3,ymm0 + vaddps ymm5,ymm3,ymm5 + vbroadcastss ymm3,DWORD PTR [ecx+ebx+BroadcastOffset] + vmulps ymm3,ymm3,ymm0 + vaddps ymm7,ymm3,ymm7 +ENDIF + + ENDM + +; +; ComputeBlockAvxLoop +; +; This macro generates code to execute the block compute macro multiple +; times and advancing the matrix A and matrix B data pointers. +; +; Arguments: +; +; ComputeBlock - Supplies the macro to compute a single block. +; +; Count - Supplies the number of rows to access from matrix A. +; +; Implicit Arguments: +; +; ebx - Supplies the number of bytes to the next row of matrix A. +; +; ecx - Supplies the address into the matrix A data. +; +; edx - Supplies the address into the matrix B data. +; +; edi - Supplies the number of columns from matrix A and the number of rows +; from matrix B to iterate over. +; +; ymm4-ymm7 - Supplies the block accumulators. +; + +ComputeBlockAvxLoop MACRO ComputeBlock, Count + + LOCAL ComputeBlockBy4Loop + LOCAL ProcessRemainingBlocks + LOCAL ComputeBlockBy1Loop + LOCAL OutputBlock + + sub edi,4 + jl ProcessRemainingBlocks + +ComputeBlockBy4Loop: + ComputeBlock Count, 0, 0 + ComputeBlock Count, 16*4, 4 + sub edx,-32*4 ; advance matrix B by 32 columns + ComputeBlock Count, 0, 8 + ComputeBlock Count, 16*4, 12 + sub edx,-32*4 ; advance matrix B by 32 columns + add ecx,4*4 ; advance matrix A by 4 columns + sub edi,4 + jge ComputeBlockBy4Loop + +ProcessRemainingBlocks: + add edi,4 ; correct for over-subtract above + jz OutputBlock + +ComputeBlockBy1Loop: + ComputeBlock Count, 0, 0 + add edx,16*4 ; advance matrix B by 16 columns + add ecx,4 ; advance matrix A by 1 column + dec edi + jne ComputeBlockBy1Loop + +OutputBlock: + + ENDM + + SUBTTL "SGEMM kernel for processors supporting AVX instructions" +;++ +; +; Routine Description: +; +; This routine is an inner kernel to compute matrix multiplication for a +; set of rows. +; +; This implementation uses AVX instructions. +; +; Arguments: +; +; A - Supplies the address of matrix A. +; +; B - Supplies the address of matrix B. The matrix data has been packed using +; MlasSgemmCopyPackB or MlasSgemmTransposePackB. +; +; C - Supplies the address of matrix C. +; +; CountK - Supplies the number of columns from matrix A and the number of rows +; from matrix B to iterate over. +; +; CountM - Supplies the maximum number of rows that can be processed for +; matrix A and matrix C. The actual number of rows handled for this +; invocation depends on the kernel implementation. +; +; CountN - Supplies the number of columns from matrix B and matrix C to iterate +; over. +; +; lda - Supplies the first dimension of matrix A. +; +; ldc - Supplies the first dimension of matrix C. +; +; Alpha - Supplies the scaler multiplier (see SGEMM definition). +; +; Return Value: +; +; Returns the number of rows handled. +; +;-- + +SgemmKernelAvxFunction MACRO Mode + +cPublicProc _MlasSgemmKernel&Mode&Avx,9 + + vzeroall + SgemmKernelEntry + +; +; Process 2 rows of the matrices. +; + + cmp SgemmKernelFrame.CountM[esp],2 + jl ProcessCountMLessThan2 + mov BYTE PTR SgemmKernelFrame.CountM[esp],2 + mov eax,SgemmKernelFrame.ldc[esp] + mov ebx,SgemmKernelFrame.lda[esp] + shl eax,2 ; convert ldc to bytes + shl ebx,2 ; convert lda to bytes + cmp ebp,8 + jle ProcessRemainingCountN2 + +ProcessNextColumnLoop16x2: + mov edi,SgemmKernelFrame.CountK[esp] + mov ecx,SgemmKernelFrame.MatrixA[esp] + vxorps xmm4,xmm4,xmm4 ; clear block accumulators + vxorps xmm5,xmm5,xmm5 + vxorps xmm6,xmm6,xmm6 + vxorps xmm7,xmm7,xmm7 + ComputeBlockAvxLoop ComputeBlockAvxBy16,2 + vbroadcastss ymm2,DWORD PTR SgemmKernelFrame.Alpha[esp] + vmulps ymm4,ymm4,ymm2 ; multiply by alpha + vmulps ymm5,ymm5,ymm2 + vmulps ymm6,ymm6,ymm2 + vmulps ymm7,ymm7,ymm2 + sub ebp,16 + jl OutputMasked16x2Block +IFIDNI , + vaddps ymm4,ymm4,YMMWORD PTR [esi] + vaddps ymm5,ymm5,YMMWORD PTR [esi+32] + vaddps ymm6,ymm6,YMMWORD PTR [esi+eax] + vaddps ymm7,ymm7,YMMWORD PTR [esi+eax+32] +ENDIF + vmovups YMMWORD PTR [esi],ymm4 + vmovups YMMWORD PTR [esi+32],ymm5 + vmovups YMMWORD PTR [esi+eax],ymm6 + vmovups YMMWORD PTR [esi+eax+32],ymm7 + add esi,16*4 ; advance matrix C by 16 columns + cmp ebp,8 + jg ProcessNextColumnLoop16x2 + test ebp,ebp + jz ExitKernel + +ProcessRemainingCountN2: + mov edi,SgemmKernelFrame.CountK[esp] + mov ecx,SgemmKernelFrame.MatrixA[esp] + vxorps xmm5,xmm5,xmm5 ; clear block accumulators + vxorps xmm7,xmm7,xmm7 + ComputeBlockAvxLoop ComputeBlockAvxBy8,2 + vbroadcastss ymm2,DWORD PTR SgemmKernelFrame.Alpha[esp] + vmulps ymm5,ymm5,ymm2 ; multiply by alpha + vmulps ymm7,ymm7,ymm2 + cmp ebp,8 + jl OutputMasked8x2Block +IFIDNI , + vaddps ymm5,ymm5,YMMWORD PTR [esi] + vaddps ymm7,ymm7,YMMWORD PTR [esi+eax] +ENDIF + vmovups YMMWORD PTR [esi],ymm5 + vmovups YMMWORD PTR [esi+eax],ymm7 + +; +; Restore non-volatile registers and return. +; + +ExitKernel: + movzx eax,BYTE PTR SgemmKernelFrame.CountM[esp] + vzeroupper + SgemmKernelExit + stdRET _MlasSgemmKernel&Mode&Avx + +OutputMasked16x2Block: +IFIDNI , + vaddps ymm4,ymm4,YMMWORD PTR [esi] + vaddps ymm6,ymm6,YMMWORD PTR [esi+eax] +ENDIF + vmovups YMMWORD PTR [esi],ymm4 + vmovups YMMWORD PTR [esi+eax],ymm6 + add esi,8*4 ; advance matrix C by 8 columns + add ebp,8 ; correct for over-subtract above + +OutputMasked8x2Block: + mov SgemmKernelFrame.CountN[esp],ebp + vbroadcastss xmm0,SgemmKernelFrame.CountN[esp] + vpcmpgtd xmm1,xmm0,XMMWORD PTR [_MlasMaskMoveAvx+16] + vpcmpgtd xmm0,xmm0,XMMWORD PTR [_MlasMaskMoveAvx] + vinsertf128 ymm0,ymm0,xmm1,1 +IFIDNI , + vmaskmovps ymm4,ymm0,YMMWORD PTR [esi] + vmaskmovps ymm6,ymm0,YMMWORD PTR [esi+eax] + vaddps ymm5,ymm5,ymm4 + vaddps ymm7,ymm7,ymm6 +ENDIF + vmaskmovps YMMWORD PTR [esi],ymm0,ymm5 + vmaskmovps YMMWORD PTR [esi+eax],ymm0,ymm7 + jmp ExitKernel + +; +; Process 1 row of the matrices. +; + +ProcessCountMLessThan2: + mov BYTE PTR SgemmKernelFrame.CountM[esp],1 + mov ebx,SgemmKernelFrame.MatrixA[esp] + vbroadcastss ymm2,DWORD PTR SgemmKernelFrame.Alpha[esp] + cmp ebp,8 + jle ProcessRemainingCountN1 + +ProcessNextColumnLoop16x1: + mov edi,SgemmKernelFrame.CountK[esp] + mov ecx,ebx ; reload matrix A + vxorps xmm4,xmm4,xmm4 ; clear block accumulators + vxorps xmm5,xmm5,xmm5 + ComputeBlockAvxLoop ComputeBlockAvxBy16,1 + vmulps ymm4,ymm4,ymm2 ; multiply by alpha + vmulps ymm5,ymm5,ymm2 + sub ebp,16 + jl OutputMasked16x1Block +IFIDNI , + vaddps ymm4,ymm4,YMMWORD PTR [esi] + vaddps ymm5,ymm5,YMMWORD PTR [esi+32] +ENDIF + vmovups YMMWORD PTR [esi],ymm4 + vmovups YMMWORD PTR [esi+32],ymm5 + add esi,16*4 ; advance matrix C by 16 columns + cmp ebp,8 + jg ProcessNextColumnLoop16x1 + test ebp,ebp + jz ExitKernel + +ProcessRemainingCountN1: + mov edi,SgemmKernelFrame.CountK[esp] + mov ecx,ebx ; reload matrix A + vxorps xmm5,xmm5,xmm5 ; clear block accumulators + ComputeBlockAvxLoop ComputeBlockAvxBy8,1 + vmulps ymm5,ymm5,ymm2 ; multiply by alpha + cmp ebp,8 + jl OutputMasked8x1Block +IFIDNI , + vaddps ymm5,ymm5,YMMWORD PTR [esi] +ENDIF + vmovups YMMWORD PTR [esi],ymm5 + jmp ExitKernel + +OutputMasked16x1Block: +IFIDNI , + vaddps ymm4,ymm4,YMMWORD PTR [esi] +ENDIF + vmovups YMMWORD PTR [esi],ymm4 + add esi,8*4 ; advance matrix C by 8 columns + add ebp,8 ; correct for over-subtract above + +OutputMasked8x1Block: + mov SgemmKernelFrame.CountN[esp],ebp + vbroadcastss xmm0,SgemmKernelFrame.CountN[esp] + vpcmpgtd xmm1,xmm0,XMMWORD PTR [_MlasMaskMoveAvx+16] + vpcmpgtd xmm0,xmm0,XMMWORD PTR [_MlasMaskMoveAvx] + vinsertf128 ymm0,ymm0,xmm1,1 +IFIDNI , + vmaskmovps ymm4,ymm0,YMMWORD PTR [esi] + vaddps ymm5,ymm5,ymm4 +ENDIF + vmaskmovps YMMWORD PTR [esi],ymm0,ymm5 + jmp ExitKernel + +stdENDP _MlasSgemmKernel&Mode&Avx + + ENDM + +SgemmKernelAvxFunction Zero +SgemmKernelAvxFunction Add + +_TEXT ENDS + + END diff --git a/onnxruntime/core/mlas/lib/mlasi.h b/onnxruntime/core/mlas/lib/mlasi.h new file mode 100644 index 0000000000000..b03cd4c84c68e --- /dev/null +++ b/onnxruntime/core/mlas/lib/mlasi.h @@ -0,0 +1,485 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + mlasi.h + +Abstract: + + This module contains the private data structures and procedure prototypes + for the Microsoft Machine Learning algebra subprogram library. + +--*/ + +#pragma once +// clang-format off + +#include +#include +#include +#include + +#if defined(_WIN32) +#include +#include +#else +#if defined(__arm__) || defined(__aarch64__) +#include +#endif +#if defined(__x86_64__) || defined(__i386__) +#include +#include +#endif +#if defined(__x86_64__) +#include "x86_64/xgetbv.h" +#endif +#endif + +// +// Macro to place variables at a specified alignment. +// + +#ifdef _WIN32 +#define MLAS_DECLSPEC_ALIGN(variable, alignment) DECLSPEC_ALIGN(alignment) variable +#else +#define MLAS_DECLSPEC_ALIGN(variable, alignment) variable __attribute__ ((aligned(alignment))) +#endif + +// +// Macro to suppress unreferenced parameter warnings. +// + +#define MLAS_UNREFERENCED_PARAMETER(parameter) ((void)(parameter)) + +// +// Select the target architecture. +// + +#if defined(_M_AMD64) || defined(__x86_64__) +#define MLAS_TARGET_AMD64 +#endif +#if (defined(_M_IX86) && !defined(_M_HYBRID_X86_ARM64)) || defined(__i386__) +#define MLAS_TARGET_IX86 +#endif +#if defined(MLAS_TARGET_AMD64) || defined(MLAS_TARGET_IX86) +#define MLAS_TARGET_AMD64_IX86 +#endif +#if defined(_M_ARM64) || defined(__aarch64__) +#define MLAS_TARGET_ARM64 +#endif +#if defined(_M_ARM) || defined(__arm__) +#define MLAS_TARGET_ARM +#endif + +// +// Select the threading model. +// + +#if defined(_OPENMP) +#include +#define MLAS_USE_OPENMP +#elif defined(_WIN32) +#define MLAS_USE_WIN32_THREADPOOL +#endif + +// +// Define the maximum number of threads supported by this implementation. +// + +#define MLAS_MAXIMUM_THREAD_COUNT 16 + +// +// Define the default strides to step through slices of the input matrices. +// + +#define MLAS_SGEMM_STRIDEN 128 +#define MLAS_SGEMM_STRIDEK 128 + +// +// Define the alignment for segmenting a SGEMM operation across multiple +// threads. +// +// All of the SGEMM kernels can efficiently handle 16 elements. AVX512F can +// efficiently handle 32 elements, but making this value dynamic is not worth +// the effort at this time. +// + +#define MLAS_SGEMM_STRIDEN_THREAD_ALIGN 16 + +// +// Define the prototypes of the SGEMM platform optimized routines. +// + +typedef +size_t +(MLASCALL MLAS_SGEMM_KERNEL_ROUTINE)( + const float* A, + const float* B, + float* C, + size_t CountK, + size_t CountM, + size_t CountN, + size_t lda, + size_t ldc, + float alpha + ); + +typedef MLAS_SGEMM_KERNEL_ROUTINE* PMLAS_SGEMM_KERNEL_ROUTINE; + +typedef +void +(MLASCALL MLAS_SGEMM_KERNEL_M1_ROUTINE)( + const float* A, + const float* B, + float* C, + size_t CountK, + size_t CountN, + size_t ldb, + float beta + ); + +typedef MLAS_SGEMM_KERNEL_M1_ROUTINE* PMLAS_SGEMM_KERNEL_M1_ROUTINE; + +typedef +void +(MLASCALL MLAS_SGEMM_TRANSPOSE_PACKB_BLOCK_ROUTINE)( + float* D, + const float* B, + size_t ldb + ); + +typedef MLAS_SGEMM_TRANSPOSE_PACKB_BLOCK_ROUTINE* PMLAS_SGEMM_TRANSPOSE_PACKB_BLOCK_ROUTINE; + +extern "C" { + + MLAS_SGEMM_KERNEL_ROUTINE MlasSgemmKernelZero; + MLAS_SGEMM_KERNEL_ROUTINE MlasSgemmKernelAdd; +#if defined(MLAS_TARGET_AMD64_IX86) + MLAS_SGEMM_KERNEL_ROUTINE MlasSgemmKernelZeroSse; + MLAS_SGEMM_KERNEL_ROUTINE MlasSgemmKernelAddSse; + MLAS_SGEMM_KERNEL_ROUTINE MlasSgemmKernelZeroAvx; + MLAS_SGEMM_KERNEL_ROUTINE MlasSgemmKernelAddAvx; +#endif +#if defined(MLAS_TARGET_AMD64) + MLAS_SGEMM_KERNEL_ROUTINE MlasSgemmKernelZeroFma3; + MLAS_SGEMM_KERNEL_ROUTINE MlasSgemmKernelAddFma3; + MLAS_SGEMM_KERNEL_ROUTINE MlasSgemmKernelZeroAvx512F; + MLAS_SGEMM_KERNEL_ROUTINE MlasSgemmKernelAddAvx512F; +#endif + +#if defined(MLAS_TARGET_AMD64) + MLAS_SGEMM_KERNEL_M1_ROUTINE MlasSgemmKernelM1Avx; + MLAS_SGEMM_KERNEL_M1_ROUTINE MlasSgemmKernelM1TransposeBAvx; +#endif + +#if defined(MLAS_TARGET_AMD64) + MLAS_SGEMM_TRANSPOSE_PACKB_BLOCK_ROUTINE MlasSgemmTransposePackB16x4Sse; + MLAS_SGEMM_TRANSPOSE_PACKB_BLOCK_ROUTINE MlasSgemmTransposePackB16x4Avx; +#endif + +} + +// +// Define the target number of per-thread multiplies before using another +// thread to perform additional work. +// +// The number is derived from performance results running SGEMM across a +// range of workloads and observing the ideal number of threads to complete +// that workload. See EvaluateThreadingPerformance() in the unit test. +// + +#if defined(MLAS_USE_OPENMP) +#define MLAS_SGEMM_THREAD_COMPLEXITY (64 * 1024) +#else +#if defined(MLAS_TARGET_AMD64) +#define MLAS_SGEMM_THREAD_COMPLEXITY (2 * 1024 * 1024) +#else +#define MLAS_SGEMM_THREAD_COMPLEXITY (1 * 1024 * 1024) +#endif +#endif + +// +// Single-threaded single precision matrix/matrix multiply operation. +// + +void +MlasSgemmOperation( + CBLAS_TRANSPOSE TransA, + CBLAS_TRANSPOSE TransB, + size_t M, + size_t N, + size_t K, + float alpha, + const float* A, + size_t lda, + const float* B, + size_t ldb, + float beta, + float* C, + size_t ldc + ); + +// +// Environment information class. +// + +struct MLAS_PLATFORM { + + MLAS_PLATFORM(void); + +#if defined(MLAS_TARGET_AMD64_IX86) + PMLAS_SGEMM_KERNEL_ROUTINE KernelZeroRoutine; + PMLAS_SGEMM_KERNEL_ROUTINE KernelAddRoutine; +#endif + +#if defined(MLAS_TARGET_AMD64) + PMLAS_SGEMM_KERNEL_M1_ROUTINE KernelM1Routine; + PMLAS_SGEMM_KERNEL_M1_ROUTINE KernelM1TransposeBRoutine; + PMLAS_SGEMM_TRANSPOSE_PACKB_BLOCK_ROUTINE TransposePackB16x4Routine; +#endif + +#if defined(MLAS_USE_WIN32_THREADPOOL) + uint32_t MaximumThreadCount; +#endif + + uint32_t + GetMaximumThreadCount( + void + ) + { +#if defined(MLAS_USE_OPENMP) + return (omp_get_num_threads() == 1) ? omp_get_max_threads() : 1; +#elif defined(MLAS_USE_WIN32_THREADPOOL) + return MaximumThreadCount; +#else + return 1; +#endif + } +}; + +extern MLAS_PLATFORM MlasPlatform; + +// +// Cross-platform wrappers for vector intrinsics. +// + +#if defined(MLAS_TARGET_ARM) +#define MLAS_NEON_INTRINSICS +#define MLAS_NEON32_INTRINSICS +#elif defined(MLAS_TARGET_ARM64) || defined(_M_HYBRID_X86_ARM64) +#define MLAS_NEON_INTRINSICS +#define MLAS_NEON64_INTRINSICS +#elif defined(MLAS_TARGET_AMD64_IX86) +#define MLAS_SSE2_INTRINSICS +#else +#error Unsupported architecture. +#endif + +#if defined(MLAS_NEON_INTRINSICS) +typedef float32x4_t MLAS_FLOAT32X4; +#elif defined(MLAS_SSE2_INTRINSICS) +typedef __m128 MLAS_FLOAT32X4; +#endif + +inline +MLAS_FLOAT32X4 +MlasZeroFloat32x4(void) +{ +#if defined(MLAS_NEON_INTRINSICS) + return vdupq_n_f32(0.0f); +#elif defined(MLAS_SSE2_INTRINSICS) + return _mm_setzero_ps(); +#endif +} + +inline +MLAS_FLOAT32X4 +MlasLoadFloat32x4(const float* Buffer) +{ +#if defined(MLAS_NEON_INTRINSICS) + return vld1q_f32(Buffer); +#elif defined(MLAS_SSE2_INTRINSICS) + return _mm_loadu_ps(Buffer); +#endif +} + +inline +void +MlasStoreFloat32x4(float* Buffer, MLAS_FLOAT32X4 Vector) +{ +#if defined(MLAS_NEON_INTRINSICS) + vst1q_f32(Buffer, Vector); +#elif defined(MLAS_SSE2_INTRINSICS) + _mm_storeu_ps(Buffer, Vector); +#endif +} + +inline +void +MlasStoreAlignedFloat32x4(float* Buffer, MLAS_FLOAT32X4 Vector) +{ +#if defined(MLAS_NEON_INTRINSICS) + vst1q_f32(Buffer, Vector); +#elif defined(MLAS_SSE2_INTRINSICS) + _mm_store_ps(Buffer, Vector); +#endif +} + +inline +void +MlasStoreLowHalfFloat32x4(float* Buffer, MLAS_FLOAT32X4 Vector) +{ +#if defined(MLAS_NEON_INTRINSICS) + vst1_f32(Buffer, vget_low_f32(Vector)); +#elif defined(MLAS_SSE2_INTRINSICS) + _mm_storel_pi((__m64*)Buffer, Vector); +#endif +} + +template +inline +void +MlasStoreLaneFloat32x4(float* Buffer, MLAS_FLOAT32X4 Vector) +{ +#if defined(MLAS_NEON_INTRINSICS) + vst1q_lane_f32(Buffer, Vector, Lane); +#elif defined(MLAS_SSE2_INTRINSICS) + // N.B. When building with AVX instructions, compilers optimize the following + // to a single vextractps instruction. + _mm_store_ss(Buffer, _mm_shuffle_ps(Vector, Vector, _MM_SHUFFLE(Lane, Lane, Lane, Lane))); +#endif +} + +#if defined(MLAS_SSE2_INTRINSICS) + +template<> +inline +void +MlasStoreLaneFloat32x4<0>(float* Buffer, MLAS_FLOAT32X4 Vector) +{ + _mm_store_ss(Buffer, Vector); +} + +#endif + +inline +MLAS_FLOAT32X4 +MlasBroadcastFloat32x4(float Value) +{ +#if defined(MLAS_NEON_INTRINSICS) + return vdupq_n_f32(Value); +#elif defined(MLAS_SSE2_INTRINSICS) + return _mm_set_ps1(Value); +#endif +} + +inline +MLAS_FLOAT32X4 +MlasAddFloat32x4(MLAS_FLOAT32X4 Vector1, MLAS_FLOAT32X4 Vector2) +{ +#if defined(MLAS_NEON_INTRINSICS) + return vaddq_f32(Vector1, Vector2); +#elif defined(MLAS_SSE2_INTRINSICS) + return _mm_add_ps(Vector1, Vector2); +#endif +} + +inline +MLAS_FLOAT32X4 +MlasSubtractFloat32x4(MLAS_FLOAT32X4 Vector1, MLAS_FLOAT32X4 Vector2) +{ +#if defined(MLAS_NEON_INTRINSICS) + return vsubq_f32(Vector1, Vector2); +#elif defined(MLAS_SSE2_INTRINSICS) + return _mm_sub_ps(Vector1, Vector2); +#endif +} + +inline +MLAS_FLOAT32X4 +MlasMultiplyFloat32x4(MLAS_FLOAT32X4 Vector1, MLAS_FLOAT32X4 Vector2) +{ +#if defined(MLAS_NEON_INTRINSICS) + return vmulq_f32(Vector1, Vector2); +#elif defined(MLAS_SSE2_INTRINSICS) + return _mm_mul_ps(Vector1, Vector2); +#endif +} + +inline +MLAS_FLOAT32X4 +MlasDivideFloat32x4(MLAS_FLOAT32X4 Vector1, MLAS_FLOAT32X4 Vector2) +{ +#if defined(MLAS_NEON64_INTRINSICS) + return vdivq_f32(Vector1, Vector2); +#elif defined(MLAS_NEON32_INTRINSICS) + Vector1 = vsetq_lane_f32(vgetq_lane_f32(Vector1, 0) / vgetq_lane_f32(Vector2, 0), Vector1, 0); + Vector1 = vsetq_lane_f32(vgetq_lane_f32(Vector1, 1) / vgetq_lane_f32(Vector2, 1), Vector1, 1); + Vector1 = vsetq_lane_f32(vgetq_lane_f32(Vector1, 2) / vgetq_lane_f32(Vector2, 2), Vector1, 2); + Vector1 = vsetq_lane_f32(vgetq_lane_f32(Vector1, 3) / vgetq_lane_f32(Vector2, 3), Vector1, 3); + return Vector1; +#elif defined(MLAS_SSE2_INTRINSICS) + return _mm_div_ps(Vector1, Vector2); +#endif +} + +inline +MLAS_FLOAT32X4 +MlasMaximumFloat32x4(MLAS_FLOAT32X4 Vector1, MLAS_FLOAT32X4 Vector2) +{ +#if defined(MLAS_NEON_INTRINSICS) + return vmaxq_f32(Vector1, Vector2); +#elif defined(MLAS_SSE2_INTRINSICS) + return _mm_max_ps(Vector1, Vector2); +#endif +} + +inline +MLAS_FLOAT32X4 +MlasMinimumFloat32x4(MLAS_FLOAT32X4 Vector1, MLAS_FLOAT32X4 Vector2) +{ +#if defined(MLAS_NEON_INTRINSICS) + return vminq_f32(Vector1, Vector2); +#elif defined(MLAS_SSE2_INTRINSICS) + return _mm_min_ps(Vector1, Vector2); +#endif +} + +// +// Reads a platform specific time stamp counter. +// + +inline +uint64_t +MlasReadTimeStampCounter(void) +{ +#ifdef _WIN32 +#if defined(MLAS_TARGET_AMD64_IX86) + return ReadTimeStampCounter(); +#else + LARGE_INTEGER PerformanceCounter; + + QueryPerformanceCounter(&PerformanceCounter); + + return (ULONG64)PerformanceCounter.QuadPart; +#endif +#else +#if defined(MLAS_TARGET_AMD64) + uint32_t eax, edx; + + __asm__ __volatile__ + ( + "rdtsc" + : "=a" (eax), "=d" (edx) + ); + + return ((uint64_t)edx << 32) | eax; +#else + return 0; +#endif +#endif +} diff --git a/onnxruntime/core/mlas/lib/platform.cpp b/onnxruntime/core/mlas/lib/platform.cpp new file mode 100644 index 0000000000000..79ae776554e41 --- /dev/null +++ b/onnxruntime/core/mlas/lib/platform.cpp @@ -0,0 +1,147 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + platform.cpp + +Abstract: + + This module implements logic to select the best configuration for the + this platform. + +--*/ + +#include "mlasi.h" + +// +// Stores the platform information. +// + +MLAS_PLATFORM MlasPlatform; + +MLAS_PLATFORM::MLAS_PLATFORM( + void + ) +/*++ + +Routine Description: + + This routine initializes the platform support for this library. + +Arguments: + + None. + +Return Value: + + None. + +--*/ +{ + +#if defined(MLAS_TARGET_AMD64_IX86) + + // + // Default to the baseline SSE2 support. + // + + this->KernelZeroRoutine = MlasSgemmKernelZeroSse; + this->KernelAddRoutine = MlasSgemmKernelAddSse; +#if defined(MLAS_TARGET_AMD64) + this->TransposePackB16x4Routine = MlasSgemmTransposePackB16x4Sse; +#endif + + // + // Check if the processor supports the AVX and OSXSAVE features. + // + + unsigned Cpuid1[4]; +#if defined(_WIN32) + __cpuid((int*)Cpuid1, 1); +#else + __cpuid(1, Cpuid1[0], Cpuid1[1], Cpuid1[2], Cpuid1[3]); +#endif + + if ((Cpuid1[2] & 0x18000000) == 0x18000000) { + + // + // Check if the operating system supports saving SSE and AVX states. + // + +#if defined(_WIN32) + uint64_t xcr0 = _xgetbv(_XCR_XFEATURE_ENABLED_MASK); +#else + uint64_t xcr0 = xgetbv(_XCR_XFEATURE_ENABLED_MASK); +#endif + + if ((xcr0 & 0x6) == 0x6) { + +#if defined(MLAS_TARGET_IX86) + + this->KernelZeroRoutine = MlasSgemmKernelZeroAvx; + this->KernelAddRoutine = MlasSgemmKernelAddAvx; + +#else + + // + // Check if the processor supports AVX512F (and the operating + // system supports saving AVX512F state) or AVX2/FMA3 features. + // + + unsigned Cpuid7[4]; +#if defined(_WIN32) + __cpuidex((int*)Cpuid7, 7, 0); +#else + __cpuid_count(7, 0, Cpuid7[0], Cpuid7[1], Cpuid7[2], Cpuid7[3]); +#endif + + if (((Cpuid1[2] & 0x1000) != 0) && ((Cpuid7[1] & 0x20) != 0)) { + + if (((Cpuid7[1] & 0x10000) != 0) && ((xcr0 & 0xE0) == 0xE0)) { + this->KernelZeroRoutine = MlasSgemmKernelZeroAvx512F; + this->KernelAddRoutine = MlasSgemmKernelAddAvx512F; + } else { + this->KernelZeroRoutine = MlasSgemmKernelZeroFma3; + this->KernelAddRoutine = MlasSgemmKernelAddFma3; + } + + } else { + + this->KernelZeroRoutine = MlasSgemmKernelZeroAvx; + this->KernelAddRoutine = MlasSgemmKernelAddAvx; + } + + this->KernelM1Routine = MlasSgemmKernelM1Avx; + this->KernelM1TransposeBRoutine = MlasSgemmKernelM1TransposeBAvx; + this->TransposePackB16x4Routine = MlasSgemmTransposePackB16x4Avx; + +#endif + + } + } + +#endif + +#if defined(MLAS_USE_WIN32_THREADPOOL) + + // + // Retrieve the number of processors in the system. + // + + SYSTEM_INFO SystemInfo; + + GetSystemInfo(&SystemInfo); + + if (SystemInfo.dwNumberOfProcessors <= MLAS_MAXIMUM_THREAD_COUNT) { + this->MaximumThreadCount = SystemInfo.dwNumberOfProcessors; + } else { + this->MaximumThreadCount = MLAS_MAXIMUM_THREAD_COUNT; + } + +#endif + +} diff --git a/onnxruntime/core/mlas/lib/pooling.cpp b/onnxruntime/core/mlas/lib/pooling.cpp new file mode 100644 index 0000000000000..f535d339d2ca2 --- /dev/null +++ b/onnxruntime/core/mlas/lib/pooling.cpp @@ -0,0 +1,1175 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + pooling.cpp + +Abstract: + + This module implements the pooling operation. + +--*/ + +#include "mlasi.h" + +// +// Define the parameters to execute segments of a pooling operation on worker +// threads. +// + +struct MLAS_WORK_BLOCK { + MLAS_POOLING_KIND PoolingKind; + size_t InputShape[3]; + size_t OutputShape[3]; + int64_t KernelShape[3]; + int64_t Padding[6]; + int64_t StrideShape[3]; +}; + +// +// Define the prototype of the pooling kernel routine. +// + +typedef +void +(MLAS_POOL_KERNEL_ROUTINE)( + const MLAS_WORK_BLOCK* WorkBlock, + size_t ChannelCount, + const float* Input, + float* Output + ); + +typedef MLAS_POOL_KERNEL_ROUTINE* PMLAS_POOL_KERNEL_ROUTINE; + +// +// Define the number of elements to allocate on the stack for the reduction +// buffer in the vectorized kernels. +// + +#define MLAS_POOL_REDUCTION_BUFFER_STACK 2048 + +// +// Define the number of reduction buffer elements reserved for over-reading +// an entire vector to avoid special handling at the right edge of the +// buffer. +// + +#define MLAS_POOL_REDUCTION_BUFFER_PADDING ((sizeof(MLAS_FLOAT32X4) / sizeof(float)) - 1) + +// +// Abstraction for maximum pooling. +// + +struct MLAS_MAXIMUM_POOLING +{ + static float InitialValue() + { + return std::numeric_limits::lowest(); + } + + static MLAS_FLOAT32X4 InitialVector() + { + return MlasBroadcastFloat32x4(InitialValue()); + } + + static float Reduce(float Reduction, float Value) + { + return (std::max)(Reduction, Value); + } + + static MLAS_FLOAT32X4 Reduce(MLAS_FLOAT32X4 Reduction, MLAS_FLOAT32X4 Value) + { + return MlasMaximumFloat32x4(Reduction, Value); + } + + static float AveragePool(float Reduction, float Size) + { + MLAS_UNREFERENCED_PARAMETER(Size); + + return Reduction; + } + + struct DividerVectorContext + { + void PrepareExcludePad(size_t PaddingLeftWidth, size_t InputWidth, size_t KernelWidth) + { + MLAS_UNREFERENCED_PARAMETER(PaddingLeftWidth); + MLAS_UNREFERENCED_PARAMETER(InputWidth); + MLAS_UNREFERENCED_PARAMETER(KernelWidth); + } + + void PrepareIncludePad(size_t KernelSize) + { + MLAS_UNREFERENCED_PARAMETER(KernelSize); + } + + void StartNextOutputRow(size_t InputRowsCount) + { + MLAS_UNREFERENCED_PARAMETER(InputRowsCount); + } + + MLAS_FLOAT32X4 DivideExcludePad(MLAS_FLOAT32X4 Reduction) + { + return Reduction; + } + + MLAS_FLOAT32X4 DivideIncludePad(MLAS_FLOAT32X4 Reduction) + { + return Reduction; + } + }; +}; + +// +// Abstraction for average pooling. +// + +MLAS_DECLSPEC_ALIGN(static const float MlasInitialReductionInputIndex[], sizeof(MLAS_FLOAT32X4)) = { 0.0f, 1.0f, 2.0f, 3.0f }; + +struct MLAS_AVERAGE_POOLING +{ + static float InitialValue() + { + return 0.0f; + } + + static MLAS_FLOAT32X4 InitialVector() + { + return MlasZeroFloat32x4(); + } + + static float Reduce(float Reduction, float Value) + { + return Reduction + Value; + } + + static MLAS_FLOAT32X4 Reduce(MLAS_FLOAT32X4 Reduction, MLAS_FLOAT32X4 Value) + { + return MlasAddFloat32x4(Reduction, Value); + } + + static float AveragePool(float Reduction, float Size) + { + return Reduction / Size; + } + + struct DividerVectorContext + { + MLAS_FLOAT32X4 KernelSizeBroadcast; + MLAS_FLOAT32X4 KernelWidthBroadcast; + MLAS_FLOAT32X4 PaddingLowerBound; + MLAS_FLOAT32X4 PaddingUpperBound; + MLAS_FLOAT32X4 ReductionInputIndex; + MLAS_FLOAT32X4 InputRowsBroadcast; + + void PrepareExcludePad(size_t PaddingLeftWidth, size_t InputWidth, size_t KernelWidth) + { + KernelWidthBroadcast = MlasBroadcastFloat32x4(float(unsigned(KernelWidth))); + PaddingLowerBound = MlasBroadcastFloat32x4(float(unsigned(PaddingLeftWidth))); + PaddingUpperBound = MlasBroadcastFloat32x4(float(unsigned(PaddingLeftWidth + InputWidth))); + } + + void PrepareIncludePad(size_t KernelSize) + { + KernelSizeBroadcast = MlasBroadcastFloat32x4(float(unsigned(KernelSize))); + } + + void StartNextOutputRow(size_t InputRowsCount) + { + ReductionInputIndex = MlasLoadFloat32x4(MlasInitialReductionInputIndex); + InputRowsBroadcast = MlasBroadcastFloat32x4(float(unsigned(InputRowsCount))); + } + + MLAS_FLOAT32X4 DivideExcludePad(MLAS_FLOAT32X4 Reduction) + { + MLAS_FLOAT32X4 Divisor; + + // + // Compute the ending input index for each column and bound the index + // range by the padding indices, then compute the number of input + // column contributions from the delta. + // + + MLAS_FLOAT32X4 ReductionInputEndingIndex = + MlasAddFloat32x4(ReductionInputIndex, KernelWidthBroadcast); + + MLAS_FLOAT32X4 LowerInputIndex = + MlasMaximumFloat32x4(ReductionInputIndex, PaddingLowerBound); + MLAS_FLOAT32X4 UpperInputIndex = + MlasMinimumFloat32x4(ReductionInputEndingIndex, PaddingUpperBound); + + MLAS_FLOAT32X4 InputIndexDelta = + MlasSubtractFloat32x4(UpperInputIndex, LowerInputIndex); + + // + // Advance the input index vector for the next iteration. + // + + ReductionInputIndex = + MlasAddFloat32x4(ReductionInputIndex, MlasBroadcastFloat32x4(4.0f)); + + // + // Compute the per-column number of input elements used for the sum. + // + // At the end of the input row, the index range computed above may be + // zero for unused trailing vector elements, so avoid any divide by zero + // penalty by enforcing a minimum of 1.0f. + // + + Divisor = MlasMultiplyFloat32x4(InputIndexDelta, InputRowsBroadcast); + Divisor = MlasMaximumFloat32x4(Divisor, MlasBroadcastFloat32x4(1.0f)); + + return MlasDivideFloat32x4(Reduction, Divisor); + } + + MLAS_FLOAT32X4 DivideIncludePad(MLAS_FLOAT32X4 Reduction) + { + return MlasDivideFloat32x4(Reduction, KernelSizeBroadcast); + } + }; +}; + +template +void +MlasPool1DKernel( + const MLAS_WORK_BLOCK* WorkBlock, + size_t ChannelCount, + const float* Input, + float* Output + ) +/*++ + +Routine Description: + + This routine implements the 1D pooling operation using generic constructs. + +Arguments: + + WorkBlock - Supplies the structure that contains the pooling parameters. + + ChannelCount - Supplies the number of channels to process. + + Input - Supplies the input tensor. + + Output - Supplies the output tensor. + +Return Value: + + None. + +--*/ +{ + constexpr size_t WidthShapeIndex = 0; + + const MLAS_POOLING_KIND PoolingKind = WorkBlock->PoolingKind; + + const size_t InputWidth = WorkBlock->InputShape[WidthShapeIndex]; + const size_t OutputWidth = WorkBlock->OutputShape[WidthShapeIndex]; + + const int64_t KernelWidth = WorkBlock->KernelShape[WidthShapeIndex]; + const int64_t PaddingLeftWidth = WorkBlock->Padding[WidthShapeIndex]; + const int64_t StrideWidth = WorkBlock->StrideShape[WidthShapeIndex]; + + for (size_t c = 0; c < ChannelCount; c++) { + + for (size_t pw = 0; pw < OutputWidth; pw++) { + + const int64_t iwStart64 = pw * StrideWidth - PaddingLeftWidth; + const int64_t iwEnd64 = iwStart64 + KernelWidth; + + const size_t iwStart = size_t((std::max)(iwStart64, int64_t(0))); + const size_t iwEnd = size_t((std::min)(iwEnd64, int64_t(InputWidth))); + + float m = PoolingType::InitialValue(); + + for (size_t iw = size_t(iwStart); iw < size_t(iwEnd); iw++) { + m = PoolingType::Reduce(m, Input[iw]); + } + + if (PoolingKind == MlasAveragePoolingExcludePad) { + m = PoolingType::AveragePool(m, float(iwEnd - iwStart)); + } else { + m = PoolingType::AveragePool(m, float(KernelWidth)); + } + + *Output++ = m; + } + + Input += InputWidth; + } +} + +template +void +MlasPool2DKernel( + const MLAS_WORK_BLOCK* WorkBlock, + size_t ChannelCount, + const float* Input, + float* Output + ) +/*++ + +Routine Description: + + This routine implements the 2D pooling operation using generic constructs. + +Arguments: + + WorkBlock - Supplies the structure that contains the pooling parameters. + + ChannelCount - Supplies the number of channels to process. + + Input - Supplies the input tensor. + + Output - Supplies the output tensor. + +Return Value: + + None. + +--*/ +{ + constexpr size_t HeightShapeIndex = 0; + constexpr size_t WidthShapeIndex = 1; + + const MLAS_POOLING_KIND PoolingKind = WorkBlock->PoolingKind; + + const size_t InputHeight = WorkBlock->InputShape[HeightShapeIndex]; + const size_t InputWidth = WorkBlock->InputShape[WidthShapeIndex]; + const size_t OutputHeight = WorkBlock->OutputShape[HeightShapeIndex]; + const size_t OutputWidth = WorkBlock->OutputShape[WidthShapeIndex]; + + const int64_t KernelHeight = WorkBlock->KernelShape[HeightShapeIndex]; + const int64_t KernelWidth = WorkBlock->KernelShape[WidthShapeIndex]; + const int64_t PaddingLeftHeight = WorkBlock->Padding[HeightShapeIndex]; + const int64_t PaddingLeftWidth = WorkBlock->Padding[WidthShapeIndex]; + const int64_t StrideHeight = WorkBlock->StrideShape[HeightShapeIndex]; + const int64_t StrideWidth = WorkBlock->StrideShape[WidthShapeIndex]; + + for (size_t c = 0; c < ChannelCount; c++) { + + for (size_t ph = 0; ph < OutputHeight; ph++) { + + const int64_t ihStart64 = ph * StrideHeight - PaddingLeftHeight; + const int64_t ihEnd64 = ihStart64 + KernelHeight; + + const size_t ihStart = size_t((std::max)(ihStart64, int64_t(0))); + const size_t ihEnd = size_t((std::min)(ihEnd64, int64_t(InputHeight))); + + for (size_t pw = 0; pw < OutputWidth; pw++) { + + const int64_t iwStart64 = pw * StrideWidth - PaddingLeftWidth; + const int64_t iwEnd64 = iwStart64 + KernelWidth; + + const size_t iwStart = size_t((std::max)(iwStart64, int64_t(0))); + const size_t iwEnd = size_t((std::min)(iwEnd64, int64_t(InputWidth))); + + float m = PoolingType::InitialValue(); + + for (size_t ih = ihStart; ih < ihEnd; ih++) { + for (size_t iw = iwStart; iw < iwEnd; iw++) { + m = PoolingType::Reduce(m, Input[ih * InputWidth + iw]); + } + } + + if (PoolingKind == MlasAveragePoolingExcludePad) { + m = PoolingType::AveragePool(m, float((ihEnd - ihStart) * (iwEnd - iwStart))); + } else { + m = PoolingType::AveragePool(m, float(KernelHeight * KernelWidth)); + } + + *Output++ = m; + } + } + + Input += InputHeight * InputWidth; + } +} + +template +void +MlasPool2DVectorKernel( + const MLAS_WORK_BLOCK* WorkBlock, + size_t ChannelCount, + const float* Input, + float* Output + ) +/*++ + +Routine Description: + + This routine implements an optimized 2D pooling operation using vector + instructions. + +Arguments: + + WorkBlock - Supplies the structure that contains the pooling parameters. + + ChannelCount - Supplies the number of channels to process. + + Input - Supplies the input tensor. + + Output - Supplies the output tensor. + +Return Value: + + None. + +--*/ +{ + constexpr size_t Dimensions = 2; + + constexpr size_t HeightShapeIndex = 0; + constexpr size_t WidthShapeIndex = 1; + + const MLAS_POOLING_KIND PoolingKind = WorkBlock->PoolingKind; + + const size_t InputHeight = WorkBlock->InputShape[HeightShapeIndex]; + const size_t InputWidth = WorkBlock->InputShape[WidthShapeIndex]; + const size_t OutputHeight = WorkBlock->OutputShape[HeightShapeIndex]; + const size_t OutputWidth = WorkBlock->OutputShape[WidthShapeIndex]; + + const size_t KernelHeight = size_t(WorkBlock->KernelShape[HeightShapeIndex]); + const size_t KernelWidth = size_t(WorkBlock->KernelShape[WidthShapeIndex]); + const size_t PaddingLeftHeight = size_t(WorkBlock->Padding[HeightShapeIndex]); + const size_t PaddingLeftWidth = size_t(WorkBlock->Padding[WidthShapeIndex]); + const size_t PaddingRightWidth = size_t(WorkBlock->Padding[Dimensions + WidthShapeIndex]); + const size_t StrideHeight = size_t(WorkBlock->StrideShape[HeightShapeIndex]); + const size_t StrideWidth = size_t(WorkBlock->StrideShape[WidthShapeIndex]); + + float ReductionBuffer[MLAS_POOL_REDUCTION_BUFFER_STACK]; + + // + // Fill the edges of the reduction buffer with the padding value. + // + + float* FillReductionBuffer = ReductionBuffer; + float* FillReductionBufferEnd = FillReductionBuffer + PaddingLeftWidth; + + while (FillReductionBuffer < FillReductionBufferEnd) { + *FillReductionBuffer++ = PoolingType::InitialValue(); + } + + FillReductionBuffer = FillReductionBuffer + InputWidth; + FillReductionBufferEnd = FillReductionBuffer + PaddingRightWidth + MLAS_POOL_REDUCTION_BUFFER_PADDING; + + while (FillReductionBuffer < FillReductionBufferEnd) { + *FillReductionBuffer++ = PoolingType::InitialValue(); + } + + // + // Apply the pooling operation to each channel. + // + + typename PoolingType::DividerVectorContext divider; + divider.PrepareExcludePad(PaddingLeftWidth, InputWidth, KernelWidth); + divider.PrepareIncludePad(KernelHeight * KernelWidth); + + for (size_t c = 0; c < ChannelCount; c++) { + + for (size_t ph = 0; ph < OutputHeight; ph++) { + + size_t ihStart = ph * StrideHeight - PaddingLeftHeight; + size_t ihEnd = ihStart + KernelHeight; + + if (ihStart >= InputHeight) { + ihStart = 0; + } + + if (ihEnd > InputHeight) { + ihEnd = InputHeight; + } + + divider.StartNextOutputRow(ihEnd - ihStart); + + // + // Reduce the input across the kernel height and store in a local + // reduction buffer. + // + + const float* InputRowStart = &Input[ihStart * InputWidth]; + const size_t InputRowsCount = ihEnd - ihStart - 1; + size_t InputWidthRemaining = InputWidth; + float* ReductionOutput = &ReductionBuffer[PaddingLeftWidth]; + + while (InputWidthRemaining >= 4) { + + const float* InputRow = InputRowStart; + size_t InputRowsRemaining = InputRowsCount; + MLAS_FLOAT32X4 Reduction = MlasLoadFloat32x4(InputRow); + + while (InputRowsRemaining > 0) { + InputRow += InputWidth; + Reduction = PoolingType::Reduce(Reduction, MlasLoadFloat32x4(InputRow)); + InputRowsRemaining--; + } + + MlasStoreFloat32x4(ReductionOutput, Reduction); + ReductionOutput += 4; + + InputRowStart += 4; + InputWidthRemaining -= 4; + } + + while (InputWidthRemaining > 0) { + + const float* InputRow = InputRowStart; + size_t InputRowsRemaining = InputRowsCount; + float Reduction = *InputRow; + + while (InputRowsRemaining > 0) { + InputRow += InputWidth; + Reduction = PoolingType::Reduce(Reduction, *InputRow); + InputRowsRemaining--; + } + + *ReductionOutput++ = Reduction; + + InputRowStart += 1; + InputWidthRemaining -= 1; + } + + // + // Reduce the input across the kernel width and store to the output + // tensor. + // + + size_t OutputWidthRemaining = OutputWidth; + const float* ReductionInputStart = ReductionBuffer; + + do { + + const float* ReductionInput = ReductionInputStart; + const float* ReductionInputEnd = ReductionInput + KernelWidth; + MLAS_FLOAT32X4 Reduction = MlasLoadFloat32x4(ReductionInput++); + + while (ReductionInput < ReductionInputEnd) { + Reduction = PoolingType::Reduce(Reduction, MlasLoadFloat32x4(ReductionInput++)); + } + + if (PoolingKind == MlasAveragePoolingExcludePad) { + Reduction = divider.DivideExcludePad(Reduction); + } else { + Reduction = divider.DivideIncludePad(Reduction); + } + + if (StrideWidth == 1) { + + if (OutputWidthRemaining < 4) { + + if (OutputWidthRemaining >= 2) { + + MlasStoreLowHalfFloat32x4(Output, Reduction); + + if (OutputWidthRemaining > 2) { + MlasStoreLaneFloat32x4<2>(Output + 2, Reduction); + } + + } else { + MlasStoreLaneFloat32x4<0>(Output, Reduction); + } + + Output += OutputWidthRemaining; + + break; + } + + MlasStoreFloat32x4(Output, Reduction); + + Output += 4; + OutputWidthRemaining -= 4; + + } else { + + if (OutputWidthRemaining == 1) { + MlasStoreLaneFloat32x4<0>(Output++, Reduction); + break; + } + +#if defined(MLAS_NEON_INTRINSICS) + MlasStoreLaneFloat32x4<0>(Output, Reduction); + MlasStoreLaneFloat32x4<2>(Output + 1, Reduction); +#elif defined(MLAS_SSE2_INTRINSICS) + Reduction = _mm_shuffle_ps(Reduction, Reduction, _MM_SHUFFLE(2, 0, 2, 0)); + MlasStoreLowHalfFloat32x4(Output, Reduction); +#else +#error Unsupported architecture. +#endif + + Output += 2; + OutputWidthRemaining -= 2; + } + + ReductionInputStart += 4; + + } while (OutputWidthRemaining > 0); + } + + Input += InputHeight * InputWidth; + } +} + +template +void +MlasPool3DKernel( + const MLAS_WORK_BLOCK* WorkBlock, + size_t ChannelCount, + const float* Input, + float* Output + ) +/*++ + +Routine Description: + + This routine implements the 3D pooling operation using generic constructs. + +Arguments: + + WorkBlock - Supplies the structure that contains the pooling parameters. + + ChannelCount - Supplies the number of channels to process. + + Input - Supplies the input tensor. + + Output - Supplies the output tensor. + +Return Value: + + None. + +--*/ +{ + constexpr size_t DepthShapeIndex = 0; + constexpr size_t HeightShapeIndex = 1; + constexpr size_t WidthShapeIndex = 2; + + const MLAS_POOLING_KIND PoolingKind = WorkBlock->PoolingKind; + + const size_t InputDepth = WorkBlock->InputShape[DepthShapeIndex]; + const size_t InputHeight = WorkBlock->InputShape[HeightShapeIndex]; + const size_t InputWidth = WorkBlock->InputShape[WidthShapeIndex]; + const size_t OutputDepth = WorkBlock->OutputShape[DepthShapeIndex]; + const size_t OutputHeight = WorkBlock->OutputShape[HeightShapeIndex]; + const size_t OutputWidth = WorkBlock->OutputShape[WidthShapeIndex]; + + const int64_t KernelDepth = WorkBlock->KernelShape[DepthShapeIndex]; + const int64_t KernelHeight = WorkBlock->KernelShape[HeightShapeIndex]; + const int64_t KernelWidth = WorkBlock->KernelShape[WidthShapeIndex]; + const int64_t PaddingLeftDepth = WorkBlock->Padding[DepthShapeIndex]; + const int64_t PaddingLeftHeight = WorkBlock->Padding[HeightShapeIndex]; + const int64_t PaddingLeftWidth = WorkBlock->Padding[WidthShapeIndex]; + const int64_t StrideDepth = WorkBlock->StrideShape[DepthShapeIndex]; + const int64_t StrideHeight = WorkBlock->StrideShape[HeightShapeIndex]; + const int64_t StrideWidth = WorkBlock->StrideShape[WidthShapeIndex]; + + for (size_t c = 0; c < ChannelCount; c++) { + + for (size_t pd = 0; pd < OutputDepth; pd++) { + + const int64_t idStart64 = pd * StrideDepth - PaddingLeftDepth; + const int64_t idEnd64 = idStart64 + KernelDepth; + + const size_t idStart = size_t((std::max)(idStart64, int64_t(0))); + const size_t idEnd = size_t((std::min)(idEnd64, int64_t(InputDepth))); + + for (size_t ph = 0; ph < OutputHeight; ph++) { + + const int64_t ihStart64 = ph * StrideHeight - PaddingLeftHeight; + const int64_t ihEnd64 = ihStart64 + KernelHeight; + + const size_t ihStart = size_t((std::max)(ihStart64, int64_t(0))); + const size_t ihEnd = size_t((std::min)(ihEnd64, int64_t(InputHeight))); + + for (size_t pw = 0; pw < OutputWidth; pw++) { + + const int64_t iwStart64 = pw * StrideWidth - PaddingLeftWidth; + const int64_t iwEnd64 = iwStart64 + KernelWidth; + + const size_t iwStart = size_t((std::max)(iwStart64, int64_t(0))); + const size_t iwEnd = size_t((std::min)(iwEnd64, int64_t(InputWidth))); + + float m = PoolingType::InitialValue(); + + for (size_t id = idStart; id < idEnd; id++) { + for (size_t ih = ihStart; ih < ihEnd; ih++) { + for (size_t iw = iwStart; iw < iwEnd; iw++) { + m = PoolingType::Reduce(m, Input[id * InputHeight * InputWidth + ih * InputWidth + iw]); + } + } + } + + if (PoolingKind == MlasAveragePoolingExcludePad) { + m = PoolingType::AveragePool(m, float((idEnd - idStart) * (ihEnd - ihStart) * (iwEnd - iwStart))); + } else { + m = PoolingType::AveragePool(m, float(KernelDepth * KernelHeight * KernelWidth)); + } + + *Output++ = m; + } + } + } + + Input += InputDepth * InputHeight * InputWidth; + } +} + +template +void +MlasPool3DVectorKernel( + const MLAS_WORK_BLOCK* WorkBlock, + size_t ChannelCount, + const float* Input, + float* Output + ) +/*++ + +Routine Description: + + This routine implements an optimized 2D pooling operation using vector + instructions. + +Arguments: + + WorkBlock - Supplies the structure that contains the pooling parameters. + + ChannelCount - Supplies the number of channels to process. + + Input - Supplies the input tensor. + + Output - Supplies the output tensor. + +Return Value: + + None. + +--*/ +{ + constexpr size_t Dimensions = 3; + + constexpr size_t DepthShapeIndex = 0; + constexpr size_t HeightShapeIndex = 1; + constexpr size_t WidthShapeIndex = 2; + + const MLAS_POOLING_KIND PoolingKind = WorkBlock->PoolingKind; + + const size_t InputDepth = WorkBlock->InputShape[DepthShapeIndex]; + const size_t InputHeight = WorkBlock->InputShape[HeightShapeIndex]; + const size_t InputWidth = WorkBlock->InputShape[WidthShapeIndex]; + const size_t OutputDepth = WorkBlock->OutputShape[DepthShapeIndex]; + const size_t OutputHeight = WorkBlock->OutputShape[HeightShapeIndex]; + const size_t OutputWidth = WorkBlock->OutputShape[WidthShapeIndex]; + + const size_t KernelDepth = size_t(WorkBlock->KernelShape[DepthShapeIndex]); + const size_t KernelHeight = size_t(WorkBlock->KernelShape[HeightShapeIndex]); + const size_t KernelWidth = size_t(WorkBlock->KernelShape[WidthShapeIndex]); + const size_t PaddingLeftDepth = size_t(WorkBlock->Padding[DepthShapeIndex]); + const size_t PaddingLeftHeight = size_t(WorkBlock->Padding[HeightShapeIndex]); + const size_t PaddingLeftWidth = size_t(WorkBlock->Padding[WidthShapeIndex]); + const size_t PaddingRightWidth = size_t(WorkBlock->Padding[Dimensions + WidthShapeIndex]); + const size_t StrideDepth = size_t(WorkBlock->StrideShape[DepthShapeIndex]); + const size_t StrideHeight = size_t(WorkBlock->StrideShape[HeightShapeIndex]); + const size_t StrideWidth = size_t(WorkBlock->StrideShape[WidthShapeIndex]); + + float ReductionBuffer[MLAS_POOL_REDUCTION_BUFFER_STACK]; + + // + // Fill the edges of the reduction buffer with the padding value. + // + + float* FillReductionBuffer = ReductionBuffer; + float* FillReductionBufferEnd = FillReductionBuffer + PaddingLeftWidth; + + while (FillReductionBuffer < FillReductionBufferEnd) { + *FillReductionBuffer++ = PoolingType::InitialValue(); + } + + FillReductionBuffer = FillReductionBuffer + InputWidth; + FillReductionBufferEnd = FillReductionBuffer + PaddingRightWidth + MLAS_POOL_REDUCTION_BUFFER_PADDING; + + while (FillReductionBuffer < FillReductionBufferEnd) { + *FillReductionBuffer++ = PoolingType::InitialValue(); + } + + // + // Apply the pooling operation to each channel. + // + + typename PoolingType::DividerVectorContext divider; + divider.PrepareExcludePad(PaddingLeftWidth, InputWidth, KernelWidth); + divider.PrepareIncludePad(KernelDepth * KernelHeight * KernelWidth); + + for (size_t c = 0; c < ChannelCount; c++) { + + for (size_t pd = 0; pd < OutputDepth; pd++) { + + size_t idStart = pd * StrideDepth - PaddingLeftDepth; + size_t idEnd = idStart + KernelDepth; + + if (idStart >= InputDepth) { + idStart = 0; + } + + if (idEnd > InputDepth) { + idEnd = InputDepth; + } + + for (size_t ph = 0; ph < OutputHeight; ph++) { + + size_t ihStart = ph * StrideHeight - PaddingLeftHeight; + size_t ihEnd = ihStart + KernelHeight; + + if (ihStart >= InputHeight) { + ihStart = 0; + } + + if (ihEnd > InputHeight) { + ihEnd = InputHeight; + } + + divider.StartNextOutputRow((idEnd - idStart) * (ihEnd - ihStart)); + + // + // Reduce the input across the kernel height and store in a local + // reduction buffer. + // + + const float* InputRowStart = &Input[idStart * InputHeight * InputWidth + ihStart * InputWidth]; + const size_t InputPlanesCount = idEnd - idStart; + const size_t InputRowsCount = ihEnd - ihStart; + size_t InputWidthRemaining = InputWidth; + float* ReductionOutput = &ReductionBuffer[PaddingLeftWidth]; + const size_t InputAdvancePlane = (InputHeight - InputRowsCount) * InputWidth; + + while (InputWidthRemaining >= 4) { + + const float* InputRow = InputRowStart; + size_t InputPlanesRemaining = InputPlanesCount; + MLAS_FLOAT32X4 Reduction = PoolingType::InitialVector(); + + do { + + size_t InputRowsRemaining = InputRowsCount; + + do { + + Reduction = PoolingType::Reduce(Reduction, MlasLoadFloat32x4(InputRow)); + InputRow += InputWidth; + InputRowsRemaining--; + + } while (InputRowsRemaining > 0); + + InputRow += InputAdvancePlane; + InputPlanesRemaining--; + + } while (InputPlanesRemaining > 0); + + MlasStoreFloat32x4(ReductionOutput, Reduction); + ReductionOutput += 4; + + InputRowStart += 4; + InputWidthRemaining -= 4; + } + + while (InputWidthRemaining > 0) { + + const float* InputRow = InputRowStart; + size_t InputPlanesRemaining = InputPlanesCount; + float Reduction = PoolingType::InitialValue(); + + do { + + size_t InputRowsRemaining = InputRowsCount; + + do { + + Reduction = PoolingType::Reduce(Reduction, *InputRow); + InputRow += InputWidth; + InputRowsRemaining--; + + } while (InputRowsRemaining > 0); + + InputRow += InputAdvancePlane; + InputPlanesRemaining--; + + } while (InputPlanesRemaining > 0); + + *ReductionOutput++ = Reduction; + + InputRowStart += 1; + InputWidthRemaining -= 1; + } + + // + // Reduce the input across the kernel width and store to the output + // tensor. + // + + size_t OutputWidthRemaining = OutputWidth; + const float* ReductionInputStart = ReductionBuffer; + + do { + + const float* ReductionInput = ReductionInputStart; + const float* ReductionInputEnd = ReductionInput + KernelWidth; + MLAS_FLOAT32X4 Reduction = MlasLoadFloat32x4(ReductionInput++); + + while (ReductionInput < ReductionInputEnd) { + Reduction = PoolingType::Reduce(Reduction, MlasLoadFloat32x4(ReductionInput++)); + } + + if (PoolingKind == MlasAveragePoolingExcludePad) { + Reduction = divider.DivideExcludePad(Reduction); + } else { + Reduction = divider.DivideIncludePad(Reduction); + } + + if (StrideWidth == 1) { + + if (OutputWidthRemaining < 4) { + + if (OutputWidthRemaining >= 2) { + + MlasStoreLowHalfFloat32x4(Output, Reduction); + + if (OutputWidthRemaining > 2) { + MlasStoreLaneFloat32x4<2>(Output + 2, Reduction); + } + + } else { + MlasStoreLaneFloat32x4<0>(Output, Reduction); + } + + Output += OutputWidthRemaining; + + break; + } + + MlasStoreFloat32x4(Output, Reduction); + + Output += 4; + OutputWidthRemaining -= 4; + + } else { + + if (OutputWidthRemaining == 1) { + MlasStoreLaneFloat32x4<0>(Output++, Reduction); + break; + } + +#if defined(MLAS_NEON_INTRINSICS) + MlasStoreLaneFloat32x4<0>(Output, Reduction); + MlasStoreLaneFloat32x4<2>(Output + 1, Reduction); +#elif defined(MLAS_SSE2_INTRINSICS) + Reduction = _mm_shuffle_ps(Reduction, Reduction, _MM_SHUFFLE(2, 0, 2, 0)); + MlasStoreLowHalfFloat32x4(Output, Reduction); +#else +#error Unsupported architecture. +#endif + + Output += 2; + OutputWidthRemaining -= 2; + } + + ReductionInputStart += 4; + + } while (OutputWidthRemaining > 0); + } + } + + Input += InputDepth * InputHeight * InputWidth; + } +} + +// +// Stores pointers to the pooling kernel routines. +// + +static const PMLAS_POOL_KERNEL_ROUTINE MlasPoolGenericKernels[][3] = +{ + { + MlasPool1DKernel, + MlasPool2DKernel, + MlasPool3DKernel, + }, + { + MlasPool1DKernel, + MlasPool2DKernel, + MlasPool3DKernel, + }, + { + MlasPool1DKernel, + MlasPool2DKernel, + MlasPool3DKernel, + }, +}; + +static const PMLAS_POOL_KERNEL_ROUTINE MlasPoolVectorKernels[][2] = +{ + { + MlasPool2DVectorKernel, + MlasPool3DVectorKernel, + }, + { + MlasPool2DVectorKernel, + MlasPool3DVectorKernel, + }, + { + MlasPool2DVectorKernel, + MlasPool3DVectorKernel, + }, +}; + +void +MLASCALL +MlasPool( + MLAS_POOLING_KIND PoolingKind, + size_t Dimensions, + const int64_t* InputShape, + const int64_t* KernelShape, + const int64_t* Padding, + const int64_t* StrideShape, + const int64_t* OutputShape, + const float* Input, + float* Output + ) +/*++ + +Routine Description: + + This routine prepares for a pooling operation by computing required WorkBlock. + +Arguments: + + PoolingKind - Supplies the kind of pooling operation to perform. + + Dimensions - Supplies the number of dimensions. + + InputShape - Supplies the shape of the input tensor. + + KernelShape - Supplies the shape of the kernel transform. + + Padding - Supplies the number of padding elements at the edge of the input + tensor. + + StrideShape - Supplies the shape of the stride. + + OutputShape - Supplies the shape of the output tensor. + + Input - Supplies the input tensor. + + Output - Supplies the output tensor. + +Return Value: + + None. + +--*/ +{ + MLAS_WORK_BLOCK WorkBlock; + + WorkBlock.PoolingKind = PoolingKind; + + // + // Compute the total number of channels to process and advance the input + // and output shapes over the batch and channel counts. + // + + size_t TotalChannelCount = size_t(InputShape[0]) * size_t(InputShape[1]); + + InputShape += 2; + OutputShape += 2; + + // + // Save the pooling parameters. + // + + size_t InputSize = 1; + size_t OutputSize = 1; + + bool AllKernelsAreSmall = true; + + for (size_t dim = 0; dim < Dimensions; dim++) { + + WorkBlock.InputShape[dim] = size_t(InputShape[dim]); + WorkBlock.OutputShape[dim] = size_t(OutputShape[dim]); + + if (KernelShape != nullptr) { + WorkBlock.KernelShape[dim] = KernelShape[dim]; + } else { + WorkBlock.KernelShape[dim] = InputShape[dim]; + } + + if (Padding != nullptr) { + WorkBlock.Padding[dim] = Padding[dim]; + WorkBlock.Padding[dim + Dimensions] = Padding[dim + Dimensions]; + } else { + WorkBlock.Padding[dim] = 0; + WorkBlock.Padding[dim + Dimensions] = 0; + } + + if (StrideShape != nullptr) { + WorkBlock.StrideShape[dim] = StrideShape[dim]; + } else { + WorkBlock.StrideShape[dim] = 1; + } + + InputSize *= WorkBlock.InputShape[dim]; + OutputSize *= WorkBlock.OutputShape[dim]; + + AllKernelsAreSmall &= (WorkBlock.KernelShape[dim] <= 32); + } + + // + // Determine which pooling kernel routine to use. + // + // The vectorized kernels only support strides of 1 or 2. The kernel size + // should be kept low in order to keep the divisors for average pooling to + // be exactly representable as float. The input width plus padding must fit + // in the reduction buffer. + // + + PMLAS_POOL_KERNEL_ROUTINE PoolKernelRoutine = MlasPoolGenericKernels[PoolingKind][Dimensions - 1]; + + if (Dimensions >= 2 && WorkBlock.StrideShape[Dimensions - 1] <= 2 && AllKernelsAreSmall) { + + int64_t ReductionBufferRemaining = MLAS_POOL_REDUCTION_BUFFER_STACK - MLAS_POOL_REDUCTION_BUFFER_PADDING; + + if (ReductionBufferRemaining >= WorkBlock.Padding[Dimensions - 1]) { + ReductionBufferRemaining -= WorkBlock.Padding[Dimensions - 1]; + } else { + ReductionBufferRemaining = 0; + } + + if (ReductionBufferRemaining >= WorkBlock.Padding[Dimensions * 2 - 1]) { + ReductionBufferRemaining -= WorkBlock.Padding[Dimensions * 2 - 1]; + } else { + ReductionBufferRemaining = 0; + } + + if (ReductionBufferRemaining >= int64_t(WorkBlock.InputShape[Dimensions - 1])) { + PoolKernelRoutine = MlasPoolVectorKernels[PoolingKind][Dimensions - 2]; + } + } + + // + // Execute the pooling kernel routine. + // + +#if defined(MLAS_USE_OPENMP) + + #pragma omp parallel for + for (int64_t c = 0; c < int64_t(TotalChannelCount); c++) { + PoolKernelRoutine(&WorkBlock, 1, Input + c * InputSize, Output + c * OutputSize); + } + +#else + + PoolKernelRoutine(&WorkBlock, TotalChannelCount, Input, Output); + +#endif + +} diff --git a/onnxruntime/core/mlas/lib/sgemm.cpp b/onnxruntime/core/mlas/lib/sgemm.cpp new file mode 100644 index 0000000000000..11f0d22121260 --- /dev/null +++ b/onnxruntime/core/mlas/lib/sgemm.cpp @@ -0,0 +1,1332 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + sgemm.cpp + +Abstract: + + This module implements the single precision matrix/matrix multiply + operation (SGEMM). + +--*/ + +#include "mlasi.h" + +// +// Define the number of rows from matrix A to transpose to a local buffer. +// +// N.B. AVX processes a maximum of 4 rows, FMA3 processes a maximum of 6 +// rows, and AVX512F processes a maximum of 12 rows. +// + +#define MLAS_SGEMM_TRANSA_ROWS 12 + +// +// Define the parameters to execute segments of a SGEMM operation on worker +// threads. +// + +struct MLAS_SGEMM_WORK_BLOCK { +#if defined(MLAS_USE_WIN32_THREADPOOL) + volatile LONG Counter; + CBLAS_TRANSPOSE TransA; + CBLAS_TRANSPOSE TransB; + size_t K; + size_t lda; + size_t ldb; + size_t ldc; + float alpha; + float beta; +#endif + struct SEGMENT { + size_t M; + size_t N; + const float* A; + const float* B; + float* C; + } Segments[MLAS_MAXIMUM_THREAD_COUNT]; +}; + +#if defined(MLAS_TARGET_AMD64_IX86) + +// +// Stores a vector to build a conditional load/store mask for vmaskmovps. +// + +extern "C" MLAS_DECLSPEC_ALIGN(const uint32_t MlasMaskMoveAvx[8], 8 * sizeof(float)) = { 0, 1, 2, 3, 4, 5, 6, 7 }; + +#endif + +void +MlasSgemmMultiplyBeta( + float* C, + size_t CountM, + size_t CountN, + size_t ldc, + float beta + ) +/*++ + +Routine Description: + + This routine multiplies all elements of the output matrix by the beta + scalar value. + +Arguments: + + C - Supplies the address of matrix C. + + CountM - Supplies the number of rows from matrix C. + + CountN - Supplies the number of columns from matrix C. + + ldc - Supplies the first dimension of matrix C. + + beta - Supplies the scaler multiplier (see SGEMM definition). + +Return Value: + + None. + +--*/ +{ + MLAS_FLOAT32X4 BetaBroadcast = MlasBroadcastFloat32x4(beta); + + do { + + float* c = C; + size_t n = CountN; + + while (n >= 4) { + MlasStoreFloat32x4(c, MlasMultiplyFloat32x4(MlasLoadFloat32x4(c), BetaBroadcast)); + c += 4; + n -= 4; + } + + while (n > 0) { +#if defined(MLAS_SSE2_INTRINSICS) + _mm_store_ss(c, _mm_mul_ss(_mm_load_ss(c), BetaBroadcast)); +#else + *c = *c * beta; +#endif + c += 1; + n -= 1; + } + + C += ldc; + CountM--; + + } while (CountM > 0); +} + +void +MlasSgemmTransposeA( + float* D, + const float* A, + size_t lda, + size_t CountY, + size_t CountX + ) +/*++ + +Routine Description: + + This routine transposes elements from the source matrix to the destination + buffer. + +Arguments: + + D - Supplies the address of the destination buffer. + + A - Supplies the address of the source matrix. + + lda - Supplies the number of elements per row of the source matrix. + + CountY - Supplies the number of rows of the source matrix to transpose. + + CountX - Supplies the number of columns of the source matrix to transpose. + +Return Value: + + None. + +--*/ +{ + size_t ldd = CountX; + + // + // Transpose elements from matrix A into the destination buffer 4 columns + // at a time. + // + + while (CountX >= 4) { + + float* d = D; + const float* a = A; + size_t y = CountY; + + do { + + float t0 = a[0]; + float t1 = a[lda]; + float t2 = a[lda * 2]; + float t3 = a[lda * 3]; + + d[0] = t0; + d[1] = t1; + d[2] = t2; + d[3] = t3; + + d += ldd; + a += 1; + y--; + + } while (y > 0); + + D += 4; + A += lda * 4; + CountX -= 4; + } + + // + // Transpose elements from matrix A into the destination buffer for the + // remaining columns. + // + + if (CountX >= 2) { + + float* d = D; + const float* a = A; + size_t y = CountY; + + do { + + float t0 = a[0]; + float t1 = a[lda]; + + d[0] = t0; + d[1] = t1; + + d += ldd; + a += 1; + y--; + + } while (y > 0); + + D += 2; + A += lda * 2; + CountX -= 2; + } + + if (CountX >= 1) { + + float* d = D; + const float* a = A; + size_t y = CountY; + + do { + + d[0] = a[0]; + + d += ldd; + a += 1; + y--; + + } while (y > 0); + } +} + +void +MlasSgemmCopyPackB( + float* D, + const float* B, + size_t ldb, + size_t CountX, + size_t CountY + ) +/*++ + +Routine Description: + + This routine copies elements from the source matrix to the destination + packed buffer. + + Columns of 16 elements from the source matrix are unrolled to be physically + contiguous for better locality inside the SGEMM kernels. Any remaining + columns less than 16 elements wide are zero-padded. + +Arguments: + + D - Supplies the address of the destination packed buffer. + + B - Supplies the address of the source matrix. + + ldb - Supplies the number of elements per row of the source matrix. + + CountX - Supplies the number of columns of the source matrix to copy. + + CountY - Supplies the number of rows of the source matrix to copy. + +Return Value: + + None. + +--*/ +{ + // + // Copy data from matrix B into the destination buffer 16 columns at a + // time. + // + + while (CountX >= 16) { + + const float* b = B; + size_t y = CountY; + + do { + +#if defined(MLAS_NEON_INTRINSICS) + vst4q_f32(D, vld4q_f32(b)); +#else + MLAS_FLOAT32X4 t0 = MlasLoadFloat32x4(b); + MLAS_FLOAT32X4 t1 = MlasLoadFloat32x4(b + 4); + MLAS_FLOAT32X4 t2 = MlasLoadFloat32x4(b + 8); + MLAS_FLOAT32X4 t3 = MlasLoadFloat32x4(b + 12); + + MlasStoreAlignedFloat32x4(D, t0); + MlasStoreAlignedFloat32x4(D + 4, t1); + MlasStoreAlignedFloat32x4(D + 8, t2); + MlasStoreAlignedFloat32x4(D + 12, t3); +#endif + + D += 16; + b += ldb; + y--; + + } while (y > 0); + + B += 16; + CountX -= 16; + } + + // + // Special case the handling of the remaining columns less than 16 elements + // wide. + // + + if (CountX > 0) { + + MLAS_FLOAT32X4 ZeroFloat32x4 = MlasZeroFloat32x4(); + +#if defined(MLAS_NEON_INTRINSICS) + float32x4x4_t ZeroFloat32x4x4 = { ZeroFloat32x4, ZeroFloat32x4, ZeroFloat32x4, ZeroFloat32x4 }; +#endif + + size_t y = CountY; + + do { + + float* d = D; + const float* b = B; + +#if defined(MLAS_NEON_INTRINSICS) + vst4q_f32(D, ZeroFloat32x4x4); +#else + MlasStoreAlignedFloat32x4(d, ZeroFloat32x4); + MlasStoreAlignedFloat32x4(d + 4, ZeroFloat32x4); + MlasStoreAlignedFloat32x4(d + 8, ZeroFloat32x4); + MlasStoreAlignedFloat32x4(d + 12, ZeroFloat32x4); +#endif + + if ((CountX & 8) != 0) { + + MLAS_FLOAT32X4 t0 = MlasLoadFloat32x4(b); + MLAS_FLOAT32X4 t1 = MlasLoadFloat32x4(b + 4); + + MlasStoreAlignedFloat32x4(d, t0); + MlasStoreAlignedFloat32x4(d + 4, t1); + + d += 8; + b += 8; + } + + if ((CountX & 4) != 0) { + + MlasStoreAlignedFloat32x4(d, MlasLoadFloat32x4(b)); + + d += 4; + b += 4; + } + + if ((CountX & 2) != 0) { + + float t0 = b[0]; + float t1 = b[1]; + + d[0] = t0; + d[1] = t1; + + d += 2; + b += 2; + } + + if ((CountX & 1) != 0) { + *d = *b; + } + + D += 16; + B += ldb; + y--; + + } while (y > 0); + } +} + +#if defined(MLAS_SSE2_INTRINSICS) || defined(MLAS_NEON_INTRINSICS) + +inline +void +MlasSgemmTransposePackB16x4( + float* D, + const float* B, + size_t ldb + ) +/*++ + +Routine Description: + + This routine transposes elements from the source matrix to the destination + packed buffer. + + 4 columns of 16 rows from the source matrix are transposed to 16 columns of 4 + rows in the destination packed buffer. + +Arguments: + + D - Supplies the address of the destination packed buffer. + + B - Supplies the address of the source matrix. + + ldb - Supplies the number of elements per row of the source matrix. + +Return Value: + + None. + +--*/ +{ + for (unsigned n = 0; n < 4; n++) { + + MLAS_FLOAT32X4 t0 = MlasLoadFloat32x4(&B[ldb * 0]); + MLAS_FLOAT32X4 t1 = MlasLoadFloat32x4(&B[ldb * 1]); + MLAS_FLOAT32X4 t2 = MlasLoadFloat32x4(&B[ldb * 2]); + MLAS_FLOAT32X4 t3 = MlasLoadFloat32x4(&B[ldb * 3]); + +#if defined(MLAS_NEON_INTRINSICS) + float32x4x2_t z0 = vzipq_f32(t0, t2); + float32x4x2_t z1 = vzipq_f32(t1, t3); + float32x4x2_t o0 = vzipq_f32(z0.val[0], z1.val[0]); + float32x4x2_t o1 = vzipq_f32(z0.val[1], z1.val[1]); + t0 = o0.val[0]; + t1 = o0.val[1]; + t2 = o1.val[0]; + t3 = o1.val[1]; +#else + _MM_TRANSPOSE4_PS(t0, t1, t2, t3); +#endif + + MlasStoreAlignedFloat32x4(&D[0], t0); + MlasStoreAlignedFloat32x4(&D[16], t1); + MlasStoreAlignedFloat32x4(&D[32], t2); + MlasStoreAlignedFloat32x4(&D[48], t3); + + D += 4; + B += ldb * 4; + } +} + +#endif + +void +MlasSgemmTransposePackB( + float* D, + const float* B, + size_t ldb, + size_t CountY, + size_t CountX + ) +/*++ + +Routine Description: + + This routine transposes elements from the source matrix to the destination + packed buffer. + + Columns of 16 elements from the source matrix are unrolled to be physically + contiguous for better locality inside the SGEMM kernels. Any remaining + columns less than 16 elements wide are zero-padded. + +Arguments: + + D - Supplies the address of the destination packed buffer. + + B - Supplies the address of the source matrix. + + ldb - Supplies the number of elements per row of the source matrix. + + CountY - Supplies the number of rows of the source matrix to transpose. + + CountX - Supplies the number of columns of the source matrix to transpose. + +Return Value: + + None. + +--*/ +{ + // + // Transpose elements from matrix B into the packed buffer 16 rows at a + // time. + // + + while (CountY >= 16) { + + const float* b = B; + size_t x = CountX; + +#if defined(MLAS_TARGET_AMD64) + + PMLAS_SGEMM_TRANSPOSE_PACKB_BLOCK_ROUTINE SgemmTransposePackB16x4Routine = + MlasPlatform.TransposePackB16x4Routine; + + while (x >= 4) { + + SgemmTransposePackB16x4Routine(&D[0], &b[0], ldb); + + b += 4; + D += 16 * 4; + x -= 4; + } + +#elif defined(MLAS_SSE2_INTRINSICS) || defined(MLAS_NEON_INTRINSICS) + + while (x >= 4) { + + MlasSgemmTransposePackB16x4(&D[0], &b[0], ldb); + + b += 4; + D += 16 * 4; + x -= 4; + } + +#endif + + while (x > 0) { + + float t0 = b[0]; + float t1 = b[ldb]; + float t2 = b[ldb * 2]; + float t3 = b[ldb * 3]; + float t4 = b[ldb * 4]; + float t5 = b[ldb * 5]; + float t6 = b[ldb * 6]; + float t7 = b[ldb * 7]; + float t8 = b[ldb * 8]; + float t9 = b[ldb * 9]; + float t10 = b[ldb * 10]; + float t11 = b[ldb * 11]; + float t12 = b[ldb * 12]; + float t13 = b[ldb * 13]; + float t14 = b[ldb * 14]; + float t15 = b[ldb * 15]; + + D[0] = t0; + D[1] = t1; + D[2] = t2; + D[3] = t3; + D[4] = t4; + D[5] = t5; + D[6] = t6; + D[7] = t7; + D[8] = t8; + D[9] = t9; + D[10] = t10; + D[11] = t11; + D[12] = t12; + D[13] = t13; + D[14] = t14; + D[15] = t15; + + D += 16; + b += 1; + x--; + } + + B += ldb * 16; + CountY -= 16; + } + + if (CountY > 0) { + + // + // Zero pad the remaining entries of the packed buffer. + // + + memset(D, 0, CountX * 16 * sizeof(float)); + + // + // Transpose elements from matrix B into the packed buffer for the + // remaining rows. + // + + if (CountY >= 8) { + + float* d = D; + const float* b = B; + size_t x = CountX; + + do { + + float t0 = b[0]; + float t1 = b[ldb]; + float t2 = b[ldb * 2]; + float t3 = b[ldb * 3]; + float t4 = b[ldb * 4]; + float t5 = b[ldb * 5]; + float t6 = b[ldb * 6]; + float t7 = b[ldb * 7]; + + d[0] = t0; + d[1] = t1; + d[2] = t2; + d[3] = t3; + d[4] = t4; + d[5] = t5; + d[6] = t6; + d[7] = t7; + + d += 16; + b += 1; + x--; + + } while (x > 0); + + D += 8; + B += ldb * 8; + CountY -= 8; + } + + if (CountY >= 4) { + + float* d = D; + const float* b = B; + size_t x = CountX; + + do { + + float t0 = b[0]; + float t1 = b[ldb]; + float t2 = b[ldb * 2]; + float t3 = b[ldb * 3]; + + d[0] = t0; + d[1] = t1; + d[2] = t2; + d[3] = t3; + + d += 16; + b += 1; + x--; + + } while (x > 0); + + D += 4; + B += ldb * 4; + CountY -= 4; + } + + if (CountY >= 2) { + + float* d = D; + const float* b = B; + size_t x = CountX; + + do { + + float t0 = b[0]; + float t1 = b[ldb]; + + d[0] = t0; + d[1] = t1; + + d += 16; + b += 1; + x--; + + } while (x > 0); + + D += 2; + B += ldb * 2; + CountY -= 2; + } + + if (CountY >= 1) { + + float* d = D; + const float* b = B; + size_t x = CountX; + + do { + + d[0] = b[0]; + + d += 16; + b += 1; + x--; + + } while (x > 0); + } + } +} + +void +MlasSgemmOperation( + CBLAS_TRANSPOSE TransA, + CBLAS_TRANSPOSE TransB, + size_t M, + size_t N, + size_t K, + float alpha, + const float* A, + size_t lda, + const float* B, + size_t ldb, + float beta, + float* C, + size_t ldc + ) +/*++ + +Routine Description: + + This routine implements the single precision matrix/matrix multiply + operation (SGEMM). + +Arguments: + + TransA - Supplies the transpose operation for matrix A. + + TransB - Supplies the transpose operation for matrix B. + + M - Supplies the number of rows of matrix A and matrix C. + + N - Supplies the number of columns of matrix B and matrix C. + + K - Supplies the number of columns of matrix A and the number of rows of + matrix B. + + alpha - Supplies the scaler alpha multiplier (see SGEMM definition). + + A - Supplies the address of matrix A. + + lda - Supplies the first dimension of matrix A. + + B - Supplies the address of matrix B. + + ldb - Supplies the first dimension of matrix B. + + beta - Supplies the scaler beta multiplier (see SGEMM definition). + + C - Supplies the address of matrix C. + + ldc - Supplies the first dimension of matrix C. + +Return Value: + + None. + +--*/ +{ + float PanelA[MLAS_SGEMM_TRANSA_ROWS * MLAS_SGEMM_STRIDEK]; + MLAS_DECLSPEC_ALIGN(float PanelB[MLAS_SGEMM_STRIDEN * MLAS_SGEMM_STRIDEK], 16 * sizeof(float)); + + // + // Handle the special case of a small M. The data from matrix B is not + // referenced multiple times, so using a local packed buffer is a wasted + // memory copy. + // + + if (M == 1 && TransA == CblasNoTrans && alpha == 1.0f && (beta == 0.0f || beta == 1.0f)) { + +#if defined(MLAS_TARGET_AMD64) + + PMLAS_SGEMM_KERNEL_M1_ROUTINE SgemmKernelM1Routine; + + if (TransB == CblasNoTrans) { + SgemmKernelM1Routine = MlasPlatform.KernelM1Routine; + } else { + SgemmKernelM1Routine = MlasPlatform.KernelM1TransposeBRoutine; + } + + if (SgemmKernelM1Routine != nullptr) { + SgemmKernelM1Routine(A, B, C, K, N, ldb, beta); + return; + } + +#endif + + } + + // + // Compute the strides to step through slices of the input matrices. + // + // Expand the N stride if K is small or expand the K stride if N is small + // for better utilization of the B panel. Avoid changing the K stride if + // the A panel needs to be used for transposing. + // + + uint32_t StrideN = MLAS_SGEMM_STRIDEN; + uint32_t StrideK = MLAS_SGEMM_STRIDEK; + + if (N >= K) { + + while (StrideK / 2 >= K) { + StrideN *= 2; + StrideK /= 2; + } + + } else if (TransA == CblasNoTrans) { + + while (StrideN > 16 && StrideN / 2 >= N) { + StrideK *= 2; + StrideN /= 2; + } + } + + // + // Step through each slice of matrix B along the N dimension. + // + + size_t CountN; + size_t CountK; + + for (size_t n = 0; n < N; n += CountN) { + + CountN = StrideN; + + if (CountN > (N - n)) { + CountN = N - n; + } + + // + // Multiply the output matrix by beta as needed. + // + + if (beta != 0.0f && beta != 1.0f) { + MlasSgemmMultiplyBeta(C + n, M, CountN, ldc, beta); + } + + // + // Step through each slice of matrix B along the K dimension. + // + + for (size_t k = 0; k < K; k += CountK) { + + CountK = StrideK; + + if (CountK > (K - k)) { + CountK = K - k; + } + + // + // Copy or transpose a panel of matrix B to a local packed buffer. + // + + if (TransB == CblasNoTrans) { + MlasSgemmCopyPackB(PanelB, B + n + k * ldb, ldb, CountN, CountK); + } else { + MlasSgemmTransposePackB(PanelB, B + k + n * ldb, ldb, CountN, CountK); + } + + // + // Select the kernel routine to use for this panel. + // + + bool UseKernelZeroRoutine = (k == 0 && beta == 0.0f); + +#if defined(MLAS_TARGET_AMD64_IX86) + PMLAS_SGEMM_KERNEL_ROUTINE SgemmKernelRoutine = + UseKernelZeroRoutine ? MlasPlatform.KernelZeroRoutine : MlasPlatform.KernelAddRoutine; +#endif + + // + // Step through each slice of matrix A along the M dimension. + // + + float* c = C + n; + + size_t RowsRemaining = M; + size_t RowsHandled; + + if (TransA == CblasNoTrans) { + + const float* a = A + k; + + // + // Step through the rows of matrix A. + // + + do { + +#if defined(MLAS_TARGET_AMD64_IX86) + RowsHandled = SgemmKernelRoutine(a, PanelB, c, CountK, RowsRemaining, CountN, lda, ldc, alpha); +#else + if (UseKernelZeroRoutine) { + RowsHandled = MlasSgemmKernelZero(a, PanelB, c, CountK, RowsRemaining, CountN, lda, ldc, alpha); + } else { + RowsHandled = MlasSgemmKernelAdd(a, PanelB, c, CountK, RowsRemaining, CountN, lda, ldc, alpha); + } +#endif + + c += ldc * RowsHandled; + a += lda * RowsHandled; + + RowsRemaining -= RowsHandled; + + } while (RowsRemaining > 0); + + } else { + + const float* a = A + k * lda; + + do { + + // + // Transpose elements from matrix A into a local buffer. + // + + size_t RowsTransposed = RowsRemaining; + + if (RowsTransposed > MLAS_SGEMM_TRANSA_ROWS) { + RowsTransposed = MLAS_SGEMM_TRANSA_ROWS; + } + + RowsRemaining -= RowsTransposed; + + MlasSgemmTransposeA(PanelA, a, lda, RowsTransposed, CountK); + + a += RowsTransposed; + + // + // Step through the rows of the local buffer. + // + + const float* pa = PanelA; + + do { + +#if defined(MLAS_TARGET_AMD64_IX86) + RowsHandled = SgemmKernelRoutine(pa, PanelB, c, CountK, RowsTransposed, CountN, CountK, ldc, alpha); +#else + if (UseKernelZeroRoutine) { + RowsHandled = MlasSgemmKernelZero(pa, PanelB, c, CountK, RowsTransposed, CountN, CountK, ldc, alpha); + } else { + RowsHandled = MlasSgemmKernelAdd(pa, PanelB, c, CountK, RowsTransposed, CountN, CountK, ldc, alpha); + } +#endif + + c += ldc * RowsHandled; + pa += CountK * RowsHandled; + + RowsTransposed -= RowsHandled; + + } while (RowsTransposed > 0); + + } while (RowsRemaining > 0); + } + } + } +} + +#if defined(MLAS_USE_WIN32_THREADPOOL) + +void +CALLBACK +MlasSgemmWorkCallback( + PTP_CALLBACK_INSTANCE Instance, + void* Context, + PTP_WORK WorkObject + ) +/*++ + +Routine Description: + + This routine is invoked from a worker thread to execute a segment of a + SGEMM operation. + +Arguments: + + Instance - Supplies the callback instance object. + + Context - Supplies the pointer to the parameters for the SGEMM operation. + + WorkObject - Supplies the threadpool work object. + +Return Value: + + None. + +--*/ +{ + MLAS_UNREFERENCED_PARAMETER(Instance); + MLAS_UNREFERENCED_PARAMETER(WorkObject); + + MLAS_SGEMM_WORK_BLOCK* WorkBlock = (MLAS_SGEMM_WORK_BLOCK*)Context; + + LONG Index = InterlockedIncrement(&WorkBlock->Counter) - 1; + + MLAS_SGEMM_WORK_BLOCK::SEGMENT* Segment = &WorkBlock->Segments[Index]; + + MlasSgemmOperation(WorkBlock->TransA, WorkBlock->TransB, Segment->M, + Segment->N, WorkBlock->K, WorkBlock->alpha, Segment->A, WorkBlock->lda, + Segment->B, WorkBlock->ldb, WorkBlock->beta, Segment->C, + WorkBlock->ldc); +} + +#endif + +inline +bool +MlasSgemmTryMultithread( + CBLAS_TRANSPOSE TransA, + CBLAS_TRANSPOSE TransB, + size_t M, + size_t N, + size_t K, + float alpha, + const float* A, + size_t lda, + const float* B, + size_t ldb, + float beta, + float* C, + size_t ldc + ) +/*++ + +Routine Description: + + This routine attempts to launch a single precision matrix/matrix multiply + operation (SGEMM) across multiple threads. + +Arguments: + + TransA - Supplies the transpose operation for matrix A. + + TransB - Supplies the transpose operation for matrix B. + + M - Supplies the number of rows of matrix A and matrix C. + + N - Supplies the number of columns of matrix B and matrix C. + + K - Supplies the number of columns of matrix A and the number of rows of + matrix B. + + alpha - Supplies the scaler alpha multiplier (see SGEMM definition). + + A - Supplies the address of matrix A. + + lda - Supplies the first dimension of matrix A. + + B - Supplies the address of matrix B. + + ldb - Supplies the first dimension of matrix B. + + beta - Supplies the scaler beta multiplier (see SGEMM definition). + + C - Supplies the address of matrix C. + + ldc - Supplies the first dimension of matrix C. + +Return Value: + + Returns true if the operation was completed across multiple threads, else + false if the operation should fall back to a single thread. + +--*/ +{ + +#if defined(MLAS_USE_WIN32_THREADPOOL) || defined(MLAS_USE_OPENMP) + + MLAS_SGEMM_WORK_BLOCK WorkBlock; + uint32_t TargetThreadCount; + + // + // Compute the number of target threads given the complexity of the SGEMM + // operation. Small requests should run using the single threaded path. + // + + double Complexity = double(M) * double(N) * double(K); + + if (Complexity < double(MLAS_SGEMM_THREAD_COMPLEXITY * MLAS_MAXIMUM_THREAD_COUNT)) { + TargetThreadCount = uint32_t(Complexity / double(MLAS_SGEMM_THREAD_COMPLEXITY)) + 1; + } else { + TargetThreadCount = MLAS_MAXIMUM_THREAD_COUNT; + } + + uint32_t MaximumThreadCount = MlasPlatform.GetMaximumThreadCount(); + + if (TargetThreadCount >= MaximumThreadCount) { + TargetThreadCount = MaximumThreadCount; + } + + if (TargetThreadCount == 1) { + return false; + } + +#if defined(MLAS_USE_WIN32_THREADPOOL) + + // + // Create an object to submit work to the threadpool. + // + + PTP_WORK WorkObject = CreateThreadpoolWork(MlasSgemmWorkCallback, &WorkBlock, nullptr); + + if (WorkObject == nullptr) { + return false; + } + + // + // Initialize the common fields of the work block. + // + + WorkBlock.Counter = 0; + WorkBlock.TransA = TransA; + WorkBlock.TransB = TransB; + WorkBlock.K = K; + WorkBlock.lda = lda; + WorkBlock.ldb = ldb; + WorkBlock.ldc = ldc; + WorkBlock.alpha = alpha; + WorkBlock.beta = beta; + +#endif + + // + // Segment the operation across multiple threads. + // + + uint32_t Index = 0; + + if (N > M) { + + size_t StrideN = N / TargetThreadCount; + + if ((StrideN * TargetThreadCount) != N) { + StrideN++; + } + + StrideN = + (StrideN + MLAS_SGEMM_STRIDEN_THREAD_ALIGN - 1) & ~(MLAS_SGEMM_STRIDEN_THREAD_ALIGN - 1); + + size_t pldb = (TransB == CblasNoTrans) ? 1 : ldb; + + for (size_t CountN, n = 0; n < N; n += CountN) { + + CountN = StrideN; + + if (CountN > (N - n)) { + CountN = N - n; + } + + WorkBlock.Segments[Index].M = M; + WorkBlock.Segments[Index].N = CountN; + WorkBlock.Segments[Index].A = A; + WorkBlock.Segments[Index].B = B + n * pldb; + WorkBlock.Segments[Index].C = C + n; + +#if defined(MLAS_USE_WIN32_THREADPOOL) + + // + // Execute one of the segments on a worker thread. + // + + if (Index > 0) { + SubmitThreadpoolWork(WorkObject); + } + +#endif + + Index++; + } + + } else { + + size_t StrideM = M / TargetThreadCount; + + if ((StrideM * TargetThreadCount) != M) { + StrideM++; + } + + size_t plda = (TransA == CblasNoTrans) ? lda : 1; + + for (size_t CountM, m = 0; m < M; m += CountM) { + + CountM = StrideM; + + if (CountM > (M - m)) { + CountM = M - m; + } + + WorkBlock.Segments[Index].M = CountM; + WorkBlock.Segments[Index].N = N; + WorkBlock.Segments[Index].A = A + m * plda; + WorkBlock.Segments[Index].B = B; + WorkBlock.Segments[Index].C = C + m * ldc; + +#if defined(MLAS_USE_WIN32_THREADPOOL) + + // + // Execute one of the segments on a worker thread. + // + + if (Index > 0) { + SubmitThreadpoolWork(WorkObject); + } + +#endif + + Index++; + } + } + +#if defined(MLAS_USE_OPENMP) + + #pragma omp parallel num_threads(Index) + { + int tid = omp_get_thread_num(); + + MLAS_SGEMM_WORK_BLOCK::SEGMENT* Segment = &WorkBlock.Segments[tid]; + + MlasSgemmOperation(TransA, TransB, Segment->M, Segment->N, K, alpha, + Segment->A, lda, Segment->B, ldb, beta, Segment->C, ldc); + } + +#elif defined(MLAS_USE_WIN32_THREADPOOL) + + // + // Execute the remaining segment on this thread. + // + + MlasSgemmWorkCallback(nullptr, &WorkBlock, WorkObject); + + // + // Wait for the worker threads to complete. + // + + WaitForThreadpoolWorkCallbacks(WorkObject, FALSE); + CloseThreadpoolWork(WorkObject); + +#endif + + return true; + +#else + + // + // No threading implementation is available. + // + + MLAS_UNREFERENCED_PARAMETER(TransA); + MLAS_UNREFERENCED_PARAMETER(TransB); + MLAS_UNREFERENCED_PARAMETER(M); + MLAS_UNREFERENCED_PARAMETER(N); + MLAS_UNREFERENCED_PARAMETER(K); + MLAS_UNREFERENCED_PARAMETER(alpha); + MLAS_UNREFERENCED_PARAMETER(A); + MLAS_UNREFERENCED_PARAMETER(lda); + MLAS_UNREFERENCED_PARAMETER(B); + MLAS_UNREFERENCED_PARAMETER(ldb); + MLAS_UNREFERENCED_PARAMETER(beta); + MLAS_UNREFERENCED_PARAMETER(C); + MLAS_UNREFERENCED_PARAMETER(ldc); + + return false; + +#endif + +} + +void +MLASCALL +MlasSgemm( + CBLAS_TRANSPOSE TransA, + CBLAS_TRANSPOSE TransB, + size_t M, + size_t N, + size_t K, + float alpha, + const float* A, + size_t lda, + const float* B, + size_t ldb, + float beta, + float* C, + size_t ldc + ) +/*++ + +Routine Description: + + This routine implements the single precision matrix/matrix multiply + operation (SGEMM). + +Arguments: + + TransA - Supplies the transpose operation for matrix A. + + TransB - Supplies the transpose operation for matrix B. + + M - Supplies the number of rows of matrix A and matrix C. + + N - Supplies the number of columns of matrix B and matrix C. + + K - Supplies the number of columns of matrix A and the number of rows of + matrix B. + + alpha - Supplies the scaler alpha multiplier (see SGEMM definition). + + A - Supplies the address of matrix A. + + lda - Supplies the first dimension of matrix A. + + B - Supplies the address of matrix B. + + ldb - Supplies the first dimension of matrix B. + + beta - Supplies the scaler beta multiplier (see SGEMM definition). + + C - Supplies the address of matrix C. + + ldc - Supplies the first dimension of matrix C. + +Return Value: + + None. + +--*/ +{ + // + // Try to run the operation across multiple threads or fall back to a + // single thread based on the GEMM parameters and system configuration. + // + + if (!MlasSgemmTryMultithread(TransA, TransB, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc)) { + MlasSgemmOperation(TransA, TransB, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); + } +} diff --git a/onnxruntime/core/mlas/lib/x86_64/SgemmKernelAvx.S b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelAvx.S new file mode 100644 index 0000000000000..63c6d5d2c837e --- /dev/null +++ b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelAvx.S @@ -0,0 +1,557 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + SgemmKernelAvx.s + +Abstract: + + This module implements the kernels for the single precision matrix/matrix + multiply operation (SGEMM). + + This implementation uses AVX instructions. + +--*/ + +#include "asmmacro.h" + + .intel_syntax noprefix + + .equ SgemmKernelFrame_alpha, -8 + .equ SgemmKernelFrame_SavedRbx, 0 + .equ SgemmKernelFrame_SavedRbp, 8 + .equ SgemmKernelFrame_ReturnAddress, 16 + .equ SgemmKernelFrame_lda, 24 + .equ SgemmKernelFrame_ldc, 32 + + .text + +/*++ + +Macro Description: + + This macro multiplies and accumulates for a 16xN block (where N is 1,2,4) + of the output matrix. + +Arguments: + + Count - Supplies the number of rows to access from matrix A. + + VectorOffset - Supplies the byte offset from matrix B to fetch elements. + + BroadcastOffset - Supplies the byte offset from matrix A to fetch elements. + +Implicit Arguments: + + rdi - Supplies the address into the matrix A data. + + rbx - Supplies the address into the matrix A data plus 2 rows. + + rsi - Supplies the address into the matrix B data. + + r10 - Supplies the length in bytes of a row from matrix A. + + ymm8-ymm15 - Supplies the block accumulators. + +--*/ + + .macro ComputeBlockAvxBy16 Count, VectorOffset, BroadcastOffset + +.if \Count\() == 1 + vbroadcastss ymm3,DWORD PTR [rdi+\BroadcastOffset\()] + vmulps ymm4,ymm3,YMMWORD PTR [rsi+\VectorOffset\()] + vaddps ymm8,ymm8,ymm4 + vmulps ymm5,ymm3,YMMWORD PTR [rsi+\VectorOffset\()+32] + vaddps ymm9,ymm9,ymm5 +.else + vmovaps ymm0,YMMWORD PTR [rsi+\VectorOffset\()] + vmovaps ymm1,YMMWORD PTR [rsi+\VectorOffset\()+32] + vbroadcastss ymm3,DWORD PTR [rdi+\BroadcastOffset\()] + vmulps ymm4,ymm3,ymm0 + vaddps ymm8,ymm8,ymm4 + vmulps ymm5,ymm3,ymm1 + vaddps ymm9,ymm9,ymm5 +.if \Count\() >= 2 + vbroadcastss ymm3,DWORD PTR [rdi+r10+\BroadcastOffset\()] + vmulps ymm6,ymm3,ymm0 + vaddps ymm10,ymm10,ymm6 + vmulps ymm7,ymm3,ymm1 + vaddps ymm11,ymm11,ymm7 +.endif +.if \Count\() >= 4 + vbroadcastss ymm3,DWORD PTR [rbx+\BroadcastOffset\()] + vmulps ymm4,ymm3,ymm0 + vaddps ymm12,ymm12,ymm4 + vmulps ymm5,ymm3,ymm1 + vaddps ymm13,ymm13,ymm5 + vbroadcastss ymm3,DWORD PTR [rbx+r10+\BroadcastOffset\()] + vmulps ymm6,ymm3,ymm0 + vaddps ymm14,ymm14,ymm6 + vmulps ymm7,ymm3,ymm1 + vaddps ymm15,ymm15,ymm7 +.endif +.endif + + .endm + +/*++ + +Macro Description: + + This macro multiplies and accumulates for a 8xN block (where N is 1,2,4) + of the output matrix. + +Arguments: + + Count - Supplies the number of rows to access from matrix A. + + VectorOffset - Supplies the byte offset from matrix B to fetch elements. + + BroadcastOffset - Supplies the byte offset from matrix A to fetch elements. + +Implicit Arguments: + + rdi - Supplies the address into the matrix A data. + + rbx - Supplies the address into the matrix A data plus 2 rows. + + rsi - Supplies the address into the matrix B data. + + r10 - Supplies the length in bytes of a row from matrix A. + + ymm8-ymm15 - Supplies the block accumulators. + +--*/ + + .macro ComputeBlockAvxBy8 Count, VectorOffset, BroadcastOffset + +.if \Count\() == 1 + vbroadcastss ymm3,DWORD PTR [rdi+\BroadcastOffset\()] + vmulps ymm5,ymm3,YMMWORD PTR [rsi+\VectorOffset\()] + vaddps ymm9,ymm9,ymm5 +.else + vmovaps ymm0,YMMWORD PTR [rsi+\VectorOffset\()] + vbroadcastss ymm3,DWORD PTR [rdi+\BroadcastOffset\()] + vmulps ymm5,ymm3,ymm0 + vaddps ymm9,ymm9,ymm5 +.if \Count\() >= 2 + vbroadcastss ymm3,DWORD PTR [rdi+r10+\BroadcastOffset\()] + vmulps ymm7,ymm3,ymm0 + vaddps ymm11,ymm11,ymm7 +.endif +.if \Count\() >= 4 + vbroadcastss ymm3,DWORD PTR [rbx+\BroadcastOffset\()] + vmulps ymm5,ymm3,ymm0 + vaddps ymm13,ymm13,ymm5 + vbroadcastss ymm3,DWORD PTR [rbx+r10+\BroadcastOffset\()] + vmulps ymm7,ymm3,ymm0 + vaddps ymm15,ymm15,ymm7 +.endif +.endif + + .endm + +/*++ + +Macro Description: + + This macro generates code to execute the block compute macro multiple + times and advancing the matrix A and matrix B data pointers. + +Arguments: + + ComputeBlock - Supplies the macro to compute a single block. + + Count - Supplies the number of rows to access from matrix A. + +Implicit Arguments: + + rdi - Supplies the address into the matrix A data. + + rbx - Supplies the address into the matrix A data plus 2 rows. + + rsi - Supplies the address into the matrix B data. + + rcx - Supplies the number of columns from matrix A and the number of rows + from matrix B to iterate over. + + ymm4-ymm15 - Supplies the block accumulators. + +--*/ + + .macro ComputeBlockAvxLoop Mode, ComputeBlock, Count + + mov rbp,rcx # reload CountK + sub rbp,4 + jb .L\Mode\().\ComputeBlock\().\Count\().ProcessRemainingBlocks + +.L\Mode\().\ComputeBlock\().\Count\().ComputeBlockBy4Loop: + \ComputeBlock\() \Count\(), 0, 0 + \ComputeBlock\() \Count\(), 16*4, 4 + sub rsi,-32*4 # advance matrix B by 32 columns + \ComputeBlock\() \Count\(), 0, 8 + \ComputeBlock\() \Count\(), 16*4, 12 + sub rsi,-32*4 # advance matrix B by 32 columns + add rdi,4*4 # advance matrix A by 4 columns +.if \Count\() > 2 + add rbx,4*4 # advance matrix A plus rows by 4 columns +.endif + sub rbp,4 + jae .L\Mode\().\ComputeBlock\().\Count\().ComputeBlockBy4Loop + +.L\Mode\().\ComputeBlock\().\Count\().ProcessRemainingBlocks: + add rbp,4 # correct for over-subtract above + jz .L\Mode\().\ComputeBlock\().\Count\().OutputBlock + +.L\Mode\().\ComputeBlock\().\Count\().ComputeBlockBy1Loop: + \ComputeBlock\() \Count\(), 0, 0 + add rsi,16*4 # advance matrix B by 16 columns + add rdi,4 # advance matrix A by 1 column +.if \Count\() > 2 + add rbx,4 # advance matrix A plus rows by 1 column +.endif + dec rbp + jne .L\Mode\().\ComputeBlock\().\Count\().ComputeBlockBy1Loop + +.L\Mode\().\ComputeBlock\().\Count\().OutputBlock: + + .endm + +/*++ + +Routine Description: + + This routine is an inner kernel to compute matrix multiplication for a + set of rows. + +Arguments: + + A (rdi) - Supplies the address of matrix A. + + B (rsi) - Supplies the address of matrix B. The matrix data has been packed + using MlasSgemmCopyPackB or MlasSgemmTransposePackB. + + C (rdx) - Supplies the address of matrix C. + + CountK (rcx) - Supplies the number of columns from matrix A and the number + of rows from matrix B to iterate over. + + CountM (r8) - Supplies the maximum number of rows that can be processed for + matrix A and matrix C. The actual number of rows handled for this + invocation depends on the kernel implementation. + + CountN (r9) - Supplies the number of columns from matrix B and matrix C to + iterate over. + + lda - Supplies the first dimension of matrix A. + + ldc - Supplies the first dimension of matrix C. + + Alpha (xmm0) - Supplies the scaler multiplier (see SGEMM definition). + +Return Value: + + Returns the number of rows handled. + +--*/ + + .macro SgemmKernelAvxFunction Mode + + .globl C_UNDERSCORE(MlasSgemmKernel\Mode\()Avx) +C_UNDERSCORE(MlasSgemmKernel\Mode\()Avx): + + push rbp + push rbx + mov r11,rdi + mov r10,[rsp+SgemmKernelFrame_lda] + shl r10,2 # convert lda to bytes + mov rax,[rsp+SgemmKernelFrame_ldc] + shl rax,2 # convert ldc to bytes + vmovss DWORD PTR [rsp+SgemmKernelFrame_alpha],xmm0 + vbroadcastss ymm2,DWORD PTR [rsp+SgemmKernelFrame_alpha] + +// +// Process 4 rows of the matrices. +// + + cmp r8,4 + jb .L\Mode\().ProcessCountMLessThan4 + mov r8d,4 # return 4 rows handled + cmp r9,8 + jbe .L\Mode\().ProcessRemainingCountN4 + +.L\Mode\().ProcessNextColumnLoop16x4: + vxorps xmm8,xmm8,xmm8 # clear block accumulators + vxorps xmm9,xmm9,xmm9 + vxorps xmm10,xmm10,xmm10 + vxorps xmm11,xmm11,xmm11 + vxorps xmm12,xmm12,xmm12 + vxorps xmm13,xmm13,xmm13 + vxorps xmm14,xmm14,xmm14 + vxorps xmm15,xmm15,xmm15 + lea rbx,[rdi+r10*2] # compute matrix A plus 2 rows + ComputeBlockAvxLoop \Mode\(), ComputeBlockAvxBy16, 4 + vmulps ymm8,ymm8,ymm2 # multiply by alpha + vmulps ymm9,ymm9,ymm2 + vmulps ymm10,ymm10,ymm2 + vmulps ymm11,ymm11,ymm2 + vmulps ymm12,ymm12,ymm2 + vmulps ymm13,ymm13,ymm2 + vmulps ymm14,ymm14,ymm2 + vmulps ymm15,ymm15,ymm2 + lea rdi,[rdx+rax*2] # compute matrix C plus 2 rows + sub r9,16 + jb .L\Mode\().OutputMasked16x4Block +.ifeqs "\Mode\()","Add" + vaddps ymm8,ymm8,YMMWORD PTR [rdx] + vaddps ymm9,ymm9,YMMWORD PTR [rdx+32] + vaddps ymm10,ymm10,YMMWORD PTR [rdx+rax] + vaddps ymm11,ymm11,YMMWORD PTR [rdx+rax+32] + vaddps ymm12,ymm12,YMMWORD PTR [rdi] + vaddps ymm13,ymm13,YMMWORD PTR [rdi+32] + vaddps ymm14,ymm14,YMMWORD PTR [rdi+rax] + vaddps ymm15,ymm15,YMMWORD PTR [rdi+rax+32] +.endif + vmovups YMMWORD PTR [rdx],ymm8 + vmovups YMMWORD PTR [rdx+32],ymm9 + vmovups YMMWORD PTR [rdx+rax],ymm10 + vmovups YMMWORD PTR [rdx+rax+32],ymm11 + vmovups YMMWORD PTR [rdi],ymm12 + vmovups YMMWORD PTR [rdi+32],ymm13 + vmovups YMMWORD PTR [rdi+rax],ymm14 + vmovups YMMWORD PTR [rdi+rax+32],ymm15 + add rdx,16*4 # advance matrix C by 16 columns + mov rdi,r11 # reload matrix A + cmp r9,8 + ja .L\Mode\().ProcessNextColumnLoop16x4 + test r9,r9 + jz .L\Mode\().ExitKernel + +.L\Mode\().ProcessRemainingCountN4: + vxorps xmm9,xmm9,xmm9 # clear block accumulators + vxorps xmm11,xmm11,xmm11 + vxorps xmm13,xmm13,xmm13 + vxorps xmm15,xmm15,xmm15 + lea rbx,[rdi+r10*2] # compute matrix A plus 2 rows + ComputeBlockAvxLoop \Mode\(), ComputeBlockAvxBy8, 4 + vmulps ymm9,ymm9,ymm2 # multiply by alpha + vmulps ymm11,ymm11,ymm2 + vmulps ymm13,ymm13,ymm2 + vmulps ymm15,ymm15,ymm2 + lea rdi,[rdx+rax*2] # compute matrix C plus 2 rows + cmp r9,8 + jb .L\Mode\().OutputMasked8x4Block +.ifeqs "\Mode\()","Add" + vaddps ymm9,ymm9,YMMWORD PTR [rdx] + vaddps ymm11,ymm11,YMMWORD PTR [rdx+rax] + vaddps ymm13,ymm13,YMMWORD PTR [rdi] + vaddps ymm15,ymm15,YMMWORD PTR [rdi+rax] +.endif + vmovups YMMWORD PTR [rdx],ymm9 + vmovups YMMWORD PTR [rdx+rax],ymm11 + vmovups YMMWORD PTR [rdi],ymm13 + vmovups YMMWORD PTR [rdi+rax],ymm15 + jmp .L\Mode\().ExitKernel + +.L\Mode\().OutputMasked16x4Block: +.ifeqs "\Mode\()","Add" + vaddps ymm8,ymm8,YMMWORD PTR [rdx] + vaddps ymm10,ymm10,YMMWORD PTR [rdx+rax] + vaddps ymm12,ymm12,YMMWORD PTR [rdi] + vaddps ymm14,ymm14,YMMWORD PTR [rdi+rax] +.endif + vmovups YMMWORD PTR [rdx],ymm8 + vmovups YMMWORD PTR [rdx+rax],ymm10 + vmovups YMMWORD PTR [rdi],ymm12 + vmovups YMMWORD PTR [rdi+rax],ymm14 + add rdx,8*4 # advance matrix C by 8 columns + add rdi,8*4 # advance matrix C plus 2 rows by 8 columns + add r9,8 # correct for over-subtract above + +.L\Mode\().OutputMasked8x4Block: + vmovd xmm0,r9d + vshufps xmm0,xmm0,xmm0,0 + vpcmpgtd xmm1,xmm0,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip+16] + vpcmpgtd xmm0,xmm0,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] + vinsertf128 ymm0,ymm0,xmm1,1 +.ifeqs "\Mode\()","Add" + vmaskmovps ymm8,ymm0,YMMWORD PTR [rdx] + vmaskmovps ymm10,ymm0,YMMWORD PTR [rdx+rax] + vmaskmovps ymm12,ymm0,YMMWORD PTR [rdi] + vmaskmovps ymm14,ymm0,YMMWORD PTR [rdi+rax] + vaddps ymm9,ymm9,ymm8 + vaddps ymm11,ymm11,ymm10 + vaddps ymm13,ymm13,ymm12 + vaddps ymm15,ymm15,ymm14 +.endif + vmaskmovps YMMWORD PTR [rdx],ymm0,ymm9 + vmaskmovps YMMWORD PTR [rdx+rax],ymm0,ymm11 + vmaskmovps YMMWORD PTR [rdi],ymm0,ymm13 + vmaskmovps YMMWORD PTR [rdi+rax],ymm0,ymm15 + +// +// Restore non-volatile registers and return. +// + +.L\Mode\().ExitKernel: + vzeroupper + mov eax,r8d + pop rbx + pop rbp + ret + +// +// Process 2 rows of the matrices. +// + +.L\Mode\().ProcessCountMLessThan4: + cmp r8,2 + jb .L\Mode\().ProcessCountMLessThan2 + mov r8d,2 # return 2 rows handled + cmp r9,8 + jbe .L\Mode\().ProcessRemainingCountN2 + +.L\Mode\().ProcessNextColumnLoop16x2: + vxorps xmm8,xmm8,xmm8 # clear block accumulators + vxorps xmm9,xmm9,xmm9 + vxorps xmm10,xmm10,xmm10 + vxorps xmm11,xmm11,xmm11 + ComputeBlockAvxLoop \Mode\(), ComputeBlockAvxBy16, 2 + vmulps ymm8,ymm8,ymm2 # multiply by alpha + vmulps ymm9,ymm9,ymm2 + vmulps ymm10,ymm10,ymm2 + vmulps ymm11,ymm11,ymm2 + sub r9,16 + jb .L\Mode\().OutputMasked16x2Block +.ifeqs "\Mode\()","Add" + vaddps ymm8,ymm8,YMMWORD PTR [rdx] + vaddps ymm9,ymm9,YMMWORD PTR [rdx+32] + vaddps ymm10,ymm10,YMMWORD PTR [rdx+rax] + vaddps ymm11,ymm11,YMMWORD PTR [rdx+rax+32] +.endif + vmovups YMMWORD PTR [rdx],ymm8 + vmovups YMMWORD PTR [rdx+32],ymm9 + vmovups YMMWORD PTR [rdx+rax],ymm10 + vmovups YMMWORD PTR [rdx+rax+32],ymm11 + add rdx,16*4 # advance matrix C by 16 columns + mov rdi,r11 # reload matrix A + cmp r9,8 + ja .L\Mode\().ProcessNextColumnLoop16x2 + test r9,r9 + jz .L\Mode\().ExitKernel + +.L\Mode\().ProcessRemainingCountN2: + vxorps xmm9,xmm9,xmm9 # clear block accumulators + vxorps xmm11,xmm11,xmm11 + ComputeBlockAvxLoop \Mode\(), ComputeBlockAvxBy8, 2 + vmulps ymm9,ymm9,ymm2 # multiply by alpha + vmulps ymm11,ymm11,ymm2 + cmp r9,8 + jb .L\Mode\().OutputMasked8x2Block +.ifeqs "\Mode\()","Add" + vaddps ymm9,ymm9,YMMWORD PTR [rdx] + vaddps ymm11,ymm11,YMMWORD PTR [rdx+rax] +.endif + vmovups YMMWORD PTR [rdx],ymm9 + vmovups YMMWORD PTR [rdx+rax],ymm11 + jmp .L\Mode\().ExitKernel + +.L\Mode\().OutputMasked16x2Block: +.ifeqs "\Mode\()","Add" + vaddps ymm8,ymm8,YMMWORD PTR [rdx] + vaddps ymm10,ymm10,YMMWORD PTR [rdx+rax] +.endif + vmovups YMMWORD PTR [rdx],ymm8 + vmovups YMMWORD PTR [rdx+rax],ymm10 + add rdx,8*4 # advance matrix C by 8 columns + add r9,8 # correct for over-subtract above + +.L\Mode\().OutputMasked8x2Block: + vmovd xmm0,r9d + vshufps xmm0,xmm0,xmm0,0 + vpcmpgtd xmm1,xmm0,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip+16] + vpcmpgtd xmm0,xmm0,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] + vinsertf128 ymm0,ymm0,xmm1,1 +.ifeqs "\Mode\()","Add" + vmaskmovps ymm8,ymm0,YMMWORD PTR [rdx] + vmaskmovps ymm10,ymm0,YMMWORD PTR [rdx+rax] + vaddps ymm9,ymm9,ymm8 + vaddps ymm11,ymm11,ymm10 +.endif + vmaskmovps YMMWORD PTR [rdx],ymm0,ymm9 + vmaskmovps YMMWORD PTR [rdx+rax],ymm0,ymm11 + jmp .L\Mode\().ExitKernel + +// +// Process 1 row of the matrices. +// + +.L\Mode\().ProcessCountMLessThan2: + mov r8d,1 # return 1 row handled + cmp r9,8 + jbe .L\Mode\().ProcessRemainingCountN1 + +.L\Mode\().ProcessNextColumnLoop16x1: + vxorps xmm8,xmm8,xmm8 # clear block accumulators + vxorps xmm9,xmm9,xmm9 + ComputeBlockAvxLoop \Mode\(), ComputeBlockAvxBy16, 1 + vmulps ymm8,ymm8,ymm2 # multiply by alpha + vmulps ymm9,ymm9,ymm2 + sub r9,16 + jb .L\Mode\().OutputMasked16x1Block +.ifeqs "\Mode\()","Add" + vaddps ymm8,ymm8,YMMWORD PTR [rdx] + vaddps ymm9,ymm9,YMMWORD PTR [rdx+32] +.endif + vmovups YMMWORD PTR [rdx],ymm8 + vmovups YMMWORD PTR [rdx+32],ymm9 + add rdx,16*4 # advance matrix C by 16 columns + mov rdi,r11 # reload matrix A + cmp r9,8 + ja .L\Mode\().ProcessNextColumnLoop16x1 + test r9,r9 + jz .L\Mode\().ExitKernel + +.L\Mode\().ProcessRemainingCountN1: + vxorps xmm9,xmm9,xmm9 # clear block accumulators + ComputeBlockAvxLoop \Mode\(), ComputeBlockAvxBy8, 1 + vmulps ymm9,ymm9,ymm2 # multiply by alpha + cmp r9,8 + jb .L\Mode\().OutputMasked8x1Block +.ifeqs "\Mode\()","Add" + vaddps ymm9,ymm9,YMMWORD PTR [rdx] +.endif + vmovups YMMWORD PTR [rdx],ymm9 + jmp .L\Mode\().ExitKernel + +.L\Mode\().OutputMasked16x1Block: +.ifeqs "\Mode\()","Add" + vaddps ymm8,ymm8,YMMWORD PTR [rdx] +.endif + vmovups YMMWORD PTR [rdx],ymm8 + add rdx,8*4 # advance matrix C by 8 columns + add r9,8 # correct for over-subtract above + +.L\Mode\().OutputMasked8x1Block: + vmovd xmm0,r9d + vshufps xmm0,xmm0,xmm0,0 + vpcmpgtd xmm1,xmm0,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip+16] + vpcmpgtd xmm0,xmm0,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] + vinsertf128 ymm0,ymm0,xmm1,1 +.ifeqs "\Mode\()","Add" + vmaskmovps ymm8,ymm0,YMMWORD PTR [rdx] + vaddps ymm9,ymm9,ymm8 +.endif + vmaskmovps YMMWORD PTR [rdx],ymm0,ymm9 + jmp .L\Mode\().ExitKernel + + .endm + + SgemmKernelAvxFunction Zero + SgemmKernelAvxFunction Add + + .end diff --git a/onnxruntime/core/mlas/lib/x86_64/SgemmKernelAvx512F.S b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelAvx512F.S new file mode 100644 index 0000000000000..e4728ba343dae --- /dev/null +++ b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelAvx512F.S @@ -0,0 +1,693 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + SgemmKernelAvx512F.s + +Abstract: + + This module implements the kernels for the single precision matrix/matrix + multiply operation (SGEMM). + + This implementation uses AVX512F instructions. + +--*/ + +#include "asmmacro.h" + + .intel_syntax noprefix + + .equ SgemmKernelFrame_alpha, -8 + .equ SgemmKernelFrame_SavedR14, 0 + .equ SgemmKernelFrame_SavedR13, 8 + .equ SgemmKernelFrame_SavedR12, 16 + .equ SgemmKernelFrame_SavedRbx, 24 + .equ SgemmKernelFrame_SavedRbp, 32 + .equ SgemmKernelFrame_ReturnAddress, 40 + .equ SgemmKernelFrame_lda, 48 + .equ SgemmKernelFrame_ldc, 56 + + .text + +/*++ + +Macro Description: + + This macro multiplies and accumulates for a 32xN block (where N is 1,3,6,12) + of the output matrix. + +Arguments: + + Count - Supplies the number of rows to access from matrix A. + + VectorOffset - Supplies the byte offset from matrix B to fetch elements. + + BroadcastOffset - Supplies the byte offset from matrix A to fetch elements. + +Implicit Arguments: + + rcx - Supplies the address into the matrix A data. + + rbx - Supplies the address into the matrix A data plus 3 rows. + + r13 - Supplies the address into the matrix A data plus 6 rows. + + r14 - Supplies the address into the matrix A data plus 9 rows. + + rdx - Supplies the address into the matrix B data. + + r10 - Supplies the length in bytes of a row from matrix A. + + zmm4-zmm27 - Supplies the block accumulators. + +--*/ + + .macro ComputeBlockAvx512FBy32 Count, VectorOffset, BroadcastOffset + +.if \Count\() == 1 + vbroadcastss zmm3,DWORD PTR [rdi+\BroadcastOffset\()] + vfmadd231ps zmm4,zmm3,ZMMWORD PTR [rsi+\VectorOffset\()] + vfmadd231ps zmm5,zmm3,ZMMWORD PTR [rsi+r12+\VectorOffset\()] +.else + vmovaps zmm0,ZMMWORD PTR [rsi+\VectorOffset\()] + vmovaps zmm1,ZMMWORD PTR [rsi+r12+\VectorOffset\()] + vbroadcastss zmm3,DWORD PTR [rdi+\BroadcastOffset\()] + vfmadd231ps zmm4,zmm3,zmm0 + vfmadd231ps zmm5,zmm3,zmm1 +.if \Count\() >= 3 + vbroadcastss zmm3,DWORD PTR [rdi+r10+\BroadcastOffset\()] + vfmadd231ps zmm6,zmm3,zmm0 + vfmadd231ps zmm7,zmm3,zmm1 + vbroadcastss zmm3,DWORD PTR [rdi+r10*2+\BroadcastOffset\()] + vfmadd231ps zmm8,zmm3,zmm0 + vfmadd231ps zmm9,zmm3,zmm1 +.endif +.if \Count\() >= 6 + vbroadcastss zmm3,DWORD PTR [rbx+\BroadcastOffset\()] + vfmadd231ps zmm10,zmm3,zmm0 + vfmadd231ps zmm11,zmm3,zmm1 + vbroadcastss zmm3,DWORD PTR [rbx+r10+\BroadcastOffset\()] + vfmadd231ps zmm12,zmm3,zmm0 + vfmadd231ps zmm13,zmm3,zmm1 + vbroadcastss zmm3,DWORD PTR [rbx+r10*2+\BroadcastOffset\()] + vfmadd231ps zmm14,zmm3,zmm0 + vfmadd231ps zmm15,zmm3,zmm1 +.endif +.if \Count\() >= 12 + vbroadcastss zmm3,DWORD PTR [r13+\BroadcastOffset\()] + vfmadd231ps zmm16,zmm3,zmm0 + vfmadd231ps zmm17,zmm3,zmm1 + vbroadcastss zmm3,DWORD PTR [r13+r10+\BroadcastOffset\()] + vfmadd231ps zmm18,zmm3,zmm0 + vfmadd231ps zmm19,zmm3,zmm1 + vbroadcastss zmm3,DWORD PTR [r13+r10*2+\BroadcastOffset\()] + vfmadd231ps zmm20,zmm3,zmm0 + vfmadd231ps zmm21,zmm3,zmm1 + vbroadcastss zmm3,DWORD PTR [r14+\BroadcastOffset\()] + vfmadd231ps zmm22,zmm3,zmm0 + vfmadd231ps zmm23,zmm3,zmm1 + vbroadcastss zmm3,DWORD PTR [r14+r10+\BroadcastOffset\()] + vfmadd231ps zmm24,zmm3,zmm0 + vfmadd231ps zmm25,zmm3,zmm1 + vbroadcastss zmm3,DWORD PTR [r14+r10*2+\BroadcastOffset\()] + vfmadd231ps zmm26,zmm3,zmm0 + vfmadd231ps zmm27,zmm3,zmm1 +.endif +.endif + + .endm + +/*++ + +Macro Description: + + This macro multiplies and accumulates for a 16xN block (where N is 1,3,6,12) + of the output matrix. + +Arguments: + + Count - Supplies the number of rows to access from matrix A. + + VectorOffset - Supplies the byte offset from matrix B to fetch elements. + + BroadcastOffset - Supplies the byte offset from matrix A to fetch elements. + +Implicit Arguments: + + rdi - Supplies the address into the matrix A data. + + rbx - Supplies the address into the matrix A data plus 3 rows. + + r13 - Supplies the address into the matrix A data plus 6 rows. + + r14 - Supplies the address into the matrix A data plus 9 rows. + + rsi - Supplies the address into the matrix B data. + + r10 - Supplies the length in bytes of a row from matrix A. + + zmm4-zmm27 - Supplies the block accumulators. + +--*/ + + .macro ComputeBlockAvx512FBy16 Count, VectorOffset, BroadcastOffset + + vmovaps zmm0,ZMMWORD PTR [rsi+\VectorOffset\()] + vfmadd231ps zmm5,zmm0,DWORD PTR [rdi+\BroadcastOffset\()]{1to16} +.if \Count\() >= 3 + vfmadd231ps zmm7,zmm0,DWORD PTR [rdi+r10+\BroadcastOffset\()]{1to16} + vfmadd231ps zmm9,zmm0,DWORD PTR [rdi+r10*2+\BroadcastOffset\()]{1to16} +.endif +.if \Count\() >= 6 + vfmadd231ps zmm11,zmm0,DWORD PTR [rbx+\BroadcastOffset\()]{1to16} + vfmadd231ps zmm13,zmm0,DWORD PTR [rbx+r10+\BroadcastOffset\()]{1to16} + vfmadd231ps zmm15,zmm0,DWORD PTR [rbx+r10*2+\BroadcastOffset\()]{1to16} +.endif +.if \Count\() >= 12 + vfmadd231ps zmm17,zmm0,DWORD PTR [r13+\BroadcastOffset\()]{1to16} + vfmadd231ps zmm19,zmm0,DWORD PTR [r13+r10+\BroadcastOffset\()]{1to16} + vfmadd231ps zmm21,zmm0,DWORD PTR [r13+r10*2+\BroadcastOffset\()]{1to16} + vfmadd231ps zmm23,zmm0,DWORD PTR [r14+\BroadcastOffset\()]{1to16} + vfmadd231ps zmm25,zmm0,DWORD PTR [r14+r10+\BroadcastOffset\()]{1to16} + vfmadd231ps zmm27,zmm0,DWORD PTR [r14+r10*2+\BroadcastOffset\()]{1to16} +.endif + + .endm + +/*++ + +Macro Description: + + This macro generates code to execute the block compute macro multiple + times and advancing the matrix A and matrix B data pointers. + +Arguments: + + ComputeBlock - Supplies the macro to compute a single block. + + Count - Supplies the number of rows to access from matrix A. + +Implicit Arguments: + + rdi - Supplies the address into the matrix A data. + + rbx - Supplies the address into the matrix A data plus 3 rows. + + r13 - Supplies the address into the matrix A data plus 6 rows. + + r14 - Supplies the address into the matrix A data plus 9 rows. + + rsi - Supplies the address into the matrix B data. + + rcx - Supplies the number of columns from matrix A and the number of rows + from matrix B to iterate over. + + zmm4-zmm27 - Supplies the block accumulators. + +--*/ + + .macro ComputeBlockAvx512FLoop Mode, ComputeBlock, Count + + mov rbp,rcx # reload CountK + sub rbp,4 + jb .L\Mode\().\ComputeBlock\().\Count\().ProcessRemainingBlocks + +.L\Mode\().\ComputeBlock\().\Count\().ComputeBlockBy4Loop: + \ComputeBlock\() \Count\(), 0, 0 + \ComputeBlock\() \Count\(), 16*4, 4 + sub rsi,-32*4 # advance matrix B by 32 columns + \ComputeBlock\() \Count\(), 0, 8 + \ComputeBlock\() \Count\(), 16*4, 12 + sub rsi,-32*4 # advance matrix B by 32 columns + add rdi,4*4 # advance matrix A by 4 columns +.if \Count\() > 3 + add rbx,4*4 # advance matrix A plus rows by 4 columns +.if \Count\() == 12 + add r13,4*4 + add r14,4*4 +.endif +.endif + sub rbp,4 + jae .L\Mode\().\ComputeBlock\().\Count\().ComputeBlockBy4Loop + +.L\Mode\().\ComputeBlock\().\Count\().ProcessRemainingBlocks: + add rbp,4 # correct for over-subtract above + jz .L\Mode\().\ComputeBlock\().\Count\().OutputBlock + +.L\Mode\().\ComputeBlock\().\Count\().ComputeBlockBy1Loop: + \ComputeBlock\() \Count\(), 0, 0 + add rsi,16*4 # advance matrix B by 16 columns + add rdi,4 # advance matrix A by 1 column +.if \Count\() > 3 + add rbx,4 # advance matrix A plus rows by 1 column +.if \Count\() == 12 + add r13,4 + add r14,4 +.endif +.endif + dec rbp + jne .L\Mode\().\ComputeBlock\().\Count\().ComputeBlockBy1Loop + +.L\Mode\().\ComputeBlock\().\Count\().OutputBlock: + + .endm + +/*++ + +Routine Description: + + This routine is an inner kernel to compute matrix multiplication for a + set of rows. + +Arguments: + + A (rdi) - Supplies the address of matrix A. + + B (rsi) - Supplies the address of matrix B. The matrix data has been packed + using MlasSgemmCopyPackB or MlasSgemmTransposePackB. + + C (rdx) - Supplies the address of matrix C. + + CountK (rcx) - Supplies the number of columns from matrix A and the number + of rows from matrix B to iterate over. + + CountM (r8) - Supplies the maximum number of rows that can be processed for + matrix A and matrix C. The actual number of rows handled for this + invocation depends on the kernel implementation. + + CountN (r9) - Supplies the number of columns from matrix B and matrix C to + iterate over. + + lda - Supplies the first dimension of matrix A. + + ldc - Supplies the first dimension of matrix C. + + Alpha (xmm0) - Supplies the scaler multiplier (see SGEMM definition). + +Return Value: + + Returns the number of rows handled. + +--*/ + + .macro SgemmKernelAvx512FFunction Mode + + .globl C_UNDERSCORE(MlasSgemmKernel\Mode\()Avx512F) +C_UNDERSCORE(MlasSgemmKernel\Mode\()Avx512F): + + push rbp + push rbx + push r12 + push r13 + push r14 + mov r11,rdi + mov r10,[rsp+SgemmKernelFrame_lda] + shl r10,2 # convert lda to bytes + mov rax,[rsp+SgemmKernelFrame_ldc] + shl rax,2 # convert ldc to bytes + mov r12,rcx + shl r12,6 # compute 16*CountK*sizeof(float) + mov ebx,-1 + kmovw k1,ebx # update mask to write all columns + vmovss DWORD PTR [rsp+SgemmKernelFrame_alpha],xmm0 + vzeroall + vbroadcastss zmm31,DWORD PTR [rsp+SgemmKernelFrame_alpha] + +// +// Process 12 rows of the matrices. +// + + cmp r8,12 + jb .L\Mode\().ProcessCountMLessThan12 + mov r8d,12 # return 12 rows handled + cmp r9,16 + jbe .L\Mode\().ProcessRemainingCountN12 + +.L\Mode\().ProcessNextColumnLoop32x12: + vmovaps zmm16,zmm4 # clear upper block accumulators + vmovaps zmm17,zmm5 + vmovaps zmm18,zmm4 + vmovaps zmm19,zmm5 + vmovaps zmm20,zmm4 + vmovaps zmm21,zmm5 + vmovaps zmm22,zmm4 + vmovaps zmm23,zmm5 + vmovaps zmm24,zmm4 + vmovaps zmm25,zmm5 + vmovaps zmm26,zmm4 + vmovaps zmm27,zmm5 + lea rbx,[r10*2+r10] + lea r13,[rdi+rbx*2] # compute matrix A plus 6 rows + lea r14,[r13+rbx] # compute matrix A plus 9 rows + add rbx,rdi # compute matrix A plus 3 rows + ComputeBlockAvx512FLoop \Mode\(), ComputeBlockAvx512FBy32, 12 + add rsi,r12 # advance matrix B by 16*CountK floats + lea rbx,[rax*2+rax] + lea r13,[rdx+rbx*2] # compute matrix C plus 6 rows + lea r14,[r13+rbx] # compute matrix C plus 9 rows + add rbx,rdx # compute matrix C plus 3 rows +.ifnes "\Mode\()","Add" + vmulps zmm4,zmm4,zmm31 # multiply by alpha + vmulps zmm6,zmm6,zmm31 + vmulps zmm8,zmm8,zmm31 + vmulps zmm10,zmm10,zmm31 + vmulps zmm12,zmm12,zmm31 + vmulps zmm14,zmm14,zmm31 + vmulps zmm16,zmm16,zmm31 + vmulps zmm18,zmm18,zmm31 + vmulps zmm20,zmm20,zmm31 + vmulps zmm22,zmm22,zmm31 + vmulps zmm24,zmm24,zmm31 + vmulps zmm26,zmm26,zmm31 +.else + vfmadd213ps zmm4,zmm31,ZMMWORD PTR [rdx] + vfmadd213ps zmm6,zmm31,ZMMWORD PTR [rdx+rax] + vfmadd213ps zmm8,zmm31,ZMMWORD PTR [rdx+rax*2] + vfmadd213ps zmm10,zmm31,ZMMWORD PTR [rbx] + vfmadd213ps zmm12,zmm31,ZMMWORD PTR [rbx+rax] + vfmadd213ps zmm14,zmm31,ZMMWORD PTR [rbx+rax*2] + vfmadd213ps zmm16,zmm31,ZMMWORD PTR [r13] + vfmadd213ps zmm18,zmm31,ZMMWORD PTR [r13+rax] + vfmadd213ps zmm20,zmm31,ZMMWORD PTR [r13+rax*2] + vfmadd213ps zmm22,zmm31,ZMMWORD PTR [r14] + vfmadd213ps zmm24,zmm31,ZMMWORD PTR [r14+rax] + vfmadd213ps zmm26,zmm31,ZMMWORD PTR [r14+rax*2] +.endif + vmovups ZMMWORD PTR [rdx],zmm4 + vmovups ZMMWORD PTR [rdx+rax],zmm6 + vmovups ZMMWORD PTR [rdx+rax*2],zmm8 + vmovups ZMMWORD PTR [rbx],zmm10 + vmovups ZMMWORD PTR [rbx+rax],zmm12 + vmovups ZMMWORD PTR [rbx+rax*2],zmm14 + vmovups ZMMWORD PTR [r13],zmm16 + vmovups ZMMWORD PTR [r13+rax],zmm18 + vmovups ZMMWORD PTR [r13+rax*2],zmm20 + vmovups ZMMWORD PTR [r14],zmm22 + vmovups ZMMWORD PTR [r14+rax],zmm24 + vmovups ZMMWORD PTR [r14+rax*2],zmm26 + add rdx,16*4 # advance matrix C by 16 columns + sub r9,16 + +.L\Mode\().Output16x12Block: + sub r9,16 + jae .L\Mode\().Output16x12BlockWithMask + lea rcx,[r9+16] # correct for over-subtract above + mov ebx,1 + shl ebx,cl + dec ebx + kmovw k1,ebx # update mask for remaining columns + xor r9,r9 # no more columns remaining + +.L\Mode\().Output16x12BlockWithMask: + lea rbx,[rax*2+rax] + lea r13,[rdx+rbx*2] # compute matrix C plus 6 rows + lea r14,[r13+rbx] # compute matrix C plus 9 rows + add rbx,rdx # compute matrix C plus 3 rows +.ifnes "\Mode\()","Add" + vmulps zmm5,zmm5,zmm31 # multiply by alpha + vmulps zmm7,zmm7,zmm31 + vmulps zmm9,zmm9,zmm31 + vmulps zmm11,zmm11,zmm31 + vmulps zmm13,zmm13,zmm31 + vmulps zmm15,zmm15,zmm31 + vmulps zmm17,zmm17,zmm31 + vmulps zmm19,zmm19,zmm31 + vmulps zmm21,zmm21,zmm31 + vmulps zmm23,zmm23,zmm31 + vmulps zmm25,zmm25,zmm31 + vmulps zmm27,zmm27,zmm31 +.else + vfmadd213ps zmm5{k1},zmm31,ZMMWORD PTR [rdx] + vfmadd213ps zmm7{k1},zmm31,ZMMWORD PTR [rdx+rax] + vfmadd213ps zmm9{k1},zmm31,ZMMWORD PTR [rdx+rax*2] + vfmadd213ps zmm11{k1},zmm31,ZMMWORD PTR [rbx] + vfmadd213ps zmm13{k1},zmm31,ZMMWORD PTR [rbx+rax] + vfmadd213ps zmm15{k1},zmm31,ZMMWORD PTR [rbx+rax*2] + vfmadd213ps zmm17{k1},zmm31,ZMMWORD PTR [r13] + vfmadd213ps zmm19{k1},zmm31,ZMMWORD PTR [r13+rax] + vfmadd213ps zmm21{k1},zmm31,ZMMWORD PTR [r13+rax*2] + vfmadd213ps zmm23{k1},zmm31,ZMMWORD PTR [r14] + vfmadd213ps zmm25{k1},zmm31,ZMMWORD PTR [r14+rax] + vfmadd213ps zmm27{k1},zmm31,ZMMWORD PTR [r14+rax*2] +.endif + vmovups ZMMWORD PTR [rdx]{k1},zmm5 + vmovups ZMMWORD PTR [rdx+rax]{k1},zmm7 + vmovups ZMMWORD PTR [rdx+rax*2]{k1},zmm9 + vmovups ZMMWORD PTR [rbx]{k1},zmm11 + vmovups ZMMWORD PTR [rbx+rax]{k1},zmm13 + vmovups ZMMWORD PTR [rbx+rax*2]{k1},zmm15 + vmovups ZMMWORD PTR [r13]{k1},zmm17 + vmovups ZMMWORD PTR [r13+rax]{k1},zmm19 + vmovups ZMMWORD PTR [r13+rax*2]{k1},zmm21 + vmovups ZMMWORD PTR [r14]{k1},zmm23 + vmovups ZMMWORD PTR [r14+rax]{k1},zmm25 + vmovups ZMMWORD PTR [r14+rax*2]{k1},zmm27 + add rdx,16*4 # advance matrix C by 16 columns + mov rdi,r11 # reload matrix A + vzeroall + cmp r9,16 + ja .L\Mode\().ProcessNextColumnLoop32x12 + test r9,r9 + jz .L\Mode\().ExitKernel + +.L\Mode\().ProcessRemainingCountN12: + vmovaps zmm17,zmm5 # clear upper block accumulators + vmovaps zmm19,zmm5 + vmovaps zmm21,zmm5 + vmovaps zmm23,zmm5 + vmovaps zmm25,zmm5 + vmovaps zmm27,zmm5 + lea rbx,[r10*2+r10] + lea r13,[rdi+rbx*2] # compute matrix A plus 6 rows + lea r14,[r13+rbx] # compute matrix A plus 9 rows + add rbx,rdi # compute matrix A plus 3 rows + ComputeBlockAvx512FLoop \Mode\(), ComputeBlockAvx512FBy16, 12 + jmp .L\Mode\().Output16x12Block + +// +// Restore non-volatile registers and return. +// + +.L\Mode\().ExitKernel: + mov eax,r8d + pop r14 + pop r13 + pop r12 + pop rbx + pop rbp + ret + +// +// Process 6 rows of the matrices. +// + +.L\Mode\().ProcessCountMLessThan12: + cmp r8,6 + jb .L\Mode\().ProcessCountMLessThan6 + mov r8d,6 # return 6 rows handled + cmp r9,16 + jbe .L\Mode\().ProcessRemainingCountN6 + +.L\Mode\().ProcessNextColumnLoop32x6: + lea rbx,[r10*2+r10] + add rbx,rdi # compute matrix A plus 3 rows + ComputeBlockAvx512FLoop \Mode\(), ComputeBlockAvx512FBy32, 6 + add rsi,r12 # advance matrix B by 16*CountK floats + lea rdi,[rdx+rax*2] # compute matrix C plus 2 rows + lea rbx,[rdx+rax*4] # compute matrix C plus 4 rows +.ifnes "\Mode\()","Add" + vmulps zmm4,zmm4,zmm31 # multiply by alpha + vmulps zmm6,zmm6,zmm31 + vmulps zmm8,zmm8,zmm31 + vmulps zmm10,zmm10,zmm31 + vmulps zmm12,zmm12,zmm31 + vmulps zmm14,zmm14,zmm31 +.else + vfmadd213ps zmm4,zmm31,ZMMWORD PTR [rdx] + vfmadd213ps zmm6,zmm31,ZMMWORD PTR [rdx+rax] + vfmadd213ps zmm8,zmm31,ZMMWORD PTR [rdi] + vfmadd213ps zmm10,zmm31,ZMMWORD PTR [rdi+rax] + vfmadd213ps zmm12,zmm31,ZMMWORD PTR [rbx] + vfmadd213ps zmm14,zmm31,ZMMWORD PTR [rbx+rax] +.endif + vmovups ZMMWORD PTR [rdx],zmm4 + vmovups ZMMWORD PTR [rdx+rax],zmm6 + vmovups ZMMWORD PTR [rdi],zmm8 + vmovups ZMMWORD PTR [rdi+rax],zmm10 + vmovups ZMMWORD PTR [rbx],zmm12 + vmovups ZMMWORD PTR [rbx+rax],zmm14 + add rdx,16*4 # advance matrix C by 16 columns + sub r9,16 + +.L\Mode\().Output16x6Block: + sub r9,16 + jae .L\Mode\().Output16x6BlockWithMask + lea rcx,[r9+16] # correct for over-subtract above + mov ebx,1 + shl ebx,cl + dec ebx + kmovw k1,ebx # update mask for remaining columns + xor r9,r9 # no more columns remaining + +.L\Mode\().Output16x6BlockWithMask: + lea rdi,[rdx+rax*2] # compute matrix C plus 2 rows + lea rbx,[rdx+rax*4] # compute matrix C plus 4 rows +.ifnes "\Mode\()","Add" + vmulps zmm5,zmm5,zmm31 # multiply by alpha + vmulps zmm7,zmm7,zmm31 + vmulps zmm9,zmm9,zmm31 + vmulps zmm11,zmm11,zmm31 + vmulps zmm13,zmm13,zmm31 + vmulps zmm15,zmm15,zmm31 +.else + vfmadd213ps zmm5{k1},zmm31,ZMMWORD PTR [rdx] + vfmadd213ps zmm7{k1},zmm31,ZMMWORD PTR [rdx+rax] + vfmadd213ps zmm9{k1},zmm31,ZMMWORD PTR [rdi] + vfmadd213ps zmm11{k1},zmm31,ZMMWORD PTR [rdi+rax] + vfmadd213ps zmm13{k1},zmm31,ZMMWORD PTR [rbx] + vfmadd213ps zmm15{k1},zmm31,ZMMWORD PTR [rbx+rax] +.endif + vmovups ZMMWORD PTR [rdx]{k1},zmm5 + vmovups ZMMWORD PTR [rdx+rax]{k1},zmm7 + vmovups ZMMWORD PTR [rdi]{k1},zmm9 + vmovups ZMMWORD PTR [rdi+rax]{k1},zmm11 + vmovups ZMMWORD PTR [rbx]{k1},zmm13 + vmovups ZMMWORD PTR [rbx+rax]{k1},zmm15 + add rdx,16*4 # advance matrix C by 16 columns + mov rdi,r11 # reload matrix A + vzeroall + cmp r9,16 + ja .L\Mode\().ProcessNextColumnLoop32x6 + test r9,r9 + jz .L\Mode\().ExitKernel + +.L\Mode\().ProcessRemainingCountN6: + lea rbx,[r10*2+r10] + add rbx,rdi # compute matrix A plus 3 rows + ComputeBlockAvx512FLoop \Mode\(), ComputeBlockAvx512FBy16, 6 + jmp .L\Mode\().Output16x6Block + +// +// Process 3 rows of the matrices. +// + +.L\Mode\().ProcessCountMLessThan6: + cmp r8,3 + jb .L\Mode\().ProcessCountMLessThan3 + mov r8d,3 # return 3 rows handled + cmp r9,16 + jbe .L\Mode\().ProcessRemainingCountN3 + +.L\Mode\().ProcessNextColumnLoop32x3: + ComputeBlockAvx512FLoop \Mode\(), ComputeBlockAvx512FBy32, 3 + add rsi,r12 # advance matrix B by 16*CountK floats +.ifnes "\Mode\()","Add" + vmulps zmm4,zmm4,zmm31 # multiply by alpha + vmulps zmm6,zmm6,zmm31 + vmulps zmm8,zmm8,zmm31 +.else + vfmadd213ps zmm4,zmm31,ZMMWORD PTR [rdx] + vfmadd213ps zmm6,zmm31,ZMMWORD PTR [rdx+rax] + vfmadd213ps zmm8,zmm31,ZMMWORD PTR [rdx+rax*2] +.endif + vmovups ZMMWORD PTR [rdx],zmm4 + vmovups ZMMWORD PTR [rdx+rax],zmm6 + vmovups ZMMWORD PTR [rdx+rax*2],zmm8 + add rdx,16*4 # advance matrix C by 16 columns + sub r9,16 + +.L\Mode\().Output16x3Block: + sub r9,16 + jae .L\Mode\().Output16x3BlockWithMask + lea rcx,[r9+16] # correct for over-subtract above + mov ebx,1 + shl ebx,cl + dec ebx + kmovw k1,ebx # update mask for remaining columns + xor r9,r9 # no more columns remaining + +.L\Mode\().Output16x3BlockWithMask: +.ifnes "\Mode\()","Add" + vmulps zmm5,zmm5,zmm31 # multiply by alpha + vmulps zmm7,zmm7,zmm31 + vmulps zmm9,zmm9,zmm31 +.else + vfmadd213ps zmm5{k1},zmm31,ZMMWORD PTR [rdx] + vfmadd213ps zmm7{k1},zmm31,ZMMWORD PTR [rdx+rax] + vfmadd213ps zmm9{k1},zmm31,ZMMWORD PTR [rdx+rax*2] +.endif + vmovups ZMMWORD PTR [rdx]{k1},zmm5 + vmovups ZMMWORD PTR [rdx+rax]{k1},zmm7 + vmovups ZMMWORD PTR [rdx+rax*2]{k1},zmm9 + add rdx,16*4 # advance matrix C by 16 columns + mov rdi,r11 # reload matrix A + vzeroall + cmp r9,16 + ja .L\Mode\().ProcessNextColumnLoop32x3 + test r9,r9 + jz .L\Mode\().ExitKernel + +.L\Mode\().ProcessRemainingCountN3: + ComputeBlockAvx512FLoop \Mode\(), ComputeBlockAvx512FBy16, 3 + jmp .L\Mode\().Output16x3Block + +// +// Process 1 row of the matrices. +// + +.L\Mode\().ProcessCountMLessThan3: + mov r8d,1 # return 1 row handled + cmp r9,16 + jbe .L\Mode\().ProcessRemainingCountN1 + +.L\Mode\().ProcessNextColumnLoop32x1: + ComputeBlockAvx512FLoop \Mode\(), ComputeBlockAvx512FBy32, 1 + add rsi,r12 # advance matrix B by 16*CountK floats +.ifnes "\Mode\()","Add" + vmulps zmm4,zmm4,zmm31 # multiply by alpha +.else + vfmadd213ps zmm4,zmm31,ZMMWORD PTR [rdx] +.endif + vmovups ZMMWORD PTR [rdx],zmm4 + add rdx,16*4 # advance matrix C by 16 columns + sub r9,16 + +.L\Mode\().Output16x1Block: + sub r9,16 + jae .L\Mode\().Output16x1BlockWithMask + lea rcx,[r9+16] # correct for over-subtract above + mov ebx,1 + shl ebx,cl + dec ebx + kmovw k1,ebx # update mask for remaining columns + xor r9,r9 # no more columns remaining + +.L\Mode\().Output16x1BlockWithMask: +.ifnes "\Mode\()","Add" + vmulps zmm5,zmm5,zmm31 # multiply by alpha +.else + vfmadd213ps zmm5{k1},zmm31,ZMMWORD PTR [rdx] +.endif + vmovups ZMMWORD PTR [rdx]{k1},zmm5 + add rdx,16*4 # advance matrix C by 16 columns + mov rdi,r11 # reload matrix A + vzeroall + cmp r9,16 + ja .L\Mode\().ProcessNextColumnLoop32x1 + test r9,r9 + jz .L\Mode\().ExitKernel + +.L\Mode\().ProcessRemainingCountN1: + ComputeBlockAvx512FLoop \Mode\(), ComputeBlockAvx512FBy16, 1 + jmp .L\Mode\().Output16x1Block + + .endm + + SgemmKernelAvx512FFunction Zero + SgemmKernelAvx512FFunction Add + + .end diff --git a/onnxruntime/core/mlas/lib/x86_64/SgemmKernelFma3.S b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelFma3.S new file mode 100644 index 0000000000000..a7382f897946b --- /dev/null +++ b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelFma3.S @@ -0,0 +1,668 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + SgemmKernelFma3.s + +Abstract: + + This module implements the kernels for the single precision matrix/matrix + multiply operation (SGEMM). + + This implementation uses AVX fused multiply/add instructions. + +--*/ + +#include "asmmacro.h" + + .intel_syntax noprefix + + .equ SgemmKernelFrame_alpha, -8 + .equ SgemmKernelFrame_mask, -4 + .equ SgemmKernelFrame_SavedRbx, 0 + .equ SgemmKernelFrame_SavedRbp, 8 + .equ SgemmKernelFrame_ReturnAddress, 16 + .equ SgemmKernelFrame_lda, 24 + .equ SgemmKernelFrame_ldc, 32 + + .text + +/*++ + +Macro Description: + + This macro multiplies and accumulates for a 32xN block (where N is 1,3) + of the output matrix. + +Arguments: + + Count - Supplies the number of rows to access from matrix A. + + VectorOffset - Supplies the byte offset from matrix B to fetch elements. + + BroadcastOffset - Supplies the byte offset from matrix A to fetch elements. + +Implicit Arguments: + + rdi - Supplies the address into the matrix A data. + + rsi - Supplies the address into the matrix B data. + + r10 - Supplies the length in bytes of a row from matrix A. + + ymm4-ymm15 - Supplies the block accumulators. + +--*/ + + .macro ComputeBlockFma3By32 Count, VectorOffset, BroadcastOffset + +.if \Count\() == 1 + vbroadcastss ymm3,DWORD PTR [rdi+\BroadcastOffset\()] + vfmadd231ps ymm4,ymm3,YMMWORD PTR [rsi+\VectorOffset\()] + vfmadd231ps ymm5,ymm3,YMMWORD PTR [rsi+\VectorOffset\()+32] + vfmadd231ps ymm6,ymm3,YMMWORD PTR [rsi+rbx+\VectorOffset\()] + vfmadd231ps ymm7,ymm3,YMMWORD PTR [rsi+rbx+\VectorOffset\()+32] +.endif + + .endm + +/*++ + +Macro Description: + + This macro multiplies and accumulates for a 16xN block (where N is 1,3,6) + of the output matrix. + +Arguments: + + Count - Supplies the number of rows to access from matrix A. + + VectorOffset - Supplies the byte offset from matrix B to fetch elements. + + BroadcastOffset - Supplies the byte offset from matrix A to fetch elements. + +Implicit Arguments: + + rdi - Supplies the address into the matrix A data. + + rbx - Supplies the address into the matrix A data plus 3 rows. + + rsi - Supplies the address into the matrix B data. + + r10 - Supplies the length in bytes of a row from matrix A. + + ymm4-ymm15 - Supplies the block accumulators. + +--*/ + + .macro ComputeBlockFma3By16 Count, VectorOffset, BroadcastOffset + +.if \Count\() == 1 + vbroadcastss ymm3,DWORD PTR [rdi+\BroadcastOffset\()] + vfmadd231ps ymm4,ymm3,YMMWORD PTR [rsi+\VectorOffset\()] + vfmadd231ps ymm5,ymm3,YMMWORD PTR [rsi+\VectorOffset\()+32] +.else + vmovaps ymm0,YMMWORD PTR [rsi+\VectorOffset\()] + vmovaps ymm1,YMMWORD PTR [rsi+\VectorOffset\()+32] + vbroadcastss ymm3,DWORD PTR [rdi+\BroadcastOffset\()] + vfmadd231ps ymm4,ymm3,ymm0 + vfmadd231ps ymm5,ymm3,ymm1 +.if \Count\() >= 3 + vbroadcastss ymm3,DWORD PTR [rdi+r10+\BroadcastOffset\()] + vfmadd231ps ymm6,ymm3,ymm0 + vfmadd231ps ymm7,ymm3,ymm1 + vbroadcastss ymm3,DWORD PTR [rdi+r10*2+\BroadcastOffset\()] + vfmadd231ps ymm8,ymm3,ymm0 + vfmadd231ps ymm9,ymm3,ymm1 +.endif +.if \Count\() >= 6 + vbroadcastss ymm3,DWORD PTR [rbx+\BroadcastOffset\()] + vfmadd231ps ymm10,ymm3,ymm0 + vfmadd231ps ymm11,ymm3,ymm1 + vbroadcastss ymm3,DWORD PTR [rbx+r10+\BroadcastOffset\()] + vfmadd231ps ymm12,ymm3,ymm0 + vfmadd231ps ymm13,ymm3,ymm1 + vbroadcastss ymm3,DWORD PTR [rbx+r10*2+\BroadcastOffset\()] + vfmadd231ps ymm14,ymm3,ymm0 + vfmadd231ps ymm15,ymm3,ymm1 +.endif +.endif + + .endm + +/*++ + +Macro Description: + + This macro multiplies and accumulates for a 8xN block (where N is 1,3,6) + of the output matrix. + +Arguments: + + Count - Supplies the number of rows to access from matrix A. + + VectorOffset - Supplies the byte offset from matrix B to fetch elements. + + BroadcastOffset - Supplies the byte offset from matrix A to fetch elements. + +Implicit Arguments: + + rdi - Supplies the address into the matrix A data. + + rbx - Supplies the address into the matrix A data plus 3 rows. + + rsi - Supplies the address into the matrix B data. + + r10 - Supplies the length in bytes of a row from matrix A. + + ymm4-ymm15 - Supplies the block accumulators. + +--*/ + + .macro ComputeBlockFma3By8 Count, VectorOffset, BroadcastOffset + +.if \Count\() == 1 + vbroadcastss ymm3,DWORD PTR [rdi+\BroadcastOffset\()] + vfmadd231ps ymm5,ymm3,YMMWORD PTR [rsi+\VectorOffset\()] +.else + vmovaps ymm0,YMMWORD PTR [rsi+\VectorOffset\()] + vbroadcastss ymm3,DWORD PTR [rdi+\BroadcastOffset\()] + vfmadd231ps ymm5,ymm3,ymm0 +.if \Count\() >= 3 + vbroadcastss ymm3,DWORD PTR [rdi+r10+\BroadcastOffset\()] + vfmadd231ps ymm7,ymm3,ymm0 + vbroadcastss ymm3,DWORD PTR [rdi+r10*2+\BroadcastOffset\()] + vfmadd231ps ymm9,ymm3,ymm0 +.endif +.if \Count\() >= 6 + vbroadcastss ymm3,DWORD PTR [rbx+\BroadcastOffset\()] + vfmadd231ps ymm11,ymm3,ymm0 + vbroadcastss ymm3,DWORD PTR [rbx+r10+\BroadcastOffset\()] + vfmadd231ps ymm13,ymm3,ymm0 + vbroadcastss ymm3,DWORD PTR [rbx+r10*2+\BroadcastOffset\()] + vfmadd231ps ymm15,ymm3,ymm0 +.endif +.endif + + .endm + +/*++ + +Macro Description: + + This macro generates code to execute the block compute macro multiple + times and advancing the matrix A and matrix B data pointers. + +Arguments: + + ComputeBlock - Supplies the macro to compute a single block. + + Count - Supplies the number of rows to access from matrix A. + +Implicit Arguments: + + rdi - Supplies the address into the matrix A data. + + rbx - Supplies the address into the matrix A data plus 3 rows. + + rsi - Supplies the address into the matrix B data. + + rcx - Supplies the number of columns from matrix A and the number of rows + from matrix B to iterate over. + + ymm4-ymm15 - Supplies the block accumulators. + +--*/ + + .macro ComputeBlockFma3Loop Mode, ComputeBlock, Count + +// +// Reload the alpha value which is lost after each vzeroall instruction. +// + + vbroadcastss ymm2,DWORD PTR [rsp+SgemmKernelFrame_alpha] + + mov rbp,rcx # reload CountK + sub rbp,4 + jb .L\Mode\().\ComputeBlock\().\Count\().ProcessRemainingBlocks + +.L\Mode\().\ComputeBlock\().\Count\().ComputeBlockBy4Loop: + \ComputeBlock\() \Count\(), 0, 0 + \ComputeBlock\() \Count\(), 16*4, 4 + sub rsi,-32*4 # advance matrix B by 32 columns + \ComputeBlock\() \Count\(), 0, 8 + \ComputeBlock\() \Count\(), 16*4, 12 + sub rsi,-32*4 # advance matrix B by 32 columns + add rdi,4*4 # advance matrix A by 4 columns +.if \Count\() > 3 + add rbx,4*4 # advance matrix A plus rows by 4 columns +.endif + sub rbp,4 + jae .L\Mode\().\ComputeBlock\().\Count\().ComputeBlockBy4Loop + +.L\Mode\().\ComputeBlock\().\Count\().ProcessRemainingBlocks: + add rbp,4 # correct for over-subtract above + jz .L\Mode\().\ComputeBlock\().\Count\().OutputBlock + +.L\Mode\().\ComputeBlock\().\Count\().ComputeBlockBy1Loop: + \ComputeBlock\() \Count\(), 0, 0 + add rsi,16*4 # advance matrix B by 16 columns + add rdi,4 # advance matrix A by 1 column +.if \Count\() > 3 + add rbx,4 # advance matrix A plus rows by 1 column +.endif + dec rbp + jne .L\Mode\().\ComputeBlock\().\Count\().ComputeBlockBy1Loop + +.L\Mode\().\ComputeBlock\().\Count\().OutputBlock: + + .endm + +/*++ + +Routine Description: + + This routine is an inner kernel to compute matrix multiplication for a + set of rows. + +Arguments: + + A (rdi) - Supplies the address of matrix A. + + B (rsi) - Supplies the address of matrix B. The matrix data has been packed + using MlasSgemmCopyPackB or MlasSgemmTransposePackB. + + C (rdx) - Supplies the address of matrix C. + + CountK (rcx) - Supplies the number of columns from matrix A and the number + of rows from matrix B to iterate over. + + CountM (r8) - Supplies the maximum number of rows that can be processed for + matrix A and matrix C. The actual number of rows handled for this + invocation depends on the kernel implementation. + + CountN (r9) - Supplies the number of columns from matrix B and matrix C to + iterate over. + + lda - Supplies the first dimension of matrix A. + + ldc - Supplies the first dimension of matrix C. + + Alpha (xmm0) - Supplies the scaler multiplier (see SGEMM definition). + +Return Value: + + Returns the number of rows handled. + +--*/ + + .macro SgemmKernelFma3Function Mode + + .globl C_UNDERSCORE(MlasSgemmKernel\Mode\()Fma3) +C_UNDERSCORE(MlasSgemmKernel\Mode\()Fma3): + + push rbp + push rbx + mov r11,rdi + mov r10,[rsp+SgemmKernelFrame_lda] + shl r10,2 # convert lda to bytes + mov rax,[rsp+SgemmKernelFrame_ldc] + shl rax,2 # convert ldc to bytes + vmovss DWORD PTR [rsp+SgemmKernelFrame_alpha],xmm0 + vzeroall + +// +// Process 6 rows of the matrices. +// + + cmp r8,6 + jb .L\Mode\().ProcessCountMLessThan6 + mov r8d,6 # return 6 rows handled + cmp r9,8 + jbe .L\Mode\().ProcessRemainingCountN6 + +.L\Mode\().ProcessNextColumnLoop16x6: + lea rbx,[r10*2+r10] + add rbx,rdi # compute matrix A plus 3 rows + ComputeBlockFma3Loop \Mode\(), ComputeBlockFma3By16, 6 + lea rdi,[rdx+rax*2] # compute matrix C plus 2 rows + lea rbx,[rdx+rax*4] # compute matrix C plus 4 rows +.ifnes "\Mode\()","Add" + vmulps ymm4,ymm4,ymm2 # multiply by alpha + vmulps ymm5,ymm5,ymm2 + vmulps ymm6,ymm6,ymm2 + vmulps ymm7,ymm7,ymm2 + vmulps ymm8,ymm8,ymm2 + vmulps ymm9,ymm9,ymm2 + vmulps ymm10,ymm10,ymm2 + vmulps ymm11,ymm11,ymm2 + vmulps ymm12,ymm12,ymm2 + vmulps ymm13,ymm13,ymm2 + vmulps ymm14,ymm14,ymm2 + vmulps ymm15,ymm15,ymm2 +.endif + sub r9,16 + jb .L\Mode\().OutputMasked16x6Block +.ifeqs "\Mode\()","Add" + vfmadd213ps ymm4,ymm2,YMMWORD PTR [rdx] + vfmadd213ps ymm5,ymm2,YMMWORD PTR [rdx+32] + vfmadd213ps ymm6,ymm2,YMMWORD PTR [rdx+rax] + vfmadd213ps ymm7,ymm2,YMMWORD PTR [rdx+rax+32] + vfmadd213ps ymm8,ymm2,YMMWORD PTR [rdi] + vfmadd213ps ymm9,ymm2,YMMWORD PTR [rdi+32] + vfmadd213ps ymm10,ymm2,YMMWORD PTR [rdi+rax] + vfmadd213ps ymm11,ymm2,YMMWORD PTR [rdi+rax+32] + vfmadd213ps ymm12,ymm2,YMMWORD PTR [rbx] + vfmadd213ps ymm13,ymm2,YMMWORD PTR [rbx+32] + vfmadd213ps ymm14,ymm2,YMMWORD PTR [rbx+rax] + vfmadd213ps ymm15,ymm2,YMMWORD PTR [rbx+rax+32] +.endif + vmovups YMMWORD PTR [rdx],ymm4 + vmovups YMMWORD PTR [rdx+32],ymm5 + vmovups YMMWORD PTR [rdx+rax],ymm6 + vmovups YMMWORD PTR [rdx+rax+32],ymm7 + vmovups YMMWORD PTR [rdi],ymm8 + vmovups YMMWORD PTR [rdi+32],ymm9 + vmovups YMMWORD PTR [rdi+rax],ymm10 + vmovups YMMWORD PTR [rdi+rax+32],ymm11 + vmovups YMMWORD PTR [rbx],ymm12 + vmovups YMMWORD PTR [rbx+32],ymm13 + vmovups YMMWORD PTR [rbx+rax],ymm14 + vmovups YMMWORD PTR [rbx+rax+32],ymm15 + add rdx,16*4 # advance matrix C by 16 columns + mov rdi,r11 # reload matrix A + vzeroall + cmp r9,8 + ja .L\Mode\().ProcessNextColumnLoop16x6 + test r9,r9 + jz .L\Mode\().ExitKernel + +.L\Mode\().ProcessRemainingCountN6: + lea rbx,[r10*2+r10] + add rbx,rdi # compute matrix A plus 3 rows + ComputeBlockFma3Loop \Mode\(), ComputeBlockFma3By8, 6 + lea rdi,[rdx+rax*2] # compute matrix C plus 2 rows + lea rbx,[rdx+rax*4] # compute matrix C plus 4 rows +.ifnes "\Mode\()","Add" + vmulps ymm5,ymm5,ymm2 # multiply by alpha + vmulps ymm7,ymm7,ymm2 + vmulps ymm9,ymm9,ymm2 + vmulps ymm11,ymm11,ymm2 + vmulps ymm13,ymm13,ymm2 + vmulps ymm15,ymm15,ymm2 +.endif + cmp r9,8 + jb .L\Mode\().OutputMasked8x6Block +.ifeqs "\Mode\()","Add" + vfmadd213ps ymm5,ymm2,YMMWORD PTR [rdx] + vfmadd213ps ymm7,ymm2,YMMWORD PTR [rdx+rax] + vfmadd213ps ymm9,ymm2,YMMWORD PTR [rdi] + vfmadd213ps ymm11,ymm2,YMMWORD PTR [rdi+rax] + vfmadd213ps ymm13,ymm2,YMMWORD PTR [rbx] + vfmadd213ps ymm15,ymm2,YMMWORD PTR [rbx+rax] +.endif + vmovups YMMWORD PTR [rdx],ymm5 + vmovups YMMWORD PTR [rdx+rax],ymm7 + vmovups YMMWORD PTR [rdi],ymm9 + vmovups YMMWORD PTR [rdi+rax],ymm11 + vmovups YMMWORD PTR [rbx],ymm13 + vmovups YMMWORD PTR [rbx+rax],ymm15 + jmp .L\Mode\().ExitKernelAndZeroUpper + +.L\Mode\().OutputMasked16x6Block: +.ifeqs "\Mode\()","Add" + vfmadd213ps ymm4,ymm2,YMMWORD PTR [rdx] + vfmadd213ps ymm6,ymm2,YMMWORD PTR [rdx+rax] + vfmadd213ps ymm8,ymm2,YMMWORD PTR [rdi] + vfmadd213ps ymm10,ymm2,YMMWORD PTR [rdi+rax] + vfmadd213ps ymm12,ymm2,YMMWORD PTR [rbx] + vfmadd213ps ymm14,ymm2,YMMWORD PTR [rbx+rax] +.endif + vmovups YMMWORD PTR [rdx],ymm4 + vmovups YMMWORD PTR [rdx+rax],ymm6 + vmovups YMMWORD PTR [rdi],ymm8 + vmovups YMMWORD PTR [rdi+rax],ymm10 + vmovups YMMWORD PTR [rbx],ymm12 + vmovups YMMWORD PTR [rbx+rax],ymm14 + add rdx,8*4 # advance matrix C by 8 columns + add rdi,8*4 # advance matrix C plus 2 rows by 8 columns + add rbx,8*4 # advance matrix C plus 4 rows by 8 columns + add r9,8 # correct for over-subtract above + +.L\Mode\().OutputMasked8x6Block: + mov DWORD PTR [rsp+SgemmKernelFrame_mask],r9d + vbroadcastss ymm0,DWORD PTR [rsp+SgemmKernelFrame_mask] + vpcmpgtd ymm0,ymm0,YMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] +.ifeqs "\Mode\()","Add" + vmaskmovps ymm4,ymm0,YMMWORD PTR [rdx] + vmaskmovps ymm6,ymm0,YMMWORD PTR [rdx+rax] + vmaskmovps ymm8,ymm0,YMMWORD PTR [rdi] + vmaskmovps ymm10,ymm0,YMMWORD PTR [rdi+rax] + vmaskmovps ymm12,ymm0,YMMWORD PTR [rbx] + vmaskmovps ymm14,ymm0,YMMWORD PTR [rbx+rax] + vfmadd213ps ymm5,ymm2,ymm4 + vfmadd213ps ymm7,ymm2,ymm6 + vfmadd213ps ymm9,ymm2,ymm8 + vfmadd213ps ymm11,ymm2,ymm10 + vfmadd213ps ymm13,ymm2,ymm12 + vfmadd213ps ymm15,ymm2,ymm14 +.endif + vmaskmovps YMMWORD PTR [rdx],ymm0,ymm5 + vmaskmovps YMMWORD PTR [rdx+rax],ymm0,ymm7 + vmaskmovps YMMWORD PTR [rdi],ymm0,ymm9 + vmaskmovps YMMWORD PTR [rdi+rax],ymm0,ymm11 + vmaskmovps YMMWORD PTR [rbx],ymm0,ymm13 + vmaskmovps YMMWORD PTR [rbx+rax],ymm0,ymm15 + +// +// Restore non-volatile registers and return. +// + +.L\Mode\().ExitKernelAndZeroUpper: + vzeroupper + +.L\Mode\().ExitKernel: + mov eax,r8d + pop rbx + pop rbp + ret + +// +// Process 3 rows of the matrices. +// + +.L\Mode\().ProcessCountMLessThan6: + cmp r8,3 # try to process 3 rows of matrix A + jb .L\Mode\().ProcessCountMLessThan3 + mov r8d,3 # return 3 rows handled + cmp r9,8 + jbe .L\Mode\().ProcessRemainingCountN3 + +.L\Mode\().ProcessNextColumnLoop16x3: + ComputeBlockFma3Loop \Mode\(), ComputeBlockFma3By16, 3 +.ifnes "\Mode\()","Add" + vmulps ymm4,ymm4,ymm2 # multiply by alpha + vmulps ymm5,ymm5,ymm2 + vmulps ymm6,ymm6,ymm2 + vmulps ymm7,ymm7,ymm2 + vmulps ymm8,ymm8,ymm2 + vmulps ymm9,ymm9,ymm2 +.endif + sub r9,16 + jb .L\Mode\().OutputMasked16x3Block +.ifeqs "\Mode\()","Add" + vfmadd213ps ymm4,ymm2,YMMWORD PTR [rdx] + vfmadd213ps ymm5,ymm2,YMMWORD PTR [rdx+32] + vfmadd213ps ymm6,ymm2,YMMWORD PTR [rdx+rax] + vfmadd213ps ymm7,ymm2,YMMWORD PTR [rdx+rax+32] + vfmadd213ps ymm8,ymm2,YMMWORD PTR [rdx+rax*2] + vfmadd213ps ymm9,ymm2,YMMWORD PTR [rdx+rax*2+32] +.endif + vmovups YMMWORD PTR [rdx],ymm4 + vmovups YMMWORD PTR [rdx+32],ymm5 + vmovups YMMWORD PTR [rdx+rax],ymm6 + vmovups YMMWORD PTR [rdx+rax+32],ymm7 + vmovups YMMWORD PTR [rdx+rax*2],ymm8 + vmovups YMMWORD PTR [rdx+rax*2+32],ymm9 + add rdx,16*4 # advance matrix C by 16 columns + mov rdi,r11 # reload matrix A + vzeroall + cmp r9,8 + ja .L\Mode\().ProcessNextColumnLoop16x3 + test r9,r9 + jz .L\Mode\().ExitKernel + +.L\Mode\().ProcessRemainingCountN3: + ComputeBlockFma3Loop \Mode\(), ComputeBlockFma3By8, 3 +.ifnes "\Mode\()","Add" + vmulps ymm5,ymm5,ymm2 # multiply by alpha + vmulps ymm7,ymm7,ymm2 + vmulps ymm9,ymm9,ymm2 +.endif + cmp r9,8 + jb .L\Mode\().OutputMasked8x3Block +.ifeqs "\Mode\()","Add" + vfmadd213ps ymm5,ymm2,YMMWORD PTR [rdx] + vfmadd213ps ymm7,ymm2,YMMWORD PTR [rdx+rax] + vfmadd213ps ymm9,ymm2,YMMWORD PTR [rdx+rax*2] +.endif + vmovups YMMWORD PTR [rdx],ymm5 + vmovups YMMWORD PTR [rdx+rax],ymm7 + vmovups YMMWORD PTR [rdx+rax*2],ymm9 + jmp .L\Mode\().ExitKernelAndZeroUpper + +.L\Mode\().OutputMasked16x3Block: +.ifeqs "\Mode\()","Add" + vfmadd213ps ymm4,ymm2,YMMWORD PTR [rdx] + vfmadd213ps ymm6,ymm2,YMMWORD PTR [rdx+rax] + vfmadd213ps ymm8,ymm2,YMMWORD PTR [rdx+rax*2] +.endif + vmovups YMMWORD PTR [rdx],ymm4 + vmovups YMMWORD PTR [rdx+rax],ymm6 + vmovups YMMWORD PTR [rdx+rax*2],ymm8 + add rdx,8*4 # advance matrix C by 8 columns + add r9,8 # correct for over-subtract above + +.L\Mode\().OutputMasked8x3Block: + mov DWORD PTR [rsp+SgemmKernelFrame_mask],r9d + vbroadcastss ymm0,DWORD PTR [rsp+SgemmKernelFrame_mask] + vpcmpgtd ymm0,ymm0,YMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] +.ifeqs "\Mode\()","Add" + vmaskmovps ymm4,ymm0,YMMWORD PTR [rdx] + vmaskmovps ymm6,ymm0,YMMWORD PTR [rdx+rax] + vmaskmovps ymm8,ymm0,YMMWORD PTR [rdx+rax*2] + vfmadd213ps ymm5,ymm2,ymm4 + vfmadd213ps ymm7,ymm2,ymm6 + vfmadd213ps ymm9,ymm2,ymm8 +.endif + vmaskmovps YMMWORD PTR [rdx],ymm0,ymm5 + vmaskmovps YMMWORD PTR [rdx+rax],ymm0,ymm7 + vmaskmovps YMMWORD PTR [rdx+rax*2],ymm0,ymm9 + jmp .L\Mode\().ExitKernelAndZeroUpper + +// +// Process 1 row of the matrices. +// + +.L\Mode\().ProcessCountMLessThan3: + mov r8d,1 # return 1 row handled + cmp r9,32 + jb .L\Mode\().ProcessRemainingCountN1LessThan32 + mov rbx,rcx + shl rbx,6 # compute 16*CountK*sizeof(float) + +.L\Mode\().ProcessNextColumnLoop32x1: + ComputeBlockFma3Loop \Mode\(), ComputeBlockFma3By32, 1 + add rsi,rbx # advance matrix B by 16*CountK floats +.ifnes "\Mode\()","Add" + vmulps ymm4,ymm4,ymm2 # multiply by alpha + vmulps ymm5,ymm5,ymm2 + vmulps ymm6,ymm6,ymm2 + vmulps ymm7,ymm7,ymm2 +.else + vfmadd213ps ymm4,ymm2,YMMWORD PTR [rdx] + vfmadd213ps ymm5,ymm2,YMMWORD PTR [rdx+32] + vfmadd213ps ymm6,ymm2,YMMWORD PTR [rdx+64] + vfmadd213ps ymm7,ymm2,YMMWORD PTR [rdx+96] +.endif + sub r9,32 + vmovups YMMWORD PTR [rdx],ymm4 + vmovups YMMWORD PTR [rdx+32],ymm5 + vmovups YMMWORD PTR [rdx+64],ymm6 + vmovups YMMWORD PTR [rdx+96],ymm7 + add rdx,32*4 # advance matrix C by 32 columns + mov rdi,r11 # reload matrix A + vzeroall + cmp r9,32 + jae .L\Mode\().ProcessNextColumnLoop32x1 + test r9,r9 + jz .L\Mode\().ExitKernel + +.L\Mode\().ProcessRemainingCountN1LessThan32: + cmp r9,8 + jbe .L\Mode\().ProcessRemainingCountN1 + +.L\Mode\().ProcessNextColumnLoop16x1: + ComputeBlockFma3Loop \Mode\(), ComputeBlockFma3By16, 1 +.ifnes "\Mode\()","Add" + vmulps ymm4,ymm4,ymm2 # multiply by alpha + vmulps ymm5,ymm5,ymm2 +.endif + sub r9,16 + jb .L\Mode\().OutputMasked16x1Block +.ifeqs "\Mode\()","Add" + vfmadd213ps ymm4,ymm2,YMMWORD PTR [rdx] + vfmadd213ps ymm5,ymm2,YMMWORD PTR [rdx+32] +.endif + vmovups YMMWORD PTR [rdx],ymm4 + vmovups YMMWORD PTR [rdx+32],ymm5 + add rdx,16*4 # advance matrix C by 16 columns + mov rdi,r11 # reload matrix A + vzeroall + cmp r9,8 + ja .L\Mode\().ProcessNextColumnLoop16x1 + test r9,r9 + jz .L\Mode\().ExitKernel + +.L\Mode\().ProcessRemainingCountN1: + ComputeBlockFma3Loop \Mode\(), ComputeBlockFma3By8, 1 +.ifnes "\Mode\()","Add" + vmulps ymm5,ymm5,ymm2 # multiply by alpha +.endif + cmp r9,8 + jb .L\Mode\().OutputMasked8x1Block +.ifeqs "\Mode\()","Add" + vfmadd213ps ymm5,ymm2,YMMWORD PTR [rdx] +.endif + vmovups YMMWORD PTR [rdx],ymm5 + jmp .L\Mode\().ExitKernelAndZeroUpper + +.L\Mode\().OutputMasked16x1Block: +.ifeqs "\Mode\()","Add" + vfmadd213ps ymm4,ymm2,YMMWORD PTR [rdx] +.endif + vmovups YMMWORD PTR [rdx],ymm4 + add rdx,8*4 # advance matrix C by 8 columns + add r9,8 # correct for over-subtract above + +.L\Mode\().OutputMasked8x1Block: + mov DWORD PTR [rsp+SgemmKernelFrame_mask],r9d + vbroadcastss ymm0,DWORD PTR [rsp+SgemmKernelFrame_mask] + vpcmpgtd ymm0,ymm0,YMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] +.ifeqs "\Mode\()","Add" + vmaskmovps ymm4,ymm0,YMMWORD PTR [rdx] + vfmadd213ps ymm5,ymm2,ymm4 +.endif + vmaskmovps YMMWORD PTR [rdx],ymm0,ymm5 + jmp .L\Mode\().ExitKernelAndZeroUpper + + .endm + + SgemmKernelFma3Function Zero + SgemmKernelFma3Function Add + + .end diff --git a/onnxruntime/core/mlas/lib/x86_64/SgemmKernelM1Avx.S b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelM1Avx.S new file mode 100644 index 0000000000000..86bc82b23071b --- /dev/null +++ b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelM1Avx.S @@ -0,0 +1,268 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + SgemmKernelM1Avx.s + +Abstract: + + This module implements the kernels for the single precision matrix/matrix + multiply operation (SGEMM). This handles the special case of M=1. + + This implementation uses AVX instructions. + +--*/ + +#include "asmmacro.h" + + .intel_syntax noprefix + + .text + +/*++ + +Routine Description: + + This routine is an inner kernel to compute matrix multiplication for a + set of rows. This handles the special case of M=1. + + The elements in matrix B are not transposed. + +Arguments: + + A (rdi) - Supplies the address of matrix A. + + B (rsi) - Supplies the address of matrix B. + + C (rdx) - Supplies the address of matrix C. + + CountK (rcx) - Supplies the number of columns from matrix A and the number + of rows from matrix B to iterate over. + + CountN (r8) - Supplies the number of columns from matrix B and matrix C to + iterate over. + + ldb (r9) - Supplies the first dimension of matrix B. + + Beta (xmm0) - Supplies the scaler multiplier (see SGEMM definition). + +Return Value: + + None. + +--*/ + + .globl C_UNDERSCORE(MlasSgemmKernelM1Avx) +C_UNDERSCORE(MlasSgemmKernelM1Avx): + + push rbx + shl r9,2 # convert ldb to bytes + mov r10,rdx + mov r11,rsi + +// +// Compute the initial results mask for zeroing or accumulate mode. +// + + vxorps xmm1,xmm1,xmm1 + vcmpeqss xmm0,xmm1,xmm0 + vshufps xmm0,xmm0,xmm0,0 + vinsertf128 ymm0,ymm0,xmm0,1 + +// +// Compute the conditional load/store mask for an unaligned CountN. +// + + mov eax,r8d + and eax,7 + vmovd xmm7,eax + vshufps xmm7,xmm7,xmm7,0 + vpcmpgtd xmm6,xmm7,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip+16] + vpcmpgtd xmm7,xmm7,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] + vinsertf128 ymm7,ymm7,xmm6,1 + +// +// Process 4 rows of the matrices in a loop. +// + + sub rcx,4 + jb .LProcessRemainingCountK + +.LProcessRowLoop4: + vbroadcastss ymm2,DWORD PTR [rdi] + mov rax,r8 # reload CountN + vbroadcastss ymm3,DWORD PTR [rdi+4] + mov rsi,r11 # reload matrix B + vbroadcastss ymm4,DWORD PTR [rdi+8] + mov rdx,r10 # reload matrix C + vbroadcastss ymm5,DWORD PTR [rdi+12] + add rdi,4*4 # advance matrix A by 4 columns + lea r11,[rsi+r9*4] # advance matrix B by 4 rows + sub rax,16 + jb .LProcessRemainingCountN4 + +.LProcessColumnLoop4: + lea rbx,[rsi+r9*2] # compute matrix B plus 2 rows + vmulps ymm1,ymm2,YMMWORD PTR [rsi] + vmulps ymm6,ymm2,YMMWORD PTR [rsi+32] + vmulps ymm8,ymm3,YMMWORD PTR [rsi+r9] + vaddps ymm1,ymm1,ymm8 + vmulps ymm8,ymm3,YMMWORD PTR [rsi+r9+32] + vaddps ymm6,ymm6,ymm8 + vmulps ymm8,ymm4,YMMWORD PTR [rbx] + vaddps ymm1,ymm1,ymm8 + vmulps ymm8,ymm4,YMMWORD PTR [rbx+32] + vaddps ymm6,ymm6,ymm8 + vmulps ymm8,ymm5,YMMWORD PTR [rbx+r9] + vaddps ymm1,ymm1,ymm8 + vmulps ymm8,ymm5,YMMWORD PTR [rbx+r9+32] + vaddps ymm6,ymm6,ymm8 + vandnps ymm8,ymm0,YMMWORD PTR [rdx] + vaddps ymm1,ymm1,ymm8 + vandnps ymm8,ymm0,YMMWORD PTR [rdx+32] + vaddps ymm6,ymm6,ymm8 + vmovups YMMWORD PTR [rdx],ymm1 + vmovups YMMWORD PTR [rdx+32],ymm6 + add rsi,16*4 # advance matrix B by 16 columns + add rdx,16*4 # advance matrix C by 16 columns + sub rax,16 + jae .LProcessColumnLoop4 + +.LProcessRemainingCountN4: + test al,15 # test for unaligned columns + jz .LProcessedRemainingCountN4 + test al,8 # CountN >= 8? + jz .LProcessRemainingCountNSmall4 + lea rbx,[rsi+r9*2] # compute matrix B plus 2 rows + vmulps ymm1,ymm2,YMMWORD PTR [rsi] + vmulps ymm8,ymm3,YMMWORD PTR [rsi+r9] + vaddps ymm1,ymm1,ymm8 + vmulps ymm8,ymm4,YMMWORD PTR [rbx] + vaddps ymm1,ymm1,ymm8 + vmulps ymm8,ymm5,YMMWORD PTR [rbx+r9] + vaddps ymm1,ymm1,ymm8 + vandnps ymm8,ymm0,YMMWORD PTR [rdx] + vaddps ymm1,ymm1,ymm8 + vmovups YMMWORD PTR [rdx],ymm1 + add rsi,8*4 # advance matrix B by 8 columns + add rdx,8*4 # advance matrix C by 8 columns + test al,7 + jz .LProcessedRemainingCountN4 + +.LProcessRemainingCountNSmall4: + lea rbx,[rsi+r9*2] # compute matrix B plus 2 rows + vmaskmovps ymm6,ymm7,YMMWORD PTR [rsi] + vmulps ymm1,ymm2,ymm6 + vmaskmovps ymm6,ymm7,YMMWORD PTR [rsi+r9] + vmulps ymm8,ymm3,ymm6 + vaddps ymm1,ymm1,ymm8 + vmaskmovps ymm6,ymm7,YMMWORD PTR [rbx] + vmulps ymm8,ymm4,ymm6 + vaddps ymm1,ymm1,ymm8 + vmaskmovps ymm6,ymm7,YMMWORD PTR [rbx+r9] + vmulps ymm8,ymm5,ymm6 + vaddps ymm1,ymm1,ymm8 + vmaskmovps ymm6,ymm7,YMMWORD PTR [rdx] + vandnps ymm6,ymm0,ymm6 + vaddps ymm1,ymm1,ymm6 + vmaskmovps YMMWORD PTR [rdx],ymm7,ymm1 + +.LProcessedRemainingCountN4: + vxorps xmm0,xmm0,xmm0 # switch to accumulate mode + sub rcx,4 + jae .LProcessRowLoop4 + +.LProcessRemainingCountK: + test cl,2 + jnz .LProcessRowLoop2 + test cl,1 + jnz .LProcessRowLoop1 + +.LExitKernel: + vzeroupper + pop rbx + ret + +// +// Process 2 rows of the matrices. +// + +.LProcessRowLoop2: + vbroadcastss ymm2,DWORD PTR [rdi] + mov rax,r8 # reload CountN + vbroadcastss ymm3,DWORD PTR [rdi+4] + mov rsi,r11 # reload matrix B + mov rdx,r10 # reload matrix C + add rdi,2*4 # advance matrix A by 2 columns + lea r11,[rsi+r9*2] # advance matrix B by 2 rows + sub rax,8 + jb .LProcessRemainingCountN2 + +.LProcessColumnLoop2: + vmulps ymm1,ymm2,YMMWORD PTR [rsi] + vmulps ymm8,ymm3,YMMWORD PTR [rsi+r9] + vaddps ymm1,ymm1,ymm8 + vandnps ymm6,ymm0,YMMWORD PTR [rdx] + vaddps ymm1,ymm1,ymm6 + vmovups YMMWORD PTR [rdx],ymm1 + add rsi,8*4 # advance matrix B by 8 columns + add rdx,8*4 # advance matrix C by 8 columns + sub rax,8 + jae .LProcessColumnLoop2 + +.LProcessRemainingCountN2: + test al,7 # test for unaligned columns + jz .LProcessedRemainingCountN2 + vmaskmovps ymm6,ymm7,YMMWORD PTR [rsi] + vmulps ymm1,ymm2,ymm6 + vmaskmovps ymm6,ymm7,YMMWORD PTR [rsi+r9] + vmulps ymm8,ymm3,ymm6 + vaddps ymm1,ymm1,ymm8 + vmaskmovps ymm6,ymm7,YMMWORD PTR [rdx] + vandnps ymm6,ymm0,ymm6 + vaddps ymm1,ymm1,ymm6 + vmaskmovps YMMWORD PTR [rdx],ymm7,ymm1 + +.LProcessedRemainingCountN2: + test cl,1 + jz .LExitKernel + vxorps xmm0,xmm0,xmm0 # switch to accumulate mode + +// +// Process 1 row of the matrices. +// + +.LProcessRowLoop1: + vbroadcastss ymm2,DWORD PTR [rdi] + mov rax,r8 # reload CountN + mov rsi,r11 # reload matrix B + mov rdx,r10 # reload matrix C + sub rax,8 + jb .LProcessRemainingCountN1 + +.LProcessColumnLoop1: + vmulps ymm1,ymm2,YMMWORD PTR [rsi] + vandnps ymm6,ymm0,YMMWORD PTR [rdx] + vaddps ymm1,ymm1,ymm6 + vmovups YMMWORD PTR [rdx],ymm1 + add rsi,8*4 # advance matrix B by 8 columns + add rdx,8*4 # advance matrix C by 8 columns + sub rax,8 + jae .LProcessColumnLoop1 + +.LProcessRemainingCountN1: + test al,7 # test for unaligned columns + jz .LExitKernel + vmaskmovps ymm6,ymm7,YMMWORD PTR [rsi] + vmulps ymm1,ymm2,ymm6 + vmaskmovps ymm6,ymm7,YMMWORD PTR [rdx] + vandnps ymm6,ymm0,ymm6 + vaddps ymm1,ymm1,ymm6 + vmaskmovps YMMWORD PTR [rdx],ymm7,ymm1 + jmp .LExitKernel + + .end diff --git a/onnxruntime/core/mlas/lib/x86_64/SgemmKernelM1TransposeBAvx.S b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelM1TransposeBAvx.S new file mode 100644 index 0000000000000..86bc9209fa248 --- /dev/null +++ b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelM1TransposeBAvx.S @@ -0,0 +1,276 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + SgemmKernelM1TransposeBAvx.s + +Abstract: + + This module implements the kernels for the single precision matrix/matrix + multiply operation (SGEMM). This handles the special case of M=1. + + This implementation uses AVX instructions. + +--*/ + +#include "asmmacro.h" + + .intel_syntax noprefix + + .text + +/*++ + +Routine Description: + + This routine is an inner kernel to compute matrix multiplication for a + set of rows. This handles the special case of M=1. + + The elements in matrix B are transposed. + +Arguments: + + A (rdi) - Supplies the address of matrix A. + + B (rsi) - Supplies the address of matrix B. The elements are transposed. + + C (rdx) - Supplies the address of matrix C. + + CountK (rcx) - Supplies the number of columns from matrix A and the number + of columns from matrix B to iterate over. + + CountN (r8) - Supplies the number of rows from matrix B and the number of + columns from matrix C to iterate over. + + ldb (r9) - Supplies the first dimension of matrix B. + + Beta (xmm0) - Supplies the scaler multiplier (see SGEMM definition). + +Return Value: + + None. + +--*/ + + .globl C_UNDERSCORE(MlasSgemmKernelM1TransposeBAvx) +C_UNDERSCORE(MlasSgemmKernelM1TransposeBAvx): + + push rbx + shl r9,2 # convert ldb to bytes + mov r10,rdi + mov r11,rsi + +// +// Compute the results mask for zeroing or accumulate mode. +// + + vxorps xmm1,xmm1,xmm1 + vcmpeqss xmm0,xmm1,xmm0 + vshufps xmm0,xmm0,xmm0,0 + +// +// Compute the conditional load/store mask for an unaligned CountK. +// + + mov eax,ecx + and eax,7 + vmovd xmm7,eax + vshufps xmm7,xmm7,xmm7,0 + vpcmpgtd xmm6,xmm7,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip+16] + vpcmpgtd xmm7,xmm7,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] + vinsertf128 ymm7,ymm7,xmm6,1 + +// +// Process 4 rows of the matrices in a loop. +// + + sub r8,4 + jb .LProcessRemainingCountN + +.LProcessRowLoop4: + vxorps xmm2,xmm2,xmm2 # clear row accumulators + vxorps xmm3,xmm3,xmm3 + vxorps xmm4,xmm4,xmm4 + vxorps xmm5,xmm5,xmm5 + mov rdi,r10 # reload matrix A + mov rsi,r11 # reload matrix B + mov rax,rcx # reload CountK + lea r11,[rsi+r9*4] # advance matrix B by 4 rows + sub rax,8 + jb .LProcessRemainingCountK4 + +.LProcessColumnLoop4: + lea rbx,[rsi+r9*2] # compute matrix B plus 2 rows + vmovups ymm1,YMMWORD PTR [rdi] + vmulps ymm6,ymm1,YMMWORD PTR [rsi] + vaddps ymm2,ymm2,ymm6 + vmulps ymm6,ymm1,YMMWORD PTR [rsi+r9] + vaddps ymm3,ymm3,ymm6 + vmulps ymm6,ymm1,YMMWORD PTR [rbx] + vaddps ymm4,ymm4,ymm6 + vmulps ymm6,ymm1,YMMWORD PTR [rbx+r9] + vaddps ymm5,ymm5,ymm6 + add rdi,8*4 # advance matrix A by 8 columns + add rsi,8*4 # advance matrix B by 8 columns + sub rax,8 + jae .LProcessColumnLoop4 + +.LProcessRemainingCountK4: + test al,7 # test for unaligned columns + jz .LOutput4x1Block + lea rbx,[rsi+r9*2] # compute matrix B plus 2 rows + vmaskmovps ymm1,ymm7,YMMWORD PTR [rdi] + vmaskmovps ymm6,ymm7,YMMWORD PTR [rsi] + vmulps ymm6,ymm1,ymm6 + vaddps ymm2,ymm2,ymm6 + vmaskmovps ymm6,ymm7,YMMWORD PTR [rsi+r9] + vmulps ymm6,ymm1,ymm6 + vaddps ymm3,ymm3,ymm6 + vmaskmovps ymm6,ymm7,YMMWORD PTR [rbx] + vmulps ymm6,ymm1,ymm6 + vaddps ymm4,ymm4,ymm6 + vmaskmovps ymm6,ymm7,YMMWORD PTR [rbx+r9] + vmulps ymm6,ymm1,ymm6 + vaddps ymm5,ymm5,ymm6 + +// +// Reduce and output the row accumulators. +// + +.LOutput4x1Block: + vunpcklps ymm6,ymm2,ymm3 # transpose row accumulators + vunpckhps ymm1,ymm2,ymm3 + vunpcklps ymm2,ymm4,ymm5 + vunpckhps ymm3,ymm4,ymm5 + vunpcklpd ymm4,ymm6,ymm2 + vunpckhpd ymm5,ymm6,ymm2 + vaddps ymm4,ymm4,ymm5 + vunpcklpd ymm6,ymm1,ymm3 + vunpckhpd ymm2,ymm1,ymm3 + vaddps ymm4,ymm4,ymm6 + vaddps ymm4,ymm4,ymm2 + vextractf128 xmm5,ymm4,1 + vaddps xmm4,xmm4,xmm5 + vandnps xmm6,xmm0,XMMWORD PTR [rdx] + vaddps xmm4,xmm4,xmm6 + vmovups XMMWORD PTR [rdx],xmm4 + add rdx,4*4 # advance matrix C by 4 columns + sub r8,4 + jae .LProcessRowLoop4 + +.LProcessRemainingCountN: + test r8d,2 + jnz .LProcessRowLoop2 + test r8d,1 + jnz .LProcessRowLoop1 + +.LExitKernel: + vzeroupper + pop rbx + ret + +// +// Process 2 rows of the matrices. +// + +.LProcessRowLoop2: + vxorps xmm2,xmm2,xmm2 # clear row accumulators + vxorps xmm3,xmm3,xmm3 + mov rdi,r10 # reload matrix A + mov rsi,r11 # reload matrix B + mov rax,rcx # reload CountK + lea r11,[rsi+r9*2] # advance matrix B by 2 rows + sub rax,8 + jb .LProcessRemainingCountK2 + +.LProcessColumnLoop2: + vmovups ymm1,YMMWORD PTR [rdi] + vmulps ymm6,ymm1,YMMWORD PTR [rsi] + vaddps ymm2,ymm2,ymm6 + vmulps ymm6,ymm1,YMMWORD PTR [rsi+r9] + vaddps ymm3,ymm3,ymm6 + add rdi,8*4 # advance matrix A by 8 columns + add rsi,8*4 # advance matrix B by 8 columns + sub rax,8 + jae .LProcessColumnLoop2 + +.LProcessRemainingCountK2: + test al,7 # test for unaligned columns + jz .LOutput2x1Block + vmaskmovps ymm1,ymm7,YMMWORD PTR [rdi] + vmaskmovps ymm6,ymm7,YMMWORD PTR [rsi] + vmulps ymm6,ymm1,ymm6 + vaddps ymm2,ymm2,ymm6 + vmaskmovps ymm6,ymm7,YMMWORD PTR [rsi+r9] + vmulps ymm6,ymm1,ymm6 + vaddps ymm3,ymm3,ymm6 + +// +// Reduce and output the row accumulators. +// + +.LOutput2x1Block: + vunpcklps ymm4,ymm2,ymm3 # reduce row accumulators + vunpckhps ymm2,ymm2,ymm3 + vaddps ymm2,ymm2,ymm4 + vextractf128 xmm4,ymm2,1 + vaddps xmm2,xmm2,xmm4 + vmovhlps xmm4,xmm2,xmm2 + vaddps xmm2,xmm2,xmm4 + vmovsd xmm3,QWORD PTR [rdx] + vandnps xmm3,xmm0,xmm3 + vaddps xmm2,xmm2,xmm3 + vmovsd QWORD PTR [rdx],xmm2 + add rdx,2*4 # advance matrix C by 2 columns + test r8d,1 + jz .LExitKernel + +// +// Process 1 row of the matrices. +// + +.LProcessRowLoop1: + vxorps xmm2,xmm2,xmm2 # clear row accumulators + mov rdi,r10 # reload matrix A + mov rsi,r11 # reload matrix B + mov rax,rcx # reload CountK + sub rax,8 + jb .LProcessRemainingCountK1 + +.LProcessColumnLoop1: + vmovups ymm1,YMMWORD PTR [rdi] + vmulps ymm6,ymm1,YMMWORD PTR [rsi] + vaddps ymm2,ymm2,ymm6 + add rdi,8*4 # advance matrix A by 8 columns + add rsi,8*4 # advance matrix B by 8 columns + sub rax,8 + jae .LProcessColumnLoop1 + +.LProcessRemainingCountK1: + test al,7 # test for unaligned columns + jz .LOutput1x1Block + vmaskmovps ymm1,ymm7,YMMWORD PTR [rdi] + vmaskmovps ymm6,ymm7,YMMWORD PTR [rsi] + vmulps ymm6,ymm1,ymm6 + vaddps ymm2,ymm2,ymm6 + +// +// Reduce and output the row accumulators. +// + +.LOutput1x1Block: + vhaddps ymm2,ymm2,ymm2 # reduce row accumulators + vhaddps ymm2,ymm2,ymm2 + vextractf128 xmm4,ymm2,1 + vaddss xmm2,xmm2,xmm4 + vmovss xmm3,DWORD PTR [rdx] + vandnps xmm3,xmm0,xmm3 + vaddss xmm2,xmm2,xmm3 + vmovss DWORD PTR [rdx],xmm2 + jmp .LExitKernel + + .end diff --git a/onnxruntime/core/mlas/lib/x86_64/SgemmKernelSse2.S b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelSse2.S new file mode 100644 index 0000000000000..5c36ba24a6473 --- /dev/null +++ b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelSse2.S @@ -0,0 +1,509 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + SgemmKernelSse2.s + +Abstract: + + This module implements the kernels for the single precision matrix/matrix + multiply operation (SGEMM). + + This implementation uses SSE2 instructions. + +--*/ + +#include "asmmacro.h" + + .intel_syntax noprefix + + .equ SgemmKernelFrame_alpha, -8 + .equ SgemmKernelFrame_SavedRbx, 0 + .equ SgemmKernelFrame_SavedRbp, 8 + .equ SgemmKernelFrame_ReturnAddress, 16 + .equ SgemmKernelFrame_lda, 24 + .equ SgemmKernelFrame_ldc, 32 + + .text + +/*++ + +Macro Description: + + This macro multiplies and accumulates for a 16xN block (where N is 1,2) + of the output matrix. + +Arguments: + + Count - Supplies the number of rows to access from matrix A. + + VectorOffset - Supplies the byte offset from matrix B to fetch elements. + + Shuffle - Supplies the shuffle mask to extract the element from matrix A. + +Implicit Arguments: + + rsi - Supplies the address into the matrix B data. + + xmm0-xmm1 - Supplies up to four elements loaded from matrix A and matrix A + plus one row. + + xmm8-xmm15 - Supplies the block accumulators. + +--*/ + + .macro ComputeBlockSseBy16 Count, VectorOffset, Shuffle + + movaps xmm4,XMMWORD PTR [rsi+\VectorOffset\()] + movaps xmm5,XMMWORD PTR [rsi+\VectorOffset\()+16] + pshufd xmm2,xmm0,\Shuffle\() +.if \Count\() == 2 + pshufd xmm3,xmm1,\Shuffle\() + movaps xmm6,xmm4 + movaps xmm7,xmm5 +.endif + mulps xmm4,xmm2 + mulps xmm5,xmm2 + addps xmm8,xmm4 + addps xmm9,xmm5 +.if \Count\() == 2 + mulps xmm6,xmm3 + mulps xmm7,xmm3 + addps xmm12,xmm6 + addps xmm13,xmm7 +.endif + movaps xmm4,XMMWORD PTR [rsi+\VectorOffset\()+32] + movaps xmm5,XMMWORD PTR [rsi+\VectorOffset\()+48] +.if \Count\() == 2 + movaps xmm6,xmm4 + movaps xmm7,xmm5 +.endif + mulps xmm4,xmm2 + mulps xmm5,xmm2 + addps xmm10,xmm4 + addps xmm11,xmm5 +.if \Count\() == 2 + mulps xmm6,xmm3 + mulps xmm7,xmm3 + addps xmm14,xmm6 + addps xmm15,xmm7 +.endif + + .endm + +/*++ + +Routine Description: + + This routine is an inner kernel to compute matrix multiplication for a + set of rows. + +Arguments: + + A (rdi) - Supplies the address of matrix A. + + B (rsi) - Supplies the address of matrix B. The matrix data has been packed + using MlasSgemmCopyPackB or MlasSgemmTransposePackB. + + C (rdx) - Supplies the address of matrix C. + + CountK (rcx) - Supplies the number of columns from matrix A and the number + of rows from matrix B to iterate over. + + CountM (r8) - Supplies the maximum number of rows that can be processed for + matrix A and matrix C. The actual number of rows handled for this + invocation depends on the kernel implementation. + + CountN (r9) - Supplies the number of columns from matrix B and matrix C to + iterate over. + + lda - Supplies the first dimension of matrix A. + + ldc - Supplies the first dimension of matrix C. + + Alpha (xmm0) - Supplies the scaler multiplier (see SGEMM definition). + +Return Value: + + Returns the number of rows handled. + +--*/ + + .macro SgemmKernelSseFunction Mode + + .globl C_UNDERSCORE(MlasSgemmKernel\Mode\()Sse) +C_UNDERSCORE(MlasSgemmKernel\Mode\()Sse): + + push rbp + push rbx + mov r11,rdi + mov r10,[rsp+SgemmKernelFrame_lda] + shl r10,2 # convert lda to bytes + mov rax,[rsp+SgemmKernelFrame_ldc] + shl rax,2 # convert ldc to bytes + movss DWORD PTR [rsp+SgemmKernelFrame_alpha],xmm0 + +// +// Process 2 rows of the matrices. +// + + cmp r8,2 + jb .L\Mode\().ProcessCountMLessThan2 + mov r8d,2 # return 2 rows handled + +.L\Mode\().ProcessNextColumnLoop16x2: + xorps xmm8,xmm8 # clear block accumulators + xorps xmm9,xmm9 + xorps xmm10,xmm10 + xorps xmm11,xmm11 + xorps xmm12,xmm12 + xorps xmm13,xmm13 + xorps xmm14,xmm14 + xorps xmm15,xmm15 + mov rbp,rcx # reload CountK + sub rbp,4 + jb .L\Mode\().ProcessRemaining16x2Blocks + +.L\Mode\().Compute16x2BlockBy4Loop: + movups xmm0,XMMWORD PTR [rdi] + movups xmm1,XMMWORD PTR [rdi+r10] + ComputeBlockSseBy16 2, 0, 0x00 + ComputeBlockSseBy16 2, 16*4, 0x55 + sub rsi,-32*4 # advance matrix B by 32 columns + ComputeBlockSseBy16 2, 0, 0xAA + ComputeBlockSseBy16 2, 16*4, 0xFF + sub rsi,-32*4 # advance matrix B by 32 columns + add rdi,4*4 # advance matrix A by 4 columns + sub rbp,4 + jae .L\Mode\().Compute16x2BlockBy4Loop + +.L\Mode\().ProcessRemaining16x2Blocks: + add rbp,4 # correct for over-subtract above + jz .L\Mode\().Output16x2Block + +.L\Mode\().Compute16x2BlockBy1Loop: + movss xmm0,DWORD PTR [rdi] + movss xmm1,DWORD PTR [rdi+r10] + ComputeBlockSseBy16 2, 0, 0x00 + add rsi,16*4 # advance matrix B by 16 columns + add rdi,4 # advance matrix A by 1 column + dec rbp + jne .L\Mode\().Compute16x2BlockBy1Loop + +.L\Mode\().Output16x2Block: + movss xmm2,DWORD PTR [rsp+SgemmKernelFrame_alpha] + lea rdi,[rdx+rax] # compute matrix C plus 1 row + shufps xmm2,xmm2,0 + mulps xmm8,xmm2 # multiply by alpha + mulps xmm9,xmm2 + mulps xmm10,xmm2 + mulps xmm11,xmm2 + mulps xmm12,xmm2 + mulps xmm13,xmm2 + mulps xmm14,xmm2 + mulps xmm15,xmm2 + sub r9,16 + jb .L\Mode\().OutputPartial16x2Block +.ifeqs "\Mode\()","Add" + movups xmm0,XMMWORD PTR [rdx] + movups xmm1,XMMWORD PTR [rdx+16] + movups xmm2,XMMWORD PTR [rdx+32] + movups xmm3,XMMWORD PTR [rdx+48] + movups xmm4,XMMWORD PTR [rdi] + movups xmm5,XMMWORD PTR [rdi+16] + movups xmm6,XMMWORD PTR [rdi+32] + movups xmm7,XMMWORD PTR [rdi+48] + addps xmm8,xmm0 + addps xmm9,xmm1 + addps xmm10,xmm2 + addps xmm11,xmm3 + addps xmm12,xmm4 + addps xmm13,xmm5 + addps xmm14,xmm6 + addps xmm15,xmm7 +.endif + movups XMMWORD PTR [rdx],xmm8 + movups XMMWORD PTR [rdx+16],xmm9 + movups XMMWORD PTR [rdx+32],xmm10 + movups XMMWORD PTR [rdx+48],xmm11 + movups XMMWORD PTR [rdi],xmm12 + movups XMMWORD PTR [rdi+16],xmm13 + movups XMMWORD PTR [rdi+32],xmm14 + movups XMMWORD PTR [rdi+48],xmm15 + add rdx,16*4 # advance matrix C by 16 columns + mov rdi,r11 # reload matrix A + test r9,r9 + jnz .L\Mode\().ProcessNextColumnLoop16x2 + +// +// Restore non-volatile registers and return. +// + +.L\Mode\().ExitKernel: + mov eax,r8d + pop rbx + pop rbp + ret + +// +// Output a partial 16x2 block to the matrix. +// + +.L\Mode\().OutputPartial16x2Block: + add r9,16 # correct for over-subtract above + cmp r9,4 + jb .L\Mode\().OutputPartialLessThan4x2Block + cmp r9,8 + jb .L\Mode\().OutputPartialLessThan8x2Block + cmp r9,12 + jb .L\Mode\().OutputPartialLessThan12x2Block + +.ifeqs "\Mode\()","Add" + movups xmm0,XMMWORD PTR [rdx] + movups xmm1,XMMWORD PTR [rdx+16] + movups xmm2,XMMWORD PTR [rdx+32] + movups xmm3,XMMWORD PTR [rdi] + movups xmm4,XMMWORD PTR [rdi+16] + movups xmm5,XMMWORD PTR [rdi+32] + addps xmm8,xmm0 + addps xmm9,xmm1 + addps xmm10,xmm2 + addps xmm12,xmm3 + addps xmm13,xmm4 + addps xmm14,xmm5 +.endif + movups XMMWORD PTR [rdx],xmm8 + movups XMMWORD PTR [rdx+16],xmm9 + movups XMMWORD PTR [rdx+32],xmm10 + movups XMMWORD PTR [rdi],xmm12 + movups XMMWORD PTR [rdi+16],xmm13 + movups XMMWORD PTR [rdi+32],xmm14 + and r9d,3 # check if remaining count is small + jz .L\Mode\().ExitKernel + movaps xmm8,xmm11 # shift remaining elements down + movaps xmm12,xmm15 + add rdx,12*4 # advance matrix C by 12 columns + add rdi,12*4 # advance matrix C plus 1 row by 12 columns + jmp .L\Mode\().OutputPartialLessThan4x2Block + +.L\Mode\().OutputPartialLessThan12x2Block: +.ifeqs "\Mode\()","Add" + movups xmm0,XMMWORD PTR [rdx] + movups xmm1,XMMWORD PTR [rdx+16] + movups xmm2,XMMWORD PTR [rdi] + movups xmm3,XMMWORD PTR [rdi+16] + addps xmm8,xmm0 + addps xmm9,xmm1 + addps xmm12,xmm2 + addps xmm13,xmm3 +.endif + movups XMMWORD PTR [rdx],xmm8 + movups XMMWORD PTR [rdx+16],xmm9 + movups XMMWORD PTR [rdi],xmm12 + movups XMMWORD PTR [rdi+16],xmm13 + and r9d,3 # check if remaining count is small + jz .L\Mode\().ExitKernel + movaps xmm8,xmm10 # shift remaining elements down + movaps xmm12,xmm14 + add rdx,8*4 # advance matrix C by 8 columns + add rdi,8*4 # advance matrix C plus 1 row by 8 columns + jmp .L\Mode\().OutputPartialLessThan4x2Block + +.L\Mode\().OutputPartialLessThan8x2Block: +.ifeqs "\Mode\()","Add" + movups xmm0,XMMWORD PTR [rdx] + movups xmm1,XMMWORD PTR [rdi] + addps xmm8,xmm0 + addps xmm12,xmm1 +.endif + movups XMMWORD PTR [rdx],xmm8 + movups XMMWORD PTR [rdi],xmm12 + and r9d,3 # check if remaining count is small + jz .L\Mode\().ExitKernel + movaps xmm8,xmm9 # shift remaining elements down + movaps xmm12,xmm13 + add rdx,4*4 # advance matrix C by 4 columns + add rdi,4*4 # advance matrix C plus 1 row by 4 columns + +.L\Mode\().OutputPartialLessThan4x2Block: + cmp r9,2 + jb .L\Mode\().OutputPartial1x2Block +.ifeqs "\Mode\()","Add" + movsd xmm0,QWORD PTR [rdx] + movsd xmm1,QWORD PTR [rdi] + addps xmm8,xmm0 + addps xmm12,xmm1 +.endif + movsd QWORD PTR [rdx],xmm8 + movsd QWORD PTR [rdi],xmm12 + and r9d,1 # check if remaining count is odd + jz .L\Mode\().ExitKernel + movhlps xmm8,xmm8 # shift third element down + movhlps xmm12,xmm12 + add rdx,2*4 # advance matrix C by 2 columns + add rdi,2*4 # advance matrix C plus 1 row by 2 columns + +.L\Mode\().OutputPartial1x2Block: +.ifeqs "\Mode\()","Add" + addss xmm8,DWORD PTR [rdx] + addss xmm12,DWORD PTR [rdi] +.endif + movss DWORD PTR [rdx],xmm8 + movss DWORD PTR [rdi],xmm12 + jmp .L\Mode\().ExitKernel + +// +// Process 1 row of the matrices. +// + +.L\Mode\().ProcessCountMLessThan2: + mov r8d,1 # return 1 row handled + +.L\Mode\().ProcessNextColumnLoop16x1: + xorps xmm8,xmm8 # clear block accumulators + xorps xmm9,xmm9 + xorps xmm10,xmm10 + xorps xmm11,xmm11 + mov rbp,rcx # reload CountK + sub rbp,4 + jb .L\Mode\().ProcessRemaining16x1Blocks + +.L\Mode\().Compute16x1BlockBy4Loop: + movups xmm0,XMMWORD PTR [rdi] + ComputeBlockSseBy16 1, 0, 0x00 + ComputeBlockSseBy16 1, 16*4, 0x55 + sub rsi,-32*4 # advance matrix B by 32 columns + ComputeBlockSseBy16 1, 0, 0xAA + ComputeBlockSseBy16 1, 16*4, 0xFF + sub rsi,-32*4 # advance matrix B by 32 columns + add rdi,4*4 # advance matrix A by 4 columns + sub rbp,4 + jae .L\Mode\().Compute16x1BlockBy4Loop + +.L\Mode\().ProcessRemaining16x1Blocks: + add rbp,4 # correct for over-subtract above + jz .L\Mode\().Output16x1Block + +.L\Mode\().Compute16x1BlockBy1Loop: + movss xmm0,DWORD PTR [rdi] + ComputeBlockSseBy16 1, 0, 0x00 + add rsi,16*4 # advance matrix B by 16 columns + add rdi,4 # advance matrix A by 1 column + dec rbp + jne .L\Mode\().Compute16x1BlockBy1Loop + +.L\Mode\().Output16x1Block: + movss xmm2,DWORD PTR [rsp+SgemmKernelFrame_alpha] + shufps xmm2,xmm2,0 + mulps xmm8,xmm2 # multiply by alpha + mulps xmm9,xmm2 + mulps xmm10,xmm2 + mulps xmm11,xmm2 + sub r9,16 + jb .L\Mode\().OutputPartial16x1Block +.ifeqs "\Mode\()","Add" + movups xmm0,XMMWORD PTR [rdx] + movups xmm1,XMMWORD PTR [rdx+16] + movups xmm2,XMMWORD PTR [rdx+32] + movups xmm3,XMMWORD PTR [rdx+48] + addps xmm8,xmm0 + addps xmm9,xmm1 + addps xmm10,xmm2 + addps xmm11,xmm3 +.endif + movups XMMWORD PTR [rdx],xmm8 + movups XMMWORD PTR [rdx+16],xmm9 + movups XMMWORD PTR [rdx+32],xmm10 + movups XMMWORD PTR [rdx+48],xmm11 + add rdx,16*4 # advance matrix C by 16 columns + mov rdi,r11 # reload matrix A + test r9,r9 + jnz .L\Mode\().ProcessNextColumnLoop16x1 + jmp .L\Mode\().ExitKernel + +// +// Output a partial 16x1 block to the matrix. +// + +.L\Mode\().OutputPartial16x1Block: + add r9,16 # correct for over-subtract above + cmp r9,4 + jb .L\Mode\().OutputPartialLessThan4x1Block + cmp r9,8 + jb .L\Mode\().OutputPartialLessThan8x1Block + cmp r9,12 + jb .L\Mode\().OutputPartialLessThan12x1Block + +.ifeqs "\Mode\()","Add" + movups xmm0,XMMWORD PTR [rdx] + movups xmm1,XMMWORD PTR [rdx+16] + movups xmm2,XMMWORD PTR [rdx+32] + addps xmm8,xmm0 + addps xmm9,xmm1 + addps xmm10,xmm2 +.endif + movups XMMWORD PTR [rdx],xmm8 + movups XMMWORD PTR [rdx+16],xmm9 + movups XMMWORD PTR [rdx+32],xmm10 + and r9d,3 # check if remaining count is small + jz .L\Mode\().ExitKernel + movaps xmm8,xmm11 # shift remaining elements down + add rdx,12*4 # advance matrix C by 12 columns + jmp .L\Mode\().OutputPartialLessThan4x1Block + +.L\Mode\().OutputPartialLessThan12x1Block: +.ifeqs "\Mode\()","Add" + movups xmm0,XMMWORD PTR [rdx] + movups xmm1,XMMWORD PTR [rdx+16] + addps xmm8,xmm0 + addps xmm9,xmm1 +.endif + movups XMMWORD PTR [rdx],xmm8 + movups XMMWORD PTR [rdx+16],xmm9 + and r9d,3 # check if remaining count is small + jz .L\Mode\().ExitKernel + movaps xmm8,xmm10 # shift remaining elements down + add rdx,8*4 # advance matrix C by 8 columns + jmp .L\Mode\().OutputPartialLessThan4x1Block + +.L\Mode\().OutputPartialLessThan8x1Block: +.ifeqs "\Mode\()","Add" + movups xmm0,XMMWORD PTR [rdx] + addps xmm8,xmm0 +.endif + movups XMMWORD PTR [rdx],xmm8 + and r9d,3 # check if remaining count is small + jz .L\Mode\().ExitKernel + movaps xmm8,xmm9 # shift remaining elements down + add rdx,4*4 # advance matrix C by 4 columns + +.L\Mode\().OutputPartialLessThan4x1Block: + cmp r9,2 + jb .L\Mode\().OutputPartial1x1Block +.ifeqs "\Mode\()","Add" + movsd xmm0,QWORD PTR [rdx] + addps xmm8,xmm0 +.endif + movsd QWORD PTR [rdx],xmm8 + and r9d,1 # check if remaining count is odd + jz .L\Mode\().ExitKernel + movhlps xmm8,xmm8 # shift third element down + add rdx,2*4 # advance matrix C by 2 columns + +.L\Mode\().OutputPartial1x1Block: +.ifeqs "\Mode\()","Add" + addss xmm8,DWORD PTR [rdx] +.endif + movss DWORD PTR [rdx],xmm8 + jmp .L\Mode\().ExitKernel + + .endm + + SgemmKernelSseFunction Zero + SgemmKernelSseFunction Add + + .end diff --git a/onnxruntime/core/mlas/lib/x86_64/SgemmTransposePackB16x4Avx.S b/onnxruntime/core/mlas/lib/x86_64/SgemmTransposePackB16x4Avx.S new file mode 100644 index 0000000000000..0f92de07d821e --- /dev/null +++ b/onnxruntime/core/mlas/lib/x86_64/SgemmTransposePackB16x4Avx.S @@ -0,0 +1,121 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + SgemmTransposePackB16x4Avx.s + +Abstract: + + This module implements routines for packing buffers for the single precision + matrix/matrix multiply operation (SGEMM). + + This implementation uses AVX instructions. + +--*/ + +#include "asmmacro.h" + + .intel_syntax noprefix + + .text + +/*++ + +Macro Description: + + 4 columns of 8 rows from the source matrix are transposed to 8 columns of 4 + rows in the destination packed buffer. + +Arguments: + + StoreOffset - Supplies the relative byte offset into the destination packed + buffer. + +Implicit Arguments: + + rdi - Supplies the address of the destination packed buffer. + + rsi - Supplies the address of the source matrix. + + rdx - Supplies the number of elements per row of the source matrix. + +--*/ + + .macro TransposePackB8x4BlockAvx StoreOffset + +// +// Load 4 columns from 8 rows of the source matrix into the lower and upper +// halves of 4 YMM registers. +// + + lea rax,[rsi+rdx*2] + vmovups xmm0,XMMWORD PTR [rsi] + vmovups xmm1,XMMWORD PTR [rsi+rdx] + lea rsi,[rax+rdx*2] + vmovups xmm2,XMMWORD PTR [rax] + vmovups xmm3,XMMWORD PTR [rax+rdx] + lea rax,[rsi+rdx*2] + vinsertf128 ymm0,ymm0,XMMWORD PTR [rsi],1 + vinsertf128 ymm1,ymm1,XMMWORD PTR [rsi+rdx],1 + vinsertf128 ymm2,ymm2,XMMWORD PTR [rax],1 + vinsertf128 ymm3,ymm3,XMMWORD PTR [rax+rdx],1 + +// +// Transpose the lower and upper halves of the 4 YMM registers as two 4x4 +// matrices and store the output to the destination packed buffer. +// + + vunpcklps ymm4,ymm0,ymm1 + vunpckhps ymm5,ymm0,ymm1 + vunpcklps ymm0,ymm2,ymm3 + vunpckhps ymm1,ymm2,ymm3 + vunpcklpd ymm2,ymm4,ymm0 + vunpckhpd ymm3,ymm4,ymm0 + vmovaps YMMWORD PTR [rdi+16*4*0+\StoreOffset\()],ymm2 + vmovaps YMMWORD PTR [rdi+16*4*1+\StoreOffset\()],ymm3 + vunpcklpd ymm0,ymm5,ymm1 + vunpckhpd ymm4,ymm5,ymm1 + vmovaps YMMWORD PTR [rdi+16*4*2+\StoreOffset\()],ymm0 + vmovaps YMMWORD PTR [rdi+16*4*3+\StoreOffset\()],ymm4 + + .endm + +/*++ + +Routine Description: + + This routine transposes elements from the source matrix to the destination + packed buffer. + + 4 columns of 16 rows from the source matrix are transposed to 16 columns of 4 + rows in the destination packed buffer. + +Arguments: + + D (rdi) - Supplies the address of the destination packed buffer. + + B (rsi) - Supplies the address of the source matrix. + + ldb (rdx) - Supplies the number of elements per row of the source matrix. + +Return Value: + + None. + +--*/ + + .globl C_UNDERSCORE(MlasSgemmTransposePackB16x4Avx) +C_UNDERSCORE(MlasSgemmTransposePackB16x4Avx): + + shl rdx,2 # convert ldb to bytes + TransposePackB8x4BlockAvx 0*4 + lea rsi,[rax+rdx*2] + TransposePackB8x4BlockAvx 8*4 + vzeroupper + ret + + .end diff --git a/onnxruntime/core/mlas/lib/x86_64/SgemmTransposePackB16x4Sse2.S b/onnxruntime/core/mlas/lib/x86_64/SgemmTransposePackB16x4Sse2.S new file mode 100644 index 0000000000000..e39133c794f04 --- /dev/null +++ b/onnxruntime/core/mlas/lib/x86_64/SgemmTransposePackB16x4Sse2.S @@ -0,0 +1,84 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + SgemmTransposePackB16x4Sse2.s + +Abstract: + + This module implements routines for packing buffers for the single precision + matrix/matrix multiply operation (SGEMM). + + This implementation uses SSE2 instructions. + +--*/ + +#include "asmmacro.h" + + .intel_syntax noprefix + + .text + +/*++ + +Routine Description: + + This routine transposes elements from the source matrix to the destination + packed buffer. + + 4 columns of 16 rows from the source matrix are transposed to 16 columns of 4 + rows in the destination packed buffer. + +Arguments: + + D (rdi) - Supplies the address of the destination packed buffer. + + B (rsi) - Supplies the address of the source matrix. + + ldb (rdx) - Supplies the number of elements per row of the source matrix. + +Return Value: + + None. + +--*/ + + .globl C_UNDERSCORE(MlasSgemmTransposePackB16x4Sse) +C_UNDERSCORE(MlasSgemmTransposePackB16x4Sse): + + shl rdx,2 # convert ldb to bytes + mov ecx,4 # transpose four 4x4 blocks + +.LTransposeBlockLoop: + lea rax,[rsi+rdx*2] + movups xmm0,XMMWORD PTR [rsi] + movups xmm1,XMMWORD PTR [rsi+rdx] + movups xmm2,XMMWORD PTR [rax] + movups xmm3,XMMWORD PTR [rax+rdx] + movaps xmm4,xmm0 + unpcklps xmm4,xmm1 + unpckhps xmm0,xmm1 + movaps xmm5,xmm2 + unpcklps xmm5,xmm3 + unpckhps xmm2,xmm3 + movaps xmm1,xmm4 + unpcklpd xmm1,xmm5 + unpckhpd xmm4,xmm5 + movaps xmm3,xmm0 + unpcklpd xmm3,xmm2 + unpckhpd xmm0,xmm2 + movaps XMMWORD PTR [rdi+16*4*0],xmm1 + movaps XMMWORD PTR [rdi+16*4*1],xmm4 + movaps XMMWORD PTR [rdi+16*4*2],xmm3 + movaps XMMWORD PTR [rdi+16*4*3],xmm0 + add rdi,4*4 + lea rsi,[rax+rdx*2] + dec ecx + jnz .LTransposeBlockLoop + ret + + .end diff --git a/onnxruntime/core/mlas/lib/x86_64/asmmacro.h b/onnxruntime/core/mlas/lib/x86_64/asmmacro.h new file mode 100644 index 0000000000000..00f11eea3f687 --- /dev/null +++ b/onnxruntime/core/mlas/lib/x86_64/asmmacro.h @@ -0,0 +1,21 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + asmmacro.h + +Abstract: + + This module implements common macros for the assembly modules. + +--*/ + +#if defined(__APPLE__) +#define C_UNDERSCORE(symbol) _##symbol +#else +#define C_UNDERSCORE(symbol) symbol +#endif diff --git a/onnxruntime/core/mlas/lib/x86_64/xgetbv.h b/onnxruntime/core/mlas/lib/x86_64/xgetbv.h new file mode 100644 index 0000000000000..007a8e397aaff --- /dev/null +++ b/onnxruntime/core/mlas/lib/x86_64/xgetbv.h @@ -0,0 +1,41 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + xgetbv.h + +Abstract: + + This module contains a wrapper for the XGETBV instruction for compilers + lacking an intrinsic alternative. + +--*/ + +#pragma once +// clang-format off + +#if !defined(_XCR_XFEATURE_ENABLED_MASK) +#define _XCR_XFEATURE_ENABLED_MASK 0 +#endif + +inline +uint64_t +xgetbv( + unsigned int ext_ctrl_reg + ) +{ + uint32_t eax, edx; + + __asm__ + ( + "xgetbv" + : "=a" (eax), "=d" (edx) + : "c" (ext_ctrl_reg) + ); + + return ((uint64_t)edx << 32) | eax; +} diff --git a/onnxruntime/core/platform/context.h b/onnxruntime/core/platform/context.h new file mode 100644 index 0000000000000..55e2eaab32299 --- /dev/null +++ b/onnxruntime/core/platform/context.h @@ -0,0 +1,44 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +// Portions Copyright (c) Microsoft Corporation + +#pragma once + +namespace onnxruntime { + +enum class ContextKind { + // Initial state with default (empty) values. + kDefault, + // Initial state inherited from the creating or scheduling thread. + kThread, +}; + +// Context is a container for request-specific information that should be passed +// to threads that perform related work. The default constructor should capture +// all relevant context. +class Context { + public: + Context() noexcept = default; + Context(const ContextKind) noexcept {} +}; + +// Scoped object that sets the current thread's context until the object is +// destroyed. +class WithContext { + public: + explicit WithContext(const Context&) noexcept {} +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/platform/env.cc b/onnxruntime/core/platform/env.cc new file mode 100644 index 0000000000000..f6fdbc40fc761 --- /dev/null +++ b/onnxruntime/core/platform/env.cc @@ -0,0 +1,25 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +// Portions Copyright (c) Microsoft Corporation + +#include "core/platform/env.h" + +namespace onnxruntime { + +Env::Env() = default; + +Thread::~Thread() = default; + +} // namespace onnxruntime diff --git a/onnxruntime/core/platform/env.h b/onnxruntime/core/platform/env.h new file mode 100644 index 0000000000000..f4577bc7362df --- /dev/null +++ b/onnxruntime/core/platform/env.h @@ -0,0 +1,175 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +// Portions Copyright (c) Microsoft Corporation + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "core/common/common.h" +#include "core/platform/env_time.h" + +#ifndef _WIN32 +#include +#include +#endif + +namespace onnxruntime { + +class Thread; + +struct ThreadOptions; +#ifdef _WIN32 +using PIDType = unsigned long; +#else +using PIDType = pid_t; +#endif + +/// \brief An interface used by the onnxruntime implementation to +/// access operating system functionality like the filesystem etc. +/// +/// Callers may wish to provide a custom Env object to get fine grain +/// control. +/// +/// All Env implementations are safe for concurrent access from +/// multiple threads without any external synchronization. +class Env { + public: + virtual ~Env() = default; + /// for use with Eigen::ThreadPool + using EnvThread = Thread; + + /// for use with Eigen::ThreadPool + struct Task { + std::function f; + }; + /// \brief Returns a default environment suitable for the current operating + /// system. + /// + /// Sophisticated users may wish to provide their own Env + /// implementation instead of relying on this default environment. + /// + /// The result of Default() belongs to this library and must never be deleted. + static const Env& Default(); + + virtual int GetNumCpuCores() const = 0; + + /// \brief Returns the number of micro-seconds since the Unix epoch. + virtual uint64_t NowMicros() const { return env_time_->NowMicros(); } + + /// \brief Returns the number of seconds since the Unix epoch. + virtual uint64_t NowSeconds() const { return env_time_->NowSeconds(); } + + /// Sleeps/delays the thread for the prescribed number of micro-seconds. + /// On Windows, it's the min time to sleep, not the actual one. + virtual void SleepForMicroseconds(int64_t micros) const = 0; + + /// for use with Eigen::ThreadPool + virtual EnvThread* CreateThread(std::function f) const = 0; + /// for use with Eigen::ThreadPool + virtual Task CreateTask(std::function f) const = 0; + /// for use with Eigen::ThreadPool + virtual void ExecuteTask(const Task& t) const = 0; + + /// \brief Returns a new thread that is running fn() and is identified + /// (for debugging/performance-analysis) by "name". + /// + /// Caller takes ownership of the result and must delete it eventually + /// (the deletion will block until fn() stops running). + virtual Thread* StartThread(const ThreadOptions& thread_options, + const std::string& name, + std::function fn) const = 0; + +#ifdef _WIN32 + //Mainly for use with protobuf library + virtual common::Status FileOpenRd(const std::wstring& path, /*out*/ int& fd) const = 0; + //Mainly for use with protobuf library + virtual common::Status FileOpenWr(const std::wstring& path, /*out*/ int& fd) const = 0; +#endif + //Mainly for use with protobuf library + virtual common::Status FileOpenRd(const std::string& path, /*out*/ int& fd) const = 0; + //Mainly for use with protobuf library + virtual common::Status FileOpenWr(const std::string& path, /*out*/ int& fd) const = 0; + //Mainly for use with protobuf library + virtual common::Status FileClose(int fd) const = 0; + //This functions is always successful. It can't fail. + virtual PIDType GetSelfPid() const = 0; + + // \brief Load a dynamic library. + // + // Pass "library_filename" to a platform-specific mechanism for dynamically + // loading a library. The rules for determining the exact location of the + // library are platform-specific and are not documented here. + // + // On success, returns a handle to the library in "*handle" and returns + // OK from the function. + // Otherwise returns nullptr in "*handle" and an error status from the + // function. + virtual common::Status LoadDynamicLibrary(const std::string& library_filename, void** handle) const = 0; + + virtual common::Status UnloadDynamicLibrary(void* handle) const = 0; + + // \brief Get a pointer to a symbol from a dynamic library. + // + // "handle" should be a pointer returned from a previous call to LoadDynamicLibrary. + // On success, store a pointer to the located symbol in "*symbol" and return + // OK from the function. Otherwise, returns nullptr in "*symbol" and an error + // status from the function. + virtual common::Status GetSymbolFromLibrary(void* handle, const std::string& symbol_name, void** symbol) const = 0; + + // \brief build the name of dynamic library. + // + // "name" should be name of the library. + // "version" should be the version of the library or NULL + // returns the name that LoadDynamicLibrary() can use + virtual std::string FormatLibraryFileName(const std::string& name, const std::string& version) const = 0; + + protected: + Env(); + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Env); + EnvTime* env_time_ = EnvTime::Default(); +}; + +/// Represents a thread used to run a onnxruntime function. +class Thread { + public: + Thread() noexcept = default; + + /// Blocks until the thread of control stops running. + virtual ~Thread(); + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Thread); +}; + +/// \brief Options to configure a Thread. +/// +/// Note that the options are all hints, and the +/// underlying implementation may choose to ignore it. +struct ThreadOptions { + /// Thread stack size to use (in bytes). + size_t stack_size = 0; // 0: use system default value + /// Guard area size to use near thread stacks to use (in bytes) + size_t guard_size = 0; // 0: use system default value +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/platform/env_time.cc b/onnxruntime/core/platform/env_time.cc new file mode 100644 index 0000000000000..7dee7c758d589 --- /dev/null +++ b/onnxruntime/core/platform/env_time.cc @@ -0,0 +1,23 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +// Portions Copyright (c) Microsoft Corporation + +#include "core/platform/env_time.h" + +namespace onnxruntime { + +EnvTime::EnvTime() = default; + +} // namespace onnxruntime diff --git a/onnxruntime/core/platform/env_time.h b/onnxruntime/core/platform/env_time.h new file mode 100644 index 0000000000000..c3399733002b1 --- /dev/null +++ b/onnxruntime/core/platform/env_time.h @@ -0,0 +1,61 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +// Portions Copyright (c) Microsoft Corporation + +#pragma once + +#include +#include + +namespace onnxruntime { + +#ifdef _WIN32 +using TIME_SPEC = int64_t; +#else +using TIME_SPEC = timespec; +#endif + +//Get a time stamp counter +//If the function succeeds, return true. If the function fails, return false +bool GetMonotonicTimeCounter(TIME_SPEC* value); + +void SetTimeSpecToZero(TIME_SPEC* value); +void AccumulateTimeSpec(TIME_SPEC* base, TIME_SPEC* start, TIME_SPEC* end); + +//Return the interval in seconds. +//If the function fails, the return value is zero +double TimeSpecToSeconds(TIME_SPEC* value); + +/// \brief An interface used by the onnxruntime implementation to +/// access timer related operations. +class EnvTime { + public: + EnvTime(); + virtual ~EnvTime() = default; + + /// \brief Returns a default impl suitable for the current operating + /// system. + /// + /// The result of Default() belongs to this library and must never be deleted. + static EnvTime* Default(); + + /// \brief Returns the number of micro-seconds since the Unix epoch. + virtual uint64_t NowMicros() = 0; + + /// \brief Returns the number of seconds since the Unix epoch. + virtual uint64_t NowSeconds() { return NowMicros() / 1000000L; } +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/platform/notification.h b/onnxruntime/core/platform/notification.h new file mode 100644 index 0000000000000..e15740a53df7e --- /dev/null +++ b/onnxruntime/core/platform/notification.h @@ -0,0 +1,85 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +// Portions Copyright (c) Microsoft Corporation + +#ifndef CORE_PLATFORM_NOTIFICATION_H_ +#define CORE_PLATFORM_NOTIFICATION_H_ + +#include +#include // NOLINT +#include // NOLINT +#include // NOLINT + +namespace onnxruntime { + +class Notification { + public: + Notification() : notified_(false) {} + ~Notification() { + // In case the notification is being used to synchronize its own deletion, + // force any prior notifier to leave its critical section before the object + // is destroyed. + std::unique_lock l(mu_); + } + + void Notify() { + std::unique_lock l(mu_); + assert(!HasBeenNotified()); + notified_.store(true, std::memory_order_release); + cv_.notify_all(); + } + + bool HasBeenNotified() const { + return notified_.load(std::memory_order_acquire); + } + + void WaitForNotification() { + if (!HasBeenNotified()) { + std::unique_lock l(mu_); + while (!HasBeenNotified()) { + cv_.wait(l); + } + } + } + + private: + friend bool WaitForNotificationWithTimeout(Notification* n, + int64_t timeout_in_us); + bool WaitForNotificationWithTimeout(int64_t timeout_in_us) { + bool notified = HasBeenNotified(); + if (!notified) { + std::unique_lock l(mu_); + do { + notified = HasBeenNotified(); + } while (!notified && + cv_.wait_for(l, std::chrono::microseconds(timeout_in_us)) != + std::cv_status::timeout); + } + return notified; + } + + std::mutex mu_; // protects mutations of notified_ + std::condition_variable cv_; // signaled when notified_ becomes non-zero + std::atomic notified_; // mutations under mu_ +}; + +inline bool WaitForNotificationWithTimeout(Notification* n, + int64_t timeout_in_us) { + return n->WaitForNotificationWithTimeout(timeout_in_us); +} + +} // namespace onnxruntime + +#endif // CORE_PLATFORM_NOTIFICATION_H_ diff --git a/onnxruntime/core/platform/posix/env.cc b/onnxruntime/core/platform/posix/env.cc new file mode 100644 index 0000000000000..6ff7a3a5dcc42 --- /dev/null +++ b/onnxruntime/core/platform/posix/env.cc @@ -0,0 +1,183 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +// Portions Copyright (c) Microsoft Corporation + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/platform/env.h" +#include "core/common/common.h" + +namespace onnxruntime { + +namespace { + +class StdThread : public Thread { + public: + StdThread(std::function fn) + : thread_(fn) {} + ~StdThread() override { thread_.join(); } + + private: + std::thread thread_; +}; + +class PosixEnv : public Env { + public: + static PosixEnv& Instance() { + static PosixEnv default_env; + return default_env; + } + + int GetNumCpuCores() const override { + // TODO if you need the number of physical cores you'll need to parse + // /proc/cpuinfo and grep for "cpu cores". + //However, that information is not always available(output of 'grep -i core /proc/cpuinfo' is empty) + return std::thread::hardware_concurrency(); + } + + EnvThread* CreateThread(std::function fn) const override { + return new StdThread(fn); + } + + Task CreateTask(std::function f) const override { + return Task{std::move(f)}; + } + void ExecuteTask(const Task& t) const override { + t.f(); + } + + void SleepForMicroseconds(int64_t micros) const override { + while (micros > 0) { + timespec sleep_time; + sleep_time.tv_sec = 0; + sleep_time.tv_nsec = 0; + + if (micros >= 1e6) { + sleep_time.tv_sec = + std::min(micros / 1e6, std::numeric_limits::max()); + micros -= static_cast(sleep_time.tv_sec) * 1e6; + } + if (micros < 1e6) { + sleep_time.tv_nsec = 1000 * micros; + micros = 0; + } + while (nanosleep(&sleep_time, &sleep_time) != 0 && errno == EINTR) { + // Ignore signals and wait for the full interval to elapse. + } + } + } + + Thread* StartThread(const ThreadOptions& /*thread_options*/, const std::string& /*name*/, + std::function fn) const override { + return new StdThread(fn); + } + + PIDType GetSelfPid() const override { + return getpid(); + } + + common::Status FileOpenRd(const std::string& path, /*out*/ int& fd) const override { + fd = open(path.c_str(), O_RDONLY); + if (0 > fd) { + return common::Status(common::SYSTEM, errno); + } + return Status::OK(); + } + + common::Status FileOpenWr(const std::string& path, /*out*/ int& fd) const override { + fd = open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (0 > fd) { + return common::Status(common::SYSTEM, errno); + } + return Status::OK(); + } + + common::Status FileClose(int fd) const override { + int ret = close(fd); + if (0 != ret) { + return common::Status(common::SYSTEM, errno); + } + return Status::OK(); + } + + virtual common::Status LoadDynamicLibrary(const std::string& library_filename, void** handle) const override { + char* error_str = dlerror(); // clear any old error_str + *handle = dlopen(library_filename.c_str(), RTLD_NOW | RTLD_LOCAL); + error_str = dlerror(); + if (!*handle) { + return common::Status(common::ONNXRUNTIME, common::FAIL, + "Failed to load library " + library_filename + " with error: " + error_str); + } + return common::Status::OK(); + } + + virtual common::Status UnloadDynamicLibrary(void* handle) const override { + if (!handle) { + return common::Status(common::ONNXRUNTIME, common::FAIL, "Got null library handle"); + } + char* error_str = dlerror(); // clear any old error_str + int retval = dlclose(handle); + error_str = dlerror(); + if (retval != 0) { + return common::Status(common::ONNXRUNTIME, common::FAIL, + "Failed to unload library with error: " + std::string(error_str)); + } + return common::Status::OK(); + } + + virtual common::Status GetSymbolFromLibrary(void* handle, const std::string& symbol_name, void** symbol) const override { + char* error_str = dlerror(); // clear any old error str + *symbol = dlsym(handle, symbol_name.c_str()); + error_str = dlerror(); + if (error_str) { + return common::Status(common::ONNXRUNTIME, common::FAIL, + "Failed to get symbol " + symbol_name + " with error: " + error_str); + } + // it's possible to get a NULL symbol in our case when Schemas are not custom. + return common::Status::OK(); + } + + virtual std::string FormatLibraryFileName(const std::string& name, const std::string& version) const override { + std::string filename; + if (version.empty()) { + filename = "lib" + name + ".so"; + } else { + filename = "lib" + name + ".so" + "." + version; + } + return filename; + } + + private: + PosixEnv() = default; +}; + +} // namespace + +#if defined(PLATFORM_POSIX) || defined(__ANDROID__) +// REGISTER_FILE_SYSTEM("", PosixFileSystem); +// REGISTER_FILE_SYSTEM("file", LocalPosixFileSystem); +const Env& Env::Default() { + return PosixEnv::Instance(); +} +#endif + +} // namespace onnxruntime diff --git a/onnxruntime/core/platform/posix/env_time.cc b/onnxruntime/core/platform/posix/env_time.cc new file mode 100644 index 0000000000000..fd1057df97020 --- /dev/null +++ b/onnxruntime/core/platform/posix/env_time.cc @@ -0,0 +1,83 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +// Portions Copyright (c) Microsoft Corporation + +#include +#include +#include +#include "core/platform/env_time.h" + +namespace onnxruntime { + +namespace { + +class PosixEnvTime : public EnvTime { + public: + PosixEnvTime() = default; + + uint64_t NowMicros() override { + struct timeval tv; + gettimeofday(&tv, nullptr); + return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; + } +}; + +} // namespace + +#if defined(PLATFORM_POSIX) || defined(__ANDROID__) +EnvTime* EnvTime::Default() { + static PosixEnvTime default_env_time; + return &default_env_time; +} +#endif + +bool GetMonotonicTimeCounter(TIME_SPEC* value) { + return clock_gettime(CLOCK_MONOTONIC, value) == 0; +} + +void SetTimeSpecToZero(TIME_SPEC* value) { + memset(value, 0, sizeof(TIME_SPEC)); +} + +void AccumulateTimeSpec(TIME_SPEC* base, TIME_SPEC* y, TIME_SPEC* x) { + /* Perform the carry for the later subtraction by updating y. */ + if (x->tv_nsec < y->tv_nsec) { + int nsec = (y->tv_nsec - x->tv_nsec) / 1000000000 + 1; + y->tv_nsec -= 1000000000 * nsec; + y->tv_sec += nsec; + } + if (x->tv_nsec - y->tv_nsec > 1000000000) { + int nsec = (x->tv_nsec - y->tv_nsec) / 1000000000; + y->tv_nsec += 1000000000 * nsec; + y->tv_sec -= nsec; + } + + /* Compute the time remaining to wait. + tv_nsec is certainly positive. */ + base->tv_sec += x->tv_sec - y->tv_sec; + base->tv_nsec += x->tv_nsec - y->tv_nsec; + if (base->tv_nsec >= 1000000000) { + base->tv_nsec -= 1000000000; + ++base->tv_sec; + } +} + +//Return the interval in seconds. +//If the function fails, the return value is zero +double TimeSpecToSeconds(TIME_SPEC* value) { + return value->tv_sec + value->tv_nsec / (double)1000000000; +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/platform/posix/stacktrace.cc b/onnxruntime/core/platform/posix/stacktrace.cc new file mode 100644 index 0000000000000..f41ef0e1d4902 --- /dev/null +++ b/onnxruntime/core/platform/posix/stacktrace.cc @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" + +namespace onnxruntime { + +std::vector GetStackTrace() { + return {""}; +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/platform/windows/debug_alloc.cc b/onnxruntime/core/platform/windows/debug_alloc.cc new file mode 100644 index 0000000000000..c4ddbb8069eb4 --- /dev/null +++ b/onnxruntime/core/platform/windows/debug_alloc.cc @@ -0,0 +1,247 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// +// Debug Memory Leak Checking +// +// Implements a custom operator new and delete that will capture a callstack in each allocation +// It creates a separate heap at startup and walks the remaining allocations at process exit, +// dumping out the callstacks to the console and showing a message box if there were any leaks. +// +// It creates & destroys itself in init_seg(lib) so it should scope all user code +// +#ifndef NDEBUG +// TVM need to run with shared CRT, so won't work with debug heap alloc +#ifndef USE_TVM +constexpr int c_callstack_limit = 16; // Maximum depth of callstack in leak trace +#define VALIDATE_HEAP_EVERY_ALLOC 0 // Call HeapValidate on every new/delete + +#pragma warning(disable : 4073) // initializers put in library initialization area (this is intentional) +#pragma init_seg(lib) + +// as this is a debug only checker that does some very low level things and isn't used in the released code +// ignore a bunch of C++ Core Guidelines code analysis warnings +#pragma warning(disable : 26409) // r.11 Don't use 'new' explicitly. +#pragma warning(disable : 26426) // i.22 Static local variables use non-constexpr initializer. +#pragma warning(disable : 26481) // bounds.1 Don't use pointer arithmetic. +#pragma warning(disable : 26482) // bounds.2 Only index into arrays using constant expressions. +#pragma warning(disable : 26485) // bounds.3 No array to pointer decay. +#pragma warning(disable : 26490) // type.1 Don't use reinterpret_cast +#pragma warning(disable : 26493) // type.4 Don't use C-style casts + +#include +#include +#include +#include "debug_alloc.h" +#include +#pragma comment(lib, "Dbghelp.lib") + +_Ret_notnull_ _Post_writable_byte_size_(size) void* operator new(size_t size) { return DebugHeapAlloc(size, 1); } +_Ret_notnull_ _Post_writable_byte_size_(size) void* operator new[](size_t size) { return DebugHeapAlloc(size, 1); } +void operator delete(void* p) noexcept { DebugHeapFree(p); } +void operator delete[](void* p) noexcept { DebugHeapFree(p); } + +struct MemoryBlock { + MemoryBlock(unsigned framesToSkip = 1) noexcept { + unsigned i = CaptureStackBackTrace(framesToSkip + 1, _countof(m_pTraces), m_pTraces, nullptr); + for (; i < _countof(m_pTraces); i++) + m_pTraces[i] = nullptr; + } + + void* m_pTraces[c_callstack_limit]; +}; + +struct SymbolHelper { + SymbolHelper() noexcept { + SymSetOptions(SymGetOptions() | SYMOPT_DEFERRED_LOADS); + SymInitialize(GetCurrentProcess(), nullptr, true); + } + + void Lookup(std::string& string, const ULONG_PTR address) { + char buffer[2048] = {0}; + Symbol symbol; + if (SymFromAddr(GetCurrentProcess(), address, 0, &symbol) == false) { + _snprintf_s(buffer, _TRUNCATE, "0x%08IX (Unknown symbol)", address); + string.append(buffer); + return; + } + + Line line; + DWORD displacement; + if (SymGetLineFromAddr(GetCurrentProcess(), address, &displacement, &line) == false) { + _snprintf_s(buffer, _TRUNCATE, "(unknown file & line number): %s", symbol.Name); + string.append(buffer); + return; + } + + _snprintf_s(buffer, _TRUNCATE, "%s(%d): %s", line.FileName, line.LineNumber, symbol.Name); + string.append(buffer); + } + + struct Symbol : SYMBOL_INFO { + Symbol() noexcept { + SizeOfStruct = sizeof(SYMBOL_INFO); + MaxNameLen = _countof(buffer); + } + + char buffer[1024] = {0}; + }; + + struct Line : IMAGEHLP_LINE { + Line() noexcept { + SizeOfStruct = sizeof(IMAGEHLP_LINE); + } + }; +}; + +static HANDLE g_heap{}; +unsigned g_cumulativeAllocationCount{}; +unsigned g_allocationCount{}; +uint64_t g_cumulativeAllocationBytes{}; + +// Disable C6386: Buffer overrun for just this section. +// 'p' is considered a 0 byte array as it's a void*, so the write to 'p' +// in DebugHeapAlloc and DebugHeapReAlloc trigger spurious warnings. +#pragma warning(push) +#pragma warning(disable : 6386) + +void* DebugHeapAlloc(size_t size, unsigned framesToSkip) { +#if (VALIDATE_HEAP_EVERY_ALLOC) + if (HeapValidate(g_heap, 0, nullptr) == 0) + exit(-1); +#endif + + g_cumulativeAllocationCount++; + g_cumulativeAllocationBytes += size; + void* p = HeapAlloc(g_heap, 0, size + sizeof(MemoryBlock)); + if (!p) + throw std::bad_alloc(); + + g_allocationCount++; + new (p) MemoryBlock(framesToSkip + 1); + return static_cast(p) + sizeof(MemoryBlock); // Adjust outgoing pointer +} + +void* DebugHeapReAlloc(void* p, size_t size) { + if (!p) // Std library will call realloc(nullptr, size) + return DebugHeapAlloc(size); + + g_cumulativeAllocationCount++; + g_cumulativeAllocationBytes += size; + p = static_cast(p) - sizeof(MemoryBlock); // Adjust incoming pointer + p = HeapReAlloc(g_heap, 0, p, size + sizeof(MemoryBlock)); + if (!p) + throw std::bad_alloc(); + + new (p) MemoryBlock; // Redo the callstack + return static_cast(p) + sizeof(MemoryBlock); // Adjust outgoing pointer +} + +#pragma warning(pop) // buffer overrun + +void DebugHeapFree(void* p) noexcept { +#if (VALIDATE_HEAP_EVERY_ALLOC) + if (HeapValidate(g_heap, 0, nullptr) == 0) + exit(-1); +#endif + + if (!p) + return; + + g_allocationCount--; + p = static_cast(p) - sizeof(MemoryBlock); // Adjust incoming pointer + HeapFree(g_heap, 0, p); +} + +static struct Memory_LeakCheck { + Memory_LeakCheck() noexcept; + ~Memory_LeakCheck(); + Memory_LeakCheck(const Memory_LeakCheck&) = delete; + Memory_LeakCheck& operator=(const Memory_LeakCheck&) = delete; + Memory_LeakCheck(Memory_LeakCheck&&) = delete; + Memory_LeakCheck& operator=(Memory_LeakCheck&&) = delete; +} g_memory_leak_check; + +Memory_LeakCheck::Memory_LeakCheck() noexcept { + g_heap = HeapCreate(0, 0, 0); +} + +Memory_LeakCheck::~Memory_LeakCheck() { + SymbolHelper symbols; + + // Create a new heap so we can still allocate memory while dumping the memory leaks + HANDLE heap = HeapCreate(0, 0, 0); + std::swap(heap, g_heap); // Swap it out with our current heap + + unsigned leaked_bytes = 0; + unsigned leak_count = 0; + + PROCESS_HEAP_ENTRY entry{}; + while (HeapWalk(heap, &entry)) { + if ((entry.wFlags & PROCESS_HEAP_ENTRY_BUSY) == 0) + continue; + + const MemoryBlock& block = *static_cast(entry.lpData); + const BYTE* pBlock = static_cast(entry.lpData) + sizeof(MemoryBlock); + + std::string string; + char buffer[1024]; + _snprintf_s(buffer, _TRUNCATE, "%IX bytes at location 0x%08IX\n", entry.cbData - sizeof(MemoryBlock), UINT_PTR(pBlock)); + string.append(buffer); + for (auto& p : block.m_pTraces) { + if (!p) break; + symbols.Lookup(string, reinterpret_cast(p)); + string.push_back('\n'); + } + + // Google test has memory leaks that they haven't fixed. One such issue is tracked here: https://github.com/google/googletest/issues/692 + // + // In gtest-port.cc in function: static ThreadIdToThreadLocals* GetThreadLocalsMapLocked() + // static ThreadIdToThreadLocals* map = new ThreadIdToThreadLocals; + // + // In gtest-port.cc in Mutex::~Mutex() there is this comment: + // "Static mutexes are leaked intentionally. It is not thread-safe to try to clean them up." + // Which explains this leak inside of: void Mutex::ThreadSafeLazyInit() + // critical_section_ = new CRITICAL_SECTION; + if (string.find("testing::internal::Mutex::ThreadSafeLazyInit") == std::string::npos && + string.find("testing::internal::ThreadLocalRegistryImpl::GetThreadLocalsMapLocked") == std::string::npos && + string.find("testing::internal::ThreadLocalRegistryImpl::GetValueOnCurrentThread") == std::string::npos) { + if (leaked_bytes == 0) + OutputDebugStringA("\n-----Starting Heap Trace-----\n\n"); + + leak_count++; + leaked_bytes += entry.cbData - sizeof(MemoryBlock); + OutputDebugStringA(string.c_str()); + OutputDebugStringA("\n"); + } + } + + if (leaked_bytes) { + OutputDebugStringA("-----Ending Heap Trace-----\n\n"); + + std::string string; + char buffer[1024]; + _snprintf_s(buffer, _TRUNCATE, "%d bytes of memory leaked in %d allocations", leaked_bytes, leak_count); + string.append(buffer); + + // Check if we're running on the build machine, if so just exit(-1) + size_t requiredSize; + if (getenv_s(&requiredSize, nullptr, 0, "AGENT_BUILDDIRECTORY") == 0 && requiredSize > 0) { + std::cout << "\n----- MEMORY LEAKS: " << string.c_str() << "\n"; + exit(-1); + } + + // Otherwise we're running on a dev system, show a message box to get their attention + if (IsDebuggerPresent()) { + MessageBoxA(nullptr, string.c_str(), "Warning", MB_OK | MB_ICONWARNING); + } + } else { + OutputDebugStringA("\n----- No memory leaks detected -----\n\n"); + } + + HeapDestroy(heap); + HeapDestroy(g_heap); + g_heap = nullptr; // Any allocations after this point will fail +} +#endif +#endif diff --git a/onnxruntime/core/platform/windows/debug_alloc.h b/onnxruntime/core/platform/windows/debug_alloc.h new file mode 100644 index 0000000000000..ac107334cb936 --- /dev/null +++ b/onnxruntime/core/platform/windows/debug_alloc.h @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#ifndef NDEBUG +// TVM need to run with shared CRT, so won't work with debug heap alloc +#ifndef USE_TVM +void* DebugHeapAlloc(size_t size, unsigned framesToSkip = 0); +void* DebugHeapReAlloc(void* p, size_t size); +void DebugHeapFree(void* p) noexcept; + +#define calloc CallocNotImplemented +#define malloc DebugHeapAlloc +#define realloc DebugHeapReAlloc +#define free DebugHeapFree +#endif +#endif diff --git a/onnxruntime/core/platform/windows/env.cc b/onnxruntime/core/platform/windows/env.cc new file mode 100644 index 0000000000000..2a157af8de0ad --- /dev/null +++ b/onnxruntime/core/platform/windows/env.cc @@ -0,0 +1,188 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +// Portions Copyright (c) Microsoft Corporation + +#include +#include + +#include +#include +#include +#include +#include + +#include "core/common/logging/logging.h" +#include "core/platform/env.h" + +namespace onnxruntime { + +namespace { + +class StdThread : public Thread { + public: + StdThread(std::function fn) + : thread_(fn) {} + + ~StdThread() { thread_.join(); } + + private: + std::thread thread_; +}; + +class WindowsEnv : public Env { + public: + void SleepForMicroseconds(int64_t micros) const override { Sleep(static_cast(micros) / 1000); } + + Thread* StartThread(const ThreadOptions&, const std::string&, + std::function fn) const override { + return new StdThread(fn); + } + + int GetNumCpuCores() const override { + SYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer[256]; + DWORD returnLength = sizeof(buffer); + if (GetLogicalProcessorInformation(buffer, &returnLength) == FALSE) { + // try GetSystemInfo + SYSTEM_INFO sysInfo; + GetSystemInfo(&sysInfo); + if (sysInfo.dwNumberOfProcessors <= 0) { + ONNXRUNTIME_THROW("Fatal error: 0 count processors from GetSystemInfo"); + } + // This is the number of logical processors in the current group + return sysInfo.dwNumberOfProcessors; + } + int processorCoreCount = 0; + int count = (int)(returnLength / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION)); + for (int i = 0; i != count; ++i) { + if (buffer[i].Relationship == RelationProcessorCore) { + ++processorCoreCount; + } + } + if (!processorCoreCount) ONNXRUNTIME_THROW("Fatal error: 0 count processors from GetLogicalProcessorInformation"); + return processorCoreCount; + } + + static WindowsEnv& Instance() { + static WindowsEnv default_env; + return default_env; + } + + PIDType GetSelfPid() const override { + return GetCurrentProcessId(); + } + + EnvThread* CreateThread(std::function fn) const override { + return new StdThread(fn); + } + + Task CreateTask(std::function f) const override { + return Task{std::move(f)}; + } + void ExecuteTask(const Task& t) const override { + t.f(); + } + + common::Status FileOpenRd(const std::wstring& path, /*out*/ int& fd) const override { + _wsopen_s(&fd, path.c_str(), _O_RDONLY | _O_SEQUENTIAL | _O_BINARY, _SH_DENYWR, _S_IREAD | _S_IWRITE); + if (0 > fd) { + return common::Status(common::SYSTEM, errno); + } + return Status::OK(); + } + + common::Status FileOpenWr(const std::wstring& path, /*out*/ int& fd) const override { + _wsopen_s(&fd, path.c_str(), _O_CREAT | _O_SEQUENTIAL | _O_BINARY | _O_WRONLY, _SH_DENYWR, _S_IREAD | _S_IWRITE); + if (0 > fd) { + return common::Status(common::SYSTEM, errno); + } + return Status::OK(); + } + + common::Status FileOpenRd(const std::string& path, /*out*/ int& fd) const override { + _sopen_s(&fd, path.c_str(), _O_RDONLY | _O_SEQUENTIAL | _O_BINARY, _SH_DENYWR, _S_IREAD | _S_IWRITE); + if (0 > fd) { + return common::Status(common::SYSTEM, errno); + } + return Status::OK(); + } + + common::Status FileOpenWr(const std::string& path, /*out*/ int& fd) const override { + _sopen_s(&fd, path.c_str(), _O_CREAT | _O_SEQUENTIAL | _O_BINARY | _O_WRONLY, _SH_DENYWR, _S_IREAD | _S_IWRITE); + if (0 > fd) { + return common::Status(common::SYSTEM, errno); + } + return Status::OK(); + } + + common::Status FileClose(int fd) const override { + int ret = _close(fd); + if (0 != ret) { + return common::Status(common::SYSTEM, errno); + } + return Status::OK(); + } + + virtual Status LoadDynamicLibrary(const std::string& library_filename, void** handle) const override { + ONNXRUNTIME_UNUSED_PARAMETER(library_filename); + ONNXRUNTIME_UNUSED_PARAMETER(handle); + ONNXRUNTIME_NOT_IMPLEMENTED(__FUNCTION__, " is not implemented"); + } + + virtual common::Status UnloadDynamicLibrary(void* handle) const override { + ONNXRUNTIME_UNUSED_PARAMETER(handle); + ONNXRUNTIME_NOT_IMPLEMENTED(__FUNCTION__, " is not implemented"); + } + + virtual Status GetSymbolFromLibrary(void* handle, const std::string& symbol_name, void** symbol) const override { + ONNXRUNTIME_UNUSED_PARAMETER(handle); + ONNXRUNTIME_UNUSED_PARAMETER(symbol_name); + ONNXRUNTIME_UNUSED_PARAMETER(symbol); + ONNXRUNTIME_NOT_IMPLEMENTED(__FUNCTION__, " is not implemented"); + } + + virtual std::string FormatLibraryFileName(const std::string& name, const std::string& version) const override { + ONNXRUNTIME_UNUSED_PARAMETER(name); + ONNXRUNTIME_UNUSED_PARAMETER(version); + ONNXRUNTIME_NOT_IMPLEMENTED(__FUNCTION__, " is not implemented"); + } + + private: + WindowsEnv() + : GetSystemTimePreciseAsFileTime_(nullptr) { + // GetSystemTimePreciseAsFileTime function is only available in the latest + // versions of Windows. For that reason, we try to look it up in + // kernel32.dll at runtime and use an alternative option if the function + // is not available. + HMODULE module = GetModuleHandleW(L"kernel32.dll"); + if (module != nullptr) { + auto func = (FnGetSystemTimePreciseAsFileTime)GetProcAddress( + module, "GetSystemTimePreciseAsFileTime"); + GetSystemTimePreciseAsFileTime_ = func; + } + } + + typedef VOID(WINAPI* FnGetSystemTimePreciseAsFileTime)(LPFILETIME); + FnGetSystemTimePreciseAsFileTime GetSystemTimePreciseAsFileTime_; +}; + +} // namespace + +#if defined(PLATFORM_WINDOWS) +const Env& Env::Default() { + return WindowsEnv::Instance(); +} +#endif + +} // namespace onnxruntime diff --git a/onnxruntime/core/platform/windows/env_time.cc b/onnxruntime/core/platform/windows/env_time.cc new file mode 100644 index 0000000000000..f54fbfb17d480 --- /dev/null +++ b/onnxruntime/core/platform/windows/env_time.cc @@ -0,0 +1,118 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +// Portions Copyright (c) Microsoft Corporation + +#include "core/platform/env_time.h" + +#include +#include +#include +#include +#include + +namespace onnxruntime { + +namespace { + +class WindowsEnvTime : public EnvTime { + public: + WindowsEnvTime() : GetSystemTimePreciseAsFileTime_(NULL) { + // GetSystemTimePreciseAsFileTime function is only available in the latest + // versions of Windows. For that reason, we try to look it up in + // kernel32.dll at runtime and use an alternative option if the function + // is not available. + HMODULE module = GetModuleHandleW(L"kernel32.dll"); + if (module != NULL) { + auto func = (FnGetSystemTimePreciseAsFileTime)GetProcAddress( + module, "GetSystemTimePreciseAsFileTime"); + GetSystemTimePreciseAsFileTime_ = func; + } + } + + uint64_t NowMicros() override { + if (GetSystemTimePreciseAsFileTime_ != NULL) { + // GetSystemTimePreciseAsFileTime function is only available in latest + // versions of Windows, so we need to check for its existence here. + // All std::chrono clocks on Windows proved to return + // values that may repeat, which is not good enough for some uses. + constexpr int64_t kUnixEpochStartTicks = 116444736000000000i64; + constexpr int64_t kFtToMicroSec = 10; + + // This interface needs to return system time and not + // just any microseconds because it is often used as an argument + // to TimedWait() on condition variable + FILETIME system_time; + GetSystemTimePreciseAsFileTime_(&system_time); + + LARGE_INTEGER li; + li.LowPart = system_time.dwLowDateTime; + li.HighPart = system_time.dwHighDateTime; + // Subtract unix epoch start + li.QuadPart -= kUnixEpochStartTicks; + // Convert to microsecs + li.QuadPart /= kFtToMicroSec; + return li.QuadPart; + } + using namespace std::chrono; + return duration_cast(system_clock::now().time_since_epoch()) + .count(); + } + + void SleepForMicroseconds(int64_t micros) { Sleep(static_cast(micros) / 1000); } + + private: + typedef VOID(WINAPI* FnGetSystemTimePreciseAsFileTime)(LPFILETIME); + FnGetSystemTimePreciseAsFileTime GetSystemTimePreciseAsFileTime_; +}; + +} // namespace + +EnvTime* EnvTime::Default() { + static WindowsEnvTime default_time_env; + return &default_time_env; +} + +bool GetMonotonicTimeCounter(TIME_SPEC* value) { + static_assert(sizeof(LARGE_INTEGER) == sizeof(TIME_SPEC), "type mismatch"); + return QueryPerformanceCounter((LARGE_INTEGER*)value) != 0; +} + +static INIT_ONCE g_InitOnce = INIT_ONCE_STATIC_INIT; +static LARGE_INTEGER freq; + +BOOL CALLBACK InitHandleFunction( + PINIT_ONCE, + PVOID, + PVOID*) { + return QueryPerformanceFrequency(&freq); +} + +void SetTimeSpecToZero(TIME_SPEC* value) { + *value = 0; +} + +void AccumulateTimeSpec(TIME_SPEC* base, TIME_SPEC* start, TIME_SPEC* end) { + *base += std::max(0, *end - *start); +} + +//Return the interval in seconds. +//If the function fails, the return value is zero +double TimeSpecToSeconds(TIME_SPEC* value) { + BOOL initState = InitOnceExecuteOnce(&g_InitOnce, InitHandleFunction, nullptr, nullptr); + if (!initState) return 0; + return *value / (double)freq.QuadPart; +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/platform/windows/logging/etw_sink.cc b/onnxruntime/core/platform/windows/logging/etw_sink.cc new file mode 100644 index 0000000000000..b5831f795711b --- /dev/null +++ b/onnxruntime/core/platform/windows/logging/etw_sink.cc @@ -0,0 +1,132 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// EtwSink.h must come before the windows includes +#include "core/platform/windows/logging/etw_sink.h" + +#ifdef ETW_TRACE_LOGGING_SUPPORTED + +// STL includes +#include + +// ETW includes +// need space after Windows.h to prevent clang-format re-ordering breaking the build. +// TraceLoggingProvider.h must follow Windows.h +#include + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 26440) // Warning C26440 from TRACELOGGING_DEFINE_PROVIDER +#endif + +#include +#include + +namespace onnxruntime { +namespace logging { + +namespace { +TRACELOGGING_DEFINE_PROVIDER(etw_provider_handle, "ONNXRuntimeTraceLoggingProvider", + // {929DD115-1ECB-4CB5-B060-EBD4983C421D} + (0x929dd115, 0x1ecb, 0x4cb5, 0xb0, 0x60, 0xeb, 0xd4, 0x98, 0x3c, 0x42, 0x1d)); +} // namespace + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +// Class to unregister ETW provider at shutdown. +// We expect one static instance to be created for the lifetime of the program. +class EtwRegistrationManager { + public: + static EtwRegistrationManager& Register() { + const HRESULT etw_status = ::TraceLoggingRegister(etw_provider_handle); + + if (FAILED(etw_status)) { + throw std::runtime_error("ETW registration failed. Logging will be broken: " + std::to_string(etw_status)); + } + + // return an instance that is just used to unregister as the program exits + static EtwRegistrationManager instance(etw_status); + return instance; + } + + const HRESULT Status() const noexcept { + return etw_status_; + } + + ~EtwRegistrationManager() { + ::TraceLoggingUnregister(etw_provider_handle); + } + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(EtwRegistrationManager); + + EtwRegistrationManager(const HRESULT status) noexcept : etw_status_{status} {} + const HRESULT etw_status_; +}; + +void EtwSink::SendImpl(const Timestamp& timestamp, const std::string& logger_id, const Capture& message) { + UNREFERENCED_PARAMETER(timestamp); + + // register on first usage + static EtwRegistrationManager& etw_manager = EtwRegistrationManager::Register(); + + // do something (not that meaningful) with etw_manager so it doesn't get optimized out + // as we want an instance around to do the unregister + if (FAILED(etw_manager.Status())) { + return; + } + + // Do we want to output Verbose level messages via ETW at any point it time? + // TODO: Validate if this filtering makes sense. + if (message.Severity() <= Severity::kVERBOSE || message.DataType() == DataType::USER) { + return; + } + + // NOTE: Theoretically we could create an interface for all the ETW system interactions so we can separate + // out those from the logic in this class so it is more testable. + // Right now the logic is trivial, so that effort isn't worth it. + + // TraceLoggingWrite requires (painfully) a compile time constant for the TraceLoggingLevel, + // forcing us to use an ugly macro for the call. +#define ETW_EVENT_NAME "ONNXRuntimeLogEvent" +#define TRACE_LOG_WRITE(level) \ + TraceLoggingWrite(etw_provider_handle, ETW_EVENT_NAME, TraceLoggingLevel(level), \ + TraceLoggingString(logger_id.c_str(), "logger"), \ + TraceLoggingString(message.Category(), "category"), \ + TraceLoggingString(message.Location().ToString().c_str(), "location"), \ + TraceLoggingString(message.Message().c_str(), "message")) + + const auto severity{message.Severity()}; + + GSL_SUPPRESS(bounds) + GSL_SUPPRESS(type) { + switch (severity) { + case Severity::kVERBOSE: + TRACE_LOG_WRITE(TRACE_LEVEL_VERBOSE); + break; + case Severity::kINFO: + TRACE_LOG_WRITE(TRACE_LEVEL_INFORMATION); + break; + case Severity::kWARNING: + TRACE_LOG_WRITE(TRACE_LEVEL_WARNING); + break; + case Severity::kERROR: + TRACE_LOG_WRITE(TRACE_LEVEL_ERROR); + break; + case Severity::kFATAL: + TRACE_LOG_WRITE(TRACE_LEVEL_CRITICAL); + break; + default: + throw std::logic_error("Unexpected Severity of " + static_cast(severity)); + } + } + +#undef ETW_EVENT_NAME +#undef TRACE_LOG_WRITE +} +} // namespace logging +} // namespace onnxruntime + +#endif // ETW_TRACE_LOGGING_SUPPORTED diff --git a/onnxruntime/core/platform/windows/logging/etw_sink.h b/onnxruntime/core/platform/windows/logging/etw_sink.h new file mode 100644 index 0000000000000..b9c5cd54c420a --- /dev/null +++ b/onnxruntime/core/platform/windows/logging/etw_sink.h @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +// check for Windows 10 SDK or later +// https://stackoverflow.com/questions/2665755/how-can-i-determine-the-version-of-the-windows-sdk-installed-on-my-computer +#if VER_PRODUCTBUILD > 9600 +// ETW trace logging uses Windows 10 SDK's TraceLoggingProvider.h +#define ETW_TRACE_LOGGING_SUPPORTED 1 +#endif + +#ifdef ETW_TRACE_LOGGING_SUPPORTED + +#include +#include +#include +#include + +#include "core/common/logging/capture.h" +#include "core/common/logging/isink.h" + +namespace onnxruntime { +namespace logging { + +class EtwSink : public ISink { + public: + EtwSink() = default; + ~EtwSink() = default; + + constexpr static const char* kEventName = "ONNXRuntimeLogEvent"; + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(EtwSink); + + void SendImpl(const Timestamp& timestamp, const std::string& logger_id, const Capture& message) override; + + // limit to one instance of an EtwSink being around, so we can control the lifetime of + // EtwTracingManager to ensure we cleanly unregister it + static std::atomic_flag have_instance_; +}; +} // namespace logging +} // namespace onnxruntime + +#endif // ETW_TRACE_LOGGING_SUPPORTED diff --git a/onnxruntime/core/platform/windows/stacktrace.cc b/onnxruntime/core/platform/windows/stacktrace.cc new file mode 100644 index 0000000000000..05cadad70d9ea --- /dev/null +++ b/onnxruntime/core/platform/windows/stacktrace.cc @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include +#include +#include + +#include +#include + +#include "core/common/logging/logging.h" +#include "gsl/span" + +namespace onnxruntime { + +namespace detail { +class CaptureStackTrace { + public: + CaptureStackTrace() = default; + + std::vector Trace() const; + + private: + std::string Lookup(void* address_in) const; + + HANDLE process_ = GetCurrentProcess(); + static const int kCallstackLimit = 64; // Maximum depth of callstack +}; +} // namespace detail + +// Get the stack trace. Currently only enabled for a DEBUG build as we require the DbgHelp library. +std::vector GetStackTrace() { +#ifndef NDEBUG +// TVM need to run with shared CRT, so won't work with debug helper now +#ifndef USE_TVM + return detail::CaptureStackTrace().Trace(); +#else + return {}; +#endif +#else + return {}; +#endif +} + +namespace detail { +#ifndef NDEBUG +#ifndef USE_TVM +class SymbolHelper { + public: + SymbolHelper() noexcept { + SymSetOptions(SymGetOptions() | SYMOPT_DEFERRED_LOADS); + // this could have been called earlier by a higher level component, so failure doesn't necessarily mean + // this won't work. however we should only call SymCleanup if it was successful. + if (SymInitialize(process_, nullptr, true)) { + cleanup_ = true; + } else { + // Log it so we know it happened. Can't do anything else about it. + LOGS_DEFAULT(WARNING) << "Failed to initialize symbols for providing stack trace. Error: 0x" + << std::hex << GetLastError(); + } + } + + struct Symbol : SYMBOL_INFO { + Symbol() noexcept { + SizeOfStruct = sizeof(SYMBOL_INFO); + GSL_SUPPRESS(bounds .3) + MaxNameLen = _countof(buffer); + } + + char buffer[1024]; + }; + + struct Line : IMAGEHLP_LINE64 { + Line() noexcept { + SizeOfStruct = sizeof(IMAGEHLP_LINE64); + } + }; + + ~SymbolHelper() { + if (cleanup_) + SymCleanup(process_); + } + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(SymbolHelper); + + HANDLE process_ = GetCurrentProcess(); + bool cleanup_ = false; +}; + +std::vector CaptureStackTrace::Trace() const { +#pragma warning(push) +#pragma warning(disable : 26426) + static SymbolHelper sh; +#pragma warning(pop) + + std::vector stacktrace; + + PVOID frames[kCallstackLimit]; + const auto f = gsl::make_span(frames); + const auto num_frames = CaptureStackBackTrace(0, kCallstackLimit, f.data(), nullptr); + + stacktrace.reserve(num_frames); + + // hide CaptureStackTrace::Trace and GetStackTrace so the output starts with the 'real' location + const int frames_to_skip = 2; + + // we generally want to skip the first two frames, but if something weird is going on (e.g. code coverage is + // running) and we only have 1 or 2 frames, output them so there's at least something that may be meaningful + const uint16_t start_frame = num_frames > frames_to_skip ? frames_to_skip : 0; + for (uint16_t i = start_frame; i < num_frames; ++i) { + stacktrace.push_back(Lookup(f[i])); + } + + return stacktrace; +} + +std::string CaptureStackTrace::Lookup(void* address_in) const { + SymbolHelper::Symbol symbol; + std::ostringstream result; + + DWORD64 address = 0; + + GSL_SUPPRESS(type .1) { + address = reinterpret_cast(address_in); + } + + if (SymFromAddr(process_, address, 0, &symbol) == false) { + result << "0x" << std::hex << address << " (Unknown symbol)"; + } else + GSL_SUPPRESS(bounds .3) // symbol.Name converts to char* + { + SymbolHelper::Line line; + DWORD displacement; + if (SymGetLineFromAddr64(process_, address, &displacement, &line) == false) { + result << "???: " << symbol.Name; + } else { + result << line.FileName << '(' << line.LineNumber << "): " << symbol.Name; + } + } + + return result.str(); +} + +#endif +#endif +} // namespace detail +} // namespace onnxruntime diff --git a/onnxruntime/core/protobuf/onnx-ml.proto b/onnxruntime/core/protobuf/onnx-ml.proto new file mode 100644 index 0000000000000..79edbb6ef49ce --- /dev/null +++ b/onnxruntime/core/protobuf/onnx-ml.proto @@ -0,0 +1,563 @@ +// +// WARNING: This file is automatically generated! Please edit onnx.in.proto. +// + + +// Copyright (c) Facebook Inc. and Microsoft Corporation. +// Licensed under the MIT license. + +syntax = "proto2"; + +package onnx; + +// Overview +// +// ONNX is an open specification that is comprised of the following components: +// +// 1) A definition of an extensible computation graph model. +// 2) Definitions of standard data types. +// 3) Definitions of built-in operators. +// +// This document describes the syntax of models and their computation graphs, +// as well as the standard data types. Together, they are referred to as the ONNX +// Intermediate Representation, or 'IR' for short. +// +// The normative semantic specification of the ONNX IR is found in docs/IR.md. +// Definitions of the built-in neural network operators may be found in docs/Operators.md. +// Definitions of the built-in classical machine learning operators may be found in +// docs/Operators-ml.md. + +// Notes +// +// Release +// +// We are still in the very early stage of defining ONNX. The current +// version of ONNX is a starting point. While we are actively working +// towards a complete spec, we would like to get the community involved +// by sharing our working version of ONNX. +// +// Protobuf compatibility +// +// To simplify framework compatibility, ONNX is defined using the subset of protobuf +// that is compatible with both protobuf v2 and v3. This means that we do not use any +// protobuf features that are only available in one of the two versions. +// +// Here are the most notable contortions we have to carry out to work around +// these limitations: +// +// - No 'map' (added protobuf 3.0). We instead represent mappings as lists +// of key-value pairs, where order does not matter and duplicates +// are not allowed. + + +// Versioning +// +// ONNX versioning is specified in docs/IR.md and elaborated on in docs/Versioning.md +// +// To be compatible with both proto2 and proto3, we will use a version number +// that is not defined by the default value but an explicit enum number. +enum Version { + // proto3 requires the first enum value to be zero. + // We add this just to appease the compiler. + _START_VERSION = 0; + // The version field is always serialized and we will use it to store the + // version that the graph is generated from. This helps us set up version + // control. + // For the IR, we are using simple numbers starting with with 0x00000001, + // which was the version we published on Oct 10, 2017. + IR_VERSION_2017_10_10 = 0x0000000000000001; + + // IR_VERSION 2 published on Oct 30, 2017 + // - Added type discriminator to AttributeProto to support proto3 users + IR_VERSION_2017_10_30 = 0x0000000000000002; + + // IR VERSION 3 published on Nov 3, 2017 + // - For operator versioning: + // - Added new message OperatorSetIdProto + // - Added opset_import in ModelProto + // - For vendor extensions, added domain in NodeProto + IR_VERSION = 0x0000000000000003; +} + +// Attributes +// +// A named attribute containing either singular float, integer, string, graph, +// and tensor values, or repeated float, integer, string, graph, and tensor values. +// An AttributeProto MUST contain the name field, and *only one* of the +// following content fields, effectively enforcing a C/C++ union equivalent. +message AttributeProto { + + // Note: this enum is structurally identical to the OpSchema::AttrType + // enum defined in schema.h. If you rev one, you likely need to rev the other. + enum AttributeType { + UNDEFINED = 0; + FLOAT = 1; + INT = 2; + STRING = 3; + TENSOR = 4; + GRAPH = 5; + + FLOATS = 6; + INTS = 7; + STRINGS = 8; + TENSORS = 9; + GRAPHS = 10; + } + + // The name field MUST be present for this version of the IR. + optional string name = 1; // namespace Attribute + + // if ref_attr_name is not empty, ref_attr_name is the attribute name in parent function. + // In this case, this AttributeProto does not contain data, and it's a reference of attribute + // in parent scope. + // NOTE: This should ONLY be used in function (sub-graph). It's invalid to be used in main graph. + optional string ref_attr_name = 21; + + // A human-readable documentation for this attribute. Markdown is allowed. + optional string doc_string = 13; + + // The type field MUST be present for this version of the IR. + // For 0.0.1 versions of the IR, this field was not defined, and + // implementations needed to use has_field hueristics to determine + // which value field was in use. For IR_VERSION 0.0.2 or later, this + // field MUST be set and match the f|i|s|t|... field in use. This + // change was made to accomodate proto3 implementations. + optional AttributeType type = 20; // discriminator that indicates which field below is in use + + // Exactly ONE of the following fields must be present for this version of the IR + optional float f = 2; // float + optional int64 i = 3; // int + optional bytes s = 4; // UTF-8 string + optional TensorProto t = 5; // tensor value + optional GraphProto g = 6; // graph + // Do not use field below, it's deprecated. + // optional ValueProto v = 12; // value - subsumes everything but graph + + repeated float floats = 7; // list of floats + repeated int64 ints = 8; // list of ints + repeated bytes strings = 9; // list of UTF-8 strings + repeated TensorProto tensors = 10; // list of tensors + repeated GraphProto graphs = 11; // list of graph +} + +// Defines information on value, including the name, the type, and +// the shape of the value. +message ValueInfoProto { + // This field MUST be present in this version of the IR. + optional string name = 1; // namespace Value + // This field MUST be present in this version of the IR. + optional TypeProto type = 2; + // A human-readable documentation for this value. Markdown is allowed. + optional string doc_string = 3; +} + +// Nodes +// +// Computation graphs are made up of a DAG of nodes, which represent what is +// commonly called a "layer" or "pipeline stage" in machine learning frameworks. +// +// For example, it can be a node of type "Conv" that takes in an image, a filter +// tensor and a bias tensor, and produces the convolved output. +message NodeProto { + repeated string input = 1; // namespace Value + repeated string output = 2; // namespace Value + + // An optional identifier for this node in a graph. + // This field MAY be absent in ths version of the IR. + optional string name = 3; // namespace Node + + // The symbolic identifier of the Operator to execute. + optional string op_type = 4; // namespace Operator + // The domain of the OperatorSet that specifies the operator named by op_type. + optional string domain = 7; // namespace Domain + + // Additional named attributes. + repeated AttributeProto attribute = 5; + + // A human-readable documentation for this node. Markdown is allowed. + optional string doc_string = 6; +} + +// Models +// +// ModelProto is a top-level file/container format for bundling a ML model and +// associating its computation graph with metadata. +// +// The semantics of the model are described by the associated GraphProto. +message ModelProto { + // The version of the IR this model targets. See Version enum above. + // This field MUST be present. + optional int64 ir_version = 1; + + // The OperatorSets this model relies on. + // All ModelProtos MUST have at least one entry that + // specifies which version of the ONNX OperatorSet is + // being imported. + // + // All nodes in the ModelProto's graph will bind against the operator + // with the same-domain/same-op_type operator with the HIGHEST version + // in the referenced operator sets. + repeated OperatorSetIdProto opset_import = 8; + + // The name of the framework or tool used to generate this model. + // This field SHOULD be present to indicate which implementation/tool/framework + // emitted the model. + optional string producer_name = 2; + + // The version of the framework or tool used to generate this model. + // This field SHOULD be present to indicate which implementation/tool/framework + // emitted the model. + optional string producer_version = 3; + + // Domain name of the model. + // We use reverse domain names as name space indicators. For example: + // `com.facebook.fair` or `com.microsoft.cognitiveservices` + // + // Together with `model_version` and GraphProto.name, this forms the unique identity of + // the graph. + optional string domain = 4; + + // The version of the graph encoded. See Version enum below. + optional int64 model_version = 5; + + // A human-readable documentation for this model. Markdown is allowed. + optional string doc_string = 6; + + // The parameterized graph that is evaluated to execute the model. + optional GraphProto graph = 7; + // kezhan: This field is not in ONNX, and will be pushed into ONNX with good use cases in microsoft. + repeated FunctionProto functions = 100; + + // Named metadata values; keys should be distinct. + repeated StringStringEntryProto metadata_props = 14; +}; + +// StringStringEntryProto follows the pattern for cross-proto-version maps. +// See https://developers.google.com/protocol-buffers/docs/proto3#maps +message StringStringEntryProto { + optional string key = 1; + optional string value= 2; +}; + +message QuantInfo { + optional string tensor_name = 1; + // The keys used in the mapping below will be pre-defined in ONNX spec. + // For example, for linear case, “SCALE”, “ZERO_POINT” will be pre-defined as + // quantization parameter keys. + repeated StringStringEntryProto quant_parameter_tensor_names = 2; +}; + +// Graphs +// +// A graph defines the computational logic of a model and is comprised of a parameterized +// list of nodes that form a directed acyclic graph based on their inputs and outputs. +// This is the equivalent of the "network" or "graph" in many deep learning +// frameworks. +message GraphProto { + // The nodes in the graph, sorted topologically. + repeated NodeProto node = 1; + + // The name of the graph. + optional string name = 2; // namespace Graph + + // A list of named tensor values, used to specify constant inputs of the graph. + // Each TensorProto entry must have a distinct name (within the list) that + // also appears in the input list. + repeated TensorProto initializer = 5; + + // A human-readable documentation for this graph. Markdown is allowed. + optional string doc_string = 10; + + // The inputs and outputs of the graph. + repeated ValueInfoProto input = 11; + repeated ValueInfoProto output = 12; + + // Information for the values in the graph. The ValueInfoProto.name's + // must be distinct. It is optional for a value to appear in value_info list. + repeated ValueInfoProto value_info = 13; + + // This field carries information to indicate the mapping among a tensor and its + // quantization parameter tensors. For example: + // A tensor “a” and its quantization parameter tensor are “scale_a” and + // “zero_point_a” (in linear case). + repeated QuantInfo quantization_annotation = 100; + + // DO NOT USE the following fields, they were deprecated from earlier versions. + // repeated string input = 3; + // repeated string output = 4; + // optional int64 ir_version = 6; + // optional int64 producer_version = 7; + // optional string producer_tag = 8; + // optional string domain = 9; +} + +// Tensors +// +// A serialized tensor value. +message TensorProto { + enum DataType { + UNDEFINED = 0; + // Basic types. + FLOAT = 1; // float + UINT8 = 2; // uint8_t + INT8 = 3; // int8_t + UINT16 = 4; // uint16_t + INT16 = 5; // int16_t + INT32 = 6; // int32_t + INT64 = 7; // int64_t + STRING = 8; // string + BOOL = 9; // bool + + // IEEE754 half-precision floating-point format (16 bits wide). + // This format has 1 sign bit, 5 exponent bits, and 10 mantissa bits. + FLOAT16 = 10; + + DOUBLE = 11; + UINT32 = 12; + UINT64 = 13; + COMPLEX64 = 14; // complex with float32 real and imaginary components + COMPLEX128 = 15; // complex with float64 real and imaginary components + + // Non-IEEE floating-point format based on IEEE754 single-precision + // floating-point number truncated to 16 bits. + // This format has 1 sign bit, 8 exponent bits, and 7 mantissa bits. + BFLOAT16 = 16; + + // Future extensions go here. + } + + // The shape of the tensor. + repeated int64 dims = 1; + + // The data type of the tensor. + optional DataType data_type = 2; + + // For very large tensors, we may want to store them in chunks, in which + // case the following fields will specify the segment that is stored in + // the current TensorProto. + message Segment { + optional int64 begin = 1; + optional int64 end = 2; + } + optional Segment segment = 3; + + // Tensor content must be organized in row-major order. + // + // Depending on the data_type field, exactly one of the fields below with + // name ending in _data is used to store the elements of the tensor. + + // For float and complex64 values + // Complex64 tensors are encoded as a single array of floats, + // with the real components appearing in odd numbered positions, + // and the corresponding imaginary component apparing in the + // subsequent even numbered position. (e.g., [1.0 + 2.0i, 3.0 + 4.0i] + // is encoded as [1.0, 2.0 ,3.0 ,4.0] + // When this field is present, the data_type field MUST be FLOAT or COMPLEX64. + repeated float float_data = 4 [packed = true]; + + // For int32, uint8, int8, uint16, int16, bool, and float16 values + // float16 values must be bit-wise converted to an uint16_t prior + // to writing to the buffer. + // When this field is present, the data_type field MUST be + // INT32, INT16, INT8, UINT16, UINT8, BOOL, or FLOAT16 + repeated int32 int32_data = 5 [packed = true]; + + // For strings. + // Each element of string_data is a UTF-8 encoded Unicode + // string. No trailing null, no leading BOM. The protobuf "string" + // scalar type is not used to match ML community conventions. + // When this field is present, the data_type field MUST be STRING + repeated bytes string_data = 6; + + // For int64. + // When this field is present, the data_type field MUST be INT64 + repeated int64 int64_data = 7 [packed = true]; + + // Optionally, a name for the tensor. + optional string name = 8; // namespace Value + + // A human-readable documentation for this tensor. Markdown is allowed. + optional string doc_string = 12; + + // Serializations can either use one of the fields above, or use this + // raw bytes field. The only exception is the string case, where one is + // required to store the content in the repeated bytes string_data field. + // + // When this raw_data field is used to store tensor value, elements MUST + // be stored in as fixed-width, little-endian order. + // Floating-point data types MUST be stored in IEEE 754 format. + // Complex64 elements must be written as two consecutive FLOAT values, real component first. + // Complex128 elements must be written as two consecutive DOUBLE values, real component first. + // Boolean type MUST be written one byte per tensor element (00000001 for true, 00000000 for false). + // + // Note: the advantage of specific field rather than the raw_data field is + // that in some cases (e.g. int data), protobuf does a better packing via + // variable length storage, and may lead to smaller binary footprint. + // When this field is present, the data_type field MUST NOT be STRING or UNDEFINED + optional bytes raw_data = 9; + + // For double + // Complex128 tensors are encoded as a single array of doubles, + // with the real components appearing in odd numbered positions, + // and the corresponding imaginary component apparing in the + // subsequent even numbered position. (e.g., [1.0 + 2.0i, 3.0 + 4.0i] + // is encoded as [1.0, 2.0 ,3.0 ,4.0] + // When this field is present, the data_type field MUST be DOUBLE or COMPLEX128 + repeated double double_data = 10 [packed = true]; + + // For uint64 and uint32 values + // When this field is present, the data_type field MUST be + // UINT32 or UINT64 + repeated uint64 uint64_data = 11 [packed = true]; +} + +// Defines a tensor shape. A dimension can be either an integer value +// or a symbolic variable. A symbolic variable represents an unknown +// dimension. +message TensorShapeProto { + message Dimension { + oneof value { + int64 dim_value = 1; + string dim_param = 2; // namespace Shape + }; + // Standard denotation can optionally be used to denote tensor + // dimensions with standard semantic descriptions to ensure + // that operations are applied to the correct axis of a tensor. + // Refer to https://github.com/onnx/onnx/blob/master/docs/DimensionDenotation.md#denotation-definition + // for pre-defined dimension denotations. + optional string denotation = 3; + }; + repeated Dimension dim = 1; +} + +// Types +// +// The standard ONNX data types. +message TypeProto { + + message Tensor { + // This field MUST NOT have the value of UNDEFINED + // This field MUST be present for this version of the IR. + optional TensorProto.DataType elem_type = 1; + optional TensorShapeProto shape = 2; + } + + + // repeated T + message Sequence { + // The type and optional shape of each element of the sequence. + // This field MUST be present for this version of the IR. + optional TypeProto elem_type = 1; + }; + + // map + message Map { + // This field MUST be present for this version of the IR. + // This field MUST refer to an integral type ([U]INT{8|16|32|64}) or STRING + optional TensorProto.DataType key_type = 1; + // This field MUST be present for this version of the IR. + optional TypeProto value_type = 2; + }; + + message Opaque { + // When missing, the domain is the same as the model's. + optional string domain = 1; + // The name is optional but significant when provided. + optional string name = 2; + // parameters that help defining the type + // DEPRECATED do not use. + // repeated TypeProto parameters = 3; + } + + message SparseTensor { + // This field MUST NOT have the value of UNDEFINED + // This field MUST be present for this version of the IR. + optional TensorProto.DataType elem_type = 1; + optional TensorShapeProto shape = 2; + } + + + oneof value { + // The type of a tensor. + Tensor tensor_type = 1; + + + // NOTE: DNN-only implementations of ONNX MAY elect to not support non-tensor values + // as input and output to graphs and nodes. These types are needed to naturally + // support classical ML operators. DNN operators SHOULD restrict their input + // and output types to tensors. + + // The type of a sequence. + Sequence sequence_type = 4; + + // The type of a map. + Map map_type = 5; + + Opaque opaque_type = 7; + + SparseTensor sparse_tensor_type = 8; + + } + + // An optional denotation can be used to denote the whole + // type with a standard semantic description as to what is + // stored inside. Refer to https://github.com/onnx/onnx/blob/master/docs/TypeDenotation.md#type-denotation-definition + // for pre-defined type denotations. + optional string denotation = 6; +} + +// Operator Sets +// +// OperatorSets are uniquely identified by a (domain, opset_version) pair. +message OperatorSetIdProto { + // The domain of the operator set being identified. + // The empty string ("") or absence of this field implies the operator + // set that is defined as part of the ONNX specification. + // This field MUST be present in this version of the IR when referring to any other operator set. + optional string domain = 1; + + // The version of the operator set being identified. + // This field MUST be present in this version of the IR. + optional int64 version = 2; +} + +// Following messages are copied from onnx-operators-ml.proto in ONNX. +// +// Operator/function status. +enum OperatorStatus { + EXPERIMENTAL = 0; + STABLE = 1; +} + +message FunctionProto { + // The name of the function, similar usage of op_type in OperatorProto. + optional string name = 1; + + // The first version of a function set which contains this function. + // When there's any breaking change for this function, the function set + // contains the function needs to bump its version, and since_version of + // the updated function will be changed to the updated function set version. + optional int64 since_version = 2; + + // This field indicates whether the syntax, semantics, or presence + // of this function is in an experimental or stable stage. Once an + // function is published as STABLE, its syntax and semantics MUST NOT + // change in subsequent versions of the operator set. + // When a function is published as EXPERIMENTAL, the syntax and semantics + // of the function MAY change across operator set versions. + // Functions "become" stable by deprecating the experimental version and + // introducing a new stable function with the same name. + optional OperatorStatus status = 3; + + // The inputs and outputs of the function. + repeated string input = 4; + repeated string output = 5; + + // The attributes of the function. + repeated string attribute = 6; + + // The nodes in the function. + repeated NodeProto node = 7; + // A human-readable documentation for this function. Markdown is allowed. + optional string doc_string = 8; +} diff --git a/onnxruntime/core/protobuf/onnx-operators-ml.proto b/onnxruntime/core/protobuf/onnx-operators-ml.proto new file mode 100644 index 0000000000000..4c4fb2b49b940 --- /dev/null +++ b/onnxruntime/core/protobuf/onnx-operators-ml.proto @@ -0,0 +1,130 @@ +// +// WARNING: This file is automatically generated! Please edit onnx.in.proto. +// + +// Copyright (c) Facebook Inc. and Microsoft Corporation. +// Licensed under the MIT license. + +syntax = "proto2"; + +package onnx; +import "onnx-ml.proto"; + +// +// This file contains the proto definitions for OperatorSetProto and +// OperatorProto. OperatorSetProtos are used to describe a versioned +// set of operators that can be used by a ModelProto. +// +// Like ModelProto, OperatorSetProto is defined as a top-level file/wire +// format, however their usage is different. +// +// ModelProto files are used to describe executable graphs that can be +// executed directly by a framework, runtime, or engine. +// +// OperatorSetProto files are used to describe a set of operators that are +// available in a given environment. The file TBD.TBD is the OperatorSetProto +// that describes the ONNX standard operators. +// + +// An OperatorProto represents the immutable specification of the signature +// and semantics of an operator. +// +// Operators are declared as part of an OperatorSet, which also defines the +// domain name for the set. +// +// Operators are uniquely identified by a three part identifier +// (domain, op_type, since_version) +// where +// *domain* is the domain of an operator set that +// contains this operator specification. +// +// *op_type* is the name of the operator as referenced by a +// NodeProto.op_type +// +// *since_version* is the version of the operator set that +// this operator was initially declared in. +// +message OperatorProto { + // The name of the operator within a domain. + // This field MUST be present in this version of the IR. + optional string op_type = 1; + + // The version of the operator set that first introduced this + // operator. This value MUST be the same value as the + // opset_version of the operator set that first published this operator. + // Subsequent versions of the operator set MUST NOT alter the signature + // or semantics of the operator once published as STABLE. + // This field MUST be present in this version of the IR. + optional int64 since_version = 2; + + // This field indicates whether the syntax, semantics, or presence + // of this operator is in an experimental or stable stage. Once an + // operator is published as STABLE, it's syntax and semantics MUST NOT + // change in subsequent versions of the operator set. + // When an operator is published as EXPERIMENTAL, the syntax and semantics + // of the operator MAY change across operator set versions. + // Operators "become" stable by deprecating the experimental version and + // introducing a new stable operator with the same op_type. + optional OperatorStatus status = 3; + + // Eventually we will declare the signature of the operator here + + // A human-readable documentation for this operator. Markdown is allowed. + optional string doc_string = 10; +} + +// An OperatorSetProto represents an immutable set of immutable operator +// specifications. +// +// The domain of the set (OperatorSetProto.domain) is a reverse-DNS name +// that disambiguates operator sets defined by independent entities. +// +// The version of the set (opset_version) is a monotonically increasing +// integer that indicates changes to the membership of the operator set. +// +// +// Operator sets are uniquely identified by a two part identifier (domain, opset_version) +// +// Like ModelProto, OperatorSetProto is intended as a top-level file/wire format, +// and thus has the standard format headers in addition to the operator set information. +// +message OperatorSetProto { + // All OperatorSetProtos start with a distingushed byte sequence to disambiguate + // protobuf files containing OperatorSets from other content. + // This field MUST be "ONNXOPSET" + // This field MUST be present in this version of the IR + optional string magic = 1; + + // All OperatorSetProtos indicate the version of the IR syntax and semantics + // they adhere to. It is always IR_VERSION. + // This field MUST be present in this version of the IR + optional int32 ir_version = 2; + + // The prerelease component of the SemVer of the IR. + // This field MAY be absent in this version of the IR + optional string ir_version_prerelease = 3; + + // The build metadata component of the SemVer of the IR. + // This field MAY be absent in this version of the IR + optional string ir_build_metadata = 7; + + // Domain name of the operator set, in reverse DNS form (e.g., com.acme.dnnops). + optional string domain = 4; + + // The version of the set of operators. This is a simple int value + // that is monotonically increasing as new versions of operator set + // are published. All operators in this set MUST have version + // numbers no greater than opset_version. + optional int64 opset_version = 5; + + // A human-readable documentation for this set of operators. Markdown is allowed. + optional string doc_string = 6; + + // The operators specified by this operator set. + // The (name, version) MUST be unique across all OperatorProtos in operator + repeated OperatorProto operator = 8; + + // The functions specified by this operator set. + // The (name, version) MUST be unique across all OperatorProtos/FunctionProtos in operator/functions + repeated FunctionProto functions = 9; +} diff --git a/onnxruntime/core/providers/common.h b/onnxruntime/core/providers/common.h new file mode 100644 index 0000000000000..b177c201f9359 --- /dev/null +++ b/onnxruntime/core/providers/common.h @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" + +namespace onnxruntime { + +/** +Handle a potentially negative axis. Enforces negative axis is valid. +@param axis Axis to convert from negative to positive if needed. +@param tensor_rank Rank of tensor axis applies to. Tensor::Shape()::NumDimensions(). +@returns Positive axis. +*/ +inline int64_t HandleNegativeAxis(int64_t axis, int64_t tensor_rank) { + ONNXRUNTIME_ENFORCE(axis >= -tensor_rank && axis <= tensor_rank - 1, "axis ", axis, + " is not in valid range [-", tensor_rank, ",", tensor_rank - 1, "]"); + // Handle negative axis + return axis = axis < 0 ? axis + tensor_rank : axis; +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/activation/activations.cc b/onnxruntime/core/providers/cpu/activation/activations.cc new file mode 100644 index 0000000000000..7d381a9edc422 --- /dev/null +++ b/onnxruntime/core/providers/cpu/activation/activations.cc @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/activation/activations.h" + +namespace onnxruntime { + +#define REGISTER_UNARY_ELEMENTWISE_KERNEL_ALIAS(alias, x, sinceVersion) \ + ONNX_CPU_OPERATOR_KERNEL( \ + alias, \ + sinceVersion, \ + KernelDefBuilder().MayInplace(0, 0).TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + x); + +#define REGISTER_UNARY_ELEMENTWISE_KERNEL(x, sinceVersion) \ + REGISTER_UNARY_ELEMENTWISE_KERNEL_ALIAS(x, x, sinceVersion) + +REGISTER_UNARY_ELEMENTWISE_KERNEL(Elu, 6); +REGISTER_UNARY_ELEMENTWISE_KERNEL(HardSigmoid, 6); +REGISTER_UNARY_ELEMENTWISE_KERNEL(LeakyRelu, 6); +REGISTER_UNARY_ELEMENTWISE_KERNEL(ParametricSoftplus, 1); +REGISTER_UNARY_ELEMENTWISE_KERNEL(Relu, 6); +REGISTER_UNARY_ELEMENTWISE_KERNEL(ScaledTanh, 1); +REGISTER_UNARY_ELEMENTWISE_KERNEL(Selu, 6); +REGISTER_UNARY_ELEMENTWISE_KERNEL(Sigmoid, 6); +// SoftPlus is the default case for ParametricSoftPlus +REGISTER_UNARY_ELEMENTWISE_KERNEL_ALIAS(Softplus, ParametricSoftplus, 1); +REGISTER_UNARY_ELEMENTWISE_KERNEL(Softsign, 1); +REGISTER_UNARY_ELEMENTWISE_KERNEL(Tanh, 6); +REGISTER_UNARY_ELEMENTWISE_KERNEL(ThresholdedRelu, 1); + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/activation/activations.h b/onnxruntime/core/providers/cpu/activation/activations.h new file mode 100644 index 0000000000000..7b5e213d8c9b2 --- /dev/null +++ b/onnxruntime/core/providers/cpu/activation/activations.h @@ -0,0 +1,204 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/util/math_cpuonly.h" + +namespace onnxruntime { + +#define EIGEN_X ConstEigenVectorArrayMap(X->template Data(), X->Shape().Size()) +#define EIGEN_X_VAR(var) ConstEigenVectorArrayMap var(X->template Data(), X->Shape().Size()) +#define EIGEN_Y EigenVectorArrayMap(Y->template MutableData(), Y->Shape().Size()) +#define EIGEN_Y_VAR(var) EigenVectorArrayMap var(Y->template MutableData(), Y->Shape().Size()) + +template +class Elu final : public OpKernel { + public: + Elu(const OpKernelInfo& info) : OpKernel(info), alpha_(info.GetAttrOrDefault("alpha", 1.0f)) {} + + Status Compute(OpKernelContext* context) const override { + const Tensor* X = context->Input(0); + Tensor* Y = context->Output(0, X->Shape()); + EIGEN_X_VAR(xm); + EIGEN_Y = (xm >= 0).select(xm, (T)alpha_ * (xm.exp() - 1)); + return Status::OK(); + } + + private: + const float alpha_; +}; + +template +class HardSigmoid final : public OpKernel { + public: + HardSigmoid(const OpKernelInfo& info) + : OpKernel(info), alpha_(info.GetAttrOrDefault("alpha", 0.2f)), beta_(info.GetAttrOrDefault("beta", 0.5f)) {} + + Status Compute(OpKernelContext* context) const override { + const Tensor* X = context->Input(0); + Tensor* Y = context->Output(0, X->Shape()); + EIGEN_X_VAR(xm); + EIGEN_Y_VAR(ym); + ym = (((T)alpha_ * xm + (T)beta_).cwiseMin(1.0f)).cwiseMax(0.0f); + return Status::OK(); + } + + private: + const float alpha_; + const float beta_; +}; + +template +class LeakyRelu final : public OpKernel { + public: + LeakyRelu(const OpKernelInfo& info) : OpKernel(info), alpha_(info.GetAttrOrDefault("alpha", 0.01f)) {} + + Status Compute(OpKernelContext* context) const override { + const Tensor* X = context->Input(0); + Tensor* Y = context->Output(0, X->Shape()); + EIGEN_X_VAR(xm); + EIGEN_Y = (xm >= 0).select(xm, (T)alpha_ * xm); + return Status::OK(); + } + + private: + const float alpha_; +}; + +template +class ParametricSoftplus final : public OpKernel { + public: + ParametricSoftplus(const OpKernelInfo& info) + : OpKernel(info), alpha_(info.GetAttrOrDefault("alpha", 1.0f)), beta_(info.GetAttrOrDefault("beta", 1.0f)) {} + + Status Compute(OpKernelContext* context) const override { + const Tensor* X = context->Input(0); + Tensor* Y = context->Output(0, X->Shape()); + EIGEN_X_VAR(xm); + EIGEN_Y = (T)alpha_ * + (xm * (T)beta_ > 0) + .select(xm * (T)beta_ + ((-xm * (T)beta_).exp() + 1.0f).log(), ((xm * (T)beta_).exp() + 1.0f).log()); + return Status::OK(); + } + + private: + const float alpha_; + const float beta_; +}; + +template +class Relu final : public OpKernel { + public: + Relu(const OpKernelInfo& info) : OpKernel(info) {} + + Status Compute(OpKernelContext* context) const override { + const Tensor* X = context->Input(0); + Tensor* Y = context->Output(0, X->Shape()); + EIGEN_Y = EIGEN_X.cwiseMax(0); + return Status::OK(); + } +}; + +template +class ScaledTanh final : public OpKernel { + public: + ScaledTanh(const OpKernelInfo& info) + : OpKernel(info), alpha_(info.GetAttrOrDefault("alpha", 1.0f)), beta_(info.GetAttrOrDefault("beta", 1.0f)) {} + + Status Compute(OpKernelContext* context) const override { + const Tensor* X = context->Input(0); + Tensor* Y = context->Output(0, X->Shape()); + EIGEN_Y = (T)alpha_ * (EIGEN_X * (T)beta_).tanh(); + return Status::OK(); + } + + private: + const float alpha_; + const float beta_; +}; + +template +class Selu final : public OpKernel { + public: + // TODO: I don't think float can represent such a long string(1.67326319217681884765625) + Selu(const OpKernelInfo& info) + : OpKernel(info), + alpha_(info.GetAttrOrDefault("alpha", 1.67326319217681884765625f)), + gamma_(info.GetAttrOrDefault("gamma", 1.05070102214813232421875f)) {} + + Status Compute(OpKernelContext* context) const override { + const Tensor* X = context->Input(0); + Tensor* Y = context->Output(0, X->Shape()); + EIGEN_X_VAR(xm); + EIGEN_Y = (T)gamma_ * (xm.cwiseMax(0.0f) + ((T)alpha_ * (xm.array().exp() - 1.0f)).cwiseMin(0.0f)); + return Status::OK(); + } + + private: + const float alpha_; + const float gamma_; +}; + +template +class Sigmoid final : public OpKernel { + public: + Sigmoid(const OpKernelInfo& info) : OpKernel(info) {} + + Status Compute(OpKernelContext* context) const override { + const Tensor* X = context->Input(0); + Tensor* Y = context->Output(0, X->Shape()); + EIGEN_X_VAR(xm); + EIGEN_Y_VAR(ym); + ym = (xm >= 0).select(1 / (1. + (-xm.abs()).exp()), 1 - 1 / (1. + (-xm.abs()).exp())); + return Status::OK(); + } +}; + +template +class Softsign final : public OpKernel { + public: + Softsign(const OpKernelInfo& info) : OpKernel(info) {} + + Status Compute(OpKernelContext* context) const override { + const Tensor* X = context->Input(0); + Tensor* Y = context->Output(0, X->Shape()); + EIGEN_X_VAR(xm); + EIGEN_Y = (1 + xm.abs()).inverse() * xm; + return Status::OK(); + } +}; + +template +class Tanh final : public OpKernel { + public: + Tanh(const OpKernelInfo& info) : OpKernel(info) {} + + Status Compute(OpKernelContext* context) const override { + const Tensor* X = context->Input(0); + Tensor* Y = context->Output(0, X->Shape()); + EIGEN_Y = EIGEN_X.tanh(); + return Status::OK(); + } +}; + +template +class ThresholdedRelu final : public OpKernel { + public: + ThresholdedRelu(const OpKernelInfo& info) : OpKernel(info), alpha_(info.GetAttrOrDefault("alpha", 1.0f)) {} + + Status Compute(OpKernelContext* context) const override { + const Tensor* X = context->Input(0); + Tensor* Y = context->Output(0, X->Shape()); + EIGEN_X_VAR(xm); + EIGEN_Y = (xm > (T)alpha_).select(xm, 0); + return Status::OK(); + } + + private: + const float alpha_; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/controlflow/if.cc b/onnxruntime/core/providers/cpu/controlflow/if.cc new file mode 100644 index 0000000000000..df87b830c8416 --- /dev/null +++ b/onnxruntime/core/providers/cpu/controlflow/if.cc @@ -0,0 +1,221 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/controlflow/if.h" + +#include "core/framework/framework_common.h" +#include "core/framework/op_kernel_context_internal.h" +#include "core/framework/sequential_executor.h" +#include "core/framework/session_state.h" + +#include "core/framework/tensorprotoutils.h" +// #include "core/providers/cpu/tensor/utils.h" + +using namespace ONNX_NAMESPACE; +using namespace onnxruntime::common; + +namespace onnxruntime { + +/* +ONNX_OPERATOR_SET_SCHEMA( + If, + 1, + OpSchema() + .SetDoc("If conditional") + .Input(0, "cond", "Condition for the if", "B") + .Output( + 0, + "outputs", + "Values that are live-out to the enclosing scope. The return values in " + "the `then_branch` and `else_branch` must be of the same shape and same " + "data type.", + "V", + OpSchema::Variadic) + .Attr( + "then_branch", + "Graph to run if condition is true. Has N outputs: values you wish to " + "be live-out to the enclosing scope. The number of outputs must match" + " the number of outputs in the else_branch.", + AttributeProto::GRAPH) + .Attr( + "else_branch", + "Graph to run if condition is false. Has N outputs: values you wish to" + " be live-out to the enclosing scope. The number of outputs must match" + " the number of outputs in the then_branch.", + AttributeProto::GRAPH) + .TypeConstraint("V", OpSchema::all_tensor_types(), "All Tensor types") + .TypeConstraint("B", {"tensor(bool)"}, "Only bool")); +*/ + +ONNX_CPU_OPERATOR_KERNEL(If, + 1, + KernelDefBuilder() + .TypeConstraint("B", DataTypeImpl::GetTensorType()) + .TypeConstraint("V", DataTypeImpl::AllTensorTypes()), + If); + +class IfImpl { + public: + IfImpl(OpKernelContextInternal& context, + const SessionState& session_state); + + // Initialize by validating all the inputs, and allocating the output tensors + Status Initialize(); + + // Execute the batch, by iterating the sequence in each batch entry + // and calling the subgraph with each item in the sequence. + Status Execute(); + + private: + Status AllocateOutputTensors(); + + OpKernelContextInternal& context_; + const SessionState& session_state_; + const GraphViewer& subgraph_; + + int num_outputs_; + std::vector subgraph_output_names_; + std::unordered_map implicit_inputs_; + + enum class AllocationType { + Temporary, + IfOutput + }; + + // track where the fetches provided to subgraph execution were allocated. + std::vector> outputs_; +}; + +Status If::Compute(OpKernelContext* ctx) const { + auto ctx_internal = static_cast(ctx); + + auto condition = *ctx->Input(0)->Data(); + + auto attribute = condition ? "then_branch" : "else_branch"; + auto* session_state = ctx_internal->SubgraphSessionState(attribute); + ONNXRUNTIME_ENFORCE(session_state, "Subgraph SessionState was not found for '", attribute, "' attribute."); + + IfImpl impl{*ctx_internal, *session_state}; + + auto status = impl.Initialize(); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + status = impl.Execute(); + + return status; +} + +IfImpl::IfImpl(OpKernelContextInternal& context, + const SessionState& session_state) + : context_{context}, + session_state_{session_state}, + subgraph_{*session_state.GetGraphViewer()}, + implicit_inputs_{context_.GetImplicitInputs()} { + num_outputs_ = context_.OutputCount(); +} + +Status IfImpl::Initialize() { + auto& graph_outputs = subgraph_.GetOutputs(); + auto num_subgraph_outputs = graph_outputs.size(); + + if (num_subgraph_outputs != num_outputs_) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "'If' node has ", num_outputs_, + " outputs which doesn't match the subgraph's ", num_subgraph_outputs, " outputs."); + } + + subgraph_output_names_.reserve(num_subgraph_outputs); + + // save list of subgraph output names in their provided order to use when fetching the results + // from each subgraph execution. the If outputs will match this order. + for (auto& output : graph_outputs) { + subgraph_output_names_.push_back(output->Name()); + } + + auto status = AllocateOutputTensors(); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + return Status::OK(); +} + +Status IfImpl::AllocateOutputTensors() { + Status status = Status::OK(); + int index = 0; + + for (auto& graph_output : subgraph_.GetOutputs()) { + auto* graph_output_shape = graph_output->Shape(); + if (!graph_output_shape) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Subgraph must have the shape set for all outputs but ", + graph_output->Name(), " did not."); + } + + TensorShape output_shape{onnxruntime::utils::GetTensorShapeFromTensorShapeProto(*graph_output_shape)}; + + // TODO: Task 1913: Improve handling of If outputs to avoid copy when the shape is not known + // at subgraph execution time. + // + // if size < 0 we have a symbolic dimension and need to use a temporary MLValue in the subgraph execution + if (output_shape.Size() < 0) { + outputs_.push_back({AllocationType::Temporary, {}}); + } else { + auto* tensor = context_.Output(index, output_shape); + + if (!tensor) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to create output tensor for ", graph_output->Name()); + + outputs_.push_back({AllocationType::IfOutput, *context_.GetOutputMLValue(index)}); + } + + ++index; + } + + return Status::OK(); +} + +Status IfImpl::Execute() { + Status status = Status::OK(); + + NameMLValMap feeds; + + feeds.reserve(implicit_inputs_.size()); + auto& mlvalue_name_idx_map = session_state_.GetMLValueNameIdxMap(); + + // pass in implicit inputs as feeds. + for (auto& entry : implicit_inputs_) { + ONNXRUNTIME_ENFORCE(entry.second, "All implicit inputs should have MLValue instances by now. ", + entry.first, " did not."); + + // prune to values that are in this subgraph as the implicit inputs cover both 'then' and 'else' subgraphs. + // alternatively we could track implicit inputs on a per-attribute basis in the node, but that + // would make that tracking a bit more complicated. + int idx; + if (mlvalue_name_idx_map.GetIdx(entry.first, idx).IsOK()) { + feeds[entry.first] = *entry.second; + } + } + + std::vector fetches; + fetches.reserve(num_outputs_); + + for (int i = 0; i < num_outputs_; ++i) { + fetches.push_back(outputs_[i].second); + } + + SequentialExecutor executor{context_.GetTerminateFlag()}; + status = executor.Execute(session_state_, feeds, subgraph_output_names_, fetches, context_.Logger()); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + for (int i = 0; i < num_outputs_; ++i) { + // TODO: Task 1913: Improve handling of If outputs to avoid copy when the shape is not known + // at subgraph execution time. + if (outputs_[i].first == AllocationType::Temporary) { + // need to allocate If output and copy MLValue from fetches + auto& data = fetches[i].Get(); + Tensor* output = context_.Output(i, data.Shape()); + memcpy(output->MutableDataRaw(), data.DataRaw(), data.Size()); + } + } + + return status; +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/controlflow/if.h b/onnxruntime/core/providers/cpu/controlflow/if.h new file mode 100644 index 0000000000000..e227dd6cc2b0d --- /dev/null +++ b/onnxruntime/core/providers/cpu/controlflow/if.h @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include "gsl/gsl_util" + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { +class SessionState; + +class If final : public OpKernel { + public: + If(const OpKernelInfo& info) : OpKernel(info) { + // make sure the required attributes are present even though we don't need it here. + // The GraphProto attributes are loaded as a Graph instance by main Graph::Resolve, + // and a SessionState instance for executing the subgraph is created by InferenceSession. + // This is available via Info().GetSubgraphSessionState("attribute_name") when Compute is called. + ONNX_NAMESPACE::GraphProto proto; + ONNXRUNTIME_ENFORCE(info.GetAttr("then_branch", &proto).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttr("else_branch", &proto).IsOK()); + ONNXRUNTIME_IGNORE_RETURN_VALUE(proto); + } + + Status Compute(OpKernelContext* ctx) const override; + + private: +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/controlflow/loop.cc b/onnxruntime/core/providers/cpu/controlflow/loop.cc new file mode 100644 index 0000000000000..1bd429923403f --- /dev/null +++ b/onnxruntime/core/providers/cpu/controlflow/loop.cc @@ -0,0 +1,366 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// there's no way to use a raw pointer as the copy destination with std::copy_n +// (which gsl::copy uses with span::data() which returns a raw pointer) with the 14.11 toolset +// without generating a 4996 warning. going through an iterator is way too much overhead so turn off the warning. +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4996) +#endif + +#include "core/providers/cpu/controlflow/loop.h" + +#include "core/framework/framework_common.h" +#include "core/framework/op_kernel_context_internal.h" +#include "core/framework/sequential_executor.h" +#include "core/framework/session_state.h" +#include "core/framework/tensorprotoutils.h" + +#include "core/providers/cpu/tensor/utils.h" + +#include "gsl/gsl_algorithm" +#include "gsl/span" + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +using namespace ONNX_NAMESPACE; +using namespace onnxruntime::common; + +namespace onnxruntime { +/* +ONNX_OPERATOR_SET_SCHEMA( + Loop, + 1, + OpSchema() + .SetDoc(Loop_ver1_doc) + .Input( + 0, + "M", + "A maximum trip-count for the loop specified at runtime. Optional." + " Pass empty string to skip.", + "I", + OpSchema::Optional) + .Input( + 1, + "cond", + "A boolean termination condition. Optional. Pass empty string to skip.", + "B", + OpSchema::Optional) + .Input( + 2, + "v_initial", + "The initial values of any loop-carried dependencies (values that " + "change across loop iterations)", + "V", + OpSchema::Variadic) + .Output( + 0, + "v_final_and_scan_outputs", + "Final N loop carried dependency values then K scan_outputs", + "V", + OpSchema::Variadic) + .Attr( + "body", + "The graph run each iteration. It has 2+N inputs: (iteration_num, " + "condition, loop carried dependencies...). It has 1+N+K outputs: " + "(condition, loop carried dependencies..., scan_outputs...). Each " + "scan_output is created by concatenating the value of the specified " + "output value at the end of each iteration of the loop. It is an error" + " if the dimensions or data type of these scan_outputs change across loop" + " iterations.", + AttributeProto::GRAPH) + .TypeConstraint("V", OpSchema::all_tensor_types(), "All Tensor types") + .TypeConstraint("I", {"int64"}, "Only int64") + .TypeConstraint("B", {"bool"}, "Only bool") + .TypeAndShapeInferenceFunction(LoopInferenceFunction)); +*/ + +ONNX_CPU_OPERATOR_KERNEL(Loop, + 1, + KernelDefBuilder() + .TypeConstraint("I", DataTypeImpl::GetTensorType()) + .TypeConstraint("B", DataTypeImpl::GetTensorType()) + .TypeConstraint("V", DataTypeImpl::AllTensorTypes()), + Loop); + +class LoopImpl { + public: + LoopImpl(OpKernelContextInternal& context, + const SessionState& session_state); + + // Initialize by validating all the inputs, and allocating the output tensors + Status Initialize(); + + // Execute the batch, by iterating the sequence in each batch entry + // and calling the subgraph with each item in the sequence. + Status Execute(); + + private: + NameMLValMap CreateInitialFeeds(); + void UpdateFeeds(const std::vector& last_output, NameMLValMap& next_input); + + // create the single Loop output from a collection of per-iteration outputs + Status ConcatenateLoopOutput(std::vector& per_iteration_output, int output_index); + + OpKernelContextInternal& context_; + const SessionState& session_state_; + const GraphViewer& subgraph_; + + int64_t max_trip_count_; + bool condition_; + + int num_loop_carried_vars_; + int num_subgraph_inputs_; + int num_outputs_; + + std::unordered_map implicit_inputs_; + + MLValue iter_num_mlvalue_; + MLValue condition_mlvalue_; + + std::vector subgraph_input_names_; + std::vector subgraph_output_names_; + + // collection of MLValue outputs from each loop iteration for the loop outputs. + // the order from the subgraph matches the order from the loop output + std::vector> loop_output_tensors_; +}; + +Status Loop::Compute(OpKernelContext* ctx) const { + auto ctx_internal = static_cast(ctx); + auto* session_state = ctx_internal->SubgraphSessionState("body"); + ONNXRUNTIME_ENFORCE(session_state, "Subgraph SessionState was not found for 'body' attribute."); + + LoopImpl loop_impl{*ctx_internal, *session_state}; + + auto status = loop_impl.Initialize(); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + status = loop_impl.Execute(); + + return status; +} + +LoopImpl::LoopImpl(OpKernelContextInternal& context, + const SessionState& session_state) + : context_{context}, + session_state_{session_state}, + subgraph_{*session_state.GetGraphViewer()}, + implicit_inputs_{context_.GetImplicitInputs()} { + auto* max_trip_count_tensor = context.Input(0); + max_trip_count_ = max_trip_count_tensor ? *max_trip_count_tensor->Data() : INT64_MAX; + + auto cond_tensor = context.Input(1); + condition_ = cond_tensor ? *cond_tensor->Data() : true; + + num_loop_carried_vars_ = context.InputCount() - 2; // skip 'M' and 'cond' + num_subgraph_inputs_ = num_loop_carried_vars_ + 2; // iter_num, cond, loop carried vars + num_outputs_ = context_.OutputCount(); +} + +template +static MLValue MakeScalarMLValue(AllocatorPtr& allocator, T value) { + auto* data_type = DataTypeImpl::GetType(); + std::unique_ptr p_tensor = std::make_unique(data_type, + TensorShape({1}), + allocator->Alloc(data_type->Size()), + allocator->Info(), + allocator); + + *p_tensor->MutableData() = value; + + return MLValue{p_tensor.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()}; +} + +Status LoopImpl::Initialize() { + auto status = Status::OK(); + + auto& subgraph_inputs = subgraph_.GetInputs(); + + // we know how many inputs we are going to call the subgraph with based on the Loop inputs, + // and that value is in num_subgraph_inputs_. + // validate that the subgraph has that many inputs. + if (num_subgraph_inputs_ != subgraph_inputs.size()) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, + "Graph in 'body' attribute of Loop should have ", + num_subgraph_inputs_, " inputs. Found:", subgraph_.GetInputs().size()); + } + + auto& subgraph_outputs = subgraph_.GetOutputs(); + auto num_subgraph_outputs = subgraph_outputs.size(); + + // check num outputs are correct. the 'cond' output from the subgraph is not a Loop output, so diff is 1 + if (num_subgraph_outputs - 1 != num_outputs_) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "'Loop' node has ", num_outputs_, + " outputs so the subgraph requires ", num_outputs_ + 1, + " but has ", num_subgraph_outputs); + } + + AllocatorPtr allocator; + status = context_.GetTempSpaceAllocator(&allocator); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + condition_mlvalue_ = MakeScalarMLValue(allocator, condition_); + iter_num_mlvalue_ = MakeScalarMLValue(allocator, 0); + + subgraph_input_names_.reserve(num_subgraph_inputs_); + for (size_t i = 0; i < num_subgraph_inputs_; ++i) { + subgraph_input_names_.push_back(subgraph_inputs[i]->Name()); + } + + subgraph_output_names_.reserve(num_subgraph_outputs); + loop_output_tensors_.resize(num_outputs_ - num_loop_carried_vars_); + + // save list of subgraph output names in their provided order to use when fetching the results + // from each subgraph execution. the Loop outputs will match this order. + for (size_t i = 0; i < num_subgraph_outputs; ++i) { + auto& output = subgraph_outputs[i]; + subgraph_output_names_.push_back(output->Name()); + } + + return status; +} + +NameMLValMap LoopImpl::CreateInitialFeeds() { + NameMLValMap feeds; + + feeds.reserve(num_subgraph_inputs_ + implicit_inputs_.size()); + + feeds[subgraph_input_names_[0]] = iter_num_mlvalue_; + feeds[subgraph_input_names_[1]] = condition_mlvalue_; + + // populate loop carried var inputs which conveniently start at slot 2 in both the Loop and subgraph inputs + for (int i = 2; i < num_subgraph_inputs_; ++i) { + feeds[subgraph_input_names_[i]] = *context_.GetInputMLValue(i); + } + + // pass in implicit inputs as feeds. + for (auto& entry : implicit_inputs_) { + ONNXRUNTIME_ENFORCE(entry.second, "All implicit inputs should have MLValue instances by now. ", + entry.first, " did not."); + feeds[entry.first] = *entry.second; + } + + return feeds; +} + +void LoopImpl::UpdateFeeds(const std::vector& last_output, NameMLValMap& next_input) { + // last_output: cond, loop vars..., loop output... + // next_input: iter_num, cond, loop_vars. iter_num is re-used + + // simple copy for cond and loop carried vars. + for (int i = 1; i < num_subgraph_inputs_; ++i) { + next_input[subgraph_input_names_[i]] = last_output[i - 1]; // skip iter_num in input + } + + // save loop outputs as we have to concatenate at the end + for (int j = num_loop_carried_vars_; j < num_outputs_; ++j) { + loop_output_tensors_[j - num_loop_carried_vars_].push_back(last_output[j + 1]); // skip 'cond' in output + } +} + +Status LoopImpl::ConcatenateLoopOutput(std::vector& per_iteration_output, int output_index) { + const auto& first_output = per_iteration_output.front().Get(); + size_t bytes_per_iteration = first_output.Size(); + const auto& per_iteration_shape = first_output.Shape(); + const auto& per_iteration_dims = per_iteration_shape.GetDims(); + + // prepend number of iterations to the dimensions + int64_t num_iterations = gsl::narrow_cast(per_iteration_output.size()); + std::vector dims{num_iterations}; + std::copy(per_iteration_dims.cbegin(), per_iteration_dims.cend(), std::back_inserter(dims)); + TensorShape output_shape{dims}; + + Tensor* output = context_.Output(output_index, output_shape); + + // we can't easily use a C++ template for the tensor element type, + // so use a span for some protection but work in bytes + gsl::span output_span = gsl::make_span(static_cast(output->MutableDataRaw()), + output->Size()); + + for (int64_t i = 0; i < num_iterations; ++i) { + auto& mlvalue = per_iteration_output[i]; + auto& iteration_data = mlvalue.Get(); + + // sanity check + if (bytes_per_iteration != iteration_data.Size()) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Inconsistent shape in loop output for output ", output_index, + " Expected:", per_iteration_shape, " Got:", iteration_data.Shape()); + } + + auto num_bytes = iteration_data.Size(); + auto src = gsl::make_span(static_cast(iteration_data.DataRaw()), num_bytes); + auto dst = output_span.subspan(i * bytes_per_iteration, bytes_per_iteration); + gsl::copy(src, dst); + } + + return Status::OK(); +} + +Status LoopImpl::Execute() { + auto status = Status::OK(); + + NameMLValMap feeds{CreateInitialFeeds()}; + std::vector fetches; + + auto& iter_num_value = *iter_num_mlvalue_.GetMutable()->MutableData(); + + while (iter_num_value < max_trip_count_ && *condition_mlvalue_.GetMutable()->MutableData()) { + if (iter_num_value != 0) { + UpdateFeeds(fetches, feeds); + fetches.clear(); + } + + SequentialExecutor executor{context_.GetTerminateFlag()}; + status = executor.Execute(session_state_, feeds, subgraph_output_names_, fetches, context_.Logger()); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + condition_mlvalue_ = fetches[0]; + + ++iter_num_value; + } + + // TODO: Task 1913: Improve handling of If and Loop outputs to avoid copy + auto copy_tensor_from_mlvalue_to_output = [this](const MLValue& input, int output_idx) { + auto& data = input.Get(); + Tensor* output = context_.Output(output_idx, data.Shape()); + auto src = gsl::make_span(static_cast(data.DataRaw()), data.Size()); + auto dst = gsl::make_span(static_cast(output->MutableDataRaw()), output->Size()); + gsl::copy(src, dst); + }; + + // copy to Loop output + if (iter_num_value != 0) { + for (int i = 0; i < num_loop_carried_vars_; ++i) { + // need to allocate Loop output and copy MLValue from fetches + copy_tensor_from_mlvalue_to_output(fetches[i + 1], i); // skip cond + } + + for (int i = num_loop_carried_vars_; i < num_outputs_; ++i) { + // add last output + auto& per_iteration_outputs = loop_output_tensors_[i - num_loop_carried_vars_]; + per_iteration_outputs.push_back(fetches[i + 1]); // skip cond + + ONNXRUNTIME_RETURN_IF_ERROR(ConcatenateLoopOutput(per_iteration_outputs, i)); + } + } else { + // no iterations. + // copy input loop carried vars to output. + for (int i = 0; i < num_loop_carried_vars_; ++i) { + copy_tensor_from_mlvalue_to_output(feeds[subgraph_input_names_[i + 2]], i); // skip iter# and cond + } + + // create empty outputs for loop outputs + TensorShape empty; + for (int i = num_loop_carried_vars_; i < num_outputs_; ++i) { + ONNXRUNTIME_IGNORE_RETURN_VALUE(context_.Output(i, empty)); + } + } + return status; +} +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/controlflow/loop.h b/onnxruntime/core/providers/cpu/controlflow/loop.h new file mode 100644 index 0000000000000..d21e8d3d3d8b2 --- /dev/null +++ b/onnxruntime/core/providers/cpu/controlflow/loop.h @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include "gsl/gsl_util" + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { +class Loop final : public OpKernel { + public: + Loop(const OpKernelInfo& info) : OpKernel(info) { + // make sure the attribute was present even though we don't need it here. + // The GraphProto is loaded as a Graph instance by main Graph::Resolve, + // and a SessionState instance for executing the subgraph is created by InferenceSession. + // This is available via Info().GetSubgraphSessionState("attribute_name") when Compute is called. + ONNX_NAMESPACE::GraphProto proto; + ONNXRUNTIME_ENFORCE(info.GetAttr("body", &proto).IsOK()); + ONNXRUNTIME_IGNORE_RETURN_VALUE(proto); + } + + Status Compute(OpKernelContext* ctx) const override; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/controlflow/scan.cc b/onnxruntime/core/providers/cpu/controlflow/scan.cc new file mode 100644 index 0000000000000..cbd6044c8287e --- /dev/null +++ b/onnxruntime/core/providers/cpu/controlflow/scan.cc @@ -0,0 +1,628 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/controlflow/scan.h" + +#include "core/framework/framework_common.h" +#include "core/framework/mlvalue_tensor_slicer.h" +#include "core/framework/op_kernel_context_internal.h" +#include "core/framework/sequential_executor.h" +#include "core/framework/session_state.h" +#include "core/framework/tensorprotoutils.h" + +#include "core/providers/cpu/tensor/utils.h" + +using namespace ONNX_NAMESPACE; +using namespace onnxruntime::common; + +namespace onnxruntime { + +/* +ONNX_OPERATOR_SET_SCHEMA( + Scan, + 8, + OpSchema() + .SetDoc(scan_ver1_doc) + .Input( + 0, + "sequence_lens", + "Optional tensor specifying lengths of the sequences in a batch. " + "If this input is not specified, all sequences are assumed to be of " + "the maximum sequence length (the dimension of the sequence axis of " + "the scan_input tensors).", + "I", + OpSchema::Optional) + .Input( + 1, + "initial_state_and_scan_inputs", + "Initial values of the loop's N state variables followed by M scan_inputs", + "V", + OpSchema::Variadic) + .Output( + 0, + "final_state_and_scan_outputs", + "Final values of the loop's N state variables followed by K scan_outputs", + "V", + OpSchema::Variadic) + .Attr( + "body", + "The graph run each iteration. It has N+M inputs: " + "(loop state variables..., scan_input_elts...). It has N+K outputs: " + "(loop state variables..., scan_output_elts...). Each " + "scan_output is created by concatenating the value of the specified " + "scan_output_elt value at the end of each iteration of the loop. It is an error" + " if the dimensions of these values change across loop iterations.", + AttributeProto::GRAPH, + true) + .Attr( + "num_scan_inputs", + "An attribute specifying the number of scan_inputs M. ", + AttributeProto::INT, + true) + .Attr( + "directions", + "An optional list of M flags. The i-th element of the list specifies the direction " + "to be scanned for the i-th scan_input tensor: 0 indicates forward direction and 1 " + "indicates reverse direction. " + "If omitted, all scan_input tensors will be scanned in the forward direction.", + AttributeProto::INTS, + false) + .TypeConstraint("I", { "tensor(int64)" }, "Int64 tensor") + .TypeConstraint("V", OpSchema::all_tensor_types(), "All Tensor types")); +*/ + +ONNX_CPU_OPERATOR_KERNEL(Scan, + 8, + KernelDefBuilder() + .TypeConstraint("I", DataTypeImpl::GetTensorType()) + .TypeConstraint("V", DataTypeImpl::AllTensorTypes()), + Scan); + +/** +Class to provide input/output MLValue instances for a loop state variable. +The MLValue flips between two internal temporary buffers to minimize copies. +*/ +class LoopStateVariable { + public: + LoopStateVariable(const MLValue& original_value, MLValue& final_value, const int64_t sequence_len, + AllocatorPtr& allocator); + + // get current Input MLValue + const MLValue& Input() const; + + // get current Output MLValue + MLValue& Output(); + + // move to next usage of the loop state variable. call after each iteration of the subgraph. + void Next(); + + private: + int64_t iteration_num_{0}; + const int64_t sequence_len_; + + // copy original and final value from temporary MLValue provided by iterator + const MLValue original_value_; + MLValue final_value_; + + /* we use original_value and final_value once, + and alternate between a_ and b_ as input/output for each iteration to avoid copies + + Iteration Input Output + 0 original_value a_ + 1 a_ b_ + 2 b_ a_ + ... + seq len - 1 final_value + */ + MLValue a_; + MLValue b_; +}; + +class ScanImpl { + public: + ScanImpl(OpKernelContextInternal& context, + const SessionState& session_state, + int64_t num_scan_inputs, + const std::vector& directions); + + // Initialize by validating all the inputs, and allocating the output tensors + Status Initialize(); + + // Execute the batch, by iterating the sequence in each batch entry + // and calling the subgraph with each item in the sequence. + Status Execute(); + + private: + // validate inputs and setup batch size and max sequence length. + Status ValidateInput(); + Status ValidateSubgraphInput(int start_input, int end_input, bool has_seq_len_dim, + const std::vector& graph_inputs); + + Status AllocateOutput(int index, bool has_sequence_len); + Status AllocateOutputTensors(); + Status CreateLoopStateVariables(std::vector>& loop_state_variables); + + using ConstTensorSlicerIterators = std::vector::Iterator>; + using MutableTensorSlicerIterators = std::vector::Iterator>; + + Status IterateSequence(std::vector& loop_state_variables, + ConstTensorSlicerIterators& scan_input_stream_iterators, + MutableTensorSlicerIterators& scan_output_stream_iterators, + int64_t seq_length); + + OpKernelContextInternal& context_; + const SessionState& session_state_; + const GraphViewer& subgraph_; + + int num_loop_state_variables_; + int num_variadic_inputs_; + int num_variadic_outputs_; + + int64_t batch_size_ = -1; + int64_t max_sequence_len_ = -1; + + const std::vector& directions_; + const Tensor* sequence_lens_tensor_; + std::vector sequence_lens_; + + std::vector subgraph_output_names_; + + std::unordered_map implicit_inputs_; +}; + +Status Scan::Compute(OpKernelContext* ctx) const { + auto ctx_internal = static_cast(ctx); + auto* session_state = ctx_internal->SubgraphSessionState("body"); + ONNXRUNTIME_ENFORCE(session_state, "Subgraph SessionState was not found for 'body' attribute."); + + // TODO: + // Consider how usage of ExecutionFrame and SequentialExecutor can be optimized + // - initial implementation is focused on making it work, rather than optimizing. + + ScanImpl scan_impl{*ctx_internal, *session_state, num_scan_inputs_, directions_}; + + auto status = scan_impl.Initialize(); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + status = scan_impl.Execute(); + + return status; +} + +LoopStateVariable::LoopStateVariable(const MLValue& original_value, + MLValue& final_value, + const int64_t sequence_len, + AllocatorPtr& allocator) + : sequence_len_{sequence_len}, + original_value_{original_value}, + final_value_{final_value} { + auto& tensor = original_value.Get(); + auto& shape = tensor.Shape(); + + // allocate a new Tensor in an MLValue with the same shape and type as the tensor in original_value. + // the Tensor will own the buffer, and the MLValue will own the Tensor. + // the MLValue returned by Input()/Output() gets copied into the execution frame feeds/fetches + // with the Tensor being used via a shared_ptr (so remains valid during execution and is cleaned up + // automatically at the end). + // TODO: Could allocate one large chunk for all the loop state variable buffers in ScanImpl, although that + // may make it harder to parallelize processing of the batch in the future. + auto allocate_tensor_in_mlvalue = [&]() { + auto new_tensor = std::make_unique(tensor.DataType(), + shape, + allocator->Alloc(shape.Size() * tensor.DataType()->Size()), + allocator->Info(), + allocator); + + return MLValue{new_tensor.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()}; + }; + + // if length is > 1, we need a_ for the first output location. otherwise we use final_value for the output. + if (sequence_len_ > 1) { + a_ = allocate_tensor_in_mlvalue(); + } + + // if length is > 2, we need b_ for the second output location + if (sequence_len_ > 2) { + b_ = allocate_tensor_in_mlvalue(); + } +} + +const MLValue& LoopStateVariable::Input() const { + if (iteration_num_ == 0) + return original_value_; + + return iteration_num_ % 2 == 1 ? a_ : b_; +} + +MLValue& LoopStateVariable::Output() { + if (iteration_num_ + 1 == sequence_len_) { + return final_value_; + } + + return iteration_num_ % 2 == 1 ? b_ : a_; +} + +void LoopStateVariable::Next() { + ONNXRUNTIME_ENFORCE(iteration_num_ < sequence_len_, "Misuse of LoopStateVariable. Attempt to move beyond end of sequence"); + ++iteration_num_; +} + +ScanImpl::ScanImpl(OpKernelContextInternal& context, + const SessionState& session_state, + int64_t num_scan_inputs, + const std::vector& directions) + : context_{context}, + session_state_{session_state}, + subgraph_{*session_state.GetGraphViewer()}, + directions_{directions}, + implicit_inputs_{context_.GetImplicitInputs()} { + //optional first input so may be nullptr + sequence_lens_tensor_ = context.Input(0); + + num_variadic_inputs_ = context_.NumVariadicInputs(1); + num_variadic_outputs_ = context_.OutputCount(); + + num_loop_state_variables_ = num_variadic_inputs_ - gsl::narrow_cast(num_scan_inputs); +} + +Status ScanImpl::Initialize() { + auto status = ValidateInput(); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + auto& graph_outputs = subgraph_.GetOutputs(); + subgraph_output_names_.reserve(graph_outputs.size()); + + // save list of subgraph output names in their provided order to use when fetching the results + // from each subgraph execution. the Scan outputs will match this order. + for (auto& output : graph_outputs) { + subgraph_output_names_.push_back(output->Name()); + } + + status = AllocateOutputTensors(); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + return Status::OK(); +} + +// get the Scan input that is used in a call to the subgraph as a Tensor, +// skipping over the optional arg to the Scan operator +static const Tensor& GetSubgraphInputTensor(const OpKernelContext& context, int index) { + // skip the optional sequence_lens input + return *context.Input(index + 1); +} + +// get the Scan input that is used in a call to the subgraph as an MLValue, +// skipping over the optional arg to the Scan operator +static const MLValue& GetSubgraphInputMLValue(const OpKernelContextInternal& context, int index) { + // skip the optional sequence_lens input + return *context.GetInputMLValue(index + 1); +} + +// Validate that the subgraph input has valid shapes +Status ScanImpl::ValidateSubgraphInput(int start_input, int end_input, bool has_seq_len_dim, + const std::vector& graph_inputs) { + // first dim is batch size. optional sequence dim. dim/s for the data + auto min_dims_required = has_seq_len_dim ? 3 : 2; + + for (int i = start_input; i < end_input; ++i) { + auto& input_tensor = GetSubgraphInputTensor(context_, i); + const auto& input_shape = input_tensor.Shape(); + + if (input_shape.NumDimensions() < min_dims_required) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Invalid scan input:", graph_inputs[i]->Name(), + " Expected ", min_dims_required, + " dimensions or more but input had shape of ", input_shape); + + auto this_batch_size = input_shape[0]; + + if (batch_size_ < 0) + batch_size_ = this_batch_size; + else { + if (batch_size_ != this_batch_size) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Scan inputs have inconsistent batch size. Previous value was ", + batch_size_, " but ", graph_inputs[i]->Name(), " has batch size of ", + this_batch_size); + } + } + + if (has_seq_len_dim) { + auto this_seq_len = input_shape[1]; + + if (max_sequence_len_ < 0) { + max_sequence_len_ = this_seq_len; + } else { + if (max_sequence_len_ != this_seq_len) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Scan inputs have inconsistent sequence lengths. Previous value was ", + max_sequence_len_, " but ", graph_inputs[i]->Name(), + " has length of ", this_seq_len); + } + } + } + } + + return Status::OK(); +} + +Status ScanImpl::ValidateInput() { + auto& graph_inputs = subgraph_.GetInputs(); + auto num_graph_inputs = graph_inputs.size(); + + if (num_graph_inputs != num_variadic_inputs_) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "The subgraph in 'body' expects ", num_graph_inputs, + " inputs but Scan was only given ", num_variadic_inputs_); + } + + // process any loop state variables, which will set the batch size + auto status = ValidateSubgraphInput(0, num_loop_state_variables_, false, graph_inputs); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + // process the scan inputs. sets/validates batch size and sequence length + status = ValidateSubgraphInput(num_loop_state_variables_, num_variadic_inputs_, true, graph_inputs); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + if (sequence_lens_tensor_ != nullptr) { + auto num_entries = sequence_lens_tensor_->Shape().Size(); + + if (num_entries != batch_size_) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "sequence_lens length of ", num_entries, + " did not match batch size of ", batch_size_); + } + + auto d = sequence_lens_tensor_->DataAsSpan(); + sequence_lens_.assign(d.cbegin(), d.cend()); + + if (std::all_of(sequence_lens_.cbegin(), sequence_lens_.cend(), + [this](int64_t value) { return value > 0 && value <= max_sequence_len_; }) == false) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Invalid entries in sequence_lens. Max sequence length was ", max_sequence_len_); + } + + } else { + sequence_lens_ = std::vector(batch_size_, max_sequence_len_); + } + + return Status::OK(); +} + +Status ScanImpl::AllocateOutput(int index, bool has_sequence_len_dimension) { + // use the shape from the subgraph output. we require this to be specified in the model or inferable. + auto& graph_outputs = subgraph_.GetOutputs(); + auto* graph_output = graph_outputs.at(index); + auto* graph_output_shape = graph_output->Shape(); + if (!graph_output_shape) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Subgraph must have the shape set for all outputs but ", + graph_output->Name(), " did not."); + } + + TensorShape output_shape{onnxruntime::utils::GetTensorShapeFromTensorShapeProto(*graph_output_shape)}; + auto& graph_output_dims{output_shape.GetDims()}; + + std::vector scan_output_dims; + scan_output_dims.reserve(graph_output_dims.size() + 2); + + scan_output_dims.push_back(batch_size_); + + if (has_sequence_len_dimension) { + scan_output_dims.push_back(max_sequence_len_); + } + + scan_output_dims.insert(scan_output_dims.cend(), graph_output_dims.cbegin(), graph_output_dims.cend()); + + // make sure a single buffer for the full output is created upfront. + // we slice this into per-iteration pieces in Execute using MLValueTensorSlicer. + auto* tensor = context_.Output(index, TensorShape(scan_output_dims)); + + if (!tensor) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to create output tensor for ", graph_output->Name()); + + // zero out the output so that any short sequences have deterministic values in unused slots. + // strictly speaking this isn't required, and alternatively we could fill with zeros when we + // encounter a short sequence and are creating output, but one memset is easy, involves + // less code complexity, and should be relatively cheap. + memset(tensor->MutableDataRaw(), 0, tensor->Size()); + + return Status::OK(); +} + +Status ScanImpl::AllocateOutputTensors() { + Status status = Status::OK(); + auto& graph_outputs = subgraph_.GetOutputs(); + + if (graph_outputs.size() != num_variadic_outputs_) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Subgraph in 'body' produces ", graph_outputs.size(), + " outputs but Scan expects ", num_variadic_outputs_); + } + + // TODO: Need to handle shape/type inference for subgraphs. + // For now copy shape from subgraph output and expand based on batch size and sequence length + + for (int i = 0; i < num_loop_state_variables_; ++i) { + const bool has_sequence_len_dimension = false; // loop state variables don't have a sequence_len dimension; + status = AllocateOutput(i, has_sequence_len_dimension); + ONNXRUNTIME_RETURN_IF_ERROR(status); + } + + for (int i = num_loop_state_variables_, end = num_variadic_outputs_; i < end; ++i) { + status = AllocateOutput(i, true); + ONNXRUNTIME_RETURN_IF_ERROR(status); + } + + return Status::OK(); +} + +// setup the loop state variables for each batch item +Status ScanImpl::CreateLoopStateVariables(std::vector>& batch_loop_state_variables) { + // Setup loop state variables + // 1. Slice the input/output loop state variable tensors provided to Scan into the per-batch-item chunks + // (slice on the first dimension which is the batch size). + // 2. For each batch item, create the LoopStateVariable instances that can be used to pass state between + // each iteration of the subgraph. This minimizes copying of data during each iteration. + + std::vector::Iterator> loop_state_input_iterators; + std::vector::Iterator> loop_state_output_iterators; + loop_state_input_iterators.reserve(num_loop_state_variables_); + loop_state_output_iterators.reserve(num_loop_state_variables_); + + // create the input and output slice iterator for each loop state variable. + for (int i = 0; i < num_loop_state_variables_; ++i) { + const MLValue& mlvalue = GetSubgraphInputMLValue(context_, i); + MLValue* p_mlvalue = context_.GetOutputMLValue(i); + + ONNXRUNTIME_ENFORCE(p_mlvalue, "Output MLValue has not been created for loop state variable output ", i); + + loop_state_input_iterators.push_back(MLValueTensorSlicer::Create(mlvalue).begin()); + loop_state_output_iterators.push_back(MLValueTensorSlicer::Create(*p_mlvalue).begin()); + } + + batch_loop_state_variables.clear(); + batch_loop_state_variables.resize(batch_size_); + + AllocatorPtr alloc; + auto status = context_.GetTempSpaceAllocator(&alloc); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + // setup the loop state variables for each batch row + for (int64_t b = 0; b < batch_size_; ++b) { + std::vector& variables = batch_loop_state_variables[b]; + variables.reserve(num_loop_state_variables_); + + for (int i = 0; i < num_loop_state_variables_; ++i) { + auto& input_iter = loop_state_input_iterators[i]; + auto& output_iter = loop_state_output_iterators[i]; + + variables.push_back(LoopStateVariable(*input_iter, *output_iter, sequence_lens_[b], alloc)); + + ++input_iter; + ++output_iter; + } + } + + return status; +} + +Status ScanImpl::Execute() { + Status status = Status::OK(); + + // for each batch item, std::vector of LoopStateVariables + std::vector> batch_loop_state_variables; + status = CreateLoopStateVariables(batch_loop_state_variables); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + for (int64_t b = 0; b < batch_size_; ++b) { + // Setup input MLValue streams + std::vector::Iterator> scan_input_stream_iterators; + scan_input_stream_iterators.reserve(num_variadic_inputs_ - num_loop_state_variables_); + + for (int i = num_loop_state_variables_, end = num_variadic_inputs_; i < end; ++i) { + const auto& mlvalue = GetSubgraphInputMLValue(context_, i); + + // forward + if (directions_[i - num_loop_state_variables_] == static_cast(Scan::Direction::kForward)) { + // the iterator is self contained, so we don't need to keep the MLValueTensorSlicer instance around + scan_input_stream_iterators.push_back(MLValueTensorSlicer::Create(mlvalue, 1, b).begin()); + } else { // reverse + scan_input_stream_iterators.push_back(MLValueTensorSlicer::Create(mlvalue, 1, b).rbegin()); + // need to skip past the empty entries at the end of the input if sequence length is short + auto offset = max_sequence_len_ - sequence_lens_[b]; + if (offset > 0) { + // reverse iterator so += moves backwards through the input + scan_input_stream_iterators.back() += offset; + } + } + } + + // Setup output MLValue streams + std::vector::Iterator> scan_output_stream_iterators; + scan_output_stream_iterators.reserve(num_variadic_outputs_); + + for (int i = num_loop_state_variables_, end = num_variadic_outputs_; i < end; ++i) { + MLValue* p_mlvalue = context_.GetOutputMLValue(i); + ONNXRUNTIME_ENFORCE(p_mlvalue, "Output MLValue has not been created for output ", i); + + scan_output_stream_iterators.push_back(MLValueTensorSlicer::Create(*p_mlvalue, 1, b).begin()); + } + + // Call the subgraph for each item in the sequence + status = IterateSequence(batch_loop_state_variables[b], + scan_input_stream_iterators, + scan_output_stream_iterators, + sequence_lens_[b]); + + ONNXRUNTIME_RETURN_IF_ERROR(status); + } + + return status; +} + +Status ScanImpl::IterateSequence(std::vector& loop_state_variables, + ConstTensorSlicerIterators& scan_input_stream_iterators, + MutableTensorSlicerIterators& scan_output_stream_iterators, + int64_t seq_length) { + Status status = Status::OK(); + auto& graph_inputs = subgraph_.GetInputs(); + NameMLValMap feeds; + std::vector fetches; + + feeds.reserve(num_variadic_inputs_ + implicit_inputs_.size()); + fetches.resize(num_variadic_outputs_); + + // pass in implicit inputs as feeds. + for (auto& entry : implicit_inputs_) { + ONNXRUNTIME_ENFORCE(entry.second, "All implicit inputs should have MLValue instances by now. ", + entry.first, " did not."); + feeds[entry.first] = *entry.second; + } + + // as we fill all the outputs with 0 initially, just iterate seq_length not max_seq_length_ + // as we don't need to pad the output for a short sequence here. + for (int64_t seq_no = 0; seq_no < seq_length; ++seq_no) { + for (int input = 0; input < num_variadic_inputs_; ++input) { + // the ordering of the Scan inputs should match the ordering of the subgraph inputs + auto name = graph_inputs[input]->Name(); + + if (input < num_loop_state_variables_) { + // add loop state variable input + feeds[name] = loop_state_variables[input].Input(); + } else { + // add sliced input + auto& iterator = scan_input_stream_iterators[input - num_loop_state_variables_]; + feeds[name] = *iterator; + + ++iterator; + } + } + + fetches.clear(); + + for (int output = 0, end = num_variadic_outputs_; output < end; ++output) { + if (output < num_loop_state_variables_) { + // add loop state variable output + fetches.push_back(loop_state_variables[output].Output()); + } else { + // add sliced output + auto& iterator = scan_output_stream_iterators[output - num_loop_state_variables_]; + fetches.push_back(*iterator); + ++iterator; + } + } + + // Create Executor and run graph. + // TODO: Consider pulling ExecutionFrame up from within SequentialExecutor + // and separating it out a bit so we can maybe just update the feeds/fetches in the frame on each iteration. + // Many of the other pieces are constant across usages. + // Not sure how best to handle the memory pattern side of things though. + // For now just making it work. Optimization and refinement will follow. + SequentialExecutor executor{context_.GetTerminateFlag()}; + status = executor.Execute(session_state_, feeds, subgraph_output_names_, fetches, context_.Logger()); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + // cycle the LoopStateVariable input/output in preparation for the next iteration + std::for_each(loop_state_variables.begin(), loop_state_variables.end(), [](LoopStateVariable& v) { v.Next(); }); + } + + return status; +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/controlflow/scan.h b/onnxruntime/core/providers/cpu/controlflow/scan.h new file mode 100644 index 0000000000000..c7263c200b8dc --- /dev/null +++ b/onnxruntime/core/providers/cpu/controlflow/scan.h @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include "gsl/gsl_util" + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { +class Scan final : public OpKernel { + public: + Scan(const OpKernelInfo& info) : OpKernel(info) { + // make sure the attribute was present even though we don't need it here. + // The GraphProto is loaded as a Graph instance by main Graph::Resolve, + // and a SessionState instance for executing the subgraph is created by InferenceSession. + // This is available via Info().GetSubgraphSessionState("attribute_name") when Compute is called. + ONNX_NAMESPACE::GraphProto proto; + ONNXRUNTIME_ENFORCE(info.GetAttr("body", &proto).IsOK()); + (void)proto; + + ONNXRUNTIME_ENFORCE(info.GetAttr("num_scan_inputs", &num_scan_inputs_).IsOK()); + + if (info.GetAttrs("directions", directions_).IsOK()) { + ONNXRUNTIME_ENFORCE(gsl::narrow_cast(directions_.size()) == num_scan_inputs_, + "Number of entries in 'directions' was ", directions_.size(), + ". Must match 'num_scan_inputs' of ", num_scan_inputs_); + ONNXRUNTIME_ENFORCE(std::all_of(directions_.cbegin(), directions_.cend(), + [](int64_t i) { return i == static_cast(Direction::kForward) || + i == static_cast(Direction::kReverse); }), + "Invalid values in 'directions'. 0 == forward. 1 == reverse."); + } else { + // default to forward + directions_ = std::vector(num_scan_inputs_, static_cast(Direction::kForward)); + } + } + + Status Compute(OpKernelContext* ctx) const override; + + enum class Direction { kForward = 0, + kReverse = 1 }; + + private: + int64_t num_scan_inputs_; + std::vector directions_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc new file mode 100644 index 0000000000000..39ec9c6fc4d4c --- /dev/null +++ b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc @@ -0,0 +1,473 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/cpu_execution_provider.h" +#include "core/framework/op_kernel.h" +#include "core/framework/kernel_registry.h" +#include "contrib_ops/contrib_ops.h" +#include "core/framework/computation_capacity.h" + +namespace onnxruntime { + +// Forward declarations of op kernels +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, Clip); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, Elu); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, HardSigmoid); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, LeakyRelu); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, ParametricSoftplus); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, Relu); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, ScaledTanh); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, Selu); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, Sigmoid); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, Softplus); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, Softsign); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, Tanh); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, ThresholdedRelu); +class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, 9, PRelu); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, RandomNormal); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, RandomUniform); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, RandomNormalLike); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, RandomUniformLike); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, Multinomial); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, float, Add); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, int32_t, Add); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, int64_t, Add); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, float, Sub); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, int32_t, Sub); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, int64_t, Sub); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, float, Mul); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, double, Mul); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, int32_t, Mul); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, int64_t, Mul); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, float, Div); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, int32_t, Div); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, int64_t, Div); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, float, Abs); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, double, Abs); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, int8_t, Abs); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, int16_t, Abs); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, int32_t, Abs); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, int64_t, Abs); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, uint8_t, Abs); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, uint16_t, Abs); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, uint32_t, Abs); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, uint64_t, Abs); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, float, Neg); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, int8_t, Neg); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, int32_t, Neg); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, Floor); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, Ceil); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, Reciprocal); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, Sqrt); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, Pow); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, Exp); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, Log); +class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, 7, Sum); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 8, Sum); +class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, 7, Min); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 8, Min); +class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, 7, Max); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 8, Max); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, Not); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, And); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, Or); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, Xor); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, 9, float, Less); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, int32_t, Less); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, 9, float, Greater); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, int32_t, Greater); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, bool, Equal); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, int32_t, Equal); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, int64_t, Equal); +class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, 7, Mean); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 8, Mean); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, Affine); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, Sin); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, Cos); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, Tan); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, Asin); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, Acos); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, Atan); +class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, 9, Gemm); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, Hardmax); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, LogSoftmax); +class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, 9, MatMul); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, Softmax); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, TopK); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, BatchNormalization); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, Conv); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, ConvTranspose); +class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, 9, Flatten); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, InstanceNormalization); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, LpNormalization); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, LRN); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, AveragePool); +class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, 7, MaxPool); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 8, MaxPool); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 2, LpPool); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 2, GlobalLpPool); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, GlobalAveragePool); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, GlobalMaxPool); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, MaxRoiPool); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, float, ReduceL1); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, int32_t, ReduceL1); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, float, ReduceL2); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, int32_t, ReduceL2); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, float, ReduceLogSum); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, int32_t, ReduceLogSum); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, float, ReduceLogSumExp); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, int32_t, ReduceLogSumExp); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, float, ReduceMax); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, int32_t, ReduceMax); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, float, ReduceMean); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, int32_t, ReduceMean); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, float, ReduceMin); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, int32_t, ReduceMin); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, float, ReduceProd); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, int32_t, ReduceProd); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, float, ReduceSum); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, int32_t, ReduceSum); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, float, ReduceSumSquare); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, int32_t, ReduceSumSquare); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, float, ArgMax); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, int32_t, ArgMax); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, float, ArgMin); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, int32_t, ArgMin); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, GRU); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, LSTM); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, RNN); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, uint8_t, Cast); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, uint16_t, Cast); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, uint32_t, Cast); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, uint64_t, Cast); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, int8_t, Cast); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, int16_t, Cast); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, int32_t, Cast); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, int64_t, Cast); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, bool, Cast); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, float, Cast); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, double, Cast); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, MLFloat16, Cast); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 4, Concat); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, Crop); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, Gather); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, Dropout); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, Identity); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, ImageScaler); +class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, 8, MeanVarianceNormalization); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, MeanVarianceNormalization); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 2, Pad); +class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, 4, Reshape_1); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 5, Reshape); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, Shape); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, Size); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, Slice); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, Compress); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, SpaceToDepth); +class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, 4, DepthToSpace); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 2, Split); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, Squeeze); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, Tile); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, Transpose); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, Unsqueeze); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, float, Upsample); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 7, int32_t, Upsample); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 8, float, Expand); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 8, Scan); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, Scale); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, If); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, Loop); + +// Opset 9 +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, ConstantLike); + +void RegisterOnnxOperatorKernels(std::function fn) { + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + + // Opset 9 + fn(BuildKernel()); +} + +// Forward declarations of ml op kernels +namespace ml { +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, float, ArrayFeatureExtractor); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, double, ArrayFeatureExtractor); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, int32_t, ArrayFeatureExtractor); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, int64_t, ArrayFeatureExtractor); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, string, ArrayFeatureExtractor); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, Binarizer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, CastMap); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, CategoryMapper); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, string_int64_t, DictVectorizer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, string_float, DictVectorizer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, string_double, DictVectorizer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, int64_t_string, DictVectorizer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, int64_t_float, DictVectorizer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, int64_t_double, DictVectorizer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, FeatureVectorizer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, Imputer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, LabelEncoder); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, float, LinearClassifier); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, double, LinearClassifier); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, int64_t, LinearClassifier); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, int32_t, LinearClassifier); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, LinearRegressor); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, Normalizer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, int64_t, OneHotEncoder); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, float, OneHotEncoder); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, double, OneHotEncoder); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, string, OneHotEncoder); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, float, Scaler); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, double, Scaler); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, int64_t, Scaler); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, int32_t, Scaler); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, float, SVMClassifier); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, double, SVMClassifier); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, int64_t, SVMClassifier); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, int32_t, SVMClassifier); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, SVMRegressor); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, float, TreeEnsembleClassifier); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, double, TreeEnsembleClassifier); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, int64_t, TreeEnsembleClassifier); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, int32_t, TreeEnsembleClassifier); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, TreeEnsembleRegressor); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMLDomain, 1, ZipMap); + +void RegisterOnnxMLOperatorKernels(std::function fn) { + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); +} +} // namespace ml + +static void RegisterCPUKernels(std::function create_fn) { + RegisterOnnxOperatorKernels(create_fn); + ::onnxruntime::ml::RegisterOnnxMLOperatorKernels(create_fn); + ::onnxruntime::contrib::RegisterContribKernels(create_fn); +} + +std::shared_ptr CPUExecutionProvider::GetKernelRegistry() const { + static std::shared_ptr kernel_registry = std::make_shared(RegisterCPUKernels); + return kernel_registry; +} + +std::vector> +CPUExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph, + const std::vector& kernel_registries) const { + std::vector> result = IExecutionProvider::GetCapability(graph, kernel_registries); + + for (auto& rule : fuse_rules_) { + rule(graph, result); + } + return result; +} + +void CPUExecutionProvider::InsertFusedRules(FuseRuleFn rule) { + fuse_rules_.push_back(rule); +} +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/cpu_execution_provider.h b/onnxruntime/core/providers/cpu/cpu_execution_provider.h new file mode 100644 index 0000000000000..e70a4956d2924 --- /dev/null +++ b/onnxruntime/core/providers/cpu/cpu_execution_provider.h @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/framework/allocatormgr.h" +#include "core/framework/execution_provider.h" +#include "core/graph/graph_transformer.h" +#include "core/graph/constants.h" + +namespace onnxruntime { + +// Information needed to construct CPU execution providers. +struct CPUExecutionProviderInfo { + bool create_arena{true}; + + explicit CPUExecutionProviderInfo(bool use_arena) + : create_arena(use_arena) {} + CPUExecutionProviderInfo() = default; +}; + +using FuseRuleFn = std::function>&)>; +// Logical device representation. +class CPUExecutionProvider : public IExecutionProvider { + public: + explicit CPUExecutionProvider(const CPUExecutionProviderInfo& info) { + DeviceAllocatorRegistrationInfo device_info({ONNXRuntimeMemTypeDefault, [](int) { return std::make_unique(); }, std::numeric_limits::max()}); +#ifdef USE_JEMALLOC + ONNXRUNTIME_UNUSED_PARAMETER(info); + //JEMalloc already has memory pool, so just use device allocator. + InsertAllocator( + std::shared_ptr( + std::make_unique(device_info.factory(0)))); +#else + if (info.create_arena) + InsertAllocator(CreateAllocator(device_info)); + else + InsertAllocator( + std::shared_ptr( + std::make_unique(device_info.factory(0)))); +#endif + } + + std::string Type() const override { + return onnxruntime::kCpuExecutionProvider; + } + + virtual std::vector> + GetCapability(const onnxruntime::GraphViewer& graph, + const std::vector& kernel_registries) const override; + + ///requires src.buffer_deleter_ == nullptr + Status CopyTensor(const Tensor& src, Tensor& dst) const override { + ONNXRUNTIME_ENFORCE(strcmp(dst.Location().name, CPU) == 0); + + // Todo: support copy with different devices. + if (strcmp(src.Location().name, CPU) != 0) + ONNXRUNTIME_NOT_IMPLEMENTED("copy from ", src.Location().name, " is not implemented"); + + // no really copy needed if is copy to cpu. + dst.ShallowCopy(src); + + return Status::OK(); + } + + const void* GetExecutionHandle() const noexcept override { + // The CPU interface does not return anything interesting. + return nullptr; + } + + virtual std::shared_ptr GetKernelRegistry() const override; + + void InsertFusedRules(FuseRuleFn rule); + + protected: + std::vector fuse_rules_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/cpu_provider_factory.cc b/onnxruntime/core/providers/cpu/cpu_provider_factory.cc new file mode 100644 index 0000000000000..3f3f44e723d41 --- /dev/null +++ b/onnxruntime/core/providers/cpu/cpu_provider_factory.cc @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/cpu_provider_factory.h" +#include +#include "cpu_execution_provider.h" + +using namespace onnxruntime; + +namespace { +struct CpuProviderFactory { + const ONNXRuntimeProviderFactoryInterface* const cls; + std::atomic_int ref_count; + bool create_arena; + CpuProviderFactory(); +}; + +ONNXStatusPtr ONNXRUNTIME_API_STATUSCALL CreateCpu(void* this_, ONNXRuntimeProviderPtr* out) { + CPUExecutionProviderInfo info; + CpuProviderFactory* this_ptr = (CpuProviderFactory*)this_; + info.create_arena = this_ptr->create_arena; + CPUExecutionProvider* ret = new CPUExecutionProvider(info); + *out = (ONNXRuntimeProviderPtr)ret; + return nullptr; +} + +uint32_t ONNXRUNTIME_API_STATUSCALL ReleaseCpu(void* this_) { + CpuProviderFactory* this_ptr = (CpuProviderFactory*)this_; + if (--this_ptr->ref_count == 0) + delete this_ptr; + return 0; +} + +uint32_t ONNXRUNTIME_API_STATUSCALL AddRefCpu(void* this_) { + CpuProviderFactory* this_ptr = (CpuProviderFactory*)this_; + ++this_ptr->ref_count; + return 0; +} + +constexpr ONNXRuntimeProviderFactoryInterface cpu_cls = { + {AddRefCpu, + ReleaseCpu}, + CreateCpu, +}; + +CpuProviderFactory::CpuProviderFactory() : cls(&cpu_cls), ref_count(1), create_arena(true) {} +} // namespace + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeCreateCpuExecutionProviderFactory, int use_arena, _Out_ ONNXRuntimeProviderFactoryPtr** out) { + CpuProviderFactory* ret = new CpuProviderFactory(); + ret->create_arena = (use_arena != 0); + *out = (ONNXRuntimeProviderFactoryPtr*)ret; + return nullptr; +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeCreateCpuAllocatorInfo, enum ONNXRuntimeAllocatorType type, enum ONNXRuntimeMemType mem_type, _Out_ ONNXRuntimeAllocatorInfo** out) { + return ONNXRuntimeCreateAllocatorInfo(onnxruntime::CPU, type, 0, mem_type, out); +} \ No newline at end of file diff --git a/onnxruntime/core/providers/cpu/generator/constant.cc b/onnxruntime/core/providers/cpu/generator/constant.cc new file mode 100644 index 0000000000000..72ecb86d05f02 --- /dev/null +++ b/onnxruntime/core/providers/cpu/generator/constant.cc @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "constant.h" + +#include "core/common/common.h" +#include "gsl/span" +#include "core/framework/tensorprotoutils.h" + +using namespace ONNX_NAMESPACE; + +namespace onnxruntime { + +ONNX_CPU_OPERATOR_KERNEL( + ConstantLike, + 9, + KernelDefBuilder() + .TypeConstraint("T1", std::vector{ + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}) + .TypeConstraint("T2", std::vector{ + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + ConstantLike); + +template +void GenerateData(Tensor& tensor, float value) { + auto out = gsl::make_span(tensor.MutableData(), tensor.Shape().Size()); + std::for_each(out.begin(), out.end(), [&value](T& v) { v = static_cast(value); }); +} + +static Status GenerateConstantOutput(Tensor& Y, TensorProto::DataType dtype, float value) { + switch (dtype) { + case TensorProto::FLOAT: + GenerateData(Y, value); + break; + case TensorProto::DOUBLE: + GenerateData(Y, value); + break; + case TensorProto::INT8: + GenerateData(Y, value); + break; + case TensorProto::INT16: + GenerateData(Y, value); + break; + case TensorProto::INT32: + GenerateData(Y, value); + break; + case TensorProto::INT64: + GenerateData(Y, value); + break; + case TensorProto::UINT8: + GenerateData(Y, value); + break; + case TensorProto::UINT16: + GenerateData(Y, value); + break; + case TensorProto::UINT32: + GenerateData(Y, value); + break; + case TensorProto::UINT64: + GenerateData(Y, value); + break; + case TensorProto::BOOL: + GenerateData(Y, value); + break; + case TensorProto::FLOAT16: + ONNXRUNTIME_NOT_IMPLEMENTED("FLOAT16 is not supported"); + default: + ONNXRUNTIME_THROW("Unsupported data type of ", dtype); + } + + return Status::OK(); +} + +Status ConstantLike::Compute(OpKernelContext* ctx) const { + const Tensor* X = ctx->Input(0); + Tensor* Y = nullptr; + + ONNX_NAMESPACE::TensorProto::DataType dtype = dtype_; + if (nullptr != X) { + Y = ctx->Output(0, X->Shape()); + if (dtype == ONNX_NAMESPACE::TensorProto::UNDEFINED) { + dtype = utils::GetTensorProtoType(*X); + } + } else { + ONNXRUNTIME_ENFORCE(!shape_.empty(), "Neither Input tensor is not null nor shape attribute exists"); + Y = ctx->Output(0, TensorShape(shape_)); + } + + if (dtype == ONNX_NAMESPACE::TensorProto::UNDEFINED) { + dtype = TensorProto_DataType_FLOAT; + } + + return GenerateConstantOutput(*Y, dtype, value_); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/generator/constant.h b/onnxruntime/core/providers/cpu/generator/constant.h new file mode 100644 index 0000000000000..1558e02fba971 --- /dev/null +++ b/onnxruntime/core/providers/cpu/generator/constant.h @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include "core/framework/op_kernel.h" + +namespace onnxruntime { + +class ConstantLike final : public OpKernel { + public: + ConstantLike(const OpKernelInfo& info) : OpKernel(info) { + int64_t dtype = info.GetAttrOrDefault("dtype", 0 /*default_value*/); + dtype_ = static_cast(dtype); + + shape_ = info.GetAttrsOrDefault("shape"); + ONNXRUNTIME_ENFORCE(info.GetAttr("value", &value_).IsOK()); + } + + Status Compute(OpKernelContext* ctx) const override; + + private: + ONNX_NAMESPACE::TensorProto::DataType dtype_; + std::vector shape_; + float value_; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/generator/random.cc b/onnxruntime/core/providers/cpu/generator/random.cc new file mode 100644 index 0000000000000..5331d8c8f5e7a --- /dev/null +++ b/onnxruntime/core/providers/cpu/generator/random.cc @@ -0,0 +1,366 @@ +/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +// NOTE: License applies to the multinomial implementation only. +// Portions Copyright (c) Microsoft Corporation + +#include "core/providers/cpu/generator/random.h" + +// build\windows\debug\external\eigen3\unsupported\eigen\cxx11\src/Tensor/Tensor.h(76): +// warning C4554: '&': check operator precedence for possible error; use parentheses to clarify precedence +// build\windows\relwithdebinfo\eigen\src\eigen\eigen-eigen-5a0156e40feb\unsupported\eigen\cxx11\src/Tensor/TensorChipping.h(52) +// warning C4100: 'dim': unreferenced formal parameter +#ifdef _WIN32 +#pragma warning(disable : 4554 4100) +#endif + +#include +#include +#include +#include "core/util/math_cpuonly.h" +#include "core/util/eigen_common_wrapper.h" +#include "gsl/span" +using namespace ONNX_NAMESPACE; +using namespace ::onnxruntime::common; +namespace onnxruntime { + +ONNX_CPU_OPERATOR_KERNEL( + RandomNormal, + 1, + KernelDefBuilder().TypeConstraint("T", std::vector{ + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + RandomNormal); + +ONNX_CPU_OPERATOR_KERNEL( + RandomUniform, + 1, + KernelDefBuilder().TypeConstraint("T", std::vector{ + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + RandomUniform); + +ONNX_CPU_OPERATOR_KERNEL( + RandomNormalLike, + 1, + KernelDefBuilder().TypeConstraint("T1", DataTypeImpl::AllTensorTypes()).TypeConstraint("T2", std::vector{DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}), + RandomNormalLike); + +ONNX_CPU_OPERATOR_KERNEL( + RandomUniformLike, + 1, + KernelDefBuilder().TypeConstraint("T1", DataTypeImpl::AllTensorTypes()).TypeConstraint("T2", std::vector{DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}), + RandomUniformLike); + +// https://github.com/onnx/onnx/blob/master/docs/Operators.md#multinomial +ONNX_CPU_OPERATOR_KERNEL( + Multinomial, + 7, + KernelDefBuilder().TypeConstraint("T1", DataTypeImpl::GetTensorType()).TypeConstraint("T2", std::vector{DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}), + Multinomial); + +template +void GenerateData(std::default_random_engine generator, TDistribution distribution, Tensor& tensor); + +static Status RandomNormalCompute(float mean, float scale, std::default_random_engine generator, TensorProto::DataType dtype, Tensor& Y); +static Status RandomUniformCompute(float high, float low, std::default_random_engine generator, TensorProto::DataType dtype, Tensor& Y); + +// Leaving in case we need to change to this approach +//static Status CreateOutputTensorFromTensorValues(OpKernelContext* ctx, const Tensor& X,Tensor** Y); +static Status CreateOutputTensorFromTensorShape(OpKernelContext* ctx, const Tensor& X, Tensor** Y); +static TensorProto::DataType InferDataType(const Tensor& tensor); + +Status RandomNormal::Compute(OpKernelContext* ctx) const { + Tensor& Y = *ctx->Output(0, shape_); + + auto status = RandomNormalCompute(mean_, scale_, generator_, dtype_, Y); + + return status; +} + +Status RandomUniform::Compute(OpKernelContext* ctx) const { + Tensor& Y = *ctx->Output(0, shape_); + + auto status = RandomUniformCompute(low_, high_, generator_, dtype_, Y); + + return status; +} + +Status RandomNormalLike::Compute(OpKernelContext* ctx) const { + const Tensor* tensor_pointer = ctx->Input(0); + if (tensor_pointer == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + const Tensor& X = *tensor_pointer; + Tensor* Y = nullptr; + + auto status = CreateOutputTensorFromTensorShape(ctx, X, &Y); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + auto dtype = dtype_ != TensorProto_DataType_UNDEFINED ? dtype_ : InferDataType(X); + + if (dtype == TensorProto_DataType_UNDEFINED) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, + "Could not infer data type from input tensor with data type ", + X.DataType()); + + status = RandomNormalCompute(mean_, scale_, generator_, dtype, *Y); + + return status; +} + +Status RandomUniformLike::Compute(OpKernelContext* ctx) const { + const Tensor* tensor_pointer = ctx->Input(0); + if (tensor_pointer == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + const Tensor& X = *tensor_pointer; + Tensor* Y = nullptr; + + auto status = CreateOutputTensorFromTensorShape(ctx, X, &Y); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + auto dtype = dtype_ != TensorProto_DataType_UNDEFINED ? dtype_ : InferDataType(X); + + if (dtype == TensorProto_DataType_UNDEFINED) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, + "Could not infer data type from input tensor with data type ", + X.DataType()); + status = RandomUniformCompute(low_, high_, generator_, dtype, *Y); + + return status; +} + +// Rank-2 tensor (matrix) of scalar type T. +template +using Matrix = Eigen::TensorMap>; + +template +using ConstMatrix = Eigen::TensorMap>; + +template +using EigenVector = Eigen::TensorMap>; + +template +static Status MultinomialCompute(OpKernelContext* ctx, + const Tensor& X, + const int64_t batch_size, + const int64_t num_classes, + const int64_t num_samples, + std::default_random_engine generator, + Tensor& Y) { + // implementation copied from Tensorflow with some changes such as using the std::uniform_real_distribution + // instead of the Philox RNG. + Eigen::array X_dims = {{batch_size, num_classes}}; + ConstMatrix logits = ConstMatrix(X.template Data(), X_dims); + + Eigen::array Y_dims = {{batch_size, num_samples}}; + Matrix output = Matrix(Y.template MutableData(), Y_dims); + + // TODO (perf optimization) - the idea behind making this a lambda is so that we can parallelize across batches. + // When we do that this lamdba will act as one task given to a thread + auto DoWork = [ctx, num_samples, num_classes, &generator, &logits, &output](int64_t start_row, + int64_t limit_row) { + std::default_random_engine generator_copy = generator; + // BEGIN create temporary tensor + AllocatorPtr alloc; + ctx->GetTempSpaceAllocator(&alloc); + auto cdf_data = static_cast(alloc->Alloc(sizeof(double) * num_classes)); + BufferUniquePtr cdf_buffer(cdf_data, BufferDeleter(alloc)); + Eigen::array cdf_dims = {{num_classes}}; + auto cdf = EigenVector(cdf_data, cdf_dims); + // END create temporary tensor + + std::uniform_real_distribution dist(0.0, 1.0); // TODO: should this be initialized per batch? + for (int64_t b = start_row; b < limit_row; ++b) { + const float* logits_row = &(logits(b, 0)); + // Takes an along-class maximum (for numerical stability). + float maxx = std::numeric_limits::lowest(); + for (int64_t j = 0; j < num_classes; ++j) { + if (Eigen::numext::isfinite(logits_row[j])) { + maxx = std::max(maxx, logits_row[j]); + } + } + const double max_logit = static_cast(maxx); + + // Precompute cumulative probability distribution across classes. + // Note: This isn't normalized. + cdf = (logits.chip<0>(b).cast() - max_logit).exp(); + double running_total = 0; + for (int64_t j = 0; j < num_classes; ++j) { + if (Eigen::numext::isfinite(logits_row[j])) { + running_total += cdf(j); + } + cdf(j) = running_total; + } + // Generate each sample. + const double* cdf_begin = cdf.data(); + const double* cdf_end = cdf.data() + num_classes; + for (int64_t j = 0; j < num_samples; ++j) { + const double to_find = dist(generator_copy) * running_total; + auto found_iter = std::upper_bound(cdf_begin, cdf_end, to_find); + output(b, j) = static_cast(std::distance(cdf_begin, found_iter)); + } + } + }; + DoWork(0, batch_size); + return Status::OK(); +} + +Status Multinomial::Compute(OpKernelContext* ctx) const { + const Tensor* tensor_pointer = ctx->Input(0); + if (tensor_pointer == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + const Tensor& X = *tensor_pointer; + auto& X_dims = X.Shape().GetDims(); + + if (X_dims.empty()) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Empty dimensions for input tensor"); + } + + const auto batch_size = X_dims[0]; + const auto num_classes = X_dims[1]; + + // validate inputs + if (batch_size < 1) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "batch_size is < 1"); + } + if (num_classes < 1) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "num_classes is < 1"); + } + if (num_samples_ < 1) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "num_samples is < 1"); + } + + Tensor* Y = ctx->Output(0, TensorShape({batch_size, num_samples_})); + + Status status = Status::OK(); + switch (output_dtype_) { + case TensorProto::INT32: { + status = MultinomialCompute(ctx, X, batch_size, num_classes, num_samples_, generator_, *Y); + break; + } + case TensorProto::INT64: { + status = MultinomialCompute(ctx, X, batch_size, num_classes, num_samples_, generator_, *Y); + break; + } + default: + status = ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Invalid data type of ", output_dtype_); + } + + return status; +} + +/* +alternative interpretation of the spec is that the input tensor contains the dimensions as ints. +Keeping this temporarily in case we go back to that. + +// read shape information from input tensor and create output tensor with it +static Status CreateOutputTensorFromTensorValues(OpKernelContext* ctx, const Tensor& X, Tensor** Y) { + const TensorShape& shape = X.Shape(); + auto size = shape.Size(); + auto num_dims = shape.NumDimensions(); + + if (num_dims != 1) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Expected 1 dimension tensor with shape information. Dimensions=", num_dims); + } + + std::vector dims; + dims.reserve(shape.Size()); + + auto data = gsl::make_span(tensor.template Data(), shape.Size()); + dims.insert(dims.cbegin(), data.cbegin(), data.cend()); + + *Y = ctx->Output(0, TensorShape(dims)); + + return Status::OK(); +} +*/ + +// create output tensor using shape of input tensor +static Status CreateOutputTensorFromTensorShape(OpKernelContext* ctx, const Tensor& X, Tensor** Y) { + const TensorShape& shape = X.Shape(); + + *Y = ctx->Output(0, shape); + + return Status::OK(); +} + +static TensorProto::DataType InferDataType(const Tensor& tensor) { + auto tensor_type = tensor.DataType(); + TensorProto::DataType dtype = TensorProto_DataType_UNDEFINED; + + if (tensor_type == DataTypeImpl::GetType()) + dtype = TensorProto_DataType_FLOAT; + else if (tensor_type == DataTypeImpl::GetType()) + dtype = TensorProto_DataType_DOUBLE; + else { + // unsupported. return UNDEFINED + } + + return dtype; +} + +static Status RandomNormalCompute(float mean, float scale, + std::default_random_engine generator, + TensorProto::DataType dtype, Tensor& Y) { + switch (dtype) { + case TensorProto::FLOAT: { + GenerateData>( + generator, std::normal_distribution{mean, scale}, Y); + break; + } + case TensorProto::FLOAT16: { + ONNXRUNTIME_NOT_IMPLEMENTED("FLOAT16 is not supported"); + } + case TensorProto::DOUBLE: { + GenerateData>( + generator, std::normal_distribution{mean, scale}, Y); + break; + } + default: + ONNXRUNTIME_THROW("Invalid data type of ", dtype); + } + + return Status::OK(); +} + +static Status RandomUniformCompute(float low, float high, + std::default_random_engine generator, + TensorProto::DataType dtype, + Tensor& Y) { + switch (dtype) { + case TensorProto::FLOAT: { + GenerateData>( + generator, std::uniform_real_distribution{low, high}, Y); + break; + } + case TensorProto::FLOAT16: { + ONNXRUNTIME_NOT_IMPLEMENTED("FLOAT16 is not supported"); + } + case TensorProto::DOUBLE: { + GenerateData>( + generator, std::uniform_real_distribution{low, high}, Y); + break; + } + default: + ONNXRUNTIME_THROW("Invalid data type of ", dtype); + } + + return Status::OK(); +} + +template +void GenerateData(std::default_random_engine generator, TDistribution distribution, Tensor& tensor) { + auto out = gsl::make_span(tensor.template MutableData(), tensor.Shape().Size()); + + std::for_each(out.begin(), out.end(), [&generator, &distribution](T& value) { value = distribution(generator); }); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/generator/random.h b/onnxruntime/core/providers/cpu/generator/random.h new file mode 100644 index 0000000000000..fb8bdcef7b092 --- /dev/null +++ b/onnxruntime/core/providers/cpu/generator/random.h @@ -0,0 +1,174 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include "gsl/gsl_util" + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { + +class RandomNormal final : public OpKernel { + public: + RandomNormal(const OpKernelInfo& info) : OpKernel(info) { + ONNXRUNTIME_ENFORCE(info.GetAttr("mean", &mean_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttr("scale", &scale_).IsOK()); + + // read optional seed attribute and generate if not provided + float seed = 0.f; + if (!info.GetAttr("seed", &seed).IsOK()) { + seed = gsl::narrow_cast(std::chrono::high_resolution_clock::now().time_since_epoch().count()); + } + + generator_ = std::default_random_engine{gsl::narrow_cast(seed)}; + + int64_t dtype; + ONNXRUNTIME_ENFORCE(info.GetAttr("dtype", &dtype).IsOK()); + dtype_ = static_cast(dtype); + ONNXRUNTIME_ENFORCE(ONNX_NAMESPACE::TensorProto::DataType_IsValid(dtype_) && dtype_ != ONNX_NAMESPACE::TensorProto::UNDEFINED, + "Invalid dtype of ", dtype_); + + std::vector shape; + ONNXRUNTIME_ENFORCE(info.GetAttrs("shape", shape).IsOK()); + shape_ = TensorShape(shape); + } + + Status Compute(OpKernelContext* ctx) const override; + + private: + float mean_; + float scale_; + std::default_random_engine generator_; + ONNX_NAMESPACE::TensorProto::DataType dtype_; + TensorShape shape_; +}; + +class RandomNormalLike final : public OpKernel { + public: + RandomNormalLike(const OpKernelInfo& info) : OpKernel(info) { + ONNXRUNTIME_ENFORCE(info.GetAttr("mean", &mean_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttr("scale", &scale_).IsOK()); + + // read optional seed attribute and generate if not provided + float seed = 0.f; + if (!info.GetAttr("seed", &seed).IsOK()) { + seed = gsl::narrow_cast(std::chrono::high_resolution_clock::now().time_since_epoch().count()); + } + + generator_ = std::default_random_engine{gsl::narrow_cast(seed)}; + + int64_t dtype; + if (info.GetAttr("dtype", &dtype).IsOK()) { + dtype_ = static_cast(dtype); + ONNXRUNTIME_ENFORCE(ONNX_NAMESPACE::TensorProto::DataType_IsValid(dtype_) && dtype_ != ONNX_NAMESPACE::TensorProto::UNDEFINED, + "Invalid dtype of ", dtype_); + } + } + + Status Compute(OpKernelContext* ctx) const override; + + private: + float mean_; + float scale_; + std::default_random_engine generator_; + ONNX_NAMESPACE::TensorProto::DataType dtype_ = ONNX_NAMESPACE::TensorProto::DataType::TensorProto_DataType_UNDEFINED; //optional and may be inferred +}; + +class RandomUniform final : public OpKernel { + public: + RandomUniform(const OpKernelInfo& info) : OpKernel(info) { + ONNXRUNTIME_ENFORCE(info.GetAttr("high", &high_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttr("low", &low_).IsOK()); + + // read optional seed attribute and generate if not provided + float seed = 0.f; + if (!info.GetAttr("seed", &seed).IsOK()) { + seed = gsl::narrow_cast(std::chrono::high_resolution_clock::now().time_since_epoch().count()); + } + + generator_ = std::default_random_engine{gsl::narrow_cast(seed)}; + + int64_t dtype; + ONNXRUNTIME_ENFORCE(info.GetAttr("dtype", &dtype).IsOK()); + dtype_ = static_cast(dtype); + ONNXRUNTIME_ENFORCE(ONNX_NAMESPACE::TensorProto::DataType_IsValid(dtype_) && dtype_ != ONNX_NAMESPACE::TensorProto::UNDEFINED, + "Invalid dtype of ", dtype_); + + std::vector shape; + ONNXRUNTIME_ENFORCE(info.GetAttrs("shape", shape).IsOK()); + shape_ = TensorShape(shape); + } + + Status Compute(OpKernelContext* ctx) const override; + + private: + float high_; + float low_; + std::default_random_engine generator_; + ONNX_NAMESPACE::TensorProto::DataType dtype_; + TensorShape shape_; +}; + +class RandomUniformLike final : public OpKernel { + public: + RandomUniformLike(const OpKernelInfo& info) : OpKernel(info) { + ONNXRUNTIME_ENFORCE(info.GetAttr("high", &high_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttr("low", &low_).IsOK()); + // read optional seed attribute and generate if not provided + float seed = 0.f; + if (!info.GetAttr("seed", &seed).IsOK()) { + seed = gsl::narrow_cast(std::chrono::high_resolution_clock::now().time_since_epoch().count()); + } + + generator_ = std::default_random_engine{gsl::narrow_cast(seed)}; + + int64_t dtype; + if (info.GetAttr("dtype", &dtype).IsOK()) { + dtype_ = static_cast(dtype); + ONNXRUNTIME_ENFORCE(ONNX_NAMESPACE::TensorProto::DataType_IsValid(dtype_) && dtype_ != ONNX_NAMESPACE::TensorProto::UNDEFINED, + "Invalid dtype of ", dtype_); + } + } + + Status Compute(OpKernelContext* ctx) const override; + + private: + float high_; + float low_; + std::default_random_engine generator_; + ONNX_NAMESPACE::TensorProto::DataType dtype_ = ONNX_NAMESPACE::TensorProto::DataType::TensorProto_DataType_UNDEFINED; //optional and may be inferred +}; + +class Multinomial final : public OpKernel { + public: + Multinomial(const OpKernelInfo& info) : OpKernel(info) { + ONNXRUNTIME_ENFORCE(info.GetAttr("sample_size", &num_samples_).IsOK()); + + float seed = 0.f; + if (!info.GetAttr("seed", &seed).IsOK()) { + seed = gsl::narrow_cast(std::chrono::high_resolution_clock::now().time_since_epoch().count()); + } + + generator_ = std::default_random_engine{gsl::narrow_cast(seed)}; + + int64_t output_dtype_tmp; + if (!info.GetAttr("dtype", &output_dtype_tmp).IsOK()) { + output_dtype_ = ONNX_NAMESPACE::TensorProto_DataType_INT32; // default is INT32 as per spec + } else { + output_dtype_ = static_cast(output_dtype_tmp); + } + ONNXRUNTIME_ENFORCE(ONNX_NAMESPACE::TensorProto::DataType_IsValid(output_dtype_) && output_dtype_ != ONNX_NAMESPACE::TensorProto::UNDEFINED, + "Invalid dtype of ", output_dtype_); + } + + Status Compute(OpKernelContext* ctx) const override; + + private: + int64_t num_samples_; + std::default_random_engine generator_; + ONNX_NAMESPACE::TensorProto::DataType output_dtype_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/math/clip.cc b/onnxruntime/core/providers/cpu/math/clip.cc new file mode 100644 index 0000000000000..160d587df7238 --- /dev/null +++ b/onnxruntime/core/providers/cpu/math/clip.cc @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/math/clip.h" + +namespace onnxruntime { + +ONNX_CPU_OPERATOR_KERNEL( + Clip, + 6, + KernelDefBuilder().MayInplace(0, 0).TypeConstraint("T", DataTypeImpl::GetTensorType()), + Clip); + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/math/clip.h b/onnxruntime/core/providers/cpu/math/clip.h new file mode 100644 index 0000000000000..ea1feb3dc3e4d --- /dev/null +++ b/onnxruntime/core/providers/cpu/math/clip.h @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/util/math_cpuonly.h" + +namespace onnxruntime { + +template +class Clip final : public OpKernel { + public: + Clip(const OpKernelInfo& info) : OpKernel(info) { + ONNXRUNTIME_ENFORCE(info.GetAttr("max", &max_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttr("min", &min_).IsOK()); + } + + Status Compute(OpKernelContext* ctx) const override { + const Tensor* X = ctx->Input(0); + Tensor* Y = ctx->Output(0, X->Shape()); + EigenVectorMap(Y->template MutableData(), Y->Shape().Size()) = + ConstEigenVectorMap(X->template Data(), X->Shape().Size()) + .cwiseMax(min_) + .cwiseMin(max_); + return Status::OK(); + } + + private: + T max_; + T min_; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/math/element_wise_ops.cc b/onnxruntime/core/providers/cpu/math/element_wise_ops.cc new file mode 100644 index 0000000000000..ceaaf8f97b0cb --- /dev/null +++ b/onnxruntime/core/providers/cpu/math/element_wise_ops.cc @@ -0,0 +1,1173 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/math/element_wise_ops.h" + +namespace onnxruntime { + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Add, + 7, + float, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Add); + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Add, + 7, + int32_t, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Add); + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Add, + 7, + int64_t, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Add); + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Sub, + 7, + float, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Sub); + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Sub, + 7, + int32_t, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Sub); + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Sub, + 7, + int64_t, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Sub); + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Mul, + 7, + float, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Mul); + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Mul, + 7, + double, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Mul); + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Mul, + 7, + int32_t, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Mul); + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Mul, + 7, + int64_t, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Mul); + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Div, + 7, + float, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Div); + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Div, + 7, + int32_t, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Div); + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Div, + 7, + int64_t, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Div); + +#define REG_ABS_KERNEL(TYPE) \ + ONNX_CPU_OPERATOR_TYPED_KERNEL( \ + Abs, \ + 6, \ + TYPE, \ + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + Abs); + +REG_ABS_KERNEL(float) +REG_ABS_KERNEL(double) +REG_ABS_KERNEL(int8_t) +REG_ABS_KERNEL(int16_t) +REG_ABS_KERNEL(int32_t) +REG_ABS_KERNEL(int64_t) +REG_ABS_KERNEL(uint8_t) +REG_ABS_KERNEL(uint16_t) +REG_ABS_KERNEL(uint32_t) +REG_ABS_KERNEL(uint64_t) + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Neg, + 6, + float, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Neg); + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Neg, + 6, + int8_t, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Neg); + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Neg, + 6, + int32_t, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Neg); + +ONNX_CPU_OPERATOR_KERNEL( + Floor, + 6, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Floor); + +ONNX_CPU_OPERATOR_KERNEL( + Ceil, + 6, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Ceil); + +ONNX_CPU_OPERATOR_KERNEL( + Reciprocal, + 6, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Reciprocal); + +ONNX_CPU_OPERATOR_KERNEL( + Sqrt, + 6, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Sqrt); + +ONNX_CPU_OPERATOR_KERNEL( + Pow, + 7, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Pow); + +ONNX_CPU_OPERATOR_KERNEL( + Exp, + 6, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Exp); + +ONNX_CPU_OPERATOR_KERNEL( + Log, + 6, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Log); + +ONNX_CPU_OPERATOR_VERSIONED_KERNEL( + Sum, + 6, 7, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Sum_6); + +ONNX_CPU_OPERATOR_KERNEL( + Sum, + 8, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Sum_8); + +ONNX_CPU_OPERATOR_VERSIONED_KERNEL( + Min, + 6, 7, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Min_6); + +ONNX_CPU_OPERATOR_KERNEL( + Min, + 8, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Min_8); + +ONNX_CPU_OPERATOR_VERSIONED_KERNEL( + Max, + 6, 7, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Max_6); + +ONNX_CPU_OPERATOR_KERNEL( + Max, + 8, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Max_8); + +ONNX_CPU_OPERATOR_KERNEL( + Not, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Not); + +ONNX_CPU_OPERATOR_KERNEL( + And, + 7, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + And); + +ONNX_CPU_OPERATOR_KERNEL( + Or, + 7, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Or); + +ONNX_CPU_OPERATOR_KERNEL( + Xor, + 7, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Xor); + +ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL( + Less, + 7, 9, + float, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Less); + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Less, + 9, + int32_t, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Less); + +ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL( + Greater, + 7, 9, + float, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Greater); + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Greater, + 9, + int32_t, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Greater); + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Equal, + 7, + bool, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Equal); + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Equal, + 7, + int32_t, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Equal); + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Equal, + 7, + int64_t, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Equal); + +ONNX_CPU_OPERATOR_VERSIONED_KERNEL( + Mean, + 6, 7, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Mean_6); + +ONNX_CPU_OPERATOR_KERNEL( + Mean, + 8, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Mean_8); + +ONNX_CPU_OPERATOR_KERNEL( + Affine, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Affine); + +ONNX_CPU_OPERATOR_KERNEL( + Scale, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Scale); + +template +auto MakeEigenArrayMap(Tensor& t) { return EigenVectorArrayMap(t.template MutableData(), t.Shape().Size()); } +template +auto MakeEigenArrayMap(const Tensor& t) { return ConstEigenVectorArrayMap(t.template Data(), t.Shape().Size()); } + +struct BroadcastIterator { + size_t AdvanceBy(size_t delta) { + size_t index = index_; + + index_ += deltas_[0] * delta; + counters_[0] += delta; + if (counters_[0] == counts_[0]) { + counters_[0] = 0; + for (size_t counterIndex = 1; counterIndex < counters_.size(); counterIndex++) { + index_ += deltas_[counterIndex]; + if (++counters_[counterIndex] != counts_[counterIndex]) + break; + counters_[counterIndex] = 0; + } + } + return index; + } + + void Init(int64_t axis, int64_t largest) { + ONNXRUNTIME_ENFORCE(axis == 1 || axis == largest, "Attempting to broadcast an axis by a dimension other than 1. ", axis, " by ", largest); + + deltas_.push_back(axis > 1); + counts_.push_back(largest); + count_ *= axis; + } + + void Append(int64_t axis, int64_t largest) { + ONNXRUNTIME_ENFORCE(axis == 1 || axis == largest, "Attempting to broadcast an axis by a dimension other than 1. ", axis, " by ", largest); + + // If we're greater than 1, it doesn't matter what the other tensor does + if (axis > 1) { + if (deltas_.back() <= 0) // Were we broadcasting + StopBroadcasting(); + } else { // We must be 1, at this point + if (deltas_.back() > 0) + StartBroadcasting(); + } + + counts_.back() *= largest; // Just increase the last count + count_ *= axis; + } + + void StopBroadcasting() { + deltas_.push_back(count_); + counts_.push_back(1); + } + + void StartBroadcasting() { + deltas_.push_back(-count_); + counts_.push_back(1); + } + + std::vector counters_; + std::vector deltas_; + std::vector counts_; + size_t count_{1}; // Running total count of entries in tensor, used while building up the entries + + private: + size_t index_{}; +}; + +struct Broadcaster { + Broadcaster(const std::vector& shape1, const std::vector& shape2) { + size_t dimension_count_max = std::max(shape1.size(), shape2.size()); + size_t dimension_count_min = std::min(shape1.size(), shape2.size()); + output_shape_.resize(dimension_count_max); + + auto iter1 = shape1.end(); + auto iter2 = shape2.end(); + auto output_shape = output_shape_.end(); + + // Scalars are a special case, as it's always a broadcast + size_t index = 0; + if (dimension_count_min == 0) { + if (shape1.size() == 0) // Shape1 is a scalar + { + if (shape2.size() == 0) // Two scalars? + { + iterator1_.Init(1, 1); + iterator2_.Init(1, 1); + } else { + auto axis = *--iter2; + iterator1_.Init(1, axis); + iterator2_.Init(axis, axis); + *--output_shape = axis; + } + } else { // Shape2 is a scalar + auto axis = *--iter1; + iterator1_.Init(axis, axis); + iterator2_.Init(1, axis); + *--output_shape = axis; + } + index++; // Manually increment since we processed one axis + } + + for (; index < dimension_count_min; index++) { + auto axis1 = *--iter1; + auto axis2 = *--iter2; + + auto largest = std::max(axis1, axis2); + *--output_shape = largest; + + if (largest == 1 && index + 1 < dimension_count_min) // Nothing to do in this case + continue; + + iterator1_.Init(axis1, largest); + iterator2_.Init(axis2, largest); + index++; // Manually increment since we processed one axis + break; + } + + for (; index < dimension_count_min; index++) { + auto axis1 = *--iter1; + auto axis2 = *--iter2; + + auto largest = std::max(axis1, axis2); + *--output_shape = largest; + + if (largest == 1) // Nothing to do in this case + continue; + + iterator1_.Append(axis1, largest); + iterator2_.Append(axis2, largest); + } + + // If one shape is bigger than another we need to broadcast the smaller onto the bigger from this point on + for (; index < dimension_count_max; index++) { + if (dimension_count_max == shape2.size()) { + auto axis = *--iter2; + iterator1_.Append(1, axis); + iterator2_.Append(axis, axis); + *--output_shape = axis; + } else { + auto axis = *--iter1; + iterator1_.Append(axis, axis); + iterator2_.Append(1, axis); + *--output_shape = axis; + } + } + + // Allocate the counters + iterator1_.counters_.resize(iterator1_.counts_.size(), 0); + iterator2_.counters_.resize(iterator2_.counts_.size(), 0); + } + + size_t GetSpanSize() const { return std::min(iterator1_.counts_.front(), iterator2_.counts_.front()); } + + BroadcastIterator iterator1_, iterator2_; + std::vector output_shape_; +}; + +template +struct TBroadcaster { + TBroadcaster(const Tensor& input0, const Tensor& input1) + : input_tensor0_(input0), + input_tensor1_(input1) { + } + + TensorShape GetOutputShape() const { return TensorShape(broadcaster_.output_shape_); } + size_t GetSpanSize() const { return span_size_; } + + bool IsInput0Scalar() const { return broadcaster_.iterator1_.deltas_.front() == 0; } + bool IsInput1Scalar() const { return broadcaster_.iterator2_.deltas_.front() == 0; } + + T NextScalar0() { return *Next0(); } + T NextScalar1() { return *Next1(); } + + gsl::span NextSpan0() { return gsl::span(Next0(), span_size_); } + gsl::span NextSpan1() { return gsl::span(Next1(), span_size_); } + + ConstEigenVectorMap NextEigen0() { return ConstEigenVectorMap(Next0(), span_size_); } + ConstEigenVectorMap NextEigen1() { return ConstEigenVectorMap(Next1(), span_size_); } + + private: + const T* Next0() { return input0_ + broadcaster_.iterator1_.AdvanceBy(span_size_); } + const T* Next1() { return input1_ + broadcaster_.iterator2_.AdvanceBy(span_size_); } + + const Tensor& input_tensor0_; + const Tensor& input_tensor1_; + Broadcaster broadcaster_{input_tensor0_.Shape().GetDims(), input_tensor1_.Shape().GetDims()}; + size_t span_size_{broadcaster_.GetSpanSize()}; + + const T* input0_{input_tensor0_.template Data()}; + const T* input1_{input_tensor1_.template Data()}; +}; + +template +struct TBroadcastOutput { + TBroadcastOutput(size_t span_size, Tensor& tensor) + : span_size_(span_size) { + output_ = tensor.template MutableData(); + output_end_ = output_ + tensor.Shape().Size(); + } + + operator bool() const { + return output_ != output_end_; + } + + EigenVectorMap NextEigenOutput() { + return EigenVectorMap(NextOutput(), span_size_); + } + + gsl::span NextSpanOutput() { + return gsl::span(NextOutput(), span_size_); + } + + private: + T* NextOutput() { + T* output = output_; + output_ += span_size_; + return output; + } + + T* output_; + const T* output_end_; + size_t span_size_; +}; + +template +struct TensorAllocator { + TensorAllocator(OpKernelContext& context) { + ONNXRUNTIME_ENFORCE(context.GetTempSpaceAllocator(&allocator_).IsOK()); + } + + std::unique_ptr Allocate(const TensorShape& shape) { + return std::make_unique(DataTypeImpl::GetType(), + shape, + allocator_->Alloc(sizeof(T) * shape.Size()), + allocator_->Info(), + allocator_); + } + + private: + AllocatorPtr allocator_; +}; + +// Broadcast loop for when using eigen, functions are in this form: +// Input0Scalar: [](EigenVectorMap output, T input0, ConstEigenVectorMap input1) +// Input1Scalar: [](EigenVectorMap output, ConstEigenVectorMap input0, T input1) +// General : [](EigenVectorMap output, ConstEigenVectorMap input0, ConstEigenVectorMap input1) +template +void BroadcastLoop(TBroadcaster& bc, Output& output, Input0Scalar input0scalar, Input1Scalar input1scalar, General general) { + if (bc.IsInput0Scalar()) { + while (output) + input0scalar(output.NextEigenOutput(), bc.NextScalar0(), bc.NextEigen1()); + } else if (bc.IsInput1Scalar()) { + while (output) + input1scalar(output.NextEigenOutput(), bc.NextEigen0(), bc.NextScalar1()); + } else { + while (output) + general(output.NextEigenOutput(), bc.NextEigen0(), bc.NextEigen1()); + } +} + +template +Status BroadcastTwo(OpKernelContext& context, Input0Scalar input0scalar, Input1Scalar input1scalar, General general) { + TBroadcaster bc(*context.Input(0), *context.Input(1)); + TBroadcastOutput output(bc.GetSpanSize(), *context.Output(0, bc.GetOutputShape())); + BroadcastLoop(bc, output, input0scalar, input1scalar, general); + + return Status::OK(); +} + +template +Status BroadcastVariadic(const Node& node, OpKernelContext& context, Input0Scalar input0scalar, Input1Scalar input1scalar, General general) { + auto input_count = node.InputArgCount().front(); + ONNXRUNTIME_ENFORCE(input_count >= 1, "Must have 1 or more inputs"); + + // One item is trivial, just copy across and exit + if (input_count == 1) { + EigenMap(*context.Output(0, context.Input(0)->Shape())) = EigenMap(*context.Input(0)); + return Status::OK(); + } + + std::unique_ptr tempInput; + std::unique_ptr tempOutput; + + TensorAllocator tensorAllocator(context); + + // For more than 2 tensors, we sum the first two into a temporary tensor, then sum the next with the temporary tensor + for (int i = 0; i < input_count - 1; i++) { + auto& tensor0 = tempInput ? *tempInput : *context.Input(0); + auto& tensor1 = *context.Input(i + 1); + + TBroadcaster bc(tensor0, tensor1); + + // Create a temporary output for all but the last iteration, which goes to the real output + Tensor* p_output{}; + if (i == input_count - 2) + p_output = context.Output(0, bc.GetOutputShape()); + else { + tempOutput = tensorAllocator.Allocate(bc.GetOutputShape()); + p_output = tempOutput.get(); + } + + TBroadcastOutput output(bc.GetSpanSize(), *p_output); + + BroadcastLoop(bc, output, input0scalar, input1scalar, general); + + tempInput = std::move(tempOutput); + } + return Status::OK(); +} + +template +Status Add::Compute(OpKernelContext* context) const { + return BroadcastTwo( + *context, + [](EigenVectorMap output, T input0, ConstEigenVectorMap input1) { output = input0 + input1.array(); }, + [](EigenVectorMap output, ConstEigenVectorMap input0, T input1) { output = input0.array() + input1; }, + [](EigenVectorMap output, ConstEigenVectorMap input0, ConstEigenVectorMap input1) { output = input0 + input1; }); +} + +template +Status Sub::Compute(OpKernelContext* context) const { + return BroadcastTwo( + *context, + [](EigenVectorMap output, T input0, ConstEigenVectorMap input1) { output = input0 - input1.array(); }, + [](EigenVectorMap output, ConstEigenVectorMap input0, T input1) { output = input0.array() - input1; }, + [](EigenVectorMap output, ConstEigenVectorMap input0, ConstEigenVectorMap input1) { output = input0 - input1; }); +} + +template +Status Mul::Compute(OpKernelContext* context) const { + return BroadcastTwo( + *context, + [](EigenVectorMap output, T input0, ConstEigenVectorMap input1) { output = input0 * input1.array(); }, + [](EigenVectorMap output, ConstEigenVectorMap input0, T input1) { output = input0.array() * input1; }, + [](EigenVectorMap output, ConstEigenVectorMap input0, ConstEigenVectorMap input1) { output = input0.cwiseProduct(input1); }); +} + +template +Status Div::Compute(OpKernelContext* context) const { + return BroadcastTwo( + *context, + [](EigenVectorMap output, T input0, ConstEigenVectorMap input1) { output = input0 / input1.array(); }, + [](EigenVectorMap output, ConstEigenVectorMap input0, T input1) { output = input0.array() / input1; }, + [](EigenVectorMap output, ConstEigenVectorMap input0, ConstEigenVectorMap input1) { output = input0.cwiseQuotient(input1); }); +} + +template <> +Status Floor::Compute(OpKernelContext* ctx) const { + auto& X = *ctx->Input(0); + auto& Y = *ctx->Output(0, X.Shape()); + + EigenMap(Y) = EigenMap(X).array().floor(); + + return Status::OK(); +} + +template <> +Status Ceil::Compute(OpKernelContext* ctx) const { + auto& X = *ctx->Input(0); + auto& Y = *ctx->Output(0, X.Shape()); + + EigenMap(Y) = EigenMap(X).array().ceil(); + + return Status::OK(); +} + +template <> +Status Reciprocal::Compute(OpKernelContext* ctx) const { + auto& X = *ctx->Input(0); + auto& Y = *ctx->Output(0, X.Shape()); + + EigenMap(Y) = EigenMap(X).cwiseInverse(); + + return Status::OK(); +} + +template <> +Status Sqrt::Compute(OpKernelContext* ctx) const { + auto& X = *ctx->Input(0); + auto& Y = *ctx->Output(0, X.Shape()); + + EigenMap(Y) = EigenMap(X).cwiseSqrt(); + + return Status::OK(); +} + +template <> +Status Pow::Compute(OpKernelContext* context) const { + return BroadcastTwo( + *context, + [](EigenVectorMap output, float input0, ConstEigenVectorMap input1) { output = Eigen::pow(input0, input1.array()); }, + [](EigenVectorMap output, ConstEigenVectorMap input0, float input1) { output = Eigen::pow(input0.array(), input1); }, + [](EigenVectorMap output, ConstEigenVectorMap input0, ConstEigenVectorMap input1) { output = Eigen::pow(input0.array(), input1.array()); }); +} + +template <> +Status Exp::Compute(OpKernelContext* ctx) const { + auto& X = *ctx->Input(0); + auto& Y = *ctx->Output(0, X.Shape()); + + EigenMap(Y) = EigenMap(X).array().exp(); + + return Status::OK(); +} + +template <> +Status Log::Compute(OpKernelContext* ctx) const { + auto& X = *ctx->Input(0); + auto& Y = *ctx->Output(0, X.Shape()); + + EigenMap(Y) = EigenMap(X).array().log(); + + return Status::OK(); +} + +template <> +Status Sum_6::Compute(OpKernelContext* ctx) const { + auto input_count = Node().InputArgCount().front(); + ONNXRUNTIME_ENFORCE(input_count >= 1, "Must have 1 or more inputs"); + auto& data_0 = *ctx->Input(0); + auto& shape = data_0.Shape(); + auto sum = EigenMap(*ctx->Output(0, shape)); + + if (input_count == 1) { + sum = EigenMap(data_0); + } else { + auto& data_1 = *ctx->Input(1); + ONNXRUNTIME_ENFORCE(data_1.Shape() == shape, "All inputs must have the same shape"); + + sum = EigenMap(data_0) + EigenMap(data_1); + for (int index = 2; index < input_count; index++) { + auto& data_n = *ctx->Input(index); + ONNXRUNTIME_ENFORCE(data_n.Shape() == shape, "All inputs must have the same shape"); + sum += EigenMap(data_n); + } + } + + return Status::OK(); +} + +template <> +Status Sum_8::Compute(OpKernelContext* context) const { + return BroadcastVariadic( + Node(), *context, + [](EigenVectorMap output, float input0, ConstEigenVectorMap input1) { output = input0 + input1.array(); }, + [](EigenVectorMap output, ConstEigenVectorMap input0, float input1) { output = input0.array() + input1; }, + [](EigenVectorMap output, ConstEigenVectorMap input0, ConstEigenVectorMap input1) { output = input0 + input1; }); +} + +template <> +Status Min_6::Compute(OpKernelContext* ctx) const { + auto inputCount = Node().InputArgCount().front(); + ONNXRUNTIME_ENFORCE(inputCount >= 1, "Must have 1 or more inputs"); + auto& data_0 = *ctx->Input(0); + auto& shape = data_0.Shape(); + auto min = EigenMap(*ctx->Output(0, shape)); + + min = EigenMap(data_0); + for (int index = 1; index < inputCount; index++) { + auto& data_n = *ctx->Input(index); + ONNXRUNTIME_ENFORCE(data_n.Shape() == shape, "All inputs must have the same shape"); + min = min.array().min(EigenMap(data_n).array()); + } + + return Status::OK(); +} + +template <> +Status Min_8::Compute(OpKernelContext* context) const { + return BroadcastVariadic( + Node(), *context, + [](EigenVectorMap output, float input0, ConstEigenVectorMap input1) { output = input1.array().min(input0); }, + [](EigenVectorMap output, ConstEigenVectorMap input0, float input1) { output = input0.array().min(input1); }, + [](EigenVectorMap output, ConstEigenVectorMap input0, ConstEigenVectorMap input1) { output = input0.array().min(input1.array()); }); +} + +template <> +Status Max_6::Compute(OpKernelContext* ctx) const { + auto inputCount = Node().InputArgCount().front(); + ONNXRUNTIME_ENFORCE(inputCount >= 1, "Must have 1 or more inputs"); + auto& data_0 = *ctx->Input(0); + auto& shape = data_0.Shape(); + auto max = EigenMap(*ctx->Output(0, shape)); + + max = EigenMap(data_0); + for (int index = 1; index < inputCount; index++) { + auto& data_n = *ctx->Input(index); + ONNXRUNTIME_ENFORCE(data_n.Shape() == shape, "All inputs must have the same shape"); + max = max.array().max(EigenMap(data_n).array()); + } + + return Status::OK(); +} + +template <> +Status Max_8::Compute(OpKernelContext* context) const { + return BroadcastVariadic( + Node(), *context, + [](EigenVectorMap output, float input0, ConstEigenVectorMap input1) { output = input1.array().max(input0); }, + [](EigenVectorMap output, ConstEigenVectorMap input0, float input1) { output = input0.array().max(input1); }, + [](EigenVectorMap output, ConstEigenVectorMap input0, ConstEigenVectorMap input1) { output = input0.array().max(input1.array()); }); +} + +Status Not::Compute(OpKernelContext* context) const { + auto& input = *context->Input(0); + auto& output = *context->Output(0, input.Shape()); + + EigenMap(output).array() = !EigenMap(input).array(); + return Status::OK(); +} + +Status And::Compute(OpKernelContext* context) const { + // The scalar cases are special cased, since 'X && true = X' and 'X && false = false' + return BroadcastTwo( + *context, + [](EigenVectorMap output, bool input0, ConstEigenVectorMap input1) { + if (input0) + output = input1; + else + output.array() = false; + }, + [](EigenVectorMap output, ConstEigenVectorMap input0, bool input1) { + if (input1) + output = input0; + else + output.array() = false; + }, + [](EigenVectorMap output, ConstEigenVectorMap input0, ConstEigenVectorMap input1) { output = input0.array() && input1.array(); }); +} + +Status Or::Compute(OpKernelContext* context) const { + // The scalar cases are special cased, since 'X || true = true' and 'X || false = X' + return BroadcastTwo( + *context, + [](EigenVectorMap output, bool input0, ConstEigenVectorMap input1) { + if (input0) + output.array() = true; + else + output = input1; + }, + [](EigenVectorMap output, ConstEigenVectorMap input0, bool input1) { + if (input1) + output.array() = true; + else + output = input0; + }, + [](EigenVectorMap output, ConstEigenVectorMap input0, ConstEigenVectorMap input1) { output = input0.array() || input1.array(); }); +} + +Status Xor::Compute(OpKernelContext* context) const { + // The scalar cases are special cased, since 'X ^ true = !X' and 'X ^ false = X' + return BroadcastTwo( + *context, + [](EigenVectorMap output, bool input0, ConstEigenVectorMap input1) { + if (input0) + output.array() = !input1.array(); + else + output = input1; + }, + [](EigenVectorMap output, ConstEigenVectorMap input0, bool input1) { + if (input1) + output.array() = !input0.array(); + else + output = input0; + }, + [](EigenVectorMap output, ConstEigenVectorMap input0, ConstEigenVectorMap input1) { output = input0.array() ^ input1.array(); }); +} + +template +Status Equal::Compute(OpKernelContext* context) const { + return BroadcastTwo( + *context, + [](EigenVectorMap output, T input0, ConstEigenVectorMap input1) { output = input1.array() == input0; }, + [](EigenVectorMap output, ConstEigenVectorMap input0, T input1) { output = input0.array() == input1; }, + [](EigenVectorMap output, ConstEigenVectorMap input0, ConstEigenVectorMap input1) { output = input0.array() == input1.array(); }); +} + +template +Status Less::Compute(OpKernelContext* context) const { + return BroadcastTwo( + *context, + [](EigenVectorMap output, T input0, ConstEigenVectorMap input1) { output = input1.array() > input0; }, + [](EigenVectorMap output, ConstEigenVectorMap input0, T input1) { output = input0.array() < input1; }, + [](EigenVectorMap output, ConstEigenVectorMap input0, ConstEigenVectorMap input1) { output = input0.array() < input1.array(); }); +} + +template +Status Greater::Compute(OpKernelContext* context) const { + return BroadcastTwo( + *context, + [](EigenVectorMap output, T input0, ConstEigenVectorMap input1) { output = input1.array() < input0; }, + [](EigenVectorMap output, ConstEigenVectorMap input0, T input1) { output = input0.array() > input1; }, + [](EigenVectorMap output, ConstEigenVectorMap input0, ConstEigenVectorMap input1) { output = input0.array() > input1.array(); }); +} + +template <> +Status Mean_6::Compute(OpKernelContext* ctx) const { + auto inputCount = Node().InputArgCount().front(); + ONNXRUNTIME_ENFORCE(inputCount >= 1, "Must have 1 or more inputs"); + auto& data_0 = *ctx->Input(0); + auto& shape = data_0.Shape(); + auto mean = EigenMap(*ctx->Output(0, shape)); + + if (inputCount == 1) { + mean = EigenMap(data_0); + } else { + auto& data_1 = *ctx->Input(1); + ONNXRUNTIME_ENFORCE(data_1.Shape() == shape, "All inputs must have the same shape"); + + mean = EigenMap(data_0) + EigenMap(data_1); + for (int index = 2; index < inputCount; index++) { + auto& data_n = *ctx->Input(index); + ONNXRUNTIME_ENFORCE(data_n.Shape() == shape, "All inputs must have the same shape"); + mean += EigenMap(data_n); + } + } + + // Take the mean + float weight = 1.0f / static_cast(inputCount); + mean = mean * weight; + + return Status::OK(); +} + +template <> +Status Mean_8::Compute(OpKernelContext* context) const { + // Do a sum exactly the same as in Sum_8 + Status status = BroadcastVariadic( + Node(), *context, + [](EigenVectorMap output, float input0, ConstEigenVectorMap input1) { output = input0 + input1.array(); }, + [](EigenVectorMap output, ConstEigenVectorMap input0, float input1) { output = input0.array() + input1; }, + [](EigenVectorMap output, ConstEigenVectorMap input0, ConstEigenVectorMap input1) { output = input0 + input1; }); + if (!status.IsOK()) + return status; + + // Now divide by the input count to get the mean + EigenMap(*context->Output(0)) *= 1.0f / static_cast(Node().InputArgCount().front()); + return Status::OK(); +} + +template <> +Status Affine::Compute(OpKernelContext* ctx) const { + auto& X = *ctx->Input(0); + auto& Y = *ctx->Output(0, X.Shape()); + MakeEigenArrayMap(Y) = alpha_ * MakeEigenArrayMap(X) + beta_; + return Status::OK(); +} + +template +class Sin final : public OpKernel { + public: + Sin(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override { + auto& X = *context->Input(0); + auto& Y = *context->Output(0, X.Shape()); + MakeEigenArrayMap(Y) = MakeEigenArrayMap(X).sin(); + return Status::OK(); + } +}; + +ONNX_CPU_OPERATOR_KERNEL( + Sin, + 7, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Sin); + +template +class Cos final : public OpKernel { + public: + Cos(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override { + auto& X = *context->Input(0); + auto& Y = *context->Output(0, X.Shape()); + MakeEigenArrayMap(Y) = MakeEigenArrayMap(X).cos(); + return Status::OK(); + } +}; + +ONNX_CPU_OPERATOR_KERNEL( + Cos, + 7, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Cos); + +template +class Tan final : public OpKernel { + public: + Tan(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override { + auto& X = *context->Input(0); + auto& Y = *context->Output(0, X.Shape()); + MakeEigenArrayMap(Y) = MakeEigenArrayMap(X).tan(); + return Status::OK(); + } +}; + +ONNX_CPU_OPERATOR_KERNEL( + Tan, + 7, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Tan); + +template +class Asin final : public OpKernel { + public: + Asin(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override { + auto& X = *context->Input(0); + auto& Y = *context->Output(0, X.Shape()); + MakeEigenArrayMap(Y) = MakeEigenArrayMap(X).asin(); + return Status::OK(); + } +}; + +ONNX_CPU_OPERATOR_KERNEL( + Asin, + 7, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Asin); + +template +class Acos final : public OpKernel { + public: + Acos(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override { + auto& X = *context->Input(0); + auto& Y = *context->Output(0, X.Shape()); + MakeEigenArrayMap(Y) = MakeEigenArrayMap(X).acos(); + return Status::OK(); + } +}; + +ONNX_CPU_OPERATOR_KERNEL( + Acos, + 7, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Acos); + +template +class Atan final : public OpKernel { + public: + Atan(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override { + auto& X = *context->Input(0); + auto& Y = *context->Output(0, X.Shape()); + MakeEigenArrayMap(Y) = MakeEigenArrayMap(X).atan(); + return Status::OK(); + } +}; + +ONNX_CPU_OPERATOR_KERNEL( + Atan, + 7, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Atan); + +template <> +Status PRelu::Compute(OpKernelContext* context) const { + return BroadcastTwo( + *context, + [](EigenVectorMap output, float input0, ConstEigenVectorMap input1) { + if (input0 > 0) + output.array() = input0; + else + output = input0 * input1.array(); + }, + [](EigenVectorMap output, ConstEigenVectorMap input0, float input1) { + output = (input0.array() > 0).select(input0, input0 * input1); + }, + [](EigenVectorMap output, ConstEigenVectorMap input0, ConstEigenVectorMap input1) { + output = (input0.array() > 0).select(input0, input0.cwiseProduct(input1)); + }); +} + +ONNX_CPU_OPERATOR_VERSIONED_KERNEL( + PRelu, + 7, + 9, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + PRelu); + +// This is a special case version of TBroadcaster just for Expand that only has a shape as the second parameter +template +struct TBroadcasterExpand { + TBroadcasterExpand(const Tensor& input, const std::vector& shape) + : input_tensor_(input), + broadcaster_(input.Shape().GetDims(), shape) { + } + + TensorShape GetOutputShape() const { return TensorShape(broadcaster_.output_shape_); } + size_t GetSpanSize() const { return span_size_; } + + bool IsInput0Scalar() const { return broadcaster_.iterator1_.deltas_.front() == 0; } + + T NextScalar() { return *Next(); } + + ConstEigenVectorMap NextEigen() { return ConstEigenVectorMap(Next(), span_size_); } + + private: + const T* Next() { return input_ + broadcaster_.iterator1_.AdvanceBy(span_size_); } + + const Tensor& input_tensor_; + Broadcaster broadcaster_; + size_t span_size_{broadcaster_.GetSpanSize()}; + + const T* input_{input_tensor_.template Data()}; +}; + +template +Status Expand_8::Compute(OpKernelContext* context) const { + auto& tensor_shape = *context->Input(1); + ONNXRUNTIME_ENFORCE(tensor_shape.Shape().GetDims().size() == 1, "Shape must be 1 dimensional as it's tensor data is a shape"); + + // Turn the shape tensor data into an actual shape + const int64_t* p_shape = tensor_shape.template Data(); + std::vector shape{p_shape, p_shape + tensor_shape.Shape().Size()}; + + TBroadcasterExpand bc(*context->Input(0), shape); + TBroadcastOutput output(bc.GetSpanSize(), *context->Output(0, bc.GetOutputShape())); + + // This doesn't use BroadcastLoop since there is no second tensor, just duplicating the first + if (bc.IsInput0Scalar()) { + // Input0 being a scalar is the only special case here, since we're duplicating a single value + while (output) + output.NextEigenOutput().array() = bc.NextScalar(); + } else { + // Input1 being a scalar doesn't matter (as there's no actual input1). We're still duplicating Input0 in the same sized chunks + while (output) + output.NextEigenOutput() = bc.NextEigen(); + } + return Status::OK(); +} + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Expand, + 8, + float, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Expand_8); + +template <> +Status Scale::Compute(OpKernelContext* ctx) const { + auto& X = *ctx->Input(0); + auto& Y = *ctx->Output(0, X.Shape()); + EigenMap(Y) = scale_ * EigenMap(X); + return Status::OK(); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/math/element_wise_ops.h b/onnxruntime/core/providers/cpu/math/element_wise_ops.h new file mode 100644 index 0000000000000..13d9c5ea5df52 --- /dev/null +++ b/onnxruntime/core/providers/cpu/math/element_wise_ops.h @@ -0,0 +1,320 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/util/math_cpuonly.h" + +namespace onnxruntime { + +template +class Add final : public OpKernel { + public: + Add(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class Sub final : public OpKernel { + public: + Sub(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class Mul final : public OpKernel { + public: + Mul(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class Div final : public OpKernel { + public: + Div(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class Abs final : public OpKernel { + public: + Abs(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override { + auto& input = *context->Input(0); + auto& output = *context->Output(0, input.Shape()); + + EigenMap(output) = EigenMap(input).cwiseAbs(); + return Status::OK(); + } +}; + +template +class Neg final : public OpKernel { + public: + Neg(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override { + auto& input = *context->Input(0); + auto& output = *context->Output(0, input.Shape()); + + EigenMap(output) = -EigenMap(input); + return Status::OK(); + } +}; + +template +class Floor final : public OpKernel { + public: + Floor(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class Ceil final : public OpKernel { + public: + Ceil(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class Reciprocal final : public OpKernel { + public: + Reciprocal(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class Sqrt final : public OpKernel { + public: + Sqrt(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class Pow final : public OpKernel { + public: + Pow(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class Exp final : public OpKernel { + public: + Exp(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class Log final : public OpKernel { + public: + Log(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class Sum_6 final : public OpKernel { + public: + Sum_6(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class Sum_8 final : public OpKernel { + public: + Sum_8(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class Min_6 final : public OpKernel { + public: + Min_6(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class Min_8 final : public OpKernel { + public: + Min_8(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class Max_6 final : public OpKernel { + public: + Max_6(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class Max_8 final : public OpKernel { + public: + Max_8(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +class Not final : public OpKernel { + public: + Not(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +class And final : public OpKernel { + public: + And(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +class Or final : public OpKernel { + public: + Or(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +class Xor final : public OpKernel { + public: + Xor(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class Equal final : public OpKernel { + public: + Equal(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class Less final : public OpKernel { + public: + Less(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class Greater final : public OpKernel { + public: + Greater(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class Mean_6 final : public OpKernel { + public: + Mean_6(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class Mean_8 final : public OpKernel { + public: + Mean_8(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class Affine final : public OpKernel { + public: + Affine(const OpKernelInfo& info) : OpKernel(info) { + // Either model-supplied or default values should be returned for alpha and beta + ONNXRUNTIME_ENFORCE(info.GetAttr("alpha", &alpha_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttr("beta", &beta_).IsOK()); + } + + Status Compute(OpKernelContext* context) const override; + + private: + float alpha_; + float beta_; +}; + +// PRelu is activation function, but it's closer to binary elementwise ops in implementation +template +class PRelu final : public OpKernel { + public: + PRelu(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class Expand_8 final : public OpKernel { + public: + Expand_8(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class Scale final : public OpKernel { + public: + Scale(const OpKernelInfo& info) : OpKernel(info) { + ONNXRUNTIME_ENFORCE(info.GetAttr("scale", &scale_).IsOK()); + } + + Status Compute(OpKernelContext* context) const override; + + private: + float scale_; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/math/gemm.cc b/onnxruntime/core/providers/cpu/math/gemm.cc new file mode 100644 index 0000000000000..6554e3c60145f --- /dev/null +++ b/onnxruntime/core/providers/cpu/math/gemm.cc @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/math/gemm.h" + +namespace onnxruntime { + +ONNX_CPU_OPERATOR_VERSIONED_KERNEL( + Gemm, + 7, + 9, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Gemm); +} \ No newline at end of file diff --git a/onnxruntime/core/providers/cpu/math/gemm.h b/onnxruntime/core/providers/cpu/math/gemm.h new file mode 100644 index 0000000000000..b995f95a82609 --- /dev/null +++ b/onnxruntime/core/providers/cpu/math/gemm.h @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/util/math.h" +#include "core/util/math_cpuonly.h" +#include "gemm_helper.h" + +namespace onnxruntime { + +template +class Gemm final : public OpKernel { + public: + Gemm(const OpKernelInfo& info) : OpKernel(info) { + int64_t temp; + ONNXRUNTIME_ENFORCE(info.GetAttr("transA", &temp).IsOK()); + trans_A_ = temp == 0 ? CblasNoTrans : CblasTrans; + + ONNXRUNTIME_ENFORCE(info.GetAttr("transB", &temp).IsOK()); + trans_B_ = temp == 0 ? CblasNoTrans : CblasTrans; + + ONNXRUNTIME_ENFORCE(info.GetAttr("alpha", &alpha_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttr("beta", &beta_).IsOK()); + } + + Status Compute(OpKernelContext* context) const override { + const auto X = context->Input(0); + const auto W = context->Input(1); + const auto B = context->Input(2); + GemmHelper helper(X->Shape(), trans_A_ != CblasNoTrans, W->Shape(), trans_B_ != CblasNoTrans, B->Shape()); + + if (!helper.State().IsOK()) + return helper.State(); + + int64_t M = helper.M(); + int64_t N = helper.N(); + int64_t K = helper.K(); + auto Y = context->Output(0, TensorShape({M, N})); + + //bias + // Todo: we might should move this part into math::gemm to let eigen + // have better chance to further optimize it. + if (beta_ != 0) { + auto output_mat = EigenMatrixMapRowMajor( + Y->template MutableData(), + M, + N); + output_mat.setZero(); + + auto& b_shape = B->Shape(); + // if B is (), (1,) or (1, 1), add the scalar + if (b_shape.Size() == 1) { + output_mat.array() += *(B->template Data()); + } + // B is (N,) + else if (b_shape.NumDimensions() == 1) { + auto bias_vec = ConstEigenVectorMap( + B->template Data(), + N); + output_mat.rowwise() += bias_vec.transpose(); + } else if (b_shape.NumDimensions() == 2) { + // B is (M, 1) + if (b_shape[1] == 1) { + auto bias_vec = ConstEigenVectorMap( + B->template Data(), + M); + output_mat.colwise() += bias_vec; + } + // B is (1, N) + else if (b_shape[0] == 1) { + auto bias_vec = ConstEigenVectorMap( + B->template Data(), + N); + output_mat.rowwise() += bias_vec.transpose(); + } + // B is (M, N), no broadcast needed. + else { + auto bias_mat = ConstEigenMatrixMapRowMajor( + B->template Data(), + M, + N); + output_mat += bias_mat; + } + } + } + + // W * x + math::Gemm( + trans_A_, + trans_B_, + M, + N, + K, + alpha_, + X->template Data(), + W->template Data(), + beta_, + Y->template MutableData(), + &CPUMathUtil::Instance()); + + return Status::OK(); + } + + private: + CBLAS_TRANSPOSE trans_A_; + CBLAS_TRANSPOSE trans_B_; + float alpha_; + float beta_; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/math/gemm_helper.h b/onnxruntime/core/providers/cpu/math/gemm_helper.h new file mode 100644 index 0000000000000..14d259eb4bb25 --- /dev/null +++ b/onnxruntime/core/providers/cpu/math/gemm_helper.h @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/common/common.h" + +namespace onnxruntime { + +class GemmHelper { + public: + GemmHelper(const TensorShape& left, bool trans_left, const TensorShape& right, bool trans_right, const TensorShape& bias) { + //dimension check + ONNXRUNTIME_ENFORCE(left.NumDimensions() == 2); + ONNXRUNTIME_ENFORCE(right.NumDimensions() == 2); + + if (trans_left) { + M_ = left[1]; + K_ = left[0]; + } else { + M_ = left[0]; + K_ = left[1]; + } + + int k_dim; + if (trans_right) { + N_ = right[0]; + k_dim = 1; + } else { + N_ = right[1]; + k_dim = 0; + } + + if (right[k_dim] != K_) + status_ = ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "GEMM: Dimension mismatch, W: ", + right.ToString(), + " K: " + std::to_string(K_), + " N:" + std::to_string(N_)); + + if (!IsValidBroadcast(bias, M_, N_)) + status_ = common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Gemm: Invalid bias shape for broadcast"); + + ONNXRUNTIME_ENFORCE(M_ > 0 && N_ > 0 && K_ > 0); + } + + int64_t M() const { return M_; } + int64_t N() const { return N_; } + int64_t K() const { return K_; } + Status State() const { return status_; } + + private: + bool IsValidBroadcast(const TensorShape& shape, int64_t M, int64_t N) { + if (shape.NumDimensions() != 1 && shape.NumDimensions() != 2) + return false; + // shape is (1,) or (1, 1), or (,) + if (shape.Size() == 1) + return true; + // shape is (N,) or (1, N) or (M, 1) + // or (M, N), in last case no broadcast needed, but don't fail it + return ((shape.NumDimensions() == 1 && shape[0] == N) || + (shape.NumDimensions() == 2 && shape[0] == M && (shape[1] == 1 || shape[1] == N)) || + (shape.NumDimensions() == 2 && shape[0] == 1 && shape[1] == N)); + } + + private: + int64_t M_; + int64_t K_; + int64_t N_; + Status status_; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/math/hardmax.cc b/onnxruntime/core/providers/cpu/math/hardmax.cc new file mode 100644 index 0000000000000..2bb5e5b8b684a --- /dev/null +++ b/onnxruntime/core/providers/cpu/math/hardmax.cc @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/math/hardmax.h" +#include "core/util/math_cpuonly.h" +#include "core/util/math.h" + +namespace onnxruntime { + +template <> +Status Hardmax::Compute(OpKernelContext* ctx) const { + const Tensor* X = ctx->Input(0); + const TensorShape& input_shape = X->Shape(); + const float* Xdata = X->template Data(); + + size_t tmpN = input_shape.SizeToDimension(axis_); + size_t tmpD = input_shape.SizeFromDimension(axis_); + + // Math::RowwiseMax expects int N and D. + if (tmpN * tmpD > INT32_MAX || tmpN > INT32_MAX || tmpD > INT32_MAX) { + std::ostringstream ss; + ss << "Hardmax inputs N, D and N * D must be < " << INT32_MAX << ". N=" << tmpN << ", D=" << tmpD; + std::string msg = ss.str(); + + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, msg); + } + + const int N = gsl::narrow_cast(tmpN); + const int D = gsl::narrow_cast(tmpD); + + std::vector rowmax_(N); + float* rowmax_data = rowmax_.data(); + math::RowwiseMax(N, D, Xdata, rowmax_data, nullptr); + + Tensor* Y = ctx->Output(0, input_shape); + float* Ydata = Y->template MutableData(); + math::Set(input_shape.Size(), 0.f, Ydata, &CPUMathUtil::Instance()); + + for (int i = 0; i < N; ++i) { + for (int j = 0; j < D; ++j) { + if (Xdata[i * D + j] == rowmax_data[i]) { + Ydata[i * D + j] = 1; + break; + } + } + } + + return Status::OK(); +} + +ONNX_CPU_OPERATOR_KERNEL( + Hardmax, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Hardmax); + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/math/hardmax.h b/onnxruntime/core/providers/cpu/math/hardmax.h new file mode 100644 index 0000000000000..23c7bb6b36920 --- /dev/null +++ b/onnxruntime/core/providers/cpu/math/hardmax.h @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "gsl/gsl_util" + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { +template +class Hardmax final : public OpKernel { + public: + Hardmax(const OpKernelInfo& info) : OpKernel{info}, axis_{1} { + int64_t axis; + Status status = info.GetAttr("axis", &axis); + + if (status.IsOK()) { + axis_ = gsl::narrow_cast(axis); + } + + // if value was provided, make sure it was valid + ONNXRUNTIME_ENFORCE(axis_ >= 0, "Invalid axis provided."); + } + + Status Compute(OpKernelContext* context) const override; + + private: + int axis_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/math/logsoftmax.cc b/onnxruntime/core/providers/cpu/math/logsoftmax.cc new file mode 100644 index 0000000000000..6b3c108e32620 --- /dev/null +++ b/onnxruntime/core/providers/cpu/math/logsoftmax.cc @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/math/logsoftmax.h" + +#include "core/framework/op_kernel.h" +#include "core/providers/common.h" +#include "core/providers/cpu/math/softmax_shared.h" +#include "core/util/math.h" + +namespace onnxruntime { + +template <> +Status LogSoftmax::Compute(OpKernelContext* ctx) const { + const Tensor* tensor_pointer = ctx->Input(0); + if (tensor_pointer == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + const Tensor& X = *tensor_pointer; + const TensorShape input_shape{X.Shape()}; + + Tensor* Y = ctx->Output(0, input_shape); + + const int64_t axis = HandleNegativeAxis(axis_, input_shape.NumDimensions()); + + size_t N = input_shape.SizeToDimension(axis); + size_t D = input_shape.SizeFromDimension(axis); + + float* Ydata = Y->template MutableData(); + + std::vector scale_(N); + std::vector rowmax_(N); + std::vector sum_multiplier_(D, 1.f); // initialize all multiplier values to 1.0 + + const bool logarithmic = true; + auto status = SoftmaxCPU(N, D, X.template Data(), Ydata, + scale_.data(), sum_multiplier_.data(), logarithmic, rowmax_.data()); + + return status; +} + +ONNX_CPU_OPERATOR_KERNEL( + LogSoftmax, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + LogSoftmax); + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/math/logsoftmax.h b/onnxruntime/core/providers/cpu/math/logsoftmax.h new file mode 100644 index 0000000000000..e1362d87608b3 --- /dev/null +++ b/onnxruntime/core/providers/cpu/math/logsoftmax.h @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "gsl/gsl_util" + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/util/math_cpuonly.h" + +namespace onnxruntime { +template +class LogSoftmax final : public OpKernel { + public: + LogSoftmax(const OpKernelInfo& info) : OpKernel{info}, axis_{1} { + int64_t axis; + Status status = info.GetAttr("axis", &axis); + + if (status.IsOK()) { + axis_ = gsl::narrow_cast(axis); + } + } + + Status Compute(OpKernelContext* context) const override; + + private: + int axis_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/math/matmul.cc b/onnxruntime/core/providers/cpu/math/matmul.cc new file mode 100644 index 0000000000000..68ae225320821 --- /dev/null +++ b/onnxruntime/core/providers/cpu/math/matmul.cc @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/math/matmul.h" + +#include "core/util/math.h" +#include "core/util/math_cpuonly.h" +#include "matmul_helper.h" + +namespace onnxruntime { + +ONNX_CPU_OPERATOR_VERSIONED_KERNEL( + MatMul, + 1, + 9, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + MatMul); + +template <> +Status MatMul::Compute(OpKernelContext* ctx) const { + const Tensor* left_X = ctx->Input(0); + const Tensor* right_X = ctx->Input(1); + + MatMulComputeHelper helper; + ONNXRUNTIME_RETURN_IF_ERROR(helper.Compute(left_X->Shape(), right_X->Shape())); + + Tensor* Y = ctx->Output(0, helper.OutputShape()); + + // TODO: replace it with GemmBatch for performance, it's OK for now as GemmBatch unrolls as well + for (int i = 0; i < helper.OutputOffsets().size(); i++) { + math::Gemm( + CblasNoTrans, + CblasNoTrans, + static_cast(helper.M()), + static_cast(helper.N()), + static_cast(helper.K()), + /* alpha */ 1.0f, + left_X->template Data() + helper.LeftOffsets()[i], + right_X->template Data() + helper.RightOffsets()[i], + /* beta */ 0.0f, + Y->template MutableData() + helper.OutputOffsets()[i], + &CPUMathUtil::Instance()); + } + + return Status::OK(); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/math/matmul.h b/onnxruntime/core/providers/cpu/math/matmul.h new file mode 100644 index 0000000000000..02a42aa3ef874 --- /dev/null +++ b/onnxruntime/core/providers/cpu/math/matmul.h @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { + +template +class MatMul final : public OpKernel { + public: + MatMul(const OpKernelInfo& info) + : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/math/matmul_helper.h b/onnxruntime/core/providers/cpu/math/matmul_helper.h new file mode 100644 index 0000000000000..994b621998cf3 --- /dev/null +++ b/onnxruntime/core/providers/cpu/math/matmul_helper.h @@ -0,0 +1,258 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/common/common.h" +#include "core/framework/tensor.h" + +namespace onnxruntime { + +class MatMulComputeHelper { + public: + Status Compute(const TensorShape& left_shape, const TensorShape& right_shape) { + // Following numpy.matmul for shape inference: + // https://docs.scipy.org/doc/numpy/reference/generated/numpy.matmul.html + // The behavior depends on the arguments in the following way. + // * If both arguments are 2 - D they are multiplied like conventional matrices. + // * If either argument is N - D, N > 2, it is treated as a stack of matrices residing in the last two indexes and broadcast accordingly. + // * If the first argument is 1 - D, it is promoted to a matrix by prepending a 1 to its dimensions.After matrix multiplication the prepended 1 is removed. + // * If the second argument is 1 - D, it is promoted to a matrix by appending a 1 to its dimensions.After matrix multiplication the appended 1 is removed. + + size_t left_num_dims = left_shape.NumDimensions(); + size_t right_num_dims = right_shape.NumDimensions(); + ONNXRUNTIME_RETURN_IF_NOT(left_num_dims >= 1 && right_num_dims >= 1); + + // special case for right_shape being 2D and left_shape > 2D by flattening left_shape to 2D + // note that padding 1s in front of the right shape can be flattened too + if (left_num_dims >= 2 && right_num_dims >= 2 && + right_shape.SizeToDimension(right_num_dims - 1) == right_shape[right_num_dims - 2]) { + M_ = left_shape.SizeToDimension(left_num_dims - 1); + K_ = left_shape[left_num_dims - 1]; + N_ = right_shape[right_num_dims - 1]; + std::vector output_dims = left_shape.GetDims(); + output_dims[left_num_dims - 1] = N_; + output_shape_ = TensorShape(output_dims); + output_offsets_ = {0}; + left_offsets_ = {0}; + right_offsets_ = {0}; + ONNXRUNTIME_RETURN_IF_NOT(K_ == right_shape[right_num_dims - 2], "MatMul dimension mismatch"); + return Status::OK(); + } + + bool has_1D_input = (left_num_dims == 1 || right_num_dims == 1); + + size_t num_input_dims = std::max(left_num_dims, right_num_dims); + + // use padded dims to compute matrix offsets, right 1D would be padded + size_t num_dims_with_pad = num_input_dims + (right_num_dims == 1 ? 1 : 0); + + // output shape would squeeze the reduced 1D dimension + size_t num_output_dims = num_input_dims - (has_1D_input ? 1 : 0); + + left_padded_dims_ = std::vector(num_dims_with_pad, 1); + right_padded_dims_ = std::vector(num_dims_with_pad, 1); + + if (right_num_dims == 1) { + // right padded to (1,...,K,1) + right_padded_dims_[num_dims_with_pad - 2] = right_shape[0]; + + if (num_input_dims >= 2) { + // left padded to (...,1,K) + left_shape.CopyDims(&left_padded_dims_[0], left_num_dims - 1); + left_padded_dims_[num_dims_with_pad - 1] = left_shape[left_num_dims - 1]; + } else { + // pad 1 in the front + left_shape.CopyDims(&left_padded_dims_[num_dims_with_pad - left_num_dims], left_num_dims); + } + } else { + // pad 1 in the front for left + left_shape.CopyDims(&left_padded_dims_[num_dims_with_pad - left_num_dims], left_num_dims); + // pad 1 in the front for right + right_shape.CopyDims(&right_padded_dims_[num_dims_with_pad - right_num_dims], right_num_dims); + } + + // validate input shape and generate output shape + std::vector output_dims(num_output_dims); + + // broadcasting for all output dims except last two + for (int idx_dim = 0; idx_dim < num_dims_with_pad - 2; ++idx_dim) { + output_dims[idx_dim] = std::max(left_padded_dims_[idx_dim], right_padded_dims_[idx_dim]); + if (left_padded_dims_[idx_dim] != output_dims[idx_dim]) + ONNXRUNTIME_RETURN_IF_NOT(left_padded_dims_[idx_dim] == 1, "left operand cannot broadcast on dim ", idx_dim); + if (right_padded_dims_[idx_dim] != output_dims[idx_dim]) + ONNXRUNTIME_RETURN_IF_NOT(right_padded_dims_[idx_dim] == 1, "right operand cannot broadcast on dim ", idx_dim); + } + + M_ = has_1D_input ? 1 : left_shape[left_num_dims - 2]; + K_ = left_shape[left_num_dims - 1]; + N_ = (right_num_dims == 1) ? 1 : right_shape[right_num_dims - 1]; + + if (!has_1D_input) { + ONNXRUNTIME_RETURN_IF_NOT(K_ == right_shape[right_num_dims - 2], "MatMul dimension mismatch"); + // left (...M x K), right (...K x N), output (...M x N) + ONNXRUNTIME_RETURN_IF_NOT(num_dims_with_pad == num_output_dims); + output_dims[num_output_dims - 2] = M_; + output_dims[num_output_dims - 1] = N_; + } else { + if (num_output_dims == 0) { + // for left and right being both vector, output is scalar thus no shape + ONNXRUNTIME_RETURN_IF_NOT(M_ == 1 && N_ == 1); + } else { + if (left_num_dims == 1) { + ONNXRUNTIME_RETURN_IF_NOT(num_dims_with_pad - 1 == num_output_dims); + ONNXRUNTIME_RETURN_IF_NOT(K_ == right_shape[right_num_dims - 2], "MatMul dimension mismatch"); + // left (K), right (...K,N), output (...N) + output_dims[num_output_dims - 1] = N_; + } else { + ONNXRUNTIME_RETURN_IF_NOT(num_dims_with_pad - 2 == num_output_dims); + ONNXRUNTIME_RETURN_IF_NOT(K_ == right_shape[0], "MatMul dimension mismatch"); + // left(...K), right (K), output (...), already assigned + } + } + } + + // assign shape + output_shape_ = TensorShape(output_dims); + + // compute broadcast offsets + ComputeBroadcastOffsets(); + + return Status::OK(); + } + + private: + void ComputeBroadcastOffsets() { + num_broadcasted_dims_ = left_padded_dims_.size() - 2; + + if (num_broadcasted_dims_ == 0) { + left_offsets_ = {0}; + right_offsets_ = {0}; + output_offsets_ = {0}; + return; + } + + left_mat_size_ = M_ * K_; + right_mat_size_ = K_ * N_; + output_mat_size_ = M_ * N_; + + // stride in mats and dims for broadcasting + left_padded_strides_.resize(num_broadcasted_dims_); + right_padded_strides_.resize(num_broadcasted_dims_); + output_broadcast_strides_.resize(num_broadcasted_dims_); + output_broadcast_dims_.resize(num_broadcasted_dims_); + for (size_t i = num_broadcasted_dims_; i > 0; --i) { + size_t idx = i - 1; + output_broadcast_dims_[idx] = std::max(left_padded_dims_[idx], right_padded_dims_[idx]); + output_broadcast_strides_[idx] = ((i == num_broadcasted_dims_) ? 1 : output_broadcast_strides_[idx + 1] * output_broadcast_dims_[idx + 1]); + left_padded_strides_[idx] = ((i == num_broadcasted_dims_) ? 1 : left_padded_strides_[idx + 1] * left_padded_dims_[idx + 1]); + right_padded_strides_[idx] = ((i == num_broadcasted_dims_) ? 1 : right_padded_strides_[idx + 1] * right_padded_dims_[idx + 1]); + } + + size_t num_offsets = output_broadcast_dims_[0] * output_broadcast_strides_[0]; + left_offsets_.resize(num_offsets); + right_offsets_.resize(num_offsets); + output_offsets_.resize(num_offsets); + + RecursiveFill(0, 0, 0, 0); + } + + void + RecursiveFill(size_t idx_dim, size_t idx_left, size_t idx_right, size_t idx_out) { + if (idx_dim == num_broadcasted_dims_) { + left_offsets_[idx_out] = idx_left * left_mat_size_; + right_offsets_[idx_out] = idx_right * right_mat_size_; + output_offsets_[idx_out] = idx_out * output_mat_size_; + } else { + auto left_dim = left_padded_dims_[idx_dim]; + auto right_dim = right_padded_dims_[idx_dim]; + auto output_dim = output_broadcast_dims_[idx_dim]; + for (int64_t i = 0; i < output_dim; ++i) { + RecursiveFill(idx_dim + 1, + idx_left + i * (left_dim == 1 ? 0 : left_padded_strides_[idx_dim]), + idx_right + i * (right_dim == 1 ? 0 : right_padded_strides_[idx_dim]), + idx_out + i * output_broadcast_strides_[idx_dim]); + } + } + } + + private: + size_t left_mat_size_; + size_t right_mat_size_; + size_t output_mat_size_; + + size_t num_broadcasted_dims_; + + std::vector left_padded_dims_; + std::vector right_padded_dims_; + std::vector output_broadcast_dims_; + + std::vector left_padded_strides_; + std::vector right_padded_strides_; + std::vector output_broadcast_strides_; + + TensorShape output_shape_; + + int64_t M_; + int64_t N_; + int64_t K_; + + std::vector left_offsets_; + std::vector right_offsets_; + std::vector output_offsets_; + + public: + // output shape + const TensorShape& OutputShape() const { + return output_shape_; + } + + // left and output matrices' first dim + int64_t M() const { + return M_; + } + + // right and output matrices' second dim + int64_t N() const { + return N_; + } + + // left matrices' second dim, and right matrices' first dim + int64_t K() const { + return K_; + } + + // Batched Gemm offsets in left matrices + const std::vector& LeftOffsets() const { + return left_offsets_; + } + + // Batched Gemm offsets in right matrices + const std::vector& RightOffsets() const { + return right_offsets_; + } + + // Batched Gemm offsets in output matrices + const std::vector& OutputOffsets() const { + return output_offsets_; + } + + template + static void OffsetToArrays(T* p, const std::vector& offsets, gsl::span arrays) { + auto len = offsets.size(); + ONNXRUNTIME_ENFORCE(arrays.size() == gsl::narrow_cast(len)); + for (size_t i = 0; i < len; i++) { + arrays[i] = p + offsets[i]; + } + } + + template + static void OffsetToArrays(const T* p, const std::vector& offsets, gsl::span arrays) { + auto len = offsets.size(); + ONNXRUNTIME_ENFORCE(arrays.size() == gsl::narrow_cast(len)); + for (size_t i = 0; i < len; i++) { + arrays[i] = p + offsets[i]; + } + } +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/math/softmax.cc b/onnxruntime/core/providers/cpu/math/softmax.cc new file mode 100644 index 0000000000000..61ad9bc71f06e --- /dev/null +++ b/onnxruntime/core/providers/cpu/math/softmax.cc @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/math/softmax.h" + +#include "core/framework/op_kernel.h" +#include "core/providers/common.h" +#include "core/providers/cpu/math/softmax_shared.h" +#include "core/util/math.h" + +namespace onnxruntime { + +template <> +Status Softmax::Compute(OpKernelContext* ctx) const { + const Tensor* tensor_pointer = ctx->Input(0); + if (tensor_pointer == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + const Tensor& X = *tensor_pointer; + const TensorShape input_shape{X.Shape()}; + + VLOGS(ctx->Logger(), 2) << "Input tensor shape: " << input_shape; + + Tensor* Y = ctx->Output(0, input_shape); + + const int64_t axis = HandleNegativeAxis(axis_, input_shape.NumDimensions()); + + size_t N = input_shape.SizeToDimension(axis); + size_t D = input_shape.SizeFromDimension(axis); + + float* Ydata = Y->template MutableData(); + + std::vector scale_(N); + std::vector rowmax_(N); + std::vector sum_multiplier_(D, 1.f); // initialize all multiplier values to 1.0 + + const bool logarithmic = false; + auto status = SoftmaxCPU(N, D, X.template Data(), Ydata, + scale_.data(), sum_multiplier_.data(), logarithmic, rowmax_.data()); + + return status; +} + +ONNX_CPU_OPERATOR_KERNEL( + Softmax, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Softmax); + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/math/softmax.h b/onnxruntime/core/providers/cpu/math/softmax.h new file mode 100644 index 0000000000000..2ac0a98f76e60 --- /dev/null +++ b/onnxruntime/core/providers/cpu/math/softmax.h @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "gsl/gsl_util" + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/util/math_cpuonly.h" + +namespace onnxruntime { +template +class Softmax final : public OpKernel { + public: + Softmax(const OpKernelInfo& info) : OpKernel{info}, axis_{1} { + int64_t axis; + Status status = info.GetAttr("axis", &axis); + + if (status.IsOK()) { + axis_ = gsl::narrow_cast(axis); + } + } + + Status Compute(OpKernelContext* context) const override; + + private: + int axis_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/math/softmax_shared.cc b/onnxruntime/core/providers/cpu/math/softmax_shared.cc new file mode 100644 index 0000000000000..32df249f362a0 --- /dev/null +++ b/onnxruntime/core/providers/cpu/math/softmax_shared.cc @@ -0,0 +1,90 @@ +/** +* Derived from caffe2, need copy right announcement here. +*/ + +/** +* Copyright (c) 2016-present, Facebook, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +// disable noisy warning about use of std::copy_n by gsl::copy. gsl_algorithm does the same thing +// but that's too late for the disable to work +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4996) +#endif +#include +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +#include "core/providers/cpu/math/softmax_shared.h" +#include "core/util/math.h" +#include "core/util/math_cpuonly.h" + +#include "gsl/gsl_algorithm" +#include "gsl/gsl_util" + +namespace onnxruntime { + +common::Status SoftmaxCPU(const int64_t N, + const int64_t D, + const float* Xdata, + float* Ydata, + float* scale, + const float* sum_multiplier, + bool logarithmic, + float* rowmax) { + // the Math functions SoftmaxCPU uses only support int32_t as input, so enforce that + if (N * D > INT32_MAX || N > INT32_MAX || D > INT32_MAX) { + std::ostringstream ss; + ss << "SoftmaxCPU inputs N, D and N * D must be < " << INT32_MAX << ". N=" << N << ", D=" << D; + std::string msg = ss.str(); + + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, msg); + } + + const int n = gsl::narrow_cast(N); + const int d = gsl::narrow_cast(D); + const int nd = gsl::narrow_cast(N * D); + + math::RowwiseMax(n, d, Xdata, rowmax, nullptr); + + // Put the intermediate result X - max(X) into Y by first copying X to Y, and then subtracting max from each entry + gsl::copy(gsl::make_span(Xdata, nd), gsl::make_span(Ydata, nd)); + + math::Gemm(CblasNoTrans, CblasNoTrans, n, d, 1, -1, rowmax, sum_multiplier, 1, Ydata, nullptr); + + // Exponentiation + math::Exp(nd, Ydata, Ydata, nullptr); + math::Gemv(CblasNoTrans, n, d, 1, Ydata, sum_multiplier, 0, scale, nullptr); + + // Do division + if (!logarithmic) { + for (int i = 0; i < N; ++i) { + for (int j = 0; j < D; ++j) { + Ydata[i * D + j] /= scale[i]; + } + } + } else { + for (int i = 0; i < N; ++i) { + for (int j = 0; j < D; ++j) { + Ydata[i * D + j] = Xdata[i * D + j] - rowmax[i] - log(fmaxf(scale[i], 1e-20f)); + } + } + } + + return Status::OK(); +} +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/math/softmax_shared.h b/onnxruntime/core/providers/cpu/math/softmax_shared.h new file mode 100644 index 0000000000000..ab107c8370a38 --- /dev/null +++ b/onnxruntime/core/providers/cpu/math/softmax_shared.h @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/status.h" + +namespace onnxruntime { +/** +Calculate Softmax using CPU memory. +@param N Number of rows +@param D Number of elements in each row +@param Xdata Source data +@param Ydata Output data +@param scale Storage for scale calculation. Size must be >= N. +@param sum_multiplier Weights for each element. Size must be >= D. +@param logarithmic If true, compute LogSoftmax. If false compute Softmax. +@param rowmax Storage for calculation of maximum in each row. Size must be >= N. +*/ +common::Status SoftmaxCPU(const int64_t N, + const int64_t D, + const float* Xdata, + float* Ydata, + float* scale, + const float* sum_multiplier, + bool logarithmic, + float* rowmax); +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/math/top_k.cc b/onnxruntime/core/providers/cpu/math/top_k.cc new file mode 100644 index 0000000000000..2bd5127523e74 --- /dev/null +++ b/onnxruntime/core/providers/cpu/math/top_k.cc @@ -0,0 +1,134 @@ +/** +* Copyright (c) 2016-present, Facebook, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#include "core/providers/cpu/math/top_k.h" +#include "core/common/common.h" +#include "core/common/exceptions.h" +#include "core/framework/op_kernel.h" +#include "core/framework/tensor.h" +#include "core/util/math_cpuonly.h" +#include +using namespace std; +namespace onnxruntime { +// spec https://github.com/onnx/onnx/blob/master/docs/Operators.md#TopK +ONNX_CPU_OPERATOR_KERNEL( + TopK, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()).TypeConstraint("I", DataTypeImpl::GetTensorType()), + TopK); + +static int64_t SizeToDim(size_t k, const vector& dims) { + ONNXRUNTIME_ENFORCE(k <= dims.size()); + int64_t r = 1; + for (size_t i = 0; i < k; ++i) { + r *= dims[i]; + } + return r; +} + +// Define these two names to allow lookup into the 2d tensors like +// mytensor(i, j) +template +using EigenMatrixMapRowMajor = Eigen::Map< + Eigen::Matrix>; + +template +using ConstEigenMatrixMapRowMajor = Eigen::Map< + const Eigen::Matrix>; + +template +struct ValueCmp { + bool operator()( + const pair& lhs, + const pair& rhs) { + return ( + lhs.first > rhs.first || + (lhs.first == rhs.first && lhs.second < rhs.second)); + } +}; + +template <> +Status TopK::Compute(OpKernelContext* p_op_kernel_context) const { + const Tensor* X = p_op_kernel_context->Input(0); + if (X == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + const vector& in_dims = X->Shape().GetDims(); + + // Linearize input tensor except for last dimension + // e.g. [3, 4, 5] -> [12, 5] + // [5] -> [5] + if (in_dims.back() < k_) { + ostringstream err_msg; + err_msg << "k argment [" << k_ << "] should not be greater than last dim [" << in_dims.back() << "]"; + return Status(common::ONNXRUNTIME, common::FAIL, err_msg.str()); + } + + vector linear_shape = {SizeToDim(in_dims.size() - 1, in_dims), + in_dims[in_dims.size() - 1]}; + auto input_map = ConstEigenMatrixMapRowMajor( + static_cast(X->template Data()), + linear_shape[0], + linear_shape[1]); + + // Resize output tensors to be the same shape as the linearized input except + // for the last dimension, which will be of size k. E.x. for an input tensor + // of shape [3, 4, 5] and k=2, both of these will be shape [3, 4, 2] + vector output_linear_shape = {linear_shape[0], k_}; + auto* Values = p_op_kernel_context->Output(0, output_linear_shape); + auto* Indices = p_op_kernel_context->Output(1, output_linear_shape); + + // Use Eigen maps to allow indexing into the 2d tensors like Values_map(i,j) + auto Values_map = EigenMatrixMapRowMajor( + Values->template MutableData(), linear_shape[0], k_); + auto Indices_map = EigenMatrixMapRowMajor( + Indices->template MutableData(), linear_shape[0], k_); + + // Sort preserving Indices + for (int64_t i = 0; i < linear_shape[0]; ++i) { + // Build a min-heap, the heap element is pair of (value, idx) + // the top of the heap is the smallest value + priority_queue< + pair, + vector>, + ValueCmp> + min_heap; + + // Maintain the size of heap to be less or equal to k_, so the + // heap will hold the k_ largest Values + for (int64_t j = 0; j < linear_shape[1]; ++j) { + const auto value = input_map(i, j); + if (min_heap.size() < k_ || value > min_heap.top().first) { + min_heap.push({value, j}); + } + if (min_heap.size() > k_) { + min_heap.pop(); + } + } + for (int64_t j = 0; j < k_; ++j) { + auto& pqElem = min_heap.top(); + Values_map(i, k_ - j - 1) = pqElem.first; + Indices_map(i, k_ - j - 1) = pqElem.second; + min_heap.pop(); + } + } + + // Reshape output tensors to [a_1, a_2, ..., a_n, k] + auto out_dims = in_dims; + out_dims[out_dims.size() - 1] = k_; + Values->Reshape(out_dims); + Indices->Reshape(out_dims); + return Status::OK(); +} +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/math/top_k.h b/onnxruntime/core/providers/cpu/math/top_k.h new file mode 100644 index 0000000000000..80d14b5e7a1e0 --- /dev/null +++ b/onnxruntime/core/providers/cpu/math/top_k.h @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/common/exceptions.h" +#include "core/framework/op_kernel.h" +#include "core/framework/tensor.h" +#include "core/util/math_cpuonly.h" +#include "gsl/gsl_util" + +namespace onnxruntime { +template +class TopK final : public OpKernel { + public: + TopK(const OpKernelInfo& op_kernel_info) : OpKernel(op_kernel_info) { + int64_t k_temp; + ONNXRUNTIME_ENFORCE(op_kernel_info.GetAttr("k", &k_temp).IsOK()); + ONNXRUNTIME_ENFORCE(k_temp > 0); + k_ = gsl::narrow_cast(k_temp); + + int64_t axis_temp; + ONNXRUNTIME_ENFORCE(op_kernel_info.GetAttr("axis", &axis_temp).IsOK()); + axis_ = gsl::narrow_cast(axis_temp); + } + + Status Compute(OpKernelContext* p_op_kernel_context) const override; + + private: + int axis_; + unsigned k_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/array_feature_extractor.cc b/onnxruntime/core/providers/cpu/ml/array_feature_extractor.cc new file mode 100644 index 0000000000000..eb79c5b5a62f7 --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/array_feature_extractor.cc @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/ml/array_feature_extractor.h" + +/** +https://github.com/onnx/onnx/blob/master/onnx/defs/traditionalml/defs.cc +ONNX_OPERATOR_SCHEMA(ArrayFeatureExtractor) + .SetDomain("ai.onnx.ml") + .SetDoc(R"DOC( + Select a subset of the data X based on the indices provided Y. +)DOC") + .Input(0, "X", "Data to be selected", "T") + .Input( + 1, + "Y", + "The index values to select as a int64 tensor", + "tensor(int64)") + .Output(0, "Z", "Selected output data as an array", "T") + .TypeConstraint( + "T", + {"tensor(float)", + "tensor(double)", + "tensor(int64)", + "tensor(int32)", + "tensor(string)"}, + "allowed types."); +*/ +using namespace ::onnxruntime::common; +using namespace std; +namespace onnxruntime { +namespace ml { +#define REG_ARRAYFEATUREEXTRACTOR(in_type) \ + ONNX_CPU_OPERATOR_TYPED_ML_KERNEL( \ + ArrayFeatureExtractor, \ + 1, \ + in_type, \ + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + ArrayFeatureExtractorOp); + +REG_ARRAYFEATUREEXTRACTOR(float); +REG_ARRAYFEATUREEXTRACTOR(double); +REG_ARRAYFEATUREEXTRACTOR(int32_t); +REG_ARRAYFEATUREEXTRACTOR(int64_t); +REG_ARRAYFEATUREEXTRACTOR(string); + +template +ArrayFeatureExtractorOp::ArrayFeatureExtractorOp(const OpKernelInfo& info) + : OpKernel(info) { +} + +template +common::Status ArrayFeatureExtractorOp::Compute(OpKernelContext* context) const { + const Tensor& X = *context->Input(0); + const TensorShape& x_shape = X.Shape(); + const vector& x_dims = x_shape.GetDims(); + const T* x_data = X.template Data(); + + if (x_dims.empty()) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid argument: X input has empty dimensions."); + } + + int64_t stride = x_dims.size() == 1 ? x_dims[0] : x_dims[1]; + int64_t N = x_dims.size() == 1 ? 1 : x_dims[0]; + + const Tensor& Y = *context->Input(1); + const TensorShape& y_shape = Y.Shape(); + const int64_t* y_data = Y.template Data(); + int64_t num_indices = y_shape.Size(); + + // validate Y + if (num_indices == 0) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid Y argument: num_indices = 0"); + } + + if (num_indices - 1 >= stride) { + std::ostringstream err_msg; + err_msg << "Invalid Y argument: num_indices - 1 (" << num_indices - 1 << ") >= stride (" << stride << ")"; + return Status(ONNXRUNTIME, INVALID_ARGUMENT, err_msg.str()); + } + + Tensor* Z = context->Output(0, TensorShape({N, num_indices})); + T* z_data = Z->template MutableData(); + + for (int64_t n = 0; n < N; ++n) { + for (int64_t j = 0; j < num_indices; ++j) { + *z_data++ = x_data[y_data[j]]; + } + x_data += stride; + } + + return Status::OK(); +} + +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/array_feature_extractor.h b/onnxruntime/core/providers/cpu/ml/array_feature_extractor.h new file mode 100644 index 0000000000000..dbd6811209191 --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/array_feature_extractor.h @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { +namespace ml { +template +class ArrayFeatureExtractorOp final : public OpKernel { + public: + explicit ArrayFeatureExtractorOp(const OpKernelInfo& info); + common::Status Compute(OpKernelContext* context) const override; +}; +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/binarizer.cc b/onnxruntime/core/providers/cpu/ml/binarizer.cc new file mode 100644 index 0000000000000..6306316ad523e --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/binarizer.cc @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/ml/binarizer.h" +#include +/** +https://github.com/onnx/onnx/blob/master/onnx/defs/traditionalml/defs.cc + +ONNX_OPERATOR_SCHEMA(Binarizer) +.SetDomain("ai.onnx.ml") +.SetDoc(R"DOC( +Makes values 1 or 0 based on a single threshold. +)DOC") +.Input(0, "X", "Data to be binarized", "T") +.Output(0, "Y", "Binarized output data", "T") +.TypeConstraint( +"T", +{ "tensor(float)", "tensor(double)", "tensor(int64)", "tensor(int32)" }, +" allowed types.") +.Attr( +"threshold", +"Values greater than this are set to 1, else set to 0", +AttributeProto::FLOAT, +OPTIONAL); +*/ + +namespace onnxruntime { +namespace ml { +ONNX_CPU_OPERATOR_ML_KERNEL( + Binarizer, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + BinarizerOp); + +template +BinarizerOp::BinarizerOp(const OpKernelInfo& info) + : OpKernel(info), threshold_(info.GetAttrOrDefault("threshold", 1.0f)) {} + +template +common::Status BinarizerOp::Compute(OpKernelContext* context) const { + const Tensor& X = *context->Input(0); + const TensorShape& x_shape = X.Shape(); + Tensor* Y = context->Output(0, x_shape); + const T* x_data = X.template Data(); + T* y_data = Y->template MutableData(); + size_t x_size = x_shape.Size(); + + common::Status status = common::Status::OK(); + for (size_t i = 0; i < x_size; ++i) { + T x_val = x_data[i]; + T& y_val = y_data[i]; + + float tmp = static_cast(x_val); // this cast is necessary because isnan doesn't work otherwise. + if (std::isnan(tmp)) { + return common::Status(common::ONNXRUNTIME, common::FAIL, "Input data with index: " + std::to_string(i) + " is NaN"); + } + y_val = x_val > threshold_ ? static_cast(1) : static_cast(0); + } + return status; +} +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/binarizer.h b/onnxruntime/core/providers/cpu/ml/binarizer.h new file mode 100644 index 0000000000000..a2a89800c06fd --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/binarizer.h @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { +namespace ml { +template +class BinarizerOp final : public OpKernel { + public: + explicit BinarizerOp(const OpKernelInfo& info); + common::Status Compute(OpKernelContext* context) const override; + + private: + const float threshold_; +}; +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/cast_map.cc b/onnxruntime/core/providers/cpu/ml/cast_map.cc new file mode 100644 index 0000000000000..f7cfef462665d --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/cast_map.cc @@ -0,0 +1,146 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/ml/cast_map.h" +#include +#include +using namespace ::onnxruntime::common; + +namespace { +template +TCastTo Cast(const TCastFrom& from); + +template <> +float Cast(const std::string& from) { + return std::stof(from); +} + +template <> +int64_t Cast(const std::string& from) { + return std::stoll(from); +} + +template <> +std::string Cast(const std::string& from) { + return from; +} + +template <> +float Cast(const float& from) { + return from; +} + +template <> +int64_t Cast(const float& from) { + return static_cast(from); +} + +template <> +std::string Cast(const float& from) { + return std::to_string(from); +} +} // namespace +namespace onnxruntime { +namespace ml { + +ONNX_CPU_OPERATOR_ML_KERNEL( + CastMap, + 1, + KernelDefBuilder().TypeConstraint("T1", + std::vector{DataTypeImpl::GetType>(), + DataTypeImpl::GetType>()}) + .TypeConstraint("T2", + std::vector{DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + CastMap); + +Status CastMap::Compute(OpKernelContext* context) const { + MLDataType input_type = context->InputType(0); + + // input map value is either string or float + bool float_input = false; + + if (input_type == DataTypeImpl::GetType>()) { + float_input = true; + } else if (input_type != DataTypeImpl::GetType>()) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid input type of value: ", + input_type, + " Expected std::map or std::map"); + } + + Status status; + switch (cast_to_) { + case (CAST_TO::TO_FLOAT): { + status = float_input ? ComputeImpl(*context, 0.f) + : ComputeImpl(*context, 0.f); + break; + } + case CAST_TO::TO_INT64: { + status = float_input ? ComputeImpl(*context, 0) + : ComputeImpl(*context, 0); + break; + } + case CAST_TO::TO_STRING: { + status = float_input ? ComputeImpl(*context, "0.f") + : ComputeImpl(*context, "0.f"); + break; + } + default: + return Status(ONNXRUNTIME, + INVALID_ARGUMENT, + ("Unexpected CAST_TO value of " + std::to_string(static_cast::type>(cast_to_)))); + } + + return status; +} + +template +Status CastMap::ComputeImpl(OpKernelContext& context, TTo pad_value) const { + using InputMap = std::map; + + const auto& X = *context.Input(0); + + int64_t num_dims = map_form_ == PACK_MAP::DENSE ? gsl::narrow_cast(X.size()) : max_map_; + + // create a span for the output + Tensor* Y = context.Output(0, TensorShape({1, num_dims})); + auto out = gsl::make_span(Y->template MutableData(), Y->Shape().Size()); + auto out_iter = out.begin(); + + // for each item in the entry, use the template specialized Cast function to convert + if (map_form_ == PACK_MAP::DENSE) { + // dense map is a straight copy + std::for_each(X.cbegin(), X.cend(), + [&out_iter](const typename InputMap::value_type& entry) { + *out_iter = Cast(entry.second); + ++out_iter; + }); + } else { + // sparse map puts pad_value in all entries that aren't present in the input, up to map_max_ + auto cur_input = X.cbegin(), end_input = X.cend(); + auto out_end = out.end(); + int64_t cur_idx = 0; + + ONNXRUNTIME_ENFORCE(cur_input == end_input || cur_input->first >= 0, + "Negative index values are not permitted. First entry in map has index value of ", cur_input->first); + + // for each output value, see if we have an input value, if not use the pad value + while (out_iter < out_end) { + if (cur_input != end_input && cur_input->first == cur_idx) { + *out_iter = Cast(cur_input->second); + ++cur_input; + } else { + *out_iter = pad_value; + } + + ++out_iter; + ++cur_idx; + } + } + + return Status::OK(); +} + +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/cast_map.h b/onnxruntime/core/providers/cpu/ml/cast_map.h new file mode 100644 index 0000000000000..008bf13fc92b8 --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/cast_map.h @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cpu/ml/ml_common.h" + +namespace onnxruntime { +namespace ml { + +class CastMap final : public OpKernel { + public: + CastMap(const OpKernelInfo& info) : OpKernel(info) { + std::string attr; + + ONNXRUNTIME_ENFORCE(info.GetAttr("cast_to", &attr).IsOK()); + cast_to_ = MakeCast(attr); + + ONNXRUNTIME_ENFORCE(info.GetAttr("map_form", &attr).IsOK()); + map_form_ = MakePack(attr); + + // ignore if not found as we fall back to the default of 1 + ONNXRUNTIME_ENFORCE(info.GetAttr("max_map", &max_map_).IsOK()); + + ONNXRUNTIME_ENFORCE(map_form_ != PACK_MAP::SPARSE || max_map_ > 0, "max_map must be > 0 if map_form is SPARSE"); + } + + Status Compute(OpKernelContext* context) const override; + + private: + template + Status ComputeImpl(OpKernelContext& ctx, TTo pad_value) const; + + CAST_TO cast_to_; + PACK_MAP map_form_; + + int64_t max_map_; +}; + +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/category_mapper.cc b/onnxruntime/core/providers/cpu/ml/category_mapper.cc new file mode 100644 index 0000000000000..819cad0e886cb --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/category_mapper.cc @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/ml/category_mapper.h" +#include +#include +using namespace ::onnxruntime::common; + +namespace onnxruntime { +namespace ml { + +ONNX_CPU_OPERATOR_ML_KERNEL( + CategoryMapper, + 1, + KernelDefBuilder().TypeConstraint("T1", + std::vector{DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}) + .TypeConstraint("T2", + std::vector{DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + CategoryMapper); + +Status CategoryMapper::Compute(OpKernelContext* context) const { + const Tensor* tensor_pointer = context->Input(0); + if (tensor_pointer == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + const Tensor& X = *tensor_pointer; + const TensorShape& shape = X.Shape(); + Tensor& Y = *context->Output(0, TensorShape(shape)); + + auto input_type = X.DataType(); + + if (input_type == DataTypeImpl::GetType()) { + if (Y.DataType() != DataTypeImpl::GetType()) + return Status(ONNXRUNTIME, FAIL, "Input of string must have output of int64"); + + auto input = gsl::make_span(X.template Data(), shape.Size()); + auto output = gsl::make_span(Y.template MutableData(), shape.Size()); + auto out = output.begin(); + + // map isn't going to change so get end() once instead of calling inside the for_each loop + const auto map_end = string_to_int_map_.end(); + + std::for_each(input.cbegin(), input.cend(), + [&out, &map_end, this](const std::string& value) { + auto map_to = string_to_int_map_.find(value); + *out = map_to == map_end ? default_int_ : map_to->second; + ++out; + }); + } else { + if (Y.DataType() != DataTypeImpl::GetType()) + return Status(ONNXRUNTIME, FAIL, "Input of int64 must have output of string "); + + auto input = gsl::make_span(X.template Data(), shape.Size()); + auto output = gsl::make_span(Y.template MutableData(), shape.Size()); + auto out = output.begin(); + + const auto map_end = int_to_string_map_.end(); + + std::for_each(input.cbegin(), input.cend(), + [&out, &map_end, this](const int64_t& value) { + auto map_to = int_to_string_map_.find(value); + *out = map_to == map_end ? default_string_ : map_to->second; + ++out; + }); + } + + return Status::OK(); +} + +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/category_mapper.h b/onnxruntime/core/providers/cpu/ml/category_mapper.h new file mode 100644 index 0000000000000..4693668f4e6bd --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/category_mapper.h @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cpu/ml/ml_common.h" + +namespace onnxruntime { +namespace ml { + +class CategoryMapper final : public OpKernel { + public: + CategoryMapper(const OpKernelInfo& info) : OpKernel(info) { + std::vector string_categories; + std::vector int_categories; + + ONNXRUNTIME_ENFORCE(info.GetAttrs("cats_strings", string_categories).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttrs("cats_int64s", int_categories).IsOK()); + + ONNXRUNTIME_ENFORCE(info.GetAttr("default_string", &default_string_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttr("default_int64", &default_int_).IsOK()); + + auto num_entries = string_categories.size(); + + ONNXRUNTIME_ENFORCE(num_entries == int_categories.size()); + + string_to_int_map_.reserve(num_entries); + int_to_string_map_.reserve(num_entries); + + for (size_t i = 0; i < num_entries; ++i) { + const std::string& str = string_categories[i]; + int64_t index = int_categories[i]; + + string_to_int_map_[str] = index; + int_to_string_map_[index] = str; + } + } + + Status Compute(OpKernelContext* context) const override; + + private: + std::unordered_map string_to_int_map_; + std::unordered_map int_to_string_map_; + + std::string default_string_; + int64_t default_int_; +}; + +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/dictvectorizer.cc b/onnxruntime/core/providers/cpu/ml/dictvectorizer.cc new file mode 100644 index 0000000000000..3e1d7464c0318 --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/dictvectorizer.cc @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/ml/dictvectorizer.h" + +using namespace std; + +namespace onnxruntime { +namespace ml { + +#define REG_NAMED_KERNEL(name, T1, T2) \ + ONNX_CPU_OPERATOR_TYPED_ML_KERNEL( \ + DictVectorizer, \ + 1, \ + name, \ + KernelDefBuilder().TypeConstraint("T1", DataTypeImpl::GetType>()).TypeConstraint("T2", DataTypeImpl::GetTensorType()), \ + DictVectorizerOp); + +#define REG_MY_KERNEL(T1, T2) REG_NAMED_KERNEL(T1##_##T2, T1, T2) + +#define REG_MY_STRING_KERNEL(T) REG_MY_KERNEL(string, T); +#define REG_MY_INT64_KERNEL(T) REG_MY_KERNEL(int64_t, T); + +REG_MY_STRING_KERNEL(int64_t); +REG_MY_STRING_KERNEL(float); +REG_MY_STRING_KERNEL(double); + +REG_MY_INT64_KERNEL(string); +REG_MY_INT64_KERNEL(float); +REG_MY_INT64_KERNEL(double); + +} // namespace ml + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/dictvectorizer.h b/onnxruntime/core/providers/cpu/ml/dictvectorizer.h new file mode 100644 index 0000000000000..1a852b90d25b3 --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/dictvectorizer.h @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { +namespace ml { +template +class DictVectorizerOp final : public OpKernel { + public: + DictVectorizerOp(const OpKernelInfo& info) : OpKernel(info) { + //In some stupid models, the vocabulary could have duplicated elements. + //We must support that, otherwise some tests will be break. + ONNXRUNTIME_ENFORCE(info.GetAttrs(std::is_same::value ? "string_vocabulary" : "int64_vocabulary", vocabulary_).IsOK()); + } + common::Status Compute(OpKernelContext* ctx) const override { + auto map = ctx->Input >(0); + auto Y = ctx->Output(0, TensorShape({1, static_cast(vocabulary_.size())})); + auto* y_data = Y->template MutableData(); + for (size_t i = 0, end = vocabulary_.size(); i < end; ++i) { + auto index = map->find(vocabulary_[i]); + if (index != map->end()) { + *y_data++ = index->second; + } else { + //Any keys not present in the input dictionary, will be zero in the output array + *y_data++ = TargetType(); + } + } + return Status::OK(); + } + + std::vector vocabulary_; +}; + +} // namespace ml + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/feature_vectorizer.cc b/onnxruntime/core/providers/cpu/ml/feature_vectorizer.cc new file mode 100644 index 0000000000000..95f99487f2ebe --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/feature_vectorizer.cc @@ -0,0 +1,131 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/ml/feature_vectorizer.h" + +#include + +namespace onnxruntime { +namespace ml { + +ONNX_CPU_OPERATOR_ML_KERNEL( + FeatureVectorizer, + 1, + KernelDefBuilder().TypeConstraint("T1", std::vector{ + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + FeatureVectorizer); + +template +static void VectorizeTensor(const Tensor& input_tensor, int64_t feature_size, int64_t sum_input_dimensions, + typename gsl::span::iterator out_iter); + +template +static void CopyWithCast(typename gsl::span::const_iterator begin, + typename gsl::span::const_iterator end, + gsl::span::iterator out_iter); + +Status FeatureVectorizer::Compute(OpKernelContext* context) const { + auto input_count = context->NumVariadicInputs(0); + ONNXRUNTIME_ENFORCE(input_count == input_dimensions_.size(), + "Number of inputs (", input_count, ") does not match number of inputdimensions values (", + input_dimensions_.size(), ")."); + + const Tensor* tensor_pointer = context->Input(0); + if (tensor_pointer == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + const Tensor& X = *tensor_pointer; + const auto& x_dims = X.Shape().GetDims(); + + // assumes all inputs have the same batch size + int64_t N = X.Shape().NumDimensions() == 1 ? 1 : x_dims[0]; + + // initialize all the output to 0.f + Tensor* Y = context->Output(0, TensorShape({N, total_dimensions_})); + auto Y_data = Y->template MutableData(); + + auto out = gsl::make_span(Y_data, Y->Shape().Size()); + + // init all to 0.f so we don't need to do that each loop if we have to add padding + std::fill_n(out.data(), out.size(), 0.f); + + int64_t feature_offset = 0; + + // for each feature, write out its data in one pass + for (int index = 0; index < input_count; ++index) { + const Tensor* input_tensor_ptr = context->Input(index); + ONNXRUNTIME_ENFORCE(input_tensor_ptr != nullptr); + auto& input_tensor = *input_tensor_ptr; + + auto feature_size = input_dimensions_[index]; + + auto data_type = input_tensor.DataType(); + auto cur_out = out.begin() + feature_offset; + + if (data_type == DataTypeImpl::GetType()) { + // straight copy for float to float + VectorizeTensor(input_tensor, feature_size, total_dimensions_, cur_out); + } else if (data_type == DataTypeImpl::GetType()) { + VectorizeTensor(input_tensor, feature_size, total_dimensions_, cur_out); + } else if (data_type == DataTypeImpl::GetType()) { + VectorizeTensor(input_tensor, feature_size, total_dimensions_, cur_out); + } else if (data_type == DataTypeImpl::GetType()) { + VectorizeTensor(input_tensor, feature_size, total_dimensions_, cur_out); + } else { + // should never happen. graph validation should have failed + ONNXRUNTIME_THROW("Invalid input type:", data_type); + } + + // move to start of next feature + feature_offset += feature_size; + } + + return Status::OK(); +} // namespace ml + +template +static void VectorizeTensor(const Tensor& input_tensor, int64_t feature_size, int64_t sum_input_dimensions, + typename gsl::span::iterator out_iter) { + auto& shape = input_tensor.Shape(); + auto& input_dims = shape.GetDims(); + + auto input_size = input_dims.size() == 1 ? input_dims[0] : input_tensor.Shape().SizeFromDimension(1); + auto N = input_dims.size() == 1 ? 1 : input_dims[0]; + + // if there's extra data, ignore it + auto stride = input_size; + if (input_size > feature_size) { + stride = feature_size; + } + + auto data = input_tensor.template Data(); + auto input = gsl::make_span(data, shape.Size()); + auto input_iter = input.cbegin(); + + for (int i = 0; i < N;) { + // copy each row to the output. iters are passed by value + CopyWithCast(input_iter, input_iter + stride, out_iter); + + // skip to start of next input row, and start of next output + // if we have more input. otherwise we go past then end of the input and the bounds checking errors out + if (++i < N) { + input_iter += input_size; + out_iter += sum_input_dimensions; + } + } +} + +template +static void CopyWithCast(typename gsl::span::const_iterator begin, + typename gsl::span::const_iterator end, + gsl::span::iterator out_iter) { + std::for_each(begin, end, + [&out_iter](const typename gsl::span::const_iterator::reference value) { + *out_iter = static_cast(value); + ++out_iter; + }); +} + +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/feature_vectorizer.h b/onnxruntime/core/providers/cpu/ml/feature_vectorizer.h new file mode 100644 index 0000000000000..6a5ce854daa62 --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/feature_vectorizer.h @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +namespace onnxruntime { +namespace ml { + +// https://github.com/onnx/onnx/blob/master/docs/Operators-ml.md#ai.onnx.ml.FeatureVectorizer +class FeatureVectorizer final : public OpKernel { + public: + FeatureVectorizer(const OpKernelInfo& info) : OpKernel(info) { + auto status = info.GetAttrs("inputdimensions", input_dimensions_); + ONNXRUNTIME_ENFORCE(status.IsOK() && !input_dimensions_.empty(), "inputdimensions attribute must be provided"); + + total_dimensions_ = std::accumulate(input_dimensions_.cbegin(), input_dimensions_.cend(), 0LL); + } + + Status Compute(OpKernelContext* context) const override; + + private: + std::vector input_dimensions_; + int64_t total_dimensions_; +}; + +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/imputer.cc b/onnxruntime/core/providers/cpu/ml/imputer.cc new file mode 100644 index 0000000000000..aaadd4e69b07a --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/imputer.cc @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/ml/imputer.h" +#include +/** +https://github.com/onnx/onnx/blob/master/onnx/defs/traditionalml/defs.cc +ONNX_OPERATOR_SCHEMA(Imputer) +.SetDomain("ai.onnx.ml") +.SetDoc(R"DOC( +Replace imputs that equal replaceValue/s with imputeValue/s. +All other inputs are copied to the output unchanged. +This op is used to replace missing values where we know what a missing value looks like. +Only one of imputed_value_floats or imputed_value_int64s should be used. +The size can be 1 element, which will be reused, or the size of the feature set F in input N,F +)DOC") +.Input(0, "X", "Data to be imputed", "T") +.Output(0, "Y", "Imputed output data", "T") +.TypeConstraint( +"T", +{"tensor(float)", "tensor(double)", "tensor(int64)", "tensor(int32)"}, +" allowed types.") +.Attr( +"imputed_value_floats", +"value to change to", +AttributeProto::FLOATS, +OPTIONAL) +.Attr( +"replaced_value_float", +"value that needs replacing", +AttributeProto::FLOAT, +0.f) +.Attr( +"imputed_value_int64s", +"value to change to", +AttributeProto::INTS, +OPTIONAL) +.Attr( +"replaced_value_int64", +"value that needs replacing", +AttributeProto::INT, +static_cast(0)); +*/ +using namespace ::onnxruntime::common; + +namespace onnxruntime { +namespace ml { + +ONNX_CPU_OPERATOR_ML_KERNEL( + Imputer, + 1, + KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + ImputerOp); + +ImputerOp::ImputerOp(const OpKernelInfo& info) : OpKernel(info), + imputed_values_float_(info.GetAttrsOrDefault("imputed_value_floats")), + imputed_values_int64_(info.GetAttrsOrDefault("imputed_value_int64s")) { + if (imputed_values_float_.size() && !info.GetAttr("replaced_value_float", &replaced_value_float_).IsOK()) + ONNXRUNTIME_THROW("Expected 'replaced_value_float' attribute since 'imputed_value_floats' is specified"); + if (imputed_values_int64_.size() && !info.GetAttr("replaced_value_int64", &replaced_value_int64_).IsOK()) + ONNXRUNTIME_THROW("Expected 'replace_value_int64' attribute since 'imputed_values_int64' is specified"); + ONNXRUNTIME_ENFORCE(imputed_values_float_.empty() ^ imputed_values_int64_.empty(), + "Must provide imputed_values_float_ or imputed_values_int64_ but not both."); +} + +template +common::Status ComputeByType(OpKernelContext* context, + T replaced_value, + const std::vector& imputed_values) { + if (imputed_values.empty()) { + return Status(ONNXRUNTIME, FAIL, "Empty value of imputed values."); + } + + const Tensor* tensor_pointer = context->Input(0); + if (tensor_pointer == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + const Tensor& X = *tensor_pointer; + const TensorShape& x_shape = X.Shape(); + auto& dims = x_shape.GetDims(); + if (dims.empty()) { + return Status(ONNXRUNTIME, FAIL, "Empty input dimensions."); + } + + const T* x_data = X.template Data(); + size_t x_size = x_shape.Size(); + int64_t stride = dims.size() == 1 ? dims[0] : dims[1]; + + Tensor* Y = context->Output(0, x_shape); + T* y_data = Y->template MutableData(); + if (imputed_values.size() == static_cast(stride)) { + for (size_t i = 0; i < x_size; i++) { + if (std::isnan(static_cast(x_data[i])) && std::isnan(static_cast(replaced_value))) { + y_data[i] = imputed_values[i % stride]; + } else if (x_data[i] == replaced_value) { + y_data[i] = imputed_values[i % stride]; + } else { + y_data[i] = x_data[i]; + } + } + } else { + for (size_t i = 0; i < x_size; i++) { + if (std::isnan(static_cast(x_data[i])) && std::isnan(static_cast(replaced_value))) { + y_data[i] = imputed_values[0]; + } else if (x_data[i] == replaced_value) { + y_data[i] = imputed_values[0]; + } else { + y_data[i] = x_data[i]; + } + } + } + + return Status::OK(); +} + +common::Status ImputerOp::Compute(OpKernelContext* context) const { + const Tensor* input_tensor_ptr = context->Input(0); + ONNXRUNTIME_ENFORCE(input_tensor_ptr != nullptr); + auto input_type = input_tensor_ptr->DataType(); + if (input_type == DataTypeImpl::GetType()) { + return ComputeByType(context, replaced_value_float_, imputed_values_float_); + } else if (input_type == DataTypeImpl::GetType()) { + return ComputeByType(context, replaced_value_int64_, imputed_values_int64_); + } else { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid type"); + } +} +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/imputer.h b/onnxruntime/core/providers/cpu/ml/imputer.h new file mode 100644 index 0000000000000..793304f491985 --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/imputer.h @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { +namespace ml { +class ImputerOp final : public OpKernel { + public: + explicit ImputerOp(const OpKernelInfo& info); + common::Status Compute(OpKernelContext* context) const override; + + private: + std::vector imputed_values_float_; + float replaced_value_float_; + std::vector imputed_values_int64_; + int64_t replaced_value_int64_; +}; +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/label_encoder.cc b/onnxruntime/core/providers/cpu/ml/label_encoder.cc new file mode 100644 index 0000000000000..cd4daf878189e --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/label_encoder.cc @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/ml/label_encoder.h" +#include +#include +using namespace ::onnxruntime::common; + +namespace onnxruntime { +namespace ml { + +ONNX_CPU_OPERATOR_ML_KERNEL( + LabelEncoder, + 1, + KernelDefBuilder().TypeConstraint("T1", + std::vector{DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}) + .TypeConstraint("T2", + std::vector{DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + LabelEncoder); + +Status LabelEncoder::Compute(OpKernelContext* context) const { + const Tensor* tensor_pointer = context->Input(0); + if (tensor_pointer == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + const Tensor& X = *tensor_pointer; + const TensorShape& shape = X.Shape(); + Tensor& Y = *context->Output(0, TensorShape(shape)); + + auto input_type = X.DataType(); + + if (input_type == DataTypeImpl::GetType()) { + if (Y.DataType() != DataTypeImpl::GetType()) + return Status(ONNXRUNTIME, FAIL, "Input of tensor(string) must have output of tensor(int64)"); + + auto input = gsl::make_span(X.template Data(), shape.Size()); + auto output = gsl::make_span(Y.template MutableData(), shape.Size()); + auto out = output.begin(); + + // map isn't going to change so get end() once instead of calling inside the for_each loop + const auto map_end = string_to_int_map_.end(); + + std::for_each(input.cbegin(), input.cend(), + [&out, &map_end, this](const std::string& value) { + auto map_to = string_to_int_map_.find(value); + *out = map_to == map_end ? default_int_ : map_to->second; + ++out; + }); + } else { + if (Y.DataType() != DataTypeImpl::GetType()) + return Status(ONNXRUNTIME, FAIL, "Input of tensor(int64) must have output of tensor(string)"); + + auto input = gsl::make_span(X.template Data(), shape.Size()); + auto output = gsl::make_span(Y.template MutableData(), shape.Size()); + auto out = output.begin(); + + const auto map_end = int_to_string_map_.end(); + + std::for_each(input.cbegin(), input.cend(), + [&out, &map_end, this](const int64_t& value) { + auto map_to = int_to_string_map_.find(value); + *out = map_to == map_end ? default_string_ : map_to->second; + ++out; + }); + } + + return Status::OK(); +} + +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/label_encoder.h b/onnxruntime/core/providers/cpu/ml/label_encoder.h new file mode 100644 index 0000000000000..1855462a6ea97 --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/label_encoder.h @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cpu/ml/ml_common.h" + +namespace onnxruntime { +namespace ml { + +class LabelEncoder final : public OpKernel { + public: + LabelEncoder(const OpKernelInfo& info) : OpKernel(info) { + std::vector string_classes; + + ONNXRUNTIME_ENFORCE(info.GetAttrs("classes_strings", string_classes).IsOK()); + + ONNXRUNTIME_ENFORCE(info.GetAttr("default_string", &default_string_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttr("default_int64", &default_int_).IsOK()); + + auto num_entries = string_classes.size(); + + string_to_int_map_.reserve(num_entries); + int_to_string_map_.reserve(num_entries); + + for (size_t i = 0; i < num_entries; ++i) { + const std::string& str = string_classes[i]; + + string_to_int_map_[str] = i; + int_to_string_map_[i] = str; + } + } + + Status Compute(OpKernelContext* context) const override; + + private: + std::unordered_map string_to_int_map_; + std::unordered_map int_to_string_map_; + + std::string default_string_; + int64_t default_int_; +}; + +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/linearclassifier.cc b/onnxruntime/core/providers/cpu/ml/linearclassifier.cc new file mode 100644 index 0000000000000..0beaf833ac8d2 --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/linearclassifier.cc @@ -0,0 +1,152 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/ml/linearclassifier.h" + +namespace onnxruntime { +namespace ml { + +const std::vector linearClassifierOutputConstraints{ + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}; + +ONNX_CPU_OPERATOR_TYPED_ML_KERNEL( + LinearClassifier, + 1, + float, + KernelDefBuilder().TypeConstraint("T1", DataTypeImpl::GetTensorType()).TypeConstraint("T2", linearClassifierOutputConstraints), + LinearClassifier); + +ONNX_CPU_OPERATOR_TYPED_ML_KERNEL( + LinearClassifier, + 1, + double, + KernelDefBuilder().TypeConstraint("T1", DataTypeImpl::GetTensorType()).TypeConstraint("T2", linearClassifierOutputConstraints), + LinearClassifier); + +ONNX_CPU_OPERATOR_TYPED_ML_KERNEL( + LinearClassifier, + 1, + int64_t, + KernelDefBuilder().TypeConstraint("T1", DataTypeImpl::GetTensorType()).TypeConstraint("T2", linearClassifierOutputConstraints), + LinearClassifier); + +ONNX_CPU_OPERATOR_TYPED_ML_KERNEL( + LinearClassifier, + 1, + int32_t, + KernelDefBuilder().TypeConstraint("T1", DataTypeImpl::GetTensorType()).TypeConstraint("T2", linearClassifierOutputConstraints), + LinearClassifier); + +template +LinearClassifier::LinearClassifier(const OpKernelInfo& info) : OpKernel(info), + multi_class_(info.GetAttrOrDefault("multi_class", 0)), + post_transform_(MakeTransform(info.GetAttrOrDefault("post_transform", "NONE"))), + intercepts_(info.GetAttrsOrDefault("intercepts")), + classlabels_strings_(info.GetAttrsOrDefault("classlabels_strings")), + classlabels_ints_(info.GetAttrsOrDefault("classlabels_ints")) { + if (!info.GetAttrs("coefficients", coefficients_).IsOK()) + ONNXRUNTIME_ENFORCE(!coefficients_.empty()); + + using_strings_ = !classlabels_strings_.empty(); + class_count_ = static_cast(intercepts_.size()); +} + +template +Status LinearClassifier::Compute(OpKernelContext* ctx) const { + const Tensor* X = ctx->Input(0); + const TensorShape& shape = X->Shape(); + if (shape.NumDimensions() == 0) { + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, + "Input shape needs to be at least a single dimension."); + } + + int64_t stride = shape.NumDimensions() == 1 ? shape[0] : shape[1]; + int64_t N = shape.NumDimensions() == 1 ? 1 : shape[0]; + Tensor* Y = ctx->Output(0, TensorShape({N})); + + int64_t output_classes = class_count_; + bool add_second_class = false; + if (intercepts_.size() == 1 && ((using_strings_ && classlabels_strings_.size() == 2) || (!using_strings_ && classlabels_ints_.size() == 2))) { + output_classes = 2; + add_second_class = true; + } + Tensor* Z = ctx->Output(1, TensorShape({N, output_classes})); + + int64_t zindex = 0; + const auto* x_data = X->template Data(); + + size_t class_count = static_cast(class_count_); + std::vector scores; + scores.reserve(class_count); + for (int64_t i = 0; i < N; i++) //for each point + { + scores.clear(); + size_t current_weight_0 = i * stride; + int maxclass = -1; + float maxweight = 0.f; + for (int j = 0; j < class_count; j++) //for each class + { + size_t current_coeff_0 = j * stride; + float weight = 0.f; + for (int64_t k = 0; k < stride; k++) //for each weight + { + weight += static_cast(x_data[current_weight_0 + k] * coefficients_[current_coeff_0 + k]); + } + if (intercepts_.size() == class_count) { + weight += intercepts_[j]; + } + scores.push_back(weight); + if (weight > maxweight || maxclass == -1) { + maxweight = weight; + maxclass = j; + } + } + //write top class + if (intercepts_.size() == 1) //binary + { + if (using_strings_) { + if (classlabels_strings_.size() == 2 && maxweight > 0) { + Y->template MutableData()[i] = classlabels_strings_[1]; //positive label + } else if (classlabels_strings_.size() == 2) { + Y->template MutableData()[i] = classlabels_strings_[0]; //negative label + } else if (maxweight > 0) { + Y->template MutableData()[i] = "1"; //positive label + } else { + Y->template MutableData()[i] = "0"; //negative label + } + } else //no strings + { + if (classlabels_ints_.size() == 2 && maxweight > 0) { + Y->template MutableData()[i] = classlabels_ints_[1]; //positive label + } else if (classlabels_ints_.size() == 2) { + Y->template MutableData()[i] = classlabels_ints_[0]; //negative label + } else if (maxweight > 0) { + Y->template MutableData()[i] = 1; //positive label + } else { + Y->template MutableData()[i] = 0; //negative label + } + } + } else //multiclass + { + if (using_strings_) { + Y->template MutableData()[i] = classlabels_strings_[maxclass]; + } else { + Y->template MutableData()[i] = classlabels_ints_[maxclass]; + } + } + //write float values + if (add_second_class && maxweight > 0) { + ::onnxruntime::ml::write_scores(scores, post_transform_, zindex, Z, 0); + } else if (add_second_class) { + ::onnxruntime::ml::write_scores(scores, post_transform_, zindex, Z, 1); + } else { + ::onnxruntime::ml::write_scores(scores, post_transform_, zindex, Z, -1); + } + zindex += scores.size(); + } //for each point + return Status::OK(); +} + +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/linearclassifier.h b/onnxruntime/core/providers/cpu/ml/linearclassifier.h new file mode 100644 index 0000000000000..53855435b31a5 --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/linearclassifier.h @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/util/math_cpuonly.h" +#include "ml_common.h" + +namespace onnxruntime { +namespace ml { + +template +class LinearClassifier final : public OpKernel { + public: + LinearClassifier(const OpKernelInfo& info); + Status Compute(OpKernelContext* context) const override; + + private: + int64_t multi_class_; + int64_t class_count_; + POST_EVAL_TRANSFORM post_transform_; + bool using_strings_; + std::vector coefficients_; + std::vector intercepts_; + std::vector classlabels_strings_; + std::vector classlabels_ints_; +}; + +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/linearregressor.cc b/onnxruntime/core/providers/cpu/ml/linearregressor.cc new file mode 100644 index 0000000000000..a3c97132170b6 --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/linearregressor.cc @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/ml/linearregressor.h" + +namespace onnxruntime { +namespace ml { + +ONNX_CPU_OPERATOR_ML_KERNEL( + LinearRegressor, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + LinearRegressor); + +template +LinearRegressor::LinearRegressor(const OpKernelInfo& info) : OpKernel(info), + intercepts_(info.GetAttrsOrDefault("intercepts")), + post_transform_(MakeTransform(info.GetAttrOrDefault("post_transform", "NONE"))) { + ONNXRUNTIME_ENFORCE(info.GetAttr("targets", &targets_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttrs("coefficients", coefficients_).IsOK()); +} + +template <> +Status LinearRegressor::Compute(OpKernelContext* ctx) const { + const Tensor* X = ctx->Input(0); + if (X->Shape().NumDimensions() == 0) { + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, + "Input shape needs to be at least a single dimension."); + } + + int64_t stride = X->Shape().NumDimensions() == 1 ? X->Shape()[0] : X->Shape()[1]; + int64_t N = X->Shape().NumDimensions() == 1 ? 1 : X->Shape()[0]; + Tensor* Y = ctx->Output(0, TensorShape({N, targets_})); + const auto* Xdata = X->template Data(); + int64_t yindex = 0; + + bool useIntercepts = intercepts_.size() == static_cast(targets_) ? true : false; + for (int64_t i = 0; i < N; i++) //for each point + { + std::vector scores; + int64_t current_weight_0 = i * stride; + for (int j = 0; j < targets_; j++) //for each target + { + int64_t current_coeff_0 = j * stride; + float weight = 0.f; + for (int64_t k = 0; k < stride; k++) //for each weight + { + weight = weight + Xdata[current_weight_0 + k] * coefficients_[current_coeff_0 + k]; + } + if (useIntercepts) { + weight = weight + intercepts_[j]; + } + scores.push_back(weight); + } + ::onnxruntime::ml::write_scores(scores, post_transform_, yindex, Y, -1); + yindex += scores.size(); + } + return Status::OK(); +} + +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/linearregressor.h b/onnxruntime/core/providers/cpu/ml/linearregressor.h new file mode 100644 index 0000000000000..50ea0e55fe783 --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/linearregressor.h @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/util/math_cpuonly.h" +#include "ml_common.h" + +namespace onnxruntime { +namespace ml { + +template +class LinearRegressor final : public OpKernel { + public: + LinearRegressor(const OpKernelInfo& info); + Status Compute(OpKernelContext* context) const override; + + private: + int64_t targets_; + std::vector coefficients_; + std::vector intercepts_; + POST_EVAL_TRANSFORM post_transform_; +}; + +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/ml_common.h b/onnxruntime/core/providers/cpu/ml/ml_common.h new file mode 100644 index 0000000000000..4154982405c24 --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/ml_common.h @@ -0,0 +1,328 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/util/math_cpuonly.h" + +namespace onnxruntime { +namespace ml { // name space for onnx.ml operators + +enum class OUTPUT_MODE { + TOPCLASS, + TOPCLASS_ANDSCORE, + ALL_SCORES +}; + +enum class NODE_MODE { + BRANCH_LEQ, + BRANCH_LT, + BRANCH_GTE, + BRANCH_GT, + BRANCH_EQ, + BRANCH_NEQ, + LEAF +}; + +static inline NODE_MODE MakeTreeNodeMode(const std::string& input) { + if (input == "BRANCH_LEQ") { + return NODE_MODE::BRANCH_LEQ; + } else if (input == "LEAF") { + return NODE_MODE::LEAF; + } else if (input == "BRANCH_LT") { + return NODE_MODE::BRANCH_LT; + } else if (input == "BRANCH_GTE") { + return NODE_MODE::BRANCH_GTE; + } else if (input == "BRANCH_GT") { + return NODE_MODE::BRANCH_GT; + } else if (input == "BRANCH_EQ") { + return NODE_MODE::BRANCH_EQ; + } else { + return NODE_MODE::BRANCH_NEQ; + } +} + +enum class POST_EVAL_TRANSFORM { + NONE, + LOGISTIC, + SOFTMAX, + SOFTMAX_ZERO, + PROBIT +}; + +static inline POST_EVAL_TRANSFORM MakeTransform(const std::string& input) { + if (input == "NONE") { + return POST_EVAL_TRANSFORM::NONE; + } else if (input == "LOGISTIC") { + return POST_EVAL_TRANSFORM::LOGISTIC; + } else if (input == "SOFTMAX") { + return POST_EVAL_TRANSFORM::SOFTMAX; + } else if (input == "SOFTMAX_ZERO") { + return POST_EVAL_TRANSFORM::SOFTMAX_ZERO; + } else { + return POST_EVAL_TRANSFORM::PROBIT; + } +} + +enum class AGGREGATE_FUNCTION { + AVERAGE, + SUM, + MIN, + MAX +}; + +static inline AGGREGATE_FUNCTION MakeAggregateFunction(const std::string& input) { + if (input == "AVERAGE") { + return AGGREGATE_FUNCTION::AVERAGE; + } else if (input == "SUM") { + return AGGREGATE_FUNCTION::SUM; + } else if (input == "MIN") { + return AGGREGATE_FUNCTION::MIN; + } else { + return AGGREGATE_FUNCTION::MAX; + } +} + +enum class CAST_TO { + TO_FLOAT, + TO_STRING, + TO_INT64 +}; + +static inline CAST_TO MakeCast(const std::string& input) { + if (input == "TO_FLOAT") { + return CAST_TO::TO_FLOAT; + } else if (input == "TO_STRING") { + return CAST_TO::TO_STRING; + } else if (input == "TO_INT64") { + return CAST_TO::TO_INT64; + } else { + ONNXRUNTIME_THROW("Invalid CAST_TO value of ", input, " Expected TO_FLOAT, TO_STRING or TO_INT64"); + } +} + +enum PACK_MAP { + DENSE, + SPARSE +}; + +static inline PACK_MAP MakePack(const std::string& input) { + if (input == "DENSE") { + return PACK_MAP::DENSE; + } else if (input == "SPARSE") { + return PACK_MAP::SPARSE; + } else { + ONNXRUNTIME_THROW("Invalid PACK_MAP value of ", input, " Expected DENSE or SPARSE"); + } +} + +enum KERNEL { + LINEAR, + POLY, + RBF, + SIGMOID +}; + +static inline KERNEL MakeKernel(const std::string& input) { + if (input == "LINEAR") { + return KERNEL::LINEAR; + } else if (input == "POLY") { + return KERNEL::POLY; + } else if (input == "RBF") { + return KERNEL::RBF; + } else { + return KERNEL::SIGMOID; + } +} + +enum NORMALIZE { + NMAX, + L1, + L2 +}; + +static inline NORMALIZE MakeNormalize(const std::string& input) { + if (input == "MAX") { + return NORMALIZE::NMAX; + } else if (input == "L1") { + return NORMALIZE::L1; + } else if (input == "L2") { + return NORMALIZE::L2; + } else { + ONNXRUNTIME_THROW("Invalid normalize value of ", input); + } +} + +enum class SVM_TYPE { + SVM_LINEAR, + SVM_SVC +}; + +static inline float ml_inv_erf(float x) { + float sgn = x < 0 ? -1.0f : 1.0f; + x = (1 - x) * (1 + x); + float log = std::log(x); + float v = 2 / (3.14159f * 0.147f) + 0.5f * log; + float v2 = 1 / (0.147f) * log; + float v3 = -v + std::sqrt(v * v - v2); + x = sgn * std::sqrt(v3); + return x; +} + +//https://www.csie.ntu.edu.tw/~cjlin/papers/svmprob/svmprob.pdf +static inline void multiclass_probability(int64_t classcount, const std::vector& r, std::vector& p) { + int64_t sized2 = classcount * classcount; + std::vector Q; + std::vector Qp; + for (int64_t k = 0; k < sized2; k++) { + Q.push_back(0); + } + for (int64_t k = 0; k < classcount; k++) { + Qp.push_back(0); + } + float eps = 0.005f / static_cast(classcount); + for (int64_t i = 0; i < classcount; i++) { + p[i] = 1.0f / static_cast(classcount); // Valid if k = 1 + for (int64_t j = 0; j < i; j++) { + Q[i * classcount + i] += r[j * classcount + i] * r[j * classcount + i]; + Q[i * classcount + j] = Q[j * classcount + i]; + } + for (int64_t j = i + 1; j < classcount; j++) { + Q[i * classcount + i] += r[j * classcount + i] * r[j * classcount + i]; + Q[i * classcount + j] = -r[j * classcount + i] * r[i * classcount + j]; + } + } + for (int64_t loop = 0; loop < 100; loop++) { + // stopping condition, recalculate QP,pQP for numerical accuracy + float pQp = 0; + for (int64_t i = 0; i < classcount; i++) { + Qp[i] = 0; + for (int64_t j = 0; j < classcount; j++) { + Qp[i] += Q[i * classcount + j] * p[j]; + } + pQp += p[i] * Qp[i]; + } + float max_error = 0; + for (int64_t i = 0; i < classcount; i++) { + float error = std::fabs(Qp[i] - pQp); + if (error > max_error) { + max_error = error; + } + } + if (max_error < eps) break; + + for (int64_t i = 0; i < classcount; i++) { + float diff = (-Qp[i] + pQp) / Q[i * classcount + i]; + p[i] += diff; + pQp = (pQp + diff * (diff * Q[i * classcount + i] + 2 * Qp[i])) / (1 + diff) / (1 + diff); + for (int64_t j = 0; j < classcount; j++) { + Qp[j] = (Qp[j] + diff * Q[i * classcount + j]) / (1 + diff); + p[j] /= (1 + diff); + } + } + } +} + +static inline float ml_logit(float val) { + float v = 1 / (1 + std::exp(-std::abs(val))); + return (val < 0) ? (1 - v) : v; +} + +static inline float sigmoid_probability(float score, float proba, float probb) { + float val = score * proba + probb; + return 1 - ml_logit(val); // ref: https://github.com/arnaudsj/libsvm/blob/eaaefac5ebd32d0e07902e1ae740e038eaaf0826/svm.cpp#L1818 +} + +static const float ml_sqrt2 = 1.41421356f; + +static inline void compute_softmax(std::vector& values) { + std::vector newscores; + // compute exp with negative number to be numerically stable + float v_max = -std::numeric_limits::max(); + for (float value : values) { + if (value > v_max) + v_max = value; + } + float this_sum = 0.f; + for (float value : values) { + float val2 = std::exp(value - v_max); + this_sum += val2; + newscores.push_back(val2); + } + for (int64_t k = 0; k < static_cast(values.size()); k++) { + values[k] = newscores[k] / this_sum; + } +} + +//this function skips zero values (since exp(0) is non zero) +static inline void compute_softmax_zero(std::vector& values) { + std::vector newscores; + // compute exp with negative number to be numerically stable + float v_max = -std::numeric_limits::max(); + for (float value : values) { + if (value > v_max) + v_max = value; + } + float exp_neg_v_max = std::exp(-v_max); + float this_sum = 0.f; + for (float value : values) { + if (value > 0.0000001f || value < -0.0000001f) { + float val2 = std::exp(value - v_max); + this_sum += val2; + newscores.push_back(val2); + } else { + newscores.push_back(value * exp_neg_v_max); + } + } + for (int64_t k = 0; k < static_cast(values.size()); k++) { + values[k] = newscores[k] / this_sum; + } +} + +static inline void write_scores(std::vector& scores, POST_EVAL_TRANSFORM post_transform, int64_t write_index, Tensor* Z, int add_second_class) { + if (post_transform == POST_EVAL_TRANSFORM::PROBIT && scores.size() == 1) { + scores[0] = ml_sqrt2 * ml_inv_erf(2 * scores[0] - 1); + Z->template MutableData()[write_index] = scores[0]; + } else if (scores.size() >= 2) { //multiclass + if (post_transform == POST_EVAL_TRANSFORM::LOGISTIC) { + for (float& score : scores) { + score = ml_logit(score); + } + } else if (post_transform == POST_EVAL_TRANSFORM::SOFTMAX) { + compute_softmax(scores); + } else if (post_transform == POST_EVAL_TRANSFORM::SOFTMAX_ZERO) { + compute_softmax_zero(scores); + } + } else { //binary case + if (add_second_class == 0 && scores.size() == 1) { //0=all positive weights, winning class is positive + scores.push_back(scores[0]); + scores[0] = 1.f - scores[0]; //put opposite score in positive slot + } else if (add_second_class == 1 && scores.size() == 1) { //1 = all positive weights, winning class is negative + scores.push_back(scores[0]); + scores[0] = 1.f - scores[0]; //put opposite score in positive slot + } else if (add_second_class == 2 && scores.size() == 1) { //2 = mixed weights, winning class is positive + if (post_transform == POST_EVAL_TRANSFORM::LOGISTIC) { + scores.push_back(ml_logit(scores[0])); //ml_logit(scores[k]); + scores[0] = ml_logit(-scores[0]); + } else { + scores.push_back(scores[0]); + scores[0] = -scores[0]; + } + } else if (add_second_class == 3 && scores.size() == 1) { //3 = mixed weights, winning class is negative + if (post_transform == POST_EVAL_TRANSFORM::LOGISTIC) { + scores.push_back(ml_logit(scores[0])); //ml_logit(scores[k]); + scores[0] = ml_logit(-scores[0]); + } else { + scores.push_back(-scores[0]); + } + } + } + for (float score : scores) { + Z->template MutableData()[write_index] = score; + write_index++; + } +} + +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/normalizer.cc b/onnxruntime/core/providers/cpu/ml/normalizer.cc new file mode 100644 index 0000000000000..211b61d1f7b52 --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/normalizer.cc @@ -0,0 +1,192 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/ml/normalizer.h" + +#include +#include "gsl/span" + +/* +ONNX_OPERATOR_SCHEMA(Normalizer) + .SetDomain("ai.onnx.ml") + .SetDoc(R"DOC( + Normalize the input. There are three normalization modes, + which have the corresponding formulas: + Max .. math:: max(x_i) + L1 .. math:: z = ||x||_1 = \sum_{i=1}^{n} |x_i| + L2 .. math:: z = ||x||_2 = \sqrt{\sum_{i=1}^{n} x_i^2} +)DOC") + .Input(0, "X", "Data to be encoded", "T") + .Output(0, "Y", "encoded output data", "tensor(float)") + .TypeConstraint( + "T", + {"tensor(float)", "tensor(double)", "tensor(int64)", "tensor(int32)"}, + " allowed types.") + .Attr( + "norm", + "enum 'MAX', 'L1', 'L2'", + AttributeProto::STRING, + std::string("MAX")); +*/ + +namespace onnxruntime { +namespace ml { + +ONNX_CPU_OPERATOR_ML_KERNEL( + Normalizer, + 1, + KernelDefBuilder().MayInplace(0, 0) // input is 4 or 8 byte, output is 4 byte + .TypeConstraint("T", std::vector{DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + Normalizer); + +Status Normalizer::Compute(OpKernelContext* context) const { + const Tensor* input_tensor_ptr = context->Input(0); + ONNXRUNTIME_ENFORCE(input_tensor_ptr != nullptr); + MLDataType input_type = input_tensor_ptr->DataType(); + + if (input_type == DataTypeImpl::GetType()) { + Normalize(context); + } else if (input_type == DataTypeImpl::GetType()) { + Normalize(context); + } else if (input_type == DataTypeImpl::GetType()) { + Normalize(context); + } else if (input_type == DataTypeImpl::GetType()) { + Normalize(context); + } else { + ONNXRUNTIME_THROW("Invalid input type of ", input_type); + } + + return Status::OK(); +} + +template +void NormalizeMax(const gsl::span& in, gsl::span& out, + int64_t offset, int64_t stride, int64_t increment_by) { + float max = std::numeric_limits::lowest(); + + for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) { + max = std::max(max, static_cast(in[i])); + } + + if (max != 0.f) { + for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) { + out[i] = static_cast(in[i]) / max; + } + } else { + for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) { + out[i] = static_cast(in[i]); + } + } +} + +template +void NormalizeL1(const gsl::span& in, gsl::span& out, + int64_t offset, int64_t stride, int64_t increment_by) { + float sum = 0.f; + + for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) { + sum += static_cast(std::abs(in[i])); + } + + if (sum != 0.f) { + for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) { + out[i] = static_cast(in[i]) / sum; + } + } else { + for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) { + out[i] = static_cast(in[i]); + } + } +} + +template +void NormalizeL2(const gsl::span& in, gsl::span& out, + int64_t offset, int64_t stride, int64_t increment_by) { + float sum = 0.f; + for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) { + auto x = in[i]; + auto x_sq = static_cast(x * x); + out[i] = x_sq; + sum += x_sq; + } + + if (sum != 0.f) { + for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) { + auto x = in[i]; + auto x_sq = out[i]; + + if (x < 0) + out[i] = std::sqrt(x_sq / sum) * -1; + else + out[i] = std::sqrt(x_sq / sum); + } + } else { + for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) { + out[i] = static_cast(in[i]); + } + } +} + +template +void Normalizer::Normalize(OpKernelContext* context) const { + const Tensor& X = *context->Input(0); + const TensorShape& x_shape = X.Shape(); + const auto data_size = x_shape.Size(); + const auto& x_dims = x_shape.GetDims(); + + Tensor* Y = context->Output(0, x_shape); + + auto input = gsl::make_span(X.template Data(), data_size); + auto output = gsl::make_span(Y->template MutableData(), data_size); + + int64_t stride = x_dims.size() == 1 ? x_dims[0] : x_dims[1]; + int64_t loops = data_size / stride; + + // we normalize on axis 1 so if there are more than 2 dimensions we need to increment the index + // by more than 1 as we process the stride + // for 1 and 2 dimension tensors we're normalizing across the row/s, so increment_by is 1 + // + // e.g. if you have a tensor of shape {2, 2, 3} + // [[[ 1, 2, 3], + // [ 4, 5, 6]], + // [[ 7, 8, 9], + // [10, 11, 12]]] + // + // we want to normalize (1, 4), (2, 5), (3, 6), + // (7, 10), (8, 11), (9, 12) + // so the stride would be 2, and the increment_by would be 3. + // + // we process a block of stride * increment_by entries before we need to skip to the next row of the 2nd dimension. + // the offset starts at 0 and increases by 1 each loop, for increment_by loops. + // the offset then jumps by stride * increment + + int64_t increment_by = x_dims.size() > 1 ? x_shape.SizeFromDimension(2) : 1; + + for (int64_t n = 0; n < loops; ++n) { + int64_t offset = (n % increment_by) + ((n / increment_by) * (stride * increment_by)); + + switch (normalization_) { + case NORMALIZE::NMAX: { + NormalizeMax(input, output, offset, stride, increment_by); + break; + } + case NORMALIZE::L1: { + NormalizeL1(input, output, offset, stride, increment_by); + break; + } + case NORMALIZE::L2: { + NormalizeL2(input, output, offset, stride, increment_by); + break; + } + default: { + ONNXRUNTIME_THROW("Unexpected NORMALIZE value of ", normalization_); + } + } + } +} + +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/normalizer.h b/onnxruntime/core/providers/cpu/ml/normalizer.h new file mode 100644 index 0000000000000..ae8b633e9b351 --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/normalizer.h @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cpu/ml/ml_common.h" + +#include "gsl/span" + +namespace onnxruntime { +namespace ml { + +class Normalizer final : public OpKernel { + public: + Normalizer(const OpKernelInfo& info) : OpKernel(info) { + std::string norm; + ONNXRUNTIME_ENFORCE(info.GetAttr("norm", &norm).IsOK()); + + normalization_ = MakeNormalize(norm); + } + + Status Compute(OpKernelContext* context) const override; + + private: + template + void Normalize(OpKernelContext* context) const; + + NORMALIZE normalization_; +}; + +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/onehotencoder.cc b/onnxruntime/core/providers/cpu/ml/onehotencoder.cc new file mode 100644 index 0000000000000..1e986915d1a5d --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/onehotencoder.cc @@ -0,0 +1,120 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/ml/onehotencoder.h" +/** +https://github.com/onnx/onnx/blob/master/onnx/defs/traditionalml/defs.cc +ONNX_OPERATOR_SCHEMA(OneHotEncoder) +.SetDomain("ai.onnx.ml") +.SetDoc(R"DOC( +Replace the inputs with an array of ones and zeros, where the only +one is the zero-based category that was passed in. The total category count +will determine the length of the vector. For example if we pass a +tensor with a single value of 4, and a category count of 8, the +output will be a tensor with 0,0,0,0,1,0,0,0 . + +This operator assumes every input in X is of the same category set +(meaning there is only one category count). + +If the input is a tensor of float, int32, or double, the data will be cast +to int64s and the cats_int64s category list will be used for the lookups. +)DOC") +.Input(0, "X", "Data to be encoded", "T") +.Output(0, "Y", "encoded output data", "tensor(float)") +.TypeConstraint("T", { "tensor(string)", "tensor(int64)","tensor(int32)", "tensor(float)","tensor(double)" }, " allowed types.") +.Attr("cats_int64s", "list of categories, ints", AttributeProto::INTS, OPTIONAL) +.Attr("cats_strings", "list of categories, strings", AttributeProto::STRINGS, OPTIONAL) +.Attr( +"zeros", +"if true and category is not present, will return all zeros, if false and missing category, operator will return false", +AttributeProto::INT, +OPTIONAL); +*/ +using namespace ::onnxruntime::common; +using namespace std; +namespace onnxruntime { +namespace ml { + +#define REG_KERNEL(TYPE) \ + ONNX_CPU_OPERATOR_TYPED_ML_KERNEL( \ + OneHotEncoder, \ + 1, \ + TYPE, \ + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + OneHotEncoderOp); + +REG_KERNEL(int64_t); +REG_KERNEL(float); +REG_KERNEL(double); +REG_KERNEL(string); + +template +OneHotEncoderOp::OneHotEncoderOp(const OpKernelInfo& info) : OpKernel(info), zeros_(info.GetAttrOrDefault("zeros", 1)), num_categories_(0) { + std::vector tmp_cats_int64s = info.GetAttrsOrDefault("cats_int64s"); + std::vector tmp_cats_strings = info.GetAttrsOrDefault("cats_strings"); + ONNXRUNTIME_ENFORCE(tmp_cats_int64s.empty() || tmp_cats_strings.empty()); + if (!tmp_cats_int64s.empty()) { + num_categories_ = tmp_cats_int64s.size(); + for (size_t idx = 0, end = tmp_cats_int64s.size(); idx < end; ++idx) { + cats_int64s_[tmp_cats_int64s[idx]] = idx; + } + } else { + num_categories_ = tmp_cats_strings.size(); + for (size_t idx = 0, end = tmp_cats_strings.size(); idx < end; ++idx) { + cats_strings_[tmp_cats_strings[idx]] = idx; + } + } + ONNXRUNTIME_ENFORCE(num_categories_ > 0); +} + +template +common::Status OneHotEncoderOp::Compute(OpKernelContext* context) const { + const Tensor* X = context->Input(0); + const TensorShape& input_shape = X->Shape(); + ONNXRUNTIME_ENFORCE(input_shape.NumDimensions() <= 2); + + std::vector output_shape(input_shape.GetDims()); + output_shape.push_back(num_categories_); + + Tensor* Y = context->Output(0, TensorShape(output_shape)); + auto y_data = Y->template MutableData(); + std::fill_n(y_data, Y->Shape().Size(), 0.0f); + + auto x_data = X->template Data(); + std::unordered_map::const_iterator idx; + for (int64_t i = 0; i < input_shape.Size(); ++i) { + auto int_idx = cats_int64s_.find(static_cast(x_data[i])); + if (int_idx != cats_int64s_.cend()) + y_data[i * num_categories_ + int_idx->second] = 1.0f; + else if (!zeros_) + return Status(ONNXRUNTIME, FAIL, "Unknown Category and zeros = 0."); + } + return Status::OK(); +} + +template <> +common::Status OneHotEncoderOp::Compute(OpKernelContext* context) const { + const Tensor* X = context->Input(0); + const TensorShape& input_shape = X->Shape(); + ONNXRUNTIME_ENFORCE(input_shape.NumDimensions() <= 2); + + std::vector output_shape(input_shape.GetDims()); + output_shape.push_back(num_categories_); + + Tensor* Y = context->Output(0, TensorShape(output_shape)); + auto y_data = Y->template MutableData(); + std::fill_n(y_data, Y->Shape().Size(), 0.0f); + + auto x_data = X->template Data(); + for (int64_t i = 0; i < input_shape.Size(); ++i) { + auto str_idx = cats_strings_.find(x_data[i]); + if (str_idx != cats_strings_.cend()) + y_data[i * num_categories_ + str_idx->second] = 1.0f; + else if (!zeros_) + return Status(ONNXRUNTIME, FAIL, "Unknown Category and zeros = 0."); + } + return Status::OK(); +} + +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/onehotencoder.h b/onnxruntime/core/providers/cpu/ml/onehotencoder.h new file mode 100644 index 0000000000000..93904bb0c66f5 --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/onehotencoder.h @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { +namespace ml { +template +class OneHotEncoderOp final : public OpKernel { + public: + explicit OneHotEncoderOp(const OpKernelInfo& info); + common::Status Compute(OpKernelContext* context) const override; + + private: + std::unordered_map cats_int64s_; + std::unordered_map cats_strings_; + int64_t zeros_; + int64_t num_categories_; +}; +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/scaler.cc b/onnxruntime/core/providers/cpu/ml/scaler.cc new file mode 100644 index 0000000000000..5139de339b887 --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/scaler.cc @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/ml/scaler.h" + +/** +https://github.com/onnx/onnx/blob/master/onnx/defs/traditionalml/defs.cc +ONNX_OPERATOR_SCHEMA(Scaler) +.SetDomain("ai.onnx.ml") +.SetDoc(R"DOC( + Rescale input data, for example to standardize features by removing the mean and scaling to unit variance. + )DOC") +.Input(0, "X", "Data to be scaled", "T") +.Output(0, "Y", "Scaled output data", "tensor(float)") +.TypeConstraint( + "T", + { "tensor(float)", "tensor(double)", "tensor(int64)", "tensor(int32)" }, + " allowed types.") +.Attr( + "scale", + "second, multiply by this, can be length of features or length 1", + AttributeProto::FLOATS, + OPTIONAL) +.Attr( + "offset", + "first, offset by this, must be same length as scale", + AttributeProto::FLOATS, + OPTIONAL); +*/ +using namespace ::onnxruntime::common; +using namespace std; +namespace onnxruntime { +namespace ml { + +ONNX_CPU_OPERATOR_TYPED_ML_KERNEL( + Scaler, + 1, + float, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()).MayInplace(0, 0), + ScalerOp); + +ONNX_CPU_OPERATOR_TYPED_ML_KERNEL( + Scaler, + 1, + double, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()).MayInplace(0, 0), + ScalerOp); + +ONNX_CPU_OPERATOR_TYPED_ML_KERNEL( + Scaler, + 1, + int64_t, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()).MayInplace(0, 0), + ScalerOp); + +ONNX_CPU_OPERATOR_TYPED_ML_KERNEL( + Scaler, + 1, + int32_t, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()).MayInplace(0, 0), + ScalerOp); + +template +ScalerOp::ScalerOp(const OpKernelInfo& info) : OpKernel(info), + scale_(info.GetAttrsOrDefault("scale")), + offset_(info.GetAttrsOrDefault("offset")) { + ONNXRUNTIME_ENFORCE(!scale_.empty(), "Empty scale in attributes"); + ONNXRUNTIME_ENFORCE(scale_.size() == offset_.size(), + "Scale size: (" + std::to_string(scale_.size()) + ") != (" + std::to_string(offset_.size()) + ")"); +} + +template +common::Status ScalerOp::Compute(OpKernelContext* context) const { + const Tensor& X = *context->Input(0); + const TensorShape& x_shape = X.Shape(); + Tensor* Y = context->Output(0, x_shape); + const T* x_data = X.template Data(); + float* y_data = Y->template MutableData(); + const vector& x_dims = x_shape.GetDims(); + if (x_dims.empty()) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid argument: input has empty dimensions."); + } + + size_t x_size = x_shape.Size(); + int64_t stride = x_dims.size() == 1 ? x_dims[0] : x_dims[1]; + if (static_cast(offset_.size()) == stride && + static_cast(scale_.size()) == stride) { + for (size_t i = 0; i < x_size; i++) { + y_data[i] = static_cast((x_data[i] - offset_[i % stride]) * scale_[i % stride]); + } + } else if (offset_.size() == 1 && scale_.size() == 1) { + for (size_t i = 0; i < x_size; i++) { + y_data[i] = static_cast((x_data[i] - offset_[0]) * scale_[0]); + } + } else { + std::ostringstream err_msg; + err_msg << "Either both scale and offset can be of feature size (" << stride << ") or 1"; + return Status(ONNXRUNTIME, INVALID_ARGUMENT, err_msg.str()); + } + return Status::OK(); +} +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/scaler.h b/onnxruntime/core/providers/cpu/ml/scaler.h new file mode 100644 index 0000000000000..76dd8b0eb6d37 --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/scaler.h @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { +namespace ml { +template +class ScalerOp final : public OpKernel { + public: + explicit ScalerOp(const OpKernelInfo& info); + common::Status Compute(OpKernelContext* context) const override; + + private: + std::vector scale_; + std::vector offset_; +}; +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/svmclassifier.cc b/onnxruntime/core/providers/cpu/ml/svmclassifier.cc new file mode 100644 index 0000000000000..a740d989ae078 --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/svmclassifier.cc @@ -0,0 +1,259 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/ml/svmclassifier.h" + +namespace onnxruntime { +namespace ml { + +#define ADD_IN_TYPE_SVM_CLASSIFIER_OP(in_type) \ + ONNX_CPU_OPERATOR_TYPED_ML_KERNEL( \ + SVMClassifier, \ + 1, \ + in_type, \ + KernelDefBuilder().TypeConstraint("T1", DataTypeImpl::GetTensorType()).TypeConstraint("T2", {DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}), \ + SVMClassifier); + +ADD_IN_TYPE_SVM_CLASSIFIER_OP(float); +ADD_IN_TYPE_SVM_CLASSIFIER_OP(double); +ADD_IN_TYPE_SVM_CLASSIFIER_OP(int64_t); +ADD_IN_TYPE_SVM_CLASSIFIER_OP(int32_t); + +template +SVMClassifier::SVMClassifier(const OpKernelInfo& info) + : OpKernel(info), + SVMCommon(info), + vectors_per_class_(info.GetAttrsOrDefault("vectors_per_class")), + proba_(info.GetAttrsOrDefault("prob_a")), + probb_(info.GetAttrsOrDefault("prob_b")), + support_vectors_(info.GetAttrsOrDefault("support_vectors")), + post_transform_(MakeTransform(info.GetAttrOrDefault("post_transform", "NONE"))) { + ONNXRUNTIME_ENFORCE(info.GetAttrs("rho", rho_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttrs("coefficients", coefficients_).IsOK()); + + // prob_a and prob_b are optional for Z output + ONNXRUNTIME_ENFORCE(proba_.size() == probb_.size()); + + // one of these should be valid + ONNXRUNTIME_ENFORCE(info.GetAttrs("classlabels_strings", classlabels_strings_).IsOK() || + info.GetAttrs("classlabels_ints", classlabels_ints_).IsOK()); + + vector_count_ = 0; + feature_count_ = 0; + class_count_ = 0; + for (int64_t i = 0; i < static_cast(vectors_per_class_.size()); i++) { + starting_vector_.push_back(vector_count_); + vector_count_ += vectors_per_class_[i]; + } + + using_strings_ = false; + if (classlabels_strings_.size() > 0) { + using_strings_ = true; + class_count_ = classlabels_strings_.size(); + } else if (classlabels_ints_.size() > 0) { + class_count_ = classlabels_ints_.size(); + } else { + class_count_ = 1; + } + if (vector_count_ > 0) { + feature_count_ = support_vectors_.size() / vector_count_; //length of each support vector + mode_ = SVM_TYPE::SVM_SVC; + } else { + feature_count_ = coefficients_.size() / class_count_; //liblinear mode + mode_ = SVM_TYPE::SVM_LINEAR; + set_kernel_type(KERNEL::LINEAR); + } + ONNXRUNTIME_ENFORCE(classlabels_strings_.size() > 0 || classlabels_ints_.size() > 0); + ONNXRUNTIME_ENFORCE(proba_.size() == probb_.size()); + ONNXRUNTIME_ENFORCE(coefficients_.size() > 0); + weights_are_all_positive_ = true; + for (int64_t i = 0; i < static_cast(coefficients_.size()); i++) { + if (coefficients_[i] < 0) { + weights_are_all_positive_ = false; + break; + } + } +} + +template +Status SVMClassifier::Compute(OpKernelContext* ctx) const { + const Tensor* X = ctx->Input(0); + + int64_t stride = X->Shape().NumDimensions() == 1 ? X->Shape()[0] : X->Shape()[1]; + int64_t N = X->Shape().NumDimensions() == 1 ? 1 : X->Shape()[0]; + + Tensor* Y = ctx->Output(0, TensorShape({N})); + Tensor* Z; + + std::vector dims; + if (mode_ == SVM_TYPE::SVM_SVC && proba_.size() == 0) + dims = {static_cast(N), static_cast(class_count_ * (class_count_ - 1) / 2)}; + else + dims = {static_cast(N), static_cast(class_count_)}; + Z = ctx->Output(1, TensorShape(dims)); + + const auto* x_data = X->template Data(); + int64_t zindex = 0; + + for (int64_t n = 0; n < N; n++) //for each example + { + int64_t current_weight_0 = n * stride; + int64_t maxclass = -1; + double maxweight = 0.f; + std::vector decisions; + std::vector scores; + std::vector kernels; + std::vector votes; + + if (mode_ == SVM_TYPE::SVM_SVC) { + for (int64_t j = 0; j < vector_count_; j++) { + float val = kernel_dot(x_data, current_weight_0, support_vectors_, feature_count_ * j, feature_count_, get_kernel_type()); + kernels.push_back(val); + } + for (int64_t j = 0; j < class_count_; j++) { + votes.push_back(0); + } + int evals = 0; + for (int64_t i = 0; i < class_count_; i++) { //for each class + for (int64_t j = i + 1; j < class_count_; j++) { //for each class + float sum = 0; + int64_t start_index_i = starting_vector_[i]; // *feature_count_; + int64_t start_index_j = starting_vector_[j]; // *feature_count_; + + int64_t class_i_support_count = vectors_per_class_[i]; + int64_t class_j_support_count = vectors_per_class_[j]; + + int64_t pos1 = (vector_count_) * (j - 1); + int64_t pos2 = (vector_count_) * (i); + for (int64_t m = 0; m < class_i_support_count; m++) { + float val1 = coefficients_[pos1 + start_index_i + m]; + float val2 = kernels[start_index_i + m]; + sum += val1 * val2; + } + for (int64_t m = 0; m < class_j_support_count; m++) { + float val1 = coefficients_[pos2 + start_index_j + m]; + float val2 = kernels[start_index_j + m]; + sum += val1 * val2; + } + + sum += rho_[evals]; + scores.push_back(sum); + if (sum > 0) { + votes[i]++; + } else { + votes[j]++; + } + evals++; //index into rho + } + } + } else if (mode_ == SVM_TYPE::SVM_LINEAR) { //liblinear + for (int64_t j = 0; j < class_count_; j++) { //for each class + float val = kernel_dot(x_data, current_weight_0, coefficients_, feature_count_ * j, feature_count_, get_kernel_type()); + val += rho_[0]; + scores.push_back(val); + } + } + if (proba_.size() > 0 && mode_ == SVM_TYPE::SVM_SVC) { + //compute probabilities from the scores + std::vector estimates; + std::vector probsp2; + int64_t num = class_count_ * class_count_; + for (int64_t m = 0; m < num; m++) { + probsp2.push_back(0.f); //min prob + } + for (int64_t m = 0; m < class_count_; m++) { + estimates.push_back(0.f); //min prob + } + int64_t index = 0; + for (int64_t i = 0; i < class_count_; i++) { + for (int64_t j = i + 1; j < class_count_; j++) { + float val1 = sigmoid_probability(scores[index], proba_[index], probb_[index]); + float val2 = std::max(val1, 1.0e-7f); + probsp2[i * class_count_ + j] = std::min(val2, 1 - 1.0e-7f); + probsp2[j * class_count_ + i] = 1 - probsp2[i * class_count_ + j]; + index++; + } + } + multiclass_probability(class_count_, probsp2, estimates); + //copy probabilities back into scores + scores.resize(estimates.size()); + for (int64_t k = 0; k < static_cast(estimates.size()); k++) { + scores[k] = estimates[k]; + } + } + int64_t maxvotes = 0; + if (votes.size() > 0) { + for (int64_t k = 0; k < static_cast(votes.size()); k++) { + if (votes[k] > maxvotes) { + maxvotes = votes[k]; + maxclass = k; + } + } + } else { + for (int64_t k = 0; k < static_cast(scores.size()); k++) { + if (scores[k] > maxweight) { + maxclass = k; + maxweight = scores[k]; + } + } + } + //write top class + int write_additional_scores = -1; + if (rho_.size() == 1) //binary + { + if (using_strings_) { + if (classlabels_strings_.size() == 2 && weights_are_all_positive_ && maxweight >= 0.5 && proba_.size() == 0) { + Y->template MutableData()[n] = classlabels_strings_[1]; //positive label + write_additional_scores = 0; + } else if (classlabels_strings_.size() == 2 && maxweight > 0 && !weights_are_all_positive_ && proba_.size() == 0) { + Y->template MutableData()[n] = classlabels_strings_[1]; //positive label + write_additional_scores = 0; + } else if (classlabels_strings_.size() == 2 && proba_.size() > 0) { //this case all classes are in their rightful spot + Y->template MutableData()[n] = classlabels_strings_[maxclass]; //whichever label + write_additional_scores = -1; + } else if (classlabels_strings_.size() == 2) { + Y->template MutableData()[n] = classlabels_strings_[0]; //negative label + write_additional_scores = 1; + } else if (maxweight > 0) { + Y->template MutableData()[n] = "1"; //positive label + } else { + Y->template MutableData()[n] = "0"; //negative label + } + } else //no strings + { + if (classlabels_ints_.size() == 2 && weights_are_all_positive_ && maxweight >= 0.5 && proba_.size() == 0) { + Y->template MutableData()[n] = classlabels_ints_[1]; //positive label + write_additional_scores = 0; + } else if (classlabels_ints_.size() == 2 && maxweight > 0 && !weights_are_all_positive_ && proba_.size() == 0) { + Y->template MutableData()[n] = classlabels_ints_[0]; //pos label + write_additional_scores = 0; + } else if (classlabels_ints_.size() == 2 && proba_.size() > 0) //this case all classes are in their rightful spot + { + Y->template MutableData()[n] = classlabels_ints_[maxclass]; //whichever label + write_additional_scores = -1; + } else if (classlabels_ints_.size() == 2) { + Y->template MutableData()[n] = classlabels_ints_[0]; //negative label + write_additional_scores = 1; + } else if (maxweight > 0) { + Y->template MutableData()[n] = 1; //positive label + } else { + Y->template MutableData()[n] = 0; //negative label + } + } + } else { //multiclass + if (using_strings_) { + Y->template MutableData()[n] = classlabels_strings_[maxclass]; + } else { + Y->template MutableData()[n] = classlabels_ints_[maxclass]; + } + } + + write_scores(scores, post_transform_, zindex, Z, write_additional_scores); + zindex += scores.size(); + } + + return Status::OK(); +} + +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/svmclassifier.h b/onnxruntime/core/providers/cpu/ml/svmclassifier.h new file mode 100644 index 0000000000000..f51a6be5d75f0 --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/svmclassifier.h @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/util/math_cpuonly.h" +#include "ml_common.h" + +namespace onnxruntime { +namespace ml { + +// stuffs shared by SVMClassifier and SVMRegressor +template +class SVMCommon { + protected: + SVMCommon(const OpKernelInfo& info) : kernel_type_(MakeKernel(info.GetAttrOrDefault("kernel_type", "LINEAR"))) { + std::vector kernel_params; + ONNXRUNTIME_ENFORCE(info.GetAttrs("kernel_params", kernel_params).IsOK()); + + if (kernel_params.size() > 0) { + gamma_ = kernel_params[0]; + coef0_ = kernel_params[1]; + degree_ = kernel_params[2]; + } + } + + void set_kernel_type(KERNEL new_kernel_type) { kernel_type_ = new_kernel_type; } + KERNEL get_kernel_type() const { return kernel_type_; } + + float kernel_dot(const T* A, int64_t a, const std::vector& B, int64_t b, int64_t len, KERNEL k) const { + float sum = 0.f; + if (k == KERNEL::POLY) { + for (int64_t i = 0; i < len; i++) { + sum += B[b + i] * static_cast(A[a + i]); + } + sum = gamma_ * sum + coef0_; + sum = std::pow(sum, degree_); + } else if (k == KERNEL::SIGMOID) { + for (int64_t i = 0; i < len; i++) { + sum += B[b + i] * static_cast(A[a + i]); + } + sum = gamma_ * sum + coef0_; + sum = std::tanh(sum); + } else if (k == KERNEL::RBF) { + for (int64_t i = 0; i < len; i++) { + float val = static_cast(A[a + i]) - B[b + i]; + sum += (val * val); + } + sum = std::exp(-gamma_ * sum); + } else if (k == KERNEL::LINEAR) { + for (int64_t i = 0; i < len; i++) { + sum += B[b + i] * static_cast(A[a + i]); + } + } + return sum; + } + + private: + KERNEL kernel_type_; + float gamma_; + float coef0_; + float degree_; +}; + +template +class SVMClassifier final : public OpKernel, private SVMCommon { + using SVMCommon::kernel_dot; + using SVMCommon::set_kernel_type; + using SVMCommon::get_kernel_type; + + public: + SVMClassifier(const OpKernelInfo& info); + Status Compute(OpKernelContext* context) const override; + + private: + bool weights_are_all_positive_; + int64_t feature_count_; + int64_t class_count_; + int64_t vector_count_; + bool using_strings_; + std::vector vectors_per_class_; + std::vector starting_vector_; + std::vector rho_; + std::vector proba_; + std::vector probb_; + std::vector coefficients_; + std::vector support_vectors_; + std::vector classlabels_ints_; + std::vector classlabels_strings_; + POST_EVAL_TRANSFORM post_transform_; + SVM_TYPE mode_; //how are we computing SVM? 0=LibSVC, 1=LibLinear +}; + +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/svmregressor.cc b/onnxruntime/core/providers/cpu/ml/svmregressor.cc new file mode 100644 index 0000000000000..2a3c47c5e3134 --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/svmregressor.cc @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/ml/svmregressor.h" + +namespace onnxruntime { +namespace ml { + +ONNX_CPU_OPERATOR_ML_KERNEL( + SVMRegressor, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + SVMRegressor); + +template +SVMRegressor::SVMRegressor(const OpKernelInfo& info) + : OpKernel(info), + SVMCommon(info), + vector_count_(info.GetAttrOrDefault("n_supports", 0)), + support_vectors_(info.GetAttrsOrDefault("support_vectors")), + post_transform_(MakeTransform(info.GetAttrOrDefault("post_transform", "NONE"))) { + ONNXRUNTIME_ENFORCE(info.GetAttrs("rho", rho_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttrs("coefficients", coefficients_).IsOK()); + ONNXRUNTIME_ENFORCE(coefficients_.size() > 0); + + int64_t onec = info.GetAttrOrDefault("one_class", 0); + one_class_ = (onec != 0); + + if (vector_count_ > 0) { + feature_count_ = support_vectors_.size() / vector_count_; //length of each support vector + mode_ = SVM_TYPE::SVM_SVC; + } else { + feature_count_ = coefficients_.size(); + mode_ = SVM_TYPE::SVM_LINEAR; + set_kernel_type(KERNEL::LINEAR); + } +} + +template +Status SVMRegressor::Compute(OpKernelContext* ctx) const { + const Tensor* X = ctx->Input(0); + + int64_t stride = X->Shape().NumDimensions() == 1 ? X->Shape()[0] : X->Shape()[1]; + int64_t N = X->Shape().NumDimensions() == 1 ? 1 : X->Shape()[0]; + + Tensor* Y = ctx->Output(0, TensorShape({N, 1})); // this op outputs for one target only + const auto* x_data = X->template Data(); + + for (int64_t n = 0; n < N; n++) { //for each example + int64_t current_weight_0 = n * stride; + std::vector scores; + + float sum = 0.f; + if (mode_ == SVM_TYPE::SVM_SVC) { + for (int64_t j = 0; j < vector_count_; j++) { + float val1 = kernel_dot(x_data, current_weight_0, support_vectors_, feature_count_ * j, feature_count_, get_kernel_type()); + float val2 = coefficients_[j]; + float val3 = val1 * val2; + sum += val3; + } + sum += rho_[0]; + } else if (mode_ == SVM_TYPE::SVM_LINEAR) { //liblinear + sum = kernel_dot(x_data, current_weight_0, coefficients_, 0, feature_count_, get_kernel_type()); + sum += rho_[0]; + } + if (one_class_ && sum > 0) { + Y->template MutableData()[n] = 1.f; + } else if (one_class_) { + Y->template MutableData()[n] = -1.f; + } else { + Y->template MutableData()[n] = sum; + } + } + + return Status::OK(); +} + +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/svmregressor.h b/onnxruntime/core/providers/cpu/ml/svmregressor.h new file mode 100644 index 0000000000000..37d5a81bc154e --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/svmregressor.h @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/util/math_cpuonly.h" +#include "ml_common.h" +#include "svmclassifier.h" + +namespace onnxruntime { +namespace ml { + +template +class SVMRegressor final : public OpKernel, private SVMCommon { + using SVMCommon::kernel_dot; + using SVMCommon::set_kernel_type; + using SVMCommon::get_kernel_type; + + public: + SVMRegressor(const OpKernelInfo& info); + Status Compute(OpKernelContext* context) const override; + + private: + bool one_class_; + int64_t feature_count_; + int64_t vector_count_; + std::vector rho_; + std::vector coefficients_; + std::vector support_vectors_; + POST_EVAL_TRANSFORM post_transform_; + SVM_TYPE mode_; //how are we computing SVM? 0=LibSVC, 1=LibLinear +}; + +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/tree_ensemble_classifier.cc b/onnxruntime/core/providers/cpu/ml/tree_ensemble_classifier.cc new file mode 100644 index 0000000000000..cc26bdcc8d1b5 --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/tree_ensemble_classifier.cc @@ -0,0 +1,513 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/ml/tree_ensemble_classifier.h" + +/** +https://github.com/onnx/onnx/blob/master/onnx/defs/traditionalml/defs.cc +ONNX_OPERATOR_SCHEMA(TreeEnsembleClassifier) +.SetDomain("ai.onnx.ml") +.SetDoc(R"DOC( +Tree Ensemble classifier. Returns the top class for each input in N. +All args with nodes_ are fields of a tuple of tree nodes, and +it is assumed they are the same length, and an index i will decode the +tuple across these inputs. Each node id can appear only once +for each tree id. +All fields prefixed with class_ are tuples of votes at the leaves. +A leaf may have multiple votes, where each vote is weighted by +the associated class_weights index. +It is expected that either classlabels_strings or classlabels_int64s +will be passed and the class_ids are an index into this list. +Mode enum is BRANCH_LEQ, BRANCH_LT, BRANCH_GTE, BRANCH_GT, BRANCH_EQ, BRANCH_NEQ, LEAF +)DOC") +.Input(0, "X", "Input N,F", "T1") +.Output(0, "Y", "N, Top class for each point", "T2") +.Output( +1, +"Z", +"N,E the class score for each class, for each point", +"tensor(float)") +.TypeConstraint( +"T1", +{"tensor(float)", "tensor(double)", "tensor(int64)", "tensor(int32)"}, +" allowed types.") +.TypeConstraint( +"T2", +{"tensor(string)", "tensor(int64)"}, +" allowed types.") +.Attr( +"nodes_treeids", +"tree id for this node", +AttributeProto::INTS, +OPTIONAL) +.Attr( +"nodes_nodeids", +"node id for this node, node ids may restart at zero for each tree (but not required).", +AttributeProto::INTS, +OPTIONAL) +.Attr( +"nodes_featureids", +"feature id for this node", +AttributeProto::INTS, +OPTIONAL) +.Attr( +"nodes_values", +"thresholds to do the splitting on for this node.", +AttributeProto::FLOATS, +OPTIONAL) +.Attr("nodes_hitrates", "", AttributeProto::FLOATS, OPTIONAL) +.Attr( +"nodes_modes", +"enum of behavior for this node 'BRANCH_LEQ', 'BRANCH_LT', 'BRANCH_GTE', 'BRANCH_GT', 'BRANCH_EQ', 'BRANCH_NEQ', 'LEAF'", +AttributeProto::STRINGS, +OPTIONAL) +.Attr( +"nodes_truenodeids", +"child node if expression is true", +AttributeProto::INTS, +OPTIONAL) +.Attr( +"nodes_falsenodeids", +"child node if expression is false", +AttributeProto::INTS, +OPTIONAL) +.Attr( +"nodes_missing_value_tracks_true", +"for each node, decide if the value is missing (nan) then use true branch, this field can be left unset and will assume false for all nodes", +AttributeProto::INTS, +OPTIONAL) +.Attr( +"class_treeids", +"tree that this node is in", +AttributeProto::INTS, +OPTIONAL) +.Attr( +"class_nodeids", +"node id that this weight is for", +AttributeProto::INTS, +OPTIONAL) +.Attr( +"class_ids", +"index of the class list that this weight is for", +AttributeProto::INTS, +OPTIONAL) +.Attr( +"class_weights", +"the weight for the class in class_id", +AttributeProto::FLOATS, +OPTIONAL) +.Attr( +"classlabels_strings", +"class labels if using string labels", +AttributeProto::STRINGS, +OPTIONAL) +.Attr( +"classlabels_int64s", +"class labels if using int labels", +AttributeProto::INTS, +OPTIONAL) +.Attr( +"post_transform", +"post eval transform for score, enum NONE, SOFTMAX, LOGISTIC, SOFTMAX_ZERO, PROBIT", +AttributeProto::STRING, +std::string("NONE")) +.Attr( +"base_values", +"base values for classification, added to final class score, size must be the same as classes or can be left unassigned (assumed 0)", +AttributeProto::FLOATS, +OPTIONAL); +*/ +using namespace ::onnxruntime::common; +using namespace std; +namespace onnxruntime { +namespace ml { + +#define ADD_IN_TYPE_TREE_ENSEMBLE_CLASSIFIER_OP(in_type) \ + ONNX_CPU_OPERATOR_TYPED_ML_KERNEL( \ + TreeEnsembleClassifier, \ + 1, \ + in_type, \ + KernelDefBuilder().TypeConstraint("T1", DataTypeImpl::GetTensorType()).TypeConstraint("T2", {DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}), \ + TreeEnsembleClassifier); + +ADD_IN_TYPE_TREE_ENSEMBLE_CLASSIFIER_OP(float); +ADD_IN_TYPE_TREE_ENSEMBLE_CLASSIFIER_OP(double); +ADD_IN_TYPE_TREE_ENSEMBLE_CLASSIFIER_OP(int64_t); +ADD_IN_TYPE_TREE_ENSEMBLE_CLASSIFIER_OP(int32_t); + +template +TreeEnsembleClassifier::TreeEnsembleClassifier(const OpKernelInfo& info) + : OpKernel(info), + nodes_treeids_(info.GetAttrsOrDefault("nodes_treeids")), + nodes_nodeids_(info.GetAttrsOrDefault("nodes_nodeids")), + nodes_featureids_(info.GetAttrsOrDefault("nodes_featureids")), + nodes_values_(info.GetAttrsOrDefault("nodes_values")), + nodes_hitrates_(info.GetAttrsOrDefault("nodes_hitrates")), + nodes_modes_names_(info.GetAttrsOrDefault("nodes_modes")), + nodes_truenodeids_(info.GetAttrsOrDefault("nodes_truenodeids")), + nodes_falsenodeids_(info.GetAttrsOrDefault("nodes_falsenodeids")), + missing_tracks_true_(info.GetAttrsOrDefault("nodes_missing_value_tracks_true")), + class_nodeids_(info.GetAttrsOrDefault("class_nodeids")), + class_treeids_(info.GetAttrsOrDefault("class_treeids")), + class_ids_(info.GetAttrsOrDefault("class_ids")), + class_weights_(info.GetAttrsOrDefault("class_weights")), + base_values_(info.GetAttrsOrDefault("base_values")), + classlabels_strings_(info.GetAttrsOrDefault("classlabels_strings")), + classlabels_int64s_(info.GetAttrsOrDefault("classlabels_int64s")), + post_transform_(MakeTransform(info.GetAttrOrDefault("post_transform", "NONE"))) { + ONNXRUNTIME_ENFORCE(!nodes_treeids_.empty()); + ONNXRUNTIME_ENFORCE(class_nodeids_.size() == class_ids_.size()); + ONNXRUNTIME_ENFORCE(class_nodeids_.size() == class_weights_.size()); + ONNXRUNTIME_ENFORCE(nodes_nodeids_.size() == nodes_featureids_.size()); + ONNXRUNTIME_ENFORCE(nodes_nodeids_.size() == nodes_modes_names_.size()); + ONNXRUNTIME_ENFORCE(nodes_nodeids_.size() == nodes_values_.size()); + ONNXRUNTIME_ENFORCE(nodes_nodeids_.size() == nodes_truenodeids_.size()); + ONNXRUNTIME_ENFORCE(nodes_nodeids_.size() == nodes_falsenodeids_.size()); + ONNXRUNTIME_ENFORCE((nodes_nodeids_.size() == nodes_hitrates_.size()) || (nodes_hitrates_.empty())); + + ONNXRUNTIME_ENFORCE(classlabels_strings_.empty() ^ classlabels_int64s_.empty(), + "Must provide classlabels_strings or classlabels_int64s but not both."); + + // in the absence of bool type supported by GetAttrs this ensure that we don't have any negative + // values so that we can check for the truth condition without worrying about negative values. + ONNXRUNTIME_ENFORCE(std::all_of( + std::begin(missing_tracks_true_), + std::end(missing_tracks_true_), [](int64_t elem) { return elem >= 0; })); + + Initialize(); +} + +template +void TreeEnsembleClassifier::Initialize() { + int64_t current_tree_id = 1234567891L; + std::vector tree_offsets; + weights_are_all_positive_ = true; + + for (int64_t i = 0, size_node_treeids = static_cast(nodes_treeids_.size()); + i < size_node_treeids; + ++i) { + if (nodes_treeids_[i] != current_tree_id) { + tree_offsets.push_back(nodes_nodeids_[i]); + current_tree_id = nodes_treeids_[i]; + } + int64_t offset = tree_offsets[tree_offsets.size() - 1]; + nodes_nodeids_[i] = nodes_nodeids_[i] - offset; + if (nodes_falsenodeids_[i] >= 0) { + nodes_falsenodeids_[i] = nodes_falsenodeids_[i] - offset; + } + if (nodes_truenodeids_[i] >= 0) { + nodes_truenodeids_[i] = nodes_truenodeids_[i] - offset; + } + } + for (int64_t i = 0, size_class_nodeids = static_cast(class_nodeids_.size()); + i < size_class_nodeids; + ++i) { + int64_t offset = tree_offsets[class_treeids_[i]]; + class_nodeids_[i] = class_nodeids_[i] - offset; + if (class_weights_[i] < 0) { + weights_are_all_positive_ = false; + } + } + + nodes_modes_.reserve(nodes_modes_names_.size()); + for (size_t i = 0, end = nodes_modes_names_.size(); i < end; ++i) { + nodes_modes_.push_back(MakeTreeNodeMode(nodes_modes_names_[i])); + } + + // leafnode data, these are the votes that leaves do + for (size_t i = 0, end = class_nodeids_.size(); i < end; ++i) { + leafnodedata_.push_back(std::make_tuple(class_treeids_[i], class_nodeids_[i], class_ids_[i], class_weights_[i])); + weights_classes_.insert(class_ids_[i]); + } + std::sort(std::begin(leafnodedata_), std::end(leafnodedata_), [](auto const& t1, auto const& t2) { + if (std::get<0>(t1) != std::get<0>(t2)) + return std::get<0>(t1) < std::get<0>(t2); + else + return std::get<1>(t1) < std::get<1>(t2); + }); + // make an index so we can find the leafnode data quickly when evaluating + int64_t field0 = -1; + int64_t field1 = -1; + for (size_t i = 0, end = leafnodedata_.size(); i < end; ++i) { + int64_t id0 = std::get<0>(leafnodedata_[i]); + int64_t id1 = std::get<1>(leafnodedata_[i]); + if (id0 != field0 || id1 != field1) { + int64_t id = id0 * kOffset_ + id1; + int64_t position = static_cast(i); + auto p3 = std::make_pair(id, position); + leafdata_map_.insert(p3); + field0 = id; + field1 = position; + } + } + + // treenode ids, some are roots_, and roots_ have no parents + std::unordered_map parents; // holds count of all who point to you + std::unordered_map indices; + // add all the nodes to a map, and the ones that have parents are not roots_ + std::unordered_map::iterator it; + for (size_t i = 0, end = nodes_treeids_.size(); i < end; ++i) { + // make an index to look up later + int64_t id = nodes_treeids_[i] * kOffset_ + nodes_nodeids_[i]; + int64_t position = static_cast(i); + auto p3 = std::make_pair(id, position); + indices.insert(p3); + it = parents.find(id); + if (it == parents.end()) { + // start counter at 0 + int64_t b = (int64_t)0L; + auto p1 = std::make_pair(id, b); + parents.insert(p1); + } + } + // all true nodes arent roots_ + for (size_t i = 0, end = nodes_truenodeids_.size(); i < end; ++i) { + if (nodes_modes_[i] == NODE_MODE::LEAF) continue; + // they must be in the same tree + int64_t id = nodes_treeids_[i] * kOffset_ + nodes_truenodeids_[i]; + it = parents.find(id); + ONNXRUNTIME_ENFORCE(it != parents.end()); + it->second++; + } + // all false nodes arent roots_ + for (size_t i = 0, end = nodes_falsenodeids_.size(); i < end; ++i) { + if (nodes_modes_[i] == NODE_MODE::LEAF) continue; + // they must be in the same tree + int64_t id = nodes_treeids_[i] * kOffset_ + nodes_falsenodeids_[i]; + it = parents.find(id); + ONNXRUNTIME_ENFORCE(it != parents.end()); + it->second++; + } + // find all the nodes that dont have other nodes pointing at them + for (auto& parent : parents) { + if (parent.second == 0) { + int64_t id = parent.first; + it = indices.find(id); + roots_.push_back(it->second); + } + } + class_count_ = !classlabels_strings_.empty() ? classlabels_strings_.size() : classlabels_int64s_.size(); + using_strings_ = !classlabels_strings_.empty(); + ONNXRUNTIME_ENFORCE(base_values_.empty() || + base_values_.size() == static_cast(class_count_) || + base_values_.size() == weights_classes_.size()); +} + +template +common::Status TreeEnsembleClassifier::Compute(OpKernelContext* context) const { + const Tensor& X = *context->Input(0); + const TensorShape& x_shape = X.Shape(); + vector x_dims = x_shape.GetDims(); + if (x_dims.empty()) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "X dims is empty."); + } + + int64_t stride = x_dims.size() == 1 ? x_dims[0] : x_dims[1]; // TODO(task 495): how does this work in the case of 3D tensors? + int64_t N = x_dims.size() == 1 ? 1 : x_dims[0]; + Tensor* Y = context->Output(0, TensorShape({N})); + auto* Z = context->Output(1, TensorShape({N, class_count_})); + + int64_t zindex = 0; + const T* x_data = X.template Data(); + + // for each class + std::vector scores; + scores.reserve(class_count_); + for (int64_t i = 0; i < N; ++i) { + scores.clear(); + int64_t current_weight_0 = i * stride; + std::map classes; + // fill in base values, this might be empty but that is ok + for (int64_t k = 0, end = static_cast(base_values_.size()); k < end; ++k) { + auto p1 = std::make_pair(k, base_values_[k]); + classes.insert(p1); + } + // walk each tree from its root + for (size_t j = 0, end = roots_.size(); j < end; ++j) { + ONNXRUNTIME_RETURN_IF_ERROR(ProcessTreeNode(classes, roots_[j], x_data, current_weight_0)); + } + float maxweight = 0.f; + int64_t maxclass = -1; + // write top class + int write_additional_scores = -1; + if (class_count_ > 2) { + for (auto& classe : classes) { + if (maxclass == -1 || classe.second > maxweight) { + maxclass = classe.first; + maxweight = classe.second; + } + } + if (using_strings_) { + Y->template MutableData()[i] = classlabels_strings_[maxclass]; + } else { + Y->template MutableData()[i] = classlabels_int64s_[maxclass]; + } + } else // binary case + { + maxweight = classes.size() > 0 ? classes[0] : 0.f; // only 1 class + if (using_strings_) { + auto* y_data = Y->template MutableData(); + if (classlabels_strings_.size() == 2 && + weights_are_all_positive_ && + maxweight > 0.5 && + weights_classes_.size() == 1) { + y_data[i] = classlabels_strings_[1]; // positive label + write_additional_scores = 0; + } else if (classlabels_strings_.size() == 2 && + weights_are_all_positive_ && + maxweight <= 0.5 && + weights_classes_.size() == 1) { + y_data[i] = classlabels_strings_[0]; // negative label + write_additional_scores = 1; + } else if (classlabels_strings_.size() == 2 && + maxweight > 0 && + !weights_are_all_positive_ && weights_classes_.size() == 1) { + y_data[i] = classlabels_strings_[1]; // pos label + write_additional_scores = 2; + } else if (classlabels_strings_.size() == 2 && + maxweight <= 0 && + !weights_are_all_positive_ && + weights_classes_.size() == 1) { + y_data[i] = classlabels_strings_[0]; // neg label + write_additional_scores = 3; + } else if (maxweight > 0) { + y_data[i] = "1"; // positive label + } else { + y_data[i] = "0"; // negative label + } + } else { + auto* y_data = Y->template MutableData(); + if (classlabels_int64s_.size() == 2 && + weights_are_all_positive_ && + maxweight > 0.5 && + weights_classes_.size() == 1) { + y_data[i] = classlabels_int64s_[1]; // positive label + write_additional_scores = 0; + } else if (classlabels_int64s_.size() == 2 && + weights_are_all_positive_ && + maxweight <= 0.5 && + weights_classes_.size() == 1) { + y_data[i] = classlabels_int64s_[0]; // negative label + write_additional_scores = 1; + } else if (classlabels_int64s_.size() == 2 && + maxweight > 0 && + !weights_are_all_positive_ && + weights_classes_.size() == 1) { + y_data[i] = classlabels_int64s_[1]; // pos label + write_additional_scores = 2; + } else if (classlabels_int64s_.size() == 2 && + maxweight <= 0 && + !weights_are_all_positive_ && + weights_classes_.size() == 1) { + y_data[i] = classlabels_int64s_[0]; // neg label + write_additional_scores = 3; + } else if (maxweight > 0) { + y_data[i] = 1; // positive label + } else { + y_data[i] = 0; // negative label + } + } + } + // write float values, might not have all the classes in the output yet + // for example a 10 class case where we only found 2 classes in the leaves + if (weights_classes_.size() == static_cast(class_count_)) { + for (int64_t k = 0; k < class_count_; ++k) { + auto it_classes = classes.find(k); + if (it_classes != classes.end()) { + scores.push_back(it_classes->second); + } else { + scores.push_back(0.f); + } + } + } else { + for (auto& classe : classes) { + scores.push_back(classe.second); + } + } + write_scores(scores, post_transform_, zindex, Z, write_additional_scores); + zindex += scores.size(); + } // for every batch + return Status::OK(); +} + +template +common::Status TreeEnsembleClassifier::ProcessTreeNode(std::map& classes, + int64_t treeindex, + const T* x_data, + int64_t feature_base) const { + // walk down tree to the leaf + NODE_MODE mode = static_cast(nodes_modes_[treeindex]); + int64_t loopcount = 0; + int64_t root = treeindex; + while (mode != NODE_MODE::LEAF) { + T val = x_data[feature_base + nodes_featureids_[treeindex]]; + bool tracktrue = true; + if (missing_tracks_true_.size() != nodes_truenodeids_.size()) { + tracktrue = false; + } else { + tracktrue = missing_tracks_true_[treeindex] && std::isnan(static_cast(val)); + } + float threshold = nodes_values_[treeindex]; + switch (mode) { + case NODE_MODE::BRANCH_LEQ: + treeindex = val <= threshold || tracktrue ? nodes_truenodeids_[treeindex] : nodes_falsenodeids_[treeindex]; + break; + case NODE_MODE::BRANCH_LT: + treeindex = val < threshold || tracktrue ? nodes_truenodeids_[treeindex] : nodes_falsenodeids_[treeindex]; + break; + case NODE_MODE::BRANCH_GTE: + treeindex = val >= threshold || tracktrue ? nodes_truenodeids_[treeindex] : nodes_falsenodeids_[treeindex]; + break; + case NODE_MODE::BRANCH_GT: + treeindex = val > threshold || tracktrue ? nodes_truenodeids_[treeindex] : nodes_falsenodeids_[treeindex]; + break; + case NODE_MODE::BRANCH_EQ: + treeindex = val == threshold || tracktrue ? nodes_truenodeids_[treeindex] : nodes_falsenodeids_[treeindex]; + break; + case NODE_MODE::BRANCH_NEQ: + treeindex = val != threshold || tracktrue ? nodes_truenodeids_[treeindex] : nodes_falsenodeids_[treeindex]; + break; + default: { + std::ostringstream err_msg; + err_msg << "Invalid mode of value: " << static_cast::type>(mode); + return Status(ONNXRUNTIME, INVALID_ARGUMENT, err_msg.str()); + } + } + ONNXRUNTIME_ENFORCE(treeindex >= 0); + treeindex = treeindex + root; + mode = static_cast(nodes_modes_[treeindex]); + loopcount++; + if (loopcount > kMaxTreeDepth_) break; + } + // should be at leaf + int64_t id = nodes_treeids_[treeindex] * kOffset_ + nodes_nodeids_[treeindex]; + auto it_lp = leafdata_map_.find(id); + if (it_lp == leafdata_map_.end()) { // if not found, simply return + return Status::OK(); + } + int64_t index = it_lp->second; + int64_t treeid = std::get<0>(leafnodedata_[index]); + int64_t nodeid = std::get<1>(leafnodedata_[index]); + while (treeid == nodes_treeids_[treeindex] && nodeid == nodes_nodeids_[treeindex]) { + int64_t classid = std::get<2>(leafnodedata_[index]); + float weight = std::get<3>(leafnodedata_[index]); + std::map::iterator it_classes; + it_classes = classes.find(classid); + if (it_classes != classes.end()) { + it_classes->second += weight; + } else { + auto p1 = std::make_pair(classid, weight); + classes.insert(p1); + } + ++index; + // some tree node will be last + if (index >= static_cast(leafnodedata_.size())) { + break; + } + treeid = std::get<0>(leafnodedata_[index]); + nodeid = std::get<1>(leafnodedata_[index]); + } + return Status::OK(); +} +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/tree_ensemble_classifier.h b/onnxruntime/core/providers/cpu/ml/tree_ensemble_classifier.h new file mode 100644 index 0000000000000..34f3b2d2f635a --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/tree_ensemble_classifier.h @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "ml_common.h" + +namespace onnxruntime { +namespace ml { +template +class TreeEnsembleClassifier final : public OpKernel { + public: + explicit TreeEnsembleClassifier(const OpKernelInfo& info); + common::Status Compute(OpKernelContext* context) const override; + + private: + void Initialize(); + common::Status ProcessTreeNode(std::map& classes, + int64_t treeindex, + const T* x_data, + int64_t feature_base) const; + + std::vector nodes_treeids_; + std::vector nodes_nodeids_; + std::vector nodes_featureids_; + std::vector nodes_values_; + std::vector nodes_hitrates_; + std::vector nodes_modes_names_; + std::vector nodes_modes_; + std::vector nodes_truenodeids_; + std::vector nodes_falsenodeids_; + std::vector missing_tracks_true_; // no bool type + + std::vector class_nodeids_; + std::vector class_treeids_; + std::vector class_ids_; + std::vector class_weights_; + int64_t class_count_; + std::set weights_classes_; + + std::vector base_values_; + std::vector classlabels_strings_; + std::vector classlabels_int64s_; + bool using_strings_; + + std::vector> leafnodedata_; + std::unordered_map leafdata_map_; + std::vector roots_; + const int64_t kOffset_ = 4000000000L; + const int64_t kMaxTreeDepth_ = 1000; + POST_EVAL_TRANSFORM post_transform_; + bool weights_are_all_positive_; +}; +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/treeregressor.cc b/onnxruntime/core/providers/cpu/ml/treeregressor.cc new file mode 100644 index 0000000000000..4fecec46d76a6 --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/treeregressor.cc @@ -0,0 +1,278 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/ml/treeregressor.h" + +namespace onnxruntime { +namespace ml { + +ONNX_CPU_OPERATOR_ML_KERNEL( + TreeEnsembleRegressor, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()).MayInplace(0, 0), + TreeEnsembleRegressor); + +template +TreeEnsembleRegressor::TreeEnsembleRegressor(const OpKernelInfo& info) + : OpKernel(info), + nodes_treeids_(info.GetAttrsOrDefault("nodes_treeids")), + nodes_nodeids_(info.GetAttrsOrDefault("nodes_nodeids")), + nodes_featureids_(info.GetAttrsOrDefault("nodes_featureids")), + nodes_values_(info.GetAttrsOrDefault("nodes_values")), + nodes_hitrates_(info.GetAttrsOrDefault("nodes_hitrates")), + nodes_truenodeids_(info.GetAttrsOrDefault("nodes_truenodeids")), + nodes_falsenodeids_(info.GetAttrsOrDefault("nodes_falsenodeids")), + missing_tracks_true_(info.GetAttrsOrDefault("nodes_missing_value_tracks_true")), + target_nodeids_(info.GetAttrsOrDefault("target_nodeids")), + target_treeids_(info.GetAttrsOrDefault("target_treeids")), + target_ids_(info.GetAttrsOrDefault("target_ids")), + target_weights_(info.GetAttrsOrDefault("target_weights")), + base_values_(info.GetAttrsOrDefault("base_values")), + transform_(::onnxruntime::ml::MakeTransform(info.GetAttrOrDefault("post_transform", "NONE"))), + aggregate_function_(::onnxruntime::ml::MakeAggregateFunction(info.GetAttrOrDefault("aggregate_function", "SUM"))) { + ONNXRUNTIME_ENFORCE(info.GetAttr("n_targets", &n_targets_).IsOK()); + + //update nodeids to start at 0 + ONNXRUNTIME_ENFORCE(!nodes_treeids_.empty()); + int64_t current_tree_id = 1234567891L; + std::vector tree_offsets; + + for (size_t i = 0; i < nodes_treeids_.size(); i++) { + if (nodes_treeids_[i] != current_tree_id) { + tree_offsets.push_back(nodes_nodeids_[i]); + current_tree_id = nodes_treeids_[i]; + } + int64_t offset = tree_offsets[tree_offsets.size() - 1]; + nodes_nodeids_[i] = nodes_nodeids_[i] - offset; + if (nodes_falsenodeids_[i] >= 0) { + nodes_falsenodeids_[i] = nodes_falsenodeids_[i] - offset; + } + if (nodes_truenodeids_[i] >= 0) { + nodes_truenodeids_[i] = nodes_truenodeids_[i] - offset; + } + } + for (size_t i = 0; i < target_nodeids_.size(); i++) { + int64_t offset = tree_offsets[target_treeids_[i]]; + target_nodeids_[i] = target_nodeids_[i] - offset; + } + + std::vector modes = info.GetAttrsOrDefault("nodes_modes"); + + for (const auto& mode : modes) { + nodes_modes_.push_back(::onnxruntime::ml::MakeTreeNodeMode(mode)); + } + + size_t nodes_id_size = nodes_nodeids_.size(); + ONNXRUNTIME_ENFORCE(target_nodeids_.size() == target_ids_.size()); + ONNXRUNTIME_ENFORCE(target_nodeids_.size() == target_weights_.size()); + ONNXRUNTIME_ENFORCE(nodes_id_size == nodes_featureids_.size()); + ONNXRUNTIME_ENFORCE(nodes_id_size == nodes_values_.size()); + ONNXRUNTIME_ENFORCE(nodes_id_size == nodes_modes_.size()); + ONNXRUNTIME_ENFORCE(nodes_id_size == nodes_truenodeids_.size()); + ONNXRUNTIME_ENFORCE(nodes_id_size == nodes_falsenodeids_.size()); + ONNXRUNTIME_ENFORCE((nodes_id_size == nodes_hitrates_.size()) || (0 == nodes_hitrates_.size())); + + max_tree_depth_ = 1000; + offset_ = four_billion_; + //leafnode data, these are the votes that leaves do + for (size_t i = 0; i < target_nodeids_.size(); i++) { + leafnode_data_.push_back(std::make_tuple(target_treeids_[i], target_nodeids_[i], target_ids_[i], target_weights_[i])); + } + std::sort(begin(leafnode_data_), end(leafnode_data_), [](auto const& t1, auto const& t2) { + if (std::get<0>(t1) != std::get<0>(t2)) + return std::get<0>(t1) < std::get<0>(t2); + else + return std::get<1>(t1) < std::get<1>(t2); + }); + //make an index so we can find the leafnode data quickly when evaluating + int64_t field0 = -1; + int64_t field1 = -1; + for (size_t i = 0; i < leafnode_data_.size(); i++) { + int64_t id0 = std::get<0>(leafnode_data_[i]); + int64_t id1 = std::get<1>(leafnode_data_[i]); + if (id0 != field0 || id1 != field1) { + int64_t id = id0 * four_billion_ + id1; + auto p3 = std::make_pair(id, i); // position is i + leafdata_map_.insert(p3); + field0 = id; + field1 = static_cast(i); + } + } + //treenode ids, some are roots, and roots have no parents + std::unordered_map parents; //holds count of all who point to you + std::unordered_map indices; + //add all the nodes to a map, and the ones that have parents are not roots + std::unordered_map::iterator it; + size_t start_counter = 0L; + for (size_t i = 0; i < nodes_treeids_.size(); i++) { + //make an index to look up later + int64_t id = nodes_treeids_[i] * four_billion_ + nodes_nodeids_[i]; + auto p3 = std::make_pair(id, i); // i is the position + indices.insert(p3); + it = parents.find(id); + if (it == parents.end()) { + //start counter at 0 + auto p1 = std::make_pair(id, start_counter); + parents.insert(p1); + } + } + //all true nodes aren't roots + for (size_t i = 0; i < nodes_truenodeids_.size(); i++) { + if (nodes_modes_[i] == ::onnxruntime::ml::NODE_MODE::LEAF) continue; + //they must be in the same tree + int64_t id = nodes_treeids_[i] * offset_ + nodes_truenodeids_[i]; + it = parents.find(id); + ONNXRUNTIME_ENFORCE(it != parents.end()); + it->second++; + } + //all false nodes aren't roots + for (size_t i = 0; i < nodes_falsenodeids_.size(); i++) { + if (nodes_modes_[i] == ::onnxruntime::ml::NODE_MODE::LEAF) continue; + //they must be in the same tree + int64_t id = nodes_treeids_[i] * offset_ + nodes_falsenodeids_[i]; + it = parents.find(id); + ONNXRUNTIME_ENFORCE(it != parents.end()); + it->second++; + } + //find all the nodes that dont have other nodes pointing at them + for (auto& parent : parents) { + if (parent.second == 0) { + int64_t id = parent.first; + it = indices.find(id); + ONNXRUNTIME_ENFORCE(it != indices.end()); + roots_.push_back(it->second); + } + } + ONNXRUNTIME_ENFORCE(base_values_.empty() || base_values_.size() == static_cast(n_targets_)); +} + +template +common::Status TreeEnsembleRegressor::ProcessTreeNode(std::unordered_map& classes, int64_t treeindex, const T* Xdata, int64_t feature_base) const { + //walk down tree to the leaf + ::onnxruntime::ml::NODE_MODE mode = static_cast<::onnxruntime::ml::NODE_MODE>(nodes_modes_[treeindex]); + int64_t loopcount = 0; + int64_t root = treeindex; + while (mode != ::onnxruntime::ml::NODE_MODE::LEAF) { + T val = Xdata[feature_base + nodes_featureids_[treeindex]]; + bool tracktrue = true; + if (missing_tracks_true_.size() != nodes_truenodeids_.size()) { + tracktrue = false; + } else { + tracktrue = (missing_tracks_true_[treeindex] != 0) && std::isnan(val); + } + float threshold = nodes_values_[treeindex]; + if (mode == ::onnxruntime::ml::NODE_MODE::BRANCH_LEQ) { + treeindex = val <= threshold || tracktrue ? nodes_truenodeids_[treeindex] : nodes_falsenodeids_[treeindex]; + } else if (mode == ::onnxruntime::ml::NODE_MODE::BRANCH_LT) { + treeindex = val < threshold || tracktrue ? nodes_truenodeids_[treeindex] : nodes_falsenodeids_[treeindex]; + } else if (mode == ::onnxruntime::ml::NODE_MODE::BRANCH_GTE) { + treeindex = val >= threshold || tracktrue ? nodes_truenodeids_[treeindex] : nodes_falsenodeids_[treeindex]; + } else if (mode == ::onnxruntime::ml::NODE_MODE::BRANCH_GT) { + treeindex = val > threshold || tracktrue ? nodes_truenodeids_[treeindex] : nodes_falsenodeids_[treeindex]; + } else if (mode == ::onnxruntime::ml::NODE_MODE::BRANCH_EQ) { + treeindex = val == threshold || tracktrue ? nodes_truenodeids_[treeindex] : nodes_falsenodeids_[treeindex]; + } else if (mode == ::onnxruntime::ml::NODE_MODE::BRANCH_NEQ) { + treeindex = val != threshold || tracktrue ? nodes_truenodeids_[treeindex] : nodes_falsenodeids_[treeindex]; + } + + if (treeindex < 0) { + return common::Status(common::ONNXRUNTIME, common::RUNTIME_EXCEPTION, + "treeindex evaluated to a negative value, which should not happen."); + } + treeindex = treeindex + root; + mode = (::onnxruntime::ml::NODE_MODE)nodes_modes_[treeindex]; + loopcount++; + if (loopcount > max_tree_depth_) break; + } + //should be at leaf + int64_t id = nodes_treeids_[treeindex] * four_billion_ + nodes_nodeids_[treeindex]; + //auto it_lp = leafdata_map.find(id); + auto it_lp = leafdata_map_.find(id); + if (it_lp != leafdata_map_.end()) { + size_t index = it_lp->second; + int64_t treeid = std::get<0>(leafnode_data_[index]); + int64_t nodeid = std::get<1>(leafnode_data_[index]); + while (treeid == nodes_treeids_[treeindex] && nodeid == nodes_nodeids_[treeindex]) { + int64_t classid = std::get<2>(leafnode_data_[index]); + float weight = std::get<3>(leafnode_data_[index]); + auto it_classes = classes.find(classid); + if (it_classes != classes.end()) { + it_classes->second += weight; + } else { + auto p1 = std::make_pair(classid, weight); + classes.insert(p1); + } + index++; + if (index >= leafnode_data_.size()) { + break; + } + treeid = std::get<0>(leafnode_data_[index]); + nodeid = std::get<1>(leafnode_data_[index]); + } + } + return common::Status::OK(); +} + +template +common::Status TreeEnsembleRegressor::Compute(OpKernelContext* context) const { + const Tensor* X = context->Input(0); + if (X == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + if (X->Shape().NumDimensions() == 0) { + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, + "Input shape needs to be at least a single dimension."); + } + + int64_t stride = X->Shape().NumDimensions() == 1 ? X->Shape()[0] : X->Shape()[1]; + int64_t N = X->Shape().NumDimensions() == 1 ? 1 : X->Shape()[0]; + Tensor* Y = context->Output(0, TensorShape({N, n_targets_})); + + int64_t write_index = 0; + const auto* x_data = X->template Data(); + + for (int64_t i = 0; i < N; i++) //for each class + { + int64_t current_weight_0 = i * stride; + std::unordered_map scores; + //for each tree + for (size_t j = 0; j < roots_.size(); j++) { + //walk each tree from its root + ONNXRUNTIME_RETURN_IF_ERROR(ProcessTreeNode(scores, roots_[j], x_data, current_weight_0)); + } + //find aggregate, could use a heap here if there are many classes + std::vector outputs; + for (int64_t j = 0; j < n_targets_; j++) { + //reweight scores based on number of voters + auto it_scores = scores.find(j); + float val = base_values_.size() == (size_t)n_targets_ ? base_values_[j] : 0.f; + if (it_scores != scores.end()) { + if (aggregate_function_ == ::onnxruntime::ml::AGGREGATE_FUNCTION::AVERAGE) { + val += scores[j] / roots_.size(); + } else if (aggregate_function_ == ::onnxruntime::ml::AGGREGATE_FUNCTION::SUM) { + val += scores[j]; + } else if (aggregate_function_ == ::onnxruntime::ml::AGGREGATE_FUNCTION::MIN) { + if (scores[j] < val) val = scores[j]; + } else if (aggregate_function_ == ::onnxruntime::ml::AGGREGATE_FUNCTION::MAX) { + if (scores[j] > val) val = scores[j]; + } + } + outputs.push_back(val); + } + if (transform_ == ::onnxruntime::ml::POST_EVAL_TRANSFORM::LOGISTIC) { + for (float& output : outputs) { + output = ::onnxruntime::ml::ml_logit(output); + } + } else if (transform_ == ::onnxruntime::ml::POST_EVAL_TRANSFORM::SOFTMAX) { + ::onnxruntime::ml::compute_softmax(outputs); + } else if (transform_ == ::onnxruntime::ml::POST_EVAL_TRANSFORM::SOFTMAX_ZERO) { + ::onnxruntime::ml::compute_softmax_zero(outputs); + } + for (float output : outputs) { + Y->template MutableData()[write_index] = output; + write_index++; + } + } + return Status::OK(); +} + +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/treeregressor.h b/onnxruntime/core/providers/cpu/ml/treeregressor.h new file mode 100644 index 0000000000000..2216e5c74b689 --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/treeregressor.h @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "ml_common.h" + +namespace onnxruntime { +namespace ml { +template +class TreeEnsembleRegressor final : public OpKernel { + public: + explicit TreeEnsembleRegressor(const OpKernelInfo& info); + common::Status Compute(OpKernelContext* context) const override; + + private: + common::Status ProcessTreeNode(std::unordered_map& classes, int64_t treeindex, const T* Xdata, int64_t feature_base) const; + + std::vector nodes_treeids_; + std::vector nodes_nodeids_; + std::vector nodes_featureids_; + std::vector nodes_values_; + std::vector nodes_hitrates_; + std::vector nodes_modes_; + std::vector nodes_truenodeids_; + std::vector nodes_falsenodeids_; + std::vector missing_tracks_true_; + + std::vector target_nodeids_; + std::vector target_treeids_; + std::vector target_ids_; + std::vector target_weights_; + + std::vector base_values_; + int64_t n_targets_; + ::onnxruntime::ml::POST_EVAL_TRANSFORM transform_; + ::onnxruntime::ml::AGGREGATE_FUNCTION aggregate_function_; + std::vector> leafnode_data_; + std::unordered_map leafdata_map_; + std::vector roots_; + int64_t offset_; + int64_t max_tree_depth_; + const int64_t four_billion_ = 4000000000L; +}; +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/zipmap.cc b/onnxruntime/core/providers/cpu/ml/zipmap.cc new file mode 100644 index 0000000000000..db4e52488642c --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/zipmap.cc @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/ml/zipmap.h" +#include "core/util/math_cpuonly.h" +/** +https://github.com/onnx/onnx/blob/master/onnx/defs/traditionalml/defs.cc +ONNX_OPERATOR_SCHEMA(ZipMap) +.SetDomain("ai.onnx.ml") +.SetDoc(R"DOC( +Makes a map from the input and the attributes. +Assumes input 0 are the values, and the keys are specified by the attributes. +Must provide keys in either classlabels_strings or classlabels_int64s (but not both). +Input 0 may have a batch size larger than 1, +but each input in the batch must be the size of the keys specified by the attributes. +The order of the input and attributes determines the key-value mapping. +)DOC") +.Input(0, "X", "The input values", "tensor(float)") +.Output(0, "Z", "The output map", "T") +.TypeConstraint( +"T", +{ "seq(map(string, float))", "seq(map(int64, float))" }, +" allowed types.") +.Attr("classlabels_strings", "keys if using string keys", AttributeProto::STRINGS, OPTIONAL) +.Attr("classlabels_int64s", "keys if using int keys", AttributeProto::INTS, OPTIONAL); +*/ +using namespace ::onnxruntime::common; +using namespace std; +namespace onnxruntime { +namespace ml { +ONNX_CPU_OPERATOR_ML_KERNEL( + ZipMap, + 1, + KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetType>>(), + DataTypeImpl::GetType>>()}), + ZipMapOp); + +ZipMapOp::ZipMapOp(const OpKernelInfo& info) + : OpKernel(info), + classlabels_int64s_(info.GetAttrsOrDefault("classlabels_int64s")), + classlabels_strings_(info.GetAttrsOrDefault("classlabels_strings")) { + ONNXRUNTIME_ENFORCE(classlabels_strings_.empty() ^ classlabels_int64s_.empty(), + "Must provide classlabels_strings or classlabels_int64s but not both."); + using_strings_ = !classlabels_strings_.empty(); +} + +common::Status ZipMapOp::Compute(OpKernelContext* context) const { + const Tensor* tensor_pointer = context->Input(0); + if (tensor_pointer == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + const Tensor& X = *tensor_pointer; + const TensorShape& x_shape = X.Shape(); + const vector x_dims = x_shape.GetDims(); + + if (x_dims.empty()) { + return Status(ONNXRUNTIME, + INVALID_ARGUMENT, + "Zipmap does not support empty dim count"); + } + + int64_t batch_size = x_dims.size() > 1 ? x_dims[0] : 1; + int64_t features_per_batch = x_dims[x_dims.size() - 1]; + + if (x_dims.size() > 2) { + return Status(ONNXRUNTIME, + INVALID_ARGUMENT, + "Zipmap only supports 1D or 2D input tensors"); + } + + const float* x_data = X.template Data(); + + if (using_strings_) { + if (features_per_batch != static_cast(classlabels_strings_.size())) { + return Status(ONNXRUNTIME, + INVALID_ARGUMENT, + "Input features_per_batch[" + std::to_string(features_per_batch) + + "] != number of classlabels[" + std::to_string(classlabels_strings_.size()) + "]"); + } + auto* y_data = context->Output>>(0); + if (y_data == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + + //auto* y_data = Y->template MutableData>>(); + y_data->resize(batch_size); + int64_t current_weight_0 = 0; + for (int n = 0; n < batch_size; n++) { + std::map map1; + for (int j = 0; j < features_per_batch; j++) { + map1[classlabels_strings_[j]] = x_data[current_weight_0 + j]; + } + current_weight_0 += features_per_batch; + (*y_data)[n] = std::move(map1); + } + } else { + if (features_per_batch != static_cast(classlabels_int64s_.size())) { + return Status(ONNXRUNTIME, + INVALID_ARGUMENT, + "Input features_per_batch[" + std::to_string(features_per_batch) + + "] != number of classlabels[" + std::to_string(classlabels_int64s_.size()) + "]"); + } + auto* y_data = context->Output>>(0); + if (y_data == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + //auto* y_data = Y->template MutableData>>(); + y_data->resize(batch_size); + int64_t current_weight_0 = 0; + for (int n = 0; n < batch_size; n++) { + std::map map2; + for (int j = 0; j < features_per_batch; j++) { + map2[classlabels_int64s_[j]] = x_data[current_weight_0 + j]; + } + current_weight_0 += features_per_batch; + (*y_data)[n] = std::move(map2); + } + } + return common::Status::OK(); +} +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/ml/zipmap.h b/onnxruntime/core/providers/cpu/ml/zipmap.h new file mode 100644 index 0000000000000..82874f69ffe7a --- /dev/null +++ b/onnxruntime/core/providers/cpu/ml/zipmap.h @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +namespace onnxruntime { +namespace ml { + +class ZipMapOp final : public OpKernel { + public: + explicit ZipMapOp(const OpKernelInfo& info); + common::Status Compute(OpKernelContext* context) const override; + + private: + bool using_strings_; + std::vector classlabels_int64s_; + std::vector classlabels_strings_; +}; + +} // namespace ml +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/autopad_type.h b/onnxruntime/core/providers/cpu/nn/autopad_type.h new file mode 100644 index 0000000000000..a98a88ff5da70 --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/autopad_type.h @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/common/exceptions.h" + +namespace onnxruntime { + +enum class AutoPadType { + NOTSET = 0, + VALID = 1, + SAME_UPPER = 2, + SAME_LOWER = 3, +}; + +inline AutoPadType StringToAutoPadType(const std::string& str) { + if (str.empty()) { + return AutoPadType::NOTSET; + } else if (str == "NOTSET") { // in onnx spec, default value is "NOTSET" + return AutoPadType::NOTSET; + } else if (str == "VALID") { + return AutoPadType::VALID; + } else if (str == "SAME_UPPER") { + return AutoPadType::SAME_UPPER; + } else if (str == "SAME_LOWER") { + return AutoPadType::SAME_LOWER; + } else { + ONNXRUNTIME_ENFORCE(false, "Unknown AutoPadType String"); + } +} +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/batch_norm.cc b/onnxruntime/core/providers/cpu/nn/batch_norm.cc new file mode 100644 index 0000000000000..c0fd62944e120 --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/batch_norm.cc @@ -0,0 +1,77 @@ +/** +* Copyright (c) 2016-present, Facebook, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +/* Modifications Copyright (c) Microsoft. */ + +#include "core/providers/cpu/nn/batch_norm.h" +#include "core/providers/cpu/nn/batch_norm_helper.h" + +namespace onnxruntime { +// spec: https://github.com/onnx/onnx/blob/master/docs/Operators.md#BatchNormalization +ONNX_CPU_OPERATOR_KERNEL( + BatchNormalization, + 7, + KernelDefBuilder().TypeConstraint("X", DataTypeImpl::GetTensorType()).TypeConstraint("scale", DataTypeImpl::GetTensorType()).TypeConstraint("B", DataTypeImpl::GetTensorType()).TypeConstraint("mean", DataTypeImpl::GetTensorType()).TypeConstraint("var", DataTypeImpl::GetTensorType()), + BatchNorm); + +template <> +Status BatchNorm::Compute(OpKernelContext* p_op_kernel_context) const { + const Tensor* X = p_op_kernel_context->Input(0); + const Tensor* scale = p_op_kernel_context->Input(1); + const Tensor* B = p_op_kernel_context->Input(2); + const Tensor* mean = p_op_kernel_context->Input(3); + const Tensor* var = p_op_kernel_context->Input(4); + + ONNXRUNTIME_RETURN_IF_ERROR(BatchNormHelper::ValidateInputs(X, scale, B, mean, var)); + + const TensorShape& x_shape = X->Shape(); + Tensor* Y = p_op_kernel_context->Output(0, x_shape); + + const auto& dims_vec = x_shape.GetDims(); + const size_t N = dims_vec[0]; + const size_t C = dims_vec[1]; // assume NCHW as per the spec + + // calculate sample_size + size_t sample_size = 1; + for (size_t i = 2; i < dims_vec.size(); ++i) { + sample_size *= dims_vec[i]; + } + + ConstEigenVectorArrayMap scale_arr(scale->template Data(), C); + ConstEigenVectorArrayMap bias_arr(B->template Data(), C); + + // Regardless of training or testing, we will apply the estimated mean + // and standard deviation to the input. For testing, they are + // specified directly by the input, and for training, they are computed + // by the op. + Eigen::Array inv_std(C); + ConstEigenVectorArrayMap var_arr(var->template Data(), C); + inv_std = (var_arr + epsilon_).sqrt().inverse(); + ConstEigenVectorArrayMap mean_arr(mean->template Data(), C); + // We can fuse the output computation as follows: + // ((x - est_mean) * (inv_var) * scale + bias + // to + // (x * inv_var * scale) + (bias - est_mean * inv_var * scale) + Eigen::Array new_scale = inv_std * scale_arr; + Eigen::Array new_bias = bias_arr - mean_arr * new_scale; + EigenArrayMap Y_arr(Y->template MutableData(), sample_size, N * C); + ConstEigenArrayMap X_arr(X->template Data(), sample_size, N * C); + for (size_t nc = 0; nc < N * C; ++nc) { + Y_arr.col(nc) = X_arr.col(nc) * new_scale(nc % C) + new_bias(nc % C); + } + + return Status::OK(); +} +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/batch_norm.h b/onnxruntime/core/providers/cpu/nn/batch_norm.h new file mode 100644 index 0000000000000..f5e66849d6a26 --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/batch_norm.h @@ -0,0 +1,45 @@ +/** +* Copyright (c) 2016-present, Facebook, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +/* Modifications Copyright (c) Microsoft. */ + +#pragma once + +#include "core/common/common.h" +#include "core/common/exceptions.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cpu/nn/autopad_type.h" +#include "core/framework/tensor.h" +#include "core/util/math_cpuonly.h" + +namespace onnxruntime { + +template +class BatchNorm final : public OpKernel { + public: + BatchNorm(const OpKernelInfo& op_kernel_info) : OpKernel(op_kernel_info) { + float tmp_eplison; + if (op_kernel_info.GetAttr("epsilon", &tmp_eplison).IsOK()) { + epsilon_ = tmp_eplison; + } + } + + Status Compute(OpKernelContext* p_op_kernel_context) const override; + + private: + float epsilon_ = 1e-5f; + int64_t is_test_; // ignored in this implementation since we're doing inferencing only. +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/batch_norm_helper.h b/onnxruntime/core/providers/cpu/nn/batch_norm_helper.h new file mode 100644 index 0000000000000..d6f7861bccfae --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/batch_norm_helper.h @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/status.h" +#include "core/framework/tensor.h" +#include + +namespace onnxruntime { +class BatchNormHelper { + public: + static common::Status ValidateInputs(const Tensor* X, + const Tensor* scale, + const Tensor* B, + const Tensor* mean, + const Tensor* var) { + // defined as per spec and used for validation + constexpr int kNumInputScaleDimensions = 1; + constexpr int kNumInputBiasDimensions = 1; + constexpr int kNumInputMeanDimensions = 1; + constexpr int kNumInputVarianceDimensions = 1; + //constexpr int kMinCudaNumDims = 4; + //constexpr int kMaxCudaNumDims = 5; + + if (X->Shape().GetDims().empty()) { + return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Invalid input X: Empty dimensions"); + } + + int64_t num_channels = X->Shape().GetDims()[1]; + + if (scale->Shape().NumDimensions() != kNumInputScaleDimensions) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid input scale: NumDimensions() != ", kNumInputScaleDimensions); + } + if (scale->Shape().GetDims()[0] != num_channels) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid input scale: 0th dimension != ", num_channels); + } + + if (B->Shape().NumDimensions() != kNumInputBiasDimensions) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid input B: NumDimensions() != ", kNumInputBiasDimensions); + } + if (B->Shape().GetDims()[0] != num_channels) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid input B: 0th dimension != ", num_channels); + } + + if (mean->Shape().NumDimensions() != kNumInputMeanDimensions) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid input mean: NumDimensions() != ", kNumInputMeanDimensions); + } + if (mean->Shape().GetDims()[0] != num_channels) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid input mean: 0th dimension != ", num_channels); + } + + if (var->Shape().NumDimensions() != kNumInputVarianceDimensions) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid input var: NumDimensions() != ", kNumInputVarianceDimensions); + } + if (var->Shape().GetDims()[0] != num_channels) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid input var: 0th dimension != ", num_channels); + } + + return common::Status::OK(); + } + + static void NormalizeDims(const TensorShape& x_shape, std::vector& new_dims) { + new_dims.clear(); + auto& orig_dims = x_shape.GetDims(); + if (orig_dims.size() == 4 /*supported size by CUDA*/ || + orig_dims.size() == 5 /*supported size by CUDA*/) { + new_dims = orig_dims; + return; + } + + auto rank = x_shape.NumDimensions(); + auto num_samples = rank > 0 ? orig_dims[0] : 1; // NCHW + auto num_channels = rank > 1 ? orig_dims[1] : 1; + auto width = rank > 3 ? orig_dims[3] : 1; + auto height = rank > 2 ? orig_dims[2] : 1; + new_dims = {num_samples, num_channels, height, width}; + } +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/conv.cc b/onnxruntime/core/providers/cpu/nn/conv.cc new file mode 100644 index 0000000000000..01b38657e8e8b --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/conv.cc @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/nn/conv_impl.h" + +namespace onnxruntime { +ONNX_CPU_OPERATOR_KERNEL( + Conv, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Conv); +} diff --git a/onnxruntime/core/providers/cpu/nn/conv.h b/onnxruntime/core/providers/cpu/nn/conv.h new file mode 100644 index 0000000000000..cf6484ab417a2 --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/conv.h @@ -0,0 +1,33 @@ +/** +* Copyright (c) 2016-present, Facebook, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +/* Modifications Copyright (c) Microsoft. */ + +#pragma once + +#include "core/providers/cpu/nn/conv_base.h" + +namespace onnxruntime { + +template +class Conv : public OpKernel, public ConvBase { + public: + Conv(const OpKernelInfo& info) : OpKernel(info), ConvBase(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/conv_base.h b/onnxruntime/core/providers/cpu/nn/conv_base.h new file mode 100644 index 0000000000000..68c2cbdf28f89 --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/conv_base.h @@ -0,0 +1,187 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/common/exceptions.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cpu/nn/autopad_type.h" +#include "core/util/math.h" + +namespace onnxruntime { +namespace { + +// helper function +template +Status ComputePadAndOutputShape( + const int64_t in_dim, + const int64_t stride, + const int64_t kernel, + const int64_t dilation, + AutoPadType pad_type, + int64_t* pad_head, + int64_t* pad_tail, + int64_t* out_dim) { + const int64_t dkernel = dilation * (kernel - 1) + 1; + + if (pad_type == AutoPadType::NOTSET) { + *out_dim = static_cast(static_cast(in_dim + *pad_head + *pad_tail - dkernel) / stride + 1); + } else { + switch (pad_type) { + case AutoPadType::VALID: + *pad_head = 0; + *pad_tail = 0; + *out_dim = (in_dim - dkernel) / stride + 1; + break; + case AutoPadType::SAME_UPPER: + case AutoPadType::SAME_LOWER: { + ONNXRUNTIME_ENFORCE(dilation == 1, "Dilation not supported for AutoPadType::SAME_UPPER or AutoPadType::SAME_LOWER."); + int64_t legacy_target_size = (in_dim + stride - 1) / stride; + int64_t pad_needed = (legacy_target_size - 1) * stride + kernel - in_dim; + *out_dim = (in_dim + pad_needed - dkernel) / stride + 1; + + // make sure padding is symmetric + if (ForceSymmetricAutoPadding) + pad_needed = math::roundUpPow2(pad_needed); + + if (pad_type == AutoPadType::SAME_LOWER) { + *pad_head = (pad_needed + 1) / 2; + } else { + *pad_head = pad_needed / 2; + } + *pad_tail = pad_needed - *pad_head; + } break; + default: + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "pad type not supported."); + } + } + return Status::OK(); +} +} // namespace + +// base class used by Conv and ConvTranspose +class ConvBase { + protected: + ConvBase(const OpKernelInfo& info) { + std::string auto_pad; + auto status = info.GetAttr("auto_pad", &auto_pad); + auto_pad_ = status.IsOK() ? StringToAutoPadType(auto_pad) : AutoPadType::NOTSET; + + kernel_shape_specified_ = info.GetAttrs("kernel_shape", kernel_shape_).IsOK(); + + status = info.GetAttrs("strides", strides_); + if (!status.IsOK()) { + strides_.resize(kernel_shape_.size(), 1); + } + + status = info.GetAttrs("pads", pads_); + if (!status.IsOK()) { + pads_.resize(kernel_shape_.size() * 2, 0); + } + + status = info.GetAttrs("dilations", dilations_); + if (!status.IsOK()) { + dilations_.resize(kernel_shape_.size(), 1); + } + + status = info.GetAttr("group", &group_); + if (!status.IsOK()) { + group_ = 1; + } + +#if false + // TODO: Re-enable when attributes values are guaranteed to be filled. + std::string auto_pad; + ONNXRUNTIME_ENFORCE(info.GetAttr("auto_pad", &auto_pad).IsOK()); + auto_pad_ = StringToAutoPadType(auto_pad); + ONNXRUNTIME_ENFORCE(info.GetAttr("group", &group_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttrs("kernel_shape", kernel_shape_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttrs("strides", strides_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttrs("pads", pads_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttrs("dilations", dilations_).IsOK()); +#endif + } + + ~ConvBase() {} + + protected: + std::vector ComputeKernelShape(const TensorShape& weight_shape) const { + if (kernel_shape_specified_) + return kernel_shape_; + else { + auto& weight_dims = weight_shape.GetDims(); + std::vector result(weight_dims.begin() + 2, weight_dims.end()); + return result; + } + } + + Status ValidateInputShape(const Tensor* X, const Tensor* W) const { + const int64_t C = X->Shape()[1]; + const int64_t M = W->Shape()[0]; + + if (X->Shape().NumDimensions() != W->Shape().NumDimensions()) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "X num_dims does not match W num_dims.", + " X: ", X->Shape().ToString().c_str(), + " W: ", W->Shape().ToString().c_str()); + } + + if (C != W->Shape()[1] * group_) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Input channels C is not equal to kernel channels * group.", + " C: ", C, + " kernel channels: ", W->Shape()[1], + " group: ", group_); + } + + if (M % group_ != 0) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Output channels M is not divisible by group.", + " M: ", M, + " group: ", group_); + } + return Status::OK(); + } + + template + Status InferOutputShape(const TensorShape& input_shape, + const std::vector& kernel_shape, + const std::vector& strides, + const std::vector& dilations, + std::vector* pads, + std::vector* output_shape) const { + int rank = gsl::narrow_cast(input_shape.NumDimensions()); + for (int dim = 0; dim < rank; ++dim) { + if (dim >= strides.size() || dim >= kernel_shape.size() || + dim >= dilations.size() || dim >= pads->size() || + rank + dim >= pads->size()) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Out of bound access to array"); + } + int64_t dim_size = 0; + ONNXRUNTIME_RETURN_IF_ERROR(ComputePadAndOutputShape( + input_shape[dim], + strides[dim], + kernel_shape[dim], + dilations[dim], + auto_pad_, + &pads->at(dim), + &pads->at(input_shape.NumDimensions() + dim), + &dim_size)); + if (dim_size <= 0) { + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Invalid input shape: " + input_shape.ToString()); + } + output_shape->push_back(dim_size); + } + return Status::OK(); + } + + AutoPadType auto_pad_; + int64_t group_; + bool kernel_shape_specified_; + std::vector strides_; + std::vector pads_; + std::vector dilations_; + + private: + std::vector kernel_shape_; // must use ComputeKernelShape(...), instead of kernel_shape_ +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/conv_impl.h b/onnxruntime/core/providers/cpu/nn/conv_impl.h new file mode 100644 index 0000000000000..44fb16c353053 --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/conv_impl.h @@ -0,0 +1,301 @@ +/** +* Copyright (c) 2016-present, Facebook, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +/* Modifications Copyright (c) Microsoft. */ + +#pragma once + +#include "core/providers/cpu/nn/conv.h" +#include "core/util/math.h" +#include "core/util/math_cpuonly.h" +#include "core/mlas/inc/mlas.h" + +namespace onnxruntime { + +template +Status Conv::Compute(OpKernelContext* context) const { + size_t num_inputs = OpKernel::Node().InputDefs().size(); + + const Tensor* X = context->Input(0); + const Tensor* W = context->Input(1); + const Tensor* B = num_inputs == 3 ? context->Input(2) : nullptr; + const int64_t N = X->Shape()[0]; + const int64_t C = X->Shape()[1]; + const int64_t M = W->Shape()[0]; + ONNXRUNTIME_RETURN_IF_ERROR(ValidateInputShape(X, W)); + + std::vector kernel_shape = ComputeKernelShape(W->Shape()); + + if (kernel_shape.size() + 2 != W->Shape().NumDimensions()) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "kernel_shape num_dims is not compatible with W num_dims.", + " kernel_shape: ", TensorShape(kernel_shape).ToString().c_str(), + " W: ", W->Shape().ToString().c_str()); + } + + for (size_t i = 0; i < kernel_shape.size(); ++i) { + if (kernel_shape[i] != W->Shape()[i + 2]) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "kernel_shape is not compatible with W shape.", + " kernel_shape: ", TensorShape(kernel_shape).ToString().c_str(), + " W: ", W->Shape().ToString().c_str()); + } + } + + bool Is2DKernel = kernel_shape.size() == 2; + std::vector pads(pads_); + if (pads.empty()) { + pads.resize(kernel_shape.size() * 2, 0); + } + std::vector dilations(dilations_); + if (dilations.empty()) { + dilations.resize(kernel_shape.size(), 1); + } + std::vector strides(strides_); + if (strides.empty()) { + strides.resize(kernel_shape.size(), 1); + } + + std::vector Y_dims; + Y_dims.insert(Y_dims.begin(), {N, M}); + TensorShape input_shape = X->Shape().Slice(2); + ONNXRUNTIME_RETURN_IF_ERROR(InferOutputShape(input_shape, kernel_shape, strides, dilations, &pads, &Y_dims)); + Tensor* Y = context->Output(0, TensorShape(Y_dims)); + TensorShape output_shape = Y->Shape().Slice(2); + + const int64_t input_image_size = input_shape.Size(); + const int64_t output_image_size = output_shape.Size(); + const int64_t kernel_size = TensorShape(kernel_shape).Size(); + const int64_t X_offset = C / group_ * input_image_size; + const int64_t Y_offset = Y->Shape().Size() / Y->Shape()[0] / group_; + const int64_t W_offset = W->Shape().Size() / group_; + const int64_t kernel_dim = C / group_ * kernel_size; + const int64_t col_buffer_size = kernel_dim * output_image_size; + + AllocatorPtr alloc; + ONNXRUNTIME_RETURN_IF_ERROR(context->GetTempSpaceAllocator(&alloc)); + + auto col_data = alloc->Alloc(sizeof(T) * col_buffer_size); + BufferUniquePtr col_buffer(col_data, BufferDeleter(alloc)); + T* col_buffer_data = static_cast(col_buffer.get()); + + const T* Xdata = X->template Data(); + T* Ydata = Y->template MutableData(); + + TensorShape image_shape = X->Shape().Slice(1); + std::vector col_buffer_shape{kernel_dim}; + col_buffer_shape.insert(col_buffer_shape.end(), output_shape.GetDims().begin(), + output_shape.GetDims().end()); + + for (int image_id = 0; image_id < N; ++image_id) { + for (int group_id = 0; group_id < group_; ++group_id) { + if (Is2DKernel) { + math::Im2col( + Xdata + group_id * X_offset, + C / group_, + input_shape[0], + input_shape[1], + kernel_shape[0], + kernel_shape[1], + dilations[0], + dilations[1], + pads[0], + pads[1], + pads[2], + pads[3], + strides[0], + strides[1], + col_buffer_data, + &CPUMathUtil::Instance()); + } else { + math::Im2colNd( + Xdata + group_id * X_offset, + image_shape.GetDims().data(), + col_buffer_shape.data(), + C * input_image_size, + col_buffer_size, + kernel_shape.data(), + strides.data(), + dilations.data(), + pads.data(), + static_cast(kernel_shape.size()), + col_buffer_data, + &CPUMathUtil::Instance()); + } + math::Gemm( + CblasNoTrans, + CblasNoTrans, + M / group_, + output_image_size, + kernel_dim, + 1, + W->template Data() + group_id * W_offset, + col_buffer_data, + 0, + Ydata + group_id * Y_offset, + &CPUMathUtil::Instance()); + } + + if (B != nullptr) { + auto Ymatrix = EigenMatrixMap(Ydata, output_image_size, M); + auto Bvec = ConstEigenVectorMap(B->template Data(), M); + Ymatrix.rowwise() += Bvec.transpose(); + } + + Xdata += X_offset * group_; + Ydata += Y_offset * group_; + } + + return Status::OK(); +} + +template <> +Status Conv::Compute(OpKernelContext* context) const { + size_t num_inputs = OpKernel::Node().InputDefs().size(); + const Tensor* X = context->Input(0); + const Tensor* W = context->Input(1); + const Tensor* B = num_inputs == 3 ? context->Input(2) : nullptr; + const int64_t N = X->Shape()[0]; + const int64_t C = X->Shape()[1]; + const int64_t M = W->Shape()[0]; + ONNXRUNTIME_RETURN_IF_ERROR(ValidateInputShape(X, W)); + + std::vector kernel_shape = ComputeKernelShape(W->Shape()); + + if (kernel_shape.size() + 2 != W->Shape().NumDimensions()) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "kernel_shape num_dims is not compatible with W num_dims.", + " kernel_shape: ", TensorShape(kernel_shape).ToString().c_str(), + " W: ", W->Shape().ToString().c_str()); + } + + for (size_t i = 0; i < kernel_shape.size(); ++i) { + if (kernel_shape[i] != W->Shape()[i + 2]) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "kernel_shape is not compatible with W shape.", + " kernel_shape: ", TensorShape(kernel_shape).ToString().c_str(), + " W: ", W->Shape().ToString().c_str()); + } + } + + std::vector pads(pads_); + if (pads.empty()) { + pads.resize(kernel_shape.size() * 2, 0); + } + std::vector dilations(dilations_); + if (dilations.empty()) { + dilations.resize(kernel_shape.size(), 1); + } + std::vector strides(strides_); + if (strides.empty()) { + strides.resize(kernel_shape.size(), 1); + } + + std::vector Y_dims; + Y_dims.insert(Y_dims.begin(), {N, M}); + TensorShape input_shape = X->Shape().Slice(2); + ONNXRUNTIME_RETURN_IF_ERROR(InferOutputShape(input_shape, kernel_shape, strides, dilations, &pads, &Y_dims)); + Tensor* Y = context->Output(0, TensorShape(Y_dims)); + TensorShape output_shape = Y->Shape().Slice(2); + + const int64_t input_image_size = input_shape.Size(); + const int64_t output_image_size = output_shape.Size(); + const int64_t kernel_size = TensorShape(kernel_shape).Size(); + + AllocatorPtr alloc; + ONNXRUNTIME_RETURN_IF_ERROR(context->GetTempSpaceAllocator(&alloc)); + + const float* Xdata = X->template Data(); + float* Ydata = Y->template MutableData(); + + MLAS_CONV_PARAMETERS Parameters; + size_t WorkingBufferSize; + if (MlasConvPrepare(&Parameters, + kernel_shape.size(), + static_cast(N), + static_cast(group_), + static_cast(C / group_), + input_shape.GetDims().data(), + kernel_shape.data(), + dilations.data(), + pads.data(), + strides.data(), + static_cast(M / group_), + &WorkingBufferSize)) { + auto working_data = WorkingBufferSize > 0 ? alloc->Alloc(sizeof(float) * WorkingBufferSize) : nullptr; + BufferUniquePtr working_buffer(working_data, BufferDeleter(alloc)); + + MlasConv(&Parameters, + Xdata, + W->template Data(), + B != nullptr ? B->template Data() : nullptr, + static_cast(working_buffer.get()), + Ydata); + } else { + const int64_t X_offset = C / group_ * input_image_size; + const int64_t Y_offset = Y->Shape().Size() / Y->Shape()[0] / group_; + const int64_t W_offset = W->Shape().Size() / group_; + const int64_t kernel_dim = C / group_ * kernel_size; + const int64_t col_buffer_size = kernel_dim * output_image_size; + + auto col_data = alloc->Alloc(sizeof(float) * col_buffer_size); + BufferUniquePtr col_buffer(col_data, BufferDeleter(alloc)); + float* col_buffer_data = static_cast(col_buffer.get()); + + TensorShape image_shape = X->Shape().Slice(1); + std::vector col_buffer_shape{kernel_dim}; + col_buffer_shape.insert(col_buffer_shape.end(), output_shape.GetDims().begin(), + output_shape.GetDims().end()); + + for (int image_id = 0; image_id < N; ++image_id) { + for (int group_id = 0; group_id < group_; ++group_id) { + math::Im2colNd( + Xdata + group_id * X_offset, + image_shape.GetDims().data(), + col_buffer_shape.data(), + C * input_image_size, + col_buffer_size, + kernel_shape.data(), + strides.data(), + dilations.data(), + pads.data(), + static_cast(kernel_shape.size()), + col_buffer_data, + &CPUMathUtil::Instance()); + math::Gemm( + CblasNoTrans, + CblasNoTrans, + M / group_, + output_image_size, + kernel_dim, + 1, + W->template Data() + group_id * W_offset, + col_buffer_data, + 0, + Ydata + group_id * Y_offset, + &CPUMathUtil::Instance()); + } + + if (B != nullptr) { + auto Ymatrix = EigenMatrixMap(Ydata, output_image_size, M); + auto Bvec = ConstEigenVectorMap(B->template Data(), M); + Ymatrix.rowwise() += Bvec.transpose(); + } + + Xdata += X_offset * group_; + Ydata += Y_offset * group_; + } + } + + return Status::OK(); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/conv_transpose.cc b/onnxruntime/core/providers/cpu/nn/conv_transpose.cc new file mode 100644 index 0000000000000..939269d190238 --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/conv_transpose.cc @@ -0,0 +1,295 @@ +/** +* Copyright (c) 2016-present, Facebook, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +/* Modifications Copyright (c) Microsoft. */ + +#include "core/providers/cpu/nn/conv_transpose.h" +#include "core/util/math.h" +#include "core/util/math_cpuonly.h" + +namespace onnxruntime { + +ONNX_CPU_OPERATOR_KERNEL( + ConvTranspose, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + ConvTranspose); + +inline void ComputeTransposePadAndOutputShape( + const int64_t in_size, + const int64_t stride, + const int64_t kernel, + const int64_t adj, + AutoPadType pad_type, + int64_t* pad_head, + int64_t* pad_tail, + int64_t* out_size) { + if (*out_size != -1) { + ONNXRUNTIME_ENFORCE(*out_size >= 0); + // total padding size + int64_t paddings = std::max(0, (in_size - 1) * stride + kernel + adj - *out_size); + if (pad_type == AutoPadType::SAME_UPPER) { // pad more on head when paddings are odd. + *pad_head = paddings - paddings / 2; + *pad_tail = paddings / 2; + } else { + // for pad_type is NOTSET, SAME_LOWER or VALID + // set pad_head as paddings/2, pad_tail as paddings-paddings/2. + // That said, we pad more on tail when paddings are odd. + *pad_head = paddings / 2; + *pad_tail = paddings - paddings / 2; + } + return; + } else { + if (pad_type != AutoPadType::NOTSET) { + switch (pad_type) { + // We handle cases of AutoPadType::VALID and AutoPadType::SAME_UPPER/LOWER, + // the same way + case AutoPadType::VALID: + case AutoPadType::SAME_UPPER: + case AutoPadType::SAME_LOWER: + *pad_head = 0; + *pad_tail = 0; + *out_size = (in_size - 1) * stride + kernel + adj; + break; + default: + throw NotImplementedException("pad type not supported"); + } + } else { + *out_size = + (in_size - 1) * stride + kernel + adj - *pad_head - *pad_tail; + } + } +} + +Status ConvTransposeBase::PrepareForCompute(OpKernelContext* context, bool has_bias, ConvTransposeBase::Prepare& p) const { + const Tensor* X = context->Input(0); + const Tensor* F = context->Input(1); + const Tensor* B = has_bias ? context->Input(2) : nullptr; + const TensorShape& input_shape = X->Shape(); + + // input validations + if (group_ <= 0) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "group count is <= 0", + " group: ", group_); + } + + if (input_shape.NumDimensions() != 4) { + // This condition is not true for two tests in ONNX tests series: + // test_convtranspose_1d_cpu, test_convtranspose_3d_cpu. + // TODO: the error message should tell which operator raises it. + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input X must be 4-dimensional.", + " X: ", X->Shape().ToString().c_str()); + } + + if (input_shape.NumDimensions() != F->Shape().NumDimensions()) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "X num_dims does not match W num_dims.", + " X: ", X->Shape().ToString().c_str(), + " W: ", F->Shape().ToString().c_str()); + } + + const int64_t num_input_channels = input_shape[1]; + + if (F->Shape()[0] != num_input_channels) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "filter number not equal to input channel number.", + " filter_number: ", F->Shape()[0], + " num_input_channels: ", num_input_channels); + } + + const int64_t N = input_shape[0]; + const int64_t H = input_shape[2]; + const int64_t W = input_shape[3]; + const int64_t num_output_channels_multiplier = F->Shape()[1]; + const int64_t num_output_channels = num_output_channels_multiplier * group_; + + // it looks like num_output_channels is really k*group_ similar to how in the conv case + // num_input_channels is k*group_. hence removing the check for num_output_channels here. + + if (num_input_channels % group_ != 0) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input channels is not divisible by group.", + " num_input_channels: ", num_input_channels, + " group: ", group_); + } + + std::vector kernel_shape = ComputeKernelShape(F->Shape()); + + if (kernel_shape[0] != F->Shape()[2]) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "kernel height does not match filter height.", + " kernel_height: ", kernel_shape[0], + " filter_height: ", F->Shape()[2]); + } + + if (kernel_shape[1] != F->Shape()[3]) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "kernel width does not match filter width.", + " kernel_width: ", kernel_shape[1], + " filter_width: ", F->Shape()[3]); + } + + std::vector output_padding(output_padding_); + if (output_padding.empty()) { + output_padding.resize(kernel_shape.size(), 0); + } + std::vector pads(pads_); + if (pads.empty()) { + pads.resize(kernel_shape.size() * 2, 0); + } + std::vector dilations(dilations_); + if (dilations.empty()) { + dilations.resize(kernel_shape.size(), 1); + } + std::vector strides(strides_); + if (strides.empty()) { + strides.resize(kernel_shape.size(), 1); + } + + std::vector Y_dims; + + ComputePadsAndOutputShape(input_shape, num_output_channels, kernel_shape, strides, output_padding, &pads, &Y_dims); + TensorShape Yshape(Y_dims); + Tensor* Y = context->Output(0, Yshape); + + p.X = X; + p.F = F; + p.B = B; + p.Y = Y; + p.N = N; + p.H = H; + p.W = W; + p.num_input_channels = num_input_channels; + p.num_output_channels = num_output_channels; + p.kernel_shape = std::move(kernel_shape); + p.pads = std::move(pads); + p.strides = std::move(strides); + p.dilations = std::move(dilations); + return Status::OK(); +} + +void ConvTransposeBase::ComputePadsAndOutputShape( + const TensorShape input_shape, + const int64_t output_channel, + const std::vector& kernel_shape, + const std::vector& strides, + const std::vector& output_padding, + std::vector* pads, + std::vector* output_shape) const { + const int64_t N = input_shape[0]; + const int64_t H = input_shape[2]; + const int64_t W = input_shape[3]; + int64_t output_height = -1, output_width = -1; + size_t output_shape_size = output_shape_.size(); + + if (output_shape_size != 0) { + output_height = output_shape_[output_shape_size - 2]; + output_width = output_shape_[output_shape_size - 1]; + ONNXRUNTIME_ENFORCE(output_height >= H, "Output height cannot be smaller than input height."); + ONNXRUNTIME_ENFORCE(output_width >= W, "Output width cannot be smaller than input width."); + } + + ComputeTransposePadAndOutputShape( + H, + strides[0], + kernel_shape[0], + output_padding[0], + auto_pad_, + &pads->at(0), + &pads->at(2), + &output_height); + + ComputeTransposePadAndOutputShape( + W, + strides[1], + kernel_shape[1], + output_padding[1], + auto_pad_, + &pads->at(1), + &pads->at(3), + &output_width); + + output_shape->insert(output_shape->begin(), {N, output_channel, output_height, output_width}); +} + +template +Status ConvTranspose::Compute(OpKernelContext* context) const { + size_t num_inputs = OpKernel::Node().InputDefs().size(); + Prepare p; + ONNXRUNTIME_RETURN_IF_ERROR(PrepareForCompute(context, num_inputs == 3, p)); + + const int64_t input_image_size = p.H * p.W; + const int64_t X_offset = p.num_input_channels / group_ * input_image_size; + const int64_t Y_offset = p.Y->Shape().Size() / p.Y->Shape()[0] / group_; + const int64_t W_offset = p.F->Shape().Size() / group_; + const int64_t kernel_dim = p.num_output_channels / group_ * p.kernel_shape[0] * p.kernel_shape[1]; + const int64_t output_image_size = p.Y->Shape()[2] * p.Y->Shape()[3]; + + AllocatorPtr alloc; + ONNXRUNTIME_RETURN_IF_ERROR(context->GetTempSpaceAllocator(&alloc)); + + auto col_data = alloc->Alloc(sizeof(T) * kernel_dim * p.H * p.W); + BufferUniquePtr col_buffer(col_data, BufferDeleter(alloc)); + T* col_buffer_data = static_cast(col_buffer.get()); + + const T* Xdata = p.X->template Data(); + const T* filter_data = p.F->template Data(); + T* Ydata = p.Y->template MutableData(); + + for (auto image_id = 0; image_id < p.N; ++image_id) { + for (int group_id = 0; group_id < group_; ++group_id) { + // Weight term + math::Gemm( + CblasTrans, + CblasNoTrans, + kernel_dim, + input_image_size, + p.num_input_channels / group_, + 1, + filter_data + group_id * W_offset, + Xdata + group_id * X_offset, + 0, + col_buffer_data, + &CPUMathUtil::Instance()); + + // Col2im + math::Col2im( + col_buffer_data, + p.num_output_channels / group_, + p.Y->Shape()[2], + p.Y->Shape()[3], + p.kernel_shape[0], + p.kernel_shape[1], + 1, + 1, + p.pads[0], + p.pads[1], + p.pads[2], + p.pads[3], + p.strides[0], + p.strides[1], + Ydata + group_id * Y_offset, + &CPUMathUtil::Instance()); + } + + if (p.B != nullptr) { + auto Ymatrix = EigenMatrixMap(Ydata, output_image_size, p.num_output_channels); + auto Bvec = ConstEigenVectorMap(p.B->template Data(), p.num_output_channels); + Ymatrix.rowwise() += Bvec.transpose(); + } + + Xdata += X_offset * group_; + Ydata += Y_offset * group_; + } + + return Status::OK(); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/conv_transpose.h b/onnxruntime/core/providers/cpu/nn/conv_transpose.h new file mode 100644 index 0000000000000..0e50c0fb128b5 --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/conv_transpose.h @@ -0,0 +1,71 @@ +/** +* Copyright (c) 2016-present, Facebook, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +/* Modifications Copyright (c) Microsoft. */ + +#pragma once + +#include "core/providers/cpu/nn/conv_base.h" + +namespace onnxruntime { + +class ConvTransposeBase : public ConvBase { + protected: + ConvTransposeBase(const OpKernelInfo& info) + : ConvBase(info), + output_padding_(info.GetAttrsOrDefault("output_padding")), + output_shape_(info.GetAttrsOrDefault("output_shape")) { + } + + struct Prepare { + const Tensor* X; + const Tensor* F; + const Tensor* B; + Tensor* Y; + int64_t N; + int64_t H; + int64_t W; + int64_t num_input_channels; + int64_t num_output_channels; + std::vector kernel_shape; + std::vector pads; + std::vector dilations; + std::vector strides; + }; + + Status PrepareForCompute(OpKernelContext* context, bool has_bias, Prepare& p) const; + + void ComputePadsAndOutputShape( + const TensorShape input_shape, + const int64_t output_channel, + const std::vector& kernel_shape, + const std::vector& strides, + const std::vector& output_padding, + std::vector* pads, + std::vector* output_shape) const; + + const std::vector output_padding_; + const std::vector output_shape_; +}; + +template +class ConvTranspose : public OpKernel, public ConvTransposeBase { + public: + ConvTranspose(const OpKernelInfo& info) : OpKernel(info), ConvTransposeBase(info) {} + + Status Compute(OpKernelContext* context) const override; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/flatten.cc b/onnxruntime/core/providers/cpu/nn/flatten.cc new file mode 100644 index 0000000000000..e07e93357e6e6 --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/flatten.cc @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/nn/flatten.h" + +namespace onnxruntime { +ONNX_CPU_OPERATOR_VERSIONED_KERNEL( + Flatten, + 1, + 9, + KernelDefBuilder() + .Alias(0, 0) + .TypeConstraint("T", DataTypeImpl::AllTensorTypes()), + Flatten); +} diff --git a/onnxruntime/core/providers/cpu/nn/flatten.h b/onnxruntime/core/providers/cpu/nn/flatten.h new file mode 100644 index 0000000000000..a1c8881f35328 --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/flatten.h @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "gsl/gsl_util" +#include "core/providers/cpu/tensor/utils.h" + +namespace onnxruntime { + +class Flatten final : public OpKernel { + public: + Flatten(const OpKernelInfo& info) : OpKernel(info) { + ONNXRUNTIME_ENFORCE(info.GetAttr("axis", &axis_).IsOK()); + } + + Status Compute(OpKernelContext* context) const override { + const Tensor* X = context->Input(0); + if (X == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + + const TensorShape& X_shape = X->Shape(); + ONNXRUNTIME_ENFORCE(gsl::narrow_cast(X_shape.NumDimensions()) >= axis_, "The rank of input tensor must be >= axis"); + + Tensor* Y = context->Output(0, TensorShape({X_shape.SizeToDimension(axis_), X_shape.SizeFromDimension(axis_)})); + + CopyCpuTensor(X, Y); + + return Status::OK(); + } + + private: + int64_t axis_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/instance_norm.cc b/onnxruntime/core/providers/cpu/nn/instance_norm.cc new file mode 100644 index 0000000000000..3903b76dffc84 --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/instance_norm.cc @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/nn/instance_norm.h" +#include "core/providers/cpu/nn/instance_norm_helper.h" +#include "core/util/math_cpuonly.h" +using namespace ::onnxruntime::common; + +namespace onnxruntime { + +ONNX_CPU_OPERATOR_KERNEL( + InstanceNormalization, + 6, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + InstanceNorm); + +template <> +Status InstanceNorm::Compute(OpKernelContext* p_op_kernel_context) const { + const Tensor* input = p_op_kernel_context->Input(0); + const Tensor* scale = p_op_kernel_context->Input(1); + const Tensor* B = p_op_kernel_context->Input(2); + + ONNXRUNTIME_RETURN_IF_ERROR(InstanceNormHelper::ValidateInputs(input, scale, B)); + const int64_t N = input->Shape().GetDims()[0]; + const int64_t C = input->Shape().GetDims()[1]; + const int64_t W = input->Shape().SizeFromDimension(2); + + const TensorShape& x_shape = input->Shape(); + Tensor* Y = p_op_kernel_context->Output(0, x_shape); + + for (auto i = 0; i < N * C; ++i) { + ConstEigenVectorArrayMap Xi(input->template Data() + W * i, W); + const float Xi_mean = Xi.mean(); + const float squared_norm = (Xi - Xi_mean).matrix().squaredNorm(); + const float inv_stdev = 1.0f / std::sqrt(squared_norm / W + epsilon_); + EigenVectorArrayMap Yi(Y->template MutableData() + W * i, W); + const float channel_scale = inv_stdev * scale->template Data()[i % C]; + const float channel_shift = B->template Data()[i % C] - Xi_mean * channel_scale; + Yi = Xi * channel_scale + channel_shift; + } + + return Status::OK(); +} +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/instance_norm.h b/onnxruntime/core/providers/cpu/nn/instance_norm.h new file mode 100644 index 0000000000000..0a6f35e07f08f --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/instance_norm.h @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/framework/op_kernel.h" + +namespace onnxruntime { + +template +class InstanceNorm final : public OpKernel { + public: + InstanceNorm(const OpKernelInfo& op_kernel_info) : OpKernel(op_kernel_info) { + ONNXRUNTIME_ENFORCE(op_kernel_info.GetAttr("epsilon", &epsilon_).IsOK()); + } + + Status Compute(OpKernelContext* p_op_kernel_context) const override; + + private: + float epsilon_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/instance_norm_helper.h b/onnxruntime/core/providers/cpu/nn/instance_norm_helper.h new file mode 100644 index 0000000000000..bdf6f90b3828c --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/instance_norm_helper.h @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/status.h" +#include "core/framework/tensor.h" +#include + +namespace onnxruntime { + +class InstanceNormHelper { + public: + static common::Status ValidateInputs(const Tensor* input, const Tensor* scale, const Tensor* B) { + if (input->Shape().NumDimensions() < 3) { + std::ostringstream ostr; + ostr << "Invalid input data: number of dimensions is less than 3: " << input->Shape().NumDimensions(); + return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, ostr.str()); + } + if (scale->Shape().NumDimensions() != 1) { + std::ostringstream ostr; + ostr << "Invalid input scale: number of dimensions is not 1: " << scale->Shape().NumDimensions(); + return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, ostr.str()); + } + if (scale->Shape().Size() != input->Shape().GetDims()[1]) { + std::ostringstream ostr; + ostr << "Mismatch between input data and scale: size of scale != input channel count " + << scale->Shape().Size() << " vs. " << input->Shape().GetDims()[1]; + return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, ostr.str()); + } + + if (B->Shape().NumDimensions() != 1) { + std::ostringstream ostr; + ostr << "Invalid input B: number of dimensions is not 1: " << B->Shape().NumDimensions(); + return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, ostr.str()); + } + + if (B->Shape().Size() != input->Shape().GetDims()[1]) { + std::ostringstream ostr; + ostr << "Mismatch between input data and B: size of B != input channel count " + << B->Shape().Size() << " vs. " << input->Shape().GetDims()[1]; + return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, ostr.str()); + } + + return common::Status::OK(); + } +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/lp_norm.cc b/onnxruntime/core/providers/cpu/nn/lp_norm.cc new file mode 100644 index 0000000000000..1a12e77dc3830 --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/lp_norm.cc @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/nn/lp_norm.h" +#include "core/util/math_cpuonly.h" + +namespace onnxruntime { +ONNX_CPU_OPERATOR_KERNEL( + LpNormalization, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + LpNorm); + +using InnerStride = Eigen::InnerStride; +using StridedVec = Eigen::Map, 0, InnerStride>; +using ConstStridedVec = Eigen::Map, 0, InnerStride>; + +void DoNormalizeP2( + const float* xData, + float* yData, + const int64_t m, + const int64_t n, + const int64_t sf) { + for (int i = 0; i < n; ++i) { + auto base = (i / sf) * sf * m + (i % sf); + ConstStridedVec xVec(xData + base, 1, m, InnerStride(sf)); + auto norm = xVec.template lpNorm<2>(); + if (norm != 0) { + StridedVec yVec(yData + base, 1, m, InnerStride(sf)); + yVec = xVec / norm; + } + } +}; + +void DoNormalizeP1( + const float* xData, + float* yData, + const int64_t m, + const int64_t n, + const int64_t sf) { + for (int i = 0; i < n; ++i) { + auto base = (i / sf) * sf * m + (i % sf); + ConstStridedVec xVec(xData + base, 1, m, InnerStride(sf)); + auto norm = xVec.template lpNorm<1>(); + if (norm != 0) { + StridedVec yVec(yData + base, 1, m, InnerStride(sf)); + yVec = xVec / norm; + } + } +}; + +template <> +Status LpNorm::Compute(OpKernelContext* p_op_kernel_context) const { + const Tensor* input = p_op_kernel_context->Input(0); + const TensorShape& input_shape = input->Shape(); + Tensor* output = p_op_kernel_context->Output(0, input_shape); + + const auto canonical_axis = axis_ != -1 ? axis_ : (input_shape.NumDimensions() - 1); + const int64_t m = input_shape.GetDims()[canonical_axis]; + const int64_t n = input_shape.Size() / m; + const int64_t sf = input_shape.SizeFromDimension(canonical_axis + 1); + + if (p_ == 1) { + DoNormalizeP1(input->template Data(), output->template MutableData(), m, n, sf); + } else if (p_ == 2) { + DoNormalizeP2(input->template Data(), output->template MutableData(), m, n, sf); + } + + return Status::OK(); +} +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/lp_norm.h b/onnxruntime/core/providers/cpu/nn/lp_norm.h new file mode 100644 index 0000000000000..932776ec2a19f --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/lp_norm.h @@ -0,0 +1,39 @@ +/** +* Copyright (c) 2016-present, Facebook, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +/* Modifications Copyright (c) Microsoft. */ + +#pragma once + +#include "core/framework/op_kernel.h" + +namespace onnxruntime { + +template +class LpNorm final : public OpKernel { + public: + LpNorm(const OpKernelInfo& op_kernel_info) : OpKernel(op_kernel_info) { + ONNXRUNTIME_ENFORCE(op_kernel_info.GetAttr("axis", &axis_).IsOK()); + ONNXRUNTIME_ENFORCE(op_kernel_info.GetAttr("p", &p_).IsOK()); + ONNXRUNTIME_ENFORCE(p_ == 1 || p_ == 2); + } + + Status Compute(OpKernelContext* p_op_kernel_context) const override; + + private: + int64_t axis_; + int64_t p_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/lrn.cc b/onnxruntime/core/providers/cpu/nn/lrn.cc new file mode 100644 index 0000000000000..eb1978b16c898 --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/lrn.cc @@ -0,0 +1,99 @@ +/** +* Copyright (c) 2016-present, Facebook, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +/* Modifications Copyright (c) Microsoft. */ + +#include "core/providers/cpu/nn/lrn.h" +#include "core/util/math.h" +#include "core/util/math_cpuonly.h" + +namespace onnxruntime { + +template <> +Status LRN::Compute(OpKernelContext* context) const { + const Tensor* X = context->Input(0); + if (X == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + + Tensor* Y = context->Output(0, X->Shape()); + + // Supports NCHW image format. + ONNXRUNTIME_ENFORCE(X->Shape().NumDimensions() == 4); + const int N = gsl::narrow_cast(X->Shape()[0]); + const int C = gsl::narrow_cast(X->Shape()[1]); + const int H = gsl::narrow_cast(X->Shape()[2]); + const int W = gsl::narrow_cast(X->Shape()[3]); + const int image_size = C * H * W; + const int pre_pad = (size_ - 1) / 2; + + const float* Xdata = X->template Data(); + float* Ydata = Y->template MutableData(); + + AllocatorPtr alloc; + ONNXRUNTIME_RETURN_IF_ERROR(context->GetTempSpaceAllocator(&alloc)); + + const int Xsize = gsl::narrow_cast(X->Shape().Size()); + auto sdata = alloc->Alloc(sizeof(float) * Xsize); + BufferUniquePtr scale_buffer(sdata, BufferDeleter(alloc)); + float* scale_data = static_cast(scale_buffer.get()); + math::Set(Xsize, bias_, scale_data, &CPUMathUtil::Instance()); + + const size_t padded_square_size = (C + size_ - 1) * H * W; + auto psdata = alloc->Alloc(sizeof(float) * padded_square_size); + BufferUniquePtr padded_square_buffer(psdata, BufferDeleter(alloc)); + float* padded_square_data = static_cast(padded_square_buffer.get()); + math::Set(padded_square_size, 0.0f, padded_square_data, &CPUMathUtil::Instance()); + + const float alpha_over_size = alpha_ / size_; + // go through the images + for (int n = 0; n < N; ++n) { + // compute the padded square + math::Sqr(image_size, Xdata + image_size * n, + padded_square_data + pre_pad * H * W, + &CPUMathUtil::Instance()); + // Create the first channel scale + for (int c = 0; c < size_; ++c) { + math::Axpy( + H * W, alpha_over_size, padded_square_data + c * H * W, + scale_data + image_size * n, &CPUMathUtil::Instance()); + } + + for (int c = 1; c < C; ++c) { + float* this_scale_slice = scale_data + n * image_size + c * H * W; + // copy previous scale + memcpy(this_scale_slice, this_scale_slice - H * W, H * W * sizeof(float)); + // add head + math::Axpy( + H * W, alpha_over_size, padded_square_data + (c + size_ - 1) * H * W, + this_scale_slice, &CPUMathUtil::Instance()); + // subtract tail + math::Axpy( + H * W, -alpha_over_size, padded_square_data + (c - 1) * H * W, + this_scale_slice, &CPUMathUtil::Instance()); + } + } + + math::Powx(Xsize, scale_data, -beta_, Ydata, &CPUMathUtil::Instance()); + math::Mul(Xsize, Ydata, Xdata, Ydata, &CPUMathUtil::Instance()); + + return Status::OK(); +} + +ONNX_CPU_OPERATOR_KERNEL( + LRN, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + LRN); + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/lrn.h b/onnxruntime/core/providers/cpu/nn/lrn.h new file mode 100644 index 0000000000000..031189cf613b8 --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/lrn.h @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "gsl/gsl_util" + +#include "core/common/common.h" +#include "core/common/exceptions.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { + +template +class LRN : public OpKernel { + public: + LRN(const OpKernelInfo& info) : OpKernel(info) { + int64_t size; + ONNXRUNTIME_ENFORCE(info.GetAttr("size", &size).IsOK()); + size_ = gsl::narrow_cast(size); + ONNXRUNTIME_ENFORCE(size_ > 0); + ONNXRUNTIME_ENFORCE(size_ % 2 == 1); + ONNXRUNTIME_ENFORCE(info.GetAttr("alpha", &alpha_).IsOK()); + ONNXRUNTIME_ENFORCE(alpha_ > 0.0f); + ONNXRUNTIME_ENFORCE(info.GetAttr("beta", &beta_).IsOK()); + ONNXRUNTIME_ENFORCE(beta_ > 0.0f); + Status status = info.GetAttr("bias", &bias_); + if (!status.IsOK()) { + bias_ = 1.0f; + } + } + + Status Compute(OpKernelContext* p_op_kernel_context) const override; + + protected: + float alpha_; + float beta_; + float bias_; + int size_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/pool.cc b/onnxruntime/core/providers/cpu/nn/pool.cc new file mode 100644 index 0000000000000..a76322ef31670 --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/pool.cc @@ -0,0 +1,413 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/nn/pool.h" +#include +using namespace ::onnxruntime::common; + +namespace onnxruntime { + +template +Status Pool::Compute(OpKernelContext* context) const { + const Tensor* X = context->Input(0); + const TensorShape& x_shape = X->Shape(); + + ONNXRUNTIME_RETURN_IF_NOT(x_shape.NumDimensions() >= 3, "Input dimension cannot be less than 3."); + + std::vector pads = pads_; + std::vector kernel_shape = kernel_shape_; + + if (global_pooling_) { + const auto& input_dims = x_shape.GetDims(); + kernel_shape.assign(input_dims.begin() + 2, input_dims.end()); + pads.assign(kernel_shape.size(), 0); + } + + std::vector output_dims = PoolBase::SetOutputSize(x_shape, x_shape[1], &pads); + Tensor* Y = context->Output(0, TensorShape(output_dims)); + + const float* X_data = X->template Data(); + float* Y_data = Y->template MutableData(); + + // The main loop + int64_t channels = x_shape[1]; + int64_t height = x_shape[2]; + int64_t width = kernel_shape.size() > 1 ? x_shape[3] : 1; + int64_t depth = kernel_shape.size() > 2 ? x_shape[4] : 1; + int64_t pooled_height = output_dims[2]; + int64_t pooled_width = kernel_shape.size() > 1 ? output_dims[3] : 1; + int64_t pooled_depth = kernel_shape.size() > 2 ? output_dims[4] : 1; + + switch (kernel_shape.size()) { + case 1: { + int64_t x_step = height; + int64_t y_step = pooled_height; + const int64_t total_channels = x_shape[0] * channels; + +#pragma omp parallel for + for (int64_t c = 0; c < total_channels; ++c) { + const float* x_d = X_data + c * x_step; + float* y_d = Y_data + c * y_step; + + for (int64_t ph = 0; ph < pooled_height; ++ph) { + int64_t hstart = ph * stride_h() - pads[0]; + int64_t hend = std::min(hstart + kernel_shape[0], height); + hstart = std::max(hstart, static_cast(0)); + T Yh = PoolType::Initialize(); + for (int64_t h = hstart; h < hend; ++h) { + PoolType::Process(x_d[h], Yh, pool_context_); + } + if (count_include_pad_) { + PoolType::Finalize(kernel_shape[0], Yh, pool_context_); + } else { + PoolType::Finalize(hend - hstart, Yh, pool_context_); + } + y_d[ph] = Yh; + } + } + + break; + } + + case 2: { + int64_t x_step = height * width; + int64_t y_step = pooled_height * pooled_width; + const int64_t total_channels = x_shape[0] * channels; + +#pragma omp parallel for + for (int64_t c = 0; c < total_channels; ++c) { + const float* x_d = X_data + c * x_step; + float* y_d = Y_data + c * y_step; + + for (int64_t ph = 0; ph < pooled_height; ++ph) { + int64_t hstart = ph * stride_h() - pads[0]; + int64_t hend = std::min(hstart + kernel_shape[0], height); + hstart = std::max(hstart, static_cast(0)); + for (int64_t pw = 0; pw < pooled_width; ++pw) { + int64_t wstart = pw * stride_w() - pads[1]; + int64_t wend = std::min(wstart + kernel_shape[1], width); + wstart = std::max(wstart, static_cast(0)); + const int64_t pool_index = ph * pooled_width + pw; + T Yh = PoolType::Initialize(); + for (int64_t h = hstart; h < hend; ++h) { + for (int64_t w = wstart; w < wend; ++w) { + const int64_t input_index = h * width + w; + PoolType::Process(x_d[input_index], Yh, pool_context_); + } + } + if (count_include_pad_) { + PoolType::Finalize(kernel_shape[0] * kernel_shape[1], Yh, pool_context_); + } else { + PoolType::Finalize((hend - hstart) * (wend - wstart), Yh, pool_context_); + } + y_d[pool_index] = Yh; + } + } + } + + break; + } + case 3: { + int64_t x_step = height * width * depth; + int64_t y_step = pooled_height * pooled_width * pooled_depth; + const int64_t total_channels = x_shape[0] * channels; + +#pragma omp parallel for + for (int64_t c = 0; c < total_channels; ++c) { + const float* x_d = X_data + c * x_step; + float* y_d = Y_data + c * y_step; + + for (int64_t ph = 0; ph < pooled_height; ++ph) { + int64_t hstart = ph * stride_h() - pads[0]; + int64_t hend = std::min(hstart + kernel_shape[0], height); + hstart = std::max(hstart, static_cast(0)); + for (int64_t pw = 0; pw < pooled_width; ++pw) { + int64_t wstart = pw * stride_w() - pads[1]; + int64_t wend = std::min(wstart + kernel_shape[1], width); + wstart = std::max(wstart, static_cast(0)); + for (int64_t pd = 0; pd < pooled_depth; ++pd) { + int64_t dstart = pd * stride_d() - pads[2]; + int64_t dend = std::min(dstart + kernel_shape[2], depth); + dstart = std::max(dstart, static_cast(0)); + const int64_t pool_index = + ph * pooled_width * pooled_depth + pw * pooled_depth + pd; + T Yh = PoolType::Initialize(); + for (int64_t h = hstart; h < hend; ++h) { + for (int64_t w = wstart; w < wend; ++w) { + for (int64_t d = dstart; d < dend; ++d) { + const int64_t input_index = h * width * depth + w * depth + d; + PoolType::Process(x_d[input_index], Yh, pool_context_); + } + } + } + if (count_include_pad_) { + PoolType::Finalize(kernel_shape[0] * kernel_shape[1] * kernel_shape[2], Yh, pool_context_); + } else { + PoolType::Finalize( + (hend - hstart) * (wend - wstart) * (dend - dstart), Yh, pool_context_); + } + y_d[pool_index] = Yh; + } + } + } + } + + break; + } + default: + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Unsupported pooling size : "); + } + + return Status::OK(); +} + +Status PoolBase::Compute(OpKernelContext* context, MLAS_POOLING_KIND kind) const { + const Tensor* X = context->Input(0); + const TensorShape& x_shape = X->Shape(); + + size_t input_dims = x_shape.NumDimensions(); + ONNXRUNTIME_RETURN_IF_NOT(input_dims >= 3, "Input dimension cannot be less than 3."); + + size_t pooling_dims = input_dims - 2; + if (pooling_dims > 3) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Unsupported pooling size."); + } + if (!global_pooling_) { + ONNXRUNTIME_RETURN_IF_NOT(pooling_dims == kernel_shape_.size(), "kernel_shape num_dims is not compatible with X num_dims."); + } + + std::vector pads = pads_; + std::vector output_dims = PoolBase::SetOutputSize(x_shape, x_shape[1], &pads); + Tensor* Y = context->Output(0, TensorShape(output_dims)); + + MlasPool(kind, + pooling_dims, + X->Shape().GetDims().data(), + global_pooling_ ? nullptr : kernel_shape_.data(), + global_pooling_ ? nullptr : pads.data(), + global_pooling_ ? nullptr : strides_.data(), + output_dims.data(), + X->template Data(), + Y->template MutableData()); + + return Status::OK(); +} + +template <> +Status Pool>::Compute(OpKernelContext* context) const { + return PoolBase::Compute(context, MlasMaximumPooling); +} + +template <> +Status Pool::Compute(OpKernelContext* context) const { + return PoolBase::Compute(context, count_include_pad_ ? MlasAveragePoolingIncludePad : MlasAveragePoolingExcludePad); +} + +template <> +Status Pool>::Compute(OpKernelContext* context) const { + // Use MLAS pooling if the index output tensor is not used. + if (OpKernel::Node().OutputDefs().size() == 1) { + return PoolBase::Compute(context, MlasMaximumPooling); + } + + const Tensor* X = context->Input(0); + const TensorShape& x_shape = X->Shape(); + + ONNXRUNTIME_RETURN_IF_NOT(x_shape.NumDimensions() >= 3, "Input dimension cannot be less than 3."); + + std::vector pads = pads_; + std::vector kernel_shape = kernel_shape_; + + std::vector output_dims = PoolBase::SetOutputSize(x_shape, x_shape[1], &pads); + Tensor* Y = context->Output(0, TensorShape(output_dims)); + Tensor* I = context->Output(1, TensorShape(output_dims)); + + const float* X_data = X->template Data(); + float* Y_data = Y->template MutableData(); + int64_t* I_data = I != nullptr ? I->template MutableData() : nullptr; + + // The main loop + int64_t channels = x_shape[1]; + int64_t height = x_shape[2]; + int64_t width = kernel_shape.size() > 1 ? x_shape[3] : 1; + int64_t depth = kernel_shape.size() > 2 ? x_shape[4] : 1; + int64_t pooled_height = output_dims[2]; + int64_t pooled_width = kernel_shape.size() > 1 ? output_dims[3] : 1; + int64_t pooled_depth = kernel_shape.size() > 2 ? output_dims[4] : 1; + + switch (kernel_shape.size()) { + case 1: { + int64_t x_step = height; + int64_t y_step = pooled_height; + const int64_t total_channels = x_shape[0] * channels; + +#pragma omp parallel for + for (int64_t c = 0; c < total_channels; ++c) { + const float* x_d = X_data + c * x_step; + float* y_d = Y_data + c * y_step; + int64_t* i_d = I_data ? I_data + c * y_step : nullptr; + for (int64_t ph = 0; ph < pooled_height; ++ph) { + int64_t hstart = ph * stride_h() - pads[0]; + int64_t hend = std::min(hstart + kernel_shape[0], height); + hstart = std::max(hstart, static_cast(0)); + float Yh = std::numeric_limits::lowest(); + int64_t h_index = -1; + for (int64_t h = hstart; h < hend; ++h) { + if (x_d[h] > Yh) { + Yh = x_d[h]; + h_index = h; + } + } + y_d[ph] = Yh; + if (i_d != nullptr) i_d[ph] = c * x_step + h_index; + } + } + + break; + } + + case 2: { + int64_t x_step = height * width; + int64_t y_step = pooled_height * pooled_width; + const int64_t total_channels = x_shape[0] * channels; + +#pragma omp parallel for + for (int64_t c = 0; c < total_channels; ++c) { + const float* x_d = X_data + c * x_step; + float* y_d = Y_data + c * y_step; + int64_t* i_d = I_data ? I_data + c * y_step : nullptr; + + for (int64_t ph = 0; ph < pooled_height; ++ph) { + int64_t hstart = ph * stride_h() - pads[0]; + int64_t hend = std::min(hstart + kernel_shape[0], height); + hstart = std::max(hstart, static_cast(0)); + for (int64_t pw = 0; pw < pooled_width; ++pw) { + int64_t wstart = pw * stride_w() - pads[1]; + int64_t wend = std::min(wstart + kernel_shape[1], width); + wstart = std::max(wstart, static_cast(0)); + const int64_t pool_index = ph * pooled_width + pw; + float Yh = std::numeric_limits::lowest(); + int64_t h_index = -1; + int64_t w_index = -1; + for (int64_t h = hstart; h < hend; ++h) { + for (int64_t w = wstart; w < wend; ++w) { + const int64_t input_index = h * width + w; + if (x_d[input_index] > Yh) { + Yh = x_d[input_index]; + h_index = h; + w_index = w; + } + } + } + y_d[pool_index] = Yh; + if (i_d != nullptr) + i_d[pool_index] = storage_order_ == 0 ? c * x_step + h_index * width + w_index + : c * x_step + h_index + w_index * height; + } + } + } + break; + } + case 3: { + int64_t x_step = height * width * depth; + int64_t y_step = pooled_height * pooled_width * pooled_depth; + const int64_t total_channels = x_shape[0] * channels; + +#pragma omp parallel for + for (int64_t c = 0; c < total_channels; ++c) { + const float* x_d = X_data + c * x_step; + float* y_d = Y_data + c * y_step; + int64_t* i_d = I_data ? I_data + c * y_step : nullptr; + + for (int64_t ph = 0; ph < pooled_height; ++ph) { + int64_t hstart = ph * stride_h() - pads[0]; + int64_t hend = std::min(hstart + kernel_shape[0], height); + hstart = std::max(hstart, static_cast(0)); + for (int64_t pw = 0; pw < pooled_width; ++pw) { + int64_t wstart = pw * stride_w() - pads[1]; + int64_t wend = std::min(wstart + kernel_shape[1], width); + wstart = std::max(wstart, static_cast(0)); + for (int64_t pd = 0; pd < pooled_depth; ++pd) { + int64_t dstart = pd * stride_d() - pads[2]; + int64_t dend = std::min(dstart + kernel_shape[2], depth); + dstart = std::max(dstart, static_cast(0)); + const int64_t pool_index = + ph * pooled_width * pooled_depth + pw * pooled_depth + pd; + float Yh = std::numeric_limits::lowest(); + int64_t h_index = -1; + int64_t w_index = -1; + int64_t d_index = -1; + for (int64_t h = hstart; h < hend; ++h) { + for (int64_t w = wstart; w < wend; ++w) { + for (int64_t d = dstart; d < dend; ++d) { + const int64_t input_index = h * width * depth + w * depth + d; + if (x_d[input_index] > Yh) { + Yh = x_d[input_index]; + h_index = h; + w_index = w; + d_index = d; + } + } + } + } + y_d[pool_index] = Yh; + if (i_d != nullptr) + i_d[pool_index] = storage_order_ == 0 ? c * x_step + h_index * width * depth + w_index * depth + d_index + : c * x_step + h_index + w_index * height + d_index * height * width; + } + } + } + } + break; + } + default: + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Unsupported pooling size : "); + } + + return Status::OK(); +} // namespace onnxruntime + +ONNX_CPU_OPERATOR_KERNEL( + AveragePool, + 7, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Pool); + +ONNX_CPU_OPERATOR_VERSIONED_KERNEL( + MaxPool, + 1, + 7, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Pool>); + +ONNX_CPU_OPERATOR_KERNEL( + MaxPool, + 8, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()).TypeConstraint("I", DataTypeImpl::GetTensorType()), + Pool>); + +ONNX_CPU_OPERATOR_KERNEL( + LpPool, + 2, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Pool); + +ONNX_CPU_OPERATOR_KERNEL( + GlobalLpPool, + 2, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Pool); + +ONNX_CPU_OPERATOR_KERNEL( + GlobalAveragePool, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Pool); + +ONNX_CPU_OPERATOR_KERNEL( + GlobalMaxPool, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Pool>); + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/pool.h b/onnxruntime/core/providers/cpu/nn/pool.h new file mode 100644 index 0000000000000..40284ea7d5b8b --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/pool.h @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/providers/cpu/nn/pool_base.h" + +namespace onnxruntime { + +template +class Pool : public OpKernel, public PoolBase { + public: + Pool(const OpKernelInfo& info) : OpKernel(info), PoolBase(info) { + const std::string& op_name = info.GetKernelDef().OpName(); + if (op_name == "LpPool" || op_name == "GlobalLpPool") { + pool_context_.init(info); + } + } + + ~Pool() override{}; + + Status Compute(OpKernelContext* context) const override; + + private: + PoolProcessContext pool_context_; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/pool_base.h b/onnxruntime/core/providers/cpu/nn/pool_base.h new file mode 100644 index 0000000000000..124ea992e5a37 --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/pool_base.h @@ -0,0 +1,246 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cpu/nn/autopad_type.h" +#include "core/mlas/inc/mlas.h" + +namespace onnxruntime { + +enum class PoolType : uint8_t { + kMaxPool, + kAveragePool, + kLpPool +}; + +class LpPool; + +class PoolProcessContext { + private: + int64_t p_; + + public: + friend class LpPool; + PoolProcessContext() {} + void init(const OpKernelInfo& info) { + ONNXRUNTIME_ENFORCE(info.GetAttr("p", &p_).IsOK()); + } +}; + +class AveragePool { + public: + static float Initialize() { + return 0.0; + } + + template + static void Process(const T& x_data, T& y_data, const PoolProcessContext& /*cxt*/) { + y_data += x_data; + } + + template + static void Finalize(const int64_t size, T& y_data, const PoolProcessContext& /*cxt*/) { + y_data /= size; + } + + static const PoolType type = PoolType::kAveragePool; +}; + +template +class MaxPool; + +template <> +class MaxPool<1 /*VERSION*/> { + public: + static float Initialize() { + return std::numeric_limits::lowest(); + } + + template + static void Process(const T& x_data, T& y_data, const PoolProcessContext& /*cxt*/) { + if (x_data > y_data) { + y_data = x_data; + } + } + + template + static void Finalize(const int64_t /*size*/, T& /*y_data*/, const PoolProcessContext& /*cxt*/) {} + + static const PoolType type = PoolType::kMaxPool; +}; + +template <> +class MaxPool<8 /*VERSION*/> { + public: + static const PoolType type = PoolType::kMaxPool; +}; + +class LpPool { + public: + static float Initialize() { + return 0.0f; + } + + template + static void Process(const T& x_data, T& y_data, const PoolProcessContext& cxt) { + y_data += static_cast(std::pow(std::abs(x_data), cxt.p_)); + } + + template + static void Finalize(const int64_t /*size*/, T& y_data, const PoolProcessContext& cxt) { + y_data = static_cast(std::pow(y_data, 1.0f / cxt.p_)); + } + static const PoolType type = PoolType::kLpPool; +}; + +class PoolBase { + protected: + PoolBase(const OpKernelInfo& info) { + op_name_ = info.GetKernelDef().OpName(); + global_pooling_ = (op_name_ == "GlobalAveragePool" || op_name_ == "GlobalMaxPool" || op_name_ == "GlobalLpPool"); + + if (!global_pooling_) { + ONNXRUNTIME_ENFORCE(info.GetAttrs("kernel_shape", kernel_shape_).IsOK(), + "No kernel shape is set."); + + std::string auto_padding; + ONNXRUNTIME_ENFORCE(info.GetAttr("auto_pad", &auto_padding).IsOK()); + auto_pad_ = StringToAutoPadType(auto_padding); + + if (!info.GetAttrs("pads", pads_).IsOK() || pads_.empty()) { + pads_.resize(kernel_shape_.size() * 2, 0); + } + + if (!info.GetAttrs("strides", strides_).IsOK() || strides_.empty()) { + strides_.resize(kernel_shape_.size(), 1); + } + + if (op_name_ == "AveragePool") { + int64_t temp; + ONNXRUNTIME_ENFORCE(info.GetAttr("count_include_pad", &temp).IsOK()); + count_include_pad_ = (temp != 0); + } + + if (op_name_ == "MaxPool") { + int start, end; + info.GetKernelDef().SinceVersion(&start, &end); + if (start == 8) { + storage_order_ = info.GetAttrOrDefault("storage_order", 0 /*default_value*/); + } + } + + for (size_t dim = 0; dim < kernel_shape_.size(); ++dim) { + ONNXRUNTIME_ENFORCE(kernel_shape_[dim] > 0); + ONNXRUNTIME_ENFORCE(pads_[dim] < kernel_shape_[dim] && pads_[dim + kernel_shape_.size()] < kernel_shape_[dim], + "Pad should be smaller than kernel."); + } + + ONNXRUNTIME_ENFORCE(strides_.size() == kernel_shape_.size()); + } + } + + ~PoolBase(){}; + + std::vector SetOutputSize(const TensorShape& input_shape, + int64_t output_channel, + std::vector* pads) const { + ONNXRUNTIME_ENFORCE(input_shape.Size() > 0); + std::vector output_dims; + int64_t N = input_shape[0]; + InferOutputSize(input_shape.GetDims(), &output_dims, pads); + + output_dims.insert(output_dims.begin(), {N, output_channel}); + + return output_dims; + } + + inline void InferOutputSize(const std::vector& input_dims, + std::vector* output_dims, + std::vector* pads) const { + ONNXRUNTIME_ENFORCE(input_dims.size() >= 2); + if (global_pooling_) { + output_dims->assign(input_dims.size() - 2, 1); + } else { + for (size_t dim = 0; dim < input_dims.size() - 2; ++dim) { + int64_t dim_size = 0; + ComputeSizeAndPad(static_cast(input_dims[dim + 2]), + strides_[dim], + kernel_shape_[dim], + &pads->at(dim), + &pads->at(input_dims.size() + dim - 2), + &dim_size); + output_dims->push_back(dim_size); + } + } + } + + inline void ComputeSizeAndPad(const int64_t in_size, + const int64_t stride, + const int64_t kernel, + int64_t* pad_head, + int64_t* pad_tail, + int64_t* out_size) const { + if (auto_pad_ != AutoPadType::NOTSET) { + switch (auto_pad_) { + case AutoPadType::VALID: + *pad_head = 0; + *pad_tail = 0; + *out_size = (in_size - kernel) / stride + 1; + break; + case AutoPadType::SAME_LOWER: { + int64_t legacy_target_size = (in_size + stride - 1) / stride; + int64_t pad_needed = (legacy_target_size - 1) * stride + kernel - in_size; + *pad_head = (pad_needed + 1) / 2; + *pad_tail = pad_needed - *pad_head; + *out_size = (in_size + pad_needed - kernel) / stride + 1; + break; + } + case AutoPadType::SAME_UPPER: { + int64_t legacy_target_size = (in_size + stride - 1) / stride; + int64_t pad_needed = (legacy_target_size - 1) * stride + kernel - in_size; + *pad_head = pad_needed / 2; + *pad_tail = pad_needed - *pad_head; + *out_size = (in_size + pad_needed - kernel) / stride + 1; + break; + } + default: { + ONNXRUNTIME_THROW("Unsupported AutoPad Type."); + } + } + } else { + *out_size = static_cast( + static_cast(in_size + *pad_head + *pad_tail - kernel) / stride + 1); + } + } + + Status Compute(OpKernelContext* context, MLAS_POOLING_KIND kind) const; + + protected: + std::string op_name_; + bool global_pooling_{}; + bool count_include_pad_{}; + int64_t storage_order_{0}; // MaxPool_8 only. 0 is row major, and 1 is column major. Default is 0. + std::vector kernel_shape_; + std::vector pads_; + std::vector strides_; + + AutoPadType auto_pad_; + + inline int64_t stride_h() const { + return global_pooling_ ? 1 : strides_[0]; + } + + inline int64_t stride_w() const { + return global_pooling_ ? 1 : strides_[1]; + } + + inline int64_t stride_d() const { + return global_pooling_ ? 1 : strides_[2]; + } +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/roi_pool.cc b/onnxruntime/core/providers/cpu/nn/roi_pool.cc new file mode 100644 index 0000000000000..4ab1b0bd03bfc --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/roi_pool.cc @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/nn/roi_pool.h" +#include + +namespace onnxruntime { +ONNX_CPU_OPERATOR_KERNEL( + MaxRoiPool, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + RoiPool); + +template <> +Status RoiPool::Compute(OpKernelContext* context) const { + const Tensor* X = context->Input(0); + const Tensor* R = context->Input(1); + if (X == nullptr || R == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + + int batch_size = static_cast(X->Shape()[0]); + int channels = static_cast(X->Shape()[1]); + int height = static_cast(X->Shape()[2]); + int width = static_cast(X->Shape()[3]); + int num_rois = static_cast(R->Shape()[0]); + + // Each ROI is of the form [batch_index x1 y1 x2 y2] + ONNXRUNTIME_ENFORCE(R->Shape()[1] == 5); + + std::vector output_dims({num_rois, channels, pooled_height_, pooled_width_}); + + Tensor* Y = context->Output(0, TensorShape(output_dims)); + + const float* Xdata = X->template Data(); + const float* rois = R->template Data(); + + float* Ydata = Y->template MutableData(); + + for (int n = 0; n < num_rois; n++) { + int roi_batch_id = static_cast(rois[0]); + int roi_start_w = static_cast(round(rois[1] * spatial_scale_)); + int roi_start_h = static_cast(round(rois[2] * spatial_scale_)); + int roi_end_w = static_cast(round(rois[3] * spatial_scale_)); + int roi_end_h = static_cast(round(rois[4] * spatial_scale_)); + ONNXRUNTIME_ENFORCE(roi_batch_id >= 0); + ONNXRUNTIME_ENFORCE(roi_batch_id < batch_size); + + // Force malformed ROIs to be 1x1 + int roi_height = std::max(roi_end_h - roi_start_h + 1, 1); + int roi_width = std::max(roi_end_w - roi_start_w + 1, 1); + + const float bin_size_h = + static_cast(roi_height) / static_cast(pooled_height_); + const float bin_size_w = + static_cast(roi_width) / static_cast(pooled_width_); + + const float* batch_data = Xdata + roi_batch_id * X->Shape().SizeFromDimension(1); + + for (int c = 0; c < channels; ++c) { + for (int ph = 0; ph < pooled_height_; ++ph) { + for (int pw = 0; pw < pooled_width_; ++pw) { + // Compute pooling region for this output unit: + // start (included) = floor(ph * roi_height / pooled_height_) + // end (excluded) = ceil((ph + 1) * roi_height / pooled_height_) + int hstart = + static_cast(floor(static_cast(ph) * bin_size_h)); + int wstart = + static_cast(floor(static_cast(pw) * bin_size_w)); + int hend = + static_cast(ceil(static_cast(ph + 1) * bin_size_h)); + int wend = + static_cast(ceil(static_cast(pw + 1) * bin_size_w)); + + // Add roi offsets and clip to input boundaries + hstart = std::min(std::max(hstart + roi_start_h, 0), height); + hend = std::min(std::max(hend + roi_start_h, 0), height); + wstart = std::min(std::max(wstart + roi_start_w, 0), width); + wend = std::min(std::max(wend + roi_start_w, 0), width); + + const int pool_index = static_cast(ph * pooled_width_ + pw); + + // Define an empty pooling region to be zero + bool is_empty = (hend <= hstart) || (wend <= wstart); + Ydata[pool_index] = is_empty ? 0 : std::numeric_limits::min(); + + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { + const int index = h * width + w; + Ydata[pool_index] = std::max(batch_data[index], Ydata[pool_index]); + } + } + } + } + // Increment all data pointers by one channel + batch_data += X->Shape().SizeFromDimension(2); + Ydata += Y->Shape().SizeFromDimension(2); + } + // Increment ROI data pointer + rois += R->Shape().SizeFromDimension(1); + } + + return Status::OK(); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/roi_pool.h b/onnxruntime/core/providers/cpu/nn/roi_pool.h new file mode 100644 index 0000000000000..73e775cc00368 --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/roi_pool.h @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cpu/nn/autopad_type.h" + +namespace onnxruntime { + +template +class RoiPool : public OpKernel { + public: + RoiPool(const OpKernelInfo& info) : OpKernel(info) { + std::vector pooled_shape; + ONNXRUNTIME_ENFORCE(info.GetAttrs("pooled_shape", pooled_shape).IsOK()); + ONNXRUNTIME_ENFORCE(pooled_shape.size() == 2); + + pooled_height_ = pooled_shape[0]; + pooled_width_ = pooled_shape[1]; + ONNXRUNTIME_ENFORCE(pooled_height_ > 0); + ONNXRUNTIME_ENFORCE(pooled_width_ > 0); + + ONNXRUNTIME_ENFORCE(info.GetAttr("spatial_scale", &spatial_scale_).IsOK()); + ONNXRUNTIME_ENFORCE(spatial_scale_ > 0); + } + + ~RoiPool() override = default; + + Status Compute(OpKernelContext* context) const override; + + protected: + int64_t pooled_height_, pooled_width_; + float spatial_scale_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/reduction/reduction_ops.cc b/onnxruntime/core/providers/cpu/reduction/reduction_ops.cc new file mode 100644 index 0000000000000..ae17d9c563ac4 --- /dev/null +++ b/onnxruntime/core/providers/cpu/reduction/reduction_ops.cc @@ -0,0 +1,383 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/reduction/reduction_ops.h" +#include "core/util/math_cpuonly.h" +using namespace std; +namespace onnxruntime { + +#define REGISTER_UNARY_ELEMENTWISE_KERNEL(x, sinceVersion) \ + ONNX_CPU_OPERATOR_TYPED_KERNEL( \ + x, \ + sinceVersion, \ + float, \ + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + x); \ + \ + ONNX_CPU_OPERATOR_TYPED_KERNEL( \ + x, \ + sinceVersion, \ + int32_t, \ + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + x); + +REGISTER_UNARY_ELEMENTWISE_KERNEL(ReduceL1, 1); +REGISTER_UNARY_ELEMENTWISE_KERNEL(ReduceL2, 1); +REGISTER_UNARY_ELEMENTWISE_KERNEL(ReduceLogSum, 1); +REGISTER_UNARY_ELEMENTWISE_KERNEL(ReduceLogSumExp, 1); +REGISTER_UNARY_ELEMENTWISE_KERNEL(ReduceMax, 1); +REGISTER_UNARY_ELEMENTWISE_KERNEL(ReduceMean, 1); +REGISTER_UNARY_ELEMENTWISE_KERNEL(ReduceMin, 1); +REGISTER_UNARY_ELEMENTWISE_KERNEL(ReduceProd, 1); +REGISTER_UNARY_ELEMENTWISE_KERNEL(ReduceSum, 1); +REGISTER_UNARY_ELEMENTWISE_KERNEL(ReduceSumSquare, 1); +REGISTER_UNARY_ELEMENTWISE_KERNEL(ArgMax, 1); +REGISTER_UNARY_ELEMENTWISE_KERNEL(ArgMin, 1); + +template +void PrepareForReduce(OpKernelContext* ctx, + std::vector& transposedInputData, + Tensor** reducedTensor, + int64_t& block_size, + int64_t& blocks, + const std::vector& axes_, + bool keepdims_) { + const Tensor* input_tensor_ptr = ctx->Input(0); + ONNXRUNTIME_ENFORCE(input_tensor_ptr != nullptr); + const Tensor& input = *input_tensor_ptr; + + size_t ndim = input.Shape().GetDims().size(); + for (int64_t axe : axes_) { + ONNXRUNTIME_ENFORCE(axe >= 0 && axe < (int64_t)ndim, "Axis attribute out of range"); + } + + transposedInputData.resize(input.Shape().Size(), 0); + + std::vector axes = axes_; + if (axes.empty()) { + // This is the default case for non-arg kind reductions. Reduce on all dimensions. + for (int i = 0; i < ndim; i++) + axes.push_back(i); + } + + std::sort(axes.begin(), axes.end()); + + vector keep_axis(ndim, true); + for (auto i : axes) { + keep_axis[i] = false; + } + + //transpose the input so that all to-be-reduced axes are at the head + vector transposed_axes(axes.begin(), axes.end()); + for (int i = 0; i < ndim; ++i) { + if (keep_axis[i]) { + transposed_axes.push_back(i); + } + } + + vector new_dims_(transposed_axes.size()); + for (int i = 0; i < transposed_axes.size(); ++i) { + new_dims_[i] = input.Shape().GetDims().at(transposed_axes[i]); + } + + int num_axes = static_cast(transposed_axes.size()); + auto in_dims = input.Shape().GetDims(); + + // Measure amount of contiguous data we can copy at once + int64_t blocksize = 1; + int n_shared_idxs = 0; + for (int i = num_axes - 1; i >= 0; --i) { + if (transposed_axes[i] == i) { + blocksize *= new_dims_[i]; + ++n_shared_idxs; + } else { + break; + } + } + + const T* from_data = input.template Data(); + T* to_data = &transposedInputData[0]; + size_t count = input.Shape().Size(); + + //set to-be-reduced axes to one. squeeze is keepdims_ is false + int64_t first_dim = 1; + std::vector reduced_dims; + for (int i = 0; i < in_dims.size(); i++) { + if (keep_axis[i]) { + reduced_dims.push_back(in_dims[i]); + } else { + first_dim *= in_dims[i]; + if (keepdims_) { + reduced_dims.push_back(1); + } + } + } + + *reducedTensor = ctx->Output(0, reduced_dims); + block_size = input.Shape().Size() / first_dim; + blocks = first_dim; + + if (num_axes < 2 || n_shared_idxs == num_axes) { + memcpy(to_data, from_data, count * sizeof(T)); + return; + } + + int itr_axes = num_axes - n_shared_idxs; + + // Calculate strides + std::vector stride_x(itr_axes, 0); + for (size_t i = 0; i < itr_axes; i++) { + stride_x[i] = 1; + for (size_t j = transposed_axes[i] + 1; j < itr_axes; j++) { + stride_x[i] *= in_dims[j]; + } + } + + std::vector itr_idxs(itr_axes, 0); + + // Branch here to avoid branching within the loop + if (blocksize > 1) { + for (size_t index = 0; index < (count / blocksize); index++) { + int64_t from_index = 0; + for (int i = 0; i < itr_axes; ++i) { + from_index += stride_x[i] * itr_idxs[i]; + } + + memcpy( + to_data + blocksize * index, + from_data + blocksize * from_index, + blocksize * sizeof(T)); + + ++itr_idxs[itr_axes - 1]; + for (int i = itr_axes - 1; i >= 1; --i) { + auto expected_dim = new_dims_[i]; + if (itr_idxs[i] < expected_dim) { + break; + } + itr_idxs[i] %= expected_dim; + ++itr_idxs[i - 1]; + } + } + } else { + for (size_t index = 0; index < count; index++) { + int64_t from_index = 0; + for (int i = 0; i < itr_axes; ++i) { + from_index += stride_x[i] * itr_idxs[i]; + } + + *(to_data + index) = *(from_data + from_index); + + ++itr_idxs[itr_axes - 1]; + for (int i = itr_axes - 1; i >= 1; --i) { + auto expected_dim = new_dims_[i]; + if (itr_idxs[i] < expected_dim) { + break; + } + itr_idxs[i] %= expected_dim; + ++itr_idxs[i - 1]; + } + } + } +} + +template +Status ReduceL1::Compute(OpKernelContext* ctx) const { + std::vector transposedInputData; + int64_t block_size, blocks; + Tensor* reduced; + PrepareForReduce(ctx, transposedInputData, &reduced, block_size, blocks, axes_, keepdims_); + + T* output_data = reduced->template MutableData(); + + EigenVectorMap out_vec(output_data, block_size); + out_vec = ConstEigenMatrixMap(&transposedInputData[0], block_size, blocks).cwiseAbs().rowwise().sum(); + + return Status::OK(); +} + +template +Status ReduceL2::Compute(OpKernelContext* ctx) const { + std::vector transposedInputData; + int64_t block_size, blocks; + Tensor* reduced; + PrepareForReduce(ctx, transposedInputData, &reduced, block_size, blocks, axes_, keepdims_); + + T* output_data = reduced->template MutableData(); + + EigenVectorMap out_vec(output_data, block_size); + out_vec = ConstEigenMatrixMap(&transposedInputData[0], block_size, blocks).rowwise().norm(); + + return Status::OK(); +} + +template +Status ReduceLogSum::Compute(OpKernelContext* ctx) const { + std::vector transposedInputData; + int64_t block_size, blocks; + Tensor* reduced; + PrepareForReduce(ctx, transposedInputData, &reduced, block_size, blocks, axes_, keepdims_); + + T* output_data = reduced->template MutableData(); + + EigenVectorMap out_vec(output_data, block_size); + out_vec = ConstEigenMatrixMap(&transposedInputData[0], block_size, blocks).rowwise().sum(); + for (int j = 0; j < block_size; ++j) { + *(output_data) = static_cast(std::log(*(output_data))); + ++output_data; + } + + return Status::OK(); +} + +template +Status ReduceLogSumExp::Compute(OpKernelContext* ctx) const { + std::vector transposedInputData; + int64_t block_size, blocks; + Tensor* reduced; + PrepareForReduce(ctx, transposedInputData, &reduced, block_size, blocks, axes_, keepdims_); + + T* output_data = reduced->template MutableData(); + + for (int j = 0; j < block_size; ++j) { + T max_value = std::numeric_limits::lowest(); + for (int i = 0; i < blocks; ++i) { + max_value = std::max(max_value, transposedInputData[i * block_size + j]); + } + T scaled_exp_sum = 0; + for (int i = 0; i < blocks; ++i) { + scaled_exp_sum += static_cast(std::exp(transposedInputData[i * block_size + j] - max_value)); + } + *(output_data++) = static_cast(std::log(scaled_exp_sum) + max_value); + } + return Status::OK(); +} + +template +Status ReduceMax::Compute(OpKernelContext* ctx) const { + std::vector transposedInputData; + int64_t block_size, blocks; + Tensor* reduced; + PrepareForReduce(ctx, transposedInputData, &reduced, block_size, blocks, axes_, keepdims_); + + T* output_data = reduced->template MutableData(); + + EigenVectorMap out_vec(output_data, block_size); + out_vec = ConstEigenMatrixMap(&transposedInputData[0], block_size, blocks).rowwise().maxCoeff(); + + return Status::OK(); +} + +template +Status ReduceMean::Compute(OpKernelContext* ctx) const { + std::vector transposedInputData; + int64_t block_size, blocks; + Tensor* reduced; + PrepareForReduce(ctx, transposedInputData, &reduced, block_size, blocks, axes_, keepdims_); + + T* output_data = reduced->template MutableData(); + + EigenVectorMap out_vec(output_data, block_size); + out_vec = ConstEigenMatrixMap(&transposedInputData[0], block_size, blocks).rowwise().mean(); + + return Status::OK(); +} + +template +Status ReduceMin::Compute(OpKernelContext* ctx) const { + std::vector transposedInputData; + int64_t block_size, blocks; + Tensor* reduced; + PrepareForReduce(ctx, transposedInputData, &reduced, block_size, blocks, axes_, keepdims_); + + T* output_data = reduced->template MutableData(); + + EigenVectorMap out_vec(output_data, block_size); + out_vec = ConstEigenMatrixMap(&transposedInputData[0], block_size, blocks).rowwise().minCoeff(); + + return Status::OK(); +} + +template +Status ReduceProd::Compute(OpKernelContext* ctx) const { + std::vector transposedInputData; + int64_t block_size, blocks; + Tensor* reduced; + PrepareForReduce(ctx, transposedInputData, &reduced, block_size, blocks, axes_, keepdims_); + + T* output_data = reduced->template MutableData(); + + EigenVectorMap out_vec(output_data, block_size); + out_vec = ConstEigenMatrixMap(&transposedInputData[0], block_size, blocks).rowwise().prod(); + + return Status::OK(); +} + +template +Status ReduceSum::Compute(OpKernelContext* ctx) const { + std::vector transposedInputData; + int64_t block_size, blocks; + Tensor* reduced; + PrepareForReduce(ctx, transposedInputData, &reduced, block_size, blocks, axes_, keepdims_); + + T* output_data = reduced->template MutableData(); + + EigenVectorMap out_vec(output_data, block_size); + out_vec = ConstEigenMatrixMap(&transposedInputData[0], block_size, blocks).rowwise().sum(); + + return Status::OK(); +} + +template +Status ReduceSumSquare::Compute(OpKernelContext* ctx) const { + std::vector transposedInputData; + int64_t block_size, blocks; + Tensor* reduced; + PrepareForReduce(ctx, transposedInputData, &reduced, block_size, blocks, axes_, keepdims_); + + T* output_data = reduced->template MutableData(); + + EigenVectorMap out_vec(output_data, block_size); + out_vec = ConstEigenMatrixMap(&transposedInputData[0], block_size, blocks).rowwise().squaredNorm(); + + return Status::OK(); +} + +template +Status ArgMax::Compute(OpKernelContext* ctx) const { + std::vector transposedInputData; + int64_t block_size, blocks; + Tensor* reduced; + PrepareForReduce(ctx, transposedInputData, &reduced, block_size, blocks, axes_, keepdims_); + + int64_t* output_data = reduced->template MutableData(); + + Eigen::MatrixXf::Index maxIndex; + auto matrixData = ConstEigenMatrixMap(&transposedInputData[0], block_size, blocks); + for (int i = 0; i < block_size; ++i) { + matrixData.row(i).maxCoeff(&maxIndex); + *(output_data++) = maxIndex; + } + + return Status::OK(); +} + +template +Status ArgMin::Compute(OpKernelContext* ctx) const { + std::vector transposedInputData; + int64_t block_size, blocks; + Tensor* reduced; + PrepareForReduce(ctx, transposedInputData, &reduced, block_size, blocks, axes_, keepdims_); + + int64_t* output_data = reduced->template MutableData(); + + Eigen::MatrixXf::Index minIndex; + auto matrixData = ConstEigenMatrixMap(&transposedInputData[0], block_size, blocks); + for (int i = 0; i < block_size; ++i) { + matrixData.row(i).minCoeff(&minIndex); + *(output_data++) = minIndex; + } + + return Status::OK(); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/reduction/reduction_ops.h b/onnxruntime/core/providers/cpu/reduction/reduction_ops.h new file mode 100644 index 0000000000000..9c3b986c10052 --- /dev/null +++ b/onnxruntime/core/providers/cpu/reduction/reduction_ops.h @@ -0,0 +1,147 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#ifndef CORE_PROVIDERS_CPU_REDUCTION_OPS_H +#define CORE_PROVIDERS_CPU_REDUCTION_OPS_H + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { + +template +class ReduceKernelBase { + protected: + ReduceKernelBase(const OpKernelInfo& info) { + if (allow_multi_axes) { + axes_ = info.GetAttrsOrDefault("axes"); + } else { + int64_t v = info.GetAttrOrDefault("axis", 0); + axes_.push_back(v); + } + int64_t keepdims = 1; + ONNXRUNTIME_ENFORCE(info.GetAttr("keepdims", &keepdims).IsOK()); + keepdims_ = (keepdims == 1); + } + + std::vector axes_; + bool keepdims_; +}; + +template +class ReduceKernel : public OpKernel, public ReduceKernelBase { + protected: + ReduceKernel(const OpKernelInfo& info) : OpKernel(info), ReduceKernelBase(info) {} +}; + +template +class ReduceL1 final : public ReduceKernel { + public: + ReduceL1(const OpKernelInfo& info) : ReduceKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class ReduceL2 final : public ReduceKernel { + public: + ReduceL2(const OpKernelInfo& info) : ReduceKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class ReduceLogSum final : public ReduceKernel { + public: + ReduceLogSum(const OpKernelInfo& info) : ReduceKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class ReduceLogSumExp final : public ReduceKernel { + public: + ReduceLogSumExp(const OpKernelInfo& info) : ReduceKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class ReduceMax final : public ReduceKernel { + public: + ReduceMax(const OpKernelInfo& info) : ReduceKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class ReduceMean final : public ReduceKernel { + public: + ReduceMean(const OpKernelInfo& info) : ReduceKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class ReduceMin final : public ReduceKernel { + public: + ReduceMin(const OpKernelInfo& info) : ReduceKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class ReduceProd final : public ReduceKernel { + public: + ReduceProd(const OpKernelInfo& info) : ReduceKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class ReduceSum final : public ReduceKernel { + public: + ReduceSum(const OpKernelInfo& info) : ReduceKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class ReduceSumSquare final : public ReduceKernel { + public: + ReduceSumSquare(const OpKernelInfo& info) : ReduceKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class ArgMax final : public ReduceKernel { + public: + ArgMax(const OpKernelInfo& info) : ReduceKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class ArgMin final : public ReduceKernel { + public: + ArgMin(const OpKernelInfo& info) : ReduceKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +} // namespace onnxruntime + +#endif // !CORE_PROVIDERS_CPU_REDUCTION_OPS_H diff --git a/onnxruntime/core/providers/cpu/rnn/deep_cpu_gru.cc b/onnxruntime/core/providers/cpu/rnn/deep_cpu_gru.cc new file mode 100644 index 0000000000000..9be1d1ffd05fa --- /dev/null +++ b/onnxruntime/core/providers/cpu/rnn/deep_cpu_gru.cc @@ -0,0 +1,1098 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// there's no way to use a raw pointer as the copy destination with std::copy_n +// (which gsl::copy uses with span::data() which returns a raw pointer) with the 14.11 toolset +// without generating a 4996 warning. going through an iterator is way too much overhead so turn off the warning. +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4996) +#endif + +#include "core/providers/cpu/rnn/deep_cpu_gru.h" + +#include +#include +#include + +#include "core/common/logging/logging.h" +#include "core/framework/allocator.h" +#include "core/framework/tensor.h" + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +/* +ONNX_OPERATOR_SCHEMA(GRU) + .SetDoc(R"DOC( +Computes an one-layer GRU. This operator is usually supported via some custom +implementation such as CuDNN. + +Notations: + +`X` - input tensor +`z` - update gate +`r` - reset gate +`h` - hidden gate +`t` - time step (t-1 means previous time step) +`W[zrh]` - W parameter weight matrix for update, reset, and hidden gates +`R[zrh]` - R recurrence weight matrix for update, reset, and hidden gates +`Wb[zrh]` - W bias vectors for update, reset, and hidden gates +`Rb[zrh]` - R bias vectors for update, reset, and hidden gates +`WB[zrh]` - W parameter weight matrix for backward update, reset, and hidden gates +`RB[zrh]` - R recurrence weight matrix for backward update, reset, and hidden gates +`WBb[zrh]` - W bias vectors for backward update, reset, and hidden gates +`RBb[zrh]` - R bias vectors for backward update, reset, and hidden gates +`H` - Hidden state +`num_directions` - 2 if direction == bidirectional else 1 + +Activation functions: + + Relu(x) - max(0, x) + Tanh(x) - (1 - e^{-2x})/(1 + e^{-2x}) + Sigmoid(x) - 1/(1 + e^{-x}) + + (NOTE: Below are optional) + Affine(x) - alpha*x + beta + LeakyRelu(x) - x if x >= 0 else alpha * x + ThresholdedRelu(x) - x if x >= alpha else 0 + ScaledTanh(x) - alpha*Tanh(beta*x) + HardSigmoid(x) - min(max(alpha*x + beta, 0), 1) + Elu(x) - x if x >= 0 else alpha*(e^x - 1) + Softsign(x) - x/(1 + |x|) + Softplus(x) - log(1 + e^x) + +Equations (Default: f=Sigmoid, g=Tanh): + - zt = f(Xt*(Wz^T) + Ht-1*Rz + Wbz + Rbz) + - rt = f(Xt*(Wr^T) + Ht-1*Rr + Wbr + Rbr) + - ht = g(Xt*(Wh^T) + (rt (.) Ht-1)*Rh + Rbh + Wbh) # default, when linear_before_reset = 0 + - ht = g(Xt*(Wh^T) + (rt (.) (Ht-1*Rh + Rbh) + Wbh) # when linear_before_reset != 0 + - Ht = (1 - zt) (.) ht + zt (.) Ht-1 +)DOC") + .SinceVersion(3) + .Attr("direction", "Specify if the RNN is forward, reverse, or bidirectional. " + "Must be one of forward (default), reverse, or bidirectional.", + AttributeProto::STRING, + std::string("forward")) + .Attr("hidden_size", "Number of neurons in the hidden layer", AttributeProto::INT, OPTIONAL) + .Attr("activations", "A list of 2 (or 4 if bidirectional) activation functions " + "for update, reset, and hidden gates. The activation functions must be one " + "of the activation functions specified above. Optional: See the equations " + "for default if not specified.", + AttributeProto::STRINGS, + OPTIONAL) + .Attr("activation_alpha", + "Optional scaling values used by some activation functions. The values " + "are consumed in the order of activation functions, for example (f, g, h) " + "in LSTM.", + AttributeProto::FLOATS, + OPTIONAL) + .Attr("activation_beta", + "Optional scaling values used by some activation functions. The values " + "are consumed in the order of activation functions, for example (f, g, h) " + "in LSTM.", + AttributeProto::FLOATS, + OPTIONAL) + .Attr("output_sequence", + "The sequence output for the hidden is optional if 0. Default 0.", + AttributeProto::INT, + static_cast(0)); + .Attr("clip", "Cell clip threshold. Clipping bounds the elements of a tensor " + "in the range of [-threshold, +threshold] and is applied to the input " + "of activations. No clip if not specified.", AttributeProto::FLOAT, OPTIONAL) + .Attr("linear_before_reset", "When computing the output of the hidden gate, " + "apply the linear transformation before multiplying by the output of the " + "reset gate.", + AttributeProto::INT, + static_cast(0)) + .Input(0, "X", + "The input sequences packed (and potentially padded) into one 3-D " + "tensor with the shape of `[seq_length, batch_size, input_size]`.", "T") + .Input(1, "W", + "The weight tensor for the gates. Concatenation of `W[zrh]` and `WB[zrh]` " + "(if bidirectional) along dimension 0. This tensor has shape " + "`[num_directions, 3*hidden_size, input_size]`.", "T") + .Input(2, "R", + "The recurrence weight tensor. Concatenation of `R[zrh]` and `RB[zrh]` " + "(if bidirectional) along dimension 0. This tensor has shape " + "`[num_directions, 3*hidden_size, hidden_size]`.", "T") + .Input(3, "B", + "The bias tensor for the gates. Concatenation of `[Wb[zrh], Rb[zrh]]` and " + "`[WBb[zrh], RBb[zrh]]` (if bidirectional) along dimension 0. This tensor " + "has shape `[num_directions, 6*hidden_size]`. Optional: If not specified " + "- assumed to be 0", "T", + OpSchema::Optional) + .Input(4, "sequence_lens", + "Optional tensor specifying lengths of the sequences in a batch. " + "If not specified - assumed all sequences in the batch to have " + "length `seq_length`. It has shape `[batch_size]`.", "T1", + OpSchema::Optional) + .Input(5, "initial_h", + "Optional initial value of the hidden. If not specified - assumed " + "to be 0. It has shape `[num_directions, batch_size, hidden_size]`.", + " T", OpSchema::Optional) + .Output(0, "Y", + "A tensor that concats all the intermediate output values of the hidden. " + "It has shape `[seq_length, num_directions, batch_size, hidden_size]`. ", + "T", OpSchema::Optional); + .Output(1, "Y_h", + "The last output value of the hidden. It has shape " + "`[num_directions, batch_size, hidden_size]`.", "T", OpSchema::Optional); + .TypeConstraint("T", { "tensor(float16)", "tensor(float)", "tensor(double)" }, + "Constrain input and output types to float tensors.") + .TypeConstraint("T1", { "tensor(int32)" }, "Constrain seq_lens to integer tensor."); +*/ + +namespace onnxruntime { + +ONNX_CPU_OPERATOR_KERNEL( + GRU, + 7, + KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}) + .TypeConstraint("T1", DataTypeImpl::GetTensorType()), + DeepCpuGruOp); + +using namespace rnn::detail; + +// internal helper code +namespace detail { + +/// The class represents DeepCPU implementation of a gated recurrent unit (GRU) operator. +/// For details, refer to http://aka.ms/dl-optimization/. +template +class UniDirectionalGru { + public: + UniDirectionalGru(AllocatorPtr allocator, + const logging::Logger& logger, + const int seq_length, + const int batch_size, + const int input_size, + const int hidden_size, + const bool linear_before_reset, + Direction direction, + const gsl::span& bias, + const gsl::span& initial_hidden_state, + const ActivationFuncs::Entry& activation_func_f, + const ActivationFuncs::Entry& activation_func_g, + const float clip, + TaskThreadPool& ttp); + + void Compute(const gsl::span& inputs, + const gsl::span& sequence_lengths, + const int num_directions, + const gsl::span& input_weights, + const gsl::span& recurrent_weights, + gsl::span& outputs, + gsl::span& final_hidden_state); + + ~UniDirectionalGru() = default; + + private: + AllocatorPtr allocator_; + const logging::Logger& logger_; + TaskThreadPool& ttp_; + + int seq_length_; + int batch_size_; + int input_size_; + int hidden_size_; + bool linear_before_reset_; + + const float clip_; + + Direction direction_; + bool use_bias_; + bool batch_parallel_; + + int hidden_num_threads_ = -1; + + IAllocatorUniquePtr outputZRH_ptr_; + gsl::span outputZRH_; + + IAllocatorUniquePtr cur_h_ptr_; + IAllocatorUniquePtr batched_hidden0_ptr_; + IAllocatorUniquePtr sequence_lengths_ptr_; + gsl::span cur_h_; + gsl::span batched_hidden0_; + gsl::span sequence_lengths_; + + // Wb[zr] and Rb[zr] can always be added together upfront, and repeated to match the batch size for + // faster GEMM calculations, so these two members are all the + // Wb[z] + Rb[z] values added together, repeated batch_size_ times + IAllocatorUniquePtr batched_bias_WRz_ptr_, batched_bias_WRr_ptr_; + gsl::span batched_bias_WRz_, batched_bias_WRr_; + + // Wbh and Rbh can only be combined upfront if linear_before_reset_ is false + IAllocatorUniquePtr batched_bias_WRh_ptr_; + gsl::span batched_bias_WRh_; + + // if linear_before_reset_ is true, we need to setup Wbh and Rbh separately + IAllocatorUniquePtr batched_bias_Wh_ptr_, batched_bias_Rh_ptr_; + gsl::span batched_bias_Wh_, batched_bias_Rh_; + + IAllocatorUniquePtr linear_output_ptr_; + gsl::span linear_output_; + + IAllocatorUniquePtr inputs_reverse_ptr_; + IAllocatorUniquePtr outputs_reverse_ptr_; + gsl::span inputs_reverse_; + gsl::span outputs_reverse_; + + deepcpu::ClipWithBiasFuncPtr clip_with_bias_ptr_ = nullptr; + + float zr_alpha_ = 0.f, zr_beta_ = 0.f; + float h_alpha_ = 0.f, h_beta_ = 0.f; + + deepcpu::GruResetGateFuncPtr reset_gate_ = nullptr; + deepcpu::ActivationFuncPtr update_gate_ = nullptr; + deepcpu::GruOutputGateFuncPtr output_gate_ = nullptr; + + void AllocateBuffers(); + void SetNumThreads(); +}; +} // namespace detail + +// #define DUMP_MATRIXES to provide lots of diagnostic output +#if defined(DUMP_MATRIXES) +#define DumpMatrix(...) ::onnxruntime::rnn::detail::DumpMatrixImpl(__VA_ARGS__) +#else +#define DumpMatrix(...) ((void)0) +#endif + +Status DeepCpuGruOp::Compute(OpKernelContext* context) const { + const Tensor& X = *context->Input(0); // inputs. [seq_length, batch_size, input_size] + + Status status; + // auto& logger = context->Logger(); + + auto data_type = X.DataType(); + if (data_type == DataTypeImpl::GetType()) + status = ComputeImpl(*context); + else if (data_type == DataTypeImpl::GetType()) { + /* Need to update all the helpers to support double... + status = ComputeImpl(*context); */ + ONNXRUNTIME_NOT_IMPLEMENTED("GRU operator does not support double yet"); + } else + ONNXRUNTIME_THROW("Invalid data type for GRU operator of ", data_type); + + return status; +} + +template +Status DeepCpuGruOp::ComputeImpl(OpKernelContext& context) const { + auto& logger = context.Logger(); + + const Tensor& X = *context.Input(0); // inputs. [seq_length, batch_size, input_size] + const Tensor& W = *context.Input(1); // weights. [num_directions, 3*hidden_size, input_size] + const Tensor& R = *context.Input(2); // recurrence weights. [num_directions, 3*hidden_size, hidden_size] + + // optional + const Tensor* B = context.Input(3); // bias. [num_directions, 6*hidden_size] + const Tensor* sequence_lens = context.Input(4); // [batch_size] + const Tensor* initial_h = context.Input(5); // initial hidden. [num_directions, batch_size, hidden_size] + + auto& X_shape = X.Shape(); + + int seq_length = gsl::narrow(X_shape[0]); + int batch_size = gsl::narrow(X_shape[1]); + int input_size = gsl::narrow(X_shape[2]); + + auto status = ValidateCommonRnnInputs(X, W, R, B, 3, sequence_lens, initial_h, num_directions_, hidden_size_); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + // GRU outputs are optional but must be in the same order + TensorShape Y_dims{seq_length, num_directions_, batch_size, hidden_size_}; + Tensor* Y = context.Output(/*index*/ 0, Y_dims); // TODO: Adjust for however optional outputs gets implemented + + TensorShape Y_h_dims{num_directions_, batch_size, hidden_size_}; + Tensor* Y_h = context.Output(/*index*/ 1, Y_h_dims); + + AllocatorPtr alloc; + status = context.GetTempSpaceAllocator(&alloc); + ONNXRUNTIME_RETURN_IF_ERROR(status); + gsl::span input_weights = W.DataAsSpan(); + gsl::span recurrent_weights = R.DataAsSpan(); + gsl::span bias = B != nullptr ? B->DataAsSpan() : gsl::span(); + + // spans for first direction + const size_t input_weights_size_per_direction = 3 * hidden_size_ * input_size; + const size_t recurrent_weights_size_per_direction = 3 * hidden_size_ * hidden_size_; + const size_t bias_size_per_direction = 6 * hidden_size_; + + gsl::span input_weights_1 = input_weights.subspan(0, input_weights_size_per_direction); + gsl::span recurrent_weights_1 = recurrent_weights.subspan(0, recurrent_weights_size_per_direction); + gsl::span bias_1 = bias.empty() ? bias : bias.subspan(0, bias_size_per_direction); + + gsl::span input = X.DataAsSpan(); + gsl::span sequence_lens_span = sequence_lens != nullptr ? sequence_lens->DataAsSpan() + : gsl::span(); + + const size_t initial_hidden_size_per_direction = batch_size * hidden_size_; + gsl::span initial_hidden = initial_h != nullptr ? initial_h->DataAsSpan() : gsl::span(); + gsl::span initial_hidden_1 = initial_hidden.empty() + ? initial_hidden + : initial_hidden.subspan(0, initial_hidden_size_per_direction); + + // output shape is [seq_length, num_directions, batch_size, hidden_size] + // so it's not a case of all the output for one direction being first. + // due to that we can only easily check that the end of the output for each direction is valid. + const size_t output_size = Y != nullptr ? Y->Shape().Size() : 0; + const size_t per_direction_offset = batch_size * hidden_size_; + gsl::span output = Y != nullptr ? Y->MutableDataAsSpan() : gsl::span(); + gsl::span output_1 = output.empty() + ? output + : output.subspan(0, output_size - (num_directions_ - 1) * per_direction_offset); + + // UniDirectionalGru needs somewhere to write output, so even if we aren't returning Y_h + // we provide an appropriately sized buffer for that purpose. + const size_t hidden_output_size_per_direction = batch_size * hidden_size_; + IAllocatorUniquePtr local_hidden_output; + gsl::span hidden_output = + Y_h ? Y_h->MutableDataAsSpan() + : Allocate(alloc, hidden_output_size_per_direction * num_directions_, local_hidden_output); + + gsl::span hidden_output_1 = hidden_output.subspan(0, hidden_output_size_per_direction); + + if (direction_ == Direction::kBidirectional) { + // spans for second direction + gsl::span input_weights_2 = input_weights.subspan(input_weights_size_per_direction, + input_weights_size_per_direction); + gsl::span recurrent_weights_2 = recurrent_weights.subspan(recurrent_weights_size_per_direction, + recurrent_weights_size_per_direction); + gsl::span bias_2 = bias.empty() ? bias : bias.subspan(bias_size_per_direction, bias_size_per_direction); + + gsl::span initial_hidden_2 = initial_hidden.empty() + ? initial_hidden + : initial_hidden.subspan(initial_hidden_size_per_direction, + initial_hidden_size_per_direction); + gsl::span output_2 = output.empty() + ? output + : output.subspan(per_direction_offset, output_size - per_direction_offset); + + gsl::span hidden_output_2 = hidden_output.subspan(hidden_output_size_per_direction, + hidden_output_size_per_direction); + + // run backward first as it needs an extra reverse sequence operation + std::packaged_task task_bw{ + [&]() { + std::unique_ptr> bw = std::make_unique>( + alloc, logger, + seq_length, batch_size, input_size, hidden_size_, linear_before_reset_, Direction::kReverse, + bias_2, initial_hidden_2, + activation_funcs_.Entries()[2], + activation_funcs_.Entries()[3], + clip_, ttp_); + bw->Compute(input, sequence_lens_span, num_directions_, input_weights_2, recurrent_weights_2, output_2, hidden_output_2); + }}; + auto task_results_bw = task_bw.get_future(); + ttp_.RunTask(std::move(task_bw)); + + std::packaged_task task_fw{ + [&]() { + std::unique_ptr> fw = std::make_unique>( + alloc, logger, + seq_length, batch_size, input_size, hidden_size_, linear_before_reset_, Direction::kForward, + bias_1, initial_hidden_1, + activation_funcs_.Entries()[0], + activation_funcs_.Entries()[1], + clip_, ttp_); + fw->Compute(input, sequence_lens_span, num_directions_, input_weights_1, recurrent_weights_1, output_1, hidden_output_1); + }}; + auto task_results_fw = task_fw.get_future(); + ttp_.RunTask(std::move(task_fw)); + + task_results_fw.get(); + task_results_bw.get(); + } else { + std::unique_ptr> gru_p = std::make_unique>( + alloc, logger, + seq_length, batch_size, input_size, hidden_size_, linear_before_reset_, direction_, + bias_1, initial_hidden_1, + activation_funcs_.Entries()[0], + activation_funcs_.Entries()[1], + clip_, ttp_); + + gru_p->Compute(input, sequence_lens_span, num_directions_, input_weights_1, recurrent_weights_1, output_1, hidden_output_1); + } + + if (!output.empty()) + DumpMatrix("Y", output.data(), seq_length * num_directions_ * batch_size, hidden_size_); + + DumpMatrix("Y_h", hidden_output.data(), num_directions_ * batch_size, hidden_size_); + + return Status::OK(); +} + +// +// Implementation of internal helper code +namespace detail { + +template +UniDirectionalGru::UniDirectionalGru(AllocatorPtr allocator, + const logging::Logger& logger, + const int seq_length, + const int batch_size, + const int input_size, + const int hidden_size, + const bool linear_before_reset, + Direction direction, + const gsl::span& bias, + const gsl::span& initial_hidden_state, + const ActivationFuncs::Entry& activation_func_f, + const ActivationFuncs::Entry& activation_func_g, + const float clip, + TaskThreadPool& ttp) + : allocator_(allocator), + logger_(logger), + ttp_(ttp), + seq_length_(seq_length), + batch_size_(batch_size), + input_size_(input_size), + hidden_size_(hidden_size), + linear_before_reset_(linear_before_reset), + clip_(clip), + direction_(direction), + use_bias_(!bias.empty()) { + // + clip_with_bias_ptr_ = use_bias_ ? deepcpu::clip_add_bias : deepcpu::clip_ignore_bias; + + // setup activation function pointers and alpha/beta values to use with them + reset_gate_ = deepcpu::GruResetGateFuncByName(activation_func_f.name); + update_gate_ = deepcpu::ActivationFuncByName(activation_func_f.name); + output_gate_ = deepcpu::GruOutputGateFuncByName(activation_func_g.name); + + zr_alpha_ = activation_func_f.alpha; + zr_beta_ = activation_func_f.beta; + h_alpha_ = activation_func_g.alpha; + h_beta_ = activation_func_g.beta; + + SetNumThreads(); + AllocateBuffers(); + + if (use_bias_) { + auto bias_Wz = bias.subspan(0 * hidden_size_, hidden_size_); + auto bias_Wr = bias.subspan(1 * hidden_size_, hidden_size_); + auto bias_Wo = bias.subspan(2 * hidden_size_, hidden_size_); + auto bias_Rz = bias.subspan(3 * hidden_size_, hidden_size_); + auto bias_Rr = bias.subspan(4 * hidden_size_, hidden_size_); + auto bias_Ro = bias.subspan(5 * hidden_size_, hidden_size_); + + // add Wb[zr] and Rb[zr] and replicate so we have batch_size_ copies of the result + auto combine_and_replicate = [&](gsl::span& bias_w, + gsl::span& bias_r, + gsl::span& output) { + // add once + for (int i = 0; i < hidden_size_; ++i) { + output[i] = bias_w[i] + bias_r[i]; + } + + // replicate what we just wrote to the start of the output span so we have batch_size_ copies + auto values = output.cbegin(); + ONNXRUNTIME_IGNORE_RETURN_VALUE(RepeatVectorToConstructArray(values, values + hidden_size_, + output.begin() + hidden_size_, // skip the first batch + batch_size_ - 1)); // and replicate batch size - 1 times + }; + + // we can always combine the z and r weights + combine_and_replicate(bias_Wz, bias_Rz, batched_bias_WRz_); + combine_and_replicate(bias_Wr, bias_Rr, batched_bias_WRr_); + + // how we treat the h weight depends on whether linear_before_reset_ is set + if (linear_before_reset_) { + // need to replicate Wb[o] and Rb[o] separately + ONNXRUNTIME_IGNORE_RETURN_VALUE(RepeatVectorToConstructArray(bias_Wo.cbegin(), bias_Wo.cend(), batched_bias_Wh_.begin(), batch_size_)); + ONNXRUNTIME_IGNORE_RETURN_VALUE(RepeatVectorToConstructArray(bias_Ro.cbegin(), bias_Ro.cend(), batched_bias_Rh_.begin(), batch_size_)); + } else { + combine_and_replicate(bias_Wo, bias_Ro, batched_bias_WRh_); + } + } + + if (!initial_hidden_state.empty()) { + gsl::copy(initial_hidden_state, batched_hidden0_); + } +} + +template +void UniDirectionalGru::Compute(const gsl::span& inputs_arg, + const gsl::span& sequence_lengths_arg, + const int num_directions, + const gsl::span& input_weights, + const gsl::span& recurrent_weights, + gsl::span& outputs, + gsl::span& final_hidden_state) { + using span_T_const_iter = typename gsl::span::const_iterator; + using span_T_iter = typename gsl::span::iterator; + + // copy inputs_arg as we may change it to point to inputs_reverse_ + gsl::span inputs = inputs_arg; + gsl::span sequence_lengths = sequence_lengths_arg; + + // if sequence lengths weren't provided, use internal array and init all to seq_length + if (sequence_lengths.empty()) { + sequence_lengths_ = Allocate(allocator_, batch_size_, sequence_lengths_ptr_, true, seq_length_); + sequence_lengths = sequence_lengths_; + } + + DumpMatrix("Inputs", inputs.data(), seq_length_ * batch_size_, input_size_); + DumpMatrix("input_weights", input_weights.data(), 3 * hidden_size_, input_size_); + DumpMatrix("recurrent_weights", recurrent_weights.data(), 3 * hidden_size_, hidden_size_); + + gsl::span recurrent_weightsZR = recurrent_weights.subspan(0, 2 * hidden_size_ * hidden_size_); + gsl::span recurrent_weightsH = recurrent_weights.subspan(2 * hidden_size_ * hidden_size_, hidden_size_ * hidden_size_); + + gsl::span original_outputs = outputs; + const bool output_sequence = !outputs.empty(); + + if (direction_ == kReverse) { + ReverseSequence(inputs, inputs_reverse_, sequence_lengths, seq_length_, batch_size_, input_size_, 1); + // DumpMatrix("Reversed inputs", inputs_reverse_.data(), seq_length_ * batch_size_, input_size_); + + inputs = inputs_reverse_; + + if (output_sequence) { + outputs = outputs_reverse_; + } + } + + // Calculate the max and min length + int32_t max_sequence_length = *std::max_element(sequence_lengths.cbegin(), sequence_lengths.cend()); + int32_t min_sequence_length = std::min(seq_length_, *std::min_element(sequence_lengths.cbegin(), + sequence_lengths.cend())); + + const int hidden_size_x2 = 2 * hidden_size_; + const int hidden_size_x3 = 3 * hidden_size_; + const int total_rows = max_sequence_length * batch_size_; + + float alpha = 1.0f; + float beta = 0.0f; // zero out outputZRH_ when calling ComputeGemm. + + // apply weights to all the inputs + ComputeGemm(total_rows, hidden_size_x3, input_size_, alpha, + inputs.cbegin(), inputs.cend(), + input_size_, + input_weights.cbegin(), input_weights.cend(), + input_size_, beta, + outputZRH_.begin(), outputZRH_.end(), + hidden_size_x3); + + DumpMatrix("inputs with weights applied", outputZRH_.data(), seq_length_ * batch_size_ * 3, hidden_size_); + + // set to 1 so the weighted inputs in outputZRH_ are added to the result in the next call to ComputeGemm + beta = 1.0f; + + // output shape is [seq_length, num_directions, batch_size, hidden_size] + // if we are doing 2 directions and this is the forward pass we're writing to the real output so + // need to include num_directions in the step length. + // we do not need to do that if there are two directions and we're doing the backwards pass as we + // are writing to a temporary buffer (as outputs == outputs_reverse_) which is later copied + // to the real output by ReverseSequence. this later copy includes num_directions in the step length. + int output_step_length = batch_size_ * hidden_size_; + if (direction_ == kForward && num_directions == 2) + output_step_length = 2 * batch_size_ * hidden_size_; + + // convenience end iterators we use in the loops below to detect any bounds issues + span_T_const_iter batched_bias_WRz_local_end = batched_bias_WRz_.cend(); + span_T_const_iter batched_bias_WRr_local_end = batched_bias_WRr_.cend(); + span_T_const_iter batched_bias_Wh_local_end = batched_bias_Wh_.cend(); + span_T_const_iter batched_bias_Rh_local_end = batched_bias_Rh_.cend(); + span_T_const_iter batched_bias_WRh_local_end = batched_bias_WRh_.cend(); + + if (batch_parallel_) { + int fused_hidden_rows = batch_size_ / hidden_num_threads_; + if (batch_size_ % hidden_num_threads_ != 0) + fused_hidden_rows++; + + // lambda executed by TaskThreadPool + auto hidden_gemm_and_activations = [&](const int row) { + //handling boundaries + int local_fused_hidden_rows = fused_hidden_rows; + if ((row + fused_hidden_rows) > batch_size_) + local_fused_hidden_rows = batch_size_ - row; + + size_t out_added_offset; + span_T_const_iter prev_Ht = batched_hidden0_.cbegin() + row * hidden_size_; // Ht-1 + span_T_const_iter prev_Ht_end = batched_hidden0_.cend(); + span_T_iter cur_h_local = cur_h_.begin() + row * hidden_size_; + span_T_iter cur_h_local_end = cur_h_.end(); + span_T_iter linear_output_local; + span_T_iter linear_output_local_end; + + span_T_const_iter batched_bias_WRz_local; + span_T_const_iter batched_bias_WRr_local; + span_T_const_iter batched_bias_WRh_local; + span_T_const_iter batched_bias_Wh_local; + span_T_const_iter batched_bias_Rh_local; + + if (use_bias_) { + batched_bias_WRz_local = batched_bias_WRz_.cbegin() + row * hidden_size_; + batched_bias_WRr_local = batched_bias_WRr_.cbegin() + row * hidden_size_; + + if (linear_before_reset_) { + batched_bias_Wh_local = batched_bias_Wh_.cbegin() + row * hidden_size_; + batched_bias_Rh_local = batched_bias_Rh_.cbegin() + row * hidden_size_; + linear_output_local = linear_output_.begin() + row * hidden_size_; + linear_output_local_end = linear_output_.end(); + } else { + batched_bias_WRh_local = batched_bias_WRh_.cbegin() + row * hidden_size_; + } + } + + for (int step = 0; step < max_sequence_length; step++) { + const std::string row_str = " [row=" + std::to_string(row) + ",seqno=" + std::to_string(step) + "]"; + + DumpMatrix("Ht-1" + row_str, &*prev_Ht, local_fused_hidden_rows, hidden_size_); + + out_added_offset = (step * batch_size_ + row) * hidden_size_x3; + + // calculate Ht-1*R[zr], and add to the weighted inputs that are in outputZRH_ + ComputeGemm(local_fused_hidden_rows, hidden_size_x2, hidden_size_, alpha, + prev_Ht, prev_Ht_end, + hidden_size_, + recurrent_weightsZR.cbegin(), recurrent_weightsZR.cend(), + hidden_size_, beta, + outputZRH_.begin() + out_added_offset, outputZRH_.end(), + hidden_size_x3); + + DumpMatrix("Xt*(W[zr]^T) + Ht-1 * R[zr]" + row_str, + outputZRH_.data() + out_added_offset, local_fused_hidden_rows, hidden_size_x2, 0, hidden_size_x3); + + if (linear_before_reset_) { + // copy Rbh to linear output + gsl::copy(batched_bias_Rh_.subspan(batched_bias_Rh_local - batched_bias_Rh_.begin(), local_fused_hidden_rows * hidden_size_), + linear_output_.subspan(linear_output_local - linear_output_.begin(), linear_output_local_end - linear_output_local)); + + // compute Ht-1 * (Rh^T) + Rbh + ComputeGemm(local_fused_hidden_rows, hidden_size_, hidden_size_, alpha, + prev_Ht, prev_Ht_end, // Ht-1 + hidden_size_, + recurrent_weightsH.cbegin(), recurrent_weightsH.cend(), // Rh^T + hidden_size_, beta, + linear_output_local, linear_output_.end(), // pre: Rbh, post:output + hidden_size_); + + DumpMatrix("Ht-1 * (Rh^T) + Rbh " + row_str, &*linear_output_local, batch_size_, hidden_size_); + } + + // 1st Set Of Activations + for (int r = 0; r < local_fused_hidden_rows; r++) { + const T* p_bias_r = use_bias_ ? SafeRawConstPointer(batched_bias_WRr_local + r * hidden_size_, + batched_bias_WRr_local_end, hidden_size_) + : nullptr; + + // initialize p_rt with input to calculate rt. outputZRH_ has Xt*(Wr^T) + Ht-1*(Rr^T). + T* p_rt = SafeRawPointer(outputZRH_, out_added_offset + r * hidden_size_x3 + hidden_size_, hidden_size_); + + // add the bias and clip. post: p_rt == Xt*(Wr^T) + Ht-1*(Rr^T) + Wbr + Rbr + clip_with_bias_ptr_(clip_, p_bias_r, p_rt, hidden_size_); + + if (linear_before_reset_) { + // p_linear_output = Ht-1 * (Rh^T) + Rbh + T* p_linear_output = SafeRawPointer(linear_output_local + r * hidden_size_, + linear_output_local_end, hidden_size_); + T* p_cur_h = SafeRawPointer(cur_h_local + r * hidden_size_, cur_h_local_end, hidden_size_); + + // calculate rt in-place [p_rt = f(p_rt)] + // calculate rt (.) (Ht-1 * (Rh^T) + Rbh) using p_linear_output. write to p_cur_h + reset_gate_(p_linear_output, p_rt, p_cur_h, hidden_size_, zr_alpha_, zr_beta_); + + } else { + const T* p_prev_Ht = SafeRawConstPointer(prev_Ht + r * hidden_size_, prev_Ht_end, hidden_size_); + T* p_cur_h = SafeRawPointer(cur_h_local + r * hidden_size_, cur_h_local_end, hidden_size_); + + // calculate rt in-place [p_rt = f(p_rt)] + // calculate rt (.) Ht-1 using p_prev_Ht, and write to p_cur_h + reset_gate_(p_prev_Ht, p_rt, p_cur_h, hidden_size_, zr_alpha_, zr_beta_); + } + } + + std::string label = linear_before_reset_ ? "rt (.) (Ht-1 * (Rh^T) + Rbh)" : "rt (.) Ht-1"; + DumpMatrix(label + row_str, &*cur_h_local, local_fused_hidden_rows, hidden_size_); + + if (linear_before_reset_) { + // input contains rt (.) (Ht-1*(Rh^T) + Rbh) + auto input = cur_h_local; + // out_H currently contains Xt*(W[zrh]^T). + auto out_H = outputZRH_.begin() + out_added_offset; + + for (int r = 0; r < local_fused_hidden_rows; r++) { + // skip over the inputs with Z and R weights + out_H += hidden_size_x2; + for (int h = 0; h < hidden_size_; ++h) { + *out_H += *input; + ++out_H; + ++input; + } + } + } else { + label += " * Rh^T"; + ComputeGemm(local_fused_hidden_rows, hidden_size_, hidden_size_, alpha, + cur_h_local, cur_h_local_end, + hidden_size_, + recurrent_weightsH.cbegin(), recurrent_weightsH.cend(), + hidden_size_, beta, + outputZRH_.begin() + out_added_offset + hidden_size_x2, outputZRH_.end(), + hidden_size_x3); + } + + DumpMatrix("Xt*(Wh^T) + (" + label + ")" + row_str, + outputZRH_.data() + out_added_offset, local_fused_hidden_rows, hidden_size_, + hidden_size_x2, hidden_size_x3); + + // 2nd Set of Activations + span_T_iter output; + span_T_iter output_end; + if (output_sequence) { + output = outputs.begin() + step * output_step_length + row * hidden_size_; + output_end = outputs.end(); + + } else { + output = final_hidden_state.begin() + row * hidden_size_; + output_end = final_hidden_state.end(); + } + + for (int r = 0; r < local_fused_hidden_rows; r++) { + if (step >= min_sequence_length && step >= sequence_lengths[row + r]) { + if (output_sequence) { + auto fill_output = output + r * hidden_size_; + std::fill_n(fill_output, hidden_size_, T{}); + } + + continue; + } + + const T* p_bias_z = use_bias_ ? SafeRawConstPointer(batched_bias_WRz_local, batched_bias_WRz_local_end, + hidden_size_) + : nullptr; + + // initialize p_zt with Xt*(Wz^T) + Ht-1*(Rz^T), which is most of the input to calculate zt: + T* p_zt = SafeRawPointer(outputZRH_, out_added_offset + r * hidden_size_x3, hidden_size_); + + // using p_zt, add bias and clip in-place + clip_with_bias_ptr_(clip_, p_bias_z, p_zt, hidden_size_); + + // calculate zt in-place. p_zt = f(p_zt) + update_gate_(p_zt, hidden_size_, zr_alpha_, zr_beta_); + + DumpMatrix("zt[" + std::to_string(r) + "]" + row_str, p_zt, 1, hidden_size_); + + const T* p_bias_h = nullptr; + if (use_bias_) { + if (linear_before_reset_) { + // Wbh + p_bias_h = SafeRawConstPointer(batched_bias_Wh_local + r * hidden_size_, + batched_bias_Wh_local_end, hidden_size_); + + } else { + // Wbh + Wrh + p_bias_h = SafeRawConstPointer(batched_bias_WRh_local + r * hidden_size_, + batched_bias_WRh_local_end, hidden_size_); + } + } + + // setup p_ht with input to calculate ht + // p_ht = Xt*(Wh^T) + (rt (.) Ht-1 * Rh^T) # linear_before_reset_ == false + // = Xt*(Wh^T) + (rt (.) (Ht-1*(Rh^T) + Rbh)) # linear_before_reset_ == true + T* p_ht = SafeRawPointer(outputZRH_, out_added_offset + r * hidden_size_x3 + hidden_size_x2, hidden_size_); + + // add Wbh [and Wrh] and clip + clip_with_bias_ptr_(clip_, p_bias_h, p_ht, hidden_size_); // post: p_ht = input to g() for calculating ht + + DumpMatrix("ht input [" + std::to_string(r) + "]" + row_str, p_ht, 1, hidden_size_); + + const T* p_prev_Ht = SafeRawConstPointer(prev_Ht + r * hidden_size_, prev_Ht_end, hidden_size_); + T* p_Ht = SafeRawPointer(output + r * hidden_size_, output_end, hidden_size_); + + // calculate ht = g(p_ht) and write in-place to p_ht + // calculate Ht = (1 - zt) (.) ht + zt (.) Ht-1 and write to p_Ht + output_gate_(p_ht, p_zt, p_prev_Ht, p_Ht, hidden_size_, h_alpha_, h_beta_); + } + + DumpMatrix("output" + row_str, &*output, 1, hidden_size_); + + prev_Ht = output; + prev_Ht_end = output_end; + } + }; + + ExecuteLambdaInParallel("Processing batch", hidden_gemm_and_activations, batch_size_, fused_hidden_rows, + ttp_, logger_); + } else { + size_t out_added_offset; + + span_T_const_iter prev_Ht = batched_hidden0_.cbegin(); // Ht-1 + span_T_const_iter prev_Ht_end = batched_hidden0_.cend(); + span_T_iter cur_h_local = cur_h_.begin(); + span_T_iter cur_h_local_end = cur_h_.end(); + + span_T_const_iter batched_bias_WRz_local; + span_T_const_iter batched_bias_WRr_local; + span_T_const_iter batched_bias_WRh_local; + span_T_const_iter batched_bias_Wh_local; + span_T_const_iter batched_bias_Rh_local; + + if (use_bias_) { + batched_bias_WRz_local = batched_bias_WRz_.cbegin(); + batched_bias_WRr_local = batched_bias_WRr_.cbegin(); + + if (linear_before_reset_) { + batched_bias_Wh_local = batched_bias_Wh_.cbegin(); + batched_bias_Rh_local = batched_bias_Rh_.cbegin(); + } else { + batched_bias_WRh_local = batched_bias_WRh_.cbegin(); + } + } + + // for each item in sequence run all calculations + for (int step = 0; step < max_sequence_length; step++) { + const std::string seqno_str = " [seqno=" + std::to_string(step) + "]"; + + DumpMatrix("Ht-1" + seqno_str, &*prev_Ht, batch_size_, hidden_size_); + + out_added_offset = (step * batch_size_) * hidden_size_x3; + + // calculate Ht-1*R[zr], and add to the weighted inputs that are in outputZRH_ + // Ht-1 * R[zr] + Xt*(W[zr]^T) + ComputeGemm(batch_size_, hidden_size_x2, hidden_size_, alpha, + prev_Ht, prev_Ht_end, + hidden_size_, + recurrent_weightsZR.cbegin(), recurrent_weightsZR.cend(), + hidden_size_, beta, + outputZRH_.begin() + out_added_offset, outputZRH_.end(), + hidden_size_x3); + + DumpMatrix("Ht-1 * R[zr] + Xt*(W[zr]^T)" + seqno_str, + outputZRH_.data() + out_added_offset, batch_size_, hidden_size_x2, 0, hidden_size_x3); + + if (linear_before_reset_) { + // copy Rbh to linear output + gsl::copy(batched_bias_Rh_.subspan(batched_bias_Rh_local - batched_bias_Rh_.begin(), batched_bias_Rh_local_end - batched_bias_Rh_local), linear_output_); + + // compute Ht-1 * (Rh^T) + Rbh + ComputeGemm(batch_size_, hidden_size_, hidden_size_, alpha, + prev_Ht, prev_Ht_end, // Ht-1 + hidden_size_, + recurrent_weightsH.cbegin(), recurrent_weightsH.cend(), // Rh^T + hidden_size_, beta, + linear_output_.begin(), linear_output_.end(), // pre: Rbh, post:output + hidden_size_); + + DumpMatrix("Ht-1 * (Rh^T) + Rbh " + seqno_str, linear_output_.data(), batch_size_, hidden_size_); + } + + // 1st Set Of Activations + for (int r = 0; r < batch_size_; r++) { + const T* p_bias_r = use_bias_ ? SafeRawConstPointer(batched_bias_WRr_local + r * hidden_size_, + batched_bias_WRr_local_end, hidden_size_) + : nullptr; + + // initialize p_rt with input to calculate rt. outputZRH_ has Xt*(Wr^T) + Ht-1*(Rr^T). + T* p_rt = SafeRawPointer(outputZRH_, out_added_offset + r * hidden_size_x3 + hidden_size_, hidden_size_); + + // add the bias and clip. post: p_rt == Xt*(Wr^T) + Ht-1*(Rr^T) + Wbr + Rbr + clip_with_bias_ptr_(clip_, p_bias_r, p_rt, hidden_size_); + + if (linear_before_reset_) { + // p_linear_output = Ht-1 * (Rh^T) + Rbh + T* p_linear_output = SafeRawPointer(linear_output_, r * hidden_size_, hidden_size_); + T* p_cur_h = SafeRawPointer(cur_h_local + r * hidden_size_, cur_h_local_end, hidden_size_); + + // calculate rt in-place [p_rt = f(p_rt)] + // calculate rt (.) (Ht-1 * (Rh^T) + Rbh) using p_linear_output. write to p_cur_h + reset_gate_(p_linear_output, p_rt, p_cur_h, hidden_size_, zr_alpha_, zr_beta_); + + } else { + const T* p_prev_Ht = SafeRawConstPointer(prev_Ht + r * hidden_size_, prev_Ht_end, hidden_size_); + T* p_cur_h = SafeRawPointer(cur_h_local + r * hidden_size_, cur_h_local_end, hidden_size_); + + // calculate rt in-place [p_rt = f(p_rt)] + // calculate rt (.) Ht-1 using p_prev_Ht, and write to p_cur_h + reset_gate_(p_prev_Ht, p_rt, p_cur_h, hidden_size_, zr_alpha_, zr_beta_); + } + } + + std::string label = linear_before_reset_ ? "rt (.) (Ht-1 * (Rh^T) + Rbh)" : "rt (.) Ht-1"; + DumpMatrix(label + seqno_str, &*cur_h_local, batch_size_, hidden_size_); + + if (linear_before_reset_) { + // input contains rt (.) (Ht-1*(Rh^T) + Rbh) + auto input = cur_h_local; + // out_H currently contains Xt*(W[zrh]^T). + auto out_H = outputZRH_.begin() + out_added_offset; + + for (int r = 0; r < batch_size_; r++) { + // skip over the inputs with Z and R weights + out_H += hidden_size_x2; + for (int h = 0; h < hidden_size_; ++h) { + *out_H += *input; + ++out_H; + ++input; + } + } + } else { + label += " * Rh^T"; + + // out_H currently contains Xt*(Wh^T). + auto out_H = outputZRH_.begin() + out_added_offset + hidden_size_x2; + + // Calculate Xt*(Wh^T) + rt (.) Ht-1 * Rh + ComputeGemm(batch_size_, hidden_size_, hidden_size_, alpha, + cur_h_local, cur_h_local_end, // rt (.) Ht-1 + hidden_size_, + recurrent_weightsH.cbegin(), recurrent_weightsH.cend(), // Rh^T + hidden_size_, beta, + out_H, outputZRH_.end(), + hidden_size_x3); + } + + DumpMatrix("Xt*(Wh^T) + (" + label + ")" + seqno_str, outputZRH_.data() + out_added_offset, + batch_size_, hidden_size_, hidden_size_x2, hidden_size_x3); + + //2nd Set of Activations + span_T_iter output; + span_T_iter output_end; + if (output_sequence) { + output = outputs.begin() + step * output_step_length; + output_end = outputs.end(); + + } else { + output = final_hidden_state.begin(); + output_end = final_hidden_state.end(); + } + + for (int r = 0; r < batch_size_; r++) { + if (step >= min_sequence_length && step >= sequence_lengths[r]) { + if (output_sequence) { + auto fill_output = output + r * hidden_size_; + std::fill_n(fill_output, hidden_size_, T{}); + } + + continue; + } + + const T* p_bias_z = use_bias_ ? SafeRawConstPointer(batched_bias_WRz_local, + batched_bias_WRz_local_end, hidden_size_) + : nullptr; + + // initialize p_zt with Xt*(Wz^T) + Ht-1*(Rz^T), which is most of the input to calculate zt: + T* p_zt = SafeRawPointer(outputZRH_, out_added_offset + r * hidden_size_x3, hidden_size_); + + // using p_zt, add bias and clip in-place + clip_with_bias_ptr_(clip_, p_bias_z, p_zt, hidden_size_); + + // calculate zt in-place. p_zt = f(p_zt) + update_gate_(p_zt, hidden_size_, zr_alpha_, zr_beta_); + + DumpMatrix("zt[" + std::to_string(r) + "]" + seqno_str, p_zt, 1, hidden_size_); + + const T* p_bias_h = nullptr; + if (use_bias_) { + if (linear_before_reset_) { + // Wbh + p_bias_h = SafeRawConstPointer(batched_bias_Wh_local + r * hidden_size_, + batched_bias_Wh_local_end, hidden_size_); + + } else { + // Wbh + Wrh + p_bias_h = SafeRawConstPointer(batched_bias_WRh_local + r * hidden_size_, + batched_bias_WRh_local_end, hidden_size_); + } + } + + // setup p_ht with input to calculate ht + // p_ht = Xt*(Wh^T) + (rt (.) Ht-1 * Rh^T) # linear_before_reset_ == false + // = Xt*(Wh^T) + (rt (.) (Ht-1*(Rh^T) + Rbh)) # linear_before_reset_ == true + T* p_ht = SafeRawPointer(outputZRH_, out_added_offset + r * hidden_size_x3 + hidden_size_x2, hidden_size_); + + // add Wbh [and Wrh] and clip + clip_with_bias_ptr_(clip_, p_bias_h, p_ht, hidden_size_); // post: p_ht == input to g() for calculating ht + + DumpMatrix("ht input [" + std::to_string(r) + "]" + seqno_str, p_ht, 1, hidden_size_); + + const T* p_prev_Ht = SafeRawConstPointer(prev_Ht + r * hidden_size_, prev_Ht_end, hidden_size_); + T* p_Ht = SafeRawPointer(output + r * hidden_size_, output_end, hidden_size_); + + // calculate ht = g(p_ht) and write in-place to p_ht + // calculate Ht = (1 - zt) (.) ht + zt (.) Ht-1 and write to p_Ht + output_gate_(p_ht, p_zt, p_prev_Ht, p_Ht, hidden_size_, h_alpha_, h_beta_); // calculate ht and Ht + } + + DumpMatrix("output" + seqno_str, &*output, batch_size_, hidden_size_); + + prev_Ht = output; + prev_Ht_end = output_end; + } + } + + if (output_sequence) { + // copy last output to final_hidden_state + for (int i = 0; i < batch_size_; i++) { + const int seq_len = sequence_lengths[i]; + auto src = outputs.subspan((seq_len - 1) * output_step_length + i * hidden_size_, hidden_size_); + auto dest = final_hidden_state.subspan(i * hidden_size_, hidden_size_); + gsl::copy(src, dest); + } + + if (direction_ == kReverse) { + ReverseSequence(outputs, original_outputs, + sequence_lengths, seq_length_, + batch_size_, hidden_size_, num_directions); + } + } +} + +template +void UniDirectionalGru::AllocateBuffers() { + cur_h_ = Allocate(allocator_, hidden_size_ * batch_size_, cur_h_ptr_); + batched_hidden0_ = Allocate(allocator_, batch_size_ * hidden_size_, batched_hidden0_ptr_, true); + + if (use_bias_) { + batched_bias_WRz_ = Allocate(allocator_, batch_size_ * hidden_size_, batched_bias_WRz_ptr_); + batched_bias_WRr_ = Allocate(allocator_, batch_size_ * hidden_size_, batched_bias_WRr_ptr_); + + if (linear_before_reset_) { + batched_bias_Wh_ = Allocate(allocator_, batch_size_ * hidden_size_, batched_bias_Wh_ptr_); + batched_bias_Rh_ = Allocate(allocator_, batch_size_ * hidden_size_, batched_bias_Rh_ptr_); + linear_output_ = Allocate(allocator_, batch_size_ * hidden_size_, linear_output_ptr_); + } else { + batched_bias_WRh_ = Allocate(allocator_, batch_size_ * hidden_size_, batched_bias_WRh_ptr_); + } + } + + auto batch_times_seq_length = batch_size_ * seq_length_; + + outputZRH_ = Allocate(allocator_, hidden_size_ * 3 * batch_times_seq_length, outputZRH_ptr_, true); + + if (direction_ == kReverse) { + inputs_reverse_ = Allocate(allocator_, batch_times_seq_length * input_size_, inputs_reverse_ptr_); + outputs_reverse_ = Allocate(allocator_, batch_times_seq_length * hidden_size_, outputs_reverse_ptr_); + } +} + +template +void UniDirectionalGru::SetNumThreads() { + int threads = std::thread::hardware_concurrency() - 1; + + if (threads < 1) + threads = 1; + + hidden_num_threads_ = threads; + batch_parallel_ = false; + + // for readability of the below logic + const auto num_rows = batch_size_; + const auto num_columns = hidden_size_; + + // parallelize by partitioning the batch rows + if (num_rows > 4 || + (num_rows >= 2 && num_columns <= 256) || + (num_rows >= 3 && num_columns <= 512)) { + batch_parallel_ = true; + VLOGS(logger_, 1) << "Hidden Threads : " << hidden_num_threads_; + } + + ONNXRUNTIME_ENFORCE(hidden_num_threads_ >= 1); +} +} // namespace detail +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/rnn/deep_cpu_gru.h b/onnxruntime/core/providers/cpu/rnn/deep_cpu_gru.h new file mode 100644 index 0000000000000..5deb6a05cec06 --- /dev/null +++ b/onnxruntime/core/providers/cpu/rnn/deep_cpu_gru.h @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include "core/common/task_thread_pool.h" +#include "core/framework/allocator.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cpu/rnn/rnn_helpers.h" + +namespace onnxruntime { + +/// The class represents GRU operator using DeepCPU implementation for +/// fast inference computation on CPU machines. +class DeepCpuGruOp final : public OpKernel { + public: + DeepCpuGruOp(const OpKernelInfo& info) : OpKernel(info) { + // required attributes + std::string direction; + ONNXRUNTIME_ENFORCE(info.GetAttr("direction", &direction).IsOK()); + + int64_t int64_value; + ONNXRUNTIME_ENFORCE(info.GetAttr("linear_before_reset", &int64_value).IsOK()); + linear_before_reset_ = gsl::narrow(int64_value); + + ONNXRUNTIME_ENFORCE(info.GetAttr("hidden_size", &int64_value).IsOK() && int64_value > 0); + hidden_size_ = gsl::narrow(int64_value); + + // optional attributes + std::vector activation_func_names = info.GetAttrsOrDefault("activations"); + std::vector activation_func_alphas = info.GetAttrsOrDefault("activation_alpha"); + std::vector activation_func_betas = info.GetAttrsOrDefault("activation_beta"); + + clip_ = info.GetAttrOrDefault("clip", std::numeric_limits::max()); + ONNXRUNTIME_ENFORCE(clip_ > 0.f); + + direction_ = rnn::detail::MakeDirection(direction); + num_directions_ = direction_ == rnn::detail::Direction::kBidirectional ? 2 : 1; + + if (activation_func_names.empty()) { + for (int i = 0; i < num_directions_; ++i) { + activation_func_names.emplace_back("sigmoid"); + activation_func_names.emplace_back("tanh"); + } + } + + ONNXRUNTIME_ENFORCE(activation_func_names.size() == num_directions_ * 2); + + activation_funcs_ = rnn::detail::ActivationFuncs(activation_func_names, + activation_func_alphas, + activation_func_betas); + } + + Status Compute(OpKernelContext* context) const override; + + ~DeepCpuGruOp() override = default; + + private: + rnn::detail::Direction direction_; + int num_directions_; + + int hidden_size_ = 0; + float clip_; + int linear_before_reset_ = 0; + + rnn::detail::ActivationFuncs activation_funcs_; + + // Threadpool for operator. If concurrent Compute calls are possible, it will be shared + // across them. mutable due to this. + // The alternative would be to create a threadpool in each call to Compute but that would incur thread creation + // cost on every call. + mutable TaskThreadPool ttp_{std::thread::hardware_concurrency()}; + + template + Status ComputeImpl(OpKernelContext& context) const; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.cc b/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.cc new file mode 100644 index 0000000000000..1a2ae2fd9d38b --- /dev/null +++ b/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.cc @@ -0,0 +1,1120 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// there's no way to use a raw pointer as the copy destination with std::copy_n +// (which gsl::copy uses with span::data() which returns a raw pointer) with the 14.11 toolset +// without generating a 4996 warning. going through an iterator is way too much overhead so turn off the warning. +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4996) +#endif + +#include "core/providers/cpu/rnn/deep_cpu_lstm.h" + +#include "core/common/common.h" +#include "core/common/logging/logging.h" +#include "core/common/task_thread_pool.h" +#include "core/framework/allocator.h" + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +/* +ONNX_OPERATOR_SCHEMA(LSTM) + .SetDoc(R"DOC( +Computes an one-layer LSTM. This operator is usually supported via some +custom implementation such as CuDNN. + +Notations: + +`X` - input tensor +`i` - input gate +`o` - output gate +`f` - forget gate +`c` - cell gate +`t` - time step (t-1 means previous time step) + +`W[iofc]` - W parameter weight matrix for input, output, forget, and cell gates +`R[iofc]` - R recurrence weight matrix for input, output, forget, and cell gates +`Wb[iofc]` - W bias vectors for input, output, forget, and cell gates +`Rb[iofc]` - R bias vectors for input, output, forget, and cell gates +`P[iof]` - P peephole weight vector for input, output, and forget gates +`WB[iofc]` - W parameter weight matrix for backward input, output, forget, and cell gates +`RB[iofc]` - R recurrence weight matrix for backward input, output, forget, and cell gates +`WBb[iofc]` - W bias vectors for backward input, output, forget, and cell gates +`RBb[iofc]` - R bias vectors for backward input, output, forget, and cell gates +`PB[iof]` - P peephole weight vector for backward input, output, and forget gates + +`H` - Hidden state +`num_directions` - 2 if direction == bidirectional else 1 + +Activation functions: + + Relu(x) - max(0, x) + Tanh(x) - (1 - e^{-2x})/(1 + e^{-2x}) + Sigmoid(x) - 1/(1 + e^{-x}) + + (NOTE: Below are optional) + Affine(x) - alpha*x + beta + LeakyRelu(x) - x if x >= 0 else alpha * x + ThresholdedRelu(x) - x if x >= alpha else 0 + ScaledTanh(x) - alpha*Tanh(beta*x) + HardSigmoid(x) - min(max(alpha*x + beta, 0), 1) + Elu(x) - x if x >= 0 else alpha*(e^x - 1) + Softsign(x) - x/(1 + |x|) + Softplus(x) - log(1 + e^x) + +Equations (Default: f=Sigmoid, g=Tanh, h=Tanh): + - it = f(Xt*(Wi^T) + Ht-1*Ri + Pi (.) Ct-1 + Wbi + Rbi) + - ft = f(Xt*(Wf^T) + Ht-1*Rf + Pf (.) Ct-1 + Wbf + Rbf) + - ct = g(Xt*(Wc^T) + Ht-1*Rc + Wbc + Rbc) + - Ct = ft (.) Ct-1 + it (.) ct + - ot = f(Xt*(Wo^T) + Ht-1*Ro + Po (.) Ct + Wbo + Rbo) + - Ht = ot (.) h(Ct) +)DOC") + .Attr("direction", "Specify if the RNN is forward, reverse, or bidirectional. " + "Must be one of forward (default), reverse, or bidirectional.", + AttributeProto::STRING, + std::string("forward")) + .Attr("hidden_size", "Number of neurons in the hidden layer", AttributeProto::INT, OPTIONAL) + .Attr("activations", "A list of 3 (or 6 if bidirectional) activation functions " + "for input, output, forget, cell, and hidden. The activation functions must " + "be one of the activation functions specified above. Optional: See the equations " + "for default if not specified.", + AttributeProto::STRINGS, + OPTIONAL) + .Attr("activation_alpha", + "Optional scaling values used by some activation functions. The values " + "are consumed in the order of activation functions, for example (f, g, h) " + "in LSTM.", + AttributeProto::FLOATS, + OPTIONAL) + .Attr("activation_beta", + "Optional scaling values used by some activation functions. The values " + "are consumed in the order of activation functions, for example (f, g, h) " + "in LSTM.", + AttributeProto::FLOATS, + OPTIONAL) + .Attr("output_sequence", + "The sequence output for the hidden is optional if 0. Default 0.", + AttributeProto::INT, + static_cast(0)); + .Attr("clip", "Cell clip threshold. Clipping bounds the elements of a tensor " + "in the range of [-threshold, +threshold] and is applied to the input " + "of activations. No clip if not specified.", AttributeProto::FLOAT, OPTIONAL) + .Attr("input_forget", "Couple the input and forget gates if 1, default 0.", + AttributeProto::INT, + static_cast(0)) + .Input(0, "X", + "The input sequences packed (and potentially padded) into one 3-D " + "tensor with the shape of `[seq_length, batch_size, input_size]`.", "T") + .Input(1, "W", + "The weight tensor for the gates. Concatenation of `W[iofc]` and " + "`WB[iofc]` (if bidirectional) along dimension 0. The tensor has shape " + "`[num_directions, 4*hidden_size, input_size]`.", "T") + .Input(2, "R", + "The recurrence weight tensor. Concatenation of `R[iofc]` and " + "`RB[iofc]` (if bidirectional) along dimension 0. This tensor has shape " + "`[num_directions, 4*hidden_size, hidden_size]`.", "T") + .Input(3, "B", + "The bias tensor for input gate. Concatenation of `[Wb[iofc], Rb[iofc]]`, " + "and `[WBb[iofc], RBb[iofc]]` (if bidirectional) along dimension 0. This " + "tensor has shape `[num_directions, 8*hidden_size]`. Optional: If not " + "specified - assumed to be 0.", "T", + OpSchema::Optional) + .Input(4, "sequence_lens", + "Optional tensor specifying lengths of the sequences in a batch. " + "If not specified - assumed all sequences in the batch to have " + "length `seq_length`. It has shape `[batch_size]`.", "T1", + OpSchema::Optional) + .Input(5, "initial_h", + "Optional initial value of the hidden. If not specified - assumed " + "to be 0. It has shape `[num_directions, batch_size, hidden_size]`.", + "T", OpSchema::Optional) + .Input(6, "initial_c", + "Optional initial value of the cell. If not specified - assumed " + "to be 0. It has shape `[num_directions, batch_size, hidden_size]`.", +"T", OpSchema::Optional) + .Input(7, "P", + "The weight tensor for peepholes. Concatenation of `P[iof]` and " + "`PB[iof]` (if bidirectional) along dimension 0. It has shape " + "`[num_directions, 3*hidde_size]`. Optional: If not specified - " + "assumed to be 0.", "T", + OpSchema::Optional) + .Output(0, "Y", + "A tensor that concats all the intermediate output values of the hidden. " + "It has shape `[seq_length, num_directions, batch_size, hidden_size]`. ", + "T", OpSchema::Optional); + .Output(1, "Y_h", + "The last output value of the hidden. It has shape " + "`[num_directions, batch_size, hidden_size]`.", "T", OpSchema::Optional); + .Output(2, "Y_c", + "The last output value of the cell. It has shape " + "`[num_directions, batch_size, hidden_size]`.", "T", OpSchema::Optional); + .TypeConstraint("T", { "tensor(float16)", "tensor(float)", "tensor(double)" }, + "Constrain input and output types to float tensors.") + .TypeConstraint("T1", { "tensor(int32)" }, "Constrain seq_lens to integer tensor."); + +*/ + +namespace onnxruntime { + +/* LSTM operator */ +ONNX_CPU_OPERATOR_KERNEL( + LSTM, + 7, + KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}) + .TypeConstraint("T1", DataTypeImpl::GetTensorType()), + DeepCpuLstmOp); + +using namespace rnn::detail; + +// LSTM details +namespace detail { + +// Helper struct for an activation function call information +template +struct ActivationInfo { + TFunc func; + float alpha; + float beta; +}; + +// copying the peephole values into UniDirectionalLstm seems unnecessary. don't do that until proven necessary +#define LSTM_NO_PEEPHOLE_COPY + +template +class UniDirectionalLstm { + public: + UniDirectionalLstm(AllocatorPtr allocator, + const logging::Logger& logger, + const int seq_length, + const int batch_size, + const int input_size, + const int hidden_size, + Direction direction, + const bool input_forget, + const gsl::span& bias, + const gsl::span& peephole_weights, + const gsl::span& initial_hidden_state, + const gsl::span& initial_cell_state, + const ActivationFuncs::Entry& activation_func_f, + const ActivationFuncs::Entry& activation_func_g, + const ActivationFuncs::Entry& activation_func_h, + const float clip, + TaskThreadPool& ttp); + + void Compute(const gsl::span& inputs, + const gsl::span& sequence_lengths, + const int num_directions, + const gsl::span& input_weights, + const gsl::span& recurrent_weights, + gsl::span& outputs, + gsl::span& final_hidden_state, + gsl::span& final_cell_state); + + ~UniDirectionalLstm() = default; + + private: + using span_T_const_iter = typename gsl::span::const_iterator; + using span_T_iter = typename gsl::span::iterator; + + void SetNumThreads(); + + void GateComputations(span_T_iter& out, span_T_iter& out_end, + span_T_iter& C_prev, span_T_iter& C_prev_end, // Ct-1 value not 'ct'. using 'C' for clarity + span_T_iter& C_prev_clipped, span_T_iter& C_prev_clipped_end, + span_T_iter& batched_output, span_T_iter& batched_output_end, + const gsl::span& seq_lengths, + const int min_sequence_length, + const int step, + const int row, + const int local_fused_hidden_rows, + bool output_sequence); + + void AllocateBuffers(); + + void InitializeBuffers(const gsl::span& initial_hidden_state, + const gsl::span& initial_cell_state); + + void LoadPeepholeWeights(const gsl::span& peephole_weights); + void LoadBias(const gsl::span& WbRb_values); + + AllocatorPtr allocator_; + const logging::Logger& logger_; + + int seq_length_; + int batch_size_; + int input_size_; + int hidden_size_; + + Direction direction_; + bool input_forget_; + float clip_; + + bool batch_parallel_; + + bool use_bias_; + bool use_peepholes_; + + int hidden_num_threads_ = -1; + + IAllocatorUniquePtr output_iofc_ptr_; + IAllocatorUniquePtr hidden0_ptr_, batched_hidden0_ptr_; + gsl::span output_iofc_; + gsl::span hidden0_, batched_hidden0_; + + IAllocatorUniquePtr internal_memory_prev_ptr_, batched_internal_memory_prev_ptr_; + IAllocatorUniquePtr internal_memory_cur_ptr_, batched_internal_memory_cur_ptr_; + IAllocatorUniquePtr batched_internal_memory_clipped_ptr_; + gsl::span internal_memory_prev_, batched_internal_memory_prev_; + gsl::span internal_memory_cur_, batched_internal_memory_cur_; + gsl::span batched_internal_memory_clipped_; + + IAllocatorUniquePtr bias_WRi_ptr_, bias_WRf_ptr_, bias_WRo_ptr_, bias_WRc_ptr_; + IAllocatorUniquePtr batched_bias_WRi_ptr_, batched_bias_WRf_ptr_, batched_bias_WRo_ptr_, batched_bias_WRc_ptr_; + IAllocatorUniquePtr peephole_i_ptr_, peephole_f_ptr_, peephole_o_ptr_; + IAllocatorUniquePtr inputs_reverse_ptr_, outputs_reverse_ptr_; + gsl::span bias_WRi_, bias_WRf_, bias_WRo_, bias_WRc_; + gsl::span batched_bias_WRi_, batched_bias_WRf_, batched_bias_WRo_, *batched_bias_WRc_; + gsl::span inputs_reverse_, outputs_reverse_; + +#if defined(LSTM_NO_PEEPHOLE_COPY) + gsl::span peephole_i_, peephole_f_, peephole_o_; +#else + gsl::span peephole_i_, peephole_f_, peephole_o_; +#endif + + IAllocatorUniquePtr sequence_lengths_ptr_; + gsl::span sequence_lengths_; + + deepcpu::ClipWithBiasFuncPtr clip_with_bias_ptr_; + + ActivationInfo activation_f_; + ActivationInfo activation_g_; + ActivationInfo activation_h_; + + TaskThreadPool& ttp_; +}; + +} // namespace detail + +Status +DeepCpuLstmOp::Compute(OpKernelContext* context) const { + const Tensor& X = *context->Input(0); // inputs. [seq_length, batch_size, input_size] + + Status status; + // auto& logger = context->Logger(); + + auto data_type = X.DataType(); + if (data_type == DataTypeImpl::GetType()) + status = ComputeImpl(*context); + else if (data_type == DataTypeImpl::GetType()) { + /* Need to update all the helpers to support double... + status = ComputeImpl(*context); */ + ONNXRUNTIME_NOT_IMPLEMENTED("LSTM operator does not support double yet"); + } else + ONNXRUNTIME_THROW("Invalid data type for LSTM operator of ", data_type); + + return status; +} + +// #define DUMP_MATRIXES to provide lots of diagnostic output +#if defined(DUMP_MATRIXES) +#define DumpMatrix(...) ::onnxruntime::rnn::detail::DumpMatrixImpl(__VA_ARGS__) +#else +#define DumpMatrix(...) ((void)0) +#endif + +template +Status DeepCpuLstmOp::ComputeImpl(OpKernelContext& context) const { + auto& logger = context.Logger(); + + const Tensor& X = *context.Input(0); // inputs. [seq_length, batch_size, input_size] + const Tensor& W = *context.Input(1); // weights. [num_directions, 4*hidden_size, input_size] + const Tensor& R = *context.Input(2); // recurrence weights. [num_directions, 4*hidden_size, hidden_size] + + // optional + const Tensor* B = context.Input(3); // bias. [num_directions, 8*hidden_size] + const Tensor* sequence_lens = context.Input(4); // [batch_size] + const Tensor* initial_h = context.Input(5); // initial hidden. [num_directions, batch_size, hidden_size] + const Tensor* initial_c = context.Input(6); // initial cell. [num_directions, batch_size, hidden_size] + const Tensor* P = context.Input(7); // peephole weights. [num_directions, 3*hidden_size] + + auto& X_shape = X.Shape(); + + int seq_length = gsl::narrow(X_shape[0]); + int batch_size = gsl::narrow(X_shape[1]); + int input_size = gsl::narrow(X_shape[2]); + + Status status = ValidateInputs(X, W, R, B, sequence_lens, initial_h, initial_c, P, batch_size); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + // LSTM outputs are optional but must be in the same order + TensorShape Y_dims{seq_length, num_directions_, batch_size, hidden_size_}; + Tensor* Y = context.Output(/*index*/ 0, Y_dims); + + TensorShape Y_h_dims{num_directions_, batch_size, hidden_size_}; + Tensor* Y_h = context.Output(/*index*/ 1, Y_h_dims); + + TensorShape Y_c_dims{num_directions_, batch_size, hidden_size_}; + Tensor* Y_c = context.Output(/*index*/ 2, Y_c_dims); + + AllocatorPtr alloc; + status = context.GetTempSpaceAllocator(&alloc); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + gsl::span input_weights = W.DataAsSpan(); + gsl::span recurrent_weights = R.DataAsSpan(); + gsl::span bias = B != nullptr ? B->DataAsSpan() : gsl::span(); + gsl::span peephole_weights = P != nullptr ? P->DataAsSpan() : gsl::span(); + + // spans for first direction + const size_t input_weights_size_per_direction = 4 * hidden_size_ * input_size; + const size_t hidden_weights_size_per_direction = 4 * hidden_size_ * hidden_size_; + const size_t bias_size_per_direction = 8 * hidden_size_; + const size_t peephole_weights_size_per_direction = 3 * hidden_size_; + + gsl::span input_weights_1 = input_weights.subspan(0, input_weights_size_per_direction); + gsl::span recurrent_weights_1 = recurrent_weights.subspan(0, hidden_weights_size_per_direction); + gsl::span bias_1 = bias.empty() ? bias : bias.subspan(0, bias_size_per_direction); + gsl::span peephole_weights_1 = + peephole_weights.empty() ? peephole_weights + : peephole_weights.subspan(0, peephole_weights_size_per_direction); + + gsl::span input = X.DataAsSpan(); + gsl::span sequence_lens_span = sequence_lens != nullptr ? sequence_lens->DataAsSpan() + : gsl::span(); + + const size_t initial_hidden_size_per_direction = batch_size * hidden_size_; + gsl::span initial_hidden = initial_h != nullptr ? initial_h->DataAsSpan() : gsl::span(); + gsl::span initial_hidden_1 = + initial_hidden.empty() ? initial_hidden + : initial_hidden.subspan(0, initial_hidden_size_per_direction); + + const size_t initial_cell_size_per_direction = batch_size * hidden_size_; + gsl::span initial_cell = initial_c != nullptr ? initial_c->DataAsSpan() : gsl::span(); + gsl::span initial_cell_1 = + initial_cell.empty() ? initial_cell + : initial_cell.subspan(0, initial_cell_size_per_direction); + + // output shape is [seq_length, num_directions, batch_size, hidden_size] + // so it's not a case of all the output for one direction being first. + // due to that we can only easily check that the end of the output for each direction is valid. + const size_t output_size = Y != nullptr ? Y->Shape().Size() : 0; + const size_t per_direction_offset = batch_size * hidden_size_; + gsl::span output = Y != nullptr ? Y->MutableDataAsSpan() : gsl::span(); + gsl::span output_1 = + output.empty() ? output + : output.subspan(0, output_size - (num_directions_ - 1) * per_direction_offset); + + // UniDirectionalLstm needs somewhere to write output, so even if we aren't returning Y_h and Y_c + // we provide an appropriately sized buffer for that purpose. + const size_t hidden_output_size_per_direction = batch_size * hidden_size_; + IAllocatorUniquePtr local_hidden_output; + gsl::span hidden_output = + Y_h ? Y_h->MutableDataAsSpan() + : Allocate(alloc, hidden_output_size_per_direction * num_directions_, local_hidden_output); + + gsl::span hidden_output_1 = hidden_output.subspan(0, hidden_output_size_per_direction); + + const size_t last_cell_size_per_direction = batch_size * hidden_size_; + IAllocatorUniquePtr local_last_cell; + gsl::span last_cell = + Y_c ? Y_c->MutableDataAsSpan() + : Allocate(alloc, last_cell_size_per_direction * num_directions_, local_last_cell); + + gsl::span last_cell_1 = last_cell.subspan(0, last_cell_size_per_direction); + + std::unique_ptr> fw; + std::unique_ptr> bw; + + if (direction_ == Direction::kBidirectional) { + // spans for second direction + gsl::span input_weights_2 = input_weights.subspan(input_weights_size_per_direction, + input_weights_size_per_direction); + gsl::span hidden_weights_2 = recurrent_weights.subspan(hidden_weights_size_per_direction, + hidden_weights_size_per_direction); + gsl::span bias_2 = bias.empty() ? bias : bias.subspan(bias_size_per_direction, bias_size_per_direction); + gsl::span peephole_weights_2 = + peephole_weights.empty() ? peephole_weights + : peephole_weights.subspan(peephole_weights_size_per_direction, + peephole_weights_size_per_direction); + + gsl::span initial_hidden_2 = + initial_hidden.empty() ? initial_hidden + : initial_hidden.subspan(initial_hidden_size_per_direction, + initial_hidden_size_per_direction); + gsl::span initial_cell_2 = + initial_cell.empty() ? initial_cell + : initial_cell.subspan(initial_cell_size_per_direction, + initial_cell_size_per_direction); + gsl::span output_2 = + output.empty() ? output : output.subspan(per_direction_offset, output_size - per_direction_offset); + + gsl::span hidden_output_2 = hidden_output.subspan(hidden_output_size_per_direction, + hidden_output_size_per_direction); + gsl::span last_cell_2 = last_cell.subspan(last_cell_size_per_direction, + last_cell_size_per_direction); + + fw = std::make_unique>(alloc, logger, + seq_length, batch_size, input_size, + hidden_size_, Direction::kForward, input_forget_, + bias_1, peephole_weights_1, initial_hidden_1, initial_cell_1, + activation_funcs_.Entries()[0], + activation_funcs_.Entries()[1], + activation_funcs_.Entries()[2], + clip_, ttp_); + + bw = std::make_unique>(alloc, logger, + seq_length, batch_size, input_size, + hidden_size_, Direction::kReverse, input_forget_, + bias_2, peephole_weights_2, initial_hidden_2, initial_cell_2, + activation_funcs_.Entries()[3], + activation_funcs_.Entries()[4], + activation_funcs_.Entries()[5], + clip_, ttp_); + + fw->Compute(input, sequence_lens_span, num_directions_, input_weights_1, recurrent_weights_1, output_1, hidden_output_1, last_cell_1); + bw->Compute(input, sequence_lens_span, num_directions_, input_weights_2, hidden_weights_2, output_2, hidden_output_2, last_cell_2); + } else { + fw = std::make_unique>(alloc, logger, + seq_length, batch_size, input_size, + hidden_size_, direction_, input_forget_, + bias_1, peephole_weights_1, initial_hidden_1, initial_cell_1, + activation_funcs_.Entries()[0], + activation_funcs_.Entries()[1], + activation_funcs_.Entries()[2], + clip_, ttp_); + + fw->Compute(input, sequence_lens_span, num_directions_, input_weights_1, recurrent_weights_1, output_1, hidden_output_1, last_cell_1); + } + + if (!output.empty()) + DumpMatrix("Y", output.data(), seq_length * num_directions_ * batch_size, hidden_size_); + + // these always get written to regardless of whether we're returning them as optional output or not + DumpMatrix("Y_h", hidden_output.data(), num_directions_ * batch_size, hidden_size_); + DumpMatrix("Y_c", last_cell.data(), num_directions_ * batch_size, hidden_size_); + + return Status::OK(); +} + +Status DeepCpuLstmOp::ValidateInputs(const Tensor& X, const Tensor& W, const Tensor& R, const Tensor* B, + const Tensor* sequence_lens, const Tensor* initial_h, const Tensor* initial_c, + const Tensor* P, int batch_size) const { + auto status = rnn::detail::ValidateCommonRnnInputs(X, W, R, B, 4, sequence_lens, initial_h, + num_directions_, hidden_size_); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + if (initial_c != nullptr) { + auto& initial_c_shape = initial_c->Shape(); + + if (initial_c_shape.NumDimensions() != 3 || + initial_c_shape[0] != num_directions_ || + initial_c_shape[1] != batch_size || + initial_c_shape[2] != hidden_size_) + + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Input initial_c must have shape {", + num_directions_, ",", batch_size, ",", hidden_size_, "}. Actual:", initial_c_shape); + } + + if (P != nullptr) { + auto& p_shape = P->Shape(); + + if (p_shape.NumDimensions() != 2 || + p_shape[0] != num_directions_ || + p_shape[1] != 3 * hidden_size_) + + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Input P must have shape {", + num_directions_, ",", 3 * hidden_size_, "}. Actual:", p_shape); + } + + return Status::OK(); +} + +/************************************* +* +* Implementation of UniDirectionalLstm +* +*/ +namespace detail { + +template +UniDirectionalLstm::UniDirectionalLstm(AllocatorPtr allocator, + const logging::Logger& logger, + const int seq_length, + const int batch_size, + const int input_size, + const int hidden_size, + Direction direction, + const bool input_forget, + const gsl::span& bias, + const gsl::span& peephole_weights, + const gsl::span& initial_hidden_state, + const gsl::span& initial_cell_state, + const ActivationFuncs::Entry& activation_func_f, + const ActivationFuncs::Entry& activation_func_g, + const ActivationFuncs::Entry& activation_func_h, + const float clip, + TaskThreadPool& ttp) + : allocator_(allocator), + logger_(logger), + seq_length_(seq_length), + batch_size_(batch_size), + input_size_(input_size), + hidden_size_(hidden_size), + direction_(direction), + input_forget_(input_forget), + clip_(clip), + use_bias_(!bias.empty()), + use_peepholes_(!peephole_weights.empty()), + ttp_(ttp) { + activation_f_ = {deepcpu::ActivationFuncByName(activation_func_f.name), + activation_func_f.alpha, + activation_func_f.beta}; + + activation_g_ = {deepcpu::ActivationFuncByName(activation_func_g.name), + activation_func_g.alpha, + activation_func_g.beta}; + + activation_h_ = {deepcpu::LstmMergeGatesFuncByName(activation_func_h.name), + activation_func_h.alpha, + activation_func_h.beta}; + + clip_with_bias_ptr_ = use_bias_ ? deepcpu::clip_add_bias : deepcpu::clip_ignore_bias; + + SetNumThreads(); + AllocateBuffers(); + InitializeBuffers(initial_hidden_state, initial_cell_state); + + if (!peephole_weights.empty()) + LoadPeepholeWeights(peephole_weights); + if (!bias.empty()) + LoadBias(bias); +} + +template +void UniDirectionalLstm::AllocateBuffers() { + // allocate and fill with 0's. + const bool fill = true; + hidden0_ = Allocate(allocator_, hidden_size_, hidden0_ptr_, fill); + internal_memory_prev_ = Allocate(allocator_, hidden_size_, internal_memory_prev_ptr_, fill); + internal_memory_cur_ = Allocate(allocator_, hidden_size_, internal_memory_cur_ptr_, fill); + batched_hidden0_ = Allocate(allocator_, batch_size_ * hidden_size_, batched_hidden0_ptr_, fill); + + batched_internal_memory_prev_ = Allocate(allocator_, batch_size_ * hidden_size_, + batched_internal_memory_prev_ptr_, fill); + batched_internal_memory_cur_ = Allocate(allocator_, batch_size_ * hidden_size_, + batched_internal_memory_cur_ptr_, fill); + batched_internal_memory_clipped_ = Allocate(allocator_, batch_size_ * hidden_size_, + batched_internal_memory_clipped_ptr_, fill); + + output_iofc_ = Allocate(allocator_, hidden_size_ * 4 * batch_size_ * seq_length_, output_iofc_ptr_, fill); + + if (use_bias_) { + bias_WRi_ = Allocate(allocator_, hidden_size_, bias_WRi_ptr_); + bias_WRf_ = Allocate(allocator_, hidden_size_, bias_WRf_ptr_); + bias_WRo_ = Allocate(allocator_, hidden_size_, bias_WRo_ptr_); + bias_WRc_ = Allocate(allocator_, hidden_size_, bias_WRc_ptr_); + } + + if (direction_ == kReverse) { + inputs_reverse_ = Allocate(allocator_, seq_length_ * batch_size_ * input_size_, inputs_reverse_ptr_); + outputs_reverse_ = Allocate(allocator_, seq_length_ * batch_size_ * hidden_size_, outputs_reverse_ptr_); + } + +#if !defined(LSTM_NO_PEEPHOLE_COPY) + if (use_peepholes_) { + peephole_i_ = Allocate(allocator_, hidden_size_, peephole_i_ptr_); + peephole_f_ = Allocate(allocator_, hidden_size_, peephole_f_ptr_); + peephole_o_ = Allocate(allocator_, hidden_size_, peephole_o_ptr_); + } +#endif +} + +template +void UniDirectionalLstm::InitializeBuffers(const gsl::span& initial_hidden_state, + const gsl::span& initial_cell_state) { + if (!initial_hidden_state.empty()) { + gsl::copy(initial_hidden_state, batched_hidden0_); + } else { + std::fill_n(batched_hidden0_.data(), batched_hidden0_.size(), T{}); + } + + if (!initial_cell_state.empty()) { + gsl::copy(initial_cell_state, batched_internal_memory_prev_); + } else { + std::fill_n(batched_internal_memory_prev_.data(), batched_internal_memory_prev_.size(), T{}); + } +} + +template +void UniDirectionalLstm::LoadPeepholeWeights(const gsl::span& peephole_weights) { + int i = 0; +#if defined(LSTM_NO_PEEPHOLE_COPY) + + // just use spans. we don't change these values so there's no point copying to them + peephole_i_ = peephole_weights.subspan((i++ * hidden_size_), hidden_size_); + peephole_o_ = peephole_weights.subspan((i++ * hidden_size_), hidden_size_); + peephole_f_ = peephole_weights.subspan((i++ * hidden_size_), hidden_size_); + +#else + DumpMatrix("P[i]", peephole_weights.data() + (i++ * hidden_size_), 1, hidden_size_); + DumpMatrix("P[o]", peephole_weights.data() + (i++ * hidden_size_), 1, hidden_size_); + DumpMatrix("P[f]", peephole_weights.data() + (i++ * hidden_size_), 1, hidden_size_); + + auto copy_weight = [this, &peephole_weights](int offset, gsl::span& out) { + typename gsl::span src = peephole_weights.subspan(offset, hidden_size_); + gsl::copy(src, out); + }; + + i = 0; + copy_weight((i++ * hidden_size_), peephole_i_); + copy_weight((i++ * hidden_size_), peephole_o_); + copy_weight((i++ * hidden_size_), peephole_f_); +#endif + + /* + DumpMatrix("peephole_i_", peephole_i_.data(), 1, hidden_size_); + DumpMatrix("peephole_o_", peephole_o_.data(), 1, hidden_size_); + DumpMatrix("peephole_f_", peephole_f_.data(), 1, hidden_size_); + */ +} + +template +void UniDirectionalLstm::LoadBias(const gsl::span& WbRb_values) { + // add Wb and Rb + auto copy_fused_bias = [this, &WbRb_values](int offset, gsl::span& out) { + // gap between Wb and Wb value for an entry + const int Wb_to_Rb_offset = 4 * hidden_size_; + + for (int j = 0; j < hidden_size_; ++j) + out[j] = WbRb_values[j + offset] + WbRb_values[j + offset + Wb_to_Rb_offset]; + }; + + int i = 0; + copy_fused_bias((i++) * hidden_size_, bias_WRi_); + copy_fused_bias((i++) * hidden_size_, bias_WRo_); + copy_fused_bias((i++) * hidden_size_, bias_WRf_); + copy_fused_bias((i++) * hidden_size_, bias_WRc_); + + /* + i = 0; + DumpMatrix("Wb[i]", WbRb_values.data() + (i++ * hidden_size_), 1, hidden_size_); + DumpMatrix("Wb[o]", WbRb_values.data() + (i++ * hidden_size_), 1, hidden_size_); + DumpMatrix("Wb[f]", WbRb_values.data() + (i++ * hidden_size_), 1, hidden_size_); + DumpMatrix("Wb[c]", WbRb_values.data() + (i++ * hidden_size_), 1, hidden_size_); + DumpMatrix("Rb[i]", WbRb_values.data() + (i++ * hidden_size_), 1, hidden_size_); + DumpMatrix("Rb[o]", WbRb_values.data() + (i++ * hidden_size_), 1, hidden_size_); + DumpMatrix("Rb[f]", WbRb_values.data() + (i++ * hidden_size_), 1, hidden_size_); + DumpMatrix("Rb[c]", WbRb_values.data() + (i++ * hidden_size_), 1, hidden_size_); + + DumpMatrix("Wb[i]+Rb[i]", bias_WRi_.data(), 1, hidden_size_); + DumpMatrix("Wb[o]+Rb[o]", bias_WRo_.data(), 1, hidden_size_); + DumpMatrix("Wb[f]+Rb[f]", bias_WRf_.data(), 1, hidden_size_); + DumpMatrix("Wb[c]+Rb[c]", bias_WRc_.data(), 1, hidden_size_); + */ +} + +template +void UniDirectionalLstm::Compute(const gsl::span& inputs_arg, + const gsl::span& sequence_lengths_arg, + const int num_directions, + const gsl::span& input_weights, + const gsl::span& recurrent_weights, + gsl::span& outputs, + gsl::span& final_hidden_state, + gsl::span& final_cell_state) { + // copy spans (just T* and size, not data in span) as we may change them + gsl::span inputs = inputs_arg; + gsl::span sequence_lengths = sequence_lengths_arg; + + // if sequence lengths weren't provided, use internal array and init all to seq_length + if (sequence_lengths.empty()) { + sequence_lengths_ = Allocate(allocator_, batch_size_, sequence_lengths_ptr_, true, seq_length_); + sequence_lengths = sequence_lengths_; + } + + // LSTM Layer + gsl::span batched_hidden_state_one_step = batched_hidden0_; + gsl::span batched_internal_state_prev_one_step = batched_internal_memory_prev_; + gsl::span batched_internal_state_clipped_one_step = batched_internal_memory_clipped_; + + int output_step_length = batch_size_ * hidden_size_; + + // The bidirectional LSTM wrapper wraps this LSTM class and produces bi-directional output + // the output has layout [seq,num_direction,batch,neurons]. + // When num_direction is 2, then this class will compute forward or backward LSTM. + // The outputs corresponds to either [seq,0,batch,neurons] or [seq,1,batch,neurons] + // Setting output_step_length this way allows writing the output directly without requiring + // additional memcpy. Note that if direction is kReverse, we write to output_reverse buffer + // which is then copied to output buffer, and ReverseSequence method handles the step length. + if (direction_ == kForward && num_directions == 2) + output_step_length = 2 * batch_size_ * hidden_size_; + + gsl::span original_outputs = outputs; + const bool output_sequence = !outputs.empty(); + + if (direction_ == kReverse) { + ReverseSequence(inputs, inputs_reverse_, sequence_lengths, seq_length_, batch_size_, input_size_, 1); + inputs = inputs_reverse_; + + if (output_sequence) + outputs = outputs_reverse_; + } + + // DumpMatrix("Input", inputs.data(), seq_length_, batch_size_ * input_size_); + + // Calculate the max and min length + int32_t max_sequence_length = *std::max_element(sequence_lengths.cbegin(), sequence_lengths.cend()); + int32_t min_sequence_length = std::min(seq_length_, *std::min_element(sequence_lengths.cbegin(), + sequence_lengths.cend())); + + ///**************************LSTM Calculations****************************/ + float alpha = 1.0f; + float beta = 0.0f; // first call to ComputeGemm zeros out any existing data + + const int hidden_size_x4 = 4 * hidden_size_; + const int total_rows = max_sequence_length * batch_size_; + + // apply the weights to all the inputs and save to output_IOFC + ComputeGemm(total_rows, hidden_size_x4, input_size_, alpha, + inputs.cbegin(), inputs.cend(), + input_size_, + input_weights.cbegin(), input_weights.cend(), // W[iofc] + input_size_, beta, + output_iofc_.begin(), output_iofc_.end(), + hidden_size_x4); + + DumpMatrix("Xt*(W[iofc]^T)", output_iofc_.data(), total_rows, hidden_size_x4); + + beta = 1.0f; // calls to ComputeGemm now add to existing data + + // NOTE: we could refine the bounds checking in the calls below that use these values to instead + // explicitly check just the range for each iteration, however if it's going to run over + // it should also run over on the last iteration, so this should be good enough to catch any + // logic errors causing bounds violations. + span_T_iter C_prev_end = batched_internal_state_prev_one_step.end(); + span_T_iter C_prev_clipped_end = batched_internal_state_clipped_one_step.end(); + span_T_const_iter previous_state_end = batched_hidden_state_one_step.end(); + + if (batch_parallel_) { + int fused_hidden_rows = batch_size_ / hidden_num_threads_; + if (batch_size_ % hidden_num_threads_ != 0) + fused_hidden_rows++; + + // lambda to do all processing on fused_hidden_rows rows + auto hidden_gemm_and_activations = [&](int row) { + //handling boundaries + int local_fused_hidden_rows = fused_hidden_rows; + if ((row + fused_hidden_rows) > batch_size_) + local_fused_hidden_rows = batch_size_ - row; + + // these are all batch * hidden_size_ and get updated in-place when running GateComputations so non-const iters + span_T_iter c_prev = batched_internal_state_prev_one_step.begin() + row * hidden_size_; + span_T_iter c_prev_clipped = batched_internal_state_clipped_one_step.begin() + row * hidden_size_; + + // hidden state can be provided as input for first step, so need to special case that. + // after the first step this will switch to the output from the previous step + span_T_const_iter previous_state = batched_hidden_state_one_step.cbegin() + row * hidden_size_; + + // run through steps sequentially + for (int step = 0; step < max_sequence_length; step++) { +#if defined(DUMP_MATRIXES) + const std::string row_str = " [row=" + std::to_string(row) + ",seqno=" + std::to_string(step) + "]"; +#endif + + span_T_iter step_out_IOFC = output_iofc_.begin() + (step * batch_size_ + row) * hidden_size_x4; + + // calculate Xt*(W[iofc]^T) + Ht-t*R[iofc] + ComputeGemm(local_fused_hidden_rows, hidden_size_x4, hidden_size_, alpha, + previous_state, previous_state_end, // Ht-1 + hidden_size_, + recurrent_weights.cbegin(), recurrent_weights.cend(), // R[iofc] + hidden_size_, beta, + step_out_IOFC, output_iofc_.end(), // input contains Xt*(W[iofc]^T) + hidden_size_x4); + + DumpMatrix("Xt*(W[iofc]^T) + Ht-t*R[iofc]" + row_str, + &*step_out_IOFC, local_fused_hidden_rows, hidden_size_x4); + + span_T_iter batched_output, batched_output_end; + if (output_sequence) { + batched_output = outputs.begin() + step * output_step_length; + batched_output_end = outputs.end(); + + } else { + batched_output = final_hidden_state.begin(); + batched_output_end = final_hidden_state.end(); + } + + span_T_iter step_out_IOFC_end = step_out_IOFC + local_fused_hidden_rows * hidden_size_x4; + GateComputations(step_out_IOFC, step_out_IOFC_end, + c_prev, C_prev_end, + c_prev_clipped, C_prev_clipped_end, + batched_output, batched_output_end, + sequence_lengths, min_sequence_length, step, row, local_fused_hidden_rows, output_sequence); + + // copy last row to final_cell_state + for (int lrow = row; lrow < row + local_fused_hidden_rows; ++lrow) { + if ((step + 1) == sequence_lengths[lrow]) { + gsl::span src = batched_internal_memory_prev_.subspan(lrow * hidden_size_, hidden_size_); + gsl::span dst = final_cell_state.subspan(lrow * hidden_size_, hidden_size_); + gsl::copy(src, dst); + } + } + + if (output_sequence) { + // set to 0 if step >= sequence_length + for (int lrow = row; lrow < row + local_fused_hidden_rows; lrow++) { + if (step >= min_sequence_length && step >= sequence_lengths[lrow]) { + auto output_lrow = outputs.begin() + step * output_step_length + lrow * hidden_size_; + std::fill_n(output_lrow, hidden_size_, (T)0); + } + } + } + + previous_state = batched_output + row * hidden_size_; + previous_state_end = batched_output_end; + } + }; + + ExecuteLambdaInParallel("Processing batch", hidden_gemm_and_activations, batch_size_, fused_hidden_rows, + ttp_, logger_); + + } else { + span_T_iter c_prev = batched_internal_state_prev_one_step.begin(); + span_T_iter c_prev_clipped = batched_internal_state_clipped_one_step.begin(); + + // hidden state can be provided as input for first step, so need to special case that. + // after the first step this will switch to the output from the previous step + span_T_const_iter previous_state = batched_hidden_state_one_step.cbegin(); + + //run through steps sequentially + for (int step = 0; step < max_sequence_length; step++) { +#if defined(DUMP_MATRIXES) + const std::string seqno_str = " [seqno=" + std::to_string(step) + "]"; +#endif + + DumpMatrix("previous_state" + seqno_str, &*previous_state, batch_size_, hidden_size_); + + span_T_iter step_out_IOFC = output_iofc_.begin() + (step * batch_size_) * hidden_size_x4; + + // calculate Xt*(W[iofc]^T) + Ht-t*R[iofc] + ComputeGemm(batch_size_, hidden_size_x4, hidden_size_, alpha, + previous_state, previous_state_end, // Ht-1 + hidden_size_, + recurrent_weights.cbegin(), recurrent_weights.cend(), // R[iofc] + hidden_size_, beta, + step_out_IOFC, output_iofc_.end(), // input contains Xt*(W[iofc]^T) + hidden_size_x4); + + span_T_iter batched_output, batched_output_end; + if (output_sequence) { + batched_output = outputs.begin() + step * output_step_length; + batched_output_end = outputs.end(); + + } else { + batched_output = final_hidden_state.begin(); + batched_output_end = final_hidden_state.end(); + } + + span_T_iter step_out_IOFC_end = step_out_IOFC + batch_size_ * hidden_size_x4; + GateComputations(step_out_IOFC, step_out_IOFC_end, + c_prev, C_prev_end, + c_prev_clipped, C_prev_clipped_end, + batched_output, batched_output_end, + sequence_lengths, min_sequence_length, step, 0, batch_size_, output_sequence); + + // copy last row to final_cell_state + for (int lrow = 0; lrow < batch_size_; lrow++) { + if ((step + 1) == sequence_lengths[lrow]) { + gsl::copy(batched_internal_memory_prev_.subspan(lrow * hidden_size_, hidden_size_), + final_cell_state.subspan(lrow * hidden_size_, hidden_size_)); + } + } + + if (output_sequence) { + //set to 0 if step >= sequence_length + for (int lrow = 0; lrow < batch_size_; lrow++) { + if (step >= min_sequence_length && step >= sequence_lengths[lrow]) { + auto dst = outputs.begin() + step * output_step_length + lrow * hidden_size_; + std::fill_n(dst, hidden_size_, (T)0); + } + } + } + + previous_state = batched_output; + previous_state_end = batched_output_end; + } + } + + if (output_sequence) { + // copy last output to final_hidden_state + for (int i = 0; i < batch_size_; i++) { + const int seq_len = sequence_lengths[i]; + auto src = outputs.subspan((seq_len - 1) * output_step_length + i * hidden_size_, hidden_size_); + auto dest = final_hidden_state.subspan(i * hidden_size_, hidden_size_); + gsl::copy(src, dest); + } + + if (direction_ == Direction::kReverse) + ReverseSequence(outputs, original_outputs, sequence_lengths, seq_length_, + batch_size_, hidden_size_, num_directions); + } +} + +// #define PREVIOUS_BROKEN_VERSION + +template +void UniDirectionalLstm::GateComputations(span_T_iter& out, span_T_iter& out_end, + span_T_iter& C_prev, span_T_iter& C_prev_end, // Ct-1 value not 'ct'. using 'C' for clarity + span_T_iter& C_prev_clipped, span_T_iter& C_prev_clipped_end, + span_T_iter& batched_output, span_T_iter& batched_output_end, + const gsl::span& seq_lengths, + const int min_sequence_length, + const int step, + const int row, + const int local_fused_hidden_rows, + bool output_sequence) { + int hidden_size_x4 = 4 * hidden_size_; + + // Activation gates. + for (int b = 0; b < local_fused_hidden_rows; b++) { + if (step >= min_sequence_length && step >= seq_lengths[row + b]) { + if (output_sequence) { + auto fill_output = batched_output + (row + b) * hidden_size_; + std::fill(fill_output, fill_output + hidden_size_, T{}); + } + + continue; + } + + // std::string row_str = " row[" + std::to_string(row + b) + "]"; + + // check that we have hidden_size_x4 left starting at cur_out + b * hidden_size_x4, and get a raw pointer to that + float* pi = SafeRawPointer(out + b * hidden_size_x4, out_end, hidden_size_x4); + float* po = pi + hidden_size_; + float* pf = po + hidden_size_; + float* pc = pf + hidden_size_; + +#ifdef PREVIOUS_BROKEN_VERSION + float* pCprev_hidden_size = SafeRawPointer(C_prev, C_prev_end, hidden_size_); +#else + float* pCprev_hidden_size = SafeRawPointer(C_prev + b * hidden_size_, C_prev_end, hidden_size_); +#endif + + // DumpMatrix("C_prev" + row_str, pCprev_hidden_size, 1, hidden_size_); + + // Input Gate + if (use_peepholes_) { + deepcpu::elementwise_product(pCprev_hidden_size, SafeRawConstPointer(peephole_i_, 0, hidden_size_), + pi, hidden_size_); + } + + const float* pBi = use_bias_ ? SafeRawConstPointer(bias_WRi_, 0, hidden_size_) : nullptr; + clip_with_bias_ptr_(clip_, pBi, pi, hidden_size_); // post: pi has input to f() to calculate i + activation_f_.func(pi, hidden_size_, activation_f_.alpha, activation_f_.beta); + // DumpMatrix("i" + row_str, pi, 1, hidden_size_); + + // Forget Gate + if (input_forget_) { + for (int i = 0; i < hidden_size_; i++) + pf[i] = 1.0f - pi[i]; + } else { + if (use_peepholes_) { + deepcpu::elementwise_product(pCprev_hidden_size, SafeRawConstPointer(peephole_f_, 0, hidden_size_), + pf, hidden_size_); + } + + const float* pBf = use_bias_ ? SafeRawConstPointer(bias_WRf_, 0, hidden_size_) : nullptr; + clip_with_bias_ptr_(clip_, pBf, pf, hidden_size_); + activation_f_.func(pf, hidden_size_, activation_f_.alpha, activation_f_.beta); + } + + // DumpMatrix("f" + row_str, pf, 1, hidden_size_); + + // Block Gate + const float* pBc = use_bias_ ? SafeRawConstPointer(bias_WRc_, 0, hidden_size_) : nullptr; + clip_with_bias_ptr_(clip_, pBc, pc, hidden_size_); + activation_g_.func(pc, hidden_size_, activation_g_.alpha, activation_g_.beta); + + // DumpMatrix("c" + row_str, pc, 1, hidden_size_); + + // C_current. use previous C value as input, and update in-place + float* pC_cur = pCprev_hidden_size; +#ifdef PREVIOUS_BROKEN_VERSION + deepcpu::merge_lstm_gates_to_memory(pCprev_hidden_size + b * hidden_size_, pi, pf, pc, pCprev_hidden_size + b * hidden_size_, hidden_size_); + // DumpMatrix("C", pCprev_hidden_size + b * hidden_size_, 1, hidden_size_); +#else + deepcpu::merge_lstm_gates_to_memory(pCprev_hidden_size, pi, pf, pc, pC_cur, hidden_size_); + // DumpMatrix("C", pC_cur, 1, hidden_size_); +#endif + + // Output Gate + if (use_peepholes_) + deepcpu::elementwise_product(pCprev_hidden_size, SafeRawConstPointer(peephole_o_, 0, hidden_size_), + po, hidden_size_); + + // calculate 'ot' + const float* pBo = use_bias_ ? SafeRawConstPointer(bias_WRo_, 0, hidden_size_) : nullptr; + clip_with_bias_ptr_(clip_, pBo, po, hidden_size_); + activation_f_.func(po, hidden_size_, activation_f_.alpha, activation_f_.beta); + // DumpMatrix("o" + row_str, po, 1, hidden_size_); + + // calculate 'Ht' + float* pH = SafeRawPointer(batched_output + row * hidden_size_ + b * hidden_size_, + batched_output_end, hidden_size_); + + // the C_prev_clipped location is not actually used as input - it's temporary storage for writing + // the clipped Ct value to, before calling h(). As such a) it could just be a local variable + // of std::vector with size of hidden_size_, b) the previous version wasn't 'broken' by never + // incrementing what C_prev_clipped pointed to. +#ifdef PREVIOUS_BROKEN_VERSION + float* pC_prev_clipped = SafeRawPointer(C_prev_clipped, C_prev_clipped_end, hidden_size_); +#else + float* pC_prev_clipped = SafeRawPointer(C_prev_clipped + b * hidden_size_, C_prev_clipped_end, hidden_size_); +#endif + + activation_h_.func(pC_cur, pC_prev_clipped, po, pH, hidden_size_, activation_h_.alpha, activation_h_.beta); + + // DumpMatrix("H" + row_str, pH, 1, hidden_size_); + } + + auto num_rows = local_fused_hidden_rows - row; + std::string rows_str = " rows[" + std::to_string(row) + ".." + std::to_string(num_rows) + "]"; + + DumpMatrix("i" + rows_str, &*out, num_rows, hidden_size_, 0, hidden_size_x4); + DumpMatrix("o" + rows_str, &*out, num_rows, hidden_size_, 1 * hidden_size_, hidden_size_x4); + DumpMatrix("f" + rows_str, &*out, num_rows, hidden_size_, 2 * hidden_size_, hidden_size_x4); + DumpMatrix("c" + rows_str, &*out, num_rows, hidden_size_, 3 * hidden_size_, hidden_size_x4); + DumpMatrix("C" + rows_str, &*C_prev, num_rows, hidden_size_); // Ct overwrites the input C_prev value + DumpMatrix("H" + rows_str, &*batched_output, num_rows, hidden_size_); +} + +template +void UniDirectionalLstm::SetNumThreads() { + int threads = std::thread::hardware_concurrency() - 1; + + if (threads < 1) + threads = 1; + + hidden_num_threads_ = threads; + batch_parallel_ = false; + + // for readability of the below logic + const auto num_rows = batch_size_; + const auto num_columns = hidden_size_; + + // parallelize by partitioning the batch rows + if (num_rows > 4 || (num_rows >= 2 && num_columns <= 256)) { + batch_parallel_ = true; + VLOGS(logger_, 1) << "Hidden Threads : " << hidden_num_threads_; + } +} + +} // namespace detail +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.h b/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.h new file mode 100644 index 0000000000000..790268b94acf1 --- /dev/null +++ b/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.h @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include "core/framework/op_kernel.h" +#include "core/common/task_thread_pool.h" +#include "core/providers/cpu/rnn/rnn_helpers.h" + +namespace onnxruntime { + +/// The class represents DeepCPU implementation of a long short term memory (LSTM) operator. +/// For details, refer to http://aka.ms/dl-optimization/. +class DeepCpuLstmOp final : public OpKernel { + public: + DeepCpuLstmOp(const OpKernelInfo& info) + : OpKernel(info), clip_(info.GetAttrOrDefault("clip", std::numeric_limits::max())) { + std::string direction; + ONNXRUNTIME_ENFORCE(info.GetAttr("direction", &direction).IsOK()); + + int64_t int64_value; + ONNXRUNTIME_ENFORCE(info.GetAttr("hidden_size", &int64_value).IsOK() && int64_value > 0); + hidden_size_ = gsl::narrow(int64_value); + + // optional attributes + std::vector activation_func_names = info.GetAttrsOrDefault("activations"); + std::vector activation_func_alphas = info.GetAttrsOrDefault("activation_alpha"); + std::vector activation_func_betas = info.GetAttrsOrDefault("activation_beta"); + ONNXRUNTIME_ENFORCE(clip_ > 0.f); + + if (info.GetAttr("input_forget", &int64_value).IsOK()) + input_forget_ = int64_value != 0; + + direction_ = rnn::detail::MakeDirection(direction); + num_directions_ = direction_ == rnn::detail::Direction::kBidirectional ? 2 : 1; + + if (activation_func_names.empty()) { + for (int i = 0; i < num_directions_; ++i) { + activation_func_names.emplace_back("sigmoid"); + activation_func_names.emplace_back("tanh"); + activation_func_names.emplace_back("tanh"); + } + } + + ONNXRUNTIME_ENFORCE(activation_func_names.size() == num_directions_ * 3); + + activation_funcs_ = rnn::detail::ActivationFuncs(activation_func_names, + activation_func_alphas, + activation_func_betas); + } + + Status Compute(OpKernelContext* context) const override; + + ~DeepCpuLstmOp() override = default; + + private: + template + Status ComputeImpl(OpKernelContext& context) const; + + Status ValidateInputs(const Tensor& X, + const Tensor& W, + const Tensor& R, + const Tensor* B, + const Tensor* sequence_lens, + const Tensor* initial_h, + const Tensor* initial_c, + const Tensor* P, + int batch_size) const; + + rnn::detail::Direction direction_; + int num_directions_; + + int hidden_size_ = 0; + float clip_; + bool input_forget_ = false; + + rnn::detail::ActivationFuncs activation_funcs_; + + // Threadpool for operator. If concurrent Compute calls are possible, it will be shared + // across them. mutable due to this. + // The alternative would be to create a threadpool in each call to Compute but that would incur thread creation + // cost on every call. + mutable TaskThreadPool ttp_{std::thread::hardware_concurrency()}; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/rnn/rnn.cc b/onnxruntime/core/providers/cpu/rnn/rnn.cc new file mode 100644 index 0000000000000..4a3f2f5ee050d --- /dev/null +++ b/onnxruntime/core/providers/cpu/rnn/rnn.cc @@ -0,0 +1,239 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/rnn/rnn.h" +#include "core/providers/cpu/rnn/rnn_activation_functors.h" +#include "core/providers/cpu/rnn/rnn_helpers.h" +#include "core/util/math.h" +#include "core/util/math_cpuonly.h" + +namespace onnxruntime { +ONNX_CPU_OPERATOR_KERNEL( + RNN, + 7, + KernelDefBuilder() + .TypeConstraint("T", DataTypeImpl::GetTensorType()) + .TypeConstraint("T1", DataTypeImpl::GetTensorType()), + RNN); + +// #define DUMP_MATRIXES to provide diagnostic output +#if defined(DUMP_MATRIXES) +#define DumpMatrix(...) ::onnxruntime::rnn::detail::DumpMatrixImpl(__VA_ARGS__) +#else +#define DumpMatrix(...) ((void)0) +#endif + +template +T Clip(const T& x, T clip) { + if (clip < 0) + return x; + else + return std::max(std::min(x, clip), -clip); +} + +template +void ApplyActivationToBatches(const Tensor* sequence_lens, const T* h_prev, T* Y_buffer_data_current_frame, + int64_t time_step, int64_t batch_size, int64_t hidden_size, + T alpha, T beta, T clip, std::function activation_func) { + const int* seq_len_data = sequence_lens ? sequence_lens->template Data() : nullptr; + + for (int batch = 0; batch < batch_size; batch++) { + bool valid = true; + if (nullptr != seq_len_data) { + // sequence_lens is already validated to have batch_size entries + valid = time_step < seq_len_data[batch]; + } + + for (int feature = 0; feature < hidden_size; ++feature) { + int64_t y_index = batch * hidden_size + feature; + if (!valid) { + // copy from previous time_step if available + Y_buffer_data_current_frame[y_index] = h_prev ? h_prev[batch * hidden_size + feature] : 0.f; + } else { + Y_buffer_data_current_frame[y_index] = activation_func( + Clip(Y_buffer_data_current_frame[y_index], clip), alpha, beta); + } + } + } +} + +template +void Assign_Y_h(const T* Y_buffer_data, Tensor* Y_h, const Tensor* sequence_lens, + int64_t num_directions, int direction, bool isReverse, int64_t batch_size, int64_t seq_length, int64_t hidden_size) { + for (int batch = 0; batch < batch_size; batch++) { + int64_t last_time_step = isReverse ? 0 : seq_length - 1; + if (nullptr != sequence_lens && !isReverse) + last_time_step = sequence_lens->template Data()[batch] - 1; + int64_t y_offset = last_time_step * num_directions * batch_size * hidden_size + + direction * batch_size * hidden_size + + batch * hidden_size; + int64_t Y_h_offset = direction * batch_size * hidden_size + batch * hidden_size; + math::CopyVector(static_cast(hidden_size), Y_buffer_data + y_offset, + Y_h->template MutableData() + Y_h_offset, + &CPUMathUtil::Instance()); + } +} + +template +void ClearMissingFrames(T* Y_buffer_data, const Tensor* sequence_lens, + int64_t num_directions, int64_t batch_size, int64_t seq_length, int64_t hidden_size) { + for (int direction = 0; direction < num_directions; direction++) { + for (int batch = 0; batch < batch_size; batch++) { + if (sequence_lens->template Data()[batch] < seq_length) { + for (int seq = sequence_lens->template Data()[batch]; seq < seq_length; seq++) { + int64_t offset = + seq * num_directions * batch_size * hidden_size + + direction * batch_size * hidden_size + + batch * hidden_size; + math::Set(hidden_size, 0, Y_buffer_data + offset, &CPUMathUtil::Instance()); + } + } + } + } +} + +template +using EigenMatrixMapRowMajor = Eigen::Map< + Eigen::Matrix>; + +template <> +Status RNN::Compute(OpKernelContext* ctx) const { + using namespace rnn::detail; + + // inputs + const Tensor& X = *ctx->Input(0); + const Tensor& W = *ctx->Input(1); + const Tensor& R = *ctx->Input(2); + + // optional inputs + const Tensor* B = ctx->Input(3); + const Tensor* sequence_lens = ctx->Input(4); + const Tensor* initial_h = ctx->Input(5); + + int64_t num_directions = direction_ == "bidirectional" ? 2 : 1; + int64_t seq_length = X.Shape()[0]; + int64_t batch_size = X.Shape()[1]; + int64_t input_size = X.Shape()[2]; + + auto status = rnn::detail::ValidateCommonRnnInputs(X, W, R, B, 1, sequence_lens, initial_h, + num_directions, hidden_size_); + ONNXRUNTIME_RETURN_IF_ERROR(status); + + // RNN outputs are optional + std::vector Y_dims({seq_length, num_directions, batch_size, hidden_size_}); + Tensor* Y = ctx->Output(0, Y_dims); + + std::vector Y_h_dims({num_directions, batch_size, hidden_size_}); + Tensor* Y_h = ctx->Output(1, Y_h_dims); + + AllocatorPtr alloc; + ONNXRUNTIME_RETURN_IF_ERROR(ctx->GetTempSpaceAllocator(&alloc)); + + // X * W^t, each direction has shape of [seq_length, batch_size, hidden_size] + auto x_matmul_data = alloc->Alloc(sizeof(float) * seq_length * batch_size * hidden_size_); + BufferUniquePtr x_matmul_buffer(x_matmul_data, BufferDeleter(alloc)); + float* x_matmul_w_buffer_data = static_cast(x_matmul_buffer.get()); + + float* Y_buffer_data; + void* Y_data; + BufferUniquePtr Y_matmul_buffer; + if (Y != nullptr) + Y_buffer_data = Y->template MutableData(); + else { + Y_data = alloc->Alloc(sizeof(float) * seq_length * num_directions * batch_size * hidden_size_); + Y_matmul_buffer = BufferUniquePtr(Y_data, BufferDeleter(alloc)); + Y_buffer_data = static_cast(Y_matmul_buffer.get()); + } + + int64_t Y_frame_size = batch_size * hidden_size_; + + for (int direction = 0; direction < num_directions; direction++) { + auto activation_func = GetFuncByName(activations_[direction], "Tanh"); + bool isReverse = direction_ == "reverse" || direction == 1; + + if (B != nullptr) { + EigenMatrixMapRowMajor(x_matmul_w_buffer_data, seq_length * batch_size, hidden_size_).rowwise() = + ConstEigenVectorMap(B->template Data() + direction * 2 * hidden_size_, hidden_size_).transpose() + + ConstEigenVectorMap(B->template Data() + direction * 2 * hidden_size_ + hidden_size_, hidden_size_).transpose(); + } else { + math::Set(seq_length * batch_size * hidden_size_, 0, x_matmul_w_buffer_data, &CPUMathUtil::Instance()); + } + + // X * W[direction]^t + B + math::Gemm( + CblasNoTrans, + CblasTrans, + static_cast(seq_length * batch_size), + static_cast(hidden_size_), + static_cast(input_size), + 1, + X.template Data(), + W.template Data() + direction * hidden_size_ * input_size, + 1, + x_matmul_w_buffer_data, + &CPUMathUtil::Instance()); + + for (int64_t t = 0; t < seq_length; t++) { + int64_t time_step = isReverse ? (seq_length - t - 1) : t; + int64_t Y_frame_offset = (time_step * num_directions + direction) * Y_frame_size; + float* Y_buffer_data_current_frame = Y_buffer_data + Y_frame_offset; + auto y_frame_mat = EigenMatrixMapRowMajor(Y_buffer_data_current_frame, batch_size, hidden_size_); + + const float* h_prev = nullptr; + if (t == 0) { + if (initial_h != nullptr) + h_prev = initial_h->template Data(); + } else { + if (isReverse) + h_prev = Y_buffer_data_current_frame + num_directions * Y_frame_size; + else + h_prev = Y_buffer_data_current_frame - num_directions * Y_frame_size; + } + + if (h_prev != nullptr) { + // H_t_1 * R[direction]^t + math::Gemm( + CblasNoTrans, + CblasTrans, + static_cast(batch_size), + static_cast(hidden_size_), + static_cast(hidden_size_), + 1, + h_prev, + R.template Data() + direction * hidden_size_ * hidden_size_, + 0, + Y_buffer_data_current_frame, + &CPUMathUtil::Instance()); + } else { + math::Set(batch_size * hidden_size_, 0, Y_buffer_data_current_frame, &CPUMathUtil::Instance()); + } + + // X[time_step] * W^t + H_t_1 * R^t + y_frame_mat += EigenMatrixMapRowMajor(&x_matmul_w_buffer_data[time_step * Y_frame_size], batch_size, hidden_size_); + + // apply activation + ApplyActivationToBatches(sequence_lens, h_prev, Y_buffer_data_current_frame, + time_step, batch_size, hidden_size_, + activation_alpha_[direction], activation_beta_[direction], clip_, activation_func); + } // close sequence loop + + if (Y_h) + Assign_Y_h(Y_buffer_data, Y_h, sequence_lens, + num_directions, direction, isReverse, batch_size, seq_length, hidden_size_); + } + + // Now the full sequence is completed. Set missing frames to zero. + if (nullptr != sequence_lens) { + ClearMissingFrames(Y_buffer_data, sequence_lens, + num_directions, batch_size, seq_length, hidden_size_); + } + + if (Y != nullptr) + DumpMatrix("Y", Y_buffer_data, (int)(seq_length * num_directions * batch_size), (int)hidden_size_); + + if (Y_h != nullptr) + DumpMatrix("Y_h", Y_h->template Data(), (int)(num_directions * batch_size), (int)hidden_size_); + + return Status::OK(); +} +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/rnn/rnn.h b/onnxruntime/core/providers/cpu/rnn/rnn.h new file mode 100644 index 0000000000000..5c59e1a42df87 --- /dev/null +++ b/onnxruntime/core/providers/cpu/rnn/rnn.h @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include "core/common/common.h" +#include "core/common/exceptions.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { +template +class RNN : public OpKernel { + const std::set allowed_activations{"Relu", "Tanh", "Sigmoid", "Affine", "LeakyRelu", "ThresholdedRelu", "ScaledTanh", "HardSigmoid", "Elu", "Softsign", "Softplus"}; + const std::set allowed_directions{"forward", "reverse", "bidirectional"}; + + public: + RNN(const OpKernelInfo& info) : OpKernel(info), + clip_(info.GetAttrOrDefault("clip", -1.0f)) { + ONNXRUNTIME_ENFORCE(info.GetAttr("direction", &direction_).IsOK()); + ONNXRUNTIME_ENFORCE(allowed_directions.find(direction_) != allowed_directions.end()); + const int num_directions = direction_ == "bidirectional" ? 2 : 1; + + activation_alpha_ = info.GetAttrsOrDefault("activation_alpha", std::vector(num_directions, 0.0F)); + activation_beta_ = info.GetAttrsOrDefault("activation_beta", std::vector(num_directions, 0.0F)); + ONNXRUNTIME_ENFORCE(info.GetAttrs("activations", activations_).IsOK()); + //TODO: is it optional or not? + ONNXRUNTIME_ENFORCE(info.GetAttr("hidden_size", &hidden_size_).IsOK()); + + if (activations_.size() == 2 && num_directions == 1) { + // ONNX RNN default activations are {"Tanh", "Tanh"} + // In this case, take the first default activation. + activations_.resize(1); + } + + ONNXRUNTIME_ENFORCE(activations_.size() == num_directions); + for (int direction = 1; direction < num_directions; direction++) { + ONNXRUNTIME_ENFORCE(allowed_activations.find(activations_[direction]) != allowed_activations.end()); + } + } + + Status Compute(OpKernelContext* context) const override; + + private: + // optional, default values tied to the activation function + std::vector activation_alpha_; + + // optional, default values tied to the activation function + std::vector activation_beta_; + + // optional, default = "Tanh" + std::vector activations_; + + // optional, default no clip_ + float clip_; + + // optional + std::string direction_; + + // required + int64_t hidden_size_; + + // const std::string default_activation = "Tanh"; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/rnn/rnn_activation_functors.h b/onnxruntime/core/providers/cpu/rnn/rnn_activation_functors.h new file mode 100644 index 0000000000000..dde46b425175f --- /dev/null +++ b/onnxruntime/core/providers/cpu/rnn/rnn_activation_functors.h @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#ifdef _WIN32 +#pragma warning(push) + +#pragma warning(disable : 4100) +#endif +#define RNN_UNUSED_PARAMETER ONNXRUNTIME_ATTRIBUTE_UNUSED = 0 +namespace onnxruntime { +namespace rnn { +namespace detail { + +template +inline T Affine(T x, T alpha, T beta) { + return alpha * x + beta; +} + +template +inline T Relu(T x, T alpha RNN_UNUSED_PARAMETER, T beta RNN_UNUSED_PARAMETER) { + return std::max(0.0f, static_cast(x)); +} + +template +inline T LeakyRelu(T x, T alpha, T beta RNN_UNUSED_PARAMETER) { + return x >= 0 ? x : alpha * x; +} + +template +inline T ThresholdedRelu(T x, T alpha, T beta RNN_UNUSED_PARAMETER) { + return x > alpha ? x : 0; +} + +template +inline T Sigmoid(T x, T alpha RNN_UNUSED_PARAMETER, T beta RNN_UNUSED_PARAMETER) { + if (x >= 0) { + return 1 / (1 + exp(-x)); + } else { + return exp(x) / (1 + exp(x)); + } +} + +template +inline T Tanh(T x, T alpha RNN_UNUSED_PARAMETER, T beta RNN_UNUSED_PARAMETER) { + return 2.0f * Sigmoid(2.0f * x) - 1.0f; +} + +template +inline T ScaledTanh(T x, T alpha, T beta) { + return alpha * Tanh(beta * x); +} + +template +inline T HardSigmoid(T x, T alpha, T beta) { + return std::min(1.0f, std::max(0.0f, alpha * x + beta)); +} + +template +inline T Elu(T x, T alpha, T beta RNN_UNUSED_PARAMETER) { + return x >= 0 ? x : alpha * (exp(x) - 1); +} + +template +inline T Softsign(T x, T alpha, T beta); + +template <> +inline float Softsign(float x, float alpha ONNXRUNTIME_ATTRIBUTE_UNUSED, float beta ONNXRUNTIME_ATTRIBUTE_UNUSED) { + return x / (1 + fabs(x)); +} + +template <> +inline double Softsign(double x, double alpha ONNXRUNTIME_ATTRIBUTE_UNUSED, double beta ONNXRUNTIME_ATTRIBUTE_UNUSED) { + return x / (1 + fabs(x)); +} + +template +inline T Softplus(T x, T alpha RNN_UNUSED_PARAMETER, T beta RNN_UNUSED_PARAMETER) { + return log(1 + exp(x)); +} + +#ifdef _WIN32 +#pragma warning(pop) +#endif + +template +std::function GetFuncByName(const std::string& name, const std::string& default_name) { + static std::unordered_map> NameToFuncMap( + {{"Affine", Affine}, + {"Relu", Relu}, + {"LeakyRelu", LeakyRelu}, + {"ThresholdedRelu", ThresholdedRelu}, + {"Tanh", Tanh}, + {"ScaledTanh", ScaledTanh}, + {"Sigmoid", Sigmoid}, + {"HardSigmoid", HardSigmoid}, + {"Elu", Elu}, + {"Softsign", Softsign}, + {"Softplus", Softplus}}); + + if (NameToFuncMap.find(name) == NameToFuncMap.end()) { + return NameToFuncMap[default_name]; + } + return NameToFuncMap[name]; +} + +} // namespace detail +} // namespace rnn +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/rnn/rnn_helpers.cc b/onnxruntime/core/providers/cpu/rnn/rnn_helpers.cc new file mode 100644 index 0000000000000..c455c67c99a9c --- /dev/null +++ b/onnxruntime/core/providers/cpu/rnn/rnn_helpers.cc @@ -0,0 +1,884 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/rnn/rnn_helpers.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cpu/rnn/rnn_activation_functors.h" +#include "core/util/math.h" +#include "core/util/math_cpuonly.h" + +namespace onnxruntime { +namespace rnn { +namespace detail { + +using namespace ::onnxruntime::common; + +Status ValidateCommonRnnInputs(const Tensor& X, + const Tensor& W, + const Tensor& R, + const Tensor* B, + int WRB_dim_1_multipler, + const Tensor* sequence_lens, + const Tensor* initial_h, + int64_t num_directions, + int64_t hidden_size) { + auto& X_shape = X.Shape(); + auto& W_shape = W.Shape(); + auto& R_shape = R.Shape(); + + int64_t seq_length = X_shape[0]; + int64_t batch_size = X_shape[1]; + int64_t input_size = X_shape[2]; + + if (X_shape.NumDimensions() != 3) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input X must have 3 dimensions only. Actual:", X_shape); + + if (W_shape.NumDimensions() != 3 || + W_shape[0] != num_directions || + W_shape[1] != hidden_size * WRB_dim_1_multipler || + W_shape[2] != input_size) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input W must have shape {", + num_directions, ",", WRB_dim_1_multipler, "*", hidden_size, ",", + input_size, "}. Actual:", W_shape); + + if (R_shape.NumDimensions() != 3 || + R_shape[0] != num_directions || + R_shape[1] != hidden_size * WRB_dim_1_multipler || + R_shape[2] != hidden_size) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input R must have shape {", + num_directions, ",", WRB_dim_1_multipler, "*", hidden_size, ",", + hidden_size, "}. Actual:", R_shape); + + if (B != nullptr) { + auto& B_shape = B->Shape(); + if (B_shape.NumDimensions() != 2 || + B_shape[0] != num_directions || + B_shape[1] != 2 * WRB_dim_1_multipler * hidden_size) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input B must have shape {", + num_directions, ",", 2 * WRB_dim_1_multipler, "*", hidden_size, "}. Actual:", B_shape); + } + + if (sequence_lens != nullptr) { + auto& sequence_lens_shape = sequence_lens->Shape(); + if (sequence_lens_shape.NumDimensions() != 1 || + sequence_lens_shape[0] != batch_size) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input sequence_lens must have shape {", + batch_size, "}. Actual:", sequence_lens_shape); + } + + auto sequence_len_entries = sequence_lens->DataAsSpan(); + if (std::any_of(sequence_len_entries.cbegin(), + sequence_len_entries.cend(), + [seq_length](int len) { return len <= 0 || len > seq_length; })) { + return ONNXRUNTIME_MAKE_STATUS( + ONNXRUNTIME, INVALID_ARGUMENT, + "Invalid value/s in sequence_lens. All values must be > 0 and < seq_length. seq_length=", seq_length); + } + } + + if (initial_h != nullptr) { + auto& initial_h_shape = initial_h->Shape(); + + if (initial_h_shape.NumDimensions() != 3 || + initial_h_shape[0] != num_directions || + initial_h_shape[1] != batch_size || + initial_h_shape[2] != hidden_size) + + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Input initial_h must have shape {", + num_directions, ",", batch_size, ",", hidden_size, "}. Actual:", initial_h_shape); + } + + return Status::OK(); +} // namespace detail + +// map of arg name and whether the alpha and/or beta arguments are required +static std::unordered_map> + NameToArgUsageMap{{"affine", {1, 1}}, + {"relu", {0, 0}}, + {"leakyrelu", {1, 0}}, + {"thresholdedrelu", {1, 0}}, + {"tanh", {0, 0}}, + {"scaledtanh", {1, 1}}, + {"sigmoid", {0, 0}}, + {"hardsigmoid", {1, 1}}, + {"elu", {1, 0}}, + {"softsign", {0, 0}}, + {"softplus", {0, 0}}}; + +// map of alpha/beta defaults +static std::unordered_map> + NameToArgDefaultsMap{{"leakyrelu", {0.01f, 0.f}}, + {"hardsigmoid", {0.2f, 0.5f}}, + {"elu", {1.0f, 0.f}}}; + +std::string NormalizeActivationArgumentAndGetAlphaBetaCount(const std::string& activation, + std::vector::const_iterator& cur_alpha, + const std::vector::const_iterator& end_alpha, + std::vector::const_iterator& cur_beta, + const std::vector::const_iterator& end_beta, + float& alpha, float& beta) { + std::string name(activation); + std::transform(name.begin(), name.end(), name.begin(), + [](const unsigned char i) { return static_cast(::tolower(i)); }); + + auto usage_entry = NameToArgUsageMap.find(name); + if (usage_entry == NameToArgUsageMap.end()) { + ONNXRUNTIME_THROW( + "Expecting activation to be one of Affine, Relu, LeakyRelu, " + "ThresholdedRelu, Tanh, ScaledTanh, Sigmoid, HardSigmoid, " + "Elu, Softsign, Softplus. Got " + + activation); + } + + bool needs_alpha = usage_entry->second.first; + bool needs_beta = usage_entry->second.second; + + auto set_if_needed = [](bool needed, + std::vector::const_iterator& in, + const std::vector::const_iterator& in_end, + const float default_val, + float& out) { + if (needed) { + if (in != in_end) { + out = *in; + ++in; + } else { + out = default_val; + } + } + }; + + auto defaults_entry = NameToArgDefaultsMap.find(name); + if (defaults_entry != NameToArgDefaultsMap.end()) { + set_if_needed(needs_alpha, cur_alpha, end_alpha, defaults_entry->second.first, alpha); + set_if_needed(needs_beta, cur_beta, end_beta, defaults_entry->second.second, beta); + } else { + set_if_needed(needs_alpha, cur_alpha, end_alpha, 0.f, alpha); + set_if_needed(needs_beta, cur_beta, end_beta, 0.f, beta); + } + + return name; +} + +ActivationFuncs::ActivationFuncs(const std::vector& funcs, + const std::vector& alphas, + const std::vector& betas) { + auto cur_alpha = alphas.cbegin(); + auto end_alpha = alphas.cend(); + auto cur_beta = betas.cbegin(); + auto end_beta = betas.cend(); + + for (auto input_func : funcs) { + float alpha = 0.f; + float beta = 0.f; + std::string func = detail::NormalizeActivationArgumentAndGetAlphaBetaCount( + input_func, cur_alpha, end_alpha, cur_beta, end_beta, alpha, beta); + + entries_.push_back(Entry{func, alpha, beta}); + } +} + +void DumpMatrixImpl(const std::string& name, const float* src, int row, int col, int offset, int col_width) { + std::cout << "Dump matrix: " << name << std::endl; + + if (col_width == -1) col_width = col; + + for (int r = 0; r < row; r++) { + for (int c = 0; c < col; c++) { + int index = r * col_width + offset + c; + std::cout << std::setw(12) << std::setprecision(8) << src[index]; + } + std::cout << std::endl; + } + std::cout << std::endl; +} + +namespace deepcpu { + +const float alpha_1 = 4.89352455891786e-03f; +const float alpha_3 = 6.37261928875436e-04f; +const float alpha_5 = 1.48572235717979e-05f; +const float alpha_7 = 5.12229709037114e-08f; +const float alpha_9 = -8.60467152213735e-11f; +const float alpha_11 = 2.00018790482477e-13f; +const float alpha_13 = -2.76076847742355e-16f; + +const float beta_0 = 4.89352518554385e-03f; +const float beta_2 = 2.26843463243900e-03f; +const float beta_4 = 1.18534705686654e-04f; +const float beta_6 = 1.19825839466702e-06f; + +const float sigmoid_bound = 20.0f; +const float tanh_bound = 10.0f; + +inline void clip_for_sigmoid_in_place(float* ps, int c) { + for (int i = 0; i < c; i++) { + if (ps[i] < -sigmoid_bound) + ps[i] = -sigmoid_bound; + else if (ps[i] > sigmoid_bound) + ps[i] = sigmoid_bound; + } +} + +inline void clip_for_tanh_in_place(float* ps, int c) { + for (int i = 0; i < c; i++) { + if (ps[i] < -tanh_bound) + ps[i] = -tanh_bound; + else if (ps[i] > tanh_bound) + ps[i] = tanh_bound; + } +} + +inline void clip_for_sigmoid(const float* ps, float* pd, int c) { + for (int i = 0; i < c; i++) { + if (ps[i] < -sigmoid_bound) + pd[i] = -sigmoid_bound; + else if (ps[i] > sigmoid_bound) + pd[i] = sigmoid_bound; + else + pd[i] = ps[i]; + } +} + +inline void clip_for_tanh(const float* ps, float* pd, int c) { + for (int i = 0; i < c; i++) { + if (ps[i] < -tanh_bound) + pd[i] = -tanh_bound; + else if (ps[i] > tanh_bound) + pd[i] = tanh_bound; + else + pd[i] = ps[i]; + } +} + +void add_bias_into_ignore(const float* ps, float* pd, const int c) { + ONNXRUNTIME_UNUSED_PARAMETER(ps); + ONNXRUNTIME_UNUSED_PARAMETER(pd); + ONNXRUNTIME_UNUSED_PARAMETER(c); +} + +void add_bias_into(const float* ps, float* pd, const int c) { + for (int i = 0; i < c; i++) { + pd[i] += ps[i]; + } +} + +void clip(const float b, float* pd, const int c) { + for (int i = 0; i < c; i++) { + float x = pd[i]; + if (x > b) + pd[i] = b; + else if (x < -b) + pd[i] = -b; + } +} + +void clip_ignore_bias(const float b, const float* pb, float* pd, const int c) { + ONNXRUNTIME_UNUSED_PARAMETER(pb); + + for (int i = 0; i < c; i++) { + float x = pd[i]; + if (x > b) + pd[i] = b; + else if (x < -b) + pd[i] = -b; + else + pd[i] = x; + } +} + +void clip_add_bias(const float b, const float* pb, float* pd, const int c) { + for (int i = 0; i < c; i++) { + float x = pd[i] + pb[i]; + if (x > b) + pd[i] = b; + else if (x < -b) + pd[i] = -b; + else + pd[i] = x; + } +} + +void sigmoid_m(const float* ps1, float* ps1_c, const float* ps2, float* pd, int c, + const float alpha, const float beta) { + ONNXRUNTIME_UNUSED_PARAMETER(alpha); + ONNXRUNTIME_UNUSED_PARAMETER(beta); + + clip_for_sigmoid(ps1, ps1_c, c); + + for (int i = 0; i < c; i++) { + float x = 0.5f * ps1_c[i]; + float x2 = x * x; + float p = x2 * alpha_13 + alpha_11; + p = x2 * p + alpha_9; + p = x2 * p + alpha_7; + p = x2 * p + alpha_5; + p = x2 * p + alpha_3; + p = x2 * p + alpha_1; + p = x * p; + float q = x2 * beta_6 + beta_4; + q = x2 * q + beta_2; + q = x2 * q + beta_0; + pd[i] = ps2[i] * 0.5f * (1 + (p / q)); + } +} + +void tanh_m(const float* ps1, float* ps1_c, const float* ps2, float* pd, int c, + const float alpha, const float beta) { + ONNXRUNTIME_UNUSED_PARAMETER(alpha); + ONNXRUNTIME_UNUSED_PARAMETER(beta); + + clip_for_tanh(ps1, ps1_c, c); + + for (int i = 0; i < c; i++) { + float x = ps1_c[i]; + float x2 = x * x; + float p = x2 * alpha_13 + alpha_11; + p = x2 * p + alpha_9; + p = x2 * p + alpha_7; + p = x2 * p + alpha_5; + p = x2 * p + alpha_3; + p = x2 * p + alpha_1; + p = x * p; + float q = x2 * beta_6 + beta_4; + q = x2 * q + beta_2; + q = x2 * q + beta_0; + pd[i] = ps2[i] * p / q; + } +} + +void relu_m(const float* ps1, float* ps1_c, const float* ps2, float* pd, int c, + const float alpha, const float beta) { + ONNXRUNTIME_UNUSED_PARAMETER(ps1_c); + ONNXRUNTIME_UNUSED_PARAMETER(alpha); + ONNXRUNTIME_UNUSED_PARAMETER(beta); + + for (int i = 0; i < c; i++) { + const float max = ps1[i] > 0 ? ps1[i] : 0.0f; + pd[i] = ps2[i] * max; + } +} + +void composed_m(const float* ps1, float* ps1_c, const float* ps2, float* pd, int c, + std::function func, + const float alpha, const float beta) { + ONNXRUNTIME_UNUSED_PARAMETER(ps1_c); + for (int i = 0; i < c; i++) { + pd[i] = ps2[i] * func(ps1[i], alpha, beta); + } +} + +void sigmoid_exact_m(const float* ps1, float* ps1_c, const float* ps2, float* pd, int c, + const float alpha, const float beta) { + ONNXRUNTIME_UNUSED_PARAMETER(ps1_c); + ONNXRUNTIME_UNUSED_PARAMETER(alpha); + ONNXRUNTIME_UNUSED_PARAMETER(beta); + + for (int i = 0; i < c; i++) { + float x = ps1[i]; + pd[i] = ps2[i] / (1 + ::std::exp(-x)); + } +} + +void tanh_exact_m(const float* ps1, float* ps1_c, const float* ps2, float* pd, int c, + const float alpha, const float beta) { + ONNXRUNTIME_UNUSED_PARAMETER(ps1_c); + ONNXRUNTIME_UNUSED_PARAMETER(alpha); + ONNXRUNTIME_UNUSED_PARAMETER(beta); + + for (int i = 0; i < c; i++) { + pd[i] = ::std::tanh(ps1[i]) * ps2[i]; + } +} + +void sigmoid(float* pd, int c, const float alpha, const float beta) { + ONNXRUNTIME_UNUSED_PARAMETER(alpha); + ONNXRUNTIME_UNUSED_PARAMETER(beta); + + clip_for_sigmoid_in_place(pd, c); + + for (int i = 0; i < c; i++) { + float x = 0.5f * pd[i]; + float x2 = x * x; + float p = x2 * alpha_13 + alpha_11; + p = x2 * p + alpha_9; + p = x2 * p + alpha_7; + p = x2 * p + alpha_5; + p = x2 * p + alpha_3; + p = x2 * p + alpha_1; + p = x * p; + float q = x2 * beta_6 + beta_4; + q = x2 * q + beta_2; + q = x2 * q + beta_0; + pd[i] = 0.5f * (1 + (p / q)); + } +} + +void tanh(float* pd, int c, const float alpha, const float beta) { + ONNXRUNTIME_UNUSED_PARAMETER(alpha); + ONNXRUNTIME_UNUSED_PARAMETER(beta); + + clip_for_tanh_in_place(pd, c); + + for (int i = 0; i < c; i++) { + float x = pd[i]; + float x2 = x * x; + float p = x2 * alpha_13 + alpha_11; + p = x2 * p + alpha_9; + p = x2 * p + alpha_7; + p = x2 * p + alpha_5; + p = x2 * p + alpha_3; + p = x2 * p + alpha_1; + p = x * p; + float q = x2 * beta_6 + beta_4; + q = x2 * q + beta_2; + q = x2 * q + beta_0; + pd[i] = p / q; + } +} + +void relu(float* pd, int c, const float alpha, const float beta) { + ONNXRUNTIME_UNUSED_PARAMETER(alpha); + ONNXRUNTIME_UNUSED_PARAMETER(beta); + + for (int i = 0; i < c; i++) { + if (pd[i] < 0) + pd[i] = 0.0f; + } +} + +void sigmoid_exact(float* pd, int c, const float alpha, const float beta) { + ONNXRUNTIME_UNUSED_PARAMETER(alpha); + ONNXRUNTIME_UNUSED_PARAMETER(beta); + + for (int i = 0; i < c; i++) { + float x = pd[i]; + pd[i] = 1.0f / (1 + ::std::exp(-x)); + } +} + +void tanh_exact(float* pd, int c, const float alpha, const float beta) { + ONNXRUNTIME_UNUSED_PARAMETER(alpha); + ONNXRUNTIME_UNUSED_PARAMETER(beta); + + for (int i = 0; i < c; i++) { + float x = pd[i]; + pd[i] = ::std::tanh(x); + } +} + +void merge_lstm_gates_to_memory(const float* pprev, const float* pi, const float* pf, const float* pg, + float* pcurr, const int c) { + for (int i = 0; i < c; i++) { + pcurr[i] = pprev[i] * pf[i] + pi[i] * pg[i]; + } +} + +void gru_reset_gate_tanh(const float* ps1, float* ps2, float* pd, const int c, + const float alpha, const float beta) { + ONNXRUNTIME_UNUSED_PARAMETER(alpha); + ONNXRUNTIME_UNUSED_PARAMETER(beta); + + clip_for_tanh_in_place(ps2, c); + + for (int i = 0; i < c; i++) { + float x = ps2[i]; + float x2 = x * x; + float p = x2 * alpha_13 + alpha_11; + p = x2 * p + alpha_9; + p = x2 * p + alpha_7; + p = x2 * p + alpha_5; + p = x2 * p + alpha_3; + p = x2 * p + alpha_1; + p = x * p; + float q = x2 * beta_6 + beta_4; + q = x2 * q + beta_2; + q = x2 * q + beta_0; + pd[i] = ps1[i] * p / q; + } +} + +void gru_reset_gate_sigmoid(const float* ps1, float* ps2, float* pd, const int c, + const float alpha, const float beta) { + ONNXRUNTIME_UNUSED_PARAMETER(alpha); + ONNXRUNTIME_UNUSED_PARAMETER(beta); + + clip_for_sigmoid_in_place(ps2, c); + + for (int i = 0; i < c; i++) { + float x = 0.5f * ps2[i]; + float x2 = x * x; + float p = x2 * alpha_13 + alpha_11; + p = x2 * p + alpha_9; + p = x2 * p + alpha_7; + p = x2 * p + alpha_5; + p = x2 * p + alpha_3; + p = x2 * p + alpha_1; + p = x * p; + float q = x2 * beta_6 + beta_4; + q = x2 * q + beta_2; + q = x2 * q + beta_0; + pd[i] = ps1[i] * 0.5f * (1 + p / q); + } +} + +void gru_reset_gate_relu(const float* ps1, float* ps2, float* pd, const int c, + const float alpha, const float beta) { + ONNXRUNTIME_UNUSED_PARAMETER(alpha); + ONNXRUNTIME_UNUSED_PARAMETER(beta); + + for (int i = 0; i < c; i++) { + const auto max = ps2[i] > 0 ? ps2[i] : 0.0f; + pd[i] = ps1[i] * max; + } +} + +void gru_reset_gate_composed(const float* ps1, float* ps2, float* pd, const int c, + std::function func, + const float alpha, const float beta) { + for (int i = 0; i < c; i++) { + pd[i] = ps1[i] * func(ps2[i], alpha, beta); + } +} + +void gru_output_gate_tanh(float* ph, const float* pz, const float* ps, float* po, const int c, + const float alpha, const float beta) { + ONNXRUNTIME_UNUSED_PARAMETER(alpha); + ONNXRUNTIME_UNUSED_PARAMETER(beta); + + clip_for_tanh_in_place(ph, c); + + for (int i = 0; i < c; i++) { + float x = ph[i]; + float x2 = x * x; + float p = x2 * alpha_13 + alpha_11; + p = x2 * p + alpha_9; + p = x2 * p + alpha_7; + p = x2 * p + alpha_5; + p = x2 * p + alpha_3; + p = x2 * p + alpha_1; + p = x * p; + float q = x2 * beta_6 + beta_4; + q = x2 * q + beta_2; + q = x2 * q + beta_0; + po[i] = (1 - pz[i]) * (p / q) + pz[i] * ps[i]; + } +} + +void gru_output_gate_relu(float* ph, const float* pz, const float* ps, float* po, const int c, + const float alpha, const float beta) { + ONNXRUNTIME_UNUSED_PARAMETER(alpha); + ONNXRUNTIME_UNUSED_PARAMETER(beta); + + for (int i = 0; i < c; i++) { + float max = ph[i] > 0 ? ph[i] : 0.0f; + po[i] = (1 - pz[i]) * max + pz[i] * ps[i]; + } +} + +void gru_output_gate_composed(float* ph, const float* pz, const float* ps, float* po, const int c, + std::function func, + const float alpha, const float beta) { + for (int i = 0; i < c; i++) { + po[i] = (1 - pz[i]) * func(ph[i], alpha, beta) + pz[i] * ps[i]; + } +} + +void gru_output_gate_sigmoid(float* ph, const float* pz, const float* ps, float* po, const int c, + const float alpha, const float beta) { + ONNXRUNTIME_UNUSED_PARAMETER(alpha); + ONNXRUNTIME_UNUSED_PARAMETER(beta); + + clip_for_sigmoid_in_place(ph, c); + + for (int i = 0; i < c; i++) { + float x = 0.5f * (ph[i]); + float x2 = x * x; + float p = x2 * alpha_13 + alpha_11; + p = x2 * p + alpha_9; + p = x2 * p + alpha_7; + p = x2 * p + alpha_5; + p = x2 * p + alpha_3; + p = x2 * p + alpha_1; + p = x * p; + float q = x2 * beta_6 + beta_4; + q = x2 * q + beta_2; + q = x2 * q + beta_0; + po[i] = (1 - pz[i]) * 0.5f * (1 + (p / q)) + pz[i] * ps[i]; + } +} + +void composed_activation_func(float* ps, const int c, + std::function func, + const float alpha, const float beta) { + for (int i = 0; i < c; i++) { + ps[i] = func(ps[i], alpha, beta); + } +} + +void composed_lstm_merge_gates_func(float* ps, const int c, + std::function func, + const float alpha, const float beta) { + for (int i = 0; i < c; i++) { + ps[i] = func(ps[i], alpha, beta); + } +} + +void composed_gru_reset_gate_func(float* ps, const int c, + std::function func, + const float alpha, const float beta) { + for (int i = 0; i < c; i++) { + ps[i] = func(ps[i], alpha, beta); + } +} + +void composed_gru_output_gate_func(float* ps, const int c, + std::function func, + const float alpha, const float beta) { + for (int i = 0; i < c; i++) { + ps[i] = func(ps[i], alpha, beta); + } +} + +ActivationFuncPtr ActivationFuncByName(const std::string& func) { + if (func == "sigmoid") + return sigmoid; + + if (func == "tanh") + return tanh; + + if (func == "relu") + return relu; + + if (func == "affine") + return [](float* ps, const int c, const float alpha, const float beta) { + composed_activation_func(ps, c, Affine, alpha, beta); + }; + + if (func == "leakyrelu") + return [](float* ps, const int c, const float alpha, const float beta) { + composed_activation_func(ps, c, LeakyRelu, alpha, beta); + }; + + if (func == "thresholdedrelu") + return [](float* ps, const int c, const float alpha, const float beta) { + composed_activation_func(ps, c, ThresholdedRelu, alpha, beta); + }; + + if (func == "scaledtanh") + return [](float* ps, const int c, const float alpha, const float beta) { + composed_activation_func(ps, c, ScaledTanh, alpha, beta); + }; + + if (func == "hardsigmoid") + return [](float* ps, const int c, const float alpha, const float beta) { + composed_activation_func(ps, c, HardSigmoid, alpha, beta); + }; + + if (func == "elu") + return [](float* ps, const int c, const float alpha, const float beta) { + composed_activation_func(ps, c, Elu, alpha, beta); + }; + + if (func == "softsign") + return [](float* ps, const int c, const float alpha, const float beta) { + composed_activation_func(ps, c, Softsign, alpha, beta); + }; + + if (func == "softplus") + return [](float* ps, const int c, const float alpha, const float beta) { + composed_activation_func(ps, c, Softplus, alpha, beta); + }; + + ONNXRUNTIME_THROW("Invalid activation function of ", func); +} + +LstmMergeGatesFuncPtr LstmMergeGatesFuncByName(const std::string& func) { + if (func == "sigmoid") + return sigmoid_m; + + if (func == "tanh") + return tanh_m; + + if (func == "relu") + return relu_m; + + if (func == "affine") + return [](const float* ps1, float* ps1_c, const float* ps2, float* ps3, const int c, + const float alpha, const float beta) { + composed_m(ps1, ps1_c, ps2, ps3, c, Affine, alpha, beta); + }; + + if (func == "leakyrelu") + return [](const float* ps1, float* ps1_c, const float* ps2, float* ps3, const int c, + const float alpha, const float beta) { + composed_m(ps1, ps1_c, ps2, ps3, c, LeakyRelu, alpha, beta); + }; + + if (func == "thresholdedrelu") + return [](const float* ps1, float* ps1_c, const float* ps2, float* ps3, const int c, + const float alpha, const float beta) { + composed_m(ps1, ps1_c, ps2, ps3, c, ThresholdedRelu, alpha, beta); + }; + + if (func == "scaledtanh") + return [](const float* ps1, float* ps1_c, const float* ps2, float* ps3, const int c, + const float alpha, const float beta) { + composed_m(ps1, ps1_c, ps2, ps3, c, ScaledTanh, alpha, beta); + }; + + if (func == "hardsigmoid") + return [](const float* ps1, float* ps1_c, const float* ps2, float* ps3, const int c, + const float alpha, const float beta) { + composed_m(ps1, ps1_c, ps2, ps3, c, HardSigmoid, alpha, beta); + }; + + if (func == "elu") + return [](const float* ps1, float* ps1_c, const float* ps2, float* ps3, const int c, + const float alpha, const float beta) { + composed_m(ps1, ps1_c, ps2, ps3, c, Elu, alpha, beta); + }; + + if (func == "softsign") + return [](const float* ps1, float* ps1_c, const float* ps2, float* ps3, const int c, + const float alpha, const float beta) { + composed_m(ps1, ps1_c, ps2, ps3, c, Softsign, alpha, beta); + }; + + if (func == "softplus") + return [](const float* ps1, float* ps1_c, const float* ps2, float* ps3, const int c, + const float alpha, const float beta) { + composed_m(ps1, ps1_c, ps2, ps3, c, Softplus, alpha, beta); + }; + + ONNXRUNTIME_THROW("Invalid LSTM merge activation function of ", func); +} + +GruResetGateFuncPtr GruResetGateFuncByName(const std::string& func) { + if (func == "sigmoid") + return gru_reset_gate_sigmoid; + + if (func == "tanh") + return gru_reset_gate_tanh; + + if (func == "relu") + return gru_reset_gate_relu; + + if (func == "affine") + return [](const float* ps1, float* ps2, float* ps3, const int c, const float alpha, const float beta) { + gru_reset_gate_composed(ps1, ps2, ps3, c, Affine, alpha, beta); + }; + + if (func == "leakyrelu") + return [](const float* ps1, float* ps2, float* ps3, const int c, const float alpha, const float beta) { + gru_reset_gate_composed(ps1, ps2, ps3, c, LeakyRelu, alpha, beta); + }; + + if (func == "thresholdedrelu") + return [](const float* ps1, float* ps2, float* ps3, const int c, const float alpha, const float beta) { + gru_reset_gate_composed(ps1, ps2, ps3, c, ThresholdedRelu, alpha, beta); + }; + + if (func == "scaledtanh") + return [](const float* ps1, float* ps2, float* ps3, const int c, const float alpha, const float beta) { + gru_reset_gate_composed(ps1, ps2, ps3, c, ScaledTanh, alpha, beta); + }; + + if (func == "hardsigmoid") + return [](const float* ps1, float* ps2, float* ps3, const int c, const float alpha, const float beta) { + gru_reset_gate_composed(ps1, ps2, ps3, c, HardSigmoid, alpha, beta); + }; + + if (func == "elu") + return [](const float* ps1, float* ps2, float* ps3, const int c, const float alpha, const float beta) { + gru_reset_gate_composed(ps1, ps2, ps3, c, Elu, alpha, beta); + }; + + if (func == "softsign") + return [](const float* ps1, float* ps2, float* ps3, const int c, const float alpha, const float beta) { + gru_reset_gate_composed(ps1, ps2, ps3, c, Softsign, alpha, beta); + }; + + if (func == "softplus") + return [](const float* ps1, float* ps2, float* ps3, const int c, const float alpha, const float beta) { + gru_reset_gate_composed(ps1, ps2, ps3, c, Softplus, alpha, beta); + }; + + ONNXRUNTIME_THROW("Invalid GRU reset gate activation function: ", func); +} + +GruOutputGateFuncPtr GruOutputGateFuncByName(const std::string& func) { + if (func == "sigmoid") + return gru_output_gate_sigmoid; + + if (func == "tanh") + return gru_output_gate_tanh; + + if (func == "relu") + return gru_output_gate_relu; + + if (func == "affine") + return [](float* ps1, const float* ps2, const float* ph, float* ps3, const int c, + const float alpha, const float beta) { + gru_output_gate_composed(ps1, ps2, ph, ps3, c, Affine, alpha, beta); + }; + + if (func == "leakyrelu") + return [](float* ps1, const float* ps2, const float* ph, float* ps3, const int c, + const float alpha, const float beta) { + gru_output_gate_composed(ps1, ps2, ph, ps3, c, LeakyRelu, alpha, beta); + }; + + if (func == "thresholdedrelu") + return [](float* ps1, const float* ps2, const float* ph, float* ps3, const int c, + const float alpha, const float beta) { + gru_output_gate_composed(ps1, ps2, ph, ps3, c, ThresholdedRelu, alpha, beta); + }; + + if (func == "scaledtanh") + return [](float* ps1, const float* ps2, const float* ph, float* ps3, const int c, + const float alpha, const float beta) { + gru_output_gate_composed(ps1, ps2, ph, ps3, c, ScaledTanh, alpha, beta); + }; + + if (func == "hardsigmoid") + return [](float* ps1, const float* ps2, const float* ph, float* ps3, const int c, + const float alpha, const float beta) { + gru_output_gate_composed(ps1, ps2, ph, ps3, c, HardSigmoid, alpha, beta); + }; + + if (func == "elu") + return [](float* ps1, const float* ps2, const float* ph, float* ps3, const int c, + const float alpha, const float beta) { + gru_output_gate_composed(ps1, ps2, ph, ps3, c, Elu, alpha, beta); + }; + + if (func == "softsign") + return [](float* ps1, const float* ps2, const float* ph, float* ps3, const int c, + const float alpha, const float beta) { + gru_output_gate_composed(ps1, ps2, ph, ps3, c, Softsign, alpha, beta); + }; + + if (func == "softplus") + return [](float* ps1, const float* ps2, const float* ph, float* ps3, const int c, + const float alpha, const float beta) { + gru_output_gate_composed(ps1, ps2, ph, ps3, c, Softplus, alpha, beta); + }; + + ONNXRUNTIME_THROW("Invalid GRU hidden gate activation function: ", func); +} + +} // namespace deepcpu +} // namespace detail +} // namespace rnn +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/rnn/rnn_helpers.h b/onnxruntime/core/providers/cpu/rnn/rnn_helpers.h new file mode 100644 index 0000000000000..fe56ae7ee7c1c --- /dev/null +++ b/onnxruntime/core/providers/cpu/rnn/rnn_helpers.h @@ -0,0 +1,325 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "gsl/span" +#include "gsl/gsl_algorithm" + +#include "core/common/common.h" +#include "core/common/task_thread_pool.h" +#include "core/common/logging/logging.h" +#include "core/framework/allocator.h" +#include "core/util/math.h" +#include "core/util/math_cpuonly.h" + +namespace onnxruntime { +class Tensor; +class OpKernelContext; + +namespace rnn { +namespace detail { + +enum Direction { + kForward = 0, + kReverse = 1, + kBidirectional = 2 +}; + +inline Direction MakeDirection(const std::string& direction) { + if (direction == "forward") { + return kForward; + } else if (direction == "reverse") { + return kReverse; + } else if (direction == "bidirectional") { + return kBidirectional; + } else { + ONNXRUNTIME_THROW("Invalid 'direction' argument of '", direction, + "'. Must be one of 'forward', 'reverse', or 'bidirectional'."); + } +} + +/** Allocate a unique_ptr using allocator_, and return a span to the allocated memory so usage is safe +@param allocator IAllocator to use for the allocation. +@param size Allocation size. Number of elements of type TAlloc, or total size if TAlloc is 'void'. +@param unique_ptr unique_ptr that will control the lifetime of the allocated memory. +@param fill If true, fill the allocated memory with fill_value. +@param fill_value Value to use if 'fill' is true. +@returns A span to provide bounds checked access to the allocated memory. +*/ +template +gsl::span Allocate(std::shared_ptr allocator, + size_t size, + IAllocatorUniquePtr& unique_ptr, + bool fill = false, TAlloc fill_value = TAlloc{}) { + unique_ptr = IAllocator::MakeUniquePtr(allocator, size); + auto span = gsl::make_span(unique_ptr.get(), size); + + if (fill) { + // Do't use span.begin() it will cause performance issue and stop compiler to optimize the code + std::fill_n(unique_ptr.get(), size, fill_value); + } + + return span; +} + +// validate the common inputs to RNN, LSTM and GRU operators +Status ValidateCommonRnnInputs(const Tensor& X, + const Tensor& W, + const Tensor& R, + const Tensor* B, + int WRB_dim_1_multipler, // multiplier used with hidden_size for W, R and B inputs + const Tensor* sequence_lens, + const Tensor* initial_h, + int64_t num_directions, + int64_t hidden_size); + +/// Copy an input array repeatedly to an output array +/// @param input_begin Beginning of input +/// @param input_end End of input +/// @param output Output iterator +/// @param repetitions Number of times to repeat copy. Assumes output is sufficiently sized. +/// @returns Position of output iterator after copy is completed +template +TOutIter RepeatVectorToConstructArray(TInIter input_begin, + TInIter input_end, + TOutIter output, + int64_t repetitions) { + for (int64_t i = 0; i < repetitions; i++) { + output = std::copy(input_begin, input_end, output); + } + + return output; +} + +// reverse an LSTM or GRU sequence which has shape [seq_length, batch_size, hidden_size] +// and output to shape [seq_length, num_directions, batch_size, hidden_size] +template +void ReverseSequence(gsl::span inputs, + gsl::span inputs_reverse, + gsl::span sequence_lengths, + const int max_sequence_length, + const int batch_size, + const int input_size, + const int num_directions) { + for (int i = 0; i < batch_size; i++) { + int seq_len = sequence_lengths[i]; + + if (seq_len == 0) + continue; + + // Parallel execute the loop. + #pragma omp for + for (int j = 0; j < seq_len; j++) { + gsl::span src = inputs.subspan(j * batch_size * input_size + i * input_size, input_size); + gsl::span dest = inputs_reverse.subspan(num_directions * (seq_len - j - 1) * batch_size * input_size + i * input_size, input_size); + + // Use gsl::copy instead of std::copy() to allow compiler to optimize the code + gsl::copy(src, dest); + } + + #pragma omp for + for (int j = seq_len; j < max_sequence_length; j++) { + gsl::span src = inputs.subspan(j * batch_size * input_size + i * input_size, input_size); + gsl::span dest = inputs_reverse.subspan(num_directions * j * batch_size * input_size + i * input_size, input_size); + + // Use gsl::copy instead of std::copy() to allow compiler to optimize the code + gsl::copy(src, dest); + } + } +} + +// A has size M x K, B has size N x K (transposed), and C has size M x N +// We check that A, B and C are large enough before calling the lower level GEMM implementation +template +void ComputeGemm(const int M, + const int N, + const int K, + const float alpha, + TSpanAIter A, + TSpanAIter A_end, + const int lda, + TSpanBIter B, + TSpanBIter B_end, + const int ldb, + const float beta, + TSpanCIter C, + TSpanCIter C_end, + const int ldc) { + // validate all the inputs + // need to use the lda/ldb/ldc strides which should be >= the columns for the span + ONNXRUNTIME_ENFORCE(lda >= K && ldb >= K && ldc >= N); + ONNXRUNTIME_ENFORCE(A + (M * lda - (lda - K)) <= A_end); + ONNXRUNTIME_ENFORCE(B + (N * ldb - (ldb - K)) <= B_end); + ONNXRUNTIME_ENFORCE(C + (M * ldc - (ldc - N)) <= C_end); + + ::onnxruntime::math::GemmEx( + CblasNoTrans, CblasTrans, + M, N, K, alpha, + &*A, lda, + &*B, ldb, beta, + &*C, ldc, &CPUMathUtil::Instance()); +} + +// helper to convert a span to a raw pointer +// after validating the memory covered by the span supports the size required +template +const T* SafeRawConstPointer(typename gsl::span::const_iterator cur, + typename gsl::span::const_iterator end, + size_t size) { + ONNXRUNTIME_ENFORCE(cur + size <= end); + return &*cur; +} + +// helper to convert a span to a raw pointer +// after validating the memory covered by the span supports the size required +template +const T* SafeRawConstPointer(gsl::span span, size_t offset, size_t size) { + ONNXRUNTIME_ENFORCE(offset + size <= size_t(span.size())); + return span.data(); +} + +// helper to convert a span to a raw pointer +// after validating the memory covered by the span supports the size required +template +T* SafeRawPointer(typename gsl::span::iterator cur, + typename gsl::span::iterator end, + size_t size) { + ONNXRUNTIME_ENFORCE(cur + size <= end); + return &*cur; +} + +// helper to convert a span to a raw pointer +// after validating the memory covered by the span supports the size required +template +T* SafeRawPointer(typename gsl::span span, size_t offset, size_t size) { + ONNXRUNTIME_ENFORCE(offset + size <= size_t(span.size())); + return span.data() + offset; +} + +template +void ExecuteLambdaInParallel(const std::string& name, TLambda lambda, int max, int step, + TaskThreadPool& ttp, const ::onnxruntime::logging::Logger& logger) { + // #define NOTHREADS to execute the lambdas directly and in order if you need to do that to debug + +#ifdef NOTHREADS + ONNXRUNTIME_UNUSED_PARAMETER(ttp); + ONNXRUNTIME_UNUSED_PARAMETER(logger); + + for (int i = 0; i < max; i += step) { + (void)name; + std::bind(lambda, i)(); + } +#else + std::vector > task_results{}; + task_results.reserve(static_cast(std::ceil(max / step))); + + for (int i = 0; i < max; i += step) { + std::packaged_task task{std::bind(lambda, i)}; + task_results.emplace_back(task.get_future()); + ttp.RunTask(std::move(task)); + } + + try { + // wait for all and propagate any exceptions + for (auto& future : task_results) + future.get(); + } catch (const std::exception& ex) { + LOGS(logger, ERROR) << name << " - exception running tasks: " << ex.what(); + throw; + } +#endif +} + +void DumpMatrixImpl(const std::string& name, const float* src, int row, int col, + int offset = 0, int col_width = -1); + +// Helper class to wrap the processing of the activation funcs and any alpha/beta values. +// The alpha/beta values are consumed in the order of the activation funcs. once they run out +// defaults will be used as needed. +// The Entries property contains the normalized function names and the alpha/beta value to use. +class ActivationFuncs { + public: + struct Entry { + const std::string name; + const float alpha; + const float beta; + }; + + ActivationFuncs() = default; + + ActivationFuncs(const std::vector& funcs, + const std::vector& alphas, + const std::vector& betas); + + const std::vector& Entries() const { + return entries_; + } + + private: + std::vector entries_; +}; + +namespace deepcpu { + +using AddBiasIntoFuncPtr = void (*)(const float*, float*, const int); +using ClipWithBiasFuncPtr = void (*)(const float, const float*, float*, const int); +using ActivationFuncPtr = void (*)(float*, const int, const float, const float); +using ActivationFuncBPtr = void (*)(const float*, float*, const int, const float, const float); +using LstmMergeGatesFuncPtr = void (*)(const float*, float*, const float*, float*, const int, const float, const float); +using GruResetGateFuncPtr = void (*)(const float*, float*, float*, const int, const float, const float); +using GruOutputGateFuncPtr = void (*)(float*, const float*, const float*, float*, const int, const float, const float); + +ActivationFuncPtr ActivationFuncByName(const std::string& func); +LstmMergeGatesFuncPtr LstmMergeGatesFuncByName(const std::string& func); +GruResetGateFuncPtr GruResetGateFuncByName(const std::string& func); +GruOutputGateFuncPtr GruOutputGateFuncByName(const std::string& func); + +void add_bias_into_ignore(const float* ignored, float* pd, const int c); +void add_bias_into(const float* ps, float* pd, const int c); +void clip(const float b, float* pd, const int c); +void clip_add_bias(const float b, const float* pb, float* pd, const int c); +void clip_ignore_bias(const float b, const float* pb, float* pd, const int c); +void sigmoid_m(const float* ps1, float* ps1_c, const float* ps2, float* pd, int c, const float alpha, const float beta); +void tanh_m(const float* ps1, float* ps1_c, const float* ps2, float* pd, int c, const float alpha, const float beta); +void relu_m(const float* ps1, float* ps1_c, const float* ps2, float* pd, int c, const float alpha, const float beta); +void sigmoid_exact_m(const float* ps1, float* ps1_c, const float* ps2, float* pd, int c, const float alpha, const float beta); +void tanh_exact_m(const float* ps1, float* ps1_c, const float* ps2, float* pd, int c, const float alpha, const float beta); +void sigmoid(float* pd, int c, const float alpha, const float beta); +void tanh(float* pd, int c, const float alpha, const float beta); +void relu(float* pd, int c, const float alpha, const float beta); +void sigmoid_exact(float* pd, int c, const float alpha, const float beta); +void tanh_exact(float* pd, int c, const float alpha, const float beta); +void merge_lstm_gates_to_memory(const float* pprev, const float* pi, const float* pf, const float* pg, float* pcurr, const int c); +void gru_reset_gate_tanh(const float* ps1, float* ps2, float* pd, const int c, const float alpha, const float beta); +void gru_reset_gate_sigmoid(const float* ps1, float* ps2, float* pd, const int c, const float alpha, const float beta); +void gru_reset_gate_relu(const float* ps1, float* ps2, float* pd, const int c, const float alpha, const float beta); +void gru_output_gate_tanh(float* ph, const float* pz, const float* ps, float* po, const int c, const float alpha, const float beta); +void gru_output_gate_sigmoid(float* ph, const float* pz, const float* ps, float* po, const int c, const float alpha, const float beta); +void gru_output_gate_relu(float* ph, const float* pz, const float* ps, float* po, const int c, const float alpha, const float beta); + +inline void elementwise_product(const float* op1, const float* op2, float* dest, const int size) { + for (int i = 0; i < size; i++) + dest[i] += op1[i] * op2[i]; +} + +inline void elementwise_sum1(const float* src, float* dest, const int size) { + for (int i = 0; i < size; i++) + dest[i] += src[i]; +} + +inline void elementwise_sum2(const float* src1, const float* src2, float* dest, const int size) { + for (int i = 0; i < size; i++) + dest[i] += src1[i] + src2[i]; +} + +} // namespace deepcpu +} // namespace detail +} // namespace rnn +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/symbols.txt b/onnxruntime/core/providers/cpu/symbols.txt new file mode 100644 index 0000000000000..723523d7b24d0 --- /dev/null +++ b/onnxruntime/core/providers/cpu/symbols.txt @@ -0,0 +1,68 @@ +ONNXRuntimeAddCustomOp +ONNXRuntimeAddRefToObject +ONNXRuntimeAllocatorInfoGetId +ONNXRuntimeAllocatorInfoGetMemType +ONNXRuntimeAllocatorInfoGetName +ONNXRuntimeAllocatorInfoGetType +ONNXRuntimeCloneSessionOptions +ONNXRuntimeCompareAllocatorInfo +ONNXRuntimeCreateAllocatorInfo +ONNXRuntimeCreateCpuAllocatorInfo +ONNXRuntimeCreateCpuExecutionProviderFactory +ONNXRuntimeCreateDefaultAllocator +ONNXRuntimeCreateInferenceSession +ONNXRuntimeCreateRunOptions +ONNXRuntimeCreateSessionOptions +ONNXRuntimeCreateTensorAsONNXValue +ONNXRuntimeCreateTensorTypeAndShapeInfo +ONNXRuntimeCreateTensorWithDataAsONNXValue +ONNXRuntimeDisableCpuMemArena +ONNXRuntimeDisableMemPattern +ONNXRuntimeDisableProfiling +ONNXRuntimeDisableSequentialExecution +ONNXRuntimeEnableCpuMemArena +ONNXRuntimeEnableMemPattern +ONNXRuntimeEnableProfiling +ONNXRuntimeEnableSequentialExecution +ONNXRuntimeFillStringTensor +ONNXRuntimeGetDimensions +ONNXRuntimeGetErrorCode +ONNXRuntimeGetErrorMessage +ONNXRuntimeGetNumOfDimensions +ONNXRuntimeGetStringTensorContent +ONNXRuntimeGetStringTensorDataLength +ONNXRuntimeGetTensorElementType +ONNXRuntimeGetTensorMutableData +ONNXRuntimeGetTensorShapeAndType +ONNXRuntimeGetTensorShapeElementCount +ONNXRuntimeInferenceSessionGetInputCount +ONNXRuntimeInferenceSessionGetInputName +ONNXRuntimeInferenceSessionGetOutputCount +ONNXRuntimeInferenceSessionGetOutputName +ONNXRuntimeInitialize +ONNXRuntimeInitializeWithCustomLogger +ONNXRuntimeIsTensor +ONNXRuntimeONNXValueListGetNthValue +ONNXRuntimeReleaseObject +ONNXRuntimeRunInference +ONNXRuntimeRunInferenceAndFetchAll +ONNXRuntimeRunOptionsGetRunLogVerbosityLevel +ONNXRuntimeRunOptionsGetRunTag +ONNXRuntimeRunOptionsSetRunLogVerbosityLevel +ONNXRuntimeRunOptionsSetRunTag +ONNXRuntimeSessionOptionsAppendExecutionProvider +ONNXRuntimeSetDims +ONNXRuntimeSetSessionLogId +ONNXRuntimeSetSessionLogVerbosityLevel +ONNXRuntimeSetSessionThreadPoolSize +ONNXRuntimeSetTensorElementType +ONNXRuntimeTensorProtoToONNXValue +ReleaseONNXEnv +ReleaseONNXRuntimeAllocatorInfo +ReleaseONNXSession +ReleaseONNXStatus +ReleaseONNXValue +ReleaseONNXValueList +ONNXRuntimeAllocatorAlloc +ONNXRuntimeAllocatorFree +ONNXRuntimeAllocatorGetInfo \ No newline at end of file diff --git a/onnxruntime/core/providers/cpu/tensor/cast_op.cc b/onnxruntime/core/providers/cpu/tensor/cast_op.cc new file mode 100644 index 0000000000000..236b6c4b52413 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/cast_op.cc @@ -0,0 +1,173 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/tensor/cast_op.h" +#include +#include "core/common/common.h" + +using namespace ONNX_NAMESPACE; +namespace onnxruntime { + +const std::vector castOpTypeConstraints{ + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}; + +#define ADD_FROM_CAST_OP(in_type) \ + ONNX_CPU_OPERATOR_TYPED_KERNEL( \ + Cast, \ + 6, \ + in_type, \ + KernelDefBuilder().TypeConstraint("T1", DataTypeImpl::GetTensorType()).TypeConstraint("T2", castOpTypeConstraints), \ + Cast); \ + \ + template <> \ + Status Cast::Compute(OpKernelContext* context) const { \ + const Tensor* X = context->Input(0); \ + if (X == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); \ + const TensorShape& shape = X->Shape(); \ + Tensor* Y = context->Output(0, TensorShape(shape)); \ + \ + switch (to_) { \ + case TensorProto_DataType_BOOL: \ + CastData(X, Y, shape); \ + break; \ + case TensorProto_DataType_INT16: \ + CastData(X, Y, shape); \ + break; \ + case TensorProto_DataType_INT32: \ + CastData(X, Y, shape); \ + break; \ + case TensorProto_DataType_INT64: \ + CastData(X, Y, shape); \ + break; \ + case TensorProto_DataType_UINT8: \ + CastData(X, Y, shape); \ + break; \ + case TensorProto_DataType_UINT16: \ + CastData(X, Y, shape); \ + break; \ + case TensorProto_DataType_UINT32: \ + CastData(X, Y, shape); \ + break; \ + case TensorProto_DataType_UINT64: \ + CastData(X, Y, shape); \ + break; \ + case TensorProto_DataType_FLOAT: \ + CastData(X, Y, shape); \ + break; \ + case TensorProto_DataType_DOUBLE: \ + CastData(X, Y, shape); \ + break; \ + case TensorProto_DataType_INT8: \ + CastData(X, Y, shape); \ + break; \ + case TensorProto_DataType_FLOAT16: \ + if (std::is_same::value) { \ + CastData(X, Y, shape); \ + } else { \ + auto st = CastFloat16Data(X, Y, shape, context); \ + if (!st.IsOK()) return st; \ + } \ + break; \ + case TensorProto_DataType_STRING: \ + ONNXRUNTIME_THROW("Casting to and from strings is not supported yet."); /*break;*/ \ + case TensorProto_DataType_UNDEFINED: \ + ONNXRUNTIME_THROW("Cast op must have 'to' argument of type DataType"); /*break;*/ \ + default: \ + ONNXRUNTIME_THROW("Unexpected 'to' argument value: ", to_); \ + } \ + return Status::OK(); \ + } + +ADD_FROM_CAST_OP(uint8_t); +ADD_FROM_CAST_OP(uint16_t); +ADD_FROM_CAST_OP(uint32_t); +ADD_FROM_CAST_OP(uint64_t); +ADD_FROM_CAST_OP(int8_t); +ADD_FROM_CAST_OP(int16_t); +ADD_FROM_CAST_OP(int32_t); +ADD_FROM_CAST_OP(int64_t); +ADD_FROM_CAST_OP(bool); +ADD_FROM_CAST_OP(float); +ADD_FROM_CAST_OP(double); + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Cast, + 6, + MLFloat16, + KernelDefBuilder().TypeConstraint("T1", DataTypeImpl::GetTensorType()).TypeConstraint("T2", castOpTypeConstraints), + Cast); + +template <> +Status Cast::Compute(OpKernelContext* context) const { + const Tensor* X = context->Input(0); + if (X == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + const TensorShape& shape = X->Shape(); + Tensor* Y = context->Output(0, TensorShape(shape)); + Status st; + switch (to_) { + case TensorProto_DataType_BOOL: + st = CastFloat16Data(X, Y, shape, context); + break; + case TensorProto_DataType_INT16: + st = CastFloat16Data(X, Y, shape, context); + break; + case TensorProto_DataType_INT32: + st = CastFloat16Data(X, Y, shape, context); + break; + case TensorProto_DataType_INT64: + st = CastFloat16Data(X, Y, shape, context); + break; + case TensorProto_DataType_UINT8: + st = CastFloat16Data(X, Y, shape, context); + break; + case TensorProto_DataType_UINT16: + st = CastFloat16Data(X, Y, shape, context); + break; + case TensorProto_DataType_UINT32: + st = CastFloat16Data(X, Y, shape, context); + break; + case TensorProto_DataType_UINT64: + st = CastFloat16Data(X, Y, shape, context); + break; + case TensorProto_DataType_FLOAT: + CastData(X, Y, shape); + break; + case TensorProto_DataType_FLOAT16: { + auto X_type = X->DataType(); + const void* source = X->DataRaw(X_type); + void* target = Y->MutableDataRaw(X_type); + // if source and target pointers are not equal, we need to copy the data. + if (target != source) { + memcpy(target, source, shape.Size() * X_type->Size()); + } + st = Status::OK(); + break; + } + case TensorProto_DataType_DOUBLE: + st = CastFloat16Data(X, Y, shape, context); + break; + case TensorProto_DataType_INT8: + st = CastFloat16Data(X, Y, shape, context); + break; + case TensorProto_DataType_STRING: + ONNXRUNTIME_THROW("Casting to and from strings is not supported yet."); /*break;*/ + case TensorProto_DataType_UNDEFINED: + ONNXRUNTIME_THROW("Cast op must have 'to' argument of type DataType"); /*break;*/ + default: + ONNXRUNTIME_THROW("Unexpected 'to' argument value: ", to_); + } + return st; +} + +} //namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/cast_op.h b/onnxruntime/core/providers/cpu/tensor/cast_op.h new file mode 100644 index 0000000000000..83e2b560a981f --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/cast_op.h @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/util/math_cpuonly.h" +#include "Eigen/src/Core/arch/CUDA/Half.h" +#include "core/util/math_cpuonly.h" + +#if defined(USE_MLAS) && defined(_M_AMD64) +#include "core/mlas/inc/mlas.h" +#endif + +namespace onnxruntime { + +template +inline void CastData(const Tensor* in, Tensor* out, const TensorShape& shape) { + auto shape_size = shape.Size(); + auto in_vector = ConstEigenVectorMap(in->template Data(), shape_size); + auto output_vector = EigenVectorMap(out->template MutableData(), shape_size); + output_vector = in_vector.template cast(); +} + +template <> +inline void CastData(const Tensor* in, Tensor* out, const TensorShape& shape) { + auto out_data = out->template MutableData(); + auto shape_size = shape.Size(); + auto in_vector = ConstEigenVectorMap(in->template Data(), shape_size); + auto output_vector = EigenVectorMap(static_cast(static_cast(out_data)), shape_size); + output_vector = in_vector.template cast(); +} + +template <> +inline void CastData(const Tensor* in, Tensor* out, const TensorShape& shape) { + auto out_data = out->template MutableData(); + auto in_data = in->template Data(); + auto shape_size = shape.Size(); +#if defined(USE_MLAS) && defined(_M_AMD64) + MlasConvertHalfToFloatBuffer(&in_data[0].val, out_data, shape_size); +#else + auto in_vector = ConstEigenVectorMap(static_cast(static_cast(in_data)), shape_size); + auto output_vector = EigenVectorMap(out_data, shape_size); + output_vector = in_vector.template cast(); +#endif +} + +template +inline void CastFloat16Data(const Tensor* in, Tensor* out, const TensorShape& shape, const AllocatorPtr& allocator) { + ONNXRUNTIME_ENFORCE(allocator != nullptr); + const int64_t len = shape.Size(); + ONNXRUNTIME_ENFORCE(len > 0); + void* buffer = allocator->AllocArray(sizeof(float), len); + ONNXRUNTIME_ENFORCE(buffer); + Tensor tmp_tensor(DataTypeImpl::GetType(), shape, buffer, allocator->Info(), nullptr); + if (std::is_same::value) { + CastData(in, &tmp_tensor, shape); // first cast to float + CastData(&tmp_tensor, out, shape); // then cast to the destination type. + } else if (std::is_same::value) { + CastData(in, &tmp_tensor, shape); + CastData(&tmp_tensor, out, shape); + } + allocator->Free(buffer); +} + +template +class Cast final : public OpKernel { + public: + Cast(const OpKernelInfo& info) : OpKernel(info) { + int64_t to; + Status status = info.GetAttr("to", &to); + ONNXRUNTIME_ENFORCE(status.IsOK(), "Attribute to is not set."); + to_ = gsl::narrow_cast(to); + } + + Status Compute(OpKernelContext* context) const override; + + private: + template + void CastData(const Tensor* in, Tensor* out, const TensorShape& shape) const { + ::onnxruntime::CastData(in, out, shape); + } + + template + Status CastFloat16Data(const Tensor* in, Tensor* out, const TensorShape& shape, OpKernelContext* context) const { + AllocatorPtr allocator; + ONNXRUNTIME_RETURN_IF_ERROR(context->GetTempSpaceAllocator(&allocator)); + ::onnxruntime::CastFloat16Data(in, out, shape, allocator); + return Status::OK(); + } + + ONNX_NAMESPACE::TensorProto_DataType to_; +}; + +} //namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/compress.cc b/onnxruntime/core/providers/cpu/tensor/compress.cc new file mode 100644 index 0000000000000..1598449e158cb --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/compress.cc @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/tensor/compress.h" +using namespace ::onnxruntime::common; + +namespace onnxruntime { + +ONNX_CPU_OPERATOR_KERNEL( + Compress, + 9, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::AllTensorTypes()), + Compress); + +Status Compress::Compute(OpKernelContext* ctx) const { + const Tensor* input_tensor = ctx->Input(0); + size_t rank = input_tensor->Shape().NumDimensions(); + auto& input_dimensions = input_tensor->Shape().GetDims(); + if (has_axis_) { + ONNXRUNTIME_ENFORCE(axis_ < static_cast(rank), "axis greater than input data dimension!"); + } + + const Tensor* condition = ctx->Input(1); + auto condition_length = condition->Shape().Size(); + auto condition_data = condition->template Data(); + + int64_t positive_condition_count = 0; + // if has axis, we need to compress on dimension[axis], otherwise compress on the flattened input data + int64_t compress_input_length = has_axis_ ? input_dimensions[axis_] : input_tensor->Shape().Size(); + int64_t valid_condition_length = compress_input_length < condition_length ? compress_input_length : condition_length; + + // Figure out output shape + for (int i = 0; i < valid_condition_length; ++i) { + if (condition_data[i]) { + ++positive_condition_count; + } + } + + std::vector output_dims(input_dimensions); + if (has_axis_) { + output_dims[axis_] = positive_condition_count; + } else { + output_dims.resize(1); + output_dims[0] = positive_condition_count; + } + + TensorShape output_shape(output_dims); + auto output_tensor = ctx->Output(0, output_shape); + if (positive_condition_count <= 0) { + return Status::OK(); + } + + const uint8_t* input_data = static_cast(input_tensor->DataRaw()); + uint8_t* output_data = static_cast(output_tensor->MutableDataRaw()); + auto element_bytes = input_tensor->DataType()->Size(); + bool is_string_type = input_tensor->DataType() == DataTypeImpl::GetType(); + int64_t output_index = 0; + + if (has_axis_) { + int64_t axes_left_stride = 1; + int64_t axes_right_stride = 1; + for (int i = 0; i < axis_; ++i) { + axes_left_stride *= input_dimensions[i]; + } + + for (int i = static_cast(axis_ + 1); i < rank; ++i) { + axes_right_stride *= input_dimensions[i]; + } + int64_t axes_included_right_stride = axes_right_stride * input_dimensions[axis_]; + int64_t axes_included_right_stride_bytes = axes_included_right_stride * element_bytes; + int64_t axes_right_stride_bytes = axes_right_stride * element_bytes; + + for (int i = 0; i < axes_left_stride; ++i) { + for (int j = 0; j < valid_condition_length; ++j) { + if (!condition_data[j]) { + continue; + } + if (is_string_type) { + for (int idxItem = 0; idxItem < axes_right_stride; ++idxItem) { + reinterpret_cast(output_data)[output_index + idxItem] = + reinterpret_cast(input_data)[i * axes_included_right_stride + j * axes_right_stride + idxItem]; + } + output_index += axes_right_stride; + } else { + memcpy(output_data + output_index, input_data + i * axes_included_right_stride_bytes + j * axes_right_stride_bytes, axes_right_stride_bytes); + output_index += axes_right_stride_bytes; + } + } + } + } else { + for (int i = 0; i < valid_condition_length; ++i) { + if (!condition_data[i]) { + continue; + } + if (is_string_type) { + reinterpret_cast(output_data)[output_index] = reinterpret_cast(input_data)[i]; + } else { + memcpy(output_data + output_index * element_bytes, input_data + i * element_bytes, element_bytes); + } + ++output_index; + } + } + +return Status::OK(); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/compress.h b/onnxruntime/core/providers/cpu/tensor/compress.h new file mode 100644 index 0000000000000..ef96babee047e --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/compress.h @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { + +class Compress final : public OpKernel { + public: + Compress(const OpKernelInfo& info) : OpKernel(info) { + has_axis_ = info.GetAttr("axis", &axis_).IsOK(); + } + + Status Compute(OpKernelContext* context) const override; + + private: + int64_t axis_; + bool has_axis_; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/concat.cc b/onnxruntime/core/providers/cpu/tensor/concat.cc new file mode 100644 index 0000000000000..d07b3ed732562 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/concat.cc @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/tensor/concat.h" + +namespace onnxruntime { + +ONNX_CPU_OPERATOR_KERNEL( + Concat, + 4, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::AllTensorTypes()), + Concat); + +Status ConcatBase::PrepareForCompute(OpKernelContext* ctx, int input_count, Prepare& p) const { + ONNXRUNTIME_RETURN_IF_NOT(input_count >= 1, "Must have 1 or more inputs"); + const Tensor* tensor_pointer = ctx->Input(0); + if (tensor_pointer == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + const Tensor& inputs_0 = *tensor_pointer; + + // Ensure all of the non concatenated axes match each other + for (int index = 1; index < input_count; index++) { + tensor_pointer = ctx->Input(index); + if (tensor_pointer == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + auto& data_n = *tensor_pointer; + // Ensure all the other axes match + auto dimension_count = inputs_0.Shape().NumDimensions(); + for (int axis_index = 0; axis_index < dimension_count; axis_index++) { + if (axis_index == axis_) + continue; + ONNXRUNTIME_RETURN_IF_NOT(data_n.Shape()[axis_index] == inputs_0.Shape()[axis_index], "Non concat axis dimensions must match: Axis ", axis_index, " has mismatched dimensions of ", data_n.Shape()[axis_index], " and ", inputs_0.Shape()[axis_index]); + } + } + + // Calculate the size of the concatenated axis, and verify all other dimensions match + size_t concat_axis_size = 0; + for (int index = 0; index < input_count; index++) { + tensor_pointer = ctx->Input(index); + if (tensor_pointer == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + concat_axis_size += tensor_pointer->Shape()[int(axis_)]; + } + + // Calculate the shape of the output tensor + std::vector dims; + for (int dimension_index = 0; dimension_index < inputs_0.Shape().NumDimensions(); dimension_index++) + dims.emplace_back(inputs_0.Shape()[dimension_index]); + dims[axis_] = concat_axis_size; + TensorShape outputShape(dims); + + // The output_axis_pitch is the number of elements to add to move to the next split axis in the output + p.output_axis_pitch = 1; + for (auto i = int64_t(dims.size()); i-- > axis_;) + p.output_axis_pitch *= dims[i]; + + auto& concat_result = *ctx->Output(0, outputShape); + p.output_tensor = &concat_result; + + for (int input_index = 0; input_index < input_count; input_index++) { + const Tensor* data_n_ptr = ctx->Input(input_index); + ONNXRUNTIME_ENFORCE(data_n_ptr != nullptr); + auto& data_n = *data_n_ptr; + + ONNXRUNTIME_RETURN_IF_NOT(data_n.DataType() == concat_result.DataType()); + + // The input_axis_pitch is the number of elements to add to move to the next split axis in the input + int64_t input_axis_pitch = 1; + for (int i = int(data_n.Shape().NumDimensions()); i-- > axis_;) + input_axis_pitch *= data_n.Shape()[i]; + + p.inputs.push_back({&data_n, input_axis_pitch}); + } + + return Status::OK(); +} + +Status Concat::Compute(OpKernelContext* ctx) const { + auto input_count = Node().InputArgCount().front(); + + Prepare p; + ONNXRUNTIME_RETURN_IF_ERROR(PrepareForCompute(ctx, input_count, p)); + + auto is_string_type = ctx->Input(0)->DataType() == DataTypeImpl::GetType(); + + int64_t output_offset = 0; + auto element_bytes = p.output_tensor->DataType()->Size(); + for (int input_index = 0; input_index < input_count; input_index++) { + const auto& prep = p.inputs[input_index]; + auto input_axis_pitch = prep.axis_pitch; + const uint8_t* input = static_cast(prep.tensor->DataRaw()); + auto input_size = prep.tensor->Shape().Size(); + + // Copy the data across. For every 'input_axis_pitch' values copied, we move over by the 'output_axis_pitch' + uint8_t* output = static_cast(p.output_tensor->MutableDataRaw()); + for (int idxCopy = 0; idxCopy < input_size / input_axis_pitch; ++idxCopy) { + if (is_string_type) { + for (int idxItem = 0; idxItem < input_axis_pitch; ++idxItem) + reinterpret_cast(output)[output_offset + idxCopy * p.output_axis_pitch + idxItem] = + reinterpret_cast(input)[idxCopy * input_axis_pitch + idxItem]; + } else + memcpy( + output + (output_offset + idxCopy * p.output_axis_pitch) * element_bytes, + input + idxCopy * input_axis_pitch * element_bytes, + input_axis_pitch * element_bytes); + } + output_offset += input_axis_pitch; + } + return Status::OK(); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/concat.h b/onnxruntime/core/providers/cpu/tensor/concat.h new file mode 100644 index 0000000000000..23f1f13cd4003 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/concat.h @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/util/math_cpuonly.h" +#include "core/framework/tensor.h" + +namespace onnxruntime { + +class ConcatBase { + protected: + ConcatBase(const OpKernelInfo& info) { + if (!info.GetAttr("axis", &axis_).IsOK()) { + ONNXRUNTIME_ENFORCE(false, "Must have valid 'axis' attribute"); + } + } + + struct Prepare { + struct InputInfo { + const Tensor* tensor; + int64_t axis_pitch; + }; + std::vector inputs; + int64_t output_axis_pitch; + Tensor* output_tensor; + }; + + Status PrepareForCompute(OpKernelContext* ctx, int input_count, Prepare& p) const; + + private: + int64_t axis_; +}; + +class Concat final : public OpKernel, public ConcatBase { + public: + Concat(const OpKernelInfo& info) : OpKernel(info), ConcatBase(info) {} + + Status Compute(OpKernelContext* context) const override; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/crop.cc b/onnxruntime/core/providers/cpu/tensor/crop.cc new file mode 100644 index 0000000000000..53698123eb8bd --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/crop.cc @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/tensor/crop.h" + +namespace onnxruntime { +ONNX_CPU_OPERATOR_KERNEL( + Crop, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Crop); +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/crop.h b/onnxruntime/core/providers/cpu/tensor/crop.h new file mode 100644 index 0000000000000..e406d44e206f7 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/crop.h @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +#include "gsl/gsl_util" + +namespace onnxruntime { + +class CropBase { + protected: + CropBase(const OpKernelInfo& info) + : border_(info.GetAttrsOrDefault("border")), + scale_(info.GetAttrsOrDefault("scale")) { + } + + Status ValidateInput(const Tensor* X) const { + if (border_.size() < 4) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Attribute border needs to be specified with four border elements, got ", border_.size()); + } + + const auto dims = X->Shape().GetDims(); + + if (dims.size() < 4) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Input is expected to have four dimensions corresponding to [N,C,H,W], got ", dims.size()); + } + + const int64_t H = dims[2]; + const int64_t W = dims[3]; + + // find the cropped region, and copy it to the destination matrix + int64_t leftBorder = border_[0], + topBorder = border_[1], + rightBorder = border_[2], + bottomBorder = border_[3]; + + if (H < topBorder + bottomBorder) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input's height (", H, ") needs to be greater than the topBorder (", topBorder, ") + bottomBorder (", bottomBorder, ")"); + } + + if (W < leftBorder + rightBorder) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input's width (", W, ") needs to be greater than the leftBorder (", leftBorder, ") + rightBorder (", rightBorder, ")"); + } + + int64_t bottomLimit = H - bottomBorder; + int64_t rightLimit = W - rightBorder; + + // scale = (height, width) + if (!scale_.empty()) { + bottomLimit = topBorder + scale_[0]; + rightLimit = leftBorder + scale_[1]; + + if (H < bottomLimit) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Input's height (", H, ") needs to be greater than the topBorder (", topBorder, ") + scale_[0] (", scale_[0], ")"); + } + + if (W < rightLimit) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input's width (", W, ") needs to be greater than the leftBorder (", leftBorder, ") + scale_[1] (", scale_[1], ")"); + } + } + + return Status::OK(); + } + + const std::vector border_; // (leftBorder, topBorder, rightBorder, bottomBorder) + const std::vector scale_; // (height, width) +}; + +template +class Crop final : public CropBase, public OpKernel { + public: + Crop(const OpKernelInfo& info) : CropBase(info), OpKernel(info) { + } + + common::Status Compute(OpKernelContext* context) const override { + const Tensor* X = context->Input(0); + ONNXRUNTIME_RETURN_IF_ERROR(ValidateInput(X)); + + const auto dims = X->Shape().GetDims(); + const int64_t N = dims[0]; + const int64_t C = dims[1]; + const int64_t H = dims[2]; + const int64_t W = dims[3]; + + // find the cropped region, and copy it to the destination matrix + int64_t leftBorder = border_[0], + topBorder = border_[1], + rightBorder = border_[2], + bottomBorder = border_[3]; + + int64_t bottomLimit = H - bottomBorder; + int64_t rightLimit = W - rightBorder; + + // scale = (height, width) + if (!scale_.empty()) { + bottomLimit = topBorder + scale_[0]; + rightLimit = leftBorder + scale_[1]; + } + + Tensor* Y = context->Output(0, TensorShape({N, C, bottomLimit - topBorder, rightLimit - leftBorder})); + const T* Xdata = X->template Data(); + T* Ydata = Y->template MutableData(); + + int64_t dest_idx = 0; + int64_t HW = H * W; + int64_t CHW = C * HW; + int64_t nCHW; + int64_t nCHW_p_cHW; + int64_t nCHW_p_cHW_p_hW; + int64_t source_idx; + for (int64_t n = 0; n < N; ++n) { + nCHW = n * CHW; + for (int64_t c = 0; c < C; ++c) { + nCHW_p_cHW = nCHW + c * HW; + for (int64_t h = topBorder; h < bottomLimit; ++h) { + nCHW_p_cHW_p_hW = nCHW_p_cHW + h * W; + for (int64_t w = leftBorder; w < rightLimit; ++w) { + source_idx = nCHW_p_cHW_p_hW + w; + Ydata[dest_idx++] = Xdata[source_idx]; + } + } + } + } + return Status::OK(); + } +}; + +} //namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/gather.cc b/onnxruntime/core/providers/cpu/tensor/gather.cc new file mode 100644 index 0000000000000..5a8295c7420a1 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/gather.cc @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//https://github.com/onnx/onnx/blob/master/docs/Operators.md#Gather +#include "core/providers/cpu/tensor/gather.h" +#include "core/common/common.h" + +namespace onnxruntime { + +ONNX_CPU_OPERATOR_KERNEL( + Gather, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::AllTensorTypes()).TypeConstraint("Tind", std::vector{DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}), + Gather); + +Status GatherBase::PrepareForCompute(OpKernelContext* context, Prepare& p) const { + p.input_tensor = context->Input(0); + const TensorShape& input_data_shape = p.input_tensor->Shape(); + p.indices_tensor = context->Input(1); + const TensorShape& indices_shape = p.indices_tensor->Shape(); + + p.axis = HandleNegativeAxis(axis_, input_data_shape.NumDimensions()); + + std::vector shape(indices_shape.GetDims().begin(), indices_shape.GetDims().end()); + shape.insert(shape.begin(), input_data_shape.GetDims().begin(), input_data_shape.GetDims().begin() + p.axis); + shape.insert(shape.end(), input_data_shape.GetDims().begin() + p.axis + 1, input_data_shape.GetDims().end()); + + p.output_tensor = context->Output(0, TensorShape(shape)); + + return Status::OK(); +} + +template +Status GatherCopyData(const Tensor* indices_tensor, const uint8_t* src_base, uint8_t* dst_base, bool is_string_type, + const size_t element_bytes, const int64_t block_size, const int64_t M, + const int64_t N, const int64_t data_batch_bytes, const int64_t gathered_batch_bytes, + const TensorShape& input_data_shape, const int64_t axis) { + const Tin* indices_data = indices_tensor->template Data(); + + // Check the indices first in case there's a out of bound index. + // We can't merge this code in the omp loop below as omp does not allow return in the loop + for (int64_t i = 0; i < N; ++i) { + Tin idx = indices_data[i]; + if (idx < 0 || idx >= input_data_shape[axis]) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "indices element out of data bounds, idx=", idx, + " data_dim=", input_data_shape[axis]); + } + } + +#pragma omp parallel for + for (int64_t batch = 0; batch < M; ++batch) { + const int64_t src_offset_batch = batch * data_batch_bytes; + const int64_t dst_offset_batch = batch * gathered_batch_bytes; + for (int64_t i = 0; i < N; ++i) { + Tin idx = indices_data[i]; + const int64_t src_offset = src_offset_batch + idx * block_size; + const int64_t dst_offset = dst_offset_batch + i * block_size; + + if (is_string_type) { + reinterpret_cast(dst_base)[dst_offset / element_bytes] = + reinterpret_cast(src_base)[src_offset / element_bytes]; + } else { + memcpy(dst_base + dst_offset, src_base + src_offset, block_size); + } + } + } + + return Status::OK(); +} + +Status Gather::Compute(OpKernelContext* context) const { + Prepare p; + ONNXRUNTIME_RETURN_IF_ERROR(PrepareForCompute(context, p)); + + const TensorShape& input_data_shape = p.input_tensor->Shape(); + + bool is_string_type = p.input_tensor->DataType() == DataTypeImpl::GetType(); + + const size_t element_bytes = p.input_tensor->DataType()->Size(); + const int64_t block = input_data_shape.SizeFromDimension(p.axis + 1); + const int64_t block_size = block * element_bytes; + const int64_t M = input_data_shape.SizeToDimension(p.axis); + const int64_t N = p.indices_tensor->Shape().Size(); + const int64_t data_batch_bytes = input_data_shape.SizeFromDimension(p.axis) * element_bytes; + const int64_t gathered_batch_bytes = N * block * element_bytes; + + const uint8_t* src_base = static_cast(p.input_tensor->DataRaw()); + uint8_t* dst_base = static_cast(p.output_tensor->MutableDataRaw()); + + MLDataType Tind_type = p.indices_tensor->DataType(); + if (Tind_type == DataTypeImpl::GetType()) { + return GatherCopyData(p.indices_tensor, src_base, dst_base, is_string_type, element_bytes, + block_size, M, N, data_batch_bytes, gathered_batch_bytes, input_data_shape, p.axis); + } else if (Tind_type == DataTypeImpl::GetType()) { + return GatherCopyData(p.indices_tensor, src_base, dst_base, is_string_type, element_bytes, + block_size, M, N, data_batch_bytes, gathered_batch_bytes, input_data_shape, p.axis); + } + + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, NOT_IMPLEMENTED, "Type for Tind not supported yet in Gather."); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/gather.h b/onnxruntime/core/providers/cpu/tensor/gather.h new file mode 100644 index 0000000000000..e6df9dd68ffef --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/gather.h @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/providers/common.h" + +namespace onnxruntime { + +class GatherBase { + protected: + GatherBase(const OpKernelInfo& info) { + ONNXRUNTIME_ENFORCE(info.GetAttr("axis", &axis_).IsOK(), "Missing/Invalid 'axis' attribute value"); + } + + struct Prepare { + const Tensor* input_tensor; + const Tensor* indices_tensor; + Tensor* output_tensor; + int64_t axis; + }; + + Status PrepareForCompute(OpKernelContext* context, Prepare& p) const; + + private: + int64_t axis_; +}; + +class Gather final : public OpKernel, public GatherBase { + public: + Gather(const OpKernelInfo& info) : OpKernel(info), GatherBase(info) {} + + Status Compute(OpKernelContext* context) const override; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/identity_op.cc b/onnxruntime/core/providers/cpu/tensor/identity_op.cc new file mode 100644 index 0000000000000..e66e28821d41b --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/identity_op.cc @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/tensor/identity_op.h" + +namespace onnxruntime { + +ONNX_CPU_OPERATOR_KERNEL( + Dropout, + 7, + KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}), + IdentityOp); + +ONNX_CPU_OPERATOR_KERNEL( + Identity, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::AllTensorTypes()).Alias(0, 0), + IdentityOp); + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/identity_op.h b/onnxruntime/core/providers/cpu/tensor/identity_op.h new file mode 100644 index 0000000000000..d7b54c4ba7267 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/identity_op.h @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4996) +#endif +#include "core/common/common.h" +#ifdef _MSC_VER +#pragma warning(pop) +#endif +#include "core/framework/op_kernel.h" + +namespace onnxruntime { + +template +class IdentityOp final : public OpKernel { + public: + IdentityOp(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override { + const Tensor* X = context->Input(0); + ONNXRUNTIME_ENFORCE(X != nullptr); + const TensorShape& shape = X->Shape(); + Tensor* Y = context->Output(0, shape); + auto X_type = X->DataType(); + + const void* source = X->DataRaw(X_type); + void* target = Y->MutableDataRaw(X_type); + //If source and target pointers are not equal, we need to copy the data. + if (target != source) { + if (X_type != DataTypeImpl::GetType()) { + memcpy(target, source, shape.Size() * X_type->Size()); + } else { + // handle std::string + const std::string* src = X->template Data(); + std::string* dst = Y->template MutableData(); + std::copy(src, src + shape.Size(), dst); + } + } + + if (is_dropout) { + context->Output(1, std::vector()); + } + + return Status::OK(); + } +}; + +} //namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/image_scaler.cc b/onnxruntime/core/providers/cpu/tensor/image_scaler.cc new file mode 100644 index 0000000000000..98c7240b59b64 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/image_scaler.cc @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/tensor/image_scaler.h" + +namespace onnxruntime { +ONNX_CPU_OPERATOR_KERNEL( + ImageScaler, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + ImageScaler); +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/image_scaler.h b/onnxruntime/core/providers/cpu/tensor/image_scaler.h new file mode 100644 index 0000000000000..d16f9a0cbc02a --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/image_scaler.h @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/util/math_cpuonly.h" + +namespace onnxruntime { + +template +class ImageScaler final : public OpKernel { + public: + ImageScaler(const OpKernelInfo& info) : OpKernel(info) { + ONNXRUNTIME_ENFORCE(info.GetAttr("scale", &scale_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttrs("bias", bias_).IsOK()); + } + + Status Compute(OpKernelContext* context) const override { + const Tensor* X = context->Input(0); + if (X == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + + const auto dims = X->Shape().GetDims(); + + if (dims.size() < 4) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Input is expected to have four dimensions corresponding to [N,C,H,W], got ", dims.size()); + } + + const int64_t N = dims[0]; + const int64_t C = dims[1]; + const int64_t H = dims[2]; + const int64_t W = dims[3]; + + if (!bias_.empty() && bias_.size() != static_cast(C)) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Bias size (", bias_.size(), ") does not match the number of channels (", C, ")"); + } + + Tensor* Y = context->Output(0, TensorShape({N, C, H, W})); + ConstEigenArrayMap X_arr(X->template Data(), H * W, N * C); + EigenArrayMap Y_arr(Y->template MutableData(), H * W, N * C); + + for (int64_t nc = 0; nc < N * C; ++nc) { + Y_arr.col(nc) = scale_ * X_arr.col(nc) + bias_[nc % C]; + } + return Status::OK(); + } + + protected: + float scale_; + std::vector bias_; +}; + +} //namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/mean_variance_normalization.cc b/onnxruntime/core/providers/cpu/tensor/mean_variance_normalization.cc new file mode 100644 index 0000000000000..67c6feab46312 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/mean_variance_normalization.cc @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/tensor/mean_variance_normalization.h" + +namespace onnxruntime { +ONNX_CPU_OPERATOR_VERSIONED_KERNEL( + MeanVarianceNormalization, + 1, + 8, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + MeanVarianceNormalization_0); + +ONNX_CPU_OPERATOR_KERNEL( + MeanVarianceNormalization, + 9, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + MeanVarianceNormalization_1); +} diff --git a/onnxruntime/core/providers/cpu/tensor/mean_variance_normalization.h b/onnxruntime/core/providers/cpu/tensor/mean_variance_normalization.h new file mode 100644 index 0000000000000..b02b2e30c9671 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/mean_variance_normalization.h @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/util/math_cpuonly.h" + +#include "gsl/gsl_util" +namespace onnxruntime { +template +class MeanVarianceNormalization_0 : public OpKernel { + public: + MeanVarianceNormalization_0(const OpKernelInfo& info, bool old_attr = true) : OpKernel(info) { + if (old_attr) { + ONNXRUNTIME_ENFORCE(info.GetAttr("across_channels", &across_channels_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttr("normalize_variance", &normalize_variance_).IsOK()); + } + } + + Status Compute(OpKernelContext* context) const override { + const Tensor* X = context->Input(0); + if (X == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + + const auto dims = X->Shape().GetDims(); + + if (dims.size() < 4) { + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, + "Input is expected to have four dimensions corresponding to [N,C,H,W]"); + } + + const int64_t N = dims[0]; + const int64_t C = dims[1]; + const int64_t H = dims[2]; + const int64_t W = dims[3]; + + Tensor* Y = context->Output(0, TensorShape({N, C, H, W})); + const T* Xdata = X->template Data(); + T* Ydata = Y->template MutableData(); + + const int64_t sample_size = H * W; + Eigen::Array mean(C, 1); + Eigen::Array var(C, 1); + mean.setZero(); + var.setZero(); + + ConstEigenArrayMap X_arr(Xdata, sample_size, N * C); + for (int nc = 0; nc < N * C; ++nc) { + mean(nc % C) += X_arr.col(nc).sum(); + } + mean /= gsl::narrow_cast(N * sample_size); + for (int64_t nc = 0; nc < N * C; ++nc) { + var(nc % C) += (X_arr.col(nc) - mean(nc % C)).matrix().squaredNorm(); + } + var /= gsl::narrow_cast(N * sample_size); + + Eigen::Array inv_std; + EigenArrayMap Y_arr(Ydata, sample_size, N * C); + + if (across_channels_) { + // m_c = sum(m_i) / n + float global_mean = mean.mean(); + + // var_c = [(var_1 + (m_1 - m_c)^2) + ... + (var_n + (m_n - m_c)^2)] / n + // = [sum(var_i) + squared_norm(m_i - m_c)] / n + float global_var = ((mean - global_mean).matrix().squaredNorm() + var.sum()) / C; + + // For across channels we can directly use eigen because global_mean and global_var + // are just scalars. + if (!normalize_variance_) { + Y_arr = X_arr - global_mean; + } else { + float inv_std_scalar = 1 / std::sqrt(global_var); + Y_arr = (X_arr - global_mean) * inv_std_scalar; + } + } else { + if (!normalize_variance_) { + // inv_std = 1 + for (int64_t nc = 0; nc < N * C; ++nc) { + // y = (x - mean) + Y_arr.col(nc) = (X_arr.col(nc) - mean(nc % C)); + } + } else { + inv_std = var.sqrt().inverse(); + for (int64_t nc = 0; nc < N * C; ++nc) { + // y = (x - mean) * (inv_std) + Y_arr.col(nc) = (X_arr.col(nc) - mean(nc % C)) * inv_std(nc % C); + } + } + } + return Status::OK(); + } + + protected: + int64_t across_channels_; + int64_t normalize_variance_; +}; + +template +class MeanVarianceNormalization_1 final : public MeanVarianceNormalization_0 { + public: + MeanVarianceNormalization_1(const OpKernelInfo& info) : MeanVarianceNormalization_0(info, false) { + std::vector axes; + if (!info.GetAttrs("axes", axes).IsOK()) { + axes = {0, 2, 3}; + } + if (find(axes.begin(), axes.end(), 1) != axes.end()) { + this->across_channels_ = true; + } else { + this->across_channels_ = false; + } + this->normalize_variance_ = 1; + } +}; + +} //namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/pad.cc b/onnxruntime/core/providers/cpu/tensor/pad.cc new file mode 100644 index 0000000000000..5d8b19c5016c8 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/pad.cc @@ -0,0 +1,180 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// there's no way to use a raw pointer as the copy destination with std::copy_n +// (which gsl::copy uses with span::data() which returns a raw pointer) with the 14.11 toolset +// without generating a 4996 warning. going through an iterator is way too much overhead so turn off the warning. +#ifdef _MSC_VER +#pragma warning(disable : 4996) +#endif +#include "core/providers/cpu/tensor/pad.h" +#include "core/providers/cpu/tensor/utils.h" + +namespace onnxruntime { + +ONNX_CPU_OPERATOR_KERNEL( + Pad, + 2, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Pad); + +// This is the general padding method to n-dimensionally do edge or reflection padding (based on the inputDelta values) +template +static void PadAxis(T* output, T* input, ptrdiff_t input_delta, ptrdiff_t input_pitch, size_t block_size, size_t block_count) { + for (size_t block_index = 0; block_index < block_count; block_index++) { + for (size_t i = 0; i < block_size; i++) { + *output++ = *input; + input += input_delta; + } + input += input_pitch; + } +} + +// These are optimizations of PadAxis. The inner loop is removed since the innermost axis has a blockSize of 1, +// and inputPitch and inputDelta are just a single value added each iteration. +template +static void PadInnermostAxis(T* output, T* input, ptrdiff_t input_delta, size_t block_count) { + for (size_t block_index = 0; block_index < block_count; block_index++) { + *output++ = *input; + input += input_delta; + } +} + +// For constant padding, there is no input, just a size to write the constant to +template +static void PadAxisConstant(T* output, T constant, size_t size) { + for (size_t i = 0; i < size; i++) + *output++ = constant; +} + +template <> +Status Pad::Compute(OpKernelContext* ctx) const { + auto& input_tensor = *ctx->Input(0); + std::vector output_dims(input_tensor.Shape().GetDims()); + size_t dimension_count = output_dims.size(); + + ONNXRUNTIME_ENFORCE(dimension_count * 2 == pads_.size(), "'pads' attribute has wrong number of values"); + + std::vector input_starts; + std::vector input_extents; + + // Calculate output dimensions, and handle any negative padding + for (size_t i = 0; i < dimension_count; i++) { + input_starts.push_back(slices_[i]); + input_extents.push_back(output_dims[i] + slices_[i] + slices_[i + dimension_count]); + output_dims[i] += pads_[i] + pads_[i + dimension_count] + slices_[i] + slices_[i + dimension_count]; + } + TensorShape output_shape(output_dims); + + SliceIterator input(input_tensor, input_starts, input_extents); + auto& output_tensor = *ctx->Output(0, output_shape); + auto* output = output_tensor.template MutableData(); + + TensorPitches output_pitches(output_tensor); + size_t alignSkip = 0; // Amount to skip to align to where the next input tensor data needs to be written + + // Initial skip, sum up the begin padding on each axis + for (size_t i = 0; i < dimension_count; i++) + alignSkip += pads_[i] * output_pitches[i]; + + size_t inner_axis = dimension_count - 1; + TensorAxisCounters input_counters(input_tensor); + + switch (mode_) { + case Mode::Constant: + // Loop over the output tensor, writing out padding between the blocks of copied data + // On loop entry, 'pad' is already set to the first continuous block of padding, and + // after every pass through the inner loop it gets set to the next continuous pad size. + while (input_counters) { + output += alignSkip; + { + float* axisStart = output; + output = input.CopyInnermostAxis(output); + + int64_t prePad = pads_[inner_axis]; + int64_t postPad = pads_[inner_axis + dimension_count]; + PadAxisConstant(axisStart - prePad, value_, prePad); + PadAxisConstant(output, value_, postPad); + output += postPad; + alignSkip = prePad; + } + // Calculate the size of the next block of padding (skipping over the innermost axis since that's already done) + while (input_counters.Increment()) { + ptrdiff_t inner_pitch = output_pitches[input_counters.Axis()]; + float* axisStart = output - inner_pitch * input_extents[input_counters.Axis()]; + int64_t prePad = pads_[input_counters.Axis()]; + int64_t postPad = pads_[input_counters.Axis() + dimension_count]; + PadAxisConstant(axisStart - prePad * inner_pitch, value_, prePad * inner_pitch); + PadAxisConstant(output, value_, postPad * inner_pitch); + output += inner_pitch * postPad; + alignSkip += inner_pitch * prePad; + } + } + break; + + case Mode::Edge: + // Loop over the output tensor, writing out padding between the blocks of copied data + // On loop entry, 'pad' is already set to the first continuous block of padding, and + // after every pass through the inner loop it gets set to the next continuous pad size. + while (input_counters) { + output += alignSkip; + { + float* axisStart = output; + output = input.CopyInnermostAxis(output); + + int64_t prePad = pads_[inner_axis]; + int64_t postPad = pads_[inner_axis + dimension_count]; + PadAxisConstant(axisStart - prePad, *axisStart, prePad); + PadAxisConstant(output, *(output - 1), postPad); + output += postPad; + alignSkip = prePad; + } + // Calculate the size of the next block of padding (skipping over the innermost axis since that's already done) + while (input_counters.Increment()) { + ptrdiff_t inner_pitch = output_pitches[input_counters.Axis()]; + float* axisStart = output - inner_pitch * input_extents[input_counters.Axis()]; + int64_t prePad = pads_[input_counters.Axis()]; + int64_t postPad = pads_[input_counters.Axis() + dimension_count]; + PadAxis(axisStart - prePad * inner_pitch, axisStart, 1, -inner_pitch, inner_pitch, prePad); + PadAxis(output, output - inner_pitch, 1, -inner_pitch, inner_pitch, postPad); + output += inner_pitch * postPad; + alignSkip += inner_pitch * prePad; + } + } + break; + + case Mode::Reflect: + // Loop over the output tensor, writing out padding between the blocks of copied data + // On loop entry, 'pad' is already set to the first continuous block of padding, and + // after every pass through the inner loop it gets set to the next continuous pad size. + while (input_counters) { + output += alignSkip; + { + float* axisStart = output; + output = input.CopyInnermostAxis(output); + + int64_t prePad = pads_[inner_axis]; + int64_t postPad = pads_[inner_axis + dimension_count]; + PadInnermostAxis(axisStart - prePad, axisStart + prePad, -1 /* inputDelta */, prePad); + PadInnermostAxis(output, output - 2, -1 /* inputDelta */, postPad); + output += postPad; + alignSkip = prePad; + } + // Calculate the size of the next block of padding (skipping over the innermost axis since that's already done) + while (input_counters.Increment()) { + ptrdiff_t inner_pitch = output_pitches[input_counters.Axis()]; + float* axisStart = output - inner_pitch * input_extents[input_counters.Axis()]; + int64_t prePad = pads_[input_counters.Axis()]; + int64_t postPad = pads_[input_counters.Axis() + dimension_count]; + PadAxis(axisStart - prePad * inner_pitch, axisStart + prePad * inner_pitch, 1, -inner_pitch * 2, inner_pitch, prePad); + PadAxis(output, output - 2 * inner_pitch, 1, -inner_pitch * 2, inner_pitch, postPad); + output += inner_pitch * postPad; + alignSkip += inner_pitch * prePad; + } + } + break; + } + + return Status::OK(); +} +}; // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/pad.h b/onnxruntime/core/providers/cpu/tensor/pad.h new file mode 100644 index 0000000000000..50a02a4ca3edb --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/pad.h @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { + +class PadBase { + protected: + PadBase(const OpKernelInfo& info) : value_(info.GetAttrOrDefault("value", 0.f)) { + std::string mode; + if (info.GetAttr("mode", &mode).IsOK()) { + if (mode == "constant") + mode_ = Mode::Constant; + else if (mode == "reflect") + mode_ = Mode::Reflect; + else if (mode == "edge") + mode_ = Mode::Edge; + else + ONNXRUNTIME_THROW("Invalid 'mode' attribute value"); + } + if (!info.GetAttrs("pads", pads_).IsOK()) + ONNXRUNTIME_THROW("Invalid 'pads' attribute value"); + + // Separate out any negative pads_ into the slices_ array + slices_.resize(pads_.size(), 0); + for (size_t index = 0; index < pads_.size(); index++) { + if (pads_[index] < 0) { + slices_[index] = pads_[index]; + pads_[index] = 0; + } + } + + ; // Value is optional and initialized to 0 by default + } + + ~PadBase() {} + + enum class Mode : int { + Constant = 0, + Reflect, + Edge + }; + Mode mode_{Mode::Constant}; + std::vector pads_; // After construction, only >=0 values are in here + std::vector slices_; // All of the negative padding values are separated out into slices_ + const float value_; +}; + +template +struct Pad final : public OpKernel, public PadBase { + Pad(const OpKernelInfo& info) : OpKernel(info), PadBase(info) {} + + Status Compute(OpKernelContext* context) const override; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/reshape.cc b/onnxruntime/core/providers/cpu/tensor/reshape.cc new file mode 100644 index 0000000000000..daaabcbd0f169 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/reshape.cc @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/tensor/reshape.h" +namespace onnxruntime { + +ONNX_CPU_OPERATOR_KERNEL( + Reshape, + 5, + KernelDefBuilder() + .Alias(0, 0) + .TypeConstraint("T", DataTypeImpl::AllTensorTypes()) + .TypeConstraint("shape", DataTypeImpl::GetTensorType()), + Reshape); + +ONNX_CPU_OPERATOR_VERSIONED_KERNEL( + Reshape_1, + 1, + 4, + KernelDefBuilder() + .Alias(0, 0) + .TypeConstraint("T", DataTypeImpl::AllTensorTypes()), + Reshape_1); + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/reshape.h b/onnxruntime/core/providers/cpu/tensor/reshape.h new file mode 100644 index 0000000000000..b8e52a2af2236 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/reshape.h @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "gsl/gsl_util" +#include "reshape_helper.h" +#include "utils.h" + +namespace onnxruntime { + +class Reshape final : public OpKernel { + public: + Reshape(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override { + // Copy the second input tensor into the shape vector + const Tensor* shapeTensor = context->Input(1); + ONNXRUNTIME_ENFORCE(shapeTensor->Shape().NumDimensions() == 1, + "A shape tensor must be a vector tensor."); + size_t nDims = static_cast(shapeTensor->Shape()[0]); + const int64_t* data = shapeTensor->template Data(); + std::vector shape(data, data + nDims); + + const Tensor* X = context->Input(0); + const TensorShape& X_shape = X->Shape(); + + ReshapeHelper helper(X_shape, shape); + + Tensor* Y = context->Output(0, TensorShape(shape)); + + CopyCpuTensor(X, Y); + + return Status::OK(); + } +}; + +class Reshape_1 final : public OpKernel { + public: + Reshape_1(const OpKernelInfo& info) : OpKernel(info) { + Status status = info.GetAttrs("shape", shape_); + ONNXRUNTIME_ENFORCE(status.IsOK(), "Attribute shape is not set."); + } + + Status Compute(OpKernelContext* context) const override { + std::vector shape = shape_; + const Tensor* X = context->Input(0); + const TensorShape& X_shape = X->Shape(); + + ReshapeHelper helper(X_shape, shape); + + Tensor* Y = context->Output(0, TensorShape(shape)); + + CopyCpuTensor(X, Y); + + return Status::OK(); + } + + private: + std::vector shape_; +}; + +} //namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/reshape_helper.h b/onnxruntime/core/providers/cpu/tensor/reshape_helper.h new file mode 100644 index 0000000000000..abb45e652c5fb --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/reshape_helper.h @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/tensor.h" +#include "gsl/gsl_util" + +namespace onnxruntime { + +// Verify and convert unknown dim during reshape +class ReshapeHelper { + public: + ReshapeHelper(const TensorShape& input_shape, std::vector& requested_shape) { + auto nDims = requested_shape.size(); + int64_t unknown_dim = -1; + int64_t size = 1; + for (size_t i = 0; i < nDims; ++i) { + ONNXRUNTIME_ENFORCE(requested_shape[i] >= -1, "A dimension cannot be less than -1."); + if (requested_shape[i] == -1) { + ONNXRUNTIME_ENFORCE(unknown_dim == -1, "At most one dimension can be -1."); + unknown_dim = i; + } else { + if (requested_shape[i] == 0) { + ONNXRUNTIME_ENFORCE(i < input_shape.NumDimensions(), + "The dimension with value zero exceeds" + " the dimension size of the input tensor."); + requested_shape[i] = input_shape[i]; + } + size *= requested_shape[i]; + } + } + + if (unknown_dim != -1) { + // calculate unknown dimension + ONNXRUNTIME_ENFORCE((input_shape.Size() % size) == 0, + "The input tensor cannot be reshaped to the requested shape. Input shape:", input_shape); + requested_shape[unknown_dim] = input_shape.Size() / size; + } else { + // check if the output shape is valid. + ONNXRUNTIME_ENFORCE(gsl::narrow_cast(input_shape.Size()) == size, + "The input tensor cannot be reshaped to the requested shape. Input shape:", input_shape); + } + } +}; + +} //namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/shape_op.cc b/onnxruntime/core/providers/cpu/tensor/shape_op.cc new file mode 100644 index 0000000000000..aba5b45c853d0 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/shape_op.cc @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/tensor/shape_op.h" + +namespace onnxruntime { + +const std::vector shapeOpTypeConstraints{ + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}; + +ONNX_CPU_OPERATOR_KERNEL( + Shape, + 1, + KernelDefBuilder().TypeConstraint("T", shapeOpTypeConstraints).TypeConstraint("T1", DataTypeImpl::GetTensorType()), + Shape); + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/shape_op.h b/onnxruntime/core/providers/cpu/tensor/shape_op.h new file mode 100644 index 0000000000000..7148301cda222 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/shape_op.h @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +#include "gsl/gsl_util" + +namespace onnxruntime { + +class Shape final : public OpKernel { + public: + Shape(const OpKernelInfo& info) : OpKernel(info) { + } + + // Takes a tensor as input and outputs an 1D int64 tensor + // containing the shape of the input tensor. + Status Compute(OpKernelContext* context) const override { + const Tensor* input = context->Input(0); + const TensorShape& inputShape = input->Shape(); + + size_t nDims = inputShape.NumDimensions(); + TensorShape outputShape({gsl::narrow_cast(nDims)}); + Tensor* output = context->Output(0, TensorShape(outputShape)); + + inputShape.CopyDims(output->template MutableData(), nDims); + return Status::OK(); + } +}; +} //namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/size.cc b/onnxruntime/core/providers/cpu/tensor/size.cc new file mode 100644 index 0000000000000..ab4af299ae3bf --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/size.cc @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/tensor/size.h" +#include + +namespace onnxruntime { + +Status Size::Compute(OpKernelContext* ctx) const { + const Tensor* input_tensor = ctx->Input(0); + if (input_tensor == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + TensorShape scalar_shape; + Tensor* p_output_tensor = ctx->Output(0, scalar_shape); + assert(p_output_tensor->Size() == sizeof(int64_t)); + int64_t* p_output_scalar = p_output_tensor->template MutableData(); + + *p_output_scalar = input_tensor->Shape().Size(); + + return Status::OK(); +} + +// The implementation of Size works for tensors of any type. The types listed below are +// based on the ones the datatypes in data_types.cc. +// TODO: we should not have to add the TypeConstraint below, since it is meant to be in +// addition to the ONNX specification. But the registration doesn't seem to work if we +// omit this. +// TODO: Both onnxruntime and ONNX lists of types seem somewhat incomplete and incomparable. + +ONNX_CPU_OPERATOR_KERNEL( + Size, + 1, + KernelDefBuilder().TypeConstraint("T", + std::vector({DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()})), + Size); + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/size.h b/onnxruntime/core/providers/cpu/tensor/size.h new file mode 100644 index 0000000000000..64e6b94356cbe --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/size.h @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { + +class Size final : public OpKernel { + public: + Size(const OpKernelInfo& info) : OpKernel{info} {} + + Status Compute(OpKernelContext* context) const override; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/slice.cc b/onnxruntime/core/providers/cpu/tensor/slice.cc new file mode 100644 index 0000000000000..fe7729c86db08 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/slice.cc @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/tensor/slice.h" +#include "core/providers/cpu/tensor/utils.h" +using namespace ::onnxruntime::common; + +namespace onnxruntime { + +ONNX_CPU_OPERATOR_KERNEL( + Slice, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Slice); +namespace { +// std::clamp doesn't exist until C++17 so create a local version +template +const T& clamp(const T& v, const T& lo, const T& hi) { + if (v < lo) return lo; + if (v > hi) return hi; + return v; +} +} // namespace +Status SliceBase::PrepareForCompute(const size_t dimension_count, const std::vector& input_dimensions, + std::vector& starts, std::vector& output_dims) const { + // Initialize axes to the provided axes attribute or to the default sequence + std::vector axes(axes_); + if (!has_axes_) { + //axes are omitted, they are set to[0, ..., ndim - 1] + axes.resize(starts.size()); + for (size_t i = 0; i < starts.size(); i++) + axes[i] = i; + + if (axes.size() > starts_.size()) + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "'axes' has more entries than the 'starts' attribute holds"); + if (axes.size() > ends_.size()) + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "'axes' has more entries than the 'ends' attribute holds"); + } + + // Iterate through the provided axes and override the start/end ranges + for (size_t axesIndex = 0; axesIndex < axes.size(); axesIndex++) { + auto axis = static_cast(axes[axesIndex]); + if (axis >= dimension_count) + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "'axes' has an axis outside of the tensor dimension count"); + auto start = starts_[axesIndex]; + if (start < 0) + start += input_dimensions[axis]; + starts[axis] = clamp(start, int64_t{0}, input_dimensions[axis]); + + auto end = ends_[axesIndex]; + if (end < 0) + end += input_dimensions[axis]; + output_dims[axis] = clamp(end, int64_t{0}, input_dimensions[axis]) - starts[axis]; + if (output_dims[axis] < 0) + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "'starts' and 'ends' values resulted in a negative dimension"); + } + + return Status::OK(); +} + +template <> +Status Slice::Compute(OpKernelContext* ctx) const { + const Tensor* input_tensor_ptr = ctx->Input(0); + ONNXRUNTIME_ENFORCE(input_tensor_ptr != nullptr); + auto& input_tensor = *input_tensor_ptr; + auto& input_dimensions = input_tensor.Shape().GetDims(); + + // Initialize the starts & ends to the actual tensor shape + const size_t dimension_count = input_dimensions.size(); + std::vector starts(dimension_count, 0); + std::vector output_dims(input_dimensions); + + ONNXRUNTIME_RETURN_IF_ERROR(PrepareForCompute(dimension_count, input_dimensions, starts, output_dims)); + + TensorShape output_shape(output_dims); + auto& output_tensor = *ctx->Output(0, output_shape); + auto* output = output_tensor.template MutableData(); + const auto* output_end = output + output_shape.Size(); + + SliceIterator input_iterator(input_tensor, starts, output_dims); + while (output != output_end) + *output++ = *input_iterator++; + + return Status::OK(); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/slice.h b/onnxruntime/core/providers/cpu/tensor/slice.h new file mode 100644 index 0000000000000..26adcd9fdeb46 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/slice.h @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/util/math_cpuonly.h" + +namespace onnxruntime { + +class SliceBase { + protected: + SliceBase(const OpKernelInfo& info) { + has_axes_ = info.GetAttrs("axes", axes_).IsOK(); + + ONNXRUNTIME_ENFORCE(info.GetAttrs("starts", starts_).IsOK(), "Invalid 'starts' attribute value"); + ONNXRUNTIME_ENFORCE(info.GetAttrs("ends", ends_).IsOK(), "Invalid 'ends' attribute value"); + + if (has_axes_) { + if (axes_.size() > starts_.size()) + ONNXRUNTIME_THROW("'axes' has more entries than the 'starts' attribute holds"); + if (axes_.size() > ends_.size()) + ONNXRUNTIME_THROW("'axes' has more entries than the 'ends' attribute holds"); + } + } + + Status PrepareForCompute(const size_t dimension_count, const std::vector& input_dimensions, + std::vector& starts, std::vector& output_dims) const; + + std::vector axes_; + bool has_axes_; + std::vector starts_, ends_; +}; + +template +struct Slice final : public OpKernel, public SliceBase { + Slice(const OpKernelInfo& info) : OpKernel(info), SliceBase(info) {} + + Status Compute(OpKernelContext* context) const override; +}; // namespace onnxruntime + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/space_depth_ops.cc b/onnxruntime/core/providers/cpu/tensor/space_depth_ops.cc new file mode 100644 index 0000000000000..1532cf1ee8bf1 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/space_depth_ops.cc @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/tensor/space_depth_ops.h" + +// warning raised from Eigen Tensor code. +#ifdef _MSC_VER +#pragma warning(disable : 4554) +#endif +#include "core/util/eigen_common_wrapper.h" +#include + +namespace onnxruntime { + +ONNX_CPU_OPERATOR_KERNEL( + SpaceToDepth, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + SpaceToDepth); + +ONNX_CPU_OPERATOR_VERSIONED_KERNEL( + DepthToSpace, + 1, + 4, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + DepthToSpace); + +// intemediate tensor shapes are: +// (batch, blocksize, blocksize, input_depth / (blocksize * blocksize), input_height, input_width) for DepthToSpace +// (batch, input_depth, input_height / blocksize, blocksize, input_width / blocksize, blocksize) for SpaceToDepth +const int IntermediateTensorRank = 6; +typedef Eigen::TensorMap, + Eigen::Aligned> + EigenTensorMap; + +template <> +Status SpaceToDepth::Compute(OpKernelContext* context) const { + const Tensor* tensor_pointer = context->Input(0); + if (tensor_pointer == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + const Tensor& input = *tensor_pointer; + ONNXRUNTIME_ENFORCE(input.Shape().NumDimensions() == 4); + const int64_t batch = input.Shape().GetDims().at(0); + const int64_t input_depth = input.Shape().GetDims().at(1); + const int64_t input_height = input.Shape().GetDims().at(2); + const int64_t input_width = input.Shape().GetDims().at(3); + ONNXRUNTIME_ENFORCE(input_height % this->blocksize_ == 0); + ONNXRUNTIME_ENFORCE(input_width % this->blocksize_ == 0); + + const int64_t output_depth = input_depth * blocksize_ * blocksize_; + const int64_t output_height = input_height / blocksize_; + const int64_t output_width = input_width / blocksize_; + Tensor& output = *context->Output(0, {batch, output_depth, output_height, output_width}); + + std::array permutation{{0, 3, 5, 1, 2, 4}}; + EigenTensorMap(output.template MutableData(), batch, blocksize_, blocksize_, + input_depth, input_height / blocksize_, input_width / blocksize_) = + EigenTensorMap(const_cast(input.template Data()), batch, + input_depth, input_height / blocksize_, blocksize_, + input_width / blocksize_, blocksize_) + .shuffle(permutation); + + return Status::OK(); +} + +template <> +Status DepthToSpace::Compute(OpKernelContext* context) const { + const Tensor* tensor_pointer = context->Input(0); + if (tensor_pointer == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); + const Tensor& input = *tensor_pointer; + ONNXRUNTIME_ENFORCE(input.Shape().NumDimensions() == 4); + + const int64_t batch = input.Shape().GetDims().at(0); + const int64_t input_depth = input.Shape().GetDims().at(1); + const int64_t input_height = input.Shape().GetDims().at(2); + const int64_t input_width = input.Shape().GetDims().at(3); + ONNXRUNTIME_ENFORCE(input_depth % (blocksize_ * blocksize_) == 0); + + const int64_t output_depth = input_depth / blocksize_ / blocksize_; + const int64_t output_height = input_height * blocksize_; + const int64_t output_width = input_width * blocksize_; + + Tensor& output = *context->Output(0, {batch, output_depth, output_height, output_width}); + + std::array permutation{{0, 3, 4, 1, 5, 2}}; + EigenTensorMap(output.template MutableData(), batch, input_depth / blocksize_ / blocksize_, + input_height, blocksize_, input_width, blocksize_) = + EigenTensorMap(const_cast(input.template Data()), batch, + blocksize_, blocksize_, input_depth / blocksize_ / blocksize_, + input_height, input_width) + .shuffle(permutation); + + return Status::OK(); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/space_depth_ops.h b/onnxruntime/core/providers/cpu/tensor/space_depth_ops.h new file mode 100644 index 0000000000000..fec8c0caae3e8 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/space_depth_ops.h @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/framework/op_kernel.h" + +namespace onnxruntime { + +class SpaceDepthBase : public OpKernel { + public: + SpaceDepthBase(const OpKernelInfo& info) : OpKernel(info) { + ONNXRUNTIME_ENFORCE(info.GetAttr("blocksize", &blocksize_).IsOK(), + "Attribute blocksize is not set."); + } + + protected: + int64_t blocksize_; +}; + +template +class SpaceToDepth final : public SpaceDepthBase { + public: + SpaceToDepth(const OpKernelInfo& info) : SpaceDepthBase(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +template +class DepthToSpace final : public SpaceDepthBase { + public: + DepthToSpace(const OpKernelInfo& info) : SpaceDepthBase(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +} //namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/split.cc b/onnxruntime/core/providers/cpu/tensor/split.cc new file mode 100644 index 0000000000000..afc31d6380aef --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/split.cc @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/tensor/split.h" +#include "core/providers/common.h" +#include "core/util/math.h" +#include "core/util/math_cpuonly.h" + +#include "gsl/gsl_util" + +namespace onnxruntime { + +ONNX_CPU_OPERATOR_KERNEL( + Split, + 2, + KernelDefBuilder().TypeConstraint("T", + std::vector{ + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + }), + Split); + +Status Split::Compute(OpKernelContext* context) const { + const Tensor& input = *context->Input(0); + + Status status; + auto data_type = input.DataType(); + + if (data_type == DataTypeImpl::GetType()) + status = ComputeImpl(*context, input); + else if (data_type == DataTypeImpl::GetType()) { + /* Need to update CopyMatrix to support double... + status = ComputeImpl(*context, input); */ + ONNXRUNTIME_NOT_IMPLEMENTED("Split operator does not support double yet"); + } else + ONNXRUNTIME_THROW("Invalid data type for Split operator of ", data_type); + + return status; +} + +template +Status Split::ComputeImpl(OpKernelContext& context, const Tensor& input) const { + auto& input_shape = input.Shape(); + auto& input_dims = input_shape.GetDims(); + const int64_t num_dimensions = gsl::narrow_cast(input_shape.NumDimensions()); + const int64_t axis = HandleNegativeAxis(axis_, num_dimensions); // handle negative and enforce axis is valid + const int64_t split_dim_size = input_dims[axis]; + + auto num_outputs = context.OutputCount(); + std::vector outputs; + outputs.reserve(num_outputs); + + int before_dims = gsl::narrow(input_shape.SizeToDimension(axis)); + int after_dims_including_split_axis = gsl::narrow(input_shape.SizeFromDimension(axis)); + int after_dims_excluding_split = (axis + 1 == num_dimensions) + ? 1 // we multiply by this value so must be 1 not 0 + : gsl::narrow(input_shape.SizeFromDimension(axis + 1)); + + std::vector split_sizes; + + if (split_sizes_.empty()) { + // equal split based on number of outputs + if (split_dim_size % num_outputs != 0) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Input cannot be split evenly on selected axis. Input shape=", input_shape, + " Axis=", axis_, " NumOutputs=", num_outputs); + } + + // populate split_sizes with the same size for each output + split_sizes = std::vector(num_outputs, split_dim_size / num_outputs); + } else { + if (split_sizes_.size() != num_outputs || split_size_sum_ != split_dim_size) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, + "Cannot split using values in 'split' attribute. Axis=", axis_, + " Input shape=", input_shape, + " NumOutputs=", num_outputs, + " Num entries in 'split' (must equal number of outputs) was ", split_sizes_.size(), + " Sum of sizes in 'split' (must equal size of selected axis) was ", split_size_sum_); + + split_sizes = split_sizes_; + } + + // copy dimensions so we can update the selected axis in place + std::vector output_dimensions{input_dims}; + + int64_t input_offset = 0; + const T* input_data = input.template Data(); + + for (int i = 0; i < num_outputs; ++i) { + // update size of dimension for axis we're splitting on + auto split_size = gsl::narrow(split_sizes[i]); + output_dimensions[axis] = split_size; + + Tensor* output = context.Output(i, TensorShape{output_dimensions}); + T* output_data = output->template MutableData(); + + ::onnxruntime::math::CopyMatrix( + sizeof(T), + before_dims, // M + split_size * after_dims_excluding_split, // N + static_cast(input_data + input_offset), // A + after_dims_including_split_axis, // lda + static_cast(output_data), // B + split_size * after_dims_excluding_split, // ldb + &CPUMathUtil::Instance()); + + input_offset += split_size * after_dims_excluding_split; // offset by the N data we used in this iteration + } + + return Status::OK(); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/split.h b/onnxruntime/core/providers/cpu/tensor/split.h new file mode 100644 index 0000000000000..1d61d91483ba2 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/split.h @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { + +class Split final : public OpKernel { + public: + Split(const OpKernelInfo& info) : OpKernel(info) { + // required with default of 0 + if (!info.GetAttr("axis", &axis_).IsOK()) + ONNXRUNTIME_THROW("Missing 'axis' attribute value"); + + // optional + if (info.GetAttrs("split", split_sizes_).IsOK()) { + split_size_sum_ = std::accumulate(split_sizes_.cbegin(), split_sizes_.cend(), 0LL); + ONNXRUNTIME_ENFORCE(std::all_of(split_sizes_.cbegin(), split_sizes_.cend(), [](int64_t value) { return value > 0; }), + "Invalid value in 'split' attribute. All values must be > 0"); + } + } + + Status Compute(OpKernelContext* context) const override; + + private: + template + Status ComputeImpl(OpKernelContext& context, const Tensor& input) const; + + int64_t axis_; + std::vector split_sizes_; + int64_t split_size_sum_ = 0; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/squeeze.cc b/onnxruntime/core/providers/cpu/tensor/squeeze.cc new file mode 100644 index 0000000000000..b29af6ac67bda --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/squeeze.cc @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/tensor/squeeze.h" + +namespace onnxruntime { + +ONNX_CPU_OPERATOR_KERNEL( + Squeeze, + 1, + KernelDefBuilder() + .TypeConstraint("T", DataTypeImpl::AllTensorTypes()) + .Alias(0, 0), + Squeeze); + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/squeeze.h b/onnxruntime/core/providers/cpu/tensor/squeeze.h new file mode 100644 index 0000000000000..addd3f95221b8 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/squeeze.h @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "utils.h" + +namespace onnxruntime { + +class SqueezeBase { + protected: + SqueezeBase(const OpKernelInfo& info) { + std::vector axes; + Status status = info.GetAttrs("axes", axes); + ONNXRUNTIME_ENFORCE(status.IsOK(), "Attribute axes is not set."); + + // Handle out of order and repeating dims. + std::sort(axes.begin(), axes.end()); + axes.erase(std::unique(axes.begin(), axes.end()), axes.end()); + axes_ = axes; + } + + static std::vector ComputeOutputShape( + std::vector input_shape, + std::vector axes) { + int j = 0; + std::vector output_shape; + for (size_t i = 0; i < input_shape.size(); ++i) { + if (j < axes.size() && axes[j] == static_cast(i)) { + ONNXRUNTIME_ENFORCE(input_shape[i] == 1, "Dimension of input ", i, + " must be 1 instead of ", input_shape[i]); + ++j; + continue; + } + output_shape.push_back(input_shape[i]); + } + return output_shape; + } + + std::vector axes_; +}; + +class Squeeze final : public OpKernel, public SqueezeBase { + public: + Squeeze(const OpKernelInfo& info) : OpKernel(info), SqueezeBase(info) {} + + Status Compute(OpKernelContext* context) const override { + const Tensor* X = context->Input(0); + const TensorShape& X_shape = X->Shape(); + std::vector output_shape = ComputeOutputShape(X_shape.GetDims(), axes_); + + Tensor* Y = context->Output(0, TensorShape(output_shape)); + + CopyCpuTensor(X, Y); + + return Status::OK(); + } +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/tile.cc b/onnxruntime/core/providers/cpu/tensor/tile.cc new file mode 100644 index 0000000000000..b478541b32601 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/tile.cc @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// there's no way to use a raw pointer as the copy destination with std::copy_n +// (which gsl::copy uses with span::data() which returns a raw pointer) with the 14.11 toolset +// without generating a 4996 warning. going through an iterator is way too much overhead so turn off the warning. +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4996) +#endif + +#include "gsl/gsl_algorithm" +#include "core/providers/cpu/tensor/tile.h" +#include "core/providers/cpu/tensor/utils.h" + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +using namespace ::onnxruntime::common; + +namespace onnxruntime { + +ONNX_CPU_OPERATOR_KERNEL( + Tile, + 6, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Tile); + +template <> +Status Tile::Compute(OpKernelContext* ctx) const { + const Tensor* tensor_pointer = ctx->Input(0); + if (tensor_pointer == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "Input count of Tile OP mismatch, the first one is empty"); + const Tensor& input_tensor = *tensor_pointer; + tensor_pointer = ctx->Input(1); + if (tensor_pointer == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "Input count of Tile OP mismatch, the second one is empty"); + const Tensor& repeats_tensor = *tensor_pointer; + + size_t dimension_count = input_tensor.Shape().NumDimensions(); + + if (repeats_tensor.Shape().NumDimensions() != 1) + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "'repeat' input tensor must be 1 dimensional"); + if (size_t(repeats_tensor.Shape().Size()) != input_tensor.Shape().NumDimensions()) + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "'repeat' input tensor must have the same length as the 'input' tensor"); + + // Calculate the shape of the output tensor + auto* repeats = repeats_tensor.template Data(); + std::vector output_dims = input_tensor.Shape().GetDims(); + for (auto axis = 0; axis < input_tensor.Shape().NumDimensions(); axis++) + output_dims[axis] *= repeats[axis]; + TensorShape outputShape(output_dims); + auto& output_tensor = *ctx->Output(0, outputShape); + + auto* output = output_tensor.template MutableData(); + auto* input = input_tensor.template Data(); + + TensorPitches output_pitches(output_tensor); + TensorAxisCounters input_counters(input_tensor); + + while (input_counters) { + // Copy the input data over + size_t input_pitch = input_tensor.Shape().GetDims().back(); + for (size_t i = 0; i < input_pitch; i++) + *output++ = *input++; + + // Tile it for the innermost axis + const auto* copy = output - input_tensor.Shape()[dimension_count - 1]; + for (int64_t repeat = (repeats[dimension_count - 1] - 1) * input_pitch; repeat-- > 0;) + *output++ = *copy++; + + // Tile it in the other axes + while (input_counters.Increment()) { + ptrdiff_t pitch = output_pitches[input_counters.Axis()] * input_tensor.Shape()[input_counters.Axis()]; + copy = output - pitch; + for (int64_t repeat = (repeats[input_counters.Axis()] - 1) * pitch; repeat-- > 0;) { + *output++ = *copy++; + } + } + } + return Status::OK(); +} +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/tile.h b/onnxruntime/core/providers/cpu/tensor/tile.h new file mode 100644 index 0000000000000..d7cf00097bd81 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/tile.h @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/util/math_cpuonly.h" + +namespace onnxruntime { + +template +struct Tile final : OpKernel { + Tile(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; + + private: +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/transpose.cc b/onnxruntime/core/providers/cpu/tensor/transpose.cc new file mode 100644 index 0000000000000..d7828870c4cd0 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/transpose.cc @@ -0,0 +1,150 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/tensor/transpose.h" + +namespace onnxruntime { + +/* A permutation [a,b,c,...] indicates that + - The 0-th dimension of the output corresponds to the a-th dimension of input + - The 1-st dimension of the output corresponds to the b-th dimension of input + - The 2-nd dimension of the output corresponds to the c-th dimension of input + etc. + */ + +// ComputeOffset: compute offset into a tensor. This is essentially the dot-product of +// index and stride, restricted to the specified number of axes. +size_t ComputeOffset(const std::vector& index, const std::vector& stride, int64_t num_axes) { + size_t offset = 0; + for (int64_t j = 0; j < num_axes; ++j) { + offset += index[j] * stride[j]; + } + return offset; +} + +// IncrementIndex: Increment an index into a tensor (in lexicographic ordering), wrapping +// around the specified upper_bound. +void IncrementIndex(std::vector& index, const std::vector& upper_bound, int64_t num_axes) { + for (int64_t k = num_axes - 1; k >= 0; --k) { + index[k]++; + if (index[k] < upper_bound[k]) break; + index[k] = 0; + } +} + +// DoTranspose: copies source tensor to target, transposing elements. +// The stride vector indicates the transposition. +void DoTranspose(int64_t num_axes, const std::vector& target_dims, + size_t num_blocks, size_t num_elts_in_block, + const std::vector& stride, + float* target, const float* source) { + size_t blocksize = num_elts_in_block * sizeof(float); + // index used to iterate over target iteration-space + std::vector target_index(num_axes, 0); + for (size_t i = 0; i < num_blocks; ++i) { + // convert target_index into an offset in source data + size_t source_offset = ComputeOffset(target_index, stride, num_axes); + + // copy + memcpy(target, source + source_offset, blocksize); + + // increment target_index: + IncrementIndex(target_index, target_dims, num_axes); + target += num_elts_in_block; + } +} + +// DoTransposeEltWise: specialization of DoTranspose for the num_elts_in_block=1 case. +// copies source tensor to target, transposing elements. +// The stride vector indicates the transposition. +void DoTransposeEltWise(int64_t num_axes, const std::vector& target_dims, + size_t num_blocks, + const std::vector& stride, + float* target, const float* source) { + // index used to iterate over target iteration-space + std::vector target_index(num_axes, 0); + for (size_t i = 0; i < num_blocks; ++i) { + // convert target_index into an offset in source data + size_t source_offset = ComputeOffset(target_index, stride, num_axes); + + // copy + *target = *(source + source_offset); + + // increment target_index: + IncrementIndex(target_index, target_dims, num_axes); + target++; + } +} + +// DoTransposeSingleBlock: specialization of DoTranspose for the num_blocks=1 case. +// copies source tensor to target, transposing elements. +// The stride vector indicates the transposition. +void DoTransposeSingleBlock(size_t num_elts_in_block, float* target, const float* source) { + size_t blocksize = num_elts_in_block * sizeof(float); + // copy + memcpy(target, source, blocksize); +} + +template <> +Status Transpose::Compute(OpKernelContext* ctx) const { + // Get input and output: + const Tensor* input_tensor_ptr = ctx->Input(0); + ONNXRUNTIME_ENFORCE(input_tensor_ptr != nullptr); + const Tensor& X = *input_tensor_ptr; + const TensorShape& input_shape = X.Shape(); + const std::vector& input_dims = input_shape.GetDims(); + size_t rank = input_dims.size(); + + std::vector output_dims(rank); + const std::vector* p_perm; + std::vector default_perm(rank); + ComputeOutputShape(X, output_dims, default_perm, p_perm); + + std::vector stride(rank); + for (int i = 0; i < rank; i++) { + size_t inpdim = (*p_perm)[i]; + if (inpdim + 1 < rank) + stride[i] = input_shape.SizeFromDimension(inpdim + 1); + else + stride[i] = 1; + } + + TensorShape output_shape{output_dims}; + Tensor* Y = ctx->Output(0, output_shape); + const float* Xdata = X.template Data(); + float* Ydata = Y->template MutableData(); + + // Partition the permutation into a prefix and the largest suffix such that + // every axis i in the suffix is mapped to i. + int64_t num_axes_in_prefix = 0; // number of axes in prefix + size_t suffix_blocksize = 1; // product of dimensions in the suffix + size_t prefix_blocksize = 1; // product of dimensions in the prefix + bool is_suffix = true; + for (int64_t i = rank - 1; i >= 0; --i) { + int64_t inpaxis = (*p_perm)[i]; + if (is_suffix && (inpaxis == i)) { + suffix_blocksize *= input_dims[inpaxis]; + } else { + is_suffix = false; + prefix_blocksize *= input_dims[inpaxis]; + ++num_axes_in_prefix; + } + } + + if (1 == prefix_blocksize) + DoTransposeSingleBlock(suffix_blocksize, Ydata, Xdata); + else if (1 == suffix_blocksize) + DoTransposeEltWise(num_axes_in_prefix, output_dims, prefix_blocksize, stride, Ydata, Xdata); + else + DoTranspose(num_axes_in_prefix, output_dims, prefix_blocksize, suffix_blocksize, stride, Ydata, Xdata); + + return Status::OK(); +} + +ONNX_CPU_OPERATOR_KERNEL( + Transpose, + 1, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Transpose); + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/transpose.h b/onnxruntime/core/providers/cpu/tensor/transpose.h new file mode 100644 index 0000000000000..2f8657ff35081 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/transpose.h @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "gsl/gsl_util" +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { + +class TransposeBase { + protected: + TransposeBase(const OpKernelInfo& info) { + Status status = info.GetAttrs("perm", perm_); + + if (status.IsOK()) { + perm_specified_ = true; + size_t rank = perm_.size(); + std::vector seen(rank, false); + // Check that perm_ is a valid permutation of [0,rank-1] + for (auto i : perm_) { + if ((i < 0) || (i >= gsl::narrow(rank))) + ONNXRUNTIME_THROW("Attribute perm of Transpose has an invalid value. Value ", i, " is outside range."); + if (seen[i]) + ONNXRUNTIME_THROW("Attribute perm of Transpose has an invalid value. Value ", i, " is repeated."); + seen[i] = true; + } + } + } + + void ComputeOutputShape(const Tensor& X, std::vector& output_dims, + std::vector& default_perm, const std::vector*& p_perm) const { + size_t rank = X.Shape().NumDimensions(); + const auto& input_dims = X.Shape().GetDims(); + + // Determine permutation to use: + // If no permutation was specified in the attributes, the default is [rank-1, ..., 0] + default_perm.resize(rank); + + if (perm_specified_) + p_perm = &perm_; + else { + for (int i = 0; i < rank; ++i) + default_perm[i] = rank - i - 1; + p_perm = &default_perm; + } + + // Determine shape of output, as well as stride to be used: + // stride[i] indicates the stride for the input-tensor dimension corresponding + // to the i-th dimension of the output + + output_dims.resize(rank); + for (int i = 0; i < rank; i++) { + size_t inpdim = (*p_perm)[i]; + output_dims[i] = input_dims[inpdim]; + } + } + + bool perm_specified_ = false; + std::vector perm_; +}; + +template +class Transpose final : public OpKernel, public TransposeBase { + public: + Transpose(const OpKernelInfo& info) : OpKernel(info), TransposeBase(info) {} + + Status Compute(OpKernelContext* context) const override; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/unsqueeze.cc b/onnxruntime/core/providers/cpu/tensor/unsqueeze.cc new file mode 100644 index 0000000000000..4a8f406226cf4 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/unsqueeze.cc @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/tensor/unsqueeze.h" +#include "utils.h" +using namespace ::onnxruntime::common; + +namespace onnxruntime { + +ONNX_CPU_OPERATOR_KERNEL( + Unsqueeze, + 1, + KernelDefBuilder() + .Alias(0, 0) + .TypeConstraint("T", DataTypeImpl::AllTensorTypes()), + Unsqueeze); + +Status UnsqueezeBase::PrepareCompute(OpKernelContext* ctx, Prepare& p) const { + const Tensor* X = ctx->Input(0); + ONNXRUNTIME_ENFORCE(X != nullptr); + auto& input_tensor = *X; + + // New dimension count is the current dimensions + the number of entries in axes_ + // Initialize output_dims to 0 in each axis initially + std::vector output_dims(axes_.size() + input_tensor.Shape().GetDims().size(), 0); + + // Set all axes_ indices to 1 in output_dims and check for duplicates + for (size_t axis : axes_) { + if (axis >= output_dims.size()) + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "'axes' has an out of range axis"); + if (output_dims[axis] != 0) + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "'axes' has a duplicate axis"); + output_dims[axis] = 1; + } + + // Now fill in the zero entries with the existing shape + { + auto begin = input_tensor.Shape().GetDims().cbegin(); + for (auto& axisSize : output_dims) { + if (axisSize == 0) + axisSize = *begin++; + } + assert(begin == input_tensor.Shape().GetDims().cend()); + } + + TensorShape output_shape(output_dims); + p.output_tensor = ctx->Output(0, output_shape); + p.input_tensor = &input_tensor; + return Status::OK(); +} + +Status Unsqueeze::Compute(OpKernelContext* ctx) const { + Prepare p; + ONNXRUNTIME_RETURN_IF_ERROR(PrepareCompute(ctx, p)); + + CopyCpuTensor(p.input_tensor, p.output_tensor); + + return Status::OK(); +} +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/unsqueeze.h b/onnxruntime/core/providers/cpu/tensor/unsqueeze.h new file mode 100644 index 0000000000000..3e4740bc8ad14 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/unsqueeze.h @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/util/math_cpuonly.h" +#include "core/framework/tensor.h" + +namespace onnxruntime { + +class UnsqueezeBase { + protected: + UnsqueezeBase(const OpKernelInfo& info) { + ONNXRUNTIME_ENFORCE(info.GetAttrs("axes", axes_).IsOK(), "Missing/Invalid 'axes' attribute value"); + } + + struct Prepare { + const Tensor* input_tensor; + Tensor* output_tensor; + }; + + Status PrepareCompute(OpKernelContext* context, Prepare& p) const; + + private: + std::vector axes_; +}; + +class Unsqueeze final : public OpKernel, public UnsqueezeBase { + public: + Unsqueeze(const OpKernelInfo& info) : OpKernel(info), UnsqueezeBase(info) {} + Status Compute(OpKernelContext* context) const override; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/upsample.cc b/onnxruntime/core/providers/cpu/tensor/upsample.cc new file mode 100644 index 0000000000000..904a503f3b5db --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/upsample.cc @@ -0,0 +1,231 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/tensor/upsample.h" +#include //for fabs + +using namespace ::onnxruntime::common; +using namespace std; +namespace onnxruntime { + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Upsample, + 7, + float, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Upsample); + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Upsample, + 7, + int32_t, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Upsample); + +void upsampleNearest2x( + int64_t batch_size, + int64_t num_channels, + int64_t input_height, + int64_t input_width, + const float* input, + float* output) { + const int64_t output_height = input_height * 2; + const int64_t output_width = input_width * 2; + for (int64_t n = 0; n < batch_size; ++n) { + for (int64_t c = 0; c < num_channels; ++c) { + for (int64_t y = 0; y < output_height; ++y) { + const int64_t in_y = y / 2; + for (int64_t x = 0; x < input_width; ++x) { + const float v = input[in_y * input_width + x]; + const int64_t oidx = output_width * y + x * 2; + output[oidx + 0] = v; + output[oidx + 1] = v; + } + } + input += input_height * input_width; + output += output_height * output_width; + } + } +} + +template +Status upsampleNearest(const T* input, + T* output, + const TensorShape& input_shape, + const TensorShape& output_shape, + const vector& scales) { + if (!input || !output) + return Status(ONNXRUNTIME, FAIL, "Upsample: input/output value is nullptr"); + if (input_shape.NumDimensions() != output_shape.NumDimensions()) + return Status(ONNXRUNTIME, FAIL, "Upsample: input/output value's dimension mismatch"); + auto n_dim = input_shape.NumDimensions(); + for (size_t i = 0, size = output_shape.Size(); i < size; i++) { + size_t old_idx = 0; + size_t cur_idx = i; + + int64_t base = 1; + for (int64_t j = static_cast(n_dim - 1); j >= 0; j--) { + auto tmp = cur_idx % output_shape[j]; + old_idx += (std::min(static_cast(tmp / scales[j]), input_shape[j] - 1)) * base; + base *= input_shape[j]; + cur_idx /= output_shape[j]; + } + + output[i] = input[old_idx]; + } + return Status::OK(); +} + +//This is a generic upsample in linear mode for N-D tensor. +//But what's the correct behavior for linear mode is not clear right now. +//this function is not enabled yet. +template +Status upsampleLiner(const T* input, + T* output, + const TensorShape& input_shape, + const TensorShape& output_shape, + const vector& scales) { + if (!input || !output) + return Status(ONNXRUNTIME, FAIL, "Upsample: input/output value is nullptr"); + if (input_shape.NumDimensions() != output_shape.NumDimensions()) + return Status(ONNXRUNTIME, FAIL, "Upsample: input/output value's dimension mismatch"); + auto n_dim = input_shape.NumDimensions(); + for (size_t i = 0, size = output_shape.Size(); i < size; i++) { + std::vector val1, val2; + std::vector d1, d2; + size_t cur_idx = i; + //val1, vla2, d1, d2 are in reverse order + for (int64_t j = static_cast(n_dim - 1); j >= 0; j--) { + T v = std::min((cur_idx % output_shape[j]) / scales[j], static_cast(input_shape[j] - 1)); + auto v1 = std::min(static_cast(v), input_shape[j] - 1); + auto v2 = std::min(v1 + 1, input_shape[j] - 1); + if (v1 == v2) { + d1.push_back(0.5f); + d2.push_back(0.5f); + } else { + d1.push_back(std::abs(v - v1)); + d2.push_back(std::abs(v - v2)); + } + val1.push_back(v1); + val2.push_back(v2); + cur_idx /= output_shape[j]; + } + + output[i] = 0; + int64_t step = static_cast(1 << n_dim) - 1; + while (step >= 0) { + auto cur = step; + float w = 1.0f; + size_t old_idx = 0; + size_t base = 1; + for (int64_t j = static_cast(n_dim - 1); j >= 0; j--) { + int64_t reverse_idx = static_cast(n_dim - 1) - j; + w *= (cur % 2) ? d1[reverse_idx] : d2[reverse_idx]; + old_idx += ((cur % 2) ? val2[reverse_idx] : val1[reverse_idx]) * base; + base *= input_shape[j]; + cur >>= 1; + } + output[i] += input[old_idx] * w; + step--; + } + } + return Status::OK(); +} + +template +void upsampleBilinear( + int64_t batch_size, + int64_t num_channels, + int64_t input_height, + int64_t input_width, + float height_scale, + float width_scale, + const T* Xdata, + T* Ydata) { + int64_t output_width = static_cast(input_width * width_scale); + int64_t output_height = static_cast(input_height * height_scale); + + for (int64_t n = 0; n < batch_size; ++n) { + for (int64_t c = 0; c < num_channels; ++c) { + for (int64_t y = 0; y < output_height; ++y) { + float in_y = std::min(y / height_scale, static_cast(input_height - 1)); + const int64_t in_y1 = std::min(static_cast(in_y), input_height - 1); + const int64_t in_y2 = std::min(in_y1 + 1, input_height - 1); + float dy1 = fabs(in_y - in_y1); + float dy2 = fabs(in_y - in_y2); + if (in_y1 == in_y2) { + dy1 = 0.5f; + dy2 = 0.5f; + } + + const int64_t input_width_mul_y1 = input_width * in_y1; + const int64_t input_width_mul_y2 = input_width * in_y2; + + for (int64_t x = 0; x < output_width; ++x) { + float in_x = std::min(x / width_scale, static_cast(input_width - 1)); + const int64_t in_x1 = std::min(static_cast(in_x), input_width - 1); + const int64_t in_x2 = std::min(in_x1 + 1, input_width - 1); + + float dx1 = std::abs(in_x - in_x1); + float dx2 = std::abs(in_x - in_x2); + if (in_x1 == in_x2) { + dx1 = 0.5f; + dx2 = 0.5f; + } + + T X11 = Xdata[input_width_mul_y1 + in_x1]; + T X21 = Xdata[input_width_mul_y1 + in_x2]; + T X12 = Xdata[input_width_mul_y2 + in_x1]; + T X22 = Xdata[input_width_mul_y2 + in_x2]; + + Ydata[output_width * y + x] = static_cast(dx2 * dy2 * X11 + + dx1 * dy2 * X21 + + dx2 * dy1 * X12 + + dx1 * dy1 * X22); + } + } + Xdata += input_height * input_width; + Ydata += output_width * output_height; + } + } +} + +template +Status Upsample::Compute(OpKernelContext* context) const { + const Tensor* X = context->Input(0); + ONNXRUNTIME_ENFORCE(X != nullptr); + + const std::vector& dims = X->Shape().GetDims(); + if (dims.size() != scales_.size()) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Upsample: input tensor's dimension does not match the scales."); + } + + std::vector Y_dims; + for (std::size_t i = 0; i < dims.size(); i++) { + Y_dims.push_back(static_cast(scales_[i] * dims[i])); + } + Tensor* Y = context->Output(0, Y_dims); + + switch (mode_) { + case UpsampleMode::NN: + return upsampleNearest(X->template Data(), Y->template MutableData(), X->Shape(), Y->Shape(), scales_); + case UpsampleMode::LINEAR: { + //What's the correct behavior of linear mode is not clear right now, + //Only support bilinear with 4D tensor to keep consistent with previous behavior + if (dims.size() != 4) + return Status(ONNXRUNTIME, FAIL, "Upsample: linear mode upsample only support 4-D tensor with NCHW layout"); + + const int64_t batch_size = dims[0], num_channels = dims[1]; + const int64_t input_height = dims[2], input_width = dims[3]; + + upsampleBilinear(batch_size, num_channels, input_height, input_width, + scales_[2], scales_[3], X->template Data(), Y->template MutableData()); + return Status::OK(); + //return upsampleLiner(X->template Data(), Y->template MutableData(), X->Shape(), Y->Shape(), scales_); + } + default: + return Status(ONNXRUNTIME, FAIL, "Upsample: unexpected mode"); + } +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/upsample.h b/onnxruntime/core/providers/cpu/tensor/upsample.h new file mode 100644 index 0000000000000..3281da6e0ee73 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/upsample.h @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/framework/op_kernel.h" + +namespace onnxruntime { + +constexpr const char* UpsampleModeNN = "nearest"; +constexpr const char* UpsampleModeLinear = "linear"; + + + enum UpsampleMode { + NN = 0, // nearest neighbour + LINEAR = 1, // linear interpolation + }; + +class UpsampleBase { + protected: + UpsampleBase(OpKernelInfo info) { + std::string mode; + ONNXRUNTIME_ENFORCE(info.GetAttr("mode", &mode).IsOK()); + + mode_ = StringToUpsampleMode(mode); + + ONNXRUNTIME_ENFORCE(info.GetAttrs("scales", scales_).IsOK()); + for (auto& scale : scales_) { + ONNXRUNTIME_ENFORCE(scale >= 1, "Scale value should be greater than or equal to 1."); + } + + if (UpsampleMode::LINEAR == mode_) { + ONNXRUNTIME_ENFORCE(((scales_[0] == 1) && (scales_[1] == 1)), + "Upsample: linear mode upsample only support bilinear, the first 2 scales should be 1."); + } + } + + UpsampleMode mode_; + + std::vector scales_; + + UpsampleMode StringToUpsampleMode(const std::string& mode) { + if (strcmp(mode.c_str(), UpsampleModeNN) == 0) { + return UpsampleMode::NN; + } else if (strcmp(mode.c_str(), UpsampleModeLinear) == 0) { + return UpsampleMode::LINEAR; + } else { + ONNXRUNTIME_THROW("mode attribute is " + mode + ". It can only be " + + UpsampleModeNN + "(default) or " + UpsampleModeLinear + "."); + } + } +}; + +template +class Upsample : public UpsampleBase, public OpKernel { + public: + Upsample(OpKernelInfo info) : UpsampleBase(info), OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/utils.h b/onnxruntime/core/providers/cpu/tensor/utils.h new file mode 100644 index 0000000000000..5890412601239 --- /dev/null +++ b/onnxruntime/core/providers/cpu/tensor/utils.h @@ -0,0 +1,171 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "gsl/gsl_algorithm" +namespace onnxruntime { + +struct TensorPitches : std::vector { + TensorPitches(const Tensor& tensor, size_t rank = 0) : TensorPitches(tensor.Shape(), rank) {} + TensorPitches(const TensorShape& shape, size_t rank = 0) : TensorPitches(shape.GetDims(), rank) {} + TensorPitches(const std::vector& dims, size_t rank = 0) + : std::vector(std::max(rank, dims.size()), 0) { + Calculate(gsl::span(data(), size()), dims); + } + + static bool Calculate(gsl::span p, const std::vector& dims) { + // The pitches is the size of the next inner axis. Aka the amount to move by one of the next inner axis. + // For a tensor with shape(2,3,4,5) the values would be: (3*4*5, 4*5, 5, 1) + // Note that the outermost '2' is never used, as you never need to move by the entire size of the outermost axis + + auto tensor_rank = dims.size(); + auto pitch_rank = p.size(); + auto padded_rank = pitch_rank - tensor_rank; + if (gsl::narrow_cast(padded_rank) < 0) + return false; + + *(p.rbegin()) = 1; // The innermost axis is 1 (single values) + if (tensor_rank > 1) { + for (size_t i = tensor_rank - 1; i-- > 0;) { + p.operator[](i + padded_rank) = p.operator[](i + 1 + padded_rank) * dims[i + 1]; + } + } + + if (padded_rank >= 1) { + for (size_t i = 0; i < padded_rank; ++i) { + if (i == 0) + p.operator[](padded_rank - 1) = p.operator[](padded_rank) * dims[0]; + else + p.operator[](padded_rank - 1 - i) = p.operator[](padded_rank - 1); + } + } + return true; + } +}; + +// This class is to iterate through the axes of an arbitrarily shaped tensor +// For example, a tensor with shape (2,3,4) will be iterated in this order: +// (0,0,x) (0,1,x) (0,2,x) (1,0,x) (1,1,x) (1,2,x) +// Note: The innermost axis is not iterated over since it's always special cased +struct TensorAxisCounters { + TensorAxisCounters(const Tensor& tensor) : tensor_(tensor) { + indices_.resize(tensor_.Shape().NumDimensions() - 1, 0); + axis_ = indices_.size(); + + // If a tensor has a shape, but one of the axes is 0 in size, there are no elements, so nothing to iterate + if (tensor_.Shape().Size() == 0) + running_ = false; + } + + // Returns true if there was a carry to the next axis + bool Increment() { + if (axis_-- == 0) { + running_ = false; + return false; + } + + if (++indices_[axis_] != tensor_.Shape()[axis_]) { + axis_ = indices_.size(); + return false; + } + + indices_[axis_] = 0; // Reset the counter for this axis + return true; // There was a carry + } + + size_t Axis() const { return axis_; } + operator bool() const { return running_; } + + private: + const Tensor& tensor_; + bool running_{true}; + size_t axis_; + std::vector indices_; // There is no index for innermost axis since it's a special case +}; + +// A std::vector that holds the number of entries to skip to go to the next axis start given an extent in each axis +// This is used by the SliceIterator to iterate over a slice of a tensor +struct SliceSkips : std::vector { + SliceSkips(const Tensor& tensor, gsl::span extents) + : std::vector(tensor.Shape().NumDimensions(), 0) { + auto& dims = tensor.Shape().GetDims(); + ONNXRUNTIME_ENFORCE(static_cast(dims.size()) == extents.size()); + size_t pitch = dims.back(); + back() = pitch - extents[size() - 1]; + for (size_t i = size() - 1; i-- > 0;) { + auto prevPitch = pitch; + pitch *= dims[i]; + operator[](i) = pitch - prevPitch * extents[i]; + } + } +}; + +// This provides easy sequential iteration over a subset of a tensor given a span of starts & etents +template +struct SliceIterator { + SliceIterator(const Tensor& tensor, gsl::span starts, gsl::span extents) + : tensor_(tensor), extents_(extents), skips_(tensor, extents), indices_(extents.size(), 0) { + auto& dims = tensor_.Shape().GetDims(); + ONNXRUNTIME_ENFORCE(static_cast(dims.size()) == starts.size() && static_cast(dims.size()) == extents.size()); + + size_t pitch = 1; + // Initial skip, so that input_ points to the first element to copy + for (size_t i = dims.size(); i-- > 0;) { + input_ += pitch * starts[i]; + pitch *= dims[i]; + } + + inner_extent_ = extents_[dims.size() - 1]; + } + + void AdvanceOverInnerExtent() { + size_t axis = skips_.size() - 1; + input_ += skips_[axis]; + while (axis-- && ++indices_[axis] == extents_[axis]) { + indices_[axis] = 0; + input_ += skips_[axis]; + } + } + + const T* operator++(int) { + const T* input = input_++; + if (++inner_counter_ == inner_extent_) { + inner_counter_ = 0; + AdvanceOverInnerExtent(); + } + return input; + } + + T* CopyInnermostAxis(T* output) { + gsl::copy(gsl::make_span(input_, inner_extent_), gsl::make_span(output, inner_extent_)); + input_ += inner_extent_; + output += inner_extent_; + AdvanceOverInnerExtent(); + return output; + } + + private: + const Tensor& tensor_; + const T* input_{tensor_.template Data()}; + gsl::span extents_; + size_t inner_counter_{}, inner_extent_; + SliceSkips skips_; + std::vector indices_; // There is no index for innermost axis since it's a special case +}; + +inline void CopyCpuTensor(const Tensor* src, Tensor* tgt) { + void* target = tgt->MutableDataRaw(); + const void* source = src->DataRaw(); + + if (target != source) { + auto is_string_type = (src->DataType() == DataTypeImpl::GetType()); + if (is_string_type) { + for (int64_t i = 0; i < src->Shape().Size(); ++i) + static_cast(target)[i] = static_cast(source)[i]; + } else { + memcpy(target, source, src->Shape().Size() * src->DataType()->Size()); + } + } +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/activation/activations.cc b/onnxruntime/core/providers/cuda/activation/activations.cc new file mode 100644 index 0000000000000..cbdefe8e9905f --- /dev/null +++ b/onnxruntime/core/providers/cuda/activation/activations.cc @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "activations.h" + +namespace onnxruntime { +namespace cuda { + +#define REGISTER_ACTIVATION_KERNEL(x, ver, T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + x, \ + kOnnxDomain, \ + ver, \ + T, \ + kCudaExecutionProvider, \ + KernelDefBuilder() \ + .TypeConstraint("T", DataTypeImpl::GetTensorType()) \ + .MayInplace(0, 0), \ + x); + +#define UNARY_ACTIVATION_COMPUTE(x, T) \ + template <> \ + Status x::ComputeInternal(OpKernelContext* context) const { \ + UnaryElementwisePreparation p; \ + UnaryElementwise::Prepare(context, &p); \ + CudaAsyncBuffer func_ctx(this, 0, MakeFuncCtx()); \ + if (!std::is_same::value) \ + ONNXRUNTIME_RETURN_IF_ERROR(func_ctx.CopyToGpu()); \ + Impl_##x::MappedType>( \ + reinterpret_cast::MappedType*>(p.input_tensor->template Data()), \ + reinterpret_cast::MappedType*>(p.output_tensor->template MutableData()), \ + func_ctx.GpuPtr(), \ + p.output_tensor->Shape().Size()); \ + \ + return Status::OK(); \ + } + +#define UNARY_ACTIVATION_OP_TYPED(name, ver, T) \ + REGISTER_ACTIVATION_KERNEL(name, ver, T) \ + UNARY_ACTIVATION_COMPUTE(name, T) + +#define UNARY_ACTIVATION_OP_HFD(name, ver) \ + UNARY_ACTIVATION_OP_TYPED(name, ver, MLFloat16) \ + UNARY_ACTIVATION_OP_TYPED(name, ver, float) \ + UNARY_ACTIVATION_OP_TYPED(name, ver, double) + +UNARY_ACTIVATION_OP_HFD(Affine, 1); +UNARY_ACTIVATION_OP_HFD(Elu, 6); +UNARY_ACTIVATION_OP_HFD(HardSigmoid, 6); +UNARY_ACTIVATION_OP_HFD(LeakyRelu, 6); +UNARY_ACTIVATION_OP_HFD(ParametricSoftplus, 1); +UNARY_ACTIVATION_OP_HFD(Relu, 6); +UNARY_ACTIVATION_OP_HFD(ScaledTanh, 1); +UNARY_ACTIVATION_OP_HFD(Selu, 6); +UNARY_ACTIVATION_OP_HFD(Sigmoid, 6); +UNARY_ACTIVATION_OP_HFD(Softplus, 1); +UNARY_ACTIVATION_OP_HFD(Softsign, 1); +UNARY_ACTIVATION_OP_HFD(Tanh, 6); +UNARY_ACTIVATION_OP_HFD(ThresholdedRelu, 1); + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/activation/activations.h b/onnxruntime/core/providers/cuda/activation/activations.h new file mode 100644 index 0000000000000..55d1eac1ae5c3 --- /dev/null +++ b/onnxruntime/core/providers/cuda/activation/activations.h @@ -0,0 +1,228 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/providers/cuda/cuda_common.h" +#include "core/providers/cuda/math/unary_elementwise_ops.h" +#include "core/providers/cuda/math/binary_elementwise_ops.h" +#include "activations_impl.h" + +namespace onnxruntime { +namespace cuda { + +#define MAKE_FUNC_CTX_ALPHA() \ + inline CtxAlpha MakeFuncCtx() const { \ + CtxAlpha ctx; \ + ctx.alpha = alpha_; \ + return ctx; \ + } + +#define MAKE_FUNC_CTX_ALPHA_BETA() \ + inline CtxAlphaBeta MakeFuncCtx() const { \ + CtxAlphaBeta ctx; \ + ctx.alpha = alpha_; \ + ctx.beta = beta_; \ + return ctx; \ + } + +#define MAKE_FUNC_CTX_ALPHA_GAMMA() \ + inline CtxAlphaGamma MakeFuncCtx() const { \ + CtxAlphaGamma ctx; \ + ctx.alpha = alpha_; \ + ctx.gamma = gamma_; \ + return ctx; \ + } + +#define MAKE_FUNC_CTX_NULL() \ + inline CtxNull MakeFuncCtx() const { \ + CtxNull ctx; \ + return ctx; \ + } + +template +class Affine final : public UnaryElementwise { + public: + Affine(const OpKernelInfo& info) : UnaryElementwise(info) { + ONNXRUNTIME_ENFORCE(info.GetAttr("alpha", &alpha_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttr("beta", &beta_).IsOK()); + } + + Status ComputeInternal(OpKernelContext* context) const override; + + private: + MAKE_FUNC_CTX_ALPHA_BETA() + + float alpha_; + float beta_; +}; + +template +class Elu final : public UnaryElementwise { + public: + Elu(const OpKernelInfo& info) : UnaryElementwise(info) { + ONNXRUNTIME_ENFORCE(info.GetAttr("alpha", &alpha_).IsOK()); + } + + Status ComputeInternal(OpKernelContext* context) const override; + + private: + MAKE_FUNC_CTX_ALPHA() + + float alpha_; +}; + +template +class HardSigmoid final : public UnaryElementwise { + public: + HardSigmoid(const OpKernelInfo& info) : UnaryElementwise(info) { + ONNXRUNTIME_ENFORCE(info.GetAttr("alpha", &alpha_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttr("beta", &beta_).IsOK()); + } + + Status ComputeInternal(OpKernelContext* context) const override; + + private: + MAKE_FUNC_CTX_ALPHA_BETA() + + float alpha_; + float beta_; +}; + +template +class LeakyRelu final : public UnaryElementwise { + public: + LeakyRelu(const OpKernelInfo& info) : UnaryElementwise(info) { + ONNXRUNTIME_ENFORCE(info.GetAttr("alpha", &alpha_).IsOK()); + } + + Status ComputeInternal(OpKernelContext* context) const override; + + private: + MAKE_FUNC_CTX_ALPHA() + + float alpha_; +}; + +template +class ParametricSoftplus final : public UnaryElementwise { + public: + ParametricSoftplus(const OpKernelInfo& info) : UnaryElementwise(info) { + ONNXRUNTIME_ENFORCE(info.GetAttr("alpha", &alpha_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttr("beta", &beta_).IsOK()); + } + + Status ComputeInternal(OpKernelContext* context) const override; + + private: + MAKE_FUNC_CTX_ALPHA_BETA() + + float alpha_; + float beta_; +}; + +template +class Relu final : public UnaryElementwise { + public: + Relu(const OpKernelInfo& info) : UnaryElementwise(info) {} + + Status ComputeInternal(OpKernelContext* context) const override; + + private: + MAKE_FUNC_CTX_NULL() +}; + +template +class ScaledTanh final : public UnaryElementwise { + public: + ScaledTanh(const OpKernelInfo& info) : UnaryElementwise(info) { + ONNXRUNTIME_ENFORCE(info.GetAttr("alpha", &alpha_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttr("beta", &beta_).IsOK()); + } + + Status ComputeInternal(OpKernelContext* context) const override; + + private: + MAKE_FUNC_CTX_ALPHA_BETA() + + float alpha_; + float beta_; +}; + +template +class Selu final : public UnaryElementwise { + public: + Selu(const OpKernelInfo& info) : UnaryElementwise(info) { + ONNXRUNTIME_ENFORCE(info.GetAttr("alpha", &alpha_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttr("gamma", &gamma_).IsOK()); + } + + Status ComputeInternal(OpKernelContext* context) const override; + + private: + MAKE_FUNC_CTX_ALPHA_GAMMA() + + float alpha_; + float gamma_; +}; + +template +class Sigmoid final : public UnaryElementwise { + public: + Sigmoid(const OpKernelInfo& info) : UnaryElementwise(info) {} + + Status ComputeInternal(OpKernelContext* context) const override; + + private: + MAKE_FUNC_CTX_NULL() +}; + +template +class Softplus final : public UnaryElementwise { + public: + Softplus(const OpKernelInfo& info) : UnaryElementwise(info) {} + + Status ComputeInternal(OpKernelContext* context) const override; + + private: + MAKE_FUNC_CTX_NULL() +}; + +template +class Softsign final : public UnaryElementwise { + public: + Softsign(const OpKernelInfo& info) : UnaryElementwise(info) {} + + Status ComputeInternal(OpKernelContext* context) const override; + + private: + MAKE_FUNC_CTX_NULL() +}; + +template +class Tanh final : public UnaryElementwise { + public: + Tanh(const OpKernelInfo& info) : UnaryElementwise(info) {} + + Status ComputeInternal(OpKernelContext* context) const override; + + private: + MAKE_FUNC_CTX_NULL() +}; + +template +class ThresholdedRelu final : public UnaryElementwise { + public: + ThresholdedRelu(const OpKernelInfo& info) : UnaryElementwise(info) { + ONNXRUNTIME_ENFORCE(info.GetAttr("alpha", &alpha_).IsOK()); + } + + Status ComputeInternal(OpKernelContext* context) const override; + + private: + MAKE_FUNC_CTX_ALPHA() + float alpha_; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/activation/activations_impl.cu b/onnxruntime/core/providers/cuda/activation/activations_impl.cu new file mode 100644 index 0000000000000..173ba6a69abbf --- /dev/null +++ b/onnxruntime/core/providers/cuda/activation/activations_impl.cu @@ -0,0 +1,133 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include "activations_impl.h" +#include "core/providers/cuda/cu_inc/common.cuh" +#include "core/providers/cuda/cu_inc/unary_elementwise_impl.cuh" + +namespace onnxruntime { +namespace cuda { + +template +struct OP_Affine : public CtxAffine { + __device__ __inline__ T operator()(const T& a) const { + return a * (T)alpha + (T)beta; + } +}; + +template +struct OP_Elu : public CtxElu { + __device__ __inline__ T operator()(const T& a) const { + return a > (T)0 ? a : (T)alpha * (_Exp(a) - (T)1); + } +}; + +template +struct OP_HardSigmoid : public CtxHardSigmoid { + __device__ __inline__ T operator()(const T& a) const { + return _Max(_Min((T)alpha * a + (T)beta, (T)1), (T)0); + } +}; + +template +struct OP_LeakyRelu : public CtxLeakyRelu { + __device__ __inline__ T operator()(const T& a) const { + return a > (T)0 ? a : (T)alpha * a; + } +}; + +template +struct OP_ParametricSoftplus : public CtxParametricSoftplus { + __device__ __inline__ T operator()(const T& a) const { + if (a > (T)0) + return (T)alpha * (a * (T)beta + _Log(_Exp(-a * (T)beta) + (T)1)); + else + return (T)alpha * _Log(_Exp(a * (T)beta) + (T)1); + } +}; + +template +struct OP_Relu : public CtxRelu { + __device__ __inline__ T operator()(const T& a) const { + return _Max(a, (T)0); + } +}; + +template +struct OP_ScaledTanh : public CtxScaledTanh { + __device__ __inline__ T operator()(const T& a) const { + return (T)alpha * _Tanh(a * (T)beta); + } +}; + +template +struct OP_Selu : public CtxSelu { + __device__ __inline__ T operator()(const T& a) const { + return (T)gamma * (_Max(a, (T)0) + _Min((T)alpha * (_Exp(a) - (T)1), (T)0)); + } +}; + +template +struct OP_Sigmoid : public CtxSigmoid { + __device__ __inline__ T operator()(const T& a) const { + return a > T(0) ? (T)1 / ((T)1. + _Exp(-_Abs(a))) : (T)1 - (T)1 / ((T)1 + _Exp(-_Abs(a))); + } +}; + +template +struct OP_Softplus : public CtxSoftplus { + __device__ __inline__ T operator()(const T& a) const { + if (a > (T)0) + return a + _Log(_Exp(-a) + (T)1); + else + return _Log(_Exp(a) + (T)1); + } +}; + +template +struct OP_Softsign : public CtxSoftsign { + __device__ __inline__ T operator()(const T& a) const { + return a / ((T)1. + _Abs(a)); + } +}; + +template +struct OP_Tanh : public CtxTanh { + __device__ __inline__ T operator()(const T& a) const { + return _Tanh(a); + } +}; + +template +struct OP_ThresholdedRelu : public CtxThresholdedRelu { + __device__ __inline__ T operator()(const T& a) const { + return a > (T)alpha ? a : (T)0; + } +}; + +#define UNARY_ACTIVATION_IMPL(name) \ + UNARY_ACTIVATION_IMPL_DECLARATION(name) { \ + UnaryElementWiseImpl(input_data, \ + output_data, \ + *reinterpret_cast*>(func_ctx), \ + count); \ + } + +#define SPECIALIZED_UNARY_ACTIVATION_IMPL(name, T) \ + template void Impl_##name(const T* input_data, T* output_data, const Ctx##name* func_ctx, size_t count); + +#define SPECIALIZED_UNARY_ACTIVATIONL_HFD(name) \ + SPECIALIZED_UNARY_ACTIVATION_IMPL(name, half) \ + SPECIALIZED_UNARY_ACTIVATION_IMPL(name, float) \ + SPECIALIZED_UNARY_ACTIVATION_IMPL(name, double) + +#define UNARY_ACTIVATION_OP_NAME(name) \ + UNARY_ACTIVATION_IMPL(name); \ + SPECIALIZED_UNARY_ACTIVATIONL_HFD(name) + +UNARY_ACTIVATION_OPS() +#undef UNARY_ACTIVATION_OP_NAME + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/activation/activations_impl.h b/onnxruntime/core/providers/cuda/activation/activations_impl.h new file mode 100644 index 0000000000000..90c0281e8c526 --- /dev/null +++ b/onnxruntime/core/providers/cuda/activation/activations_impl.h @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +namespace onnxruntime { +namespace cuda { + +struct CtxAlpha { + float alpha; +}; + +struct CtxAlphaBeta { + float alpha; + float beta; +}; + +struct CtxAlphaGamma { + float alpha; + float gamma; +}; + +struct CtxNull { +}; + +typedef CtxAlphaBeta CtxAffine; +typedef CtxAlpha CtxElu; +typedef CtxAlphaBeta CtxHardSigmoid; +typedef CtxAlpha CtxLeakyRelu; +typedef CtxAlphaBeta CtxParametricSoftplus; +typedef CtxNull CtxRelu; +typedef CtxAlphaBeta CtxScaledTanh; +typedef CtxAlphaGamma CtxSelu; +typedef CtxNull CtxSigmoid; +typedef CtxNull CtxSoftplus; +typedef CtxNull CtxSoftsign; +typedef CtxNull CtxTanh; +typedef CtxAlpha CtxThresholdedRelu; + +#define UNARY_ACTIVATION_OPS() \ + UNARY_ACTIVATION_OP_NAME(Affine) \ + UNARY_ACTIVATION_OP_NAME(Elu) \ + UNARY_ACTIVATION_OP_NAME(HardSigmoid) \ + UNARY_ACTIVATION_OP_NAME(LeakyRelu) \ + UNARY_ACTIVATION_OP_NAME(ParametricSoftplus) \ + UNARY_ACTIVATION_OP_NAME(Relu) \ + UNARY_ACTIVATION_OP_NAME(ScaledTanh) \ + UNARY_ACTIVATION_OP_NAME(Selu) \ + UNARY_ACTIVATION_OP_NAME(Sigmoid) \ + UNARY_ACTIVATION_OP_NAME(Softplus) \ + UNARY_ACTIVATION_OP_NAME(Softsign) \ + UNARY_ACTIVATION_OP_NAME(Tanh) \ + UNARY_ACTIVATION_OP_NAME(ThresholdedRelu) + +#define UNARY_ACTIVATION_IMPL_DECLARATION(name) \ + template \ + void Impl_##name( \ + const T* input_data, \ + T* output_data, \ + const Ctx##name* func_ctx, \ + size_t count) + +#define UNARY_ACTIVATION_OP_NAME(name) UNARY_ACTIVATION_IMPL_DECLARATION(name); +UNARY_ACTIVATION_OPS() +#undef UNARY_ACTIVATION_OP_NAME + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cu_inc/binary_elementwise_impl.cuh b/onnxruntime/core/providers/cuda/cu_inc/binary_elementwise_impl.cuh new file mode 100644 index 0000000000000..e40eb8e967cdc --- /dev/null +++ b/onnxruntime/core/providers/cuda/cu_inc/binary_elementwise_impl.cuh @@ -0,0 +1,199 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include "core/providers/cuda/shared_inc/cuda_utils.h" +#include "common.cuh" + +namespace onnxruntime { +namespace cuda { + +// broadcast by computing output coordinate from offset, using fast_divmod +template +__global__ void _BinaryElementWise( + size_t output_rank, + const int64_t* lhs_padded_strides, + const T* lhs_data, + const int64_t* rhs_padded_strides, + const T* rhs_data, + const fast_divmod* fdm_output_strides, + T* output_data, + const FuncT& functor, + CUDA_LONG N) { + CALCULATE_ELEMENTWISE_INDEX_OR_EXIT(id, N); + CUDA_LONG lhs_index = (lhs_need_compute ? 0 : id); + CUDA_LONG rhs_index = (rhs_need_compute ? 0 : id); + // compute indexes with broadcasting rules: https://github.com/onnx/onnx/blob/master/docs/Broadcasting.md + CUDA_LONG offset = id; + for (int dim = 0; dim < output_rank; dim++) { + int q, r; + fdm_output_strides[dim].divmod(offset, q, r); + // compute index increase based on stride and broadcast + // note that stride[i-1] == stride[i] means dim[i] is 1 (broadcasting) + if (lhs_need_compute) { + if (lhs_padded_strides[dim] != lhs_padded_strides[dim + 1]) + lhs_index += static_cast(lhs_padded_strides[dim + 1]) * q; + } + + if (rhs_need_compute) { + if (rhs_padded_strides[dim] != rhs_padded_strides[dim + 1]) + rhs_index += static_cast(rhs_padded_strides[dim + 1]) * q; + } + offset = r; + } + output_data[id] = functor(lhs_data[lhs_index], rhs_data[rhs_index]); +} + +// for scalar broadcast or non-broadcast case +template +__global__ void _BinaryElementWiseSimple( + const T* lhs_data, + const T* rhs_data, + T* output_data, + FuncT func, + CUDA_LONG N) { + CALCULATE_ELEMENTWISE_INDEX_OR_EXIT(id, N); + output_data[id] = func(lhs_data[IncL ? id : 0], rhs_data[IncR ? id : 0]); +} + +// for rhs per-channel broadcast case +template +__global__ void _BinaryElementWiseRhsPerChannelBatch1( + const T* lhs_data, + const T* rhs_data, + const fast_divmod fdm_H, + T* output_data, + FuncT func, + CUDA_LONG N) { + CALCULATE_ELEMENTWISE_INDEX_OR_EXIT(id, N); + CUDA_LONG rhs_id = fdm_H.div(id); + output_data[id] = func(lhs_data[id], rhs_data[rhs_id]); +} + +template +__global__ void _BinaryElementWiseRhsPerChannelBatchN( + const T* lhs_data, + const T* rhs_data, + const fast_divmod fdm_H, + const fast_divmod fdm_C, + T* output_data, + FuncT func, + CUDA_LONG N) { + CALCULATE_ELEMENTWISE_INDEX_OR_EXIT(id, N); + CUDA_LONG rhs_id = fdm_H.div(id); + int q, r; + fdm_C.divmod(rhs_id, q, r); + rhs_id = r; + output_data[id] = func(lhs_data[id], rhs_data[rhs_id]); +} + +template +void BinaryElementWiseNoBroadcastImpl( + const T* lhs_data, + const T* rhs_data, + T* output_data, + const FuncT& func, + size_t count) { + int blocksPerGrid = (int)(ceil(static_cast(count) / GridDim::maxThreadsPerBlock)); + CUDA_LONG N = static_cast(count); + _BinaryElementWiseSimple<<>>( + lhs_data, + rhs_data, + output_data, + func, + N); +} + +template +void BinaryElementWiseImpl( + size_t output_rank_or_simple_broadcast, + const int64_t* lhs_padded_strides, + const T* lhs_data, + const int64_t* rhs_padded_strides, + const T* rhs_data, + const fast_divmod* fdm_output_strides, + const fast_divmod& fdm_H, + const fast_divmod& fdm_C, + T* output_data, + const FuncT& func, + size_t count) { + int blocksPerGrid = (int)(ceil(static_cast(count) / GridDim::maxThreadsPerBlock)); + CUDA_LONG N = static_cast(count); + if (output_rank_or_simple_broadcast == static_cast(SimpleBroadcast::NoBroadcast)) { + _BinaryElementWiseSimple<<>>( + lhs_data, + rhs_data, + output_data, + func, + N); + } else if (output_rank_or_simple_broadcast == static_cast(SimpleBroadcast::LeftScalar)) { + _BinaryElementWiseSimple<<>>( + lhs_data, + rhs_data, + output_data, + func, + N); + } else if (output_rank_or_simple_broadcast == static_cast(SimpleBroadcast::RightScalar)) { + _BinaryElementWiseSimple<<>>( + lhs_data, + rhs_data, + output_data, + func, + N); + } else if (output_rank_or_simple_broadcast == static_cast(SimpleBroadcast::RightPerChannelBatch1)) { + _BinaryElementWiseRhsPerChannelBatch1<<>>( + lhs_data, + rhs_data, + fdm_H, + output_data, + func, + N); + } else if (output_rank_or_simple_broadcast == static_cast(SimpleBroadcast::RightPerChannelBatchN)) { + _BinaryElementWiseRhsPerChannelBatchN<<>>( + lhs_data, + rhs_data, + fdm_H, + fdm_C, + output_data, + func, + N); + } else { + if (lhs_padded_strides && rhs_padded_strides) + _BinaryElementWise<<>>( + output_rank_or_simple_broadcast, + lhs_padded_strides, + lhs_data, + rhs_padded_strides, + rhs_data, + fdm_output_strides, + output_data, + func, + N); + else if (lhs_padded_strides) + _BinaryElementWise<<>>( + output_rank_or_simple_broadcast, + lhs_padded_strides, + lhs_data, + rhs_padded_strides, + rhs_data, + fdm_output_strides, + output_data, + func, + N); + else + _BinaryElementWise<<>>( + output_rank_or_simple_broadcast, + lhs_padded_strides, + lhs_data, + rhs_padded_strides, + rhs_data, + fdm_output_strides, + output_data, + func, + N); + } +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cu_inc/common.cuh b/onnxruntime/core/providers/cuda/cu_inc/common.cuh new file mode 100644 index 0000000000000..750c7416d8e21 --- /dev/null +++ b/onnxruntime/core/providers/cuda/cu_inc/common.cuh @@ -0,0 +1,267 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include +#include +#include +#include +#include "core/providers/cuda/shared_inc/cuda_call.h" + +namespace onnxruntime { +namespace cuda { + +// float16 arithmetic is supported after sm5.3 with intrinsics, and cuda does not provide fallback for lower versions +#if __CUDA_ARCH__ < 530 +__device__ __forceinline__ half operator+(const half& lh, const half& rh) { return half((float)lh + (float)rh); } +__device__ __forceinline__ half operator-(const half& lh, const half& rh) { return half((float)lh - (float)rh); } +__device__ __forceinline__ half operator*(const half& lh, const half& rh) { return half((float)lh * (float)rh); } +__device__ __forceinline__ half operator/(const half& lh, const half& rh) { return half((float)lh / (float)rh); } + +__device__ __forceinline__ half& operator+=(half& lh, const half& rh) { + lh = half((float)lh + (float)rh); + return lh; +} +__device__ __forceinline__ half& operator-=(half& lh, const half& rh) { + lh = half((float)lh - (float)rh); + return lh; +} +__device__ __forceinline__ half& operator*=(half& lh, const half& rh) { + lh = half((float)lh * (float)rh); + return lh; +} +__device__ __forceinline__ half& operator/=(half& lh, const half& rh) { + lh = half((float)lh / (float)rh); + return lh; +} + +/* Note for increment and decrement we use the raw value 0x3C00 equating to half(1.0f), to avoid the extra conversion */ +__device__ __forceinline__ __half& operator++(__half& h) { + h = half((float)h + 1.0f); + return h; +} +__device__ __forceinline__ __half& operator--(__half& h) { + h = half((float)h - 1.0f); + return h; +} +__device__ __forceinline__ __half operator++(__half& h, int) { + half ret = h; + h = half((float)h + 1); + return ret; +} +__device__ __forceinline__ __half operator--(__half& h, int) { + half ret = h; + h = half((float)h - 1); + return ret; +} + +/* Unary plus and inverse operators */ +__device__ __forceinline__ half operator+(const half& h) { return h; } +__device__ __forceinline__ half operator-(const half& h) { return half(-(float)h); } + +/* Some basic comparison operations to make it look like a builtin */ +__device__ __forceinline__ bool operator==(const half& lh, const half& rh) { return (float)lh == (float)rh; } +__device__ __forceinline__ bool operator!=(const half& lh, const half& rh) { return (float)lh != (float)rh; } +__device__ __forceinline__ bool operator>(const half& lh, const half& rh) { return (float)lh > (float)rh; } +__device__ __forceinline__ bool operator<(const half& lh, const half& rh) { return (float)lh < (float)rh; } +__device__ __forceinline__ bool operator>=(const half& lh, const half& rh) { return (float)lh >= (float)rh; } +__device__ __forceinline__ bool operator<=(const half& lh, const half& rh) { return (float)lh <= (float)rh; } +#endif + +template +__device__ __inline__ T _Ceil(T a); + +template <> +__device__ __inline__ float _Ceil(float a) { return ceilf(a); } + +template <> +__device__ __inline__ double _Ceil(double a) { return ceil(a); } + +template <> +__device__ __inline__ half _Ceil(half a) { return half(ceilf((float)a)); } + +template +__device__ __inline__ T _Floor(T a); + +template <> +__device__ __inline__ float _Floor(float a) { return floorf(a); } + +template <> +__device__ __inline__ double _Floor(double a) { return floor(a); } + +template <> +__device__ __inline__ half _Floor(half a) { return half(floorf((float)a)); } + +template +__device__ __inline__ T _Sqrt(T a); + +template <> +__device__ __inline__ float _Sqrt(float a) { return sqrtf(a); } + +template <> +__device__ __inline__ double _Sqrt(double a) { return sqrt(a); } + +template <> +__device__ __inline__ half _Sqrt(half a) { return half(sqrtf((float)a)); } + +template +__device__ __inline__ T _Exp(T a); + +template <> +__device__ __inline__ float _Exp(float a) { return expf(a); } + +template <> +__device__ __inline__ double _Exp(double a) { return exp(a); } + +template <> +__device__ __inline__ half _Exp(half a) { return half(expf((float)a)); } + +template +__device__ __inline__ T _Log(T a); + +template <> +__device__ __inline__ float _Log(float a) { return logf(a); } + +template <> +__device__ __inline__ double _Log(double a) { return log(a); } + +template <> +__device__ __inline__ half _Log(half a) { return half(logf((float)a)); } + +template +__device__ __inline T _Tanh(T a); + +template <> +__device__ __inline__ float _Tanh(float a) { return tanhf(a); } + +template <> +__device__ __inline__ double _Tanh(double a) { return tanh(a); } + +template <> +__device__ __inline__ half _Tanh(half a) { return half(tanhf((float)a)); } + +template +__device__ __inline__ T _Pow(T a, T b); + +template <> +__device__ __inline__ float _Pow(float a, float b) { return powf(a, b); } + +template <> +__device__ __inline__ double _Pow(double a, double b) { return pow(a, b); } + +template <> +__device__ __inline__ half _Pow(half a, half b) { return half(powf((float)a, (float)b)); } + +template +__device__ __inline__ T _Min(T a, T b) { return a < b ? a : b; } + +template +__device__ __inline__ T _Max(T a, T b) { return a > b ? a : b; } + +template +__device__ __inline__ T _Abs(T a) { return a > (T)0 ? a : -a; } + +// We would like to use 64-bit integer to support large matrices. However, CUDA seems to support only 32-bit integer +// For now, use int32_t to ensure that both Linux and Windows see this as 32 bit integer type. + +#ifndef CUDA_LONG +#define CUDA_LONG int32_t +#endif + +#define IDX2C(i, j, ld) (((j) * (ld)) + (i)) // 0 based indexing + +// --------------------------------------------------------------------------- +// GridDim -- helper to choose the CUDA grid dimensions +// --------------------------------------------------------------------------- + +template +static INT CeilDiv(INT a, INT2 b) // ceil(a/b) +{ + return (INT)(((size_t)a + (size_t)b - 1) / (size_t)b); // these size_t casts are necessary since b may be INT_MAX (for maxGridSize[]) +} + +struct GridDim { + enum : CUDA_LONG { + maxThreadsPerBlock = 1024, // use this many threads per block + maxWarpsPerBlock = 32, // use this many warps per block. This means 1024 threads for warpSize=32 + }; + + // use these for launching + // GridDim grid(NN); + // kernel<<>>(...) + int blocks_per_grid_, threads_per_block_; // (these may in the future be extended to multi-dimensional ones) + CUDA_LONG N_; + + GridDim(CUDA_LONG N) // linear grid + { + N_ = N; + if (N == 0) // CUDA will fail to launch with 0 blocks + N = 1; + + // get device information + const auto& props = GetDeviceProps(); + CUDA_LONG numProcs = props.multiProcessorCount; + CUDA_LONG warpSize = props.warpSize; + + // distribute warps evenly over processors + CUDA_LONG warpsPerProc = CeilDiv(N, numProcs * warpSize); + + // if too many warps per block then reduce #warps + // This limits the number of threads to 512. + if (warpsPerProc > maxWarpsPerBlock) { + CUDA_LONG overBy = CeilDiv(warpsPerProc, maxWarpsPerBlock); // we are over by this factor + warpsPerProc = CeilDiv(warpsPerProc, overBy); + } + + // put it back together + threads_per_block_ = warpsPerProc * warpSize; // =a multiple of 32 that is as close to 1024 as makes sense given NN + blocks_per_grid_ = CeilDiv(N, threads_per_block_); + if (blocks_per_grid_ == 1) + threads_per_block_ = N; // don't launch more than necessary + assert(blocks_per_grid_ * threads_per_block_ >= N); + } + + static const std::vector& GetCachedDeviceProps() { + std::call_once(s_cachedDevicePropsInitFlag, [=] { + int numDevices; + // must wait GPU idle, otherwise cudaGetDeviceProperties might fail + CUDA_CALL_THROW(cudaDeviceSynchronize()); + CUDA_CALL_THROW(cudaGetDeviceCount(&numDevices)); + s_cachedDeviceProps.resize(numDevices); + for (int i = 0; i < numDevices; i++) + CUDA_CALL_THROW(cudaGetDeviceProperties(&s_cachedDeviceProps[i], i)); + }); + + return s_cachedDeviceProps; + } + + static size_t GetCurrentDeviceId() { + int deviceId; + cudaGetDevice(&deviceId); + return (size_t)deviceId; + } + + // get device properties of current device + static const cudaDeviceProp& GetDeviceProps() { + const auto& cachedDevicesProps = GetCachedDeviceProps(); + return cachedDevicesProps[GetCurrentDeviceId()]; + } + + // compute our location on the grid + static __device__ CUDA_LONG GetLinearThreadId() { + return blockDim.x * blockIdx.x + threadIdx.x; + } + + private: + static std::vector s_cachedDeviceProps; + static std::once_flag s_cachedDevicePropsInitFlag; +}; + +#define CALCULATE_ELEMENTWISE_INDEX_OR_EXIT(id, N) \ + CUDA_LONG id = GridDim::GetLinearThreadId(); \ + if (id >= N) \ + return; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cu_inc/unary_elementwise_impl.cuh b/onnxruntime/core/providers/cuda/cu_inc/unary_elementwise_impl.cuh new file mode 100644 index 0000000000000..50c8b73128bf1 --- /dev/null +++ b/onnxruntime/core/providers/cuda/cu_inc/unary_elementwise_impl.cuh @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include "core/providers/cuda/shared_inc/cuda_utils.h" +#include "common.cuh" + +namespace onnxruntime { +namespace cuda { + +template +__global__ void _UnaryElementWise( + const InT* input_data, + OutT* output_data, + const FuncT& functor, + CUDA_LONG N) { + CALCULATE_ELEMENTWISE_INDEX_OR_EXIT(id, N); + output_data[id] = functor(input_data[id]); +} + +template +void UnaryElementWiseImpl( + const InT* input_data, + OutT* output_data, + const FuncT& func, + size_t count) { + int blocksPerGrid = (int)(ceil(static_cast(count) / GridDim::maxThreadsPerBlock)); + CUDA_LONG N = static_cast(count); + _UnaryElementWise<<>>( + input_data, + output_data, + func, + N); +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cuda_allocator.cc b/onnxruntime/core/providers/cuda/cuda_allocator.cc new file mode 100644 index 0000000000000..5b0ca35924502 --- /dev/null +++ b/onnxruntime/core/providers/cuda/cuda_allocator.cc @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "cuda_common.h" +#include "cuda_allocator.h" +#include "core/framework/allocatormgr.h" +#include "core/framework/session_state.h" +#include "cuda_fence.h" + +namespace onnxruntime { + +static const CUDAExecutionProvider* GetCUDAExecutionProvider(const SessionState* session_state) { + return dynamic_cast( + session_state->GetExecutionProviders().Get(onnxruntime::kCudaExecutionProvider)); +} + +void CUDAAllocator::CheckDevice() const { +#ifndef NDEBUG + // check device to match at debug build + // if it's expected to change, call cudaSetDevice instead of the check + int current_device; + CUDA_CALL_THROW(cudaGetDevice(¤t_device)); + ONNXRUNTIME_ENFORCE(current_device == device_id_); +#endif +} + +void* CUDAAllocator::Alloc(size_t size) { + CheckDevice(); + void* p = nullptr; + if (size > 0) { + CUDA_CALL_THROW(cudaMalloc((void**)&p, size)); + } + return p; +} + +void CUDAAllocator::Free(void* p) { + CheckDevice(); + cudaFree(p); // do not throw error since it's OK for cudaFree to fail during shutdown +} + +const ONNXRuntimeAllocatorInfo& CUDAAllocator::Info() const { + return info_; +} + +FencePtr CUDAAllocator::CreateFence(const SessionState* session_state) { + return std::make_shared(GetCUDAExecutionProvider(session_state)); +} + +void* CUDAPinnedAllocator::Alloc(size_t size) { + void* p = nullptr; + if (size > 0) { + CUDA_CALL_THROW(cudaMallocHost((void**)&p, size)); + } + return p; +} + +void CUDAPinnedAllocator::Free(void* p) { + CUDA_CALL_THROW(cudaFreeHost(p)); +} + +const ONNXRuntimeAllocatorInfo& CUDAPinnedAllocator::Info() const { + static constexpr ONNXRuntimeAllocatorInfo cuda_allocator_info(CUDA_PINNED, ONNXRuntimeDeviceAllocator, 0, ONNXRuntimeMemTypeCPUOutput); + return cuda_allocator_info; +} + +FencePtr CUDAPinnedAllocator::CreateFence(const SessionState* session_state) { + return std::make_shared(GetCUDAExecutionProvider(session_state)); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cuda_allocator.h b/onnxruntime/core/providers/cuda/cuda_allocator.h new file mode 100644 index 0000000000000..0cbb4af7367a8 --- /dev/null +++ b/onnxruntime/core/providers/cuda/cuda_allocator.h @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/framework/allocator.h" + +namespace onnxruntime { +constexpr const char* CUDA = "Cuda"; +constexpr const char* CUDA_PINNED = "CudaPinned"; + +class CUDAAllocator : public IDeviceAllocator { + public: + CUDAAllocator(int device_id) : device_id_(device_id), info_(CUDA, ONNXRuntimeAllocatorType::ONNXRuntimeDeviceAllocator, device_id, ONNXRuntimeMemTypeDefault) {} + virtual void* Alloc(size_t size) override; + virtual void Free(void* p) override; + virtual const ONNXRuntimeAllocatorInfo& Info() const override; + virtual FencePtr CreateFence(const SessionState* session_state) override; + + private: + void CheckDevice() const; + + private: + const int device_id_; + const ONNXRuntimeAllocatorInfo info_; +}; + +//TODO: add a default constructor +class CUDAPinnedAllocator : public IDeviceAllocator { + public: + virtual void* Alloc(size_t size) override; + virtual void Free(void* p) override; + virtual const ONNXRuntimeAllocatorInfo& Info() const override; + virtual FencePtr CreateFence(const SessionState* session_state) override; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cuda_call.cc b/onnxruntime/core/providers/cuda/cuda_call.cc new file mode 100644 index 0000000000000..d5a4b03e384d9 --- /dev/null +++ b/onnxruntime/core/providers/cuda/cuda_call.cc @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "shared_inc/cuda_call.h" +#include "core/common/common.h" +#include "core/common/status.h" +#include "core/common/logging/logging.h" + +#ifdef _WIN32 +#else // POSIX +#include +#include +#endif + +namespace onnxruntime { + +using namespace common; + +template +const char* CudaErrString(ERRTYPE x) { + ONNXRUNTIME_NOT_IMPLEMENTED(); +} + +#define CASE_ENUM_TO_STR(x) \ + case x: \ + return #x + +template <> +const char* CudaErrString(cudaError_t x) { + cudaDeviceSynchronize(); + return cudaGetErrorString(x); +} +template <> +const char* CudaErrString(cublasStatus_t e) { + cudaDeviceSynchronize(); + + switch (e) { + CASE_ENUM_TO_STR(CUBLAS_STATUS_SUCCESS); + CASE_ENUM_TO_STR(CUBLAS_STATUS_NOT_INITIALIZED); + CASE_ENUM_TO_STR(CUBLAS_STATUS_ALLOC_FAILED); + CASE_ENUM_TO_STR(CUBLAS_STATUS_INVALID_VALUE); + CASE_ENUM_TO_STR(CUBLAS_STATUS_ARCH_MISMATCH); + CASE_ENUM_TO_STR(CUBLAS_STATUS_MAPPING_ERROR); + CASE_ENUM_TO_STR(CUBLAS_STATUS_EXECUTION_FAILED); + CASE_ENUM_TO_STR(CUBLAS_STATUS_INTERNAL_ERROR); + CASE_ENUM_TO_STR(CUBLAS_STATUS_NOT_SUPPORTED); + CASE_ENUM_TO_STR(CUBLAS_STATUS_LICENSE_ERROR); + default: + return "(look for CUBLAS_STATUS_xxx in cublas_api.h)"; + } +} + +template <> +const char* CudaErrString(curandStatus) { + cudaDeviceSynchronize(); + return "(see curand.h & look for curandStatus or CURAND_STATUS_xxx)"; +} + +template <> +const char* CudaErrString(cudnnStatus_t e) { + cudaDeviceSynchronize(); + return cudnnGetErrorString(e); +} + +template +bool CudaCall(ERRTYPE retCode, const char* exprString, const char* libName, ERRTYPE successCode, const char* msg) { + if (retCode != successCode) { + try { +#ifdef _WIN32 + auto del = [](char* p) { free(p); }; + std::unique_ptr hostname_ptr(nullptr, del); + size_t hostname_len = 0; + char* hostname = nullptr; + if (-1 == _dupenv_s(&hostname, &hostname_len, "COMPUTERNAME")) + hostname = "?"; + else + hostname_ptr.reset(hostname); +#else + char hostname[HOST_NAME_MAX]; + if (gethostname(hostname, HOST_NAME_MAX) != 0) + strcpy(hostname, "?"); +#endif + int currentCudaDevice; + cudaGetDevice(¤tCudaDevice); + static char str[1024]; + snprintf(str, 1024, "%s failure %d: %s ; GPU=%d ; hostname=%s ; expr=%s; %s", + libName, (int)retCode, CudaErrString(retCode), currentCudaDevice, + hostname, + exprString, msg); + if (THRW) { + ONNXRUNTIME_THROW(str); + } else { + LOGS_DEFAULT(ERROR) << str; + } + } catch (const std::exception& e) { // catch, log, and rethrow since CUDA code sometimes hangs in destruction, so we'd never get to see the error + if (THRW) { + ONNXRUNTIME_THROW(e.what()); + } else { + LOGS_DEFAULT(ERROR) << e.what(); + } + } + return false; + } + return true; +} + +template bool CudaCall(cudaError retCode, const char* exprString, const char* libName, cudaError successCode, const char* msg); +template bool CudaCall(cudaError retCode, const char* exprString, const char* libName, cudaError successCode, const char* msg); +template bool CudaCall(cublasStatus_t retCode, const char* exprString, const char* libName, cublasStatus_t successCode, const char* msg); +template bool CudaCall(cublasStatus_t retCode, const char* exprString, const char* libName, cublasStatus_t successCode, const char* msg); +template bool CudaCall(cudnnStatus_t retCode, const char* exprString, const char* libName, cudnnStatus_t successCode, const char* msg); +template bool CudaCall(cudnnStatus_t retCode, const char* exprString, const char* libName, cudnnStatus_t successCode, const char* msg); + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cuda_common.h b/onnxruntime/core/providers/cuda/cuda_common.h new file mode 100644 index 0000000000000..0fc1e30d5cf42 --- /dev/null +++ b/onnxruntime/core/providers/cuda/cuda_common.h @@ -0,0 +1,178 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "cuda_pch.h" +#include "core/common/status.h" +#include "core/framework/op_kernel.h" +#include "core/graph/graph.h" +#include "shared_inc/cuda_call.h" +#include "cuda_execution_provider.h" +#include "shared_inc/fast_divmod.h" +#include "core/util/math.h" +#include "cuda_fwd.h" + +namespace onnxruntime { +namespace cuda { + +#define CUDA_RETURN_IF_ERROR(expr) ONNXRUNTIME_RETURN_IF_ERROR(CUDA_CALL(expr) ? common::Status::OK() : common::Status(common::ONNXRUNTIME, common::FAIL)) +#define CUBLAS_RETURN_IF_ERROR(expr) ONNXRUNTIME_RETURN_IF_ERROR(CUBLAS_CALL(expr) ? common::Status::OK() : common::Status(common::ONNXRUNTIME, common::FAIL)) +#define CUSPARSE_RETURN_IF_ERROR(expr) ONNXRUNTIME_RETURN_IF_ERROR(CUSPARSE_CALL(expr) ? common::Status::OK() : common::Status(common::ONNXRUNTIME, common::FAIL)) +#define CURAND_RETURN_IF_ERROR(expr) ONNXRUNTIME_RETURN_IF_ERROR(CURAND_CALL(expr) ? common::Status::OK() : common::Status(common::ONNXRUNTIME, common::FAIL)) +#define CUDNN_RETURN_IF_ERROR(expr) ONNXRUNTIME_RETURN_IF_ERROR(CUDNN_CALL(expr) ? common::Status::OK() : common::Status(common::ONNXRUNTIME, common::FAIL)) +#define CUDNN2_RETURN_IF_ERROR(expr, m) ONNXRUNTIME_RETURN_IF_ERROR(CUDNN_CALL2(expr, m) ? common::Status::OK() : common::Status(common::ONNXRUNTIME, common::FAIL)) + +// ----------------------------------------------------------------------- +// Base class for CUDA kernels +// ----------------------------------------------------------------------- +class CudaKernel : public OpKernel { + public: + explicit CudaKernel(const OpKernelInfo& info) + : OpKernel(info), + // Is this OK to have a non-const execution provider? + provider_(const_cast(dynamic_cast(info.GetExecutionProvider()))) { + } + + Status Compute(OpKernelContext* p_op_kernel_context) const override { + auto s = ComputeInternal(p_op_kernel_context); + // use this to precisely locate the node where CUDA failure comes from + // if (cudaSuccess != cudaDeviceSynchronize()) + // __debugbreak(); + return s; + } + + virtual Status ComputeInternal(OpKernelContext* p_op_kernel_context) const = 0; + + template + inline IAllocatorUniquePtr AllocateBufferOnCPUPinned(int id, size_t count_or_bytes) const { + AllocatorPtr allocator = provider_->GetAllocator(id, ONNXRuntimeMemTypeCPU); + if (!allocator) + return nullptr; + return IAllocator::MakeUniquePtr(allocator, count_or_bytes); + } + + template + inline IAllocatorUniquePtr GetScratchBuffer(size_t count_or_bytes) const { + return provider_->GetScratchBuffer(count_or_bytes); + } + + inline void AddDeferredReleaseCPUPtr(void* p) const { + provider_->AddDeferredReleaseCPUPtr(p); + } + + // To support cudaMemcpyAsync, the cpu memory should be allocated in pinned memory + // and it can only be released after the copy has finished + template + class CudaAsyncBuffer { + public: + CudaAsyncBuffer(const CudaKernel* op_kernel) : gpu_copy_(nullptr), count_(0), op_kernel_(op_kernel) {} + + CudaAsyncBuffer(const CudaKernel* op_kernel, int device_id, size_t count) : CudaAsyncBuffer(op_kernel) { + AllocCpuPtr(device_id, count); + } + + CudaAsyncBuffer(const CudaKernel* op_kernel, int device_id, const T& value) : CudaAsyncBuffer(op_kernel, device_id, 1) { + *CpuPtr() = value; + } + + CudaAsyncBuffer(const CudaKernel* op_kernel, int device_id, const std::vector& vec) : CudaAsyncBuffer(op_kernel, device_id, vec.size()) { + memcpy(CpuPtr(), vec.data(), vec.size() * sizeof(T)); + } + + void AllocCpuPtr(int id, size_t count) { + cpu_pinned_copy_ = op_kernel_->AllocateBufferOnCPUPinned(id, count); + if (cpu_pinned_copy_ == nullptr) + throw std::runtime_error("alloc failed"); + count_ = count; + } + + Status CopyToGpu() { + if (cpu_pinned_copy_) { + gpu_copy_ = op_kernel_->GetScratchBuffer(count_); + CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(gpu_copy_.get(), cpu_pinned_copy_.get(), count_ * sizeof(T), cudaMemcpyHostToDevice)); + op_kernel_->AddDeferredReleaseCPUPtr(cpu_pinned_copy_.release()); + } + return Status::OK(); + } + + T* CpuPtr() const { + return cpu_pinned_copy_.get(); + } + + gsl::span CpuSpan() const { + return gsl::span(CpuPtr(), count_); + } + + T* GpuPtr() const { + return gpu_copy_.get(); + } + + size_t count() const { + return count_; + } + + protected: + IAllocatorUniquePtr gpu_copy_; + IAllocatorUniquePtr cpu_pinned_copy_; + size_t count_; + const CudaKernel* op_kernel_; + }; + + protected: + inline cublasHandle_t CublasHandle() const { + return provider_->PerThreadCublasHandle(); + } + + inline cudnnHandle_t CudnnHandle() const { + return provider_->PerThreadCudnnHandle(); + } + + template + inline const T* GetConstOnes(size_t count) const { + return provider_->template GetConstOnes(count); + } + + inline Status CopyTensor(const Tensor& src, Tensor& dst) const { + return provider_->CopyTensor(src, dst); + } + + private: + CUDAExecutionProvider* provider_; +}; + +// Type mapping for MLFloat16 to half +template +class ToCudaType { + public: + typedef T MappedType; + static MappedType FromFloat(float f) { + return static_cast(f); + } +}; + +template <> +class ToCudaType { + public: + typedef half MappedType; + static MappedType FromFloat(float f) { + uint16_t h = math::floatToHalf(f); + return *reinterpret_cast(&h); + } +}; + +inline bool CalculateFdmStrides(gsl::span p, const std::vector& dims) { + int stride = 1; + if (dims.empty() || p.size() < gsl::narrow_cast(dims.size())) + return false; + std::ptrdiff_t rank = p.size(); + for (std::ptrdiff_t i = 0; i < rank; i++) { + p[rank - 1 - i] = fast_divmod(stride); + if (static_cast(i) < dims.size() - 1) { + stride *= static_cast(dims[dims.size() - 1 - i]); + } + } + return true; +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider.cc b/onnxruntime/core/providers/cuda/cuda_execution_provider.cc new file mode 100644 index 0000000000000..cdb7ad9a40f52 --- /dev/null +++ b/onnxruntime/core/providers/cuda/cuda_execution_provider.cc @@ -0,0 +1,852 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "cuda_common.h" +#include "cuda_execution_provider.h" +#include "core/framework/transformer_memcpy.h" +#include "core/framework/memcpy.h" +#include "cuda_fence.h" +#include "cuda_allocator.h" +#include "core/framework/kernel_registry.h" +#include "core/framework/computation_capacity.h" + +using namespace onnxruntime::common; + +namespace onnxruntime { + +namespace cuda { + +ONNX_OPERATOR_KERNEL_EX( + MemcpyFromHost, + kOnnxDomain, + 1, + kCudaExecutionProvider, + KernelDefBuilder() + .InputMemoryType(0) + .ExecQueueId(kCudaStreamCopyIn) + .TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()), + Memcpy); + +ONNX_OPERATOR_KERNEL_EX( + MemcpyToHost, + kOnnxDomain, + 1, + kCudaExecutionProvider, + KernelDefBuilder() + .OutputMemoryType(0) + .ExecQueueId(kCudaStreamCopyOut) + .TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()), + Memcpy); + +} // namespace cuda + +thread_local std::shared_ptr CUDAExecutionProvider::per_thread_context_; +thread_local AllocatorPtr CUDAExecutionProvider::per_thread_default_allocator_; + +CUDAExecutionProvider::PerThreadContext::PerThreadContext(int device_id) { + CUDA_CALL_THROW(cudaSetDevice(device_id)); + CUBLAS_CALL_THROW(cublasCreate(&cublas_handle_)); + CUDNN_CALL_THROW(cudnnCreate(&cudnn_handle_)); +} + +CUDAExecutionProvider::PerThreadContext::~PerThreadContext() { + CUBLAS_CALL_THROW(cublasDestroy(cublas_handle_)); + CUDNN_CALL_THROW(cudnnDestroy(cudnn_handle_)); +} + +CUDAExecutionProvider::CUDAExecutionProvider(const CUDAExecutionProviderInfo& info) + : device_id_(info.device_id) { + CUDA_CALL_THROW(cudaSetDevice(device_id_)); + // create streams, default is nullptr + streams_[kCudaStreamDefault] = nullptr; + CUDA_CALL_THROW(cudaStreamCreateWithFlags(&streams_[kCudaStreamCopyIn], cudaStreamNonBlocking)); + CUDA_CALL_THROW(cudaStreamCreateWithFlags(&streams_[kCudaStreamCopyOut], cudaStreamNonBlocking)); + + DeviceAllocatorRegistrationInfo default_allocator_info({ONNXRuntimeMemTypeDefault, + [](int id) { return std::make_unique(id); }, std::numeric_limits::max()}); + InsertAllocator(CreateAllocator(default_allocator_info, device_id_)); + + DeviceAllocatorRegistrationInfo pinned_allocator_info({ONNXRuntimeMemTypeCPUOutput, + [](int) { return std::make_unique(); }, std::numeric_limits::max()}); + InsertAllocator(CreateAllocator(pinned_allocator_info, device_id_)); +} + +CUDAExecutionProvider::~CUDAExecutionProvider() { + auto cpu_alloc = GetAllocator(0, ONNXRuntimeMemTypeCPU); + std::lock_guard lock(deferred_release_cpu_ptr_mutex_); + auto it = deferred_release_cpu_ptr_.begin(); + while (it != deferred_release_cpu_ptr_.end()) { + auto& e = it->first; + auto& v = it->second; + if (v.recorded) + CUDA_CALL_THROW(cudaEventSynchronize(e)); + for (auto p : v.cpu_ptrs) { + cpu_alloc->Free(p); + } + CUDA_CALL_THROW(cudaEventDestroy(e)); + it = deferred_release_cpu_ptr_.erase(it); + } + CUDA_CALL_THROW(cudaStreamDestroy(streams_[kCudaStreamCopyIn])); + CUDA_CALL_THROW(cudaStreamDestroy(streams_[kCudaStreamCopyOut])); + + ReleasePerThreadStuffs(); +} + +void CUDAExecutionProvider::ReleasePerThreadStuffs() const { + if (per_thread_default_allocator_) { + std::lock_guard lock(default_allocator_pool_mutex_); + default_allocator_pool_.push_back(per_thread_default_allocator_); + per_thread_default_allocator_.reset(); + } + if (per_thread_context_) { + std::lock_guard lock(context_pool_mutex_); + context_pool_.push_back(per_thread_context_); + per_thread_context_.reset(); + } +} + +AllocatorPtr CUDAExecutionProvider::GetAllocator(int id, ONNXRuntimeMemType mem_type) const { + // Pinned memory allocator is shared between threads, but CUDA memory allocator is per-thread or it may cause result changes + // A hypothesis is that arena allocator is not aligned with CUDA output cache, and data from different kernel writes may + // cause cacheline to contain dirty data. + if (mem_type == ONNXRuntimeMemTypeDefault) { + if (!per_thread_default_allocator_) { + std::lock_guard lock(default_allocator_pool_mutex_); + if (default_allocator_pool_.empty()) { + DeviceAllocatorRegistrationInfo default_allocator_info( + {ONNXRuntimeMemTypeDefault, + [](int id) { return std::make_unique(id); }, std::numeric_limits::max()}); + per_thread_default_allocator_ = CreateAllocator(default_allocator_info, device_id_); + } else { + per_thread_default_allocator_ = default_allocator_pool_.back(); + default_allocator_pool_.pop_back(); + } + } + return per_thread_default_allocator_; + } else { + return IExecutionProvider::GetAllocator(id, mem_type); + } +} + +Status CUDAExecutionProvider::Sync() const { + CUDA_RETURN_IF_ERROR(cudaDeviceSynchronize()); + return Status::OK(); +} + +void CUDAExecutionProvider::AddDeferredReleaseCPUPtr(void* p) { + // when not running in InferenceSession (e.g. Test) + // it's OK to not remember the deferred release ptr + // as the actual memory will be cleaned in arena allocator dtor + auto current_deferred_release_event = per_thread_context_->GetCurrentDeferredReleaseEvent(); + if (current_deferred_release_event) { + std::lock_guard lock(deferred_release_cpu_ptr_mutex_); + auto iter = deferred_release_cpu_ptr_.find(current_deferred_release_event); + ONNXRUNTIME_ENFORCE(iter != deferred_release_cpu_ptr_.end()); + iter->second.cpu_ptrs.push_back(p); + } +} + +Status CUDAExecutionProvider::OnRunStart() { + auto cpu_alloc = GetAllocator(0, ONNXRuntimeMemTypeCPU); + // check if cudaEvents has passed for deferred release + // note that we need to take a mutex in case of multi-threaded Run() + std::lock_guard lock(deferred_release_cpu_ptr_mutex_); + auto it = deferred_release_cpu_ptr_.begin(); + while (it != deferred_release_cpu_ptr_.end()) { + auto& e = it->first; + auto& v = it->second; + // note that cudaEventQuery returns cudaSucess before first cudaEventRecord + if (v.recorded && cudaSuccess == cudaEventQuery(e)) { + for (auto p : v.cpu_ptrs) { + cpu_alloc->Free(p); + } + cudaEvent_t expired_event = it->first; + it = deferred_release_cpu_ptr_.erase(it); + CUDA_RETURN_IF_ERROR(cudaEventDestroy(expired_event)); + } else { + ++it; + } + } + // start a new per_thread context, store in TLS + { + std::lock_guard ctx_lock(context_pool_mutex_); + if (context_pool_.empty()) { + per_thread_context_ = std::make_shared(device_id_); + } else { + per_thread_context_ = context_pool_.back(); + context_pool_.pop_back(); + } + } + auto& current_deferred_release_event = per_thread_context_->GetCurrentDeferredReleaseEvent(); + CUDA_RETURN_IF_ERROR(cudaEventCreate(¤t_deferred_release_event, cudaEventDisableTiming)); + deferred_release_cpu_ptr_.emplace(current_deferred_release_event, DeferredReleaseCPUPtrs()); + return Status::OK(); +} + +Status CUDAExecutionProvider::OnRunEnd() { + ONNXRUNTIME_RETURN_IF_NOT(per_thread_context_ != nullptr); + // record deferred release event on default stream, and release per_thread_context + auto current_deferred_release_event = per_thread_context_->GetCurrentDeferredReleaseEvent(); + CUDA_RETURN_IF_ERROR(cudaEventRecord(current_deferred_release_event, nullptr)); + ReleasePerThreadStuffs(); + std::lock_guard lock(deferred_release_cpu_ptr_mutex_); + deferred_release_cpu_ptr_[current_deferred_release_event].recorded = true; + return Status::OK(); +} + +Status CUDAExecutionProvider::CopyTensor(const Tensor& src, Tensor& dst) const { + return CopyTensor(src, dst, kCudaStreamDefault); +} + +Status CUDAExecutionProvider::CopyTensor(const Tensor& src, Tensor& dst, int exec_queue_id) const { + if (src.Shape().Size() != dst.Shape().Size()) { + return Status(ONNXRUNTIME, FAIL, "Tensor size mismatch"); + } + + if (strcmp(src.Location().name, CUDA) != 0 && strcmp(src.Location().name, CUDA_PINNED) != 0 && + strcmp(dst.Location().name, CUDA) != 0 && strcmp(dst.Location().name, CUDA_PINNED) != 0) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unsupported tensor location: src_location is: ", src.Location().name, " and dst_location is: ", dst.Location().name); + } + + size_t bytes = src.DataType()->Size() * src.Shape().Size(); + + const void* src_data = src.DataRaw(); + void* dst_data = dst.MutableDataRaw(); + + if (strcmp(dst.Location().name, CUDA) == 0) { + if (strcmp(src.Location().name, CUDA_PINNED) == 0) { + // copy from pinned memory to GPU, this is non-blocking + CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(dst_data, src_data, bytes, cudaMemcpyHostToDevice, streams_[exec_queue_id])); + } else if (strcmp(src.Location().name, CUDA) == 0) { + // copying between GPU, this is non-blocking + CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(dst_data, src_data, bytes, cudaMemcpyDeviceToDevice, streams_[kCudaStreamDefault])); + } else { + // copy from other CPU memory to GPU, this is blocking + CUDA_RETURN_IF_ERROR(cudaMemcpy(dst_data, src_data, bytes, cudaMemcpyHostToDevice)); + } + } else if (strcmp(src.Location().name, CUDA) == 0) { + if (strcmp(dst.Location().name, CUDA_PINNED) == 0) { + // copying from GPU to pinned memory, this is non-blocking + CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(dst_data, src_data, bytes, cudaMemcpyDeviceToHost, streams_[exec_queue_id])); + } else { + // copying from GPU to CPU memory, this is blocking + CUDA_RETURN_IF_ERROR(cudaMemcpy(dst_data, src_data, bytes, cudaMemcpyDeviceToHost)); + } + } else { + // copying between cpu memory + memcpy(dst_data, src_data, bytes); + } + + return Status::OK(); +} + +namespace cuda { +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MemcpyFromHost); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MemcpyToHost); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 4, Concat); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, Unsqueeze); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, Flatten); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, Squeeze); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, Identity); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, Dropout); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, Gather); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, float, Gemm); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, double, Gemm); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, MLFloat16, Gemm); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, MatMul); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, MatMul); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, MatMul); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, float, Tile); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, double, Tile); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, MLFloat16, Tile); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, Affine); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, Affine); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, Affine); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, float, Elu); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, double, Elu); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, MLFloat16, Elu); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, float, HardSigmoid); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, double, HardSigmoid); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, MLFloat16, HardSigmoid); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, float, LeakyRelu); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, double, LeakyRelu); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, MLFloat16, LeakyRelu); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, ParametricSoftplus); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, ParametricSoftplus); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, ParametricSoftplus); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, float, Relu); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, double, Relu); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, MLFloat16, Relu); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, ScaledTanh); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, ScaledTanh); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, ScaledTanh); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, float, Selu); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, double, Selu); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, MLFloat16, Selu); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, float, Sigmoid); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, double, Sigmoid); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, MLFloat16, Sigmoid); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, Softsign); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, Softsign); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, Softsign); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, float, Tanh); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, double, Tanh); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, MLFloat16, Tanh); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, ThresholdedRelu); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, ThresholdedRelu); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, ThresholdedRelu); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, Softplus); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, Softplus); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, Softplus); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, Softmax); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, Softmax); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, Softmax); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, float, Pow); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, double, Pow); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, MLFloat16, Pow); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, float, PRelu); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, double, PRelu); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, MLFloat16, PRelu); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, bool, And); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, bool, Or); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, bool, Xor); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, 7, int32_t, Sum); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, 7, int64_t, Sum); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, 7, uint32_t, Sum); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, 7, uint64_t, Sum); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, 7, float, Sum); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, 7, double, Sum); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, 7, MLFloat16, Sum); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 8, int32_t, Sum); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 8, int64_t, Sum); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 8, uint32_t, Sum); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 8, uint64_t, Sum); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 8, float, Sum); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 8, double, Sum); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 8, MLFloat16, Sum); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, int32_t, Add); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, int64_t, Add); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, uint32_t, Add); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, uint64_t, Add); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, float, Add); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, double, Add); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, MLFloat16, Add); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, int32_t, Sub); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, int64_t, Sub); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, uint32_t, Sub); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, uint64_t, Sub); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, float, Sub); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, double, Sub); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, MLFloat16, Sub); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, int32_t, Mul); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, int64_t, Mul); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, uint32_t, Mul); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, uint64_t, Mul); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, float, Mul); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, double, Mul); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, MLFloat16, Mul); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, int32_t, Div); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, int64_t, Div); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, uint32_t, Div); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, uint64_t, Div); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, float, Div); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, double, Div); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, MLFloat16, Div); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, int8_t, Abs); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, int16_t, Abs); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, int32_t, Abs); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, int64_t, Abs); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, uint8_t, Abs); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, uint16_t, Abs); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, uint32_t, Abs); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, uint64_t, Abs); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, float, Abs); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, double, Abs); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, MLFloat16, Abs); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, int8_t, Neg); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, int16_t, Neg); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, int32_t, Neg); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, int64_t, Neg); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, float, Neg); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, double, Neg); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, MLFloat16, Neg); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, float, Floor); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, double, Floor); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, MLFloat16, Floor); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, float, Ceil); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, double, Ceil); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, MLFloat16, Ceil); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, float, Reciprocal); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, double, Reciprocal); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, MLFloat16, Reciprocal); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, float, Sqrt); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, double, Sqrt); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, MLFloat16, Sqrt); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, float, Log); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, double, Log); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, MLFloat16, Log); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, float, Exp); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, double, Exp); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, MLFloat16, Exp); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, float, BatchNormalization); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, double, BatchNormalization); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, MLFloat16, BatchNormalization); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, LRN); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, LRN); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, LRN); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, Conv); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, Conv); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, Conv); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, ConvTranspose); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, ConvTranspose); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, ConvTranspose); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, float, AveragePool); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, double, AveragePool); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, MLFloat16, AveragePool); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, GlobalAveragePool); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, GlobalAveragePool); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, GlobalAveragePool); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 8, float, MaxPool); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 8, double, MaxPool); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 8, MLFloat16, MaxPool); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, 7, float, MaxPool); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, 7, double, MaxPool); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, 7, MLFloat16, MaxPool); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, GlobalMaxPool); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, GlobalMaxPool); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, GlobalMaxPool); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, ArgMax); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, ArgMax); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, ArgMax); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, ArgMin); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, ArgMin); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, ArgMin); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, ReduceL1); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, ReduceL1); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, ReduceL1); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, ReduceL2); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, ReduceL2); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, ReduceL2); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, ReduceMax); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, ReduceMax); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, ReduceMax); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, ReduceMean); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, ReduceMean); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, ReduceMean); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, ReduceMin); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, ReduceMin); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, ReduceMin); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, ReduceProd); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, ReduceProd); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, ReduceProd); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, ReduceSum); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, ReduceSum); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, ReduceSum); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, ReduceLogSum); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, ReduceLogSum); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, ReduceLogSum); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, ReduceSumSquare); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, ReduceSumSquare); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, ReduceSumSquare); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, ReduceLogSumExp); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, ReduceLogSumExp); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, ReduceLogSumExp); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, float, Cast); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, double, Cast); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, MLFloat16, Cast); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, int8_t, Cast); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, int16_t, Cast); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, int32_t, Cast); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, int64_t, Cast); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, uint8_t, Cast); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, uint16_t, Cast); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, uint32_t, Cast); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, uint64_t, Cast); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, bool, Cast); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 2, float, Pad); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 2, double, Pad); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 2, MLFloat16, Pad); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 5, Reshape); +class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, 4, Reshape_1); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, Shape); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, Crop); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, Crop); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, Crop); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, float, Tile); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, double, Tile); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, MLFloat16, Tile); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, Transpose); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, Transpose); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, Transpose); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, float, ImageScaler); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, double, ImageScaler); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, MLFloat16, ImageScaler); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, float, InstanceNormalization); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, double, InstanceNormalization); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, MLFloat16, InstanceNormalization); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, float, RNN); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, double, RNN); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, MLFloat16, RNN); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, float, GRU); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, double, GRU); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, MLFloat16, GRU); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, float, LSTM); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, double, LSTM); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, MLFloat16, LSTM); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, Slice); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 9, Compress); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, float, Upsample); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, double, Upsample); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, MLFloat16, Upsample); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 7, int32_t, Upsample); + +static void RegisterCudaKernels(std::function fn) { + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); +} + +} // namespace cuda + +std::shared_ptr CUDAExecutionProvider::GetKernelRegistry() const { + static std::shared_ptr kernel_registry = std::make_shared(onnxruntime::cuda::RegisterCudaKernels); + return kernel_registry; +} + +bool CUDAExecutionProvider::RNNNeedFallbackToCPU(const onnxruntime::Node& node, + const std::vector activations_supported, + const std::string& op_type) const { + auto node_attributes = node.GetAttributes(); + // Check attributes + for (auto& attr : node_attributes) { + auto attr_name = attr.first; + auto attr_value = attr.second; + + if ("activation_alpha" == attr_name || "activation_beta" == attr_name || "clip" == attr_name) { + return true; + } + + if ("activations" == attr_name && + ::onnx::AttributeProto_AttributeType::AttributeProto_AttributeType_STRINGS == attr_value.type()) { + for (int i = 0; i < attr_value.strings_size(); ++i) { + if (activations_supported[i] != attr_value.strings(i)) { + return true; + } + } + } + + if ("LSTM" == op_type && + "input_forget" == attr_name && + ::onnx::AttributeProto_AttributeType::AttributeProto_AttributeType_INT == attr_value.type()) { + if (0 != attr_value.i()) { + return true; + } + } + + if ("GRU" == op_type && + "linear_before_reset" == attr_name && + ::onnx::AttributeProto_AttributeType::AttributeProto_AttributeType_INT == attr_value.type()) { + // cudnn GRU only support linear_before_reset = 1 + if (1 != attr_value.i()) { + return true; + } + } + } + + if ("LSTM" == op_type) { + // cudnn LSTM not support peephole + auto input_defs = node.InputDefs(); + if (8 == input_defs.size()) { + auto peephole = input_defs.at(7); + if (peephole->Exists()) { + return true; + } + } + } + return false; +} + +std::vector> +CUDAExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph, + const std::vector& kernel_registries) const { + std::vector> result = IExecutionProvider::GetCapability(graph, kernel_registries); + + for (auto& node : graph.Nodes()) { + bool fallback_to_cpu_provider = false; + if ("LSTM" == node.OpType()) { + std::vector activations_supported{"Sigmoid", "Tanh", "Tanh"}; + fallback_to_cpu_provider = RNNNeedFallbackToCPU(node, activations_supported, node.OpType()); + } else if ("RNN" == node.OpType()) { + std::vector activations_supported{"Tanh", "Tanh"}; + fallback_to_cpu_provider = RNNNeedFallbackToCPU(node, activations_supported, node.OpType()); + } else if ("GRU" == node.OpType()) { + std::vector activations_supported{"Sigmoid", "Tanh"}; + fallback_to_cpu_provider = RNNNeedFallbackToCPU(node, activations_supported, node.OpType()); + } + + if (fallback_to_cpu_provider) { + LOGS_DEFAULT(WARNING) << "Fallback to CPU execution provider for Op type: " << node.OpType() << " node name: " << node.Name(); + auto update_node = const_cast(graph.GetNode(node.Index())); + update_node->SetExecutionProviderType(onnxruntime::kCpuExecutionProvider); + } + } + + return result; +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider.h b/onnxruntime/core/providers/cuda/cuda_execution_provider.h new file mode 100644 index 0000000000000..247841bee9f22 --- /dev/null +++ b/onnxruntime/core/providers/cuda/cuda_execution_provider.h @@ -0,0 +1,179 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "cuda_pch.h" +#include "core/graph/graph_transformer.h" +#include "core/framework/allocatormgr.h" +#include "core/framework/execution_provider.h" +#include "shared_inc/cuda_utils.h" +#include + +namespace onnxruntime { + +// Information needed to construct CUDA execution providers. +struct CUDAExecutionProviderInfo { + int device_id{0}; +}; + +enum CUDAStreamType : int { + kCudaStreamDefault = 0, + kCudaStreamCopyIn, + kCudaStreamCopyOut, + kTotalCudaStreams, +}; + +// Logical device representation. +class CUDAExecutionProvider : public IExecutionProvider { + public: + explicit CUDAExecutionProvider(const CUDAExecutionProviderInfo& info); + virtual ~CUDAExecutionProvider(); + + AllocatorPtr GetAllocator(int id, ONNXRuntimeMemType mem_type = ONNXRuntimeMemTypeDefault) const override; + + std::string Type() const override { + return onnxruntime::kCudaExecutionProvider; + } + + Status Sync() const override; + + Status OnRunStart() override; + + Status OnRunEnd() override; + + Status CopyTensor(const Tensor& src, Tensor& dst) const override; + + Status CopyTensor(const Tensor& src, Tensor& dst, int exec_queue_id) const override; + + const void* GetExecutionHandle() const noexcept override { + // The CUDA interface does not return anything interesting. + return nullptr; + } + + cublasHandle_t PerThreadCublasHandle() { + // Assure each thread has its TLS context. + if (!per_thread_context_) + per_thread_context_ = std::make_shared(device_id_); + return per_thread_context_->CublasHandle(); + } + + cudnnHandle_t PerThreadCudnnHandle() { + // Assure each thread has its TLS context. + // TODO: improve its performance when calling cuda functions from multiple threads. + if (!per_thread_context_) + per_thread_context_ = std::make_shared(device_id_); + return per_thread_context_->CudnnHandle(); + } + + cudaStream_t GetStream(int queue_id) const { + ONNXRUNTIME_ENFORCE(queue_id >= 0 && queue_id < kTotalCudaStreams); + return streams_[queue_id]; + } + + template + const T* GetConstOnes(size_t count) { + // Assure each thread has its TLS context. + if (!per_thread_context_) + per_thread_context_ = std::make_shared(device_id_); + return per_thread_context_->template GetConstOnes(count); + } + + void AddDeferredReleaseCPUPtr(void* p); + + template + inline IAllocatorUniquePtr GetScratchBuffer(size_t count_or_bytes) const { + if (count_or_bytes == 0) + return nullptr; + + return IAllocator::MakeUniquePtr(GetAllocator(ONNXRuntimeMemTypeDefault), count_or_bytes); + } + + virtual std::shared_ptr GetKernelRegistry() const override; + + virtual std::vector> + GetCapability(const onnxruntime::GraphViewer& graph, + const std::vector& kernel_registries) const override; + private: + cudaStream_t streams_[kTotalCudaStreams]; + int device_id_; + + struct DeferredReleaseCPUPtrs { + bool recorded = false; + std::vector cpu_ptrs; + }; + std::unordered_map deferred_release_cpu_ptr_; + std::mutex deferred_release_cpu_ptr_mutex_; + + class PerThreadContext final { + public: + PerThreadContext(int device_id); + ~PerThreadContext(); + + cublasHandle_t CublasHandle() const { + return cublas_handle_; + } + + cudnnHandle_t CudnnHandle() const { + return cudnn_handle_; + } + + cudaEvent_t& GetCurrentDeferredReleaseEvent() { + return current_deferred_release_event_; + } + + template + const T* GetConstOnes(size_t count) { + if (std::is_same::value) { + if (!constant_ones_float_) { + constant_ones_float_ = cuda::CreateConstantOnes(); + } + return reinterpret_cast(constant_ones_float_->GetBuffer(count)); + } else if (std::is_same::value) { + if (!constant_ones_double_) { + constant_ones_double_ = cuda::CreateConstantOnes(); + } + return reinterpret_cast(constant_ones_double_->GetBuffer(count)); + } else if (std::is_same::value) { + if (!constant_ones_half_) { + constant_ones_half_ = cuda::CreateConstantOnes(); + } + return reinterpret_cast(constant_ones_half_->GetBuffer(count)); + } else { + return nullptr; + } + } + + private: + cublasHandle_t cublas_handle_ = nullptr; + cudnnHandle_t cudnn_handle_ = nullptr; + + // deferred release for temporary CPU pinned memory used in cudaMemcpyAsync + // note that cudaEvent will be assigned at OnRunEnd() when PerThreadContext destory + // so the ownership is passed to deferred_release_cpu_ptr_ + cudaEvent_t current_deferred_release_event_ = nullptr; + + std::unique_ptr> constant_ones_float_; + std::unique_ptr> constant_ones_double_; + std::unique_ptr> constant_ones_half_; + }; + + // thread local context during execution + static thread_local std::shared_ptr per_thread_context_; + + // thread local GPU memory allocator. could be used before execution + static thread_local AllocatorPtr per_thread_default_allocator_; + + // reuse thread local GPU memory allocator for memory pattern + mutable std::deque default_allocator_pool_; + mutable std::mutex default_allocator_pool_mutex_; + + // reuse thread local context + mutable std::deque> context_pool_; + mutable std::mutex context_pool_mutex_; + + void ReleasePerThreadStuffs() const; + + bool RNNNeedFallbackToCPU(const onnxruntime::Node& node, const std::vector activations_supported, const std::string& op_type) const; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cuda_fence.cc b/onnxruntime/core/providers/cuda/cuda_fence.cc new file mode 100644 index 0000000000000..754e66bf1933c --- /dev/null +++ b/onnxruntime/core/providers/cuda/cuda_fence.cc @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "cuda_common.h" +#include "cuda_fence.h" + +namespace onnxruntime { + +CUDAFence::CUDAFence(const CUDAExecutionProvider* provider) : provider_(provider) { + // NOTE: cudaEventBlockingSync may leads to longer wait time because of thread yield/switching in kernel + // if lower CPU usage is more important than latency, we should use this flag to avoid spin-loop in WaitOnCPU + int event_flags = /*cudaEventBlockingSync |*/ cudaEventDisableTiming; + CUDA_CALL_THROW(cudaEventCreate(&read_event_, event_flags)); + CUDA_CALL_THROW(cudaEventCreate(&write_event_, event_flags)); +} + +CUDAFence::~CUDAFence() { + CUDA_CALL_THROW(cudaEventDestroy(read_event_)); + CUDA_CALL_THROW(cudaEventDestroy(write_event_)); +} + +void CUDAFence::BeforeUsingAsInput(onnxruntime::ProviderType provider_type, int async_queue_id) { + if (provider_type == onnxruntime::kCudaExecutionProvider) { + // sync in GPU, the call is non-blocking on CPU + CUDA_CALL_THROW(cudaStreamWaitEvent(provider_->GetStream(async_queue_id), write_event_, 0)); + } else { + // sync on CPU for all other providers, this is blocking + CUDA_CALL_THROW(cudaEventSynchronize(write_event_)); + } +} + +void CUDAFence::BeforeUsingAsOutput(onnxruntime::ProviderType provider_type, int queue_id) { + if (provider_type == onnxruntime::kCudaExecutionProvider) { + // sync in GPU, the call is non-blocking on CPU + cudaStream_t stream = provider_->GetStream(queue_id); + CUDA_CALL_THROW(cudaStreamWaitEvent(stream, read_event_, 0)); + CUDA_CALL_THROW(cudaStreamWaitEvent(stream, write_event_, 0)); + } else { + // sync on CPU for all other providers, this is blocking + CUDA_CALL_THROW(cudaEventSynchronize(read_event_)); + CUDA_CALL_THROW(cudaEventSynchronize(write_event_)); + } +} + +void CUDAFence::AfterUsedAsInput(int queue_id) { + // update read fence + cudaStream_t stream = provider_->GetStream(queue_id); + CUDA_CALL_THROW(cudaEventRecord(read_event_, stream)); +} + +void CUDAFence::AfterUsedAsOutput(int queue_id) { + // update write fence + cudaStream_t stream = provider_->GetStream(queue_id); + CUDA_CALL_THROW(cudaEventRecord(write_event_, stream)); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cuda_fence.h b/onnxruntime/core/providers/cuda/cuda_fence.h new file mode 100644 index 0000000000000..e622ca61e0ebd --- /dev/null +++ b/onnxruntime/core/providers/cuda/cuda_fence.h @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/framework/tensor.h" +#include "cuda_execution_provider.h" +namespace onnxruntime { + +class CUDAFence : public IFence { + public: + CUDAFence(const CUDAExecutionProvider* provider); + virtual ~CUDAFence(); + virtual void BeforeUsingAsInput(onnxruntime::ProviderType provider_type, int queue_id) override; + virtual void BeforeUsingAsOutput(onnxruntime::ProviderType provider_type, int queue_id) override; + virtual void AfterUsedAsInput(int queue_id) override; + virtual void AfterUsedAsOutput(int queue_id) override; + + private: + cudaEvent_t read_event_; + cudaEvent_t write_event_; + const CUDAExecutionProvider* provider_; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cuda_fwd.h b/onnxruntime/core/providers/cuda/cuda_fwd.h new file mode 100644 index 0000000000000..0001933e54728 --- /dev/null +++ b/onnxruntime/core/providers/cuda/cuda_fwd.h @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +namespace onnxruntime { +namespace cuda { +template +KernelCreateInfo BuildKernel(); +} +} diff --git a/onnxruntime/core/providers/cuda/cuda_pch.cc b/onnxruntime/core/providers/cuda/cuda_pch.cc new file mode 100644 index 0000000000000..d312b20f6d160 --- /dev/null +++ b/onnxruntime/core/providers/cuda/cuda_pch.cc @@ -0,0 +1,4 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "cuda_pch.h" diff --git a/onnxruntime/core/providers/cuda/cuda_pch.h b/onnxruntime/core/providers/cuda/cuda_pch.h new file mode 100644 index 0000000000000..6fd31e0c08b53 --- /dev/null +++ b/onnxruntime/core/providers/cuda/cuda_pch.h @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#ifndef CUDA_API_PER_THREAD_DEFAULT_STREAM +#define CUDA_API_PER_THREAD_DEFAULT_STREAM +#endif +#include +#include +#include +#include +#include +#include diff --git a/onnxruntime/core/providers/cuda/cuda_provider_factory.cc b/onnxruntime/core/providers/cuda/cuda_provider_factory.cc new file mode 100644 index 0000000000000..fa12db2dd18e3 --- /dev/null +++ b/onnxruntime/core/providers/cuda/cuda_provider_factory.cc @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cuda/cuda_provider_factory.h" +#include +#include "cuda_execution_provider.h" + +using namespace onnxruntime; + +namespace { +struct CUDAProviderFactory { + const ONNXRuntimeProviderFactoryInterface* const cls; + std::atomic_int ref_count; + int device_id; + CUDAProviderFactory(); +}; + +ONNXStatusPtr ONNXRUNTIME_API_STATUSCALL CreateCuda(void* this_, ONNXRuntimeProviderPtr* out) { + CUDAExecutionProviderInfo info; + CUDAProviderFactory* this_ptr = (CUDAProviderFactory*)this_; + info.device_id = this_ptr->device_id; + CUDAExecutionProvider* ret = new CUDAExecutionProvider(info); + *out = (ONNXRuntimeProviderPtr)ret; + return nullptr; +} + +uint32_t ONNXRUNTIME_API_STATUSCALL ReleaseCuda(void* this_) { + CUDAProviderFactory* this_ptr = (CUDAProviderFactory*)this_; + if (--this_ptr->ref_count == 0) + delete this_ptr; + return 0; +} + +uint32_t ONNXRUNTIME_API_STATUSCALL AddRefCuda(void* this_) { + CUDAProviderFactory* this_ptr = (CUDAProviderFactory*)this_; + ++this_ptr->ref_count; + return 0; +} + +constexpr ONNXRuntimeProviderFactoryInterface cuda_cls = { + AddRefCuda, + ReleaseCuda, + CreateCuda, +}; + +CUDAProviderFactory::CUDAProviderFactory() : cls(&cuda_cls), ref_count(1), device_id(0) {} +} // namespace + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeCreateCUDAExecutionProviderFactory, int device_id, _Out_ ONNXRuntimeProviderFactoryPtr** out) { + CUDAProviderFactory* ret = new CUDAProviderFactory(); + ret->device_id = device_id; + *out = (ONNXRuntimeProviderFactoryPtr*)ret; + return nullptr; +} diff --git a/onnxruntime/core/providers/cuda/cuda_utils.cu b/onnxruntime/core/providers/cuda/cuda_utils.cu new file mode 100644 index 0000000000000..3790d4e7455de --- /dev/null +++ b/onnxruntime/core/providers/cuda/cuda_utils.cu @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Thrust code needs to be compiled with nvcc +#include +#include +#include +#include +#include +#include "core/providers/cuda/shared_inc/cuda_utils.h" +#include "core/providers/cuda/cu_inc/common.cuh" + +namespace onnxruntime { +namespace cuda { + +template +class ConstantBufferImpl : public IConstantBuffer { + public: + ConstantBufferImpl(T val) : val_(val) {} + + virtual const T* GetBuffer(size_t count) { + buffer_.resize(count); + thrust::fill(buffer_.begin(), buffer_.end(), val_); + return buffer_.data().get(); + } + + private: + thrust::device_vector buffer_; + T val_; +}; + +template +std::unique_ptr> CreateConstantOnes() { + return std::make_unique>((T)1); +} + +template std::unique_ptr> CreateConstantOnes(); +template std::unique_ptr> CreateConstantOnes(); +template std::unique_ptr> CreateConstantOnes(); + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cudnn_common.cc b/onnxruntime/core/providers/cuda/cudnn_common.cc new file mode 100644 index 0000000000000..0d09fcdb328b2 --- /dev/null +++ b/onnxruntime/core/providers/cuda/cudnn_common.cc @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "cudnn_common.h" +#include "gsl/gsl_util" +#include "shared_inc/cuda_call.h" +#include "core/providers/cpu/tensor/utils.h" + +namespace onnxruntime { +namespace cuda { + +CudnnTensor::CudnnTensor() + : tensor_(nullptr) { +} + +CudnnTensor::~CudnnTensor() { + if (tensor_ != nullptr) { + cudnnDestroyTensorDescriptor(tensor_); + tensor_ = nullptr; + } +} + +Status CudnnTensor::CreateTensorIfNeeded() { + if (!tensor_) + CUDNN_RETURN_IF_ERROR(cudnnCreateTensorDescriptor(&tensor_)); + return Status::OK(); +} + +Status CudnnTensor::Set(const std::vector& input_dims, cudnnDataType_t dataType) { + ONNXRUNTIME_RETURN_IF_ERROR(CreateTensorIfNeeded()); + + int rank = gsl::narrow_cast(input_dims.size()); + TensorPitches pitches(input_dims); + std::vector dims(rank); + std::vector strides(rank); + for (int i = 0; i < rank; i++) { + dims[i] = gsl::narrow_cast(input_dims[i]); + strides[i] = gsl::narrow_cast(pitches[i]); + } + CUDNN_RETURN_IF_ERROR(cudnnSetTensorNdDescriptor(tensor_, dataType, static_cast(rank), dims.data(), strides.data())); + return Status::OK(); +} + +Status CudnnTensor::Set(const CudnnTensor& x_desc, cudnnBatchNormMode_t mode) { + ONNXRUNTIME_RETURN_IF_ERROR(CreateTensorIfNeeded()); + CUDNN_RETURN_IF_ERROR(cudnnDeriveBNTensorDescriptor(tensor_, x_desc, mode)); + return Status::OK(); +} + +template +cudnnDataType_t CudnnTensor::GetDataType() { + if (typeid(ElemType) == typeid(float)) + return CUDNN_DATA_FLOAT; + else if (typeid(ElemType) == typeid(double)) + return CUDNN_DATA_DOUBLE; + else if (typeid(ElemType) == typeid(half)) + return CUDNN_DATA_HALF; + else + ONNXRUNTIME_THROW("cuDNN engine currently supports only single/double/half precision data types."); +} + +CudnnFilterDescriptor::CudnnFilterDescriptor() : desc_(nullptr) { + cudnnCreateFilterDescriptor(&desc_); +} + +CudnnFilterDescriptor::~CudnnFilterDescriptor() { + if (desc_ != nullptr) { + cudnnDestroyFilterDescriptor(desc_); + desc_ = nullptr; + } +} + +Status CudnnFilterDescriptor::Set(const std::vector& filter_dims, cudnnDataType_t data_type) { + if (!desc_) + CUDNN_RETURN_IF_ERROR(cudnnCreateFilterDescriptor(&desc_)); + + int rank = gsl::narrow_cast(filter_dims.size()); + std::vector w_dims(rank); + for (int i = 0; i < rank; i++) { + w_dims[i] = gsl::narrow_cast(filter_dims[i]); + } + + CUDNN_RETURN_IF_ERROR(cudnnSetFilterNdDescriptor(desc_, + data_type, + CUDNN_TENSOR_NCHW, + rank, + w_dims.data())); + return Status::OK(); +} + +template cudnnDataType_t CudnnTensor::GetDataType(); +template cudnnDataType_t CudnnTensor::GetDataType(); +template cudnnDataType_t CudnnTensor::GetDataType(); + +template <> +const float Consts::One = 1; + +template <> +const double Consts::One = 1; + +template <> +const float Consts::Zero = 0; + +template <> +const double Consts::Zero = 0; + +const float Consts::Zero = 0; + +const float Consts::One = 1; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cudnn_common.h b/onnxruntime/core/providers/cuda/cudnn_common.h new file mode 100644 index 0000000000000..3b56b9c68a392 --- /dev/null +++ b/onnxruntime/core/providers/cuda/cudnn_common.h @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "cuda_common.h" +#include "core/framework/tensor.h" +#include + +namespace onnxruntime { +namespace cuda { + +class CudnnTensor final { + public: + CudnnTensor(); + ~CudnnTensor(); + + Status Set(const std::vector& input_dims, cudnnDataType_t dataType); + Status Set(const CudnnTensor& x_desc, cudnnBatchNormMode_t mode); + + operator cudnnTensorDescriptor_t() const { return tensor_; } + + template + static cudnnDataType_t GetDataType(); + + private: + Status CreateTensorIfNeeded(); + + cudnnTensorDescriptor_t tensor_; +}; + +class CudnnFilterDescriptor final { + public: + CudnnFilterDescriptor(); + ~CudnnFilterDescriptor(); + + Status Set(const std::vector& filter_dims, cudnnDataType_t data_typ); + + operator cudnnFilterDescriptor_t() const { return desc_; } + + private: + cudnnFilterDescriptor_t desc_; +}; + +template +struct Consts { + static const ElemType Zero; + static const ElemType One; +}; + +template <> +struct Consts { + static const float Zero; + static const float One; +}; + +inline double ClampCudnnBatchNormEpsilon(double epsilon) { + if (epsilon < CUDNN_BN_MIN_EPSILON) { + if (CUDNN_BN_MIN_EPSILON - epsilon > FLT_EPSILON) + LOGS_DEFAULT(WARNING) << "Provided epsilon is smaller than CUDNN_BN_MIN_EPSILON. Setting it to CUDNN_BN_MIN_EPSILON"; + return CUDNN_BN_MIN_EPSILON; + } + return epsilon; +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/fpgeneric.cu b/onnxruntime/core/providers/cuda/fpgeneric.cu new file mode 100644 index 0000000000000..f00992ff50a48 --- /dev/null +++ b/onnxruntime/core/providers/cuda/fpgeneric.cu @@ -0,0 +1,157 @@ +// +// Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +// + +// Make generic operators for floating point types +/* This file contains: + Generalized library calls + kernels to be called for not supported data type +*/ +// NV_TODO: optimize speed -- pass things needed in, optimize kernel speed, add half2 +// NV_TODO: investigate cub support for half + +#include "core/providers/cuda/cu_inc/common.cuh" +#include + +#define TRANS_TILE_DIM 32 +#define BLOCK_ROWS 8 +#define COPY_TILE_DIM 1024 +#define COPY_BLOCK_DIM 256 + +// kernel(s) for half functions with no library support +namespace { + +__global__ void transposeNoOverlap(half* odata, const half* idata, const int m, const int n) { + __shared__ half tile[TRANS_TILE_DIM][TRANS_TILE_DIM + 1]; + + int x = blockIdx.x * TRANS_TILE_DIM + threadIdx.x; + int y = blockIdx.y * TRANS_TILE_DIM + threadIdx.y; + + for (int j = 0; j < TRANS_TILE_DIM; j += BLOCK_ROWS) + tile[threadIdx.y + j][threadIdx.x] = idata[(y + j) * m + x]; + + __syncthreads(); + + x = blockIdx.y * TRANS_TILE_DIM + threadIdx.x; // transpose block offset + y = blockIdx.x * TRANS_TILE_DIM + threadIdx.y; + + if (x >= n) return; + + for (int j = 0; j < TRANS_TILE_DIM; j += BLOCK_ROWS) { + if ((y + j) >= m) return; + odata[(y + j) * n + x] = tile[threadIdx.x][threadIdx.y + j]; + } +} +// set up curand state, need to move up layer to remove calling for each generate call +__global__ void setup_state(curandState* state, unsigned long long seed) { + curand_init(seed, 0, 0, state); +} + +__global__ void GenerateUniformHalf(curandState* state, half* result, int n) { + int id = blockIdx.x * blockDim.x + threadIdx.x; + if (id >= n) return; + + curandState localState = *state; + + float x; + skipahead(id, &localState); + x = curand_uniform(&localState); + + result[id] = x; + if (id == n - 1) *state = localState; +} + +__global__ void GenerateNormalHalf(curandState* state, half* result, int n, half mean, half stddev) { + int id = blockIdx.x * blockDim.x + threadIdx.x; + if (id >= n) return; + + curandState localState = *state; + + float x; + skipahead(id, &localState); + x = curand_normal(&localState); + + result[id] = (float)mean + (float)stddev * x; + if (id == n - 1) *state = localState; +} + +// kernels can convert matrix between half and float. speed currently not optimized, may need to add half2 +/* +__global__ void copyHalf2Float(float *odata, const half *idata, const int n) +{ + float tmp[COPY_TILE_DIM/COPY_BLOCK_DIM]; + + int x = blockIdx.x * COPY_TILE_DIM + threadIdx.x; + + for (int j = 0; j < COPY_TILE_DIM/COPY_BLOCK_DIM; j++) + tmp[j] = (float) idata[x + j*COPY_BLOCK_DIM]; + + for (int j = 0; j < COPY_TILE_DIM/COPY_BLOCK_DIM; j++) + if(x + j*COPY_BLOCK_DIM < n) odata[x + j*COPY_BLOCK_DIM] = tmp[j]; +} + +__global__ void copyFloat2Half(half *odata, const float *idata, const int n) +{ + float tmp[COPY_TILE_DIM/COPY_BLOCK_DIM]; + + int x = blockIdx.x * COPY_TILE_DIM + threadIdx.x; + + for (int j = 0; j < COPY_TILE_DIM/COPY_BLOCK_DIM; j++) + tmp[j] = idata[x + j*COPY_BLOCK_DIM]; + + for (int j = 0; j < COPY_TILE_DIM/COPY_BLOCK_DIM; j++) + if(x + j*COPY_BLOCK_DIM < n) odata[x + j*COPY_BLOCK_DIM] = tmp[j]; +} +*/ + +__global__ void CopyVectorHalf(const half* x, int incx, half* y, int incy, int n) { + int id = blockIdx.x * blockDim.x + threadIdx.x; + if (id >= n) return; + y[id * incy] = x[id * incx]; +} + +} // namespace + +cublasStatus_t cublasTransposeHelper(cublasHandle_t, cublasOperation_t, cublasOperation_t, int m, int n, half*, half* A, int, half*, half*, int, half* C, int) { + if (C != A) { + dim3 dimGrid((n + TRANS_TILE_DIM - 1) / TRANS_TILE_DIM, (m + TRANS_TILE_DIM - 1) / TRANS_TILE_DIM, 1); + dim3 dimBlock(TRANS_TILE_DIM, BLOCK_ROWS, 1); + + transposeNoOverlap<<>>(C, A, n, m); + } else { + return CUBLAS_STATUS_NOT_SUPPORTED; + } + return CUBLAS_STATUS_SUCCESS; +} + +cublasStatus_t cublasCopyHelper(cublasHandle_t, int n, const half* x, int incx, half* y, int incy) { + dim3 dimGrid((unsigned int)(n + COPY_BLOCK_DIM - 1) / COPY_BLOCK_DIM, 1, 1); + dim3 dimBlock(COPY_BLOCK_DIM, 1, 1); + CopyVectorHalf<<>>(x, incx, y, incy, n); + return CUBLAS_STATUS_SUCCESS; +} + +curandStatus_t curandGenerateUniformHelper(curandGenerator_t, half* outputPtr, size_t num) { + curandState* devStates; + cudaMalloc((void**)&devStates, sizeof(curandState)); + setup_state<<<1, 1>>>(devStates, time(NULL)); // What does curandGenerateUniform actually doing? should also pass in state here + + dim3 dimGrid((unsigned int)(num + COPY_BLOCK_DIM - 1) / COPY_BLOCK_DIM, 1, 1); + dim3 dimBlock(COPY_BLOCK_DIM, 1, 1); + GenerateUniformHalf<<>>(devStates, outputPtr, (int)num); + + return (curandStatus_t)0; +} + +curandStatus_t curandGenerateNormalHelper(curandGenerator_t, half* outputPtr, size_t n, half mean, half stddev) { + curandState* devStates; + cudaMalloc((void**)&devStates, sizeof(curandState)); + setup_state<<<1, 1>>>(devStates, time(NULL)); // What does curandGenerateUniform actually doing? should also pass in state here + + dim3 dimGrid((unsigned int)(n + COPY_BLOCK_DIM - 1) / COPY_BLOCK_DIM, 1, 1); + dim3 dimBlock(COPY_BLOCK_DIM, 1, 1); + GenerateNormalHalf<<>>(devStates, outputPtr, (int)n, mean, stddev); + + return (curandStatus_t)0; +} diff --git a/onnxruntime/core/providers/cuda/math/binary_elementwise_ops.cc b/onnxruntime/core/providers/cuda/math/binary_elementwise_ops.cc new file mode 100644 index 0000000000000..0cd19bb4ac3ca --- /dev/null +++ b/onnxruntime/core/providers/cuda/math/binary_elementwise_ops.cc @@ -0,0 +1,251 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "binary_elementwise_ops.h" +#include "binary_elementwise_ops_impl.h" +using namespace onnxruntime::common; +namespace onnxruntime { +namespace cuda { + +template <> +Status BinaryElementwise::Prepare(OpKernelContext* context, int /*device_id*/, BinaryElementwisePreparation* p) const { + p->lhs_tensor = context->Input(0); + p->rhs_tensor = context->Input(1); + if (!(p->lhs_tensor->Shape() == p->rhs_tensor->Shape())) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, Node().Name(), ": mismatching input shapes: ", p->lhs_tensor->Shape().ToString(), " != ", p->rhs_tensor->Shape().ToString()); + p->output_tensor = context->Output(0, p->lhs_tensor->Shape()); + p->output_rank_or_simple_broadcast = static_cast(SimpleBroadcast::NoBroadcast); + return Status::OK(); +} + +static Status ComputeOutputShape(const std::string& node_name, const TensorShape& lhs_shape, const TensorShape& rhs_shape, TensorShape& out_shape) { + size_t lhs_rank = lhs_shape.NumDimensions(); + size_t rhs_rank = rhs_shape.NumDimensions(); + size_t out_rank = std::max(lhs_rank, rhs_rank); + + std::vector output_dims(out_rank, 0); + for (int i = 0; i < out_rank; ++i) { + int64_t lhs_dim = 1; + if (i < lhs_rank) + lhs_dim = lhs_shape[lhs_rank - 1 - i]; + int64_t rhs_dim = 1; + if (i < rhs_rank) + rhs_dim = rhs_shape[rhs_rank - 1 - i]; + int64_t out_dim = std::max(lhs_dim, rhs_dim); + if (lhs_dim != out_dim && lhs_dim != 1) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, node_name, ": left operand cannot broadcast on dim ", lhs_rank - 1 - i, + " LeftShape: ", lhs_shape.ToString(), ", RightShape: ", rhs_shape.ToString()); + if (rhs_dim != out_dim && rhs_dim != 1) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, node_name, ": right operand cannot broadcast on dim ", rhs_rank - 1 - i, + " LeftShape: ", lhs_shape.ToString(), ", RightShape: ", rhs_shape.ToString()); + output_dims[out_rank - 1 - i] = out_dim; + } + out_shape = TensorShape(output_dims); + return Status::OK(); +} + +Status BinaryElementwiseBroadcastPrepare( + int device_id, const Tensor* lhs_tensor, + const Tensor* rhs_tensor, + Tensor* output_tensor, + BinaryElementwisePreparation* p, + const TensorShape* override_lhs_shape = nullptr, + const TensorShape* override_rhs_shape = nullptr) { + p->lhs_tensor = lhs_tensor; + p->rhs_tensor = rhs_tensor; + const auto& lhs_shape = override_lhs_shape ? *override_lhs_shape : lhs_tensor->Shape(); + const auto& rhs_shape = override_rhs_shape ? *override_rhs_shape : rhs_tensor->Shape(); + + p->output_tensor = output_tensor; + const auto& output_shape = output_tensor->Shape(); + + ONNXRUNTIME_RETURN_IF_ERROR(p->BinaryElementwiseBroadcastPrepareHelper(device_id, lhs_shape, rhs_shape, output_shape)); + + return Status::OK(); +} + +template <> +Status BinaryElementwise::Prepare(OpKernelContext* context, int device_id, BinaryElementwisePreparation* p) const { + auto lhs_tensor = context->Input(0); + auto rhs_tensor = context->Input(1); + const auto& lhs_shape = lhs_tensor->Shape(); + const auto& rhs_shape = rhs_tensor->Shape(); + + TensorShape output_shape; + ONNXRUNTIME_RETURN_IF_ERROR(ComputeOutputShape(Node().Name(), lhs_shape, rhs_shape, output_shape)); + auto output_tensor = context->Output(0, output_shape); + + ONNXRUNTIME_RETURN_IF_ERROR(BinaryElementwiseBroadcastPrepare(device_id, lhs_tensor, rhs_tensor, output_tensor, p)); + + return Status::OK(); +} + +#define BINARY_ELEMENTWISE_REGISTER_KERNEL_TYPED(x, ver, T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + x, \ + kOnnxDomain, \ + ver, \ + T, \ + kCudaExecutionProvider, \ + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + x); + +#define BINARY_ELEMENTWISE_REGISTER_KERNEL_VERSIONED_TYPED(x, startver, endver, T) \ + ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ + x, \ + kOnnxDomain, \ + startver, \ + endver, \ + T, \ + kCudaExecutionProvider, \ + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + x); + +#define BINARY_ELEMENTWISE_COMPUTE(x, T) \ + template <> \ + Status x::ComputeInternal(OpKernelContext* context) const { \ + BinaryElementwisePreparation prepare(this); \ + Prepare(context, 0, &prepare); \ + ONNXRUNTIME_RETURN_IF_ERROR(prepare.CopyToGpu()); \ + Impl_##x::MappedType>( \ + prepare.output_rank_or_simple_broadcast, \ + prepare.lhs_padded_strides.GpuPtr(), \ + reinterpret_cast::MappedType*>(prepare.lhs_tensor->template Data()), \ + prepare.rhs_padded_strides.GpuPtr(), \ + reinterpret_cast::MappedType*>(prepare.rhs_tensor->template Data()), \ + prepare.fdm_output_strides.GpuPtr(), \ + prepare.fdm_H, \ + prepare.fdm_C, \ + reinterpret_cast::MappedType*>(prepare.output_tensor->template MutableData()), \ + prepare.output_tensor->Shape().Size()); \ + return Status::OK(); \ + } + +#define BINARY_OP_TYPED(name, ver, T) \ + BINARY_ELEMENTWISE_REGISTER_KERNEL_TYPED(name, ver, T) \ + BINARY_ELEMENTWISE_COMPUTE(name, T) + + // since different ops has different types, we cannot use BINARY_OPS() directly + // the postfix of means the types supported by the op: + // B: uint8_t + // W: uint16_t + // U: uint32_t + // Z: uint64_t + // C: int8_t + // S: int16_t + // I: int32_t + // L: int64_t + // H: float16 + // F: float + // D: double + // O: bool + +#define BINARY_OP_HFD(name, ver) \ + BINARY_OP_TYPED(name, ver, MLFloat16) \ + BINARY_OP_TYPED(name, ver, float) \ + BINARY_OP_TYPED(name, ver, double) + +#define BINARY_OP_UZILHFD(name, ver) \ + BINARY_OP_TYPED(name, ver, uint32_t) \ + BINARY_OP_TYPED(name, ver, uint64_t) \ + BINARY_OP_TYPED(name, ver, int32_t) \ + BINARY_OP_TYPED(name, ver, int64_t) \ + BINARY_OP_HFD(name, ver) + +#define BINARY_OP_REGISTER_HFD(name, ver) \ + BINARY_ELEMENTWISE_REGISTER_KERNEL_TYPED(name, ver, MLFloat16) \ + BINARY_ELEMENTWISE_REGISTER_KERNEL_TYPED(name, ver, float) \ + BINARY_ELEMENTWISE_REGISTER_KERNEL_TYPED(name, ver, double) + +#define BINARY_OP_REGISTER_UZILHFD(name, ver) \ + BINARY_ELEMENTWISE_REGISTER_KERNEL_TYPED(name, ver, uint32_t) \ + BINARY_ELEMENTWISE_REGISTER_KERNEL_TYPED(name, ver, uint64_t) \ + BINARY_ELEMENTWISE_REGISTER_KERNEL_TYPED(name, ver, int32_t) \ + BINARY_ELEMENTWISE_REGISTER_KERNEL_TYPED(name, ver, int64_t) \ + BINARY_OP_REGISTER_HFD(name, ver) + +#define BINARY_OP_REGISTER_VERSIONED_HFD(name, startver, endver) \ + BINARY_ELEMENTWISE_REGISTER_KERNEL_VERSIONED_TYPED(name, startver, endver, MLFloat16) \ + BINARY_ELEMENTWISE_REGISTER_KERNEL_VERSIONED_TYPED(name, startver, endver, float) \ + BINARY_ELEMENTWISE_REGISTER_KERNEL_VERSIONED_TYPED(name, startver, endver, double) + +#define BINARY_OP_REGISTER_VERSIONED_UZILHFD(name, startver, endver) \ + BINARY_ELEMENTWISE_REGISTER_KERNEL_VERSIONED_TYPED(name, startver, endver, uint32_t) \ + BINARY_ELEMENTWISE_REGISTER_KERNEL_VERSIONED_TYPED(name, startver, endver, uint64_t) \ + BINARY_ELEMENTWISE_REGISTER_KERNEL_VERSIONED_TYPED(name, startver, endver, int32_t) \ + BINARY_ELEMENTWISE_REGISTER_KERNEL_VERSIONED_TYPED(name, startver, endver, int64_t) \ + BINARY_OP_REGISTER_VERSIONED_HFD(name, startver, endver) + +BINARY_OP_UZILHFD(Add, 7) +BINARY_OP_UZILHFD(Sub, 7) +BINARY_OP_UZILHFD(Mul, 7) +BINARY_OP_UZILHFD(Div, 7) +BINARY_OP_HFD(Pow, 7) +BINARY_OP_TYPED(And, 7, bool) +BINARY_OP_TYPED(Or, 7, bool) +BINARY_OP_TYPED(Xor, 7, bool) +BINARY_OP_HFD(PRelu, 7) + +template +Status Sum::ComputeInternal(OpKernelContext* context) const { + typedef typename ToCudaType::MappedType CudaT; + const auto& node = Node(); + const auto& node_name = node.Name(); + auto input_count = node.InputArgCount().front(); + ONNXRUNTIME_RETURN_IF_NOT(input_count >= 1, "Must have 1 or more inputs"); + + if (input_count == 1) { + auto input_tensor = context->Input(0); + const auto& input_shape = input_tensor->Shape(); + auto output_tensor = context->Output(0, input_shape); + CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(output_tensor->MutableDataRaw(), input_tensor->DataRaw(), sizeof(CudaT) * input_shape.Size(), cudaMemcpyDeviceToDevice)); + } else { + // compute output shape first, using broadcast rule + TensorShape output_shape; + ONNXRUNTIME_RETURN_IF_ERROR(ComputeOutputShape(node_name, context->Input(0)->Shape(), context->Input(1)->Shape(), output_shape)); + for (int index = 2; index < input_count; index++) { + TensorShape previous_output_shape = output_shape; + ONNXRUNTIME_RETURN_IF_ERROR(ComputeOutputShape(node_name, previous_output_shape, context->Input(index)->Shape(), output_shape)); + } + Tensor* output_tensor = context->Output(0, output_shape); + BinaryElementwisePreparation prepare(this); + if (input_count == 2) { + // special case for 2 tensors to avoid memset zero + ONNXRUNTIME_RETURN_IF_ERROR(BinaryElementwiseBroadcastPrepare(0, context->Input(0), context->Input(1), output_tensor, &prepare)); + Impl_Add( + prepare.output_rank_or_simple_broadcast, + prepare.lhs_padded_strides.GpuPtr(), + reinterpret_cast(prepare.lhs_tensor->template Data()), + prepare.rhs_padded_strides.GpuPtr(), + reinterpret_cast(prepare.rhs_tensor->template Data()), + prepare.fdm_output_strides.GpuPtr(), + prepare.fdm_H, + prepare.fdm_C, + reinterpret_cast(prepare.output_tensor->template MutableData()), + prepare.output_tensor->Shape().Size()); + } else { + // for more than 2 inputs, we need to accumulate into output tensor, as the shape from input0 + input1 might be different from output shape + CUDA_RETURN_IF_ERROR(cudaMemset(output_tensor->MutableDataRaw(), 0, output_shape.Size() * sizeof(CudaT))); + for (int index = 0; index < input_count; index++) { + ONNXRUNTIME_RETURN_IF_ERROR(BinaryElementwiseBroadcastPrepare(0, output_tensor, context->Input(index), output_tensor, &prepare)); + Impl_Add( + prepare.output_rank_or_simple_broadcast, + prepare.lhs_padded_strides.GpuPtr(), + reinterpret_cast(prepare.lhs_tensor->template Data()), + prepare.rhs_padded_strides.GpuPtr(), + reinterpret_cast(prepare.rhs_tensor->template Data()), + prepare.fdm_output_strides.GpuPtr(), + prepare.fdm_H, + prepare.fdm_C, + reinterpret_cast(prepare.output_tensor->template MutableData()), + prepare.output_tensor->Shape().Size()); + } + } + } + return Status::OK(); +} + +BINARY_OP_REGISTER_UZILHFD(Sum, 8) +BINARY_OP_REGISTER_VERSIONED_UZILHFD(Sum, 6, 7) +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/math/binary_elementwise_ops.h b/onnxruntime/core/providers/cuda/math/binary_elementwise_ops.h new file mode 100644 index 0000000000000..a8517d9ee83d6 --- /dev/null +++ b/onnxruntime/core/providers/cuda/math/binary_elementwise_ops.h @@ -0,0 +1,201 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/providers/cuda/cuda_common.h" +#include "core/providers/cuda/shared_inc/fast_divmod.h" +#include "core/providers/cpu/tensor/utils.h" + +namespace onnxruntime { +namespace cuda { + +struct BinaryElementwisePreparation { + const Tensor* lhs_tensor = nullptr; + const Tensor* rhs_tensor = nullptr; + Tensor* output_tensor = nullptr; + size_t output_rank_or_simple_broadcast = 0; // for no_broadcast|left_scalar|right_scalar cases, output_rank uses SimpleBroadcast enums + CudaKernel::CudaAsyncBuffer lhs_padded_strides; // for lhs shape == output shape, this is nullptr + CudaKernel::CudaAsyncBuffer rhs_padded_strides; // for rhs shape == output shape, this is nullptr + CudaKernel::CudaAsyncBuffer fdm_output_strides; + + // these are for RightPerChannel case + fast_divmod fdm_H; + fast_divmod fdm_C; + + BinaryElementwisePreparation(const CudaKernel* op_kernel) : lhs_padded_strides(op_kernel), + rhs_padded_strides(op_kernel), + fdm_output_strides(op_kernel) {} + + Status CopyToGpu() { + ONNXRUNTIME_RETURN_IF_ERROR(lhs_padded_strides.CopyToGpu()); + ONNXRUNTIME_RETURN_IF_ERROR(rhs_padded_strides.CopyToGpu()); + ONNXRUNTIME_RETURN_IF_ERROR(fdm_output_strides.CopyToGpu()); + return Status::OK(); + } + + Status BinaryElementwiseBroadcastPrepareHelper(int device_id, const TensorShape& lhs_shape, + const TensorShape& rhs_shape, + const TensorShape& output_shape) { + size_t lhs_rank = lhs_shape.NumDimensions(); + size_t rhs_rank = rhs_shape.NumDimensions(); + size_t out_rank = std::max(lhs_rank, rhs_rank); + + // early return when shapes match + if (lhs_shape == rhs_shape) { + output_rank_or_simple_broadcast = static_cast(SimpleBroadcast::NoBroadcast); + return Status::OK(); + } + + // early return if one operand is scalar + if (lhs_shape.Size() <= 1 || rhs_shape.Size() <= 1) { + output_rank_or_simple_broadcast = static_cast(lhs_shape.Size() <= 1 ? SimpleBroadcast::LeftScalar : SimpleBroadcast::RightScalar); + return Status::OK(); + } + + // special case for lhs(N,C,H) and rhs (C,1) which is used in conv bias + // when N == 1: out[id] = op(lhs[id], rhs[id / H]) + // When N > 1: out[id] = op(lhs[id], rhs[id / H % C]) + if (lhs_shape == output_shape) { + const auto& rhs_dims = rhs_shape.GetDims(); + int64_t C; + if (1 == std::count_if(rhs_dims.begin(), rhs_dims.end(), [&C](int64_t dim) { if (dim > 1) C = dim; return (dim > 1); })) { + auto dim_C = std::find(rhs_dims.begin(), rhs_dims.end(), C) - rhs_dims.begin() + output_shape.NumDimensions() - rhs_shape.NumDimensions(); + int64_t N = output_shape.SizeToDimension(dim_C); + int64_t H = (dim_C < out_rank - 1 ? output_shape.SizeFromDimension(dim_C + 1) : 1); + + std::vector new_output_dims; + if (N == 1) { + output_rank_or_simple_broadcast = static_cast(SimpleBroadcast::RightPerChannelBatch1); + fdm_H = fast_divmod(gsl::narrow_cast(H)); + } else { + output_rank_or_simple_broadcast = static_cast(SimpleBroadcast::RightPerChannelBatchN); + fdm_H = fast_divmod(gsl::narrow_cast(H)); + fdm_C = fast_divmod(gsl::narrow_cast(C)); + } + return Status::OK(); + } + } + + output_rank_or_simple_broadcast = out_rank; + + if (lhs_shape != output_shape) { + // compute strides with 1 more dim than out_rank, and use strides[0] == strides[1] + // to decide if dim0 needs broadcast + lhs_padded_strides.AllocCpuPtr(device_id, out_rank + 1); + ONNXRUNTIME_RETURN_IF_NOT(TensorPitches::Calculate(lhs_padded_strides.CpuSpan(), lhs_shape.GetDims())); + if (lhs_shape[0] > 1 && lhs_rank == out_rank) + lhs_padded_strides.CpuPtr()[0] = 0; + } + + if (rhs_shape != output_shape) { + rhs_padded_strides.AllocCpuPtr(device_id, out_rank + 1); + ONNXRUNTIME_RETURN_IF_NOT(TensorPitches::Calculate(rhs_padded_strides.CpuSpan(), rhs_shape.GetDims())); + if (rhs_shape[0] > 1 && rhs_rank == out_rank) + rhs_padded_strides.CpuPtr()[0] = 0; + } + + fdm_output_strides.AllocCpuPtr(device_id, out_rank); + ONNXRUNTIME_RETURN_IF_NOT(CalculateFdmStrides(fdm_output_strides.CpuSpan(), output_shape.GetDims())); + return Status::OK(); + } +}; + + +// trait classes to indicate if the kernel supports broadcast +class ShouldBroadcast { +}; + +class ShouldNotBroadcast { +}; + +template +class BinaryElementwise : public CudaKernel { + protected: + typedef BroadcastTrait broadcast_type; + + BinaryElementwise(const OpKernelInfo& info) : CudaKernel(info) {} + Status ComputeInternal(OpKernelContext*) const override { + return Status(common::ONNXRUNTIME, common::FAIL); // should not reach here + } + Status Prepare(OpKernelContext* context, int device_id, BinaryElementwisePreparation* p) const; +}; + +template +class Add final : public BinaryElementwise { + public: + Add(const OpKernelInfo& info) : BinaryElementwise(info) {} + Status ComputeInternal(OpKernelContext* context) const override; +}; + +template +class Sub final : public BinaryElementwise { + public: + Sub(const OpKernelInfo& info) : BinaryElementwise(info) {} + Status ComputeInternal(OpKernelContext* context) const override; +}; + +template +class Mul final : public BinaryElementwise { + public: + Mul(const OpKernelInfo& info) : BinaryElementwise(info) {} + Status ComputeInternal(OpKernelContext* context) const override; +}; + +template +class Div final : public BinaryElementwise { + public: + Div(const OpKernelInfo& info) : BinaryElementwise(info) {} + Status ComputeInternal(OpKernelContext* context) const override; +}; + +template +class Pow final : public BinaryElementwise { + public: + Pow(const OpKernelInfo& info) : BinaryElementwise(info) {} + Status ComputeInternal(OpKernelContext* context) const override; +}; + +template +class And final : public BinaryElementwise { + public: + And(const OpKernelInfo& info) : BinaryElementwise(info) {} + Status ComputeInternal(OpKernelContext* context) const override; +}; + +template +class Or final : public BinaryElementwise { + public: + Or(const OpKernelInfo& info) : BinaryElementwise(info) {} + Status ComputeInternal(OpKernelContext* context) const override; +}; + +template +class Xor final : public BinaryElementwise { + public: + Xor(const OpKernelInfo& info) : BinaryElementwise(info) {} + Status ComputeInternal(OpKernelContext* context) const override; +}; + +// PRelu is activation function, but it's closer to binary elementwise ops in implementation +template +class PRelu final : public BinaryElementwise { + public: + PRelu(const OpKernelInfo& info) : BinaryElementwise(info) { + } + + Status ComputeInternal(OpKernelContext* context) const override; +}; + +// Sum allows varadic inputs, and it uses binary elementwise Add in implementation +template +class Sum final : public CudaKernel { + public: + Sum(const OpKernelInfo& info) : CudaKernel(info) { + } + + Status ComputeInternal(OpKernelContext* context) const override; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/math/binary_elementwise_ops_impl.cu b/onnxruntime/core/providers/cuda/math/binary_elementwise_ops_impl.cu new file mode 100644 index 0000000000000..f0bb85cb686e7 --- /dev/null +++ b/onnxruntime/core/providers/cuda/math/binary_elementwise_ops_impl.cu @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include "binary_elementwise_ops_impl.h" +#include "core/providers/cuda/cu_inc/common.cuh" +#include "core/providers/cuda/cu_inc/binary_elementwise_impl.cuh" + +namespace onnxruntime { +namespace cuda { + +#define OP(name, expr) \ + template \ + struct OP_##name { \ + __device__ __inline__ T operator()(T a, T b) const { \ + return (expr); \ + } \ + }; + +#define BINARY_ELEMENTWISE_IMPL(name) \ + BINARY_ELEMENTWISE_IMPL_DECLARATION(name) { \ + BinaryElementWiseImpl(output_rank_or_simple_broadcast, \ + lhs_padded_strides, \ + lhs_data, \ + rhs_padded_strides, \ + rhs_data, \ + fdm_output_strides, \ + fdm_H, \ + fdm_C, \ + output_data, \ + OP_##name(), \ + count); \ + } + +#define SPECIALIZED_BINARY_ELEMENTWISE_IMPL(x, T) \ + template void Impl_##x(size_t output_rank, const int64_t* lhs_padded_strides, const T* lhs_data, const int64_t* rhs_padded_strides, const T* rhs_data, const fast_divmod* fdm_output_strides, const fast_divmod& fdm_H, const fast_divmod& fdm_C, T* output_data, size_t count); + +#define SPECIALIZED_BINARY_ELEMENTWISE_IMPL_UZILHFD(x) \ + SPECIALIZED_BINARY_ELEMENTWISE_IMPL(x, uint32_t) \ + SPECIALIZED_BINARY_ELEMENTWISE_IMPL(x, uint64_t) \ + SPECIALIZED_BINARY_ELEMENTWISE_IMPL(x, int32_t) \ + SPECIALIZED_BINARY_ELEMENTWISE_IMPL(x, int64_t) \ + SPECIALIZED_BINARY_ELEMENTWISE_IMPL(x, half) \ + SPECIALIZED_BINARY_ELEMENTWISE_IMPL(x, float) \ + SPECIALIZED_BINARY_ELEMENTWISE_IMPL(x, double) + +#define SPECIALIZED_BINARY_ELEMENTWISE_IMPL_HFD(x) \ + SPECIALIZED_BINARY_ELEMENTWISE_IMPL(x, half) \ + SPECIALIZED_BINARY_ELEMENTWISE_IMPL(x, float) \ + SPECIALIZED_BINARY_ELEMENTWISE_IMPL(x, double) + +// create declarations for op and impl +#define BINARY_OP_NAME_EXPR(name, expr) \ + OP(name, expr) \ + BINARY_ELEMENTWISE_IMPL(name) + +BINARY_OPS() +#undef BINARY_OP_NAME_EXPR + +// create specialized impl +// the postfix of means the types supported by the op: +// B: uint8_t +// W: uint16_t +// U: uint32_t +// Z: uint64_t +// C: int8_t +// S: int16_t +// I: int32_t +// L: int64_t +// H: float16 +// F: float +// D: double +// O: bool + +SPECIALIZED_BINARY_ELEMENTWISE_IMPL_UZILHFD(Add) +SPECIALIZED_BINARY_ELEMENTWISE_IMPL_UZILHFD(Sub) +SPECIALIZED_BINARY_ELEMENTWISE_IMPL_UZILHFD(Mul) +SPECIALIZED_BINARY_ELEMENTWISE_IMPL_UZILHFD(Div) +SPECIALIZED_BINARY_ELEMENTWISE_IMPL_HFD(Pow) +SPECIALIZED_BINARY_ELEMENTWISE_IMPL(And, bool) +SPECIALIZED_BINARY_ELEMENTWISE_IMPL(Or, bool) +SPECIALIZED_BINARY_ELEMENTWISE_IMPL(Xor, bool) +SPECIALIZED_BINARY_ELEMENTWISE_IMPL_HFD(PRelu) + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/math/binary_elementwise_ops_impl.h b/onnxruntime/core/providers/cuda/math/binary_elementwise_ops_impl.h new file mode 100644 index 0000000000000..1e76680b2ddd2 --- /dev/null +++ b/onnxruntime/core/providers/cuda/math/binary_elementwise_ops_impl.h @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include "core/providers/cuda/shared_inc/cuda_utils.h" + +namespace onnxruntime { +namespace cuda { + +// These macros simplifies coding. To add a new op with following steps: +// 1. Add a new entry in BINARY_OPS() list +// 2. (optional) Define templated single element operator in binary_elementwise_ops_impl.cu +// 3. (optional) Implement specialized single element operator +// 4. Add op kernel class definition in binary_elementwise_ops.h +// 5. Add op kernel registration and compute specialization in binary_elementwise_ops.cc + +#define BINARY_OPS() \ + BINARY_OP_NAME_EXPR(Add, (a + b)) \ + BINARY_OP_NAME_EXPR(Sub, (a - b)) \ + BINARY_OP_NAME_EXPR(Mul, (a * b)) \ + BINARY_OP_NAME_EXPR(Div, (a / b)) \ + BINARY_OP_NAME_EXPR(Pow, _Pow(a, b)) \ + BINARY_OP_NAME_EXPR(And, (a & b)) \ + BINARY_OP_NAME_EXPR(Or, (a | b)) \ + BINARY_OP_NAME_EXPR(Xor, (a ^ b)) \ + BINARY_OP_NAME_EXPR(PRelu, (a > (T)0 ? a : a * b)) + +// NOTE that cu files are compiled with nvcc and should not refer to any onnxruntime headers +// so struct BinaryElementwisePreparation cannot be used here + +#define BINARY_ELEMENTWISE_IMPL_DECLARATION(name) \ + template \ + void Impl_##name( \ + size_t output_rank_or_simple_broadcast, \ + const int64_t* lhs_padded_strides, \ + const T* lhs_data, \ + const int64_t* rhs_padded_strides, \ + const T* rhs_data, \ + const fast_divmod* fdm_output_strides, \ + const fast_divmod& fdm_H, \ + const fast_divmod& fdm_C, \ + T* output_data, \ + size_t count) + +#define BINARY_OP_NAME_EXPR(name, expr) BINARY_ELEMENTWISE_IMPL_DECLARATION(name); +BINARY_OPS() +#undef BINARY_OP_NAME_EXPR + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/math/gemm.cc b/onnxruntime/core/providers/cuda/math/gemm.cc new file mode 100644 index 0000000000000..9b4ef08678360 --- /dev/null +++ b/onnxruntime/core/providers/cuda/math/gemm.cc @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gemm.h" +#include "core/providers/cpu/math/gemm_helper.h" +#include "core/providers/cuda/cudnn_common.h" +#include "core/providers/cuda/shared_inc/fpgeneric.h" + +namespace onnxruntime { +namespace cuda { + +#define REGISTER_KERNEL_TYPED(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + Gemm, \ + kOnnxDomain, \ + 7, \ + T, \ + kCudaExecutionProvider, \ + KernelDefBuilder() \ + .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + Gemm); + +REGISTER_KERNEL_TYPED(float) +REGISTER_KERNEL_TYPED(double) +REGISTER_KERNEL_TYPED(MLFloat16) + +template +Status Gemm::ComputeInternal(OpKernelContext* ctx) const { + typedef typename ToCudaType::MappedType CudaT; + + const auto X = ctx->Input(0); + const auto W = ctx->Input(1); + const auto B = ctx->Input(2); + GemmHelper helper(X->Shape(), trans_A_, W->Shape(), trans_B_, B->Shape()); + + if (!helper.State().IsOK()) + return helper.State(); + + int M = gsl::narrow_cast(helper.M()); + int N = gsl::narrow_cast(helper.N()); + int K = gsl::narrow_cast(helper.K()); + auto Y = ctx->Output(0, TensorShape(std::vector{M, N})); + CudaT* out_data = reinterpret_cast(Y->template MutableData()); + + CudaT one = ToCudaType::FromFloat(1.0f); + CudaT zero = ToCudaType::FromFloat(0.0f); + + // broadcast bias if needed + if (beta_ != 0) { + auto& b_shape = B->Shape(); + const CudaT* b_data = reinterpret_cast(B->template Data()); + + if (b_shape.Size() == 1) { + // if B is (), (1,) or (1, 1), broadcast the scalar + CUBLAS_RETURN_IF_ERROR(cublasCopyHelper( + CublasHandle(), + M * N, + b_data, + 0, + out_data, + 1)); + } else if (b_shape.NumDimensions() == 1 || b_shape[0] == 1) { + // B is (N,) or (1, N), broadcast using Y(N,M) = 1 * B(N,1) x ones(1,M) + 0 * Y + CUBLAS_RETURN_IF_ERROR(cublasGemmHelper( + CublasHandle(), + CUBLAS_OP_N, + CUBLAS_OP_N, + N, M, 1, + /*alpha*/ &one, + b_data, N, + GetConstOnes(M), 1, + /*beta*/ &zero, + out_data, N)); + } else if (b_shape.NumDimensions() == 2 && b_shape[1] == 1) { + // B is (M, 1), broadcast using Y(N,M) = 1 * ones(N,1) x B(1,M) + 0 * Y + CUBLAS_RETURN_IF_ERROR(cublasGemmHelper( + CublasHandle(), + CUBLAS_OP_N, + CUBLAS_OP_N, + N, M, 1, + /*alpha*/ &one, + GetConstOnes(N), N, + b_data, 1, + /*beta*/ &zero, + out_data, N)); + } else { + // B is (M, N), no broadcast needed. + CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(out_data, b_data, M * N * sizeof(float), cudaMemcpyDeviceToDevice)); + } + } + + CudaT alpha = ToCudaType::FromFloat(alpha_); + CudaT beta = ToCudaType::FromFloat(beta_); + // Gemm, note that CUDA assumes col-major, so Y(N,M) = alpha * op(W) x op(X) + beta * Y + CUBLAS_RETURN_IF_ERROR(cublasGemmHelper( + CublasHandle(), + trans_B_ ? CUBLAS_OP_T : CUBLAS_OP_N, + trans_A_ ? CUBLAS_OP_T : CUBLAS_OP_N, + N, M, K, + &alpha, + reinterpret_cast(W->template Data()), + (trans_B_ ? K : N), + reinterpret_cast(X->template Data()), + (trans_A_ ? M : K), + &beta, + out_data, N)); + + return Status::OK(); +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/math/gemm.h b/onnxruntime/core/providers/cuda/math/gemm.h new file mode 100644 index 0000000000000..8c9895f1c42af --- /dev/null +++ b/onnxruntime/core/providers/cuda/math/gemm.h @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/providers/cuda/cuda_common.h" + +namespace onnxruntime { +namespace cuda { +template +class Gemm final : public CudaKernel { + using Base = CudaKernel; + + public: + Gemm(const OpKernelInfo& info) : CudaKernel(info) { + int64_t temp; + ONNXRUNTIME_ENFORCE(info.GetAttr("transA", &temp).IsOK()); + trans_A_ = (temp != 0); + + ONNXRUNTIME_ENFORCE(info.GetAttr("transB", &temp).IsOK()); + trans_B_ = (temp != 0); + + ONNXRUNTIME_ENFORCE(info.GetAttr("alpha", &alpha_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttr("beta", &beta_).IsOK()); + } + + Status ComputeInternal(OpKernelContext* context) const override; + + private: + bool trans_A_; + bool trans_B_; + float alpha_; + float beta_; +}; +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/math/matmul.cc b/onnxruntime/core/providers/cuda/math/matmul.cc new file mode 100644 index 0000000000000..7372ccd2b3380 --- /dev/null +++ b/onnxruntime/core/providers/cuda/math/matmul.cc @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "matmul.h" +#include "core/providers/cpu/math/matmul_helper.h" +#include "core/providers/cuda/shared_inc/fpgeneric.h" +#include "core/providers/cuda/cuda_allocator.h" + +namespace onnxruntime { +namespace cuda { + +#define REGISTER_KERNEL_TYPED(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + MatMul, \ + kOnnxDomain, \ + 1, \ + T, \ + kCudaExecutionProvider, \ + KernelDefBuilder() \ + .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + MatMul); + +REGISTER_KERNEL_TYPED(float) +REGISTER_KERNEL_TYPED(double) +REGISTER_KERNEL_TYPED(MLFloat16) + +template +Status MatMul::ComputeInternal(OpKernelContext* ctx) const { + typedef typename ToCudaType::MappedType CudaT; + + const Tensor* left_X = ctx->Input(0); + const Tensor* right_X = ctx->Input(1); + + MatMulComputeHelper helper; + ONNXRUNTIME_RETURN_IF_ERROR(helper.Compute(left_X->Shape(), right_X->Shape())); + + Tensor* Y = ctx->Output(0, helper.OutputShape()); + ONNXRUNTIME_RETURN_IF_NOT(strcmp(Y->Location().name, CUDA) == 0, "Output should be allocated on CUDA"); + + CudaT one = ToCudaType::FromFloat(1.0f); + CudaT zero = ToCudaType::FromFloat(0.0f); + + if (helper.OutputOffsets().size() == 1) { + CUBLAS_RETURN_IF_ERROR(cublasGemmHelper( + Base::CublasHandle(), + CUBLAS_OP_N, + CUBLAS_OP_N, + static_cast(helper.N()), + static_cast(helper.M()), + static_cast(helper.K()), + &one, + reinterpret_cast(right_X->template Data()), + static_cast(helper.N()), + reinterpret_cast(left_X->template Data()), + static_cast(helper.K()), + &zero, + reinterpret_cast(Y->template MutableData()), + static_cast(helper.N()))); + return Status::OK(); + } + int device_id = 0; + CudaAsyncBuffer left_arrays(this, device_id, helper.LeftOffsets().size()); + CudaAsyncBuffer right_arrays(this, device_id, helper.RightOffsets().size()); + CudaAsyncBuffer output_arrays(this, device_id, helper.OutputOffsets().size()); + MatMulComputeHelper::OffsetToArrays(reinterpret_cast(left_X->template Data()), helper.LeftOffsets(), left_arrays.CpuSpan()); + MatMulComputeHelper::OffsetToArrays(reinterpret_cast(right_X->template Data()), helper.RightOffsets(), right_arrays.CpuSpan()); + MatMulComputeHelper::OffsetToArrays(reinterpret_cast(Y->template MutableData()), helper.OutputOffsets(), output_arrays.CpuSpan()); + ONNXRUNTIME_RETURN_IF_ERROR(left_arrays.CopyToGpu()); + ONNXRUNTIME_RETURN_IF_ERROR(right_arrays.CopyToGpu()); + ONNXRUNTIME_RETURN_IF_ERROR(output_arrays.CopyToGpu()); + + // note that onnxruntime MLValue is row major, while cublas is column major, + // so swap left/right operands + CUBLAS_RETURN_IF_ERROR(cublasGemmBatchedHelper( + Base::CublasHandle(), + CUBLAS_OP_N, + CUBLAS_OP_N, + static_cast(helper.N()), + static_cast(helper.M()), + static_cast(helper.K()), + &one, + right_arrays.GpuPtr(), + static_cast(helper.N()), + left_arrays.GpuPtr(), + static_cast(helper.K()), + &zero, + output_arrays.GpuPtr(), + static_cast(helper.N()), + static_cast(helper.OutputOffsets().size()))); + + return Status::OK(); +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/math/matmul.h b/onnxruntime/core/providers/cuda/math/matmul.h new file mode 100644 index 0000000000000..98a95988da603 --- /dev/null +++ b/onnxruntime/core/providers/cuda/math/matmul.h @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/providers/cuda/cuda_common.h" + +namespace onnxruntime { +namespace cuda { +template +class MatMul final : public CudaKernel { + using Base = CudaKernel; + + public: + MatMul(const OpKernelInfo& info) + : CudaKernel(info) { + } + + Status ComputeInternal(OpKernelContext* context) const override; +}; +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/math/softmax.cc b/onnxruntime/core/providers/cuda/math/softmax.cc new file mode 100644 index 0000000000000..b444e2a25bba0 --- /dev/null +++ b/onnxruntime/core/providers/cuda/math/softmax.cc @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "softmax.h" +#include "core/providers/common.h" +#include "core/providers/cuda/cudnn_common.h" + +namespace onnxruntime { +namespace cuda { + +#define REGISTER_KERNEL_TYPED(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + Softmax, \ + kOnnxDomain, \ + 1, \ + T, \ + kCudaExecutionProvider, \ + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + Softmax); + +template +Status Softmax::ComputeInternal(OpKernelContext* ctx) const { + typedef typename ToCudaType::MappedType CudaT; + const Tensor& X = *ctx->Input(0); + const TensorShape input_shape{X.Shape()}; + + Tensor* Y = ctx->Output(0, input_shape); + + const int64_t axis = HandleNegativeAxis(axis_, input_shape.NumDimensions()); + + int64_t N = input_shape.SizeToDimension(axis); + int64_t D = input_shape.SizeFromDimension(axis); + std::vector dims({N, 1, 1, D}); // cudnn expects 4D shape in NCHW format + + auto y_data = reinterpret_cast(Y->template MutableData()); + auto x_data = reinterpret_cast(X.template Data()); + + const auto alpha = Consts::One; + const auto beta = Consts::Zero; + CudnnTensor input_tensor; + CudnnTensor output_tensor; + ONNXRUNTIME_RETURN_IF_ERROR(input_tensor.Set(dims, CudnnTensor::GetDataType())); + ONNXRUNTIME_RETURN_IF_ERROR(output_tensor.Set(dims, CudnnTensor::GetDataType())); + CUDNN_RETURN_IF_ERROR(cudnnSoftmaxForward(CudnnHandle(), CUDNN_SOFTMAX_ACCURATE, CUDNN_SOFTMAX_MODE_INSTANCE, &alpha, input_tensor, x_data, &beta, output_tensor, y_data)); + + return Status::OK(); +} + +#define SPECIALIZED_COMPUTE(T) \ + REGISTER_KERNEL_TYPED(T) \ + template Status Softmax::ComputeInternal(OpKernelContext* ctx) const; + +SPECIALIZED_COMPUTE(float) +SPECIALIZED_COMPUTE(double) +SPECIALIZED_COMPUTE(MLFloat16) + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/math/softmax.h b/onnxruntime/core/providers/cuda/math/softmax.h new file mode 100644 index 0000000000000..a12ca8be287ad --- /dev/null +++ b/onnxruntime/core/providers/cuda/math/softmax.h @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "gsl/gsl_util" +#include "core/providers/cuda/cuda_common.h" + +namespace onnxruntime { +namespace cuda { + +template +class Softmax final : public CudaKernel { + public: + Softmax(const OpKernelInfo& info) : CudaKernel{info} { + info.GetAttrOrDefault("axis", &axis_, static_cast(1)); + } + + Status ComputeInternal(OpKernelContext* context) const override; + + private: + int64_t axis_; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/math/unary_elementwise_ops.cc b/onnxruntime/core/providers/cuda/math/unary_elementwise_ops.cc new file mode 100644 index 0000000000000..de118b9ee742f --- /dev/null +++ b/onnxruntime/core/providers/cuda/math/unary_elementwise_ops.cc @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "unary_elementwise_ops.h" +#include "unary_elementwise_ops_impl.h" + +namespace onnxruntime { +namespace cuda { + +Status UnaryElementwise::Prepare(OpKernelContext* context, UnaryElementwisePreparation* p) const { + p->input_tensor = context->Input(0); + p->output_tensor = context->Output(0, p->input_tensor->Shape()); + return Status::OK(); +} + +#define UNARY_ELEMENTWISE_REGISTER_KERNEL(x, ver, T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + x, \ + kOnnxDomain, \ + ver, \ + T, \ + kCudaExecutionProvider, \ + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + x); + +#define UNARY_ELEMENTWISE_COMPUTE(x, T) \ + template <> \ + Status x::ComputeInternal(OpKernelContext* context) const { \ + UnaryElementwisePreparation p; \ + UnaryElementwise::Prepare(context, &p); \ + Impl_##x( \ + reinterpret_cast::MappedType*>(p.input_tensor->template Data()), \ + reinterpret_cast::MappedType*>(p.output_tensor->template MutableData()),\ + p.output_tensor->Shape().Size()); \ + \ + return Status::OK(); \ + } + +#define UNARY_OP_TYPED(name, ver, T) \ + UNARY_ELEMENTWISE_REGISTER_KERNEL(name, ver, T) \ + UNARY_ELEMENTWISE_COMPUTE(name, T) + +// the postfix of means the types supported by the op: +// B: uint8_t +// W: uint16_t +// U: uint32_t +// Z: uint64_t +// C: int8_t +// S: int16_t +// I: int32_t +// L: int64_t +// H: float16 +// F: float +// D: double +// O: bool + +#define UNARY_OP_HFD(name, ver) \ + UNARY_OP_TYPED(name, ver, MLFloat16) \ + UNARY_OP_TYPED(name, ver, float) \ + UNARY_OP_TYPED(name, ver, double) + +#define UNARY_OP_CSILHFD(name, ver) \ + UNARY_OP_TYPED(name, ver, int8_t) \ + UNARY_OP_TYPED(name, ver, int16_t) \ + UNARY_OP_TYPED(name, ver, int32_t) \ + UNARY_OP_TYPED(name, ver, int64_t) \ + UNARY_OP_HFD(name, ver) + +#define UNARY_OP_BWUZCSILHFD(name, ver) \ + UNARY_OP_TYPED(name, ver, uint8_t) \ + UNARY_OP_TYPED(name, ver, uint16_t) \ + UNARY_OP_TYPED(name, ver, uint32_t) \ + UNARY_OP_TYPED(name, ver, uint64_t) \ + UNARY_OP_CSILHFD(name, ver) + +UNARY_OP_BWUZCSILHFD(Abs, 6) +UNARY_OP_CSILHFD(Neg, 6) +UNARY_OP_HFD(Floor, 6) +UNARY_OP_HFD(Ceil, 6) +UNARY_OP_HFD(Reciprocal, 6) +UNARY_OP_HFD(Sqrt, 6) +UNARY_OP_HFD(Log, 6) +UNARY_OP_HFD(Exp, 6) + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/math/unary_elementwise_ops.h b/onnxruntime/core/providers/cuda/math/unary_elementwise_ops.h new file mode 100644 index 0000000000000..5b719ae4e87e4 --- /dev/null +++ b/onnxruntime/core/providers/cuda/math/unary_elementwise_ops.h @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/providers/cuda/cuda_common.h" + +namespace onnxruntime { +namespace cuda { + +struct UnaryElementwisePreparation { + const Tensor* input_tensor = nullptr; + Tensor* output_tensor = nullptr; +}; + +class UnaryElementwise : public CudaKernel { + protected: + UnaryElementwise(const OpKernelInfo& info) : CudaKernel(info) {} + Status ComputeInternal(OpKernelContext*) const override { + return Status(common::ONNXRUNTIME, common::FAIL); // should not reach here + } + Status Prepare(OpKernelContext* context, UnaryElementwisePreparation* p) const; +}; + +template +class Abs final : public UnaryElementwise { + public: + Abs(const OpKernelInfo& info) : UnaryElementwise(info) {} + Status ComputeInternal(OpKernelContext* context) const override; +}; + +template +class Neg final : public UnaryElementwise { + public: + Neg(const OpKernelInfo& info) : UnaryElementwise(info) {} + Status ComputeInternal(OpKernelContext* context) const override; +}; + +template +class Floor final : public UnaryElementwise { + public: + Floor(const OpKernelInfo& info) : UnaryElementwise(info) {} + Status ComputeInternal(OpKernelContext* context) const override; +}; + +template +class Ceil final : public UnaryElementwise { + public: + Ceil(const OpKernelInfo& info) : UnaryElementwise(info) {} + Status ComputeInternal(OpKernelContext* context) const override; +}; + +template +class Reciprocal final : public UnaryElementwise { + public: + Reciprocal(const OpKernelInfo& info) : UnaryElementwise(info) {} + Status ComputeInternal(OpKernelContext* context) const override; +}; + +template +class Sqrt final : public UnaryElementwise { + public: + Sqrt(const OpKernelInfo& info) : UnaryElementwise(info) {} + Status ComputeInternal(OpKernelContext* context) const override; +}; + +template +class Log final : public UnaryElementwise { + public: + Log(const OpKernelInfo& info) : UnaryElementwise(info) {} + Status ComputeInternal(OpKernelContext* context) const override; +}; + +template +class Exp final : public UnaryElementwise { + public: + Exp(const OpKernelInfo& info) : UnaryElementwise(info) {} + Status ComputeInternal(OpKernelContext* context) const override; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/math/unary_elementwise_ops_impl.cu b/onnxruntime/core/providers/cuda/math/unary_elementwise_ops_impl.cu new file mode 100644 index 0000000000000..49f05b3a75900 --- /dev/null +++ b/onnxruntime/core/providers/cuda/math/unary_elementwise_ops_impl.cu @@ -0,0 +1,143 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include "unary_elementwise_ops_impl.h" +#include "core/providers/cuda/cu_inc/common.cuh" +#include "core/providers/cuda/cu_inc/unary_elementwise_impl.cuh" + +namespace onnxruntime { +namespace cuda { + +#define OP(name, expr) \ + template \ + struct OP_##name { \ + __device__ __inline__ T operator()(const T& a) const { \ + return expr; \ + } \ + }; + +#define UNARY_ELEMENTWISE_IMPL(name) \ + UNARY_ELEMENTWISE_IMPL_DECLARATION(name) { \ + UnaryElementWiseImpl(input_data, \ + output_data, \ + OP_##name(), \ + count); \ + } + +#define SPECIALIZED_UNARY_ELEMENTWISE_IMPL(name, T) \ + template void Impl_##name(const T* input_data, T* output_data, size_t count); + +#define UNARY_OP_NAME_EXPR(name, expr) \ + OP(name, expr) \ + UNARY_ELEMENTWISE_IMPL(name) + +UNARY_OPS() +#undef UNARY_OP_NAME_EXPR + +// the postfix of means the types supported by the op: +// B: uint8_t +// W: uint16_t +// U: uint32_t +// Z: uint64_t +// C: int8_t +// S: int16_t +// I: int32_t +// L: int64_t +// H: float16 +// F: float +// D: double +// O: bool + +#define SPECIALIZED_UNARY_ELEMENTWISE_IMPL_HFD(name) \ + SPECIALIZED_UNARY_ELEMENTWISE_IMPL(name, half) \ + SPECIALIZED_UNARY_ELEMENTWISE_IMPL(name, float) \ + SPECIALIZED_UNARY_ELEMENTWISE_IMPL(name, double) + +#define SPECIALIZED_UNARY_ELEMENTWISE_IMPL_CSILHFD(name) \ + SPECIALIZED_UNARY_ELEMENTWISE_IMPL(name, int8_t) \ + SPECIALIZED_UNARY_ELEMENTWISE_IMPL(name, int16_t) \ + SPECIALIZED_UNARY_ELEMENTWISE_IMPL(name, int32_t) \ + SPECIALIZED_UNARY_ELEMENTWISE_IMPL(name, int64_t) \ + SPECIALIZED_UNARY_ELEMENTWISE_IMPL_HFD(name) + +#define SPECIALIZED_UNARY_ELEMENTWISE_IMPL_BWUZCSILHFD(name) \ + SPECIALIZED_UNARY_ELEMENTWISE_IMPL(name, uint8_t) \ + SPECIALIZED_UNARY_ELEMENTWISE_IMPL(name, uint16_t) \ + SPECIALIZED_UNARY_ELEMENTWISE_IMPL(name, uint32_t) \ + SPECIALIZED_UNARY_ELEMENTWISE_IMPL(name, uint64_t) \ + SPECIALIZED_UNARY_ELEMENTWISE_IMPL_CSILHFD(name) + +SPECIALIZED_UNARY_ELEMENTWISE_IMPL_BWUZCSILHFD(Abs) +SPECIALIZED_UNARY_ELEMENTWISE_IMPL_CSILHFD(Neg) +SPECIALIZED_UNARY_ELEMENTWISE_IMPL_HFD(Floor) +SPECIALIZED_UNARY_ELEMENTWISE_IMPL_HFD(Ceil) +SPECIALIZED_UNARY_ELEMENTWISE_IMPL_HFD(Reciprocal) +SPECIALIZED_UNARY_ELEMENTWISE_IMPL_HFD(Sqrt) +SPECIALIZED_UNARY_ELEMENTWISE_IMPL_HFD(Log) +SPECIALIZED_UNARY_ELEMENTWISE_IMPL_HFD(Exp) + +// When casting, half needs to be converted via float type from most other types +template +struct ViaTypeMap { + typedef T ViaT; +}; + +template <> +struct ViaTypeMap { + typedef float ViaT; +}; + +template +struct OP_Cast { + __device__ __inline__ OutT operator()(const InT& a) const { + const bool any_float16 = std::is_same::value || std::is_same::value; + typedef typename std::conditional::type T; + typedef typename ViaTypeMap::ViaT ViaT; + return (OutT)((ViaT)a); + } +}; + +template +void Impl_Cast( + const InT* input_data, + OutT* output_data, + size_t count) { + UnaryElementWiseImpl(input_data, + output_data, + OP_Cast(), + count); +} + +#define SPECIALIZED_CAST_IMPL2(InT, OutT) \ + template void Impl_Cast(const InT* input_data, OutT* output_data, size_t count); + +#define SPECIALIZED_CAST_FROM(T) \ + SPECIALIZED_CAST_IMPL2(T, half) \ + SPECIALIZED_CAST_IMPL2(T, float) \ + SPECIALIZED_CAST_IMPL2(T, double) \ + SPECIALIZED_CAST_IMPL2(T, int8_t) \ + SPECIALIZED_CAST_IMPL2(T, int16_t) \ + SPECIALIZED_CAST_IMPL2(T, int32_t) \ + SPECIALIZED_CAST_IMPL2(T, int64_t) \ + SPECIALIZED_CAST_IMPL2(T, uint8_t) \ + SPECIALIZED_CAST_IMPL2(T, uint16_t) \ + SPECIALIZED_CAST_IMPL2(T, uint32_t) \ + SPECIALIZED_CAST_IMPL2(T, uint64_t) \ + SPECIALIZED_CAST_IMPL2(T, bool) + +SPECIALIZED_CAST_FROM(half) +SPECIALIZED_CAST_FROM(float) +SPECIALIZED_CAST_FROM(double) +SPECIALIZED_CAST_FROM(int8_t) +SPECIALIZED_CAST_FROM(int16_t) +SPECIALIZED_CAST_FROM(int32_t) +SPECIALIZED_CAST_FROM(int64_t) +SPECIALIZED_CAST_FROM(uint8_t) +SPECIALIZED_CAST_FROM(uint16_t) +SPECIALIZED_CAST_FROM(uint32_t) +SPECIALIZED_CAST_FROM(uint64_t) +SPECIALIZED_CAST_FROM(bool) + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/math/unary_elementwise_ops_impl.h b/onnxruntime/core/providers/cuda/math/unary_elementwise_ops_impl.h new file mode 100644 index 0000000000000..db990bc692c5b --- /dev/null +++ b/onnxruntime/core/providers/cuda/math/unary_elementwise_ops_impl.h @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +namespace onnxruntime { +namespace cuda { + +// This macro simplifies coding to add a new op with following steps: +// 1. Add a new entry in UNARY_OPS() list +// 2. (optional) Define templated single element operator in unary_elementwise_ops_impl.cu +// 3. (optional) Implement specialized single element operator +// 4. Add op kernel class definition in unary_elementwise_ops.h +// 5. Add op kernel registration and compute specialization in unary_elementwise_ops.cc + +#define UNARY_OPS() \ + UNARY_OP_NAME_EXPR(Abs, _Abs(a)) \ + UNARY_OP_NAME_EXPR(Neg, -a) \ + UNARY_OP_NAME_EXPR(Ceil, _Ceil(a)) \ + UNARY_OP_NAME_EXPR(Floor, _Floor(a)) \ + UNARY_OP_NAME_EXPR(Reciprocal, T(1) / a) \ + UNARY_OP_NAME_EXPR(Sqrt, _Sqrt(a)) \ + UNARY_OP_NAME_EXPR(Exp, _Exp(a)) \ + UNARY_OP_NAME_EXPR(Log, _Log(a)) + +#define UNARY_ELEMENTWISE_IMPL_DECLARATION(name) \ + template \ + void Impl_##name( \ + const T* input_data, \ + T* output_data, \ + size_t count) + +#define UNARY_OP_NAME_EXPR(name, expr) UNARY_ELEMENTWISE_IMPL_DECLARATION(name); +UNARY_OPS() +#undef UNARY_OP_NAME_EXPR + +template +void Impl_Cast( + const InT* input_data, + OutT* output_data, + size_t count); + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/nn/batch_norm.cc b/onnxruntime/core/providers/cuda/nn/batch_norm.cc new file mode 100644 index 0000000000000..0cc7623aafb03 --- /dev/null +++ b/onnxruntime/core/providers/cuda/nn/batch_norm.cc @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "batch_norm.h" +#include "core/providers/common.h" +#include "core/providers/cuda/cudnn_common.h" +#include "core/providers/cpu/nn/batch_norm_helper.h" +using namespace std; +namespace onnxruntime { +namespace cuda { + +#define REGISTER_KERNEL_TYPED(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + BatchNormalization, \ + kOnnxDomain, \ + 7, \ + T, \ + kCudaExecutionProvider, \ + KernelDefBuilder() \ + .TypeConstraint("X", DataTypeImpl::GetTensorType()) \ + .TypeConstraint("scale", DataTypeImpl::GetTensorType()) \ + .TypeConstraint("B", DataTypeImpl::GetTensorType()) \ + .TypeConstraint("mean", DataTypeImpl::GetTensorType()) \ + .TypeConstraint("var", DataTypeImpl::GetTensorType()), \ + BatchNorm); + +template +Status BatchNorm::ComputeInternal(OpKernelContext* p_op_kernel_context) const { + typedef typename ToCudaType::MappedType CudaT; + + const Tensor* X = p_op_kernel_context->Input(0); + const Tensor* scale = p_op_kernel_context->Input(1); + const Tensor* B = p_op_kernel_context->Input(2); + const Tensor* mean = p_op_kernel_context->Input(3); + const Tensor* var = p_op_kernel_context->Input(4); + + ONNXRUNTIME_RETURN_IF_ERROR(BatchNormHelper::ValidateInputs(X, scale, B, mean, var)); + + const TensorShape& x_shape = X->Shape(); + Tensor* Y = p_op_kernel_context->Output(0, x_shape); + + auto y_data = reinterpret_cast(Y->template MutableData()); + auto x_data = reinterpret_cast(X->template Data()); + auto scale_data = reinterpret_cast(scale->template Data()); + auto b_data = reinterpret_cast(B->template Data()); + auto mean_data = reinterpret_cast(mean->template Data()); + auto var_data = reinterpret_cast(var->template Data()); + + CudnnTensor data_desc; + vector new_dims; + BatchNormHelper::NormalizeDims(x_shape, new_dims); + ONNXRUNTIME_RETURN_IF_ERROR(data_desc.Set(new_dims, CudnnTensor::GetDataType())); + + CudnnTensor bn_tensor_desc; + ONNXRUNTIME_RETURN_IF_ERROR(bn_tensor_desc.Set(data_desc, cudnn_batch_norm_mode_)); + + const auto alpha = Consts::One; + const auto beta = Consts::Zero; + + CUDNN_RETURN_IF_ERROR(cudnnBatchNormalizationForwardInference( + CudnnHandle(), + cudnn_batch_norm_mode_, + &alpha, + &beta, + data_desc, + x_data, + data_desc, + y_data, + bn_tensor_desc, + scale_data, + b_data, + mean_data, + var_data, + epsilon_)); + + return Status::OK(); +} + +#define SPECIALIZED_COMPUTE(T) \ + REGISTER_KERNEL_TYPED(T) \ + template Status BatchNorm::ComputeInternal(OpKernelContext* ctx) const; + +SPECIALIZED_COMPUTE(float) +SPECIALIZED_COMPUTE(double) +SPECIALIZED_COMPUTE(MLFloat16) + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/nn/batch_norm.h b/onnxruntime/core/providers/cuda/nn/batch_norm.h new file mode 100644 index 0000000000000..162ef6d6fe693 --- /dev/null +++ b/onnxruntime/core/providers/cuda/nn/batch_norm.h @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "gsl/gsl_util" +#include "core/providers/cuda/cudnn_common.h" + +namespace onnxruntime { +namespace cuda { + +template +class BatchNorm final : public CudaKernel { + public: + BatchNorm(const OpKernelInfo& op_kernel_info) + : CudaKernel{op_kernel_info}, + cudnn_batch_norm_mode_(CUDNN_BATCHNORM_SPATIAL) { + float tmp_epsilon; + ONNXRUNTIME_ENFORCE(op_kernel_info.GetAttr("epsilon", &tmp_epsilon).IsOK()); + epsilon_ = ClampCudnnBatchNormEpsilon(tmp_epsilon); + + // spatial or not + int64_t tmp_spatial; + if (op_kernel_info.GetAttr("spatial", &tmp_spatial).IsOK()) { + spatial_ = tmp_spatial; + } + + if (spatial_ == 0) { + cudnn_batch_norm_mode_ = CUDNN_BATCHNORM_PER_ACTIVATION; // TODO add test case for this when implemented in CPU as well. + } + } + + Status ComputeInternal(OpKernelContext* context) const override; + + private: + double epsilon_; + int64_t spatial_ = 1; // default as per spec + cudnnBatchNormMode_t cudnn_batch_norm_mode_; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/nn/conv.cc b/onnxruntime/core/providers/cuda/nn/conv.cc new file mode 100644 index 0000000000000..192acfcc2ba83 --- /dev/null +++ b/onnxruntime/core/providers/cuda/nn/conv.cc @@ -0,0 +1,224 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/common.h" +#include "core/providers/cuda/cuda_common.h" +#include "core/providers/cuda/nn/conv.h" +#include "core/providers/cuda/shared_inc/fpgeneric.h" + +namespace onnxruntime { +namespace cuda { + +#define REGISTER_KERNEL_TYPED(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + Conv, \ + kOnnxDomain, \ + 1, \ + T, \ + kCudaExecutionProvider, \ + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + Conv); + +REGISTER_KERNEL_TYPED(float) +REGISTER_KERNEL_TYPED(double) +REGISTER_KERNEL_TYPED(MLFloat16) + +template +Status Conv::ComputeInternal(OpKernelContext* context) const { + typedef typename ToCudaType::MappedType CudaT; + + const Tensor* X = context->Input(0); + const TensorShape& x_shape = X->Shape(); + const auto& x_dims = x_shape.GetDims(); + auto x_data = reinterpret_cast(X->template Data()); + + const Tensor* W = context->Input(1); + const TensorShape& w_shape = W->Shape(); + std::vector w_dims = w_shape.GetDims(); + auto w_data = reinterpret_cast(W->template Data()); + + size_t num_inputs = OpKernel::Node().InputDefs().size(); + bool has_bias = (num_inputs == 3); + + CudaT* y_data = nullptr; + + { + std::lock_guard lock(s_.mutex); + // TODO: add a global cache if need to handle cases for multiple frames running simultaneuously with different batch_size + bool input_dims_changed = (s_.last_x_dims != x_dims); + bool w_dims_changed = (s_.last_w_dims != w_dims); + if (input_dims_changed || w_dims_changed) { + if (input_dims_changed) + s_.last_x_dims = x_dims; + + if (w_dims_changed) + s_.last_w_dims = w_dims; + + const int64_t N = X->Shape()[0]; + const int64_t M = W->Shape()[0]; + + ONNXRUNTIME_RETURN_IF_ERROR(ValidateInputShape(X, W)); + + std::vector kernel_shape = ComputeKernelShape(W->Shape()); + auto rank = kernel_shape.size(); + std::vector pads(pads_); + if (pads.empty()) { + pads.resize(rank * 2, 0); + } + std::vector dilations(dilations_); + if (dilations.empty()) { + dilations.resize(rank, 1); + } + std::vector strides(strides_); + if (strides.empty()) { + strides.resize(rank, 1); + } + + std::vector y_dims; + y_dims.insert(y_dims.begin(), {N, M}); + ONNXRUNTIME_RETURN_IF_ERROR(InferOutputShape(x_shape.Slice(2), kernel_shape, strides, dilations, &pads, &y_dims)); + s_.y_dims = y_dims; + + std::vector x_dims_cudnn = x_dims; + std::vector y_dims_cudnn = y_dims; + if (rank < 2) { + // cudnn only takes 4D or 5D input, so pad dimensions if needed + x_dims_cudnn.push_back(1); + y_dims_cudnn.push_back(1); + w_dims.push_back(1); + pads.insert(pads.begin() + rank, 0); + pads.insert(pads.end(), 0); + kernel_shape.push_back(1); + strides.push_back(1); + dilations.push_back(1); + } + ONNXRUNTIME_RETURN_IF_ERROR(s_.x_tensor.Set(x_dims_cudnn, CudnnTensor::GetDataType())); + ONNXRUNTIME_RETURN_IF_ERROR(s_.y_tensor.Set(y_dims_cudnn, CudnnTensor::GetDataType())); + + if (w_dims_changed) + ONNXRUNTIME_RETURN_IF_ERROR(s_.filter_desc.Set(w_dims, CudnnTensor::GetDataType())); + + cudnnConvolutionMode_t mode = CUDNN_CROSS_CORRELATION; + ONNXRUNTIME_RETURN_IF_ERROR(s_.conv_desc.Set(kernel_shape.size(), pads, strides, dilations, mode, CudnnTensor::GetDataType())); + CUDNN_RETURN_IF_ERROR(cudnnSetConvolutionGroupCount(s_.conv_desc, gsl::narrow_cast(group_))); + + IAllocatorUniquePtr algo_search_workspace = GetScratchBuffer(AlgoSearchWorkspaceSize); + + if (has_bias) { + const Tensor* B = context->Input(2); + const auto& b_shape = B->Shape(); + ONNXRUNTIME_RETURN_IF_NOT(b_shape.NumDimensions() == 1, "bias should be 1D"); + std::vector b_dims(2 + kernel_shape.size()); + b_dims[0] = 1; // N + b_dims[1] = b_shape[0]; // C + for (int i = 0; i < kernel_shape.size(); i++) + b_dims[2 + i] = 1; + + ONNXRUNTIME_RETURN_IF_ERROR(s_.b_tensor.Set(b_dims, CudnnTensor::GetDataType())); + } + + Tensor* Y = context->Output(0, TensorShape(s_.y_dims)); + y_data = reinterpret_cast(Y->template MutableData()); + + // set math type to tensor core before algorithm search + if (std::is_same::value) + CUDNN_RETURN_IF_ERROR(cudnnSetConvolutionMathType(s_.conv_desc, CUDNN_TENSOR_OP_MATH)); + + cudnnConvolutionFwdAlgoPerf_t perf; + int algo_count = 1; + CUDNN_RETURN_IF_ERROR(cudnnFindConvolutionForwardAlgorithmEx( + CudnnHandle(), + s_.x_tensor, + x_data, + s_.filter_desc, + w_data, + s_.conv_desc, + s_.y_tensor, + y_data, + 1, + &algo_count, + &perf, + algo_search_workspace.get(), + AlgoSearchWorkspaceSize)); + CUDNN_RETURN_IF_ERROR(cudnnSetConvolutionMathType(s_.conv_desc, perf.mathType)); + s_.algo = perf.algo; + s_.workspace_bytes = perf.memory; + } + } + + if (!y_data) { + Tensor* Y = context->Output(0, TensorShape(s_.y_dims)); + y_data = reinterpret_cast(Y->template MutableData()); + } + + const auto alpha = Consts::One; + const auto beta = Consts::Zero; + + IAllocatorUniquePtr workspace = GetScratchBuffer(s_.workspace_bytes); + + CUDNN_RETURN_IF_ERROR(cudnnConvolutionForward(CudnnHandle(), + &alpha, + s_.x_tensor, + x_data, + s_.filter_desc, + w_data, + s_.conv_desc, + s_.algo, + workspace.get(), + s_.workspace_bytes, + &beta, + s_.y_tensor, + y_data)); + + if (has_bias) { + const Tensor* B = context->Input(2); + auto b_data = reinterpret_cast(B->template Data()); + CUDNN_RETURN_IF_ERROR(cudnnAddTensor(CudnnHandle(), &alpha, s_.b_tensor, b_data, &alpha, s_.y_tensor, y_data)); + } + + return Status::OK(); +} + +CudnnConvolutionDescriptor::CudnnConvolutionDescriptor() : desc_(nullptr) { +} + +CudnnConvolutionDescriptor::~CudnnConvolutionDescriptor() { + if (desc_ != nullptr) { + cudnnDestroyConvolutionDescriptor(desc_); + desc_ = nullptr; + } +} + +Status CudnnConvolutionDescriptor::Set( + size_t rank, + const std::vector& pads, + const std::vector& strides, + const std::vector& dilations, + cudnnConvolutionMode_t mode, + cudnnDataType_t data_type) { + if (!desc_) + CUDNN_RETURN_IF_ERROR(cudnnCreateConvolutionDescriptor(&desc_)); + + std::vector pad_dims(rank); + std::vector stride_dims(rank); + std::vector dilation_dims(rank); + for (int i = 0; i < rank; i++) { + pad_dims[i] = gsl::narrow_cast(pads[i]); + stride_dims[i] = gsl::narrow_cast(strides[i]); + dilation_dims[i] = gsl::narrow_cast(dilations[i]); + } + + CUDNN_RETURN_IF_ERROR(cudnnSetConvolutionNdDescriptor( + desc_, + gsl::narrow_cast(rank), + pad_dims.data(), + stride_dims.data(), + dilation_dims.data(), + mode, + data_type)); + + return Status::OK(); +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/nn/conv.h b/onnxruntime/core/providers/cuda/nn/conv.h new file mode 100644 index 0000000000000..08278978168b8 --- /dev/null +++ b/onnxruntime/core/providers/cuda/nn/conv.h @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cuda/cudnn_common.h" +#include "core/providers/cpu/nn/conv_base.h" + +namespace onnxruntime { +namespace cuda { + +class CudnnConvolutionDescriptor final { + public: + CudnnConvolutionDescriptor(); + ~CudnnConvolutionDescriptor(); + + Status Set(size_t rank, + const std::vector& pads, + const std::vector& strides, + const std::vector& dilations, + cudnnConvolutionMode_t mode, + cudnnDataType_t data_type); + + operator cudnnConvolutionDescriptor_t() const { return desc_; } + + private: + cudnnConvolutionDescriptor_t desc_; +}; + +// cached cudnn descriptors +template +struct CudnnConvState { + // if x/w dims changed, update algo and cudnnTensors + std::vector last_x_dims; + std::vector last_w_dims; + + // these would be recomputed if x/w dims change + std::vector y_dims; + size_t workspace_bytes; + AlgoType algo; + CudnnTensor x_tensor; + CudnnFilterDescriptor filter_desc; + CudnnTensor b_tensor; + CudnnTensor y_tensor; + CudnnConvolutionDescriptor conv_desc; + + // note that conv objects are shared between execution frames, and a lock is needed to avoid multi-thread racing + std::mutex mutex; +}; + +enum : size_t { + AlgoSearchWorkspaceSize = 32 * 1024 * 1024, +}; + +template +class Conv : public CudaKernel, public ConvBase { + public: + Conv(const OpKernelInfo& info) : CudaKernel(info), ConvBase(info) { + auto pads_size = pads_.size(); + ONNXRUNTIME_ENFORCE(pads_size % 2 == 0); + auto rank = pads_size / 2; + for (size_t i = 0; i < rank; i++) { + ONNXRUNTIME_ENFORCE(pads_[i] == pads_[i + rank], "cudnn only supports symmetric padding"); + } + } + + Status ComputeInternal(OpKernelContext* context) const override; + + private: + mutable CudnnConvState s_; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/nn/conv_transpose.cc b/onnxruntime/core/providers/cuda/nn/conv_transpose.cc new file mode 100644 index 0000000000000..f39a23338340c --- /dev/null +++ b/onnxruntime/core/providers/cuda/nn/conv_transpose.cc @@ -0,0 +1,148 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "conv_transpose.h" + +namespace onnxruntime { +namespace cuda { + +#define REGISTER_KERNEL_TYPED(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + ConvTranspose, \ + kOnnxDomain, \ + 1, \ + T, \ + kCudaExecutionProvider, \ + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + ConvTranspose); + +REGISTER_KERNEL_TYPED(float) +REGISTER_KERNEL_TYPED(double) +REGISTER_KERNEL_TYPED(MLFloat16) + +template +Status ConvTranspose::ComputeInternal(OpKernelContext* context) const { + typedef typename ToCudaType::MappedType CudaT; + + const Tensor* X = context->Input(0); + const TensorShape& x_shape = X->Shape(); + const auto& x_dims = x_shape.GetDims(); + auto x_data = reinterpret_cast(X->template Data()); + + const Tensor* W = context->Input(1); + const TensorShape& w_shape = W->Shape(); + std::vector w_dims = w_shape.GetDims(); + auto w_data = reinterpret_cast(W->template Data()); + + size_t num_inputs = OpKernel::Node().InputDefs().size(); + bool has_bias = (num_inputs == 3); + + CudaT* y_data = nullptr; + + { + std::lock_guard lock(s_.mutex); + // TODO: add a global cache if need to handle cases for multiple frames running simultaneuously with different batch_size + bool input_dims_changed = (s_.last_x_dims != x_dims); + bool w_dims_changed = (s_.last_w_dims != w_dims); + if (input_dims_changed || w_dims_changed) { + if (input_dims_changed) + s_.last_x_dims = x_dims; + + if (w_dims_changed) + s_.last_w_dims = w_dims; + + Prepare p; + ONNXRUNTIME_RETURN_IF_ERROR(PrepareForCompute(context, has_bias, p)); + + const auto& y_dims = p.Y->Shape().GetDims(); + s_.y_dims = y_dims; + + ONNXRUNTIME_RETURN_IF_ERROR(s_.x_tensor.Set(x_dims, CudnnTensor::GetDataType())); + ONNXRUNTIME_RETURN_IF_ERROR(s_.y_tensor.Set(y_dims, CudnnTensor::GetDataType())); + + if (w_dims_changed) + ONNXRUNTIME_RETURN_IF_ERROR(s_.filter_desc.Set(w_dims, CudnnTensor::GetDataType())); + + cudnnConvolutionMode_t mode = CUDNN_CROSS_CORRELATION; + ONNXRUNTIME_RETURN_IF_ERROR(s_.conv_desc.Set(p.kernel_shape.size(), p.pads, p.strides, p.dilations, mode, CudnnTensor::GetDataType())); + CUDNN_RETURN_IF_ERROR(cudnnSetConvolutionGroupCount(s_.conv_desc, gsl::narrow_cast(group_))); + + IAllocatorUniquePtr algo_search_workspace = GetScratchBuffer(AlgoSearchWorkspaceSize); + + if (has_bias) { + const auto& b_shape = p.B->Shape(); + ONNXRUNTIME_RETURN_IF_NOT(b_shape.NumDimensions() == 1, "bias should be 1D"); + std::vector b_dims(2 + p.kernel_shape.size()); + b_dims[0] = 1; // N + b_dims[1] = b_shape[0]; // C + for (size_t i = 0; i < p.kernel_shape.size(); i++) + b_dims[2 + i] = 1; + + ONNXRUNTIME_RETURN_IF_ERROR(s_.b_tensor.Set(b_dims, CudnnTensor::GetDataType())); + } + + y_data = reinterpret_cast(p.Y->template MutableData()); + + // set math type to tensor core before algorithm search + if (std::is_same::value) + CUDNN_RETURN_IF_ERROR(cudnnSetConvolutionMathType(s_.conv_desc, CUDNN_TENSOR_OP_MATH)); + + cudnnConvolutionBwdDataAlgoPerf_t perf; + int algo_count = 1; + CUDNN_RETURN_IF_ERROR(cudnnFindConvolutionBackwardDataAlgorithmEx( + CudnnHandle(), + s_.filter_desc, + w_data, + s_.x_tensor, + x_data, + s_.conv_desc, + s_.y_tensor, + y_data, + 1, + &algo_count, + &perf, + algo_search_workspace.get(), + AlgoSearchWorkspaceSize)); + CUDNN_RETURN_IF_ERROR(cudnnSetConvolutionMathType(s_.conv_desc, perf.mathType)); + s_.algo = perf.algo; + s_.workspace_bytes = perf.memory; + } + } + + if (!y_data) { + Tensor* Y = context->Output(0, TensorShape(s_.y_dims)); + y_data = reinterpret_cast(Y->template MutableData()); + } + + const auto alpha = Consts::One; + const auto beta = Consts::Zero; + + IAllocatorUniquePtr workspace = GetScratchBuffer(s_.workspace_bytes); + + CUDNN_RETURN_IF_ERROR( + cudnnConvolutionBackwardData( + CudnnHandle(), + &alpha, + s_.filter_desc, + w_data, + s_.x_tensor, + x_data, + s_.conv_desc, + s_.algo, + workspace.get(), + s_.workspace_bytes, + &beta, + s_.y_tensor, + y_data)); + + if (has_bias) { + const Tensor* B = context->Input(2); + auto b_data = reinterpret_cast(B->template Data()); + CUDNN_RETURN_IF_ERROR(cudnnAddTensor(CudnnHandle(), &alpha, s_.b_tensor, b_data, &alpha, s_.y_tensor, y_data)); + } + + return Status::OK(); +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/nn/conv_transpose.h b/onnxruntime/core/providers/cuda/nn/conv_transpose.h new file mode 100644 index 0000000000000..93846950b2119 --- /dev/null +++ b/onnxruntime/core/providers/cuda/nn/conv_transpose.h @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/providers/cuda/cudnn_common.h" +#include "core/providers/cpu/nn/conv_transpose.h" +#include "conv.h" + +namespace onnxruntime { +namespace cuda { + +template +class ConvTranspose : public CudaKernel, public ConvTransposeBase { + public: + ConvTranspose(const OpKernelInfo& info) : CudaKernel(info), ConvTransposeBase(info){}; + Status ComputeInternal(OpKernelContext* context) const override; + + private: + mutable CudnnConvState s_; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/nn/instance_norm.cc b/onnxruntime/core/providers/cuda/nn/instance_norm.cc new file mode 100644 index 0000000000000..65c868cc91eb2 --- /dev/null +++ b/onnxruntime/core/providers/cuda/nn/instance_norm.cc @@ -0,0 +1,154 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "instance_norm.h" +#include "instance_norm_impl.h" +#include "core/providers/cpu/nn/instance_norm_helper.h" +#include "core/providers/cpu/nn/batch_norm_helper.h" + +namespace onnxruntime { +namespace cuda { + +#define REGISTER_KERNEL_TYPED(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + InstanceNormalization, \ + kOnnxDomain, \ + 6, \ + T, \ + kCudaExecutionProvider, \ + KernelDefBuilder() \ + .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + InstanceNorm); + +REGISTER_KERNEL_TYPED(float) +REGISTER_KERNEL_TYPED(double) +REGISTER_KERNEL_TYPED(MLFloat16) + +template +InstanceNorm::InstanceNorm(const OpKernelInfo& op_kernel_info) + : CudaKernel(op_kernel_info) { + float tmp_epsilon; + ONNXRUNTIME_ENFORCE(op_kernel_info.GetAttr("epsilon", &tmp_epsilon).IsOK()); + epsilon_ = ClampCudnnBatchNormEpsilon(tmp_epsilon); +} + +template +Status InstanceNorm::ComputeInternal(OpKernelContext* p_op_kernel_context) const { + typedef typename ToCudaType::MappedType CudaT; + + const Tensor* X = p_op_kernel_context->Input(0); + const Tensor* scale = p_op_kernel_context->Input(1); + const Tensor* bias = p_op_kernel_context->Input(2); + + ONNXRUNTIME_RETURN_IF_ERROR(InstanceNormHelper::ValidateInputs(X, scale, bias)); + + const TensorShape& x_shape = X->Shape(); + Tensor* Y = p_op_kernel_context->Output(0, x_shape); + + auto y_data = reinterpret_cast(Y->template MutableData()); + auto x_data = reinterpret_cast(X->template Data()); + auto scale_data = reinterpret_cast(scale->template Data()); + auto bias_data = reinterpret_cast(bias->template Data()); + + const auto& x_dims = x_shape.GetDims(); + const int64_t N = x_dims[0]; + const int64_t C = x_dims[1]; + const auto one = Consts::One; + const auto zero = Consts::Zero; + + if (N == 1) { + // when N == 1, we can treat it as spatial batch normalization in training + // as the mean/variance would be computed from input + + CudnnTensor data_desc; + std::vector new_dims; + BatchNormHelper::NormalizeDims(x_shape, new_dims); + ONNXRUNTIME_RETURN_IF_ERROR(data_desc.Set(new_dims, CudnnTensor::GetDataType())); + + CudnnTensor stats_desc; + ONNXRUNTIME_RETURN_IF_ERROR(stats_desc.Set(data_desc, CUDNN_BATCHNORM_SPATIAL)); + + CUDNN_RETURN_IF_ERROR(cudnnBatchNormalizationForwardTraining( + CudnnHandle(), + CUDNN_BATCHNORM_SPATIAL, + &one, + &zero, + data_desc, + x_data, + data_desc, + y_data, + stats_desc, + scale_data, + bias_data, + 1.0f, + nullptr, + nullptr, + epsilon_, + nullptr, + nullptr)); + } else { + // we use cudnnBatchNormalizationForwardTraining to compute mean/variance + // so collapsing NC into channel + + auto input_count = x_shape.Size(); // N * C * H * W + auto stats_count = x_shape.SizeToDimension(2); // N * C + auto image_size = input_count / stats_count; + + CudnnTensor data_desc; + ONNXRUNTIME_RETURN_IF_ERROR(data_desc.Set({1, stats_count, image_size, 1}, CudnnTensor::GetDataType())); + + CudnnTensor stats_desc; + ONNXRUNTIME_RETURN_IF_ERROR(stats_desc.Set({1, stats_count, 1, 1}, CudnnTensor::GetDataType())); + + auto mean = GetScratchBuffer(stats_count); + auto variance = GetScratchBuffer(stats_count); + auto unused_scale = GetScratchBuffer(stats_count); + auto unused_bias = GetScratchBuffer(stats_count); + + // first, compute mean and variance per-instance per-channel using cudnnBatchNorm training + CUDNN_RETURN_IF_ERROR(cudnnBatchNormalizationForwardTraining( + CudnnHandle(), + CUDNN_BATCHNORM_SPATIAL, + &one, + &zero, + data_desc, + x_data, + data_desc, + y_data, // use y temporarily, would be rewritten later + stats_desc, + unused_scale.get(), + unused_bias.get(), + 1.0f, + mean.get(), + variance.get(), + CUDNN_BN_MIN_EPSILON, + nullptr, + nullptr)); + + // Y = scale * (x - mean) / sqrt (variance + epsilon) + B + // X/Y is (N,C,H,W) + // scale/bias is (1,C,1,1) + // mean/stddev is (N,C,1,1) + // NOTE cudnnBatchNormalization computes unbiased variance sum((Xi - mean)^2) / (count - 1) + // and it needs to be corrected with (count - 1) / count + fast_divmod fdm_HW(gsl::narrow_cast(image_size)); + fast_divmod fdm_C(gsl::narrow_cast(C)); + + InstanceNormImpl( + x_data, + scale_data, + bias_data, + mean.get(), + variance.get(), + (image_size - 1.0) / image_size, + static_cast(epsilon_), + fdm_HW, + fdm_C, + y_data, + input_count); + } + return Status::OK(); +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/nn/instance_norm.h b/onnxruntime/core/providers/cuda/nn/instance_norm.h new file mode 100644 index 0000000000000..fcc66c146cc10 --- /dev/null +++ b/onnxruntime/core/providers/cuda/nn/instance_norm.h @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/providers/cuda/cudnn_common.h" + +namespace onnxruntime { +namespace cuda { + +template +class InstanceNorm final : public CudaKernel { + public: + InstanceNorm(const OpKernelInfo& op_kernel_info); + Status ComputeInternal(OpKernelContext* p_op_kernel_context) const override; + + private: + double epsilon_; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/nn/instance_norm_impl.cu b/onnxruntime/core/providers/cuda/nn/instance_norm_impl.cu new file mode 100644 index 0000000000000..98cf179601081 --- /dev/null +++ b/onnxruntime/core/providers/cuda/nn/instance_norm_impl.cu @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cuda/cu_inc/common.cuh" +#include "instance_norm_impl.h" + +namespace onnxruntime { +namespace cuda { + +template +__global__ void _InstanceNormKernel( + const T* input_data, + const T* scale, + const T* bias, + const T* mean, + const T* variance, + const double variance_correction, + const double epsilon, + const fast_divmod fdm_HW, + const fast_divmod fdm_C, + T* output_data, + const CUDA_LONG N) { + CALCULATE_ELEMENTWISE_INDEX_OR_EXIT(id, N); + int nc = fdm_HW.div(id); + int n, c; + fdm_C.divmod(nc, n, c); + + // Y = scale * (x - mean) / sqrt (std * std + epsilon) + B + output_data[id] = scale[c] * (input_data[id] - mean[nc]) / _Sqrt(variance[nc] * (T)variance_correction + (T)epsilon) + bias[c]; +} + +template +void InstanceNormImpl( + const T* input_data, + const T* scale, + const T* bias, + const T* mean, + const T* variance, + const double variance_correction, + const double epsilon, + const fast_divmod& fdm_HW, + const fast_divmod& fdm_C, + T* output_data, + size_t N) { + int blocksPerGrid = (int)(ceil(static_cast(N) / GridDim::maxThreadsPerBlock)); + _InstanceNormKernel<<>>( + input_data, scale, bias, mean, variance, variance_correction, epsilon, fdm_HW, fdm_C, output_data, (CUDA_LONG)N); +} + +#define SPECIALIZED_IMPL(T) \ + template void InstanceNormImpl(const T* input_data, const T* scale, const T* bias, const T* mean, const T* stddev, const double variance_correction, const double epsilon, const fast_divmod& fdm_HW, const fast_divmod& fdm_C, T* output_data, size_t count); + +SPECIALIZED_IMPL(float) +SPECIALIZED_IMPL(double) +SPECIALIZED_IMPL(half) + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/nn/instance_norm_impl.h b/onnxruntime/core/providers/cuda/nn/instance_norm_impl.h new file mode 100644 index 0000000000000..5748746db65ac --- /dev/null +++ b/onnxruntime/core/providers/cuda/nn/instance_norm_impl.h @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/providers/cuda/shared_inc/fast_divmod.h" +namespace onnxruntime { +namespace cuda { + +template +void InstanceNormImpl( + const T* input_data, + const T* scale, + const T* bias, + const T* mean, + const T* variance, + const double variance_correction, + const double epsilon, + const fast_divmod& fdm_HW, + const fast_divmod& fdm_C, + T* output_data, + size_t count); + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/nn/lrn.cc b/onnxruntime/core/providers/cuda/nn/lrn.cc new file mode 100644 index 0000000000000..c25464cbf7a27 --- /dev/null +++ b/onnxruntime/core/providers/cuda/nn/lrn.cc @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "lrn.h" + +namespace onnxruntime { +namespace cuda { + +#define REGISTER_KERNEL_TYPED(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + LRN, \ + kOnnxDomain, \ + 1, \ + T, \ + kCudaExecutionProvider, \ + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + LRN); + +REGISTER_KERNEL_TYPED(float) +REGISTER_KERNEL_TYPED(double) +REGISTER_KERNEL_TYPED(MLFloat16) + +template +LRN::LRN(const OpKernelInfo& info) : CudaKernel(info) { + int64_t size; + ONNXRUNTIME_ENFORCE(info.GetAttr("size", &size).IsOK()); + ONNXRUNTIME_ENFORCE(size > 0); + ONNXRUNTIME_ENFORCE(size % 2 == 1); + + float alpha; + float beta; + ONNXRUNTIME_ENFORCE(info.GetAttr("alpha", &alpha).IsOK()); + ONNXRUNTIME_ENFORCE(alpha > 0.0f); + ONNXRUNTIME_ENFORCE(info.GetAttr("beta", &beta).IsOK()); + ONNXRUNTIME_ENFORCE(beta > 0.0f); + float bias = info.GetAttrOrDefault("bias", 1.0f); + + ONNXRUNTIME_ENFORCE(norm_desc_.Set( + gsl::narrow_cast(size), + static_cast(alpha), + static_cast(beta), + static_cast(bias)) + .IsOK()); +} + +template +Status LRN::ComputeInternal(OpKernelContext* context) const { + typedef typename ToCudaType::MappedType CudaT; + + const Tensor* X = context->Input(0); + + auto rank = X->Shape().NumDimensions(); + if (rank != 4 && rank != 5) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "cudnn LRN only supports 4D or 5D input"); + + Tensor* Y = context->Output(0, X->Shape()); + + CudnnTensor x_tensor; + ONNXRUNTIME_RETURN_IF_ERROR(x_tensor.Set(X->Shape().GetDims(), CudnnTensor::GetDataType())); + + const auto one = Consts::One; + const auto zero = Consts::Zero; + + CUDNN_RETURN_IF_ERROR(cudnnLRNCrossChannelForward( + CudnnHandle(), + norm_desc_, + CUDNN_LRN_CROSS_CHANNEL_DIM1, + &one, + x_tensor, + reinterpret_cast(X->template Data()), + &zero, + x_tensor, + reinterpret_cast(Y->template MutableData()))); + + return Status::OK(); +} + +CudnnLRNDescriptor::CudnnLRNDescriptor() : desc_(nullptr) { +} + +CudnnLRNDescriptor::~CudnnLRNDescriptor() { + if (desc_) { + cudnnDestroyLRNDescriptor(desc_); + desc_ = nullptr; + } +} + +Status CudnnLRNDescriptor::Set(uint32_t N, double alpha, double beta, double K) { + if (!desc_) + CUDNN_RETURN_IF_ERROR(cudnnCreateLRNDescriptor(&desc_)); + + CUDNN_RETURN_IF_ERROR(cudnnSetLRNDescriptor(desc_, N, alpha, beta, K)); + return Status::OK(); +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/nn/lrn.h b/onnxruntime/core/providers/cuda/nn/lrn.h new file mode 100644 index 0000000000000..727ec0cb9d47a --- /dev/null +++ b/onnxruntime/core/providers/cuda/nn/lrn.h @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/framework/op_kernel.h" +#include "core/providers/cuda/cudnn_common.h" + +namespace onnxruntime { +namespace cuda { + +class CudnnLRNDescriptor final { + public: + CudnnLRNDescriptor(); + ~CudnnLRNDescriptor(); + Status Set(uint32_t N, double alpha, double beta, double K); + operator cudnnLRNDescriptor_t() const { return desc_; } + + private: + cudnnLRNDescriptor_t desc_; +}; + +template +class LRN : public CudaKernel { + public: + LRN(const OpKernelInfo& info); + Status ComputeInternal(OpKernelContext* p_op_kernel_context) const override; + + private: + CudnnLRNDescriptor norm_desc_; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/nn/max_pool_with_index.cu b/onnxruntime/core/providers/cuda/nn/max_pool_with_index.cu new file mode 100644 index 0000000000000..29b83ecdefa6d --- /dev/null +++ b/onnxruntime/core/providers/cuda/nn/max_pool_with_index.cu @@ -0,0 +1,171 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "max_pool_with_index.h" + +#include + +#include "core/providers/cuda/cu_inc/common.cuh" +#include "core/providers/cuda/shared_inc/fast_divmod.h" + +namespace onnxruntime { +namespace cuda { +template +__global__ void MaxPoolWithIndexKernel( + int64_t batch, + int64_t channels, + int64_t height, + int64_t width, + int64_t depth, + int64_t pooled_height, + int64_t pooled_width, + int64_t pooled_depth, + int64_t kernel_h, + int64_t kernel_w, + int64_t kernel_d, + int64_t stride_h, + int64_t stride_w, + int64_t stride_d, + int64_t pad_h, + int64_t pad_w, + int64_t pad_d, + fast_divmod fdm_c, + fast_divmod fdm_h, + fast_divmod fdm_w, + fast_divmod fdm_d, + int64_t storage_order, + const T* p_input, + int64_t output_size, + T* p_output, + int64_t* p_indices) { + int id = blockIdx.x * blockDim.x + threadIdx.x; + if (id >= output_size) return; + + int d_index, w_index, h_index, c_index, n_index, id_tmp; + fdm_d.divmod(id, id_tmp, d_index); + fdm_w.divmod(id_tmp, id_tmp, w_index); + fdm_h.divmod(id_tmp, id_tmp, h_index); + fdm_c.divmod(id_tmp, n_index, c_index); + + int64_t d_start = d_index * stride_d - pad_d; + int64_t w_start = w_index * stride_w - pad_w; + int64_t h_start = h_index * stride_h - pad_h; + + int64_t d_end = _Min(d_start + kernel_d, depth); + int64_t w_end = _Min(w_start + kernel_w, width); + int64_t h_end = _Min(h_start + kernel_h, height); + + d_start = _Max(d_start, 0); + w_start = _Max(w_start, 0); + h_start = _Max(h_start, 0); + int64_t d_index_max = -1; + int64_t w_index_max = -1; + int64_t h_index_max = -1; + int64_t offset = (n_index * channels + c_index) * height * width * depth; + const T* p_slice = p_input + offset; + T maxval = p_slice[h_start * width * depth + w_start * depth + d_start] - (T)1; + for (int64_t d = d_start; d < d_end; ++d) { + for (int64_t w = w_start; w < w_end; ++w) { + for (int64_t h = h_start; h < h_end; ++h) { + if (p_slice[h * width * depth + w * depth + d] > maxval) { + h_index_max = h; + w_index_max = w; + d_index_max = d; + maxval = static_cast(p_slice[h * width * depth + w * depth + d]); + } + } + } + } + p_output[id] = p_input[offset + h_index_max * width * depth + w_index_max * depth + d_index_max]; + p_indices[id] = storage_order == 0 ? offset + h_index_max * width * depth + w_index_max * depth + d_index_max + : offset + h_index_max + w_index_max * height + d_index_max * width * height; +} + +template +void MaxPoolWithIndex( + const TensorShape& input_shape, + const TensorShape& output_shape, + const std::vector& kernel_shape, + const std::vector& stride_shape, + const std::vector& pads, + int64_t storage_order, + const T* p_input, + T* p_output, + int64_t* p_indices) { + size_t input_dim_count = input_shape.NumDimensions(); + + int64_t batchs = input_shape[0]; + int64_t channels = input_shape[1]; + int64_t height = input_shape[2]; + int64_t width = kernel_shape.size() > 1 ? input_shape[3] : 1; + int64_t depth = kernel_shape.size() > 2 ? input_shape[4] : 1; + int64_t pooled_height = output_shape[2]; + int64_t pooled_width = kernel_shape.size() > 1 ? output_shape[3] : 1; + int64_t pooled_depth = kernel_shape.size() > 2 ? output_shape[4] : 1; + int64_t kernel_h = kernel_shape[0]; + int64_t kernel_w = kernel_shape.size() > 1 ? kernel_shape[1] : 1; + int64_t kernel_d = kernel_shape.size() > 2 ? kernel_shape[2] : 1; + int64_t stride_h = stride_shape[0]; + int64_t stride_w = stride_shape.size() > 1 ? stride_shape[1] : 1; + int64_t stride_d = stride_shape.size() > 2 ? stride_shape[2] : 1; + //pads in the format of [x1_begin, x2_begin...x1_end, x2_end,...], + //where xi_begin the number of pixels added at the beginning of axis i + //and xi_end, the number of pixels added at the end of axis i. + int64_t pad_h = pads[0]; + int64_t pad_w = pads.size() == 4 ? pads[1] : 0; + int64_t pad_d = pads.size() == 6 ? pads[2] : 0; + int64_t output_size = output_shape.Size(); + + fast_divmod fdm_c(static_cast(channels)); + fast_divmod fdm_h(static_cast(pooled_height)); + fast_divmod fdm_w(static_cast(pooled_width)); + fast_divmod fdm_d(static_cast(pooled_depth)); + + int blocksPerGrid = (int)((output_size + GridDim::maxThreadsPerBlock - 1) / GridDim::maxThreadsPerBlock); + MaxPoolWithIndexKernel<<>>( + batchs, + channels, + height, + width, + depth, + pooled_height, + pooled_width, + pooled_depth, + kernel_h, + kernel_w, + kernel_d, + stride_h, + stride_w, + stride_d, + pad_h, + pad_w, + pad_d, + fdm_c, + fdm_h, + fdm_w, + fdm_d, + storage_order, + p_input, + output_size, + p_output, + p_indices); +} + +#define INSTANTIATEMAXPOOLWITHINDEX(T) \ + template void MaxPoolWithIndex( \ + const TensorShape& input_shape, \ + const TensorShape& output_shape, \ + const std::vector& kernel_shape, \ + const std::vector& stride_shape, \ + const std::vector& pads, \ + int64_t storage_order, \ + const T* p_input, \ + T* p_output, \ + int64_t* p_indices); + +INSTANTIATEMAXPOOLWITHINDEX(float) +INSTANTIATEMAXPOOLWITHINDEX(double) +INSTANTIATEMAXPOOLWITHINDEX(half) + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/nn/max_pool_with_index.h b/onnxruntime/core/providers/cuda/nn/max_pool_with_index.h new file mode 100644 index 0000000000000..60fa7f806b4a3 --- /dev/null +++ b/onnxruntime/core/providers/cuda/nn/max_pool_with_index.h @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include + +#include "core/framework/tensor_shape.h" + +namespace onnxruntime { +namespace cuda { +template +void MaxPoolWithIndex( + const TensorShape& input_shape, + const TensorShape& output_shape, + const std::vector& kernel_shape, + const std::vector& stride_shape, + const std::vector& pads, + int64_t storage_order, + const T* p_input, + T* p_output, + int64_t* p_indices); +} //namespace cuda +} //namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/nn/pool.cc b/onnxruntime/core/providers/cuda/nn/pool.cc new file mode 100644 index 0000000000000..89d5dcb4b584c --- /dev/null +++ b/onnxruntime/core/providers/cuda/nn/pool.cc @@ -0,0 +1,204 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cuda/nn/pool.h" +#include "core/providers/common.h" +#include "core/providers/cuda/cudnn_common.h" +#include "core/providers/cuda/nn/max_pool_with_index.h" +using namespace onnxruntime::common; +namespace onnxruntime { +namespace cuda { + +#define POOLING_KERNEL(op_name, data_type, pool_type, since_version) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + op_name, \ + kOnnxDomain, \ + since_version, \ + data_type, \ + kCudaExecutionProvider, \ + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + Pool); + +#define POOLING_KERNEL_VERSIONED(op_name, data_type, pool_type, since_version, end_version) \ + ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ + op_name, \ + kOnnxDomain, \ + since_version, \ + end_version, \ + data_type, \ + kCudaExecutionProvider, \ + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()).TypeConstraint("I", DataTypeImpl::GetTensorType()), \ + Pool); + +POOLING_KERNEL(AveragePool, float, AveragePool, 7) +POOLING_KERNEL(AveragePool, double, AveragePool, 7) +POOLING_KERNEL(AveragePool, MLFloat16, AveragePool, 7) +POOLING_KERNEL(GlobalAveragePool, float, AveragePool, 1) +POOLING_KERNEL(GlobalAveragePool, double, AveragePool, 1) +POOLING_KERNEL(GlobalAveragePool, MLFloat16, AveragePool, 1) +POOLING_KERNEL_VERSIONED(MaxPool, float, MaxPool<1>, 1, 7) +POOLING_KERNEL_VERSIONED(MaxPool, double, MaxPool<1>, 1, 7) +POOLING_KERNEL_VERSIONED(MaxPool, MLFloat16, MaxPool<1>, 1, 7) +POOLING_KERNEL(MaxPool, float, MaxPool<8>, 8) +POOLING_KERNEL(MaxPool, double, MaxPool<8>, 8) +POOLING_KERNEL(MaxPool, MLFloat16, MaxPool<8>, 8) +POOLING_KERNEL(GlobalMaxPool, float, MaxPool<1>, 1) +POOLING_KERNEL(GlobalMaxPool, double, MaxPool<1>, 1) +POOLING_KERNEL(GlobalMaxPool, MLFloat16, MaxPool<1>, 1) + +class CudnnPoolingDescriptor final { + public: + CudnnPoolingDescriptor() : desc_(nullptr) { + } + + ~CudnnPoolingDescriptor() { + if (desc_ != nullptr) { + cudnnDestroyPoolingDescriptor(desc_); + desc_ = nullptr; + } + } + + Status Set(cudnnPoolingMode_t mode, + const std::vector& kernel_shape, + const std::vector& pads, + const std::vector& strides) { + if (!desc_) + CUDNN_RETURN_IF_ERROR(cudnnCreatePoolingDescriptor(&desc_)); + + int rank = gsl::narrow_cast(kernel_shape.size()); + std::vector window(rank); + std::vector padding(rank); + std::vector stride(rank); + for (int i = 0; i < rank; i++) { + window[i] = gsl::narrow_cast(kernel_shape[i]); + } + for (int i = 0; i < rank; i++) { + padding[i] = gsl::narrow_cast(pads[i]); + } + for (int i = 0; i < rank; i++) { + stride[i] = gsl::narrow_cast(strides[i]); + } + CUDNN_RETURN_IF_ERROR(cudnnSetPoolingNdDescriptor( + desc_, + mode, + CUDNN_PROPAGATE_NAN, + rank, + window.data(), + padding.data(), + stride.data())); + + return Status::OK(); + } + + operator cudnnPoolingDescriptor_t() const { return desc_; } + + private: + cudnnPoolingDescriptor_t desc_; +}; + +template +Status Pool::ComputeInternal(OpKernelContext* context) const { + typedef typename ToCudaType::MappedType CudaT; + const Tensor* X = context->Input(0); + const TensorShape& x_shape = X->Shape(); + const auto& x_dims = x_shape.GetDims(); + + if (x_shape.NumDimensions() < 3) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Input dimension cannot be less than 3."); + } + + std::vector kernel_shape = kernel_shape_; + std::vector pads = pads_; + std::vector strides = strides_; + + if (global_pooling_) { + kernel_shape.assign(x_dims.begin() + 2, x_dims.end()); + pads.assign(kernel_shape.size(), 0); + strides.assign(kernel_shape.size(), 1); + } + + std::vector y_dims = PoolBase::SetOutputSize(x_shape, x_shape[1], &pads); + Tensor* Y = context->Output(0, TensorShape(y_dims)); + + auto x_data = reinterpret_cast(X->template Data()); + auto y_data = reinterpret_cast(Y->template MutableData()); + + std::vector x_dims_cudnn = x_dims; + std::vector y_dims_cudnn = y_dims; + if (kernel_shape.size() < 2) { + // cudnn only takes 4D or 5D input, so pad dimensions if needed + x_dims_cudnn.push_back(1); + y_dims_cudnn.push_back(1); + pads.insert(pads.begin() + kernel_shape.size(), 0); + pads.insert(pads.end(), 0); + kernel_shape.push_back(1); + strides.push_back(1); + } + + const auto alpha = Consts::One; + const auto beta = Consts::Zero; + CudnnTensor x_tensor; + CudnnTensor y_tensor; + ONNXRUNTIME_RETURN_IF_ERROR(x_tensor.Set(x_dims_cudnn, CudnnTensor::GetDataType())); + ONNXRUNTIME_RETURN_IF_ERROR(y_tensor.Set(y_dims_cudnn, CudnnTensor::GetDataType())); + + cudnnPoolingMode_t mode = CUDNN_POOLING_MAX; + if (PoolType::type == onnxruntime::PoolType::kAveragePool) { + mode = count_include_pad_ ? CUDNN_POOLING_AVERAGE_COUNT_INCLUDE_PADDING : CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING; + } + CudnnPoolingDescriptor pooling_desc; + ONNXRUNTIME_RETURN_IF_ERROR(pooling_desc.Set(mode, kernel_shape, pads, strides)); + + CUDNN_RETURN_IF_ERROR(cudnnPoolingForward(CudnnHandle(), pooling_desc, &alpha, x_tensor, x_data, &beta, y_tensor, y_data)); + + return Status::OK(); +} + +template +Status Pool>::ComputeInternal(OpKernelContext* context) const { + typedef typename ToCudaType::MappedType CudaT; + const Tensor* X = context->Input(0); + const TensorShape& x_shape = X->Shape(); + const auto& x_dims = x_shape.GetDims(); + + if (x_shape.NumDimensions() < 3) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Input dimension cannot be less than 3."); + } + + std::vector kernel_shape = this->kernel_shape_; + std::vector pads = this->pads_; + std::vector strides = this->strides_; + + if (this->global_pooling_) { + kernel_shape.assign(x_dims.begin() + 2, x_dims.end()); + pads.assign(kernel_shape.size(), 0); + strides.assign(kernel_shape.size(), 1); + } + + std::vector y_dims = PoolBase::SetOutputSize(x_shape, x_shape[1], &pads); + Tensor* Y = context->Output(0, TensorShape(y_dims)); + + auto x_data = reinterpret_cast(X->template Data()); + auto y_data = reinterpret_cast(Y->template MutableData()); + + Tensor* I = context->Output(1, TensorShape(y_dims)); + if (nullptr != I) { + auto i_data = I->template MutableData(); + MaxPoolWithIndex( + x_shape, + TensorShape(y_dims), + kernel_shape, + strides, + pads, + this->storage_order_, + x_data, + y_data, + i_data); + } else { + Pool>::ComputeInternal(context); + } + return Status::OK(); +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/nn/pool.h b/onnxruntime/core/providers/cuda/nn/pool.h new file mode 100644 index 0000000000000..ba371c5039249 --- /dev/null +++ b/onnxruntime/core/providers/cuda/nn/pool.h @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cuda/cudnn_common.h" +#include "core/providers/cpu/nn/pool_base.h" + +namespace onnxruntime { +namespace cuda { + +template +class Pool : public CudaKernel, public PoolBase { + public: + Pool(OpKernelInfo info) : CudaKernel(info), PoolBase(info) {} + + Status ComputeInternal(OpKernelContext* context) const override; +}; + +template +class Pool> final : public Pool> { + public: + Pool(OpKernelInfo info) : Pool>(info) {} + + Status ComputeInternal(OpKernelContext* context) const override; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/reduction/reduction_ops.cc b/onnxruntime/core/providers/cuda/reduction/reduction_ops.cc new file mode 100644 index 0000000000000..9d3cba282beaf --- /dev/null +++ b/onnxruntime/core/providers/cuda/reduction/reduction_ops.cc @@ -0,0 +1,247 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "reduction_ops.h" +#include "core/providers/common.h" +#include "core/providers/cuda/cudnn_common.h" +#include "core/providers/cuda/math/unary_elementwise_ops_impl.h" +#include "core/providers/cuda/math/binary_elementwise_ops_impl.h" +#include "core/providers/cuda/math/binary_elementwise_ops.h" +#include "core/providers/cpu/tensor/utils.h" + +using namespace onnxruntime::common; +namespace onnxruntime { +namespace cuda { + +#define REGISTER_KERNEL_TYPED(name, T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + name, \ + kOnnxDomain, \ + 1, \ + T, \ + kCudaExecutionProvider, \ + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + name); + +class CudnnReduceDescriptor final { + public: + CudnnReduceDescriptor() : desc_(nullptr) { + } + + ~CudnnReduceDescriptor() { + if (desc_ != nullptr) { + cudnnDestroyReduceTensorDescriptor(desc_); + desc_ = nullptr; + } + } + + Status Set(cudnnReduceTensorOp_t op, cudnnDataType_t type, cudnnReduceTensorIndices_t indices) { + if (!desc_) + CUDNN_RETURN_IF_ERROR(cudnnCreateReduceTensorDescriptor(&desc_)); + + CUDNN_RETURN_IF_ERROR(cudnnSetReduceTensorDescriptor( + desc_, + op, + type, + CUDNN_PROPAGATE_NAN, + indices, + CUDNN_32BIT_INDICES)); // currently only the 32-bit (unsigned int) type is supported. + return Status::OK(); + } + + operator cudnnReduceTensorDescriptor_t() const { return desc_; } + + private: + cudnnReduceTensorDescriptor_t desc_; +}; + +template +template +Status ReduceKernel::ComputeImpl(OpKernelContext* ctx, cudnnReduceTensorOp_t cudnnReduceOp) const { + typedef typename ToCudaType::MappedType CudaT; + const Tensor* X = ctx->Input(0); + ONNXRUNTIME_ENFORCE(nullptr != X); + const TensorShape input_shape{X->Shape()}; + const auto rank = input_shape.NumDimensions(); + + if (rank > 8) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "cuDNN only supports up to 8-D tensors in reduction"); + } + + const auto& input_dims = input_shape.GetDims(); + std::vector output_dims; + std::vector reduced(rank, false); + std::vector squeezed_output_dims; + if (axes_.size() > 0) { + output_dims = input_dims; + for (auto reduced_axis : axes_) { + const int64_t axis = HandleNegativeAxis(reduced_axis, rank); + output_dims[axis] = 1; + reduced[axis] = true; + } + } else { + output_dims = std::vector(rank, 1); + } + + if (keepdims_) { + squeezed_output_dims = output_dims; + } else { + for (size_t i = 0; i < rank; ++i) { + if (!reduced[i]) + squeezed_output_dims.push_back(input_dims[i]); + } + } + + Tensor* Y = ctx->Output(0, TensorShape(squeezed_output_dims)); + + int64_t input_count = input_shape.Size(); + IAllocatorUniquePtr temp_X; + cudnnDataType_t cudnn_type_X = CudnnTensor::GetDataType(); + if (ReduceTensorIndices == CUDNN_REDUCE_TENSOR_FLATTENED_INDICES && std::is_same::value) { + // ArgMax/ArgMin with FP16 are not supported by cudnn, so convert input to fp32 then call cudnn + temp_X = GetScratchBuffer(input_count); + cudnn_type_X = CUDNN_DATA_FLOAT; + Impl_Cast(reinterpret_cast(X->template Data()), temp_X.get(), X->Shape().Size()); + } + + // CUDNN requires at least 3D input, so pad 1s if needed + std::vector input_dims_cudnn = input_dims; + std::vector output_dims_cudnn = output_dims; + if (rank < 3) { + std::vector pads(3 - rank, 1); + input_dims_cudnn.insert(input_dims_cudnn.end(), pads.begin(), pads.end()); + output_dims_cudnn.insert(output_dims_cudnn.end(), pads.begin(), pads.end()); + } + + CudnnReduceDescriptor reduce_desc; + ONNXRUNTIME_RETURN_IF_ERROR(reduce_desc.Set(cudnnReduceOp, cudnn_type_X, ReduceTensorIndices)); + const auto one = Consts::One; + const auto zero = Consts::Zero; + CudnnTensor input_tensor; + CudnnTensor output_tensor; + ONNXRUNTIME_RETURN_IF_ERROR(input_tensor.Set(input_dims_cudnn, cudnn_type_X)); + ONNXRUNTIME_RETURN_IF_ERROR(output_tensor.Set(output_dims_cudnn, cudnn_type_X)); + size_t workspace_bytes = 0; + CUDNN_RETURN_IF_ERROR(cudnnGetReductionWorkspaceSize(CudnnHandle(), reduce_desc, input_tensor, output_tensor, &workspace_bytes)); + auto workspace_cuda = GetScratchBuffer(workspace_bytes); + + // need to allocate a separate buffer for ArgMin/ArgMax comparsion output + auto output_count = Y->Shape().Size(); + + if (ReduceTensorIndices == CUDNN_REDUCE_TENSOR_NO_INDICES) { + CudaT* input_data = nullptr; + if (calculate_sqt_) { + input_data = reinterpret_cast(GetScratchBuffer(input_count).get()); + fast_divmod tmp_div; + Impl_Mul(static_cast(SimpleBroadcast::NoBroadcast), nullptr, + reinterpret_cast(X->template Data()), nullptr, + reinterpret_cast(X->template Data()), nullptr, + tmp_div, tmp_div, + input_data, input_count); + } else if (log_sum_exp_) { + // Reduce max + CudnnReduceDescriptor reduce_max_desc; + ONNXRUNTIME_RETURN_IF_ERROR(reduce_max_desc.Set(CUDNN_REDUCE_TENSOR_MAX, cudnn_type_X, CUDNN_REDUCE_TENSOR_NO_INDICES)); + CUDNN_RETURN_IF_ERROR(cudnnReduceTensor( + CudnnHandle(), reduce_max_desc, nullptr, 0, workspace_cuda.get(), workspace_bytes, + &one, input_tensor, reinterpret_cast(X->template Data()), + &zero, output_tensor, reinterpret_cast(Y->template MutableData()))); + + // Exp(X-ReduceMax) + const TensorShape output_shape(output_dims); + auto exp_result = GetScratchBuffer(input_count).get(); + auto log_sum_result = GetScratchBuffer(output_count).get(); + BinaryElementwisePreparation prepare(this); + prepare.BinaryElementwiseBroadcastPrepareHelper(0, input_shape, output_shape, input_shape); + prepare.CopyToGpu(); + Impl_Sub(prepare.output_rank_or_simple_broadcast, + prepare.lhs_padded_strides.GpuPtr(), + reinterpret_cast(X->template Data()), + prepare.rhs_padded_strides.GpuPtr(), + reinterpret_cast(Y->template MutableData()), + prepare.fdm_output_strides.GpuPtr(), + prepare.fdm_H, prepare.fdm_C, + reinterpret_cast(exp_result), input_count); + + Impl_Exp(reinterpret_cast(exp_result), + reinterpret_cast(exp_result), + input_count); + + // ReduceSum + CUDNN_RETURN_IF_ERROR(cudnnReduceTensor( + CudnnHandle(), reduce_desc, nullptr, 0, workspace_cuda.get(), workspace_bytes, + &one, input_tensor, exp_result, + &zero, output_tensor, reinterpret_cast(log_sum_result))); + + // Log(Sum) + Impl_Log(reinterpret_cast(log_sum_result), + reinterpret_cast(log_sum_result), + output_count); + + // Log + ReduceMax + fast_divmod tmp_div; + Impl_Add(static_cast(SimpleBroadcast::NoBroadcast), nullptr, + reinterpret_cast(log_sum_result), nullptr, + reinterpret_cast(Y->template MutableData()), nullptr, + tmp_div, tmp_div, + reinterpret_cast(Y->template MutableData()), output_count); + + return Status::OK(); + } + + CUDNN_RETURN_IF_ERROR(cudnnReduceTensor( + CudnnHandle(), reduce_desc, nullptr, 0, workspace_cuda.get(), workspace_bytes, + &one, input_tensor, calculate_sqt_ ? input_data : reinterpret_cast(X->template Data()), + &zero, output_tensor, reinterpret_cast(Y->template MutableData()))); + } else { + size_t indices_bytes = 0; + CUDNN_RETURN_IF_ERROR(cudnnGetReductionIndicesSize(CudnnHandle(), reduce_desc, input_tensor, output_tensor, &indices_bytes)); + auto indices_cuda = GetScratchBuffer(indices_bytes); + + if (temp_X) { + auto temp_output = GetScratchBuffer(output_count); + CUDNN_RETURN_IF_ERROR(cudnnReduceTensor( + CudnnHandle(), reduce_desc, indices_cuda.get(), indices_bytes, workspace_cuda.get(), workspace_bytes, + &one, input_tensor, temp_X.get(), + &zero, output_tensor, temp_output.get())); + } else { + auto temp_output = GetScratchBuffer(output_count); + CUDNN_RETURN_IF_ERROR(cudnnReduceTensor( + CudnnHandle(), reduce_desc, indices_cuda.get(), indices_bytes, workspace_cuda.get(), workspace_bytes, + &one, input_tensor, reinterpret_cast(X->template Data()), + &zero, output_tensor, temp_output.get())); + } + + // CUDA reduction index is uint32_t for now, cast it to int64_t according to ONNX spec + Impl_Cast(reinterpret_cast(indices_cuda.get()), Y->template MutableData(), output_count); + } + + if (calculate_log_) { + Impl_Log(reinterpret_cast(Y->template MutableData()), + reinterpret_cast(Y->template MutableData()), + output_count); + } + + return Status::OK(); +} + +#define REGISTER_KERNEL_HFD(name) \ + REGISTER_KERNEL_TYPED(name, MLFloat16) \ + REGISTER_KERNEL_TYPED(name, float) \ + REGISTER_KERNEL_TYPED(name, double) + +REGISTER_KERNEL_HFD(ArgMax) +REGISTER_KERNEL_HFD(ArgMin) +REGISTER_KERNEL_HFD(ReduceL1) +REGISTER_KERNEL_HFD(ReduceL2) +REGISTER_KERNEL_HFD(ReduceMax) +REGISTER_KERNEL_HFD(ReduceMean) +REGISTER_KERNEL_HFD(ReduceMin) +REGISTER_KERNEL_HFD(ReduceProd) +REGISTER_KERNEL_HFD(ReduceSum) +REGISTER_KERNEL_HFD(ReduceLogSum) +REGISTER_KERNEL_HFD(ReduceSumSquare) +REGISTER_KERNEL_HFD(ReduceLogSumExp) + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/reduction/reduction_ops.h b/onnxruntime/core/providers/cuda/reduction/reduction_ops.h new file mode 100644 index 0000000000000..3436c4ada7f6d --- /dev/null +++ b/onnxruntime/core/providers/cuda/reduction/reduction_ops.h @@ -0,0 +1,158 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/providers/cuda/cuda_common.h" +#include "core/providers/cpu/reduction/reduction_ops.h" + +namespace onnxruntime { +namespace cuda { + +template +class ReduceKernel : public CudaKernel, public ReduceKernelBase { + protected: + ReduceKernel(const OpKernelInfo& info) : CudaKernel(info), + ReduceKernelBase(info), + calculate_log_(false), + calculate_sqt_(false), + log_sum_exp_(false) {} + + template + Status ComputeImpl(OpKernelContext* ctx, cudnnReduceTensorOp_t cudnnReduceOp) const; + + using ReduceKernelBase::axes_; + using ReduceKernelBase::keepdims_; + + bool calculate_log_; + bool calculate_sqt_; + bool log_sum_exp_; +}; + +template +class ArgMax final : public ReduceKernel { + public: + ArgMax(const OpKernelInfo& info) : ReduceKernel(info) {} + + Status ComputeInternal(OpKernelContext* ctx) const override { + return ComputeImpl(ctx, CUDNN_REDUCE_TENSOR_MAX); + } +}; + +template +class ArgMin final : public ReduceKernel { + public: + ArgMin(const OpKernelInfo& info) : ReduceKernel(info) {} + + Status ComputeInternal(OpKernelContext* ctx) const override { + return ComputeImpl(ctx, CUDNN_REDUCE_TENSOR_MIN); + } +}; + +template +class ReduceL1 final : public ReduceKernel { + public: + ReduceL1(const OpKernelInfo& info) : ReduceKernel(info) {} + + Status ComputeInternal(OpKernelContext* ctx) const override { + return ComputeImpl(ctx, CUDNN_REDUCE_TENSOR_NORM1); + } +}; + +template +class ReduceL2 final : public ReduceKernel { + public: + ReduceL2(const OpKernelInfo& info) : ReduceKernel(info) {} + + Status ComputeInternal(OpKernelContext* ctx) const override { + return ComputeImpl(ctx, CUDNN_REDUCE_TENSOR_NORM2); + } +}; + +template +class ReduceMax final : public ReduceKernel { + public: + ReduceMax(const OpKernelInfo& info) : ReduceKernel(info) {} + + Status ComputeInternal(OpKernelContext* ctx) const override { + return ComputeImpl(ctx, CUDNN_REDUCE_TENSOR_MAX); + } +}; + +template +class ReduceMean final : public ReduceKernel { + public: + ReduceMean(const OpKernelInfo& info) : ReduceKernel(info) {} + + Status ComputeInternal(OpKernelContext* ctx) const override { + return ComputeImpl(ctx, CUDNN_REDUCE_TENSOR_AVG); + } +}; + +template +class ReduceMin final : public ReduceKernel { + public: + ReduceMin(const OpKernelInfo& info) : ReduceKernel(info) {} + + Status ComputeInternal(OpKernelContext* ctx) const override { + return ComputeImpl(ctx, CUDNN_REDUCE_TENSOR_MIN); + } +}; + +template +class ReduceProd final : public ReduceKernel { + public: + ReduceProd(const OpKernelInfo& info) : ReduceKernel(info) {} + + Status ComputeInternal(OpKernelContext* ctx) const override { + return ComputeImpl(ctx, CUDNN_REDUCE_TENSOR_MUL); + } +}; + +template +class ReduceSum final : public ReduceKernel { + public: + ReduceSum(const OpKernelInfo& info) : ReduceKernel(info) {} + + Status ComputeInternal(OpKernelContext* ctx) const override { + return ComputeImpl(ctx, CUDNN_REDUCE_TENSOR_ADD); + } +}; + +template +class ReduceLogSum final : public ReduceKernel { + public: + ReduceLogSum(const OpKernelInfo& info) : ReduceKernel(info) { + ReduceKernel::calculate_log_ = true; + } + + Status ComputeInternal(OpKernelContext* ctx) const override { + return ComputeImpl(ctx, CUDNN_REDUCE_TENSOR_ADD); + } +}; + +template +class ReduceSumSquare final : public ReduceKernel { + public: + ReduceSumSquare(const OpKernelInfo& info) : ReduceKernel(info) { + ReduceKernel::calculate_sqt_ = true; + } + + Status ComputeInternal(OpKernelContext* ctx) const override { + return ComputeImpl(ctx, CUDNN_REDUCE_TENSOR_ADD); + } +}; + +template +class ReduceLogSumExp final : public ReduceKernel { + public: + ReduceLogSumExp(const OpKernelInfo& info) : ReduceKernel(info) { + ReduceKernel::log_sum_exp_ = true; + } + + Status ComputeInternal(OpKernelContext* ctx) const override { + return ComputeImpl(ctx, CUDNN_REDUCE_TENSOR_ADD); + } +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/rnn/cudnn_rnn_base.cc b/onnxruntime/core/providers/cuda/rnn/cudnn_rnn_base.cc new file mode 100644 index 0000000000000..907e831ce984f --- /dev/null +++ b/onnxruntime/core/providers/cuda/rnn/cudnn_rnn_base.cc @@ -0,0 +1,308 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "cudnn_rnn_base.h" +#include "rnn_impl.h" +#include "core/providers/cpu/rnn/rnn_helpers.h" + +namespace onnxruntime { +namespace cuda { + +template +void CudnnRnnBase::SetWeightBias(const cudnnHandle_t handle, + const cudnnRNNDescriptor_t rnn_desc, + const int pseudo_layer, + const cudnnTensorDescriptor_t x_desc, + const cudnnFilterDescriptor_t w_desc, + const cudnnFilterDescriptor_t filter_desc, + const void* w_data, + const int lin_layer_id, + const T* pos, + int& offset, + bool is_matrix) const { + int numDims; + std::vector matDims(3); + cudnnDataType_t dt; + cudnnTensorFormat_t tf; + T* mem_offset; + + if (is_matrix) { + cudnnGetRNNLinLayerMatrixParams(handle, rnn_desc, pseudo_layer, x_desc, w_desc, w_data, lin_layer_id, filter_desc, (void**)&mem_offset); + } else { + cudnnGetRNNLinLayerBiasParams(handle, rnn_desc, pseudo_layer, x_desc, w_desc, w_data, lin_layer_id, filter_desc, (void**)&mem_offset); + } + + cudnnGetFilterNdDescriptor(filter_desc, 3, &dt, &tf, &numDims, matDims.data()); + int count = matDims[0] * matDims[1] * matDims[2]; + cudaMemcpyAsync(mem_offset, pos + offset, count * sizeof(T), cudaMemcpyDeviceToDevice); + offset += count; +} +template +Status CudnnRnnBase::SetCudnnRnnWeightBias(const cudnnHandle_t cudnn_handle, + const cudnnRNNDescriptor_t rnn_desc, + const cudnnTensorDescriptor_t x_desc, + const cudnnFilterDescriptor_t w_desc, + void* w_data, + const T* W_data, + const T* R_data, + const T* B_data) const { + //Onnx only support 1 layer + CudnnFilterDescriptor filter_desc; + int w_offset = 0; + int r_offset = 0; + int bias_offset = 0; + for (int layer = 0; layer < num_layers_ * num_directions_; ++layer) { + for (int idx = 0; idx < W_lin_layer_id_.size(); ++idx) { + SetWeightBias(cudnn_handle, rnn_desc, layer, x_desc, w_desc, filter_desc, w_data, W_lin_layer_id_[idx], W_data, w_offset, true); + if (B_data != nullptr) { + SetWeightBias(cudnn_handle, rnn_desc, layer, x_desc, w_desc, filter_desc, w_data, W_lin_layer_id_[idx], B_data, bias_offset, false); + } + } + for (int idx = 0; idx < R_lin_layer_id_.size(); ++idx) { + SetWeightBias(cudnn_handle, rnn_desc, layer, x_desc, w_desc, filter_desc, w_data, R_lin_layer_id_[idx], R_data, r_offset, true); + if (B_data != nullptr) { + SetWeightBias(cudnn_handle, rnn_desc, layer, x_desc, w_desc, filter_desc, w_data, R_lin_layer_id_[idx], B_data, bias_offset, false); + } + } + } + + return Status::OK(); +} + +template +Status CudnnRnnBase::SetCudnnRnnDesc() { + typedef typename ToCudaType::MappedType CudaT; + + cudnnDirectionMode_t cudnn_direction = CUDNN_UNIDIRECTIONAL; + if (direction_ == "bidirectional") { + cudnn_direction = CUDNN_BIDIRECTIONAL; + } else if (direction_ == "forward") { + cudnn_direction = CUDNN_UNIDIRECTIONAL; + } else if (direction_ == "reverse") { + cudnn_direction = CUDNN_UNIDIRECTIONAL; + // need to reverse data + reverse_ = true; + } + + CudnnDropout cudnn_dropout_desc; + cudnn_dropout_desc.Set(CudnnHandle()); + ONNXRUNTIME_RETURN_IF_ERROR(rnn_desc_.Set(CudnnHandle(), hidden_size_, num_layers_, cudnn_dropout_desc, + cudnn_direction, rnn_mode_, CudnnTensor::GetDataType())); + + return Status::OK(); +} + +template +Status CudnnRnnBase::ReorganizeWeights(const Tensor* W, const Tensor* R, const Tensor* B, + IAllocatorUniquePtr& target_w_data, + CudnnFilterDescriptor& target_w_desc) const { + typedef typename ToCudaType::MappedType CudaT; + int64_t input_size = W->Shape()[2]; + // RNN W[num_directions_, hidden_size_, input_size] + // RNN R[num_directions_, hidden_size_, hidden_size_] + // RNN B[num_directions_, 2*hidden_size_] + // GRU W[num_directions_, 3*hidden_size_, input_size] + // GRU R[num_directions_, 3*hidden_size_, hidden_size_] + // GRU B[num_directions_, 6*hidden_size_] + // LSTM W[num_directions_, 4*hidden_size_, input_size] + // LSTM R[num_directions_, 4*hidden_size_, hidden_size_] + // LSTM B[num_directions_, 8*hidden_size_] + size_t number = W_lin_layer_id_.size(); + int64_t w_size = num_directions_ * (number * hidden_size_ * (input_size + hidden_size_ + 2)); + std::vector dims_w({w_size, 1, 1}); + ONNXRUNTIME_RETURN_IF_ERROR(target_w_desc.Set(dims_w, CudnnTensor::GetDataType())); + + std::vector fake_dims_x({1, input_size, 1}); + CudnnTensor fake_x_desc; + fake_x_desc.Set(fake_dims_x, CudnnTensor::GetDataType()); + + // Prepare the weight data + target_w_data = GetScratchBuffer(w_size * sizeof(T)); + + const T* W_data = W->template Data(); + const T* R_data = R->template Data(); + const T* B_data = B == nullptr ? nullptr : B->template Data(); + + ONNXRUNTIME_RETURN_IF_ERROR(SetCudnnRnnWeightBias(CudnnHandle(), rnn_desc_, fake_x_desc, target_w_desc, + target_w_data.get(), W_data, R_data, B_data)); + + return Status::OK(); +} + +template +Status CudnnRnnBase::CacheCudnnRnnWeights(const OpKernelInfo& info) { + // Cache the weight + const Tensor* W; + const Tensor* R; + const Tensor* B; + bool get_W = info.TryGetConstantInput(Input_Index::W, &W); + bool get_R = info.TryGetConstantInput(Input_Index::R, &R); + + if (get_W && get_R) { + info.TryGetConstantInput(Input_Index::B, &B); + ONNXRUNTIME_RETURN_IF_ERROR(ReorganizeWeights(W, R, B, w_data_cache_, w_desc_cache_)); + weight_cached_ = true; + } + + return Status::OK(); +} + +template +Status CudnnRnnBase::ComputeInternal(OpKernelContext* ctx) const { + typedef typename ToCudaType::MappedType CudaT; + + // inputs + const Tensor* X = ctx->Input(Input_Index::X); // inputs. [seq_length, batch_size, input_size] + ONNXRUNTIME_ENFORCE(nullptr != X); + + // optional inputs + const Tensor* sequence_lens = ctx->Input(Input_Index::sequence_lens); // [batch_size] + const Tensor* initial_h = ctx->Input(Input_Index::initial_h); // initial hidden. [num_directions_, batch_size, hidden_size_] + const Tensor* initial_c(nullptr); + if (rnn_mode_ == CUDNN_LSTM) { + initial_c = ctx->Input(Input_Index::initial_c); // initial cell. [num_directions_, batch_size, hidden_size_] + } + + int64_t seq_length = X->Shape()[0]; + int64_t batch_size = X->Shape()[1]; + int64_t input_size = X->Shape()[2]; + + // optional outputs + std::vector dims_Y({seq_length, num_directions_, batch_size, hidden_size_}); + std::vector dims_hxy({num_layers_ * num_directions_, batch_size, hidden_size_}); + std::vector dims_yc{num_directions_, batch_size, hidden_size_}; + Tensor* Y = ctx->Output(Output_Index::Y, dims_Y); + Tensor* Y_h = ctx->Output(Output_Index::Y_h, dims_hxy); + Tensor* Y_c = ctx->Output(Output_Index::Y_c, dims_yc); + + std::vector dims_x({batch_size, input_size, 1}); + std::vector dims_y({batch_size, hidden_size_ * num_directions_, 1}); + + CudnnTensor x_desc_temp; + x_desc_temp.Set(dims_x, CudnnTensor::GetDataType()); + CudnnTensor y_desc_temp; + y_desc_temp.Set(dims_y, CudnnTensor::GetDataType()); + std::vector x_desc(seq_length, x_desc_temp); + std::vector y_desc(seq_length, y_desc_temp); + + CudnnTensor hx_desc; + CudnnTensor cx_desc; + CudnnTensor y_h_desc; + CudnnTensor y_c_desc; + ONNXRUNTIME_RETURN_IF_ERROR(hx_desc.Set(dims_hxy, CudnnTensor::GetDataType())); + ONNXRUNTIME_RETURN_IF_ERROR(cx_desc.Set(dims_hxy, CudnnTensor::GetDataType())); + ONNXRUNTIME_RETURN_IF_ERROR(y_h_desc.Set(dims_hxy, CudnnTensor::GetDataType())); + ONNXRUNTIME_RETURN_IF_ERROR(y_c_desc.Set(dims_hxy, CudnnTensor::GetDataType())); + + // Prepare the weight data + IAllocatorUniquePtr w_data; + CudnnFilterDescriptor w_desc; + if (!weight_cached_) { + const Tensor& W = *ctx->Input(Input_Index::W); + const Tensor& R = *ctx->Input(Input_Index::R); + const Tensor* B = ctx->Input(Input_Index::B); + ReorganizeWeights(&W, &R, B, w_data, w_desc); + } + + IAllocatorUniquePtr x_reversed_data; + T* x_data = const_cast(X->template Data()); + if (reverse_) { + // reverse input data + x_reversed_data = GetScratchBuffer(seq_length * batch_size * input_size); + ReverseBySequence(gsl::narrow_cast(seq_length), + gsl::narrow_cast(batch_size), + gsl::narrow_cast(input_size), + reinterpret_cast(x_data), + reinterpret_cast(x_reversed_data.get()), + seq_length * batch_size * input_size); + x_data = x_reversed_data.get(); + } + + const T* hx_data = (initial_h == nullptr) ? nullptr : initial_h->template Data(); + const T* cx_data = (initial_c == nullptr) ? nullptr : initial_c->template Data(); + T* y_h_data = (Y_h == nullptr) ? nullptr : Y_h->template MutableData(); + T* y_c_data = (Y_c == nullptr) ? nullptr : Y_c->template MutableData(); + int64_t output_size = seq_length * num_directions_ * batch_size * hidden_size_; + T* y_data = nullptr; + IAllocatorUniquePtr y_alloc_data; + if (Y != nullptr) { + y_data = Y->template MutableData(); + } else { + y_alloc_data = GetScratchBuffer(output_size); + y_data = y_alloc_data.get(); + } + const int32_t* sequence_lens_data = (sequence_lens == nullptr) ? nullptr : sequence_lens->template Data(); + + size_t workspace_bytes; + CUDNN_RETURN_IF_ERROR(cudnnGetRNNWorkspaceSize(CudnnHandle(), rnn_desc_, gsl::narrow_cast(seq_length), x_desc.data(), &workspace_bytes)); + workspace_bytes *= num_directions_; + auto workspace_cuda = GetScratchBuffer(workspace_bytes); + + CUDNN_RETURN_IF_ERROR(cudnnRNNForwardInference(CudnnHandle(), + rnn_desc_, + gsl::narrow_cast(seq_length), + x_desc.data(), + x_data, + hx_desc, + hx_data, + cx_desc, + cx_data, + weight_cached_ ? w_desc_cache_ : w_desc, + weight_cached_ ? w_data_cache_.get() : w_data.get(), + y_desc.data(), + y_data, + y_h_desc, + y_h_data, + y_c_desc, + y_c_data, + workspace_cuda.get(), + workspace_bytes)); + IAllocatorUniquePtr y_reorganized_data; + if (reverse_ || num_directions_ == 2) { + //reverse output + y_reorganized_data = GetScratchBuffer(output_size); + if (reverse_) { + //reverse output data + ReverseBySequence(gsl::narrow_cast(seq_length), + gsl::narrow_cast(batch_size), + gsl::narrow_cast(hidden_size_), + reinterpret_cast(y_data), + reinterpret_cast(y_reorganized_data.get()), + output_size); + } else { + ReorderBidirectionalDataInSequence(gsl::narrow_cast(seq_length), + gsl::narrow_cast(batch_size), + gsl::narrow_cast(hidden_size_), + reinterpret_cast(y_data), + reinterpret_cast(y_reorganized_data.get()), + output_size); + } + + if (Y != nullptr) { + // User specified this optional output, so need to copy the reversed data to orignial place + cudaMemcpyAsync(y_data, y_reorganized_data.get(), output_size * sizeof(T), cudaMemcpyDeviceToDevice); + } else { + y_data = y_reorganized_data.get(); + } + } + + if (sequence_lens_data != nullptr && y_h_data != nullptr && y_data != nullptr) { + RnnMaskImpl(gsl::narrow_cast(num_directions_), + gsl::narrow_cast(seq_length), + gsl::narrow_cast(batch_size), + gsl::narrow_cast(hidden_size_), + sequence_lens_data, + reinterpret_cast(y_data), + reinterpret_cast(y_h_data), + output_size); + } + + return Status::OK(); +} + +template class CudnnRnnBase; +template class CudnnRnnBase; +template class CudnnRnnBase; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/rnn/cudnn_rnn_base.h b/onnxruntime/core/providers/cuda/rnn/cudnn_rnn_base.h new file mode 100644 index 0000000000000..18c82863a52ba --- /dev/null +++ b/onnxruntime/core/providers/cuda/rnn/cudnn_rnn_base.h @@ -0,0 +1,176 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "gsl/gsl_util" +#include "core/providers/cuda/cudnn_common.h" +#include "core/providers/cuda/cuda_common.h" +#include + +namespace onnxruntime { +namespace cuda { + +class CudnnDropout { + public: + CudnnDropout() : dropout_desc_(nullptr) { + } + + Status Set(const cudnnHandle_t& cudnnHandle, float dropout = 0.0f, unsigned long long seed = 1) { + CUDNN_RETURN_IF_ERROR(cudnnCreateDropoutDescriptor(&dropout_desc_)); + size_t stateSize; + void* states; + CUDNN_RETURN_IF_ERROR(cudnnDropoutGetStatesSize(cudnnHandle, &stateSize)); + + CUDA_CALL(cudaMalloc(&states, stateSize)); + + CUDNN_RETURN_IF_ERROR(cudnnSetDropoutDescriptor(dropout_desc_, + cudnnHandle, + dropout, + states, + stateSize, + seed)); + + return Status::OK(); + } + + ~CudnnDropout() { + if (dropout_desc_ != nullptr) { + cudnnDestroyDropoutDescriptor(dropout_desc_); + } + } + + operator cudnnDropoutDescriptor_t() const { + return dropout_desc_; + } + + private: + cudnnDropoutDescriptor_t dropout_desc_; +}; + +class CudnnRNN { + public: + CudnnRNN() : rnn_desc_(nullptr) { + } + + ~CudnnRNN() { + if (rnn_desc_ != nullptr) { + cudnnDestroyRNNDescriptor(rnn_desc_); + rnn_desc_ = nullptr; + } + } + + Status Set(const cudnnHandle_t& cudnnHandle, int64_t hidden_size, int num_layers, + cudnnDropoutDescriptor_t cudnn_dropout_desc, cudnnDirectionMode_t cudnn_direction_model, + cudnnRNNMode_t rnn_mode, cudnnDataType_t dataType) { + if (!rnn_desc_) + CUDNN_RETURN_IF_ERROR(cudnnCreateRNNDescriptor(&rnn_desc_)); + + CUDNN_RETURN_IF_ERROR(cudnnSetRNNDescriptor(cudnnHandle, + rnn_desc_, + gsl::narrow_cast(hidden_size), + num_layers, + cudnn_dropout_desc, + CUDNN_LINEAR_INPUT, // We can also skip the input matrix transformation + cudnn_direction_model, + rnn_mode, + CUDNN_RNN_ALGO_STANDARD, //CUDNN_RNN_ALGO_PERSIST_STATIC, CUDNN_RNN_ALGO_PERSIST_DYNAMIC + dataType)); + + return Status::OK(); + } + + operator cudnnRNNDescriptor_t() const { + return rnn_desc_; + } + + private: + cudnnRNNDescriptor_t rnn_desc_; +}; + +template +class CudnnRnnBase : public CudaKernel { + const std::set allowed_directions{"forward", "reverse", "bidirectional"}; + + public: + CudnnRnnBase(const OpKernelInfo& info) : CudaKernel{info} { + reverse_ = false; + ONNXRUNTIME_ENFORCE(info.GetAttr("direction", &direction_).IsOK()); + num_directions_ = direction_ == "bidirectional" ? 2 : 1; + ONNXRUNTIME_ENFORCE(allowed_directions.find(direction_) != allowed_directions.end()); + + ONNXRUNTIME_ENFORCE(info.GetAttr("hidden_size", &hidden_size_).IsOK() && hidden_size_ > 0); + rnn_mode_ = CUDNN_LSTM; + num_layers_ = 1; + weight_cached_ = false; + w_data_cache_ = nullptr; + } + + Status SetCudnnRnnDesc(); + + Status CacheCudnnRnnWeights(const OpKernelInfo& info); + + Status ComputeInternal(OpKernelContext* ctx) const override; + + private: + Status SetCudnnRnnWeightBias(const cudnnHandle_t cudnn_handle, + const cudnnRNNDescriptor_t rnn_desc, + const cudnnTensorDescriptor_t x_desc, + const cudnnFilterDescriptor_t w_desc, + void* w_data, + const T* W_data, + const T* R_data, + const T* B_data) const; + + Status ReorganizeWeights(const Tensor* W, const Tensor* R, const Tensor* B, + IAllocatorUniquePtr& target_w_data, + CudnnFilterDescriptor& target_w_desc) const; + + void SetWeightBias(const cudnnHandle_t handle, + const cudnnRNNDescriptor_t rnn_desc, + const int pseudo_layer, + const cudnnTensorDescriptor_t x_desc, + const cudnnFilterDescriptor_t w_desc, + const cudnnFilterDescriptor_t filter_desc, + const void* w_data, + const int lin_layer_id, + const T* pos, + int& offset, + bool is_matrix) const; + + protected: + int64_t num_directions_; + // required + int64_t hidden_size_; + cudnnRNNMode_t rnn_mode_; + std::vector W_lin_layer_id_; + std::vector R_lin_layer_id_; + CudnnRNN rnn_desc_; + bool reverse_; + int num_layers_; + + private: + // optional + std::string direction_; + CudnnFilterDescriptor w_desc_cache_; + IAllocatorUniquePtr w_data_cache_; + bool weight_cached_; + + enum Input_Index { + X = 0, + W = 1, + R = 2, + B = 3, + sequence_lens = 4, + initial_h = 5, + initial_c = 6 + }; + enum Output_Index { + Y = 0, + Y_h = 1, + Y_c = 2 + }; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/rnn/gru.cc b/onnxruntime/core/providers/cuda/rnn/gru.cc new file mode 100644 index 0000000000000..38f311008c7c8 --- /dev/null +++ b/onnxruntime/core/providers/cuda/rnn/gru.cc @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gru.h" +#include "rnn_impl.h" +#include "core/providers/common.h" +#include "core/providers/cuda/cudnn_common.h" +#include "core/providers/cpu/math/gemm_helper.h" + +namespace onnxruntime { +namespace cuda { + +#define REGISTER_KERNEL_TYPED(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + GRU, \ + kOnnxDomain, \ + 7, \ + T, \ + kCudaExecutionProvider, \ + KernelDefBuilder() \ + .TypeConstraint("T", DataTypeImpl::GetTensorType()) \ + .TypeConstraint("T1", DataTypeImpl::GetTensorType()), \ + GRU); + +REGISTER_KERNEL_TYPED(float); +REGISTER_KERNEL_TYPED(double); +REGISTER_KERNEL_TYPED(MLFloat16); + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/rnn/gru.h b/onnxruntime/core/providers/cuda/rnn/gru.h new file mode 100644 index 0000000000000..d69c122ee4dff --- /dev/null +++ b/onnxruntime/core/providers/cuda/rnn/gru.h @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "cudnn_rnn_base.h" +#include "gsl/gsl_util" +#include "core/providers/cuda/cuda_common.h" +#include + +namespace onnxruntime { +namespace cuda { + +template +class GRU final : public CudnnRnnBase { + public: + GRU(const OpKernelInfo& info) : CudnnRnnBase(info) { + CudnnRnnBase::rnn_mode_ = CUDNN_GRU; + CudnnRnnBase::SetCudnnRnnDesc(); + + // ONNX W layout is Wzrh, WBzrh, mapping to RNNLinLayerMatrixParams the linLayerID is 0, 3, 1, 2 + CudnnRnnBase::W_lin_layer_id_.assign({1, 0, 2}); + // ONNX R layout is Rzrh, RBzrh, mapping to RNNLinLayerMatrixParams the linLayerID is 4, 7, 5, 6 + CudnnRnnBase::R_lin_layer_id_.assign({4, 3, 5}); + // ONNX B layout is Wbzrh, Rbzrh, mapping to RNNLinLayerMatrixParams + // the linLayerID is 1, 0, 2, 4, 3, 5, we can reuse it from W_lin_layer_id & R_lin_layer_id + + CudnnRnnBase::CacheCudnnRnnWeights(info); + } +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/rnn/lstm.cc b/onnxruntime/core/providers/cuda/rnn/lstm.cc new file mode 100644 index 0000000000000..e78ddf534f294 --- /dev/null +++ b/onnxruntime/core/providers/cuda/rnn/lstm.cc @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "lstm.h" +#include "core/providers/cpu/rnn/rnn_helpers.h" +#include "core/providers/common.h" + +namespace onnxruntime { +namespace cuda { + +#define REGISTER_KERNEL_TYPED(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + LSTM, \ + kOnnxDomain, \ + 7, \ + T, \ + kCudaExecutionProvider, \ + KernelDefBuilder() \ + .TypeConstraint("T", DataTypeImpl::GetTensorType()) \ + .TypeConstraint("T1", DataTypeImpl::GetTensorType()), \ + LSTM); + +REGISTER_KERNEL_TYPED(float); +REGISTER_KERNEL_TYPED(double); +REGISTER_KERNEL_TYPED(MLFloat16); + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/rnn/lstm.h b/onnxruntime/core/providers/cuda/rnn/lstm.h new file mode 100644 index 0000000000000..3ba719d61750d --- /dev/null +++ b/onnxruntime/core/providers/cuda/rnn/lstm.h @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "cudnn_rnn_base.h" + +namespace onnxruntime { +namespace cuda { + +template +class LSTM final : public CudnnRnnBase { + + public: + LSTM(const OpKernelInfo& info) : CudnnRnnBase(info) { + CudnnRnnBase::rnn_mode_ = CUDNN_LSTM; + CudnnRnnBase::SetCudnnRnnDesc(); + + // ONNX W layout is W[iofc], WB[iofc], mapping to RNNLinLayerMatrixParams the linLayerID is 0, 3, 1, 2 + CudnnRnnBase::W_lin_layer_id_.assign({0, 3, 1, 2}); + // ONNX R layout is R[iofc], RB[iofc], mapping to RNNLinLayerMatrixParams the linLayerID is 4, 7, 5, 6 + CudnnRnnBase::R_lin_layer_id_.assign({4, 7, 5, 6}); + // ONNX B layout is Wb[iofc], Rb[iofc], mapping to RNNLinLayerMatrixParams + // the linLayerID is 0, 3, 1, 2, 4, 7, 5, 6, we can reuse it from W_lin_layer_id & R_lin_layer_id + + CudnnRnnBase::CacheCudnnRnnWeights(info); + } +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/rnn/rnn.cc b/onnxruntime/core/providers/cuda/rnn/rnn.cc new file mode 100644 index 0000000000000..277b5a88b6e62 --- /dev/null +++ b/onnxruntime/core/providers/cuda/rnn/rnn.cc @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "rnn.h" +#include "rnn_impl.h" +#include "core/providers/common.h" +#include "core/providers/cuda/cudnn_common.h" +#include "core/providers/cpu/math/gemm_helper.h" + +namespace onnxruntime { +namespace cuda { + +#define REGISTER_KERNEL_TYPED(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + RNN, \ + kOnnxDomain, \ + 7, \ + T, \ + kCudaExecutionProvider, \ + KernelDefBuilder() \ + .TypeConstraint("T", DataTypeImpl::GetTensorType()) \ + .TypeConstraint("T1", DataTypeImpl::GetTensorType()), \ + RNN); + +REGISTER_KERNEL_TYPED(float); +REGISTER_KERNEL_TYPED(double); +REGISTER_KERNEL_TYPED(MLFloat16); + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/rnn/rnn.h b/onnxruntime/core/providers/cuda/rnn/rnn.h new file mode 100644 index 0000000000000..30cadec069f6f --- /dev/null +++ b/onnxruntime/core/providers/cuda/rnn/rnn.h @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "cudnn_rnn_base.h" +#include "gsl/gsl_util" +#include "core/providers/cuda/cuda_common.h" +#include + +namespace onnxruntime { +namespace cuda { + +template +class RNN final : public CudnnRnnBase { + const std::set allowed_activations{"Relu", "Tanh" /*, "Sigmoid"*/}; + + public: + RNN(const OpKernelInfo& info) : CudnnRnnBase(info) { + std::vector activations_; + ONNXRUNTIME_ENFORCE(info.GetAttrs("activations", activations_).IsOK()); + if (activations_[0] == "Relu") + CudnnRnnBase::rnn_mode_ = CUDNN_RNN_RELU; + else if (activations_[0] == "Tanh") + CudnnRnnBase::rnn_mode_ = CUDNN_RNN_TANH; + + CudnnRnnBase::SetCudnnRnnDesc(); + + // ONNX W mapping to RNNLinLayerMatrixParams the linLayerID is 0 + CudnnRnnBase::W_lin_layer_id_.assign({0}); + // ONNX R mapping to RNNLinLayerMatrixParams the linLayerID is 1 + CudnnRnnBase::R_lin_layer_id_.assign({1}); + // ONNX B layout is Wb, Rb, mapping to RNNLinLayerMatrixParams + // the linLayerID is 0, 1, we can reuse it from W_lin_layer_id & R_lin_layer_id + + CudnnRnnBase::CacheCudnnRnnWeights(info); + } +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/rnn/rnn_impl.cu b/onnxruntime/core/providers/cuda/rnn/rnn_impl.cu new file mode 100644 index 0000000000000..fb7b9f4753685 --- /dev/null +++ b/onnxruntime/core/providers/cuda/rnn/rnn_impl.cu @@ -0,0 +1,159 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cuda/cu_inc/common.cuh" +#include "rnn_impl.h" + +namespace onnxruntime { +namespace cuda { + +template +__global__ void _ReverseBySequenceKernel(const int32_t seq_length, + const int32_t block_size, + const fast_divmod div_batch_block, + const T* data, + T* reversed_data, + const CUDA_LONG N) { + CALCULATE_ELEMENTWISE_INDEX_OR_EXIT(id, N); + int seq_id, offset; + div_batch_block.divmod(id, seq_id, offset); + int org_id = (seq_length - seq_id - 1) * block_size + offset; + reversed_data[id] = data[org_id]; +} + +template +void ReverseBySequence(const int32_t seq_length, + const int32_t batch_size, + const int32_t input_or_hidden_size, + const T* data, + T* reversed_data, + const size_t N) { + // kerneral + int32_t block_size = batch_size * input_or_hidden_size; + fast_divmod div_batch_block(block_size); + int blocksPerGrid = (int)(ceil(static_cast(N) / GridDim::maxThreadsPerBlock)); + _ReverseBySequenceKernel<<>>( + seq_length, block_size, div_batch_block, data, reversed_data, (CUDA_LONG)N); +} + +template +__global__ void _BidirectionalDataKernel(const int32_t seq_length, + const int32_t batch_size, + const int32_t hidden_size, + const int32_t seq_block_size, + const fast_divmod div_seq_block, + const fast_divmod div_output_block, + const T* data, + T* reordered_data, + const CUDA_LONG N) { + CALCULATE_ELEMENTWISE_INDEX_OR_EXIT(id, N); + int seq_id, seq_offset, output_id, offset; + div_seq_block.divmod(id, seq_id, seq_offset); + div_output_block.divmod(seq_offset, output_id, offset); + int org_output_id = 0; + if (output_id < batch_size) { + org_output_id = 2 * output_id; + } else { + org_output_id = (output_id - batch_size) * 2 + 1; + } + int org_id = seq_id * seq_block_size + org_output_id * hidden_size + offset; + reordered_data[id] = data[org_id]; +} + +template +void ReorderBidirectionalDataInSequence(const int32_t seq_length, + const int32_t batch_size, + const int32_t hidden_size, + const T* data, + T* reordered_data, + const size_t N) { + // The cudnn Y output is organize like [Y1, YB1] [Y2, YB2] ... + // need to reorganize it to [Y1, Y2, ...] [YB1, YB2, ...] + int32_t seq_block_size = 2 * batch_size * hidden_size; + fast_divmod div_seq_block(seq_block_size); + fast_divmod div_output_block(hidden_size); + int blocksPerGrid = (int)(ceil(static_cast(N) / GridDim::maxThreadsPerBlock)); + + _BidirectionalDataKernel<<>>( + seq_length, batch_size, hidden_size, seq_block_size, + div_seq_block, div_output_block, + data, reordered_data, (CUDA_LONG)N); +} + +template +__global__ void _RnnMaskKernel(const int32_t seq_length, + const int32_t batch_size, + const int32_t hidden_size, + const int32_t* sequence_lens, + const fast_divmod div_seq_block, + const fast_divmod div_batch_block, + T* y_output_data, + T* y_h_output_data, + const CUDA_LONG N) { + CALCULATE_ELEMENTWISE_INDEX_OR_EXIT(id, N); + + int seq_id, offset, batch_id, batch_offset; + div_seq_block.divmod(id, seq_id, offset); + div_batch_block.divmod(offset, batch_id, batch_offset); + int32_t batch_seq_length = sequence_lens[batch_id]; + + if (batch_id >= batch_size || batch_seq_length == seq_length) { + return; + } + + if (seq_id >= batch_seq_length) { + y_output_data[id] = 0; + return; + } + + if ((y_h_output_data != nullptr) && (batch_seq_length != seq_length) && ((seq_id + 1) == batch_seq_length)) { + int hy_idx = batch_id * hidden_size + batch_offset; + y_h_output_data[hy_idx] = y_output_data[id]; + } +} + +template +void RnnMaskImpl(const int32_t num_directions, + const int32_t seq_length, + const int32_t batch_size, + const int32_t hidden_size, + const int32_t* sequence_lens, + T* y_output_data, + T* y_h_output_data, + const size_t N) { + fast_divmod div_seq_block(batch_size * hidden_size * num_directions); + fast_divmod div_batch_block(hidden_size); + int blocksPerGrid = (int)(ceil(static_cast(N) / GridDim::maxThreadsPerBlock)); + _RnnMaskKernel<<>>( + seq_length, batch_size, hidden_size, sequence_lens, + div_seq_block, div_batch_block, y_output_data, y_h_output_data, (CUDA_LONG)N); +} + +#define SPECIALIZED_RNN_IMPL(T) \ + template void RnnMaskImpl(const int32_t num_directions, \ + const int32_t seq_length, \ + const int32_t batch_size, \ + const int32_t hidden_size, \ + const int32_t* sequence_lens, \ + T* y_output_data, \ + T* y_h_output_data, \ + const size_t N); \ + template void ReverseBySequence(const int32_t seq_length, \ + const int32_t batch_size, \ + const int32_t hidden_size, \ + const T* data, \ + T* reversed_data, \ + const size_t N); \ + template void ReorderBidirectionalDataInSequence(const int32_t seq_length, \ + const int32_t batch_size, \ + const int32_t hidden_size,\ + const T* data, \ + T* reordered_data, \ + const size_t N); + +SPECIALIZED_RNN_IMPL(half) +SPECIALIZED_RNN_IMPL(float) +SPECIALIZED_RNN_IMPL(double) + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/rnn/rnn_impl.h b/onnxruntime/core/providers/cuda/rnn/rnn_impl.h new file mode 100644 index 0000000000000..d25d71aed3fb1 --- /dev/null +++ b/onnxruntime/core/providers/cuda/rnn/rnn_impl.h @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include "core/providers/cuda/shared_inc/cuda_utils.h" + +namespace onnxruntime { +namespace cuda { + +template +void ReverseBySequence(const int32_t seq_length, + const int32_t batch_size, + const int32_t input_or_hidden_size, + const T* data, + T* reversed_data, + const size_t N); + +template +void ReorderBidirectionalDataInSequence(const int32_t seq_length, + const int32_t batch_size, + const int32_t hidden_size, + const T* data, + T* reordered_data, + const size_t N); + +template +void RnnMaskImpl(const int32_t num_directions, + const int32_t seq_length, + const int32_t batch_size, + const int32_t hidden_size, + const int32_t* sequence_lens, + T* y_output_data, + T* y_h_output_data, + const size_t N); + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/shared_inc/cuda_call.h b/onnxruntime/core/providers/cuda/shared_inc/cuda_call.h new file mode 100644 index 0000000000000..34b049afe5586 --- /dev/null +++ b/onnxruntime/core/providers/cuda/shared_inc/cuda_call.h @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/providers/cuda/cuda_pch.h" + +namespace onnxruntime { + +// ----------------------------------------------------------------------- +// Error handling +// ----------------------------------------------------------------------- + +template +bool CudaCall(ERRTYPE retCode, const char* exprString, const char* libName, ERRTYPE successCode, const char* msg = ""); + +#define CUDA_CALL(expr) (CudaCall((expr), #expr, "CUDA", cudaSuccess)) +#define CUBLAS_CALL(expr) (CudaCall((expr), #expr, "CUBLAS", CUBLAS_STATUS_SUCCESS)) +#define CUSPARSE_CALL(expr) (CudaCall((expr), #expr, "CUSPARSE", CUSPARSE_STATUS_SUCCESS)) +#define CURAND_CALL(expr) (CudaCall((expr), #expr, "CURAND", CURAND_STATUS_SUCCESS)) +#define CUDNN_CALL(expr) (CudaCall((expr), #expr, "CUDNN", CUDNN_STATUS_SUCCESS)) +#define CUDNN_CALL2(expr, m) (CudaCall((expr), #expr, "CUDNN", CUDNN_STATUS_SUCCESS, m)) + +#define CUDA_CALL_THROW(expr) (CudaCall((expr), #expr, "CUDA", cudaSuccess)) +#define CUBLAS_CALL_THROW(expr) (CudaCall((expr), #expr, "CUBLAS", CUBLAS_STATUS_SUCCESS)) +#define CUSPARSE_CALL_THROW(expr) (CudaCall((expr), #expr, "CUSPARSE", CUSPARSE_STATUS_SUCCESS)) +#define CURAND_CALL_THROW(expr) (CudaCall((expr), #expr, "CURAND", CURAND_STATUS_SUCCESS)) +#define CUDNN_CALL_THROW(expr) (CudaCall((expr), #expr, "CUDNN", CUDNN_STATUS_SUCCESS)) +#define CUDNN_CALL_THROW2(expr, m) (CudaCall((expr), #expr, "CUDNN", CUDNN_STATUS_SUCCESS, m)) + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/shared_inc/cuda_utils.h b/onnxruntime/core/providers/cuda/shared_inc/cuda_utils.h new file mode 100644 index 0000000000000..741087685f534 --- /dev/null +++ b/onnxruntime/core/providers/cuda/shared_inc/cuda_utils.h @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// for things shared between nvcc and onnxruntime +// as currently nvcc cannot compile all onnxruntime headers + +#pragma once +#include +#include +#include "fast_divmod.h" + +namespace onnxruntime { +namespace cuda { + +enum class SimpleBroadcast : size_t { + NoBroadcast = (size_t)-1, + LeftScalar = (size_t)-2, + RightScalar = (size_t)-3, + RightPerChannelBatch1 = (size_t)-4, + RightPerChannelBatchN = (size_t)-5, +}; + +template +class IConstantBuffer { + public: + virtual ~IConstantBuffer(){}; + virtual const T* GetBuffer(size_t count) = 0; +}; + +template +std::unique_ptr> CreateConstantOnes(); + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/shared_inc/fast_divmod.h b/onnxruntime/core/providers/cuda/shared_inc/fast_divmod.h new file mode 100644 index 0000000000000..f392290f1cbd0 --- /dev/null +++ b/onnxruntime/core/providers/cuda/shared_inc/fast_divmod.h @@ -0,0 +1,131 @@ +// +// Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +// + +#pragma once + +#include +#include + +namespace onnxruntime { +namespace cuda { + +__host__ __device__ __inline__ int mulhi(const int M, const int n) { +#ifdef __CUDA_ARCH__ + return __mulhi(M, n); +#else + return (((unsigned long long)((long long)M * (long long)n)) >> 32); +#endif +} + +// Based on code from Chapter 10 of "Hacker's Delight, 2nd ed." +class fast_divmod { + public: + fast_divmod(int d = 1) : d_(d), a_(0) { find_magic_numbers(); } + + fast_divmod(const fast_divmod& other) : d_(other.d_), M_(other.M_), s_(other.s_), a_(other.a_){}; + + __host__ __device__ __inline__ int div(int n) const { + // get high 32 bits of M * n + int q = mulhi(M_, n); + + // deal with add / subs if needed + q += a_ * n; + + // shift if necessary + if (s_ >= 0) { + q >>= s_; + q += ((unsigned int)q >> 31); + } + + return q; + } + + __host__ __device__ __inline__ void divmod(int n, int& q, int& r) const { + // handle special cases + if (d_ == 1) { + q = n; + r = 0; + } else if (d_ == -1) { + q = -n; + r = 0; + } else { + // general case + q = div(n); + r = n - q * d_; + } + } + + public: + int d_, M_, s_, a_; + + private: + // Based on code from Hacker's delight 2.ed + // Chapter 10, figure 10-1 + void find_magic_numbers() { + // special case for d = 1, -1 + if (d_ == 1) { + M_ = 0; + s_ = 0; + a_ = 1; + return; + } else if (d_ == -1) { + M_ = 0; + s_ = -1; + a_ = -1; + return; + } + // general case + const unsigned two31 = 0x80000000; + unsigned abs_d = (d_ == 0) ? 1 : abs(d_); + unsigned t = two31 + ((unsigned)d_ >> 31); // t = 2^31 + (d < 0) ? 1 : 0 + unsigned abs_nc = t - 1 - (t % abs_d); // |n_c| = t - 1 - rem(t, |d|) + int p = 31; + unsigned q1 = two31 / abs_nc; // Init q_1 = 2^31 / |n_c| + unsigned r1 = two31 - q1 * abs_nc; // Init r_1 = rem(q_1, |n_c|) + unsigned q2 = two31 / abs_d; // Init q_2 = 2^31 / |d| + unsigned r2 = two31 - q2 * abs_d; // Init r_2 = rem(q_2, |d|) + + unsigned delta; + // iterate p until + // 2^p < n_c * (d - rem(2^p, d)) is satisfied + do { + ++p; + q1 *= 2; + r1 *= 2; + + if (r1 >= abs_nc) { + q1 += 1; + r1 -= abs_nc; + } + q2 *= 2; + r2 *= 2; + + if (r2 >= abs_d) { + q2 += 1; + r2 -= abs_d; + } + delta = abs_d - r2; + } while (q1 < delta || + (q1 == delta && r1 == 0)); + + // store magic numbers + M_ = q2 + 1; + if (d_ < 0) M_ = -M_; + s_ = p - 32; + + // generate sentinel for correct adds / subs + // "generate the add if d > 0 and M < 0" + if ((d_ > 0) && (M_ < 0)) a_ = 1; + // "generate the sub if d < 0 and M > 0" + else if ((d_ < 0) && (M_ > 0)) + a_ = -1; + // Otherwise no add / sub needed + else + a_ = 0; + } +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/shared_inc/fpgeneric.h b/onnxruntime/core/providers/cuda/shared_inc/fpgeneric.h new file mode 100644 index 0000000000000..bad41a1bb2ef2 --- /dev/null +++ b/onnxruntime/core/providers/cuda/shared_inc/fpgeneric.h @@ -0,0 +1,269 @@ +// +// Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +// + +// Make generic operators for floating point types +/* This file contains: + Generalized library calls + kernels to be called for not supported data type +*/ +// NV_TODO: optimize speed -- pass things needed in, optimize kernel speed, add half2 +// NV_TODO: investigate cub support for half + +#pragma once + +#include "core/providers/cuda/cuda_common.h" + +// Generalize library calls to be use in template functions + +// gemm +inline cublasStatus_t cublasGemmHelper(cublasHandle_t handle, cublasOperation_t transa, cublasOperation_t transb, int m, int n, int k, const float* alpha, const float* A, int lda, const float* B, int ldb, const float* beta, float* C, int ldc) { + return cublasSgemm(handle, transa, transb, m, n, k, alpha, A, lda, B, ldb, beta, C, ldc); +} +inline cublasStatus_t cublasGemmHelper(cublasHandle_t handle, cublasOperation_t transa, cublasOperation_t transb, int m, int n, int k, const double* alpha, const double* A, int lda, const double* B, int ldb, const double* beta, double* C, int ldc) { + return cublasDgemm(handle, transa, transb, m, n, k, alpha, A, lda, B, ldb, beta, C, ldc); +} +inline cublasStatus_t cublasGemmHelper(cublasHandle_t handle, cublasOperation_t transa, cublasOperation_t transb, int m, int n, int k, const half* alpha, const half* A, int lda, const half* B, int ldb, const half* beta, half* C, int ldc) { + // This does true FP16 computation which is slow for non-Volta GPUs + //return cublasHgemm(handle, transa, transb, m, n, k, alpha, A, lda, B, ldb, beta, C, ldc); + // This does pseudo FP16 computation (input/output in fp16, computation in fp32) + float h_a = onnxruntime::math::halfToFloat(*reinterpret_cast(alpha)); + float h_b = onnxruntime::math::halfToFloat(*reinterpret_cast(beta)); + cublasSetMathMode(handle, CUBLAS_TENSOR_OP_MATH); + return cublasGemmEx(handle, transa, transb, m, n, k, &h_a, A, CUDA_R_16F, lda, B, CUDA_R_16F, ldb, &h_b, C, CUDA_R_16F, ldc, CUDA_R_32F, CUBLAS_GEMM_DFALT); +} + +// batched gemm +inline cublasStatus_t cublasGemmBatchedHelper(cublasHandle_t handle, cublasOperation_t transa, cublasOperation_t transb, int m, int n, int k, const float* alpha, const float* Aarray[], int lda, const float* Barray[], int ldb, const float* beta, float* Carray[], int ldc, int batchCount) { + return cublasSgemmBatched(handle, transa, transb, m, n, k, alpha, Aarray, lda, Barray, ldb, beta, Carray, ldc, batchCount); +} +inline cublasStatus_t cublasGemmBatchedHelper(cublasHandle_t handle, cublasOperation_t transa, cublasOperation_t transb, int m, int n, int k, const double* alpha, const double* Aarray[], int lda, const double* Barray[], int ldb, const double* beta, double* Carray[], int ldc, int batchCount) { + return cublasDgemmBatched(handle, transa, transb, m, n, k, alpha, Aarray, lda, Barray, ldb, beta, Carray, ldc, batchCount); +} +inline cublasStatus_t cublasGemmBatchedHelper(cublasHandle_t handle, cublasOperation_t transa, cublasOperation_t transb, int m, int n, int k, const half* alpha, const half* Aarray[], int lda, const half* Barray[], int ldb, const half* beta, half* Carray[], int ldc, int batchCount) { + cublasSetMathMode(handle, CUBLAS_TENSOR_OP_MATH); + return cublasHgemmBatched(handle, transa, transb, m, n, k, alpha, (const __half**)Aarray, lda, (const __half**)Barray, ldb, beta, (__half**)Carray, ldc, batchCount); +} + +// axpy +inline cublasStatus_t cublasAxpyHelper(cublasHandle_t handle, int n, const float* alpha, const float* x, int incx, float* y, int incy) { + return cublasSaxpy(handle, n, alpha, x, incx, y, incy); +} +inline cublasStatus_t cublasAxpyHelper(cublasHandle_t handle, int n, const double* alpha, const double* x, int incx, double* y, int incy) { + return cublasDaxpy(handle, n, alpha, x, incx, y, incy); +} +inline cublasStatus_t cublasAxpyHelper(cublasHandle_t handle, int n, const half* alpha, const half* x, int incx, half* y, int incy) { + float tmp_alpha = onnxruntime::math::halfToFloat(*reinterpret_cast(alpha)); + return cublasAxpyEx(handle, n, (void*)&tmp_alpha, CUDA_R_32F, (void*)x, CUDA_R_16F, incx, (void*)y, CUDA_R_16F, incy, CUDA_R_32F); +} + +// transpose using geam +inline cublasStatus_t cublasTransposeHelper(cublasHandle_t handle, cublasOperation_t transa, cublasOperation_t transb, int m, int n, float* alpha, float* A, int lda, float* beta, float* B, int ldb, float* C, int ldc) { + return cublasSgeam(handle, transa, transb, m, n, alpha, A, lda, beta, B, ldb, C, ldc); +} +inline cublasStatus_t cublasTransposeHelper(cublasHandle_t handle, cublasOperation_t transa, cublasOperation_t transb, int m, int n, double* alpha, double* A, int lda, double* beta, double* B, int ldb, double* C, int ldc) { + return cublasDgeam(handle, transa, transb, m, n, alpha, A, lda, beta, B, ldb, C, ldc); +} +cublasStatus_t cublasTransposeHelper(cublasHandle_t, cublasOperation_t, cublasOperation_t, int m, int n, half*, half* A, int, half*, half*, int, half* C, int); + +// asum +inline cublasStatus_t cublasAsumHelper(cublasHandle_t handle, int n, const float* x, int incx, float* result) { + return cublasSasum(handle, n, x, incx, result); +} +inline cublasStatus_t cublasAsumHelper(cublasHandle_t handle, int n, const double* x, int incx, double* result) { + return cublasDasum(handle, n, x, incx, result); +} +inline cublasStatus_t cublasAsumHelper(cublasHandle_t, int n, const half* x, int incx, half* result) { + // pass in cudnn handle/descriptor to remove overhead? + cudnnHandle_t cudnnHandle; + cudnnTensorDescriptor_t srcTensorDesc, dstTensorDesc; + cudnnReduceTensorDescriptor_t reduceTensorDesc; + + cudnnCreate(&cudnnHandle); + cudnnCreateTensorDescriptor(&srcTensorDesc); + cudnnCreateTensorDescriptor(&dstTensorDesc); + cudnnCreateReduceTensorDescriptor(&reduceTensorDesc); + + cudnnSetTensor4dDescriptorEx(srcTensorDesc, CUDNN_DATA_HALF, 1, 1, 1, n, 1, 1, 1, incx); + cudnnSetTensor4dDescriptorEx(dstTensorDesc, CUDNN_DATA_HALF, 1, 1, 1, 1, 1, 1, 1, 1); + cudnnSetReduceTensorDescriptor(reduceTensorDesc, + CUDNN_REDUCE_TENSOR_NORM1, + CUDNN_DATA_FLOAT, + CUDNN_NOT_PROPAGATE_NAN, + CUDNN_REDUCE_TENSOR_NO_INDICES, + CUDNN_32BIT_INDICES); + + void* workspace = NULL; + size_t workspaceSizeInBytes = 0; + cudnnGetReductionWorkspaceSize(cudnnHandle, reduceTensorDesc, srcTensorDesc, dstTensorDesc, &workspaceSizeInBytes); + if (workspaceSizeInBytes > 0) cudaMalloc(&workspace, workspaceSizeInBytes); + + float alpha = 1.0f; + float beta = 0.0f; + + void* d_res; + cudaMalloc(&d_res, sizeof(half)); + + cudnnReduceTensor(cudnnHandle, + reduceTensorDesc, + NULL, + 0, + workspace, + workspaceSizeInBytes, + &alpha, + srcTensorDesc, + (void*)x, + &beta, + dstTensorDesc, + d_res); + + cudaMemcpy((void*)result, d_res, sizeof(half), cudaMemcpyDeviceToHost); + + cudnnDestroyReduceTensorDescriptor(reduceTensorDesc); + cudnnDestroyTensorDescriptor(srcTensorDesc); + cudnnDestroyTensorDescriptor(dstTensorDesc); + cudnnDestroy(cudnnHandle); + cudaFree(d_res); + cudaFree(workspace); + + return (cublasStatus_t)0; +} + +// amax +inline cublasStatus_t cublasAmaxHelper(cublasHandle_t handle, int n, const float* x, int incx, int* result) { + return cublasIsamax(handle, n, x, incx, result); +} +inline cublasStatus_t cublasAmaxHelper(cublasHandle_t handle, int n, const double* x, int incx, int* result) { + return cublasIdamax(handle, n, x, incx, result); +} +inline cublasStatus_t cublasAmaxHelper(cublasHandle_t, int n, const half* x, int incx, int* result) { + unsigned int h_result_uint = 0; + // pass in cudnn handle/descriptor to remove overhead? + cudnnHandle_t cudnnHandle; + cudnnTensorDescriptor_t srcTensorDesc, dstTensorDesc; + cudnnReduceTensorDescriptor_t reduceTensorDesc; + + cudnnCreate(&cudnnHandle); + cudnnCreateTensorDescriptor(&srcTensorDesc); + cudnnCreateTensorDescriptor(&dstTensorDesc); + cudnnCreateReduceTensorDescriptor(&reduceTensorDesc); + + cudnnSetTensor4dDescriptorEx(srcTensorDesc, CUDNN_DATA_HALF, 1, 1, 1, n, 1, 1, 1, incx); + cudnnSetTensor4dDescriptorEx(dstTensorDesc, CUDNN_DATA_HALF, 1, 1, 1, 1, 1, 1, 1, 1); + cudnnSetReduceTensorDescriptor(reduceTensorDesc, + CUDNN_REDUCE_TENSOR_AMAX, + CUDNN_DATA_FLOAT, + CUDNN_NOT_PROPAGATE_NAN, + CUDNN_REDUCE_TENSOR_FLATTENED_INDICES, + CUDNN_32BIT_INDICES); + + void* workspace = NULL; + size_t workspaceSizeInBytes = 0; + cudnnGetReductionWorkspaceSize(cudnnHandle, reduceTensorDesc, srcTensorDesc, dstTensorDesc, &workspaceSizeInBytes); + if (workspaceSizeInBytes > 0) cudaMalloc(&workspace, workspaceSizeInBytes); + + float alpha = 1.0f; + float beta = 0.0f; + void* d_max; + cudaMalloc(&d_max, sizeof(half)); + void* d_result_uint; + cudaMalloc(&d_result_uint, sizeof(unsigned int)); + + cudnnReduceTensor(cudnnHandle, + reduceTensorDesc, + d_result_uint, + sizeof(unsigned int), + workspace, + workspaceSizeInBytes, + &alpha, + srcTensorDesc, + (void*)x, + &beta, + dstTensorDesc, + d_max); + + cudaMemcpy(&h_result_uint, d_result_uint, sizeof(unsigned int), cudaMemcpyDeviceToHost); + + cudnnDestroyReduceTensorDescriptor(reduceTensorDesc); + cudnnDestroyTensorDescriptor(srcTensorDesc); + cudnnDestroyTensorDescriptor(dstTensorDesc); + cudnnDestroy(cudnnHandle); + cudaFree(workspace); + cudaFree(d_max); + cudaFree(d_result_uint); + + *result = (int)h_result_uint; + return (cublasStatus_t)0; +} + +// scal +inline cublasStatus_t cublasScalHelper(cublasHandle_t handle, int n, const float* alpha, float* x, int incx) { + return cublasSscal(handle, n, alpha, x, incx); +} +inline cublasStatus_t cublasScalHelper(cublasHandle_t handle, int n, const double* alpha, double* x, int incx) { + return cublasDscal(handle, n, alpha, x, incx); +} +inline cublasStatus_t cublasScalHelper(cublasHandle_t handle, int n, const half* alpha, half* x, int incx) { + float tmp_alpha = onnxruntime::math::halfToFloat(*reinterpret_cast(alpha)); + return cublasScalEx(handle, n, (void*)&tmp_alpha, CUDA_R_32F, (void*)x, CUDA_R_16F, incx, CUDA_R_32F); +} +inline cublasStatus_t cublasScalHelper(cublasHandle_t, int, const char*, char*, int) { + ONNXRUNTIME_NOT_IMPLEMENTED("Unsupported template argument(char) in cublas_scal"); +} +inline cublasStatus_t cublasScalHelper(cublasHandle_t, int, const short*, short*, int) { + ONNXRUNTIME_NOT_IMPLEMENTED("Unsupported template argument(short) in cublas_scal"); +} + +// dot +inline cublasStatus_t cublasDotHelper(cublasHandle_t handle, int n, const float* x, int incx, const float* y, int incy, float* result) { + return cublasSdot(handle, n, x, incx, y, incy, result); +} +inline cublasStatus_t cublasDotHelper(cublasHandle_t handle, int n, const double* x, int incx, const double* y, int incy, double* result) { + return cublasDdot(handle, n, x, incx, y, incy, result); +} +inline cublasStatus_t cublasDotHelper(cublasHandle_t handle, int n, const half* x, int incx, const half* y, int incy, half* result) { + return cublasDotEx(handle, n, (void*)x, CUDA_R_16F, incx, (void*)y, CUDA_R_16F, incy, (void*)result, CUDA_R_16F, CUDA_R_32F); +} + +// copy +inline cublasStatus_t cublasCopyHelper(cublasHandle_t handle, int n, const float* x, int incx, float* y, int incy) { + return cublasScopy(handle, n, x, incx, y, incy); +} +inline cublasStatus_t cublasCopyHelper(cublasHandle_t handle, int n, const double* x, int incx, double* y, int incy) { + return cublasDcopy(handle, n, x, incx, y, incy); +} +cublasStatus_t cublasCopyHelper(cublasHandle_t handle, int n, const half* x, int incx, half* y, int incy); + +// curand +inline curandStatus_t curandGenerateUniformHelper(curandGenerator_t generator, float* outputPtr, size_t num) { + return curandGenerateUniform(generator, outputPtr, num); +} +inline curandStatus_t curandGenerateUniformHelper(curandGenerator_t generator, double* outputPtr, size_t num) { + return curandGenerateUniformDouble(generator, outputPtr, num); +} +curandStatus_t curandGenerateUniformHelper(curandGenerator_t, half* outputPtr, size_t num); + +inline curandStatus_t curandGenerateUniformHelper(curandGenerator_t, char*, size_t) { + ONNXRUNTIME_NOT_IMPLEMENTED("Unsupported template argument(char) in GPUSparseMatrix"); +} + +inline curandStatus_t curandGenerateUniformHelper(curandGenerator_t, short*, size_t) { + ONNXRUNTIME_NOT_IMPLEMENTED("Unsupported template argument(short) in GPUSparseMatrix"); +} + +inline curandStatus_t curandGenerateNormalHelper(curandGenerator_t generator, float* outputPtr, size_t n, float mean, float stddev) { + return curandGenerateNormal(generator, outputPtr, n, mean, stddev); +} +inline curandStatus_t curandGenerateNormalHelper(curandGenerator_t generator, double* outputPtr, size_t n, double mean, double stddev) { + return curandGenerateNormalDouble(generator, outputPtr, n, mean, stddev); +} +curandStatus_t curandGenerateNormalHelper(curandGenerator_t, half* outputPtr, size_t n, half mean, half stddev); + +inline curandStatus_t curandGenerateNormalHelper(curandGenerator_t, char*, size_t, char, char) { + ONNXRUNTIME_NOT_IMPLEMENTED("Unsupported template argument(char) in GPUSparseMatrix"); +} + +inline curandStatus_t curandGenerateNormalHelper(curandGenerator_t, short*, size_t, short, short) { + ONNXRUNTIME_NOT_IMPLEMENTED("Unsupported template argument(short) in GPUSparseMatrix"); +} diff --git a/onnxruntime/core/providers/cuda/symbols.txt b/onnxruntime/core/providers/cuda/symbols.txt new file mode 100644 index 0000000000000..ed660e42fe162 --- /dev/null +++ b/onnxruntime/core/providers/cuda/symbols.txt @@ -0,0 +1 @@ +ONNXRuntimeCreateCUDAExecutionProviderFactory diff --git a/onnxruntime/core/providers/cuda/tensor/cast_op.cc b/onnxruntime/core/providers/cuda/tensor/cast_op.cc new file mode 100644 index 0000000000000..a0db09da86bfa --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/cast_op.cc @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "cast_op.h" +#include "core/providers/cuda/math/unary_elementwise_ops_impl.h" + +using namespace ONNX_NAMESPACE; +using namespace onnxruntime::common; +namespace onnxruntime { +namespace cuda { + +const std::vector castOpTypeConstraints{ + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}; + +#define REGISTER_KERNEL_TYPED(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + Cast, \ + kOnnxDomain, \ + 6, \ + T, \ + kCudaExecutionProvider, \ + KernelDefBuilder() \ + .TypeConstraint("T1", DataTypeImpl::GetTensorType()) \ + .TypeConstraint("T2", castOpTypeConstraints), \ + Cast); + +template +Status Cast::ComputeInternal(OpKernelContext* context) const { + typedef typename ToCudaType::MappedType CudaSrcT; + const Tensor* X = context->Input(0); + const TensorShape& shape = X->Shape(); + Tensor* Y = context->Output(0, TensorShape(shape)); + const auto* x_data = reinterpret_cast(X->template Data()); + size_t count = shape.Size(); + +#define CASE(TP_TYPE, DstT) \ + case TP_TYPE: \ + Impl_Cast::MappedType>( \ + x_data, \ + reinterpret_cast::MappedType*>(Y->template MutableData()), \ + count); \ + break; + + switch (to_) { + CASE(TensorProto_DataType_FLOAT16, MLFloat16) + CASE(TensorProto_DataType_FLOAT, float) + CASE(TensorProto_DataType_DOUBLE, double) + CASE(TensorProto_DataType_INT8, int8_t) + CASE(TensorProto_DataType_INT16, int16_t) + CASE(TensorProto_DataType_INT32, int32_t) + CASE(TensorProto_DataType_INT64, int64_t) + CASE(TensorProto_DataType_UINT8, uint8_t) + CASE(TensorProto_DataType_UINT16, uint16_t) + CASE(TensorProto_DataType_UINT32, uint32_t) + CASE(TensorProto_DataType_UINT64, uint64_t) + CASE(TensorProto_DataType_BOOL, bool) + case TensorProto_DataType_STRING: + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Casting to and from strings is not supported yet."); + case TensorProto_DataType_UNDEFINED: + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Cast op must have 'to' argument of type DataType"); + default: + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unexpected 'to' argument value: ", to_); + } + return Status::OK(); +} + +#define SPECIALIZE_IMPL(T) \ + REGISTER_KERNEL_TYPED(T) \ + template Status Cast::ComputeInternal(OpKernelContext* context) const; + +SPECIALIZE_IMPL(MLFloat16) +SPECIALIZE_IMPL(float) +SPECIALIZE_IMPL(double) +SPECIALIZE_IMPL(int8_t) +SPECIALIZE_IMPL(int16_t) +SPECIALIZE_IMPL(int32_t) +SPECIALIZE_IMPL(int64_t) +SPECIALIZE_IMPL(uint8_t) +SPECIALIZE_IMPL(uint16_t) +SPECIALIZE_IMPL(uint32_t) +SPECIALIZE_IMPL(uint64_t) +SPECIALIZE_IMPL(bool) + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/cast_op.h b/onnxruntime/core/providers/cuda/tensor/cast_op.h new file mode 100644 index 0000000000000..c88f2afee0c89 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/cast_op.h @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/providers/cuda/cuda_common.h" + +namespace onnxruntime { +namespace cuda { + +template +class Cast final : public CudaKernel { + public: + Cast(const OpKernelInfo& info) : CudaKernel(info) { + int64_t to; + Status status = info.GetAttr("to", &to); + ONNXRUNTIME_ENFORCE(status.IsOK(), "Attribute to is not set."); + to_ = gsl::narrow_cast(to); + } + + Status ComputeInternal(OpKernelContext* context) const override; + + private: + ONNX_NAMESPACE::TensorProto_DataType to_; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/compress.cc b/onnxruntime/core/providers/cuda/tensor/compress.cc new file mode 100644 index 0000000000000..354a85483ad79 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/compress.cc @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "compress.h" +#include "core/providers/cpu/tensor/utils.h" +#include "compress_impl.h" + +namespace onnxruntime { +namespace cuda { + +ONNX_OPERATOR_KERNEL_EX( + Compress, + kOnnxDomain, + 9, + kCudaExecutionProvider, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()), + Compress); + +Status Compress::ComputeInternal(OpKernelContext* ctx) const { + const Tensor* input_tensor = ctx->Input(0); + ONNXRUNTIME_ENFORCE(input_tensor); + size_t rank = input_tensor->Shape().NumDimensions(); + auto& input_dimensions = input_tensor->Shape().GetDims(); + if (has_axis_) { + ONNXRUNTIME_ENFORCE(axis_ < static_cast(rank), "axis greater than input data dimension!"); + } + + const Tensor* condition = ctx->Input(1); + ONNXRUNTIME_ENFORCE(condition); + auto condition_length = condition->Shape().Size(); + auto condition_data = condition->template Data(); + + // if has axis, we need to compress on dimension[axis], otherwise compress on the flattened input data + int64_t input_size = input_tensor->Shape().Size(); + int64_t compress_input_length = has_axis_ ? input_dimensions[axis_] : input_size; + int64_t valid_condition_length = compress_input_length < condition_length ? compress_input_length : condition_length; + + auto condition_cumulative_sum = GetScratchBuffer(valid_condition_length).get(); + PrefixSumImpl(reinterpret_cast(condition_data), condition_cumulative_sum, valid_condition_length); + + int32_t positive_condition_count = 0; + CUDA_RETURN_IF_ERROR(cudaMemcpy(&positive_condition_count, condition_cumulative_sum + valid_condition_length - 1, sizeof(int32_t), cudaMemcpyDeviceToHost)); + + std::vector output_dims(input_dimensions); + if (has_axis_) { + output_dims[axis_] = positive_condition_count; + } else { + output_dims.resize(1); + output_dims[0] = positive_condition_count; + } + + TensorShape output_shape(output_dims); + auto output_tensor = ctx->Output(0, output_shape); + if (positive_condition_count <= 0) { + return Status::OK(); + } + + auto element_bytes = input_tensor->DataType()->Size(); + + int64_t axis_right_stride = 1; + if (has_axis_) { + for (int i = static_cast(axis_ + 1); i < rank; ++i) { + axis_right_stride *= input_dimensions[i]; + } + } + + ONNXRUNTIME_RETURN_IF_ERROR(CompressImpl(element_bytes, + gsl::narrow_cast(valid_condition_length), + gsl::narrow_cast(axis_right_stride), + has_axis_ ? gsl::narrow_cast(input_dimensions[axis_]) : gsl::narrow_cast(input_size), + gsl::narrow_cast(positive_condition_count), + condition_cumulative_sum, + condition_data, + input_tensor->DataRaw(), + output_tensor->MutableDataRaw(), + input_size)); + + return Status::OK(); +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/compress.h b/onnxruntime/core/providers/cuda/tensor/compress.h new file mode 100644 index 0000000000000..c5befc61be8a7 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/compress.h @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/providers/cuda/cuda_common.h" + +namespace onnxruntime { +namespace cuda { + +class Compress final : public CudaKernel { + public: + Compress(const OpKernelInfo& info) : CudaKernel(info) { + has_axis_ = info.GetAttr("axis", &axis_).IsOK(); + } + + Status ComputeInternal(OpKernelContext* context) const override; + + private: + int64_t axis_; + bool has_axis_; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/compress_impl.cu b/onnxruntime/core/providers/cuda/tensor/compress_impl.cu new file mode 100644 index 0000000000000..6e889a9d81655 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/compress_impl.cu @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cuda/cu_inc/common.cuh" +#include "core/providers/cuda/cuda_common.h" +#include "compress_impl.h" + +#include +#include + +namespace onnxruntime { +namespace cuda { + +void PrefixSumImpl(const int8_t* condition_data, + int32_t* condition_cumulative_sum, + const size_t length) { + thrust::inclusive_scan(thrust::device, condition_data, condition_data + length, condition_cumulative_sum); +} + +template +__global__ void _CompressKernel(const int32_t valid_condition_length, + const fast_divmod axis_right_stride_div, + const fast_divmod input_axis_included_stride_div, + const int32_t output_axis_included_stride, + const int32_t* condition_cumulative_sum, + const bool* condition_data, + const T* input_data, + T* output_data, + const CUDA_LONG N) { + CALCULATE_ELEMENTWISE_INDEX_OR_EXIT(id, N); + CUDA_LONG output_index = 0; + + int div, mod; + input_axis_included_stride_div.divmod(id, div, mod); + output_index = output_axis_included_stride * div; + axis_right_stride_div.divmod(mod, div, mod); + + if (div < valid_condition_length && condition_data[div]) { + output_index += (condition_cumulative_sum[div] - 1) * axis_right_stride_div.d_ + mod; + output_data[output_index] = input_data[id]; + } +} + +Status CompressImpl(const size_t element_bytes, + const int32_t valid_condition_length, + const int32_t axis_right_stride, + const int32_t input_axis_dim_length, + const int32_t output_axis_dim_length, + const int32_t* condition_cumulative_sum, + const bool* condition_data, + const void* input_data, + void* output_data, + const size_t N) { + int blocksPerGrid = (int)(ceil(static_cast(N) / GridDim::maxThreadsPerBlock)); + + fast_divmod axis_right_stride_div(axis_right_stride); + fast_divmod input_axis_included_stride_div(axis_right_stride * input_axis_dim_length); + int output_axis_included_stride = axis_right_stride * output_axis_dim_length; + + switch (element_bytes) { + case sizeof(int8_t): + _CompressKernel<<>>( + valid_condition_length, + axis_right_stride_div, + input_axis_included_stride_div, + output_axis_included_stride, + condition_cumulative_sum, + condition_data, + reinterpret_cast::MappedType*>(input_data), + reinterpret_cast::MappedType*>(output_data), + (CUDA_LONG)N); + break; + case sizeof(int16_t): + _CompressKernel<<>>( + valid_condition_length, + axis_right_stride_div, + input_axis_included_stride_div, + output_axis_included_stride, + condition_cumulative_sum, + condition_data, + reinterpret_cast::MappedType*>(input_data), + reinterpret_cast::MappedType*>(output_data), + (CUDA_LONG)N); + break; + case sizeof(int32_t): + _CompressKernel<<>>( + valid_condition_length, + axis_right_stride_div, + input_axis_included_stride_div, + output_axis_included_stride, + condition_cumulative_sum, + condition_data, + reinterpret_cast::MappedType*>(input_data), + reinterpret_cast::MappedType*>(output_data), + (CUDA_LONG)N); + break; + case sizeof(int64_t): + _CompressKernel<<>>( + valid_condition_length, + axis_right_stride_div, + input_axis_included_stride_div, + output_axis_included_stride, + condition_cumulative_sum, + condition_data, + reinterpret_cast::MappedType*>(input_data), + reinterpret_cast::MappedType*>(output_data), + (CUDA_LONG)N); + break; + default: + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Type not supported for Compress operator"); + } + + return Status::OK(); +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/compress_impl.h b/onnxruntime/core/providers/cuda/tensor/compress_impl.h new file mode 100644 index 0000000000000..08005944cc5d4 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/compress_impl.h @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include "core/providers/cuda/shared_inc/cuda_utils.h" +#include "core/common/common.h" + +namespace onnxruntime { +namespace cuda { + +void PrefixSumImpl(const int8_t* condition_data, + int32_t* condition_cumulative_sum, + const size_t length); + +Status CompressImpl(const size_t element_bytes, + const int32_t valid_condition_length, + const int32_t axis_right_stride, + const int32_t input_axis_dim_length, + const int32_t output_axis_dim_length, + const int32_t* condition_cumulative_sum, + const bool* condition_data, + const void* input_data, + void* output_data, + const size_t N); + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/concat.cc b/onnxruntime/core/providers/cuda/tensor/concat.cc new file mode 100644 index 0000000000000..a6a7100ec967c --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/concat.cc @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "concat.h" + +namespace onnxruntime { +namespace cuda { +ONNX_OPERATOR_KERNEL_EX( + Concat, + kOnnxDomain, + 4, + kCudaExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()), + Concat); + +Status Concat::ComputeInternal(OpKernelContext* ctx) const { + auto input_count = Node().InputArgCount().front(); + + Prepare p; + ONNXRUNTIME_RETURN_IF_ERROR(PrepareForCompute(ctx, input_count, p)); + + int64_t output_offset = 0; + auto element_bytes = p.output_tensor->DataType()->Size(); + for (int input_index = 0; input_index < input_count; input_index++) { + const auto& prep = p.inputs[input_index]; + + // Copy the data across. For every 'input_axis_pitch' values copied, we move over by the 'output_axis_pitch' + CUDA_RETURN_IF_ERROR(cudaMemcpy2DAsync( + static_cast(p.output_tensor->MutableDataRaw()) + output_offset * element_bytes, + p.output_axis_pitch * element_bytes, + prep.tensor->DataRaw(), + prep.axis_pitch * element_bytes, + prep.axis_pitch * element_bytes, + prep.tensor->Shape().Size() / prep.axis_pitch, + cudaMemcpyDeviceToDevice)); + + output_offset += prep.axis_pitch; + } + return Status::OK(); +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/concat.h b/onnxruntime/core/providers/cuda/tensor/concat.h new file mode 100644 index 0000000000000..7c542709bcb4e --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/concat.h @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cuda/cuda_common.h" +#include "core/providers/cpu/tensor/concat.h" + +namespace onnxruntime { +namespace cuda { + +class Concat final : public CudaKernel, public ConcatBase { + public: + Concat(const OpKernelInfo& info) : ConcatBase(info), CudaKernel(info) {} + Status ComputeInternal(OpKernelContext* context) const override; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/crop.cc b/onnxruntime/core/providers/cuda/tensor/crop.cc new file mode 100644 index 0000000000000..fa1d2bf2b9556 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/crop.cc @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "crop.h" +#include "crop_impl.h" + +namespace onnxruntime { +namespace cuda { + +#define REGISTER_KERNEL_TYPED(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + Crop, \ + kOnnxDomain, \ + 1, \ + T, \ + kCudaExecutionProvider, \ + KernelDefBuilder() \ + .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + Crop); + +REGISTER_KERNEL_TYPED(float) +REGISTER_KERNEL_TYPED(double) +REGISTER_KERNEL_TYPED(MLFloat16) + +template +Status Crop::ComputeInternal(OpKernelContext* context) const { + const Tensor* X = context->Input(0); + ONNXRUNTIME_RETURN_IF_ERROR(ValidateInput(X)); + + const auto dims = X->Shape().GetDims(); + const int64_t N = dims[0]; + const int64_t C = dims[1]; + const int64_t H = dims[2]; + const int64_t W = dims[3]; + + // find the cropped region, and copy it to the destination matrix + int64_t leftBorder = border_[0]; + int64_t topBorder = border_[1]; + int64_t rightBorder = border_[2]; + int64_t bottomBorder = border_[3]; + + int64_t bottomLimit = H - bottomBorder; + int64_t rightLimit = W - rightBorder; + + // scale = (height, width) + if (!scale_.empty()) { + bottomLimit = topBorder + scale_[0]; + rightLimit = leftBorder + scale_[1]; + } + + Tensor* Y = context->Output(0, TensorShape({N, C, bottomLimit - topBorder, rightLimit - leftBorder})); + + typedef typename ToCudaType::MappedType CudaT; + fast_divmod fdm_YW(gsl::narrow_cast(rightLimit - leftBorder)); + fast_divmod fdm_YHW(gsl::narrow_cast((bottomLimit - topBorder) * (rightLimit - leftBorder))); + + CropImpl( + reinterpret_cast(X->template Data()), + gsl::narrow_cast(leftBorder), + gsl::narrow_cast(topBorder), + gsl::narrow_cast(W), + gsl::narrow_cast(W * H), + fdm_YW, + fdm_YHW, + reinterpret_cast(Y->template MutableData()), + Y->Shape().Size()); + + return Status::OK(); +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/crop.h b/onnxruntime/core/providers/cuda/tensor/crop.h new file mode 100644 index 0000000000000..50ef2af0fe1e7 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/crop.h @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/providers/cuda/cuda_common.h" +#include "core/providers/cpu/tensor/crop.h" + +namespace onnxruntime { +namespace cuda { + +template +class Crop final : public CropBase, public CudaKernel { + public: + Crop(const OpKernelInfo& info) : CropBase(info), CudaKernel(info) { + } + + Status ComputeInternal(OpKernelContext* context) const override; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/crop_impl.cu b/onnxruntime/core/providers/cuda/tensor/crop_impl.cu new file mode 100644 index 0000000000000..cf34767ff8178 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/crop_impl.cu @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cuda/cu_inc/common.cuh" +#include "image_scaler_impl.h" + +namespace onnxruntime { +namespace cuda { + +template +__global__ void _CropKernel( + const T* input_data, + const int src_start_x, + const int src_start_y, + const int src_w, + const int src_hw, + const fast_divmod fdm_dst_w, + const fast_divmod fdm_dst_hw, + T* output_data, + const CUDA_LONG N) { + CALCULATE_ELEMENTWISE_INDEX_OR_EXIT(id, N); + int dst_xy, dst_nc; + fdm_dst_hw.divmod(id, dst_nc, dst_xy); + int dst_x, dst_y; + fdm_dst_w.divmod(dst_xy, dst_y, dst_x); + output_data[id] = input_data[dst_nc * src_hw + (dst_y + src_start_y) * src_w + (dst_x + src_start_x)]; +} + +template +void CropImpl( + const T* input_data, + const int src_start_x, + const int src_start_y, + const int src_w, + const int src_hw, + const fast_divmod& fdm_dst_w, + const fast_divmod& fdm_dst_hw, + T* output_data, + const size_t N) { + int blocksPerGrid = (int)(ceil(static_cast(N) / GridDim::maxThreadsPerBlock)); + _CropKernel<<>>( + input_data, src_start_x, src_start_y, src_w, src_hw, fdm_dst_w, fdm_dst_hw, output_data, (CUDA_LONG)N); +} + +#define SPECIALIZED_IMPL(T) \ + template void CropImpl(const T* input_data, const int src_start_x, const int src_start_y, const int src_w, const int src_hw, const fast_divmod& fdm_dst_w, const fast_divmod& fdm_dst_hw, T* output_data, const size_t N); + +SPECIALIZED_IMPL(float) +SPECIALIZED_IMPL(double) +SPECIALIZED_IMPL(half) + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/crop_impl.h b/onnxruntime/core/providers/cuda/tensor/crop_impl.h new file mode 100644 index 0000000000000..f2343fa0161a1 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/crop_impl.h @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/providers/cuda/shared_inc/cuda_utils.h" + +namespace onnxruntime { +namespace cuda { + +template +void CropImpl( + const T* input_data, + const int src_start_x, + const int src_start_y, + const int src_w, + const int src_hw, + const fast_divmod& fdm_dst_w, + const fast_divmod& fdm_dst_hw, + T* output_data, + const size_t N); + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/flatten.cc b/onnxruntime/core/providers/cuda/tensor/flatten.cc new file mode 100644 index 0000000000000..f5b959bbdb7de --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/flatten.cc @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "flatten.h" + +namespace onnxruntime { +namespace cuda { + +ONNX_OPERATOR_KERNEL_EX( + Flatten, + kOnnxDomain, + 1, + kCudaExecutionProvider, + KernelDefBuilder() + .Alias(0, 0) + .TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()), + Flatten); + +Status Flatten::ComputeInternal(OpKernelContext* ctx) const { + const Tensor* X = ctx->Input(0); + const TensorShape& X_shape = X->Shape(); + ONNXRUNTIME_ENFORCE(gsl::narrow_cast(X_shape.NumDimensions()) >= axis_, "The rank of input tensor must be >= axis"); + + Tensor* Y = ctx->Output(0, TensorShape({X_shape.SizeToDimension(axis_), X_shape.SizeFromDimension(axis_)})); + //If source and target pointers are not equal (non-inplace operation), we need to copy the data. + const void* source = X->DataRaw(); + void* target = Y->MutableDataRaw(); + if (target != source) { + CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(target, source, X_shape.Size() * X->DataType()->Size(), cudaMemcpyDeviceToDevice)); + } + + return Status::OK(); +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/flatten.h b/onnxruntime/core/providers/cuda/tensor/flatten.h new file mode 100644 index 0000000000000..9e3066ddd1a89 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/flatten.h @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cuda/cuda_common.h" + +namespace onnxruntime { +namespace cuda { + +class Flatten final : public CudaKernel { + public: + Flatten(const OpKernelInfo& info) : CudaKernel(info) { + ONNXRUNTIME_ENFORCE(info.GetAttr("axis", &axis_).IsOK()); + } + + Status ComputeInternal(OpKernelContext* context) const override; + + private: + int64_t axis_; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/gather.cc b/onnxruntime/core/providers/cuda/tensor/gather.cc new file mode 100644 index 0000000000000..4f6ae9348ea2d --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/gather.cc @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gather.h" +#include "gather_impl.h" +#include "core/providers/cpu/tensor/utils.h" +#include "core/providers/common.h" + +namespace onnxruntime { +namespace cuda { +ONNX_OPERATOR_KERNEL_EX( + Gather, + kOnnxDomain, + 1, + kCudaExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()) + .TypeConstraint("Tind", std::vector{ + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + Gather); + +#define TYPED_FUNCTION_CALL(T) \ + if (T_type == DataTypeImpl::GetType()) { \ + T* output_data = p.output_tensor->template MutableData(); \ + const T* input_data = p.input_tensor->template Data(); \ + if (Tin_type == DataTypeImpl::GetType()) { \ + GatherImpl( \ + input_block_size, \ + indices_max, \ + p.indices_tensor->template Data(), \ + div_strides.GpuPtr(), \ + reinterpret_cast::MappedType*>(input_data), \ + reinterpret_cast::MappedType*>(output_data), \ + p.output_tensor->Shape().Size()); \ + return Status::OK(); \ + } \ + if (Tin_type == DataTypeImpl::GetType()) { \ + GatherImpl( \ + input_block_size, \ + indices_max, \ + p.indices_tensor->template Data(), \ + div_strides.GpuPtr(), \ + reinterpret_cast::MappedType*>(input_data), \ + reinterpret_cast::MappedType*>(output_data), \ + p.output_tensor->Shape().Size()); \ + return Status::OK(); \ + } \ + } + +Status Gather::ComputeInternal(OpKernelContext* context) const { + Prepare p; + ONNXRUNTIME_RETURN_IF_ERROR(PrepareForCompute(context, p)); + + const TensorShape& input_shape = p.input_tensor->Shape(); + + const int64_t block_size = input_shape.SizeFromDimension(p.axis + 1); + size_t N = p.indices_tensor->Shape().Size(); + const int64_t input_block_size = input_shape.SizeFromDimension(p.axis); + const int64_t output_block_size = N * block_size; + const int64_t indices_max = input_shape[p.axis]; + + // Put the output_block_size and block_size into div_strides + // for divmod calling in _GatherKernel to calculate the input index + CudaAsyncBuffer div_strides(this, 0, 2); + gsl::span div_strides_span = div_strides.CpuSpan(); + div_strides_span[0] = fast_divmod(gsl::narrow_cast(output_block_size)); + div_strides_span[1] = fast_divmod(gsl::narrow_cast(block_size)); + ONNXRUNTIME_RETURN_IF_ERROR(div_strides.CopyToGpu()); + + MLDataType T_type = p.input_tensor->DataType(); + MLDataType Tin_type = p.indices_tensor->DataType(); + + TYPED_FUNCTION_CALL(int8_t) + TYPED_FUNCTION_CALL(int16_t) + TYPED_FUNCTION_CALL(int32_t) + TYPED_FUNCTION_CALL(int64_t) + TYPED_FUNCTION_CALL(uint8_t) + TYPED_FUNCTION_CALL(uint16_t) + TYPED_FUNCTION_CALL(uint32_t) + TYPED_FUNCTION_CALL(uint64_t) + TYPED_FUNCTION_CALL(MLFloat16) + TYPED_FUNCTION_CALL(float) + TYPED_FUNCTION_CALL(double) + TYPED_FUNCTION_CALL(bool) + + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, NOT_IMPLEMENTED, "Type for Tind not supported yet in Gather."); +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/gather.h b/onnxruntime/core/providers/cuda/tensor/gather.h new file mode 100644 index 0000000000000..bc7e2508f2b81 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/gather.h @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cuda/cuda_common.h" +#include "core/providers/cpu/tensor/gather.h" + +namespace onnxruntime { +namespace cuda { + +class Gather final : public CudaKernel, public GatherBase { + public: + Gather(const OpKernelInfo& info) : GatherBase(info), CudaKernel(info) {} + Status ComputeInternal(OpKernelContext* context) const override; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/gather_impl.cu b/onnxruntime/core/providers/cuda/tensor/gather_impl.cu new file mode 100644 index 0000000000000..bcc0ce73afedd --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/gather_impl.cu @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cuda/cu_inc/common.cuh" +#include "gather_impl.h" + +namespace onnxruntime { +namespace cuda { + +template +__global__ void _GatherKernel( + const int64_t input_block_size, + const int64_t indices_max, + const Tin* indices_data, + const fast_divmod* div_strides, + const T* input_data, + T* output_data, + const CUDA_LONG N) { + CALCULATE_ELEMENTWISE_INDEX_OR_EXIT(id, N); + CUDA_LONG input_index = 0; + int input_block_index, block_offset; + div_strides[0].divmod(id, input_block_index, block_offset); + int indices_index, offset; + div_strides[1].divmod(block_offset, indices_index, offset); + int block_size = div_strides[1].d_; + int64_t idx = indices_data[indices_index]; + if (idx < 0 || idx >= indices_max) { + output_data[id] = 0; + return; + } + + input_index = input_block_index * input_block_size + idx * block_size + offset; + output_data[id] = input_data[input_index]; +} + +template +void GatherImpl( + const int64_t input_block_size, + const int64_t indices_max, + const Tin* indices_data, + const fast_divmod* div_strides, + const T* input_data, + T* output_data, + const size_t N) { + int blocksPerGrid = (int)(ceil(static_cast(N) / GridDim::maxThreadsPerBlock)); + _GatherKernel<<>>( + input_block_size, indices_max, indices_data, div_strides, input_data, output_data, (CUDA_LONG)N); +} + +#define SPECIALIZED_IMPL(T) \ + template void GatherImpl(const int64_t input_block_size, const int64_t indices_max, const int32_t* indices_data, const fast_divmod* div_strides, const T* input_data, T* output_data, const size_t N); \ + template void GatherImpl(const int64_t input_block_size, const int64_t indices_max, const int64_t* indices_data, const fast_divmod* div_strides, const T* input_data, T* output_data, const size_t N); + +SPECIALIZED_IMPL(int8_t) +SPECIALIZED_IMPL(int16_t) +SPECIALIZED_IMPL(int32_t) +SPECIALIZED_IMPL(int64_t) +SPECIALIZED_IMPL(uint8_t) +SPECIALIZED_IMPL(uint16_t) +SPECIALIZED_IMPL(uint32_t) +SPECIALIZED_IMPL(uint64_t) +SPECIALIZED_IMPL(half) +SPECIALIZED_IMPL(float) +SPECIALIZED_IMPL(double) +SPECIALIZED_IMPL(bool) + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/gather_impl.h b/onnxruntime/core/providers/cuda/tensor/gather_impl.h new file mode 100644 index 0000000000000..6f3254564ee06 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/gather_impl.h @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include "core/providers/cuda/shared_inc/cuda_utils.h" + +namespace onnxruntime { +namespace cuda { + +template +void GatherImpl( + const int64_t input_block_size, + const int64_t indices_max, + const Tin* indices_data, + const fast_divmod* output_strides, + const T* input_data, + T* output_data, + const size_t N); + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/identity_op.cc b/onnxruntime/core/providers/cuda/tensor/identity_op.cc new file mode 100644 index 0000000000000..e9a4125c43188 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/identity_op.cc @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "identity_op.h" + +namespace onnxruntime { +namespace cuda { +ONNX_OPERATOR_KERNEL_EX( + Dropout, + kOnnxDomain, + 7, + kCudaExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T", {DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}) + .Alias(0, 0), + IdentityOp); + +ONNX_OPERATOR_KERNEL_EX( + Identity, + kOnnxDomain, + 1, + kCudaExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()) + .Alias(0, 0), + IdentityOp); +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/identity_op.h b/onnxruntime/core/providers/cuda/tensor/identity_op.h new file mode 100644 index 0000000000000..f66a11faf72dc --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/identity_op.h @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cuda/cuda_common.h" + +namespace onnxruntime { +namespace cuda { + +template +class IdentityOp final : public CudaKernel { + public: + IdentityOp(const OpKernelInfo& info) : CudaKernel(info) { + } + + Status ComputeInternal(OpKernelContext* context) const override { + const Tensor* X = context->Input(0); + const TensorShape& shape = X->Shape(); + Tensor* Y = context->Output(0, shape); + auto X_type = X->DataType(); + + const void* source = X->DataRaw(X_type); + void* target = Y->MutableDataRaw(X_type); + //If source and target pointers are not equal, we need to copy the data. + if (target != source) { + CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(target, source, X->Shape().Size() * X->DataType()->Size(), cudaMemcpyDeviceToDevice)); + } + + if (is_dropout) { + context->Output(1, std::vector()); + } + + return Status::OK(); + } +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/image_scaler.cc b/onnxruntime/core/providers/cuda/tensor/image_scaler.cc new file mode 100644 index 0000000000000..be672b134cce9 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/image_scaler.cc @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "image_scaler.h" +#include "image_scaler_impl.h" + +namespace onnxruntime { +namespace cuda { + +#define REGISTER_KERNEL_TYPED(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + ImageScaler, \ + kOnnxDomain, \ + 1, \ + T, \ + kCudaExecutionProvider, \ + KernelDefBuilder() \ + .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + ImageScaler); + +REGISTER_KERNEL_TYPED(float) +REGISTER_KERNEL_TYPED(double) +REGISTER_KERNEL_TYPED(MLFloat16) + +template +ImageScaler::ImageScaler(const OpKernelInfo& info) : CudaKernel(info) { + ONNXRUNTIME_ENFORCE(info.GetAttr("scale", &scale_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttrs("bias", bias_).IsOK()); + + b_data_ = GetScratchBuffer(bias_.size()); + CUDA_CALL_THROW(cudaMemcpy(b_data_.get(), bias_.data(), sizeof(float) * bias_.size(), cudaMemcpyHostToDevice)); +} + +template +Status ImageScaler::ComputeInternal(OpKernelContext* context) const { + const Tensor* X = context->Input(0); + const auto dims = X->Shape().GetDims(); + + if (dims.size() != 4) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Input is expected to have four dimensions corresponding to [N,C,H,W], got ", dims.size()); + } + + const int64_t C = dims[1]; // dims are NCHW + + if (!bias_.empty() && bias_.size() != static_cast(C)) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Bias size (", bias_.size(), ") does not match the number of channels (", C, ")"); + } + + Tensor* Y = context->Output(0, X->Shape()); + + typedef typename ToCudaType::MappedType CudaT; + ImageScalerImpl( + reinterpret_cast(X->template Data()), + scale_, + b_data_.get(), + dims.data(), + reinterpret_cast(Y->template MutableData()), + X->Shape().Size()); + + return Status::OK(); +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/image_scaler.h b/onnxruntime/core/providers/cuda/tensor/image_scaler.h new file mode 100644 index 0000000000000..9e4010408d195 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/image_scaler.h @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cuda/cudnn_common.h" + +namespace onnxruntime { +namespace cuda { + +template +class ImageScaler final : public CudaKernel { + public: + ImageScaler(const OpKernelInfo& info); + Status ComputeInternal(OpKernelContext* context) const override; + + private: + float scale_; + std::vector bias_; + IAllocatorUniquePtr b_data_; // gpu copy of bias +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/image_scaler_impl.cu b/onnxruntime/core/providers/cuda/tensor/image_scaler_impl.cu new file mode 100644 index 0000000000000..870f43a6230f8 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/image_scaler_impl.cu @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cuda/cu_inc/common.cuh" +#include "image_scaler_impl.h" + +namespace onnxruntime { +namespace cuda { + +template +__global__ void _ImageScalerKernel( + const T* input_data, + const float scale, + const float* bias_data, + const fast_divmod fdm_C, + const fast_divmod fdm_HW, + T* output_data, + const size_t N) { + CALCULATE_ELEMENTWISE_INDEX_OR_EXIT(id, N); + int n, c; + if (batch1) + c = fdm_HW.div(id); + else + fdm_C.divmod(fdm_HW.div(id), n, c); + output_data[id] = input_data[id] * (T)scale + (T)bias_data[c]; +} + +template +void ImageScalerImpl( + const T* input_data, + const float scale, + const float* bias_data, + const int64_t dims[4], // NCHW + T* output_data, + const size_t N) { + int blocksPerGrid = (int)(ceil(static_cast(N) / GridDim::maxThreadsPerBlock)); + fast_divmod fdm_HW((int)(dims[2] * dims[3])); + fast_divmod fdm_C; + if (dims[0] == 1) { + _ImageScalerKernel<<>>( + input_data, scale, bias_data, fdm_C, fdm_HW, output_data, N); + } else { + fdm_C = fast_divmod((int)dims[1]); + _ImageScalerKernel<<>>( + input_data, scale, bias_data, fdm_C, fdm_HW, output_data, N); + } +} + +#define SPECIALIZED_IMPL(T) \ + template void ImageScalerImpl(const T* input_data, const float scale, const float* bias_data, const int64_t dims[4], T* output_data, const size_t N); + +SPECIALIZED_IMPL(float) +SPECIALIZED_IMPL(double) +SPECIALIZED_IMPL(half) + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/image_scaler_impl.h b/onnxruntime/core/providers/cuda/tensor/image_scaler_impl.h new file mode 100644 index 0000000000000..b35bc7bd7fded --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/image_scaler_impl.h @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/providers/cuda/shared_inc/cuda_utils.h" + +namespace onnxruntime { +namespace cuda { + +template +void ImageScalerImpl( + const T* input_data, + const float scale, + const float* bias_data, + const int64_t dims[4], + T* output_data, + const size_t N); + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/pad.cc b/onnxruntime/core/providers/cuda/tensor/pad.cc new file mode 100644 index 0000000000000..809980235fbe4 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/pad.cc @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "pad.h" +#include "pad_impl.h" +#include "core/providers/cpu/tensor/utils.h" + +namespace onnxruntime { +namespace cuda { + +#define REGISTER_KERNEL_TYPED(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + Pad, \ + kOnnxDomain, \ + 2, \ + T, \ + kCudaExecutionProvider, \ + KernelDefBuilder() \ + .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + Pad); + +template +Status Pad::ComputeInternal(OpKernelContext* ctx) const { + const auto& input_tensor = *ctx->Input(0); + auto const& input_shape = input_tensor.Shape(); + auto dimension_count = input_shape.NumDimensions(); + int device_id = 0; + CudaAsyncBuffer input_dims(this, device_id, input_shape.GetDims()); + CudaAsyncBuffer input_strides(this, device_id, dimension_count); + CudaAsyncBuffer lower_pads(this, device_id, dimension_count); + CudaAsyncBuffer upper_pads(this, device_id, dimension_count); + CudaAsyncBuffer fdm_output_strides(this, device_id, dimension_count); + + TensorPitches::Calculate(input_strides.CpuSpan(), input_shape.GetDims()); + std::vector output_dims(input_shape.GetDims()); + + ONNXRUNTIME_ENFORCE(dimension_count * 2 == pads_.size(), "'pads' attribute has wrong number of values"); + + // Calculate output dimensions, and handle any negative padding + auto lower_pads_span = lower_pads.CpuSpan(); + auto upper_pads_span = upper_pads.CpuSpan(); + for (size_t i = 0; i < dimension_count; i++) { + lower_pads_span[i] = pads_[i] + slices_[i]; + upper_pads_span[i] = pads_[i + dimension_count] + slices_[i + dimension_count]; + output_dims[i] += lower_pads_span[i] + upper_pads_span[i]; + } + TensorShape output_shape(output_dims); + auto& output_tensor = *ctx->Output(0, output_shape); + ONNXRUNTIME_ENFORCE(CalculateFdmStrides(fdm_output_strides.CpuSpan(), output_dims)); + ONNXRUNTIME_RETURN_IF_ERROR(input_dims.CopyToGpu()); + ONNXRUNTIME_RETURN_IF_ERROR(input_strides.CopyToGpu()); + ONNXRUNTIME_RETURN_IF_ERROR(lower_pads.CopyToGpu()); + ONNXRUNTIME_RETURN_IF_ERROR(upper_pads.CopyToGpu()); + ONNXRUNTIME_RETURN_IF_ERROR(fdm_output_strides.CopyToGpu()); + + PadImpl( + dimension_count, + input_dims.GpuPtr(), + input_strides.GpuPtr(), + lower_pads.GpuPtr(), + upper_pads.GpuPtr(), + value_, + static_cast(mode_), + reinterpret_cast::MappedType*>(input_tensor.template Data()), + fdm_output_strides.GpuPtr(), + reinterpret_cast::MappedType*>(output_tensor.template MutableData()), + output_tensor.Shape().Size()); + + return Status::OK(); +} + +#define SPECIALIZED_COMPUTE(T) \ + REGISTER_KERNEL_TYPED(T) \ + template Status Pad::ComputeInternal(OpKernelContext* ctx) const; + +SPECIALIZED_COMPUTE(float) +SPECIALIZED_COMPUTE(double) +SPECIALIZED_COMPUTE(MLFloat16) + +} // namespace cuda +}; // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/pad.h b/onnxruntime/core/providers/cuda/tensor/pad.h new file mode 100644 index 0000000000000..6c0a360cee87f --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/pad.h @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cuda/cuda_common.h" +#include "core/providers/cpu/tensor/pad.h" + +namespace onnxruntime { +namespace cuda { + +template +class Pad final : public PadBase, public CudaKernel { + public: + Pad(const OpKernelInfo& info) : PadBase(info), CudaKernel(info) {} + + Status ComputeInternal(OpKernelContext* context) const override; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/pad_impl.cu b/onnxruntime/core/providers/cuda/tensor/pad_impl.cu new file mode 100644 index 0000000000000..7ace55590c5ee --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/pad_impl.cu @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cuda/cu_inc/common.cuh" +#include "pad_impl.h" + +namespace onnxruntime { +namespace cuda { + +// PadMode enum from core/providers/cpu/tensor/pad.h, cannot use that header because of nvcc/onnxruntime incompatibility +enum class PadMode : int { + Constant = 0, + Reflect, + Edge +}; + +template +__global__ void _PadKernel( + const size_t shape_rank, + const int64_t* input_dims, + const int64_t* input_strides, + const int64_t* lower_pads, + const int64_t* upper_pads, + const float pad_value, + const T* input_data, + const fast_divmod* fdm_output_strides, + T* output_data, + const size_t N) { + CALCULATE_ELEMENTWISE_INDEX_OR_EXIT(id, N); + CUDA_LONG input_index = 0; + CUDA_LONG output_index = id; + bool use_pad_value = false; + for (int dim = 0; dim < shape_rank && !use_pad_value; ++dim) { + int out_coord, r; + fdm_output_strides[dim].divmod(output_index, out_coord, r); + output_index = r; + int in_coord = 0; + if (out_coord < lower_pads[dim]) { + switch ((PadMode)pad_mode) { + case PadMode::Constant: + use_pad_value = true; + break; + case PadMode::Edge: + in_coord = 0; + break; + case PadMode::Reflect: + in_coord = lower_pads[dim] - out_coord; + break; + } + } else if (out_coord >= lower_pads[dim] + input_dims[dim]) { + switch ((PadMode)pad_mode) { + case PadMode::Constant: + use_pad_value = true; + break; + case PadMode::Edge: + in_coord = input_dims[dim] - 1; + break; + case PadMode::Reflect: + in_coord = input_dims[dim] - 2 - (out_coord - (lower_pads[dim] + input_dims[dim])); + break; + } + } else { + in_coord = out_coord - lower_pads[dim]; + } + input_index += input_strides[dim] * in_coord; + } + output_data[id] = use_pad_value ? (T)pad_value : input_data[input_index]; +} + +template +void PadImpl( + const size_t shape_rank, + const int64_t* input_dims, + const int64_t* input_strides, + const int64_t* lower_pads, + const int64_t* upper_pads, + const float pad_value, + const int pad_mode, + const T* input_data, + const fast_divmod* fdm_output_strides, + T* output_data, + const size_t N) { + int blocksPerGrid = (int)(ceil(static_cast(N) / GridDim::maxThreadsPerBlock)); + switch (pad_mode) { + case 0: + _PadKernel<<>>( + shape_rank, input_dims, input_strides, lower_pads, upper_pads, + pad_value, input_data, fdm_output_strides, output_data, N); + break; + case 1: + _PadKernel<<>>( + shape_rank, input_dims, input_strides, lower_pads, upper_pads, + pad_value, input_data, fdm_output_strides, output_data, N); + break; + case 2: + _PadKernel<<>>( + shape_rank, input_dims, input_strides, lower_pads, upper_pads, + pad_value, input_data, fdm_output_strides, output_data, N); + break; + } +} + +#define SPECIALIZED_IMPL(T) \ + template void PadImpl(const size_t shape_rank, const int64_t* input_dims, const int64_t* input_strides, const int64_t* lower_pads, const int64_t* upper_pads, const float pad_value, const int pad_mode, const T* input_data, const fast_divmod* fdm_output_strides, T* output_data, const size_t N); + +SPECIALIZED_IMPL(float) +SPECIALIZED_IMPL(double) +SPECIALIZED_IMPL(half) + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/pad_impl.h b/onnxruntime/core/providers/cuda/tensor/pad_impl.h new file mode 100644 index 0000000000000..d886f1fdc5e40 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/pad_impl.h @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include "core/providers/cuda/shared_inc/cuda_utils.h" + +namespace onnxruntime { +namespace cuda { + +template +void PadImpl( + const size_t shape_rank, + const int64_t* input_dims, + const int64_t* input_strides, + const int64_t* lower_pads, + const int64_t* upper_pads, + const float pad_value, + const int pad_mode, + const T* input_data, + const fast_divmod* fdm_output_strides, + T* output_data, + const size_t N); + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/reshape.cc b/onnxruntime/core/providers/cuda/tensor/reshape.cc new file mode 100644 index 0000000000000..501336d552ae9 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/reshape.cc @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "reshape.h" + +namespace onnxruntime { +namespace cuda { + +ONNX_OPERATOR_KERNEL_EX( + Reshape, + kOnnxDomain, + 5, + kCudaExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()) + .TypeConstraint("shape", DataTypeImpl::GetTensorType()) + .Alias(0, 0) + .InputMemoryType(1), + Reshape); + +ONNX_OPERATOR_VERSIONED_KERNEL_EX( + Reshape_1, + kOnnxDomain, + 1, + 4, + kCudaExecutionProvider, + KernelDefBuilder() + .Alias(0, 0) + .TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()), + Reshape_1); + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/reshape.h b/onnxruntime/core/providers/cuda/tensor/reshape.h new file mode 100644 index 0000000000000..56169f027a98c --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/reshape.h @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cuda/cuda_common.h" +#include "gsl/gsl_util" +#include "core/providers/cpu/tensor/reshape_helper.h" + +namespace onnxruntime { +namespace cuda { + +class Reshape final : public CudaKernel { + public: + Reshape(const OpKernelInfo& info) : CudaKernel(info) { + } + + Status ComputeInternal(OpKernelContext* context) const override { + // Copy the second input tensor into the shape vector + const Tensor* shapeTensor = context->Input(1); + ONNXRUNTIME_ENFORCE(shapeTensor->Shape().NumDimensions() == 1, + "A shape tensor must be a vector tensor."); + size_t nDims = static_cast(shapeTensor->Shape()[0]); + const int64_t* data = shapeTensor->template Data(); + std::vector shape; + for (size_t i = 0; i < nDims; ++i) + shape.push_back(data[i]); + + const Tensor* X = context->Input(0); + const TensorShape& X_shape = X->Shape(); + + ReshapeHelper helper(X_shape, shape); + + Tensor* Y = context->Output(0, TensorShape(shape)); + const void* source = X->DataRaw(); + void* target = Y->MutableDataRaw(); + //If source and target pointers are not equal (non-inplace operation), we need to copy the data. + if (target != source) { + CopyTensor(*X, *Y); + } + + return Status::OK(); + } +}; + +class Reshape_1 final : public CudaKernel { + public: + Reshape_1(const OpKernelInfo& info) : CudaKernel(info) { + Status status = info.GetAttrs("shape", shape_); + ONNXRUNTIME_ENFORCE(status.IsOK(), "Attribute shape is not set."); + } + + Status ComputeInternal(OpKernelContext* context) const override { + std::vector shape = shape_; + const Tensor* X = context->Input(0); + const TensorShape& X_shape = X->Shape(); + + ReshapeHelper helper(X_shape, shape); + + Tensor* Y = context->Output(0, TensorShape(shape)); + const void* source = X->DataRaw(); + void* target = Y->MutableDataRaw(); + //If source and target pointers are not equal (non-inplace operation), we need to copy the data. + if (target != source) { + CopyTensor(*X, *Y); + } + + return Status::OK(); + } + + private: + std::vector shape_; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/shape_op.cc b/onnxruntime/core/providers/cuda/tensor/shape_op.cc new file mode 100644 index 0000000000000..b0608dd364cd3 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/shape_op.cc @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/tensor/shape_op.h" +#include "core/providers/cuda/cuda_fwd.h" + +namespace onnxruntime { +namespace cuda { + +const std::vector shapeOpTypeConstraints{ + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}; + +ONNX_OPERATOR_KERNEL_EX( + Shape, + kOnnxDomain, + 1, + kCudaExecutionProvider, + KernelDefBuilder() + .OutputMemoryType(0) + .TypeConstraint("T", shapeOpTypeConstraints) + .TypeConstraint("T1", DataTypeImpl::GetTensorType()), + Shape); + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/slice.cc b/onnxruntime/core/providers/cuda/tensor/slice.cc new file mode 100644 index 0000000000000..59f4f5f796261 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/slice.cc @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "slice.h" +#include "core/providers/cpu/tensor/utils.h" +#include "slice_impl.h" + +namespace onnxruntime { +namespace cuda { + +ONNX_OPERATOR_KERNEL_EX( + Slice, + kOnnxDomain, + 1, + kCudaExecutionProvider, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()), + Slice); + +Status Slice::ComputeInternal(OpKernelContext* ctx) const { + auto input_tensor = ctx->Input(0); + ONNXRUNTIME_ENFORCE(nullptr != input_tensor); + auto& input_dimensions = input_tensor->Shape().GetDims(); + + // Initialize the starts & ends to the actual tensor shape + const size_t dimension_count = input_dimensions.size(); + std::vector starts(dimension_count, 0); + std::vector output_dims(input_dimensions); + + ONNXRUNTIME_RETURN_IF_ERROR(PrepareForCompute(dimension_count, input_dimensions, starts, output_dims)); + + TensorShape output_shape(output_dims); + auto output_tensor = ctx->Output(0, output_shape); + int64_t output_size = output_shape.Size(); + if (output_size == 0) { + return Status::OK(); + } + int device_id = 0; + CudaAsyncBuffer starts_buffer(this, device_id, dimension_count); + gsl::span starts_buffer_span = starts_buffer.CpuSpan(); + for (int i = 0; i < dimension_count; ++i) { + starts_buffer_span[i] = starts[i]; + } + starts_buffer.CopyToGpu(); + + CudaAsyncBuffer input_strides(this, device_id, dimension_count); + ONNXRUNTIME_ENFORCE(TensorPitches::Calculate(input_strides.CpuSpan(), input_dimensions)); + input_strides.CopyToGpu(); + + TensorPitches output_pitches(output_dims); + + CudaAsyncBuffer div_strides(this, device_id, dimension_count); + gsl::span div_strides_span = div_strides.CpuSpan(); + for (int i = 0; i < dimension_count; ++i) { + div_strides_span[i] = fast_divmod(gsl::narrow_cast(output_pitches[i])); + } + div_strides.CopyToGpu(); + + size_t element_size = input_tensor->DataType()->Size(); + + ONNXRUNTIME_RETURN_IF_ERROR(SliceImpl(element_size, + gsl::narrow_cast(dimension_count), + starts_buffer.GpuPtr(), + input_strides.GpuPtr(), + div_strides.GpuPtr(), + input_tensor->DataRaw(), + output_tensor->MutableDataRaw(), + output_size)); + + return Status::OK(); +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/slice.h b/onnxruntime/core/providers/cuda/tensor/slice.h new file mode 100644 index 0000000000000..442f3c368d027 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/slice.h @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/providers/cuda/cuda_common.h" +#include "core/providers/cpu/tensor/slice.h" + +namespace onnxruntime { +namespace cuda { + +class Slice final : public CudaKernel, public SliceBase { + public: + Slice(const OpKernelInfo& info) : CudaKernel(info), SliceBase(info) {} + Status ComputeInternal(OpKernelContext* context) const override; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/slice_impl.cu b/onnxruntime/core/providers/cuda/tensor/slice_impl.cu new file mode 100644 index 0000000000000..712148c08c7e2 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/slice_impl.cu @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cuda/cu_inc/common.cuh" +#include "slice_impl.h" +#include "core/providers/cuda/cuda_common.h" + +namespace onnxruntime { +namespace cuda { + +template +__global__ void _SliceKernel(const int32_t dimension_count, + const int64_t* starts, + const int64_t* input_strides, + const fast_divmod* div_strides, + const T* input_data, + T* output_data, + const CUDA_LONG N) { + CALCULATE_ELEMENTWISE_INDEX_OR_EXIT(id, N); + CUDA_LONG input_index = 0; + int div; + int mod = id; + int value = id; + int dim_idx = 0; + for (; dim_idx < dimension_count - 1; ++dim_idx) { + div_strides[dim_idx].divmod(value, div, mod); + input_index += (starts[dim_idx] + div) * input_strides[dim_idx]; + value = mod; + } + input_index += starts[dim_idx] + mod; + output_data[id] = input_data[input_index]; +} + +Status SliceImpl(const size_t element_size, + const int32_t dimension_count, + const int64_t* starts, + const int64_t* input_strides, + const fast_divmod* output_div_strides, + const void* input_data, + void* output_data, + const size_t N) { + int blocksPerGrid = (int)(ceil(static_cast(N) / GridDim::maxThreadsPerBlock)); + + switch (element_size) { + case sizeof(int8_t): + _SliceKernel<<>>( + dimension_count, starts, input_strides, output_div_strides, + reinterpret_cast::MappedType*>(input_data), + reinterpret_cast::MappedType*>(output_data), + (CUDA_LONG)N); + break; + case sizeof(int16_t): + _SliceKernel<<>>( + dimension_count, starts, input_strides, output_div_strides, + reinterpret_cast::MappedType*>(input_data), + reinterpret_cast::MappedType*>(output_data), + (CUDA_LONG)N); + break; + case sizeof(int32_t): + _SliceKernel<<>>( + dimension_count, starts, input_strides, output_div_strides, + reinterpret_cast::MappedType*>(input_data), + reinterpret_cast::MappedType*>(output_data), + (CUDA_LONG)N); + break; + case sizeof(int64_t): + _SliceKernel<<>>( + dimension_count, starts, input_strides, output_div_strides, + reinterpret_cast::MappedType*>(input_data), + reinterpret_cast::MappedType*>(output_data), + (CUDA_LONG)N); + break; + default: + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Type not supported for Slice operator"); + } + + return Status::OK(); +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/slice_impl.h b/onnxruntime/core/providers/cuda/tensor/slice_impl.h new file mode 100644 index 0000000000000..123b3e408a00d --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/slice_impl.h @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include "core/providers/cuda/shared_inc/cuda_utils.h" +#include "core/common/common.h" + +namespace onnxruntime { +namespace cuda { + +Status SliceImpl(const size_t element_size, + const int32_t dimension_count, + const int64_t* starts, + const int64_t* input_strides, + const fast_divmod* output_div_strides, + const void* input_data, + void* output_data, + const size_t N); + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/squeeze.cc b/onnxruntime/core/providers/cuda/tensor/squeeze.cc new file mode 100644 index 0000000000000..bc1f86b92b6c3 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/squeeze.cc @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "squeeze.h" + +namespace onnxruntime { +namespace cuda { + +ONNX_OPERATOR_KERNEL_EX( + Squeeze, + kOnnxDomain, + 1, + kCudaExecutionProvider, + KernelDefBuilder() + .Alias(0, 0) + .TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()), + Squeeze); + +Status Squeeze::ComputeInternal(OpKernelContext* ctx) const { + const Tensor* X = ctx->Input(0); + const TensorShape& X_shape = X->Shape(); + std::vector output_shape = ComputeOutputShape(X_shape.GetDims(), axes_); + + Tensor* Y = ctx->Output(0, TensorShape(output_shape)); + + const void* input = X->DataRaw(); + void* output = Y->MutableDataRaw(); + if (input == output) + return Status::OK(); + + auto count = X->Shape().Size(); + auto element_bytes = X->DataType()->Size(); + CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(output, input, count * element_bytes, cudaMemcpyDeviceToDevice)); + + return Status::OK(); +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/squeeze.h b/onnxruntime/core/providers/cuda/tensor/squeeze.h new file mode 100644 index 0000000000000..aa3fbfb8e6b4e --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/squeeze.h @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cuda/cuda_common.h" +#include "core/providers/cpu/tensor/squeeze.h" + +namespace onnxruntime { +namespace cuda { + +class Squeeze final : public SqueezeBase, public CudaKernel { + public: + Squeeze(const OpKernelInfo& info) : SqueezeBase(info), CudaKernel(info) {} + Status ComputeInternal(OpKernelContext* context) const override; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/tile.cc b/onnxruntime/core/providers/cuda/tensor/tile.cc new file mode 100644 index 0000000000000..39d279ea1cef8 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/tile.cc @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cuda/tensor/tile.h" +#include "core/providers/cpu/tensor/utils.h" +#include "tile_impl.h" +using namespace onnxruntime::common; +namespace onnxruntime { +namespace cuda { + +#define REGISTER_KERNEL_TYPED(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + Tile, \ + kOnnxDomain, \ + 6, \ + T, \ + kCudaExecutionProvider, \ + KernelDefBuilder() \ + .InputMemoryType(1) \ + .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + Tile); + +template +Status Tile::ComputeInternal(OpKernelContext* ctx) const { + auto& input_tensor = *ctx->Input(0); + auto& repeats_tensor = *ctx->Input(1); + size_t rank = input_tensor.Shape().NumDimensions(); + + if (repeats_tensor.Shape().NumDimensions() != 1) + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "'repeat' input tensor must be 1 dimensional"); + if (size_t(repeats_tensor.Shape().Size()) != rank) + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "'repeat' input tensor must have the same length as the 'input' tensor"); + + // Calculate the shape of the output tensor + auto* repeats = repeats_tensor.template Data(); + const auto& input_shape = input_tensor.Shape().GetDims(); + std::vector output_dims(input_shape); + for (auto axis = 0; axis < rank; axis++) + output_dims[axis] *= repeats[axis]; + TensorShape outputShape(output_dims); + auto& output_tensor = *ctx->Output(0, outputShape); + + T* output_data = output_tensor.template MutableData(); + const T* input_data = input_tensor.template Data(); + int device_id = 0; + CudaAsyncBuffer input_strides(this, device_id, rank); + CudaAsyncBuffer fdm_input_shape(this, device_id, rank); + CudaAsyncBuffer fdm_output_strides(this, device_id, rank); + + ONNXRUNTIME_ENFORCE(TensorPitches::Calculate(input_strides.CpuSpan(), input_shape)); + ONNXRUNTIME_ENFORCE(CalculateFdmStrides(fdm_output_strides.CpuSpan(), output_dims)); + + auto fdm_input_shape_span = fdm_input_shape.CpuSpan(); + for (size_t i = 0; i < input_shape.size(); ++i) + fdm_input_shape_span[i] = fast_divmod(gsl::narrow_cast(input_shape[i])); + + ONNXRUNTIME_RETURN_IF_ERROR(fdm_input_shape.CopyToGpu()); + ONNXRUNTIME_RETURN_IF_ERROR(input_strides.CopyToGpu()); + ONNXRUNTIME_RETURN_IF_ERROR(fdm_output_strides.CopyToGpu()); + + TileImpl( + rank, + fdm_input_shape.GpuPtr(), + input_strides.GpuPtr(), + reinterpret_cast::MappedType*>(input_data), + fdm_output_strides.GpuPtr(), + reinterpret_cast::MappedType*>(output_data), + output_tensor.Shape().Size()); + + return Status::OK(); +} + +#define SPECIALIZED_COMPUTE(T) \ + REGISTER_KERNEL_TYPED(T) \ + template Status Tile::ComputeInternal(OpKernelContext* ctx) const; + +SPECIALIZED_COMPUTE(float) +SPECIALIZED_COMPUTE(double) +SPECIALIZED_COMPUTE(MLFloat16) + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/tile.h b/onnxruntime/core/providers/cuda/tensor/tile.h new file mode 100644 index 0000000000000..e4c215ddb27e2 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/tile.h @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cuda/cuda_common.h" + +namespace onnxruntime { +namespace cuda { +template +struct Tile final : CudaKernel { + Tile(const OpKernelInfo& info) : CudaKernel(info) { + } + + Status ComputeInternal(OpKernelContext* context) const override; +}; +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/tile_impl.cu b/onnxruntime/core/providers/cuda/tensor/tile_impl.cu new file mode 100644 index 0000000000000..a27431849b856 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/tile_impl.cu @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cuda/cu_inc/common.cuh" +#include "tile_impl.h" + +namespace onnxruntime { +namespace cuda { + +template +__global__ void _TileKernel( + const size_t shape_rank, + const fast_divmod* fdm_input_shape, + const int64_t* input_strides, + const T* input_data, + const fast_divmod* fdm_output_strides, + T* output_data, + const CUDA_LONG N) { + CALCULATE_ELEMENTWISE_INDEX_OR_EXIT(id, N); + CUDA_LONG input_index = 0; + CUDA_LONG output_index = id; + for (int dim = 0; dim < shape_rank; ++dim) { + int out_coord, r; + fdm_output_strides[dim].divmod(output_index, out_coord, r); + output_index = r; + int q, in_coord; + fdm_input_shape[dim].divmod(out_coord, q, in_coord); + input_index += input_strides[dim] * in_coord; + } + output_data[id] = input_data[input_index]; +} + +template +void TileImpl( + const size_t shape_rank, + const fast_divmod* fdm_input_shape, + const int64_t* input_stride, + const T* input_data, + const fast_divmod* fdm_output_strides, + T* output_data, + const size_t N) { + int blocksPerGrid = (int)(ceil(static_cast(N) / GridDim::maxThreadsPerBlock)); + _TileKernel<<>>( + shape_rank, fdm_input_shape, input_stride, input_data, + fdm_output_strides, output_data, (CUDA_LONG)N); +} + +#define SPECIALIZED_IMPL(T) \ + template void TileImpl(const size_t shape_rank, const fast_divmod* fdm_input_shape, const int64_t* input_stride, const T* input_data, const fast_divmod* fdm_output_strides, T* output_data, const size_t N); + +SPECIALIZED_IMPL(float) +SPECIALIZED_IMPL(double) +SPECIALIZED_IMPL(half) + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/tile_impl.h b/onnxruntime/core/providers/cuda/tensor/tile_impl.h new file mode 100644 index 0000000000000..59751da799a08 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/tile_impl.h @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include "core/providers/cuda/shared_inc/cuda_utils.h" + +namespace onnxruntime { +namespace cuda { + +template +void TileImpl( + const size_t shape_rank, + const fast_divmod* input_shape, + const int64_t* input_strides, + const T* input_data, + const fast_divmod* fdm_output_strides, + T* output_data, + const size_t N); + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/transpose.cc b/onnxruntime/core/providers/cuda/tensor/transpose.cc new file mode 100644 index 0000000000000..3255718a1a3d3 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/transpose.cc @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "transpose.h" +#include "transpose_impl.h" +#include "core/providers/cpu/tensor/utils.h" + +namespace onnxruntime { +namespace cuda { + +#define REGISTER_KERNEL_TYPED(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + Transpose, \ + kOnnxDomain, \ + 1, \ + T, \ + kCudaExecutionProvider, \ + KernelDefBuilder() \ + .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + Transpose); + +template +Status Transpose::ComputeInternal(OpKernelContext* ctx) const { + const Tensor& X = *ctx->Input(0); + const TensorShape& input_shape = X.Shape(); + const std::vector& input_dims = input_shape.GetDims(); + size_t rank = input_dims.size(); + + std::vector output_dims(rank); + std::vector default_perm(rank); + const std::vector* p_perm = nullptr; + ComputeOutputShape(X, output_dims, default_perm, p_perm); + + TensorShape output_shape{output_dims}; + Tensor* Y = ctx->Output(0, output_shape); + int device_id = 0; + CudaAsyncBuffer input_strides(this, device_id, rank); + CudaAsyncBuffer perm(this, device_id, *p_perm); + CudaAsyncBuffer fdm_output_strides(this, device_id, rank); + ONNXRUNTIME_ENFORCE(TensorPitches::Calculate(input_strides.CpuSpan(), input_dims)); + ONNXRUNTIME_ENFORCE(CalculateFdmStrides(fdm_output_strides.CpuSpan(), output_dims)); + + ONNXRUNTIME_RETURN_IF_ERROR(input_strides.CopyToGpu()); + ONNXRUNTIME_RETURN_IF_ERROR(perm.CopyToGpu()); + ONNXRUNTIME_RETURN_IF_ERROR(fdm_output_strides.CopyToGpu()); + + TransposeImpl( + rank, + input_strides.GpuPtr(), + perm.GpuPtr(), + reinterpret_cast::MappedType*>(X.template Data()), + fdm_output_strides.GpuPtr(), + reinterpret_cast::MappedType*>(Y->template MutableData()), + output_shape.Size()); + + return Status::OK(); +} + +#define SPECIALIZED_COMPUTE(T) \ + REGISTER_KERNEL_TYPED(T) \ + template Status Transpose::ComputeInternal(OpKernelContext* ctx) const; + +SPECIALIZED_COMPUTE(float) +SPECIALIZED_COMPUTE(double) +SPECIALIZED_COMPUTE(MLFloat16) + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/transpose.h b/onnxruntime/core/providers/cuda/tensor/transpose.h new file mode 100644 index 0000000000000..2bb2326b7f6c5 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/transpose.h @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "gsl/gsl_util" +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cuda/cuda_common.h" +#include "core/providers/cpu/tensor/transpose.h" + +namespace onnxruntime { +namespace cuda { + +template +class Transpose final : public CudaKernel, public TransposeBase { + public: + Transpose(const OpKernelInfo& info) : CudaKernel(info), TransposeBase(info) {} + + Status ComputeInternal(OpKernelContext* context) const override; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/transpose_impl.cu b/onnxruntime/core/providers/cuda/tensor/transpose_impl.cu new file mode 100644 index 0000000000000..32580f928f6f0 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/transpose_impl.cu @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cuda/cu_inc/common.cuh" +#include "transpose_impl.h" + +namespace onnxruntime { +namespace cuda { + +template +__global__ void _TransposeKernel( + const size_t shape_rank, + const int64_t* input_strides, + const int64_t* perm, + const T* input_data, + const fast_divmod* fdm_output_strides, + T* output_data, + const size_t N) { + CALCULATE_ELEMENTWISE_INDEX_OR_EXIT(id, N); + CUDA_LONG input_index = 0; + CUDA_LONG output_index = id; + + for (int dim = 0; dim < shape_rank; ++dim) { + int out_coord, r; + fdm_output_strides[dim].divmod(output_index, out_coord, r); + output_index = r; + input_index += input_strides[perm[dim]] * out_coord; + } + output_data[id] = input_data[input_index]; +} + +template +void TransposeImpl( + const size_t shape_rank, + const int64_t* input_strides, + const int64_t* perm, + const T* input_data, + const fast_divmod* fdm_output_strides, + T* output_data, + const size_t N) { + int blocksPerGrid = (int)(ceil(static_cast(N) / GridDim::maxThreadsPerBlock)); + _TransposeKernel<<>>( + shape_rank, input_strides, perm, input_data, + fdm_output_strides, output_data, N); +} + +#define SPECIALIZED_IMPL(T) \ + template void TransposeImpl( \ + const size_t shape_rank, \ + const int64_t* input_strides, \ + const int64_t* perm, \ + const T* input_data, \ + const fast_divmod* fdm_output_strides, \ + T* output_data, \ + const size_t N); + +SPECIALIZED_IMPL(float) +SPECIALIZED_IMPL(double) +SPECIALIZED_IMPL(half) + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/transpose_impl.h b/onnxruntime/core/providers/cuda/tensor/transpose_impl.h new file mode 100644 index 0000000000000..5adc22f63d5ef --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/transpose_impl.h @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include "core/providers/cuda/shared_inc/cuda_utils.h" + +namespace onnxruntime { +namespace cuda { + +template +void TransposeImpl( + const size_t shape_rank, + const int64_t* input_strides, + const int64_t* perm, + const T* input_data, + const fast_divmod* fdm_output_strides, + T* output_data, + const size_t N); + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/unsqueeze.cc b/onnxruntime/core/providers/cuda/tensor/unsqueeze.cc new file mode 100644 index 0000000000000..3312d57d17436 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/unsqueeze.cc @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cuda/tensor/unsqueeze.h" + +namespace onnxruntime { +namespace cuda { + +ONNX_OPERATOR_KERNEL_EX( + Unsqueeze, + kOnnxDomain, + 1, + kCudaExecutionProvider, + KernelDefBuilder() + .Alias(0, 0) + .TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()), + Unsqueeze); + +Status Unsqueeze::ComputeInternal(OpKernelContext* ctx) const { + Prepare p; + ONNXRUNTIME_RETURN_IF_ERROR(PrepareCompute(ctx, p)); + + const void* input = p.input_tensor->DataRaw(); + void* output = p.output_tensor->MutableDataRaw(); + if (input == output) + return Status::OK(); + + auto count = p.input_tensor->Shape().Size(); + auto element_bytes = p.input_tensor->DataType()->Size(); + CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(output, input, count * element_bytes, cudaMemcpyDeviceToDevice)); + + return Status::OK(); +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/unsqueeze.h b/onnxruntime/core/providers/cuda/tensor/unsqueeze.h new file mode 100644 index 0000000000000..6c41fd9bbe6d6 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/unsqueeze.h @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cuda/cuda_common.h" +#include "core/providers/cpu/tensor/unsqueeze.h" + +namespace onnxruntime { +namespace cuda { + +class Unsqueeze final : public UnsqueezeBase, public CudaKernel { + public: + Unsqueeze(const OpKernelInfo& info) : UnsqueezeBase(info), CudaKernel(info) {} + Status ComputeInternal(OpKernelContext* context) const override; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/upsample.cc b/onnxruntime/core/providers/cuda/tensor/upsample.cc new file mode 100644 index 0000000000000..f7312ef9716de --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/upsample.cc @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "upsample.h" +#include "upsample_impl.h" +#include "core/providers/cpu/tensor/utils.h" + +using namespace onnxruntime::common; + +namespace onnxruntime { +namespace cuda { + +#define REGISTER_KERNEL_TYPED(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + Upsample, \ + kOnnxDomain, \ + 7, \ + T, \ + kCudaExecutionProvider, \ + KernelDefBuilder() \ + .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + Upsample); + +REGISTER_KERNEL_TYPED(float) +REGISTER_KERNEL_TYPED(double) +REGISTER_KERNEL_TYPED(MLFloat16) +REGISTER_KERNEL_TYPED(int32_t) + +template +Status Upsample::ComputeInternal(OpKernelContext* context) const { + const Tensor* X = context->Input(0); + ONNXRUNTIME_ENFORCE(nullptr != X); + const std::vector& X_dims = X->Shape().GetDims(); + auto rank = X_dims.size(); + if (rank == 0) + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Upsample: input tensor cannot be scalar."); + + if (rank != scales_.size()) + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Upsample: input tensor's dimension does not match the scales."); + + std::vector Y_dims; + for (std::size_t i = 0; i < rank; i++) { + Y_dims.push_back(static_cast(scales_[i] * X_dims[i])); + } + Tensor* Y = context->Output(0, Y_dims); + typedef typename ToCudaType::MappedType CudaT; + + // kernel + int device_id = 0; + TensorPitches input_pitches(X_dims); + CudaAsyncBuffer input_strides(this, device_id, rank); + gsl::span input_stride_span = input_strides.CpuSpan(); + + TensorPitches output_pitches(Y_dims); + CudaAsyncBuffer output_div_pitches(this, device_id, rank); + gsl::span div_strides_span = output_div_pitches.CpuSpan(); + + + CudaAsyncBuffer scales_div(this, device_id, rank); + gsl::span scales_div_span = scales_div.CpuSpan(); + + for (int i = 0; i < rank; ++i) { + input_stride_span[i] = input_pitches[i]; + div_strides_span[i] = fast_divmod(gsl::narrow_cast(output_pitches[i])); + scales_div_span[i] = fast_divmod(gsl::narrow_cast(ceil(scales_[i]))); + } + input_strides.CopyToGpu(); + output_div_pitches.CopyToGpu(); + scales_div.CopyToGpu(); + + size_t output_count = Y->Shape().Size(); + + if (UpsampleMode::LINEAR == mode_) { + if (rank != 4) + return Status(ONNXRUNTIME, FAIL, "Upsample: linear mode upsample only support 4-D tensor with NCHW layout"); + } + + UpampleImpl(mode_, + rank, + (UpsampleMode::LINEAR == mode_) ? X_dims[2] : 0, + input_strides.GpuPtr(), + output_div_pitches.GpuPtr(), + scales_div.GpuPtr(), + reinterpret_cast(X->template Data()), + reinterpret_cast(Y->template MutableData()), + output_count); + + return Status::OK(); +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/upsample.h b/onnxruntime/core/providers/cuda/tensor/upsample.h new file mode 100644 index 0000000000000..d6dfdea2c6e67 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/upsample.h @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/providers/cuda/cuda_common.h" +#include "core/providers/cpu/tensor/upsample.h" + +namespace onnxruntime { +namespace cuda { + +struct TVMState; + +template +class Upsample : public UpsampleBase, public CudaKernel { + public: + Upsample(OpKernelInfo info) : UpsampleBase(info), CudaKernel(info) { + } + + Status ComputeInternal(OpKernelContext* context) const override; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/upsample_impl.cu b/onnxruntime/core/providers/cuda/tensor/upsample_impl.cu new file mode 100644 index 0000000000000..baf1ac8c0b3f5 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/upsample_impl.cu @@ -0,0 +1,132 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cuda/cu_inc/common.cuh" +#include "upsample_impl.h" + +namespace onnxruntime { +namespace cuda { + +template +__global__ void _UpampleNearestKernel(const size_t rank, + const int64_t* input_pitches, + const fast_divmod* output_div_pitches, + const fast_divmod* scales_div, + const T* input_data, + T* output_data, + const size_t N) { + CALCULATE_ELEMENTWISE_INDEX_OR_EXIT(id, N); + CUDA_LONG input_index = 0; + CUDA_LONG output_index = id; + + int div, mod; + for (int dim = 0; dim < rank; ++dim) { + output_div_pitches[dim].divmod(output_index, div, mod); + output_index = mod; + if (scales_div[dim].d_ != 1 && div > 0) { + scales_div[dim].divmod(div, div, mod); + } + input_index += input_pitches[dim] * div; + } + output_data[id] = input_data[input_index]; +} + +template +__global__ void _UpampleBilinearKernel(const int64_t input_dim2, + const int64_t* input_pitches, + const fast_divmod* output_div_pitches, + const fast_divmod* scales_div, + const T* input_data, + T* output_data, + const size_t N) { + CALCULATE_ELEMENTWISE_INDEX_OR_EXIT(id, N); + CUDA_LONG input_index = 0; + + // For bilinear mode, scales[0]=scales[1]=1 + int mod; + int index_of_dim0, index_of_dim1, index_of_dim2, index_of_dim3; + output_div_pitches[0].divmod(id, index_of_dim0, mod); + output_div_pitches[1].divmod(mod, index_of_dim1, mod); + output_div_pitches[2].divmod(mod, index_of_dim2, mod); + index_of_dim3 = mod; + int index_of_input_dim2, index_of_input_dim3, x_offset, y_offset; + scales_div[2].divmod(index_of_dim2, index_of_input_dim2, y_offset); + scales_div[3].divmod(index_of_dim3, index_of_input_dim3, x_offset); + + input_index = index_of_dim0 * input_pitches[0] + + index_of_dim1 * input_pitches[1] + + index_of_input_dim2 * input_pitches[2] + + index_of_input_dim3; + + T x00 = input_data[input_index]; + T x10, x01, x11; + + bool end_of_dim2 = false; + if (index_of_input_dim2 == (input_dim2 - 1)) { + // It's the end in dimension 2 + x01 = x00; + end_of_dim2 = true; + } else { + x01 = input_data[input_index + input_pitches[2]]; + } + + if (index_of_input_dim3 == (input_pitches[2] - 1)) { + // It's the end in dimension 3 + x10 = x00; + x11 = x01; + } + else { + x10 = input_data[input_index + 1]; + x11 = end_of_dim2 ? x10 : input_data[input_index + input_pitches[2] + 1]; + } + + T y_offset_T = static_cast(y_offset); + T x_offset_T = static_cast(x_offset); + T scales_div2_T = static_cast(scales_div[2].d_); + T scales_div3_T = static_cast(scales_div[3].d_); + T y0 = x00 + static_cast(y_offset_T * (x01 - x00) / scales_div2_T); + T y1 = x10 + static_cast(y_offset_T * (x11 - x10) / scales_div2_T); + + output_data[id] = y0 + static_cast(x_offset_T * (y1 - y0) / scales_div3_T); +} + +template +void UpampleImpl(const onnxruntime::UpsampleMode upsample_mode, + const size_t rank, + const int64_t input_dim2, + const int64_t* input_pitches, + const fast_divmod* output_div_pitches, + const fast_divmod* scales_div, + const T* input_data, + T* output_data, + const size_t N) { + int blocksPerGrid = (int)(ceil(static_cast(N) / GridDim::maxThreadsPerBlock)); + if (onnxruntime::UpsampleMode::NN == upsample_mode) { + _UpampleNearestKernel<<>>( + rank, input_pitches, output_div_pitches, scales_div, + input_data, output_data, N); + } else if (onnxruntime::UpsampleMode::LINEAR == upsample_mode) { + _UpampleBilinearKernel<<>>( + input_dim2, input_pitches, output_div_pitches, scales_div, + input_data, output_data, N); + } +} + +#define SPECIALIZED_IMPL(T) \ + template void UpampleImpl(const onnxruntime::UpsampleMode upsample_mode, \ + const size_t rank, \ + const int64_t input_dim2, \ + const int64_t* input_pitches, \ + const fast_divmod* output_div_pitches, \ + const fast_divmod* scales_div, \ + const T* input_data, \ + T* output_data, \ + const size_t N); + +SPECIALIZED_IMPL(float) +SPECIALIZED_IMPL(double) +SPECIALIZED_IMPL(half) +SPECIALIZED_IMPL(int32_t) + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/upsample_impl.h b/onnxruntime/core/providers/cuda/tensor/upsample_impl.h new file mode 100644 index 0000000000000..0ff2000908584 --- /dev/null +++ b/onnxruntime/core/providers/cuda/tensor/upsample_impl.h @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include "core/providers/cuda/shared_inc/cuda_utils.h" +#include "core/common/common.h" +#include "core/providers/cpu/tensor/upsample.h" + +namespace onnxruntime { +namespace cuda { + +template +void UpampleImpl(const onnxruntime::UpsampleMode upsample_mode, + const size_t rank, + const int64_t input_dim2, + const int64_t* input_pitches, + const fast_divmod* output_div_pitches, + const fast_divmod* scales_div, + const T* input_data, + T* output_data, + const size_t N); + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/math/gemm.cc b/onnxruntime/core/providers/mkldnn/math/gemm.cc new file mode 100644 index 0000000000000..c2648b159c189 --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/math/gemm.cc @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gemm.h" +#include "core/providers/cpu/math/gemm_helper.h" +#include "core/util/math_cpuonly.h" +#include "mkldnn.h" +#include "core/providers/mkldnn/mkldnn_fwd.h" + +namespace onnxruntime { +namespace mkl_dnn { + +ONNX_OPERATOR_KERNEL_EX( + Gemm, + kOnnxDomain, + 7, + kMklDnnExecutionProvider, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Gemm); + +template <> +Status Gemm::Compute(OpKernelContext* ctx) const { + const auto X = ctx->Input(0); + const auto W = ctx->Input(1); + const auto B = ctx->Input(2); + GemmHelper helper(X->Shape(), trans_A_, W->Shape(), trans_B_, B->Shape()); + + if (!helper.State().IsOK()) + return helper.State(); + + int M = gsl::narrow_cast(helper.M()); + int N = gsl::narrow_cast(helper.N()); + int K = gsl::narrow_cast(helper.K()); + auto Y = ctx->Output(0, TensorShape({M, N})); + + if (beta_ != 0) { + auto output_mat = EigenMatrixMapRowMajor( + Y->template MutableData(), + M, + N); + output_mat.setZero(); + + auto& b_shape = B->Shape(); + // if B is (), (1,) or (1, 1), add the scalar + if (b_shape.Size() == 1) { + output_mat.array() += *(B->template Data()); + } + // B is (N,) + else if (b_shape.NumDimensions() == 1) { + auto bias_vec = ConstEigenVectorMap( + B->template Data(), + N); + output_mat.rowwise() += bias_vec.transpose(); + } else if (b_shape.NumDimensions() == 2) { + // B is (M, 1) + if (b_shape[1] == 1) { + auto bias_vec = ConstEigenVectorMap( + B->template Data(), + M); + output_mat.colwise() += bias_vec; + } + // B is (1, N) + else if (b_shape[0] == 1) { + auto bias_vec = ConstEigenVectorMap( + B->template Data(), + N); + output_mat.rowwise() += bias_vec.transpose(); + } + // B is (M, N), no broadcast needed. + else { + auto bias_mat = ConstEigenMatrixMapRowMajor( + B->template Data(), + M, + N); + output_mat += bias_mat; + } + } + } + + // mkldnn_sgemm expects col major matrices, so we need to swap the operands A and B + auto status = mkldnn_sgemm(trans_B_ ? "T" : "N", + trans_A_ ? "T" : "N", + &N, &M, &K, + &alpha_, W->template Data(), + trans_B_ ? &K : &N, + X->template Data(), + trans_A_ ? &M : &K, + &beta_, Y->template MutableData(), &N); + if (status == mkldnn_success) { + return Status::OK(); + } else { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "mkldnn_sgemm failed with status: ", status); + } +} + +} // namespace mkl_dnn +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/math/gemm.h b/onnxruntime/core/providers/mkldnn/math/gemm.h new file mode 100644 index 0000000000000..d5d23b7bbd0ca --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/math/gemm.h @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/framework/op_kernel.h" + +namespace onnxruntime { +namespace mkl_dnn { +template +class Gemm final : public OpKernel { + public: + Gemm(const OpKernelInfo& info) : OpKernel(info) { + int64_t temp; + ONNXRUNTIME_ENFORCE(info.GetAttr("transA", &temp).IsOK()); + trans_A_ = (temp != 0); + + ONNXRUNTIME_ENFORCE(info.GetAttr("transB", &temp).IsOK()); + trans_B_ = (temp != 0); + + ONNXRUNTIME_ENFORCE(info.GetAttr("alpha", &alpha_).IsOK()); + ONNXRUNTIME_ENFORCE(info.GetAttr("beta", &beta_).IsOK()); + } + + Status Compute(OpKernelContext* context) const override; + + private: + bool trans_A_; + bool trans_B_; + float alpha_; + float beta_; +}; +} // namespace mkl_dnn +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/mkldnn_allocator.cc b/onnxruntime/core/providers/mkldnn/mkldnn_allocator.cc new file mode 100644 index 0000000000000..a7db234a33ebb --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/mkldnn_allocator.cc @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "mkldnn_allocator.h" +#include "core/framework/allocatormgr.h" + +namespace onnxruntime { + +const ONNXRuntimeAllocatorInfo& MKLDNNAllocator::Info() const { + static constexpr ONNXRuntimeAllocatorInfo mkl_allocator_info(MKLDNN, ONNXRuntimeAllocatorType::ONNXRuntimeDeviceAllocator, 0, ONNXRuntimeMemTypeDefault); + return mkl_allocator_info; +} + +const ONNXRuntimeAllocatorInfo& MKLDNNCPUAllocator::Info() const { + static constexpr ONNXRuntimeAllocatorInfo mkl_cpu_allocator_info(MKLDNN_CPU, ONNXRuntimeAllocatorType::ONNXRuntimeDeviceAllocator, 0, ONNXRuntimeMemTypeCPUOutput); + return mkl_cpu_allocator_info; +} +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/mkldnn_allocator.h b/onnxruntime/core/providers/mkldnn/mkldnn_allocator.h new file mode 100644 index 0000000000000..afcc7b55397ae --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/mkldnn_allocator.h @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/framework/allocator.h" + +// Placeholder for an MKL allocators +namespace onnxruntime { +constexpr const char* MKLDNN = "MklDnn"; +constexpr const char* MKLDNN_CPU = "MklDnnCpu"; + +class MKLDNNAllocator : public CPUAllocator { + public: + const ONNXRuntimeAllocatorInfo& Info() const override; +}; +class MKLDNNCPUAllocator : public CPUAllocator { + public: + const ONNXRuntimeAllocatorInfo& Info() const override; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/mkldnn_common.h b/onnxruntime/core/providers/mkldnn/mkldnn_common.h new file mode 100644 index 0000000000000..d103791a2a2b1 --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/mkldnn_common.h @@ -0,0 +1,175 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/common/common.h" +#include "mkldnn.hpp" +#include + +namespace onnxruntime { +namespace mkl_dnn { + +template +static mkldnn::memory::data_type MklDnnType(); + +// Add more types here as needed. +template <> +mkldnn::memory::data_type MklDnnType() { + return mkldnn::memory::data_type::f32; +} + +static mkldnn::engine& GetEngine() { + static mkldnn::engine cpu_engine = mkldnn::engine(mkldnn::engine::cpu, 0); + return cpu_engine; +} + +static void AddDimsToKey(std::string& key, const mkldnn::memory::dims& dims) { + key.append(1, '#'); + for (size_t i = 0; i < dims.size(); i++) { + key.append(std::to_string(dims[i])); + key.append(1, '_'); + } + key.append(1, '#'); +} + +class PrimitiveBase { + public: + virtual ~PrimitiveBase() = default; +}; + +template +class PrimitivePool { + public: + PrimitivePool() = default; + ~PrimitivePool() = default; + + void SetPrimitive(const std::string& key, std::unique_ptr primitive) { + auto& map = PrimitivePool::GetMap(); + auto iter = map.find(key); + // We should not find a primitive already using this key. + ONNXRUNTIME_ENFORCE(iter == map.end(), "duplicate key: " + key); + map.insert(std::make_pair(key, std::move(primitive))); + } + + PrimitiveBase* GetPrimitive(const std::string& key) { + const auto& map = PrimitivePool::GetMap(); + auto iter = map.find(key); + if (iter != map.end()) { + return iter->second.get(); + } else { + return nullptr; + } + } + + private: + // For thread safety, the map needs to be kept in thread local storage. + static inline std::unordered_map>& GetMap() { + static thread_local std::unordered_map> map; + return map; + } +}; + +// Struct which encapsulates parameters for MKLDNN memory reorder primitive. +struct MemoryReorderParams { + const mkldnn::memory& src; + const mkldnn::memory& dst; + + MemoryReorderParams(const mkldnn::memory& src, const mkldnn::memory& dst) : src(src), dst(dst) {} + + // Used as the key for MemoryReorder primitive reuse pool. + std::string ToString() const { + std::string key; + key.reserve(64); + key.append("reorder_"); + const auto& src_desc = src.get_primitive_desc().desc().data; + const auto& dst_desc = dst.get_primitive_desc().desc().data; + mkldnn::memory::dims src_dims(src_desc.dims, &src_desc.dims[src_desc.ndims]); + mkldnn::memory::dims dst_dims(dst_desc.dims, &dst_desc.dims[dst_desc.ndims]); + key.append(std::to_string(src_desc.format)); + key.append(1, '_'); + key.append(std::to_string(dst_desc.data_type)); + AddDimsToKey(key, src_dims); + key.append(std::to_string(dst_desc.format)); + key.append(1, '_'); + key.append(std::to_string(dst_desc.data_type)); + AddDimsToKey(key, dst_dims); + return key; + } +}; + +// Encapsulates an MKLDNN memory reorder primitive. +// these are needed to convert the source/weight/destination memory layout +// to one that is identified by MKLDNN to be optimal for performance. +class MemoryReorderPrimitive : public PrimitiveBase { + public: + explicit MemoryReorderPrimitive(const MemoryReorderParams& params) : cpu_engine_(GetEngine()) { + Initialize(params); + } + ~MemoryReorderPrimitive() = default; + + std::shared_ptr GetPrimitive() { + return context_.primitive; + } + + void SetMemory(const MemoryReorderParams& params) { + context_.src_mem->set_data_handle(params.src.get_data_handle()); + context_.dst_mem->set_data_handle(params.dst.get_data_handle()); + } + + private: + struct MemoryReorderContext { + std::shared_ptr src_mem; + std::shared_ptr dst_mem; + std::shared_ptr primitive; + MemoryReorderContext() : src_mem(nullptr), dst_mem(nullptr), primitive(nullptr) { + } + } context_; + + mkldnn::engine& cpu_engine_; + + void Initialize(const MemoryReorderParams& params) { + context_.src_mem = std::make_shared( + mkldnn::memory({params.src.get_primitive_desc().desc(), cpu_engine_}, nullptr)); + context_.dst_mem = std::make_shared( + mkldnn::memory({params.dst.get_primitive_desc().desc(), cpu_engine_}, nullptr)); + context_.primitive = std::make_shared( + mkldnn::reorder(*context_.src_mem, *context_.dst_mem)); + } +}; + +// Pool which allows for reuse of MKLDNN memory reorder primitives which are expensive to instantiate. +// To address thread safety, the primitives are stored in a map on thread local storage. +template +class MemoryReorderPrimitivePool : public PrimitivePool { + public: + static MemoryReorderPrimitivePool& GetInstance() { + static MemoryReorderPrimitivePool pool; + return pool; + } + + static MemoryReorderPrimitive* Get(const MemoryReorderParams& params) { + MemoryReorderPrimitive* primitive = static_cast( + MemoryReorderPrimitivePool::GetInstance().GetPrimitive(params.ToString())); + if (primitive == nullptr) { + auto reorder_primitive = std::make_unique(params); + primitive = reorder_primitive.get(); + MemoryReorderPrimitivePool::GetInstance().SetPrimitive(params.ToString(), std::move(reorder_primitive)); + } + primitive->SetMemory(params); + return primitive; + } + + private: + MemoryReorderPrimitivePool() = default; + ~MemoryReorderPrimitivePool() = default; +}; + +template +static void DoReorder(const MemoryReorderParams& params) { + std::vector net; + net.push_back(*(MemoryReorderPrimitivePool::Get(params)->GetPrimitive())); + mkldnn::stream(mkldnn::stream::kind::eager).submit(net).wait(); +} + +} // namespace mkl_dnn +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.cc b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.cc new file mode 100644 index 0000000000000..f98ce0c68bd48 --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.cc @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "mkldnn_allocator.h" +#include "mkldnn_execution_provider.h" +#include "core/framework/allocator.h" +#include "core/framework/transformer_memcpy.h" +#include "core/framework/memcpy.h" +#include "core/framework/kernel_registry.h" +#include "mkldnn_fwd.h" + +namespace onnxruntime { +namespace mkl_dnn { + +ONNX_OPERATOR_KERNEL_EX( + MemcpyFromHost, + kOnnxDomain, + 1, + kMklDnnExecutionProvider, + KernelDefBuilder().InputMemoryType(0).TypeConstraint("T", DataTypeImpl::AllTensorTypes()), + Memcpy); + +ONNX_OPERATOR_KERNEL_EX( + MemcpyToHost, + kOnnxDomain, + 1, + kMklDnnExecutionProvider, + KernelDefBuilder().OutputMemoryType(0).TypeConstraint("T", DataTypeImpl::AllTensorTypes()), + Memcpy); + +} // namespace mkl_dnn + +MKLDNNExecutionProvider::MKLDNNExecutionProvider(const MKLDNNExecutionProviderInfo& /*info*/) { + DeviceAllocatorRegistrationInfo default_allocator_info({ONNXRuntimeMemTypeDefault, + [](int) { return std::make_unique(); }, std::numeric_limits::max()}); + InsertAllocator(CreateAllocator(default_allocator_info)); + + DeviceAllocatorRegistrationInfo cpu_allocator_info({ONNXRuntimeMemTypeCPUOutput, + [](int) { return std::make_unique(); }, std::numeric_limits::max()}); + InsertAllocator(CreateAllocator(cpu_allocator_info)); +} + +MKLDNNExecutionProvider::~MKLDNNExecutionProvider() { +} + +Status MKLDNNExecutionProvider::CopyTensor(const Tensor& src, Tensor& dst) const { + // Support CPU <-> MKLDNN for now + if (!(strcmp(src.Location().name, MKLDNN) == 0 && strcmp(dst.Location().name, CPU) == 0) && + !(strcmp(src.Location().name, CPU) == 0 && strcmp(dst.Location().name, MKLDNN) == 0) && + !(strcmp(src.Location().name, MKLDNN) == 0 && strcmp(dst.Location().name, MKLDNN_CPU) == 0)) { + ONNXRUNTIME_NOT_IMPLEMENTED(src.Location().name, " copy to ", dst.Location().name, " is not implemented"); + } + + // Todo: Copy for now. May optimize later to avoid copy. + size_t bytes = src.DataType()->Size() * src.Shape().Size(); + const void* src_data = src.DataRaw(); + void* dst_data = dst.MutableDataRaw(); + memcpy(dst_data, src_data, bytes); + + return Status::OK(); +} + +namespace mkl_dnn { +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 1, Conv); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 7, Gemm); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 1, MemcpyFromHost); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 1, MemcpyToHost); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 7, 8, float, AveragePool); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 1, 8, float, GlobalAveragePool); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 1, 7, float, MaxPool); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 8, 8, float, MaxPool); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 1, 8, float, GlobalMaxPool); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 1, float, LRN); + +void RegisterMKLDNNKernels(std::function fn) { + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); +} +} // namespace mkl_dnn + +std::shared_ptr MKLDNNExecutionProvider::GetKernelRegistry() const { + static std::shared_ptr kernel_registry = std::make_shared(onnxruntime::mkl_dnn::RegisterMKLDNNKernels); + return kernel_registry; +} +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h new file mode 100644 index 0000000000000..e5820f8951514 --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include "core/framework/allocatormgr.h" +#include "core/framework/execution_provider.h" +#include "core/graph/graph_transformer.h" + +namespace onnxruntime { + +// Information needed to construct MKL-DNN execution providers. +struct MKLDNNExecutionProviderInfo { + bool create_arena{true}; + + explicit MKLDNNExecutionProviderInfo(bool use_arena) + : create_arena(use_arena) {} + MKLDNNExecutionProviderInfo() = default; +}; + +// Logical device representation. +class MKLDNNExecutionProvider : public IExecutionProvider { + public: + explicit MKLDNNExecutionProvider(const MKLDNNExecutionProviderInfo& info); + virtual ~MKLDNNExecutionProvider(); + + std::string Type() const override { + return onnxruntime::kMklDnnExecutionProvider; + } + + Status CopyTensor(const Tensor& src, Tensor& dst) const override; + + const void* GetExecutionHandle() const noexcept override { + return nullptr; + } + + virtual std::shared_ptr GetKernelRegistry() const override; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/mkldnn_fwd.h b/onnxruntime/core/providers/mkldnn/mkldnn_fwd.h new file mode 100644 index 0000000000000..47564d2cab014 --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/mkldnn_fwd.h @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +namespace onnxruntime { +namespace mkl_dnn { +template +KernelCreateInfo BuildKernel(); +} +} diff --git a/onnxruntime/core/providers/mkldnn/mkldnn_provider_factory.cc b/onnxruntime/core/providers/mkldnn/mkldnn_provider_factory.cc new file mode 100644 index 0000000000000..86ae8f3476cac --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/mkldnn_provider_factory.cc @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/mkldnn/mkldnn_provider_factory.h" +#include +#include "mkldnn_execution_provider.h" + +using namespace onnxruntime; + +namespace { +struct MkldnnProviderFactory { + const ONNXRuntimeProviderFactoryInterface* const cls; + std::atomic_int ref_count; + bool create_arena; + MkldnnProviderFactory(); +}; + +ONNXStatusPtr ONNXRUNTIME_API_STATUSCALL CreateMkldnn(void* this_, ONNXRuntimeProviderPtr* out) { + MKLDNNExecutionProviderInfo info; + MkldnnProviderFactory* this_ptr = (MkldnnProviderFactory*)this_; + info.create_arena = this_ptr->create_arena; + MKLDNNExecutionProvider* ret = new MKLDNNExecutionProvider(info); + *out = (ONNXRuntimeProviderPtr)ret; + return nullptr; +} + +uint32_t ONNXRUNTIME_API_STATUSCALL ReleaseMkldnn(void* this_) { + MkldnnProviderFactory* this_ptr = (MkldnnProviderFactory*)this_; + if (--this_ptr->ref_count == 0) + delete this_ptr; + return 0; +} + +uint32_t ONNXRUNTIME_API_STATUSCALL AddRefMkldnn(void* this_) { + MkldnnProviderFactory* this_ptr = (MkldnnProviderFactory*)this_; + ++this_ptr->ref_count; + return 0; +} + +constexpr ONNXRuntimeProviderFactoryInterface mkl_cls = { + {AddRefMkldnn, + ReleaseMkldnn}, + CreateMkldnn, +}; + +MkldnnProviderFactory::MkldnnProviderFactory() : cls(&mkl_cls), ref_count(1), create_arena(true) {} +} // namespace + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeCreateMkldnnExecutionProviderFactory, int use_arena, _Out_ ONNXRuntimeProviderFactoryPtr** out) { + MkldnnProviderFactory* ret = new MkldnnProviderFactory(); + ret->create_arena = (use_arena != 0); + *out = (ONNXRuntimeProviderFactoryPtr*)ret; + return nullptr; +} \ No newline at end of file diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.cc b/onnxruntime/core/providers/mkldnn/nn/conv.cc new file mode 100644 index 0000000000000..52307dc617cc2 --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/nn/conv.cc @@ -0,0 +1,407 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#ifdef _WIN32 +#pragma warning(disable : 4244) +#endif + +#include "core/providers/mkldnn/mkldnn_common.h" +#include "core/providers/mkldnn/nn/conv.h" +#include "core/providers/mkldnn/mkldnn_fwd.h" + +namespace onnxruntime { +namespace mkl_dnn { + +namespace { +// Struct which encapsulates parameters for MKLDNN Conv2d primitive. +struct Conv2dParams { + mkldnn::memory::dims& src_dims; + mkldnn::memory::dims& filter_dims; + mkldnn::memory::dims& bias_dims; + mkldnn::memory::dims& dst_dims; + mkldnn::memory::dims& strides; + mkldnn::memory::dims& dilations; + mkldnn::memory::dims& padding_left; + mkldnn::memory::dims& padding_right; + + Conv2dParams(mkldnn::memory::dims& src_dims, mkldnn::memory::dims& filter_dims, + mkldnn::memory::dims& bias_dims, mkldnn::memory::dims& dst_dims, + mkldnn::memory::dims& strides, mkldnn::memory::dims& dilations, + mkldnn::memory::dims& padding_left, mkldnn::memory::dims& padding_right) + : src_dims(src_dims), + filter_dims(filter_dims), + bias_dims(bias_dims), + dst_dims(dst_dims), + strides(strides), + dilations(dilations), + padding_left(padding_left), + padding_right(padding_right) {} + + // Used as the key for Conv2d Primitive Reuse Pool. + std::string ToString() const { + std::string key; + key.reserve(128); + key.append("conv2d_"); + AddDimsToKey(key, src_dims); + AddDimsToKey(key, filter_dims); + AddDimsToKey(key, bias_dims); + AddDimsToKey(key, dst_dims); + AddDimsToKey(key, strides); + AddDimsToKey(key, dilations); + AddDimsToKey(key, padding_left); + AddDimsToKey(key, padding_right); + return key; + } +}; + +template +class Conv2dPrimitive : public PrimitiveBase { + public: + explicit Conv2dPrimitive(const Conv2dParams& params) + : cpu_engine_(GetEngine()) { + context_.stream.reset(new mkldnn::stream(mkldnn::stream::kind::eager)); + if (context_.conv_fwd == nullptr) { + Initialize(params); + } + } + + ~Conv2dPrimitive() = default; + + void Compute(const T* src_data, const T* filter_data, + const T* dst_data, const T* bias_data = nullptr) { + context_.src_mem->set_data_handle( + static_cast(const_cast(src_data))); + context_.filter_mem->set_data_handle( + static_cast(const_cast(filter_data))); + if (bias_data != nullptr) { + context_.bias_mem->set_data_handle( + static_cast(const_cast(bias_data))); + } + context_.dst_mem->set_data_handle( + static_cast(const_cast(dst_data))); + context_.stream->submit(context_.net); + + context_.src_mem->set_data_handle(nullptr); + context_.filter_mem->set_data_handle(nullptr); + if (bias_data != nullptr) { + context_.bias_mem->set_data_handle(nullptr); + } + context_.dst_mem->set_data_handle(nullptr); + return; + } + + mkldnn::memory::format GetSrcMemoryFormat() const { return context_.src_fmt; } + + mkldnn::memory::format GetFilterMemoryFormat() const { return context_.filter_fmt; } + + mkldnn::memory::format GetDstMemoryFormat() const { return context_.dst_fmt; } + + mkldnn::convolution_forward::primitive_desc* GetPrimitiveDesc() const { + return context_.conv_fwd_pd.get(); + } + + private: + struct Conv2dContext { + mkldnn::memory::format src_fmt; + mkldnn::memory::format filter_fmt; + mkldnn::memory::format dst_fmt; + + std::unique_ptr src_mem; + std::unique_ptr filter_mem; + std::unique_ptr bias_mem; + std::unique_ptr dst_mem; + + std::unique_ptr fwd_desc; + + std::unique_ptr src_md; + std::unique_ptr filter_md; + std::unique_ptr bias_md; + std::unique_ptr dst_md; + + std::unique_ptr conv_fwd_pd; + std::unique_ptr conv_fwd; + + std::unique_ptr stream; + std::vector net; + + Conv2dContext() + : src_fmt(mkldnn::memory::format::any), + filter_fmt(mkldnn::memory::format::any), + dst_fmt(mkldnn::memory::format::any), + src_mem(nullptr), + filter_mem(nullptr), + bias_mem(nullptr), + dst_mem(nullptr), + fwd_desc(nullptr), + src_md(nullptr), + filter_md(nullptr), + bias_md(nullptr), + conv_fwd_pd(nullptr), + conv_fwd(nullptr), + stream(nullptr) {} + }; + + void Initialize(const Conv2dParams& params) { + // Set the memory descriptors to format::any to allow MKLDNN to decide what the optimal memory layout should be + // for the computation given the input params. + context_.src_md.reset(new mkldnn::memory::desc( + {params.src_dims}, MklDnnType(), mkldnn::memory::format::any)); + context_.filter_md.reset(new mkldnn::memory::desc( + {params.filter_dims}, MklDnnType(), mkldnn::memory::format::any)); + context_.dst_md.reset(new mkldnn::memory::desc( + {params.dst_dims}, MklDnnType(), mkldnn::memory::format::any)); + if (!params.bias_dims.empty()) + context_.bias_md.reset(new mkldnn::memory::desc( + {params.bias_dims}, MklDnnType(), mkldnn::memory::format::any)); + + if (!params.bias_dims.empty()) { + context_.fwd_desc.reset(new mkldnn::convolution_forward::desc( + mkldnn::prop_kind::forward, mkldnn::convolution_direct, *context_.src_md, + *context_.filter_md, *context_.bias_md, *context_.dst_md, + params.strides, params.dilations, params.padding_left, + params.padding_right, mkldnn::padding_kind::zero)); + } else { + context_.fwd_desc.reset(new mkldnn::convolution_forward::desc( + mkldnn::prop_kind::forward, mkldnn::convolution_direct, *context_.src_md, + *context_.filter_md, *context_.dst_md, params.strides, + params.dilations, params.padding_left, + params.padding_right, mkldnn::padding_kind::zero)); + } + + context_.conv_fwd_pd.reset(new mkldnn::convolution_forward::primitive_desc( + *context_.fwd_desc, cpu_engine_)); + + context_.src_fmt = static_cast( + context_.conv_fwd_pd.get()->src_primitive_desc().desc().data.format); + + context_.filter_fmt = static_cast( + context_.conv_fwd_pd.get()->weights_primitive_desc().desc().data.format); + + context_.dst_fmt = static_cast( + context_.conv_fwd_pd.get()->dst_primitive_desc().desc().data.format); + + context_.src_mem.reset( + new mkldnn::memory(context_.conv_fwd_pd.get()->src_primitive_desc(), nullptr)); + context_.filter_mem.reset( + new mkldnn::memory(context_.conv_fwd_pd.get()->weights_primitive_desc(), nullptr)); + context_.dst_mem.reset( + new mkldnn::memory(context_.conv_fwd_pd.get()->dst_primitive_desc(), nullptr)); + + if (!params.bias_dims.empty()) { + context_.bias_mem.reset(new mkldnn::memory( + {{{params.bias_dims}, MklDnnType(), mkldnn::memory::format::x}, + cpu_engine_}, + nullptr)); + context_.conv_fwd.reset(new mkldnn::convolution_forward( + *context_.conv_fwd_pd, *context_.src_mem, *context_.filter_mem, + *context_.bias_mem, *context_.dst_mem)); + } else { + context_.conv_fwd.reset( + new mkldnn::convolution_forward(*context_.conv_fwd_pd, *context_.src_mem, + *context_.filter_mem, *context_.dst_mem)); + } + + context_.net.push_back(*context_.conv_fwd); + } + + Conv2dContext context_; + mkldnn::engine& cpu_engine_; +}; + +// Pool which allows for reuse of MKLDNN Conv2d primitives which are expensive to instantiate. +// To address thread safety, the primitives are stored in a map on thread local storage. +template +class Conv2dPrimitivePool : public PrimitivePool { + public: + static Conv2dPrimitive* Get(const Conv2dParams& params) { + Conv2dPrimitive* primitive = dynamic_cast*>( + Conv2dPrimitivePool::GetInstance().GetPrimitive(params.ToString())); + + if (primitive == nullptr) { + auto conv2d_primitive = std::make_unique>(params); + primitive = conv2d_primitive.get(); + Conv2dPrimitivePool::GetInstance().SetPrimitive(params.ToString(), std::move(conv2d_primitive)); + } + return primitive; + } + + private: + Conv2dPrimitivePool() = default; + ~Conv2dPrimitivePool() = default; + + static Conv2dPrimitivePool& GetInstance() { + static Conv2dPrimitivePool pool; + return pool; + } +}; +} // namespace + +template +Status Conv::Compute(OpKernelContext* context) const { + size_t num_inputs = OpKernel::Node().InputDefs().size(); + + const Tensor* X = context->Input(0); + const Tensor* W = context->Input(1); + const Tensor* B = num_inputs == 3 ? context->Input(2) : nullptr; + const int64_t N = X->Shape()[0]; + const int64_t M = W->Shape()[0]; + const int group_mkl = static_cast(onnxruntime::ConvBase::group_); + + ONNXRUNTIME_RETURN_IF_ERROR(onnxruntime::ConvBase::ValidateInputShape(X, W)); + + std::vector kernel_shape = onnxruntime::ConvBase::ComputeKernelShape(W->Shape()); + + // TODO: Support more than 2d kernels + if (kernel_shape.size() != 2) { + // Fall Back to CPU implementation. + return onnxruntime::Conv::Compute(context); + } + + if (kernel_shape.size() + 2 != W->Shape().NumDimensions()) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "kernel_shape num_dims is not compatible with W num_dims.", + " kernel_shape: ", TensorShape(kernel_shape).ToString().c_str(), + " W: ", W->Shape().ToString().c_str()); + } + + for (size_t i = 0; i < kernel_shape.size(); ++i) { + if (kernel_shape[i] != W->Shape()[i + 2]) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "kernel_shape is not compatible with W shape.", + " kernel_shape: ", TensorShape(kernel_shape).ToString().c_str(), + " W: ", W->Shape().ToString().c_str()); + } + } + + std::vector pads(onnxruntime::ConvBase::pads_); + if (pads.empty()) { + pads.resize(kernel_shape.size() * 2, 0); + } + std::vector dilations(onnxruntime::ConvBase::dilations_); + if (dilations.empty()) { + dilations.resize(kernel_shape.size(), 1); + } + std::vector strides(onnxruntime::ConvBase::strides_); + if (strides.empty()) { + strides.resize(kernel_shape.size(), 1); + } + + std::vector Y_dims; + Y_dims.insert(Y_dims.begin(), {N, M}); + TensorShape input_shape = X->Shape().Slice(2); + ONNXRUNTIME_RETURN_IF_ERROR(onnxruntime::ConvBase::InferOutputShape(input_shape, kernel_shape, strides, dilations, &pads, &Y_dims)); + Tensor* Y = context->Output(0, TensorShape(Y_dims)); + TensorShape output_shape = Y->Shape().Slice(2); + + mkldnn::memory::dims src_dims_mkl(X->Shape().GetDims().begin(), X->Shape().GetDims().end()); + mkldnn::memory::dims filter_dims_mkl; + if (group_mkl == 1) { + filter_dims_mkl.assign(W->Shape().GetDims().begin(), W->Shape().GetDims().end()); + } else { + filter_dims_mkl.assign({ + group_mkl, + static_cast(W->Shape()[0] / group_mkl), + static_cast(W->Shape()[1]), + static_cast(W->Shape()[2]), + static_cast(W->Shape()[3]), + }); + } + mkldnn::memory::dims strides_mkl(strides.begin(), strides.end()); + mkldnn::memory::dims dilations_mkl(dilations.begin(), dilations.end()); + // mkldnn dilations start from 0 so we need to subtract 1 from each dim. + dilations_mkl[0] -= 1; + dilations_mkl[1] -= 1; + mkldnn::memory::dims padding_left_mkl(pads.begin(), pads.begin() + 2); + mkldnn::memory::dims padding_right_mkl(pads.begin() + 2, pads.end()); + mkldnn::memory::dims dst_dims_mkl(Y_dims.begin(), Y_dims.end()); + mkldnn::memory::dims bias_dims_mkl; + if (B != nullptr) { + bias_dims_mkl.assign(B->Shape().GetDims().begin(), B->Shape().GetDims().end()); + } + + AllocatorPtr alloc; + ONNXRUNTIME_RETURN_IF_ERROR(context->GetTempSpaceAllocator(&alloc)); + IAllocatorUniquePtr src_reorder_buffer; + IAllocatorUniquePtr filter_reorder_buffer; + IAllocatorUniquePtr dst_reorder_buffer; + + const T* src_data = X->template Data(); + const T* filter_data = W->template Data(); + T* dst_data = Y->template MutableData(); + const T* bias_data = nullptr; + if (B != nullptr) { + bias_data = B->template Data(); + } + + try { + Conv2dParams conv2d_params(src_dims_mkl, filter_dims_mkl, bias_dims_mkl, + dst_dims_mkl, strides_mkl, dilations_mkl, + padding_left_mkl, padding_right_mkl); + Conv2dPrimitive* conv2d_primitive = Conv2dPrimitivePool::Get(conv2d_params); + auto conv_fwd_pd = conv2d_primitive->GetPrimitiveDesc(); + + mkldnn::engine& cpu_engine = GetEngine(); + + // Per ONNX spec, + // X (src) is NCHW, W (filter) is OIHW/GOIHW, and Y (dst) is NCHW + auto src_md = mkldnn::memory::desc(src_dims_mkl, MklDnnType(), mkldnn::memory::format::nchw); + auto filter_format = group_mkl == 1 ? mkldnn::memory::format::oihw : mkldnn::memory::format::goihw; + auto dst_md = mkldnn::memory::desc(dst_dims_mkl, MklDnnType(), mkldnn::memory::format::nchw); + + // Reorder src memory layout if necessary. + if (src_md.data.format != conv2d_primitive->GetSrcMemoryFormat()) { + auto pd = mkldnn::memory::primitive_desc(src_md, cpu_engine); + mkldnn::memory src = mkldnn::memory(pd, (void*)src_data); + src_reorder_buffer = IAllocator::MakeUniquePtr(alloc, sizeof(T) * X->Shape().Size()); + mkldnn::memory dst = mkldnn::memory(conv_fwd_pd->src_primitive_desc(), src_reorder_buffer.get()); + MemoryReorderParams params(src, dst); + DoReorder(params); + src_data = static_cast(dst.get_data_handle()); + } + + // Reorder filter memory layout if necessary. + if (filter_format != conv2d_primitive->GetFilterMemoryFormat()) { + auto pd = mkldnn::memory::primitive_desc(mkldnn::memory::desc(filter_dims_mkl, + MklDnnType(), + filter_format), + cpu_engine); + mkldnn::memory src = mkldnn::memory(pd, (void*)filter_data); + filter_reorder_buffer = IAllocator::MakeUniquePtr(alloc, sizeof(T) * W->Shape().Size()); + mkldnn::memory dst = mkldnn::memory(conv_fwd_pd->weights_primitive_desc(), filter_reorder_buffer.get()); + MemoryReorderParams params(src, dst); + DoReorder(params); + filter_data = static_cast(dst.get_data_handle()); + } + + // Allocate dst buffer if reorder is necessary + if (dst_md.data.format != conv2d_primitive->GetDstMemoryFormat()) { + dst_reorder_buffer = IAllocator::MakeUniquePtr(alloc, sizeof(T) * Y->Shape().Size()); + dst_data = static_cast(dst_reorder_buffer.get()); + } + + conv2d_primitive->Compute(src_data, filter_data, dst_data, bias_data); + + // Reorder dst memory layout if necessary + if (dst_md.data.format != conv2d_primitive->GetDstMemoryFormat()) { + mkldnn::memory src = mkldnn::memory(conv_fwd_pd->dst_primitive_desc(), (void*)dst_data); + auto pd = mkldnn::memory::primitive_desc(dst_md, cpu_engine); + mkldnn::memory dst = mkldnn::memory(pd, Y->template MutableData()); + MemoryReorderParams params(src, dst); + DoReorder(params); + } + + } catch (mkldnn::error& e) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Status: ", e.status, ", message: ", e.message.c_str()); + } + + return Status::OK(); +} + +ONNX_OPERATOR_KERNEL_EX( + Conv, + kOnnxDomain, + 1, + kMklDnnExecutionProvider, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Conv); + +} // namespace mkl_dnn +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.h b/onnxruntime/core/providers/mkldnn/nn/conv.h new file mode 100644 index 0000000000000..8189a7ca6ba10 --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/nn/conv.h @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/framework/op_kernel.h" +#include "core/providers/cpu/nn/conv.h" + +namespace onnxruntime { +namespace mkl_dnn { +template +class Conv final : public onnxruntime::Conv { + public: + Conv(const OpKernelInfo& info) : onnxruntime::Conv(info) { + } + + Status Compute(OpKernelContext* context) const override; + + private: +}; +} // namespace mkl_dnn +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/nn/lrn.cc b/onnxruntime/core/providers/mkldnn/nn/lrn.cc new file mode 100644 index 0000000000000..6bdba04f62593 --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/nn/lrn.cc @@ -0,0 +1,226 @@ + +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#ifdef _WIN32 +#pragma warning(disable : 4244) +#endif + +#include "core/providers/mkldnn/mkldnn_common.h" +#include "core/providers/mkldnn/nn/lrn.h" +#include "core/providers/mkldnn/mkldnn_fwd.h" + +namespace onnxruntime { +namespace mkl_dnn { + +ONNX_OPERATOR_TYPED_KERNEL_EX( + LRN, + kOnnxDomain, + 1, + float, + kMklDnnExecutionProvider, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + LRN); + +namespace { +// Struct which encapsulates parameters for MKLDNN LRN primitive. +struct LRNParams { + mkldnn::memory::dims& dims_; + float alpha_; + float beta_; + float bias_; + int size_; + + LRNParams(mkldnn::memory::dims& dims, float alpha, float beta, float bias, int size) + : dims_(dims), alpha_(alpha), beta_(beta), bias_(bias), size_(size) {} + + // Used as the key for LRN Primitive Reuse LRN. + std::string ToString() const { + std::string key; + key.reserve(128); + key.append("lrn"); + AddDimsToKey(key, dims_); + key.append('#' + std::to_string(alpha_) + '#'); + key.append('#' + std::to_string(beta_) + '#'); + key.append('#' + std::to_string(bias_) + '#'); + key.append('#' + std::to_string(size_) + '#'); + + return key; + } +}; + +template +class LRNPrimitive : public PrimitiveBase { + public: + explicit LRNPrimitive(const LRNParams& params) + : cpu_engine_(GetEngine()) { + context_.stream.reset(new mkldnn::stream(mkldnn::stream::kind::eager)); + if (context_.lrn_fwd == nullptr) { + Initialize(params); + } + } + + ~LRNPrimitive() = default; + + void Compute(const T* src_data, const T* dst_data) { + context_.src_mem->set_data_handle(static_cast(const_cast(src_data))); + context_.dst_mem->set_data_handle(static_cast(const_cast(dst_data))); + context_.stream->submit(context_.net); + + context_.src_mem->set_data_handle(nullptr); + context_.dst_mem->set_data_handle(nullptr); + return; + } + + mkldnn::memory::format GetSrcMemoryFormat() const { return context_.src_fmt; } + + mkldnn::lrn_forward::primitive_desc* GetPrimitiveDesc() const { + return context_.fwd_primitive_desc.get(); + } + + private: + struct LRNContext { + mkldnn::memory::format src_fmt; + std::unique_ptr src_md; + + std::unique_ptr src_mem; + std::unique_ptr dst_mem; + + std::unique_ptr fwd_desc; + std::unique_ptr fwd_primitive_desc; + std::unique_ptr lrn_fwd; + + std::unique_ptr stream; + std::vector net; + + LRNContext() + : src_fmt(mkldnn::memory::format::any), + src_md(nullptr), + src_mem(nullptr), + dst_mem(nullptr), + fwd_desc(nullptr), + fwd_primitive_desc(nullptr), + lrn_fwd(nullptr), + stream(nullptr) {} + }; + + void Initialize(const LRNParams& params) { + context_.src_md.reset(new mkldnn::memory::desc({params.dims_}, MklDnnType(), mkldnn::memory::format::nchw)); + + mkldnn::algorithm algo = mkldnn::algorithm::lrn_across_channels; + context_.fwd_desc.reset(new mkldnn::lrn_forward::desc( + mkldnn::prop_kind::forward_scoring, algo, *context_.src_md, + params.size_, params.alpha_, params.beta_, params.bias_)); + + context_.fwd_primitive_desc.reset(new mkldnn::lrn_forward::primitive_desc( + *context_.fwd_desc, cpu_engine_)); + + context_.src_fmt = static_cast( + context_.fwd_primitive_desc.get()->src_primitive_desc().desc().data.format); + + context_.src_mem.reset(new mkldnn::memory(context_.fwd_primitive_desc.get()->src_primitive_desc(), nullptr)); + context_.dst_mem.reset(new mkldnn::memory(context_.fwd_primitive_desc.get()->dst_primitive_desc(), nullptr)); + context_.lrn_fwd.reset( + new mkldnn::lrn_forward(*context_.fwd_primitive_desc, *context_.src_mem, *context_.dst_mem)); + context_.net.push_back(*context_.lrn_fwd); + } + + LRNContext context_; + mkldnn::engine& cpu_engine_; +}; + +// Pool which allows for reuse of MKLDNN Pool primitives which are expensive to instantiate. +// To address thread safety, the primitives are stored in a map on thread local storage. +template +class LRNPrimitivePool : public PrimitivePool { + public: + static LRNPrimitive* Get(const LRNParams& params) { + LRNPrimitive* primitive = dynamic_cast*>( + LRNPrimitivePool::GetInstance().GetPrimitive(params.ToString())); + if (primitive == nullptr) { + auto pool_primitive = std::make_unique>(params); + primitive = pool_primitive.get(); + LRNPrimitivePool::GetInstance().SetPrimitive(params.ToString(), std::move(pool_primitive)); + } + return primitive; + } + + private: + LRNPrimitivePool() = default; + ~LRNPrimitivePool() = default; + + static LRNPrimitivePool& GetInstance() { + static LRNPrimitivePool pool; + return pool; + } +}; +} // namespace + +template +Status LRN::Compute(OpKernelContext* context) const { + const Tensor* X = context->Input(0); + const T* src_data = X->template Data(); + + const TensorShape& x_shape = X->Shape(); + if (x_shape.NumDimensions() != 4) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Support NCHW image only."); + } + + const auto& x_dims = x_shape.GetDims(); + mkldnn::memory::dims dims_mkl(x_dims.begin(), x_dims.end()); + + Tensor* Y = context->Output(0, TensorShape(x_dims)); + T* dst_data = Y->template MutableData(); + + AllocatorPtr alloc; + ONNXRUNTIME_RETURN_IF_ERROR(context->GetTempSpaceAllocator(&alloc)); + IAllocatorUniquePtr src_reorder_buffer; + IAllocatorUniquePtr dst_reorder_buffer; + + try { + LRNParams lrn_params(dims_mkl, this->alpha_, this->beta_, this->bias_, this->size_); + LRNPrimitive* lrn_primitive = LRNPrimitivePool::Get(lrn_params); + auto fwd_primitive_desc = lrn_primitive->GetPrimitiveDesc(); + + mkldnn::engine& cpu_engine = GetEngine(); + mkldnn::memory::format mem_format = dims_mkl.size() == 5 ? mkldnn::memory::format::ncdhw : mkldnn::memory::format::nchw; + // Per ONNX spec, X (src) is NCHW and Y (dst) is NCHW + auto src_md = mkldnn::memory::desc(dims_mkl, MklDnnType(), mem_format); + auto dst_md = mkldnn::memory::desc(dims_mkl, MklDnnType(), mem_format); + + // Reorder src memory layout if necessary. + if (src_md.data.format != lrn_primitive->GetSrcMemoryFormat()) { + auto pd = mkldnn::memory::primitive_desc(src_md, cpu_engine); + mkldnn::memory src = mkldnn::memory(pd, (void*)src_data); + src_reorder_buffer = IAllocator::MakeUniquePtr(alloc, sizeof(T) * X->Shape().Size()); + mkldnn::memory dst = mkldnn::memory(fwd_primitive_desc->src_primitive_desc(), src_reorder_buffer.get()); + MemoryReorderParams params(src, dst); + DoReorder(params); + src_data = static_cast(dst.get_data_handle()); + } + + // Allocate dst buffer if reorder is necessary + if (src_md.data.format != lrn_primitive->GetSrcMemoryFormat()) { + dst_reorder_buffer = IAllocator::MakeUniquePtr(alloc, sizeof(T) * Y->Shape().Size()); + dst_data = static_cast(dst_reorder_buffer.get()); + } + + lrn_primitive->Compute(src_data, dst_data); + + // Reorder dst memory layout if necessary + if (src_md.data.format != lrn_primitive->GetSrcMemoryFormat()) { + mkldnn::memory src = mkldnn::memory(fwd_primitive_desc->dst_primitive_desc(), (void*)dst_data); + auto pd = mkldnn::memory::primitive_desc(dst_md, cpu_engine); + mkldnn::memory dst = mkldnn::memory(pd, Y->template MutableData()); + MemoryReorderParams params(src, dst); + DoReorder(params); + } + } catch (mkldnn::error& e) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Status: ", e.status, ", message: ", e.message.c_str()); + } + + return Status::OK(); +} + +} // namespace onnxruntime +} diff --git a/onnxruntime/core/providers/mkldnn/nn/lrn.h b/onnxruntime/core/providers/mkldnn/nn/lrn.h new file mode 100644 index 0000000000000..fe2a4c0373844 --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/nn/lrn.h @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/framework/op_kernel.h" +#include "core/providers/cpu/nn/lrn.h" + +namespace onnxruntime { +namespace mkl_dnn { + +template +class LRN final : public onnxruntime::LRN { + public: + LRN(const OpKernelInfo& info) : onnxruntime::LRN(info) {} + + Status Compute(OpKernelContext* p_op_kernel_context) const override; +}; + +} +} \ No newline at end of file diff --git a/onnxruntime/core/providers/mkldnn/nn/pool.cc b/onnxruntime/core/providers/mkldnn/nn/pool.cc new file mode 100644 index 0000000000000..b7c576c05c934 --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/nn/pool.cc @@ -0,0 +1,322 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#ifdef _WIN32 +#pragma warning(disable : 4244) +#endif + +#include "core/common/cpuid_info.h" +#include "core/providers/mkldnn/mkldnn_common.h" +#include "core/providers/mkldnn/nn/pool.h" +#include "core/providers/mkldnn/mkldnn_fwd.h" + +namespace onnxruntime { +namespace mkl_dnn { + +#define POOLING_KERNEL(op_name, data_type, pool_type, since_version, end_version) \ + ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ + op_name, \ + kOnnxDomain, \ + since_version, \ + end_version, \ + data_type, \ + kMklDnnExecutionProvider, \ + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + Pool); + +POOLING_KERNEL(AveragePool, float, AveragePool, 7, 8) +POOLING_KERNEL(GlobalAveragePool, float, AveragePool, 1, 8) +POOLING_KERNEL(MaxPool, float, MaxPool<1>, 1, 7) +POOLING_KERNEL(MaxPool, float, MaxPool<8>, 8, 8) +POOLING_KERNEL(GlobalMaxPool, float, MaxPool<1>, 1, 8) + +namespace { +// Struct which encapsulates parameters for MKLDNN Pool primitive. +struct PoolParams { + std::string op_name; + // Pooling primitive needs version as part of key because there + // are multiple versions of mkldnn pool kernels. + std::string version; + mkldnn::memory::dims& src_dims; + mkldnn::memory::dims& dst_dims; + mkldnn::memory::dims& kernel; + mkldnn::memory::dims& strides; + mkldnn::memory::dims& padding_left; + mkldnn::memory::dims& padding_right; + bool count_include_pad; + + PoolParams(std::string op_name, std::string version, + mkldnn::memory::dims& src_dims, mkldnn::memory::dims& dst_dims, + mkldnn::memory::dims& kernel, mkldnn::memory::dims& strides, + mkldnn::memory::dims& padding_left, mkldnn::memory::dims& padding_right, + bool count_include_pad) + : op_name(op_name), + version(version), + src_dims(src_dims), + dst_dims(dst_dims), + kernel(kernel), + strides(strides), + padding_left(padding_left), + padding_right(padding_right), + count_include_pad(count_include_pad) {} + + // Used as the key for Pool Primitive Reuse Pool. + std::string ToString() const { + std::string key; + key.reserve(128); + key.append(op_name); + key.append(version); + AddDimsToKey(key, src_dims); + AddDimsToKey(key, dst_dims); + AddDimsToKey(key, kernel); + AddDimsToKey(key, strides); + AddDimsToKey(key, padding_left); + AddDimsToKey(key, padding_right); + key.append(count_include_pad ? "true" : "false"); + return key; + } +}; + +template +class PoolPrimitive : public PrimitiveBase { + public: + explicit PoolPrimitive(const PoolParams& params) + : cpu_engine_(GetEngine()) { + context_.stream.reset(new mkldnn::stream(mkldnn::stream::kind::eager)); + if (context_.pool_fwd == nullptr) { + Initialize(params); + } + } + + ~PoolPrimitive() = default; + + void Compute(const T* src_data, const T* dst_data) { + context_.src_mem->set_data_handle(static_cast(const_cast(src_data))); + context_.dst_mem->set_data_handle(static_cast(const_cast(dst_data))); + context_.stream->submit(context_.net); + + context_.src_mem->set_data_handle(nullptr); + context_.dst_mem->set_data_handle(nullptr); + return; + } + + mkldnn::memory::format GetSrcMemoryFormat() const { return context_.src_fmt; } + mkldnn::memory::format GetDstMemoryFormat() const { return context_.dst_fmt; } + + // std::unique_ptr GetDstMemoryDesc() const { return context_.dst_md; } + mkldnn::pooling_forward::primitive_desc* GetPrimitiveDesc() const { + return context_.fwd_primitive_desc.get(); + } + + private: + struct PoolContext { + mkldnn::memory::format src_fmt; + mkldnn::memory::format dst_fmt; + + std::unique_ptr src_mem; + std::unique_ptr dst_mem; + + std::unique_ptr fwd_desc; + + std::unique_ptr src_md; + std::unique_ptr dst_md; + + std::unique_ptr fwd_primitive_desc; + + std::unique_ptr pool_fwd; + + std::unique_ptr stream; + std::vector net; + + PoolContext() + : src_fmt(mkldnn::memory::format::any), + dst_fmt(mkldnn::memory::format::any), + src_mem(nullptr), + dst_mem(nullptr), + fwd_desc(nullptr), + src_md(nullptr), + fwd_primitive_desc(nullptr), + pool_fwd(nullptr), + stream(nullptr) {} + }; + + void Initialize(const PoolParams& params) { + bool is_2D = params.src_dims.size() == 4 ? true : false; + mkldnn::memory::format fmt = mkldnn::memory::format::any; + if (CPUIDInfo::GetCPUIDInfo().HasAVX512f()) { + fmt = is_2D ? mkldnn::memory::format::nChw16c : mkldnn::memory::format::nCdhw16c; + } else if (CPUIDInfo::GetCPUIDInfo().HasAVX2() && (params.src_dims[1] % 8 == 0)) { + fmt = is_2D ? mkldnn::memory::format::nChw8c : mkldnn::memory::format::ncdhw; + } else { + fmt = is_2D ? mkldnn::memory::format::nchw : mkldnn::memory::format::ncdhw; + } + context_.src_md.reset(new mkldnn::memory::desc( + {params.src_dims}, MklDnnType(), fmt)); + context_.dst_md.reset(new mkldnn::memory::desc( + {params.dst_dims}, MklDnnType(), mkldnn::memory::format::any)); + + mkldnn::algorithm algo = mkldnn::algorithm::pooling_max; + if (PoolType::type == onnxruntime::PoolType::kAveragePool) { + algo = mkldnn::algorithm::pooling_avg_exclude_padding; + if (params.count_include_pad) { + algo = mkldnn::algorithm::pooling_avg_include_padding; + } + } + context_.fwd_desc.reset(new mkldnn::pooling_forward::desc( + mkldnn::prop_kind::forward_inference, algo, + *context_.src_md, *context_.dst_md, + params.strides, params.kernel, + params.padding_left, params.padding_right, + mkldnn::padding_kind::zero)); + + context_.fwd_primitive_desc.reset(new mkldnn::pooling_forward::primitive_desc( + *context_.fwd_desc, cpu_engine_)); + + context_.src_fmt = static_cast( + context_.fwd_primitive_desc.get()->src_primitive_desc().desc().data.format); + + context_.dst_fmt = static_cast( + context_.fwd_primitive_desc.get()->dst_primitive_desc().desc().data.format); + + context_.src_mem.reset( + new mkldnn::memory(context_.fwd_primitive_desc.get()->src_primitive_desc(), nullptr)); + context_.dst_mem.reset( + new mkldnn::memory(context_.fwd_primitive_desc.get()->dst_primitive_desc(), nullptr)); + context_.pool_fwd.reset( + new mkldnn::pooling_forward(*context_.fwd_primitive_desc, *context_.src_mem, *context_.dst_mem)); + context_.net.push_back(*context_.pool_fwd); + } + + PoolContext context_; + mkldnn::engine& cpu_engine_; +}; + +// Pool which allows for reuse of MKLDNN Pool primitives which are expensive to instantiate. +// To address thread safety, the primitives are stored in a map on thread local storage. +template +class PoolPrimitivePool : public PrimitivePool { + public: + static PoolPrimitive* Get(const PoolParams& params) { + PoolPrimitive* primitive = dynamic_cast*>( + PoolPrimitivePool::GetInstance().GetPrimitive(params.ToString())); + if (primitive == nullptr) { + auto pool_primitive = std::make_unique>(params); + primitive = pool_primitive.get(); + PoolPrimitivePool::GetInstance().SetPrimitive(params.ToString(), std::move(pool_primitive)); + } + return primitive; + } + + private: + PoolPrimitivePool() = default; + ~PoolPrimitivePool() = default; + + static PoolPrimitivePool& GetInstance() { + static PoolPrimitivePool pool; + return pool; + } +}; +} // namespace + +template +Status Pool::Compute(OpKernelContext* context) const { + const Tensor* X = context->Input(0); + const TensorShape& x_shape = X->Shape(); + const auto& x_dims = x_shape.GetDims(); + + if (x_shape.NumDimensions() < 3) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Input dimension cannot be less than 3."); + } + + if (x_shape.NumDimensions() == 3) { + // Fall Back to CPU implementation. + return onnxruntime::Pool::Compute(context); + } + + std::vector kernel_shape = this->kernel_shape_; + std::vector pads = this->pads_; + std::vector strides = this->strides_; + + if (this->global_pooling_) { + kernel_shape.assign(x_dims.begin() + 2, x_dims.end()); + pads.assign(kernel_shape.size() * 2, 0); + strides.assign(kernel_shape.size(), 1); + } + + std::vector y_dims = PoolBase::SetOutputSize(x_shape, x_shape[1], &pads); + Tensor* Y = context->Output(0, TensorShape(y_dims)); + + size_t num_outputs = OpKernel::Node().OutputDefs().size(); + if (num_outputs == 2) { + Tensor* I = context->Output(1, TensorShape(y_dims)); + if (nullptr != I) { + return onnxruntime::Pool::Compute(context); + } + } + + const T* src_data = X->template Data(); + T* dst_data = Y->template MutableData(); + + mkldnn::memory::dims src_dims_mkl(x_dims.begin(), x_dims.end()); + mkldnn::memory::dims dst_dims_mkl(y_dims.begin(), y_dims.end()); + mkldnn::memory::dims kernel_mkl(kernel_shape.begin(), kernel_shape.end()); + mkldnn::memory::dims strides_mkl(strides.begin(), strides.end()); + mkldnn::memory::dims padding_left_mkl(pads.begin(), pads.begin() + (pads.size() / 2)); + mkldnn::memory::dims padding_right_mkl(pads.begin() + (pads.size() / 2), pads.end()); + + AllocatorPtr alloc; + ONNXRUNTIME_RETURN_IF_ERROR(context->GetTempSpaceAllocator(&alloc)); + IAllocatorUniquePtr src_reorder_buffer; + IAllocatorUniquePtr dst_reorder_buffer; + + try { + PoolParams pool_params(this->op_name_, this->opset_version_, + src_dims_mkl, dst_dims_mkl, + kernel_mkl, strides_mkl, + padding_left_mkl, padding_right_mkl, + this->count_include_pad_); + PoolPrimitive* pool_primitive = PoolPrimitivePool::Get(pool_params); + auto fwd_primitive_desc = pool_primitive->GetPrimitiveDesc(); + + mkldnn::engine& cpu_engine = GetEngine(); + mkldnn::memory::format mem_format = src_dims_mkl.size() == 5 ? mkldnn::memory::format::ncdhw : mkldnn::memory::format::nchw; + // Per ONNX spec, X (src) is NCHW and Y (dst) is NCHW + auto src_md = mkldnn::memory::desc(src_dims_mkl, MklDnnType(), mem_format); + auto dst_md = mkldnn::memory::desc(dst_dims_mkl, MklDnnType(), mem_format); + + // Reorder src memory layout if necessary. + if (src_md.data.format != pool_primitive->GetSrcMemoryFormat()) { + auto pd = mkldnn::memory::primitive_desc(src_md, cpu_engine); + mkldnn::memory src = mkldnn::memory(pd, (void*)src_data); + src_reorder_buffer = IAllocator::MakeUniquePtr(alloc, sizeof(T) * X->Shape().Size()); + mkldnn::memory dst = mkldnn::memory(fwd_primitive_desc->src_primitive_desc(), src_reorder_buffer.get()); + MemoryReorderParams params(src, dst); + DoReorder(params); + src_data = static_cast(dst.get_data_handle()); + } + + // Allocate dst buffer if reorder is necessary + if (dst_md.data.format != pool_primitive->GetDstMemoryFormat()) { + dst_reorder_buffer = IAllocator::MakeUniquePtr(alloc, sizeof(T) * Y->Shape().Size()); + dst_data = static_cast(dst_reorder_buffer.get()); + } + + pool_primitive->Compute(src_data, dst_data); + + // Reorder dst memory layout if necessary + if (dst_md.data.format != pool_primitive->GetDstMemoryFormat()) { + mkldnn::memory src = mkldnn::memory(fwd_primitive_desc->dst_primitive_desc(), (void*)dst_data); + auto pd = mkldnn::memory::primitive_desc(dst_md, cpu_engine); + mkldnn::memory dst = mkldnn::memory(pd, Y->template MutableData()); + MemoryReorderParams params(src, dst); + DoReorder(params); + } + } catch (mkldnn::error& e) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Status: ", e.status, ", message: ", e.message.c_str()); + } + + return Status::OK(); +} + +} // namespace mkl_dnn +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/nn/pool.h b/onnxruntime/core/providers/mkldnn/nn/pool.h new file mode 100644 index 0000000000000..bf8fc5f802733 --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/nn/pool.h @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/framework/op_kernel.h" +#include "core/providers/cpu/nn/pool.h" + +namespace onnxruntime { +namespace mkl_dnn { + +template +class Pool final : public onnxruntime::Pool { + public: + Pool(const OpKernelInfo& info) : onnxruntime::Pool(info) { + // Since there are multiple versions of Pooling kernels, we need to use + // the opset version as part of the key for caching Pooling Primitives. + int start, end; + OpKernel::KernelDef().SinceVersion(&start, &end); + opset_version_ = std::to_string(start); + } + + Status Compute(OpKernelContext* context) const override; + + private: + std::string opset_version_; +}; + +} // namespace mkl_dnn +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/symbols.txt b/onnxruntime/core/providers/mkldnn/symbols.txt new file mode 100644 index 0000000000000..c3e14d5b73889 --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/symbols.txt @@ -0,0 +1 @@ +ONNXRuntimeCreateMkldnnExecutionProviderFactory diff --git a/onnxruntime/core/session/CustomOpsLoader.cc b/onnxruntime/core/session/CustomOpsLoader.cc new file mode 100644 index 0000000000000..5d5dc9c5a848a --- /dev/null +++ b/onnxruntime/core/session/CustomOpsLoader.cc @@ -0,0 +1,136 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/CustomOpsLoader.h" + +#include "core/framework/custom_ops_author.h" +#include "core/platform/env.h" +#include "core/common/logging/logging.h" +#include "core/common/logging/severity.h" +#include + +using namespace ::onnxruntime::common; +using namespace ::onnxruntime::logging; + +namespace onnxruntime { +CustomOpsLoader::~CustomOpsLoader() { + using FreeKernelsContainerFn = void (*)(KernelsContainer*); + using FreeSchemasContainerFn = void (*)(SchemasContainer*); + + try { + for (auto& elem : dso_name_data_map_) { + LOGS_DEFAULT(INFO) << "Unloading DSO " << elem.first; + + // free memory + void* lib_handle = elem.second.lib_handle; + if (!lib_handle) { + continue; + } + + // free the kernels container + if (elem.second.kernels_container) { + void* free_all_kernels_symbol_handle = nullptr; + Env::Default().GetSymbolFromLibrary(lib_handle, + kFreeKernelsContainerSymbol, + &free_all_kernels_symbol_handle); + if (!free_all_kernels_symbol_handle) { + LOGS_DEFAULT(WARNING) << "Got nullptr for " + kFreeKernelsContainerSymbol + " for DSO " + elem.first; + } else { + FreeKernelsContainerFn free_all_kernels_fn = reinterpret_cast(free_all_kernels_symbol_handle); + free_all_kernels_fn(elem.second.kernels_container); + } + } + + // free the schemas container + if (elem.second.schemas_container) { + void* free_all_schemas_symbol_handle = nullptr; + Env::Default().GetSymbolFromLibrary(lib_handle, + kFreeSchemasContainerSymbol, + &free_all_schemas_symbol_handle); + + if (!free_all_schemas_symbol_handle) { + LOGS_DEFAULT(WARNING) << "Got nullptr for " + kFreeSchemasContainerSymbol + " for DSO " + elem.first; + } else { + FreeSchemasContainerFn free_all_schemas_fn = reinterpret_cast(free_all_schemas_symbol_handle); + free_all_schemas_fn(elem.second.schemas_container); + } + } + + // unload the DSO + if (!Env::Default().UnloadDynamicLibrary(lib_handle).IsOK()) { + LOGS_DEFAULT(WARNING) << "Failed to unload DSO: " << elem.first; + } + } + } catch (std::exception& ex) { // make sure exceptions don't leave the destructor + LOGS_DEFAULT(WARNING) << "Caught exception while destructing CustomOpsLoader with message: " << ex.what(); + } +} + +Status CustomOpsLoader::LoadCustomOps(const std::string& dso_file_path, + std::shared_ptr& custom_registry) { + try { + if (dso_name_data_map_.count(dso_file_path)) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "A dso with name " + dso_file_path + " has already been loaded."); + } + + using GetAllKernelsFn = KernelsContainer* (*)(); + using GetAllSchemasFn = SchemasContainer* (*)(); + void* lib_handle = nullptr; + ONNXRUNTIME_RETURN_IF_ERROR(Env::Default().LoadDynamicLibrary(dso_file_path, &lib_handle)); + dso_name_data_map_[dso_file_path].lib_handle = lib_handle; + + // get symbol for GetAllKernels + void* get_all_kernels_symbol_handle = nullptr; + ONNXRUNTIME_RETURN_IF_ERROR(Env::Default().GetSymbolFromLibrary(lib_handle, + kGetAllKernelsSymbol, + &get_all_kernels_symbol_handle)); + if (!get_all_kernels_symbol_handle) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, + "Got null handle for " + kGetAllKernelsSymbol + " for DSO " + dso_file_path); + } + + GetAllKernelsFn get_all_kernels_fn = reinterpret_cast(get_all_kernels_symbol_handle); + KernelsContainer* kernels_container = get_all_kernels_fn(); + if (!kernels_container) { + LOGS_DEFAULT(WARNING) << "Got nullptr for KernelsContainer from the custom op library " << dso_file_path; + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Got nullptr for KernelsContainer from the custom op library " + dso_file_path); + } + dso_name_data_map_[dso_file_path].kernels_container = kernels_container; + + // register the kernels + custom_registry.reset(); + custom_registry = std::make_shared(); + + for (auto& i : kernels_container->kernels_list) { + ONNXRUNTIME_RETURN_IF_ERROR(custom_registry->RegisterCustomKernel(i)); + } + + // get symbol for GetAllSchemas + void* get_all_schemas_symbol_handle = nullptr; + ONNXRUNTIME_RETURN_IF_ERROR(Env::Default().GetSymbolFromLibrary(lib_handle, + kGetAllSchemasSymbol, + &get_all_schemas_symbol_handle)); + + if (!get_all_schemas_symbol_handle) { // a custom schema may not be registered + return Status::OK(); + } + + GetAllSchemasFn get_all_schemas_fn = reinterpret_cast(get_all_schemas_symbol_handle); + SchemasContainer* schemas_container = get_all_schemas_fn(); + if (!schemas_container) { + LOGS_DEFAULT(WARNING) << "Got nullptr for SchemasContainer from the custom op library " << dso_file_path; + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Got nullptr for SchemasContainer from the custom op library " + dso_file_path); + } + dso_name_data_map_[dso_file_path].schemas_container = schemas_container; + + // register the schemas if present + ONNXRUNTIME_RETURN_IF_ERROR(custom_registry->RegisterOpSet(schemas_container->schemas_list, + schemas_container->domain, + schemas_container->baseline_opset_version, + schemas_container->opset_version)); + return Status::OK(); + } catch (const std::exception& ex) { + return Status(ONNXRUNTIME, FAIL, "Caught exception while loading custom ops with message: " + std::string(ex.what())); + } +} +} // namespace onnxruntime diff --git a/onnxruntime/core/session/CustomOpsLoader.h b/onnxruntime/core/session/CustomOpsLoader.h new file mode 100644 index 0000000000000..db9e790cf950c --- /dev/null +++ b/onnxruntime/core/session/CustomOpsLoader.h @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include + +#include "core/common/common.h" +#include "core/common/status.h" +#include "core/framework/customregistry.h" +#include "core/framework/custom_ops_author.h" + +namespace onnxruntime { +class CustomOpsLoader final { + public: + CustomOpsLoader() = default; + common::Status LoadCustomOps(const std::string& dso_file_path, + std::shared_ptr& custom_registry); + ~CustomOpsLoader(); + + private: + const std::string kGetAllKernelsSymbol = "GetAllKernels"; + const std::string kGetAllSchemasSymbol = "GetAllSchemas"; + const std::string kFreeKernelsContainerSymbol = "FreeKernelsContainer"; + const std::string kFreeSchemasContainerSymbol = "FreeSchemasContainer"; + + struct DsoData { + void* lib_handle = nullptr; + KernelsContainer* kernels_container = nullptr; + SchemasContainer* schemas_container = nullptr; + }; + std::map dso_name_data_map_; + + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(CustomOpsLoader); +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/session/IOBinding.cc b/onnxruntime/core/session/IOBinding.cc new file mode 100644 index 0000000000000..1387ab045f014 --- /dev/null +++ b/onnxruntime/core/session/IOBinding.cc @@ -0,0 +1,190 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/IOBinding.h" +#include "core/common/logging/logging.h" +#include "core/framework/session_state.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { +IOBinding::IOBinding(const SessionState& session_state) : session_state_(session_state) { +} + +common::Status IOBinding::BindInput(const std::string& name, const MLValue& ml_value) { + if (!ml_value.IsTensor()) { + feeds_[name] = ml_value; + return Status::OK(); + } + + MLValue new_mlvalue; + ONNXRUNTIME_RETURN_IF_ERROR(CopyOneInputAcrossDevices(session_state_, name, ml_value, new_mlvalue)); + feeds_[name] = new_mlvalue; + return Status::OK(); +} + +static common::Status AllocateHelper(const SessionState& session_state, + int id, onnxruntime::ProviderType provider_type, + const MLValue& fetched_mlvalue, + MLValue& output_mlvalue) { + auto* p_provider = session_state.GetExecutionProviders().Get(provider_type); + ONNXRUNTIME_ENFORCE(p_provider); + auto allocator = p_provider->GetAllocator(id, ONNXRuntimeMemTypeDefault); + ONNXRUNTIME_ENFORCE(allocator != nullptr); + auto& fetched_tensor = fetched_mlvalue.Get(); + void* buffer = allocator->Alloc(fetched_tensor.Size()); + ONNXRUNTIME_ENFORCE(buffer); + std::unique_ptr p_tensor = std::make_unique(fetched_tensor.DataType(), + fetched_tensor.Shape(), + buffer, + allocator->Info(), + allocator); + output_mlvalue.Init(p_tensor.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); + + return Status::OK(); +} + +// TODO should we handle the case of one input name feeding 2 nodes placed on different +// devices. +common::Status IOBinding::CopyOneInputAcrossDevices(const SessionState& session_state, + const std::string& input_name, + const MLValue& orig_mlvalue, + MLValue& new_mlvalue) { + //TODO: make it configurable + const int target_device_id = 0; + std::vector node_info_vec; + ONNXRUNTIME_RETURN_IF_ERROR(session_state.GetInputNodeInfo(input_name, node_info_vec)); + + for (auto& node_info : node_info_vec) { + size_t index = node_info.index; + auto& node = *node_info.p_node; + const KernelCreateInfo* kci = node_info.kci; + const auto* node_input_mem_types = (kci != nullptr) ? &kci->kernel_def->InputMemoryType() : nullptr; + + // node may declare input_mem_type to be on CPU explicitly + bool node_input_on_cpu = node_input_mem_types && MemTypeOnCpuExplicitly(*node_input_mem_types, index); + auto& required_provider_type = node_input_on_cpu ? onnxruntime::kCpuExecutionProvider : node.GetExecutionProviderType(); + if (!orig_mlvalue.IsTensor()) { + // copying not supported for non-tensor types + new_mlvalue = orig_mlvalue; + return Status::OK(); + } + auto& input_tensor = orig_mlvalue.Get(); + auto& input_tensor_loc = input_tensor.Location(); + auto& exec_providers = session_state.GetExecutionProviders(); + + auto* p_input_provider = exec_providers.Get(input_tensor_loc); + if (!p_input_provider) { + p_input_provider = exec_providers.Get(onnxruntime::kCpuExecutionProvider); + ONNXRUNTIME_ENFORCE(p_input_provider); + } + + auto input_provider_type = p_input_provider->Type(); + if (input_provider_type == required_provider_type && input_tensor_loc.mem_type == ONNXRuntimeMemTypeDefault) { + new_mlvalue = orig_mlvalue; + return Status::OK(); + } + + //If node require input on cpu and input tensor is allocated with pinned memory allocator, don't do copy + if (node_input_on_cpu && (input_tensor_loc.mem_type == ONNXRuntimeMemTypeCPU || input_tensor_loc.mem_type == ONNXRuntimeMemTypeCPUOutput)) { + new_mlvalue = orig_mlvalue; + return Status::OK(); + } + + auto* node_provider = exec_providers.Get(required_provider_type); + ONNXRUNTIME_ENFORCE(node_provider); + ONNXRUNTIME_RETURN_IF_ERROR(AllocateHelper(session_state, target_device_id, required_provider_type, orig_mlvalue, new_mlvalue)); + auto* new_tensor = new_mlvalue.GetMutable(); + auto* node_exec_provider = exec_providers.Get(required_provider_type); + ONNXRUNTIME_ENFORCE(node_exec_provider); + + // our CPU exec provider doesn't support copy from GPU->CPU + if (required_provider_type != onnxruntime::kCpuExecutionProvider) { + ONNXRUNTIME_RETURN_IF_ERROR(node_exec_provider->CopyTensor(input_tensor, *new_tensor)); + } else { + ONNXRUNTIME_RETURN_IF_ERROR(p_input_provider->CopyTensor(input_tensor, *new_tensor)); + } + } + + return Status::OK(); +} + +static common::Status SyncProviders(const SessionState::NameNodeInfoMapType& node_info_map, + const SessionState& session_state) { + std::set providers; + for (auto& pair : node_info_map) { + for (auto& node_info : pair.second) { + if (node_info.p_node->GetExecutionProviderType() != onnxruntime::kCpuExecutionProvider) { + providers.insert(node_info.p_node->GetExecutionProviderType()); + } + } + } + for (auto& provider_type : providers) { + auto* p_provider = session_state.GetExecutionProviders().Get(provider_type); + if (!p_provider) { + continue; + } + + ONNXRUNTIME_RETURN_IF_ERROR(p_provider->Sync()); + } + return Status::OK(); +} + +common::Status IOBinding::SynchronizeInputs() { + ONNXRUNTIME_RETURN_IF_ERROR(SyncProviders(session_state_.GetInputNodeInfoMap(), session_state_)); + return Status::OK(); +} + +common::Status IOBinding::SynchronizeOutputs() { + ONNXRUNTIME_RETURN_IF_ERROR(SyncProviders(session_state_.GetOutputNodeInfoMap(), session_state_)); + return Status::OK(); +} + +static std::pair Contains(const std::vector& output_names, const std::string& oname) { + auto it = std::find(std::begin(output_names), std::end(output_names), oname); + if (it == std::end(output_names)) { + return {false, 0}; + } + return {true, it - std::begin(output_names)}; +} + +common::Status IOBinding::BindOutput(const std::string& name, const MLValue& ml_value) { + auto rc = Contains(output_names_, name); + if (rc.first) { + outputs_[rc.second] = ml_value; + return Status::OK(); + } + + output_names_.push_back(name); + outputs_.push_back(ml_value); + return Status::OK(); +} + +const std::vector& IOBinding::GetOutputNames() const { + return output_names_; +} + +std::vector& IOBinding::GetOutputs() { + return outputs_; +} + +const std::unordered_map& IOBinding::GetInputs() const { + return feeds_; +} + +AllocatorPtr IOBinding::GetCPUAllocator(int id, onnxruntime::ProviderType provider_type) const { + auto& exec_providers = session_state_.GetExecutionProviders(); + auto* p_provider = exec_providers.Get(provider_type); + ONNXRUNTIME_ENFORCE(p_provider); + auto allocator = p_provider->GetAllocator(id, ONNXRuntimeMemTypeCPU); + + // if the provider does not implement CPU allocator, fall back to CPU + if (allocator) + return allocator; + + auto* cpu_provider = exec_providers.Get(onnxruntime::kCpuExecutionProvider); + return cpu_provider->GetAllocator(0, ONNXRuntimeMemTypeDefault); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/session/IOBinding.h b/onnxruntime/core/session/IOBinding.h new file mode 100644 index 0000000000000..c927a76b7a707 --- /dev/null +++ b/onnxruntime/core/session/IOBinding.h @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include +#include + +#include "core/framework/execution_provider.h" +#include "core/common/status.h" +#include "core/graph/basic_types.h" +#include "core/framework/ml_value.h" +#include "core/session/inference_session.h" +#include "core/common/logging/logging.h" + +namespace onnxruntime { +class SessionState; +/** + * Input/Output binding. + * Usage is as follows: + * + * InferenceSession session; + * session.Load(); + * session.Initialize(); + * ... + * shared_ptr io_binding; + * session.NewIOBinding("DML", &io_binding); + * io_binding->BindInput(...); + * io_binding->BindInput(...); + * io_binding->SynchronizeInputs(); + * + * io_binding->BindOutput(...); + * io_binding->BindOutput(...); + * + * session.Run(io_binding); + * + * vector& outputs = io_binding->GetOutputs(); + */ +class IOBinding { + public: + /** + * Call repeatedly to bind as many inputs as required. + * If the input mlvalue is not at the desired location (specified by the execution provider), this will + * copy it to the desired location. This copy may or may not be async. It depends on the exec provider. + * If the input mlvalue is not at the desired location, it should be preallocated + * If the input mlvalue isn't preallocated, it should have memtype of ONNXRuntimeMemTypeDefault + * For copying it leverages IExecutionProvider::CopyTensor(). + */ + common::Status BindInput(const std::string& name, const MLValue& ml_value); + + /** + * If the BindInput calls are async this function acts as a barrier to ensure all inputs are fully copied + * before you call the Run() method. There is no point calling Run() if you're inputs are not ready at the + * desired location. + * This is a blocking call and is a wrapper over IExecutionProvider::Sync(). + * Call InferenceSession::Run() only after calling this method or else you'll end up wasting cycles inside Run(). + */ + common::Status SynchronizeInputs(); + common::Status SynchronizeOutputs(); + /** + * This simply provides the names and optionally allocated output containers. + */ + common::Status BindOutput(const std::string& name, const MLValue& ml_value); + + /** + * This simply collects the outputs obtained after calling Run() inside the @param outputs. + */ + const std::vector& GetOutputNames() const; + std::vector& GetOutputs(); + + const std::unordered_map& GetInputs() const; + + /** + * Get a CPU allocator from provider for async copy later if the provider supports that + * If it doesn't support that, return the default allocator from CPU provider + * \return a nonnull pointer + */ + AllocatorPtr GetCPUAllocator(int id, onnxruntime::ProviderType provider_type) const; + + private: + friend InferenceSession; + + IOBinding(const SessionState& session_state); + const SessionState& session_state_; + std::unordered_map feeds_; + std::vector output_names_; + std::vector outputs_; + + static common::Status CopyOneInputAcrossDevices(const SessionState& session_state, + const std::string& input_name, + const MLValue& orig_mlvalue, + MLValue& new_mlvalue); + + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(IOBinding); +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/session/abi_session_options.cc b/onnxruntime/core/session/abi_session_options.cc new file mode 100644 index 0000000000000..bed078ec5faa6 --- /dev/null +++ b/onnxruntime/core/session/abi_session_options.cc @@ -0,0 +1,122 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/onnxruntime_c_api.h" +#include +#include "core/session/inference_session.h" +#include "abi_session_options_impl.h" + +uint32_t ONNXRUNTIME_API_STATUSCALL ReleaseCuda(void* this_) { + ONNXRuntimeSessionOptions* this_ptr = static_cast(this_); + if (--this_ptr->ref_count == 0) + delete this_ptr; + return 0; +} + +uint32_t ONNXRUNTIME_API_STATUSCALL AddRefCuda(void* this_) { + ONNXRuntimeSessionOptions* this_ptr = static_cast(this_); + ++this_ptr->ref_count; + return 0; +} + +constexpr ONNXObject mkl_cls = { + AddRefCuda, + ReleaseCuda, +}; + +ONNXRuntimeSessionOptions::ONNXRuntimeSessionOptions() : cls(&mkl_cls), ref_count(1) { +} + +ONNXRuntimeSessionOptions::~ONNXRuntimeSessionOptions() { + for (ONNXRuntimeProviderFactoryPtr* p : provider_factories) { + ONNXRuntimeReleaseObject(p); + } +} + +ONNXRuntimeSessionOptions& ONNXRuntimeSessionOptions::operator=(const ONNXRuntimeSessionOptions&) { + throw std::runtime_error("not implemented"); +} +ONNXRuntimeSessionOptions::ONNXRuntimeSessionOptions(const ONNXRuntimeSessionOptions& other) + : cls(&mkl_cls), ref_count(1), value(other.value), custom_op_paths(other.custom_op_paths), provider_factories(other.provider_factories) { + for (ONNXRuntimeProviderFactoryPtr* p : other.provider_factories) { + ONNXRuntimeAddRefToObject(p); + } +} +ONNXRUNTIME_API(ONNXRuntimeSessionOptions*, ONNXRuntimeCreateSessionOptions) { + std::unique_ptr options = std::make_unique(); + return options.release(); +} + +ONNXRUNTIME_API(ONNXRuntimeSessionOptions*, ONNXRuntimeCloneSessionOptions, ONNXRuntimeSessionOptions* input) { + try { + return new ONNXRuntimeSessionOptions(*input); + } catch (std::exception&) { + return nullptr; + } +} + +ONNXRUNTIME_API(void, ONNXRuntimeSessionOptionsAppendExecutionProvider, _In_ ONNXRuntimeSessionOptions* options, _In_ ONNXRuntimeProviderFactoryPtr* f) { + ONNXRuntimeAddRefToObject(f); + options->provider_factories.push_back(f); +} + +ONNXRUNTIME_API(void, ONNXRuntimeEnableSequentialExecution, _In_ ONNXRuntimeSessionOptions* options) { + options->value.enable_sequential_execution = true; +} +ONNXRUNTIME_API(void, ONNXRuntimeDisableSequentialExecution, _In_ ONNXRuntimeSessionOptions* options) { + options->value.enable_sequential_execution = false; +} + +// enable profiling for this session. +ONNXRUNTIME_API(void, ONNXRuntimeEnableProfiling, _In_ ONNXRuntimeSessionOptions* options, _In_ const char* profile_file_prefix) { + options->value.enable_profiling = true; + options->value.profile_file_prefix = profile_file_prefix; +} +ONNXRUNTIME_API(void, ONNXRuntimeDisableProfiling, _In_ ONNXRuntimeSessionOptions* options) { + options->value.enable_profiling = false; + options->value.profile_file_prefix.clear(); +} + +// enable the memory pattern optimization. +// The idea is if the input shapes are the same, we could trace the internal memory allocation +// and generate a memory pattern for future request. So next time we could just do one allocation +// with a big chunk for all the internal memory allocation. +ONNXRUNTIME_API(void, ONNXRuntimeEnableMemPattern, _In_ ONNXRuntimeSessionOptions* options) { + options->value.enable_mem_pattern = true; +} +ONNXRUNTIME_API(void, ONNXRuntimeDisableMemPattern, _In_ ONNXRuntimeSessionOptions* options) { + options->value.enable_mem_pattern = false; +} + +// enable the memory arena on CPU +// Arena may pre-allocate memory for future usage. +// set this option to false if you don't want it. +ONNXRUNTIME_API(void, ONNXRuntimeEnableCpuMemArena, _In_ ONNXRuntimeSessionOptions* options) { + options->value.enable_cpu_mem_arena = true; +} + +ONNXRUNTIME_API(void, ONNXRuntimeDisableCpuMemArena, _In_ ONNXRuntimeSessionOptions* options) { + options->value.enable_cpu_mem_arena = false; +} + +///< logger id to use for session output +ONNXRUNTIME_API(void, ONNXRuntimeSetSessionLogId, _In_ ONNXRuntimeSessionOptions* options, const char* logid) { + options->value.session_logid = logid; +} + +///< applies to session load, initialization, etc +ONNXRUNTIME_API(void, ONNXRuntimeSetSessionLogVerbosityLevel, _In_ ONNXRuntimeSessionOptions* options, uint32_t session_log_verbosity_level) { + options->value.session_log_verbosity_level = session_log_verbosity_level; +} + +///How many threads in the session thread pool. +ONNXRUNTIME_API(int, ONNXRuntimeSetSessionThreadPoolSize, _In_ ONNXRuntimeSessionOptions* options, int session_thread_pool_size) { + if (session_thread_pool_size <= 0) return -1; + options->value.session_thread_pool_size = session_thread_pool_size; + return 0; +} + + +ONNXRUNTIME_API(void, ONNXRuntimeAddCustomOp, _In_ ONNXRuntimeSessionOptions* options, const char* custom_op_path) { + options->custom_op_paths.emplace_back(custom_op_path); +} \ No newline at end of file diff --git a/onnxruntime/core/session/abi_session_options_impl.h b/onnxruntime/core/session/abi_session_options_impl.h new file mode 100644 index 0000000000000..ea0ba68ce14d1 --- /dev/null +++ b/onnxruntime/core/session/abi_session_options_impl.h @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include +#include "core/session/inference_session.h" +#include "core/session/onnxruntime_c_api.h" + +struct ONNXRuntimeSessionOptions { + const ONNXObject* const cls; + std::atomic_int ref_count; + onnxruntime::SessionOptions value; + std::vector custom_op_paths; + std::vector provider_factories; + ONNXRuntimeSessionOptions(); + ~ONNXRuntimeSessionOptions(); + ONNXRuntimeSessionOptions(const ONNXRuntimeSessionOptions& other); + ONNXRuntimeSessionOptions& operator=(const ONNXRuntimeSessionOptions& other); +}; \ No newline at end of file diff --git a/onnxruntime/core/session/allocator_impl.h b/onnxruntime/core/session/allocator_impl.h new file mode 100644 index 0000000000000..12169ecf0dd77 --- /dev/null +++ b/onnxruntime/core/session/allocator_impl.h @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/session/allocator.h" +#include "core/framework/allocator.h" + +namespace onnxruntime { +class AllocatorWrapper : public IAllocator { + public: + AllocatorWrapper(ONNXRuntimeAllocator* impl) : impl_(impl) { + (*impl)->parent.AddRef(impl); + } + ~AllocatorWrapper() { + (*impl_)->parent.Release(impl_); + } + void* Alloc(size_t size) override { + return (*impl_)->Alloc(impl_, size); + } + void Free(void* p) override { + return (*impl_)->Free(impl_, p); + } + const ONNXRuntimeAllocatorInfo& Info() const override { + return *(ONNXRuntimeAllocatorInfo*)(*impl_)->Info(impl_); + } + + private: + ONNXRuntimeAllocator* impl_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/session/default_cpu_allocator_c_api.cc b/onnxruntime/core/session/default_cpu_allocator_c_api.cc new file mode 100644 index 0000000000000..1f1418e486c99 --- /dev/null +++ b/onnxruntime/core/session/default_cpu_allocator_c_api.cc @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/allocator.h" +#include +#include +#include "core/framework/allocator_info.h" +#include "core/session/onnxruntime_cxx_api.h" +#include + +#define ONNXRUNTIME_ALLOCATOR_IMPL_BEGIN(CLASS_NAME) \ + class CLASS_NAME { \ + private: \ + const ONNXRuntimeAllocatorInteface* vtable_ = &table_; \ + std::atomic_int ref_count_; \ + static void* ONNXRUNTIME_API_STATUSCALL Alloc_(void* this_ptr, size_t size) { \ + return ((CLASS_NAME*)this_ptr)->Alloc(size); \ + } \ + static void ONNXRUNTIME_API_STATUSCALL Free_(void* this_ptr, void* p) { \ + return ((CLASS_NAME*)this_ptr)->Free(p); \ + } \ + static const ONNXRuntimeAllocatorInfo* ONNXRUNTIME_API_STATUSCALL Info_(const void* this_ptr) { \ + return ((const CLASS_NAME*)this_ptr)->Info(); \ + } \ + static uint32_t ONNXRUNTIME_API_STATUSCALL AddRef_(void* this_) { \ + CLASS_NAME* this_ptr = (CLASS_NAME*)this_; \ + return ++this_ptr->ref_count_; \ + } \ + static uint32_t ONNXRUNTIME_API_STATUSCALL Release_(void* this_) { \ + CLASS_NAME* this_ptr = (CLASS_NAME*)this_; \ + uint32_t ret = --this_ptr->ref_count_; \ + if (ret == 0) \ + delete this_ptr; \ + return 0; \ + } \ + static ONNXRuntimeAllocatorInteface table_; + +#define ONNXRUNTIME_ALLOCATOR_IMPL_END \ + } \ + ; + +ONNXRUNTIME_ALLOCATOR_IMPL_BEGIN(ONNXRuntimeDefaultAllocator) +private: +ONNXRuntimeAllocatorInfo* cpuAllocatorInfo; +ONNXRuntimeDefaultAllocator() : ref_count_(1){ + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateAllocatorInfo("Cpu", ONNXRuntimeDeviceAllocator, 0, ONNXRuntimeMemTypeDefault, &cpuAllocatorInfo)); +} +~ONNXRuntimeDefaultAllocator() { + assert(ref_count_ == 0); + ReleaseONNXRuntimeAllocatorInfo(cpuAllocatorInfo); +} + +public: +ONNXRuntimeDefaultAllocator(const ONNXRuntimeDefaultAllocator&) = delete; +ONNXRuntimeDefaultAllocator& operator=(const ONNXRuntimeDefaultAllocator&) = delete; +ONNXRuntimeAllocatorInteface** Upcast() { + return const_cast(&vtable_); +} +static ONNXRuntimeAllocatorInteface** Create() { + return (ONNXRuntimeAllocatorInteface**)new ONNXRuntimeDefaultAllocator(); +} +void* Alloc(size_t size) { + return ::malloc(size); +} +void Free(void* p) { + return ::free(p); +} +const ONNXRuntimeAllocatorInfo* Info() const { + return cpuAllocatorInfo; +} +ONNXRUNTIME_ALLOCATOR_IMPL_END + +#define API_IMPL_BEGIN try { +#define API_IMPL_END \ + } \ + catch (std::exception & ex) { \ + return CreateONNXStatus(ONNXRUNTIME_RUNTIME_EXCEPTION, ex.what()); \ + } + +ONNXRuntimeAllocatorInteface ONNXRuntimeDefaultAllocator::table_ = { + {ONNXRuntimeDefaultAllocator::AddRef_, ONNXRuntimeDefaultAllocator::Release_}, ONNXRuntimeDefaultAllocator::Alloc_, ONNXRuntimeDefaultAllocator::Free_, ONNXRuntimeDefaultAllocator::Info_}; + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeCreateDefaultAllocator, _Out_ ONNXRuntimeAllocator** out){ + API_IMPL_BEGIN + *out = ONNXRuntimeDefaultAllocator::Create(); + return nullptr; + API_IMPL_END +} \ No newline at end of file diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc new file mode 100644 index 0000000000000..31b1400d25cd8 --- /dev/null +++ b/onnxruntime/core/session/inference_session.cc @@ -0,0 +1,1156 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/inference_session.h" + +#include +#include +#include +#include +#include + +#include "core/common/logging/logging.h" +#include "core/common/task_thread_pool.h" +#include "core/graph/graph.h" +#include "core/graph/graph_transformer.h" +#include "core/graph/graph_transformer_mgr.h" +#include "core/graph/model.h" +#include "core/framework/allocatormgr.h" +#include "core/framework/customregistry.h" +#include "core/framework/execution_frame.h" +#include "core/framework/graph_partitioner.h" +#include "core/framework/insert_cast_transformer.h" +#include "core/framework/kernel_def_builder.h" +#include "core/framework/kernel_registry.h" +#include "core/framework/ml_value_patterns_planner.h" +#include "core/framework/mldata_type_utils.h" +#include "core/framework/mlvalue_name_idx_map.h" +#include "core/framework/sequential_executor.h" +#include "core/framework/parallel_executor.h" +#include "core/framework/session_state.h" +#include "core/framework/session_state_initializer.h" +#include "core/framework/tensorprotoutils.h" +#include "core/framework/tensorutils.h" +#include "core/framework/transformer_memcpy.h" +#include "core/framework/utils.h" +#include "core/platform/notification.h" +#include "core/providers/cpu/cpu_execution_provider.h" +#include "core/session/CustomOpsLoader.h" +#include "core/session/IOBinding.h" + +using namespace ONNX_NAMESPACE; + +namespace onnxruntime { + +class InferenceSession::Impl { + public: + Impl(const SessionOptions& session_options, logging::LoggingManager* logging_manager) + : session_options_{session_options}, + graph_transformation_mgr_{session_options_.max_num_graph_transformation_steps}, + logging_manager_{logging_manager}, + session_state_{execution_providers_}, + insert_cast_transformer_{"CastFloat16Transformer"} { + InitLogger(logging_manager); + + // currently the threadpool is used by the parallel executor only and hence + // there is no point creating it when only sequential execution is enabled. + if (!session_options.enable_sequential_execution) { + int pool_size = session_options_.session_thread_pool_size == 0 + ? std::thread::hardware_concurrency() / 2 + : session_options_.session_thread_pool_size; + thread_pool_ = std::make_unique(pool_size); + } + + session_state_.SetThreadPool(thread_pool_.get()); + session_state_.SetEnableMemoryPattern(session_options.enable_mem_pattern); + session_profiler_.Initialize(session_logger_); + session_state_.SetProfiler(session_profiler_); + if (session_options.enable_profiling) { + StartProfiling(session_options.profile_file_prefix); + } + } + + common::Status RegisterExecutionProvider(std::unique_ptr p_exec_provider) { + if (p_exec_provider == nullptr) { + return Status(common::ONNXRUNTIME, common::FAIL, "Received nullptr for exec provider"); + } + + std::string provider_type = p_exec_provider->Type(); + VLOGS(*session_logger_, 1) << "Adding execution provider of type: " << provider_type; + execution_providers_.Add(provider_type, std::move(p_exec_provider)); + + return Status::OK(); + } + + common::Status RegisterGraphTransformer(std::unique_ptr p_graph_transformer) { + if (p_graph_transformer == nullptr) { + return Status(common::ONNXRUNTIME, common::FAIL, "Received nullptr for graph transformer"); + } + return graph_transformation_mgr_.Register(std::move(p_graph_transformer)); + } + + common::Status LoadCustomOps(const std::vector& dso_list) { + if (dso_list.empty()) { + return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Empty list of shared libraries in the input."); + } + for (auto& dso_file_path : dso_list) { + std::shared_ptr custom_registry; + ONNXRUNTIME_RETURN_IF_ERROR(custom_ops_loader_.LoadCustomOps(dso_file_path, custom_registry)); + if (!custom_registry) { + return Status(common::ONNXRUNTIME, common::FAIL, "Null custom_registry after loading custom ops."); + } + ONNXRUNTIME_RETURN_IF_ERROR(RegisterCustomRegistry(custom_registry)); + } + return Status::OK(); + } + + common::Status RegisterCustomRegistry(std::shared_ptr& custom_registry) { + if (custom_registry == nullptr) { + return Status(common::ONNXRUNTIME, common::FAIL, "Received nullptr for custom registry"); + } + + // Insert session-level customized kernel registry. + kernel_registry_manager_.RegisterKernelRegistry(custom_registry, KernelRegistryPriority::HighPriority); + custom_schema_registries_.push_back(custom_registry); + return Status::OK(); + } + + template + common::Status Load(const T& model_uri) { + auto tp = session_profiler_.StartTime(); + try { + std::lock_guard l(session_mutex_); + if (is_model_loaded_) { // already loaded + LOGS(*session_logger_, ERROR) << "This session already contains a loaded model."; + return common::Status(common::ONNXRUNTIME, common::MODEL_LOADED, "This session already contains a loaded model."); + } + + std::shared_ptr p_tmp_model; + ONNXRUNTIME_RETURN_IF_ERROR(onnxruntime::Model::Load(model_uri, p_tmp_model, + HasLocalSchema() ? &custom_schema_registries_ : nullptr)); + model_ = p_tmp_model; + + ONNXRUNTIME_RETURN_IF_ERROR(DoPostLoadProcessing(*model_.get())); + + // all steps complete, mark the model as loaded. + is_model_loaded_ = true; + } catch (const std::exception& ex) { + return Status(common::ONNXRUNTIME, common::FAIL, "Exception during loading: " + std::string(ex.what())); + } catch (...) { + LOGS(*session_logger_, ERROR) << "Unknown exception in Load()"; + return Status(common::ONNXRUNTIME, common::RUNTIME_EXCEPTION, "Encountered unknown exception in Load()"); + } + session_profiler_.EndTimeAndRecordEvent(profiling::SESSION_EVENT, "model_loading_uri", tp); + return common::Status::OK(); + } + + common::Status Load(const ModelProto& model_proto) { + auto tp = session_profiler_.StartTime(); + try { + LOGS(*session_logger_, INFO) << "Loading model using model_proto"; + std::lock_guard l(session_mutex_); + if (is_model_loaded_) { // already loaded + LOGS(*session_logger_, ERROR) << "This session already contains a loaded model."; + return common::Status(common::ONNXRUNTIME, common::MODEL_LOADED, "This session already contains a loaded model."); + } + + std::shared_ptr p_tmp_model; + ONNXRUNTIME_RETURN_IF_ERROR(onnxruntime::Model::Load(model_proto, p_tmp_model, + HasLocalSchema() ? &custom_schema_registries_ : nullptr)); + model_ = p_tmp_model; + + ONNXRUNTIME_RETURN_IF_ERROR(DoPostLoadProcessing(*model_.get())); + + // all steps complete, mark the model as loaded. + is_model_loaded_ = true; + + LOGS(*session_logger_, INFO) << "Model successfully loaded."; + } catch (const std::exception& ex) { + return Status(common::ONNXRUNTIME, common::FAIL, "Exception during loading: " + std::string(ex.what())); + } catch (...) { + LOGS(*session_logger_, ERROR) << "Unknown exception in Load()"; + return Status(common::ONNXRUNTIME, common::RUNTIME_EXCEPTION, "Encountered unknown exception in Load()"); + } + session_profiler_.EndTimeAndRecordEvent(profiling::SESSION_EVENT, "model_loading_proto", tp); + return Status::OK(); + } + + common::Status Load(std::unique_ptr p_model_proto) { + auto tp = session_profiler_.StartTime(); + try { + LOGS(*session_logger_, INFO) << "Loading model using model_proto"; + std::lock_guard l(session_mutex_); + if (is_model_loaded_) { // already loaded + LOGS(*session_logger_, ERROR) << "This session already contains a loaded model."; + return common::Status(common::ONNXRUNTIME, common::MODEL_LOADED, "This session already contains a loaded model."); + } + + std::shared_ptr p_tmp_model; + ONNXRUNTIME_RETURN_IF_ERROR(onnxruntime::Model::Load(std::move(p_model_proto), p_tmp_model, + HasLocalSchema() ? &custom_schema_registries_ : nullptr)); + model_ = p_tmp_model; + + ONNXRUNTIME_RETURN_IF_ERROR(DoPostLoadProcessing(*model_.get())); + + // all steps complete, mark the model as loaded. + is_model_loaded_ = true; + + LOGS(*session_logger_, INFO) << "Model successfully loaded."; + } catch (const std::exception& ex) { + return Status(common::ONNXRUNTIME, common::FAIL, "Exception during loading: " + std::string(ex.what())); + } catch (...) { + LOGS(*session_logger_, ERROR) << "Unknown exception in Load()"; + return Status(common::ONNXRUNTIME, common::RUNTIME_EXCEPTION, "Encountered unknown exception in Load()"); + } + session_profiler_.EndTimeAndRecordEvent(profiling::SESSION_EVENT, "model_loading_proto", tp); + return Status::OK(); + } + + common::Status Load(std::istream& model_istream) { + auto tp = session_profiler_.StartTime(); + try { + LOGS(*session_logger_, INFO) << "Loading model using istream"; + std::lock_guard l(session_mutex_); + if (is_model_loaded_) { // already loaded + LOGS(*session_logger_, ERROR) << "This session already contains a loaded model."; + return common::Status(common::ONNXRUNTIME, common::MODEL_LOADED, "This session already contains a loaded model."); + } + + ModelProto model_proto; + const bool result = model_proto.ParseFromIstream(&model_istream); + if (!result) { + return Status(common::ONNXRUNTIME, common::INVALID_PROTOBUF, "Failed to load model because protobuf parsing failed."); + } + + std::shared_ptr p_tmp_model; + ONNXRUNTIME_RETURN_IF_ERROR(onnxruntime::Model::Load(model_proto, p_tmp_model, + HasLocalSchema() ? &custom_schema_registries_ : nullptr)); + model_ = p_tmp_model; + + ONNXRUNTIME_RETURN_IF_ERROR(DoPostLoadProcessing(*model_.get())); + + // all steps complete, mark the model as loaded. + is_model_loaded_ = true; + + LOGS(*session_logger_, INFO) << "Model successfully loaded."; + } catch (const std::exception& ex) { + return Status(common::ONNXRUNTIME, common::FAIL, "Exception during loading: " + std::string(ex.what())); + } catch (...) { + LOGS(*session_logger_, ERROR) << "Unknown exception in Load()"; + return Status(common::ONNXRUNTIME, common::RUNTIME_EXCEPTION, "Encountered unknown exception in Load()"); + } + session_profiler_.EndTimeAndRecordEvent(profiling::SESSION_EVENT, "model_loading_istream", tp); + return common::Status::OK(); + } + + // memory allocations for a subgraph that are owned by InferenceSession + struct SubgraphMemory { + std::unique_ptr session_state; + std::map weights_buffers; + }; + + /// iterate nodes in graph looking for ones with graph attribute/s + /// @param graph The graph to iterate + /// @param session_state The SessionState instance for 'graph'. + /// @remarks We pass in graph and session_state so we can handled nested subgraphs in the future + common::Status InitializeSubgraphSessions(Graph& graph, SessionState& session_state) { + for (auto& node : graph.Nodes()) { + for (auto& attribute : node.GetAttributes()) { + auto& name = attribute.first; + auto& proto = attribute.second; + + // check if it has a subgraph + if (proto.has_g()) { + Graph* subgraph = graph.GetMutableSubgraph(node.Index(), name); + ONNXRUNTIME_ENFORCE(subgraph, "Main Graph instance should have populated all subgraphs when being resolved."); + + SubgraphMemory subgraph_info; + // create SessionState for executing subgraph + subgraph_info.session_state = std::make_unique(execution_providers_); + subgraph_info.session_state->SetProfiler(session_profiler_); + + // setup everything required to execute the subgraph and save it in subgraph_session_state + SessionStateInitializer initializer{*subgraph, *subgraph_info.session_state, + execution_providers_, kernel_registry_manager_, *session_logger_}; + + ONNXRUNTIME_RETURN_IF_ERROR(initializer.CreatePlan(graph_transformation_mgr_, insert_cast_transformer_, + session_options_.enable_sequential_execution)); + + ONNXRUNTIME_RETURN_IF_ERROR(initializer.InitializeAndSave(session_state_.GetEnableMemoryPattern(), + subgraph_info.weights_buffers)); + + // add the subgraph SessionState instance to the parent graph SessionState so it can be retrieved + // by Compute() via OpKernelContextInternal. + session_state.AddSubgraphSessionState(node.Index(), name, *subgraph_info.session_state); + + // save subgraph_info as InferenceSession owns these so they remain valid + // for the entire InferenceSession. + subgraph_memory_.push_back(std::move(subgraph_info)); + } + } + } + + return Status::OK(); + } + + common::Status Initialize() { + Status status = Status::OK(); + auto tp = session_profiler_.StartTime(); + + try { + LOGS(*session_logger_, INFO) << "Initializing session."; + std::lock_guard l(session_mutex_); + if (!is_model_loaded_) { + LOGS(*session_logger_, ERROR) << "Model was not loaded"; + return common::Status(common::ONNXRUNTIME, common::FAIL, "Model was not loaded."); + } + + if (is_inited_) { // already initialized + LOGS(*session_logger_, INFO) << "Session has already been initialized."; + return common::Status::OK(); + } + + // Register default CPUExecutionProvider if user didn't provide it through the Register() calls + if (!execution_providers_.Get(onnxruntime::kCpuExecutionProvider)) { + LOGS(*session_logger_, INFO) << "Adding default CPU execution provider."; + CPUExecutionProviderInfo epi{session_options_.enable_cpu_mem_arena}; + execution_providers_.Add(onnxruntime::kCpuExecutionProvider, + std::make_unique(epi)); + } + + onnxruntime::Graph& graph = model_->MainGraph(); + + // Collect the kernel registries from execution provider instances; + // There are 2 kinds of kernel registries with priority from high to low as below, + // 1. Custom execution provider type specific kernel registries. + // 2. common execution provider type specific kernel registries. + // The 1st and 2nd ones are shared across sessions. + // The 1st ones should have already been registered via session-level API into KernelRegistryManager. + // + // Register 2nd registries into KernelRegistryManager. + kernel_registry_manager_.RegisterKernels(execution_providers_); + + insert_cast_transformer_.AddKernelRegistries(kernel_registry_manager_.GetAllKernelRegistries()); + + SessionStateInitializer session_initializer{graph, session_state_, execution_providers_, + kernel_registry_manager_, *session_logger_}; + + ONNXRUNTIME_RETURN_IF_ERROR(session_initializer.CreatePlan(graph_transformation_mgr_, insert_cast_transformer_, + session_options_.enable_sequential_execution)); + + ONNXRUNTIME_RETURN_IF_ERROR(session_initializer.InitializeAndSave(session_state_.GetEnableMemoryPattern(), + weights_buffers_)); + + // handle any subgraphs + ONNXRUNTIME_RETURN_IF_ERROR(InitializeSubgraphSessions(graph, session_state_)); + + is_inited_ = true; + + LOGS(*session_logger_, INFO) << "Session successfully initialized."; + } catch (const NotImplementedException& ex) { + status = ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, NOT_IMPLEMENTED, "Exception during initialization: ", ex.what()); + LOGS(*session_logger_, ERROR) << status.ErrorMessage(); + } catch (const std::exception& ex) { + status = ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "Exception during initialization: ", ex.what()); + LOGS(*session_logger_, ERROR) << status.ErrorMessage(); + } catch (...) { + status = ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, RUNTIME_EXCEPTION, "Encountered unknown exception in Initialize()"); + LOGS(*session_logger_, ERROR) << status.ErrorMessage(); + } + + session_profiler_.EndTimeAndRecordEvent(profiling::SESSION_EVENT, "session_initialization", tp); + return status; + } + + int GetCurrentNumRuns() const { + return current_num_runs_.load(); + } + + common::Status Run(const NameMLValMap& feeds, + const std::vector& output_names, + std::vector* p_fetches) { + RunOptions run_options; + return Run(run_options, feeds, output_names, p_fetches); + } + + static common::Status CheckTypes(MLDataType actual, MLDataType expected) { + if (actual == expected) { + return Status::OK(); + } + auto actual_name = std::string(typeid(*actual).name()); + auto expected_name = std::string(typeid(*expected).name()); + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, + "Unexpected input data type. Actual: (" + actual_name + ") , expected: (" + expected_name + ")"); + } + + common::Status ValidateInputTypes(const NameMLValMap& feeds) { + for (auto& arg : input_def_list_) { + auto& arg_name = arg->Name(); + if (arg_name.empty() || !feeds.count(arg_name)) { + continue; + } + + auto& input_ml_value = feeds.at(arg_name); + auto input_type = input_ml_value.Type(); + auto expected_type = utils::GetMLDataType(*arg); + + if (!input_ml_value.IsTensor()) { + auto retval = CheckTypes(input_type, expected_type); + if (!retval.IsOK()) { + return retval; + } + continue; + } + + auto expected_element_type = expected_type->AsTensorType()->GetElementType(); + auto input_element_type = input_ml_value.Get().DataType(); + auto retval = CheckTypes(input_element_type, expected_element_type); + if (!retval.IsOK()) { + return retval; + } + } + return Status::OK(); + } + + common::Status ValidateInputNames(const NameMLValMap& feeds) { + if (model_input_names_.size() != feeds.size()) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "The number of feeds is not same as the number of the model input, expect ", + model_input_names_.size(), " got ", feeds.size()); + } + + bool valid = true; + std::ostringstream invalid_names; + for (const auto& pair : feeds) { + if (model_input_names_.find(pair.first) == model_input_names_.end()) { + valid = false; + invalid_names << " " << pair.first; + } + } + + if (!valid) { + std::ostringstream ostr; + std::for_each(std::begin(model_input_names_), + std::end(model_input_names_), + [&ostr](const std::string& elem) { + ostr << elem << " "; + }); + return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, + "Invalid Feed Input Names:" + invalid_names.str() + + " Valid input names are: " + ostr.str()); + } + + return Status::OK(); + } + + common::Status ValidateInputs(const NameMLValMap& feeds) { + ONNXRUNTIME_RETURN_IF_ERROR(ValidateInputNames(feeds)); + ONNXRUNTIME_RETURN_IF_ERROR(ValidateInputTypes(feeds)); + return Status::OK(); + } + + common::Status ValidateOutputs(const std::vector& output_names, + const std::vector* p_fetches) { + if (!p_fetches) { + return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, + "Output vector pointer is NULL"); + } + + if (output_names.empty()) { + return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, + "At least one output should be requested."); + } + + if (!p_fetches->empty() && + (output_names.size() != p_fetches->size())) { + std::ostringstream ostr; + ostr << "Output vector incorrectly sized: output_names.size(): " << output_names.size() + << "p_fetches->size(): " << p_fetches->size(); + return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, ostr.str()); + } + + bool valid = true; + std::ostringstream invalid_names; + for (const auto& name : output_names) { + if (model_output_names_.find(name) == model_output_names_.end()) { + valid = false; + invalid_names << " " << name; + } + } + + if (!valid) { + std::ostringstream ostr; + std::for_each(std::begin(model_output_names_), + std::end(model_output_names_), + [&ostr](const std::string& elem) { + ostr << elem << " "; + }); + return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, + "Invalid Output Names:" + invalid_names.str() + + " Valid output names are: " + ostr.str()); + } + + // TODO add more validation here like checking shape of the allocated buffers + + return common::Status::OK(); + } + + // copies inputs across devices only if required + common::Status CopyInputsAcrossDevices(const SessionState& session_state, + const NameMLValMap& orig_feeds, + NameMLValMap& new_feeds) { + for (auto& pair : orig_feeds) { + MLValue new_mlvalue; + auto& input_name = pair.first; + auto& orig_mlvalue = pair.second; + ONNXRUNTIME_RETURN_IF_ERROR(IOBinding::CopyOneInputAcrossDevices(session_state, + input_name, + orig_mlvalue, + new_mlvalue)); + new_feeds[input_name] = new_mlvalue; + } + return Status::OK(); + } + + // ensures pre-allocated outputs match the node providers. + common::Status MatchOutputsWithProviders(const std::vector& output_names, + std::vector& fetches, + std::vector& new_fetches) { + if (fetches.empty()) { + fetches.resize(output_names.size()); + } + new_fetches.resize(output_names.size()); + + std::set seen_outputs; + auto p_graph = session_state_.GetGraphViewer(); + ONNXRUNTIME_ENFORCE(p_graph); + + std::pair found; + for (auto& node : p_graph->Nodes()) { // TODO optimize this + if (seen_outputs.size() == fetches.size()) { + break; + } + for (auto* arg : node.OutputDefs()) { + if (!arg->Exists() || + arg->Name().empty() || + !(found = Contains(output_names, arg->Name())).first) { + continue; + } + + seen_outputs.insert(arg->Name()); + size_t idx = found.second; + MLValue orig_mlvalue = fetches[idx]; + if (orig_mlvalue.IsAllocated()) { + if (!orig_mlvalue.IsTensor()) { + new_fetches[idx] = fetches[idx]; + continue; + } + + auto& node_provider_type = node.GetExecutionProviderType(); + auto& orig_tensor = orig_mlvalue.Get(); + auto& orig_tensor_loc = orig_tensor.Location(); + auto* tensor_provider = execution_providers_.Get(orig_tensor_loc); + if (!tensor_provider) { + tensor_provider = execution_providers_.Get(onnxruntime::kCpuExecutionProvider); + } + + auto tensor_provider_type = tensor_provider->Type(); + if (node_provider_type == tensor_provider_type) { + new_fetches[idx] = fetches[idx]; + continue; + } + // leave the new_fetches[idx] as it is since it'll get allocated on the appropriate + // provider by the op kernel context when requested. + continue; + + } else { + new_fetches[idx] = fetches[idx]; + continue; + } + } + } + + // If we've already seen all the outputs requested just return. + if (seen_outputs.size() == output_names.size()) { + return Status::OK(); + } + + // Handle the case when a constant is an output but has been folded into a weight + // and hence it doesn't show up in any of the OutputDefs before. + // assume that the weight has already been placed in the appropriate device before + auto& defs = p_graph->GetOutputs(); + auto& mlvalue_name_idx_map{session_state_.GetMLValueNameIdxMap()}; + auto& weights = session_state_.GetInitializedTensors(); + + for (auto& one_def : defs) { + if (!one_def->Exists() || + one_def->Name().empty() || + seen_outputs.count(one_def->Name()) || + !(found = Contains(output_names, one_def->Name())).first) { + continue; + } + + auto& def_name = one_def->Name(); + size_t idx = found.second; + int mlvalue_idx; + ONNXRUNTIME_RETURN_IF_ERROR(mlvalue_name_idx_map.GetIdx(def_name, mlvalue_idx)); + if (!weights.count(mlvalue_idx)) { + LOGS(*session_logger_, INFO) << "Output with name " << def_name << " is not a weight."; + continue; + } + seen_outputs.insert(def_name); + const auto& weight = weights.at(mlvalue_idx); + new_fetches[idx] = weight; + } + + if (seen_outputs.size() != output_names.size()) // make sure we've seen all outputs + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "output size mismatch, expected ", output_names.size(), + " got ", seen_outputs.size()); + + return Status::OK(); + } + + common::Status AllocateHelper(onnxruntime::ProviderType provider_type, + int device_id, + const Tensor& fetched_tensor, + MLValue& output_mlvalue) { + auto* p_provider = execution_providers_.Get(provider_type); + if (!p_provider) + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "invalid provider_type"); + + auto allocator = p_provider->GetAllocator(device_id, ONNXRuntimeMemTypeDefault); + if (!allocator) + return Status(common::ONNXRUNTIME, common::FAIL, "invalid allocator"); + + void* buffer = nullptr; + if (fetched_tensor.Shape().Size() != 0) { + buffer = allocator->Alloc(fetched_tensor.DataType()->Size() * fetched_tensor.Shape().Size()); + if (!buffer) + return Status(common::ONNXRUNTIME, common::FAIL, "invalid buffer"); + } + + std::unique_ptr p_tensor = std::make_unique(fetched_tensor.DataType(), + fetched_tensor.Shape(), + buffer, + allocator->Info(), + allocator); + output_mlvalue.Init(p_tensor.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); + + return Status::OK(); + } + + // copies outputs across devices only if required + common::Status CopyOutputsAcrossDevices(std::vector& fetches, + std::vector& user_fetches) { + for (size_t idx = 0, end = fetches.size(); idx < end; ++idx) { + auto& fetched_mlvalue = fetches[idx]; + if (!fetched_mlvalue.IsTensor()) { + user_fetches[idx] = fetched_mlvalue; + continue; + } + + auto& fetched_tensor = fetched_mlvalue.Get(); + auto& fetched_tensor_location = fetched_tensor.Location(); + auto* p_fetched_provider = execution_providers_.Get(fetched_tensor_location); + if (!p_fetched_provider) { + p_fetched_provider = execution_providers_.Get(onnxruntime::kCpuExecutionProvider); + ONNXRUNTIME_ENFORCE(p_fetched_provider); + } + + auto fetched_provider_type = p_fetched_provider->Type(); + + auto& output_mlvalue = user_fetches[idx]; + if (!output_mlvalue.IsAllocated()) { + if (fetched_provider_type != onnxruntime::kCpuExecutionProvider) { + ONNXRUNTIME_RETURN_IF_ERROR(AllocateHelper(onnxruntime::kCpuExecutionProvider, 0, + fetched_tensor, + output_mlvalue)); + } else { + user_fetches[idx] = fetched_mlvalue; + continue; + } + } + + Tensor* p_output_tensor = output_mlvalue.GetMutable(); + auto& output_tensor_loc = p_output_tensor->Location(); + auto* p_output_provider = execution_providers_.Get(output_tensor_loc); + if (!p_output_provider) { + p_output_provider = execution_providers_.Get(onnxruntime::kCpuExecutionProvider); + ONNXRUNTIME_ENFORCE(p_output_provider); + } + + auto output_provider_type = p_output_provider->Type(); + + if (output_provider_type == fetched_provider_type || fetched_tensor_location.mem_type == ONNXRuntimeMemTypeCPUOutput) { + user_fetches[idx] = fetched_mlvalue; + continue; + } + + // our CPU exec provider doesn't support copy from GPU->CPU + if (fetched_provider_type != onnxruntime::kCpuExecutionProvider) { + ONNXRUNTIME_RETURN_IF_ERROR(p_fetched_provider->CopyTensor(fetched_tensor, *p_output_tensor)); + } else { + ONNXRUNTIME_RETURN_IF_ERROR(p_output_provider->CopyTensor(fetched_tensor, *p_output_tensor)); + } + } + + return Status::OK(); + } + + Status Run(const RunOptions& run_options, + const NameMLValMap& feeds, + const std::vector& output_names, + std::vector* p_fetches) { + auto tp = session_profiler_.StartTime(); + Status retval = Status::OK(); + + try { + { + std::lock_guard l(session_mutex_); + if (!is_inited_) { + LOGS(*session_logger_, ERROR) << "Session was not initialized"; + retval = Status(common::ONNXRUNTIME, common::FAIL, "Session not initialized."); + } + } + + ONNXRUNTIME_CHECK_AND_SET_RETVAL(ValidateInputs(feeds)); + + // if the output vector is non-empty, ensure that its the same size as the output_names + ONNXRUNTIME_CHECK_AND_SET_RETVAL(ValidateOutputs(output_names, p_fetches)); + + if (!run_options.run_tag.empty()) { + LOGS(*session_logger_, INFO) << "Running with tag: " << run_options.run_tag; + } + + ++current_num_runs_; + + // TODO should we add this exec to the list of executors? i guess its not needed now? + + // scope of owned_run_logger is just the call to Execute. + // If Execute ever becomes async we need a different approach + std::unique_ptr owned_run_logger; + auto run_logger = CreateLoggerForRun(run_options, owned_run_logger); + + // info all execution providers InferenceSession:Run started + // TODO: only call OnRunStart for all providers in-use + for (auto& xp : execution_providers_) + ONNXRUNTIME_CHECK_AND_SET_RETVAL(xp->OnRunStart()); + + NameMLValMap copied_feeds; + ONNXRUNTIME_CHECK_AND_SET_RETVAL(CopyInputsAcrossDevices(session_state_, feeds, copied_feeds)); + + std::vector new_fetches; + ONNXRUNTIME_CHECK_AND_SET_RETVAL(MatchOutputsWithProviders(output_names, *p_fetches, new_fetches)); + + std::unique_ptr p_exec; + + if (retval.IsOK()) { + if (session_options_.enable_sequential_execution) { + p_exec = std::unique_ptr(new SequentialExecutor(run_options.terminate)); + } else { + p_exec = std::unique_ptr(new ParallelExecutor(session_state_, run_options.terminate)); + } + } + + ONNXRUNTIME_CHECK_AND_SET_RETVAL(p_exec->Execute(session_state_, copied_feeds, output_names, new_fetches, run_logger)); + ONNXRUNTIME_CHECK_AND_SET_RETVAL(CopyOutputsAcrossDevices(new_fetches, *p_fetches)); + + } catch (const std::exception& e) { + retval = Status(common::ONNXRUNTIME, common::FAIL, e.what()); + } catch (...) { + retval = Status(common::ONNXRUNTIME, common::RUNTIME_EXCEPTION, "Encountered unknown exception in Run()"); + } + + // info all execution providers InferenceSession:Run ended + for (auto& xp : execution_providers_) + ONNXRUNTIME_CHECK_AND_SET_RETVAL(xp->OnRunEnd()); + + --current_num_runs_; + session_profiler_.EndTimeAndRecordEvent(profiling::SESSION_EVENT, "model_run", tp); + return retval; + } + + std::pair GetModelMetadata() const { + { + std::lock_guard l(session_mutex_); + if (!is_model_loaded_) { + LOGS(*session_logger_, ERROR) << "Model was not loaded"; + return std::make_pair(common::Status(common::ONNXRUNTIME, common::FAIL, "Model was not loaded."), + nullptr); + } + } + + return std::make_pair(common::Status::OK(), &model_metadata_); + } + + std::pair GetModelInputs() const { + { + std::lock_guard l(session_mutex_); + if (!is_model_loaded_) { + LOGS(*session_logger_, ERROR) << "Model was not loaded"; + return std::make_pair(common::Status(common::ONNXRUNTIME, common::FAIL, "Model was not loaded."), + nullptr); + } + } + + return std::make_pair(common::Status::OK(), &input_def_list_); + } + + std::pair GetModelOutputs() const { + { + std::lock_guard l(session_mutex_); + if (!is_model_loaded_) { + LOGS(*session_logger_, ERROR) << "Model was not loaded"; + return std::make_pair(common::Status(common::ONNXRUNTIME, common::FAIL, "Model was not loaded."), + nullptr); + } + } + + return std::make_pair(common::Status::OK(), &output_def_list_); + } + + common::Status NewIOBinding(std::unique_ptr* io_binding) { + { + std::lock_guard l(session_mutex_); + if (!is_inited_) { + LOGS(*session_logger_, ERROR) << "Session was not initialized"; + return common::Status(common::ONNXRUNTIME, common::FAIL, "Session not initialized."); + } + } + + // private constructor, can't use make_unique + *io_binding = std::unique_ptr(new IOBinding(session_state_)); + return Status::OK(); + } + + common::Status Run(const RunOptions& run_options, IOBinding& io_binding) { + // TODO should Run() call io_binding.SynchronizeInputs() or should it let the callers do it? + // io_binding.SynchronizeInputs(); + return Run(run_options, io_binding.feeds_, io_binding.output_names_, &io_binding.outputs_); + } + + common::Status Run(IOBinding& io_binding) { + RunOptions run_options; + return Run(run_options, io_binding); + } + + void StartProfiling(const std::string& file_prefix) { + std::ostringstream ss; + ss << file_prefix << "_" << GetCurrentTimeString() << ".json"; + session_profiler_.StartProfiling(ss.str()); + } + + void StartProfiling(const logging::Logger* logger_ptr) { + session_profiler_.StartProfiling(logger_ptr); + } + + std::string EndProfiling() { + if (is_model_loaded_) { + return session_profiler_.EndProfiling(); + } + LOGS(*session_logger_, ERROR) << "Could not write a profile because no model was loaded."; + return std::string(); + } + + private: + static std::pair Contains(const std::vector& output_names, + const std::string& name) { + auto it = std::find(std::begin(output_names), std::end(output_names), name); + if (it == output_names.end()) { + return {false, 0}; + } + return {true, it - output_names.begin()}; + } + + bool HasLocalSchema() const { + return !custom_schema_registries_.empty(); + } + + // assumes model has already been loaded before + common::Status DoPostLoadProcessing(onnxruntime::Model& model) { + // TODO add other post load processing here + common::Status status = SaveModelMetadata(model); + return status; + } + + common::Status SaveModelMetadata(const onnxruntime::Model& model) { + VLOGS(*session_logger_, 1) << "Saving model metadata"; + const onnxruntime::Graph& graph = model.MainGraph(); + + // save model metadata + model_metadata_.producer_name = model.ProducerName(); + model_metadata_.description = model.DocString(); + model_metadata_.domain = model.Domain(); + model_metadata_.version = model.ModelVersion(); + model_metadata_.custom_metadata_map = model.MetaData(); + model_metadata_.graph_name = graph.Name(); + + // save inputs + auto& inputs = graph.GetInputs(); // inputs excluding initializers + input_def_list_.reserve(inputs.size()); + for (const auto& elem : inputs) { + if (!elem) { + return common::Status(common::ONNXRUNTIME, common::FAIL, "Got null input nodearg ptr"); + } + + input_def_list_.push_back(elem); + model_input_names_.insert(elem->Name()); + } + + // save outputs + auto& outputs = graph.GetOutputs(); + output_def_list_.reserve(outputs.size()); + for (const auto& elem : outputs) { + if (!elem) { + return common::Status(common::ONNXRUNTIME, common::FAIL, "Got null output nodearg ptr"); + } + output_def_list_.push_back(elem); + model_output_names_.insert(elem->Name()); + } + VLOGS(*session_logger_, 1) << "Done saving model metadata"; + return common::Status::OK(); + } + + // Create a Logger for a single execution if possible. Otherwise use the default logger. + // If a new logger is created, it will also be stored in new_run_logger, + // which must remain valid for the duration of the execution. + // If the default logger is used, new_run_logger will remain empty. + // The returned value should be used in the execution. + const logging::Logger& CreateLoggerForRun(const RunOptions& run_options, + std::unique_ptr& new_run_logger) { + const logging::Logger* run_logger; + + // create a per-run logger if we can + if (logging_manager_ != nullptr) { + std::string run_log_id{session_options_.session_logid}; + + if (!session_options_.session_logid.empty() && !run_options.run_tag.empty()) { + run_log_id += ":"; + } + + run_log_id += run_options.run_tag; + + if (run_options.run_log_verbosity_level > 0) { + new_run_logger = logging_manager_->CreateLogger(run_log_id, + logging::Severity::kVERBOSE, + false, + run_options.run_log_verbosity_level); + } else { + new_run_logger = logging_manager_->CreateLogger(run_log_id); + } + + run_logger = new_run_logger.get(); + VLOGS(*run_logger, 1) << "Created logger for run with id of " << run_log_id; + } else { + // fallback to using default logger. this does NOT have any session or run specific id/tag in it + run_logger = session_logger_; + VLOGS(*run_logger, 1) << "Using default logger for run " << run_options.run_tag; + } + + return *run_logger; + } + + void InitLogger(logging::LoggingManager* logging_manager) { + // create logger for session, using provided logging manager if possible + if (logging_manager != nullptr) { + std::string session_logid = !session_options_.session_logid.empty() + ? session_options_.session_logid + : "InferenceSession"; // there's probably a better default... + + if (session_options_.session_log_verbosity_level > 0) { + owned_session_logger_ = logging_manager->CreateLogger(session_logid, + logging::Severity::kVERBOSE, + false, + session_options_.session_log_verbosity_level); + } else { + owned_session_logger_ = logging_manager->CreateLogger(session_logid); + } + session_logger_ = owned_session_logger_.get(); + } else { + session_logger_ = &logging::LoggingManager::DefaultLogger(); + } + + session_state_.SetLogger(*session_logger_); + } + + common::Status WaitForNotification(Notification* p_executor_done, int64_t timeout_in_ms) { + if (timeout_in_ms > 0) { + ONNXRUNTIME_NOT_IMPLEMENTED(__FUNCTION__, "timeout_in_ms >0 is not supported"); // TODO + } + p_executor_done->WaitForNotification(); + + return Status::OK(); + } + + CustomOpsLoader custom_ops_loader_; + + const SessionOptions session_options_; + + onnxruntime::GraphTransformerManager graph_transformation_mgr_; + + /// Logging manager if provided. + logging::LoggingManager* logging_manager_; + + /// Logger for this session. WARNING: Will contain nullptr if logging_manager_ is nullptr. + std::unique_ptr owned_session_logger_; + + /// convenience pointer to logger. should always be the same as session_state_.Logger(); + const logging::Logger* session_logger_; + + // Profiler for this session. + profiling::Profiler session_profiler_; + + ExecutionProviders execution_providers_; + + KernelRegistryManager kernel_registry_manager_; + std::list> custom_schema_registries_; + + // The model served by this inference session instance. + // Currently this has to be a shared ptr because the Model::Load method + // returns a shared_ptr only. Ideally factory functions should always return + // unique_ptr for maximum flexibility. Client can always upgrade it to shared_ptr + // if they need. + std::shared_ptr model_; + + // A set of executors that can run in parallel. + std::vector> executors_; // TODO do we need this vector? + + // Immutable state for each op in the model. Shared by all executors. + SessionState session_state_; + + ModelMetadata model_metadata_; + InputDefList input_def_list_; + OutputDefList output_def_list_; + + // names of model inputs and outputs used for quick validation. + std::unordered_set model_input_names_; + std::unordered_set model_output_names_; + + // Environment for this session + // not used now; we'll need it when we introduce threadpool + // statically allocated pointer, no need to manage its lifetime. + //Env* env_; + + // Threadpool for this session + //thread::ThreadPool thread_pool_; // not used for now; will add it later when implementing RunAsync + std::unique_ptr thread_pool_; + + // Number of concurrently running executors + std::atomic current_num_runs_; + + mutable std::mutex session_mutex_; // to ensure only one thread can invoke Load/Initialize + bool is_model_loaded_ = false; // GUARDED_BY(session_mutex_) + bool is_inited_ = false; // GUARDED_BY(session_mutex_) + + std::map weights_buffers_; + InsertCastTransformer insert_cast_transformer_; + + // memory allocations for any subgraphs + std::vector subgraph_memory_; +}; // namespace onnxruntime + +// +// InferenceSession +// +InferenceSession::InferenceSession(const SessionOptions& session_options, + logging::LoggingManager* logging_manager) + : impl_(std::make_unique(session_options, logging_manager)) { +} + +InferenceSession::~InferenceSession() = default; + +common::Status InferenceSession::Load(const std::string& model_uri) { + return impl_->Load(model_uri); +} +#ifdef _WIN32 +common::Status InferenceSession::Load(const std::wstring& model_uri) { + return impl_->Load(model_uri); +} +#endif +common::Status InferenceSession::Load(std::istream& model_istream) { + return impl_->Load(model_istream); +} + +common::Status InferenceSession::Initialize() { + return impl_->Initialize(); +} + +common::Status InferenceSession::Run(const NameMLValMap& feeds, + const std::vector& output_names, + std::vector* p_fetches) { + return impl_->Run(feeds, output_names, p_fetches); +} + +common::Status InferenceSession::Run(const RunOptions& run_options, + const NameMLValMap& feeds, + const std::vector& output_names, + std::vector* p_fetches) { + return impl_->Run(run_options, feeds, output_names, p_fetches); +} + +std::pair InferenceSession::GetModelMetadata() const { + return impl_->GetModelMetadata(); +} + +std::pair InferenceSession::GetModelInputs() const { + return impl_->GetModelInputs(); +} + +std::pair InferenceSession::GetModelOutputs() const { + return impl_->GetModelOutputs(); +} + +int InferenceSession::GetCurrentNumRuns() { + return impl_->GetCurrentNumRuns(); +} + +void InferenceSession::StartProfiling(const std::string& file_prefix) { + impl_->StartProfiling(file_prefix); +} + +void InferenceSession::StartProfiling(const logging::Logger* custom_logger) { + impl_->StartProfiling(custom_logger); +} + +std::string InferenceSession::EndProfiling() { + return impl_->EndProfiling(); +} + +common::Status InferenceSession::RegisterExecutionProvider(std::unique_ptr p_exec_provider) { + return impl_->RegisterExecutionProvider(std::move(p_exec_provider)); +} + +common::Status InferenceSession::RegisterGraphTransformer(std::unique_ptr p_graph_transformer) { + return impl_->RegisterGraphTransformer(std::move(p_graph_transformer)); +} + +common::Status InferenceSession::RegisterCustomRegistry(std::shared_ptr custom_registry) { + return impl_->RegisterCustomRegistry(custom_registry); +} + +common::Status InferenceSession::Load(const ModelProto& model_proto) { + return impl_->Load(model_proto); +} + +common::Status InferenceSession::Load(std::unique_ptr p_model_proto) { + return impl_->Load(std::move(p_model_proto)); +} + +common::Status InferenceSession::NewIOBinding(std::unique_ptr* io_binding) { + return impl_->NewIOBinding(io_binding); +} + +common::Status InferenceSession::Run(const RunOptions& run_options, IOBinding& io_binding) { + return impl_->Run(run_options, io_binding); +} + +common::Status InferenceSession::Run(IOBinding& io_binding) { + return impl_->Run(io_binding); +} + +common::Status InferenceSession::LoadCustomOps(const std::vector& dso_list) { + return impl_->LoadCustomOps(dso_list); +} +} // namespace onnxruntime diff --git a/onnxruntime/core/session/inference_session.h b/onnxruntime/core/session/inference_session.h new file mode 100644 index 0000000000000..65a7ef991063f --- /dev/null +++ b/onnxruntime/core/session/inference_session.h @@ -0,0 +1,276 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include + +#include "core/common/common.h" +#include "core/common/status.h" +#include "core/framework/framework_common.h" +#include "core/graph/basic_types.h" +#include "core/common/logging/logging.h" + +namespace onnxruntime { // forward declarations +class GraphTransformer; +} // namespace onnxruntime + +namespace ONNX_NAMESPACE { +class ModelProto; +} // namespace ONNX_NAMESPACE + +namespace onnxruntime { +class IExecutionProvider; // forward decl +class IOBinding; + +class CustomRegistry; + +namespace logging { +class LoggingManager; +} + +/** + * Configuration information for a session. + */ +struct SessionOptions { + //int num_threads; // not used now until we re-introduce threadpools for async execution + bool enable_sequential_execution = true; // TODO: should we default to sequential execution? + + // enable profiling for this session. + bool enable_profiling = false; + + // enable the memory pattern optimization. + // The idea is if the input shapes are the same, we could trace the internal memory allocation + // and generate a memory pattern for future request. So next time we could just do one allocation + // with a big chunk for all the internal memory allocation. + bool enable_mem_pattern = true; + + // enable the memory arena on CPU + // Arena may pre-allocate memory for future usage. + // set this option to false if you don't want it. + bool enable_cpu_mem_arena = true; + + // the prefix of the profile file. The current time will be appended to the file name. + std::string profile_file_prefix = "onnxruntime_profile_"; + + std::string session_logid; ///< logger id to use for session output + unsigned session_log_verbosity_level = 0; ///< applies to session load, initialization, etc + + unsigned max_num_graph_transformation_steps = 5; // TODO choose a good default here? + + // How many threads in the session thread pool. + int session_thread_pool_size = 0; +}; + +/** + * Pre-defined and custom metadata about the model. + */ +struct ModelMetadata { + std::string producer_name; + std::string graph_name; + std::string domain; + std::string description; + int64_t version; + std::unordered_map custom_metadata_map; +}; + +/** + * @brief This is the main class used to Run a model. + * Sample simple usage: + * CPUExecutionProviderInfo epi; + * ProviderOption po{"CPUExecutionProvider", epi}; + * SessionOptions so(vector{po}); + * InferenceSession session_object{so}; + * common::Status status = session_object.Load(MODEL_URI); + * common::Status status = session_object.Initialize(); + * + * NameMLValMap feeds; + * feeds.insert({}); + * ... + * std::vector output_names; + * output_names.insert(...); + * ... + * std::vector fetches; + * common::Status status = session_object.Run(run_options, feeds, output_names, &fetches); + * process the output here... + */ + +class InferenceSession { + public: + /** + Create a new InferenceSession + @param session_options Session options. + @param logging_manager + Optional logging manager instance that will enable per session logger output using + session_options.session_logid as the logger id in messages. + If nullptr, the default LoggingManager MUST have been created previously as it will be used + for logging. This will use the default logger id in messages. + See core/common/logging/logging.h for details, and how LoggingManager::DefaultLogger works. + */ + explicit InferenceSession(const SessionOptions& session_options, + logging::LoggingManager* logging_manager = nullptr); + + virtual ~InferenceSession(); + + /** + * Register an execution provider. If you've one to register, call this before invoking Initialize(). + * The order of invocation indicates the preference order as well. In other words call this method + * on your most preferred execution provider first followed by the less preferred ones. + * Calling this API is optional in which case onnxruntime will use its internal CPU execution provider. + * @return OK if success. + */ + common::Status RegisterExecutionProvider(std::unique_ptr p_exec_provider); + + /** + * Register a graph transformer. If you've one to register, call this before invoking Initialize(). + * Calling this API is optional. + * @return OK if success. + */ + common::Status RegisterGraphTransformer(std::unique_ptr p_graph_transformer); + + /** + * Load custom ops implemented in a dynamically linked shared library. + * @param dso_list list of library file paths containing the custom ops implementation. + * In order to implement a custom op please see file: custom_ops_author.h + * TODO add sample code + * @return OK if success + */ + common::Status LoadCustomOps(const std::vector& dso_list); + + /** + * Register a custom registry for operator schema and kernels. If you've one to register, + * call this before invoking Initialize(). + * The order of invocation indicates the reversed preference order: Register your most + * preferred registry at the end. + * Calling this API is optional. + * @return OK if success. + */ + common::Status RegisterCustomRegistry(std::shared_ptr custom_registry); + + /** + * Load an ONNX model. + * @param model_uri absolute path of the model file. + * @return OK if success. + */ + common::Status Load(const std::string& model_uri); +#ifdef _WIN32 + common::Status Load(const std::wstring& model_uri); +#endif + /** + * Load an ONNX model. + * @param istream object of the model. + * @return OK if success. + */ + common::Status Load(std::istream& model_istream); + + /** + * Initializes a previously loaded model. Initialization includes but is not + * limited to graph transformations, construction of kernels, etc. + * This method assumes that a method has been loaded previously. + * @return OK if success + */ + common::Status Initialize(); + + /** + * Run a pre-loaded and pre-intialized model. + * Multiple threads are allowed to run this function; hence its thread-safe. + * @param feeds named inputs owned by client code and should not be changed during + * execution of this function. + * @param output_names output names + * @param p_fetches output values in the order specified by output_names. + * This should not be changed during execution of this function. + * @return OK if success. + */ + common::Status Run(const NameMLValMap& feeds, + const std::vector& output_names, + std::vector* p_fetches); + + /** + * See Run(const NameMLValMap& feeds, const std::vector& output_names, std::vector* p_fetches) + * for details. + * @param run_options use this to tune the Run call to your needs. + */ + common::Status Run(const RunOptions& run_options, + const NameMLValMap& feeds, + const std::vector& output_names, + std::vector* p_fetches); + + /** + * Creates a new binding object for binding inputs and outputs. + * @param provider_type specifies the location where the inputs need to be potentially copied. + * See IOBinding class for more info. + */ + common::Status NewIOBinding(std::unique_ptr* io_binding); + + common::Status Run(const RunOptions& run_options, IOBinding& io_binding); + common::Status Run(IOBinding& io_binding); + + /** + * @return pair.first = OK; FAIL otherwise. pair.second is non-NULL when pair.first = OK. + * @note lifetime of the returned pointer is valid as long as the Session object is live. + */ + std::pair GetModelMetadata() const; + + /** + * Get all input definitions of the model. This does not include weights. Use this + * to get the name/type/shapes of the inputs. + * @return pair.first = OK; FAIL otherwise. pair.second is non-NULL when pair.first = OK. + * @note lifetime of the returned pointer is valid as long as the Session object is live. + */ + std::pair GetModelInputs() const; + + /** + * Get all output definitions of the model. Use this to get the name/type/shapes of the outputs. + * @return pair.first = OK; FAIL otherwise. pair.second is non-NULL when pair.first = OK. + * @note lifetime of the returned pointer is valid as long as the Session object is live. + */ + std::pair GetModelOutputs() const; + + /** + * Get the current number of in-progress concurrent Run calls. + */ + int GetCurrentNumRuns(); + + /** + * Start profiling on this inference session. This simply turns on profiling events to be + * recorded. A corresponding EndProfiling has to follow to write profiling data to a file. + *@param file_prefix is the prefix of the profile file. It can include a directory path. + */ + void StartProfiling(const std::string& file_prefix); + + /** + * Start profiling on this inference session. This simply turns on profiling events to be + * recorded. A corresponding EndProfiling has to follow to send profiling events through the logger's ISink. + *@param logger_ptr is pointer to the logger where profiling events will be sent to. + */ + void StartProfiling(const logging::Logger* logger_ptr); + + /** + * Write captured profile events in chromium format. + @return the name of the profile file. + */ + std::string EndProfiling(); + + protected: + /** + * Load an ONNX model. + * @param protobuf object corresponding to the model file. model_proto will be copied by the API. + * @return OK if success. + */ + common::Status Load(const ONNX_NAMESPACE::ModelProto& model_proto); + + /** + * Load an ONNX model. + * @param protobuf object corresponding to the model file. This is primarily to support large models. + * @return OK if success. + */ + common::Status Load(std::unique_ptr p_model_proto); + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(InferenceSession); + + class Impl; + std::unique_ptr impl_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/session/onnxruntime_c_api.cc b/onnxruntime/core/session/onnxruntime_c_api.cc new file mode 100644 index 0000000000000..c0822c0ebe1a2 --- /dev/null +++ b/onnxruntime/core/session/onnxruntime_c_api.cc @@ -0,0 +1,706 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/graph/onnx_protobuf.h" //TODO: remove this +#include "core/session/onnxruntime_c_api.h" +#include "core/session/allocator_impl.h" +#include "core/framework/error_code_helper.h" +#include "core/framework/execution_provider.h" +#include +#include +#include + +#include "core/common/logging/logging.h" +#include "core/common/logging/sinks/clog_sink.h" +#include "core/common/status.h" +#include "core/framework/allocator.h" +#include "core/framework/tensor.h" +#include "core/framework/ml_value.h" +#include "core/framework/environment.h" +#include "core/framework/tensorprotoutils.h" +#include "core/session/inference_session.h" +#include "core/graph/graph_base.h" +#include "abi_session_options_impl.h" + +using namespace onnxruntime::logging; +using onnxruntime::DataTypeImpl; +using onnxruntime::Environment; +using onnxruntime::IAllocator; +using onnxruntime::InputDefList; +using onnxruntime::MLFloat16; +using onnxruntime::MLStatus; +using onnxruntime::MLValue; +using onnxruntime::OutputDefList; +using onnxruntime::Tensor; +using onnxruntime::ToONNXStatus; +using onnxruntime::common::Status; + +#define ONNXRUNTIME_API_RETURN_IF_ERROR(expr) \ + do { \ + auto _status = (expr); \ + if (_status) return _status; \ + } while (0) + +struct ONNXEnv { + ONNXEnv(Environment* value1, LoggingManager* loggingManager1) : value(value1), loggingManager(loggingManager1) { + } + /** + * This function will call ::google::protobuf::ShutdownProtobufLibrary + */ + ~ONNXEnv() { + delete loggingManager; + delete value; + } + Environment* value; + LoggingManager* loggingManager; + ONNXRUNTIME_DISALLOW_COPY_AND_ASSIGNMENT(ONNXEnv); +}; + +#define API_IMPL_BEGIN try { +#define API_IMPL_END \ + } \ + catch (std::exception & ex) { \ + return CreateONNXStatus(ONNXRUNTIME_RUNTIME_EXCEPTION, ex.what()); \ + } + +#define TENSOR_READ_API_BEGIN \ + API_IMPL_BEGIN \ + auto v = reinterpret_cast<::onnxruntime::MLValue*>(value); \ + auto& tensor = v->Get(); + +#define TENSOR_READWRITE_API_BEGIN \ + API_IMPL_BEGIN \ + auto v = reinterpret_cast<::onnxruntime::MLValue*>(value); \ + auto tensor = v->GetMutable(); + +class LoggingWrapper : public ISink { + public: + LoggingWrapper(ONNXRuntimeLoggingFunction logging_function, void* logger_param) + : logging_function_{logging_function}, logger_param_{logger_param} { + } + + void SendImpl(const Timestamp& /*timestamp*/ /*timestamp*/, const std::string& logger_id, + const Capture& message) override { + std::string s = message.Location().ToString(); + logging_function_(logger_param_, static_cast(message.Severity()), message.Category(), + logger_id.c_str(), s.c_str(), message.Message().c_str()); + } + + private: + ONNXRuntimeLoggingFunction logging_function_; + void* logger_param_; +}; + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeInitializeWithCustomLogger, ONNXRuntimeLoggingFunction logging_function, + void* logger_param, ONNXRuntimeLoggingLevel default_warning_level, _In_ const char* logid, + _Out_ ONNXEnv** out) { + API_IMPL_BEGIN + std::string name = logid; + std::unique_ptr logger = std::make_unique(logging_function, logger_param); + auto default_logging_manager = std::make_unique(std::move(logger), + static_cast(default_warning_level), false, + LoggingManager::InstanceType::Default, + &name); + std::unique_ptr env; + Status status = Environment::Create(env); + if (status.IsOK()) + *out = new ONNXEnv(env.release(), default_logging_manager.release()); + return ToONNXStatus(status); + API_IMPL_END +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeInitialize, ONNXRuntimeLoggingLevel default_warning_level, + _In_ const char* logid, _Out_ ONNXEnv** out) { + API_IMPL_BEGIN + std::string name = logid; + auto default_logging_manager = std::make_unique(std::unique_ptr{new CLogSink{}}, + static_cast(default_warning_level), false, + LoggingManager::InstanceType::Default, + &name); + std::unique_ptr env; + Status status = Environment::Create(env); + if (status.IsOK()) + *out = new ONNXEnv(env.release(), default_logging_manager.release()); + return ToONNXStatus(status); + API_IMPL_END +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeGetStringTensorDataLength, _In_ ONNXValuePtr value, _Out_ size_t* out) { + TENSOR_READ_API_BEGIN + const auto* src = tensor.Data(); + int64_t len = tensor.Shape().Size(); + if (len >= 0) { + size_t ret = 0; + for (int64_t i = 0; i != len; ++i) { + ret += src[i].size(); + } + *out = ret; + } else + return CreateONNXStatus(ONNXRUNTIME_INVALID_ARGUMENT, "shape is invalid"); + return nullptr; + API_IMPL_END +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeFillStringTensor, _In_ ONNXValuePtr value, _In_ const char* s[], size_t s_len) { + TENSOR_READWRITE_API_BEGIN + auto* dst = tensor->MutableData(); + auto len = static_cast(tensor->Shape().Size()); + if (s_len < len) { + return CreateONNXStatus(ONNXRUNTIME_INVALID_ARGUMENT, "input array is too short"); + } + for (size_t i = 0; i != len; ++i) { + //allocate and copy + dst[i] = s[i]; + } + return nullptr; + API_IMPL_END +} + +template +void CreateTensorImpl(const size_t* shape, size_t shape_len, std::shared_ptr& allocator, + std::unique_ptr* out) { + size_t elem_count = 1; + std::vector shapes(shape_len); + for (size_t i = 0; i != shape_len; ++i) { + elem_count *= shape[i]; + shapes[i] = shape[i]; + } + + size_t size_to_allocate = sizeof(T) * elem_count; + void* p_data = allocator->Alloc(size_to_allocate); + *out = std::make_unique(DataTypeImpl::GetType(), + onnxruntime::TensorShape(shapes.data(), shape_len), + static_cast(p_data), + allocator->Info(), + allocator); +} + +template +ONNXStatusPtr CreateTensorImpl(const size_t* shape, size_t shape_len, const ONNXRuntimeAllocatorInfo* info, + void* p_data, size_t p_data_len, std::unique_ptr* out) { + size_t elem_count = 1; + std::vector shapes(shape_len); + for (size_t i = 0; i != shape_len; ++i) { + elem_count *= shape[i]; + shapes[i] = shape[i]; + } + + size_t size_to_allocate = sizeof(T) * elem_count; + if (size_to_allocate > p_data_len) { + std::ostringstream oss; + oss << "not enough space: expected " << size_to_allocate << ", got " << p_data_len; + return CreateONNXStatus(ONNXRUNTIME_INVALID_ARGUMENT, oss.str().c_str()); + } + *out = std::make_unique(DataTypeImpl::GetType(), + onnxruntime::TensorShape(shapes.data(), shape_len), + p_data, + *info, + nullptr); + return nullptr; +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeCreateTensorWithDataAsONNXValue, _In_ const ONNXRuntimeAllocatorInfo* info, + _In_ void* p_data, size_t p_data_len, _In_ const size_t* shape, size_t shape_len, + OnnxRuntimeTensorElementDataType type, _Out_ ONNXValuePtr* out) { + API_IMPL_BEGIN + std::unique_ptr tensor; + switch (type) { + case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, info, p_data, p_data_len, &tensor)); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8: + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, info, p_data, p_data_len, &tensor)); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8: + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, info, p_data, p_data_len, &tensor)); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16: + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, info, p_data, p_data_len, &tensor)); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16: + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, info, p_data, p_data_len, &tensor)); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32: + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, info, p_data, p_data_len, &tensor)); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, info, p_data, p_data_len, &tensor)); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING: + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, info, p_data, p_data_len, &tensor)); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL: + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, info, p_data, p_data_len, &tensor)); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16: + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, info, p_data, p_data_len, &tensor)); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE: + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, info, p_data, p_data_len, &tensor)); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32: + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, info, p_data, p_data_len, &tensor)); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64: + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, info, p_data, p_data_len, &tensor)); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64: + case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128: + default: { + std::ostringstream oss; + oss << "type " << type << " is not supported in this function"; + std::string errmsg = oss.str(); + return CreateONNXStatus(ONNXRUNTIME_NOT_IMPLEMENTED, errmsg.c_str()); + } + } + std::unique_ptr value = std::make_unique(); + value->Init(tensor.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); + *out = reinterpret_cast(value.release()); + return nullptr; + API_IMPL_END +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeCreateTensorAsONNXValue, _Inout_ ONNXRuntimeAllocator* allocator, + _In_ const size_t* shape, size_t shape_len, OnnxRuntimeTensorElementDataType type, + _Out_ ONNXValuePtr* out) { + API_IMPL_BEGIN + std::shared_ptr allocator_ = std::make_shared(allocator); + std::unique_ptr tensor; + switch (type) { + case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: + CreateTensorImpl(shape, shape_len, allocator_, &tensor); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8: + CreateTensorImpl(shape, shape_len, allocator_, &tensor); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8: + CreateTensorImpl(shape, shape_len, allocator_, &tensor); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16: + CreateTensorImpl(shape, shape_len, allocator_, &tensor); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16: + CreateTensorImpl(shape, shape_len, allocator_, &tensor); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32: + CreateTensorImpl(shape, shape_len, allocator_, &tensor); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: + CreateTensorImpl(shape, shape_len, allocator_, &tensor); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING: + CreateTensorImpl(shape, shape_len, allocator_, &tensor); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL: + CreateTensorImpl(shape, shape_len, allocator_, &tensor); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16: + CreateTensorImpl(shape, shape_len, allocator_, &tensor); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE: + CreateTensorImpl(shape, shape_len, allocator_, &tensor); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32: + CreateTensorImpl(shape, shape_len, allocator_, &tensor); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64: + CreateTensorImpl(shape, shape_len, allocator_, &tensor); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64: + case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128: + case ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16: + default: { + std::ostringstream oss; + oss << "type " << type << " is not supported in this function"; + std::string errmsg = oss.str(); + return CreateONNXStatus(ONNXRUNTIME_NOT_IMPLEMENTED, errmsg.c_str()); + } + } + std::unique_ptr value = std::make_unique(); + value->Init(tensor.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); + *out = reinterpret_cast(value.release()); + return nullptr; + API_IMPL_END +} + +template +static ONNXStatusPtr CreateInferenceSessionImpl(_In_ ONNXEnv* env, _In_ T model_path, + _In_ const ONNXRuntimeSessionOptions* options, + _Out_ ONNXSessionPtr* out) { + API_IMPL_BEGIN + auto sess = std::make_unique<::onnxruntime::InferenceSession>(options->value, env->loggingManager); + Status status; + if (!options->custom_op_paths.empty()) { + status = sess->LoadCustomOps(options->custom_op_paths); + if (!status.IsOK()) + return ToONNXStatus(status); + } + for (ONNXRuntimeProviderFactoryPtr* p : options->provider_factories) { + ONNXRuntimeProviderPtr provider; + ONNXStatusPtr error_code = (*p)->CreateProvider(p, &provider); + if (error_code) + return error_code; + sess->RegisterExecutionProvider(std::unique_ptr( + reinterpret_cast(provider))); + } + status = sess->Load(model_path); + if (!status.IsOK()) + return ToONNXStatus(status); + status = sess->Initialize(); + if (!status.IsOK()) + return ToONNXStatus(status); + *out = reinterpret_cast(sess.release()); + return nullptr; + API_IMPL_END +} + +#ifdef _WIN32 +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeCreateInferenceSession, _In_ ONNXEnv* env, _In_ const wchar_t* model_path, + _In_ const ONNXRuntimeSessionOptions* options, _Out_ ONNXSessionPtr* out) { + API_IMPL_BEGIN + return CreateInferenceSessionImpl(env, model_path, options, out); + API_IMPL_END +} +#else +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeCreateInferenceSession, _In_ ONNXEnv* env, _In_ const char* model_path, + _In_ const ONNXRuntimeSessionOptions* options, _Out_ ONNXSessionPtr* out) { + API_IMPL_BEGIN + return CreateInferenceSessionImpl(env, model_path, options, out); + API_IMPL_END +} +#endif + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeRunInference, _In_ ONNXSessionPtr sess, + _In_ const char* input_names[], _In_ ONNXValuePtr* input, size_t input_len, + _In_ const char* output_names1[], size_t output_names_len, _Out_ ONNXValuePtr* output) { + API_IMPL_BEGIN + ONNXRuntimeRunOptions run_options{}; + return ONNXRuntimeRunInferenceWithRunOptions(sess, &run_options, input_names, input, input_len, + output_names1, output_names_len, output); + API_IMPL_END +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeRunInferenceWithRunOptions, _In_ ONNXSessionPtr sess, + _In_ ONNXRuntimeRunOptionsPtr run_options, + _In_ const char* input_names[], _In_ ONNXValuePtr* input, size_t input_len, + _In_ const char* output_names1[], size_t output_names_len, _Out_ ONNXValuePtr* output) { + API_IMPL_BEGIN + auto session = reinterpret_cast<::onnxruntime::InferenceSession*>(sess); + ::onnxruntime::NameMLValMap in; + const int queue_id = 0; + for (size_t i = 0; i != input_len; ++i) { + auto kvp = in.insert(std::make_pair(std::string(input_names[i]), + *reinterpret_cast<::onnxruntime::MLValue*>(input[i]))); + if (!kvp.second) { + return CreateONNXStatus(ONNXRUNTIME_INVALID_ARGUMENT, "duplicated input name"); + } + ::onnxruntime::MLValue& value = kvp.first->second; + if (value.Fence()) + value.Fence()->BeforeUsingAsInput(onnxruntime::kCpuExecutionProvider, queue_id); + } + // Create output feed + std::vector output_names(output_names_len); + for (size_t i = 0; i != output_names_len; ++i) { + if (output_names1[i] == nullptr || output_names1[i][0] == '\0') { + return CreateONNXStatus(ONNXRUNTIME_INVALID_ARGUMENT, "output name cannot be empty"); + } + output_names[i] = output_names1[i]; + } + + std::vector fetches(output_names_len); + for (size_t i = 0; i != output_names_len; ++i) { + if (output[i] != nullptr) { + ::onnxruntime::MLValue& value = *reinterpret_cast<::onnxruntime::MLValue*>(output[i]); + if (value.Fence()) + value.Fence()->BeforeUsingAsOutput(onnxruntime::kCpuExecutionProvider, queue_id); + fetches[i] = value; + } + } + auto status = session->Run(*run_options, in, output_names, &fetches); + if (!status.IsOK()) + return ToONNXStatus(status); + for (size_t i = 0; i != output_names_len; ++i) { + ::onnxruntime::MLValue& value = fetches[i]; + if (value.Fence()) + value.Fence()->BeforeUsingAsInput(onnxruntime::kCpuExecutionProvider, queue_id); + if (output[i] == nullptr) { + output[i] = reinterpret_cast(new MLValue(value)); + } + } + return nullptr; + API_IMPL_END +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeRunInferenceAndFetchAll, _In_ ONNXSessionPtr sess, + _In_ const char* input_names[], _In_ ONNXValuePtr* input, size_t input_len, + _Out_ ONNXValueListPtr* output, _Out_ size_t* output_len) { + API_IMPL_BEGIN + ONNXRuntimeRunOptions run_options{}; + return ONNXRuntimeRunInferenceAndFetchAllWithRunOptions(sess, &run_options, input_names, input, input_len, + output, output_len); + API_IMPL_END +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeRunInferenceAndFetchAllWithRunOptions, _In_ ONNXSessionPtr sess, + _In_ ONNXRuntimeRunOptionsPtr run_options, + _In_ const char* input_names[], _In_ ONNXValuePtr* input, size_t input_len, + _Out_ ONNXValueListPtr* output, _Out_ size_t* output_len) { + API_IMPL_BEGIN + auto session = reinterpret_cast<::onnxruntime::InferenceSession*>(sess); + ::onnxruntime::NameMLValMap in; + for (size_t i = 0; i != input_len; ++i) { + auto kvp = in.insert(std::make_pair(std::string(input_names[i]), + *reinterpret_cast<::onnxruntime::MLValue*>(input[i]))); + if (!kvp.second) { + return CreateONNXStatus(ONNXRUNTIME_INVALID_ARGUMENT, "duplicated input name"); + } + } + // Create output feed + std::vector output_names; + for (auto const& outp : *(session->GetModelOutputs().second)) { + output_names.push_back(outp->Name()); + } + std::vector fetches; + auto status = session->Run(*run_options, in, output_names, &fetches); + if (!status.IsOK()) + return ToONNXStatus(status); + auto* out = new MLValue[fetches.size()]; + const int queue_id = 0; + for (size_t i = 0; i != fetches.size(); ++i) { + if (fetches[i].Fence()) + fetches[i].Fence()->BeforeUsingAsInput(onnxruntime::kCpuExecutionProvider, queue_id); + out[i] = fetches[i]; + } + *output_len = fetches.size(); + *output = reinterpret_cast(out); + return nullptr; + API_IMPL_END +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeGetTensorMutableData, _In_ ONNXValuePtr value, _Out_ void** output) { + TENSOR_READWRITE_API_BEGIN + //TODO: test if it's a string tensor + *output = tensor->MutableDataRaw(); + return nullptr; + API_IMPL_END +} + +inline OnnxRuntimeTensorElementDataType MLDataTypeToOnnxRuntimeTensorElementDataType( + const onnxruntime::DataTypeImpl* cpp_type) { + OnnxRuntimeTensorElementDataType type; + if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; + } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8; + } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8; + } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16; + } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16; + } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32; + } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64; + } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING; + } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL; + } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16; + } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE; + } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32; + } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64; + } else { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_MAX; + } + return type; +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeGetTensorShapeAndType, _In_ const ONNXValuePtr value, + _Out_ ONNXRuntimeTensorTypeAndShapeInfo** out) { + TENSOR_READ_API_BEGIN + OnnxRuntimeTensorElementDataType type = MLDataTypeToOnnxRuntimeTensorElementDataType(tensor.DataType()); + if (ONNX_TENSOR_ELEMENT_DATA_TYPE_MAX == type) { + return CreateONNXStatus(ONNXRUNTIME_FAIL, "Not implemented"); + } + const onnxruntime::TensorShape& shape = tensor.Shape(); + ONNXRuntimeTensorTypeAndShapeInfo* ret = ONNXRuntimeCreateTensorTypeAndShapeInfo(); + auto status = ONNXRuntimeSetTensorElementType(ret, type); + if (status != nullptr) { + ONNXRuntimeReleaseObject(ret); + return status; + } + status = ONNXRuntimeSetDims(ret, shape.GetDims().data(), shape.GetDims().size()); + if (status != nullptr) { + ONNXRuntimeReleaseObject(ret); + return status; + } + *out = ret; + return nullptr; + API_IMPL_END +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeGetStringTensorContent, _In_ ONNXValuePtr value, + _Out_ void* s, size_t s_len, _Out_ size_t* offsets, size_t offsets_len) { + TENSOR_READ_API_BEGIN + const auto* input = tensor.Data(); + auto len = static_cast(tensor.Shape().Size()); + if (offsets_len < len) { + return CreateONNXStatus(ONNXRUNTIME_FAIL, "space is not enough"); + } + { + size_t ret = 0; + for (size_t i = 0; i != len; ++i) { + ret += input[i].size(); + } + if (s_len < ret) { + return CreateONNXStatus(ONNXRUNTIME_FAIL, "space is not enough"); + } + } + size_t f = 0; + char* p = static_cast(s); + for (size_t i = 0; i != offsets_len; ++i, ++offsets) { + memcpy(p, input[i].data(), input[i].size()); + p += input[i].size(); + *offsets = f; + f += input[i].size(); + } + return nullptr; + API_IMPL_END +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeTensorProtoToONNXValue, _Inout_ ONNXRuntimeAllocator* allocator, + const void* input, int input_len, _Out_ ONNXValuePtr* out) { + API_IMPL_BEGIN + std::shared_ptr allocator_ = std::make_shared(allocator); + ::ONNX_NAMESPACE::TensorProto proto; + if (!proto.ParseFromArray(input, input_len)) { + return CreateONNXStatus(ONNXRUNTIME_FAIL, "parse input tensor proto failed"); + } + std::unique_ptr value = std::make_unique(); + Status st = onnxruntime::utils::TensorProtoToMLValue(proto, allocator_, nullptr, 0, *value); + if (!st.IsOK()) + return ToONNXStatus(st); + *out = reinterpret_cast(value.release()); + return nullptr; + API_IMPL_END +} + +ONNXRUNTIME_API(ONNXValuePtr, ONNXRuntimeONNXValueListGetNthValue, ONNXValueListPtr list, size_t index) { + auto v = reinterpret_cast<::onnxruntime::MLValue*>(list); + return reinterpret_cast(v + index); +} + +#define DEFINE_RELEASE_ONNX_RUNTIME_OBJECT_FUNCTION(INPUT_TYPE, REAL_TYPE) \ + ONNXRUNTIME_API(void, Release##INPUT_TYPE, INPUT_TYPE##Ptr value) { \ + delete reinterpret_cast(value); \ + } + +#define DEFINE_RELEASE_ONNX_RUNTIME_OBJECT_FUNCTION_FOR_ARRAY(INPUT_TYPE, REAL_TYPE) \ + ONNXRUNTIME_API(void, Release##INPUT_TYPE, INPUT_TYPE##Ptr value) { \ + delete[] reinterpret_cast(value); \ + } + +ONNXRUNTIME_API(void, ReleaseONNXEnv, ONNXEnv* env) { + delete env; +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeInferenceSessionGetInputCount, _In_ ONNXSessionPtr sess, _Out_ size_t* out) { + API_IMPL_BEGIN + auto session = reinterpret_cast<::onnxruntime::InferenceSession*>(sess); + std::pair p = session->GetModelInputs(); + if (!p.first.IsOK()) + return ToONNXStatus(p.first); + *out = p.second->size(); + return nullptr; + API_IMPL_END +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeInferenceSessionGetOutputCount, _In_ ONNXSessionPtr sess, _Out_ size_t* out) { + API_IMPL_BEGIN + auto session = reinterpret_cast<::onnxruntime::InferenceSession*>(sess); + std::pair p = session->GetModelOutputs(); + if (!p.first.IsOK()) + return ToONNXStatus(p.first); + *out = p.second->size(); + return nullptr; + API_IMPL_END +} + +static char* StrDup(const std::string& str, ONNXRuntimeAllocator* allocator) { + char* output_string = reinterpret_cast((*allocator)->Alloc(allocator, str.size() + 1)); + memcpy(output_string, str.c_str(), str.size()); + output_string[str.size()] = '\0'; + return output_string; +} + +static ONNXStatusPtr GetInputOutputNameImpl(_In_ ONNXSessionPtr sess, size_t index, + _Inout_ ONNXRuntimeAllocator* allocator, bool is_input, + _Out_ char** output) { + auto session = reinterpret_cast<::onnxruntime::InferenceSession*>(sess); + std::pair p = is_input ? session->GetModelInputs() : session->GetModelOutputs(); + if (!p.first.IsOK()) + return ToONNXStatus(p.first); + if (p.second == nullptr) + return CreateONNXStatus(ONNXRUNTIME_FAIL, "internal error"); + const InputDefList& defs = *p.second; + if (index >= defs.size()) + return CreateONNXStatus(ONNXRUNTIME_FAIL, "index out of range"); + *output = StrDup(defs[index]->Name(), allocator); + return nullptr; +} + +ONNXRUNTIME_API(int, ONNXRuntimeIsTensor, _In_ ONNXValuePtr value) { + auto v = reinterpret_cast<::onnxruntime::MLValue*>(value); + return v->IsTensor() ? 1 : 0; +} + +ONNXRUNTIME_API(void*, ONNXRuntimeAllocatorAlloc, _Inout_ ONNXRuntimeAllocator* ptr, size_t size) { + try { + return (*ptr)->Alloc(ptr, size); + } catch (std::exception&) { + return nullptr; + } +} + +ONNXRUNTIME_API(void, ONNXRuntimeAllocatorFree, _Inout_ ONNXRuntimeAllocator* ptr, void* p) { + try { + (*ptr)->Free(ptr, p); + } catch (std::exception&) { + } +} + +ONNXRUNTIME_API(const struct ONNXRuntimeAllocatorInfo*, ONNXRuntimeAllocatorGetInfo, _In_ const ONNXRuntimeAllocator* ptr) { + try { + return (*ptr)->Info(ptr); + } catch (std::exception&) { + return nullptr; + } +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeInferenceSessionGetInputName, _In_ ONNXSessionPtr sess, size_t index, + _Inout_ ONNXRuntimeAllocator* allocator, _Out_ char** output) { + API_IMPL_BEGIN + return GetInputOutputNameImpl(sess, index, allocator, true, output); + API_IMPL_END +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeInferenceSessionGetOutputName, _In_ ONNXSessionPtr sess, size_t index, + _Inout_ ONNXRuntimeAllocator* allocator, _Out_ char** output) { + API_IMPL_BEGIN + return GetInputOutputNameImpl(sess, index, allocator, false, output); + API_IMPL_END +} + +DEFINE_RELEASE_ONNX_RUNTIME_OBJECT_FUNCTION(ONNXValue, MLValue) +DEFINE_RELEASE_ONNX_RUNTIME_OBJECT_FUNCTION(ONNXSession, ::onnxruntime::InferenceSession) +DEFINE_RELEASE_ONNX_RUNTIME_OBJECT_FUNCTION_FOR_ARRAY(ONNXValueList, ::onnxruntime::MLValue) +DEFINE_RELEASE_ONNX_RUNTIME_OBJECT_FUNCTION_FOR_ARRAY(ONNXStatus, char) diff --git a/onnxruntime/core/session/tensor_type_and_shape.cc b/onnxruntime/core/session/tensor_type_and_shape.cc new file mode 100644 index 0000000000000..9f707380ef35b --- /dev/null +++ b/onnxruntime/core/session/tensor_type_and_shape.cc @@ -0,0 +1,90 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/tensor_type_and_shape_c_api.h" +#include "core/framework/onnx_object.h" +#include "core/framework/tensor_shape.h" +#include +#include +#include + +struct ONNXRuntimeTensorTypeAndShapeInfo { + public: + const ONNXObject* const cls; + std::atomic_int ref_count; + OnnxRuntimeTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; + onnxruntime::TensorShape shape; + + static ONNXRuntimeTensorTypeAndShapeInfo* Create() { + return new ONNXRuntimeTensorTypeAndShapeInfo(); + } + static uint32_t ONNXRUNTIME_API_STATUSCALL ReleaseImpl(void* this_) { + ONNXRuntimeTensorTypeAndShapeInfo* this_ptr = static_cast(this_); + if (--this_ptr->ref_count == 0) + delete this_ptr; + return 0; + } + + static uint32_t ONNXRUNTIME_API_STATUSCALL AddRefImpl(void* this_) { + ONNXRuntimeTensorTypeAndShapeInfo* this_ptr = static_cast(this_); + ++this_ptr->ref_count; + return 0; + } + + private: + ONNXRuntimeTensorTypeAndShapeInfo(); + ~ONNXRuntimeTensorTypeAndShapeInfo() { + assert(ref_count == 0); + } + ONNXRuntimeTensorTypeAndShapeInfo(const ONNXRuntimeTensorTypeAndShapeInfo& other) = delete; + ONNXRuntimeTensorTypeAndShapeInfo& operator=(const ONNXRuntimeTensorTypeAndShapeInfo& other) = delete; +}; + +constexpr ONNXObject shape_cls = { + ONNXRuntimeTensorTypeAndShapeInfo::AddRefImpl, + ONNXRuntimeTensorTypeAndShapeInfo::ReleaseImpl, +}; + +#define API_IMPL_BEGIN try { +#define API_IMPL_END \ + } \ + catch (std::exception & ex) { \ + return CreateONNXStatus(ONNXRUNTIME_RUNTIME_EXCEPTION, ex.what()); \ + } + +ONNXRUNTIME_API(ONNXRuntimeTensorTypeAndShapeInfo*, ONNXRuntimeCreateTensorTypeAndShapeInfo) { + return ONNXRuntimeTensorTypeAndShapeInfo::Create(); +} + +ONNXRuntimeTensorTypeAndShapeInfo::ONNXRuntimeTensorTypeAndShapeInfo() : cls(&shape_cls), ref_count(1) { +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeSetTensorElementType, _In_ ONNXRuntimeTensorTypeAndShapeInfo* this_ptr, enum OnnxRuntimeTensorElementDataType type) { + API_IMPL_BEGIN + this_ptr->type = type; + return nullptr; + API_IMPL_END +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeSetDims, _In_ ONNXRuntimeTensorTypeAndShapeInfo* this_ptr, _In_ const int64_t* dim_values, size_t dim_count) { + API_IMPL_BEGIN + this_ptr->shape = onnxruntime::TensorShape(dim_values, dim_count); + return nullptr; + API_IMPL_END +} + +ONNXRUNTIME_API(enum OnnxRuntimeTensorElementDataType, ONNXRuntimeGetTensorElementType, _In_ const struct ONNXRuntimeTensorTypeAndShapeInfo* info) { + return info->type; +} + +ONNXRUNTIME_API(size_t, ONNXRuntimeGetNumOfDimensions, _In_ const struct ONNXRuntimeTensorTypeAndShapeInfo* info) { + return info->shape.NumDimensions(); +} + +ONNXRUNTIME_API(void, ONNXRuntimeGetDimensions, _In_ const struct ONNXRuntimeTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, size_t dim_values_length) { + info->shape.CopyDims(dim_values, dim_values_length); +} + +ONNXRUNTIME_API(int64_t, ONNXRuntimeGetTensorShapeElementCount, _In_ const ONNXRuntimeTensorTypeAndShapeInfo* this_ptr) { + return this_ptr->shape.Size(); +} diff --git a/onnxruntime/core/util/eigen_common_wrapper.h b/onnxruntime/core/util/eigen_common_wrapper.h new file mode 100644 index 0000000000000..30c9feb2ff6a7 --- /dev/null +++ b/onnxruntime/core/util/eigen_common_wrapper.h @@ -0,0 +1,23 @@ +//----------------------------------------------------------------------------- +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +// +//----------------------------------------------------------------------------- +#pragma once + +// build/external/eigen/unsupported/Eigen/CXX11/src/Tensor/TensorEvaluator.h:162:71: +// error: ignoring attributes on template argument "Eigen::PacketType::type {aka __vector(4) float}" [-Werror=ignored-attributes] +#if defined(__GNUC__) +#pragma GCC diagnostic push +#if __GNUC__ >= 6 +#pragma GCC diagnostic ignored "-Wignored-attributes" +#endif +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +#include "unsupported/Eigen/CXX11/Tensor" + +#if defined(__GNUC__) && __GNUC__ >= 6 +#pragma GCC diagnostic pop +#endif diff --git a/onnxruntime/core/util/math.h b/onnxruntime/core/util/math.h new file mode 100644 index 0000000000000..62db5e9fabe51 --- /dev/null +++ b/onnxruntime/core/util/math.h @@ -0,0 +1,477 @@ +/** +* Derived from caffe2, need copyright announcement here. +*/ + +/** +* Copyright (c) 2016-present, Facebook, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#pragma once + +// This is a simple translation from the old Caffe math interfaces. We aim to +// still keep it simple, so all platforms would be able to support it fairly +// easily. + +// We include the cblas header here so that we can obtain the macros from cblas. +extern "C" { +#include "core/framework/cblas.h" +} + +#include "core/common/common.h" +#include "core/framework/data_types.h" +#include "core/framework/tensor.h" + +namespace onnxruntime { + +enum StorageOrder { + UNKNOWN = 0, + NHWC = 1, + NCHW = 2, +}; + +#define FLOAT_TYPE DataTypeImpl::GetType() + +namespace math { + +template +void Exp(const int N, const T* x, T* y, Provider* provider); +template +void Log(const int N, const T* x, T* y, Provider* provider); +template +void Cos(const int N, const T* x, T* y, Provider* provider); +template +void Sin(const int N, const T* x, T* y, Provider* provider); +template +void SinCos(const int N, const T* x, T* ys, T* yc, Provider* provider); +template +void Abs(const int N, const T* x, T* y, Provider* provider); +template +void Sqrt(const int N, const T* x, T* y, Provider* provider); +template +void InvSqrt(const int N, const T* x, T* y, Provider* provider); +template +void Sqr(const int N, const T* x, T* y, Provider* provider); + +template +void Not(const int N, const T* x, T* y, Provider* provider); + +template +void Powx(const int N, const T* a, const T b, T* y, Provider* provider); + +#define DECLARE_BINARY_OP_BINARY_RESULT(name) \ + template \ + void name(const int N, const T* a, const T* b, bool* y, Provider* provider); \ + template \ + void name##ToRow( \ + const int M, \ + const int N, \ + const T* a, \ + const T* b, \ + bool* y, \ + Provider* provider); + +DECLARE_BINARY_OP_BINARY_RESULT(LT); +DECLARE_BINARY_OP_BINARY_RESULT(LE); +DECLARE_BINARY_OP_BINARY_RESULT(GT); +DECLARE_BINARY_OP_BINARY_RESULT(GE); + +DECLARE_BINARY_OP_BINARY_RESULT(And); +DECLARE_BINARY_OP_BINARY_RESULT(Or); +DECLARE_BINARY_OP_BINARY_RESULT(Xor); + +#undef DECLARE_BINARY_OP_BINARY_RESULT + +#define DECLARE_BINARY_OP(name) \ + template \ + void name(const int N, const T* a, const T* b, T* y, Provider* provider); \ + template \ + void name##ToRow( \ + const int M, \ + const int N, \ + const T* a, \ + const T* b, \ + T* y, \ + Provider* provider); \ + template \ + void name##ToRow( \ + const int M, const int N, const T* x, T* y, Provider* provider); \ + template \ + void name##ToCol( \ + const int M, const int N, const T* x, T* y, Provider* provider); + +DECLARE_BINARY_OP(Add); +DECLARE_BINARY_OP(Sub); +DECLARE_BINARY_OP(Mul); +DECLARE_BINARY_OP(Div); + +#undef DECLARE_BINARY_OP + +template +void ReduceMin( + const int N, + const T* x, + T* y, + Tensor* scratch_ptr, + Provider* provider); +template +void ReduceMax( + const int N, + const T* x, + T* y, + Tensor* scratch_ptr, + Provider* provider); + +// Adds batch sub-tensors elementwise to output. Stripe is the stripe length +// and N is the number of elements to add (size of Y). +template +void AddStripedBatch( + const int N, + const T* first, + T* y, + const int stripe, + const int batch, + Provider* provider); + +// Compute the row-wise max of a N*D matrix X, and write it to a N +// dimensional vector y. +template +void RowwiseMax(const int N, const int D, const T* x, T* y, + Provider* provider); + +// Compute the column-wise max of a N*D matrix X, and write it to a D +// dimensional vector y. +template +void ColwiseMax(const int N, const int D, const T* x, T* y, + Provider* provider); + +// Elemwise maximum of vector x and vector y. z[i] = max(x[i], y[i]) +template +void ElemwiseMax(const int N, const T* x, const T* y, T* z, Provider* provider); + +// Elemwise maximum of vector x and scalar alpha. y[i] = max(x[i], alpha) +template +void Maximum( + const int N, + const float alpha, + const T* x, + T* y, + Provider* provider); + +// Decaf gemm provides a simpler interface to the gemm functions, with the +// limitation that the data has to be contiguous in memory. +template +void Gemm( + const CBLAS_TRANSPOSE TransA, + const CBLAS_TRANSPOSE TransB, + const int64_t M, + const int64_t N, + const int64_t K, + const float alpha, + const T* A, + const T* B, + const float beta, + T* C, + Provider* provider, + //Caffe2 use this type to control on GPU, what presicion do we want to do the calculation + //But not sure is this a good design for us. Keep it here for now. + MLDataType math_type = FLOAT_TYPE); + +// We also provide a gemm that has explicit lda, ldb and ldc specified. +// In most cases you probably want to use the function above, though. +template +void GemmEx( + const CBLAS_TRANSPOSE TransA, + const CBLAS_TRANSPOSE TransB, + const int M, + const int N, + const int K, + const T alpha, + const T* A, + const int lda, + const T* B, + const int ldb, + const T beta, + T* C, + const int ldc, + Provider* provider); + +// GemmBatched provides a simple abstraction into library routines +template +void GemmBatched( + const CBLAS_TRANSPOSE TransA, + const CBLAS_TRANSPOSE TransB, + const int A_size, + const int A_batches, + const int B_size, + const int B_batches, + const int M, + const int N, + const int K, + const float alpha, + const T* A, + const T* B, + const float beta, + T* C, + Provider* provider, + Tensor* scratch = nullptr, + MLDataType math_type = DataTypeImpl::FLOAT_TYPE); + +// Gemv always takes in a M*N matrix A, and depending on whether we set TransA +// to Trans, the output is: +// CblasNoTrans: x is an N dim vector and y is an M dim vector. +// CblasTrans: x is an M dim vector and y is an N dim vector. +template +void Gemv( + const CBLAS_TRANSPOSE TransA, + const int M, + const int N, + const float alpha, + const T* A, + const T* x, + const float beta, + T* y, + Provider* provider, + MLDataType math_type = DataTypeImpl::FLOAT_TYPE); +template +void Set(const int64_t N, const T alpha, T* X, Provider* provider); + +template +void RandUniform(const int n, const T a, const T b, T* r, + Provider* provider); + +template +void RandUniformUnique( + const size_t n, + const T a, + const T b, + T* r, + const size_t m, + const T* avoid, + Provider* provider); + +template +void RandGaussian( + const int n, + const T mean, + const T std, + T* r, + Provider* provider); + +// Dot matrix of vector a and b, and writes the result to a single value y. +template +void Dot(const int N, const T* a, const T* b, T* y, Provider* provider); + +// Sum of vector x, and writes the result to a single value y. +template +void Sum(const int N, const T* x, T* y, Provider* provider, + Tensor* scratch_ptr = nullptr); + +// Sum of squares of vector x, and writes the result to a single value y. +template +void SumSqr( + const int N, + const T* x, + T* y, + Provider* provider, + Tensor* scratch_ptr = nullptr); + +// Select does index selection of the rows a N*D matrix x, and gives the N +// dimensional vector y that contains the selected data. +template +void Select(const int N, const int D, const T* x, const int* idx, T* y, + Provider* provider); + +template +void Scale(const int N, const float alpha, const T* x, T* y, Provider* provider); + +// Different from the Scale function above, if alpha is passed in +// as a pointer, we will assume that it lives on the correct execution provider, +// for example on GPU. +template +void Scale(const int N, const float* alpha, const T* x, T* y, Provider* provider); + +template +void Axpy(const int N, const float alpha, const T* x, T* y, Provider* provider); + +// Different from the Axpy function above, if alpha is passed in +// as a pointer, we will assume that it lives on the correct execution provider, +// for example on GPU. +template +void Axpy(const int N, const float* alpha, const T* x, T* y, Provider* provider); + +template +void Axpby( + const int N, + const float alpha, + const T* x, + const T b, + T* y, + Provider* provider); + +template +void Im2colNd( + const T* data_img, + const int64_t* im_shape, + const int64_t* col_shape, + const int64_t img_size, + const int64_t col_size, + const int64_t* kernel_shape, + const int64_t* stride, + const int64_t* dilation, + const int64_t* pad, + const int64_t N, + T* data_col, + Provider* provider, + bool accumulate_output = false); + +template +void Col2imNd( + const T* data_col, + const int64_t* img_shape, + const int64_t* col_shape, + const int64_t img_size, + const int64_t col_size, + const int64_t* kernel_shape, + const int64_t* stride, + const int64_t* dilation, + const int64_t* pad, + const int64_t N, + T* data_img, + Provider* provider); + +template +void Im2col( + const T* data_im, + const int64_t channels, + const int64_t height, + const int64_t width, + const int64_t kernel_h, + const int64_t kernel_w, + const int64_t dilation_h, + const int64_t dilation_w, + const int64_t pad_t, + const int64_t pad_l, + const int64_t pad_b, + const int64_t pad_r, + const int64_t stride_h, + const int64_t stride_w, + T* data_col, + Provider* provider); + +template +void Col2im( + const T* data_col, + const int64_t channels, + const int64_t height, + const int64_t width, + const int64_t patch_h, + const int64_t patch_w, + const int64_t dilation_h, + const int64_t dilation_w, + const int64_t pad_t, + const int64_t pad_l, + const int64_t pad_b, + const int64_t pad_r, + const int64_t stride_h, + const int64_t stride_w, + T* data_im, + Provider* provider); + +// Todo: should we provide a better copy method signature? +using TypedCopy = void (*)(const void*, void*, size_t); + +template +void CopyMatrix( + const size_t item_size, + const int M, + const int N, + const void* A, + const int lda, + void* B, + const int ldb, + Provider* provider, + TypedCopy copy = nullptr); + +template +void CopyVector(const int N, const T* A, T* B, Provider* provider); + +uint32_t randomNumberSeed(); + +// Function uses casting from int64_t to uint64_t to compare if value of +// parameter a is greater or equal to zero and lower than value of +// parameter b. The b parameter is of type signed and is always +// positive, +// therefore its value is always lower than 0x800... where casting +// negative value of a parameter converts it to value higher than +// 0x800... +// The casting allows to use one condition instead of two. +inline bool is_a_ge_zero_and_a_lt_b(int64_t a, int64_t b) { + return static_cast(a) < static_cast(b); +} + +// Calculates ceil(a / b). User must be careful to ensure that there +// is no overflow or underflow in the calculation. +template +constexpr T divUp(T a, T b) { + return (a + b - (T)1) / b; +} + +// Rounds a up to the next highest multiple of b. User must be careful +// to ensure that there is no overflow or underflow in the calculation +// of divUp. +template +constexpr T roundUp(T a, T b) { + return divUp(a, b) * b; +} + +// Returns true if the given integer type is a power-of-2 (positive only) +// Note(jiayq): windows reported an error per +// https://github.com/caffe2/caffe2/issues/997 +// and as a result will make it a macro. +#ifdef _MSC_VER +#define integerIsPowerOf2(v) ((v) && !((v) & ((v)-1))) +#else // _MSC_VER +template +constexpr bool integerIsPowerOf2(T v) { + return (v && !(v & (v - 1))); +} +#endif // _MSC_VER + +// Returns log2(n) for a positive integer type +template +constexpr int integerLog2(T n, int p = 0) { + return (n <= 1) ? p : integerLog2(n / 2, p + 1); +} + +// Returns the next highest power-of-2 for an integer type +template +constexpr T integerNextHighestPowerOf2(T v) { + return (integerIsPowerOf2(v) ? (T)2 * v : ((T)1 << (integerLog2(v) + 1))); +} + +// Rounds a up to the next highest multiple of b, which is power-of-2. User must be careful +// to ensure that there is no overflow or underflow in the calculation +// of divUp. +template +constexpr T roundUpPow2(T a) { + return (a + (b - 1)) & (~(b - 1)); +} + +uint16_t floatToHalf(float f); + +float halfToFloat(uint16_t h); + +} // namespace math +} // namespace onnxruntime diff --git a/onnxruntime/core/util/math_cpu.cc b/onnxruntime/core/util/math_cpu.cc new file mode 100644 index 0000000000000..9805a17410cac --- /dev/null +++ b/onnxruntime/core/util/math_cpu.cc @@ -0,0 +1,1377 @@ +/** +* Copyright (c) 2016-present, Facebook, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +// Implements the math functions for CPU. +// The implementation in this file allows us to route the underlying numerical +// computation library to different backends. Notably: +// (1) For all BLAS-related functions, one can explicitly request a BLAS backend +// such as MKL, openblas or Atlas. To see the set of supported backends +// currently provided, check //third_party/blas/. +// (2) If one chooses to link against MKL, we utilize MKL's vector math library +// (VML) for a few functions such as Exp and Log. +// (3) Fallback implementations are provided in Eigen for cross-platform +// support. Since Eigen is a header-only library and supports a number of +// platforms, it allows one to quickly port Caffe2 to different platforms +// where BLAS may not be present. +// Modifications Copyright (c) Microsoft. + +#include +#include +#include +#include +#include +#include "core/platform/env.h" +#include "core/common/logging/logging.h" +#include "core/providers/cpu/cpu_execution_provider.h" +#include "core/util/math.h" +#include "core/util/math_cpuonly.h" +#include "Eigen/src/Core/arch/CUDA/Half.h" + +#if defined(USE_MLAS) +#include "core/mlas/inc/mlas.h" +#endif + +#ifdef USE_MKLDNN +#include "mkldnn.h" +#endif + +namespace onnxruntime { +namespace math { + +//////////////////////////////////////////////////////////////////////////////// +// BLAS alternatives. +// Depending on whether we have specified an external BLAS library or not, we +// will delegate the Caffe math functions that are BLAS-related to either the +// CBLAS call or the Eigen implementation. +//////////////////////////////////////////////////////////////////////////////// +#ifdef USE_EIGEN_FOR_BLAS + +// Caffe2 gemm provides a simpler interface to the gemm functions, with the +// limitation that the data has to be contiguous in memory. +// +// The gemm call implements the following operation: +// +// C = alpha * op(A) * op(B) + beta * C +// +// where op(A) has size M x K, op(B) has size K x N, and C has size M x N. Each +// of A, B, and C are matrices and alpha and beta are scalars. Note that the +// most common use case of gemm will involve setting alpha to 1 and beta to 0. +// +// op(A) and op(B) represent the transformations that are done to A and B before +// the matrix multiply; depending on the flags set, op(A) is equal to A or A^T +// (transpose) if the argument TransA or TransB is set to CblasNoTrans or +// CblasTrans, respectively, for each of A and B. +template <> +void Gemm( + const CBLAS_TRANSPOSE TransA, + const CBLAS_TRANSPOSE TransB, + const int64_t M, + const int64_t N, + const int64_t K, + const float alpha, + const float* A, + const float* B, + const float beta, + float* C, + CPUMathUtil* /*provider*/, + MLDataType /*math_type*/) { +#if defined(USE_MKLDNN) + int lda = (int)((TransA == CblasTrans) ? M : K); + int ldb = (int)((TransB == CblasTrans) ? K : N); + int M_ = (int)M; + int N_ = (int)N; + int K_ = (int)K; + // mkldnn_sgemm expects col major matrices, so we need to swap the operands A and B + auto status = mkldnn_sgemm(TransB == CblasNoTrans ? "N" : "T", + TransA == CblasNoTrans ? "N" : "T", + &N_, &M_, &K_, + &alpha, B, &ldb, + A, &lda, + &beta, C, &N_); + if (status != mkldnn_success) { + ONNXRUNTIME_THROW("mkldnn_sgemm failed with status: ", status); + } +#elif defined(USE_MLAS) + int lda = (int)((TransA == CblasNoTrans) ? K : M); + int ldb = (int)((TransB == CblasNoTrans) ? N : K); + MlasSgemm(TransA, TransB, M, N, K, alpha, A, lda, B, ldb, beta, C, N); +#else + auto C_mat = EigenMatrixMap(C, N, M); + if (beta == 0) { + C_mat.setZero(); + } else { + C_mat *= beta; + } + switch (TransA) { + case CblasNoTrans: { + switch (TransB) { + case CblasNoTrans: + C_mat.noalias() += alpha * (ConstEigenMatrixMap(B, N, K) * + ConstEigenMatrixMap(A, K, M)); + return; + case CblasTrans: + C_mat.noalias() += alpha * (ConstEigenMatrixMap(B, K, N).transpose() * + ConstEigenMatrixMap(A, K, M)); + return; + default: + ONNXRUNTIME_THROW("CblasNoTrans Unexpected CBLAS_TRANSPOSE for TransB of ", TransB); + } + } + case CblasTrans: { + switch (TransB) { + case CblasNoTrans: + C_mat.noalias() += alpha * (ConstEigenMatrixMap(B, N, K) * + ConstEigenMatrixMap(A, M, K).transpose()); + return; + case CblasTrans: + C_mat.noalias() += alpha * (ConstEigenMatrixMap(B, K, N).transpose() * + ConstEigenMatrixMap(A, M, K).transpose()); + return; + default: + ONNXRUNTIME_THROW("CblasTrans Unexpected CBLAS_TRANSPOSE for TransB of ", TransB); + } + } + default: + ONNXRUNTIME_THROW("Unexpected CBLAS_TRANSPOSE for TransA of ", TransA); + } +#endif +} + +template <> +void GemmEx( + const CBLAS_TRANSPOSE TransA, + const CBLAS_TRANSPOSE TransB, + const int M, + const int N, + const int K, + const float alpha, + const float* A, + const int lda, + const float* B, + const int ldb, + const float beta, + float* C, + const int ldc, + CPUMathUtil*) { +#if defined(USE_MKLDNN) + // mkldnn_sgemm expects col major matrices, so we need to swap the operands A and B + auto status = mkldnn_sgemm(TransB == CblasNoTrans ? "N" : "T", + TransA == CblasNoTrans ? "N" : "T", + &N, &M, &K, + &alpha, B, &ldb, + A, &lda, + &beta, C, &ldc); + if (status != mkldnn_success) { + ONNXRUNTIME_THROW("mkldnn_sgemm failed with status: ", status); + } +#elif defined(USE_MLAS) + MlasSgemm(TransA, TransB, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); +#else + using OuterStride = Eigen::OuterStride; + using StridedMap = Eigen::Map; + using ConstStridedMap = Eigen::Map; + auto C_mat = StridedMap(C, N, M, OuterStride(ldc)); + if (beta == 0) { + C_mat.setZero(); + } else { + C_mat *= beta; + } + switch (TransA) { + case CblasNoTrans: { + switch (TransB) { + case CblasNoTrans: + C_mat.noalias() += + alpha * (ConstStridedMap(B, N, K, OuterStride(ldb)) * + ConstStridedMap(A, K, M, OuterStride(lda))); + return; + case CblasTrans: + C_mat.noalias() += + alpha * (ConstStridedMap(B, K, N, OuterStride(ldb)).transpose() * + ConstStridedMap(A, K, M, OuterStride(lda))); + return; + default: + ONNXRUNTIME_THROW("CblasNoTrans Unexpected CBLAS_TRANSPOSE for TransB of ", TransB); + } + } + case CblasTrans: { + switch (TransB) { + case CblasNoTrans: + C_mat.noalias() += + alpha * (ConstStridedMap(B, N, K, OuterStride(ldb)) * + ConstStridedMap(A, M, K, OuterStride(lda)).transpose()); + return; + case CblasTrans: + C_mat.noalias() += + alpha * (ConstStridedMap(B, K, N, OuterStride(ldb)).transpose() * + ConstStridedMap(A, M, K, OuterStride(lda)).transpose()); + return; + default: + ONNXRUNTIME_THROW("CblasTrans Unexpected CBLAS_TRANSPOSE for TransB of ", TransB); + } + } + default: + ONNXRUNTIME_THROW("Unexpected CBLAS_TRANSPOSE for TransA of ", TransA); + } +#endif +} + +template <> +void Gemv( + const CBLAS_TRANSPOSE TransA, + const int M, + const int N, + const float alpha, + const float* A, + const float* x, + const float beta, + float* y, + CPUMathUtil* /*provider*/, + MLDataType /*math_type*/) { + EigenVectorMap y_vec(y, TransA == CblasNoTrans ? M : N); + if (beta == 0) { + // In Caffe2 we often do a lazy initialization, which may contain NaNs in + // the float values. As a result, if beta is 0, we explicitly do a setzero. + y_vec.setZero(); + } else { + y_vec *= beta; + } + switch (TransA) { + case CblasNoTrans: { + y_vec.noalias() += alpha * (ConstEigenMatrixMap(A, N, M).transpose() * + ConstEigenVectorMap(x, N)); + return; + } + case CblasTrans: { + y_vec.noalias() += alpha * (ConstEigenMatrixMap(A, N, M) * + ConstEigenVectorMap(x, M)); + return; + } + default: + ONNXRUNTIME_THROW("Gemv float found an unexpected CBLAS_TRANSPOSE input of", TransA); + } +} + +#define SPECIALIZED_SCALE(T) \ + template <> \ + void Scale( \ + const int n, const float alpha, const T* x, T* y, CPUMathUtil* /*provider*/) { \ + EigenVectorMap(y, n) = ConstEigenVectorMap(x, n) * alpha; \ + } \ + template <> \ + void Scale( \ + const int n, \ + const float* alpha, \ + const T* x, \ + T* y, \ + CPUMathUtil* /*provider*/) { \ + EigenVectorMap(y, n) = ConstEigenVectorMap(x, n) * (*alpha); \ + } +SPECIALIZED_SCALE(float) +#undef SPECIALIZED_SCALE + +#define SPECIALIZED_DOT(T) \ + template <> \ + void Dot( \ + const int N, const T* a, const T* b, T* y, \ + CPUMathUtil* /*provider*/) { \ + *y = ConstEigenVectorMap(a, N).dot(ConstEigenVectorMap(b, N)); \ + } +SPECIALIZED_DOT(float) +#undef SPECIALIZED_DOT + +#define SPECIALIZED_AXPY(T) \ + template <> \ + void Axpy( \ + const int N, const T alpha, const T* x, T* Y, CPUMathUtil* /*provider*/) { \ + EigenVectorMap(Y, N) += ConstEigenVectorMap(x, N) * alpha; \ + } \ + template <> \ + void Axpy( \ + const int N, const T* alpha, const T* x, T* Y, CPUMathUtil* /*provider*/) { \ + EigenVectorMap(Y, N) += ConstEigenVectorMap(x, N) * (*alpha); \ + } +SPECIALIZED_AXPY(float) +#undef SPECIALIZED_AXPY + +#define SPECIALIZED_AXPBY(T) \ + template <> \ + void Axpby(const int N, const T alpha, const T* x, \ + const T beta, T* y, CPUMathUtil* /*context*/) { \ + EigenVectorMap y_vec(y, N); \ + y_vec = y_vec * beta + ConstEigenVectorMap(x, N) * alpha; \ + } +SPECIALIZED_AXPBY(float) +#undef SPECIALIZED_AXPBY + +#else // USE_EIGEN_FOR_BLAS + +template <> +void Gemm( + const CBLAS_TRANSPOSE TransA, + const CBLAS_TRANSPOSE TransB, + const int64_t M, + const int64_t N, + const int64_t K, + const float alpha, + const float* A, + const float* B, + const float beta, + float* C, + CPUMathUtil* /*context*/, + MLDataType /*math_type*/) { + int lda = (TransA == CblasNoTrans) ? K : M; + int ldb = (TransB == CblasNoTrans) ? N : K; + cblas_sgemm(CblasRowMajor, TransA, TransB, M, N, K, alpha, A, lda, B, ldb, + beta, C, N); +} + +template <> +void GemmEx( + const CBLAS_TRANSPOSE TransA, + const CBLAS_TRANSPOSE TransB, + const int M, + const int N, + const int K, + const float alpha, + const float* A, + const int lda, + const float* B, + const int ldb, + const float beta, + float* C, + const int ldc, + CPUMathUtil* /*context*/) { + cblas_sgemm(CblasRowMajor, TransA, TransB, M, N, K, alpha, A, lda, B, ldb, + beta, C, ldc); +} + +template <> +void Gemv( + const CBLAS_TRANSPOSE TransA, + const int M, + const int N, + const float alpha, + const float* A, + const float* x, + const float beta, + float* y, + CPUMathUtil* /*context*/, + MLDataType /*math_type*/) { + cblas_sgemv(CblasRowMajor, TransA, M, N, alpha, A, N, x, 1, beta, y, 1); +} + +#define CAFFE2_SPECIALIZED_SCALE(T, prefix) \ + template <> \ + void Scale( \ + const int n, const float alpha, const T* x, T* y, CPUMathUtil*) { \ + if (y != x) \ + cblas_##prefix##copy(n, x, 1, y, 1); \ + cblas_##prefix##scal(n, static_cast(alpha), y, 1); \ + } \ + template <> \ + void Scale( \ + const int n, const float* alpha, const T* x, T* y, CPUMathUtil*) { \ + if (y != x) \ + cblas_##prefix##copy(n, x, 1, y, 1); \ + cblas_##prefix##scal(n, static_cast(*alpha), y, 1); \ + } +CAFFE2_SPECIALIZED_SCALE(float, s) +#undef CAFFE2_SPECIALIZED_SCALE + +#define CAFFE2_SPECIALIZED_DOT(T, prefix) \ + template <> \ + void Dot( \ + const int N, const T* a, const T* b, T* y, CPUMathUtil*) { \ + *y = cblas_##prefix##dot(N, a, 1, b, 1); \ + } +CAFFE2_SPECIALIZED_DOT(float, s) +#undef CAFFE2_SPECIALIZED_DOT + +#define CAFFE2_SPECIALIZED_AXPY(T, prefix) \ + template <> \ + void Axpy( \ + const int N, const T alpha, const T* x, T* y, CPUMathUtil*) { \ + cblas_##prefix##axpy(N, alpha, x, 1, y, 1); \ + } \ + template <> \ + void Axpy( \ + const int N, const T* alpha, const T* x, T* y, CPUMathUtil*) { \ + cblas_##prefix##axpy(N, *alpha, x, 1, y, 1); \ + } +CAFFE2_SPECIALIZED_AXPY(float, s) +#undef CAFFE2_SPECIALIZED_AXPY + +#define CAFFE2_SPECIALIZED_AXPBY(T, prefix) \ + template <> \ + void Axpby( \ + const int N, \ + const T alpha, \ + const T* x, \ + const T beta, \ + T* y, \ + CPUMathUtil*) { \ + cblas_##prefix##scal(N, beta, y, 1); \ + cblas_##prefix##axpy(N, alpha, x, 1, y, 1); \ + } +CAFFE2_SPECIALIZED_AXPBY(float, s) +#undef CAFFE2_SPECIALIZED_AXPBY + +#endif // USE_EIGEN_FOR_BLAS + +template <> +void GemmBatched( + const CBLAS_TRANSPOSE TransA, + const CBLAS_TRANSPOSE TransB, + const int A_size, + const int A_batches, + const int B_size, + const int B_batches, + const int M, + const int N, + const int K, + const float /*alpha*/, + const float* A, + const float* B, + const float /*beta*/, + float* C, + CPUMathUtil* provider, + Tensor*, /* scratch */ + MLDataType /* math_type */) { + auto a_offset = A_size / A_batches; + auto b_offset = B_size / B_batches; + auto y_offset = M * N; + // loop over matrices in the batch + for (int i = 0; i < A_batches; ++i) { + math::Gemm( + TransA, + TransB, + M, + N, + K, + 1, + A + a_offset * i, + B + b_offset * i, + 0, + C + y_offset * i, + provider); + } +} + +// MKL will be implmenet as an execution provider +//////////////////////////////////////////////////////////////////////////////// +// MKL VML alternatives. +// Depending on whether we are using MKL, we will delegate the Caffe math +// functions that are VML-related to either the VML call or the Eigen +// implementation. If you are setting the flags (such as AVX) right for your CPU +// architecture, usually Eigen will deliver a throughput as fast as the VML +// functions. +//////////////////////////////////////////////////////////////////////////////// + +#define DELEGATE_SIMPLE_UNARY_FUNCTION(T, Funcname, expr) \ + template <> \ + void Funcname(const int N, const T* x, T* y, CPUMathUtil*) { \ + EigenVectorMap(y, N) = ConstEigenVectorMap(x, N).array().expr(); \ + } +DELEGATE_SIMPLE_UNARY_FUNCTION(float, Exp, exp) +DELEGATE_SIMPLE_UNARY_FUNCTION(float, Log, log) +DELEGATE_SIMPLE_UNARY_FUNCTION(float, Cos, cos) +DELEGATE_SIMPLE_UNARY_FUNCTION(float, Sin, sin) +DELEGATE_SIMPLE_UNARY_FUNCTION(float, Abs, abs) +DELEGATE_SIMPLE_UNARY_FUNCTION(float, Sqrt, sqrt) +DELEGATE_SIMPLE_UNARY_FUNCTION(float, InvSqrt, rsqrt) +DELEGATE_SIMPLE_UNARY_FUNCTION(float, Sqr, square) +#undef DELEGATE_SIMPLE_UNARY_FUNCTION + +#define DELEGATE_SINCOS_FUNCTION(T) \ + template <> \ + void SinCos( \ + const int N, const T* x, T* ys, T* yc, CPUMathUtil*) { \ + EigenVectorMap(ys, N) = ConstEigenVectorMap(x, N).array().sin(); \ + EigenVectorMap(yc, N) = ConstEigenVectorMap(x, N).array().cos(); \ + } +DELEGATE_SINCOS_FUNCTION(float) +DELEGATE_SINCOS_FUNCTION(double) +#undef DELEGATE_SINCOS_FUNCTION + +#define DELEGATE_POWX_FUNCTION(T) \ + template <> \ + void Powx(const int N, const T* a, T b, T* y, CPUMathUtil*) { \ + EigenVectorMap(y, N) = ConstEigenVectorMap(a, N).array().pow(b); \ + } +DELEGATE_POWX_FUNCTION(float) +#undef DELEGATE_POWX_FUNCTION + +#define EIGEN_SIMPLE_BINARY_FUNCTION(T, Funcname, expr) \ + template <> \ + void Funcname( \ + const int N, const T* a, const T* b, T* y, \ + CPUMathUtil*) { \ + EigenVectorMap(y, N) = \ + ConstEigenVectorMap(a, N).array() expr \ + ConstEigenVectorMap(b, N) \ + .array(); \ + } + +#define DEFINE_SIMPLE_BINARY_FUNCTION(Funcname, expr) \ + EIGEN_SIMPLE_BINARY_FUNCTION(float, Funcname, expr) \ + EIGEN_SIMPLE_BINARY_FUNCTION(int32_t, Funcname, expr) \ + EIGEN_SIMPLE_BINARY_FUNCTION(int64_t, Funcname, expr) + +DEFINE_SIMPLE_BINARY_FUNCTION(Add, +) +DEFINE_SIMPLE_BINARY_FUNCTION(Sub, -) +DEFINE_SIMPLE_BINARY_FUNCTION(Mul, *) +DEFINE_SIMPLE_BINARY_FUNCTION(Div, /) + +#undef EIGEN_SIMPLE_BINARY_FUNCTION +#undef DEFINE_FLOAT_BINARY_FUNCTION + +//////////////////////////////////////////////////////////////////////////////// +// common math functions being used in Caffe that do not have a BLAS or MKL +// equivalent. For all these functions, we will simply implement them either via +// Eigen or via custom code. +//////////////////////////////////////////////////////////////////////////////// + +#define SPECIALIZED_REDUCEMIN(T) \ + template <> \ + void ReduceMin( \ + const int N, \ + const T* x, \ + T* y, \ + Tensor* /*scratch_ptr*/, \ + CPUMathUtil* /*context*/) { \ + *y = *std::min_element(x, x + N); \ + } +SPECIALIZED_REDUCEMIN(float) +#undef SPECIALIZED_REDUCEMIN + +#define SPECIALIZED_REDUCEMAX(T) \ + template <> \ + void ReduceMax( \ + const int N, \ + const T* x, \ + T* y, \ + Tensor* /*scratch_ptr*/, \ + CPUMathUtil* /*context*/) { \ + *y = *std::max_element(x, x + N); \ + } +SPECIALIZED_REDUCEMAX(float) +SPECIALIZED_REDUCEMAX(int32_t) +SPECIALIZED_REDUCEMAX(int64_t) + +#undef SPECIALIZED_REDUCEMAX + +#define SPECIALIZED_ROWWISEMAX(T) \ + template <> \ + void RowwiseMax( \ + const int N, const int D, const T* x, T* y, CPUMathUtil*) { \ + EigenVectorMap(y, N) = \ + ConstEigenMatrixMap(x, D, N).colwise().maxCoeff(); \ + } +SPECIALIZED_ROWWISEMAX(float) +#undef SPECIALIZED_ROWWISEMAX + +#define SPECIALIZED_COLWISEMAX(T) \ + template <> \ + void ColwiseMax( \ + const int N, const int D, const T* x, T* y, CPUMathUtil*) { \ + EigenVectorMap(y, D) = \ + ConstEigenMatrixMap(x, D, N).rowwise().maxCoeff(); \ + } +SPECIALIZED_COLWISEMAX(float) +#undef SPECIALIZED_COLWISEMAX + +#define SPECIALIZED_ELEMWISEMAX(T) \ + template <> \ + void ElemwiseMax( \ + const int N, const T* x, const T* y, T* z, CPUMathUtil* /*context*/) { \ + std::transform(x, x + N, y, z, [](const T& x_i, const T& y_i) { \ + return std::max(x_i, y_i); \ + }); \ + } +SPECIALIZED_ELEMWISEMAX(float) +#undef SPECIALIZED_ELEMWISEMAX + +#define SPECIALIZED_MAXIMUM(T) \ + template <> \ + void Maximum( \ + const int N, const float alpha, const T* x, T* y, CPUMathUtil* /*provider*/) { \ + std::transform( \ + x, x + N, y, [&alpha](const T& x_i) { return std::max(x_i, alpha); }); \ + } +SPECIALIZED_MAXIMUM(float) +#undef SPECIALIZED_MAXIMUM + +// AddToRow and AddToCol adds the corresponding row/col vector b to the matrix a +// of shape M x N. The actual implementation uses eigen which is column major, +// so notice the row/column swap in the actual implementation. +#define DELEGATE_BROADCAST_BINARY_FUNCTION(T, Funcname, expr) \ + template <> \ + void Funcname##ToRow( \ + const int M, const int N, const T* a, const T* b, T* y, CPUMathUtil*) { \ + EigenArrayMap(y, N, M) = ConstEigenArrayMap(a, N, M).colwise() \ + expr ConstEigenVectorArrayMap(b, N); \ + } \ + /* inplace versions */ \ + template <> \ + void Funcname##ToRow( \ + const int M, const int N, const T* x, T* y, CPUMathUtil*) { \ + EigenArrayMap(y, N, M).colwise() expr## = \ + ConstEigenVectorArrayMap(x, N); \ + } \ + template <> \ + void Funcname##ToCol( \ + const int M, const int N, const T* x, T* y, CPUMathUtil*) { \ + EigenArrayMap(y, N, M).rowwise() expr## = \ + ConstEigenVectorArrayMap(x, M).transpose(); \ + } + +#define DEFINE_BROADCAST_BINARY_FUNCTION(name, op) \ + DELEGATE_BROADCAST_BINARY_FUNCTION(int32_t, name, op) \ + DELEGATE_BROADCAST_BINARY_FUNCTION(int64_t, name, op) \ + DELEGATE_BROADCAST_BINARY_FUNCTION(float, name, op) + +DEFINE_BROADCAST_BINARY_FUNCTION(Add, +) +DEFINE_BROADCAST_BINARY_FUNCTION(Sub, -) +DEFINE_BROADCAST_BINARY_FUNCTION(Mul, *) +DEFINE_BROADCAST_BINARY_FUNCTION(Div, /) + +#undef DEFINE_BROADCAST_BINARY_FUNCTION +#undef DELEGATE_BROADCAST_BINARY_FUNCTION + +#define SPECIALIZED_SET(T) \ + template <> \ + void Set(const int64_t N, const T alpha, T* Y, CPUMathUtil*) { \ + if (alpha == (T)0) { \ + memset(Y, 0, N * sizeof(T)); \ + } else { \ + EigenVectorMap(Y, N).setConstant(alpha); \ + } \ + } + +SPECIALIZED_SET(float); +SPECIALIZED_SET(double); +SPECIALIZED_SET(int8_t); +SPECIALIZED_SET(int16_t); +SPECIALIZED_SET(int32_t); +SPECIALIZED_SET(int64_t); +SPECIALIZED_SET(bool); +SPECIALIZED_SET(char); +SPECIALIZED_SET(uint8_t); +SPECIALIZED_SET(uint16_t); +#undef SPECIALIZED_SET + +#define INSTANTIATE_BINARY_OP(name, op, T) \ + template <> \ + void name( \ + const int n, const T* a, const T* b, bool* y, CPUMathUtil*) { \ + for (int i = 0; i < n; ++i) { \ + y[i] = a[i] op b[i]; \ + } \ + } \ + template <> \ + void name##ToRow( \ + const int m, \ + const int n, \ + const T* a, \ + const T* b, \ + bool* y, \ + CPUMathUtil*) { \ + for (int i = 0; i < n * m; ++i) { \ + y[i] = a[i] op b[i % n]; \ + } \ + } + +#define DEFINE_BINARY_OP(name, op) \ + INSTANTIATE_BINARY_OP(name, op, float) \ + INSTANTIATE_BINARY_OP(name, op, int32_t) \ + INSTANTIATE_BINARY_OP(name, op, int64_t) + +DEFINE_BINARY_OP(LT, <); +DEFINE_BINARY_OP(LE, <=); +DEFINE_BINARY_OP(GT, >); +DEFINE_BINARY_OP(GE, >=); + +INSTANTIATE_BINARY_OP(Or, |, bool); +INSTANTIATE_BINARY_OP(And, &, bool); +INSTANTIATE_BINARY_OP(Xor, ^, bool); + +template <> +void Not( + const int n, + const bool* x, + bool* y, + CPUMathUtil* /*context*/) { + for (int i = 0; i < n; ++i) { + y[i] = !x[i]; + } +} + +#undef DEFINE_BINARY_OP +#undef INSTANTIATE_BINARY_OP + +#define SPECIALIZED_CPU_ADD_STRIPED_BATCH(T) \ + template <> \ + void AddStripedBatch( \ + const int N, \ + const T* first, \ + T* y, \ + const int stripe, \ + const int batch, \ + CPUMathUtil* provider) { \ + for (int j = 0; j < batch; j++) { \ + Add(N, first + j * stripe, y, y, provider); \ + } \ + } + +SPECIALIZED_CPU_ADD_STRIPED_BATCH(float); +#undef SPECIALIZED_CPU_ADD_STRIPED_BATCH + +template <> +void RandUniform( + const int n, const float a, const float b, float* r, + CPUMathUtil* /*provider*/) { + std::uniform_real_distribution distribution(a, b); + //todo: need implmenet "RandGenerator()" in execution provider + ONNXRUNTIME_UNUSED_PARAMETER(n); + ONNXRUNTIME_UNUSED_PARAMETER(r); + ONNXRUNTIME_NOT_IMPLEMENTED(__FUNCTION__, " is not implemented"); + /*for (int i = 0; i < n; ++i) { + r[i] = distribution(context->RandGenerator()); + }*/ +} + +template <> +void RandUniform( + const int n, const int a, const int b, int* r, + CPUMathUtil* /*provider*/) { + std::uniform_int_distribution distribution(a, b); + //todo: need implmenet "RandGenerator()" in execution provider + ONNXRUNTIME_UNUSED_PARAMETER(n); + ONNXRUNTIME_UNUSED_PARAMETER(r); + ONNXRUNTIME_NOT_IMPLEMENTED(__FUNCTION__, " is not implemented"); + /*for (int i = 0; i < n; ++i) { + r[i] = distribution(context->RandGenerator()); + }*/ +} + +//todo: need implmenet "RandGenerator()" in execution provider + +//#define CAFFE2_SPECIALIZED_RAND_UNIFORM_UNIQUE(T) \ +// template <> \ +// void RandUniformUnique( \ +// const size_t n, \ +// const T a, \ +// const T b, \ +// T* r, \ +// const size_t m, \ +// const T* avoid, \ +// CPUContext* context) { \ +// CAFFE_ENFORCE_LE( \ +// n, b - a - m + 1, "Cannot satisfy the unique requirement"); \ +// std::unordered_set avoid_set(n); \ +// if (m) { \ +// avoid_set.insert(avoid, avoid + m); \ +// CAFFE_ENFORCE_EQ(m, avoid_set.size(), "Avoid should be unique"); \ +// } \ +// std::uniform_int_distribution distribution(a, b); \ +// T v = 0; \ +// for (size_t i = 0; i < n; ++i) { \ +// do { \ +// v = distribution(context->RandGenerator()); \ +// } while (avoid_set.count(v)); \ +// r[i] = v; \ +// avoid_set.insert(v); \ +// } \ +// } +// +// CAFFE2_SPECIALIZED_RAND_UNIFORM_UNIQUE(int32_t); +// CAFFE2_SPECIALIZED_RAND_UNIFORM_UNIQUE(int64_t); +//#undef CAFFE2_SPECIALIZED_RAND_UNIFORM_UNIQUE + +template <> +void RandGaussian( + const int n, const float mean, const float std, float* r, + CPUMathUtil* /*provider*/) { + std::normal_distribution distribution(mean, std); + ONNXRUNTIME_UNUSED_PARAMETER(n); + ONNXRUNTIME_UNUSED_PARAMETER(r); + ONNXRUNTIME_NOT_IMPLEMENTED(__FUNCTION__, " is not implemented"); + /*for (int i = 0; i < n; ++i) { + r[i] = distribution(context->RandGenerator()); + }*/ +} + +#define SPECIALIZED_SUM(T) \ + template <> \ + void Sum( \ + const int N, \ + const T* x, \ + T* y, \ + CPUMathUtil* /* unused */, \ + Tensor* /* unused */) { \ + *y = ConstEigenVectorMap(x, N).sum(); \ + } + +SPECIALIZED_SUM(float); +SPECIALIZED_SUM(int32_t); +SPECIALIZED_SUM(int64_t); + +#undef SPECIALIZED_SUM + +template <> +void SumSqr( + const int N, + const float* x, + float* y, + CPUMathUtil* /*context*/ /* unused */, + Tensor* /*scratch_ptr*/ /* unused */) { + *y = ConstEigenVectorMap(x, N).squaredNorm(); +} + +template <> +void Select( + const int N, + const int D, + const float* x, + const int* idx, + float* y, + CPUMathUtil* /*context*/) { + for (int i = 0; i < N; ++i) { + ONNXRUNTIME_ENFORCE(idx[i] < D); + y[i] = x[i * D + idx[i]]; + } +} +// Ported from caffe 1. +template <> +void Im2colNd( + const float* data_img, + const int64_t* im_shape, + const int64_t* col_shape, + const int64_t /* img_size*/, + const int64_t /* col_size*/, + const int64_t* kernel_shape, + const int64_t* stride, + const int64_t* dilation, + const int64_t* pad, + const int64_t N, + float* data_col, + CPUMathUtil* /* context */, + bool accumulate_output) { + int64_t kernel_size = 1; + for (int64_t i = 0; i < N; ++i) { + kernel_size *= kernel_shape[i]; + } + const int64_t channels_col = col_shape[0]; + std::vector d_offset(N, 0); + std::vector d_iter(N, 0); + for (int64_t c_col = 0; c_col < channels_col; ++c_col) { + // Loop over spatial axes in reverse order to compute a per-axis offset. + int64_t offset = c_col; + for (int64_t d_i = N - 1; d_i >= 0; --d_i) { + if (d_i < N - 1) { + offset /= kernel_shape[d_i + 1]; + } + d_offset[d_i] = offset % kernel_shape[d_i]; + } + for (bool incremented = true; incremented;) { + // Loop over spatial axes in forward order to compute the indices in the + // image and column, and whether the index lies in the padding. + int64_t index_col = c_col; + int64_t index_im = c_col / kernel_size; + bool is_padding = false; + for (int64_t d_i = 0; d_i < N; ++d_i) { + const int64_t d = d_iter[d_i]; + const int64_t d_im = + d * stride[d_i] - pad[d_i] + d_offset[d_i] * dilation[d_i]; + is_padding |= d_im < 0 || d_im >= im_shape[d_i + 1]; + index_col *= col_shape[d_i + 1]; + index_col += d; + index_im *= im_shape[d_i + 1]; + index_im += d_im; + } + if (!accumulate_output) { + if (is_padding) { + data_col[index_col] = 0; + } else { + data_col[index_col] = data_img[index_im]; + } + } else if (!is_padding) { // col2im + data_col[index_im] += data_img[index_col]; + } + // Loop over spatial axes in reverse order to choose an index, + // like counting. + incremented = false; + for (int64_t d_i = N - 1; d_i >= 0; --d_i) { + const int64_t d_max = col_shape[d_i + 1]; + ONNXRUNTIME_ENFORCE(d_iter[d_i] < d_max); + if (d_iter[d_i] == d_max - 1) { + d_iter[d_i] = 0; + } else { // d_iter[d_i] < d_max - 1 + ++d_iter[d_i]; + incremented = true; + break; + } + } + } // while(incremented) { + } // for (int c = 0; c < channels_col; ++c) { +} + +template <> +void Col2imNd( + const float* data_col, + const int64_t* img_shape, + const int64_t* col_shape, + const int64_t img_size, + const int64_t col_size, + const int64_t* kernel_shape, + const int64_t* stride, + const int64_t* dilation, + const int64_t* pad, + const int64_t N, + float* data_img, + CPUMathUtil* context) { + Set(img_size, 0, data_img, context); + Im2colNd( + data_col, + img_shape, + col_shape, + img_size, + col_size, + kernel_shape, + stride, + dilation, + pad, + N, + data_img, + context, + true); +} + +static void Im2colWithEqualPadding(int64_t output_h, int64_t output_w, const float* data_im, + const int64_t channels, + const int64_t height, + const int64_t width, + const int64_t kernel_h, + const int64_t kernel_w, + const int64_t dilation_h, + const int64_t dilation_w, + const int64_t pad_t, + const int64_t pad_l, + const int64_t stride_h, + const int64_t stride_w, + float* data_col) { + // From Intel, https://github.com/BVLC/caffe/pull/3536 + const int64_t pad_h = pad_t; + const int64_t pad_w = pad_l; + const int64_t channel_size = height * width; + for (int64_t channel = channels; channel--; data_im += channel_size) { + for (int64_t kernel_row = 0; kernel_row < kernel_h; kernel_row++) { + for (int64_t kernel_col = 0; kernel_col < kernel_w; kernel_col++) { + int64_t input_row = -pad_h + kernel_row * dilation_h; + for (int64_t output_rows = output_h; output_rows; output_rows--) { + if (!is_a_ge_zero_and_a_lt_b(input_row, height)) { + memset(data_col, 0, output_w * sizeof(float)); + data_col += output_w; + } else { + int64_t input_col = -pad_w + kernel_col * dilation_w; + const float* rdptr = data_im + input_row * width + input_col; + for (int64_t i = 0; i != output_w; ++i) { + if (is_a_ge_zero_and_a_lt_b(input_col, width)) { + *(data_col++) = rdptr[i * stride_w]; + } else { + *(data_col++) = 0; + } + input_col += stride_w; + } + } + input_row += stride_h; + } + } + } + } +} +template <> +void Im2col( + const float* data_im, + const int64_t channels, + const int64_t height, + const int64_t width, + const int64_t kernel_h, + const int64_t kernel_w, + const int64_t dilation_h, + const int64_t dilation_w, + const int64_t pad_t, + const int64_t pad_l, + const int64_t pad_b, + const int64_t pad_r, + const int64_t stride_h, + const int64_t stride_w, + float* data_col, + CPUMathUtil* /*context*/) { + const int64_t output_h = + (height + pad_b + pad_t - (dilation_h * (kernel_h - 1) + 1)) / stride_h + + 1; + const int64_t output_w = + (width + pad_l + pad_r - (dilation_w * (kernel_w - 1) + 1)) / stride_w + + 1; + + // Fast path for zero padding and no dilation + // From Torch, THNN_(unfolded_copy) + if (dilation_h == 1 && dilation_w == 1 && pad_l == 0 && pad_r == 0 && + pad_t == 0 && pad_b == 0) { + for (auto k = 0; k < channels * kernel_h * kernel_w; k++) { + const auto nip = k / (kernel_h * kernel_w); + const auto rest = k % (kernel_h * kernel_w); + const auto kh = rest / kernel_w; + const auto kw = rest % kernel_w; + auto* dst = data_col + nip * (kernel_h * kernel_w * output_h * output_w) + + kh * (kernel_w * output_h * output_w) + kw * (output_h * output_w); + const auto* src = data_im + nip * (height * width); + for (auto y = 0; y < output_h; y++) { + const auto iy = y * stride_h + kh; + const auto ix = kw; + if (stride_w == 1) { + memcpy( + dst + (y * output_w), + src + (iy * width + ix), + sizeof(float) * output_w); + } else { + for (auto x = 0; x < output_w; x++) { + memcpy( + dst + (y * output_w + x), + src + (iy * width + ix + x * stride_w), + sizeof(float)); + } + } + } + } + return; + } + + // Fast path for equal padding + if (pad_l == pad_r && pad_t == pad_b) { + Im2colWithEqualPadding(output_h, output_w, data_im, channels, height, width, kernel_h, kernel_w, dilation_h, dilation_w, pad_t, pad_l, stride_h, stride_w, data_col); + return; + } + + // Baseline + const int64_t dkernel_h = dilation_h * (kernel_h - 1) + 1; + const int64_t dkernel_w = dilation_w * (kernel_w - 1) + 1; + + int64_t height_col = (height + pad_t + pad_b - dkernel_h) / stride_h + 1; + int64_t width_col = (width + pad_l + pad_r - dkernel_w) / stride_w + 1; + + int64_t channels_col = channels * kernel_h * kernel_w; + for (int64_t c = 0; c < channels_col; ++c) { + int64_t w_offset = c % kernel_w; + int64_t h_offset = (c / kernel_w) % kernel_h; + int64_t c_im = c / kernel_h / kernel_w; + for (int64_t h = 0; h < height_col; ++h) { + for (int64_t w = 0; w < width_col; ++w) { + int64_t h_pad = h * stride_h - pad_t + h_offset * dilation_h; + int64_t w_pad = w * stride_w - pad_l + w_offset * dilation_w; + if (h_pad >= 0 && h_pad < height && w_pad >= 0 && w_pad < width) + data_col[(c * height_col + h) * width_col + w] = + data_im[(c_im * height + h_pad) * width + w_pad]; + else + data_col[(c * height_col + h) * width_col + w] = 0; + } + } + } +} + +template <> +void Im2col( + const float* data_im, + const int64_t channels, + const int64_t height, + const int64_t width, + const int64_t kernel_h, + const int64_t kernel_w, + const int64_t dilation_h, + const int64_t dilation_w, + const int64_t pad_t, + const int64_t pad_l, + const int64_t pad_b, + const int64_t pad_r, + const int64_t stride_h, + const int64_t stride_w, + float* data_col, + CPUMathUtil* /*context*/) { + const int64_t dkernel_h = dilation_h * (kernel_h - 1) + 1; + const int64_t dkernel_w = dilation_w * (kernel_w - 1) + 1; + + int64_t height_col = (height + pad_t + pad_b - dkernel_h) / stride_h + 1; + int64_t width_col = (width + pad_l + pad_r - dkernel_w) / stride_w + 1; + + int64_t h_pad = -pad_t; + for (int64_t h = 0; h < height_col; ++h) { + int64_t w_pad = -pad_l; + for (int64_t w = 0; w < width_col; ++w) { + for (int64_t ih = h_pad; ih < h_pad + dkernel_h; ih += dilation_h) { + for (int64_t iw = w_pad; iw < w_pad + dkernel_w; iw += dilation_w) { + if (ih >= 0 && ih < height && iw >= 0 && iw < width) { + memcpy(data_col, data_im + (ih * width + iw) * channels, + sizeof(float) * channels); + } else { + // This should be simply padded with zero. + memset(data_col, 0, sizeof(float) * channels); + } + data_col += channels; + } + } + w_pad += stride_w; + } + h_pad += stride_h; + } +} + +template <> +void Col2im( + const float* data_col, + const int64_t channels, + const int64_t height, + const int64_t width, + const int64_t kernel_h, + const int64_t kernel_w, + const int64_t dilation_h, + const int64_t dilation_w, + const int64_t pad_t, + const int64_t pad_l, + const int64_t pad_b, + const int64_t pad_r, + const int64_t stride_h, + const int64_t stride_w, + float* data_im, + CPUMathUtil* context) { + const int64_t output_h = + (height + pad_b + pad_t - (dilation_h * (kernel_h - 1) + 1)) / stride_h + + 1; + const int64_t output_w = + (width + pad_l + pad_r - (dilation_w * (kernel_w - 1) + 1)) / stride_w + + 1; + + Set(height * width * channels, 0, data_im, context); + + // Fast path for zero padding and no dilation + // From Torch, modified THNN_(unfolded_acc) + if (dilation_h == 1 && dilation_w == 1 && pad_l == 0 && pad_r == 0 && + pad_t == 0 && pad_b == 0) { + for (auto k = 0; k < channels * kernel_h * kernel_w; k++) { + const auto nip = k / (kernel_h * kernel_w); + const auto rest = k % (kernel_h * kernel_w); + const auto kh = rest / kernel_w; + const auto kw = rest % kernel_w; + const auto* dst = data_col + + nip * (kernel_h * kernel_w * output_h * output_w) + + kh * (kernel_w * output_h * output_w) + kw * (output_h * output_w); + auto* src = data_im + nip * (height * width); + for (auto y = 0; y < output_h; y++) { + const auto iy = y * stride_h + kh; + const auto ix = kw; + if (stride_w == 1) { + auto offsrc = src + (iy * width + ix); + const auto offdst = dst + (y * output_w); + for (auto i = 0; i < output_w; ++i) { + offsrc[i] += offdst[i]; + } + } else { + for (auto x = 0; x < output_w; x++) { + auto offsrc = src + (iy * width + ix + x * stride_w); + const auto offdst = dst + (y * output_w + x); + *offsrc += *offdst; + } + } + } + } + return; + } + + // Fast path for equal padding + if (pad_l == pad_r && pad_t == pad_b) { + // From Intel, https://github.com/BVLC/caffe/pull/3536 + const int64_t pad_h = pad_t; + const int64_t pad_w = pad_l; + const int64_t channel_size = height * width; + for (int64_t channel = channels; channel--; data_im += channel_size) { + for (int64_t kernel_row = 0; kernel_row < kernel_h; kernel_row++) { + for (int64_t kernel_col = 0; kernel_col < kernel_w; kernel_col++) { + int64_t input_row = -pad_h + kernel_row * dilation_h; + for (int64_t output_rows = output_h; output_rows; output_rows--) { + if (!is_a_ge_zero_and_a_lt_b(input_row, height)) { + data_col += output_w; + } else { + int64_t input_col = -pad_w + kernel_col * dilation_w; + for (int64_t output_col = output_w; output_col; output_col--) { + if (is_a_ge_zero_and_a_lt_b(input_col, width)) { + data_im[input_row * width + input_col] += *data_col; + } + data_col++; + input_col += stride_w; + } + } + input_row += stride_h; + } + } + } + } + return; + } + + // Fallback + const int64_t dkernel_h = dilation_h * (kernel_h - 1) + 1; + const int64_t dkernel_w = dilation_w * (kernel_w - 1) + 1; + + int64_t height_col = (height + pad_t + pad_b - dkernel_h) / stride_h + 1; + int64_t width_col = (width + pad_l + pad_r - dkernel_w) / stride_w + 1; + int64_t channels_col = channels * kernel_h * kernel_w; + for (int64_t c = 0; c < channels_col; ++c) { + int64_t w_offset = c % kernel_w; + int64_t h_offset = (c / kernel_w) % kernel_h; + int64_t c_im = c / kernel_h / kernel_w; + for (int64_t h = 0; h < height_col; ++h) { + for (int64_t w = 0; w < width_col; ++w) { + int64_t h_pad = h * stride_h - pad_t + h_offset * dilation_h; + int64_t w_pad = w * stride_w - pad_l + w_offset * dilation_w; + if (h_pad >= 0 && h_pad < height && w_pad >= 0 && w_pad < width) { + data_im[(c_im * height + h_pad) * width + w_pad] += + data_col[(c * height_col + h) * width_col + w]; + } + } + } + } +} + +template <> +void Col2im( + const float* data_col, + const int64_t channels, + const int64_t height, + const int64_t width, + const int64_t kernel_h, + const int64_t kernel_w, + const int64_t dilation_h, + const int64_t dilation_w, + const int64_t pad_t, + const int64_t pad_l, + const int64_t pad_b, + const int64_t pad_r, + const int64_t stride_h, + const int64_t stride_w, + float* data_im, + CPUMathUtil* context) { + const int64_t dkernel_h = dilation_h * (kernel_h - 1) + 1; + const int64_t dkernel_w = dilation_w * (kernel_w - 1) + 1; + + Set(height * width * channels, 0, data_im, context); + int64_t height_col = (height + pad_t + pad_b - dkernel_h) / stride_h + 1; + int64_t width_col = (width + pad_l + pad_r - dkernel_w) / stride_w + 1; + int64_t h_pad = -pad_t; + for (int64_t h = 0; h < height_col; ++h) { + int64_t w_pad = -pad_l; + for (int64_t w = 0; w < width_col; ++w) { + for (int64_t ih = h_pad; ih < h_pad + dkernel_h; ih += dilation_h) { + for (int64_t iw = w_pad; iw < w_pad + dkernel_w; iw += dilation_w) { + if (ih >= 0 && ih < height && iw >= 0 && iw < width) { + auto* data_im_patch = data_im + (ih * width + iw) * channels; + Add( + static_cast(channels), data_im_patch, data_col, data_im_patch, context); + } + data_col += channels; + } + } + w_pad += stride_w; + } + h_pad += stride_h; + } +} + +template <> +void CopyMatrix( + const size_t itemsize, + const int M, + const int N, + const void* A, + const int lda, + void* B, + const int ldb, + CPUMathUtil*, + TypedCopy copy) { + if (lda == N && ldb == N) { + // can coalese to a single memcpy of size M * N + if (copy) { + copy(static_cast(A), static_cast(B), N * M); + } else { + memcpy( + static_cast(B), static_cast(A), itemsize * N * M); + } + return; + } + + for (int i = 0; i < M; ++i) { + if (copy) { + copy( + static_cast(A) + lda * i * itemsize, + static_cast(B) + ldb * i * itemsize, + N); + } else { + memcpy( + static_cast(B) + ldb * i * itemsize, + static_cast(A) + lda * i * itemsize, + itemsize * N); + } + } +} + +#define SPECIALIZED_COPYVECTOR(T) \ + template <> \ + void CopyVector( \ + const int N, const T* src, T* dst, CPUMathUtil* /*context*/) { \ + if (src != dst && N > 0) { \ + memcpy(dst, src, sizeof(T) * N); \ + } \ + } +SPECIALIZED_COPYVECTOR(float) +#undef SPECIALIZED_COPYVECTOR + +uint32_t randomNumberSeed() { + // Originally copied from folly::randomNumberSeed (at 418ad4) + // modified to use chrono instead of sys/time.h + static std::atomic seedInput(0); + auto tv = std::chrono::system_clock::now().time_since_epoch(); + uint64_t usec = static_cast( + std::chrono::duration_cast(tv).count()); + uint32_t tv_sec = static_cast(usec / 1000000); + uint32_t tv_usec = static_cast(usec % 1000000); + const uint32_t kPrime0 = 51551; + const uint32_t kPrime1 = 61631; + const uint32_t kPrime2 = 64997; + const uint32_t kPrime3 = 111857; + static const uint32_t pid = static_cast(Env::Default().GetSelfPid()); + return kPrime0 * (seedInput++) + kPrime1 * pid + + kPrime2 * tv_sec + kPrime3 * tv_usec; +} + +uint16_t floatToHalf(float f) { + return Eigen::half_impl::float_to_half_rtne(f).x; +} + +float halfToFloat(uint16_t h) { + return Eigen::half_impl::half_to_float(Eigen::half_impl::raw_uint16_to_half(h)); +} + +} // namespace math +} // namespace onnxruntime diff --git a/onnxruntime/core/util/math_cpuonly.h b/onnxruntime/core/util/math_cpuonly.h new file mode 100644 index 0000000000000..091eff61dd26c --- /dev/null +++ b/onnxruntime/core/util/math_cpuonly.h @@ -0,0 +1,79 @@ +/** +* Copyright (c) 2016-present, Facebook, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +// Modifications Copyright (c) Microsoft. + +#pragma once + +// external/eigen/Eigen/src/Core/AssignEvaluator.h:86:63: +// error: enum constant in boolean context [-Werror=int-in-bool-context] +#if defined(__GNUC__) && __GNUC__>=7 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wint-in-bool-context" +#endif + +#include "Eigen/Core" + +#if defined(__GNUC__) && __GNUC__>=7 +#pragma GCC diagnostic pop +#endif + +#include "Eigen/Dense" + +namespace onnxruntime { + +// common Eigen types that we will often use +template +using EigenMatrixMap = Eigen::Map>; +template +using EigenArrayMap = Eigen::Map>; +template +using EigenVectorMap = Eigen::Map>; +template +using EigenVectorArrayMap = Eigen::Map>; +template +using ConstEigenMatrixMap = Eigen::Map>; +template +using ConstEigenArrayMap = Eigen::Map>; +template +using ConstEigenVectorMap = Eigen::Map>; +template +using ConstEigenVectorArrayMap = Eigen::Map>; +template +using EigenMatrixMapRowMajor = Eigen::Map< + Eigen::Matrix>; +template +using ConstEigenMatrixMapRowMajor = Eigen::Map< + const Eigen::Matrix>; + +template +auto EigenMap(Tensor& t) { return EigenVectorMap(t.template MutableData(), t.Shape().Size()); } +template +auto EigenMap(const Tensor& t) { return ConstEigenVectorMap(t.template Data(), t.Shape().Size()); } + +class CPUMathUtil { + public: + /*CPUMathUtil contains some help method like generate a + random seed. We only need a single instance for it.*/ + static CPUMathUtil& Instance() { + static CPUMathUtil p; + return p; + } + //todo: the random generate interface. + private: + CPUMathUtil() {} +}; + +} // namespace onnxruntime diff --git a/onnxruntime/python/__init__.py b/onnxruntime/python/__init__.py new file mode 100644 index 0000000000000..1a8a615c070e1 --- /dev/null +++ b/onnxruntime/python/__init__.py @@ -0,0 +1,4 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +#-------------------------------------------------------------------------- diff --git a/onnxruntime/python/_pybind_state.py b/onnxruntime/python/_pybind_state.py new file mode 100644 index 0000000000000..2feda322a331d --- /dev/null +++ b/onnxruntime/python/_pybind_state.py @@ -0,0 +1,12 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +#-------------------------------------------------------------------------- +import sys +import os +import warnings + +try: + from onnxruntime.capi.onnxruntime_pybind11_state import * # noqa +except ImportError as e: + warnings.warn("Cannot load onnxruntime.capi. Error: '{0}'".format(str(e))) diff --git a/onnxruntime/python/backend/__init__.py b/onnxruntime/python/backend/__init__.py new file mode 100644 index 0000000000000..876766046bb4f --- /dev/null +++ b/onnxruntime/python/backend/__init__.py @@ -0,0 +1,6 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +#-------------------------------------------------------------------------- + +from .backend import is_compatible, prepare, run, supports_device diff --git a/onnxruntime/python/backend/backend.py b/onnxruntime/python/backend/backend.py new file mode 100644 index 0000000000000..2a29ccdaa5268 --- /dev/null +++ b/onnxruntime/python/backend/backend.py @@ -0,0 +1,112 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +#-------------------------------------------------------------------------- +""" +Implements ONNX's backend API. +""" +from onnx.checker import check_model +from onnx.backend.base import Backend +from onnxruntime import InferenceSession, RunOptions, SessionOptions, get_device +from onnxruntime.backend.backend_rep import OnnxRuntimeBackendRep + + +class OnnxRuntimeBackend(Backend): + """ + Implements + `ONNX's backend API `_ + with *ONNX Runtime*. + The backend is mostly used when you need to switch between + multiple runtimes with the same API. + `Importing models from ONNX to Caffe2 `_ + shows how to use *caffe2* as a backend for a converted model. + """ + + @classmethod + def is_compatible(cls, model, device=None, **kwargs): + """ + Return whether the model is compatible with the backend. + + :param model: unused + :param device: None to use the default device or a string (ex: `'CPU'`) + :return: boolean + """ + if device is None: + device = get_device() + return cls.supports_device(device) + + @classmethod + def supports_device(cls, device): + """ + Check whether the backend is compiled with particular device support. + In particular it's used in the testing suite. + """ + return device in get_device() + + @classmethod + def prepare(cls, model, device=None, **kwargs): + """ + Load the model and creates a :class:`onnxruntime.InferenceSession` + ready to be used as a backend. + + :param model: ModelProto (returned by `onnx.load`), + string for a filename or bytes for a serialized model + :param device: requested device for the computation, + None means the default one which depends on + the compilation settings + :param kwargs: see :class:`onnxruntime.SessionOptions` + :return: :class:`onnxruntime.InferenceSession` + """ + if isinstance(model, OnnxRuntimeBackendRep): + return model + elif isinstance(model, InferenceSession): + return OnnxRuntimeBackendRep(model) + elif isinstance(model, (str, bytes)): + options = SessionOptions() + for k, v in kwargs.items(): + if hasattr(options, k): + setattr(options, k, v) + inf = InferenceSession(model, options) + if device is not None and not cls.supports_device(device): + raise RuntimeError("Incompatible device expected '{0}', got '{1}'".format(device, get_device())) + return cls.prepare(inf, device, **kwargs) + else: + # type: ModelProto + check_model(model) + bin = model.SerializeToString() + return cls.prepare(bin, device, **kwargs) + + @classmethod + def run_model(cls, model, inputs, device=None, **kwargs): + """ + Compute the prediction. + + :param model: :class:`onnxruntime.InferenceSession` returned + by function *prepare* + :param inputs: inputs + :param device: requested device for the computation, + None means the default one which depends on + the compilation settings + :param kwargs: see :class:`onnxruntime.RunOptions` + :return: predictions + """ + rep = cls.prepare(model, device, **kwargs) + options = RunOptions() + for k, v in kwargs.items(): + if hasattr(options, k): + setattr(options, k, v) + return rep.run(inputs, options) + + @classmethod + def run_node(cls, node, inputs, device=None, outputs_info=None, **kwargs): + ''' + This method is not implemented as it is much more efficient + to run a whole model than every node independently. + ''' + raise NotImplementedError("It is much more efficient to run a whole model than every node independently.") + + +is_compatible = OnnxRuntimeBackend.is_compatible +prepare = OnnxRuntimeBackend.prepare +run = OnnxRuntimeBackend.run_model +supports_device = OnnxRuntimeBackend.supports_device diff --git a/onnxruntime/python/backend/backend_rep.py b/onnxruntime/python/backend/backend_rep.py new file mode 100644 index 0000000000000..63aa586ade6a5 --- /dev/null +++ b/onnxruntime/python/backend/backend_rep.py @@ -0,0 +1,45 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +#-------------------------------------------------------------------------- +""" +Implements ONNX's backend API. +""" +import numpy as np +from onnx.backend.base import BackendRep + + +class OnnxRuntimeBackendRep(BackendRep): + """ + Computes the prediction for a pipeline converted into + an :class:`onnxruntime.InferenceSession` node. + """ + + def __init__(self, session): + """ + :param session: :class:`onnxruntime.InferenceSession` + """ + self._session = session + + + def run(self, inputs, **kwargs): # type: (Any, **Any) -> Tuple[Any, ...] + """ + Computes the prediction. + See :meth:`onnxruntime.InferenceSession.run`. + """ + if isinstance(inputs, list): + inps = {} + for i, inp in enumerate(self._session.get_inputs()): + inps[inp.name] = inputs[i] + outs = self._session.run(None, inps) + if isinstance(outs, list): + return outs + else: + output_names = [o.name for o in self._session.get_outputs()] + return [outs[name] for name in output_names] + else: + inp = self._session.get_inputs() + if len(inp) != 1: + raise RuntimeError("Model expect {0} inputs".format(len(inp))) + inps = {inp[0].name: inputs} + return self._session.run(None, inps) diff --git a/onnxruntime/python/datasets/__init__.py b/onnxruntime/python/datasets/__init__.py new file mode 100644 index 0000000000000..e1ab4a0e44077 --- /dev/null +++ b/onnxruntime/python/datasets/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +""" +Short examples used in the documentation. +""" +import os + + +def get_example(name): + """ + Retrieves the absolute file name of an example. + """ + this = os.path.abspath(os.path.dirname(__file__)) + full = os.path.join(this, name) + if not os.path.exists(full): + raise FileNotFoundError("Unable to find example '{0}'".format(name)) + return full diff --git a/onnxruntime/python/datasets/logreg_iris.onnx b/onnxruntime/python/datasets/logreg_iris.onnx new file mode 100644 index 0000000000000..e15b27a5da40e Binary files /dev/null and b/onnxruntime/python/datasets/logreg_iris.onnx differ diff --git a/onnxruntime/python/datasets/mul_1.pb b/onnxruntime/python/datasets/mul_1.pb new file mode 100644 index 0000000000000..60606b699062e Binary files /dev/null and b/onnxruntime/python/datasets/mul_1.pb differ diff --git a/onnxruntime/python/datasets/sigmoid.onnx b/onnxruntime/python/datasets/sigmoid.onnx new file mode 100644 index 0000000000000..f03fcd156af58 --- /dev/null +++ b/onnxruntime/python/datasets/sigmoid.onnx @@ -0,0 +1,13 @@ + backend-test:Q + +xy"Sigmoid test_sigmoidZ +x + + + +b +y + + + +B \ No newline at end of file diff --git a/onnxruntime/python/exported_symbols.lst b/onnxruntime/python/exported_symbols.lst new file mode 100644 index 0000000000000..9ee2cde243220 --- /dev/null +++ b/onnxruntime/python/exported_symbols.lst @@ -0,0 +1 @@ +_PyInit_onnxruntime_pybind11_state diff --git a/onnxruntime/python/onnxruntime_pybind_mlvalue.cc b/onnxruntime/python/onnxruntime_pybind_mlvalue.cc new file mode 100644 index 0000000000000..f3db01441f7e8 --- /dev/null +++ b/onnxruntime/python/onnxruntime_pybind_mlvalue.cc @@ -0,0 +1,417 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "onnxruntime_pybind_mlvalue.h" + +#define NO_IMPORT_ARRAY +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#define PY_ARRAY_UNIQUE_SYMBOL onnxruntime_python_ARRAY_API +#include + +#include "core/graph/graph_base.h" +#include "core/framework/tensor_shape.h" +#include "core/framework/tensor.h" + +using namespace std; +namespace onnxruntime { +namespace python { + +namespace py = pybind11; +using namespace onnxruntime::logging; + +int OnnxRuntimeTensorToNumpyType(const MLDataType& tensor_type) { + static std::map type_map{ + {DataTypeImpl::GetType(), NPY_BOOL}, + {DataTypeImpl::GetType(), NPY_FLOAT}, + {DataTypeImpl::GetType(), NPY_DOUBLE}, + {DataTypeImpl::GetType(), NPY_INT}, + {DataTypeImpl::GetType(), NPY_INT8}, + {DataTypeImpl::GetType(), NPY_UINT8}, + {DataTypeImpl::GetType(), NPY_INT16}, + {DataTypeImpl::GetType(), NPY_UINT16}, + {DataTypeImpl::GetType(), NPY_LONGLONG}, + {DataTypeImpl::GetType(), NPY_ULONGLONG}, + {DataTypeImpl::GetType(), NPY_OBJECT}, + }; + + const auto it = type_map.find(tensor_type); + if (it == type_map.end()) { + throw std::runtime_error("No corresponding Numpy type for Tensor Type."); + } else { + return it->second; + } +} + +const MLDataType& NumpyToOnnxRuntimeTensorType(int numpy_type) { + static std::map type_map{ + {NPY_BOOL, DataTypeImpl::GetType()}, + {NPY_FLOAT, DataTypeImpl::GetType()}, + {NPY_DOUBLE, DataTypeImpl::GetType()}, + {NPY_INT, DataTypeImpl::GetType()}, + {NPY_INT8, DataTypeImpl::GetType()}, + {NPY_UINT8, DataTypeImpl::GetType()}, + {NPY_INT16, DataTypeImpl::GetType()}, + {NPY_UINT16, DataTypeImpl::GetType()}, + {NPY_LONG, + sizeof(long) == sizeof(int) ? DataTypeImpl::GetType() + : DataTypeImpl::GetType()}, + {NPY_LONGLONG, DataTypeImpl::GetType()}, + {NPY_ULONGLONG, DataTypeImpl::GetType()}, + {NPY_UNICODE, DataTypeImpl::GetType()}, + {NPY_STRING, DataTypeImpl::GetType()}, + {NPY_OBJECT, DataTypeImpl::GetType()}, + {NPY_VOID, DataTypeImpl::GetType()}}; + + const auto it = type_map.find(numpy_type); + if (it == type_map.end()) { + throw std::runtime_error("Numpy_type " + std::to_string(numpy_type) + + " can't be converted to MLDataType."); + } else { + return it->second; + } +} + +bool PyObjectCheck_Array(PyObject* o) { + return PyObject_HasAttrString(o, "__array_finalize__"); +} + +void CreateTensorMLValue(AllocatorPtr alloc, const std::string& name_input, PyArrayObject* pyObject, MLValue* p_mlvalue) { + PyArrayObject* darray = PyArray_GETCONTIGUOUS(pyObject); + if (darray == NULL) { + throw std::runtime_error(std::string("The object must be a contiguous array for input '") + name_input + std::string("'.")); + } + bool dref = false; + try { + const int npy_type = PyArray_TYPE(darray); + + // numpy requires long int as its dims. + int ndim = PyArray_NDIM(darray); + npy_intp* npy_dims = PyArray_DIMS(darray); + std::vector dims(ndim); + for (int i = 0; i < ndim; ++i) { + dims[i] = npy_dims[i]; + } + + TensorShape shape(dims); + auto element_type = NumpyToOnnxRuntimeTensorType(npy_type); + void* buffer = alloc->Alloc(element_type->Size() * shape.Size()); + + if (npy_type != NPY_UNICODE && npy_type != NPY_OBJECT) { + memcpy(buffer, static_cast(PyArray_DATA(darray)), element_type->Size() * shape.Size()); + } + + std::unique_ptr p_tensor = std::make_unique(element_type, + shape, + static_cast(buffer), + alloc->Info(), alloc); + + if (npy_type == NPY_UNICODE) { + // Copy string data which needs to be done after Tensor is allocated. + // Strings are Python strings or numpy.unicode string. + std::string* dst = static_cast(buffer); + auto item_size = PyArray_ITEMSIZE(darray); + auto num_chars = item_size / PyUnicode_4BYTE_KIND; + char* src = static_cast(PyArray_DATA(darray)); + const char* str; + Py_ssize_t size; + PyObject* pStr; + for (int i = 0; i < shape.Size(); i++, src += item_size) { + // Python unicode strings are assumed to be USC-4. Strings are stored as UTF-8. + pStr = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, src, num_chars); + str = PyUnicode_AsUTF8AndSize(pStr, &size); + if (str == NULL) { + dst[i] = ""; + } else { + // Size is equal to the longest string size, numpy stores + // strings in a single array. Those code assumes a string ends with a final 0. + dst[i] = str; + } + Py_XDECREF(pStr); + } + } else if (npy_type == NPY_STRING || npy_type == NPY_VOID) { + // Copy string data which needs to be done after Tensor is allocated. + // Strings are given as bytes (encoded strings). + // NPY_VOID does not trim final 0. + // NPY_STRING assumes bytes string ends with a final 0. + std::string* dst = static_cast(buffer); + auto item_size = PyArray_ITEMSIZE(darray); + char* src = static_cast(PyArray_DATA(darray)); + for (int i = 0; i < shape.Size(); i++, src += item_size) { + if (npy_type == NPY_STRING) { + dst[i] = src; + } else { + dst[i].resize(item_size); + memcpy((void*)dst[i].c_str(), src, item_size); + } + } + } else if (npy_type == NPY_OBJECT) { + // Converts object into string. + std::string* dst = static_cast(buffer); + auto item_size = PyArray_ITEMSIZE(darray); + char* src = static_cast(PyArray_DATA(darray)); + PyObject *item, *pStr; + for (int i = 0; i < shape.Size(); ++i, src += item_size) { + // Python unicode strings are assumed to be USC-4. Strings are stored as UTF-8. + item = PyArray_GETITEM(darray, src); + pStr = PyObject_Str(item); + dst[i] = py::reinterpret_borrow(pStr); + Py_XDECREF(pStr); + } + } + + p_mlvalue->Init(p_tensor.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); + } catch (...) { + if (!dref) { + Py_XDECREF(darray); + dref = true; + } + + // allocator should be able to gc the memory created by it. + // ... + + throw; + } + + if (!dref) { + Py_XDECREF(darray); + } +} + +std::string _get_type_name(int64_t&) { + return std::string("int64_t"); +} + +std::string _get_type_name(float&) { + return std::string("float"); +} + +std::string _get_type_name(std::string&) { + return std::string("string"); +} + +template +void CreateMapMLValue_LoopIntoMap(Py_ssize_t& pos, PyObject*& key, const std::string& name_input, PyObject*& value, + PyObject* item, std::map& current, + KeyGetterType keyGetter, ValueGetterType valueGetter) { + KeyType ckey; + ValueType cvalue; + do { + if (!keyGetter(key, ckey)) { + PyObject* pType = PyObject_Type(key); + auto pStr = PyObject_Str(pType); + py::str spyType = py::reinterpret_borrow(pStr); + std::string sType = spyType; + Py_XDECREF(pStr); + Py_XDECREF(pType); + Py_XDECREF(item); + throw std::runtime_error(std::string("Unexpected key type ") + sType + + std::string(", it cannot be linked to C type ") + + _get_type_name(ckey) + std::string(" for input '") + + name_input + std::string("'.")); + } + + if (!valueGetter(value, cvalue)) { + PyObject* pType = PyObject_Type(value); + auto pStr = PyObject_Str(pType); + py::str spyType = py::reinterpret_borrow(pStr); + std::string sType = spyType; + Py_XDECREF(pStr); + Py_XDECREF(pType); + Py_XDECREF(item); + throw std::runtime_error(std::string("Unexpected value type ") + sType + + std::string(", it cannot be linked to C type ") + + _get_type_name(ckey) + std::string(" for input '") + + name_input + std::string("'.")); + } + current[ckey] = cvalue; + } while (PyDict_Next(item, &pos, &key, &value)); +} + +template +void CreateMapMLValue_Map(Py_ssize_t& pos, PyObject*& key, const std::string& name_input, PyObject*& value, + PyObject* item, + AllocatorPtr /*alloc*/, MLValue* p_mlvalue, + KeyGetterType keyGetter, ValueGetterType valueGetter) { + std::unique_ptr> dst; + dst = std::make_unique>(); + CreateMapMLValue_LoopIntoMap(pos, key, name_input, value, item, *dst, keyGetter, valueGetter); + p_mlvalue->Init(dst.release(), DataTypeImpl::GetType>(), + DataTypeImpl::GetType>()->GetDeleteFunc()); +} + +template +void CreateMapMLValue_VectorMap(Py_ssize_t& pos, PyObject*& key, const std::string& name_input, PyObject*& value, + PyObject* iterator, PyObject* item, + AllocatorPtr /*alloc*/, MLValue* p_mlvalue, + KeyGetterType keyGetter, ValueGetterType valueGetter) { + std::unique_ptr>> dstVector; + dstVector = std::make_unique>>(); + int index = 0; + do { + dstVector->push_back(std::map()); + CreateMapMLValue_LoopIntoMap(pos, key, name_input, value, item, (*dstVector)[index], keyGetter, valueGetter); + Py_DECREF(item); + ++index; + item = iterator == NULL ? NULL : PyIter_Next(iterator); + } while (item != NULL); + p_mlvalue->Init(dstVector.release(), DataTypeImpl::GetType>>(), + DataTypeImpl::GetType>>()->GetDeleteFunc()); +} + +void CreateMapMLValue_AgnosticMap(Py_ssize_t& pos, PyObject*& key, const std::string& name_input, PyObject*& value, + PyObject* iterator, PyObject* item, + AllocatorPtr alloc, MLValue* p_mlvalue) { + // If iterator is NULL, it returns a single Map, + // if is not NULL, it returns a VectorMap. + auto int64Getter = [](PyObject* obj, int64_t& value) -> bool { + value = PyLong_AsLong(obj); + return !PyErr_Occurred(); + }; + + auto floatGetter = [](PyObject* obj, float& value) -> bool { + if (PyFloat_Check(obj)) { + value = (float)PyFloat_AS_DOUBLE(obj); + return true; + } else if (PyNumber_Check(obj)) { + value = (float)PyFloat_AsDouble(obj); + return true; + } else { + return false; + } + }; + + auto stringGetter = [](PyObject* obj, std::string& value) -> bool { + PyObject* pStr = PyObject_Str(obj); + if (pStr == NULL) { + return false; + } + value = py::reinterpret_borrow(pStr); + Py_DECREF(pStr); + return true; + }; + + if (iterator == NULL) { + if (PyLong_Check(key)) { + // Regular Python. + CreateMapMLValue_Map(pos, key, name_input, value, item, alloc, p_mlvalue, int64Getter, floatGetter); + } else if (PyNumber_Check(key)) { + // For numpy type. + CreateMapMLValue_Map(pos, key, name_input, value, item, alloc, p_mlvalue, int64Getter, floatGetter); + } else if (PyUnicode_Check(key)) { + CreateMapMLValue_Map(pos, key, name_input, value, item, alloc, p_mlvalue, stringGetter, floatGetter); + } else { + PyObject* pType = PyObject_Type(key); + PyObject* pStr = PyObject_Str(pType); + py::str spyType = py::reinterpret_borrow(pStr); + std::string sType = spyType; + Py_XDECREF(pType); + Py_XDECREF(pStr); + throw std::runtime_error(std::string("Key type must be int or string (not ") + sType + + std::string(") for input '") + name_input + std::string("'.")); + } + } else { + if (PyLong_Check(key)) { + CreateMapMLValue_VectorMap(pos, key, name_input, value, iterator, item, alloc, p_mlvalue, int64Getter, floatGetter); + } else if (PyNumber_Check(key)) { + // For numpy type. + CreateMapMLValue_VectorMap(pos, key, name_input, value, iterator, item, alloc, p_mlvalue, int64Getter, floatGetter); + } else if (PyUnicode_Check(key)) { + CreateMapMLValue_VectorMap(pos, key, name_input, value, iterator, item, alloc, p_mlvalue, stringGetter, floatGetter); + } else { + PyObject* pType = PyObject_Type(value); + PyObject* pStr = PyObject_Str(pType); + py::str spyType = py::reinterpret_borrow(pStr); + std::string sType = spyType; + Py_XDECREF(pType); + Py_XDECREF(pStr); + throw std::runtime_error(std::string("Key type must be int or string (not ") + sType + + std::string(") for input '") + name_input + std::string("'.")); + } + } +} + +void CreateMapMLValue_AgnosticVectorMap(PyObject* iterator, PyObject* item, AllocatorPtr alloc, const std::string& name_input, MLValue* p_mlvalue) { + // CreateMapMLValue is called by CreateGenericTerableMLValue + // or CreateGenericMLValue which ensures + // item is a dictionary, no need to check type again. + // This functions starts to iterate on the first + // element of the dictionary and calls CreateMapMLValue_AgnosticMap + // which determines the container type. This type + // is based on the first pair of the dictionary + // and all the function assumes the key and value type remain the same + // for all pairs in the dictionary. + + // If iterator is NULL, it returns a single Map, + // if is not NULL, it returns a VectorMap. + + PyObject *key, *value; + Py_ssize_t pos = 0; + + if (PyDict_Next(item, &pos, &key, &value)) { + CreateMapMLValue_AgnosticMap(pos, key, name_input, value, iterator, item, alloc, p_mlvalue); + } else { + throw std::runtime_error("Size of dictionary is empty, unable to run the prediction."); + } +} + +void CreateGenericIterableMLValue(PyObject* iterator, AllocatorPtr alloc, const std::string& name_input, MLValue* p_mlvalue) { + PyObject* item; + MLValue ml_value; + item = PyIter_Next(iterator); + if (item == NULL) { + throw std::runtime_error("Input '" + name_input + "' must not be empty."); + } + if (PyObjectCheck_Array(item)) { + PyObject* pType = PyObject_Type(item); + PyObject* pStr = PyObject_Str(pType); + py::str spyType = py::reinterpret_borrow(pStr); + std::string sType = spyType; + Py_XDECREF(pType); + Py_XDECREF(pStr); + throw std::runtime_error("Iterable of " + sType + " should be given as array for input '" + + name_input + std::string("'.")); + } else { + // We expect a dictionary. + if (!PyDict_Check(item)) { + throw std::runtime_error("Input must be a list of dictionaries or a single numpy array for input '" + + name_input + std::string("'.")); + } + CreateMapMLValue_AgnosticVectorMap(iterator, item, alloc, name_input, p_mlvalue); + } +} + +void CreateGenericMLValue(AllocatorPtr alloc, const std::string& name_input, py::object& value, MLValue* p_mlvalue) { + if (PyObjectCheck_Array(value.ptr())) { + // The most frequent case: input comes as an array. + PyArrayObject* arr = reinterpret_cast(value.ptr()); + CreateTensorMLValue(alloc, name_input, arr, p_mlvalue); + } else if (PyDict_Check(value.ptr())) { + CreateMapMLValue_AgnosticVectorMap((PyObject*)NULL, value.ptr(), alloc, name_input, p_mlvalue); + } else { + auto iterator = PyObject_GetIter(value.ptr()); + if (iterator == NULL) { + // The pype cannot be handled. + PyObject* pType = PyObject_Type(value.ptr()); + PyObject* pStr = PyObject_Str(pType); + py::str spyType = py::reinterpret_borrow(pStr); + std::string sType = spyType; + Py_XDECREF(pType); + Py_XDECREF(pStr); + throw std::runtime_error(std::string("Unable to handle object of type ") + sType); + } + // We assume the object is iterable. + // iterator should not be NULL due to previous test. + try { + CreateGenericIterableMLValue(iterator, alloc, name_input, p_mlvalue); + } catch (const std::runtime_error&) { + Py_DECREF(iterator); + throw; + } + Py_DECREF(iterator); + } +} + +} // namespace python +} // namespace onnxruntime diff --git a/onnxruntime/python/onnxruntime_pybind_mlvalue.h b/onnxruntime/python/onnxruntime_pybind_mlvalue.h new file mode 100644 index 0000000000000..4673893d37a46 --- /dev/null +++ b/onnxruntime/python/onnxruntime_pybind_mlvalue.h @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include + +#include "core/common/logging/logging.h" +#include "core/common/logging/sinks/clog_sink.h" +#include "core/common/logging/sinks/cerr_sink.h" +#include "core/framework/allocatormgr.h" +#include "core/framework/environment.h" +#include "core/framework/ml_value.h" +#include "core/session/inference_session.h" + + +using namespace std; +namespace onnxruntime { +namespace python { + +namespace py = pybind11; + +int OnnxRuntimeTensorToNumpyType(const MLDataType& tensor_type); + +void CreateGenericMLValue(AllocatorPtr alloc, const std::string& name_input, py::object& value, MLValue* p_mlvalue); + +} // namespace python +} // namespace onnxruntime diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc new file mode 100644 index 0000000000000..d2c7d0b52cb1f --- /dev/null +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -0,0 +1,434 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "onnxruntime_pybind_mlvalue.h" + +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#define PY_ARRAY_UNIQUE_SYMBOL onnxruntime_python_ARRAY_API +#include + +#include "core/graph/graph.h" + +#if USE_CUDA +#define BACKEND_PROC "GPU" +#else +#define BACKEND_PROC "CPU" +#endif + +#if USE_OPENMP +#define BACKEND_OPENMP "-OPENMP" +#else +#define BACKEND_OPENMP "" +#endif + +#if USE_MKLDNN +#define BACKEND_MKLDNN "-MKL-DNN" +#include "core/providers/mkldnn/mkldnn_execution_provider.h" +#else +#define BACKEND_MKLDNN "" +#endif + +#if USE_MKLML +#define BACKEND_MKLML "-MKL-ML" +#else +#define BACKEND_MKLML "" +#endif + +#if USE_OPENBLAS +#define BACKEND_OPENBLAS "-OPENBLAS" +#else +#define BACKEND_OPENBLAS "" +#endif + +#define BACKEND_DEVICE BACKEND_PROC BACKEND_MKLDNN BACKEND_MKLML BACKEND_OPENBLAS +#include "core/session/onnxruntime_cxx_api.h" +#include "core/providers/cpu/cpu_execution_provider.h" +#include "core/providers/cpu/cpu_provider_factory.h" + +#ifdef USE_CUDA +#include "core/providers/cuda/cuda_provider_factory.h" +#endif +#ifdef USE_MKLDNN +#include "core/providers/mkldnn/mkldnn_provider_factory.h" +#endif +#ifdef USE_NUPHAR +#include "core/providers/nuphar/nuphar_provider_factory.h" +#endif +#if defined(_MSC_VER) +#pragma warning(disable : 4267 4996 4503 4003) +#endif // _MSC_VER + +#include + +#if defined(_MSC_VER) +#pragma warning(disable : 4267 4996 4503 4003) +#endif // _MSC_VER + +using namespace std; +namespace onnxruntime { +namespace python { + +namespace py = pybind11; +using namespace onnxruntime; +using namespace onnxruntime::logging; + +static AllocatorPtr& GetAllocator() { + static AllocatorPtr alloc = std::make_shared(); + return alloc; +} + +static const SessionOptions& GetDefaultCPUSessionOptions() { + static SessionOptions so; + return so; +} + +template +void AddNonTensor(onnxruntime::MLValue& val, vector& pyobjs) { + pyobjs.push_back(py::cast(val.Get())); +} +void AddNonTensorAsPyObj(onnxruntime::MLValue& val, vector& pyobjs) { + // Should be in sync with core/framework/datatypes.h + if (val.Type() == DataTypeImpl::GetType()) { + AddNonTensor(val, pyobjs); + } else if (val.Type() == DataTypeImpl::GetType()) { + AddNonTensor(val, pyobjs); + } else if (val.Type() == DataTypeImpl::GetType()) { + AddNonTensor(val, pyobjs); + } else if (val.Type() == DataTypeImpl::GetType()) { + AddNonTensor(val, pyobjs); + } else if (val.Type() == DataTypeImpl::GetType()) { + AddNonTensor(val, pyobjs); + } else if (val.Type() == DataTypeImpl::GetType()) { + AddNonTensor(val, pyobjs); + } else if (val.Type() == DataTypeImpl::GetType()) { + AddNonTensor(val, pyobjs); + } else if (val.Type() == DataTypeImpl::GetType()) { + AddNonTensor(val, pyobjs); + } else if (val.Type() == DataTypeImpl::GetType()) { + AddNonTensor(val, pyobjs); + } else if (val.Type() == DataTypeImpl::GetType()) { + AddNonTensor(val, pyobjs); + } else if (val.Type() == DataTypeImpl::GetType()) { + AddNonTensor(val, pyobjs); + } else if (val.Type() == DataTypeImpl::GetType()) { + AddNonTensor(val, pyobjs); + } else if (val.Type() == DataTypeImpl::GetType()) { + AddNonTensor(val, pyobjs); + } else if (val.Type() == DataTypeImpl::GetType()) { + AddNonTensor(val, pyobjs); + } else { + throw std::runtime_error("Output is a non-tensor type which is not supported."); + } +} + +void AddTensorAsPyObj(onnxruntime::MLValue& val, vector& pyobjs) { + const Tensor& rtensor = val.Get(); + std::vector npy_dims; + const TensorShape& shape = rtensor.Shape(); + + for (size_t n = 0; n < shape.NumDimensions(); ++n) { + npy_dims.push_back(shape[n]); + } + + MLDataType dtype = rtensor.DataType(); + const int numpy_type = OnnxRuntimeTensorToNumpyType(dtype); + py::object obj = py::reinterpret_steal(PyArray_SimpleNew( + shape.NumDimensions(), npy_dims.data(), numpy_type)); + + void* outPtr = static_cast( + PyArray_DATA(reinterpret_cast(obj.ptr()))); + + if (numpy_type != NPY_OBJECT) { + memcpy(outPtr, rtensor.DataRaw(dtype), dtype->Size() * shape.Size()); + } else { + // Handle string type. + py::object* outObj = static_cast(outPtr); + const std::string* src = rtensor.template Data(); + for (int i = 0; i < rtensor.Shape().Size(); i++, src++) { + outObj[i] = py::cast(*src); + } + } + pyobjs.push_back(obj); +} + +class SessionObjectInitializer { + public: + typedef const SessionOptions& Arg1; + typedef logging::LoggingManager* Arg2; + operator Arg1() { + return GetDefaultCPUSessionOptions(); + } + + operator Arg2() { + static std::string default_logger_id{"Default"}; + static LoggingManager default_logging_manager{std::unique_ptr{new CErrSink{}}, + Severity::kWARNING, false, LoggingManager::InstanceType::Default, + &default_logger_id}; + return &default_logging_manager; + } + + static SessionObjectInitializer Get() { + return SessionObjectInitializer(); + } +}; + +inline void RegisterExecutionProvider(InferenceSession* sess, ONNXRuntimeProviderFactoryPtr* f) { + ONNXRuntimeProviderPtr p; + (*f)->CreateProvider(f, &p); + std::unique_ptr q((onnxruntime::IExecutionProvider*)p); + auto status = sess->RegisterExecutionProvider(std::move(q)); + if (!status.IsOK()) { + throw std::runtime_error(status.ErrorMessage().c_str()); + } +} + +#define FACTORY_PTR_HOLDER \ + std::unique_ptr ptr_holder_(f, ONNXRuntimeReleaseObject); + +void InitializeSession(InferenceSession* sess) { + onnxruntime::common::Status status; +#ifdef USE_CUDA + { + ONNXRuntimeProviderFactoryPtr* f; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateCUDAExecutionProviderFactory(0, &f)); + RegisterExecutionProvider(sess, f); + FACTORY_PTR_HOLDER; + } +#endif + +#ifdef USE_MKLDNN + { + const bool enable_cpu_mem_arena = true; + ONNXRuntimeProviderFactoryPtr* f; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateMkldnnExecutionProviderFactory(enable_cpu_mem_arena ? 1 : 0, &f)); + RegisterExecutionProvider(sess, f); + FACTORY_PTR_HOLDER; + } +#endif +#if 0 //USE_NUPHAR + { + ONNXRuntimeProviderFactoryPtr* f; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateNupharExecutionProviderFactory(0, "", &f)); + RegisterExecutionProvider(sess, f); + FACTORY_PTR_HOLDER; + } +#endif + + status = sess->Initialize(); + if (!status.IsOK()) { + throw std::runtime_error(status.ToString().c_str()); + } +} // namespace python + +void addGlobalMethods(py::module& m) { + m.def("get_session_initializer", &SessionObjectInitializer::Get, "Return a default session object initializer."); + m.def("get_device", []() -> std::string { return BACKEND_DEVICE; }, + "Return the device used to compute the prediction (CPU, MKL, ...)"); +} + +void addObjectMethods(py::module& m) { + // allow unit tests to redirect std::cout and std::cerr to sys.stdout and sys.stderr + py::add_ostream_redirect(m, "onnxruntime_ostream_redirect"); + py::class_(m, "SessionOptions", R"pbdoc(Configuration information for a session.)pbdoc") + .def(py::init()) + .def_readwrite("enable_profiling", &SessionOptions::enable_profiling, + R"pbdoc(Enable profiling for this session.)pbdoc") + .def_readwrite("profile_file_prefix", &SessionOptions::profile_file_prefix, + R"pbdoc(The prefix of the profile file. The current time will be appended to the file name.)pbdoc") + .def_readwrite("session_logid", &SessionOptions::session_logid, + R"pbdoc(Logger id to use for session output.)pbdoc") + .def_readwrite("session_log_verbosity_level", &SessionOptions::session_log_verbosity_level, + R"pbdoc(Applies to session load, initialization, etc.)pbdoc"); + + py::class_(m, "RunOptions", R"pbdoc(Configuration information for a single Run.)pbdoc") + .def(py::init()) + .def_readwrite("run_log_verbosity_level", &RunOptions::run_log_verbosity_level, + "Applies to a particular Run() invocation.") + .def_readwrite("run_tag", &RunOptions::run_tag, + "To identify logs generated by a particular Run() invocation."); + + py::class_(m, "ModelMetadata", R"pbdoc(Pre-defined and custom metadata about the model. +It is usually used to identify the model used to run the prediction and +facilitate the comparison.)pbdoc") + .def_readwrite("producer_name", &ModelMetadata::producer_name, "producer name") + .def_readwrite("graph_name", &ModelMetadata::graph_name, "graph name") + .def_readwrite("domain", &ModelMetadata::domain, "ONNX domain") + .def_readwrite("description", &ModelMetadata::description, "description of the model") + .def_readwrite("version", &ModelMetadata::version, "version of the model") + .def_readwrite("custom_metadata_map", &ModelMetadata::custom_metadata_map, "additional metadata"); + + py::class_(m, "NodeArg", R"pbdoc(Node argument definition, for both input and output, +including arg name, arg type (contains both type and shape).)pbdoc") + .def_property_readonly("name", &onnxruntime::NodeArg::Name, "node name") + .def_property_readonly("type", [](const onnxruntime::NodeArg& na) -> std::string { + return *(na.Type()); + }, "node type") + .def("__str__", [](const onnxruntime::NodeArg& na) -> std::string { + std::ostringstream res; + res << "NodeArg(name='" << na.Name() << "', type='" << *(na.Type()) << "', shape="; + auto shape = na.Shape(); + std::vector arr; + if (shape == nullptr || shape->dim_size() == 0) { + res << "[]"; + } else { + res << "["; + for (int i = 0; i < shape->dim_size(); ++i) { + if (shape->dim(i).has_dim_value()) { + res << shape->dim(i).dim_value(); + } else if (shape->dim(i).has_dim_param()) { + res << "None"; + } + if (i < shape->dim_size() - 1) { + res << ", "; + } + } + res << "]"; + } + res << ")"; + + return std::string(res.str()); + }, "converts the node into a readable string") + .def_property_readonly("shape", [](const onnxruntime::NodeArg& na) -> std::vector { + auto shape = na.Shape(); + std::vector arr; + if (shape == nullptr || shape->dim_size() == 0) { + return arr; + } + + arr.resize(shape->dim_size()); + for (int i = 0; i < shape->dim_size(); ++i) { + if (shape->dim(i).has_dim_value()) { + arr[i] = py::cast(shape->dim(i).dim_value()); + } else if (shape->dim(i).has_dim_param()) { + arr[i] = py::none(); + } + } + return arr; + }, "node shape (assuming the node holds a tensor)"); + + py::class_(m, "SessionObjectInitializer"); + py::class_(m, "InferenceSession", R"pbdoc(This is the main class used to run a model.)pbdoc") + .def(py::init()) + .def(py::init()) + .def("load_model", [](InferenceSession* sess, const std::string& path) { + auto status = sess->Load(path); + if (!status.IsOK()) { + throw std::runtime_error(status.ToString().c_str()); + } + InitializeSession(sess); + }, + R"pbdoc(Load a model saved in ONNX format.)pbdoc") + .def("read_bytes", [](InferenceSession* sess, const py::bytes& serializedModel) { + std::istringstream buffer(serializedModel); + auto status = sess->Load(buffer); + if (!status.IsOK()) { + throw std::runtime_error(status.ToString().c_str()); + } + InitializeSession(sess); + }, + R"pbdoc(Load a model serialized in ONNX format.)pbdoc") + .def("run", [](InferenceSession* sess, std::vector output_names, std::map pyfeeds, RunOptions* run_options = nullptr) -> std::vector { + NameMLValMap feeds; + for (auto _ : pyfeeds) { + MLValue ml_value; + CreateGenericMLValue(GetAllocator(), _.first, _.second, &ml_value); + if (PyErr_Occurred()) { + PyObject *ptype, *pvalue, *ptraceback; + PyErr_Fetch(&ptype, &pvalue, &ptraceback); + + PyObject* pStr = PyObject_Str(ptype); + std::string sType = py::reinterpret_borrow(pStr); + Py_XDECREF(pStr); + pStr = PyObject_Str(pvalue); + sType += ": "; + sType += py::reinterpret_borrow(pStr); + Py_XDECREF(pStr); + throw std::runtime_error(sType); + } + feeds.insert(std::make_pair(_.first, ml_value)); + } + + std::vector fetches; + common::Status status; + + if (run_options != nullptr) { + status = sess->Run(*run_options, feeds, output_names, &fetches); + } else { + status = sess->Run(feeds, output_names, &fetches); + } + + if (!status.IsOK()) { + auto mes = status.ToString(); + throw std::runtime_error(std::string("Method run failed due to: ") + std::string(mes.c_str())); + } + + std::vector rfetch; + rfetch.reserve(fetches.size()); + for (auto _ : fetches) { + if (_.IsTensor()) { + AddTensorAsPyObj(_, rfetch); + } else { + AddNonTensorAsPyObj(_, rfetch); + } + } + return rfetch; + }) + .def("end_profiling", [](InferenceSession* sess) -> std::string { + return sess->EndProfiling(); + }) + .def_property_readonly("inputs_meta", [](const InferenceSession* sess) -> const std::vector& { + auto res = sess->GetModelInputs(); + if (!res.first.IsOK()) { + throw std::runtime_error(res.first.ToString().c_str()); + } else { + return *(res.second); + } + }) + .def_property_readonly("outputs_meta", [](const InferenceSession* sess) -> const std::vector& { + auto res = sess->GetModelOutputs(); + if (!res.first.IsOK()) { + throw std::runtime_error(res.first.ToString().c_str()); + } else { + return *(res.second); + } + }) + .def_property_readonly("model_meta", [](const InferenceSession* sess) -> const onnxruntime::ModelMetadata& { + auto res = sess->GetModelMetadata(); + if (!res.first.IsOK()) { + throw std::runtime_error(res.first.ToString().c_str()); + } else { + return *(res.second); + } + }); +} + +PYBIND11_MODULE(onnxruntime_pybind11_state, m) { + m.doc() = "pybind11 stateful interface to ONNX runtime"; + + auto initialize = [&]() { + // Initialization of the module + ([]() -> void { + // import_array1() forces a void return value. + import_array1(); + })(); + + static std::unique_ptr env; + auto status = Environment::Create(env); + if (!status.IsOK()) { + throw std::runtime_error(status.ToString().c_str()); + } + + static bool initialized = false; + if (initialized) { + return; + } + initialized = true; + }; + initialize(); + + addGlobalMethods(m); + addObjectMethods(m); +} + +} // namespace python +} // namespace onnxruntime diff --git a/onnxruntime/python/onnxruntime_validation.py b/onnxruntime/python/onnxruntime_validation.py new file mode 100644 index 0000000000000..b337395d87c0a --- /dev/null +++ b/onnxruntime/python/onnxruntime_validation.py @@ -0,0 +1,60 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +#-------------------------------------------------------------------------- +""" +Check OS requirements for ONNX Runtime Python Bindings. +""" +import platform +import linecache +import warnings + +def check_distro_info(): + __my_distro__ = '' + __my_distro_ver__ = '' + __my_system__ = platform.system().lower() + __my_arch__ = platform.architecture()[0].lower() + + __OS_RELEASE_FILE__ = '/etc/os-release' + __LSB_RELEASE_FILE__ = '/etc/lsb-release' + + if __my_arch__ != '64bit': + warnings.warn('Unsupported architecture (%s). ONNX Runtime supports 64bit architecture, only.' % __my_arch__) + + if __my_system__ == 'windows': + __my_distro__ = __my_system__ + __my_distro_ver__ = platform.release().lower() + + if __my_distro_ver__ != '10': + warnings.warn('Unsupported Windows version (%s). ONNX Runtime supports Windows 10 and above, only.' % __my_distro_ver__) + elif __my_system__ == 'linux': + ''' Although the 'platform' python module for getting Distro information works well on standard OS images running on real + hardware, it is not acurate when running on Azure VMs, Git Bash, Cygwin, etc. The returned values for release and version + are unpredictable for virtualized or emulated environments. + /etc/os-release and /etc/lsb_release files, on the other hand, are guaranteed to exist and have standard values in all + OSes supported by onnxruntime. The former is the current standard file to check OS info and the latter is its antecessor. + ''' + # Newer systems have /etc/os-release with relevant distro info + __my_distro__ = linecache.getline(__OS_RELEASE_FILE__, 3)[3:-1] + __my_distro_ver__ = linecache.getline(__OS_RELEASE_FILE__, 6)[12:-2] + + # Older systems may have /etc/os-release instead + if not __my_distro__: + __my_distro__ = linecache.getline(__LSB_RELEASE_FILE__, 1)[11:-1] + __my_distro_ver__ = linecache.getline(__LSB_RELEASE_FILE__, 2)[16:-1] + + # Instead of trying to parse distro specific files, + # warn the user ONNX Runtime may not work out of the box + __my_distro__ = __my_distro__.lower() + __my_distro_ver__ = __my_distro_ver__.lower() + + if __my_distro__ != 'ubuntu' and __my_distro_ver__ != '16.04': + warnings.warn('Unsupported Linux distribution (%s-%s). ONNX Runtime supports Ubuntu 16.04 only.' % (__my_distro__, __my_distro_ver__)) + elif __my_system__ == 'darwin': + __my_distro__ = __my_system__ + __my_distro_ver__ = platform.release().lower() + + if int(__my_distro_ver__.split('.')[0]) < 11: + warnings.warn('Unsupported macOS version (%s). ONNX Runtime supports macOS 11.0 or later.' % (__my_distro_ver__)) + else: + warnings.warn('Unsupported platform (%s). ONNX Runtime supports Linux, macOS and Windows platforms, only.' % __my_system__) diff --git a/onnxruntime/python/session.py b/onnxruntime/python/session.py new file mode 100644 index 0000000000000..a97b8685fa926 --- /dev/null +++ b/onnxruntime/python/session.py @@ -0,0 +1,80 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +#-------------------------------------------------------------------------- + +import sys +import os + +from onnxruntime.capi import _pybind_state as C + + +class InferenceSession: + """ + This is the main class used to run a model. + """ + def __init__(self, path_or_bytes, sess_options=None): + """ + :param path_or_bytes: filename or serialized model in a byte string + :param sess_options: session options + """ + if sess_options: + self._sess = C.InferenceSession( + sess_options, C.get_session_initializer()) + else: + self._sess = C.InferenceSession( + C.get_session_initializer(), C.get_session_initializer()) + + if isinstance(path_or_bytes, str): + self._sess.load_model(path_or_bytes) + elif isinstance(path_or_bytes, bytes): + self._sess.read_bytes(path_or_bytes) + elif isinstance(path_or_bytes, tuple): + # to remove, hidden trick + self._sess.load_model_no_init(path_or_bytes[0]) + else: + raise TypeError("Unable to load from type '{0}'".format(type(path_or_bytes))) + self._inputs_meta = self._sess.inputs_meta + self._outputs_meta = self._sess.outputs_meta + self._model_meta = self._sess.model_meta + + def get_inputs(self): + "Return the inputs metadata as a list of :class:`onnxruntime.NodeArg`." + return self._inputs_meta + + def get_outputs(self): + "Return the outputs metadata as a list of :class:`onnxruntime.NodeArg`." + return self._outputs_meta + + def get_modelmeta(self): + "Return the metadata. See :class:`onnxruntime.ModelMetadata`." + return self._model_meta + + def run(self, output_names, input_feed, run_options=None): + """ + Compute the predictions. + + :param output_names: name of the outputs + :param input_feed: dictionary ``{ input_name: input_value }`` + :param run_options: See :class:`onnxruntime.RunOptions`. + + :: + + sess.run([output_name], {input_name: x}) + """ + num_required_inputs = len(self._inputs_meta) + num_inputs = len(input_feed) + if num_inputs != num_required_inputs: + raise ValueError("Model requires {} inputs. Input Feed contains {}".format(num_required_inputs, num_inputs)) + if not output_names: + output_names = [output.name for output in self._outputs_meta] + return self._sess.run(output_names, input_feed, run_options) + + def end_profiling(self): + """ + End profiling and return results in a file. + + The results are stored in a filename if the option + :meth:`onnxruntime.SessionOptions.enable_profiling`. + """ + return self._sess.end_profiling() diff --git a/onnxruntime/python/tools/__init__.py b/onnxruntime/python/tools/__init__.py new file mode 100644 index 0000000000000..1a8a615c070e1 --- /dev/null +++ b/onnxruntime/python/tools/__init__.py @@ -0,0 +1,4 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +#-------------------------------------------------------------------------- diff --git a/onnxruntime/python/tools/onnxruntime_test.py b/onnxruntime/python/tools/onnxruntime_test.py new file mode 100644 index 0000000000000..56223b1981270 --- /dev/null +++ b/onnxruntime/python/tools/onnxruntime_test.py @@ -0,0 +1,95 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +#-------------------------------------------------------------------------- + +import argparse +import onnxruntime as onnxrt +import numpy as np +import os +import sys +from timeit import default_timer as timer + +float_dict = { + 'tensor(float16)': 'float16', + 'tensor(float)': 'float32', + 'tensor(double)': 'float64' +} + +integer_dict = { + 'tensor(int32)': 'int32', + 'tensor(int8)': 'int8', + 'tensor(uint8)': 'uint8', + 'tensor(int16)': 'int16', + 'tensor(uint16)': 'uint16', + 'tensor(int64)': 'int64', + 'tensor(uint64)': 'uint64' +} + +# simple test program for loading onnx model, feeding all inputs and running the model num_iters times. + + +def main(): + parser = argparse.ArgumentParser(description='Simple ONNX Runtime Test Tool.') + parser.add_argument('model_path', help='model path') + parser.add_argument('num_iters', nargs='?', type=int, + default=1000, help='model run iterations. default=1000') + parser.add_argument('--debug', action='store_true', + help='pause execution to allow attaching a debugger.') + parser.add_argument('--profile', action='store_true', + help='enable chrome timeline trace profiling.') + args = parser.parse_args() + iters = args.num_iters + + if args.debug: + print("Pausing execution ready for debugger to attach to pid: {}".format( + os.getpid())) + print("Press key to continue.") + sys.stdin.read(1) + + sess_options = None + if args.profile: + sess_options = onnxrt.SessionOptions() + sess_options.enable_profiling = True + sess_options.profile_file_prefix = os.path.basename(args.model_path) + + sess = onnxrt.InferenceSession(args.model_path, sess_options) + meta = sess.get_modelmeta() + + feeds = {} + for input_meta in sess.get_inputs(): + # replace any symbolic dimensions (value is None) with 1 + shape = [dim if dim else 1 for dim in input_meta.shape] + if input_meta.type in float_dict: + feeds[input_meta.name] = np.random.rand( + *shape).astype(float_dict[input_meta.type]) + elif input_meta.type in integer_dict: + feeds[input_meta.name] = np.random.uniform( + high=1000, size=tuple(shape)).astype(integer_dict[input_meta.type]) + elif input_meta.type == 'tensor(bool)': + feeds[input_meta.name] = np.random.randint( + 2, size=tuple(shape)).astype('bool') + else: + print("unsupported input type {} for input {}".format( + input_meta.type, input_meta.name)) + sys.exit(-1) + + start = timer() + for i in range(iters): + sess.run([], feeds) # fetch all outputs + end = timer() + + print("model: {}".format(meta.graph_name)) + print("version: {}".format(meta.version)) + print("iterations: {}".format(iters)) + print("avg latency: {} ms".format(((end - start)*1000)/iters)) + + if args.profile: + trace_file = sess.end_profiling() + print("trace file written to: {}".format(trace_file)) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/onnxruntime/python/version_script.lds b/onnxruntime/python/version_script.lds new file mode 100644 index 0000000000000..4e017d0dcdccb --- /dev/null +++ b/onnxruntime/python/version_script.lds @@ -0,0 +1,9 @@ +#_init and _fini should be local +VERS_1.0 { + global: + PyInit_onnxruntime_pybind11_state; + + # Hide everything else. + local: + *; +}; diff --git a/onnxruntime/test/common/logging/helpers.h b/onnxruntime/test/common/logging/helpers.h new file mode 100644 index 0000000000000..c8d027022850c --- /dev/null +++ b/onnxruntime/test/common/logging/helpers.h @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include "date/date.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "core/common/logging/capture.h" +#include "core/common/logging/isink.h" + +class MockSink : public ::onnxruntime::logging::ISink { + public: + MOCK_METHOD3(SendImpl, void(const ::onnxruntime::logging::Timestamp& timestamp, + const std::string& logger_id, + const ::onnxruntime::logging::Capture& message)); +}; + +// The ACTION*() macros trigger warning C4100 (unreferenced formal +// parameter) in MSVC with -W4. Unfortunately they cannot be fixed in +// the macro definition, as the warnings are generated when the macro +// is expanded and macro expansion cannot contain #pragma. Therefore +// we suppress them here. +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4100) +#endif + +ACTION(PrintArgs) { + using date::operator<<; + + // const Timestamp ×tamp, const std::string &logger_id, const Message &message + // arg0 arg1 arg2 + std::cout << arg1 << "@" << arg0 << " " + << arg2.SeverityPrefix() << ":" << arg2.Category() << ":" + << arg2.Location().ToString(::onnxruntime::CodeLocation::kFilenameAndPath) << " " << arg2.Message() << std::endl; +}; + +#ifdef _MSC_VER +#pragma warning(pop) +#endif diff --git a/onnxruntime/test/common/logging/logging_test.cc b/onnxruntime/test/common/logging/logging_test.cc new file mode 100644 index 0000000000000..b9b6ae49df472 --- /dev/null +++ b/onnxruntime/test/common/logging/logging_test.cc @@ -0,0 +1,234 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include +#include + +#include "core/common/logging/isink.h" +#include "core/common/logging/logging.h" +#include "core/common/logging/sinks/clog_sink.h" + +#include "test/common/logging/helpers.h" + +using namespace onnxruntime; +using namespace ::onnxruntime::logging; +using InstanceType = LoggingManager::InstanceType; + +// if we pull in the whole 'testing' namespace we get warnings from date.h as both use '_' in places. +// to avoid that we explicitly pull in the pieces we are using +using testing::Eq; +using testing::Field; +using testing::Ge; +using testing::HasSubstr; +using testing::Property; + +static std::string default_logger_id{"TestFixtureDefaultLogger"}; + +// class to provide single default instance of LoggingManager for use with macros involving 'DEFAULT' +class LoggingTestsFixture : public ::testing::Test { + protected: + static void SetUpTestCase() { + // logger uses kWARNING so we can test filtering of kVERBOSE output, + // and filters user data so that can also be tested +#if !defined(SKIP_DEFAULT_LOGGER_TESTS) + const bool filter_user_data = false; + default_logging_manager_ = std::make_unique( + std::unique_ptr{new CLogSink {}}, Severity::kWARNING, filter_user_data, + InstanceType::Default, &default_logger_id, /*default_max_vlog_level*/ -1); +#endif + } + + static void TearDownTestCase() { + } + + // Objects declared here can be used by all tests in the test case for Foo. + static std::unique_ptr default_logging_manager_; +}; + +std::unique_ptr LoggingTestsFixture::default_logging_manager_; + +/// +/// Tests that the ONNXRUNTIME_WHERE macro populates all fields correctly. +/// +TEST_F(LoggingTestsFixture, TestWhereMacro) { + const std::string logid{"TestWhereMacro"}; + const std::string message{"Testing the WHERE macro."}; + const Severity min_log_level = Severity::kVERBOSE; + + const std::string file = __FILE__; + const std::string function = __FUNCTION__; + int log_line = 0; + + std::cout << function << std::endl; + + MockSink* sink_ptr = new MockSink(); + + EXPECT_CALL(*sink_ptr, SendImpl(testing::_, HasSubstr(logid), + Property(&Capture::Location, + AllOf(Field(&CodeLocation::line_num, Eq(std::ref(log_line))), + Field(&CodeLocation::file_and_path, HasSubstr("onnxruntime")), // path + Field(&CodeLocation::file_and_path, HasSubstr("logging_test.cc")), // filename + Field(&CodeLocation::function, HasSubstr(function)))))) + .WillRepeatedly(PrintArgs()); + + LoggingManager manager{std::unique_ptr(sink_ptr), min_log_level, false, InstanceType::Temporal}; + + std::unique_ptr logger = manager.CreateLogger(logid); + + log_line = __LINE__ + 1; + LOGS(*logger, ERROR) << message; +} + +/// +/// Tests that the logging manager filters based on severity and user data correctly. +/// +TEST_F(LoggingTestsFixture, TestDefaultFiltering) { + const std::string logid{"TestDefaultFiltering"}; + const Severity min_log_level = Severity::kWARNING; + const bool filter_user_data = true; + + MockSink* sink_ptr = new MockSink(); + + EXPECT_CALL(*sink_ptr, SendImpl(testing::_, HasSubstr(logid), testing::_)) // Property(&Capture::Severity, Ge(min_log_level)))) + .Times(1) + .WillRepeatedly(PrintArgs()); + + LoggingManager manager{std::unique_ptr(sink_ptr), min_log_level, filter_user_data, + InstanceType::Temporal}; + + auto logger = manager.CreateLogger(logid); + + LOGS(*logger, VERBOSE) << "Filtered by severity"; + LOGS_USER(*logger, ERROR) << "Filtered user data"; + LOGF(*logger, ERROR, "%s", "hello"); // not filtered + LOGF_USER(*logger, ERROR, "Filtered %s", "user data"); + + LOGS_DEFAULT(WARNING) << "Warning"; // not filtered + LOGS_USER_DEFAULT(ERROR) << "Default logger doesn't filter user data"; + LOGF_DEFAULT(VERBOSE, "Filtered by severity"); + LOGF_USER_DEFAULT(WARNING, "Default logger doesn't filter user data"); +} + +/// +/// Tests that the logger filter overrides work correctly. +/// +TEST_F(LoggingTestsFixture, TestLoggerFiltering) { + const std::string logid{"TestLoggerFiltering"}; + const bool default_filter_user_data = true; + const int default_max_vlog_level = -1; + + MockSink* sink_ptr = new MockSink(); + + int num_expected_calls = 2; +#ifndef NDEBUG + ++num_expected_calls; // VLOG output enabled in DEBUG +#endif + EXPECT_CALL(*sink_ptr, SendImpl(testing::_, HasSubstr(logid), testing::_)) // Property(&Capture::Severity, Ge(min_log_level)))) + .Times(num_expected_calls) + .WillRepeatedly(PrintArgs()); + + LoggingManager manager{std::unique_ptr(sink_ptr), Severity::kERROR, default_filter_user_data, + InstanceType::Temporal, nullptr, default_max_vlog_level}; + + bool filter_user_data = false; + int max_vlog_level = 2; + auto logger = manager.CreateLogger(logid, Severity::kVERBOSE, filter_user_data, max_vlog_level); + + LOGS(*logger, VERBOSE) << "VERBOSE enabled in this logger"; + LOGS_USER(*logger, ERROR) << "USER data not filtered in this logger"; + VLOGS(*logger, 2) << "VLOG enabled up to " << max_vlog_level; +} + +/// +/// Tests that the logging manager constructor validates its usage correctly. +/// +TEST_F(LoggingTestsFixture, TestLoggingManagerCtor) { + // throw if sink is null + EXPECT_THROW((LoggingManager{std::unique_ptr{nullptr}, Severity::kINFO, false, + InstanceType::Temporal}), + std::logic_error); + + // can't have two logging managers with InstanceType of Default. + // this should clash with LoggingTestsFixture::default_logging_manager_ + EXPECT_THROW((LoggingManager{std::unique_ptr{new MockSink{}}, Severity::kINFO, false, + InstanceType::Default}), + std::logic_error); +} + +/// +/// Tests that the conditional logging macros work correctly. +/// +TEST_F(LoggingTestsFixture, TestConditionalMacros) { + const std::string logger_id{"TestConditionalMacros"}; + const Severity min_log_level = Severity::kVERBOSE; + const bool filter_user_data = false; + + MockSink* sink_ptr = new MockSink(); + + // two logging calls that are true using default logger which won't hit our MockSink + + // two logging calls that are true using non-default logger + EXPECT_CALL(*sink_ptr, SendImpl(testing::_, HasSubstr(logger_id), testing::_)) + .Times(2) + .WillRepeatedly(PrintArgs()); + + LoggingManager manager{std::unique_ptr(sink_ptr), min_log_level, filter_user_data, + InstanceType::Temporal}; + + auto logger = manager.CreateLogger(logger_id); + + // macros to use local logger + LOGS_IF(logger != nullptr, *logger, INFO) << "Valid logger"; // true + LOGF_USER_IF(logger != nullptr, *logger, INFO, "Logger is %p", logger.get()); // true + + // macros to test LoggingTestsFixture::default_logging_manager_ + LOGS_DEFAULT_IF(logger == nullptr, INFO) << "Null logger"; // false + LOGS_USER_DEFAULT_IF(logger != nullptr, INFO) << "Valid logger"; // true + LOGF_DEFAULT_IF(logger == nullptr, INFO, "Logger is %p", logger.get()); // false + LOGF_USER_DEFAULT_IF(logger != nullptr, INFO, "Logger is %p", logger.get()); // true +} + +/// +/// Tests that the VLOG* macros produce the expected output. +/// Disabled in Release build, so should be no calls to SendImpl in that case. +/// +TEST_F(LoggingTestsFixture, TestVLog) { + const std::string logid{"TestVLog"}; + + MockSink* sink_ptr = new MockSink(); + + // we only get the non-default calls from below in this sink + EXPECT_CALL(*sink_ptr, SendImpl(testing::_, HasSubstr(logid), testing::_)) +#ifndef NDEBUG + .Times(2) + .WillRepeatedly(PrintArgs()); +#else + .Times(0); +#endif + + const bool filter_user_data = false; + LoggingManager manager{std::unique_ptr(sink_ptr), Severity::kVERBOSE, filter_user_data, InstanceType::Temporal}; + + int max_vlog_level = 2; + auto logger = manager.CreateLogger(logid, Severity::kVERBOSE, filter_user_data, max_vlog_level); + + // test local logger + VLOGS(*logger, max_vlog_level) << "Stream"; // logged + VLOGF(*logger, max_vlog_level + 1, "Printf %d", 1); // ignored due to level + VLOGS_USER(*logger, max_vlog_level + 1) << "User data"; // ignored due to level + VLOGF_USER(*logger, 0, "User Id %d", 1); // logged + + // test default logger - just using macros to check they compile as we can't + // automatically validate the output + VLOGS_DEFAULT(0) << "Stream"; // ignored due to level + VLOGF_DEFAULT(10, "Printf %d", 1); // ignored due to level + VLOGS_USER_DEFAULT(0) << "User data"; // ignored due to level + VLOGF_USER_DEFAULT(0, "User Id %d", 1); // ignored due to level + +#ifndef NDEBUG + // test we can globally disable + logging::vlog_enabled = false; + VLOGS(*logger, 0) << "Should be ignored."; // ignored as disabled +#endif +} diff --git a/onnxruntime/test/common/logging/sinks_test.cc b/onnxruntime/test/common/logging/sinks_test.cc new file mode 100644 index 0000000000000..a748e004f2687 --- /dev/null +++ b/onnxruntime/test/common/logging/sinks_test.cc @@ -0,0 +1,146 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/logging/capture.h" +#include "core/common/logging/logging.h" +#include "core/common/logging/sinks/cerr_sink.h" +#include "core/common/logging/sinks/clog_sink.h" +#include "core/common/logging/sinks/composite_sink.h" +#include "core/common/logging/sinks/file_sink.h" + +#include "test/common/logging/helpers.h" + +using namespace ::onnxruntime::logging; +using InstanceType = LoggingManager::InstanceType; + +namespace { +void CheckStringInFile(const std::string& filename, const std::string& look_for) { + std::ifstream ifs{filename}; + std::string content(std::istreambuf_iterator{ifs}, + std::istreambuf_iterator{}); + + EXPECT_NE(content.find(look_for), std::string::npos); +} + +void DeleteFile(const std::string& filename) { + int result = std::remove(filename.c_str()); + EXPECT_EQ(result, 0); +} +} // namespace + +/// +/// Tests that the std::clog sink produces the expected output. +/// +TEST(LoggingTests, TestCLogSink) { + const std::string filename{"TestCLogSink.out"}; + const std::string logid{"CLogSink"}; + const std::string message{"Test clog message"}; + const Severity min_log_level = Severity::kWARNING; + + // redirect clog to a file so we can check the output + std::ofstream ofs(filename); + + auto old_rdbuf = std::clog.rdbuf(); + std::clog.rdbuf(ofs.rdbuf()); + + // create scoped manager so sink gets destroyed once done + { + LoggingManager manager{std::unique_ptr{new CLogSink{}}, min_log_level, false, + InstanceType::Temporal}; + + auto logger = manager.CreateLogger(logid); + + LOGS(*logger, WARNING) << message; + } + + // check message was flushed to file before we close ofs. + CheckStringInFile(filename, message); + + // revert redirection + std::clog.rdbuf(old_rdbuf); + ofs.close(); + + DeleteFile(filename); +} + +/// +/// Tests that the std::cerr sink produces the expected output. +/// +TEST(LoggingTests, TestCErrSink) { + const std::string filename{"TestCErrSink.out"}; + const std::string logid{"CErrSink"}; + const std::string message{"Test cerr message"}; + const Severity min_log_level = Severity::kWARNING; + + // redirect clog to a file so we can check the output + std::ofstream ofs(filename); + ofs << std::unitbuf; // turn off buffering so we replicate how std::cerr behaves. + + auto old_rdbuf = std::cerr.rdbuf(); + std::cerr.rdbuf(ofs.rdbuf()); + + // create scoped manager so sink gets destroyed once done + { + LoggingManager manager{std::unique_ptr{new CErrSink{}}, min_log_level, false, + InstanceType::Temporal}; + + auto logger = manager.CreateLogger(logid); + + LOGS(*logger, WARNING) << message; + } + + // check message was flushed to file before we close ofs. + CheckStringInFile(filename, message); + + // revert redirection + std::cerr.rdbuf(old_rdbuf); + ofs.close(); + + DeleteFile(filename); +} + +/// +/// Tests that the file_sink produces the expected output. +/// +TEST(LoggingTests, TestFileSink) { + const std::string filename{"TestFileSink.out"}; + const std::string logid{"FileSink"}; + const std::string message{"Test message"}; + const Severity min_log_level = Severity::kWARNING; + + // create scoped manager so sink gets destroyed once done + { + LoggingManager manager{std::unique_ptr{new FileSink{filename, false, false}}, + min_log_level, false, InstanceType::Temporal}; + + auto logger = manager.CreateLogger(logid); + + LOGS(*logger, WARNING) << message; + } + + CheckStringInFile(filename, message); + DeleteFile(filename); +} + +/// +/// Tests that a composite_sink works correctly. +/// +TEST(LoggingTests, TestCompositeSink) { + const std::string logid{"TestCompositeSink"}; + const Severity min_log_level = Severity::kWARNING; + + MockSink* sink_ptr1 = new MockSink(); + MockSink* sink_ptr2 = new MockSink(); + + // both should be called for a single log statement + EXPECT_CALL(*sink_ptr1, SendImpl(testing::_, testing::_, testing::_)).Times(1); + EXPECT_CALL(*sink_ptr2, SendImpl(testing::_, testing::_, testing::_)).Times(1); + + CompositeSink* sink = new CompositeSink(); + sink->AddSink(std::unique_ptr{sink_ptr1}).AddSink(std::unique_ptr{sink_ptr2}); + LoggingManager manager{std::unique_ptr(sink), min_log_level, false, InstanceType::Temporal}; + + auto logger = manager.CreateLogger(logid); + + LOGS_CATEGORY(*logger, WARNING, "ArbitraryCategory") << "Warning"; +} diff --git a/onnxruntime/test/common/test_main.cc b/onnxruntime/test/common/test_main.cc new file mode 100644 index 0000000000000..2f20a160d9014 --- /dev/null +++ b/onnxruntime/test/common/test_main.cc @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/test_environment.h" + +GTEST_API_ int main(int argc, char** argv) { + int status = 0; + + try { + const bool create_default_logger = false; + onnxruntime::test::TestEnvironment environment{argc, argv, create_default_logger}; + + status = RUN_ALL_TESTS(); + } catch (const std::exception& ex) { + std::cerr << ex.what(); + status = -1; + } + + return status; +} diff --git a/onnxruntime/test/contrib_ops/attention_lstm_data_gen.py b/onnxruntime/test/contrib_ops/attention_lstm_data_gen.py new file mode 100644 index 0000000000000..50a09eaa42965 --- /dev/null +++ b/onnxruntime/test/contrib_ops/attention_lstm_data_gen.py @@ -0,0 +1,158 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import tensorflow as tf + +batchSize = 2 + +memMaxStep = 3 +memDepth = 3 +queryMaxStep = 4 +queryDepth = 3 +am_attn_size: int = 2 +cell_hidden_size = 3 +aw_attn_size: int = 2 +am_context_size: int = memDepth + +root_variable_scope = 'LstmAttention' + +with tf.variable_scope(root_variable_scope): + query = tf.get_variable( + "input", + initializer=tf.constant([0.25, -1.5, 1.0, 0.25, -0.5, -1.5, 0.1, 1.5, 0.25, 0.0, 0.0, 0.0, + 0.1, -0.125, 0.25, -0.5, 0.25, 0.1, 1.0, 0.5, -1.5, 0.0, 0.0, 0.0], + shape=[batchSize, queryMaxStep, queryDepth])) + + querySeqLen = tf.Variable(tf.constant([queryMaxStep-1, queryMaxStep-2], tf.int32), name="query_seq_len") + + memory = tf.get_variable( + "memory", + initializer=tf.constant([0.1, -0.25, 1.0, 1.0, -1.0, -1.5, 1.0, 0.25, -0.125, + 0.1, -0.25, 0.5, -0.25, -1.25, 0.25, -1.0, 1.5, -1.250], + shape=[batchSize, memMaxStep, memDepth])) + + memSeqLen = tf.Variable(tf.constant([memMaxStep, memMaxStep-1], dtype=tf.int32), name="mem_seq_len") + + with tf.variable_scope("fwBahdanau"): + fw_mem_layer_weights = tf.get_variable( + "memory_layer/kernel", + initializer=tf.constant([4.0, 2.0, 0.5, -8.0, -2.0, -2.0], shape=[memDepth, am_attn_size])) + + fw_query_layer_weights = tf.get_variable( + "bidirectional_rnn/fw/attention_wrapper/bahdanau_attention/query_layer/kernel", + initializer=tf.constant([-0.125, -0.25, 0.1, -0.125, -0.5, 1.5], + shape=[cell_hidden_size, am_attn_size])) + + fw_aw_attn_weights = tf.get_variable( + "bidirectional_rnn/fw/attention_wrapper/attention_layer/kernel", + initializer=tf.constant([1.5, 1.0, 0.1, -0.25, 0.1, 1.0, -0.25, -0.125, -1.5, -1.5, -0.25, 1.5], + shape=[am_context_size + cell_hidden_size, aw_attn_size])) + + fw_am_attention_v = tf.get_variable( + "bidirectional_rnn/fw/attention_wrapper/bahdanau_attention/attention_v", + initializer=tf.constant([-0.25, 0.1], shape=[am_attn_size])) + + fw_lstm_cell_kernel = tf.get_variable( + "bidirectional_rnn/fw/attention_wrapper/lstm_cell/kernel", + initializer=tf.constant([ + -1.0, -1.5, -0.5, -1.5, 0.1, -0.5, 0.5, -1.5, -0.25, 1.0, -0.125, -0.25, + -1.0, -0.5, 0.25, -0.125, -0.25, -1.0, 1.5, 1.0, -1.5, 0.25, 0.5, 0.5, + 1.5, -0.5, -1.0, -0.5, 0.1, 1.0, 0.1, -0.5, -0.125, -1.5, 0.1, 1.5, + 1.0, -0.5, -0.5, -1.5, -0.125, -0.125, 0.25, -0.25, -0.25, 0.1, -0.5, -0.25, + 0.25, -0.5, 0.1, -0.5, -0.25, 0.25, 0.1, 0.5, -1.5, -0.125, 1.5, 0.5, + -1.5, 1.0, 0.1, -0.5, -1.5, 0.5, -1.0, 0.25, -0.25, 1.0, 0.25, 0.5, + -0.125, 0.1, -1.0, -1.0, 0.1, 1.5, -1.5, 0.1, 1.5, 0.5, 0.25, 1.0, + 1.0, -1.5, -0.25, 0.5, -0.25, 1.0, -1.0, 0.25, -0.5, 0.5, -1.5, 0.5], + shape=[aw_attn_size + queryDepth + cell_hidden_size, 4 * cell_hidden_size])) + + fw_lstm_cell_bias = tf.get_variable( + "bidirectional_rnn/fw/attention_wrapper/lstm_cell/bias", + initializer=tf.constant([0.25, -0.25, 0.1, 1.0, 1.5, -1.5, 1.5, -1.0, -0.25, 1.0, -0.25, 1.0], + shape=[4 * cell_hidden_size])) + + with tf.variable_scope("bwBahdanau"): + fw_mem_layer_weights = tf.get_variable( + "memory_layer/kernel", + initializer=tf.constant([4.0, 2.0, 0.5, -8.0, -2.0, -2.0], shape=[memDepth, am_attn_size])) + + bw_query_layer_weights = tf.get_variable( + "bidirectional_rnn/bw/attention_wrapper/bahdanau_attention/query_layer/kernel", + initializer=tf.constant([-0.125, -0.25, 0.1, -0.125, -0.5, 1.5], + shape=[cell_hidden_size, am_attn_size])) + + bw_aw_attn_weights = tf.get_variable( + "bidirectional_rnn/bw/attention_wrapper/attention_layer/kernel", + initializer=tf.constant([1.5, 1.0, 0.1, -0.25, 0.1, 1.0, -0.25, -0.125, -1.5, -1.5, -0.25, 1.5], + shape=[am_context_size + cell_hidden_size, aw_attn_size])) + + bw_am_attention_v = tf.get_variable( + "bidirectional_rnn/bw/attention_wrapper/bahdanau_attention/attention_v", + initializer=tf.constant([-0.25, 0.1], shape=[am_attn_size])) + + bw_lstm_cell_kernel = tf.get_variable( + "bidirectional_rnn/bw/attention_wrapper/lstm_cell/kernel", + initializer=tf.constant([ + -1.0, -1.5, -0.5, -1.5, 0.1, -0.5, 0.5, -1.5, -0.25, 1.0, -0.125, -0.25, + -1.0, -0.5, 0.25, -0.125, -0.25, -1.0, 1.5, 1.0, -1.5, 0.25, 0.5, 0.5, + 1.5, -0.5, -1.0, -0.5, 0.1, 1.0, 0.1, -0.5, -0.125, -1.5, 0.1, 1.5, + 1.0, -0.5, -0.5, -1.5, -0.125, -0.125, 0.25, -0.25, -0.25, 0.1, -0.5, -0.25, + 0.25, -0.5, 0.1, -0.5, -0.25, 0.25, 0.1, 0.5, -1.5, -0.125, 1.5, 0.5, + -1.5, 1.0, 0.1, -0.5, -1.5, 0.5, -1.0, 0.25, -0.25, 1.0, 0.25, 0.5, + -0.125, 0.1, -1.0, -1.0, 0.1, 1.5, -1.5, 0.1, 1.5, 0.5, 0.25, 1.0, + 1.0, -1.5, -0.25, 0.5, -0.25, 1.0, -1.0, 0.25, -0.5, 0.5, -1.5, 0.5], + shape=[aw_attn_size + queryDepth + cell_hidden_size, 4 * cell_hidden_size])) + + bw_lstm_cell_bias = tf.get_variable( + "bidirectional_rnn/bw/attention_wrapper/lstm_cell/bias", + initializer=tf.constant([0.25, -0.25, 0.1, 1.0, 1.5, -1.5, 1.5, -1.0, -0.25, 1.0, -0.25, 1.0], + shape=[4 * cell_hidden_size])) + +reuse = tf.AUTO_REUSE # tf.AUTO_REUSE or TRUE +with tf.variable_scope(root_variable_scope, reuse=reuse): + with tf.variable_scope("fwBahdanau", reuse=reuse): + fw_am = tf.contrib.seq2seq.BahdanauAttention(am_attn_size, memory, memSeqLen) + with tf.variable_scope("bwBahdanau", reuse=reuse): + bw_am = tf.contrib.seq2seq.BahdanauAttention(am_attn_size, memory, memSeqLen) + + fw_cell = tf.contrib.rnn.LSTMCell(num_units=cell_hidden_size, forget_bias=0.0) + bw_cell = tf.contrib.rnn.LSTMCell(num_units=cell_hidden_size, forget_bias=0.0) + + fw_attn_wrapper = tf.contrib.seq2seq.AttentionWrapper( + fw_cell, fw_am, attention_layer_size=aw_attn_size, output_attention=False) + bw_attn_wrapper = tf.contrib.seq2seq.AttentionWrapper( + bw_cell, bw_am, attention_layer_size=aw_attn_size, output_attention=False) + + outputs, states = tf.nn.bidirectional_dynamic_rnn( + fw_attn_wrapper, bw_attn_wrapper, query, querySeqLen, dtype=tf.float32) + + +tensors = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=root_variable_scope) + +init = tf.global_variables_initializer() + +with tf.Session() as sess: + sess.run(init) + sess.run([outputs, states]) + + tensors.append(outputs[0]) + tensors.append(outputs[1]) + + fw_state = states[0] # output_state_fw + cell = fw_state.cell_state.c + attention = fw_state.attention + alignments = fw_state.alignments + sess.run(tf.Print(cell, [cell], '====FinalState(fw)', summarize=10000)) + sess.run(tf.Print(alignments, [alignments], '====Final Alignment(fw)', summarize=10000)) + sess.run(tf.Print(attention, [attention], '====Final Attention Context(fw)', summarize=10000)) + + bw_state = states[1] # output_state_bw + cell = bw_state.cell_state.c + attention = bw_state.attention + alignments = bw_state.alignments + sess.run(tf.Print(cell, [cell], '====FinalState(bw)', summarize=10000)) + sess.run(tf.Print(alignments, [alignments], '====Final Alignment(bw)', summarize=10000)) + sess.run(tf.Print(attention, [attention], '====Final Attention Context(bw)', summarize=10000)) + + for t in tensors: + shape_str = '[' + ','.join(list(map(lambda x: str(x.__int__()), t.get_shape()))) + ']' + sess.run(tf.Print(t, [tf.reshape(t, [-1])], '\t'.join([t.name, shape_str, '']), summarize=10000)) diff --git a/onnxruntime/test/contrib_ops/attention_lstm_op_test.cc b/onnxruntime/test/contrib_ops/attention_lstm_op_test.cc new file mode 100644 index 0000000000000..a7e4d4858956a --- /dev/null +++ b/onnxruntime/test/contrib_ops/attention_lstm_op_test.cc @@ -0,0 +1,589 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +#include +#include +#include +#include + +namespace onnxruntime { +namespace test { + +// copy the contents of the container to the end so the original values are duplicated +template +T DuplicateContainer(const T& container) { + T doubled; + doubled.reserve(container.size() * 2); // need to avoid reallocation when inserting + std::copy(container.cbegin(), container.cend(), std::back_inserter(doubled)); + std::copy(container.cbegin(), container.cend(), std::back_inserter(doubled)); + + return doubled; +} + +// Test data is generated using attention_lstm_data_gen.py, which covers the all options +// that need to be specified to generate different cases here. Also from the file, it is +// easy to understand how the operate works in Tensorflow. +static void RunAttnLstmTest( + const std::vector& X_data, + const std::vector& W_data, + const std::vector& R_data, + const std::vector& Y_data, + const std::vector& Y_h_data, + const std::vector& Y_c_data, + + const std::vector& MW_data, // memory layer weight: [num_directions, memory_depth, am_attn_depth] + const std::vector& QW_data, // query layer weight: [num_directions, query_layer_depth=cell_hidden_size, am_attn_depth] + const std::vector& attn_v_data, // [num_directions, am_attn_depth] + const std::vector& M_data, // memory sequence: [batch_size, maxMemoryTimeStep, memory_depth] + const std::vector* memory_sequence_lengths, // [batch_size] + const std::vector* attn_layer_weights, // [num_directions, cell_hidden_size + memory_depth, aw_hidden_depth] + + int64_t x_depth, + int batch_size, + int64_t hidden_size, + int64_t seq_length, + + int64_t memory_max_time, + int64_t memory_depth, + int64_t am_attn_size, + int64_t aw_attn_size, + + const std::vector* B_data = nullptr, + const std::vector* P_data = nullptr, + const std::vector* initial_h_data = nullptr, + const std::vector* initial_c_data = nullptr, + const std::vector* sequence_lengths = nullptr, + const std::string& direction = "forward", + float clip = -9999.f, + bool output_sequence = true, + bool input_forget = false, + // copy the following vectors as we may modify them + std::vector activations = {}, + std::vector activation_alphas = {}, + std::vector activation_betas = {}) { + const int64_t input_size = x_depth + aw_attn_size; + + OpTester test("AttnLSTM", 1, onnxruntime::kMSDomain); + + int num_directions = (direction == "bidirectional") ? 2 : 1; + + if (activations.empty()) { + activations = {"sigmoid", "tanh", "tanh"}; + } + + if (num_directions == 2 && activations.size() == 3) { + activations = DuplicateContainer(activations); + } + + test.AddAttribute>("activations", activations); + if (!activation_alphas.empty()) + test.AddAttribute>("activation_alpha", activation_alphas); + if (!activation_betas.empty()) + test.AddAttribute>("activation_beta", activation_betas); + + test.AddAttribute("direction", direction); + test.AddAttribute("hidden_size", hidden_size); + // test.AddAttribute("output_sequence", output_sequence); + test.AddAttribute("input_forget", input_forget); + if (clip < 0.0f) clip = std::numeric_limits::max(); + test.AddAttribute("clip", clip); + + std::vector X_dims = {seq_length, batch_size, x_depth}; + std::vector W_dims = {num_directions, 4 * hidden_size, input_size}; + std::vector R_dims = {num_directions, 4 * hidden_size, hidden_size}; + + test.AddInput("X", X_dims, X_data); + test.AddInput("W", W_dims, W_data); + test.AddInput("R", R_dims, R_data); + + if (B_data) { + std::vector B_dims = {num_directions, 8 * hidden_size}; + test.AddInput("B", B_dims, *B_data); + } else { + test.AddMissingOptionalInput(); + } + + if (sequence_lengths) { + std::vector sequence_lens_dims{batch_size}; + test.AddInput("sequence_lens", sequence_lens_dims, *sequence_lengths); + } else { + test.AddMissingOptionalInput(); + } + + if (initial_h_data && !initial_h_data->empty()) { + std::vector initial_h_dims = {num_directions, batch_size, hidden_size}; + test.AddInput("initial_h", initial_h_dims, *initial_h_data); + } else { + test.AddMissingOptionalInput(); + } + + if (initial_c_data && !initial_c_data->empty()) { + std::vector initial_c_dims = {num_directions, batch_size, hidden_size}; + test.AddInput("initial_c", initial_c_dims, *initial_c_data); + } else { + test.AddMissingOptionalInput(); + } + + if (P_data && !P_data->empty()) { + std::vector P_dims = {num_directions, 3 * hidden_size}; + test.AddInput("P", P_dims, *P_data); + } else { + test.AddMissingOptionalInput(); + } + + std::vector QW_dims{num_directions, hidden_size, am_attn_size}; + test.AddInput("QW", QW_dims, QW_data); + + std::vector MW_dims{num_directions, memory_depth, am_attn_size}; + test.AddInput("MW", MW_dims, MW_data); + + std::vector attn_v_dims{num_directions, am_attn_size}; + test.AddInput("V", attn_v_dims, attn_v_data); + + std::vector M_dims{batch_size, memory_max_time, memory_depth}; + test.AddInput("M", M_dims, M_data); + + if (memory_sequence_lengths) { + std::vector M_seq_dims{batch_size}; + test.AddInput("memory_seq_lens", M_seq_dims, *memory_sequence_lengths); + } else { + test.AddMissingOptionalInput(); + } + + if (attn_layer_weights) { + std::vector attn_layer_weight_dims{num_directions, memory_depth + hidden_size, aw_attn_size}; + test.AddInput("AW", attn_layer_weight_dims, *attn_layer_weights); + } else { + test.AddMissingOptionalInput(); + } + + if (output_sequence != 0 && !Y_data.empty()) { + std::vector Y_dims = {seq_length, num_directions, batch_size, hidden_size}; + test.AddOutput("Y", Y_dims, Y_data); + } else { + // add placeholder so node counts match as Y_h will always be the second Y_data, + // so Y must exist as the first Y_data + test.AddMissingOptionalOutput(); + } + + if (!Y_h_data.empty()) { + std::vector Y_h_dims{num_directions, batch_size, hidden_size}; + test.AddOutput("Y_h", Y_h_dims, Y_h_data); + } else { + test.AddMissingOptionalOutput(); + } + + if (!Y_c_data.empty()) { + std::vector Y_c_dims{num_directions, batch_size, hidden_size}; + test.AddOutput("Y_c", Y_c_dims, Y_c_data); + } else { + test.AddMissingOptionalOutput(); + } + + test.Run(); +} + +template +static std::vector ConcatDup(const std::vector& src) { + std::vector dst(2 * src.size()); + std::copy(src.cbegin(), src.cend(), dst.begin()); + std::copy(src.cbegin(), src.cend(), dst.begin() + src.size()); + return dst; +} + +template +static std::vector ConcatLastDim(const std::vector& a, const std::vector& b, int depth) { + std::vector dst(2 * a.size()); + for (int s = 0; s < a.size(); s += depth) { + std::copy(a.cbegin() + s, a.cbegin() + s + depth, dst.begin() + 2 * s); + std::copy(b.cbegin() + s, b.cbegin() + s + depth, dst.begin() + 2 * s + depth); + } + return dst; +} + +template +static std::vector Transpose2D(const std::vector& src, int num_rows, int num_cols) { + std::vector dst(src.size()); + int pos = 0; + for (int x = 0; x < num_rows; ++x) { + for (int y = 0; y < num_cols; ++y) { + dst[y * num_rows + x] = src[pos++]; + } + } + return dst; +} + +template +static std::vector ConvertBatchSeqToSeqBatch( + const std::vector& bs, int batch_size, int max_step, int depth) { + std::vector sb(bs.size()); + int pos = 0; + for (int b = 0; b < batch_size; ++b) { + for (int t = 0; t < max_step; ++t) { + for (int i = 0; i < depth; ++i) { + int tp = t * batch_size * depth + b * depth + i; + sb[tp] = bs[pos++]; + } + } + } + return sb; +} + +// This convert seq in [*, 4*cell_hidden_size] from: I, J(C), F, O into: I O, F, C(J) +// for weights from semantic TF to Onnx semantic. +template +static std::vector ConvertIcfoToIofc(const std::vector& icfo, int cell_hidden_size) { + std::vector iofc(icfo.size()); + for (int i = 0; i < icfo.size(); i += 4 * cell_hidden_size) { + auto src = icfo.cbegin() + i; + auto dst = iofc.begin() + i; + + std::copy(src + 0 * cell_hidden_size, src + 1 * cell_hidden_size, dst + 0 * cell_hidden_size); + std::copy(src + 3 * cell_hidden_size, src + 4 * cell_hidden_size, dst + 1 * cell_hidden_size); + std::copy(src + 2 * cell_hidden_size, src + 3 * cell_hidden_size, dst + 2 * cell_hidden_size); + std::copy(src + 1 * cell_hidden_size, src + 2 * cell_hidden_size, dst + 3 * cell_hidden_size); + } + return iofc; +} + +//Settings for this group of test data +static const int batch_size = 1; +static const int memory_max_step = 3; +static const int memory_depth = 3; +static const int input_max_step = 3; +static const int input_only_depth = 3; +static const int am_attn_size = 2; +static const int cell_hidden_size = 3; +static const int aw_attn_size = 2; +static const int input_size = input_only_depth + aw_attn_size; + +// [batch_size=1, memory_max_step=3, memory_depth=3] +static std::vector s_M_data{0.1f, -0.25f, 1.0f, 1.0f, -1.0f, -1.5f, 1.0f, 0.25f, -0.125f}; +static const std::vector s_M_2batch{0.1f, -0.25f, 1.0f, 1.0f, -1.0f, -1.5f, 1.0f, 0.25f, -0.125f, + 0.1f, -0.25f, 0.5f, -0.25f, -1.25f, 0.25f, -1.0f, 1.5f, -1.25f}; + +//real seq lens for memory +static std::vector s_mem_seq_lenghts{3}; +static const std::vector s_mem_seq_lenghts_2batch{3, 2}; + +// [batch_size=1, input_max_step=3, input_only_depth=3] +static std::vector s_X_T_data{ + 0.25f, + -1.5f, + 1.0f, + 0.25f, + -0.5f, + -1.5f, + 0.1f, + 1.5f, + 0.25f, +}; + +//real seq lens for X +static std::vector s_seq_lengths{3}; + +// [num_directions, memory_depth=3, am_attn_size=2] +static std::vector s_memory_layer_weight{4.0f, 2.0f, 0.5f, -8.0f, -2.0f, -2.0f}; + +// [num_directions, query_depth(cell_hidden_size)=3, am_attn_size=2] +static std::vector s_query_layer_weight{-0.125f, -0.25f, 0.1f, -0.125f, -0.5f, 1.5f}; + +// [num_directions, memory_depth+cell_hidden_size=3+3=6, aw_attn_size=2] +static std::vector s_attn_layer_weight{1.5f, 1.0f, 0.1f, -0.25f, 0.1f, 1.0f, -0.25f, -0.125f, -1.5f, -1.5f, -0.25f, 1.5f}; + +//[2] +static std::vector s_attn_v{-0.25f, 0.1f}; + +//lstm kernel weights, [8, 12] 8 = x_depth + aw_attn_size + cell_hidden_size, 12 = 4 * cell_hidden_size (ijfo) +static std::vector s_WR_T_data_ICFO{ + // ---- x_depth lines of attention input weight + -1.0f, -1.5f, -0.5f, -1.5f, 0.1f, -0.5f, 0.5f, -1.5f, -0.25f, 1.0f, -0.125f, -0.25f, + -1.0f, -0.5f, 0.25f, -0.125f, -0.25f, -1.0f, 1.5f, 1.0f, -1.5f, 0.25f, 0.5f, 0.5f, + 1.5f, -0.5f, -1.0f, -0.5f, 0.1f, 1.0f, 0.1f, -0.5f, -0.125f, -1.5f, 0.1f, 1.5f, + // ---- aw_attn_size lines of attention input weight + 1.0f, -0.5f, -0.5f, -1.5f, -0.125f, -0.125f, 0.25f, -0.25f, -0.25f, 0.1f, -0.5f, -0.25f, + 0.25f, -0.5f, 0.1f, -0.5f, -0.25f, 0.25f, 0.1f, 0.5f, -1.5f, -0.125f, 1.5f, 0.5f, + // ---- cell_hidden_size lines of attention input weight + -1.5f, 1.0f, 0.1f, -0.5f, -1.5f, 0.5f, -1.0f, 0.25f, -0.25f, 1.0f, 0.25f, 0.5f, + -0.125f, 0.1f, -1.0f, -1.0f, 0.1f, 1.5f, -1.5f, 0.1f, 1.5f, 0.5f, 0.25f, 1.0f, + 1.0f, -1.5f, -0.25f, 0.5f, -0.25f, 1.0f, -1.0f, 0.25f, -0.5f, 0.5f, -1.5f, 0.5f}; + +//lstm_cell_bias, [12] = 4 * 3, append extra zero for onnix +std::vector s_lstm_cell_bias_ICFO{ + 0.25f, -0.25f, 0.1f, 1.0f, 1.5f, -1.5f, 1.5f, -1.0f, -0.25f, 1.0f, -0.25f, 1.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; + +TEST(AttnLSTMTest, ForwardLstmWithBahdanauAMZeroAttention) { + std::vector X_data = ConvertBatchSeqToSeqBatch(s_X_T_data, batch_size, input_max_step, input_only_depth); + + //clear attention layer weight, so that no attention value will be in effective + std::vector zero_attn_layer_weight(s_attn_layer_weight.size(), 0.0f); + + std::vector WR_T_data = ConvertIcfoToIofc(s_WR_T_data_ICFO, cell_hidden_size); + + const size_t W_data_size = 5 * 12; + std::vector W_T_data(&(WR_T_data[0]), &(WR_T_data[0]) + W_data_size); + // Fake zero for attention input weight now + std::fill(W_T_data.begin() + 3 * 12, W_T_data.begin() + W_data_size, 0.0f); + + std::vector R_T_data(&(WR_T_data[0]) + W_data_size, &(WR_T_data[0]) + WR_T_data.size()); + + // transpose W and R for onnx sematic + std::vector W_data = Transpose2D(W_T_data, input_size, 4 * cell_hidden_size); + std::vector R_data = Transpose2D(R_T_data, cell_hidden_size, 4 * cell_hidden_size); + + std::vector B_data = ConvertIcfoToIofc(s_lstm_cell_bias_ICFO, cell_hidden_size); + + // [1, 3, 3] + std::vector Y_T_data{ + 0.0978363082f, 0.105625421f, 0.116753615f, + 0.277829766f, 0.166462898f, -0.119725525f, + 0.244476393f, 0.344343185f, -0.470122069f}; + + // convert to onnx output semantic, should be same in this case. + std::vector Y_data = ConvertBatchSeqToSeqBatch( + Y_T_data, batch_size, input_max_step, cell_hidden_size); + + const std::vector Y_h_data{}; + const std::vector Y_c_data{}; + + RunAttnLstmTest( + X_data, W_data, R_data, Y_data, Y_h_data, Y_c_data, + s_memory_layer_weight, s_query_layer_weight, s_attn_v, s_M_data, &s_mem_seq_lenghts, &zero_attn_layer_weight, + input_only_depth, batch_size, cell_hidden_size, input_max_step, + memory_max_step, memory_depth, am_attn_size, aw_attn_size, + &B_data, nullptr, nullptr, nullptr, &s_seq_lengths, + "forward", -9999.f, true, false); +} + +TEST(AttnLSTMTest, ForwardLstmWithBahdanauAM) { + std::vector X_data = ConvertBatchSeqToSeqBatch(s_X_T_data, batch_size, input_max_step, input_only_depth); + + std::vector WR_T_data = ConvertIcfoToIofc(s_WR_T_data_ICFO, cell_hidden_size); + + const size_t W_data_size = 5 * 12; + std::vector W_T_data(&(WR_T_data[0]), &(WR_T_data[0]) + W_data_size); + std::vector R_T_data(&(WR_T_data[0]) + W_data_size, &(WR_T_data[0]) + WR_T_data.size()); + + // transpose W and R for onnx sematic + std::vector W_data = Transpose2D(W_T_data, input_size, 4 * cell_hidden_size); + std::vector R_data = Transpose2D(R_T_data, cell_hidden_size, 4 * cell_hidden_size); + + std::vector B_data = ConvertIcfoToIofc(s_lstm_cell_bias_ICFO, cell_hidden_size); + + // [1, 3, 3] + std::vector Y_T_data{ + 0.0978363082f, 0.105625421f, 0.116753615f, + 0.236107856f, 0.195716992f, -0.133973882f, + -0.029754376f, 0.274325848f, -0.387993187f}; + + // convert to onnx output semantic, should be same in this case. + std::vector Y_data = ConvertBatchSeqToSeqBatch( + Y_T_data, batch_size, input_max_step, cell_hidden_size); + + const std::vector Y_h_data{}; + const std::vector Y_c_data{}; + + RunAttnLstmTest( + X_data, W_data, R_data, Y_data, Y_h_data, Y_c_data, + s_memory_layer_weight, s_query_layer_weight, s_attn_v, s_M_data, &s_mem_seq_lenghts, &s_attn_layer_weight, + input_only_depth, batch_size, cell_hidden_size, input_max_step, + memory_max_step, memory_depth, am_attn_size, aw_attn_size, + &B_data, nullptr, nullptr, nullptr, &s_seq_lengths, + "forward", -9999.f, true, false); +} + +TEST(AttnLSTMTest, ForwardLstmWithBahdanauAMShortenSeqLength) { + std::vector X_data = ConvertBatchSeqToSeqBatch(s_X_T_data, batch_size, input_max_step, input_only_depth); + + std::vector WR_T_data = ConvertIcfoToIofc(s_WR_T_data_ICFO, cell_hidden_size); + + const size_t W_data_size = 5 * 12; + std::vector W_T_data(&(WR_T_data[0]), &(WR_T_data[0]) + W_data_size); + std::vector R_T_data(&(WR_T_data[0]) + W_data_size, &(WR_T_data[0]) + WR_T_data.size()); + + // transpose W and R for onnx sematic + std::vector W_data = Transpose2D(W_T_data, input_size, 4 * cell_hidden_size); + std::vector R_data = Transpose2D(R_T_data, cell_hidden_size, 4 * cell_hidden_size); + + std::vector B_data = ConvertIcfoToIofc(s_lstm_cell_bias_ICFO, cell_hidden_size); + + // [1, 3, 3] + std::vector Y_T_data{ + 0.0978363082f, 0.105625421f, 0.116753615f, + 0.236107856f, 0.195716992f, -0.133973882f, + 0.0f, 0.0f, 0.0f}; + + // convert to onnx output semantic, should be same in this case. + std::vector Y_data = ConvertBatchSeqToSeqBatch( + Y_T_data, batch_size, input_max_step, cell_hidden_size); + + const std::vector Y_h_data{}; + const std::vector Y_c_data{}; + + std::vector shortenSeqLen{2}; + + RunAttnLstmTest( + X_data, W_data, R_data, Y_data, Y_h_data, Y_c_data, + s_memory_layer_weight, s_query_layer_weight, s_attn_v, s_M_data, &s_mem_seq_lenghts, &s_attn_layer_weight, + input_only_depth, batch_size, cell_hidden_size, input_max_step, + memory_max_step, memory_depth, am_attn_size, aw_attn_size, + &B_data, nullptr, nullptr, nullptr, &shortenSeqLen, + "forward", -9999.f, true, false); +} + +TEST(AttnLSTMTest, ReverseLstmWithBahdanauAMShortenSeqLength) { + std::vector X_data = ConvertBatchSeqToSeqBatch(s_X_T_data, batch_size, input_max_step, input_only_depth); + + std::vector WR_T_data = ConvertIcfoToIofc(s_WR_T_data_ICFO, cell_hidden_size); + + const size_t W_data_size = 5 * 12; + std::vector W_T_data(&(WR_T_data[0]), &(WR_T_data[0]) + W_data_size); + std::vector R_T_data(&(WR_T_data[0]) + W_data_size, &(WR_T_data[0]) + WR_T_data.size()); + + // transpose W and R for onnx sematic + std::vector W_data = Transpose2D(W_T_data, input_size, 4 * cell_hidden_size); + std::vector R_data = Transpose2D(R_T_data, cell_hidden_size, 4 * cell_hidden_size); + + std::vector B_data = ConvertIcfoToIofc(s_lstm_cell_bias_ICFO, cell_hidden_size); + + // [1, 3, 3] + std::vector Y_T_data{ + -0.229537353f, 0.136488736f, -0.414591223f, + 0.127119571f, 0.164731115f, -0.1136849f, + 0.0f, 0.0f, 0.0f}; + + // convert to onnx output semantic, should be same in this case. + std::vector Y_data = ConvertBatchSeqToSeqBatch( + Y_T_data, batch_size, input_max_step, cell_hidden_size); + + const std::vector Y_h_data{}; + const std::vector Y_c_data{}; + + std::vector shortenSeqLen{2}; + + RunAttnLstmTest( + X_data, W_data, R_data, Y_data, Y_h_data, Y_c_data, + s_memory_layer_weight, s_query_layer_weight, s_attn_v, s_M_data, &s_mem_seq_lenghts, &s_attn_layer_weight, + input_only_depth, batch_size, cell_hidden_size, input_max_step, + memory_max_step, memory_depth, am_attn_size, aw_attn_size, + &B_data, nullptr, nullptr, nullptr, &shortenSeqLen, + "reverse", -9999.f, true, false); +} + +TEST(AttnLSTMTest, BidirectionLstmWithBahdanauAMShortenSeqLength) { + std::vector X_data = ConvertBatchSeqToSeqBatch(s_X_T_data, batch_size, input_max_step, input_only_depth); + + std::vector WR_T_data = ConvertIcfoToIofc(s_WR_T_data_ICFO, cell_hidden_size); + + const size_t W_data_size = 5 * 12; + std::vector W_T_data(&(WR_T_data[0]), &(WR_T_data[0]) + W_data_size); + std::vector R_T_data(&(WR_T_data[0]) + W_data_size, &(WR_T_data[0]) + WR_T_data.size()); + + // transpose W and R for onnx sematic + std::vector W_data = Transpose2D(W_T_data, input_size, 4 * cell_hidden_size); + std::vector R_data = Transpose2D(R_T_data, cell_hidden_size, 4 * cell_hidden_size); + + std::vector B_data = ConvertIcfoToIofc(s_lstm_cell_bias_ICFO, cell_hidden_size); + + // concat two result sequence from tf + std::vector Y_data = ConcatLastDim( + ConvertBatchSeqToSeqBatch( + std::vector{ + 0.0978363082f, 0.105625421f, 0.116753615f, + 0.236107856f, 0.195716992f, -0.133973882f, + 0.0f, 0.0f, 0.0f}, + batch_size, input_max_step, cell_hidden_size), + ConvertBatchSeqToSeqBatch( + std::vector{ + -0.229537353f, 0.136488736f, -0.414591223f, + 0.127119571f, 0.164731115f, -0.1136849f, + 0.0f, 0.0f, 0.0f}, + batch_size, input_max_step, cell_hidden_size), + cell_hidden_size * batch_size); + + const std::vector Y_h_data{}; + const std::vector Y_c_data{}; + + std::vector shortenSeqLen{2}; + + auto d_W_data = ConcatDup(W_data); + auto d_R_data = ConcatDup(R_data); + auto d_B_data = ConcatDup(B_data); + + auto d_memory_layer_weight = ConcatDup(s_memory_layer_weight); + auto d_query_layer_weight = ConcatDup(s_query_layer_weight); + auto d_attn_v = ConcatDup(s_attn_v); + auto d_attn_layer_weight = ConcatDup(s_attn_layer_weight); + + RunAttnLstmTest( + X_data, d_W_data, d_R_data, Y_data, Y_h_data, Y_c_data, + d_memory_layer_weight, d_query_layer_weight, d_attn_v, s_M_data, &s_mem_seq_lenghts, &d_attn_layer_weight, + input_only_depth, batch_size, cell_hidden_size, input_max_step, + memory_max_step, memory_depth, am_attn_size, aw_attn_size, + &d_B_data, nullptr, nullptr, nullptr, &shortenSeqLen, + "bidirectional", -9999.f, true, false); +} + +TEST(AttnLSTMTest, BidirectionLstmWithBahdanauAM2BatchShortenSeqLen) { + const int batch2Size = 2; + const int inputMaxStep4 = 4; + + static const std::vector s_X_T_2batch{0.25f, -1.5f, 1.0f, 0.25f, -0.5f, -1.5f, 0.1f, 1.5f, 0.25f, 0.0f, 0.0f, 0.0f, + 0.1f, -0.125f, 0.25f, -0.5f, 0.25f, 0.1f, 1.0f, 0.5f, -1.5f, 0.0f, 0.0f, 0.0f}; + static const std::vector s_seq_lengths_2batch{3, 2}; + + std::vector X_data = ConvertBatchSeqToSeqBatch(s_X_T_2batch, batch2Size, inputMaxStep4, input_only_depth); + + std::vector WR_T_data = ConvertIcfoToIofc(s_WR_T_data_ICFO, cell_hidden_size); + + const size_t W_data_size = 5 * 12; + std::vector W_T_data(&(WR_T_data[0]), &(WR_T_data[0]) + W_data_size); + std::vector R_T_data(&(WR_T_data[0]) + W_data_size, &(WR_T_data[0]) + WR_T_data.size()); + + // transpose W and R for onnx sematic + std::vector W_data = Transpose2D(W_T_data, input_size, 4 * cell_hidden_size); + std::vector R_data = Transpose2D(R_T_data, cell_hidden_size, 4 * cell_hidden_size); + + std::vector B_data = ConvertIcfoToIofc(s_lstm_cell_bias_ICFO, cell_hidden_size); + + // concat two result sequence from tf + std::vector Y_data = ConcatLastDim( + ConvertBatchSeqToSeqBatch( + std::vector{ + 0.0978363082f, 0.105625421f, 0.116753615f, 0.236107856f, 0.195716992f, -0.133973882f, -0.029754376f, 0.274325848f, -0.387993187f, 0.0f, 0.0f, 0.0f, + 0.261070877f, 0.144692719f, -0.274273455f, -0.272313654f, 0.324584424f, -0.298215479f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}, + batch2Size, inputMaxStep4, cell_hidden_size), + ConvertBatchSeqToSeqBatch( + std::vector{ + -0.248873845f, 0.139064044f, -0.596312642f, 0.134674609f, 0.255465984f, -0.119320348f, 0.10030812f, 0.110956885f, -0.438956916f, 0.0f, 0.0f, 0.0f, + -0.230028018f, 0.230880141f, -0.193421111f, 0.328211069f, 0.230195627f, -0.3777861f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}, + batch2Size, inputMaxStep4, cell_hidden_size), + cell_hidden_size * batch2Size); + + const std::vector Y_h_data{}; + const std::vector Y_c_data{}; + + auto d_W_data = ConcatDup(W_data); + auto d_R_data = ConcatDup(R_data); + auto d_B_data = ConcatDup(B_data); + + auto d_memory_layer_weight = ConcatDup(s_memory_layer_weight); + auto d_query_layer_weight = ConcatDup(s_query_layer_weight); + auto d_attn_v = ConcatDup(s_attn_v); + auto d_attn_layer_weight = ConcatDup(s_attn_layer_weight); + + RunAttnLstmTest( + X_data, d_W_data, d_R_data, Y_data, Y_h_data, Y_c_data, + d_memory_layer_weight, d_query_layer_weight, d_attn_v, s_M_2batch, &s_mem_seq_lenghts_2batch, &d_attn_layer_weight, + input_only_depth, batch2Size, cell_hidden_size, inputMaxStep4, + memory_max_step, memory_depth, am_attn_size, aw_attn_size, + &d_B_data, nullptr, nullptr, nullptr, &s_seq_lengths_2batch, + "bidirectional", -9999.f, true, false); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/contrib_ops/expand_dims_test.cc b/onnxruntime/test/contrib_ops/expand_dims_test.cc new file mode 100644 index 0000000000000..df34a5eb8d30f --- /dev/null +++ b/onnxruntime/test/contrib_ops/expand_dims_test.cc @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(ContribOpTest, ExpandDims_0) { + OpTester test("ExpandDims", 1, onnxruntime::kMSDomain); + test.AddShapeToTensorData(false); // TODO: re-enable shape inference test + test.AddInput("X", {2, 3}, std::vector(6, 1.0f)); + test.AddInput("axis", {}, {-1}); + test.AddOutput("Y", {2, 3, 1}, std::vector(6, 1.0f)); + test.Run(); +} + +TEST(ContribOpTest, ExpandDims_1) { + OpTester test("ExpandDims", 1, onnxruntime::kMSDomain); + test.AddShapeToTensorData(false); // TODO: re-enable shape inference test + test.AddInput("X", {2, 3}, std::vector(6, 1.0f)); + test.AddInput("axis", {}, {1}); + test.AddOutput("Y", {2, 1, 3}, std::vector(6, 1.0f)); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/contrib_ops/isnan_test.cc b/onnxruntime/test/contrib_ops/isnan_test.cc new file mode 100644 index 0000000000000..43a7e781ce174 --- /dev/null +++ b/onnxruntime/test/contrib_ops/isnan_test.cc @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" +#include // NAN + +namespace onnxruntime { +namespace test { + +TEST(ContribOpTest, IsNaN) { + OpTester test("IsNaN", 1, onnxruntime::kMSDomain); + std::vector dims{2, 2}; + test.AddInput("X", dims, {1.0f, NAN, 2.0f, NAN}); + test.AddOutput("Y", dims, {false, true, false, true}); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/contrib_ops/sample_op_test.cc b/onnxruntime/test/contrib_ops/sample_op_test.cc new file mode 100644 index 0000000000000..53d47acc99cc6 --- /dev/null +++ b/onnxruntime/test/contrib_ops/sample_op_test.cc @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(MLOpTest, SampleOpFloat) { + OpTester test("SampleOp", 1, onnxruntime::kMSDomain); + std::vector X = {0.8f, -0.5f, 0.0f, 1.f, 1.0f}; + std::vector expected_output = X; + const int64_t N = static_cast(X.size()); + test.AddInput("X", {N}, X); + test.AddOutput("Y", {N}, expected_output); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/custom_op_shared_lib/test_custom_op.cc b/onnxruntime/test/custom_op_shared_lib/test_custom_op.cc new file mode 100644 index 0000000000000..0abd22539cf5f --- /dev/null +++ b/onnxruntime/test/custom_op_shared_lib/test_custom_op.cc @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// example custom op + +#include "core/framework/custom_ops_author.h" +#include "core/common/visibility_macros.h" + +using namespace onnxruntime; +using namespace onnxruntime::common; +using namespace ONNX_NAMESPACE; + +class FooKernel : public OpKernel { + public: + FooKernel(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* ctx) const override { + const Tensor* X = ctx->Input(0); + const Tensor* W = ctx->Input(1); + auto* X_data = X->template Data(); + auto* W_data = W->template Data(); + Tensor* Y = ctx->Output(0, X->Shape()); + auto* Y_data = Y->template MutableData(); + + for (int64_t i = 0; i < X->Shape().Size(); i++) { + Y_data[i] = X_data[i] + W_data[i]; + } + + return Status::OK(); + } +}; + +ONNX_RUNTIME_EXPORT KernelsContainer* GetAllKernels() { + KernelsContainer* kc = new KernelsContainer; + + KernelDefBuilder def_builder; + def_builder.SetName("Foo") + .SetDomain(onnxruntime::kOnnxDomain) + .SinceVersion(7) + .Provider(onnxruntime::kCpuExecutionProvider) + .TypeConstraint("T", DataTypeImpl::GetTensorType()); + KernelCreateFn kernel_create_fn = [](const OpKernelInfo& info) -> OpKernel* { return new FooKernel(info); }; + KernelCreateInfo create_info(def_builder.Build(), kernel_create_fn); + kc->kernels_list.push_back(std::move(create_info)); + return kc; +} + +ONNX_RUNTIME_EXPORT SchemasContainer* GetAllSchemas() { + SchemasContainer* sc = new SchemasContainer; + sc->domain = onnxruntime::kOnnxDomain; + sc->baseline_opset_version = 5; + sc->opset_version = 7; + + ONNX_NAMESPACE::OpSchema schema("Foo", "unknown", 0); + schema.Input(0, + "A", + "First operand, should share the type with the second operand.", + "T"); + schema.Input( + 1, + "B", + "Second operand. With broadcasting can be of smaller size than A. " + "If broadcasting is disabled it should be of the same size.", + "T"); + schema.Output(0, "C", "Result, has same dimensions and type as A", "T"); + schema.TypeConstraint( + "T", + OpSchema::numeric_types_for_math_reduction(), + "Constrain input and output types to high-precision numeric tensors."); + schema.SinceVersion(7); + + sc->schemas_list.push_back(schema); + return sc; +} + +ONNX_RUNTIME_EXPORT void FreeKernelsContainer(KernelsContainer* kc) { + delete kc; +} + +ONNX_RUNTIME_EXPORT void FreeSchemasContainer(SchemasContainer* sc) { + delete sc; +} diff --git a/onnxruntime/test/framework/TestAllocatorManager.cc b/onnxruntime/test/framework/TestAllocatorManager.cc new file mode 100644 index 0000000000000..499ef7468c5cb --- /dev/null +++ b/onnxruntime/test/framework/TestAllocatorManager.cc @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "test/framework/TestAllocatorManager.h" +#include "core/framework/allocatormgr.h" +#ifdef USE_CUDA +#include "core/providers/cuda/cuda_allocator.h" +#endif // USE_CUDA + +namespace onnxruntime { +namespace test { + +static std::string GetAllocatorId(const std::string& name, const int id, const bool isArena) { + std::ostringstream ss; + if (isArena) + ss << "arena_"; + else + ss << "device_"; + ss << name << "_" << id; + return ss.str(); +} + +static Status RegisterAllocator(std::unordered_map& map, + std::unique_ptr allocator, size_t /*memory_limit*/, + bool use_arena) { + auto& info = allocator->Info(); + auto allocator_id = GetAllocatorId(info.name, info.id, use_arena); + + auto status = Status::OK(); + if (map.find(allocator_id) != map.end()) + status = Status(common::ONNXRUNTIME, common::FAIL, "allocator already exists"); + else { + if (use_arena) + map[allocator_id] = std::make_shared(std::move(allocator)); + else + map[allocator_id] = std::move(allocator); + } + + return status; +} + +AllocatorManager& AllocatorManager::Instance() { + static AllocatorManager s_instance_; + return s_instance_; +} + +AllocatorManager::AllocatorManager() { + InitializeAllocators(); +} + +Status AllocatorManager::InitializeAllocators() { + auto cpu_alocator = std::make_unique(); + ONNXRUNTIME_RETURN_IF_ERROR(RegisterAllocator(map_, std::move(cpu_alocator), std::numeric_limits::max(), true)); +#ifdef USE_CUDA + auto cuda_alocator = std::make_unique(0); + ONNXRUNTIME_RETURN_IF_ERROR(RegisterAllocator(map_, std::move(cuda_alocator), std::numeric_limits::max(), true)); + + auto cuda_pinned_alocator = std::make_unique(); + ONNXRUNTIME_RETURN_IF_ERROR(RegisterAllocator(map_, std::move(cuda_pinned_alocator), std::numeric_limits::max(), true)); +#endif // USE_CUDA + + return Status::OK(); +} + +AllocatorManager::~AllocatorManager() { +} + +AllocatorPtr AllocatorManager::GetAllocator(const std::string& name, const int id, bool arena) { + auto allocator_id = GetAllocatorId(name, id, arena); + auto entry = map_.find(allocator_id); + ONNXRUNTIME_ENFORCE(entry != map_.end(), "Allocator not found:", allocator_id); + return entry->second; +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/TestAllocatorManager.h b/onnxruntime/test/framework/TestAllocatorManager.h new file mode 100644 index 0000000000000..44e3cb926a2ab --- /dev/null +++ b/onnxruntime/test/framework/TestAllocatorManager.h @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/framework/arena.h" +namespace onnxruntime { +namespace test { +class AllocatorManager { + public: + // the allocator manager is a just for onnx runner to allocate space for input/output tensors. + // onnxruntime session will use the allocator owned by execution provider. + static AllocatorManager& Instance(); + + /** + Destruct th AllocatorManager. Will unset Instance(). + */ + ~AllocatorManager(); + + AllocatorPtr GetAllocator(const std::string& name, const int id = 0, bool arena = true); + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(AllocatorManager); + + AllocatorManager(); + Status InitializeAllocators(); + + std::unordered_map map_; +}; +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/allocation_planner_test.cc b/onnxruntime/test/framework/allocation_planner_test.cc new file mode 100644 index 0000000000000..dbdbc5c14f0da --- /dev/null +++ b/onnxruntime/test/framework/allocation_planner_test.cc @@ -0,0 +1,431 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include +#include +#include +#include "core/framework/session_state.h" +#include "core/graph/model.h" +#include "gtest/gtest.h" +#include "core/framework/op_kernel.h" +#include "test/framework/model_builder_utils.h" +#include "core/framework/allocation_planner.h" +#include "core/providers/cpu/cpu_execution_provider.h" +using namespace ONNX_NAMESPACE; + +namespace onnxruntime { +namespace test { + +namespace modelbuilder { + +class NodeCounter { + private: + static int node_count_; + + public: + static int Next() { return ++node_count_; } +}; + +int NodeCounter::node_count_ = 0; + +struct UnaryNode { + std::vector input_args; + std::vector output_args; + onnxruntime::Node* p_node; + + UnaryNode(onnxruntime::Graph& graph, const std::string& op, + onnxruntime::NodeArg* p_input_arg, onnxruntime::NodeArg* p_output_arg) + : input_args({p_input_arg}), output_args({p_output_arg}) { + int num = NodeCounter::Next(); + p_node = graph.AddNode("node" + std::to_string(num), op, "test op", input_args, output_args); + } + + UnaryNode(onnxruntime::Graph& graph, onnxruntime::NodeArg* p_input_arg, onnxruntime::NodeArg* p_output_arg) + : UnaryNode(graph, "Transpose", p_input_arg, p_output_arg) {} +}; + +class DummyOpKernel : public OpKernel { + public: + DummyOpKernel(const OpKernelInfo& p) : OpKernel(p) {} + Status Compute(OpKernelContext* context) const { + ONNXRUNTIME_UNUSED_PARAMETER(context); + return Status::OK(); + } + Status ComputeAsync(OpKernelContext* context, DoneCallback done) const { + ONNXRUNTIME_UNUSED_PARAMETER(context); + ONNXRUNTIME_UNUSED_PARAMETER(done); + return Status::OK(); + } +}; + +} // namespace modelbuilder + +using namespace modelbuilder; + +class AllocationPlanTestUtility { + public: + static void CheckAllocationKind(const SequentialExecutionPlan& plan, std::vector& expected) { + ASSERT_EQ(plan.allocation_plan.size(), expected.size()) << "Allocation plan of wrong size"; + for (int i = 0; i < expected.size(); ++i) { + EXPECT_EQ(plan.allocation_plan[i].alloc_kind, expected[i]) << "Error in allocation kind at position " << i; + } + } + + static void CheckToBeFreed(const SequentialExecutionPlan& plan, const std::vector& expected) { + ASSERT_EQ(plan.to_be_freed.size(), expected.size()) << "Allocation plan's to_be_freed of wrong size"; + for (int i = 0; i < expected.size(); ++i) { + EXPECT_EQ(plan.to_be_freed[i], expected[i]) << "Error in to_be_freed at position " << i; + } + } + + static void CheckFreedAtEachStep(const SequentialExecutionPlan& plan, const std::vector& expected_num_freed) { + ASSERT_EQ(plan.execution_plan.size(), expected_num_freed.size()) << "Execution plan is of wrong size"; + int start = 0; + for (int i = 0; i < expected_num_freed.size(); ++i) { + if (expected_num_freed[i] > 0) { + EXPECT_EQ(plan.execution_plan[i].free_from_index, start) << "Error in free_from_index at position " << i; + EXPECT_EQ(plan.execution_plan[i].free_to_index, start + expected_num_freed[i] - 1) + << "Error in free_to_index at position " << i; + start = start + expected_num_freed[i]; + } else { + // "free_from_index > free_to_index" indicates nothing is to be freed + EXPECT_GT(plan.execution_plan[i].free_from_index, plan.execution_plan[i].free_to_index); + } + } + } + + static void BasicIntegrityCheck(const SequentialExecutionPlan& plan, size_t num_ml_values) { + // Sanity checks for plan.to_be_freed + std::unordered_set freed; + for (MLValueIndex index : plan.to_be_freed) { + // Every index should be in the valid range [0, num_ml_values-1] + EXPECT_GE(index, 0); + EXPECT_LT(index, num_ml_values); + // An index should not be freed more than once + EXPECT_EQ(freed.count(index), 0) << "MLValue " << index << " freed multiple times"; + freed.insert(index); + } + // Check the free-index information for every execution step: they should cover the + // range [0, plan.to_be_freed.size()-1] properly. + int next_free_index = 0; + int max_free_index = ((int)plan.to_be_freed.size()) - 1; + for (const SequentialExecutionPlan::NodeExecutionPlan& step : plan.execution_plan) { + if (step.free_from_index <= step.free_to_index) { + EXPECT_EQ(step.free_from_index, next_free_index); + EXPECT_LE(step.free_to_index, max_free_index); + next_free_index = step.free_to_index + 1; + } // else nothing needs to be freed in this step + } + } +}; + +typedef std::unordered_map ShapeMap; + +class SequentialPlannerTestContext : public ISequentialPlannerContext { + public: + SequentialPlannerTestContext(ShapeMap* shape_map) : shape_map_(shape_map) {} + + virtual TensorShapeProto* GetShape(const onnxruntime::NodeArg& arg) const override { + auto iter = shape_map_->find(&arg); + return (shape_map_->end() != iter) ? iter->second : nullptr; + } + + private: + ShapeMap* shape_map_; +}; + +class PlannerTest : public ::testing::Test { + private: + void index(const std::string& name, int& out) { + ASSERT_TRUE(state_.GetMLValueNameIdxMap().GetIdx(name, out).IsOK()); + } + + onnxruntime::Model model_; + onnxruntime::Graph& graph_; + + // some standard components used to build test-cases: + Type float_type_; + + std::unique_ptr<::onnxruntime::KernelDef> std_kernel_; // a unary kernel with no-aliasing and no-in-place + std::unique_ptr<::onnxruntime::KernelDef> in_place_kernel_; // a unary kernel with in-place + + std::unordered_map name_to_arg_; + std::vector> nodes_; + std::vector> op_kernel_infos_; + std::vector> kernel_bindings_; + ExecutionProviders execution_providers_; + SessionState state_; + ShapeMap shape_map_; + std::unique_ptr plan_; + + public: + PlannerTest() : model_("test"), graph_{model_.MainGraph()}, state_{execution_providers_} { + std_kernel_ = KernelDefBuilder().SetName("Transpose").Build(); + in_place_kernel_ = KernelDefBuilder().SetName("Clip").MayInplace(0, 0).Build(); + CPUExecutionProviderInfo epi; + auto execution_provider = std::make_unique(epi); + execution_providers_.Add("CPUExecutionProvider", std::move(execution_provider)); + } + + ~PlannerTest() = default; + + onnxruntime::NodeArg* Arg(const std::string& name) { + auto iter = name_to_arg_.find(name); + if (name_to_arg_.end() != iter) return iter->second; + return (name_to_arg_[name] = &graph_.GetOrCreateNodeArg(name, &float_type_.value)); + } + + onnxruntime::Node* AddNode(::onnxruntime::KernelDef& kernel_def, std::string& input, std::string& output) { + auto node = std::make_unique(graph_, kernel_def.OpName(), Arg(input), Arg(output)); + auto* p_node = node->p_node; + p_node->SetExecutionProviderType(onnxruntime::kCpuExecutionProvider); + nodes_.push_back(std::move(node)); + kernel_bindings_.emplace_back(p_node, kernel_def); + return p_node; + } + + onnxruntime::Node* AddNormalNode(std::string& input, std::string& output) { + return AddNode(*std_kernel_, input, output); + } + + onnxruntime::Node* AddInplaceNode(std::string& input, std::string& output) { + return AddNode(*in_place_kernel_, input, output); + } + + void BindKernel(onnxruntime::Node* p_node, ::onnxruntime::KernelDef& kernel_def) { + auto info = std::make_unique(*p_node, kernel_def, *execution_providers_.Get(*p_node), state_); + auto dummy = std::make_unique(*info); + op_kernel_infos_.push_back(std::move(info)); + state_.AddKernel(p_node->Index(), std::move(dummy)); + } + + void SetShape(std::string& name, TensorShapeProto* shape) { + shape_map_[Arg(name)] = shape; + } + + void SetShape(std::initializer_list> shapes) { + for (auto& pair : shapes) { + SetShape(pair.first, pair.second); + } + } + + void CreatePlan() { + EXPECT_EQ(graph_.Resolve(), Status::OK()); + state_.SetGraphViewer(std::make_unique(graph_)); + + MLValueNameIdxMap& mlvalue_name_idx_map{state_.GetMLValueNameIdxMap()}; + + int count = 0; + for (auto& pair : name_to_arg_) { + EXPECT_EQ(mlvalue_name_idx_map.Add(pair.first), count++); + } + + for (auto& binding : kernel_bindings_) { + BindKernel(binding.first, binding.second); + } + + auto cpu_execution_provider = std::make_unique(CPUExecutionProviderInfo()); + KernelRegistryManager kernel_registry_manager; + kernel_registry_manager.RegisterKernelRegistry(cpu_execution_provider->GetKernelRegistry(), KernelRegistryPriority::LowPriority); + + ExecutionProviders execution_providers; + execution_providers.Add(onnxruntime::kCpuExecutionProvider, std::move(cpu_execution_provider)); + + SequentialPlannerTestContext test_context(&shape_map_); + auto status = SequentialPlanner::CreatePlan( + graph_, execution_providers, kernel_registry_manager, mlvalue_name_idx_map, test_context, plan_); + + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + AllocationPlanTestUtility::BasicIntegrityCheck(*plan_, name_to_arg_.size()); + } + + void CheckAllocKind(const std::string& name, AllocKind kind) { + int id; + index(name, id); + EXPECT_EQ(plan_->allocation_plan[id].alloc_kind, kind) << "Error in allocation kind for " << name; + } + + void CheckFreed(int step_number, std::initializer_list freed_items) { + // create set and check equality + std::unordered_set expected; + for (auto& name : freed_items) { + int id; + index(name, id); + expected.insert(id); + } + std::unordered_set plan_result; + auto& step_plan = plan_->execution_plan[step_number]; + for (int i = step_plan.free_from_index; i <= step_plan.free_to_index; ++i) + plan_result.insert(plan_->to_be_freed[i]); + EXPECT_EQ(plan_result, expected) << "Freed items incorrect for step " << step_number; + } + + protected: + Graph& GetGraph() { return graph_; } + const SequentialExecutionPlan& GetPlan() const { return *plan_; } + const SessionState& GetState() const { return state_; } +}; + +TEST_F(PlannerTest, ChainTest) { + // tensor variables: + std::string W("W"), X("X"), B("B"), Y("Y"), Z("Z"); + + // graph structure: + + ONNX_NAMESPACE::TensorProto tensor; + tensor.add_dims(1); + tensor.add_float_data(1.0f); + tensor.set_data_type(TensorProto_DataType_FLOAT); + tensor.set_name("W"); + GetGraph().AddInitializedTensor(tensor); + + AddNormalNode(W, X); + AddNormalNode(X, B); + AddNormalNode(B, Y); + AddNormalNode(Y, Z); + + // simulate shape-inference results: + Shape shape1{50, 100}; + auto shape = &shape1.value; + SetShape({{X, shape}, {B, shape}, {Y, shape}, {Z, shape}}); + + CreatePlan(); + + // Expected plan: + // W: kAllocateStatically; X: kAllocate; B: kAllocate; Y: kReuse (X); post-node3: free(B); X is returned output + CheckAllocKind(W, AllocKind::kAllocateStatically); + CheckAllocKind(X, AllocKind::kAllocate); + CheckAllocKind(B, AllocKind::kAllocate); + CheckAllocKind(Y, AllocKind::kReuse); + CheckAllocKind(Z, AllocKind::kAllocateOutput); + + CheckFreed(0, {}); + CheckFreed(1, {}); + CheckFreed(2, {"B"}); + CheckFreed(3, {"X"}); +} + +/* InputOutputTest: Test that: +(a) All inputs are classified as kPreExisting, +(b) All outputs are classified as kAllocate (in this example), +(c) Neither input nor outputs are freed. +*/ +TEST_F(PlannerTest, InputOutputTest) { + // tensor variables: + std::string X1("X1"), X2("X2"), Y1("Y1"), Y2("Y2"); + + // graph structure: + AddNormalNode(X1, Y1); + AddNormalNode(X2, Y2); + + // simulate no shape-inference: + + CreatePlan(); + + // X1: kPreExisting, X2: kPreExisting, Y1: kAllocate, Y2: kAllocate + CheckAllocKind(X1, AllocKind::kPreExisting); + CheckAllocKind(X2, AllocKind::kPreExisting); + CheckAllocKind(Y1, AllocKind::kAllocateOutput); + CheckAllocKind(Y2, AllocKind::kAllocateOutput); + + // Nothing should be freed (since they are either inputs or outputs) + CheckFreed(0, {}); + CheckFreed(1, {}); +} + +// InPlaceTest: Check that we reuse when Inplace allows us to. + +TEST_F(PlannerTest, InPlaceTest) { + // tensor variables: + std::string X1("X1"), X2("X2"), X3("X3"), X4("X4"); + + // graph structure: + AddNormalNode(X1, X2); // no in-place operator; X1: input; X2: temporary + AddInplaceNode(X2, X3); // may-in-place operator; X3: temporary + AddNormalNode(X3, X4); // no in-place operator; X4: output + + // simulate shape-inference results: + Shape shape1{"M", "N"}; + auto shape = &shape1.value; + SetShape({{X1, shape}, {X2, shape}, {X3, shape}, {X4, shape}}); + + CreatePlan(); + + // check allocation kind: + CheckAllocKind(X1, AllocKind::kPreExisting); + CheckAllocKind(X2, AllocKind::kAllocate); + CheckAllocKind(X3, AllocKind::kReuse); + CheckAllocKind(X4, AllocKind::kAllocateOutput); + + // check each ml-value is freed at appropriate step + CheckFreed(0, {}); + CheckFreed(1, {}); + CheckFreed(2, {X2}); +} + +// InPlaceSizeMismatchTest: Check that Inplace reuse is not allowed when sizes don't match. +// Also tests reuse of disjoint lifetime tensors. +TEST_F(PlannerTest, InPlaceSizeMismatchTest) { + // tensor variables: + std::string X1("X1"), X2("X2"), X3("X3"), X4("X4"), X5("X5"); + + // graph structure: + AddNormalNode(X1, X2); // no in-place operator; X1: input; X2: temporary + AddInplaceNode(X2, X3); // may-in-place operator; X3: temporary + AddNormalNode(X3, X4); // no in-place operator; X4: temporary + AddInplaceNode(X4, X5); // may-in-place operator; X5 output + + // simulate shape-inference results: + Shape shape1w{"M", "N"}; + auto shape1 = &shape1w.value; + Shape shape2w{"M", "K"}; + auto shape2 = &shape2w.value; + SetShape({{X1, shape1}, {X2, shape1}, {X3, shape2}, {X4, shape1}, {X5, shape1}}); + + CreatePlan(); + + // check allocation kind: + CheckAllocKind(X1, AllocKind::kPreExisting); + CheckAllocKind(X2, AllocKind::kAllocate); + CheckAllocKind(X3, AllocKind::kAllocate); + CheckAllocKind(X4, AllocKind::kReuse); + CheckAllocKind(X5, AllocKind::kAllocateOutput); + + // check each ml-value is freed at appropriate step + CheckFreed(0, {}); + CheckFreed(1, {}); + CheckFreed(2, {X3}); + CheckFreed(3, {X2}); +} + +// Test operator<< to output details of an allocation & execution plan. +TEST_F(PlannerTest, PlanOutputTest) { + // tensor variables: + std::string X1("X1"), X2("X2"), X3("X3"), X4("X4"); + + // graph structure: + AddNormalNode(X1, X2); // no in-place operator; X1: input; X2: temporary + AddInplaceNode(X2, X3); // may-in-place operator; X3: temporary + AddNormalNode(X3, X4); // no in-place operator; X4: output + + // simulate shape-inference results: + Shape shape1{"M", "N"}; + auto shape = &shape1.value; + SetShape({{X1, shape}, {X2, shape}, {X3, shape}, {X4, shape}}); + + CreatePlan(); + + try { + std::ostringstream output; + output << std::make_pair(&GetPlan(), &GetState()); + auto output_size = output.str().size(); + // Currently, we don't check details of the output, as it may change over time. + EXPECT_GT(output_size, 0); + } catch (const std::exception& ex) { + EXPECT_TRUE(false) << "Exception in producing output: " << ex.what(); + } +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/allocator_test.cc b/onnxruntime/test/framework/allocator_test.cc new file mode 100644 index 0000000000000..587f753b1b5f7 --- /dev/null +++ b/onnxruntime/test/framework/allocator_test.cc @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/allocatormgr.h" +#include "core/framework/allocator.h" +#include "test_utils.h" +#include "gtest/gtest.h" + +namespace onnxruntime { +namespace test { +TEST(AllocatorTest, CPUAllocatorTest) { + auto cpu_arena = TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault); + + ASSERT_STREQ(cpu_arena->Info().name, CPU); + EXPECT_EQ(cpu_arena->Info().id, 0); + EXPECT_EQ(cpu_arena->Info().type, ONNXRuntimeAllocatorType::ONNXRuntimeArenaAllocator); + + size_t size = 1024; + auto bytes = cpu_arena->Alloc(size); + EXPECT_TRUE(bytes); + //test the bytes are ok for read/write + memset(bytes, -1, 1024); + + EXPECT_EQ(*((int*)bytes), -1); + cpu_arena->Free(bytes); + //todo: test the used / max api. +} + +// helper class to validate values in Alloc and Free calls made via IAllocator::MakeUniquePtr +class TestAllocator : public IAllocator { + public: + TestAllocator(size_t expected_size) : expected_size_{expected_size} {} + + void* Alloc(size_t size) override { + EXPECT_EQ(size, expected_size_); + // return a pointer to the expected size in the result. + // this isn't valid as a real allocator would return memory of the correct size, + // however the unit test won't be using the memory and via this mechanism we can validate + // that the Free argument matches. + size_t* result = new size_t(size); + return result; + } + + void Free(void* p) override { + // the IAllocatorUniquePtr should be calling this with the contents of what was returned from the Alloc + size_t* p_sizet = (size_t*)p; + EXPECT_EQ(*p_sizet, expected_size_); + delete p_sizet; + } + + virtual const ONNXRuntimeAllocatorInfo& Info() const override { + static ONNXRuntimeAllocatorInfo info("test", ONNXRuntimeDeviceAllocator, 0); + return info; + } + + private: + size_t expected_size_; +}; + +// test that IAllocator::MakeUniquePtr allocates buffers of the expected size +TEST(AllocatorTest, MakeUniquePtrTest) { + // test float creates buffer of size * sizeof(float) + size_t num_floats = 16; + + // create allocator that will check the call to Alloc matches the expected size + auto allocator = std::make_shared(num_floats * sizeof(float)); + IAllocatorUniquePtr float_ptr = IAllocator::MakeUniquePtr(allocator, num_floats); + float_ptr = nullptr; // reset so TestAllocator.Free is called here + + // void should create buffer of size 16 for void* + // Create new TestAllocator to validate that. + allocator = std::make_shared(16); + auto void_ptr = IAllocator::MakeUniquePtr(allocator, 16); + void_ptr = nullptr; +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/bfc_arena_test.cc b/onnxruntime/test/framework/bfc_arena_test.cc new file mode 100644 index 0000000000000..62e87f5b56cd5 --- /dev/null +++ b/onnxruntime/test/framework/bfc_arena_test.cc @@ -0,0 +1,238 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/bfc_arena.h" +#include "gtest/gtest.h" +#include + +namespace onnxruntime { +namespace test { +static void CheckStats(BFCArena* a, int64_t num_allocs, int64_t bytes_in_use, + int64_t max_bytes_in_use, int64_t max_alloc_size) { + AllocatorStats stats; + a->GetStats(&stats); + EXPECT_EQ(stats.bytes_in_use, bytes_in_use); + EXPECT_EQ(stats.max_bytes_in_use, max_bytes_in_use); + EXPECT_EQ(stats.num_allocs, num_allocs); + EXPECT_EQ(stats.max_alloc_size, max_alloc_size); +} + +TEST(BFCArenaTest, NoDups) { + BFCArena a(std::unique_ptr(new CPUAllocator()), 1 << 30); + CheckStats(&a, 0, 0, 0, 0); + + // Allocate a lot of raw pointers + std::vector ptrs; + for (int s = 1; s < 1024; s++) { + void* raw = a.Alloc(s); + ptrs.push_back(raw); + } + CheckStats(&a, 1023, 654336, 654336, 1024); + + std::sort(ptrs.begin(), ptrs.end()); + + // Make sure none of them are equal, and that none of them overlap. + for (size_t i = 1; i < ptrs.size(); i++) { + ASSERT_NE(ptrs[i], ptrs[i - 1]); // No dups + size_t req_size = a.RequestedSize(ptrs[i - 1]); + ASSERT_GT(req_size, 0); + ASSERT_GE(static_cast(static_cast(ptrs[i]) - static_cast(ptrs[i - 1])), + req_size); + } + + for (size_t i = 0; i < ptrs.size(); i++) { + a.Free(ptrs[i]); + } + CheckStats(&a, 1023, 0, 654336, 1024); +} + +TEST(BFCArenaTest, AllocationsAndDeallocations) { + BFCArena a(std::unique_ptr(new CPUAllocator()), 1 << 30); + // Allocate 256 raw pointers of sizes between 100 bytes and about a meg + std::srand(static_cast(std::time(nullptr))); + + std::vector initial_ptrs; + for (int s = 1; s < 256; s++) { + size_t size = std::min( + std::max(rand() % 1048576, 100), 1048576); + void* raw = a.Alloc(size); + + initial_ptrs.push_back(raw); + } + + // Deallocate half of the memory, and keep track of the others. + std::vector existing_ptrs; + for (size_t i = 0; i < initial_ptrs.size(); i++) { + if (i % 2 == 1) { + a.Free(initial_ptrs[i]); + } else { + existing_ptrs.push_back(initial_ptrs[i]); + } + } + + // Ensure out of memory errors work and do not prevent future allocations from + // working. + void* out_of_memory_ptr = a.Alloc((1 << 30) + 1); + EXPECT_EQ(out_of_memory_ptr, nullptr); + + // Allocate a lot of raw pointers + for (int s = 1; s < 256; s++) { + size_t size = std::min( + std::max(rand() % 1048576, 100), 1048576); + void* raw = a.Alloc(size); + existing_ptrs.push_back(raw); + } + + std::sort(existing_ptrs.begin(), existing_ptrs.end()); + // Make sure none of them are equal + for (size_t i = 1; i < existing_ptrs.size(); i++) { + EXPECT_NE(existing_ptrs[i], existing_ptrs[i - 1]); // No dups + + size_t req_size = a.RequestedSize(existing_ptrs[i - 1]); + ASSERT_GT(req_size, 0); + + // Check that they don't overlap. + ASSERT_GE(static_cast(static_cast(existing_ptrs[i]) - + static_cast(existing_ptrs[i - 1])), + req_size); + } + + for (size_t i = 0; i < existing_ptrs.size(); i++) { + a.Free(existing_ptrs[i]); + } +} + +TEST(BFCArenaTest, ExerciseCoalescing) { + BFCArena a(std::unique_ptr(new CPUAllocator()), 1 << 30); + CheckStats(&a, 0, 0, 0, 0); + + void* first_ptr = a.Alloc(4096); + a.Free(first_ptr); + CheckStats(&a, 1, 0, 4096, 4096); + for (int i = 0; i < 1024; ++i) { + // Allocate several buffers of different sizes, and then clean them + // all up. We should be able to repeat this endlessly without + // causing fragmentation and growth. + void* t1 = a.Alloc(4096); + + void* t2 = a.Alloc(1048576 * sizeof(int64_t)); + void* t3 = a.Alloc(2048 * sizeof(double)); + void* t4 = a.Alloc(1048576 * sizeof(int64_t) * sizeof(float)); + + a.Free(t1); + a.Free(t2); + a.Free(t3); + a.Free(t4); + } + CheckStats(&a, 4097, 0, + 1024 * sizeof(float) + 1048576 * sizeof(int64_t) + + 2048 * sizeof(double) + 1048576 * sizeof(int64_t) * sizeof(float), + 1048576 * sizeof(int64_t) * sizeof(float)); + + // At the end, we should have coalesced all memory into one region + // starting at the beginning, so validate that allocating a pointer + // starts from this region. + void* first_ptr_after = a.Alloc(1024 * sizeof(float)); + EXPECT_EQ(first_ptr, first_ptr_after); + a.Free(first_ptr_after); +} + +TEST(BFCArenaTest, AllocateZeroBufSize) { + BFCArena a(std::unique_ptr(new CPUAllocator()), 1 << 30); + void* ptr = a.Alloc(0); + EXPECT_EQ(nullptr, ptr); +} + +TEST(BFCArenaTest, AllocatedVsRequested) { + BFCArena a(std::unique_ptr(new CPUAllocator()), 1 << 30); + void* t1 = a.Alloc(4); + EXPECT_EQ(4, a.RequestedSize(t1)); + EXPECT_EQ(256, a.AllocatedSize(t1)); + a.Free(t1); +} + +TEST(BFCArenaTest, TestCustomMemoryLimit) { + // Configure a 1MiB byte limit + BFCArena a(std::unique_ptr(new CPUAllocator()), 1 << 20); + + void* first_ptr = a.Alloc(sizeof(float) * (1 << 6)); + void* second_ptr = a.Alloc(sizeof(float) * (1 << 20)); + + EXPECT_NE(nullptr, first_ptr); + EXPECT_EQ(nullptr, second_ptr); + a.Free(first_ptr); +} + +TEST(BFCArenaTest, AllocationsAndDeallocationsWithGrowth) { + // Max of 2GiB, but starts out small. + BFCArena a(std::unique_ptr(new CPUAllocator()), 1LL << 31); + + // Allocate 10 raw pointers of sizes between 100 bytes and about + // 64 megs. + std::srand(static_cast(std::time(nullptr))); + + const int32_t max_mem = 1 << 27; + + std::vector initial_ptrs; + for (int s = 1; s < 10; s++) { + size_t size = std::min( + std::max(rand() % max_mem, 100), max_mem); + void* raw = a.Alloc(size); + + initial_ptrs.push_back(raw); + } + + // Deallocate half of the memory, and keep track of the others. + std::vector existing_ptrs; + for (size_t i = 0; i < initial_ptrs.size(); i++) { + if (i % 2 == 1) { + a.Free(initial_ptrs[i]); + } else { + existing_ptrs.push_back(initial_ptrs[i]); + } + } + + const int32_t max_mem_2 = 1 << 26; + // Allocate a lot of raw pointers between 100 bytes and 64 megs. + for (int s = 1; s < 10; s++) { + size_t size = std::min( + std::max(rand() % max_mem_2, 100), max_mem_2); + void* raw = a.Alloc(size); + existing_ptrs.push_back(raw); + } + + std::sort(existing_ptrs.begin(), existing_ptrs.end()); + // Make sure none of them are equal + for (size_t i = 1; i < existing_ptrs.size(); i++) { + EXPECT_NE(existing_ptrs[i], existing_ptrs[i - 1]); // No dups + + size_t req_size = a.RequestedSize(existing_ptrs[i - 1]); + ASSERT_GT(req_size, 0); + + // Check that they don't overlap. + ASSERT_GE(static_cast( + static_cast(existing_ptrs[i]) - + static_cast(existing_ptrs[i - 1])), + req_size); + } + + for (size_t i = 0; i < existing_ptrs.size(); i++) { + a.Free(existing_ptrs[i]); + } +} + +TEST(BFCArenaTest, TestReserve) { + // Configure a 1MiB byte limit + BFCArena a(std::unique_ptr(new CPUAllocator()), 1 << 30); + + void* first_ptr = a.Alloc(sizeof(float) * (1 << 6)); + void* second_ptr = a.Reserve(sizeof(float) * (1 << 20)); + a.Free(first_ptr); + a.Free(second_ptr); + + AllocatorStats stats; + a.GetStats(&stats); + EXPECT_EQ(stats.total_allocated_bytes, 1048576); +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/cuda/allocator_cuda_test.cc b/onnxruntime/test/framework/cuda/allocator_cuda_test.cc new file mode 100644 index 0000000000000..dc36bab3e81f8 --- /dev/null +++ b/onnxruntime/test/framework/cuda/allocator_cuda_test.cc @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/allocatormgr.h" +#include "test/framework/test_utils.h" +#include "gtest/gtest.h" +#include "cuda_runtime.h" +#include "core/providers/cuda/cuda_allocator.h" + +namespace onnxruntime { +namespace test { +TEST(AllocatorTest, CUDAAllocatorTest) { + int cuda_device_id = 0; + DeviceAllocatorRegistrationInfo default_allocator_info({ONNXRuntimeMemTypeDefault, + [](int id) { return std::make_unique(id); }, std::numeric_limits::max()}); + + auto cuda_arena = CreateAllocator(default_allocator_info, cuda_device_id); + + size_t size = 1024; + + EXPECT_STREQ(cuda_arena->Info().name, CUDA); + EXPECT_EQ(cuda_arena->Info().id, cuda_device_id); + EXPECT_EQ(cuda_arena->Info().mem_type, ONNXRuntimeMemTypeDefault); + EXPECT_EQ(cuda_arena->Info().type, ONNXRuntimeArenaAllocator); + + //test cuda allocation + auto cuda_addr = cuda_arena->Alloc(size); + EXPECT_TRUE(cuda_addr); + + DeviceAllocatorRegistrationInfo pinned_allocator_info({ONNXRuntimeMemTypeCPUOutput, + [](int) { return std::make_unique(); }, std::numeric_limits::max()}); + + auto pinned_allocator = CreateAllocator(pinned_allocator_info); + + EXPECT_STREQ(pinned_allocator->Info().name, CUDA_PINNED); + EXPECT_EQ(pinned_allocator->Info().id, 0); + EXPECT_EQ(pinned_allocator->Info().mem_type, ONNXRuntimeMemTypeCPUOutput); + EXPECT_EQ(pinned_allocator->Info().type, ONNXRuntimeArenaAllocator); + + //test pinned allocation + auto pinned_addr = pinned_allocator->Alloc(size); + EXPECT_TRUE(pinned_addr); + + const auto& cpu_arena = TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault); + EXPECT_STREQ(cpu_arena->Info().name, CPU); + EXPECT_EQ(cpu_arena->Info().id, 0); + EXPECT_EQ(cpu_arena->Info().mem_type, ONNXRuntimeMemTypeDefault); + EXPECT_EQ(cpu_arena->Info().type, ONNXRuntimeArenaAllocator); + + auto cpu_addr_a = cpu_arena->Alloc(size); + EXPECT_TRUE(cpu_addr_a); + auto cpu_addr_b = cpu_arena->Alloc(size); + EXPECT_TRUE(cpu_addr_b); + memset(cpu_addr_a, -1, 1024); + + //test host-device memory copy + cudaMemcpy(cuda_addr, cpu_addr_a, size, cudaMemcpyHostToDevice); + cudaMemcpy(cpu_addr_b, cuda_addr, size, cudaMemcpyDeviceToHost); + EXPECT_EQ(*((int*)cpu_addr_b), -1); + + cudaMemcpyAsync(pinned_addr, cuda_addr, size, cudaMemcpyDeviceToHost); + cudaDeviceSynchronize(); + EXPECT_EQ(*((int*)pinned_addr), -1); + + cpu_arena->Free(cpu_addr_a); + cpu_arena->Free(cpu_addr_b); + cuda_arena->Free(cuda_addr); + pinned_allocator->Free(pinned_addr); +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/cuda/fence_cuda_test.cc b/onnxruntime/test/framework/cuda/fence_cuda_test.cc new file mode 100644 index 0000000000000..9b39fca92cd76 --- /dev/null +++ b/onnxruntime/test/framework/cuda/fence_cuda_test.cc @@ -0,0 +1,256 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/inference_session.h" + +#include +#include +#include +#include + +#include "core/common/status.h" +#include "core/common/logging/logging.h" +#include "core/framework/execution_provider.h" +#include "core/framework/op_kernel.h" +#include "core/framework/session_state.h" +#include "core/graph/graph.h" +#include "core/graph/model.h" +#include "core/graph/op.h" +#include "core/providers/cuda/cuda_execution_provider.h" +#include "core/providers/cpu/math/element_wise_ops.h" +#include "core/framework/tensorprotoutils.h" +#include "test/capturing_sink.h" +#include "test/test_environment.h" +#include "test/framework/test_utils.h" +#include "gtest/gtest.h" + +using namespace std; +using namespace ONNX_NAMESPACE; +using namespace onnxruntime::logging; + +namespace onnxruntime { +namespace test { + +typedef std::vector ArgMap; + +size_t CountCopyNodes(const onnxruntime::Graph& graph) { + size_t num_copy_nodes = 0; + for (auto& p : graph.Nodes()) + num_copy_nodes += (p.OpType().substr(0, 6) == "Memcpy"); + return num_copy_nodes; +} + +static common::Status LoadInferenceSessionFromModel(InferenceSession& session, onnxruntime::Model& model) { + std::stringstream s1; + model.ToProto().SerializeToOstream(&s1); + return session.Load(s1); +} + +#define CREATE_INITIALIZER_FUNC(T, PROTO_DATATYPE, PROTO_ADD_DATA) \ + onnxruntime::NodeArg& CreateInitializer(onnxruntime::Graph& graph, const std::string& name, \ + const std::vector& shape, const std::vector& value) { \ + ONNX_NAMESPACE::TensorProto tensor_proto; \ + for (auto dim : shape) tensor_proto.add_dims(dim); \ + tensor_proto.set_data_type(PROTO_DATATYPE); \ + for (auto v : value) tensor_proto.PROTO_ADD_DATA(v); \ + tensor_proto.set_name(name); \ + graph.AddInitializedTensor(tensor_proto); \ + TypeProto type_proto; \ + type_proto.mutable_tensor_type()->set_elem_type(PROTO_DATATYPE); \ + return graph.GetOrCreateNodeArg(name, &type_proto); \ + } + +CREATE_INITIALIZER_FUNC(float, TensorProto_DataType_FLOAT, add_float_data) +CREATE_INITIALIZER_FUNC(int64_t, TensorProto_DataType_INT64, add_int64_data) +// TO DO: Figure out a way to enable it again +TEST(CUDAFenceTests, DISABLED_PartOnCPU) { + std::unique_ptr model = std::make_unique("test"); + onnxruntime::Graph& graph = model->MainGraph(); + TypeProto tensor_float; + tensor_float.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + onnxruntime::NodeArg x1_def("X1", &tensor_float); + onnxruntime::NodeArg y_def("Y", &tensor_float); + onnxruntime::NodeArg z_def("Z", &tensor_float); + onnxruntime::NodeArg out_def("Out", &tensor_float); + + auto& w_def = CreateInitializer(graph, "W", std::vector({2, 2}), std::vector({-1, 2, 3, -4})); + + auto p_node = graph.AddNode("node1", "MatMul", "MatMul operator", ArgMap{&w_def, &x1_def}, ArgMap{&y_def}); + p_node->SetExecutionProviderType(onnxruntime::kCudaExecutionProvider); + p_node = graph.AddNode("node2", "Add", "Add operator", ArgMap{&y_def, &w_def}, ArgMap{&z_def}); + p_node->SetExecutionProviderType(onnxruntime::kCpuExecutionProvider); + p_node = graph.AddNode("node3", "Add", "Add operator", ArgMap{&y_def, &z_def}, ArgMap{&out_def}); + p_node->SetExecutionProviderType(onnxruntime::kCpuExecutionProvider); + + // add and then delete a node to test node iteration against nullptr + p_node = graph.AddNode("node_to_delete", "Add", "Add operator", ArgMap{&y_def, &z_def}, ArgMap{&out_def}); + graph.RemoveNode(p_node->Index()); + + ASSERT_TRUE(graph.Resolve().IsOK()); + + auto cpu_allocator = TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault); + auto element_type = DataTypeImpl::GetType(); + TensorShape shape({2, 2}); + float data[4] = {-1, 2, 3, -4}; + void* buffer = cpu_allocator->Alloc(element_type->Size() * shape.Size()); + memcpy(buffer, data, sizeof(data)); + + //create fake ml value with owned buffer. + std::unique_ptr p_tensor = std::make_unique( + element_type, + shape, + buffer, + cpu_allocator->Info(), + cpu_allocator); + MLValue value; + value.Init(p_tensor.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); + + SessionOptions so; + InferenceSession session(so); + LoadInferenceSessionFromModel(session, *model); + CUDAExecutionProviderInfo xp_info; + session.RegisterExecutionProvider(std::make_unique(xp_info)); + ASSERT_TRUE(session.Initialize().IsOK()); + ASSERT_TRUE(1 == CountCopyNodes(graph)); + + vector outputs; + session.Run(std::unordered_map{{"X1", value}}, + std::vector{"Out"}, + &outputs); + ASSERT_TRUE(1 == outputs.size()); + const Tensor& output = outputs[0].Get(); + EXPECT_EQ(output.Shape(), shape); + EXPECT_EQ(output.DataType(), DataTypeImpl::GetType()); + + float expected_output[4] = {13.0f, -18.0f, -27.0f, 40.0f}; + for (int i = 0; i < 4; ++i) { + EXPECT_EQ(output.template Data()[i], expected_output[i]); + } +} + +TEST(CUDAFenceTests, TileWithInitializer) { + std::unique_ptr model = std::make_unique("test"); + onnxruntime::Graph& graph = model->MainGraph(); + TypeProto tensor_float; + tensor_float.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + onnxruntime::NodeArg x1_def("X1", &tensor_float); + onnxruntime::NodeArg y_def("Y", &tensor_float); + auto& tile_repeat_def = CreateInitializer(graph, "tile_repeat", std::vector({2}), std::vector({1, 2})); + + auto p_node = graph.AddNode("node1", "Tile", "Tile operator", ArgMap{&x1_def, &tile_repeat_def}, ArgMap{&y_def}); + p_node->SetExecutionProviderType(onnxruntime::kCudaExecutionProvider); + ASSERT_TRUE(graph.Resolve().IsOK()); + + ASSERT_TRUE(0 == CountCopyNodes(graph)); + + auto cpu_allocator = TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault); + auto element_type = DataTypeImpl::GetType(); + TensorShape shape({2, 2}); + float data[4] = {-1, 2, 3, -4}; + void* buffer = cpu_allocator->Alloc(element_type->Size() * shape.Size()); + memcpy(buffer, data, sizeof(data)); + + //create fake ml value with owned buffer. + std::unique_ptr p_tensor = std::make_unique( + element_type, + shape, + buffer, + cpu_allocator->Info(), + cpu_allocator); + MLValue value; + value.Init(p_tensor.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); + + SessionOptions so; + InferenceSession session(so); + LoadInferenceSessionFromModel(session, *model); + CUDAExecutionProviderInfo xp_info; + session.RegisterExecutionProvider(std::make_unique(xp_info)); + ASSERT_TRUE(session.Initialize().IsOK()); + + vector outputs; + session.Run(std::unordered_map{{"X1", value}}, + std::vector{"Y"}, + &outputs); + ASSERT_TRUE(1 == outputs.size()); + const Tensor& output = outputs[0].Get(); + EXPECT_EQ(output.Shape(), TensorShape({2, 4})); + EXPECT_EQ(output.DataType(), DataTypeImpl::GetType()); + + float expected_output[8] = {-1, 2, -1, 2, 3, -4, 3, -4}; + for (int i = 0; i < 8; ++i) { + EXPECT_EQ(output.template Data()[i], expected_output[i]); + } +} + +TEST(CUDAFenceTests, TileWithComputedInput) { + std::unordered_map domain_to_version; + domain_to_version[onnxruntime::kOnnxDomain] = 7; + std::unique_ptr model = std::make_unique("test", true, ModelMetaData(), IOnnxRuntimeOpSchemaRegistryList(), domain_to_version); + onnxruntime::Graph& graph = model->MainGraph(); + TypeProto tensor_float; + tensor_float.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + TypeProto tensor_int64; + tensor_int64.mutable_tensor_type()->set_elem_type(TensorProto_DataType_INT64); + onnxruntime::NodeArg x1_def("X1", &tensor_float); + onnxruntime::NodeArg y_def("Y", &tensor_float); + onnxruntime::NodeArg s_def("S", &tensor_int64); + onnxruntime::NodeArg out_def("Out", &tensor_float); + auto& w_def = CreateInitializer(graph, "W", std::vector({2, 2}), std::vector({-1, 2, 3, -4})); + + auto p_node = graph.AddNode("node1", "MatMul", "MatMul operator", ArgMap{&x1_def, &w_def}, ArgMap{&y_def}); + p_node->SetExecutionProviderType(onnxruntime::kCudaExecutionProvider); + p_node = graph.AddNode("node2", "Shape", "Shape operator", ArgMap{&y_def}, ArgMap{&s_def}); + p_node->SetExecutionProviderType(onnxruntime::kCpuExecutionProvider); + p_node = graph.AddNode("node3", "Tile", "Tile operator", ArgMap{&y_def, &s_def}, ArgMap{&out_def}); + p_node->SetExecutionProviderType(onnxruntime::kCudaExecutionProvider); + ASSERT_TRUE(graph.Resolve().IsOK()); + + ASSERT_TRUE(0 == CountCopyNodes(graph)); + + auto cpu_allocator = TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault); + auto element_type = DataTypeImpl::GetType(); + TensorShape shape({2, 2}); + float data[4] = {-1, 2, 3, -4}; + void* buffer = cpu_allocator->Alloc(element_type->Size() * shape.Size()); + memcpy(buffer, data, sizeof(data)); + + //create fake ml value with owned buffer. + std::unique_ptr p_tensor = std::make_unique( + element_type, + shape, + buffer, + cpu_allocator->Info(), + cpu_allocator); + MLValue value; + value.Init(p_tensor.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); + + SessionOptions so; + InferenceSession session(so); + LoadInferenceSessionFromModel(session, *model); + CUDAExecutionProviderInfo xp_info; + session.RegisterExecutionProvider(std::make_unique(xp_info)); + ASSERT_TRUE(session.Initialize().IsOK()); + + vector outputs; + session.Run(std::unordered_map{{"X1", value}}, + std::vector{"Out"}, + &outputs); + ASSERT_TRUE(1 == outputs.size()); + const Tensor& output = outputs[0].Get(); + EXPECT_EQ(output.Shape(), TensorShape({4, 4})); + EXPECT_EQ(output.DataType(), DataTypeImpl::GetType()); + + float expected_output[16] = {7, -10, 7, -10, -15, 22, -15, 22, 7, -10, 7, -10, -15, 22, -15, 22}; + for (int i = 0; i < 16; ++i) { + EXPECT_EQ(output.template Data()[i], expected_output[i]); + } +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/data_types_test.cc b/onnxruntime/test/framework/data_types_test.cc new file mode 100644 index 0000000000000..d6d0f8f98d791 --- /dev/null +++ b/onnxruntime/test/framework/data_types_test.cc @@ -0,0 +1,547 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include + +#include "core/framework/data_types.h" +#include "core/graph/onnx_protobuf.h" +#include "gtest/gtest.h" + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wignored-qualifiers" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif +#include "onnx/defs/data_type_utils.h" +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + +namespace onnxruntime { + +template +struct TestMap { + using key_type = K; + using mapped_type = V; +}; + +// Try recursive type registration and compatibility tests +using TestMapToMapInt64ToFloat = TestMap; +using TestMapStringToVectorInt64 = TestMap; + +// Trial to see if we resolve the setter properly +// a map with a key that has not been registered in data_types.cc +using TestMapMLFloat16ToFloat = TestMap; + +template +struct TestSequence { + using value_type = T; +}; + +using TestSequenceOfSequence = TestSequence; + +/// Adding an Opaque type with type parameters +struct TestOpaqueType_1 {}; +struct TestOpaqueType_2 {}; + +// String arrays must be extern to make them unique +// so the instantiated template would produce a unique type as well. +extern const char TestOpaqueDomain_1[] = "test_domain_1"; +extern const char TestOpaqueName_1[] = "test_name_1"; + +extern const char TestOpaqueDomain_2[] = "test_domain_2"; +extern const char TestOpaqueName_2[] = "test_name_2"; + +extern const char TestOpaqueEmpty[] = ""; + +struct TestOpaqueDomainOnly {}; +struct TestOpaqueNameOnly {}; +struct TestOpaqueNoNames {}; + +// Register Maps using Opaque types as values. Note that we +// use the same cpp runtime types but due to Opaque type domain, name +// and optional parameters we produce separate MLDataTypes that are NOT +// compatible with each other. +using MyOpaqueMapCpp_1 = std::map; +using MyOpaqueMapCpp_2 = std::map; + +// Register Sequence as containing an Opaque type +using MyOpaqueSeqCpp_1 = std::vector; +using MyOpaqueSeqCpp_2 = std::vector; + +ONNXRUNTIME_REGISTER_MAP(MyOpaqueMapCpp_1); +ONNXRUNTIME_REGISTER_MAP(MyOpaqueMapCpp_2); + +ONNXRUNTIME_REGISTER_MAP(TestMapToMapInt64ToFloat); +ONNXRUNTIME_REGISTER_MAP(TestMapStringToVectorInt64); +ONNXRUNTIME_REGISTER_MAP(TestMapMLFloat16ToFloat); + +ONNXRUNTIME_REGISTER_SEQ(MyOpaqueSeqCpp_1); +ONNXRUNTIME_REGISTER_SEQ(MyOpaqueSeqCpp_2); +ONNXRUNTIME_REGISTER_SEQ(TestSequenceOfSequence); + +ONNXRUNTIME_REGISTER_OPAQUE_TYPE(TestOpaqueType_1, TestOpaqueDomain_1, TestOpaqueName_1); +ONNXRUNTIME_REGISTER_OPAQUE_TYPE(TestOpaqueType_2, TestOpaqueDomain_2, TestOpaqueName_2); +// Special cases +ONNXRUNTIME_REGISTER_OPAQUE_TYPE(TestOpaqueDomainOnly, TestOpaqueDomain_1, TestOpaqueEmpty); +ONNXRUNTIME_REGISTER_OPAQUE_TYPE(TestOpaqueNameOnly, TestOpaqueEmpty, TestOpaqueName_1); +ONNXRUNTIME_REGISTER_OPAQUE_TYPE(TestOpaqueNoNames, TestOpaqueEmpty, TestOpaqueEmpty); + +#define REGISTER_ONNX_PROTO(TYPE) \ + { \ + MLDataType mltype = DataTypeImpl::GetType(); \ + DataTypeImpl::RegisterDataType(mltype); \ + } + +void RegisterTestTypes() { + REGISTER_ONNX_PROTO(MyOpaqueMapCpp_1); + REGISTER_ONNX_PROTO(MyOpaqueMapCpp_2); + + REGISTER_ONNX_PROTO(TestMapToMapInt64ToFloat); + REGISTER_ONNX_PROTO(TestMapStringToVectorInt64); + REGISTER_ONNX_PROTO(TestMapMLFloat16ToFloat); + + REGISTER_ONNX_PROTO(MyOpaqueSeqCpp_1); + REGISTER_ONNX_PROTO(MyOpaqueSeqCpp_2); + REGISTER_ONNX_PROTO(TestSequenceOfSequence); + + REGISTER_ONNX_PROTO(TestOpaqueType_1); + REGISTER_ONNX_PROTO(TestOpaqueType_2); + REGISTER_ONNX_PROTO(TestOpaqueDomainOnly); + REGISTER_ONNX_PROTO(TestOpaqueNameOnly); + REGISTER_ONNX_PROTO(TestOpaqueNoNames); +} + +namespace test { +using namespace ONNX_NAMESPACE; + +template +struct DimSetter; + +template <> +struct DimSetter<> { + static void set(TensorShapeProto&) {} +}; + +inline void AddDim(TensorShapeProto& proto, int d) { + proto.add_dim()->set_dim_value(d); +} + +template +struct DimSetter { + static void set(TensorShapeProto& proto) { + AddDim(proto, d); + } +}; + +template +struct DimSetter { + static void set(TensorShapeProto& proto) { + AddDim(proto, d); + DimSetter::set(proto); + } +}; + +template +struct TensorShapeTypeProto : public TensorShapeProto { + TensorShapeTypeProto() { + DimSetter::set(*this); + } +}; + +template <> +struct TensorShapeTypeProto<> : public TensorShapeProto {}; + +template +struct TensorTypeProto : public TypeProto { + TensorTypeProto() { + mutable_tensor_type()->set_elem_type(T); + } +}; + +template +struct SparseTensorTypeProto : public TypeProto { + SparseTensorTypeProto() { + mutable_sparse_tensor_type()->set_elem_type(T); + } + void SetShape(const TensorShapeProto& shape) { + mutable_sparse_tensor_type()->mutable_shape()->CopyFrom(shape); + } + void SetShape(TensorShapeProto&& shape) { + *mutable_sparse_tensor_type()->mutable_shape() = std::move(shape); + } + void ClearShape() { + mutable_sparse_tensor_type()->clear_shape(); + } +}; + +template +struct MapTypeProto : public TypeProto { + MapTypeProto() { + mutable_map_type()->set_key_type(key); + mutable_map_type()->mutable_value_type()->mutable_tensor_type()->set_elem_type(value); + } +}; + +class DataTypeTest : public testing::Test { + public: + static void SetUpTestCase() { + RegisterTestTypes(); + } +}; + +TEST_F(DataTypeTest, OpaqueRegistrationTest) { + // No parameters + TypeProto opaque_proto_1; + auto* mop = opaque_proto_1.mutable_opaque_type(); + mop->mutable_domain()->assign(TestOpaqueDomain_1); + mop->mutable_name()->assign(TestOpaqueName_1); + + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(opaque_proto_1)); + // OpaqueType_2 has the same domain and name but also has parameters + // so it is not compatible + EXPECT_FALSE(DataTypeImpl::GetType()->IsCompatible(opaque_proto_1)); + + // Now change domain and name for that of OpaqueType_3 + // now we are supposed to be compatible with OpaqueType_2 but not + // OpaqueType_1 + mop->mutable_domain()->assign(TestOpaqueDomain_2); + mop->mutable_name()->assign(TestOpaqueName_2); + EXPECT_FALSE(DataTypeImpl::GetType()->IsCompatible(opaque_proto_1)); + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(opaque_proto_1)); + + // assign back original domain/name and add params + mop->mutable_domain()->assign(TestOpaqueDomain_2); + mop->mutable_name()->assign(TestOpaqueName_2); + + EXPECT_FALSE(DataTypeImpl::GetType()->IsCompatible(opaque_proto_1)); + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(opaque_proto_1)); +} + +TEST_F(DataTypeTest, MapStringStringTest) { + TensorTypeProto tensor_type; + EXPECT_TRUE(DataTypeImpl::GetTensorType()->IsCompatible(tensor_type)); + EXPECT_FALSE(DataTypeImpl::GetTensorType()->IsCompatible(tensor_type)); + EXPECT_FALSE(DataTypeImpl::GetType()->IsCompatible(tensor_type)); + + MapTypeProto maps2s_type; + MapTypeProto maps2i_type; + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(maps2s_type)); + EXPECT_FALSE(DataTypeImpl::GetType()->IsCompatible(maps2i_type)); +} + +TEST_F(DataTypeTest, MapStringInt64Test) { + MapTypeProto maps2s_type; + MapTypeProto maps2i_type; + TensorTypeProto tensor_type; + EXPECT_FALSE(DataTypeImpl::GetType()->IsCompatible(maps2s_type)); + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(maps2i_type)); + EXPECT_FALSE(DataTypeImpl::GetType()->IsCompatible(tensor_type)); +} + +TEST_F(DataTypeTest, MapStringFloatTest) { + MapTypeProto maps2f_type; + MapTypeProto maps2i_type; + TensorTypeProto tensor_type; + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(maps2f_type)); + EXPECT_FALSE(DataTypeImpl::GetType()->IsCompatible(maps2i_type)); + EXPECT_FALSE(DataTypeImpl::GetType()->IsCompatible(tensor_type)); +} + +TEST_F(DataTypeTest, MapStringDoubleTest) { + MapTypeProto maps2d_type; + MapTypeProto maps2i_type; + TensorTypeProto tensor_type; + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(maps2d_type)); + EXPECT_FALSE(DataTypeImpl::GetType()->IsCompatible(maps2i_type)); + EXPECT_FALSE(DataTypeImpl::GetType()->IsCompatible(tensor_type)); +} + +TEST_F(DataTypeTest, MapInt64StringTest) { + MapTypeProto mapi2s_type; + MapTypeProto mapi2i_type; + TensorTypeProto tensor_type; + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(mapi2s_type)); + EXPECT_FALSE(DataTypeImpl::GetType()->IsCompatible(mapi2i_type)); + EXPECT_FALSE(DataTypeImpl::GetType()->IsCompatible(tensor_type)); +} + +TEST_F(DataTypeTest, MapInt64DoubleTest) { + MapTypeProto mapi2d_type; + MapTypeProto mapi2i_type; + TensorTypeProto tensor_type; + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(mapi2d_type)); + EXPECT_FALSE(DataTypeImpl::GetType()->IsCompatible(mapi2i_type)); + EXPECT_FALSE(DataTypeImpl::GetType()->IsCompatible(tensor_type)); +} + +TEST_F(DataTypeTest, RecursiveMapTest) { + TypeProto map_int64_to_map_int64_to_float; + auto* mut_map = map_int64_to_map_int64_to_float.mutable_map_type(); + mut_map->set_key_type(TensorProto_DataType_INT64); + mut_map = mut_map->mutable_value_type()->mutable_map_type(); + mut_map->set_key_type(TensorProto_DataType_INT64); + mut_map->mutable_value_type()->mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + + TypeProto map_string_to_vector_of_int64; + mut_map = map_string_to_vector_of_int64.mutable_map_type(); + mut_map->set_key_type(TensorProto_DataType_STRING); + mut_map->mutable_value_type()->mutable_sequence_type()->mutable_elem_type()->mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(map_int64_to_map_int64_to_float)); + EXPECT_FALSE(DataTypeImpl::GetType()->IsCompatible(map_string_to_vector_of_int64)); + + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(map_int64_to_map_int64_to_float)); + EXPECT_FALSE(DataTypeImpl::GetType()->IsCompatible(map_string_to_vector_of_int64)); + + // Map that contains an Opaque_1 + const auto* op1_proto = DataTypeImpl::GetType(); + TypeProto unod_map_int64_to_op1; + mut_map = unod_map_int64_to_op1.mutable_map_type(); + mut_map->set_key_type(TensorProto_DataType_INT64); + mut_map->mutable_value_type()->CopyFrom(*op1_proto->GetTypeProto()); + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(unod_map_int64_to_op1)); + + // Map that contains an Opaque_2 + const auto* op2_proto = DataTypeImpl::GetType(); + TypeProto unod_map_int64_to_op2; + mut_map = unod_map_int64_to_op2.mutable_map_type(); + mut_map->set_key_type(TensorProto_DataType_INT64); + mut_map->mutable_value_type()->CopyFrom(*op2_proto->GetTypeProto()); + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(unod_map_int64_to_op2)); +} + +TEST_F(DataTypeTest, RecursiveVectorTest) { + TypeProto seq_of_seq_string; + auto* mut_seq = seq_of_seq_string.mutable_sequence_type(); + mut_seq = mut_seq->mutable_elem_type()->mutable_sequence_type(); + mut_seq->mutable_elem_type()->mutable_tensor_type()->set_elem_type(TensorProto_DataType_STRING); + + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(seq_of_seq_string)); + EXPECT_FALSE(DataTypeImpl::GetType()->IsCompatible(seq_of_seq_string)); +} + +TEST_F(DataTypeTest, VectorMapStringToFloatTest) { + TypeProto vector_map_string_to_float; + vector_map_string_to_float.mutable_sequence_type()->mutable_elem_type()->mutable_map_type()->set_key_type(TensorProto_DataType_STRING); + vector_map_string_to_float.mutable_sequence_type()->mutable_elem_type()->mutable_map_type()->mutable_value_type()->mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + + MapTypeProto mapi2d_type; + MapTypeProto mapi2i_type; + TensorTypeProto tensor_type; + + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(vector_map_string_to_float)); + EXPECT_FALSE(DataTypeImpl::GetType()->IsCompatible(mapi2d_type)); + EXPECT_FALSE(DataTypeImpl::GetType()->IsCompatible(mapi2i_type)); + EXPECT_FALSE(DataTypeImpl::GetType()->IsCompatible(tensor_type)); +} + +TEST_F(DataTypeTest, VectorMapInt64ToFloatTest) { + TypeProto type_proto; + type_proto.mutable_sequence_type()->mutable_elem_type()->mutable_map_type()->set_key_type(TensorProto_DataType_INT64); + type_proto.mutable_sequence_type()->mutable_elem_type()->mutable_map_type()->mutable_value_type()->mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + + MapTypeProto mapi2d_type; + MapTypeProto mapi2i_type; + TensorTypeProto tensor_type; + + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(type_proto)); + EXPECT_FALSE(DataTypeImpl::GetType()->IsCompatible(mapi2d_type)); + EXPECT_FALSE(DataTypeImpl::GetType()->IsCompatible(mapi2i_type)); + EXPECT_FALSE(DataTypeImpl::GetType()->IsCompatible(tensor_type)); +} + +TEST_F(DataTypeTest, DataUtilsTest) { + using namespace ONNX_NAMESPACE::Utils; + // Test Tensor + { + const std::string tensor_uint64("tensor(uint64)"); + const auto* ten_proto = DataTypeImpl::GetTensorType()->GetTypeProto(); + EXPECT_NE(ten_proto, nullptr); + DataType ten_dt = DataTypeUtils::ToType(*ten_proto); + EXPECT_NE(ten_dt, nullptr); + EXPECT_EQ(tensor_uint64, *ten_dt); + DataType ten_from_str = DataTypeUtils::ToType(*ten_dt); + // Expect internalized strings + EXPECT_EQ(ten_dt, ten_from_str); + const auto& from_dt_proto = DataTypeUtils::ToTypeProto(ten_dt); + EXPECT_TRUE(DataTypeImpl::GetTensorType()->IsCompatible(from_dt_proto)); + } + // SparseTensor + // Currently test only with proto, no MLDataType yet. + { + const std::string tensor_uint64("sparse_tensor(uint64)"); + // We expect that the above string will be matched in both cases + // where we have shape and where we don't + SparseTensorTypeProto sparse_proto; + DataType ten_dt = DataTypeUtils::ToType(sparse_proto); + EXPECT_NE(ten_dt, nullptr); + EXPECT_EQ(tensor_uint64, *ten_dt); + DataType ten_from_str = DataTypeUtils::ToType(*ten_dt); + // Expect internalized strings + EXPECT_EQ(ten_dt, ten_from_str); + + // Now add empty shape, we expect the same string + TensorShapeTypeProto<> shape_no_dims; + sparse_proto.SetShape(shape_no_dims); + ten_dt = DataTypeUtils::ToType(sparse_proto); + EXPECT_NE(ten_dt, nullptr); + EXPECT_EQ(tensor_uint64, *ten_dt); + ten_from_str = DataTypeUtils::ToType(*ten_dt); + // Expect internalized strings + EXPECT_EQ(ten_dt, ten_from_str); + + // Now add shape with dimensions, we expect no difference + sparse_proto.ClearShape(); + TensorShapeTypeProto<10, 12> shape_with_dim; + sparse_proto.SetShape(shape_with_dim); + ten_dt = DataTypeUtils::ToType(sparse_proto); + EXPECT_NE(ten_dt, nullptr); + EXPECT_EQ(tensor_uint64, *ten_dt); + ten_from_str = DataTypeUtils::ToType(*ten_dt); + // Expect internalized strings + EXPECT_EQ(ten_dt, ten_from_str); + } + // Test Simple map + { + const std::string map_string_string("map(string,tensor(string))"); + const auto* map_proto = DataTypeImpl::GetType()->GetTypeProto(); + EXPECT_NE(map_proto, nullptr); + DataType map_dt = DataTypeUtils::ToType(*map_proto); + EXPECT_NE(map_dt, nullptr); + EXPECT_EQ(map_string_string, *map_dt); + DataType map_from_str = DataTypeUtils::ToType(*map_dt); + // Expect internalized strings + EXPECT_EQ(map_dt, map_from_str); + const auto& from_dt_proto = DataTypeUtils::ToTypeProto(map_dt); + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(from_dt_proto)); + } + // Test map with recursive value + { + const std::string map_int_map_int_float("map(int64,map(int64,tensor(float)))"); + const auto* map_proto = DataTypeImpl::GetType()->GetTypeProto(); + EXPECT_NE(map_proto, nullptr); + DataType map_dt = DataTypeUtils::ToType(*map_proto); + EXPECT_NE(map_dt, nullptr); + EXPECT_EQ(map_int_map_int_float, *map_dt); + DataType map_from_str = DataTypeUtils::ToType(*map_dt); + // Expect internalized strings + EXPECT_EQ(map_dt, map_from_str); + const auto& from_dt_proto = DataTypeUtils::ToTypeProto(map_dt); + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(from_dt_proto)); + } + { + const std::string opaque_map_2("map(int64,opaque(test_domain_2,test_name_2))"); + const auto* map_proto = DataTypeImpl::GetType()->GetTypeProto(); + EXPECT_NE(map_proto, nullptr); + DataType map_dt = DataTypeUtils::ToType(*map_proto); + EXPECT_NE(map_dt, nullptr); + EXPECT_EQ(opaque_map_2, *map_dt); + DataType map_from_str = DataTypeUtils::ToType(*map_dt); + // Expect internalized strings + EXPECT_EQ(map_dt, map_from_str); + const auto& from_dt_proto = DataTypeUtils::ToTypeProto(map_dt); + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(from_dt_proto)); + } + // Test simple seq + { + const std::string seq_float("seq(tensor(float))"); + const auto* seq_proto = DataTypeImpl::GetType()->GetTypeProto(); + EXPECT_NE(seq_proto, nullptr); + DataType seq_dt = DataTypeUtils::ToType(*seq_proto); + EXPECT_NE(seq_dt, nullptr); + EXPECT_EQ(seq_float, *seq_dt); + DataType seq_from_str = DataTypeUtils::ToType(*seq_dt); + // Expect internalized strings + EXPECT_EQ(seq_dt, seq_from_str); + const auto& from_dt_proto = DataTypeUtils::ToTypeProto(seq_dt); + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(from_dt_proto)); + } + // Test Sequence with recursion + { + const std::string seq_map_str_float("seq(map(string,tensor(float)))"); + const auto* seq_proto = DataTypeImpl::GetType()->GetTypeProto(); + EXPECT_NE(seq_proto, nullptr); + DataType seq_dt = DataTypeUtils::ToType(*seq_proto); + EXPECT_NE(seq_dt, nullptr); + EXPECT_EQ(seq_map_str_float, *seq_dt); + DataType seq_from_str = DataTypeUtils::ToType(*seq_dt); + // Expect internalized strings + EXPECT_EQ(seq_dt, seq_from_str); + const auto& from_dt_proto = DataTypeUtils::ToTypeProto(seq_dt); + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(from_dt_proto)); + } + // Test Sequence with opaque_2 + { + const std::string seq_opaque_2("seq(opaque(test_domain_2,test_name_2))"); + const auto* seq_proto = DataTypeImpl::GetType()->GetTypeProto(); + EXPECT_NE(seq_proto, nullptr); + DataType seq_dt = DataTypeUtils::ToType(*seq_proto); + EXPECT_NE(seq_dt, nullptr); + EXPECT_EQ(seq_opaque_2, *seq_dt); + DataType seq_from_str = DataTypeUtils::ToType(*seq_dt); + // Expect internalized strings + EXPECT_EQ(seq_dt, seq_from_str); + const auto& from_dt_proto = DataTypeUtils::ToTypeProto(seq_dt); + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(from_dt_proto)); + } + // Test Opaque type opaque_1 + { + const std::string opaque_q("seq(opaque(test_domain_1,test_name_1))"); + const auto* op_proto = DataTypeImpl::GetType()->GetTypeProto(); + EXPECT_NE(op_proto, nullptr); + DataType op_dt = DataTypeUtils::ToType(*op_proto); + EXPECT_NE(op_dt, nullptr); + EXPECT_EQ(opaque_q, *op_dt); + DataType op_from_str = DataTypeUtils::ToType(*op_dt); + // Expect internalized strings + EXPECT_EQ(op_dt, op_from_str); + const auto& from_dt_proto = DataTypeUtils::ToTypeProto(op_dt); + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(from_dt_proto)); + } + // Test TestOpaqueDomainOnly + { + const std::string opaque_q("opaque(test_domain_1,)"); + const auto* op_proto = DataTypeImpl::GetType()->GetTypeProto(); + EXPECT_NE(op_proto, nullptr); + DataType op_dt = DataTypeUtils::ToType(*op_proto); + EXPECT_NE(op_dt, nullptr); + EXPECT_EQ(opaque_q, *op_dt); + DataType op_from_str = DataTypeUtils::ToType(*op_dt); + // Expect internalized strings + EXPECT_EQ(op_dt, op_from_str); + const auto& from_dt_proto = DataTypeUtils::ToTypeProto(op_dt); + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(from_dt_proto)); + } + // Test TestOpaqueNameOnly + { + const std::string opaque_q("opaque(test_name_1)"); + const auto* op_proto = DataTypeImpl::GetType()->GetTypeProto(); + EXPECT_NE(op_proto, nullptr); + DataType op_dt = DataTypeUtils::ToType(*op_proto); + EXPECT_NE(op_dt, nullptr); + EXPECT_EQ(opaque_q, *op_dt); + DataType op_from_str = DataTypeUtils::ToType(*op_dt); + // Expect internalized strings + EXPECT_EQ(op_dt, op_from_str); + const auto& from_dt_proto = DataTypeUtils::ToTypeProto(op_dt); + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(from_dt_proto)); + } + // Test TestOpaqueNoNames + { + const std::string opaque_q("opaque()"); + const auto* op_proto = DataTypeImpl::GetType()->GetTypeProto(); + EXPECT_NE(op_proto, nullptr); + DataType op_dt = DataTypeUtils::ToType(*op_proto); + EXPECT_NE(op_dt, nullptr); + EXPECT_EQ(opaque_q, *op_dt); + DataType op_from_str = DataTypeUtils::ToType(*op_dt); + // Expect internalized strings + EXPECT_EQ(op_dt, op_from_str); + const auto& from_dt_proto = DataTypeUtils::ToTypeProto(op_dt); + EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(from_dt_proto)); + } +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/execution_frame_test.cc b/onnxruntime/test/framework/execution_frame_test.cc new file mode 100644 index 0000000000000..3176783f08508 --- /dev/null +++ b/onnxruntime/test/framework/execution_frame_test.cc @@ -0,0 +1,265 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/execution_frame.h" +#include "core/framework/op_kernel.h" +#include "core/framework/session_state.h" +#include "core/graph/model.h" +#include "core/providers/cpu/cpu_execution_provider.h" +#include "test_utils.h" +#include "gtest/gtest.h" + +using namespace ONNX_NAMESPACE; +using namespace std; + +namespace onnxruntime { +namespace test { +typedef std::vector ArgMap; + +std::shared_ptr DummyGraphWithClip() { + auto model = std::make_shared("test"); + onnxruntime::Graph& graph = model->MainGraph(); + TypeProto tensor_float; + tensor_float.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + onnxruntime::NodeArg input_def("X", &tensor_float), output_def("Y", &tensor_float); + + graph.AddNode("node1", "Clip", "clip operator", ArgMap{&input_def}, ArgMap{&output_def}); + return model; +} + +std::unique_ptr CreateCPUExecutionProvider() { + CPUExecutionProviderInfo info; + return std::make_unique(info); +} + +TEST(ExecutionFrameTest, TensorAllocationTest) { + onnxruntime::Model model("test"); + onnxruntime::Graph& graph = model.MainGraph(); + TypeProto tensor_float; + tensor_float.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + onnxruntime::NodeArg input_def("X", &tensor_float), output_def("Y", &tensor_float); + + graph.AddNode("node1", "Clip", "Clip operator", ArgMap{&input_def}, ArgMap{&output_def}); + onnxruntime::Node* node = graph.GetNode(graph.NumberOfNodes() - 1); + + Status status = graph.Resolve(); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + + auto cpu_xp = CreateCPUExecutionProvider(); + auto xp_typ = cpu_xp->Type(); + + KernelRegistryManager kernel_registry_manager; + kernel_registry_manager.RegisterKernelRegistry(cpu_xp->GetKernelRegistry(), KernelRegistryPriority::LowPriority); + + ExecutionProviders execution_providers; + execution_providers.Add(xp_typ, std::move(cpu_xp)); + + SessionState state{execution_providers}; + state.SetGraphViewer(std::make_unique(graph)); + + MLValueNameIdxMap& mlvalue_name_idx_map{state.GetMLValueNameIdxMap()}; + mlvalue_name_idx_map.Add("X"); + mlvalue_name_idx_map.Add("Y"); + + node->SetExecutionProviderType(xp_typ); + + std::unique_ptr p_seq_exec_plan; + // TODO below line is for testing only. In production use SequentialPlanner::CreatePlan() + status = SequentialPlanner::CreatePlan(graph, execution_providers, kernel_registry_manager, mlvalue_name_idx_map, + p_seq_exec_plan); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + state.SetExecutionPlan(std::move(p_seq_exec_plan)); + + vector outputs; + ExecutionFrame frame(std::unordered_map{}, + std::vector{}, + outputs, + state); + + int start_index = frame.GetFirstArgIndex(node->Index()); + EXPECT_EQ(start_index, 0); + + TensorShape shape(std::vector{2, 3}); + status = frame.AllocateTensorWithSelfOwnBuffer(start_index, DataTypeImpl::GetType(), + execution_providers.Get(xp_typ)->GetAllocator(0, ONNXRuntimeMemTypeDefault)->Info(), shape); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + + MLValue* p_ml_value = frame.GetMutableNodeInputOrOutputMLValue(0); + Tensor* p_tensor = p_ml_value ? p_ml_value->GetMutable() : nullptr; + EXPECT_TRUE(p_tensor); + EXPECT_EQ(p_tensor->Shape(), shape); + EXPECT_EQ(p_tensor->DataType(), DataTypeImpl::GetType()); + + //test share memory from tensor + TensorShape shape2(std::vector{3, 2}); + status = frame.AllocateTensorWithPreAllocateBuffer( + start_index + 1, + p_tensor->template MutableData(), + DataTypeImpl::GetType(), + p_tensor->Location(), + shape2); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + + const MLValue* p_ml_value_const = frame.GetNodeInputOrOutputMLValue(1); + auto tensor2 = p_ml_value_const ? &(p_ml_value_const->Get()) : nullptr; + EXPECT_TRUE(tensor2); + EXPECT_EQ(tensor2->Shape(), shape2); + EXPECT_EQ(tensor2->template Data(), p_tensor->template Data()); +} + +TEST(ExecutionFrameTest, FeedInDataTest) { + onnxruntime::Model model("test"); + onnxruntime::Graph& graph = model.MainGraph(); + TypeProto tensor_float; + tensor_float.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + onnxruntime::NodeArg input_def("X", &tensor_float), output_def("Y", &tensor_float); + + graph.AddNode("node1", "Clip", "Clip operator", ArgMap{&input_def}, ArgMap{&output_def}); + graph.Resolve(); + auto cpu_allocator = TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault); + auto element_type = DataTypeImpl::GetType(); + TensorShape shape({3, 2}); + void* buffer = cpu_allocator->Alloc(element_type->Size() * shape.Size()); + //create fake ml value with owned buffer. + std::unique_ptr p_tensor = std::make_unique(element_type, + shape, + buffer, + cpu_allocator->Info(), + cpu_allocator); + MLValue value; + value.Init(p_tensor.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); + + auto cpu_xp = CreateCPUExecutionProvider(); + auto xp_typ = cpu_xp->Type(); + + KernelRegistryManager kernel_registry_manager; + kernel_registry_manager.RegisterKernelRegistry(cpu_xp->GetKernelRegistry(), KernelRegistryPriority::LowPriority); + + ExecutionProviders execution_providers; + execution_providers.Add("", std::move(cpu_xp)); + + SessionState state{execution_providers}; + state.SetGraphViewer(std::make_unique(graph)); + + MLValueNameIdxMap& mlvalue_name_idx_map{state.GetMLValueNameIdxMap()}; + mlvalue_name_idx_map.Add("X"); + mlvalue_name_idx_map.Add("Y"); + + vector outputs; + ExecutionFrame frame(std::unordered_map{{"X", value}}, + std::vector{}, + outputs, + state); + + MLValue* p_ml_value = frame.GetMutableNodeInputOrOutputMLValue(0); + Tensor* p_tensor_arg_0 = p_ml_value ? p_ml_value->GetMutable() : nullptr; + EXPECT_TRUE(p_tensor_arg_0); + EXPECT_EQ(p_tensor_arg_0->Shape(), shape); + EXPECT_EQ(p_tensor_arg_0->DataType(), DataTypeImpl::GetType()); + EXPECT_EQ(p_tensor_arg_0->template MutableData(), buffer); +} + +TEST(ExecutionFrameTest, MemPatternTest) { + auto cpu_xp = CreateCPUExecutionProvider(); + auto xp_type = cpu_xp->Type(); + std::unordered_map domain_to_version; + domain_to_version[onnxruntime::kOnnxDomain] = 7; + onnxruntime::Model model("test", true, ModelMetaData(), IOnnxRuntimeOpSchemaRegistryList(), domain_to_version); + onnxruntime::Graph& graph = model.MainGraph(); + TypeProto tensor_float; + tensor_float.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + onnxruntime::NodeArg input_def1("X1", &tensor_float), + input_def2("X2", &tensor_float), + input_def3("X3", &tensor_float), + gemm1_out_def("T1", &tensor_float), + gemm2_out_def("T2", &tensor_float), + clip_out_def("T3", &tensor_float); + + graph.AddNode("node1", "MatMul", "gemm1", ArgMap{&input_def1, &input_def2}, ArgMap{&gemm1_out_def}) + ->SetExecutionProviderType(xp_type); + graph.AddNode("node2", "MatMul", "gemm2", ArgMap{&gemm1_out_def, &input_def3}, ArgMap{&gemm2_out_def}) + ->SetExecutionProviderType(xp_type); + graph.AddNode("node3", "Clip", "clip1", ArgMap{&gemm2_out_def}, ArgMap{&clip_out_def}) + ->SetExecutionProviderType(xp_type); + + auto status = graph.Resolve(); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + + KernelRegistryManager kernel_registry_manager; + kernel_registry_manager.RegisterKernelRegistry(cpu_xp->GetKernelRegistry(), KernelRegistryPriority::LowPriority); + + ExecutionProviders execution_providers; + execution_providers.Add(xp_type, std::move(cpu_xp)); + + //1. prepare input + SessionState state{execution_providers}; + state.SetGraphViewer(std::make_unique(graph)); + + MLValueNameIdxMap& mlvalue_name_idx_map{state.GetMLValueNameIdxMap()}; + + mlvalue_name_idx_map.Add("X1"); + mlvalue_name_idx_map.Add("X2"); + mlvalue_name_idx_map.Add("X3"); + mlvalue_name_idx_map.Add("T1"); + mlvalue_name_idx_map.Add("T2"); + mlvalue_name_idx_map.Add("T3"); + + auto cpu_allocator = execution_providers.Get(xp_type)->GetAllocator(0, ONNXRuntimeMemTypeDefault); + + MLValue v1, v2, v3; + CreateMLValue(cpu_allocator, + std::vector{1, 2}, + std::vector{1.0f, 1.0f}, &v1); + CreateMLValue(cpu_allocator, + std::vector{2, 2}, + std::vector(4, 1.0f), &v2); + CreateMLValue(cpu_allocator, + std::vector{2, 3}, + std::vector(6, 1.0f), &v3); + + std::unique_ptr p_seq_exec_plan = std::make_unique(); + status = SequentialPlanner::CreatePlan(graph, execution_providers, kernel_registry_manager, mlvalue_name_idx_map, + p_seq_exec_plan); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + + state.SetExecutionPlan(std::move(p_seq_exec_plan)); + + vector outputs; + ExecutionFrame frame(std::unordered_map{{"X1", v1}, {"X2", v2}, {"X3", v3}}, + std::vector{"T3"}, + outputs, + state); + + status = frame.AllocateMLValueTensorSelfOwnBuffer(3, + DataTypeImpl::GetType(), + cpu_allocator->Info(), + TensorShape(std::vector{2, 2})); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + + status = frame.AllocateMLValueTensorSelfOwnBuffer(4, + DataTypeImpl::GetType(), + cpu_allocator->Info(), + TensorShape(std::vector{2, 3})); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + + status = frame.AllocateMLValueTensorSelfOwnBuffer(5, + DataTypeImpl::GetType(), + cpu_allocator->Info(), + TensorShape(std::vector{2, 3})); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + + MemoryPatternGroup pattern; + status = frame.GeneratePatterns(&pattern); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + + EXPECT_EQ(pattern.patterns.size(), pattern.locations.size()); + EXPECT_EQ(pattern.patterns.size(), 1); + auto p = pattern.GetPatterns(cpu_allocator->Info()); + EXPECT_EQ(p->PeakSize(), sizeof(float) * (4 + 6)); + EXPECT_EQ(p->GetBlock(3)->offset_, 0); + EXPECT_EQ(p->GetBlock(4)->offset_, sizeof(float) * 4); +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/float_16_test.cc b/onnxruntime/test/framework/float_16_test.cc new file mode 100644 index 0000000000000..7ddf63532dffc --- /dev/null +++ b/onnxruntime/test/framework/float_16_test.cc @@ -0,0 +1,177 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/inference_session.h" + +#include +#include +#include +#include + +#include "core/common/logging/logging.h" +#include "core/framework/execution_provider.h" +#include "core/framework/op_kernel.h" +#include "core/framework/session_state.h" +#include "core/graph/graph.h" +#include "core/graph/model.h" +#include "core/graph/op.h" +#include "core/providers/cpu/cpu_execution_provider.h" +#include "core/providers/cpu/math/element_wise_ops.h" +#include "core/framework/tensorprotoutils.h" +#include "core/framework/data_types.h" +#include "core/common/status.h" +#include "test/capturing_sink.h" +#include "test/test_environment.h" +#include "test_utils.h" +#include "gtest/gtest.h" +#include "core/graph/schema_registry.h" +#include "core/framework/customregistry.h" +#include "core/util/math.h" + +using namespace ONNX_NAMESPACE; +using namespace onnxruntime::common; + +namespace onnxruntime { +namespace test { + +class MulFP16Kernel final : public OpKernel { + public: + MulFP16Kernel(const OpKernelInfo& info) : OpKernel(info) {} + + Status Compute(OpKernelContext* p_context) const { + const auto* X = p_context->Input(0); + const auto* W = p_context->Input(1); + + auto X_Data = X->Data(); + auto W_Data = W->Data(); + + auto& shape = X->Shape().GetDims(); + auto* Y = p_context->Output(0, shape); + auto* Y_Data = Y->MutableData(); + + size_t size = 1; + for (size_t i = 0; i < shape.size(); i++) { + size *= shape[i]; + } + + for (size_t i = 0; i < size; i++) { + Y_Data[i].val = math::floatToHalf( + math::halfToFloat(X_Data[i].val) * + math::halfToFloat(W_Data[i].val)); + } + + return Status::OK(); + } +}; + +//For test purpose, we register this MulFP16Kernel kernel to Mul op. +//Once the custom schema is ready, should update this. +KernelDefBuilder MulFP16KernelDef() { + KernelDefBuilder def; + def.SetName("Mul16") + .SetDomain(onnxruntime::kOnnxDomain) + .SinceVersion(6) + .Provider(onnxruntime::kCpuExecutionProvider) + .TypeConstraint("T", DataTypeImpl::GetTensorType()); + return def; +} + +ONNX_NAMESPACE::OpSchema GetMulFP16Schema() { + ONNX_NAMESPACE::OpSchema schema("Mul16", "unknown", 0); + schema.Input(0, + "A", + "First operand, should share the type with the second operand.", + "T"); + schema.Input( + 1, + "B", + "Second operand. With broadcasting can be of smaller size than A. ", + "T"); + schema.Output(0, "C", "Result, has same dimensions and type as A", "T"); + schema.TypeConstraint( + "T", + OpSchema::all_numeric_types(), + "Constrain input and output types to high-precision numeric tensors."); + schema.SinceVersion(6); + return schema; +} + +static const std::string MUL_MODEL_URI = "testdata/mul_16.pb"; + +void RunSession(InferenceSession& session_object, + RunOptions& run_options, + std::vector& dims_x, + std::vector& values_x, + std::vector& dims_y, + std::vector& values_y) { + // prepare inputs + MLValue ml_value; + CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault), dims_x, values_x, &ml_value); + NameMLValMap feeds; + feeds.insert(std::make_pair("X", ml_value)); + + // prepare outputs + std::vector output_names; + output_names.push_back("Y"); + std::vector fetches; + + // Now run + common::Status st = session_object.Run(run_options, feeds, output_names, &fetches); + std::cout << "Run returned status: " << st.ErrorMessage() << std::endl; + EXPECT_TRUE(st.IsOK()); + ASSERT_EQ(1, fetches.size()); + auto& rtensor = fetches.front().Get(); + TensorShape expected_shape(dims_y); + EXPECT_EQ(expected_shape, rtensor.Shape()); + const std::vector found(rtensor.template Data(), rtensor.template Data() + expected_shape.Size()); + ASSERT_EQ(found.size(), values_y.size()); + for (size_t i = 0; i < found.size(); i++) + ASSERT_EQ(found[i].val, values_y[i].val); +} + +TEST(Float16_Tests, Mul_16_Test) { + SessionOptions so; + + so.session_logid = "InferenceSessionTests.NoTimeout"; + + std::shared_ptr registry = std::make_shared(); + InferenceSession session_object{so, &DefaultLoggingManager()}; + EXPECT_TRUE(session_object.RegisterCustomRegistry(registry).IsOK()); + auto mulfp16_schema = GetMulFP16Schema(); + std::vector schemas = {mulfp16_schema}; + + EXPECT_TRUE(registry->RegisterOpSet(schemas, onnxruntime::kOnnxDomain, 5, 7).IsOK()); + + auto def = MulFP16KernelDef(); + //Register a foo kernel which is doing Add, but bind to Mul. + KernelCreateFn kernel_create_fn = [](const OpKernelInfo& info) -> OpKernel* { return new MulFP16Kernel(info); }; + EXPECT_TRUE(registry->RegisterCustomKernel(def, kernel_create_fn).IsOK()); + + EXPECT_TRUE(session_object.Load(MUL_MODEL_URI).IsOK()); + EXPECT_TRUE(session_object.Initialize().IsOK()); + + RunOptions run_options; + run_options.run_tag = "one session/one tag"; + + // prepare inputs + std::vector dims_x = {3, 2}; + std::vector values_x_32 = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; + std::vector values_x; + for (float i : values_x_32) { + values_x.push_back(MLFloat16(math::floatToHalf(i))); + } + + // prepare expected inputs and outputs + std::vector expected_dims_y = {3, 2}; + // now the expected value should be Add's result. + std::vector expected_values_y_32 = {1.0f, 4.0f, 9.0f, 16.0f, 25.0f, 36.0f}; + std::vector expected_values_y; + for (float i : expected_values_y_32) { + expected_values_y.push_back(MLFloat16(math::floatToHalf(i))); + } + + // Now run + RunSession(session_object, run_options, dims_x, values_x, expected_dims_y, expected_values_y); +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/header_files_test.cc b/onnxruntime/test/framework/header_files_test.cc new file mode 100644 index 0000000000000..dde3a9eb3f263 --- /dev/null +++ b/onnxruntime/test/framework/header_files_test.cc @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#if defined(__MACH__) +#else + +#include +#include +#include +#include +#include + +#include + +using namespace std::experimental::filesystem::v1; + +namespace onnxruntime { +namespace test { + +std::set RetrieveAllHeaders(const path& include_folder_root) { + std::set headers; + std::vector paths{include_folder_root}; + while (!paths.empty()) { + path node_data_root_path = paths.back(); + paths.pop_back(); + for (directory_iterator file_entry(node_data_root_path), end; file_entry != end; ++file_entry) { + if (is_directory(*file_entry)) { + paths.push_back(file_entry->path()); + continue; + } + + if (!file_entry->path().has_extension()) continue; + if (file_entry->path().extension() != ".h") continue; + headers.insert(file_entry->path()); + } + } + return headers; +} + +std::vector RetrieveHeaderDependencies(const path& header, const path& include_folder_root) { + std::vector header_dependencies; + std::ifstream header_stream(header); + const std::size_t pos_off = 10; // length of "#include \"" + if (!header_stream.good()) return header_dependencies; + std::string line; + while (std::getline(header_stream, line)) { + line.erase(line.find_last_not_of(" \r\n\t") + 1); + std::size_t pos = line.find("#include \"core/"); + if (pos != std::string::npos) { + header_dependencies.push_back(include_folder_root / line.substr(pos + pos_off, line.length() - pos - pos_off - 1)); + } else { + pos = line.find("#include headers(RetrieveAllHeaders(include_folder_path)); + + for (auto header_iterator = headers.begin(); header_iterator != headers.end(); header_iterator++) { + std::vector header_dependencies(RetrieveHeaderDependencies(*header_iterator, include_folder_path)); + for (auto dependency_iterator = header_dependencies.begin(); dependency_iterator != header_dependencies.end(); dependency_iterator++) { + EXPECT_TRUE(headers.find(*dependency_iterator) != headers.end()) << *header_iterator << " depends on " << *dependency_iterator << " that is not in include folder"; + } + } +} + +} // namespace test +} // namespace onnxruntime + +#endif diff --git a/onnxruntime/test/framework/inference_session_test.cc b/onnxruntime/test/framework/inference_session_test.cc new file mode 100644 index 0000000000000..45058099e9b83 --- /dev/null +++ b/onnxruntime/test/framework/inference_session_test.cc @@ -0,0 +1,988 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/inference_session.h" + +#include +#include +#include +#include +#include + +#include "core/common/logging/logging.h" +#include "core/common/profiler.h" +#include "core/framework/execution_provider.h" +#include "core/framework/kernel_registry.h" +#include "core/framework/op_kernel.h" +#include "core/framework/session_state.h" +#include "core/graph/graph.h" +#include "core/framework/computation_capacity.h" +#include "core/graph/model.h" +#include "core/graph/op.h" +#include "core/providers/cpu/cpu_execution_provider.h" +#include "core/providers/cpu/math/element_wise_ops.h" +#include "core/framework/tensorprotoutils.h" +#include "core/session/IOBinding.h" +#include "test/capturing_sink.h" +#include "test/test_environment.h" +#include "test_utils.h" +#include "gtest/gtest.h" + +using namespace std; +using namespace ONNX_NAMESPACE; +using namespace ::onnxruntime::logging; + +namespace onnxruntime { +class FuseAdd : public OpKernel { + public: + FuseAdd(const OpKernelInfo& info) : OpKernel(info) {} + + Status Compute(OpKernelContext* context) const override { + auto X = context->Input(0); + auto Y = context->Input(1); + auto Z = context->Input(2); + auto& shape = X->Shape(); + auto M = context->Output(0, shape)->template MutableData(); + for (int i = 0; i < shape.Size(); ++i) { + *(M + i) = *(X->template Data() + i) + *(Y->template Data() + i) + *(Z->template Data() + i); + } + return Status::OK(); + } +}; +std::string kFuseTest = "FuseTest"; +std::string kFuseExecutionProvider = "FuseExecutionProvider"; +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kFuseExecutionProvider, kFuseTest, 1, FuseAdd); +ONNX_OPERATOR_KERNEL_EX(FuseAdd, + kFuseTest, + 1, + kFuseExecutionProvider, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + FuseAdd); + +void RegisterOperatorKernels(std::function fn) { + fn(BuildKernel()); +} + +class FuseExecutionProvider : public IExecutionProvider { + public: + explicit FuseExecutionProvider() { + DeviceAllocatorRegistrationInfo device_info({ONNXRuntimeMemTypeDefault, [](int) { return std::make_unique(); }, std::numeric_limits::max()}); + InsertAllocator(std::shared_ptr( + std::make_unique(device_info.factory(0)))); + } + + std::vector> + GetCapability(const onnxruntime::GraphViewer& graph, + const std::vector& /*kernel_registries*/) const override { + // Fuse two add into one. + std::vector> result; + std::unique_ptr sub_graph = std::make_unique(); + for (auto& node : graph.Nodes()) { + sub_graph->nodes.push_back(node.Index()); + } + auto meta_def = std::make_unique<::onnxruntime::IndexedSubGraph::MetaDef>(); + meta_def->name = "FuseAdd"; + meta_def->domain = "FuseTest"; + meta_def->inputs = {"X", "Y", "Z"}; + meta_def->outputs = {"M"}; + meta_def->since_version = 1; + meta_def->status = ONNX_NAMESPACE::EXPERIMENTAL; + sub_graph->SetMetaDef(meta_def); + result.push_back(std::make_unique(std::move(sub_graph), nullptr)); + return result; + } + + std::shared_ptr<::onnxruntime::KernelRegistry> GetKernelRegistry() const override { + static std::shared_ptr<::onnxruntime::KernelRegistry> kernel_registry = std::make_shared<::onnxruntime::KernelRegistry>(RegisterOperatorKernels); + return kernel_registry; + } + + common::Status CopyTensor(const Tensor& src, Tensor& dst) const override { + ONNXRUNTIME_UNUSED_PARAMETER(src); + ONNXRUNTIME_UNUSED_PARAMETER(dst); + return Status::OK(); + } + + const void* GetExecutionHandle() const noexcept override { + return nullptr; + } + + std::string Type() const override { + return "FuseExecutionProvider"; + } +}; +namespace test { +static void VerifyOutputs(const std::vector& fetches, + const std::vector& expected_dims, + const std::vector& expected_values); +static const std::string MODEL_URI = "testdata/mul_1.pb"; +static const std::string MODEL_URI_NO_OPSET = "testdata/mul_1.pb.noopset"; +//static const std::string MODEL_URI = "./testdata/squeezenet/model.onnx"; // TODO enable this after we've weights? + +static void CreateMatMulModel(std::unique_ptr& p_model, ProviderType provider_type) { + std::unordered_map domain_to_version; + domain_to_version[onnxruntime::kOnnxDomain] = 7; + // Generate the input & output def lists + p_model = std::make_unique("test", true, ModelMetaData(), IOnnxRuntimeOpSchemaRegistryList(), domain_to_version); + onnxruntime::Graph& graph = p_model->MainGraph(); + + TypeProto tensor_float; + tensor_float.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + + std::vector input_defs; + auto& input_arg_a = graph.GetOrCreateNodeArg("A", &tensor_float); + input_defs.push_back(&input_arg_a); + + auto& input_arg_b = graph.GetOrCreateNodeArg("B", &tensor_float); + input_defs.push_back(&input_arg_b); + + std::vector output_defs; + auto& output_arg = graph.GetOrCreateNodeArg("Y", &tensor_float); + output_defs.push_back(&output_arg); + + // Create a simple model + auto& node = *graph.AddNode("node1", "MatMul", "MatMul", input_defs, output_defs, nullptr, onnxruntime::kOnnxDomain); + if (provider_type == kCpuExecutionProvider) { + node.SetExecutionProviderType(provider_type); + } else { +#ifdef USE_CUDA + node.SetExecutionProviderType(provider_type); +#endif + } + Status status = graph.Resolve(); + ASSERT_TRUE(status.IsOK()) << status.ErrorMessage(); +} + +void VerifyOutputs(const std::vector& fetches, + const std::vector& expected_dims, + const std::vector& expected_values) { + ASSERT_EQ(1, fetches.size()); + auto& rtensor = fetches.front().Get(); + TensorShape expected_shape(expected_dims); + ASSERT_EQ(expected_shape, rtensor.Shape()); + const std::vector found(rtensor.template Data(), rtensor.template Data() + expected_values.size()); + ASSERT_EQ(expected_values, found); +} + +void RunModel(InferenceSession& session_object, + const RunOptions& run_options, + bool is_preallocate_output_vec = false) { + // prepare inputs + std::vector dims_mul_x = {3, 2}; + std::vector values_mul_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; + MLValue ml_value; + CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault), dims_mul_x, values_mul_x, &ml_value); + NameMLValMap feeds; + feeds.insert(std::make_pair("X", ml_value)); + + // prepare outputs + std::vector output_names; + output_names.push_back("Y"); + std::vector fetches; + + if (is_preallocate_output_vec) { + fetches.resize(output_names.size()); + for (auto& elem : fetches) { + CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault), dims_mul_x, values_mul_x, &elem); + } + } + + // prepare expected inputs and outputs + std::vector expected_dims_mul_y = {3, 2}; + std::vector expected_values_mul_y = {1.0f, 4.0f, 9.0f, 16.0f, 25.0f, 36.0f}; + + // Now run + common::Status st = session_object.Run(run_options, feeds, output_names, &fetches); + if (!st.IsOK()) { + std::cout << "Run returned status: " << st.ErrorMessage() << std::endl; + } + ASSERT_TRUE(st.IsOK()); + VerifyOutputs(fetches, expected_dims_mul_y, expected_values_mul_y); +} + +void RunModelWithBindingMatMul(InferenceSession& session_object, + const RunOptions& run_options, + ProviderType bind_provider_type, + bool is_preallocate_output_vec = false, + ProviderType allocation_provider = kCpuExecutionProvider) { + unique_ptr io_binding; + Status st = session_object.NewIOBinding(&io_binding); + ASSERT_TRUE(st.IsOK()); + auto input_allocator = io_binding->GetCPUAllocator(0, bind_provider_type); + + // prepare inputs + std::vector values_mul_x = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f}; + /* + 0 1 2 3 0 1 2 + 4 5 6 7 3 4 5 + 8 9 10 11 6 7 8 + 9 10 11 + */ + // bind one input to cpu allocator from bind_provider_type, and another on user provided CPU memory + // so both code pathes are covered + MLValue input_ml_value_A; + std::vector dims_mul_x_A = {3, 4}; + CreateMLValue(input_allocator, dims_mul_x_A, values_mul_x, &input_ml_value_A); + + MLValue input_ml_value_B; + std::vector dims_mul_x_B = {4, 3}; + CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault), dims_mul_x_B, values_mul_x, &input_ml_value_B); + + io_binding->BindInput("A", input_ml_value_A); + io_binding->BindInput("B", input_ml_value_B); + + // prepare outputs + std::vector expected_output_dims = {3, 3}; + MLValue output_ml_value; + if (is_preallocate_output_vec) { + if (allocation_provider == kCpuExecutionProvider) { + AllocateMLValue(TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault), expected_output_dims, &output_ml_value); + } else if (allocation_provider == kCudaExecutionProvider) { +#ifdef USE_CUDA + AllocateMLValue(TestCudaExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault), expected_output_dims, &output_ml_value); +#endif + } else { + ONNXRUNTIME_THROW("Unsupported provider"); + } + } + io_binding->BindOutput("Y", output_ml_value); + ASSERT_TRUE(io_binding->SynchronizeInputs().IsOK()); + + // prepare expected inputs and outputs + std::vector expected_values_mul_y = {42, 48, 54, 114, 136, 158, 186, 224, 262}; + + // Now run + st = session_object.Run(run_options, *io_binding.get()); + + std::cout << "Run returned status: " << st.ErrorMessage() << std::endl; + ASSERT_TRUE(st.IsOK()); + + if (is_preallocate_output_vec && + allocation_provider == kCudaExecutionProvider) { +#ifdef USE_CUDA + // in this case we need to copy the tensor from cuda to cpu + vector& outputs = io_binding->GetOutputs(); + ASSERT_EQ(1, outputs.size()); + auto& rtensor = outputs.front().Get(); + auto element_type = rtensor.DataType(); + auto& shape = rtensor.Shape(); + auto cpu_allocator = TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault); + void* buffer = cpu_allocator->Alloc(element_type->Size() * shape.Size()); + ONNXRUNTIME_ENFORCE(buffer); + std::unique_ptr cpu_tensor = std::make_unique(element_type, + shape, + buffer, + cpu_allocator->Info(), + cpu_allocator); + st = TestCudaExecutionProvider()->CopyTensor(rtensor, *cpu_tensor.get()); + ASSERT_TRUE(st.IsOK()); + MLValue ml_value; + ml_value.Init(cpu_tensor.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); + VerifyOutputs({ml_value}, expected_output_dims, expected_values_mul_y); +#endif + } else { + if (allocation_provider == kCudaExecutionProvider) { +#ifdef USE_CUDA + TestCudaExecutionProvider()->Sync(); +#endif + } + VerifyOutputs(io_binding->GetOutputs(), expected_output_dims, expected_values_mul_y); + } +} + +TEST(InferenceSessionTests, NoTimeout) { + SessionOptions so; + + so.session_logid = "InferenceSessionTests.NoTimeout"; + + InferenceSession session_object{so, &DefaultLoggingManager()}; + ASSERT_TRUE(session_object.Load(MODEL_URI).IsOK()); + ASSERT_TRUE(session_object.Initialize().IsOK()); + + RunOptions run_options; + run_options.run_tag = "one session/one tag"; + RunModel(session_object, run_options); +} + +TEST(InferenceSessionTests, DisableCPUArena) { + SessionOptions so; + + so.session_logid = "InferenceSessionTests.DisableCPUArena"; + so.enable_cpu_mem_arena = false; + + InferenceSession session_object{so, &DefaultLoggingManager()}; + ASSERT_TRUE(session_object.Load(MODEL_URI).IsOK()); + ASSERT_TRUE(session_object.Initialize().IsOK()); + + RunOptions run_options; + run_options.run_tag = "one session/one tag"; + RunModel(session_object, run_options); +} + +#ifdef ONNXRUNTIME_RUN_EXTERNAL_ONNX_TESTS +static bool Compare(const InputDefList& f_arg, const InputDefList& s_arg) { + if (f_arg.size() != s_arg.size()) { + cout << "Sizes differ: f_arg size: " << f_arg.size() << " s_arg size: " << s_arg.size() << endl; + return false; + } + + for (size_t i = 0; i < f_arg.size(); ++i) { + const onnxruntime::NodeArg* x = f_arg[i]; + const onnxruntime::NodeArg* y = s_arg[i]; + if ((x->Shape() == nullptr) ^ (y->Shape() == nullptr)) { + return false; + } + if (!x->Shape()) { + continue; + } + vector x_shape = utils::GetTensorShapeFromTensorShapeProto(*x->Shape()); + vector y_shape = utils::GetTensorShapeFromTensorShapeProto(*y->Shape()); + if (x->Name() == y->Name() && x_shape == y_shape && *x->Type() == *y->Type()) { + continue; + } + return false; + } + + return true; +} + +TEST(InferenceSessionTests, ModelMetadata) { + SessionOptions so; + + so.session_logid = "InferenceSessionTests.ModelMetadata"; + InferenceSession session_object{so, &DefaultLoggingManager()}; + string model_uri = "../models/opset8/test_squeezenet/model.onnx"; + ASSERT_TRUE(session_object.Load(model_uri).IsOK()); + + std::shared_ptr p_model; + Status st = onnxruntime::Model::Load(model_uri, p_model); + ASSERT_TRUE(st.IsOK()); + const onnxruntime::Graph& graph = p_model->MainGraph(); + + // 1. first test the model meta + { + auto retval = session_object.GetModelMetadata(); + ASSERT_TRUE(retval.first.IsOK()); + const ModelMetadata* m = retval.second; + ASSERT_TRUE(m->custom_metadata_map == p_model->MetaData() && + m->description == p_model->DocString() && + m->domain == p_model->Domain() && + m->graph_name == graph.Name() && + m->producer_name == p_model->ProducerName() && + m->version == p_model->ModelVersion()); + } + + { + // 2. test inputs + auto& inputs = graph.GetInputs(); + auto weights = graph.GetAllInitializedTensors(); + + // skip the weights + InputDefList inputs_no_weights; + for (auto& elem : inputs) { + if (weights.find(elem->Name()) != weights.end()) { + continue; + } else { + inputs_no_weights.push_back(elem); + } + } + + auto retval = session_object.GetModelInputs(); + cout << "weights size: " << weights.size() + << " inputs.size(): " << inputs.size() + << " from session: " << retval.second->size() << endl; + ASSERT_TRUE(retval.first.IsOK()); + ASSERT_TRUE(Compare(inputs_no_weights, *retval.second)); + } + + // 3. test outputs + { + auto retval = session_object.GetModelOutputs(); + ASSERT_TRUE(retval.first.IsOK()); + + auto& outputs = graph.GetOutputs(); + retval = session_object.GetModelOutputs(); + ASSERT_TRUE(retval.first.IsOK()); + ASSERT_TRUE(Compare(outputs, *retval.second)); + } +} +#endif +TEST(InferenceSessionTests, CheckRunLogger) { + SessionOptions so; + + so.session_logid = "CheckRunLogger"; + + // create CapturingSink. LoggingManager will own it, but as long as the logging_manager + // is around our pointer stays valid. + auto capturing_sink = new CapturingSink(); + + auto logging_manager = std::make_unique( + std::unique_ptr(capturing_sink), logging::Severity::kVERBOSE, false, LoggingManager::InstanceType::Temporal); + + InferenceSession session_object{so, logging_manager.get()}; + ASSERT_TRUE(session_object.Load(MODEL_URI).IsOK()); + ASSERT_TRUE(session_object.Initialize().IsOK()); + + RunOptions run_options; + run_options.run_tag = "RunTag"; + RunModel(session_object, run_options); + +#ifndef NDEBUG + // check for some VLOG output to make sure tag was correct. VLOG is not enabled in release build + auto& msgs = capturing_sink->Messages(); + std::copy(msgs.begin(), msgs.end(), std::ostream_iterator(std::cout, "\n")); + bool have_log_entry_with_run_tag = + (std::find_if(msgs.begin(), msgs.end(), + [&run_options](std::string msg) { return msg.find(run_options.run_tag) != string::npos; }) != msgs.end()); + + ASSERT_TRUE(have_log_entry_with_run_tag); +#endif +} + +TEST(InferenceSessionTests, CheckRunProfilerWithSessionOptions) { + SessionOptions so; + + so.session_logid = "CheckRunProfiler"; + so.enable_profiling = true; + so.profile_file_prefix = "onnxprofile_profile_test"; + + InferenceSession session_object(so); + ASSERT_TRUE(session_object.Load(MODEL_URI).IsOK()); + ASSERT_TRUE(session_object.Initialize().IsOK()); + + RunOptions run_options; + run_options.run_tag = "RunTag"; + + RunModel(session_object, run_options); + std::string profile_file = session_object.EndProfiling(); + + std::ifstream profile(profile_file); + ASSERT_TRUE(profile); + std::string line; + + std::vector tags = {"pid", "dur", "ts", "ph", "X", "name", "args"}; + int count = 0; + while (std::getline(profile, line)) { + if (count == 0) { + ASSERT_TRUE(line.find("[") != string::npos); + } else if (count <= 7) { + for (auto& s : tags) { + ASSERT_TRUE(line.find(s) != string::npos); + } + } else { + ASSERT_TRUE(line.find("]") != string::npos); + } + + if (count == 1) { + ASSERT_TRUE(line.find("model_loading_uri") != string::npos); + } + count++; + } +} + +TEST(InferenceSessionTests, CheckRunProfilerWithStartProfile) { + SessionOptions so; + + so.session_logid = "CheckRunProfiler"; + + InferenceSession session_object(so); + ASSERT_TRUE(session_object.Load(MODEL_URI).IsOK()); + ASSERT_TRUE(session_object.Initialize().IsOK()); + + RunOptions run_options; + run_options.run_tag = "RunTag"; + + session_object.StartProfiling("onnxruntime_profile_custom"); + RunModel(session_object, run_options); + std::string profile_file = session_object.EndProfiling(); + + std::ifstream profile(profile_file); + std::string line; + + std::vector tags = {"pid", "dur", "ts", "ph", "X", "name", "args"}; + int count = 0; + while (std::getline(profile, line)) { + if (count == 0) { + ASSERT_TRUE(line.find("[") != string::npos); + } else if (count <= 5) { + for (auto& s : tags) { + ASSERT_TRUE(line.find(s) != string::npos); + } + } else { + ASSERT_TRUE(line.find("]") != string::npos); + } + + if (count == 1) { + ASSERT_TRUE(line.find("mul_1_fence_before") != string::npos); + } + count++; + } +} + +TEST(InferenceSessionTests, MultipleSessionsNoTimeout) { + SessionOptions session_options; + + session_options.session_logid = "InferenceSessionTests.MultipleSessionsNoTimeout"; + InferenceSession session_object{session_options, &DefaultLoggingManager()}; + ASSERT_TRUE(session_object.Load(MODEL_URI).IsOK()); + ASSERT_TRUE(session_object.Initialize().IsOK()); + + std::thread thread1{[&session_object]() { + RunOptions run_options; + run_options.run_tag = "one session/thread 1"; + RunModel(session_object, run_options); + }}; + + std::thread thread2{[&session_object]() { + RunOptions run_options; + run_options.run_tag = "one session/thread 2"; + RunModel(session_object, run_options); + }}; + + thread1.join(); + thread2.join(); +} + +TEST(InferenceSessionTests, PreAllocateOutputVector) { + SessionOptions so; + + so.session_logid = "InferenceSessionTests.PreAllocateOutputVector"; + + InferenceSession session_object{so, &DefaultLoggingManager()}; + ASSERT_TRUE(session_object.Load(MODEL_URI).IsOK()); + ASSERT_TRUE(session_object.Initialize().IsOK()); + + RunOptions run_options; + run_options.run_tag = "InferenceSessionTests.PreAllocateOutputVector"; + bool is_preallocate_output_vec = true; + RunModel(session_object, run_options, is_preallocate_output_vec); +} + +TEST(InferenceSessionTests, ConfigureVerbosityLevel) { + SessionOptions so; + + so.session_logid = "ConfigureVerbosityLevel"; + so.session_log_verbosity_level = 1; + + // create CapturingSink. LoggingManager will own it, but as long as the logging_manager + // is around our pointer stays valid. + auto capturing_sink = new CapturingSink(); + + auto logging_manager = std::make_unique( + std::unique_ptr(capturing_sink), + logging::Severity::kVERBOSE, + false, + LoggingManager::InstanceType::Temporal); + + InferenceSession session_object{so, logging_manager.get()}; + ASSERT_TRUE(session_object.Load(MODEL_URI).IsOK()); + ASSERT_TRUE(session_object.Initialize().IsOK()); + + RunOptions run_options; + run_options.run_tag = "ConfigureVerbosityLevel"; + run_options.run_log_verbosity_level = 1; + RunModel(session_object, run_options); + +#ifndef NDEBUG + // check for some VLOG output to make sure tag was correct. VLOG is not enabled in release build + auto& msgs = capturing_sink->Messages(); + std::copy(msgs.begin(), msgs.end(), std::ostream_iterator(std::cout, "\n")); + bool have_log_entry_with_vlog_session_msg = + (std::find_if(msgs.begin(), msgs.end(), + [&](std::string msg) { return msg.find("Added input argument with name") != string::npos; }) != + msgs.end()); + + ASSERT_TRUE(have_log_entry_with_vlog_session_msg); + + bool have_log_entry_with_vlog_run_msg = + (std::find_if(msgs.begin(), msgs.end(), + [&](std::string msg) { return msg.find("Size of execution plan vector") != string::npos; }) != + msgs.end()); + + ASSERT_TRUE(have_log_entry_with_vlog_run_msg); +#endif +} + +TEST(InferenceSessionTests, TestWithIstream) { + SessionOptions so; + + so.session_logid = "InferenceSessionTests.TestWithIstream"; + + InferenceSession session_object{so}; + + std::ifstream model_file_stream(MODEL_URI, ios::in | ios::binary); + ASSERT_TRUE(model_file_stream.good()); + ASSERT_TRUE(session_object.Load(model_file_stream).IsOK()); + ASSERT_TRUE(session_object.Initialize().IsOK()); + + RunOptions run_options; + run_options.run_tag = "InferenceSessionTests.TestWithIstream"; + RunModel(session_object, run_options); +} + +TEST(InferenceSessionTests, TestRegisterExecutionProvider) { + SessionOptions so; + + so.session_logid = "InferenceSessionTests.TestWithIstream"; + + InferenceSession session_object{so}; + CPUExecutionProviderInfo epi; + ASSERT_TRUE(session_object.RegisterExecutionProvider(std::make_unique(epi)).IsOK()); + + std::ifstream model_file_stream(MODEL_URI, ios::in | ios::binary); + ASSERT_TRUE(model_file_stream.good()); + ASSERT_TRUE(session_object.Load(model_file_stream).IsOK()); + ASSERT_TRUE(session_object.Initialize().IsOK()); + + RunOptions run_options; + run_options.run_tag = "InferenceSessionTests.TestWithIstream"; + RunModel(session_object, run_options); +} + +static void TestBindHelper(const std::string& log_str, + ProviderType bind_provider_type, + ProviderType run_provider_type, + bool preallocate_output, + ProviderType allocation_provider = kCpuExecutionProvider) { + SessionOptions so; + + so.session_logid = "InferenceSessionTests." + log_str; + so.session_log_verbosity_level = 1; // change to 1 for detailed logging + + InferenceSession session_object{so, &DefaultLoggingManager()}; + + if (bind_provider_type == kCudaExecutionProvider || run_provider_type == kCudaExecutionProvider) { +#ifdef USE_CUDA + CUDAExecutionProviderInfo epi; + epi.device_id = 0; + EXPECT_TRUE(session_object.RegisterExecutionProvider(std::make_unique(epi)).IsOK()); +#endif + } + + std::unique_ptr p_model; + CreateMatMulModel(p_model, run_provider_type); + + std::stringstream s1; + p_model->ToProto().SerializeToOstream(&s1); + ASSERT_TRUE(session_object.Load(s1).IsOK()); + ASSERT_TRUE(session_object.Initialize().IsOK()); + + RunOptions run_options; + run_options.run_log_verbosity_level = so.session_log_verbosity_level; + run_options.run_tag = so.session_logid; + RunModelWithBindingMatMul(session_object, + run_options, + bind_provider_type, + preallocate_output, + allocation_provider); +} + +TEST(InferenceSessionTests, TestBindCpu) { + TestBindHelper("TestBindCpu", + kCpuExecutionProvider, + kCpuExecutionProvider, + false /* don't preallocate output */); +} + +TEST(InferenceSessionTests, TestIOBindingReuse) { + SessionOptions so; + InferenceSession session_object(so); + std::unique_ptr p_model; + CreateMatMulModel(p_model, kCpuExecutionProvider); + + std::stringstream s1; + p_model->ToProto().SerializeToOstream(&s1); + ASSERT_TRUE(session_object.Load(s1).IsOK()); + ASSERT_TRUE(session_object.Initialize().IsOK()); + unique_ptr io_binding; + Status st = session_object.NewIOBinding(&io_binding); + ASSERT_TRUE(st.IsOK()); + + MLValue ml_value1; + vector v1{2.f}; + CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault), {1}, v1, &ml_value1); + io_binding->BindOutput("foo", ml_value1); + ASSERT_TRUE(io_binding->GetOutputs().size() == 1); + auto span = io_binding->GetOutputs()[0].Get().DataAsSpan(); + ASSERT_TRUE(static_cast(span.size()) == v1.size()); + for (size_t i = 0; i < v1.size(); ++i) { + ASSERT_TRUE(v1[i] == span[i]); + } + + MLValue ml_value2; + vector v2{3.f}; + CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault), {1}, v2, &ml_value2); + io_binding->BindOutput("foo", ml_value2); + ASSERT_TRUE(io_binding->GetOutputs().size() == 1); + span = io_binding->GetOutputs()[0].Get().DataAsSpan(); + ASSERT_TRUE(static_cast(span.size()) == v2.size()); + for (size_t i = 0; i < v2.size(); ++i) { + ASSERT_TRUE(v2[i] == span[i]); + } +} + +TEST(InferenceSessionTests, InvalidInputTypeOfTensorElement) { + SessionOptions so; + + so.session_logid = "InferenceSessionTests.InvalidInputTypeOfTensorElement"; + + InferenceSession session_object{so, &DefaultLoggingManager()}; + ASSERT_TRUE(session_object.Load(MODEL_URI).IsOK()); + ASSERT_TRUE(session_object.Initialize().IsOK()); + + RunOptions run_options; + run_options.run_tag = so.session_logid; + + // prepare inputs + std::vector dims_mul_x = {3, 2}; + std::vector values_mul_x = {1, 2, 3, 4, 5, 6}; + MLValue ml_value; + CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault), dims_mul_x, values_mul_x, &ml_value); + NameMLValMap feeds; + feeds.insert(std::make_pair("X", ml_value)); + + // prepare outputs + std::vector output_names; + output_names.push_back("Y"); + std::vector fetches; + + // prepare expected inputs and outputs + std::vector expected_dims_mul_y = {3, 2}; + std::vector expected_values_mul_y = {1.0f, 4.0f, 9.0f, 16.0f, 25.0f, 36.0f}; + + // Now run + common::Status st = session_object.Run(run_options, feeds, output_names, &fetches); + if (!st.IsOK()) { + std::cout << "Run returned status: " << st.ErrorMessage() << std::endl; + } + ASSERT_TRUE(!st.IsOK()); +} + +#ifdef USE_CUDA + +TEST(InferenceSessionTests, TestBindCuda) { + TestBindHelper("TestBindCuda", + kCudaExecutionProvider, + kCudaExecutionProvider, + false /* don't preallocate output */); +} + +TEST(InferenceSessionTests, TestBindCudaPreallocateOutputOnCuda) { + TestBindHelper("TestBindCudaPreallocateOutputOnCuda", + kCudaExecutionProvider, + kCudaExecutionProvider, + true /* preallocate output on GPU */, + kCudaExecutionProvider); +} + +TEST(InferenceSessionTests, TestBindCudaPreallocateOutputOnCpu) { + TestBindHelper("TestBindCudaPreallocateOutputOnCpu", + kCudaExecutionProvider, + kCudaExecutionProvider, + true /* preallocate output on CPU */, + kCpuExecutionProvider); +} + +TEST(InferenceSessionTests, TestBindCudaPreallocateOutputOnCpu2) { + TestBindHelper("TestBindCudaPreallocateOutputOnCpu2", + kCudaExecutionProvider, + kCpuExecutionProvider, + true /* preallocate output on CPU */, + kCpuExecutionProvider); +} + +#endif + +TEST(InferenceSessionTests, ModelWithoutOpset) { + SessionOptions so; + + so.session_logid = "InferenceSessionTests.ModelWithoutOpset"; + + InferenceSession session_object{so, &DefaultLoggingManager()}; + Status retval = session_object.Load(MODEL_URI_NO_OPSET); + ASSERT_FALSE(retval.IsOK()); + if (!retval.IsOK()) { + ASSERT_TRUE(retval.ErrorMessage().find("Missing opset in the model") != std::string::npos); + } +} + +TEST(ExecutionProviderTest, FunctionTest) { + onnxruntime::Model model("graph_1"); + auto& graph = model.MainGraph(); + std::vector inputs; + std::vector outputs; + + // FLOAT tensor. + ONNX_NAMESPACE::TypeProto float_tensor; + float_tensor.mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_FLOAT); + float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(3); + float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(2); + + auto& input_arg_1 = graph.GetOrCreateNodeArg("X", &float_tensor); + auto& input_arg_2 = graph.GetOrCreateNodeArg("Y", &float_tensor); + inputs.push_back(&input_arg_1); + inputs.push_back(&input_arg_2); + auto& output_arg = graph.GetOrCreateNodeArg("node_1_out_1", &float_tensor); + outputs.push_back(&output_arg); + graph.AddNode("node_1", "Add", "node 1.", inputs, outputs); + + auto& input_arg_3 = graph.GetOrCreateNodeArg("Z", &float_tensor); + inputs.clear(); + inputs.push_back(&output_arg); + inputs.push_back(&input_arg_3); + auto& output_arg_2 = graph.GetOrCreateNodeArg("M", &float_tensor); + outputs.clear(); + outputs.push_back(&output_arg_2); + graph.AddNode("node_2", "Add", "node 2.", inputs, outputs); + + auto status = graph.Resolve(); + ASSERT_TRUE(status.IsOK()); + std::string model_file_name = "execution_provider_test_graph.onnx"; + status = onnxruntime::Model::Save(model, model_file_name); + + SessionOptions so; + so.session_logid = "ExecutionProviderTest.FunctionTest"; + InferenceSession session_object{so}; + status = session_object.Load(model_file_name); + ASSERT_TRUE(status.IsOK()); + status = session_object.Initialize(); + ASSERT_TRUE(status.IsOK()); + + RunOptions run_options; + run_options.run_tag = so.session_logid; + + std::vector dims_mul_x = {3, 2}; + std::vector values_mul_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; + MLValue ml_value_x; + CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault), dims_mul_x, values_mul_x, &ml_value_x); + MLValue ml_value_y; + CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault), dims_mul_x, values_mul_x, &ml_value_y); + MLValue ml_value_z; + CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault), dims_mul_x, values_mul_x, &ml_value_z); + NameMLValMap feeds; + feeds.insert(std::make_pair("X", ml_value_x)); + feeds.insert(std::make_pair("Y", ml_value_y)); + feeds.insert(std::make_pair("Z", ml_value_z)); + + // prepare outputs + std::vector output_names; + output_names.push_back("M"); + std::vector fetches; + + // prepare expected inputs and outputs + std::vector expected_dims_mul_m = {3, 2}; + std::vector expected_values_mul_m = {3.0f, 6.0f, 9.0f, 12.0f, 15.0f, 18.0f}; + + // Now run + status = session_object.Run(run_options, feeds, output_names, &fetches); + ASSERT_TRUE(status.IsOK()); + VerifyOutputs(fetches, expected_dims_mul_m, expected_values_mul_m); + + InferenceSession session_object_2{so}; + session_object_2.RegisterExecutionProvider(std::make_unique<::onnxruntime::FuseExecutionProvider>()); + status = session_object_2.Load(model_file_name); + ASSERT_TRUE(status.IsOK()); + status = session_object_2.Initialize(); + ASSERT_TRUE(status.IsOK()); + status = session_object_2.Run(run_options, feeds, output_names, &fetches); + ASSERT_TRUE(status.IsOK()); + VerifyOutputs(fetches, expected_dims_mul_m, expected_values_mul_m); +} + +TEST(ExecutionProviderTest, FunctionInlineTest) { + onnxruntime::Model model("graph_1"); + + ONNX_NAMESPACE::FunctionProto fc_proto; + fc_proto.set_name("FC"); + fc_proto.set_doc_string("this is a full connection function."); + fc_proto.set_since_version(7); + fc_proto.add_input("w"); + fc_proto.add_input("x"); + fc_proto.add_input("b"); + fc_proto.add_output("y"); + NodeProto* node0 = fc_proto.add_node(); + node0->set_name("node0"); + node0->set_domain(""); + node0->set_doc_string("This is a matmul testing node "); + node0->set_op_type("MatMul"); + node0->add_input("w"); + node0->add_input("x"); + node0->add_output("y_1"); + NodeProto* node1 = fc_proto.add_node(); + node1->set_name("node1"); + node1->set_domain(""); + node1->set_doc_string("This is a add testing node "); + node1->set_op_type("Add"); + node1->add_input("y_1"); + node1->add_input("b"); + node1->add_output("y"); + model.AddFunction(fc_proto); + + auto& graph = model.MainGraph(); + std::vector inputs; + std::vector outputs; + + // FLOAT tensor. + ONNX_NAMESPACE::TypeProto float_tensor; + float_tensor.mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_FLOAT); + float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(2); + float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(2); + + auto& input_arg_1 = graph.GetOrCreateNodeArg("X", &float_tensor); + auto& input_arg_2 = graph.GetOrCreateNodeArg("Y", &float_tensor); + auto& input_arg_3 = graph.GetOrCreateNodeArg("Z", &float_tensor); + inputs.push_back(&input_arg_1); + inputs.push_back(&input_arg_2); + inputs.push_back(&input_arg_3); + auto& output_arg = graph.GetOrCreateNodeArg("M", &float_tensor); + outputs.push_back(&output_arg); + graph.AddNode("node_1", "FC", "node 1.", inputs, outputs); + + auto status = graph.Resolve(); + ASSERT_TRUE(status.IsOK()); + std::string model_file_name = "inline_test_graph.onnx"; + status = onnxruntime::Model::Save(model, model_file_name); + + SessionOptions so; + so.session_logid = "ExecutionProviderTest.FunctionInlineTest"; + InferenceSession session_object{so}; + status = session_object.Load(model_file_name); + ASSERT_TRUE(status.IsOK()); + status = session_object.Initialize(); + ASSERT_TRUE(status.IsOK()); + + RunOptions run_options; + run_options.run_tag = so.session_logid; + + std::vector dims_mul_x = {2, 2}; + std::vector values_mul_x = {1.0f, 2.0f, 3.0f, 4.0f}; + MLValue ml_value_x; + CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault), dims_mul_x, values_mul_x, &ml_value_x); + MLValue ml_value_y; + CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault), dims_mul_x, values_mul_x, &ml_value_y); + MLValue ml_value_z; + CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault), dims_mul_x, values_mul_x, &ml_value_z); + NameMLValMap feeds; + feeds.insert(std::make_pair("X", ml_value_x)); + feeds.insert(std::make_pair("Y", ml_value_y)); + feeds.insert(std::make_pair("Z", ml_value_z)); + + // prepare outputs + std::vector output_names; + output_names.push_back("M"); + std::vector fetches; + + // prepare expected inputs and outputs + std::vector expected_dims_mul_m = {2, 2}; + std::vector expected_values_mul_m = {8.0f, 12.0f, 18.0f, 26.0f}; + + // Now run + status = session_object.Run(run_options, feeds, output_names, &fetches); + ASSERT_TRUE(status.IsOK()); + VerifyOutputs(fetches, expected_dims_mul_m, expected_values_mul_m); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/insert_cast_transformer_test.cc b/onnxruntime/test/framework/insert_cast_transformer_test.cc new file mode 100644 index 0000000000000..89532a9cf9b68 --- /dev/null +++ b/onnxruntime/test/framework/insert_cast_transformer_test.cc @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/allocatormgr.h" +#include "core/framework/allocator.h" +#include "core/framework/insert_cast_transformer.h" +#include "core/graph/model.h" +#include "gtest/gtest.h" +#include "test_utils.h" + +using namespace ONNX_NAMESPACE; +namespace onnxruntime { +namespace test { +typedef std::vector ArgMap; +TEST(TransformerTest, InsertCastGPUTest) { + auto model = std::make_shared("test"); + onnxruntime::Graph& graph = model->MainGraph(); + + TypeProto tensor_float_16; + tensor_float_16.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT16); + onnxruntime::NodeArg i1_def("I1", &tensor_float_16), + i2_def("I2", &tensor_float_16), + i3_def("I3", &tensor_float_16), + o1_def("O1", &tensor_float_16), + o2_def("O2", &tensor_float_16), + o3_def("O3", &tensor_float_16); + + auto node1 = graph.AddNode("node1", "MatMul", "cpu operator1", ArgMap{&i1_def, &i2_def}, ArgMap{&o1_def}); + auto node2 = graph.AddNode("node2", "MatMul", "gpu operator1", ArgMap{&o1_def, &i3_def}, ArgMap{&o2_def}); + node2->SetExecutionProviderType(onnxruntime::kCudaExecutionProvider); + auto node3 = graph.AddNode("node3", "Clip", "cpu operator2", ArgMap{&o2_def}, ArgMap{&o3_def}); + + auto status = graph.Resolve(); + ASSERT_TRUE(status.IsOK()) << status.ErrorMessage(); + auto cpu_execution_provider = TestCPUExecutionProvider(); + InsertCastTransformer transformer("Test"); + transformer.AddKernelRegistry(*cpu_execution_provider->GetKernelRegistry().get()); + +#ifdef USE_CUDA + auto cuda_execution_provider = TestCudaExecutionProvider(); + transformer.AddKernelRegistry(*cuda_execution_provider->GetKernelRegistry().get()); +#endif + + bool modified = true; + status = transformer.Apply(graph, modified); + EXPECT_TRUE(status.IsOK()); + status = graph.Resolve(); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + EXPECT_EQ(graph.NumberOfNodes(), 8); + for (auto it = node1->InputNodesBegin(); it != node1->InputNodesEnd(); ++it) { + EXPECT_EQ((*it)->OpType(), "Cast"); + } + for (auto it = node1->OutputNodesBegin(); it != node1->OutputNodesEnd(); ++it) { + EXPECT_EQ((*it)->OpType(), "Cast"); + } + for (auto it = node2->InputNodesBegin(); it != node2->InputNodesEnd(); ++it) { + EXPECT_EQ((*it)->OpType(), "Cast"); + } + for (auto it = node2->OutputNodesBegin(); it != node2->OutputNodesEnd(); ++it) { + EXPECT_EQ((*it)->OpType(), "Cast"); + } + for (auto it = node3->InputNodesBegin(); it != node3->InputNodesEnd(); ++it) { + EXPECT_EQ((*it)->OpType(), "Cast"); + } + for (auto it = node3->OutputNodesBegin(); it != node3->OutputNodesEnd(); ++it) { + EXPECT_EQ((*it)->OpType(), "Cast"); + } +} + +TEST(TransformerTest, InsertCastAllCPUTest) { + auto model = std::make_shared("test"); + onnxruntime::Graph& graph = model->MainGraph(); + + TypeProto tensor_float_16; + tensor_float_16.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT16); + onnxruntime::NodeArg i1_def("I1", &tensor_float_16), + i2_def("I2", &tensor_float_16), + i3_def("I3", &tensor_float_16), + o1_def("O1", &tensor_float_16), + o2_def("O2", &tensor_float_16), + o3_def("O3", &tensor_float_16); + + auto node1 = graph.AddNode("node1", "MatMul", "cpu operator1", ArgMap{&i1_def, &i2_def}, ArgMap{&o1_def}); + auto node2 = graph.AddNode("node2", "MatMul", "gpu operator1", ArgMap{&o1_def, &i3_def}, ArgMap{&o2_def}); + auto node3 = graph.AddNode("node3", "Clip", "cpu operator2", ArgMap{&o2_def}, ArgMap{&o3_def}); + + auto status = graph.Resolve(); + ASSERT_TRUE(status.IsOK()) << status.ErrorMessage(); + + auto cpu_execution_provider = TestCPUExecutionProvider(); + InsertCastTransformer transformer("Test"); + transformer.AddKernelRegistry(*cpu_execution_provider->GetKernelRegistry().get()); + +#ifdef USE_CUDA + auto cuda_execution_provider = TestCudaExecutionProvider(); + transformer.AddKernelRegistry(*cuda_execution_provider->GetKernelRegistry().get()); +#endif + + bool modified = true; + EXPECT_TRUE(transformer.Apply(graph, modified).IsOK()); + status = graph.Resolve(); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + EXPECT_EQ(graph.NumberOfNodes(), 7); + for (auto it = node1->InputNodesBegin(); it != node1->InputNodesEnd(); ++it) { + EXPECT_EQ((*it)->OpType(), "Cast"); + } + for (auto it = node1->OutputNodesBegin(); it != node1->OutputNodesEnd(); ++it) { + EXPECT_NE((*it)->OpType(), "Cast"); + } + for (auto it = node2->OutputNodesBegin(); it != node2->OutputNodesEnd(); ++it) { + EXPECT_NE((*it)->OpType(), "Cast"); + } + for (auto it = node3->InputNodesBegin(); it != node3->InputNodesEnd(); ++it) { + EXPECT_NE((*it)->OpType(), "Cast"); + } + for (auto it = node3->OutputNodesBegin(); it != node3->OutputNodesEnd(); ++it) { + EXPECT_EQ((*it)->OpType(), "Cast"); + } +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/local_kernel_registry_test.cc b/onnxruntime/test/framework/local_kernel_registry_test.cc new file mode 100644 index 0000000000000..300d4bb266a5b --- /dev/null +++ b/onnxruntime/test/framework/local_kernel_registry_test.cc @@ -0,0 +1,329 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/inference_session.h" + +#include +#include +#include +#include + +#include "core/common/logging/logging.h" +#include "core/framework/execution_provider.h" +#include "core/framework/op_kernel.h" +#include "core/framework/session_state.h" +#include "core/graph/graph.h" +#include "core/graph/model.h" +#include "core/graph/op.h" +#include "core/providers/cpu/cpu_execution_provider.h" +#include "core/providers/cpu/math/element_wise_ops.h" +#include "core/framework/tensorprotoutils.h" +#include "test/capturing_sink.h" +#include "test/test_environment.h" +#include "test_utils.h" +#include "gtest/gtest.h" +#include "core/graph/schema_registry.h" +#include "core/framework/customregistry.h" +using namespace ONNX_NAMESPACE; +using namespace onnxruntime::common; + +namespace onnxruntime { +namespace test { + +// Foo kernel which is doing Add +template +class FooKernel : public OpKernel { + public: + FooKernel(const OpKernelInfo& info) : OpKernel(info) {} + + Status Compute(OpKernelContext* context) const { + const auto* X = context->Input(0); + const auto* W = context->Input(1); + + auto X_Data = X->Data(); + auto W_Data = W->Data(); + + auto shape = X->Shape().GetDims(); + + auto* Y = context->Output(0, shape); + auto* Y_Data = Y->MutableData(); + + size_t size = 1; + for (size_t i = 0; i < shape.size(); i++) { + size *= shape[i]; + } + + for (size_t i = 0; i < size; i++) { + Y_Data[i] = X_Data[i] + W_Data[i]; + } + + return Status::OK(); + } +}; + +ONNX_NAMESPACE::OpSchema GetFooSchema() { + ONNX_NAMESPACE::OpSchema schema("Foo", "unknown", 0); + schema.Input(0, + "A", + "First operand, should share the type with the second operand.", + "T"); + schema.Input( + 1, + "B", + "Second operand. With broadcasting can be of smaller size than A. " + "If broadcasting is disabled it should be of the same size.", + "T"); + schema.Output(0, "C", "Result, has same dimensions and type as A", "T"); + schema.TypeConstraint( + "T", + OpSchema::numeric_types_for_math_reduction(), + "Constrain input and output types to high-precision numeric tensors."); + schema.SinceVersion(7); + return schema; +} + +//For test purpose, we register this Foo kernel to Mul op. +//Once the custom schema is ready, should update this. +KernelDefBuilder FooKernelDef(const char* schema_name) { + KernelDefBuilder def; + def.SetName(schema_name) + .SetDomain(onnxruntime::kOnnxDomain) + .SinceVersion(7) + .Provider(onnxruntime::kCpuExecutionProvider) + .TypeConstraint("T", DataTypeImpl::GetTensorType()); + return def; +} + +OpKernel* CreateFooKernel(const OpKernelInfo& kernel_info) { + return new FooKernel(kernel_info); +} + +// kernel with optional outputs +KernelDefBuilder OptionalKernelDef() { + KernelDefBuilder def; + def.SetName("OptionalOp") + .SetDomain(onnxruntime::kOnnxDomain) + .SinceVersion(6) + .Provider(onnxruntime::kCpuExecutionProvider) + .TypeConstraint("T", DataTypeImpl::GetTensorType()); + return def; +} + +ONNX_NAMESPACE::OpSchema GetOptionalOpSchema() { + ONNX_NAMESPACE::OpSchema schema("OptionalOp", "unknown", 0); + schema.Input(0, + "X", + "First operand, should share the type with the second operand.", + "T"); + schema.Input( + 1, + "W", + "Second operand. If provided, add it to the output", + "T", + OpSchema::Optional); + schema.Output(0, "Y", "Result, has same dimensions and type as A", "T"); + schema.Output(1, "Y2", "Result, has same dimensions and type as A", "T", OpSchema::Optional); + schema.TypeConstraint( + "T", + OpSchema::numeric_types_for_math_reduction(), + "Constrain input and output types to high-precision numeric tensors."); + schema.SinceVersion(6); + return schema; +} + +template +class OptionalOpKernel : public OpKernel { + public: + OptionalOpKernel(const OpKernelInfo& info) : OpKernel(info) {} + + Status Compute(OpKernelContext* context) const { + const auto* X = context->Input(0); + const auto* W = context->Input(1); + + auto* X_Data = X->Data(); + auto& shape = X->Shape().GetDims(); + auto* Y = context->Output(0, shape); + auto* Y_Data = Y->MutableData(); + size_t size = 1; + for (size_t i = 0; i < shape.size(); i++) { + size *= shape[i]; + } + + for (size_t i = 0; i < size; i++) { + Y_Data[i] = X_Data[i]; + } + + auto* Y2 = context->Output(1, shape); + // Y2 is used or not + if (Y2) { + auto Y2_Data = Y2->MutableData(); + for (size_t i = 0; i < size; i++) { + Y2_Data[i] = X_Data[i]; + } + } + + //W is used or not + if (W) { + auto* W_Data = W->Data(); + for (size_t i = 0; i < size; i++) { + Y_Data[i] += W_Data[i]; + } + if (Y2) { + auto* Y2_Data = Y2->MutableData(); + for (size_t i = 0; i < size; i++) { + Y2_Data[i] += W_Data[i]; + } + } + } + + return Status::OK(); + } +}; + +OpKernel* CreateOptionalOpKernel(const OpKernelInfo& kernel_info) { + return new OptionalOpKernel(kernel_info); +} + +static const std::string MUL_MODEL_URI = "testdata/mul_1.pb"; +static const std::string FOO_MODEL_URI = "testdata/foo_1.pb"; +static const std::string FOO_TRUNCATE_MODEL_URI = "testdata/foo_2.pb"; + +static const std::string OPTIONAL_MODEL1_URI = "testdata/optional_1.pb"; + +void RunSession(InferenceSession& session_object, + RunOptions& run_options, + std::vector& dims_x, + std::vector& values_x, + std::vector& dims_y, + std::vector& values_y) { + // prepare inputs + MLValue ml_value; + CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault), dims_x, values_x, &ml_value); + NameMLValMap feeds; + feeds.insert(std::make_pair("X", ml_value)); + + // prepare outputs + std::vector output_names; + output_names.push_back("Y"); + std::vector fetches; + + // Now run + common::Status st = session_object.Run(run_options, feeds, output_names, &fetches); + std::cout << "Run returned status: " << st.ErrorMessage() << std::endl; + EXPECT_TRUE(st.IsOK()); + ASSERT_EQ(1, fetches.size()); + auto& rtensor = fetches.front().Get(); + TensorShape expected_shape(dims_y); + EXPECT_EQ(expected_shape, rtensor.Shape()); + const std::vector found(rtensor.template Data(), rtensor.template Data() + expected_shape.Size()); + ASSERT_EQ(values_y, found); +} + +TEST(CustomKernelTests, CustomKernelWithBuildInSchema) { + SessionOptions so; + + so.session_logid = "InferenceSessionTests.NoTimeout"; + + // Register a foo kernel which is doing Add, but bind to Mul. + std::shared_ptr registry = std::make_shared(); + + InferenceSession session_object{so, &DefaultLoggingManager()}; + EXPECT_TRUE(session_object.RegisterCustomRegistry(registry).IsOK()); + auto def = FooKernelDef("Mul"); + + EXPECT_TRUE(registry->RegisterCustomKernel(def, CreateFooKernel).IsOK()); + + EXPECT_TRUE(session_object.Load(MUL_MODEL_URI).IsOK()); + EXPECT_TRUE(session_object.Initialize().IsOK()); + + RunOptions run_options; + run_options.run_tag = "one session/one tag"; + + // prepare inputs + std::vector dims_x = {3, 2}; + std::vector values_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; + + // prepare expected inputs and outputs + std::vector expected_dims_y = {3, 2}; + // now the expected value should be Add's result. + std::vector expected_values_y = {2.0f, 4.0f, 6.0f, 8.0f, 10.0f, 12.0f}; + + // Now run + RunSession(session_object, run_options, dims_x, values_x, expected_dims_y, expected_values_y); +} + +TEST(CustomKernelTests, CustomKernelWithCustomSchema) { + SessionOptions so; + + so.session_logid = "InferenceSessionTests.NoTimeout"; + + std::shared_ptr registry = std::make_shared(); + + InferenceSession session_object{so, &DefaultLoggingManager()}; + EXPECT_TRUE(session_object.RegisterCustomRegistry(registry).IsOK()); + + //register foo schema + auto foo_schema = GetFooSchema(); + std::vector schemas = {foo_schema}; + EXPECT_TRUE(registry->RegisterOpSet(schemas, onnxruntime::kOnnxDomain, 5, 7).IsOK()); + auto def = FooKernelDef("Foo"); + //Register a foo kernel which is doing Add, but bind to Mul. + EXPECT_TRUE(registry->RegisterCustomKernel(def, CreateFooKernel).IsOK()); + + EXPECT_TRUE(session_object.Load(FOO_MODEL_URI).IsOK()); + EXPECT_TRUE(session_object.Initialize().IsOK()); + + RunOptions run_options; + run_options.run_tag = "one session/one tag"; + + // prepare inputs + std::vector dims_x = {3, 2}; + std::vector values_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; + + // prepare expected inputs and outputs + std::vector expected_dims_y = {3, 2}; + // now the expected value should be Add's result. + std::vector expected_values_y = {2.0f, 4.0f, 6.0f, 8.0f, 10.0f, 12.0f}; + + // Now run + RunSession(session_object, run_options, dims_x, values_x, expected_dims_y, expected_values_y); +} + +TEST(CustomKernelTests, CustomKernelWithOptionalOutput) { + SessionOptions so; + + so.session_logid = "InferenceSessionTests.NoTimeout"; + + //reigster optional schema + auto optional_schema = GetOptionalOpSchema(); + std::vector schemas = {optional_schema}; + + std::shared_ptr registry = std::make_shared(); + + EXPECT_TRUE(registry->RegisterOpSet(schemas, onnxruntime::kOnnxDomain, 5, 7).IsOK()); + auto def = OptionalKernelDef(); + //Register a foo kernel which is doing Add, but bind to Mul. + EXPECT_TRUE(registry->RegisterCustomKernel(def, CreateOptionalOpKernel).IsOK()); + + InferenceSession session_object{so, &DefaultLoggingManager()}; + EXPECT_TRUE(session_object.RegisterCustomRegistry(registry).IsOK()); + EXPECT_TRUE(session_object.Load(OPTIONAL_MODEL1_URI).IsOK()); + EXPECT_TRUE(session_object.Initialize().IsOK()); + + RunOptions run_options; + run_options.run_tag = "one session/one tag"; + + // prepare inputs + std::vector dims_x = {3, 2}; + std::vector values_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; + + // prepare expected inputs and outputs + std::vector expected_dims_y = {3, 2}; + // now the expected value should be equal result. + std::vector expected_values_y = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; + + // Now run + RunSession(session_object, run_options, dims_x, values_x, expected_dims_y, expected_values_y); +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/math_test.cc b/onnxruntime/test/framework/math_test.cc new file mode 100644 index 0000000000000..617e1fb0ab7f8 --- /dev/null +++ b/onnxruntime/test/framework/math_test.cc @@ -0,0 +1,187 @@ +/** +* Copyright (c) 2016-present, Facebook, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +// Modifications Copyright (c) Microsoft. + +#include "core/util/math.h" +#include +#include "core/util/math_cpuonly.h" +namespace onnxruntime { + +#define VECTOR_HEAD(x) x.size() > 0 ? &x[0] : NULL + +TEST(MathTest, GemmNoTransNoTrans) { + auto& provider = CPUMathUtil::Instance(); + std::vector X(50); // 5 * 10 + std::vector W(60); // 10 * 6 + std::vector Y(30); // 5 * 6 + math::Set(X.size(), 1, VECTOR_HEAD(X), &provider); + math::Set(W.size(), 1, VECTOR_HEAD(W), &provider); + EXPECT_EQ(Y.size(), 30); + for (int i = 0; i < X.size(); ++i) { + EXPECT_EQ(X[i], 1); + } + for (int i = 0; i < W.size(); ++i) { + EXPECT_EQ(W[i], 1); + } + + const float kOne = 1.0; + const float kPointFive = 0.5; + const float kZero = 0.0; + math::Gemm(CblasNoTrans, CblasNoTrans, 5, 6, 10, kOne, + VECTOR_HEAD(X), VECTOR_HEAD(W), kZero, VECTOR_HEAD(Y), + &provider); + EXPECT_EQ(Y.size(), 30); + for (int i = 0; i < Y.size(); ++i) { + EXPECT_EQ(Y[i], 10) << i; + } + // Test Accumulate + math::Gemm(CblasNoTrans, CblasNoTrans, 5, 6, 10, kOne, + VECTOR_HEAD(X), VECTOR_HEAD(W), kPointFive, + VECTOR_HEAD(Y), &provider); + EXPECT_EQ(Y.size(), 30); + for (int i = 0; i < Y.size(); ++i) { + EXPECT_EQ(Y[i], 15) << i; + } + // Test Accumulate + math::Gemm(CblasNoTrans, CblasNoTrans, 5, 6, 10, + kPointFive, + VECTOR_HEAD(X), VECTOR_HEAD(W), kOne, VECTOR_HEAD(Y), + &provider); + EXPECT_EQ(Y.size(), 30); + for (int i = 0; i < Y.size(); ++i) { + EXPECT_EQ(Y[i], 20) << i; + } +} + +TEST(MathTest, GemmNoTransTrans) { + auto& provider = CPUMathUtil::Instance(); + std::vector X(50); // 5 * 10 + std::vector W(60); // 10 * 6 + std::vector Y(30); // 5 * 6 + math::Set(X.size(), 1, VECTOR_HEAD(X), &provider); + math::Set(W.size(), 1, VECTOR_HEAD(W), &provider); + EXPECT_EQ(Y.size(), 30); + for (int i = 0; i < X.size(); ++i) { + EXPECT_EQ(X[i], 1); + } + for (int i = 0; i < W.size(); ++i) { + EXPECT_EQ(W[i], 1); + } + + const float kOne = 1.0; + const float kPointFive = 0.5; + const float kZero = 0.0; + math::Gemm(CblasNoTrans, CblasTrans, 5, 6, 10, kOne, + VECTOR_HEAD(X), VECTOR_HEAD(W), kZero, VECTOR_HEAD(Y), + &provider); + EXPECT_EQ(Y.size(), 30); + for (int i = 0; i < Y.size(); ++i) { + EXPECT_EQ(Y[i], 10) << i; + } + // Test Accumulate + math::Gemm(CblasNoTrans, CblasTrans, 5, 6, 10, kOne, + VECTOR_HEAD(X), VECTOR_HEAD(W), kPointFive, + VECTOR_HEAD(Y), &provider); + EXPECT_EQ(Y.size(), 30); + for (int i = 0; i < Y.size(); ++i) { + EXPECT_EQ(Y[i], 15) << i; + } + math::Gemm(CblasNoTrans, CblasTrans, 5, 6, 10, kPointFive, + VECTOR_HEAD(X), VECTOR_HEAD(W), kOne, VECTOR_HEAD(Y), + &provider); + EXPECT_EQ(Y.size(), 30); + for (int i = 0; i < Y.size(); ++i) { + EXPECT_EQ(Y[i], 20) << i; + } +} + +TEST(MathTest, GemvNoTrans) { + auto& provider = CPUMathUtil::Instance(); + std::vector A(50); // 5 * 10 + std::vector X(10); + std::vector Y(5); + math::Set(A.size(), 1, VECTOR_HEAD(A), &provider); + math::Set(X.size(), 1, VECTOR_HEAD(X), &provider); + EXPECT_EQ(Y.size(), 5); + for (int i = 0; i < A.size(); ++i) { + EXPECT_EQ(A[i], 1); + } + for (int i = 0; i < X.size(); ++i) { + EXPECT_EQ(X[i], 1); + } + + const float kOne = 1.0; + const float kPointFive = 0.5; + const float kZero = 0.0; + math::Gemv(CblasNoTrans, 5, 10, kOne, VECTOR_HEAD(A), VECTOR_HEAD(X), + kZero, VECTOR_HEAD(Y), &provider); + for (int i = 0; i < Y.size(); ++i) { + EXPECT_EQ(Y[i], 10) << i; + } + // Test Accumulate + math::Gemv(CblasNoTrans, 5, 10, kOne, VECTOR_HEAD(A), VECTOR_HEAD(X), + kPointFive, VECTOR_HEAD(Y), &provider); + for (int i = 0; i < Y.size(); ++i) { + EXPECT_EQ(Y[i], 15) << i; + } + // Test Accumulate + math::Gemv(CblasNoTrans, 5, 10, kPointFive, VECTOR_HEAD(A), + VECTOR_HEAD(X), kOne, VECTOR_HEAD(Y), + &provider); + for (int i = 0; i < Y.size(); ++i) { + EXPECT_EQ(Y[i], 20) << i; + } +} + +TEST(MathTest, GemvTrans) { + auto& provider = CPUMathUtil::Instance(); + std::vector A(60); // 6 * 10 + std::vector X(6); + std::vector Y(10); + math::Set(A.size(), 1, VECTOR_HEAD(A), &provider); + math::Set(X.size(), 1, VECTOR_HEAD(X), &provider); + EXPECT_EQ(Y.size(), 10); + for (int i = 0; i < A.size(); ++i) { + EXPECT_EQ(A[i], 1); + } + for (int i = 0; i < X.size(); ++i) { + EXPECT_EQ(X[i], 1); + } + + const float kOne = 1.0; + const float kPointFive = 0.5; + const float kZero = 0.0; + math::Gemv(CblasTrans, 6, 10, kOne, VECTOR_HEAD(A), VECTOR_HEAD(X), + kZero, VECTOR_HEAD(Y), &provider); + for (int i = 0; i < Y.size(); ++i) { + EXPECT_EQ(Y[i], 6) << i; + } + // Test Accumulate + math::Gemv(CblasTrans, 6, 10, kOne, VECTOR_HEAD(A), VECTOR_HEAD(X), + kPointFive, VECTOR_HEAD(Y), &provider); + for (int i = 0; i < Y.size(); ++i) { + EXPECT_EQ(Y[i], 9) << i; + } + // Test Accumulate + math::Gemv(CblasTrans, 6, 10, kPointFive, VECTOR_HEAD(A), + VECTOR_HEAD(X), kOne, VECTOR_HEAD(Y), + &provider); + for (int i = 0; i < Y.size(); ++i) { + EXPECT_EQ(Y[i], 12) << i; + } +} + +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/mem_pattern_planner_test.cc b/onnxruntime/test/framework/mem_pattern_planner_test.cc new file mode 100644 index 0000000000000..eb905566f2327 --- /dev/null +++ b/onnxruntime/test/framework/mem_pattern_planner_test.cc @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/mem_pattern_planner.h" +#include "gtest/gtest.h" + +namespace onnxruntime { +namespace test { +TEST(MemPatternPlannerTest, TraceAllocaitonTest) { + MemPatternPlanner planner; + planner.TraceAllocation(0, 1024); + planner.TraceAllocation(1, 256); + planner.TraceAllocation(2, 512); + planner.TraceAllocation(3, 1024); + + auto pattern = planner.GenerateMemPattern(); + + EXPECT_EQ(pattern.PeakSize(), 1024 + 256 + 512 + 1024); + EXPECT_EQ(pattern.GetBlock(0)->offset_, 0); + EXPECT_EQ(pattern.GetBlock(1)->offset_, 1024); + EXPECT_EQ(pattern.GetBlock(2)->offset_, 1024 + 256); + EXPECT_EQ(pattern.GetBlock(3)->offset_, 1024 + 256 + 512); + + planner.TraceFree(1); + planner.TraceAllocation(4, 512); + planner.TraceFree(3); + planner.TraceAllocation(5, 600); + planner.TraceAllocation(6, 200); + + pattern = planner.GenerateMemPattern(); + + EXPECT_EQ(pattern.PeakSize(), 1024 + 256 + 512 + 1024 + 512); + EXPECT_EQ(pattern.GetBlock(4)->offset_, 1024 + 256 + 512 + 1024); + EXPECT_EQ(pattern.GetBlock(5)->offset_, 1024 + 256 + 512); + EXPECT_EQ(pattern.GetBlock(6)->offset_, 1024); +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/memcpy_transformer_test.cc b/onnxruntime/test/framework/memcpy_transformer_test.cc new file mode 100644 index 0000000000000..8e5b6f83cc9ba --- /dev/null +++ b/onnxruntime/test/framework/memcpy_transformer_test.cc @@ -0,0 +1,165 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include "core/framework/transformer_memcpy.h" +#include "core/graph/model.h" +#include "gtest/gtest.h" +#include "test_utils.h" +using namespace ONNX_NAMESPACE; +namespace onnxruntime { +namespace test { + +typedef std::vector ArgMap; + +void ExpectSame(const onnxruntime::Node* source, const onnxruntime::Node* target, int argnum) { + // Check that target's argnum-th input comes from the source node (without copy): + auto* source_output = source->OutputDefs()[0]; + auto* target_input = target->InputDefs()[argnum]; + EXPECT_EQ(source_output, target_input); +} + +void ExpectCopy(const onnxruntime::Node* source, const std::string copy_op, + const onnxruntime::Node* target, int argnum) { + // Check that source's output is consumed by a copy_op; + for (auto it = source->OutputNodesBegin(); it != source->OutputNodesEnd(); ++it) { + auto* copy_node = *it; + if (copy_node->OpType() == copy_op) { + // Check that target's argnum-th input comes from the copy node: + auto* copy_output = copy_node->OutputDefs()[0]; + auto* target_input = target->InputDefs()[argnum]; + EXPECT_EQ(copy_output, target_input); + return; + } + } + EXPECT_TRUE(false) << "Copy node expected but not found"; +} + +void ExpectCopy(const onnxruntime::NodeArg* source_arg, const std::string copy_op, + const onnxruntime::Node* target, int argnum) { + auto* target_input = target->InputDefs()[argnum]; + for (auto it = target->InputNodesBegin(); it != target->InputNodesEnd(); ++it) { + auto* copy_node = *it; + // Check if target's argnum-th input comes from this node: + auto* copy_output = copy_node->OutputDefs()[0]; + if (copy_output == target_input) { + EXPECT_EQ(copy_node->OpType(), copy_op); + auto* copy_input = copy_node->InputDefs()[0]; + EXPECT_EQ(copy_input, source_arg); + return; + } + } + EXPECT_TRUE(false) << "Copy node expected but not found"; +} + +void ExpectCopy(const onnxruntime::Node* source, const std::string copy_op, + const onnxruntime::NodeArg* target_arg) { + // Check that source's output is consumed by a copy_op; + for (auto it = source->OutputNodesBegin(); it != source->OutputNodesEnd(); ++it) { + auto* copy_node = *it; + if (copy_node->OpType() == copy_op) { + auto* copy_output = copy_node->OutputDefs()[0]; + EXPECT_EQ(copy_output, target_arg); + return; + } + } + EXPECT_TRUE(false) << "Copy node expected but not found"; +} + +TEST(TransformerTest, MemcpyTransformerTest) { + auto model = std::make_shared("test"); + onnxruntime::Graph& graph = model->MainGraph(); + + TypeProto tensor_float_type; + tensor_float_type.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + onnxruntime::NodeArg i1_def("I1", &tensor_float_type), + i2_def("I2", &tensor_float_type), + i3_def("I3", &tensor_float_type), + o1_def("O1", &tensor_float_type), + o2_def("O2", &tensor_float_type), + o3_def("O3", &tensor_float_type), + o4_def("O4", &tensor_float_type); + + auto node1 = graph.AddNode("node1", "MatMul", "cpu operator1", ArgMap{&i1_def, &i2_def}, ArgMap{&o1_def}); + node1->SetExecutionProviderType(onnxruntime::kCpuExecutionProvider); + auto node2 = graph.AddNode("node2", "MatMul", "gpu operator1", ArgMap{&o1_def, &i3_def}, ArgMap{&o2_def}); + node2->SetExecutionProviderType(onnxruntime::kCudaExecutionProvider); + auto node3 = graph.AddNode("node3", "Clip", "cpu operator2", ArgMap{&o2_def}, ArgMap{&o3_def}); + node3->SetExecutionProviderType(onnxruntime::kCpuExecutionProvider); + auto node4 = graph.AddNode("node4", "MatMul", "gpu operator2", ArgMap{&o2_def, &o2_def}, ArgMap{&o4_def}); + node4->SetExecutionProviderType(onnxruntime::kCudaExecutionProvider); + + auto status = graph.Resolve(); + ASSERT_TRUE(status.IsOK()) << status.ErrorMessage(); + + auto cpu_execution_provider = TestCPUExecutionProvider(); + KernelRegistryManager test_registry_manager; + test_registry_manager.RegisterKernelRegistry(cpu_execution_provider->GetKernelRegistry(), KernelRegistryPriority::LowPriority); + + TransformerMemcpyImpl transformer(graph, onnxruntime::kCudaExecutionProvider); + + bool modified = transformer.ModifyGraph(test_registry_manager); + EXPECT_TRUE(modified); + + status = graph.Resolve(); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + + // Expect: copy of O1 from cpu to gpu + ExpectCopy(node1, "MemcpyFromHost", node2, 0); + + // Expect: copy O2 from gpu to cpu + ExpectCopy(node2, "MemcpyToHost", node3, 0); + ExpectSame(node2, node4, 0); + ExpectSame(node2, node4, 1); +} + +TEST(TransformerTest, MemcpyTransformerTestCudaFirst) { + auto model = std::make_shared("test"); + onnxruntime::Graph& graph = model->MainGraph(); + + TypeProto tensor_float_type; + tensor_float_type.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + onnxruntime::NodeArg i1_def("I1", &tensor_float_type), + i2_def("I2", &tensor_float_type), + i3_def("I3", &tensor_float_type), + o1_def("O1", &tensor_float_type), + o2_def("O2", &tensor_float_type), + o3_def("O3", &tensor_float_type), + o4_def("O4", &tensor_float_type); + + auto node1 = graph.AddNode("node1", "MatMul", "gpu operator1", ArgMap{&i1_def, &i2_def}, ArgMap{&o1_def}); + node1->SetExecutionProviderType(onnxruntime::kCudaExecutionProvider); + auto node2 = graph.AddNode("node2", "MatMul", "cpu operator1", ArgMap{&o1_def, &i3_def}, ArgMap{&o2_def}); + node2->SetExecutionProviderType(onnxruntime::kCpuExecutionProvider); + auto node3 = graph.AddNode("node3", "Clip", "gpu operator2", ArgMap{&o2_def}, ArgMap{&o3_def}); + node3->SetExecutionProviderType(onnxruntime::kCudaExecutionProvider); + auto node4 = graph.AddNode("node4", "MatMul", "cpu operator2", ArgMap{&o2_def, &o2_def}, ArgMap{&o4_def}); + node4->SetExecutionProviderType(onnxruntime::kCpuExecutionProvider); + + auto status = graph.Resolve(); + ASSERT_TRUE(status.IsOK()) << status.ErrorMessage(); + + auto cpu_execution_provider = TestCPUExecutionProvider(); + KernelRegistryManager test_registry_manager; + test_registry_manager.RegisterKernelRegistry(cpu_execution_provider->GetKernelRegistry(), + KernelRegistryPriority::LowPriority); + + TransformerMemcpyImpl transformer(graph, onnxruntime::kCudaExecutionProvider); + + bool modified = transformer.ModifyGraph(test_registry_manager); + EXPECT_TRUE(modified); + + status = graph.Resolve(); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + + // Expect: copy of O1 from gpu to cpu + ExpectCopy(node1, "MemcpyToHost", node2, 0); + + // Expect: copy O2 from cpu to gpu + ExpectCopy(node2, "MemcpyFromHost", node3, 0); + ExpectSame(node2, node4, 0); + ExpectSame(node2, node4, 1); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/model_builder_utils.h b/onnxruntime/test/framework/model_builder_utils.h new file mode 100644 index 0000000000000..b959d5cb2b571 --- /dev/null +++ b/onnxruntime/test/framework/model_builder_utils.h @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/graph/model.h" + +namespace onnxruntime { +namespace test { + +namespace modelbuilder { + +// Shape: a wrapper to build a TensorShapeProto +struct Shape { + ONNX_NAMESPACE::TensorShapeProto value; + + // construct a shape with given constant dimensions + Shape(std::initializer_list dims) { + for (auto d : dims) { + auto dim = value.add_dim(); + dim->set_dim_value(d); + } + } + + // construct a shape with given symbolic dimensions + Shape(std::initializer_list dims) { + for (auto d : dims) { + auto dim = value.add_dim(); + dim->set_dim_param(d); + } + } +}; + +// Type: a wrapper to build a TypeProto +struct Type { + ONNX_NAMESPACE::TypeProto value; + + // construct a float-tensor-type + Type() { + value.mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_FLOAT); + } + + // construct a float-tensor-type with given constant dimensions + Type(std::initializer_list dims) { + value.mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_FLOAT); + auto p_shape = value.mutable_tensor_type()->mutable_shape(); + for (auto d : dims) { + auto dim = p_shape->add_dim(); + dim->set_dim_value(d); + } + } + + // construct a float-tensor-type with given symbolic dimensions + Type(std::initializer_list symbolic_dims) { + value.mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_FLOAT); + auto p_shape = value.mutable_tensor_type()->mutable_shape(); + for (auto d : symbolic_dims) { + auto dim = p_shape->add_dim(); + dim->set_dim_param(d); + } + } +}; + +} // namespace modelbuilder +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/op_kernel_test.cc b/onnxruntime/test/framework/op_kernel_test.cc new file mode 100644 index 0000000000000..b20933d3a3d1a --- /dev/null +++ b/onnxruntime/test/framework/op_kernel_test.cc @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/inference_session.h" +#include "core/framework/op_kernel.h" +#include "core/framework/session_state.h" +#include "core/graph/graph.h" +#include "core/graph/model.h" +#include "core/graph/op.h" +#include "core/providers/cpu/cpu_execution_provider.h" +#include "gtest/gtest.h" +#include "test_utils.h" +using namespace std; +using namespace ONNX_NAMESPACE; +using namespace ::onnxruntime::logging; + +namespace onnxruntime { +namespace test { + +class XPUExecutionProvider : public IExecutionProvider { + public: + XPUExecutionProvider() = default; + + std::string Type() const override { + return onnxruntime::kCpuExecutionProvider; + } + + Status CopyTensor(const Tensor& src, Tensor& dst) const override { + ONNXRUNTIME_UNUSED_PARAMETER(src); + ONNXRUNTIME_UNUSED_PARAMETER(dst); + return Status::OK(); + } + + virtual const void* GetExecutionHandle() const noexcept override { + // The XPU interface does not return anything interesting. + return nullptr; + } +}; + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/opaque_kernels_test.cc b/onnxruntime/test/framework/opaque_kernels_test.cc new file mode 100644 index 0000000000000..71808d5c7be08 --- /dev/null +++ b/onnxruntime/test/framework/opaque_kernels_test.cc @@ -0,0 +1,402 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include + +#include "core/framework/data_types.h" +#include "core/framework/execution_providers.h" +#include "core/framework/op_kernel.h" +#include "core/framework/session_state.h" +#include "core/graph/graph.h" +#include "core/graph/model.h" +#include "core/graph/op.h" +#include "core/providers/cpu/cpu_execution_provider.h" +#include "core/session/inference_session.h" +#include "gtest/gtest.h" + +#include "test/providers/provider_test_utils.h" +#include "test_utils.h" + +using namespace ONNX_NAMESPACE; +using namespace onnxruntime::common; + +const char* experimental_using_opaque = R"DOC( +The operator constructs an instance of sparse one dimensional tensor +represented by a SparseTensorSample type. It uses 3 supplied inputs each +in a form of a single dimensional tensor. +)DOC"; + +namespace onnxruntime { +// We will use this class to implement Sparse Tensor and +// register it as an Opaque type emulating some experimental type + +/** + * @brief This class implements a SparseTensor as an example + * of using custom experimental type outside of ONNXRuntime. + * + * @details The class captures the 3 necessary elements of a Sparse Tensor + * values - a vector of non-zero sparse tensor values + * indices - a vector of indices of non zero values + * shape - a scalar tensor that indicates the size of a single dimension + * It is assumed that all of the values for the tensors are int64 + * we use tensor datatypes as effective memory managers. + */ + +// This type is a result of the construct_sparse OpKernel. +class SparseTensorSample final { + public: + SparseTensorSample() = default; + ~SparseTensorSample() = default; + + SparseTensorSample(const SparseTensorSample&) = default; + SparseTensorSample& operator=(const SparseTensorSample&) = default; + SparseTensorSample(SparseTensorSample&&) = default; + SparseTensorSample& operator=(SparseTensorSample&&) = default; + + const auto& Values() const { + return values_; + } + + const auto& Indicies() const { + return indicies_; + } + + const auto& Size() const { + return size_; + } + + auto& Values() { + return values_; + } + + auto& Indicies() { + return indicies_; + } + + auto& Size() { + return size_; + } + + private: + std::vector values_; + std::vector indicies_; + int64_t size_; // The value of a single dimension +}; + +// We will then register this class as an Opaque type as if created and used by +// a 3rd party for experiments. +extern const char kTestDomain[] = "ai.onnx"; +extern const char kSparseTensorName[] = "SparseTensorSample"; + +ONNXRUNTIME_REGISTER_OPAQUE_TYPE(SparseTensorSample, kTestDomain, kSparseTensorName); + +class OpaqueTypeTests : public testing::Test { + public: + static void SetUpTestCase() { + MLDataType mltype = DataTypeImpl::GetType(); + DataTypeImpl::RegisterDataType(mltype); + } +}; + +/** + * @brief This class represents an operator kernel which takes as input 3 tensors + * + * The OpKernel takes 3 tensors as input named as follows: + * - sparse_values - Tensor + * - sparse_indicies - Tensor + * - sparse_shape - Tensor + * + * Output - TestSparseTensorType - Opaque type + */ +class ConstructSparseTensor final : public OpKernel { + public: + ConstructSparseTensor(const OpKernelInfo& info) : OpKernel{info} {} + + Status Compute(OpKernelContext* ctx) const override { + ONNXRUNTIME_ENFORCE(ctx->InputCount() == 3, "Expecting 3 inputs"); + + const Tensor& values_tensor = *ctx->Input(0); + const Tensor& indicies_tensor = *ctx->Input(1); + const Tensor& shape_tensor = *ctx->Input(2); + + // Shapes of values and indicies should be the same since they refer to the same + // values + const TensorShape& val_shape = values_tensor.Shape(); + const TensorShape& ind_shape = indicies_tensor.Shape(); + ONNXRUNTIME_ENFORCE(val_shape.NumDimensions() == 1, "Expecting vectors"); + ONNXRUNTIME_ENFORCE(val_shape.NumDimensions() == ind_shape.NumDimensions()); + + // Copy data. With some effort we could hold shallow copies of the input Tensors + // but I will leave this for now. + SparseTensorSample* output_sparse_tensor = ctx->Output(0); + ONNXRUNTIME_ENFORCE(output_sparse_tensor != nullptr); + output_sparse_tensor->Values().assign(values_tensor.Data(), + values_tensor.Data() + val_shape[0]); + output_sparse_tensor->Indicies().assign(indicies_tensor.Data(), + indicies_tensor.Data() + ind_shape[0]); + output_sparse_tensor->Size() = *shape_tensor.Data(); + + return Status::OK(); + } +}; + +/** + * @brief This class represents an operator kernel that fetches and returns + * sparse tensor shape from an Opaque type + * + * + * Output - Scalar Tensor + */ +class FetchSparseTensorShape final : public OpKernel { + public: + FetchSparseTensorShape(const OpKernelInfo& info) : OpKernel{info} {} + + Status Compute(OpKernelContext* ctx) const override { + ONNXRUNTIME_ENFORCE(ctx->InputCount() == 1, "Expecting a single SparseTensorSample input"); + const SparseTensorSample* sparse_input = ctx->Input(0); + // Always a single dimension of 1 bc we are storing a single number + const int64_t dims[1] = {1}; + TensorShape output_shape(dims, 1); + Tensor* sparse_shape = ctx->Output(0, output_shape); + int64_t* shape_data = sparse_shape->MutableData(); + ONNXRUNTIME_ENFORCE(shape_data != nullptr); + *shape_data = sparse_input->Size(); + + return Status::OK(); + } +}; + +namespace test { + +KernelDefBuilder ConstructSparseTensorDef() { + KernelDefBuilder def; + def.SetName("ConstructSparseTensor") + .SetDomain(onnxruntime::kMLDomain) + .SinceVersion(8) + .Provider(onnxruntime::kCpuExecutionProvider) + .TypeConstraint("sparse_values", + DataTypeImpl::GetTensorType()) + .TypeConstraint("sparse_indicies", + DataTypeImpl::GetTensorType()) + .TypeConstraint("sparse_shape", + DataTypeImpl::GetTensorType()) + .TypeConstraint("sparse_rep", + DataTypeImpl::GetType()); + return def; +} + +KernelDefBuilder ConstructFetchSparseShape() { + KernelDefBuilder def; + def.SetName("FetchSparseTensorShape") + .SetDomain(onnxruntime::kMLDomain) + .SinceVersion(8) + .Provider(onnxruntime::kCpuExecutionProvider) + .TypeConstraint("sparse_rep", + DataTypeImpl::GetType()) + .TypeConstraint("sparse_tensor_shape", + DataTypeImpl::GetTensorType()); + return def; +} + +ONNX_NAMESPACE::OpSchema GetConstructSparseTensorSchema() { + ONNX_NAMESPACE::OpSchema schema("ConstructSparseTensor", __FILE__, __LINE__); + schema.SetDoc(experimental_using_opaque) + .SetDomain(onnxruntime::kMLDomain) + .Input( + 0, + "sparse_values", + "Single dimensional Tensor that holds all non-zero values", + "T1", + OpSchema::Single) + .Input( + 1, + "sparse_indicies", + "Single dimensional tensor that holds indicies of non-zero values", + "T2", + OpSchema::Single) + .Input( + 2, + "sparse_shape", + "Single dimensional tensor that holds sparse tensor shape", + "T3", + OpSchema::Single) + .Output( + 0, + "sparse_rep", + "SparseTensorSample opaque object", + "T", + OpSchema::Single) + .TypeConstraint( + "T1", + {"tensor(int64)"}, + "Only int64 is allowed") + .TypeConstraint( + "T2", + {"tensor(int64)"}, + "Only int64 is allowed") + .TypeConstraint( + "T3", + {"tensor(int64)"}, + "Only int64 is allowed") + .TypeConstraint( + "T", + {"opaque(ai.onnx,SparseTensorSample)"}, + "Opaque object"); + schema.SinceVersion(8); + return schema; +} + +ONNX_NAMESPACE::OpSchema GetFetchSparseShapeSchema() { + ONNX_NAMESPACE::OpSchema schema("FetchSparseTensorShape", __FILE__, __LINE__); + schema.SetDoc(experimental_using_opaque) + .SetDomain(onnxruntime::kMLDomain) + .Input( + 0, + "sparse_rep", + "Opaque SparseTensorSample", + "T1", + OpSchema::Single) + .Output( + 0, + "sparse_tensor_shape", + "Single dimensional tensor that holds sparse tensor shape", + "T", + OpSchema::Single) + .TypeConstraint( + "T1", + {"opaque(ai.onnx,SparseTensorSample)"}, + "Only int64 is allowed") + .TypeConstraint( + "T", + {"tensor(int64)"}, + "Only int64 is allowed"); + schema.SinceVersion(8); + return schema; +} + +TEST_F(OpaqueTypeTests, RunModel) { + SessionOptions so; + so.enable_sequential_execution = true; + so.session_logid = "SparseTensorTest"; + so.session_log_verbosity_level = 1; + + // Both the session and the model need custom registries + // so we construct it here before the model + std::shared_ptr registry = std::make_shared(); + InferenceSession session_object{so, &DefaultLoggingManager()}; + EXPECT_TRUE(session_object.RegisterCustomRegistry(registry).IsOK()); + + auto ops_schema = GetConstructSparseTensorSchema(); + auto shape_schema = GetFetchSparseShapeSchema(); + std::vector schemas = {ops_schema, shape_schema}; + EXPECT_TRUE(registry->RegisterOpSet(schemas, onnxruntime::kMLDomain, 8, 9).IsOK()); + // Register our kernels here + auto ctor_def = ConstructSparseTensorDef(); + EXPECT_TRUE(registry->RegisterCustomKernel(ctor_def, [](const OpKernelInfo& info) { return new ConstructSparseTensor(info); }).IsOK()); + auto shape_def = ConstructFetchSparseShape(); + EXPECT_TRUE(registry->RegisterCustomKernel(shape_def, [](const OpKernelInfo& info) { return new FetchSparseTensorShape(info); }).IsOK()); + + IOnnxRuntimeOpSchemaRegistryList custom_schema_registries_ = {registry}; + std::unordered_map domain_to_version = {{onnxruntime::kMLDomain, 8}}; + + Model model("SparseTensorTest", false, ModelMetaData(), custom_schema_registries_, domain_to_version); + auto& graph = model.MainGraph(); + + std::vector inputs; + std::vector outputs; + + TypeProto input_tensor_proto(*DataTypeImpl::GetTensorType()->GetTypeProto()); + + { + // Sparse tensor will contain total 5 elements but only 2 of them a non-zero + TypeProto input_values(input_tensor_proto); + input_values.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(2); + auto& sparse_values_arg = graph.GetOrCreateNodeArg("sparse_values", &input_values); + inputs.push_back(&sparse_values_arg); + + TypeProto input_indicies(input_tensor_proto); + input_indicies.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(2); + auto& sparse_indicies_arg = graph.GetOrCreateNodeArg("sparse_indicies", &input_indicies); + inputs.push_back(&sparse_indicies_arg); + + // Shape tensor will contain only one value + TypeProto input_shape(input_tensor_proto); + input_shape.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + auto& sparse_shape_arg = graph.GetOrCreateNodeArg("sparse_shape", &input_shape); + inputs.push_back(&sparse_shape_arg); + + //Output is our custom data type + TypeProto output_sparse_tensor(*DataTypeImpl::GetType()->GetTypeProto()); + auto& output_sparse_tensor_arg = graph.GetOrCreateNodeArg("sparse_rep", &output_sparse_tensor); + outputs.push_back(&output_sparse_tensor_arg); + + auto* node = graph.AddNode("ConstructSparseTensor", "ConstructSparseTensor", "Create a sparse tensor representation", + inputs, outputs, nullptr, onnxruntime::kMLDomain); + node->SetExecutionProviderType(onnxruntime::kCpuExecutionProvider); + } + { + // We start the input from previous node output + inputs = std::move(outputs); + outputs.clear(); + + TypeProto output_shape(input_tensor_proto); + output_shape.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + auto& output_shape_arg = graph.GetOrCreateNodeArg("sparse_tensor_shape", &output_shape); + outputs.push_back(&output_shape_arg); + auto* node = graph.AddNode("FetchSparseTensorShape", "FetchSparseTensorShape", "Fetch shape from sparse tensor", + inputs, outputs, nullptr, onnxruntime::kMLDomain); + node->SetExecutionProviderType(onnxruntime::kCpuExecutionProvider); + } + + EXPECT_TRUE(graph.Resolve().IsOK()); + + // Get a proto and load from it + std::stringstream serialized_model; + auto model_proto = model.ToProto(); + EXPECT_TRUE(model_proto.SerializeToOstream(&serialized_model)); + + EXPECT_TRUE(session_object.Load(serialized_model).IsOK()); + EXPECT_TRUE(session_object.Initialize().IsOK()); + + RunOptions run_options; + + // Prepare inputs/outputs + std::vector val_dims = {2}; + std::vector values = {1, 2}; + // prepare inputs + MLValue ml_values; + CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault), val_dims, values, &ml_values); + + std::vector ind_dims = {2}; + std::vector indicies = {1, 4}; + MLValue ml_indicies; + CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault), ind_dims, indicies, &ml_indicies); + + std::vector shape_dims = {1}; + std::vector shape = {5}; + MLValue ml_shape; + CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault), shape_dims, shape, &ml_shape); + + NameMLValMap feeds; + feeds.insert(std::make_pair("sparse_values", ml_values)); + feeds.insert(std::make_pair("sparse_indicies", ml_indicies)); + feeds.insert(std::make_pair("sparse_shape", ml_shape)); + + // Output + std::vector output_shape_dims = {1}; + std::vector output_shape = {0}; + + std::vector output_names; + output_names.push_back("sparse_tensor_shape"); + std::vector fetches; + + EXPECT_TRUE(session_object.Run(run_options, feeds, output_names, &fetches).IsOK()); + ASSERT_EQ(1, fetches.size()); + auto& rtensor = fetches.front().Get(); + // Should get the original shape back in the form of a tensor + EXPECT_EQ(1, rtensor.Shape().NumDimensions()); + EXPECT_EQ(5, *rtensor.template Data()); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/session_state_test.cc b/onnxruntime/test/framework/session_state_test.cc new file mode 100644 index 0000000000000..876eecc961bed --- /dev/null +++ b/onnxruntime/test/framework/session_state_test.cc @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include + +#include "core/framework/execution_providers.h" +#include "core/framework/op_kernel.h" +#include "core/framework/session_state.h" +#include "core/graph/graph.h" +#include "core/graph/model.h" +#include "core/graph/op.h" +#include "core/providers/cpu/cpu_execution_provider.h" +#include "gtest/gtest.h" + +using namespace ONNX_NAMESPACE; +using namespace std; + +namespace onnxruntime { +namespace test { +class TestOpKernel : public OpKernel { + public: + TestOpKernel(const OpKernelInfo& p) : OpKernel(p) {} + Status Compute(OpKernelContext* context) const { + ONNXRUNTIME_UNUSED_PARAMETER(context); + return Status::OK(); + } + Status ComputeAsync(OpKernelContext* context, DoneCallback done) const { + ONNXRUNTIME_UNUSED_PARAMETER(context); + ONNXRUNTIME_UNUSED_PARAMETER(done); + return Status::OK(); + } +}; + +TEST(SessionStateTest, AddGetKernelTest) { + ONNX_OPERATOR_SCHEMA(Variable) + .SetDoc("Input variable.") + .Output(0, "output_1", "docstr for output_1.", "tensor(int32)"); + ExecutionProviders execution_providers; + SessionState s{execution_providers}; + + onnxruntime::Model model("graph_1"); + auto& graph = model.MainGraph(); + std::vector inputs; + std::vector outputs; + TypeProto output_type; + output_type.mutable_tensor_type()->set_elem_type(TensorProto_DataType_INT32); + output_type.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + onnxruntime::NodeArg output_arg("node_1_out_1", &output_type); + outputs.push_back(&output_arg); + onnxruntime::Node* p_node = graph.AddNode("node_1", "Variable", "node 1.", inputs, outputs); + auto status = graph.Resolve(); + EXPECT_TRUE(status.IsOK()); + KernelDef kernel_def; + CPUExecutionProvider execution_provider{CPUExecutionProviderInfo{"CPUExecutionProvider"}}; + + OpKernelInfo p_info(*p_node, kernel_def, execution_provider, s); + unique_ptr p_kernel; + p_kernel.reset(new TestOpKernel(p_info)); + size_t orig_num_outputs = p_kernel->Node().OutputDefs().size(); + std::cout << "node_idx: " << p_node->Index() << std::endl; + + s.SetGraphViewer(std::make_unique(graph)); + s.AddKernel(p_node->Index(), std::move(p_kernel)); + auto test_kernel = s.GetKernel(p_node->Index()); + std::cout << "orig: " << orig_num_outputs << " new: " << test_kernel->Node().OutputDefs().size() << std::endl; + EXPECT_EQ(orig_num_outputs, test_kernel->Node().OutputDefs().size()); +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/shape_inference_test.cc b/onnxruntime/test/framework/shape_inference_test.cc new file mode 100644 index 0000000000000..699a47f3fb661 --- /dev/null +++ b/onnxruntime/test/framework/shape_inference_test.cc @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include + +#include "gtest/gtest.h" +#include "core/graph/model.h" +#include "test/framework/model_builder_utils.h" +using namespace ONNX_NAMESPACE; +using namespace std; + +namespace onnxruntime { +namespace test { + +using namespace modelbuilder; + +class ShapeInferenceTest : public ::testing::Test { + protected: + onnxruntime::Model model_; + int node_count_; + std::unordered_map> name_to_arg_; + + public: + ShapeInferenceTest() : model_("Test"), node_count_(0) {} + + void Input(const std::string& name, const Type& type) { + name_to_arg_[name] = std::make_unique(name, &type.value); + } + + onnxruntime::NodeArg* Arg(const std::string& name) { + if (name_to_arg_.count(name) == 0) + name_to_arg_[name] = std::make_unique(name, nullptr); + return name_to_arg_[name].get(); + } + + onnxruntime::Node* Node(const std::string& op, const std::string& input, const std::string& output) { + std::vector input_args({Arg(input)}); + std::vector output_args({Arg(output)}); + int num = node_count_++; + return model_.MainGraph().AddNode("node" + std::to_string(num), op, "test op", input_args, output_args); + } + + void DoShapeInference() { + auto status = model_.MainGraph().Resolve(); + EXPECT_TRUE(status.IsOK()) << "Graph resolve failed: " << status.ErrorMessage(); + } + + const TensorShapeProto* InputShape(onnxruntime::Node* node, int arg_num = 0) { + return node->InputDefs()[arg_num]->Shape(); + } + + const TensorShapeProto* OutputShape(onnxruntime::Node* node, int arg_num = 0) { + return node->OutputDefs()[arg_num]->Shape(); + } + + void CheckShapeEquality(const TensorShapeProto* shape1, const TensorShapeProto* shape2) { + EXPECT_NE(shape1, nullptr); + EXPECT_NE(shape2, nullptr); + if ((shape1 != nullptr) && (shape2 != nullptr)) { + EXPECT_EQ(shape1->dim_size(), shape2->dim_size()) << "Shapes do not have same rank"; + auto min_dims = std::min(shape1->dim_size(), shape2->dim_size()); + for (int i = 0; i < min_dims; ++i) { + auto dim1 = shape1->dim(i); + auto dim2 = shape2->dim(i); + EXPECT_EQ(dim1.has_dim_value(), dim2.has_dim_value()); + if (dim1.has_dim_value()) { + EXPECT_EQ(dim1.dim_value(), dim2.dim_value()); + } + EXPECT_EQ(dim1.has_dim_param(), dim2.has_dim_param()); + if (dim1.has_dim_param()) { + EXPECT_EQ(dim1.dim_param(), dim2.dim_param()); + } + } + } + } + +}; // namespace test + +TEST_F(ShapeInferenceTest, BasicTest) { + Type type1({1, 50, 100}); + Input("X1", type1); + + auto p_node = Node("Cast", "X1", "Y1"); + //AttributeProto squeezed_axes; + //squeezed_axes.set_name("axes"); + //squeezed_axes.set_type(ONNX_NAMESPACE::AttributeProto_AttributeType_INTS); + //squeezed_axes.add_ints(0); + //p_node->AddAttribute("axes", squeezed_axes); + AttributeProto cast_to; + cast_to.set_name("to"); + cast_to.set_type(ONNX_NAMESPACE::AttributeProto_AttributeType_INT); + cast_to.set_i(ONNX_NAMESPACE::TensorProto_DataType_INT32); + //cast_to.set_type(ONNX_NAMESPACE::AttributeProto_AttributeType_STRING); + //cast_to.set_s("INT16"); + p_node->AddAttribute("to", cast_to); + + DoShapeInference(); + // check inferred shapes + Shape expected_shape({1, 50, 100}); + CheckShapeEquality(OutputShape(p_node), &expected_shape.value); + CheckShapeEquality(InputShape(p_node), OutputShape(p_node)); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/tensor_test.cc b/onnxruntime/test/framework/tensor_test.cc new file mode 100644 index 0000000000000..d545ae65c3bc3 --- /dev/null +++ b/onnxruntime/test/framework/tensor_test.cc @@ -0,0 +1,220 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/tensor.h" +#include "core/framework/allocatormgr.h" +#include "test_utils.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include + +namespace onnxruntime { +namespace test { +template +void CPUTensorTest(std::vector dims, const int offset = 0) { + //not own the buffer + TensorShape shape(dims); + auto alloc = TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault); + auto data = alloc->Alloc(sizeof(T) * (shape.Size() + offset)); + EXPECT_TRUE(data); + Tensor t(DataTypeImpl::GetType(), shape, data, alloc->Info(), nullptr, offset); + auto tensor_shape = t.Shape(); + EXPECT_EQ(shape, tensor_shape); + EXPECT_EQ(t.DataType(), DataTypeImpl::GetType()); + auto& location = t.Location(); + EXPECT_STREQ(location.name, CPU); + EXPECT_EQ(location.id, 0); + + auto t_data = t.template MutableData(); + EXPECT_TRUE(t_data); + memset(t_data, 0, sizeof(T) * shape.Size()); + EXPECT_EQ(*(T*)((char*)data + offset), (T)0); + alloc->Free(data); + + // owned buffer + data = alloc->Alloc(sizeof(T) * (shape.Size() + offset)); + EXPECT_TRUE(data); + Tensor new_t(DataTypeImpl::GetType(), shape, data, alloc->Info(), alloc, offset); + + tensor_shape = new_t.Shape(); + EXPECT_EQ(shape, tensor_shape); + EXPECT_EQ(new_t.DataType(), DataTypeImpl::GetType()); + auto& new_location = new_t.Location(); + ASSERT_STREQ(new_location.name, CPU); + EXPECT_EQ(new_location.id, 0); + + auto new_data = new_t.template MutableData(); + EXPECT_TRUE(new_data); + memset(new_data, 0, sizeof(T) * shape.Size()); + EXPECT_EQ(*(T*)((char*)new_data + offset), (T)0); + //no free op as the tensor own the buffer +} + +TEST(TensorTest, CPUFloatTensorTest) { + CPUTensorTest(std::vector({3, 2, 4})); +} + +TEST(TensorTest, CPUInt32TensorTest) { + CPUTensorTest(std::vector({3, 2, 4})); +} + +TEST(TensorTest, CPUUInt8TensorTest) { + CPUTensorTest(std::vector({3, 2, 4})); +} + +TEST(TensorTest, CPUUInt16TensorTest) { + CPUTensorTest(std::vector({3, 2, 4})); +} + +TEST(TensorTest, CPUInt16TensorTest) { + CPUTensorTest(std::vector({3, 2, 4})); +} + +TEST(TensorTest, CPUInt64TensorTest) { + CPUTensorTest(std::vector({3, 2, 4})); +} + +TEST(TensorTest, CPUDoubleTensorTest) { + CPUTensorTest(std::vector({3, 2, 4})); +} + +TEST(TensorTest, CPUUInt32TensorTest) { + CPUTensorTest(std::vector({3, 2, 4})); +} + +TEST(TensorTest, CPUUInt64TensorTest) { + CPUTensorTest(std::vector({3, 2, 4})); +} + +TEST(TensorTest, CPUFloatTensorOffsetTest) { + CPUTensorTest(std::vector({3, 2, 4}), 5); +} + +TEST(TensorTest, CPUInt32TensorOffsetTest) { + CPUTensorTest(std::vector({3, 2, 4}), 5); +} + +TEST(TensorTest, CPUUInt8TensorOffsetTest) { + CPUTensorTest(std::vector({3, 2, 4}), 5); +} + +TEST(TensorTest, CPUUInt16TensorOffsetTest) { + CPUTensorTest(std::vector({3, 2, 4}), 5); +} + +TEST(TensorTest, CPUInt16TensorOffsetTest) { + CPUTensorTest(std::vector({3, 2, 4}), 5); +} + +TEST(TensorTest, CPUInt64TensorOffsetTest) { + CPUTensorTest(std::vector({3, 2, 4}), 5); +} + +TEST(TensorTest, CPUDoubleTensorOffsetTest) { + CPUTensorTest(std::vector({3, 2, 4}), 5); +} + +TEST(TensorTest, CPUUInt32TensorOffsetTest) { + CPUTensorTest(std::vector({3, 2, 4}), 5); +} + +TEST(TensorTest, CPUUInt64TensorOffsetTest) { + CPUTensorTest(std::vector({3, 2, 4}), 5); +} + +TEST(TensorTest, EmptyTensorTest) { + auto type = DataTypeImpl::GetType(); + Tensor t(type, TensorShape({1, 0}), nullptr, TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault)->Info()); + auto& shape = t.Shape(); + EXPECT_EQ(shape.Size(), 0); + EXPECT_EQ(t.DataType(), type); + + auto data = t.template MutableData(); + EXPECT_TRUE(!data); + + auto& location = t.Location(); + ASSERT_STREQ(location.name, CPU); + EXPECT_EQ(location.id, 0); + EXPECT_EQ(location.type, ONNXRuntimeAllocatorType::ONNXRuntimeArenaAllocator); +} + +TEST(TensorTest, TensorCopyAssignOpTest) { + TensorShape shape({1, 2, 3}); + auto alloc = TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault); + auto data = alloc->Alloc(sizeof(int) * shape.Size()); + EXPECT_TRUE(data); + Tensor t1(DataTypeImpl::GetType(), shape, data, alloc->Info()); + Tensor t2 = t1; + EXPECT_EQ(t2.DataType(), DataTypeImpl::GetType()); + EXPECT_EQ(t2.Shape(), shape); + auto location = t2.Location(); + ASSERT_STREQ(location.name, CPU); + EXPECT_EQ(location.id, 0); + EXPECT_EQ(location.type, ONNXRuntimeAllocatorType::ONNXRuntimeArenaAllocator); + auto t_data = t2.template Data(); + EXPECT_EQ((void*)t_data, data); + alloc->Free(data); +} + +TEST(TensorTest, StringTensorTest) { +//add scope to explicitly delete tensor +#ifdef _MSC_VER + std::string* string_ptr = nullptr; +#else + std::string* string_ptr __attribute__((unused)) = nullptr; +#endif + { + TensorShape shape({2, 3}); + auto alloc = TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault); + auto buffer = alloc->Alloc(sizeof(std::string) * (shape.Size())); + Tensor t(DataTypeImpl::GetType(), shape, buffer, alloc->Info(), alloc); + + auto& tensor_shape = t.Shape(); + EXPECT_EQ(shape, tensor_shape); + EXPECT_EQ(t.DataType(), DataTypeImpl::GetType()); + auto& location = t.Location(); + ASSERT_STREQ(location.name, CPU); + EXPECT_EQ(location.id, 0); + + std::string* new_data = t.template MutableData(); + EXPECT_TRUE(new_data); + new_data[0] = "a"; + new_data[1] = "b"; + + auto tensor_data = t.template Data(); + EXPECT_EQ(tensor_data[0], "a"); + EXPECT_EQ(tensor_data[1], "b"); + string_ptr = new_data; + } + // on msvc, check does the ~string be called when release tensor + // It may be not stable as access to a deleted pointer could have + // undefined behavior. If we find it is failure on other platform + // go ahead to remove it. +#ifdef _MSC_VER + EXPECT_EQ(string_ptr->size(), 0); + EXPECT_EQ((string_ptr + 1)->size(), 0); +#endif +} + +TEST(TensorTest, ConvertToString) { + TensorShape shape({2, 3, 4}); + + EXPECT_EQ(shape.ToString(), "{2,3,4}"); + + std::ostringstream ss; + ss << shape; + EXPECT_EQ(ss.str(), "{2,3,4}"); +} + +TEST(TensorTest, Int64PtrConstructor) { + int64_t dimensions[] = {2, 3, 4}; + TensorShape shape(dimensions, 2); // just use first 2 + EXPECT_EQ(shape.Size(), 6); + EXPECT_EQ(shape.NumDimensions(), 2); + EXPECT_THAT(shape.GetDims(), testing::ElementsAre(2, 3)); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/tensorprotoutils_test.cc b/onnxruntime/test/framework/tensorprotoutils_test.cc new file mode 100644 index 0000000000000..3898629a80809 --- /dev/null +++ b/onnxruntime/test/framework/tensorprotoutils_test.cc @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/platform/env.h" +#include "core/framework/tensor.h" +#include "core/graph/onnx_protobuf.h" +#include "core/framework/tensorprotoutils.h" +#include "gtest/gtest.h" +#include +#include + +namespace onnxruntime { +namespace test { +#ifdef ONNXRUNTIME_RUN_EXTERNAL_ONNX_TESTS +TEST(TensorProtoUtilsTest, test1) { + const char* filename = "../models/opset8/test_resnet50/test_data_set_0/input_0.pb"; + int test_data_pb_fd; + common::Status st = Env::Default().FileOpenRd(filename, test_data_pb_fd); + ASSERT_TRUE(st.IsOK()); + google::protobuf::io::FileInputStream f(test_data_pb_fd); + f.SetCloseOnDelete(true); + ONNX_NAMESPACE::TensorProto proto; + ASSERT_TRUE(proto.ParseFromZeroCopyStream(&f)); + std::unique_ptr tensor; + ::onnxruntime::AllocatorPtr cpu_allocator = std::make_shared<::onnxruntime::CPUAllocator>(); + st = ::onnxruntime::utils::GetTensorFromTensorProto(proto, &tensor, cpu_allocator); + ASSERT_TRUE(st.IsOK()); +} +#endif +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/tensorutils_test.cc b/onnxruntime/test/framework/tensorutils_test.cc new file mode 100644 index 0000000000000..f666429876e40 --- /dev/null +++ b/onnxruntime/test/framework/tensorutils_test.cc @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/tensorutils.h" +#include "core/graph/onnx_protobuf.h" +#include "gtest/gtest.h" + +using namespace ::onnxruntime::utils; +using namespace ONNX_NAMESPACE; + +namespace onnxruntime { +namespace test { + +//T must be float for double, and it must match with the 'type' argument +template +void test_unpack_float_tensor(TensorProto_DataType type) { + TensorProto float_tensor_proto; + float_tensor_proto.set_data_type(type); + T f[4] = {1.1f, 2.2f, 3.3f, 4.4f}; + const size_t len = sizeof(T) * 4; + char rawdata[len]; + for (int i = 0; i < 4; ++i) { + memcpy(rawdata + i * sizeof(T), &(f[i]), sizeof(T)); + } + float_tensor_proto.set_raw_data(rawdata, len); + T float_data2[4]; + auto status = TensorUtils::UnpackTensor(float_tensor_proto, float_data2, 4); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + EXPECT_EQ(1.1f, float_data2[0]); + EXPECT_EQ(2.2f, float_data2[1]); + EXPECT_EQ(3.3f, float_data2[2]); + EXPECT_EQ(4.4f, float_data2[3]); +} + +TEST(TensorParseTest, TensorUtilsTest) { + TensorProto bool_tensor_proto; + bool_tensor_proto.set_data_type(TensorProto_DataType_BOOL); + bool_tensor_proto.add_int32_data(1); + + bool bool_data[1]; + auto status = TensorUtils::UnpackTensor(bool_tensor_proto, bool_data, 1); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + EXPECT_TRUE(bool_data[0]); + + float float_data[1]; + status = TensorUtils::UnpackTensor(bool_tensor_proto, float_data, 1); + EXPECT_FALSE(status.IsOK()); + + test_unpack_float_tensor(TensorProto_DataType_FLOAT); + test_unpack_float_tensor(TensorProto_DataType_DOUBLE); + + TensorProto string_tensor_proto; + string_tensor_proto.set_data_type(TensorProto_DataType_STRING); + string_tensor_proto.add_string_data("a"); + string_tensor_proto.add_string_data("b"); + + std::string string_data[2]; + status = TensorUtils::UnpackTensor(string_tensor_proto, string_data, 2); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + EXPECT_EQ("a", string_data[0]); + EXPECT_EQ("b", string_data[1]); + + status = TensorUtils::UnpackTensor(bool_tensor_proto, string_data, 2); + EXPECT_FALSE(status.IsOK()); +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/test_main.cc b/onnxruntime/test/framework/test_main.cc new file mode 100644 index 0000000000000..50413690bd832 --- /dev/null +++ b/onnxruntime/test/framework/test_main.cc @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/environment.h" +#include "gtest/gtest.h" +#include "test/test_environment.h" + +GTEST_API_ int main(int argc, char** argv) { + int status = 0; + + try { + onnxruntime::test::TestEnvironment test_environment{argc, argv}; + + status = RUN_ALL_TESTS(); + } catch (const std::exception& ex) { + std::cerr << ex.what(); + status = -1; + } + + return status; +} diff --git a/onnxruntime/test/framework/test_utils.cc b/onnxruntime/test/framework/test_utils.cc new file mode 100644 index 0000000000000..19df409604c34 --- /dev/null +++ b/onnxruntime/test/framework/test_utils.cc @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "test_utils.h" +namespace onnxruntime { +namespace test { +IExecutionProvider* TestCPUExecutionProvider() { + static CPUExecutionProviderInfo info; + static CPUExecutionProvider cpu_provider(info); + return &cpu_provider; +} + +#ifdef USE_CUDA +IExecutionProvider* TestCudaExecutionProvider() { + static CUDAExecutionProviderInfo info; + static CUDAExecutionProvider cuda_provider(info); + return &cuda_provider; +} +#endif +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/test_utils.h b/onnxruntime/test/framework/test_utils.h new file mode 100644 index 0000000000000..24f8851102e20 --- /dev/null +++ b/onnxruntime/test/framework/test_utils.h @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "core/framework/allocatormgr.h" +#include "core/framework/execution_provider.h" +#include "core/providers/cpu/cpu_execution_provider.h" +#include "core/framework/ml_value.h" +#ifdef USE_CUDA +#include "core/providers/cuda/cuda_execution_provider.h" +#endif + +namespace onnxruntime { +namespace test { +IExecutionProvider* TestCPUExecutionProvider(); + +#ifdef USE_CUDA +IExecutionProvider* TestCudaExecutionProvider(); +#endif + +template +void CreateMLValue(AllocatorPtr alloc, + const std::vector& dims, + const std::vector& value, + MLValue* p_mlvalue) { + TensorShape shape(dims); + auto location = alloc->Info(); + auto element_type = DataTypeImpl::GetType(); + void* buffer = alloc->Alloc(element_type->Size() * shape.Size()); + if (value.size() > 0) { + memcpy(buffer, &value[0], element_type->Size() * shape.Size()); + } + + std::unique_ptr p_tensor = std::make_unique(element_type, + shape, + buffer, + location, + alloc); + p_mlvalue->Init(p_tensor.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); +} + +template +void AllocateMLValue(AllocatorPtr alloc, + const std::vector& dims, + MLValue* p_mlvalue) { + TensorShape shape(dims); + auto location = alloc->Info(); + auto element_type = DataTypeImpl::GetType(); + void* buffer = alloc->Alloc(element_type->Size() * shape.Size()); + std::unique_ptr p_tensor = std::make_unique(element_type, + shape, + buffer, + location, + alloc); + p_mlvalue->Init(p_tensor.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/ir/common_data_structure_test.cc b/onnxruntime/test/ir/common_data_structure_test.cc new file mode 100644 index 0000000000000..be7299777c2be --- /dev/null +++ b/onnxruntime/test/ir/common_data_structure_test.cc @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/graph/record.h" +#include "gtest/gtest.h" + +using namespace ::onnxruntime::common; + +namespace onnxruntime { +namespace test { +TEST(RecordTest, CommonDataStructureTest) { + Record string_record; + const std::string* n2; + EXPECT_FALSE(string_record.GetName(0, &n2).IsOK()); + + // One way to store feature vector using Record. + std::vector names = {"featureName", "featureValue"}; + std::tuple values("streamLength", 2.0f); + Record record(names, values); + + const std::string* name; + auto status = record.GetName(2, &name); + EXPECT_FALSE(status.IsOK()); + + record.GetName(0, &name); + auto& value = std::get<0>(record.GetValues()); + EXPECT_EQ("featureName", *name); + EXPECT_EQ("streamLength", value); + + record.GetName(1, &name); + auto& value2 = std::get<1>(record.GetValues()); + EXPECT_EQ("featureValue", *name); + EXPECT_EQ(2.0f, value2); + + // Another way to store feature vector using Record. + names = {"streamLength"}; + std::tuple values2(2.0f); + Record record2(names, values2); + + record2.GetName(0, &name); + auto& value3 = std::get<0>(record2.GetValues()); + EXPECT_EQ("streamLength", *name); + EXPECT_EQ(2.0f, value3); +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/ir/graph_test.cc b/onnxruntime/test/ir/graph_test.cc new file mode 100644 index 0000000000000..f9d90a0953567 --- /dev/null +++ b/onnxruntime/test/ir/graph_test.cc @@ -0,0 +1,869 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#ifdef _MSC_VER +#pragma warning(push) +// 'identifier' : unreferenced formal parameter +#pragma warning(disable : 4100) +// 'type' : forcing value to bool 'true' or 'false' (performance warning) +#pragma warning(disable : 4800) +#else +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif +#include "google/protobuf/util/message_differencer.h" +#ifdef _MSC_VER +#pragma warning(pop) +#else +#pragma GCC diagnostic pop +#endif +#include "core/graph/graph.h" +#include "core/graph/model.h" +#include "core/graph/op.h" +#include "gtest/gtest.h" + +#ifdef __GNUC__ +#define UNUSED __attribute__((unused)) +#else +#define UNUSED +#endif + +#define OPERATOR_SCHEMA UNUSED ONNX_OPERATOR_SCHEMA + +using namespace ONNX_NAMESPACE; +namespace onnxruntime { +namespace test { +using google::protobuf::util::MessageDifferencer; + +static bool RegisterCustomSchemas() { + OPERATOR_SCHEMA(Variable_DFS) + .SetDoc("Input variable.") + .Input(0, "input_1", "docstr for input_1.", "tensor(int32)") + .Output(0, "output_1", "docstr for output_1.", "tensor(int32)"); + OPERATOR_SCHEMA(Add_DFS) + .SetDoc("Add two integers.") + .Input(0, "input_1", "docstr for input_1.", "tensor(int32)") + .Input(1, "input_2", "docstr for input_2.", "tensor(int32)") + .Output(0, "output_1", "docstr for output_1.", "tensor(int32)"); + OPERATOR_SCHEMA(NoOp_DFS) + .SetDoc("Operator doing nothing.") + .Input(0, "input_1", "docstr for input_1.", "tensor(int32)") + .Output(0, "output_1", "docstr for output_1.", "tensor(int32)"); + + OPERATOR_SCHEMA(Variable_Fake) + .SetDoc("Input variable.") + .Input(0, "input_1", "docstr for input_1.", "tensor(int32)") + .Output(0, "output_1", "docstr for output_1.", "tensor(int32)"); + OPERATOR_SCHEMA(Add_Fake) + .SetDoc("Add two integers.") + .Input(0, "input_1", "docstr for input_1.", "tensor(int32)") + .Input(1, "input_2", "docstr for input_2.", "tensor(int32)") + .Output(0, "output_1", "docstr for output_1.", "tensor(int32)"); + OPERATOR_SCHEMA(NoOp_Fake) + .SetDoc("Operator doing nothing.") + .Input(0, "input_1", "docstr for input_1.", "tensor(int32)") + .Output(0, "output_1", "docstr for output_1.", "tensor(int32)"); + + OPERATOR_SCHEMA(Identity_Fake) + .SetDoc("Identity.") + .Input(0, "input_1", "docstr for input_1.", "tensor(int32)") + .Output(0, "output_1", "docstr for output_1.", "tensor(int32)"); + OPERATOR_SCHEMA(Merge_Fake) + .SetDoc("Merge.") + .Input(0, "input_1", "docstr for input_1.", "tensor(int32)") + .Input(1, "input_2", "docstr for input_2.", "tensor(int32)") + .Output(0, "output_1", "docstr for output_1.", "tensor(int32)"); + + // we need more than 8 outputs to trigger the unordered_map that's used in Graph::SetGraphInputsOutputs to + // re-allocate and re-order to prove the code works. + OPERATOR_SCHEMA(Split_Fake) + .SetDoc("Split.") + .Input(0, "input_1", "docstr for input_1.", "tensor(int32)") + .Output(0, "output_1", "docstr for output_1.", "tensor(int32)") + .Output(1, "output_2", "docstr for output_2.", "tensor(int32)") + .Output(2, "output_3", "docstr for output_3.", "tensor(int32)") + .Output(3, "output_4", "docstr for output_4.", "tensor(int32)") + .Output(4, "output_5", "docstr for output_5.", "tensor(int32)") + .Output(5, "output_6", "docstr for output_6.", "tensor(int32)") + .Output(6, "output_7", "docstr for output_7.", "tensor(int32)") + .Output(7, "output_8", "docstr for output_8.", "tensor(int32)") + .Output(8, "output_9", "docstr for output_9.", "tensor(int32)") + .Output(9, "output_10", "docstr for output_10.", "tensor(int32)"); + + OPERATOR_SCHEMA(Variable2_Fake) + .SetDoc("Input variable.") + .Input(0, "input_1", "docstr for input_1.", "T") + .Output(0, "output_1", "docstr for output_1.", "T") + .TypeConstraint("T", {"tensor(int32)", "tensor(float)"}, "input/output types"); + + OPERATOR_SCHEMA(Max_Fake) + .SetDoc("Add two integers.") + .Input(0, "input_1", "docstr for input_1.", "T") + .Input(1, "input_2", "docstr for input_2.", "T") + .Input(2, "input_3", "docstr for input_3.", "T") + .Output(0, "output_1", "docstr for output_1.", "T") + .TypeConstraint("T", {"tensor(int32)", "tensor(float)"}, "input/output types"); + + return true; +} + +static const bool kSchemasRegistered = RegisterCustomSchemas(); + +TEST(GraphTraversalTest, ReverseDFS) { + ASSERT_TRUE(kSchemasRegistered); + + Model model("graph_1"); + auto& graph = model.MainGraph(); + + // Case 1: A normal graph. + // SouceNode + // / \ + // node_1 (Variable) node_2 (Variable) + // \ / + // node_3 (Add) + // | + // node_4 (NoOp) + // | + // SinkNode + std::vector inputs; + std::vector outputs; + + TypeProto tensor_int32; + tensor_int32.mutable_tensor_type()->set_elem_type(TensorProto_DataType_INT32); + tensor_int32.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + + auto& input_arg = graph.GetOrCreateNodeArg("node_1_in_1", &tensor_int32); + inputs.push_back(&input_arg); + auto& output_arg = graph.GetOrCreateNodeArg("node_1_out_1", &tensor_int32); + outputs.push_back(&output_arg); + auto node_1 = graph.AddNode("node_1", "Variable_DFS", "node 1", inputs, outputs); + + auto& input_arg2 = graph.GetOrCreateNodeArg("node_2_in_1", &tensor_int32); + inputs.clear(); + inputs.push_back(&input_arg2); + auto& output_arg2 = graph.GetOrCreateNodeArg("node_2_out_1", &tensor_int32); + outputs.clear(); + outputs.push_back(&output_arg2); + graph.AddNode("node_2", "Variable_DFS", "node 2", inputs, outputs); + + inputs.clear(); + inputs.push_back(&output_arg); + inputs.push_back(&output_arg2); + auto& output_arg3 = graph.GetOrCreateNodeArg("node_3_out_1", &tensor_int32); + outputs.clear(); + outputs.push_back(&output_arg3); + auto node_3 = graph.AddNode("node_3", "Add_DFS", "node 3", inputs, outputs); + + inputs.clear(); + inputs.push_back(&output_arg3); + auto& output_arg4 = graph.GetOrCreateNodeArg("node_4_out_1", &tensor_int32); + outputs.clear(); + outputs.push_back(&output_arg4); + graph.AddNode("node_4", "NoOp_DFS", "node 4", inputs, outputs); + auto status = graph.Resolve(); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + + // Remove/Add edge should not ask for resolving again. + graph.RemoveEdge(node_1->Index(), node_3->Index(), *graph.GetNodeArg("node_1_out_1")); + graph.AddEdge(node_1->Index(), node_3->Index(), *graph.GetNodeArg("node_1_out_1")); + + std::vector from; + for (auto& node : graph.Nodes()) { + if (node.OutputEdgesBegin() == node.OutputEdgesEnd()) { + // This is a leaf node. + from.push_back(&node); + } + } + + std::vector enter_leave_sequence; + + struct NodeCompareName { + bool operator()(const Node* n1, const Node* n2) const { + return n1->Name() < n2->Name(); + } + }; + + graph.ReverseDFSFrom( + from, + [&enter_leave_sequence](const Node* n) { + std::string s("enter:"); + s += n->Name(); + enter_leave_sequence.push_back(s); + }, + [&enter_leave_sequence](const Node* n) { + std::string s("leave:"); + s += n->Name(); + enter_leave_sequence.push_back(s); + }, + NodeCompareName()); + + EXPECT_EQ(enter_leave_sequence.size(), 8); + EXPECT_EQ("enter:node_4", enter_leave_sequence.at(0)); + EXPECT_EQ("enter:node_3", enter_leave_sequence.at(1)); + EXPECT_EQ("enter:node_2", enter_leave_sequence.at(2)); + EXPECT_EQ("leave:node_2", enter_leave_sequence.at(3)); + EXPECT_EQ("enter:node_1", enter_leave_sequence.at(4)); + EXPECT_EQ("leave:node_1", enter_leave_sequence.at(5)); + EXPECT_EQ("leave:node_3", enter_leave_sequence.at(6)); + EXPECT_EQ("leave:node_4", enter_leave_sequence.at(7)); +} + +TEST(ResolvingGraphTest, GraphConstruction_VerifyNoDuplicateName) { + Model model("graph_1"); + auto& graph = model.MainGraph(); + + EXPECT_EQ("graph_1", graph.Name()); + + std::vector inputs; + std::vector outputs; + + // INT32 vector. + TypeProto output_type; + output_type.mutable_tensor_type()->set_elem_type(TensorProto_DataType_INT32); + output_type.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + + auto& output_arg = graph.GetOrCreateNodeArg("node_1_out_1", &output_type); + outputs.push_back(&output_arg); + graph.AddNode("node_1", "Variable", "node 1.", inputs, outputs); + + // Case 1: Adding two nodes with same node name should fail. + auto node_with_dup_name = graph.AddNode("node_1", "Variable", "node 2", inputs, outputs); + auto status = graph.Resolve(); + EXPECT_FALSE(status.IsOK()); + EXPECT_EQ("Error: two nodes with same node name (node_1).", status.ErrorMessage()); + graph.RemoveNode(node_with_dup_name->Index()); + + // Case 2: Adding two nodes with same output arg name should fail. + graph.AddNode("node_2", "Variable", "node 2", inputs, outputs); + status = graph.Resolve(); + EXPECT_FALSE(status.IsOK()); + bool duplicate_error_found = status.ErrorMessage().find("Duplicate") != std::string::npos; + EXPECT_TRUE(duplicate_error_found); +} + +TEST(ResolvingGraphTest, GraphConstruction_VerifyNodeAndOpMatch) { + Model model("graph_1"); + auto& graph = model.MainGraph(); + + std::vector inputs; + std::vector outputs; + + // INT32 vector. + TypeProto output_type; + output_type.mutable_tensor_type()->set_elem_type(TensorProto_DataType_INT32); + output_type.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + + auto& output_arg = graph.GetOrCreateNodeArg("node_1_out_1", &output_type); + outputs.push_back(&output_arg); + // Case: Adding node refering to non-existing operator should fail. + graph.AddNode("node_1", "OpNotExist", "node 1", inputs, outputs); + auto status = graph.Resolve(); + EXPECT_FALSE(status.IsOK()); + EXPECT_EQ(0, status.ErrorMessage().find_first_of("No Schema registered for OpNotExist")); +} + +TEST(ResolvingGraphTest, GraphConstruction_CheckIsAcyclic) { + ASSERT_TRUE(kSchemasRegistered); + + Model model("graph_1"); + auto& graph = model.MainGraph(); + + // A normal graph. + // SouceNode + // / \ + // node_1 (Variable) node_2 (Variable) + // \ / + // node_3 (Add) + // | + // node_4 (NoOp) + // | + // SinkNode + std::vector inputs; + std::vector outputs; + + std::unordered_map, std::vector>> + expected_node_name_to_input_output_args; + + TypeProto tensor_int32; + tensor_int32.mutable_tensor_type()->set_elem_type(TensorProto_DataType_INT32); + tensor_int32.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + + auto& input_arg1 = graph.GetOrCreateNodeArg("node_1_in_1", &tensor_int32); + inputs.push_back(&input_arg1); + auto& output_arg1 = graph.GetOrCreateNodeArg("node_1_out_1", &tensor_int32); + outputs.push_back(&output_arg1); + expected_node_name_to_input_output_args["node_1"] = {inputs, outputs}; + graph.AddNode("node_1", "Variable_Fake", "node 1", inputs, outputs); + + auto& input_arg2 = graph.GetOrCreateNodeArg("node_2_in_1", &tensor_int32); + inputs.clear(); + inputs.push_back(&input_arg2); + auto& output_arg2 = graph.GetOrCreateNodeArg("node_2_out_1", &tensor_int32); + outputs.clear(); + outputs.push_back(&output_arg2); + expected_node_name_to_input_output_args["node_2"] = {inputs, outputs}; + graph.AddNode("node_2", "Variable_Fake", "node 2", inputs, outputs); + + inputs.clear(); + inputs.push_back(&output_arg1); + inputs.push_back(&output_arg2); + auto& output_arg3 = graph.GetOrCreateNodeArg("node_3_out_1", &tensor_int32); + outputs.clear(); + outputs.push_back(&output_arg3); + expected_node_name_to_input_output_args["node_3"] = {inputs, outputs}; + graph.AddNode("node_3", "Add_Fake", "node 3", inputs, outputs); + + inputs.clear(); + inputs.push_back(&output_arg3); + auto& output_arg4 = graph.GetOrCreateNodeArg("node_4_out_1", &tensor_int32); + outputs.clear(); + outputs.push_back(&output_arg4); + expected_node_name_to_input_output_args["node_4"] = {inputs, outputs}; + graph.AddNode("node_4", "NoOp_Fake", "node 4", inputs, outputs); + auto status = graph.Resolve(); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + + EXPECT_TRUE(Model::Save(model, "graph_1.pb").IsOK()); + std::shared_ptr model2; + EXPECT_TRUE(Model::Load("graph_1.pb", model2).IsOK()); + + auto model_proto = model.ToProto(); + auto model_proto2 = model2->ToProto(); + bool equal_proto_1_and_2 = MessageDifferencer::MessageDifferencer::Equals(model_proto, model_proto2); + std::string diff; + if (!equal_proto_1_and_2) { + MessageDifferencer d; + d.ReportDifferencesToString(&diff); + d.Compare(model_proto, model_proto2); + } else { + diff = "it's fine"; + } + EXPECT_TRUE(equal_proto_1_and_2) << diff; + + // Load the model again to ensure that it's still the right thing. + //EXPECT_EQ(Model::Load(model_proto2, &model2), Status::OK()); + model2.reset(new Model(model_proto2)); + Graph& graph2 = model2->MainGraph(); + for (auto& node : graph2.Nodes()) { + auto node_name_to_input_output_iter = expected_node_name_to_input_output_args.find(node.Name()); + EXPECT_FALSE(node_name_to_input_output_iter == expected_node_name_to_input_output_args.end()); + + EXPECT_EQ(node_name_to_input_output_iter->second.first.size(), node.InputDefs().size()); + for (size_t i = 0; i < node_name_to_input_output_iter->second.first.size(); ++i) { + EXPECT_EQ(node_name_to_input_output_iter->second.first[i]->Name(), node.InputDefs()[i]->Name()); + EXPECT_EQ(node_name_to_input_output_iter->second.first[i]->Type(), node.InputDefs()[i]->Type()); + } + + EXPECT_EQ(node_name_to_input_output_iter->second.second.size(), node.OutputDefs().size()); + for (size_t i = 0; i < node_name_to_input_output_iter->second.second.size(); ++i) { + EXPECT_EQ(node_name_to_input_output_iter->second.second[i]->Name(), node.OutputDefs()[i]->Name()); + EXPECT_EQ(node_name_to_input_output_iter->second.second[i]->Type(), node.OutputDefs()[i]->Type()); + } + } +} + +TEST(ResolvingGraphTest, GraphConstruction_CheckInputNodeOrderMaintained) { + ASSERT_TRUE(kSchemasRegistered); + + Model model("graph_1"); + auto& graph = model.MainGraph(); + + // node_1 (Identity) node_2 (Identity) + // | | + // node_4 (Identity) node_3 (Identity) Cross inputs over so node_1 and node_2 would get swapped if we didn't + // \ / maintain order. + // node_5 (Merge) + // | + + std::unordered_map, std::vector>> + expected_node_name_to_input_output_args; + + TypeProto tensor_int32; + tensor_int32.mutable_tensor_type()->set_elem_type(TensorProto_DataType_INT32); + tensor_int32.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + + auto& input_arg1 = graph.GetOrCreateNodeArg("node_1_in_1", &tensor_int32); + auto& output_arg1 = graph.GetOrCreateNodeArg("node_1_out_1", &tensor_int32); + + auto& input_arg2 = graph.GetOrCreateNodeArg("node_2_in_1", &tensor_int32); + auto& output_arg2 = graph.GetOrCreateNodeArg("node_2_out_1", &tensor_int32); + + auto& output_arg3 = graph.GetOrCreateNodeArg("node_3_out_1", &tensor_int32); + auto& output_arg4 = graph.GetOrCreateNodeArg("node_4_out_1", &tensor_int32); + auto& output_arg5 = graph.GetOrCreateNodeArg("node_5_out_1", &tensor_int32); + + std::vector inputs; + std::vector outputs; + + inputs.push_back(&input_arg1); + outputs.push_back(&output_arg1); + expected_node_name_to_input_output_args["node_1"] = {inputs, outputs}; + graph.AddNode("node_1", "Identity_Fake", "node 1", inputs, outputs); + + inputs[0] = &input_arg2; + outputs[0] = &output_arg2; + expected_node_name_to_input_output_args["node_2"] = {inputs, outputs}; + graph.AddNode("node_2", "Identity_Fake", "node 2", inputs, outputs); + + inputs[0] = &output_arg2; + outputs[0] = &output_arg3; + expected_node_name_to_input_output_args["node_3"] = {inputs, outputs}; + graph.AddNode("node_3", "Identity_Fake", "node 3", inputs, outputs); + + inputs[0] = &output_arg1; + outputs[0] = &output_arg4; + expected_node_name_to_input_output_args["node_4"] = {inputs, outputs}; + graph.AddNode("node_4", "Identity_Fake", "node 4", inputs, outputs); + + inputs.resize(2); + inputs[0] = &output_arg4; + inputs[1] = &output_arg3; + outputs[0] = &output_arg5; + expected_node_name_to_input_output_args["node_5"] = {inputs, outputs}; + graph.AddNode("node_5", "Merge_Fake", "node 3", inputs, outputs); + + auto status = graph.Resolve(); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + GraphViewer graph_viewer(graph); + auto& topological_order = graph_viewer.GetNodesInTopologicalOrder(); + bool seen1 = false; + bool seen2 = false; + + for (auto i : topological_order) { + auto node = graph.GetNode(i); + + if (node->Name() == "node_1") { + EXPECT_TRUE(!seen2) << "node_1 should remain before node_2 after the topological sort."; + seen1 = true; + } else if (node->Name() == "node_2") { + EXPECT_TRUE(seen1) << "node_1 should be before node_2 after the topological sort."; + seen2 = true; + } + } +} + +TEST(ResolvingGraphTest, GraphConstruction_CheckGraphInputOutputOrderMaintained) { + ASSERT_TRUE(kSchemasRegistered); + + Model model("graph_1"); + auto& graph = model.MainGraph(); + + std::unordered_map map; + + for (auto i = 0; i < 20; ++i) { + map.insert({std::to_string(i), i}); + + std::cout << "Insert " << i << "\n"; + for (auto pair : map) { + std::cout << pair.first << ":" << pair.second << " "; + } + std::cout << "\n"; + } + + // | | + // b (Identity) a (Identity) values + // \ / + // c (Merge) + // | + // d (Split) + // / \ + // 1 .. 10 + std::unordered_map, std::vector>> + expected_node_name_to_input_output_args; + + TypeProto tensor_int32; + tensor_int32.mutable_tensor_type()->set_elem_type(TensorProto_DataType_INT32); + tensor_int32.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + + auto& input_arg_a = graph.GetOrCreateNodeArg("node_a_in_1", &tensor_int32); + auto& output_arg_a = graph.GetOrCreateNodeArg("node_a_out_1", &tensor_int32); + + auto& input_arg_b = graph.GetOrCreateNodeArg("node_b_in_1", &tensor_int32); + auto& output_arg_b = graph.GetOrCreateNodeArg("node_b_out_1", &tensor_int32); + + auto& output_arg_c = graph.GetOrCreateNodeArg("node_c_out_1", &tensor_int32); + + std::vector split_outputs; + for (int i = 0; i < 10; ++i) { + split_outputs.push_back(&graph.GetOrCreateNodeArg("node_d_out_" + std::to_string(i + 1), &tensor_int32)); + } + + std::vector inputs; + std::vector outputs; + + inputs.push_back(&input_arg_a); + outputs.push_back(&output_arg_a); + expected_node_name_to_input_output_args["a"] = {inputs, outputs}; + graph.AddNode("a", "Identity_Fake", "a", inputs, outputs); + + inputs.resize(2); + inputs[0] = &output_arg_b; + inputs[1] = &output_arg_a; + outputs[0] = &output_arg_c; + expected_node_name_to_input_output_args["c"] = {inputs, outputs}; + graph.AddNode("c", "Merge_Fake", "c", inputs, outputs); + + // deliberately add 'b' after 'c' to mix up the inputs as well + inputs.resize(1); + inputs[0] = &input_arg_b; + outputs[0] = &output_arg_b; + expected_node_name_to_input_output_args["b"] = {inputs, outputs}; + graph.AddNode("b", "Identity_Fake", "b", inputs, outputs); + + inputs[0] = &output_arg_c; + expected_node_name_to_input_output_args["d"] = {inputs, split_outputs}; + graph.AddNode("d", "Split_Fake", "d", inputs, split_outputs); + + auto validate_inputs_outputs = [&split_outputs](const Graph& graph) { + auto inputs = graph.GetInputs(); + auto outputs = graph.GetOutputs(); + + ASSERT_TRUE(inputs.size() == 2); + + EXPECT_TRUE(inputs[0]->Name() == "node_a_in_1"); // 'a' was added first + EXPECT_TRUE(inputs[1]->Name() == "node_b_in_1"); + + ASSERT_TRUE(outputs.size() == 10); + for (int i = 0; i < 10; ++i) { + EXPECT_TRUE(split_outputs[i]->Name() == outputs[i]->Name()); + } + }; + + auto status = graph.Resolve(); + ASSERT_TRUE(status.IsOK()) << status.ErrorMessage(); + + validate_inputs_outputs(graph); + + // serialize and reload so we check the loaded from proto path in SetGraphInputsOutputs + auto proto = model.ToProto(); + std::stringstream s1; + model.ToProto().SerializeToOstream(&s1); + + ModelProto model_proto; + const bool result = model_proto.ParseFromIstream(&s1); + ASSERT_TRUE(result) << "Failed to load model from serialized protobuf"; + + std::shared_ptr p_tmp_model; + auto x = onnxruntime::Model::Load(model_proto, p_tmp_model, nullptr); + + auto& graph2 = p_tmp_model->MainGraph(); + status = graph2.Resolve(); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + + validate_inputs_outputs(graph2); +} + +// Validate that an unused initializer doesn't break graph loading/resolution +// and is removed as expected. +TEST(ResolvingGraphTest, UnusedInitializerIsIgnored) { + OPERATOR_SCHEMA(Identity_Fake) + .SetDoc("Identity.") + .Input(0, "input_1", "docstr for input_1.", "tensor(int32)") + .Output(0, "output_1", "docstr for output_1.", "tensor(int32)"); + + Model model("UnusedInitializerIsIgnored"); + auto& graph = model.MainGraph(); + + std::vector inputs; + std::vector outputs; + + TypeProto tensor_int32; + tensor_int32.mutable_tensor_type()->set_elem_type(TensorProto_DataType_INT32); + tensor_int32.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + + auto& input_arg_a = graph.GetOrCreateNodeArg("node_a_in_1", &tensor_int32); + auto& output_arg_a = graph.GetOrCreateNodeArg("node_a_out_1", &tensor_int32); + + inputs.push_back(&input_arg_a); + outputs.push_back(&output_arg_a); + graph.AddNode("a", "Identity_Fake", "a", inputs, outputs); + + TensorProto initializer_tensor; + initializer_tensor.set_name("unused"); + initializer_tensor.add_dims(1); + initializer_tensor.add_float_data(1.f); + initializer_tensor.set_data_type(onnx::TensorProto_DataType_FLOAT); + + graph.AddInitializedTensor(initializer_tensor); + ASSERT_TRUE(graph.GetAllInitializedTensors().size() == 1); + + auto status = graph.Resolve(); + ASSERT_TRUE(status.IsOK()) << status.ErrorMessage(); + ASSERT_TRUE(graph.GetAllInitializedTensors().empty()); + + // serialize and reload so we check the loaded from proto path in SetGraphInputsOutputs + auto proto = model.ToProto(); + std::stringstream s1; + model.ToProto().SerializeToOstream(&s1); + + ModelProto model_proto; + const bool result = model_proto.ParseFromIstream(&s1); + ASSERT_TRUE(result) << "Failed to load model from serialized protobuf"; + + std::shared_ptr p_tmp_model; + auto x = onnxruntime::Model::Load(model_proto, p_tmp_model, nullptr); + + auto& graph2 = p_tmp_model->MainGraph(); + status = graph2.Resolve(); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + ASSERT_TRUE(graph.GetAllInitializedTensors().empty()); +} + +TEST(ResolvingGraphTest, GraphConstruction_CheckIsNotAcyclic) { + // A cyclic graph + // SouceNode + // | + // --> node_1 (Add) + // ^ | + // | <- node_2 (NoOp) + + ASSERT_TRUE(kSchemasRegistered); + + std::vector inputs; + std::vector outputs; + + TypeProto tensor_int32; + tensor_int32.mutable_tensor_type()->set_elem_type(TensorProto_DataType_INT32); + tensor_int32.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + + Model model("graph_1"); + auto& graph = model.MainGraph(); + auto& input_arg1 = graph.GetOrCreateNodeArg("node_1_in_1", &tensor_int32); + auto& output_arg1 = graph.GetOrCreateNodeArg("node_1_out_1", &tensor_int32); + auto& output_arg2 = graph.GetOrCreateNodeArg("node_2_out_1", &tensor_int32); + inputs.push_back(&input_arg1); + inputs.push_back(&output_arg2); + outputs.push_back(&output_arg1); + graph.AddNode("node_1", "Add_Fake", "node 1", inputs, outputs); + + inputs.clear(); + inputs.push_back(&output_arg1); + outputs.clear(); + outputs.push_back(&output_arg2); + graph.AddNode("node_2", "NoOp_Fake", "node 2", inputs, outputs); + + auto status = graph.Resolve(); + EXPECT_FALSE(status.IsOK()); + EXPECT_EQ("Error: the graph is not acyclic.", status.ErrorMessage()); +} + +TEST(ResolvingGraphTest, GraphConstruction_OnlyInitializer) { + onnxruntime::Model model("graph"); + auto& graph = model.MainGraph(); + + ONNX_NAMESPACE::TensorProto weight; + weight.add_dims(1); + weight.set_data_type(TensorProto_DataType_STRING); + weight.add_string_data("test"); + weight.set_name("node_1_in_2"); + graph.AddInitializedTensor(weight); + + auto status = graph.Resolve(); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + + auto& iii = graph.GetInputsIncludingInitializers(); + EXPECT_TRUE(iii.size() == 1); + EXPECT_TRUE(iii.front()->Name() == "node_1_in_2"); +} + +TEST(ResolvingGraphTest, GraphConstruction_TypeInference) { + ASSERT_TRUE(kSchemasRegistered); + + Model model("graph_1"); + auto& graph = model.MainGraph(); + + // Case 1: A normal graph. + // SourceNode + // / | \ + // node_1 (Variable) node_2 (Variable) node_3 (Variable) + // \ | / (it's all 3 nodes above outputs to the one input of node_4) + // node_4 (Max) + // | + // SinkNode + std::vector inputs; + std::vector outputs; + + TypeProto tensor_int32; + tensor_int32.mutable_tensor_type()->set_elem_type(TensorProto_DataType_INT32); + tensor_int32.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + + auto& input_arg = graph.GetOrCreateNodeArg("node_1_in_1", &tensor_int32); + inputs.push_back(&input_arg); + auto& output_arg = graph.GetOrCreateNodeArg("node_1_out_1", &tensor_int32); + outputs.push_back(&output_arg); + graph.AddNode("node_1", "Variable2_Fake", "node 1", inputs, outputs); + + inputs.clear(); + inputs.push_back(&input_arg); + auto& output_arg2 = graph.GetOrCreateNodeArg("node_2_out_1", &tensor_int32); + outputs.clear(); + outputs.push_back(&output_arg2); + graph.AddNode("node_2", "Variable2_Fake", "node 2", inputs, outputs); + + auto& input_arg3 = graph.GetOrCreateNodeArg("node_3_in_1", &tensor_int32); + inputs.clear(); + inputs.push_back(&input_arg3); + auto& output_arg3 = graph.GetOrCreateNodeArg("node_3_out_1", &tensor_int32); + outputs.clear(); + outputs.push_back(&output_arg3); + graph.AddNode("node_3", "Variable2_Fake", "node 3", inputs, outputs); + + inputs.clear(); + inputs.push_back(&output_arg); + inputs.push_back(&output_arg2); + inputs.push_back(&output_arg3); + auto& output_arg4 = graph.GetOrCreateNodeArg("node_4_out_1", &tensor_int32); + outputs.clear(); + outputs.push_back(&output_arg4); + auto node_4 = graph.AddNode("node_4", "Max_Fake", "node 4", inputs, outputs); + EXPECT_NE(node_4, nullptr); + auto status = graph.Resolve(); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + + std::unordered_set expected_graph_inputs = {"node_1_in_1", "node_3_in_1"}; + EXPECT_EQ(2, graph.GetInputs().size()); + for (auto& graph_input : graph.GetInputs()) { + EXPECT_TRUE(expected_graph_inputs.find(graph_input->Name()) != expected_graph_inputs.end()); + } + EXPECT_EQ(1, graph.GetOutputs().size()); + EXPECT_EQ("node_4_out_1", graph.GetOutputs()[0]->Name()); + EXPECT_EQ(2, graph.GetInputs().size()); + + EXPECT_TRUE(Model::Save(model, "model_x.pb").IsOK()); + std::shared_ptr loaded_model; + EXPECT_TRUE(Model::Load("model_x.pb", loaded_model).IsOK()); + EXPECT_EQ(2, loaded_model->MainGraph().GetInputs().size()); + + auto& graph_proto = graph.ToGraphProto(); + EXPECT_EQ(2, graph_proto.input_size()); + for (auto& graphProtoInput : graph_proto.input()) { + EXPECT_TRUE(expected_graph_inputs.find(graphProtoInput.name()) != expected_graph_inputs.end()); + } + EXPECT_EQ(1, graph_proto.output_size()); + EXPECT_EQ("node_4_out_1", graph_proto.output(0).name()); +} + +TEST(TestAddAttribute, AddTensorAttribute) { + OPERATOR_SCHEMA(__Constant) + .SetDoc("Constant Op.") + .Attr(kConstantValue, "constant value", AttrType::AttributeProto_AttributeType_TENSOR) + .Output(0, "output_1", "docstr for output_1.", "tensor(int64)"); + std::vector inputs; + std::vector outputs; + Model model("graph_1"); + auto& graph = model.MainGraph(); + TypeProto output_type; + output_type.mutable_tensor_type()->set_elem_type(TensorProto_DataType_INT64); + TensorShapeProto output_shape; + output_shape.mutable_dim()->Add()->set_dim_value(1); + output_shape.mutable_dim()->Add()->set_dim_value(3); + *(output_type.mutable_tensor_type()->mutable_shape()) = output_shape; + auto& output_arg = graph.GetOrCreateNodeArg("node_1_out_1", &output_type); + outputs.push_back(&output_arg); + auto node_1 = graph.AddNode("node_1", "__Constant", "node 1.", inputs, outputs); + TensorProto t; + t.set_data_type(TensorProto_DataType_INT64); + *(t.mutable_int64_data()->Add()) = 1; + *(t.mutable_int64_data()->Add()) = 2; + *(t.mutable_int64_data()->Add()) = 3; + *(t.mutable_dims()->Add()) = 1; + *(t.mutable_dims()->Add()) = 3; + node_1->AddAttribute(kConstantValue, t); + auto status = graph.Resolve(); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); +} + +void AddAttribute(onnxruntime::Node* p_node, const std::string& attr_name, int64_t attr_value) { + AttributeProto attr; + attr.set_name(attr_name); + attr.set_type(AttributeProto_AttributeType_INT); + attr.set_i(attr_value); + p_node->AddAttribute(attr_name, attr); +} + +void AddAttribute(onnxruntime::Node* p_node, const std::string& attr_name, std::initializer_list attr_value) { + AttributeProto attr; + attr.set_name(attr_name); + attr.set_type(AttributeProto_AttributeType_INTS); + for (auto v : attr_value) { + attr.add_ints(v); + } + p_node->AddAttribute(attr_name, attr); +} + +// Test that output type can be inferred for ops with a type-attribute +TEST(TypeInferenceTest, TypeAttribute) { + std::vector inputs; + std::vector outputs; + Model model("graph_1"); + auto& graph = model.MainGraph(); + auto& output_arg = graph.GetOrCreateNodeArg("node_1_out_1", nullptr); + outputs.push_back(&output_arg); + auto node_1 = graph.AddNode("node_1", "RandomNormal", "node 1.", inputs, outputs); + AddAttribute(node_1, "dtype", TensorProto_DataType_FLOAT); + AddAttribute(node_1, "shape", {2, 3}); + auto status = graph.Resolve(); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); +} + +void CheckTensorEltType(const TypeProto* ptype, TensorProto_DataType elt_type) { + EXPECT_NE(ptype, nullptr); + EXPECT_TRUE(ptype->has_tensor_type()); + EXPECT_TRUE(ptype->tensor_type().has_elem_type()); + EXPECT_EQ(ptype->tensor_type().elem_type(), elt_type); +} + +// Test that output type can be inferred for ops with variadic outputs +TEST(TypeInferenceTest, VariadicOutput) { + std::vector inputs; + std::vector outputs; + TypeProto tensor_type; + tensor_type.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + Model model("graph_1"); + auto& graph = model.MainGraph(); + auto& X = graph.GetOrCreateNodeArg("X", &tensor_type); + inputs.push_back(&X); + auto& Y = graph.GetOrCreateNodeArg("Y", nullptr); + outputs.push_back(&Y); + auto& Z = graph.GetOrCreateNodeArg("Z", nullptr); + outputs.push_back(&Z); + graph.AddNode("node_1", "Split", "node 1.", inputs, outputs); + auto status = graph.Resolve(); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + CheckTensorEltType(Y.TypeAsProto(), TensorProto_DataType_FLOAT); + CheckTensorEltType(Z.TypeAsProto(), TensorProto_DataType_FLOAT); +} + +// Test that Graph::Resolve identifies name-duplication across initializer and node-output-arg +TEST(NameResolutionTest, DuplicateName) { + Model model("graph_1"); + auto& graph = model.MainGraph(); + + ONNX_NAMESPACE::TensorProto weight; + weight.set_data_type(TensorProto_DataType_FLOAT); + weight.add_dims(1); + weight.add_float_data(1.0f); + weight.set_name("W"); + graph.AddInitializedTensor(weight); + + std::vector inputs; + std::vector outputs; + TypeProto tensor_type; + tensor_type.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + tensor_type.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(2); + auto& X = graph.GetOrCreateNodeArg("X", &tensor_type); + inputs.push_back(&X); + auto& Y = graph.GetOrCreateNodeArg("Y", nullptr); + outputs.push_back(&Y); + auto& W = graph.GetOrCreateNodeArg("W", nullptr); + outputs.push_back(&W); + graph.AddNode("node_1", "Split", "node 1.", inputs, outputs); + + auto status = graph.Resolve(); + EXPECT_FALSE(status.IsOK()); + bool duplicate_error_found = status.ErrorMessage().find("Duplicate") != std::string::npos; + EXPECT_TRUE(duplicate_error_found); +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/ir/graph_transform_test.cc b/onnxruntime/test/ir/graph_transform_test.cc new file mode 100644 index 0000000000000..3d1c754dce497 --- /dev/null +++ b/onnxruntime/test/ir/graph_transform_test.cc @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/inference_session.h" +#include "core/graph/graph.h" +#include "core/graph/model.h" +#include "core/graph/graph_transformer.h" +#include "core/graph/identity_elimination.h" +#include "core/graph/unsqueeze_elimination.h" +#include "core/graph/conv_bn_fusion.h" +#include "core/graph/conv_mul_fusion.h" +#include "core/graph/conv_add_fusion.h" + +#include "test/capturing_sink.h" +#include "test/test_environment.h" +#include "gtest/gtest.h" + +using namespace std; +using namespace ONNX_NAMESPACE; + +using namespace onnx; + +namespace onnxruntime { +namespace test { + +static const std::string MODEL_FOLDER = "testdata/transform/"; + +TEST(GraphTransformationTests, IdentityElimination) { + string model_uri = MODEL_FOLDER + "abs-id-max.onnx"; + + SessionOptions so; + so.session_logid = "GraphTransformationTests.LoadModelToTransform"; + InferenceSession session_object{so, &DefaultLoggingManager()}; + ASSERT_TRUE(session_object.Load(model_uri).IsOK()); + + std::shared_ptr p_model; + ASSERT_TRUE(Model::Load(model_uri, p_model).IsOK()); + //Graph& p_graph = p_model->MainGraph(); + + std::unique_ptr rule_transformer = + std::make_unique("RuleTransformer1", "First rule transformer"); + + rule_transformer->Register("Identity", std::make_unique()); + + session_object.RegisterGraphTransformer(std::move(rule_transformer)); + + ASSERT_TRUE(session_object.Initialize().IsOK()); +} + +TEST(GraphTransformationTests, FuseConvBNMulAddUnsqueeze) { + string model_uri = MODEL_FOLDER + "fusion/fuse-conv-bn-mul-add-unsqueeze.onnx"; + + SessionOptions so; + so.session_logid = "GraphTransformationTests.LoadModelToTransform"; + InferenceSession session_object{so, &DefaultLoggingManager()}; + ASSERT_TRUE(session_object.Load(model_uri).IsOK()); + + std::shared_ptr p_model; + ASSERT_TRUE(Model::Load(model_uri, p_model).IsOK()); + + std::unique_ptr Unsqueeze_transformer = std::make_unique(); + std::unique_ptr ConvBNFusion_transformer = std::make_unique(); + std::unique_ptr ConvMulFusion_transformer = std::make_unique(); + std::unique_ptr ConvAddFusion_transformer = std::make_unique(); + + session_object.RegisterGraphTransformer(std::move(Unsqueeze_transformer)); + session_object.RegisterGraphTransformer(std::move(ConvBNFusion_transformer)); + session_object.RegisterGraphTransformer(std::move(ConvMulFusion_transformer)); + session_object.RegisterGraphTransformer(std::move(ConvAddFusion_transformer)); + + ASSERT_TRUE(session_object.Initialize().IsOK()); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/ir/onnx_model_test.cc b/onnxruntime/test/ir/onnx_model_test.cc new file mode 100644 index 0000000000000..dfbc75c05dd49 --- /dev/null +++ b/onnxruntime/test/ir/onnx_model_test.cc @@ -0,0 +1,150 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include +#include "core/platform/env.h" +#include "core/graph/graph.h" +#include "core/graph/model.h" +#include "core/graph/op.h" +#include "gtest/gtest.h" + +using namespace onnxruntime; +using namespace ONNX_NAMESPACE; +namespace onnxruntime { +namespace test { +#ifdef ONNXRUNTIME_RUN_EXTERNAL_ONNX_TESTS +// Tests that Resolve() properly clears the state of topological sorted nodes, +// inputs, outputs and valueInfo. +// Assumes the graph passed in has been previously resolved. +static void TestResolve(onnxruntime::Graph& graph) { + GraphViewer graph_viewer(graph); + auto& nodes_before = graph_viewer.GetNodesInTopologicalOrder(); + auto& inputs_before = graph.GetInputs(); + auto& outputs_before = graph.GetOutputs(); + auto& value_info_before = graph.GetValueInfo(); + + // Touch the graph to force Resolve() to recompute. + graph.SetGraphResolveNeeded(); + graph.SetGraphProtoSyncNeeded(); + EXPECT_TRUE(graph.Resolve().IsOK()); + + GraphViewer graph_viewer_2(graph); + auto& nodes_after = graph_viewer_2.GetNodesInTopologicalOrder(); + auto& inputs_after = graph.GetInputs(); + auto& outputs_after = graph.GetOutputs(); + auto& value_info_after = graph.GetValueInfo(); + + // Multiple calls to Resolve() should not alter the sorted nodes, + // inputs, outputs and valueInfo. The internal state should be + // cleared. + EXPECT_EQ(nodes_before, nodes_after); + EXPECT_EQ(inputs_before, inputs_after); + EXPECT_EQ(outputs_before, outputs_after); + EXPECT_EQ(value_info_before, value_info_after); +} + +TEST(ONNXModelsTest, squeeze_net) { + // NOTE: this requires the current directory to be where onnxruntime_ir_UT.exe is located + std::shared_ptr model; + ASSERT_TRUE(Model::Load("../models/opset8/test_squeezenet/model.onnx", model).IsOK()); + TestResolve(model->MainGraph()); +#ifdef _WIN32 + // wstring version + std::shared_ptr model2; + ASSERT_TRUE(Model::Load(L"../models/opset8/test_squeezenet/model.onnx", model2).IsOK()); + TestResolve(model2->MainGraph()); +#endif +} +#endif + +TEST(ONNXModelsTest, non_existing_model) { + // NOTE: this requires the current directory to be where onnxruntime_ir_UT.exe is located + std::shared_ptr model; + common::Status st = Model::Load("./testdata/non_existing_model_XXXXXX/model.onnx", model); + ASSERT_FALSE(st.IsOK()); + ASSERT_EQ(st.Code(), common::NO_SUCHFILE); +#ifdef _WIN32 + // wstring version + std::shared_ptr model2; + ASSERT_FALSE(Model::Load(L"./testdata/non_existing_model_XXXXXX/model.onnx", model2).IsOK()); + ASSERT_EQ(st.Code(), common::NO_SUCHFILE); +#endif +} + +#ifdef ONNXRUNTIME_RUN_EXTERNAL_ONNX_TESTS +TEST(ONNXModelsTest1, bvlc_alexnet_1) { + using ::google::protobuf::io::CodedInputStream; + using ::google::protobuf::io::FileInputStream; + using ::google::protobuf::io::ZeroCopyInputStream; + int fd; + ASSERT_TRUE(Env::Default().FileOpenRd("../models/opset8/test_bvlc_alexnet/model.onnx", fd).IsOK()); + ASSERT_TRUE(fd > 0); + std::unique_ptr raw_input(new FileInputStream(fd)); + std::unique_ptr coded_input(new CodedInputStream(raw_input.get())); + // Allows protobuf library versions < 3.2.0 to parse messages greater than 64MB. + coded_input->SetTotalBytesLimit(INT_MAX, INT_MAX); + ModelProto model_proto; + bool result = model_proto.ParseFromCodedStream(coded_input.get()); + coded_input.reset(); + raw_input.reset(); + EXPECT_TRUE(result); + ASSERT_TRUE(Env::Default().FileClose(fd).IsOK()); + + std::shared_ptr model; + ASSERT_TRUE(Model::Load("../models/opset8/test_bvlc_alexnet/model.onnx", model).IsOK()); + + // Check the graph input/output/value_info should have the same size as specified in the model file. + EXPECT_EQ(model_proto.graph().value_info_size(), model->MainGraph().GetValueInfo().size()); + EXPECT_EQ(model_proto.graph().input_size(), model->MainGraph().GetInputs().size() + model->MainGraph().GetAllInitializedTensors().size()); + EXPECT_EQ(model_proto.graph().output_size(), model->MainGraph().GetOutputs().size()); + TestResolve(model->MainGraph()); +} + +class ONNXModelsTest : public ::testing::TestWithParam { + // You can implement all the usual fixture class members here. + // To access the test parameter, call GetParam() from class + // TestWithParam. + public: + std::string GetModelFileName() const { + std::ostringstream oss; + oss << "../models/opset7/test_" << GetParam() << "/model.onnx"; + return oss.str(); + } +}; + +TEST_P(ONNXModelsTest, LoadFromFile) { + std::shared_ptr model; + ASSERT_TRUE(Model::Load(GetModelFileName(), model).IsOK()); + TestResolve(model->MainGraph()); +} + +TEST_P(ONNXModelsTest, LoadFromProtobuf) { + using ::google::protobuf::io::CodedInputStream; + using ::google::protobuf::io::FileInputStream; + using ::google::protobuf::io::ZeroCopyInputStream; + int fd; + auto st = Env::Default().FileOpenRd(GetModelFileName(), fd); + ASSERT_TRUE(st.IsOK()) << st.ErrorMessage(); + ASSERT_TRUE(fd > 0); + std::unique_ptr raw_input(new FileInputStream(fd)); + std::unique_ptr coded_input(new CodedInputStream(raw_input.get())); + coded_input->SetTotalBytesLimit(INT_MAX, INT_MAX); + std::unique_ptr model_proto = std::make_unique(); + bool result = model_proto->ParseFromCodedStream(coded_input.get()); + coded_input.reset(); + raw_input.reset(); + ASSERT_TRUE(result); + ASSERT_TRUE(Env::Default().FileClose(fd).IsOK()); + std::shared_ptr model; + ASSERT_TRUE(Model::Load(std::move(model_proto), model).IsOK()); + TestResolve(model->MainGraph()); +} + +INSTANTIATE_TEST_CASE_P(ONNXModelsTests, + ONNXModelsTest, + ::testing::Values("bvlc_alexnet", "bvlc_googlenet", "bvlc_reference_caffenet", "bvlc_reference_rcnn_ilsvrc13", "densenet121", "emotion_ferplus", "inception_v1", "inception_v2", "mnist", "resnet50", "shufflenet", "squeezenet", "tiny_yolov2", "vgg19", "zfnet512")); + +#endif +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/ir/op_reg_test.cc b/onnxruntime/test/ir/op_reg_test.cc new file mode 100644 index 0000000000000..17dc14cb63055 --- /dev/null +++ b/onnxruntime/test/ir/op_reg_test.cc @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include "core/graph/model.h" +#include "core/graph/op.h" +#include "core/graph/onnx_protobuf.h" +#include "gtest/gtest.h" + +using namespace ONNX_NAMESPACE; + +namespace onnxruntime { +namespace test { +TEST(OpRegistrationTest, AffineOp) { + auto op = OpSchemaRegistry::Schema("Affine"); + EXPECT_TRUE(nullptr != op); + size_t input_size = op->inputs().size(); + EXPECT_EQ(input_size, 1); + EXPECT_EQ(op->inputs()[0].GetTypes(), op->outputs()[0].GetTypes()); + size_t attr_size = op->attributes().size(); + EXPECT_EQ(attr_size, 2); + auto attr_alpha = op->attributes().find("alpha")->second; + EXPECT_EQ(attr_alpha.name, "alpha"); + EXPECT_EQ(attr_alpha.type, AttrType::AttributeProto_AttributeType_FLOAT); + auto attr_beta = op->attributes().find("beta")->second; + EXPECT_EQ(attr_beta.name, "beta"); + EXPECT_EQ(attr_beta.type, AttrType::AttributeProto_AttributeType_FLOAT); +} + +TEST(FeatureVectorizerTest, TraditionalMlOpTest) { + Model model("traditionalMl"); + auto& graph = model.MainGraph(); + + // Case: A traditional ml graph. + // SouceNode + // | + // node_1(CastMap) + // (tensor(float)) + // | + // node_5 (FeatureVectorizer) + // | + // SinkNode + + std::vector inputs; + std::vector outputs; + + // Type: tensor(float) + TypeProto tensor_float; + tensor_float.mutable_tensor_type()->set_elem_type(TensorProto::FLOAT); + + // Type: map(int64,float); + TypeProto map_int64_float; + auto map_type = map_int64_float.mutable_map_type(); + map_type->set_key_type(TensorProto::INT64); + auto map_value_type = map_type->mutable_value_type()->mutable_tensor_type(); + map_value_type->set_elem_type(TensorProto::FLOAT); + map_value_type->mutable_shape(); + + NodeArg* input_arg1 = new NodeArg("node_1_in_1", &map_int64_float); + inputs.clear(); + inputs.push_back(input_arg1); + NodeArg* output_arg1 = new NodeArg("node_1_out_1", &tensor_float); + outputs.clear(); + outputs.push_back(output_arg1); + graph.AddNode("node_1", "CastMap", "node 1", inputs, outputs, nullptr, kMLDomain); + + inputs.clear(); + inputs.push_back(output_arg1); + + NodeArg* output_arg4 = new NodeArg("node_4_out_1", &tensor_float); + outputs.clear(); + outputs.push_back(output_arg4); + graph.AddNode("node_4", "FeatureVectorizer", "node 4", inputs, outputs, nullptr, kMLDomain); + auto status = graph.Resolve(); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + + delete input_arg1; + delete output_arg1; + delete output_arg4; +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/ir/op_test.cc b/onnxruntime/test/ir/op_test.cc new file mode 100644 index 0000000000000..3673b5bc54e4b --- /dev/null +++ b/onnxruntime/test/ir/op_test.cc @@ -0,0 +1,252 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/graph/op.h" +#include +#include "core/graph/onnx_protobuf.h" +#include "gtest/gtest.h" +#include "core/graph/schema_registry.h" + +using namespace ONNX_NAMESPACE; + +#ifdef __GNUC__ +#define UNUSED __attribute__((unused)) +#else +#define UNUSED +#endif + +#define OPERATOR_SCHEMA UNUSED ONNX_OPERATOR_SCHEMA + +namespace onnxruntime { +namespace test { +TEST(FormalParamTest, Success) { + OpSchema::FormalParameter p("input", "desc: integer input", "tensor(int32)"); + EXPECT_EQ("input", p.GetName()); + EXPECT_EQ("tensor(int32)", p.GetTypeStr()); + EXPECT_EQ("desc: integer input", p.GetDescription()); + // TODO: change onnx to make formal parameter construction self-contain. + //EXPECT_EQ(Utils::DataTypeUtils::ToType("tensor(int32)"), *p.GetTypes().begin()); +} + +TEST(OpRegistrationTest, OpRegTest) { + OPERATOR_SCHEMA(__TestOpReg) + .SetDoc("Op Registration Basic Test.") + .Input(0, "input_1", "docstr for input_1.", "tensor(int32)") + .Input(1, "input_2", "docstr for input_2.", "tensor(int32)") + .Output(0, "output_1", "docstr for output_1.", "tensor(int32)"); + const OpSchema* op_schema = OpSchemaRegistry::Schema("__TestOpReg"); + EXPECT_TRUE(nullptr != op_schema); + EXPECT_EQ(op_schema->inputs().size(), 2); + EXPECT_EQ(op_schema->inputs()[0].GetName(), "input_1"); + EXPECT_EQ(op_schema->inputs()[0].GetTypes().size(), 1); + EXPECT_EQ(**op_schema->inputs()[0].GetTypes().find(Utils::DataTypeUtils::ToType("tensor(int32)")), "tensor(int32)"); + EXPECT_EQ(op_schema->inputs()[1].GetName(), "input_2"); + EXPECT_EQ(op_schema->inputs()[1].GetTypes().size(), 1); + EXPECT_EQ(**op_schema->inputs()[1].GetTypes().find(Utils::DataTypeUtils::ToType("tensor(int32)")), "tensor(int32)"); + EXPECT_EQ(op_schema->outputs().size(), 1); + EXPECT_EQ(op_schema->outputs()[0].GetName(), "output_1"); + EXPECT_EQ(op_schema->outputs()[0].GetTypes().size(), 1); + EXPECT_EQ(**op_schema->outputs()[0].GetTypes().find(Utils::DataTypeUtils::ToType("tensor(int32)")), "tensor(int32)"); +} + +ONNX_NAMESPACE::OpSchema CreateTestSchema(const char* name, const char* domain, int sinceVersion) { + return ONNX_NAMESPACE::OpSchema().SetName(name).SinceVersion(sinceVersion).SetDomain(domain).Output(0, "output_1", "docstr for output", "tensor(int32)"); +} + +TEST(OpRegistrationTest, OpsetRegTest) { + std::shared_ptr registry = std::make_shared(); + + // Register op-set version 1 with baseline version 0 + std::vector schema = {CreateTestSchema("Op1", "Domain1", 1), CreateTestSchema("Op2", "Domain1", 1)}; + EXPECT_TRUE(registry->RegisterOpSet(schema, "Domain1", 0, 1).IsOK()); + + // Get the schema + EXPECT_TRUE(registry->GetSchema("Op1", 1, "Domain1")->Name() == "Op1"); + EXPECT_TRUE(registry->GetSchema("Op2", 1, "Domain1")->Name() == "Op2"); + + // Getting schema with wrong name, domain, and version will fail + EXPECT_TRUE(registry->GetSchema("Op1", 1, "WrongDomain") == nullptr); + EXPECT_TRUE(registry->GetSchema("WrongOp", 1, "Domain1") == nullptr); + EXPECT_TRUE(registry->GetSchema("Op1", 2, "Domain1") == nullptr); + EXPECT_TRUE(registry->GetSchema("Op1", 0, "Domain1") == nullptr); + + // Registering a new op-set in the same domain will fail. This (currently) requires the caller to + // use multiple registry instances and a registry manager. + std::vector schemaV2 = {CreateTestSchema("Op1", "Domain1", 2)}; + EXPECT_FALSE(registry->RegisterOpSet(schemaV2, "Domain1", 1, 2).IsOK()); + + // Registering an op-set with schema in a different domain than the op-set will fail + std::shared_ptr temp_reg = std::make_shared(); + EXPECT_FALSE(temp_reg->RegisterOpSet(schema, "WrongDomain", 0, 1).IsOK()); + + // Registering a second op-set in a different domain should succeed + std::vector schemaDomain2 = {CreateTestSchema("Op2", "Domain2", 1)}; + EXPECT_TRUE(registry->RegisterOpSet(schemaDomain2, "Domain2", 0, 1).IsOK()); + EXPECT_TRUE(registry->GetSchema("Op1", 1, "Domain1")->Name() == "Op1"); + EXPECT_TRUE(registry->GetSchema("Op2", 1, "Domain2")->Name() == "Op2"); + + // Overriding existing op-set schema will fail + std::vector schemaOverride = {CreateTestSchema("Op1", "Domain1", 1)}; + EXPECT_FALSE(registry->RegisterOpSet(schema, "Domain1", 0, 1).IsOK()); + + // Create a second registry, combined with the first through a manager + std::shared_ptr registry2 = std::make_shared(); + SchemaRegistryManager manager; + manager.RegisterRegistry(registry); + manager.RegisterRegistry(registry2); + + // Register the second version of the same op-set on the second registry, overriding one operator + EXPECT_TRUE(registry2->RegisterOpSet(schemaV2, "Domain1", 1, 2).IsOK()); + EXPECT_TRUE(manager.GetSchema("Op1", 1, "Domain1")->since_version() == 1); + EXPECT_TRUE(manager.GetSchema("Op1", 2, "Domain1")->since_version() == 2); + EXPECT_TRUE(manager.GetSchema("Op2", 1, "Domain1")->since_version() == 1); + + // Op2 is provided only in opset v1, and in the first registry. The absence of Op2 in the second + // registry will trigger the first registry to be queried using V1 rather than V2 here. + EXPECT_TRUE(manager.GetSchema("Op2", 2, "Domain1")->since_version() == 1); + + // Add a new operator set which is verion 5, with a baseline of version 4, meaning that + // there is a gap at version 3. + std::shared_ptr registryV5 = std::make_shared(); + manager.RegisterRegistry(registryV5); + std::vector schemaV5 = { + CreateTestSchema("Op3", "Domain1", 4), + CreateTestSchema("Op4", "Domain1", 5), + CreateTestSchema("Op5", "Domain1", 1)}; + EXPECT_TRUE(registryV5->RegisterOpSet(schemaV5, "Domain1", 4, 5).IsOK()); + + // Query the new version 5 op. This will be missing for earlier versions + EXPECT_TRUE(manager.GetSchema("Op4", 5, "Domain1")->since_version() == 5); + EXPECT_TRUE(manager.GetSchema("Op4", 4, "Domain1") == nullptr); + + // The only schema with SinceVersion < 3 which can be queried as version 5 are those which are registered on + // the v5 registry itself. Those schema may be queried for any version between their sinceVersion and the + // opset's version. + EXPECT_TRUE(manager.GetSchema("Op1", 5, "Domain1") == nullptr); + EXPECT_TRUE(manager.GetSchema("Op3", 5, "Domain1")->since_version() == 4); + EXPECT_TRUE(manager.GetSchema("Op3", 4, "Domain1")->since_version() == 4); + + // Note that "Op5" has SinceVersion equal to 1, but a V1 operator set was already registered + // without this operator. This would normally be invalid, and the registry with the missing + // operator could trigger the operator lookup to fail. Version 1 is a special case to allow + // for experimental operators, and is accomplished by not reducing the targetted version to + // zero in OnnxRuntimeOpSchemaRegistry::GetSchemaAndHistory. + // TODO - Consider making the registration algorithm robust to this invalid usage in general + EXPECT_TRUE(manager.GetSchema("Op5", 5, "Domain1")->since_version() == 1); + EXPECT_TRUE(manager.GetSchema("Op5", 1, "Domain1")->since_version() == 1); +} + +TEST(OpRegistrationTest, TypeConstraintTest) { + OPERATOR_SCHEMA(__TestTypeConstraint) + .SetDoc("Op with Type Constraint.") + .Input(0, "input_1", "docstr for input_1.", "T") + .Input(1, "input_2", "docstr for input_2.", "T") + .Output(0, "output_1", "docstr for output_1.", "T") + .TypeConstraint("T", {"tensor(float16)", "tensor(float)", "tensor(double)"}, "Constrain input and output types to floats."); + const OpSchema* op_schema = OpSchemaRegistry::Schema("__TestTypeConstraint"); + EXPECT_TRUE(nullptr != op_schema); + EXPECT_EQ(op_schema->inputs().size(), 2); + EXPECT_EQ(op_schema->inputs()[0].GetName(), "input_1"); + EXPECT_EQ(op_schema->inputs()[0].GetTypes().size(), 3); + EXPECT_EQ(**op_schema->inputs()[0].GetTypes().find(Utils::DataTypeUtils::ToType("tensor(float16)")), "tensor(float16)"); + EXPECT_EQ(**op_schema->inputs()[0].GetTypes().find(Utils::DataTypeUtils::ToType("tensor(float)")), "tensor(float)"); + EXPECT_EQ(**op_schema->inputs()[0].GetTypes().find(Utils::DataTypeUtils::ToType("tensor(double)")), "tensor(double)"); + + EXPECT_EQ(op_schema->inputs()[1].GetName(), "input_2"); + EXPECT_EQ(op_schema->inputs()[1].GetTypes().size(), 3); + EXPECT_EQ(**op_schema->inputs()[1].GetTypes().find(Utils::DataTypeUtils::ToType("tensor(float16)")), "tensor(float16)"); + EXPECT_EQ(**op_schema->inputs()[1].GetTypes().find(Utils::DataTypeUtils::ToType("tensor(float)")), "tensor(float)"); + EXPECT_EQ(**op_schema->inputs()[1].GetTypes().find(Utils::DataTypeUtils::ToType("tensor(double)")), "tensor(double)"); + + EXPECT_EQ(op_schema->outputs().size(), 1); + EXPECT_EQ(op_schema->outputs()[0].GetName(), "output_1"); + EXPECT_EQ(op_schema->outputs()[0].GetTypes().size(), 3); + EXPECT_EQ(**op_schema->outputs()[0].GetTypes().find(Utils::DataTypeUtils::ToType("tensor(float16)")), "tensor(float16)"); + EXPECT_EQ(**op_schema->outputs()[0].GetTypes().find(Utils::DataTypeUtils::ToType("tensor(float)")), "tensor(float)"); + EXPECT_EQ(**op_schema->outputs()[0].GetTypes().find(Utils::DataTypeUtils::ToType("tensor(double)")), "tensor(double)"); +} + +TEST(OpRegistrationTest, AttributeDefaultValueTest) { + OPERATOR_SCHEMA(__TestAttrDefaultValue) + .SetDoc("Op with attributes that have default values") + .Attr("my_attr_int", "attr with default value of 99.", AttrType::AttributeProto_AttributeType_INT, int64_t(99)) + .Attr("my_attr_float", "attr with default value of 0.99.", AttrType::AttributeProto_AttributeType_FLOAT, float(0.99)) + .Attr("my_attr_string", "attr with default value of \"99\".", AttrType::AttributeProto_AttributeType_STRING, std::string("99")); + const OpSchema* op_schema = OpSchemaRegistry::Schema("__TestAttrDefaultValue"); + EXPECT_TRUE(nullptr != op_schema); + EXPECT_EQ(op_schema->attributes().size(), 3); + + auto attr_int = op_schema->attributes().find("my_attr_int")->second; + EXPECT_EQ(attr_int.name, "my_attr_int"); + EXPECT_EQ(attr_int.type, AttrType::AttributeProto_AttributeType_INT); + EXPECT_FALSE(attr_int.required); + EXPECT_EQ(attr_int.default_value.name(), "my_attr_int"); + EXPECT_TRUE(attr_int.default_value.has_i()); + EXPECT_EQ(attr_int.default_value.i(), 99LL); + + auto attr_float = op_schema->attributes().find("my_attr_float")->second; + EXPECT_EQ(attr_float.name, "my_attr_float"); + EXPECT_EQ(attr_float.type, AttrType::AttributeProto_AttributeType_FLOAT); + EXPECT_FALSE(attr_float.required); + EXPECT_EQ(attr_float.default_value.name(), "my_attr_float"); + EXPECT_TRUE(attr_float.default_value.has_f()); + EXPECT_EQ(attr_float.default_value.f(), 0.99f); + + auto attr_string = op_schema->attributes().find("my_attr_string")->second; + EXPECT_EQ(attr_string.name, "my_attr_string"); + EXPECT_EQ(attr_string.type, AttrType::AttributeProto_AttributeType_STRING); + EXPECT_FALSE(attr_string.required); + EXPECT_EQ(attr_string.default_value.name(), "my_attr_string"); + EXPECT_TRUE(attr_string.default_value.has_s()); + EXPECT_EQ(attr_string.default_value.s(), "99"); +} + +TEST(OpRegistrationTest, AttributeDefaultValueListTest) { + OPERATOR_SCHEMA(__TestAttrDefaultValueList) + .SetDoc("Op with attributes that have default list of values.") + .Attr("my_attr_ints", "attr with default value of [98, 99, 100].", AttrType::AttributeProto_AttributeType_INTS, std::vector{int64_t(98), int64_t(99), int64_t(100)}) + .Attr("my_attr_floats", "attr with default value of [0.98, 0.99, 1.00].", AttrType::AttributeProto_AttributeType_FLOATS, std::vector{float(0.98), float(0.99), float(1.00)}) + .Attr("my_attr_strings", "attr with default value of [\"98\", \"99\", \"100\"].", AttrType::AttributeProto_AttributeType_STRINGS, std::vector{"98", "99", "100"}); + const OpSchema* op_schema = OpSchemaRegistry::Schema("__TestAttrDefaultValueList"); + EXPECT_TRUE(nullptr != op_schema); + EXPECT_EQ(op_schema->attributes().size(), 3); + + auto attr_ints = op_schema->attributes().find("my_attr_ints")->second; + EXPECT_EQ(attr_ints.name, "my_attr_ints"); + EXPECT_EQ(attr_ints.type, AttrType::AttributeProto_AttributeType_INTS); + EXPECT_FALSE(attr_ints.required); + EXPECT_EQ(attr_ints.default_value.name(), "my_attr_ints"); + int size = attr_ints.default_value.ints_size(); + EXPECT_EQ(size, 3); + std::vector expected_ints = {98LL, 99LL, 100LL}; + for (int i = 0; i < size; i++) { + EXPECT_EQ(attr_ints.default_value.ints(i), expected_ints[i]); + } + + auto attr = op_schema->attributes().find("my_attr_floats")->second; + EXPECT_EQ(attr.name, "my_attr_floats"); + EXPECT_EQ(attr.type, AttrType::AttributeProto_AttributeType_FLOATS); + EXPECT_FALSE(attr.required); + EXPECT_EQ(attr.default_value.name(), "my_attr_floats"); + size = attr.default_value.floats_size(); + EXPECT_EQ(size, 3); + std::vector expected_floats = {0.98f, 0.99f, 1.00f}; + for (int i = 0; i < size; i++) { + EXPECT_EQ(attr.default_value.floats(i), expected_floats[i]); + } + + auto attr2 = op_schema->attributes().find("my_attr_strings")->second; + EXPECT_EQ(attr2.name, "my_attr_strings"); + EXPECT_EQ(attr2.type, AttrType::AttributeProto_AttributeType_STRINGS); + EXPECT_FALSE(attr2.required); + EXPECT_EQ(attr2.default_value.name(), "my_attr_strings"); + size = attr2.default_value.strings_size(); + EXPECT_EQ(size, 3); + std::vector expected_strings = {"98", "99", "100"}; + for (int i = 0; i < size; i++) { + EXPECT_EQ(attr2.default_value.strings(i), expected_strings[i]); + } +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/ir/test_main.cc b/onnxruntime/test/ir/test_main.cc new file mode 100644 index 0000000000000..ed47ee6a24968 --- /dev/null +++ b/onnxruntime/test/ir/test_main.cc @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/test_environment.h" +#include "core/graph/constants.h" +#include "core/graph/op.h" + +GTEST_API_ int main(int argc, char** argv) { + int status = 0; + + try { + onnxruntime::test::TestEnvironment test_environment{argc, argv}; + + // Register Microsoft domain with min/max op_set version as 1/1. + ONNX_NAMESPACE::OpSchemaRegistry::DomainToVersionRange::Instance().AddDomainToVersion(onnxruntime_ir::kMSDomain, 1, 1); + + // Register Microsoft domain ops. + onnxruntime_ir::MsOpRegistry::RegisterMsOps(); + + status = RUN_ALL_TESTS(); + } catch (const std::exception& ex) { + std::cerr << ex.what(); + status = -1; + } + + return status; +} diff --git a/onnxruntime/test/ir/utils_test.cc b/onnxruntime/test/ir/utils_test.cc new file mode 100644 index 0000000000000..1f46ee01de648 --- /dev/null +++ b/onnxruntime/test/ir/utils_test.cc @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "core/graph/onnx_protobuf.h" + +using ONNX_NAMESPACE::Utils::DataTypeUtils; +using namespace ONNX_NAMESPACE; + +namespace onnxruntime { +namespace test { + +TEST(OpUtilsTest, TestPTYPE) { + DataType p1 = DataTypeUtils::ToType("tensor(int32)"); + DataType p2 = DataTypeUtils::ToType("tensor(int32)"); + DataType p3 = DataTypeUtils::ToType("tensor(int32)"); + EXPECT_EQ(p1, p2); + EXPECT_EQ(p2, p3); + EXPECT_EQ(p1, p3); + DataType p4 = DataTypeUtils::ToType("seq(tensor(int32))"); + DataType p5 = DataTypeUtils::ToType("seq(tensor(int32))"); + DataType p6 = DataTypeUtils::ToType("seq(tensor(int32))"); + EXPECT_EQ(p4, p5); + EXPECT_EQ(p5, p6); + EXPECT_EQ(p4, p6); + + TypeProto t1 = DataTypeUtils::ToTypeProto(p1); + EXPECT_TRUE(t1.has_tensor_type()); + EXPECT_TRUE(t1.tensor_type().has_elem_type()); + EXPECT_EQ(t1.tensor_type().elem_type(), TensorProto_DataType::TensorProto_DataType_INT32); + TypeProto t2 = DataTypeUtils::ToTypeProto(p2); + EXPECT_TRUE(t2.has_tensor_type()); + EXPECT_TRUE(t2.tensor_type().has_elem_type()); + EXPECT_EQ(t2.tensor_type().elem_type(), TensorProto_DataType::TensorProto_DataType_INT32); + TypeProto t3 = DataTypeUtils::ToTypeProto(p3); + EXPECT_TRUE(t3.has_tensor_type()); + EXPECT_TRUE(t3.tensor_type().has_elem_type()); + EXPECT_EQ(t3.tensor_type().elem_type(), TensorProto_DataType::TensorProto_DataType_INT32); + TypeProto t4 = DataTypeUtils::ToTypeProto(p4); + EXPECT_TRUE(t4.has_sequence_type()); + EXPECT_TRUE(t4.sequence_type().has_elem_type()); + EXPECT_TRUE(t4.sequence_type().elem_type().has_tensor_type()); + EXPECT_TRUE(t4.sequence_type().elem_type().tensor_type().has_elem_type()); + EXPECT_EQ(t4.sequence_type().elem_type().tensor_type().elem_type(), TensorProto_DataType::TensorProto_DataType_INT32); + TypeProto t5 = DataTypeUtils::ToTypeProto(p5); + EXPECT_TRUE(t5.has_sequence_type()); + EXPECT_TRUE(t5.sequence_type().has_elem_type()); + EXPECT_TRUE(t5.sequence_type().elem_type().has_tensor_type()); + EXPECT_TRUE(t5.sequence_type().elem_type().tensor_type().has_elem_type()); + EXPECT_EQ(t5.sequence_type().elem_type().tensor_type().elem_type(), TensorProto_DataType::TensorProto_DataType_INT32); + TypeProto t6 = DataTypeUtils::ToTypeProto(p6); + EXPECT_TRUE(t6.has_sequence_type()); + EXPECT_TRUE(t6.sequence_type().has_elem_type()); + EXPECT_TRUE(t6.sequence_type().elem_type().has_tensor_type()); + EXPECT_TRUE(t6.sequence_type().elem_type().tensor_type().has_elem_type()); + EXPECT_EQ(t6.sequence_type().elem_type().tensor_type().elem_type(), TensorProto_DataType::TensorProto_DataType_INT32); +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/mlas/unittest.cpp b/onnxruntime/test/mlas/unittest.cpp new file mode 100644 index 0000000000000..2fd079f090ae3 --- /dev/null +++ b/onnxruntime/test/mlas/unittest.cpp @@ -0,0 +1,1245 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + unittest.cpp + +Abstract: + + This module implements unit tests of the MLAS library. + +--*/ + +#include +#include +#include +#include +#include + +#if defined(_WIN32) +#include +#else +#include +#endif + +#if !defined(_countof) +#define _countof(_Array) (sizeof(_Array) / sizeof(_Array[0])) +#endif + +class MatrixGuardBuffer +{ +public: + MatrixGuardBuffer(size_t Elements, bool ReadOnly) + { + Construct(Elements, ReadOnly); + } + + ~MatrixGuardBuffer(void) + { +#if defined(_WIN32) + VirtualFree(_BaseBuffer, 0, MEM_RELEASE); +#else + munmap(_BaseBuffer, _BaseBufferSize); +#endif + } + + float* GetBuffer(size_t Elements) + { + return _GuardAddress - Elements; + } + +private: + void Construct(size_t Elements, bool ReadOnly) + { + const size_t PageSize = 4096; + const size_t GuardPadding = 256 * 1024; + + size_t MatrixSize = Elements * sizeof(float); + size_t AlignedMatrixSize = (MatrixSize + PageSize - 1) & ~(PageSize - 1); + + _BaseBufferSize = AlignedMatrixSize + GuardPadding; + +#if defined(_WIN32) + _BaseBuffer = VirtualAlloc(NULL, _BaseBufferSize, MEM_RESERVE, PAGE_NOACCESS); + VirtualAlloc(_BaseBuffer, AlignedMatrixSize, MEM_COMMIT, PAGE_READWRITE); +#else + _BaseBuffer = mmap(0, _BaseBufferSize, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + mprotect(_BaseBuffer, AlignedMatrixSize, PROT_READ | PROT_WRITE); +#endif + + float* GuardAddress = (float*)((unsigned char*)_BaseBuffer + AlignedMatrixSize); + + const int MinimumFillValue = -23; + const int MaximumFillValue = 23; + + int FillValue = MinimumFillValue; + float* FillAddress = (float*)((unsigned char*)GuardAddress - MatrixSize); + + while (FillAddress < GuardAddress) { + + *FillAddress++ = (float)FillValue; + + FillValue++; + + if (FillValue > MaximumFillValue) { + FillValue = MinimumFillValue; + } + } + + if (ReadOnly) { +#if defined(_WIN32) + DWORD OldProtect; + VirtualProtect(_BaseBuffer, AlignedMatrixSize, PAGE_READONLY, &OldProtect); +#else + mprotect(_BaseBuffer, AlignedMatrixSize, PROT_READ); +#endif + } + + _GuardAddress = GuardAddress; + } + +private: + void* _BaseBuffer; + size_t _BaseBufferSize; + float* _GuardAddress; +}; + +void +ReferenceSgemm( + CBLAS_TRANSPOSE TransA, + CBLAS_TRANSPOSE TransB, + size_t M, + size_t N, + size_t K, + float alpha, + const float* A, + size_t lda, + const float* B, + size_t ldb, + float beta, + float* C, + size_t ldc + ) +{ + if (TransA == CblasNoTrans) { + + if (TransB == CblasNoTrans) { + + for (size_t m = 0; m < M; m++) { + + for (size_t n = 0; n < N; n++) { + + const float* a = A + (m * lda); + const float* b = B + n; + float* c = C + (m * ldc) + n; + float sum = 0.0f; + + for (size_t k = 0; k < K; k++) { + sum += (*b * *a); + b += ldb; + a += 1; + } + + *c = (*c * beta) + (sum * alpha); + } + } + + } else { + + for (size_t m = 0; m < M; m++) { + + for (size_t n = 0; n < N; n++) { + + const float* a = A + (m * lda); + const float* b = B + (n * ldb); + float* c = C + (m * ldc) + n; + float sum = 0.0f; + + for (size_t k = 0; k < K; k++) { + sum += (*b * *a); + b += 1; + a += 1; + } + + *c = (*c * beta) + (sum * alpha); + } + } + } + + } else { + + if (TransB == CblasNoTrans) { + + for (size_t m = 0; m < M; m++) { + + for (size_t n = 0; n < N; n++) { + + const float* a = A + m; + const float* b = B + n; + float* c = C + (m * ldc) + n; + float sum = 0.0f; + + for (size_t k = 0; k < K; k++) { + sum += (*b * *a); + b += ldb; + a += lda; + } + + *c = (*c * beta) + (sum * alpha); + } + } + + } else { + + for (size_t m = 0; m < M; m++) { + + for (size_t n = 0; n < N; n++) { + + const float* a = A + m; + const float* b = B + (n * ldb); + float* c = C + (m * ldc) + n; + float sum = 0.0f; + + for (size_t k = 0; k < K; k++) { + sum += (*b * *a); + b += 1; + a += lda; + } + + *c = (*c * beta) + (sum * alpha); + } + } + } + } +} + +void +TrialSgemm( + CBLAS_TRANSPOSE TransA, + CBLAS_TRANSPOSE TransB, + size_t M, + size_t N, + size_t K, + float alpha, + const float* A, + size_t lda, + const float* B, + size_t ldb, + float beta, + float* C, + float* CReference, + size_t ldc + ) +{ + for (size_t f = 0; f < M * N; f++) { + C[f] = -0.5f; + CReference[f] = -0.5f; + } + + MlasSgemm(TransA, TransB, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); + ReferenceSgemm(TransA, TransB, M, N, K, alpha, A, lda, B, ldb, beta, CReference, ldc); + + for (size_t f = 0; f < M * N; f++) { + // Sensitive to comparing positive/negative zero. + if (C[f] != CReference[f]) { + printf("mismatch TransA=%d, TransB=%d, M=%zd, N=%zd, K=%zd, alpha=%f, beta=%f!\n", TransA, TransB, M, N, K, alpha, beta); + } + } +} + +void +TrialSgemm( + size_t M, + size_t N, + size_t K, + float alpha, + MatrixGuardBuffer& BufferA, + MatrixGuardBuffer& BufferB, + float beta, + MatrixGuardBuffer& BufferC, + MatrixGuardBuffer& BufferCReference + ) +{ + const float* A = BufferA.GetBuffer(K * M); + const float* B = BufferB.GetBuffer(N * K); + float* C = BufferC.GetBuffer(N * M); + float* CReference = BufferCReference.GetBuffer(N * M); + + TrialSgemm(CblasNoTrans, CblasNoTrans, M, N, K, alpha, A, K, B, N, beta, C, CReference, N); + TrialSgemm(CblasNoTrans, CblasTrans, M, N, K, alpha, A, K, B, K, beta, C, CReference, N); + TrialSgemm(CblasTrans, CblasNoTrans, M, N, K, alpha, A, M, B, N, beta, C, CReference, N); + TrialSgemm(CblasTrans, CblasTrans, M, N, K, alpha, A, M, B, K, beta, C, CReference, N); +} + +void +ExecuteSgemmTests( + void + ) +{ + constexpr size_t MaximumDimension = 320; + + MatrixGuardBuffer BufferA(MaximumDimension * MaximumDimension, true); + MatrixGuardBuffer BufferB(MaximumDimension * MaximumDimension, true); + MatrixGuardBuffer BufferC(MaximumDimension * MaximumDimension, false); + MatrixGuardBuffer BufferCReference(MaximumDimension * MaximumDimension, false); + + // Trial balloons. + for (size_t b = 1; b < 16; b++) { + TrialSgemm(b, b, b, 1.0f, BufferA, BufferB, 0.0f, BufferC, BufferCReference); + } + for (size_t b = 16; b <= 256; b <<= 1) { + TrialSgemm(b, b, b, 1.0f, BufferA, BufferB, 0.0f, BufferC, BufferCReference); + } + for (size_t b = 256; b < 320; b += 32) { + TrialSgemm(b, b, b, 1.0f, BufferA, BufferB, 0.0f, BufferC, BufferCReference); + } + + static const float multipliers[] = { 0.0f, -0.0f, 0.25f, -0.5f, 1.0f, -1.0f }; + + for (size_t N = 1; N < 128; N++) { + for (size_t K = 1; K < 128; K++) { + for (size_t a = 0; a < _countof(multipliers); a++) { + for (size_t b = 0; b < _countof(multipliers); b++) { + TrialSgemm(1, N, K, multipliers[a], BufferA, BufferB, multipliers[b], BufferC, BufferCReference); + } + } + } + } + + for (size_t a = 0; a < _countof(multipliers); a++) { + float alpha = multipliers[a]; + + for (size_t b = 0; b < _countof(multipliers); b++) { + float beta = multipliers[b]; + + for (size_t M = 16; M < 160; M += 32) { + for (size_t N = 16; N < 160; N += 32) { + + static const size_t ks[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 20, 32, 48, 64, 118, 119, 120, 121, 122, 160, 240, 320 }; + for (size_t k = 0; k < _countof(ks); k++) { + size_t K = ks[k]; + + TrialSgemm(M, N, K, alpha, BufferA, BufferB, beta, BufferC, BufferCReference); + TrialSgemm(M + 1, N, K, alpha, BufferA, BufferB, beta, BufferC, BufferCReference); + TrialSgemm(M, N + 1, K, alpha, BufferA, BufferB, beta, BufferC, BufferCReference); + TrialSgemm(M + 1, N + 1, K, alpha, BufferA, BufferB, beta, BufferC, BufferCReference); + TrialSgemm(M + 3, N + 2, K, alpha, BufferA, BufferB, beta, BufferC, BufferCReference); + TrialSgemm(M + 4, N, K, alpha, BufferA, BufferB, beta, BufferC, BufferCReference); + TrialSgemm(M, N + 4, K, alpha, BufferA, BufferB, beta, BufferC, BufferCReference); + TrialSgemm(M + 4, N + 4, K, alpha, BufferA, BufferB, beta, BufferC, BufferCReference); + TrialSgemm(M + 3, N + 7, K, alpha, BufferA, BufferB, beta, BufferC, BufferCReference); + TrialSgemm(M + 8, N, K, alpha, BufferA, BufferB, beta, BufferC, BufferCReference); + TrialSgemm(M, N + 8, K, alpha, BufferA, BufferB, beta, BufferC, BufferCReference); + TrialSgemm(M + 12, N + 12, K, alpha, BufferA, BufferB, beta, BufferC, BufferCReference); + TrialSgemm(M + 13, N, K, alpha, BufferA, BufferB, beta, BufferC, BufferCReference); + TrialSgemm(M, N + 15, K, alpha, BufferA, BufferB, beta, BufferC, BufferCReference); + TrialSgemm(M + 15, N + 15, K, alpha, BufferA, BufferB, beta, BufferC, BufferCReference); + } + } + printf("a %zd/%zd b %zd/%zd M %zd\n", a, _countof(multipliers), b, _countof(multipliers), M); + } + } + } + + for (size_t M = 1; M < 160; M++) { + for (size_t N = 1; N < 160; N++) { + for (size_t K = 1; K < 160; K++) { + TrialSgemm(M, N, K, 1.0f, BufferA, BufferB, 0.0f, BufferC, BufferCReference); + } + } + printf("M %zd\n", M); + } + + for (size_t M = 160; M < 320; M += 24) { + for (size_t N = 112; N < 320; N += 24) { + for (size_t K = 1; K < 16; K++) { + TrialSgemm(M, N, K, 1.0f, BufferA, BufferB, 0.0f, BufferC, BufferCReference); + } + for (size_t K = 16; K < 160; K += 32) { + TrialSgemm(M, N, K, 1.0f, BufferA, BufferB, 0.0f, BufferC, BufferCReference); + } + } + printf("M %zd\n", M); + } +} + +void +ReferenceConv2D( + size_t BatchCount, + size_t GroupCount, + size_t InputChannels, + size_t InputHeight, + size_t InputWidth, + size_t FilterCount, + size_t KernelHeight, + size_t KernelWidth, + size_t PaddingLeftHeight, + size_t PaddingLeftWidth, + size_t DilationHeight, + size_t DilationWidth, + size_t StrideHeight, + size_t StrideWidth, + size_t OutputHeight, + size_t OutputWidth, + const float* Input, + const float* Filter, + const float* Bias, + float* Output + ) +{ + size_t InputSize = InputHeight * InputWidth; + size_t OutputSize = OutputHeight * OutputWidth; + size_t KernelSize = KernelHeight * KernelWidth; + + size_t K = InputChannels * KernelSize; + size_t Im2ColBufferElements = OutputSize * K; + + MatrixGuardBuffer BufferIm2Col(Im2ColBufferElements, false); + + for (size_t b = 0; b < BatchCount; b++) { + + const float* filter = Filter; + const float* bias = Bias; + + for (size_t g = 0; g < GroupCount; g++) { + + // + // Transform the image using IM2COL and invoke the GEMM. + // + + float* Im2Col = BufferIm2Col.GetBuffer(Im2ColBufferElements); + float* Im2ColOut = Im2Col; + + for (size_t c = 0; c < InputChannels; c++) { + + for (size_t ky = 0; ky < KernelHeight; ky++) { + + for (size_t kx = 0; kx < KernelWidth; kx++) { + + for (size_t oh = 0; oh < OutputHeight; oh++) { + + size_t ih = oh * StrideHeight + ky * DilationHeight - PaddingLeftHeight; + + for (size_t ow = 0; ow < OutputWidth; ow++) { + + size_t iw = ow * StrideWidth + kx * DilationWidth - PaddingLeftWidth; + + *Im2ColOut++ = (ih < InputHeight && iw < InputWidth) ? + Input[ih * InputWidth + iw] : 0; + } + } + } + } + + Input += InputSize; + } + + MlasSgemm(CblasNoTrans, CblasNoTrans, FilterCount, OutputSize, K, 1.0f, + filter, K, Im2Col, OutputSize, 0.0f, Output, OutputSize); + + // + // Apply the bias. + // + + for (size_t f = 0; f < FilterCount; f++) { + + float biasValue = *bias++; + + for (size_t o = 0; o < OutputSize; o++) { + *Output++ += biasValue; + } + } + + filter += FilterCount * InputChannels * KernelSize; + } + } +} + +void +TrialConv2D( + size_t BatchCount, + size_t GroupCount, + size_t InputChannels, + size_t InputHeight, + size_t InputWidth, + size_t FilterCount, + size_t KernelHeight, + size_t KernelWidth, + size_t PaddingLeftHeight, + size_t PaddingLeftWidth, + size_t PaddingRightHeight, + size_t PaddingRightWidth, + size_t DilationHeight, + size_t DilationWidth, + size_t StrideHeight, + size_t StrideWidth + ) +{ + int64_t OutputHeight64 = + ((int64_t(InputHeight) + int64_t(PaddingLeftHeight) + int64_t(PaddingRightHeight)) - + (int64_t(DilationHeight) * (int64_t(KernelHeight) - 1) + 1)) / int64_t(StrideHeight) + 1; + int64_t OutputWidth64 = + ((int64_t(InputWidth) + int64_t(PaddingLeftWidth) + int64_t(PaddingRightWidth)) - + (int64_t(DilationWidth) * (int64_t(KernelWidth) - 1) + 1)) / int64_t(StrideWidth) + 1; + + if (OutputHeight64 <= 0 || OutputWidth64 <= 0) { + return; + } + + int64_t InputShape[] = { int64_t(InputHeight), int64_t(InputWidth) }; + int64_t KernelShape[] = { int64_t(KernelHeight), int64_t(KernelWidth) }; + int64_t DilationShape[] = { int64_t(DilationHeight), int64_t(DilationWidth) }; + int64_t Padding[] = { int64_t(PaddingLeftHeight), int64_t(PaddingLeftWidth), int64_t(PaddingRightHeight), int64_t(PaddingRightWidth) }; + int64_t StrideShape[] = { int64_t(StrideHeight), int64_t(StrideWidth) }; + + MLAS_CONV_PARAMETERS Parameters; + size_t WorkingBufferSize; + + if (!MlasConvPrepare(&Parameters, + 2, + BatchCount, + GroupCount, + InputChannels, + InputShape, + KernelShape, + DilationShape, + Padding, + StrideShape, + FilterCount, + &WorkingBufferSize)) { + printf("MlasConvPrepare failed!!!\n"); + return; + } + + size_t OutputHeight = size_t(OutputHeight64); + size_t OutputWidth = size_t(OutputWidth64); + + size_t InputSize = InputHeight * InputWidth; + size_t KernelSize = KernelHeight * KernelWidth; + size_t OutputSize = OutputHeight * OutputWidth; + + size_t InputBufferElements = BatchCount * GroupCount * InputChannels * InputSize; + size_t FilterBufferElements = GroupCount * FilterCount * InputChannels * KernelSize; + size_t BiasBufferElements = GroupCount * FilterCount; + size_t OutputBufferElements = BatchCount * GroupCount * FilterCount * OutputSize; + + MatrixGuardBuffer BufferInput(InputBufferElements, true); + MatrixGuardBuffer BufferFilter(FilterBufferElements, true); + MatrixGuardBuffer BufferBias(BiasBufferElements, true); + MatrixGuardBuffer BufferOutput(OutputBufferElements, false); + MatrixGuardBuffer BufferOutputReference(OutputBufferElements, false); + + const float* Input = BufferInput.GetBuffer(InputBufferElements); + const float* Filter = BufferFilter.GetBuffer(FilterBufferElements); + const float* Bias = BufferBias.GetBuffer(BiasBufferElements); + float* Output = BufferOutput.GetBuffer(OutputBufferElements); + float* OutputReference = BufferOutputReference.GetBuffer(OutputBufferElements); + + MatrixGuardBuffer BufferWorking(WorkingBufferSize, false); + + MlasConv(&Parameters, + Input, + Filter, + Bias, + BufferWorking.GetBuffer(WorkingBufferSize), + Output); + + ReferenceConv2D(BatchCount, + GroupCount, + InputChannels, + InputHeight, InputWidth, + FilterCount, + KernelHeight, KernelWidth, + PaddingLeftHeight, PaddingLeftWidth, + DilationHeight, DilationWidth, + StrideHeight, StrideWidth, + OutputHeight, OutputWidth, + Input, + Filter, + Bias, + OutputReference); + + if (memcmp(Output, OutputReference, OutputBufferElements * sizeof(float)) != 0) { + printf("mismatch: batch=%zd,group=%zd,input(%zd,%zd,%zd),filter=%zd,kernel(%zd,%zd)!!!\n", + BatchCount, GroupCount, InputChannels, InputHeight, InputWidth, FilterCount, + KernelHeight, KernelWidth); + } +} + +void +ExecuteConvTests( + void + ) +{ + static const unsigned cs[] = { 32, 14, 1 }; + static const unsigned is[] = { 53, 11, 5, 1 }; + + for (unsigned i = 1; i < 256; i <<= 1) { + TrialConv2D(1, 1, 16, i, i, 32, 3, 3, 0, 0, 0, 0, 1, 1, 1, 1); + TrialConv2D(1, 1, 16, i, i, 32, 3, 3, 0, 0, 0, 0, 1, 1, 2, 2); + TrialConv2D(1, 1, 16, i, i, 32, 3, 3, 0, 0, 0, 0, 2, 2, 1, 1); + TrialConv2D(1, 1, 16, i, i, 32, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1); + TrialConv2D(1, 1, 16, i, i, 32, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1); + TrialConv2D(1, 1, 16, i, i, 32, i, 1, 0, 0, 0, 0, 1, 1, 1, 1); + TrialConv2D(1, 1, 16, i, i, 32, 1, i, 0, 0, 0, 0, 1, 1, 1, 1); + } + + for (unsigned i = 1; i <= 32; i++) { + TrialConv2D(4, 18, 1, 32, 89, 48, i, 89, 0, 0, 0, 0, 1, 1, 1, 1); + TrialConv2D(4, 18, 1, 32, 89, 48, i, 89, 1, 1, 1, 1, 1, 1, 1, 1); + TrialConv2D(4, 18, 2, 32, 89, 48, i, 89, 0, 0, 0, 0, 1, 1, 1, 1); + } + + for (unsigned b = 1; b < 64; b++) { + TrialConv2D(b, 1, 64, 11, 11, 128, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1); + } + + for (unsigned ic = 0; ic < _countof(cs); ic++) { + for (unsigned ih = 0; ih < _countof(is); ih++) { + for (unsigned iw = 0; iw < _countof(is); iw++) { + fprintf(stderr, "Handling %dx%dx%d\n", cs[ic], is[ih], is[iw]); + for (unsigned fc = 0; fc < _countof(cs); fc++) { + for (unsigned kh = 1; kh <= 5; kh++) { + if (kh == 4) continue; + for (unsigned kw = 1; kw <= 5; kw++) { + if (kw == 4) continue; + for (unsigned p0 = 0; p0 < 2; p0++) { + for (unsigned p1 = 0; p1 < 2; p1++) { + for (unsigned p2 = 0; p2 < 2; p2++) { + for (unsigned p3 = 0; p3 < 2; p3++) { + for (unsigned dh = 1; dh <= 2; dh++) { + for (unsigned dw = 1; dw <= 2; dw++) { + for (unsigned sh = 1; sh <= 2; sh++) { + for (unsigned sw = 1; sw <= 2; sw++) { + TrialConv2D(1, 1, cs[ic], is[ih], is[iw], cs[fc], kh, kw, p0, p1, p2, p3, dh, dw, sh, sw); + } + } + } + } + } + } + } + } + } + } + } + } + } + } +} + +void +ReferenceMaximumPool2D( + const int64_t* InputShape, + const int64_t* KernelShape, + const int64_t* Padding, + const int64_t* StrideShape, + const float* Input, + float* Output + ) +{ + int64_t ChannelCount = InputShape[0] * InputShape[1]; + + int64_t InputHeight = InputShape[2]; + int64_t InputWidth = InputShape[3]; + + int64_t KernelHeight = KernelShape[0]; + int64_t KernelWidth = KernelShape[1]; + + int64_t PaddingLeftY = Padding[0]; + int64_t PaddingLeftX = Padding[1]; + int64_t PaddingRightY = Padding[2]; + int64_t PaddingRightX = Padding[3]; + + int64_t StrideHeight = StrideShape[0]; + int64_t StrideWidth = StrideShape[1]; + + int64_t OutputHeight = (InputHeight + PaddingLeftY + PaddingRightY - KernelHeight) / StrideHeight + 1; + int64_t OutputWidth = (InputWidth + PaddingLeftX + PaddingRightX - KernelWidth) / StrideWidth + 1; + + for (int64_t c = 0; c < ChannelCount; c++) { + + for (int64_t ph = 0; ph < OutputHeight; ph++) { + + int64_t ihStart = ph * StrideHeight - PaddingLeftY; + int64_t ihEnd = ihStart + KernelHeight; + + ihStart = (std::max)(ihStart, int64_t(0)); + ihEnd = (std::min)(ihEnd, InputHeight); + + for (int64_t pw = 0; pw < OutputWidth; pw++) { + + int64_t iwStart = pw * StrideWidth - PaddingLeftX; + int64_t iwEnd = iwStart + KernelWidth; + + iwStart = (std::max)(iwStart, int64_t(0)); + iwEnd = (std::min)(iwEnd, InputWidth); + + float m = std::numeric_limits::lowest(); + + for (int64_t ih = ihStart; ih < ihEnd; ih++) { + for (int64_t iw = iwStart; iw < iwEnd; iw++) { + m = (std::max)(m, Input[ih * InputWidth + iw]); + } + } + + Output[ph * OutputWidth + pw] = m; + } + } + + Input += InputHeight * InputWidth; + Output += OutputHeight * OutputWidth; + } +} + +void +ReferenceMaximumPool3D( + const int64_t* InputShape, + const int64_t* KernelShape, + const int64_t* Padding, + const int64_t* StrideShape, + const float* Input, + float* Output + ) +{ + int64_t ChannelCount = InputShape[0] * InputShape[1]; + + int64_t InputDepth = InputShape[2]; + int64_t InputHeight = InputShape[3]; + int64_t InputWidth = InputShape[4]; + + int64_t KernelDepth = KernelShape[0]; + int64_t KernelHeight = KernelShape[1]; + int64_t KernelWidth = KernelShape[2]; + + int64_t PaddingLeftZ = Padding[0]; + int64_t PaddingLeftY = Padding[1]; + int64_t PaddingLeftX = Padding[2]; + int64_t PaddingRightZ = Padding[3]; + int64_t PaddingRightY = Padding[4]; + int64_t PaddingRightX = Padding[5]; + + int64_t StrideDepth = StrideShape[0]; + int64_t StrideHeight = StrideShape[1]; + int64_t StrideWidth = StrideShape[2]; + + int64_t OutputDepth = (InputDepth + PaddingLeftZ + PaddingRightZ - KernelDepth) / StrideDepth + 1; + int64_t OutputHeight = (InputHeight + PaddingLeftY + PaddingRightY - KernelHeight) / StrideHeight + 1; + int64_t OutputWidth = (InputWidth + PaddingLeftX + PaddingRightX - KernelWidth) / StrideWidth + 1; + + for (int64_t c = 0; c < ChannelCount; c++) { + + for (int64_t pd = 0; pd < OutputDepth; pd++) { + + int64_t idStart = pd * StrideDepth - PaddingLeftZ; + int64_t idEnd = idStart + KernelDepth; + + idStart = (std::max)(idStart, int64_t(0)); + idEnd = (std::min)(idEnd, InputDepth); + + for (int64_t ph = 0; ph < OutputHeight; ph++) { + + int64_t ihStart = ph * StrideHeight - PaddingLeftY; + int64_t ihEnd = ihStart + KernelHeight; + + ihStart = (std::max)(ihStart, int64_t(0)); + ihEnd = (std::min)(ihEnd, InputHeight); + + for (int64_t pw = 0; pw < OutputWidth; pw++) { + + int64_t iwStart = pw * StrideWidth - PaddingLeftX; + int64_t iwEnd = iwStart + KernelWidth; + + iwStart = (std::max)(iwStart, int64_t(0)); + iwEnd = (std::min)(iwEnd, InputWidth); + + float m = std::numeric_limits::lowest(); + + for (int64_t id = idStart; id < idEnd; id++) { + for (int64_t ih = ihStart; ih < ihEnd; ih++) { + for (int64_t iw = iwStart; iw < iwEnd; iw++) { + m = (std::max)(m, Input[id * InputHeight * InputWidth + ih * InputWidth + iw]); + } + } + } + + Output[pd * OutputHeight * OutputWidth + ph * OutputWidth + pw] = m; + } + } + } + + Input += InputDepth * InputHeight * InputWidth; + Output += OutputDepth * OutputHeight * OutputWidth; + } +} + +void +ReferenceAveragePool2D( + const int64_t* InputShape, + const int64_t* KernelShape, + const int64_t* Padding, + const int64_t* StrideShape, + const float* Input, + float* Output, + bool CountIncludePad + ) +{ + int64_t ChannelCount = InputShape[0] * InputShape[1]; + + int64_t InputHeight = InputShape[2]; + int64_t InputWidth = InputShape[3]; + + int64_t KernelHeight = KernelShape[0]; + int64_t KernelWidth = KernelShape[1]; + + int64_t PaddingLeftY = Padding[0]; + int64_t PaddingLeftX = Padding[1]; + int64_t PaddingRightY = Padding[2]; + int64_t PaddingRightX = Padding[3]; + + int64_t StrideHeight = StrideShape[0]; + int64_t StrideWidth = StrideShape[1]; + + int64_t OutputHeight = (InputHeight + PaddingLeftY + PaddingRightY - KernelHeight) / StrideHeight + 1; + int64_t OutputWidth = (InputWidth + PaddingLeftX + PaddingRightX - KernelWidth) / StrideWidth + 1; + + for (int64_t c = 0; c < ChannelCount; c++) { + + for (int64_t ph = 0; ph < OutputHeight; ph++) { + + int64_t ihStart = ph * StrideHeight - PaddingLeftY; + int64_t ihEnd = ihStart + KernelHeight; + + ihStart = (std::max)(ihStart, int64_t(0)); + ihEnd = (std::min)(ihEnd, InputHeight); + + for (int64_t pw = 0; pw < OutputWidth; pw++) { + + int64_t iwStart = pw * StrideWidth - PaddingLeftX; + int64_t iwEnd = iwStart + KernelWidth; + + iwStart = (std::max)(iwStart, int64_t(0)); + iwEnd = (std::min)(iwEnd, InputWidth); + + float m = 0.0f; + + for (int64_t ih = ihStart; ih < ihEnd; ih++) { + for (int64_t iw = iwStart; iw < iwEnd; iw++) { + m += Input[ih * InputWidth + iw]; + } + } + + if (CountIncludePad) { + m /= (KernelHeight * KernelWidth); + } else { + m /= (ihEnd - ihStart) * (iwEnd - iwStart); + } + + Output[ph * OutputWidth + pw] = m; + } + } + + Input += InputHeight * InputWidth; + Output += OutputHeight * OutputWidth; + } +} + +void +ReferenceAveragePool3D( + const int64_t* InputShape, + const int64_t* KernelShape, + const int64_t* Padding, + const int64_t* StrideShape, + const float* Input, + float* Output, + bool CountIncludePad + ) +{ + int64_t ChannelCount = InputShape[0] * InputShape[1]; + + int64_t InputDepth = InputShape[2]; + int64_t InputHeight = InputShape[3]; + int64_t InputWidth = InputShape[4]; + + int64_t KernelDepth = KernelShape[0]; + int64_t KernelHeight = KernelShape[1]; + int64_t KernelWidth = KernelShape[2]; + + int64_t PaddingLeftZ = Padding[0]; + int64_t PaddingLeftY = Padding[1]; + int64_t PaddingLeftX = Padding[2]; + int64_t PaddingRightZ = Padding[3]; + int64_t PaddingRightY = Padding[4]; + int64_t PaddingRightX = Padding[5]; + + int64_t StrideDepth = StrideShape[0]; + int64_t StrideHeight = StrideShape[1]; + int64_t StrideWidth = StrideShape[2]; + + int64_t OutputDepth = (InputDepth + PaddingLeftZ + PaddingRightZ - KernelDepth) / StrideDepth + 1; + int64_t OutputHeight = (InputHeight + PaddingLeftY + PaddingRightY - KernelHeight) / StrideHeight + 1; + int64_t OutputWidth = (InputWidth + PaddingLeftX + PaddingRightX - KernelWidth) / StrideWidth + 1; + + for (int64_t c = 0; c < ChannelCount; c++) { + + for (int64_t pd = 0; pd < OutputDepth; pd++) { + + int64_t idStart = pd * StrideDepth - PaddingLeftZ; + int64_t idEnd = idStart + KernelDepth; + + idStart = (std::max)(idStart, int64_t(0)); + idEnd = (std::min)(idEnd, InputDepth); + + for (int64_t ph = 0; ph < OutputHeight; ph++) { + + int64_t ihStart = ph * StrideHeight - PaddingLeftY; + int64_t ihEnd = ihStart + KernelHeight; + + ihStart = (std::max)(ihStart, int64_t(0)); + ihEnd = (std::min)(ihEnd, InputHeight); + + for (int64_t pw = 0; pw < OutputWidth; pw++) { + + int64_t iwStart = pw * StrideWidth - PaddingLeftX; + int64_t iwEnd = iwStart + KernelWidth; + + iwStart = (std::max)(iwStart, int64_t(0)); + iwEnd = (std::min)(iwEnd, InputWidth); + + float m = 0.0f; + + for (int64_t id = idStart; id < idEnd; id++) { + for (int64_t ih = ihStart; ih < ihEnd; ih++) { + for (int64_t iw = iwStart; iw < iwEnd; iw++) { + m += Input[id * InputHeight * InputWidth + ih * InputWidth + iw]; + } + } + } + + if (CountIncludePad) { + m /= (KernelDepth * KernelHeight * KernelWidth); + } else { + m /= (idEnd - idStart) * (ihEnd - ihStart) * (iwEnd - iwStart); + } + + Output[pd * OutputHeight * OutputWidth + ph * OutputWidth + pw] = m; + } + } + } + + Input += InputDepth * InputHeight * InputWidth; + Output += OutputDepth * OutputHeight * OutputWidth; + } +} + +void +TrialPool2D( + size_t BatchCount, + size_t InputChannels, + size_t InputHeight, + size_t InputWidth, + size_t KernelHeight, + size_t KernelWidth, + size_t PaddingLeftHeight, + size_t PaddingLeftWidth, + size_t PaddingRightHeight, + size_t PaddingRightWidth, + size_t StrideHeight, + size_t StrideWidth + ) +{ + int64_t InputShape[] = { int64_t(BatchCount), int64_t(InputChannels), int64_t(InputHeight), int64_t(InputWidth) }; + int64_t KernelShape[] = { int64_t(KernelHeight), int64_t(KernelWidth) }; + int64_t Padding[] = { int64_t(PaddingLeftHeight), int64_t(PaddingLeftWidth), int64_t(PaddingRightHeight), int64_t(PaddingRightWidth) }; + int64_t StrideShape[] = { int64_t(StrideHeight), int64_t(StrideWidth) }; + int64_t OutputShape[] = { int64_t(BatchCount), int64_t(InputChannels), 0, 0 }; + + OutputShape[2] = (InputShape[2] + Padding[0] + Padding[2] - KernelShape[0]) / StrideShape[0] + 1; + OutputShape[3] = (InputShape[3] + Padding[1] + Padding[3] - KernelShape[1]) / StrideShape[1] + 1; + + size_t InputBufferElements = size_t(InputShape[0] * InputShape[1] * InputShape[2] * InputShape[3]); + size_t OutputBufferElements = size_t(OutputShape[0] * OutputShape[1] * OutputShape[2] * OutputShape[3]); + + MatrixGuardBuffer BufferInput(InputBufferElements, true); + MatrixGuardBuffer BufferOutput(OutputBufferElements, false); + MatrixGuardBuffer BufferOutputReference(OutputBufferElements, false); + + const float* Input = BufferInput.GetBuffer(InputBufferElements); + float* Output = BufferOutput.GetBuffer(OutputBufferElements); + float* OutputReference = BufferOutputReference.GetBuffer(OutputBufferElements); + + MlasPool(MlasMaximumPooling, 2, InputShape, KernelShape, Padding, StrideShape, OutputShape, Input, Output); + ReferenceMaximumPool2D(InputShape, KernelShape, Padding, StrideShape, Input, OutputReference); + + if (memcmp(Output, OutputReference, OutputBufferElements * sizeof(float)) != 0) { + printf("mismatch: maximum input(%zd,%zd,%zd),kernel(%zd,%zd)!!!\n", + InputChannels, InputHeight, InputWidth, KernelHeight, KernelWidth); + } + + MlasPool(MlasAveragePoolingExcludePad, 2, InputShape, KernelShape, Padding, StrideShape, OutputShape, Input, Output); + ReferenceAveragePool2D(InputShape, KernelShape, Padding, StrideShape, Input, OutputReference, false); + + if (memcmp(Output, OutputReference, OutputBufferElements * sizeof(float)) != 0) { + printf("mismatch: averageexcpad input(%zd,%zd,%zd),kernel(%zd,%zd)!!!\n", + InputChannels, InputHeight, InputWidth, KernelHeight, KernelWidth); + } + + MlasPool(MlasAveragePoolingIncludePad, 2, InputShape, KernelShape, Padding, StrideShape, OutputShape, Input, Output); + ReferenceAveragePool2D(InputShape, KernelShape, Padding, StrideShape, Input, OutputReference, true); + + if (memcmp(Output, OutputReference, OutputBufferElements * sizeof(float)) != 0) { + printf("mismatch: averageincpad input(%zd,%zd,%zd),kernel(%zd,%zd)!!!\n", + InputChannels, InputHeight, InputWidth, KernelHeight, KernelWidth); + } +} + +void +TrialPool3D( + size_t BatchCount, + size_t InputChannels, + size_t InputDepth, + size_t InputHeight, + size_t InputWidth, + size_t KernelDepth, + size_t KernelHeight, + size_t KernelWidth, + size_t PaddingLeftDepth, + size_t PaddingLeftHeight, + size_t PaddingLeftWidth, + size_t PaddingRightDepth, + size_t PaddingRightHeight, + size_t PaddingRightWidth, + size_t StrideDepth, + size_t StrideHeight, + size_t StrideWidth + ) +{ + int64_t InputShape[] = { int64_t(BatchCount), int64_t(InputChannels), int64_t(InputDepth), int64_t(InputHeight), int64_t(InputWidth) }; + int64_t KernelShape[] = { int64_t(KernelDepth), int64_t(KernelHeight), int64_t(KernelWidth) }; + int64_t Padding[] = { int64_t(PaddingLeftDepth), int64_t(PaddingLeftHeight), int64_t(PaddingLeftWidth), int64_t(PaddingRightDepth), int64_t(PaddingRightHeight), int64_t(PaddingRightWidth) }; + int64_t StrideShape[] = { int64_t(StrideDepth), int64_t(StrideHeight), int64_t(StrideWidth) }; + int64_t OutputShape[] = { int64_t(BatchCount), int64_t(InputChannels), 0, 0, 0 }; + + OutputShape[2] = (InputShape[2] + Padding[0] + Padding[3] - KernelShape[0]) / StrideShape[0] + 1; + OutputShape[3] = (InputShape[3] + Padding[1] + Padding[4] - KernelShape[1]) / StrideShape[1] + 1; + OutputShape[4] = (InputShape[4] + Padding[2] + Padding[5] - KernelShape[2]) / StrideShape[2] + 1; + + size_t InputBufferElements = size_t(InputShape[0] * InputShape[1] * InputShape[2] * InputShape[3] * InputShape[4]); + size_t OutputBufferElements = size_t(OutputShape[0] * OutputShape[1] * OutputShape[2] * OutputShape[3] * OutputShape[4]); + + MatrixGuardBuffer BufferInput(InputBufferElements, true); + MatrixGuardBuffer BufferOutput(OutputBufferElements, false); + MatrixGuardBuffer BufferOutputReference(OutputBufferElements, false); + + const float* Input = BufferInput.GetBuffer(InputBufferElements); + float* Output = BufferOutput.GetBuffer(OutputBufferElements); + float* OutputReference = BufferOutputReference.GetBuffer(OutputBufferElements); + + MlasPool(MlasMaximumPooling, 3, InputShape, KernelShape, Padding, StrideShape, OutputShape, Input, Output); + ReferenceMaximumPool3D(InputShape, KernelShape, Padding, StrideShape, Input, OutputReference); + + if (memcmp(Output, OutputReference, OutputBufferElements * sizeof(float)) != 0) { + printf("mismatch: maximum input(%zd,%zd,%zd,%zd),kernel(%zd,%zd,%zd)!!!\n", + InputChannels, InputDepth, InputHeight, InputWidth, KernelDepth, KernelHeight, KernelWidth); + } + + MlasPool(MlasAveragePoolingExcludePad, 3, InputShape, KernelShape, Padding, StrideShape, OutputShape, Input, Output); + ReferenceAveragePool3D(InputShape, KernelShape, Padding, StrideShape, Input, OutputReference, false); + + if (memcmp(Output, OutputReference, OutputBufferElements * sizeof(float)) != 0) { + printf("mismatch: averageexcpad input(%zd,%zd,%zd,%zd),kernel(%zd,%zd,%zd)!!!\n", + InputChannels, InputDepth, InputHeight, InputWidth, KernelDepth, KernelHeight, KernelWidth); + } + + MlasPool(MlasAveragePoolingIncludePad, 3, InputShape, KernelShape, Padding, StrideShape, OutputShape, Input, Output); + ReferenceAveragePool3D(InputShape, KernelShape, Padding, StrideShape, Input, OutputReference, true); + + if (memcmp(Output, OutputReference, OutputBufferElements * sizeof(float)) != 0) { + printf("mismatch: averageincpad input(%zd,%zd,%zd,%zd),kernel(%zd,%zd,%zd)!!!\n", + InputChannels, InputDepth, InputHeight, InputWidth, KernelDepth, KernelHeight, KernelWidth); + } +} + +void +ExecutePool2DTests( + void + ) +{ + static const unsigned is[] = { 53, 17, 11, 5, 4, 3, 2, 1 }; + + for (unsigned i = 1; i < 2058; i++) { + TrialPool2D(1, 1, 4, i, 2, 4, 0, 2, 0, 1, 1, 1); + } + + for (unsigned ih = 0; ih < _countof(is); ih++) { + for (unsigned iw = 0; iw < _countof(is); iw++) { + fprintf(stderr, "Handling %dx%d\n", is[ih], is[iw]); + TrialPool2D(1, 1, is[ih], is[iw], is[ih], is[iw], 0, 0, 0, 0, 1, 1); + TrialPool2D(1, 1, is[ih], is[iw], is[ih], 1, 0, 0, 0, 0, 1, 1); + TrialPool2D(1, 1, is[ih], is[iw], 1, is[iw], 0, 0, 0, 0, 1, 1); + for (unsigned kh = 1; kh <= 5; kh++) { + if (kh > is[ih]) break; + for (unsigned kw = 1; kw <= 5; kw++) { + if (kw > is[iw]) break; + for (unsigned sh = 1; sh <= 3; sh++) { + for (unsigned sw = 1; sw <= 3; sw++) { + for (unsigned p0 = 0; p0 < kh; p0++) { + for (unsigned p1 = 0; p1 < kw; p1++) { + for (unsigned p2 = 0; p2 < kh; p2++) { + for (unsigned p3 = 0; p3 < kw; p3++) { + TrialPool2D(2, 3, is[ih], is[iw], kh, kw, p0, p1, p2, p3, sh, sw); + } + } + } + } + } + } + } + } + } + } +} + +void +ExecutePool3DTests( + void + ) +{ + static const unsigned is[] = { 11, 5, 4, 3, 2, 1 }; + + for (unsigned id = 0; id < _countof(is); id++) { + for (unsigned ih = 0; ih < _countof(is); ih++) { + for (unsigned iw = 0; iw < _countof(is); iw++) { + fprintf(stderr, "Handling %dx%dx%d\n", is[id], is[ih], is[iw]); + TrialPool3D(1, 1, is[id], is[ih], is[iw], is[id], is[ih], is[iw], 0, 0, 0, 0, 0, 0, 1, 1, 1); + for (unsigned kd = 1; kd <= 4; kd++) { + if (kd > is[id]) break; + for (unsigned kh = 1; kh <= 4; kh++) { + if (kh > is[ih]) break; + for (unsigned kw = 1; kw <= 4; kw++) { + if (kw > is[iw]) break; + for (unsigned sd = 1; sd <= 3; sd++) { + for (unsigned sh = 1; sh <= 3; sh++) { + for (unsigned sw = 1; sw <= 3; sw++) { + for (unsigned p0 = 0; p0 < kd; p0++) { + for (unsigned p1 = 0; p1 < kh; p1++) { + for (unsigned p2 = 0; p2 < kw; p2++) { + for (unsigned p3 = 0; p3 < kd; p3++) { + for (unsigned p4 = 0; p4 < kh; p4++) { + for (unsigned p5 = 0; p5 < kw; p5++) { + TrialPool3D(1, 1, is[id], is[ih], is[iw], kd, kh, kw, p0, p1, p2, p3, p4, p5, sd, sh, sw); + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } +} + +#if 0 +#if defined(_WIN32) + +extern uint32_t MlasMaximumThreadCount; + +void +EvaluateThreadingPerformance( + void + ) +{ + SYSTEM_INFO SystemInfo; + + GetSystemInfo(&SystemInfo); + + const size_t MaximumDimension = 4096; + + MatrixGuardBuffer BufferA(MaximumDimension, true); + MatrixGuardBuffer BufferB(MaximumDimension, true); + MatrixGuardBuffer BufferC(MaximumDimension, false); + + for (size_t M = 16; M <= MaximumDimension; M <<= 1) { + for (size_t N = 16; N <= MaximumDimension; N <<= 1) { + for (size_t K = 16; K <= MaximumDimension; K <<= 1) { + + const float* A = BufferA.GetBuffer(K * M); + const float* B = BufferB.GetBuffer(N * K); + float* C = BufferC.GetBuffer(N * M); + + // + // Compute the number of iterations to run for at least five seconds. + // + + MlasMaximumThreadCount = 1; + + ULONG64 NumberIterations = 0; + DWORD start = GetTickCount(); + DWORD stop; + do { + MlasSgemm(CblasNoTrans, CblasNoTrans, M, N, K, 1.0f, A, K, B, N, 0.0f, C, N); + stop = GetTickCount(); + NumberIterations++; + } while ((stop - start) <= 5000); + + // + // Determine the performance for a range of thread counts. + // + + static DWORD cpus[] = { 1, 2, 4, 8, 12 }; + for (size_t cpu = 0; cpu < _countof(cpus); cpu++) { + + if (cpus[cpu] <= SystemInfo.dwNumberOfProcessors) { + MlasMaximumThreadCount = cpus[cpu]; + } else { + break; + } + + start = GetTickCount(); + for (size_t iters = 0; iters < NumberIterations; iters++) { + MlasSgemm(CblasNoTrans, CblasNoTrans, M, N, K, 1.0f, A, K, B, N, 0.0f, C, N); + stop = GetTickCount(); + if ((stop - start) > 20000) { + break; + } + } + + printf("%zd,%zd,%zd cpus=%d, iters=%I64d, time=%d %c\n", M, N, K, + MlasMaximumThreadCount, NumberIterations, stop - start, ((stop - start) > 5100) ? '!' : '\0'); + } + + fflush(stdout); + } + } + } +} + +#endif +#endif + +int +#if defined(_WIN32) +__cdecl +#endif +main( + void + ) +{ +// ExecuteSgemmTests(); +// ExecuteConvTests(); + ExecutePool2DTests(); + ExecutePool3DTests(); +// EvaluateThreadingPerformance(); + + return 0; +} diff --git a/onnxruntime/test/onnx/FixedCountFinishCallback.h b/onnxruntime/test/onnx/FixedCountFinishCallback.h new file mode 100644 index 0000000000000..73117105e20a0 --- /dev/null +++ b/onnxruntime/test/onnx/FixedCountFinishCallback.h @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "sync_api.h" + +#include + +template +class FixedCountFinishCallbackImpl { + private: + //remain tasks + int s_; + std::mutex m_; + ONNXRUNTIME_EVENT finish_event_; + bool failed = false; + std::vector> results_; + + public: + FixedCountFinishCallbackImpl(const FixedCountFinishCallbackImpl&) = delete; + FixedCountFinishCallbackImpl& operator=(const FixedCountFinishCallbackImpl&) = delete; + + const std::vector>& getResults() const { + return results_; + } + + FixedCountFinishCallbackImpl(int s) : s_(s), results_(s) { + ONNXRUNTIME_ENFORCE(CreateOnnxRuntimeEvent(&finish_event_).IsOK()); + } + + ~FixedCountFinishCallbackImpl() { + if (finish_event_) ONNXRuntimeCloseEvent(finish_event_); + } + + ::onnxruntime::common::Status fail(ONNXRUNTIME_CALLBACK_INSTANCE pci) { + { + std::lock_guard g(m_); + failed = true; + s_ = 0; //fail earlier + } + return OnnxRuntimeSetEventWhenCallbackReturns(pci, finish_event_); + } + + ::onnxruntime::common::Status onFinished(size_t task_index, std::shared_ptr result, ONNXRUNTIME_CALLBACK_INSTANCE pci) { + int v; + { + std::lock_guard g(m_); + v = --s_; + results_.at(task_index) = result; + } + if (v == 0) { + return OnnxRuntimeSetEventWhenCallbackReturns(pci, finish_event_); + } + return ::onnxruntime::common::Status::OK(); + } + + bool shouldStop() { + std::lock_guard g(m_); + return failed; + } + //this function can only be invoked once + bool wait() { + ONNXRUNTIME_ENFORCE(WaitAndCloseEvent(finish_event_).IsOK()); + { + std::lock_guard g(m_); + finish_event_ = nullptr; + return !failed; + } + } +}; diff --git a/onnxruntime/test/onnx/README.txt b/onnxruntime/test/onnx/README.txt new file mode 100644 index 0000000000000..22e18a66f44c9 --- /dev/null +++ b/onnxruntime/test/onnx/README.txt @@ -0,0 +1,48 @@ +onnxruntime_test_runner [options...] +Options: + -j [models]: Specifies the number of models to run simultaneously. + -c [runs]: Specifies the number of Session::Run() to invoke simultaneously for each model. + -n [test_case_name]: Specifies a single test case to run. + -p [PLANNER_TYPE]: PLANNER_TYPE could be 'seq' or 'simple'. Default: 'simple'. + -e [EXECUTION_PROVIDER]: EXECUTION_PROVIDER could be 'cpu' or 'cuda'. Default: 'cpu'. + -h: help + +The debug version of this program depends on dbghelp.dll. Please make sure it's in your PATH. + +How to run node tests: +1. Install onnx from onnxruntime\cmake\external\onnx + Steps: + "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvars64.bat" + C: + cd C:\src\onnxruntime\onnxruntime\cmake\external\onnx + set PATH="C:\Program Files\CMake\bin";C:\python35;C:\python35\scripts;C:\Program Files\CMake\bin;%PATH% + set PATH=%PATH%;C:\os\onnx\protoc + set INCLUDE=%INCLUDE%;C:\os\protobuf-2.6.1\src + set LIB=%LIB%;C:\os\protobuf-2.6.1\vsprojects\x64\Release + python setup.py install + +2. Execute test data generator: + backend-test-tools generate-data -o + e.g. + backend-test-tools generate-data -o C:\testdata + backend-test-tools is a tool under C:\Python35\Scripts (If your python was installed to C:\Python35) + +3. compile onnxruntime_test_runner and run + onnxruntime_test_runner + e.g. + onnxruntime_test_runner C:\testdata\node + + +How to run model tests: +1. Download test data from VSTS drop + 1) Download drop app from https://aiinfra.artifacts.visualstudio.com/_apis/drop/client/exe + Unzip the downloaded file and add lib/net45 dir to your PATH + 2) Download the test data by using this command: + drop get -a -s https://aiinfra.artifacts.visualstudio.com/DefaultCollection -n Lotus/testdata/onnx/model/16 -d C:\testdata + You may change C:\testdata to any directory in your disk. + Full document: https://www.1eswiki.com/wiki/VSTS_Drop + +2. compile onnxruntime_test_runner and run + onnxruntime_test_runner + e.g. + onnxruntime_test_runner C:\testdata diff --git a/onnxruntime/test/onnx/TestCase.cc b/onnxruntime/test/onnx/TestCase.cc new file mode 100644 index 0000000000000..1485b441088c6 --- /dev/null +++ b/onnxruntime/test/onnx/TestCase.cc @@ -0,0 +1,563 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//TODO: switch to use onnxruntime public api + +#include "TestCase.h" +#include +#include +#include "core/common/logging/logging.h" +#include "core/common/common.h" +#include "core/platform/env.h" +#include "core/session/onnxruntime_cxx_api.h" +#include "path_lib.h" +//TODO: delete this +#include "core/framework/data_types.h" +#include "core/framework/ml_value.h" +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wignored-qualifiers" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#else +#pragma warning(push) +#pragma warning(disable : 4018) /*'expression' : signed/unsigned mismatch */ +#pragma warning(disable : 4065) /*switch statement contains 'default' but no 'case' labels*/ +#pragma warning(disable : 4100) +#pragma warning(disable : 4146) /*unary minus operator applied to unsigned type, result still unsigned*/ +#pragma warning(disable : 4244) /*'conversion' conversion from 'type1' to 'type2', possible loss of data*/ +#pragma warning(disable : 4251) /*'identifier' : class 'type' needs to have dll-interface to be used by clients of class 'type2'*/ +#pragma warning(disable : 4267) /*'var' : conversion from 'size_t' to 'type', possible loss of data*/ +#pragma warning(disable : 4305) /*'identifier' : truncation from 'type1' to 'type2'*/ +#pragma warning(disable : 4307) /*'operator' : integral constant overflow*/ +#pragma warning(disable : 4309) /*'conversion' : truncation of constant value*/ +#pragma warning(disable : 4334) /*'operator' : result of 32-bit shift implicitly converted to 64 bits (was 64-bit shift intended?)*/ +#pragma warning(disable : 4355) /*'this' : used in base member initializer list*/ +#pragma warning(disable : 4506) /*no definition for inline function 'function'*/ +#pragma warning(disable : 4800) /*'type' : forcing value to bool 'true' or 'false' (performance warning)*/ +#pragma warning(disable : 4996) /*The compiler encountered a deprecated declaration.*/ +#endif +#include +#include +#include "tml.pb.h" +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#else +#pragma warning(pop) +#endif + +using namespace onnxruntime; +using namespace onnxruntime::common; + +namespace { +template +Status ConvertVector(const InputType& data, OutputType** vec) { + OutputType* v = new OutputType(); + for (const auto& i : data) { + typename OutputType::value_type new_value; + for (const auto& j : i.v()) { + new_value[j.first] = j.second; + } + v->push_back(new_value); + } + *vec = v; + return Status::OK(); +} + +template +Status Convert(const InputType& tensor_proto, OutputType** p_tensor); + +template <> +Status Convert(const google::protobuf::RepeatedPtrField& data, VectorMapInt64ToFloat** vec) { + return ConvertVector, VectorMapInt64ToFloat>(data, vec); +} + +template <> +Status Convert(const google::protobuf::RepeatedPtrField& data, VectorMapStringToFloat** vec) { + return ConvertVector, VectorMapStringToFloat>(data, vec); +} + +template +void ConvertMap(const InputType& data, OutputType** out) { + OutputType* ret = new OutputType(); + for (const auto& pv : data) { + (*ret)[pv.first] = pv.second; + } + *out = ret; +} + +template <> +Status Convert(const google::protobuf::Map& data, MapStringToString** out) { + ConvertMap(data, out); + return Status::OK(); +} + +template <> +Status Convert(const google::protobuf::Map& data, MapStringToInt64** out) { + ConvertMap(data, out); + return Status::OK(); +} + +template <> +Status Convert(const google::protobuf::Map& data, MapStringToFloat** out) { + ConvertMap(data, out); + return Status::OK(); +} + +template <> +Status Convert(const google::protobuf::Map& data, MapStringToDouble** out) { + ConvertMap(data, out); + return Status::OK(); +} + +template <> +Status Convert(const google::protobuf::Map& data, MapInt64ToString** out) { + ConvertMap(data, out); + return Status::OK(); +} + +template <> +Status Convert(const google::protobuf::Map& data, MapInt64ToInt64** out) { + ConvertMap(data, out); + return Status::OK(); +} + +template <> +Status Convert(const google::protobuf::Map& data, MapInt64ToFloat** out) { + ConvertMap(data, out); + return Status::OK(); +} + +template <> +Status Convert(const google::protobuf::Map& data, MapInt64ToDouble** out) { + ConvertMap(data, out); + return Status::OK(); +} +template +Status RichTypeProtoToMLValue(const InputType& input, MLValue& value) { + OutputType* tensor = nullptr; + Status st = Convert(input, &tensor); + if (!st.IsOK()) return st; + value.Init(tensor, + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); + return Status::OK(); +} + +template +static int ExtractFileNo(const std::basic_string& name) { + size_t p1 = name.rfind('.'); + size_t p2 = name.rfind('_', p1); + ++p2; + std::basic_string number_str = name.substr(p2, p1 - p2); + const CHAR_T* start = number_str.c_str(); + const CHAR_T* end = number_str.c_str(); + long ret = MyStrtol(start, const_cast(&end), 10); + if (end == start) { + ONNXRUNTIME_THROW("parse file name failed"); + } + return static_cast(ret); +} +typedef std::basic_string PATH_STRING_TYPE; + +template +static Status SortTensorFileNames(std::vector>& input_pb_files) { + if (input_pb_files.size() <= 1) return Status::OK(); + std::sort(input_pb_files.begin(), input_pb_files.end(), + [](const std::basic_string& left, const std::basic_string& right) -> bool { + std::basic_string leftname = GetLastComponent(left); + std::basic_string rightname = GetLastComponent(right); + int left1 = ExtractFileNo(leftname); + int right1 = ExtractFileNo(rightname); + return left1 < right1; + }); + for (size_t i = 0; i != input_pb_files.size(); ++i) { + int fileno = ExtractFileNo(GetLastComponent(input_pb_files[i])); + if (static_cast(fileno) != i) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "illegal input file name:", ToMBString(input_pb_files[i])); + } + } + return Status::OK(); +} + +Status LoopDataFile(int test_data_pb_fd, ONNXRuntimeAllocator* env, + const std::vector value_info, std::unordered_map& name_data_map, std::ostringstream& oss) { + google::protobuf::io::FileInputStream f(test_data_pb_fd); + f.SetCloseOnDelete(true); + google::protobuf::io::CodedInputStream coded_input(&f); + bool clean_eof = false; + Status st; + int item_id = 1; + for (proto::TraditionalMLData data; google::protobuf::util::ParseDelimitedFromCodedStream(&data, &coded_input, &clean_eof); ++item_id, data.Clear()) { + std::unique_ptr gvalue(nullptr, ReleaseONNXValue); + MLValue value; + bool is_tensor = false; + switch (data.values_case()) { + case proto::TraditionalMLData::kVectorMapStringToFloat: + st = RichTypeProtoToMLValue(data.vector_map_string_to_float().v(), value); + break; + case proto::TraditionalMLData::kVectorMapInt64ToFloat: + st = RichTypeProtoToMLValue(data.vector_map_int64_to_float().v(), value); + break; + case proto::TraditionalMLData::kMapStringToString: + st = RichTypeProtoToMLValue(data.map_string_to_string().v(), value); + break; + case proto::TraditionalMLData::kMapStringToInt64: + st = RichTypeProtoToMLValue(data.map_string_to_int64().v(), value); + break; + case proto::TraditionalMLData::kMapStringToFloat: + st = RichTypeProtoToMLValue(data.map_string_to_float().v(), value); + break; + case proto::TraditionalMLData::kMapStringToDouble: + st = RichTypeProtoToMLValue(data.map_string_to_double().v(), value); + break; + case proto::TraditionalMLData::kMapInt64ToString: + st = RichTypeProtoToMLValue(data.map_int64_to_string().v(), value); + break; + case proto::TraditionalMLData::kMapInt64ToInt64: + st = RichTypeProtoToMLValue(data.map_int64_to_int64().v(), value); + break; + case proto::TraditionalMLData::kMapInt64ToFloat: + st = RichTypeProtoToMLValue(data.map_int64_to_float().v(), value); + break; + case proto::TraditionalMLData::kMapInt64ToDouble: + st = RichTypeProtoToMLValue(data.map_int64_to_double().v(), value); + break; + case proto::TraditionalMLData::kTensor: { + ONNXValuePtr temp_value; + std::string s = data.tensor().SerializeAsString(); + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeTensorProtoToONNXValue(env, s.data(), (int)s.size(), &temp_value)); + gvalue.reset(temp_value); + is_tensor = true; + } break; + default: + st = Status(ONNXRUNTIME, NOT_IMPLEMENTED, "unknown data type inside TraditionalMLData"); + } + if (!st.IsOK()) break; + if (!data.debug_info().empty()) { + oss << ":" << data.debug_info(); + } + std::string value_name = data.name(); + if (value_name.empty()) + value_name = value_info[name_data_map.size()].name(); + + auto pv = name_data_map.insert(std::make_pair(value_name, is_tensor ? gvalue.release() : (ONNXValuePtr) new MLValue(value))); + if (!pv.second) { + st = Status(ONNXRUNTIME, FAIL, "duplicated test data name"); + break; + } + } + if (!st.IsOK()) return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "load the ", item_id, "-th item failed,", st.ErrorMessage()); + if (!clean_eof) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "parse input file failed, has extra unparsed data"); + } + return Status::OK(); +} + +template +static void RepeatedPtrFieldToVector(const ::google::protobuf::RepeatedPtrField& input_value_info, std::vector& out) { + for (int i = 0; i != input_value_info.size(); ++i) { + out.push_back(input_value_info[i]); + } +} +} // namespace + +/** + * test_case_dir must have contents of: + * model.onnx + * ???/input_??.pb + * ???/output_??.pb + * ???/input_??.pb + * ???/output_??.pb + */ +class OnnxTestCase : public ITestCase { + private: + std::string test_case_name_; + std::basic_string model_url_; + ONNXRuntimeAllocator* allocator; + std::vector debuginfo_strings; + std::mutex m_; + std::vector input_value_info_; + std::vector output_value_info_; + + std::vector> test_data_dirs_; + Status loadModelFile(const PATH_CHAR_TYPE* model_url, ONNX_NAMESPACE::ModelProto** model_pb); + + std::string GetDatasetDebugInfoString(size_t dataset_id) override { + std::lock_guard l(m_); + if (dataset_id < debuginfo_strings.size()) { + return debuginfo_strings[dataset_id]; + } + // return empty string + return std::string(); + } + //If we cannot get input name from input_pbs, we'll use names like "data_0","data_1",... It's dirty hack + // for https://github.com/onnx/onnx/issues/679 + ::onnxruntime::common::Status ConvertTestData(ONNXSessionPtr session, const std::vector& test_data_pbs, + bool is_input, std::unordered_map& out); + std::string node_name_; + std::once_flag model_parsed_; + std::once_flag config_parsed_; + double per_sample_tolerance_; + double relative_per_sample_tolerance_; + bool post_processing_; + Status ParseModel(); + Status ParseConfig(); + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(OnnxTestCase); + + public: + OnnxTestCase(ONNXRuntimeAllocator* env, const std::string& test_case_name); + explicit OnnxTestCase(const std::string& test_case_name) : test_case_name_(test_case_name) {} + Status GetPerSampleTolerance(double* value) override; + Status GetRelativePerSampleTolerance(double* value) override; + Status GetPostProcessing(bool* value) override; + + const ONNX_NAMESPACE::ValueInfoProto& GetOutputInfoFromModel(size_t i) const override { + return output_value_info_[i]; + } + + size_t GetDataCount() const override { + return test_data_dirs_.size(); + } + Status GetNodeName(std::string* out) override { + Status st = ParseModel(); + if (st.IsOK()) *out = node_name_; + return st; + } +#ifdef _WIN32 + Status SetModelPath(const wchar_t* path) override; +#else + Status SetModelPath(const char* path) override; +#endif + + const PATH_CHAR_TYPE* GetModelUrl() const override { return model_url_.c_str(); } + const std::string& GetTestCaseName() const override { + return test_case_name_; + } + ::onnxruntime::common::Status LoadTestData(ONNXSessionPtr session, size_t id, std::unordered_map&, bool is_input) override; +}; + +Status OnnxTestCase::loadModelFile(const PATH_CHAR_TYPE* model_url, ONNX_NAMESPACE::ModelProto** model_pb) { + int model_fd; + ONNXRUNTIME_RETURN_IF_ERROR(Env::Default().FileOpenRd(model_url, model_fd)); + google::protobuf::io::FileInputStream f(model_fd); + f.SetCloseOnDelete(true); + ONNX_NAMESPACE::ModelProto* ret = new ONNX_NAMESPACE::ModelProto(); + if (!ret->ParseFromZeroCopyStream(&f)) { + return Status(ONNXRUNTIME, INVALID_PROTOBUF, "Failed to load model because protobuf parsing failed."); + } + *model_pb = ret; + return Status::OK(); +} +ITestCase* CreateOnnxTestCase(ONNXRuntimeAllocator* ptr, const std::string& test_case_name) { + return new OnnxTestCase(ptr, test_case_name); +} + +Status OnnxTestCase::GetPerSampleTolerance(double* value) { + Status st = ParseConfig(); + if (!st.IsOK()) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, MODEL_LOADED, "parse test config failed:", st.ErrorMessage()); + + *value = per_sample_tolerance_; + return Status::OK(); +} + +Status OnnxTestCase::GetRelativePerSampleTolerance(double* value) { + Status st = ParseConfig(); + if (!st.IsOK()) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, MODEL_LOADED, "parse test config failed:", st.ErrorMessage()); + *value = relative_per_sample_tolerance_; + return Status::OK(); +} + +Status OnnxTestCase::GetPostProcessing(bool* value) { + Status st = ParseConfig(); + if (!st.IsOK()) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, MODEL_LOADED, "parse test config failed:", st.ErrorMessage()); + } + *value = post_processing_; + return Status::OK(); +} + +Status OnnxTestCase::ParseConfig() { + std::call_once(config_parsed_, [this]() { + std::basic_string config_path = + ReplaceFilename>(model_url_, ONNXRUNTIME_TSTR("config.txt")); + int config_fd; + auto st = Env::Default().FileOpenRd(config_path, config_fd); + if (st.IsOK()) { + google::protobuf::io::FileInputStream f(config_fd); + f.SetCloseOnDelete(true); + //parse model + onnxruntime::proto::TestCaseConfig config_pb; + if (!google::protobuf::TextFormat::Parse(&f, &config_pb)) { + LOGF_DEFAULT(ERROR, "Parse config failed"); + return; + } else { + per_sample_tolerance_ = config_pb.per_sample_tolerance(); + relative_per_sample_tolerance_ = config_pb.relative_per_sample_tolerance(); + post_processing_ = config_pb.post_processing(); + return; + } + } + per_sample_tolerance_ = 1e-3; + relative_per_sample_tolerance_ = 1e-5; +#ifdef USE_CUDA + relative_per_sample_tolerance_ = 0.017; // to resolve random MNIST test failure +#endif + post_processing_ = false; + return; + }); + return Status::OK(); +} + +Status OnnxTestCase::ParseModel() { + Status st = Status::OK(); + std::call_once(model_parsed_, [this, &st]() { + //parse model + ONNX_NAMESPACE::ModelProto* model_pb; + st = loadModelFile(model_url_.c_str(), &model_pb); + if (!st.IsOK()) return; + const ONNX_NAMESPACE::GraphProto& graph = model_pb->graph(); + if (graph.node().size() == 1) { + node_name_ = graph.node()[0].op_type(); + } + RepeatedPtrFieldToVector(graph.input(), input_value_info_); + RepeatedPtrFieldToVector(graph.output(), output_value_info_); + st = Status::OK(); + delete model_pb; + }); + return st; +} + +Status OnnxTestCase::SetModelPath(const PATH_CHAR_TYPE* m) { + model_url_ = m; + std::basic_string test_case_dir = GetDirNameFromFilePath(model_url_); + LoopDir(test_case_dir, [&test_case_dir, this](const PATH_CHAR_TYPE* filename, FileType f_type) -> bool { + if (filename[0] == '.') return true; + if (f_type == FileType::TYPE_DIR) { + std::basic_string p = ConcatPathComponent(test_case_dir, filename); + test_data_dirs_.push_back(p); + debuginfo_strings.push_back(ToMBString(p)); + } + return true; + }); + return Status::OK(); +} + +//load tensors from disk +template +static Status LoadTensors(const std::vector& pb_files, + std::vector* input_pbs) { + for (size_t i = 0; i != pb_files.size(); ++i) { + int tensor_fd; + auto st = Env::Default().FileOpenRd(pb_files.at(i), tensor_fd); + ONNXRUNTIME_RETURN_IF_ERROR(st); + google::protobuf::io::FileInputStream f(tensor_fd); + f.SetCloseOnDelete(true); + ONNX_NAMESPACE::TensorProto tensor; + if (!tensor.ParseFromZeroCopyStream(&f)) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "parse file '", ToMBString(pb_files.at(i)), "' failed"); + } + input_pbs->emplace_back(tensor); + } + return Status::OK(); +} + +Status OnnxTestCase::LoadTestData(ONNXSessionPtr session, size_t id, std::unordered_map& name_data_map, bool is_input) { + if (id >= test_data_dirs_.size()) + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "out of bound"); + + Status st = ParseModel(); + if (!st.IsOK()) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, MODEL_LOADED, "parse model failed:", st.ErrorMessage()); + + PATH_STRING_TYPE test_data_pb = ConcatPathComponent( + test_data_dirs_[id], (is_input ? ONNXRUNTIME_TSTR("inputs.pb") : ONNXRUNTIME_TSTR("outputs.pb"))); + int test_data_pb_fd; + st = Env::Default().FileOpenRd(test_data_pb, test_data_pb_fd); + if (st.IsOK()) { //has an all-in-one input file + std::ostringstream oss; + { + std::lock_guard l(m_); + oss << debuginfo_strings[id]; + } + st = LoopDataFile(test_data_pb_fd, allocator, is_input ? input_value_info_ : output_value_info_, name_data_map, oss); + { + std::lock_guard l(m_); + debuginfo_strings[id] = oss.str(); + } + if (!st.IsOK()) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, MODEL_LOADED, "parse data file \"", ToMBString(test_data_pb), + "\" failed:", st.ErrorMessage()); + return Status::OK(); + } + + std::vector test_data_pb_files; + const PATH_STRING_TYPE& dir_path = test_data_dirs_[id]; + LoopDir(dir_path, + [&test_data_pb_files, &dir_path, is_input](const PATH_CHAR_TYPE* filename, FileType f_type) -> bool { + if (filename[0] == '.') return true; + if (f_type != FileType::TYPE_REG) return true; + std::basic_string filename_str = filename; + if (!HasExtensionOf(filename_str, ONNXRUNTIME_TSTR("pb"))) return true; + const std::basic_string file_prefix = + is_input ? ONNXRUNTIME_TSTR("input_") : ONNXRUNTIME_TSTR("output_"); + if (!filename_str.compare(0, file_prefix.length(), file_prefix.c_str())) { + std::basic_string p = ConcatPathComponent(dir_path, filename_str); + test_data_pb_files.push_back(p); + } + return true; + }); + ONNXRUNTIME_RETURN_IF_ERROR(SortTensorFileNames(test_data_pb_files)); + + std::vector test_data_pbs; + ONNXRUNTIME_RETURN_IF_ERROR(LoadTensors(test_data_pb_files, &test_data_pbs)); + ONNXRUNTIME_RETURN_IF_ERROR(ConvertTestData(session, test_data_pbs, is_input, name_data_map)); + return Status::OK(); +} + +Status OnnxTestCase::ConvertTestData(ONNXSessionPtr session, const std::vector& test_data_pbs, + bool is_input, std::unordered_map& out) { + bool has_valid_names = true; + std::vector var_names(test_data_pbs.size()); + for (size_t input_index = 0; input_index != test_data_pbs.size(); ++input_index) { + std::string name = test_data_pbs[input_index].name(); + if (name.empty()) { + has_valid_names = false; + break; + } + var_names[input_index] = name; + } + if (!has_valid_names) { + size_t count; + if (is_input) { + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeInferenceSessionGetInputCount(session, &count)); + } else { + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeInferenceSessionGetOutputCount(session, &count)); + } + if (count != test_data_pbs.size()) + ONNXRUNTIME_THROW("data count mismatch"); + for (size_t i = 0; i != count; ++i) { + char* temp_name; + if (is_input) { + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeInferenceSessionGetInputName(session, i, allocator, &temp_name)); + } else { + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeInferenceSessionGetOutputName(session, i, allocator, &temp_name)); + } + var_names[i] = temp_name; + (*allocator)->Free(allocator, temp_name); + } + } + for (size_t input_index = 0; input_index != test_data_pbs.size(); ++input_index) { + std::string name = var_names[input_index]; + const onnx::TensorProto& input = test_data_pbs[input_index]; + std::string s = input.SerializeAsString(); + MLValue* v1; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeTensorProtoToONNXValue(allocator, s.data(), (int)s.size(), (ONNXValuePtr*)&v1)); + out.insert(std::make_pair(name, (ONNXValuePtr)v1)); + } + return Status::OK(); +} + +OnnxTestCase::OnnxTestCase(ONNXRuntimeAllocator* ptr, const std::string& test_case_name) : test_case_name_(test_case_name), allocator(ptr) { +} diff --git a/onnxruntime/test/onnx/TestCase.h b/onnxruntime/test/onnx/TestCase.h new file mode 100644 index 0000000000000..39ce453a15b75 --- /dev/null +++ b/onnxruntime/test/onnx/TestCase.h @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include +#include +#include +#include +#include "path_lib.h" + +namespace ONNX_NAMESPACE { +class ValueInfoProto; +} + +//One test case is for one model file +//One test case can contain multiple test data(input/output pairs) +class ITestCase { + public: + //must be called before calling the other functions + virtual ::onnxruntime::common::Status SetModelPath(_In_ const PATH_CHAR_TYPE* path) ONNXRUNTIME_ALL_ARGS_NONNULL = 0; + virtual ::onnxruntime::common::Status LoadTestData(ONNXSessionPtr session, size_t id, std::unordered_map& name_data_map, bool is_input) = 0; + virtual const PATH_CHAR_TYPE* GetModelUrl() const = 0; + virtual const std::string& GetTestCaseName() const = 0; + //a string to help identify the dataset + virtual std::string GetDatasetDebugInfoString(size_t dataset_id) = 0; + virtual ::onnxruntime::common::Status GetNodeName(std::string* out) = 0; + //The number of input/output pairs + virtual size_t GetDataCount() const = 0; + virtual const ONNX_NAMESPACE::ValueInfoProto& GetOutputInfoFromModel(size_t i) const = 0; + virtual ~ITestCase() {} + virtual ::onnxruntime::common::Status GetPerSampleTolerance(double* value) = 0; + virtual ::onnxruntime::common::Status GetRelativePerSampleTolerance(double* value) = 0; + virtual ::onnxruntime::common::Status GetPostProcessing(bool* value) = 0; +}; + +ITestCase* CreateOnnxTestCase(ONNXRuntimeAllocator* ptr, const std::string& test_case_name); diff --git a/onnxruntime/test/onnx/TestCaseResult.cc b/onnxruntime/test/onnx/TestCaseResult.cc new file mode 100644 index 0000000000000..ecb054028e752 --- /dev/null +++ b/onnxruntime/test/onnx/TestCaseResult.cc @@ -0,0 +1,8 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "TestCaseResult.h" + +void TestCaseResult::SetResult(size_t task_id, EXECUTE_RESULT r) { + excution_result_[task_id] = r; +} diff --git a/onnxruntime/test/onnx/TestCaseResult.h b/onnxruntime/test/onnx/TestCaseResult.h new file mode 100644 index 0000000000000..ce967facf1d0d --- /dev/null +++ b/onnxruntime/test/onnx/TestCaseResult.h @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include +#include +#include + +//result of a single test run: 1 model with 1 test dataset +enum class EXECUTE_RESULT { + SUCCESS = 0, + UNKNOWN_ERROR = -1, + WITH_EXCEPTION = -2, + RESULT_DIFFERS = -3, + SHAPE_MISMATCH = -4, + TYPE_MISMATCH = -5, + NOT_SUPPORT = -6, + LOAD_MODEL_FAILED = -7, + INVALID_GRAPH = -8, + INVALID_ARGUMENT = -9, + MODEL_SHAPE_MISMATCH = -10, + MODEL_TYPE_MISMATCH = -11, +}; + +class TestCaseResult { + public: + TestCaseResult(size_t count, EXECUTE_RESULT result, const std::string& node_name1) : node_name(node_name1), excution_result_(count, result) { + ::onnxruntime::SetTimeSpecToZero(&spent_time_); + } + + void SetResult(size_t task_id, EXECUTE_RESULT r); + + const std::vector& GetExcutionResult() const { + return excution_result_; + } + + //Time spent in Session::Run. It only make sense when SeqTestRunner was used + ::onnxruntime::TIME_SPEC GetSpentTime() const { + return spent_time_; + } + + //Time spent in Session::Run. It only make sense when SeqTestRunner was used + void SetSpentTime(const ::onnxruntime::TIME_SPEC& input) const { + memcpy((void*)&spent_time_, &input, sizeof(input)); + } + + //only valid for single node tests; + const std::string node_name; + + private: + ::onnxruntime::TIME_SPEC spent_time_; + std::vector excution_result_; +}; diff --git a/onnxruntime/test/onnx/TestResultStat.cc b/onnxruntime/test/onnx/TestResultStat.cc new file mode 100644 index 0000000000000..7fb0102d46f4d --- /dev/null +++ b/onnxruntime/test/onnx/TestResultStat.cc @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include +#include +#include +#include "TestResultStat.h" + +namespace { +template +std::string containerToStr(const T1& input) { + std::ostringstream oss; + bool is_first = true; + std::vector vec(input.begin(), input.end()); + std::sort(vec.begin(), vec.end()); + for (const auto& s : vec) { + if (!is_first) oss << ", "; + oss << s; + is_first = false; + } + return oss.str(); +} +} // namespace + +std::string TestResultStat::ToString() { + std::string not_implemented_kernels_str = containerToStr(this->not_implemented_kernels); + std::string failed_kernels_str = containerToStr(this->failed_kernels); + int failed = static_cast(this->total_test_case_count) - this->succeeded - this->skipped - this->not_implemented; + int other_reason_failed = failed - this->load_model_failed - this->result_differs - this->throwed_exception - this->invalid_graph; + std::ostringstream oss; + oss << "result: " + "\n\tModels: " + << this->total_model_count + << "\n\tTotal test cases: " + << this->total_test_case_count + << "\n\t\tSucceeded: " << this->succeeded + << "\n\t\tNot implemented: " << this->not_implemented + << "\n\t\tFailed: " << failed << "\n"; + if (this->invalid_graph) + oss << "\t\t\tGraph is invalid:" << this->invalid_graph << "\n"; + if (this->load_model_failed) + oss << "\t\t\tGot exception while loading model: " << this->load_model_failed << "\n"; + if (this->throwed_exception) + oss << "\t\t\tGot exception while running: " << this->throwed_exception << "\n"; + if (this->result_differs) + oss << "\t\t\tResult differs: " << this->result_differs << "\n"; + if (other_reason_failed != 0) oss << "\t\t\tOther reason:" << other_reason_failed << "\n"; + oss << "\tStats by Operator type:\n"; + oss << "\t\tNot implemented(" << this->not_implemented_kernels.size() << "): " << not_implemented_kernels_str << "\n\t\tFailed:" + << failed_kernels_str << "\n"; + oss << "Failed Test Cases:" << containerToStr(failed_test_cases) << "\n"; + return oss.str(); +} diff --git a/onnxruntime/test/onnx/TestResultStat.h b/onnxruntime/test/onnx/TestResultStat.h new file mode 100644 index 0000000000000..c6a29a67f292a --- /dev/null +++ b/onnxruntime/test/onnx/TestResultStat.h @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include +#include +#include + +class TestResultStat { + public: + size_t total_test_case_count = 0; + size_t total_model_count = 0; + std::atomic_int succeeded; + std::atomic_int not_implemented; + std::atomic_int load_model_failed; + std::atomic_int throwed_exception; + std::atomic_int result_differs; + std::atomic_int skipped; + std::atomic_int invalid_graph; + + TestResultStat() : succeeded(0), not_implemented(0), load_model_failed(0), throwed_exception(0), result_differs(0), skipped(0), invalid_graph(0) {} + + void AddNotImplementedKernels(const std::string& s) { + std::lock_guard l(m_); + not_implemented_kernels.insert(s); + } + + void AddFailedKernels(const std::string& s) { + std::lock_guard l(m_); + failed_kernels.insert(s); + } + + void AddFailedTest(const std::string& s) { + std::lock_guard l(m_); + failed_test_cases.insert(s); + } + + std::unordered_set GetFailedTest() { + std::lock_guard l(m_); + return failed_test_cases; + } + + std::string ToString(); + + private: + std::mutex m_; + std::unordered_set not_implemented_kernels; + std::unordered_set failed_kernels; + std::unordered_set failed_test_cases; +}; diff --git a/onnxruntime/test/onnx/gen_test_models.py b/onnxruntime/test/onnx/gen_test_models.py new file mode 100644 index 0000000000000..cc186a9dc9f7d --- /dev/null +++ b/onnxruntime/test/onnx/gen_test_models.py @@ -0,0 +1,54 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import onnx +import numpy as np +import os +import argparse +from onnx import numpy_helper +from onnx import helper +from onnx import utils +from onnx import AttributeProto, TensorProto, GraphProto + +def parse_arguments(): + parser = argparse.ArgumentParser() + parser.add_argument("--output_dir", required=True, help="Path to the build directory.") + return parser.parse_args() + +def generate_test(type, X, test_folder): + data_dir = os.path.join(test_folder,"test_data_0") + os.makedirs(data_dir, exist_ok=True) + # Create one output (ValueInfoProto) + Y = helper.make_tensor_value_info('Y', type, X.shape) + X_INFO = helper.make_tensor_value_info('X', type, X.shape) + tensor_x = onnx.helper.make_tensor(name='X', data_type=type, dims=X.shape, vals=X.tobytes(),raw=True) + # Create a node (NodeProto) + node_def = helper.make_node('Abs', inputs=['X'], outputs=['Y']) + + # Create the graph (GraphProto) + graph_def = helper.make_graph( [node_def], 'test-model', [X_INFO], [Y], [tensor_x]) + # Create the model (ModelProto) + model_def = helper.make_model(graph_def, producer_name='onnx-example') + #print('The model is:\n{}'.format(model_def)) + final_model = onnx.utils.polish_model(model_def) + onnx.save(final_model, os.path.join(test_folder, 'model.onnx')) + expected_output_array = np.abs(X) + expected_output_tensor = numpy_helper.from_array(expected_output_array) + with open(os.path.join(data_dir,"output_0.pb"),"wb") as f: + f.write(expected_output_tensor.SerializeToString()) + + +args = parse_arguments() +os.makedirs(args.output_dir,exist_ok=True) +generate_test(TensorProto.FLOAT, np.random.randn(3, 4, 5).astype(np.float32), os.path.join(args.output_dir,'test_abs_float')) +generate_test(TensorProto.DOUBLE, np.random.randn(3, 4, 5).astype(np.float64), os.path.join(args.output_dir,'test_abs_double')) +generate_test(TensorProto.INT8, np.int8([-127, -4, 0, 3, 127]), os.path.join(args.output_dir, 'test_abs_int8')) +generate_test(TensorProto.UINT8, np.uint8([0, 1, 20, 255]), os.path.join(args.output_dir, 'test_abs_uint8')) +generate_test(TensorProto.INT16, np.int16([-32767, -4, 0, 3, 32767]), os.path.join(args.output_dir, 'test_abs_int16')) +generate_test(TensorProto.UINT16, np.uint16([-32767, -4, 0, 3, 32767]), os.path.join(args.output_dir, 'test_abs_uint16')) +generate_test(TensorProto.INT32, np.int32([-2147483647, -4, 0, 3, 2147483647]), os.path.join(args.output_dir, 'test_abs_int32')) +generate_test(TensorProto.UINT32, np.uint32([0, 1, 20, 4294967295]), os.path.join(args.output_dir, 'test_abs_uint32')) +number_info = np.iinfo(np.int64) +generate_test(TensorProto.INT64, np.int64([-number_info.max, -4, 0, 3, number_info.max]), os.path.join(args.output_dir, 'test_abs_int64')) +number_info = np.iinfo(np.uint64) +generate_test(TensorProto.UINT64, np.uint64([0, 1, 20, number_info.max]), os.path.join(args.output_dir, 'test_abs_uint64')) diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc new file mode 100644 index 0000000000000..9d4149996bf56 --- /dev/null +++ b/onnxruntime/test/onnx/main.cc @@ -0,0 +1,363 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include +#include +#ifdef _WIN32 +#include "getopt.h" +#else +#include +#include +#endif +#include "TestResultStat.h" +#include "testenv.h" +#include "runner.h" +#include "path_lib.h" +#include "sync_api.h" +#include "providers.h" +#include "core/session/onnxruntime_cxx_api.h" + +using namespace onnxruntime; + +namespace { +void usage() { + printf( + "onnx_test_runner [options...] \n" + "Options:\n" + "\t-j [models]: Specifies the number of models to run simultaneously.\n" + "\t-A : Disable memory arena\n" + "\t-c [runs]: Specifies the number of Session::Run() to invoke simultaneously for each model.\n" + "\t-r [repeat]: Specifies the number of times to repeat\n" + "\t-v: verbose\n" + "\t-n [test_case_name]: Specifies a single test case to run.\n" + "\t-e [EXECUTION_PROVIDER]: EXECUTION_PROVIDER could be 'cpu', 'cuda' or 'mkldnn'. Default: 'cpu'.\n" + "\t-x: Use parallel executor, default (without -x): sequential executor.\n" + "\t-h: help\n"); +} + +#ifdef _WIN32 +int GetNumCpuCores() { + SYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer[256]; + DWORD returnLength = sizeof(buffer); + if (GetLogicalProcessorInformation(buffer, &returnLength) == FALSE) { + // try GetSystemInfo + SYSTEM_INFO sysInfo; + GetSystemInfo(&sysInfo); + if (sysInfo.dwNumberOfProcessors <= 0) { + ONNXRUNTIME_THROW("Fatal error: 0 count processors from GetSystemInfo"); + } + // This is the number of logical processors in the current group + return sysInfo.dwNumberOfProcessors; + } + int processorCoreCount = 0; + int count = (int)(returnLength / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION)); + for (int i = 0; i != count; ++i) { + if (buffer[i].Relationship == RelationProcessorCore) { + ++processorCoreCount; + } + } + if (!processorCoreCount) ONNXRUNTIME_THROW("Fatal error: 0 count processors from GetLogicalProcessorInformation"); + return processorCoreCount; +} +#else +int GetNumCpuCores() { + return std::thread::hardware_concurrency(); +} +#endif +} // namespace + +#ifdef _WIN32 +int real_main(int argc, wchar_t* argv[]) { +#else +int real_main(int argc, char* argv[]) { +#endif + //if this var is not empty, only run the tests with name in this list + std::vector > whitelisted_test_cases; + int concurrent_session_runs = GetNumCpuCores(); + bool enable_cpu_mem_arena = true; + bool enable_sequential_execution = true; + int repeat_count = 1; + int p_models = GetNumCpuCores(); + bool enable_cuda = false; + bool enable_mkl = false; + bool enable_nuphar = false; + ONNXRuntimeLoggingLevel logging_level = ONNXRUNTIME_LOGGING_LEVEL_kWARNING; + { + int ch; + while ((ch = getopt(argc, argv, ONNXRUNTIME_TSTR("Ac:hj:m:n:r:e:xv"))) != -1) { + switch (ch) { + case 'A': + enable_cpu_mem_arena = false; + break; + case 'v': + logging_level = ONNXRUNTIME_LOGGING_LEVEL_kINFO; + break; + case 'c': + concurrent_session_runs = static_cast(MyStrtol(optarg, nullptr, 10)); + if (concurrent_session_runs <= 0) { + usage(); + return -1; + } + break; + case 'j': + p_models = static_cast(MyStrtol(optarg, nullptr, 10)); + if (p_models <= 0) { + usage(); + return -1; + } + break; + case 'r': + repeat_count = static_cast(MyStrtol(optarg, nullptr, 10)); + if (repeat_count <= 0) { + usage(); + return -1; + } + break; + case 'm': + //ignore. + break; + case 'n': + //run only some whitelisted tests + //TODO: parse name str to an array + whitelisted_test_cases.emplace_back(optarg); + break; + case 'e': + if (!MyStrCmp(optarg, ONNXRUNTIME_TSTR("cpu"))) { + //do nothing + } else if (!MyStrCmp(optarg, ONNXRUNTIME_TSTR("cuda"))) { + enable_cuda = true; + } else if (!MyStrCmp(optarg, ONNXRUNTIME_TSTR("mkldnn"))) { + enable_mkl = true; + } else if (!MyStrCmp(optarg, ONNXRUNTIME_TSTR("nuphar"))) { + enable_nuphar = true; + } else { + usage(); + return -1; + } + break; + case 'x': + enable_sequential_execution = false; + break; + case '?': + case 'h': + default: + usage(); + return -1; + } + } + } + if (concurrent_session_runs > 1 && repeat_count > 1) { + fprintf(stderr, "when you use '-r [repeat]', please set '-c' to 1\n"); + usage(); + return -1; + } + argc -= optind; + argv += optind; + if (argc < 1) { + fprintf(stderr, "please specify a test data dir\n"); + usage(); + return -1; + } + std::unique_ptr env(nullptr, ReleaseONNXEnv); + { + ONNXEnv* t; + ONNXStatusPtr ost = ONNXRuntimeInitialize(logging_level, "Default", &t); + if (ost != nullptr) { + fprintf(stderr, "Error creating environment: %s \n", ONNXRuntimeGetErrorMessage(ost)); + ReleaseONNXStatus(ost); + return -1; + } + env.reset(t); + } + std::vector > data_dirs; + TestResultStat stat; + + std::unique_ptr default_allocator; + { + ONNXRuntimeAllocator* p; + ONNXStatusPtr ost = ONNXRuntimeCreateDefaultAllocator(&p); + if (ost != nullptr) { + fprintf(stderr, "Error creating environment: %s \n", ONNXRuntimeGetErrorMessage(ost)); + ReleaseONNXStatus(ost); + return -1; + } + default_allocator.reset(p); + } + for (int i = 0; i != argc; ++i) { + data_dirs.emplace_back(argv[i]); + } + { + std::vector tests = LoadTests(data_dirs, whitelisted_test_cases, default_allocator.get()); + SessionOptionsWrapper sf(env.get()); + if (enable_cpu_mem_arena) + sf.EnableCpuMemArena(); + else + sf.DisableCpuMemArena(); + if (enable_sequential_execution) + sf.EnableSequentialExecution(); + else + sf.DisableSequentialExecution(); + if (enable_cuda) { +#ifdef USE_CUDA + ONNXRuntimeProviderFactoryPtr* f; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateCUDAExecutionProviderFactory(0, &f)); + sf.AppendExecutionProvider(f); + ONNXRuntimeReleaseObject(f); +#else + fprintf(stderr, "CUDA is supported in this build"); + return -1; +#endif + } + if (enable_nuphar) { +#ifdef USE_NUPHAR + ONNXRuntimeProviderFactoryPtr* f; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateNupharExecutionProviderFactory(0, "", &f)); + sf.AppendExecutionProvider(f); + ONNXRuntimeReleaseObject(f); +#else + fprintf(stderr, "Nuphar is supported in this build"); + return -1; +#endif + } + if (enable_mkl) { +#ifdef USE_MKLDNN + ONNXRuntimeProviderFactoryPtr* f; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateMkldnnExecutionProviderFactory(enable_cpu_mem_arena ? 1 : 0, &f)); + sf.AppendExecutionProvider(f); + ONNXRuntimeReleaseObject(f); +#else + fprintf(stderr, "MKL-DNN is supported in this build"); + return -1; +#endif + } + TestEnv args(tests, stat, sf); + Status st = RunTests(args, p_models, concurrent_session_runs, static_cast(repeat_count), GetDefaultThreadPool(Env::Default())); + if (!st.IsOK()) { + fprintf(stderr, "%s\n", st.ErrorMessage().c_str()); + return -1; + } + + std::string res = stat.ToString(); + fwrite(res.c_str(), 1, res.size(), stdout); + for (ITestCase* l : tests) { + delete l; + } + } + std::map broken_tests{ + {"AvgPool1d", "disable reason"}, + {"AvgPool1d_stride", "disable reason"}, + {"AvgPool2d", "disable reason"}, + {"AvgPool2d_stride", "disable reason"}, + {"AvgPool3d", "disable reason"}, + {"AvgPool3d_stride", "disable reason"}, + {"AvgPool3d_stride1_pad0_gpu_input", "disable reason"}, + {"BatchNorm1d_3d_input_eval", "disable reason"}, + {"BatchNorm2d_eval", "disable reason"}, + {"BatchNorm2d_momentum_eval", "disable reason"}, + {"BatchNorm3d_eval", "disable reason"}, + {"BatchNorm3d_momentum_eval", "disable reason"}, + {"GLU", "disable reason"}, + {"GLU_dim", "disable reason"}, + {"Linear", "disable reason"}, + {"PReLU_1d", "disable reason"}, + {"PReLU_1d_multiparam", "disable reason"}, + {"PReLU_2d", "disable reason"}, + {"PReLU_2d_multiparam", "disable reason"}, + {"PReLU_3d", "disable reason"}, + {"PReLU_3d_multiparam", "disable reason"}, + {"PoissonNLLLLoss_no_reduce", "disable reason"}, + {"Softsign", "disable reason"}, + {"convtranspose_1d", "disable reason"}, + {"convtranspose_3d", "disable reason"}, + {"dynamic_slice", "disable reason"}, + {"dynamic_slice_default_axes", "disable reason"}, + {"dynamic_slice_end_out_of_bounds", "disable reason"}, + {"dynamic_slice_neg", "disable reason"}, + {"dynamic_slice_start_out_of_bounds", "disable reason"}, + {"eyelike_populate_off_main_diagonal", "disable reason"}, + {"eyelike_with_dtype", "disable reason"}, + {"eyelike_without_dtype", "disable reason"}, + {"flatten_axis0", "disable reason"}, + {"flatten_axis1", "disable reason"}, + {"flatten_axis2", "disable reason"}, + {"flatten_axis3", "disable reason"}, + {"flatten_default_axis", "disable reason"}, + {"gemm_broadcast", "disable reason"}, + {"gemm_nobroadcast", "disable reason"}, + {"greater", "disable reason"}, + {"greater_bcast", "disable reason"}, + {"less", "disable reason"}, + {"less_bcast", "disable reason"}, + {"matmul_2d", "disable reason"}, + {"matmul_3d", "disable reason"}, + {"matmul_4d", "disable reason"}, + {"mvn", "disable reason"}, + {"operator_add_broadcast", "disable reason"}, + {"operator_add_size1_broadcast", "disable reason"}, + {"operator_add_size1_right_broadcast", "disable reason"}, + {"operator_add_size1_singleton_broadcast", "disable reason"}, + {"operator_addconstant", "disable reason"}, + {"operator_addmm", "disable reason"}, + {"operator_basic", "disable reason"}, + {"operator_lstm", "disable reason"}, + {"operator_mm", "disable reason"}, + {"operator_non_float_params", "disable reason"}, + {"operator_params", "disable reason"}, + {"operator_pow", "disable reason"}, + {"operator_rnn", "disable reason"}, + {"operator_rnn_single_layer", "disable reason"}, + {"prelu_broadcast", "disable reason"}, + {"prelu_example", "disable reason"}, + {"cntk_simple_seg", "mkldnn test failed"}, + {"maxunpool_export_with_output_shape", "opset 9 not supported yet"}, + {"maxunpool_export_without_output_shape", "opset 9 not supported yet"}, + {"upsample_nearest", "opset 9 not supported yet"}, + {"onehot_with_axis", "opset 9 not supported yet"}, + {"onehot_without_axis", "opset 9 not supported yet"}, // also has bug in current test re: output type. Spandan to fix. + {"sinh", "opset 9 not supported yet"}, + {"cosh", "opset 9 not supported yet"}, + {"asinh", "opset 9 not supported yet"}, + {"acosh", "opset 9 not supported yet"}, + {"atanh", "opset 9 not supported yet"}, + {"sinh_example", "opset 9 not supported yet"}, + {"cosh_example", "opset 9 not supported yet"}, + {"asinh_example", "opset 9 not supported yet"}, + {"acosh_example", "opset 9 not supported yet"}, + {"atanh_example", "opset 9 not supported yet"}}; + +#ifdef USE_CUDA + broken_tests["maxpool_2d_default"] = "cudnn pooling only support input dimension >= 3"; + broken_tests["maxpool_2d_pads"] = "cudnn pooling only support input dimension >= 3"; + broken_tests["maxpool_2d_precomputed_strides"] = "cudnn pooling only support input dimension >= 3"; + broken_tests["maxpool_2d_precomputed_pads"] = "cudnn pooling only support input dimension >= 3"; + broken_tests["maxpool_2d_strides"] = "cudnn pooling only support input dimension >= 3"; + broken_tests["maxpool_2d_precomputed_same_upper"] = "cudnn pooling only support input dimension >= 3"; + broken_tests["maxpool_2d_same_upper"] = "cudnn pooling only support input dimension >= 3"; + broken_tests["maxpool_2d_same_lower"] = "cudnn pooling only support input dimension >= 3"; + broken_tests["maxpool_3d_default"] = "cudnn pooling only support input dimension >= 3"; + broken_tests["maxpool_1d_default"] = "cudnn pooling only support input dimension >= 3"; +#endif + + int result = 0; + for (const std::string& s : stat.GetFailedTest()) { + if (broken_tests.find(s) == broken_tests.end()) { + fprintf(stderr, "test %s failed, please fix it\n", s.c_str()); + result = -1; + } + } + + return result; +} +#ifdef _WIN32 +int wmain(int argc, wchar_t* argv[]) { +#else +int main(int argc, char* argv[]) { +#endif + try { + return real_main(argc, argv); + } catch (std::exception& ex) { + fprintf(stderr, "%s\n", ex.what()); + return -1; + } +} diff --git a/onnxruntime/test/onnx/microbenchmark/main.cc b/onnxruntime/test/onnx/microbenchmark/main.cc new file mode 100644 index 0000000000000..d5cb722f9914f --- /dev/null +++ b/onnxruntime/test/onnx/microbenchmark/main.cc @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace onnxruntime; + +static void BM_CPUAllocator(benchmark::State& state) { + AllocatorPtr cpu_allocator = std::make_shared(); + const size_t len = state.range(0); + for (auto _ : state) { + void* p = cpu_allocator->Alloc(len); + cpu_allocator->Free(p); + } +} +BENCHMARK(BM_CPUAllocator)->Arg(4)->Arg(sizeof(Tensor)); + +static void BM_ResolveGraph(benchmark::State& state) { + std::shared_ptr model_copy; + auto st = onnxruntime::Model::Load("../models/opset8/test_tiny_yolov2/model.onnx", model_copy); + if (!st.IsOK()) { + printf("Parse model failed: %s", st.ErrorMessage().c_str()); + abort(); + } + auto proto = model_copy->ToProto(); + model_copy.reset(); + for (auto _ : state) { + state.PauseTiming(); + std::shared_ptr model = std::make_shared(proto); + onnxruntime::Graph& graph = model->MainGraph(); + state.ResumeTiming(); + st = graph.Resolve(); + if (!st.IsOK()) { + printf("Resolve graph failed: %s", st.ErrorMessage().c_str()); + abort(); + } + } +} + +BENCHMARK(BM_ResolveGraph); + +int main(int argc, char** argv) { + ::benchmark::Initialize(&argc, argv); + if (::benchmark::ReportUnrecognizedArguments(argc, argv)) return -1; + std::string default_logger_id{"Default"}; + logging::LoggingManager default_logging_manager{std::unique_ptr{new logging::CLogSink{}}, + logging::Severity::kWARNING, false, + logging::LoggingManager::InstanceType::Default, + &default_logger_id}; + + std::unique_ptr env; + auto status = Environment::Create(env); + ::benchmark::RunSpecifiedBenchmarks(); + return 0; +} diff --git a/onnxruntime/test/onnx/microbenchmark/modeltest.cc b/onnxruntime/test/onnx/microbenchmark/modeltest.cc new file mode 100644 index 0000000000000..e021d1daec38b --- /dev/null +++ b/onnxruntime/test/onnx/microbenchmark/modeltest.cc @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include + +static void BM_LoadModel(benchmark::State& state) { + for (auto _ : state) { + std::shared_ptr yolomodel; + auto st = onnxruntime::Model::Load("../models/opset8/test_tiny_yolov2/model.onnx", yolomodel); + if (!st.IsOK()) { + state.SkipWithError(st.ErrorMessage().c_str()); + break; + } + } +} + +BENCHMARK(BM_LoadModel); diff --git a/onnxruntime/test/onnx/onnxruntime_event.h b/onnxruntime/test/onnx/onnxruntime_event.h new file mode 100644 index 0000000000000..bfbc65eb1d201 --- /dev/null +++ b/onnxruntime/test/onnx/onnxruntime_event.h @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include +#include + +struct OnnxRuntimeEvent { + public: + pthread_mutex_t finish_event_mutex = PTHREAD_MUTEX_INITIALIZER; + pthread_cond_t finish_event_data = PTHREAD_COND_INITIALIZER; + bool finished = false; + OnnxRuntimeEvent() = default; + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(OnnxRuntimeEvent); +}; + +using ONNXRUNTIME_EVENT = OnnxRuntimeEvent*; diff --git a/onnxruntime/test/onnx/path_lib.h b/onnxruntime/test/onnx/path_lib.h new file mode 100644 index 0000000000000..36dda779d46eb --- /dev/null +++ b/onnxruntime/test/onnx/path_lib.h @@ -0,0 +1,263 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/common/visibility_macros.h" +#include +#include +#include +#include +#ifdef _WIN32 +#include +#include +#else +#include +#include +#include +#endif + +#ifdef _WIN32 +typedef wchar_t PATH_CHAR_TYPE; +#define ONNXRUNTIME_TSTR(X) L##X +#else +typedef char PATH_CHAR_TYPE; +#define ONNXRUNTIME_TSTR(X) (X) +#endif + +template +long MyStrtol(const T* nptr, T** endptr, int base); + +template <> +inline long MyStrtol(const char* nptr, char** endptr, int base) { + return strtol(nptr, endptr, base); +} + +template <> +inline long MyStrtol(const wchar_t* nptr, wchar_t** endptr, int base) { + return wcstol(nptr, endptr, base); +} + +template +inline int MyStrCmp(const T* s1, const T* s2); + +template <> +inline int MyStrCmp(const char* s1, const char* s2) { + return strcmp(s1, s2); +} + +template <> +inline int MyStrCmp(const wchar_t* s1, const wchar_t* s2) { + return wcscmp(s1, s2); +} + +enum class FileType { TYPE_BLK, TYPE_CHR, TYPE_DIR, TYPE_FIFO, TYPE_LNK, TYPE_REG, TYPE_SOCK, TYPE_UNKNOWN }; + +template +PATH_CHAR_TYPE GetPathSep(); + +template +PATH_CHAR_TYPE GetDot(); + +template +bool HasExtensionOf(const std::basic_string& s1, _In_ const PATH_CHAR_TYPE* s2) { + typename std::basic_string::size_type pos = s1.rfind(GetDot()); + if (pos == std::basic_string::npos || pos == s1.size() - 1) { + return false; + } + ++pos; + size_t extension_length = s1.size() - pos; + return s1.compare(pos, extension_length, s2) == 0; +} + +template <> +inline char GetDot() { + return '.'; +} + +template <> +inline wchar_t GetDot() { + return L'.'; +} + +#ifdef _WIN32 +template <> +inline char GetPathSep() { + return '\\'; +} + +template <> +inline wchar_t GetPathSep() { + return L'\\'; +} +#else +template <> +inline char GetPathSep() { + return '/'; +} + +template <> +inline wchar_t GetPathSep() { + return L'/'; +} +#endif + +template +std::basic_string ConcatPathComponent(const std::basic_string& left, + const std::basic_string& right) { + std::basic_string ret(left); + ret.append(1, GetPathSep()).append(right); + return ret; +} + +inline std::string ToMBString(const std::string& s) { return s; } + +#ifdef _WIN32 +inline FileType DTToFileType(DWORD dwFileAttributes) { + if (dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + return FileType::TYPE_DIR; + } + // TODO: test if it is reg + return FileType::TYPE_REG; +} +inline std::string FormatErrorCode(DWORD dw) { + char* lpMsgBuf; + FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, dw, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&lpMsgBuf, 0, NULL); + std::string s(lpMsgBuf); + LocalFree(lpMsgBuf); + return s; +} +template +void LoopDir(const std::wstring& dir_name, T func) { + std::wstring pattern = dir_name + L"\\*"; + WIN32_FIND_DATAW ffd; + std::unique_ptr hFind(FindFirstFileW(pattern.c_str(), &ffd), FindClose); + if (hFind.get() == INVALID_HANDLE_VALUE) { + DWORD dw = GetLastError(); + std::string s = FormatErrorCode(dw); + throw std::runtime_error(s); + } + do { + if (!func(ffd.cFileName, DTToFileType(ffd.dwFileAttributes))) return; + } while (FindNextFileW(hFind.get(), &ffd) != 0); + DWORD dwError = GetLastError(); + if (dwError != ERROR_NO_MORE_FILES) { + DWORD dw = GetLastError(); + std::string s = FormatErrorCode(dw); + throw std::runtime_error(s); + } +} + +inline std::string ToMBString(const std::wstring& s) { + if (s.size() >= std::numeric_limits::max()) throw std::runtime_error("length overflow"); + const int src_len = static_cast(s.size() + 1); + const int len = WideCharToMultiByte(CP_ACP, 0, s.data(), src_len, nullptr, 0, nullptr, nullptr); + std::string ret(len, '\0'); + const int r = WideCharToMultiByte(CP_ACP, 0, s.data(), src_len, (char*)ret.data(), len, nullptr, nullptr); + assert(len == r); + ret.resize(r - 1); + return ret; +} +inline std::wstring GetDirNameFromFilePath(const std::wstring& s) { + std::wstring input = s; + if (input.empty()) throw std::runtime_error("illegal input path"); + if (input.back() == L'\\') input.resize(input.size() - 1); + std::wstring ret(input); + if (PathCchRemoveFileSpec(const_cast(ret.data()), ret.length() + 1) != S_OK) { + throw std::runtime_error("illegal input path"); + } + ret.resize(wcslen(ret.c_str())); + return ret; +} + +template +inline std::basic_string GetLastComponent(const std::basic_string& s) { + if (s.empty()) return std::basic_string(1, GetDot()); + std::basic_string input = s; + typename std::basic_string::size_type pos = input.length(); + PATH_CHAR_TYPE sep = GetPathSep(); + // remove trailing backslash + for (; pos > 1 && input[pos - 1] == sep; --pos) + ; + input.resize(pos); + for (; pos != 0 && input[pos - 1] != sep; --pos) + ; + return input.substr(pos); +} + +#else +inline std::string GetDirNameFromFilePath(const std::string& input) { + char* s = strdup(input.c_str()); + std::string ret = dirname(s); + free(s); + return ret; +} + +inline std::string GetLastComponent(const std::string& input) { + char* s = strdup(input.c_str()); + std::string ret = basename(s); + free(s); + return ret; +} + +inline FileType DTToFileType(unsigned char t) { + switch (t) { + case DT_BLK: + return FileType::TYPE_BLK; + case DT_CHR: + return FileType::TYPE_CHR; + case DT_DIR: + return FileType::TYPE_DIR; + case DT_FIFO: + return FileType::TYPE_FIFO; + case DT_LNK: + return FileType::TYPE_LNK; + case DT_REG: + return FileType::TYPE_REG; + case DT_SOCK: + return FileType::TYPE_SOCK; + default: + return FileType::TYPE_UNKNOWN; + } +} + +template +void LoopDir(const std::string& dir_name, T func) { + DIR* dir = opendir(dir_name.c_str()); + if (dir == nullptr) { + auto e = errno; + char buf[1024]; + char* msg; +#ifdef _GNU_SOURCE + msg = strerror_r(e, buf, sizeof(buf)); +#else + // for Mac OS X + if (strerror_r(e, buf, sizeof(buf)) != 0) { + buf[0] = '\0'; + } + msg = buf; +#endif + std::ostringstream oss; + oss << "couldn't open '" << dir_name << "':" << msg; + std::string s = oss.str(); + throw std::runtime_error(s); + } + try { + struct dirent* dp; + while ((dp = readdir(dir)) != nullptr) { + if (!func(dp->d_name, DTToFileType(dp->d_type))) { + break; + } + } + } catch (std::exception& ex) { + closedir(dir); + throw; + } + closedir(dir); +} +#endif +template +inline T ReplaceFilename(const T& input, const T& new_value) { + T ret = GetDirNameFromFilePath(input); + return ConcatPathComponent(ret, new_value); +} \ No newline at end of file diff --git a/onnxruntime/test/onnx/runner.cc b/onnxruntime/test/onnx/runner.cc new file mode 100644 index 0000000000000..e0684af1f222e --- /dev/null +++ b/onnxruntime/test/onnx/runner.cc @@ -0,0 +1,524 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "core/graph/onnx_protobuf.h" +#include "runner.h" + +#include +#include + +#include +#include +#include +#include +#include "test_allocator.h" +#include "path_lib.h" +#ifdef _WIN32 +#include +#else +#include +#include +#endif +#include +#include "TestCase.h" +#include "FixedCountFinishCallback.h" + +using namespace onnxruntime; +using ::onnxruntime::common::Status; + +void ONNXRUNTIME_CALLBACK RunTestCase(ONNXRUNTIME_CALLBACK_INSTANCE pci, void* context, ONNXRUNTIME_WORK work) { + OnnxRuntimeCloseThreadpoolWork(work); + assert(context != nullptr); + TestCaseTask* task((TestCaseTask*)context); + ITestCase* info = task->env.tests[task->task_id]; + std::shared_ptr ret; + try { + RunSingleTestCase(info, task->env.sf, task->concurrent_runs, task->repeat_count, task->pool, pci, [task](std::shared_ptr result, ONNXRUNTIME_CALLBACK_INSTANCE pci) { + return OnTestCaseFinished(pci, task, result); + }); + return; + } catch (std::exception& ex) { + LOGF_DEFAULT(ERROR, "Test %s failed:%s", info->GetTestCaseName().c_str(), ex.what()); + std::string node_name; + (void)info->GetNodeName(&node_name); + ret = std::make_shared(info->GetDataCount(), EXECUTE_RESULT::WITH_EXCEPTION, node_name); + } + auto status = OnTestCaseFinished(pci, task, ret); + if (!status.IsOK()) { + LOGF_DEFAULT(ERROR, "FATAL ERROR"); + abort(); + } +} + +void PTestRunner::Start(ONNXRUNTIME_CALLBACK_INSTANCE, size_t concurrent_runs) { + concurrent_runs = std::min(std::max(1, concurrent_runs), c_->GetDataCount()); + next_test_to_run = 0; + for (size_t i = 0; i != concurrent_runs; ++i) { + if (!ScheduleNew()) { + throw std::runtime_error("ScheduleNew task failed"); + } + } +} + +bool PTestRunner::ScheduleNew() { + size_t next_test = next_test_to_run++; + if (next_test >= c_->GetDataCount()) return false; + DataTask* t = new DataTask{this, next_test}; + Status st = CreateAndSubmitThreadpoolWork(RunSingleDataItem, t, tpool_); + if (!st.IsOK()) { + delete t; + LOGF_DEFAULT(ERROR, "schedule test task failed: %s\n", st.ErrorMessage().c_str()); + return false; + } + return true; +} + +void PTestRunner::OnTaskFinished(size_t, EXECUTE_RESULT, ONNXRUNTIME_CALLBACK_INSTANCE pci) noexcept { + try { + ScheduleNew(); + if (++finished == c_->GetDataCount()) { + //For each test case, only one DataTask can reach here + finish(pci); + } + } catch (std::exception& ex) { + LOGF_DEFAULT(ERROR, "%s:unrecoverable error:%s,exit...\n", c_->GetTestCaseName().c_str(), ex.what()); + abort(); + } catch (...) { + LOGF_DEFAULT(ERROR, "%s:unrecoverable error,exit...\n", c_->GetTestCaseName().c_str()); + abort(); + } +} + +PTestRunner::PTestRunner(ONNXSessionPtr session1, + ITestCase* c, PThreadPool tpool, + TestCaseCallBack on_finished1) : DataRunner(session1, c->GetTestCaseName(), c, on_finished1), next_test_to_run(0), finished(0), tpool_(tpool) { +} + +void ONNXRUNTIME_CALLBACK RunSingleDataItem(ONNXRUNTIME_CALLBACK_INSTANCE instance, void* context, ONNXRUNTIME_WORK work) { + OnnxRuntimeCloseThreadpoolWork(work); + DataTask* task((DataTask*)context); + PTestRunner* env = task->env; + const size_t task_id = task->task_id; + delete task; + env->RunTask(task_id, instance, true); +} + +Status OnTestCaseFinished(ONNXRUNTIME_CALLBACK_INSTANCE pci, TestCaseTask* task, std::shared_ptr result) { + FixedCountFinishCallback* finished = task->env.finished; + auto task_id = task->task_id; + bool failed = false; + { + std::unique_ptr unused(task); + TestEnv& env = task->env; + int next_test = env.next_test_to_run++; + if (static_cast(next_test) < env.tests.size()) { + //schedule the next TestCase + std::unique_ptr t(new TestCaseTask{env, next_test, task->concurrent_runs, task->repeat_count, task->pool}); + Status st = CreateAndSubmitThreadpoolWork(RunTestCase, t.get(), task->pool); + if (st.IsOK()) { + t.release(); + } else + return st; + } + } + if (failed) + return finished->fail(pci); + return finished->onFinished(task_id, result, pci); +} + +//Do not run this function in the thread pool passed in +static Status ParallelRunTests(TestEnv& env, int p_models, size_t current_runs, size_t repeat_count, PThreadPool pool) { + p_models = (int)std::min(p_models, env.tests.size()); + LOGF_DEFAULT(ERROR, "Running tests in parallel: at most %d models at any time", p_models); + env.next_test_to_run = p_models; + for (int i = 0; i != p_models; ++i) { + TestCaseTask* t(new TestCaseTask{env, i, current_runs, repeat_count, pool}); + try { + auto st = CreateAndSubmitThreadpoolWork(RunTestCase, t, pool); + if (!st.IsOK()) { + delete t; + return st; + } + } catch (std::exception&) { + delete t; + throw; + } + } + bool ret = env.finished->wait(); + if (!ret) { + return Status(::onnxruntime::common::ONNXRUNTIME, ::onnxruntime::common::FAIL, "ParallelRunTests failed"); + } + LOGF_DEFAULT(ERROR, "Running tests finished. Generating report"); + return Status::OK(); +} + +Status RunTests(TestEnv& env, int p_models, int concurrent_runs, size_t repeat_count, PThreadPool tpool) { + TestResultStat& stat = env.stat; + stat.total_model_count = env.tests.size(); + stat.total_test_case_count = std::accumulate(env.tests.begin(), env.tests.end(), static_cast(0), [](size_t v, const ITestCase* info) { + return info->GetDataCount() + v; + }); + std::vector> results; + if (p_models > 1 && env.tests.size() > 1) { + ONNXRUNTIME_RETURN_IF_ERROR(ParallelRunTests(env, p_models, concurrent_runs, repeat_count, tpool)); + results = env.finished->getResults(); + } else { + //run models one by one + for (size_t i = 0; i != env.tests.size(); ++i) { + const char* test_case_name = env.tests[i]->GetTestCaseName().c_str(); + ONNXRUNTIME_EVENT ev; + ONNXRUNTIME_RETURN_IF_ERROR(CreateOnnxRuntimeEvent(&ev)); + try { + RunSingleTestCase(env.tests[i], env.sf, concurrent_runs, repeat_count, tpool, nullptr, [repeat_count, &results, ev, concurrent_runs, test_case_name](std::shared_ptr result, ONNXRUNTIME_CALLBACK_INSTANCE pci) { + //TODO:output this information to a xml + if (concurrent_runs == 1) { + TIME_SPEC ts = result->GetSpentTime(); + double spent = TimeSpecToSeconds(&ts); + double spent2 = spent / result->GetExcutionResult().size() / repeat_count; + LOGF_DEFAULT(ERROR, "Test %s finished in %.3g seconds, took %.3g for each input", test_case_name, spent, spent2); + } + results.push_back(result); + return OnnxRuntimeSetEventWhenCallbackReturns(pci, ev); + }); + ONNXRUNTIME_RETURN_IF_ERROR(WaitAndCloseEvent(ev)); + } catch (std::exception& ex) { + LOGF_DEFAULT(ERROR, "Test %s failed:%s", test_case_name, ex.what()); + std::string node_name; + (void)env.tests[i]->GetNodeName(&node_name); + results.push_back( + std::make_shared(env.tests[i]->GetDataCount(), EXECUTE_RESULT::WITH_EXCEPTION, node_name)); + ONNXRuntimeCloseEvent(ev); + } + } + } + for (size_t i = 0; i != env.tests.size(); ++i) { + if (!results[i]) { + stat.AddFailedTest(env.tests[i]->GetTestCaseName()); + continue; + } + const TestCaseResult& r = *results[i]; + for (const EXECUTE_RESULT res : r.GetExcutionResult()) { + if (res != EXECUTE_RESULT::SUCCESS && res != EXECUTE_RESULT::NOT_SUPPORT) { + stat.AddFailedTest(env.tests[i]->GetTestCaseName()); + } + switch (res) { + case EXECUTE_RESULT::SUCCESS: + stat.succeeded++; + break; + case EXECUTE_RESULT::INVALID_ARGUMENT: + case EXECUTE_RESULT::UNKNOWN_ERROR: + if (!r.node_name.empty()) stat.AddFailedKernels(r.node_name); + break; + case EXECUTE_RESULT::INVALID_GRAPH: + stat.invalid_graph++; + break; + case EXECUTE_RESULT::WITH_EXCEPTION: + stat.throwed_exception++; + if (!r.node_name.empty()) stat.AddFailedKernels(r.node_name); + break; + case EXECUTE_RESULT::RESULT_DIFFERS: + stat.result_differs++; + if (!r.node_name.empty()) stat.AddFailedKernels(r.node_name); + break; + case EXECUTE_RESULT::MODEL_SHAPE_MISMATCH: + case EXECUTE_RESULT::SHAPE_MISMATCH: + case EXECUTE_RESULT::MODEL_TYPE_MISMATCH: + case EXECUTE_RESULT::TYPE_MISMATCH: + stat.result_differs++; + if (!r.node_name.empty()) stat.AddFailedKernels(r.node_name); + break; + case EXECUTE_RESULT::NOT_SUPPORT: + stat.not_implemented++; + if (!r.node_name.empty()) stat.AddNotImplementedKernels(r.node_name); + break; + case EXECUTE_RESULT::LOAD_MODEL_FAILED: + stat.load_model_failed++; + if (!r.node_name.empty()) stat.AddFailedKernels(r.node_name); + break; + default: + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "unknown result"); + } + } + } + return common::Status::OK(); +} + +std::vector LoadTests(const std::vector>& input_paths, + const std::vector>& whitelisted_test_cases, + ONNXRuntimeAllocator* env) { + std::vector tests; + std::vector> paths(input_paths); + while (!paths.empty()) { + std::basic_string node_data_root_path = paths.back(); + paths.pop_back(); + std::basic_string my_dir_name = GetLastComponent(node_data_root_path); + LoopDir(node_data_root_path, [&](const PATH_CHAR_TYPE* filename, FileType f_type) -> bool { + if (filename[0] == '.') return true; + if (f_type == FileType::TYPE_DIR) { + std::basic_string p = ConcatPathComponent(node_data_root_path, filename); + paths.push_back(p); + return true; + } + std::basic_string filename_str = filename; + if (!HasExtensionOf(filename_str, ONNXRUNTIME_TSTR("onnx"))) return true; + + std::basic_string test_case_name = my_dir_name; + if (test_case_name.compare(0, 5, ONNXRUNTIME_TSTR("test_")) == 0) test_case_name = test_case_name.substr(5); + if (!whitelisted_test_cases.empty() && std::find(whitelisted_test_cases.begin(), whitelisted_test_cases.end(), test_case_name) == whitelisted_test_cases.end()) { + return true; + } + std::basic_string p = ConcatPathComponent(node_data_root_path, filename_str); + + ITestCase* l = CreateOnnxTestCase(env, ToMBString(test_case_name)); + auto status = l->SetModelPath(p.c_str()); + if (!status.IsOK()) { + LOGF_DEFAULT(ERROR, "load data from %s failed:%s\n", p.c_str(), status.ErrorMessage().c_str()); + delete l; + return true; + } + tests.push_back(l); + return true; + }); + } + return tests; +} + +SeqTestRunner::SeqTestRunner(ONNXSessionPtr session1, + ITestCase* c, size_t repeat_count, + TestCaseCallBack on_finished1) : DataRunner(session1, c->GetTestCaseName(), c, on_finished1), repeat_count_(repeat_count) { +} + +DataRunner::DataRunner(ONNXSessionPtr session1, const std::string& test_case_name1, ITestCase* c, TestCaseCallBack on_finished1) : test_case_name_(test_case_name1), c_(c), session(session1), on_finished(on_finished1), default_allocator(MockedONNXRuntimeAllocator::Create()) { + std::string s; + c->GetNodeName(&s); + result = std::make_shared(c->GetDataCount(), EXECUTE_RESULT::UNKNOWN_ERROR, s); + SetTimeSpecToZero(&spent_time_); +} + +void DataRunner::RunTask(size_t task_id, ONNXRUNTIME_CALLBACK_INSTANCE pci, bool store_result) { + EXECUTE_RESULT res = EXECUTE_RESULT::UNKNOWN_ERROR; + try { + res = RunTaskImpl(task_id); + } catch (std::exception& ex) { + res = EXECUTE_RESULT::WITH_EXCEPTION; + LOGF_DEFAULT(ERROR, "%s:%s", c_->GetTestCaseName().c_str(), ex.what()); + } + if (store_result) { + result->SetResult(task_id, res); + } + OnTaskFinished(task_id, res, pci); +} + +std::pair CompareGenericValue(const ONNXValuePtr o, const ONNXValuePtr expected_mlvalue, double per_sample_tolerance, double relative_per_sample_tolerance, + bool post_processing) { + return onnxruntime::CompareMLValue(*(MLValue*)o, *(MLValue*)expected_mlvalue, per_sample_tolerance, relative_per_sample_tolerance, post_processing); +} +EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) { + std::unordered_map feeds; + common::Status status = c_->LoadTestData(session, task_id, feeds, true); + if (!status.IsOK()) { + LOGF_DEFAULT(ERROR, "%s", status.ErrorMessage().c_str()); + return StatusCodeToExecuteResult(status.Code()); + } + + // Create output feed + size_t output_count; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeInferenceSessionGetOutputCount(session, &output_count)); + std::vector output_names(output_count); + for (size_t i = 0; i != output_count; ++i) { + char* output_name = nullptr; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeInferenceSessionGetOutputName(session, i, default_allocator, &output_name)); + assert(output_name != nullptr); + output_names[i] = output_name; + (*default_allocator)->Free(default_allocator, output_name); + } + + TIME_SPEC start_time, end_time; + GetMonotonicTimeCounter(&start_time); + std::vector input_names(feeds.size()); + std::vector input_values(feeds.size()); + size_t input_index = 0; + for (auto& kvp : feeds) { + input_names[input_index] = kvp.first.c_str(); + input_values[input_index] = kvp.second; + ++input_index; + } + std::vector output_values(output_count); + { + std::vector output_names_raw_ptr(output_count); + for (size_t i = 0; i != output_count; ++i) { + output_names_raw_ptr[i] = output_names[i].c_str(); + } + auto onnx_status = ONNXRuntimeRunInference(session, input_names.data(), input_values.data(), input_index, output_names_raw_ptr.data(), output_count, output_values.data()); + if (onnx_status != nullptr) { + std::string onnx_runtime_error_message = ONNXRuntimeGetErrorMessage(onnx_status); + ReleaseONNXStatus(onnx_status); + for (auto& kvp : feeds) { + ReleaseONNXValue(kvp.second); + } + throw std::runtime_error(onnx_runtime_error_message); + } + } + GetMonotonicTimeCounter(&end_time); + AccumulateTimeSpec(&spent_time_, &start_time, &end_time); + for (auto& kvp : feeds) { + ReleaseONNXValue(kvp.second); + } + if (!status.IsOK()) { + LOGF_DEFAULT(ERROR, "%s:%s\n", test_case_name_.c_str(), status.ErrorMessage().c_str()); + return StatusCodeToExecuteResult(status.Code()); + } + + double per_sample_tolerance, relative_per_sample_tolerance; + bool post_procesing; + + if (!(status = c_->GetPerSampleTolerance(&per_sample_tolerance)).IsOK()) { + LOGF_DEFAULT(ERROR, "%s", status.ErrorMessage().c_str()); + return StatusCodeToExecuteResult(status.Code()); + } + if (!(status = c_->GetRelativePerSampleTolerance(&relative_per_sample_tolerance)).IsOK()) { + LOGF_DEFAULT(ERROR, "%s", status.ErrorMessage().c_str()); + return StatusCodeToExecuteResult(status.Code()); + } + if (!(status = c_->GetPostProcessing(&post_procesing)).IsOK()) { + LOGF_DEFAULT(ERROR, "%s", status.ErrorMessage().c_str()); + return StatusCodeToExecuteResult(status.Code()); + } + + //TODO: if there are no output value files, just skip the validation + std::unordered_map expected_output_values; + status = c_->LoadTestData(session, task_id, expected_output_values, false); + if (!status.IsOK()) { + LOGF_DEFAULT(ERROR, "%s", status.ErrorMessage().c_str()); + return StatusCodeToExecuteResult(status.Code()); + } + std::unordered_map name_fetch_output_map; + std::unordered_map name_output_value_info_proto; + int i = 0; + for (auto& output_name : output_names) { + // p_fetches is filled in the order of output_names. + name_fetch_output_map[output_name] = output_values[i]; + const onnx::ValueInfoProto& infoProto = c_->GetOutputInfoFromModel(i); + name_output_value_info_proto.insert(std::make_pair(infoProto.name(), &infoProto)); + i++; + } + + EXECUTE_RESULT res = EXECUTE_RESULT::SUCCESS; + for (auto& output : expected_output_values) { + ONNXValuePtr expected_output_value = output.second; + const std::string& output_name = output.first; + auto iter = name_fetch_output_map.find(output_name); + if (iter == name_fetch_output_map.end()) { + res = EXECUTE_RESULT::INVALID_GRAPH; + LOGF_DEFAULT(ERROR, "cannot find %s in the outputs", output_name.c_str()); + break; + } + ONNXValuePtr actual_output_value = iter->second; + std::pair ret = CompareGenericValue(actual_output_value, expected_output_value, per_sample_tolerance, relative_per_sample_tolerance, post_procesing); + COMPARE_RESULT compare_result = ret.first; + if (compare_result == COMPARE_RESULT::SUCCESS) { + const onnx::ValueInfoProto& v = *name_output_value_info_proto[output_name]; + ret = VerifyValueInfo(v, *(MLValue*)actual_output_value); + compare_result = ret.first; + if (compare_result != COMPARE_RESULT::SUCCESS) { + switch (compare_result) { + case COMPARE_RESULT::NOT_SUPPORT: + res = EXECUTE_RESULT::NOT_SUPPORT; + break; + case COMPARE_RESULT::SHAPE_MISMATCH: + res = EXECUTE_RESULT::MODEL_SHAPE_MISMATCH; + break; + case COMPARE_RESULT::TYPE_MISMATCH: + res = EXECUTE_RESULT::MODEL_TYPE_MISMATCH; + break; + default: + res = EXECUTE_RESULT::UNKNOWN_ERROR; + } + } + } else { + switch (compare_result) { + case COMPARE_RESULT::NOT_SUPPORT: + res = EXECUTE_RESULT::NOT_SUPPORT; + break; + case COMPARE_RESULT::RESULT_DIFFERS: + res = EXECUTE_RESULT::RESULT_DIFFERS; + break; + case COMPARE_RESULT::SHAPE_MISMATCH: + res = EXECUTE_RESULT::SHAPE_MISMATCH; + break; + case COMPARE_RESULT::TYPE_MISMATCH: + res = EXECUTE_RESULT::TYPE_MISMATCH; + break; + default: + res = EXECUTE_RESULT::UNKNOWN_ERROR; + } + } + if (compare_result != COMPARE_RESULT::SUCCESS && !ret.second.empty()) { + LOGF_DEFAULT(ERROR, "%s:%s", test_case_name_.c_str(), ret.second.c_str()); + } + if (compare_result != COMPARE_RESULT::SUCCESS) { + break; + } + } + for (auto& kvp : expected_output_values) { + ReleaseONNXValue(kvp.second); + } + for (ONNXValuePtr p : output_values) { + ReleaseONNXValue(p); + } + return res; +} + +void SeqTestRunner::Start(ONNXRUNTIME_CALLBACK_INSTANCE pci, size_t) { + const size_t data_count = c_->GetDataCount(); + for (size_t idx_repeat = 0; idx_repeat != repeat_count_; ++idx_repeat) + for (size_t idx_data = 0; idx_data != data_count; ++idx_data) { + RunTask(idx_data, nullptr, idx_repeat == 0); + } + finish(pci); +} + +void RunSingleTestCase(ITestCase* info, const onnxruntime::SessionOptionsWrapper& sf, size_t concurrent_runs, size_t repeat_count, PThreadPool tpool, ONNXRUNTIME_CALLBACK_INSTANCE pci, TestCaseCallBack on_finished) { + std::shared_ptr ret; + size_t data_count = info->GetDataCount(); + { + DataRunner* r = nullptr; + std::string node_name; + Status status = info->GetNodeName(&node_name); + if (!status.IsOK()) { + LOGF_DEFAULT(ERROR, "load model %s failed:%s\n", info->GetTestCaseName().c_str(), status.ErrorMessage().c_str()); + ret = std::make_shared(data_count, StatusCodeToExecuteResult(status.Code()), node_name); + goto end; + } + auto sf2 = sf.clone(); + sf2.SetSessionLogId(info->GetTestCaseName().c_str()); + std::unique_ptr session_object( + sf2.ONNXRuntimeCreateInferenceSession(info->GetModelUrl()), ReleaseONNXSession); + LOGF_DEFAULT(INFO, "testing %s\n", info->GetTestCaseName().c_str()); + //temp hack. Because we have no resource control. We may not have enough memory to run this test in parallel + if (info->GetTestCaseName() == "coreml_FNS-Candy_ImageNet") + concurrent_runs = 1; + if (concurrent_runs > 1 && data_count > 1) { + r = new PTestRunner(session_object.get(), info, tpool, on_finished); + } else { + r = new SeqTestRunner(session_object.get(), info, repeat_count, on_finished); + } + session_object.release(); + r->Start(pci, concurrent_runs); + return; + } +end: + on_finished(ret, pci); +} + +EXECUTE_RESULT StatusCodeToExecuteResult(int input) { + switch (input) { + case common::NOT_IMPLEMENTED: + return EXECUTE_RESULT::NOT_SUPPORT; + case common::INVALID_GRAPH: + return EXECUTE_RESULT::INVALID_GRAPH; + case common::INVALID_ARGUMENT: + return EXECUTE_RESULT::INVALID_ARGUMENT; + default: + return EXECUTE_RESULT::UNKNOWN_ERROR; + } +} diff --git a/onnxruntime/test/onnx/runner.h b/onnxruntime/test/onnx/runner.h new file mode 100644 index 0000000000000..1a13c076946cb --- /dev/null +++ b/onnxruntime/test/onnx/runner.h @@ -0,0 +1,138 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include +#include "core/common/common.h" +#include "core/common/logging/logging.h" +#include "core/platform/env_time.h" +#include "core/session/onnxruntime_cxx_api.h" + +#include "TestCase.h" +#include "TestCaseResult.h" + +#include "testenv.h" +#include "sync_api.h" + +typedef std::function<::onnxruntime::common::Status(std::shared_ptr result, ONNXRUNTIME_CALLBACK_INSTANCE pci)> TestCaseCallBack; + +struct TestCaseTask { + TestEnv& env; + const int task_id; + //The max number of concurrent Session::Run() for each model + const size_t concurrent_runs; + const size_t repeat_count; + const PThreadPool pool; +}; + +void ONNXRUNTIME_CALLBACK RunTestCase(ONNXRUNTIME_CALLBACK_INSTANCE instance, void* context, ONNXRUNTIME_WORK work); +//TODO: implement this function for Linux +void ONNXRUNTIME_CALLBACK RunSingleDataItem(ONNXRUNTIME_CALLBACK_INSTANCE instance, void* context, ONNXRUNTIME_WORK work); +::onnxruntime::common::Status OnTestCaseFinished(ONNXRUNTIME_CALLBACK_INSTANCE pci, TestCaseTask* task, std::shared_ptr result); + +class DataRunner { + protected: + typedef TestCaseCallBack CALL_BACK; + std::shared_ptr result; + std::string test_case_name_; + ITestCase* c_; + //Time spent in Session::Run. It only make sense when SeqTestRunner was used + ::onnxruntime::TIME_SPEC spent_time_; + + private: + ONNXSessionPtr session; + CALL_BACK on_finished; + ONNXRuntimeAllocatorInteface** const default_allocator; + EXECUTE_RESULT RunTaskImpl(size_t task_id); + ONNXRUNTIME_DISALLOW_COPY_AND_ASSIGNMENT(DataRunner); + + public: + DataRunner(ONNXSessionPtr session1, const std::string& test_case_name1, ITestCase* c, TestCaseCallBack on_finished1); + virtual void OnTaskFinished(size_t task_id, EXECUTE_RESULT res, ONNXRUNTIME_CALLBACK_INSTANCE pci) noexcept = 0; + void RunTask(size_t task_id, ONNXRUNTIME_CALLBACK_INSTANCE pci, bool store_result); + virtual ~DataRunner() { + ReleaseONNXSession(session); + ONNXRuntimeReleaseObject(default_allocator); + } + + virtual void Start(ONNXRUNTIME_CALLBACK_INSTANCE pci, size_t concurrent_runs) = 0; + + void finish(ONNXRUNTIME_CALLBACK_INSTANCE pci) { + std::shared_ptr res = result; + CALL_BACK callback = on_finished; + res->SetSpentTime(spent_time_); + const std::vector& er = res->GetExcutionResult(); + for (size_t i = 0; i != er.size(); ++i) { + EXECUTE_RESULT r = er[i]; + if (r == EXECUTE_RESULT::SUCCESS) continue; + std::string s = c_->GetDatasetDebugInfoString(i); + switch (r) { + case EXECUTE_RESULT::RESULT_DIFFERS: + LOGF_DEFAULT(ERROR, "%s: result differs. Dataset:%s\n", test_case_name_.c_str(), s.c_str()); + break; + case EXECUTE_RESULT::SHAPE_MISMATCH: + LOGF_DEFAULT(ERROR, "%s: shape mismatch. Dataset:%s\n", test_case_name_.c_str(), s.c_str()); + break; + case EXECUTE_RESULT::TYPE_MISMATCH: + LOGF_DEFAULT(ERROR, "%s: type mismatch. Dataset:%s\n", test_case_name_.c_str(), s.c_str()); + break; + case EXECUTE_RESULT::MODEL_SHAPE_MISMATCH: + LOGF_DEFAULT(ERROR, "%s: shape in model file mismatch. Dataset:%s\n", test_case_name_.c_str(), s.c_str()); + break; + case EXECUTE_RESULT::MODEL_TYPE_MISMATCH: + LOGF_DEFAULT(ERROR, "%s: type in model file mismatch. Dataset:%s\n", test_case_name_.c_str(), s.c_str()); + default: + //nothing to do + break; + } + break; + } + delete this; + callback(res, pci); + } +}; + +class SeqTestRunner : public DataRunner { + private: + size_t repeat_count_; + + public: + SeqTestRunner(ONNXSessionPtr session1, + ITestCase* c, size_t repeat_count, + TestCaseCallBack on_finished1); + + void Start(ONNXRUNTIME_CALLBACK_INSTANCE pci, size_t concurrent_runs) override; + void OnTaskFinished(size_t, EXECUTE_RESULT, ONNXRUNTIME_CALLBACK_INSTANCE) noexcept override {} +}; + +class PTestRunner : public DataRunner { + private: + std::atomic next_test_to_run; + std::atomic finished; + void OnTaskFinished(size_t task_id, EXECUTE_RESULT res, ONNXRUNTIME_CALLBACK_INSTANCE pci) noexcept override; + + public: + void Start(ONNXRUNTIME_CALLBACK_INSTANCE pci, size_t concurrent_runs) override; + + PTestRunner(ONNXSessionPtr session1, + ITestCase* c, PThreadPool tpool, + TestCaseCallBack on_finished1); + + private: + bool ScheduleNew(); + const PThreadPool tpool_; +}; + +struct DataTask { + PTestRunner* env; + const size_t task_id; +}; + +std::vector LoadTests(const std::vector>& input_paths, + const std::vector>& whitelisted_test_cases, + ONNXRuntimeAllocator* env); +//Do not run this function in the thread pool passed in +::onnxruntime::common::Status RunTests(TestEnv& env, int p_models, int concurrent_runs, size_t repeat_count, PThreadPool tpool); +EXECUTE_RESULT StatusCodeToExecuteResult(int input); +void RunSingleTestCase(ITestCase* info, const onnxruntime::SessionOptionsWrapper& sf, size_t concurrent_runs, size_t repeat_count, PThreadPool tpool, ONNXRUNTIME_CALLBACK_INSTANCE pci, TestCaseCallBack on_finished); diff --git a/onnxruntime/test/onnx/simple_thread_pool.h b/onnxruntime/test/onnx/simple_thread_pool.h new file mode 100644 index 0000000000000..fab060e7df9cb --- /dev/null +++ b/onnxruntime/test/onnx/simple_thread_pool.h @@ -0,0 +1,151 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Benoit Steiner +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#pragma once +#include + +//: copied from Eigen, with just one tiny modification: remove the default vaule of the constructor of SimpleThreadPoolTempl +namespace onnxruntime { + +// The implementation of the ThreadPool type ensures that the Schedule method +// runs the functions it is provided in FIFO order when the scheduling is done +// by a single thread. +// Environment provides a way to create threads and also allows to intercept +// task submission and execution. +template +class SimpleThreadPoolTempl : public Eigen::ThreadPoolInterface { + public: + // Construct a pool that contains "num_threads" threads. + explicit SimpleThreadPoolTempl(int num_threads, const Environment& env) + : env_(env), threads_(num_threads), waiters_(num_threads) { + for (int i = 0; i < num_threads; i++) { + threads_.push_back(env.CreateThread([this, i]() { WorkerLoop(i); })); + } + } + + // Wait until all scheduled work has finished and then destroy the + // set of threads. + ~SimpleThreadPoolTempl() { + { + // Wait for all work to get done. + std::unique_lock l(mu_); + while (!pending_.empty()) { + empty_.wait(l); + } + exiting_ = true; + + // Wakeup all waiters. + for (auto w : waiters_) { + w->ready = true; + w->task.f = nullptr; + w->cv.notify_one(); + } + } + + // Wait for threads to finish. + for (auto t : threads_) { + delete t; + } + } + + // Schedule fn() for execution in the pool of threads. The functions are + // executed in the order in which they are scheduled. + void Schedule(std::function fn) final { + Task t = env_.CreateTask(std::move(fn)); + std::unique_lock l(mu_); + if (waiters_.empty()) { + pending_.push_back(std::move(t)); + } else { + Waiter* w = waiters_.back(); + waiters_.pop_back(); + w->ready = true; + w->task = std::move(t); + w->cv.notify_one(); + } + } + + int NumThreads() const final { + return static_cast(threads_.size()); + } + + int CurrentThreadId() const final { + const PerThread* pt = this->GetPerThread(); + if (pt->pool == this) { + return pt->thread_id; + } else { + return -1; + } + } + + protected: + void WorkerLoop(int thread_id) { + std::unique_lock l(mu_); + PerThread* pt = GetPerThread(); + pt->pool = this; + pt->thread_id = thread_id; + Waiter w; + Task t; + while (!exiting_) { + if (pending_.empty()) { + // Wait for work to be assigned to me + w.ready = false; + waiters_.push_back(&w); + while (!w.ready) { + w.cv.wait(l); + } + t = w.task; + w.task.f = nullptr; + } else { + // Pick up pending work + t = std::move(pending_.front()); + pending_.pop_front(); + if (pending_.empty()) { + empty_.notify_all(); + } + } + if (t.f) { + mu_.unlock(); + env_.ExecuteTask(t); + t.f = nullptr; + mu_.lock(); + } + } + } + + private: + typedef typename Environment::Task Task; + typedef typename Environment::EnvThread Thread; + + struct Waiter { + std::condition_variable cv; + Task task; + bool ready; + }; + + struct PerThread { + constexpr PerThread() : pool(NULL), thread_id(-1) {} + SimpleThreadPoolTempl* pool; // Parent pool, or null for normal threads. + int thread_id; // Worker thread index in pool. + }; + + const Environment& env_; + std::mutex mu_; + Eigen::MaxSizeVector threads_; // All threads + Eigen::MaxSizeVector waiters_; // Stack of waiting threads. + std::deque pending_; // Queue of pending work + std::condition_variable empty_; // Signaled on pending_.empty() + bool exiting_ = false; + + PerThread* GetPerThread() const { + EIGEN_THREAD_LOCAL PerThread per_thread; + return &per_thread; + } +}; + +} // namespace onnxruntime diff --git a/onnxruntime/test/onnx/sync_api.h b/onnxruntime/test/onnx/sync_api.h new file mode 100644 index 0000000000000..00c9e83a6592a --- /dev/null +++ b/onnxruntime/test/onnx/sync_api.h @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#ifdef _WIN32 +#include +#else +#include +#endif +#include +#include +#include + +#ifdef _WIN32 +using ONNXRUNTIME_CALLBACK_INSTANCE = PTP_CALLBACK_INSTANCE; +using ONNXRUNTIME_EVENT = HANDLE; +#define ONNXRUNTIME_CALLBACK __stdcall +using ONNXRUNTIME_WORK = PTP_WORK; +using PThreadPool = PTP_CALLBACK_ENVIRON; +using ONNXRUNTIME_CALLBACK_FUNCTION = PTP_WORK_CALLBACK; +#define OnnxRuntimeCloseThreadpoolWork CloseThreadpoolWork +inline PThreadPool GetDefaultThreadPool(const ::onnxruntime::Env&) { + return nullptr; +} +#else +#define ONNXRUNTIME_CALLBACK +namespace Eigen { +class ThreadPoolInterface; +} +using PThreadPool = Eigen::ThreadPoolInterface*; +#define ONNXRUNTIME_WORK void* +struct OnnxRuntimeEvent; +using ONNXRUNTIME_EVENT = OnnxRuntimeEvent*; + +class OnnxRuntimeCallbackInstance; +using ONNXRUNTIME_CALLBACK_INSTANCE = OnnxRuntimeCallbackInstance*; +using ONNXRUNTIME_CALLBACK_FUNCTION = void ONNXRUNTIME_CALLBACK (*)(ONNXRUNTIME_CALLBACK_INSTANCE pci, void* context, ONNXRUNTIME_WORK work); +//Do nothing +inline void OnnxRuntimeCloseThreadpoolWork(ONNXRUNTIME_WORK) {} +#endif + +//The returned value will be used with CreateAndSubmitThreadpoolWork function +PThreadPool GetDefaultThreadPool(const ::onnxruntime::Env& env); +//On Windows, the last parameter can be null, in that case it will use the default thread pool. +//On Linux, there is no per process default thread pool. You have to pass a non-null pointer. +//Caller must delete the data pointer if this function returns a non-ok status. Otherwise, the ownership is transferred +::onnxruntime::common::Status CreateAndSubmitThreadpoolWork(ONNXRUNTIME_CALLBACK_FUNCTION callback, void* data, PThreadPool pool); +::onnxruntime::common::Status CreateOnnxRuntimeEvent(ONNXRUNTIME_EVENT* out); +//pci is a pointer, can be NULL. If pci is NULL, signal the event immediately +::onnxruntime::common::Status OnnxRuntimeSetEventWhenCallbackReturns(ONNXRUNTIME_CALLBACK_INSTANCE pci, ONNXRUNTIME_EVENT finish_event); +::onnxruntime::common::Status WaitAndCloseEvent(ONNXRUNTIME_EVENT finish_event); +void ONNXRuntimeCloseEvent(ONNXRUNTIME_EVENT finish_event); diff --git a/onnxruntime/test/onnx/sync_api_linux.cc b/onnxruntime/test/onnx/sync_api_linux.cc new file mode 100644 index 0000000000000..c35ed430284ce --- /dev/null +++ b/onnxruntime/test/onnx/sync_api_linux.cc @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "sync_api.h" +#include +#include +#include +#include +#include "simple_thread_pool.h" +#include "onnxruntime_event.h" + +using onnxruntime::common::Status; + +//this can be passed to one of the following functions: +//OnnxRuntimeSetEventWhenCallbackReturns +class OnnxRuntimeCallbackInstance { + private: + std::vector events_to_signal_; + + public: + void AddEvent(ONNXRUNTIME_EVENT event); + onnxruntime::common::Status SignalAllEvents(); +}; + +Status WaitAndCloseEvent(ONNXRUNTIME_EVENT finish_event) { + if (finish_event == nullptr) + return Status(onnxruntime::common::ONNXRUNTIME, onnxruntime::common::INVALID_ARGUMENT, ""); + pthread_mutex_lock(&finish_event->finish_event_mutex); + while (!finish_event->finished) { + pthread_cond_wait(&finish_event->finish_event_data, &finish_event->finish_event_mutex); + } + pthread_mutex_unlock(&finish_event->finish_event_mutex); + delete finish_event; + return Status::OK(); +} + +Status CreateAndSubmitThreadpoolWork(ONNXRUNTIME_CALLBACK_FUNCTION callback, void* data, PThreadPool pool) { + if (callback == nullptr) + return Status(onnxruntime::common::ONNXRUNTIME, onnxruntime::common::INVALID_ARGUMENT, "callback cannot be NULL"); + if (pool == nullptr) + return Status(onnxruntime::common::ONNXRUNTIME, onnxruntime::common::INVALID_ARGUMENT, "pool cannot be NULL"); + pool->Schedule([=]() { + OnnxRuntimeCallbackInstance instance; + callback(&instance, data, nullptr); + Status st = instance.SignalAllEvents(); + if (!st.IsOK()) { + LOGF_DEFAULT(ERROR, "SignalAllEvents failed:%s. aborting...\n", st.ErrorMessage().c_str()); + abort(); + } + }); + return Status::OK(); +} + +using DefaultThreadPoolType = onnxruntime::SimpleThreadPoolTempl; +static std::unique_ptr default_pool; +static std::once_flag default_pool_init; + +PThreadPool GetDefaultThreadPool(const onnxruntime::Env& env) { + std::call_once(default_pool_init, [&env] { + int core_num = env.GetNumCpuCores(); + default_pool.reset(new DefaultThreadPoolType(core_num, env)); + }); + return default_pool.get(); +} + +Status OnnxRuntimeSetEventWhenCallbackReturns(ONNXRUNTIME_CALLBACK_INSTANCE pci, ONNXRUNTIME_EVENT finish_event) { + if (finish_event == nullptr) + return Status(onnxruntime::common::ONNXRUNTIME, onnxruntime::common::INVALID_ARGUMENT, ""); + + if (pci == nullptr) { + if (pthread_mutex_lock(&finish_event->finish_event_mutex)) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "lock failed"); + } + finish_event->finished = true; + if (pthread_mutex_unlock(&finish_event->finish_event_mutex)) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "unlock failed"); + if (!pthread_cond_broadcast(&finish_event->finish_event_data)) + return Status::OK(); + else + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "pthread_cond_broadcast failed"); + } else { + pci->AddEvent(finish_event); + return Status::OK(); + } +} + +void OnnxRuntimeCallbackInstance::AddEvent(ONNXRUNTIME_EVENT event) { + events_to_signal_.push_back(event); +} + +Status OnnxRuntimeCallbackInstance::SignalAllEvents() { + for (ONNXRUNTIME_EVENT finish_event : events_to_signal_) { + if (pthread_mutex_lock(&finish_event->finish_event_mutex)) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "lock failed"); + } + finish_event->finished = true; + if (pthread_mutex_unlock(&finish_event->finish_event_mutex)) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "unlock failed"); + if (pthread_cond_broadcast(&finish_event->finish_event_data)) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "pthread_cond_broadcast failed"); + } + return Status::OK(); +} + +Status CreateOnnxRuntimeEvent(ONNXRUNTIME_EVENT* out) { + if (out == nullptr) + return Status(onnxruntime::common::ONNXRUNTIME, onnxruntime::common::INVALID_ARGUMENT, ""); + *out = new OnnxRuntimeEvent(); + return Status::OK(); +} + +void ONNXRuntimeCloseEvent(ONNXRUNTIME_EVENT finish_event) { + delete finish_event; +} diff --git a/onnxruntime/test/onnx/sync_api_win.cc b/onnxruntime/test/onnx/sync_api_win.cc new file mode 100644 index 0000000000000..28f4e16a12a4b --- /dev/null +++ b/onnxruntime/test/onnx/sync_api_win.cc @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "sync_api.h" +#include + +using ::onnxruntime::common::Status; + +Status CreateAndSubmitThreadpoolWork(ONNXRUNTIME_CALLBACK_FUNCTION callback, void* data, PThreadPool pool) { + PTP_WORK work = CreateThreadpoolWork(callback, data, pool); + if (!work) { + return Status(::onnxruntime::common::ONNXRUNTIME, ::onnxruntime::common::FAIL, "create thread pool task failed"); + } + SubmitThreadpoolWork(work); + return Status::OK(); +} + +Status WaitAndCloseEvent(ONNXRUNTIME_EVENT finish_event) { + DWORD dwWaitResult = WaitForSingleObject(finish_event, INFINITE); + (void)CloseHandle(finish_event); + if (dwWaitResult != WAIT_OBJECT_0) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "WaitForSingleObject failed"); + } + return Status::OK(); +} + +Status CreateOnnxRuntimeEvent(ONNXRUNTIME_EVENT* out) { + if (out == nullptr) + return Status(::onnxruntime::common::ONNXRUNTIME, ::onnxruntime::common::INVALID_ARGUMENT, ""); + HANDLE finish_event = CreateEvent( + NULL, // default security attributes + TRUE, // manual-reset event + FALSE, // initial state is nonsignaled + NULL); + if (finish_event == NULL) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "unable to create finish event"); + } + *out = finish_event; + return Status::OK(); +} + +Status OnnxRuntimeSetEventWhenCallbackReturns(ONNXRUNTIME_CALLBACK_INSTANCE pci, ONNXRUNTIME_EVENT finish_event) { + if (finish_event == nullptr) + return Status(::onnxruntime::common::ONNXRUNTIME, ::onnxruntime::common::INVALID_ARGUMENT, ""); + if (pci) + SetEventWhenCallbackReturns(pci, finish_event); + else if (!SetEvent(finish_event)) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "SetEvent failed"); + } + return Status::OK(); +} + +void ONNXRuntimeCloseEvent(ONNXRUNTIME_EVENT finish_event) { + (void)CloseHandle(finish_event); +} \ No newline at end of file diff --git a/onnxruntime/test/onnx/testenv.cc b/onnxruntime/test/onnx/testenv.cc new file mode 100644 index 0000000000000..1826f60931283 --- /dev/null +++ b/onnxruntime/test/onnx/testenv.cc @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "testenv.h" +#include "FixedCountFinishCallback.h" +#include + +using onnxruntime::SessionOptionsWrapper; + +using onnxruntime::Status; +TestEnv::TestEnv(const std::vector& tests1, TestResultStat& stat1, SessionOptionsWrapper& sf1) + : tests(tests1), next_test_to_run(0), stat(stat1), finished(new FixedCountFinishCallback(static_cast(tests1.size()))), sf(sf1) { +} + +TestEnv::~TestEnv() { + delete finished; +} diff --git a/onnxruntime/test/onnx/testenv.h b/onnxruntime/test/onnx/testenv.h new file mode 100644 index 0000000000000..f77ac7badc30a --- /dev/null +++ b/onnxruntime/test/onnx/testenv.h @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include +#include "TestResultStat.h" +#include +#include + +class ITestCase; +class TestCaseResult; +template +class FixedCountFinishCallbackImpl; +using FixedCountFinishCallback = FixedCountFinishCallbackImpl; + +class TestEnv { + public: + std::vector tests; + std::atomic_int next_test_to_run; + TestResultStat& stat; + FixedCountFinishCallback* finished; + const onnxruntime::SessionOptionsWrapper& sf; + TestEnv(const std::vector& tests, TestResultStat& stat1, onnxruntime::SessionOptionsWrapper& sf1); + ~TestEnv(); + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(TestEnv); +}; diff --git a/onnxruntime/test/onnx/vstest_logger.cc b/onnxruntime/test/onnx/vstest_logger.cc new file mode 100644 index 0000000000000..8f8e54e36bf33 --- /dev/null +++ b/onnxruntime/test/onnx/vstest_logger.cc @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "vstest_logger.h" +#include +#include +#include +#include + +#include "date/date.h" + +#include "core/common/logging/capture.h" +#include "core/common/logging/isink.h" + +void VsTestSink::SendImpl(const ::onnxruntime::logging::Timestamp& timestamp, const std::string& logger_id_, const ::onnxruntime::logging::Capture& message) { + // operator for formatting of timestamp in ISO8601 format including microseconds + using date::operator<<; + + // Two options as there may be multiple calls attempting to write to the same sink at once: + // 1) Use mutex to synchronize access to the stream. + // 2) Create the message in an ostringstream and output in one call. + // + // Going with #2 as it should scale better at the cost of creating the message in memory first + // before sending to the stream. + + std::ostringstream msg; + + msg << timestamp << " [" << message.SeverityPrefix() << ":" << message.Category() << ":" << logger_id_ << ", " + << message.Location().ToString() << "] " << message.Message(); + std::string s = msg.str(); + Microsoft::VisualStudio::CppUnitTestFramework::Logger::WriteMessage(s.c_str()); +} diff --git a/onnxruntime/test/onnx/vstest_logger.h b/onnxruntime/test/onnx/vstest_logger.h new file mode 100644 index 0000000000000..569d8ae473554 --- /dev/null +++ b/onnxruntime/test/onnx/vstest_logger.h @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include "core/common/logging/isink.h" +#include "TestCaseResult.h" +#include + +class VsTestSink : public onnxruntime::logging::ISink { + public: + void SendImpl(const onnxruntime::logging::Timestamp& timestamp, const std::string& logger_id_, const onnxruntime::logging::Capture& message) override; +}; + +namespace Microsoft { +namespace VisualStudio { +namespace CppUnitTestFramework { +template <> +std::wstring ToString<>(const EXECUTE_RESULT& q) { + switch (q) { + case EXECUTE_RESULT::SUCCESS: + return L"SUCCESS"; + case EXECUTE_RESULT::UNKNOWN_ERROR: + return L"UNKNOWN_ERROR"; + case EXECUTE_RESULT::WITH_EXCEPTION: + return L"WITH_EXCEPTION"; + case EXECUTE_RESULT::RESULT_DIFFERS: + return L"RESULT_DIFFERS"; + case EXECUTE_RESULT::SHAPE_MISMATCH: + return L"SHAPE_MISMATCH"; + case EXECUTE_RESULT::TYPE_MISMATCH: + return L"TYPE_MISMATCH"; + case EXECUTE_RESULT::NOT_SUPPORT: + return L"NOT_SUPPORT"; + case EXECUTE_RESULT::LOAD_MODEL_FAILED: + return L"LOAD_MODEL_FAILED"; + case EXECUTE_RESULT::INVALID_GRAPH: + return L"INVALID_GRAPH"; + case EXECUTE_RESULT::INVALID_ARGUMENT: + return L"INVALID_ARGUMENT"; + case EXECUTE_RESULT::MODEL_SHAPE_MISMATCH: + return L"MODEL_SHAPE_MISMATCH"; + case EXECUTE_RESULT::MODEL_TYPE_MISMATCH: + return L"MODEL_TYPE_MISMATCH"; + } + return L"UNKNOWN_RETURN_CODE"; +} +} // namespace CppUnitTestFramework +} // namespace VisualStudio +} // namespace Microsoft diff --git a/onnxruntime/test/onnx/vstest_main.cc b/onnxruntime/test/onnx/vstest_main.cc new file mode 100644 index 0000000000000..83584796b35a0 --- /dev/null +++ b/onnxruntime/test/onnx/vstest_main.cc @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include +#include +#include +#include +#include "core/session/onnxruntime_cxx_api.h" +#include "runner.h" +#include "test_allocator.h" +#include +#include +//#include "vstest_logger.h" +using onnxruntime::SessionOptionsWrapper; +using namespace Microsoft::VisualStudio::CppUnitTestFramework; +using std::experimental::filesystem::v1::path; +namespace Microsoft { +namespace VisualStudio { +namespace CppUnitTestFramework { +template <> +std::wstring ToString<>(const EXECUTE_RESULT& q) { + switch (q) { + case EXECUTE_RESULT::SUCCESS: + return L"SUCCESS"; + case EXECUTE_RESULT::UNKNOWN_ERROR: + return L"UNKNOWN_ERROR"; + case EXECUTE_RESULT::WITH_EXCEPTION: + return L"WITH_EXCEPTION"; + case EXECUTE_RESULT::RESULT_DIFFERS: + return L"RESULT_DIFFERS"; + case EXECUTE_RESULT::SHAPE_MISMATCH: + return L"SHAPE_MISMATCH"; + case EXECUTE_RESULT::TYPE_MISMATCH: + return L"TYPE_MISMATCH"; + case EXECUTE_RESULT::NOT_SUPPORT: + return L"NOT_SUPPORT"; + case EXECUTE_RESULT::LOAD_MODEL_FAILED: + return L"LOAD_MODEL_FAILED"; + case EXECUTE_RESULT::INVALID_GRAPH: + return L"INVALID_GRAPH"; + case EXECUTE_RESULT::INVALID_ARGUMENT: + return L"INVALID_ARGUMENT"; + case EXECUTE_RESULT::MODEL_SHAPE_MISMATCH: + return L"MODEL_SHAPE_MISMATCH"; + case EXECUTE_RESULT::MODEL_TYPE_MISMATCH: + return L"MODEL_TYPE_MISMATCH"; + } + return L"UNKNOWN_RETURN_CODE"; +} +} // namespace CppUnitTestFramework +} // namespace VisualStudio +} // namespace Microsoft + +static void run(ONNXEnv* env, SessionOptionsWrapper& sf, const wchar_t* test_folder) { + char buf[1024]; + std::vector res; + { + //Current working directory is the one who contains 'onnx_test_runner_vstest.dll' + //We want debug build and release build share the same test data files, so it should + //be one level up. + std::wstring test_folder_full_path(L"..\\models\\"); + test_folder_full_path.append(test_folder); + path p1(test_folder_full_path); + std::unique_ptr default_allocator(MockedONNXRuntimeAllocator::Create()); + std::vector tests = LoadTests({p1.c_str()}, {}, default_allocator.get()); + Assert::AreEqual((size_t)1, tests.size()); + int p_models = ::onnxruntime::Env::Default().GetNumCpuCores(); + if (tests[0]->GetTestCaseName() == "coreml_FNS-Candy_ImageNet") { + p_models = 2; + } + snprintf(buf, sizeof(buf), "running test %s with %d cores", tests[0]->GetTestCaseName().c_str(), p_models); + Logger::WriteMessage(buf); + ONNXRUNTIME_EVENT finish_event; + ::onnxruntime::Status status = CreateOnnxRuntimeEvent(&finish_event); + Assert::IsTrue(status.IsOK()); + Assert::IsNotNull(finish_event); + RunSingleTestCase(tests[0], sf, p_models, 1, GetDefaultThreadPool(::onnxruntime::Env::Default()), nullptr, [finish_event, &res](std::shared_ptr result, PTP_CALLBACK_INSTANCE pci) { + res = result->GetExcutionResult(); + return OnnxRuntimeSetEventWhenCallbackReturns(pci, finish_event); + }); + status = WaitAndCloseEvent(finish_event); + Assert::IsTrue(status.IsOK()); + Assert::AreEqual(tests[0]->GetDataCount(), res.size()); + delete tests[0]; + } + for (EXECUTE_RESULT r : res) { + Assert::AreEqual(EXECUTE_RESULT::SUCCESS, r); + } +} + +static ONNXEnv* env; + +TEST_MODULE_INITIALIZE(ModuleInitialize) { + ONNXStatusPtr ost = ONNXRuntimeInitialize(ONNXRUNTIME_LOGGING_LEVEL_kWARNING, "Default", &env); + if (ost != nullptr) { + Logger::WriteMessage(ONNXRuntimeGetErrorMessage(ost)); + ReleaseONNXStatus(ost); + Assert::Fail(L"create onnx env failed"); + } +} + +TEST_MODULE_CLEANUP(ModuleCleanup) { + ReleaseONNXEnv(env); +} +// clang-format off +TEST_CLASS(ONNX_TEST){ +public : +#include "vsts/tests.inc" +}; diff --git a/onnxruntime/test/onnxruntime_exec/CmdParser.h b/onnxruntime/test/onnxruntime_exec/CmdParser.h new file mode 100644 index 0000000000000..145f14d979569 --- /dev/null +++ b/onnxruntime/test/onnxruntime_exec/CmdParser.h @@ -0,0 +1,32 @@ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +// + +#pragma once + +#include +#include + +class CmdParser { + public: + CmdParser(int argc, const char* argsv[]) { + if (argc > 2) { + for (int i = 1; i < argc; i += 2) { + cmd_map_.insert({argsv[i], argsv[i + 1]}); + } + } + } + + const std::string* GetCommandArg(const std::string& option) const { + auto value = cmd_map_.find(option); + if (value != cmd_map_.cend()) { + return &value->second; + } + + return nullptr; + } + + private: + std::map cmd_map_; +}; diff --git a/onnxruntime/test/onnxruntime_exec/Model.h b/onnxruntime/test/onnxruntime_exec/Model.h new file mode 100644 index 0000000000000..917a6a2fbc26b --- /dev/null +++ b/onnxruntime/test/onnxruntime_exec/Model.h @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#if !defined(_MSC_VER) +// HRESULT is a 4-byte long on MSVC. We'll just make it a signed int here. +typedef int HRESULT; +// Success codes +#define S_OK ((HRESULT)0L) +#define S_FALSE ((HRESULT)1L) +#endif + +#include "Runtime.h" + +enum class ExecutionStatus { + OK = 0, + MODEL_LOADING_FAILURE = 1, + DATA_LOADING_FAILURE = 2, + PREDICTION_FAILURE = 3, + ONNXRUNTIME_NOT_IMPLEMENTED = 5 +}; + +class Model { + public: + Model(const std::string& modelfile) { + runtime_ = std::make_unique(); + LoadModel(modelfile); + } + + void Execute(const std::string& datafile) { + struct stat s; + if (stat(datafile.c_str(), &s) == 0) { + if (s.st_mode & S_IFDIR) { + exec_status_ = ExecutionStatus::ONNXRUNTIME_NOT_IMPLEMENTED; + return; + } + } + + auto input_reader = LoadTestFile(datafile); + if (!input_reader) { + exec_status_ = ExecutionStatus::DATA_LOADING_FAILURE; + return; + } + + int sample = 0; + while (!input_reader->Eof()) { + std::map> outputs; + + // Perform the test + int hr = runtime_->Run(*input_reader); + if (hr != 0) { + std::cerr << "Failed to execute example" << std::endl; + exec_status_ = ExecutionStatus::PREDICTION_FAILURE; + return; + } + sample++; + } + } + + ExecutionStatus GetStatus() const { + return exec_status_; + } + + std::string GetStatusString() const { + return GetStatusString(exec_status_); + } + + static std::string GetStatusString(ExecutionStatus exec_status) { + switch (exec_status) { + case ExecutionStatus::OK: + return "OK"; + case ExecutionStatus::MODEL_LOADING_FAILURE: + return "MODEL_LOADING_FAILURE"; + case ExecutionStatus::DATA_LOADING_FAILURE: + return "DATA_LOADING_FAILURE"; + case ExecutionStatus::PREDICTION_FAILURE: + return "PREDICTION_FAILURE"; + default: + return "UNKNOWN"; + } + } + + private: + void LoadModel(const std::string& strfilepath) { + std::wstring filepath(strfilepath.begin(), strfilepath.end()); + + auto status = runtime_->LoadModel(filepath); + if (status.IsOK()) { + std::cerr << "'" << strfilepath.c_str() << "' loaded successfully." << std::endl; + } else { + std::cerr << "Loading failed for '" << strfilepath.c_str() << "'" << std::endl; + std::cerr << "-----------------------------" << std::endl; + std::cerr << status.ErrorMessage() << std::endl; + exec_status_ = ExecutionStatus::MODEL_LOADING_FAILURE; + } + } + + std::unique_ptr LoadTestFile(const std::string& filepath) { + std::wstring testfilepath(filepath.begin(), filepath.end()); + auto reader = TestDataReader::OpenReader(testfilepath); + + if (!reader) { + std::cerr << "Unable to load test data file " << filepath << std::endl; + } + + return reader; + } + + std::unique_ptr runtime_; + ExecutionStatus exec_status_ = ExecutionStatus::OK; +}; diff --git a/onnxruntime/test/onnxruntime_exec/README.md b/onnxruntime/test/onnxruntime_exec/README.md new file mode 100644 index 0000000000000..71c9098e2c5e7 --- /dev/null +++ b/onnxruntime/test/onnxruntime_exec/README.md @@ -0,0 +1,62 @@ +# Compile and Build Onnxruntime Executor +``` +>> bld.bat +``` + +Assuming the command was executed from (PATH_2_MONTREAL)\tests\onnxruntime_exec folder. This command will retrieve the latest version of Onnxruntime and build it in debug mode. +Once build is done, open the onnxruntime_exec.sln and build it in debug mode for debugging. + +# Loading Model +``` +>> loturt_exec.exe -m modelfile +``` + +The above command will load the model from modelfile and return the status of model loading [success/fail]. + +# Predicting with Model +``` +>> loturt_exec.exe -m modelfile [-t testfile] +``` + +[-t testfile] is optional. When specified, the loturt_exec.exe will compute prediction/score/probability/etc. on every row of the file using the model and output it to stdout in CSV form. +The format of input testfile is CSV without any header. As of now 11/14, only ints/floats are supported type in CSV. + + +# Model Debugging +* Install python runtime for project Montreal. + + ``` + >> Powershell ./build.ps1 + ``` + + * The first run will take the longest time. This is because we are setting up a python environment for the first time. Subsequent times will be quicker. + * The python environment is located at runtime\Python + * The build script will create and install the winmltools python package. To update this package, you will need to re-run the build script or copy the changed files into python\lib\site-packages\winmltools. + +* Install CoreMLTools for Python 3: + ``` + mkdir coremltools + git clone --recursive https://github.com/apple/coremltools.git + runtime\python\python.exe -m pip install -e coremltools/ + ``` + +* Run the test in local mode (don't run the following script in 'mode' other than local model for your testing.). You can use any python enviroment as long as winmltools are there. + + ``` + >> $(PATH_2_MONTREAL)\runtime\Python\python.exe fn_model_conversion.py -m local -j TAEF_JSONS -s MODEL_SAVE_PATH + ``` + +* Investigate the stdout of the above script to see where the model has problem. The model file is save in MODEL_SAVE_PATH. If its MODEL_LOADING_FAILURE or PREDICTION_FAILURE, run it through loturt_exec.exe. Unless loturt_exec.exe returns some prediction the model has problem. + + +* Please note that **loturt_exec.exe is currently not working on Image models.** + +* Get text representation of coreml model on console + ``` + tests\scrtips\model_viewer_coreml.py model.mlmodel + ``` + +* Get text representation of winml(onnx) model on console. + ``` + tests\scrtips\model_viewer.py model.onnx + ``` diff --git a/onnxruntime/test/onnxruntime_exec/Runtime.h b/onnxruntime/test/onnxruntime_exec/Runtime.h new file mode 100644 index 0000000000000..c83418885b53d --- /dev/null +++ b/onnxruntime/test/onnxruntime_exec/Runtime.h @@ -0,0 +1,378 @@ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +// + +#pragma once + +#include "TestDataReader.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/graph/onnx_protobuf.h" +#include "core/common/logging/sinks/clog_sink.h" +#include "core/common/logging/logging.h" +#include "core/framework/environment.h" +#include "core/framework/data_types.h" +#include "core/session/inference_session.h" +#include "core/providers/cpu/cpu_execution_provider.h" + +#include "test/compare_mlvalue.h" + +#if !defined(_MSC_VER) +#include + +#define ERROR_FILE_NOT_FOUND 2L +#define ERROR_BAD_FORMAT 11L + +#define O_BINARY 0x0000 +#endif + +class WinMLRuntime { + public: + WinMLRuntime() { + using namespace onnxruntime; + using namespace ::onnxruntime::logging; + + static std::unique_ptr<::onnxruntime::Environment> onnxruntime_env = nullptr; + static std::once_flag env_flag; + std::call_once(env_flag, []() { ::onnxruntime::Environment::Create(onnxruntime_env); }); + + static LoggingManager& s_default_logging_manager = DefaultLoggingManager(); + SessionOptions so; + so.session_logid = "WinMLRuntime"; + + inference_session_ = std::make_unique<::onnxruntime::InferenceSession>(so, &s_default_logging_manager); + } + + ::onnxruntime::common::Status LoadModel(const std::wstring& model_path) { + ::onnxruntime::common::Status result = inference_session_->Load(wstr2str(model_path)); + if (result.IsOK()) + result = inference_session_->Initialize(); + + return result; + } + + void FillInBatchSize(std::vector& shape, int input_size, int feature_size) { + if ((input_size % feature_size != 0) && (feature_size != -1)) + throw DataValidationException("Input count is not a multiple of dimension."); + + int batch_size = feature_size == -1 ? 1 : input_size / feature_size; + shape.insert(shape.begin(), batch_size); + } + + ::onnxruntime::MLValue ReadTensorStrings(::onnxruntime::AllocatorPtr alloc, TestDataReader& inputs_reader, + int feature_size, std::vector dims, bool variable_batch_size) { + using namespace onnxruntime; + + auto vec = inputs_reader.GetSampleStrings(feature_size, variable_batch_size); + + std::vector vec2; + for (int i = 0; i < vec.size(); i++) { + std::string str(vec[i].begin(), vec[i].end()); + vec2.push_back(str); + } + + if (variable_batch_size) + FillInBatchSize(dims, gsl::narrow_cast(vec.size()), feature_size); + + TensorShape shape(dims); + auto element_type = DataTypeImpl::GetType(); + + void* buffer = alloc->Alloc(sizeof(std::string) * shape.Size()); + std::unique_ptr p_tensor = std::make_unique(element_type, + shape, + buffer, + alloc->Info(), alloc); + + std::string* p = p_tensor->template MutableData(); + for (int i = 0; i < vec.size(); i++) { + p[i] = std::string(vec[i].begin(), vec[i].end()); + } + + ::onnxruntime::MLValue result; + result.Init(p_tensor.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); + + return result; + } + + template + ::onnxruntime::MLValue ReadTensor(::onnxruntime::AllocatorPtr alloc, TestDataReader& inputs_reader, + int feature_size, std::vector dims, bool variable_batch_size) { + using namespace onnxruntime; + + auto vec = inputs_reader.GetSample(feature_size, variable_batch_size); + + if (variable_batch_size) + FillInBatchSize(dims, gsl::narrow_cast(vec.size()), feature_size); + + ::onnxruntime::TensorShape shape(dims); + auto location = alloc->Info(); + auto element_type = ::onnxruntime::DataTypeImpl::GetType(); + void* buffer = alloc->Alloc(element_type->Size() * shape.Size()); + + if (vec.size() > 0) { + memcpy(buffer, &vec[0], element_type->Size() * shape.Size()); + } + + std::unique_ptr p_tensor = std::make_unique<::onnxruntime::Tensor>(element_type, + shape, + buffer, + location, + alloc); + + ::onnxruntime::MLValue result; + result.Init(p_tensor.release(), + ::onnxruntime::DataTypeImpl::GetType<::onnxruntime::Tensor>(), + ::onnxruntime::DataTypeImpl::GetType<::onnxruntime::Tensor>()->GetDeleteFunc()); + + return result; + } + + template + onnxruntime::MLValue ReadTensorForMapStringToScalar(TestDataReader& inputs_reader) { + auto vec = inputs_reader.GetSample(-1); + + auto data = std::make_unique>(); + for (int i = 0; i < vec.size(); i++) { + // keys start at "1" so convert index to string key based on that + data->insert({std::to_string(i + 1), vec[i]}); + } + + ::onnxruntime::MLValue result; + result.Init(data.release(), + ::onnxruntime::DataTypeImpl::GetType>(), + ::onnxruntime::DataTypeImpl::GetType>()->GetDeleteFunc()); + + return result; + } + + int Run(TestDataReader& inputs_reader) { + using namespace onnxruntime; + int hr = 0; + std::wstring_convert> converter; + + // Create CPU input tensors + ::onnxruntime::NameMLValMap feed; + inputs_reader.BufferNextSample(); + if (inputs_reader.Eof()) + return 0; + + bool variable_batch_size = false; + auto inputs_pairs = inference_session_->GetModelInputs(); + if (!inputs_pairs.first.IsOK()) { + auto error = inputs_pairs.first.ErrorMessage(); + return inputs_pairs.first.Code(); + } + + auto& inputs = *(inputs_pairs.second); + for (size_t index = 0, end = inputs.size(); index < end; ++index) { + MLValue mlvalue; + const onnxruntime::NodeArg& input = *(inputs[index]); + const ONNX_NAMESPACE::TensorShapeProto* input_shape = input.Shape(); + if (input.Name().empty()) + continue; + + auto type = input.Type(); + + std::vector shape; + int feature_size = -1; + + //Previous graph input was variable length that consumed entire input line_ so fetch new input line_. + if (variable_batch_size) + inputs_reader.BufferNextSample(); + + //This graph input may or may not be variable length. + //REVIEW mzs: this can cause issues if we had variable-input followed by fixed input followed by variable-input where + //fixed-input consumed all of the input line_. *Ideally each graph input should be on its own line_*. + variable_batch_size = false; + + //If the shape is not available then read everything into the input tensor. + //feature_size = -1 indicates this condition. + if (input_shape) { + feature_size = 0; + auto dims = input_shape->dim(); + for (auto dim : dims) { + if (dim.has_dim_param()) + variable_batch_size = true; + else { + auto dim_value = dim.dim_value(); + shape.push_back(dim_value); + feature_size = gsl::narrow_cast(feature_size ? feature_size * dim_value : dim_value); + } + } + } + + //REVIEW mzs: Here an assumption is made that all the input columns are for the map. + //The supported map types in onnxruntime seen so far are or . + if (*type == "map(string,tensor(int64))") { + // check if really map(string, int64), which is all we currently support + bool is_map_value_scalar = input.TypeAsProto()->map_type().value_type().tensor_type().shape().dim_size() == 0; + + if (is_map_value_scalar) { + mlvalue = ReadTensorForMapStringToScalar(inputs_reader); + feed.insert(std::make_pair(input.Name(), mlvalue)); + } else { + throw DataValidationException("Unsupported input type: " + std::string(*type)); + } + } else if (*type == "map(string,tensor(float))" || *type == "map(string,tensor(double))") { + // check if really map(string, float) or map(string, double), which is all we currently support + bool is_map_value_scalar = input.TypeAsProto()->map_type().value_type().tensor_type().shape().dim_size() == 0; + + if (is_map_value_scalar) { + mlvalue = ReadTensorForMapStringToScalar(inputs_reader); + feed.insert({input.Name(), mlvalue}); + } else { + throw DataValidationException("Unsupported input type: " + std::string(*type)); + } + } else { + if (*type == "tensor(double)" || *type == "tensor(float)") { + // If double is used in the following statement, following error occurs. + // Tensor type mismatch, caller expects elements to be float while tensor contains double Error from operator + mlvalue = ReadTensor(TestCPUExecutionProvider().GetAllocator(0, ONNXRuntimeMemTypeDefault), inputs_reader, feature_size, shape, variable_batch_size); + } else if (*type == "tensor(int64)") + mlvalue = ReadTensor(TestCPUExecutionProvider().GetAllocator(0, ONNXRuntimeMemTypeDefault), inputs_reader, feature_size, shape, variable_batch_size); + else if (*type == "tensor(string)") + mlvalue = ReadTensorStrings(TestCPUExecutionProvider().GetAllocator(0, ONNXRuntimeMemTypeDefault), inputs_reader, feature_size, shape, variable_batch_size); + else + throw DataValidationException("Unsupported input type: " + std::string(*type)); + + feed.insert(std::make_pair(input.Name(), mlvalue)); + } + } + + // Create output feed + std::vector output_names; + for (auto const& outp : *(inference_session_->GetModelOutputs().second)) { + output_names.push_back(outp->Name()); + } + + std::cout.precision(12); + std::string separator = ""; + // Invoke the net + std::vector<::onnxruntime::MLValue> outputMLValue; + RunOptions run_options; + ::onnxruntime::common::Status result = inference_session_->Run(run_options, feed, output_names, &outputMLValue); + if (result.IsOK()) { + auto outputMeta = inference_session_->GetModelOutputs().second; + // Peel the data off the CPU + for (unsigned int i = 0; i < output_names.size(); i++) { + ::onnxruntime::MLValue& output = outputMLValue[i]; + const ::onnxruntime::Tensor* ctensor = nullptr; + + if (output.IsTensor()) { + ctensor = &output.Get(); + + ONNX_NAMESPACE::ValueInfoProto expected_output_info = (*outputMeta)[i]->ToProto(); + std::pair ret = VerifyValueInfo(expected_output_info, output); + COMPARE_RESULT compare_result = ret.first; + compare_result = ret.first; + if (compare_result != COMPARE_RESULT::SUCCESS) { + switch (compare_result) { + case COMPARE_RESULT::NOT_SUPPORT: + throw std::runtime_error("Unsupported output type in onnxruntime model: " + std::string((*outputMeta)[i]->Name())); + break; + case COMPARE_RESULT::SHAPE_MISMATCH: + throw std::runtime_error("Output shape mismatch in onnxruntime model: " + std::string((*outputMeta)[i]->Name())); + break; + case COMPARE_RESULT::TYPE_MISMATCH: + throw std::runtime_error("Output type mismatch in onnxruntime model: " + std::string((*outputMeta)[i]->Name())); + break; + default: + throw std::runtime_error("Unknown error in onnxruntime model: " + std::string((*outputMeta)[i]->Name())); + } + } + + //REVIEW mzs: Map output types are not tested because I couldn't find any tests for that. + if (ctensor->DataType() == ::onnxruntime::DataTypeImpl::GetType>()) { + const std::map* ci = &output.Get>(); + for (const auto& p : *ci) { + std::cout << separator << p.second; + separator = ","; + } + } else if (ctensor->DataType() == ::onnxruntime::DataTypeImpl::GetType>()) { + const std::map* ci = &output.Get>(); + for (const auto& p : *ci) { + std::cout << separator << p.second; + separator = ","; + } + } else if (ctensor->DataType() == ::onnxruntime::DataTypeImpl::GetType()) { + const float* cdata = ctensor->template Data(); + for (int ci = 0; ci < ctensor->Shape().Size(); ci++) { + std::cout << separator << cdata[ci]; + separator = ","; + } + } else if (ctensor->DataType() == ::onnxruntime::DataTypeImpl::GetType()) { + const int64_t* cdata = ctensor->template Data(); + for (int ci = 0; ci < ctensor->Shape().Size(); ci++) { + std::cout << separator << cdata[ci]; + separator = ","; + } + } else if (ctensor->DataType() == ::onnxruntime::DataTypeImpl::GetType()) { + const std::string* cdata = ctensor->template Data(); + for (int ci = 0; ci < ctensor->Shape().Size(); ci++) { + std::cout << separator << cdata[ci]; + separator = ","; + } + } else { + throw DataValidationException("Unsupported output type in onnxruntime model: " + std::string((*outputMeta)[i]->Name())); + } + } else if (output.Type() == ::onnxruntime::DataTypeImpl::GetType<::onnxruntime::VectorMapStringToFloat>()) { + auto& cdata = output.Get<::onnxruntime::VectorMapStringToFloat>(); + for (int ci = 0; ci < cdata.size(); ci++) { + for (const auto& p : cdata[ci]) { + std::cout << separator << p.second; + separator = ","; + } + } + } else if (output.Type() == ::onnxruntime::DataTypeImpl::GetType<::onnxruntime::VectorMapInt64ToFloat>()) { + auto& cdata = output.Get<::onnxruntime::VectorMapInt64ToFloat>(); + for (int ci = 0; ci < cdata.size(); ci++) { + for (const auto& p : cdata[ci]) { + std::cout << separator << p.second; + separator = ","; + } + } + } + } + + std::cout << std::endl; + } else { + std::cerr << result.ErrorMessage() << std::endl; + hr = result.Code(); + } + + return hr; + } + + private: + std::unique_ptr<::onnxruntime::InferenceSession> inference_session_; + + static ::onnxruntime::logging::LoggingManager& DefaultLoggingManager() { + using namespace onnxruntime; + std::string default_logger_id{"Default"}; + + static logging::LoggingManager default_logging_manager{ + std::unique_ptr{new ::onnxruntime::logging::CLogSink{}}, + logging::Severity::kWARNING, false, + logging::LoggingManager::InstanceType::Default, + &default_logger_id}; + + return default_logging_manager; + } + + static ::onnxruntime::IExecutionProvider& TestCPUExecutionProvider() { + static ::onnxruntime::CPUExecutionProviderInfo info; + static ::onnxruntime::CPUExecutionProvider cpu_provider(info); + return cpu_provider; + } +}; diff --git a/onnxruntime/test/onnxruntime_exec/TestDataReader.h b/onnxruntime/test/onnxruntime_exec/TestDataReader.h new file mode 100644 index 0000000000000..cfc4719426146 --- /dev/null +++ b/onnxruntime/test/onnxruntime_exec/TestDataReader.h @@ -0,0 +1,148 @@ +/* +* Copyright (c) Microsoft Corporation. All rights reserved. +* Licensed under the MIT License. +*/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +std::string wstr2str(const std::wstring& wstr) { + std::string str = std::wstring_convert>().to_bytes(wstr); + return str; +} + +class DataValidationException : public std::exception { + public: + DataValidationException(const std::string& str) : str_(str) { + } + const char* what() const noexcept override { + return str_.c_str(); + } + + private: + std::string str_; +}; + +class TestDataReader { + public: + static std::unique_ptr OpenReader(std::wstring data_file); + + void BufferNextSample(); + bool Eof(); + + template + std::vector GetSample(int sample_count, bool variable_batch_size = false); + + std::vector GetSampleStrings(int sample_count, bool variable_batch_size = false); + + private: + std::wstring line_; + std::wifstream reader_stream_; + std::unique_ptr row_stream_; +}; + +bool TestDataReader::Eof() { + return reader_stream_.eof(); +} + +void TestDataReader::BufferNextSample() { + if (Eof()) + return; + + std::getline(reader_stream_, line_); + + if (Eof()) + return; + + row_stream_ = std::make_unique(line_); + std::wstring feature; + std::getline(*row_stream_, feature, L','); //Skip the Label which is actually. +} + +template +std::vector TestDataReader::GetSample(int sample_count, bool variable_batch_size) { + assert(sample_count == -1 || sample_count > 0); + + std::wstring feature; + std::vector result; + + int s = 0; + while ((s++ < sample_count || sample_count == -1 || variable_batch_size) && + std::getline(*row_stream_, feature, L',')) // -1 means read all data in the sample + { + T feature_value; + std::wstringstream feature_convert(feature); + feature_convert >> feature_value; + if (feature_convert.fail()) { + feature_value = (T)NAN; + } + + result.push_back(feature_value); + } + + if (line_.length() > 0 && line_.back() == L',') + result.push_back((T)NAN); + + if (sample_count != -1 && !variable_batch_size) { + //Remove the last NAN inserted if it is not part of this feature. + if (result.size() == sample_count + 1) + result.pop_back(); + + if (result.size() != sample_count) + throw DataValidationException("Not enough features in sample."); + } + + if (variable_batch_size && (result.size() % sample_count != 0) && (sample_count != -1)) + throw DataValidationException("Input count is not a multiple of dimension."); + + return result; +} + +std::vector TestDataReader::GetSampleStrings(int sample_count, bool variable_batch_size) { + std::wstring feature; + std::vector result; + + int s = 0; + while (s < sample_count || sample_count == -1 || variable_batch_size) // -1 means read all data in the sample + { + if (std::getline(*row_stream_, feature, L',')) + result.push_back(feature); + else { + if (sample_count == -1 || variable_batch_size) + break; + + throw DataValidationException("Not enough features in sample."); + } + s++; + } + + if (line_.length() > 0 && line_.back() == L',') + result.push_back(L""); + + if (variable_batch_size && (result.size() % sample_count != 0) && (sample_count != -1)) + throw DataValidationException("Input count is not a multiple of dimension."); + + return result; +} + +std::unique_ptr TestDataReader::OpenReader(std::wstring dataFile) { + auto reader = std::make_unique(); + + reader->reader_stream_.open(wstr2str(dataFile)); + if (!reader->reader_stream_) { + reader = nullptr; + } + + return reader; +} diff --git a/onnxruntime/test/onnxruntime_exec/main.cc b/onnxruntime/test/onnxruntime_exec/main.cc new file mode 100644 index 0000000000000..c3c9ba2abf3e4 --- /dev/null +++ b/onnxruntime/test/onnxruntime_exec/main.cc @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include + +#include "CmdParser.h" +#include "Model.h" +#include "TestDataReader.h" + +void print_cmd_option() { + std::cerr << "onnxruntime_exec.exe -m model_file [-t testdata]" << std::endl; +} + +int main(int argc, const char* args[]) { + try { + CmdParser parser(argc, args); + const std::string* modelfile = parser.GetCommandArg("-m"); + if (!modelfile) { + std::cerr << "WinML model file is required." << std::endl; + print_cmd_option(); + return -1; + } + + Model model(*modelfile); + + if (model.GetStatus() == ExecutionStatus::OK) { + std::cerr << "Done loading model: " << modelfile->c_str() << std::endl; + const std::string* testfile = parser.GetCommandArg("-t"); + if (testfile) { + model.Execute(*testfile); + } + } + + std::cerr << "Execution Status: " << model.GetStatusString() << std::endl; + } catch (const DataValidationException& e) { + std::cerr << "Execution Status: " << Model::GetStatusString(ExecutionStatus::DATA_LOADING_FAILURE) << std::endl; + std::cout << "Exception msg: " << e.what() << std::endl; + } catch (const std::exception& e) { + std::cerr << "Execution Status: " << Model::GetStatusString(ExecutionStatus::PREDICTION_FAILURE) << std::endl; + std::cout << "Exception msg: " << e.what() << std::endl; + } +} diff --git a/onnxruntime/test/perftest/README.md b/onnxruntime/test/perftest/README.md new file mode 100644 index 0000000000000..07760877e05ea --- /dev/null +++ b/onnxruntime/test/perftest/README.md @@ -0,0 +1,25 @@ +# ONNXRuntime Performance Test + +onnxruntime_perf_test [options...] model_path result_file +Options: + -m [test_mode]: Specifies the test mode. Value coulde be 'duration' or 'times'. + Provide 'duration' to run the test for a fix duration, and 'times' to repeated for a certain times. Default:'duration'. + -e [cpu|cuda|mkldnn]: Specifies the provider 'cpu','cuda','mkldnn'. Default:'cpu'. + -r [repeated_times]: Specifies the repeated times if running in 'times' test mode.Default:1000. + -t [seconds_to_run]: Specifies the seconds to run for 'duration' mode. Default:600. + -p [profile_file]: Specifies the profile name to enable profiling and dump the profile data to the file. + -s: Show statistics result, like P75, P90. + -v: Show verbose information. + -x: Use parallel executor, default (without -x): sequential executor. + -h: help + +Model path and input data dependency: + Performance test uses the same input structure as onnx_test_runner. It requrires the directory trees as below: + + --ModelName + --test_data_set_0 + --input0.pb + --test_data_set_2 + --input0.pb + --model.onnx + The path of model.onnx needs to be provided as argument. diff --git a/onnxruntime/test/perftest/ReadMe.txt b/onnxruntime/test/perftest/ReadMe.txt new file mode 100644 index 0000000000000..4142beefbd034 --- /dev/null +++ b/onnxruntime/test/perftest/ReadMe.txt @@ -0,0 +1,37 @@ +onnxruntime_perf_test [options...] + +Options: + -m [test_mode]: Specifies the test mode. Value coulde be 'duration' or 'times'. Provide 'duration' to run the test for a fix duration, and 'times' to repeated for a certain times. Default:'duration'. + -e [cpu|cuda]: Specifies the provider 'cpu' or 'cuda'. Default:'cpu'.\n" + -r [repeated_times]: Specifies the repeated times if running in 'times' test mode.Default:1000. + -t [seconds_to_run]: Specifies the seconds to run for 'duration' mode. Default:600. + -p [profile_file]: Specifies the profile name to enable profiling and dump the profile data to the file. + -v: Show verbose information + -h: help + +Model path and input data dependency: + Performance test uses the same input structure as onnx_test_runner. It requrires the direcotry trees as below: + + --ModelName + --test_data_set_0 + --input0.pb + --test_data_set_2 + --input0.pb + --model.onnx + The path of model.onnx needs to be provided as argument. + +How to download sample test data from VSTS drop: + 1) Download drop app from https://aiinfra.artifacts.visualstudio.com/_apis/drop/client/exe + Unzip the downloaded file and add lib/net45 dir to your PATH + 2) Download the test data by using this command: + drop get -a -s https://aiinfra.artifacts.visualstudio.com/DefaultCollection -n Lotus/testdata/onnx/model/16 -d C:\testdata + You may change C:\testdata to any directory in your disk. + Full document: https://www.1eswiki.com/wiki/VSTS_Drop + + +How to run performance tests for batch of models: + 1) Download the driver by using this command: + drop get -a -s https://aiinfra.artifacts.visualstudio.com/DefaultCollection -n Lotus/test/perfdriver/$(perfTestDriverVersion) -d C:\perfdriver + You may change C:\perfdriver to any directory in your disk. + Currently, the $(perfTestDriverVersion) is 6 + 2) Run the PerfTestDriver.py under python environment with proper arguments. \ No newline at end of file diff --git a/onnxruntime/test/perftest/TestCase.cc b/onnxruntime/test/perftest/TestCase.cc new file mode 100644 index 0000000000000..b47016d4978b7 --- /dev/null +++ b/onnxruntime/test/perftest/TestCase.cc @@ -0,0 +1,549 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "TestCase.h" +#include +#include + +#include "core/platform/env.h" +#include "core/framework/tensorprotoutils.h" + + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wignored-qualifiers" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#else +#pragma warning(push) +#pragma warning(disable : 4018) /*'expression' : signed/unsigned mismatch */ +#pragma warning(disable : 4065) /*switch statement contains 'default' but no 'case' labels*/ +#pragma warning(disable : 4100) +#pragma warning(disable : 4146) /*unary minus operator applied to unsigned type, result still unsigned*/ +#pragma warning(disable : 4244) /*'conversion' conversion from 'type1' to 'type2', possible loss of data*/ +#pragma warning(disable : 4251) /*'identifier' : class 'type' needs to have dll-interface to be used by clients of class 'type2'*/ +#pragma warning(disable : 4267) /*'var' : conversion from 'size_t' to 'type', possible loss of data*/ +#pragma warning(disable : 4305) /*'identifier' : truncation from 'type1' to 'type2'*/ +#pragma warning(disable : 4307) /*'operator' : integral constant overflow*/ +#pragma warning(disable : 4309) /*'conversion' : truncation of constant value*/ +#pragma warning(disable : 4334) /*'operator' : result of 32-bit shift implicitly converted to 64 bits (was 64-bit shift intended?)*/ +#pragma warning(disable : 4355) /*'this' : used in base member initializer list*/ +#pragma warning(disable : 4505) /*unreferenced local function has been removed*/ +#pragma warning(disable : 4506) /*no definition for inline function 'function'*/ +#pragma warning(disable : 4800) /*'type' : forcing value to bool 'true' or 'false' (performance warning)*/ +#pragma warning(disable : 4996) /*The compiler encountered a deprecated declaration.*/ +#endif +#include +#include +#include "tml.pb.h" +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#else +#pragma warning(pop) +#endif + +using namespace std::experimental::filesystem::v1; + +using namespace onnxruntime; +using namespace onnxruntime::common; + +namespace { +template +Status ConvertVector(const InputType& data, OutputType** vec) { + //void* p = allocator->Alloc(sizeof(OutputType)); + //if (p == nullptr) + // return Status(ONNXRUNTIME, FAIL, "out of memory"); + //OutputType* v = new (p) OutputType(); + //TODO: non-tensor type has no deleter inside it. So, cannot use allocator + OutputType* v = new OutputType(); + for (const auto& i : data) { + typename OutputType::value_type new_value; + for (const auto& j : i.v()) { + new_value[j.first] = j.second; + } + v->push_back(new_value); + } + *vec = v; + return Status::OK(); +} + +template +Status Convert(const InputType& tensor_proto, OutputType** p_tensor); + +template <> +Status Convert(const google::protobuf::RepeatedPtrField& data, VectorMapInt64ToFloat** vec) { + return ConvertVector, VectorMapInt64ToFloat>(data, vec); +} + +template <> +Status Convert(const google::protobuf::RepeatedPtrField& data, VectorMapStringToFloat** vec) { + return ConvertVector, VectorMapStringToFloat>(data, vec); +} + +template +void ConvertMap(const InputType& data, OutputType** out) { + OutputType* ret = new OutputType(); + for (const auto& pv : data) { + (*ret)[pv.first] = pv.second; + } + *out = ret; +} + +template <> +Status Convert(const google::protobuf::Map& data, MapStringToString** out) { + ConvertMap(data, out); + return Status::OK(); +} + +template <> +Status Convert(const google::protobuf::Map& data, MapStringToInt64** out) { + ConvertMap(data, out); + return Status::OK(); +} + +template <> +Status Convert(const google::protobuf::Map& data, MapStringToFloat** out) { + ConvertMap(data, out); + return Status::OK(); +} + +template <> +Status Convert(const google::protobuf::Map& data, MapStringToDouble** out) { + ConvertMap(data, out); + return Status::OK(); +} + +template <> +Status Convert(const google::protobuf::Map& data, MapInt64ToString** out) { + ConvertMap(data, out); + return Status::OK(); +} + +template <> +Status Convert(const google::protobuf::Map& data, MapInt64ToInt64** out) { + ConvertMap(data, out); + return Status::OK(); +} + +template <> +Status Convert(const google::protobuf::Map& data, MapInt64ToFloat** out) { + ConvertMap(data, out); + return Status::OK(); +} + +template <> +Status Convert(const google::protobuf::Map& data, MapInt64ToDouble** out) { + ConvertMap(data, out); + return Status::OK(); +} +template +Status RichTypeProtoToMLValue(const InputType& input, MLValue& value) { + OutputType* tensor = nullptr; + Status st = Convert(input, &tensor); + if (!st.IsOK()) return st; + value.Init(tensor, + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); + return Status::OK(); +} + +static int ExtractFileNo(const std::string& name) { + size_t p1 = name.rfind('.'); + size_t p2 = name.rfind('_', p1); + ++p2; + std::string number_str = name.substr(p2, p1 - p2); + const char* start = number_str.c_str(); + const char* end = number_str.c_str(); + long ret = strtol(start, const_cast(&end), 10); + if (end == start) { + ONNXRUNTIME_THROW("parse file name failed"); + } + return static_cast(ret); +} + +static Status SortTensorFileNames(std::vector& input_pb_files) { + if (input_pb_files.size() <= 1) return Status::OK(); + std::sort(input_pb_files.begin(), input_pb_files.end(), [](const path& left, const path& right) -> bool { + std::string leftname = left.filename().string(); + std::string rightname = right.filename().string(); + int left1 = ExtractFileNo(leftname); + int right1 = ExtractFileNo(rightname); + return left1 < right1; + }); + for (size_t i = 0; i != input_pb_files.size(); ++i) { + int fileno = ExtractFileNo(input_pb_files[i].filename().string()); + if (fileno != i) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "illegal input file name:", input_pb_files[i].filename().string()); + } + } + return Status::OK(); +} + +//Doesn't support file size >2 GB +Status LoopDataFile(int test_data_pb_fd, AllocatorPtr allocator, + const std::vector value_info, onnxruntime::NameMLValMap& name_data_map, std::ostringstream& oss) { + google::protobuf::io::FileInputStream f(test_data_pb_fd); + f.SetCloseOnDelete(true); + google::protobuf::io::CodedInputStream coded_input(&f); + bool clean_eof = false; + Status st; + int item_id = 1; + for (proto::TraditionalMLData data; google::protobuf::util::ParseDelimitedFromCodedStream(&data, &coded_input, &clean_eof); ++item_id, data.Clear()) { + MLValue value; + switch (data.values_case()) { + case proto::TraditionalMLData::kVectorMapStringToFloat: + st = RichTypeProtoToMLValue(data.vector_map_string_to_float().v(), value); + break; + case proto::TraditionalMLData::kVectorMapInt64ToFloat: + st = RichTypeProtoToMLValue(data.vector_map_int64_to_float().v(), value); + break; + case proto::TraditionalMLData::kMapStringToString: + st = RichTypeProtoToMLValue(data.map_string_to_string().v(), value); + break; + case proto::TraditionalMLData::kMapStringToInt64: + st = RichTypeProtoToMLValue(data.map_string_to_int64().v(), value); + break; + case proto::TraditionalMLData::kMapStringToFloat: + st = RichTypeProtoToMLValue(data.map_string_to_float().v(), value); + break; + case proto::TraditionalMLData::kMapStringToDouble: + st = RichTypeProtoToMLValue(data.map_string_to_double().v(), value); + break; + case proto::TraditionalMLData::kMapInt64ToString: + st = RichTypeProtoToMLValue(data.map_int64_to_string().v(), value); + break; + case proto::TraditionalMLData::kMapInt64ToInt64: + st = RichTypeProtoToMLValue(data.map_int64_to_int64().v(), value); + break; + case proto::TraditionalMLData::kMapInt64ToFloat: + st = RichTypeProtoToMLValue(data.map_int64_to_float().v(), value); + break; + case proto::TraditionalMLData::kMapInt64ToDouble: + st = RichTypeProtoToMLValue(data.map_int64_to_double().v(), value); + break; + case proto::TraditionalMLData::kTensor: + st = utils::TensorProtoToMLValue(data.tensor(), allocator, nullptr, 0, value); + break; + default: + st = Status(ONNXRUNTIME, NOT_IMPLEMENTED, "unknown data type inside TraditionalMLData"); + } + if (!st.IsOK()) break; + if (!data.debug_info().empty()) { + oss << ":" << data.debug_info(); + } + std::string value_name = data.name(); + if (value_name.empty()) + value_name = value_info[name_data_map.size()].name(); + + auto pv = name_data_map.insert(std::make_pair(value_name, value)); + if (!pv.second) { + st = Status(ONNXRUNTIME, FAIL, "duplicated test data name"); + break; + } + } + if (!st.IsOK()) return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "load the ", item_id, "-th item failed,", st.ErrorMessage()); + if (!clean_eof) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "parse input file failed, has extra unparsed data"); + } + return Status::OK(); +} + +Status loadModel(std::istream& model_istream, ONNX_NAMESPACE::ModelProto* p_model_proto) { + if (!model_istream.good()) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid istream object."); + } + if (!p_model_proto) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Null model_proto ptr."); + } + const bool result = p_model_proto->ParseFromIstream(&model_istream); + if (!result) { + return Status(ONNXRUNTIME, INVALID_PROTOBUF, "Failed to load model because protobuf parsing failed."); + } + return Status::OK(); +} + +Status loadModelFile(const std::string& model_url, ONNX_NAMESPACE::ModelProto* model_pb) { + std::ifstream input(model_url, std::ios::in | std::ios::binary); + if (!input) { + std::ostringstream oss; + oss << "open file " << model_url << " failed"; + return Status(ONNXRUNTIME, NO_SUCHFILE, oss.str()); + } + return loadModel(input, model_pb); +} + +template +static void RepeatedPtrFieldToVector(const ::google::protobuf::RepeatedPtrField& input_value_info, std::vector& out) { + for (int i = 0; i != input_value_info.size(); ++i) { + out.push_back(input_value_info[i]); + } +} +} // namespace + +/** + * test_case_dir must have contents of: + * model.onnx + * ???/input_??.pb + * ???/output_??.pb + * ???/input_??.pb + * ???/output_??.pb + */ +class OnnxTestCase : public ITestCase { + private: + std::string test_case_name_; + std::experimental::filesystem::v1::path model_url_; + AllocatorPtr allocator_; + std::vector debuginfo_strings; + std::mutex m_; + std::vector input_value_info_; + std::vector output_value_info_; + + Status FromPbFiles(const std::vector& files, std::vector& output_values); + std::vector test_data_dirs_; + + std::string GetDatasetDebugInfoString(size_t dataset_id) override { + std::lock_guard l(m_); + if (dataset_id < debuginfo_strings.size()) + return debuginfo_strings[dataset_id]; + return test_data_dirs_.at(dataset_id).string(); + } + //If we cannot get input name from input_pbs, we'll use names like "data_0","data_1",... It's dirty hack + // for https://github.com/onnx/onnx/issues/679 + ::onnxruntime::common::Status ConvertTestData(const std::vector& test_data_pbs, + const std::vector value_info, onnxruntime::NameMLValMap& out); + std::string node_name_; + std::once_flag model_parsed_; + + Status ParseModel(); + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(OnnxTestCase); + + public: + OnnxTestCase(const AllocatorPtr&, const std::string& test_case_name); + explicit OnnxTestCase(const std::string& test_case_name) : test_case_name_(test_case_name) {} + void SetAllocator(const AllocatorPtr&) override; + + const ONNX_NAMESPACE::ValueInfoProto& GetOutputInfoFromModel(size_t i) const override { + return output_value_info_[i]; + } + size_t GetDataCount() const override { + return test_data_dirs_.size(); + } + Status GetNodeName(std::string* out) override { + Status st = ParseModel(); + if (st.IsOK()) *out = node_name_; + return st; + } + Status SetModelPath(const std::experimental::filesystem::v1::path& path) override; + + const std::experimental::filesystem::v1::path& GetModelUrl() const override { + return model_url_; + } + const std::string& GetTestCaseName() const override { + return test_case_name_; + } + ::onnxruntime::common::Status LoadTestData(size_t id, onnxruntime::NameMLValMap& name_data_map, bool is_input) override; +}; + +ITestCase* CreateOnnxTestCase(const AllocatorPtr& ptr, const std::string& test_case_name) { + return new OnnxTestCase(ptr, test_case_name); +} +ITestCase* CreateOnnxTestCase(const std::string& test_case_name) { + return new OnnxTestCase(test_case_name); +} + + + +Status OnnxTestCase::ParseModel() { + Status st = Status::OK(); + std::call_once(model_parsed_, [this, &st]() { + //parse model + ONNX_NAMESPACE::ModelProto model_pb; + st = loadModelFile(model_url_.string(), &model_pb); + if (!st.IsOK()) return; + const ONNX_NAMESPACE::GraphProto& graph = model_pb.graph(); + if (graph.node().size() == 1) { + node_name_ = graph.node()[0].op_type(); + } + RepeatedPtrFieldToVector(graph.input(), input_value_info_); + RepeatedPtrFieldToVector(graph.output(), output_value_info_); + st = Status::OK(); + }); + return st; +} +Status OnnxTestCase::SetModelPath(const path& m) { + model_url_ = m; + path test_case_dir = m.parent_path(); + for (directory_iterator test_data_set(test_case_dir), end2; test_data_set != end2; ++test_data_set) { + if (!is_directory(*test_data_set)) { + continue; + } + test_data_dirs_.push_back(test_data_set->path()); + debuginfo_strings.push_back(test_data_set->path().string()); + } + return Status::OK(); +} + +//load tensors from disk +static Status LoadTensors(const std::vector& pb_files, std::vector* input_pbs) { + for (size_t i = 0; i != pb_files.size(); ++i) { + ONNX_NAMESPACE::TensorProto tensor; + std::ifstream input(pb_files.at(i), std::ios::in | std::ios::binary); + if (!input) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "open file '", pb_files.at(i), "' failed"); + } + if (!tensor.ParseFromIstream(&input)) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "parse file '", pb_files.at(i), "' failed"); + } + input_pbs->emplace_back(tensor); + } + return Status::OK(); +} + +Status OnnxTestCase::LoadTestData(size_t id, onnxruntime::NameMLValMap& name_data_map, bool is_input) { + if (id >= test_data_dirs_.size()) + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "out of bound"); + + Status st = ParseModel(); + if (!st.IsOK()) + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, MODEL_LOADED, "parse model failed:", st.ErrorMessage()); + + path test_data_pb = test_data_dirs_[id] / (is_input ? "inputs.pb" : "outputs.pb"); + int test_data_pb_fd; + st = Env::Default().FileOpenRd(test_data_pb, test_data_pb_fd); + if (st.IsOK()) { //has an all-in-one input file + std::ostringstream oss; + { + std::lock_guard l(m_); + oss << debuginfo_strings[id]; + } + st = LoopDataFile(test_data_pb_fd, allocator_, is_input ? input_value_info_ : output_value_info_, name_data_map, oss); + { + std::lock_guard l(m_); + debuginfo_strings[id] = oss.str(); + } + return st; + } + + std::vector test_data_pb_files; + const path pb(".pb"); + + for (directory_iterator pb_file(test_data_dirs_[id]), end3; pb_file != end3; ++pb_file) { + path f = *pb_file; + if (!is_regular_file(f)) continue; + if (f.extension() != pb) continue; + std::string filename = f.filename().string(); + std::string file_prefix = is_input ? "input_" : "output_"; + if (!filename.compare(0, file_prefix.length(), file_prefix.c_str())) { + test_data_pb_files.push_back(f); + } + } + ONNXRUNTIME_RETURN_IF_ERROR(SortTensorFileNames(test_data_pb_files)); + + std::vector test_data_pbs; + ONNXRUNTIME_RETURN_IF_ERROR(LoadTensors(test_data_pb_files, &test_data_pbs)); + ONNXRUNTIME_RETURN_IF_ERROR(ConvertTestData(test_data_pbs, is_input ? input_value_info_ : output_value_info_, name_data_map)); + return Status::OK(); +} + +Status OnnxTestCase::FromPbFiles(const std::vector& files, std::vector& output_values) { + for (const path& f : files) { + if (!f.has_extension()) return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, NOT_IMPLEMENTED, "unknown file type, path = ", f); + std::string s = f.extension().string(); + if (s != ".pb") + continue; + ONNX_NAMESPACE::TensorProto tensor; + { + std::ifstream input(f, std::ios::in | std::ios::binary); + if (!input) { + return Status(ONNXRUNTIME, FAIL, "open file failed"); + } + if (!tensor.ParseFromIstream(&input)) { + return Status(ONNXRUNTIME, FAIL, "parse file failed"); + } + } + MLValue value; + ONNXRUNTIME_RETURN_IF_ERROR(onnxruntime::utils::TensorProtoToMLValue(tensor, allocator_, nullptr, 0, value)); + output_values.emplace_back(value); + } + return Status::OK(); +} + +Status OnnxTestCase::ConvertTestData(const std::vector& test_data_pbs, + const std::vector value_info, onnxruntime::NameMLValMap& out) { + int len = static_cast(value_info.size()); + bool has_valid_names = true; + //"0","1",... + bool use_number_names = true; + //"data_0","data_1",... + bool use_data_number_names = true; + //"gpu_0/data_0","gpu_0/data_1",... + bool use_gpu_data_number_names = true; + + std::vector var_names(test_data_pbs.size()); + for (size_t input_index = 0; input_index != test_data_pbs.size(); ++input_index) { + std::string name = test_data_pbs[input_index].name(); + if (name.empty()) { + has_valid_names = false; + break; + } + var_names[input_index] = name; + } + if (!has_valid_names) { + if (len == test_data_pbs.size()) { + for (int i = 0; i != len; ++i) { + std::string vname = value_info[i].name(); + var_names[i] = vname; + } + } else { + char buf[64]; + char buf2[64]; + char buf3[64]; + for (int i = 0; i != test_data_pbs.size(); ++i) { + snprintf(buf, sizeof(buf), "%d", i); + snprintf(buf2, sizeof(buf2), "data_%d", i); + snprintf(buf3, sizeof(buf3), "gpu_0/data_%d", i); + if (use_number_names && std::find_if(value_info.begin(), value_info.end(), [buf](const onnx::ValueInfoProto& info) { + return info.name() == buf; + }) == value_info.end()) use_number_names = false; + if (use_data_number_names && std::find_if(value_info.begin(), value_info.end(), [buf2](const onnx::ValueInfoProto& info) { + return info.name() == buf2; + }) == value_info.end()) use_data_number_names = false; + if (use_data_number_names && std::find_if(value_info.begin(), value_info.end(), [buf3](const onnx::ValueInfoProto& info) { + return info.name() == buf3; + }) == value_info.end()) use_gpu_data_number_names = false; + } + } + for (size_t input_index = 0; input_index != test_data_pbs.size(); ++input_index) { + std::string name = var_names[input_index]; + char buf[64]; + if (name.empty()) { + if (use_number_names) { + snprintf(buf, sizeof(buf), "%d", static_cast(input_index)); + var_names[input_index] = buf; + } else if (use_data_number_names) { + snprintf(buf, sizeof(buf), "data_%d", static_cast(input_index)); + var_names[input_index] = buf; + } else if (use_gpu_data_number_names) { + snprintf(buf, sizeof(buf), "gpu_0/data_%d", static_cast(input_index)); + var_names[input_index] = buf; + } else + return Status(ONNXRUNTIME, NOT_IMPLEMENTED, "cannot guess a valid input name"); + } + } + } + for (size_t input_index = 0; input_index != test_data_pbs.size(); ++input_index) { + std::string name = var_names[input_index]; + const onnx::TensorProto& input = test_data_pbs[input_index]; + MLValue v1; + ONNXRUNTIME_RETURN_IF_ERROR(utils::TensorProtoToMLValue(input, allocator_, nullptr, 0, v1)); + out.insert(std::make_pair(name, v1)); + } + return Status::OK(); +} + +OnnxTestCase::OnnxTestCase(const AllocatorPtr& allocator, const std::string& test_case_name) : test_case_name_(test_case_name) { + SetAllocator(allocator); +} + +void OnnxTestCase::SetAllocator(const AllocatorPtr& allocator) { + allocator_ = allocator; +} diff --git a/onnxruntime/test/perftest/TestCase.h b/onnxruntime/test/perftest/TestCase.h new file mode 100644 index 0000000000000..b2aaeaaa64f85 --- /dev/null +++ b/onnxruntime/test/perftest/TestCase.h @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include +#include +#include +#include +#ifdef _MSC_VER +#include +#endif + +namespace ONNX_NAMESPACE { +class ValueInfoProto; +} + +//One test case is for one model file +//One test case can contain multiple test data(input/output pairs) +class ITestCase { + public: + //must be called before calling the other functions + virtual ::onnxruntime::common::Status SetModelPath(const std::experimental::filesystem::v1::path& path) = 0; + virtual ::onnxruntime::common::Status LoadTestData(size_t id, onnxruntime::NameMLValMap& name_data_map, bool is_input) = 0; + virtual const std::experimental::filesystem::v1::path& GetModelUrl() const = 0; + virtual const std::string& GetTestCaseName() const = 0; + virtual void SetAllocator(const ::onnxruntime::AllocatorPtr&) = 0; + //a string to help identify the dataset + virtual std::string GetDatasetDebugInfoString(size_t dataset_id) = 0; + virtual ::onnxruntime::common::Status GetNodeName(std::string* out) = 0; + //The number of input/output pairs + virtual size_t GetDataCount() const = 0; + virtual const ONNX_NAMESPACE::ValueInfoProto& GetOutputInfoFromModel(size_t i) const = 0; + virtual ~ITestCase() {} +}; + +ITestCase* CreateOnnxTestCase(const ::onnxruntime::AllocatorPtr&, const std::string& test_case_name); +ITestCase* CreateOnnxTestCase(const std::string& test_case_name); diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc new file mode 100644 index 0000000000000..b61cb6e12dd90 --- /dev/null +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "command_args_parser.h" + +#include +#include + +// Windows Specific +#ifdef _WIN32 +#include "getopt.h" +#include "windows.h" +#else +#include +#endif + +#include + +#include "test_configuration.h" + +namespace onnxruntime { +namespace perftest { + +/*static*/ void CommandLineParser::ShowUsage() { + printf( + "perf_test [options...] model_path result_file\n" + "Options:\n" + "\t-m [test_mode]: Specifies the test mode. Value coulde be 'duration' or 'times'.\n" + "\t\tProvide 'duration' to run the test for a fix duration, and 'times' to repeated for a certain times. Default:'duration'.\n" + "\t-e [cpu|cuda|mkldnn]: Specifies the provider 'cpu','cuda','mkldnn'. Default:'cpu'.\n" + "\t-r [repeated_times]: Specifies the repeated times if running in 'times' test mode.Default:1000.\n" + "\t-t [seconds_to_run]: Specifies the seconds to run for 'duration' mode. Default:600.\n" + "\t-p [profile_file]: Specifies the profile name to enable profiling and dump the profile data to the file.\n" + "\t-s: Show statistics result, like P75, P90.\n" + "\t-v: Show verbose information.\n" + "\t-x: Use parallel executor, default (without -x): sequential executor.\n" + "\t-h: help\n"); +} + +/*static*/ bool CommandLineParser::ParseArguments(PerformanceTestConfig& test_config, int argc, char* argv[]) { + int ch; + while ((ch = getopt(argc, argv, "m:e:r:t:p:xvhs")) != -1) { + switch (ch) { + case 'm': + if (!strcmp(optarg, "duration")) { + test_config.run_config.test_mode = TestMode::kFixDurationMode; + } else if (!strcmp(optarg, "times")) { + test_config.run_config.test_mode = TestMode::KFixRepeatedTimesMode; + } else { + return false; + } + break; + case 'p': + test_config.run_config.profile_file = optarg; + break; + case 'e': + if (!strcmp(optarg, "cpu")) { + test_config.machine_config.provider_type_name = onnxruntime::kCpuExecutionProvider; + } else if (!strcmp(optarg, "cuda")) { + test_config.machine_config.provider_type_name = onnxruntime::kCudaExecutionProvider; + } else if (!strcmp(optarg, "mkldnn")) { + test_config.machine_config.provider_type_name = onnxruntime::kMklDnnExecutionProvider; + } else if (!strcmp(optarg, "brainslice")) { + test_config.machine_config.provider_type_name = onnxruntime::kBrainSliceExecutionProvider; + } else { + return false; + } + break; + case 'r': + test_config.run_config.repeated_times = static_cast(strtol(optarg, nullptr, 10)); + if (test_config.run_config.repeated_times <= 0) { + return false; + } + break; + case 't': + test_config.run_config.duration_in_seconds = static_cast(strtol(optarg, nullptr, 10)); + if (test_config.run_config.repeated_times <= 0) { + return false; + } + break; + case 's': + test_config.run_config.f_dump_statistics = true; + break; + case 'v': + test_config.run_config.f_verbose = true; + break; + case 'x': + test_config.run_config.enable_sequential_execution = false; + break; + case '?': + case 'h': + default: + return false; + } + } + + // parse model_path and result_file_path + argc -= optind; + argv += optind; + if (argc != 2) return false; + + test_config.model_info.model_file_path = argv[0]; + test_config.model_info.result_file_path = argv[1]; + + return true; +} + +} // namespace perftest +} // namespace onnxruntime diff --git a/onnxruntime/test/perftest/command_args_parser.h b/onnxruntime/test/perftest/command_args_parser.h new file mode 100644 index 0000000000000..552cf73d17936 --- /dev/null +++ b/onnxruntime/test/perftest/command_args_parser.h @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +namespace onnxruntime { +namespace perftest { + +struct PerformanceTestConfig; + +class CommandLineParser { + public: + static void ShowUsage(); + + static bool ParseArguments(PerformanceTestConfig& test_config, int argc, char* argv[]); +}; + +} // namespace perftest +} // namespace onnxruntime diff --git a/onnxruntime/test/perftest/main.cc b/onnxruntime/test/perftest/main.cc new file mode 100644 index 0000000000000..2fdca31880180 --- /dev/null +++ b/onnxruntime/test/perftest/main.cc @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// onnxruntime dependencies +#include +#include +#include +#include + +#include "command_args_parser.h" +#include "performance_runner.h" + +using namespace onnxruntime; + +int main(int argc, char* args[]) { + std::string default_logger_id{"Default"}; + logging::LoggingManager default_logging_manager{std::unique_ptr{new logging::CLogSink{}}, + logging::Severity::kWARNING, false, + logging::LoggingManager::InstanceType::Default, + &default_logger_id}; + + std::unique_ptr env; + auto status = Environment::Create(env); + if (!status.IsOK()) { + LOGF_DEFAULT(ERROR, "failed to create environment:%s", status.ErrorMessage().c_str()); + return -1; + } + + ::onnxruntime::perftest::PerformanceTestConfig test_config; + if (!::onnxruntime::perftest::CommandLineParser::ParseArguments(test_config, argc, args)) { + ::onnxruntime::perftest::CommandLineParser::ShowUsage(); + return -1; + } + + ::onnxruntime::perftest::PerformanceRunner perf_runner(test_config); + status = perf_runner.Run(); + if (!status.IsOK()) { + LOGF_DEFAULT(ERROR, "Run failed:%s", status.ErrorMessage().c_str()); + return -1; + } + + perf_runner.SerializeResult(); + + return 0; +} diff --git a/onnxruntime/test/perftest/performance_runner.cc b/onnxruntime/test/perftest/performance_runner.cc new file mode 100644 index 0000000000000..7bd09070683d6 --- /dev/null +++ b/onnxruntime/test/perftest/performance_runner.cc @@ -0,0 +1,131 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "performance_runner.h" +#include "TestCase.h" +#include +#ifdef _MSC_VER +#include +#endif +#include "core/graph/graph.h" //for onnxruntime::NodeArg +#include "utils.h" +#include "testenv.h" +#include "providers.h" + +using namespace std::experimental::filesystem::v1; +using onnxruntime::Status; + +namespace onnxruntime { +namespace perftest { +Status PerformanceRunner::Run() { + if (!Initialize()) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "failed to initialize."); + } + + // warm up + RunOneIteration(true /*isWarmup*/); + + if (!performance_test_config_.run_config.profile_file.empty()) + session_object_->StartProfiling(performance_test_config_.run_config.profile_file); + + std::unique_ptr p_ICPUUsage = utils::CreateICPUUsage(); + switch (performance_test_config_.run_config.test_mode) { + case TestMode::kFixDurationMode: + ONNXRUNTIME_RETURN_IF_ERROR(RunFixDuration()); + break; + case TestMode::KFixRepeatedTimesMode: + ONNXRUNTIME_RETURN_IF_ERROR(RunRepeatedTimes()); + break; + default: + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "unknown test mode."); + } + performance_result_.average_CPU_usage = p_ICPUUsage->GetUsage(); + performance_result_.peak_workingset_size = utils::GetPeakWorkingSetSize(); + + if (!performance_test_config_.run_config.profile_file.empty()) + session_object_->EndProfiling(); + + std::cout << "Total time cost:" << performance_result_.total_time_cost << std::endl + << "Total iterations:" << performance_result_.time_costs.size() << std::endl + << "Average time cost:" << performance_result_.total_time_cost / performance_result_.time_costs.size() << std::endl; + return Status::OK(); +} + +bool PerformanceRunner::Initialize() { + path model_path(performance_test_config_.model_info.model_file_path); + if (model_path.extension() != ".onnx") { + LOGF_DEFAULT(ERROR, "input path is not a valid model"); + return false; + } + + // TO DO: remove the input and model name's dependency on directory tree + std::string model_name = model_path.parent_path().filename().string(); + if (model_name.compare(0, 5, "test_") == 0) model_name = model_name.substr(5); + performance_result_.model_name = model_name; + + // TO DO: remove depedency on OnnxTestCase. + std::unique_ptr test_case(CreateOnnxTestCase(model_name)); + + if (!test_case->SetModelPath(model_path).IsOK()) { + LOGF_DEFAULT(ERROR, "load model failed"); + return false; + } + + std::vector provider_types; + if (performance_test_config_.machine_config.provider_type_name == onnxruntime::kCpuExecutionProvider) { + provider_types = {onnxruntime::kMklDnnExecutionProvider, onnxruntime::kCpuExecutionProvider}; + } + provider_types = {performance_test_config_.machine_config.provider_type_name}; + SessionFactory sf(provider_types, true, true); + sf.enable_sequential_execution = performance_test_config_.run_config.enable_sequential_execution; + sf.session_thread_pool_size = 6; + + sf.create(session_object_, test_case->GetModelUrl(), test_case->GetTestCaseName()); + + // Initialize IO Binding + if (!session_object_->NewIOBinding(&io_binding_).IsOK()) { + LOGF_DEFAULT(ERROR, "Failed to init session and IO binding"); + return false; + } + + auto provider_type = performance_test_config_.machine_config.provider_type_name; + // Place input tensor on cpu memory if mkldnn provider type to avoid CopyTensor logic in CopyInputAcrossDevices + // TODO: find a better way to do this. + if (provider_type == onnxruntime::kMklDnnExecutionProvider) { + provider_type = onnxruntime::kCpuExecutionProvider; + } + AllocatorPtr cpu_allocator = io_binding_->GetCPUAllocator(0, provider_type); + test_case->SetAllocator(cpu_allocator); + + if (test_case->GetDataCount() <= 0) { + LOGF_DEFAULT(ERROR, "there is no test data for model %s", test_case->GetTestCaseName().c_str()); + return false; + } + + std::unordered_map feeds; + test_case->LoadTestData(0 /* id */, feeds, true); + for (auto feed : feeds) { + io_binding_->BindInput(feed.first, feed.second); + } + auto outputs = session_object_->GetModelOutputs(); + auto status = outputs.first; + if (!outputs.first.IsOK()) { + LOGF_DEFAULT(ERROR, "GetOutputs failed, TestCaseName:%s, ErrorMessage:%s", + test_case->GetTestCaseName().c_str(), + status.ErrorMessage().c_str()); + return false; + } + + std::vector output_mlvalues(outputs.second->size()); + for (size_t i_output = 0; i_output < outputs.second->size(); ++i_output) { + auto output = outputs.second->at(i_output); + if (!output) continue; + io_binding_->BindOutput(output->Name(), output_mlvalues[i_output]); + } + + return true; +} + +} // namespace perftest + +} // namespace onnxruntime diff --git a/onnxruntime/test/perftest/performance_runner.h b/onnxruntime/test/perftest/performance_runner.h new file mode 100644 index 0000000000000..1143d59e5548d --- /dev/null +++ b/onnxruntime/test/perftest/performance_runner.h @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include +#include + +// onnxruntime dependencies +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test_configuration.h" + +namespace onnxruntime { +namespace perftest { + +struct PerformanceResult { + size_t peak_workingset_size{0}; + short average_CPU_usage{0}; + double total_time_cost{0}; + std::vector time_costs; + std::string model_name; + + void DumpToFile(const std::string& path, bool f_include_statistics = false) const { + std::ofstream outfile; + outfile.open(path, std::ofstream::out | std::ofstream::app); + if (!outfile.good()) { + LOGF_DEFAULT(ERROR, "failed to open result file"); + return; + } + + for (size_t runs = 0; runs < time_costs.size(); runs++) { + outfile << model_name << "," << time_costs[runs] << "," << peak_workingset_size << "," << average_CPU_usage << "," << runs << std::endl; + } + + if (time_costs.size() > 0 && f_include_statistics) { + std::vector sorted_time = time_costs; + + size_t total = sorted_time.size(); + size_t n50 = static_cast(total * 0.5); + size_t n90 = static_cast(total * 0.9); + size_t n95 = static_cast(total * 0.95); + size_t n99 = static_cast(total * 0.99); + size_t n999 = static_cast(total * 0.999); + + std::sort(sorted_time.begin(), sorted_time.end()); + + outfile << std::endl; + outfile << "P50 Latency is " << sorted_time[n50] << "sec" << std::endl; + outfile << "P90 Latency is " << sorted_time[n90] << "sec" << std::endl; + outfile << "P95 Latency is " << sorted_time[n95] << "sec" << std::endl; + outfile << "P99 Latency is " << sorted_time[n99] << "sec" << std::endl; + outfile << "P999 Latency is " << sorted_time[n999] << "sec" << std::endl; + } + + outfile.close(); + } +}; + +class PerformanceRunner { + public: + PerformanceRunner(const PerformanceTestConfig& test_config) : performance_test_config_(test_config) {} + + Status Run(); + + inline const PerformanceResult& GetResult() const { return performance_result_; } + + inline void SerializeResult() const { performance_result_.DumpToFile(performance_test_config_.model_info.result_file_path, performance_test_config_.run_config.f_dump_statistics); } + + private: + bool Initialize(); + + inline Status RunOneIteration(bool isWarmup = false) { + auto start = std::chrono::high_resolution_clock::now(); + ONNXRUNTIME_RETURN_IF_ERROR(session_object_->Run(*io_binding_)); + auto end = std::chrono::high_resolution_clock::now(); + + if (!isWarmup) { + std::chrono::duration duration_seconds = end - start; + performance_result_.time_costs.emplace_back(duration_seconds.count()); + performance_result_.total_time_cost += duration_seconds.count(); + if (performance_test_config_.run_config.f_verbose) { + std::cout << "iteration:" << performance_result_.time_costs.size() << "," + << "time_cost:" << performance_result_.time_costs.back() << std::endl; + } + } + return Status::OK(); + } + + inline Status RunFixDuration() { + while (performance_result_.total_time_cost < performance_test_config_.run_config.duration_in_seconds) { + ONNXRUNTIME_RETURN_IF_ERROR(RunOneIteration()); + } + return Status::OK(); + } + + inline Status RunRepeatedTimes() { + for (size_t ite = 0; ite < performance_test_config_.run_config.repeated_times; ite++) { + ONNXRUNTIME_RETURN_IF_ERROR(RunOneIteration()); + } + return Status::OK(); + } + + private: + PerformanceResult performance_result_; + PerformanceTestConfig performance_test_config_; + + std::shared_ptr<::onnxruntime::InferenceSession> session_object_; + std::unique_ptr io_binding_; +}; +} // namespace perftest +} // namespace onnxruntime diff --git a/onnxruntime/test/perftest/posix/utils.cc b/onnxruntime/test/perftest/posix/utils.cc new file mode 100644 index 0000000000000..931274fcab7e7 --- /dev/null +++ b/onnxruntime/test/perftest/posix/utils.cc @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "test/perftest/utils.h" + +#include + +#include +#include + +#include "core/platform/env.h" + +namespace onnxruntime { +namespace perftest { +namespace utils { + +std::size_t GetPeakWorkingSetSize() { + struct rusage rusage; + getrusage(RUSAGE_SELF, &rusage); + return static_cast(rusage.ru_maxrss * 1024L); +} + +class CPUUsage : public ICPUUsage { + public: + CPUUsage() { + Reset(); + } + + short GetUsage() const override { + struct tms time_sample; + clock_t total_clock_now = times(&time_sample); + if (total_clock_now <= total_clock_start_ || + time_sample.tms_stime < proc_sys_clock_start_ || + time_sample.tms_utime < proc_user_clock_start_) { + // overflow detection + return -1; + } else { + clock_t proc_total_clock_diff = (time_sample.tms_stime - proc_sys_clock_start_) + (time_sample.tms_utime - proc_user_clock_start_); + clock_t total_clock_diff = total_clock_now - total_clock_start_; + return static_cast(100.0 * proc_total_clock_diff / total_clock_diff / onnxruntime::Env::Default().GetNumCpuCores()); + } + } + + void Reset() override { + struct tms time_sample; + total_clock_start_ = times(&time_sample); + proc_sys_clock_start_ = time_sample.tms_stime; + proc_user_clock_start_ = time_sample.tms_utime; + } + + private: + clock_t total_clock_start_; + clock_t proc_sys_clock_start_; + clock_t proc_user_clock_start_; +}; + +std::unique_ptr CreateICPUUsage() { + return std::make_unique(); +} + +} // namespace utils +} // namespace perftest +} // namespace onnxruntime diff --git a/onnxruntime/test/perftest/test_configuration.h b/onnxruntime/test/perftest/test_configuration.h new file mode 100644 index 0000000000000..263835e4890b3 --- /dev/null +++ b/onnxruntime/test/perftest/test_configuration.h @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include + +#include "core/graph/constants.h" + +namespace onnxruntime { +namespace perftest { + +enum class TestMode : std::uint8_t { + kFixDurationMode = 0, + KFixRepeatedTimesMode +}; + +enum class Platform : std::uint8_t { + kWindows = 0, + kLinux +}; + +struct ModelInfo { + std::string model_name; + std::string model_file_path; + std::string input_file_path; + std::string result_file_path; +}; + +struct MachineConfig { + Platform platform{Platform::kWindows}; + std::string provider_type_name{onnxruntime::kCpuExecutionProvider}; +}; + +struct RunConfig { + std::string profile_file; + TestMode test_mode{TestMode::kFixDurationMode}; + size_t repeated_times{1000}; + size_t duration_in_seconds{600}; + bool f_dump_statistics{false}; + bool f_verbose{false}; + bool enable_sequential_execution{true}; +}; + +struct PerformanceTestConfig { + ModelInfo model_info; + MachineConfig machine_config; + RunConfig run_config; +}; + +} // namespace perftest +} // namespace onnxruntime diff --git a/onnxruntime/test/perftest/testenv.cc b/onnxruntime/test/perftest/testenv.cc new file mode 100644 index 0000000000000..f4378f63e75cf --- /dev/null +++ b/onnxruntime/test/perftest/testenv.cc @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "testenv.h" +#include +#include +#include +#include +#include +#include +#ifdef _MSC_VER +#include +#endif +#include "providers.h" + +using namespace std::experimental::filesystem::v1; +using onnxruntime::Status; + +inline void RegisterExecutionProvider(onnxruntime::InferenceSession* sess, ONNXRuntimeProviderFactoryPtr* f) { + ONNXRuntimeProviderPtr p; + (*f)->CreateProvider(f, &p); + std::unique_ptr q((onnxruntime::IExecutionProvider*)p); + auto status = sess->RegisterExecutionProvider(std::move(q)); + if (!status.IsOK()) { + throw std::runtime_error(status.ErrorMessage().c_str()); + } +} +#define FACTORY_PTR_HOLDER \ + std::unique_ptr ptr_holder_(f, ONNXRuntimeReleaseObject); + +Status SessionFactory::create(std::shared_ptr<::onnxruntime::InferenceSession>& sess, const path& model_url, const std::string& logid) const { + ::onnxruntime::SessionOptions so; + so.session_logid = logid; + so.enable_cpu_mem_arena = enable_cpu_mem_arena_; + so.enable_mem_pattern = enable_mem_pattern_; + so.enable_sequential_execution = enable_sequential_execution; + so.session_thread_pool_size = session_thread_pool_size; + sess.reset(new ::onnxruntime::InferenceSession(so)); + + Status status; + for (const std::string& provider : providers_) { + if (provider == onnxruntime::kCudaExecutionProvider) { +#ifdef USE_CUDA + ONNXRuntimeProviderFactoryPtr* f; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateCUDAExecutionProviderFactory(0, &f)); + FACTORY_PTR_HOLDER; + RegisterExecutionProvider(sess.get(), f); +#else + ONNXRUNTIME_THROW("CUDA is not supported in this build"); +#endif + } else if (provider == onnxruntime::kMklDnnExecutionProvider) { +#ifdef USE_MKLDNN + ONNXRuntimeProviderFactoryPtr* f; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateMkldnnExecutionProviderFactory(enable_cpu_mem_arena_ ? 1 : 0, &f)); + FACTORY_PTR_HOLDER; + RegisterExecutionProvider(sess.get(), f); +#else + ONNXRUNTIME_THROW("CUDA is not supported in this build"); +#endif + } else if (provider == onnxruntime::kNupharExecutionProvider) { +#ifdef USE_NUPHAR + ONNXRuntimeProviderFactoryPtr* f; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateNupharExecutionProviderFactory(0, "", &f)); + RegisterExecutionProvider(sess.get(), f); + FACTORY_PTR_HOLDER; +#else + ONNXRUNTIME_THROW("CUDA is not supported in this build"); +#endif + } else if (provider == onnxruntime::kBrainSliceExecutionProvider) { +#if USE_BRAINSLICE + ONNXRuntimeProviderFactoryPtr* f; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateBrainSliceExecutionProviderFactory(0, true, "testdata/firmwares/onnx_rnns/instructions.bin", "testdata/firmwares/onnx_rnns/data.bin", "testdata/firmwares/onnx_rnns/schema.bin", &f)); + RegisterExecutionProvider(sess.get(), f); + FACTORY_PTR_HOLDER; +#else + ONNXRUNTIME_THROW("This executable was not built with BrainSlice"); +#endif + } + //TODO: add more + } + + status = sess->Load(model_url.string()); + ONNXRUNTIME_RETURN_IF_ERROR(status); + LOGS_DEFAULT(INFO) << "successfully loaded model from " << model_url; + status = sess->Initialize(); + if (status.IsOK()) + LOGS_DEFAULT(INFO) << "successfully initialized model from " << model_url; + return status; +} diff --git a/onnxruntime/test/perftest/testenv.h b/onnxruntime/test/perftest/testenv.h new file mode 100644 index 0000000000000..3e25c449888a1 --- /dev/null +++ b/onnxruntime/test/perftest/testenv.h @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include +#include +#include + +#include +#ifdef _MSC_VER +#include +#endif + +class ITestCase; +class TestCaseResult; +template +class FixedCountFinishCallbackImpl; +using FixedCountFinishCallback = FixedCountFinishCallbackImpl; + +class SessionFactory { + private: + const std::vector providers_; + bool enable_mem_pattern_ = true; + bool enable_cpu_mem_arena_ = true; + + public: + SessionFactory(std::vector providers, bool enable_mem_pattern, bool enable_cpu_mem_arena) : providers_(std::move(providers)), enable_mem_pattern_(enable_mem_pattern), enable_cpu_mem_arena_(enable_cpu_mem_arena) {} + //Create an initialized session from a given model url + ::onnxruntime::common::Status create(std::shared_ptr<::onnxruntime::InferenceSession>& sess, const std::experimental::filesystem::v1::path& model_url, const std::string& logid) const; + + bool enable_sequential_execution = true; + int session_thread_pool_size = 0; +}; diff --git a/onnxruntime/test/perftest/utils.h b/onnxruntime/test/perftest/utils.h new file mode 100644 index 0000000000000..f22abc04fa99e --- /dev/null +++ b/onnxruntime/test/perftest/utils.h @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +namespace onnxruntime { +namespace perftest { +namespace utils { + +size_t GetPeakWorkingSetSize(); + +class ICPUUsage { + public: + virtual ~ICPUUsage() = default; + + virtual short GetUsage() const = 0; + + virtual void Reset() = 0; +}; + +std::unique_ptr CreateICPUUsage(); + +} // namespace utils +} // namespace perftest +} // namespace onnxruntime diff --git a/onnxruntime/test/perftest/windows/utils.cc b/onnxruntime/test/perftest/windows/utils.cc new file mode 100644 index 0000000000000..4ebdd64b58ce0 --- /dev/null +++ b/onnxruntime/test/perftest/windows/utils.cc @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "test/perftest/utils.h" + +#include + +#include +#include + +namespace onnxruntime { +namespace perftest { +namespace utils { + +size_t GetPeakWorkingSetSize() { + PROCESS_MEMORY_COUNTERS pmc; + if (GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc))) { + return pmc.PeakWorkingSetSize; + } + + return 0; +} + +static std::uint64_t SubtractFILETIME(const FILETIME& ft_a, const FILETIME& ft_b) { + LARGE_INTEGER a, b; + a.LowPart = ft_a.dwLowDateTime; + a.HighPart = ft_a.dwHighDateTime; + + b.LowPart = ft_b.dwLowDateTime; + b.HighPart = ft_b.dwHighDateTime; + + return a.QuadPart - b.QuadPart; +} + +class CPUUsage : public ICPUUsage { + public: + CPUUsage() { + Reset(); + } + + short GetUsage() const override { + FILETIME sys_idle_ft, sys_kernel_ft, sys_user_ft, proc_creation_ft, proc_exit_ft, proc_kernel_ft, proc_user_ft; + GetSystemTimes(&sys_idle_ft, &sys_kernel_ft, &sys_user_ft); + GetProcessTimes(GetCurrentProcess(), &proc_creation_ft, &proc_exit_ft, &proc_kernel_ft, &proc_user_ft); + + std::uint64_t sys_kernel_ft_diff = SubtractFILETIME(sys_kernel_ft, sys_kernel_ft_); + std::uint64_t sys_user_ft_diff = SubtractFILETIME(sys_user_ft, sys_user_ft_); + + std::uint64_t proc_kernel_diff = SubtractFILETIME(proc_kernel_ft, proc_kernel_ft_); + std::uint64_t proc_user_diff = SubtractFILETIME(proc_user_ft, proc_user_ft_); + + std::uint64_t total_sys = sys_kernel_ft_diff + sys_user_ft_diff; + std::uint64_t total_proc = proc_kernel_diff + proc_user_diff; + + return total_sys > 0 ? static_cast((100.0 * total_proc) / total_sys) : 0; + } + + void Reset() override { + FILETIME sys_idle_ft, proc_creation_ft, proc_exit_ft; + GetSystemTimes(&sys_idle_ft, &sys_kernel_ft_, &sys_user_ft_); + GetProcessTimes(GetCurrentProcess(), &proc_creation_ft, &proc_exit_ft, &proc_kernel_ft_, &proc_user_ft_); + } + + private: + //system total times + FILETIME sys_kernel_ft_; + FILETIME sys_user_ft_; + + //process times + FILETIME proc_kernel_ft_; + FILETIME proc_user_ft_; +}; + +std::unique_ptr CreateICPUUsage() { + return std::make_unique(); +} + +} // namespace utils +} // namespace perftest +} // namespace onnxruntime diff --git a/onnxruntime/test/platform/windows/logging/HowToValidateEtwSinkOutput.md b/onnxruntime/test/platform/windows/logging/HowToValidateEtwSinkOutput.md new file mode 100644 index 0000000000000..2f8d06d66d576 --- /dev/null +++ b/onnxruntime/test/platform/windows/logging/HowToValidateEtwSinkOutput.md @@ -0,0 +1,34 @@ +## Validating ETW Sink unit test output + +## Setup +Install Windows Performance Toolkit from +You get to select components when installing, so can select just the performance toolkit. + +Overview of the steps is at if you want more detail. + +## Capturing ETW trace output + +From an elevated prompt, assuming wpr.exe and wpa.exe are in the path (if not, most likely they can be found under C:\Program Files (x86)\Windows Kits\10\Windows Performance Toolkit\) do the following: + +Start the ETW tracing + `\onnxruntime\test\platform\windows\logging> wpr -start .\etw_provider.wprp` + + Note: If you add '-start GeneralProfile' a huge amount of data will be captured. Not necessary for checking log messages are produced. It will also result in a huge number of ngen.exe calls when you attempt to stop logging the first type. + +Run the ETW sink unit tests + + * Run the tests in etw_sink_test.cc and then stop WPR. + * This can be done by executing the specific tests in Visual Studio, + * or by running the exe with all the tests from the platform library + * Assuming debug build on Windows run `\build\Windows\Debug\Debug\onnxruntime_test_framework.exe` + +Stop the ETW tracing + `\onnxruntime\test\platform\windows\logging> wpr -stop TraceCaptureFile.etl EtwSinkTest` + +## View the output + +Open TraceCaptureFile.etl file Windows Performance Analyzer. + +Expand the "System Activity" dropdown in the left pane, and double-click "Generic Events". +That should open events in an Analysis window in the right pane. You should see an event +with provider of ONNXRuntimeTraceLoggingProvider, and the log message output. diff --git a/onnxruntime/test/platform/windows/logging/etw_provider.wprp b/onnxruntime/test/platform/windows/logging/etw_provider.wprp new file mode 100644 index 0000000000000..737fd00552e61 --- /dev/null +++ b/onnxruntime/test/platform/windows/logging/etw_provider.wprp @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/onnxruntime/test/platform/windows/logging/etw_sink_test.cc b/onnxruntime/test/platform/windows/logging/etw_sink_test.cc new file mode 100644 index 0000000000000..c71bab2119fad --- /dev/null +++ b/onnxruntime/test/platform/windows/logging/etw_sink_test.cc @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/platform/windows/logging/etw_sink.h" + +#ifdef ETW_TRACE_LOGGING_SUPPORTED + +#include "core/common/logging/capture.h" +#include "core/common/logging/logging.h" +#include "core/common/logging/sinks/composite_sink.h" + +#include "test/common/logging/helpers.h" + +namespace onnxruntime { +namespace test { + +using namespace ::onnxruntime::logging; + +/// +/// Test usage of the ETW sinks does not fail. +/// +TEST(LoggingTests, TestEtwSink) { + const std::string logid{"ETW"}; + const std::string message{"Test message"}; + + // create scoped manager so sink gets destroyed once done and we check disposal + // within the scope of this test + { + LoggingManager manager{std::unique_ptr{new EtwSink{}}, Severity::kWARNING, false, + LoggingManager::InstanceType::Temporal}; + + auto logger = manager.CreateLogger(logid); + + LOGS(*logger, WARNING) << message; + + // can't test much else without creating an interface for ETW, using that in EtwSink + // and mocking that interface here. too much work given how trivial the logic in EtwSink is. + } +} + +/// +/// Test that attempting to create two ETW sinks is fine. +/// We register the ETW handler for the duration of the program so it can be shared +/// across multiple sinks. +/// +TEST(LoggingTests, TestEtwSinkCtor) { + CompositeSink* sinks = new CompositeSink(); + sinks->AddSink(std::unique_ptr(new EtwSink())) + .AddSink(std::unique_ptr(new EtwSink())); + + LoggingManager manager{std::unique_ptr{sinks}, + Severity::kWARNING, + false, + LoggingManager::InstanceType::Temporal}; + + auto logger = manager.CreateLogger("logid"); + + LOGS(*logger, WARNING) << "Two sinks aren't better than one"; +} + +} // namespace test +} // namespace onnxruntime +#endif // ETW_TRACE_LOGGING_SUPPORTED diff --git a/onnxruntime/test/platform/windows/stacktrace_test.cc b/onnxruntime/test/platform/windows/stacktrace_test.cc new file mode 100644 index 0000000000000..c7da48daadc57 --- /dev/null +++ b/onnxruntime/test/platform/windows/stacktrace_test.cc @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#if defined(_MSC_VER) && !defined(NDEBUG) + +#include "core/common/common.h" + +#include + +#include "gtest/gtest.h" +#include "gmock/gmock.h" + +namespace onnxruntime { +namespace test { + +using namespace ::testing; +//TVM is not working with StackTrace now. +#ifndef USE_TVM +TEST(StacktraceTests, BasicTests) { + auto result = ::onnxruntime::GetStackTrace(); + + // if we are running code coverage the Windows CaptureStackBackTrace function only returns a single + // frame that is unknown. adjust for that. + // works fine when running unit tests normally. + const bool have_working_stacktrace = result.size() > 1; + + if (have_working_stacktrace) + // this method name should be the first on the stack as we hide the calls to the infrastructure that + // creates the stack trace + EXPECT_THAT(result[0], HasSubstr("BasicTests")); + else + // check that we have + EXPECT_THAT(result[0], HasSubstr("Unknown symbol")); + + try { + ONNXRUNTIME_THROW("Testing"); + } catch (const OnnxRuntimeException& ex) { + auto msg = ex.what(); + std::cout << msg; + + if (have_working_stacktrace) + // unit tests are run by main() in test_main.cc, so make sure that is present + EXPECT_THAT(msg, HasSubstr("test_main.cc")); + else + // otherwise just make sure we captured where the throw was from + EXPECT_THAT(msg, HasSubstr("BasicTests")); + } +} +#endif +} // namespace test +} // namespace onnxruntime +#endif diff --git a/onnxruntime/test/proto/tml.proto b/onnxruntime/test/proto/tml.proto new file mode 100644 index 0000000000000..205d23f94a098 --- /dev/null +++ b/onnxruntime/test/proto/tml.proto @@ -0,0 +1,105 @@ +syntax = "proto3"; + +import "onnx-ml.proto"; +package onnxruntime.proto; + +//must sync with data_types.h +//MapStringToString +//MapStringToInt64 +//MapStringToFloat +//MapStringToDouble +//MapInt64ToString +//MapInt64ToInt64 +//MapInt64ToFloat +//MapInt64ToDouble +//VectorString +//VectorFloat +//VectorInt64 +//VectorDouble +//VectorMapStringToFloat +//VectorMapInt64ToFloat + +message MapStringToString { + map v = 1; +} + +message MapStringToInt64 { + map v = 1; +} + +message MapStringToDouble { + map v = 1; +} + +message MapStringToFloat { + map v = 1; +} + +message MapInt64ToString { + map v = 1; +} + +message MapInt64ToInt64 { + map v = 1; +} + +message MapInt64ToFloat { + map v = 1; +} + +message MapInt64ToDouble { + map v = 1; +} + +message VectorString { + repeated string v = 1; +} + +message VectorFloat { + repeated float v = 1; +} + +message VectorInt64 { + repeated int64 v = 1; +} + +message VectorDouble { + repeated double v = 1; +} + +message VectorMapStringToFloat { + repeated MapStringToFloat v = 1; +} + +message VectorMapInt64ToFloat { + repeated MapInt64ToFloat v = 1; +} + +message TraditionalMLData { + oneof values { + MapStringToString map_string_to_string = 1; + MapStringToInt64 map_string_to_int64 = 2; + MapStringToFloat map_string_to_float = 3; + MapStringToDouble map_string_to_double = 4; + MapInt64ToString map_int64_to_string = 5; + MapInt64ToInt64 map_int64_to_int64 = 6; + MapInt64ToFloat map_int64_to_float = 7; + MapInt64ToDouble map_int64_to_double = 8; + VectorString vector_string = 9; + VectorFloat vector_float = 10; + VectorInt64 vector_int64 = 11; + VectorDouble vector_double = 12; + VectorMapStringToFloat vector_map_string_to_float = 13; + VectorMapInt64ToFloat vector_map_int64_to_float = 14; + onnx.TensorProto tensor = 16; + } + // Optionally, a name for the tensor. + string name = 15; + string debug_info = 17; //A human-readable string for helping debugging +} + +message TestCaseConfig { + double per_sample_tolerance = 1; + double relative_per_sample_tolerance = 2; + bool post_processing = 3; +} diff --git a/onnxruntime/test/providers/cpu/activation/activation_op_test.cc b/onnxruntime/test/providers/cpu/activation/activation_op_test.cc new file mode 100644 index 0000000000000..00ff67eb5070d --- /dev/null +++ b/onnxruntime/test/providers/cpu/activation/activation_op_test.cc @@ -0,0 +1,220 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/activation/activations.h" +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +void TestUnaryElementwiseOp(const char* szOp, std::vector& input_vals, + std::function expected_func, + const std::unordered_map attribs = {}) { + OpTester test(szOp); + + for (auto attr : attribs) + test.AddAttribute(attr.first, attr.second); + + std::vector dims{(int64_t)input_vals.size()}; + + std::vector expected_vals; + for (const auto& iv : input_vals) + expected_vals.push_back(expected_func(iv)); + + test.AddInput("X", dims, input_vals); + test.AddOutput("Y", dims, expected_vals); + test.Run(); +} + +std::vector input_vals = { + -1.0f, 0, 1.0f, // normal input values for activation + 100.0f, -100.0f, 1000.0f, -1000.0f, // input values that leads to exp() overflow + FLT_MIN, FLT_MIN / 10, -FLT_MIN / 10, // min, denorm, -denorm + FLT_MAX, -FLT_MAX, std::numeric_limits::infinity()}; // max, -max, inf + +std::vector no_inf_input_vals = { + -1.0f, 0, 1.0f, // normal input values for activation + FLT_MIN, FLT_MIN / 10, -FLT_MIN / 10, // min, denorm, -denorm + FLT_MAX, -FLT_MAX}; // max, -max + +TEST(ActivationOpTest, Sigmoid) { + TestUnaryElementwiseOp("Sigmoid", + input_vals, + [](float x) { + auto y = 1.f / (1.f + std::exp(-std::abs(x))); // safe sigmoid + y = x > 0 ? y : 1 - y; + return y; + }); +} + +TEST(ActivationOpTest, HardSigmoid) { + float alpha = 0.2f; + float beta = 0.5f; + TestUnaryElementwiseOp("HardSigmoid", + input_vals, + [alpha, beta](float x) { + return std::max(std::min((alpha * x + beta), 1.0f), 0.0f); + }, + {{"alpha", alpha}, {"beta", beta}}); +} + +TEST(ActivationOpTest, Tanh) { + TestUnaryElementwiseOp("Tanh", + input_vals, + [](float x) { return std::tanh(x); }); +} + +TEST(ActivationOpTest, Relu) { + TestUnaryElementwiseOp("Relu", + input_vals, + [](float x) { return std::max(x, 0.0f); }); +} + +TEST(ActivationOpTest, Elu) { + float alpha = 0.1f; + TestUnaryElementwiseOp("Elu", + input_vals, + [alpha](float x) { return (x >= 0) ? x : alpha * (exp(x) - 1); }, + {{"alpha", alpha}}); +} + +TEST(ActivationOpTest, LeakyRelu) { + float alpha = 0.1f; + TestUnaryElementwiseOp("LeakyRelu", + input_vals, + [alpha](float x) { return (x >= 0) ? x : alpha * x; }, + {{"alpha", alpha}}); +} + +TEST(ActivationOpTest, ThresholdedRelu) { + float alpha = 0.1f; + TestUnaryElementwiseOp("ThresholdedRelu", + input_vals, + [alpha](float x) { return (x >= alpha) ? x : 0; }, + {{"alpha", alpha}}); +} + +TEST(ActivationOpTest, ScaledTanh) { + static constexpr float alpha = 2.0f; + static constexpr float beta = 1.5f; + + TestUnaryElementwiseOp("ScaledTanh", + input_vals, + [](float x) { return alpha * tanh(beta * x); }, + {{"alpha", alpha}, {"beta", beta}}); +} + +TEST(ActivationOpTest, Selu) { + static constexpr float alpha = 1.6732f; + static constexpr float gamma = 1.0507f; + + TestUnaryElementwiseOp("Selu", + input_vals, + [](float x) { return x <= 0 ? gamma * (alpha * exp(x) - alpha) : gamma * x; }, + {{"alpha", alpha}, {"gamma", gamma}}); +} + +TEST(ActivationOpTest, Selu_Attributes) { + static constexpr float alpha = 1.8f; + static constexpr float gamma = 0.5f; + + TestUnaryElementwiseOp("Selu", + input_vals, + [](float x) { return x <= 0 ? gamma * (alpha * exp(x) - alpha) : gamma * x; }, + {{"alpha", alpha}, {"gamma", gamma}}); +} + +TEST(ActivationOpTest, PRelu) { + OpTester test("PRelu"); + + auto formula = [](float x, float slope) { return x < 0 ? slope * x : x; }; + + std::vector inputs{1.0f, -4.0f, 0.0f, -9.0f}; + std::vector slopes{1.0f, -2.0f, 3.0f, -4.0f}; + std::vector outputs; + for (unsigned i = 0; i < inputs.size(); i++) + outputs.push_back(formula(inputs[i], slopes[i])); + + std::vector dims{2, 2}; + test.AddInput("X", dims, inputs); + test.AddInput("slope", dims, slopes); + test.AddOutput("Y", dims, outputs); + test.Run(); +} + +TEST(ActivationOpTest, PRelu_SingleSlope) { + OpTester test("PRelu"); + + auto formula = [](float x, float slope) { return x < 0 ? slope * x : x; }; + + auto inputs = {1.0f, -4.0f, 0.0f, -9.0f}; + auto slope = 1.5f; + std::vector outputs; + for (auto& input : inputs) + outputs.push_back(formula(input, slope)); + + std::vector dims{2, 2}; + test.AddInput("X", dims, inputs); + test.AddInput("slope", {}, {slope}); + test.AddOutput("Y", dims, outputs); + test.Run(); +} + +TEST(ActivationOpTest, PRelu_MultiChannel) { + OpTester test("PRelu"); + + auto formula = [](float x, float slope) { return x < 0 ? slope * x : x; }; + + std::vector inputs{1.0f, 2.0f, -4.0f, 3.0f, 0.0f, 5.0f, -9.0f, 8.0f}; + std::vector slopes{1.0f, -2.0f}; + std::vector outputs; + const int64_t num_images = 2; + const int64_t num_channels = 2; + const int64_t num_pixels = 2; + for (unsigned i = 0; i < inputs.size(); i++) + outputs.push_back(formula(inputs[i], slopes[i / num_pixels % num_channels])); + + std::vector x_dims{num_images, num_channels, num_pixels}; + std::vector slope_dims{num_channels, 1}; + test.AddInput("X", x_dims, inputs); + test.AddInput("slope", slope_dims, slopes); + test.AddOutput("Y", x_dims, outputs); + test.Run(); +} + +TEST(ActivationOpTest, ParametricSoftplus) { + static constexpr float alpha = 2.0f; + static constexpr float beta = 1.5f; + + TestUnaryElementwiseOp("ParametricSoftplus", + input_vals, + [](float x) { + float bx = beta * x; + if (bx > 0) + return alpha * (bx + logf(expf(-bx) + 1)); + else + return alpha * logf(expf(bx) + 1); + }, + {{"alpha", alpha}, {"beta", beta}}); +} + +TEST(ActivationOpTest, Softplus) { + TestUnaryElementwiseOp("Softplus", + input_vals, + [](float x) { + if (x > 0) + return x + logf(expf(-x) + 1); + else + return logf(expf(x) + 1); + }); +} + +TEST(ActivationOpTest, Softsign) { + TestUnaryElementwiseOp("Softsign", + no_inf_input_vals, + [](float x) { return x / (1 + std::abs(x)); }); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/controlflow/if_test.cc b/onnxruntime/test/providers/cpu/controlflow/if_test.cc new file mode 100644 index 0000000000000..b53b451a0d4eb --- /dev/null +++ b/onnxruntime/test/providers/cpu/controlflow/if_test.cc @@ -0,0 +1,265 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "gmock/gmock.h" +// #include "core/framework/customregistry.h" +#include "core/framework/session_state.h" +#include "core/providers/cpu/controlflow/if.h" +#include "test/providers/provider_test_utils.h" +#include "core/session/inference_session.h" + +using namespace ONNX_NAMESPACE; + +namespace onnxruntime { +namespace test { + +struct RunOptions { + bool include_dim_values_in_main_graph = false; + bool symbolic_dim_values_in_main_graph = false; + bool include_dim_values_in_subgraph = true; +}; + +static const ONNX_NAMESPACE::GraphProto CreateSubgraph(bool then_branch, const RunOptions& options); + +/* + Main graph + + split_input if_cond if_graph_input_0, + | | | + [Split] | [Identity] + | | | + | | if_input_0 + | split_out_0 | | + ------------------[If]-------------- (see below for then/else subgraphs in If node) + split_out_1 | + | + if_out_0 +*/ + +class IfOpTester : public OpTester { + public: + IfOpTester(const RunOptions& options) : OpTester("If"), options_{options} { + } + + protected: + void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& /*add_attribute_funcs*/) override { + // Graph inputs are 0:Split input, 1:Cond for If, 2:if input + ASSERT_EQ(graph_input_defs.size(), 3); + ASSERT_EQ(graph_output_defs.size(), 1); + + NodeArg* split_input = graph_input_defs[0]; + NodeArg* if_cond_input = graph_input_defs[1]; + NodeArg* if_input = graph_input_defs[2]; + + std::vector inputs; + std::vector outputs; + + // add Split node + { + TypeProto split_out_type; + split_out_type.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + auto& split_out_0 = graph.GetOrCreateNodeArg("split_out_0", &split_out_type); + auto& split_out_1 = graph.GetOrCreateNodeArg("split_out_1", &split_out_type); + + inputs = {split_input}; + outputs = {&split_out_0, &split_out_1}; + + graph.AddNode("split", "Split", "Split into 2", inputs, outputs); + } + + // add If node + { + inputs = {if_cond_input}; + outputs = {graph_output_defs[0]}; + + auto if_node = graph.AddNode("if", "If", "If node", inputs, outputs); + + auto then_proto = CreateSubgraph(true, options_); + auto else_proto = CreateSubgraph(false, options_); + if_node->AddAttribute("then_branch", {then_proto}); + if_node->AddAttribute("else_branch", {else_proto}); + } + + // add Identity node so if_graph_input_0 comes from graph inputs + { + MTypeProto map_type; + + inputs = {if_input}; + outputs = {&graph.GetOrCreateNodeArg("if_input_0", if_input->TypeAsProto())}; + graph.AddNode("identity", "Identity", "Pass if input through from graph inputs.", inputs, outputs); + } + } + + private: + RunOptions options_; +}; + +/* Subgraphs looks like this. All inputs come from outer scope so we just + create a NodeArg with the input name. The numbers in [] are the values the tests are expected to produce + as output from each node. + +THEN branch + split_out_0 if_input_0 [1] + \ | + [1] \ | + \------[Add] + | + add_out_0 [2] + +ELSE branch + split_out_1 if_input_0 [1] + \ | + [10] \ | + \------[Add] + | + add_out_1 [11] +*/ + +static const ONNX_NAMESPACE::GraphProto CreateSubgraph(bool then_branch, const RunOptions& options) { + bool include_dim_values = options.include_dim_values_in_subgraph; + + Model model(then_branch ? "If_then" : "If_else"); + auto& graph = model.MainGraph(); + + std::vector inputs; + std::vector outputs; + + const std::string suffix = then_branch ? "0" : "1"; + + // graph input has to have type and rank even though it's an outer scope value. + TypeProto input_tensor_type; + input_tensor_type.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + auto* mutable_dim = input_tensor_type.mutable_tensor_type()->mutable_shape()->add_dim(); + if (include_dim_values) { + mutable_dim->set_dim_value(1); + } + + // outer scope values + auto& split_output = graph.GetOrCreateNodeArg("split_out_" + suffix, &input_tensor_type); + auto& if_input = graph.GetOrCreateNodeArg("if_input_0", &input_tensor_type); + + // add so that we don't end up with it being considered a graph input + graph.AddOuterScopeNodeArg("split_out_" + suffix); + graph.AddOuterScopeNodeArg("if_input_0"); + + { + // Add + + // graph output has to have type and shape + TypeProto add_output_tensor; + add_output_tensor.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + mutable_dim = add_output_tensor.mutable_tensor_type()->mutable_shape()->add_dim(); + if (include_dim_values) { + mutable_dim->set_dim_value(1); + } + + auto& add_out = graph.GetOrCreateNodeArg("add_out_" + suffix, &add_output_tensor); + + inputs = {&split_output, &if_input}; + outputs = {&add_out}; + + auto* add = graph.AddNode("add", "Add", "Add two inputs.", inputs, outputs); + (void)add; + } + + auto status = graph.Resolve(); + EXPECT_EQ(status, Status::OK()); + + auto& proto = graph.ToGraphProto(); + + return proto; +} + +void RunTest(bool condition_value, + RunOptions options, + OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess, + const std::string& failure_message = "") { + IfOpTester test{options}; + + test.AddShapeToTensorData(options.include_dim_values_in_main_graph, + options.symbolic_dim_values_in_main_graph); + + // add the main graph inputs and outputs. + // we will handle the 'If' inputs in the AddNodes override, and as 'If' is the last node + // it's outputs are 1:1 with the graph outputs. + + // simple tensor that we split into 2, and use one output for the 'then' and one for the 'else' branch in the If + test.AddInput("split_input", {2}, {1.f, 10.f}); + + // graph input to specify which branch to take + test.AddInput("if_cond", {1}, {condition_value}); + + test.AddInput("if_graph_input_0", {1}, {1.f}); + + std::vector output_shape{1}; + if (condition_value) { + test.AddOutput("if_out_0", output_shape, {2.f}); + } else { + test.AddOutput("if_out_0", output_shape, {11.f}); + } + + test.Run(expect_result, failure_message); +} + +TEST(If, ShapeInMainGraph_NoShapeInSubgraph_True) { + RunOptions options{}; + options.include_dim_values_in_main_graph = true; + options.include_dim_values_in_subgraph = false; + + RunTest(true, options); +} + +TEST(If, ShapeInMainGraph_NoShapeInSubgraph_False) { + RunOptions options{}; + options.include_dim_values_in_main_graph = true; + options.include_dim_values_in_subgraph = false; + + RunTest(false, options); +} + +TEST(If, NoShapeInMainGraph_ShapeInSubgraph_True) { + RunOptions options{}; + options.include_dim_values_in_main_graph = false; + options.include_dim_values_in_subgraph = true; + + RunTest(true, options); +} + +TEST(If, NoShapeInMainGraph_ShapeInSubgraph_False) { + RunOptions options{}; + options.include_dim_values_in_main_graph = false; + options.include_dim_values_in_subgraph = true; + + RunTest(false, options); +} + +/* +These tests require subgraphs with nodes that support symbolic dimensions. +'Add' does not. +TODO: Task 1913: Improve handling of If outputs to avoid copy when the shape is not known + at subgraph execution time. + +TEST(If, SymbolicShapeInMainGraph_NoShapeInSubgraph_True) { + RunOptions options; + options.include_dim_values_in_main_graph = true; + options.symbolic_dim_values_in_main_graph = true; + options.include_dim_values_in_subgraph = false; + + RunTest(true, options); +} + +TEST(If, SymbolicShapeInMainGraph_NoShapeInSubgraph_False) { + RunOptions options; + options.include_dim_values_in_main_graph = true; + options.symbolic_dim_values_in_main_graph = true; + options.include_dim_values_in_subgraph = false; + + RunTest(false, options); +} +*/ +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/controlflow/loop_test.cc b/onnxruntime/test/providers/cpu/controlflow/loop_test.cc new file mode 100644 index 0000000000000..531469d3f5bdb --- /dev/null +++ b/onnxruntime/test/providers/cpu/controlflow/loop_test.cc @@ -0,0 +1,466 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include "gtest/gtest.h" +#include "gmock/gmock.h" + +#include "core/common/logging/logging.h" +#include "core/framework/session_state.h" +#include "test/providers/provider_test_utils.h" +#include "core/session/inference_session.h" + +using namespace ONNX_NAMESPACE; + +namespace onnxruntime { +namespace test { + +struct RunOptions { + bool include_dim_values_in_main_graph = true; + bool include_dim_values_in_subgraph = false; + bool include_types_in_subgraph = false; +}; + +static const ONNX_NAMESPACE::GraphProto CreateSubgraph(const RunOptions& options); + +static const float kOuterNodeAddValue = 3.f; +static const float kSumMax = 8.f; + +class LoopOpTester : public OpTester { + public: + using SubgraphFunc = std::function; + + LoopOpTester(const RunOptions& options, SubgraphFunc create_subgraph = CreateSubgraph) + : OpTester("Loop", 8), options_{options}, create_subgraph_{create_subgraph} { + } + + protected: + void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& /*add_attribute_funcs*/) override { + // add outer_scope_0 node + { + TypeProto float_scalar; + float_scalar.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + auto mutable_dim = float_scalar.mutable_tensor_type()->mutable_shape()->add_dim(); + mutable_dim->set_dim_value(1); + + { + auto& output_arg = graph.GetOrCreateNodeArg("outer_scope_0", &float_scalar); + auto* constant = graph.AddNode("outer_scope_constant", "Constant", "Constant in outer scope", {}, + {&output_arg}); + + TensorProto value_tensor; + value_tensor.add_dims(1); + value_tensor.add_float_data(kOuterNodeAddValue); + value_tensor.set_data_type(onnx::TensorProto_DataType_FLOAT); + + constant->AddAttribute("value", value_tensor); + } + } + + // add Loop node + { + auto loop_node = graph.AddNode("loop", "Loop", "Loop node", graph_input_defs, graph_output_defs); + + auto body = create_subgraph_(options_); + loop_node->AddAttribute("body", {body}); + } + } + + private: + RunOptions options_; + SubgraphFunc create_subgraph_; +}; + +static const ONNX_NAMESPACE::GraphProto CreateSubgraph(const RunOptions& options) { + bool include_dim_value = options.include_dim_values_in_subgraph; + bool include_types = options.include_types_in_subgraph; + + // a subgraph output needs the type/shape to come from somewhere, + // so if the main graph isn't providing it, it has to come from here. + bool graph_output_shape_required = options.include_dim_values_in_main_graph == false; + + bool use_null_typeproto = !include_types && !include_dim_value && !graph_output_shape_required; + + Model model("Loop subgraph"); + auto& graph = model.MainGraph(); + + std::vector inputs; + std::vector outputs; + + /* Subgraph Adds outer_scope_0 to loop_var_0_in, + Concats the iter_num to loop_var_1_in (test loop var that changes shape) so each iteration appends the iter_num + to loop_var_1 + Loop output is the iter_num and sum for that iteration, so each iteration adds a pair to the overall output + Inputs require Identity nodes to fix their order. + + Inputs: iter_num, cond_in, loop_var_in + + iter_num_in loop_var_0_in [outer_scope_0] loop_var_1_in cond_in + | | / | (unused) + [Cast] [Add]-----/ | + | | | [Constant] + iter_num_float sum_0 | sum_0 / + | / | \ | \ / + [Concat]------/ | \---------------------[Concat] \--[Less]--/ + | | | | + | [Identity] | | + | | | | + loop_out_0 loop_var_0_out loop_var_1_out cond_out + + */ + + // graph inputs. must have type and at least rank + TypeProto int64_scalar; + int64_scalar.mutable_tensor_type()->set_elem_type(TensorProto_DataType_INT64); + int64_scalar.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + + TypeProto bool_scalar; + bool_scalar.mutable_tensor_type()->set_elem_type(TensorProto_DataType_BOOL); + bool_scalar.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + + TypeProto float_scalar; + float_scalar.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + auto* mutable_dim = float_scalar.mutable_tensor_type()->mutable_shape()->add_dim(); + if (include_dim_value) { + mutable_dim->set_dim_value(1); + } + + TypeProto float_tensor; + float_tensor.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + + // dimension value changes on each iteration so just add a dimension but no value + TypeProto float_tensor_single_dim{float_tensor}; + mutable_dim = float_tensor_single_dim.mutable_tensor_type()->mutable_shape()->add_dim(); + + // graph inputs + auto& iter_num_in = graph.GetOrCreateNodeArg("iter_num_in", &int64_scalar); + auto& cond_in = graph.GetOrCreateNodeArg("cond_in", &bool_scalar); + auto& loop_var_0_in = graph.GetOrCreateNodeArg("loop_var_0_in", &float_scalar); + auto& loop_var_1_in = graph.GetOrCreateNodeArg("loop_var_1_in", &float_tensor_single_dim); + + auto& iter_num_float = graph.GetOrCreateNodeArg("iter_num_float", &float_scalar); + + // outer scope values. need type but not shape. + auto& outer_scope_0 = graph.GetOrCreateNodeArg("outer_scope_0", &float_tensor); + + // add so that we don't end up with it being considered a graph input + graph.AddOuterScopeNodeArg("outer_scope_0"); + + // graph outputs + NodeArg* cond_out = nullptr; + NodeArg* loop_var_0_out = nullptr; + NodeArg* loop_var_1_out = nullptr; + NodeArg* loop_out_0 = nullptr; + + TypeProto sum_tensor; + auto* mutable_tensor_type = sum_tensor.mutable_tensor_type(); + mutable_tensor_type->set_elem_type(TensorProto_DataType_FLOAT); + if (include_dim_value) { + mutable_tensor_type->mutable_shape()->add_dim()->set_dim_value(1); + } /*else if (graph_output_shape_required) { + mutable_tensor_type->mutable_shape()->add_dim(); + }*/ + + TypeProto* type_proto = use_null_typeproto ? nullptr : &sum_tensor; + auto& sum_0 = graph.GetOrCreateNodeArg("sum_0", type_proto); + + // Add + { + inputs = {&outer_scope_0, &loop_var_0_in}; + outputs = {&sum_0}; + + graph.AddNode("add", "Add", "Add 1 to the loop carried var", inputs, outputs); + } + + // Convert iter_num to float + { + auto* cast = graph.AddNode("iter_num_cast", "Cast", "Cast iter_num to float", {&iter_num_in}, {&iter_num_float}); + cast->AddAttribute("to", int64_t{TensorProto_DataType_FLOAT}); + } + + // Concat iter_num and sum to create loop_out_0 + { + TypeProto loop_out_type; + mutable_tensor_type = loop_out_type.mutable_tensor_type(); + mutable_tensor_type->set_elem_type(TensorProto_DataType_FLOAT); + + if (include_dim_value) { + loop_out_type.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(2); + } else { + // tensor type must have rank + loop_out_type.mutable_tensor_type()->mutable_shape()->add_dim(); + } + + loop_out_0 = &graph.GetOrCreateNodeArg("loop_out_0", &loop_out_type); + + inputs = {&iter_num_float, &sum_0}; + outputs = {loop_out_0}; + + auto* concat = graph.AddNode("concat_0", "Concat", "Combine iter num and current sum", inputs, outputs); + concat->AddAttribute("axis", int64_t{0}); + } + + // output sum_0 as loop_var_0_out + { + TypeProto loop_var_0_out_type; + mutable_tensor_type = loop_var_0_out_type.mutable_tensor_type(); + mutable_tensor_type->set_elem_type(TensorProto_DataType_FLOAT); + + if (include_dim_value) { + loop_var_0_out_type.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + } else { + loop_var_0_out_type.mutable_tensor_type()->mutable_shape()->add_dim(); + } + + loop_var_0_out = &graph.GetOrCreateNodeArg("loop_var_0_out", &loop_var_0_out_type); + + inputs = {&sum_0}; + outputs = {loop_var_0_out}; + + graph.AddNode("identity", "Identity", "Output sum as loop_var_0_out", inputs, outputs); + } + + // Concat sum with loop_var_1 + { + TypeProto loop_var_1_out_type; + mutable_tensor_type = loop_var_1_out_type.mutable_tensor_type(); + mutable_tensor_type->set_elem_type(TensorProto_DataType_FLOAT); + // As we accumulate on each iteration the shape can only have a rank and not a specific value for the dimension. + // as the type will be inferred it must have a shape + loop_var_1_out_type.mutable_tensor_type()->mutable_shape()->add_dim(); + + loop_var_1_out = &graph.GetOrCreateNodeArg("loop_var_1_out", &loop_var_1_out_type); + + inputs = {&loop_var_1_in, &sum_0}; + outputs = {loop_var_1_out}; + + auto* concat = graph.AddNode("concat_1", "Concat", "Append value of sum to loop_var_1_out", inputs, outputs); + concat->AddAttribute("axis", int64_t{0}); + } + + // Update cond by checking if sum is < kSumMax + { + { + auto& max_value_out = graph.GetOrCreateNodeArg("max_value_out", &float_scalar); + auto* constant = graph.AddNode("constant_max_value", "Constant", "Constant with value kSumMax", + {}, {&max_value_out}); + + TensorProto value_tensor; + value_tensor.add_dims(1); + value_tensor.add_float_data(kSumMax); + value_tensor.set_data_type(onnx::TensorProto_DataType_FLOAT); + + constant->AddAttribute("value", value_tensor); + + cond_out = &graph.GetOrCreateNodeArg("cond_out", &bool_scalar); + + inputs = {&sum_0, &max_value_out}; + outputs = {cond_out}; + + graph.AddNode("sum_less_than_max", "Less", "Check sum < kSumMax", inputs, outputs); + } + } + + graph.SetInputOrder({&iter_num_in, &cond_in, &loop_var_0_in, &loop_var_1_in}); + graph.SetOutputOrder({cond_out, loop_var_0_out, loop_var_1_out, loop_out_0}); + + auto status = graph.Resolve(); + EXPECT_EQ(status, Status::OK()); + + return graph.ToGraphProto(); +} + +void RunTest(int64_t max_iterations, + float loop_var_0_final, + std::vector& loop_var_1_final_shape, + std::vector& loop_var_1_final, + std::vector& loop_out_0_final_shape, + std::vector& loop_out_0_final, + const RunOptions& options, + OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess, + const std::string& failure_message = "") { + LoopOpTester test{options}; + + test.AddShapeToTensorData(options.include_dim_values_in_main_graph); + + test.AddInput("M", {1}, {max_iterations}); + test.AddInput("cond", {1}, {true}); + + test.AddInput("loop_var_0_orig", {1}, {0.f}); + test.AddInput("loop_var_0_orig", {1}, {0.f}); + + test.AddOutput("loop_var_0_final", {1}, {loop_var_0_final}); + test.AddOutput("loop_var_1_final", loop_var_1_final_shape, loop_var_1_final); + test.AddOutput("loop_out_0_final", loop_out_0_final_shape, loop_out_0_final); + + test.Run(expect_result, failure_message); +} + +// exit due to hitting condition that the sum is < kSumMax which is 8 +// this should take 3 iterations as we add 3 each time. +void ExitDueToCond(const RunOptions& options) { + int64_t max_iterations = 5; + const int64_t expected_num_iterations = 3; + + float loop_var_0_final = kOuterNodeAddValue * expected_num_iterations; + + std::vector loop_var_1_final_shape{1 + expected_num_iterations}; + std::vector loop_var_1_final{0.f, 3.f, 6.f, 9.f}; + + std::vector loop_out_0_final_shape{expected_num_iterations, 2}; + std::vector loop_out_0_final{0.f, 3.f, // iter #, sum for each iteration + 1.f, 6.f, + 2.f, 9.f}; + + RunTest(max_iterations, + loop_var_0_final, + loop_var_1_final_shape, loop_var_1_final, + loop_out_0_final_shape, loop_out_0_final, + options); +} + +TEST(Loop, ExitDueToCond_DimsInMainGraph) { + RunOptions options{}; + options.include_dim_values_in_main_graph = true; + options.include_dim_values_in_subgraph = false; + options.include_dim_values_in_subgraph = false; + + ExitDueToCond(options); +} + +TEST(Loop, ExitDueToCond_DimsInSubgraph) { + RunOptions options{}; + options.include_dim_values_in_main_graph = false; + options.include_dim_values_in_subgraph = true; + options.include_types_in_subgraph = false; + + ExitDueToCond(options); +} + +TEST(Loop, ExitDueToMaxIterations) { + int64_t max_iterations = 2; + const int64_t expected_num_iterations = 2; + + float loop_var_0_final = kOuterNodeAddValue * expected_num_iterations; + + std::vector loop_var_1_final_shape{1 + expected_num_iterations}; + std::vector loop_var_1_final{0.f, 3.f, 6.f}; + + std::vector loop_out_0_final_shape{expected_num_iterations, 2}; + std::vector loop_out_0_final{0.f, 3.f, // iter #, sum for each iteration + 1.f, 6.f}; + + RunTest(max_iterations, + loop_var_0_final, + loop_var_1_final_shape, loop_var_1_final, + loop_out_0_final_shape, loop_out_0_final, + {}); +} + +TEST(Loop, InfiniteLoopTermination) { + auto create_subgraph = [](const RunOptions&) { + Model model("Infinite Loop subgraph"); + auto& graph = model.MainGraph(); + + std::vector inputs; + std::vector outputs; + + /* Never change cond_in so loop is infinite + Inputs: iter_num, cond_in, loop carried state variables. + + iter_num_in cond_in [outer_scope_0] + (unused) | | + [Identity] [Identity] + | | + cond_out loop_var_0_out + */ + + // graph inputs types. must have type and at least rank + TypeProto int64_scalar; + int64_scalar.mutable_tensor_type()->set_elem_type(TensorProto_DataType_INT64); + int64_scalar.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + + TypeProto bool_scalar; + bool_scalar.mutable_tensor_type()->set_elem_type(TensorProto_DataType_BOOL); + bool_scalar.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + + TypeProto float_tensor; + float_tensor.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + float_tensor.mutable_tensor_type()->mutable_shape()->add_dim(); + + // graph inputs + auto& iter_num_in = graph.GetOrCreateNodeArg("iter_num_in", &int64_scalar); + auto& cond_in = graph.GetOrCreateNodeArg("cond_in", &bool_scalar); + + // outer scope value. need type but not shape. + auto& outer_scope_0 = graph.GetOrCreateNodeArg("outer_scope_0", &float_tensor); + + // add so that we don't end up with it being considered a graph input + graph.AddOuterScopeNodeArg("outer_scope_0"); + + // graph outputs + auto& cond_out = graph.GetOrCreateNodeArg("cond_out", &bool_scalar); + auto& loop_var_0_out = graph.GetOrCreateNodeArg("loop_var_0_out", &float_tensor); + + // cond_in -> cond_out + { + inputs = {&cond_in}; + outputs = {&cond_out}; + + graph.AddNode("cond_in_identity", "Identity", "Forward cond_in to cond_out", inputs, outputs); + } + + // outer_scope_0 -> loop_var_0_out + { + inputs = {&outer_scope_0}; + outputs = {&loop_var_0_out}; + + graph.AddNode("loop_var_out", "Identity", "Forward outer_scope_0 to loop_var_0_out", inputs, outputs); + } + + graph.SetInputOrder({&iter_num_in, &cond_in, &outer_scope_0}); + graph.SetOutputOrder({&cond_out, &loop_var_0_out}); + + auto status = graph.Resolve(); + EXPECT_EQ(status, Status::OK()); + + return graph.ToGraphProto(); + }; + + LoopOpTester test{{}, create_subgraph}; + + test.AddInput("M", {1}, {INT64_MAX}); + test.AddInput("cond", {1}, {true}); + test.AddInput("fake", {1}, {0.f}); + + test.AddOutput("loop_var_0_final", {1}, {0.f}); + + ONNXRuntimeRunOptions session_run_options; + session_run_options.run_tag = "Loop.InfiniteLoopTermination"; + + auto terminator = [&session_run_options]() { + std::this_thread::sleep_for(std::chrono::seconds(3)); + LOGS_DEFAULT(WARNING) << "Setting terminate flag in run options."; + session_run_options.terminate = true; + return; + }; + + std::packaged_task task{terminator}; + std::future terminator_result = task.get_future(); + std::thread terminator_thread{std::move(task)}; + + test.Run(OpTester::ExpectResult::kExpectFailure, "Exiting due to terminate flag being set to true", {}, + &session_run_options); + + // call get to propagate any exception + terminator_result.get(); + + // done with the thread + terminator_thread.join(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/controlflow/scan_test.cc b/onnxruntime/test/providers/cpu/controlflow/scan_test.cc new file mode 100644 index 0000000000000..ca9d2e5189208 --- /dev/null +++ b/onnxruntime/test/providers/cpu/controlflow/scan_test.cc @@ -0,0 +1,665 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "gmock/gmock.h" +#include "core/framework/session_state.h" +#include "test/providers/provider_test_utils.h" +#include "core/session/inference_session.h" + +using namespace ONNX_NAMESPACE; + +namespace onnxruntime { +namespace test { + +struct RunOptions { + bool include_dim_values_in_main_graph = false; + bool include_dim_values_in_subgraph = true; + bool include_types_in_subgraph = true; + bool include_outer_scope_add = false; + bool add_bad_shape = false; +}; + +static void CreateSubgraph(Graph& graph, RunOptions& options, const std::string& failure_message = ""); + +static const float kOuterNodeAddValue = 42.f; + +class ScanOpTester : public OpTester { + public: + ScanOpTester() : OpTester("Scan", 8) { + } + + protected: + void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& add_attribute_funcs) override { + // add outer_scope_0 node + { + TypeProto float_scalar; + float_scalar.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + auto mutable_dim = float_scalar.mutable_tensor_type()->mutable_shape()->add_dim(); + mutable_dim->set_dim_value(1); + + { + auto& output_arg = graph.GetOrCreateNodeArg("outer_scope_0", &float_scalar); + auto* constant = graph.AddNode("outer_scope_constant", "Constant", "Constant with value kOuterNodeAddValue", + {}, {&output_arg}); + + TensorProto value_tensor; + value_tensor.add_dims(1); + value_tensor.add_float_data(kOuterNodeAddValue); + value_tensor.set_data_type(onnx::TensorProto_DataType_FLOAT); + + constant->AddAttribute("value", value_tensor); + } + } + + // call base implementation to add the Scan node as per usual + OpTester::AddNodes(graph, graph_input_defs, graph_output_defs, add_attribute_funcs); + } +}; + +static void CreateSubgraph(Graph& graph, RunOptions& options, const std::string& failure_message) { + bool include_shapes = options.include_dim_values_in_subgraph; + bool include_types = options.include_types_in_subgraph; + + std::vector inputs; + std::vector outputs; + + /* Subgraph looks like this. + + [constant_1] loop_state_in_1 concat_in_0 concat_in_1 + \ | \ / + \--------[Add] [Concat] + | | + | concat_out_1 + | | + | [Add]----------outer_scope_0 < Optional + | | < based on RunOptions + | add_out_1 < + | | < + | [Split] + | / | | \ + loop_state_out_1 split_out_0 ... split_out_3 + */ + + // constant of 1 to add to the loop state on each iteration + { + inputs = {}; + outputs = {}; + + TypeProto float_scalar; + // inputs must have type information and a rank + float_scalar.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + auto mutable_dim = float_scalar.mutable_tensor_type()->mutable_shape()->add_dim(); + if (include_shapes) + mutable_dim->set_dim_value(1); + + { + auto& output_arg = graph.GetOrCreateNodeArg("constant_1", &float_scalar); + outputs.push_back(&output_arg); + + auto* constant = graph.AddNode("constant", "Constant", "Constant with value 1", inputs, outputs); + + TensorProto value_tensor; + value_tensor.add_dims(1); + value_tensor.add_float_data(1.f); + value_tensor.set_data_type(onnx::TensorProto_DataType_FLOAT); + + constant->AddAttribute("value", value_tensor); + } + + inputs = outputs; // start with output from Constant node + outputs = {}; + + auto& input_arg = graph.GetOrCreateNodeArg("loop_state_in_1", &float_scalar); + inputs.push_back(&input_arg); + + TypeProto loop_state_output_tensor; + loop_state_output_tensor.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + + // as this is a subgraph output we need a shape to come from somewhere, so if the main graph isn't providing it, + // it has to come from here. + bool type_and_shape_required = options.include_dim_values_in_main_graph == false; + + if (include_shapes || type_and_shape_required) + loop_state_output_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + + TypeProto* type_proto = include_types || type_and_shape_required ? &loop_state_output_tensor : nullptr; + auto& output_arg = graph.GetOrCreateNodeArg("loop_state_out_1", type_proto); + outputs.push_back(&output_arg); + + auto* add = graph.AddNode("add", "Add", "Add 1 to the loop state", inputs, outputs); + (void)add; + } + + // subgraph with multiple inputs and outputs to test variadic behaviour. + // 2 inputs of 2 that are concatenated and then split into 4 outputs of 1 + + // Concat node + { + inputs = {}; + outputs = {}; + + // input of 2 x {2} tensors + TypeProto concat_input_tensor; + // inputs must have type information and rank, but dimension can have no value if we're not providing shape info. + concat_input_tensor.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + auto mutable_dim = concat_input_tensor.mutable_tensor_type()->mutable_shape()->add_dim(); + if (include_shapes) { + mutable_dim->set_dim_value(2); + + if (options.add_bad_shape) { + concat_input_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(99); + } + } + + for (int i = 0, num_inputs = 2; i < num_inputs; ++i) { + auto& input_arg = graph.GetOrCreateNodeArg("concat_in_" + std::to_string(i), &concat_input_tensor); + inputs.push_back(&input_arg); + } + + // one output from concatenate of {4} tensor + TypeProto concat_output_tensor; + concat_output_tensor.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + if (include_shapes) + concat_output_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(4); + + TypeProto* type_proto = include_types ? &concat_output_tensor : nullptr; + auto& output_arg = graph.GetOrCreateNodeArg("concat_out_1", type_proto); + outputs.push_back(&output_arg); + + auto* concat = graph.AddNode("concat", "Concat", "concat 2 inputs", inputs, outputs); + + concat->AddAttribute("axis", int64_t{0}); + } + + // Post-Concat Add Node + if (options.include_outer_scope_add) { + TypeProto float_scalar; + float_scalar.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + float_scalar.mutable_tensor_type()->mutable_shape(); // create it to make checker happy but don't put any dims in it + + auto& outer_scope_input_arg = graph.GetOrCreateNodeArg("outer_scope_0", &float_scalar); + + // add so that we don't end up with it being considered a graph input + graph.AddOuterScopeNodeArg(outer_scope_input_arg.Name()); + + inputs = outputs; + outputs = {}; + + inputs.push_back(&outer_scope_input_arg); + + TypeProto post_concat_add_output_tensor; // should be same type/shape as concat_out_0 + post_concat_add_output_tensor.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + post_concat_add_output_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(4); + + auto& output_arg = graph.GetOrCreateNodeArg("add_out_1", &post_concat_add_output_tensor); + outputs.push_back(&output_arg); + + graph.AddNode("post_concat_add", "Add", "Add outer scope value to concat output", inputs, outputs); + } + + // Split node + { + // setup Split to run using the Concat output + inputs = outputs; + outputs = {}; + + // split output of 4 x {1} tensors + TypeProto split_output_tensor; + split_output_tensor.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + // the Split shape inferencing assumes that the shape is already set to each value in the + // 'split' attribute. that seems reasonable (if you set that attribute you can set the shape) + // so we must set the shape here. + split_output_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + + for (int i = 0, num_outputs = 4; i < num_outputs; ++i) { + auto& output_arg = graph.GetOrCreateNodeArg("split_out_" + std::to_string(i), &split_output_tensor); + outputs.push_back(&output_arg); + } + + auto* split = graph.AddNode("split", "Split", "split into 4 outputs", inputs, outputs); + split->AddAttribute("axis", int64_t{0}); + split->AddAttribute("split", std::vector{1, 1, 1, 1}); + } + + auto status = graph.Resolve(); + + if (failure_message.empty()) { + EXPECT_EQ(status, Status::OK()); + } else { + EXPECT_TRUE(!status.IsOK()); + EXPECT_THAT(status.ErrorMessage(), testing::HasSubstr(failure_message)); + } +} + +void RunTest(const std::string test_name, int64_t batch_size, int64_t max_sequence_len, int64_t input_size, + std::vector* directions, + std::vector* sequence_lens, + std::vector& loop_state_in_0, + std::vector input_0, + std::vector input_1, + std::vector& loop_state_out_0, + std::vector output_0, + std::vector output_1, + std::vector output_2, + std::vector output_3, + RunOptions options = {}, + OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess, + const std::string& failure_message = "") { + // create model that will be used to initialize subgraph. currently there's no direct way to create a Graph instance. + Model model(test_name); + auto& graph = model.MainGraph(); + CreateSubgraph(graph, options, options.add_bad_shape ? failure_message : ""); + auto& proto = graph.ToGraphProto(); + + ScanOpTester test; + + test.AddShapeToTensorData(options.include_dim_values_in_main_graph); + + test.AddAttribute("body", proto); + test.AddAttribute("num_scan_inputs", 2); + + if (directions != nullptr) { + test.AddAttribute>("directions", *directions); + } + + if (sequence_lens == nullptr) { + test.AddMissingOptionalInput(); + } else { + std::vector sequence_lens_dims{batch_size}; + test.AddInput("sequence_lens", sequence_lens_dims, *sequence_lens); + } + + test.AddInput("scan_loop_state_in_0", {batch_size, 1}, loop_state_in_0); + + std::vector input_shape{batch_size, max_sequence_len, input_size}; + test.AddInput("scan_input_0", input_shape, input_0); + test.AddInput("scan_input_1", input_shape, input_1); + + test.AddOutput("scan_loop_state_out_0", {batch_size, 1}, loop_state_out_0); + + std::vector output_shape{batch_size, max_sequence_len, 1}; + test.AddOutput("scan_output_0", output_shape, output_0); + test.AddOutput("scan_output_1", output_shape, output_1); + test.AddOutput("scan_output_2", output_shape, output_2); + test.AddOutput("scan_output_3", output_shape, output_3); + + test.Run(expect_result, failure_message); +} + +void ShortSequenceOneInBatchOneLoopStateVar(const RunOptions& options, const std::string& expected_error = "") { + const int64_t batch_size = 1; + const int64_t sequence_len = 2; + const int64_t input_size = 2; + + std::vector iteration_count_in{0.f}; + + // batch_size, max_sequence_len, input_size + std::vector input_0{1.f, 2.f, + 4.f, 3.f}; + std::vector input_1{3.f, 4.f, + 2.f, 1.f}; + + std::vector iteration_count_out{2.f}; // iteration_count_in + 1 for each item in sequence + + float output_adjust = options.include_outer_scope_add ? kOuterNodeAddValue : 0.f; + + // batch_size, max_sequence_len, 1 + std::vector output_0{1.f + output_adjust, 4.f + output_adjust}; + std::vector output_1{2.f + output_adjust, 3.f + output_adjust}; + std::vector output_2{3.f + output_adjust, 2.f + output_adjust}; + std::vector output_3{4.f + output_adjust, 1.f + output_adjust}; + + RunTest("ShortSequenceOneInBatchOneLoopStateVar", batch_size, sequence_len, input_size, + nullptr, nullptr, + iteration_count_in, input_0, input_1, + iteration_count_out, output_0, output_1, output_2, output_3, + options, + expected_error.empty() ? OpTester::ExpectResult::kExpectSuccess : OpTester::ExpectResult::kExpectFailure, + expected_error); +} + +TEST(Scan, ShortSequenceOneInBatchOneLoopStateVar_NoShapeInMainGraph_TypeAndShapeInSubgraph) { + RunOptions options{}; + options.include_dim_values_in_main_graph = false; + options.include_types_in_subgraph = true; + options.include_dim_values_in_subgraph = true; + + ShortSequenceOneInBatchOneLoopStateVar(options); +} + +TEST(Scan, ShortSequenceOneInBatchOneLoopStateVar_ShapeInMainGraph_NoTypeAndShapeInSubgraph) { + RunOptions options{}; + options.include_dim_values_in_main_graph = true; + options.include_types_in_subgraph = false; + options.include_dim_values_in_subgraph = false; + + ShortSequenceOneInBatchOneLoopStateVar(options); +} + +TEST(Scan, ShortSequenceOneInBatchOneLoopStateVar_NoShapeInMainGraph_NoTypeAndShapeInSubgraph) { + RunOptions options{}; + options.include_dim_values_in_main_graph = false; + options.include_types_in_subgraph = false; + options.include_dim_values_in_subgraph = false; + + ShortSequenceOneInBatchOneLoopStateVar(options); +} + +// test when there is an operator in the subgraph that uses a value coming from outer scope +TEST(Scan, OuterScopeAccess_NoShapeInMainGraph_TypeAndShapeInSubgraph) { + RunOptions options{}; + options.include_dim_values_in_main_graph = false; + options.include_types_in_subgraph = true; + options.include_dim_values_in_subgraph = true; + + options.include_outer_scope_add = true; + + ShortSequenceOneInBatchOneLoopStateVar(options); +} + +TEST(Scan, OuterScopeAccess_ShapeInMainGraph_NoTypeAndShapeInSubgraph) { + RunOptions options{}; + options.include_dim_values_in_main_graph = true; + options.include_types_in_subgraph = false; + options.include_dim_values_in_subgraph = false; + + options.include_outer_scope_add = true; + + ShortSequenceOneInBatchOneLoopStateVar(options); +} + +TEST(Scan, OuterScopeAccess_NoShapeInMainGraph_NoTypeAndShapeInSubgraph) { + RunOptions options{}; + options.include_dim_values_in_main_graph = false; + options.include_types_in_subgraph = false; + options.include_dim_values_in_subgraph = false; + + options.include_outer_scope_add = true; + + ShortSequenceOneInBatchOneLoopStateVar(options); +} + +TEST(Scan, BadShape) { + RunOptions options{}; + options.include_dim_values_in_main_graph = false; + options.include_types_in_subgraph = true; + options.include_dim_values_in_subgraph = true; + options.add_bad_shape = true; + + ShortSequenceOneInBatchOneLoopStateVar( + options, + "Node:concat Output:concat_out_1 [ShapeInferenceError] Mismatch between number of source and target dimensions. " + "Source=2 Target=1"); +} + +TEST(Scan, ShortSequenceTwoInBatchOneLoopStateVar) { + const int64_t batch_size = 2; + const int64_t sequence_len = 2; + const int64_t input_size = 2; + + std::vector iteration_count_in{0.f, 10.f}; // start at 0 for first item in batch, and 10 for second + + // batch_size, max_sequence_len, input_size + std::vector input_0{1.f, 2.f, + 4.f, 3.f, + + -1.f, -2.f, + -4.f, -3.f}; + + std::vector input_1{3.f, 4.f, + 2.f, 1.f, + + -3.f, -4.f, + -2.f, -1.f}; + + std::vector iteration_count_out{2.f, 12.f}; // iteration_count_in + 1 for each item in sequence + + // batch_size, max_sequence_len, 1 + std::vector output_0{1.f, 4.f, -1.f, -4.f}; + std::vector output_1{2.f, 3.f, -2.f, -3.f}; + std::vector output_2{3.f, 2.f, -3.f, -2.f}; + std::vector output_3{4.f, 1.f, -4.f, -1.f}; + + RunTest("ShortSequenceTwoInBatchOneLoopStateVar", batch_size, sequence_len, input_size, + nullptr, nullptr, + iteration_count_in, input_0, input_1, + iteration_count_out, output_0, output_1, output_2, output_3); +} + +TEST(Scan, MixedSequenceLens) { + const int64_t batch_size = 2; + const int64_t max_sequence_len = 2; + const int64_t input_size = 2; + + std::vector sequence_lens{1, 2}; + + std::vector iteration_count_in{0.f, 10.f}; // start at 0 for first item in batch, and 10 for second + + // batch_size, max_sequence_len, input_size + std::vector input_0{1.f, 2.f, + 4.f, 3.f, // <- this should be ignored + + -1.f, -2.f, + -4.f, -3.f}; + + std::vector input_1{3.f, 4.f, + 2.f, 1.f, // <- this should be ignored + + -3.f, -4.f, + -2.f, -1.f}; + + // iteration_count_in + 1 for each item in sequence. + // as sequence_len is 1 for the first item in the batch, the final value should be 0 + 1. + // as sequence_len is 2 for the second item in the batch, the final value should be 10 + 1 + 1. + std::vector iteration_count_out{1.f, 12.f}; + + // batch_size, max_sequence_len, 1 + // as sequence_len is 1 for the first item in the batch we expect 0.f's for the second value in the output + // (which technically is undefined, but 0.f is consistent with other RNN ops) + std::vector output_0{1.f, 0.f, -1.f, -4.f}; + std::vector output_1{2.f, 0.f, -2.f, -3.f}; + std::vector output_2{3.f, 0.f, -3.f, -2.f}; + std::vector output_3{4.f, 0.f, -4.f, -1.f}; + + RunTest("MixedSequenceLens", batch_size, max_sequence_len, input_size, + nullptr, &sequence_lens, + iteration_count_in, input_0, input_1, + iteration_count_out, output_0, output_1, output_2, output_3); +} + +TEST(Scan, MixedSequenceLensReverse) { + const int64_t batch_size = 2; + const int64_t max_sequence_len = 2; + const int64_t input_size = 2; + + std::vector sequence_lens{1, 2}; + std::vector directions{1, 1}; // reverse both inputs + + std::vector iteration_count_in{0.f, 10.f}; // start at 0 for first item in batch, and 10 for second + + // batch_size, max_sequence_len, input_size + std::vector input_0{1.f, 2.f, + 400.f, 300.f, // <- this should be ignored + + -1.f, -2.f, + -4.f, -3.f}; + + std::vector input_1{3.f, 4.f, + 200.f, 100.f, // <- this should be ignored + + -3.f, -4.f, + -2.f, -1.f}; + + // iteration_count_in + 1 for each item in sequence. + // as sequence_len is 1 for the first item in the batch, the final value should be 0 + 1. + // as sequence_len is 2 for the second item in the batch, the final value should be 10 + 1 + 1. + std::vector iteration_count_out{1.f, 12.f}; + + // batch_size, max_sequence_len, 1 + // as sequence_len is 1 for the first item in the batch we expect 0.f's for the second value in the output + // (which technically is undefined, but 0.f is consistent with other RNN ops) + // as the first sequence only contains one entry, the output should actually be the same as if the direction + // was forward. + std::vector output_0{1.f, 0.f, -4.f, -1.f}; + std::vector output_1{2.f, 0.f, -3.f, -2.f}; + std::vector output_2{3.f, 0.f, -2.f, -3.f}; + std::vector output_3{4.f, 0.f, -1.f, -4.f}; + + RunTest("MixedSequenceLensReverse", batch_size, max_sequence_len, input_size, + &directions, &sequence_lens, + iteration_count_in, input_0, input_1, + iteration_count_out, output_0, output_1, output_2, output_3); +} + +TEST(Scan, ShortSequenceTwoInBatchOneLoopStateVarReverseFirstInput) { + const int64_t batch_size = 2; + const int64_t sequence_len = 2; + const int64_t input_size = 2; + + std::vector iteration_count_in{0.f, 10.f}; // start at 0 for first item in batch, and 10 for second + + std::vector directions{1, 0}; // reverse for input_0, forward for input_1 + + // batch_size, max_sequence_len, input_size + std::vector input_0{1.f, 2.f, + 4.f, 3.f, + + -1.f, -2.f, + -4.f, -3.f}; + + std::vector input_1{3.f, 4.f, + 2.f, 1.f, + + -3.f, -4.f, + -2.f, -1.f}; + + std::vector iteration_count_out{2.f, 12.f}; // iteration_count_in + 1 for each item in sequence + + // batch_size, max_sequence_len, 1 + // the sequence of input0 is reversed, so the subgraph will get {4.f, 3.f} then {1.f, 2.f} for batch 0 + // and {-4.f, -3.f} then {-1.f, -2.f} for batch 0. + std::vector output_0{4.f, 1.f, -4.f, -1.f}; + std::vector output_1{3.f, 2.f, -3.f, -2.f}; + std::vector output_2{3.f, 2.f, -3.f, -2.f}; + std::vector output_3{4.f, 1.f, -4.f, -1.f}; + + RunTest("ShortSequenceTwoInBatchOneLoopStateVarReverseFirstInput", batch_size, sequence_len, input_size, + &directions, nullptr, + iteration_count_in, input_0, input_1, + iteration_count_out, output_0, output_1, output_2, output_3); +} + +TEST(Scan, InvalidInput) { + const int64_t batch_size = 1; + const int64_t sequence_len = 1; + const int64_t input_size = 2; + + std::vector iteration_count_in{0.f}; + + // batch_size, max_sequence_len, input_size + std::vector input_0{1.f, 2.f}; + std::vector input_1{3.f, 4.f}; + + std::vector iteration_count_out{1.f}; + + // batch_size, max_sequence_len, 1 + std::vector output_0{1.f}; + std::vector output_1{2.f}; + std::vector output_2{3.f}; + std::vector output_3{4.f}; + + // invalid direction value - only 0 or 1 are valid + std::vector directions = {2, 1}; + + RunTest("InvalidDirectionsValue", batch_size, sequence_len, input_size, + &directions, nullptr, + iteration_count_in, input_0, input_1, + iteration_count_out, output_0, output_1, output_2, output_3, + {}, + OpTester::ExpectResult::kExpectFailure, + "Invalid values in 'directions'."); + + // mismatch between direction entries and num inputs + directions = {1, 0, 1}; // too many entries - should match the number of inputs (2) + + RunTest("InvalidNumEntriesInDirections", batch_size, sequence_len, input_size, + &directions, nullptr, + iteration_count_in, input_0, input_1, + iteration_count_out, output_0, output_1, output_2, output_3, + {}, + OpTester::ExpectResult::kExpectFailure, + "Number of entries in 'directions' was 3. Must match 'num_scan_inputs' of 2"); +} + +// Test usage of multiple inputs of different types for variadic inputs +TEST(Scan, MixedTypeInputs) { + // Construct scan body subgraph with 2 state variables, 2 scan inputs, 2 scan outputs + // of different types (1 float and 1 int64 of each): + // state-in-1 => scan-out-1 + // scan-in-1 => state-out-1 + // state-in-2 => scan-out-2 + // scan-in-2 => state-out-2 + + Model model("ScanBody"); + auto& graph = model.MainGraph(); + + TypeProto float_tensor; + float_tensor.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + TypeProto int_tensor; + int_tensor.mutable_tensor_type()->set_elem_type(TensorProto_DataType_INT64); + int_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + + auto& state_in_1 = graph.GetOrCreateNodeArg("state_in_1", &float_tensor); + auto& state_in_2 = graph.GetOrCreateNodeArg("state_in_2", &int_tensor); + auto& scan_in_1 = graph.GetOrCreateNodeArg("scan_in_1", &float_tensor); + auto& scan_in_2 = graph.GetOrCreateNodeArg("scan_in_2", &int_tensor); + + auto& state_out_1 = graph.GetOrCreateNodeArg("state_out_1", &float_tensor); + auto& state_out_2 = graph.GetOrCreateNodeArg("state_out_2", &int_tensor); + auto& scan_out_1 = graph.GetOrCreateNodeArg("scan_out_1", &float_tensor); + auto& scan_out_2 = graph.GetOrCreateNodeArg("scan_out_2", &int_tensor); + + graph.AddNode("node1", "Identity", "Copy state_in_1 to scan_out_1", {&state_in_1}, {&scan_out_1}); + graph.AddNode("node2", "Identity", "Copy state_in_2 to scan_out_2", {&state_in_2}, {&scan_out_2}); + graph.AddNode("node3", "Identity", "Copy scan_in_1 to state_out_1", {&scan_in_1}, {&state_out_1}); + graph.AddNode("node4", "Identity", "Copy scan_in_2 to state_out_2", {&scan_in_2}, {&state_out_2}); + + graph.SetInputOrder({&state_in_1, &state_in_2, &scan_in_1, &scan_in_2}); + graph.SetOutputOrder({&state_out_1, &state_out_2, &scan_out_1, &scan_out_2}); + + auto status = graph.Resolve(); + EXPECT_EQ(status, Status::OK()); + + auto& scan_body = graph.ToGraphProto(); + + // Construct and run scan test + ScanOpTester test; + + int64_t batch_size = 1, sequence_len = 3, input_size = 1; + std::vector seq_shape{batch_size, sequence_len, input_size}; + std::vector state_shape{batch_size, input_size}; + + test.AddAttribute("body", scan_body); + test.AddAttribute("num_scan_inputs", 2); + + test.AddMissingOptionalInput(); + test.AddInput("initial_state_1", state_shape, {0.0}); + test.AddInput("initial_state_2", state_shape, {0}); + test.AddInput("scan_input_1", seq_shape, {1.0, 2.0, 3.0}); + test.AddInput("scan_input_2", seq_shape, {1, 2, 3}); + + test.AddOutput("final_state_1", state_shape, {3.0}); + test.AddOutput("final_state_2", state_shape, {3}); + test.AddOutput("scan_output_1", seq_shape, {0.0, 1.0, 2.0}); + test.AddOutput("scan_output_2", seq_shape, {0, 1, 2}); + + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/cpu_execution_provider_test.cc b/onnxruntime/test/providers/cpu/cpu_execution_provider_test.cc new file mode 100644 index 0000000000000..0f941a10454be --- /dev/null +++ b/onnxruntime/test/providers/cpu/cpu_execution_provider_test.cc @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/cpu_execution_provider.h" +#include "gtest/gtest.h" + +namespace onnxruntime { +namespace test { +TEST(CPUExecutionProviderTest, MetadataTest) { + CPUExecutionProviderInfo info; + auto provider = std::make_unique(info); + EXPECT_TRUE(provider != nullptr); + ASSERT_STREQ(provider->GetAllocator(0, ONNXRuntimeMemTypeDefault)->Info().name, CPU); +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/generator/constant_test.cc b/onnxruntime/test/providers/cpu/generator/constant_test.cc new file mode 100644 index 0000000000000..addc2c1bf2020 --- /dev/null +++ b/onnxruntime/test/providers/cpu/generator/constant_test.cc @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +using namespace ONNX_NAMESPACE; +namespace onnxruntime { +namespace test { + +TEST(ConstantLike, ConstantLike_with_input) { + OpTester test("ConstantLike", 9); + + std::vector dims{4, 3, 2}; + + test.AddInput("X", dims, + {0, 1, 2, 3, + 4, 5, 7, 8, + 0, 1, 2, 3, + 4, 5, 7, 8, + 0, 1, 2, 3, + 4, 5, 7, 8}); + + test.AddAttribute("value", 1.0f); + + std::vector expected_output( + {1, 1, 1, 1, + 1, 1, 1, 1, + 1, 1, 1, 1, + 1, 1, 1, 1, + 1, 1, 1, 1, + 1, 1, 1, 1}); + + test.AddOutput("Y", dims, expected_output); + test.Run(); +} + +TEST(ConstantLike, ConstantLike_without_input) { + OpTester test("ConstantLike", 9); + + std::vector dims{4, 3, 2}; + test.AddAttribute("shape", dims); + test.AddAttribute("value", 2.0f); + + std::vector expected_output( + {2.0f, 2.0f, 2.0f, 2.0f, + 2.0f, 2.0f, 2.0f, 2.0f, + 2.0f, 2.0f, 2.0f, 2.0f, + 2.0f, 2.0f, 2.0f, 2.0f, + 2.0f, 2.0f, 2.0f, 2.0f, + 2.0f, 2.0f, 2.0f, 2.0f}); + + test.AddOutput("Y", dims, expected_output); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/generator/random_test.cc b/onnxruntime/test/providers/cpu/generator/random_test.cc new file mode 100644 index 0000000000000..350efbab230de --- /dev/null +++ b/onnxruntime/test/providers/cpu/generator/random_test.cc @@ -0,0 +1,311 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +#include +#include +using namespace ONNX_NAMESPACE; +namespace onnxruntime { +namespace test { + +TEST(Random, RandomNormal2DDouble) { + OpTester test("RandomNormal"); + + std::vector dims{20, 50}; + + float scale = 10.f; + float mean = 0.f; + float seed = 123.f; + + test.AddAttribute("scale", scale); + test.AddAttribute("mean", mean); + test.AddAttribute("seed", seed); + test.AddAttribute("dtype", TensorProto::DOUBLE); + test.AddAttribute("shape", dims); + + std::default_random_engine generator{gsl::narrow_cast(seed)}; + std::normal_distribution distribution{mean, scale}; + + std::vector expected_output(TensorShape(dims).Size()); + std::for_each(expected_output.begin(), expected_output.end(), + [&generator, &distribution](double& value) { value = distribution(generator); }); + + test.AddOutput("Y", dims, expected_output); + test.Run(); +} + +void RunRandomNormalLike3DFloat(bool infer_dtype = false) { + OpTester test("RandomNormalLike"); + + std::vector dims{2, 2, 3}; + + float scale = 10.f; + float mean = 0.f; + float seed = 123.f; + + test.AddAttribute("scale", scale); + test.AddAttribute("mean", mean); + test.AddAttribute("seed", seed); + + if (!infer_dtype) + test.AddAttribute("dtype", TensorProto::FLOAT); + + test.AddInput("X", dims, + {0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, + + 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f}); + + std::default_random_engine generator{gsl::narrow_cast(seed)}; + std::normal_distribution distribution{mean, scale}; + + std::vector expected_output(TensorShape(dims).Size()); + std::for_each(expected_output.begin(), expected_output.end(), + [&generator, &distribution](float& value) { value = distribution(generator); }); + + test.AddOutput("Y", dims, expected_output); + + test.Run(); +} + +TEST(Random, RandomNormalLike3DDouble) { + RunRandomNormalLike3DFloat(); +} + +TEST(Random, RandomNormalLikeInferDType) { + const bool infer_dtype = true; + RunRandomNormalLike3DFloat(infer_dtype); +} + +TEST(Random, RandomUniform1DFloat) { + OpTester test("RandomUniform"); + + std::vector dims{10}; + + float low = 0.f; + float high = 100.f; + float seed = 123.f; + + test.AddAttribute("low", low); + test.AddAttribute("high", high); + test.AddAttribute("seed", seed); + test.AddAttribute("dtype", TensorProto::FLOAT); + test.AddAttribute("shape", dims); + + std::default_random_engine generator{gsl::narrow_cast(seed)}; + std::uniform_real_distribution distribution{low, high}; + + std::vector expected_output(TensorShape(dims).Size()); + std::for_each(expected_output.begin(), expected_output.end(), + [&generator, &distribution](float& value) { value = distribution(generator); }); + + test.AddOutput("Y", dims, expected_output); + + test.Run(); +} + +void RunRandomUniformLikeTest(bool infer_dtype = false) { + OpTester test("RandomUniformLike"); + + std::vector dims{2, 6}; + + float low = 0.f; + float high = 100.f; + float seed = 123.f; + + test.AddAttribute("low", low); + test.AddAttribute("high", high); + test.AddAttribute("seed", seed); + + if (!infer_dtype) + test.AddAttribute("dtype", TensorProto::DOUBLE); + + test.AddInput("X", dims, + {0., 0., 0., 0., 0., 0., + 0., 0., 0., 0., 0., 0.}); + + std::default_random_engine generator{gsl::narrow_cast(seed)}; + std::uniform_real_distribution distribution{low, high}; + + std::vector expected_output(TensorShape(dims).Size()); + std::for_each(expected_output.begin(), expected_output.end(), + [&generator, &distribution](double& value) { value = distribution(generator); }); + + test.AddOutput("Y", dims, expected_output); + + test.Run(); +} + +TEST(Random, RandomUniformLike2DDouble) { + RunRandomUniformLikeTest(); +} + +TEST(Random, RandomUniformLikeInferDType) { + const bool infer_dtype = true; + RunRandomUniformLikeTest(infer_dtype); +} + +TEST(Random, InvalidDType) { + float seed = 123.f; + + std::vector dims{1, 4}; + std::vector input{0, 0, 0, 0}; + std::vector expected_output{0., 0., 0., 0.}; + + { + OpTester test("RandomNormal"); + + float scale = 10.f; + float mean = 0.f; + + test.AddAttribute("scale", scale); + test.AddAttribute("mean", mean); + test.AddAttribute("seed", seed); + test.AddAttribute("dtype", 999); + test.AddAttribute("shape", dims); + + test.AddOutput("Y", dims, expected_output); + test.Run(OpTester::ExpectResult::kExpectFailure, "Attribute dtype does not specify a valid type."); + } + + { + OpTester test("RandomUniform"); + + float low = 0.f; + float high = 100.f; + + test.AddAttribute("low", low); + test.AddAttribute("high", high); + test.AddAttribute("seed", seed); + test.AddAttribute("dtype", 999); + test.AddAttribute("shape", dims); + + test.AddOutput("Y", dims, expected_output); + test.Run(OpTester::ExpectResult::kExpectFailure, "Attribute dtype does not specify a valid type."); + } + + { + OpTester test("RandomNormalLike"); + + float scale = 10.f; + float mean = 0.f; + + test.AddAttribute("scale", scale); + test.AddAttribute("mean", mean); + test.AddAttribute("seed", seed); + test.AddAttribute("dtype", 999); + + test.AddInput("X", dims, input); + test.AddOutput("Y", dims, expected_output); + test.Run(OpTester::ExpectResult::kExpectFailure, "Attribute dtype does not specify a valid type."); + } + + { + OpTester test("RandomUniformLike"); + + float low = 0.f; + float high = 100.f; + + test.AddAttribute("low", low); + test.AddAttribute("high", high); + test.AddAttribute("seed", seed); + test.AddAttribute("dtype", 999); + + test.AddInput("X", dims, input); + test.AddOutput("Y", dims, expected_output); + test.Run(OpTester::ExpectResult::kExpectFailure, "Attribute dtype does not specify a valid type."); + } +} + +/* +Note: There are no reference tests that can be reused in this case. I tried to use the tensorflow +test cases but they use a different RNG (Philox) and hence the test results differ. Since the implementation +of the op is same as tensorflow, for now I've just relied on the output generated by this code as ground truth +for verification. +*/ +TEST(Random, MultinomialGoodCase) { + OpTester test("Multinomial"); + + const int64_t num_samples = 10; + const float seed = 1618.f; + const int batch_size = 2; + const int num_classes = 3; + + const std::vector input_dims{batch_size, num_classes}; + std::vector input(TensorShape(input_dims).Size()); + std::fill(input.begin(), input.end(), -10.f); + test.AddInput("X", input_dims, input); + + test.AddAttribute("sample_size", num_samples); + test.AddAttribute("seed", seed); + test.AddAttribute("dtype", TensorProto::INT64); + + const std::vector output_dims{batch_size, num_samples}; +#ifdef _WIN32 + const std::vector expected_output{2, 0, 0, 2, 2, 2, 0, 2, 2, 1, 1, 2, 1, 1, 1, 1, 2, 1, 2, 0}; +#elif defined(__MACH__) + const std::vector expected_output{1, 1, 2, 2, 0, 2, 2, 2, 0, 2, 1, 1, 2, 0, 2, 2, 0, 2, 1, 1}; +#else + const std::vector expected_output{2, 0, 0, 1, 0, 1, 2, 0, 1, 0, 0, 1, 1, 0, 1, 0, 2, 0, 2, 0}; +#endif + test.AddOutput("Y", output_dims, expected_output); + + test.Run(); +} + +TEST(Random, MultinomialDefaultDType) { + OpTester test("Multinomial"); + + const int64_t num_samples = 10; + const int batch_size = 2; + const float seed = 1618.f; + + const std::vector input_dims{2, 3}; + std::vector input(TensorShape(input_dims).Size()); + std::fill(input.begin(), input.end(), -10.f); + test.AddInput("X", input_dims, input); + + test.AddAttribute("sample_size", num_samples); + test.AddAttribute("seed", seed); + + const std::vector output_dims{batch_size, num_samples}; +#ifdef _WIN32 + const std::vector expected_output{2, 0, 0, 2, 2, 2, 0, 2, 2, 1, 1, 2, 1, 1, 1, 1, 2, 1, 2, 0}; +#elif defined(__MACH__) + const std::vector expected_output{1, 1, 2, 2, 0, 2, 2, 2, 0, 2, 1, 1, 2, 0, 2, 2, 0, 2, 1, 1}; +#else + const std::vector expected_output{2, 0, 0, 1, 0, 1, 2, 0, 1, 0, 0, 1, 1, 0, 1, 0, 2, 0, 2, 0}; +#endif + test.AddOutput("Y", output_dims, expected_output); + + test.Run(); +} + +TEST(Random, MultinomialInvalidDtype) { + OpTester test("Multinomial"); + + const int64_t num_samples = 10; + const int batch_size = 2; + const int num_classes = 3; + const float seed = 1618.f; + + const std::vector input_dims{batch_size, num_classes}; + std::vector input(TensorShape(input_dims).Size()); + std::fill(input.begin(), input.end(), -10.f); + test.AddInput("X", input_dims, input); + + test.AddAttribute("sample_size", num_samples); + test.AddAttribute("seed", seed); + test.AddAttribute("dtype", 999); + + const std::vector output_dims{batch_size, num_samples}; + const std::vector expected_output{2, 0, 0, 2, 2, 2, 0, 2, 2, 1, 1, 2, 1, 1, 1, 1, 2, 1, 2, 0}; + test.AddOutput("Y", output_dims, expected_output); + + test.Run(OpTester::ExpectResult::kExpectFailure, "Output type must be int32 or int64"); +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/math/clip_test.cc b/onnxruntime/test/providers/cpu/math/clip_test.cc new file mode 100644 index 0000000000000..7a80f6a9835be --- /dev/null +++ b/onnxruntime/test/providers/cpu/math/clip_test.cc @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(MathOpTest, Clip) { + OpTester test("Clip"); + + test.AddAttribute("min", -10.0f); + test.AddAttribute("max", 10.0f); + + std::vector dims{3, 3}; + test.AddInput("X", dims, + {11.0f, 4.4f, 432.3f, + -1.3f, 3.5f, 64.0f, + -5.4f, 9.3f, 82.4f}); + test.AddOutput("Y", dims, + {10.0f, 4.4f, 10.0f, + -1.3f, 3.5f, 10.0f, + -5.4f, 9.3f, 10.0f}); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc b/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc new file mode 100644 index 0000000000000..e6eaf6c99e782 --- /dev/null +++ b/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc @@ -0,0 +1,874 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(MathOpTest, Add_int32) { + OpTester test("Add"); + test.AddInput("A", {3}, {1, 2, 3}); + test.AddInput("B", {3}, {4, 5, 6}); + test.AddOutput("C", {3}, {5, 7, 9}); + test.Run(); +} + +TEST(MathOpTest, Add_int64) { + OpTester test("Add"); + test.AddInput("A", {3}, {1, 2, 3}); + test.AddInput("B", {3}, {4, 5, 6}); + test.AddOutput("C", {3}, {5, 7, 9}); + test.Run(); +} + +TEST(MathOpTest, Add) { + OpTester test("Add"); + std::vector dims{3, 3}; + test.AddInput("A", dims, + {1.0f, 2.0f, -1.0f, + 0.0f, 1.5f, -100.0f, + -5.4f, 9.3f, -10'000.0f}); + test.AddInput("B", dims, + {-1.0f, 4.4f, 432.3f, + 0.0f, 3.5f, 64.0f, + -5.4f, 9.3f, 10'000.0f}); + test.AddOutput("C", dims, + {0.0f, 6.4f, 431.3f, + 0.0f, 5.0f, -36.0f, + -10.8f, 18.6f, 0.0f}); + test.Run(); +} + +TEST(MathOpTest, Add_Broadcast_Axis) { + OpTester test("Add"); + + std::vector dims{3, 3}; + test.AddInput("A", dims, + {1.0f, 2.0f, 3.0f, + 4.0f, 5.0f, 6.0f, + 7.0f, 8.0f, 9.0f}); + test.AddInput("B", {3, 1}, + {3.0f, + 2.0f, + 1.0f}); + test.AddOutput("C", dims, + {4.0f, 5.0f, 6.0f, + 6.0f, 7.0f, 8.0f, + 8.0f, 9.0f, 10.0f}); + test.Run(); +} + +TEST(MathOpTest, Add_Broadcast_0x0) { + OpTester test("Add"); + + test.AddInput("A", {}, {10.0f}); + test.AddInput("B", {}, {2.0f}); + test.AddOutput("C", {}, {12.0f}); + test.Run(); +} + +TEST(MathOpTest, Add_Broadcast_0x1) { + OpTester test("Add"); + + test.AddInput("A", {}, {10.0f}); + test.AddInput("B", {1}, {2.0f}); + test.AddOutput("C", {1}, {12.0f}); + test.Run(); +} + +TEST(MathOpTest, Add_Broadcast_1x0) { + OpTester test("Add"); + + test.AddInput("A", {1}, {10.0f}); + test.AddInput("B", {}, {2.0f}); + test.AddOutput("C", {1}, {12.0f}); + test.Run(); +} + +TEST(MathOpTest, Add_Broadcast_1x1) { + OpTester test("Add"); + + test.AddInput("A", {1}, {10.0f}); + test.AddInput("B", {1}, {2.0f}); + test.AddOutput("C", {1}, {12.0f}); + test.Run(); +} + +TEST(MathOpTest, Add_Broadcast_3x2_3x1) { + OpTester test("Add"); + + std::vector dims{3, 2}; + test.AddInput("A", dims, + {1.0f, 2.0f, + 3.0f, 4.0f, + 5.0f, 6.0f}); + test.AddInput("B", {3, 1}, + {1.0f, + 2.0f, + 3.0f}); + test.AddOutput("C", dims, + {2.0f, 3.0f, + 5.0f, 6.0f, + 8.0f, 9.0f}); + test.Run(); +} + +TEST(MathOpTest, Add_Broadcast_2x1x4_1x3x1) { + OpTester test("Add"); + + test.AddInput("A", {2, 1, 4}, + {101.0f, 102.0f, 103.0f, 104.0f, + 201.0f, 202.0f, 203.0f, 204.0f}); + test.AddInput("B", {1, 3, 1}, + {010.0f, 020.0f, 030.0f}); + test.AddOutput("C", {2, 3, 4}, + {111.0f, 112.0f, 113.0f, 114.0f, + 121.0f, 122.0f, 123.0f, 124.0f, + 131.0f, 132.0f, 133.0f, 134.0f, + + 211.0f, 212.0f, 213.0f, 214.0f, + 221.0f, 222.0f, 223.0f, 224.0f, + 231.0f, 232.0f, 233.0f, 234.0f}); + test.Run(); +} + +TEST(MathOpTest, Add_Broadcast_2x1x1_3x4) { + OpTester test("Add"); + + test.AddInput("A", {2, 1, 1}, + {100.0f, 200.0f}); + test.AddInput("B", {3, 4}, + {011.0f, 012.0f, 013.0f, 014.0f, + 021.0f, 022.0f, 023.0f, 024.0f, + 031.0f, 032.0f, 033.0f, 034.0f}); + test.AddOutput("C", {2, 3, 4}, + {111.0f, 112.0f, 113.0f, 114.0f, + 121.0f, 122.0f, 123.0f, 124.0f, + 131.0f, 132.0f, 133.0f, 134.0f, + + 211.0f, 212.0f, 213.0f, 214.0f, + 221.0f, 222.0f, 223.0f, 224.0f, + 231.0f, 232.0f, 233.0f, 234.0f}); + test.Run(); +} + +TEST(MathOpTest, Sub_int32) { + OpTester test("Sub"); + test.AddInput("A", {3}, {1, 4, 3}); + test.AddInput("B", {3}, {4, 2, 4}); + test.AddOutput("C", {3}, {-3, 2, -1}); + test.Run(); +} + +TEST(MathOpTest, Sub_int64) { + OpTester test("Sub"); + test.AddInput("A", { 3 }, { 1, 5, 6 }); + test.AddInput("B", { 3 }, { 4, 5, 3 }); + test.AddOutput("C", { 3 }, { -3, 0, 3 }); + test.Run(); +} + +TEST(MathOpTest, Sub) { + OpTester test("Sub"); + std::vector dims{3, 3}; + test.AddInput("A", dims, + {1.0f, 2.0f, -1.0f, + 0.0f, 1.5f, -100.0f, + -5.4f, 9.3f, -10'000.0f}); + test.AddInput("B", dims, + {-1.0f, 4.4f, 432.3f, + 0.0f, 3.5f, 64.0f, + -5.4f, 9.3f, 10'000.0f}); + test.AddOutput("C", dims, + {2.0f, -2.4f, -433.3f, + 0.0f, -2.0f, -164.0f, + 0.0f, 0.0f, -20'000.0f}); + test.Run(); +} + +TEST(MathOpTest, Sub_Broadcast_Scalar) { + OpTester test("Sub"); + std::vector dims{3, 3}; + test.AddInput("A", dims, + {1.0f, 2.0f, -1.0f, + 0.0f, 1.5f, -100.0f, + -5.4f, 9.3f, -10'000.0f}); + test.AddInput("B", {}, {5.0f}); + test.AddOutput("C", dims, + {-4.0f, -3.0f, -6.0f, + -5.0f, -3.5f, -105.0f, + -10.4f, 4.3f, -10'005.0f}); + test.Run(); +} + +TEST(MathOpTest, Mul_int32) { + OpTester test("Mul"); + test.AddInput("A", {3}, {1, 2, 3}); + test.AddInput("B", {3}, {4, -3, 6}); + test.AddOutput("C", {3}, {4, -6, 18}); + test.Run(); +} + +TEST(MathOpTest, Mul_int64) { + OpTester test("Mul"); + test.AddInput("A", { 3 }, { 3, 6, -3 }); + test.AddInput("B", { 3 }, { 4, -3, -2 }); + test.AddOutput("C", { 3 }, { 12, -18, 6 }); + test.Run(); +} + +TEST(MathOpTest, Mul) { + OpTester test("Mul"); + std::vector dims{3, 3}; + test.AddInput("A", dims, + {1.0f, 2.0f, -1.0f, + 0.0f, 1.5f, -100.0f, -5.4f, + 9.30f, -10'000.0f}); + test.AddInput("B", dims, + {-1.0f, 4.4f, 432.3f, + 0.0f, 3.5f, 64.0f, -5.4f, + 9.30f, 10'000.0f}); + test.AddOutput("C", dims, + {-1.0f, 8.8f, -432.3f, + 0.0f, 5.25f, -6'400.0f, + 29.16f, 86.49f, -100'000'000.0f}); + test.Run(); +} + +TEST(MathOpTest, Div_int32) { + OpTester test("Div"); + test.AddInput("A", {3}, {4, 8, 8}); + test.AddInput("B", {3}, {1, 3, 2}); + test.AddOutput("C", {3}, {4, 2, 4}); + test.Run(); +} + +TEST(MathOpTest, Div_int64) { + OpTester test("Div"); + test.AddInput("A", { 3 }, { 4, 8, 8 }); + test.AddInput("B", { 3 }, { 2, 3, 4 }); + test.AddOutput("C", { 3 }, { 2, 2, 2 }); + test.Run(); +} + +TEST(MathOpTest, Div) { + OpTester test("Div"); + std::vector dims{2, 3}; + test.AddInput("A", dims, + {1'000.0f, 1.0f, 6.0f, + 0.0f, -10.0f, -1.0f}); + test.AddInput("B", dims, + {1'000.0f, 2.0f, 3.0f, + 1.0f, -1.0f, 4.0f}); + test.AddOutput("C", dims, + {1.0f, 0.5f, 2.0f, + 0.0f, 10.0f, -0.25f}); + test.Run(); +} + +TEST(MathOpTest, Abs) { + OpTester test("Abs"); + std::vector dims{2, 2}; + test.AddInput("X", dims, {1.0f, -2.0f, -0.0f, -10.0f}); + test.AddOutput("Y", dims, {1.0f, 2.0f, 0.0f, 10.0f}); + test.Run(); +} + +TEST(MathOpTest, Abs_int8) { + OpTester test("Abs"); + std::vector dims{4}; + test.AddInput("X", dims, {1, 2, -1, -5}); + test.AddOutput("Y", dims, {1, 2, 1, 5}); + test.Run(); +} + +TEST(MathOpTest, Abs_int32) { + OpTester test("Abs"); + std::vector dims{4}; + test.AddInput("X", dims, {1, 2, -1, -5}); + test.AddOutput("Y", dims, {1, 2, 1, 5}); + test.Run(); +} + +TEST(MathOpTest, Neg) { + OpTester test("Neg"); + std::vector dims{2, 2}; + test.AddInput("X", dims, + {1.0f, -2.0f, + 0.0f, -10.0f}); + test.AddOutput("Y", dims, + {-1.0f, 2.0f, + -0.0f, 10.0f}); + test.Run(); +} + +TEST(MathOpTest, Neg_int8) { + OpTester test("Neg"); + std::vector dims{4}; + test.AddInput("X", dims, {1, -2, 0, -10}); + test.AddOutput("Y", dims, {-1, 2, 0, 10}); + test.Run(); +} + +TEST(MathOpTest, Neg_int32) { + OpTester test("Neg"); + std::vector dims{4}; + test.AddInput("X", dims, {1, -2, 0, -10}); + test.AddOutput("Y", dims, {-1, 2, 0, 10}); + test.Run(); +} + +TEST(MathOpTest, Floor) { + OpTester test("Floor"); + std::vector dims{2, 2}; + test.AddInput("X", dims, + {-1.5f, 0.2f, + -0.5f, 10.3f}); + test.AddOutput("Y", dims, + {-2.0f, 0.0f, + -1.0f, 10.0f}); + test.Run(); +} + +TEST(MathOpTest, Ceil) { + OpTester test("Ceil"); + std::vector dims{2, 2}; + test.AddInput("X", dims, + {-1.5f, 0.2f, + -0.5f, 10.3f}); + test.AddOutput("Y", dims, + {-1.0f, 1.0f, + 0.0f, 11.0f}); + test.Run(); +} + +TEST(MathOpTest, Reciprocal) { + OpTester test("Reciprocal"); + std::vector dims{2, 2}; + test.AddInput("X", dims, + {1.0f, 2.0f, + -1.0f, -2.0f}); + test.AddOutput("Y", dims, + {1.0f, 0.5f, + -1.0f, -0.5f}); + test.Run(); +} + +TEST(MathOpTest, Sqrt) { + OpTester test("Sqrt"); + std::vector dims{2, 2}; + test.AddInput("X", dims, + {1.0f, 4.0f, + 0.0f, 9.0f}); + test.AddOutput("Y", dims, + {1.0f, 2.0f, + 0.0f, 3.0f}); + test.Run(); +} + +TEST(MathOpTest, Pow) { + OpTester test("Pow"); + std::vector dims{2, 2}; + test.AddInput("X", dims, + {2.0f, 2.0f, + sqrt(2.0f), 1.0f}); + test.AddInput("Y", dims, + {0.0f, 8.0f, + 2.0f, 9.0f}); + test.AddOutput("Z", dims, + {1.0f, 256.0f, + 2.0f, 1.0f}); + test.Run(); +} + +TEST(MathOpTest, Pow_Broadcast_Scalar0) { + OpTester test("Pow"); + + std::vector dims{3}; + test.AddInput("X", {}, {2.0f}); + test.AddInput("Y", dims, {1.0f, 2.0f, 3.0f}); + test.AddOutput("Z", dims, {2.0f, 4.0f, 8.0f}); + test.Run(); +} + +TEST(MathOpTest, Pow_Broadcast_Scalar1) { + OpTester test("Pow"); + + std::vector dims{3}; + test.AddInput("X", dims, {1.0f, 2.0f, 3.0f}); + test.AddInput("Y", {}, {2.0f}); + test.AddOutput("Z", dims, {1.0f, 4.0f, 9.0f}); + test.Run(); +} + +TEST(MathOpTest, Exp) { + OpTester test("Exp"); + std::vector dims{2, 2}; + test.AddInput("X", dims, + {0.0f, 1.0f, + 2.0f, 10.0f}); + test.AddOutput("Y", dims, + {1.0f, exp(1.0f), + exp(2.0f), exp(10.0f)}); + test.SetOutputRelErr("Y", 1e-7f); + test.Run(); +} + +TEST(MathOpTest, Log) { + OpTester test("Log"); + std::vector dims{2, 2}; + test.AddInput("X", dims, + {1.0f, 2.0f, + 5.0f, 10.0f}); + test.AddOutput("Y", dims, + {0.0f, log(2.0f), + log(5.0f), log(10.0f)}); + test.Run(); +} + +TEST(MathOpTest, Sum_6) { + OpTester test("Sum", 6); + std::vector dims{3, 3}; + test.AddInput("data_0", dims, + {1.0f, 0.0f, 1.0f, + -1.0f, 1.1f, -100.0f, + -5.4f, 0.01f, -10'000.0f}); + test.AddInput("data_1", dims, + {1.0f, 0.0f, 2.0f, + -2.0f, 2.2f, 64.0f, + -1.0f, 0.02f, 0.25f}); + test.AddInput("data_3", dims, + {1.0f, 0.0f, 3.0f, + -3.0f, 3.3f, 64.0f, + 5.4f, 0.03f, 10'000.0f}); + test.AddOutput("sum", dims, + {3.0f, 0.0f, 6.0f, + -6.0f, 6.6f, 28.0f, + -1.0f, 0.06f, 0.25f}); + test.Run(); +} + +TEST(MathOpTest, Sum_8_Test1) { + OpTester test("Sum", 8); + test.AddInput("data_0", {3}, {1.0f, 2.0f, 3.0f}); + test.AddInput("data_1", {3, 1}, {10.0f, 20.0f, 30.0f}); + test.AddInput("data_2", {3, 1, 1}, {100.0f, 200.0f, 300.0f}); + test.AddOutput("sum", {3, 3, 3}, + {111.0f, 112.0f, 113.0f, + 121.0f, 122.0f, 123.0f, + 131.0f, 132.0f, 133.0f, + + 211.0f, 212.0f, 213.0f, + 221.0f, 222.0f, 223.0f, + 231.0f, 232.0f, 233.0f, + + 311.0f, 312.0f, 313.0f, + 321.0f, 322.0f, 323.0f, + 331.0f, 332.0f, 333.0f}); + test.Run(); +} + +TEST(MathOpTest, Sum_8_Test2) { + OpTester test("Sum", 8); + std::vector dims{3, 3}; + test.AddInput("data_0", dims, + { + 1.0f, + 0.0f, + 1.0f, + -1.0f, + 1.1f, + -100.0f, + -5.4f, + 0.01f, + -74.0f, + }); + std::vector dims_1{3}; + test.AddInput("data_1", dims_1, + {1.0f, 0.0f, 2.0f}); + std::vector dims_2{3, 1}; + test.AddInput("data_2", dims_2, + {-3.0f, 3.3f, 64.0f}); + test.AddOutput("sum", dims, + {-1.0f, -3.0f, 0.0f, + 3.3f, 4.4f, -94.7f, + 59.6f, 64.01f, -8.0f}); + + test.Run(OpTester::ExpectResult::kExpectSuccess, "Sum is not correct"); +} + +TEST(MathOpTest, Min_6) { + OpTester test("Min", 6); + std::vector dims{3, 3}; + test.AddInput("data_0", dims, + {1.0f, 0.0f, 1.0f, + -1.0f, 1.1f, -100.0f, + -5.4f, 0.01f, -10'000.0f}); + test.AddInput("data_1", dims, + {1.0f, 0.0f, 2.0f, + -2.0f, 2.2f, 64.0f, + -1.0f, 0.02f, 0.1f}); + test.AddInput("data_3", dims, + {1.0f, 0.0f, 3.0f, + -3.0f, 3.3f, 64.0f, + 5.4f, 0.03f, 10'000.0f}); + test.AddOutput("sum", dims, + {1.0f, 0.0f, 1.0f, + -3.0f, 1.1f, -100.0f, + -5.4f, 0.01f, -10'000.0f}); + test.Run(); +} + +TEST(MathOpTest, Min_8) { + OpTester test("Min", 8); + std::vector dims{3, 3}; + test.AddInput("data_0", dims, + {1.0f, 0.0f, 1.0f, + -1.0f, 1.1f, -100.0f, + -5.4f, 0.01f, -10'000.0f}); + test.AddInput("data_1", dims, + {1.0f, 0.0f, 2.0f, + -2.0f, 2.2f, 64.0f, + -1.0f, 0.02f, 0.1f}); + test.AddInput("data_3", dims, + {1.0f, 0.0f, 3.0f, + -3.0f, 3.3f, 64.0f, + 5.4f, 0.03f, 10'000.0f}); + test.AddOutput("min", dims, + {1.0f, 0.0f, 1.0f, + -3.0f, 1.1f, -100.0f, + -5.4f, 0.01f, -10'000.0f}); + test.Run(); +} + +TEST(MathOpTest, Max_6) { + OpTester test("Max", 6); + std::vector dims{3, 3}; + test.AddInput("data_0", dims, + {1.0f, 0.0f, 1.0f, + -1.0f, 1.1f, -100.0f, + -5.4f, 0.01f, -10'000.0f}); + test.AddInput("data_1", dims, + {1.0f, 0.0f, 2.0f, + -2.0f, 2.2f, 64.0f, + -1.0f, 0.02f, 0.1f}); + test.AddInput("data_2", dims, + {1.0f, 0.0f, 3.0f, + -3.0f, 3.3f, 64.0f, + 5.4f, 0.03f, 10'000.0f}); + test.AddOutput("max", dims, + {1.0f, 0.0f, 3.0f, + -1.0f, 3.3f, 64.0f, + 5.4f, 0.03f, 10'000.0f}); + test.Run(); +} + +TEST(MathOpTest, Max_8) { + OpTester test("Max", 8); + test.AddInput("data_0", {1, 3}, + {1.0f, 2.0f, 3.0f}); + test.AddInput("data_2", {3, 3}, + {10.0f, 20.0f, 30.0f, + 40.0f, 50.0f, 60.0f, + 70.0f, 80.0f, 90.0f}); + test.AddInput("data_1", {3, 1}, + {-1.0f, -2.0f, 300.0f}); + test.AddOutput("max", {3, 3}, + {10.0f, 20.0f, 30.0f, + 40.0f, 50.0f, 60.0f, + 300.0f, 300.0f, 300.0f}); + test.Run(); +} + +TEST(MathOpTest, Not) { + OpTester test("Not"); + std::vector dims{2}; + test.AddInput("X", dims, {false, true}); + test.AddOutput("Y", dims, {true, false}); + test.Run(); +} + +TEST(MathOpTest, And) { + OpTester test("And"); + std::vector dims{4}; + test.AddInput("A", dims, {false, true, false, true}); + test.AddInput("B", dims, {false, false, true, true}); + test.AddOutput("C", dims, {false, false, false, true}); + test.Run(); +} + +TEST(MathOpTest, Or) { + OpTester test("Or"); + std::vector dims{4}; + test.AddInput("A", dims, {false, true, false, true}); + test.AddInput("B", dims, {false, false, true, true}); + test.AddOutput("C", dims, {false, true, true, true}); + test.Run(); +} + +TEST(MathOpTest, Xor) { + OpTester test("Xor"); + std::vector dims{4}; + test.AddInput("A", dims, {false, true, false, true}); + test.AddInput("B", dims, {false, false, true, true}); + test.AddOutput("C", dims, {false, true, true, false}); + test.Run(); +} + +TEST(MathOpTest, Xor_bcast3v2d) { + OpTester test("Xor"); + + test.AddInput("A", {2, 3, 4}, + {false, true, false, true, + false, true, false, true, + false, true, false, true, + + false, true, false, true, + false, true, false, true, + false, true, false, true}); + test.AddInput("B", {3, 4}, + {false, false, true, true, + false, false, true, true, + false, false, true, true}); + test.AddOutput("C", {2, 3, 4}, + {false, true, true, false, + false, true, true, false, + false, true, true, false, + + false, true, true, false, + false, true, true, false, + false, true, true, false}); + test.Run(); +} + +TEST(MathOpTest, Less) { + OpTester test("Less"); + std::vector dims{4}; + test.AddInput("A", dims, {1.0f, 0.0f, -1.0f, -1.0f}); + test.AddInput("B", dims, {1.0f, 1.0f, 2.0f, -1.0f}); + test.AddOutput("C", dims, {false, true, true, false}); + test.Run(); +} + +TEST(MathOpTest, Less_Scalar0) { + OpTester test("Less"); + test.AddInput("A", {1}, {1.0f}); + test.AddInput("B", {4}, {1.0f, 1.5f, 2.0f, -1.0f}); + test.AddOutput("C", {4}, {false, true, true, false}); + test.Run(); +} + +TEST(MathOpTest, Less_Scalar1) { + OpTester test("Less"); + test.AddInput("A", {4}, {1.0f, 0.5f, 2.0f, -1.0f}); + test.AddInput("B", {1}, {1.0f}); + test.AddOutput("C", {4}, {false, true, false, true}); + test.Run(); +} + +TEST(MathOpTest, Greater) { + OpTester test("Greater"); + std::vector dims{4}; + test.AddInput("A", dims, {1.0f, 0.0f, -1.0f, -1.0f}); + test.AddInput("B", dims, {1.0f, 1.0f, 2.0f, -1.0f}); + test.AddOutput("C", dims, {false, false, false, false}); + test.Run(); +} + +TEST(MathOpTest, Equal_bool) { + OpTester test("Equal"); + std::vector dims{4}; + test.AddInput("A", dims, {false, true, false, true}); + test.AddInput("B", dims, {false, false, true, true}); + test.AddOutput("C", dims, {true, false, false, true}); + test.Run(); +} + +TEST(MathOpTest, Equal_bool_scalar0) { + OpTester test("Equal"); + test.AddInput("A", {1}, {false}); + test.AddInput("B", {4}, {false, false, true, true}); + test.AddOutput("C", {4}, {true, true, false, false}); + test.Run(); +} + +TEST(MathOpTest, Equal_bool_scalar1) { + OpTester test("Equal"); + test.AddInput("A", {4}, {false, false, true, true}); + test.AddInput("B", {1}, {false}); + test.AddOutput("C", {4}, {true, true, false, false}); + test.Run(); +} + +TEST(MathOpTest, Equal_int32) { + OpTester test("Equal"); + std::vector dims{4}; + test.AddInput("A", dims, {1, 0, -1, -1}); + test.AddInput("B", dims, {1, 1, 2, -1}); + test.AddOutput("C", dims, {true, false, false, true}); + test.Run(); +} + +TEST(MathOpTest, Equal_int64) { + OpTester test("Equal"); + std::vector dims{4}; + test.AddInput("A", dims, {1, 0, -1, -1}); + test.AddInput("B", dims, {1, 1, 2, -1}); + test.AddOutput("C", dims, {true, false, false, true}); + test.Run(); +} + +TEST(MathOpTest, Mean_6) { + OpTester test("Mean", 6); + std::vector dims{3, 3}; + test.AddInput("data_0", dims, + {1.0f, 0.0f, 1.0f, + -1.0f, 1.1f, -100.0f, + -5.0f, 0.01f, -10.0f}); + test.AddInput("data_1", dims, + {1.0f, 0.0f, 2.0f, + -2.0f, 2.2f, 65.0f, + -1.0f, 0.02f, -1.0f}); + test.AddInput("data_3", dims, + {1.0f, 0.0f, 3.0f, + -3.0f, 3.3f, 65.0f, + -3.0f, 0.03f, -1.0f}); + test.AddOutput("mean", dims, + {1.0f, 0.0f, 2.0f, + -2.0f, 2.2f, 10.0f, + -3.0f, 0.02f, -4.0f}); + test.Run(); +} + +TEST(MathOpTest, Mean_8) { + OpTester test("Mean", 8); + test.AddInput("data_0", {1}, {1.0f}); + test.AddInput("data_1", {3, 1}, + {1.0f, 2.0f, 3.0f}); + test.AddInput("data_3", {3, 3}, + {10.0f, 20.0f, 30.0f, + 40.0f, 50.0f, 60.0f, + 70.0f, 80.0f, 90.0f}); + test.AddOutput("mean", {3, 3}, + {12.0f / 3.0f, 22.0f / 3.0f, 32.0f / 3.0f, + 43.0f / 3.0f, 53.0f / 3.0f, 63.0f / 3.0f, + 74.0f / 3.0f, 84.0f / 3.0f, 94.0f / 3.0f}); + test.Run(); +} + +TEST(MathOpTest, AffineDefaultAttributes) { + OpTester test("Affine"); + std::vector dims{2, 2}; + test.AddInput("A", dims, {0.0f, 1.0f, 2.0f, 3.0f}); + test.AddOutput("B", dims, {0.0f, 1.0f, 2.0f, 3.0f}); + test.Run(); +} + +TEST(MathOpTest, Affine) { + OpTester test("Affine"); + std::vector dims{2, 2}; + test.AddAttribute("alpha", 2.0f); + test.AddAttribute("beta", 1.0f); + test.AddInput("A", dims, {0.0f, 1.0f, 2.0f, 3.0f}); + test.AddOutput("B", dims, {1.0f, 3.0f, 5.0f, 7.0f}); + test.Run(); +} + +template +void TrigTest(OpTester& test, std::initializer_list input) { + std::vector dims{static_cast(input.size())}; + + std::vector output; + for (auto v : input) + output.push_back(op(v)); + + test.AddInput("X", dims, input); + test.AddOutput("Y", dims, output); + test.Run(); +} + +TEST(MathOpTest, Sin) { + OpTester test("Sin"); + TrigTest(test, {1.1f, -1.1f, 2.2f, -2.2f}); +} + +TEST(MathOpTest, Cos) { + OpTester test("Cos"); + TrigTest(test, {1.1f, -1.1f, 2.2f, -2.2f}); +} + +TEST(MathOpTest, Tan) { + OpTester test("Tan"); + TrigTest(test, {-100.0f, -50.0f, 0.0f, 50.0f, 100.0f}); +} + +TEST(MathOpTest, Asin) { + OpTester test("Asin"); + TrigTest(test, {-1.0f, -0.5f, 0.0f, 0.5f, 1.0f}); +} + +TEST(MathOpTest, Acos) { + OpTester test("Acos"); + TrigTest(test, {-1.0f, -0.5f, 0.0f, 0.5f, 1.0f}); +} + +TEST(MathOpTest, Atan) { + OpTester test("Atan"); + TrigTest(test, {-10.0f, -5.0f, 0.0f, 5.0f, 10.0f}); +} + +TEST(MathOpTest, Expand_8_3x3) { + OpTester test("Expand", 8); + test.AddInput("data_0", {1}, {1.0f}); + test.AddInput("data_1", {2}, {3, 3}); + test.AddOutput("result", {3, 3}, + {1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f}); + test.Run(); +} + +TEST(MathOpTest, Expand_8_3x1) { + OpTester test("Expand", 8); + test.AddInput("data_0", {3}, {1.0f, 2.0f, 3.0f}); + test.AddInput("data_1", {2}, {3, 1}); + test.AddOutput("result", {3, 3}, + {1.0f, 2.0f, 3.0f, + 1.0f, 2.0f, 3.0f, + 1.0f, 2.0f, 3.0f}); + test.Run(); +} + +TEST(MathOpTest, Expand_8_1x3) { + OpTester test("Expand", 8); + test.AddInput("data_0", {3, 1}, {1.0f, 2.0f, 3.0f}); + test.AddInput("data_1", {2}, {1, 3}); + test.AddOutput("result", {3, 3}, + {1.0f, 1.0f, 1.0f, + 2.0f, 2.0f, 2.0f, + 3.0f, 3.0f, 3.0f}); + test.Run(); +} + +TEST(MathOpTest, Scale) { + OpTester test("Scale"); + std::vector dims{2, 2}; + test.AddAttribute("scale", 2.0f); + test.AddInput("A", dims, {0.0f, 1.0f, 2.0f, 3.0f}); + test.AddOutput("B", dims, {0.0f, 2.0f, 4.0f, 6.0f}); + test.Run(); +} + +TEST(MathOpTest, Scale_Default) { + OpTester test("Scale"); + std::vector dims{2, 2}; + test.AddInput("A", dims, {0.0f, 1.0f, 2.0f, 3.0f}); + test.AddOutput("B", dims, {0.0f, 1.0f, 2.0f, 3.0f}); + test.Run(); +} + +} // namespace test + +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/math/gemm_test.cc b/onnxruntime/test/providers/cpu/math/gemm_test.cc new file mode 100644 index 0000000000000..c915f2215f2ea --- /dev/null +++ b/onnxruntime/test/providers/cpu/math/gemm_test.cc @@ -0,0 +1,165 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(MathOpTest, GemmNoTrans) { + OpTester test("Gemm"); + + test.AddAttribute("transA", (int64_t)0); + test.AddAttribute("transB", (int64_t)0); + test.AddAttribute("alpha", 1.0f); + test.AddAttribute("beta", 1.0f); + + test.AddInput("A", {2, 4}, + {1.0f, 2.0f, 3.0f, 4.0f, + -1.0f, -2.0f, -3.0f, -4.0f}); + test.AddInput("B", {4, 3}, std::vector(12, 1.0f)); + test.AddInput("C", {2, 3}, std::vector(6, 1.0f)); + test.AddOutput("Y", {2, 3}, + {11.0f, 11.0f, 11.0f, + -9.0f, -9.0f, -9.0f}); + test.Run(); +} + +TEST(MathOpTest, GemmBroadcast) { + OpTester test("Gemm"); + + test.AddAttribute("transA", (int64_t)0); + test.AddAttribute("transB", (int64_t)0); + test.AddAttribute("alpha", 1.0f); + test.AddAttribute("beta", 1.0f); + + test.AddInput("A", {2, 4}, + {1.0f, 2.0f, 3.0f, 4.0f, + -1.0f, -2.0f, -3.0f, -4.0f}); + test.AddInput("B", {4, 3}, std::vector(12, 1.0f)); + test.AddInput("C", {3}, std::vector{1.0f, 2.0f, 3.0f}); + test.AddOutput("Y", {2, 3}, + {11.0f, 12.0f, 13.0f, + -9.0f, -8.0f, -7.0f}); + test.Run(); +} + +TEST(MathOpTest, GemmTrans) { + OpTester test("Gemm"); + + test.AddAttribute("transA", (int64_t)1); + test.AddAttribute("transB", (int64_t)1); + test.AddAttribute("alpha", 1.0f); + test.AddAttribute("beta", 1.0f); + + test.AddInput("A", {4, 2}, + {1.0f, -1.0f, + 2.0f, -2.0f, + 3.0f, -3.0f, + 4.0f, -4.0f}); + test.AddInput("B", {3, 4}, std::vector(12, 1.0f)); + test.AddInput("C", {3}, std::vector(3, 1.0f)); + test.AddOutput("Y", {2, 3}, + {11.0f, 11.0f, 11.0f, + -9.0f, -9.0f, -9.0f}); + test.Run(); +} + +TEST(MathOpTest, GemmAlphaBeta) { + OpTester test("Gemm"); + + test.AddAttribute("transA", (int64_t)0); + test.AddAttribute("transB", (int64_t)0); + test.AddAttribute("alpha", 0.5f); + test.AddAttribute("beta", 2.0f); + + test.AddInput("A", {2, 4}, + {1.0f, 2.0f, 3.0f, 4.0f, + -1.0f, -2.0f, -3.0f, -4.0f}); + test.AddInput("B", {4, 3}, std::vector(12, 1.0f)); + test.AddInput("C", {3}, std::vector(3, 1.0f)); + test.AddOutput("Y", {2, 3}, + {7.0f, 7.0f, 7.0f, + -3.0f, -3.0f, -3.0f}); + test.Run(); +} + +TEST(MathOpTest, GemmNaN) { + OpTester test("Gemm"); + + test.AddAttribute("transA", (int64_t)0); + test.AddAttribute("transB", (int64_t)0); + test.AddAttribute("alpha", 1.0f); + test.AddAttribute("beta", 0.0f); + + test.AddInput("A", {2, 4}, + {1.0f, 2.0f, 3.0f, 4.0f, + -1.0f, -2.0f, -3.0f, -4.0f}); + test.AddInput("B", {4, 3}, std::vector(12, 1.0f)); + test.AddInput("C", {2, 3}, std::vector(6, 1.0f)); + test.AddOutput("Y", {2, 3}, + {10.0f, 10.0f, 10.0f, + -10.0f, -10.0f, -10.0f}); + test.Run(); +} + +TEST(MathOpTest, GemmScalarBroadcast) { + OpTester test("Gemm"); + + test.AddAttribute("transA", (int64_t)0); + test.AddAttribute("transB", (int64_t)0); + test.AddAttribute("alpha", 1.0f); + test.AddAttribute("beta", 1.0f); + + test.AddInput("A", {2, 4}, + {1.0f, 2.0f, 3.0f, 4.0f, + -1.0f, -2.0f, -3.0f, -4.0f}); + test.AddInput("B", {4, 3}, std::vector(12, 1.0f)); + test.AddInput("C", {1}, std::vector{1.0f}); + test.AddOutput("Y", {2, 3}, + {11.0f, 11.0f, 11.0f, + -9.0f, -9.0f, -9.0f}); + test.Run(); +} + +TEST(MathOpTest, Gemm2DBroadcast) { + OpTester test("Gemm"); + + test.AddAttribute("transA", (int64_t)0); + test.AddAttribute("transB", (int64_t)0); + test.AddAttribute("alpha", 1.0f); + test.AddAttribute("beta", 1.0f); + + test.AddInput("A", {2, 4}, + {1.0f, 2.0f, 3.0f, 4.0f, + -1.0f, -2.0f, -3.0f, -4.0f}); + test.AddInput("B", {4, 3}, std::vector(12, 1.0f)); + test.AddInput("C", {2, 1}, std::vector{1.0f, 2.0f}); + test.AddOutput("Y", {2, 3}, + {11.0f, 11.0f, 11.0f, + -8.0f, -8.0f, -8.0f}); + test.Run(); +} + +TEST(MathOpTest, GemmFalseBroadcast) { + OpTester test("Gemm"); + + test.AddAttribute("transA", (int64_t)0); + test.AddAttribute("transB", (int64_t)0); + test.AddAttribute("alpha", 1.0f); + test.AddAttribute("beta", 1.0f); + + test.AddInput("A", {2, 4}, + {1.0f, 2.0f, 3.0f, 4.0f, + -1.0f, -2.0f, -3.0f, -4.0f}); + test.AddInput("B", {4, 3}, std::vector(12, 1.0f)); + test.AddInput("C", {2, 3}, std::vector{1.0f, 1.0f, 1.0f, 2.0f, 2.0f, 2.0f}); + test.AddOutput("Y", {2, 3}, + {11.0f, 11.0f, 11.0f, + -8.0f, -8.0f, -8.0f}); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/math/hardmax_test.cc b/onnxruntime/test/providers/cpu/math/hardmax_test.cc new file mode 100644 index 0000000000000..1cab384c255a7 --- /dev/null +++ b/onnxruntime/test/providers/cpu/math/hardmax_test.cc @@ -0,0 +1,152 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/math/softmax_shared.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +static void RunTest(const std::vector& x_vals, + const std::vector& expected_vals, + const std::vector& dimensions, + int64_t axis = 1, + OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess, + const std::string& expected_err_str = "") { + OpTester test("Hardmax"); + + if (axis != 1) { + test.AddAttribute("axis", axis); + } + + test.AddInput("X", dimensions, x_vals); + test.AddOutput("Y", dimensions, expected_vals); + test.Run(expect_result, expected_err_str); +} + +TEST(HardmaxOperator, Simple) { + // https://github.com/onnx/onnx/blob/master/docs/Operators.md#Hardmax + std::vector x_vals = {-1.0f, 0.0f, 1.0f}; + std::vector expected_vals = {0.0f, 0.0f, 1.0f}; + std::vector dimensions = {1, 3}; + + RunTest(x_vals, expected_vals, dimensions); +} + +TEST(HardmaxOperator, LargeNumber) { + // x = np.array([[0, 1, 2, 3], [10000, 10001, 10002, 10003]]).astype(np.float32) + // expected output[[0.0f, 0.0f, 0.0f, 1.0f], + // [0.0f, 0.0f, 0.0f, 1.0f]] + + std::vector x_vals = {0.0f, 1.0f, 2.0f, 3.0f, 10000.0f, 10001.0f, 10002.0f, 10003.0f}; + std::vector expected_vals = {0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 1.0f}; + std::vector dimensions = {2, 4}; + + RunTest(x_vals, expected_vals, dimensions); +} + +//np.random.seed(123) # Use a seed so we can replicate the input and expected values here and in python +//x = np.abs(np.random.randn(3, 4, 5).astype(np.float32)) +static std::vector three_dimensions = {3, 4, 5}; +static std::vector x_vals_3dims = { + 1.0856307f, 0.99734545f, 0.2829785f, 1.5062947f, 0.5786002f, + 1.6514366f, 2.4266791f, 0.42891264f, 1.2659363f, 0.8667404f, + 0.6788862f, 0.09470897f, 1.4913896f, 0.638902f, 0.44398195f, + 0.43435127f, 2.20593f, 2.1867862f, 1.004054f, 0.3861864f, + + 0.7373686f, 1.4907321f, 0.9358339f, 1.175829f, 1.2538806f, + 0.6377515f, 0.9071052f, 1.4286807f, 0.14006872f, 0.8617549f, + 0.25561938f, 2.798589f, 1.7715331f, 0.69987726f, 0.92746246f, + 0.17363568f, 0.002845916f, 0.6882227f, 0.87953633f, 0.28362733f, + + 0.8053665f, 1.7276695f, 0.3908998f, 0.57380587f, 0.33858904f, + 0.011830495f, 2.3923652f, 0.41291216f, 0.978736f, 2.2381434f, + 1.2940853f, 1.0387882f, 1.7437122f, 0.79806274f, 0.02968323f, + 1.0693159f, 0.8907064f, 1.7548862f, 1.4956441f, 1.0693927f}; + +TEST(HardmaxOperator, ThreeDimsAxis0) { + // x = + // import cntk as C + // expected = C.hardmax(x.reshape(1,60)).eval().reshape(3, 4, 5) + std::vector expected_vals = { + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; + + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 0); +} + +TEST(HardmaxOperator, ThreeDimsAxis1) { + // x = + // import cntk as C + // expected = C.hardmax(x.reshape(3,20)).eval().reshape(3, 4, 5) + std::vector expected_vals = { + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; + + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 1); +} + +TEST(HardmaxOperator, ThreeDimsAxis2) { + // x = + // import cntk as C + // expected = C.hardmax(x.reshape(12,5)).eval().reshape(3, 4, 5) + std::vector expected_vals = { + 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, + 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, + + 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, + 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, + + 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, 0.0f}; + + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 2); +} + +TEST(HardmaxOperator, InvalidAxis) { + std::vector x_vals = {-1.0f, 0.0f, 1.0f}; + std::vector expected_vals = {0.0f, 0.0f, 0.0f}; + std::vector dimensions = {1, 3}; + + RunTest(x_vals, + expected_vals, + dimensions, + /* invalid axis */ -1, + OpTester::ExpectResult::kExpectFailure, + "Invalid axis provided."); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/math/logsoftmax_test.cc b/onnxruntime/test/providers/cpu/math/logsoftmax_test.cc new file mode 100644 index 0000000000000..13f22423708b9 --- /dev/null +++ b/onnxruntime/test/providers/cpu/math/logsoftmax_test.cc @@ -0,0 +1,195 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/math/logsoftmax.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +static void RunTest(const std::vector& x_vals, + const std::vector& expected_vals, + const std::vector& dimensions, + int64_t axis = 1, + OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess, + const std::string& error_msg = "") { + OpTester tester("LogSoftmax"); + + if (axis != 1) { + tester.AddAttribute("axis", axis); + } + + tester.AddInput("X", dimensions, x_vals); + tester.AddOutput("Y", dimensions, expected_vals); + + tester.Run(expect_result, error_msg); +} + +TEST(LogSoftmaxOperator, Simple) { + // https://github.com/onnx/onnx/blob/master/docs/Operators.md#LogSoftmax + // x = np.array([[-1, 0, 1]]).astype(np.float32) + // # expected output[[-2.40760589, -1.40760589, -0.40760589]] + + std::vector x_vals = {-1.0f, 0.0f, 1.0f}; + std::vector expected_vals = {-2.40760589f, -1.40760589f, -0.40760589f}; + std::vector dimensions = {1, 3}; + + RunTest(x_vals, expected_vals, dimensions); +} + +TEST(LogSoftmaxOperator, LargeNumber) { + // x = np.array([[0, 1, 2, 3], + // [10000, 10001, 10002, 10003]]).astype(np.float32) + // expected output[[-3.4401896, -2.4401896, -1.44018972, -0.44018969], + // [-3.4401896, -2.4401896, -1.44018972, -0.44018969]] + + std::vector x_vals = {0.0f, 1.0f, 2.0f, 3.0f, + 10000.0f, 10001.0f, 10002.0f, 10003.0f}; + std::vector expected_vals = {-3.4401896f, -2.4401896f, -1.44018972f, -0.44018969f, + -3.4401896f, -2.4401896f, -1.44018972f, -0.44018969f}; + std::vector dimensions = {2, 4}; + + RunTest(x_vals, expected_vals, dimensions); +} + +//np.random.seed(123) # Use a seed so we can replicate the input and expected values here and in python +//x = np.abs(np.random.randn(3, 4, 5).astype(np.float32)) +static std::vector three_dimensions = {3, 4, 5}; +static std::vector x_vals_3dims = { + 1.0856307f, 0.99734545f, 0.2829785f, 1.5062947f, 0.5786002f, + 1.6514366f, 2.4266791f, 0.42891264f, 1.2659363f, 0.8667404f, + 0.6788862f, 0.09470897f, 1.4913896f, 0.638902f, 0.44398195f, + 0.43435127f, 2.20593f, 2.1867862f, 1.004054f, 0.3861864f, + + 0.7373686f, 1.4907321f, 0.9358339f, 1.175829f, 1.2538806f, + 0.6377515f, 0.9071052f, 1.4286807f, 0.14006872f, 0.8617549f, + 0.25561938f, 2.798589f, 1.7715331f, 0.69987726f, 0.92746246f, + 0.17363568f, 0.002845916f, 0.6882227f, 0.87953633f, 0.28362733f, + + 0.8053665f, 1.7276695f, 0.3908998f, 0.57380587f, 0.33858904f, + 0.011830495f, 2.3923652f, 0.41291216f, 0.978736f, 2.2381434f, + 1.2940853f, 1.0387882f, 1.7437122f, 0.79806274f, 0.02968323f, + 1.0693159f, 0.8907064f, 1.7548862f, 1.4956441f, 1.0693927f}; + +TEST(LogSoftmaxOperator, ThreeDimsAxis0) { + // x = + // node = onnx.helper.make_node('LogSoftmax', inputs = ['x'], outputs = ['y'], axis = 0) + // y = logsoftmax_2d(x.reshape(1, 60)).reshape(3, 4, 5) + // expect(node, inputs = [x], outputs = [y], + // name = 'test_logsoftmax_axis_0') + + std::vector expected_vals = { + -4.2514257f, -4.339711f, -5.054078f, -3.8307617f, -4.758456f, + -3.6856198f, -2.9103773f, -4.908144f, -4.0711203f, -4.470316f, + -4.65817f, -5.2423477f, -3.845667f, -4.6981544f, -4.8930745f, + -4.902705f, -3.1311264f, -3.1502702f, -4.3330026f, -4.95087f, + + -4.5996876f, -3.8463244f, -4.401222f, -4.161227f, -4.0831757f, + -4.6993046f, -4.429951f, -3.9083757f, -5.1969876f, -4.4753017f, + -5.081437f, -2.5384674f, -3.5655231f, -4.6371794f, -4.409594f, + -5.1634207f, -5.3342104f, -4.6488338f, -4.45752f, -5.053429f, + + -4.5316896f, -3.609387f, -4.9461565f, -4.7632504f, -4.9984674f, + -5.325226f, -2.9446912f, -4.9241443f, -4.35832f, -3.098913f, + -4.042971f, -4.2982683f, -3.5933442f, -4.538994f, -5.307373f, + -4.2677402f, -4.44635f, -3.5821702f, -3.8414123f, -4.267664f}; + + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 0); +} + +TEST(LogSoftmaxOperator, ThreeDimsAxis1) { + // x = + // node = onnx.helper.make_node('LogSoftmax', inputs = ['x'], outputs = ['y'], axis = 1) + // y = logsoftmax_2d(x.reshape(3, 20)).reshape(3, 4, 5) + // expect(node, inputs = [x], outputs = [y], + // name = 'test_logsoftmax_axis_1') + + std::vector expected_vals = { + -3.1908588f, -3.2791443f, -3.9935112f, -2.770195f, -3.6978893f, + -2.625053f, -1.8498105f, -3.847577f, -3.0105534f, -3.409749f, + -3.5976033f, -4.181781f, -2.7851f, -3.6375875f, -3.8325076f, + -3.8421383f, -2.0705595f, -2.0897036f, -3.2724357f, -3.8903031f, + + -3.4205704f, -2.667207f, -3.222105f, -2.98211f, -2.9040585f, + -3.5201874f, -3.250834f, -2.7292585f, -4.0178704f, -3.296184f, + -3.90232f, -1.3593501f, -2.386406f, -3.4580617f, -3.2304766f, + -3.9843035f, -4.155093f, -3.4697165f, -3.2784028f, -3.874312f, + + -3.4709241f, -2.5486212f, -3.8853908f, -3.7024848f, -3.9377017f, + -4.26446f, -1.8839254f, -3.8633785f, -3.2975547f, -2.0381472f, + -2.9822054f, -3.2375026f, -2.5325785f, -3.4782279f, -4.246608f, + -3.2069747f, -3.3855844f, -2.5214045f, -2.7806466f, -3.206898f}; + + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 1); +} + +TEST(LogSoftmaxOperator, ThreeDimsAxis2) { + // x = + // node = onnx.helper.make_node('LogSoftmax', inputs = ['x'], outputs = ['y'], axis = 2) + // y = logsoftmax_2d(x.reshape(12, 5)).reshape(3, 4, 5) + // expect(node, inputs = [x], outputs = [y], + // name = 'test_logsoftmax_axis_2') + + std::vector expected_vals = { + -1.5016061f, -1.5898913f, -2.3042583f, -1.080942f, -2.0086365f, + -1.5264852f, -0.7512426f, -2.7490091f, -1.9119854f, -2.3111813f, + -1.716058f, -2.3002353f, -0.9035546f, -1.7560422f, -1.9509623f, + -2.7323837f, -0.96080494f, -0.97994876f, -2.162681f, -2.7805486f, + + -2.024213f, -1.2708496f, -1.8257477f, -1.5857526f, -1.507701f, + -1.8521607f, -1.582807f, -1.0612315f, -2.3498435f, -1.6281573f, + -3.0813656f, -0.538396f, -1.5654519f, -2.6371078f, -2.4095225f, + -1.8958019f, -2.0665917f, -1.3812149f, -1.1899012f, -1.7858102f, + + -1.7220669f, -0.79976386f, -2.1365335f, -1.9536276f, -2.1888442f, + -3.2268262f, -0.84629166f, -2.8257446f, -2.259921f, -1.0005134f, + -1.4430928f, -1.6983899f, -0.9934659f, -1.9391153f, -2.7074947f, + -1.8489327f, -2.027542f, -1.1633625f, -1.4226046f, -1.848856f}; + + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 2); +} + +TEST(LogSoftmaxOperator, ThreeDimsNegativeAxis) { + // x = + // node = onnx.helper.make_node('LogSoftmax', inputs = ['x'], outputs = ['y'], axis = 2) + // y = logsoftmax_2d(x.reshape(12, 5)).reshape(3, 4, 5) + // expect(node, inputs = [x], outputs = [y], + // name = 'test_logsoftmax_axis_2') + + std::vector expected_vals = { + -1.5016061f, -1.5898913f, -2.3042583f, -1.080942f, -2.0086365f, + -1.5264852f, -0.7512426f, -2.7490091f, -1.9119854f, -2.3111813f, + -1.716058f, -2.3002353f, -0.9035546f, -1.7560422f, -1.9509623f, + -2.7323837f, -0.96080494f, -0.97994876f, -2.162681f, -2.7805486f, + + -2.024213f, -1.2708496f, -1.8257477f, -1.5857526f, -1.507701f, + -1.8521607f, -1.582807f, -1.0612315f, -2.3498435f, -1.6281573f, + -3.0813656f, -0.538396f, -1.5654519f, -2.6371078f, -2.4095225f, + -1.8958019f, -2.0665917f, -1.3812149f, -1.1899012f, -1.7858102f, + + -1.7220669f, -0.79976386f, -2.1365335f, -1.9536276f, -2.1888442f, + -3.2268262f, -0.84629166f, -2.8257446f, -2.259921f, -1.0005134f, + -1.4430928f, -1.6983899f, -0.9934659f, -1.9391153f, -2.7074947f, + -1.8489327f, -2.027542f, -1.1633625f, -1.4226046f, -1.848856f}; + + // -1 is last axis so same as axis == 2 + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ -1); +} + +TEST(LogSoftmaxOperator, InvalidAxis) { + std::vector x_vals = {-1.0f, 0.0f, 1.0f}; + std::vector expected_vals = {0.0f, 0.0f, 0.0f}; + std::vector dimensions = {1, 3}; + + RunTest(x_vals, + expected_vals, + dimensions, + /* invalid axis */ -7, + OpTester::ExpectResult::kExpectFailure, + "-7 is not in valid range [-2,1]"); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/math/matmul_test.cc b/onnxruntime/test/providers/cpu/math/matmul_test.cc new file mode 100644 index 0000000000000..19e9c0bb929cf --- /dev/null +++ b/onnxruntime/test/providers/cpu/math/matmul_test.cc @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(MathOpTest, MatMul) { + std::vector vals{0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f}; + + struct MatMulTest { + std::string name; + std::vector input0_dims; + std::vector input1_dims; + std::vector expected_dims; + std::vector expected_vals; + }; + + MatMulTest testcases[] = { + {"test padding and broadcast", + {3, 1, 1, 2}, + {2, 2, 2}, + {3, 2, 1, 2}, + {2, 3, 6, 7, 6, 11, 26, 31, 10, 19, 46, 55}}, + {"test padding and broadcast", + {2, 3, 2}, + {3, 2, 2, 1}, + {3, 2, 3, 1}, + {1, 3, 5, 33, 43, 53, 5, 23, 41, 85, 111, 137, 9, 43, 77, 137, 179, 221}}, + {"test left 1D", + {2}, + {3, 2, 1}, + {3, 1}, + {1, 3, 5}}, + {"test right 1D", + {3, 1, 2}, + {2}, + {3, 1}, + {1, 3, 5}}, + {"test scalar output", + {3}, + {3}, + {}, + {5}}, + {"test 2D", + {3, 4}, + {4, 3}, + {3, 3}, + {42, 48, 54, 114, 136, 158, 186, 224, 262}}, + {"test 2D special", + {2, 2, 3}, + {3, 4}, + {2, 2, 4}, + {20, 23, 26, 29, 56, 68, 80, 92, 92, 113, 134, 155, 128, 158, 188, 218}}, + {"test 2D special 2", + {2, 2, 3}, + {1, 3, 4}, + {2, 2, 4}, + {20, 23, 26, 29, 56, 68, 80, 92, 92, 113, 134, 155, 128, 158, 188, 218}}, + }; + + for (auto t : testcases) { + OpTester test("MatMul"); + + int64_t size0 = TensorShape::ReinterpretBaseType(t.input0_dims).SizeHelper(0, t.input0_dims.size()); + std::vector input0_vals(vals.cbegin(), vals.cbegin() + size0); + test.AddInput("A", t.input0_dims, input0_vals); + + int64_t size1 = TensorShape::ReinterpretBaseType(t.input1_dims).SizeHelper(0, t.input1_dims.size()); + std::vector input1_vals(vals.cbegin(), vals.cbegin() + size1); + test.AddInput("B", t.input1_dims, input1_vals); + + test.AddOutput("Y", t.expected_dims, t.expected_vals); + test.Run(); + } +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/math/softmax_test.cc b/onnxruntime/test/providers/cpu/math/softmax_test.cc new file mode 100644 index 0000000000000..82704dd61e7e4 --- /dev/null +++ b/onnxruntime/test/providers/cpu/math/softmax_test.cc @@ -0,0 +1,232 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/math/softmax_shared.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +static void RunTest(const std::vector& x_vals, + const std::vector& expected_vals, + const std::vector& dimensions, + int64_t axis = 1, + OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess, + const std::string& error_msg = "") { + OpTester test("Softmax"); + + if (axis != 1) { + test.AddAttribute("axis", axis); + } + + test.AddInput("X", dimensions, x_vals); + test.AddOutput("Y", dimensions, expected_vals); + test.Run(expect_result, error_msg); +} + +TEST(SoftmaxOperator, Simple) { + // https://github.com/onnx/onnx/blob/master/docs/Operators.md#Softmax + // x = np.array([[-1, 0, 1]]).astype(np.float32) + // y = np.exp(x) / np.sum(np.exp(x), axis = 1) #expected output[[0.09003058, 0.24472848, 0.66524094]] + + std::vector x_vals = {-1.0f, 0.0f, 1.0f}; + std::vector expected_vals = {0.09003058f, 0.24472848f, 0.66524094f}; + std::vector dimensions = {1, 3}; + + RunTest(x_vals, expected_vals, dimensions); +} + +TEST(SoftmaxOperator, LargeNumber) { + // x = np.array([[0, 1, 2, 3], [10000, 10001, 10002, 10003]]).astype(np.float32) + // expected output[[0.0320586, 0.08714432, 0.23688284, 0.64391428], + // [0.0320586, 0.08714432, 0.23688284, 0.64391428]] + + std::vector x_vals = {0.0f, 1.0f, 2.0f, 3.0f, 10000.0f, 10001.0f, 10002.0f, 10003.0f}; + std::vector expected_vals = {0.0320586f, 0.08714432f, 0.23688284f, 0.64391428f, 0.0320586f, 0.08714432f, 0.23688284f, 0.64391428f}; + std::vector dimensions = {2, 4}; + + RunTest(x_vals, expected_vals, dimensions); +} + +//np.random.seed(123) # Use a seed so we can replicate the input and expected values here and in python +//x = np.abs(np.random.randn(3, 4, 5).astype(np.float32)) +static std::vector three_dimensions = {3, 4, 5}; +static std::vector x_vals_3dims = { + 1.0856307f, 0.99734545f, 0.2829785f, 1.5062947f, 0.5786002f, + 1.6514366f, 2.4266791f, 0.42891264f, 1.2659363f, 0.8667404f, + 0.6788862f, 0.09470897f, 1.4913896f, 0.638902f, 0.44398195f, + 0.43435127f, 2.20593f, 2.1867862f, 1.004054f, 0.3861864f, + + 0.7373686f, 1.4907321f, 0.9358339f, 1.175829f, 1.2538806f, + 0.6377515f, 0.9071052f, 1.4286807f, 0.14006872f, 0.8617549f, + 0.25561938f, 2.798589f, 1.7715331f, 0.69987726f, 0.92746246f, + 0.17363568f, 0.002845916f, 0.6882227f, 0.87953633f, 0.28362733f, + + 0.8053665f, 1.7276695f, 0.3908998f, 0.57380587f, 0.33858904f, + 0.011830495f, 2.3923652f, 0.41291216f, 0.978736f, 2.2381434f, + 1.2940853f, 1.0387882f, 1.7437122f, 0.79806274f, 0.02968323f, + 1.0693159f, 0.8907064f, 1.7548862f, 1.4956441f, 1.0693927f}; + +TEST(SoftmaxOperator, ThreeDimsAxis0) { + // x = + // node = onnx.helper.make_node('Softmax', inputs = ['x'], outputs = ['y'], axis = 0) + // y = softmax_2d(x.reshape(1, 60)).reshape(3, 4, 5) + // expect(node, inputs = [x], outputs = [y], + // name = 'test_softmax_axis_0') + + std::vector expected_vals = { + 0.01424391f, 0.013040296f, 0.0063832495f, 0.021693084f, 0.0085788425f, + 0.02508162f, 0.054455176f, 0.007386185f, 0.017058268f, 0.011443698f, + 0.009483798f, 0.0052878284f, 0.021372143f, 0.009112078f, 0.0074983323f, + 0.0074264654f, 0.04366858f, 0.04284054f, 0.01312807f, 0.007077248f, + + 0.010054973f, 0.021358095f, 0.01226234f, 0.015588412f, 0.016853856f, + 0.009101601f, 0.01191507f, 0.020073075f, 0.0055332067f, 0.0113867875f, + 0.0062109763f, 0.07898735f, 0.028282179f, 0.009684978f, 0.012160114f, + 0.005722092f, 0.004823717f, 0.009572758f, 0.011591072f, 0.006387392f, + + 0.010762473f, 0.027068434f, 0.007110686f, 0.00853781f, 0.0067482805f, + 0.004867251f, 0.0526183f, 0.007268943f, 0.0127998665f, 0.045098193f, + 0.017545262f, 0.0135920765f, 0.027506188f, 0.010684152f, 0.0049549243f, + 0.01401341f, 0.011721271f, 0.027815264f, 0.021463264f, 0.014014485f}; + + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 0); +} + +TEST(SoftmaxOperator, ThreeDimsAxis1) { + // x = + // node = onnx.helper.make_node('Softmax', inputs = ['x'], outputs = ['y'], axis = 1) + // y = softmax_2d(x.reshape(3, 20)).reshape(3, 4, 5) + // expect(node, inputs = [x], outputs = [y], + // name = 'test_softmax_axis_1') + + std::vector expected_vals = { + 0.04113652f, 0.037660476f, 0.018434875f, 0.0626498f, 0.024775764f, + 0.072435915f, 0.15726697f, 0.021331362f, 0.049264412f, 0.03304949f, + 0.027389284f, 0.015271291f, 0.061722923f, 0.026315752f, 0.021655245f, + 0.021447688f, 0.1261152f, 0.12372383f, 0.03791397f, 0.020439148f, + + 0.032693777f, 0.069445916f, 0.039871037f, 0.05068577f, 0.054800365f, + 0.029593885f, 0.03874189f, 0.06526767f, 0.01799124f, 0.037024178f, + 0.02019501f, 0.25682762f, 0.091959596f, 0.031490736f, 0.03953865f, + 0.018605402f, 0.01568433f, 0.031125855f, 0.037688408f, 0.020768626f, + + 0.031088287f, 0.0781894f, 0.020539802f, 0.024662167f, 0.019492965f, + 0.014059456f, 0.15199229f, 0.020996941f, 0.036973465f, 0.13026986f, + 0.050680935f, 0.03926183f, 0.079453886f, 0.030862054f, 0.014312706f, + 0.040478885f, 0.033857856f, 0.080346674f, 0.06199841f, 0.040481992f}; + + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 1); +} + +TEST(SoftmaxOperator, ThreeDimsAxis2) { + // x = + // node = onnx.helper.make_node('Softmax', inputs = ['x'], outputs = ['y'], axis = 2) + // y = softmax_2d(x.reshape(12, 5)).reshape(3, 4, 5) + // expect(node, inputs = [x], outputs = [y], + // name = 'test_softmax_axis_2') + + std::vector expected_vals = { + 0.22277209f, 0.20394778f, 0.09983283f, 0.33927578f, 0.13417149f, + 0.21729809f, 0.47177994f, 0.06399124f, 0.14778666f, 0.099144064f, + 0.1797734f, 0.10023525f, 0.40512702f, 0.17272712f, 0.14213723f, + 0.06506401f, 0.3825848f, 0.37533033f, 0.11501635f, 0.062004484f, + + 0.13209775f, 0.28059313f, 0.16109712f, 0.2047936f, 0.22141843f, + 0.1568978f, 0.20539774f, 0.3460294f, 0.0953841f, 0.19629094f, + 0.045896534f, 0.5836837f, 0.20899355f, 0.07156797f, 0.08985819f, + 0.15019783f, 0.1266166f, 0.2512731f, 0.30425128f, 0.16766116f, + + 0.17869644f, 0.44943509f, 0.11806339f, 0.1417589f, 0.112046175f, + 0.03968324f, 0.42900288f, 0.059264507f, 0.10435873f, 0.36769062f, + 0.23619612f, 0.1829779f, 0.37029108f, 0.14383113f, 0.0667037f, + 0.15740506f, 0.13165872f, 0.31243387f, 0.24108529f, 0.15741715f}; + + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 2); +} + +TEST(SoftmaxOperator, ThreeDimsNegativeAxis) { + // x = + // node = onnx.helper.make_node('Softmax', inputs = ['x'], outputs = ['y'], axis = 2) + // y = softmax_2d(x.reshape(12, 5)).reshape(3, 4, 5) + // expect(node, inputs = [x], outputs = [y], + // name = 'test_softmax_axis_2') + + std::vector expected_vals = { + 0.22277209f, 0.20394778f, 0.09983283f, 0.33927578f, 0.13417149f, + 0.21729809f, 0.47177994f, 0.06399124f, 0.14778666f, 0.099144064f, + 0.1797734f, 0.10023525f, 0.40512702f, 0.17272712f, 0.14213723f, + 0.06506401f, 0.3825848f, 0.37533033f, 0.11501635f, 0.062004484f, + + 0.13209775f, 0.28059313f, 0.16109712f, 0.2047936f, 0.22141843f, + 0.1568978f, 0.20539774f, 0.3460294f, 0.0953841f, 0.19629094f, + 0.045896534f, 0.5836837f, 0.20899355f, 0.07156797f, 0.08985819f, + 0.15019783f, 0.1266166f, 0.2512731f, 0.30425128f, 0.16766116f, + + 0.17869644f, 0.44943509f, 0.11806339f, 0.1417589f, 0.112046175f, + 0.03968324f, 0.42900288f, 0.059264507f, 0.10435873f, 0.36769062f, + 0.23619612f, 0.1829779f, 0.37029108f, 0.14383113f, 0.0667037f, + 0.15740506f, 0.13165872f, 0.31243387f, 0.24108529f, 0.15741715f}; + + // -1 is last axis so same as axis == 2 + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ -1); +} + +TEST(SoftmaxOperator, InvalidAxis) { + std::vector x_vals = {-1.0f, 0.0f, 1.0f}; + std::vector expected_vals = {0.0f, 0.0f, 0.0f}; + std::vector dimensions = {1, 3}; + + RunTest(x_vals, + expected_vals, + dimensions, + /* invalid axis */ -10, + OpTester::ExpectResult::kExpectFailure, + "-10 is not in valid range [-2,1]"); +} + +TEST(SoftmaxOperator, TestInputTooLarge) { + float* ignored = nullptr; + + // N > INT32_MAX + int64_t N = int64_t(INT32_MAX) + 1; + int64_t D = 1; + auto status = SoftmaxCPU(N, D, ignored, ignored, ignored, ignored, true, ignored); + EXPECT_EQ(status.Code(), common::INVALID_ARGUMENT); + + // D > INT32_MAX + N = 1; + D = int64_t(INT32_MAX) + 1; + status = SoftmaxCPU(N, D, ignored, ignored, ignored, ignored, true, ignored); + EXPECT_EQ(status.Code(), common::INVALID_ARGUMENT); + + // N * D > INT32_MAX + N = int64_t(INT32_MAX) / 2; + D = 3; + status = SoftmaxCPU(N, D, ignored, ignored, ignored, ignored, true, ignored); + EXPECT_EQ(status.Code(), common::INVALID_ARGUMENT); + + /* + common::Status SoftmaxCPU(const int64_t N, + const int64_t D, + const float* Xdata, + float* Ydata, + float* scale, + const float* sum_multiplier, + bool logarithmic, + float* rowmax) + { + // the Math functions SoftmaxCPU uses only support int32_t as input, so enforce that + if (N * D > INT32_MAX || N > INT32_MAX || D > INT32_MAX) + { + std::ostringstream ss; + ss << "SoftmaxCPU inputs N, D and N * D must be < " << INT32_MAX << ". N=" << N << ", D=" << D; + std::string msg = ss.str(); + + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, msg); + }*/ +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/math/topk_op_test.cc b/onnxruntime/test/providers/cpu/math/topk_op_test.cc new file mode 100644 index 0000000000000..645f37475bfcb --- /dev/null +++ b/onnxruntime/test/providers/cpu/math/topk_op_test.cc @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +static void RunTest(int64_t k, + const std::vector& input_vals, + const std::vector& input_dimensions, + const std::vector& expected_vals, + const std::vector& expected_indices, + const std::vector& expected_dimensions, + int64_t axis = 1, + OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess, + const std::string& expected_err_str = "") { + OpTester test("TopK"); + test.AddAttribute("k", k); + if (axis != 1) { + test.AddAttribute("axis", axis); + } + + test.AddInput("X", input_dimensions, input_vals); + test.AddOutput("Values", expected_dimensions, expected_vals); + test.AddOutput("Indices", expected_dimensions, expected_indices); + test.Run(expect_result, expected_err_str); +} + +TEST(TopKOperator, Top1) { + std::vector input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.3f, 0.2f}; + std::vector input_dimensions = {2, 4}; + std::vector expected_vals = {0.4f, 0.3f}; + std::vector expected_indices = {3, 1}; + std::vector expected_dimensions = {2, 1}; + RunTest(1, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions); +} + +TEST(TopKOperator, Top2) { + std::vector input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.4f, 0.2f}; + std::vector input_dimensions = {2, 4}; + std::vector expected_vals = {0.4f, 0.3f, 0.4f, 0.3f}; + std::vector expected_indices = {3, 1, 2, 1}; + std::vector expected_dimensions = {2, 2}; + RunTest(2, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions); +} + +TEST(TopKOperator, Top3) { + std::vector input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.4f, 0.2f}; + std::vector input_dimensions = {2, 4}; + std::vector expected_vals = {0.4f, 0.3f, 0.2f, 0.4f, 0.3f, 0.2f}; + std::vector expected_indices = {3, 1, 2, 2, 1, 3}; + std::vector expected_dimensions = {2, 3}; + RunTest(3, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions); +} + +TEST(TopKOperator, TopAll) { + std::vector input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.3f, 0.2f}; + std::vector input_dimensions = {2, 4}; + std::vector expected_vals = {0.4f, 0.3f, 0.2f, 0.1f, 0.3f, 0.3f, 0.2f, 0.1f}; + std::vector expected_indices = {3, 1, 2, 0, 1, 2, 3, 0}; + std::vector expected_dimensions = {2, 4}; + RunTest(4, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions); +} + +TEST(TopKOperator, InvalidK) { + std::vector input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.3f, 0.2f}; + std::vector input_dimensions = {2, 4}; + std::vector expected_vals = {0.4f, 0.3f, 0.2f, 0.1f, 0.3f, 0.3f, 0.2f, 0.1f}; + std::vector expected_indices = {3, 1, 2, 0, 1, 2, 3, 0}; + std::vector expected_dimensions = {2, 4}; + RunTest(0, + input_vals, + input_dimensions, + expected_vals, + expected_indices, + expected_dimensions, + 1, + OpTester::ExpectResult::kExpectFailure, + "Invalid value for attribute k"); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/ml/array_feature_extractor_test.cc b/onnxruntime/test/providers/cpu/ml/array_feature_extractor_test.cc new file mode 100644 index 0000000000000..164e143fbcf3d --- /dev/null +++ b/onnxruntime/test/providers/cpu/ml/array_feature_extractor_test.cc @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" +#include "gsl/gsl" +using namespace std; +namespace onnxruntime { +namespace test { + +TEST(MLOpTest, ArrayFeatureExtractorTest) { + OpTester test("ArrayFeatureExtractor", 1, onnxruntime::kMLDomain); + const int N = 3; + const std::vector X = {0.8f, -1.5f, 2.0f, 3.8f, -4.0f, 5.0f, + 6.8f, -7.5f, 8.0f, 9.8f, -9.0f, 4.0f, + 4.8f, -4.5f, 4.0f, 4.8f, -4.0f, 4.0f}; + const int kCols = 6; + const vector x_dims = {N, kCols}; + test.AddInput("X", x_dims, X); + + const std::vector Y = {1L, 2L, 4L}; + const vector y_dims = {1, 3}; + test.AddInput("Y", y_dims, Y); + + // prepare expected output + vector expected_output; + for (int i = 0; i < N; ++i) { + auto offset = i * kCols; + for (size_t j = 0; j < Y.size(); ++j) { + expected_output.push_back(X[offset + Y[j]]); + } + } + const vector expected_dims{N, gsl::narrow_cast(Y.size())}; + test.AddOutput("Z", expected_dims, expected_output); + + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/ml/binarizer_test.cc b/onnxruntime/test/providers/cpu/ml/binarizer_test.cc new file mode 100644 index 0000000000000..dd4c24a2286f6 --- /dev/null +++ b/onnxruntime/test/providers/cpu/ml/binarizer_test.cc @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" +using namespace std; +namespace onnxruntime { +namespace test { + +TEST(MLOpTest, BinarizerOp) { + OpTester test("Binarizer", 1, onnxruntime::kMLDomain); + float threshold = 0.3f; + test.AddAttribute("threshold", threshold); + vector input{0.8f, -0.5f, 0.2f, 0.8f, -1.0f, 0.1f}; + + // setup expected output + vector expected_output; + for (auto& elem : input) { + expected_output.push_back(elem > threshold ? 1.f : 0.f); + } + + vector dims{2, 3}; + test.AddInput("X", dims, input); + test.AddOutput("Y", dims, expected_output); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/ml/cast_map_test.cc b/onnxruntime/test/providers/cpu/ml/cast_map_test.cc new file mode 100644 index 0000000000000..287734f7ae54b --- /dev/null +++ b/onnxruntime/test/providers/cpu/ml/cast_map_test.cc @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +// cast from map to Tensor +template +static void RunTest(const std::map& input, + const std::vector& output, + const std::string& cast_to, + int64_t max_map = -1, + OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess) { + OpTester test("CastMap", 1, onnxruntime::kMLDomain); + + test.AddAttribute("cast_to", cast_to); + + if (max_map <= 0) { + test.AddAttribute("map_form", "DENSE"); + } else { + test.AddAttribute("map_form", "SPARSE"); + test.AddAttribute("max_map", max_map); + } + + test.AddInput("X", input); + + std::vector dims{1, gsl::narrow_cast(output.size())}; + test.AddOutput("Y", dims, output); + + test.Run(expect_result); +} + +/* +Cast to Tensor. +*/ + +// test dense input, converting from string to float +// also validate the output is ordered based on the index in the map, and not the order the entries were added. +TEST(CastMap, DenseStringToFloat) { + std::map map = {{1, "1.0"}, + {2, "2"}, + {3, "-3.0f"}, + {0, "-1"}}; // this should be first in the output + + std::vector output{-1.0f, 1.0f, 2.0f, -3.0f}; + + RunTest(map, output, "TO_FLOAT"); +} // namespace test + +// Test sparse input, converting from float to float +TEST(CastMap, SparseFloatToFloat) { + // test with gaps at start and end + std::map map{{1, 1.0f}, {2, 2.0f}}; + + std::vector output{0.0f, 1.0f, 2.0f, 0.0f}; + + RunTest(map, output, "TO_FLOAT", 4); +} + +/* +Cast to Tensor +*/ +TEST(CastMap, SparseStringToInt64) { + // gaps in middle + std::map map{{0, "-1.0"}, {3, "3"}}; + std::vector output{-1, 0, 0, 3}; + + RunTest(map, output, "TO_INT64", 4); +} + +TEST(CastMap, DenseFloatToInt64) { + // float to int64 is just a static_cast, so no rounding + std::map map{{0, -1.f}, {1, 1.9f}, {2, -2.4f}, {3, 3.0f}}; + std::vector output{-1, 1, -2, 3}; + + RunTest(map, output, "TO_INT64"); +} + +/* +Cast to Tensor +*/ +TEST(CastMap, StringToString) { + std::map map{{0, "-1.0f"}, {1, "3"}}; + + std::vector output{"-1.0f", "3"}; + + RunTest(map, output, "TO_STRING"); +} + +TEST(CastMap, FloatToString) { + std::map map{{0, -1.0f}, {1, 3.0f}}; + + // std::stof converts to these values. + std::vector output{"-1.000000", "3.000000"}; + + RunTest(map, output, "TO_STRING"); +} + +/* +Miscellaneous tests +*/ + +// cast from map to Tensor +void RunBadAttributeTest(const std::string& cast_to, + const std::string& map_form, + int64_t max_map = -1, + OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess) { + OpTester test("CastMap", 1, onnxruntime::kMLDomain); + + test.AddAttribute("cast_to", cast_to); + test.AddAttribute("map_form", map_form); + test.AddAttribute("max_map", max_map); + + std::map input{{0, 1.0f}}; + std::vector output{1.0f}; + + test.AddInput("X", input); + + std::vector dims{1, gsl::narrow_cast(output.size())}; + test.AddOutput("Y", dims, output); + + test.Run(expect_result); +} + +// test invalid attributes are detected +TEST(CastMap, InvalidAttributes) { + // bad cast_to + RunBadAttributeTest("UNKNOWN", "DENSE", -1, OpTester::ExpectResult::kExpectFailure); + + // bad map_form + RunBadAttributeTest("TO_FLOAT", "UNKNOWN", -1, OpTester::ExpectResult::kExpectFailure); + + // bad max_map + RunBadAttributeTest("TO_FLOAT", "SPARSE", -2, OpTester::ExpectResult::kExpectFailure); +} + +TEST(CastMap, InvalidIndexInMap) { + // negative index values in map aren't allowed + std::map map{{-3, "-3"}, {0, "0"}}; + std::vector output{0, 0}; + + RunTest(map, output, "TO_INT64", 5, OpTester::ExpectResult::kExpectFailure); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/ml/category_mapper_test.cc b/onnxruntime/test/providers/cpu/ml/category_mapper_test.cc new file mode 100644 index 0000000000000..6f08d74a3fc65 --- /dev/null +++ b/onnxruntime/test/providers/cpu/ml/category_mapper_test.cc @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +template +static void RunTest(const std::vector& dims, const std::vector& input, const std::vector& output) { + OpTester test("CategoryMapper", 1, onnxruntime::kMLDomain); + + static const std::vector categories = {"Three", "Two", "One"}; + static const std::vector indexes = {3, 2, 1}; + + test.AddAttribute("cats_strings", categories); + test.AddAttribute("cats_int64s", indexes); + + test.AddAttribute("default_string", "default"); + test.AddAttribute("default_int64", 99); + + test.AddInput("X", dims, input); + test.AddOutput("Y", dims, output); + + test.Run(); +} + +TEST(CategoryMapper, StringToInt) { + std::vector dims{2, 2, 2}; + + std::vector input{"Unknown", "Two", "Three", "B", "A", "One", "one", "two"}; + std::vector output{99, 2, 3, 99, 99, 1, 99, 99}; + + RunTest(dims, input, output); +} + +TEST(CategoryMapper, IntToString) { + std::vector dims{2, 3}; + + std::vector input{1, 2, 3, 4, 5, 6}; + std::vector output{"One", "Two", "Three", "default", "default", "default"}; + + RunTest(dims, input, output); +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/ml/dictvectorizer_test.cc b/onnxruntime/test/providers/cpu/ml/dictvectorizer_test.cc new file mode 100644 index 0000000000000..4432855808614 --- /dev/null +++ b/onnxruntime/test/providers/cpu/ml/dictvectorizer_test.cc @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(MLOpTest, DictVectorizerStringInput) { + OpTester test("DictVectorizer", 1, onnxruntime::kMLDomain); + + test.AddAttribute("string_vocabulary", std::vector{"a", "b", "c", "d"}); + + std::map map; + map["a"] = 1; + map["c"] = 2; + map["d"] = 3; + + test.AddInput("X", map); + + std::vector dims{1, 4}; + test.AddOutput("Y", dims, + {1, 0, 2, 3}); + test.Run(); +} + +TEST(MLOpTest, DictVectorizerInt64Input) { + OpTester test("DictVectorizer", 1, onnxruntime::kMLDomain); + + test.AddAttribute("int64_vocabulary", std::vector{1, 2, 3, 4}); + + std::map map; + map[1] = "a"; + map[3] = "c"; + map[4] = "d"; + + test.AddInput("X", map); + + std::vector dims{1, 4}; + test.AddOutput("Y", dims, {"a", "", "c", "d"}); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/ml/feature_vectorizer_test.cc b/onnxruntime/test/providers/cpu/ml/feature_vectorizer_test.cc new file mode 100644 index 0000000000000..6695efe93390e --- /dev/null +++ b/onnxruntime/test/providers/cpu/ml/feature_vectorizer_test.cc @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(FeatureVectorizer, BasicFunctionality) { + OpTester test("FeatureVectorizer", 1, onnxruntime::kMLDomain); + + test.AddAttribute("inputdimensions", std::vector{3, 2, 1, 4}); + + std::vector input0_dims = {1, 3}; + test.AddInput("X0", input0_dims, {1, 2, 3}); + + std::vector input1_dims = {1, 2}; + test.AddInput("X1", input1_dims, {4, 5}); + + std::vector input2_dims = {1}; + test.AddInput("X2", input2_dims, {6}); + + std::vector input3_dims = {1, 4}; + test.AddInput("X3", input3_dims, {7, 8, 9, 10}); + + test.AddOutput("Y", std::vector{1, 10}, + {1.f, 2.f, 3.f, 4.f, 5.f, 6.f, 7.f, 8.f, 9.f, 10.f}); + test.Run(); +} + +TEST(FeatureVectorizer, HandleInputDimensionMismatch) { + OpTester test("FeatureVectorizer", 1, onnxruntime::kMLDomain); + + test.AddAttribute("inputdimensions", std::vector{2, 3}); + + std::vector input0_dims = {1, 3}; // long - ignore extra + test.AddInput("X0", input0_dims, {1, 2, 3}); + + std::vector input1_dims = {1, 2}; // short - pad with 0.f + test.AddInput("X1", input1_dims, {1, 2}); + + test.AddOutput("Y", std::vector{1, 5}, {1.f, 2.f, 1.f, 2.f, 0.f}); + + test.Run(); +} + +// test with batch size of 2. +TEST(FeatureVectorizer, Batch) { + OpTester test("FeatureVectorizer", 1, onnxruntime::kMLDomain); + + test.AddAttribute("inputdimensions", std::vector{2, 2}); + + std::vector input0_dims = {2, 2}; + test.AddInput("X0", input0_dims, {1., 2., 3., 4.}); + + std::vector input1_dims = {2, 2}; + test.AddInput("X1", input1_dims, {10., 11., 12., 13.}); + + test.AddOutput("Y", std::vector{2, 4}, + {1.f, 2.f, 10.f, 11.f, + 3.f, 4.f, 12.f, 13.f}); + + test.Run(); +} + +// test with batch size of 2. +TEST(FeatureVectorizer, BatchWith3DInput) { + OpTester test("FeatureVectorizer", 1, onnxruntime::kMLDomain); + + test.AddAttribute("inputdimensions", std::vector{2, 4}); + + std::vector input0_dims = {2, 2}; + test.AddInput("X0", input0_dims, {1., 2., 3., 4.}); + + std::vector input1_dims = {2, 2, 2}; + test.AddInput("X1", input1_dims, + {10., 11., 12., 13., + 14., 15., 16., 17.}); + + test.AddOutput("Y", std::vector{2, 6}, + {1.f, 2.f, 10.f, 11.f, 12.f, 13.f, + 3.f, 4.f, 14.f, 15.f, 16.f, 17.f}); + + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/ml/imputer_test.cc b/onnxruntime/test/providers/cpu/ml/imputer_test.cc new file mode 100644 index 0000000000000..964efce562401 --- /dev/null +++ b/onnxruntime/test/providers/cpu/ml/imputer_test.cc @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(MLOpTest, ImputerOpFloat) { + OpTester test("Imputer", 1, onnxruntime::kMLDomain); + const int N = 5; + std::vector impute = {10.0f}; + float replace = 1.f; + test.AddAttribute("imputed_value_floats", impute); + test.AddAttribute("replaced_value_float", replace); + std::vector X = {0.8f, -0.5f, 0.0f, 1.f, 1.0f}; + + // setup expected output + std::vector expected_output; + for (auto& elem : X) { + if (elem == replace) { + expected_output.push_back(impute[0]); + } else { + expected_output.push_back(elem); + } + } + test.AddInput("X", {N}, X); + test.AddOutput("Y", {N}, expected_output); + test.Run(); +} + +TEST(MLOpTest, ImputerOpInts) { + OpTester test("Imputer", 1, onnxruntime::kMLDomain); + std::vector impute = {10, 20, 30, 40, 50}; + int64_t replace = 2; + test.AddAttribute("imputed_value_int64s", impute); + test.AddAttribute("replaced_value_int64", replace); + std::vector X = {2, 0, 2, 1, 1}; + + // setup expected output + std::vector expected_output; + int impute_idx = 0; + for (auto& elem : X) { + if (elem == replace) { + expected_output.push_back(impute[impute_idx]); + } else { + expected_output.push_back(elem); + } + ++impute_idx; + } + std::vector dims{1, 5}; + test.AddInput("X", dims, X); + test.AddOutput("Y", dims, expected_output); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/ml/label_encoder_test.cc b/onnxruntime/test/providers/cpu/ml/label_encoder_test.cc new file mode 100644 index 0000000000000..c52f43d682abb --- /dev/null +++ b/onnxruntime/test/providers/cpu/ml/label_encoder_test.cc @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +template +static void RunTest(const std::vector& dims, const std::vector& input, const std::vector& output) { + OpTester test("LabelEncoder", 1, onnxruntime::kMLDomain); + + static const std::vector labels = {"Beer", "Wine", "Tequila"}; + + test.AddAttribute("classes_strings", labels); + + test.AddAttribute("default_string", "Water"); + test.AddAttribute("default_int64", 99); + + test.AddInput("X", dims, input); + test.AddOutput("Y", dims, output); + + test.Run(); +} + +TEST(LabelEncoder, StringToInt) { + std::vector dims{2, 2, 2}; + + std::vector input{"Beer", "Burger", "Tequila", "Burrito", "Wine", "Cheese", "Tequila", "Floor"}; + std::vector output{0, 99, 2, 99, 1, 99, 2, 99}; + + RunTest(dims, input, output); +} + +TEST(LabelEncoder, IntToString) { + std::vector dims{2, 3}; + + std::vector input{0, 10, 2, 3, 1, -1}; + std::vector output{"Beer", "Water", "Tequila", "Water", "Wine", "Water"}; + + RunTest(dims, input, output); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/ml/linearclassifer_test.cc b/onnxruntime/test/providers/cpu/ml/linearclassifer_test.cc new file mode 100644 index 0000000000000..04fb84039f07b --- /dev/null +++ b/onnxruntime/test/providers/cpu/ml/linearclassifer_test.cc @@ -0,0 +1,147 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(MLOpTest, LinearClassifierMulticlass) { + OpTester test("LinearClassifier", 1, onnxruntime::kMLDomain); + + std::vector coefficients = {-0.22562418f, 0.34188559f, 0.68346153f, -0.68051993f, -0.1975279f, 0.03748541f}; + std::vector classes = {1, 2, 3}; + int64_t multi_class = 0; + std::vector X = {1.f, 0.f, 3.f, 44.f, 23.f, 11.3f}; + + //three estimates, for 3 points each, so 9 predictions + std::vector predictions = {-4.14164229f, 1.1092185f, -0.06021539f, 10.45007543f, -27.46673545f, 1.19408663f, -5.24206713f, 8.45549693f, -3.98224414f}; + std::vector intercepts = {-3.91601811f, 0.42575697f, 0.13731251f}; + std::vector predicted_class = {2, 1, 2}; + + test.AddAttribute("coefficients", coefficients); + test.AddAttribute("intercepts", intercepts); + test.AddAttribute("classlabels_ints", classes); + test.AddAttribute("multi_class", multi_class); + + test.AddInput("X", {3, 2}, X); + test.AddOutput("Y", {3}, predicted_class); + test.AddOutput("Z", {3, 3}, predictions); + + test.Run(); +} + +TEST(MLOpTest, LinearClassifierMulticlassProb) { + OpTester test("LinearClassifier", 1, onnxruntime::kMLDomain); + + std::vector coefficients = {-0.22562418f, 0.34188559f, 0.68346153f, -0.68051993f, -0.1975279f, 0.03748541f}; + std::vector classes = {1, 2, 3}; + std::vector X = {1.f, 0.f, 3.f, 44.f, 23.f, 11.3f}; + + //three estimates, for 3 points each, so 9 predictions + std::vector predictions = {-4.14164229f, 1.1092185f, -0.06021539f, 10.45007543f, -27.46673545f, 1.19408663f, -5.24206713f, 8.45549693f, -3.98224414f}; + std::vector intercepts = {-3.91601811f, 0.42575697f, 0.13731251f}; + std::vector predicted_class = {2, 1, 2}; + + test.AddAttribute("coefficients", coefficients); + test.AddAttribute("intercepts", intercepts); + test.AddAttribute("classlabels_ints", classes); + + test.AddInput("X", {3, 2}, X); + test.AddOutput("Y", {3}, predicted_class); + test.AddOutput("Z", {3, 3}, predictions); + test.SetOutputAbsErr("Z", 0.00001f); + test.Run(); +} + +TEST(MLOpTest, LinearClassifierMulticlassProbSigmoid) { + OpTester test("LinearClassifier", 1, onnxruntime::kMLDomain); + + std::vector coefficients = {-0.22562418f, 0.34188559f, 0.68346153f, -0.68051993f, -0.1975279f, 0.03748541f}; + std::vector classes = {1, 2, 3}; + std::vector X = {1.f, 0.f, 3.f, 44.f, 23.f, 11.3f}; + + //three estimates, for 3 points each, so 9 predictions + std::vector predictions = {0.015647972f, 0.751983387f, 0.484950699f, 0.999971055f, 1.17855E-12f, 0.767471158f, 0.005261482f, 0.999787317f, 0.018302525f}; + std::vector intercepts = {-3.91601811f, 0.42575697f, 0.13731251f}; + std::vector predicted_class = {2, 1, 2}; + + std::string trans("LOGISTIC"); + test.AddAttribute("coefficients", coefficients); + test.AddAttribute("intercepts", intercepts); + test.AddAttribute("classlabels_ints", classes); + test.AddAttribute("post_transform", trans); + + test.AddInput("X", {3, 2}, X); + test.AddOutput("Y", {3}, predicted_class); + test.AddOutput("Z", {3, 3}, predictions); + test.SetOutputAbsErr("Z", 0.0001f); + test.Run(); +} + +TEST(MLOpTest, LinearClassifierBinary) { + OpTester test("LinearClassifier", 1, onnxruntime::kMLDomain); + + std::vector coefficients = {0.00085401f, -0.00314063f}; + std::vector X = {1.f, 0.f, 3.f, 44.f, 23.f, 11.3f}; + std::vector intercepts = {0.03930598f}; + std::vector predicted_class = {1, 0, 1}; + std::vector scores = {0.0401599929f, -0.0963197052f, 0.0234590918f}; + + test.AddAttribute("coefficients", coefficients); + test.AddAttribute("intercepts", intercepts); + + test.AddInput("X", {3, 2}, X); + test.AddOutput("Y", {3}, predicted_class); + test.AddOutput("Z", {3, 1}, scores); + test.Run(); +} + +TEST(MLOpTest, LinearClassifierBinaryWithLabels) { + OpTester test("LinearClassifier", 1, onnxruntime::kMLDomain); + + std::vector coefficients = {0.00085401f, -0.00314063f}; + std::vector X = {1.f, 0.f, 3.f, 44.f, 23.f, 11.3f}; + std::vector intercepts = {0.03930598f}; + std::vector labels = {"not_so_good", "pretty_good"}; + std::vector predicted_class = {"pretty_good", "not_so_good", "pretty_good"}; + std::vector scores = {0.959840000f, 0.0401599929f, 1.09631968f, -0.0963197052f, 0.976540923f, 0.0234590918f}; + + test.AddAttribute("coefficients", coefficients); + test.AddAttribute("intercepts", intercepts); + test.AddAttribute("classlabels_strings", labels); + + test.AddInput("X", {3, 2}, X); + test.AddOutput("Y", {3}, predicted_class); + test.AddOutput("Z", {3, 2}, scores); + test.Run(); +} + +TEST(MLOpTest, LinearClassifierMulticlassInt64Input) { + OpTester test("LinearClassifier", 1, onnxruntime::kMLDomain); + + std::vector coefficients = {-0.22562418f, 0.34188559f, 0.68346153f, -0.68051993f, -0.1975279f, 0.03748541f}; + std::vector classes = {1, 2, 3}; + int64_t multi_class = 0; + std::vector X = {1, 0, 3, 44, 23, 11}; + + //three estimates, for 3 points each, so 9 predictions + std::vector predictions = {-4.14164229f, 1.1092185f, -0.06021539f, 10.45007543f, -27.46673545f, 1.19408663f, -5.3446321487426758f, 8.6596536636352539f, -3.9934897422790527}; + std::vector intercepts = {-3.91601811f, 0.42575697f, 0.13731251f}; + std::vector predicted_class = {2, 1, 2}; + + test.AddAttribute("coefficients", coefficients); + test.AddAttribute("intercepts", intercepts); + test.AddAttribute("classlabels_ints", classes); + test.AddAttribute("multi_class", multi_class); + + test.AddInput("X", {3, 2}, X); + test.AddOutput("Y", {3}, predicted_class); + test.AddOutput("Z", {3, 3}, predictions); + + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/ml/linearregressor_test.cc b/onnxruntime/test/providers/cpu/ml/linearregressor_test.cc new file mode 100644 index 0000000000000..c5e75ecdafce8 --- /dev/null +++ b/onnxruntime/test/providers/cpu/ml/linearregressor_test.cc @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(MLOpTest, LinearRegressorUniTarget) { + /* TEST MODEL TRAINING + from sklearn import linear_model + X = [[0., 0.5], [1., 1.5], [2., 2.9], [3., 13.3]] + Z = [[41.], [32.], [23.], [14.]] + model = linear_model.LinearRegression() + r3 =model.fit(X, Z) + r4 =model.predict ([[1, 0.],[3.,44.],[23.,11.3]]) + r3.coef_ + array([-9.00000000e+00,  -1.99600736e-16]) + + r3.intercept_ + array([4.10000000e+01]) + + r4 + array([[32.], [14.], [-166.]]) + */ + + OpTester test("LinearRegressor", 1, onnxruntime::kMLDomain); + std::vector coefficients = {-9.00000000f, -1.99600736e-16f}; + std::vector intercepts = {41.0000000f}; + test.AddAttribute("intercepts", intercepts); + test.AddAttribute("coefficients", coefficients); + + test.AddInput("X", {3, 2}, {1.f, 0.f, 3.f, 44.f, 23.f, 11.3f}); + test.AddOutput("Y", {3, 1}, {32.0f, 14.0f, -166.0f}); + test.Run(); +} + +TEST(MLOpTest, LinearRegressorMultiTarget) { + /* TEST MODEL TRAINING + from sklearn import linear_model + X = [[0., 0.5], [1., 1.5], [2., 2.9], [3., 13.3]] + Z = [[0., 41.], [1., 32.], [2., 23.], [3., 14.]] + model = linear_model.LinearRegression() + r3 =model.fit(X, Z) + r4 =model.predict ([[1, 0.],[3.,44.],[23.,11.3]]) + r3.coef_ + array([[  1.00000000e+00,  -2.49500920e-17], + [ -9.00000000e+00,  -1.99600736e-16]]) + r3.intercept_ + array([  2.22044605e-16,   4.10000000e+01]) + r4 + array([[   1.,   32.], [   3.,   14.], [  23., -166.]]) + */ + + OpTester test("LinearRegressor", 1, onnxruntime::kMLDomain); + std::vector coefficients = {1.00000000f, -2.49500920e-17f, -9.00000000f, -1.99600736e-16f}; + std::vector intercepts = {2.22044605e-16f, 41.0000000f}; + test.AddAttribute("intercepts", intercepts); + test.AddAttribute("coefficients", coefficients); + int64_t targets = 2; + test.AddAttribute("targets", targets); + + test.AddInput("X", {3, 2}, {1.f, 0.f, 3.f, 44.f, 23.f, 11.3f}); + test.AddOutput("Y", {3, 2}, {1.0f, 32.0f, 3.0f, 14.0f, 23.0f, -166.0f}); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/ml/normalizer_test.cc b/onnxruntime/test/providers/cpu/ml/normalizer_test.cc new file mode 100644 index 0000000000000..5faf012c349bb --- /dev/null +++ b/onnxruntime/test/providers/cpu/ml/normalizer_test.cc @@ -0,0 +1,182 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" +using namespace std; +namespace onnxruntime { +namespace test { + +template +static void RunTest(const vector& input, + const vector& dims, + const vector& output, + const std::string& norm, + OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess, + const std::string& expect_error_message = "") { + OpTester test("Normalizer", 1, onnxruntime::kMLDomain); + + test.AddAttribute("norm", norm); + + test.AddInput("X", dims, input); + test.AddOutput("Y", dims, output); + + test.Run(expect_result, expect_error_message); +} + +template +static void RunTests(const vector& input, + const vector& dims, + const vector& max_output, + const vector& l1_output, + const vector& l2_output) { + RunTest(input, dims, max_output, "MAX"); + RunTest(input, dims, l1_output, "L1"); + RunTest(input, dims, l2_output, "L2"); +} + +/* +Test values from this script, which is based on functions in +https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/preprocessing/data.py + +import numpy as np + +def _handle_zeros_in_scale(scale): + ''' Makes sure that whenever scale is zero, we handle it correctly. + This happens in most scalers when we have constant features.''' + + # if we are fitting on 1D arrays, scale might be a scalar + if np.isscalar(scale): + if scale == .0: + scale = 1. + return scale + elif isinstance(scale, np.ndarray): + scale[scale == 0.0] = 1.0 + return scale + + +def normalize(X, norm='l2', axis=1): + if norm not in ('l1', 'l2', 'max'): + raise ValueError("'%s' is not a supported norm" % norm) + + if norm == 'l1': + norms = np.abs(X).sum(axis=axis).astype(np.float32) + elif norm == 'l2': + norms = np.sqrt((X * X).sum(axis=axis)).astype(np.float32) + elif norm == 'max': + norms = np.max(X, axis=axis).astype(np.float32) + + norms = _handle_zeros_in_scale(norms) + + x_float = X.astype(np.float32) + + # special case single dimension input + if np.isscalar(norms) == 1: + x_float /= norms + else: + x_float /= norms[:, np.newaxis] + + return x_float + + +def RunNormalize(X, axis=1): + print("Normalizing") + print(X) + print("\nmax") + y = normalize(X, 'max', axis) + print(y) + print("\nL1") + y = normalize(X, 'l1', axis) + print(y) + print("\nL2") + y = normalize(X, 'l2', axis) + print(y) + print("\n\n") + + +x = np.array([-1, 0, 1]).astype(np.float32) +RunNormalize(x, 0) + +np.random.seed(123) +x = np.random.randn(2, 3).astype(np.double) +RunNormalize(x) + +x = (100 * np.random.randn(2, 3, 4)).astype(np.int) +RunNormalize(x, 1) + +*/ +TEST(Normalizer, SingleDimensionFloat) { + std::vector dims = {3}; + std::vector input = {-1.f, 0.f, 1.f}; + + std::vector max_output{-1.0f, 0.f, 1.0f}; + std::vector l1_output{-0.5f, 0.f, 0.5f}; + std::vector l2_output{-0.70710677f, 0.f, 0.70710677f}; + + RunTests(input, dims, max_output, l1_output, l2_output); +} + +TEST(Normalizer, TwoDimensionDouble) { + std::vector dims = {2, 3}; + std::vector input = {-1.0856306, 0.99734545, 0.2829785, + -1.50629471, -0.57860025f, 1.65143654}; + + std::vector max_output{-1.0885202f, 1.f, 0.2837317f, + -0.91211176f, -0.35036176f, 1.f}; + + std::vector l1_output{-0.45885524f, 0.42154038f, 0.11960436f, + -0.40314806f, -0.15485784f, 0.44199413f}; + + std::vector l2_output{-0.7232126f, 0.6643998f, 0.18851127f, + -0.65239084f, -0.25059736f, 0.7152532f}; + + RunTests(input, dims, max_output, l1_output, l2_output); +} + +TEST(Normalizer, ThreeDimensionInt32) { + std::vector dims = {2, 3, 4}; + std::vector input = {-242, -42, 126, -86, + -67, -9, 149, -63, + -44, -43, 220, 218, + + 100, 38, 73, 149, + -93, 117, -125, -63, + 90, -142, -14, -86}; + + std::vector max_output{5.5f, 4.6666665f, 0.57272726f, -0.39449543f, + 1.5227273f, 1.f, 0.67727274f, -0.28899083f, + 1.f, 4.7777777f, 1.f, 1.f, + + 1.f, 0.32478634f, 1.f, 1.f, + -0.93f, 1.f, -1.7123288f, -0.42281878f, + 0.9f, -1.2136753f, -0.19178082f, -0.5771812f}; + + std::vector l1_output{-0.6855524f, -0.44680852f, 0.25454545f, -0.23433243f, + -0.1898017f, -0.09574468f, 0.3010101f, -0.17166212f, + -0.12464589f, -0.4574468f, 0.44444445f, 0.59400547f, + + 0.3533569f, 0.12794612f, 0.3443396f, 0.5f, + -0.3286219f, 0.3939394f, -0.5896226f, -0.21140939f, + 0.3180212f, -0.4781145f, -0.06603774f, -0.2885906f}; + + std::vector l2_output{-0.94928247f, -0.6910363f, 0.4284698f, -0.3543899f, + -0.26281786f, -0.1480792f, 0.5066825f, -0.25961122f, + -0.17259681f, -0.7074895f, 0.74812186f, 0.89833724f, + + 0.6114293f, 0.2022622f, 0.5019584f, 0.8132732f, + -0.56862926f, 0.62275463f, -0.85951775f, -0.34386718f, + 0.55028635f, -0.7558219f, -0.09626599f, -0.469406f}; + + RunTests(input, dims, max_output, l1_output, l2_output); +} + +TEST(Normalizer, InvalidNorm) { + std::vector dims = {3}; + std::vector input = {-1.f, 0.f, 1.f}; + std::vector output{-1.0f, 0.f, 1.0f}; + + RunTest(input, dims, output, "InvalidNormValue", OpTester::ExpectResult::kExpectFailure); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/ml/onehotencoder_test.cc b/onnxruntime/test/providers/cpu/ml/onehotencoder_test.cc new file mode 100644 index 0000000000000..05af2af22c218 --- /dev/null +++ b/onnxruntime/test/providers/cpu/ml/onehotencoder_test.cc @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" +using namespace std; +namespace onnxruntime { +namespace test { + +template +void TestIntCategory(std::vector& input) { + std::vector categories{0, 1, 2, 3, 4, 5, 6, 7}; + std::vector expected_output; + for (size_t i = 0; i < input.size(); ++i) + for (size_t j = 0; j < categories.size(); ++j) + if (static_cast(input[i]) != categories[j]) + expected_output.push_back(0.0); + else + expected_output.push_back(1.0); + + // Test Matrix [Batch * Labels] + OpTester test_matrix("OneHotEncoder", 1, onnxruntime::kMLDomain); + test_matrix.AddAttribute("cats_int64s", categories); + test_matrix.AddInput("X", {1, 7}, input); + test_matrix.AddOutput("Y", {1, 7, 8}, expected_output); + + test_matrix.AddAttribute("zeros", int64_t{1}); + test_matrix.Run(); + + test_matrix.AddAttribute("zeros", int64_t{0}); + test_matrix.Run(OpTester::ExpectResult::kExpectFailure); + + // Test Vector [Labels] + OpTester test_vector("OneHotEncoder", 1, onnxruntime::kMLDomain); + test_vector.AddAttribute("cats_int64s", categories); + test_vector.AddInput("X", {7}, input); + test_vector.AddOutput("Y", {7, 8}, expected_output); + + test_vector.AddAttribute("zeros", int64_t{1}); + test_vector.Run(); + + test_vector.AddAttribute("zeros", int64_t{0}); + test_vector.Run(OpTester::ExpectResult::kExpectFailure); +} + +TEST(OneHotEncoderOpTest, IntegerWithInt64) { + vector input{8, 1, 0, 0, 3, 7, 4}; + TestIntCategory(input); +} + +/* +TEST(OneHotEncoderOpTest, IntegerWithInt32) { + vector input{ 8, 1, 0, 0, 3, 7, 4 }; + TestIntCategory(input); +} + +TEST(OneHotEncoderOpTest, IntegerWithDouble) { + vector input{ 8.1f, 1.2f, 0.0f, 0.7f, 3.4f, 7.9f, 4.4f }; + TestIntCategory(input); +} + +*/ +TEST(OneHotEncoderOpTest, String) { + std::vector categories{"Apple", "Orange", "Watermelon", "Blueberry", "Coconut", "Mango", "Tangerine"}; + vector input{"Watermelon", "Orange", "Tangerine", "Apple", "Kit"}; + vector expected_output; + + for (size_t i = 0; i < input.size(); ++i) + for (size_t j = 0; j < categories.size(); ++j) + if (input[i] != categories[j]) + expected_output.push_back(0.0); + else + expected_output.push_back(1.0); + + // Test Matrix [Batch, Labels] + OpTester test_matrix("OneHotEncoder", 1, onnxruntime::kMLDomain); + test_matrix.AddAttribute("cats_strings", categories); + test_matrix.AddInput("X", {1, 5}, input); + test_matrix.AddOutput("Y", {1, 5, 7}, expected_output); + + test_matrix.AddAttribute("zeros", int64_t{1}); + test_matrix.Run(); + + test_matrix.AddAttribute("zeros", int64_t{0}); + test_matrix.Run(OpTester::ExpectResult::kExpectFailure); + + // Test Vector [Labels] + OpTester test_vector("OneHotEncoder", 1, onnxruntime::kMLDomain); + test_vector.AddAttribute("cats_strings", categories); + test_vector.AddInput("X", {5}, input); + test_vector.AddOutput("Y", {5, 7}, expected_output); + + test_vector.AddAttribute("zeros", int64_t{1}); + test_vector.Run(); + + test_vector.AddAttribute("zeros", int64_t{0}); + test_vector.Run(OpTester::ExpectResult::kExpectFailure); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/ml/scaler_test.cc b/onnxruntime/test/providers/cpu/ml/scaler_test.cc new file mode 100644 index 0000000000000..39e616069f95e --- /dev/null +++ b/onnxruntime/test/providers/cpu/ml/scaler_test.cc @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" +using namespace std; +namespace onnxruntime { +namespace test { + +template +void TestScalar() { + OpTester test("Scaler", 1, onnxruntime::kMLDomain); + vector scale{3.f, -4.f, 3.0f}; + vector offset{4.8f, -0.5f, 77.0f}; + test.AddAttribute("scale", scale); + test.AddAttribute("offset", offset); + vector input{1, -2, 3, 4, 5, -6}; + vector dims{2, 3}; + + // prepare expected output + vector expected_output; + for (size_t i = 0; i < input.size(); ++i) { + expected_output.push_back((static_cast(input[i]) - offset[i % dims[1]]) * scale[i % dims[1]]); + } + + test.AddInput("X", dims, input); + test.AddOutput("Y", dims, expected_output); + test.Run(); +} + +TEST(MLOpTest, ScalerOp) { + TestScalar(); + TestScalar(); + TestScalar(); + TestScalar(); +} + +TEST(MLOpTest, ScalerOpScaleOffsetSize1) { + OpTester test("Scaler", 1, onnxruntime::kMLDomain); + vector scale{3.f}; + vector offset{4.8f}; + test.AddAttribute("scale", scale); + test.AddAttribute("offset", offset); + vector input{0.8f, -0.5f, 0.0f, 0.8f, 1.0f, 1.0f}; + vector dims{2, 3}; + + // prepare expected output + vector expected_output; + for (size_t i = 0; i < input.size(); ++i) { + expected_output.push_back((input[i] - offset[0]) * scale[0]); + } + + test.AddInput("X", dims, input); + test.AddOutput("Y", dims, expected_output); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/ml/svmclassifier_test.cc b/onnxruntime/test/providers/cpu/ml/svmclassifier_test.cc new file mode 100644 index 0000000000000..9b75ebe5616b2 --- /dev/null +++ b/onnxruntime/test/providers/cpu/ml/svmclassifier_test.cc @@ -0,0 +1,119 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(MLOpTest, SVMClassifierMulticlassSVC) { + OpTester test("SVMClassifier", 1, onnxruntime::kMLDomain); + + std::vector dual_coefficients = {1.14360327f, 1.95968249f, -1.175683f, -1.92760275f, -1.32575698f, -1.32575698f, 0.66332785f, 0.66242913f, 0.53120854f, 0.53510444f, -1.06631298f, -1.06631298f, 0.66332785f, 0.66242913f, 0.53120854f, 0.53510444f, 1.f, -1.f}; + std::vector support_vectors = {0.f, 0.5f, 32.f, 2.f, 2.9f, -32.f, 1.f, 1.5f, 1.f, 3.f, 13.3f, -11.f, 12.f, 12.9f, -312.f, 43.f, 413.3f, -114.f}; + std::vector classes = {0, 1, 2, 3}; + std::vector vectors_per_class = {2, 2, 1, 1}; + std::vector rho = {0.5279583f, 0.32605162f, 0.32605162f, 0.06663721f, 0.06663721f, 0.f}; + std::vector kernel_params = {0.001f, 0.f, 3.f}; //gamma, coef0, degree + + std::vector X = {1.f, 0.0f, 0.4f, 3.0f, 44.0f, -3.f, 12.0f, 12.9f, -312.f, 23.0f, 11.3f, -222.f, 23.0f, 11.3f, -222.f, 23.0f, 3311.3f, -222.f, 23.0f, 11.3f, -222.f, 43.0f, 413.3f, -114.f}; + std::vector predictions = {1, 1, 2, 0, 0, 0, 0, 3}; + std::vector scores = { + -0.956958294f, 0.799815655f, 0.799815655f, 0.988598406f, 0.988598406f, 0, + -0.159782529f, 0.407864451f, 0.407864451f, 0.347750872f, 0.347750872f, 0, + 0.527958274f, -0.999705434f, 0.326051623f, -0.999675810f, 0.0666372105f, 1.00000000f, + 0.527958274f, 0.325695992f, 0.326051623f, 0.0663511604f, 0.0666372105f, 0.000268258271f, + 0.527958274f, 0.325695992f, 0.326051623f, 0.0663511604f, 0.0666372105f, 0.000268258271f, + 0.527958274f, 0.326051623f, 0.326051623f, 0.0666372105f, 0.0666372105f, 0, + 0.527958274f, 0.325695992f, 0.326051623f, 0.0663511604f, 0.0666372105f, 0.000268258271f, + 0.527958274f, 0.326051623f, -0.999705434f, 0.0666372105f, -0.999675810f, -1.00000000f}; + + test.AddAttribute("kernel_type", std::string("RBF")); + test.AddAttribute("coefficients", dual_coefficients); + test.AddAttribute("support_vectors", support_vectors); + test.AddAttribute("vectors_per_class", vectors_per_class); + test.AddAttribute("rho", rho); + test.AddAttribute("kernel_params", kernel_params); + test.AddAttribute("classlabels_ints", classes); + + test.AddInput("X", {8, 3}, X); + test.AddOutput("Y", {8}, predictions); + test.AddOutput("Z", {8, 6}, scores); + + test.Run(); +} + +TEST(MLOpTest, SVMClassifierMulticlassLinearSVC) { + OpTester test("SVMClassifier", 1, onnxruntime::kMLDomain); + + std::vector dual_coefficients = {-1.55181212e-01f, 2.42698956e-01f, 7.01893432e-03f, 4.07614474e-01f, -3.24927823e-02f, 2.79897536e-04f, -1.95771302e-01f, -3.52437368e-01f, -2.15973096e-02f, -4.38190277e-01f, 4.56869105e-02f, -1.29375499e-02f}; + std::vector classes = {0, 1, 2, 3}; + std::vector rho = {-0.07489691f, -0.1764396f, -0.21167431f, -0.51619097f}; + std::vector kernel_params = {0.001f, 0.f, 3.f}; //gamma, coef0, degree + + std::vector X = {1.f, 0.0f, 0.4f, 3.0f, 44.0f, -3.f, 12.0f, 12.9f, -312.f, 23.0f, 11.3f, -222.f, 23.0f, 11.3f, -222.f, 23.0f, 3311.3f, -222.f, 23.0f, 11.3f, -222.f, 43.0f, 413.3f, -114.f}; + std::vector predictions = {1, 0, 1, 1, 1, 0, 1, 0}; + std::vector scores = { + -0.227270544f, 0.332829535f, -0.279307127f, -0.518262208f, + 10.1172562f, -0.282575697f, -16.1046638f, 0.659568906f, + -0.996162534f, 4.30999184f, -0.232234091f, -0.707304120f, + -2.45976996f, 8.87092972f, -3.76557732f, -6.76487541f, + -2.45976996f, 8.87092972f, -3.76557732f, -6.76487541f, + 798.446777f, -98.3552551f, -1166.80896f, 144.001923f, + -2.45976996f, 8.87092972f, -3.76557732f, -6.76487541f, + 92.7596283f, 3.99134970f, -151.693329f, 1.44020212f}; + + test.AddAttribute("kernel_type", std::string("RBF")); + test.AddAttribute("coefficients", dual_coefficients); + test.AddAttribute("rho", rho); + test.AddAttribute("kernel_params", kernel_params); + test.AddAttribute("classlabels_ints", classes); + + test.AddInput("X", {8, 3}, X); + test.AddOutput("Y", {8}, predictions); + test.AddOutput("Z", {8, 4}, scores); + + test.Run(); +} + +TEST(MLOpTest, SVMClassifierSVCProbabilities) { + OpTester test("SVMClassifier", 1, onnxruntime::kMLDomain); + + std::vector coefficients = {1.14360327f, 1.95968249f, -1.175683f, -1.92760275f, -1.32575698f, -1.32575698f, 0.66332785f, 0.66242913f, 0.53120854f, 0.53510444f, -1.06631298f, -1.06631298f, 0.66332785f, 0.66242913f, 0.53120854f, 0.53510444f, 1.f, -1.f}; + std::vector support_vectors = {0.f, 0.5f, 32.f, 2.f, 2.9f, -32.f, 1.f, 1.5f, 1.f, 3.f, 13.3f, -11.f, 12.f, 12.9f, -312.f, 43.f, 413.3f, -114.f}; + std::vector rho = {0.5279583f, 0.32605162f, 0.32605162f, 0.06663721f, 0.06663721f, 0.f}; + std::vector kernel_params = {0.001f, 0.f, 3.f}; //gamma, coef0, degree + std::vector proba = {-3.8214362f, 1.82177748f, 1.82177748f, 7.17655643f, 7.17655643f, 0.69314718f}; + std::vector probb = {-1.72839673e+00f, -1.12863030e+00f, -1.12863030e+00f, -6.48340925e+00f, -6.48340925e+00f, 2.39189538e-16f}; + std::vector classes = {0, 1, 2, 3}; + std::vector vectors_per_class = {2, 2, 1, 1}; + + std::vector X = {1.f, 0.0f, 0.4f, 3.0f, 44.0f, -3.f, 12.0f, 12.9f, -312.f, 23.0f, 11.3f, -222.f, 23.0f, 11.3f, -222.f}; + std::vector prob_predictions = { + 0.13766955f, 0.21030431f, 0.32596754f, 0.3260586f, + 0.45939931f, 0.26975416f, 0.13539588f, 0.13545066f, + 0.71045899f, 0.07858939f, 0.05400437f, 0.15694726f, + 0.58274772f, 0.10203105f, 0.15755227f, 0.15766896f, + 0.58274772f, 0.10203105f, 0.15755227f, 0.15766896f}; + std::vector class_predictions = {1, 1, 2, 0, 0}; + + test.AddAttribute("kernel_type", std::string("RBF")); + test.AddAttribute("coefficients", coefficients); + test.AddAttribute("support_vectors", support_vectors); + test.AddAttribute("vectors_per_class", vectors_per_class); + test.AddAttribute("rho", rho); + test.AddAttribute("kernel_params", kernel_params); + test.AddAttribute("classlabels_ints", classes); + test.AddAttribute("prob_a", proba); + test.AddAttribute("prob_b", probb); + + test.AddInput("X", {5, 3}, X); + test.AddOutput("Y", {5}, class_predictions); + test.AddOutput("Z", {5, 4}, prob_predictions); + + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/ml/svmregressor_test.cc b/onnxruntime/test/providers/cpu/ml/svmregressor_test.cc new file mode 100644 index 0000000000000..3df00417615b2 --- /dev/null +++ b/onnxruntime/test/providers/cpu/ml/svmregressor_test.cc @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(MLOpTest, SVMRegressorSVC) { + OpTester test("SVMRegressor", 1, onnxruntime::kMLDomain); + + std::vector dual_coefficients = {-1.54236563f, 0.53485162f, -1.5170623f, 0.69771864f, 1.82685767f}; + std::vector support_vectors = {0.f, 0.5f, 32.f, 1.f, 1.5f, 1.f, 2.f, 2.9f, -32.f, 12.f, 12.9f, -312.f, 43.f, 413.3f, -114.f}; + std::vector rho = {1.96292297f}; + std::vector kernel_params = {0.001f, 0.f, 3.f}; //gamma, coef0, degree + + //three estimates, for 3 points each, so 9 predictions + std::vector X = {1.f, 0.0f, 0.4f, 3.0f, 44.0f, -3.f, 12.0f, 12.9f, -312.f, 23.0f, 11.3f, -222.f, 23.0f, 11.3f, -222.f, 23.0f, 3311.3f, -222.f, 23.0f, 11.3f, -222.f, 43.0f, 413.3f, -114.f}; + std::vector predictions = {1.40283655f, 1.86065906f, 2.66064161f, 1.96311014f, 1.96311014f, 1.96292297f, 1.96311014f, 3.78978065f}; + + test.AddAttribute("kernel_type", std::string("RBF")); + test.AddAttribute("coefficients", dual_coefficients); + test.AddAttribute("support_vectors", support_vectors); + test.AddAttribute("rho", rho); + test.AddAttribute("kernel_params", kernel_params); + test.AddAttribute("n_supports", static_cast(5)); + + test.AddInput("X", {8, 3}, X); + test.AddOutput("Y", {8, 1}, predictions); + + test.Run(); +} + +TEST(MLOpTest, SVMRegressorNuSVC) { + OpTester test("SVMRegressor", 1, onnxruntime::kMLDomain); + + std::vector dual_coefficients = {-1.7902966f, 1.05962596f, -1.54324389f, -0.43658884f, 0.79025169f, 1.92025169f}; + std::vector support_vectors = {0.f, 0.5f, 32.f, 1.f, 1.5f, 1.f, 2.f, 2.9f, -32.f, 3.f, 13.3f, -11.f, 12.f, 12.9f, -312.f, 43.f, 413.3f, -114.f}; + std::vector rho = {1.96923464f}; + std::vector kernel_params = {0.001f, 0.f, 3.f}; //gamma, coef0, degree + + //three estimates, for 3 points each, so 9 predictions + std::vector X = {1.f, 0.0f, 0.4f, 3.0f, 44.0f, -3.f, 12.0f, 12.9f, -312.f, 23.0f, 11.3f, -222.f, 23.0f, 11.3f, -222.f, 23.0f, 3311.3f, -222.f, 23.0f, 11.3f, -222.f, 43.0f, 413.3f, -114.f}; + std::vector predictions = {1.51230766f, 1.77893206f, 2.75948633f, 1.96944663f, 1.96944663f, 1.96923464f, 1.96944663f, 3.88948633f}; + + test.AddAttribute("kernel_type", std::string("RBF")); + test.AddAttribute("coefficients", dual_coefficients); + test.AddAttribute("support_vectors", support_vectors); + test.AddAttribute("rho", rho); + test.AddAttribute("kernel_params", kernel_params); + test.AddAttribute("n_supports", static_cast(6)); + + test.AddInput("X", {8, 3}, X); + test.AddOutput("Y", {8, 1}, predictions); + + test.Run(); +} + +TEST(MLOpTest, SVMRegressorNuSVCPolyKernel) { + OpTester test("SVMRegressor", 1, onnxruntime::kMLDomain); + + std::vector dual_coefficients = {-2.74322388e+01f, 5.81893108e+01f, -1.00000000e+02f, 6.91693781e+01f, 7.62161261e-02f, -2.66618042e-03f}; + std::vector support_vectors = {0.f, 0.5f, 32.f, 1.f, 1.5f, 1.f, 2.f, 2.9f, -32.f, 3.f, 13.3f, -11.f, 12.f, 12.9f, -312.f, 43.f, 413.3f, -114.f}; + std::vector rho = {1.5004596f}; + std::vector kernel_params = {0.001f, 0.f, 3.f}; //gamma, coef0, degree + + //three estimates, for 3 points each, so 9 predictions + std::vector X = {1.f, 0.0f, 0.4f, 3.0f, 44.0f, -3.f, 12.0f, 12.9f, -312.f, 23.0f, 11.3f, -222.f, 23.0f, 11.3f, -222.f, 23.0f, 3311.3f, -222.f, 23.0f, 11.3f, -222.f, 43.0f, 413.3f, -114.f}; + std::vector predictions = {1.50041863e+00f, 3.49624795e-01f, 2.75850969e+00f, -2.28659294e+02f, -2.28659294e+02f, -6.09640826e+05f, -2.28659294e+02f, 3.89055773e+00f}; + + test.AddAttribute("kernel_type", std::string("POLY")); + test.AddAttribute("coefficients", dual_coefficients); + test.AddAttribute("support_vectors", support_vectors); + test.AddAttribute("rho", rho); + test.AddAttribute("kernel_params", kernel_params); + test.AddAttribute("n_supports", static_cast(6)); + + test.AddInput("X", {8, 3}, X); + test.AddOutput("Y", {8, 1}, predictions); + test.SetOutputRelErr("Y", 0.01f); + test.Run(); +} + +TEST(MLOpTest, SVMRegressorLinear) { + OpTester test("SVMRegressor", 1, onnxruntime::kMLDomain); + std::vector coefficients = {0.28290501f, -0.0266512f, 0.01674867f}; + std::vector rho = {1.24032312f}; + std::vector kernel_params = {0.001f, 0.f, 3.f}; //gamma, coef0, degree + + //three estimates, for 3 points each, so 9 predictions + std::vector X = {1.f, 0.0f, 0.4f, 3.0f, 44.0f, -3.f, 12.0f, 12.9f, -312.f, 23.0f, 11.3f, -222.f, 23.0f, 11.3f, -222.f, 23.0f, 3311.3f, -222.f, 23.0f, 11.3f, -222.f, 43.0f, 413.3f, -114.f}; + std::vector predictions = {1.52992759f, 0.8661395f, -0.93420165f, 3.72777548f, 3.72777548f, -84.22117216f, 3.72777548f, 0.48095091f}; + + test.AddAttribute("kernel_type", std::string("LINEAR")); + test.AddAttribute("coefficients", coefficients); + test.AddAttribute("rho", rho); + test.AddAttribute("kernel_params", kernel_params); + test.AddAttribute("n_supports", static_cast(0)); + + test.AddInput("X", {8, 3}, X); + test.AddOutput("Y", {8, 1}, predictions); + + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/ml/tree_ensembler_classifier_test.cc b/onnxruntime/test/providers/cpu/ml/tree_ensembler_classifier_test.cc new file mode 100644 index 0000000000000..36318226ef7b6 --- /dev/null +++ b/onnxruntime/test/providers/cpu/ml/tree_ensembler_classifier_test.cc @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(MLOpTest, TreeEnsembleClassifier) { + OpTester test("TreeEnsembleClassifier", 1, onnxruntime::kMLDomain); + + std::vector lefts = {1, -1, 3, -1, -1, 1, -1, 3, 4, -1, -1, -1, 1, 2, -1, 4, -1, -1, -1}; + std::vector rights = {2, -1, 4, -1, -1, 2, -1, 6, 5, -1, -1, -1, 6, 3, -1, 5, -1, -1, -1}; + std::vector treeids = {0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2}; + std::vector nodeids = {0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6}; + std::vector featureids = {2, -2, 0, -2, -2, 0, -2, 2, 1, -2, -2, -2, 0, 2, -2, 1, -2, -2, -2}; + std::vector thresholds = {-172.f, -2.f, 2.5f, -2.f, -2.f, 1.5f, -2.f, -62.5f, 213.09999084f, + -2.f, -2.f, -2.f, 27.5f, -172.f, -2.f, 8.10000038f, -2.f, -2.f, -2.f}; + std::vector modes = {"BRANCH_LEQ", "LEAF", "BRANCH_LEQ", "LEAF", "LEAF", "BRANCH_LEQ", + "LEAF", "BRANCH_LEQ", "BRANCH_LEQ", "LEAF", "LEAF", "LEAF", + "BRANCH_LEQ", "BRANCH_LEQ", "LEAF", "BRANCH_LEQ", "LEAF", "LEAF", "LEAF"}; + std::vector class_treeids = {0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2}; + std::vector class_nodeids = {1, 3, 4, 1, 4, 5, 6, 2, 4, 5, 6}; + std::vector class_classids = {2, 0, 1, 0, 2, 3, 1, 2, 0, 1, 3}; + std::vector class_weights = {1.f, 4.f, 1.f, 2.f, 1.f, 1.f, 2.f, 1.f, 1.f, 1.f, 3.f}; + std::vector classes = {0, 1, 2, 3}; + std::vector X = {1.f, 0.0f, 0.4f, 3.0f, 44.0f, -3.f, 12.0f, 12.9f, -312.f, 23.0f, 11.3f, -222.f, 23.0f, + 11.3f, -222.f, 23.0f, 3311.3f, -222.f, 23.0f, 11.3f, -222.f, 43.0f, 413.3f, -114.f}; + std::vector results = {0, 1, 2, 2, 2, 2, 2, 3}; + std::vector scores{7, 0, 0, 0, 0, 4, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, + 0, 0, 3, 0, 0, 0, 2, 1, 0, 0, 3, 0, 0, 1, 0, 4}; + std::vector probs = {}; + std::vector log_probs = {}; + + //define the context of the operator call + const int N = 8; + test.AddAttribute("nodes_truenodeids", lefts); + test.AddAttribute("nodes_falsenodeids", rights); + test.AddAttribute("nodes_treeids", treeids); + test.AddAttribute("nodes_nodeids", nodeids); + test.AddAttribute("nodes_featureids", featureids); + test.AddAttribute("nodes_values", thresholds); + test.AddAttribute("nodes_modes", modes); + test.AddAttribute("class_treeids", class_treeids); + test.AddAttribute("class_nodeids", class_nodeids); + test.AddAttribute("class_ids", class_classids); + test.AddAttribute("class_weights", class_weights); + test.AddAttribute("classlabels_int64s", classes); + + test.AddInput("X", {N, 3}, X); + test.AddOutput("Y", {N}, results); + test.AddOutput("Z", {N, static_cast(classes.size())}, scores); + test.Run(); +} + +TEST(MLOpTest, TreeEnsembleClassifierLabels) { + OpTester test("TreeEnsembleClassifier", 1, onnxruntime::kMLDomain); + + std::vector lefts = {1, -1, 3, -1, -1, 1, -1, 3, 4, -1, -1, -1, 1, 2, -1, 4, -1, -1, -1}; + std::vector rights = {2, -1, 4, -1, -1, 2, -1, 6, 5, -1, -1, -1, 6, 3, -1, 5, -1, -1, -1}; + std::vector treeids = {0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2}; + std::vector nodeids = {0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6}; + std::vector featureids = {2, -2, 0, -2, -2, 0, -2, 2, 1, -2, -2, -2, 0, 2, -2, 1, -2, -2, -2}; + std::vector thresholds = {-172.f, -2.f, 2.5f, -2.f, -2.f, 1.5f, -2.f, -62.5f, 213.09999084f, + -2.f, -2.f, -2.f, 27.5f, -172.f, -2.f, 8.10000038f, -2.f, -2.f, -2.f}; + std::vector modes = {"BRANCH_LEQ", "LEAF", "BRANCH_LEQ", "LEAF", "LEAF", "BRANCH_LEQ", + "LEAF", "BRANCH_LEQ", "BRANCH_LEQ", "LEAF", "LEAF", "LEAF", "BRANCH_LEQ", + "BRANCH_LEQ", "LEAF", "BRANCH_LEQ", "LEAF", "LEAF", "LEAF"}; + + std::vector class_treeids = {0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2}; + std::vector class_nodeids = {1, 3, 4, 1, 4, 5, 6, 2, 4, 5, 6}; + std::vector class_classids = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + std::vector class_weights = {-1.f, 4.f, -1.f, 2.f, -1.f, +1.f, -2.f, 1.f, -1.f, 2.f, -3.f}; + std::vector labels = {"label0", "label1"}; + std::vector X = {1.f, 0.0f, 0.4f, 3.0f, 44.0f, -3.f, 12.0f, 12.9f, -312.f, 23.0f, 11.3f, -222.f, 23.0f, + 11.3f, -222.f, 23.0f, 3311.3f, -222.f, 23.0f, 11.3f, -222.f, 43.0f, 413.3f, -114.f}; + std::vector results = {"label1", "label0", "label0", "label0", "label0", "label1", "label0", "label0"}; + std::vector probs = {}; + std::vector log_probs = {}; + std::vector scores{-5, 5, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -3, 3}; + + //define the context of the operator call + const int N = 8; + test.AddAttribute("nodes_truenodeids", lefts); + test.AddAttribute("nodes_falsenodeids", rights); + test.AddAttribute("nodes_treeids", treeids); + test.AddAttribute("nodes_nodeids", nodeids); + test.AddAttribute("nodes_featureids", featureids); + test.AddAttribute("nodes_values", thresholds); + test.AddAttribute("nodes_modes", modes); + test.AddAttribute("class_treeids", class_treeids); + test.AddAttribute("class_nodeids", class_nodeids); + test.AddAttribute("class_ids", class_classids); + test.AddAttribute("class_weights", class_weights); + test.AddAttribute("classlabels_strings", labels); + + test.AddInput("X", {N, 3}, X); + test.AddOutput("Y", {N}, results); + test.AddOutput("Z", {N, static_cast(labels.size())}, scores); + + test.Run(); +} + +TEST(MLOpTest, TreeEnsembleClassifierBinary) { + OpTester test("TreeEnsembleClassifier", 1, onnxruntime::kMLDomain); + + std::vector lefts = {1, -1, 3, -1, -1, 1, -1, 3, 4, -1, -1, -1, 1, 2, -1, 4, -1, -1, -1}; + std::vector rights = {2, -1, 4, -1, -1, 2, -1, 6, 5, -1, -1, -1, 6, 3, -1, 5, -1, -1, -1}; + std::vector treeids = {0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2}; + std::vector nodeids = {0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6}; + std::vector featureids = {2, -2, 0, -2, -2, 0, -2, 2, 1, -2, -2, -2, 0, 2, -2, 1, -2, -2, -2}; + std::vector thresholds = {-172.f, -2.f, 2.5f, -2.f, -2.f, 1.5f, -2.f, -62.5f, 213.09999084f, -2.f, + -2.f, -2.f, 27.5f, -172.f, -2.f, 8.10000038f, -2.f, -2.f, -2.f}; + std::vector modes = {"BRANCH_LEQ", "LEAF", "BRANCH_LEQ", "LEAF", "LEAF", "BRANCH_LEQ", + "LEAF", "BRANCH_LEQ", "BRANCH_LEQ", "LEAF", "LEAF", "LEAF", + "BRANCH_LEQ", "BRANCH_LEQ", "LEAF", "BRANCH_LEQ", "LEAF", "LEAF", "LEAF"}; + //std::vector classes = {0, 1, 2, 3}; + std::vector class_treeids = {0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2}; + std::vector class_nodeids = {1, 3, 4, 1, 4, 5, 6, 2, 4, 5, 6}; + std::vector class_classids = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + std::vector class_weights = {-1.f, 4.f, -1.f, 2.f, -1.f, +1.f, -2.f, 1.f, -1.f, 2.f, -3.f}; + std::vector classes = {1}; + std::vector X = {1.f, 0.0f, 0.4f, 3.0f, 44.0f, -3.f, 12.0f, 12.9f, -312.f, 23.0f, 11.3f, -222.f, + 23.0f, 11.3f, -222.f, 23.0f, 3311.3f, -222.f, 23.0f, 11.3f, -222.f, 43.0f, 413.3f, + -114.f}; + std::vector results = {1, 0, 0, 0, 0, 1, 0, 0}; + std::vector probs = {}; + std::vector log_probs = {}; + std::vector scores{5, -1, -1, -1, -1, 1, -1, -3}; + + //define the context of the operator call + const int N = 8; + test.AddAttribute("nodes_truenodeids", lefts); + test.AddAttribute("nodes_falsenodeids", rights); + test.AddAttribute("nodes_treeids", treeids); + test.AddAttribute("nodes_nodeids", nodeids); + test.AddAttribute("nodes_featureids", featureids); + test.AddAttribute("nodes_values", thresholds); + test.AddAttribute("nodes_modes", modes); + test.AddAttribute("class_treeids", class_treeids); + test.AddAttribute("class_nodeids", class_nodeids); + test.AddAttribute("class_ids", class_classids); + test.AddAttribute("class_weights", class_weights); + test.AddAttribute("classlabels_int64s", classes); + + test.AddInput("X", {N, 3}, X); + test.AddOutput("Y", {N}, results); + test.AddOutput("Z", {N, 1}, scores); + + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/ml/treeregressor_test.cc b/onnxruntime/test/providers/cpu/ml/treeregressor_test.cc new file mode 100644 index 0000000000000..335c17b856714 --- /dev/null +++ b/onnxruntime/test/providers/cpu/ml/treeregressor_test.cc @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(MLOpTest, TreeRegressorMultiTarget) { + OpTester test("TreeEnsembleRegressor", 1, onnxruntime::kMLDomain); + + //tree + std::vector lefts = {1, 2, -1, -1, -1, 1, -1, 3, -1, -1, 1, -1, -1}; + std::vector rights = {4, 3, -1, -1, -1, 2, -1, 4, -1, -1, 2, -1, -1}; + std::vector treeids = {0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2}; + std::vector nodeids = {0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2}; + std::vector featureids = {2, 1, -2, -2, -2, 0, -2, 2, -2, -2, 1, -2, -2}; + std::vector thresholds = {10.5f, 13.10000038f, -2.f, -2.f, -2.f, 1.5f, -2.f, -213.f, -2.f, -2.f, 13.10000038f, -2.f, -2.f}; + std::vector modes = {"BRANCH_LEQ", "BRANCH_LEQ", "LEAF", "LEAF", "LEAF", "BRANCH_LEQ", "LEAF", "BRANCH_LEQ", "LEAF", "LEAF", "BRANCH_LEQ", "LEAF", "LEAF"}; + + std::vector target_treeids = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2}; + std::vector target_nodeids = {0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 0, 0, 1, 1, 2, 2}; + std::vector target_classids = {0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1}; + std::vector target_weights = {1.5f, 27.5f, 2.25f, 20.75f, 2.f, 23.f, 3.f, 14.f, 0.f, 41.f, 1.83333333f, 24.5f, 0.f, 41.f, 2.75f, 16.25f, 2.f, 23.f, 3.f, 14.f, 2.66666667f, 17.f, 2.f, 23.f, 3.f, 14.f}; + std::vector classes = {0, 1}; + + //test data + std::vector X = {1.f, 0.0f, 0.4f, 3.0f, 44.0f, -3.f, 12.0f, 12.9f, -312.f, 23.0f, 11.3f, -222.f, 23.0f, 11.3f, -222.f, 23.0f, 3311.3f, -222.f, 23.0f, 11.3f, -222.f, 43.0f, 413.3f, -114.f}; + std::vector results = {1.33333333f, 29.f, 3.f, 14.f, 2.f, 23.f, 2.f, 23.f, 2.f, 23.f, 2.66666667f, 17.f, 2.f, 23.f, 3.f, 14.f}; + + //add attributes + test.AddAttribute("nodes_truenodeids", lefts); + test.AddAttribute("nodes_falsenodeids", rights); + test.AddAttribute("nodes_treeids", treeids); + test.AddAttribute("nodes_nodeids", nodeids); + test.AddAttribute("nodes_featureids", featureids); + test.AddAttribute("nodes_values", thresholds); + test.AddAttribute("nodes_modes", modes); + test.AddAttribute("target_treeids", target_treeids); + test.AddAttribute("target_nodeids", target_nodeids); + test.AddAttribute("target_ids", target_classids); + test.AddAttribute("target_weights", target_weights); + + test.AddAttribute("n_targets", (int64_t)2); + test.AddAttribute("aggregate_function", "AVERAGE"); + //fill input data + test.AddInput("X", {8, 3}, X); + test.AddOutput("Y", {8, 2}, results); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/ml/zipmap_test.cc b/onnxruntime/test/providers/cpu/ml/zipmap_test.cc new file mode 100644 index 0000000000000..8ce74d7a2aa8f --- /dev/null +++ b/onnxruntime/test/providers/cpu/ml/zipmap_test.cc @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" +using namespace std; +namespace onnxruntime { +namespace test { +template +void TestHelper(const std::vector& classes, + const std::string& type, + const vector& input_dims, + OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess) { + OpTester test("ZipMap", 1, onnxruntime::kMLDomain); + + std::vector input{1.f, 0.f, 3.f, 44.f, 23.f, 11.3f}; + + if (type == "string") { + test.AddAttribute("classlabels_strings", classes); + } else if (type == "int64_t") { + test.AddAttribute("classlabels_int64s", classes); + } else { + ONNXRUNTIME_THROW("Invalid type: ", type); + } + + int64_t batch_size = (input_dims.size() > 1) ? input_dims[0] : 1; + + // prepare expected output + std::vector> expected_output; + if (expect_result == OpTester::ExpectResult::kExpectSuccess) { + for (int64_t i = 0; i < batch_size; ++i) { + std::map var_map; + for (size_t j = 0; j < classes.size(); ++j) { + var_map.emplace(classes[j], input[i * 3 + j]); + } + expected_output.push_back(var_map); + } + } + + test.AddInput("X", input_dims, input); + test.AddOutput("Z", expected_output); + test.Run(expect_result); +} + +// Positive test cases +TEST(MLOpTest, ZipMapOpStringFloat) { + TestHelper({"class1", "class2", "class3"}, "string", {2, 3}); +} + +TEST(MLOpTest, ZipMapOpInt64Float) { + TestHelper({10, 20, 30}, "int64_t", {2, 3}); +} + +TEST(MLOpTest, ZipMapOpInt64Float1D) { + TestHelper({10, 20, 30, 40, 50, 60}, "int64_t", {6}); +} + +// Negative test cases +TEST(MLOpTest, ZipMapOpStringFloatStrideMoreThanNumLabels) { + TestHelper({"class1", "class2", "class3"}, "string", {1, 6}, OpTester::ExpectResult::kExpectFailure); +} + +TEST(MLOpTest, ZipMapOpStringFloatStrideLessThanNumLabels) { + TestHelper({"class1", "class2", "class3"}, "string", {3, 2}, OpTester::ExpectResult::kExpectFailure); +} + +TEST(MLOpTest, ZipMapOpInt64FloatStrideMoreThanNumLabels) { + TestHelper({10, 20, 30}, "int64_t", {1, 6}, OpTester::ExpectResult::kExpectFailure); +} + +TEST(MLOpTest, ZipMapOpInt64FloatStrideLessThanNumLabels) { + TestHelper({10, 20, 30}, "int64_t", {3, 2}, OpTester::ExpectResult::kExpectFailure); +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/nn/batch_norm_op_test.cc b/onnxruntime/test/providers/cpu/nn/batch_norm_op_test.cc new file mode 100644 index 0000000000000..06bc8e9c0c657 --- /dev/null +++ b/onnxruntime/test/providers/cpu/nn/batch_norm_op_test.cc @@ -0,0 +1,516 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/tensor.h" +#include "core/session/inference_session.h" +#include "test/providers/provider_test_utils.h" + +#include "gtest/gtest.h" +#include "gmock/gmock.h" + +using namespace std; + +namespace onnxruntime { +namespace test { + +using InputDataMap = unordered_map>; +using InputShapesMap = unordered_map>; + +void TestBatchNorm(const InputDataMap& input_data_map, + const InputShapesMap& input_shapes_map, + optional epsilon, + const std::initializer_list& expected_output, + const vector& expected_output_shape, + int64_t spatial_mode = 1, + OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess, + const std::string& err_str = "") { + OpTester test("BatchNormalization"); + if (epsilon.has_value()) { + test.AddAttribute("epsilon", epsilon.value()); + } + test.AddAttribute("spatial", spatial_mode); + test.AddInput("X", input_shapes_map.at("X"), input_data_map.at("X")); + test.AddInput("scale", input_shapes_map.at("scale"), input_data_map.at("scale")); + test.AddInput("B", input_shapes_map.at("B"), input_data_map.at("B")); + test.AddInput("mean", input_shapes_map.at("mean"), input_data_map.at("mean")); + test.AddInput("var", input_shapes_map.at("var"), input_data_map.at("var")); + test.AddOutput("output", expected_output_shape, expected_output); + test.Run(expect_result, err_str); +} + +TEST(BatchNormTest, PositiveTestCase) { + // This input was taken from the SpatialBN_1.pb, SpatialBN_1_input.pb and SpatialBN_1_output.pb files. + vector X{0.329876f, -0.287158f, -0.411425f, 0.473621f, 0.18156f, -0.170596f, -0.329516f, -0.170733f, -0.121664f, 0.4372f, + -0.485668f, 0.218049f, -0.360263f, 0.107016f, 0.45358f, 0.325056f, 0.15995f, 0.098852f, -0.283453f, -0.373051f, + 0.257542f, 0.0614853f, -0.0592363f, 0.434488f, -0.0179583f, 0.398374f, -0.451602f, -0.132009f, -0.174468f, + -0.0247169f, 0.418897f, -0.47159f, -0.131925f, 0.470943f, 0.118357f, 0.155664f, 0.370062f, -0.279229f, 0.240311f, + -0.451034f, 0.249178f, -0.294496f, 0.13683f, -0.0806475f, -0.309849f, -0.450604f, -0.28048f, -0.420197f, -0.433369f}; + vector scale{0.589433f}; + vector B{-0.384622f}; + vector mean{-2.45673f}; + vector var{1.37998f}; + + InputDataMap input_data_map; + input_data_map.insert({"X", X}); + input_data_map.insert({"scale", scale}); + input_data_map.insert({"B", B}); + input_data_map.insert({"mean", mean}); + input_data_map.insert({"var", var}); + + InputShapesMap input_shapes_map; + vector input_shape{1, 1, 7, 7, 1}; + input_shapes_map.insert({"X", input_shape}); + input_shapes_map.insert({"scale", {1}}); + input_shapes_map.insert({"B", {1}}); + input_shapes_map.insert({"mean", {1}}); + input_shapes_map.insert({"var", {1}}); + + auto expected_output = {1.01359f, 0.703983f, 0.641631f, 1.08571f, 0.939167f, 0.762469f, 0.682729f, 0.762401f, 0.787021f, + 1.06744f, 0.604378f, 0.957476f, 0.667302f, 0.901764f, 1.07566f, 1.01117f, 0.928324f, 0.897667f, + 0.705842f, 0.660885f, 0.977291f, 0.878918f, 0.818345f, 1.06608f, 0.839057f, 1.04796f, 0.621471f, + 0.781831f, 0.760527f, 0.835665f, 1.05825f, 0.611442f, 0.781873f, 1.08437f, 0.907454f, 0.926173f, + 1.03375f, 0.707961f, 0.968646f, 0.621757f, 0.973095f, 0.700301f, 0.916723f, 0.807602f, 0.692598f, + 0.621972f, 0.707334f, 0.63723f, 0.63062f}; + float epsilon = 1e-05f; + TestBatchNorm(input_data_map, input_shapes_map, epsilon, expected_output, input_shape); +} + +TEST(BatchNormTest, PositiveTestCaseDefaultEpsilon) { + // This input was taken from the SpatialBN_1.pb, SpatialBN_1_input.pb and SpatialBN_1_output.pb files. + vector X{0.329876f, -0.287158f, -0.411425f, 0.473621f, 0.18156f, -0.170596f, -0.329516f, -0.170733f, -0.121664f, 0.4372f, + -0.485668f, 0.218049f, -0.360263f, 0.107016f, 0.45358f, 0.325056f, 0.15995f, 0.098852f, -0.283453f, -0.373051f, + 0.257542f, 0.0614853f, -0.0592363f, 0.434488f, -0.0179583f, 0.398374f, -0.451602f, -0.132009f, -0.174468f, + -0.0247169f, 0.418897f, -0.47159f, -0.131925f, 0.470943f, 0.118357f, 0.155664f, 0.370062f, -0.279229f, 0.240311f, + -0.451034f, 0.249178f, -0.294496f, 0.13683f, -0.0806475f, -0.309849f, -0.450604f, -0.28048f, -0.420197f, -0.433369f}; + vector scale{0.589433f}; + vector B{-0.384622f}; + vector mean{-2.45673f}; + vector var{1.37998f}; + + InputDataMap input_data_map; + input_data_map.insert({"X", X}); + input_data_map.insert({"scale", scale}); + input_data_map.insert({"B", B}); + input_data_map.insert({"mean", mean}); + input_data_map.insert({"var", var}); + + InputShapesMap input_shapes_map; + vector input_shape{1, 1, 7, 7, 1}; + input_shapes_map.insert({"X", input_shape}); + input_shapes_map.insert({"scale", {1}}); + input_shapes_map.insert({"B", {1}}); + input_shapes_map.insert({"mean", {1}}); + input_shapes_map.insert({"var", {1}}); + + auto expected_output = {1.01359f, 0.703983f, 0.641631f, 1.08571f, 0.939167f, 0.762469f, 0.682729f, 0.762401f, 0.787021f, + 1.06744f, 0.604378f, 0.957476f, 0.667302f, 0.901764f, 1.07566f, 1.01117f, 0.928324f, 0.897667f, + 0.705842f, 0.660885f, 0.977291f, 0.878918f, 0.818345f, 1.06608f, 0.839057f, 1.04796f, 0.621471f, + 0.781831f, 0.760527f, 0.835665f, 1.05825f, 0.611442f, 0.781873f, 1.08437f, 0.907454f, 0.926173f, + 1.03375f, 0.707961f, 0.968646f, 0.621757f, 0.973095f, 0.700301f, 0.916723f, 0.807602f, 0.692598f, + 0.621972f, 0.707334f, 0.63723f, 0.63062f}; + optional epsilon; + TestBatchNorm(input_data_map, input_shapes_map, epsilon, expected_output, input_shape); +} + +TEST(BatchNormTest, BatchNorm1d_3d_Pytorch) { + vector X{0.948241f, 1.23591f, -0.39321f, 1.4254f, -0.730771f, 0.439872f, 0.0265089f, 0.8748f, + -0.197505f, 0.962646f, 0.421469f, 1.94512f, 0.234179f, -0.931897f, -0.214905f, -0.982965f, + -0.495436f, 0.81949f, -0.796605f, -0.758605f, 0.665557f, 0.0909539f, 1.10448f, 1.91214f, -1.97433f, + -2.26429f, -0.384419f, -0.226564f, 0.230568f, 0.533968f, -1.31382f, -0.156257f, 0.532323f, + -0.16714f, 0.971087f, 0.600249f, 0.858778f, 0.423108f, -0.414433f, -1.17608f, 0.673753f, 0.278517f, + -2.19044f, -0.161453f, 1.17092f, -0.155138f, -0.094729f, 0.19479f, -1.17344f, -0.213813f, 0.118659f, + -2.39525f, 0.257687f, 0.784609f, 0.297942f, 1.10277f, -1.58026f, 0.197625f, 0.0432784f, 1.12924f}; + vector scale{0.36102f, 0.592982f, 0.808513f, 0.0531484f, 0.0960613f}; + vector B{0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; + vector mean{0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; + vector var{1.0f, 1.0f, 1.0f, 1.0f, 1.0f}; + + InputDataMap input_data_map; + input_data_map.insert({"X", X}); + input_data_map.insert({"scale", scale}); + input_data_map.insert({"B", B}); + input_data_map.insert({"mean", mean}); + input_data_map.insert({"var", var}); + + InputShapesMap input_shapes_map; + vector input_shape{4, 5, 3}; + input_shapes_map.insert({"X", input_shape}); + input_shapes_map.insert({"scale", {5}}); + input_shapes_map.insert({"B", {5}}); + input_shapes_map.insert({"mean", {5}}); + input_shapes_map.insert({"var", {5}}); + + auto expected_output = {0.342332f, 0.446184f, -0.141956f, 0.845231f, -0.433332f, 0.260835f, 0.0214327f, 0.707284f, + -0.159685f, 0.0511628f, 0.0224003f, 0.10338f, 0.0224955f, -0.0895188f, -0.020644f, -0.354868f, + -0.178861f, 0.295851f, -0.47237f, -0.449836f, 0.394661f, 0.073537f, 0.892985f, 1.54598f, + -0.104932f, -0.120343f, -0.0204312f, -0.021764f, 0.0221486f, 0.0512934f, -0.474312f, + -0.0564117f, 0.192178f, -0.0991106f, 0.575834f, 0.355935f, 0.69433f, 0.342087f, -0.335073f, + -0.0625065f, 0.0358087f, 0.0148027f, -0.210415f, -0.0155093f, 0.11248f, -0.0560077f, -0.0341989f, + 0.0703226f, -0.695826f, -0.126787f, 0.0703623f, -1.93658f, 0.208342f, 0.634363f, 0.0158351f, + 0.0586101f, -0.0839879f, 0.018984f, 0.00415736f, 0.108476f}; + float epsilon = 1e-05f; + TestBatchNorm(input_data_map, input_shapes_map, epsilon, expected_output, input_shape); +} + +TEST(BatchNormTest, BatchNorm2d_Pytorch) { + vector X{-0.91221f, -0.283559f, 0.937637f, 2.09818f, -0.100199f, -0.608113f, 0.444562f, -1.07505f, 0.940591f, + -0.922262f, 0.0931303f, 0.69611f, 1.55187f, 0.159808f, 0.914874f, -1.24856f, -1.98928f, -0.331621f, + 2.33131f, 0.260409f, 0.0944811f, 0.442397f, 0.76461f, -0.203334f, -0.244228f, -0.387267f, -1.65039f, + -0.815409f, 0.931696f, -1.15328f, 0.773952f, -1.28195f, -0.437349f, 0.0644882f, -0.087637f, 1.74999f, + 0.640154f, -0.505641f, -1.84014f, -0.00135415f, 0.782006f, -1.21172f, -0.621273f, -0.0977471f, + -0.941333f, -0.170302f, 0.18923f, 0.436322f, 0.870412f, -0.582312f, 0.679017f, 0.510252f, 0.0786005f, + 0.160138f, -2.61889f, 0.402828f, 0.551144f, -1.39366f, -1.15191f, 0.160008f, -0.57735f, 0.210758f, + 1.0541f, -2.12569f, 0.101656f, 1.10223f, 0.725811f, -1.5019f, -0.0892582f, 0.063546f, 0.822734f, + 1.67707f, 0.478121f, -1.07438f, -0.0487855f, 0.0972885f, -1.54122f, 2.47422f, 0.596108f, 0.0026957f, + -0.967677f, -2.08882f, 0.469692f, 0.630784f, 0.196915f, -1.91331f, 1.26255f, 0.0491993f, -0.358415f, + 0.720588f, 0.976776f, -0.418116f, 1.70979f, 2.49971f, 1.30942f, -1.18304f, -1.64901f, -1.11048f, + 1.41467f, -0.275486f, -1.20602f, -0.545566f, -0.918059f, 1.48513f, 2.04224f, -0.96909f, -1.92804f, + 0.634147f, -1.02079f, -0.000786079f, 0.72428f, 0.893569f, 1.14604f, -1.3423f, -1.05061f, -0.617524f, + -0.12619f, -0.203127f, -0.941956f, 2.06916f, 2.03025f, 0.37269f, -0.340471f, -1.27962f, 0.159472f, + 0.643999f, 0.881773f, -0.50873f, 1.04599f, -0.287968f, 1.84344f, -0.728637f, 0.668021f, -2.00452f, + -0.585523f, -0.24982f, -0.379091f, 0.213692f, 1.21336f, -0.499157f, -1.50841f, -1.01256f, 0.745338f, + 0.591107f, 1.33781f, -0.258927f, -1.87304f, 0.884799f, 1.63174f, 0.500887f, 1.80608f, -1.25441f, + -0.655316f, 1.22439f, 0.384174f, 0.401395f, 1.43172f, 1.85338f, -0.644909f, -1.46975f, -1.06138f, + 1.09724f, 0.013438f, 0.589742f, -0.695768f, 0.758401f, 0.924533f, -0.0988563f, -0.197066f, 1.01118f, + 0.195163f, 0.975466f, 1.7682f, 0.977977f, -0.88963f, -0.251431f, 0.115828f, -0.230065f, -1.08882f, + 1.62318f, 0.502684f, 0.789724f, 1.13057f, -0.890021f, -0.614755f, 1.11055f, -1.21681f, 0.133085f, + 0.564458f, 0.723117f, 1.67088f, -0.111012f, 1.39732f, -0.846095f, -0.194408f, -0.381931f, -0.735943f, + -0.788814f, -0.910318f, 1.16345f, -1.98542f, 0.742905f, -0.749476f, -0.110805f, 0.307949f, -1.66811f, + 0.294031f, -0.522837f, -0.774399f, -0.264072f, -0.426894f, 0.965971f, 0.173348f, -0.991018f, 1.9406f, + 0.0853744f}; + vector scale{0.736494f, 0.580251f, 0.374834f}; + vector B{0.0f, 0.0f, 0.0f}; + vector mean{0.0f, 0.0f, 0.0f}; + vector var{1.0f, 1.0f, 1.0f}; + + InputDataMap input_data_map; + input_data_map.insert({"X", X}); + input_data_map.insert({"scale", scale}); + input_data_map.insert({"B", B}); + input_data_map.insert({"mean", mean}); + input_data_map.insert({"var", var}); + + InputShapesMap input_shapes_map; + vector input_shape{2, 3, 6, 6}; + input_shapes_map.insert({"X", input_shape}); + input_shapes_map.insert({"scale", {3}}); + input_shapes_map.insert({"B", {3}}); + input_shapes_map.insert({"mean", {3}}); + input_shapes_map.insert({"var", {3}}); + + auto expected_output = {-0.671834f, -0.208838f, 0.69056f, 1.54529f, -0.0737958f, -0.447869f, 0.327415f, -0.791764f, + 0.692736f, -0.679237f, 0.0685895f, 0.512678f, 1.14294f, 0.117697f, 0.673796f, -0.91955f, + -1.46508f, -0.244235f, 1.71699f, 0.191789f, 0.0695843f, 0.325821f, 0.563128f, -0.149753f, + -0.179871f, -0.285218f, -1.2155f, -0.60054f, 0.686185f, -0.84938f, 0.570008f, -0.944146f, + -0.322103f, 0.0474949f, -0.0645437f, 1.28885f, 0.371448f, -0.293397f, -1.06774f, -0.00078574f, + 0.453757f, -0.703098f, -0.360492f, -0.0567175f, -0.546206f, -0.0988174f, 0.1098f, 0.253175f, + 0.505055f, -0.337885f, 0.393998f, 0.296073f, 0.0456077f, 0.0929198f, -1.5196f, 0.23374f, 0.3198f, + -0.808671f, -0.668391f, 0.0928441f, -0.335006f, 0.122292f, 0.611642f, -1.23343f, 0.0589855f, + 0.639564f, 0.42115f, -0.871472f, -0.0517919f, 0.0368724f, 0.477389f, 0.973116f, 0.179215f, + -0.402711f, -0.0182864f, 0.0364668f, -0.577699f, 0.927416f, 0.22344f, 0.00101043f, -0.362716f, + -0.782957f, 0.176056f, 0.236438f, 0.0738101f, -0.717172f, 0.473243f, 0.0184415f, -0.134346f, + 0.2701f, 0.366127f, -0.156723f, 0.640883f, 0.936974f, 0.490814f, -0.443443f, -0.618102f, + -0.416243f, 0.530263f, -0.103261f, -0.452055f, -0.204496f, -0.344118f, 0.556675f, 0.765497f, + -0.363246f, -0.722693f, 0.237699f, -0.751804f, -0.00057894f, 0.533425f, 0.658105f, 0.844047f, + -0.988591f, -0.773767f, -0.4548f, -0.0929374f, -0.149601f, -0.693741f, 1.52392f, 1.49526f, + 0.274482f, -0.250754f, -0.942429f, 0.11745f, 0.474299f, 0.649417f, -0.374675f, 0.770361f, + -0.212086f, 1.35768f, -0.536634f, 0.491991f, -1.47631f, -0.431231f, -0.18399f, -0.279197f, + 0.157382f, 0.89363f, -0.367624f, -1.11093f, -0.745738f, 0.548934f, 0.435344f, 0.776261f, + -0.150242f, -1.08683f, 0.513403f, 0.946813f, 0.290638f, 1.04797f, -0.72787f, -0.380246f, + 0.710451f, 0.222916f, 0.232908f, 0.830753f, 1.07542f, -0.374207f, -0.852818f, -0.615864f, + 0.63667f, 0.00779735f, 0.342196f, -0.403718f, 0.44006f, 0.536458f, -0.0573611f, -0.114347f, + 0.586737f, 0.113243f, 0.566012f, 1.02599f, 0.567469f, -0.516206f, -0.145892f, 0.0672092f, + -0.133495f, -0.631785f, 0.941846f, 0.188422f, 0.296014f, 0.423775f, -0.333609f, -0.23043f, + 0.416269f, -0.456101f, 0.0498845f, 0.211577f, 0.271048f, 0.626298f, -0.0416111f, 0.523762f, + -0.317144f, -0.0728705f, -0.14316f, -0.275855f, -0.295673f, -0.341217f, 0.436097f, -0.7442f, + 0.278465f, -0.280928f, -0.0415335f, 0.115429f, -0.625263f, 0.110212f, -0.195976f, -0.29027f, + -0.0989828f, -0.160014f, 0.362077f, 0.0649763f, -0.371465f, 0.727401f, 0.0320011f}; + float epsilon = 1e-05f; + TestBatchNorm(input_data_map, input_shapes_map, epsilon, expected_output, input_shape); +} + +TEST(BatchNormTest, BatchNorm3d_Pytorch) { + vector X{2.02384f, -0.935186f, 0.488569f, -0.513934f, -1.27082f, -0.131913f, -1.806f, -0.37904f, 0.667796f, + -1.14826f, 1.2522f, 0.0300339f, 2.4758f, 1.55511f, 0.385341f, 1.46645f, -1.09355f, -2.56309f, + 0.976015f, -1.47036f, 0.89486f, 0.580989f, -1.12418f, -0.339189f, 1.3314f, 0.418893f, -0.301401f, + -1.2983f, -0.839063f, 0.170261f, 1.15486f, -0.255735f, -0.589851f, -0.416289f, -0.952648f, -0.360487f, + 0.253287f, 0.437195f, 0.32023f, 0.209606f, -0.279519f, -0.546527f, 0.265286f, -1.07383f, -1.65879f, + 1.1222f, 0.946612f, 0.822549f, 0.64689f, -0.292639f, -0.73995f, -0.694949f, 1.33899f, -0.0652476f, + 1.61791f, 1.49692f, -0.761145f, -0.201874f, -1.15431f, -1.83111f, -0.705267f, -0.143026f, -0.129819f, + -0.799425f, 0.168795f, 0.740422f, -0.377683f, 0.432598f, -2.07414f, -2.85251f, 0.273531f, 0.0532606f, + 1.31052f, -0.769382f, 0.9976f, 0.850536f, -1.53812f, -0.00496016f, 0.931242f, 0.0517056f, -0.497829f, + 0.275869f, 0.860001f, 1.23747f, 0.179686f, 1.5914f, 0.740327f, 0.798208f, 2.12478f, 1.74205f, + -0.322054f, -0.0112451f, 0.204525f, -0.431252f, -1.3114f, 0.186204f, 0.780569f, -1.42994f, 1.63344f, + -0.00839034f, -0.187035f, 1.8406f, 1.32053f, -0.636963f, 0.408944f, -1.50846f, -1.2076f, -0.129118f, + -0.0441307f, 1.47558f, 1.07251f, 1.05295f, -0.420297f, -1.13402f, -0.524053f, 3.20754f, -0.588935f, + -0.527549f, 0.591928f, -1.10529f, 0.520412f, 0.19404f, -1.21229f, -0.399594f, -0.280935f, -0.363324f, + -0.00804771f, 1.43102f, -0.523222f, 1.17608f, -0.53195f, 0.914993f, 2.69308f, -0.517211f, 0.472273f, + -0.464725f, -0.929768f, -0.631145f, 0.919709f, -0.27391f, 1.76689f, 0.894897f, 0.235798f, 1.2544f, + 0.858985f, -0.139707f, 0.354544f, 0.200878f, 0.353255f, 0.0722632f, -1.56074f, 1.03685f, 1.73434f, + 0.193269f, -0.864609f, 0.842739f, -0.372717f, 0.584484f, 0.16315f, 1.60674f, -0.0611289f, -1.24544f, + 1.33361f, -0.961942f, -0.15732f, -0.348637f, 0.361842f, 0.7386f, 0.517256f, 1.20406f, -2.07277f, + -1.01983f, -1.9163f, 0.239934f, 0.177979f, 0.464564f, 0.988822f, 0.284607f, -1.56099f, -0.429143f, + 0.111043f, -0.0853688f, -0.319176f, -0.279777f, 0.520971f, -1.078f, -0.670242f, 0.065652f, 0.468538f, + -0.825062f, 0.370068f, 1.68751f, -1.16928f, -0.411782f, 1.61624f, -0.973004f, 2.64703f, -0.220014f, + -1.43954f, -0.018692f, 1.34982f, -0.95197f, -1.72586f, 1.32725f, 0.280984f, 0.00847463f, 0.512869f, + 0.0378154f, 0.13898f, 0.35758f, -0.084558f, 1.04045f, -1.79933f, 1.3002f, 0.390457f, 1.22267f, 0.959344f, + -0.964296f, -0.0935597f, 0.288953f, -0.158046f, 0.532672f, -0.500988f, 0.25187f, -2.14384f, -0.633315f, + 1.24612f, -1.41525f, 0.36494f, -0.00714732f, -0.608963f, 0.508496f, 0.995365f, 1.21159f, -0.169055f, + -0.968783f, 1.52779f, -0.082381f, 2.2049f, 0.928655f, 0.120245f, 0.911429f, -0.885258f, -1.2072f, + 0.770694f, 2.36621f, 1.08456f, -1.60069f, 0.0345025f, 0.359559f, -0.785411f, 0.466532f, -0.78543f, + 0.024879f, 1.59337f, 1.13718f, -1.27073f, -0.263788f, -1.7702f, 0.203263f, 1.34631f, 1.11914f, + -2.04911f, -0.804137f, 0.466763f, 2.18386f, 1.4689f, 0.898297f, -0.648948f, 0.252202f, 1.12501f, + -0.204563f, 0.124608f, 0.377214f, 0.894327f, -0.249118f, 0.709188f, 0.999397f, -1.4079f, 0.193873f, + 0.657753f, -0.709732f, 1.09897f, -0.145793f, 0.779199f, 0.88378f, -1.2676f, 1.15709f, 0.62295f, + -0.370894f, -0.103268f, -1.55949f, -0.470747f, 0.100394f, 0.422334f, -0.0685312f, -0.434488f, + -0.568974f, -0.256987f, 2.01276f, -0.923322f, -0.613144f, 1.50676f, 0.65756f, 1.20524f, 1.10395f, + -0.975241f, 2.44035f, 1.08276f, 0.330393f, -0.508918f, -1.25545f, 0.189815f, -0.156263f, -0.960866f, + 1.0859f, -0.674478f, 2.76743f, 1.21399f, 1.71666f, -1.73198f, -1.1062f, 0.951285f, -0.713336f, + 1.61586f, 1.96514f, 0.002603f, 0.0953297f, 0.949256f, -1.76552f, 0.372816f, -0.781229f, 1.50532f, + 1.28462f, 1.31116f, 0.731908f, 1.54835f, 0.371081f, 0.409244f, -0.106938f, -1.79396f, -1.61198f, + -0.80869f, -1.10381f, 1.1872f, -0.832439f, 0.0755941f, -1.09553f, 0.960059f, 1.44252f, -0.196482f, + -1.07364f, 0.165547f, 0.630078f, 1.56569f, -0.669592f, 1.15974f, 0.0953399f, -0.202313f, 0.812631f, + -0.318567f, -0.16644f, 0.887062f, -0.0264821f, -0.740725f, 0.0797577f, -1.1037f, 0.90236f, 1.13427f, + 0.364186f, -2.01043f, -0.415748f, 0.116046f, 0.369949f, 0.317886f, 0.530332f, 1.48341f, 0.74666f, + -1.64142f, 0.22569f, 1.18015f, 1.31827f, -1.33904f, -0.101125f}; + vector scale{0.241661f, 0.960798f, 0.474727f}; + vector B{0.0f, 0.0f, 0.0f}; + vector mean{0.0f, 0.0f, 0.0f}; + vector var{1.0f, 1.0f, 1.0f}; + + InputDataMap input_data_map; + input_data_map.insert({"X", X}); + input_data_map.insert({"scale", scale}); + input_data_map.insert({"B", B}); + input_data_map.insert({"mean", mean}); + input_data_map.insert({"var", var}); + + InputShapesMap input_shapes_map; + vector input_shape{2, 3, 4, 4, 4}; + input_shapes_map.insert({"X", input_shape}); + input_shapes_map.insert({"scale", {3}}); + input_shapes_map.insert({"B", {3}}); + input_shapes_map.insert({"mean", {3}}); + input_shapes_map.insert({"var", {3}}); + + auto expected_output = {0.489082f, -0.225997f, 0.118068f, -0.124197f, -0.307105f, -0.031878f, -0.436439f, -0.0915989f, + 0.16138f, -0.277489f, 0.302606f, 0.007258f, 0.598301f, 0.375807f, 0.0931215f, 0.354382f, + -0.264267f, -0.619395f, 0.235864f, -0.355328f, 0.216252f, 0.140402f, -0.271669f, -0.0819684f, + 0.321747f, 0.10123f, -0.0728365f, -0.313746f, -0.202768f, 0.0411454f, 0.279085f, -0.0618009f, + -0.142543f, -0.1006f, -0.230217f, -0.0871152f, 0.0612094f, 0.105652f, 0.0773867f, 0.0506533f, + -0.0675486f, -0.132074f, 0.064109f, -0.259501f, -0.400863f, 0.271191f, 0.228758f, 0.198777f, + 0.156327f, -0.0707191f, -0.178816f, -0.167941f, 0.323581f, -0.0157677f, 0.390985f, 0.361745f, + -0.183938f, -0.0487849f, -0.27895f, -0.442507f, -0.170435f, -0.0345637f, -0.031372f, + -0.193189f, 0.162177f, 0.711393f, -0.362876f, 0.415637f, -1.99282f, -2.74067f, 0.262807f, + 0.0511725f, 1.25914f, -0.739217f, 0.958488f, 0.817189f, -1.47782f, -0.00476569f, 0.894731f, + 0.0496784f, -0.478311f, 0.265053f, 0.826283f, 1.18895f, 0.172641f, 1.52901f, 0.711301f, + 0.766913f, 2.04147f, 1.67375f, -0.309427f, -0.0108042f, 0.196507f, -0.414344f, -1.25999f, + 0.178903f, 0.749965f, -1.37387f, 1.5694f, -0.00806138f, -0.179702f, 1.76844f, 1.26875f, + -0.61199f, 0.392911f, -1.44932f, -1.16025f, -0.124055f, -0.0424004f, 1.41773f, 1.03046f, + 1.01167f, -0.403818f, -1.08956f, -0.503507f, 3.08178f, -0.565845f, -0.506866f, 0.56872f, + -1.06196f, 0.500008f, 0.186433f, -1.16476f, -0.383928f, -0.269921f, -0.349079f, -0.00773219f, + 1.37492f, -0.248386f, 0.558316f, -0.25253f, 0.43437f, 1.27847f, -0.245533f, 0.2242f, + -0.220617f, -0.441384f, -0.29962f, 0.436609f, -0.130032f, 0.838785f, 0.424829f, 0.111939f, + 0.595496f, 0.407781f, -0.0663221f, 0.168311f, 0.0953618f, 0.167699f, 0.0343051f, -0.74092f, + 0.492219f, 0.823334f, 0.0917494f, -0.410451f, 0.400069f, -0.176938f, 0.277469f, 0.0774512f, + 0.762761f, -0.0290194f, -0.59124f, 0.6331f, -0.456657f, -0.0746837f, -0.165507f, 0.171775f, + 0.350631f, 0.245554f, 0.571595f, -0.983996f, -0.484139f, -0.909715f, 0.113902f, 0.0844908f, + 0.22054f, 0.469418f, 0.13511f, -0.741041f, -0.203725f, 0.0527148f, -0.0405267f, -0.151521f, + -0.132817f, 0.247318f, -0.511752f, -0.31818f, 0.0311666f, 0.222426f, -0.391677f, 0.17568f, + 0.801104f, -0.282569f, -0.0995112f, 0.39058f, -0.235136f, 0.639682f, -0.0531687f, -0.347878f, + -0.0045171f, 0.326198f, -0.230053f, -0.41707f, 0.320744f, 0.0679025f, 0.00204798f, 0.12394f, + 0.00913847f, 0.0335859f, 0.0864127f, -0.0204343f, 0.251436f, -0.434827f, 0.314206f, 0.0943579f, + 0.295471f, 0.231835f, -0.233032f, -0.0226096f, 0.0698283f, -0.0381934f, 0.128725f, -0.121069f, + 0.060867f, -0.51808f, -0.153047f, 0.301137f, -0.342009f, 0.0881915f, -0.00172722f, -0.147162f, + 0.122883f, 0.24054f, 0.292792f, -0.0408538f, -0.234116f, 0.369206f, -0.0199082f, 0.532835f, + 0.224419f, 0.0290583f, 0.220256f, -0.213931f, -0.291733f, 0.186246f, 0.571817f, 0.262095f, + -0.386822f, 0.00833788f, 0.086891f, -0.189802f, 0.112742f, -0.189807f, 0.00601226f, 0.385054f, + 0.274811f, -1.22091f, -0.253445f, -1.7008f, 0.195294f, 1.29353f, 1.07526f, -1.96877f, -0.772609f, + 0.448463f, 2.09824f, 1.4113f, 0.863078f, -0.623505f, 0.242314f, 1.0809f, -0.196543f, 0.119722f, + 0.362425f, 0.859263f, -0.239351f, 0.681383f, 0.960214f, -1.3527f, 0.186272f, 0.631964f, + -0.681905f, 1.05588f, -0.140077f, 0.748649f, 0.84913f, -1.2179f, 1.11172f, 0.598526f, + -0.356353f, -0.099219f, -1.49835f, -0.452291f, 0.0964582f, 0.405776f, -0.0658444f, + -0.417454f, -0.546667f, -0.246911f, 1.93385f, -0.887121f, -0.589104f, 1.44769f, 0.631779f, + 1.15798f, 1.06067f, -0.937005f, 2.34467f, 1.04031f, 0.31744f, -0.488965f, -1.20623f, 0.182373f, + -0.150136f, -0.923194f, 1.04332f, -0.648034f, 2.65893f, 1.1664f, 1.64935f, -0.822216f, + -0.525139f, 0.451599f, -0.338638f, 0.767087f, 0.932899f, 0.00123571f, 0.0452554f, 0.450635f, + -0.838136f, 0.176985f, -0.370868f, 0.714614f, 0.60984f, 0.622438f, 0.347455f, 0.73504f, + 0.176161f, 0.194278f, -0.0507662f, -0.851639f, -0.765246f, -0.383905f, -0.524005f, 0.563593f, + -0.395179f, 0.0358864f, -0.520076f, 0.455763f, 0.684801f, -0.093275f, -0.509682f, 0.0785892f, + 0.299113f, 0.743272f, -0.317872f, 0.550556f, 0.0452602f, -0.0960432f, 0.385776f, -0.151232f, + -0.079013f, 0.42111f, -0.0125717f, -0.35164f, 0.0378629f, -0.523955f, 0.428372f, 0.538468f, + 0.172888f, -0.954402f, -0.197366f, 0.0550898f, 0.175624f, 0.150908f, 0.251761f, 0.704209f, + 0.354458f, -0.779221f, 0.107141f, 0.560244f, 0.625814f, -0.635675f, -0.0480064f}; + float epsilon = 1e-05f; + TestBatchNorm(input_data_map, input_shapes_map, epsilon, expected_output, input_shape); +} + +TEST(BatchNormTest, InvalidScaleDim) { + vector X{0.329876f, -0.287158f, -0.411425f, 0.473621f, 0.18156f, -0.170596f, -0.329516f, -0.170733f, -0.121664f, 0.4372f, + -0.485668f, 0.218049f, -0.360263f, 0.107016f, 0.45358f, 0.325056f, 0.15995f, 0.098852f, -0.283453f, -0.373051f, + 0.257542f, 0.0614853f, -0.0592363f, 0.434488f, -0.0179583f, 0.398374f, -0.451602f, -0.132009f, -0.174468f, + -0.0247169f, 0.418897f, -0.47159f, -0.131925f, 0.470943f, 0.118357f, 0.155664f, 0.370062f, -0.279229f, 0.240311f, + -0.451034f, 0.249178f, -0.294496f, 0.13683f, -0.0806475f, -0.309849f, -0.450604f, -0.28048f, -0.420197f, -0.433369f}; + vector scale{0.589433f, 0.589433f}; + vector B{-0.384622f}; + vector mean{-2.45673f}; + vector var{1.37998f}; + + InputDataMap input_data_map; + input_data_map.insert({"X", X}); + input_data_map.insert({"scale", scale}); + input_data_map.insert({"B", B}); + input_data_map.insert({"mean", mean}); + input_data_map.insert({"var", var}); + + InputShapesMap input_shapes_map; + input_shapes_map.insert({"X", {1, 1, 7, 7}}); + input_shapes_map.insert({"scale", {1, 2}}); // invalid + input_shapes_map.insert({"B", {1}}); + input_shapes_map.insert({"mean", {1}}); + input_shapes_map.insert({"var", {1}}); + + vector expected_output_shape{1, 1, 7, 7}; + auto expected_output = {1.01359f, 0.703983f, 0.641631f, 1.08571f, 0.939167f, 0.762469f, 0.682729f, 0.762401f, 0.787021f, + 1.06744f, 0.604378f, 0.957476f, 0.667302f, 0.901764f, 1.07566f, 1.01117f, 0.928324f, 0.897667f, + 0.705842f, 0.660885f, 0.977291f, 0.878918f, 0.818345f, 1.06608f, 0.839057f, 1.04796f, 0.621471f, + 0.781831f, 0.760527f, 0.835665f, 1.05825f, 0.611442f, 0.781873f, 1.08437f, 0.907454f, 0.926173f, + 1.03375f, 0.707961f, 0.968646f, 0.621757f, 0.973095f, 0.700301f, 0.916723f, 0.807602f, 0.692598f, + 0.621972f, 0.707334f, 0.63723f, 0.63062f}; + float epsilon = 1e-05f; + TestBatchNorm(input_data_map, + input_shapes_map, + epsilon, + expected_output, + expected_output_shape, 1, + OpTester::ExpectResult::kExpectFailure, + "Invalid input scale"); +} + +TEST(BatchNormTest, InvalidBDim) { + vector X{0.329876f, -0.287158f, -0.411425f, 0.473621f, 0.18156f, -0.170596f, -0.329516f, -0.170733f, -0.121664f, 0.4372f, + -0.485668f, 0.218049f, -0.360263f, 0.107016f, 0.45358f, 0.325056f, 0.15995f, 0.098852f, -0.283453f, -0.373051f, + 0.257542f, 0.0614853f, -0.0592363f, 0.434488f, -0.0179583f, 0.398374f, -0.451602f, -0.132009f, -0.174468f, + -0.0247169f, 0.418897f, -0.47159f, -0.131925f, 0.470943f, 0.118357f, 0.155664f, 0.370062f, -0.279229f, 0.240311f, + -0.451034f, 0.249178f, -0.294496f, 0.13683f, -0.0806475f, -0.309849f, -0.450604f, -0.28048f, -0.420197f, -0.433369f}; + vector scale{0.589433f}; + vector B{-0.384622f, -0.384622f}; + vector mean{-2.45673f}; + vector var{1.37998f}; + + InputDataMap input_data_map; + input_data_map.insert({"X", X}); + input_data_map.insert({"scale", scale}); + input_data_map.insert({"B", B}); + input_data_map.insert({"mean", mean}); + input_data_map.insert({"var", var}); + + InputShapesMap input_shapes_map; + input_shapes_map.insert({"X", {1, 1, 7, 7}}); + input_shapes_map.insert({"scale", {1}}); + input_shapes_map.insert({"B", {1, 2}}); // invalid + input_shapes_map.insert({"mean", {1}}); + input_shapes_map.insert({"var", {1}}); + + vector expected_output_shape{1, 1, 7, 7}; + auto expected_output = {1.01359f, 0.703983f, 0.641631f, 1.08571f, 0.939167f, 0.762469f, 0.682729f, 0.762401f, 0.787021f, + 1.06744f, 0.604378f, 0.957476f, 0.667302f, 0.901764f, 1.07566f, 1.01117f, 0.928324f, 0.897667f, + 0.705842f, 0.660885f, 0.977291f, 0.878918f, 0.818345f, 1.06608f, 0.839057f, 1.04796f, 0.621471f, + 0.781831f, 0.760527f, 0.835665f, 1.05825f, 0.611442f, 0.781873f, 1.08437f, 0.907454f, 0.926173f, + 1.03375f, 0.707961f, 0.968646f, 0.621757f, 0.973095f, 0.700301f, 0.916723f, 0.807602f, 0.692598f, + 0.621972f, 0.707334f, 0.63723f, 0.63062f}; + float epsilon = 1e-05f; + TestBatchNorm(input_data_map, + input_shapes_map, + epsilon, + expected_output, + expected_output_shape, 1, + OpTester::ExpectResult::kExpectFailure, + "Invalid input B"); +} + +TEST(BatchNormTest, InvalidMeanDim) { + vector X{0.329876f, -0.287158f, -0.411425f, 0.473621f, 0.18156f, -0.170596f, -0.329516f, -0.170733f, -0.121664f, 0.4372f, + -0.485668f, 0.218049f, -0.360263f, 0.107016f, 0.45358f, 0.325056f, 0.15995f, 0.098852f, -0.283453f, -0.373051f, + 0.257542f, 0.0614853f, -0.0592363f, 0.434488f, -0.0179583f, 0.398374f, -0.451602f, -0.132009f, -0.174468f, + -0.0247169f, 0.418897f, -0.47159f, -0.131925f, 0.470943f, 0.118357f, 0.155664f, 0.370062f, -0.279229f, 0.240311f, + -0.451034f, 0.249178f, -0.294496f, 0.13683f, -0.0806475f, -0.309849f, -0.450604f, -0.28048f, -0.420197f, -0.433369f}; + vector scale{0.589433f}; + vector B{-0.384622f}; + vector mean{-2.45673f, -2.45673f}; + vector var{1.37998f}; + + InputDataMap input_data_map; + input_data_map.insert({"X", X}); + input_data_map.insert({"scale", scale}); + input_data_map.insert({"B", B}); + input_data_map.insert({"mean", mean}); + input_data_map.insert({"var", var}); + + InputShapesMap input_shapes_map; + input_shapes_map.insert({"X", {1, 1, 7, 7}}); + input_shapes_map.insert({"scale", {1}}); + input_shapes_map.insert({"B", {1}}); + input_shapes_map.insert({"mean", {1, 2}}); // invalid + input_shapes_map.insert({"var", {1}}); + + vector expected_output_shape{1, 1, 7, 7}; + auto expected_output = {1.01359f, 0.703983f, 0.641631f, 1.08571f, 0.939167f, 0.762469f, 0.682729f, 0.762401f, 0.787021f, + 1.06744f, 0.604378f, 0.957476f, 0.667302f, 0.901764f, 1.07566f, 1.01117f, 0.928324f, 0.897667f, + 0.705842f, 0.660885f, 0.977291f, 0.878918f, 0.818345f, 1.06608f, 0.839057f, 1.04796f, 0.621471f, + 0.781831f, 0.760527f, 0.835665f, 1.05825f, 0.611442f, 0.781873f, 1.08437f, 0.907454f, 0.926173f, + 1.03375f, 0.707961f, 0.968646f, 0.621757f, 0.973095f, 0.700301f, 0.916723f, 0.807602f, 0.692598f, + 0.621972f, 0.707334f, 0.63723f, 0.63062f}; + float epsilon = 1e-05f; + TestBatchNorm(input_data_map, + input_shapes_map, + epsilon, + expected_output, + expected_output_shape, 1, + OpTester::ExpectResult::kExpectFailure, + "Invalid input mean"); +} + +TEST(BatchNormTest, InvalidVarDim) { + vector X{0.329876f, -0.287158f, -0.411425f, 0.473621f, 0.18156f, -0.170596f, -0.329516f, -0.170733f, -0.121664f, 0.4372f, + -0.485668f, 0.218049f, -0.360263f, 0.107016f, 0.45358f, 0.325056f, 0.15995f, 0.098852f, -0.283453f, -0.373051f, + 0.257542f, 0.0614853f, -0.0592363f, 0.434488f, -0.0179583f, 0.398374f, -0.451602f, -0.132009f, -0.174468f, + -0.0247169f, 0.418897f, -0.47159f, -0.131925f, 0.470943f, 0.118357f, 0.155664f, 0.370062f, -0.279229f, 0.240311f, + -0.451034f, 0.249178f, -0.294496f, 0.13683f, -0.0806475f, -0.309849f, -0.450604f, -0.28048f, -0.420197f, -0.433369f}; + vector scale{0.589433f}; + vector B{-0.384622f}; + vector mean{-2.45673f}; + vector var{1.37998f, 1.37998f}; + + InputDataMap input_data_map; + input_data_map.insert({"X", X}); + input_data_map.insert({"scale", scale}); + input_data_map.insert({"B", B}); + input_data_map.insert({"mean", mean}); + input_data_map.insert({"var", var}); + + InputShapesMap input_shapes_map; + input_shapes_map.insert({"X", {1, 1, 7, 7}}); + input_shapes_map.insert({"scale", {1}}); + input_shapes_map.insert({"B", {1}}); + input_shapes_map.insert({"mean", {1}}); + input_shapes_map.insert({"var", {1, 2}}); // invalid + + vector expected_output_shape{1, 1, 7, 7}; + auto expected_output = {1.01359f, 0.703983f, 0.641631f, 1.08571f, 0.939167f, 0.762469f, 0.682729f, 0.762401f, 0.787021f, + 1.06744f, 0.604378f, 0.957476f, 0.667302f, 0.901764f, 1.07566f, 1.01117f, 0.928324f, 0.897667f, + 0.705842f, 0.660885f, 0.977291f, 0.878918f, 0.818345f, 1.06608f, 0.839057f, 1.04796f, 0.621471f, + 0.781831f, 0.760527f, 0.835665f, 1.05825f, 0.611442f, 0.781873f, 1.08437f, 0.907454f, 0.926173f, + 1.03375f, 0.707961f, 0.968646f, 0.621757f, 0.973095f, 0.700301f, 0.916723f, 0.807602f, 0.692598f, + 0.621972f, 0.707334f, 0.63723f, 0.63062f}; + float epsilon = 1e-05f; + TestBatchNorm(input_data_map, + input_shapes_map, + epsilon, + expected_output, + expected_output_shape, 1, + OpTester::ExpectResult::kExpectFailure, + "Invalid input var"); +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/nn/conv_op_test.cc b/onnxruntime/test/providers/cpu/nn/conv_op_test.cc new file mode 100644 index 0000000000000..d7edd4dacd9f1 --- /dev/null +++ b/onnxruntime/test/providers/cpu/nn/conv_op_test.cc @@ -0,0 +1,546 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" +using namespace std; +namespace onnxruntime { +namespace test { + +namespace { + +struct ConvOpAttributes { + string auto_pad; + vector dilations; + int64_t group; + vector kernel_shape; + vector pads; + vector strides; +}; + +void TestConvOp(const ConvOpAttributes& attributes, + const vector>& inputs, + const vector>& input_shapes, + const std::initializer_list& expected_output, + const vector& expected_output_shape, + bool is_cuda_supported = true, + bool is_mkldnn_supported = true, + OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess, + const std::string& err_str = "") { + OpTester test("Conv"); + test.AddAttribute("auto_pad", attributes.auto_pad); + test.AddAttribute("dilations", attributes.dilations); + test.AddAttribute("group", attributes.group); + test.AddAttribute("kernel_shape", attributes.kernel_shape); + if (!attributes.pads.empty()) { + test.AddAttribute("pads", attributes.pads); + } + test.AddAttribute("strides", attributes.strides); + + ONNXRUNTIME_ENFORCE(inputs.size() <= 3, "Our name array is only setup to handle 3 inputs"); + const char* szNames[] = {"X", "W", "B"}; + for (size_t i = 0; i < inputs.size(); i++) { + test.AddInput(szNames[i], input_shapes[i], inputs[i]); + } + test.AddOutput("Y", expected_output_shape, expected_output); + std::unordered_set excluded_providers; + if (!is_cuda_supported) { + excluded_providers.insert(kCudaExecutionProvider); + } + if (!is_mkldnn_supported) { + excluded_providers.insert(kMklDnnExecutionProvider); + } + test.Run(expect_result, err_str, excluded_providers); +} + +} // namespace + +// Conv +TEST(ConvTest, Conv1D_1) { + ConvOpAttributes attrs = { + "", // auto_pad + vector{1}, // dilations + 1, // group + vector{1}, // kernel_shape + vector{0, 0}, // pads + vector{1} // strides + }; + vector X = {-0.21559301018714905f, 0.4691687822341919f, 0.4426700472831726f, -0.4517466723918915f, + -0.05216419696807861f, 0.29067182540893555f, 0.251010000705719f}; + vector X_shape = {1, 1, 7}; + vector W = {0.24472862482070923f}; + vector W_shape = {1, 1, 1}; + vector Y_shape = {1, 1, 7}; + auto expected_vals = {-0.052761781960725784f, 0.11481902748346329f, 0.10833403468132019f, -0.11055534332990646f, + -0.012766072526574135f, 0.07113571465015411f, 0.061429332941770554f}; + + TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); // Conv1d not yet optimized for MKLDNN XP +} + +// Conv3 +TEST(ConvTest, Conv1D_2) { + ConvOpAttributes attrs = { + "", // auto_pad + vector{2}, // dilations + 1, // group + vector{2}, // kernel_shape + vector{2, 2}, // pads + vector{2} // strides + }; + vector X = {0.11094123125076294f, -0.0038032233715057373f, 0.3896123170852661f, 0.33259105682373047f, + 0.02794349193572998f, -0.08360505104064941f, -0.4100455045700073f, -0.09502679109573364f, + -0.11361867189407349f, -0.025495320558547974f, 0.3696536421775818f, 0.3529144525527954f, + -0.34991076588630676f, -0.22024285793304443f, 0.23085933923721313f, -0.4575521945953369f, + -0.17685726284980774f, -0.06030535697937012f, -0.3996139168739319f, -0.19385704398155212f, + -0.10454908013343811f, -0.14503943920135498f, -0.31941986083984375f, -0.15372398495674133f}; + vector X_shape = {3, 1, 8}; + vector W = {0.13225573301315308f, 0.09750443696975708f, 0.3469849228858948f, 0.4743430018424988f}; + vector W_shape = {2, 1, 2}; + vector Y_shape = {3, 2, 5}; + auto expected_vals = {0.010817262344062328f, 0.05266154557466507f, 0.054253075271844864f, -0.03628557175397873f, + -0.05423086881637573f, 0.05262419581413269f, 0.22330480813980103f, 0.14844439923763275f, + -0.1848062425851822f, -0.14227961003780365f, -0.011078324168920517f, 0.02101614698767662f, + 0.014770962297916412f, -0.023767895996570587f, 0.03053247183561325f, -0.053894221782684326f, + 0.13591864705085754f, -0.03771348297595978f, -0.011907249689102173f, 0.08010470867156982f, + -0.01724436692893505f, -0.06235451623797417f, -0.06304522603750229f, -0.044972069561481476f, + -0.042245108634233475f, -0.08389100432395935f, -0.2509208619594574f, -0.18825212121009827f, + -0.18779152631759644f, -0.11083387583494186f}; + + TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); // Conv1d not yet optimized for MKLDNN XP +} + +// Conv1 +TEST(ConvTest, Conv1D_Bias) { + ConvOpAttributes attrs = { + "", // auto_pad + vector{2}, // dilations + 1, // group + vector{1}, // kernel_shape + vector{1, 1}, // pads + vector{3} // strides + }; + + vector X = {0.4582272171974182f, 0.3877705931663513f, -0.05413919687271118f, -0.3013981878757477f, + 0.19299334287643433f, -0.4758569598197937f, 0.4670986533164978f, 0.4078403115272522f, + 0.24010121822357178f, 0.41645896434783936f, -0.038333237171173096f, 0.22969317436218262f, + 0.3565492033958435f, 0.12812334299087524f, 0.10096627473831177f, 0.25682520866394043f, + 0.41700226068496704f, 0.34114283323287964f, -0.429997980594635f, 0.3545404076576233f, + 0.40339237451553345f, 0.10174298286437988f, 0.45713120698928833f, 0.08574831485748291f, + 0.38086581230163574f, 0.16378509998321533f, 0.12321442365646362f, -0.19936135411262512f, + 0.26019394397735596f, -0.18406429886817932f, 0.3110783100128174f, 0.15553230047225952f, + -0.14629846811294556f, -0.1779327094554901f, -0.01390346884727478f, -0.09264758229255676f}; + vector X_shape = {2, 2, 9}; + vector W = {-0.17206084728240967f, 0.3236315846443176f}; + vector W_shape = {1, 2, 1}; + vector B = {0.37892162799835205f}; + vector B_shape = {1}; + vector Y_shape = {2, 1, 4}; + auto expected_vals = {0.37892162799835205f, 0.4625728130340576f, 0.4934738576412201f, 0.44801419973373413f, + 0.37892162799835205f, 0.2499445676803589f, 0.31682088971138f, 0.32773756980895996f}; + TestConvOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape); // Conv1d not yet optimized for MKLDNN XP +} + +// Conv47 +TEST(ConvTest, Conv2D_1) { + ConvOpAttributes attrs = { + "", // auto_pad + vector{1, 1}, // dilations + 1, // group + vector{3, 3}, // kernel_shape + vector{1, 1, 1, 2}, // pads + vector{3, 1} // strides + }; + vector X = {-0.09103918075561523f, -0.32513630390167236f}; + vector X_shape = {2, 1, 1, 1}; + vector W = {0.4312484860420227f, -0.12559029459953308f, 0.44889551401138306f, -0.3100617825984955f, + 0.13522827625274658f, -0.06791308522224426f, 0.22671669721603394f, -0.17391827702522278f, + -0.31299442052841187f, -0.31545522809028625f, 0.06560015678405762f, 0.2656586766242981f, + 0.41363757848739624f, 0.31231558322906494f, -0.376018226146698f, -0.005708813667297363f, + 0.34922850131988525f, 0.45095211267471313f}; + vector W_shape = {2, 1, 3, 3}; + vector Y_shape = {2, 2, 1, 2}; + auto expected_vals = {-0.012311071157455444f, 0.02822777070105076f, -0.028432954102754593f, -0.037657227367162704f, + -0.04396762326359749f, 0.10081233829259872f, -0.10154513269662857f, -0.13448859751224518f}; + TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape, false, true); // asymmetric padding is not supported by cudnn +} + +TEST(ConvTest, Conv1D_Invalid_Input_Shape) { + ConvOpAttributes attrs = { + "", // auto_pad + vector{1}, // dilations + 1, // group + vector{2}, // kernel_shape + vector{0, 0}, // pads + vector{1} // strides + }; + vector X = vector(1, 1.0f); + vector X_shape = {1, 1, 1}; + vector dummy_shape = {1, 1, 2}; + auto dummy_vals = {0.0f, 0.0f}; + TestConvOp(attrs, {X, dummy_vals}, {X_shape, dummy_shape}, dummy_vals, dummy_shape, true, true, + OpTester::ExpectResult::kExpectFailure, "Invalid input shape: {1}"); +} + +TEST(ConvTest, Conv2D_Invalid_Input_Shape) { + ConvOpAttributes attrs = { + "", // auto_pad + vector{1, 1}, // dilations + 1, // group + vector{3, 3}, // kernel_shape + vector{0, 0, 0, 0}, // pads + vector{1, 1} // strides + }; + vector X = vector(1 * 3 * 1 * 111, 1.0f); + vector X_shape = {1, 3, 1, 111}; + vector dummy_shape = {2, 2, 1, 2}; + auto dummy_vals = {-0.0f, 0.0f, -0.0f, -0.0f, + -0.0f, 0.0f, -0.0f, -0.0f}; + TestConvOp(attrs, {X, dummy_vals}, {X_shape, dummy_shape}, dummy_vals, dummy_shape, true, true, + OpTester::ExpectResult::kExpectFailure, "Input channels C is not equal to kernel channels * group."); +} + +// Conv30 +TEST(ConvTest, Conv2D_2) { + ConvOpAttributes attrs = { + "", // auto_pad + vector{1, 1}, // dilations + 1, // group + vector{1, 1}, // kernel_shape + vector{0, 0, 0, 0}, // pads + vector{1, 1} // strides + }; + + vector X = {0.45246148109436035f, 0.15498268604278564f, 0.11199361085891724f, -0.39421093463897705f, + 0.2626858949661255f, 0.13414543867111206f, -0.27184486389160156f, -0.43028733134269714f, + -0.26825493574142456f, 0.3893144130706787f, -0.13631996512413025f, -0.009590476751327515f, + -0.48771554231643677f, -0.25256502628326416f, -0.2812897562980652f, 0.4043201804161072f, + 0.07795023918151855f, 0.326981782913208f, 0.13114392757415771f, -0.4416425824165344f, + 0.12446999549865723f, 0.36739975214004517f, 0.1698915958404541f, 0.2008744478225708f, + 0.23339951038360596f, 0.38613730669021606f, 0.11117297410964966f, 0.3877097964286804f, + 0.20812749862670898f, -0.34297940135002136f, -0.029246658086776733f, -0.20483523607254028f, + -0.19244328141212463f, -0.11104947328567505f, -0.32830488681793213f, -0.01800677180290222f, + 0.3618946671485901f, -0.40949052572250366f, -0.18248388171195984f, -0.3349453806877136f, + -0.34091079235076904f, 0.006497859954833984f, 0.4537564516067505f, 0.08006560802459717f, + -0.14788749814033508f, 0.034442365169525146f, -0.33322954177856445f, 0.06049239635467529f, + 0.42619407176971436f}; + vector X_shape = {1, 1, 7, 7}; + vector W = {-0.4406261742115021f}; + vector W_shape = {1, 1, 1, 1}; + vector Y_shape = {1, 1, 7, 7}; + auto expected_vals = {-0.19936637580394745f, -0.06828942894935608f, -0.04934731498360634f, 0.17369966208934784f, + -0.11574628204107285f, -0.05910799279808998f, 0.1197819635272026f, 0.18959586322307587f, + 0.1182001456618309f, -0.17154212296009064f, 0.06006614491343498f, 0.0042258151806890965f, + 0.21490024030208588f, 0.11128675937652588f, 0.12394362688064575f, -0.17815405130386353f, + -0.034346915781497955f, -0.14407673478126526f, -0.05778544768691063f, 0.19459928572177887f, + -0.05484473705291748f, -0.16188594698905945f, -0.07485868036746979f, -0.08851054310798645f, + -0.10284193605184555f, -0.17014220356941223f, -0.04898572340607643f, -0.17083507776260376f, + -0.09170642495155334f, 0.1511256992816925f, 0.012886842712759972f, 0.09025576710700989f, + 0.08479554951190948f, 0.0489313043653965f, 0.14465972781181335f, 0.007934254594147205f, + -0.15946026146411896f, 0.1804322451353073f, 0.08040717244148254f, 0.1475857049226761f, + 0.15021422505378723f, -0.0028631272725760937f, -0.19993697106838226f, -0.03527900204062462f, + 0.06516310572624207f, -0.015176207758486271f, 0.14682966470718384f, -0.02665453404188156f, + -0.18779225647449493f}; + TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); +} + +TEST(ConvTest, Conv2D_Bias_1) { + ConvOpAttributes attrs = { + "", // auto_pad + vector{1, 1}, // dilations + 1, // group + vector{2, 2}, // kernel_shape + vector{0, 0, 0, 0}, // pads + vector{1, 1} // strides + }; + vector X = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f}; + vector X_shape = {1, 1, 3, 3}; + vector W = {1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f}; + vector W_shape = {2, 1, 2, 2}; + vector Y_shape = {1, 2, 2, 2}; + vector B = {1.0f, -1.0f}; + vector B_shape = {2}; + auto expected_vals = {13.0f, 17.0f, 25.0f, 29.0f, 11.0f, 15.0f, 23.0f, 27.0f}; + + TestConvOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape); +} + +// Conv48 +TEST(ConvTest, Conv2D_Bias_2) { + ConvOpAttributes attrs = { + "", // auto_pad + vector{1, 1}, // dilations + 1, // group + vector{4, 4}, // kernel_shape + vector{1, 2, 3, 1}, // pads + vector{2, 3} // strides + }; + vector X = {-0.22904816269874573f, -0.20278319716453552f, -0.4723144471645355f, 0.027880489826202393f, + 0.2685856819152832f, -0.19361668825149536f, -0.39857280254364014f, 0.40285515785217285f, + 0.20966708660125732f, -0.39234158396720886f, -0.07502302527427673f, 0.4662899374961853f, + -0.2567148208618164f, -0.1186269223690033f, -0.1897754967212677f, -0.3967694342136383f, + -0.4268943667411804f, -0.344584584236145f, -0.4483465552330017f, -0.41608482599258423f, + -0.23649904131889343f, -0.4195239543914795f, 0.3277903199195862f, -0.11628741025924683f, + 0.2873995900154114f, 0.21717703342437744f, -0.26514798402786255f, 0.08272713422775269f, + 0.0050997138023376465f, -0.41409194469451904f, 0.2826550006866455f, 0.4891064763069153f, + -0.1522480845451355f, -0.2554396986961365f, 0.04099029302597046f, -0.35793858766555786f, + 0.2557554841041565f, 0.41162675619125366f, -0.06953108310699463f, 0.029517710208892822f, + 0.32956594228744507f, 0.4615175127983093f, -0.3216847777366638f, 0.15545696020126343f, + -0.3779126703739166f, -0.01712372899055481f, 0.07461833953857422f, 0.38875824213027954f, + 0.1980893611907959f, -0.19913813471794128f, -0.011296629905700684f, 0.30053526163101196f, + 0.4461088180541992f, 0.025034189224243164f, -0.3370230793952942f, -0.21012544631958008f, + -0.41627752780914307f, -0.43801137804985046f, 0.13566172122955322f, -0.47898364067077637f, + -0.45526939630508423f, -0.3007912039756775f, 0.06994932889938354f, -0.0749855637550354f, + -0.22754916548728943f, -0.469131737947464f, 0.08644282817840576f, 0.06157493591308594f, + -0.3920745849609375f, 0.458797812461853f, 0.18890488147735596f, 0.40145808458328247f}; + vector X_shape = {1, 2, 6, 6}; + vector W = {-0.48007914423942566f, -0.21048793196678162f, 0.2505034804344177f, 0.1610567569732666f, + -0.24951639771461487f, 0.1918455958366394f, 0.44247758388519287f, 0.06943017244338989f, + -0.10510382056236267f, -0.41663575172424316f, -0.3053555488586426f, -0.19126328825950623f, + -0.42332321405410767f, 0.498790979385376f, 0.081226646900177f, -0.21777048707008362f, + 0.46603143215179443f, -0.43488776683807373f, -0.3080252408981323f, -0.3844330906867981f, + -0.17214277386665344f, -0.3650006353855133f, 0.21724021434783936f, 0.1636529564857483f, + -0.22924479842185974f, 0.044009625911712646f, 0.274614155292511f, -0.06811442971229553f, + 0.450619637966156f, 0.4611729383468628f, 0.20782196521759033f, -0.3136714696884155f}; + vector W_shape = {1, 2, 4, 4}; + vector B = {-0.40378910303115845f}; + vector B_shape = {1}; + vector Y_shape = {1, 1, 4, 2}; + auto expected_vals = {-0.3419531583786011f, -0.6116723418235779f, -0.39677709341049194f, -0.7316848039627075f, + -0.5647197365760803f, 0.02788025140762329f, -0.30450713634490967f, -0.6786775588989258f}; + TestConvOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape, false, true); // asymmetric padding is not supported by cudnn +} + +TEST(ConvTest, Conv2D_AutoPad1) { + ConvOpAttributes attrs = { + "SAME_UPPER", // auto_pad + vector{1, 1}, // dilations + 1, // group + vector{3, 3}, // kernel_shape + {}, // pads + vector{1, 1} // strides + }; + vector X = vector(25, 1.0f); + vector X_shape = {1, 1, 5, 5}; + vector W = {0.0f, 1.0f, 2.0f, + 3.0f, 4.0f, 5.0f, + 6.0f, 7.0f, 8.0f}; + + vector W_shape = {1, 1, 3, 3}; + vector Y_shape = {1, 1, 5, 5}; + auto expected_vals = {24.0f, 33.0f, 33.0f, 33.0f, 20.0f, + 27.0f, 36.0f, 36.0f, 36.0f, 21.0f, + 27.0f, 36.0f, 36.0f, 36.0f, 21.0f, + 27.0f, 36.0f, 36.0f, 36.0f, 21.0f, + 12.0f, 15.0f, 15.0f, 15.0f, 8.0f}; + TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); +} + +TEST(ConvTest, Conv2D_AutoPad2) { + ConvOpAttributes attrs = { + "SAME_LOWER", // auto_pad + vector{1, 1}, // dilations + 1, // group + vector{3, 3}, // kernel_shape + {}, // pads + vector{1, 1} // strides + }; + vector X = {1.0f, 0.0f, 1.0f, 0.0f, 1.0f, + 1.0f, 0.0f, 1.0f, 0.0f, 1.0f, + 1.0f, 0.0f, 1.0f, 0.0f, 1.0f, + 1.0f, 0.0f, 1.0f, 0.0f, 1.0f, + 1.0f, 0.0f, 1.0f, 0.0f, 1.0f}; + vector X_shape = {1, 1, 5, 5}; + vector W = {0.0f, 1.0f, 2.0f, + 3.0f, 4.0f, 5.0f, + 6.0f, 7.0f, 8.0f}; + + vector W_shape = {1, 1, 3, 3}; + vector Y_shape = {1, 1, 5, 5}; + auto expected_vals = {11.0f, 22.0f, 11.0f, 22.0f, 11.0f, + 12.0f, 24.0f, 12.0f, 24.0f, 12.0f, + 12.0f, 24.0f, 12.0f, 24.0f, 12.0f, + 12.0f, 24.0f, 12.0f, 24.0f, 12.0f, + 5.0f, 10.0f, 5.0f, 10.0f, 5.0f}; + TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); +} + +// Conv10 +TEST(ConvTest, Conv3D_1) { + ConvOpAttributes attrs = { + "", // auto_pad + vector{1, 1, 1}, // dilations + 1, // group + vector{1, 1, 1}, // kernel_shape + vector{0, 0, 0, 0, 0, 0}, // pads + vector{1, 1, 1} // strides + }; + vector X = {-0.43337246775627136f, -0.48385289311408997f, -0.30954962968826294f, + 0.16074687242507935f, -0.46670910716056824f, 0.46576786041259766f, + -0.37056273221969604f, 0.40604978799819946f, -0.035478413105010986f, + -0.3125576674938202f, 0.42677170038223267f, 0.39851123094558716f, + -0.3906140625476837f, 0.2590462565422058f, -0.20646807551383972f, + 0.1382436752319336f, -0.20149192214012146f, 0.10030072927474976f, + -0.2413364052772522f, 0.1231224536895752f, 0.032734215259552f, + 0.29610633850097656f, -0.23117440938949585f, 0.3345826268196106f, + 0.02567422389984131f, 0.24579226970672607f, 0.11724984645843506f}; + vector X_shape = {1, 1, 3, 3, 3}; + vector W = {-0.44214117527008057f}; + vector W_shape = {1, 1, 1, 1, 1}; + vector Y_shape = {1, 1, 3, 3, 3}; + auto expected_vals = {0.19161181151866913f, 0.21393129229545593f, 0.13686463236808777f, + -0.07107280939817429f, 0.20635131001472473f, -0.20593515038490295f, + 0.16384103894233704f, -0.17953133583068848f, 0.01568646728992462f, + 0.13819462060928345f, -0.1886933445930481f, -0.17619822919368744f, + 0.17270655930042267f, -0.11453501880168915f, 0.09128803759813309f, + -0.06112322211265564f, 0.08908787369728088f, -0.04434708133339882f, + 0.10670476406812668f, -0.054437506943941116f, -0.014473143965005875f, + -0.13092079758644104f, 0.10221172869205475f, -0.1479327529668808f, + -0.011351631954312325f, -0.10867488384246826f, -0.05184098333120346f}; + TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); // Conv3d not yet optimized for MKLDNN XP +} + +// Conv22 +TEST(ConvTest, Conv3D_2) { + ConvOpAttributes attrs = { + "", // auto_pad + vector{1, 1, 1}, // dilations + 1, // group + vector{1, 1, 1}, // kernel_shape + vector{2, 2, 2, 2, 2, 2}, // pads + vector{2, 2, 2} // strides + }; + vector X = {0.010772407054901123f, -0.43806642293930054f, 0.455391526222229f, -0.28657248616218567f, + 0.45676887035369873f, -0.0320507287979126f, 0.4229400157928467f, -0.18730869889259338f, + -0.45851585268974304f, 0.042054951190948486f, -0.13332295417785645f, -0.25374430418014526f, + -0.23845627903938293f, 0.12214112281799316f, -0.1778157651424408f, 0.1891845464706421f, + 0.37962496280670166f, -0.033982306718826294f, 0.12737131118774414f, -0.040284961462020874f, + 0.46427029371261597f, -0.22687292098999023f, 0.17398333549499512f, -0.3014046251773834f, + -0.4043419063091278f, -0.33206477761268616f, 0.04655301570892334f, -0.4947906732559204f, + 0.0755157470703125f, 0.1173025369644165f, 0.47043120861053467f, 0.4824737310409546f, + -0.37734976410865784f, -0.056491583585739136f, -0.10790631175041199f, 0.043476223945617676f, + 0.24469023942947388f, -0.4100031852722168f, 0.0616222620010376f, 0.2296960949897766f, + 0.27883386611938477f, 0.08150351047515869f, 0.2453773021697998f, 0.08250969648361206f, + -0.1471814215183258f, -0.43011274933815f, 0.027180075645446777f, 0.3605625033378601f, + 0.24954384565353394f, -0.22505927085876465f, -0.36272895336151123f, -0.47674262523651123f, + 0.11275297403335571f, 0.49773406982421875f, 0.2686365246772766f, 0.025525271892547607f, + -0.3037869930267334f, 0.41126757860183716f, 0.36149072647094727f, 0.00883406400680542f, + -0.07959523797035217f, 0.3601323366165161f, 0.17322391271591187f, -0.012007325887680054f}; + vector X_shape = {1, 1, 4, 4, 4}; + vector W = {0.32824617624282837f}; + vector W_shape = {1, 1, 1, 1, 1}; + vector Y_shape = {1, 1, 4, 4, 4}; + auto expected_vals = {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0035360013134777546f, 0.14948052167892456f, 0.0f, + 0.0f, -0.15050607919692993f, -0.043762750923633575f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, -0.12386361509561539f, -0.03541983291506767f, 0.0f, + 0.0f, 0.09152615070343018f, 0.08054415881633759f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; + TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); // Conv3d not yet optimized for MKLDNN XP +} + +// Conv23 +TEST(ConvTest, Conv3D_Bias) { + ConvOpAttributes attrs = { + "", // auto_pad + vector{2, 2, 2}, // dilations + 1, // group + vector{2, 2, 2}, // kernel_shape + vector{2, 2, 2, 2, 2, 2}, // pads + vector{2, 2, 2} // strides + }; + + vector X = {0.46796226501464844f, -0.4613912105560303f, 0.33512794971466064f, -0.4010460674762726f, + 0.41722816228866577f, -0.048133403062820435f, 0.20415884256362915f, 0.03189706802368164f, + -0.04779183864593506f, -0.0795503556728363f, 0.4987630844116211f, 0.3506373167037964f, + 0.48065757751464844f, 0.269855260848999f, -0.2463444471359253f, 0.19044137001037598f, + -0.11830493807792664f, -0.2576887905597687f, -0.33940935134887695f, -0.257951021194458f, + -0.08279827237129211f, 0.3513314127922058f, -0.29122066497802734f, -0.43358397483825684f, + -0.13429927825927734f, 0.44032156467437744f, 0.05308258533477783f, -0.3499870300292969f, + -0.28474611043930054f, -0.44209951162338257f, -0.07418054342269897f, -0.10919415950775146f, + 0.2845439314842224f, 0.3498746156692505f, -0.19313520193099976f, 0.32609254121780396f, + 0.4880145788192749f, 0.05574071407318115f, -0.46457427740097046f, -0.02524462342262268f, + -0.18780940771102905f, -0.14720159769058228f, 0.207585871219635f, 0.47157740592956543f, + -0.05567386746406555f, -0.49871665239334106f, 0.2274145483970642f, 0.4589425325393677f, + -0.4725189805030823f, -0.4358765780925751f, 0.2841453552246094f, -0.27037882804870605f, + 0.34227508306503296f, 0.33575427532196045f, -0.19485199451446533f, -0.27679920196533203f, + -0.4238079786300659f, -0.4385119676589966f, 0.43724071979522705f, 0.3065117597579956f, + 0.45696544647216797f, 0.05291992425918579f, -0.023618370294570923f, -0.1860884726047516f, + 0.08669537305831909f, 0.32541000843048096f, 0.1846179962158203f, -0.1984834372997284f, + -0.2754465937614441f, 0.32004624605178833f, -0.34846532344818115f, 0.0999596118927002f, + -0.11374691128730774f, 0.21225297451019287f, -0.02315312623977661f, 0.1671370267868042f, + 0.22319108247756958f, 0.03609824180603027f, -0.1587022840976715f, 0.059984564781188965f, + -0.03951650857925415f, -0.4841443598270416f, 0.32919085025787354f, -0.23115816712379456f, + 0.39441078901290894f, -0.3554944396018982f, -0.17022761702537537f, -0.055081307888031006f, + 0.15856128931045532f, -0.4183449149131775f, -0.2474445104598999f, 0.03603637218475342f, + -0.2836887538433075f, 0.4602506160736084f, 0.29092925786972046f, -0.199321448802948f, + 0.380856454372406f, -0.13847029209136963f, -0.238397479057312f, -0.1907123327255249f, + -0.11061936616897583f, -0.08717870712280273f, 0.24449139833450317f, -0.14727482199668884f, + 0.1437196135520935f, 0.3955056071281433f, -0.12538021802902222f, 0.11590522527694702f, + 0.4598066806793213f, -0.30005723237991333f, -0.46578651666641235f, -0.33955082297325134f, + -0.2671887278556824f, 0.3611910939216614f, -0.11423084139823914f, -0.08382436633110046f, + -0.31819307804107666f, 0.14515334367752075f, 0.3157258629798889f, 0.33179205656051636f, + -0.2558857202529907f, 0.11888682842254639f, 0.12824326753616333f, -0.33106181025505066f, + 0.2549159526824951f, -0.46760573983192444f, -0.11983257532119751f, 0.1834418773651123f}; + vector X_shape = {2, 1, 4, 4, 4}; + vector W = {0.388077974319458f, -0.16366064548492432f, -0.42871910333633423f, 0.4276432394981384f, + 0.21517693996429443f, 0.007908165454864502f, 0.33897721767425537f, 0.21843165159225464f, + 0.34095364809036255f, -0.17043980956077576f, -0.013571739196777344f, -0.26793742179870605f, + -0.34863436222076416f, -0.2672275900840759f, -0.36691007018089294f, 0.37296557426452637f}; + vector W_shape = {2, 1, 2, 2, 2}; + vector B = {0.4310183525085449f, -0.4564093053340912f}; + vector B_shape = {2}; + vector Y_shape = {2, 2, 3, 3, 3}; + + auto expected_vals = {0.5332361459732056f, 0.6628494262695312f, 0.544619083404541f, 0.4242798388004303f, + 0.6271085739135742f, 0.6721994876861572f, 0.43064039945602417f, 0.4246789515018463f, + 0.53834068775177f, 0.6932926177978516f, 0.42797625064849854f, 0.2218741625547409f, + 0.29522019624710083f, 0.8329390287399292f, 0.37605351209640503f, 0.43735477328300476f, + 0.2920728623867035f, 0.6692450046539307f, 0.5527016520500183f, 0.22643595933914185f, + 0.5138190984725952f, 0.3041342794895172f, 0.7423423528671265f, 0.26707080006599426f, + 0.4617553651332855f, 0.32416003942489624f, 0.511577844619751f, -0.28187549114227295f, + -0.5031181573867798f, -0.5793710947036743f, -0.5992864370346069f, -0.5055556893348694f, + -0.7562476396560669f, -0.44363799691200256f, -0.5730307102203369f, -0.6302952766418457f, + -0.4756688177585602f, -0.728988528251648f, -0.3900943398475647f, -0.6694478988647461f, + -0.38822290301322937f, -0.35774707794189453f, -0.39807581901550293f, -0.547709047794342f, + -0.35872578620910645f, -0.5326492786407471f, -0.40852290391921997f, -0.4537881314754486f, + -0.4545857608318329f, -0.379546195268631f, -0.5250767469406128f, -0.42439910769462585f, + -0.5558245182037354f, -0.38563215732574463f, 0.44995537400245667f, 0.5007325410842896f, + 0.49359965324401855f, 0.40685802698135376f, 0.407518208026886f, 0.4628955125808716f, + 0.4301188290119171f, 0.40635955333709717f, 0.4260363280773163f, 0.55128413438797f, + 0.5498291254043579f, 0.27105778455734253f, 0.40259143710136414f, 0.5747092962265015f, + 0.4187920391559601f, 0.4507707953453064f, 0.420598566532135f, 0.3950541913509369f, + 0.593889057636261f, 0.16578882932662964f, 0.5332239270210266f, 0.43014785647392273f, + 0.50260329246521f, 0.39225444197654724f, 0.4074971079826355f, 0.5073125958442688f, + 0.3823610544204712f, -0.4240749180316925f, -0.41936254501342773f, -0.5241475105285645f, + -0.5220003724098206f, -0.502869725227356f, -0.5122783780097961f, -0.4260129928588867f, + -0.4105660617351532f, -0.4483373165130615f, -0.33759188652038574f, -0.735706090927124f, + -0.3714444637298584f, -0.4888814687728882f, -0.6191370487213135f, -0.2640320658683777f, + -0.47542816400527954f, -0.5078460574150085f, -0.4205915927886963f, -0.5584549903869629f, + -0.39770257472991943f, -0.45317384600639343f, -0.5598302483558655f, -0.2542789578437805f, + -0.5359901785850525f, -0.48090484738349915f, -0.38603779673576355f, -0.4991581439971924f}; + TestConvOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape); // Conv3d not yet optimized for MKLDNN XP +} + +TEST(ConvTest, Conv2D_group) { + ConvOpAttributes attrs = { + "", // auto_pad + vector{1, 1}, // dilations + 2, // group + vector{1, 1}, // kernel_shape + vector{0, 0, 0, 0}, // pads + vector{1, 1} // strides + }; + vector X = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f, 16.0f, 17.0f}; + vector X_shape = {1, 2, 3, 3}; + vector W = {1.0f, 2.0f}; + vector W_shape = {2, 1, 1, 1}; + vector Y_shape = {1, 2, 3, 3}; + auto expected_vals = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 18.0f, 20.0f, 22.0f, 24.0f, 26.0f, 28.0f, 30.0f, 32.0f, 34.0f}; + + TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/nn/conv_transpose_op_test.cc b/onnxruntime/test/providers/cpu/nn/conv_transpose_op_test.cc new file mode 100644 index 0000000000000..f7813b10e3a87 --- /dev/null +++ b/onnxruntime/test/providers/cpu/nn/conv_transpose_op_test.cc @@ -0,0 +1,338 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" +using namespace std; +namespace onnxruntime { +namespace test { + +namespace { + +struct ConvTransposeOpAttributes { + vector kernel_shape; + vector output_padding; + vector output_shape; + vector pads; + vector strides; + int64_t group; +}; + +void TestConvTransposeOp(const ConvTransposeOpAttributes& attributes, + const vector>& inputs, + const vector>& input_shapes, + const std::initializer_list& expected_output, + const vector& expected_output_shape, + OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess, + const std::string& err_str = "") { + OpTester test("ConvTranspose"); + test.AddAttribute("kernel_shape", attributes.kernel_shape); + if (!attributes.output_padding.empty()) { + test.AddAttribute("output_padding", attributes.output_padding); + } + if (!attributes.output_shape.empty()) { + test.AddAttribute("output_shape", attributes.output_shape); + } + test.AddAttribute("pads", attributes.pads); + test.AddAttribute("strides", attributes.strides); + test.AddAttribute("group", attributes.group); + + ONNXRUNTIME_ENFORCE(inputs.size() <= 3, "Our name array is only setup to handle 3 inputs"); + const char* szNames[] = {"X", "W", "B"}; + for (size_t i = 0; i < inputs.size(); i++) { + test.AddInput(szNames[i], input_shapes[i], inputs[i]); + } + test.AddOutput("Y", expected_output_shape, expected_output); + test.Run(expect_result, err_str); +} +} // namespace + +TEST(ConvTransposeTest, ConvTranspose_1) { + ConvTransposeOpAttributes attrs = { + vector{3, 3}, // kernel_shape + vector{1, 1}, // output_padding + {}, // output_shape + vector{1, 1, 1, 1}, // pads + vector{2, 2}, // strides + 1 // group + }; + vector X = {0.16857791f, -0.15161794f, 0.08540368f, + 0.1820628f, -0.21746576f, 0.08245695f, + 0.1431433f, -0.43156421f, 0.30591947f}; + vector X_shape = {1, 1, 3, 3}; + vector W = {-0.06230065f, 0.37932432f, -0.25388849f, + 0.33878803f, 0.43709868f, -0.22477469f, + 0.04118127f, -0.44696793f, 0.06373066f}; + vector W_shape = {1, 1, 3, 3}; + vector Y_shape = {1, 1, 6, 6}; + auto expected_vals = {0.07368518f, -0.08925839f, -0.06627201f, 0.06301362f, 0.03732984f, -0.01919658f, + -0.00628807f, -0.02817563f, -0.01472169f, 0.04392925f, -0.00689478f, -0.01549204f, + 0.07957941f, -0.11459791f, -0.09505399f, 0.07681622f, 0.03604182f, -0.01853423f, + -0.0270785f, -0.00680824f, -0.06650258f, 0.08004665f, 0.07918708f, -0.0724144f, + 0.06256775f, -0.17838378f, -0.18863615f, 0.20064656f, 0.133717f, -0.06876295f, + -0.06398046f, -0.00864975f, 0.19289537f, -0.01490572f, -0.13673618f, 0.01949645f}; + TestConvTransposeOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); +} + +TEST(ConvTransposeTest, ConvTranspose_Bias_1) { + ConvTransposeOpAttributes attrs = { + vector{3, 3}, // kernel_shape + vector{0, 0}, // output_padding + {}, // output_shape + vector{1, 1, 1, 1}, // pads + vector{1, 1}, // strides + 1 // group + }; + vector X = {0.22572887f, -0.07105902f, -0.40399021f, -0.14461157f, 0.05367219f, + -0.08353302f, 0.41023391f, 0.42745841f, -0.3769345f, -0.42057109f, + -0.1372498f, 0.05485916f, 0.34602994f, -0.06402895f, -0.06000063f, + 0.07891446f, -0.09410021f, 0.26251942f, -0.11043271f, 0.47966552f, + 0.34682763f, -0.04511502f, 0.22414422f, 0.24618894f, -0.21480265f}; + vector X_shape = {1, 1, 5, 5}; + vector W = {-0.0962126f, 0.19827795f, 0.03667754f, + 0.36756599f, -0.01076147f, -0.11781135f, + -0.11574665f, -0.38404959f, 0.44403327f}; + vector W_shape = {1, 1, 3, 3}; + vector B = {0.04676145f}; + vector B_shape = {1}; + vector Y_shape = {1, 1, 5, 5}; + auto expected_vals = {-0.03781903f, -0.09041066f, 0.14239404f, 0.09704495f, -0.03399426f, + 0.08749044f, 0.35613984f, 0.07240347f, -0.27841991f, -0.00337578f, + 0.07770107f, -0.09561026f, 0.13388641f, 0.30945939f, 0.14015588f, + 0.13079405f, -0.00488365f, -0.06758944f, 0.45621645f, 0.01566098f, + 0.00703105f, 0.12956856f, 0.0103332f, 0.04221053f, -0.21318194f}; + TestConvTransposeOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape); +} + +TEST(ConvTransposeTest, ConvTranspose_Bias_2) { + ConvTransposeOpAttributes attrs = { + vector{2, 2}, // kernel_shape + vector{0, 0}, // output_padding + {}, // output_shape + vector{0, 0, 0, 0}, // pads + vector{1, 1}, // strides + 1 // group + }; + vector X = {0.01270282f, 0.09657472f, -0.36909008f, -0.08085269f, + 0.0242992f, 0.40873009f, -0.46927932f, 0.34412372f, + -0.39574206f, 0.26234281f, 0.27352369f, -0.22265741f, + 0.43270493f, -0.24710381f, -0.03418651f, -0.04413456f, + -0.16414353f, 0.3158558f, 0.1087395f, -0.38577938f, + -0.38986659f, -0.09614426f, 0.17591673f, 0.40140027f, + -0.0869683f, -0.47193506f, -0.05010766f, 0.29325962f, + 0.22680271f, -0.0793834f, -0.36764491f, 0.20451134f, + 0.46361887f, -0.12190259f, 0.03413916f, 0.12307656f, + 0.28569579f, -0.392129f, 0.17179191f, 0.27161086f, + -0.12766263f, 0.1371125f, 0.28137422f, -0.39899838f, + 0.23824286f, -0.19693244f, 0.32956779f, 0.46209556f, + -0.46913007f}; + vector X_shape = {1, 1, 7, 7}; + vector W = {-0.34922412f, 0.1114341f, -0.01778314f, 0.46861196f}; + vector W_shape = {1, 1, 2, 2}; + vector B = {0.17402864f}; + vector B_shape = {1}; + vector Y_shape = {1, 1, 8, 8}; + auto expected_vals = {0.1695925f, 0.14171794f, 0.31368554f, 0.16113512f, + 0.15653302f, 0.033998f, 0.38345876f, 0.12173492f, + 0.05362644f, 0.35481372f, 0.09013268f, -0.06378071f, + 0.24394518f, 0.00222442f, 0.50842237f, -0.07341707f, + 0.17984779f, 0.35392997f, 0.03631867f, 0.16350585f, + 0.30338728f, 0.2088346f, 0.47435546f, 0.0147884f, + 0.20821247f, 0.08664516f, 0.03569011f, 0.16659322f, + 0.47522858f, 0.19675478f, -0.10781619f, 0.02401161f, + 0.0965334f, 0.1788421f, 0.36887163f, 0.2512877f, + 0.00254938f, 0.04799958f, 0.11982619f, 0.31525785f, + 0.12701407f, 0.19566584f, 0.31214368f, -0.10558143f, + 0.18591091f, 0.46830338f, 0.05418756f, 0.20530567f, + 0.07357728f, 0.39731777f, 0.1872202f, 0.08253923f, + 0.11266428f, 0.17892915f, 0.32709083f, 0.1860041f, + 0.16902491f, 0.3129794f, -0.01718347f, 0.28917417f, + 0.07588299f, 0.32025051f, 0.39891475f, -0.04581133f}; + TestConvTransposeOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape); +} + +TEST(ConvTransposeTest, ConvTranspose_Output_Shape) { + ConvTransposeOpAttributes attrs = { + vector{3, 3}, // kernel_shape + {}, // output_padding + vector{1, 3, 4, 4}, // output_shape + vector{0, 0, 0, 0}, // pads + vector{1, 1}, // strides + 1 // group + }; + int image_size = 4 * 4; + int input_channels = 3; + int output_channels = 3; + std::vector X; + for (int i = 0; i < input_channels * image_size; i++) + X.push_back(1.0f); + std::vector W; + int kernel_size = output_channels * input_channels * 3 * 3; + for (int i = 0; i < kernel_size; i++) + W.push_back(1.0f); + + vector X_shape = {1, 3, 4, 4}; + vector W_shape = {3, 3, 3, 3}; + + vector Y_shape = {1, 3, 4, 4}; + auto expected_vals = {12.0f, 18.0f, 18.0f, 12.0f, + 18.0f, 27.0f, 27.0f, 18.0f, + 18.0f, 27.0f, 27.0f, 18.0f, + 12.0f, 18.0f, 18.0f, 12.0f, + 12.0f, 18.0f, 18.0f, 12.0f, + 18.0f, 27.0f, 27.0f, 18.0f, + 18.0f, 27.0f, 27.0f, 18.0f, + 12.0f, 18.0f, 18.0f, 12.0f, + 12.0f, 18.0f, 18.0f, 12.0f, + 18.0f, 27.0f, 27.0f, 18.0f, + 18.0f, 27.0f, 27.0f, 18.0f, + 12.0f, 18.0f, 18.0f, 12.0f}; + TestConvTransposeOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); +} + +TEST(ConvTransposeTest, ConvTranspose_Output_Shape2) { + ConvTransposeOpAttributes attrs = { + vector{1, 5}, // kernel_shape + {}, // output_padding + vector{1, 1, 1, 14}, // output_shape + vector{0, 0, 0, 0}, // pads + vector{1, 1}, // strides + 1 // group + }; + vector X = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f}; + vector X_shape = {1, 1, 1, 10}; + vector W = {1.0f, 2.0f, 3.0f, 2.0f, 1.0f}; + vector W_shape = {1, 1, 1, 5}; + vector B = {1.0f}; + vector B_shape = {1}; + vector Y_shape = {1, 1, 1, 14}; + auto expected_vals = {1.0f, 2.0f, 5.0f, 11.0f, 19.0f, 28.0f, 37.0f, 46.0f, 55.0f, 64.0f, 63.0f, 51.0f, 27.0f, 10.0f}; + TestConvTransposeOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape); +} + +TEST(ConvTransposeTest, ConvTranspose_Output_Shape_Batch) { + ConvTransposeOpAttributes attrs = { + vector{1, 5}, // kernel_shape + {}, // output_padding + vector{2, 1, 1, 14}, // output_shape + vector{0, 0, 0, 0}, // pads + vector{1, 1}, // strides + 1 // group + }; + vector X = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, + 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f, 16.0f, 17.0f, 18.0f, 19.0f}; + vector X_shape = {2, 1, 1, 10}; + vector W = {1.0f, 2.0f, 3.0f, 2.0f, 1.0f}; + vector W_shape = {1, 1, 1, 5}; + vector B = {1.0f}; + vector B_shape = {1}; + vector Y_shape = {2, 1, 1, 14}; + auto expected_vals = {1.0f, 2.0f, 5.0f, 11.0f, 19.0f, 28.0f, 37.0f, 46.0f, 55.0f, 64.0f, 63.0f, 51.0f, 27.0f, 10.0f, + 11.0f, 32.0f, 65.0f, 91.0f, 109.0f, 118.0f, 127.0f, 136.0f, 145.0f, 154.0f, 143.0f, 111.0f, 57.0f, 20.0f}; + TestConvTransposeOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape); +} + +TEST(ConvTransposeTest, ConvTranspose_Invalid_Kernel_Shape) { + ConvTransposeOpAttributes attrs = { + vector{1, 1, 1, 5}, // invalid kernel_shape, should be [1, 5] + {}, // output_padding + vector{2, 1, 1, 14}, // output_shape + vector{0, 0, 0, 0}, // pads + vector{1, 1}, // strides + 1 // group + }; + vector X = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, + 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f, 16.0f, 17.0f, 18.0f, 19.0f}; + vector X_shape = {2, 1, 1, 10}; + vector W = {1.0f, 2.0f, 3.0f, 2.0f, 1.0f}; + vector W_shape = {1, 1, 1, 5}; + vector B = {1.0f}; + vector B_shape = {1}; + vector Y_shape = {2, 1, 1, 14}; + auto expected_vals = {1.0f, 2.0f, 5.0f, 11.0f, 19.0f, 28.0f, 37.0f, 46.0f, 55.0f, 64.0f, 63.0f, 51.0f, 27.0f, 10.0f, + 11.0f, 32.0f, 65.0f, 91.0f, 109.0f, 118.0f, 127.0f, 136.0f, 145.0f, 154.0f, 143.0f, 111.0f, 57.0f, 20.0f}; + TestConvTransposeOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape, + OpTester::ExpectResult::kExpectFailure, + "kernel width does not match filter width. kernel_width: 1 filter_width: 5"); +} + +TEST(ConvTransposeTest, ConvTranspose_onnx) { + ConvTransposeOpAttributes attrs = { + vector{3, 3}, // kernel_shape + {}, // output_padding + {}, // output_shape + vector{0, 0, 0, 0}, // pads + vector{1, 1}, // strides + 1 // group + }; + vector X = {0., 1., 2., 3., 4., 5., 6., 7., 8.}; + vector X_shape = {1, 1, 3, 3}; + vector W = {0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., 17.}; + vector W_shape = {1, 2, 3, 3}; + vector Y_shape = {1, 2, 5, 5}; + auto expected_vals = { + 0.f, 0.f, 1.f, 4.f, 4.f, + 0.f, 6.f, 20.f, 26.f, 20.f, + 9.f, 36.f, 84.f, 84.f, 57.f, + 36.f, 90.f, 164.f, 134.f, 80.f, + 36.f, 84.f, 145.f, 112.f, 64.f, + 0.f, 9.f, 28.f, 31.f, 22.f, + 27.f, 78.f, 155.f, 134.f, 83.f, + 90.f, 225.f, 408.f, 327.f, 192.f, + 117.f, 270.f, 461.f, 350.f, 197.f, + 90.f, 201.f, 334.f, 247.f, 136.f}; + + TestConvTransposeOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); +} + +TEST(ConvTransposeTest, ConvTranspose_onnx2) { + ConvTransposeOpAttributes attrs = { + vector{2, 2}, // kernel_shape + {}, // output_padding + {}, // output_shape + vector{0, 0, 0, 0}, // pads + vector{1, 1}, // strides + 1 // group + }; + vector X = {0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., 17.}; + vector X_shape = {1, 2, 3, 3}; + vector W = {0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., 17., 18., 19., 20., 21., 22., 23.}; + vector W_shape = {2, 3, 2, 2}; // this requires weight transpose + vector Y_shape = {1, 3, 4, 4}; + auto expected_vals = { + 108.f, 237.f, 263.f, 145.f, + 270.f, 592.f, 652.f, 358.f, + 354.f, 772.f, 832.f, 454.f, + 222.f, 481.f, 515.f, 279.f, + 144.f, 317.f, 359.f, 197.f, + 366.f, 800.f, 892.f, 486.f, + 498.f, 1076.f, 1168.f, 630.f, + 306.f, 657.f, 707.f, 379.f, + 180.f, 397.f, 455.f, 249.f, + 462.f, 1008.f, 1132.f, 614.f, + 642.f, 1380.f, 1504.f, 806.f, + 390.f, 833.f, 899.f, 479.f}; + + TestConvTransposeOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); +} + +TEST(ConvTransposeTest, ConvTranspose_onnx_group) { + ConvTransposeOpAttributes attrs = { + vector{1, 1}, // kernel_shape + {}, // output_padding + {}, // output_shape + vector{0, 0, 0, 0}, // pads + vector{1, 1}, // strides + 4 // group + }; + vector X = {0.f, 1.f, 2.f, 3.f, 4.f, 5.f, 6.f, 7.f, 8.f, 9.f, 10.f, 11.f, 12.f, 13.f, 14.f, 15.f}; + vector X_shape = {1, 16, 1, 1}; + vector W = {0.f, 1.f, 2.f, 3.f, 4.f, 5.f, 6.f, 7.f, 8.f, 9.f, 10.f, 11.f, 12.f, 13.f, 14.f, 15.f, 16.f, 17.f, 18.f, 19.f, 20.f, 21.f, 22.f, 23.f, 24.f, 25.f, 26.f, 27.f, 28.f, 29.f, 30.f, 31.0f}; + vector W_shape = {16, 2, 1, 1}; + vector Y_shape = {1, 8, 1, 1}; + auto expected_vals = {28.f, 34.f, 252.f, 274.f, 732.f, 770.f, 1468.f, 1522.f}; + TestConvTransposeOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/nn/flatten_op_test.cc b/onnxruntime/test/providers/cpu/nn/flatten_op_test.cc new file mode 100644 index 0000000000000..e51e3a7dcec8d --- /dev/null +++ b/onnxruntime/test/providers/cpu/nn/flatten_op_test.cc @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +class FlattenOpTest : public testing::Test { + public: + FlattenOpTest() : test_("Flatten"), data0_(120, 1.0f) {} + + protected: + OpTester test_; + const std::vector data0_; + const std::vector data1_ = {0.0f, 0.1f, 0.2f, 0.3f, + 1.0f, 1.1f, 1.2f, 1.3f, + 2.0f, 2.1f, 2.2f, 2.3f, + 3.0f, 3.1f, 3.2f, 3.3f}; +}; + +TEST_F(FlattenOpTest, Flatten_axis0) { + test_.AddAttribute("axis", 0L); + test_.AddInput("data", {2L, 3L, 4L, 5L}, data0_); + test_.AddOutput("output", {1L, 120L}, data0_); + test_.Run(); +} + +TEST_F(FlattenOpTest, Flatten_default_axis) { + // default axis value = 1 + test_.AddInput("data", {2L, 3L, 4L, 5L}, data0_); + test_.AddOutput("output", {2L, 60L}, data0_); + test_.Run(); +} + +TEST_F(FlattenOpTest, Flatten_invalid_axis) { + test_.AddAttribute("axis", 5L); + test_.AddInput("data", {1L, 2L, 4L, 2L}, data1_); + test_.AddOutput("output", {1L, 16L}, data1_); + test_.Run(OpTester::ExpectResult::kExpectFailure, "Invalid value(5) for attribute 'axis'"); +} + +TEST_F(FlattenOpTest, Flatten_axis3) { + test_.AddAttribute("axis", 3L); + test_.AddInput("data", {2L, 3L, 4L, 5L}, data0_); + test_.AddOutput("output", {24L, 5L}, data0_); + test_.Run(); +} + +TEST_F(FlattenOpTest, Flatten_axis4) { + test_.AddAttribute("axis", 4L); + test_.AddInput("data", {1L, 2L, 4L, 2L}, data1_); + test_.AddOutput("output", {16L, 1L}, data1_); + test_.Run(); +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/nn/instance_norm_op_test.cc b/onnxruntime/test/providers/cpu/nn/instance_norm_op_test.cc new file mode 100644 index 0000000000000..7d36bd9ee1a83 --- /dev/null +++ b/onnxruntime/test/providers/cpu/nn/instance_norm_op_test.cc @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" +using namespace std; +namespace onnxruntime { +namespace test { + +TEST(InstanceNormalizationOpTest, InstanceNorm) { + OpTester test("InstanceNormalization"); + test.AddAttribute("epsilon", 0.3F); + + vector input = {3.1513367F, 9.283596F, 1.4546119F, 5.4617004F, + 8.519701F, 1.2382338F, 1.7930176F, 5.1099434F, + 7.9195533F, 7.638727F, 8.065445F, 3.8082376F, + + 2.3667817F, 2.8248506F, 3.7754705F, 5.861325F, + 5.058735F, 3.2787242F, 3.6843839F, 9.755121F, + 2.7902672F, 7.3974323F, 8.283609F, 8.488337F}; + vector input_dims = {2, 3, 4}; + test.AddInput("input", input_dims, input); + + // vector scale = {2.1F, 0.1F, 1.F}; + vector scale = {1.0F, 1.0F, 1.F}; + vector scale_dims = {3}; + test.AddInput("scale", scale_dims, scale); + + // vector B = {2.3F, 1.5F, 0.F}; + vector B = {0.0F, 0.0F, 0.F}; + vector B_dims = {3}; + test.AddInput("B", B_dims, B); + + vector expected_output = {-0.56495477F, 1.48930046F, -1.13334329F, 0.20899761F, + 1.46688162F, -0.98600774F, -0.79911913F, 0.31824524F, + 0.57370438F, 0.42193634F, 0.6525492F, -1.64818992F, + + -0.92380346F, -0.60808484F, 0.04711878F, 1.48476953F, + -0.14644464F, -0.82262872F, -0.66852817F, 1.63760153F, + -1.65898662F, 0.27618144F, 0.64840618F, 0.734399F}; + test.AddOutput("Y", input_dims, expected_output); + test.Run(); +} + +TEST(InstanceNormalizationOpTest, InstanceNormBatch1) { + OpTester test("InstanceNormalization"); + test.AddAttribute("epsilon", 0.3F); + + vector input = {3.1513367F, 9.283596F, 1.4546119F, 5.4617004F, + 8.519701F, 1.2382338F, 1.7930176F, 5.1099434F, + 7.9195533F, 7.638727F, 8.065445F, 3.8082376F}; + vector input_dims = {1, 3, 4}; + test.AddInput("input", input_dims, input); + + // vector scale = {2.1F, 0.1F, 1.F}; + vector scale = {1.0F, 1.0F, 1.F}; + vector scale_dims = {3}; + test.AddInput("scale", scale_dims, scale); + + // vector B = {2.3F, 1.5F, 0.F}; + vector B = {0.0F, 0.0F, 0.F}; + vector B_dims = {3}; + test.AddInput("B", B_dims, B); + + vector expected_output = {-0.56495477F, 1.48930046F, -1.13334329F, 0.20899761F, + 1.46688162F, -0.98600774F, -0.79911913F, 0.31824524F, + 0.57370438F, 0.42193634F, 0.6525492F, -1.64818992F}; + test.AddOutput("Y", input_dims, expected_output); + test.Run(); +} + +TEST(InstanceNormalizationOpTest, InstanceNorm_2) { + OpTester test("InstanceNormalization"); + test.AddAttribute("epsilon", 0.3F); + + vector input = {2.676342F, 4.1100464F, 4.570907F, + 5.8493505F, 4.772751F, 7.1669755F, + 2.8400702F, 8.903057F, 1.2464883F, + 7.034208F, 4.755743F, 6.0282083F, + 2.2634823F, 2.7829134F, 8.206701F, + 9.7143545F, 3.8208177F, 7.2309036F, + 8.887503F, 9.05146F, 1.7653979F, + 1.351493F, 2.5284739F, 8.903282F, + 1.8851215F, 4.7899685F, 9.621006F, + 5.7984877F, 7.226894F, 3.8396406F, + 7.1785083F, 8.511631F, 1.1645945F, + 7.751299F, 9.89975F, 7.733491F}; + vector input_dims = {2, 3, 2, 1, 3}; + test.AddInput("input", input_dims, input); + + vector scale = {4.753198F, 7.4829206F, 1.0010294F}; + vector scale_dims = {3}; + test.AddInput("scale", scale_dims, scale); + + vector B = {3.720993F, 2.320803F, 1.8310473F}; + vector B_dims = {3}; + test.AddInput("B", B_dims, B); + + vector expected_output = {-3.1855264F, 1.3537457F, 2.8128836F, + 6.860582F, 3.451944F, 11.032334F, + -4.252796F, 13.116836F, -8.8181925F, + 7.7628374F, 1.2353455F, 4.880785F, + 0.6543751F, 0.83380324F, 2.7073524F, + 3.2281437F, 1.1923285F, 2.3702807F, + 8.316614F, 8.533577F, -1.1079268F, + -1.6556396F, -0.098163605F, 8.337496F, + -8.482306F, 0.1348517F, 14.466003F, + 3.1265988F, 7.3639297F, -2.684272F, + 1.88028F, 2.353724F, -0.25549555F, + 2.0837004F, 2.8466992F, 2.0773761F}; + test.AddOutput("Y", input_dims, expected_output); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/nn/lp_norm_op_test.cc b/onnxruntime/test/providers/cpu/nn/lp_norm_op_test.cc new file mode 100644 index 0000000000000..d21f821fccc03 --- /dev/null +++ b/onnxruntime/test/providers/cpu/nn/lp_norm_op_test.cc @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" +using namespace std; +namespace onnxruntime { +namespace test { + +TEST(LpNormalizationTest, L1Normalization) { + OpTester test("LpNormalization"); + test.AddAttribute("axis", (int64_t)1); + test.AddAttribute("p", (int64_t)1); + + vector input = {5.93932154F, 7.4367043F, 6.42487038F, 5.90394865F, + 4.81289319F, 6.81304702F, 4.9382849F, 9.02595701F, + 9.67296484F, 4.45097367F, 8.12552534F, 5.76005428F, + + 6.11240105F, 9.33036974F, 1.63932452F, 1.7841637F, + 1.18196558F, 8.49357861F, 8.00341076F, 8.83010933F, + 9.80756508F, 8.19242708F, 5.15331426F, 8.02476259F}; + vector input_dims = {2, 3, 4}; + test.AddInput("input", input_dims, input); + + vector expected_output = {0.2907843F, 0.3976693F, 0.3296719F, 0.28535331F, + 0.23563529F, 0.36431994F, 0.25339247F, 0.43624816F, + 0.47358041F, 0.23801075F, 0.41693563F, 0.27839852F, + + 0.35740998F, 0.3586345F, 0.11079474F, 0.09572189F, + 0.06911299F, 0.32647048F, 0.54091538F, 0.47374282F, + 0.57347703F, 0.31489502F, 0.34828988F, 0.43053529F}; + test.AddOutput("Y", input_dims, expected_output); + test.Run(); +} + +TEST(LpNormalizationTest, L2Normalization) { + OpTester test("LpNormalization"); + test.AddAttribute("axis", (int64_t)1); + test.AddAttribute("p", (int64_t)2); + + vector input = {5.93932154F, 7.4367043F, 6.42487038F, 5.90394865F, + 4.81289319F, 6.81304702F, 4.9382849F, 9.02595701F, + 9.67296484F, 4.45097367F, 8.12552534F, 5.76005428F, + + 6.11240105F, 9.33036974F, 1.63932452F, 1.7841637F, + 1.18196558F, 8.49357861F, 8.00341076F, 8.83010933F, + 9.80756508F, 8.19242708F, 5.15331426F, 8.02476259F}; + vector input_dims = {2, 3, 4}; + test.AddInput("input", input_dims, input); + + vector expected_output = { + 0.48173351F, 0.67457895F, 0.55987147F, 0.48285641F, + 0.39036983F, 0.61800737F, 0.4303285F, 0.73819091F, + 0.78456626F, 0.40374513F, 0.70806873F, 0.47108796F, + + 0.52617536F, 0.62021826F, 0.16971778F, 0.14788607F, + 0.10174744F, 0.56459419F, 0.82858584F, 0.73191164F, + 0.8442671F, 0.54457572F, 0.53351794F, 0.66515792F}; + test.AddOutput("Y", input_dims, expected_output); + test.Run(); +} + +TEST(LpNormalizationTest, LpNormalizationDefaultAxisAndP) { + OpTester test("LpNormalization"); + + vector input = { + 0.0f, 0.5f, 2.0f, 2.0f, + 1.0f, 0.5f, 2.0f, 2.5f, + 1.0f, 1.5f, 3.0f, 3.0f, + 1.5f, 2.0f, 3.5f, 3.5f}; + + vector input_dims = {16}; + test.AddInput("input", input_dims, input); + + vector expected_output = { + 0.0f, 0.059028134f, 0.236112535f, 0.236112535f, + 0.118056267f, 0.059028134f, 0.236112535f, 0.295140654f, + 0.118056267f, 0.177084401f, 0.354168802f, 0.354168802f, + 0.177084401f, 0.236112535f, 0.413196921f, 0.413196921f}; + test.AddOutput("Y", input_dims, expected_output); + test.Run(); +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/nn/lrn_op_test.cc b/onnxruntime/test/providers/cpu/nn/lrn_op_test.cc new file mode 100644 index 0000000000000..8f8d55e81fc10 --- /dev/null +++ b/onnxruntime/test/providers/cpu/nn/lrn_op_test.cc @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" +using namespace std; +namespace onnxruntime { +namespace test { + +TEST(LRNTest, LRN_1) { + OpTester test("LRN"); + test.AddAttribute("alpha", .001f); + test.AddAttribute("beta", .75f); + test.AddAttribute("bias", 2.0f); + test.AddAttribute("size", int64_t(5)); + + vector X = {0.93150997f, 0.22777775f, 0.0239904f, 0.68148804f, 0.38188118f, + 0.37675565f, 0.56248045f, 0.38941276f, 0.80423731f, 0.06973697f, + 0.22548851f, 0.92550498f, 0.99807096f, 0.13079064f, 0.56333995f, + 0.68052572f, 0.40238085f, 0.36750308f, 0.21684977f, 0.34298277f, + 0.41941738f, 0.09853589f, 0.67301852f, 0.17712493f, 0.12575327f, + 0.25551194f, 0.99063486f, 0.55772477f, 0.25830251f, 0.65443498f, + 0.19408275f, 0.47776002f, 0.50554532f, 0.7642616f, 0.39078581f, + 0.5375315f, 0.02074649f, 0.05360929f, 0.79922867f, 0.66012126f, + 0.76942754f, 0.14644335f, 0.09823465f, 0.79470968f, 0.60738826f, + 0.19192833f, 0.53565669f, 0.57413113f, 0.27340019f, 0.17495774f}; + vector shape = {1, 2, 5, 5}; + auto expected_output = {0.5538404f, 0.13542697f, 0.01426445f, 0.40519908f, 0.22705813f, + 0.22401723f, 0.33443925f, 0.23153915f, 0.47815821f, 0.04146536f, + 0.13407286f, 0.55027318f, 0.5934121f, 0.07776476f, 0.33494502f, + 0.40461099f, 0.23925379f, 0.21851626f, 0.12893309f, 0.20393133f, + 0.24938308f, 0.05858848f, 0.40015569f, 0.10531828f, 0.07477307f, + 0.15191767f, 0.58898938f, 0.33161741f, 0.15358147f, 0.38911262f, + 0.11540074f, 0.28406623f, 0.30058989f, 0.4543907f, 0.2323599f, + 0.31961f, 0.01233515f, 0.03187389f, 0.47520086f, 0.3924883f, + 0.45746815f, 0.08707454f, 0.05841004f, 0.47251317f, 0.36114204f, + 0.11411944f, 0.31849629f, 0.34136036f, 0.16256343f, 0.10403012f}; + test.AddInput("X", shape, X); + test.AddOutput("Y", shape, expected_output); + test.Run(); +} + +TEST(LRNTest, LRN_2) { + OpTester test("LRN"); + test.AddAttribute("alpha", .0001f); + test.AddAttribute("beta", .75f); + // default bias attribute value is 1.0f + test.AddAttribute("size", int64_t(5)); + + vector X = {0.97540224f, 0.76555133f, 0.44334042f, 0.81262767f, + 0.80408305f, 0.45893553f, 0.39771056f, 0.34420514f, + 0.94965851f, 0.93253171f, 0.42878076f, 0.85962552f, + 0.14810622f, 0.89759219f, 0.34574565f, 0.70201623f, + 0.15821661f, 0.48984697f, 0.94124645f, 0.32628751f, + 0.15926595f, 0.59950596f, 0.88879001f, 0.98674315f, + 0.80416244f, 0.71297693f, 0.94821811f, 0.48053992f, + 0.16935933f, 0.11691149f, 0.22425655f, 0.89018708f, + 0.75404555f, 0.68191183f, 0.31341696f, 0.86113745f, + 0.11833113f, 0.67812026f, 0.92196965f, 0.2754015f, + 0.80969357f, 0.06198973f, 0.01612644f, 0.12104732f, + 0.26516402f, 0.55688673f, 0.88051248f, 0.02725011f, + 0.05843787f, 0.75477105f, 0.14542344f, 0.97296566f, + 0.90956807f, 0.43174571f, 0.76335925f, 0.45990494f, + 0.40780696f, 0.29402575f, 0.54538655f, 0.28858703f, + 0.56942707f, 0.75392908f, 0.66404897f, 0.0093868f, + 0.6678884f, 0.60093129f, 0.54297262f, 0.16187565f, + 0.81088668f, 0.93738687f, 0.17667364f, 0.61121237f, + 0.46496955f, 0.11731055f, 0.09468836f, 0.80042875f, + 0.63450789f, 0.6306234f, 0.01279899f, 0.06116414f, + 0.61256766f, 0.5070824f, 0.60149539f, 0.84250051f, + 0.99870878f, 0.08305351f, 0.25943553f, 0.58913916f, + 0.19776763f, 0.29520017f, 0.21649282f, 0.46786523f, + 0.93280208f, 0.87324554f, 0.63763618f, 0.58810955f}; + vector shape = {2, 3, 4, 4}; + auto expected_output = {0.97537965f, 0.76553655f, 0.44333258f, 0.81260926f, + 0.80407488f, 0.45892847f, 0.39769977f, 0.34419912f, + 0.94962716f, 0.93251234f, 0.42877379f, 0.85961282f, + 0.14810593f, 0.89757699f, 0.34574077f, 0.7020027f, + 0.15821294f, 0.48983753f, 0.94122976f, 0.32628012f, + 0.15926433f, 0.59949672f, 0.88876593f, 0.98672587f, + 0.80413586f, 0.71296215f, 0.94820267f, 0.48053282f, + 0.169359f, 0.1169095f, 0.22425337f, 0.89016992f, + 0.75402808f, 0.68189865f, 0.31341141f, 0.86111796f, + 0.11832992f, 0.67810982f, 0.92194468f, 0.27539667f, + 0.80966681f, 0.06198845f, 0.01612618f, 0.12104553f, + 0.26516351f, 0.55687732f, 0.88050002f, 0.02724958f, + 0.05843714f, 0.75475758f, 0.14542197f, 0.9729411f, + 0.90953422f, 0.43173879f, 0.76335144f, 0.45989853f, + 0.40780437f, 0.29402491f, 0.54538363f, 0.28858295f, + 0.56941342f, 0.75390953f, 0.66404057f, 0.00938675f, + 0.66788006f, 0.60092056f, 0.54296708f, 0.16187157f, + 0.81085652f, 0.93737185f, 0.17667183f, 0.61120385f, + 0.46496657f, 0.11731022f, 0.09468785f, 0.80041742f, + 0.6344927f, 0.63060707f, 0.01279883f, 0.06116382f, + 0.61256003f, 0.50707334f, 0.60148925f, 0.84247929f, + 0.99867165f, 0.08305217f, 0.25943288f, 0.58913094f, + 0.19776636f, 0.29519933f, 0.21649165f, 0.46785861f, + 0.93277973f, 0.87322289f, 0.63762808f, 0.58810645f}; + + test.AddInput("X", shape, X); + test.AddOutput("Y", shape, expected_output); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/nn/max_roi_pool_op_test.cc b/onnxruntime/test/providers/cpu/nn/max_roi_pool_op_test.cc new file mode 100644 index 0000000000000..fa407eb425023 --- /dev/null +++ b/onnxruntime/test/providers/cpu/nn/max_roi_pool_op_test.cc @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/nn/roi_pool.h" +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(RoIPoolTest, MaxRoiPool) { + OpTester test("MaxRoiPool"); + + const int64_t pooled_height = 1, pooled_width = 1; + test.AddAttribute("pooled_shape", std::vector{pooled_height, pooled_width}); + + const int H = 6, W = 6; + const int image_size = H * W; + const int input_channels = 3; + std::vector input; + for (int i = 0; i < input_channels * image_size; i++) + input.push_back(1.0f * i / 10); + std::vector rois = { + 0, 1, 1, 2, 3, + 0, 1, 1, 2, 3, + 0, 1, 1, 2, 3}; + + std::vector x_dims = {1, 3, H, W}; + + test.AddInput("X", x_dims, input); + std::vector rois_dims = {3, 5}; + test.AddInput("rois", rois_dims, rois); + + const std::vector expected_vals = { + 2.0f, 5.6f, 9.2f, + 2.0f, 5.6f, 9.2f, + 2.0f, 5.6f, 9.2f}; + std::vector expected_dims = {3, 3, pooled_height, pooled_width}; + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/nn/pool_op_test.cc b/onnxruntime/test/providers/cpu/nn/pool_op_test.cc new file mode 100644 index 0000000000000..f2b08680d6a3b --- /dev/null +++ b/onnxruntime/test/providers/cpu/nn/pool_op_test.cc @@ -0,0 +1,1052 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/nn/pool.h" +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" +using namespace std; +namespace onnxruntime { +namespace test { + +TEST(PoolTest, MaxPool) { + OpTester test("MaxPool"); + + test.AddAttribute("auto_pad", ""); + test.AddAttribute("strides", std::vector{1, 1}); + test.AddAttribute("pads", vector{0, 0, 0, 0}); + test.AddAttribute("kernel_shape", vector{8, 8}); + + std::vector x_vals = { + 0.19151945412158966, 0.6221087574958801, 0.43772774934768677, 0.7853586077690125, 0.7799758315086365, 0.27259260416030884, 0.2764642536640167, 0.801872193813324, + 0.9581393599510193, 0.8759326338768005, 0.35781726241111755, 0.5009950995445251, 0.683462917804718, 0.7127020359039307, 0.37025076150894165, 0.5611962080001831, + 0.5030831694602966, 0.013768449425697327, 0.772826611995697, 0.8826411962509155, 0.36488598585128784, 0.6153962016105652, 0.07538124173879623, 0.3688240051269531, + 0.9331400990486145, 0.6513781547546387, 0.39720258116722107, 0.7887301445007324, 0.3168361186981201, 0.5680986642837524, 0.8691273927688599, 0.4361734092235565, + 0.802147626876831, 0.14376682043075562, 0.7042609453201294, 0.7045813202857971, 0.2187921106815338, 0.9248676300048828, 0.44214075803756714, 0.9093159437179565, + 0.05980922281742096, 0.18428708612918854, 0.047355279326438904, 0.6748809218406677, 0.5946247577667236, 0.5333101749420166, 0.043324064463377, 0.5614330768585205, + 0.32966843247413635, 0.5029668211936951, 0.11189431697130203, 0.6071937084197998, 0.5659446716308594, 0.006764062214642763, 0.617441713809967, 0.912122905254364, + 0.7905241250991821, 0.9920814633369446, 0.9588017463684082, 0.7919641137123108, 0.2852509617805481, 0.6249167323112488, 0.47809380292892456, 0.19567517936229706, + + 0.382317453622818, 0.053873684257268906, 0.45164841413497925, 0.9820047616958618, 0.12394270300865173, 0.1193808987736702, 0.7385230660438538, 0.587303638458252, + 0.47163254022598267, 0.10712681710720062, 0.22921857237815857, 0.8999651670455933, 0.41675353050231934, 0.5358516573905945, 0.0062085166573524475, 0.3006417155265808, + 0.43689316511154175, 0.6121490001678467, 0.9181980490684509, 0.625736653804779, 0.7059975862503052, 0.14983370900154114, 0.7460634112358093, 0.8310070037841797, + 0.6337257623672485, 0.4383098781108856, 0.15257278084754944, 0.5684096217155457, 0.5282242894172668, 0.9514287710189819, 0.48035916686058044, 0.5025595426559448, + 0.5368781685829163, 0.8192020654678345, 0.05711563676595688, 0.6694217324256897, 0.7671166062355042, 0.7081153392791748, 0.7968671917915344, 0.5577608346939087, + 0.9658365249633789, 0.14715689420700073, 0.02964700013399124, 0.5938934683799744, 0.11406569927930832, 0.9508098363876343, 0.32570740580558777, 0.19361868500709534, + 0.4578116536140442, 0.9204025864601135, 0.8790691494941711, 0.252615749835968, 0.34800878167152405, 0.18258872628211975, 0.9017960429191589, 0.7065281867980957, + 0.7266584634780884, 0.900087833404541, 0.7791637778282166, 0.5991547703742981, 0.29112523794174194, 0.1513952612876892, 0.33517464995384216, 0.6575517654418945, + + 0.07334254682064056, 0.055006396025419235, 0.32319480180740356, 0.5904818177223206, 0.8538985848426819, 0.2870624363422394, 0.17306722700595856, 0.13402120769023895, + 0.9946538209915161, 0.1794978678226471, 0.3175468146800995, 0.568291425704956, 0.009348574094474316, 0.9006485939025879, 0.9772414565086365, 0.5568946599960327, + 0.08477384597063065, 0.3330024778842926, 0.7284286618232727, 0.14243537187576294, 0.5524689555168152, 0.2730432450771332, 0.9744951128959656, 0.6677868962287903, + 0.2556532919406891, 0.1083114966750145, 0.7761807441711426, 0.7824779748916626, 0.7616038918495178, 0.9144031405448914, 0.6586228013038635, 0.568367600440979, + 0.20175568759441376, 0.6982963681221008, 0.952195405960083, 0.8899632692337036, 0.9935673475265503, 0.8187035322189331, 0.5451221466064453, 0.45125406980514526, + 0.8905571699142456, 0.9732648134231567, 0.5934113264083862, 0.36607450246810913, 0.3230946958065033, 0.8714232444763184, 0.2156340628862381, 0.7349451780319214, + 0.36561909317970276, 0.8016026020050049, 0.7827355861663818, 0.7013553977012634, 0.6227765679359436, 0.4936826527118683, 0.8405377268791199, 0.7120969891548157, + 0.4439089894294739, 0.031034860759973526, 0.36323976516723633, 0.7307217717170715, 0.475566565990448, 0.3444169759750366, 0.6408804059028625, 0.12620532512664795}; + std::vector x_dims = {1, 3, 8, 8}; + std::vector expected_dims = {1, 3, 1, 1}; + std::vector expected_vals = {0.9920814633369446, 0.9820047616958618, 0.9946538209915161}; + + test.AddInput("X", x_dims, x_vals); + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +static void MaxPool_8_WithIndexTest(bool has_index, int64_t storage_order=0) { + OpTester test("MaxPool", 8); + + test.AddAttribute("auto_pad", ""); + test.AddAttribute("strides", std::vector{1, 1}); + test.AddAttribute("pads", vector{0, 0, 0, 0}); + test.AddAttribute("kernel_shape", vector{8, 8}); + test.AddAttribute("storage_order", storage_order); + + std::vector x_vals = { + 0.19151945412158966, 0.6221087574958801, 0.43772774934768677, 0.7853586077690125, 0.7799758315086365, 0.27259260416030884, 0.2764642536640167, 0.801872193813324, + 0.9581393599510193, 0.8759326338768005, 0.35781726241111755, 0.5009950995445251, 0.683462917804718, 0.7127020359039307, 0.37025076150894165, 0.5611962080001831, + 0.5030831694602966, 0.013768449425697327, 0.772826611995697, 0.8826411962509155, 0.36488598585128784, 0.6153962016105652, 0.07538124173879623, 0.3688240051269531, + 0.9331400990486145, 0.6513781547546387, 0.39720258116722107, 0.7887301445007324, 0.3168361186981201, 0.5680986642837524, 0.8691273927688599, 0.4361734092235565, + 0.802147626876831, 0.14376682043075562, 0.7042609453201294, 0.7045813202857971, 0.2187921106815338, 0.9248676300048828, 0.44214075803756714, 0.9093159437179565, + 0.05980922281742096, 0.18428708612918854, 0.047355279326438904, 0.6748809218406677, 0.5946247577667236, 0.5333101749420166, 0.043324064463377, 0.5614330768585205, + 0.32966843247413635, 0.5029668211936951, 0.11189431697130203, 0.6071937084197998, 0.5659446716308594, 0.006764062214642763, 0.617441713809967, 0.912122905254364, + 0.7905241250991821, 0.9920814633369446, 0.9588017463684082, 0.7919641137123108, 0.2852509617805481, 0.6249167323112488, 0.47809380292892456, 0.19567517936229706, + + 0.382317453622818, 0.053873684257268906, 0.45164841413497925, 0.9820047616958618, 0.12394270300865173, 0.1193808987736702, 0.7385230660438538, 0.587303638458252, + 0.47163254022598267, 0.10712681710720062, 0.22921857237815857, 0.8999651670455933, 0.41675353050231934, 0.5358516573905945, 0.0062085166573524475, 0.3006417155265808, + 0.43689316511154175, 0.6121490001678467, 0.9181980490684509, 0.625736653804779, 0.7059975862503052, 0.14983370900154114, 0.7460634112358093, 0.8310070037841797, + 0.6337257623672485, 0.4383098781108856, 0.15257278084754944, 0.5684096217155457, 0.5282242894172668, 0.9514287710189819, 0.48035916686058044, 0.5025595426559448, + 0.5368781685829163, 0.8192020654678345, 0.05711563676595688, 0.6694217324256897, 0.7671166062355042, 0.7081153392791748, 0.7968671917915344, 0.5577608346939087, + 0.9658365249633789, 0.14715689420700073, 0.02964700013399124, 0.5938934683799744, 0.11406569927930832, 0.9508098363876343, 0.32570740580558777, 0.19361868500709534, + 0.4578116536140442, 0.9204025864601135, 0.8790691494941711, 0.252615749835968, 0.34800878167152405, 0.18258872628211975, 0.9017960429191589, 0.7065281867980957, + 0.7266584634780884, 0.900087833404541, 0.7791637778282166, 0.5991547703742981, 0.29112523794174194, 0.1513952612876892, 0.33517464995384216, 0.6575517654418945, + + 0.07334254682064056, 0.055006396025419235, 0.32319480180740356, 0.5904818177223206, 0.8538985848426819, 0.2870624363422394, 0.17306722700595856, 0.13402120769023895, + 0.9946538209915161, 0.1794978678226471, 0.3175468146800995, 0.568291425704956, 0.009348574094474316, 0.9006485939025879, 0.9772414565086365, 0.5568946599960327, + 0.08477384597063065, 0.3330024778842926, 0.7284286618232727, 0.14243537187576294, 0.5524689555168152, 0.2730432450771332, 0.9744951128959656, 0.6677868962287903, + 0.2556532919406891, 0.1083114966750145, 0.7761807441711426, 0.7824779748916626, 0.7616038918495178, 0.9144031405448914, 0.6586228013038635, 0.568367600440979, + 0.20175568759441376, 0.6982963681221008, 0.952195405960083, 0.8899632692337036, 0.9935673475265503, 0.8187035322189331, 0.5451221466064453, 0.45125406980514526, + 0.8905571699142456, 0.9732648134231567, 0.5934113264083862, 0.36607450246810913, 0.3230946958065033, 0.8714232444763184, 0.2156340628862381, 0.7349451780319214, + 0.36561909317970276, 0.8016026020050049, 0.7827355861663818, 0.7013553977012634, 0.6227765679359436, 0.4936826527118683, 0.8405377268791199, 0.7120969891548157, + 0.4439089894294739, 0.031034860759973526, 0.36323976516723633, 0.7307217717170715, 0.475566565990448, 0.3444169759750366, 0.6408804059028625, 0.12620532512664795}; + std::vector x_dims = {1, 3, 8, 8}; + std::vector expected_dims = {1, 3, 1, 1}; + std::vector expected_vals = {0.9920814633369446, 0.9820047616958618, 0.9946538209915161}; + std::vector expected_indices_row = {57, 67, 136}; + std::vector expected_indices_col = {15, 88, 129}; + + test.AddInput("X", x_dims, x_vals); + test.AddOutput("Y", expected_dims, expected_vals); + if (has_index) { + storage_order == 0 ? test.AddOutput("Indices", expected_dims, expected_indices_row) + : test.AddOutput("Indices", expected_dims, expected_indices_col); + } + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kMklDnnExecutionProvider}); +} + +TEST(PoolTest, MaxPool_8_With_Index) { + MaxPool_8_WithIndexTest(false); // row major + MaxPool_8_WithIndexTest(true, 0 /*storage_order*/); // row major + MaxPool_8_WithIndexTest(true, 1 /*storage_order*/); // col major +} + +TEST(PoolTest, MaxPool1D) { + OpTester test("MaxPool"); + + test.AddAttribute("auto_pad", ""); + test.AddAttribute("strides", std::vector{2}); + test.AddAttribute("pads", vector{0, 0}); + test.AddAttribute("kernel_shape", vector{2}); + + std::vector x_vals = {1, 2, 3, 4, 5, 6, 7, 8}; + std::vector x_dims = {1, 2, 4}; + std::vector expected_dims = {1, 2, 2}; + std::vector expected_vals = {2, 4, 6, 8}; + + test.AddInput("X", x_dims, x_vals); + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +static void MaxPool1D_8_WithIndexTest(int64_t storage_order) { + OpTester test("MaxPool", 8); + + test.AddAttribute("auto_pad", ""); + test.AddAttribute("strides", std::vector{2}); + test.AddAttribute("pads", vector{0, 0}); + test.AddAttribute("kernel_shape", vector{2}); + test.AddAttribute("storage_order", storage_order); + + std::vector x_vals = {1, 2, 3, 4, 5, 6, 7, 8}; + std::vector x_dims = {1, 2, 4}; + std::vector expected_dims = {1, 2, 2}; + std::vector expected_vals = {2, 4, 6, 8}; + std::vector expected_indices = {1, 3, 5, 7}; + + test.AddInput("X", x_dims, x_vals); + test.AddOutput("Y", expected_dims, expected_vals); + test.AddOutput("Indices", expected_dims, expected_indices); + test.Run(); +} + +TEST(PoolTest, MaxPool1D_8_With_Index) { + MaxPool1D_8_WithIndexTest(0 /*storage_order*/); + MaxPool1D_8_WithIndexTest(1 /*storage_order*/); +} + +TEST(PoolTest, GlobalMaxPool) { + OpTester test("GlobalMaxPool"); + + std::vector x_vals = {0.19151945412158966, 0.6221087574958801, 0.43772774934768677, + 0.7853586077690125, 0.7799758315086365, 0.27259260416030884, + 0.2764642536640167, 0.801872193813324, 0.9581393599510193, + 0.8759326338768005, 0.35781726241111755, 0.5009950995445251, + 0.683462917804718, 0.7127020359039307, 0.37025076150894165, + 0.5611962080001831, 0.5030831694602966, 0.013768449425697327, + 0.772826611995697, 0.8826411962509155, 0.36488598585128784, + 0.6153962016105652, 0.07538124173879623, 0.3688240051269531, + 0.9331400990486145, 0.6513781547546387, 0.39720258116722107, + 0.7887301445007324, 0.3168361186981201, 0.5680986642837524, + 0.8691273927688599, 0.4361734092235565, 0.802147626876831, + 0.14376682043075562, 0.7042609453201294, 0.7045813202857971, + 0.2187921106815338, 0.9248676300048828, 0.44214075803756714, + 0.9093159437179565, 0.05980922281742096, 0.18428708612918854, + 0.047355279326438904, 0.6748809218406677, 0.5946247577667236, + 0.5333101749420166, 0.043324064463377, 0.5614330768585205, + 0.32966843247413635, 0.5029668211936951, 0.11189431697130203, + 0.6071937084197998, 0.5659446716308594, 0.006764062214642763, + 0.617441713809967, 0.912122905254364, 0.7905241250991821, + 0.9920814633369446, 0.9588017463684082, 0.7919641137123108, + 0.2852509617805481, 0.6249167323112488, 0.47809380292892456, + 0.19567517936229706, 0.382317453622818, 0.053873684257268906, + 0.45164841413497925, 0.9820047616958618, 0.12394270300865173, + 0.1193808987736702, 0.7385230660438538, 0.587303638458252, + 0.47163254022598267, 0.10712681710720062, 0.22921857237815857, + 0.8999651670455933, 0.41675353050231934, 0.5358516573905945, + 0.0062085166573524475, 0.3006417155265808, 0.43689316511154175, + 0.6121490001678467, 0.9181980490684509, 0.625736653804779, + 0.7059975862503052, 0.14983370900154114, 0.7460634112358093, + 0.8310070037841797, 0.6337257623672485, 0.4383098781108856, + 0.15257278084754944, 0.5684096217155457, 0.5282242894172668, + 0.9514287710189819, 0.48035916686058044, 0.5025595426559448, + 0.5368781685829163, 0.8192020654678345, 0.05711563676595688, + 0.6694217324256897, 0.7671166062355042, 0.7081153392791748, + 0.7968671917915344, 0.5577608346939087, 0.9658365249633789, + 0.14715689420700073, 0.02964700013399124, 0.5938934683799744, + 0.11406569927930832, 0.9508098363876343, 0.32570740580558777, + 0.19361868500709534, 0.4578116536140442, 0.9204025864601135, + 0.8790691494941711, 0.252615749835968, 0.34800878167152405, + 0.18258872628211975, 0.9017960429191589, 0.7065281867980957, + 0.7266584634780884, 0.900087833404541, 0.7791637778282166, + 0.5991547703742981, 0.29112523794174194, 0.1513952612876892, + 0.33517464995384216, 0.6575517654418945, 0.07334254682064056, + 0.055006396025419235, 0.32319480180740356, 0.5904818177223206, + 0.8538985848426819, 0.2870624363422394, 0.17306722700595856, + 0.13402120769023895, 0.9946538209915161, 0.1794978678226471, + 0.3175468146800995, 0.568291425704956, 0.009348574094474316, + 0.9006485939025879, 0.9772414565086365, 0.5568946599960327, + 0.08477384597063065, 0.3330024778842926, 0.7284286618232727, + 0.14243537187576294, 0.5524689555168152, 0.2730432450771332, + 0.9744951128959656, 0.6677868962287903, 0.2556532919406891, + 0.1083114966750145, 0.7761807441711426, 0.7824779748916626, + 0.7616038918495178, 0.9144031405448914, 0.6586228013038635, + 0.568367600440979, 0.20175568759441376, 0.6982963681221008, + 0.952195405960083, 0.8899632692337036, 0.9935673475265503, + 0.8187035322189331, 0.5451221466064453, 0.45125406980514526, + 0.8905571699142456, 0.9732648134231567, 0.5934113264083862, + 0.36607450246810913, 0.3230946958065033, 0.8714232444763184, + 0.2156340628862381, 0.7349451780319214, 0.36561909317970276, + 0.8016026020050049, 0.7827355861663818, 0.7013553977012634, + 0.6227765679359436, 0.4936826527118683, 0.8405377268791199, + 0.7120969891548157, 0.4439089894294739, 0.031034860759973526, + 0.36323976516723633, 0.7307217717170715, 0.475566565990448, + 0.3444169759750366, 0.6408804059028625, 0.12620532512664795}; + std::vector x_dims = {1, 3, 8, 8}; + std::vector expected_dims = {1, 3, 1, 1}; + std::vector expected_vals = {0.9920814633369446, 0.9820047616958618, 0.9946538209915161}; + + test.AddInput("X", x_dims, x_vals); + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(PoolTest, GlobalMaxPool3D) { + OpTester test("GlobalMaxPool"); + + std::vector x_vals = {0.19151945412158966, 0.6221087574958801, 0.43772774934768677, + 0.7853586077690125, 0.7799758315086365, 0.27259260416030884, + 0.2764642536640167, 0.801872193813324, 0.9581393599510193, + 0.8759326338768005, 0.35781726241111755, 0.5009950995445251, + 0.683462917804718, 0.7127020359039307, 0.37025076150894165, + 0.5611962080001831, 0.5030831694602966, 0.013768449425697327, + 0.772826611995697, 0.8826411962509155, 0.36488598585128784, + 0.6153962016105652, 0.07538124173879623, 0.3688240051269531, + 0.9331400990486145, 0.6513781547546387, 0.39720258116722107, + 0.7887301445007324, 0.3168361186981201, 0.5680986642837524, + 0.8691273927688599, 0.4361734092235565, 0.802147626876831, + 0.14376682043075562, 0.7042609453201294, 0.7045813202857971, + 0.2187921106815338, 0.9248676300048828, 0.44214075803756714, + 0.9093159437179565, 0.05980922281742096, 0.18428708612918854, + 0.047355279326438904, 0.6748809218406677, 0.5946247577667236, + 0.5333101749420166, 0.043324064463377, 0.5614330768585205, + 0.32966843247413635, 0.5029668211936951, 0.11189431697130203, + 0.6071937084197998, 0.5659446716308594, 0.006764062214642763, + 0.617441713809967, 0.912122905254364, 0.7905241250991821, + 0.9920814633369446, 0.9588017463684082, 0.7919641137123108, + 0.2852509617805481, 0.6249167323112488, 0.47809380292892456, + 0.19567517936229706, 0.382317453622818, 0.053873684257268906, + 0.45164841413497925, 0.9820047616958618, 0.12394270300865173, + 0.1193808987736702, 0.7385230660438538, 0.587303638458252, + 0.47163254022598267, 0.10712681710720062, 0.22921857237815857, + 0.8999651670455933, 0.41675353050231934, 0.5358516573905945, + 0.0062085166573524475, 0.3006417155265808, 0.43689316511154175, + 0.6121490001678467, 0.9181980490684509, 0.625736653804779, + 0.7059975862503052, 0.14983370900154114, 0.7460634112358093, + 0.8310070037841797, 0.6337257623672485, 0.4383098781108856, + 0.15257278084754944, 0.5684096217155457, 0.5282242894172668, + 0.9514287710189819, 0.48035916686058044, 0.5025595426559448, + 0.5368781685829163, 0.8192020654678345, 0.05711563676595688, + 0.6694217324256897, 0.7671166062355042, 0.7081153392791748, + 0.7968671917915344, 0.5577608346939087, 0.9658365249633789, + 0.14715689420700073, 0.02964700013399124, 0.5938934683799744, + 0.11406569927930832, 0.9508098363876343, 0.32570740580558777, + 0.19361868500709534, 0.4578116536140442, 0.9204025864601135, + 0.8790691494941711, 0.252615749835968, 0.34800878167152405, + 0.18258872628211975, 0.9017960429191589, 0.7065281867980957, + 0.7266584634780884, 0.900087833404541, 0.7791637778282166, + 0.5991547703742981, 0.29112523794174194, 0.1513952612876892, + 0.33517464995384216, 0.6575517654418945, 0.07334254682064056, + 0.055006396025419235, 0.32319480180740356, 0.5904818177223206, + 0.8538985848426819, 0.2870624363422394, 0.17306722700595856, + 0.13402120769023895, 0.9946538209915161, 0.1794978678226471, + 0.3175468146800995, 0.568291425704956, 0.009348574094474316, + 0.9006485939025879, 0.9772414565086365, 0.5568946599960327, + 0.08477384597063065, 0.3330024778842926, 0.7284286618232727, + 0.14243537187576294, 0.5524689555168152, 0.2730432450771332, + 0.9744951128959656, 0.6677868962287903, 0.2556532919406891, + 0.1083114966750145, 0.7761807441711426, 0.7824779748916626, + 0.7616038918495178, 0.9144031405448914, 0.6586228013038635, + 0.568367600440979, 0.20175568759441376, 0.6982963681221008, + 0.952195405960083, 0.8899632692337036, 0.9935673475265503, + 0.8187035322189331, 0.5451221466064453, 0.45125406980514526, + 0.8905571699142456, 0.9732648134231567, 0.5934113264083862, + 0.36607450246810913, 0.3230946958065033, 0.8714232444763184, + 0.2156340628862381, 0.7349451780319214, 0.36561909317970276, + 0.8016026020050049, 0.7827355861663818, 0.7013553977012634, + 0.6227765679359436, 0.4936826527118683, 0.8405377268791199, + 0.7120969891548157, 0.4439089894294739, 0.031034860759973526, + 0.36323976516723633, 0.7307217717170715, 0.475566565990448, + 0.3444169759750366, 0.6408804059028625, 0.12620532512664795}; + std::vector x_dims = {1, 3, 8, 4, 2}; + std::vector expected_dims = {1, 3, 1, 1, 1}; + std::vector expected_vals = {0.9920814633369446, 0.9820047616958618, 0.9946538209915161}; + + test.AddInput("X", x_dims, x_vals); + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(PoolTest, AveragePool) { + OpTester test("AveragePool"); + + test.AddAttribute("auto_pad", ""); + test.AddAttribute("strides", std::vector{1, 1}); + test.AddAttribute("pads", vector{0, 0, 0, 0}); + test.AddAttribute("kernel_shape", vector{8, 8}); + + std::vector x_vals = {0.3337382376194, 0.8794041872024536, 0.33745908737182617, + 0.666634202003479, 0.44255536794662476, 0.6473854184150696, + 0.7674617171287537, 0.8822641968727112, 0.8852233290672302, + 0.7453723549842834, 0.2818361520767212, 0.8706393241882324, + 0.5406734347343445, 0.02016347087919712, 0.8047968745231628, + 0.6037390828132629, 0.6242085099220276, 0.15702469646930695, + 0.19581079483032227, 0.7122684717178345, 0.06907976418733597, + 0.5234333872795105, 0.9091887474060059, 0.4319673180580139, + 0.8100792169570923, 0.10371053218841553, 0.3888828456401825, + 0.27514228224754333, 0.2670423686504364, 0.04306316748261452, + 0.36913928389549255, 0.8686641454696655, 0.6307396292686462, + 0.7112566232681274, 0.25298961997032166, 0.5131869316101074, + 0.11016560345888138, 0.20159587264060974, 0.43771353363990784, + 0.7566956877708435, 0.10168474912643433, 0.7238786220550537, + 0.4961036741733551, 0.9173188209533691, 0.6056748032569885, + 0.250592976808548, 0.4755987823009491, 0.904503583908081, + 0.4725301265716553, 0.8506938219070435, 0.13940207660198212, + 0.9848986864089966, 0.6715511083602905, 0.8943559527397156, + 0.40052708983421326, 0.0880642905831337, 0.8935731649398804, + 0.3453705310821533, 0.8090538382530212, 0.19269756972789764, + 0.03951506316661835, 0.027226323261857033, 0.8117656111717224, + 0.7711597084999084, 0.8593372702598572, 0.20363913476467133, + 0.7842649817466736, 0.29195329546928406, 0.5064213871955872, + 0.7418627142906189, 0.1069103255867958, 0.5893736481666565, + 0.2143796980381012, 0.15637169778347015, 0.1684667021036148, + 0.7528857588768005, 0.5846885442733765, 0.9133154153823853, + 0.6781020760536194, 0.21141840517520905, 0.05769576504826546, + 0.49993178248405457, 0.2309824675321579, 0.05175522714853287, + 0.6969341039657593, 0.47234174609184265, 0.11310867220163345, + 0.6184650659561157, 0.896835207939148, 0.6077945232391357, + 0.3074592649936676, 0.07904505729675293, 0.048881493508815765, + 0.24833321571350098, 0.9844338893890381, 0.4520559012889862, + 0.26799046993255615, 0.7592704892158508, 0.37819114327430725, + 0.30964234471321106, 0.8839467167854309, 0.0934458002448082, + 0.379569411277771, 0.09841523319482803, 0.6000676155090332, + 0.7950544357299805, 0.45938217639923096, 0.5537487864494324, + 0.38861554861068726, 0.4074040949344635, 0.38612639904022217, + 0.89164137840271, 0.21732182800769806, 0.6711451411247253, + 0.5769082307815552, 0.9865275621414185, 0.03840707615017891, + 0.1573856621980667, 0.09340689331293106, 0.9288106560707092, + 0.16059239208698273, 0.8247162103652954, 0.422741562128067, + 0.987165629863739, 0.9476590752601624, 0.9242128133773804, + 0.9987634420394897, 0.32882997393608093, 0.011870949529111385, + 0.984099805355072, 0.09365611523389816, 0.33463314175605774, + 0.6386845111846924, 0.9860017895698547, 0.4672822654247284, + 0.9529699683189392, 0.15891511738300323, 0.7175184488296509, + 0.024524977430701256, 0.8217390179634094, 0.14411452412605286, + 0.45218998193740845, 0.4429023861885071, 0.9931989312171936, + 0.8507111072540283, 0.13051295280456543, 0.07811085134744644, + 0.943297803401947, 0.030969098210334778, 0.21649038791656494, + 0.9491124749183655, 0.5731316804885864, 0.5927708745002747, + 0.7653813362121582, 0.5627018809318542, 0.01101104449480772, + 0.7299126982688904, 0.3900069296360016, 0.0853394865989685, + 0.43255582451820374, 0.8431127071380615, 0.5303983092308044, + 0.6451488137245178, 0.16481569409370422, 0.35921016335487366, + 0.036783039569854736, 0.5699883103370667, 0.5847001075744629, + 0.9650961756706238, 0.9053892493247986, 0.2933308482170105, + 0.2615077495574951, 0.48302537202835083, 0.5642899870872498, + 0.20961439609527588, 0.37418732047080994, 0.4921484887599945, + 0.7827269434928894, 0.28503814339637756, 0.4663805067539215, + 0.1988927721977234, 0.20202897489070892, 0.3183555603027344, + 0.4528728425502777, 0.2815922796726227, 0.820142388343811, + 0.4963360130786896, 0.46687841415405273, 0.7405545115470886, + 0.40191709995269775, 0.21238186955451965, 0.46927347779273987}; + std::vector x_dims = {1, 3, 8, 8}; + std::vector expected_dims = {1, 3, 1, 1}; + std::vector expected_vals = {0.5146896243095398, 0.4851023256778717, 0.4756942689418793}; + + test.AddInput("X", x_dims, x_vals); + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(PoolTest, AveragePool_IncludePadPixel) { + OpTester test("AveragePool"); + + test.AddAttribute("auto_pad", ""); + test.AddAttribute("strides", std::vector{1, 1}); + test.AddAttribute("pads", vector{1, 1, 1, 1}); + test.AddAttribute("kernel_shape", vector{2, 2}); + test.AddAttribute("count_include_pad", (int64_t)1); + std::vector x_vals = {0.3337f, 0.8794f, 0.3375f, + 0.6666f, 0.4426f, 0.6474f, + 0.7675f, 0.8823f, 0.8852f}; + + std::vector x_dims = {1, 1, 3, 3}; + std::vector expected_dims = {1, 1, 4, 4}; + std::vector expected_vals = {0.0834f, 0.3033f, 0.3042f, 0.0844f, + 0.2501f, 0.5806f, 0.5767f, 0.2462f, + 0.3585f, 0.6897f, 0.7144f, 0.3832f, + 0.1919f, 0.4124f, 0.4419f, 0.2213f}; + + test.AddInput("X", x_dims, x_vals); + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(PoolTest, GlobalAveragePool) { + OpTester test("GlobalAveragePool"); + + std::vector x_vals = {0.3337382376194, 0.8794041872024536, 0.33745908737182617, + 0.666634202003479, 0.44255536794662476, 0.6473854184150696, + 0.7674617171287537, 0.8822641968727112, 0.8852233290672302, + 0.7453723549842834, 0.2818361520767212, 0.8706393241882324, + 0.5406734347343445, 0.02016347087919712, 0.8047968745231628, + 0.6037390828132629, 0.6242085099220276, 0.15702469646930695, + 0.19581079483032227, 0.7122684717178345, 0.06907976418733597, + 0.5234333872795105, 0.9091887474060059, 0.4319673180580139, + 0.8100792169570923, 0.10371053218841553, 0.3888828456401825, + 0.27514228224754333, 0.2670423686504364, 0.04306316748261452, + 0.36913928389549255, 0.8686641454696655, 0.6307396292686462, + 0.7112566232681274, 0.25298961997032166, 0.5131869316101074, + 0.11016560345888138, 0.20159587264060974, 0.43771353363990784, + 0.7566956877708435, 0.10168474912643433, 0.7238786220550537, + 0.4961036741733551, 0.9173188209533691, 0.6056748032569885, + 0.250592976808548, 0.4755987823009491, 0.904503583908081, + 0.4725301265716553, 0.8506938219070435, 0.13940207660198212, + 0.9848986864089966, 0.6715511083602905, 0.8943559527397156, + 0.40052708983421326, 0.0880642905831337, 0.8935731649398804, + 0.3453705310821533, 0.8090538382530212, 0.19269756972789764, + 0.03951506316661835, 0.027226323261857033, 0.8117656111717224, + 0.7711597084999084, 0.8593372702598572, 0.20363913476467133, + 0.7842649817466736, 0.29195329546928406, 0.5064213871955872, + 0.7418627142906189, 0.1069103255867958, 0.5893736481666565, + 0.2143796980381012, 0.15637169778347015, 0.1684667021036148, + 0.7528857588768005, 0.5846885442733765, 0.9133154153823853, + 0.6781020760536194, 0.21141840517520905, 0.05769576504826546, + 0.49993178248405457, 0.2309824675321579, 0.05175522714853287, + 0.6969341039657593, 0.47234174609184265, 0.11310867220163345, + 0.6184650659561157, 0.896835207939148, 0.6077945232391357, + 0.3074592649936676, 0.07904505729675293, 0.048881493508815765, + 0.24833321571350098, 0.9844338893890381, 0.4520559012889862, + 0.26799046993255615, 0.7592704892158508, 0.37819114327430725, + 0.30964234471321106, 0.8839467167854309, 0.0934458002448082, + 0.379569411277771, 0.09841523319482803, 0.6000676155090332, + 0.7950544357299805, 0.45938217639923096, 0.5537487864494324, + 0.38861554861068726, 0.4074040949344635, 0.38612639904022217, + 0.89164137840271, 0.21732182800769806, 0.6711451411247253, + 0.5769082307815552, 0.9865275621414185, 0.03840707615017891, + 0.1573856621980667, 0.09340689331293106, 0.9288106560707092, + 0.16059239208698273, 0.8247162103652954, 0.422741562128067, + 0.987165629863739, 0.9476590752601624, 0.9242128133773804, + 0.9987634420394897, 0.32882997393608093, 0.011870949529111385, + 0.984099805355072, 0.09365611523389816, 0.33463314175605774, + 0.6386845111846924, 0.9860017895698547, 0.4672822654247284, + 0.9529699683189392, 0.15891511738300323, 0.7175184488296509, + 0.024524977430701256, 0.8217390179634094, 0.14411452412605286, + 0.45218998193740845, 0.4429023861885071, 0.9931989312171936, + 0.8507111072540283, 0.13051295280456543, 0.07811085134744644, + 0.943297803401947, 0.030969098210334778, 0.21649038791656494, + 0.9491124749183655, 0.5731316804885864, 0.5927708745002747, + 0.7653813362121582, 0.5627018809318542, 0.01101104449480772, + 0.7299126982688904, 0.3900069296360016, 0.0853394865989685, + 0.43255582451820374, 0.8431127071380615, 0.5303983092308044, + 0.6451488137245178, 0.16481569409370422, 0.35921016335487366, + 0.036783039569854736, 0.5699883103370667, 0.5847001075744629, + 0.9650961756706238, 0.9053892493247986, 0.2933308482170105, + 0.2615077495574951, 0.48302537202835083, 0.5642899870872498, + 0.20961439609527588, 0.37418732047080994, 0.4921484887599945, + 0.7827269434928894, 0.28503814339637756, 0.4663805067539215, + 0.1988927721977234, 0.20202897489070892, 0.3183555603027344, + 0.4528728425502777, 0.2815922796726227, 0.820142388343811, + 0.4963360130786896, 0.46687841415405273, 0.7405545115470886, + 0.40191709995269775, 0.21238186955451965, 0.46927347779273987}; + std::vector x_dims = {1, 3, 8, 8}; + std::vector expected_dims = {1, 3, 1, 1}; + std::vector expected_vals = {0.5146896243095398, 0.4851023256778717, 0.4756942689418793}; + + test.AddInput("X", x_dims, x_vals); + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(PoolTest, LpPool) { + OpTester test("LpPool"); + + test.AddAttribute("auto_pad", ""); + test.AddAttribute("strides", std::vector{1, 1}); + test.AddAttribute("pads", vector{0, 0, 0, 0}); + test.AddAttribute("kernel_shape", vector{3, 3}); + + std::vector x_vals = {0.688458621501922607421875, + 0.8835647106170654296875, + 0.782541573047637939453125, + 0.300049364566802978515625, + 0.8066387176513671875, + 0.4520850479602813720703125, + 0.598959147930145263671875, + 0.4597113132476806640625, + 0.8161861896514892578125, + 0.7667262554168701171875, + 0.840198040008544921875, + 0.583297073841094970703125, + 0.708858668804168701171875, + 0.4728293716907501220703125, + 0.4992314875125885009765625, + 0.2504110038280487060546875, + 0.3881411850452423095703125, + 0.517398893833160400390625, + 0.657192409038543701171875, + 0.7325098514556884765625, + 0.10206781327724456787109375, + 0.2179393768310546875, + 0.0616043470799922943115234375, + 0.475992143154144287109375, + 0.737536609172821044921875, + 0.9689886569976806640625, + 0.4474093914031982421875, + 0.4323260486125946044921875, + 0.648917853832244873046875, + 0.701454102993011474609375, + 0.107639573514461517333984375, + 0.811198413372039794921875, + 0.725269258022308349609375, + 0.7497208118438720703125, + 0.3340204060077667236328125, + 0.87611293792724609375, + 0.6691205501556396484375, + 0.87189638614654541015625, + 0.971237838268280029296875, + 0.11620916426181793212890625, + 0.0249019563198089599609375, + 0.752140820026397705078125, + 0.865541160106658935546875, + 0.015474068932235240936279296875, + 0.5126011371612548828125, + 0.45315420627593994140625, + 0.1925573647022247314453125, + 0.98408544063568115234375, + 0.14754636585712432861328125, + 0.54990971088409423828125, + 0.903382003307342529296875, + 0.2905881404876708984375, + 0.33750665187835693359375, + 0.3232279717922210693359375, + 0.07346880435943603515625, + 0.3991589844226837158203125, + 0.903037011623382568359375, + 0.083290748298168182373046875, + 0.20850212872028350830078125, + 0.05971308052539825439453125, + 0.4810305535793304443359375, + 0.087783016264438629150390625, + 0.2952007353305816650390625, + 0.2153458297252655029296875, + 0.4049233496189117431640625, + 0.7175214290618896484375, + 0.872620165348052978515625, + 0.522788941860198974609375, + 0.43519175052642822265625, + 0.0193175189197063446044921875, + 0.846780240535736083984375, + 0.5219886302947998046875, + 0.242856085300445556640625, + 0.2003507316112518310546875, + 0.8327982425689697265625, + 0.18934874236583709716796875, + 0.917275846004486083984375, + 0.658357441425323486328125, + 0.847428977489471435546875, + 0.81426322460174560546875, + 0.036692313849925994873046875, + 0.132266581058502197265625, + 0.357086241245269775390625, + 0.4745192825794219970703125, + 0.821886956691741943359375, + 0.2454545795917510986328125, + 0.1065533459186553955078125, + 0.791345179080963134765625, + 0.545370578765869140625, + 0.3979628086090087890625, + 0.49180948734283447265625, + 0.1297818124294281005859375, + 0.36476039886474609375, + 0.3085542619228363037109375, + 0.899958193302154541015625, + 0.4159581661224365234375, + 0.675307571887969970703125, + 0.829472124576568603515625, + 0.2064842283725738525390625, + 0.64016926288604736328125, + 0.20317254960536956787109375, + 0.61657464504241943359375, + 0.290811240673065185546875, + 0.26665222644805908203125, + 0.3393469750881195068359375, + 0.2539980709552764892578125, + 0.791014850139617919921875, + 0.940179288387298583984375, + 0.827880084514617919921875, + 0.460959732532501220703125, + 0.63165509700775146484375, + 0.1342843472957611083984375, + 0.583048880100250244140625, + 0.4310896396636962890625, + 0.070260427892208099365234375, + 0.518509685993194580078125, + 0.255076229572296142578125, + 0.588839232921600341796875, + 0.13979454338550567626953125, + 0.816810190677642822265625, + 0.506142139434814453125, + 0.780538499355316162109375, + 0.70891857147216796875, + 0.775202929973602294921875, + 0.33364391326904296875, + 0.21829630434513092041015625, + 0.794861137866973876953125, + 0.440593779087066650390625, + 0.51086711883544921875, + 0.059619002044200897216796875, + 0.626003265380859375, + 0.831237018108367919921875, + 0.775263965129852294921875, + 0.48013699054718017578125, + 0.98830425739288330078125, + 0.5461161136627197265625, + 0.0545087419450283050537109375, + 0.067873962223529815673828125, + 0.334798395633697509765625, + 0.083531044423580169677734375, + 0.1419331729412078857421875, + 0.62124884128570556640625, + 0.4215275943279266357421875, + 0.349430382251739501953125, + 0.645228683948516845703125, + 0.15098969638347625732421875, + 0.789717197418212890625, + 0.59648799896240234375, + 0.3775124251842498779296875, + 0.2767163217067718505859375, + 0.558230340480804443359375, + 0.991863429546356201171875, + 0.813561499118804931640625, + 0.79598820209503173828125, + 0.567295074462890625, + 0.4774146378040313720703125, + 0.3510249555110931396484375, + 0.681096494197845458984375, + 0.745837032794952392578125, + 0.681192934513092041015625, + 0.88084888458251953125, + 0.52995645999908447265625, + 0.087239809334278106689453125, + 0.414192855358123779296875, + 0.539312899112701416015625, + 0.23079840838909149169921875, + 0.548077642917633056640625, + 0.3750600516796112060546875, + 0.3628396093845367431640625, + 0.078880332410335540771484375, + 0.95263445377349853515625, + 0.41051447391510009765625, + 0.820193827152252197265625, + 0.4604322016239166259765625, + 0.3603973090648651123046875, + 0.5672309398651123046875, + 0.685865581035614013671875, + 0.7147781848907470703125, + 0.772135257720947265625, + 0.623492062091827392578125, + 0.7632234096527099609375, + 0.877109348773956298828125, + 0.096309013664722442626953125, + 0.21554203331470489501953125, + 0.254471242427825927734375, + 0.58027327060699462890625, + 0.3754498958587646484375, + 0.717136919498443603515625, + 0.2995398044586181640625, + 0.931284368038177490234375, + 0.011751591227948665618896484375, + 0.07255984842777252197265625, + 0.87918460369110107421875, + 0.02955267764627933502197265625, + 0.889126598834991455078125, + 0.0329551957547664642333984375, + 0.23701806366443634033203125, + 0.5436298847198486328125, + 0.4716108739376068115234375, + 0.1311373412609100341796875, + 0.983278572559356689453125, + 0.571916878223419189453125, + 0.739863812923431396484375, + 0.28372323513031005859375, + 0.18242438137531280517578125, + 0.522270500659942626953125, + 0.880189239978790283203125, + 0.530347883701324462890625, + 0.3022750318050384521484375, + 0.02125177718698978424072265625, + 0.76706016063690185546875, + 0.666437804698944091796875, + 0.5887668132781982421875, + 0.3817012608051300048828125, + 0.069761075079441070556640625, + 0.13000230491161346435546875, + 0.3799968063831329345703125, + 0.92774105072021484375, + 0.2970103323459625244140625, + 0.2885017096996307373046875, + 0.644755303859710693359375, + 0.4826243221759796142578125, + 0.02549990825355052947998046875, + 0.845977962017059326171875, + 0.1354812681674957275390625, + 0.59001064300537109375, + 0.786619603633880615234375, + 0.808787405490875244140625, + 0.850969374179840087890625, + 0.864635884761810302734375, + 0.9816544055938720703125, + 0.704220354557037353515625, + 0.406329214572906494140625, + 0.4230716228485107421875, + 0.410357534885406494140625, + 0.7462520599365234375, + 0.251948177814483642578125, + 0.3785230815410614013671875, + 0.704321324825286865234375, + 0.0714503824710845947265625, + 0.906627714633941650390625, + 0.0333719812333583831787109375, 0.654077053070068359375}; + std::vector x_dims = {1, 3, 9, 9}; + std::vector expected_dims = {1, 3, 7, 7}; + std::vector expected_vals = {2.1165919303894043, 1.9042642116546631, 1.5751385688781738, + 1.4826388359069824, 1.5885931253433228, 1.7165449857711792, + 1.8440124988555908, 1.9269057512283325, 1.7515288591384888, + 1.5131627321243286, 1.5648597478866577, 1.7481330633163452, + 1.8362259864807129, 1.8987786769866943, 2.056734561920166, + 1.7989484071731567, 1.476754903793335, 1.4329502582550049, + 1.9585609436035156, 2.0552983283996582, 2.0338289737701416, + 2.1123726367950439, 1.9154638051986694, 1.8470758199691772, + 1.7075581550598145, 2.0650856494903564, 1.8786256313323975, + 1.6601848602294922, 2.0838139057159424, 1.9302912950515747, + 1.7651937007904053, 1.3319482803344727, 1.6723839044570923, + 1.6038172245025635, 1.281104564666748, 1.7076961994171143, + 1.8572235107421875, 1.9256408214569092, 1.5551244020462036, + 1.3944330215454102, 1.4710251092910767, 1.2723797559738159, + 1.5805213451385498, 1.786491870880127, 1.9965716600418091, + 1.6089824438095093, 1.6536226272583008, 1.7216441631317139, + 1.6427503824234009, 1.2622216939926147, 1.3339006900787354, + 1.5921475887298584, 1.4477853775024414, 1.5451828241348267, + 1.7485626935958862, 1.9603283405303955, 1.5874154567718506, + 1.174997091293335, 1.5267566442489624, 1.3757904767990112, + 1.4901281595230103, 1.6068876981735229, 1.7605991363525391, + 1.7780805826187134, 1.441672682762146, 1.6808938980102539, + 1.4773738384246826, 1.5793166160583496, 1.5747464895248413, + 1.6349068880081177, 1.8485732078552246, 1.4251554012298584, + 1.7163872718811035, 1.7315287590026855, 1.9817506074905396, + 1.7880076169967651, 1.7050145864486694, 1.557621955871582, + 1.2333823442459106, 1.5207540988922119, 1.6104618310928345, + 1.9518419504165649, 1.8223953247070312, 1.8038734197616577, + 1.567004919052124, 1.2572110891342163, 1.3791522979736328, + 1.3418225049972534, 1.6210030317306519, 1.8650168180465698, + 2.1098208427429199, 1.5974785089492798, 1.3397328853607178, + 1.435505747795105, 1.3628946542739868, 1.558194637298584, + 1.9369972944259644, 2.0405406951904297, 1.69834303855896, + 1.5347113609313965, 1.1952571868896484, 1.36539626121521, + 1.5550618171691895, 1.6876083612442017, 1.8127884864807129, + 1.8130189180374146, 1.6180311441421509, 1.2502912282943726, + 1.7129987478256226, 1.6241954565048218, 1.848590612411499, + 1.6104695796966553, 1.8547911643981934, 1.7072041034698486, + 1.6555715799331665, 1.722585916519165, 1.4128400087356567, + 1.4920854568481445, 1.4759902954101562, 1.5659612417221069, + 1.664239764213562, 1.9113870859146118, 1.9744715690612793, + 1.5460153818130493, 1.315888524055481, 1.2653214931488037, + 1.6348875761032104, 1.7388149499893188, 1.8604984283447266, + 1.751006007194519, 1.4989807605743408, 1.3538862466812134, + 1.4081637859344482, 1.7571074962615967, 1.9259833097457886, + 1.9354615211486816, 2.1322348117828369, 1.9475457668304443, + 1.7524666786193848, 1.3199115991592407, 1.8716570138931274, + 1.9475260972976685, 1.8482059240341187, 1.9523605108261108, + 2.0444071292877197, 1.8444844484329224, 1.6809544563293457}; + + test.AddInput("X", x_dims, x_vals); + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(PoolTest, GlobalLpPool) { + OpTester test("GlobalLpPool"); + test.AddAttribute("p", static_cast(3)); + std::vector x_vals = {0.688458621501922607421875, + 0.8835647106170654296875, + 0.782541573047637939453125, + 0.300049364566802978515625, + 0.8066387176513671875, + 0.4520850479602813720703125, + 0.598959147930145263671875, + 0.4597113132476806640625, + 0.8161861896514892578125, + 0.7667262554168701171875, + 0.840198040008544921875, + 0.583297073841094970703125, + 0.708858668804168701171875, + 0.4728293716907501220703125, + 0.4992314875125885009765625, + 0.2504110038280487060546875, + 0.3881411850452423095703125, + 0.517398893833160400390625, + 0.657192409038543701171875, + 0.7325098514556884765625, + 0.10206781327724456787109375, + 0.2179393768310546875, + 0.0616043470799922943115234375, + 0.475992143154144287109375, + 0.737536609172821044921875, + 0.9689886569976806640625, + 0.4474093914031982421875, + 0.4323260486125946044921875, + 0.648917853832244873046875, + 0.701454102993011474609375, + 0.107639573514461517333984375, + 0.811198413372039794921875, + 0.725269258022308349609375, + 0.7497208118438720703125, + 0.3340204060077667236328125, + 0.87611293792724609375, + 0.6691205501556396484375, + 0.87189638614654541015625, + 0.971237838268280029296875, + 0.11620916426181793212890625, + 0.0249019563198089599609375, + 0.752140820026397705078125, + 0.865541160106658935546875, + 0.015474068932235240936279296875, + 0.5126011371612548828125, + 0.45315420627593994140625, + 0.1925573647022247314453125, + 0.98408544063568115234375, + 0.14754636585712432861328125, + 0.54990971088409423828125, + 0.903382003307342529296875, + 0.2905881404876708984375, + 0.33750665187835693359375, + 0.3232279717922210693359375, + 0.07346880435943603515625, + 0.3991589844226837158203125, + 0.903037011623382568359375, + 0.083290748298168182373046875, + 0.20850212872028350830078125, + 0.05971308052539825439453125, + 0.4810305535793304443359375, + 0.087783016264438629150390625, + 0.2952007353305816650390625, + 0.2153458297252655029296875, + 0.4049233496189117431640625, + 0.7175214290618896484375, + 0.872620165348052978515625, + 0.522788941860198974609375, + 0.43519175052642822265625, + 0.0193175189197063446044921875, + 0.846780240535736083984375, + 0.5219886302947998046875, + 0.242856085300445556640625, + 0.2003507316112518310546875, + 0.8327982425689697265625, + 0.18934874236583709716796875, + 0.917275846004486083984375, + 0.658357441425323486328125, + 0.847428977489471435546875, + 0.81426322460174560546875, + 0.036692313849925994873046875, + 0.132266581058502197265625, + 0.357086241245269775390625, + 0.4745192825794219970703125, + 0.821886956691741943359375, + 0.2454545795917510986328125, + 0.1065533459186553955078125, + 0.791345179080963134765625, + 0.545370578765869140625, + 0.3979628086090087890625, + 0.49180948734283447265625, + 0.1297818124294281005859375, + 0.36476039886474609375, + 0.3085542619228363037109375, + 0.899958193302154541015625, + 0.4159581661224365234375, + 0.675307571887969970703125, + 0.829472124576568603515625, + 0.2064842283725738525390625, + 0.64016926288604736328125, + 0.20317254960536956787109375, + 0.61657464504241943359375, + 0.290811240673065185546875, + 0.26665222644805908203125, + 0.3393469750881195068359375, + 0.2539980709552764892578125, + 0.791014850139617919921875, + 0.940179288387298583984375, + 0.827880084514617919921875, + 0.460959732532501220703125, + 0.63165509700775146484375, + 0.1342843472957611083984375, + 0.583048880100250244140625, + 0.4310896396636962890625, + 0.070260427892208099365234375, + 0.518509685993194580078125, + 0.255076229572296142578125, + 0.588839232921600341796875, + 0.13979454338550567626953125, + 0.816810190677642822265625, + 0.506142139434814453125, + 0.780538499355316162109375, + 0.70891857147216796875, + 0.775202929973602294921875, + 0.33364391326904296875, + 0.21829630434513092041015625, + 0.794861137866973876953125, + 0.440593779087066650390625, + 0.51086711883544921875, + 0.059619002044200897216796875, + 0.626003265380859375, + 0.831237018108367919921875, + 0.775263965129852294921875, + 0.48013699054718017578125, + 0.98830425739288330078125, + 0.5461161136627197265625, + 0.0545087419450283050537109375, + 0.067873962223529815673828125, + 0.334798395633697509765625, + 0.083531044423580169677734375, + 0.1419331729412078857421875, + 0.62124884128570556640625, + 0.4215275943279266357421875, + 0.349430382251739501953125, + 0.645228683948516845703125, + 0.15098969638347625732421875, + 0.789717197418212890625, + 0.59648799896240234375, + 0.3775124251842498779296875, + 0.2767163217067718505859375, + 0.558230340480804443359375, + 0.991863429546356201171875, + 0.813561499118804931640625, + 0.79598820209503173828125, + 0.567295074462890625, + 0.4774146378040313720703125, + 0.3510249555110931396484375, + 0.681096494197845458984375, + 0.745837032794952392578125, + 0.681192934513092041015625, + 0.88084888458251953125, + 0.52995645999908447265625, + 0.087239809334278106689453125, + 0.414192855358123779296875, + 0.539312899112701416015625, + 0.23079840838909149169921875, + 0.548077642917633056640625, + 0.3750600516796112060546875, + 0.3628396093845367431640625, + 0.078880332410335540771484375, + 0.95263445377349853515625, + 0.41051447391510009765625, + 0.820193827152252197265625, + 0.4604322016239166259765625, + 0.3603973090648651123046875, + 0.5672309398651123046875, + 0.685865581035614013671875, + 0.7147781848907470703125, + 0.772135257720947265625, + 0.623492062091827392578125, + 0.7632234096527099609375, + 0.877109348773956298828125, + 0.096309013664722442626953125, + 0.21554203331470489501953125, + 0.254471242427825927734375, + 0.58027327060699462890625, + 0.3754498958587646484375, + 0.717136919498443603515625, + 0.2995398044586181640625, + 0.931284368038177490234375, + 0.011751591227948665618896484375, + 0.07255984842777252197265625, + 0.87918460369110107421875, + 0.02955267764627933502197265625, + 0.889126598834991455078125, + 0.0329551957547664642333984375, + 0.23701806366443634033203125, + 0.5436298847198486328125, + 0.4716108739376068115234375, + 0.1311373412609100341796875, + 0.983278572559356689453125, + 0.571916878223419189453125, + 0.739863812923431396484375, + 0.28372323513031005859375, + 0.18242438137531280517578125, + 0.522270500659942626953125, + 0.880189239978790283203125, + 0.530347883701324462890625, + 0.3022750318050384521484375, + 0.02125177718698978424072265625, + 0.76706016063690185546875, + 0.666437804698944091796875, + 0.5887668132781982421875, + 0.3817012608051300048828125, + 0.069761075079441070556640625, + 0.13000230491161346435546875, + 0.3799968063831329345703125, + 0.92774105072021484375, + 0.2970103323459625244140625, + 0.2885017096996307373046875, + 0.644755303859710693359375, + 0.4826243221759796142578125, + 0.02549990825355052947998046875, + 0.845977962017059326171875, + 0.1354812681674957275390625, + 0.59001064300537109375, + 0.786619603633880615234375, + 0.808787405490875244140625, + 0.850969374179840087890625, + 0.864635884761810302734375, + 0.9816544055938720703125, + 0.704220354557037353515625, + 0.406329214572906494140625, + 0.4230716228485107421875, + 0.410357534885406494140625, + 0.7462520599365234375, + 0.251948177814483642578125, + 0.3785230815410614013671875, + 0.704321324825286865234375, + 0.0714503824710845947265625, + 0.906627714633941650390625, + 0.0333719812333583831787109375, 0.654077053070068359375}; + std::vector x_dims = {1, 3, 9, 9}; + std::vector expected_dims = {1, 3, 1, 1}; + std::vector expected_vals = {2.7576668262481689453125, 2.6182243824005126953125, + 2.682276248931884765625}; + + test.AddInput("X", x_dims, x_vals); + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc b/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc new file mode 100644 index 0000000000000..42e5928afcb8d --- /dev/null +++ b/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc @@ -0,0 +1,1107 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/reduction/reduction_ops.h" +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" +#include "test/providers/cpu/reduction/reduction_test_cases.h" + +namespace onnxruntime { +namespace test { + +template +void TestReduceOp(const std::string& op, + const std::vector& input_dims, + const std::vector& data, + const std::vector& axes, + int64_t keepdims, + const std::vector& expected_dims, + const std::vector& expected_data) + +{ + OpTester test(op.c_str()); + if (!axes.empty()) { + if (op.compare("ArgMax") == 0 || op.compare("ArgMin") == 0) + test.AddAttribute("axis", axes[0]); + else + test.AddAttribute("axes", axes); + } + test.AddAttribute("keepdims", keepdims); + test.AddInput("data", input_dims, data); + test.AddOutput("reduced", expected_dims, expected_data); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider}); +} + +TEST(ReductionOpTest, ReductionVariationTest) { + const std::vector& input_data = testcases.input_data; + const std::vector& input_dims = testcases.input_dims; + OpAttributesResultMap& opAttributesResultMap = testcases.map_op_attribute_expected; + + for (auto a : opAttributesResultMap) { + const ReductionAttribute& attributes = std::get<0>(a.second); + const std::vector expected_dims = std::get<1>(a.second); + if (a.first.compare("ArgMax") == 0 || a.first.compare("ArgMin") == 0) { + std::vector expected_values; + for (auto v : std::get<2>(a.second)) + expected_values.push_back(static_cast(v)); + TestReduceOp(a.first, input_dims, input_data, attributes.axes_, attributes.keep_dims_, + expected_dims, expected_values); + } else { + const std::vector expected_values = std::get<2>(a.second); + TestReduceOp(a.first, input_dims, input_data, attributes.axes_, attributes.keep_dims_, + expected_dims, expected_values); + } + } +} + +TEST(ReductionOpTest, ReduceL1_default_axes_keepdims) { + OpTester test("ReduceL1"); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {1, 1, 1}, {78.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceL1_do_not_keepdims) { + OpTester test("ReduceL1"); + test.AddAttribute("axes", std::vector{2}); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {3, 2}, {3.0f, 7.0f, 11.0f, 15.0f, 19.0f, 23.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceL1_do_not_keepdims_2) { + OpTester test("ReduceL1"); + test.AddAttribute("axes", std::vector{0}); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3}, + {1.0f, 2.0f, 3.0f}); + test.AddOutput("reduced", {}, {6.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceL1_keepdims) { + OpTester test("ReduceL1"); + test.AddAttribute("axes", std::vector{2}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {3, 2, 1}, {3.0f, 7.0f, 11.0f, 15.0f, 19.0f, 23.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceL1) { + OpTester test("ReduceL1"); + test.AddAttribute("axes", std::vector{0, 2}); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {1, 2, 1}, {33.0f, 45.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceL1_int32) { + OpTester test("ReduceL1"); + test.AddAttribute("axes", std::vector{0, 2}); + test.AddInput("data", {3, 2, 2}, + {1, 2, + 3, 4, + + 5, 6, + 7, 8, + + 9, 10, + 11, 12}); + test.AddOutput("reduced", {1, 2, 1}, {33, 45}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceL2_default_axes_keepdims) { + OpTester test("ReduceL2"); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {1, 1, 1}, {25.49509757f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceL2_do_not_keepdims) { + OpTester test("ReduceL2"); + test.AddAttribute("axes", std::vector{2}); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {3, 2}, {2.23606798f, 5.0f, 7.81024968f, 10.63014581f, 13.45362405f, 16.2788206f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceL2_do_not_keepdims_2) { + OpTester test("ReduceL2"); + test.AddAttribute("axes", std::vector{0}); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3}, + {1.0f, 2.0f, 3.0f}); + test.AddOutput("reduced", {}, {3.741657387f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceL2_keepdims) { + OpTester test("ReduceL2"); + test.AddAttribute("axes", std::vector{2}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {3, 2, 1}, {2.23606798f, 5.0f, 7.81024968f, 10.63014581f, 13.45362405f, 16.2788206f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceL2) { + OpTester test("ReduceL2"); + test.AddAttribute("axes", std::vector{0, 2}); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {2}, {15.71623325f, 20.07485962f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceL2_int32) { + OpTester test("ReduceL2"); + test.AddAttribute("axes", std::vector{0, 2}); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3, 2, 2}, + {1, 2, + 3, 4, + + 5, 6, + 7, 8, + + 9, 10, + 11, 12}); + test.AddOutput("reduced", {2}, {15, 20}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceLogSum) { + OpTester test("ReduceLogSum"); + test.AddAttribute("axes", std::vector{1}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {3, 1, 2}, + {1.38629436f, 1.79175949f, + 2.48490667f, 2.6390574f, + 2.99573231f, 3.09104252f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceLogSum_do_not_keepdims_2) { + OpTester test("ReduceLogSum"); + test.AddAttribute("axes", std::vector{0}); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3}, + {1.0f, 2.0f, 3.0f}); + test.AddOutput("reduced", {}, {1.79175947f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceLogSumExp_default_axes_keepdims) { + OpTester test("ReduceLogSumExp"); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {5.0f, 1.0f, + 20.0f, 2.0f, + + 30.0f, 1.0f, + 40.0f, 2.0f, + + 55.0f, 1.0f, + 60.0f, 2.0f}); + test.AddOutput("reduced", {1, 1, 1}, {60.00671387f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceLogSumExp_do_not_keepdims) { + OpTester test("ReduceLogSumExp"); + test.AddAttribute("axes", std::vector{1}); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3, 2, 2}, + {5.0f, 1.0f, + 20.0f, 2.0f, + + 30.0f, 1.0f, + 40.0f, 2.0f, + + 55.0f, 1.0f, + 60.0f, 2.0f}); + test.AddOutput("reduced", {3, 2}, {20.0f, 2.31326175f, 40.00004578f, 2.31326175f, 60.00671387f, 2.31326175f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceLogSumExp_do_not_keepdims_2) { + OpTester test("ReduceLogSumExp"); + test.AddAttribute("axes", std::vector{0}); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3}, + {1.0f, 2.0f, 3.0f}); + test.AddOutput("reduced", {}, {3.40760596f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceLogSumExp_keepdims) { + OpTester test("ReduceLogSumExp"); + test.AddAttribute("axes", std::vector{1}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {5.0f, 1.0f, + 20.0f, 2.0f, + + 30.0f, 1.0f, + 40.0f, 2.0f, + + 55.0f, 1.0f, + 60.0f, 2.0f}); + test.AddOutput("reduced", {3, 1, 2}, {20.0f, 2.31326175f, 40.00004578f, 2.31326175f, 60.00671387f, 2.31326175f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceLogSumExp) { + OpTester test("ReduceLogSumExp"); + test.AddAttribute("axes", std::vector{0, 2}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {1, 2, 1}, {10.33174133f, 12.33174133f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceLogSumExp_int32) { + OpTester test("ReduceLogSumExp"); + test.AddAttribute("axes", std::vector{0, 2}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {1, 2, + 3, 4, + + 5, 6, + 7, 8, + + 9, 10, + 11, 12}); + test.AddOutput("reduced", {1, 2, 1}, {10, 12}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceMax_default_axes_keepdims) { + OpTester test("ReduceMax"); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {5.0f, 1.0f, + 20.0f, 2.0f, + + 30.0f, 1.0f, + 40.0f, 2.0f, + + 55.0f, 1.0f, + 60.0f, 2.0f}); + test.AddOutput("reduced", {1, 1, 1}, {60.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceMax_do_not_keepdims) { + OpTester test("ReduceMax"); + test.AddAttribute("axes", std::vector{1}); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3, 2, 2}, + {5.0f, 1.0f, + 20.0f, 2.0f, + + 30.0f, 1.0f, + 40.0f, 2.0f, + + 55.0f, 1.0f, + 60.0f, 2.0f}); + test.AddOutput("reduced", {3, 2}, {20.0f, 2.0f, 40.0f, 2.0f, 60.0f, 2.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceMax_do_not_keepdims_2) { + OpTester test("ReduceMax"); + test.AddAttribute("axes", std::vector{0}); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3}, + {5.0f, 1.0f, 20.0f}); + test.AddOutput("reduced", {}, {20.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceMax_keepdims) { + OpTester test("ReduceMax"); + test.AddAttribute("axes", std::vector{1}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {5.0f, 1.0f, + 20.0f, 2.0f, + + 30.0f, 1.0f, + 40.0f, 2.0f, + + 55.0f, 1.0f, + 60.0f, 2.0f}); + test.AddOutput("reduced", {3, 1, 2}, {20.0f, 2.0f, 40.0f, 2.0f, 60.0f, 2.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceMax) { + OpTester test("ReduceMax"); + test.AddAttribute("axes", std::vector{1, 2}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {3, 1, 1}, {4.0f, 8.0f, 12.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceMax_int32) { + OpTester test("ReduceMax"); + test.AddAttribute("axes", std::vector{1, 2}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {1, 2, + 3, 4, + + 5, 6, + 7, 8, + + 9, 10, + 11, 12}); + test.AddOutput("reduced", {3, 1, 1}, {4, 8, 12}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceMean_default_axes_keepdims) { + OpTester test("ReduceMean"); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {5.0f, 1.0f, + 20.0f, 2.0f, + + 30.0f, 1.0f, + 40.0f, 2.0f, + + 55.0f, 1.0f, + 60.0f, 2.0f}); + test.AddOutput("reduced", {1, 1, 1}, {18.25f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceMean_do_not_keepdims) { + OpTester test("ReduceMean"); + test.AddAttribute("axes", std::vector{1}); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3, 2, 2}, + {5.0f, 1.0f, + 20.0f, 2.0f, + + 30.0f, 1.0f, + 40.0f, 2.0f, + + 55.0f, 1.0f, + 60.0f, 2.0f}); + test.AddOutput("reduced", {3, 2}, {12.5f, 1.5f, 35.0f, 1.5f, 57.5f, 1.5f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceMean_do_not_keepdims_2) { + OpTester test("ReduceMean"); + test.AddAttribute("axes", std::vector{0}); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3}, + {1.0f, 2.0f, 3.0f}); + test.AddOutput("reduced", {}, {2.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceMean_keepdims) { + OpTester test("ReduceMean"); + test.AddAttribute("axes", std::vector{1}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {5.0f, 1.0f, + 20.0f, 2.0f, + + 30.0f, 1.0f, + 40.0f, 2.0f, + + 55.0f, 1.0f, + 60.0f, 2.0f}); + test.AddOutput("reduced", {3, 1, 2}, {12.5f, 1.5f, 35.0f, 1.5f, 57.5f, 1.5f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceMean) { + OpTester test("ReduceMean"); + test.AddAttribute("axes", std::vector{0, 2}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {1, 2, 1}, {5.5f, 7.5f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceMean_int32) { + OpTester test("ReduceMean"); + test.AddAttribute("axes", std::vector{0, 2}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {10, 20, + 30, 40, + + 50, 60, + 70, 80, + + 90, 100, + 110, 120}); + test.AddOutput("reduced", {1, 2, 1}, {55, 75}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceMin_default_axes_keepdims) { + OpTester test("ReduceMin"); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {5.0f, 1.0f, + 20.0f, 2.0f, + + 30.0f, 1.0f, + 40.0f, 2.0f, + + 55.0f, 1.0f, + 60.0f, 2.0f}); + test.AddOutput("reduced", {1, 1, 1}, {1.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceMin_do_not_keepdims) { + OpTester test("ReduceMin"); + test.AddAttribute("axes", std::vector{1}); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3, 2, 2}, + {5.0f, 1.0f, + 20.0f, 2.0f, + + 30.0f, 1.0f, + 40.0f, 2.0f, + + 55.0f, 1.0f, + 60.0f, 2.0f}); + test.AddOutput("reduced", {3, 2}, {5.0f, 1.0f, 30.0f, 1.0f, 55.0f, 1.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceMin_do_not_keepdims_2) { + OpTester test("ReduceMin"); + test.AddAttribute("axes", std::vector{0}); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3}, + {5.0f, 1.0f, 20.0f}); + test.AddOutput("reduced", {}, {1.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceMin_keepdims) { + OpTester test("ReduceMin"); + test.AddAttribute("axes", std::vector{1}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {5.0f, 1.0f, + 20.0f, 2.0f, + + 30.0f, 1.0f, + 40.0f, 2.0f, + + 55.0f, 1.0f, + 60.0f, 2.0f}); + test.AddOutput("reduced", {3, 1, 2}, {5.0f, 1.0f, 30.0f, 1.0f, 55.0f, 1.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceMin) { + OpTester test("ReduceMin"); + test.AddAttribute("axes", std::vector{0, 2}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {1, 2, 1}, {1.0f, 3.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceMin_int32) { + OpTester test("ReduceMin"); + test.AddAttribute("axes", std::vector{0, 2}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {1, 2, + 3, 4, + + 5, 6, + 7, 8, + + 9, 10, + 11, 12}); + test.AddOutput("reduced", {1, 2, 1}, {1, 3}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceSum) { + OpTester test("ReduceSum"); + test.AddAttribute("axes", std::vector{0, 2}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {1, 2, 1}, {33.0f, 45.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceSum_axes01) { + OpTester test("ReduceSum"); + test.AddAttribute("axes", std::vector{2}); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {3, 2}, {3.0f, 7.0f, 11.0f, 15.0f, 19.0f, 23.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceSum_axes02) { + OpTester test("ReduceSum"); + test.AddAttribute("axes", std::vector{1}); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {3, 2}, {4.0f, 6.0f, 12.0f, 14.0f, 20.0f, 22.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceSum_int32) { + OpTester test("ReduceSum"); + test.AddAttribute("axes", std::vector{0, 2}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {1, 2, + 3, 4, + + 5, 6, + 7, 8, + + 9, 10, + 11, 12}); + test.AddOutput("reduced", {1, 2, 1}, {33, 45}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceSum_default_axes_keepdims) { + OpTester test("ReduceSum"); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {1, 1, 1}, {78.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceSum_do_not_keepdims) { + OpTester test("ReduceSum"); + test.AddAttribute("axes", std::vector{1}); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {1, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f}); + test.AddOutput("reduced", {1, 2}, {4.0f, 6.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceSum_do_not_keepdims_2) { + OpTester test("ReduceSum"); + test.AddAttribute("axes", std::vector{0}); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3}, + {1.0f, 2.0f, 3.0f}); + test.AddOutput("reduced", {}, {6.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceSum_keepdims) { + OpTester test("ReduceSum"); + test.AddAttribute("axes", std::vector{1}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {3, 1, 2}, {4.0f, 6.0f, 12.0f, 14.0f, 20.0f, 22.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceSumSquare) { + OpTester test("ReduceSumSquare"); + test.AddAttribute("axes", std::vector{0, 2}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {1, 2, 1}, {247.0f, 403.f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceSumSquare_int32) { + OpTester test("ReduceSumSquare"); + test.AddAttribute("axes", std::vector{0, 2}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {1, 2, + 3, 4, + + 5, 6, + 7, 8, + + 9, 10, + 11, 12}); + test.AddOutput("reduced", {1, 2, 1}, {247, 403}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceSumSquare_default_axes_keepdims) { + OpTester test("ReduceSumSquare"); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {1, 1, 1}, {650.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceSumSquare_do_not_keepdims) { + OpTester test("ReduceSumSquare"); + test.AddAttribute("axes", std::vector{1}); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {3, 2}, {10.0f, 20.0f, 74.0f, 100.0f, 202.0f, 244.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceSumSquare_do_not_keepdims_2) { + OpTester test("ReduceSumSquare"); + test.AddAttribute("axes", std::vector{0}); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3}, + {1.0f, 2.0f, 3.0f}); + test.AddOutput("reduced", {}, {14.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceSumSquare_keepdims) { + OpTester test("ReduceSumSquare"); + test.AddAttribute("axes", std::vector{1}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {3, 1, 2}, {10.0f, 20.0f, 74.0f, 100.0f, 202.0f, 244.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceProd_default_axes_keepdims) { + OpTester test("ReduceProd"); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {1, 1, 1}, {479001600.f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceProd_do_not_keepdims) { + OpTester test("ReduceProd"); + test.AddAttribute("axes", std::vector{1}); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {3, 2}, {3.f, 8.f, 35.f, 48.f, 99.f, 120.f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceProd_do_not_keepdims_2) { + OpTester test("ReduceProd"); + test.AddAttribute("axes", std::vector{0}); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3}, + {1.0f, 2.0f, 3.0f}); + test.AddOutput("reduced", {}, {6.0f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceProd_keepdims) { + OpTester test("ReduceProd"); + test.AddAttribute("axes", std::vector{1}); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {3, 1, 2}, {3.f, 8.f, 35.f, 48.f, 99.f, 120.f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceProd) { + OpTester test("ReduceProd"); + test.AddAttribute("axes", std::vector{0, 2}); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {1, 2, 1}, {5400.f, 88704.f}); + test.Run(); +} + +TEST(ReductionOpTest, ReduceProd_int32) { + OpTester test("ReduceProd"); + test.AddAttribute("axes", std::vector{0, 2}); + test.AddInput("data", {3, 2, 2}, + {1, 2, + 3, 4, + + 5, 6, + 7, 8, + + 9, 10, + 11, 12}); + test.AddOutput("reduced", {1, 2, 1}, {5400, 88704}); + test.Run(); +} + +TEST(ReductionOpTest, ArgMax) { + OpTester test("ArgMax"); + test.AddAttribute("axis", (int64_t)1); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {3, 1, 2}, + {1, 1, + 1, 1, + 1, 1}); + test.Run(); +} + +TEST(ReductionOpTest, ArgMax_do_not_keepdims) { + OpTester test("ArgMax"); + test.AddAttribute("axis", (int64_t)1); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {3, 2}, + {1, 1, + 1, 1, + 1, 1}); + test.Run(); +} + +TEST(ReductionOpTest, ArgMax_do_not_keepdims_2) { + OpTester test("ArgMax"); + test.AddAttribute("axis", (int64_t)0); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3}, + {1.0f, 2.0f, 3.0f}); + test.AddOutput("reduced", {}, + {2}); + test.Run(); +} + +TEST(ReductionOpTest, ArgMax_int32) { + OpTester test("ArgMax"); + test.AddAttribute("axis", (int64_t)1); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {1, 2, + 3, 4, + + 5, 6, + 7, 8, + + 9, 10, + 11, 12}); + test.AddOutput("reduced", {3, 1, 2}, + {1, 1, + 1, 1, + 1, 1}); + test.Run(); +} + +TEST(ReductionOpTest, ArgMax2D) { + OpTester test("ArgMax"); + test.AddAttribute("axis", (int64_t)1); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2}, + {1.0f, 2.0f, + 6.0f, 5.0f, + 9.0f, 10.0f}); + test.AddOutput("reduced", {3, 1}, + {1, 0, 1}); + test.Run(); +} + +TEST(ReductionOpTest, ArgMin) { + OpTester test("ArgMin"); + test.AddAttribute("axis", (int64_t)0); + test.AddAttribute("keepdims", (int64_t)1); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {1, 2, 2}, + {0, 0, + 0, 0}); + test.Run(); +} + +TEST(ReductionOpTest, ArgMin_do_not_keepdims) { + OpTester test("ArgMin"); + test.AddAttribute("axis", (int64_t)0); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3, 2, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + + 9.0f, 10.0f, + 11.0f, 12.0f}); + test.AddOutput("reduced", {2, 2}, + {0, 0, + 0, 0}); + test.Run(); +} + +TEST(ReductionOpTest, ArgMin_do_not_keepdims_2) { + OpTester test("ArgMin"); + test.AddAttribute("axis", (int64_t)0); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3}, + {1.0f, 2.0f, 3.0f}); + test.AddOutput("reduced", {}, {0}); + test.Run(); +} + +TEST(ReductionOpTest, ArgMin_int32) { + OpTester test("ArgMin"); + test.AddAttribute("axis", (int64_t)0); + test.AddAttribute("keepdims", (int64_t)0); + test.AddInput("data", {3, 2, 2}, + {1, 2, + 3, 4, + + 5, 6, + 7, 8, + + 9, 10, + 11, 12}); + test.AddOutput("reduced", {2, 2}, + {0, 0, + 0, 0}); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/reduction/reduction_test_cases.h b/onnxruntime/test/providers/cpu/reduction/reduction_test_cases.h new file mode 100644 index 0000000000000..1adce706fa285 --- /dev/null +++ b/onnxruntime/test/providers/cpu/reduction/reduction_test_cases.h @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +namespace onnxruntime { +namespace test { +struct ReductionAttribute { + std::vector axes_; + int64_t keep_dims_; +}; + +typedef std::tuple, std::vector> OpAttributesResult; +typedef std::multimap OpAttributesResultMap; +struct ReductionTestCases { + std::vector input_data; + std::vector input_dims; + + OpAttributesResultMap map_op_attribute_expected; +}; + +// python generated testcases +#include "reduction_test_cases.inl" + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/reduction/reduction_test_cases.inl b/onnxruntime/test/providers/cpu/reduction/reduction_test_cases.inl new file mode 100644 index 0000000000000..afd0eb730e8e9 --- /dev/null +++ b/onnxruntime/test/providers/cpu/reduction/reduction_test_cases.inl @@ -0,0 +1,5357 @@ +// Please don't manually edit this file. Generated from reduction_test_cases_generator.py +ReductionTestCases testcases = { + // input_data + { + 0.548814f, + 0.715189f, + 0.602763f, + 0.544883f, + 0.423655f, + 0.645894f, + 0.437587f, + 0.891773f, + 0.963663f, + 0.383442f, + 0.791725f, + 0.528895f, + 0.568045f, + 0.925597f, + 0.071036f, + 0.087129f, + 0.020218f, + 0.832620f, + 0.778157f, + 0.870012f, + 0.978618f, + 0.799159f, + 0.461479f, + 0.780529f, + 0.118274f, + 0.639921f, + 0.143353f, + 0.944669f, + 0.521848f, + 0.414662f, + 0.264556f, + 0.774234f, + 0.456150f, + 0.568434f, + 0.018790f, + 0.617635f, + 0.612096f, + 0.616934f, + 0.943748f, + 0.681820f, + 0.359508f, + 0.437032f, + 0.697631f, + 0.060225f, + 0.666767f, + 0.670638f, + 0.210383f, + 0.128926f, + 0.315428f, + 0.363711f, + 0.570197f, + 0.438602f, + 0.988374f, + 0.102045f, + 0.208877f, + 0.161310f, + 0.653108f, + 0.253292f, + 0.466311f, + 0.244426f, + 0.158970f, + 0.110375f, + 0.656330f, + 0.138183f, + 0.196582f, + 0.368725f, + 0.820993f, + 0.097101f, + 0.837945f, + 0.096098f, + 0.976459f, + 0.468651f, + }, + // input_dims + {2, 3, 2, 2, 3}, + // map_op_attribute_expected + { + {"ReduceL1", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 3}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 3}, + // expected values + { + 1.914725f, + 2.822342f, + 2.741215f, + 2.232489f, + 2.277307f, + 2.662803f, + 1.895933f, + 1.954793f, + 1.631801f, + 2.662185f, + 1.247050f, + 2.176473f, + 1.216198f, + 1.979705f, + 1.569775f, + 1.214244f, + 1.380518f, + 2.331651f, + })}, + {"ReduceL2", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 3}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 3}, + // expected values + { + 0.967783f, + 1.453639f, + 1.410276f, + 1.254770f, + 1.351674f, + 1.505065f, + 1.139954f, + 1.132085f, + 0.884331f, + 1.332661f, + 0.746821f, + 1.242118f, + 0.632181f, + 1.163028f, + 0.906549f, + 0.853013f, + 1.006841f, + 1.220046f, + })}, + {"ReduceLogSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 3}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 3}, + // expected values + { + 0.649574f, + 1.037567f, + 1.008401f, + 0.803117f, + 0.822993f, + 0.979379f, + 0.639711f, + 0.670284f, + 0.489684f, + 0.979147f, + 0.220781f, + 0.777706f, + 0.195730f, + 0.682948f, + 0.450933f, + 0.194122f, + 0.322459f, + 0.846577f, + })}, + {"ReduceLogSumExp", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 3}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 3}, + // expected values + { + 1.867469f, + 2.106419f, + 2.086068f, + 1.981963f, + 2.017188f, + 2.106127f, + 1.911890f, + 1.912408f, + 1.808374f, + 2.052358f, + 1.719563f, + 1.974530f, + 1.694135f, + 1.930749f, + 1.804090f, + 1.739433f, + 1.806803f, + 1.985549f, + })}, + {"ReduceMax", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 3}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 3}, + // expected values + { + 0.548814f, + 0.891773f, + 0.963663f, + 0.799159f, + 0.925597f, + 0.978618f, + 0.944669f, + 0.774234f, + 0.617635f, + 0.697631f, + 0.616934f, + 0.943748f, + 0.438602f, + 0.988374f, + 0.653108f, + 0.820993f, + 0.976459f, + 0.837945f, + })}, + {"ReduceMean", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 3}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 3}, + // expected values + { + 0.478681f, + 0.705586f, + 0.685304f, + 0.558122f, + 0.569327f, + 0.665701f, + 0.473983f, + 0.488698f, + 0.407950f, + 0.665546f, + 0.311762f, + 0.544118f, + 0.304050f, + 0.494926f, + 0.392444f, + 0.303561f, + 0.345130f, + 0.582913f, + })}, + {"ReduceMin", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 3}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 3}, + // expected values + { + 0.383442f, + 0.423655f, + 0.528895f, + 0.087129f, + 0.020218f, + 0.071036f, + 0.118274f, + 0.018790f, + 0.143353f, + 0.612096f, + 0.060225f, + 0.128926f, + 0.208877f, + 0.161310f, + 0.102045f, + 0.096098f, + 0.097101f, + 0.368725f, + })}, + {"ReduceProd", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 3}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 3}, + // expected values + { + 0.050176f, + 0.213925f, + 0.198428f, + 0.030778f, + 0.007514f, + 0.045178f, + 0.016802f, + 0.004858f, + 0.016747f, + 0.195255f, + 0.002810f, + 0.035456f, + 0.007320f, + 0.027040f, + 0.009289f, + 0.001733f, + 0.002057f, + 0.095036f, + })}, + {"ReduceSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 3}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 3}, + // expected values + { + 1.914725f, + 2.822342f, + 2.741215f, + 2.232489f, + 2.277307f, + 2.662803f, + 1.895933f, + 1.954793f, + 1.631801f, + 2.662185f, + 1.247050f, + 2.176473f, + 1.216198f, + 1.979705f, + 1.569775f, + 1.214244f, + 1.380518f, + 2.331651f, + })}, + {"ReduceSumSquare", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 3}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 3}, + // expected values + { + 0.936604f, + 2.113067f, + 1.988879f, + 1.574448f, + 1.827022f, + 2.265222f, + 1.299495f, + 1.281615f, + 0.782041f, + 1.775985f, + 0.557741f, + 1.542857f, + 0.399652f, + 1.352635f, + 0.821832f, + 0.727631f, + 1.013729f, + 1.488512f, + })}, + {"ReduceL1", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 3}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 1, 1, 3}, + // expected values + { + 1.914725f, + 2.822342f, + 2.741215f, + 2.232489f, + 2.277307f, + 2.662803f, + 1.895933f, + 1.954793f, + 1.631801f, + 2.662185f, + 1.247050f, + 2.176473f, + 1.216198f, + 1.979705f, + 1.569775f, + 1.214244f, + 1.380518f, + 2.331651f, + })}, + {"ReduceL2", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 3}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 1, 1, 3}, + // expected values + { + 0.967783f, + 1.453639f, + 1.410276f, + 1.254770f, + 1.351674f, + 1.505065f, + 1.139954f, + 1.132085f, + 0.884331f, + 1.332661f, + 0.746821f, + 1.242118f, + 0.632181f, + 1.163028f, + 0.906549f, + 0.853013f, + 1.006841f, + 1.220046f, + })}, + {"ReduceLogSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 3}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 1, 1, 3}, + // expected values + { + 0.649574f, + 1.037567f, + 1.008401f, + 0.803117f, + 0.822993f, + 0.979379f, + 0.639711f, + 0.670284f, + 0.489684f, + 0.979147f, + 0.220781f, + 0.777706f, + 0.195730f, + 0.682948f, + 0.450933f, + 0.194122f, + 0.322459f, + 0.846577f, + })}, + {"ReduceLogSumExp", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 3}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 1, 1, 3}, + // expected values + { + 1.867469f, + 2.106419f, + 2.086068f, + 1.981963f, + 2.017188f, + 2.106127f, + 1.911890f, + 1.912408f, + 1.808374f, + 2.052358f, + 1.719563f, + 1.974530f, + 1.694135f, + 1.930749f, + 1.804090f, + 1.739433f, + 1.806803f, + 1.985549f, + })}, + {"ReduceMax", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 3}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 1, 1, 3}, + // expected values + { + 0.548814f, + 0.891773f, + 0.963663f, + 0.799159f, + 0.925597f, + 0.978618f, + 0.944669f, + 0.774234f, + 0.617635f, + 0.697631f, + 0.616934f, + 0.943748f, + 0.438602f, + 0.988374f, + 0.653108f, + 0.820993f, + 0.976459f, + 0.837945f, + })}, + {"ReduceMean", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 3}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 1, 1, 3}, + // expected values + { + 0.478681f, + 0.705586f, + 0.685304f, + 0.558122f, + 0.569327f, + 0.665701f, + 0.473983f, + 0.488698f, + 0.407950f, + 0.665546f, + 0.311762f, + 0.544118f, + 0.304050f, + 0.494926f, + 0.392444f, + 0.303561f, + 0.345130f, + 0.582913f, + })}, + {"ReduceMin", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 3}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 1, 1, 3}, + // expected values + { + 0.383442f, + 0.423655f, + 0.528895f, + 0.087129f, + 0.020218f, + 0.071036f, + 0.118274f, + 0.018790f, + 0.143353f, + 0.612096f, + 0.060225f, + 0.128926f, + 0.208877f, + 0.161310f, + 0.102045f, + 0.096098f, + 0.097101f, + 0.368725f, + })}, + {"ReduceProd", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 3}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 1, 1, 3}, + // expected values + { + 0.050176f, + 0.213925f, + 0.198428f, + 0.030778f, + 0.007514f, + 0.045178f, + 0.016802f, + 0.004858f, + 0.016747f, + 0.195255f, + 0.002810f, + 0.035456f, + 0.007320f, + 0.027040f, + 0.009289f, + 0.001733f, + 0.002057f, + 0.095036f, + })}, + {"ReduceSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 3}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 1, 1, 3}, + // expected values + { + 1.914725f, + 2.822342f, + 2.741215f, + 2.232489f, + 2.277307f, + 2.662803f, + 1.895933f, + 1.954793f, + 1.631801f, + 2.662185f, + 1.247050f, + 2.176473f, + 1.216198f, + 1.979705f, + 1.569775f, + 1.214244f, + 1.380518f, + 2.331651f, + })}, + {"ReduceSumSquare", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 3}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 1, 1, 3}, + // expected values + { + 0.936604f, + 2.113067f, + 1.988879f, + 1.574448f, + 1.827022f, + 2.265222f, + 1.299495f, + 1.281615f, + 0.782041f, + 1.775985f, + 0.557741f, + 1.542857f, + 0.399652f, + 1.352635f, + 0.821832f, + 0.727631f, + 1.013729f, + 1.488512f, + })}, + {"ReduceL1", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 1, 4}, + // keep_dims_ + 0, + }, + // expected dims + {2, 2}, + // expected values + { + 10.747742f, + 9.385667f, + 8.551745f, + 7.226055f, + })}, + {"ReduceL2", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 1, 4}, + // keep_dims_ + 0, + }, + // expected dims + {2, 2}, + // expected values + { + 2.813811f, + 2.480093f, + 2.342597f, + 2.047636f, + })}, + {"ReduceLogSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 1, 4}, + // keep_dims_ + 0, + }, + // expected dims + {2, 2}, + // expected values + { + 2.374696f, + 2.239184f, + 2.146135f, + 1.977693f, + })}, + {"ReduceLogSumExp", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 1, 4}, + // keep_dims_ + 0, + }, + // expected dims + {2, 2}, + // expected values + { + 3.527107f, + 3.445045f, + 3.404396f, + 3.330581f, + })}, + {"ReduceMax", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 1, 4}, + // keep_dims_ + 0, + }, + // expected dims + {2, 2}, + // expected values + { + 0.978618f, + 0.944669f, + 0.943748f, + 0.988374f, + })}, + {"ReduceMean", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 1, 4}, + // keep_dims_ + 0, + }, + // expected dims + {2, 2}, + // expected values + { + 0.597097f, + 0.521426f, + 0.475097f, + 0.401447f, + })}, + {"ReduceMin", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 1, 4}, + // keep_dims_ + 0, + }, + // expected dims + {2, 2}, + // expected values + { + 0.071036f, + 0.018790f, + 0.060225f, + 0.096098f, + })}, + {"ReduceProd", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 1, 4}, + // keep_dims_ + 0, + }, + // expected dims + {2, 2}, + // expected values + { + 0.000002f, + 0.000000f, + 0.000000f, + 0.000000f, + })}, + {"ReduceSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 1, 4}, + // keep_dims_ + 0, + }, + // expected dims + {2, 2}, + // expected values + { + 10.747742f, + 9.385667f, + 8.551745f, + 7.226055f, + })}, + {"ReduceSumSquare", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 1, 4}, + // keep_dims_ + 0, + }, + // expected dims + {2, 2}, + // expected values + { + 7.917535f, + 6.150859f, + 5.487762f, + 4.192813f, + })}, + {"ReduceL1", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 1, 4}, + // keep_dims_ + 1, + }, + // expected dims + {2, 1, 1, 2, 1}, + // expected values + { + 10.747742f, + 9.385667f, + 8.551745f, + 7.226055f, + })}, + {"ReduceL2", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 1, 4}, + // keep_dims_ + 1, + }, + // expected dims + {2, 1, 1, 2, 1}, + // expected values + { + 2.813811f, + 2.480093f, + 2.342597f, + 2.047636f, + })}, + {"ReduceLogSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 1, 4}, + // keep_dims_ + 1, + }, + // expected dims + {2, 1, 1, 2, 1}, + // expected values + { + 2.374696f, + 2.239184f, + 2.146135f, + 1.977693f, + })}, + {"ReduceLogSumExp", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 1, 4}, + // keep_dims_ + 1, + }, + // expected dims + {2, 1, 1, 2, 1}, + // expected values + { + 3.527107f, + 3.445045f, + + 3.404396f, + 3.330581f, + })}, + {"ReduceMax", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 1, 4}, + // keep_dims_ + 1, + }, + // expected dims + {2, 1, 1, 2, 1}, + // expected values + { + 0.978618f, + 0.944669f, + 0.943748f, + 0.988374f, + })}, + {"ReduceMean", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 1, 4}, + // keep_dims_ + 1, + }, + // expected dims + {2, 1, 1, 2, 1}, + // expected values + { + 0.597097f, + 0.521426f, + 0.475097f, + 0.401447f, + })}, + {"ReduceMin", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 1, 4}, + // keep_dims_ + 1, + }, + // expected dims + {2, 1, 1, 2, 1}, + // expected values + { + 0.071036f, + 0.018790f, + 0.060225f, + 0.096098f, + })}, + {"ReduceProd", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 1, 4}, + // keep_dims_ + 1, + }, + // expected dims + {2, 1, 1, 2, 1}, + // expected values + { + 0.000002f, + 0.000000f, + 0.000000f, + 0.000000f, + })}, + {"ReduceSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 1, 4}, + // keep_dims_ + 1, + }, + // expected dims + {2, 1, 1, 2, 1}, + // expected values + { + 10.747742f, + 9.385667f, + 8.551745f, + 7.226055f, + })}, + {"ReduceSumSquare", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2, 1, 4}, + // keep_dims_ + 1, + }, + // expected dims + {2, 1, 1, 2, 1}, + // expected values + { + 7.917535f, + 6.150859f, + 5.487762f, + 4.192813f, + })}, + {"ReduceL1", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0, 2, 3}, + // keep_dims_ + 0, + }, + // expected dims + {3, 3}, + // expected values + { + 4.576911f, + 4.069392f, + 4.917688f, + 3.448687f, + 4.257011f, + 4.232579f, + 3.110177f, + 3.335311f, + 3.963452f, + })}, + {"ReduceL2", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0, 2, 3}, + // keep_dims_ + 0, + }, + // expected dims + {3, 3}, + // expected values + { + 1.646994f, + 1.634261f, + 1.879291f, + 1.405027f, + 1.783159f, + 1.757001f, + 1.423772f, + 1.515039f, + 1.506836f, + })}, + {"ReduceLogSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0, 2, 3}, + // keep_dims_ + 0, + }, + // expected dims + {3, 3}, + // expected values + { + 1.521024f, + 1.403494f, + 1.592839f, + 1.237994f, + 1.448567f, + 1.442811f, + 1.134680f, + 1.204566f, + 1.377115f, + })}, + {"ReduceLogSumExp", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0, 2, 3}, + // keep_dims_ + 0, + }, + // expected dims + {3, 3}, + // expected values + { + 2.657328f, + 2.624730f, + 2.725001f, + 2.541516f, + 2.668049f, + 2.659616f, + 2.522522f, + 2.554146f, + 2.594027f, + })}, + {"ReduceMax", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0, 2, 3}, + // keep_dims_ + 0, + }, + // expected dims + {3, 3}, + // expected values + { + 0.697631f, + 0.891773f, + 0.963663f, + 0.799159f, + 0.988374f, + 0.978618f, + 0.944669f, + 0.976459f, + 0.837945f, + })}, + {"ReduceMean", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0, 2, 3}, + // keep_dims_ + 0, + }, + // expected dims + {3, 3}, + // expected values + { + 0.572114f, + 0.508674f, + 0.614711f, + 0.431086f, + 0.532126f, + 0.529072f, + 0.388772f, + 0.416914f, + 0.495431f, + })}, + {"ReduceMin", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0, 2, 3}, + // keep_dims_ + 0, + }, + // expected dims + {3, 3}, + // expected values + { + 0.383442f, + 0.060225f, + 0.128926f, + 0.087129f, + 0.020218f, + 0.071036f, + 0.096098f, + 0.018790f, + 0.143353f, + })}, + {"ReduceProd", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0, 2, 3}, + // keep_dims_ + 0, + }, + // expected dims + {3, 3}, + // expected values + { + 0.009797f, + 0.000601f, + 0.007035f, + 0.000225f, + 0.000203f, + 0.000420f, + 0.000029f, + 0.000010f, + 0.001592f, + })}, + {"ReduceSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0, 2, 3}, + // keep_dims_ + 0, + }, + // expected dims + {3, 3}, + // expected values + { + 4.576911f, + 4.069392f, + 4.917688f, + 3.448687f, + 4.257011f, + 4.232579f, + 3.110177f, + 3.335311f, + 3.963452f, + })}, + {"ReduceSumSquare", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0, 2, 3}, + // keep_dims_ + 0, + }, + // expected dims + {3, 3}, + // expected values + { + 2.712588f, + 2.670808f, + 3.531736f, + 1.974101f, + 3.179657f, + 3.087053f, + 2.027126f, + 2.295344f, + 2.270554f, + })}, + {"ReduceL1", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0, 2, 3}, + // keep_dims_ + 1, + }, + // expected dims + {1, 3, 1, 1, 3}, + // expected values + { + 4.576911f, + 4.069392f, + 4.917688f, + 3.448687f, + 4.257011f, + 4.232579f, + 3.110177f, + 3.335311f, + 3.963452f, + })}, + {"ReduceL2", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0, 2, 3}, + // keep_dims_ + 1, + }, + // expected dims + {1, 3, 1, 1, 3}, + // expected values + { + 1.646994f, + 1.634261f, + 1.879291f, + 1.405027f, + 1.783159f, + 1.757001f, + 1.423772f, + 1.515039f, + 1.506836f, + })}, + {"ReduceLogSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0, 2, 3}, + // keep_dims_ + 1, + }, + // expected dims + {1, 3, 1, 1, 3}, + // expected values + { + 1.521024f, + 1.403494f, + 1.592839f, + 1.237994f, + 1.448567f, + 1.442811f, + 1.134680f, + 1.204566f, + 1.377115f, + })}, + {"ReduceLogSumExp", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0, 2, 3}, + // keep_dims_ + 1, + }, + // expected dims + {1, 3, 1, 1, 3}, + // expected values + { + 2.657328f, + 2.624730f, + 2.725001f, + 2.541516f, + 2.668049f, + 2.659616f, + 2.522522f, + 2.554146f, + 2.594027f, + })}, + {"ReduceMax", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0, 2, 3}, + // keep_dims_ + 1, + }, + // expected dims + {1, 3, 1, 1, 3}, + // expected values + { + 0.697631f, + 0.891773f, + 0.963663f, + 0.799159f, + 0.988374f, + 0.978618f, + 0.944669f, + 0.976459f, + 0.837945f, + })}, + {"ReduceMean", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0, 2, 3}, + // keep_dims_ + 1, + }, + // expected dims + {1, 3, 1, 1, 3}, + // expected values + { + 0.572114f, + 0.508674f, + 0.614711f, + 0.431086f, + 0.532126f, + 0.529072f, + 0.388772f, + 0.416914f, + 0.495431f, + })}, + {"ReduceMin", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0, 2, 3}, + // keep_dims_ + 1, + }, + // expected dims + {1, 3, 1, 1, 3}, + // expected values + { + 0.383442f, + 0.060225f, + 0.128926f, + 0.087129f, + 0.020218f, + 0.071036f, + 0.096098f, + 0.018790f, + 0.143353f, + })}, + {"ReduceProd", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0, 2, 3}, + // keep_dims_ + 1, + }, + // expected dims + {1, 3, 1, 1, 3}, + // expected values + { + 0.009797f, + 0.000601f, + 0.007035f, + 0.000225f, + 0.000203f, + 0.000420f, + 0.000029f, + 0.000010f, + 0.001592f, + })}, + {"ReduceSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0, 2, 3}, + // keep_dims_ + 1, + }, + // expected dims + {1, 3, 1, 1, 3}, + // expected values + { + 4.576911f, + 4.069392f, + 4.917688f, + 3.448687f, + 4.257011f, + 4.232579f, + 3.110177f, + 3.335311f, + 3.963452f, + })}, + {"ReduceSumSquare", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0, 2, 3}, + // keep_dims_ + 1, + }, + // expected dims + {1, 3, 1, 1, 3}, + // expected values + { + 2.712588f, + 2.670808f, + 3.531736f, + 1.974101f, + 3.179657f, + 3.087053f, + 2.027126f, + 2.295344f, + 2.270554f, + })}, + {"ReduceL1", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0}, + // keep_dims_ + 0, + }, + // expected dims + {3, 2, 2, 3}, + // expected values + { + 1.160909f, + 1.332123f, + 1.546511f, + 1.226703f, + 0.783163f, + 1.082926f, + 1.135218f, + 0.951998f, + 1.630429f, + 1.054079f, + 1.002108f, + 0.657821f, + 0.883473f, + 1.289307f, + 0.641233f, + 0.525731f, + 1.008592f, + 0.934665f, + 0.987034f, + 1.031322f, + 1.631727f, + 1.052450f, + 0.927790f, + 1.024955f, + 0.277244f, + 0.750296f, + 0.799683f, + 1.082852f, + 0.718431f, + 0.783387f, + 1.085549f, + 0.871335f, + 1.294095f, + 0.664532f, + 0.995249f, + 1.086287f, + })}, + {"ReduceL2", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0}, + // keep_dims_ + 0, + }, + // expected dims + {3, 2, 2, 3}, + // expected values + { + 0.822105f, + 0.944512f, + 1.119814f, + 0.872798f, + 0.555634f, + 0.779856f, + 0.823512f, + 0.893804f, + 1.171846f, + 0.772517f, + 0.819200f, + 0.544382f, + 0.649746f, + 0.994492f, + 0.574605f, + 0.447172f, + 0.988581f, + 0.838850f, + 0.805703f, + 0.884840f, + 1.176539f, + 0.838338f, + 0.656056f, + 0.817906f, + 0.198142f, + 0.649370f, + 0.671803f, + 0.954722f, + 0.557647f, + 0.554890f, + 0.862566f, + 0.780299f, + 0.954057f, + 0.576500f, + 0.976640f, + 0.775311f, + })}, + {"ReduceLogSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0}, + // keep_dims_ + 0, + }, + // expected dims + {3, 2, 2, 3}, + // expected values + { + 0.149204f, + 0.286774f, + 0.436002f, + 0.204330f, + -0.244415f, + 0.079667f, + 0.126825f, + -0.049192f, + 0.488843f, + 0.052668f, + 0.002105f, + -0.418822f, + -0.123895f, + 0.254105f, + -0.444363f, + -0.642966f, + 0.008556f, + -0.067567f, + -0.013051f, + 0.030841f, + 0.489639f, + 0.051121f, + -0.074950f, + 0.024648f, + -1.282857f, + -0.287287f, + -0.223540f, + 0.079598f, + -0.330686f, + -0.244128f, + 0.082086f, + -0.137729f, + 0.257812f, + -0.408672f, + -0.004762f, + 0.082765f, + })}, + {"ReduceLogSumExp", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0}, + // keep_dims_ + 0, + }, + // expected dims + {3, 2, 2, 3}, + // expected values + { + 1.274102f, + 1.360415f, + 1.480867f, + 1.308841f, + 1.085243f, + 1.240053f, + 1.269186f, + 1.253199f, + 1.519340f, + 1.230462f, + 1.235864f, + 1.041923f, + 1.142839f, + 1.376757f, + 1.044591f, + 0.971375f, + 1.310300f, + 1.225764f, + 1.226638f, + 1.270319f, + 1.522197f, + 1.256165f, + 1.157045f, + 1.241128f, + 0.831976f, + 1.102946f, + 1.125527f, + 1.313763f, + 1.065529f, + 1.085104f, + 1.274135f, + 1.185066f, + 1.358306f, + 1.053046f, + 1.301283f, + 1.239063f, + })}, + {"ReduceMax", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0}, + // keep_dims_ + 0, + }, + // expected dims + {3, 2, 2, 3}, + // expected values + { + 0.612096f, + 0.715189f, + 0.943748f, + 0.681820f, + 0.423655f, + 0.645894f, + 0.697631f, + 0.891773f, + 0.963663f, + 0.670638f, + 0.791725f, + 0.528895f, + 0.568045f, + 0.925597f, + 0.570197f, + 0.438602f, + 0.988374f, + 0.832620f, + 0.778157f, + 0.870012f, + 0.978618f, + 0.799159f, + 0.466311f, + 0.780529f, + 0.158970f, + 0.639921f, + 0.656330f, + 0.944669f, + 0.521848f, + 0.414662f, + 0.820993f, + 0.774234f, + 0.837945f, + 0.568434f, + 0.976459f, + 0.617635f, + })}, + {"ReduceMean", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0}, + // keep_dims_ + 0, + }, + // expected dims + {3, 2, 2, 3}, + // expected values + { + 0.580455f, + 0.666062f, + 0.773256f, + 0.613352f, + 0.391581f, + 0.541463f, + 0.567609f, + 0.475999f, + 0.815215f, + 0.527040f, + 0.501054f, + 0.328911f, + 0.441736f, + 0.644654f, + 0.320616f, + 0.262865f, + 0.504296f, + 0.467332f, + 0.493517f, + 0.515661f, + 0.815863f, + 0.526225f, + 0.463895f, + 0.512477f, + 0.138622f, + 0.375148f, + 0.399841f, + 0.541426f, + 0.359215f, + 0.391694f, + 0.542774f, + 0.435667f, + 0.647048f, + 0.332266f, + 0.497625f, + 0.543143f, + })}, + {"ReduceMin", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0}, + // keep_dims_ + 0, + }, + // expected dims + {3, 2, 2, 3}, + // expected values + { + 0.548814f, + 0.616934f, + 0.602763f, + 0.544883f, + 0.359508f, + 0.437032f, + 0.437587f, + 0.060225f, + 0.666767f, + 0.383442f, + 0.210383f, + 0.128926f, + 0.315428f, + 0.363711f, + 0.071036f, + 0.087129f, + 0.020218f, + 0.102045f, + 0.208877f, + 0.161310f, + 0.653108f, + 0.253292f, + 0.461479f, + 0.244426f, + 0.118274f, + 0.110375f, + 0.143353f, + 0.138183f, + 0.196582f, + 0.368725f, + 0.264556f, + 0.097101f, + 0.456150f, + 0.096098f, + 0.018790f, + 0.468651f, + })}, + {"ReduceProd", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0}, + // keep_dims_ + 0, + }, + // expected dims + {3, 2, 2, 3}, + // expected values + { + 0.335926f, + 0.441225f, + 0.568857f, + 0.371512f, + 0.152307f, + 0.282276f, + 0.305274f, + 0.053707f, + 0.642538f, + 0.257150f, + 0.166565f, + 0.068188f, + 0.179177f, + 0.336649f, + 0.040505f, + 0.038215f, + 0.019983f, + 0.084965f, + 0.162539f, + 0.140341f, + 0.639144f, + 0.202420f, + 0.215193f, + 0.190781f, + 0.018802f, + 0.070631f, + 0.094087f, + 0.130537f, + 0.102586f, + 0.152896f, + 0.217198f, + 0.075179f, + 0.382229f, + 0.054626f, + 0.018347f, + 0.289456f, + })}, + {"ReduceSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0}, + // keep_dims_ + 0, + }, + // expected dims + {3, 2, 2, 3}, + // expected values + { + 1.160909f, + 1.332123f, + 1.546511f, + 1.226703f, + 0.783163f, + 1.082926f, + 1.135218f, + 0.951998f, + 1.630429f, + 1.054079f, + 1.002108f, + 0.657821f, + 0.883473f, + 1.289307f, + 0.641233f, + 0.525731f, + 1.008592f, + 0.934665f, + 0.987034f, + 1.031322f, + 1.631727f, + 1.052450f, + 0.927790f, + 1.024955f, + 0.277244f, + 0.750296f, + 0.799683f, + 1.082852f, + 0.718431f, + 0.783387f, + 1.085549f, + 0.871335f, + 1.294095f, + 0.664532f, + 0.995249f, + 1.086287f, + })}, + {"ReduceSumSquare", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0}, + // keep_dims_ + 0, + }, + // expected dims + {3, 2, 2, 3}, + // expected values + { + 0.675857f, + 0.892103f, + 1.253984f, + 0.761777f, + 0.308729f, + 0.608176f, + 0.678172f, + 0.798886f, + 1.373224f, + 0.596783f, + 0.671089f, + 0.296352f, + 0.422170f, + 0.989015f, + 0.330170f, + 0.199963f, + 0.977292f, + 0.703669f, + 0.649157f, + 0.782942f, + 1.384244f, + 0.702811f, + 0.430409f, + 0.668970f, + 0.039260f, + 0.421682f, + 0.451319f, + 0.911494f, + 0.310970f, + 0.307903f, + 0.744020f, + 0.608866f, + 0.910225f, + 0.332352f, + 0.953826f, + 0.601108f, + })}, + {"ArgMax", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0}, + // keep_dims_ + 0, + }, + // expected dims + {3, 2, 2, 3}, + // expected values + { + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + })}, + {"ArgMin", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0}, + // keep_dims_ + 0, + }, + // expected dims + {3, 2, 2, 3}, + // expected values + { + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + })}, + {"ReduceL1", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0}, + // keep_dims_ + 1, + }, + // expected dims + {1, 3, 2, 2, 3}, + // expected values + { + 1.160909f, + 1.332123f, + 1.546511f, + 1.226703f, + 0.783163f, + 1.082926f, + 1.135218f, + 0.951998f, + 1.630429f, + 1.054079f, + 1.002108f, + 0.657821f, + 0.883473f, + 1.289307f, + 0.641233f, + 0.525731f, + 1.008592f, + 0.934665f, + 0.987034f, + 1.031322f, + 1.631727f, + 1.052450f, + 0.927790f, + 1.024955f, + 0.277244f, + 0.750296f, + 0.799683f, + 1.082852f, + 0.718431f, + 0.783387f, + 1.085549f, + 0.871335f, + 1.294095f, + 0.664532f, + 0.995249f, + 1.086287f, + })}, + {"ReduceL2", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0}, + // keep_dims_ + 1, + }, + // expected dims + {1, 3, 2, 2, 3}, + // expected values + { + 0.822105f, + 0.944512f, + 1.119814f, + 0.872798f, + 0.555634f, + 0.779856f, + 0.823512f, + 0.893804f, + 1.171846f, + 0.772517f, + 0.819200f, + 0.544382f, + 0.649746f, + 0.994492f, + 0.574605f, + 0.447172f, + 0.988581f, + 0.838850f, + 0.805703f, + 0.884840f, + 1.176539f, + 0.838338f, + 0.656056f, + 0.817906f, + 0.198142f, + 0.649370f, + 0.671803f, + 0.954722f, + 0.557647f, + 0.554890f, + 0.862566f, + 0.780299f, + 0.954057f, + 0.576500f, + 0.976640f, + 0.775311f, + })}, + {"ReduceLogSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0}, + // keep_dims_ + 1, + }, + // expected dims + {1, 3, 2, 2, 3}, + // expected values + { + 0.149204f, + 0.286774f, + 0.436002f, + 0.204330f, + -0.244415f, + 0.079667f, + 0.126825f, + -0.049192f, + 0.488843f, + 0.052668f, + 0.002105f, + -0.418822f, + -0.123895f, + 0.254105f, + -0.444363f, + -0.642966f, + 0.008556f, + -0.067567f, + -0.013051f, + 0.030841f, + 0.489639f, + 0.051121f, + -0.074950f, + 0.024648f, + -1.282857f, + -0.287287f, + -0.223540f, + 0.079598f, + -0.330686f, + -0.244128f, + 0.082086f, + -0.137729f, + 0.257812f, + -0.408672f, + -0.004762f, + 0.082765f, + })}, + {"ReduceLogSumExp", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0}, + // keep_dims_ + 1, + }, + // expected dims + {1, 3, 2, 2, 3}, + // expected values + { + 1.274102f, + 1.360415f, + 1.480867f, + 1.308841f, + 1.085243f, + 1.240053f, + 1.269186f, + 1.253199f, + 1.519340f, + 1.230462f, + 1.235864f, + 1.041923f, + 1.142839f, + 1.376757f, + 1.044591f, + 0.971375f, + 1.310300f, + 1.225764f, + 1.226638f, + 1.270319f, + 1.522197f, + 1.256165f, + 1.157045f, + 1.241128f, + 0.831976f, + 1.102946f, + 1.125527f, + 1.313763f, + 1.065529f, + 1.085104f, + 1.274135f, + 1.185066f, + 1.358306f, + 1.053046f, + 1.301283f, + 1.239063f, + })}, + {"ReduceMax", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0}, + // keep_dims_ + 1, + }, + // expected dims + {1, 3, 2, 2, 3}, + // expected values + { + 0.612096f, + 0.715189f, + 0.943748f, + 0.681820f, + 0.423655f, + 0.645894f, + 0.697631f, + 0.891773f, + 0.963663f, + 0.670638f, + 0.791725f, + 0.528895f, + 0.568045f, + 0.925597f, + 0.570197f, + 0.438602f, + 0.988374f, + 0.832620f, + 0.778157f, + 0.870012f, + 0.978618f, + 0.799159f, + 0.466311f, + 0.780529f, + 0.158970f, + 0.639921f, + 0.656330f, + 0.944669f, + 0.521848f, + 0.414662f, + 0.820993f, + 0.774234f, + 0.837945f, + 0.568434f, + 0.976459f, + 0.617635f, + })}, + {"ReduceMean", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0}, + // keep_dims_ + 1, + }, + // expected dims + {1, 3, 2, 2, 3}, + // expected values + { + 0.580455f, + 0.666062f, + 0.773256f, + 0.613352f, + 0.391581f, + 0.541463f, + 0.567609f, + 0.475999f, + 0.815215f, + 0.527040f, + 0.501054f, + 0.328911f, + 0.441736f, + 0.644654f, + 0.320616f, + 0.262865f, + 0.504296f, + 0.467332f, + 0.493517f, + 0.515661f, + 0.815863f, + 0.526225f, + 0.463895f, + 0.512477f, + 0.138622f, + 0.375148f, + 0.399841f, + 0.541426f, + 0.359215f, + 0.391694f, + 0.542774f, + 0.435667f, + 0.647048f, + 0.332266f, + 0.497625f, + 0.543143f, + })}, + {"ReduceMin", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0}, + // keep_dims_ + 1, + }, + // expected dims + {1, 3, 2, 2, 3}, + // expected values + { + 0.548814f, + 0.616934f, + 0.602763f, + 0.544883f, + 0.359508f, + 0.437032f, + 0.437587f, + 0.060225f, + 0.666767f, + 0.383442f, + 0.210383f, + 0.128926f, + 0.315428f, + 0.363711f, + 0.071036f, + 0.087129f, + 0.020218f, + 0.102045f, + 0.208877f, + 0.161310f, + 0.653108f, + 0.253292f, + 0.461479f, + 0.244426f, + 0.118274f, + 0.110375f, + 0.143353f, + 0.138183f, + 0.196582f, + 0.368725f, + 0.264556f, + 0.097101f, + 0.456150f, + 0.096098f, + 0.018790f, + 0.468651f, + })}, + {"ReduceProd", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0}, + // keep_dims_ + 1, + }, + // expected dims + {1, 3, 2, 2, 3}, + // expected values + { + 0.335926f, + 0.441225f, + 0.568857f, + 0.371512f, + 0.152307f, + 0.282276f, + 0.305274f, + 0.053707f, + 0.642538f, + 0.257150f, + 0.166565f, + 0.068188f, + 0.179177f, + 0.336649f, + 0.040505f, + 0.038215f, + 0.019983f, + 0.084965f, + 0.162539f, + 0.140341f, + 0.639144f, + 0.202420f, + 0.215193f, + 0.190781f, + 0.018802f, + 0.070631f, + 0.094087f, + 0.130537f, + 0.102586f, + 0.152896f, + 0.217198f, + 0.075179f, + 0.382229f, + 0.054626f, + 0.018347f, + 0.289456f, + })}, + {"ReduceSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0}, + // keep_dims_ + 1, + }, + // expected dims + {1, 3, 2, 2, 3}, + // expected values + { + 1.160909f, + 1.332123f, + 1.546511f, + 1.226703f, + 0.783163f, + 1.082926f, + 1.135218f, + 0.951998f, + 1.630429f, + 1.054079f, + 1.002108f, + 0.657821f, + 0.883473f, + 1.289307f, + 0.641233f, + 0.525731f, + 1.008592f, + 0.934665f, + 0.987034f, + 1.031322f, + 1.631727f, + 1.052450f, + 0.927790f, + 1.024955f, + 0.277244f, + 0.750296f, + 0.799683f, + 1.082852f, + 0.718431f, + 0.783387f, + 1.085549f, + 0.871335f, + 1.294095f, + 0.664532f, + 0.995249f, + 1.086287f, + })}, + {"ReduceSumSquare", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0}, + // keep_dims_ + 1, + }, + // expected dims + {1, 3, 2, 2, 3}, + // expected values + { + 0.675857f, + 0.892103f, + 1.253984f, + 0.761777f, + 0.308729f, + 0.608176f, + 0.678172f, + 0.798886f, + 1.373224f, + 0.596783f, + 0.671089f, + + 0.296352f, + 0.422170f, + 0.989015f, + 0.330170f, + 0.199963f, + 0.977292f, + 0.703669f, + 0.649157f, + 0.782942f, + 1.384244f, + 0.702811f, + 0.430409f, + 0.668970f, + 0.039260f, + 0.421682f, + 0.451319f, + 0.911494f, + 0.310970f, + 0.307903f, + 0.744020f, + 0.608866f, + 0.910225f, + 0.332352f, + 0.953826f, + 0.601108f, + })}, + {"ArgMax", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0}, + // keep_dims_ + 1, + }, + // expected dims + {1, 3, 2, 2, 3}, + // expected values + { + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + })}, + {"ArgMin", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {0}, + // keep_dims_ + 1, + }, + // expected dims + {1, 3, 2, 2, 3}, + // expected values + { + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + })}, + {"ReduceL1", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 2, 3}, + // expected values + { + 0.986401f, + 1.606962f, + 1.566426f, + 0.928325f, + 1.215380f, + 1.174789f, + 1.346201f, + 1.795609f, + 1.049654f, + 0.886288f, + 0.481698f, + 1.613149f, + 0.382830f, + 1.414155f, + 0.599504f, + 1.513103f, + 0.540638f, + 1.032297f, + 1.309727f, + 0.677159f, + 1.610515f, + 1.352458f, + 0.569890f, + 0.565958f, + 0.524305f, + 0.525020f, + 1.223305f, + 0.691893f, + 1.454685f, + 0.346470f, + 0.979963f, + 0.207476f, + 1.494274f, + 0.234281f, + 1.173042f, + 0.837376f, + })}, + {"ReduceL2", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 2, 3}, + // expected values + { + 0.701911f, + 1.143134f, + 1.136648f, + 0.666277f, + 0.897949f, + 0.834811f, + 0.963433f, + 1.270295f, + 0.981193f, + 0.803894f, + 0.461922f, + 1.141263f, + 0.289790f, + 1.004458f, + 0.478146f, + 1.102505f, + 0.522186f, + 0.743921f, + 0.928090f, + 0.619867f, + 1.155525f, + 0.956365f, + 0.416541f, + 0.455652f, + 0.378318f, + 0.397877f, + 0.866992f, + 0.506486f, + 1.092853f, + 0.264872f, + 0.836242f, + 0.147008f, + 1.064387f, + 0.168313f, + 0.996051f, + 0.596316f, + })}, + {"ReduceLogSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 2, 3}, + // expected values + { + -0.013693f, + 0.474346f, + 0.448797f, + -0.074374f, + 0.195057f, + 0.161089f, + 0.297287f, + 0.585344f, + 0.048461f, + -0.120713f, + -0.730438f, + 0.478188f, + -0.960164f, + 0.346532f, + -0.511653f, + 0.414162f, + -0.615005f, + 0.031787f, + 0.269819f, + -0.389848f, + 0.476554f, + 0.301924f, + -0.562311f, + -0.569235f, + -0.645681f, + -0.644318f, + 0.201556f, + -0.368324f, + 0.374789f, + -1.059958f, + -0.020241f, + -1.572738f, + 0.401641f, + -1.451232f, + 0.159600f, + -0.177482f, + })}, + {"ReduceLogSumExp", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 2, 3}, + // expected values + { + 1.187893f, + 1.500521f, + 1.492554f, + 1.160564f, + 1.317677f, + 1.282252f, + 1.371756f, + 1.591338f, + 1.317586f, + 1.198369f, + 0.958140f, + 1.500061f, + 0.887235f, + 1.402478f, + 1.005080f, + 1.467289f, + 0.994772f, + 1.214437f, + 1.348925f, + 1.069977f, + 1.507964f, + 1.369392f, + 0.980870f, + 0.987946f, + 0.956718f, + 0.960769f, + 1.305659f, + 1.043380f, + 1.454178f, + 0.868914f, + 1.236941f, + 0.796907f, + 1.444402f, + 0.810509f, + 1.353842f, + 1.113083f, + })}, + {"ReduceMax", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 2, 3}, + // expected values + { + 0.548814f, + 0.891773f, + 0.963663f, + 0.544883f, + 0.791725f, + 0.645894f, + 0.778157f, + 0.925597f, + 0.978618f, + 0.799159f, + 0.461479f, + 0.832620f, + 0.264556f, + 0.774234f, + 0.456150f, + 0.944669f, + 0.521848f, + 0.617635f, + 0.697631f, + 0.616934f, + 0.943748f, + 0.681820f, + 0.359508f, + 0.437032f, + 0.315428f, + 0.363711f, + 0.653108f, + 0.438602f, + 0.988374f, + 0.244426f, + 0.820993f, + 0.110375f, + 0.837945f, + 0.138183f, + 0.976459f, + 0.468651f, + })}, + {"ReduceMean", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 2, 3}, + // expected values + { + 0.493200f, + 0.803481f, + 0.783213f, + 0.464162f, + 0.607690f, + 0.587395f, + 0.673101f, + 0.897804f, + 0.524827f, + 0.443144f, + 0.240849f, + 0.806575f, + 0.191415f, + 0.707077f, + 0.299752f, + 0.756551f, + 0.270319f, + 0.516149f, + 0.654863f, + 0.338580f, + 0.805257f, + 0.676229f, + 0.284945f, + 0.282979f, + 0.262153f, + 0.262510f, + 0.611653f, + 0.345947f, + 0.727342f, + 0.173235f, + 0.489981f, + 0.103738f, + 0.747137f, + 0.117141f, + 0.586521f, + 0.418688f, + })}, + {"ReduceMin", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 2, 3}, + // expected values + { + 0.437587f, + 0.715189f, + 0.602763f, + 0.383442f, + 0.423655f, + 0.528895f, + 0.568045f, + 0.870012f, + 0.071036f, + 0.087129f, + 0.020218f, + 0.780529f, + 0.118274f, + 0.639921f, + 0.143353f, + 0.568434f, + 0.018790f, + 0.414662f, + 0.612096f, + 0.060225f, + 0.666767f, + 0.670638f, + 0.210383f, + 0.128926f, + 0.208877f, + 0.161310f, + 0.570197f, + 0.253292f, + 0.466311f, + 0.102045f, + 0.158970f, + 0.097101f, + 0.656330f, + 0.096098f, + 0.196582f, + 0.368725f, + })}, + {"ReduceProd", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 2, 3}, + // expected values + { + 0.240154f, + 0.637787f, + 0.580861f, + 0.208931f, + 0.335418f, + 0.341610f, + 0.442028f, + 0.805280f, + 0.069517f, + 0.069630f, + 0.009330f, + 0.649884f, + 0.031290f, + 0.495448f, + 0.065391f, + 0.536982f, + 0.009805f, + 0.256110f, + 0.427017f, + 0.037155f, + 0.629260f, + 0.457255f, + 0.075634f, + 0.056345f, + 0.065886f, + 0.058670f, + 0.372400f, + 0.111094f, + 0.460889f, + 0.024942f, + 0.130513f, + 0.010718f, + 0.549968f, + 0.013279f, + 0.191955f, + 0.172803f, + })}, + {"ReduceSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 2, 3}, + // expected values + { + 0.986401f, + 1.606962f, + 1.566426f, + 0.928325f, + 1.215380f, + 1.174789f, + 1.346201f, + 1.795609f, + 1.049654f, + 0.886288f, + 0.481698f, + 1.613149f, + 0.382830f, + 1.414155f, + 0.599504f, + 1.513103f, + 0.540638f, + 1.032297f, + 1.309727f, + 0.677159f, + 1.610515f, + 1.352458f, + 0.569890f, + 0.565958f, + 0.524305f, + 0.525020f, + 1.223305f, + 0.691893f, + 1.454685f, + 0.346470f, + 0.979963f, + 0.207476f, + 1.494274f, + 0.234281f, + 1.173042f, + 0.837376f, + })}, + {"ReduceSumSquare", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 2, 3}, + // expected values + { + 0.492679f, + 1.306755f, + 1.291970f, + 0.443925f, + 0.806312f, + 0.696909f, + 0.928203f, + 1.613650f, + 0.962740f, + 0.646246f, + 0.213372f, + 1.302482f, + 0.083979f, + 1.008937f, + 0.228623f, + 1.215517f, + 0.272679f, + 0.553418f, + 0.861350f, + 0.384235f, + 1.335238f, + 0.914634f, + 0.173507f, + 0.207619f, + 0.143125f, + 0.158306f, + 0.751675f, + 0.256528f, + 1.194329f, + 0.070157f, + 0.699301f, + 0.021611f, + 1.132920f, + 0.028329f, + 0.992118f, + 0.355592f, + })}, + {"ArgMax", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 2, 3}, + // expected values + { + 0.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + })}, + {"ArgMin", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 2, 3}, + // expected values + { + 1.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + })}, + {"ReduceL1", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 1, 2, 3}, + // expected values + { + 0.986401f, + 1.606962f, + 1.566426f, + 0.928325f, + 1.215380f, + 1.174789f, + 1.346201f, + 1.795609f, + 1.049654f, + 0.886288f, + 0.481698f, + 1.613149f, + 0.382830f, + 1.414155f, + 0.599504f, + 1.513103f, + 0.540638f, + 1.032297f, + 1.309727f, + 0.677159f, + 1.610515f, + 1.352458f, + 0.569890f, + 0.565958f, + 0.524305f, + 0.525020f, + 1.223305f, + 0.691893f, + 1.454685f, + 0.346470f, + 0.979963f, + 0.207476f, + 1.494274f, + 0.234281f, + 1.173042f, + 0.837376f, + })}, + {"ReduceL2", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 1, 2, 3}, + // expected values + { + 0.701911f, + 1.143134f, + 1.136648f, + 0.666277f, + 0.897949f, + 0.834811f, + 0.963433f, + 1.270295f, + 0.981193f, + 0.803894f, + 0.461922f, + 1.141263f, + 0.289790f, + 1.004458f, + 0.478146f, + 1.102505f, + 0.522186f, + 0.743921f, + 0.928090f, + 0.619867f, + 1.155525f, + 0.956365f, + 0.416541f, + 0.455652f, + 0.378318f, + 0.397877f, + 0.866992f, + 0.506486f, + 1.092853f, + 0.264872f, + 0.836242f, + 0.147008f, + 1.064387f, + 0.168313f, + 0.996051f, + 0.596316f, + })}, + {"ReduceLogSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 1, 2, 3}, + // expected values + { + -0.013693f, + 0.474346f, + 0.448797f, + -0.074374f, + 0.195057f, + 0.161089f, + 0.297287f, + 0.585344f, + 0.048461f, + -0.120713f, + -0.730438f, + 0.478188f, + -0.960164f, + 0.346532f, + -0.511653f, + 0.414162f, + -0.615005f, + 0.031787f, + 0.269819f, + -0.389848f, + 0.476554f, + 0.301924f, + -0.562311f, + -0.569235f, + -0.645681f, + -0.644318f, + 0.201556f, + -0.368324f, + 0.374789f, + -1.059958f, + -0.020241f, + -1.572738f, + 0.401641f, + -1.451232f, + 0.159600f, + -0.177482f, + })}, + {"ReduceLogSumExp", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 1, 2, 3}, + // expected values + { + 1.187893f, + 1.500521f, + 1.492554f, + 1.160564f, + 1.317677f, + 1.282252f, + 1.371756f, + 1.591338f, + 1.317586f, + 1.198369f, + 0.958140f, + 1.500061f, + 0.887235f, + 1.402478f, + 1.005080f, + 1.467289f, + 0.994772f, + 1.214437f, + 1.348925f, + 1.069977f, + 1.507964f, + 1.369392f, + 0.980870f, + 0.987946f, + 0.956718f, + 0.960769f, + 1.305659f, + 1.043380f, + 1.454178f, + 0.868914f, + 1.236941f, + 0.796907f, + 1.444402f, + 0.810509f, + 1.353842f, + 1.113083f, + })}, + {"ReduceMax", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 1, 2, 3}, + // expected values + { + 0.548814f, + 0.891773f, + 0.963663f, + 0.544883f, + 0.791725f, + 0.645894f, + 0.778157f, + 0.925597f, + 0.978618f, + 0.799159f, + 0.461479f, + 0.832620f, + 0.264556f, + 0.774234f, + 0.456150f, + 0.944669f, + 0.521848f, + 0.617635f, + 0.697631f, + 0.616934f, + 0.943748f, + 0.681820f, + 0.359508f, + 0.437032f, + 0.315428f, + 0.363711f, + 0.653108f, + 0.438602f, + 0.988374f, + 0.244426f, + 0.820993f, + 0.110375f, + 0.837945f, + 0.138183f, + 0.976459f, + 0.468651f, + })}, + {"ReduceMean", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 1, 2, 3}, + // expected values + { + 0.493200f, + 0.803481f, + 0.783213f, + 0.464162f, + 0.607690f, + 0.587395f, + 0.673101f, + 0.897804f, + 0.524827f, + 0.443144f, + 0.240849f, + 0.806575f, + 0.191415f, + 0.707077f, + 0.299752f, + 0.756551f, + 0.270319f, + 0.516149f, + 0.654863f, + 0.338580f, + 0.805257f, + 0.676229f, + 0.284945f, + 0.282979f, + 0.262153f, + 0.262510f, + 0.611653f, + 0.345947f, + 0.727342f, + 0.173235f, + 0.489981f, + 0.103738f, + 0.747137f, + 0.117141f, + 0.586521f, + 0.418688f, + })}, + {"ReduceMin", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 1, 2, 3}, + // expected values + { + 0.437587f, + 0.715189f, + 0.602763f, + 0.383442f, + 0.423655f, + 0.528895f, + 0.568045f, + 0.870012f, + 0.071036f, + 0.087129f, + 0.020218f, + 0.780529f, + 0.118274f, + 0.639921f, + 0.143353f, + 0.568434f, + 0.018790f, + 0.414662f, + 0.612096f, + 0.060225f, + 0.666767f, + 0.670638f, + 0.210383f, + 0.128926f, + 0.208877f, + 0.161310f, + 0.570197f, + 0.253292f, + 0.466311f, + 0.102045f, + 0.158970f, + 0.097101f, + 0.656330f, + 0.096098f, + 0.196582f, + 0.368725f, + })}, + {"ReduceProd", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 1, 2, 3}, + // expected values + { + 0.240154f, + 0.637787f, + 0.580861f, + 0.208931f, + 0.335418f, + 0.341610f, + 0.442028f, + 0.805280f, + 0.069517f, + 0.069630f, + 0.009330f, + 0.649884f, + 0.031290f, + 0.495448f, + 0.065391f, + 0.536982f, + 0.009805f, + 0.256110f, + 0.427017f, + 0.037155f, + 0.629260f, + 0.457255f, + 0.075634f, + 0.056345f, + 0.065886f, + 0.058670f, + 0.372400f, + 0.111094f, + 0.460889f, + 0.024942f, + 0.130513f, + 0.010718f, + 0.549968f, + 0.013279f, + 0.191955f, + 0.172803f, + })}, + {"ReduceSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 1, 2, 3}, + // expected values + { + 0.986401f, + 1.606962f, + 1.566426f, + 0.928325f, + 1.215380f, + 1.174789f, + 1.346201f, + 1.795609f, + 1.049654f, + 0.886288f, + 0.481698f, + 1.613149f, + 0.382830f, + 1.414155f, + 0.599504f, + 1.513103f, + 0.540638f, + 1.032297f, + 1.309727f, + 0.677159f, + 1.610515f, + 1.352458f, + 0.569890f, + 0.565958f, + 0.524305f, + 0.525020f, + 1.223305f, + 0.691893f, + 1.454685f, + 0.346470f, + 0.979963f, + 0.207476f, + 1.494274f, + 0.234281f, + 1.173042f, + 0.837376f, + })}, + {"ReduceSumSquare", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 1, 2, 3}, + // expected values + { + 0.492679f, + 1.306755f, + 1.291970f, + 0.443925f, + 0.806312f, + 0.696909f, + 0.928203f, + 1.613650f, + 0.962740f, + 0.646246f, + 0.213372f, + 1.302482f, + 0.083979f, + 1.008937f, + 0.228623f, + 1.215517f, + 0.272679f, + 0.553418f, + 0.861350f, + 0.384235f, + 1.335238f, + 0.914634f, + 0.173507f, + 0.207619f, + 0.143125f, + 0.158306f, + 0.751675f, + 0.256528f, + 1.194329f, + 0.070157f, + 0.699301f, + 0.021611f, + 1.132920f, + 0.028329f, + 0.992118f, + 0.355592f, + })}, + {"ArgMax", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 1, 2, 3}, + // expected values + { + 0.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + })}, + {"ArgMin", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {2}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 1, 2, 3}, + // expected values + { + 1.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + })}, + {"ReduceL1", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {4}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 2, 2}, + // expected values + { + 1.866766f, + 1.614432f, + 2.293023f, + 1.704061f, + 1.564677f, + 0.939968f, + 2.626787f, + 2.041167f, + 0.901549f, + 1.881179f, + 1.494940f, + 1.204859f, + 2.172778f, + 1.478360f, + 1.424623f, + 1.009947f, + 1.249336f, + 1.529020f, + 1.023295f, + 0.964028f, + 0.925674f, + 0.703490f, + 1.756039f, + 1.541209f, + })}, + {"ReduceL2", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {4}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 2, 2}, + // expected values + { + 1.084443f, + 0.945283f, + 1.383975f, + 1.026443f, + 1.088324f, + 0.837410f, + 1.523202f, + 1.208654f, + 0.666362f, + 1.156144f, + 0.936750f, + 0.839609f, + 1.282938f, + 0.886071f, + 0.966899f, + 0.714589f, + 0.746261f, + 1.086125f, + 0.704415f, + 0.584248f, + 0.684268f, + 0.440111f, + 1.177119f, + 1.087355f, + })}, + {"ReduceLogSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {4}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 2, 2}, + // expected values + { + 0.624208f, + 0.478983f, + 0.829871f, + 0.533015f, + 0.447680f, + -0.061910f, + 0.965762f, + 0.713522f, + -0.103641f, + 0.631899f, + 0.402086f, + 0.186363f, + 0.776006f, + 0.390933f, + 0.353907f, + 0.009898f, + 0.222612f, + 0.424627f, + 0.023027f, + -0.036635f, + -0.077233f, + -0.351701f, + 0.563061f, + 0.432567f, + })}, + {"ReduceLogSumExp", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {4}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 2, 2}, + // expected values + { + 1.723290f, + 1.640865f, + 1.888538f, + 1.681123f, + 1.679301f, + 1.484055f, + 1.977571f, + 1.790545f, + 1.429359f, + 1.752838f, + 1.619342f, + 1.534500f, + 1.835318f, + 1.601044f, + 1.613229f, + 1.464930f, + 1.521288f, + 1.676130f, + 1.465301f, + 1.425334f, + 1.439054f, + 1.337973f, + 1.738067f, + 1.677796f, + })}, + {"ReduceMax", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {4}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 2, 2}, + // expected values + { + 0.715189f, + 0.645894f, + 0.963663f, + 0.791725f, + 0.925597f, + 0.832620f, + 0.978618f, + 0.799159f, + 0.639921f, + 0.944669f, + 0.774234f, + 0.617635f, + 0.943748f, + 0.681820f, + 0.697631f, + 0.670638f, + 0.570197f, + 0.988374f, + 0.653108f, + 0.466311f, + 0.656330f, + 0.368725f, + 0.837945f, + 0.976459f, + })}, + {"ReduceMean", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {4}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 2, 2}, + // expected values + { + 0.622255f, + 0.538144f, + 0.764341f, + 0.568020f, + 0.521559f, + 0.313323f, + 0.875596f, + 0.680389f, + 0.300516f, + 0.627060f, + 0.498313f, + 0.401620f, + 0.724259f, + 0.492787f, + 0.474874f, + 0.336649f, + 0.416445f, + 0.509673f, + 0.341098f, + 0.321343f, + 0.308558f, + 0.234497f, + 0.585346f, + 0.513736f, + })}, + {"ReduceMin", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {4}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 2, 2}, + // expected values + { + 0.548814f, + 0.423655f, + 0.437587f, + 0.383442f, + 0.071036f, + 0.020218f, + 0.778157f, + 0.461479f, + 0.118274f, + 0.414662f, + 0.264556f, + 0.018790f, + 0.612096f, + 0.359508f, + 0.060225f, + 0.128926f, + 0.315428f, + 0.102045f, + 0.161310f, + 0.244426f, + 0.110375f, + 0.138183f, + 0.097101f, + 0.096098f, + })}, + {"ReduceProd", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {4}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 2, 2}, + // expected values + { + 0.236588f, + 0.149100f, + 0.376049f, + 0.160562f, + 0.037349f, + 0.001467f, + 0.662530f, + 0.287855f, + 0.010850f, + 0.204418f, + 0.093432f, + 0.006597f, + 0.356381f, + 0.107125f, + 0.028014f, + 0.018190f, + 0.065416f, + 0.044237f, + 0.022006f, + 0.028870f, + 0.011516f, + 0.010016f, + 0.066801f, + 0.043976f, + })}, + {"ReduceSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {4}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 2, 2}, + // expected values + { + 1.866766f, + 1.614432f, + 2.293023f, + 1.704061f, + 1.564677f, + 0.939968f, + 2.626787f, + 2.041167f, + 0.901549f, + 1.881179f, + 1.494940f, + 1.204859f, + 2.172778f, + 1.478360f, + 1.424623f, + 1.009947f, + 1.249336f, + 1.529020f, + 1.023295f, + 0.964028f, + 0.925674f, + 0.703490f, + 1.756039f, + 1.541209f, + })}, + {"ReduceSumSquare", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {4}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 2, 2}, + // expected values + { + 1.176016f, + 0.893560f, + 1.915388f, + 1.053586f, + 1.184450f, + 0.701256f, + 2.320143f, + 1.460843f, + 0.444038f, + 1.336670f, + 0.877501f, + 0.704944f, + 1.645929f, + 0.785122f, + 0.934894f, + 0.510638f, + 0.556905f, + 1.179667f, + 0.496201f, + 0.341346f, + 0.468223f, + 0.193697f, + 1.385610f, + 1.182342f, + })}, + {"ArgMax", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {4}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 2, 2}, + // expected values + { + 1.000000f, + 2.000000f, + 2.000000f, + 1.000000f, + 1.000000f, + 2.000000f, + 2.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 2.000000f, + 2.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 2.000000f, + 1.000000f, + 2.000000f, + 1.000000f, + 2.000000f, + 2.000000f, + 2.000000f, + 1.000000f, + })}, + {"ArgMin", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {4}, + // keep_dims_ + 0, + }, + // expected dims + {2, 3, 2, 2}, + // expected values + { + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 2.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 2.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 2.000000f, + 0.000000f, + 2.000000f, + 1.000000f, + 2.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + })}, + {"ReduceL1", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {4}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 2, 2, 1}, + // expected values + { + 1.866766f, + 1.614432f, + 2.293023f, + 1.704061f, + 1.564677f, + 0.939968f, + 2.626787f, + 2.041167f, + 0.901549f, + 1.881179f, + 1.494940f, + 1.204859f, + 2.172778f, + 1.478360f, + 1.424623f, + 1.009947f, + 1.249336f, + 1.529020f, + 1.023295f, + 0.964028f, + 0.925674f, + 0.703490f, + 1.756039f, + 1.541209f, + })}, + {"ReduceL2", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {4}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 2, 2, 1}, + // expected values + { + 1.084443f, + 0.945283f, + 1.383975f, + 1.026443f, + 1.088324f, + 0.837410f, + 1.523202f, + 1.208654f, + 0.666362f, + 1.156144f, + 0.936750f, + 0.839609f, + 1.282938f, + 0.886071f, + 0.966899f, + 0.714589f, + 0.746261f, + 1.086125f, + 0.704415f, + 0.584248f, + 0.684268f, + 0.440111f, + 1.177119f, + 1.087355f, + })}, + {"ReduceLogSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {4}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 2, 2, 1}, + // expected values + { + 0.624208f, + 0.478983f, + 0.829871f, + 0.533015f, + 0.447680f, + -0.061910f, + 0.965762f, + 0.713522f, + -0.103641f, + 0.631899f, + 0.402086f, + 0.186363f, + 0.776006f, + 0.390933f, + 0.353907f, + 0.009898f, + 0.222612f, + 0.424627f, + 0.023027f, + -0.036635f, + -0.077233f, + -0.351701f, + 0.563061f, + 0.432567f, + })}, + {"ReduceLogSumExp", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {4}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 2, 2, 1}, + // expected values + { + 1.723290f, + 1.640865f, + 1.888538f, + 1.681123f, + 1.679301f, + 1.484055f, + 1.977571f, + 1.790545f, + 1.429359f, + 1.752838f, + 1.619342f, + 1.534500f, + 1.835318f, + 1.601044f, + 1.613229f, + 1.464930f, + 1.521288f, + 1.676130f, + 1.465301f, + 1.425334f, + 1.439054f, + 1.337973f, + 1.738067f, + 1.677796f, + })}, + {"ReduceMax", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {4}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 2, 2, 1}, + // expected values + { + 0.715189f, + 0.645894f, + 0.963663f, + 0.791725f, + 0.925597f, + 0.832620f, + 0.978618f, + 0.799159f, + 0.639921f, + 0.944669f, + 0.774234f, + 0.617635f, + 0.943748f, + 0.681820f, + 0.697631f, + 0.670638f, + 0.570197f, + 0.988374f, + 0.653108f, + 0.466311f, + 0.656330f, + 0.368725f, + 0.837945f, + 0.976459f, + })}, + {"ReduceMean", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {4}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 2, 2, 1}, + // expected values + { + 0.622255f, + 0.538144f, + 0.764341f, + 0.568020f, + 0.521559f, + 0.313323f, + 0.875596f, + 0.680389f, + 0.300516f, + 0.627060f, + 0.498313f, + 0.401620f, + 0.724259f, + 0.492787f, + 0.474874f, + 0.336649f, + 0.416445f, + 0.509673f, + 0.341098f, + 0.321343f, + 0.308558f, + 0.234497f, + 0.585346f, + 0.513736f, + })}, + {"ReduceMin", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {4}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 2, 2, 1}, + // expected values + { + 0.548814f, + 0.423655f, + 0.437587f, + 0.383442f, + 0.071036f, + 0.020218f, + + 0.778157f, + 0.461479f, + 0.118274f, + 0.414662f, + 0.264556f, + 0.018790f, + 0.612096f, + 0.359508f, + 0.060225f, + 0.128926f, + 0.315428f, + 0.102045f, + 0.161310f, + 0.244426f, + 0.110375f, + 0.138183f, + 0.097101f, + 0.096098f, + })}, + {"ReduceProd", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {4}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 2, 2, 1}, + // expected values + { + 0.236588f, + 0.149100f, + 0.376049f, + 0.160562f, + 0.037349f, + 0.001467f, + 0.662530f, + 0.287855f, + 0.010850f, + 0.204418f, + 0.093432f, + 0.006597f, + 0.356381f, + 0.107125f, + 0.028014f, + 0.018190f, + 0.065416f, + 0.044237f, + 0.022006f, + 0.028870f, + 0.011516f, + 0.010016f, + 0.066801f, + 0.043976f, + })}, + {"ReduceSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {4}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 2, 2, 1}, + // expected values + { + 1.866766f, + 1.614432f, + 2.293023f, + 1.704061f, + 1.564677f, + 0.939968f, + 2.626787f, + 2.041167f, + 0.901549f, + 1.881179f, + 1.494940f, + 1.204859f, + 2.172778f, + 1.478360f, + 1.424623f, + 1.009947f, + 1.249336f, + 1.529020f, + 1.023295f, + 0.964028f, + 0.925674f, + 0.703490f, + 1.756039f, + 1.541209f, + })}, + {"ReduceSumSquare", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {4}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 2, 2, 1}, + // expected values + { + 1.176016f, + 0.893560f, + 1.915388f, + 1.053586f, + 1.184450f, + 0.701256f, + 2.320143f, + 1.460843f, + 0.444038f, + 1.336670f, + 0.877501f, + 0.704944f, + 1.645929f, + 0.785122f, + 0.934894f, + 0.510638f, + 0.556905f, + 1.179667f, + 0.496201f, + 0.341346f, + 0.468223f, + 0.193697f, + 1.385610f, + 1.182342f, + })}, + {"ArgMax", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {4}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 2, 2, 1}, + // expected values + { + 1.000000f, + 2.000000f, + 2.000000f, + 1.000000f, + 1.000000f, + 2.000000f, + 2.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 2.000000f, + 2.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 2.000000f, + 1.000000f, + 2.000000f, + 1.000000f, + 2.000000f, + 2.000000f, + 2.000000f, + 1.000000f, + })}, + {"ArgMin", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {4}, + // keep_dims_ + 1, + }, + // expected dims + {2, 3, 2, 2, 1}, + // expected values + { + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 2.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 2.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 2.000000f, + 0.000000f, + 2.000000f, + 1.000000f, + 2.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + })}, + {"ReduceL1", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {}, + // keep_dims_ + 0, + }, + // expected dims + {}, + // expected values + { + 35.911209f, + })}, + {"ReduceL2", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {}, + // keep_dims_ + 0, + }, + // expected dims + {}, + // expected values + { + 4.873291f, + })}, + {"ReduceLogSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {}, + // keep_dims_ + 0, + }, + // expected dims + {}, + // expected values + { + 3.581049f, + })}, + {"ReduceLogSumExp", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {}, + // keep_dims_ + 0, + }, + // expected dims + {}, + // expected values + { + 4.815600f, + })}, + {"ReduceMax", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {}, + // keep_dims_ + 0, + }, + // expected dims + {}, + // expected values + { + 0.988374f, + })}, + {"ReduceMean", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {}, + // keep_dims_ + 0, + }, + // expected dims + {}, + // expected values + { + 0.498767f, + })}, + {"ReduceMin", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {}, + // keep_dims_ + 0, + }, + // expected dims + {}, + // expected values + { + 0.018790f, + })}, + {"ReduceProd", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {}, + // keep_dims_ + 0, + }, + // expected dims + {}, + // expected values + { + 0.000000f, + })}, + {"ReduceSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {}, + // keep_dims_ + 0, + }, + // expected dims + {}, + // expected values + { + 35.911209f, + })}, + {"ReduceSumSquare", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {}, + // keep_dims_ + 0, + }, + // expected dims + {}, + // expected values + { + 23.748968f, + })}, + {"ArgMax", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {}, + // keep_dims_ + 0, + }, + // expected dims + {3, 2, 2, 3}, + // expected values + { + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + })}, + {"ArgMin", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {}, + // keep_dims_ + 0, + }, + // expected dims + {3, 2, 2, 3}, + // expected values + { + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + })}, + {"ReduceL1", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {}, + // keep_dims_ + 1, + }, + // expected dims + {1, 1, 1, 1, 1}, + // expected values + { + 35.911209f, + })}, + {"ReduceL2", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {}, + // keep_dims_ + 1, + }, + // expected dims + {1, 1, 1, 1, 1}, + // expected values + { + 4.873291f, + })}, + {"ReduceLogSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {}, + // keep_dims_ + 1, + }, + // expected dims + {1, 1, 1, 1, 1}, + // expected values + { + 3.581049f, + })}, + {"ReduceLogSumExp", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {}, + // keep_dims_ + 1, + }, + // expected dims + {1, 1, 1, 1, 1}, + // expected values + { + 4.815600f, + })}, + {"ReduceMax", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {}, + // keep_dims_ + 1, + }, + // expected dims + {1, 1, 1, 1, 1}, + // expected values + { + 0.988374f, + })}, + {"ReduceMean", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {}, + // keep_dims_ + 1, + }, + // expected dims + {1, 1, 1, 1, 1}, + // expected values + { + 0.498767f, + })}, + {"ReduceMin", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {}, + // keep_dims_ + 1, + }, + // expected dims + {1, 1, 1, 1, 1}, + // expected values + { + 0.018790f, + })}, + {"ReduceProd", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {}, + // keep_dims_ + 1, + }, + // expected dims + {1, 1, 1, 1, 1}, + // expected values + { + 0.000000f, + })}, + {"ReduceSum", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {}, + // keep_dims_ + 1, + }, + // expected dims + {1, 1, 1, 1, 1}, + // expected values + { + 35.911209f, + })}, + {"ReduceSumSquare", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {}, + // keep_dims_ + 1, + }, + // expected dims + {1, 1, 1, 1, 1}, + // expected values + { + 23.748968f, + })}, + {"ArgMax", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {}, + // keep_dims_ + 1, + }, + // expected dims + {1, 3, 2, 2, 3}, + // expected values + { + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + })}, + {"ArgMin", + OpAttributesResult( + // ReductionAttribute + { + // axes_ + {}, + // keep_dims_ + 1, + }, + // expected dims + {1, 3, 2, 2, 3}, + // expected values + { + 0.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 0.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 1.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + 0.000000f, + 1.000000f, + })}, + }}; diff --git a/onnxruntime/test/providers/cpu/reduction/reduction_test_cases_generator.py b/onnxruntime/test/providers/cpu/reduction/reduction_test_cases_generator.py new file mode 100644 index 0000000000000..b8bf6f126dd6a --- /dev/null +++ b/onnxruntime/test/providers/cpu/reduction/reduction_test_cases_generator.py @@ -0,0 +1,103 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import os +import numpy as np + +def TestReduction(op, data, axes, keepdims): + if op == "ReduceL1": + return np.sum(a=np.abs(data), axis=axes, keepdims=keepdims) + elif op == "ReduceL2": + return np.sqrt(np.sum(a=np.square(data), axis=axes, keepdims=keepdims)) + elif op == "ReduceLogSum": + return np.log(np.sum(data, axis=axes, keepdims=keepdims)) + elif op == "ReduceLogSumExp": + return np.log(np.sum(np.exp(data), axis=axes, keepdims=keepdims)) + elif op == "ReduceMax": + return np.max(data, axis=axes, keepdims=keepdims) + elif op == "ReduceMean": + return np.mean(data, axis=axes, keepdims=keepdims) + elif op == "ReduceMin": + return np.min(data, axis=axes, keepdims=keepdims) + elif op == "ReduceProd": + return np.prod(data, axis=axes, keepdims=keepdims) + elif op == "ReduceSum": + return np.sum(data, axis=axes, keepdims=keepdims) + elif op == "ReduceSumSquare": + return np.sum(np.square(data), axis=axes, keepdims=keepdims) + elif op == "ArgMax": + axis = axes[0] if axes else 0 + res = np.argmax(data, axis) + if keepdims: + res = np.expand_dims(res, axis) + return res + elif op == "ArgMin": + axis = axes[0] if axes else 0 + res = np.argmin(data, axis) + if keepdims: + res = np.expand_dims(res, axis) + return res + +def PrintResult(op, axes, keepdims, res): + print(" {\"%s\"," % op) + print("OpAttributesResult(") + print(" // ReductionAttribute") + print(" {") + print (" // axes_") + print ("{", end='') + print(*axes, sep=", ", end='') if axes else print("") + print ("},") + print (" // keep_dims_") + print (keepdims, ",") + print ("},") + + print (" // expected dims") + print ("{", end='') + print(*res.shape, sep=", ", end='') + print ("},") + + print (" // expected values") + print ("{", end='') + for i in range(0, res.size): + print("%5.6ff," % res.item(i)) + + print ("})},") + +if __name__ == "__main__": + from itertools import product + input_shape = [2,3,2,2,3] + np.random.seed(0) + input_data = np.random.uniform(size=input_shape) + axes_options = [(2,3), (2, 1, 4), (0, 2, 3), (0,), (2,), (4,), None] + keepdims_options = [0, 1] + ops = ["ReduceL1", "ReduceL2", "ReduceLogSum", "ReduceLogSumExp", "ReduceMax", "ReduceMean", + "ReduceMin", "ReduceProd", "ReduceSum", "ReduceSumSquare", "ArgMax", "ArgMin"] + print ("// Please don't manually edit this file. Generated from reduction_test_cases_generator.py") + print ("ReductionTestCases testcases = {") + print ("// input_data") + print ("{") + for i in range(0, input_data.size): + print("%5.6ff," % input_data.item(i),) + print ("},") + print ("// input_dims") + print ("{", end='') + print(*input_shape, sep=", ", end='') + print ("},") + + print(" // map_op_attribute_expected") + print ("{") + + for config in product(axes_options, keepdims_options, ops): + axes, keepdims, op = config + + #ArgMax and ArgMin only take single axis (default 0) + skip = False; + if op == "ArgMax" or op == "ArgMin": + skip = axes is not None and len(axes) > 1 + + if not skip: + res = TestReduction(op, input_data, axes, keepdims) + PrintResult(op, axes, keepdims, res) + + print ("}") + print ("};") diff --git a/onnxruntime/test/providers/cpu/rnn/GRU.py b/onnxruntime/test/providers/cpu/rnn/GRU.py new file mode 100644 index 0000000000000..3389c1d52b3a6 --- /dev/null +++ b/onnxruntime/test/providers/cpu/rnn/GRU.py @@ -0,0 +1,381 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import numpy as np + +DebugOutput = False +np.set_printoptions(suppress=True) #, precision=16, floatmode='maxprec') + +def print_with_shape(name, a, force_output=False): + if (force_output or DebugOutput): + print(name + " [shape: ", a.shape, "]\n", a) + +def print_results(Y): + print("*************************") + print_with_shape("Y", Y, True) + print("*************************") + +class GRU_Helper(): + + def __init__(self, **params): + # Match the ONNXRuntime/CNTK behavior + # If False use the python from the ONNX spec + self.match_onnxruntime = True + + required_inputs = ['X', 'W', 'R'] + for i in required_inputs: + assert i in params, "Missing Required Input: {0}".format(i) + + num_directions = params['W'].shape[0] + sequence_length = params['X'].shape[0] + + hidden_size = params['R'].shape[-1] + batch_size = params['X'].shape[1] + + X = params['X'] + W = params['W'] + R = params['R'] + B = params['B'] if 'B' in params else np.zeros(num_directions * 6 * hidden_size).reshape(num_directions, 6 * hidden_size) + H_0 = params['initial_h'] if 'initial_h' in params else np.zeros((num_directions, batch_size, hidden_size)).reshape(num_directions, batch_size, hidden_size) + LBR = params['linear_before_reset'] if 'linear_before_reset' in params else 0 + self.direction = params['direction'] if 'direction' in params else 'forward' + + if (num_directions == 1): + if (self.direction == 'forward'): + self.one = OneDirectionGRU(X, W, R, B, H_0, LBR) + else: + # flip input so we process in reverse + self.one = OneDirectionGRU(np.flip(X,0), W, R, B, H_0, LBR) + + self.two = None + + else: + # split the inputs which have per direction rows + Wfw, Wbw = np.vsplit(W, 2) + Rfw, Rbw = np.vsplit(R, 2) + Bfw, Bbw = np.vsplit(B, 2) + H_0fw, H_0bw = np.vsplit(H_0, 2) + + self.one = OneDirectionGRU(X, Wfw, Rfw, Bfw, H_0fw, LBR) + self.two = OneDirectionGRU(np.flip(X, 0), Wbw, Rbw, Bbw, H_0bw, LBR) + + def run(self): + + if (self.direction == 'bidirectional'): + f_output = self.one.execute() + r_output = self.two.execute() + + # flip reverse output it matches the original input order + r_output_orig_input_order = np.flip(r_output, 0) + + # create merged output by merging the forward and reverse rows for seq_length + # 0 rows, 2 directions, batch size, hidden_size + seq_length = f_output.shape[0] + batch_size = f_output.shape[2] + hidden_size = f_output.shape[3] + + output = np.empty((0, 2, batch_size, hidden_size), np.float32) + for x in range(0, seq_length): + output = np.append(output, f_output[x]) + output = np.append(output, r_output_orig_input_order[x]) + + output = output.reshape(seq_length, 2, batch_size, hidden_size) + else: + output = self.one.execute() + if (self.direction == 'reverse'): + # flip so it's back in the original order of the inputs + output = np.flip(output, 0) + + return output + +class OneDirectionGRU(): + + def __init__(self, X, W, R, B, initial_h, LBR): + + self.X = X + # remove num_directions axis for W, R, B, H_0 + self.W = np.squeeze(W, axis=0) + self.R = np.squeeze(R, axis=0) + self.B = np.squeeze(B, axis=0) + self.H_0 = np.squeeze(initial_h, axis=0) + self.LBR = LBR + + def f(self, x): + return 1 / (1 + np.exp(-x)) + + def g(self, x): + return np.tanh(x) + + def execute(self): + print_with_shape("X", self.X) + + [w_z, w_r, w_h] = np.split(self.W, 3) + [r_z, r_r, r_h] = np.split(self.R, 3) + [w_bz, w_br, w_bh, r_bz, r_br, r_bh] = np.split(self.B, 6) + + #print_with_shape("w_z", w_z) + #print_with_shape("w_r", w_r) + #print_with_shape("w_h", w_h) + + #print_with_shape("r_z", r_z) + #print_with_shape("r_r", r_r) + #print_with_shape("r_h", r_h) + + #print_with_shape("w_bz", w_bz) + #print_with_shape("w_br", w_br) + #print_with_shape("w_bh", w_bh) + #print_with_shape("r_bz", r_bz) + #print_with_shape("r_br", r_br) + #print_with_shape("r_bh", r_bh) + + seq_len = self.X.shape[0] + num_directions = 1 + hidden_size = self.R.shape[-1] + batch_size = self.X.shape[1] + + output = np.empty((0, num_directions, batch_size, hidden_size), np.float32) + + for row in self.X: + z = self.f(np.dot(row, np.transpose(w_z)) + np.dot(self.H_0, np.transpose(r_z)) + w_bz + r_bz) + r = self.f(np.dot(row, np.transpose(w_r)) + np.dot(self.H_0, np.transpose(r_r)) + w_br + r_br) + h_default = self.g(np.dot(row, np.transpose(w_h)) + np.dot(r * self.H_0, np.transpose(r_h)) + w_bh + r_bh) + h_linear = self.g(np.dot(row, np.transpose(w_h)) + r * (np.dot(self.H_0, np.transpose(r_h)) + r_bh) + w_bh) + + h = h_linear if self.LBR else h_default + + print_with_shape("z", z) + print_with_shape("r", r) + print_with_shape("h", h) + + H = (1 - z) * h + z * self.H_0 + + print_with_shape("H", H) + output = np.append(output, H.reshape(1, 1, batch_size, hidden_size), axis=0) + + self.H_0 = H + + return output + +class ONNXRuntimeTestContext(): + + @staticmethod + def OneDirectionWeights(): + + hidden_size = 2 + + W = np.array([[[-0.494659, 0.0453352], # Wz + [-0.487793, 0.417264], + [-0.0091708, -0.255364], # Wr + [-0.106952, -0.266717], + [-0.0888852, -0.428709], # Wh + [-0.283349, 0.208792]]]).astype(np.float32) + + R = np.array([[[0.146626, -0.0620289], # Rz + [-0.0815302, 0.100482], + [-0.228172, 0.405972], # Rr + [0.31576, 0.281487], + [-0.394864, 0.42111], # Rh + [-0.386624, -0.390225]]]).astype(np.float32) + + W_B = np.array([[0.381619, 0.0323954, # Wbz + -0.258721, 0.45056, # Wbr + -0.250755, 0.0967895]]).astype(np.float32) # Wbh + R_B = np.zeros((1, 3 * hidden_size)).astype(np.float32) + B = np.concatenate((W_B, R_B), axis=1) + + return W, R, B + + @staticmethod + def BidirectionalWeights(): + + W1, R1, B1 = ONNXRuntimeTestContext.OneDirectionWeights() + + hidden_size = R1.shape[-1] + input_size = W1.shape[-1] + + W = np.tile(W1, (2, 1)).reshape(2, 3 * hidden_size, input_size) + R = np.tile(R1, (2, 1)).reshape(2, 3 * hidden_size, hidden_size) + B = np.tile(B1, (2,1)) + + return W, R, B + +# replicate ONNXRuntime unit tests inputs to validate output +class GRU_ONNXRuntimeUnitTests(): + + @staticmethod + def ForwardDefaultActivationsSimpleWeightsNoBiasTwoRows(): + + print(GRU_ONNXRuntimeUnitTests.ForwardDefaultActivationsSimpleWeightsNoBiasTwoRows.__name__) + + seq_length = 2 + batch_size = 2 + input_size = 1 + hidden_size = 3 + input = np.array([1., 2., 10., 11.]).astype(np.float32).reshape(seq_length, batch_size, input_size) + + W = np.array([0.1, 0.2, 0.3, 1, 2, 3, 10, 11, 12]).astype(np.float32).reshape(1, 3 * hidden_size, input_size) + + weight_scale = 0.1 + R = weight_scale * np.ones((1, 3 * hidden_size, hidden_size)).astype(np.float32) + + gru = GRU_Helper(X=input,W=W,R=R,direction='forward') + fw_output = gru.run() + print_results(fw_output) + + @staticmethod + def ReverseDefaultActivationsSimpleWeightsNoBiasTwoRows(): + + print(GRU_ONNXRuntimeUnitTests.ReverseDefaultActivationsSimpleWeightsNoBiasTwoRows.__name__) + + seq_length = 2 + batch_size = 2 + input_size = 1 + hidden_size = 3 + input = np.array([[[1.], [2.]], + [[10.], [11.]]]).astype(np.float32) + + + W = np.array([0.1, 0.2, 0.3, 1, 2, 3, 10, 11, 12]).astype(np.float32).reshape(1, 3 * hidden_size, input_size) + + weight_scale = 0.1 + R = weight_scale * np.ones((1, 3 * hidden_size, hidden_size)).astype(np.float32) + + gru = GRU_Helper(X=input,W=W,R=R,direction='reverse') + fw_output = gru.run() + print_results(fw_output) + + @staticmethod + def BidirectionalDefaultActivationsSimpleWeightsNoBiasTwoRows(): + + print(GRU_ONNXRuntimeUnitTests.BidirectionalDefaultActivationsSimpleWeightsNoBiasTwoRows.__name__) + + seq_length = 2 + batch_size = 2 + input_size = 1 + hidden_size = 3 + + input = np.array([[[1.], [2.]], + [[10.], [11.]]]).astype(np.float32) + + W = np.array([0.1, 0.2, 0.3, 1, 2, 3, 10, 11, 12]).astype(np.float32).reshape(1, 3 * hidden_size, input_size) + + weight_scale = 0.1 + R = weight_scale * np.ones((1, 3 * hidden_size, hidden_size)).astype(np.float32) + + # duplicate the W and R inputs so we use the same values for both forward and reverse + gru = GRU_Helper(X=input, + W=np.tile(W,(2,1)).reshape(2, 3 * hidden_size, input_size), + R=np.tile(R,(2,1)).reshape(2, 3 * hidden_size, hidden_size), + direction='bidirectional') + + fw_output = gru.run() + print_results(fw_output) + + @staticmethod + def DefaultActivationsSimpleWeightsWithBias(rows=2, direction="forward", linear_before_reset=0): + + print(GRU_ONNXRuntimeUnitTests.DefaultActivationsSimpleWeightsWithBias.__name__ + + " batch_parallel=" + str(rows != 1) + + " direction=" + direction + + " linear_before_reset=" + str(linear_before_reset)) + + seq_length = 2 + batch_size = rows + input_size = 1 + hidden_size = 3 + + if (batch_size == 1): + input = [-0.1, -0.3] + else: + input = [-0.1, 0.2, -0.3, 0.4] + + input = np.array(input).astype(np.float32).reshape(seq_length, batch_size, input_size) + + W = np.array([0.1, 0.2, 0.3, 0.2, 0.3, 0.1, 0.3, 0.1, 0.2]).astype(np.float32).reshape(1, 3 * hidden_size, input_size) + + weight_scale = 0.1 + R = weight_scale * np.ones((1, 3 * hidden_size, hidden_size)).astype(np.float32) + + # Wb[zrh] Rb[zrh] + B = np.array([-0.01, 0.1, 0.01, -0.2, -0.02, 0.02, 0.3, -0.3, -0.3, + -0.03, 0.5, -0.7, 0.05, -0.7, 0.3, 0.07, -0.03, 0.5 ]).astype(np.float32).reshape(1, 6 * hidden_size) + + gru = GRU_Helper(X=input,W=W,R=R, B=B, direction=direction, linear_before_reset=linear_before_reset) + fw_output = gru.run() + print_results(fw_output) + + @staticmethod + def ForwardDefaultActivationsSimpleWeightsWithBiasBatchParallel(): + GRU_ONNXRuntimeUnitTests.DefaultActivationsSimpleWeightsWithBias() + + @staticmethod + def ForwardDefaultActivationsSimpleWeightsWithBiasBatchParallelLinearBeforeReset(): + + GRU_ONNXRuntimeUnitTests.DefaultActivationsSimpleWeightsWithBias(linear_before_reset=1) + + @staticmethod + def ReverseDefaultActivationsSimpleWeightsWithBiasBatchParallelLinearBeforeReset(linear_before_reset=0): + + GRU_ONNXRuntimeUnitTests.DefaultActivationsSimpleWeightsWithBias(direction="reverse", linear_before_reset=1) + + @staticmethod + def ForwardDefaultActivationsSimpleWeightsWithBiasLinearBeforeReset(): + + GRU_ONNXRuntimeUnitTests.DefaultActivationsSimpleWeightsWithBias(rows=1, linear_before_reset=1) + + @staticmethod + def ReverseDefaultActivationsSimpleWeightsWithBiasLinearBeforeReset(linear_before_reset=0): + + GRU_ONNXRuntimeUnitTests.DefaultActivationsSimpleWeightsWithBias(rows=1, direction="reverse", linear_before_reset=1) + + @staticmethod + def Legacy_TestGRUOpForwardBasic(): + + print(GRU_ONNXRuntimeUnitTests.Legacy_TestGRUOpForwardBasic.__name__) + + input = np.array([[[-0.455351, -0.276391]], + [[-0.185934, -0.269585]]]).astype(np.float32) + + W, R, B = ONNXRuntimeTestContext.OneDirectionWeights() + gru = GRU_Helper(X=input, W=W, R=R, B=B) + output = gru.run() + print_results(output) + + @staticmethod + def Legacy_TestGRUOpBackwardBasic(): + print(GRU_ONNXRuntimeUnitTests.Legacy_TestGRUOpBackwardBasic.__name__) + + input = np.array([[[-0.185934, -0.269585]], + [[-0.455351, -0.276391]]]).astype(np.float32) + + W, R, B = ONNXRuntimeTestContext.OneDirectionWeights() + gru = GRU_Helper(X=input, W=W, R=R, B=B, direction='reverse') + output = gru.run() + print_results(output) + + @staticmethod + def Legacy_TestGRUOpBidirectionalBasic(): + + print(GRU_ONNXRuntimeUnitTests.Legacy_TestGRUOpBidirectionalBasic.__name__) + + input = np.array([[[-0.455351, -0.276391]], + [[-0.185934, -0.269585]]]).astype(np.float32) + + W, R, B = ONNXRuntimeTestContext.BidirectionalWeights() + gru = GRU_Helper(X=input, W=W, R=R, B=B, direction='bidirectional') + output = gru.run() + print_results(output) + +GRU_ONNXRuntimeUnitTests.ForwardDefaultActivationsSimpleWeightsNoBiasTwoRows() +GRU_ONNXRuntimeUnitTests.ReverseDefaultActivationsSimpleWeightsNoBiasTwoRows() +GRU_ONNXRuntimeUnitTests.BidirectionalDefaultActivationsSimpleWeightsNoBiasTwoRows() + +GRU_ONNXRuntimeUnitTests.ForwardDefaultActivationsSimpleWeightsWithBiasBatchParallel() +GRU_ONNXRuntimeUnitTests.ForwardDefaultActivationsSimpleWeightsWithBiasBatchParallelLinearBeforeReset() +GRU_ONNXRuntimeUnitTests.ReverseDefaultActivationsSimpleWeightsWithBiasBatchParallelLinearBeforeReset() +GRU_ONNXRuntimeUnitTests.ForwardDefaultActivationsSimpleWeightsWithBiasLinearBeforeReset() +GRU_ONNXRuntimeUnitTests.ReverseDefaultActivationsSimpleWeightsWithBiasLinearBeforeReset() + +GRU_ONNXRuntimeUnitTests.Legacy_TestGRUOpForwardBasic() +GRU_ONNXRuntimeUnitTests.Legacy_TestGRUOpBackwardBasic() +GRU_ONNXRuntimeUnitTests.Legacy_TestGRUOpBidirectionalBasic() diff --git a/onnxruntime/test/providers/cpu/rnn/LSTM.py b/onnxruntime/test/providers/cpu/rnn/LSTM.py new file mode 100644 index 0000000000000..247ce57600eda --- /dev/null +++ b/onnxruntime/test/providers/cpu/rnn/LSTM.py @@ -0,0 +1,613 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import numpy as np # type: ignore +from typing import Any, Tuple + +#import onnx +#from ..base import Base +#from . import expect + +DebugOutput = True +np.set_printoptions(suppress=True) #, precision=16, floatmode='maxprec') + +def print_with_shape(name, a, force_output=False): + if (force_output or DebugOutput): + print(name + " [shape: ", a.shape, "]\n", a) + + +def print_results(Y, Y_h, Y_c): + print("*************************") + print_with_shape("Y", Y, True) + print("---------") + print_with_shape("Y_h", Y_h, True) + print("---------") + print_with_shape("Y_c", Y_c, True) + print("*************************") + +class LSTM_Helper(): + + def __init__(self, **params): # type: (*Any) -> None + + required_inputs = ['X', 'W', 'R'] + for i in required_inputs: + assert i in params, "Missing Required Input: {0}".format(i) + + X = params['X'] + W = params['W'] + R = params['R'] + + num_directions = W.shape[0] + sequence_length = X.shape[0] + batch_size = X.shape[1] + hidden_size = R.shape[-1] + + B = params['B'] if 'B' in params else np.zeros(num_directions * 8 * hidden_size).reshape(num_directions, 8 * hidden_size) + P = params['P'] if 'P' in params else np.zeros(num_directions * 3 * hidden_size).reshape(num_directions, 3 * hidden_size) + h_0 = params['initial_h'] if 'initial_h' in params else np.zeros((num_directions, batch_size, hidden_size)).reshape(num_directions, batch_size, hidden_size) + c_0 = params['initial_c'] if 'initial_c' in params else np.zeros((num_directions, batch_size, hidden_size)).reshape(num_directions, batch_size, hidden_size) + + f = params['f'] if 'f' in params else ActivationFuncs.sigmoid + g = params['g'] if 'g' in params else ActivationFuncs.tanh + h = params['h'] if 'h' in params else ActivationFuncs.tanh + input_forget = params['input_forget'] if 'input_forget' in params else False + clip = params['clip'] if 'clip' in params else 9999.0 + + self.direction = params['direction'] if 'direction' in params else 'forward' + + if (num_directions == 1): + if (self.direction == 'forward'): + self.one = OneDirectionLSTM(X, W, R, B, P, h_0, c_0, f, g, h, input_forget, clip) + else: + # flip input so we process in reverse + self.one = OneDirectionLSTM(np.flip(X, 0), W, R, B, P, h_0, c_0, f, g, h, input_forget, clip) + + self.two = None + + else: + # split the inputs which have per direction rows + Wfw, Wbw = np.vsplit(W, 2) + Rfw, Rbw = np.vsplit(R, 2) + Bfw, Bbw = np.vsplit(B, 2) + Pfw, Pbw = np.vsplit(P, 2) + h_0fw, h_0bw = np.vsplit(h_0, 2) + c_0fw, c_0bw = np.vsplit(c_0, 2) + + self.one = OneDirectionLSTM(X, Wfw, Rfw, Bfw, Pfw, h_0fw, c_0fw, f, g, h, input_forget, clip) + self.two = OneDirectionLSTM(np.flip(X, 0), Wbw, Rbw, Bbw, Pfw, h_0bw, c_0fw, f, g, h, input_forget, clip) + + def run(self): + + if (self.direction == 'bidirectional'): + f_output, f_Y_h, f_Y_c = self.one.execute() + r_output, r_Y_h, r_Y_c = self.two.execute() + + # flip reverse output it matches the original input order + r_output_orig_input_order = np.flip(r_output, 0) + + # create merged output by merging the forward and reverse rows for seq_length + # 0 rows, 2 directions, batch size, hidden_size + seq_length = f_output.shape[0] + batch_size = f_output.shape[2] + hidden_size = f_output.shape[3] + + output = np.empty((0, 2, batch_size, hidden_size), np.float32) + #Y_h = np.empty((0, 2, batch_size, hidden_size), np.float32) + #Y_c = np.empty((0, 2, hidden_size, hidden_size), np.float32) + for x in range(0, seq_length): + output = np.append(output, f_output[x]) + output = np.append(output, r_output_orig_input_order[x]) + + output = output.reshape(seq_length, 2, batch_size, hidden_size) + + Y_h = np.append(f_Y_h, r_Y_h) + Y_c = np.append(f_Y_c, r_Y_c) + + else: + output, Y_h, Y_c = self.one.execute() + if (self.direction == 'reverse'): + # flip so it's back in the original order of the inputs + output = np.flip(output, 0) + + return output, Y_h, Y_c + +class ActivationFuncs: + @staticmethod + def sigmoid(x): + return 1 / (1 + np.exp(-x)) + + @staticmethod + def tanh(x): + return np.tanh(x) + +class OneDirectionLSTM: + + def __init__(self, X, W, R, B, P, initial_h, initial_c, + f = ActivationFuncs.sigmoid, g = ActivationFuncs.tanh, h = ActivationFuncs.tanh, + input_forget = False, clip = 9999.0): + + self.X = X + # remove num_directions axis for W, R, B, P, H_0, C_0 + self.W = np.squeeze(W, axis=0) + self.R = np.squeeze(R, axis=0) + self.B = np.squeeze(B, axis=0) + self.P = np.squeeze(P, axis=0) + self.h_0 = np.squeeze(initial_h, axis=0) + self.c_0 = np.squeeze(initial_c, axis=0) + + print_with_shape("X", self.X) + print_with_shape("W", self.W) + print_with_shape("R", self.R) + print_with_shape("B", self.B) + print_with_shape("P", self.P) + print_with_shape("h_0", self.h_0) + print_with_shape("c_0", self.c_0) + + self.f = f + self.g = g + self.h = h + self.input_forget = input_forget + self.clip = clip + + def execute(self): # type: () -> Tuple[np.ndarray, np.ndarray] + + [p_i, p_o, p_f] = np.split(self.P, 3) + h_list = [] + + H_t = self.h_0 + C_t = self.c_0 + + for x in np.split(self.X, self.X.shape[0], axis=0): + print_with_shape("Xt1", x) + + #gates = np.dot(x, np.transpose(self.W)) + np.dot(H_t, np.transpose(self.R)) + np.add(*np.split(self.B, 2)) + + print_with_shape("W^T", np.transpose(self.W)) + # t0 == t-1, t1 == current + Xt1_W = np.dot(x, np.transpose(self.W)) + print_with_shape("Xt1_W^T", Xt1_W) + Ht0_R = np.dot(H_t, np.transpose(self.R)) + print_with_shape("Ht-1*R", Ht0_R) + WbRb = np.add(*np.split(self.B, 2)) + print_with_shape("Wb + Rb", WbRb) + gates = Xt1_W + Ht0_R + WbRb + + # input to it, ft, ct, ot + it_in, ot_in, ft_in, ct_in = np.split(gates, 4, -1) + print_with_shape("it_in", it_in) + print_with_shape("ot_in", ot_in) + print_with_shape("ft_in", ft_in) + print_with_shape("ct_in", ct_in) + + i = self.f(np.clip((it_in + p_i * C_t), -self.clip, self.clip)) + if (self.input_forget): + f = 1.0 - i # this is what ONNXRuntime does + else: + f = self.f(np.clip((ft_in + p_f * C_t), -self.clip, self.clip)) + c = self.g(np.clip(ct_in, -self.clip, self.clip)) + C = f * C_t + i * c + o = self.f(np.clip((ot_in + p_o * C), -self.clip, self.clip)) + H = o * self.h(C) + h_list.append(H) + H_t = H + C_t = C + + print_with_shape("i", i) + print_with_shape("f", f) + print_with_shape("c", c) + print_with_shape("o", o) + print_with_shape("C", C) + print_with_shape("H", i) + + concatenated = np.concatenate(h_list) + output = np.expand_dims(concatenated, 1) + return output, h_list[-1], C + + +class LSTM: # Base): + + @staticmethod + def SimpleWeightsNoBiasTwoRows(direction): # type: () -> None + + print(LSTM.SimpleWeightsNoBiasTwoRows.__name__ + " direction=" + direction) + + seq_length = 2 + batch_size = 2 + input_size = 1 + hidden_size = 3 + number_of_gates = 4 + + input = np.array([[[1.], [2.]], + [[10.], [11.]]]).astype(np.float32) + + W = np.array([0.1, 0.2, 0.3, 0.4, + 1, 2, 3, 4, + 10, 11, 12, 13]).astype(np.float32).reshape(1, number_of_gates * hidden_size, input_size) + + weight_scale = 0.1 + R = weight_scale * np.ones((1, number_of_gates * hidden_size, hidden_size)).astype(np.float32) + + + if (direction == 'bidirectional'): + W = W=np.tile(W, (2, 1)).reshape(2, number_of_gates * hidden_size, input_size) + R = R=np.tile(R, (2, 1)).reshape(2, number_of_gates * hidden_size, hidden_size) + + lstm = LSTM_Helper(X=input, W=W, R=R, direction=direction) + + Y, Y_h, Y_c = lstm.run() + # expect(node, inputs=[input, W, R], outputs=[Y_h.astype(np.float32)], name='test_lstm_defaults') + print_results(Y, Y_h, Y_c) + + @staticmethod + def LargeBatchWithClip(clip): + + print(LSTM.LargeBatchWithClip.__name__ + " clip=" + str(clip)) + + seq_length = 2 + batch_size = 32 + input_size = 1 + hidden_size = 3 + number_of_gates = 4 + + # sequentialvalues from 1 to 32 + input = np.array(range(1,seq_length * batch_size + 1, 1)).astype(np.float32).reshape(seq_length, batch_size, input_size) + + W = np.array([0.1, 0.2, 0.3, 0.4, + 1, 2, 3, 4, + 10, 11, 12, 13]).astype(np.float32).reshape(1, number_of_gates * hidden_size, input_size) + + weight_scale = 0.1 + R = weight_scale * np.ones((1, number_of_gates * hidden_size, hidden_size)).astype(np.float32) + + lstm = LSTM_Helper(X=input, W=W, R=R, clip=clip) + + Y, Y_h, Y_c = lstm.run() + print_results(Y, Y_h, Y_c) + + @staticmethod + def BatchParallelFalseSeqLengthGreaterThanOne(): + print(LSTM.BatchParallelFalseSeqLengthGreaterThanOne.__name__) + + seq_length = 2 + batch_size = 1 + input_size = 1 + hidden_size = 2 + number_of_gates = 4 + + input = np.array([1, 2]).astype(np.float32).reshape(seq_length, batch_size, input_size) + + W = np.array([0.1, 0.2, 0.3, 0.4, + 1, 2, 3, 4]).astype(np.float32).reshape(1, number_of_gates * hidden_size, input_size) + + weight_scale = 0.1 + R = weight_scale * np.ones((1, number_of_gates * hidden_size, hidden_size)).astype(np.float32) + + lstm = LSTM_Helper(X=input, W=W, R=R) + + Y, Y_h, Y_c = lstm.run() + print_results(Y, Y_h, Y_c) + + + @staticmethod + def export_initial_bias(): # type: () -> None + + print(LSTM.export_initial_bias.__name__) + + input = np.array([[[1., 2., 3.], [4., 5., 6.], [7., 8., 9.]]]).astype(np.float32) + + input_size = 3 + hidden_size = 4 + weight_scale = 0.1 + custom_bias = 0.1 + number_of_gates = 4 + + # node = onnx.helper.make_node( + # 'LSTM', + # inputs=['X', 'W', 'R', 'B'], + # outputs=['', 'Y'], + # hidden_size=hidden_size + # ) + + W = weight_scale * np.ones((1, number_of_gates * hidden_size, input_size)).astype(np.float32) + R = weight_scale * np.ones((1, number_of_gates * hidden_size, hidden_size)).astype(np.float32) + + # Adding custom bias + W_B = custom_bias * np.ones((1, number_of_gates * hidden_size)).astype(np.float32) + R_B = np.zeros((1, number_of_gates * hidden_size)).astype(np.float32) + B = np.concatenate((W_B, R_B), 1) + + lstm = LSTM_Helper(X=input, W=W, R=R, B=B) + Y, Y_h, Y_c = lstm.run() + print_results(Y, Y_h, Y_c) + # expect(node, inputs=[input, W, R, B], outputs=[Y_h.astype(np.float32)], name='test_lstm_with_initial_bias') + + @staticmethod + def export_peepholes(): # type: () -> None + input = np.array([[[1., 2., 3., 4.], [5., 6., 7., 8.]]]).astype(np.float32) + + input_size = 4 + hidden_size = 3 + weight_scale = 0.1 + number_of_gates = 4 + number_of_peepholes = 3 + + # node = onnx.helper.make_node( + # 'LSTM', + # inputs=['X', 'W', 'R', 'B', 'sequence_lens', 'initial_h', 'initial_c', 'P'], + # outputs=['', 'Y'], + # hidden_size=hidden_size + # ) + + # Initializing Inputs + W = weight_scale * np.ones((1, number_of_gates * hidden_size, input_size)).astype(np.float32) + R = weight_scale * np.ones((1, number_of_gates * hidden_size, hidden_size)).astype(np.float32) + B = np.zeros((1, 2 * number_of_gates * hidden_size)).astype(np.float32) + seq_lens = np.repeat(input.shape[0], input.shape[1]).astype(np.int32) + init_h = np.zeros((1, input.shape[1], hidden_size)).astype(np.float32) + init_c = np.zeros((1, input.shape[1], hidden_size)).astype(np.float32) + P = weight_scale * np.ones((1, number_of_peepholes * hidden_size)).astype(np.float32) + + lstm = LSTM_Helper(X=input, W=W, R=R, B=B, P=P, initial_c=init_c, initial_h=init_h) + Y, Y_h, Y_c = lstm.run() + print_results(Y, Y_h, Y_c) + # expect(node, inputs=[input, W, R, B, seq_lens, init_h, init_c, P], outputs=[Y_h.astype(np.float32)], + # name='test_lstm_with_peepholes') + +class ONNXRuntimeTestContext: + + hidden_size = 2 + input_size = 2 + + @staticmethod + def OneDirectionWeights(): + + num_directions = 1 + hidden_size = ONNXRuntimeTestContext.hidden_size + input_size = ONNXRuntimeTestContext.input_size + + W = np.array([ + -0.494659, 0.0453352, + -0.487793, 0.417264, + + -0.0175329, 0.489074, + -0.446013, 0.414029, + + -0.0091708, -0.255364, + -0.106952, -0.266717, + + -0.0888852, -0.428709, + -0.283349, 0.208792]).reshape(num_directions, 4 * hidden_size, input_size).astype(np.float32) + + R = np.array([ + 0.146626, -0.0620289, + -0.0815302, 0.100482, + + -0.219535, -0.306635, + -0.28515, -0.314112, + + -0.228172, 0.405972, + 0.31576, 0.281487, + + -0.394864, 0.42111, + -0.386624, -0.390225]).reshape(num_directions, 4 * hidden_size, hidden_size).astype(np.float32) + + P = np.array([ + 0.2345, 0.5235, + 0.4378, 0.3475, + 0.8927, 0.3456]).reshape(num_directions, 3 * hidden_size).astype(np.float32) + + # // [8*hidden] + B = np.array([ + 0.381619, 0.0323954, + -0.14449, 0.420804, + -0.258721, 0.45056, + -0.250755, 0.0967895, + + # peephole bias + 0.0, 0.0, + 0.0, 0.0, + 0.0, 0.0, + 0.0, 0.0]).reshape(num_directions, 8 * hidden_size).astype(np.float32) + + return W, R, B, P + + @staticmethod + def BidirectionalWeights(): + + hidden_size = ONNXRuntimeTestContext.hidden_size + input_size = ONNXRuntimeTestContext.input_size + + W1, R1, B1, P1 = ONNXRuntimeTestContext.OneDirectionWeights() + + W = np.tile(W1, (2, 1)).reshape(2, 4 * hidden_size, input_size) + R = np.tile(R1, (2, 1)).reshape(2, 4 * hidden_size, hidden_size) + B = np.tile(B1, (2,1)) + P = np.tile(P1, (2,1)) + + return W, R, B, P + + @staticmethod + def DefaultInput(): + seq_length = 2 + batch_size = 1 + input_size = 2 + + input = np.array([-0.455351, -0.276391, -0.185934, -0.269585])\ + .reshape(seq_length, batch_size, input_size)\ + .astype(np.float32) + + return input + +class ONNXRuntimeUnitTests: + + @staticmethod + def ONNXRuntime_TestLSTMBidirectionalBasic(): + print(ONNXRuntimeUnitTests.ONNXRuntime_TestLSTMBidirectionalBasic.__name__) + + input = ONNXRuntimeTestContext.DefaultInput() + W, R, B, P = ONNXRuntimeTestContext.BidirectionalWeights() + lstm = LSTM_Helper(X=input, W=W, R=R, B=B, P=P, direction='bidirectional') + Y, Y_h, Y_c = lstm.run() + print_results(Y, Y_h, Y_c) + + @staticmethod + def ONNXRuntime_TestLSTMForwardNoBiasUsePeepholes(): + print(ONNXRuntimeUnitTests.ONNXRuntime_TestLSTMForwardNoBiasUsePeepholes.__name__) + input = ONNXRuntimeTestContext.DefaultInput() + W, R, B, P = ONNXRuntimeTestContext.OneDirectionWeights() + lstm = LSTM_Helper(X=input, W=W, R=R, P=P) # no bias + Y, Y_h, Y_c = lstm.run() + print_results(Y, Y_h, Y_c) + + @staticmethod + def ONNXRuntime_TestLSTMForwardInputForget(): + print(ONNXRuntimeUnitTests.ONNXRuntime_TestLSTMForwardInputForget.__name__) + + input = ONNXRuntimeTestContext.DefaultInput() + W, R, B, P = ONNXRuntimeTestContext.OneDirectionWeights() + lstm = LSTM_Helper(X=input, W=W, R=R, B=B, P=P, input_forget=True) + Y, Y_h, Y_c = lstm.run() + print_results(Y, Y_h, Y_c) + + @staticmethod + def ONNXRuntime_TestLSTMForwardClip(): + print(ONNXRuntimeUnitTests.ONNXRuntime_TestLSTMForwardClip.__name__) + + input = ONNXRuntimeTestContext.DefaultInput() + W, R, B, P = ONNXRuntimeTestContext.OneDirectionWeights() + lstm = LSTM_Helper(X=input, W=W, R=R, B=B, P=P, clip=0.1) + Y, Y_h, Y_c = lstm.run() + print_results(Y, Y_h, Y_c) + + @staticmethod + def ONNXRuntime_TestLSTMBackward(): + print(ONNXRuntimeUnitTests.ONNXRuntime_TestLSTMBackward.__name__) + + input = ONNXRuntimeTestContext.DefaultInput() + W, R, B, P = ONNXRuntimeTestContext.OneDirectionWeights() + lstm = LSTM_Helper(X=input, W=W, R=R, B=B, P=P, direction='reverse') + Y, Y_h, Y_c = lstm.run() + print_results(Y, Y_h, Y_c) + + @staticmethod + def ONNXRuntime_TestLSTMForwardHiddenState(): + print(ONNXRuntimeUnitTests.ONNXRuntime_TestLSTMForwardHiddenState.__name__) + + input = ONNXRuntimeTestContext.DefaultInput() + W, R, B, P = ONNXRuntimeTestContext.OneDirectionWeights() + initial_h = np.array([0.34, 0.72]).reshape(1,1,2).astype(np.float32) + lstm = LSTM_Helper(X=input, W=W, R=R, B=B, initial_h=initial_h) + Y, Y_h, Y_c = lstm.run() + print_results(Y, Y_h, Y_c) + + @staticmethod + def ONNXRuntime_TestLSTMForwardCellState(): + print(ONNXRuntimeUnitTests.ONNXRuntime_TestLSTMForwardCellState.__name__) + + input = ONNXRuntimeTestContext.DefaultInput() + W, R, B, P = ONNXRuntimeTestContext.OneDirectionWeights() + initial_h = np.array([0.34, 0.72]).reshape(1,1,2).astype(np.float32) + initial_c = np.array([0.63, 0.21]).reshape(1,1,2).astype(np.float32) + lstm = LSTM_Helper(X=input, W=W, R=R, B=B, initial_h=initial_h, initial_c=initial_c) + Y, Y_h, Y_c = lstm.run() + print_results(Y, Y_h, Y_c) + + @staticmethod + def ONNXRuntime_TestLSTMActivation(): + + print(ONNXRuntimeUnitTests.ONNXRuntime_TestLSTMActivation.__name__) + + input = ONNXRuntimeTestContext.DefaultInput() + W, R, B, P = ONNXRuntimeTestContext.OneDirectionWeights() + lstm = LSTM_Helper(X=input, W=W, R=R, B=B, + f=ActivationFuncs.tanh, g=ActivationFuncs.sigmoid, h=ActivationFuncs.tanh) + Y, Y_h, Y_c = lstm.run() + print_results(Y, Y_h, Y_c) + + @staticmethod + def ONNXRuntime_TestLSTMBatchReallocation(): + + print(ONNXRuntimeUnitTests.ONNXRuntime_TestLSTMBatchReallocation.__name__) + seq_length = 2 + batch_size = 1 + input_size = 2 + + input = ONNXRuntimeTestContext.DefaultInput() + W, R, B, P = ONNXRuntimeTestContext.OneDirectionWeights() + lstm = LSTM_Helper(X=input, W=W, R=R, B=B, + f=ActivationFuncs.tanh, g=ActivationFuncs.sigmoid, h=ActivationFuncs.tanh) + Y, Y_h, Y_c = lstm.run() + print_results(Y, Y_h, Y_c) + print ("===============") + + batch_size = 3 + input = np.array([-0.455351, -0.476391, + -0.555351, -0.376391, + -0.655351, -0.276391, + + -0.185934, -0.869585, + -0.285934, -0.769585, + -0.385934, -0.669585]).reshape(seq_length, batch_size, input_size).astype(np.float32) + + W, R, B, P = ONNXRuntimeTestContext.OneDirectionWeights() + lstm = LSTM_Helper(X=input, W=W, R=R, B=B, + f=ActivationFuncs.tanh, g=ActivationFuncs.sigmoid, h=ActivationFuncs.tanh) + Y, Y_h, Y_c = lstm.run() + print_results(Y, Y_h, Y_c) + + @staticmethod + def ONNXRuntime_TestLSTMOutputWrite(): + + print(ONNXRuntimeUnitTests.ONNXRuntime_TestLSTMOutputWrite.__name__) + seq_length = 2 + batch_size = 1 + input_size = 2 + + input = ONNXRuntimeTestContext.DefaultInput() + W, R, B, P = ONNXRuntimeTestContext.BidirectionalWeights() + lstm = LSTM_Helper(X=input, W=W, R=R, B=B, direction='bidirectional', + f=ActivationFuncs.tanh, g=ActivationFuncs.sigmoid, h=ActivationFuncs.tanh) + Y, Y_h, Y_c = lstm.run() + print_results(Y, Y_h, Y_c) + + print ("===============") + + batch_size = 3 + input = np.array([-0.455351, -0.776391, + -0.355351, -0.576391, + -0.255351, -0.376391, + + -0.185934, -0.169585, + -0.285934, -0.469585, + -0.385934, -0.669585]).reshape(seq_length, batch_size, input_size).astype(np.float32) + + W, R, B, P = ONNXRuntimeTestContext.BidirectionalWeights() + lstm = LSTM_Helper(X=input, W=W, R=R, B=B, direction='bidirectional', + f=ActivationFuncs.tanh, g=ActivationFuncs.sigmoid, h=ActivationFuncs.tanh) + Y, Y_h, Y_c = lstm.run() + print_results(Y, Y_h, Y_c) + + + +DebugOutput = False +LSTM.SimpleWeightsNoBiasTwoRows('forward') +LSTM.SimpleWeightsNoBiasTwoRows('reverse') +LSTM.SimpleWeightsNoBiasTwoRows('bidirectional') +LSTM.LargeBatchWithClip(99999.0) # too large to affect output +LSTM.LargeBatchWithClip(4.0) +LSTM.BatchParallelFalseSeqLengthGreaterThanOne() +ONNXRuntimeUnitTests.ONNXRuntime_TestLSTMBidirectionalBasic() +ONNXRuntimeUnitTests.ONNXRuntime_TestLSTMForwardNoBiasUsePeepholes() +ONNXRuntimeUnitTests.ONNXRuntime_TestLSTMForwardInputForget() +ONNXRuntimeUnitTests.ONNXRuntime_TestLSTMForwardClip() +ONNXRuntimeUnitTests.ONNXRuntime_TestLSTMBackward() +ONNXRuntimeUnitTests.ONNXRuntime_TestLSTMForwardHiddenState() +ONNXRuntimeUnitTests.ONNXRuntime_TestLSTMForwardCellState() +ONNXRuntimeUnitTests.ONNXRuntime_TestLSTMActivation() +ONNXRuntimeUnitTests.ONNXRuntime_TestLSTMBatchReallocation() +ONNXRuntimeUnitTests.ONNXRuntime_TestLSTMOutputWrite() diff --git a/onnxruntime/test/providers/cpu/rnn/deep_cpu_gru_op_test.cc b/onnxruntime/test/providers/cpu/rnn/deep_cpu_gru_op_test.cc new file mode 100644 index 0000000000000..607b0fbc735e3 --- /dev/null +++ b/onnxruntime/test/providers/cpu/rnn/deep_cpu_gru_op_test.cc @@ -0,0 +1,785 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" + +#include +#include + +#include "core/providers/cpu/rnn/deep_cpu_gru.h" +#include "test/providers/provider_test_utils.h" +using namespace std; +namespace onnxruntime { +namespace test { + +static void RunGruTest(bool run_on_gpu, + const std::vector& X_data, + const std::vector& W_data, + const std::vector& R_data, + const std::vector& Y_data, + const std::vector& Y_h_data, + int64_t input_size, + int batch_size, + int64_t hidden_size, + int64_t seq_length, + const std::vector* B_data = nullptr, + const std::vector* initial_h_data = nullptr, + const std::vector* sequence_lengths = nullptr, + const std::string& direction = "forward", + float clip = 9999.0, + bool output_sequence = true, + bool linear_before_reset = false, + // copy the following vectors as we may modify them + std::vector activations = {"sigmoid", "tanh"}, + std::vector activation_alphas = {}, + std::vector activation_betas = {}) { + OpTester test("GRU"); + + test.AddShapeToTensorData(); + + int num_directions = (direction == "bidirectional") ? 2 : 1; + + if (num_directions == 2 && activations.size() == 2) { + activations.reserve(4); // need to avoid reallocation when inserting + // default to copying the activations so the same are used for forward and backwards + std::copy(activations.cbegin(), activations.cend(), std::back_inserter(activations)); + } + + test.AddAttribute>("activations", activations); + if (!activation_alphas.empty()) + test.AddAttribute>("activation_alpha", activation_alphas); + if (!activation_betas.empty()) + test.AddAttribute>("activation_beta", activation_betas); + + test.AddAttribute("direction", direction); + test.AddAttribute("hidden_size", hidden_size); + // test.AddAttribute("output_sequence", output_sequence); + test.AddAttribute("linear_before_reset", linear_before_reset); + // if clip is a very big number (usually it is default value), don't set the clip + if (clip < 999.f) + test.AddAttribute("clip", clip); + + std::vector X_dims = {seq_length, batch_size, input_size}; + std::vector W_dims = {num_directions, 3 * hidden_size, input_size}; + std::vector R_dims = {num_directions, 3 * hidden_size, hidden_size}; + + test.AddInput("X", X_dims, X_data); + test.AddInput("W", W_dims, W_data, true); + test.AddInput("R", R_dims, R_data, true); + + if (B_data) { + std::vector B_dims = {num_directions, 6 * hidden_size}; + test.AddInput("B", B_dims, *B_data, true); + } + + if (sequence_lengths) { + std::vector sequence_lens_dims{batch_size}; + test.AddInput("sequence_lens", sequence_lens_dims, *sequence_lengths); + } + + if (initial_h_data) { + std::vector initial_h_dims = {num_directions, batch_size, hidden_size}; + test.AddInput("initial_h", initial_h_dims, *initial_h_data); + } + + if (output_sequence != 0) { + std::vector Y_dims = {seq_length, num_directions, batch_size, hidden_size}; + test.AddOutput("Y", Y_dims, Y_data); + } else { + test.AddMissingOptionalOutput(); + } + + if (!Y_h_data.empty()) { + std::vector Y_h_dims{num_directions, batch_size, hidden_size}; + test.AddOutput("Y_h", Y_h_dims, Y_h_data); + } else { + test.AddMissingOptionalOutput(); + } + + std::unordered_set excluded_provider_types; + if (!run_on_gpu) { + excluded_provider_types.insert(kCudaExecutionProvider); + } + test.Run(OpTester::ExpectResult::kExpectSuccess, "", excluded_provider_types); +} + +void DefaultActivationsSimpleWeightsNoBias(bool run_on_gpu, + std::string direction, + const std::vector& Y_data, + const std::vector& Y_h_data) { + int64_t seq_length = 2; + int batch_size = 2; + int64_t input_size = 1; + int64_t hidden_size = 3; + + int num_directions = direction == "bidirectional" ? 2 : 1; + + std::vector X_data{1.f, 2.f, + 10.f, 11.f}; + + std::vector W_data{0.1f, 0.2f, 0.3f, // wz + 1.f, 2.f, 3.f, // wr + 10.f, 11.f, 12.f}; // wh + + // duplicate for bidirectional + if (num_directions == 2) { + W_data.reserve(W_data.size() * 2); // need to avoid reallocation when inserting + std::copy(W_data.cbegin(), W_data.cend(), std::back_inserter(W_data)); + } + + std::vector R_data(num_directions * 3 * hidden_size * hidden_size, 0.1f); + + RunGruTest(run_on_gpu, X_data, W_data, R_data, Y_data, Y_h_data, input_size, batch_size, hidden_size, seq_length, + nullptr, nullptr, nullptr, direction); + + // if Y_h_data is empty that tests Y_h not being returned. we need to have at least one output or + // the node will get removed, so only test with output_sequence == false (no Y as output) if Y_h is not optional + if (!Y_h_data.empty()) + RunGruTest(run_on_gpu, X_data, W_data, R_data, Y_data, Y_h_data, input_size, batch_size, hidden_size, seq_length, + nullptr, nullptr, nullptr, direction, 9999.0, /* output_sequence*/ false); +} + +TEST(GRUTest, ForwardDefaultActivationsSimpleWeightsNoBiasTwoRows) { + std::vector Y_data{ + 0.4750208f, 0.450166f, 0.4255575f, + 0.45016602f, 0.40131235f, 0.35434368f, + + 0.6027093f, 0.5083023f, 0.44950223f, + 0.5754369f, 0.45485455f, 0.3747841f}; + + std::vector Y_h_data{ + 0.6027093f, 0.5083023f, 0.44950223f, + 0.5754369f, 0.45485455f, 0.3747841f}; + + bool run_on_gpu = true; + DefaultActivationsSimpleWeightsNoBias(run_on_gpu, "forward", Y_data, Y_h_data); + + // test Y_h not being returned + DefaultActivationsSimpleWeightsNoBias(run_on_gpu, "forward", Y_data, {}); +} + +TEST(GRUTest, ReverseDefaultActivationsSimpleWeightsNoBiasTwoRows) { + std::vector Y_data{ + 0.6082785f, 0.50623393f, 0.4426924f, + 0.5803454f, 0.4527356f, 0.36886263f, + + 0.26894143f, 0.11920292f, 0.04742587f, + 0.24973989f, 0.09975048f, 0.03557118f}; + + std::vector Y_h_data{ + 0.6082785f, 0.50623393f, 0.4426924f, + 0.5803454f, 0.4527356f, 0.36886263f}; + + bool run_on_gpu = false; // cudnn implementation only support linear_before_reset = true + DefaultActivationsSimpleWeightsNoBias(run_on_gpu, "reverse", Y_data, Y_h_data); +} + +TEST(GRUTest, BidirectionalDefaultActivationsSimpleWeightsNoBiasTwoRows) { + std::vector Y_data{ + // forward output for input sequence 0 + 0.4750208f, 0.450166f, 0.4255575f, + 0.45016602f, 0.40131235f, 0.35434368f, + + // reverse output for input sequence 0 [sequence 1 in reversed input] + 0.6082785f, 0.50623393f, 0.4426924f, + 0.5803454f, 0.4527356f, 0.36886263f, + + // forward output for input sequence 1 + 0.6027093f, 0.5083023f, 0.44950223f, + 0.5754369f, 0.45485455f, 0.3747841f, + + // reverse output for input sequence 1 [sequence 0 in reversed input] + 0.26894143f, 0.11920292f, 0.04742587f, + 0.24973989f, 0.09975048f, 0.03557118f}; + + std::vector Y_h_data{ + // we did the forward processing of input[1] last + 0.6027093f, 0.5083023f, 0.44950223f, + 0.5754369f, 0.45485455f, 0.3747841f, + + // and the reverse processing of input[0] last as the input order was reversed + 0.6082785f, 0.50623393f, 0.4426924f, + 0.5803454f, 0.4527356f, 0.36886263f}; + + bool run_on_gpu = true; + DefaultActivationsSimpleWeightsNoBias(run_on_gpu, "bidirectional", Y_data, Y_h_data); +} + +void DefaultActivationsSimpleWeightsWithBias(bool run_on_gpu, + std::string direction, + const std::vector& Y_data, + bool linear_before_reset = false, + bool one_row = false) { + int64_t seq_length = 2; + int batch_size = one_row ? 1 : 2; // if 2 take batch_parallel_ path. if 1, don't. + int64_t input_size = 1; + int64_t hidden_size = 3; + + int num_directions = direction == "bidirectional" ? 2 : 1; + + std::vector X_data; + + if (batch_size == 2) + X_data = {-0.1f, 0.2f, -0.3f, 0.4f}; + else + X_data = {-0.1f, -0.3f}; + + std::vector W_data{0.1f, 0.2f, 0.3f, // wz + 0.2f, 0.3f, 0.1f, // wr + 0.3f, 0.1f, 0.2f}; // wh + + std::vector B_data{ + -0.01f, 0.1f, 0.01f, // Wb[zrh] + -0.2f, -0.02f, 0.02f, + 0.3f, -0.3f, -0.3f, + + -0.03f, 0.5f, -0.7f, // Rb[zrh] + 0.05f, -0.7f, 0.3f, + 0.07f, -0.03f, 0.5f}; + + // duplicate for bidirectional + auto duplicate_data = [](std::vector& data) { + data.reserve(data.size() * 2); // need to avoid reallocation when inserting + std::copy(data.cbegin(), data.cend(), std::back_inserter(data)); + }; + + if (num_directions == 2) { + duplicate_data(W_data); + duplicate_data(B_data); + } + + std::vector R_data(num_directions * 3 * hidden_size * hidden_size, 0.1f); + + RunGruTest(run_on_gpu, X_data, W_data, R_data, Y_data, {}, input_size, batch_size, hidden_size, seq_length, + &B_data, nullptr, nullptr, direction, 999.f, /* output_sequence*/ true, linear_before_reset); +} // namespace test + +TEST(GRUTest, ForwardDefaultActivationsSimpleWeightsWithBiasBatchParallel) { + std::vector Y_data{ + 0.16783132f, -0.11754231f, 0.11977843f, + 0.2046872f, -0.10372487f, 0.15365849f, + + 0.22688604f, -0.19698407f, 0.14017843f, + 0.33386092f, -0.15799662f, 0.2381169f}; + + bool run_on_gpu = false; + DefaultActivationsSimpleWeightsWithBias(run_on_gpu, "forward", Y_data); +} + +TEST(GRUTest, ForwardDefaultActivationsSimpleWeightsWithBiasBatchParallelLinearBeforeReset) { + std::vector Y_data{ + 0.15024948f, -0.11097029f, -0.02121867f, + 0.18887489f, -0.09747667f, 0.02093463f, + + 0.19538902f, -0.19016478f, -0.05644283f, + 0.30856851f, -0.15190377f, 0.05999807f}; + + bool run_on_gpu = true; + const bool linear_before_reset = true; + DefaultActivationsSimpleWeightsWithBias(run_on_gpu, "forward", Y_data, linear_before_reset); +} + +TEST(GRUTest, ReverseDefaultActivationsSimpleWeightsWithBiasBatchParallelLinearBeforeReset) { + std::vector Y_data{ + 0.20910699f, -0.18880953f, -0.04005555f, + 0.29700265f, -0.15308119f, 0.04537245f, + + 0.12252139f, -0.12032216f, -0.05064924f, + 0.21249877f, -0.08884402f, 0.04751285f}; + + bool run_on_gpu = false; + const bool linear_before_reset = true; + DefaultActivationsSimpleWeightsWithBias(run_on_gpu, "reverse", Y_data, linear_before_reset); +} + +// test forward !batch_parallel_ path with linear_before_reset +TEST(GRUTest, ForwardDefaultActivationsSimpleWeightsWithBiasLinearBeforeReset) { + std::vector Y_data{ + 0.15024948f, -0.11097029f, -0.02121867f, + 0.19538902f, -0.19016478f, -0.05644283f}; + + bool run_on_gpu = true; + const bool linear_before_reset = true; + const bool one_row = true; + DefaultActivationsSimpleWeightsWithBias(run_on_gpu, "forward", Y_data, linear_before_reset, one_row); +} + +// test reverse !batch_parallel_ path with linear_before_reset +TEST(GRUTest, ReverseDefaultActivationsSimpleWeightsWithBiasLinearBeforeReset) { + std::vector Y_data{ + 0.20910699f, -0.18880953f, -0.04005555f, + 0.12252139f, -0.12032216f, -0.05064924f}; + + bool run_on_gpu = false; + const bool linear_before_reset = true; + const bool one_row = true; + DefaultActivationsSimpleWeightsWithBias(run_on_gpu, "reverse", Y_data, linear_before_reset, one_row); +} + +/******************* +* Tests from ONNXRuntime +*/ +class DeepCpuGruOpTestContext { + public: + DeepCpuGruOpTestContext(std::string direction, + const std::vector& activations, + const bool use_bias = true, + const std::vector& alpha = {}, + const std::vector& beta = {}, + bool large_hidden = false, + int input_size = 2); + + ~DeepCpuGruOpTestContext() = default; + + void RunTest(bool run_on_gpu, + const std::vector& X, + const int batch, + const int seq_length, + const std::vector& sequence_length, + const std::vector* initial_h, + const std::vector& expected_Y, + const std::vector& expected_Y_h); + + private: + const int input_size_; + const int hidden_dim_; + const bool use_bias_; + const std::string direction_; + int num_directions_; + const std::vector activation_func_names_; + const std::vector alphas_; + const std::vector betas_; + std::vector gru_input_weights_; + std::vector gru_recurrent_weights_; + std::vector gru_bias_; +}; // namespace test + +DeepCpuGruOpTestContext::DeepCpuGruOpTestContext(const std::string direction, + const std::vector& activations, + const bool use_bias, + const std::vector& alpha, + const std::vector& beta, + bool large_hidden, + int input_size) + : input_size_(input_size), + hidden_dim_(large_hidden ? 32 : 2), + use_bias_(use_bias), + direction_(direction), + activation_func_names_(activations), + alphas_(alpha), + betas_(beta) { + if (direction == "bidirectional") + num_directions_ = 2; + else + num_directions_ = 1; + + if (large_hidden) { + const int input_weight_block_size = input_size_ * hidden_dim_; + const int hidden_weight_block_size = hidden_dim_ * hidden_dim_; + std::vector gru_input_weights(3 * input_weight_block_size); + std::vector gru_hidden_weights(3 * hidden_weight_block_size); + std::vector gru_bias(6 * hidden_dim_); + + // Construction of input weights. + for (int i = 0; i < input_weight_block_size; i++) { + gru_input_weights[i] = 0.1f; + gru_input_weights[i + input_weight_block_size] = 0.2f; + gru_input_weights[i + 2 * input_weight_block_size] = 0.3f; + } + + for (int i = 0; i < hidden_dim_; i++) { + int diag_index = i * hidden_dim_ + i; + gru_hidden_weights[diag_index] = 0.1f + 0.1f * (i % 2); + gru_hidden_weights[diag_index + hidden_weight_block_size] = 0.1f + 0.1f * (i % 3); + gru_hidden_weights[diag_index + 2 * hidden_weight_block_size] = 0.1f + 0.1f * (i % 5); + } + + for (int i = 0; i < hidden_dim_; i++) { + gru_bias[i] = 0.1f; + gru_bias[i + hidden_dim_] = 0.2f; + gru_bias[i + 2 * hidden_dim_] = 0.3f; + } + + for (int i = 0; i < 3 * hidden_dim_; i++) { + gru_bias[i + 3 * hidden_dim_] = 0.0f; + } + + for (int i = 0; i < num_directions_; i++) { + gru_input_weights_.insert(gru_input_weights_.end(), gru_input_weights.begin(), gru_input_weights.end()); + gru_recurrent_weights_.insert(gru_recurrent_weights_.end(), gru_hidden_weights.begin(), gru_hidden_weights.end()); + gru_bias_.insert(gru_bias_.end(), gru_bias.begin(), gru_bias.end()); + } + + } else { + if (num_directions_ == 2) { + // kBidirectional weights. + gru_input_weights_ = { + -0.494659f, 0.0453352f, -0.487793f, 0.417264f, // Wz + -0.0091708f, -0.255364f, -0.106952f, -0.266717f, // Wr + -0.0888852f, -0.428709f, -0.283349f, 0.208792f, // Wh + -0.494659f, 0.0453352f, -0.487793f, 0.417264f, // WBz + -0.0091708f, -0.255364f, -0.106952f, -0.266717f, // WBr + -0.0888852f, -0.428709f, -0.283349f, 0.208792f}; // WBh + + gru_recurrent_weights_ = { + 0.146626f, -0.0620289f, -0.0815302f, 0.100482f, // Rz + -0.228172f, 0.405972f, 0.31576f, 0.281487f, // Rr + -0.394864f, 0.42111f, -0.386624f, -0.390225f, // Rh + 0.146626f, -0.0620289f, -0.0815302f, 0.100482f, // RBz + -0.228172f, 0.405972f, 0.31576f, 0.281487f, // RBr + -0.394864f, 0.42111f, -0.386624f, -0.390225f}; // RBh + + gru_bias_ = { + 0.381619f, 0.0323954f, // Wbz + -0.258721f, 0.45056f, // Wbr + -0.250755f, 0.0967895f, // Wbh + 0.0f, 0.0f, // Rbz + -0.0f, 0.0f, // Rbr + -0.0f, 0.0f, // Rbh + 0.381619f, 0.0323954f, // WBbz + -0.258721f, 0.45056f, // WBbr + -0.250755f, 0.0967895f, // WBbh + 0.0f, 0.0f, // RBbz + -0.0f, 0.0f, // RBbr + -0.0f, 0.0f}; // RBbh + } else { + // Unidirectional weights. + gru_input_weights_ = { + -0.494659f, 0.0453352f, -0.487793f, 0.417264f, // Wz + -0.0091708f, -0.255364f, -0.106952f, -0.266717f, // Wr + -0.0888852f, -0.428709f, -0.283349f, 0.208792f // Wh + }; + + gru_recurrent_weights_ = { + 0.146626f, -0.0620289f, -0.0815302f, 0.100482f, // Rz + -0.228172f, 0.405972f, 0.31576f, 0.281487f, // Rr + -0.394864f, 0.42111f, -0.386624f, -0.390225f}; // Rh + + gru_bias_ = { + 0.381619f, 0.0323954f, // Wbz + -0.258721f, 0.45056f, // Wbr + -0.250755f, 0.0967895f, // Wbh + 0.0f, 0.0f, // Rbz + -0.0f, 0.0f, // Rbr + -0.0f, 0.0f // Rbh + }; + } + } +} + +void DeepCpuGruOpTestContext::RunTest(bool run_on_gpu, + const std::vector& X, + const int batch_size, + const int seq_length, + const std::vector& sequence_lens, + const std::vector* initial_h, + const std::vector& expected_Y, + const std::vector& expected_Y_h) { + // run with and without output_sequence + ::onnxruntime::test::RunGruTest(run_on_gpu, X, gru_input_weights_, gru_recurrent_weights_, + expected_Y, expected_Y_h, + input_size_, batch_size, hidden_dim_, seq_length, + use_bias_ ? &gru_bias_ : nullptr, + initial_h, + &sequence_lens, + direction_, + 9999999999.f, + /*output_sequence*/ true, + false, + activation_func_names_, + alphas_, + betas_); + + ::onnxruntime::test::RunGruTest(run_on_gpu, X, gru_input_weights_, gru_recurrent_weights_, + expected_Y, expected_Y_h, + input_size_, batch_size, hidden_dim_, seq_length, + use_bias_ ? &gru_bias_ : nullptr, + initial_h, + &sequence_lens, + direction_, + 9999999999.f, + /*output_sequence*/ false, + false, + activation_func_names_, + alphas_, + betas_); +} + +TEST(GRUTest, ONNXRuntime_TestGRUOpForwardBasic) { + const std::string direction = "forward"; + const std::vector activations = {"sigmoid", "tanh"}; + + DeepCpuGruOpTestContext ctx(direction, activations); + + const int batch = 1; + const int seq_length = 2; + std::vector X = {-0.455351f, -0.276391f, + -0.185934f, -0.269585f}; + std::vector sequence_length = {2}; + std::vector initial_h = {0.0f, 0.0f}; + std::vector expected_Y = {-0.03255286f, 0.0774838f, -0.05556786f, 0.0785508f}; + std::vector expected_Y_h = {-0.05556786f, 0.0785508f}; + + bool run_on_gpu = true; + ctx.RunTest(run_on_gpu, X, batch, seq_length, sequence_length, &initial_h, expected_Y, expected_Y_h); +} + +TEST(GRUTest, ONNXRuntime_TestGRUOpBackwardBasic) { + const std::string direction = "reverse"; + const std::vector activations = {"sigmoid", "tanh"}; + + DeepCpuGruOpTestContext ctx(direction, activations); + + const int batch_size = 1; + const int seq_length = 2; + std::vector X = {-0.185934f, -0.269585f, + -0.455351f, -0.276391f}; + std::vector sequence_length = {2}; + std::vector initial_h = {0.0f, 0.0f}; + std::vector expected_Y = {-0.05556786f, 0.0785508f, + -0.03255286f, 0.0774838f}; + std::vector expected_Y_h = {-0.05556786f, 0.0785508f}; + + bool run_on_gpu = true; + ctx.RunTest(run_on_gpu, X, batch_size, seq_length, sequence_length, &initial_h, expected_Y, expected_Y_h); +} + +TEST(GRUTest, ONNXRuntime_TestGRUOpBidirectionalBasic) { + const std::string direction = "bidirectional"; + const std::vector activations = {"sigmoid", "tanh", "sigmoid", "tanh"}; + + DeepCpuGruOpTestContext ctx(direction, activations); + + const int batch_size = 1; + const int seq_length = 2; + std::vector X = {-0.455351f, -0.276391f, + -0.185934f, -0.269585f}; + std::vector sequence_length = {2}; + std::vector initial_h = {0.0f, 0.0f, 0.0f, 0.0f}; + std::vector expected_Y = {-0.03255286f, 0.0774838f, + -0.05469977f, 0.1004222f, + + -0.05556786f, 0.0785508f, + -0.04566499f, 0.04621252f}; + std::vector expected_Y_h = {-0.05556786f, 0.0785508f, + -0.05469977f, 0.1004222f}; + + bool run_on_gpu = true; + ctx.RunTest(run_on_gpu, X, batch_size, seq_length, sequence_length, &initial_h, expected_Y, expected_Y_h); +} + +TEST(GRUTest, ONNXRuntime_TestGRUOpForwardActivation) { + const std::string direction = "forward"; + const std::vector activations = {"tanh", "sigmoid"}; + + DeepCpuGruOpTestContext ctx(direction, activations); + + const int batch_size = 1; + const int seq_length = 2; + std::vector X = {-0.455351f, -0.276391f, + -0.185934f, -0.269585f}; + std::vector sequence_length = {2}; + std::vector initial_h = {0.0f, 0.0f}; + std::vector expected_Y = {0.222789f, 0.4669829f, + 0.3810334f, 0.4944591f}; + std::vector expected_Y_h = {0.3810334f, 0.4944591f}; + + bool run_on_gpu = false; // cudnn only support activation {sigmoid, tanh} + ctx.RunTest(run_on_gpu, X, batch_size, seq_length, sequence_length, &initial_h, expected_Y, expected_Y_h); +} + +TEST(GRUTest, ONNXRuntime_TestGRUOpForwardInitialHiddenState) { + const std::string direction = "forward"; + const std::vector activations = {"sigmoid", "tanh"}; + + DeepCpuGruOpTestContext ctx(direction, activations); + + const int batch_size = 1; + const int seq_length = 2; + std::vector X = {-0.455351f, -0.276391f, + -0.185934f, -0.269585f}; + std::vector sequence_length = {2}; + std::vector initial_h = {0.5f, -0.5f}; + std::vector expected_Y = {0.2366661f, -0.1500429f, + 0.07378622f, -0.02782359f}; + std::vector expected_Y_h = {0.07378622f, -0.02782359f}; + + bool run_on_gpu = false; // cudnn implementation only support linear_before_reset = true + ctx.RunTest(run_on_gpu, X, batch_size, seq_length, sequence_length, &initial_h, expected_Y, expected_Y_h); +} + +TEST(GRUTest, ONNXRuntime_TestGRUOpForwardBatch) { + const std::string direction = "forward"; + const std::vector activations = {"sigmoid", "tanh"}; + + DeepCpuGruOpTestContext ctx(direction, activations); + + const int batch_size = 2; + const int seq_length = 2; + std::vector X = {-0.455351f, -0.276391f, + -0.455351f, -0.276391f, + + -0.185934f, -0.269585f, + -0.185934f, -0.269585f}; + std::vector sequence_length = {2, 2}; + std::vector initial_h = {0.5f, -0.5f, 0.0f, 0.0f}; + std::vector expected_Y = {0.2366661f, -0.1500429f, + -0.03255286f, 0.0774838f, + + 0.07378622f, -0.02782359f, + -0.05556786f, 0.0785508f}; + + std::vector expected_Y_h = {0.07378622f, -0.02782359f, + -0.05556786f, 0.0785508f}; + + bool run_on_gpu = false; // cudnn implementation only support linear_before_reset = true + ctx.RunTest(run_on_gpu, X, batch_size, seq_length, sequence_length, &initial_h, expected_Y, expected_Y_h); +} + +TEST(GRUTest, ONNXRuntime_TestGRUOpGrowBatchSequenceLength) { + const std::string direction = "forward"; + const std::vector activations = {"sigmoid", "tanh"}; + + DeepCpuGruOpTestContext ctx(direction, activations); + + const int batch_size = 1; + const int seq_length = 2; + std::vector X = {-0.455351f, -0.276391f, + -0.185934f, -0.269585f}; + std::vector sequence_length = {2}; + std::vector initial_h = {0.0f, 0.0f}; + std::vector expected_Y = {-0.03255286f, 0.0774838f, + -0.05556786f, 0.0785508f}; + std::vector expected_Y_h = {-0.05556786f, 0.0785508f}; + + bool run_on_gpu = false; + ctx.RunTest(run_on_gpu, X, batch_size, seq_length, sequence_length, &initial_h, expected_Y, expected_Y_h); + + const int batch2 = 2; + const int seq_length2 = 2; + std::vector X2 = {-0.455351f, -0.276391f, + -0.455351f, -0.276391f, + + -0.185934f, -0.269585f, + 0.0f, 0.0f}; + std::vector sequence_length2 = {2, 1}; + std::vector initial_h2 = {0.5f, -0.5f, + 0.0f, 0.0f}; + std::vector expected_Y2 = {0.2366661f, -0.1500429f, + -0.03255286f, 0.0774838f, + + 0.07378622f, -0.02782359f, + 0.0f, 0.0f}; + + std::vector expected_Y_h2 = {0.07378622f, -0.02782359f, + -0.03255286f, 0.0774838f}; + + ctx.RunTest(run_on_gpu, X2, batch2, seq_length2, sequence_length2, &initial_h2, expected_Y2, expected_Y_h2); +} + +TEST(GRUTest, ONNXRuntime_TestGRUOpSingleBatchMultipleHiddenThreads) { + const std::string direction = "forward"; + const std::vector activations = {"sigmoid", "tanh"}; + + DeepCpuGruOpTestContext ctx(direction, activations, true, {}, {}, /*large_hidden*/ true); + + const int batch_size = 1; + const int seq_length = 1; + std::vector X = {0.1f, -0.2f}; + std::vector sequence_length = {1}; + std::vector initial_h(32); + for (int i = 0; i < 32; i++) { + initial_h[i] = 0.5f; + } + + std::vector expected_Y = + { + 0.40203814648622f, 0.416614999456787f, 0.426893838272102f, 0.438425099258723f, + 0.449074949310697f, 0.405161353080481f, 0.41381287883561f, 0.428113160854675f, + 0.438710576166608f, 0.449253502147958f, 0.402300128581669f, 0.417112500336769f, + 0.425382986540999f, 0.439390099095881f, 0.450276939756071f, 0.404653232879823f, + 0.414327989766397f, 0.428845403314675f, 0.436736277602997f, 0.45043439079097f, + 0.402560202956173f, 0.416113639384501f, 0.426141512516655f, 0.44034669402871f, + 0.447861672303443f, 0.40490822137737f, 0.414839135658386f, 0.42737488368901f, + 0.437727744598091f, 0.451604294166264f, 0.40203814648622f, 0.416614999456787f}; + std::vector expected_Y_h(expected_Y); + + bool run_on_gpu = true; + ctx.RunTest(run_on_gpu, X, batch_size, seq_length, sequence_length, &initial_h, expected_Y, expected_Y_h); +} + +TEST(GRUTest, ONNXRuntime_TestGRUPositiveActivationClipping) { + const std::string direction = "forward"; + const std::vector activations = {"sigmoid", "tanh"}; + + DeepCpuGruOpTestContext ctx(direction, activations, true, {}, {}, /*large_hidden*/ true); + + const int batch_size = 2; + const int seq_length = 1; + std::vector X = {1000.0f, 2000.0f, -1e+20f, -1e+10f}; + std::vector sequence_length = {1, 1}; + std::vector initial_h(64); + for (int i = 0; i < 64; i++) { + initial_h[i] = 0.25f; + } + + std::vector expected_Y(64); + for (int i = 0; i < 32; i++) { + expected_Y[i] = 0.25f; + } + + for (int i = 32; i < 64; i++) { + expected_Y[i] = -1.0f; + } + + std::vector expected_Y_h(expected_Y); + + bool run_on_gpu = true; + ctx.RunTest(run_on_gpu, X, batch_size, seq_length, sequence_length, &initial_h, expected_Y, expected_Y_h); +} + +TEST(GRUTest, ONNXRuntime_TestGRUPositiveActivationAlphaBeta) { + const std::string direction = "bidirectional"; + const std::vector activations = {"LeakyRelu", "Tanh", "Sigmoid", "ScaledTanh"}; + const std::vector alpha = {0.5f, 2.0f}; + const std::vector beta = {2.0f}; + + const int input_size = 2; // 4; + const int batch_size = 1; + const int seq_length = 1; + std::vector X = {1.0f, 2.0f}; // , -3.0f, -4.0f}; + + std::vector sequence_length = {1}; + std::vector initial_h(64); + for (int i = 0; i < 64; i++) { + initial_h[i] = 0.25f; + } + + std::vector expected_Y = + { + 0.589157104133446f, 0.57789190635582f, 0.596499289838044f, 0.583932266186645f, + 0.602660999282266f, 0.574613517108484f, 0.592591742002174f, 0.581152059107645f, + 0.599870466365438f, 0.58688901119379f, 0.589262946094602f, 0.578086950802852f, + 0.595908617320326f, 0.584296265507284f, 0.603117381311498f, 0.574411143084165f, + 0.592796083917448f, 0.581434103323347f, 0.599111005558765f, 0.58732791235695f, + 0.589368676977052f, 0.577696448871644f, 0.59620442543072f, 0.584658706958245f, + 0.602202148066235f, 0.574512383220923f, 0.592999994021163f, 0.580869112219442f, + 0.59949155030307f, 0.587764451689259f, 0.589157104133446f, 0.57789190635582f, + 0.92976461079108f, 0.920318863429123f, 0.931350963541001f, 0.921736050322775f, + 0.932696544850119f, 0.91952832256688f, 0.930568024045208f, 0.921062381128771f, + 0.93205941179011f, 0.922380156527733f, 0.929770910118088f, 0.920330320081968f, + 0.931318215990949f, 0.921756237467303f, 0.932720157970373f, 0.919515978936628f, + 0.930579785457139f, 0.921078414664293f, 0.932018665973048f, 0.922403708998701f, + 0.929777143371747f, 0.920307279340899f, 0.931334684571751f, 0.921776181825893f, + 0.932672631849138f, 0.919522183290344f, 0.930591417482971f, 0.921046160840421f, + 0.932039162132774f, 0.922426966110166f, 0.92976461079108f, 0.920318863429123f}; + + std::vector expected_Y_h(expected_Y); + + bool run_on_gpu = false; // cudnn implementation don't support the alpha & beta and customized activations + DeepCpuGruOpTestContext ctx(direction, activations, true, alpha, beta, /*large_hidden*/ true, input_size); + ctx.RunTest(run_on_gpu, X, batch_size, seq_length, sequence_length, &initial_h, expected_Y, expected_Y_h); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/rnn/deep_cpu_lstm_op_test.cc b/onnxruntime/test/providers/cpu/rnn/deep_cpu_lstm_op_test.cc new file mode 100644 index 0000000000000..e90baafdae106 --- /dev/null +++ b/onnxruntime/test/providers/cpu/rnn/deep_cpu_lstm_op_test.cc @@ -0,0 +1,1000 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" + +#include +#include + +#include "core/providers/cpu/rnn/deep_cpu_lstm.h" +#include "test/providers/provider_test_utils.h" +using namespace std; +namespace onnxruntime { +namespace test { + +// copy the contents of the container to the end so the original values are duplicated +template +T DuplicateContainer(const T& container) { + T doubled; + doubled.reserve(container.size() * 2); // need to avoid reallocation when inserting + std::copy(container.cbegin(), container.cend(), std::back_inserter(doubled)); + std::copy(container.cbegin(), container.cend(), std::back_inserter(doubled)); + + return doubled; +} + +static void RunLstmTest(bool run_on_gpu, + const std::vector& X_data, + const std::vector& W_data, + const std::vector& R_data, + const std::vector& Y_data, + const std::vector& Y_h_data, + const std::vector& Y_c_data, + int64_t input_size, + int batch_size, + int64_t hidden_size, + int64_t seq_length, + const std::vector* B_data = nullptr, + const std::vector* P_data = nullptr, + const std::vector* initial_h_data = nullptr, + const std::vector* initial_c_data = nullptr, + const std::vector* sequence_lengths = nullptr, + const std::string& direction = "forward", + float clip = 9999.f, + bool output_sequence = true, + bool input_forget = false, + // copy the following vectors as we may modify them + std::vector activations = {}, + std::vector activation_alphas = {}, + std::vector activation_betas = {}) { + OpTester test("LSTM"); + + int num_directions = (direction == "bidirectional") ? 2 : 1; + + if (activations.empty()) { + activations = {"sigmoid", "tanh", "tanh"}; + } + + if (num_directions == 2 && activations.size() == 3) { + activations = DuplicateContainer(activations); + } + + test.AddAttribute>("activations", activations); + if (!activation_alphas.empty()) + test.AddAttribute>("activation_alpha", activation_alphas); + if (!activation_betas.empty()) + test.AddAttribute>("activation_beta", activation_betas); + + test.AddAttribute("direction", direction); + test.AddAttribute("hidden_size", hidden_size); + // test.AddAttribute("output_sequence", output_sequence); + test.AddAttribute("input_forget", input_forget); + test.AddAttribute("clip", clip); + + std::vector X_dims = {seq_length, batch_size, input_size}; + std::vector W_dims = {num_directions, 4 * hidden_size, input_size}; + std::vector R_dims = {num_directions, 4 * hidden_size, hidden_size}; + + test.AddInput("X", X_dims, X_data); + test.AddInput("W", W_dims, W_data); + test.AddInput("R", R_dims, R_data); + + if (B_data) { + std::vector B_dims = {num_directions, 8 * hidden_size}; + test.AddInput("B", B_dims, *B_data); + } else { + test.AddMissingOptionalInput(); + } + + if (sequence_lengths) { + std::vector sequence_lens_dims{batch_size}; + test.AddInput("sequence_lens", sequence_lens_dims, *sequence_lengths); + } else { + test.AddMissingOptionalInput(); + } + + if (initial_h_data && !initial_h_data->empty()) { + std::vector initial_h_dims = {num_directions, batch_size, hidden_size}; + test.AddInput("initial_h", initial_h_dims, *initial_h_data); + } else { + test.AddMissingOptionalInput(); + } + + if (initial_c_data && !initial_c_data->empty()) { + std::vector initial_c_dims = {num_directions, batch_size, hidden_size}; + test.AddInput("initial_c", initial_c_dims, *initial_c_data); + } else { + test.AddMissingOptionalInput(); + } + + if (P_data && !P_data->empty()) { + std::vector P_dims = {num_directions, 3 * hidden_size}; + test.AddInput("P", P_dims, *P_data); + } else { + test.AddMissingOptionalInput(); + } + + if (output_sequence != 0 && !Y_data.empty()) { + std::vector Y_dims = {seq_length, num_directions, batch_size, hidden_size}; + test.AddOutput("Y", Y_dims, Y_data); + } else { + // add placeholder so node counts match as Y_h will always be the second Y_data, + // so Y must exist as the first Y_data + test.AddMissingOptionalOutput(); + } + + if (!Y_h_data.empty()) { + std::vector Y_h_dims{num_directions, batch_size, hidden_size}; + test.AddOutput("Y_h", Y_h_dims, Y_h_data); + } else { + test.AddMissingOptionalOutput(); + } + + if (!Y_c_data.empty()) { + std::vector Y_c_dims{num_directions, batch_size, hidden_size}; + test.AddOutput("Y_c", Y_c_dims, Y_c_data); + } else { + test.AddMissingOptionalOutput(); + } + + std::unordered_set excluded_providers; + if (!run_on_gpu) { + excluded_providers.insert(kCudaExecutionProvider); + } + test.Run(OpTester::ExpectResult::kExpectSuccess, "", excluded_providers); +} + +void SimpleWeightsNoBiasTwoRows(bool run_on_gpu, + std::string direction, + const std::vector& Y_data, + const std::vector& Y_h_data, + const std::vector& Y_c_data, + const std::vector* seq_lengths = nullptr) { + int64_t seq_length = 2; + int batch_size = 2; + int64_t input_size = 1; + int64_t hidden_size = 3; + + int num_directions = direction == "bidirectional" ? 2 : 1; + + std::vector X_data{1.f, 2.f, 10.f, 11.f}; + + std::vector W_data{ + 0.1f, 0.2f, 0.3f, 0.4f, + 1.f, 2.f, 3.f, 4.f, + 10.f, 11.f, 12.f, 13.f}; + + std::vector R_data(num_directions * 4 * hidden_size * hidden_size, 0.1f); + + // duplicate for bidirectional + if (num_directions == 2) { + W_data = DuplicateContainer(W_data); + } + + RunLstmTest(run_on_gpu, X_data, W_data, R_data, Y_data, Y_h_data, Y_c_data, + input_size, batch_size, hidden_size, seq_length, + nullptr, nullptr, nullptr, nullptr, seq_lengths, direction); + + // need at least one output, so we need Y_h or Y_c to be requested (non-empty output to compare against) in order + // to test Y not being returned (output_sequence == false) + if (!Y_h_data.empty() || !Y_c_data.empty()) + RunLstmTest(run_on_gpu, X_data, W_data, R_data, Y_data, Y_h_data, Y_c_data, + input_size, batch_size, hidden_size, seq_length, + nullptr, nullptr, nullptr, nullptr, seq_lengths, direction, 999.f, /* output_sequence*/ false); +} + +TEST(LSTMTest, ForwardSimpleWeightsNoBiasTwoRows) { + std::vector Y_data{ + 0.28828835f, 0.36581863f, 0.45679406f, + 0.34526032f, 0.47220859f, 0.55850911f, + + 0.84196719f, 0.89402526f, 0.91073048f, + 0.85882828f, 0.90703777f, 0.92382453f}; + + std::vector Y_h_data{ + 0.84196719f, 0.89402526f, 0.91073048f, + 0.85882828f, 0.90703777f, 0.92382453f}; + + std::vector Y_c_data{ + 1.27731147f, 1.44181041f, 1.53179041f, + 1.3249796f, 1.51063104f, 1.61451544f}; + + bool run_on_gpu = true; + SimpleWeightsNoBiasTwoRows(run_on_gpu, "forward", Y_data, Y_h_data, Y_c_data); + + // test Y_h and Y_c being optional + SimpleWeightsNoBiasTwoRows(run_on_gpu, "forward", Y_data, {}, {}); +} + +TEST(LSTMTest, ReverseSimpleWeightsNoBiasTwoRows) { + std::vector Y_data{ + 0.55391603f, 0.69201493f, 0.82696019f, + 0.64046413f, 0.82303363f, 0.91610711f, + + 0.61249432f, 0.70678632f, 0.74094619f, + 0.62759886f, 0.71640738f, 0.74624585f}; + + std::vector Y_h_data{ + 0.55391603f, 0.69201493f, 0.82696019f, + 0.64046413f, 0.82303363f, 0.91610711f}; + + std::vector Y_c_data{ + 1.27850552f, 1.46799496f, 1.57641257f, + 1.34960834f, 1.54772296f, 1.65633056f}; + + bool run_on_gpu = true; + SimpleWeightsNoBiasTwoRows(run_on_gpu, "reverse", Y_data, Y_h_data, Y_c_data); +} + +TEST(LSTMTest, BidirectionalSimpleWeightsNoBiasTwoRows) { + std::vector Y_data{ + 0.28828835f, 0.36581863f, 0.45679406f, + 0.34526032f, 0.47220859f, 0.55850911f, + + 0.55391603f, 0.69201493f, 0.82696019f, + 0.64046413f, 0.82303363f, 0.91610711f, + + 0.84196719f, 0.89402526f, 0.91073048f, + 0.85882828f, 0.90703777f, 0.92382453f, + + 0.61249432f, 0.70678632f, 0.74094619f, + 0.62759886f, 0.71640738f, 0.74624585f}; + + std::vector Y_h_data{ + // we did the forward processing of X_data[1] last + 0.84196719f, 0.89402526f, 0.91073048f, + 0.85882828f, 0.90703777f, 0.92382453f, + + // and the reverse processing of X_data[0] last as the X_data order was reversed + 0.55391603f, 0.69201493f, 0.82696019f, + 0.64046413f, 0.82303363f, 0.91610711f}; + + std::vector Y_c_data{ + 1.27731147f, 1.44181041f, 1.53179041f, + 1.3249796f, 1.51063104f, 1.61451544f, + + 1.27850552f, 1.46799496f, 1.57641257f, + 1.34960834f, 1.54772296f, 1.65633056f}; + + // cudnn don't support customized activation + bool run_on_gpu = true; + SimpleWeightsNoBiasTwoRows(run_on_gpu, "bidirectional", Y_data, Y_h_data, Y_c_data); +} + +TEST(LSTMTest, MixedSequenceLengths) { + // we don't have numpy output for this, but by testing twice and swapping which batch is smaller + // we can largely verify the behaviour by comparing to ForwardSimpleWeightsNoBiasTwoRows output. + std::vector seq_lengths{1, 2}; + + std::vector Y_data{ + 0.28828835f, 0.36581863f, 0.45679406f, + 0.34526032f, 0.47220859f, 0.55850911f, + + 0.f, 0.f, 0.f, + 0.85882828f, 0.90703777f, 0.92382453f}; + + std::vector Y_h_data{ + 0.28828835f, 0.36581863f, 0.45679406f, + 0.85882828f, 0.90703777f, 0.92382453f}; + + std::vector Y_c_data{ + 0.52497941f, 0.54983425f, 0.5744428f, // see intermediate output from ForwardSimpleWeightsNoBiasTwoRows + 1.3249796f, 1.51063104f, 1.61451544f}; + + // Not able to mask on Y_c for CUDA using cudnn lib + bool run_on_gpu = false; + SimpleWeightsNoBiasTwoRows(run_on_gpu, "forward", Y_data, Y_h_data, Y_c_data, &seq_lengths); + + // swap which one is short + seq_lengths = {2, 1}; + + Y_data = { + 0.28828835f, 0.36581863f, 0.45679406f, + 0.34526032f, 0.47220859f, 0.55850911f, + + 0.84196719f, 0.89402526f, 0.91073048f, + 0.f, 0.f, 0.f}; + + Y_h_data = { + 0.84196719f, 0.89402526f, 0.91073048f, + 0.34526032f, 0.47220859f, 0.55850911f}; + + Y_c_data = { + 1.27731147f, 1.44181041f, 1.53179041f, + 0.54983425f, 0.59868795f, 0.64565659f}; + + SimpleWeightsNoBiasTwoRows(run_on_gpu, "forward", Y_data, Y_h_data, Y_c_data, &seq_lengths); +} + +TEST(LSTMTest, MixedSequenceLengthsReverse) { + // we don't have numpy output for this, but by testing twice and swapping which batch is smaller + // we can largely verify the behaviour by comparing to ReverseSimpleWeightsNoBiasTwoRows output. + std::vector seq_lengths{1, 2}; + + std::vector Y_data{ + 0.28828844f, 0.36581877f, 0.45679423f, + 0.64046413f, 0.82303363f, 0.91610711f, + + 0.f, 0.f, 0.f, + 0.62759886f, 0.71640738f, 0.74624585f}; + + std::vector Y_h_data{ + 0.28828844f, 0.36581877f, 0.45679423f, + 0.64046413f, 0.82303363f, 0.91610711f}; + + std::vector Y_c_data{ + 0.52497941f, 0.54983425f, 0.5744428f, + 1.34960834f, 1.54772296f, 1.65633056f}; + + bool run_on_gpu = false; + SimpleWeightsNoBiasTwoRows(run_on_gpu, "reverse", Y_data, Y_h_data, Y_c_data, &seq_lengths); + + // swap which one is short + seq_lengths = {2, 1}; + + Y_data = { + 0.55391603f, 0.69201493f, 0.82696019f, + 0.34526044f, 0.47220877f, 0.55850935f, + + 0.61249432f, 0.70678632f, 0.74094619f, + 0.f, 0.f, 0.f}; + + Y_h_data = { + 0.55391603f, 0.69201493f, 0.82696019f, + 0.34526044f, 0.47220877f, 0.55850935f}; + + Y_c_data = { + 1.27850552f, 1.46799496f, 1.57641257f, + 0.54983425f, 0.59868795f, 0.64565659f}; + + SimpleWeightsNoBiasTwoRows(run_on_gpu, "reverse", Y_data, Y_h_data, Y_c_data, &seq_lengths); +} + +// test path in LSTM model where batch_parallel_ is false and there are multiple steps (seq_length > 1) +TEST(LSTMTest, BatchParallelFalseSeqLengthGreaterThanOne) { + int64_t seq_length = 2; + int batch_size = 1; + int64_t input_size = 1; + int64_t hidden_size = 2; + + int num_directions = 1; + + std::vector X_data{1.f, 2.f}; + + std::vector W_data{ + 0.1f, 0.2f, 0.3f, 0.4f, + 1.f, 2.f, 3.f, 4.f}; + + std::vector R_data(num_directions * 4 * hidden_size * hidden_size, 0.1f); + + std::vector Y_data{ + 0.27546653f, 0.29941525f, + 0.50903179f, 0.57476457f}; + + std::vector Y_c_data{ + 1.02721067f, 1.15254318f}; + + bool run_on_gpu = true; + RunLstmTest(run_on_gpu, X_data, W_data, R_data, Y_data, {}, Y_c_data, + input_size, batch_size, hidden_size, seq_length); +} + +// make sure GateComputations works correctly if batch_parallel_ is true due to large batch size +static void LargeBatchWithClip(bool run_on_gpu, const std::vector& Y_h_data, float clip = 9999.0) { + int64_t seq_length = 2; + int batch_size = 32; + int64_t input_size = 1; + int64_t hidden_size = 3; + + const std::string direction = "forward"; + int num_directions = 1; + + std::vector X_data; + + // generate input of 64 values + float i = 0.f, increment = 1.f; + std::generate_n(std::back_inserter(X_data), batch_size * seq_length, [&]() { return i += increment; }); + + std::vector W_data{0.1f, 0.2f, 0.3f, 0.4f, + 1.f, 2.f, 3.f, 4.f, + 10.f, 11.f, 12.f, 13.f}; + + std::vector R_data(num_directions * 4 * hidden_size * hidden_size, 0.1f); + + RunLstmTest(run_on_gpu, X_data, W_data, R_data, {}, Y_h_data, {}, + input_size, batch_size, hidden_size, seq_length, + nullptr, nullptr, nullptr, nullptr, nullptr, direction, clip); +} + +TEST(LSTMTest, LargeBatchNoClipping) { + std::vector Y_h_data = { + 0.90387899f, 0.9135572f, 0.91772245f, + 0.90897038f, 0.92132433f, 0.92825467f, + 0.91365823f, 0.92815113f, 0.93676105f, + 0.91799162f, 0.93406357f, 0.94344562f, + 0.92199681f, 0.93912057f, 0.94859476f, + 0.92569357f, 0.94340185f, 0.95250664f, + 0.92909964f, 0.94699686f, 0.95545127f, + 0.93223207f, 0.94999634f, 0.95765468f, + 0.93510761f, 0.9524867f, 0.95929726f, + 0.93774272f, 0.9545467f, 0.96051891f, + 0.9401536f, 0.95624603f, 0.96142619f, + 0.94235605f, 0.95764499f, 0.96209939f, + 0.94436539f, 0.95879495f, 0.96259862f, + 0.94619635f, 0.95973921f, 0.96296872f, + 0.94786299f, 0.96051397f, 0.96324302f, + 0.94937864f, 0.96114929f, 0.96344629f, + 0.95075587f, 0.96167006f, 0.96359692f, + 0.95200645f, 0.96209679f, 0.96370852f, + 0.95314133f, 0.9624464f, 0.9637912f, + 0.95417069f, 0.96273278f, 0.96385246f, + 0.95510395f, 0.96296733f, 0.96389785f, + 0.95594975f, 0.96315942f, 0.96393147f, + 0.95671607f, 0.96331673f, 0.96395638f, + 0.9574102f, 0.96344554f, 0.96397483f, + 0.9580388f, 0.96355102f, 0.9639885f, + 0.95860795f, 0.96363739f, 0.96399863f, + 0.95912322f, 0.96370811f, 0.96400613f, + 0.95958963f, 0.96376601f, 0.96401169f, + 0.96001179f, 0.96381342f, 0.96401581f, + 0.96039386f, 0.96385224f, 0.96401886f, + 0.96073964f, 0.96388402f, 0.96402112f, + 0.96105254f, 0.96391004f, 0.96402279f}; + + bool run_on_gpu = true; + LargeBatchWithClip(run_on_gpu, Y_h_data); +} + +// make sure GateComputations with clipping works correctly if batch_parallel_ is true due to large batch size +TEST(LSTMTest, LargeBatchWithClip) { + std::vector Y_h_data = { + 0.88572926f, 0.89251395f, 0.89655037f, + 0.89074291f, 0.90035688f, 0.90727429f, + 0.89535827f, 0.90727429f, 0.91596163f, + 0.89963124f, 0.91328279f, 0.9228067f, + 0.90358195f, 0.91843507f, 0.92809163f, + 0.90723279f, 0.9228067f, 0.93211437f, + 0.91038955f, 0.92648469f, 0.93514718f, + 0.91328279f, 0.92955856f, 0.93741938f, + 0.91596163f, 0.93211437f, 0.9391149f, + 0.91843507f, 0.93423112f, 0.94037686f, + 0.92071318f, 0.9359791f, 0.94131462f, + 0.9228067f, 0.93741938f, 0.94201073f, + 0.92472679f, 0.9386042f, 0.94252713f, + 0.92648469f, 0.9395777f, 0.94266769f, + 0.92809163f, 0.94037686f, 0.94266769f, + 0.92955856f, 0.94103248f, 0.94266769f, + 0.93089609f, 0.94157007f, 0.94266769f, + 0.93211437f, 0.94201073f, 0.94266769f, + 0.93322302f, 0.94237184f, 0.94266769f, + 0.93423112f, 0.94266769f, 0.94266769f, + 0.93514718f, 0.94266769f, 0.94266769f, + 0.9359791f, 0.94266769f, 0.94266769f, + 0.93673424f, 0.94266769f, 0.94266769f, + 0.93741938f, 0.94266769f, 0.94266769f, + 0.93804079f, 0.94266769f, 0.94266769f, + 0.9386042f, 0.94266769f, 0.94266769f, + 0.9391149f, 0.94266769f, 0.94266769f, + 0.9395777f, 0.94266769f, 0.94266769f, + 0.93999702f, 0.94266769f, 0.94266769f, + 0.94037686f, 0.94266769f, 0.94266769f, + 0.94072091f, 0.94266769f, 0.94266769f, + 0.94103248f, 0.94266769f, 0.94266769f}; + + bool run_on_gpu = false; + LargeBatchWithClip(run_on_gpu, Y_h_data, 4.f); +} + +// ONNXRuntime tests +class LstmOpContext2x1x2x2 { + public: + LstmOpContext2x1x2x2(const std::string direction, + const std::vector& activations = {}, + const std::vector& activation_alphas = {}, + const std::vector& activation_betas = {}) + : direction_(direction), + num_directions_(direction == "bidirectional" ? 2 : 1), + activation_func_names_{activations}, + activation_alphas_{activation_alphas}, + activation_betas_{activation_betas} { + // W[iofc] (4*hidden, X_data) + input_weights_ = { + -0.494659f, 0.0453352f, + -0.487793f, 0.417264f, + + -0.0175329f, 0.489074f, + -0.446013f, 0.414029f, + + -0.0091708f, -0.255364f, + -0.106952f, -0.266717f, + + -0.0888852f, -0.428709f, + -0.283349f, 0.208792f}; + + // R[iofc] (4*hidden, hidden) + recurrent_weights_ = { + 0.146626f, -0.0620289f, + -0.0815302f, 0.100482f, + + -0.219535f, -0.306635f, + -0.28515f, -0.314112f, + + -0.228172f, 0.405972f, + 0.31576f, 0.281487f, + + -0.394864f, 0.42111f, + -0.386624f, -0.390225f}; + + // P[iof] (3*hidden) + peephole_weights_ = { + 0.2345f, 0.5235f, + 0.4378f, 0.3475f, + 0.8927f, 0.3456f}; + + // Wb[iofc], Rb[iofc] (8*hidden) + bias_ = { + // Wb[iofc] + 0.381619f, 0.0323954f, + -0.14449f, 0.420804f, + -0.258721f, 0.45056f, + -0.250755f, 0.0967895f, + + // Rb[iofc] + 0.0f, 0.0f, + 0.0f, 0.0f, + 0.0f, 0.0f, + 0.0f, 0.0f}; + + if (num_directions_ == 2) { + input_weights_ = DuplicateContainer(input_weights_); + recurrent_weights_ = DuplicateContainer(recurrent_weights_); + peephole_weights_ = DuplicateContainer(peephole_weights_); + bias_ = DuplicateContainer(bias_); + } + + /* + + def.add_arg()->CopyFrom(caffe2::MakeArgument("direction", direction)); + def.add_arg()->CopyFrom(caffe2::MakeArgument("hidden_size", _hidden_dim)); + def.add_arg()->CopyFrom(caffe2::MakeArgument("activations", activations)); + def.add_arg()->CopyFrom(caffe2::MakeArgument("clip", clip)); + def.add_arg()->CopyFrom(caffe2::MakeArgument("input_forget", input_forget)); + def.add_arg()->CopyFrom(caffe2::MakeArgument("output_sequence", output_sequence)); + + FillTensor(&_ws, "X0", { seq_len, batch_size, _input_dim }, X_data); + + if (num_direction == 1) { + FillTensor(&_ws, "X1", { num_direction, 4 * _hidden_dim, _input_dim }, input_weights); + FillTensor(&_ws, "X2", { num_direction, 4 * _hidden_dim, _hidden_dim }, recurrent_weights); + + if (use_bias) + FillTensor(&_ws, "X3", { num_direction, 8 * _hidden_dim }, bias); + + if (seq_length.size()) + FillTensor(&_ws, "X4", { batch_size }, seq_length); + + if (hidden_state.size()) + FillTensor(&_ws, "X5", { num_direction, batch_size, _hidden_dim }, hidden_state); + + if (cell_state.size()) + FillTensor(&_ws, "X6", { num_direction, batch_size, _hidden_dim }, cell_state); + + if (use_peepholes) + FillTensor(&_ws, "X7", { num_direction, 3 * _hidden_dim }, peephole_weights); + } + if (num_direction == 2) { + FillTensor(&_ws, "X1", { num_direction, 4 * _hidden_dim, _input_dim }, bi_input_weights); + FillTensor(&_ws, "X2", { num_direction, 4 * _hidden_dim, _hidden_dim }, bi_recurrent_weights); + + if (use_bias) + FillTensor(&_ws, "X3", { num_direction, 8 * _hidden_dim }, bi_bias); + + if (seq_length.size()) + FillTensor(&_ws, "X4", { batch_size }, seq_length); + + if (hidden_state.size()) + FillTensor(&_ws, "X5", { num_direction, batch_size, _hidden_dim }, hidden_state); + + if (cell_state.size()) + FillTensor(&_ws, "X6", { num_direction, batch_size, _hidden_dim }, cell_state); + + if (use_peepholes) + FillTensor(&_ws, "X7", { num_direction, 3 * _hidden_dim }, bi_peephole_weights); + } + + _op = caffe2::CreateOperator(def, &_ws); + */ + + // RunTest(seq_len, batch_size, num_direction, Y_data, output_first); + } + + void RunTest(bool run_on_gpu, + const std::vector& X, + const int batch_size, + const int seq_length, + const std::vector* initial_h, + const std::vector* initial_c, + const std::vector& expected_Y, + const std::vector& expected_Y_h = {}, + const std::vector& expected_Y_c = {}, + const std::vector* sequence_lens = nullptr, + bool use_bias = true, + bool use_peepholes = true, + float clip = 9999.f, + bool input_forget = false) { + // run with and without output_sequence to test UniDirectionalLstm handling when Y isn't returned + ::onnxruntime::test::RunLstmTest(run_on_gpu, X, input_weights_, recurrent_weights_, + expected_Y, expected_Y_h, expected_Y_c, + input_size_, batch_size, hidden_size_, seq_length, + use_bias ? &bias_ : nullptr, + use_peepholes ? &peephole_weights_ : nullptr, + initial_h, initial_c, + sequence_lens, + direction_, + clip, + /*output_sequence*/ true, + input_forget, + activation_func_names_, + activation_alphas_, + activation_betas_); + + ::onnxruntime::test::RunLstmTest(run_on_gpu, X, input_weights_, recurrent_weights_, + expected_Y, expected_Y_h, expected_Y_c, + input_size_, batch_size, hidden_size_, seq_length, + use_bias ? &bias_ : nullptr, + use_peepholes ? &peephole_weights_ : nullptr, + initial_h, initial_c, + sequence_lens, + direction_, + clip, + /*output_sequence*/ false, + input_forget, + activation_func_names_, + activation_alphas_, + activation_betas_); + } + + private: + const int input_size_ = 2; + const int hidden_size_ = 2; + const std::string direction_; + int num_directions_; + const std::vector activation_func_names_; + const std::vector activation_alphas_; + const std::vector activation_betas_; + std::vector input_weights_; + std::vector recurrent_weights_; + std::vector bias_; + std::vector peephole_weights_; +}; + +TEST(LSTMTest, ONNXRuntime_TestLSTMForwardPeepHole) { + ///////////////Attributes//////////////////////// + const int seq_len = 2, batch_size = 1; + + std::vector input = {-0.455351f, -0.276391f, -0.185934f, -0.269585f}; + std::vector Y_data = {-0.0251062475f, 0.0561261699f, -0.03277518f, 0.05935364f}; + std::vector Y_h_data = {-0.03277518f, 0.05935364f}; + std::vector Y_c_data = {-0.0780206f, 0.098829f}; + + std::string direction = "forward"; + + //Run Test + LstmOpContext2x1x2x2 context(direction); + bool run_on_gpu = false; + context.RunTest(run_on_gpu, input, batch_size, seq_len, nullptr, nullptr, Y_data, Y_h_data, Y_c_data); +} + +TEST(LSTMTest, ONNXRuntime_TestLSTMBidirectionalBasic) { + const int seq_len = 2, batch_size = 1; + + std::vector X_data = {-0.455351f, -0.276391f, + -0.185934f, -0.269585f}; + std::vector Y_data = {-0.0251062f, 0.0561262f, + -0.0318928f, 0.0762679f, + -0.0327752f, 0.0593536f, + -0.0306872f, 0.028035f}; + std::vector Y_h_data = {-0.0327752f, 0.0593536f, + -0.0318928f, 0.0762679f}; + std::vector Y_c_data = {-0.0780206f, 0.098829f, + -0.0753684f, 0.120794f}; + + LstmOpContext2x1x2x2 context("bidirectional"); + bool run_on_gpu = false; + context.RunTest(run_on_gpu, X_data, batch_size, seq_len, nullptr, nullptr, Y_data, Y_h_data, Y_c_data); +} + +TEST(LSTMTest, ONNXRuntime_TestLSTMForwardNoBiasUsePeepholes) { + const int seq_len = 2, batch_size = 1; + + bool use_bias = false; + bool use_peepholes = true; + std::vector X_data = {-0.455351f, -0.276391f, + -0.185934f, -0.269585f}; + + std::vector Y_data = {0.04154162f, 0.01969122f, + 0.05298181f, 0.0030589f}; + std::vector Y_h_data = {0.05298181f, 0.0030589f}; + std::vector Y_c_data = {0.11169686f, 0.00625722f}; + + LstmOpContext2x1x2x2 context("forward"); + bool run_on_gpu = false; + context.RunTest(run_on_gpu, X_data, batch_size, seq_len, nullptr, nullptr, Y_data, Y_h_data, Y_c_data, nullptr, + use_bias, use_peepholes); +} + +TEST(LSTMTest, ONNXRuntime_TestLSTMForwardInputForget) { + const int seq_len = 2, batch_size = 1; + + bool use_bias = true; + bool use_peepholes = true; + bool input_forget = true; + float clip = 999.0f; + + std::vector X_data = {-0.455351f, -0.276391f, -0.185934f, -0.269585f}; + + std::vector Y_data = {-0.02510626f, 0.05612619f, + -0.0314321f, 0.05087372f}; + std::vector Y_h_data = {-0.0314321f, 0.05087372f}; + std::vector Y_c_data = {-0.07474898f, 0.08480116f}; + + LstmOpContext2x1x2x2 context("forward"); + // cudnn don't support peepholes + bool run_on_gpu = false; + context.RunTest(run_on_gpu, X_data, batch_size, seq_len, nullptr, nullptr, Y_data, Y_h_data, Y_c_data, nullptr, + use_bias, use_peepholes, clip, input_forget); +} + +TEST(LSTMTest, ONNXRuntime_TestLSTMForwardClip) { + const int seq_len = 2, batch_size = 1; + + bool use_bias = true; + bool use_peepholes = true; + float clip = 0.1f; + + std::vector X_data = {-0.455351f, -0.276391f, -0.185934f, -0.269585f}; + + std::vector Y_data = {-0.02280854f, 0.02744377f, + -0.03516197f, 0.03875681f}; + std::vector Y_h_data = {-0.03516197f, 0.03875681f}; + std::vector Y_c_data = {-0.07415761f, 0.07395997f}; + + LstmOpContext2x1x2x2 context("forward"); + bool run_on_gpu = false; + context.RunTest(run_on_gpu, X_data, batch_size, seq_len, nullptr, nullptr, Y_data, Y_h_data, Y_c_data, nullptr, + use_bias, use_peepholes, clip); +} + +TEST(LSTMTest, ONNXRuntime_TestLSTMBackward) { + const int seq_len = 2, batch_size = 1; + + std::vector X_data = {-0.455351f, -0.276391f, -0.185934f, -0.269585f}; + + std::vector Y_data = {-0.03189282f, 0.07626793f, + -0.03068724f, 0.02803503f}; + std::vector Y_h_data = {-0.03189282f, 0.07626793f}; + std::vector Y_c_data = {-0.07536839f, 0.12079399f}; + + LstmOpContext2x1x2x2 context("reverse"); + bool run_on_gpu = false; + context.RunTest(run_on_gpu, X_data, batch_size, seq_len, nullptr, nullptr, Y_data, Y_h_data, Y_c_data); +} + +TEST(LSTMTest, ONNXRuntime_TestLSTMBackward_gpu) { + const int seq_len = 2, batch_size = 1; + + std::vector X_data = {-0.455351f, -0.276391f, -0.185934f, -0.269585f}; + + std::vector Y_data = {-0.033075746f, 0.074455738f, + -0.031248707f, 0.027853041f}; + std::vector Y_h_data = {-0.033075746f, 0.074455738f}; + std::vector Y_c_data = {-0.076699793f, 0.11975205f}; + + LstmOpContext2x1x2x2 context("reverse"); + bool run_on_gpu = true; + // Disable peephole since cudnn doesn't support it + context.RunTest(run_on_gpu, X_data, batch_size, seq_len, nullptr, nullptr, Y_data, Y_h_data, Y_c_data, nullptr, true, false); +} + +TEST(LSTMTest, ONNXRuntime_TestLSTMForwardHiddenState) { + const int seq_len = 2, batch_size = 1; + + bool use_bias = true; + bool use_peepholes = false; + + std::vector X_data = {-0.455351f, -0.276391f, -0.185934f, -0.269585f}; + std::vector hidden_state = {0.34f, 0.72f}; + + std::vector Y_data = {0.01797521f, -0.07104912f, + -0.03174796f, -0.0152949f}; + std::vector Y_h_data = {-0.03174796f, -0.0152949f}; + std::vector Y_c_data = {-0.07285583f, -0.02545788f}; + + LstmOpContext2x1x2x2 context("forward"); + bool run_on_gpu = true; + context.RunTest(run_on_gpu, X_data, batch_size, seq_len, &hidden_state, nullptr, Y_data, Y_h_data, Y_c_data, + nullptr, use_bias, use_peepholes); +} + +TEST(LSTMTest, ONNXRuntime_TestLSTMForwardCellState) { + const int seq_len = 2, batch_size = 1; + + bool use_bias = true; + bool use_peepholes = false; + + std::vector X_data = {-0.455351f, -0.276391f, -0.185934f, -0.269585f}; + std::vector hidden_state = {0.34f, 0.72f}; + std::vector cell_state = {0.63f, 0.21f}; + + std::vector Y_data = {0.12797015f, 0.0097284f, + 0.02716939f, 0.01842997f}; + std::vector Y_h_data = {0.02716939f, 0.01842997f}; + std::vector Y_c_data = {0.06408449f, 0.03139432f}; + + LstmOpContext2x1x2x2 context("forward"); + bool run_on_gpu = true; + context.RunTest(run_on_gpu, X_data, batch_size, seq_len, &hidden_state, &cell_state, Y_data, Y_h_data, Y_c_data, + nullptr, use_bias, use_peepholes); +} + +TEST(LSTMTest, ONNXRuntime_TestLSTMActivation) { + const int seq_len = 2, batch_size = 1; + + std::vector activations = {"tanh", "sigmoid", "tanh"}; + + bool use_bias = true; + bool use_peepholes = false; + + std::vector X_data = {-0.455351f, -0.276391f, -0.185934f, -0.269585f}; + + std::vector Y_data = {-0.0660155f, 0.0351227f, + -0.04236888f, 0.0177365f}; + std::vector Y_h_data = {-0.04236888f, 0.0177365f}; + std::vector Y_c_data = {0.1624992f, 0.04672481f}; + + LstmOpContext2x1x2x2 context("forward", activations); + bool run_on_gpu = false; + context.RunTest(run_on_gpu, X_data, batch_size, seq_len, nullptr, nullptr, Y_data, Y_h_data, Y_c_data, + nullptr, use_bias, use_peepholes); +} + +// Original comments: +// test correctness for batch size > 1 and +// memory reallocation due to change in batch size +// The reallocation doesn't apply any more so this mainly tests larger batches with non-default activations. +TEST(LSTMTest, ONNXRuntime_TestLSTMBatchReallocation) { + ///////////////Attributes//////////////////////// + const int seq_len = 2; + int batch_size = 1; + bool use_bias = true; + bool use_peepholes = false; + + std::vector activations = {"tanh", "sigmoid", "tanh"}; + + //////////////////Inputs/////////////////////////////////// + std::string direction = "forward"; + + std::vector X_data = {-0.455351f, -0.276391f, + -0.185934f, -0.269585f}; + std::vector Y_data = {-0.0660155f, 0.0351227f, + -0.04236888f, 0.0177365f}; + std::vector Y_h_data = {-0.04236888f, 0.0177365f}; + std::vector Y_c_data = {0.1624992f, 0.04672481f}; + + LstmOpContext2x1x2x2 context(direction, activations); + bool run_on_gpu = false; + context.RunTest(run_on_gpu, X_data, batch_size, seq_len, nullptr, nullptr, Y_data, Y_h_data, Y_c_data, + nullptr, use_bias, use_peepholes); + + batch_size = 3; + + // updated from ONNXRuntime test so that it's not the same 2 values repeated 6 times each which potentially hides issues + X_data = {-0.455351f, -0.476391f, + -0.555351f, -0.376391f, + -0.655351f, -0.276391f, + -0.185934f, -0.869585f, + -0.285934f, -0.769585f, + -0.385934f, -0.669585f}; + + /* numpy */ + Y_data = {-0.090715f, 0.011908f, + -0.083193f, 0.037192f, + -0.073643f, 0.068889f, + -0.10545f, -0.01573f, + -0.10621f, -0.0056667f, + -0.10559f, 0.015734f}; + + Y_h_data = {-0.10545f, -0.01573f, + -0.10621f, -0.0056667f, + -0.10559f, 0.015734f}; + + Y_c_data = {0.21381f, -0.096022f, + 0.23038f, -0.0239f, + 0.24572f, 0.051626f}; + + context.RunTest(run_on_gpu, X_data, batch_size, seq_len, nullptr, nullptr, Y_data, Y_h_data, Y_c_data, + nullptr, use_bias, use_peepholes); +} + +// Original comments: +// test memory reallocation when sequence length increases +// test correctness for batch size > 1 and +// memory reallocation due to change in batch size +// also tests the tricky Y_data write used to avoid copying data +// Most of these aren't relevant anymore as we don't re-use buffers given Compute is stateless. +// It does test a batch > 1 with bidirectional output and custom activations though. +TEST(LSTMTest, ONNXRuntime_TestLSTMOutputWrite) { + const int seq_len = 2; + int batch_size = 1; + std::vector activations = {"tanh", "sigmoid", "tanh", "tanh", "sigmoid", "tanh"}; + + bool use_bias = true; + bool use_peepholes = false; + + std::vector X_data = {-0.455351f, -0.276391f, -0.185934f, -0.269585f}; + + std::vector Y_data = {-0.06601551f, 0.03512269f, + -0.05520744f, 0.03879774f, + + -0.04236888f, 0.01773649f, + -0.05332068f, 0.00207076f}; + std::vector Y_h_data = {-0.04236888f, 0.01773649f, + -0.05520744f, 0.03879774f}; + std::vector Y_c_data = {0.1624992f, 0.04672481f, + 0.22009919f, 0.08087098f}; + + std::string direction = "bidirectional"; + LstmOpContext2x1x2x2 context(direction, activations); + bool run_on_gpu = false; + context.RunTest(run_on_gpu, X_data, batch_size, seq_len, nullptr, nullptr, Y_data, Y_h_data, Y_c_data, + nullptr, use_bias, use_peepholes); + + batch_size = 3; + + X_data = {-0.455351f, -0.776391f, + -0.355351f, -0.576391f, + -0.255351f, -0.376391f, + + -0.185934f, -0.169585f, + -0.285934f, -0.469585f, + -0.385934f, -0.669585f}; + + Y_data = {-0.1269719f, -0.01049645f, + -0.09596697f, -0.00592083f, + -0.06777587f, -0.00001902f, + + -0.12206709f, -0.0051103f, + -0.08422903f, -0.00768428f, + -0.05224226f, -0.0042149f, + + -0.02778835f, 0.00775075f, + -0.06541093f, -0.00667958f, + -0.09953593f, -0.00899231f, + + -0.04350187f, 0.01127771f, + -0.07949658f, -0.00425178f, + -0.10883409f, -0.00926061f}; + + Y_h_data = {-0.02778835f, 0.00775075f, + -0.06541093f, -0.00667958f, + + -0.09953593f, -0.00899231f, + -0.12206709f, -0.0051103f, + + -0.08422903f, -0.00768428f, + -0.05224226f, -0.0042149f}; + + Y_c_data = {0.14675268f, 0.01759163f, + 0.19898503f, -0.01828078f, + 0.24029977f, -0.02784352f, + + 0.26577898f, -0.01694398f, + 0.22469461f, -0.02200207f, + 0.18284359f, -0.01078442f}; + + context.RunTest(run_on_gpu, X_data, batch_size, seq_len, nullptr, nullptr, Y_data, Y_h_data, Y_c_data, + nullptr, use_bias, use_peepholes); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/rnn/rnn_op_test.cc b/onnxruntime/test/providers/cpu/rnn/rnn_op_test.cc new file mode 100644 index 0000000000000..43e3aa49764a7 --- /dev/null +++ b/onnxruntime/test/providers/cpu/rnn/rnn_op_test.cc @@ -0,0 +1,787 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/rnn/rnn.h" +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" +using namespace std; +namespace onnxruntime { +namespace test { + +// test input data is generated from CNTK with shape of [batch_size, seq_length, input_size] +// onnxruntime takes input of shape [seq_length, batch_size, input_size) +template +void MemoryLayoutTransposeRNNInputCNTKToONNXRuntime(const T* X_data_cntk, T* X_data_onnx, int64_t seq_length, int64_t batch_size, int64_t input_size) { + for (int seq = 0; seq < seq_length; seq++) { + for (int batch = 0; batch < batch_size; batch++) { + for (int feature = 0; feature < input_size; feature++) { + X_data_onnx[(seq * batch_size + batch) * input_size + feature] = + X_data_cntk[(batch * seq_length + seq) * input_size + feature]; + } + } + } +} + +// test output data is generated from CNTK with shape of [batch_size, seq_length, num_directions, hidden_size]. +// onnxruntime takes output of shape [seq_length, num_directions, batch_size, hidden_size) +template +void MemoryLayoutTransposeRNNOutputCNTKToONNXRuntime(const T* X_data_cntk, T* X_data_onnx, + int64_t seq_length, int64_t num_directions, int64_t batch_size, int64_t hidden_size) { + for (int seq = 0; seq < seq_length; seq++) { + for (int dir = 0; dir < num_directions; dir++) { + for (int batch = 0; batch < batch_size; batch++) { + for (int feature = 0; feature < hidden_size; feature++) { + X_data_onnx[((seq * num_directions + dir) * batch_size + batch) * hidden_size + feature] = + // [batch, sequence, direction, feature] + X_data_cntk[((batch * seq_length + seq) * num_directions + dir) * hidden_size + feature]; + } + } + } + } +} + +TEST(RNNTest, RNN_bidirectional_bias_initial_zigged_batch) { + OpTester test("RNN"); + int64_t num_directions = 2, input_size = 2, hidden_size = 3, seq_length = 5; + + test.AddAttribute("activations", vector(num_directions, "Tanh")); + test.AddAttribute("direction", "bidirectional"); + test.AddAttribute("hidden_size", hidden_size); + + int batch_size = 2; + + std::vector X_dims = {seq_length, batch_size, input_size}; + std::vector X_data_in_batchs{1.64644051F, 2.14556813F, + 1.80829012F, 1.63464952F, + 1.27096438F, 1.93768239F, + 1.31276166F, 2.67531896F, + 2.89098835F, 1.15032458F, + + // batch 2 + 1.30798471F, 0.0777787F, + 1.64898741F, 1.30596721F, + 1.26110339F, 0.99100447F, + // + 0.0F, 0.0F, + 0.0F, 0.0F}; + std::vector X_data(seq_length * batch_size * input_size); + MemoryLayoutTransposeRNNInputCNTKToONNXRuntime(&X_data_in_batchs[0], &X_data[0], seq_length, batch_size, input_size); + test.AddInput("X", X_dims, X_data); + + std::vector W_dims = {num_directions, hidden_size, input_size}; + std::vector W_data({0.4317745F, 0.37378395F, -1.0386457F, -0.22681296F, 0.4418987F, 0.49973935F, + 0.47248289F, -0.63369429F, 0.89542073F, 0.69698066F, 0.65118814F, 1.0828459F}); + test.AddInput("W", W_dims, W_data); + + std::vector R_dims = {num_directions, hidden_size, hidden_size}; + std::vector R_data({-0.24072374F, -0.29326528F, -0.91741192F, + 0.5447638F, 0.53938544F, 0.79502326F, + -0.59813821F, 0.020413321F, -0.52225035F, + + -0.4292987F, -0.14766316F, -0.91084105F, + 0.23699039F, 0.064034894F, 0.089069292F, + -0.12803128F, -0.081178986F, 0.967533F}); + test.AddInput("R", R_dims, R_data); + + std::vector B_dims = {num_directions, 2 * hidden_size}; + std::vector B_data({-0.44529742F, 0.80094892F, -1.0028138F, 0.0F, 0.0F, 0.0F, + 0.57412368F, 0.13440208F, -0.85748988F, 0.0F, 0.0F, 0.0F}); + test.AddInput("B", B_dims, B_data); + + std::vector sequence_lens_dims({batch_size}); + std::vector sequence_lens_data{5, 3}; + test.AddInput("sequence_lens", sequence_lens_dims, sequence_lens_data); + + std::vector initial_h_dims = {num_directions, batch_size, hidden_size}; + std::vector initial_h_data({1.2F, 1.2F, 1.2F, 1.2F, 1.2F, 1.2F, 1.2F, 1.2F, 1.2F, 1.2F, 1.2F, 1.2F}); + test.AddInput("initial_h", initial_h_dims, initial_h_data); + + std::vector Y_dims = {seq_length, num_directions, batch_size, hidden_size}; + std::vector Y_data_in_batchs({-0.58767605F, 0.69586837F, -0.48001164F, -0.71697658F, 0.99646497F, 0.9980582F, + 0.8678354F, -0.94409049F, 0.8424542F, -0.37213817F, 0.99391747F, 0.99555576F, + 0.12221602F, -0.31430662F, -0.42285997F, -0.62726945F, 0.988343F, 0.9956606F, + 0.91737539F, -0.92293501F, 0.78396499F, -0.87013513F, 0.99671143F, 0.9990834F, + 0.51060104F, -0.95055139F, -0.12672578F, -0.51847482F, 0.99931973F, 0.99655205F, + // batch 2 + -0.92063117F, 0.93283325F, -0.93614483F, 0.1513377F, 0.90150106F, 0.74947751F, + 0.91569924F, -0.96036619F, 0.89311725F, -0.13006453F, 0.98576784F, 0.98875856F, + -0.28076148F, -0.04275616F, -0.75480938F, -0.84641802F, 0.98438591F, 0.96007115F, + 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, + 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F}); + std::vector Y_data(seq_length * num_directions * batch_size * hidden_size); + MemoryLayoutTransposeRNNOutputCNTKToONNXRuntime(&Y_data_in_batchs[0], &Y_data[0], + seq_length, num_directions, batch_size, hidden_size); + test.AddOutput("Y", Y_dims, Y_data); + + std::vector Y_h_dims{num_directions, batch_size, hidden_size}; + std::vector Y_h_data({0.51060104F, -0.95055139F, -0.12672578F, -0.28076148F, -0.04275616F, -0.75480938F, + -0.71697658F, 0.99646497F, 0.9980582F, 0.1513377F, 0.90150106F, 0.74947751F}); + test.AddOutput("Y_h", Y_h_dims, Y_h_data); + + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider}); +} + +TEST(RNNTest, RNN_bidirectional_zigged_batch) { + OpTester test("RNN"); + int64_t num_directions = 2, input_size = 2, hidden_size = 3, seq_length = 5; + + test.AddAttribute("activations", vector(num_directions, "Tanh")); + test.AddAttribute("direction", "bidirectional"); + test.AddAttribute("hidden_size", hidden_size); + + int batch_size = 2; + + std::vector X_dims = {seq_length, batch_size, input_size}; + std::vector X_data_in_batchs{1.64644051F, 2.14556813F, + 1.80829012F, 1.63464952F, + 1.27096438F, 1.93768239F, + 1.31276166F, 2.67531896F, + 2.89098835F, 1.15032458F, + // batch 2 + 1.30798471F, 0.0777787F, + 1.64898741F, 1.30596721F, + 1.26110339F, 0.99100447F, + 0.0F, 0.0F, + 0.0F, 0.0F}; + std::vector X_data(seq_length * batch_size * input_size); + MemoryLayoutTransposeRNNInputCNTKToONNXRuntime(&X_data_in_batchs[0], &X_data[0], seq_length, batch_size, input_size); + test.AddInput("X", X_dims, X_data); + + std::vector W_dims = {num_directions, hidden_size, input_size}; + std::vector W_data({-0.68526405F, 0.3758406F, 0.13007233F, 0.6596455F, -0.68564546F, 0.22745803F, + 0.37704858F, -0.075543992F, -0.92860377F, -0.014112951F, -1.0042796F, 0.83100969F}); + test.AddInput("W", W_dims, W_data); + + std::vector R_dims = {num_directions, hidden_size, hidden_size}; + std::vector R_data({0.22057047F, -0.25696567F, 0.93817306F, + -0.1917963F, -0.41374302F, -0.76374459F, + -0.96291065F, 0.098433927F, 0.049011F, + 0.56542879F, 0.50024462F, 0.33647421F, + -0.80293375F, 0.59855759F, -0.74431759F, + -0.003538545F, -0.73175585F, 0.65632182F}); + test.AddInput("R", R_dims, R_data); + + std::vector B_dims = {num_directions, 2 * hidden_size}; + std::vector B_data({0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F}); + test.AddInput("B", B_dims, B_data); + + std::vector sequence_lens_dims({batch_size}); + std::vector sequence_lens_data{5, 3}; + test.AddInput("sequence_lens", sequence_lens_dims, sequence_lens_data); + + std::vector initial_h_dims = {num_directions, batch_size, hidden_size}; + std::vector initial_h_data({0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F}); + test.AddInput("initial_h", initial_h_dims, initial_h_data); + + std::vector Y_dims = {seq_length, num_directions, batch_size, hidden_size}; + std::vector Y_data_in_batchs({-0.31118321F, 0.92598617F, -0.56547648F, 0.39222997F, -0.99489242F, 0.86467457F, + -0.8980186F, 0.89000309F, -0.46600604F, 0.38946036F, -0.99521333F, 0.69356728F, + -0.76437593F, 0.92218089F, 0.46116444F, 0.06449185F, -0.97850645F, 0.90903103F, + 0.13221112F, 0.87366635F, 0.50636965F, -0.09428534F, -0.94113714F, 0.76040554F, + -0.85353446F, 0.34633741F, -0.93988168F, 0.76291096F, -0.99102205F, -0.96011895F, + // batch 2 + -0.69988877F, 0.21788915F, -0.70597935F, 0.0274523F, -0.9431532F, -0.60166585F, + -0.90726709F, 0.93011433F, -0.17109135F, 0.18146965F, -0.96685904F, -0.23413686F, + -0.79737622F, 0.62769204F, 0.30727068F, 0.38049027F, -0.82903779F, -0.41610005F, + 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, + 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F}); + std::vector Y_data(seq_length * num_directions * batch_size * hidden_size); + MemoryLayoutTransposeRNNOutputCNTKToONNXRuntime(&Y_data_in_batchs[0], &Y_data[0], + seq_length, num_directions, batch_size, hidden_size); + test.AddOutput("Y", Y_dims, Y_data); + + std::vector Y_h_dims{num_directions, batch_size, hidden_size}; + std::vector Y_h_data({-0.85353446F, 0.34633741F, -0.93988168F, -0.79737622F, 0.62769204F, 0.30727068F, + 0.39222997F, -0.99489242F, 0.86467457F, 0.0274523F, -0.9431532F, -0.60166585F}); + test.AddOutput("Y_h", Y_h_dims, Y_h_data); + + test.Run(); +} + +TEST(RNNTest, RNN_reverse_direction_zigged_batch) { + OpTester test("RNN"); + int64_t num_directions = 1, input_size = 2, hidden_size = 3, seq_length = 5; + + test.AddAttribute("activations", vector(num_directions, "Tanh")); + test.AddAttribute("direction", "reverse"); + test.AddAttribute("hidden_size", hidden_size); + + int batch_size = 2; + + std::vector X_dims = {seq_length, batch_size, input_size}; + std::vector X_data_in_batchs{0.54881352F, 0.71518934F, + 0.60276335F, 0.54488319F, + 0.42365479F, 0.64589411F, + 0.4375872F, 0.89177299F, + 0.96366274F, 0.38344151F, + // batch 2 + 0.417021990F, 0.720324516F, + 0.0001143748F, 0.302332580F, + 0.146755889F, 0.0923385918F, + 0.0F, 0.0F, + 0.0F, 0.0F}; + std::vector X_data(seq_length * batch_size * input_size); + MemoryLayoutTransposeRNNInputCNTKToONNXRuntime(&X_data_in_batchs[0], &X_data[0], seq_length, batch_size, input_size); + test.AddInput("X", X_dims, X_data); + + std::vector W_dims = {num_directions, hidden_size, input_size}; + std::vector W_data({0.60482931F, 0.67304987F, 0.13166776F, -0.33417314F, 0.66345924F, -0.49411628F}); + test.AddInput("W", W_dims, W_data); + + std::vector R_dims = {num_directions, hidden_size, hidden_size}; + std::vector R_data({0.50877059F, 0.78382635F, 0.665046F, + 0.89860243F, -0.71745688F, 0.80142093F, + -0.76517141F, -0.88981366F, -0.48568386F}); + test.AddInput("R", R_dims, R_data); + + std::vector B_dims = {num_directions, 2 * hidden_size}; + std::vector B_data({0.0F, 0.0F, 0.0F, + 0.0F, 0.0F, 0.0F}); + test.AddInput("B", B_dims, B_data); + + std::vector sequence_lens_dims({batch_size}); + std::vector sequence_lens_data{5, 3}; + test.AddInput("sequence_lens", sequence_lens_dims, sequence_lens_data); + + std::vector initial_h_dims = {num_directions, batch_size, hidden_size}; + std::vector initial_h_data({0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F}); + test.AddInput("initial_h", initial_h_dims, initial_h_data); + + std::vector Y_dims = {seq_length, num_directions, batch_size, hidden_size}; + std::vector Y_data_in_batchs({0.87014002F, 0.09402763F, -0.54269236F, + 0.37661949F, 0.28492415F, 0.15850827F, + 0.8218801F, -0.33996487F, -0.7320742F, + 0.90398145F, 0.61396617F, -0.70602065F, + 0.68629962F, -0.00125255F, 0.4218055F, + // batch 2 + 0.64809889F, -0.19472955F, -0.24271242F, + 0.29596764F, 0.08308408F, -0.27175695F, + 0.14977546F, -0.01153355F, 0.05169443F, + 0.0F, 0.0F, 0.0F, + 0.0F, 0.0F, 0.0F}); + std::vector Y_data(seq_length * num_directions * batch_size * hidden_size); + MemoryLayoutTransposeRNNOutputCNTKToONNXRuntime(&Y_data_in_batchs[0], &Y_data[0], + seq_length, num_directions, batch_size, hidden_size); + test.AddOutput("Y", Y_dims, Y_data); + + std::vector Y_h_dims{num_directions, batch_size, hidden_size}; + std::vector Y_h_data({0.87014002F, 0.09402763F, -0.54269236F, 0.64809889F, -0.19472955F, -0.24271242F}); + test.AddOutput("Y_h", Y_h_dims, Y_h_data); + + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider}); +} + +TEST(RNNTest, RNN_forward_direction_zigged_batch) { + OpTester test("RNN"); + int64_t num_directions = 1, input_size = 2, hidden_size = 3, seq_length = 5; + + test.AddAttribute("activations", vector(num_directions, "Tanh")); + test.AddAttribute("direction", "forward"); + test.AddAttribute("hidden_size", hidden_size); + + int batch_size = 2; + + std::vector X_dims = {seq_length, batch_size, input_size}; + std::vector X_data_in_batchs{0.061169811F, 0.26296741F, + 0.80939841F, 0.080034949F, + 0.21000224F, 0.65772671F, + 0.20081005F, 0.95461535F, + 0.93818879F, 0.76034665F, + // batch 2 + 0.34715694F, 0.0032335778F, + 0.72840774F, 0.20933059F, + 0.01131162F, 0.15063381F, + 0.0F, 0.0F, + 0.0F, 0.0F}; + std::vector X_data(seq_length * batch_size * input_size); + MemoryLayoutTransposeRNNInputCNTKToONNXRuntime(&X_data_in_batchs[0], &X_data[0], seq_length, batch_size, input_size); + + test.AddInput("X", X_dims, X_data); + + std::vector W_dims = {num_directions, hidden_size, input_size}; + std::vector W_data({-0.49937296F, -0.082866333F, 0.40978807F, -0.33496389F, -0.40066367F, -0.72275674F}); + test.AddInput("W", W_dims, W_data); + + std::vector R_dims = {num_directions, hidden_size, hidden_size}; + std::vector R_data({0.16146433F, -0.36291042F, 0.61149812F, + -0.018460333F, -0.19345543F, 0.35175204F, + 0.84270394F, 0.94917566F, -0.76469761F}); + test.AddInput("R", R_dims, R_data); + + std::vector B_dims = {num_directions, 2 * hidden_size}; + std::vector B_data({0.0F, 0.0F, 0.0F, + 0.0F, 0.0F, 0.0F}); + test.AddInput("B", B_dims, B_data); + + std::vector sequence_lens_dims({batch_size}); + std::vector sequence_lens_data{5, 3}; + test.AddInput("sequence_lens", sequence_lens_dims, sequence_lens_data); + + std::vector initial_h_dims = {num_directions, batch_size, hidden_size}; + std::vector initial_h_data({0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F}); + test.AddInput("initial_h", initial_h_dims, initial_h_data); + + std::vector Y_dims = {seq_length, num_directions, batch_size, hidden_size}; + std::vector Y_data_in_batchs({-0.0522899628F, -0.0629346371F, -0.211336553F, + -0.482055902F, 0.238964200F, -0.313421130F, + -0.474286675F, -0.274602413F, -0.461531579F, + -0.412429035F, -0.325635254F, -0.792385221F, + -0.746264696F, -0.0781838298F, -0.751394153F, + // batch 2 + -0.171904743F, 0.140247226F, -0.140494764F, + -0.497260034F, 0.153767705F, -0.334113181F, + -0.343922496F, -0.181868196F, -0.130254388F, + 0.0F, 0.0F, 0.0F, + 0.0F, 0.0F, 0.0F}); + std::vector Y_data(seq_length * num_directions * batch_size * hidden_size); + MemoryLayoutTransposeRNNOutputCNTKToONNXRuntime(&Y_data_in_batchs[0], &Y_data[0], + seq_length, num_directions, batch_size, hidden_size); + test.AddOutput("Y", Y_dims, Y_data); + + std::vector Y_h_dims{num_directions, batch_size, hidden_size}; + std::vector Y_h_data({-0.746264696F, -0.0781838298F, -0.751394153F, -0.343922496F, -0.181868196F, -0.130254388F}); + test.AddOutput("Y_h", Y_h_dims, Y_h_data); + + test.Run(); +} + +TEST(RNNTest, RNN_bidirectional) { + OpTester test("RNN"); + int64_t num_directions = 2, input_size = 2, hidden_size = 3, batch_size = 1, seq_length = 5; + + test.AddAttribute("activations", vector(num_directions, "Tanh")); + test.AddAttribute("direction", "bidirectional"); + test.AddAttribute("hidden_size", hidden_size); + + std::vector X_dims = {seq_length, batch_size, input_size}; + std::vector X_data({0.54881352F, 0.71518934F, + 0.60276335F, 0.54488319F, + 0.42365479F, 0.64589411F, + 0.4375872F, 0.891773F, + 0.96366274F, 0.38344151F}); + + test.AddInput("X", X_dims, X_data); + + std::vector W_dims = {num_directions, hidden_size, input_size}; + std::vector W_data({-0.74535543F, 0.21360011F, 1.0782362F, 0.092641734F, -1.0087538F, -0.97021431F, + 0.88425213F, 0.93182313F, 0.767329F, -0.541361F, 0.6218195F, -0.7977342F}); + + test.AddInput("W", W_dims, W_data); + + std::vector R_dims = {num_directions, hidden_size, hidden_size}; + std::vector R_data({// forward + -0.7322467F, -0.95795155F, -0.058495734F, + -0.7271859F, -0.29820377F, -0.85114992F, + -0.097570196F, 0.82271612F, 0.1396943F, + // reverse + 0.11753198F, -0.30726218F, 0.47448817F, + -0.60847247F, 0.11959127F, -0.15468557F, + 0.18048254F, -0.27739462F, 0.40944993F}); + test.AddInput("R", R_dims, R_data); + + std::vector B_dims = {num_directions, 2 * hidden_size}; + std::vector B_data({0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, + 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F}); + test.AddInput("B", B_dims, B_data); + + std::vector sequence_lens_dims({batch_size}); + std::vector sequence_lens_data(batch_size, (int)seq_length); + test.AddInput("sequence_lens", sequence_lens_dims, sequence_lens_data); + + std::vector initial_h_dims = {num_directions, batch_size, hidden_size}; + std::vector initial_h_data({0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F}); + test.AddInput("initial_h", initial_h_dims, initial_h_data); + + std::vector Y_dims = {seq_length, num_directions, batch_size, hidden_size}; + std::vector Y_data({-0.25082839F, 0.57703555F, -0.84758246F, 0.89708149F, -0.50691134F, 0.10560472F, + -0.57328993F, 0.89210528F, -0.63864726F, 0.85242939F, -0.35763535F, 0.20078957F, + -0.51920897F, 0.83700335F, -0.33934233F, 0.80431187F, -0.51605088F, -0.060805645F, + -0.49105126F, 0.74924558F, -0.54746729F, 0.86223149F, -0.56618357F, -0.29732516F, + -0.74539614F, 0.93210655F, -0.63887376F, 0.83650553F, 0.48680621F, 0.28520593F}); + test.AddOutput("Y", Y_dims, Y_data); + + std::vector Y_h_dims{num_directions, batch_size, hidden_size}; + std::vector Y_h_data({-0.74539614F, 0.93210655F, -0.63887376F, 0.89708149F, -0.50691134F, 0.10560472F}); + test.AddOutput("Y_h", Y_h_dims, Y_h_data); + + test.Run(); +} + +typedef enum { + RNNOutputY, + RNNOutputY_h, + RNNOutputBoth +} RNNOutputOption; + +TEST(RNNTest, RNN_default_attributes_and_forward_direction) { + int64_t num_directions = 1, input_size = 2, hidden_size = 3, batch_size = 1, seq_length = 5; + + // In case of useDefault, attributes, inputs or outputs are not set. + // Otherwise they are set (with values may or may not be the same as ONNX default values). + auto run_test = [&](OpTester& test, bool useDefault, RNNOutputOption outputOption) { + std::vector X_dims = {seq_length, batch_size, input_size}; + std::vector X_data({0.061169811F, 0.26296741F, + 0.80939841F, 0.080034949F, + 0.21000224F, 0.65772671F, + 0.20081005F, 0.95461535F, + 0.93818879F, 0.76034665F}); + + test.AddInput("X", X_dims, X_data); + + std::vector W_dims = {num_directions, hidden_size, input_size}; + std::vector W_data({-0.49937296F, -0.082866333F, 0.40978807F, -0.33496389F, -0.40066367F, -0.72275674F}); + test.AddInput("W", W_dims, W_data); + + std::vector R_dims = {num_directions, hidden_size, hidden_size}; + std::vector R_data({0.16146433F, -0.36291042F, 0.61149812F, + -0.018460333F, -0.19345543F, 0.35175204F, + 0.84270394F, 0.94917566F, -0.76469761F}); + test.AddInput("R", R_dims, R_data); + + if (useDefault) { + std::vector B_dims = {num_directions, 2 * hidden_size}; + std::vector B_data({0.0F, 0.0F, 0.0F, + 0.0F, 0.0F, 0.0F}); + test.AddInput("B", B_dims, B_data); + + std::vector sequence_lens_dims({batch_size}); + std::vector sequence_lens_data(batch_size, (int)seq_length); + test.AddInput("sequence_lens", sequence_lens_dims, sequence_lens_data); + + std::vector initial_h_dims = {num_directions, batch_size, hidden_size}; + std::vector initial_h_data({0.0F, 0.0F, 0.0F}); + test.AddInput("initial_h", initial_h_dims, initial_h_data); + } else { + test.AddMissingOptionalInput(); + test.AddMissingOptionalInput(); + test.AddMissingOptionalInput(); + } + + if (outputOption == RNNOutputY || outputOption == RNNOutputBoth) { + std::vector Y_dims = {seq_length, num_directions, batch_size, hidden_size}; + std::vector Y_data({-0.052289959F, -0.062934637F, -0.21133657F, + -0.48205593F, 0.23896417F, -0.31342113F, + -0.47428668F, -0.27460238F, -0.46153161F, + -0.41242906F, -0.32563525F, -0.79238516F, + -0.74626476F, -0.07818383F, -0.75139415F}); + test.AddOutput("Y", Y_dims, Y_data); + } else { + test.AddMissingOptionalOutput(); + } + + if (outputOption == RNNOutputY_h || outputOption == RNNOutputBoth) { + std::vector Y_h_dims{num_directions, batch_size, hidden_size}; + std::vector Y_h_data({-0.74626476F, -0.07818383F, -0.75139415F}); + test.AddOutput("Y_h", Y_h_dims, Y_h_data); + } else { + test.AddMissingOptionalOutput(); + } + + test.Run(); + }; + + { + OpTester test_use_default("RNN"); + test_use_default.AddAttribute("hidden_size", hidden_size); + run_test(test_use_default, true, RNNOutputY); + } + + { + OpTester test_use_default("RNN"); + test_use_default.AddAttribute("hidden_size", hidden_size); + run_test(test_use_default, true, RNNOutputY_h); + } + + { + OpTester test_use_default("RNN"); + test_use_default.AddAttribute("hidden_size", hidden_size); + run_test(test_use_default, true, RNNOutputBoth); + } + + { + OpTester test_do_not_use_default("RNN"); + test_do_not_use_default.AddAttribute("activations", vector(num_directions, "Tanh")); + test_do_not_use_default.AddAttribute("direction", "forward"); + test_do_not_use_default.AddAttribute("hidden_size", hidden_size); + + run_test(test_do_not_use_default, false, RNNOutputY); + } + { + OpTester test_do_not_use_default("RNN"); + test_do_not_use_default.AddAttribute("activations", vector(num_directions, "Tanh")); + test_do_not_use_default.AddAttribute("direction", "forward"); + test_do_not_use_default.AddAttribute("hidden_size", hidden_size); + run_test(test_do_not_use_default, false, RNNOutputY_h); + } + + { + OpTester test_do_not_use_default("RNN"); + test_do_not_use_default.AddAttribute("activations", vector(num_directions, "Tanh")); + test_do_not_use_default.AddAttribute("direction", "forward"); + test_do_not_use_default.AddAttribute("hidden_size", hidden_size); + run_test(test_do_not_use_default, false, RNNOutputBoth); + } +} + +TEST(RNNTest, RNN_reverse_direction) { + int64_t num_directions = 1, input_size = 2, hidden_size = 3, batch_size = 1, seq_length = 5; + + // In case of useDefault, attributes, inputs or outputs are not set. + // Otherwise they are set (with values may or may not be the same as ONNX default values). + auto runTest = [&](OpTester& test, bool useDefault, RNNOutputOption outputOption) { + std::vector X_dims = {seq_length, batch_size, input_size}; + std::vector X_data({0.54881352F, 0.71518934F, + 0.60276335F, 0.54488319F, + 0.42365479F, 0.64589411F, + 0.4375872F, 0.891773F, + 0.96366274F, 0.38344151F}); + test.AddInput("X", X_dims, X_data); + + std::vector W_dims = {num_directions, hidden_size, input_size}; + std::vector W_data({-0.74535543F, 0.21360011F, 1.0782362F, 0.092641734F, -1.0087538F, -0.97021431F}); + test.AddInput("W", W_dims, W_data); + + std::vector R_dims = {num_directions, hidden_size, hidden_size}; + std::vector R_data({-0.7322467F, -0.95795155F, -0.058495734F, + -0.7271859F, -0.29820377F, -0.85114992F, + -0.097570196F, 0.82271612F, 0.1396943F}); + test.AddInput("R", R_dims, R_data); + + if (!useDefault) { + std::vector B_dims = {num_directions, 2 * hidden_size}; + std::vector B_data({0.0F, 0.0F, 0.0F, + 0.0F, 0.0F, 0.0F}); + test.AddInput("B", B_dims, B_data); + + std::vector sequence_lens_dims({batch_size}); + std::vector sequence_lens_data(batch_size, (int)seq_length); + test.AddInput("sequence_lens", sequence_lens_dims, sequence_lens_data); + + std::vector initial_h_dims = {num_directions, batch_size, hidden_size}; + std::vector initial_h_data({0.0F, 0.0F, 0.0F}); + test.AddInput("initial_h", initial_h_dims, initial_h_data); + } else { + test.AddMissingOptionalInput(); + test.AddMissingOptionalInput(); + test.AddMissingOptionalInput(); + } + + std::vector Y_dims = {seq_length, num_directions, batch_size, hidden_size}; + std::vector Y_data({-0.55397642F, 0.83026606F, -0.51471221F, + -0.55358219F, 0.8341592F, -0.44313878F, + -0.60828412F, 0.78948581F, -0.34582433F, + -0.40591392F, 0.89962566F, -0.61860478F, + -0.56242156F, 0.79118007F, -0.872658F}); + if (outputOption == RNNOutputY || outputOption == RNNOutputBoth) { + test.AddOutput("Y", Y_dims, Y_data); + } else { + test.AddMissingOptionalOutput(); + } + + std::vector Y_h_dims{num_directions, batch_size, hidden_size}; + std::vector Y_h_data({-0.55397642F, 0.83026606F, -0.51471221F}); + if (outputOption == RNNOutputY_h || outputOption == RNNOutputBoth) { + test.AddOutput("Y_h", Y_h_dims, Y_h_data); + } else { + test.AddMissingOptionalOutput(); + } + + test.Run(); + }; + + // TODO: bring in these tests + { + OpTester test_use_default("RNN"); + test_use_default.AddAttribute("direction", "reverse"); + test_use_default.AddAttribute("hidden_size", hidden_size); + runTest(test_use_default, true, RNNOutputY); + } + { + OpTester test_use_default("RNN"); + test_use_default.AddAttribute("direction", "reverse"); + test_use_default.AddAttribute("hidden_size", hidden_size); + runTest(test_use_default, true, RNNOutputY_h); + } + + { + OpTester test_use_default("RNN"); + test_use_default.AddAttribute("direction", "reverse"); + test_use_default.AddAttribute("hidden_size", hidden_size); + runTest(test_use_default, true, RNNOutputBoth); + } + + { + OpTester test_do_not_use_default("RNN"); + test_do_not_use_default.AddAttribute("activations", vector(num_directions, "Tanh")); + test_do_not_use_default.AddAttribute("direction", "reverse"); + test_do_not_use_default.AddAttribute("hidden_size", hidden_size); + runTest(test_do_not_use_default, false, RNNOutputY); + } + { + OpTester test_do_not_use_default("RNN"); + test_do_not_use_default.AddAttribute("activations", vector(num_directions, "Tanh")); + test_do_not_use_default.AddAttribute("direction", "reverse"); + test_do_not_use_default.AddAttribute("hidden_size", hidden_size); + runTest(test_do_not_use_default, false, RNNOutputY_h); + } + { + OpTester test_do_not_use_default("RNN"); + test_do_not_use_default.AddAttribute("activations", vector(num_directions, "Tanh")); + test_do_not_use_default.AddAttribute("direction", "reverse"); + test_do_not_use_default.AddAttribute("hidden_size", hidden_size); + runTest(test_do_not_use_default, false, RNNOutputBoth); + } +} + +TEST(RNNTest, RNN_invalid_sequence_lens) { + int64_t num_directions = 1, input_size = 2, hidden_size = 3, batch_size = 2, seq_length = 3; + + auto run_test = [&](const std::vector& sequence_lens, + const std::string& error_msg) { + OpTester test("RNN"); + + test.AddAttribute("activations", vector(num_directions, "Tanh")); + test.AddAttribute("direction", "forward"); + test.AddAttribute("hidden_size", hidden_size); + + std::vector X_dims = {seq_length, batch_size, input_size}; + std::vector X_data({0.1f, 0.2f, 0.3f, + 0.4f, 0.5f, 0.6f, + 0.7f, 0.8f, 0.9f, + 0.11f, 0.12f, 0.13f}); + + test.AddInput("X", X_dims, X_data); + + std::vector W_dims = {num_directions, hidden_size, input_size}; + std::vector W_data({-0.1f, 0.2f, 1.f, -2.f, -1.f, 3.f}); + + test.AddInput("W", W_dims, W_data); + + std::vector R_dims = {num_directions, hidden_size, hidden_size}; + std::vector R_data({0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f}); + test.AddInput("R", R_dims, R_data); + + std::vector B_dims = {num_directions, 2 * hidden_size}; + std::vector B_data{0.f, 0.f, 0.f, 0.f, 0.f, 0.f}; + test.AddInput("B", B_dims, B_data); + + // std::vector sequence_lens_dims({batch_size}); + std::vector sequence_lens_dims{gsl::narrow_cast(sequence_lens.size())}; + test.AddInput("sequence_lens", sequence_lens_dims, sequence_lens); + + std::vector initial_h_dims = {num_directions, batch_size, hidden_size}; + std::vector initial_h_data{0.f, 0.f, 0.f, 0.f, 0.f, 0.f}; + test.AddInput("initial_h", initial_h_dims, initial_h_data); + + // Y + test.AddMissingOptionalOutput(); + + std::vector Y_h_dims{num_directions, batch_size, hidden_size}; + std::vector Y_h_data{0.f, 0.f, 0.f, 0.f, 0.f, 0.f}; + test.AddOutput("Y_h", Y_h_dims, Y_h_data); + + test.Run(OpTester::ExpectResult::kExpectFailure, error_msg); + }; + + // should batch batch_size to be valid + std::vector invalid_num_seq_len_entries(batch_size - 1, 5); + + run_test(invalid_num_seq_len_entries, "Input sequence_lens must have shape {2}. Actual:{1}"); + + // 0 is an invalid value + std::vector bad_seq_len_entry{0, 5}; + run_test(bad_seq_len_entry, "Invalid value/s in sequence_lens. All values must be > 0 and < seq_length."); +} + +TEST(RNNTest, RNN_bidirectional_with_sequence_lens) { + // values copied from WinML RNN test + int64_t num_directions = 2, input_size = 2, hidden_size = 4, batch_size = 3, seq_length = 2; + + OpTester test("RNN"); + + test.AddAttribute("activations", vector{"Relu", "Tanh"}); + test.AddAttribute("direction", "bidirectional"); + test.AddAttribute("hidden_size", hidden_size); + + std::vector X_dims = {seq_length, batch_size, input_size}; + std::vector X_data({1.f, 2.f, + 3.f, 4.f, + 5.f, 6.f, + + 3.f, 1.f, + 4.f, 4.f, + 6.f, 3.f}); + + test.AddInput("X", X_dims, X_data); + + std::vector W_dims = {num_directions, hidden_size, input_size}; + std::vector W_data{0.1f, 0.2f, + 0.3f, 0.4f, + 0.5f, 0.6f, + 0.5f, 0.6f, + 0.7f, 0.3f, + 0.8f, 0.4f, + 0.9f, 0.5f, + 0.1f, 0.6f}; + + test.AddInput("W", W_dims, W_data); + + std::vector R_dims = {num_directions, hidden_size, hidden_size}; + std::vector R_data{0.1f, 0.1f, 0.1f, 0.1f, + 0.1f, 0.1f, 0.1f, 0.1f, + 0.1f, 0.1f, 0.1f, 0.1f, + 0.1f, 0.1f, 0.1f, 0.1f, + 0.2f, 0.2f, 0.2f, 0.2f, + 0.2f, 0.2f, 0.2f, 0.2f, + 0.2f, 0.2f, 0.2f, 0.2f, + 0.2f, 0.2f, 0.2f, 0.2f}; + test.AddInput("R", R_dims, R_data); + + std::vector B_dims = {num_directions, 2 * hidden_size}; + std::vector B_data{0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, + 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f}; + test.AddInput("B", B_dims, B_data); + + std::vector sequence_lens_dims{batch_size}; + std::vector sequence_len_data{1, 2, 2}; + test.AddInput("sequence_lens", sequence_lens_dims, sequence_len_data); + + // Y + std::vector Y_dims{seq_length, num_directions, batch_size, hidden_size}; + std::vector Y_data{ + 0.69999999f, 1.3000001f, 1.9000001f, 1.9000001f, + 1.3000001f, 2.7f, 4.0999999f, 4.0999999f, + 1.9000001f, 4.0999999f, 6.3000002f, 6.3000002f, + + 0.90514827f, 0.94680595f, 0.97045195f, 0.90514827f, + 0.99963105f, 0.99990892f, 0.99997759f, 0.99877548f, + 0.99999332f, 0.99999928f, 1.f, 0.99992537f, + + 0.f, 0.f, 0.f, 0.f, + 2.6200001f, 4.2200003f, 5.8199997f, 5.8199997f, + 3.2600002f, 5.0600004f, 6.8600001f, 6.8600001f, + + 0.f, 0.f, 0.f, 0.f, + 0.99955046f, 0.99990916f, 0.99998164f, 0.99505484f, + 0.99995017f, 0.99999166f, 0.99999857f, 0.9890275f}; + + test.AddOutput("Y", Y_dims, Y_data); + + std::vector Y_h_dims{num_directions, batch_size, hidden_size}; + std::vector Y_h_data{0.69999999f, 1.3000001f, 1.9000001f, 1.9000001f, + 2.6200001f, 4.2200003f, 5.8199997f, 5.8199997f, + 3.2600002f, 5.0600004f, 6.8600001f, 6.8600001f, + + 0.90514827f, 0.94680595f, 0.97045195f, 0.90514827f, + 0.99963105f, 0.99990892f, 0.99997759f, 0.99877548f, + 0.99999332f, 0.99999928f, 1.f, 0.99992537f}; + + test.AddOutput("Y_h", Y_h_dims, Y_h_data); + + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider}); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/tensor/compress_op.test.cc b/onnxruntime/test/providers/cpu/tensor/compress_op.test.cc new file mode 100644 index 0000000000000..5178a3382a1ce --- /dev/null +++ b/onnxruntime/test/providers/cpu/tensor/compress_op.test.cc @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(CompressTest, Compress0) { + OpTester test("Compress", 9); + + test.AddAttribute("axis", int64_t(0)); + + test.AddInput("input", {3, 2}, { + 1.0f, 2.0f, + 3.0f, 4.0f, + 5.0f, 6.0f}); + test.AddInput("condition", {3}, {0, 1, 1}); + test.AddOutput("output", {2, 2}, { + 3.0f, 4.0f, + 5.0f, 6.0f}); + test.Run(); +} + +TEST(CompressTest, Compress1) { + OpTester test("Compress", 9); + + test.AddAttribute("axis", int64_t(1)); + + test.AddInput("input", {3, 2}, { + 1.0f, 2.0f, + 3.0f, 4.0f, + 5.0f, 6.0f}); + test.AddInput("condition", {2}, {0, 1}); + test.AddOutput("output", {3, 1}, { + 2.0f, + 4.0f, + 6.0f}); + test.Run(); +} + +TEST(CompressTest, Compress_3dims) { + OpTester test("Compress", 9); + + test.AddAttribute("axis", int64_t(1)); + + test.AddInput("input", {2, 2, 3}, { + 1.0f, 2.0f, 3.0f, + 4.0f, 5.0f, 6.0f, + + 7.0f, 8.0f, 9.0f, + 10.0f, 11.0f, 12.0f}); + test.AddInput("condition", {2}, {0, 1}); + test.AddOutput("output", {2, 1, 3}, { + 4.0f, 5.0f, 6.0f, + 10.0f, 11.0f, 12.0f}); + test.Run(); +} + +TEST(CompressTest, Compress_condition_all_false) { + OpTester test("Compress", 9); + + test.AddAttribute("axis", int64_t(1)); + + test.AddInput("input", {2, 2, 3}, {1.0f, 2.0f, 3.0f, + 4.0f, 5.0f, 6.0f, + + 7.0f, 8.0f, 9.0f, + 10.0f, 11.0f, 12.0f}); + test.AddInput("condition", {2}, {0, 0}); + test.AddOutput("output", {2, 0, 3}, {}); + test.Run(); +} + +TEST(CompressTest, Compress_3dims_has_extra_condition) { + OpTester test("Compress", 9); + + test.AddAttribute("axis", int64_t(1)); + + test.AddInput("input", {2, 2, 3}, { + 1.0f, 2.0f, 3.0f, + 4.0f, 5.0f, 6.0f, + + 7.0f, 8.0f, 9.0f, + 10.0f, 11.0f, 12.0f}); + // has condition length = 3 > input_dim[axis] = 2 + test.AddInput("condition", {3}, {0, 1, 1}); + test.AddOutput("output", {2, 1, 3}, { + 4.0f, 5.0f, 6.0f, + 10.0f, 11.0f, 12.0f}); + test.Run(); +} + +TEST(CompressTest, Compress_3dims_has_extra_input) { + OpTester test("Compress", 9); + + test.AddAttribute("axis", int64_t(1)); + + test.AddInput("input", {2, 3, 3}, { + 1.0f, 2.0f, 3.0f, + 4.0f, 5.0f, 6.0f, + 7.0f, 8.0f, 9.0f, + + 10.0f, 11.0f, 12.0f, + 13.0f, 14.0f, 15.0f, + 16.0f, 17.0f, 18.0f}); + // has condition length = 2 < input_dim[axis] = 3 + test.AddInput("condition", {2}, {0, 1}); + test.AddOutput("output", {2, 1, 3}, { + 4.0f, 5.0f, 6.0f, + 13.0f, 14.0f, 15.0f}); + test.Run(); +} + +TEST(CompressTest, Compress_default_axis) { + OpTester test("Compress", 9); + + test.AddInput("input", {3, 2}, { + 1.0f, 2.0f, + 3.0f, 4.0f, + 5.0f, 6.0f}); + test.AddInput("condition", {5}, {0, 1, 0, 0, 1}); + test.AddOutput("output", {2}, {2.0f, 5.0f}); + test.Run(); +} + +TEST(CompressTest, Compress0_string) { + OpTester test("Compress", 9); + + test.AddAttribute("axis", int64_t(0)); + + test.AddInput("input", {3, 2}, { + "1", "2", + "3", "4", + "5", "6"}); + test.AddInput("condition", {3}, {0, 1, 1}); + test.AddOutput("output", {2, 2}, { + "3", "4", + "5", "6"}); + test.Run(); +} + +TEST(CompressTest, Compress_default_axis_string) { + OpTester test("Compress", 9); + + test.AddInput("input", {3, 2}, { + "1", "2", + "3", "4", + "5", "6"}); + test.AddInput("condition", {5}, {0, 1, 0, 0, 1}); + test.AddOutput("output", {2}, {"2", "5"}); + test.Run(); +} + +} // namespace Test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/tensor/concat_op_test.cc b/onnxruntime/test/providers/cpu/tensor/concat_op_test.cc new file mode 100644 index 0000000000000..8977cc9fe4299 --- /dev/null +++ b/onnxruntime/test/providers/cpu/tensor/concat_op_test.cc @@ -0,0 +1,169 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(MathOpTest, Concat1D_string) { + OpTester test("Concat"); + test.AddAttribute("axis", int64_t{0}); + + test.AddInput("input1", {1}, {"1"}); + test.AddInput("input2", {2}, {"2", "3"}); + test.AddInput("input3", {4}, {"4", "5", "6", "7"}); + test.AddOutput("concat_result", {7}, {"1", "2", "3", "4", "5", "6", "7"}); + test.Run(); +} + +TEST(MathOpTest, Concat1D_int32) { + OpTester test("Concat"); + test.AddAttribute("axis", int64_t{0}); + + test.AddInput("input1", {1}, {1}); + test.AddInput("input2", {2}, {2, 3}); + test.AddInput("input3", {4}, {4, 5, 6, 7}); + test.AddOutput("concat_result", {7}, {1, 2, 3, 4, 5, 6, 7}); + test.Run(); +} + +TEST(MathOpTest, Concat1D) { + OpTester test("Concat"); + test.AddAttribute("axis", int64_t{0}); + + test.AddInput("input1", {1}, {1.0f}); + test.AddInput("input2", {2}, {2.0f, 3.0f}); + test.AddInput("input3", {4}, {4.0f, 5.0f, 6.0f, 7.0f}); + test.AddOutput("concat_result", {7}, {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f}); + test.Run(); +} + +TEST(MathOpTest, Concat2D_1) { + OpTester test("Concat"); + test.AddAttribute("axis", int64_t{0}); + + std::vector dims{1, 4}; + test.AddInput("input1", dims, {11.0f, 12.0f, 13.0f, 14.0f}); + test.AddInput("input2", dims, {21.0f, 22.0f, 23.0f, 24.0f}); + test.AddInput("input3", dims, {31.0f, 32.0f, 33.0f, 34.0f}); + test.AddOutput("concat_result", {3, 4}, + {11.0f, 12.0f, 13.0f, 14.0f, + 21.0f, 22.0f, 23.0f, 24.0f, + 31.0f, 32.0f, 33.0f, 34.0f}); + test.Run(); +} + +TEST(MathOpTest, Concat2D_2) { + OpTester test("Concat"); + test.AddAttribute("axis", int64_t{1}); + + std::vector dims{4, 1}; + test.AddInput("input1", dims, {11.0f, 21.0f, 31.0f, 41.0f}); + test.AddInput("input2", {4, 2}, {12.0f, 13.0f, 22.0f, 23.0f, 32.0f, 33.0f, 42.0f, 43.0f}); + test.AddInput("input3", dims, {14.0f, 24.0f, 34.0f, 44.0f}); + test.AddOutput("concat_result", {4, 4}, + {11.0f, 12.0f, 13.0f, 14.0f, + 21.0f, 22.0f, 23.0f, 24.0f, + 31.0f, 32.0f, 33.0f, 34.0f, + 41.0f, 42.0f, 43.0f, 44.0f}); + test.Run(); +} + +TEST(MathOpTest, Concat3D_1) { + OpTester test("Concat"); + test.AddAttribute("axis", int64_t{0}); + + std::vector dims{1, 3, 3}; + test.AddInput("input1", dims, + {111.0f, 112.0f, 113.0f, + 121.0f, 122.0f, 123.0f, + 131.0f, 132.0f, 133.0f}); + test.AddInput("input2", dims, + {211.0f, 212.0f, 213.0f, + 221.0f, 222.0f, 223.0f, + 231.0f, 232.0f, 233.0f}); + test.AddInput("input3", dims, + {311.0f, 312.0f, 313.0f, + 321.0f, 322.0f, 323.0f, + 331.0f, 332.0f, 333.0f}); + test.AddOutput("concat_result", {3, 3, 3}, + {111.0f, 112.0f, 113.0f, + 121.0f, 122.0f, 123.0f, + 131.0f, 132.0f, 133.0f, + + 211.0f, 212.0f, 213.0f, + 221.0f, 222.0f, 223.0f, + 231.0f, 232.0f, 233.0f, + + 311.0f, 312.0f, 313.0f, + 321.0f, 322.0f, 323.0f, + 331.0f, 332.0f, 333.0f}); + test.Run(); +} + +TEST(MathOpTest, Concat3D_2) { + OpTester test("Concat"); + test.AddAttribute("axis", int64_t{1}); + + std::vector dims{3, 1, 3}; + test.AddInput("input1", dims, + {111.0f, 112.0f, 113.0f, + 211.0f, 212.0f, 213.0f, + 311.0f, 312.0f, 313.0f}); + test.AddInput("input2", dims, + {121.0f, 122.0f, 123.0f, + 221.0f, 222.0f, 223.0f, + 321.0f, 322.0f, 323.0f}); + test.AddInput("input3", dims, + {131.0f, 132.0f, 133.0f, + 231.0f, 232.0f, 233.0f, + 331.0f, 332.0f, 333.0f}); + test.AddOutput("concat_result", {3, 3, 3}, + {111.0f, 112.0f, 113.0f, + 121.0f, 122.0f, 123.0f, + 131.0f, 132.0f, 133.0f, + + 211.0f, 212.0f, 213.0f, + 221.0f, 222.0f, 223.0f, + 231.0f, 232.0f, 233.0f, + + 311.0f, 312.0f, 313.0f, + 321.0f, 322.0f, 323.0f, + 331.0f, 332.0f, 333.0f}); + test.Run(); +} + +TEST(MathOpTest, Concat3D_3) { + OpTester test("Concat"); + test.AddAttribute("axis", int64_t{1}); + + std::vector dims{2, 2, 2}; + test.AddInput("input1", dims, + {1.0f, 2.0f, + 3.0f, 4.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f}); + test.AddInput("input2", dims, + {9.0f, 10.0f, + 11.0f, 12.0f, + + 13.0f, 14.0f, + 15.0f, 16.0f}); + test.AddOutput("concat_result", {2, 4, 2}, + {1.0f, 2.0f, + 3.0f, 4.0f, + 9.0f, 10.0f, + 11.0f, 12.0f, + + 5.0f, 6.0f, + 7.0f, 8.0f, + 13.0f, 14.0f, + 15.0f, 16.0f}); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/tensor/gather_op_test.cc b/onnxruntime/test/providers/cpu/tensor/gather_op_test.cc new file mode 100644 index 0000000000000..472107bf80cef --- /dev/null +++ b/onnxruntime/test/providers/cpu/tensor/gather_op_test.cc @@ -0,0 +1,306 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(GatherOpTest, Gather_axis0) { + OpTester test("Gather"); + test.AddAttribute("axis", 0LL); + test.AddInput("data", {2, 3, 4}, + {0.0f, 0.1f, 0.2f, 0.3f, + 1.0f, 1.1f, 1.2f, 1.3f, + 2.0f, 2.1f, 2.2f, 2.3f, + 10.0f, 10.1f, 10.2f, 10.3f, + 11.0f, 11.1f, 11.2f, 11.3f, + 12.0f, 12.1f, 12.2f, 12.3f}); + test.AddInput("indices", {1}, {1LL}); + test.AddOutput("output", {1, 3, 4}, + {10.0f, 10.1f, 10.2f, 10.3f, + 11.0f, 11.1f, 11.2f, 11.3f, + 12.0f, 12.1f, 12.2f, 12.3f}); + test.Run(); +} + +TEST(GatherOpTest, Gather_negative_axis) { + OpTester test("Gather"); + test.AddAttribute("axis", -3LL); + test.AddInput("data", {2, 3, 4}, + {0.0f, 0.1f, 0.2f, 0.3f, + 1.0f, 1.1f, 1.2f, 1.3f, + 2.0f, 2.1f, 2.2f, 2.3f, + 10.0f, 10.1f, 10.2f, 10.3f, + 11.0f, 11.1f, 11.2f, 11.3f, + 12.0f, 12.1f, 12.2f, 12.3f}); + test.AddInput("indices", {1}, {1LL}); + test.AddOutput("output", {1, 3, 4}, + {10.0f, 10.1f, 10.2f, 10.3f, + 11.0f, 11.1f, 11.2f, 11.3f, + 12.0f, 12.1f, 12.2f, 12.3f}); + test.Run(); +} + +TEST(GatherOpTest, Gather_invalid_axis) { + OpTester test("Gather"); + // Invalid axis not in range [-r, r-1] + test.AddAttribute("axis", -10LL); + test.AddInput("data", {2, 3, 4}, + {0.0f, 0.1f, 0.2f, 0.3f, + 1.0f, 1.1f, 1.2f, 1.3f, + 2.0f, 2.1f, 2.2f, 2.3f, + 10.0f, 10.1f, 10.2f, 10.3f, + 11.0f, 11.1f, 11.2f, 11.3f, + 12.0f, 12.1f, 12.2f, 12.3f}); + test.AddInput("indices", {1}, {1LL}); + test.AddOutput("output", {1, 3, 4}, + {10.0f, 10.1f, 10.2f, 10.3f, + 11.0f, 11.1f, 11.2f, 11.3f, + 12.0f, 12.1f, 12.2f, 12.3f}); + test.Run(OpTester::ExpectResult::kExpectFailure, "axis must be in [-r, r-1]"); +} + +TEST(GatherOpTest, Gather_invalid_index_cpu) { + OpTester test("Gather"); + // Invalid index 3. data[3] does not exist. + test.AddAttribute("axis", 0LL); + test.AddInput("data", {3, 4}, + {0.0f, 1.0f, 2.0f, 3.0f, + 4.0f, 5.0f, 6.0f, 7.0f, + 8.0f, 9.0f, 10.0f, 11.0f}); + test.AddInput("indices", {3}, {0LL, 1L, 1000L}); + test.AddOutput("output", {1}, {1.0f}); + + test.Run(OpTester::ExpectResult::kExpectFailure, "Mismatch between number of source and target dimensions."); +} + +#ifdef USE_CUDA +TEST(GatherOpTest, Gather_invalid_index_gpu) { + OpTester test("Gather"); + // Invalid index 3. data[3] does not exist. + test.AddAttribute("axis", 0LL); + test.AddInput("data", {3, 4}, + {0.0f, 1.0f, 2.0f, 3.0f, + 4.0f, 5.0f, 6.0f, 7.0f, + 8.0f, 9.0f, 10.0f, 11.0f}); + test.AddInput("indices", {3}, {0LL, 1LL, 1000LL}); + test.AddOutput("output", {3, 4}, + {0.0f, 1.0f, 2.0f, 3.0f, + 4.0f, 5.0f, 6.0f, 7.0f, + 0.0f, 0.0f, 0.0f, 0.0f}); + + //On GPU, just set the value to 0 instead of report error. exclude all other providers + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCpuExecutionProvider, kMklDnnExecutionProvider, kNupharExecutionProvider}); +} +#endif + +TEST(GatherOpTest, Gather_axis1) { + OpTester test("Gather"); + test.AddAttribute("axis", 1LL); + test.AddInput("data", {2, 3, 4}, + {0.0f, 0.1f, 0.2f, 0.3f, + 1.0f, 1.1f, 1.2f, 1.3f, + 2.0f, 2.1f, 2.2f, 2.3f, + 10.0f, 10.1f, 10.2f, 10.3f, + 11.0f, 11.1f, 11.2f, 11.3f, + 12.0f, 12.1f, 12.2f, 12.3f}); + test.AddInput("indices", {2}, {2LL, 0LL}); + test.AddOutput("output", {2, 2, 4}, + {2.0f, 2.1f, 2.2f, 2.3f, + 0.0f, 0.1f, 0.2f, 0.3f, + 12.0f, 12.1f, 12.2f, 12.3f, + 10.0f, 10.1f, 10.2f, 10.3f}); + test.Run(); +} + +TEST(GatherOpTest, Gather_axis2) { + OpTester test("Gather"); + test.AddAttribute("axis", 2LL); + test.AddInput("data", {2, 3, 4}, + {0.0f, 0.1f, 0.2f, 0.3f, + 1.0f, 1.1f, 1.2f, 1.3f, + 2.0f, 2.1f, 2.2f, 2.3f, + 10.0f, 10.1f, 10.2f, 10.3f, + 11.0f, 11.1f, 11.2f, 11.3f, + 12.0f, 12.1f, 12.2f, 12.3f}); + test.AddInput("indices", {3}, {1LL, 0LL, 2LL}); + test.AddOutput("output", {2, 3, 3}, + {0.1f, 0.0f, 0.2f, + 1.1f, 1.0f, 1.2f, + 2.1f, 2.0f, 2.2f, + 10.1f, 10.0f, 10.2f, + 11.1f, 11.0f, 11.2f, + 12.1f, 12.0f, 12.2f}); + test.Run(); +} + +TEST(GatherOpTest, Gather_axis0_indices2d) { + OpTester test("Gather"); + test.AddAttribute("axis", 0LL); + test.AddInput("data", {3, 3}, + {0.0f, 0.1f, 0.2f, + 1.0f, 1.1f, 1.2f, + 2.0f, 2.1f, 2.2f}); + test.AddInput("indices", {2LL, 2LL}, + {1LL, 0LL, + 2LL, 1LL}); + test.AddOutput("output", {2, 2, 3}, + {1.0f, 1.1f, 1.2f, 0.0f, 0.1f, 0.2f, + 2.0f, 2.1f, 2.2f, 1.0f, 1.1f, 1.2f}); + test.Run(); +} + +TEST(GatherOpTest, Gather_axis1_indices2d) { + OpTester test("Gather"); + test.AddAttribute("axis", 1LL); + test.AddInput("data", {3, 3}, + {0.0f, 0.1f, 0.2f, + 1.0f, 1.1f, 1.2f, + 2.0f, 2.1f, 2.2f}); + test.AddInput("indices", {2LL, 2LL}, + {1LL, 0LL, + 2LL, 1LL}); + test.AddOutput("output", {3, 2, 2}, + {0.1f, 0.0f, 0.2f, 0.1f, + 1.1f, 1.0f, 1.2f, 1.1f, + 2.1f, 2.0f, 2.2f, 2.1f}); + test.Run(); +} + +TEST(GatherOpTest, Gather_axis1_indices2d_int32) { + OpTester test("Gather"); + test.AddAttribute("axis", 1LL); + test.AddInput("data", {3, 3}, + {0, 1, 2, + 10, 11, 12, + 20, 21, 22}); + test.AddInput("indices", {2, 2}, + {1, 0, + 2, 1}); + test.AddOutput("output", {3, 2, 2}, + {1, 0, 2, 1, + 11, 10, 12, 11, + 21, 20, 22, 21}); + test.Run(); +} + +TEST(GatherOpTest, Gather_axis1_indices2d_uint32) { + OpTester test("Gather"); + test.AddAttribute("axis", 1LL); + test.AddInput("data", {3, 3}, + {0, 1, 2, + 10, 11, 12, + 20, 21, 22}); + test.AddInput("indices", {2, 2}, + {1, 0, + 2, 1}); + test.AddOutput("output", {3, 2, 2}, + {1, 0, 2, 1, + 11, 10, 12, 11, + 21, 20, 22, 21}); + test.Run(); +} + +TEST(GatherOpTest, Gather_axis1_indices2d_int16) { + OpTester test("Gather"); + test.AddAttribute("axis", 1LL); + test.AddInput("data", {3, 3}, + {0, 1, 2, + 10, 11, 12, + 20, 21, 22}); + test.AddInput("indices", {2, 2}, + {1, 0, + 2, 1}); + test.AddOutput("output", {3, 2, 2}, + {1, 0, 2, 1, + 11, 10, 12, 11, + 21, 20, 22, 21}); + test.Run(); +} + +TEST(GatherOpTest, Gather_axis1_indices2d_uint16) { + OpTester test("Gather"); + test.AddAttribute("axis", 1LL); + test.AddInput("data", {3, 3}, + {0, 1, 2, + 10, 11, 12, + 20, 21, 22}); + test.AddInput("indices", {2, 2}, + {1, 0, + 2, 1}); + test.AddOutput("output", {3, 2, 2}, + {1, 0, 2, 1, + 11, 10, 12, 11, + 21, 20, 22, 21}); + test.Run(); +} + +TEST(GatherOpTest, Gather_axis1_indices2d_int8) { + OpTester test("Gather"); + test.AddAttribute("axis", 1LL); + test.AddInput("data", {3, 3}, + {0, 1, 2, + 10, 11, 12, + 20, 21, 22}); + test.AddInput("indices", {2, 2}, + {1, 0, + 2, 1}); + test.AddOutput("output", {3, 2, 2}, + {1, 0, 2, 1, + 11, 10, 12, 11, + 21, 20, 22, 21}); + test.Run(); +} + +TEST(GatherOpTest, Gather_axis1_indices2d_string) { + OpTester test("Gather"); + test.AddAttribute("axis", 1LL); + test.AddInput("data", {3, 3}, + {"0", "1", "2", + "10", "11", "12", + "20", "21", "22"}); + test.AddInput("indices", {2, 2}, + {1, 0, + 2, 1}); + test.AddOutput("output", {3, 2, 2}, + {"1", "0", "2", "1", + "11", "10", "12", "11", + "21", "20", "22", "21"}); + test.Run(); +} + +TEST(GatherOpTest, Gather_axis1_indices2d_bool) { + OpTester test("Gather"); + test.AddAttribute("axis", 1LL); + test.AddInput("data", {3, 3}, + {true, false, true, + true, true, false, + false, true, false}); + test.AddInput("indices", {2, 2}, + {1, 0, + 2, 1}); + test.AddOutput("output", {3, 2, 2}, + {false, true, true, false, + true, true, false, true, + true, false, false, true}); + test.Run(); +} + +TEST(GatherOpTest, Gather_perf) { + OpTester test("Gather"); + test.AddAttribute("axis", 0LL); + std::vector input(50000 * 100, 1); + + std::vector indices(800, 5); + + std::vector output(800 * 100, 1); + + test.AddInput("data", {50000, 100}, input); + test.AddInput("indices", {800, 1}, indices); + test.AddOutput("output", {800, 1, 100}, output); + test.Run(); +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/tensor/pad_test.cc b/onnxruntime/test/providers/cpu/tensor/pad_test.cc new file mode 100644 index 0000000000000..8a625436ba90c --- /dev/null +++ b/onnxruntime/test/providers/cpu/tensor/pad_test.cc @@ -0,0 +1,118 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(TensorOpTest, Pad_Constant_1D) { + OpTester test("Pad"); + + test.AddAttribute("pads", std::vector{1, 2}); + test.AddAttribute("value", 1234.0f); + test.AddInput("data", {2}, {1.0f, 2.0f}); + test.AddOutput("output", {5}, {1234.0f, 1.0f, 2.0f, 1234.0f, 1234.0f}); + test.Run(); +} + +TEST(TensorOpTest, Pad_Constant_1D_Zero) { + OpTester test("Pad"); + + test.AddAttribute("pads", std::vector{0, 0}); + test.AddAttribute("value", 1234.0f); + test.AddInput("data", {2}, {1.0f, 2.0f}); + test.AddOutput("output", {2}, {1.0f, 2.0f}); + test.Run(); +} + +TEST(TensorOpTest, Pad_Constant_2D) { + OpTester test("Pad"); + + test.AddAttribute("pads", std::vector{1, 2, 1, 2}); + test.AddAttribute("value", 1234.0f); + test.AddInput("data", {2, 2}, + {11.0f, 21.0f, + 12.0f, 22.0f}); + test.AddOutput("output", {4, 6}, + {1234.0f, 1234.0f, 1234.0f, 1234.0f, 1234.0f, 1234.0f, + 1234.0f, 1234.0f, 11.0f, 21.0f, 1234.0f, 1234.0f, + 1234.0f, 1234.0f, 12.0f, 22.0f, 1234.0f, 1234.0f, + 1234.0f, 1234.0f, 1234.0f, 1234.0f, 1234.0f, 1234.0f}); + test.Run(); +} + +TEST(TensorOpTest, Pad_Edge_2D) { + OpTester test("Pad"); + + test.AddAttribute("pads", std::vector{2, 2, 2, 2}); + test.AddAttribute("mode", "edge"); + test.AddInput("data", {2, 3}, + {11.0f, 21.0f, 31.0f, + 12.0f, 22.0f, 32.0f}); + test.AddOutput("output", {6, 7}, + {11.0f, 11.0f, 11.0f, 21.0f, 31.0f, 31.0f, 31.0f, + 11.0f, 11.0f, 11.0f, 21.0f, 31.0f, 31.0f, 31.0f, + 11.0f, 11.0f, 11.0f, 21.0f, 31.0f, 31.0f, 31.0f, + 12.0f, 12.0f, 12.0f, 22.0f, 32.0f, 32.0f, 32.0f, + 12.0f, 12.0f, 12.0f, 22.0f, 32.0f, 32.0f, 32.0f, + 12.0f, 12.0f, 12.0f, 22.0f, 32.0f, 32.0f, 32.0f}); + test.Run(); +} + +TEST(TensorOpTest, Pad_Edge_3D) { + OpTester test("Pad"); + + test.AddAttribute("pads", std::vector{1, 2, 2, 1, 2, 2}); + test.AddAttribute("mode", "edge"); + test.AddInput("data", {1, 2, 3}, + {11.0f, 21.0f, 31.0f, + 12.0f, 22.0f, 32.0f}); + test.AddOutput("output", {3, 6, 7}, + {11.0f, 11.0f, 11.0f, 21.0f, 31.0f, 31.0f, 31.0f, + 11.0f, 11.0f, 11.0f, 21.0f, 31.0f, 31.0f, 31.0f, + 11.0f, 11.0f, 11.0f, 21.0f, 31.0f, 31.0f, 31.0f, + 12.0f, 12.0f, 12.0f, 22.0f, 32.0f, 32.0f, 32.0f, + 12.0f, 12.0f, 12.0f, 22.0f, 32.0f, 32.0f, 32.0f, + 12.0f, 12.0f, 12.0f, 22.0f, 32.0f, 32.0f, 32.0f, + + 11.0f, 11.0f, 11.0f, 21.0f, 31.0f, 31.0f, 31.0f, + 11.0f, 11.0f, 11.0f, 21.0f, 31.0f, 31.0f, 31.0f, + 11.0f, 11.0f, 11.0f, 21.0f, 31.0f, 31.0f, 31.0f, + 12.0f, 12.0f, 12.0f, 22.0f, 32.0f, 32.0f, 32.0f, + 12.0f, 12.0f, 12.0f, 22.0f, 32.0f, 32.0f, 32.0f, + 12.0f, 12.0f, 12.0f, 22.0f, 32.0f, 32.0f, 32.0f, + + 11.0f, 11.0f, 11.0f, 21.0f, 31.0f, 31.0f, 31.0f, + 11.0f, 11.0f, 11.0f, 21.0f, 31.0f, 31.0f, 31.0f, + 11.0f, 11.0f, 11.0f, 21.0f, 31.0f, 31.0f, 31.0f, + 12.0f, 12.0f, 12.0f, 22.0f, 32.0f, 32.0f, 32.0f, + 12.0f, 12.0f, 12.0f, 22.0f, 32.0f, 32.0f, 32.0f, + 12.0f, 12.0f, 12.0f, 22.0f, 32.0f, 32.0f, 32.0f}); + + test.Run(); +} + +TEST(TensorOpTest, Pad_Reflect_2D) { + OpTester test("Pad"); + + test.AddAttribute("pads", std::vector{2, 2, 2, 2}); + test.AddAttribute("mode", "reflect"); + test.AddInput("data", {3, 3}, + {11.0f, 21.0f, 31.0f, + 12.0f, 22.0f, 32.0f, + 13.0f, 23.0f, 33.0f}); + test.AddOutput("output", {7, 7}, + {33.0f, 23.0f, 13.0f, 23.0f, 33.0f, 23.0f, 13.0f, + 32.0f, 22.0f, 12.0f, 22.0f, 32.0f, 22.0f, 12.0f, + 31.0f, 21.0f, 11.0f, 21.0f, 31.0f, 21.0f, 11.0f, + 32.0f, 22.0f, 12.0f, 22.0f, 32.0f, 22.0f, 12.0f, + 33.0f, 23.0f, 13.0f, 23.0f, 33.0f, 23.0f, 13.0f, + 32.0f, 22.0f, 12.0f, 22.0f, 32.0f, 22.0f, 12.0f, + 31.0f, 21.0f, 11.0f, 21.0f, 31.0f, 21.0f, 11.0f}); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/tensor/size_test.cc b/onnxruntime/test/providers/cpu/tensor/size_test.cc new file mode 100644 index 0000000000000..48fa6691e0710 --- /dev/null +++ b/onnxruntime/test/providers/cpu/tensor/size_test.cc @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +template +void TestSizeOp(const std::initializer_list& dims) { + OpTester test("Size"); + + int64_t actual_size = 1; + for (auto dim : dims) + actual_size *= dim; + + std::vector dim_vector(dims); + std::vector input(actual_size); + test.AddInput("A", dim_vector, input); + + std::vector scalar_dims; + test.AddOutput("B", scalar_dims, {actual_size}); + + test.Run(); +} + +// Single-dimensional float tensor +TEST(SizeOpTest, Float1000Test) { + TestSizeOp({1000}); +} + +// Two-dimensional float tensor +TEST(SizeOpTest, Float3x3Test) { + TestSizeOp({3, 3}); +} + +// Three-dimensional float tensor +TEST(SizeOpTest, Float3x3x4Test) { + TestSizeOp({3, 3, 4}); +} + +// Int tensor +TEST(SizeOpTest, Int3x3Test) { + TestSizeOp({3, 3}); +} + +// Int scalar +TEST(SizeOpTest, IntScalarTest) { + TestSizeOp({}); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/tensor/slice_op.test.cc b/onnxruntime/test/providers/cpu/tensor/slice_op.test.cc new file mode 100644 index 0000000000000..65f44d595b722 --- /dev/null +++ b/onnxruntime/test/providers/cpu/tensor/slice_op.test.cc @@ -0,0 +1,138 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(SliceTest, Slice1D) { + OpTester test("Slice"); + + test.AddAttribute("axes", std::vector{0}); + test.AddAttribute("starts", std::vector{2}); + test.AddAttribute("ends", std::vector{4}); + + test.AddInput("data", {6}, {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f}); + test.AddOutput("output", {2}, {2.0f, 3.0f}); + test.Run(); +} + +TEST(SliceTest, Slice1D_Perf) { + OpTester test("Slice"); + + test.AddAttribute("axes", std::vector{0}); + test.AddAttribute("starts", std::vector{2}); + test.AddAttribute("ends", std::vector{502}); + + std::vector input(1000, 2.0f); + std::vector output(500, 2.0f); + test.AddInput("data", {1000}, input); + test.AddOutput("output", {500}, output); + test.Run(); +} + +TEST(SliceTest, Slice2D_OutOfBounds) { + OpTester test("Slice"); + + test.AddAttribute("axes", std::vector{0, 1}); + test.AddAttribute("starts", std::vector{0, 1000}); + test.AddAttribute("ends", std::vector{10, 1000}); + + test.AddInput("data", {2, 3}, {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}); + test.AddOutput("output", {2, 0}, {}); + test.Run(); +} + +TEST(SliceTest, Slice2D_OneAxis) { + OpTester test("Slice"); + + test.AddAttribute("axes", std::vector{0}); + test.AddAttribute("starts", std::vector{1}); + test.AddAttribute("ends", std::vector{3}); + + test.AddInput("data", {6, 4}, + {00.0f, 01.0f, 02.0f, 03.0f, + 10.0f, 11.0f, 12.0f, 13.0f, + 20.0f, 21.0f, 22.0f, 23.0f, + 30.0f, 31.0f, 32.0f, 33.0f, + 40.0f, 41.0f, 42.0f, 43.0f, + 50.0f, 51.0f, 52.0f, 53.0f}); + test.AddOutput("output", {2, 4}, + {10.0f, 11.0f, 12.0f, 13.0f, + 20.0f, 21.0f, 22.0f, 23.0f}); + test.Run(); +} + +TEST(SliceTest, Slice2D_TwoAxes) { + OpTester test("Slice"); + + test.AddAttribute("axes", std::vector{1, 0}); + test.AddAttribute("starts", std::vector{2, 3}); + test.AddAttribute("ends", std::vector{1000, -1}); + + test.AddInput("data", {6, 4}, + {00.0f, 01.0f, 02.0f, 03.0f, + 10.0f, 11.0f, 12.0f, 13.0f, + 20.0f, 21.0f, 22.0f, 23.0f, + 30.0f, 31.0f, 32.0f, 33.0f, + 40.0f, 41.0f, 42.0f, 43.0f, + 50.0f, 51.0f, 52.0f, 53.0f}); + test.AddOutput("output", {2, 2}, + {32.0f, 33.0f, + 42.0f, 43.0f}); + test.Run(); +} + +TEST(SliceTest, Slice2D_TwoAxesEque) { + OpTester test("Slice"); + + test.AddAttribute("axes", std::vector{1, 0}); + test.AddAttribute("starts", std::vector{2, 3}); + test.AddAttribute("ends", std::vector{1000, 3}); + + test.AddInput("data", {6, 4}, + {00.0f, 01.0f, 02.0f, 03.0f, + 10.0f, 11.0f, 12.0f, 13.0f, + 20.0f, 21.0f, 22.0f, 23.0f, + 30.0f, 31.0f, 32.0f, 33.0f, + 40.0f, 41.0f, 42.0f, 43.0f, + 50.0f, 51.0f, 52.0f, 53.0f}); + test.AddOutput("output", {0, 2}, + {}); + test.Run(); +} + +TEST(SliceTest, Slice3D) { + OpTester test("Slice"); + + test.AddAttribute("starts", std::vector{0, 1, 1}); + test.AddAttribute("ends", std::vector{1000, 1000, 1000}); + + test.AddInput("data", {3, 3, 3}, + {111.0f, 112.0f, 113.0f, + 121.0f, 122.0f, 123.0f, + 131.0f, 132.0f, 133.0f, + + 211.0f, 212.0f, 213.0f, + 221.0f, 222.0f, 223.0f, + 231.0f, 232.0f, 233.0f, + + 311.0f, 312.0f, 313.0f, + 321.0f, 322.0f, 323.0f, + 331.0f, 332.0f, 333.0f}); + test.AddOutput("output", {3, 2, 2}, + {122.0f, 123.0f, + 132.0f, 133.0f, + + 222.0f, 223.0f, + 232.0f, 233.0f, + + 322.0f, 323.0f, + 332.0f, 333.0f}); + test.Run(); +} + +} // namespace Test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/tensor/space_depth_ops_test.cc b/onnxruntime/test/providers/cpu/tensor/space_depth_ops_test.cc new file mode 100644 index 0000000000000..8db47d2a19436 --- /dev/null +++ b/onnxruntime/test/providers/cpu/tensor/space_depth_ops_test.cc @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" +#include "core/providers/cpu/tensor/space_depth_ops.h" + +namespace onnxruntime { +namespace test { + +TEST(TensorOpTest, SpaceToDepthTest_1) { + OpTester test("SpaceToDepth"); + const int64_t blocksize = 2; + test.AddAttribute("blocksize", blocksize); + const int64_t N = 1, C = 2, H = 2, W = 4; + const std::vector X = + {0.0f, 0.1f, 0.2f, 0.3f, + 1.0f, 1.1f, 1.2f, 1.3f, + + 2.0f, 2.1f, 2.2f, 2.3f, + 3.0f, 3.1f, 3.2f, 3.3f}; + + test.AddInput("input", {N, C, H, W}, X); + + const std::vector result = { + 0.0f, 0.2f, + 2.0f, 2.2f, + + 0.1f, 0.3f, + 2.1f, 2.3f, + + 1.0f, 1.2f, + 3.0f, 3.2f, + + 1.1f, 1.3f, + 3.1f, 3.3f}; + test.AddOutput("output", {N, C * blocksize * blocksize, H / blocksize, W / blocksize}, result); + test.Run(); +} + +TEST(TensorOpTest, SpaceToDepthTest_2) { + OpTester test("SpaceToDepth"); + const int64_t blocksize = 3; + test.AddAttribute("blocksize", blocksize); + const int64_t N = 2, C = 3, H = 3, W = 6; + const std::vector X = { + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., + 11., 12., 13., 14., 15., 16., 17., 18., 19., 20., 21., + 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., + 33., 34., 35., 36., 37., 38., 39., 40., 41., 42., 43., + 44., 45., 46., 47., 48., 49., 50., 51., 52., 53., 54., + 55., 56., 57., 58., 59., 60., 61., 62., 63., 64., 65., + 66., 67., 68., 69., 70., 71., 72., 73., 74., 75., 76., + 77., 78., 79., 80., 81., 82., 83., 84., 85., 86., 87., + 88., 89., 90., 91., 92., 93., 94., 95., 96., 97., 98., + 99., 100., 101., 102., 103., 104., 105., 106., 107.}; + + test.AddInput("input", {N, C, H, W}, X); + + const std::vector result = { + 0., 3., 18., 21., 36., 39., 1., 4., 19., 22., 37., + 40., 2., 5., 20., 23., 38., 41., 6., 9., 24., 27., + 42., 45., 7., 10., 25., 28., 43., 46., 8., 11., 26., + 29., 44., 47., 12., 15., 30., 33., 48., 51., 13., 16., + 31., 34., 49., 52., 14., 17., 32., 35., 50., 53., 54., + 57., 72., 75., 90., 93., 55., 58., 73., 76., 91., 94., + 56., 59., 74., 77., 92., 95., 60., 63., 78., 81., 96., + 99., 61., 64., 79., 82., 97., 100., 62., 65., 80., 83., + 98., 101., 66., 69., 84., 87., 102., 105., 67., 70., 85., + 88., 103., 106., 68., 71., 86., 89., 104., 107.}; + test.AddOutput("output", {2, 27, 1, 2}, result); + test.Run(); +} + +TEST(TensorOpTest, DepthToSpaceTest_1) { + OpTester test("DepthToSpace"); + const int64_t blocksize = 2; + test.AddAttribute("blocksize", blocksize); + + const int64_t N = 1, C = 8, H = 1, W = 2; + const std::vector X = { + 0.0f, 0.2f, + 2.0f, 2.2f, + + 0.1f, 0.3f, + 2.1f, 2.3f, + + 1.0f, 1.2f, + 3.0f, 3.2f, + + 1.1f, 1.3f, + 3.1f, 3.3f}; + + test.AddInput("input", {N, C, H, W}, X); + + const std::vector result = + {0.0f, 0.1f, 0.2f, 0.3f, + 1.0f, 1.1f, 1.2f, 1.3f, + + 2.0f, 2.1f, 2.2f, 2.3f, + 3.0f, 3.1f, 3.2f, 3.3f}; + test.AddOutput("output", {N, C / (blocksize * blocksize), H * blocksize, W * blocksize}, result); + test.Run(); +} + +TEST(TensorOpTest, DepthToSpaceTest_2) { + OpTester test("DepthToSpace"); + const int64_t blocksize = 2; + test.AddAttribute("blocksize", blocksize); + + const int64_t N = 2, C = 12, H = 3, W = 2; + const std::vector X = { + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., + 11., 12., 13., 14., 15., 16., 17., 18., 19., 20., 21., + 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., + 33., 34., 35., 36., 37., 38., 39., 40., 41., 42., 43., + 44., 45., 46., 47., 48., 49., 50., 51., 52., 53., 54., + 55., 56., 57., 58., 59., 60., 61., 62., 63., 64., 65., + 66., 67., 68., 69., 70., 71., 72., 73., 74., 75., 76., + 77., 78., 79., 80., 81., 82., 83., 84., 85., 86., 87., + 88., 89., 90., 91., 92., 93., 94., 95., 96., 97., 98., + 99., 100., 101., 102., 103., 104., 105., 106., 107., 108., 109., + 110., 111., 112., 113., 114., 115., 116., 117., 118., 119., 120., + 121., 122., 123., 124., 125., 126., 127., 128., 129., 130., 131., + 132., 133., 134., 135., 136., 137., 138., 139., 140., 141., 142., + 143.}; + + test.AddInput("input", {N, C, H, W}, X); + + const std::vector result = { + 0., 18., 1., 19., 36., 54., 37., 55., 2., 20., 3., + 21., 38., 56., 39., 57., 4., 22., 5., 23., 40., 58., + 41., 59., 6., 24., 7., 25., 42., 60., 43., 61., 8., + 26., 9., 27., 44., 62., 45., 63., 10., 28., 11., 29., + 46., 64., 47., 65., 12., 30., 13., 31., 48., 66., 49., + 67., 14., 32., 15., 33., 50., 68., 51., 69., 16., 34., + 17., 35., 52., 70., 53., 71., 72., 90., 73., 91., 108., + 126., 109., 127., 74., 92., 75., 93., 110., 128., 111., 129., + 76., 94., 77., 95., 112., 130., 113., 131., 78., 96., 79., + 97., 114., 132., 115., 133., 80., 98., 81., 99., 116., 134., + 117., 135., 82., 100., 83., 101., 118., 136., 119., 137., 84., + 102., 85., 103., 120., 138., 121., 139., 86., 104., 87., 105., + 122., 140., 123., 141., 88., 106., 89., 107., 124., 142., 125., + 143.}; + test.AddOutput("output", {2, 3, 6, 4}, result); + test.Run(); +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/tensor/split_op_test.cc b/onnxruntime/test/providers/cpu/tensor/split_op_test.cc new file mode 100644 index 0000000000000..c2a4d07365298 --- /dev/null +++ b/onnxruntime/test/providers/cpu/tensor/split_op_test.cc @@ -0,0 +1,384 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +using ShapeAndData = std::pair, const std::vector>; +using ExpectResult = OpTester::ExpectResult; + +void RunTest(int64_t axis, const std::vector split_sizes, const ShapeAndData& input, + const std::vector& outputs, + bool expect_failure = false, const std::string& err_msg = {}) { + OpTester test("Split"); + + test.AddAttribute("axis", axis); + + if (!split_sizes.empty()) + test.AddAttribute("split", split_sizes); + + test.AddInput("input", input.first, input.second); + + int i = 0; + for (auto& output : outputs) { + auto& shape = output.first; + auto& data = output.second; + std::ostringstream oss; + oss << "output" << i++; + test.AddOutput(oss.str().c_str(), shape, data); + } + + test.Run(expect_failure ? ExpectResult::kExpectFailure : ExpectResult::kExpectSuccess, err_msg); +} + +TEST(SplitOperatorTest, Axis0EqualSplit) { + const int64_t axis = 0; + std::vector outputs; + + // input shape and data + ShapeAndData input = {{4, 2}, // shape + {1.f, 2.f, + 3.f, 4.f, + 5.f, 6.f, + 7.f, 8.f}}; + + outputs.push_back({{2, 2}, + {1.f, 2.f, + 3.f, 4.f}}); + + outputs.push_back({{2, 2}, + {5.f, 6.f, + 7.f, 8.f}}); + + RunTest(axis, {}, input, outputs); +} + +TEST(SplitOperatorTest, Axis0UnequalSplit) { + const int64_t axis = 0; + std::vector outputs; + + // input shape and data + ShapeAndData input = {{4, 2}, // shape + {1.f, 2.f, + 3.f, 4.f, + 5.f, 6.f, + 7.f, 8.f}}; + + std::vector splits{1, 3}; + + outputs.push_back({{1, 2}, {1.f, 2.f}}); + + outputs.push_back({{3, 2}, + {3.f, 4.f, + 5.f, 6.f, + 7.f, 8.f}}); + + RunTest(axis, splits, input, outputs); +} + +TEST(SplitOperatorTest, Axis1EqualSplit) { + const int64_t axis = 1; + std::vector outputs; + + // input shape and data + ShapeAndData input = {{2, 4}, + {1.f, 2.f, 3.f, 4.f, + 5.f, 6.f, 7.f, 8.f}}; + + outputs.push_back({{2, 2}, + {1.f, 2.f, + 5.f, 6.f}}); + + outputs.push_back({{2, 2}, + {3.f, 4.f, + 7.f, 8.f}}); + + RunTest(axis, {}, input, outputs); +} + +TEST(SplitOperatorTest, Axis1UnequalSplit) { + const int64_t axis = 1; + std::vector outputs; + + // input shape and data + ShapeAndData input = {{2, 4}, + {1.f, 2.f, 3.f, 4.f, + 5.f, 6.f, 7.f, 8.f}}; + + std::vector splits{3, 1}; + + outputs.push_back({{2, 3}, + {1.f, 2.f, 3.f, + 5.f, 6.f, 7.f}}); + + outputs.push_back({{2, 1}, + {4.f, + 8.f}}); + + RunTest(axis, splits, input, outputs); +} + +ShapeAndData CreateInput(std::vector shape) { + auto size = TensorShape(shape).Size(); + + float i = 0.f, increment = 1.f; + // generate the elements for the data starting at 1.f + std::vector data; + std::generate_n(std::back_inserter(data), size, [&]() { return i += increment; }); + + ShapeAndData input = {shape, data}; + + return input; +} + +TEST(SplitOperatorTest, Axis2EqualSplit) { + const int64_t axis = 2; + std::vector outputs; + + ShapeAndData input = CreateInput({2, 2, 6}); + + outputs.push_back({{2, 2, 2}, + {1.f, 2.f, + 7.f, 8.f, + + 13.f, 14.f, + 19.f, 20.f}}); + + outputs.push_back({{2, 2, 2}, + {3.f, 4.f, + 9.f, 10.f, + + 15.f, 16.f, + 21.f, 22.f}}); + + outputs.push_back({{2, 2, 2}, + {5.f, 6.f, + 11.f, 12.f, + + 17.f, 18.f, + 23.f, 24.f}}); + + RunTest(axis, {}, input, outputs); +} + +TEST(SplitOperatorTest, Axis2UnequalSplit) { + const int64_t axis = 2; + std::vector outputs; + + ShapeAndData input = CreateInput({2, 2, 6}); + + std::vector splits{1, 2, 3}; + + outputs.push_back({{2, 2, 1}, + {1.f, + 7.f, + + 13.f, + 19.f}}); + + outputs.push_back({{2, 2, 2}, + {2.f, 3.f, + 8.f, 9.f, + + 14.f, 15.f, + 20.f, 21.f}}); + + outputs.push_back({{2, 2, 3}, + {4.f, 5.f, 6.f, + 10.f, 11.f, 12.f, + + 16.f, 17.f, 18.f, + 22.f, 23.f, 24.f}}); + + RunTest(axis, splits, input, outputs); +} + +// test a split of a dimension that has leading and trailing dimensions +TEST(SplitOperatorTest, Axis1SplitMiddleDimensionEqually) { + const int64_t axis = 1; + std::vector outputs; + + ShapeAndData input = CreateInput({2, 4, 4}); + + outputs.push_back({{2, 2, 4}, + {1.f, 2.f, 3.f, 4.f, + 5.f, 6.f, 7.f, 8.f, + + 17.f, 18.f, 19.f, 20.f, + 21.f, 22.f, 23.f, 24.f}}); + + outputs.push_back({{2, 2, 4}, + {9.f, 10.f, 11.f, 12.f, + 13.f, 14.f, 15.f, 16.f, + + 25.f, 26.f, 27.f, 28.f, + 29.f, 30.f, 31.f, 32.f}}); + + RunTest(axis, {}, input, outputs); +} + +// test a split of a dimension that has leading and trailing dimensions +TEST(SplitOperatorTest, Axis1SplitMiddleDimensionUnequally) { + const int64_t axis = 1; + std::vector outputs; + + ShapeAndData input = CreateInput({2, 4, 4}); + + std::vector splits{1, 3}; + + outputs.push_back({{2, 1, 4}, + {1.f, 2.f, 3.f, 4.f, + + 17.f, 18.f, 19.f, 20.f}}); + + outputs.push_back({{2, 3, 4}, + {5.f, 6.f, 7.f, 8.f, + 9.f, 10.f, 11.f, 12.f, + 13.f, 14.f, 15.f, 16.f, + + 21.f, 22.f, 23.f, 24.f, + 25.f, 26.f, 27.f, 28.f, + 29.f, 30.f, 31.f, 32.f}}); + + RunTest(axis, splits, input, outputs); +} + +TEST(SplitOperatorTest, NegativeAxis) { + const int64_t axis = -1; // split last axis equally + std::vector outputs; + + // input shape and data + ShapeAndData input = {{2, 4}, + {1.f, 2.f, 3.f, 4.f, + 5.f, 6.f, 7.f, 8.f}}; + + outputs.push_back({{2, 2}, + {1.f, 2.f, + 5.f, 6.f}}); + + outputs.push_back({{2, 2}, + {3.f, 4.f, + 7.f, 8.f}}); + + RunTest(axis, {}, input, outputs); +} + +TEST(SplitOperatorTest, InvalidAxis) { + const int64_t axis = 2; + std::vector outputs; + + // input shape and data + ShapeAndData input = {{4, 2}, // shape + {1.f, 2.f, + 3.f, 4.f, + 5.f, 6.f, + 7.f, 8.f}}; + + outputs.push_back({{1}, {0.f}}); + + RunTest(axis, {}, input, outputs, true, "Invalid value of attribute 'axis'"); +} + +// sum of values in splits is too small +TEST(SplitOperatorTest, SplitAttributeSumTooSmall) { + const int64_t axis = 0; + std::vector outputs; + + // input shape and data + ShapeAndData input = {{4, 2}, // shape + {1.f, 2.f, + 3.f, 4.f, + 5.f, 6.f, + 7.f, 8.f}}; + + std::vector splits{1, 2}; // should sum to 4 + + outputs.push_back({{1, 2}, {1.f, 2.f}}); + outputs.push_back({{2, 2}, {3.f, 4.f, 5.f, 6.f}}); + + RunTest(axis, splits, input, outputs, true, "Cannot split using values in 'split' attribute"); +} + +TEST(SplitOperatorTest, InvalidValueInSplitAttribute) { + const int64_t axis = 0; + std::vector outputs; + + // input shape and data + ShapeAndData input = {{4, 2}, // shape + {1.f, 2.f, + 3.f, 4.f, + 5.f, 6.f, + 7.f, 8.f}}; + + std::vector splits{1, 0, 3}; // 0 is not valid + outputs.push_back({{1, 2}, {1.f, 2.f}}); + outputs.push_back({{3, 2}, {3.f, 4.f, 5.f, 6.f, 7.f, 8.f}}); + + RunTest(axis, splits, input, outputs, true, "Invalid value in 'split' attribute"); +} + +/* +Python to replicate processing: + +import numpy as np + +def SplitAxis0(): + input = np.array([1., 2., 3., 4., 5., 6.]).astype(np.float32) + + # 3 outputs, axis 0 + expected_outputs = np.split(input, 3, 0) + print("Split into 3, Axis=0") + print(expected_outputs) + + # 2 outputs, split of 2, 4 + print("Split into 2 sizes 2 and 4, Axis=0") + # numpy split is slightly different in that the values are indices rather than sizes + # so a onnxruntime split of sizes 2, 4 for input of size 6 equates to just the index 2 (:2, 3:) + # see https://docs.scipy.org/doc/numpy/reference/generated/numpy.split.html + print(np.split(input, [2])) + +def SplitAxis1(): + input = np.arange(1.0, 9).reshape(2, 4) + + # 2 outputs, axis 1 + x1, x2 = np.split(input, 2, 1) + print("Split into 2, Axis=1\nx1={}\nx2={}".format(x1, x2)) + + # 2 outputs, split of 3, 1 + x1, x2 = np.split(input, [3], 1) + print("Split into 2 sizes 3 and 1, Axis=1\nx1={}\nx2={}".format(x1, x2)) + +def SplitAxis2(): + x = np.arange(1.0, 25.0).reshape((2,2,6)) + + # equal split. 3 outputs, axis=2 + x1, x2, x3 = np.split(x, 3, 2) + print("Equal split. Axis=2\nx1={}\nx2={}\nx3={}".format(x1, x2, x3)) + + # split axis 2 into [1, 2, 3] sizes, which is index 1 (:1, 1:3, 3:) + x1, x2, x3 = np.split(x, [1, 3], 2) + print("Split into sizes 1, 3, 2. Axis=2\nx1={}\nx2={}\nx3={}".format(x1, x2, x3)) + +def SplitMiddleDimension(): + print ("SplitMiddleDimension") + x = np.arange(1.0, 33.0).reshape((2,4,4)) + + # equal split. 2 outputs, axis=1 + x1, x2 = np.split(x, 2, 1) + print("Equal split. Axis=2\nx1={}\nx2={}".format(x1, x2)) + + # split axis 2 into [1,3] sizes, which is index 1 (:1, 1:) + x1, x2 = np.split(x, [1], 1) + print("Split into sizes 1 and 3. Axis=2\nx1={}\nx2={}".format(x1, x2)) + +SplitAxis0() +SplitAxis1() +SplitAxis2() +SplitMiddleDimension() + +*/ +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/tensor/squeeze_op_test.cc b/onnxruntime/test/providers/cpu/tensor/squeeze_op_test.cc new file mode 100644 index 0000000000000..4299a631db495 --- /dev/null +++ b/onnxruntime/test/providers/cpu/tensor/squeeze_op_test.cc @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(SqueezeOpTest, Squeeze_1) { + OpTester test("Squeeze"); + test.AddAttribute("axes", std::vector{0}); + test.AddInput("data", {1, 3, 4, 5}, std::vector(60, 1.0f)); + test.AddOutput("squeezed", {3, 4, 5}, std::vector(60, 1.0f)); + test.Run(); +} + +TEST(SqueezeOpTest, Squeeze_1_int32) { + OpTester test("Squeeze"); + test.AddAttribute("axes", std::vector{0}); + test.AddInput("data", {1, 3, 4, 5}, std::vector(60, 1)); + test.AddOutput("squeezed", {3, 4, 5}, std::vector(60, 1)); + test.Run(); +} + +TEST(SqueezeOpTest, Squeeze_string) { + OpTester test("Squeeze"); + test.AddAttribute("axes", std::vector{0, 2, 4}); + test.AddInput("data", {1, 2, 1, 3, 1}, std::vector({"1", "2", "3", "4", "5", "6"})); + test.AddOutput("squeezed", {2, 3}, std::vector({"1", "2", "3", "4", "5", "6"})); + test.Run(); +} + +TEST(SqueezeOpTest, Squeeze_2) { + OpTester test("Squeeze"); + test.AddAttribute("axes", std::vector{0, 2, 3}); + test.AddInput("data", {1, 4, 1, 1, 2}, + std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}); + test.AddOutput("squeezed", {4, 2}, + std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}); + test.Run(); +} + +TEST(SqueezeOpTest, UnsortedAxes) { + OpTester test("Squeeze"); + test.AddShapeToTensorData(false); // TODO: re-enable shape inference test after ONNX fix + test.AddAttribute("axes", std::vector{3, 0, 2}); + test.AddInput("data", {1, 4, 1, 1, 2}, + std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}); + test.AddOutput("squeezed", {4, 2}, + std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}); + test.Run(); +} + +TEST(SqueezeOpTest, DuplicateAxes) { + OpTester test("Squeeze"); + test.AddShapeToTensorData(false); // TODO: re-enable shape inference test after ONNX fix + test.AddAttribute("axes", std::vector{3, 0, 2, 0, 2, 3}); + test.AddInput("data", {1, 4, 1, 1, 2}, + std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}); + test.AddOutput("squeezed", {4, 2}, + std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}); + test.Run(); +} + +TEST(SqueezeOpTest, BadAxes) { + OpTester test("Squeeze"); + test.AddShapeToTensorData(false); // TODO: re-enable shape inference test after ONNX fix + // Bad axes - should be 1 instead of 0. + test.AddAttribute("axes", std::vector{0}); + test.AddInput("data", {3, 1, 4, 5}, std::vector(60, 1.0f)); + test.AddOutput("squeezed", {3, 4, 5}, std::vector(60, 1.0f)); + + // Expect failure. + test.Run(OpTester::ExpectResult::kExpectFailure, "Dimension of input 0 must be 1 instead of 3"); +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/tensor/tensor_op_test.cc b/onnxruntime/test/providers/cpu/tensor/tensor_op_test.cc new file mode 100644 index 0000000000000..b43431ce15da9 --- /dev/null +++ b/onnxruntime/test/providers/cpu/tensor/tensor_op_test.cc @@ -0,0 +1,585 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" +#include "core/providers/cpu/tensor/cast_op.h" +#include "core/providers/cpu/tensor/crop.h" +#include "core/util/math.h" + +using namespace ONNX_NAMESPACE; +namespace onnxruntime { +namespace test { + +using ExpectResult = OpTester::ExpectResult; + +TEST(TensorOpTest, Reshape) { + OpTester test("Reshape"); + + test.AddInput("data", {2, 3}, std::vector(6, 1.0f)); + test.AddInput("shape", {3}, {-1, 0, 2}); + test.AddOutput("reshaped", {1, 3, 2}, std::vector(6, 1.0f)); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNupharExecutionProvider}); // Nuphar only supports reshape shape from initializer +} + +TEST(TensorOpTest, ReshapeWithEmptyDim) { + OpTester test("Reshape"); + + test.AddInput("data", {1, 1, 1}, std::vector(1, 1.0f)); + test.AddInput("shape", {0}, {}, true); + test.AddOutput("reshaped", {}, std::vector(1, 1.0f)); + test.Run(); +} + +TEST(TensorOpTest, ReshapeWithInitializer) { + OpTester test("Reshape"); + + test.AddInput("data", {2, 3}, std::vector(6, 1.0f)); + test.AddInput("shape", {3}, {-1, 0, 2}, true); + test.AddOutput("reshaped", {1, 3, 2}, std::vector(6, 1.0f)); + test.Run(); +} + +TEST(TensorOpTest, Identity) { + OpTester test("Identity"); + std::vector X{1.0, 2.0, 3.0, 4.0, 5.0, 6.0}; + test.AddInput("input", {2, 3}, X); + test.AddOutput("output", {2, 3}, X); + test.Run(); +} + +TEST(TensorOpTest, IdentityString) { + OpTester test("Identity"); + std::vector X{"this", "is", "a", "test", "for", "identity"}; + test.AddInput("input", {2, 3}, X); + test.AddOutput("output", {2, 3}, X); + test.Run(); +} + +TEST(TensorOpTest, ShapeTest2D) { + OpTester test("Shape"); + + test.AddInput("data", {2, 3}, std::vector(6, 1.0f)); + test.AddOutput("shape", {2}, {2, 3}); + test.Run(); +} + +TEST(TensorOpTest, ShapeTest3D) { + OpTester test("Shape"); + + test.AddInput("data", {2, 3, 4}, std::vector(24, 1.0f)); + test.AddOutput("shape", {3}, {2, 3, 4}); + test.Run(); +} + +template +void TestCastOp(const std::initializer_list& input, + const std::initializer_list& output, + const std::vector& dimensions, + int64_t toType, + ExpectResult expect_result = ExpectResult::kExpectSuccess, + const std::string& expected_failure_string = "") { + OpTester test("Cast"); + test.AddAttribute("to", toType); + test.AddInput("input", dimensions, input); + test.AddOutput("output", dimensions, output); + test.Run(expect_result, expected_failure_string); +} + +template +void TestCastFromSrc() { + std::initializer_list input_data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + const std::vector shape{3, 2, 2}; + + auto float_output = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f}; + TestCastOp(input_data, float_output, shape, TensorProto::FLOAT); + + auto double_output = {0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0}; + TestCastOp(input_data, double_output, shape, TensorProto::DOUBLE); + + auto bool_output = {false, true, true, true, true, true, true, true, true, true, true, true}; + TestCastOp(input_data, bool_output, shape, TensorProto::BOOL); + + const std::initializer_list uint8_t_output{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + TestCastOp(input_data, uint8_t_output, shape, TensorProto::UINT8); + + const std::initializer_list uint16_t_output{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + TestCastOp(input_data, uint16_t_output, shape, TensorProto::UINT16); + + const std::initializer_list uint32_t_output{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + TestCastOp(input_data, uint32_t_output, shape, TensorProto::UINT32); + + const std::initializer_list uint64_t_output{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + TestCastOp(input_data, uint64_t_output, shape, TensorProto::UINT64); + + const std::initializer_list int16_t_output{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + TestCastOp(input_data, int16_t_output, shape, TensorProto::INT16); + + const std::initializer_list int32_t_output{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + TestCastOp(input_data, int32_t_output, shape, TensorProto::INT32); + + const std::initializer_list int64_t_output{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + TestCastOp(input_data, int64_t_output, shape, TensorProto::INT64); +}; + +TEST(TensorOpTest, Cast) { + TestCastFromSrc(); + TestCastFromSrc(); + TestCastFromSrc(); + TestCastFromSrc(); + TestCastFromSrc(); + TestCastFromSrc(); + TestCastFromSrc(); + TestCastFromSrc(); + TestCastFromSrc(); + TestCastFromSrc(); +} + +TEST(TensorOpTest, CastFromBool) { + auto bool_data = {false, true, true, true, true, true, true, true, true, true, false, true}; + const std::vector shape{3, 2, 2}; + + const std::initializer_list float_output = {0.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 0.0f, 1.0f}; + TestCastOp(bool_data, float_output, shape, TensorProto::FLOAT); + + const std::initializer_list double_output = {0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 1.0}; + TestCastOp(bool_data, double_output, shape, TensorProto::DOUBLE); + + auto bool_output = {false, true, true, true, true, true, true, true, true, true, false, true}; + TestCastOp(bool_data, bool_output, shape, TensorProto::BOOL); + + const std::initializer_list uint8_t_output{0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1}; + TestCastOp(bool_data, uint8_t_output, shape, TensorProto::UINT8); + + const std::initializer_list uint16_t_output{0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1}; + TestCastOp(bool_data, uint16_t_output, shape, TensorProto::UINT16); + + const std::initializer_list uint32_t_output{0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1}; + TestCastOp(bool_data, uint32_t_output, shape, TensorProto::UINT32); + + const std::initializer_list uint64_t_output{0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1}; + TestCastOp(bool_data, uint64_t_output, shape, TensorProto::UINT64); + + const std::initializer_list int16_t_output{0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1}; + TestCastOp(bool_data, int16_t_output, shape, TensorProto::INT16); + + const std::initializer_list int32_t_output{0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1}; + TestCastOp(bool_data, int32_t_output, shape, TensorProto::INT32); + + const std::initializer_list int64_t_output{0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1}; + TestCastOp(bool_data, int64_t_output, shape, TensorProto::INT64); + + const std::initializer_list float16_output{ + MLFloat16(math::floatToHalf(0.0f)), + MLFloat16(math::floatToHalf(1.0f)), + MLFloat16(math::floatToHalf(1.0f)), + MLFloat16(math::floatToHalf(1.0f)), + MLFloat16(math::floatToHalf(1.0f)), + MLFloat16(math::floatToHalf(1.0f)), + MLFloat16(math::floatToHalf(1.0f)), + MLFloat16(math::floatToHalf(1.0f)), + MLFloat16(math::floatToHalf(1.0f)), + MLFloat16(math::floatToHalf(1.0f)), + MLFloat16(math::floatToHalf(0.0f)), + MLFloat16(math::floatToHalf(1.0f))}; + TestCastOp(bool_data, float16_output, shape, TensorProto::FLOAT16); +} + +TEST(TensorOpTest, CastToFloat16) { + const std::vector shape{3, 2, 2}; + std::initializer_list float_data = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f}; + const std::initializer_list float16_output{ + MLFloat16(math::floatToHalf(0.0f)), + MLFloat16(math::floatToHalf(1.0f)), + MLFloat16(math::floatToHalf(2.0f)), + MLFloat16(math::floatToHalf(3.0f)), + MLFloat16(math::floatToHalf(4.0f)), + MLFloat16(math::floatToHalf(5.0f)), + MLFloat16(math::floatToHalf(6.0f)), + MLFloat16(math::floatToHalf(7.0f)), + MLFloat16(math::floatToHalf(8.0f)), + MLFloat16(math::floatToHalf(9.0f)), + MLFloat16(math::floatToHalf(10.0f)), + MLFloat16(math::floatToHalf(11.0f))}; + + TestCastOp(float_data, float16_output, shape, TensorProto::FLOAT16); + + std::initializer_list uint8_t_data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + TestCastOp(uint8_t_data, float16_output, shape, TensorProto::FLOAT16); + + std::initializer_list uint16_t_data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + TestCastOp(uint16_t_data, float16_output, shape, TensorProto::FLOAT16); + + std::initializer_list uint32_t_data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + TestCastOp(uint32_t_data, float16_output, shape, TensorProto::FLOAT16); + + std::initializer_list uint64_t_data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + TestCastOp(uint64_t_data, float16_output, shape, TensorProto::FLOAT16); + + std::initializer_list int8_t_data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + TestCastOp(int8_t_data, float16_output, shape, TensorProto::FLOAT16); + + std::initializer_list int16_t_data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + TestCastOp(int16_t_data, float16_output, shape, TensorProto::FLOAT16); + + std::initializer_list int32_t_data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + TestCastOp(int32_t_data, float16_output, shape, TensorProto::FLOAT16); + + std::initializer_list int64_t_data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + TestCastOp(int64_t_data, float16_output, shape, TensorProto::FLOAT16); +} + +TEST(TensorOpTest, CastFromFloat16) { + const std::vector shape{3, 2, 2}; + const std::initializer_list float_output = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f}; + const std::initializer_list input = { + MLFloat16(math::floatToHalf(0.0f)), + MLFloat16(math::floatToHalf(1.0f)), + MLFloat16(math::floatToHalf(2.0f)), + MLFloat16(math::floatToHalf(3.0f)), + MLFloat16(math::floatToHalf(4.0f)), + MLFloat16(math::floatToHalf(5.0f)), + MLFloat16(math::floatToHalf(6.0f)), + MLFloat16(math::floatToHalf(7.0f)), + MLFloat16(math::floatToHalf(8.0f)), + MLFloat16(math::floatToHalf(9.0f)), + MLFloat16(math::floatToHalf(10.0f)), + MLFloat16(math::floatToHalf(11.0f))}; + + TestCastOp(input, float_output, shape, TensorProto::FLOAT); + + auto bool_data = {false, true, true, true, true, true, true, true, true, true, true, true}; + TestCastOp(input, bool_data, shape, TensorProto::BOOL); + + std::initializer_list uint8_t_data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + TestCastOp(input, uint8_t_data, shape, TensorProto::UINT8); + + std::initializer_list uint16_t_data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + TestCastOp(input, uint16_t_data, shape, TensorProto::UINT16); + + std::initializer_list uint32_t_data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + TestCastOp(input, uint32_t_data, shape, TensorProto::UINT32); + + std::initializer_list uint64_t_data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + TestCastOp(input, uint64_t_data, shape, TensorProto::UINT64); + + std::initializer_list int8_t_data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + TestCastOp(input, int8_t_data, shape, TensorProto::INT8); + + std::initializer_list int16_t_data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + TestCastOp(input, int16_t_data, shape, TensorProto::INT16); + + std::initializer_list int32_t_data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + TestCastOp(input, int32_t_data, shape, TensorProto::INT32); + + std::initializer_list int64_t_data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + TestCastOp(input, int64_t_data, shape, TensorProto::INT64); +} + +TEST(TensorOpTest, CropBorderOnly) { + const int N = 2, C = 1, H = 3, W = 4; + std::vector X = {1.0f, 2.0f, 3.0f, 4.0f, + 2.0f, 3.0f, 4.0f, 5.0f, + 3.0f, 4.0f, 5.0f, 6.0f, + + 4.0f, 5.0f, 6.0f, 7.0f, + 5.0f, 6.0f, 7.0f, 8.0f, + 6.0f, 7.0f, 8.0f, 9.0f}; + + const std::vector border{0, 1, 2, 1}; + std::vector output = { + 2.0f, 3.0f, + + 5.0f, 6.0f}; + + OpTester test("Crop"); + test.AddAttribute("border", border); + test.AddInput("input", {N, C, H, W}, X); + test.AddOutput("output", {N, C, (H - border[2] - border[0]), (W - border[3] - border[1])}, output); + test.Run(); +} + +TEST(TensorOpTest, CropBorderAndScale) { + const int N = 2, C = 1, H = 3, W = 4; + std::vector X = {1.0f, 2.0f, 3.0f, 4.0f, + 2.0f, 3.0f, 4.0f, 5.0f, + 3.0f, 4.0f, 5.0f, 6.0f, + + 4.0f, 5.0f, 6.0f, 7.0f, + 5.0f, 6.0f, 7.0f, 8.0f, + 6.0f, 7.0f, 8.0f, 9.0f}; + + const std::vector border = {0, 0, 0, 0}; + const std::vector scale = {2, 2}; + + std::vector output = { + 1.0f, 2.0f, + 2.0f, 3.0f, + + 4.0f, 5.0f, + 5.0f, 6.0f}; + + OpTester test("Crop"); + test.AddAttribute("border", border); + test.AddAttribute("scale", scale); + test.AddInput("input", {N, C, H, W}, X); + test.AddOutput("output", {N, C, scale[0], scale[1]}, output); + test.Run(); +} + +std::pair MeanStdev(std::vector& v) { + float sum = std::accumulate(v.begin(), v.end(), 0.0f); + float mean = sum / v.size(); + + std::vector diff(v.size()); + std::transform(v.begin(), v.end(), diff.begin(), + std::bind2nd(std::minus(), mean)); + float sq_sum = std::inner_product(diff.begin(), diff.end(), diff.begin(), 0.0f); + float stdev = std::sqrt(sq_sum / v.size()); + + return std::make_pair(mean, stdev); +} + +void Normalize(std::vector& v, + std::pair& mean_stdev, bool normalize_variance) { + float mean = mean_stdev.first; + float stdev = mean_stdev.second; + + std::transform(v.begin(), v.end(), v.begin(), + std::bind2nd(std::minus(), mean)); + + if (normalize_variance) { + std::transform(v.begin(), v.end(), v.begin(), + std::bind2nd(std::divides(), stdev)); + } +} + +void MeanVarianceNormalizationAcrossChannels(bool across_channels, bool normalize_variance) { + const int64_t N = 2, C = 2, H = 2, W = 3; + int64_t one = 1; + int64_t zero = 0; + + std::vector X = {3.0f, -3.0f, -1.0f, + 1.0f, 2.0f, -1.0f, + -2.0f, -2.0f, -2.0f, + 4.0f, 1.0f, 4.0f, + 0.0f, -2.0f, -2.0f, + -4.0f, 5.0f, 7.0f, + 5.0f, -5.0f, -5.0f, + 3.0f, 4.0f, 4.0f}; + auto mean_stdev = MeanStdev(X); + + std::vector result(X); + Normalize(result, mean_stdev, normalize_variance); + + OpTester test("MeanVarianceNormalization", 7); + test.AddAttribute("across_channels", across_channels ? one : zero); + test.AddAttribute("normalize_variance", normalize_variance ? one : zero); + test.AddInput("input", {N, C, H, W}, X); + test.AddOutput("output", {N, C, H, W}, result); + test.Run(); +} + +void MeanVarianceNormalizationPerChannel(bool across_channels, bool normalize_variance) { + const int64_t N = 2, C = 2, H = 2, W = 3; + int64_t one = 1; + int64_t zero = 0; + + std::vector N1C1 = {3.0f, -3.0f, -1.0f, + 1.0f, 2.0f, -1.0f}; + std::vector N1C2 = {-2.0f, -2.0f, -2.0f, + 4.0f, 1.0f, 4.0f}; + std::vector N2C1 = { + 0.0f, + -2.0f, + -2.0f, + -4.0f, + 5.0f, + 7.0f, + }; + std::vector N2C2 = { + 5.0f, + -5.0f, + -5.0f, + 3.0f, + 4.0f, + 4.0f, + }; + + std::vector X; + X.reserve(N * C * H * W); + X.insert(X.end(), N1C1.begin(), N1C1.end()); + X.insert(X.end(), N1C2.begin(), N1C2.end()); + X.insert(X.end(), N2C1.begin(), N2C1.end()); + X.insert(X.end(), N2C2.begin(), N2C2.end()); + + std::vector C1; + C1.reserve(N * H * W); + C1.insert(C1.end(), N1C1.begin(), N1C1.end()); + C1.insert(C1.end(), N2C1.begin(), N2C1.end()); + auto C1_meam_stdev = MeanStdev(C1); + + std::vector C2; + C2.reserve(N * H * W); + C2.insert(C2.end(), N1C2.begin(), N1C2.end()); + C2.insert(C2.end(), N2C2.begin(), N2C2.end()); + auto C2_meam_stdev = MeanStdev(C2); + + std::vector N1C1_result(N1C1), N1C2_result(N1C2), + N2C1_result(N2C1), N2C2_result(N2C2); + Normalize(N1C1_result, C1_meam_stdev, normalize_variance); + Normalize(N2C1_result, C1_meam_stdev, normalize_variance); + Normalize(N1C2_result, C2_meam_stdev, normalize_variance); + Normalize(N2C2_result, C2_meam_stdev, normalize_variance); + + std::vector result; + result.reserve(N * C * H * W); + result.insert(result.end(), N1C1_result.begin(), N1C1_result.end()); + result.insert(result.end(), N1C2_result.begin(), N1C2_result.end()); + result.insert(result.end(), N2C1_result.begin(), N2C1_result.end()); + result.insert(result.end(), N2C2_result.begin(), N2C2_result.end()); + + OpTester test("MeanVarianceNormalization", 7); + test.AddAttribute("across_channels", across_channels ? one : zero); + test.AddAttribute("normalize_variance", normalize_variance ? one : zero); + test.AddInput("input", {N, C, H, W}, X); + test.AddOutput("output", {N, C, H, W}, result); + test.Run(); +} + +void MeanVarianceNormalizationFunctionDefaultPerChannel() { + const int64_t N = 2, C = 2, H = 2, W = 3; + + std::vector N1C1 = {3.0f, -3.0f, -1.0f, + 1.0f, 2.0f, -1.0f}; + std::vector N1C2 = {-2.0f, -2.0f, -2.0f, + 4.0f, 1.0f, 4.0f}; + std::vector N2C1 = { + 0.0f, + -2.0f, + -2.0f, + -4.0f, + 5.0f, + 7.0f, + }; + std::vector N2C2 = { + 5.0f, + -5.0f, + -5.0f, + 3.0f, + 4.0f, + 4.0f, + }; + + std::vector X; + X.reserve(N * C * H * W); + X.insert(X.end(), N1C1.begin(), N1C1.end()); + X.insert(X.end(), N1C2.begin(), N1C2.end()); + X.insert(X.end(), N2C1.begin(), N2C1.end()); + X.insert(X.end(), N2C2.begin(), N2C2.end()); + + std::vector C1; + C1.reserve(N * H * W); + C1.insert(C1.end(), N1C1.begin(), N1C1.end()); + C1.insert(C1.end(), N2C1.begin(), N2C1.end()); + auto C1_meam_stdev = MeanStdev(C1); + + std::vector C2; + C2.reserve(N * H * W); + C2.insert(C2.end(), N1C2.begin(), N1C2.end()); + C2.insert(C2.end(), N2C2.begin(), N2C2.end()); + auto C2_meam_stdev = MeanStdev(C2); + + std::vector N1C1_result(N1C1), N1C2_result(N1C2), + N2C1_result(N2C1), N2C2_result(N2C2); + Normalize(N1C1_result, C1_meam_stdev, 1); + Normalize(N2C1_result, C1_meam_stdev, 1); + Normalize(N1C2_result, C2_meam_stdev, 1); + Normalize(N2C2_result, C2_meam_stdev, 1); + + std::vector result; + result.reserve(N * C * H * W); + result.insert(result.end(), N1C1_result.begin(), N1C1_result.end()); + result.insert(result.end(), N1C2_result.begin(), N1C2_result.end()); + result.insert(result.end(), N2C1_result.begin(), N2C1_result.end()); + result.insert(result.end(), N2C2_result.begin(), N2C2_result.end()); + + OpTester test("MeanVarianceNormalization", 9); + test.AddInput("input", {N, C, H, W}, X); + test.AddOutput("output", {N, C, H, W}, result); + test.Run(); +} + +void MeanVarianceNormalizationFunctionAcrossChannels(std::vector axes) { + const int64_t N = 2, C = 2, H = 2, W = 3; + + std::vector X = {3.0f, -3.0f, -1.0f, + 1.0f, 2.0f, -1.0f, + -2.0f, -2.0f, -2.0f, + 4.0f, 1.0f, 4.0f, + 0.0f, -2.0f, -2.0f, + -4.0f, 5.0f, 7.0f, + 5.0f, -5.0f, -5.0f, + 3.0f, 4.0f, 4.0f}; + auto mean_stdev = MeanStdev(X); + + std::vector result(X); + Normalize(result, mean_stdev, 1); + + OpTester test("MeanVarianceNormalization", 9); + test.AddAttribute("axes", axes); + test.AddInput("input", {N, C, H, W}, X); + test.AddOutput("output", {N, C, H, W}, result); + test.Run(); +} + +TEST(TensorOpTest, MeanVarianceNormalizationCPUTest) { + // across_channels: true, normalize_variance: true + MeanVarianceNormalizationAcrossChannels(true, true); + + // across_channels: true, normalize_variance: false + MeanVarianceNormalizationAcrossChannels(true, false); + + // across_channels: false, normalize_variance: false + MeanVarianceNormalizationPerChannel(false, false); + + // across_channels: false, normalize_variance: true + MeanVarianceNormalizationPerChannel(false, true); + + // axes: {0, 1, 2, 3} for across_channels + MeanVarianceNormalizationFunctionAcrossChannels({0, 1, 2, 3}); + + // Default (axes: {0, 2, 3}) for non across_channels + MeanVarianceNormalizationFunctionDefaultPerChannel(); +} + +TEST(TensorOpTest, ImageScalerTest) { + const int64_t N = 1, C = 2, H = 2, W = 2; + std::vector X = { + 1.0f, 3.0f, + 3.0f, 5.0f, + + 3.0f, 5.0f, + 7.0f, 9.0f}; + + float scale = 2.0f; + std::vector bias = {1.0f, 2.0f}; + + std::vector result = { + 3.0f, 7.0f, + 7.0f, 11.0f, + + 8.0f, 12.0f, + 16.0f, 20.0f}; + + OpTester test("ImageScaler"); + test.AddAttribute("scale", scale); + test.AddAttribute("bias", bias); + test.AddInput("input", {N, C, H, W}, X); + test.AddOutput("output", {N, C, H, W}, result); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/tensor/tile_op_test.cc b/onnxruntime/test/providers/cpu/tensor/tile_op_test.cc new file mode 100644 index 0000000000000..cd053f19213d8 --- /dev/null +++ b/onnxruntime/test/providers/cpu/tensor/tile_op_test.cc @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(TensorOpTest, Tile1D) { + OpTester test("Tile"); + + test.AddInput("input", {3}, {1.0f, 2.0f, 3.0f}); + test.AddInput("repeats", {1}, {3}); + test.AddOutput("output", {9}, {1.0f, 2.0f, 3.0f, 1.0f, 2.0f, 3.0f, 1.0f, 2.0f, 3.0f}); + test.Run(); +} + +TEST(TensorOpTest, Tile2D_1Axis) { + OpTester test("Tile"); + + test.AddInput("input", {2, 2}, + {11.0f, 12.0f, + 21.0f, 22.0f}); + test.AddInput("repeats", {2}, {2, 1}); + test.AddOutput("output", {4, 2}, + {11.0f, 12.0f, + 21.0f, 22.0f, + 11.0f, 12.0f, + 21.0f, 22.0f}); + + test.Run(); +} + +TEST(TensorOpTest, Tile2D_2Axes) { + OpTester test("Tile"); + + test.AddInput("input", {2, 2}, + {11.0f, 12.0f, + 21.0f, 22.0f}); + test.AddInput("repeats", {2}, {2, 2}); + test.AddOutput("output", {4, 4}, + {11.0f, 12.0f, 11.0f, 12.0f, + 21.0f, 22.0f, 21.0f, 22.0f, + 11.0f, 12.0f, 11.0f, 12.0f, + 21.0f, 22.0f, 21.0f, 22.0f}); + + test.Run(); +} + +TEST(TensorOpTest, Tile3D) { + OpTester test("Tile"); + + test.AddInput("input", {2, 1, 3}, + {111.0f, 112.0f, 113.0f, + 211.0f, 212.0f, 213.0f}); + test.AddInput("repeats", {3}, {1, 2, 1}); + test.AddOutput("output", {2, 2, 3}, + {111.0f, 112.0f, 113.0f, + 111.0f, 112.0f, 113.0f, + + 211.0f, 212.0f, 213.0f, + 211.0f, 212.0f, 213.0f}); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/tensor/transpose_test.cc b/onnxruntime/test/providers/cpu/tensor/transpose_test.cc new file mode 100644 index 0000000000000..995a2c8771b38 --- /dev/null +++ b/onnxruntime/test/providers/cpu/tensor/transpose_test.cc @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +void TransposeTest(std::vector& input_shape, + std::vector& input_vals, + std::vector* p_perm, + std::vector expected_shape, + std::initializer_list& expected_vals) { + OpTester test("Transpose"); + if (nullptr != p_perm) + test.AddAttribute("perm", *p_perm); + test.AddInput("X", input_shape, input_vals); + test.AddOutput("Y", expected_shape, expected_vals); + test.Run(); +} + +// Test 2 dimensional transpose, with no permutation attribute specified +TEST(TransposeOpTest, TwoDimNoAttr) { + std::vector input_shape({2, 3}); + std::vector input_vals = { + 1.0f, 2.0f, 3.0f, + 4.0f, 5.0f, 6.0f}; + + std::vector expected_shape({3, 2}); + auto expected_vals = { + 1.0f, 4.0f, + 2.0f, 5.0f, + 3.0f, 6.0f}; + + TransposeTest(input_shape, input_vals, nullptr, expected_shape, expected_vals); +} + +// Test 2 dimensional transpose, with permutation attribute specified +TEST(TransposeOpTest, TwoDim) { + std::vector input_shape({2, 3}); + std::vector input_vals = { + 1.0f, 2.0f, 3.0f, + 4.0f, 5.0f, 6.0f}; + + std::vector perm = {1, 0}; + std::vector expected_shape({3, 2}); + auto expected_vals = { + 1.0f, 4.0f, + 2.0f, 5.0f, + 3.0f, 6.0f}; + + TransposeTest(input_shape, input_vals, &perm, expected_shape, expected_vals); +} + +// Test 3 dimensional transpose, with permutation attribute specified +TEST(TransposeOpTest, ThreeDim) { + std::vector input_shape({4, 2, 3}); + std::vector input_vals = { + 1.0f, 2.0f, 3.0f, + 4.0f, 5.0f, 6.0f, + + 1.1f, 2.1f, 3.1f, + 4.1f, 5.1f, 6.1f, + + 1.2f, 2.2f, 3.2f, + 4.2f, 5.2f, 6.2f, + + 1.3f, 2.3f, 3.3f, + 4.3f, 5.3f, 6.3f}; + + std::vector perm = {0, 2, 1}; + std::vector expected_shape({4, 3, 2}); + auto expected_vals = { + 1.0f, + 4.0f, + 2.0f, + 5.0f, + 3.0f, + 6.0f, + + 1.1f, + 4.1f, + 2.1f, + 5.1f, + 3.1f, + 6.1f, + + 1.2f, + 4.2f, + 2.2f, + 5.2f, + 3.2f, + 6.2f, + + 1.3f, + 4.3f, + 2.3f, + 5.3f, + 3.3f, + 6.3f, + + }; + + TransposeTest(input_shape, input_vals, &perm, expected_shape, expected_vals); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/tensor/unsqueeze_op_test.cc b/onnxruntime/test/providers/cpu/tensor/unsqueeze_op_test.cc new file mode 100644 index 0000000000000..4429ef668a185 --- /dev/null +++ b/onnxruntime/test/providers/cpu/tensor/unsqueeze_op_test.cc @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(TensorOpTest, Unsqueeze_1) { + OpTester test("Unsqueeze"); + + test.AddAttribute("axes", std::vector{1}); + test.AddInput("input", {2, 3, 4}, std::vector(2 * 3 * 4, 1.0f)); + test.AddOutput("output", {2, 1, 3, 4}, std::vector(2 * 3 * 4, 1.0f)); + test.Run(); +} + +TEST(TensorOpTest, Unsqueeze_1_int32) { + OpTester test("Unsqueeze"); + + test.AddAttribute("axes", std::vector{1}); + test.AddInput("input", {2, 3, 4}, std::vector(2 * 3 * 4, 1)); + test.AddOutput("output", {2, 1, 3, 4}, std::vector(2 * 3 * 4, 1)); + test.Run(); +} + +TEST(TensorOpTest, Unsqueeze_2) { + OpTester test("Unsqueeze"); + + test.AddAttribute("axes", std::vector{0, 4}); + test.AddInput("input", {2, 3, 4}, std::vector(2 * 3 * 4, 1.0f)); + test.AddOutput("output", {1, 2, 3, 4, 1}, std::vector(2 * 3 * 4, 1.0f)); + test.Run(); +} + +TEST(TensorOpTest, Unsqueeze_3) { + OpTester test("Unsqueeze"); + + test.AddAttribute("axes", std::vector{2, 1, 0}); + test.AddInput("input", {2, 3, 4}, std::vector(2 * 3 * 4, 1.0f)); + test.AddOutput("output", {1, 1, 1, 2, 3, 4}, std::vector(2 * 3 * 4, 1.0f)); + test.Run(); +} + +TEST(TensorOpTest, Unsqueeze_Duplicate) { + OpTester test("Unsqueeze"); + + test.AddAttribute("axes", std::vector{2, 1, 0, 2}); + test.AddInput("input", {2, 3, 4}, std::vector(2 * 3 * 4, 1.0f)); + test.AddOutput("output", {1, 1, 1, 2, 3, 4}, std::vector(2 * 3 * 4, 1.0f)); + test.Run(OpTester::ExpectResult::kExpectFailure, "'axes' has a duplicate axis"); +} + +TEST(TensorOpTest, Unsqueeze_OutOfRange) { + OpTester test("Unsqueeze"); + + test.AddAttribute("axes", std::vector{4}); + test.AddInput("input", {2, 3, 4}, std::vector(2 * 3 * 4, 1.0f)); + test.AddOutput("output", {2, 1, 3, 4}, std::vector(2 * 3 * 4, 1.0f)); + test.Run(OpTester::ExpectResult::kExpectFailure, "Mismatch between number of source and target dimensions."); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/tensor/upsample_op_test.cc b/onnxruntime/test/providers/cpu/tensor/upsample_op_test.cc new file mode 100644 index 0000000000000..f3295246d7718 --- /dev/null +++ b/onnxruntime/test/providers/cpu/tensor/upsample_op_test.cc @@ -0,0 +1,323 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/tensor/upsample.h" +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(UpsampleOpTest, UpsampleOpNearestTest) { + OpTester test("Upsample"); + + std::vector scales{1.0f, 1.0f, 2.0f, 3.0f}; + test.AddAttribute("mode", "nearest"); + test.AddAttribute("scales", scales); + + const int64_t N = 1, C = 2, H = 2, W = 2; + std::vector X = {1.0f, 3.0f, + 3.0f, 5.0f, + + 3.0f, 5.0f, + 7.0f, 9.0f}; + + test.AddInput("X", {N, C, H, W}, X); + + std::vector Y = { + 1.0f, 1.0f, 1.0f, 3.0f, 3.0f, 3.0f, + 1.0f, 1.0f, 1.0f, 3.0f, 3.0f, 3.0f, + 3.0f, 3.0f, 3.0f, 5.0f, 5.0f, 5.0f, + 3.0f, 3.0f, 3.0f, 5.0f, 5.0f, 5.0f, + + 3.0f, 3.0f, 3.0f, 5.0f, 5.0f, 5.0f, + 3.0f, 3.0f, 3.0f, 5.0f, 5.0f, 5.0f, + 7.0f, 7.0f, 7.0f, 9.0f, 9.0f, 9.0f, + 7.0f, 7.0f, 7.0f, 9.0f, 9.0f, 9.0f}; + + test.AddOutput("Y", {N, C, (int64_t)(H * scales[2]), (int64_t)(W * scales[3])}, Y); + test.Run(); +} + +TEST(UpsampleOpTest, UpsampleOpNearestTest_int32) { + OpTester test("Upsample"); + + std::vector scales{1.0f, 1.0f, 2.0f, 3.0f}; + test.AddAttribute("mode", "nearest"); + test.AddAttribute("scales", scales); + + const int64_t N = 1, C = 2, H = 2, W = 2; + std::vector X = {1, 3, + 3, 5, + + 3, 5, + 7, 9}; + + test.AddInput("X", {N, C, H, W}, X); + + std::vector Y = { + 1, 1, 1, 3, 3, 3, + 1, 1, 1, 3, 3, 3, + 3, 3, 3, 5, 5, 5, + 3, 3, 3, 5, 5, 5, + + 3, 3, 3, 5, 5, 5, + 3, 3, 3, 5, 5, 5, + 7, 7, 7, 9, 9, 9, + 7, 7, 7, 9, 9, 9}; + + test.AddOutput("Y", {N, C, (int64_t)(H * scales[2]), (int64_t)(W * scales[3])}, Y); + test.Run(); +} + +TEST(UpsampleOpTest, UpsampleOpNearest2XTest) { + OpTester test("Upsample"); + + std::vector scales{1.0f, 1.0f, 2.0f, 2.0f}; + test.AddAttribute("mode", "nearest"); + test.AddAttribute("scales", scales); + + const int64_t N = 1, C = 2, H = 2, W = 2; + std::vector X = {1.0f, 3.0f, + 3.0f, 5.0f, + + 3.0f, 5.0f, + 7.0f, 9.0f}; + + test.AddInput("X", {N, C, H, W}, X); + + std::vector Y = { + 1.0f, 1.0f, 3.0f, 3.0f, + 1.0f, 1.0f, 3.0f, 3.0f, + 3.0f, 3.0f, 5.0f, 5.0f, + 3.0f, 3.0f, 5.0f, 5.0f, + + 3.0f, 3.0f, 5.0f, 5.0f, + 3.0f, 3.0f, 5.0f, 5.0f, + 7.0f, 7.0f, 9.0f, 9.0f, + 7.0f, 7.0f, 9.0f, 9.0f}; + + test.AddOutput("Y", {N, C, (int64_t)(H * scales[2]), (int64_t)(W * scales[3])}, Y); + test.Run(); +} + +TEST(UpsampleOpTest, UpsampleOpNearest222XTest) { + OpTester test("Upsample"); + + std::vector scales{2.0f, 1.0f, 2.0f, 2.0f}; + test.AddAttribute("mode", "nearest"); + test.AddAttribute("scales", scales); + + const int64_t N = 1, C = 2, H = 2, W = 2; + std::vector X = {1.0f, 3.0f, + 3.0f, 5.0f, + + 3.0f, 5.0f, + 7.0f, 9.0f}; + + test.AddInput("X", {N, C, H, W}, X); + + std::vector Y = { + 1.0f, 1.0f, 3.0f, 3.0f, + 1.0f, 1.0f, 3.0f, 3.0f, + 3.0f, 3.0f, 5.0f, 5.0f, + 3.0f, 3.0f, 5.0f, 5.0f, + + 3.0f, 3.0f, 5.0f, 5.0f, + 3.0f, 3.0f, 5.0f, 5.0f, + 7.0f, 7.0f, 9.0f, 9.0f, + 7.0f, 7.0f, 9.0f, 9.0f, + + 1.0f, 1.0f, 3.0f, 3.0f, + 1.0f, 1.0f, 3.0f, 3.0f, + 3.0f, 3.0f, 5.0f, 5.0f, + 3.0f, 3.0f, 5.0f, 5.0f, + + 3.0f, 3.0f, 5.0f, 5.0f, + 3.0f, 3.0f, 5.0f, 5.0f, + 7.0f, 7.0f, 9.0f, 9.0f, + 7.0f, 7.0f, 9.0f, 9.0f + }; + + test.AddOutput("Y", {N*2, C, (int64_t)(H * scales[2]), (int64_t)(W * scales[3])}, Y); + test.Run(); +} + +TEST(UpsampleOpTest, UpsampleOpNearest15XTest) { + OpTester test("Upsample"); + + std::vector scales{1.0f, 1.0f, 2.0f, 1.5f}; + test.AddAttribute("mode", "nearest"); + test.AddAttribute("scales", scales); + + const int64_t N = 1, C = 2, H = 2, W = 2; + std::vector X = {1.0f, 3.0f, + 3.0f, 5.0f, + + 3.0f, 5.0f, + 7.0f, 9.0f}; + + test.AddInput("X", {N, C, H, W}, X); + + std::vector Y = { + 1.0f, 1.0f, 3.0f, + 1.0f, 1.0f, 3.0f, + 3.0f, 3.0f, 5.0f, + 3.0f, 3.0f, 5.0f, + + 3.0f, 3.0f, 5.0f, + 3.0f, 3.0f, 5.0f, + 7.0f, 7.0f, 9.0f, + 7.0f, 7.0f, 9.0f}; + + test.AddOutput("Y", {N, C, (int64_t)(H * scales[2]), (int64_t)(W * scales[3])}, Y); + test.Run(); +} + +TEST(UpsampleOpTest, UpsampleOpNearest2XTest_int32) { + OpTester test("Upsample"); + + std::vector scales{1.0f, 1.0f, 2.0f, 2.0f}; + test.AddAttribute("mode", "nearest"); + test.AddAttribute("scales", scales); + + const int64_t N = 1, C = 2, H = 2, W = 2; + std::vector X = {1, 3, + 3, 5, + + 3, 5, + 7, 9}; + + test.AddInput("X", {N, C, H, W}, X); + + std::vector Y = { + 1, 1, 3, 3, + 1, 1, 3, 3, + 3, 3, 5, 5, + 3, 3, 5, 5, + + 3, 3, 5, 5, + 3, 3, 5, 5, + 7, 7, 9, 9, + 7, 7, 9, 9}; + + test.AddOutput("Y", {N, C, (int64_t)(H * scales[2]), (int64_t)(W * scales[3])}, Y); + test.Run(); +} + +TEST(UpsampleOpTest, UpsampleOpBilinearTest) { + OpTester test("Upsample"); + + std::vector scales{1.0f, 1.0f, 2.0f, 4.0f}; + test.AddAttribute("mode", "linear"); + test.AddAttribute("scales", scales); + + const int64_t N = 2, C = 1, H = 2, W = 2; + std::vector X = {1.0f, 3.0f, + 3.0f, 5.0f, + + 3.0f, 5.0f, + 7.0f, 9.0f}; + + test.AddInput("X", {N, C, H, W}, X); + + std::vector Y = { + 1.0f, 1.5f, 2.0f, 2.5f, 3.0f, 3.0f, 3.0f, 3.0f, + 2.0f, 2.5f, 3.0f, 3.5f, 4.0f, 4.0f, 4.0f, 4.0f, + 3.0f, 3.5f, 4.0f, 4.5f, 5.0f, 5.0f, 5.0f, 5.0f, + 3.0f, 3.5f, 4.0f, 4.5f, 5.0f, 5.0f, 5.0f, 5.0f, + + 3.0f, 3.5f, 4.0f, 4.5f, 5.0f, 5.0f, 5.0f, 5.0f, + 5.0f, 5.5f, 6.0f, 6.5f, 7.0f, 7.0f, 7.0f, 7.0f, + 7.0f, 7.5f, 8.0f, 8.5f, 9.0f, 9.0f, 9.0f, 9.0f, + 7.0f, 7.5f, 8.0f, 8.5f, 9.0f, 9.0f, 9.0f, 9.0f}; + + test.AddOutput("Y", {N, C, (int64_t)(H * scales[2]), (int64_t)(W * scales[3])}, Y); + test.Run(); +} + +TEST(UpsampleOpTest, UpsampleOpBilinearTest2) { + OpTester test("Upsample"); + + std::vector scales{1.0f, 1.0f, 2.0f, 4.0f}; + test.AddAttribute("mode", "linear"); + test.AddAttribute("scales", scales); + + const int64_t N = 2, C = 1, H = 2, W = 2; + std::vector X = {1.0f, 3.0f, + 4.0f, 8.0f, + + 6.0f, 2.0f, + 7.0f, 11.0f}; + + test.AddInput("X", {N, C, H, W}, X); + + std::vector Y = { + 1.0f, 1.5f, 2.0f, 2.5f, 3.0f, 3.0f, 3.0f, 3.0f, + 2.5f, 3.25f, 4.0f, 4.75f, 5.5f, 5.5f, 5.5f, 5.5f, + 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 8.0f, 8.0f, 8.0f, + 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 8.0f, 8.0f, 8.0f, + + 6.0f, 5.0f, 4.0f, 3.0f, 2.0f, 2.0f, 2.0f, 2.0f, + 6.5f, 6.5f, 6.5f, 6.5f, 6.5f, 6.5f, 6.5f, 6.5f, + 7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 11.0f, 11.0f, 11.0f, + 7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 11.0f, 11.0f, 11.0f}; + + test.AddOutput("Y", {N, C, (int64_t)(H * scales[2]), (int64_t)(W * scales[3])}, Y); + test.Run(); +} + +TEST(UpsampleOpTest, UpsampleOpBilinearTest_int32) { + OpTester test("Upsample"); + + std::vector scales{1.0f, 1.0f, 2.0f, 4.0f}; + test.AddAttribute("mode", "linear"); + test.AddAttribute("scales", scales); + + const int64_t N = 2, C = 1, H = 2, W = 2; + std::vector X = {1, 3, + 3, 5, + + 3, 5, + 7, 9}; + + test.AddInput("X", {N, C, H, W}, X); + + std::vector Y = { + 1, 1, 2, 2, 3, 3, 3, 3, + 2, 2, 3, 3, 4, 4, 4, 4, + 3, 3, 4, 4, 5, 5, 5, 5, + 3, 3, 4, 4, 5, 5, 5, 5, + + 3, 3, 4, 4, 5, 5, 5, 5, + 5, 5, 6, 6, 7, 7, 7, 7, + 7, 7, 8, 8, 9, 9, 9, 9, + 7, 7, 8, 8, 9, 9, 9, 9}; + + test.AddOutput("Y", {N, C, (int64_t)(H * scales[2]), (int64_t)(W * scales[3])}, Y); + test.Run(); +} + +TEST(UpsampleOpTest, UpsampleOpNearestTest_1D) { + OpTester test("Upsample"); + + std::vector scales{2.0f}; + test.AddAttribute("mode", "nearest"); + test.AddAttribute("scales", scales); + + std::vector X = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f}; + + test.AddInput("X", {5}, X); + + std::vector Y = { + 1.0f, 1.0f, + 2.0f, 2.0f, + 3.0f, 3.0f, + 4.0f, 4.0f, + 5.0f, 5.0f}; + + test.AddOutput("Y", {10}, Y); + test.Run(); +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/provider_test_utils.cc b/onnxruntime/test/providers/provider_test_utils.cc new file mode 100644 index 0000000000000..2c118f1372f3f --- /dev/null +++ b/onnxruntime/test/providers/provider_test_utils.cc @@ -0,0 +1,434 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include + +#include "gmock/gmock.h" +#include "test/providers/provider_test_utils.h" +#include +#include +#include +#include "core/common/logging/logging.h" +#include "core/common/logging/sinks/clog_sink.h" +#include "core/framework/tensorprotoutils.h" +#include "core/session/inference_session.h" +#include "test/util/include/default_providers.h" + +using namespace ::onnxruntime::logging; + +namespace onnxruntime { +namespace test { + +// Check functions for tensor types + +// The default implementation compares for equality, specialized versions for other types are below +template +void Check(const OpTester::Data& expected_data, const Tensor& output_tensor, const std::string& provider_type) { + auto& expected_tensor = expected_data.data_.Get(); + auto* expected = expected_tensor.template Data(); + auto* output = output_tensor.template Data(); + auto size = output_tensor.Shape().Size(); + + for (int i = 0; i < size; ++i) { + EXPECT_EQ(expected[i], output[i]) << "provider_type: " << provider_type; + } +} + +template <> +void Check(const OpTester::Data& expected_data, const Tensor& output_tensor, const std::string& provider_type) { + auto& expected_tensor = expected_data.data_.Get(); + auto* expected = expected_tensor.template Data(); + auto* output = output_tensor.template Data(); + auto size = output_tensor.Shape().Size(); + + bool has_abs_err = expected_data.absolute_error_.has_value(); + bool has_rel_err = expected_data.relative_error_.has_value(); + + float threshold = 0.001f; +#ifdef USE_CUDA + threshold = 0.005f; +#endif + + for (int i = 0; i < size; ++i) { + if (std::isinf(expected[i])) // Test infinity for equality + EXPECT_EQ(expected[i], output[i]); + else { + if (!has_abs_err && !has_rel_err) { + // the default for existing tests + EXPECT_NEAR(expected[i], output[i], threshold) << "provider_type: " << provider_type; + } else { + if (has_abs_err) { + EXPECT_NEAR(expected[i], output[i], expected_data.absolute_error_.value()) << "provider_type: " << provider_type; + } + if (has_rel_err) { + EXPECT_NEAR(expected[i], output[i], expected_data.relative_error_.value() * std::abs(expected[i])) << "provider_type: " << provider_type; + } + } + } + } +} + +template +void CheckDispatch(MLDataType type, const OpTester::Data& expected_data, const Tensor& output_tensor, const std::string& provider_type) { + if (type == DataTypeImpl::GetType()) + Check(expected_data, output_tensor, provider_type); + else + ONNXRUNTIME_THROW("OpTester:Check() not implemented for output tensor type of ", type); +} + +template +void CheckDispatch(MLDataType type, const OpTester::Data& expected_data, const Tensor& output_tensor, const std::string& provider_type) { + if (type == DataTypeImpl::GetType()) + Check(expected_data, output_tensor, provider_type); + else + CheckDispatch(type, expected_data, output_tensor, provider_type); +} + +void Check(const OpTester::Data& expected_data, const Tensor& output_tensor, const std::string& provider_type) { + ONNXRUNTIME_ENFORCE(expected_data.data_.Get().Shape() == output_tensor.Shape(), + "Expected output shape [" + expected_data.data_.Get().Shape().ToString() + + "] did not match run output shape [" + + output_tensor.Shape().ToString() + "] for " + expected_data.def_.Name()); + + CheckDispatch(output_tensor.DataType(), expected_data, output_tensor, provider_type); +} + +// Check for non tensor types + +template +void Check(const OpTester::Data& expected_data, const T& run_output, const std::string& provider_type) { + EXPECT_EQ(expected_data.data_.Get(), run_output) << "provider_type: " << provider_type; +} + +template +void CheckDispatch(MLDataType type, const OpTester::Data& expected_data, MLValue& mlvalue, const std::string& provider_type) { + if (type == DataTypeImpl::GetType()) + Check(expected_data, mlvalue.Get(), provider_type); + else + ONNXRUNTIME_THROW("OpTester:Check() not implemented for output tensor type of ", type); +} + +template +void CheckDispatch(MLDataType type, const OpTester::Data& expected_data, MLValue& mlvalue, const std::string& provider_type) { + if (type == DataTypeImpl::GetType()) + Check(expected_data, mlvalue.Get(), provider_type); + else + CheckDispatch(type, expected_data, mlvalue, provider_type); +} + +void Check(const OpTester::Data& expected_data, MLValue& mlvalue, const std::string& provider_type) { + CheckDispatch(expected_data.data_.Type(), expected_data, mlvalue, provider_type); +} + +void DebugTrap() { +#if _MSC_VER + __debugbreak(); +#else + raise(SIGTRAP); +#endif +} + +OpTester::~OpTester() { +#ifndef NDEBUG + if (!run_called_) { + std::cerr << "Someone forgot to call OpTester::Run()" << std::endl; + DebugTrap(); + } +#endif +} + +void OpTester::FillFeedsAndOutputNames(std::unordered_map& feeds, + std::vector& output_names) { + for (auto& output : output_data_) { + if (output.def_.Exists()) + output_names.push_back(output.def_.Name()); + } + + for (auto i = 0; i < input_data_.size(); ++i) { + if (std::find(initializer_index_.begin(), initializer_index_.end(), i) == initializer_index_.end() && input_data_[i].def_.Exists()) { + feeds[input_data_[i].def_.Name()] = input_data_[i].data_; + } + } +} + +void OpTester::SetOutputAbsErr(const char* name, float v) { + auto it = std::find_if( + output_data_.begin(), + output_data_.end(), + [name](Data& data) { + return (data.def_.Name() == name); + }); + ONNXRUNTIME_ENFORCE(it != output_data_.end()); + it->absolute_error_ = optional(v); +} + +void OpTester::SetOutputRelErr(const char* name, float v) { + auto it = std::find_if( + output_data_.begin(), + output_data_.end(), + [name](Data& data) { + return (data.def_.Name() == name); + }); + ONNXRUNTIME_ENFORCE(it != output_data_.end()); + it->relative_error_ = optional(v); +} + +void OpTester::AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& add_attribute_funcs) { + // default behavior is to create a single Node for the op being tested, with node inputs/outputs + // being 1:1 with graph inputs/outputs. + auto& node = *graph.AddNode("node1", op_, op_, graph_input_defs, graph_output_defs, nullptr, domain_); + + // Add the attributes if any + for (auto& add_attribute_fn : add_attribute_funcs) + add_attribute_fn(node); +} + +void OpTester::AddInitializers(onnxruntime::Graph& graph) { + for (auto index : initializer_index_) { + auto& data = input_data_[index]; + auto& tensor = data.data_.Get(); + onnx::TensorProto tensor_proto; + //1. set dimension + auto& shape = tensor.Shape(); + for (auto& dim : shape.GetDims()) { + tensor_proto.add_dims(dim); + } + //2. set type + tensor_proto.set_data_type(data.def_.TypeAsProto()->tensor_type().elem_type()); + //3. data + if (data.def_.TypeAsProto()->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto_DataType_STRING) { + const std::string* string_data = tensor.Data(); + for (auto i = 0; i < shape.Size(); i++) { + tensor_proto.add_string_data(string_data[i]); + } + } else { + auto buffer_size = tensor.DataType()->Size() * shape.Size(); + tensor_proto.set_raw_data(tensor.DataRaw(), buffer_size); + } + //4. name + tensor_proto.set_name(data.def_.Name()); + graph.AddInitializedTensor(tensor_proto); + } +} + +std::unique_ptr OpTester::BuildGraph() { + // Generate the input & output def lists + std::vector node_input_defs; + std::vector output_defs; + + for (auto i = 0; i < input_data_.size(); ++i) { + node_input_defs.push_back(&input_data_[i].def_); + } + + for (auto& data : output_data_) { + output_defs.push_back(&data.def_); + } + + // Create a simple model + std::unordered_map domain_to_version; + domain_to_version[domain_] = opset_version_; + auto p_model = std::make_unique("test", false, ModelMetaData(), + custom_schema_registries_, domain_to_version); + onnxruntime::Graph& graph = p_model->MainGraph(); + AddNodes(graph, node_input_defs, output_defs, add_attribute_funcs_); + + //Add Initializer + AddInitializers(graph); + return p_model; +} + +void OpTester::Run(ExpectResult expect_result, + const std::string& expected_failure_string, + const std::unordered_set& excluded_provider_types, + const RunOptions* run_options) { + try { +#ifndef NDEBUG + run_called_ = true; +#endif + auto p_model = BuildGraph(); + auto& graph = p_model->MainGraph(); + + Status status = Status::OK(); + if (add_shape_to_tensor_data_ && expect_result == ExpectResult::kExpectFailure) { + // capture possible exceptions from shape inference for invalid testcase + try { + status = graph.Resolve(); + } catch (const std::exception& ex) { + status = ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, ex.what()); + } + } else { + status = graph.Resolve(); + } + + if (!status.IsOK()) { + if (expect_result == ExpectResult::kExpectFailure) { + EXPECT_TRUE(!status.IsOK()); + EXPECT_THAT(status.ErrorMessage(), testing::HasSubstr(expected_failure_string)); + } else { + LOGS_DEFAULT(ERROR) << "Resolve failed with status: " << status.ErrorMessage(); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + } + } + + if (!status.IsOK()) { + return; + } + + // Hookup the inputs and outputs + std::unordered_map feeds; + std::vector output_names; + FillFeedsAndOutputNames(feeds, output_names); + + // Run the model + SessionOptions so; + so.session_logid = op_; + so.session_log_verbosity_level = 1; + + static const std::string all_provider_types[] = { + kCpuExecutionProvider, + kCudaExecutionProvider, + kMklDnnExecutionProvider, + kNupharExecutionProvider, + kBrainSliceExecutionProvider, + }; + + bool has_run = false; + + for (const std::string& provider_type : all_provider_types) { + if (excluded_provider_types.count(provider_type) > 0) + continue; + + InferenceSession session_object{so}; + + for (auto& custom_session_registry : custom_session_registries_) + session_object.RegisterCustomRegistry(custom_session_registry); + + std::unique_ptr execution_provider; + if (provider_type == onnxruntime::kCpuExecutionProvider) + execution_provider = DefaultCpuExecutionProvider(); + else if (provider_type == onnxruntime::kCudaExecutionProvider) + execution_provider = DefaultCudaExecutionProvider(); + else if (provider_type == onnxruntime::kMklDnnExecutionProvider) + execution_provider = DefaultMkldnnExecutionProvider(); + else if (provider_type == onnxruntime::kNupharExecutionProvider) + execution_provider = DefaultNupharExecutionProvider(); + else if (provider_type == onnxruntime::kBrainSliceExecutionProvider) + execution_provider = DefaultBrainSliceExecutionProvider(); + // skip if execution provider is disabled + if (execution_provider == nullptr) + continue; + + bool valid = true; + + // set execution provider for all nodes in the graph + for (auto& node : graph.Nodes()) { + if (node.OpType() == kConstant) + continue; + + //if node is not registered for the provider, skip + node.SetExecutionProviderType(provider_type); + auto reg = execution_provider->GetKernelRegistry(); + const KernelCreateInfo* kci = reg->TryFindKernel(node, execution_provider->Type()); + if (!kci) { + valid = false; + break; + } + } + + if (!valid) + continue; + + has_run = true; + + EXPECT_TRUE(session_object.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); + + std::stringstream s1; + p_model->ToProto().SerializeToOstream(&s1); + status = session_object.Load(s1); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + if (!status.IsOK()) { + LOGS_DEFAULT(ERROR) << "Load failed with status: " << status.ErrorMessage(); + return; + } + + status = session_object.Initialize(); + if (!status.IsOK()) { + if (expect_result == ExpectResult::kExpectFailure) { + EXPECT_TRUE(!status.IsOK()); + EXPECT_THAT(status.ErrorMessage(), testing::HasSubstr(expected_failure_string)); + } else { + LOGS_DEFAULT(ERROR) << "Initialize failed with status: " << status.ErrorMessage(); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + } + } + if (!status.IsOK()) { + return; + } + + RunOptions default_run_options{}; + default_run_options.run_tag = op_; + default_run_options.run_log_verbosity_level = 1; + + std::vector fetches; + status = session_object.Run(run_options ? *run_options : default_run_options, feeds, output_names, &fetches); + if (status.IsOK()) { + EXPECT_TRUE(expect_result == ExpectResult::kExpectSuccess); + if (expect_result == ExpectResult::kExpectFailure) { + return; + } + } else { + if (expect_result == ExpectResult::kExpectFailure) { + EXPECT_THAT(status.ErrorMessage(), testing::HasSubstr(expected_failure_string)); + } else { + LOGS_DEFAULT(ERROR) << "Run failed with status: " << status.ErrorMessage(); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + } + return; + } + + // Verify the outputs + // Todo: support check output with map/sequence/.... + size_t idx = 0; + for (auto& expected_data : output_data_) { + MLValue& mlvalue = fetches[idx]; + if (mlvalue.Fence()) + mlvalue.Fence()->BeforeUsingAsInput(onnxruntime::kCpuExecutionProvider, 0); + + if (expected_data.def_.Exists()) { // optional outputs won't exist + if (expected_data.data_.IsTensor()) { + // verify output shape inference when input defs have shape + if (add_shape_to_tensor_data_) { + auto out_shape_proto = expected_data.def_.Shape(); + EXPECT_TRUE(out_shape_proto != nullptr); + auto inferred_dims = utils::GetTensorShapeFromTensorShapeProto(*out_shape_proto); + const auto& expected_shape = expected_data.data_.Get().Shape(); + EXPECT_TRUE(inferred_dims.size() == expected_shape.NumDimensions()); + for (int d = 0; d < inferred_dims.size(); ++d) { + EXPECT_EQ(expected_shape[d], inferred_dims[d]); + } + } + Check(expected_data, mlvalue.Get(), provider_type); + } else { + Check(expected_data, mlvalue, provider_type); + } + ++idx; + + // skip missing trailing optional outputs + if (idx == fetches.size()) + break; + } + } + } + + EXPECT_TRUE(has_run) << "No registered execution providers were able to run the model."; + } catch (const std::exception& ex) { + std::cerr << ex.what(); + // rethrow as some tests for error handling expect this + throw; + } +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/provider_test_utils.h b/onnxruntime/test/providers/provider_test_utils.h new file mode 100644 index 0000000000000..0ce06aea34bd9 --- /dev/null +++ b/onnxruntime/test/providers/provider_test_utils.h @@ -0,0 +1,331 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/logging/logging.h" +#include "core/framework/allocatormgr.h" +#include "core/framework/customregistry.h" +#include "core/framework/execution_frame.h" +#include "core/framework/op_kernel.h" +#include "core/framework/run_options.h" +#include "core/framework/session_state.h" +#include "core/framework/tensor.h" +#include "core/graph/graph.h" +#include "core/graph/model.h" +#include "core/framework/data_types.h" +#include "test/test_environment.h" +#include "test/framework/TestAllocatorManager.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include + +namespace onnxruntime { +namespace test { +// unfortunately std::optional is in C++17 so use a miniversion of it +template +class optional { + public: + optional(T v) : has_value_(true), value_(v) {} + optional() : has_value_(false) {} + bool has_value() const { return has_value_; } + const T& value() const { + ONNXRUNTIME_ENFORCE(has_value_); + return value_; + } + + private: + bool has_value_; + T value_; +}; + +// Function templates to translate C++ types into ONNX_NAMESPACE::TensorProto_DataTypes +template +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType(); + +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_FLOAT; } + +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_DOUBLE; } + +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_INT32; } + +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_INT64; } + +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_BOOL; } + +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_INT8; } + +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_INT16; } + +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_UINT8; } + +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_UINT16; } + +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_UINT32; } + +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_UINT64; } + +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_STRING; } + +template <> +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_FLOAT16; } + +template +struct TTypeProto : ONNX_NAMESPACE::TypeProto { + TTypeProto(const std::vector* shape = nullptr) { + mutable_tensor_type()->set_elem_type(TypeToDataType()); + + if (shape) { + auto mutable_shape = mutable_tensor_type()->mutable_shape(); + for (auto i : *shape) { + mutable_shape->add_dim()->set_dim_value(i); + } + } + } +}; + +// Variable template for ONNX_NAMESPACE::TensorProto_DataTypes, s_type_proto, etc.. +template +const TTypeProto s_type_proto; + +//TypeProto for map +template +struct MTypeProto : ONNX_NAMESPACE::TypeProto { + MTypeProto() { + mutable_map_type()->set_key_type(TypeToDataType()); + mutable_map_type()->mutable_value_type()->mutable_tensor_type()->set_elem_type(TypeToDataType()); + mutable_map_type()->mutable_value_type()->mutable_tensor_type()->mutable_shape()->clear_dim(); + } +}; + +template +const MTypeProto s_map_type_proto; + +//TypeProto for vector> +template +struct VectorOfMapTypeProto : ONNX_NAMESPACE::TypeProto { + VectorOfMapTypeProto() { + auto* map_type = mutable_sequence_type()->mutable_elem_type()->mutable_map_type(); + map_type->set_key_type(TypeToDataType()); + map_type->mutable_value_type()->mutable_tensor_type()->set_elem_type(TypeToDataType()); + map_type->mutable_value_type()->mutable_tensor_type()->mutable_shape()->clear_dim(); + } +}; + +template +const VectorOfMapTypeProto s_vec_map_type_proto; + +// To use OpTester: +// 1. Create one with the op name +// 2. Call AddAttribute with any attributes +// 3. Call AddInput for all the inputs +// 4. Call AddOutput with all expected outputs +// 5. Call Run +// Not all tensor types and output types are added, if a new input type is used, add it to the TypeToDataType list above +// for new output types, add a new specialization for Check<> +// See current usage for an example, should be self explanatory +class OpTester { + public: + OpTester(const char* op, int opset_version = 7, const char* domain = onnxruntime::kOnnxDomain) + : op_(op), domain_(domain), opset_version_(opset_version) {} + ~OpTester(); + + // Set whether the NodeArg created by AddInput/AddOutput should include shape information + // for Tensor types. If not added, shape inferencing should resolve. If added, shape inferencing + // should validate. Default is to not add. + OpTester& AddShapeToTensorData(bool add_shape = true, bool add_symbolic_dim = false) { + add_shape_to_tensor_data_ = add_shape; + add_symbolic_dim_to_tensor_data_ = add_symbolic_dim; + return *this; + } + + // We have an initializer_list and vector version of the Add functions because std::vector is specialized for + // bool and we can't get the raw data out. So those cases must use an initializer_list + template + void AddInput(const char* name, const std::vector& dims, const std::initializer_list& values, bool is_initializer = false) { + AddData(input_data_, name, dims, values.begin(), values.size(), is_initializer); + } + + template + void AddInput(const char* name, const std::vector& dims, const std::vector& values, bool is_initializer = false) { + AddData(input_data_, name, dims, values.data(), values.size(), is_initializer); + } + + template + void AddInput(const char* name, const std::map& val) { + std::unique_ptr> ptr = std::make_unique>(val); + MLValue value; + value.Init(ptr.release(), + DataTypeImpl::GetType>(), + DataTypeImpl::GetType>()->GetDeleteFunc()); + input_data_.push_back({{name, &s_map_type_proto}, value, optional(), optional()}); + } + + template + void AddMissingOptionalInput() { + std::string name; // empty == input doesn't exist + input_data_.push_back({{name, &s_type_proto}, {}, optional(), optional()}); + } + + template + void AddOutput(const char* name, const std::vector& dims, const std::initializer_list& expected_values) { + AddData(output_data_, name, dims, expected_values.begin(), expected_values.size()); + } + + template + void AddOutput(const char* name, const std::vector& dims, const std::vector& expected_values) { + AddData(output_data_, name, dims, expected_values.data(), expected_values.size()); + } + + template + void AddMissingOptionalOutput() { + std::string name; // empty == input doesn't exist + output_data_.push_back({{name, &s_type_proto}, {}, optional(), optional()}); + } + + // Add non tensor output + template + void AddOutput(const char* name, const std::vector>& val) { + auto ptr = std::make_unique>>(val); + MLValue ml_value; + ml_value.Init(ptr.release(), + DataTypeImpl::GetType>>(), + DataTypeImpl::GetType>>()->GetDeleteFunc()); + output_data_.push_back({{name, &s_vec_map_type_proto}, ml_value, optional(), optional()}); + } + + void AddCustomOpRegistry(std::shared_ptr registry) { + // need to do some static casting so we can easily use this later + custom_schema_registries_.push_back(std::static_pointer_cast(registry)); + custom_session_registries_.push_back(std::static_pointer_cast(registry)); + } + + void SetOutputAbsErr(const char* name, float v); + void SetOutputRelErr(const char* name, float v); + + template + void AddAttribute(std::string name, T value) { + // Generate a the proper AddAttribute call for later + add_attribute_funcs_.emplace_back( + [name = std::move(name), value = std::move(value)](onnxruntime::Node& node) { node.AddAttribute(name, value); }); + } + + enum class ExpectResult { + kExpectSuccess, + kExpectFailure + }; + + void Run(ExpectResult expect_result = ExpectResult::kExpectSuccess, const std::string& expected_failure_string = "", + const std::unordered_set& excluded_provider_types = {}, + const RunOptions* run_options = nullptr); + + struct Data { + onnxruntime::NodeArg def_; + MLValue data_; + optional relative_error_; + optional absolute_error_; + }; + + protected: + virtual void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& add_attribute_funcs); + + void AddInitializers(onnxruntime::Graph& graph); + + void FillFeedsAndOutputNames(std::unordered_map& feeds, + std::vector& output_names); + + std::unique_ptr BuildGraph(); + + const char* op_; + +#ifndef NDEBUG + bool run_called_{}; +#endif + + private: + template + void AddData(std::vector& data, const char* name, + const std::vector& dims, const T* values, + int64_t values_count, bool is_initializer = false) { + try { + TensorShape shape{dims}; + ONNXRUNTIME_ENFORCE(shape.Size() == values_count, values_count, + " input values doesn't match tensor size of ", shape.Size()); + + auto allocator = ::onnxruntime::test::AllocatorManager::Instance().GetAllocator(CPU); + auto size_in_bytes = values_count * sizeof(T); + void* buffer = allocator->Alloc(size_in_bytes); + auto p_tensor = std::make_unique(DataTypeImpl::GetType(), + shape, + buffer, + allocator->Info(), + allocator); + + auto* data_ptr = p_tensor->template MutableData(); + for (int64_t i = 0; i < values_count; i++) { + data_ptr[i] = values[i]; + } + + std::vector dims_for_proto{dims}; + if (add_symbolic_dim_to_tensor_data_ && !dims.empty()) { + dims_for_proto[0] = -1; + } + + TTypeProto type_proto(add_shape_to_tensor_data_ ? &dims_for_proto : nullptr); + MLValue value; + value.Init(p_tensor.release(), DataTypeImpl::GetType(), DataTypeImpl::GetType()->GetDeleteFunc()); + data.push_back({{name, &type_proto}, value, optional(), optional()}); + if (is_initializer) + initializer_index_.push_back(data.size() - 1); + } catch (const std::exception& ex) { + std::cerr << "AddData for '" << name << "' threw: " << ex.what(); + throw; + } + } + + const char* domain_; + int opset_version_; + bool add_shape_to_tensor_data_ = true; + bool add_symbolic_dim_to_tensor_data_ = false; + std::vector input_data_; + std::vector output_data_; + std::vector initializer_index_; + std::vector> add_attribute_funcs_; + + IOnnxRuntimeOpSchemaRegistryList custom_schema_registries_; + std::vector> custom_session_registries_; +}; + +template +void ExpectThrow(OpTester& test, const std::string& error_msg) { + try { + test.Run(); + // should throw and not reach this + EXPECT_TRUE(false) << "Expected Run() to throw"; + } catch (TException ex) { + EXPECT_THAT(ex.what(), testing::HasSubstr(error_msg)); + } +} + +void DebugTrap(); + +void Check(const OpTester::Data& expected_data, const Tensor& output_tensor, const std::string& provider_type); + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/test_main.cc b/onnxruntime/test/providers/test_main.cc new file mode 100644 index 0000000000000..3f780fd4f6895 --- /dev/null +++ b/onnxruntime/test/providers/test_main.cc @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/environment.h" +#include "gtest/gtest.h" +#include "test/test_environment.h" + +GTEST_API_ int main(int argc, char** argv) { + int status = 0; + + try { + ::onnxruntime::test::TestEnvironment test_environment{argc, argv}; + + status = RUN_ALL_TESTS(); + } catch (const std::exception& ex) { + std::cerr << ex.what(); + status = -1; + } + + return status; +} diff --git a/onnxruntime/test/python/onnx_backend_test_series.py b/onnxruntime/test/python/onnx_backend_test_series.py new file mode 100644 index 0000000000000..f1be2cd97acc7 --- /dev/null +++ b/onnxruntime/test/python/onnx_backend_test_series.py @@ -0,0 +1,65 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import os + +import unittest +import onnx.backend.test + +import onnxruntime.backend as c2 + +pytest_plugins = 'onnx.backend.test.report', + +backend_test = onnx.backend.test.BackendTest(c2, __name__) + +# Current status +# Ran 930 tests in 0.948s OK (skipped=574) + +# We should investigate. + +# dimension issue +# Error message is: Input X must be 4-dimensional. X: {1,1,3} +# ONNX Runtime expects a 4-dimension vector for operator ConvTranspose. +# file onnxruntime/core/providers/cpu/nn/conv_transpose.cc +backend_test.exclude(r'(convtranspose)') + +# Type not supported +backend_test.exclude(r'(FLOAT16)') + +# Operator not supported +backend_test.exclude(r'(test_expand)') +backend_test.exclude(r'(maxpool)') +backend_test.exclude(r'(AvgPool)') +backend_test.exclude(r'(max_one_input)|(max_example_cpu)|(max_two_inputs)') +backend_test.exclude(r'(min_one_input)|(min_example_cpu)|(min_two_inputs)') +backend_test.exclude(r'(mean_example)|(mean_one_input)|(mean_two_inputs)') +backend_test.exclude(r'(sum_example)|(sum_one_input)|(sum_two_inputs)') +backend_test.exclude(r'(BatchNorm)') +backend_test.exclude(r'(GLU)|(PReLU)|(PoissonNLLLLoss)|(Softsign)') +backend_test.exclude(r'(Linear_cpu)') +backend_test.exclude(r'(broadcast)') +backend_test.exclude(r'(addconstant)|(addmm)|(basic)|(lstm)') +backend_test.exclude(r'(_mm)|(non_float)|(test_operator_params_cpu)') +backend_test.exclude(r'(pow_cpu)|(rnn_cpu)|(rnn_single)|(_gru_)') +backend_test.exclude("(precomputed_((pads)|(strides)))") + +# Exclude deep learning +backend_test.exclude(r'(test_vgg19|test_vgg)') # Too long. +backend_test.exclude(r'(alexnet)') # Too long. +backend_test.exclude(r'(densenet)') # Too long. +backend_test.exclude(r'(inception)') # Too long. +backend_test.exclude(r'(resnet)') # Too long. +backend_test.exclude(r'(shufflenet)') # Too long. +backend_test.exclude(r'(squeezenet)') # Too long. +backend_test.exclude(r'(zfnet)') # Too long. + +# Exclude experimental +backend_test.exclude(r'(test_mvn_cpu)') + +# import all test cases at global scope to make +# them visible to python.unittest. +globals().update(backend_test.enable_report().test_cases) + + +if __name__ == '__main__': + unittest.main() diff --git a/onnxruntime/test/python/onnxruntime_test_python.py b/onnxruntime/test/python/onnxruntime_test_python.py new file mode 100644 index 0000000000000..91851264cc940 --- /dev/null +++ b/onnxruntime/test/python/onnxruntime_test_python.py @@ -0,0 +1,462 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# -*- coding: UTF-8 -*- +import unittest +import os +import sys +import numpy as np +import onnxruntime as onnxrt +from onnxruntime.capi._pybind_state import onnxruntime_ostream_redirect + + +class TestInferenceSession(unittest.TestCase): + + def get_name(self, name): + if os.path.exists(name): + return name + rel = os.path.join("testdata", name) + if os.path.exists(rel): + return rel + this = os.path.dirname(__file__) + data = os.path.join(this, "..", "testdata") + res = os.path.join(data, name) + if os.path.exists(res): + return res + raise FileNotFoundError("Unable to find '{0}' or '{1}' or '{2}'".format(name, rel, res)) + + def testRunModel(self): + sess = onnxrt.InferenceSession(self.get_name("mul_1.pb")) + x = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32) + input_name = sess.get_inputs()[0].name + self.assertEqual(input_name, "X") + input_shape = sess.get_inputs()[0].shape + self.assertEqual(input_shape, [3, 2]) + output_name = sess.get_outputs()[0].name + self.assertEqual(output_name, "Y") + output_shape = sess.get_outputs()[0].shape + self.assertEqual(output_shape, [3, 2]) + res = sess.run([output_name], {input_name: x}) + output_expected = np.array([[1.0, 4.0], [9.0, 16.0], [25.0, 36.0]], dtype=np.float32) + np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08) + + def testRunModelFromBytes(self): + with open(self.get_name("mul_1.pb"), "rb") as f: + content = f.read() + sess = onnxrt.InferenceSession(content) + x = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32) + input_name = sess.get_inputs()[0].name + self.assertEqual(input_name, "X") + input_shape = sess.get_inputs()[0].shape + self.assertEqual(input_shape, [3, 2]) + output_name = sess.get_outputs()[0].name + self.assertEqual(output_name, "Y") + output_shape = sess.get_outputs()[0].shape + self.assertEqual(output_shape, [3, 2]) + res = sess.run([output_name], {input_name: x}) + output_expected = np.array([[1.0, 4.0], [9.0, 16.0], [25.0, 36.0]], dtype=np.float32) + np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08) + + def testRunModel2(self): + sess = onnxrt.InferenceSession(self.get_name("matmul_1.pb")) + x = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32) + input_name = sess.get_inputs()[0].name + self.assertEqual(input_name, "X") + input_shape = sess.get_inputs()[0].shape + self.assertEqual(input_shape, [3, 2]) + output_name = sess.get_outputs()[0].name + self.assertEqual(output_name, "Y") + output_shape = sess.get_outputs()[0].shape + self.assertEqual(output_shape, [3, 1]) + res = sess.run([output_name], {input_name: x}) + output_expected = np.array([[5.0], [11.0], [17.0]], dtype=np.float32) + np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08) + + def testRunDevice(self): + device = onnxrt.get_device() + self.assertTrue('CPU' in device or 'GPU' in device) + + def testRunModelSymbolicInput(self): + sess = onnxrt.InferenceSession(self.get_name("matmul_2.pb")) + x = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32) + input_name = sess.get_inputs()[0].name + self.assertEqual(input_name, "X") + input_shape = sess.get_inputs()[0].shape + # Input X has an unknown dimension. + self.assertEqual(input_shape, [None, 2]) + output_name = sess.get_outputs()[0].name + self.assertEqual(output_name, "Y") + output_shape = sess.get_outputs()[0].shape + # Output X has an unknown dimension. + self.assertEqual(output_shape, [None, 1]) + res = sess.run([output_name], {input_name: x}) + output_expected = np.array([[5.0], [11.0], [17.0]], dtype=np.float32) + np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08) + + def testBooleanInputs(self): + sess = onnxrt.InferenceSession(self.get_name("logicaland.pb")) + a = np.array([[True, True], [False, False]], dtype=np.bool) + b = np.array([[True, False], [True, False]], dtype=np.bool) + + # input1:0 is first in the protobuf, and input:0 is second + # and we maintain the original order. + a_name = sess.get_inputs()[0].name + self.assertEqual(a_name, "input1:0") + a_shape = sess.get_inputs()[0].shape + self.assertEqual(a_shape, [2, 2]) + a_type = sess.get_inputs()[0].type + self.assertEqual(a_type, 'tensor(bool)') + + b_name = sess.get_inputs()[1].name + self.assertEqual(b_name, "input:0") + b_shape = sess.get_inputs()[1].shape + self.assertEqual(b_shape, [2, 2]) + b_type = sess.get_inputs()[0].type + self.assertEqual(b_type, 'tensor(bool)') + + output_name = sess.get_outputs()[0].name + self.assertEqual(output_name, "output:0") + output_shape = sess.get_outputs()[0].shape + self.assertEqual(output_shape, [2, 2]) + output_type = sess.get_outputs()[0].type + self.assertEqual(output_type, 'tensor(bool)') + + output_expected = np.array([[True, False], [False, False]], dtype=np.bool) + res = sess.run([output_name], {a_name: a, b_name: b}) + np.testing.assert_equal(output_expected, res[0]) + + def testStringInput1(self): + sess = onnxrt.InferenceSession(self.get_name("identity_string.pb")) + x = np.array(['this', 'is', 'identity', 'test'], dtype=np.str).reshape((2,2)) + + x_name = sess.get_inputs()[0].name + self.assertEqual(x_name, "input:0") + x_shape = sess.get_inputs()[0].shape + self.assertEqual(x_shape, [2, 2]) + x_type = sess.get_inputs()[0].type + self.assertEqual(x_type, 'tensor(string)') + + output_name = sess.get_outputs()[0].name + self.assertEqual(output_name, "output:0") + output_shape = sess.get_outputs()[0].shape + self.assertEqual(output_shape, [2, 2]) + output_type = sess.get_outputs()[0].type + self.assertEqual(output_type, 'tensor(string)') + + res = sess.run([output_name], {x_name: x}) + np.testing.assert_equal(x, res[0]) + + def testStringInput2(self): + sess = onnxrt.InferenceSession(self.get_name("identity_string.pb")) + x = np.array(['Olá', '你好', '여보세요', 'hello'], dtype=np.unicode).reshape((2,2)) + + x_name = sess.get_inputs()[0].name + self.assertEqual(x_name, "input:0") + x_shape = sess.get_inputs()[0].shape + self.assertEqual(x_shape, [2, 2]) + x_type = sess.get_inputs()[0].type + self.assertEqual(x_type, 'tensor(string)') + + output_name = sess.get_outputs()[0].name + self.assertEqual(output_name, "output:0") + output_shape = sess.get_outputs()[0].shape + self.assertEqual(output_shape, [2, 2]) + output_type = sess.get_outputs()[0].type + self.assertEqual(output_type, 'tensor(string)') + + res = sess.run([output_name], {x_name: x}) + np.testing.assert_equal(x, res[0]) + + def testInputBytes(self): + sess = onnxrt.InferenceSession(self.get_name("identity_string.pb")) + x = np.array([b'this', b'is', b'identity', b'test']).reshape((2,2)) + + x_name = sess.get_inputs()[0].name + self.assertEqual(x_name, "input:0") + x_shape = sess.get_inputs()[0].shape + self.assertEqual(x_shape, [2, 2]) + x_type = sess.get_inputs()[0].type + self.assertEqual(x_type, 'tensor(string)') + + output_name = sess.get_outputs()[0].name + self.assertEqual(output_name, "output:0") + output_shape = sess.get_outputs()[0].shape + self.assertEqual(output_shape, [2, 2]) + output_type = sess.get_outputs()[0].type + self.assertEqual(output_type, 'tensor(string)') + + res = sess.run([output_name], {x_name: x}) + np.testing.assert_equal(x, res[0].astype('|S8')) + + def testInputObject(self): + sess = onnxrt.InferenceSession(self.get_name("identity_string.pb")) + x = np.array(['this', 'is', 'identity', 'test'], object).reshape((2,2)) + + x_name = sess.get_inputs()[0].name + self.assertEqual(x_name, "input:0") + x_shape = sess.get_inputs()[0].shape + self.assertEqual(x_shape, [2, 2]) + x_type = sess.get_inputs()[0].type + self.assertEqual(x_type, 'tensor(string)') + + output_name = sess.get_outputs()[0].name + self.assertEqual(output_name, "output:0") + output_shape = sess.get_outputs()[0].shape + self.assertEqual(output_shape, [2, 2]) + output_type = sess.get_outputs()[0].type + self.assertEqual(output_type, 'tensor(string)') + + res = sess.run([output_name], {x_name: x}) + np.testing.assert_equal(x, res[0]) + + def testInputVoid(self): + sess = onnxrt.InferenceSession(self.get_name("identity_string.pb")) + x = np.array([b'this', b'is', b'identity', b'test'], np.void).reshape((2,2)) + + x_name = sess.get_inputs()[0].name + self.assertEqual(x_name, "input:0") + x_shape = sess.get_inputs()[0].shape + self.assertEqual(x_shape, [2, 2]) + x_type = sess.get_inputs()[0].type + self.assertEqual(x_type, 'tensor(string)') + + output_name = sess.get_outputs()[0].name + self.assertEqual(output_name, "output:0") + output_shape = sess.get_outputs()[0].shape + self.assertEqual(output_shape, [2, 2]) + output_type = sess.get_outputs()[0].type + self.assertEqual(output_type, 'tensor(string)') + + res = sess.run([output_name], {x_name: x}) + + expr = np.array([['this\x00\x00\x00\x00', 'is\x00\x00\x00\x00\x00\x00'], + ['identity', 'test\x00\x00\x00\x00']], dtype=object) + np.testing.assert_equal(expr, res[0]) + + def testConvAutoPad(self): + sess = onnxrt.InferenceSession(self.get_name("conv_autopad.pb")) + x = np.array(25 * [1.0], dtype=np.float32).reshape((1,1,5,5)) + + x_name = sess.get_inputs()[0].name + self.assertEqual(x_name, "Input4") + x_shape = sess.get_inputs()[0].shape + self.assertEqual(x_shape, [1, 1, 5, 5]) + x_type = sess.get_inputs()[0].type + self.assertEqual(x_type, 'tensor(float)') + + output_name = sess.get_outputs()[0].name + self.assertEqual(output_name, "Convolution5_Output_0") + output_shape = sess.get_outputs()[0].shape + self.assertEqual(output_shape, [1, 1, 5, 5]) + output_type = sess.get_outputs()[0].type + self.assertEqual(output_type, 'tensor(float)') + + res = sess.run([output_name], {x_name: x}) + output_expected = np.array([[[[24., 33., 33., 33., 20.], + [27., 36., 36., 36., 21.], + [27., 36., 36., 36., 21.], + [27., 36., 36., 36., 21.], + [12., 15., 15., 15., 8.]]]], dtype=np.float32) + np.testing.assert_allclose(output_expected, res[0]) + + def testZipMapStringFloat(self): + sess = onnxrt.InferenceSession(self.get_name("zipmap_stringfloat.pb")) + x = np.array([1.0, 0.0, 3.0, 44.0, 23.0, 11.0], dtype=np.float32).reshape((2,3)) + + x_name = sess.get_inputs()[0].name + self.assertEqual(x_name, "X") + x_type = sess.get_inputs()[0].type + self.assertEqual(x_type, 'tensor(float)') + + output_name = sess.get_outputs()[0].name + self.assertEqual(output_name, "Z") + output_type = sess.get_outputs()[0].type + self.assertEqual(output_type, 'seq(map(string,tensor(float)))') + + output_expected = [{'class2': 0.0, 'class1': 1.0, 'class3': 3.0}, + {'class2': 23.0, 'class1': 44.0, 'class3': 11.0}] + res = sess.run([output_name], {x_name: x}) + self.assertEqual(output_expected, res[0]) + + def testZipMapInt64Float(self): + sess = onnxrt.InferenceSession(self.get_name("zipmap_int64float.pb")) + x = np.array([1.0, 0.0, 3.0, 44.0, 23.0, 11.0], dtype=np.float32).reshape((2,3)) + + x_name = sess.get_inputs()[0].name + self.assertEqual(x_name, "X") + x_type = sess.get_inputs()[0].type + self.assertEqual(x_type, 'tensor(float)') + + output_name = sess.get_outputs()[0].name + self.assertEqual(output_name, "Z") + output_type = sess.get_outputs()[0].type + self.assertEqual(output_type, 'seq(map(int64,tensor(float)))') + + output_expected = [{10: 1.0, 20: 0.0, 30: 3.0}, {10: 44.0, 20: 23.0, 30: 11.0}] + res = sess.run([output_name], {x_name: x}) + self.assertEqual(output_expected, res[0]) + + def testRaiseWrongNumInputs(self): + with self.assertRaises(ValueError) as context: + sess = onnxrt.InferenceSession(self.get_name("logicaland.pb")) + a = np.array([[True, True], [False, False]], dtype=np.bool) + res = sess.run([], {'input:0': a}) + + self.assertTrue('Model requires 2 inputs' in str(context.exception)) + + def testModelMeta(self): + model_path = "../models/opset8/test_squeezenet/model.onnx" + if not os.path.exists(model_path): + return + sess = onnxrt.InferenceSession(model_path) + modelmeta = sess.get_modelmeta() + self.assertEqual('onnx-caffe2', modelmeta.producer_name) + self.assertEqual('squeezenet_old', modelmeta.graph_name) + self.assertEqual('', modelmeta.domain) + self.assertEqual('', modelmeta.description) + + def testConfigureSessionVerbosityLevel(self): + so = onnxrt.SessionOptions() + so.session_log_verbosity_level = 1 + + # use onnxruntime_ostream_redirect to redirect c++ stdout/stderr to python sys.stdout and sys.stderr + with onnxruntime_ostream_redirect(stdout=True, stderr=True): + sess = onnxrt.InferenceSession(self.get_name("matmul_1.pb"), sess_options=so) + output = sys.stderr.getvalue() + self.assertTrue('[I:onnxruntime:InferenceSession, inference_session' in output) + + def testConfigureRunVerbosityLevel(self): + ro = onnxrt.RunOptions() + ro.run_log_verbosity_level = 1 + ro.run_tag = "testtag123" + + # use onnxruntime_ostream_redirect to redirect c++ stdout/stderr to python sys.stdout and sys.stderr + with onnxruntime_ostream_redirect(stdout=True, stderr=True): + sess = onnxrt.InferenceSession(self.get_name("mul_1.pb")) + x = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32) + sess.run([], {'X': x}, run_options=ro) + output = sys.stderr.getvalue() + self.assertTrue('[I:onnxruntime:testtag123,' in output) + + def testProfilerWithSessionOptions(self): + so = onnxrt.SessionOptions() + so.enable_profiling = True + sess = onnxrt.InferenceSession(self.get_name("mul_1.pb"), sess_options=so) + x = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32) + sess.run([], {'X': x}) + profile_file = sess.end_profiling() + + tags = ['pid', 'dur', 'ts', 'ph', 'X', 'name', 'args'] + with open(profile_file) as f: + lines = f.readlines() + self.assertTrue('[' in lines[0]) + for i in range(1, 8): + for tag in tags: + self.assertTrue(tag in lines[i]) + self.assertTrue(']' in lines[8]) + + def testDictVectorizer(self): + sess = onnxrt.InferenceSession(self.get_name("pipeline_vectorize.onnx")) + input_name = sess.get_inputs()[0].name + self.assertEqual(input_name, "float_input") + input_type = str(sess.get_inputs()[0].type) + self.assertEqual(input_type, "map(int64,tensor(float))") + input_shape = sess.get_inputs()[0].shape + self.assertEqual(input_shape, []) + output_name = sess.get_outputs()[0].name + self.assertEqual(output_name, "variable1") + output_type = sess.get_outputs()[0].type + self.assertEqual(output_type, "tensor(float)") + output_shape = sess.get_outputs()[0].shape + self.assertEqual(output_shape, [1, 1]) + + # Python type + x = {0: 25.0, 1: 5.13, 2: 0.0, 3: 0.453, 4: 5.966} + res = sess.run([output_name], {input_name: x}) + output_expected = np.array([[49.752754]], dtype=np.float32) + np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08) + + xwrong = x.copy() + xwrong["a"] = 5.6 + try: + res = sess.run([output_name], {input_name: xwrong}) + except RuntimeError as e: + self.assertIn("Unexpected key type , it cannot be linked to C type int64_t", str(e)) + + # numpy type + x = {np.int64(k): np.float32(v) for k, v in x.items()} + res = sess.run([output_name], {input_name: x}) + output_expected = np.array([[49.752754]], dtype=np.float32) + np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08) + + x = {np.int64(k): np.float64(v) for k, v in x.items()} + res = sess.run([output_name], {input_name: x}) + output_expected = np.array([[49.752754]], dtype=np.float32) + np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08) + + x = {np.int32(k): np.float64(v) for k, v in x.items()} + res = sess.run([output_name], {input_name: x}) + output_expected = np.array([[49.752754]], dtype=np.float32) + np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08) + + def testLabelEncoder(self): + sess = onnxrt.InferenceSession(self.get_name("LabelEncoder.pb")) + input_name = sess.get_inputs()[0].name + self.assertEqual(input_name, "input") + input_type = str(sess.get_inputs()[0].type) + self.assertEqual(input_type, "tensor(string)") + input_shape = sess.get_inputs()[0].shape + self.assertEqual(input_shape, [1, 1]) + output_name = sess.get_outputs()[0].name + self.assertEqual(output_name, "variable") + output_type = sess.get_outputs()[0].type + self.assertEqual(output_type, "tensor(int64)") + output_shape = sess.get_outputs()[0].shape + self.assertEqual(output_shape, [1, 1]) + + # Array + x = np.array([['4']]) + res = sess.run([output_name], {input_name: x}) + output_expected = np.array([[3]], dtype=np.int64) + np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08) + + # Python type + x = np.array(['4']) + res = sess.run([output_name], {input_name: x}) + output_expected = np.array([3], dtype=np.int64) + np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08) + + x = np.array(['4'], dtype=np.object) + res = sess.run([output_name], {input_name: x}) + output_expected = np.array([3], dtype=np.int64) + np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08) + + def test_run_model_mlnet(self): + sess = onnxrt.InferenceSession(self.get_name("mlnet_encoder.onnx")) + names = [_.name for _ in sess.get_outputs()] + self.assertEqual(['C00', 'C12'], names) + c0 = np.array([5.], dtype=np.float32).reshape(1, 1); + + c1 = np.array([b'A\0A\0', b"B\0B\0", b"C\0C\0"], np.void).reshape(1, 3) + res = sess.run(None, {'C0': c0, 'C1': c1}) + mat = res[1] + total = mat.sum() + self.assertEqual(total, 2) + self.assertEqual(list(mat.ravel()), + list(np.array([[[0., 0., 0., 0.], [1., 0., 0., 0.], [0., 0., 1., 0.]]]).ravel())) + + # In memory, the size of each element is fixed and equal to the + # longest element. We cannot use bytes because numpy is trimming + # every final 0 for strings and bytes before creating the array + # (to save space). It does not have this behaviour for void + # but as a result, numpy does not know anymore the size + # of each element, they all have the same size. + c1 = np.array([b'A\0A\0\0', b"B\0B\0", b"C\0C\0"], np.void).reshape(1, 3) + res = sess.run(None, {'C0': c0, 'C1': c1}) + mat = res[1] + total = mat.sum() + self.assertEqual(total, 0) + + +if __name__ == '__main__': + unittest.main(module=__name__, buffer=True) diff --git a/onnxruntime/test/python/onnxruntime_test_python_backend.py b/onnxruntime/test/python/onnxruntime_test_python_backend.py new file mode 100644 index 0000000000000..2df6fed0cc79c --- /dev/null +++ b/onnxruntime/test/python/onnxruntime_test_python_backend.py @@ -0,0 +1,62 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# -*- coding: UTF-8 -*- +import unittest +import os +import sys +import numpy as np +import onnxruntime as onnxrt +from onnxruntime import datasets +import onnxruntime.backend as backend +from onnx import load + + +class TestBackend(unittest.TestCase): + + def get_name(self, name): + if os.path.exists(name): + return name + rel = os.path.join("testdata", name) + if os.path.exists(rel): + return rel + this = os.path.dirname(__file__) + data = os.path.join(this, "..", "testdata") + res = os.path.join(data, name) + if os.path.exists(res): + return res + raise FileNotFoundError("Unable to find '{0}' or '{1}' or '{2}'".format(name, rel, res)) + + def testRunModel(self): + name = self.get_name("mul_1.pb") + rep = backend.prepare(name) + x = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32) + res = rep.run(x) + output_expected = np.array([[1.0, 4.0], [9.0, 16.0], [25.0, 36.0]], dtype=np.float32) + np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08) + + def testRunModelNonTensor(self): + name = self.get_name("pipeline_vectorize.onnx") + rep = backend.prepare(name) + x = {0: 25.0, 1: 5.13, 2: 0.0, 3: 0.453, 4: 5.966} + res = rep.run(x) + output_expected = np.array([[49.752754]], dtype=np.float32) + np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08) + + def testRunModelProto(self): + name = datasets.get_example("logreg_iris.onnx") + model = load(name) + + rep = backend.prepare(model) + x = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32) + res = rep.run(x) + output_expected = np.array([0, 0, 0], dtype=np.float32) + np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08) + output_expected = [{0: 0.950599730014801, 1: 0.027834169566631317, 2: 0.02156602405011654}, + {0: 0.9974970817565918, 1: 5.6299926654901356e-05, 2: 0.0024466661270707846}, + {0: 0.9997311234474182, 1: 1.1918064757310276e-07, 2: 0.00026869276189245284}] + self.assertEqual(output_expected, res[1]) + + +if __name__ == '__main__': + unittest.main(module=__name__, buffer=True) diff --git a/onnxruntime/test/python/onnxruntime_test_python_keras.py b/onnxruntime/test/python/onnxruntime_test_python_keras.py new file mode 100644 index 0000000000000..715a2a3490b3f --- /dev/null +++ b/onnxruntime/test/python/onnxruntime_test_python_keras.py @@ -0,0 +1,71 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# -*- coding: UTF-8 -*- +# Taken from https://github.com/onnx/onnxmltools/blob/master/tests/end2end/test_custom_op.py. +import unittest +import os +import sys +import numpy as np +import onnxmltools +import onnxruntime as onnxrt +from keras import backend as K +from keras import Sequential +from keras.layers import Layer, Conv2D, MaxPooling2D + + +class ScaledTanh(Layer): + def __init__(self, alpha=1.0, beta=1.0, **kwargs): + super(ScaledTanh, self).__init__(**kwargs) + self.alpha = alpha + self.beta = beta + + def build(self, input_shape): + super(ScaledTanh, self).build(input_shape) + + def call(self, x): + return self.alpha * K.tanh(self.beta * x) + + def compute_output_shape(self, input_shape): + return input_shape + + +def custom_activation(scope, operator, container): + # type:(ScopeBase, OperatorBase, ModelContainer) -> None + container.add_node('ScaledTanh', operator.input_full_names, operator.output_full_names, + op_version=1, alpha=operator.original_operator.alpha, beta=operator.original_operator.beta) + + +class TestInferenceSessionKeras(unittest.TestCase): + + def testRunModelConv(self): + + # keras model + N, C, H, W = 2, 3, 5, 5 + x = np.random.rand(N, H, W, C).astype(np.float32, copy=False) + + model = Sequential() + model.add(Conv2D(2, kernel_size=(1, 2), strides=(1, 1), padding='valid', input_shape=(H, W, C), + data_format='channels_last')) + model.add(ScaledTanh(0.9, 2.0)) + model.add(MaxPooling2D((2, 2), strides=(2, 2), data_format='channels_last')) + + model.compile(optimizer='sgd', loss='mse') + actual = model.predict(x) + self.assertIsNotNone(actual) + + # conversion + converted_model = onnxmltools.convert_keras(model, custom_conversion_functions={ScaledTanh: custom_activation}) + self.assertIsNotNone(converted_model) + + # runtime + content = converted_model.SerializeToString() + rt = onnxrt.InferenceSession(content) + input = {'conv2d_1_input_0': x} + actual_rt = rt.run(None, input) + self.assertEqual(len(actual_rt), 1) + np.testing.assert_allclose(actual, actual_rt[0], rtol=1e-05, atol=1e-08) + + +if __name__ == '__main__': + unittest.main(module=__name__, buffer=True) diff --git a/onnxruntime/test/shared_lib/fns_candy_style_transfer.c b/onnxruntime/test/shared_lib/fns_candy_style_transfer.c new file mode 100644 index 0000000000000..bb5930ef3b687 --- /dev/null +++ b/onnxruntime/test/shared_lib/fns_candy_style_transfer.c @@ -0,0 +1,218 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "core/session/onnxruntime_c_api.h" +#include "core/framework/allocator_info.h" +#include "providers.h" +#include +#include +#include + +#define ONNXRUNTIME_ABORT_ON_ERROR(expr) \ + do { \ + ONNXStatusPtr onnx_status = (expr); \ + if (onnx_status != NULL) { \ + const char* msg = ONNXRuntimeGetErrorMessage(onnx_status); \ + fprintf(stderr, "%s\n", msg); \ + ReleaseONNXStatus(onnx_status); \ + abort(); \ + } \ + } while (0); + +/** + * convert input from HWC format to CHW format + * \param input A single image. The byte array has length of 3*h*w + * \param h image height + * \param w image width + * \param output A float array. should be freed by caller after use + * \param output_count Array length of the `output` param + */ +static void hwc_to_chw(const png_byte* input, size_t h, size_t w, float** output, size_t* output_count) { + size_t stride = h * w; + *output_count = stride * 3; + float* output_data = (float*)malloc(*output_count * sizeof(float)); + for (size_t i = 0; i != stride; ++i) { + for (size_t c = 0; c != 3; ++c) { + output_data[c * stride + i] = input[i * 3 + c]; + } + } + *output = output_data; +} + +/** + * convert input from CHW format to HWC format + * \param input A single image. This float array has length of 3*h*w + * \param h image height + * \param w image width + * \param output A byte array. should be freed by caller after use + */ +static void chw_to_hwc(const float* input, size_t h, size_t w, png_bytep* output) { + size_t stride = h * w; + png_bytep output_data = (png_bytep)malloc(stride * 3); + for (int c = 0; c != 3; ++c) { + size_t t = c * stride; + for (size_t i = 0; i != stride; ++i) { + float f = input[t + i]; + if (f < 0.f || f > 255.0f) + f = 0; + output_data[i * 3 + c] = (png_byte)f; + } + } + *output = output_data; +} + +/** + * \param out should be freed by caller after use + * \param output_count Array length of the `out` param + */ +static int read_png_file(const char* input_file, size_t* height, size_t* width, float** out, size_t* output_count) { + png_image image; /* The control structure used by libpng */ + /* Initialize the 'png_image' structure. */ + memset(&image, 0, (sizeof image)); + image.version = PNG_IMAGE_VERSION; + if (png_image_begin_read_from_file(&image, input_file) == 0) { + return -1; + } + png_bytep buffer; + image.format = PNG_FORMAT_BGR; + size_t input_data_length = PNG_IMAGE_SIZE(image); + if (input_data_length != 720 * 720 * 3) { + printf("input_data_length:%zd\n", input_data_length); + return -1; + } + buffer = (png_bytep)malloc(input_data_length); + memset(buffer, 0, input_data_length); + if (png_image_finish_read(&image, NULL /*background*/, buffer, + 0 /*row_stride*/, NULL /*colormap*/) == 0) { + return -1; + } + hwc_to_chw(buffer, image.height, image.width, out, output_count); + free(buffer); + *width = image.width; + *height = image.height; + return 0; +} + +/** + * \param tensor should be a float tensor in [N,C,H,W] format + */ +static int write_tensor_to_png_file(const ONNXValuePtr tensor, const char* output_file) { + struct ONNXRuntimeTensorTypeAndShapeInfo* shape_info; + ONNXRUNTIME_ABORT_ON_ERROR(ONNXRuntimeGetTensorShapeAndType(tensor, &shape_info)); + size_t dim_count = ONNXRuntimeGetNumOfDimensions(shape_info); + if (dim_count != 4) { + printf("output tensor must have 4 dimensions"); + return -1; + } + int64_t dims[4]; + ONNXRuntimeGetDimensions(shape_info, dims, sizeof(dims) / sizeof(dims[0])); + if (dims[0] != 1 || dims[1] != 3) { + printf("output tensor shape error"); + return -1; + } + float* f; + ONNXRUNTIME_ABORT_ON_ERROR(ONNXRuntimeGetTensorMutableData(tensor, (void**)&f)); + png_bytep model_output_bytes; + png_image image; + memset(&image, 0, (sizeof image)); + image.version = PNG_IMAGE_VERSION; + image.format = PNG_FORMAT_BGR; + image.height = dims[2]; + image.width = dims[3]; + chw_to_hwc(f, image.height, image.width, &model_output_bytes); + int ret = 0; + if (png_image_write_to_file(&image, output_file, 0 /*convert_to_8bit*/, + model_output_bytes, 0 /*row_stride*/, NULL /*colormap*/) == 0) { + printf("write to '%s' failed:%s\n", output_file, image.message); + ret = -1; + } + free(model_output_bytes); + return ret; +} + +static void usage() { + printf("usage: \n"); +} + +int run_inference(ONNXSessionPtr session, const char* input_file, const char* output_file) { + size_t input_height; + size_t input_width; + float* model_input; + size_t model_input_ele_count; + if (read_png_file(input_file, &input_height, &input_width, &model_input, &model_input_ele_count) != 0) { + return -1; + } + if (input_height != 720 || input_width != 720) { + printf("please resize to image to 720x720\n"); + free(model_input); + return -1; + } + ONNXRuntimeAllocatorInfoPtr allocator_info; + ONNXRUNTIME_ABORT_ON_ERROR(ONNXRuntimeCreateCpuAllocatorInfo(ONNXRuntimeArenaAllocator, ONNXRuntimeMemTypeDefault, &allocator_info)); + const size_t input_shape[] = {1, 3, 720, 720}; + const size_t input_shape_len = sizeof(input_shape) / sizeof(input_shape[0]); + const size_t model_input_len = model_input_ele_count * sizeof(float); + + ONNXValuePtr input_tensor[] = {NULL}; + ONNXRUNTIME_ABORT_ON_ERROR(ONNXRuntimeCreateTensorWithDataAsONNXValue(allocator_info, model_input, model_input_len, input_shape, input_shape_len, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, &input_tensor[0])); + assert(input_tensor[0] != NULL); + assert(ONNXRuntimeIsTensor(input_tensor[0]) != 0); + ReleaseONNXRuntimeAllocatorInfo(allocator_info); + const char* input_names[] = {"inputImage"}; + const char* output_names[] = {"outputImage"}; + ONNXValuePtr output_tensor[] = {NULL}; + ONNXRUNTIME_ABORT_ON_ERROR(ONNXRuntimeRunInference(session, input_names, input_tensor, 1, output_names, 1, output_tensor)); + assert(output_tensor[0] != NULL); + assert(ONNXRuntimeIsTensor(output_tensor[0]) != 0); + int ret = 0; + if (write_tensor_to_png_file(output_tensor[0], output_file) != 0) { + ret = -1; + } + ReleaseONNXValue(output_tensor[0]); + ReleaseONNXValue(input_tensor[0]); + free(model_input); + return ret; +} + +void verify_input_output_count(ONNXSessionPtr session) { + size_t count; + ONNXRUNTIME_ABORT_ON_ERROR(ONNXRuntimeInferenceSessionGetInputCount(session, &count)); + assert(count == 1); + ONNXRUNTIME_ABORT_ON_ERROR(ONNXRuntimeInferenceSessionGetOutputCount(session, &count)); + assert(count == 1); +} + +#ifdef USE_CUDA +void enable_cuda(ONNXRuntimeSessionOptions* session_option) { + ONNXRuntimeProviderFactoryPtr* factory; + ONNXRUNTIME_ABORT_ON_ERROR(ONNXRuntimeCreateCUDAExecutionProviderFactory(0, &factory)); + ONNXRuntimeSessionOptionsAppendExecutionProvider(session_option, factory); + ONNXRuntimeReleaseObject(factory); +} +#endif + +int main(int argc, char* argv[]) { + if (argc < 4) { + usage(); + return -1; + } + char* model_path = argv[1]; + char* input_file = argv[2]; + char* output_file = argv[3]; + ONNXEnv* env; + ONNXRUNTIME_ABORT_ON_ERROR(ONNXRuntimeInitialize(ONNXRUNTIME_LOGGING_LEVEL_kWARNING, "test", &env)); + ONNXRuntimeSessionOptions* session_option = ONNXRuntimeCreateSessionOptions(); +#ifdef USE_CUDA + enable_cuda(session_option); +#endif + ONNXSessionPtr session; + ONNXRUNTIME_ABORT_ON_ERROR(ONNXRuntimeCreateInferenceSession(env, model_path, session_option, &session)); + verify_input_output_count(session); + int ret = run_inference(session, input_file, output_file); + ONNXRuntimeReleaseObject(session_option); + ReleaseONNXSession(session); + ReleaseONNXEnv(env); + if (ret != 0) { + fprintf(stderr, "fail\n"); + } + return ret; +} \ No newline at end of file diff --git a/onnxruntime/test/shared_lib/test_allocator.cc b/onnxruntime/test/shared_lib/test_allocator.cc new file mode 100644 index 0000000000000..fe04c137cc92b --- /dev/null +++ b/onnxruntime/test/shared_lib/test_allocator.cc @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/onnxruntime_cxx_api.h" +#include "core/framework/allocator.h" +#include "core/providers/cpu/cpu_provider_factory.h" +#include "test_fixture.h" + +using namespace onnxruntime; + +TEST_F(CApiTest, allocation_info) { + ONNXRuntimeAllocatorInfoPtr info1, info2; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateAllocatorInfo("Cpu", ONNXRuntimeArenaAllocator, 0, ONNXRuntimeMemTypeDefault, &info1)); + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateCpuAllocatorInfo(ONNXRuntimeArenaAllocator, ONNXRuntimeMemTypeDefault, &info2)); + ASSERT_EQ(0, ONNXRuntimeCompareAllocatorInfo(info1, info2)); + ReleaseONNXRuntimeAllocatorInfo(info1); + ReleaseONNXRuntimeAllocatorInfo(info2); +} + +TEST_F(CApiTest, DefaultAllocator) { + std::unique_ptr default_allocator; + { + ONNXRuntimeAllocator* ptr; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateDefaultAllocator(&ptr)); + default_allocator.reset(ptr); + } + char* p = (char*)ONNXRuntimeAllocatorAlloc(default_allocator.get(), 100); + ASSERT_NE(p, nullptr); + memset(p, 0, 100); + ONNXRuntimeAllocatorFree(default_allocator.get(), p); + const ONNXRuntimeAllocatorInfo* info1 = ONNXRuntimeAllocatorGetInfo(default_allocator.get()); + const ONNXRuntimeAllocatorInfo* info2 = (*default_allocator)->Info(default_allocator.get()); + ASSERT_EQ(0, ONNXRuntimeCompareAllocatorInfo(info1, info2)); +} diff --git a/onnxruntime/test/shared_lib/test_fixture.h b/onnxruntime/test/shared_lib/test_fixture.h new file mode 100644 index 0000000000000..de3e5c90f3096 --- /dev/null +++ b/onnxruntime/test/shared_lib/test_fixture.h @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/onnx_object.h" +#include "core/session/onnxruntime_cxx_api.h" +#include + +//empty +static inline void ONNXRUNTIME_API_STATUSCALL MyLoggingFunction(void*, ONNXRuntimeLoggingLevel, const char*, const char*, const char*, const char*) { +} +template +class CApiTestImpl : public ::testing::Test { + protected: + ONNXEnv* env; + + void SetUp() override { + if (use_customer_logger) { + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeInitializeWithCustomLogger(MyLoggingFunction, nullptr, ONNXRUNTIME_LOGGING_LEVEL_kINFO, "Default", &env)); + } else { + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeInitialize(ONNXRUNTIME_LOGGING_LEVEL_kINFO, "Default", &env)); + } + } + + void TearDown() override { + ReleaseONNXEnv(env); + } + + // Objects declared here can be used by all tests in the test case for Foo. +}; + +typedef CApiTestImpl CApiTest; diff --git a/onnxruntime/test/shared_lib/test_inference.cc b/onnxruntime/test/shared_lib/test_inference.cc new file mode 100644 index 0000000000000..3d778ae305a1e --- /dev/null +++ b/onnxruntime/test/shared_lib/test_inference.cc @@ -0,0 +1,220 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/onnxruntime_cxx_api.h" +#include "providers.h" +#include +#include +#include +#include +#include +#include "test_allocator.h" +#include "test_fixture.h" + +using namespace onnxruntime; + +template +void RunSession(ONNXRuntimeAllocator* env, ONNXSession* session_object, + const std::vector& dims_x, + const std::vector& values_x, + const std::vector& dims_y, + const std::vector& values_y) { + std::unique_ptr value_x(nullptr, ReleaseONNXValue); + std::vector inputs(1); + inputs[0] = ONNXRuntimeCreateTensorAsONNXValue(env, dims_x, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT); + value_x.reset(inputs[0]); + void* raw_data; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeGetTensorMutableData(inputs[0], &raw_data)); + memcpy(raw_data, values_x.data(), values_x.size() * sizeof(values_x[0])); + std::vector input_names{"X"}; + ONNXValuePtr rtensor = nullptr; + std::unique_ptr output(nullptr, ReleaseONNXValueList); + if (with_target_names) { + const char* output_names[] = {"Y"}; + std::vector t(1); + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeRunInference(session_object, input_names.data(), inputs.data(), inputs.size(), output_names, 1, t.data())); + rtensor = t[0]; + } else { + size_t output_len; + { + ONNXValueListPtr t; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeRunInferenceAndFetchAll(session_object, input_names.data(), inputs.data(), inputs.size(), &t, &output_len)); + output.reset(t); + } + + ASSERT_EQ(static_cast(1), output_len); + rtensor = ONNXRuntimeONNXValueListGetNthValue(output.get(), 0); + } + ASSERT_NE(rtensor, nullptr); + std::unique_ptr shape_info; + { + ONNXRuntimeTensorTypeAndShapeInfo* shape_info_ptr; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeGetTensorShapeAndType(rtensor, &shape_info_ptr)); + shape_info.reset(shape_info_ptr); + } + size_t rtensor_dims = ONNXRuntimeGetNumOfDimensions(shape_info.get()); + std::vector shape_array(rtensor_dims); + ONNXRuntimeGetDimensions(shape_info.get(), shape_array.data(), shape_array.size()); + ASSERT_EQ(shape_array, dims_y); + size_t total_len = 1; + for (size_t i = 0; i != rtensor_dims; ++i) { + total_len *= shape_array[i]; + } + ASSERT_EQ(values_y.size(), total_len); + float* f; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeGetTensorMutableData(rtensor, (void**)&f)); + for (size_t i = 0; i != total_len; ++i) { + ASSERT_EQ(values_y[i], f[i]); + } + if (with_target_names) { + ReleaseONNXValue(rtensor); + } +} + +template +void TestInference(ONNXEnv* env, const T& model_uri, + const std::vector& dims_x, + const std::vector& values_x, + const std::vector& expected_dims_y, + const std::vector& expected_values_y, + int provider_type, bool custom_op) { + SessionOptionsWrapper sf(env); + + if (provider_type == 1) { +#ifdef USE_CUDA + ONNXRuntimeProviderFactoryPtr* f; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateCUDAExecutionProviderFactory(0, &f)); + sf.AppendExecutionProvider(f); + ONNXRuntimeReleaseObject(f); + std::cout << "Running simple inference with cuda provider" << std::endl; +#else + return; +#endif + } else if (provider_type == 2) { +#ifdef USE_MKLDNN + ONNXRuntimeProviderFactoryPtr* f; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateMkldnnExecutionProviderFactory(1, &f)); + sf.AppendExecutionProvider(f); + ONNXRuntimeReleaseObject(f); + std::cout << "Running simple inference with mkldnn provider" << std::endl; +#else + return; +#endif + } else if (provider_type == 3) { +#ifdef USE_NUPHAR + ONNXRuntimeProviderFactoryPtr* f; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateNupharExecutionProviderFactory(0, "", &f)); + sf.AppendExecutionProvider(f); + ONNXRuntimeReleaseObject(f); + std::cout << "Running simple inference with nuphar provider" << std::endl; +#else + return; +#endif + } else { + std::cout << "Running simple inference with default provider" << std::endl; + } + if (custom_op) { + sf.AddCustomOp("libonnxruntime_custom_op_shared_lib_test.so"); + } + std::unique_ptr inference_session(sf.ONNXRuntimeCreateInferenceSession(model_uri.c_str()), ReleaseONNXSession); + std::unique_ptr default_allocator(MockedONNXRuntimeAllocator::Create()); + // Now run + RunSession(default_allocator.get(), inference_session.get(), dims_x, values_x, expected_dims_y, expected_values_y); +} + +#ifdef _WIN32 +typedef std::wstring PATH_TYPE; +#define TSTR(X) L##X +#else +#define TSTR(X) (X) +typedef std::string PATH_TYPE; +#endif + +static const PATH_TYPE MODEL_URI = TSTR("testdata/mul_1.pb"); +static const PATH_TYPE CUSTOM_OP_MODEL_URI = TSTR("testdata/foo_1.pb"); + +class CApiTestWithProvider : public CApiTest, + public ::testing::WithParamInterface { +}; + +// Tests that the Foo::Bar() method does Abc. +TEST_P(CApiTestWithProvider, simple) { + const PATH_TYPE input_filepath = TSTR("this/package/testdata/myinputfile.dat"); + const PATH_TYPE output_filepath = TSTR("this/package/testdata/myoutputfile.dat"); + // simple inference test + // prepare inputs + std::vector dims_x = {3, 2}; + std::vector values_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; + + // prepare expected inputs and outputs + std::vector expected_dims_y = {3, 2}; + std::vector expected_values_y = {1.0f, 4.0f, 9.0f, 16.0f, 25.0f, 36.0f}; + + TestInference(env, MODEL_URI, dims_x, values_x, expected_dims_y, expected_values_y, GetParam(), false); + TestInference(env, MODEL_URI, dims_x, values_x, expected_dims_y, expected_values_y, GetParam(), false); +} + +INSTANTIATE_TEST_CASE_P(CApiTestWithProviders, + CApiTestWithProvider, + ::testing::Values(0, 1, 2, 3, 4)); + +#ifndef _WIN32 +//doesn't work, failed in type comparison +TEST_F(CApiTest, DISABLED_custom_op) { + std::cout << "Running custom op inference" << std::endl; + std::vector dims_x = {3, 2}; + std::vector values_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; + + // prepare expected inputs and outputs + std::vector expected_dims_y = {3, 2}; + std::vector expected_values_y = {2.0f, 4.0f, 6.0f, 8.0f, 10.0f, 12.0f}; + + TestInference(env, CUSTOM_OP_MODEL_URI, dims_x, values_x, expected_dims_y, expected_values_y, false, true); + TestInference(env, CUSTOM_OP_MODEL_URI, dims_x, values_x, expected_dims_y, expected_values_y, false, true); +} +#endif + +TEST_F(CApiTest, create_tensor) { + const char* s[] = {"abc", "kmp"}; + size_t expected_len = 2; + std::unique_ptr default_allocator(MockedONNXRuntimeAllocator::Create()); + { + std::unique_ptr tensor( + ONNXRuntimeCreateTensorAsONNXValue(default_allocator.get(), {expected_len}, ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING), ReleaseONNXValue); + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeFillStringTensor(tensor.get(), s, expected_len)); + std::unique_ptr shape_info; + { + ONNXRuntimeTensorTypeAndShapeInfo* shape_info_ptr; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeGetTensorShapeAndType(tensor.get(), &shape_info_ptr)); + shape_info.reset(shape_info_ptr); + } + size_t len = static_cast(ONNXRuntimeGetTensorShapeElementCount(shape_info.get())); + ASSERT_EQ(len, expected_len); + std::vector shape_array(len); + + size_t data_len; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeGetStringTensorDataLength(tensor.get(), &data_len)); + std::string result(data_len, '\0'); + std::vector offsets(len); + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeGetStringTensorContent(tensor.get(), (void*)result.data(), data_len, offsets.data(), offsets.size())); + } +} + +TEST_F(CApiTest, create_tensor_with_data) { + float values[] = {3.0f, 1.0f, 2.f, 0.f}; + constexpr size_t values_length = sizeof(values) / sizeof(values[0]); + ONNXRuntimeAllocatorInfo* info; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateAllocatorInfo("Cpu", ONNXRuntimeDeviceAllocator, 0, ONNXRuntimeMemTypeDefault, &info)); + std::vector dims = {3}; + std::unique_ptr tensor( + ONNXRuntimeCreateTensorWithDataAsONNXValue(info, values, values_length * sizeof(float), dims, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT), ReleaseONNXValue); + ReleaseONNXRuntimeAllocatorInfo(info); + void* new_pointer; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeGetTensorMutableData(tensor.get(), &new_pointer)); + ASSERT_EQ(new_pointer, values); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/onnxruntime/test/shared_lib/test_run_options.cc b/onnxruntime/test/shared_lib/test_run_options.cc new file mode 100644 index 0000000000000..22d626b4072ab --- /dev/null +++ b/onnxruntime/test/shared_lib/test_run_options.cc @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/onnxruntime_cxx_api.h" +#include "core/framework/run_options_c_api.h" +#include "test_fixture.h" +using namespace onnxruntime; + +TEST_F(CApiTest, run_options) { + std::unique_ptr options(ONNXRuntimeCreateRunOptions()); + ASSERT_NE(options, nullptr); + ASSERT_EQ(ONNXRuntimeRunOptionsSetRunLogVerbosityLevel(options.get(), 1), nullptr); + ASSERT_EQ(ONNXRuntimeRunOptionsSetRunTag(options.get(), "abc"), nullptr); + ASSERT_STREQ(ONNXRuntimeRunOptionsGetRunTag(options.get()), "abc"); + ASSERT_EQ(ONNXRuntimeRunOptionsGetRunLogVerbosityLevel(options.get()), (unsigned)1); +} diff --git a/onnxruntime/test/shared_lib/test_session_options.cc b/onnxruntime/test/shared_lib/test_session_options.cc new file mode 100644 index 0000000000000..1cb8c01147812 --- /dev/null +++ b/onnxruntime/test/shared_lib/test_session_options.cc @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/onnxruntime_cxx_api.h" + +#include "test_fixture.h" +using namespace onnxruntime; + +TEST_F(CApiTest, session_options) { + std::unique_ptr options(ONNXRuntimeCreateSessionOptions()); + ASSERT_NE(options, nullptr); +} diff --git a/onnxruntime/test/testdata/CNTK/gen.py b/onnxruntime/test/testdata/CNTK/gen.py new file mode 100644 index 0000000000000..10f257bee6c8b --- /dev/null +++ b/onnxruntime/test/testdata/CNTK/gen.py @@ -0,0 +1,80 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import cntk as C +import numpy as np +import onnx +import os + +model_file = 'model.onnx' +data_dir = 'test_data_set_0' + +def SaveTensorProto(file_path, variable, data): + tp = onnx.TensorProto() + tp.name = variable.uid + for i in range(len(variable.dynamic_axes)): + tp.dims.append(1) # pad 1 for the each dynamic axis + for d in variable.shape: + tp.dims.append(d) + tp.data_type = onnx.TensorProto.FLOAT + tp.raw_data = data.tobytes() + with open(file_path, 'wb') as f: + f.write(tp.SerializeToString()) + +def SaveData(test_data_dir, prefix, variables, data_list): + if isinstance(data_list, np.ndarray): + data_list = [data_list] + for (i, d), v in zip(enumerate(data_list), variables): + SaveTensorProto(os.path.join(test_data_dir, '{0}_{1}.pb'.format(prefix, i)), v, d) + +def Save(dir, func, inputs, outputs): + if not os.path.exists(dir): + os.makedirs(dir) + func.save(os.path.join(dir,model_file), C.ModelFormat.ONNX) + + test_data_dir = os.path.join(dir, data_dir) + if not os.path.exists(test_data_dir): + os.makedirs(test_data_dir) + + SaveData(test_data_dir, 'input', func.arguments, inputs) + SaveData(test_data_dir, 'output', func.outputs, outputs) + +def GenSimple(): + x = C.input_variable((1,3,)) # TODO: fix CNTK exporter bug with shape (3,) + y = C.layers.Embedding(2)(x) + C.parameter((-1,)) + data_x = np.random.rand(*x.shape).astype(np.float32) + data_y = y.eval(data_x) + Save('test_simple', y, data_x, data_y) + +def GenSharedWeights(): + x = C.input_variable((1,3,)) + y = C.layers.Embedding(2)(x) + y = y + y.parameters[0] + data_x = np.random.rand(*x.shape).astype(np.float32) + data_y = y.eval(data_x) + Save('test_shared_weights', y, data_x, data_y) + +def GenSimpleMNIST(): + input_dim = 784 + num_output_classes = 10 + num_hidden_layers = 1 + hidden_layers_dim = 200 + + feature = C.input_variable(input_dim, np.float32) + + scaled_input = C.element_times(C.constant(0.00390625, shape=(input_dim,)), feature) + + z = C.layers.Sequential([C.layers.For(range(num_hidden_layers), lambda i: C.layers.Dense(hidden_layers_dim, activation=C.relu)), + C.layers.Dense(num_output_classes)])(scaled_input) + + model = C.softmax(z) + + data_feature = np.random.rand(*feature.shape).astype(np.float32) + data_output = model.eval(data_feature) + Save('test_simpleMNIST', model, data_feature, data_output) + +if __name__=='__main__': + np.random.seed(0) + GenSimple() + GenSharedWeights() + GenSimpleMNIST() diff --git a/onnxruntime/test/testdata/CNTK/test_LSTM.tanh.bidirectional/model.onnx b/onnxruntime/test/testdata/CNTK/test_LSTM.tanh.bidirectional/model.onnx new file mode 100644 index 0000000000000..a4164a587074d Binary files /dev/null and b/onnxruntime/test/testdata/CNTK/test_LSTM.tanh.bidirectional/model.onnx differ diff --git a/onnxruntime/test/testdata/CNTK/test_LSTM.tanh.bidirectional/test_data_set_0/input_0.pb b/onnxruntime/test/testdata/CNTK/test_LSTM.tanh.bidirectional/test_data_set_0/input_0.pb new file mode 100644 index 0000000000000..3523f13bb0263 Binary files /dev/null and b/onnxruntime/test/testdata/CNTK/test_LSTM.tanh.bidirectional/test_data_set_0/input_0.pb differ diff --git a/onnxruntime/test/testdata/CNTK/test_LSTM.tanh.bidirectional/test_data_set_0/output_0.pb b/onnxruntime/test/testdata/CNTK/test_LSTM.tanh.bidirectional/test_data_set_0/output_0.pb new file mode 100644 index 0000000000000..02fd903dc8183 --- /dev/null +++ b/onnxruntime/test/testdata/CNTK/test_LSTM.tanh.bidirectional/test_data_set_0/output_0.pb @@ -0,0 +1,2 @@ +BSplice1349_Output_0Jxč3=fĎ=%Gľs!Ű=ńŚl˝ß˝Ô˝×=A‰={Ů˝±h>őte˝)Ä*ľRÓč<'ŇĎ=b݉˝O[­=˝çŃ?}?ńHp? \ No newline at end of file diff --git a/onnxruntime/test/testdata/CNTK/test_RNN.bidirectional.one_layer.relu/test_data_set_0/output_0.pb b/onnxruntime/test/testdata/CNTK/test_RNN.bidirectional.one_layer.relu/test_data_set_0/output_0.pb new file mode 100644 index 0000000000000..9fcefe82a2ad1 Binary files /dev/null and b/onnxruntime/test/testdata/CNTK/test_RNN.bidirectional.one_layer.relu/test_data_set_0/output_0.pb differ diff --git a/onnxruntime/test/testdata/LabelEncoder.pb b/onnxruntime/test/testdata/LabelEncoder.pb new file mode 100644 index 0000000000000..8e153d0a4c037 Binary files /dev/null and b/onnxruntime/test/testdata/LabelEncoder.pb differ diff --git a/onnxruntime/test/testdata/conv_autopad.pb b/onnxruntime/test/testdata/conv_autopad.pb new file mode 100644 index 0000000000000..80a8172578b1d Binary files /dev/null and b/onnxruntime/test/testdata/conv_autopad.pb differ diff --git a/onnxruntime/test/testdata/foo_1.pb b/onnxruntime/test/testdata/foo_1.pb new file mode 100644 index 0000000000000..19a76981a4ba6 Binary files /dev/null and b/onnxruntime/test/testdata/foo_1.pb differ diff --git a/onnxruntime/test/testdata/foo_2.pb b/onnxruntime/test/testdata/foo_2.pb new file mode 100644 index 0000000000000..1cd18a0684acb Binary files /dev/null and b/onnxruntime/test/testdata/foo_2.pb differ diff --git a/onnxruntime/test/testdata/fuse_add_1.pb b/onnxruntime/test/testdata/fuse_add_1.pb new file mode 100644 index 0000000000000..801147fa4e277 Binary files /dev/null and b/onnxruntime/test/testdata/fuse_add_1.pb differ diff --git a/onnxruntime/test/testdata/gru_1.pb b/onnxruntime/test/testdata/gru_1.pb new file mode 100644 index 0000000000000..1f2c684b306b6 Binary files /dev/null and b/onnxruntime/test/testdata/gru_1.pb differ diff --git a/onnxruntime/test/testdata/identity_string.pb b/onnxruntime/test/testdata/identity_string.pb new file mode 100644 index 0000000000000..c33e4f8cc36fc --- /dev/null +++ b/onnxruntime/test/testdata/identity_string.pb @@ -0,0 +1,11 @@ +tf2onnx0.0.2.0:m +% +input:0output:0output"Identitytf2onnxRtestZ +input:0 +  + +b +output:0 +  + +B \ No newline at end of file diff --git a/onnxruntime/test/testdata/logicaland.pb b/onnxruntime/test/testdata/logicaland.pb new file mode 100644 index 0000000000000..cedb131b848ab --- /dev/null +++ b/onnxruntime/test/testdata/logicaland.pb @@ -0,0 +1,16 @@ +tf2onnx0.0.2.0:Ž +* +input:0 +input1:0output:0output"Andtf2onnxRtestZ +input1:0 +   + +Z +input:0 +   + +b +output:0 +   + +B \ No newline at end of file diff --git a/onnxruntime/test/testdata/lstm_1.onnx b/onnxruntime/test/testdata/lstm_1.onnx new file mode 100644 index 0000000000000..238712affe30c Binary files /dev/null and b/onnxruntime/test/testdata/lstm_1.onnx differ diff --git a/onnxruntime/test/testdata/matmul_1.pb b/onnxruntime/test/testdata/matmul_1.pb new file mode 100644 index 0000000000000..37c4998b01f99 Binary files /dev/null and b/onnxruntime/test/testdata/matmul_1.pb differ diff --git a/onnxruntime/test/testdata/matmul_2.pb b/onnxruntime/test/testdata/matmul_2.pb new file mode 100644 index 0000000000000..44b331ad86bc3 Binary files /dev/null and b/onnxruntime/test/testdata/matmul_2.pb differ diff --git a/onnxruntime/test/testdata/mlnet_encoder.onnx b/onnxruntime/test/testdata/mlnet_encoder.onnx new file mode 100644 index 0000000000000..7b4d916f07715 Binary files /dev/null and b/onnxruntime/test/testdata/mlnet_encoder.onnx differ diff --git a/onnxruntime/test/testdata/model_optional_inputs.pb b/onnxruntime/test/testdata/model_optional_inputs.pb new file mode 100644 index 0000000000000..638c87c14856b Binary files /dev/null and b/onnxruntime/test/testdata/model_optional_inputs.pb differ diff --git a/onnxruntime/test/testdata/mul_1.pb b/onnxruntime/test/testdata/mul_1.pb new file mode 100644 index 0000000000000..60606b699062e Binary files /dev/null and b/onnxruntime/test/testdata/mul_1.pb differ diff --git a/onnxruntime/test/testdata/mul_1.pb.noopset b/onnxruntime/test/testdata/mul_1.pb.noopset new file mode 100644 index 0000000000000..50daa54dc63e2 Binary files /dev/null and b/onnxruntime/test/testdata/mul_1.pb.noopset differ diff --git a/onnxruntime/test/testdata/mul_16.pb b/onnxruntime/test/testdata/mul_16.pb new file mode 100644 index 0000000000000..99a97cb1b11af Binary files /dev/null and b/onnxruntime/test/testdata/mul_16.pb differ diff --git a/onnxruntime/test/testdata/optional_1.pb b/onnxruntime/test/testdata/optional_1.pb new file mode 100644 index 0000000000000..19bf60e92618c Binary files /dev/null and b/onnxruntime/test/testdata/optional_1.pb differ diff --git a/onnxruntime/test/testdata/pipeline_vectorize.onnx b/onnxruntime/test/testdata/pipeline_vectorize.onnx new file mode 100644 index 0000000000000..e675f1fdc4f56 Binary files /dev/null and b/onnxruntime/test/testdata/pipeline_vectorize.onnx differ diff --git a/onnxruntime/test/testdata/squeezenet/model.onnx b/onnxruntime/test/testdata/squeezenet/model.onnx new file mode 100644 index 0000000000000..b8e1dfce26d99 Binary files /dev/null and b/onnxruntime/test/testdata/squeezenet/model.onnx differ diff --git a/onnxruntime/test/testdata/squeezenet/test_data_set_0/test_data_0_input.pb b/onnxruntime/test/testdata/squeezenet/test_data_set_0/test_data_0_input.pb new file mode 100644 index 0000000000000..f521c230e5e76 Binary files /dev/null and b/onnxruntime/test/testdata/squeezenet/test_data_set_0/test_data_0_input.pb differ diff --git a/onnxruntime/test/testdata/squeezenet/test_data_set_0/test_data_0_output.pb b/onnxruntime/test/testdata/squeezenet/test_data_set_0/test_data_0_output.pb new file mode 100644 index 0000000000000..e731eb0234cc7 Binary files /dev/null and b/onnxruntime/test/testdata/squeezenet/test_data_set_0/test_data_0_output.pb differ diff --git a/onnxruntime/test/testdata/transform/abs-2id-max.onnx b/onnxruntime/test/testdata/transform/abs-2id-max.onnx new file mode 100644 index 0000000000000..43d701735bbcb --- /dev/null +++ b/onnxruntime/test/testdata/transform/abs-2id-max.onnx @@ -0,0 +1,34 @@ +lotus-transfomrs:Č + +AB"Abs + +BC"Identity + +CD"Identity + +DE"Max abs-2id-maxZ +A + + + +b +E + + + +j +B + + + +j +C + + + +j +D + + + +B \ No newline at end of file diff --git a/onnxruntime/test/testdata/transform/abs-id-max.onnx b/onnxruntime/test/testdata/transform/abs-id-max.onnx new file mode 100644 index 0000000000000..d8999d6e0ffa1 --- /dev/null +++ b/onnxruntime/test/testdata/transform/abs-id-max.onnx @@ -0,0 +1,28 @@ +lotus-transfomrs:ś + +AB"Abs + +BC"Identity + +CD"Max +abs-id-maxZ +A + + + +b +D + + + +j +B + + + +j +C + + + +B \ No newline at end of file diff --git a/onnxruntime/test/testdata/transform/abs-id.onnx b/onnxruntime/test/testdata/transform/abs-id.onnx new file mode 100644 index 0000000000000..e9a2790ae42a3 --- /dev/null +++ b/onnxruntime/test/testdata/transform/abs-id.onnx @@ -0,0 +1,20 @@ +lotus-transfomrs:r + +XY"Abs + +YZ"Identityabs-idZ +X + + + +b +Z + + + +j +Y + + + +B \ No newline at end of file diff --git a/onnxruntime/test/testdata/transform/fusion/fuse-conv-bn-mul-add-unsqueeze.onnx b/onnxruntime/test/testdata/transform/fusion/fuse-conv-bn-mul-add-unsqueeze.onnx new file mode 100644 index 0000000000000..ac86e07847307 Binary files /dev/null and b/onnxruntime/test/testdata/transform/fusion/fuse-conv-bn-mul-add-unsqueeze.onnx differ diff --git a/onnxruntime/test/testdata/transform/fusion/test_data_set_0/input_0.pb b/onnxruntime/test/testdata/transform/fusion/test_data_set_0/input_0.pb new file mode 100644 index 0000000000000..67da09ab7c1c2 Binary files /dev/null and b/onnxruntime/test/testdata/transform/fusion/test_data_set_0/input_0.pb differ diff --git a/onnxruntime/test/testdata/transform/fusion/test_data_set_0/output_0.pb b/onnxruntime/test/testdata/transform/fusion/test_data_set_0/output_0.pb new file mode 100644 index 0000000000000..c900118107b88 Binary files /dev/null and b/onnxruntime/test/testdata/transform/fusion/test_data_set_0/output_0.pb differ diff --git a/onnxruntime/test/testdata/zipmap_int64float.pb b/onnxruntime/test/testdata/zipmap_int64float.pb new file mode 100644 index 0000000000000..9f9e53cff0bbc Binary files /dev/null and b/onnxruntime/test/testdata/zipmap_int64float.pb differ diff --git a/onnxruntime/test/testdata/zipmap_stringfloat.pb b/onnxruntime/test/testdata/zipmap_stringfloat.pb new file mode 100644 index 0000000000000..1345fabced88c Binary files /dev/null and b/onnxruntime/test/testdata/zipmap_stringfloat.pb differ diff --git a/onnxruntime/test/tvm/tvm_basic_test.cc b/onnxruntime/test/tvm/tvm_basic_test.cc new file mode 100644 index 0000000000000..5d6e96643d650 --- /dev/null +++ b/onnxruntime/test/tvm/tvm_basic_test.cc @@ -0,0 +1,281 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include +#include "core/codegen/tvm/tvm_kernel.h" +#include "core/framework/execution_provider.h" +#include "core/framework/computation_capacity.h" +#include "core/graph/graph.h" +#include "core/providers/cpu/cpu_execution_provider.h" +#include "core/session/inference_session.h" +#include "core/common/logging/logging.h" +#include "test/framework/test_utils.h" +#include "test/test_environment.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { + +tvm::Schedule DefaultTVMScheduleGenerator(const TVMGraph& tvm_graph) { + std::vector args; + for (auto& tensor : tvm_graph.outputs_) + args.push_back(tensor.tvm_tensor_->op); + return tvm::create_schedule(args); +} + +tvm::runtime::Module BuildStackVMDefaultModule(tvm::Schedule schedule, tvm::BuildConfig config, tvm::Array tvm_args, std::vector& target_func_names) { + auto target = tvm::target::stackvm(); + std::string func_name = "func"; + auto args = tvm::Array(tvm_args); + std::unordered_map binds; + auto lowered = lower(schedule, args, "func", binds, config); + target_func_names.push_back(func_name); + return build(lowered, target, tvm::Target(), config); +} + +template +class TVMFuseAddKernels : public TVMKernel { + public: + explicit TVMFuseAddKernels(const OpKernelInfo& info) : TVMKernel(info) {} + + protected: + virtual const TensorShape& GetOutputShape(OpKernelContext* context, int /*i*/) const override { + return context->Input(0)->Shape(); + } +}; + +class UnionSet { + public: + UnionSet(int n) { + for (int i = 0; i < n; ++i) { + farthers_.push_back(i); + } + } + + int get(int x) { + if (farthers_[x] == x) { + return x; + } + return farthers_[x] = get(farthers_[x]); + } + + void merge(int x, int y) { + x = get(x); + y = get(y); + if (x != y) { + farthers_[y] = x; + } + } + + std::vector farthers_; +}; + +void FuseAdd(const onnxruntime::GraphViewer& graph, std::vector>& capacities) { + std::vector add_nodes; + for (auto& node : graph.Nodes()) { + if (node.OpType() == "Add") { + add_nodes.push_back(node.Index()); + } + } + + UnionSet set(static_cast(add_nodes.size())); + for (int i = 0; i < add_nodes.size(); ++i) { + auto node = graph.GetNode(add_nodes[i]); + for (auto it = node->InputNodesBegin(); it != node->InputNodesEnd(); ++it) { + auto index_it = std::find(add_nodes.begin(), add_nodes.end(), (*it)->Index()); + if (index_it != add_nodes.end()) { + set.merge(i, static_cast(index_it - add_nodes.begin())); + } + } + } + + std::vector> groups; + groups.resize(add_nodes.size()); + for (int i = 0; i < set.farthers_.size(); ++i) { + groups[set.get(i)].push_back(add_nodes[i]); + } + + for (auto& group : groups) { + if (group.size() > 1) { + std::unique_ptr sub_graph = std::make_unique(); + std::set fused_inputs, fused_outputs; + for (auto index : group) { + sub_graph->nodes.push_back(index); + auto node = graph.GetNode(index); + for (auto input : node->InputDefs()) { + auto it = fused_outputs.find(input); + if (it != fused_outputs.end()) { + fused_outputs.erase(it); + } else { + fused_inputs.insert(input); + } + } + for (auto output : node->OutputDefs()) { + auto it = fused_inputs.find(output); + if (it != fused_inputs.end()) { + fused_inputs.erase(it); + } else { + fused_outputs.insert(output); + } + } + } + + auto meta_def = std::make_unique<::onnxruntime::IndexedSubGraph::MetaDef>(); + meta_def->name = "TVMFuseAdd"; + meta_def->domain = "FuseTest"; + for (auto input : fused_inputs) { + meta_def->inputs.push_back(input->Name()); + } + + for (auto output : fused_outputs) { + meta_def->outputs.push_back(output->Name()); + } + + meta_def->since_version = 1; + meta_def->status = ONNX_NAMESPACE::EXPERIMENTAL; + sub_graph->SetMetaDef(meta_def); + //TODO:set fuse kernel func; + capacities.push_back( + std::make_unique(std::move(sub_graph), + [](const OpKernelInfo& info) -> OpKernel* { return new TVMFuseAddKernels(info); })); + } + } +} + +namespace test { + +static void RunSession(InferenceSession& session_object, + RunOptions& run_options, + std::vector& dims_x, + std::vector& values_x, + std::vector& dims_y, + std::vector& values_y) { + // prepare inputs + MLValue ml_value; + CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, ONNXRuntimeMemTypeDefault), dims_x, values_x, &ml_value); + NameMLValMap feeds; + feeds.insert(std::make_pair("X1", ml_value)); + + // prepare outputs + std::vector output_names; + output_names.push_back("Y4"); + std::vector fetches; + + // Now run + common::Status st = session_object.Run(run_options, feeds, output_names, &fetches); + std::cout << "Run returned status: " << st.ErrorMessage() << std::endl; + EXPECT_TRUE(st.IsOK()); + ASSERT_EQ(1, fetches.size()); + auto& rtensor = fetches.front().Get(); + TensorShape expected_shape(dims_y); + EXPECT_EQ(expected_shape, rtensor.Shape()); + const std::vector found(rtensor.template Data(), rtensor.template Data() + expected_shape.Size()); + ASSERT_EQ(found.size(), values_y.size()); + for (size_t i = 0; i < found.size(); i++) + ASSERT_EQ(found[i], values_y[i]); +} + +static const std::string MODEL_URI = "testdata/fuse_add_1.pb"; + +TEST(TVMTest, Fuse_Add_Test) { + SessionOptions so; + + so.session_logid = "InferenceSessionTests.NoTimeout"; + + InferenceSession session_object{so, &DefaultLoggingManager()}; + CPUExecutionProviderInfo info; + + auto cpu_xp = std::make_unique(info); + cpu_xp->InsertFusedRules(FuseAdd); + EXPECT_TRUE(session_object.RegisterExecutionProvider(std::move(cpu_xp)).IsOK()); + EXPECT_TRUE(session_object.Load(MODEL_URI).IsOK()); + EXPECT_TRUE(session_object.Initialize().IsOK()); + + RunOptions run_options; + run_options.run_tag = "one session/one tag"; + + // prepare inputs + std::vector dims_x = { + 6, + }; + std::vector values_x = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}; + + // prepare expected inputs and outputs + std::vector expected_dims_y = { + 6, + }; + // now the expected value should be Add's result. + std::vector expected_values_y = {5.0, 10.0, 15.0, 20.0, 25.0, 30.0}; + + // Now run + RunSession(session_object, run_options, dims_x, values_x, expected_dims_y, expected_values_y); +} +} // namespace test + +} // namespace onnxruntime + +TEST(TVMTest, Basic) { + using namespace tvm; + auto n = var("n"); + Array shape; + shape.push_back(n); + auto A = placeholder(shape, Float(64), "A"); + auto B = placeholder(shape, Float(64), "B"); + auto D = placeholder(shape, Float(64), "D"); + auto C = compute(A->shape, [&A, &B](Expr i) { + return A[i] + B[i]; + }, + "C"); + auto E = compute(A->shape, [&C, &D](Expr i) { + return C[i] + D[i]; + }, + "E"); + + auto s = create_schedule({E->op}); + auto args = Array({A, B, D, E}); + std::unordered_map binds; + auto config = build_config(); +#ifdef USE_TVM_WITH_LLVM + auto target = target::llvm(); +#else + auto target = target::stackvm(); +#endif + auto lowered = lower(s, args, "func", binds, config); + auto module = build(lowered, target, Target(), config); + auto func = module.GetFunction("func"); + + DLDataType dtype; + dtype.code = kDLFloat; + dtype.bits = 64; + dtype.lanes = 1; + DLContext ctx; + ctx.device_type = DLDeviceType::kDLCPU; + ctx.device_id = 0; + + std::vector v = {1.0, 2.0, 3.0}; + int64_t len = 3; + DLTensor tensor_A = {&v[0], ctx, 1, dtype, &len, nullptr, 0}; + DLTensor tensor_B = {&v[0], ctx, 1, dtype, &len, nullptr, 0}; + DLTensor tensor_D = {&v[0], ctx, 1, dtype, &len, nullptr, 0}; + + std::vector r; + r.resize(len); + DLTensor tensor_E = {&r[0], ctx, 1, dtype, &len, nullptr, 0}; + + TVMValue lvalues[4]; + int type_codes[4] = {kNDArrayContainer, kNDArrayContainer, kNDArrayContainer, kNDArrayContainer}; + lvalues[0].v_handle = &tensor_A; + lvalues[1].v_handle = &tensor_B; + lvalues[2].v_handle = &tensor_D; + lvalues[3].v_handle = &tensor_E; + + TVMArgs tvm_args(lvalues, type_codes, 4); + TVMRetValue rvalue; + func.CallPacked(tvm_args, &rvalue); + CHECK_EQ(rvalue.type_code(), kNull); + double expected[3] = {3.0, 6.0, 9.0}; + auto data_E = static_cast(tensor_E.data); + for (int i = 0; i < 3; i++) { + EXPECT_NEAR(*(data_E + i), expected[i], 0.001f); + } +} diff --git a/onnxruntime/test/util/compare_mlvalue.cc b/onnxruntime/test/util/compare_mlvalue.cc new file mode 100644 index 0000000000000..9ae4420c4649a --- /dev/null +++ b/onnxruntime/test/util/compare_mlvalue.cc @@ -0,0 +1,345 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "test/compare_mlvalue.h" +#include +#include +#include +#include "core/graph/onnx_protobuf.h" + +#include "Eigen/Core" +#include "Eigen/src/Core/arch/CUDA/Half.h" + +using namespace onnxruntime; + +namespace { +MLDataType ElementTypeFromProto(ONNX_NAMESPACE::TensorProto_DataType type) { + switch (type) { + case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: + return DataTypeImpl::GetType(); + case ONNX_NAMESPACE::TensorProto_DataType_BOOL: + return DataTypeImpl::GetType(); + case ONNX_NAMESPACE::TensorProto_DataType_INT32: + return DataTypeImpl::GetType(); + case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: + return DataTypeImpl::GetType(); + case ONNX_NAMESPACE::TensorProto_DataType_STRING: + return DataTypeImpl::GetType(); + case ONNX_NAMESPACE::TensorProto_DataType_INT8: + return DataTypeImpl::GetType(); + case ONNX_NAMESPACE::TensorProto_DataType_UINT8: + return DataTypeImpl::GetType(); + case ONNX_NAMESPACE::TensorProto_DataType_UINT16: + return DataTypeImpl::GetType(); + case ONNX_NAMESPACE::TensorProto_DataType_INT16: + return DataTypeImpl::GetType(); + case ONNX_NAMESPACE::TensorProto_DataType_INT64: + return DataTypeImpl::GetType(); + case ONNX_NAMESPACE::TensorProto_DataType_UINT32: + return DataTypeImpl::GetType(); + case ONNX_NAMESPACE::TensorProto_DataType_UINT64: + return DataTypeImpl::GetType(); + case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16: + return DataTypeImpl::GetType(); + default: + ONNXRUNTIME_NOT_IMPLEMENTED(__FUNCTION__, ":tensor type ", type, " is not supported"); + } +} + +template +std::pair CompareFloatResult(const Tensor& outvalue, const Tensor& expected_value, + double per_sample_tolerance, + double relative_per_sample_tolerance, + bool post_processing) { + const size_t size1 = expected_value.Shape().Size(); + const FLOAT_TYPE* expected_output = expected_value.template Data(); + const FLOAT_TYPE* real_output = outvalue.template Data(); + std::pair res = std::make_pair(COMPARE_RESULT::SUCCESS, ""); + double max_diff = 0; + size_t diff_count = 0; + for (size_t di = 0; di != size1; ++di) { + const double real_value = post_processing ? std::max(0.0, std::min(255.0, real_output[di])) + : real_output[di]; + const double diff = fabs(expected_output[di] - real_value); + const double rtol = per_sample_tolerance + relative_per_sample_tolerance * fabs(expected_output[di]); + if (diff > rtol || (std::isnan(diff) && !std::isnan(expected_output[di]))) { + res.first = COMPARE_RESULT::RESULT_DIFFERS; + // update error message if this is a larger diff + if (diff > max_diff || (std::isnan(diff) && !std::isnan(max_diff))) { + std::ostringstream oss; + oss << "expected " << expected_output[di] << ", got " << real_value + << ", diff: " << diff << ", tol=" << rtol << "."; + res.second = oss.str(); + max_diff = diff; + } + ++diff_count; + } + } + + if (res.first == COMPARE_RESULT::SUCCESS) return res; + + std::ostringstream oss; + oss << res.second << " " << diff_count << " of " << size1 << " differ"; + res.second = oss.str(); + return res; +} + +template +std::pair IsResultExactlyMatch(const Tensor& outvalue, const Tensor& expected_value) { + const size_t size1 = expected_value.Shape().Size(); + const T* expected_output = expected_value.template Data(); + const T* real_output = outvalue.template Data(); + for (size_t di = 0; di != size1; ++di) { + if (expected_output[di] != real_output[di]) { + std::ostringstream oss; + oss << "expected " << expected_output[di] << ", got " << real_output[di]; + return std::make_pair(COMPARE_RESULT::RESULT_DIFFERS, oss.str()); + } + } + return std::make_pair(COMPARE_RESULT::SUCCESS, ""); +} + +std::pair CompareFloat16Result(const Tensor& outvalue, const Tensor& expected_value, + double per_sample_tolerance, + double relative_per_sample_tolerance, + bool post_processing) { + const size_t size1 = expected_value.Shape().Size(); + const MLFloat16* expected_output = expected_value.template Data(); + const MLFloat16* real_output = outvalue.template Data(); + for (size_t di = 0; di != size1; ++di) { + float expected = Eigen::half_impl::half_to_float(Eigen::half_impl::__half(expected_output[di].val)); + float real = Eigen::half_impl::half_to_float(Eigen::half_impl::__half(real_output[di].val)); + real = post_processing ? std::max(0.0f, std::min(255.0f, real)) : real; + const double diff = fabs(expected - real); + const double rtol = per_sample_tolerance + relative_per_sample_tolerance * fabs(expected); + if (diff > rtol || (std::isnan(diff) && !std::isnan(expected))) { + std::ostringstream oss; + oss << "expected " << expected << ", got " << real << ", diff: " << diff << ", tol=" << rtol; + + return std::make_pair(COMPARE_RESULT::RESULT_DIFFERS, oss.str()); + } + } + return std::make_pair(COMPARE_RESULT::SUCCESS, ""); +} + +std::pair CompareTwoTensors(const Tensor& outvalue, const Tensor& expected_tensor, + double per_sample_tolerance, + double relative_per_sample_tolerance, + bool post_processing) { + if (expected_tensor.Shape() != outvalue.Shape()) { + std::ostringstream oss; + oss << "shape mismatch, expect " << expected_tensor.Shape().ToString() << " got " << outvalue.Shape().ToString(); + return std::make_pair(COMPARE_RESULT::SHAPE_MISMATCH, oss.str()); + } + auto p1 = outvalue.DataType(); + if (p1 == DataTypeImpl::GetType()) { + return CompareFloatResult(outvalue, expected_tensor, + per_sample_tolerance, relative_per_sample_tolerance, post_processing); + } else if (p1 == DataTypeImpl::GetType()) { + return CompareFloatResult(outvalue, expected_tensor, + per_sample_tolerance, relative_per_sample_tolerance, post_processing); + } else if (p1 == DataTypeImpl::GetType()) { + return IsResultExactlyMatch(outvalue, expected_tensor); + } else if (p1 == DataTypeImpl::GetType()) { + return IsResultExactlyMatch(outvalue, expected_tensor); + } else if (p1 == DataTypeImpl::GetType()) { + return IsResultExactlyMatch(outvalue, expected_tensor); + } else if (p1 == DataTypeImpl::GetType()) { + return IsResultExactlyMatch(outvalue, expected_tensor); + } else if (p1 == DataTypeImpl::GetType()) { + return IsResultExactlyMatch(outvalue, expected_tensor); + } else if (p1 == DataTypeImpl::GetType()) { + return IsResultExactlyMatch(outvalue, expected_tensor); + } else if (p1 == DataTypeImpl::GetType()) { + return IsResultExactlyMatch(outvalue, expected_tensor); + } else if (p1 == DataTypeImpl::GetType()) { + return IsResultExactlyMatch(outvalue, expected_tensor); + } else if (p1 == DataTypeImpl::GetType()) { + return IsResultExactlyMatch(outvalue, expected_tensor); + } else if (p1 == DataTypeImpl::GetType()) { + return IsResultExactlyMatch(outvalue, expected_tensor); + } else if (p1 == DataTypeImpl::GetType()) { + return CompareFloat16Result(outvalue, expected_tensor, + per_sample_tolerance, relative_per_sample_tolerance, post_processing); + } else { + return std::make_pair(COMPARE_RESULT::NOT_SUPPORT, ""); + } +} +template +std::pair CompareSeqOfMapToFloat(const T& real_output_vector, const T& expected_value, + double per_sample_tolerance, + double relative_per_sample_tolerance, + bool post_processing) { + if (real_output_vector.size() != expected_value.size()) { + std::ostringstream oss; + oss << "vector size mismatch, expected " << expected_value.size() << ", got " << real_output_vector.size(); + return std::make_pair(COMPARE_RESULT::RESULT_DIFFERS, oss.str()); + } + for (size_t i = 0; i != real_output_vector.size(); ++i) { + const auto& expected_map = expected_value[i]; + //compare if expected_map equals real_output_vector[i] + if (real_output_vector[i].size() != expected_map.size()) { + std::ostringstream oss; + oss << "map size mismatch, expected " << expected_map.size() << ", got " << real_output_vector[i].size(); + return std::make_pair(COMPARE_RESULT::RESULT_DIFFERS, oss.str()); + } + + for (const auto& real_output_key_value_pair : real_output_vector[i]) { + auto expected_key_value_pair = expected_map.find(real_output_key_value_pair.first); + if (expected_key_value_pair == expected_map.end()) { + return std::make_pair(COMPARE_RESULT::RESULT_DIFFERS, ""); + } + const double real = post_processing + ? std::max(0.0, std::min(255.0, real_output_key_value_pair.second)) + : real_output_key_value_pair.second; + const double diff = fabs(expected_key_value_pair->second - real); + const double rtol = per_sample_tolerance + relative_per_sample_tolerance * fabs(expected_key_value_pair->second); + if (diff > rtol || (std::isnan(diff) && !std::isnan(expected_key_value_pair->second))) { + std::ostringstream oss; + oss << "expected " << expected_key_value_pair->second << ", got " << real + << ", diff: " << diff << ", tol=" << rtol; + return std::make_pair(COMPARE_RESULT::RESULT_DIFFERS, oss.str()); + } + } + } + return std::make_pair(COMPARE_RESULT::SUCCESS, ""); +} + +const char* ElementTypeToString(MLDataType type) { + if (type == DataTypeImpl::GetType()) { + return "tensor(float)"; + } else if (type == DataTypeImpl::GetType()) { + return "tensor(bool)"; + } + + else if (type == DataTypeImpl::GetType()) { + return "tensor(int32)"; + } + + else if (type == DataTypeImpl::GetType()) { + return "tensor(double)"; + } + + else if (type == DataTypeImpl::GetType()) { + return "tensor(string)"; + } + + else if (type == DataTypeImpl::GetType()) { + return "tensor(uint8)"; + } + + else if (type == DataTypeImpl::GetType()) { + return "tensor(uint16)"; + } + + else if (type == DataTypeImpl::GetType()) { + return "tensor(int16)"; + } + + else if (type == DataTypeImpl::GetType()) { + return "tensor(int64)"; + } + + else if (type == DataTypeImpl::GetType()) { + return "tensor(uint32)"; + } + + else if (type == DataTypeImpl::GetType()) { + return "tensor(uint64)"; + } + + else if (type == DataTypeImpl::GetType()) { + return "tensor(MLFloat16)"; + } else { + return "unknown"; + } +} + +//The expected_shape could contain unknown dimensions, but the real_shape cannot +bool AreShapesEqual(const TensorShape& real_shape, const ::ONNX_NAMESPACE::TensorShapeProto& expected_shape) { + const int len = expected_shape.dim_size(); + //because real_shape.NumDimensions() cannot be negative + if (len < 0) return false; + if (real_shape.NumDimensions() != static_cast(len)) return false; + for (int i = 0; i != len; ++i) { + if (!expected_shape.dim(i).has_dim_value()) { + //symbolic shape, cannot validate it right now + continue; + } + ::google::protobuf::int64 d = expected_shape.dim(i).dim_value(); + //dim value can be zero or negative, in such case, we assume it can match any value + if (d != real_shape[i]) return false; + } + return true; +} + +} // namespace + +namespace onnxruntime { +std::pair CompareMLValue(const MLValue& o, const MLValue& expected_mlvalue, + double per_sample_tolerance, + double relative_per_sample_tolerance, + bool post_processing) { + if (o.IsTensor() != expected_mlvalue.IsTensor() || o.Type() != expected_mlvalue.Type()) { + return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, ""); + } + if (!o.IsTensor()) { + if (o.Type() == DataTypeImpl::GetType()) { + return CompareSeqOfMapToFloat(o.Get(), expected_mlvalue.Get(), + per_sample_tolerance, relative_per_sample_tolerance, post_processing); + } + if (o.Type() == DataTypeImpl::GetType()) { + return CompareSeqOfMapToFloat(o.Get(), expected_mlvalue.Get(), + per_sample_tolerance, relative_per_sample_tolerance, post_processing); + } + return std::make_pair(COMPARE_RESULT::NOT_SUPPORT, ""); + } + const Tensor& outvalue = o.Get(); + const Tensor& expected_tensor = expected_mlvalue.Get(); + if (outvalue.DataType() != expected_tensor.DataType()) { + std::ostringstream oss; + oss << "expect " << ElementTypeToString(expected_tensor.DataType()) + << " got " << ElementTypeToString(outvalue.DataType()); + return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, oss.str()); + } + return CompareTwoTensors(outvalue, expected_tensor, + per_sample_tolerance, relative_per_sample_tolerance, post_processing); +} + +std::pair VerifyValueInfo(const ONNX_NAMESPACE::ValueInfoProto& v, const MLValue& o) { + if (v.has_type()) { + if (v.type().has_tensor_type()) { + if (o.Type() != DataTypeImpl::GetType()) { + return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, ""); + } + ::ONNX_NAMESPACE::TypeProto_Tensor t = v.type().tensor_type(); + //below code doesn't work + //if (((TensorTypeBase*)o.Type())->GetElementType() != DataTypeImpl::ElementTypeFromProto(t.elem_type())) { + // return COMPARE_RESULT::TYPE_MISMATCH; + //} + const Tensor& o1 = o.Get(); + if (o1.DataType() != ElementTypeFromProto(t.elem_type())) { + return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, ""); + } + if (!AreShapesEqual(o1.Shape(), t.shape())) { + std::string result; + if (!google::protobuf::TextFormat::PrintToString(t.shape(), &result)) { + result = "(unknown)"; + } + std::ostringstream oss; + oss << "Tensor shape mismatch, model file expects '" << result + << "', real output is " << o1.Shape().ToString(); + return std::make_pair(COMPARE_RESULT::SHAPE_MISMATCH, oss.str()); + } + } else { + //Cannot do this check for tensor type. + //For tensor type, o.Type() is TensorTypeBase*, but p points to a subclass of TensorTypeBase + auto p = DataTypeImpl::TypeFromProto(v.type()); + if (o.Type() != p) { + return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, ""); + } + } + } + return std::make_pair(COMPARE_RESULT::SUCCESS, ""); +} +} // namespace onnxruntime diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc new file mode 100644 index 0000000000000..8502a28772a24 --- /dev/null +++ b/onnxruntime/test/util/default_providers.cc @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "default_providers.h" +#include "providers.h" +#include "core/session/onnxruntime_cxx_api.h" +#define FACTORY_PTR_HOLDER \ + std::unique_ptr ptr_holder_(f); + +namespace onnxruntime { +namespace test { +std::unique_ptr DefaultCpuExecutionProvider(bool enable_arena) { + ONNXRuntimeProviderFactoryPtr* f; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateCpuExecutionProviderFactory(enable_arena ? 1 : 0, &f)); + FACTORY_PTR_HOLDER; + ONNXRuntimeProviderPtr out; + ONNXRUNTIME_THROW_ON_ERROR((*f)->CreateProvider(f, &out)); + return std::unique_ptr((IExecutionProvider*)out); +} + +std::unique_ptr DefaultCudaExecutionProvider() { +#ifdef USE_CUDA + ONNXRuntimeProviderFactoryPtr* f; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateCUDAExecutionProviderFactory(0, &f)); + FACTORY_PTR_HOLDER; + ONNXRuntimeProviderPtr out; + ONNXRUNTIME_THROW_ON_ERROR((*f)->CreateProvider(f, &out)); + return std::unique_ptr((IExecutionProvider*)out); +#else + return nullptr; +#endif +} + +std::unique_ptr DefaultMkldnnExecutionProvider(bool enable_arena) { +#ifdef USE_MKLDNN + ONNXRuntimeProviderFactoryPtr* f; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateMkldnnExecutionProviderFactory(enable_arena ? 1 : 0, &f)); + FACTORY_PTR_HOLDER; + ONNXRuntimeProviderPtr out; + ONNXRUNTIME_THROW_ON_ERROR((*f)->CreateProvider(f, &out)); + return std::unique_ptr((IExecutionProvider*)out); +#else + ONNXRUNTIME_UNUSED_PARAMETER(enable_arena); + return nullptr; +#endif +} + +std::unique_ptr DefaultNupharExecutionProvider() { +#ifdef USE_NUPHAR + ONNXRuntimeProviderFactoryPtr* f; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateNupharExecutionProviderFactory(0, "", &f)); + FACTORY_PTR_HOLDER; + ONNXRuntimeProviderPtr out; + ONNXRUNTIME_THROW_ON_ERROR((*f)->CreateProvider(f, &out)); + return std::unique_ptr((IExecutionProvider*)out); +#else + return nullptr; +#endif +} + +std::unique_ptr DefaultBrainSliceExecutionProvider() { +#ifdef USE_BRAINSLICE + ONNXRuntimeProviderFactoryPtr* f; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateBrainSliceExecutionProviderFactory(0, true, "testdata/firmwares/onnx_rnns/instructions.bin", "testdata/firmwares/onnx_rnns/data.bin", "testdata/firmwares/onnx_rnns/schema.bin", & f)); + FACTORY_PTR_HOLDER; + ONNXRuntimeProviderPtr out; + ONNXRUNTIME_THROW_ON_ERROR((*f)->CreateProvider(f, &out)); + return std::unique_ptr((IExecutionProvider*)out); +#else + return nullptr; +#endif +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/util/include/default_providers.h b/onnxruntime/test/util/include/default_providers.h new file mode 100644 index 0000000000000..d9bc16c369429 --- /dev/null +++ b/onnxruntime/test/util/include/default_providers.h @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/execution_provider.h" + +namespace onnxruntime { +namespace test { + +// unique_ptr providers with default values for session registration +std::unique_ptr DefaultCpuExecutionProvider(bool enable_arena = true); +std::unique_ptr DefaultCudaExecutionProvider(); +std::unique_ptr DefaultMkldnnExecutionProvider(bool enable_arena = true); +std::unique_ptr DefaultNupharExecutionProvider(); +std::unique_ptr DefaultBrainSliceExecutionProvider(); + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/util/include/providers.h b/onnxruntime/test/util/include/providers.h new file mode 100644 index 0000000000000..c421fa8f197b7 --- /dev/null +++ b/onnxruntime/test/util/include/providers.h @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/providers/cpu/cpu_provider_factory.h" + +#ifdef USE_CUDA +#include "core/providers/cuda/cuda_provider_factory.h" +#endif +#ifdef USE_MKLDNN +#include "core/providers/mkldnn/mkldnn_provider_factory.h" +#endif +#ifdef USE_NUPHAR +#include "core/providers/nuphar/nuphar_provider_factory.h" +#endif +#if USE_BRAINSLICE +#include "core/providers/brainslice/brainslice_provider_factory.h" +#endif \ No newline at end of file diff --git a/onnxruntime/test/util/include/test/capturing_sink.h b/onnxruntime/test/util/include/test/capturing_sink.h new file mode 100644 index 0000000000000..39788947602df --- /dev/null +++ b/onnxruntime/test/util/include/test/capturing_sink.h @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/logging/logging.h" +#include "core/common/logging/isink.h" + +#include "date/date.h" + +namespace onnxruntime { +namespace test { + +using namespace ::onnxruntime::logging; + +class CapturingSink : public logging::ISink { + public: + void SendImpl(const Timestamp& timestamp, const std::string& logger_id, const Capture& message) override { + // operator for formatting of timestamp in ISO8601 format including microseconds + using date::operator<<; + std::ostringstream msg; + + msg << timestamp << " [" << message.SeverityPrefix() << ":" << message.Category() << ":" << logger_id << ", " + << message.Location().ToString() << "] " << message.Message(); + + messages_.push_back(msg.str()); + } + + const std::vector& Messages() const { + return messages_; + } + + private: + std::vector messages_; +}; +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/util/include/test/compare_mlvalue.h b/onnxruntime/test/util/include/test/compare_mlvalue.h new file mode 100644 index 0000000000000..9f2b533f2ac47 --- /dev/null +++ b/onnxruntime/test/util/include/test/compare_mlvalue.h @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +//TODO(): move compare_mlvalue.{h,cc} to test dir + +#include +#include + +namespace ONNX_NAMESPACE { +class ValueInfoProto; +} +namespace onnxruntime { +enum class COMPARE_RESULT { + SUCCESS, + RESULT_DIFFERS, + TYPE_MISMATCH, + SHAPE_MISMATCH, + NOT_SUPPORT +}; +std::pair CompareMLValue(const MLValue& real, const MLValue& expected, double per_sample_tolerance, + double relative_per_sample_tolerance, bool post_processing); + +//verify if the 'value' matches the 'expected' ValueInfoProto. 'value' is a model output +std::pair VerifyValueInfo(const ONNX_NAMESPACE::ValueInfoProto& expected, const MLValue& value); +} // namespace onnxruntime diff --git a/onnxruntime/test/util/include/test/test_environment.h b/onnxruntime/test/util/include/test/test_environment.h new file mode 100644 index 0000000000000..79dd5413c3232 --- /dev/null +++ b/onnxruntime/test/util/include/test/test_environment.h @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/logging/logging.h" + +#ifdef HAVE_FRAMEWORK_LIB +#include "core/framework/environment.h" +#endif + +namespace onnxruntime { +namespace test { + +/** +Static logging manager with a CLog based sink so logging macros that use the default logger will work +Instance is created and owned by TestEnvironment. The sharing via this static is for convenience. +*/ +::onnxruntime::logging::LoggingManager& DefaultLoggingManager(); + +/** +Perform default initialization of a unit test executable. +This includes setting up google test, the default logger, and the framework runtime environment. +Keep the instance of this class until tests have completed. +*/ +class TestEnvironment { + public: + TestEnvironment(int argc, char** argv, bool create_default_logging_manager = true); + ~TestEnvironment(); + + private: + ONNXRUNTIME_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(TestEnvironment); + +#ifdef HAVE_FRAMEWORK_LIB + std::unique_ptr runtime_environment_; +#endif +}; + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/util/include/test_allocator.h b/onnxruntime/test/util/include/test_allocator.h new file mode 100644 index 0000000000000..5b6835a87bb24 --- /dev/null +++ b/onnxruntime/test/util/include/test_allocator.h @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/allocator.h" +#include +#include +#include "core/framework/allocator_info.h" +#include "core/session/onnxruntime_cxx_api.h" +#include + +#define ONNXRUNTIME_ALLOCATOR_IMPL_BEGIN(CLASS_NAME) \ + class CLASS_NAME { \ + private: \ + const ONNXRuntimeAllocatorInteface* vtable_ = &table_; \ + std::atomic_int ref_count_; \ + static void* ONNXRUNTIME_API_STATUSCALL Alloc_(void* this_ptr, size_t size) { \ + return ((CLASS_NAME*)this_ptr)->Alloc(size); \ + } \ + static void ONNXRUNTIME_API_STATUSCALL Free_(void* this_ptr, void* p) { \ + return ((CLASS_NAME*)this_ptr)->Free(p); \ + } \ + static const ONNXRuntimeAllocatorInfo* ONNXRUNTIME_API_STATUSCALL Info_(const void* this_ptr) { \ + return ((const CLASS_NAME*)this_ptr)->Info(); \ + } \ + static uint32_t ONNXRUNTIME_API_STATUSCALL AddRef_(void* this_) { \ + CLASS_NAME* this_ptr = (CLASS_NAME*)this_; \ + return ++this_ptr->ref_count_; \ + } \ + static uint32_t ONNXRUNTIME_API_STATUSCALL Release_(void* this_) { \ + CLASS_NAME* this_ptr = (CLASS_NAME*)this_; \ + uint32_t ret = --this_ptr->ref_count_; \ + if (ret == 0) \ + delete this_ptr; \ + return 0; \ + } \ + static ONNXRuntimeAllocatorInteface table_; + +#define ONNXRUNTIME_ALLOCATOR_IMPL_END \ + } \ + ; + +ONNXRUNTIME_ALLOCATOR_IMPL_BEGIN(MockedONNXRuntimeAllocator) +private: +std::atomic memory_inuse; +ONNXRuntimeAllocatorInfo* cpuAllocatorInfo; +MockedONNXRuntimeAllocator() : ref_count_(1), memory_inuse(0) { + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateAllocatorInfo("Cpu", ONNXRuntimeDeviceAllocator, 0, ONNXRuntimeMemTypeDefault, &cpuAllocatorInfo)); +} +~MockedONNXRuntimeAllocator() { + assert(ref_count_ == 0); + ReleaseONNXRuntimeAllocatorInfo(cpuAllocatorInfo); +} + +public: + MockedONNXRuntimeAllocator(const MockedONNXRuntimeAllocator&) = delete; + MockedONNXRuntimeAllocator& operator=(const MockedONNXRuntimeAllocator&) = delete; + ONNXRuntimeAllocatorInteface** Upcast() { + return const_cast(&vtable_); +} +static ONNXRuntimeAllocatorInteface** Create() { + return (ONNXRuntimeAllocatorInteface**)new MockedONNXRuntimeAllocator(); +} +void* Alloc(size_t size) { + constexpr size_t extra_len = sizeof(size_t); + memory_inuse.fetch_add(size += extra_len); + void* p = ::malloc(size); + *(size_t*)p = size; + return (char*)p + extra_len; +} +void Free(void* p) { + constexpr size_t extra_len = sizeof(size_t); + if (!p) return; + p = (char*)p - extra_len; + size_t len = *(size_t*)p; + memory_inuse.fetch_sub(len); + return ::free(p); +} +const ONNXRuntimeAllocatorInfo* Info() const { + return cpuAllocatorInfo; +} + +void LeakCheck() { + if (memory_inuse.load()) + throw std::runtime_error("memory leak!!!"); +} +ONNXRUNTIME_ALLOCATOR_IMPL_END + diff --git a/onnxruntime/test/util/test_allocator.cc b/onnxruntime/test/util/test_allocator.cc new file mode 100644 index 0000000000000..026a0ba725f5a --- /dev/null +++ b/onnxruntime/test/util/test_allocator.cc @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "test_allocator.h" + +ONNXRuntimeAllocatorInteface MockedONNXRuntimeAllocator::table_ = { + {MockedONNXRuntimeAllocator::AddRef_, MockedONNXRuntimeAllocator::Release_}, MockedONNXRuntimeAllocator::Alloc_, MockedONNXRuntimeAllocator::Free_, MockedONNXRuntimeAllocator::Info_}; diff --git a/onnxruntime/test/util/test_environment.cc b/onnxruntime/test/util/test_environment.cc new file mode 100644 index 0000000000000..440e2c6bff981 --- /dev/null +++ b/onnxruntime/test/util/test_environment.cc @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "test/test_environment.h" + +#include +#include + +#include "gtest/gtest.h" +#include "google/protobuf/stubs/common.h" + +#include "core/common/logging/logging.h" +#include "core/common/logging/sinks/clog_sink.h" + +using namespace ::onnxruntime::logging; + +namespace onnxruntime { +namespace test { + +static std::unique_ptr<::onnxruntime::logging::LoggingManager> s_default_logging_manager; + +::onnxruntime::logging::LoggingManager& DefaultLoggingManager() { + ONNXRUNTIME_ENFORCE(s_default_logging_manager != nullptr, + "Need a TestEnvironment instance to provide the default logging manager."); + + return *s_default_logging_manager; +} + +TestEnvironment::TestEnvironment(int argc, char** argv, bool create_default_logging_manager) { + ONNXRUNTIME_ENFORCE(s_default_logging_manager == nullptr, + "Only expected one instance of TestEnvironment to be created."); + + std::clog << "Initializing unit testing." << std::endl; + testing::InitGoogleTest(&argc, argv); + + if (create_default_logging_manager) { + static std::string default_logger_id{"Default"}; + s_default_logging_manager = std::make_unique(std::unique_ptr{new CLogSink{}}, + Severity::kWARNING, // TODO make this configurable through + // cmd line arguments or some other way + false, + LoggingManager::InstanceType::Default, + &default_logger_id); + + // make sure default logging manager exists and is working + auto logger = ::onnxruntime::test::DefaultLoggingManager().DefaultLogger(); + LOGS(logger, VERBOSE) << "Logging manager initialized."; + } + +#ifdef HAVE_FRAMEWORK_LIB + auto status = Environment::Create(runtime_environment_); + ONNXRUNTIME_ENFORCE(status == Status::OK(), "Failed creating runtime environment. ", status.ErrorMessage()); +#endif +} + +TestEnvironment::~TestEnvironment() { +#ifdef HAVE_FRAMEWORK_LIB + // release environment followed by logging manager so any log output from runtime environment shutdown + // using the default logger will succeed. + runtime_environment_ = nullptr; +#else + ::google::protobuf::ShutdownProtobufLibrary(); +#endif + + // dispose logging manager manually to make sure it's destructed before the default logging mutex + s_default_logging_manager = nullptr; +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/win_getopt/mb/getopt.cc b/onnxruntime/test/win_getopt/mb/getopt.cc new file mode 100644 index 0000000000000..ddd31ce9117ca --- /dev/null +++ b/onnxruntime/test/win_getopt/mb/getopt.cc @@ -0,0 +1,117 @@ +/* $NetBSD: getopt.c,v 1.29 2014/06/05 22:00:22 christos Exp $ */ + +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1987, 1993, 1994 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include + +int opterr = 1, /* if error message should be printed */ + optind = 1, /* index into parent argv vector */ + optopt, /* character checked for validity */ + optreset; /* reset getopt */ +char* optarg; /* argument associated with option */ + +#define BADCH (int)'?' +#define BADARG (int)':' +#define EMSG "" + +/* + * getopt -- + * Parse argc/argv argument vector. + */ +int getopt(int nargc, char* const nargv[], const char* ostr) { + static char* place = EMSG; /* option letter processing */ + char* oli; /* option letter list index */ + + if (optreset || *place == 0) { /* update scanning pointer */ + optreset = 0; + place = nargv[optind]; + if (optind >= nargc || *place++ != '-') { + /* Argument is absent or is not an option */ + place = EMSG; + return (-1); + } + optopt = *place++; + if (optopt == '-' && *place == 0) { + /* "--" => end of options */ + ++optind; + place = EMSG; + return (-1); + } + if (optopt == 0) { + /* Solitary '-', treat as a '-' option + if the program (eg su) is looking for it. */ + place = EMSG; + if (strchr(ostr, '-') == NULL) return (-1); + optopt = '-'; + } + } else + optopt = *place++; + + /* See if option letter is one the caller wanted... */ + if (optopt == ':' || (oli = (char*)strchr(ostr, optopt)) == NULL) { + if (*place == 0) ++optind; + if (opterr && *ostr != ':') (void)fprintf(stderr, "illegal option -- %c\n", optopt); + return (BADCH); + } + + /* Does this option need an argument? */ + if (oli[1] != ':') { + /* don't need argument */ + optarg = NULL; + if (*place == 0) ++optind; + } else { + /* Option-argument is either the rest of this argument or the + entire next argument. */ + if (*place) + optarg = place; + else if (oli[2] == ':') + /* + * GNU Extension, for optional arguments if the rest of + * the argument is empty, we return NULL + */ + optarg = NULL; + else if (nargc > ++optind) + optarg = nargv[optind]; + else { + /* option-argument absent */ + place = EMSG; + if (*ostr == ':') return (BADARG); + if (opterr) (void)fprintf(stderr, "option requires an argument -- %c\n", optopt); + return (BADCH); + } + place = EMSG; + ++optind; + } + return (optopt); /* return option letter */ +} \ No newline at end of file diff --git a/onnxruntime/test/win_getopt/mb/include/getopt.h b/onnxruntime/test/win_getopt/mb/include/getopt.h new file mode 100644 index 0000000000000..6477ee96e97e3 --- /dev/null +++ b/onnxruntime/test/win_getopt/mb/include/getopt.h @@ -0,0 +1,42 @@ +/* $NetBSD: getopt.c,v 1.29 2014/06/05 22:00:22 christos Exp $ */ + +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1987, 1993, 1994 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#pragma once + +extern int opterr, /* if error message should be printed */ + optind, /* index into parent argv vector */ + optopt, /* character checked for validity */ + optreset; /* reset getopt */ +extern char* optarg; /* argument associated with option */ + +int getopt(int nargc, char* const nargv[], const char* ostr); diff --git a/onnxruntime/test/win_getopt/wide/getopt.cc b/onnxruntime/test/win_getopt/wide/getopt.cc new file mode 100644 index 0000000000000..18fdb0d42fb69 --- /dev/null +++ b/onnxruntime/test/win_getopt/wide/getopt.cc @@ -0,0 +1,117 @@ +/* $NetBSD: getopt.c,v 1.29 2014/06/05 22:00:22 christos Exp $ */ + +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1987, 1993, 1994 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include + +int opterr = 1, /* if error message should be printed */ + optind = 1, /* index into parent argv vector */ + optopt, /* character checked for validity */ + optreset; /* reset getopt */ +wchar_t* optarg; /* argument associated with option */ + +#define BADCH (int)L'?' +#define BADARG (int)L':' +#define EMSG L"" + +/* + * getopt -- + * Parse argc/argv argument vector. + */ +int getopt(int nargc, wchar_t* const nargv[], const wchar_t* ostr) { + static wchar_t* place = EMSG; /* option letter processing */ + wchar_t* oli; /* option letter list index */ + + if (optreset || *place == 0) { /* update scanning pointer */ + optreset = 0; + place = nargv[optind]; + if (optind >= nargc || *place++ != L'-') { + /* Argument is absent or is not an option */ + place = EMSG; + return (-1); + } + optopt = *place++; + if (optopt == L'-' && *place == 0) { + /* "--" => end of options */ + ++optind; + place = EMSG; + return (-1); + } + if (optopt == 0) { + /* Solitary '-', treat as a '-' option + if the program (eg su) is looking for it. */ + place = EMSG; + if (wcschr(ostr, L'-') == NULL) return (-1); + optopt = '-'; + } + } else + optopt = *place++; + + /* See if option letter is one the caller wanted... */ + if (optopt == L':' || (oli = (wchar_t*)wcschr(ostr, (wchar_t)optopt)) == NULL) { + if (*place == 0) ++optind; + if (opterr && *ostr != L':') (void)fprintf(stderr, "illegal option -- %c\n", optopt); + return (BADCH); + } + + /* Does this option need an argument? */ + if (oli[1] != L':') { + /* don't need argument */ + optarg = NULL; + if (*place == 0) ++optind; + } else { + /* Option-argument is either the rest of this argument or the + entire next argument. */ + if (*place) + optarg = place; + else if (oli[2] == L':') + /* + * GNU Extension, for optional arguments if the rest of + * the argument is empty, we return NULL + */ + optarg = NULL; + else if (nargc > ++optind) + optarg = nargv[optind]; + else { + /* option-argument absent */ + place = EMSG; + if (*ostr == L':') return (BADARG); + if (opterr) (void)fprintf(stderr, "option requires an argument -- %c\n", optopt); + return (BADCH); + } + place = EMSG; + ++optind; + } + return (optopt); /* return option letter */ +} \ No newline at end of file diff --git a/onnxruntime/test/win_getopt/wide/include/getopt.h b/onnxruntime/test/win_getopt/wide/include/getopt.h new file mode 100644 index 0000000000000..a68af3ba84bc8 --- /dev/null +++ b/onnxruntime/test/win_getopt/wide/include/getopt.h @@ -0,0 +1,42 @@ +/* $NetBSD: getopt.c,v 1.29 2014/06/05 22:00:22 christos Exp $ */ + +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1987, 1993, 1994 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#pragma once + +extern int opterr, /* if error message should be printed */ + optind, /* index into parent argv vector */ + optopt, /* character checked for validity */ + optreset; /* reset getopt */ +extern wchar_t* optarg; /* argument associated with option */ + +int getopt(int nargc, wchar_t* const nargv[], const wchar_t* ostr); diff --git a/package/rpm/onnxruntime.spec b/package/rpm/onnxruntime.spec new file mode 100644 index 0000000000000..32aebb7d9b0bc --- /dev/null +++ b/package/rpm/onnxruntime.spec @@ -0,0 +1,63 @@ +Name: onnxruntime +Version: 0.1.4 +Release: 1%{?dist} +Summary: onnxruntime + +License: MIT +URL: https://onnx.ai +Source0: onnxruntime.tar + +BuildRequires: gcc +BuildRequires: gcc-c++ +BuildRequires: zlib-devel +BuildRequires: make +BuildRequires: git +BuildRequires: python3-devel +BuildRequires: python3-numpy +BuildRequires: python3-setuptools +BuildRequires: python3-wheel +BuildRequires: bzip2 +Requires: libstdc++ +Requires: glibc + +%description + + +%prep +%autosetup + + +%build +mkdir debug +cd debug +/opt/cmake/bin/cmake -Donnxruntime_DEV_MODE=OFF -DCMAKE_DEBUG_POSTFIX=_debug -DCMAKE_BUILD_TYPE=Debug -Deigen_SOURCE_PATH=/usr/include/eigen3 -Donnxruntime_USE_PREINSTALLED_EIGEN=ON -Donnxruntime_BUILD_SHARED_LIB=ON -DCMAKE_INSTALL_PREFIX=%{_prefix} ../cmake +%make_build +cd .. +mkdir release +cd release +/opt/cmake/bin/cmake -Donnxruntime_DEV_MODE=OFF -DCMAKE_BUILD_TYPE=RelWithDebInfo -Deigen_SOURCE_PATH=/usr/include/eigen3 -Donnxruntime_USE_PREINSTALLED_EIGEN=ON -Donnxruntime_BUILD_SHARED_LIB=ON -DCMAKE_INSTALL_PREFIX=%{_prefix} ../cmake +%make_build +cd .. + +%install +rm -rf $RPM_BUILD_ROOT +cd debug +%make_install +cd .. +cd release +%make_install +cd .. + +%files +%license LICENSE +%doc docs/* +%doc ThirdPartyNotices.txt +%{_bindir}/onnx_test_runner +%{_libdir}/libonnxruntime.so* +%{_libdir}/libonnxruntime_debug.so* +%{_includedir}/onnxruntime/* + + +%changelog +* Wed Oct 17 2018 ONNXRuntime Team +- Initial release diff --git a/rename_manylinux.sh b/rename_manylinux.sh new file mode 100755 index 0000000000000..707cc1c9c9cb1 --- /dev/null +++ b/rename_manylinux.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# hack script to modify modify whl as manylinux whl +whl=(*whl) +renamed_whl=`echo $whl | sed --expression='s/linux/manylinux1/g'` +basename=`echo $whl | awk -F'-cp3' '{print $1}'` +unzip $whl +sed -i 's/linux/manylinux1/g' ${basename}.dist-info/WHEEL +# explicitly set file perms +chmod 664 ${basename}.dist-info/* +zip -r $renamed_whl ${basename}.data ${basename}.dist-info diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000000000..67095e2aa7809 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,13 @@ +codecov +jinja2 +numpy +onnx +onnxmltools +pandas +protobuf +pytest +pytest-cov +scikit-learn +scipy +six +wheel diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000..1cf243d912f28 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +six +numpy +protobuf diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000000..ea32fa84bf5e0 --- /dev/null +++ b/setup.py @@ -0,0 +1,88 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +#-------------------------------------------------------------------------- + +from setuptools import setup, find_packages +from os import path, getcwd +import platform +import sys + +package_name = 'onnxruntime' +if '--use_cuda' in sys.argv: + package_name = 'onnxruntime-gpu' + sys.argv.remove('--use_cuda') + +try: + from wheel.bdist_wheel import bdist_wheel as _bdist_wheel + class bdist_wheel(_bdist_wheel): + def finalize_options(self): + _bdist_wheel.finalize_options(self) + self.root_is_pure = False +except ImportError: + bdist_wheel = None + +# Additional binaries +if platform.system() == 'Linux': + libs = ['onnxruntime_pybind11_state.so', 'libmkldnn.so.0', 'libmklml_intel.so', 'libiomp5.so'] +else: + libs = ['onnxruntime_pybind11_state.pyd', 'mkldnn.dll', 'mklml.dll', 'libiomp5md.dll'] + +data = [path.join('capi', x) for x in libs if path.isfile(path.join('onnxruntime', 'capi', x))] + +# Additional examples +examples_names = ["mul_1.pb", "logreg_iris.onnx", "sigmoid.onnx"] +examples = [path.join('datasets', x) for x in examples_names] + +# Extra files such as EULA and ThirdPartyNotices +extra = ["LICENSE", "ThirdPartyNotices.txt"] + +# Description +README = path.join(getcwd(), "docs/python/README.rst") +if not path.exists(README): + this = path.dirname(__file__) + README = path.join(this, "docs/python/README.rst") +if not path.exists(README): + raise FileNotFoundError("Unable to find 'README.rst'") +with open(README) as f: + long_description = f.read() + + +# Setup +setup( + name=package_name, + version='0.1.4', + description='ONNX Runtime Runtime Python bindings', + long_description=long_description, + author='Microsoft Corporation', + author_email='onnx@microsoft.com', + cmdclass={'bdist_wheel': bdist_wheel}, + license="MIT License", + packages=['onnxruntime', + 'onnxruntime.backend', + 'onnxruntime.capi', + 'onnxruntime.datasets', + 'onnxruntime.tools', + ], + package_data={ + 'onnxruntime': data + examples + extra, + }, + extras_require={ + 'backend': ['onnx>=1.2.3'], + }, + entry_points= { + 'console_scripts': [ + 'onnxruntime_test = onnxruntime.tools.onnxruntime_test:main', + ] + }, + classifiers=[ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7'], + ) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py new file mode 100644 index 0000000000000..f8c8a3a603d89 --- /dev/null +++ b/tools/ci_build/build.py @@ -0,0 +1,493 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import argparse +import fileinput +import getpass +import glob +import logging +import multiprocessing +import os +import platform +import re +import shutil +import subprocess +import sys +import warnings + +logging.basicConfig(format="%(asctime)s %(name)s [%(levelname)s] - %(message)s", level=logging.DEBUG) +log = logging.getLogger("Build") + +def parse_arguments(): + parser = argparse.ArgumentParser(description="ONNXRuntime CI build driver.", + usage=''' +Default behavior is --update --build --test. + +The Update phase will update git submodules, and run cmake to generate makefiles. +The Build phase will build all projects. +The Test phase will run all unit tests, and optionally the ONNX tests. + +Use the individual flags to only run the specified stages. + ''') + # Main arguments + parser.add_argument("--build_dir", required=True, help="Path to the build directory.") + parser.add_argument("--config", nargs="+", default=["Debug"], + choices=["Debug", "MinSizeRel", "Release", "RelWithDebInfo"], + help="Configuration(s) to build.") + parser.add_argument("--update", action='store_true', help="Update makefiles.") + parser.add_argument("--build", action='store_true', help="Build.") + parser.add_argument("--clean", action='store_true', help="Run 'cmake --build --target clean' for the selected config/s.") + parser.add_argument("--parallel", action='store_true', help='''Use parallel build. + The build setup doesn't get all dependencies right, so --parallel only works if you're just rebuilding ONNXRuntime code. + If you've done an update that fetched external dependencies you have to build without --parallel the first time. + Once that's done, run with "--build --parallel --test" to just build in parallel and run tests.''') + parser.add_argument("--test", action='store_true', help="Run unit tests.") + + # enable ONNX tests + parser.add_argument("--enable_onnx_tests", action='store_true', + help='''When running the Test phase, run onnx_test_running against available test data directories.''') + parser.add_argument("--pb_home", help="Path to protobuf installation") + # CUDA related + parser.add_argument("--use_cuda", action='store_true', help="Enable CUDA.") + parser.add_argument("--cuda_home", help="Path to CUDA home." + "Read from CUDA_HOME environment variable if --use_cuda is true and --cuda_home is not specified.") + parser.add_argument("--cudnn_home", help="Path to CUDNN home. " + "Read from CUDNN_HOME environment variable if --use_cuda is true and --cudnn_home is not specified.") + + # Python bindings + parser.add_argument("--enable_pybind", action='store_true', help="Enable Python Bindings.") + parser.add_argument("--build_wheel", action='store_true', help="Build Python Wheel. ") + + # C-Sharp bindings + parser.add_argument("--build_csharp", action='store_true', help="Build C#.Net DLL and NuGet package") + + + # Build a shared lib + parser.add_argument("--build_shared_lib", action='store_true', help="Build a shared library for the ONNXRuntime.") + + # Build options + parser.add_argument("--cmake_extra_defines", nargs="+", + help="Extra definitions to pass to CMake during build system generation. " + + "These are just CMake -D options without the leading -D.") + parser.add_argument("--x86", action='store_true', + help="Create x86 makefiles. Requires --update and no existing cache CMake setup. Delete CMakeCache.txt if needed") + + # Arguments needed by CI + parser.add_argument("--cmake_path", default="cmake", help="Path to the CMake program.") + parser.add_argument("--ctest_path", default="ctest", help="Path to the CTest program.") + parser.add_argument("--skip_submodule_sync", action='store_true', help="Don't do a 'git submodule update'. Makes the Update phase faster.") + + parser.add_argument("--use_jemalloc", action='store_true', help="use jemalloc.") + parser.add_argument("--use_openblas", action='store_true', help="Build with OpenBLAS.") + parser.add_argument("--use_mkldnn", action='store_true', help="Build with MKLDNN.") + parser.add_argument("--use_mklml", action='store_true', help="Build with MKLML.") + parser.add_argument("--use_preinstalled_eigen", action='store_true', help="Use pre-installed eigen.") + parser.add_argument("--eigen_path", help="Path to pre-installed eigen.") + parser.add_argument("--use_tvm", action="store_true", help="Build with tvm") + parser.add_argument("--use_openmp", action='store_true', help="Build with OpenMP.") + parser.add_argument("--use_llvm", action="store_true", help="Build tvm with llvm") + parser.add_argument("--enable_msinternal", action="store_true", help="Enable for Microsoft internal builds only.") + parser.add_argument("--llvm_path", help="Path to llvm dir") + parser.add_argument("--use_brainslice", action="store_true", help="Build with brain slice") + parser.add_argument("--brain_slice_package_path", help="Path to brain slice pacakges") + parser.add_argument("--brain_slice_package_name", help="Name of brain slice pakcages") + parser.add_argument("--brain_slice_client_package_name", help="Name of brainslice client package") + parser.add_argument("--use_nuphar", action='store_true', help="Build with nuphar") + return parser.parse_args() + +def is_windows(): + return sys.platform.startswith("win") + +def is_ubuntu_1604(): + return platform.linux_distribution()[0] == 'Ubuntu' and platform.linux_distribution()[1] == '16.04' + +def get_config_build_dir(build_dir, config): + # build directory per configuration + return os.path.join(build_dir, config) + +def run_subprocess(args, cwd=None, capture=False, dll_path=None, check=True): + log.debug("Running subprocess in '{0}'\n{1}".format(cwd or os.getcwd(), args)) + my_env = os.environ.copy() + if dll_path: + if is_windows(): + my_env["PATH"] += os.pathsep + dll_path + else: + if "LD_LIBRARY_PATH" in my_env: + my_env["LD_LIBRARY_PATH"] += os.pathsep + dll_path + else: + my_env["LD_LIBRARY_PATH"] = dll_path + + if (capture): + result = subprocess.run(args, cwd=cwd, check=check, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=my_env) + else: + result = subprocess.run(args, cwd=cwd, check=check, env=my_env) + + return result + +def update_submodules(source_dir): + run_subprocess(["git", "submodule", "update", "--init", "--recursive"], cwd=source_dir) + +def is_docker(): + path = '/proc/self/cgroup' + return ( + os.path.exists('/.dockerenv') or + os.path.isfile(path) and any('docker' in line for line in open(path)) + ) + +def is_sudo(): + return 'SUDO_UID' in os.environ.keys() + +def install_apt_package(package): + have = package in str(run_subprocess(["apt", "list", "--installed", package], capture=True).stdout) + if not have: + if is_sudo(): + run_subprocess(['apt-get', 'install', '-y', package]) + else: + log.error(package + " APT package missing. Please re-run this script using sudo to install.") + sys.exit(-1) + +def install_ubuntu_deps(args): + 'Check if the necessary Ubuntu dependencies are installed. Not required on docker. Provider help output if missing.' + + # check we need the packages first + if not (args.enable_pybind or args.use_openblas): + return + + # not needed on docker as packages are pre-installed + if not is_docker(): + try: + if args.enable_pybind: + install_apt_package("python3") + + if args.use_openblas: + install_apt_package("libopenblas-dev") + + except Exception as e: + log.error("Error setting up required APT packages. {}".format(str(e))) + sys.exit(-1) + +def install_python_deps(): + dep_packages = ['setuptools', 'wheel', 'numpy'] + run_subprocess([sys.executable, '-m', 'pip', 'install', '--trusted-host', 'files.pythonhosted.org'] + dep_packages) + +def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, pb_home, configs, cmake_extra_defines, args, cmake_extra_args): + log.info("Generating CMake build tree") + cmake_dir = os.path.join(source_dir, "cmake") + # TODO: fix jemalloc build so it does not conflict with onnxruntime shared lib builds. (e.g. onnxuntime_pybind) + # for now, disable jemalloc if pybind is also enabled. + cmake_args = [cmake_path, cmake_dir, + "-Donnxruntime_RUN_ONNX_TESTS=" + ("ON" if args.enable_onnx_tests else "OFF"), + "-Donnxruntime_GENERATE_TEST_REPORTS=ON", + "-Donnxruntime_DEV_MODE=ON", + "-DPYTHON_EXECUTABLE=" + sys.executable, + "-Donnxruntime_USE_CUDA=" + ("ON" if args.use_cuda else "OFF"), + "-Donnxruntime_CUDNN_HOME=" + (cudnn_home if args.use_cuda else ""), + "-Donnxruntime_USE_JEMALLOC=" + ("ON" if args.use_jemalloc else "OFF"), + "-Donnxruntime_ENABLE_PYTHON=" + ("ON" if args.enable_pybind else "OFF"), + "-Donnxruntime_BUILD_CSHARP=" + ("ON" if args.build_csharp else "OFF"), + "-Donnxruntime_BUILD_SHARED_LIB=" + ("ON" if args.build_shared_lib else "OFF"), + "-Donnxruntime_USE_EIGEN_FOR_BLAS=" + ("OFF" if args.use_openblas else "ON"), + "-Donnxruntime_USE_OPENBLAS=" + ("ON" if args.use_openblas else "OFF"), + "-Donnxruntime_USE_MKLDNN=" + ("ON" if args.use_mkldnn else "OFF"), + "-Donnxruntime_USE_MKLML=" + ("ON" if args.use_mklml else "OFF"), + "-Donnxruntime_USE_OPENMP=" + ("ON" if args.use_openmp else "OFF"), + "-Donnxruntime_USE_TVM=" + ("ON" if args.use_tvm else "OFF"), + "-Donnxruntime_USE_LLVM=" + ("ON" if args.use_llvm else "OFF"), + "-Donnxruntime_ENABLE_MICROSOFT_INTERNAL=" + ("ON" if args.enable_msinternal else "OFF"), + "-Donnxruntime_USE_BRAINSLICE=" + ("ON" if args.use_brainslice else "OFF"), + "-Donnxruntime_USE_NUPHAR=" + ("ON" if args.use_nuphar else "OFF"), + ] + if args.use_brainslice: + cmake_args += ["-Donnxruntime_BRAINSLICE_LIB_PATH=%s/%s" % (args.brain_slice_package_path, args.brain_slice_package_name), + "-Donnxruntime_BS_CLIENT_PACKAGE=%s/%s" % (args.brain_slice_package_path, args.brain_slice_client_package_name)] + + if args.use_llvm: + cmake_args += ["-DLLVM_DIR=%s" % args.llvm_path] + + if args.use_cuda and not is_windows(): + nvml_stub_path = cuda_home + "/lib64/stubs" + cmake_args += ["-DCUDA_CUDA_LIBRARY=" + nvml_stub_path] + + if args.use_preinstalled_eigen: + cmake_args += ["-Donnxruntime_USE_PREINSTALLED_EIGEN=ON", + "-Deigen_SOURCE_PATH=" + args.eigen_path] + + if pb_home: + cmake_args += ["-DONNX_CUSTOM_PROTOC_EXECUTABLE=" + os.path.join(pb_home,'bin','protoc'), '-Donnxruntime_USE_PREBUILT_PB=ON'] + + cmake_args += ["-D{}".format(define) for define in cmake_extra_defines] + + if is_windows(): + cmake_args += cmake_extra_args + if args.use_cuda: + os.environ["PATH"] += os.pathsep + os.path.join(cudnn_home, 'bin') + + for config in configs: + config_build_dir = get_config_build_dir(build_dir, config) + os.makedirs(config_build_dir, exist_ok=True) + + if args.use_tvm: + os.environ["PATH"] += os.pathsep + os.path.join(config_build_dir, "external", "tvm", config) + + run_subprocess(cmake_args + ["-DCMAKE_BUILD_TYPE={}".format(config)], cwd=config_build_dir) + #create a shortcut for test models if there is a 'models' folder in build_dir + if is_windows(): + dest_model_dir = os.path.join(config_build_dir, 'models') + src_model_dir = os.path.join(build_dir, 'models') + if os.path.exists(src_model_dir) and not os.path.exists(dest_model_dir): + subprocess.run(['mklink', '/D', '/J', dest_model_dir, src_model_dir],shell=True, check=True) + + +def clean_targets(cmake_path, build_dir, configs): + for config in configs: + log.info("Cleaning targets for %s configuration", config) + build_dir2 = get_config_build_dir(build_dir, config) + cmd_args = [cmake_path, + "--build", build_dir2, + "--config", config, + "--target", "clean"] + + run_subprocess(cmd_args) + +def build_targets(cmake_path, build_dir, configs, parallel): + for config in configs: + log.info("Building targets for %s configuration", config) + build_dir2 = get_config_build_dir(build_dir, config) + cmd_args = [cmake_path, + "--build", build_dir2, + "--config", config] + + build_tool_args = [] + if parallel: + num_cores = str(multiprocessing.cpu_count()) + if is_windows(): + build_tool_args += ["/maxcpucount:" + num_cores] + else: + build_tool_args += ["-j" + num_cores] + + if (build_tool_args): + cmd_args += [ "--" ] + cmd_args += build_tool_args + + run_subprocess(cmd_args) + +def add_dir_if_exists(dir, dir_list): + if (os.path.isdir(dir)): + dir_list.append(dir) + +def setup_cuda_vars(args): + + cuda_home = "" + cudnn_home = "" + + if (args.use_cuda): + cuda_home = args.cuda_home if args.cuda_home else os.getenv("CUDA_HOME") + cudnn_home = args.cudnn_home if args.cudnn_home else os.getenv("CUDNN_HOME") + + cuda_home_valid = (cuda_home != None and os.path.exists(cuda_home)) + cudnn_home_valid = (cudnn_home != None and os.path.exists(cudnn_home)) + + if (not cuda_home_valid or not cudnn_home_valid): + log.error("cuda_home and cudnn_home paths must be specified and valid.") + log.error("cuda_home='{}' valid={}. cudnn_home='{}' valid={}" + .format(cuda_home, cuda_home_valid, cudnn_home, cudnn_home_valid)) + sys.exit(-1) + + if (is_windows()): + os.environ["CUDA_PATH"] = cuda_home + os.environ["CUDA_TOOLKIT_ROOT_DIR"] = cuda_home + + cuda_bin_path = os.path.join(cuda_home, 'bin') + os.environ["CUDA_BIN_PATH"] = cuda_bin_path + os.environ["PATH"] += os.pathsep + cuda_bin_path + + # Add version specific CUDA_PATH_Vx_y value as the Visual Studio build files require that + version_file = os.path.join(cuda_home, 'version.txt') + if not os.path.exists(version_file): + log.error("No version file found in CUDA install directory. Looked for " + version_file) + sys.exit(-1) + + with open(version_file) as f: + # First line of version file should have something like 'CUDA Version 9.2.148' + first_line = f.readline() + m = re.match("CUDA Version (\d+).(\d+)", first_line) + if not m: + log.error("Couldn't read version from first line of " + version_file) + sys.exit(-1) + + major = m.group(1) + minor = m.group(2) + os.environ["CUDA_PATH_V{}_{}".format(major, minor)] = cuda_home + + vc_ver_str = os.getenv("VCToolsVersion") or "" + vc_ver = vc_ver_str.split(".") + if len(vc_ver) != 3: + log.warning("Unable to automatically verify VS 2017 toolset is compatible with CUDA. Will attempt to use.") + log.warning("Failed to get valid Visual C++ Tools version from VCToolsVersion environment variable value of '" + vc_ver_str + "'") + log.warning("VCToolsVersion is set in a VS 2017 Developer Command shell, or by running \"%VS2017INSTALLDIR%\\VC\\Auxiliary\\Build\\vcvars64.bat\"") + log.warning("See ReadMe.md in the root ONNXRuntime directory for instructions on installing the Visual C++ 2017 14.11 toolset if needed.") + + elif vc_ver[0] == "14" and int(vc_ver[1]) > 11: + log.error("Visual C++ Tools version not supported by CUDA. You must setup the environment to use the 14.11 toolset.") + log.info("Current version is {}. CUDA 9.2 requires version 14.11.*".format(vc_ver_str)) + log.info("If necessary manually install the 14.11 toolset using the Visual Studio 2017 updater.") + log.info("See https://blogs.msdn.microsoft.com/vcblog/2017/11/15/side-by-side-minor-version-msvc-toolsets-in-visual-studio-2017/") + log.info("Run 'C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Enterprise\\VC\\Auxiliary\\Build\\vcvarsall.bat amd64 -vcvars_ver=14.11' prior to this script, or use build.amd64.1411.bat.") + sys.exit(-1) + + return cuda_home, cudnn_home + +def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs, enable_python_tests, enable_tvm = False): + for config in configs: + log.info("Running tests for %s configuration", config) + cwd = get_config_build_dir(build_dir, config) + dll_path = os.path.join(build_dir, config, "external", "tvm", config) if enable_tvm else None + run_subprocess([ctest_path, "--build-config", config, "--verbose"], + cwd=cwd, dll_path=dll_path) + + if enable_python_tests: + if is_windows(): + cwd = os.path.join(cwd, config) + run_subprocess([sys.executable, 'onnxruntime_test_python.py'], cwd=cwd, dll_path=dll_path) + try: + import onnx + onnx_test = True + except ImportError: + warnings.warn("onnx is not installed. Following test cannot be run.") + onnx_test = False + if onnx_test: + run_subprocess([sys.executable, 'onnxruntime_test_python_backend.py'], cwd=cwd, dll_path=dll_path) + run_subprocess([sys.executable, os.path.join(source_dir,'onnxruntime','test','onnx','gen_test_models.py'),'--output_dir','test_models'], cwd=cwd) + run_subprocess([os.path.join(cwd,'onnx_test_runner'), 'test_models'], cwd=cwd) + #The following one may fail + run_subprocess([sys.executable, 'onnx_backend_test_series.py'], cwd=cwd, dll_path=dll_path, check=False) + try: + import onnxmltools + import keras + onnxml_test = True + except ImportError: + warnings.warn("onnxmltools and keras are not installed. Following test cannot be run.") + onnxml_test = False + if onnxml_test: + run_subprocess([sys.executable, 'onnxruntime_test_python_keras.py'], cwd=cwd, dll_path=dll_path) + + # shared lib tests - both simple + custom op + if args.build_shared_lib: + if is_ubuntu_1604(): + run_subprocess([cwd+'/onnxruntime_shared_lib_test'], cwd=cwd, dll_path=dll_path) + +def run_onnx_tests(build_dir, configs, onnx_test_data_dir, provider): + #TODO: enable multiple threaded executor test + for config in configs: + cwd = get_config_build_dir(build_dir, config) + if is_windows(): + exe = os.path.join(cwd, config, 'onnx_test_runner') + model_dir = os.path.join(cwd, "models") + else: + exe = os.path.join(cwd, 'onnx_test_runner') + model_dir = os.path.join(build_dir, "models") + cmd = [exe] + if provider: + cmd += ["-e", provider] + if config != 'Debug' and os.path.exists(model_dir): + cmd.append(model_dir) + if os.path.exists(onnx_test_data_dir): + cmd.append(onnx_test_data_dir) + run_subprocess(cmd, cwd=cwd) + +def build_python_wheel(source_dir, build_dir, configs, use_cuda): + for config in configs: + cwd = get_config_build_dir(build_dir, config) + if is_windows(): + cwd = os.path.join(cwd, config) + if use_cuda: + run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel', '--use_cuda'], cwd=cwd) + else: + run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel'], cwd=cwd) + if is_ubuntu_1604(): + run_subprocess([os.path.join(source_dir, 'rename_manylinux.sh')], cwd=cwd+'/dist') + +def main(): + args = parse_arguments() + + cmake_path = args.cmake_path + cmake_extra_defines = args.cmake_extra_defines if args.cmake_extra_defines else [] + + # if there was no explicit argument saying what to do, default to update, build and test. + if (args.update == False and args.clean == False and args.build == False and args.test == False): + log.debug("Defaulting to running update, build and test.") + args.update = True + args.build = True + args.test = True + + if args.build_wheel: + args.enable_pybind = True + + configs = set(args.config) + + # setup paths and directories + ctest_path = args.ctest_path + build_dir = args.build_dir + script_dir = os.path.realpath(os.path.dirname(__file__)) + source_dir = os.path.normpath(os.path.join(script_dir, "..", "..")) + + # if using cuda, setup cuda paths and env vars + cuda_home, cudnn_home = setup_cuda_vars(args) + + # directory from ONNX submodule with ONNX test data + onnx_test_data_dir = '/data/onnx' + if is_windows() or not os.path.exists(onnx_test_data_dir): + onnx_test_data_dir = os.path.join(source_dir, "cmake", "external", "onnx", "onnx", "backend", "test", "data") + + os.makedirs(build_dir, exist_ok=True) + + log.info("Build started") + + cmake_extra_args = [] + if(is_windows()): + if (args.x86): + cmake_extra_args = ['-A','Win32','-G', 'Visual Studio 15 2017'] + else: + cmake_extra_args = ['-A','x64','-T', 'host=x64', '-G', 'Visual Studio 15 2017'] + + #Add python to PATH. Please remove this after https://github.com/onnx/onnx/issues/1080 is fixed () + os.environ["PATH"] = os.path.dirname(sys.executable) + os.pathsep + os.environ["PATH"] + + if (args.update): + if is_ubuntu_1604(): + install_ubuntu_deps(args) + install_python_deps() + if (args.enable_pybind and is_windows()): + install_python_deps() + if (not args.skip_submodule_sync): + update_submodules(source_dir) + + generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, args.pb_home, configs, cmake_extra_defines, + args, cmake_extra_args) + + if (args.clean): + clean_targets(cmake_path, build_dir, configs) + + if (args.build): + build_targets(cmake_path, build_dir, configs, args.parallel) + + if (args.test): + run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs, args.enable_pybind, args.use_tvm) + + # run the onnx model tests if requested explicitly. + if (args.enable_onnx_tests): + if args.use_cuda: + run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'cuda') + else: + run_onnx_tests(build_dir, configs, onnx_test_data_dir, None) + if args.use_mkldnn: + run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'mkldnn') + + if args.build_wheel: + build_python_wheel(source_dir, build_dir, configs, args.use_cuda) + + log.info("Build complete") + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/ci_build/gen_def.py b/tools/ci_build/gen_def.py new file mode 100755 index 0000000000000..41eb75abde70e --- /dev/null +++ b/tools/ci_build/gen_def.py @@ -0,0 +1,54 @@ +#!/usr/bin/python3 +import sys +import argparse +import os + +def parse_arguments(): + parser = argparse.ArgumentParser() + parser.add_argument("--src_root", required=True, help="input symbol file") + parser.add_argument("--output", required=True, help="output file") + parser.add_argument("--version_file", required=True, help="VERSION_NUMBER file") + parser.add_argument("--style", required=True, choices=["gcc", "vc"]) + parser.add_argument("--config", required=True, nargs="+") + return parser.parse_args() + +args = parse_arguments() +print("Generating symbol file for %s" % str(args.config)) + +with open(args.version_file, 'r') as f: + VERSION_STRING=f.read().strip(); + +print("VERSION:%s" % VERSION_STRING); + +symbols = set() +for c in args.config: + file_name = os.path.join(args.src_root,'core', 'providers',c,'symbols.txt') + with open(file_name, 'r') as file: + for line in file: + line = line.strip() + if line in symbols: + print("dup symbol: %s", line) + exit(-1) + symbols.add(line) +symbols = sorted(symbols) + +symbol_index = 1 +with open(args.output, 'w') as file: + if args.style == 'vc': + file.write('LIBRARY "onnxruntime.dll"\n') + file.write('EXPORTS\n') + else: + file.write('VERS_%s {\n' % VERSION_STRING) + file.write(' global:\n') + + for symbol in symbols: + if args.style == 'vc': + file.write(" %s @%d\n" % (symbol,symbol_index)) + else: + file.write(" %s;\n" % symbol) + symbol_index +=1 + + if args.style == 'gcc': + file.write(" local:\n") + file.write(" *;\n") + file.write("}; \n") diff --git a/tools/ci_build/github/azure-pipelines/azure-pipelines.yml b/tools/ci_build/github/azure-pipelines/azure-pipelines.yml new file mode 100644 index 0000000000000..82b7b1a09c720 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/azure-pipelines.yml @@ -0,0 +1,99 @@ + +jobs: +- job: Linux_CI_Dev + + pool: Linux-CPU + + steps: + - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory)' + displayName: 'Command Line Script' + env: + AZURE_BLOB_KEY: $(onnxruntime-storage-key) + + - script: 'sudo rm -rf $(Agent.BuildDirectory)' + displayName: 'Clean build folders/files' + condition: always() + +- job: Linux_CI_GPU_Dev + + pool: Linux-GPU + + steps: + - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory)' + displayName: 'Command Line Script' + env: + AZURE_BLOB_KEY: $(onnxruntime-storage-key) + + - script: 'sudo rm -rf $(Agent.BuildDirectory)' + displayName: 'Clean build folders/files' + condition: always() + +- job: Windows_CI_Dev + + pool: Win-CPU + + steps: + - task: CmdLine@1 + displayName: 'Get ONNX testdata' + inputs: + filename: azcopy + arguments: ' /S /Source:https://onnxruntimetestdata.blob.core.windows.net/onnx-model-zoo-20181018 /Dest:$(Build.SourcesDirectory)\build\Windows\Debug\models /SourceKey:%AZURE_BLOB_KEY%' + env: + AZURE_BLOB_KEY: $(onnxruntime-storage-key) + + - task: BatchScript@1 + inputs: + filename: build.bat + arguments: ' --enable_onnx_tests' + workingFolder: "$(Build.SourcesDirectory)" + + - task: CmdLine@1 + displayName: 'Clean build folders/files' + inputs: + filename: rd + arguments: '/s /q $(Agent.BuildDirectory)' + continueOnError: true + condition: always() + +- job: Windows_CI_GPU_Dev + + pool: Win-GPU + + variables: + CUDA_VERSION: '9.0' + + steps: + - task: PowerShell@1 + displayName: 'Set CUDA path' + inputs: + scriptName: 'tools/ci_build/github/windows/set_cuda_path.ps1' + arguments: '-CudaMsbuildPath C:\local\cudaMsbuildIntegration-9.0.176-windows10-x64-2 -CudaVersion $(CUDA_VERSION)' + + - task: BatchScript@1 + displayName: 'Setup VS2017 env vars' + inputs: + filename: 'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat' + arguments: 'amd64 -vcvars_ver=14.11' + modifyEnvironment: true + + - task: CmdLine@1 + displayName: 'Get ONNX testdata' + inputs: + filename: azcopy + arguments: ' /S /Source:https://onnxruntimetestdata.blob.core.windows.net/onnx-model-zoo-20181018 /Dest:$(Build.SourcesDirectory)\build\Windows\Debug\models /SourceKey:%AZURE_BLOB_KEY%' + env: + AZURE_BLOB_KEY: $(onnxruntime-storage-key) + + - task: BatchScript@1 + inputs: + filename: build.bat + arguments: ' --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda-9.0.176-windows10-x64-0" --cudnn_home="C:\local\cudnn-9.0-windows10-x64-v7.0.5-0\cuda"' + workingFolder: "$(Build.SourcesDirectory)" + + - task: CmdLine@1 + displayName: 'Clean build folders/files' + inputs: + filename: rd + arguments: '/s /q $(Agent.BuildDirectory)' + continueOnError: true + condition: always() \ No newline at end of file diff --git a/tools/ci_build/github/linux/create_package.sh b/tools/ci_build/github/linux/create_package.sh new file mode 100755 index 0000000000000..e2f1250759147 --- /dev/null +++ b/tools/ci_build/github/linux/create_package.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -e +SCRIPT=`realpath $0` +SCRIPT_DIR=`dirname $SCRIPT` +TOP_SRC_DIR=`realpath $SCRIPT_DIR/../../../../` +mkdir -p $HOME/.cache/onnxruntime +if [ -z "$BUILD_ARTIFACTSTAGINGDIRECTORY" ]; then + sudo mkdir -p /data/a + BUILD_ARTIFACTSTAGINGDIRECTORY="/data/a" +fi + +if [ -z "$BUILD_BINARIESDIRECTORY" ]; then + sudo mkdir -p /data/b + BUILD_BINARIESDIRECTORY="/data/b" +fi + + +for version in '23'; do + docker_image=fedora$version + cd $SCRIPT_DIR/docker + docker build -t $docker_image --build-arg OS_VERSION=$version -f Dockerfile.fedora . + docker run --rm -e AZURESASKEY --volume "$HOME/.cache/onnxruntime:/root/.cache/onnxruntime" -v $BUILD_BINARIESDIRECTORY:/root/rpmbuild -v $BUILD_ARTIFACTSTAGINGDIRECTORY:/data/a -v $HOME/.ccache:/root/.ccache -v $TOP_SRC_DIR:/data/onnxruntime -w /data/b $docker_image /data/onnxruntime/tools/ci_build/github/linux/create_package_inside_docker.sh +done diff --git a/tools/ci_build/github/linux/create_package_inside_docker.sh b/tools/ci_build/github/linux/create_package_inside_docker.sh new file mode 100755 index 0000000000000..ce0972551d35e --- /dev/null +++ b/tools/ci_build/github/linux/create_package_inside_docker.sh @@ -0,0 +1,47 @@ +#!/bin/bash +#This script uses system provided eigen and protobuf, not the ones in onnxruntime repo. +export PATH=/usr/lib64/ccache:$PATH +set -e +SCRIPT_DIR="$( dirname "${BASH_SOURCE[0]}" )" +TOP_SRC_DIR=$(realpath $SCRIPT_DIR/../../../..) +echo $TOP_SRC_DIR +VERSION_NUMBER=$(cat $TOP_SRC_DIR/VERSION_NUMBER) +rpmdev-setuptree +tmp_build_dir=$(mktemp -d) +export TMP_SOURCE_DIR=$tmp_build_dir/onnxruntime-$VERSION_NUMBER +mkdir -p $TMP_SOURCE_DIR +(cd $SCRIPT_DIR/../../../.. && git archive --format=tar HEAD | (cd $TMP_SOURCE_DIR && tar xf -)) +echo "exporting submodules..." +#TODO: support submodule in submodule +(cd $SCRIPT_DIR/../../../.. && git submodule foreach --recursive " + DEST_DIR=\$TMP_SOURCE_DIR/\$path; + echo \$DEST_DIR + mkdir -p \$DEST_DIR + git archive --format=tar HEAD | tar -C \$DEST_DIR -xf - +") +tar -cf ~/rpmbuild/SOURCES/onnxruntime.tar -C $tmp_build_dir onnxruntime-$VERSION_NUMBER +rm -rf $tmp_build_dir + +/usr/bin/cp /data/onnxruntime/package/rpm/onnxruntime.spec ~/rpmbuild/SPECS +rpmbuild -ba ~/rpmbuild/SPECS/onnxruntime.spec + +#Install the packages and test it +dnf install -y /root/rpmbuild/RPMS/x86_64/onnxruntime-*$VERSION_NUMBER-*.rpm +if [ -d /data/onnx ]; then + onnx_test_runner /data/onnx +fi + +/usr/bin/cp /root/rpmbuild/RPMS/x86_64/onnxruntime-*$VERSION_NUMBER-*.rpm /data/a + + +cd /data/a +#convert rpm to tar +for filename in onnxruntime-*.rpm; do + tmp_dir=$(mktemp -d) + #older cpio doesn't support -D + rpm2cpio $filename | (cd $tmp_dir && cpio -idmv); + dest_filename="${filename//.rpm/.tar.bz2}" + dest_filename="${dest_filename//\.fc[0-9][0-9]\./.}" + tar -jcf $dest_filename -C $tmp_dir . ; + rm -rf tmp_dir +done diff --git a/tools/ci_build/github/linux/docker/Dockerfile.fedora b/tools/ci_build/github/linux/docker/Dockerfile.fedora new file mode 100644 index 0000000000000..fd7263691d614 --- /dev/null +++ b/tools/ci_build/github/linux/docker/Dockerfile.fedora @@ -0,0 +1,6 @@ +ARG OS_VERSION=27 +FROM fedora:${OS_VERSION} +ENV PATH="/opt/cmake/bin:${PATH}" +ADD scripts /tmp/scripts +RUN /tmp/scripts/install_fedora.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts + diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu new file mode 100644 index 0000000000000..6263c8bf9c17c --- /dev/null +++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu @@ -0,0 +1,31 @@ +ARG OS_VERSION=16.04 +FROM ubuntu:${OS_VERSION} + +ARG PYTHON_VERSION=3.5 + +ADD scripts /tmp/scripts +ENV PATH="/opt/cmake/bin:${PATH}" +RUN /tmp/scripts/install_ubuntu.sh -p ${PYTHON_VERSION} && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts + +WORKDIR /root + +# Build and Install LLVM +# ARG LLVM_VERSION=6.0.1 +# RUN cd /tmp && \ +# wget --no-verbose http://releases.llvm.org/$LLVM_VERSION/llvm-$LLVM_VERSION.src.tar.xz && \ +# xz -d llvm-$LLVM_VERSION.src.tar.xz && \ +# tar xvf llvm-$LLVM_VERSION.src.tar && \ +# cd llvm-$LLVM_VERSION.src && \ +# mkdir -p build && \ +# cd build && \ +# cmake .. -DCMAKE_BUILD_TYPE=Release && \ +# cmake --build . -- -j$(nproc) && \ +# cmake -DCMAKE_INSTALL_PREFIX=/usr/local/llvm-$LLVM_VERSION -DBUILD_TYPE=Release -P cmake_install.cmake && \ +# cd /tmp && \ +# rm -rf llvm* + +ENV LD_LIBRARY_PATH /usr/local/openblas/lib:$LD_LIBRARY_PATH + +ARG BUILD_USER=onnxruntimedev +WORKDIR /home/$BUILD_USER + diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu new file mode 100644 index 0000000000000..bc8602f8d0bb0 --- /dev/null +++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu @@ -0,0 +1,47 @@ +# Tag: nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04 +# Created: 2017-11-21T06:34:14.675603521Z +# Label: com.nvidia.build.id: 41212533 +# Label: com.nvidia.build.ref: e0edb5359ecb7bd3d86f0c9bfa18c2260b741ebb +# Label: com.nvidia.cuda.version: 9.0.176 +# Label: com.nvidia.cudnn.version: 7.0.4.31 +# Label: com.nvidia.nccl.version: 2.1.2 +# Ubuntu 16.04.5 +FROM nvidia/cuda@sha256:33add9c50ab76b8f3a92187c0418ed600d5bea27690fda40711122fdc28ce2f4 + +ARG PYTHON_VERSION=3.5 + +ADD scripts /tmp/scripts +ENV PATH="/opt/cmake/bin:${PATH}" +RUN /tmp/scripts/install_ubuntu.sh -p ${PYTHON_VERSION} && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts + +WORKDIR /root + +# Allow configure to pick up GDK and CuDNN where it expects it. +# (Note: $CUDNN_VERSION is defined by NVidia's base image) +RUN _CUDNN_VERSION=$(echo $CUDNN_VERSION | cut -d. -f1-2) && \ + mkdir -p /usr/local/cudnn-$_CUDNN_VERSION/cuda/include && \ + ln -s /usr/include/cudnn.h /usr/local/cudnn-$_CUDNN_VERSION/cuda/include/cudnn.h && \ + mkdir -p /usr/local/cudnn-$_CUDNN_VERSION/cuda/lib64 && \ + ln -s /etc/alternatives/libcudnn_so /usr/local/cudnn-$_CUDNN_VERSION/cuda/lib64/libcudnn.so && \ + ln -s /usr/local/cudnn{-$_CUDNN_VERSION,} + +# Build and Install LLVM +ARG LLVM_VERSION=6.0.1 +RUN cd /tmp && \ + wget --no-verbose http://releases.llvm.org/$LLVM_VERSION/llvm-$LLVM_VERSION.src.tar.xz && \ + xz -d llvm-$LLVM_VERSION.src.tar.xz && \ + tar xvf llvm-$LLVM_VERSION.src.tar && \ + cd llvm-$LLVM_VERSION.src && \ + mkdir -p build && \ + cd build && \ + cmake .. -DCMAKE_BUILD_TYPE=Release && \ + cmake --build . -- -j$(nproc) && \ + cmake -DCMAKE_INSTALL_PREFIX=/usr/local/llvm-$LLVM_VERSION -DBUILD_TYPE=Release -P cmake_install.cmake && \ + cd /tmp && \ + rm -rf llvm* + +ENV LD_LIBRARY_PATH /usr/local/openblas/lib:$LD_LIBRARY_PATH + +ARG BUILD_USER=onnxruntimedev +WORKDIR /home/$BUILD_USER + diff --git a/tools/ci_build/github/linux/docker/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/scripts/install_deps.sh new file mode 100755 index 0000000000000..e1987194ebacd --- /dev/null +++ b/tools/ci_build/github/linux/docker/scripts/install_deps.sh @@ -0,0 +1,64 @@ +#!/bin/bash +#install ninja +aria2c -q -d /tmp https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip +unzip -oq /tmp/ninja-linux.zip -d /usr/bin +rm -f /tmp/ninja-linux.zip +#install protobuf +mkdir -p /tmp/src +mkdir -p /opt/cmake +aria2c -q -d /tmp/src https://cmake.org/files/v3.12/cmake-3.12.1-Linux-x86_64.tar.gz +tar -xf /tmp/src/cmake-3.12.1-Linux-x86_64.tar.gz --strip 1 -C /opt/cmake +aria2c -q -d /tmp/src https://github.com/protocolbuffers/protobuf/archive/v3.6.1.tar.gz +tar -xf /tmp/src/protobuf-3.6.1.tar.gz -C /tmp/src +cd /tmp/src/protobuf-3.6.1 +if [ -f /etc/redhat-release ] ; then + PB_LIBDIR=lib64 +else + PB_LIBDIR=lib +fi +for build_type in 'Debug' 'Relwithdebinfo'; do + pushd . + mkdir build_$build_type + cd build_$build_type + /opt/cmake/bin/cmake -G Ninja ../cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_LIBDIR=$PB_LIBDIR -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=$build_type + ninja + ninja install + popd +done +export ONNX_ML=1 +INSTALLED_PYTHON_VERSION=$(python3 -c 'import sys; version=sys.version_info[:2]; print("{0}.{1}".format(*version));') +if [ "$INSTALLED_PYTHON_VERSION" = "3.4" ];then + echo "Python 3.5 and above is needed for running onnx tests!" 1>&2 +else + #Install ONNX + #5af210ca8a1c73aa6bae8754c9346ec54d0a756e is v1.2.3 + #bae6333e149a59a3faa9c4d9c44974373dcf5256 is v1.3.0 + #eb4b7c2cc2a0d34c0127e26c2c1cb5e712467e1e is v1.3.0 latest + for onnx_version in "5af210ca8a1c73aa6bae8754c9346ec54d0a756e" "bae6333e149a59a3faa9c4d9c44974373dcf5256" "eb4b7c2cc2a0d34c0127e26c2c1cb5e712467e1e"; do + if [ -z ${lastest_onnx_version+x} ]; then + echo "first pass"; + else + echo "deleting old onnx-${lastest_onnx_version}"; + pip3 uninstall -y onnx + fi + lastest_onnx_version=$onnx_version + aria2c -q -d /tmp/src https://github.com/onnx/onnx/archive/$onnx_version.tar.gz + tar -xf /tmp/src/onnx-$onnx_version.tar.gz -C /tmp/src + cd /tmp/src/onnx-$onnx_version + git clone https://github.com/pybind/pybind11.git third_party/pybind11 + python3 setup.py bdist_wheel + pip3 install -q dist/* + mkdir -p /data/onnx/$onnx_version + backend-test-tools generate-data -o /data/onnx/$onnx_version + done +fi + +#The last onnx version will be kept + +aria2c -q -d /tmp/src http://bitbucket.org/eigen/eigen/get/3.3.5.tar.bz2 +tar -jxf /tmp/src/eigen-eigen-b3f3d4950030.tar.bz2 -C /usr/include +mv /usr/include/eigen-eigen-b3f3d4950030 /usr/include/eigen3 + +rm -rf /tmp/src + + diff --git a/tools/ci_build/github/linux/docker/scripts/install_fedora.sh b/tools/ci_build/github/linux/docker/scripts/install_fedora.sh new file mode 100755 index 0000000000000..c39edb2dc0b5f --- /dev/null +++ b/tools/ci_build/github/linux/docker/scripts/install_fedora.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -e +dnf install -y --best --allowerasing rpmdevtools tar unzip ccache curl gcc gcc-c++ zlib-devel make git python2-devel python3-devel python3-numpy libunwind icu aria2 rsync python3-setuptools python3-wheel bzip2 diff --git a/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh b/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh new file mode 100755 index 0000000000000..9aacae059ef6b --- /dev/null +++ b/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh @@ -0,0 +1,55 @@ +#!/bin/bash +set -e +while getopts p: parameter_Option +do case "${parameter_Option}" +in +p) PYTHON_VER=${OPTARG};; +esac +done + +DEBIAN_FRONTEND=noninteractive +apt-get update && apt-get install -y software-properties-common +add-apt-repository ppa:deadsnakes/ppa +apt-get update && apt-get install -y --no-install-recommends \ + autotools-dev \ + build-essential \ + git ca-certificates \ + ca-certificates \ + pkg-config \ + wget \ + zlib1g \ + zlib1g-dev \ + libssl-dev \ + curl \ + autoconf \ + sudo \ + gfortran \ + python3-dev \ + libopenblas-dev \ + liblttng-ust0 \ + libcurl3 \ + libssl1.0.0 \ + libkrb5-3 \ + libicu55 \ + aria2 \ + bzip2 \ + unzip \ + zip \ + rsync libunwind8 libpng16-dev \ + python3-setuptools python3-numpy python3-wheel python python3-pip python3-pytest + +if [ $PYTHON_VER != "3.5" ]; then + apt-get install -y --no-install-recommends \ + python${PYTHON_VER} \ + python${PYTHON_VER}-dev + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VER} 1 + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.5 2 + update-alternatives --set python3 /usr/bin/python${PYTHON_VER} +fi + +rm -rf /var/lib/apt/lists/* + +mkdir -p /tmp/azcopy +aria2c -q -d /tmp/azcopy -o azcopy.tar.gz https://aka.ms/downloadazcopylinux64 +tar -xf /tmp/azcopy/azcopy.tar.gz -C /tmp/azcopy +/tmp/azcopy/install.sh diff --git a/tools/ci_build/github/linux/run_build.sh b/tools/ci_build/github/linux/run_build.sh new file mode 100755 index 0000000000000..8ed87f451c99e --- /dev/null +++ b/tools/ci_build/github/linux/run_build.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -e -o -x + +id + +SCRIPT_DIR="$( dirname "${BASH_SOURCE[0]}" )" + +while getopts c:d:x: parameter_Option +do case "${parameter_Option}" +in +d) BUILD_DEVICE=${OPTARG};; +esac +done + +if [ -z "$AZURE_BLOB_KEY" ]; then + echo "AZURE_BLOB_KEY is blank" +else + echo "Downloading test data from azure" + mkdir -p /home/onnxruntimedev/models/ + azcopy --recursive --source:https://onnxruntimetestdata.blob.core.windows.net/onnx-model-zoo-20181018 --destination:/home/onnxruntimedev/models/ --source-key:$AZURE_BLOB_KEY + BUILD_EXTR_PAR="--enable_onnx_tests" +fi + +if [ $BUILD_DEVICE = "gpu" ]; then + python3 $SCRIPT_DIR/../../build.py --build_dir /home/onnxruntimedev \ + --config Debug Release \ + --skip_submodule_sync \ + --parallel --build_shared_lib \ + --use_cuda \ + --cuda_home /usr/local/cuda \ + --cudnn_home /usr/local/cudnn-7.0/cuda --build_shared_lib $BUILD_EXTR_PAR + /home/onnxruntimedev/Release/onnx_test_runner -e cuda /data/onnx +else + python3 $SCRIPT_DIR/../../build.py --build_dir /home/onnxruntimedev \ + --config Debug Release --build_shared_lib \ + --skip_submodule_sync \ + --enable_pybind \ + --parallel --use_mkldnn --use_mklml --build_shared_lib $BUILD_EXTR_PAR + /home/onnxruntimedev/Release/onnx_test_runner /data/onnx +fi diff --git a/tools/ci_build/github/linux/run_dockerbuild.sh b/tools/ci_build/github/linux/run_dockerbuild.sh new file mode 100755 index 0000000000000..debdb19805cf8 --- /dev/null +++ b/tools/ci_build/github/linux/run_dockerbuild.sh @@ -0,0 +1,62 @@ +#!/bin/bash +set -e -o -x + +SCRIPT_DIR="$( dirname "${BASH_SOURCE[0]}" )" +SOURCE_ROOT=$(realpath $SCRIPT_DIR/../../../../) + +while getopts c:o:d:r:p:x: parameter_Option +do case "${parameter_Option}" +in +#ubuntu16.04 +o) BUILD_OS=${OPTARG};; +#cpu, gpu +d) BUILD_DEVICE=${OPTARG};; +r) BUILD_DIR=${OPTARG};; +#python version: 3.6 3.7 (absence means default 3.5) +p) PYTHON_VER=${OPTARG};; +# "--build_wheel --use_openblas" +x) BUILD_EXTR_PAR=${OPTARG};; +esac +done + +EXIT_CODE=1 + +echo "bo=$BUILD_OS bd=$BUILD_DEVICE bdir=$BUILD_DIR pv=$PYTHON_VER bex=$BUILD_EXTR_PAR" + +cd $SCRIPT_DIR/docker +if [ $BUILD_DEVICE = "gpu" ]; then + IMAGE="ubuntu16.04-cuda9.0-cudnn7.0" + docker build -t "onnxruntime-$IMAGE" --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu_gpu . +else + IMAGE="ubuntu16.04" + docker build -t "onnxruntime-$IMAGE" --build-arg OS_VERSION=16.04 --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu . +fi + +set +e + +if [ $BUILD_DEVICE = "cpu" ]; then + docker rm -f $HOSTNAME || true + docker run -h $HOSTNAME \ + --rm -e AZURE_BLOB_KEY \ + --name "onnxruntime-$BUILD_DEVICE" \ + --volume "$SOURCE_ROOT:/onnxruntime_src" \ + --volume "$BUILD_DIR:/home/onnxruntimedev" \ + "onnxruntime-$IMAGE" \ + /bin/bash /onnxruntime_src/tools/ci_build/github/linux/run_build.sh \ + -d $BUILD_DEVICE -x "$BUILD_EXTR_PAR" & +else + nvidia-docker run --rm -h $HOSTNAME \ + --rm -e AZURE_BLOB_KEY \ + --name "onnxruntime-$BUILD_DEVICE" \ + --volume "$SOURCE_ROOT:/onnxruntime_src" \ + --volume "$BUILD_DIR:/home/onnxruntimedev" \ + "onnxruntime-$IMAGE" \ + /bin/bash /onnxruntime_src/tools/ci_build/github/linux/run_build.sh \ + -d $BUILD_DEVICE -x "$BUILD_EXTR_PAR" & +fi +wait -n + +EXIT_CODE=$? + +set -e +exit $EXIT_CODE diff --git a/tools/ci_build/github/linux/ubuntu16.04/Dockerfile b/tools/ci_build/github/linux/ubuntu16.04/Dockerfile new file mode 100644 index 0000000000000..6ec132b370827 --- /dev/null +++ b/tools/ci_build/github/linux/ubuntu16.04/Dockerfile @@ -0,0 +1,15 @@ +FROM ubuntu:16.04 + +ADD install.sh /tmp/install.sh +ENV PATH="/opt/cmake/bin:${PATH}" +RUN /tmp/install.sh && rm /tmp/install.sh + +WORKDIR /root + +ENV LD_LIBRARY_PATH /usr/local/openblas/lib:$LD_LIBRARY_PATH + +ARG BUILD_UID=1000 +ARG BUILD_USER=onnxruntimedev +RUN adduser --gecos 'ONNXRuntime Build User' --disabled-password $BUILD_USER --uid $BUILD_UID +USER $BUILD_USER +WORKDIR /home/$BUILD_USER diff --git a/tools/ci_build/github/linux/ubuntu16.04/install.sh b/tools/ci_build/github/linux/ubuntu16.04/install.sh new file mode 100755 index 0000000000000..7c35cc3c75657 --- /dev/null +++ b/tools/ci_build/github/linux/ubuntu16.04/install.sh @@ -0,0 +1,77 @@ +#!/bin/bash +set -e +DEBIAN_FRONTEND=noninteractive +apt-get update && apt-get install -y --no-install-recommends \ + autotools-dev \ + build-essential \ + git ca-certificates \ + ca-certificates \ + pkg-config \ + wget \ + zlib1g \ + zlib1g-dev \ + libssl-dev \ + curl \ + autoconf \ + sudo \ + gfortran \ + python3-dev \ + libopenblas-dev \ + liblttng-ust0 \ + libcurl3 \ + libssl1.0.0 \ + libkrb5-3 \ + libicu55 \ + aria2 \ + bzip2 \ + unzip \ + rsync libunwind8 \ + python3-setuptools python3-numpy python3-wheel python python3-pip + +rm -rf /var/lib/apt/lists/* + +aria2c -q -d /tmp https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip +unzip -q /tmp/ninja-linux.zip -d /usr/bin +mkdir -p /tmp/azcopy +aria2c -q -d /tmp/azcopy -o azcopy.tar.gz https://aka.ms/downloadazcopylinux64 +tar -xf /tmp/azcopy/azcopy.tar.gz -C /tmp/azcopy +/tmp/azcopy/install.sh +rm -rf /tmp/azcopy +#install protobuf +mkdir -p /tmp/src +mkdir -p /opt/cmake +aria2c -q -d /tmp/src https://cmake.org/files/v3.12/cmake-3.12.1-Linux-x86_64.tar.gz +tar -xf /tmp/src/cmake-3.12.1-Linux-x86_64.tar.gz --strip 1 -C /opt/cmake +aria2c -q -d /tmp/src https://github.com/protocolbuffers/protobuf/archive/v3.6.1.tar.gz +tar -xf /tmp/src/protobuf-3.6.1.tar.gz -C /tmp/src +cd /tmp/src/protobuf-3.6.1 +for build_type in 'Debug' 'Relwithdebinfo'; do + pushd . + mkdir build_$build_type + cd build_$build_type + cmake -G Ninja ../cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=$build_type + ninja + ninja install + popd +done +export ONNX_ML=1 +#3376d4438aaadfba483399fa249b841153152bc0 is v1.2.2 +for onnx_version in "3376d4438aaadfba483399fa249b841153152bc0" "6f91908b6a894278377e2767dc9ce75ce197fb88" ; do + aria2c -q -d /tmp/src https://github.com/onnx/onnx/archive/$onnx_version.tar.gz + tar -xf /tmp/src/onnx-$onnx_version.tar.gz -C /tmp/src + cd /tmp/src/onnx-$onnx_version + git clone https://github.com/pybind/pybind11.git third_party/pybind11 + python3 setup.py bdist_wheel + pip3 install -q dist/* + mkdir -p /data/onnx/$onnx_version + backend-test-tools generate-data -o /data/onnx/$onnx_version + pip3 uninstall -y onnx +done + +aria2c -q -d /tmp/src http://bitbucket.org/eigen/eigen/get/3.3.5.tar.bz2 +tar -jxf /tmp/src/eigen-eigen-b3f3d4950030.tar.bz2 -C /usr/include +mv /usr/include/eigen-eigen-b3f3d4950030 /usr/include/eigen3 +chmod 0777 /data/onnx +rm -rf /tmp/src + + diff --git a/tools/ci_build/github/windows/set_cuda_path.ps1 b/tools/ci_build/github/windows/set_cuda_path.ps1 new file mode 100644 index 0000000000000..ce7e4fe9b197a --- /dev/null +++ b/tools/ci_build/github/windows/set_cuda_path.ps1 @@ -0,0 +1,18 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +#Copy CUDA props files +Param( + [Parameter(Mandatory=$True)] + [string]$CudaMsbuildPath, + [string]$CudaVersion +) +$CudaMsbuildPath=$(Join-Path $CudaMsbuildPath "V141\BuildCustomizations") +$Dst = $(Join-Path "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community" "Common7\IDE\VC\VCTargets\BuildCustomizations") + +Write-Host "Copy CUDA prop files" +Copy-Item $(Join-Path $CudaMsbuildPath "CUDA ${CudaVersion}.props") -Destination $Dst +Copy-Item $(Join-Path $CudaMsbuildPath "CUDA ${CudaVersion}.targets") -Destination $Dst +Copy-Item $(Join-Path $CudaMsbuildPath "CUDA ${CudaVersion}.xml") -Destination $Dst +Copy-Item $(Join-Path $CudaMsbuildPath "Nvda.Build.CudaTasks.v${CudaVersion}.dll") -Destination $Dst + diff --git a/tools/ci_build/static_analysis_filter.py b/tools/ci_build/static_analysis_filter.py new file mode 100644 index 0000000000000..254880aca9db3 --- /dev/null +++ b/tools/ci_build/static_analysis_filter.py @@ -0,0 +1,87 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import argparse +import collections +import re +import sys + +def parse_args(): + parser = argparse.ArgumentParser( + description="Filters MSBuild output for static analysis warnings.") + parser.add_argument("input", + help="Input file containing the build output.") + parser.add_argument("--tsv", + help="Writes TSV output to the specified file.") + parser.add_argument("--echo", action="store_true", + help="Echoes input to stdout.") + return parser.parse_args() + +WarningInfo = collections.namedtuple( + "WarningInfo", ["location", "code", "description", "project"]) + +def make_warning_info(location, code_str, description, project): + normalized_location = location.strip().lower() + code = int(code_str) + return WarningInfo._make([ + normalized_location, code, description, project]) + +path_pattern = r"[A-Za-z0-9_\- :./\\]+" +warning_line_pattern = r"^(" + path_pattern + r"\(\d+\)):" + \ + r"\s+warning C(\d+):" + \ + r"\s+(.*)" + \ + r"\s+\[(" + path_pattern + r")\]$" +warning_line_re = re.compile(warning_line_pattern) + +def parse_warning_line(line): + match = warning_line_re.match(line) + if not match: return None + return make_warning_info(match.group(1), match.group(2), + match.group(3), match.group(4)) + +def filter_warning(warning, min_warning_code, ignored_location_pattern_res): + if warning.code < min_warning_code: return False + for ignored_location_pattern_re in ignored_location_pattern_res: + if ignored_location_pattern_re.search(warning.location): + return False + return True + +def write_tsv(warnings, tsv_file): + with open(tsv_file, "w") as tsv_file_content: + tsv_file_content.write("location\tcode\tdescription\tproject\n") + for warning in warnings: + tsv_file_content.write( + "\t".join([warning.location, + str(warning.code), + warning.description, + warning.project]) + + "\n") + +def main(): + args = parse_args() + path_separator_pattern = r"[\\/]" + ignored_location_pattern_res = [re.compile(pattern) for pattern in [ + path_separator_pattern.join(["protobuf", "src"]), + ]] + warnings = list() + unique_warnings = set() + + with open(args.input, "r") as infile: + for line in infile: + warning = parse_warning_line(line) + if warning and \ + filter_warning(warning, 6000, ignored_location_pattern_res) and \ + warning not in unique_warnings: + warnings.append(warning) + unique_warnings.add(warning) + + if args.echo: + sys.stdout.write(line) + + if args.tsv: + write_tsv(warnings, args.tsv) + + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/nuget/create_nuspect.py b/tools/nuget/create_nuspect.py new file mode 100644 index 0000000000000..e7ab6942083f2 --- /dev/null +++ b/tools/nuget/create_nuspect.py @@ -0,0 +1,44 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import argparse +import sys + + +def parse_arguments(): + parser = argparse.ArgumentParser(description="ONNX Runtime create nuget spec script", + usage='') + # Main arguments + parser.add_argument("--source_dir", required=True, help="Path to the source directory.") + parser.add_argument("--debug_binary_root", required=True, help="Path to the debug binary directory.") + parser.add_argument("--release_binary_root", required=True, help="Path to the release binary directory.") + return parser.parse_args() + +def generate_nuspec(source_dir, debug_binary_root, release_binary_root, architecture): + template_path = '%s/tools/nuget/template.nuspec' % source_dir + with open(template_path, 'rt') as f: + template = f.read() + return template.replace('@@DebugBinaryRoot@@', debug_binary_root)\ + .replace('@@ReleaseBinaryRoot@@', release_binary_root)\ + .replace('@@MSBuildArchitecture@@', architecture)\ + .replace('@@SrcRoot@@', source_dir) + + +def generate_targets(source_dir): + template_path = '%s/tools/nuget/template.targets' % source_dir + with open(template_path, 'rt') as f: + template = f.read() + return template + + +def main(): + args = parse_arguments() + nuspec = generate_nuspec(args.source_dir, args.debug_binary_root, args.release_binary_root, 'amd64') + with open('onnxruntime.nuspec', 'wt') as f: + f.write(nuspec) + targets = generate_targets(args.source_dir) + with open('onnxruntime.targets', 'wt') as f: + f.write(targets) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/nuget/template.nuspec b/tools/nuget/template.nuspec new file mode 100644 index 0000000000000..eaee07327295f --- /dev/null +++ b/tools/nuget/template.nuspec @@ -0,0 +1,110 @@ + + + + + Onnxruntime + + + 0.0.1 + + + Onnxruntime Team + + + Onnxruntime Team + + + http://opensource.org/licenses/MS-PL + https://github.com/onnx/onnxruntime + + + http://github.com/contoso/UsefulStuff/nuget_icon.png + + + false + + + Initial Release + + + Onnxruntime is a runtime for ONNX models + + + Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. + + + ONNX DeepLearning + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tools/nuget/template.targets b/tools/nuget/template.targets new file mode 100644 index 0000000000000..3c52829d7fd47 --- /dev/null +++ b/tools/nuget/template.targets @@ -0,0 +1,22 @@ + + + + + ONNX_NAMESPACE=onnx;ONNX_ML;%(PreprocessorDefinitions) + $(MSBuildThisFileDirectory)..\..\build\native\include\;%(AdditionalIncludeDirectories) + + + + + $(MSBuildThisFileDirectory)..\..\lib\native\debug\amd64\;%(AdditionalLibraryDirectories) + onnx.lib;onnx_proto.lib;libprotobufd.lib;onnxruntime_graph.lib;onnxruntime_common.lib;/WHOLEARCHIVE:onnxruntime_framework.lib;/WHOLEARCHIVE:onnxruntime_providers.lib;%(AdditionalDependencies) + + + + + $(MSBuildThisFileDirectory)..\..\lib\native\retail\amd64\;%(AdditionalLibraryDirectories) + onnx.lib;onnx_proto.lib;libprotobuf.lib;onnxruntime_graph.lib;onnxruntime_common.lib;/WHOLEARCHIVE:onnxruntime_framework.lib;/WHOLEARCHIVE:onnxruntime_providers.lib;%(AdditionalDependencies) + + + + \ No newline at end of file diff --git a/tools/onnx/ModelOperatorParser.py b/tools/onnx/ModelOperatorParser.py new file mode 100644 index 0000000000000..0580af7defb21 --- /dev/null +++ b/tools/onnx/ModelOperatorParser.py @@ -0,0 +1,80 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import argparse +import onnx +import os +import sys + +from onnx.onnx_pb import ModelProto +from google.protobuf import text_format + +onnx_operator_list = [ + "Abs", "Acos", "Add", "And", "ArgMax", "ArgMin", "Asin", "Atan", "AveragePool", "BatchNormalization", + "Cast", "Ceil", "Clip", "Concat", "Constant", "Conv", "ConvTranspose", "Cos", "DepthToSpace", "Div", + "Dropout", "Elu", "Equal", "Exp", "Flatten", "Floor", "GRU", "Gather", "Gemm", "GlobalAveragePool", + "GlobalLpPool", "GlobalMaxPool", "Greater", "HardSigmoid", "Hardmax", "Identity", + "InstanceNormalization", "LRN", "LSTM", "LeakyRelu", "Less", "Log", "LogSoftmax", "LpNormalization", + "LpPool", "MatMul", "Max", "MaxPool", "MaxRoiPool", "Mean", "Min", "Mul", "Multinomial", "Neg", "Not", + "Or", "PRelu", "Pad", "Pow", "RNN", "RandomNormal", "RandomNormalLike", "RandomUniform", "RandomUniformLike", + "Reciprocal", "ReduceL1", "ReduceL2", "ReduceLogSum", "ReduceLogSumExp", "ReduceMax", "ReduceMean", + "ReduceMin", "ReduceProd", "ReduceSum", "ReduceSumSquare", "Relu", "Reshape", "Selu", "Shape", "Sigmoid", + "Sin", "Size", "Slice", "Softmax", "Softplus", "Softsign", "SpaceToDepth", "Split", "Sqrt", "Squeeze", + "Sub", "Sum", "Tan", "Tanh", "Tile", "TopK", "Transpose", "Unsqueeze", "Upsample", "Xor", + "ATen", "Affine", "ConstantFill", "Crop", "GRUUnit", "GivenTensorFill", "If", "ImageScaler", "Loop", + "LoopIndexTensor", "MeanVarianceNormalization", "ParametricSoftplus", "Scale", "ScaledTanh", "ThresholdedRelu" +] + +#Given a model directory, it parses the model operators and outputs in CSV formst as below: +# Abs, Acos, Add, Add, ... +# Model1, 5, 18, , 1, ... +# Model2, , 19, 3, , ... +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Operators of ONNX model.') + parser.add_argument('modelDir', nargs=1, help='Model Directory Path') + parser.add_argument('resultFile', nargs=1, help='Result File Path') + + model_dir = parser.parse_args().modelDir[0] + result_file = parser.parse_args().resultFile[0] + + with open(result_file, 'a') as result: + + for root, dirs, files in os.walk(model_dir): + for file in files: + if file.endswith('.onnx'): + abs_path = root + "/" + file + f = open(abs_path, "rb") + proto = ModelProto() + proto.ParseFromString(f.read()) + f.close() + + # Get model operator list + operators = {} + nodes = proto.graph.node + for node in nodes: + print(node.op_type) + if not node.op_type in operators: + operators[node.op_type]=1 + else: + operators[node.op_type]+=1 + + # store all the operators with format: + # ModelName[,occurences]* + + #head + head = '' + for op in onnx_operator_list: + head += ',{}'.format(op) + result.write('{}\n'.format(head)) + + #content + model_name = file[:-5] + if model_name == 'model': + model_name = os.path.basename(os.path.dirname(abs_path)) # use directory name + + model_operators_string = '{}'.format(model_name) + for op in onnx_operator_list: + model_operators_string+=',' + if op in operators: + model_operators_string+='{}'.format(operators[op]) + result.write('{}\n'.format(model_operators_string))